gnosys 5.16.0 → 5.17.0

package/README.md

@@ -51,7 +51,7 @@ That's the 60-second tour. **Everything else lives on [gnosys.ai](https://gnosys
 - **Central brain** — one `~/.gnosys/gnosys.db` unifies every project (project / user / global scopes). Sub-10ms reads, SQLite as the sole source of truth.
 - **Federated search** — tier-boosted hybrid (FTS5 keyword + semantic) search across scopes, with recency and reinforcement.
 - **MCP server** — `gnosys serve` exposes 50+ memory tools to any MCP client. Sandbox-first runtime keeps context cost near zero.
-- **Web Knowledge Base** — `gnosys web build` turns any site into a searchable index for serverless chatbots. Zero runtime deps.
+- **Web Knowledge Base** — `gnosys web build` turns any site into a searchable index for serverless chatbots, with optional build-time vectors for serverless-safe hybrid semantic search. Zero runtime deps.
 - **Dream Mode** — idle-time consolidation: confidence decay, summaries, relationship discovery. Never deletes — only suggests.
 - **Multi-machine sync** — share your brain across machines; conflict detection with skip-and-flag resolution.
 - **Obsidian export** — `gnosys export` regenerates a full vault with frontmatter, `[[wikilinks]]`, and graph data.

package/dist/cli.js

@@ -1788,6 +1788,9 @@ webCmd
     .option("--input <dir>", "Override knowledge directory")
     .option("--output <path>", "Override output file path")
     .option("--no-stop-words", "Disable stop-word filtering")
+    .option("--embeddings <provider>", "Also build gnosys-vectors.json (openai|voyage|local)")
+    .option("--embed-model <id>", "Override the embedding model")
+    .option("--no-expansions", "Skip LLM concept-expansion generation")
     .option("--json", "Output index stats as JSON")
     .action(async (opts) => {
     const { runWebBuildIndexCommand } = await import("./lib/webBuildIndexCommand.js");
@@ -1801,6 +1804,9 @@ webCmd
     .option("--no-llm", "Force structured mode (no LLM)")
     .option("--concurrency <n>", "Parallel processing limit", "3")
     .option("--dry-run", "Show what would change without writing files")
+    .option("--embeddings <provider>", "Also build gnosys-vectors.json (openai|voyage|local)")
+    .option("--embed-model <id>", "Override the embedding model")
+    .option("--no-expansions", "Skip LLM concept-expansion generation")
     .option("--json", "Output results as JSON")
     .action(async (opts) => {
     const { runWebBuildCommand } = await import("./lib/webBuildCommand.js");

package/dist/lib/staticSearch.d.ts

@@ -13,6 +13,17 @@ export interface GnosysWebIndex {
     documentCount: number;
     documents: DocumentManifest[];
     invertedIndex: Record<string, IndexEntry[]>;
+    expansions?: Record<string, string[]>;
+}
+export interface GnosysWebVectors {
+    version: number;
+    model: string;
+    dims: number;
+    quantization: "int8";
+    generated: string;
+    scale: number;
+    offset: number;
+    vectors: Record<string, number[]>;
 }
 export interface DocumentManifest {
     id: string;
@@ -36,21 +47,40 @@ export interface SearchOptions {
     category?: string;
     tags?: string[];
     boostRecent?: boolean;
+    queryVector?: number[];
+    vectors?: GnosysWebVectors;
+    expectedModel?: string;
+    expandQuery?: boolean;
 }
 export interface SearchResult {
     document: DocumentManifest;
     score: number;
     matchedTokens: string[];
+    semanticScore?: number;
 }
 /**
  * Load a pre-computed Gnosys web index from a file path or raw JSON string.
  * Caches the result for repeated calls with the same source.
  */
 export declare function loadIndex(pathOrJson: string): GnosysWebIndex;
+/**
+ * Load pre-computed Gnosys web vectors from a file path or raw JSON string.
+ * Caches the result for repeated calls with the same source.
+ */
+export declare function loadVectors(pathOrJson: string): GnosysWebVectors;
 /**
  * Clear the cached index (useful for testing).
  */
 export declare function clearIndexCache(): void;
+/**
+ * Clear the cached vectors file (useful for testing).
+ */
+export declare function clearVectorsCache(): void;
+/**
+ * Cosine similarity for equal-dimension vectors. Returns 0 for zero-magnitude
+ * vectors or dimension mismatches.
+ */
+export declare function cosineSimilarity(a: number[], b: number[]): number;
 /**
  * Search the pre-computed index and return ranked results.
  */

package/dist/lib/staticSearch.js

@@ -11,6 +11,11 @@ import { readFileSync, existsSync } from "fs";
 // ─── Module-level cache ──────────────────────────────────────────────────
 let cachedIndex = null;
 let cachedSource = null;
+let cachedVectors = null;
+let cachedVectorsSource = null;
+const RRF_K = 60;
+/** Expanded (concept-related) tokens contribute at half weight — plan §3. */
+const EXPANSION_DISCOUNT = 0.5;
 // ─── Stop words (same list used by webIndex.ts at build time) ────────────
 const STOP_WORDS = new Set([
     "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
@@ -63,7 +68,7 @@ export function loadIndex(pathOrJson) {
     if (!index.version || typeof index.version !== "number") {
         throw new Error("Invalid Gnosys web index: missing or invalid version field");
     }
-    if (index.version > 1) {
+    if (index.version > 2) {
         throw new Error(`Gnosys web index version ${index.version} is not supported by this version of gnosys/web. ` +
             `Please update the gnosys package.`);
     }
@@ -71,6 +76,36 @@ export function loadIndex(pathOrJson) {
     cachedSource = pathOrJson;
     return index;
 }
+/**
+ * Load pre-computed Gnosys web vectors from a file path or raw JSON string.
+ * Caches the result for repeated calls with the same source.
+ */
+export function loadVectors(pathOrJson) {
+    if (cachedVectors && cachedVectorsSource === pathOrJson) {
+        return cachedVectors;
+    }
+    let raw;
+    if (pathOrJson.trimStart().startsWith("{")) {
+        raw = pathOrJson;
+    }
+    else {
+        if (!existsSync(pathOrJson)) {
+            throw new Error(`Gnosys web vectors not found: ${pathOrJson}`);
+        }
+        raw = readFileSync(pathOrJson, "utf-8");
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        throw new Error("Invalid JSON in Gnosys web vectors");
+    }
+    const vectors = validateVectorsFile(parsed);
+    cachedVectors = vectors;
+    cachedVectorsSource = pathOrJson;
+    return vectors;
+}
 /**
  * Clear the cached index (useful for testing).
  */
@@ -78,31 +113,86 @@ export function clearIndexCache() {
     cachedIndex = null;
     cachedSource = null;
 }
+/**
+ * Clear the cached vectors file (useful for testing).
+ */
+export function clearVectorsCache() {
+    cachedVectors = null;
+    cachedVectorsSource = null;
+}
+/**
+ * Cosine similarity for equal-dimension vectors. Returns 0 for zero-magnitude
+ * vectors or dimension mismatches.
+ */
+export function cosineSimilarity(a, b) {
+    if (a.length !== b.length)
+        return 0;
+    let dot = 0;
+    let magA = 0;
+    let magB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        magA += a[i] * a[i];
+        magB += b[i] * b[i];
+    }
+    if (magA === 0 || magB === 0)
+        return 0;
+    return dot / (Math.sqrt(magA) * Math.sqrt(magB));
+}
 /**
  * Search the pre-computed index and return ranked results.
  */
 export function search(index, query, options = {}) {
-    const { limit = 6, minScore = 0.1, category, tags, boostRecent = false } = options;
+    const { limit = 6, minScore = 0.1, category, tags, boostRecent = false, queryVector, vectors, expectedModel, expandQuery = true, } = options;
     const queryTokens = tokenize(query);
     if (queryTokens.length === 0)
         return [];
+    const lexicalOptions = { limit, minScore, category, tags, boostRecent, expandQuery };
+    if (!queryVector || !vectors) {
+        return stripDocIndexes(buildLexicalResults(index, queryTokens, lexicalOptions));
+    }
+    if (expectedModel && expectedModel !== vectors.model) {
+        console.warn(`[gnosys/web] embedding model mismatch: index built with ${vectors.model}; ` +
+            `query uses ${expectedModel} - falling back to lexical search`);
+        return stripDocIndexes(buildLexicalResults(index, queryTokens, lexicalOptions));
+    }
+    if (queryVector.length !== vectors.dims) {
+        console.warn(`[gnosys/web] embedding dimension mismatch: index built with ${vectors.dims} dimensions; ` +
+            `query has ${queryVector.length} - falling back to lexical search`);
+        return stripDocIndexes(buildLexicalResults(index, queryTokens, lexicalOptions));
+    }
+    // For hybrid fusion, minScore is intentionally not applied: RRF scores are
+    // rank-based and much smaller than lexical TF-IDF scores.
+    const lexicalRanking = buildLexicalResults(index, queryTokens, {
+        limit: Number.POSITIVE_INFINITY,
+        minScore: Number.NEGATIVE_INFINITY,
+        category,
+        tags,
+        boostRecent,
+        expandQuery,
+    });
+    const semanticRanking = buildSemanticRanking(index, queryVector, vectors, category, tags);
+    return rrfFusion(lexicalRanking, semanticRanking).slice(0, limit);
+}
+function buildLexicalResults(index, queryTokens, options) {
+    const { limit, minScore, category, tags, boostRecent, expandQuery } = options;
     // Accumulate scores per document
     const docScores = new Map();
-    for (const token of queryTokens) {
+    for (const { token, weight } of buildWeightedQueryTokens(index, queryTokens, expandQuery)) {
         const entries = index.invertedIndex[token];
         if (!entries)
             continue;
         for (const entry of entries) {
             const existing = docScores.get(entry.docIndex);
             if (existing) {
-                existing.score += entry.score;
+                existing.score += entry.score * weight;
                 if (!existing.matchedTokens.includes(token)) {
                     existing.matchedTokens.push(token);
                 }
             }
             else {
                 docScores.set(entry.docIndex, {
-                    score: entry.score,
+                    score: entry.score * weight,
                     matchedTokens: [token],
                 });
             }
@@ -131,12 +221,151 @@ export function search(index, query, options = {}) {
         }
         if (finalScore < minScore)
             continue;
-        results.push({ document: doc, score: finalScore, matchedTokens });
+        results.push({ document: doc, score: finalScore, matchedTokens, docIndex });
     }
     // Sort by score descending
     results.sort((a, b) => b.score - a.score);
     return results.slice(0, limit);
 }
+function buildWeightedQueryTokens(index, queryTokens, expandQuery) {
+    if (!expandQuery || !index.expansions) {
+        return queryTokens.map((token) => ({ token, weight: 1 }));
+    }
+    const directTokens = new Set(queryTokens);
+    const expandedTokens = new Set();
+    const weightedTokens = queryTokens.map((token) => ({ token, weight: 1 }));
+    for (const token of queryTokens) {
+        const related = index.expansions[token];
+        if (!Array.isArray(related))
+            continue;
+        for (const relatedToken of related) {
+            if (typeof relatedToken !== "string" ||
+                directTokens.has(relatedToken) ||
+                expandedTokens.has(relatedToken)) {
+                continue;
+            }
+            expandedTokens.add(relatedToken);
+            weightedTokens.push({ token: relatedToken, weight: EXPANSION_DISCOUNT });
+        }
+    }
+    return weightedTokens;
+}
+function buildSemanticRanking(index, queryVector, vectors, category, tags) {
+    const docIndexById = new Map();
+    for (let i = 0; i < index.documents.length; i++) {
+        docIndexById.set(index.documents[i].id, i);
+    }
+    const results = [];
+    for (const [docId, quantizedVector] of Object.entries(vectors.vectors)) {
+        const docIndex = docIndexById.get(docId);
+        if (docIndex === undefined)
+            continue;
+        if (!Array.isArray(quantizedVector) || quantizedVector.length !== vectors.dims)
+            continue;
+        const doc = index.documents[docIndex];
+        if (!doc)
+            continue;
+        if (category && doc.category !== category)
+            continue;
+        if (tags && tags.length > 0 && !tags.some((t) => doc.tags.includes(t)))
+            continue;
+        const docVector = quantizedVector.map((value) => value * vectors.scale + vectors.offset);
+        const semanticScore = cosineSimilarity(queryVector, docVector);
+        if (!Number.isFinite(semanticScore))
+            continue;
+        results.push({ docIndex, document: doc, semanticScore });
+    }
+    results.sort((a, b) => b.semanticScore - a.semanticScore);
+    return results;
+}
+function rrfFusion(lexicalResults, semanticResults) {
+    const fused = new Map();
+    for (let i = 0; i < lexicalResults.length; i++) {
+        const result = lexicalResults[i];
+        fused.set(result.docIndex, {
+            score: 1 / (RRF_K + i + 1),
+            document: result.document,
+            matchedTokens: result.matchedTokens,
+        });
+    }
+    for (let i = 0; i < semanticResults.length; i++) {
+        const result = semanticResults[i];
+        const rrfScore = 1 / (RRF_K + i + 1);
+        const existing = fused.get(result.docIndex);
+        if (existing) {
+            existing.score += rrfScore;
+            existing.semanticScore = result.semanticScore;
+        }
+        else {
+            fused.set(result.docIndex, {
+                score: rrfScore,
+                document: result.document,
+                matchedTokens: [],
+                semanticScore: result.semanticScore,
+            });
+        }
+    }
+    return Array.from(fused.values()).sort((a, b) => b.score - a.score);
+}
+function stripDocIndexes(results) {
+    return results.map((result) => ({
+        document: result.document,
+        score: result.score,
+        matchedTokens: result.matchedTokens,
+    }));
+}
+function validateVectorsFile(parsed) {
+    if (!isRecord(parsed)) {
+        throw new Error("Invalid Gnosys web vectors: expected an object");
+    }
+    const version = parsed.version;
+    if (typeof version !== "number") {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid version field");
+    }
+    if (version !== 1) {
+        throw new Error(`Gnosys web vectors version ${version} is not supported by this version of gnosys/web. ` +
+            `Please update the gnosys package.`);
+    }
+    if (typeof parsed.model !== "string") {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid model field");
+    }
+    if (typeof parsed.dims !== "number" || !Number.isFinite(parsed.dims) || parsed.dims < 0) {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid dims field");
+    }
+    if (parsed.quantization !== "int8") {
+        throw new Error("Invalid Gnosys web vectors: quantization must be int8");
+    }
+    if (typeof parsed.generated !== "string") {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid generated field");
+    }
+    if (typeof parsed.scale !== "number" || !Number.isFinite(parsed.scale) || parsed.scale <= 0) {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid scale field");
+    }
+    if (typeof parsed.offset !== "number" || !Number.isFinite(parsed.offset)) {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid offset field");
+    }
+    if (!isRecord(parsed.vectors)) {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid vectors field");
+    }
+    for (const [docId, vector] of Object.entries(parsed.vectors)) {
+        if (!Array.isArray(vector) || !vector.every((value) => typeof value === "number")) {
+            throw new Error(`Invalid Gnosys web vectors: vector for ${docId} must be a number array`);
+        }
+    }
+    return {
+        version,
+        model: parsed.model,
+        dims: parsed.dims,
+        quantization: parsed.quantization,
+        generated: parsed.generated,
+        scale: parsed.scale,
+        offset: parsed.offset,
+        vectors: parsed.vectors,
+    };
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
 /**
  * Get a specific document's metadata by ID or path.
  */

package/dist/lib/webBuildCommand.d.ts

@@ -5,6 +5,9 @@ export type WebBuildCommandOptions = {
     llm: boolean;
     concurrency: string;
     dryRun?: boolean;
+    embeddings?: string;
+    embedModel?: string;
+    expansions?: boolean;
     json?: boolean;
 };
 export declare function runWebBuildCommand(getWebStorePath: GetWebStorePath, opts: WebBuildCommandOptions): Promise<void>;

package/dist/lib/webBuildCommand.js

@@ -1,14 +1,18 @@
 import path from "path";
+const VECTOR_PROVIDERS = new Set(["openai", "voyage", "local"]);
 export async function runWebBuildCommand(getWebStorePath, opts) {
     try {
         const { loadConfig } = await import("./config.js");
         const { ingestSite } = await import("./webIngest.js");
-        const { buildIndex, writeIndex } = await import("./webIndex.js");
-        const gnosysConfig = await loadConfig(await getWebStorePath());
+        const { attachExpansions, buildIndex, generateExpansions, writeIndex } = await import("./webIndex.js");
+        const storePath = await getWebStorePath();
+        const gnosysConfig = await loadConfig(storePath);
         const webConfig = gnosysConfig.web;
         if (!webConfig) {
             throw new Error("No web configuration found in gnosys.json. Run 'gnosys web init' first.");
         }
+        const embeddingProvider = parseVectorProvider(opts.embeddings);
+        const llmEnrich = opts.llm ? webConfig.llmEnrich : false;
         // Step 1: Ingest
         const ingestResult = await ingestSite({
             source: webConfig.source,
@@ -18,7 +22,7 @@ export async function runWebBuildCommand(getWebStorePath, opts) {
             outputDir: webConfig.outputDir,
             exclude: webConfig.exclude,
             categories: webConfig.categories,
-            llmEnrich: opts.llm ? webConfig.llmEnrich : false,
+            llmEnrich,
             prune: opts.prune || webConfig.prune,
             concurrency: parseInt(opts.concurrency, 10) || webConfig.concurrency,
             crawlDelayMs: webConfig.crawlDelayMs,
@@ -26,17 +30,32 @@ export async function runWebBuildCommand(getWebStorePath, opts) {
         }, gnosysConfig);
         // Step 2: Build index (skip if dry run)
         let indexStats = { documentCount: 0, tokenCount: 0 };
+        let vectorsStats;
         if (!opts.dryRun) {
-            const index = await buildIndex(webConfig.outputDir);
+            let index = await buildIndex(webConfig.outputDir);
+            if (opts.expansions !== false && llmEnrich !== false) {
+                const llmProvider = await resolveExpansionProvider(gnosysConfig);
+                if (llmProvider) {
+                    const expansions = await generateExpansions(index, llmProvider);
+                    index = attachExpansions(index, expansions);
+                }
+            }
             const indexPath = path.join(webConfig.outputDir, "gnosys-index.json");
             await writeIndex(index, indexPath);
             indexStats = {
                 documentCount: index.documentCount,
                 tokenCount: Object.keys(index.invertedIndex).length,
             };
+            if (embeddingProvider) {
+                vectorsStats = await buildCommandVectors(webConfig.outputDir, storePath, embeddingProvider, opts.embedModel);
+            }
         }
         if (opts.json) {
-            console.log(JSON.stringify({ ...ingestResult, index: indexStats }));
+            console.log(JSON.stringify({
+                ...ingestResult,
+                index: indexStats,
+                ...(vectorsStats ? { vectors: vectorsStats } : {}),
+            }));
         }
         else {
             console.log(`Web build complete (${ingestResult.duration}ms):`);
@@ -45,6 +64,10 @@ export async function runWebBuildCommand(getWebStorePath, opts) {
             console.log(`  Unchanged: ${ingestResult.unchanged.length}`);
             console.log(`  Removed:   ${ingestResult.removed.length}`);
             console.log(`  Index:     ${indexStats.documentCount} docs, ${indexStats.tokenCount} tokens`);
+            if (vectorsStats) {
+                console.log(`  Vectors:   ${vectorsStats.count} docs, ${vectorsStats.model} (${vectorsStats.dims}d)`);
+                console.log(`  Vector output: ${vectorsStats.outputPath}`);
+            }
             if (ingestResult.errors.length > 0) {
                 console.log(`  Errors:    ${ingestResult.errors.length}`);
                 for (const e of ingestResult.errors) {
@@ -63,3 +86,36 @@ export async function runWebBuildCommand(getWebStorePath, opts) {
         process.exit(1);
     }
 }
+function parseVectorProvider(provider) {
+    if (!provider)
+        return undefined;
+    const normalized = provider.trim().toLowerCase();
+    if (!VECTOR_PROVIDERS.has(normalized)) {
+        throw new Error(`Invalid embeddings provider "${provider}". Valid providers: openai, voyage, local.`);
+    }
+    return normalized;
+}
+async function resolveExpansionProvider(gnosysConfig) {
+    try {
+        const { getLLMProvider } = await import("./llm.js");
+        return getLLMProvider(gnosysConfig, "structuring");
+    }
+    catch {
+        return null;
+    }
+}
+async function buildCommandVectors(knowledgeDir, storePath, provider, model) {
+    const { buildVectors, writeVectorsFile } = await import("./webVectors.js");
+    const vectorsFile = await buildVectors(knowledgeDir, {
+        provider,
+        model,
+        storePath,
+    });
+    const outputPath = await writeVectorsFile(knowledgeDir, vectorsFile);
+    return {
+        model: vectorsFile.model,
+        dims: vectorsFile.dims,
+        count: Object.keys(vectorsFile.vectors).length,
+        outputPath,
+    };
+}

package/dist/lib/webBuildIndexCommand.d.ts

@@ -3,6 +3,9 @@ export type WebBuildIndexCommandOptions = {
     input?: string;
     output?: string;
     stopWords: boolean;
+    embeddings?: string;
+    embedModel?: string;
+    expansions?: boolean;
     json?: boolean;
 };
 export declare function runWebBuildIndexCommand(getWebStorePath: GetWebStorePath, opts: WebBuildIndexCommandOptions): Promise<void>;

package/dist/lib/webBuildIndexCommand.js

@@ -1,21 +1,35 @@
 import path from "path";
+const VECTOR_PROVIDERS = new Set(["openai", "voyage", "local"]);
 export async function runWebBuildIndexCommand(getWebStorePath, opts) {
     try {
         const { loadConfig } = await import("./config.js");
-        const { buildIndex, writeIndex } = await import("./webIndex.js");
-        const gnosysConfig = await loadConfig(await getWebStorePath());
+        const { attachExpansions, buildIndex, generateExpansions, writeIndex } = await import("./webIndex.js");
+        const storePath = await getWebStorePath();
+        const gnosysConfig = await loadConfig(storePath);
         const knowledgeDir = opts.input || gnosysConfig.web?.outputDir || "./knowledge";
         const outputPath = opts.output || path.join(knowledgeDir, "gnosys-index.json");
-        const index = await buildIndex(knowledgeDir, {
+        const embeddingProvider = parseVectorProvider(opts.embeddings);
+        let index = await buildIndex(knowledgeDir, {
             stopWords: opts.stopWords,
         });
+        if (opts.expansions !== false) {
+            const llmProvider = await resolveExpansionProvider(gnosysConfig);
+            if (llmProvider) {
+                const expansions = await generateExpansions(index, llmProvider);
+                index = attachExpansions(index, expansions);
+            }
+        }
         await writeIndex(index, outputPath);
+        const vectors = embeddingProvider
+            ? await buildCommandVectors(knowledgeDir, storePath, embeddingProvider, opts.embedModel)
+            : undefined;
         if (opts.json) {
             console.log(JSON.stringify({
                 ok: true,
                 documentCount: index.documentCount,
                 tokenCount: Object.keys(index.invertedIndex).length,
                 outputPath,
+                ...(vectors ? { vectors } : {}),
             }));
         }
         else {
@@ -23,6 +37,10 @@ export async function runWebBuildIndexCommand(getWebStorePath, opts) {
             console.log(`  Documents: ${index.documentCount}`);
             console.log(`  Tokens:    ${Object.keys(index.invertedIndex).length}`);
             console.log(`  Output:    ${outputPath}`);
+            if (vectors) {
+                console.log(`  Vectors:   ${vectors.count} docs, ${vectors.model} (${vectors.dims}d)`);
+                console.log(`  Vector output: ${vectors.outputPath}`);
+            }
         }
     }
     catch (err) {
@@ -35,3 +53,36 @@ export async function runWebBuildIndexCommand(getWebStorePath, opts) {
         process.exit(1);
     }
 }
+function parseVectorProvider(provider) {
+    if (!provider)
+        return undefined;
+    const normalized = provider.trim().toLowerCase();
+    if (!VECTOR_PROVIDERS.has(normalized)) {
+        throw new Error(`Invalid embeddings provider "${provider}". Valid providers: openai, voyage, local.`);
+    }
+    return normalized;
+}
+async function resolveExpansionProvider(gnosysConfig) {
+    try {
+        const { getLLMProvider } = await import("./llm.js");
+        return getLLMProvider(gnosysConfig, "structuring");
+    }
+    catch {
+        return null;
+    }
+}
+async function buildCommandVectors(knowledgeDir, storePath, provider, model) {
+    const { buildVectors, writeVectorsFile } = await import("./webVectors.js");
+    const vectorsFile = await buildVectors(knowledgeDir, {
+        provider,
+        model,
+        storePath,
+    });
+    const outputPath = await writeVectorsFile(knowledgeDir, vectorsFile);
+    return {
+        model: vectorsFile.model,
+        dims: vectorsFile.dims,
+        count: Object.keys(vectorsFile.vectors).length,
+        outputPath,
+    };
+}

package/dist/lib/webIndex.d.ts

@@ -11,6 +11,10 @@ export interface BuildIndexOptions {
     minTokenLength?: number;
     maxTokensPerDoc?: number;
     includeArchived?: boolean;
+    expansions?: Record<string, string[]>;
+}
+export interface ExpansionProvider {
+    generate(prompt: string, options?: unknown): Promise<string>;
 }
 /**
  * Build a search index from a directory of Gnosys markdown files.
@@ -20,6 +24,19 @@ export declare function buildIndexSync(knowledgeDir: string, options?: BuildInde
  * Build a search index (async wrapper).
  */
 export declare function buildIndex(knowledgeDir: string, options?: BuildIndexOptions): Promise<GnosysWebIndex>;
+/**
+ * Generate a token expansion map from an existing index using an injected LLM.
+ * The provider is resolved by callers so this build module remains easy to test
+ * and does not depend on runtime config or provider wiring.
+ */
+export declare function generateExpansions(index: GnosysWebIndex, llmProvider: ExpansionProvider, options?: {
+    maxTokens?: number;
+}): Promise<Record<string, string[]>>;
+/**
+ * Attach a non-empty concept expansion map to an index and bump it to v2.
+ * Empty maps are ignored so default/no-enrichment builds keep emitting v1.
+ */
+export declare function attachExpansions(index: GnosysWebIndex, expansions?: Record<string, string[]>): GnosysWebIndex;
 /**
  * Write an index to a JSON file.
  */

package/dist/lib/webIndex.js

@@ -24,6 +24,8 @@ const STOP_WORDS = new Set([
     "re", "same", "some", "such", "up", "us", "we", "what", "when",
     "where", "which", "who", "whom", "why", "you", "your",
 ]);
+const DEFAULT_EXPANSION_CANDIDATES = 100;
+const MIN_EXPANSION_TOKEN_LENGTH = 3;
 // ─── Tokenization ────────────────────────────────────────────────────────
 function tokenize(text, minLength) {
     return text
@@ -182,13 +184,16 @@ export function buildIndexSync(knowledgeDir, options = {}) {
     for (const token of Object.keys(invertedIndex).sort()) {
         sortedIndex[token] = invertedIndex[token];
     }
-    return {
+    const index = {
         version: 1,
         generated: new Date().toISOString(),
         documentCount: documents.length,
         documents,
         invertedIndex: sortedIndex,
     };
+    // Default builds intentionally remain v1; only indexes with a non-empty
+    // expansion map become v2 so older gnosys/web runtimes can still load them.
+    return attachExpansions(index, options.expansions);
 }
 /**
  * Build a search index (async wrapper).
@@ -196,6 +201,127 @@ export function buildIndexSync(knowledgeDir, options = {}) {
 export async function buildIndex(knowledgeDir, options) {
     return buildIndexSync(knowledgeDir, options);
 }
+/**
+ * Generate a token expansion map from an existing index using an injected LLM.
+ * The provider is resolved by callers so this build module remains easy to test
+ * and does not depend on runtime config or provider wiring.
+ */
+export async function generateExpansions(index, llmProvider, options = {}) {
+    const maxTokens = options.maxTokens ?? DEFAULT_EXPANSION_CANDIDATES;
+    if (maxTokens <= 0)
+        return {};
+    const candidates = selectExpansionCandidates(index, maxTokens);
+    if (candidates.length === 0)
+        return {};
+    const prompt = [
+        "Return strict JSON only: an object mapping each provided token to an array of related search tokens.",
+        "Use lowercase single-word domain concepts. Do not include explanations, markdown, or tokens not in the input list.",
+        "Input tokens:",
+        candidates.join(", "),
+    ].join("\n");
+    let raw;
+    try {
+        raw = await llmProvider.generate(prompt);
+    }
+    catch (error) {
+        console.warn(`[gnosys/web] concept expansion generation failed: ${formatErrorMessage(error)}`);
+        return {};
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw.trim());
+    }
+    catch {
+        console.warn("[gnosys/web] concept expansion provider returned invalid JSON; skipping expansions");
+        return {};
+    }
+    if (!isRecord(parsed))
+        return {};
+    const candidateSet = new Set(candidates);
+    const expansions = {};
+    for (const [rawKey, rawRelated] of Object.entries(parsed)) {
+        const key = tokenize(rawKey, MIN_EXPANSION_TOKEN_LENGTH)[0];
+        if (!key || !candidateSet.has(key) || !Array.isArray(rawRelated))
+            continue;
+        const relatedTokens = normalizeExpansionTokens(rawRelated, key);
+        if (relatedTokens.length > 0) {
+            expansions[key] = relatedTokens;
+        }
+    }
+    return sortExpansionMap(expansions);
+}
+/**
+ * Attach a non-empty concept expansion map to an index and bump it to v2.
+ * Empty maps are ignored so default/no-enrichment builds keep emitting v1.
+ */
+export function attachExpansions(index, expansions) {
+    const normalized = normalizeExpansionMap(expansions);
+    if (Object.keys(normalized).length === 0) {
+        const { expansions: _ignored, ...withoutExpansions } = index;
+        return { ...withoutExpansions, version: 1 };
+    }
+    return {
+        ...index,
+        version: 2,
+        expansions: normalized,
+    };
+}
+function selectExpansionCandidates(index, maxTokens) {
+    return Object.entries(index.invertedIndex)
+        .filter(([token, entries]) => token.length >= MIN_EXPANSION_TOKEN_LENGTH &&
+        !STOP_WORDS.has(token) &&
+        Array.isArray(entries) &&
+        entries.length > 0)
+        .sort(([tokenA, entriesA], [tokenB, entriesB]) => {
+        const frequencyDelta = entriesB.length - entriesA.length;
+        return frequencyDelta || tokenA.localeCompare(tokenB);
+    })
+        .slice(0, maxTokens)
+        .map(([token]) => token);
+}
+function normalizeExpansionMap(expansions) {
+    if (!expansions)
+        return {};
+    const normalized = {};
+    for (const [rawKey, rawRelated] of Object.entries(expansions)) {
+        const key = tokenize(rawKey, MIN_EXPANSION_TOKEN_LENGTH)[0];
+        if (!key || STOP_WORDS.has(key) || !Array.isArray(rawRelated))
+            continue;
+        const relatedTokens = normalizeExpansionTokens(rawRelated, key);
+        if (relatedTokens.length > 0) {
+            normalized[key] = relatedTokens;
+        }
+    }
+    return sortExpansionMap(normalized);
+}
+function normalizeExpansionTokens(values, key) {
+    const tokens = [];
+    for (const value of values) {
+        if (typeof value !== "string")
+            continue;
+        let relatedTokens = tokenize(value, MIN_EXPANSION_TOKEN_LENGTH);
+        relatedTokens = filterStopWords(relatedTokens);
+        for (const token of relatedTokens) {
+            if (token === key || tokens.includes(token))
+                continue;
+            tokens.push(token);
+        }
+    }
+    return tokens.sort();
+}
+function sortExpansionMap(expansions) {
+    const sorted = {};
+    for (const key of Object.keys(expansions).sort()) {
+        sorted[key] = expansions[key];
+    }
+    return sorted;
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function formatErrorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
 /**
  * Write an index to a JSON file.
  */

package/dist/lib/webInitCommand.js

@@ -153,6 +153,7 @@ export async function runWebInitCommand(getWebStorePath, opts) {
             console.log(`  ${!sitemapUrl && envVarName ? "4" : envVarName || !sitemapUrl ? "3" : "2"}. Add to package.json: ${CYAN}"postbuild": "npx gnosys web build"${RESET}`);
             console.log();
             console.log(`${DIM}Every deploy will re-crawl and rebuild the search index automatically.${RESET}`);
+            console.log(`${DIM}Tip: add semantic search with ${CYAN}gnosys web build --embeddings openai${RESET}${DIM} (see docs/web-semantic-search.md).${RESET}`);
         }
     }
     catch (err) {

package/dist/lib/webStatusCommand.js

@@ -52,6 +52,28 @@ export async function runWebStatusCommand(getWebStorePath, opts) {
                 indexInfo = { exists: true, size: stat.size };
             }
         }
+        const vectorsPath = path.join(resolvedDir, "gnosys-vectors.json");
+        let vectorsInfo = { exists: false };
+        if (existsSync(vectorsPath)) {
+            const vectorsStat = statSync(vectorsPath);
+            try {
+                const vectorsData = JSON.parse(readFileSync(vectorsPath, "utf-8"));
+                const vectorMap = typeof vectorsData.vectors === "object" && vectorsData.vectors !== null
+                    ? vectorsData.vectors
+                    : {};
+                vectorsInfo = {
+                    exists: true,
+                    model: typeof vectorsData.model === "string" ? vectorsData.model : undefined,
+                    dims: typeof vectorsData.dims === "number" ? vectorsData.dims : undefined,
+                    count: Object.keys(vectorMap).length,
+                    size: vectorsStat.size,
+                    generated: typeof vectorsData.generated === "string" ? vectorsData.generated : undefined,
+                };
+            }
+            catch {
+                vectorsInfo = { exists: true, size: vectorsStat.size };
+            }
+        }
         if (opts.json) {
             console.log(JSON.stringify({
                 ok: true,
@@ -59,6 +81,7 @@ export async function runWebStatusCommand(getWebStorePath, opts) {
                 totalFiles,
                 categoryCounts,
                 index: indexInfo,
+                vectors: vectorsInfo,
             }, null, 2));
         }
         else {
@@ -80,6 +103,17 @@ export async function runWebStatusCommand(getWebStorePath, opts) {
             else {
                 console.log(`  Index: not built (run 'gnosys web build-index')`);
             }
+            if (vectorsInfo.exists) {
+                if (vectorsInfo.model && typeof vectorsInfo.dims === "number" && typeof vectorsInfo.count === "number") {
+                    console.log(`  Vectors: ${vectorsInfo.count} docs, ${vectorsInfo.model} (${vectorsInfo.dims}d), ${((vectorsInfo.size || 0) / 1024).toFixed(1)}KB`);
+                }
+                else {
+                    console.log(`  Vectors: present, ${((vectorsInfo.size || 0) / 1024).toFixed(1)}KB (unreadable)`);
+                }
+            }
+            else {
+                console.log(`  Vectors: not built (run 'gnosys web build-index --embeddings <provider>')`);
+            }
         }
     }
     catch (err) {

package/dist/lib/webVectors.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Gnosys Web Vectors - build-time semantic vector generator.
+ *
+ * This module is intentionally build-time only. The zero-dependency
+ * `gnosys/web` runtime must not import it because the local provider reaches
+ * into the optional embeddings stack.
+ */
+export type VectorProvider = "openai" | "voyage" | "local";
+export interface WebVectorsFile {
+    version: 1;
+    model: string;
+    dims: number;
+    quantization: "int8";
+    generated: string;
+    scale: number;
+    offset: number;
+    vectors: Record<string, number[]>;
+}
+export interface BuildVectorsOptions {
+    provider: VectorProvider;
+    model?: string;
+    apiKey?: string;
+    storePath?: string;
+}
+/**
+ * Symmetric global int8 quantization. One scale/offset pair is recorded for
+ * the whole file so the runtime can dequantize without per-vector metadata.
+ */
+export declare function quantizeVector(vec: number[], scale: number): number[];
+export declare function dequantizeVector(qvec: number[], scale: number): number[];
+export declare function buildVectors(knowledgeDir: string, options: BuildVectorsOptions): Promise<WebVectorsFile>;
+export declare function writeVectorsFile(knowledgeDir: string, vectorsFile: WebVectorsFile): Promise<string>;

package/dist/lib/webVectors.js

@@ -0,0 +1,255 @@
+/**
+ * Gnosys Web Vectors - build-time semantic vector generator.
+ *
+ * This module is intentionally build-time only. The zero-dependency
+ * `gnosys/web` runtime must not import it because the local provider reaches
+ * into the optional embeddings stack.
+ */
+import fs from "fs/promises";
+import path from "path";
+import matter from "gray-matter";
+import { getOpenAIApiKey, getOpenAIBaseUrl, loadConfig } from "./config.js";
+import { GnosysEmbeddings } from "./embeddings.js";
+const VECTOR_FILE_NAME = "gnosys-vectors.json";
+const MAX_BODY_TOKENS = 512;
+const API_BATCH_SIZE = 96;
+const DEFAULT_OPENAI_MODEL = "text-embedding-3-small";
+const DEFAULT_VOYAGE_MODEL = "voyage-3-lite";
+const DEFAULT_LOCAL_MODEL = "Xenova/all-MiniLM-L6-v2";
+/**
+ * Symmetric global int8 quantization. One scale/offset pair is recorded for
+ * the whole file so the runtime can dequantize without per-vector metadata.
+ */
+export function quantizeVector(vec, scale) {
+    if (!Number.isFinite(scale) || scale <= 0) {
+        throw new Error("Cannot quantize vector with a non-positive scale");
+    }
+    return vec.map((value) => clampInt8(Math.round(value / scale)));
+}
+export function dequantizeVector(qvec, scale) {
+    if (!Number.isFinite(scale) || scale <= 0) {
+        throw new Error("Cannot dequantize vector with a non-positive scale");
+    }
+    return qvec.map((value) => value * scale);
+}
+export async function buildVectors(knowledgeDir, options) {
+    const docs = await readKnowledgeDocs(knowledgeDir);
+    const model = modelForProvider(options);
+    if (docs.length === 0) {
+        return {
+            version: 1,
+            model,
+            dims: 0,
+            quantization: "int8",
+            generated: new Date().toISOString(),
+            scale: 1,
+            offset: 0,
+            vectors: {},
+        };
+    }
+    const texts = docs.map((doc) => embeddingText(doc));
+    const embeddings = await embedTexts(texts, options, model);
+    const dims = validateEmbeddings(embeddings, docs.length);
+    const scale = computeGlobalScale(embeddings);
+    const vectors = {};
+    for (let i = 0; i < docs.length; i++) {
+        const doc = docs[i];
+        if (vectors[doc.id]) {
+            throw new Error(`Duplicate web knowledge document id: ${doc.id}`);
+        }
+        vectors[doc.id] = quantizeVector(embeddings[i], scale);
+    }
+    return {
+        version: 1,
+        model,
+        dims,
+        quantization: "int8",
+        generated: new Date().toISOString(),
+        scale,
+        offset: 0,
+        vectors,
+    };
+}
+export async function writeVectorsFile(knowledgeDir, vectorsFile) {
+    const outputPath = path.join(knowledgeDir, VECTOR_FILE_NAME);
+    await fs.mkdir(path.dirname(outputPath), { recursive: true });
+    await fs.writeFile(outputPath, JSON.stringify(vectorsFile, null, 2), "utf-8");
+    return outputPath;
+}
+async function readKnowledgeDocs(knowledgeDir) {
+    const resolvedDir = path.resolve(knowledgeDir);
+    const files = await findMarkdownFiles(resolvedDir);
+    const docs = [];
+    for (const filePath of files) {
+        let raw;
+        try {
+            raw = await fs.readFile(filePath, "utf-8");
+        }
+        catch {
+            continue;
+        }
+        let parsed;
+        try {
+            parsed = matter(raw);
+        }
+        catch {
+            continue;
+        }
+        const fm = parsed.data;
+        const status = typeof fm.status === "string" ? fm.status : "active";
+        if (status === "archived")
+            continue;
+        docs.push({
+            id: typeof fm.id === "string" ? fm.id : path.basename(filePath, ".md"),
+            title: typeof fm.title === "string" ? fm.title : path.basename(filePath, ".md"),
+            relevance: typeof fm.relevance === "string" ? fm.relevance : "",
+            body: parsed.content.trim(),
+        });
+    }
+    return docs;
+}
+async function findMarkdownFiles(dir) {
+    const results = [];
+    async function walk(current) {
+        let entries;
+        try {
+            entries = await fs.readdir(current, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            const fullPath = path.join(current, entry.name);
+            if (entry.isDirectory()) {
+                await walk(fullPath);
+            }
+            else if (entry.isFile() && entry.name.endsWith(".md")) {
+                results.push(fullPath);
+            }
+        }
+    }
+    await walk(dir);
+    return results.sort();
+}
+function embeddingText(doc) {
+    const cappedBody = doc.body.split(/\s+/).filter(Boolean).slice(0, MAX_BODY_TOKENS).join(" ");
+    return `${doc.title}\n${doc.relevance}\n${cappedBody}`;
+}
+function modelForProvider(options) {
+    if (options.model)
+        return options.model;
+    switch (options.provider) {
+        case "openai":
+            return DEFAULT_OPENAI_MODEL;
+        case "voyage":
+            return DEFAULT_VOYAGE_MODEL;
+        case "local":
+            return DEFAULT_LOCAL_MODEL;
+    }
+}
+async function embedTexts(texts, options, model) {
+    switch (options.provider) {
+        case "openai":
+            return embedOpenAI(texts, options, model);
+        case "voyage":
+            return embedVoyage(texts, options, model);
+        case "local":
+            return embedLocal(texts, options);
+    }
+}
+async function embedOpenAI(texts, options, model) {
+    const config = await loadConfig(options.storePath ?? process.cwd());
+    const apiKey = options.apiKey || getOpenAIApiKey(config) || process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+        throw new Error("OpenAI embeddings require an API key. Set OPENAI_API_KEY or configure an OpenAI key in Gnosys.");
+    }
+    const endpoint = `${getOpenAIBaseUrl(config).replace(/\/+$/, "")}/embeddings`;
+    return embedApiBatches("OpenAI", endpoint, apiKey, texts, model);
+}
+async function embedVoyage(texts, options, model) {
+    // Voyage is not part of the Gnosys config schema yet; for now it is env-only.
+    const apiKey = options.apiKey || process.env.VOYAGE_API_KEY;
+    if (!apiKey) {
+        throw new Error("Voyage embeddings require an API key. Set VOYAGE_API_KEY.");
+    }
+    return embedApiBatches("Voyage", "https://api.voyageai.com/v1/embeddings", apiKey, texts, model);
+}
+async function embedApiBatches(providerName, endpoint, apiKey, texts, model) {
+    const embeddings = [];
+    for (let i = 0; i < texts.length; i += API_BATCH_SIZE) {
+        const input = texts.slice(i, i + API_BATCH_SIZE);
+        const response = await fetch(endpoint, {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${apiKey}`,
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify({ model, input }),
+        });
+        const bodyText = await response.text();
+        if (!response.ok) {
+            const detail = bodyText ? `: ${bodyText.slice(0, 500)}` : "";
+            throw new Error(`${providerName} embedding request failed (${response.status} ${response.statusText})${detail}`);
+        }
+        embeddings.push(...parseEmbeddingResponse(providerName, bodyText));
+    }
+    return embeddings;
+}
+async function embedLocal(texts, options) {
+    const embeddings = new GnosysEmbeddings(options.storePath ?? path.resolve(process.cwd(), ".gnosys"));
+    const vectors = await embeddings.embedBatch(texts);
+    return vectors.map((vec) => Array.from(vec));
+}
+function parseEmbeddingResponse(providerName, bodyText) {
+    let payload;
+    try {
+        payload = JSON.parse(bodyText);
+    }
+    catch {
+        throw new Error(`${providerName} embedding response was not valid JSON`);
+    }
+    if (!isEmbeddingPayload(payload)) {
+        throw new Error(`${providerName} embedding response did not include data[].embedding arrays`);
+    }
+    return payload.data.map((entry) => entry.embedding);
+}
+function isEmbeddingPayload(payload) {
+    if (typeof payload !== "object" || payload === null)
+        return false;
+    const data = payload.data;
+    if (!Array.isArray(data))
+        return false;
+    return data.every((entry) => {
+        if (typeof entry !== "object" || entry === null)
+            return false;
+        const embedding = entry.embedding;
+        return Array.isArray(embedding) && embedding.every((value) => typeof value === "number" && Number.isFinite(value));
+    });
+}
+function validateEmbeddings(embeddings, expectedCount) {
+    if (embeddings.length !== expectedCount) {
+        throw new Error(`Embedding provider returned ${embeddings.length} vectors for ${expectedCount} documents`);
+    }
+    const dims = embeddings[0]?.length ?? 0;
+    if (dims === 0) {
+        throw new Error("Embedding provider returned empty vectors");
+    }
+    for (const vec of embeddings) {
+        if (vec.length !== dims) {
+            throw new Error("Embedding provider returned vectors with inconsistent dimensions");
+        }
+    }
+    return dims;
+}
+function computeGlobalScale(embeddings) {
+    let absMax = 0;
+    for (const vec of embeddings) {
+        for (const value of vec) {
+            absMax = Math.max(absMax, Math.abs(value));
+        }
+    }
+    return absMax > 0 ? absMax / 127 : 1;
+}
+function clampInt8(value) {
+    return Math.max(-128, Math.min(127, value));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gnosys",
-  "version": "5.16.0",
+  "version": "5.17.0",
   "description": "Gnosys — Persistent Memory for AI Agents. Sandbox-first runtime, central SQLite brain, federated search, Dream Mode, Web Knowledge Base, Obsidian export.",
   "type": "module",
   "main": "dist/index.js",