gnosys 5.15.3 → 5.17.0

package/dist/lib/staticSearch.js

@@ -11,6 +11,11 @@ import { readFileSync, existsSync } from "fs";
 // ─── Module-level cache ──────────────────────────────────────────────────
 let cachedIndex = null;
 let cachedSource = null;
+let cachedVectors = null;
+let cachedVectorsSource = null;
+const RRF_K = 60;
+/** Expanded (concept-related) tokens contribute at half weight — plan §3. */
+const EXPANSION_DISCOUNT = 0.5;
 // ─── Stop words (same list used by webIndex.ts at build time) ────────────
 const STOP_WORDS = new Set([
     "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
@@ -63,7 +68,7 @@ export function loadIndex(pathOrJson) {
     if (!index.version || typeof index.version !== "number") {
         throw new Error("Invalid Gnosys web index: missing or invalid version field");
     }
-    if (index.version > 1) {
+    if (index.version > 2) {
         throw new Error(`Gnosys web index version ${index.version} is not supported by this version of gnosys/web. ` +
             `Please update the gnosys package.`);
     }
@@ -71,6 +76,36 @@ export function loadIndex(pathOrJson) {
     cachedSource = pathOrJson;
     return index;
 }
+/**
+ * Load pre-computed Gnosys web vectors from a file path or raw JSON string.
+ * Caches the result for repeated calls with the same source.
+ */
+export function loadVectors(pathOrJson) {
+    if (cachedVectors && cachedVectorsSource === pathOrJson) {
+        return cachedVectors;
+    }
+    let raw;
+    if (pathOrJson.trimStart().startsWith("{")) {
+        raw = pathOrJson;
+    }
+    else {
+        if (!existsSync(pathOrJson)) {
+            throw new Error(`Gnosys web vectors not found: ${pathOrJson}`);
+        }
+        raw = readFileSync(pathOrJson, "utf-8");
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        throw new Error("Invalid JSON in Gnosys web vectors");
+    }
+    const vectors = validateVectorsFile(parsed);
+    cachedVectors = vectors;
+    cachedVectorsSource = pathOrJson;
+    return vectors;
+}
 /**
  * Clear the cached index (useful for testing).
  */
@@ -78,31 +113,86 @@ export function clearIndexCache() {
     cachedIndex = null;
     cachedSource = null;
 }
+/**
+ * Clear the cached vectors file (useful for testing).
+ */
+export function clearVectorsCache() {
+    cachedVectors = null;
+    cachedVectorsSource = null;
+}
+/**
+ * Cosine similarity for equal-dimension vectors. Returns 0 for zero-magnitude
+ * vectors or dimension mismatches.
+ */
+export function cosineSimilarity(a, b) {
+    if (a.length !== b.length)
+        return 0;
+    let dot = 0;
+    let magA = 0;
+    let magB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        magA += a[i] * a[i];
+        magB += b[i] * b[i];
+    }
+    if (magA === 0 || magB === 0)
+        return 0;
+    return dot / (Math.sqrt(magA) * Math.sqrt(magB));
+}
 /**
  * Search the pre-computed index and return ranked results.
  */
 export function search(index, query, options = {}) {
-    const { limit = 6, minScore = 0.1, category, tags, boostRecent = false } = options;
+    const { limit = 6, minScore = 0.1, category, tags, boostRecent = false, queryVector, vectors, expectedModel, expandQuery = true, } = options;
     const queryTokens = tokenize(query);
     if (queryTokens.length === 0)
         return [];
+    const lexicalOptions = { limit, minScore, category, tags, boostRecent, expandQuery };
+    if (!queryVector || !vectors) {
+        return stripDocIndexes(buildLexicalResults(index, queryTokens, lexicalOptions));
+    }
+    if (expectedModel && expectedModel !== vectors.model) {
+        console.warn(`[gnosys/web] embedding model mismatch: index built with ${vectors.model}; ` +
+            `query uses ${expectedModel} - falling back to lexical search`);
+        return stripDocIndexes(buildLexicalResults(index, queryTokens, lexicalOptions));
+    }
+    if (queryVector.length !== vectors.dims) {
+        console.warn(`[gnosys/web] embedding dimension mismatch: index built with ${vectors.dims} dimensions; ` +
+            `query has ${queryVector.length} - falling back to lexical search`);
+        return stripDocIndexes(buildLexicalResults(index, queryTokens, lexicalOptions));
+    }
+    // For hybrid fusion, minScore is intentionally not applied: RRF scores are
+    // rank-based and much smaller than lexical TF-IDF scores.
+    const lexicalRanking = buildLexicalResults(index, queryTokens, {
+        limit: Number.POSITIVE_INFINITY,
+        minScore: Number.NEGATIVE_INFINITY,
+        category,
+        tags,
+        boostRecent,
+        expandQuery,
+    });
+    const semanticRanking = buildSemanticRanking(index, queryVector, vectors, category, tags);
+    return rrfFusion(lexicalRanking, semanticRanking).slice(0, limit);
+}
+function buildLexicalResults(index, queryTokens, options) {
+    const { limit, minScore, category, tags, boostRecent, expandQuery } = options;
     // Accumulate scores per document
     const docScores = new Map();
-    for (const token of queryTokens) {
+    for (const { token, weight } of buildWeightedQueryTokens(index, queryTokens, expandQuery)) {
         const entries = index.invertedIndex[token];
         if (!entries)
             continue;
         for (const entry of entries) {
             const existing = docScores.get(entry.docIndex);
             if (existing) {
-                existing.score += entry.score;
+                existing.score += entry.score * weight;
                 if (!existing.matchedTokens.includes(token)) {
                     existing.matchedTokens.push(token);
                 }
             }
             else {
                 docScores.set(entry.docIndex, {
-                    score: entry.score,
+                    score: entry.score * weight,
                     matchedTokens: [token],
                 });
             }
@@ -131,12 +221,151 @@ export function search(index, query, options = {}) {
         }
         if (finalScore < minScore)
             continue;
-        results.push({ document: doc, score: finalScore, matchedTokens });
+        results.push({ document: doc, score: finalScore, matchedTokens, docIndex });
     }
     // Sort by score descending
     results.sort((a, b) => b.score - a.score);
     return results.slice(0, limit);
 }
+function buildWeightedQueryTokens(index, queryTokens, expandQuery) {
+    if (!expandQuery || !index.expansions) {
+        return queryTokens.map((token) => ({ token, weight: 1 }));
+    }
+    const directTokens = new Set(queryTokens);
+    const expandedTokens = new Set();
+    const weightedTokens = queryTokens.map((token) => ({ token, weight: 1 }));
+    for (const token of queryTokens) {
+        const related = index.expansions[token];
+        if (!Array.isArray(related))
+            continue;
+        for (const relatedToken of related) {
+            if (typeof relatedToken !== "string" ||
+                directTokens.has(relatedToken) ||
+                expandedTokens.has(relatedToken)) {
+                continue;
+            }
+            expandedTokens.add(relatedToken);
+            weightedTokens.push({ token: relatedToken, weight: EXPANSION_DISCOUNT });
+        }
+    }
+    return weightedTokens;
+}
+function buildSemanticRanking(index, queryVector, vectors, category, tags) {
+    const docIndexById = new Map();
+    for (let i = 0; i < index.documents.length; i++) {
+        docIndexById.set(index.documents[i].id, i);
+    }
+    const results = [];
+    for (const [docId, quantizedVector] of Object.entries(vectors.vectors)) {
+        const docIndex = docIndexById.get(docId);
+        if (docIndex === undefined)
+            continue;
+        if (!Array.isArray(quantizedVector) || quantizedVector.length !== vectors.dims)
+            continue;
+        const doc = index.documents[docIndex];
+        if (!doc)
+            continue;
+        if (category && doc.category !== category)
+            continue;
+        if (tags && tags.length > 0 && !tags.some((t) => doc.tags.includes(t)))
+            continue;
+        const docVector = quantizedVector.map((value) => value * vectors.scale + vectors.offset);
+        const semanticScore = cosineSimilarity(queryVector, docVector);
+        if (!Number.isFinite(semanticScore))
+            continue;
+        results.push({ docIndex, document: doc, semanticScore });
+    }
+    results.sort((a, b) => b.semanticScore - a.semanticScore);
+    return results;
+}
+function rrfFusion(lexicalResults, semanticResults) {
+    const fused = new Map();
+    for (let i = 0; i < lexicalResults.length; i++) {
+        const result = lexicalResults[i];
+        fused.set(result.docIndex, {
+            score: 1 / (RRF_K + i + 1),
+            document: result.document,
+            matchedTokens: result.matchedTokens,
+        });
+    }
+    for (let i = 0; i < semanticResults.length; i++) {
+        const result = semanticResults[i];
+        const rrfScore = 1 / (RRF_K + i + 1);
+        const existing = fused.get(result.docIndex);
+        if (existing) {
+            existing.score += rrfScore;
+            existing.semanticScore = result.semanticScore;
+        }
+        else {
+            fused.set(result.docIndex, {
+                score: rrfScore,
+                document: result.document,
+                matchedTokens: [],
+                semanticScore: result.semanticScore,
+            });
+        }
+    }
+    return Array.from(fused.values()).sort((a, b) => b.score - a.score);
+}
+function stripDocIndexes(results) {
+    return results.map((result) => ({
+        document: result.document,
+        score: result.score,
+        matchedTokens: result.matchedTokens,
+    }));
+}
+function validateVectorsFile(parsed) {
+    if (!isRecord(parsed)) {
+        throw new Error("Invalid Gnosys web vectors: expected an object");
+    }
+    const version = parsed.version;
+    if (typeof version !== "number") {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid version field");
+    }
+    if (version !== 1) {
+        throw new Error(`Gnosys web vectors version ${version} is not supported by this version of gnosys/web. ` +
+            `Please update the gnosys package.`);
+    }
+    if (typeof parsed.model !== "string") {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid model field");
+    }
+    if (typeof parsed.dims !== "number" || !Number.isFinite(parsed.dims) || parsed.dims < 0) {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid dims field");
+    }
+    if (parsed.quantization !== "int8") {
+        throw new Error("Invalid Gnosys web vectors: quantization must be int8");
+    }
+    if (typeof parsed.generated !== "string") {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid generated field");
+    }
+    if (typeof parsed.scale !== "number" || !Number.isFinite(parsed.scale) || parsed.scale <= 0) {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid scale field");
+    }
+    if (typeof parsed.offset !== "number" || !Number.isFinite(parsed.offset)) {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid offset field");
+    }
+    if (!isRecord(parsed.vectors)) {
+        throw new Error("Invalid Gnosys web vectors: missing or invalid vectors field");
+    }
+    for (const [docId, vector] of Object.entries(parsed.vectors)) {
+        if (!Array.isArray(vector) || !vector.every((value) => typeof value === "number")) {
+            throw new Error(`Invalid Gnosys web vectors: vector for ${docId} must be a number array`);
+        }
+    }
+    return {
+        version,
+        model: parsed.model,
+        dims: parsed.dims,
+        quantization: parsed.quantization,
+        generated: parsed.generated,
+        scale: parsed.scale,
+        offset: parsed.offset,
+        vectors: parsed.vectors,
+    };
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
 /**
  * Get a specific document's metadata by ID or path.
  */

package/dist/lib/webBuildCommand.d.ts

@@ -5,6 +5,9 @@ export type WebBuildCommandOptions = {
     llm: boolean;
     concurrency: string;
     dryRun?: boolean;
+    embeddings?: string;
+    embedModel?: string;
+    expansions?: boolean;
     json?: boolean;
 };
 export declare function runWebBuildCommand(getWebStorePath: GetWebStorePath, opts: WebBuildCommandOptions): Promise<void>;

package/dist/lib/webBuildCommand.js

@@ -1,14 +1,18 @@
 import path from "path";
+const VECTOR_PROVIDERS = new Set(["openai", "voyage", "local"]);
 export async function runWebBuildCommand(getWebStorePath, opts) {
     try {
         const { loadConfig } = await import("./config.js");
         const { ingestSite } = await import("./webIngest.js");
-        const { buildIndex, writeIndex } = await import("./webIndex.js");
-        const gnosysConfig = await loadConfig(await getWebStorePath());
+        const { attachExpansions, buildIndex, generateExpansions, writeIndex } = await import("./webIndex.js");
+        const storePath = await getWebStorePath();
+        const gnosysConfig = await loadConfig(storePath);
         const webConfig = gnosysConfig.web;
         if (!webConfig) {
             throw new Error("No web configuration found in gnosys.json. Run 'gnosys web init' first.");
         }
+        const embeddingProvider = parseVectorProvider(opts.embeddings);
+        const llmEnrich = opts.llm ? webConfig.llmEnrich : false;
         // Step 1: Ingest
         const ingestResult = await ingestSite({
             source: webConfig.source,
@@ -18,7 +22,7 @@ export async function runWebBuildCommand(getWebStorePath, opts) {
             outputDir: webConfig.outputDir,
             exclude: webConfig.exclude,
             categories: webConfig.categories,
-            llmEnrich: opts.llm ? webConfig.llmEnrich : false,
+            llmEnrich,
             prune: opts.prune || webConfig.prune,
             concurrency: parseInt(opts.concurrency, 10) || webConfig.concurrency,
             crawlDelayMs: webConfig.crawlDelayMs,
@@ -26,17 +30,32 @@ export async function runWebBuildCommand(getWebStorePath, opts) {
         }, gnosysConfig);
         // Step 2: Build index (skip if dry run)
         let indexStats = { documentCount: 0, tokenCount: 0 };
+        let vectorsStats;
         if (!opts.dryRun) {
-            const index = await buildIndex(webConfig.outputDir);
+            let index = await buildIndex(webConfig.outputDir);
+            if (opts.expansions !== false && llmEnrich !== false) {
+                const llmProvider = await resolveExpansionProvider(gnosysConfig);
+                if (llmProvider) {
+                    const expansions = await generateExpansions(index, llmProvider);
+                    index = attachExpansions(index, expansions);
+                }
+            }
             const indexPath = path.join(webConfig.outputDir, "gnosys-index.json");
             await writeIndex(index, indexPath);
             indexStats = {
                 documentCount: index.documentCount,
                 tokenCount: Object.keys(index.invertedIndex).length,
             };
+            if (embeddingProvider) {
+                vectorsStats = await buildCommandVectors(webConfig.outputDir, storePath, embeddingProvider, opts.embedModel);
+            }
         }
         if (opts.json) {
-            console.log(JSON.stringify({ ...ingestResult, index: indexStats }));
+            console.log(JSON.stringify({
+                ...ingestResult,
+                index: indexStats,
+                ...(vectorsStats ? { vectors: vectorsStats } : {}),
+            }));
         }
         else {
             console.log(`Web build complete (${ingestResult.duration}ms):`);
@@ -45,6 +64,10 @@ export async function runWebBuildCommand(getWebStorePath, opts) {
             console.log(`  Unchanged: ${ingestResult.unchanged.length}`);
             console.log(`  Removed:   ${ingestResult.removed.length}`);
             console.log(`  Index:     ${indexStats.documentCount} docs, ${indexStats.tokenCount} tokens`);
+            if (vectorsStats) {
+                console.log(`  Vectors:   ${vectorsStats.count} docs, ${vectorsStats.model} (${vectorsStats.dims}d)`);
+                console.log(`  Vector output: ${vectorsStats.outputPath}`);
+            }
             if (ingestResult.errors.length > 0) {
                 console.log(`  Errors:    ${ingestResult.errors.length}`);
                 for (const e of ingestResult.errors) {
@@ -63,3 +86,36 @@ export async function runWebBuildCommand(getWebStorePath, opts) {
         process.exit(1);
     }
 }
+function parseVectorProvider(provider) {
+    if (!provider)
+        return undefined;
+    const normalized = provider.trim().toLowerCase();
+    if (!VECTOR_PROVIDERS.has(normalized)) {
+        throw new Error(`Invalid embeddings provider "${provider}". Valid providers: openai, voyage, local.`);
+    }
+    return normalized;
+}
+async function resolveExpansionProvider(gnosysConfig) {
+    try {
+        const { getLLMProvider } = await import("./llm.js");
+        return getLLMProvider(gnosysConfig, "structuring");
+    }
+    catch {
+        return null;
+    }
+}
+async function buildCommandVectors(knowledgeDir, storePath, provider, model) {
+    const { buildVectors, writeVectorsFile } = await import("./webVectors.js");
+    const vectorsFile = await buildVectors(knowledgeDir, {
+        provider,
+        model,
+        storePath,
+    });
+    const outputPath = await writeVectorsFile(knowledgeDir, vectorsFile);
+    return {
+        model: vectorsFile.model,
+        dims: vectorsFile.dims,
+        count: Object.keys(vectorsFile.vectors).length,
+        outputPath,
+    };
+}

package/dist/lib/webBuildIndexCommand.d.ts

@@ -3,6 +3,9 @@ export type WebBuildIndexCommandOptions = {
     input?: string;
     output?: string;
     stopWords: boolean;
+    embeddings?: string;
+    embedModel?: string;
+    expansions?: boolean;
     json?: boolean;
 };
 export declare function runWebBuildIndexCommand(getWebStorePath: GetWebStorePath, opts: WebBuildIndexCommandOptions): Promise<void>;

package/dist/lib/webBuildIndexCommand.js

@@ -1,21 +1,35 @@
 import path from "path";
+const VECTOR_PROVIDERS = new Set(["openai", "voyage", "local"]);
 export async function runWebBuildIndexCommand(getWebStorePath, opts) {
     try {
         const { loadConfig } = await import("./config.js");
-        const { buildIndex, writeIndex } = await import("./webIndex.js");
-        const gnosysConfig = await loadConfig(await getWebStorePath());
+        const { attachExpansions, buildIndex, generateExpansions, writeIndex } = await import("./webIndex.js");
+        const storePath = await getWebStorePath();
+        const gnosysConfig = await loadConfig(storePath);
         const knowledgeDir = opts.input || gnosysConfig.web?.outputDir || "./knowledge";
         const outputPath = opts.output || path.join(knowledgeDir, "gnosys-index.json");
-        const index = await buildIndex(knowledgeDir, {
+        const embeddingProvider = parseVectorProvider(opts.embeddings);
+        let index = await buildIndex(knowledgeDir, {
             stopWords: opts.stopWords,
         });
+        if (opts.expansions !== false) {
+            const llmProvider = await resolveExpansionProvider(gnosysConfig);
+            if (llmProvider) {
+                const expansions = await generateExpansions(index, llmProvider);
+                index = attachExpansions(index, expansions);
+            }
+        }
         await writeIndex(index, outputPath);
+        const vectors = embeddingProvider
+            ? await buildCommandVectors(knowledgeDir, storePath, embeddingProvider, opts.embedModel)
+            : undefined;
         if (opts.json) {
             console.log(JSON.stringify({
                 ok: true,
                 documentCount: index.documentCount,
                 tokenCount: Object.keys(index.invertedIndex).length,
                 outputPath,
+                ...(vectors ? { vectors } : {}),
             }));
         }
         else {
@@ -23,6 +37,10 @@ export async function runWebBuildIndexCommand(getWebStorePath, opts) {
             console.log(`  Documents: ${index.documentCount}`);
             console.log(`  Tokens:    ${Object.keys(index.invertedIndex).length}`);
             console.log(`  Output:    ${outputPath}`);
+            if (vectors) {
+                console.log(`  Vectors:   ${vectors.count} docs, ${vectors.model} (${vectors.dims}d)`);
+                console.log(`  Vector output: ${vectors.outputPath}`);
+            }
         }
     }
     catch (err) {
@@ -35,3 +53,36 @@ export async function runWebBuildIndexCommand(getWebStorePath, opts) {
         process.exit(1);
     }
 }
+function parseVectorProvider(provider) {
+    if (!provider)
+        return undefined;
+    const normalized = provider.trim().toLowerCase();
+    if (!VECTOR_PROVIDERS.has(normalized)) {
+        throw new Error(`Invalid embeddings provider "${provider}". Valid providers: openai, voyage, local.`);
+    }
+    return normalized;
+}
+async function resolveExpansionProvider(gnosysConfig) {
+    try {
+        const { getLLMProvider } = await import("./llm.js");
+        return getLLMProvider(gnosysConfig, "structuring");
+    }
+    catch {
+        return null;
+    }
+}
+async function buildCommandVectors(knowledgeDir, storePath, provider, model) {
+    const { buildVectors, writeVectorsFile } = await import("./webVectors.js");
+    const vectorsFile = await buildVectors(knowledgeDir, {
+        provider,
+        model,
+        storePath,
+    });
+    const outputPath = await writeVectorsFile(knowledgeDir, vectorsFile);
+    return {
+        model: vectorsFile.model,
+        dims: vectorsFile.dims,
+        count: Object.keys(vectorsFile.vectors).length,
+        outputPath,
+    };
+}

package/dist/lib/webIndex.d.ts

@@ -11,6 +11,10 @@ export interface BuildIndexOptions {
     minTokenLength?: number;
     maxTokensPerDoc?: number;
     includeArchived?: boolean;
+    expansions?: Record<string, string[]>;
+}
+export interface ExpansionProvider {
+    generate(prompt: string, options?: unknown): Promise<string>;
 }
 /**
  * Build a search index from a directory of Gnosys markdown files.
@@ -20,6 +24,19 @@ export declare function buildIndexSync(knowledgeDir: string, options?: BuildInde
  * Build a search index (async wrapper).
  */
 export declare function buildIndex(knowledgeDir: string, options?: BuildIndexOptions): Promise<GnosysWebIndex>;
+/**
+ * Generate a token expansion map from an existing index using an injected LLM.
+ * The provider is resolved by callers so this build module remains easy to test
+ * and does not depend on runtime config or provider wiring.
+ */
+export declare function generateExpansions(index: GnosysWebIndex, llmProvider: ExpansionProvider, options?: {
+    maxTokens?: number;
+}): Promise<Record<string, string[]>>;
+/**
+ * Attach a non-empty concept expansion map to an index and bump it to v2.
+ * Empty maps are ignored so default/no-enrichment builds keep emitting v1.
+ */
+export declare function attachExpansions(index: GnosysWebIndex, expansions?: Record<string, string[]>): GnosysWebIndex;
 /**
  * Write an index to a JSON file.
  */