gnosys 5.15.3 → 5.17.0

package/dist/lib/webIndex.js

@@ -24,6 +24,8 @@ const STOP_WORDS = new Set([
     "re", "same", "some", "such", "up", "us", "we", "what", "when",
     "where", "which", "who", "whom", "why", "you", "your",
 ]);
+const DEFAULT_EXPANSION_CANDIDATES = 100;
+const MIN_EXPANSION_TOKEN_LENGTH = 3;
 // ─── Tokenization ────────────────────────────────────────────────────────
 function tokenize(text, minLength) {
     return text
@@ -182,13 +184,16 @@ export function buildIndexSync(knowledgeDir, options = {}) {
     for (const token of Object.keys(invertedIndex).sort()) {
         sortedIndex[token] = invertedIndex[token];
     }
-    return {
+    const index = {
         version: 1,
         generated: new Date().toISOString(),
         documentCount: documents.length,
         documents,
         invertedIndex: sortedIndex,
     };
+    // Default builds intentionally remain v1; only indexes with a non-empty
+    // expansion map become v2 so older gnosys/web runtimes can still load them.
+    return attachExpansions(index, options.expansions);
 }
 /**
  * Build a search index (async wrapper).
@@ -196,6 +201,127 @@ export function buildIndexSync(knowledgeDir, options = {}) {
 export async function buildIndex(knowledgeDir, options) {
     return buildIndexSync(knowledgeDir, options);
 }
+/**
+ * Generate a token expansion map from an existing index using an injected LLM.
+ * The provider is resolved by callers so this build module remains easy to test
+ * and does not depend on runtime config or provider wiring.
+ */
+export async function generateExpansions(index, llmProvider, options = {}) {
+    const maxTokens = options.maxTokens ?? DEFAULT_EXPANSION_CANDIDATES;
+    if (maxTokens <= 0)
+        return {};
+    const candidates = selectExpansionCandidates(index, maxTokens);
+    if (candidates.length === 0)
+        return {};
+    const prompt = [
+        "Return strict JSON only: an object mapping each provided token to an array of related search tokens.",
+        "Use lowercase single-word domain concepts. Do not include explanations, markdown, or tokens not in the input list.",
+        "Input tokens:",
+        candidates.join(", "),
+    ].join("\n");
+    let raw;
+    try {
+        raw = await llmProvider.generate(prompt);
+    }
+    catch (error) {
+        console.warn(`[gnosys/web] concept expansion generation failed: ${formatErrorMessage(error)}`);
+        return {};
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw.trim());
+    }
+    catch {
+        console.warn("[gnosys/web] concept expansion provider returned invalid JSON; skipping expansions");
+        return {};
+    }
+    if (!isRecord(parsed))
+        return {};
+    const candidateSet = new Set(candidates);
+    const expansions = {};
+    for (const [rawKey, rawRelated] of Object.entries(parsed)) {
+        const key = tokenize(rawKey, MIN_EXPANSION_TOKEN_LENGTH)[0];
+        if (!key || !candidateSet.has(key) || !Array.isArray(rawRelated))
+            continue;
+        const relatedTokens = normalizeExpansionTokens(rawRelated, key);
+        if (relatedTokens.length > 0) {
+            expansions[key] = relatedTokens;
+        }
+    }
+    return sortExpansionMap(expansions);
+}
+/**
+ * Attach a non-empty concept expansion map to an index and bump it to v2.
+ * Empty maps are ignored so default/no-enrichment builds keep emitting v1.
+ */
+export function attachExpansions(index, expansions) {
+    const normalized = normalizeExpansionMap(expansions);
+    if (Object.keys(normalized).length === 0) {
+        const { expansions: _ignored, ...withoutExpansions } = index;
+        return { ...withoutExpansions, version: 1 };
+    }
+    return {
+        ...index,
+        version: 2,
+        expansions: normalized,
+    };
+}
+function selectExpansionCandidates(index, maxTokens) {
+    return Object.entries(index.invertedIndex)
+        .filter(([token, entries]) => token.length >= MIN_EXPANSION_TOKEN_LENGTH &&
+        !STOP_WORDS.has(token) &&
+        Array.isArray(entries) &&
+        entries.length > 0)
+        .sort(([tokenA, entriesA], [tokenB, entriesB]) => {
+        const frequencyDelta = entriesB.length - entriesA.length;
+        return frequencyDelta || tokenA.localeCompare(tokenB);
+    })
+        .slice(0, maxTokens)
+        .map(([token]) => token);
+}
+function normalizeExpansionMap(expansions) {
+    if (!expansions)
+        return {};
+    const normalized = {};
+    for (const [rawKey, rawRelated] of Object.entries(expansions)) {
+        const key = tokenize(rawKey, MIN_EXPANSION_TOKEN_LENGTH)[0];
+        if (!key || STOP_WORDS.has(key) || !Array.isArray(rawRelated))
+            continue;
+        const relatedTokens = normalizeExpansionTokens(rawRelated, key);
+        if (relatedTokens.length > 0) {
+            normalized[key] = relatedTokens;
+        }
+    }
+    return sortExpansionMap(normalized);
+}
+function normalizeExpansionTokens(values, key) {
+    const tokens = [];
+    for (const value of values) {
+        if (typeof value !== "string")
+            continue;
+        let relatedTokens = tokenize(value, MIN_EXPANSION_TOKEN_LENGTH);
+        relatedTokens = filterStopWords(relatedTokens);
+        for (const token of relatedTokens) {
+            if (token === key || tokens.includes(token))
+                continue;
+            tokens.push(token);
+        }
+    }
+    return tokens.sort();
+}
+function sortExpansionMap(expansions) {
+    const sorted = {};
+    for (const key of Object.keys(expansions).sort()) {
+        sorted[key] = expansions[key];
+    }
+    return sorted;
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function formatErrorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
 /**
  * Write an index to a JSON file.
  */

package/dist/lib/webInitCommand.js

@@ -153,6 +153,7 @@ export async function runWebInitCommand(getWebStorePath, opts) {
             console.log(`  ${!sitemapUrl && envVarName ? "4" : envVarName || !sitemapUrl ? "3" : "2"}. Add to package.json: ${CYAN}"postbuild": "npx gnosys web build"${RESET}`);
             console.log();
             console.log(`${DIM}Every deploy will re-crawl and rebuild the search index automatically.${RESET}`);
+            console.log(`${DIM}Tip: add semantic search with ${CYAN}gnosys web build --embeddings openai${RESET}${DIM} (see docs/web-semantic-search.md).${RESET}`);
         }
     }
     catch (err) {

package/dist/lib/webStatusCommand.js

@@ -52,6 +52,28 @@ export async function runWebStatusCommand(getWebStorePath, opts) {
                 indexInfo = { exists: true, size: stat.size };
             }
         }
+        const vectorsPath = path.join(resolvedDir, "gnosys-vectors.json");
+        let vectorsInfo = { exists: false };
+        if (existsSync(vectorsPath)) {
+            const vectorsStat = statSync(vectorsPath);
+            try {
+                const vectorsData = JSON.parse(readFileSync(vectorsPath, "utf-8"));
+                const vectorMap = typeof vectorsData.vectors === "object" && vectorsData.vectors !== null
+                    ? vectorsData.vectors
+                    : {};
+                vectorsInfo = {
+                    exists: true,
+                    model: typeof vectorsData.model === "string" ? vectorsData.model : undefined,
+                    dims: typeof vectorsData.dims === "number" ? vectorsData.dims : undefined,
+                    count: Object.keys(vectorMap).length,
+                    size: vectorsStat.size,
+                    generated: typeof vectorsData.generated === "string" ? vectorsData.generated : undefined,
+                };
+            }
+            catch {
+                vectorsInfo = { exists: true, size: vectorsStat.size };
+            }
+        }
         if (opts.json) {
             console.log(JSON.stringify({
                 ok: true,
@@ -59,6 +81,7 @@ export async function runWebStatusCommand(getWebStorePath, opts) {
                 totalFiles,
                 categoryCounts,
                 index: indexInfo,
+                vectors: vectorsInfo,
             }, null, 2));
         }
         else {
@@ -80,6 +103,17 @@ export async function runWebStatusCommand(getWebStorePath, opts) {
             else {
                 console.log(`  Index: not built (run 'gnosys web build-index')`);
             }
+            if (vectorsInfo.exists) {
+                if (vectorsInfo.model && typeof vectorsInfo.dims === "number" && typeof vectorsInfo.count === "number") {
+                    console.log(`  Vectors: ${vectorsInfo.count} docs, ${vectorsInfo.model} (${vectorsInfo.dims}d), ${((vectorsInfo.size || 0) / 1024).toFixed(1)}KB`);
+                }
+                else {
+                    console.log(`  Vectors: present, ${((vectorsInfo.size || 0) / 1024).toFixed(1)}KB (unreadable)`);
+                }
+            }
+            else {
+                console.log(`  Vectors: not built (run 'gnosys web build-index --embeddings <provider>')`);
+            }
         }
     }
     catch (err) {

package/dist/lib/webVectors.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Gnosys Web Vectors - build-time semantic vector generator.
+ *
+ * This module is intentionally build-time only. The zero-dependency
+ * `gnosys/web` runtime must not import it because the local provider reaches
+ * into the optional embeddings stack.
+ */
+export type VectorProvider = "openai" | "voyage" | "local";
+export interface WebVectorsFile {
+    version: 1;
+    model: string;
+    dims: number;
+    quantization: "int8";
+    generated: string;
+    scale: number;
+    offset: number;
+    vectors: Record<string, number[]>;
+}
+export interface BuildVectorsOptions {
+    provider: VectorProvider;
+    model?: string;
+    apiKey?: string;
+    storePath?: string;
+}
+/**
+ * Symmetric global int8 quantization. One scale/offset pair is recorded for
+ * the whole file so the runtime can dequantize without per-vector metadata.
+ */
+export declare function quantizeVector(vec: number[], scale: number): number[];
+export declare function dequantizeVector(qvec: number[], scale: number): number[];
+export declare function buildVectors(knowledgeDir: string, options: BuildVectorsOptions): Promise<WebVectorsFile>;
+export declare function writeVectorsFile(knowledgeDir: string, vectorsFile: WebVectorsFile): Promise<string>;

package/dist/lib/webVectors.js

@@ -0,0 +1,255 @@
+/**
+ * Gnosys Web Vectors - build-time semantic vector generator.
+ *
+ * This module is intentionally build-time only. The zero-dependency
+ * `gnosys/web` runtime must not import it because the local provider reaches
+ * into the optional embeddings stack.
+ */
+import fs from "fs/promises";
+import path from "path";
+import matter from "gray-matter";
+import { getOpenAIApiKey, getOpenAIBaseUrl, loadConfig } from "./config.js";
+import { GnosysEmbeddings } from "./embeddings.js";
+const VECTOR_FILE_NAME = "gnosys-vectors.json";
+const MAX_BODY_TOKENS = 512;
+const API_BATCH_SIZE = 96;
+const DEFAULT_OPENAI_MODEL = "text-embedding-3-small";
+const DEFAULT_VOYAGE_MODEL = "voyage-3-lite";
+const DEFAULT_LOCAL_MODEL = "Xenova/all-MiniLM-L6-v2";
+/**
+ * Symmetric global int8 quantization. One scale/offset pair is recorded for
+ * the whole file so the runtime can dequantize without per-vector metadata.
+ */
+export function quantizeVector(vec, scale) {
+    if (!Number.isFinite(scale) || scale <= 0) {
+        throw new Error("Cannot quantize vector with a non-positive scale");
+    }
+    return vec.map((value) => clampInt8(Math.round(value / scale)));
+}
+export function dequantizeVector(qvec, scale) {
+    if (!Number.isFinite(scale) || scale <= 0) {
+        throw new Error("Cannot dequantize vector with a non-positive scale");
+    }
+    return qvec.map((value) => value * scale);
+}
+export async function buildVectors(knowledgeDir, options) {
+    const docs = await readKnowledgeDocs(knowledgeDir);
+    const model = modelForProvider(options);
+    if (docs.length === 0) {
+        return {
+            version: 1,
+            model,
+            dims: 0,
+            quantization: "int8",
+            generated: new Date().toISOString(),
+            scale: 1,
+            offset: 0,
+            vectors: {},
+        };
+    }
+    const texts = docs.map((doc) => embeddingText(doc));
+    const embeddings = await embedTexts(texts, options, model);
+    const dims = validateEmbeddings(embeddings, docs.length);
+    const scale = computeGlobalScale(embeddings);
+    const vectors = {};
+    for (let i = 0; i < docs.length; i++) {
+        const doc = docs[i];
+        if (vectors[doc.id]) {
+            throw new Error(`Duplicate web knowledge document id: ${doc.id}`);
+        }
+        vectors[doc.id] = quantizeVector(embeddings[i], scale);
+    }
+    return {
+        version: 1,
+        model,
+        dims,
+        quantization: "int8",
+        generated: new Date().toISOString(),
+        scale,
+        offset: 0,
+        vectors,
+    };
+}
+export async function writeVectorsFile(knowledgeDir, vectorsFile) {
+    const outputPath = path.join(knowledgeDir, VECTOR_FILE_NAME);
+    await fs.mkdir(path.dirname(outputPath), { recursive: true });
+    await fs.writeFile(outputPath, JSON.stringify(vectorsFile, null, 2), "utf-8");
+    return outputPath;
+}
+async function readKnowledgeDocs(knowledgeDir) {
+    const resolvedDir = path.resolve(knowledgeDir);
+    const files = await findMarkdownFiles(resolvedDir);
+    const docs = [];
+    for (const filePath of files) {
+        let raw;
+        try {
+            raw = await fs.readFile(filePath, "utf-8");
+        }
+        catch {
+            continue;
+        }
+        let parsed;
+        try {
+            parsed = matter(raw);
+        }
+        catch {
+            continue;
+        }
+        const fm = parsed.data;
+        const status = typeof fm.status === "string" ? fm.status : "active";
+        if (status === "archived")
+            continue;
+        docs.push({
+            id: typeof fm.id === "string" ? fm.id : path.basename(filePath, ".md"),
+            title: typeof fm.title === "string" ? fm.title : path.basename(filePath, ".md"),
+            relevance: typeof fm.relevance === "string" ? fm.relevance : "",
+            body: parsed.content.trim(),
+        });
+    }
+    return docs;
+}
+async function findMarkdownFiles(dir) {
+    const results = [];
+    async function walk(current) {
+        let entries;
+        try {
+            entries = await fs.readdir(current, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            const fullPath = path.join(current, entry.name);
+            if (entry.isDirectory()) {
+                await walk(fullPath);
+            }
+            else if (entry.isFile() && entry.name.endsWith(".md")) {
+                results.push(fullPath);
+            }
+        }
+    }
+    await walk(dir);
+    return results.sort();
+}
+function embeddingText(doc) {
+    const cappedBody = doc.body.split(/\s+/).filter(Boolean).slice(0, MAX_BODY_TOKENS).join(" ");
+    return `${doc.title}\n${doc.relevance}\n${cappedBody}`;
+}
+function modelForProvider(options) {
+    if (options.model)
+        return options.model;
+    switch (options.provider) {
+        case "openai":
+            return DEFAULT_OPENAI_MODEL;
+        case "voyage":
+            return DEFAULT_VOYAGE_MODEL;
+        case "local":
+            return DEFAULT_LOCAL_MODEL;
+    }
+}
+async function embedTexts(texts, options, model) {
+    switch (options.provider) {
+        case "openai":
+            return embedOpenAI(texts, options, model);
+        case "voyage":
+            return embedVoyage(texts, options, model);
+        case "local":
+            return embedLocal(texts, options);
+    }
+}
+async function embedOpenAI(texts, options, model) {
+    const config = await loadConfig(options.storePath ?? process.cwd());
+    const apiKey = options.apiKey || getOpenAIApiKey(config) || process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+        throw new Error("OpenAI embeddings require an API key. Set OPENAI_API_KEY or configure an OpenAI key in Gnosys.");
+    }
+    const endpoint = `${getOpenAIBaseUrl(config).replace(/\/+$/, "")}/embeddings`;
+    return embedApiBatches("OpenAI", endpoint, apiKey, texts, model);
+}
+async function embedVoyage(texts, options, model) {
+    // Voyage is not part of the Gnosys config schema yet; for now it is env-only.
+    const apiKey = options.apiKey || process.env.VOYAGE_API_KEY;
+    if (!apiKey) {
+        throw new Error("Voyage embeddings require an API key. Set VOYAGE_API_KEY.");
+    }
+    return embedApiBatches("Voyage", "https://api.voyageai.com/v1/embeddings", apiKey, texts, model);
+}
+async function embedApiBatches(providerName, endpoint, apiKey, texts, model) {
+    const embeddings = [];
+    for (let i = 0; i < texts.length; i += API_BATCH_SIZE) {
+        const input = texts.slice(i, i + API_BATCH_SIZE);
+        const response = await fetch(endpoint, {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${apiKey}`,
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify({ model, input }),
+        });
+        const bodyText = await response.text();
+        if (!response.ok) {
+            const detail = bodyText ? `: ${bodyText.slice(0, 500)}` : "";
+            throw new Error(`${providerName} embedding request failed (${response.status} ${response.statusText})${detail}`);
+        }
+        embeddings.push(...parseEmbeddingResponse(providerName, bodyText));
+    }
+    return embeddings;
+}
+async function embedLocal(texts, options) {
+    const embeddings = new GnosysEmbeddings(options.storePath ?? path.resolve(process.cwd(), ".gnosys"));
+    const vectors = await embeddings.embedBatch(texts);
+    return vectors.map((vec) => Array.from(vec));
+}
+function parseEmbeddingResponse(providerName, bodyText) {
+    let payload;
+    try {
+        payload = JSON.parse(bodyText);
+    }
+    catch {
+        throw new Error(`${providerName} embedding response was not valid JSON`);
+    }
+    if (!isEmbeddingPayload(payload)) {
+        throw new Error(`${providerName} embedding response did not include data[].embedding arrays`);
+    }
+    return payload.data.map((entry) => entry.embedding);
+}
+function isEmbeddingPayload(payload) {
+    if (typeof payload !== "object" || payload === null)
+        return false;
+    const data = payload.data;
+    if (!Array.isArray(data))
+        return false;
+    return data.every((entry) => {
+        if (typeof entry !== "object" || entry === null)
+            return false;
+        const embedding = entry.embedding;
+        return Array.isArray(embedding) && embedding.every((value) => typeof value === "number" && Number.isFinite(value));
+    });
+}
+function validateEmbeddings(embeddings, expectedCount) {
+    if (embeddings.length !== expectedCount) {
+        throw new Error(`Embedding provider returned ${embeddings.length} vectors for ${expectedCount} documents`);
+    }
+    const dims = embeddings[0]?.length ?? 0;
+    if (dims === 0) {
+        throw new Error("Embedding provider returned empty vectors");
+    }
+    for (const vec of embeddings) {
+        if (vec.length !== dims) {
+            throw new Error("Embedding provider returned vectors with inconsistent dimensions");
+        }
+    }
+    return dims;
+}
+function computeGlobalScale(embeddings) {
+    let absMax = 0;
+    for (const vec of embeddings) {
+        for (const value of vec) {
+            absMax = Math.max(absMax, Math.abs(value));
+        }
+    }
+    return absMax > 0 ? absMax / 127 : 1;
+}
+function clampInt8(value) {
+    return Math.max(-128, Math.min(127, value));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gnosys",
-  "version": "5.15.3",
+  "version": "5.17.0",
   "description": "Gnosys — Persistent Memory for AI Agents. Sandbox-first runtime, central SQLite brain, federated search, Dream Mode, Web Knowledge Base, Obsidian export.",
   "type": "module",
   "main": "dist/index.js",