@mcp-graph-workflow/mcp-graph 3.0.0

package/dist/core/rag/rag-pipeline.js

@@ -0,0 +1,276 @@
+/**
+ * RAG Semantic Pipeline — indexes nodes as embeddings and provides semantic search.
+ *
+ * Strategy: Uses lightweight TF-IDF vectorization for embeddings (no external deps).
+ * This provides semantic-like search without requiring transformers.js (~400MB).
+ * Can be upgraded to real transformer embeddings later.
+ *
+ * All processing is local — no external API calls.
+ */
+import { KnowledgeStore } from "../store/knowledge-store.js";
+import { logger } from "../utils/logger.js";
+// ── TF-IDF Vectorizer ───────────────────────────
+/**
+ * Simple tokenizer: lowercase, split on non-alphanumeric, remove stopwords.
+ */
+const STOPWORDS = new Set([
+    "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+    "have", "has", "had", "do", "does", "did", "will", "would", "shall",
+    "should", "may", "might", "can", "could", "must", "to", "of", "in",
+    "for", "on", "with", "at", "by", "from", "as", "into", "through",
+    "during", "before", "after", "above", "below", "between", "and", "but",
+    "or", "nor", "not", "so", "yet", "both", "either", "neither", "each",
+    "every", "all", "any", "few", "more", "most", "other", "some", "such",
+    "no", "only", "own", "same", "than", "too", "very", "that", "this",
+    "these", "those", "it", "its", "he", "she", "they", "them", "their",
+    "we", "our", "you", "your", "i", "me", "my", "what", "which", "who",
+    "whom", "when", "where", "why", "how",
+]);
+function tokenize(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^a-z0-9\s]/g, " ")
+        .split(/\s+/)
+        .filter((t) => t.length > 1 && !STOPWORDS.has(t));
+}
+/**
+ * Build a vocabulary from all documents and compute IDF values.
+ */
+function buildVocabulary(documents) {
+    const docFreq = new Map();
+    const allTerms = new Set();
+    for (const tokens of documents) {
+        const unique = new Set(tokens);
+        for (const term of unique) {
+            allTerms.add(term);
+            docFreq.set(term, (docFreq.get(term) ?? 0) + 1);
+        }
+    }
+    // Assign index to each term
+    const vocab = new Map();
+    let idx = 0;
+    for (const term of allTerms) {
+        vocab.set(term, idx++);
+    }
+    // Compute IDF: log(N / df)
+    const N = documents.length;
+    const idf = new Map();
+    for (const [term, df] of docFreq) {
+        idf.set(term, Math.log((N + 1) / (df + 1)) + 1); // smoothed IDF
+    }
+    return { vocab, idf };
+}
+/**
+ * Compute TF-IDF vector for a document using the given vocabulary.
+ */
+function computeTfIdfVector(tokens, vocab, idf) {
+    const vector = new Array(vocab.size).fill(0);
+    // Term frequency
+    const tf = new Map();
+    for (const token of tokens) {
+        tf.set(token, (tf.get(token) ?? 0) + 1);
+    }
+    // TF-IDF
+    for (const [term, count] of tf) {
+        const idx = vocab.get(term);
+        if (idx !== undefined) {
+            const termIdf = idf.get(term) ?? 1;
+            vector[idx] = (count / tokens.length) * termIdf;
+        }
+    }
+    // L2 normalize
+    let norm = 0;
+    for (const v of vector)
+        norm += v * v;
+    norm = Math.sqrt(norm);
+    if (norm > 0) {
+        for (let i = 0; i < vector.length; i++) {
+            vector[i] /= norm;
+        }
+    }
+    return vector;
+}
+/**
+ * TF-IDF Vectorizer — encapsulates vocabulary and IDF state per instance.
+ * Avoids module-level mutable state.
+ */
+export class TfIdfVectorizer {
+    vocab = null;
+    idf = null;
+    get vocabSize() {
+        return this.vocab?.size ?? 0;
+    }
+    /**
+     * Build vocabulary and IDF from tokenized documents.
+     */
+    fit(documents) {
+        const result = buildVocabulary(documents);
+        this.vocab = result.vocab;
+        this.idf = result.idf;
+    }
+    /**
+     * Generate an embedding for text using TF-IDF vectorization.
+     * Falls back to hash-based embedding if vocabulary not built.
+     */
+    embed(text) {
+        if (!this.vocab || !this.idf) {
+            return hashEmbed(text, 128);
+        }
+        return computeTfIdfVector(tokenize(text), this.vocab, this.idf);
+    }
+}
+/**
+ * Simple hash-based embedding fallback.
+ * Maps text to a fixed-size vector using character hashing.
+ */
+function hashEmbed(text, dim) {
+    const vector = new Array(dim).fill(0);
+    const tokens = tokenize(text);
+    for (const token of tokens) {
+        let hash = 0;
+        for (let i = 0; i < token.length; i++) {
+            hash = ((hash << 5) - hash + token.charCodeAt(i)) | 0;
+        }
+        const idx = Math.abs(hash) % dim;
+        vector[idx] += 1;
+    }
+    // L2 normalize
+    let norm = 0;
+    for (const v of vector)
+        norm += v * v;
+    norm = Math.sqrt(norm);
+    if (norm > 0) {
+        for (let i = 0; i < vector.length; i++) {
+            vector[i] /= norm;
+        }
+    }
+    return vector;
+}
+// ── Pipeline functions ──────────────────────────
+// Module-level vectorizer shared between index and search.
+// Encapsulated in class — no raw mutable `let` variables.
+let activeVectorizer = null;
+/**
+ * Index all nodes from the store as embeddings.
+ * Builds a TF-IDF vocabulary from all node texts, then stores vectors.
+ */
+export async function indexNodeEmbeddings(store, embeddingStore) {
+    const nodes = store.getAllNodes();
+    if (nodes.length === 0) {
+        logger.info("No nodes to index for embeddings");
+        return 0;
+    }
+    // Build document texts
+    const documents = nodes.map((node) => {
+        const text = [
+            node.title,
+            node.description ?? "",
+            (node.tags ?? []).join(" "),
+            node.type,
+        ].filter(Boolean).join(" | ");
+        return {
+            id: `node:${node.id}`,
+            text,
+            tokens: tokenize(text),
+        };
+    });
+    // Build vocabulary and fit vectorizer
+    const t0 = performance.now();
+    const vectorizer = new TfIdfVectorizer();
+    vectorizer.fit(documents.map((d) => d.tokens));
+    activeVectorizer = vectorizer;
+    // Generate and store embeddings
+    let indexed = 0;
+    for (const doc of documents) {
+        const vector = vectorizer.embed(doc.text);
+        embeddingStore.upsert({
+            id: doc.id,
+            source: "node",
+            sourceId: doc.id.replace("node:", ""),
+            text: doc.text,
+            embedding: vector,
+        });
+        indexed++;
+    }
+    const durationMs = Math.round(performance.now() - t0);
+    logger.debug("rag:fit+embed:nodes", { vocabSize: vectorizer.vocabSize, indexed, durationMs });
+    logger.info(`Indexed ${indexed} node embeddings (vocab size: ${vectorizer.vocabSize})`);
+    return indexed;
+}
+/**
+ * Index all sources (nodes + knowledge documents) as embeddings with a unified vocabulary.
+ * This produces cross-source similarity by fitting TF-IDF on the combined corpus.
+ */
+export async function indexAllEmbeddings(store, embeddingStore) {
+    const nodes = store.getAllNodes();
+    const knowledgeStore = new KnowledgeStore(store.getDb());
+    const knowledgeDocs = knowledgeStore.list({ limit: 10000 });
+    if (nodes.length === 0 && knowledgeDocs.length === 0) {
+        logger.info("No documents to index for embeddings");
+        return { nodes: 0, knowledge: 0 };
+    }
+    // Build combined document corpus
+    const allDocuments = [];
+    for (const node of nodes) {
+        const text = [
+            node.title,
+            node.description ?? "",
+            (node.tags ?? []).join(" "),
+            node.type,
+        ].filter(Boolean).join(" | ");
+        allDocuments.push({
+            id: `node:${node.id}`,
+            text,
+            tokens: tokenize(text),
+        });
+    }
+    for (const doc of knowledgeDocs) {
+        const text = [doc.title, doc.content].join(" | ");
+        allDocuments.push({
+            id: `knowledge:${doc.id}`,
+            text,
+            tokens: tokenize(text),
+        });
+    }
+    // Build unified vocabulary
+    const t0 = performance.now();
+    const vectorizer = new TfIdfVectorizer();
+    vectorizer.fit(allDocuments.map((d) => d.tokens));
+    activeVectorizer = vectorizer;
+    // Generate and store embeddings
+    let indexedNodes = 0;
+    let indexedKnowledge = 0;
+    for (const doc of allDocuments) {
+        const vector = vectorizer.embed(doc.text);
+        const isNode = doc.id.startsWith("node:");
+        embeddingStore.upsert({
+            id: doc.id,
+            source: isNode ? "node" : "knowledge",
+            sourceId: doc.id.replace(/^(node|knowledge):/, ""),
+            text: doc.text,
+            embedding: vector,
+        });
+        if (isNode)
+            indexedNodes++;
+        else
+            indexedKnowledge++;
+    }
+    const durationMs = Math.round(performance.now() - t0);
+    logger.debug("rag:fit+embed:all", { vocabSize: vectorizer.vocabSize, indexedNodes, indexedKnowledge, durationMs });
+    logger.info(`Indexed all embeddings (vocab size: ${vectorizer.vocabSize})`, { nodes: indexedNodes, knowledge: indexedKnowledge });
+    return { nodes: indexedNodes, knowledge: indexedKnowledge };
+}
+/**
+ * Semantic search using embeddings.
+ * Embeds the query and finds similar documents.
+ */
+export async function semanticSearch(embeddingStore, query, limit = 10) {
+    const t0 = performance.now();
+    const vectorizer = activeVectorizer ?? new TfIdfVectorizer();
+    const queryVector = vectorizer.embed(query);
+    const results = embeddingStore.findSimilar(queryVector, limit);
+    const durationMs = Math.round(performance.now() - t0);
+    logger.debug("rag:search", { query: query.slice(0, 80), resultCount: results.length, durationMs });
+    return results;
+}
+//# sourceMappingURL=rag-pipeline.js.map

package/dist/core/rag/rag-pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rag-pipeline.js","sourceRoot":"","sources":["../../../src/core/rag/rag-pipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAE7D,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,mDAAmD;AAEnD;;GAEG;AACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;IACnE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IACnE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;IAClE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS;IAChE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK;IACtE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM;IACpE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM;IACrE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAClE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IACnE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK;IACnE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK;CACtC,CAAC,CAAC;AAEH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;SAC5B,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,SAAqB;IAC5C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;IACzB,CAAC;IAED,2BAA2B;IAC3B,MAAM,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,OAAO,EAAE,CAAC;QACjC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe;IAClE,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;AACxB,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,MAAgB,EAAE,KAA0B,EAAE,GAAwB;IAChG,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE7C,iBAAiB;IACjB,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,SAAS;IACT,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,MAAM,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC;QAClD,CAAC;IACH,CAAC;IAED,eAAe;IACf,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,eAAe;IAClB,KAAK,GAA+B,IAAI,CAAC;IACzC,GAAG,GAA+B,IAAI,CAAC;IAE/C,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,SAAqB;QACvB,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAY;QAChB,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,OAAO,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAClE,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,SAAS,CAAC,IAAY,EAAE,GAAW;IAC1C,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAE9B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACxD,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;QACjC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IAED,eAAe;IACf,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,mDAAmD;AAEnD,2DAA2D;AAC3D,0DAA0D;AAC1D,IAAI,gBAAgB,GAA2B,IAAI,CAAC;AAEpD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAAkB,EAClB,cAA8B;IAE9B,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QAChD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,uBAAuB;IACvB,MAAM,SAAS,GAA0D,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1F,MAAM,IAAI,GAAG;YACX,IAAI,CAAC,KAAK;YACV,IAAI,CAAC,WAAW,IAAI,EAAE;YACtB,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAC3B,IAAI,CAAC,IAAI;SACV,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE9B,OAAO;YACL,EAAE,EAAE,QAAQ,IAAI,CAAC,EAAE,EAAE;YACrB,IAAI;YACJ,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC;SACvB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,sCAAsC;IACtC,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/C,gBAAgB,GAAG,UAAU,CAAC;IAE9B,gCAAgC;IAChC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE1C,cAAc,CAAC,MAAM,CAAC;YACpB,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACrC,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,SAAS,EAAE,MAAM;SAClB,CAAC,CAAC;QACH,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IACtD,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;IAC9F,MAAM,CAAC,IAAI,CAAC,WAAW,OAAO,iCAAiC,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC;IACxF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAAkB,EAClB,cAA8B;IAE9B,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAClC,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACzD,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IAE5D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrD,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QACpD,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IACpC,CAAC;IAED,iCAAiC;IACjC,MAAM,YAAY,GAA0D,EAAE,CAAC;IAE/E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG;YACX,IAAI,CAAC,KAAK;YACV,IAAI,CAAC,WAAW,IAAI,EAAE;YACtB,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAC3B,IAAI,CAAC,IAAI;SACV,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE9B,YAAY,CAAC,IAAI,CAAC;YAChB,EAAE,EAAE,QAAQ,IAAI,CAAC,EAAE,EAAE;YACrB,IAAI;YACJ,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC;SACvB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAElD,YAAY,CAAC,IAAI,CAAC;YAChB,EAAE,EAAE,aAAa,GAAG,CAAC,EAAE,EAAE;YACzB,IAAI;YACJ,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC;SACvB,CAAC,CAAC;IACL,CAAC;IAED,2BAA2B;IAC3B,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAClD,gBAAgB,GAAG,UAAU,CAAC;IAE9B,gCAAgC;IAChC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IAEzB,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAE1C,cAAc,CAAC,MAAM,CAAC;YACpB,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW;YACrC,QAAQ,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC;YAClD,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,SAAS,EAAE,MAAM;SAClB,CAAC,CAAC;QAEH,IAAI,MAAM;YAAE,YAAY,EAAE,CAAC;;YACtB,gBAAgB,EAAE,CAAC;IAC1B,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IACtD,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,SAAS,EAAE,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,CAAC,CAAC;IACnH,MAAM,CAAC,IAAI,CACT,uCAAuC,UAAU,CAAC,SAAS,GAAG,EAC9D,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,CACrD,CAAC;IAEF,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC;AAC9D,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,cAA8B,EAC9B,KAAa,EACb,QAAgB,EAAE;IAElB,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,UAAU,GAAG,gBAAgB,IAAI,IAAI,eAAe,EAAE,CAAC;IAC7D,MAAM,WAAW,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,cAAc,CAAC,WAAW,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IAEtD,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IACnG,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/core/rag/serena-indexer.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Serena Indexer — reads .serena/memories/*.md and stores them
+ * as knowledge documents for unified search.
+ */
+import { KnowledgeStore } from "../store/knowledge-store.js";
+export interface SerenaIndexResult {
+    memoriesFound: number;
+    documentsIndexed: number;
+    skippedDuplicates: number;
+}
+/**
+ * Index all Serena memory files into the knowledge store.
+ * Chunks large memories and deduplicates by content hash.
+ */
+export declare function indexSerenaMemories(knowledgeStore: KnowledgeStore, basePath: string): Promise<SerenaIndexResult>;
+//# sourceMappingURL=serena-indexer.d.ts.map

package/dist/core/rag/serena-indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"serena-indexer.d.ts","sourceRoot":"","sources":["../../../src/core/rag/serena-indexer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAI7D,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CACvC,cAAc,EAAE,cAAc,EAC9B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,iBAAiB,CAAC,CA2C5B"}

package/dist/core/rag/serena-indexer.js

@@ -0,0 +1,50 @@
+/**
+ * Serena Indexer — reads .serena/memories/*.md and stores them
+ * as knowledge documents for unified search.
+ */
+import { readAllSerenaMemories } from "../integrations/serena-reader.js";
+import { chunkText } from "./chunk-text.js";
+import { logger } from "../utils/logger.js";
+/**
+ * Index all Serena memory files into the knowledge store.
+ * Chunks large memories and deduplicates by content hash.
+ */
+export async function indexSerenaMemories(knowledgeStore, basePath) {
+    const memories = await readAllSerenaMemories(basePath);
+    if (memories.length === 0) {
+        logger.info("No Serena memories found to index", { basePath });
+        return { memoriesFound: 0, documentsIndexed: 0, skippedDuplicates: 0 };
+    }
+    let documentsIndexed = 0;
+    let skippedDuplicates = 0;
+    const countBefore = knowledgeStore.count("serena");
+    for (const memory of memories) {
+        const chunks = chunkText(memory.content);
+        for (const chunk of chunks) {
+            const doc = knowledgeStore.insert({
+                sourceType: "serena",
+                sourceId: `serena:${memory.name}`,
+                title: chunks.length > 1
+                    ? `${memory.name} [${chunk.index + 1}/${chunks.length}]`
+                    : memory.name,
+                content: chunk.content,
+                chunkIndex: chunk.index,
+                metadata: { sizeBytes: memory.sizeBytes, memoryName: memory.name },
+            });
+            // Check if this was a dedup hit (id already existed)
+            if (knowledgeStore.count("serena") === countBefore + documentsIndexed) {
+                skippedDuplicates++;
+            }
+            else {
+                documentsIndexed++;
+            }
+        }
+    }
+    logger.info("Serena memories indexed", {
+        memoriesFound: memories.length,
+        documentsIndexed,
+        skippedDuplicates,
+    });
+    return { memoriesFound: memories.length, documentsIndexed, skippedDuplicates };
+}
+//# sourceMappingURL=serena-indexer.js.map

package/dist/core/rag/serena-indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"serena-indexer.js","sourceRoot":"","sources":["../../../src/core/rag/serena-indexer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAEzE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAQ5C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,cAA8B,EAC9B,QAAgB;IAEhB,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAEvD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC/D,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAC;IACzE,CAAC;IAED,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,MAAM,WAAW,GAAG,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAEnD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,cAAc,CAAC,MAAM,CAAC;gBAChC,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,UAAU,MAAM,CAAC,IAAI,EAAE;gBACjC,KAAK,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;oBACtB,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG;oBACxD,CAAC,CAAC,MAAM,CAAC,IAAI;gBACf,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,UAAU,EAAE,KAAK,CAAC,KAAK;gBACvB,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE;aACnE,CAAC,CAAC;YAEH,qDAAqD;YACrD,IAAI,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,WAAW,GAAG,gBAAgB,EAAE,CAAC;gBACtE,iBAAiB,EAAE,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,gBAAgB,EAAE,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;QACrC,aAAa,EAAE,QAAQ,CAAC,MAAM;QAC9B,gBAAgB;QAChB,iBAAiB;KAClB,CAAC,CAAC;IAEH,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,MAAM,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,CAAC;AACjF,CAAC"}

package/dist/core/rag/serena-rag-query.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Serena RAG Query — semantic search over Serena memory documents.
+ *
+ * Supports three modes:
+ * - fts: keyword search via FTS5 (fast, exact match)
+ * - semantic: TF-IDF cosine similarity (broader, cross-vocabulary)
+ * - hybrid: combines FTS and semantic results with deduplication
+ */
+import type { SqliteStore } from "../store/sqlite-store.js";
+export interface SerenaRagOptions {
+    /** Search mode: fts (keyword), semantic (embeddings), hybrid (both) */
+    mode?: "fts" | "semantic" | "hybrid";
+    /** Maximum results to return */
+    limit?: number;
+}
+export interface SerenaRagResultItem {
+    id: string;
+    title: string;
+    content: string;
+    sourceId: string;
+    score: number;
+    method: "fts" | "semantic";
+}
+export interface SerenaRagResult {
+    query: string;
+    mode: "fts" | "semantic" | "hybrid";
+    results: SerenaRagResultItem[];
+    totalSerenaDocuments: number;
+}
+/**
+ * Query Serena memories using FTS, semantic search, or hybrid.
+ */
+export declare function querySerenaMemories(store: SqliteStore, query: string, options?: SerenaRagOptions): Promise<SerenaRagResult>;
+//# sourceMappingURL=serena-rag-query.d.ts.map

package/dist/core/rag/serena-rag-query.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"serena-rag-query.d.ts","sourceRoot":"","sources":["../../../src/core/rag/serena-rag-query.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAM5D,MAAM,WAAW,gBAAgB;IAC/B,uEAAuE;IACvE,IAAI,CAAC,EAAE,KAAK,GAAG,UAAU,GAAG,QAAQ,CAAC;IACrC,gCAAgC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,KAAK,GAAG,UAAU,CAAC;CAC5B;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,KAAK,GAAG,UAAU,GAAG,QAAQ,CAAC;IACpC,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC,CAyE1B"}

package/dist/core/rag/serena-rag-query.js

@@ -0,0 +1,83 @@
+/**
+ * Serena RAG Query — semantic search over Serena memory documents.
+ *
+ * Supports three modes:
+ * - fts: keyword search via FTS5 (fast, exact match)
+ * - semantic: TF-IDF cosine similarity (broader, cross-vocabulary)
+ * - hybrid: combines FTS and semantic results with deduplication
+ */
+import { KnowledgeStore } from "../store/knowledge-store.js";
+import { EmbeddingStore } from "./embedding-store.js";
+import { indexAllEmbeddings, semanticSearch } from "./rag-pipeline.js";
+import { logger } from "../utils/logger.js";
+/**
+ * Query Serena memories using FTS, semantic search, or hybrid.
+ */
+export async function querySerenaMemories(store, query, options) {
+    const mode = options?.mode ?? "fts";
+    const limit = options?.limit ?? 10;
+    const knowledgeStore = new KnowledgeStore(store.getDb());
+    const totalSerenaDocuments = knowledgeStore.count("serena");
+    if (totalSerenaDocuments === 0) {
+        logger.info("No Serena memories in knowledge store", { query });
+        return { query, mode, results: [], totalSerenaDocuments: 0 };
+    }
+    const results = [];
+    const seenIds = new Set();
+    // FTS search
+    if (mode === "fts" || mode === "hybrid") {
+        const ftsResults = knowledgeStore.search(query, limit);
+        for (const doc of ftsResults) {
+            if (doc.sourceType !== "serena")
+                continue;
+            if (seenIds.has(doc.id))
+                continue;
+            seenIds.add(doc.id);
+            results.push({
+                id: doc.id,
+                title: doc.title,
+                content: doc.content,
+                sourceId: doc.sourceId,
+                score: 1.0, // FTS results are relevance-ranked but no numeric score exposed
+                method: "fts",
+            });
+        }
+    }
+    // Semantic search
+    if (mode === "semantic" || mode === "hybrid") {
+        const embeddingStore = new EmbeddingStore(store);
+        // Ensure embeddings are indexed
+        await indexAllEmbeddings(store, embeddingStore);
+        const semanticResults = await semanticSearch(embeddingStore, query, limit);
+        for (const result of semanticResults) {
+            // Filter to only serena-sourced knowledge
+            if (result.source !== "knowledge")
+                continue;
+            const doc = knowledgeStore.getById(result.sourceId);
+            if (!doc || doc.sourceType !== "serena")
+                continue;
+            if (seenIds.has(doc.id))
+                continue;
+            seenIds.add(doc.id);
+            results.push({
+                id: doc.id,
+                title: doc.title,
+                content: doc.content,
+                sourceId: doc.sourceId,
+                score: result.similarity,
+                method: "semantic",
+            });
+        }
+    }
+    // Sort by score descending and limit
+    results.sort((a, b) => b.score - a.score);
+    const limited = results.slice(0, limit);
+    logger.info("Serena RAG query completed", {
+        query,
+        mode,
+        resultsCount: limited.length,
+        totalSerenaDocuments,
+    });
+    return { query, mode, results: limited, totalSerenaDocuments };
+}
+//# sourceMappingURL=serena-rag-query.js.map

package/dist/core/rag/serena-rag-query.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"serena-rag-query.js","sourceRoot":"","sources":["../../../src/core/rag/serena-rag-query.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAyB5C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAAkB,EAClB,KAAa,EACb,OAA0B;IAE1B,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,KAAK,CAAC;IACpC,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;IAEnC,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACzD,MAAM,oBAAoB,GAAG,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE5D,IAAI,oBAAoB,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,CAAC,IAAI,CAAC,uCAAuC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAChE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,oBAAoB,EAAE,CAAC,EAAE,CAAC;IAC/D,CAAC;IAED,MAAM,OAAO,GAA0B,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,aAAa;IACb,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACxC,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,IAAI,GAAG,CAAC,UAAU,KAAK,QAAQ;gBAAE,SAAS;YAC1C,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS;YAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACpB,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,KAAK,EAAE,GAAG,EAAE,gEAAgE;gBAC5E,MAAM,EAAE,KAAK;aACd,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7C,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,KAAK,CAAC,CAAC;QAEjD,gCAAgC;QAChC,MAAM,kBAAkB,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;QAEhD,MAAM,eAAe,GAAG,MAAM,cAAc,CAAC,cAAc,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAC3E,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;YACrC,0CAA0C;YAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW;gBAAE,SAAS;YAE5C,MAAM,GAAG,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpD,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,KAAK,QAAQ;gBAAE,SAAS;YAClD,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS;YAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAEpB,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,KAAK,EAAE,MAAM,CAAC,UAAU;gBACxB,MAAM,EAAE,UAAU;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,qCAAqC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAExC,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;QACxC,KAAK;QACL,IAAI;QACJ,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,oBAAoB;KACrB,CAAC,CAAC;IAEH,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC;AACjE,CAAC"}

package/dist/core/search/fts-search.d.ts

@@ -0,0 +1,16 @@
+import type { SqliteStore } from "../store/sqlite-store.js";
+import type { GraphNode } from "../graph/graph-types.js";
+export interface SearchResult {
+    node: GraphNode;
+    score: number;
+}
+export interface SearchOptions {
+    limit?: number;
+    rerank?: boolean;
+}
+/**
+ * Search nodes using FTS5 full-text search with BM25 ranking.
+ * Optionally applies TF-IDF reranking for better relevance.
+ */
+export declare function searchNodes(store: SqliteStore, query: string, options?: SearchOptions): SearchResult[];
+//# sourceMappingURL=fts-search.d.ts.map

package/dist/core/search/fts-search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts-search.d.ts","sourceRoot":"","sources":["../../../src/core/search/fts-search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAIzD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAqBD;;;GAGG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,aAAkB,GAC1B,YAAY,EAAE,CA0ChB"}

package/dist/core/search/fts-search.js

@@ -0,0 +1,62 @@
+import { logger } from "../utils/logger.js";
+import { rerankWithTfIdf } from "./tfidf.js";
+/**
+ * Sanitize user query for FTS5 — escape special characters and
+ * convert spaces to implicit AND (FTS5 default).
+ */
+function sanitizeFtsQuery(raw) {
+    // Remove FTS5 special operators that could cause syntax errors
+    const cleaned = raw
+        .replace(/[*"(){}[\]:^~!@#$%&|\\]/g, " ")
+        .replace(/\b(AND|OR|NOT|NEAR)\b/gi, " ")
+        .trim()
+        .replace(/\s+/g, " ");
+    if (!cleaned)
+        return '""';
+    // Wrap each term in double quotes for exact matching, join with space (implicit AND)
+    const terms = cleaned.split(" ").filter(Boolean);
+    return terms.map((t) => `"${t}"`).join(" ");
+}
+/**
+ * Search nodes using FTS5 full-text search with BM25 ranking.
+ * Optionally applies TF-IDF reranking for better relevance.
+ */
+export function searchNodes(store, query, options = {}) {
+    const { limit = 20, rerank = false } = options;
+    const sanitized = sanitizeFtsQuery(query);
+    logger.debug("FTS search start", { query, sanitized, rerank });
+    // Stage 1: FTS5 candidates (fetch extra for reranking)
+    const startMs = performance.now();
+    const candidateLimit = rerank ? Math.min(limit * 3, 100) : limit;
+    const ftsResults = store.searchNodes(sanitized, candidateLimit);
+    logger.debug("FTS search complete", {
+        resultCount: ftsResults.length,
+        durationMs: Math.round(performance.now() - startMs),
+    });
+    const resultMap = new Map();
+    for (const r of ftsResults) {
+        const { score, ...node } = r;
+        resultMap.set(node.id, node);
+    }
+    if (!rerank || ftsResults.length === 0) {
+        return ftsResults.map((r) => {
+            const { score, ...node } = r;
+            return { node: node, score };
+        });
+    }
+    // Stage 2: TF-IDF reranking
+    const candidates = ftsResults.map((r) => ({
+        id: r.id,
+        text: [r.title, r.description ?? "", r.tags?.join(" ") ?? ""].join(" "),
+    }));
+    const reranked = rerankWithTfIdf(candidates, query, limit);
+    return reranked
+        .map((r) => {
+        const node = resultMap.get(r.id);
+        if (!node)
+            return null;
+        return { node, score: r.score };
+    })
+        .filter((r) => r !== null);
+}
+//# sourceMappingURL=fts-search.js.map

package/dist/core/search/fts-search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts-search.js","sourceRoot":"","sources":["../../../src/core/search/fts-search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAY7C;;;GAGG;AACH,SAAS,gBAAgB,CAAC,GAAW;IACnC,+DAA+D;IAC/D,MAAM,OAAO,GAAG,GAAG;SAChB,OAAO,CAAC,0BAA0B,EAAE,GAAG,CAAC;SACxC,OAAO,CAAC,yBAAyB,EAAE,GAAG,CAAC;SACvC,IAAI,EAAE;SACN,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAExB,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,qFAAqF;IACrF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACjD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,KAAkB,EAClB,KAAa,EACb,UAAyB,EAAE;IAE3B,MAAM,EAAE,KAAK,GAAG,EAAE,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAC/C,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAC1C,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAE/D,uDAAuD;IACvD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACjE,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAChE,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE;QAClC,WAAW,EAAE,UAAU,CAAC,MAAM;QAC9B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;KACpD,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC;QAC7B,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAiB,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvC,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC1B,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC;YAC7B,OAAO,EAAE,IAAI,EAAE,IAAiB,EAAE,KAAK,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC;IAED,4BAA4B;IAC5B,MAAM,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,WAAW,IAAI,EAAE,EAAG,CAAoC,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;KAC5G,CAAC,CAAC,CAAC;IAEJ,MAAM,QAAQ,GAAG,eAAe,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAE3D,OAAO,QAAQ;SACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QACvB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAClC,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AAClD,CAAC"}

package/dist/core/search/tfidf.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Lightweight TF-IDF implementation for two-stage search reranking.
+ * No external dependencies — pure TypeScript.
+ */
+export declare class TfIdfIndex {
+    private docs;
+    private docFreq;
+    /**
+     * Add a document to the index.
+     */
+    addDocument(id: string, text: string): void;
+    /**
+     * Compute TF-IDF score for a query against all documents.
+     * Returns sorted results (highest score first).
+     */
+    search(query: string, limit?: number): Array<{
+        id: string;
+        score: number;
+    }>;
+}
+/**
+ * Build a TF-IDF index from search result nodes and rerank them.
+ * Two-stage: FTS5 candidates → TF-IDF reranking.
+ */
+export declare function rerankWithTfIdf(candidates: Array<{
+    id: string;
+    text: string;
+}>, query: string, limit?: number): Array<{
+    id: string;
+    score: number;
+}>;
+//# sourceMappingURL=tfidf.d.ts.map

package/dist/core/search/tfidf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tfidf.d.ts","sourceRoot":"","sources":["../../../src/core/search/tfidf.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAUH,qBAAa,UAAU;IACrB,OAAO,CAAC,IAAI,CAAuB;IACnC,OAAO,CAAC,OAAO,CAAkC;IAEjD;;OAEG;IACH,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI;IAgB3C;;;OAGG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CA+BhF;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,UAAU,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,EAC/C,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,MAAW,GACjB,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAMtC"}