@veewo/claw-core 0.1.21 → 0.1.23

package/dist/src/memory.js

@@ -1,36 +1,201 @@
 import fs from "node:fs";
+import os from "node:os";
 import path from "node:path";
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
 import { DatabaseSync } from "node:sqlite";
+import { fileURLToPath } from "node:url";
 import { resolveProjectContext, resolveTaskContext } from "./context.js";
 import { ClawError } from "./errors.js";
 import { readJsonFile, readTextFile } from "./io.js";
+import { buildProjectKeywordSearchPlan, buildProjectQueryIntent } from "./memory-query.js";
+const DEFAULT_PROJECT_REFRESH_FILE_LIMIT = 100;
+const PROJECT_SEARCH_CANDIDATE_MULTIPLIER = 8;
 export function buildMemoryIndex(input) {
     const { scope, project, task } = resolveMemoryScope(input);
     const storePath = getMemoryStorePath(project, scope, task);
     const sources = collectMemorySources(project, scope, task);
+    const embedding = scope === "project" ? resolveProjectMemoryEmbeddingConfig(project) : undefined;
     fs.mkdirSync(path.dirname(storePath), { recursive: true });
     const db = new DatabaseSync(storePath);
     try {
         prepareSchema(db);
-        db.exec("DELETE FROM docs;");
-        db.exec("DELETE FROM docs_fts;");
-        const insertDoc = db.prepare("INSERT INTO docs (source_path, kind, content) VALUES (?, ?, ?)");
-        const insertFts = db.prepare("INSERT INTO docs_fts (rowid, source_path, kind, content) VALUES (?, ?, ?, ?)");
-        for (const source of sources) {
-            const result = insertDoc.run(source.sourcePath, source.kind, source.content);
-            insertFts.run(result.lastInsertRowid, source.sourcePath, source.kind, source.content);
+        const syncResult = scope === "project"
+            ? syncProjectMemoryIndex(db, sources, embedding ?? null, input.maxFiles ?? DEFAULT_PROJECT_REFRESH_FILE_LIMIT)
+            : {
+                vectorIndex: rebuildTaskMemoryIndex(db, sources),
+                processedFileCount: sources.length,
+                pendingFileCount: 0,
+            };
+        upsertMetadata(db, "scope", scope);
+        upsertMetadata(db, "indexed_at", new Date().toISOString());
+        if (embedding) {
+            upsertMetadata(db, "embedding_config", JSON.stringify(embedding));
         }
+        else {
+            deleteMetadata(db, "embedding_config");
+        }
+        if (syncResult.vectorIndex) {
+            upsertMetadata(db, "vector_index", JSON.stringify(syncResult.vectorIndex));
+        }
+        else {
+            deleteMetadata(db, "vector_index");
+        }
+        return {
+            scope,
+            storePath,
+            indexedCount: sources.length,
+            processedFileCount: syncResult.processedFileCount,
+            pendingFileCount: syncResult.pendingFileCount,
+            sources: sources.map((entry) => entry.sourcePath),
+            ...(scope === "project" ? { embedding, vectorIndex: syncResult.vectorIndex } : {}),
+        };
     }
     finally {
         db.close();
     }
+}
+function rebuildTaskMemoryIndex(db, sources) {
+    db.exec("DELETE FROM docs;");
+    db.exec("DELETE FROM docs_fts;");
+    db.exec("DELETE FROM doc_embeddings;");
+    const insertDoc = db.prepare("INSERT INTO docs (source_path, kind, content, content_hash) VALUES (?, ?, ?, ?)");
+    const insertFts = db.prepare("INSERT INTO docs_fts (rowid, source_path, kind, content) VALUES (?, ?, ?, ?)");
+    for (const source of sources) {
+        const result = insertDoc.run(source.sourcePath, source.kind, source.content, hashMemoryContent(source.content));
+        insertFts.run(Number(result.lastInsertRowid), source.sourcePath, source.kind, source.content);
+    }
+    return null;
+}
+function syncProjectMemoryIndex(db, sources, embedding, maxFiles) {
+    const currentEmbeddingConfig = embedding ? JSON.stringify(embedding) : null;
+    const storedEmbeddingConfig = getMetadata(db, "embedding_config");
+    const shouldIndexVectors = canBuildProjectVectors(embedding);
+    const requiresVectorReset = storedEmbeddingConfig !== currentEmbeddingConfig || !shouldIndexVectors;
+    const nextSources = sources.map((source) => ({
+        ...source,
+        contentHash: hashMemoryContent(source.content),
+    }));
+    const nextByPath = new Map(nextSources.map((source) => [source.sourcePath, source]));
+    const existingDocs = db
+        .prepare("SELECT id, source_path, kind, content_hash FROM docs")
+        .all();
+    const existingByPath = new Map(existingDocs.map((doc) => [doc.source_path, doc]));
+    const docsToDelete = existingDocs.filter((doc) => {
+        const next = nextByPath.get(doc.source_path);
+        if (!next) {
+            return true;
+        }
+        if (requiresVectorReset) {
+            return true;
+        }
+        return doc.kind !== next.kind || doc.content_hash !== next.contentHash;
+    });
+    const docsToInsert = nextSources.filter((source) => {
+        const existing = existingByPath.get(source.sourcePath);
+        if (!existing) {
+            return true;
+        }
+        if (requiresVectorReset) {
+            return true;
+        }
+        return existing.kind !== source.kind || existing.content_hash !== source.contentHash;
+    });
+    const canLimitFiles = !maxFiles
+        ? false
+        : maxFiles > 0 && (!requiresVectorReset || existingDocs.length === 0);
+    const sortedDocsToInsert = [...docsToInsert].sort((left, right) => left.sourcePath.localeCompare(right.sourcePath));
+    const limitedDocsToInsert = canLimitFiles ? sortedDocsToInsert.slice(0, maxFiles) : sortedDocsToInsert;
+    const pendingFileCount = sortedDocsToInsert.length - limitedDocsToInsert.length;
+    db.exec("BEGIN");
+    try {
+        deleteDocsById(db, docsToDelete.map((doc) => doc.id));
+        if (requiresVectorReset) {
+            db.exec("DELETE FROM doc_embeddings;");
+        }
+        const indexedDocs = insertDocs(db, limitedDocsToInsert);
+        let vectorIndexingFailed = false;
+        if (shouldIndexVectors && embedding) {
+            try {
+                indexDocEmbeddings(db, indexedDocs, embedding);
+            }
+            catch (error) {
+                if (!isEmbeddingGenerationFailure(error)) {
+                    throw error;
+                }
+                vectorIndexingFailed = true;
+                db.exec("DELETE FROM doc_embeddings;");
+            }
+        }
+        db.exec("COMMIT");
+        if (vectorIndexingFailed) {
+            return {
+                vectorIndex: null,
+                processedFileCount: limitedDocsToInsert.length,
+                pendingFileCount,
+            };
+        }
+    }
+    catch (error) {
+        db.exec("ROLLBACK");
+        throw error;
+    }
+    if (!shouldIndexVectors || !embedding) {
+        return {
+            vectorIndex: null,
+            processedFileCount: limitedDocsToInsert.length,
+            pendingFileCount,
+        };
+    }
     return {
-        scope,
-        storePath,
-        indexedCount: sources.length,
-        sources: sources.map((entry) => entry.sourcePath),
+        vectorIndex: summarizeVectorIndex(db, embedding),
+        processedFileCount: limitedDocsToInsert.length,
+        pendingFileCount,
     };
 }
+function insertDocs(db, sources) {
+    const insertDoc = db.prepare("INSERT INTO docs (source_path, kind, content, content_hash) VALUES (?, ?, ?, ?)");
+    const insertFts = db.prepare("INSERT INTO docs_fts (rowid, source_path, kind, content) VALUES (?, ?, ?, ?)");
+    const indexedDocs = [];
+    for (const source of sources) {
+        const result = insertDoc.run(source.sourcePath, source.kind, source.content, source.contentHash);
+        const docId = Number(result.lastInsertRowid);
+        insertFts.run(docId, source.sourcePath, source.kind, source.content);
+        indexedDocs.push({
+            docId,
+            sourcePath: source.sourcePath,
+            kind: source.kind,
+            content: source.content,
+        });
+    }
+    return indexedDocs;
+}
+function deleteDocsById(db, docIds) {
+    if (docIds.length === 0) {
+        return;
+    }
+    const deleteFts = db.prepare("DELETE FROM docs_fts WHERE rowid = ?");
+    const deleteEmbeddings = db.prepare("DELETE FROM doc_embeddings WHERE doc_id = ?");
+    const deleteDoc = db.prepare("DELETE FROM docs WHERE id = ?");
+    for (const docId of docIds) {
+        deleteFts.run(docId);
+        deleteEmbeddings.run(docId);
+        deleteDoc.run(docId);
+    }
+}
+function upsertMetadata(db, key, value) {
+    db.prepare([
+        "INSERT INTO index_metadata (key, value) VALUES (?, ?)",
+        "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
+    ].join(" ")).run(key, value);
+}
+function deleteMetadata(db, key) {
+    db.prepare("DELETE FROM index_metadata WHERE key = ?").run(key);
+}
+function getMetadata(db, key) {
+    const row = db.prepare("SELECT value FROM index_metadata WHERE key = ?").get(key);
+    return row?.value ?? null;
+}
 export function searchMemory(input) {
     if (!input.query.trim()) {
         throw new ClawError("MEMORY_QUERY_REQUIRED", "memory search requires a non-empty query.");
@@ -47,22 +212,9 @@ export function searchMemory(input) {
     const db = new DatabaseSync(storePath);
     try {
         prepareSchema(db);
-        const limit = input.limit ?? 10;
-        const rows = db
-            .prepare([
-            "SELECT source_path, kind, snippet(docs_fts, 2, '[', ']', ' ... ', 18) AS snippet, bm25(docs_fts) AS score",
-            "FROM docs_fts",
-            "WHERE docs_fts MATCH ?",
-            "ORDER BY score ASC",
-            "LIMIT ?",
-        ].join(" "))
-            .all(input.query, limit);
-        const results = rows.map((row) => ({
-            sourcePath: row.source_path,
-            kind: row.kind,
-            snippet: row.snippet,
-            score: row.score,
-        }));
+        const results = scope === "project"
+            ? searchProjectMemoryHybrid(db, input.query, input.limit ?? 10, project)
+            : searchTaskMemoryFts(db, input.query, input.limit ?? 10);
         return {
             scope,
             storePath,
@@ -204,14 +356,53 @@ function prepareSchema(db) {
         "  id INTEGER PRIMARY KEY,",
         "  source_path TEXT NOT NULL,",
         "  kind TEXT NOT NULL,",
-        "  content TEXT NOT NULL",
+        "  content TEXT NOT NULL,",
+        "  content_hash TEXT",
         ");",
         "CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts USING fts5(",
         "  source_path,",
         "  kind,",
         "  content",
         ");",
+        "CREATE TABLE IF NOT EXISTS index_metadata (",
+        "  key TEXT PRIMARY KEY,",
+        "  value TEXT NOT NULL",
+        ");",
+        "CREATE TABLE IF NOT EXISTS doc_embeddings (",
+        "  doc_id INTEGER NOT NULL,",
+        "  chunk_index INTEGER NOT NULL,",
+        "  source_path TEXT NOT NULL,",
+        "  kind TEXT NOT NULL,",
+        "  chunk_text TEXT NOT NULL,",
+        "  embedding_json TEXT NOT NULL,",
+        "  PRIMARY KEY (doc_id, chunk_index)",
+        ");",
     ].join("\n"));
+    const docsColumns = db.prepare("PRAGMA table_info(docs)").all();
+    if (!docsColumns.some((column) => column.name === "content_hash")) {
+        db.exec("ALTER TABLE docs ADD COLUMN content_hash TEXT;");
+    }
+}
+function resolveProjectMemoryEmbeddingConfig(project) {
+    const configured = project.projectConfig?.memory?.embedding;
+    if (!configured) {
+        return null;
+    }
+    return {
+        provider: configured.provider,
+        model: configured.model,
+        ...(configured.remote ? { remote: configured.remote } : {}),
+        ...(configured.local ? { local: configured.local } : {}),
+        ...(configured.outputDimensionality ? { outputDimensionality: configured.outputDimensionality } : {}),
+        store: {
+            vector: {
+                enabled: configured.store?.vector?.enabled ?? true,
+                ...(configured.store?.vector?.extensionPath
+                    ? { extensionPath: configured.store.vector.extensionPath }
+                    : {}),
+            },
+        },
+    };
 }
 function listFiles(rootDir, matcher) {
     if (!fs.existsSync(rootDir)) {
@@ -235,6 +426,548 @@ function listFiles(rootDir, matcher) {
     return entries;
 }
 function isExternalDocFile(filePath) {
-    return /\.(md|mdx|txt|json)$/i.test(filePath);
+    return /\.md$/i.test(filePath);
+}
+function indexDocEmbeddings(db, docs, embedding) {
+    if (!embedding || embedding.store?.vector?.enabled === false) {
+        return;
+    }
+    if (!canBuildProjectVectors(embedding)) {
+        return;
+    }
+    const chunks = docs.flatMap((doc) => chunkMarkdownContent(doc.content).map((chunkText, chunkIndex) => ({
+        docId: doc.docId,
+        chunkIndex,
+        sourcePath: doc.sourcePath,
+        kind: doc.kind,
+        chunkText,
+    })));
+    if (chunks.length === 0) {
+        return;
+    }
+    const output = runEmbeddingWorker({
+        embedding,
+        texts: chunks.map((chunk) => chunk.chunkText),
+    });
+    const insertEmbedding = db.prepare([
+        "INSERT INTO doc_embeddings (doc_id, chunk_index, source_path, kind, chunk_text, embedding_json)",
+        "VALUES (?, ?, ?, ?, ?, ?)",
+    ].join(" "));
+    chunks.forEach((chunk, index) => {
+        insertEmbedding.run(chunk.docId, chunk.chunkIndex, chunk.sourcePath, chunk.kind, chunk.chunkText, JSON.stringify(output.vectors[index] ?? []));
+    });
+}
+function summarizeVectorIndex(db, embedding) {
+    const vectorCount = db
+        .prepare("SELECT COUNT(*) AS count FROM doc_embeddings")
+        .get();
+    const firstVector = db
+        .prepare("SELECT embedding_json FROM doc_embeddings ORDER BY doc_id ASC, chunk_index ASC LIMIT 1")
+        .get();
+    const dimensions = firstVector
+        ? parseEmbeddingJson(firstVector.embedding_json).length
+        : resolveEmbeddingDimensions(embedding, 0);
+    return {
+        enabled: true,
+        provider: embedding.provider,
+        model: embedding.model,
+        dimensions,
+        chunkCount: vectorCount.count,
+    };
+}
+function canBuildProjectVectors(embedding) {
+    if (!embedding || embedding.store?.vector?.enabled === false) {
+        return false;
+    }
+    if (embedding.provider === "openai" && !resolveEmbeddingApiKey(embedding)) {
+        return false;
+    }
+    return true;
+}
+function hashMemoryContent(content) {
+    return createHash("sha256").update(content, "utf8").digest("hex");
+}
+function chunkMarkdownContent(content) {
+    return content
+        .split(/\r?\n\s*\r?\n/g)
+        .map((chunk) => chunk.trim())
+        .filter((chunk) => chunk.length > 0);
+}
+function runEmbeddingWorker(input) {
+    const workerPath = fileURLToPath(new URL("./embedding-worker.js", import.meta.url));
+    const outputPath = path.join(os.tmpdir(), `claw-embedding-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}.json`);
+    const result = spawnSync(process.execPath, [workerPath], {
+        input: JSON.stringify({
+            ...input,
+            outputPath,
+        }),
+        encoding: "utf-8",
+        env: process.env,
+        maxBuffer: 10 * 1024 * 1024,
+    });
+    if (result.status !== 0) {
+        cleanupTemporaryEmbeddingOutput(outputPath);
+        throw new ClawError("PROJECT_CONFIG_INVALID", "Memory embedding generation failed.", {
+            stdout: result.stdout,
+            stderr: result.stderr,
+        });
+    }
+    try {
+        const payload = JSON.parse(stripBom(fs.readFileSync(outputPath, "utf-8")));
+        return payload;
+    }
+    finally {
+        cleanupTemporaryEmbeddingOutput(outputPath);
+    }
+}
+function stripBom(content) {
+    return content.charCodeAt(0) === 0xfeff ? content.slice(1) : content;
+}
+function cleanupTemporaryEmbeddingOutput(outputPath) {
+    if (fs.existsSync(outputPath)) {
+        fs.unlinkSync(outputPath);
+    }
+}
+function isEmbeddingGenerationFailure(error) {
+    return error instanceof ClawError &&
+        error.code === "PROJECT_CONFIG_INVALID" &&
+        error.message === "Memory embedding generation failed.";
+}
+function resolveEmbeddingApiKey(embedding) {
+    const envVar = embedding.remote?.apiKeyEnvVar?.trim();
+    if (!envVar) {
+        return null;
+    }
+    const value = process.env[envVar];
+    return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+function resolveEmbeddingDimensions(embedding, fallback) {
+    if (typeof embedding.outputDimensionality === "number" && embedding.outputDimensionality > 0) {
+        return embedding.outputDimensionality;
+    }
+    if (embedding.provider === "local") {
+        return 384;
+    }
+    return fallback > 0 ? fallback : 1536;
+}
+function searchTaskMemoryFts(db, query, limit) {
+    const rows = db
+        .prepare([
+        "SELECT source_path, kind, snippet(docs_fts, 2, '[', ']', ' ... ', 18) AS snippet, bm25(docs_fts) AS score",
+        "FROM docs_fts",
+        "WHERE docs_fts MATCH ?",
+        "ORDER BY score ASC",
+        "LIMIT ?",
+    ].join(" "))
+        .all(query, limit);
+    return rows.map((row) => ({
+        sourcePath: row.source_path,
+        kind: row.kind,
+        snippet: row.snippet,
+        score: row.score,
+    }));
+}
+function searchProjectMemoryHybrid(db, query, limit, project) {
+    const embedding = resolveProjectMemoryEmbeddingConfig(project);
+    if (!embedding || embedding.store?.vector?.enabled === false) {
+        throw new ClawError("MEMORY_VECTOR_INDEX_REQUIRED", "Project search requires memory.embedding with vector indexing enabled. Configure .claw/project.json and run `claw search index --refresh` first.");
+    }
+    const vectorIndexMetadata = db
+        .prepare("SELECT value FROM index_metadata WHERE key = ?")
+        .get("vector_index");
+    if (!vectorIndexMetadata) {
+        throw new ClawError("MEMORY_VECTOR_INDEX_REQUIRED", "Project search requires a refreshed vector index. Run `claw search index --refresh` first.");
+    }
+    const queryIntent = buildProjectQueryIntent(query);
+    const queryEmbedding = runEmbeddingWorker({
+        embedding,
+        texts: [queryIntent.embeddingText || query],
+    }).vectors[0];
+    if (!queryEmbedding?.length) {
+        throw new ClawError("MEMORY_VECTOR_INDEX_REQUIRED", "Unable to generate a query embedding for project search.");
+    }
+    const candidateLimit = Math.max(limit * PROJECT_SEARCH_CANDIDATE_MULTIPLIER, 40);
+    const projectDocs = db
+        .prepare("SELECT source_path, kind, content FROM docs")
+        .all();
+    const docSignals = new Map(projectDocs.map((row) => [
+        row.source_path,
+        buildProjectSearchSignals({
+            sourcePath: row.source_path,
+            content: row.content,
+            query,
+            queryIntent,
+        }),
+    ]));
+    const ftsRows = searchProjectMemoryKeywords(db, query, candidateLimit);
+    const signalRows = searchProjectMemorySignals(projectDocs, docSignals, candidateLimit);
+    const vectorRows = db
+        .prepare([
+        "SELECT source_path, kind, chunk_text, embedding_json",
+        "FROM doc_embeddings",
+    ].join(" "))
+        .all();
+    if (vectorRows.length === 0) {
+        throw new ClawError("MEMORY_VECTOR_INDEX_REQUIRED", "Project search requires stored vectors. Run `claw search index --refresh` first.");
+    }
+    const rankedVectors = vectorRows
+        .map((row) => {
+        const signals = docSignals.get(row.source_path);
+        return {
+            sourcePath: row.source_path,
+            kind: row.kind,
+            snippet: buildSnippet(row.chunk_text),
+            similarity: cosineSimilarity(queryEmbedding, parseEmbeddingJson(row.embedding_json)),
+            exactBoost: signals?.exactBoost ?? 0,
+        };
+    })
+        .filter((row) => Number.isFinite(row.similarity))
+        .sort((left, right) => {
+        const leftScore = left.similarity + left.exactBoost;
+        const rightScore = right.similarity + right.exactBoost;
+        return rightScore - leftScore;
+    });
+    const bestVectorBySource = new Map();
+    for (const row of rankedVectors) {
+        const existing = bestVectorBySource.get(row.sourcePath);
+        if (!existing || row.similarity > existing.similarity) {
+            bestVectorBySource.set(row.sourcePath, {
+                kind: row.kind,
+                snippet: row.snippet,
+                similarity: row.similarity,
+            });
+        }
+    }
+    const fused = new Map();
+    Array.from(bestVectorBySource.entries())
+        .sort((left, right) => {
+        const leftScore = left[1].similarity + (docSignals.get(left[0])?.exactBoost ?? 0);
+        const rightScore = right[1].similarity + (docSignals.get(right[0])?.exactBoost ?? 0);
+        return rightScore - leftScore;
+    })
+        .slice(0, candidateLimit)
+        .forEach(([sourcePath, row], index) => {
+        const signals = docSignals.get(sourcePath);
+        fused.set(sourcePath, {
+            sourcePath,
+            kind: row.kind,
+            snippet: row.snippet,
+            score: reciprocalRankScore(index + 1, 0.6) + (signals?.exactBoost ?? 0),
+            vectorRank: index + 1,
+        });
+    });
+    ftsRows.forEach((row, index) => {
+        const existing = fused.get(row.source_path);
+        const signals = docSignals.get(row.source_path);
+        const nextScore = (existing?.score ?? 0) + reciprocalRankScore(index + 1, 0.25);
+        fused.set(row.source_path, {
+            sourcePath: row.source_path,
+            kind: existing?.kind ?? row.kind,
+            snippet: row.snippet || existing?.snippet || "",
+            score: nextScore + (existing ? 0 : (signals?.exactBoost ?? 0)),
+            vectorRank: existing?.vectorRank,
+            textRank: index + 1,
+        });
+    });
+    signalRows.forEach((row, index) => {
+        const existing = fused.get(row.source_path);
+        const signals = docSignals.get(row.source_path);
+        const nextScore = (existing?.score ?? 0) + reciprocalRankScore(index + 1, 0.15);
+        fused.set(row.source_path, {
+            sourcePath: row.source_path,
+            kind: existing?.kind ?? row.kind,
+            snippet: existing?.snippet || row.snippet || "",
+            score: nextScore + (existing ? 0 : (signals?.exactBoost ?? 0)),
+            vectorRank: existing?.vectorRank,
+            textRank: existing?.textRank,
+            signalRank: index + 1,
+        });
+    });
+    return rerankProjectSearchCandidates(Array.from(fused.values()), docSignals, limit);
+}
+function searchProjectMemoryKeywords(db, query, limit) {
+    const plan = buildProjectKeywordSearchPlan(query);
+    if (plan.length === 0) {
+        return [];
+    }
+    const searchFts = db.prepare([
+        "SELECT docs.source_path, docs.kind, snippet(docs_fts, 2, '[', ']', ' ... ', 18) AS snippet, bm25(docs_fts) AS score",
+        "FROM docs_fts",
+        "JOIN docs ON docs.id = docs_fts.rowid",
+        "WHERE docs_fts MATCH ?",
+        "ORDER BY score ASC",
+        "LIMIT ?",
+    ].join(" "));
+    const bySource = new Map();
+    for (const step of plan) {
+        const rows = collectProjectKeywordRows(db, searchFts, step, limit);
+        for (const row of rows) {
+            const existing = bySource.get(row.source_path);
+            if (!existing) {
+                bySource.set(row.source_path, {
+                    ...row,
+                    matchedTerms: new Set(step.matchedTerms),
+                    exactMatches: step.matchedTerms.length > 1 ? 1 : 0,
+                });
+                continue;
+            }
+            if (row.score < existing.score) {
+                existing.score = row.score;
+            }
+            if (row.snippet && row.snippet.length > existing.snippet.length) {
+                existing.snippet = row.snippet;
+            }
+            if (step.matchedTerms.length > 1) {
+                existing.exactMatches += 1;
+            }
+            for (const term of step.matchedTerms) {
+                existing.matchedTerms.add(term);
+            }
+        }
+    }
+    return Array.from(bySource.values())
+        .sort((left, right) => {
+        if (right.matchedTerms.size !== left.matchedTerms.size) {
+            return right.matchedTerms.size - left.matchedTerms.size;
+        }
+        if (right.exactMatches !== left.exactMatches) {
+            return right.exactMatches - left.exactMatches;
+        }
+        return left.score - right.score;
+    })
+        .slice(0, limit)
+        .map(({ source_path, kind, snippet, score }) => ({
+        source_path,
+        kind,
+        snippet,
+        score,
+    }));
+}
+function collectProjectKeywordRows(db, searchFts, step, limit) {
+    const rows = [];
+    const seen = new Set();
+    if (step.query) {
+        const ftsRows = searchFts.all(step.query, limit);
+        for (const row of ftsRows) {
+            if (matchesAllSubstrings(db, row.source_path, step.substringTerms)) {
+                rows.push(row);
+                seen.add(row.source_path);
+            }
+        }
+    }
+    if (step.substringTerms.length > 0) {
+        const substringRows = searchDocsBySubstring(db, step.substringTerms, limit);
+        for (const row of substringRows) {
+            if (!seen.has(row.source_path)) {
+                rows.push(row);
+                seen.add(row.source_path);
+            }
+        }
+    }
+    return rows;
+}
+function searchDocsBySubstring(db, substringTerms, limit) {
+    const clauses = substringTerms.map(() => "content LIKE ? ESCAPE '\\'").join(" AND ");
+    const query = [
+        "SELECT source_path, kind, content",
+        "FROM docs",
+        `WHERE ${clauses}`,
+        "LIMIT ?",
+    ].join(" ");
+    const dynamicRows = db
+        .prepare(query)
+        .all(...substringTerms.map((term) => `%${escapeLikePattern(term)}%`), limit);
+    return dynamicRows.map((row) => ({
+        source_path: row.source_path,
+        kind: row.kind,
+        snippet: buildSnippet(row.content),
+        score: 0,
+    }));
+}
+function searchProjectMemorySignals(projectDocs, docSignals, limit) {
+    return projectDocs
+        .filter((row) => (docSignals.get(row.source_path)?.matchedTermCount ?? 0) > 0)
+        .sort((left, right) => {
+        const leftSignals = docSignals.get(left.source_path);
+        const rightSignals = docSignals.get(right.source_path);
+        if ((rightSignals?.strongMatchedTermCount ?? 0) !== (leftSignals?.strongMatchedTermCount ?? 0)) {
+            return (rightSignals?.strongMatchedTermCount ?? 0) - (leftSignals?.strongMatchedTermCount ?? 0);
+        }
+        if ((rightSignals?.matchedTermCount ?? 0) !== (leftSignals?.matchedTermCount ?? 0)) {
+            return (rightSignals?.matchedTermCount ?? 0) - (leftSignals?.matchedTermCount ?? 0);
+        }
+        if ((rightSignals?.exactBoost ?? 0) !== (leftSignals?.exactBoost ?? 0)) {
+            return (rightSignals?.exactBoost ?? 0) - (leftSignals?.exactBoost ?? 0);
+        }
+        if ((rightSignals?.fileNameHits ?? 0) !== (leftSignals?.fileNameHits ?? 0)) {
+            return (rightSignals?.fileNameHits ?? 0) - (leftSignals?.fileNameHits ?? 0);
+        }
+        return (rightSignals?.pathHits ?? 0) - (leftSignals?.pathHits ?? 0);
+    })
+        .slice(0, limit)
+        .map((row) => ({
+        source_path: row.source_path,
+        kind: row.kind,
+        snippet: buildSnippet(row.content),
+        score: 0,
+    }));
+}
+function matchesAllSubstrings(db, sourcePath, substringTerms) {
+    if (substringTerms.length === 0) {
+        return true;
+    }
+    const row = db
+        .prepare("SELECT content FROM docs WHERE source_path = ?")
+        .get(sourcePath);
+    if (!row) {
+        return false;
+    }
+    return substringTerms.every((term) => row.content.includes(term));
+}
+function escapeLikePattern(term) {
+    return term.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
+}
+function rerankProjectSearchCandidates(candidates, docSignals, limit) {
+    const remaining = [...candidates];
+    const selected = [];
+    const coveredStrongTerms = new Set();
+    const coveredTerms = new Set();
+    while (remaining.length > 0 && selected.length < limit) {
+        let bestIndex = 0;
+        let bestScore = Number.NEGATIVE_INFINITY;
+        for (let index = 0; index < remaining.length; index += 1) {
+            const candidate = remaining[index];
+            const signals = docSignals.get(candidate.sourcePath);
+            const routeCount = (candidate.vectorRank ? 1 : 0) + (candidate.textRank ? 1 : 0) + (candidate.signalRank ? 1 : 0);
+            const uncoveredStrongTerms = (signals?.strongMatchedTerms ?? []).filter((term) => !coveredStrongTerms.has(term));
+            const uncoveredTerms = (signals?.matchedTerms ?? []).filter((term) => !coveredTerms.has(term));
+            const adjustedScore = candidate.score
+                + uncoveredStrongTerms.length * 0.045
+                + uncoveredTerms.length * 0.01
+                + Math.max(routeCount - 1, 0) * 0.003;
+            if (adjustedScore > bestScore) {
+                bestScore = adjustedScore;
+                bestIndex = index;
+                continue;
+            }
+            if (adjustedScore === bestScore) {
+                const currentBest = remaining[bestIndex];
+                const currentSignals = docSignals.get(currentBest.sourcePath);
+                if ((signals?.strongMatchedTermCount ?? 0) !== (currentSignals?.strongMatchedTermCount ?? 0)) {
+                    if ((signals?.strongMatchedTermCount ?? 0) > (currentSignals?.strongMatchedTermCount ?? 0)) {
+                        bestIndex = index;
+                    }
+                    continue;
+                }
+                if ((signals?.matchedTermCount ?? 0) !== (currentSignals?.matchedTermCount ?? 0)) {
+                    if ((signals?.matchedTermCount ?? 0) > (currentSignals?.matchedTermCount ?? 0)) {
+                        bestIndex = index;
+                    }
+                    continue;
+                }
+                if ((signals?.exactBoost ?? 0) > (currentSignals?.exactBoost ?? 0)) {
+                    bestIndex = index;
+                }
+            }
+        }
+        const [next] = remaining.splice(bestIndex, 1);
+        const nextSignals = docSignals.get(next.sourcePath);
+        nextSignals?.strongMatchedTerms.forEach((term) => coveredStrongTerms.add(term));
+        nextSignals?.matchedTerms.forEach((term) => coveredTerms.add(term));
+        selected.push({
+            sourcePath: next.sourcePath,
+            kind: next.kind,
+            snippet: next.snippet,
+            score: next.score,
+        });
+    }
+    return selected;
+}
+function buildProjectSearchSignals(input) {
+    const normalizedQuery = input.query.trim();
+    const normalizedContent = input.content.toLowerCase();
+    const normalizedPath = input.sourcePath.toLowerCase();
+    const fileName = path.basename(input.sourcePath).toLowerCase();
+    const lowerTerms = input.queryIntent.terms.map((term) => term.toLowerCase());
+    const lowerStrongTerms = input.queryIntent.strongTerms.map((term) => term.toLowerCase());
+    const lowerWeakTerms = input.queryIntent.weakTerms.map((term) => term.toLowerCase());
+    const matchedContentTerms = lowerTerms.filter((term) => normalizedContent.includes(term));
+    const matchedPathTerms = lowerTerms.filter((term) => normalizedPath.includes(term));
+    const matchedTerms = new Set([...matchedContentTerms, ...matchedPathTerms]);
+    const strongMatchedTerms = new Set(lowerStrongTerms.filter((term) => normalizedContent.includes(term) || normalizedPath.includes(term)));
+    const weakMatchedTerms = new Set(lowerWeakTerms.filter((term) => normalizedContent.includes(term) || normalizedPath.includes(term)));
+    const matchedCharacters = Array.from(matchedTerms).reduce((sum, term) => sum + term.length, 0);
+    const normalizedLength = Math.max(Array.from(input.content.trim()).length, 1);
+    const coverageRatio = lowerTerms.length > 0 ? matchedTerms.size / lowerTerms.length : 0;
+    const strongCoverageRatio = lowerStrongTerms.length > 0 ? strongMatchedTerms.size / lowerStrongTerms.length : 0;
+    const densityRatio = matchedCharacters / normalizedLength;
+    const fileNameHits = lowerTerms.filter((term) => fileName.includes(term)).length;
+    const pathHits = matchedPathTerms.length;
+    const phraseMatch = normalizedQuery.length > 0
+        && (normalizedContent.includes(normalizedQuery.toLowerCase()) || normalizedPath.includes(normalizedQuery.toLowerCase()));
+    const weakOnlyPenalty = strongMatchedTerms.size === 0 && weakMatchedTerms.size > 0 ? 0.012 : 0;
+    const missingStrongPenalty = lowerStrongTerms.length > 0 && strongMatchedTerms.size === 0
+        ? 0.035
+        : lowerStrongTerms.length > 1 && strongMatchedTerms.size === 1
+            ? 0.01
+            : 0;
+    const indexFilePenalty = isIndexLikeDocName(fileName) ? 0.06 : 0;
+    return {
+        matchedTerms: Array.from(matchedTerms),
+        strongMatchedTerms: Array.from(strongMatchedTerms),
+        weakMatchedTerms: Array.from(weakMatchedTerms),
+        matchedTermCount: matchedTerms.size,
+        strongMatchedTermCount: strongMatchedTerms.size,
+        fileNameHits,
+        pathHits,
+        exactBoost: strongMatchedTerms.size * 0.016
+            + weakMatchedTerms.size * 0.004
+            + coverageRatio * 0.008
+            + strongCoverageRatio * 0.014
+            + densityRatio * 0.02
+            + fileNameHits * 0.025
+            + pathHits * 0.01
+            + (phraseMatch ? 0.018 : 0)
+            - weakOnlyPenalty
+            - missingStrongPenalty
+            - indexFilePenalty,
+    };
+}
+function isIndexLikeDocName(fileName) {
+    return fileName === "contents.md" || fileName === "summary.md" || fileName === "index.md" || fileName === "readme.md";
+}
+function reciprocalRankScore(rank, weight) {
+    return weight * (1 / (40 + rank));
+}
+function parseEmbeddingJson(value) {
+    const parsed = JSON.parse(value);
+    return Array.isArray(parsed) ? parsed.map((entry) => Number(entry)) : [];
+}
+function cosineSimilarity(left, right) {
+    const dimensions = Math.min(left.length, right.length);
+    if (dimensions === 0) {
+        return Number.NEGATIVE_INFINITY;
+    }
+    let dot = 0;
+    let leftNorm = 0;
+    let rightNorm = 0;
+    for (let index = 0; index < dimensions; index += 1) {
+        const leftValue = left[index] ?? 0;
+        const rightValue = right[index] ?? 0;
+        dot += leftValue * rightValue;
+        leftNorm += leftValue * leftValue;
+        rightNorm += rightValue * rightValue;
+    }
+    if (leftNorm === 0 || rightNorm === 0) {
+        return Number.NEGATIVE_INFINITY;
+    }
+    return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm));
+}
+function buildSnippet(text) {
+    const normalized = text.replace(/\s+/g, " ").trim();
+    if (normalized.length <= 180) {
+        return normalized;
+    }
+    return `${normalized.slice(0, 177)}...`;
 }
 //# sourceMappingURL=memory.js.map