lmgrep 0.1.16 → 0.1.18

package/dist/lib/store.js

@@ -6,6 +6,7 @@ import { homedir } from "node:os";
 import { join, resolve } from "node:path";
 const CHUNKS_TABLE = "chunks";
 const FILES_TABLE = "files";
+const VOCAB_TABLE = "vocab";
 const DELETE_BATCH_SIZE = 50;
 function buildInFilter(column, values) {
     const escaped = values.map((v) => `'${v.replace(/'/g, "''")}'`);
@@ -17,6 +18,40 @@ async function batchDelete(table, column, values) {
         await table.delete(buildInFilter(column, batch));
     }
 }
+/**
+ * Remove redundant search rows. Two passes, assuming `rows` is already in
+ * best-first order (ANN returns ascending distance = descending score):
+ *
+ *  1. Exact duplicates by chunk id — collapses identical rows produced by
+ *     concurrent unlocked indexing (same filePath:row:contentHash).
+ *  2. Overlapping line ranges within the same file — collapses the
+ *     fallback chunker's sliding-window overlap (and any parent/child or
+ *     near-duplicate spans), keeping the highest-scoring chunk of each
+ *     overlapping cluster. Tree-sitter chunks are node-bounded and don't
+ *     overlap, so this only ever drops genuine near-duplicates.
+ */
+function dedupeRows(rows) {
+    const seenIds = new Set();
+    const keptRanges = new Map();
+    const out = [];
+    for (const r of rows) {
+        if (seenIds.has(r.id))
+            continue;
+        seenIds.add(r.id);
+        const ranges = keptRanges.get(r.filePath);
+        if (ranges) {
+            const overlaps = ranges.some(([s, e]) => r.startLine <= e && s <= r.endLine);
+            if (overlaps)
+                continue;
+            ranges.push([r.startLine, r.endLine]);
+        }
+        else {
+            keptRanges.set(r.filePath, [[r.startLine, r.endLine]]);
+        }
+        out.push(r);
+    }
+    return out;
+}
 /**
  * Legacy indexes (pre branch-scoping) have a `files` table without a `branch`
  * column. Detect and backfill it with the current branch so queries that filter
@@ -31,6 +66,20 @@ async function migrateBranchColumn(table, currentBranch) {
         { name: "branch", valueSql: `CAST('${escaped}' AS STRING)` },
     ]);
 }
+/**
+ * Legacy chunk tables (pre version-scoping) have no `fileHash` column. Backfill
+ * it with "" — the empty string is treated as a wildcard at search time so
+ * legacy chunks keep appearing until the file is re-indexed and gets a real
+ * file-version hash.
+ */
+async function migrateFileHashColumn(table) {
+    const schema = await table.schema();
+    if (schema.fields.some((f) => f.name === "fileHash"))
+        return;
+    await table.addColumns([
+        { name: "fileHash", valueSql: `CAST('' AS STRING)` },
+    ]);
+}
 function git(cwd, ...args) {
     try {
         return execSync(`git ${args.join(" ")}`, {
@@ -267,6 +316,71 @@ export function releaseDbLock(cwd) {
     }
     catch { }
 }
+// --- Per-build write mutex ---
+//
+// The `.lock` maintainer lock (above) is held for a watcher/serve process's
+// whole lifetime and doubles as a liveness registry for `lmgrep status`. It
+// cannot also serve as a write mutex, because then a one-shot `lmgrep index`
+// could never run while a watcher is up. This separate `.writelock` is a
+// short-lived mutex acquired around each build() so that the watcher and an
+// ad-hoc `lmgrep index` serialize their writes instead of racing into
+// duplicate rows. Named `.writelock` (not `.write.lock`) so it does not match
+// the `.lock` suffix scan in discoverRunningProcesses.
+function writeLockPath(cwd) {
+    return `${getDbPath(cwd)}.writelock`;
+}
+function tryAcquireWriteLock(cwd) {
+    const lockPath = writeLockPath(cwd);
+    if (existsSync(lockPath)) {
+        try {
+            const pid = Number.parseInt(readFileSync(lockPath, "utf-8").trim(), 10);
+            if (isProcessAlive(pid))
+                return false;
+        }
+        catch {
+            // stale/corrupt lock, take over
+        }
+    }
+    mkdirSync(getDbPath(cwd), { recursive: true });
+    writeFileSync(lockPath, `${process.pid}\n`);
+    return true;
+}
+function releaseWriteLock(cwd) {
+    const lockPath = writeLockPath(cwd);
+    try {
+        // Only remove the lock if we still own it.
+        const pid = Number.parseInt(readFileSync(lockPath, "utf-8").trim(), 10);
+        if (pid === process.pid)
+            unlinkSync(lockPath);
+    }
+    catch { }
+}
+/**
+ * Run `fn` while holding the project's write mutex, so concurrent indexers
+ * (a watcher plus an ad-hoc `lmgrep index`) can't write at the same time and
+ * produce duplicate chunk rows. Waits up to `waitMs` for a busy lock, taking
+ * over a lock held by a dead process. Throws if the lock can't be acquired in
+ * time.
+ */
+export async function withWriteLock(cwd, fn, opts = {}) {
+    const waitMs = opts.waitMs ?? 120_000;
+    const pollMs = opts.pollMs ?? 200;
+    let waited = 0;
+    while (!tryAcquireWriteLock(cwd)) {
+        if (waited >= waitMs) {
+            throw new Error("Could not acquire the index write lock — another indexer is busy. " +
+                "Try again once it finishes.");
+        }
+        await new Promise((r) => setTimeout(r, pollMs));
+        waited += pollMs;
+    }
+    try {
+        return await fn();
+    }
+    finally {
+        releaseWriteLock(cwd);
+    }
+}
 /**
  * Check if a write lock is held by a live process.
  */
@@ -351,6 +465,7 @@ export class Store {
     db;
     chunksTable;
     filesTable;
+    vocabTable;
     constructor(dbPath, branch = "_default") {
         this.dbPath = dbPath;
         this.branch = branch;
@@ -373,7 +488,9 @@ export class Store {
         const conn = await this.connection();
         const tables = await conn.tableNames();
         if (tables.includes(CHUNKS_TABLE)) {
-            this.chunksTable = await conn.openTable(CHUNKS_TABLE);
+            const t = await conn.openTable(CHUNKS_TABLE);
+            await migrateFileHashColumn(t);
+            this.chunksTable = t;
             return this.chunksTable;
         }
         return undefined;
@@ -391,6 +508,100 @@ export class Store {
         }
         return undefined;
     }
+    // --- Vocab ---
+    async openVocab() {
+        if (this.vocabTable)
+            return this.vocabTable;
+        const conn = await this.connection();
+        const tables = await conn.tableNames();
+        if (tables.includes(VOCAB_TABLE)) {
+            this.vocabTable = await conn.openTable(VOCAB_TABLE);
+            return this.vocabTable;
+        }
+        return undefined;
+    }
+    async hasVocab() {
+        return (await this.openVocab()) !== undefined;
+    }
+    /**
+     * Return the set of vocab terms already embedded in the vocab table.
+     * Used at index time to skip re-embedding.
+     */
+    async getVocabTerms() {
+        const t = await this.openVocab();
+        if (!t)
+            return new Set();
+        const rows = await t.query().select(["term"]).toArray();
+        return new Set(rows.map((r) => r.term));
+    }
+    async addVocab(entries) {
+        if (entries.length === 0)
+            return;
+        // Dedup within the batch
+        const seen = new Set();
+        const batchUnique = [];
+        for (const e of entries) {
+            if (seen.has(e.term))
+                continue;
+            seen.add(e.term);
+            batchUnique.push(e);
+        }
+        // Skip terms already in the table
+        const known = await this.getVocabTerms();
+        const records = batchUnique
+            .filter((e) => !known.has(e.term))
+            .map((e) => ({ term: e.term, vector: e.vector }));
+        if (records.length === 0)
+            return;
+        const conn = await this.connection();
+        const tables = await conn.tableNames();
+        if (tables.includes(VOCAB_TABLE)) {
+            const t = await conn.openTable(VOCAB_TABLE);
+            this.vocabTable = t;
+            await t.add(records);
+        }
+        else {
+            this.vocabTable = await conn.createTable(VOCAB_TABLE, records);
+        }
+    }
+    /**
+     * ANN search against the vocab table. Returns top-N terms closest to the
+     * given vector by cosine distance.
+     */
+    async searchVocab(vector, limit, excludeTerms) {
+        const t = await this.openVocab();
+        if (!t)
+            return [];
+        const fetch = excludeTerms ? limit + excludeTerms.size : limit;
+        const rows = await t.search(vector).limit(fetch).toArray();
+        const out = [];
+        for (const r of rows) {
+            const term = r.term;
+            if (excludeTerms?.has(term))
+                continue;
+            out.push({
+                term,
+                score: r._distance != null ? 1 - r._distance : 0,
+            });
+            if (out.length >= limit)
+                break;
+        }
+        return out;
+    }
+    async vocabCount() {
+        const t = await this.openVocab();
+        if (!t)
+            return 0;
+        return t.countRows();
+    }
+    async dropVocab() {
+        const conn = await this.connection();
+        const tables = await conn.tableNames();
+        if (tables.includes(VOCAB_TABLE)) {
+            await conn.dropTable(VOCAB_TABLE);
+        }
+        this.vocabTable = undefined;
+    }
     // --- Chunks ---
     async addChunks(chunks) {
         if (chunks.length === 0)
@@ -406,6 +617,7 @@ export class Store {
             content: c.content,
             context: c.context,
             hash: c.hash,
+            fileHash: c.fileHash ?? "",
             vector: c.vector,
         }));
         const tables = await conn.tableNames();
@@ -454,13 +666,15 @@ export class Store {
             await batchDelete(t, "filePath", toDelete);
         }
     }
-    branchFilesCache;
+    branchVersionsCache;
     /**
-     * Get the set of file paths indexed on the current branch (cached).
+     * Map of filePath -> file-version hash for the current branch (cached).
+     * Used to scope search to the exact file versions this branch references,
+     * so stale chunks from another version of the same path are excluded.
      */
-    async getBranchFiles() {
-        if (this.branchFilesCache)
-            return this.branchFilesCache;
+    async getBranchFileVersions() {
+        if (this.branchVersionsCache)
+            return this.branchVersionsCache;
         const t = await this.openFiles();
         if (!t)
             return undefined;
@@ -468,14 +682,18 @@ export class Store {
         const rows = await t
             .query()
             .where(`branch = '${escaped}'`)
-            .select(["filePath"])
+            .select(["filePath", "fileHash"])
             .toArray();
-        this.branchFilesCache = new Set(rows.map((r) => r.filePath));
-        return this.branchFilesCache;
+        const map = new Map();
+        for (const r of rows) {
+            map.set(r.filePath, r.fileHash);
+        }
+        this.branchVersionsCache = map;
+        return this.branchVersionsCache;
     }
     /** Invalidate the branch files cache (call after index/import). */
     invalidateBranchFilesCache() {
-        this.branchFilesCache = undefined;
+        this.branchVersionsCache = undefined;
     }
     async search(queryVector, limit = 25, filePrefix, typeFilter, 
     /** Pass false to skip branch scoping (e.g. for cross-project search). */
@@ -484,9 +702,12 @@ export class Store {
         if (!t) {
             throw new Error("No index found. Run `lmgrep index` first.");
         }
-        const branchFiles = scopeToBranch ? await this.getBranchFiles() : undefined;
-        // Over-fetch when branch-filtering since some results will be discarded
-        const fetchLimit = branchFiles ? limit * 3 : limit;
+        const branchVersions = scopeToBranch
+            ? await this.getBranchFileVersions()
+            : undefined;
+        // Over-fetch: branch/version filtering and dedup both discard rows, so
+        // pull extra to still return `limit` distinct results.
+        const fetchLimit = branchVersions ? limit * 3 : limit * 2;
         let query = t.search(queryVector).limit(fetchLimit);
         const conditions = [];
         if (filePrefix) {
@@ -501,6 +722,48 @@ export class Store {
         }
         const results = await query.toArray();
         let mapped = results.map((r) => ({
+            id: r.id,
+            filePath: r.filePath,
+            startLine: r.startLine,
+            endLine: r.endLine,
+            type: r.type,
+            name: r.name,
+            content: r.content,
+            context: r.context,
+            score: r._distance != null ? 1 - r._distance : 0,
+            fileHash: r.fileHash ?? "",
+        }));
+        if (branchVersions) {
+            mapped = mapped.filter((r) => {
+                const want = branchVersions.get(r.filePath);
+                // "" fileHash = legacy chunk, treated as wildcard.
+                return want !== undefined && (r.fileHash === "" || r.fileHash === want);
+            });
+        }
+        return dedupeRows(mapped)
+            .slice(0, limit)
+            .map(({ id, fileHash, ...rest }) => rest);
+    }
+    /**
+     * Same as search() but also returns each chunk's vector — needed for
+     * client-side clustering (facet command).
+     */
+    async searchWithVectors(queryVector, limit = 25, filePrefix, scopeToBranch = true) {
+        const t = await this.openChunks();
+        if (!t) {
+            throw new Error("No index found. Run `lmgrep index` first.");
+        }
+        const branchVersions = scopeToBranch
+            ? await this.getBranchFileVersions()
+            : undefined;
+        const fetchLimit = branchVersions ? limit * 3 : limit * 2;
+        let query = t.search(queryVector).limit(fetchLimit);
+        if (filePrefix) {
+            query = query.where(`filePath LIKE '${filePrefix.replace(/'/g, "''")}%'`);
+        }
+        const results = await query.toArray();
+        let mapped = results.map((r) => ({
+            id: r.id,
             filePath: r.filePath,
             startLine: r.startLine,
             endLine: r.endLine,
@@ -509,11 +772,43 @@ export class Store {
             content: r.content,
             context: r.context,
             score: r._distance != null ? 1 - r._distance : 0,
+            fileHash: r.fileHash ?? "",
+            vector: Array.from(r.vector),
         }));
-        if (branchFiles) {
-            mapped = mapped.filter((r) => branchFiles.has(r.filePath));
+        if (branchVersions) {
+            mapped = mapped.filter((r) => {
+                const want = branchVersions.get(r.filePath);
+                return want !== undefined && (r.fileHash === "" || r.fileHash === want);
+            });
         }
-        return mapped.slice(0, limit);
+        return dedupeRows(mapped)
+            .slice(0, limit)
+            .map(({ fileHash, ...rest }) => rest);
+    }
+    /**
+     * Fetch chunks (with vectors) by id. Used to rehydrate faceting sessions.
+     * Missing ids are silently dropped.
+     */
+    async getChunksByIds(ids) {
+        if (ids.length === 0)
+            return [];
+        const t = await this.openChunks();
+        if (!t)
+            return [];
+        const escaped = ids.map((i) => `'${i.replace(/'/g, "''")}'`).join(",");
+        const rows = await t.query().where(`id IN (${escaped})`).toArray();
+        return rows.map((r) => ({
+            id: r.id,
+            filePath: r.filePath,
+            startLine: r.startLine,
+            endLine: r.endLine,
+            type: r.type,
+            name: r.name,
+            content: r.content,
+            context: r.context,
+            score: 0,
+            vector: Array.from(r.vector),
+        }));
     }
     async getIndexedFiles() {
         const t = await this.openChunks();
@@ -567,6 +862,25 @@ export class Store {
             return 0;
         return await t.countRows();
     }
+    /**
+     * Stream chunk texts (name + content only) in batches. Used by vocab
+     * backfill to avoid loading all chunks into memory at once.
+     */
+    async *streamChunkTexts(batchSize = 1000) {
+        const t = await this.openChunks();
+        if (!t)
+            return;
+        const stream = await t
+            .query()
+            .select(["name", "content"])
+            .toArray();
+        for (let i = 0; i < stream.length; i += batchSize) {
+            yield stream.slice(i, i + batchSize).map((r) => ({
+                name: r.name ?? "",
+                content: r.content ?? "",
+            }));
+        }
+    }
     // --- File hashes (change detection) ---
     async getFileHashes() {
         const t = await this.openFiles();
@@ -674,10 +988,79 @@ export class Store {
                 content: r.content,
                 context: r.context,
                 hash: r.hash,
+                fileHash: r.fileHash ?? "",
                 vector: Array.from(r.vector),
             }));
         }
     }
+    /**
+     * Remove redundant rows from the chunks table:
+     *  - exact duplicate ids (identical rows from concurrent unlocked indexing)
+     *  - orphaned versions: chunks whose fileHash is no longer referenced by any
+     *    branch's manifest (left behind when a file was edited on one branch
+     *    while another branch still pointed at the old path). Legacy chunks with
+     *    an empty fileHash are kept.
+     * Rewrites the table from the surviving rows. Loads chunks into memory in
+     * batches, so this is a maintenance operation, not a hot path.
+     */
+    async dedupeChunks() {
+        const t = await this.openChunks();
+        if (!t)
+            return { before: 0, after: 0, duplicateIds: 0, staleVersions: 0 };
+        const before = await t.countRows();
+        // Which (filePath, fileHash) pairs any branch still references.
+        const refs = new Map();
+        for (const e of await this.getAllFileEntries()) {
+            const set = refs.get(e.filePath) ?? new Set();
+            set.add(e.fileHash);
+            refs.set(e.filePath, set);
+        }
+        const kept = [];
+        const seenIds = new Set();
+        let duplicateIds = 0;
+        let staleVersions = 0;
+        for await (const batch of this.streamAllChunks(2000)) {
+            for (const r of batch) {
+                const id = r.id;
+                if (seenIds.has(id)) {
+                    duplicateIds++;
+                    continue;
+                }
+                seenIds.add(id);
+                const filePath = r.filePath;
+                const fileHash = r.fileHash ?? "";
+                if (fileHash !== "" && !refs.get(filePath)?.has(fileHash)) {
+                    staleVersions++;
+                    continue;
+                }
+                kept.push({
+                    id,
+                    filePath,
+                    startLine: r.startLine,
+                    endLine: r.endLine,
+                    type: r.type,
+                    name: r.name,
+                    content: r.content,
+                    context: r.context,
+                    hash: r.hash,
+                    fileHash,
+                    vector: r.vector,
+                });
+            }
+        }
+        if (duplicateIds + staleVersions === 0) {
+            return { before, after: before, duplicateIds, staleVersions };
+        }
+        // Rewrite the table from the survivors.
+        const conn = await this.connection();
+        if ((await conn.tableNames()).includes(CHUNKS_TABLE)) {
+            await conn.dropTable(CHUNKS_TABLE);
+        }
+        this.chunksTable = undefined;
+        if (kept.length > 0)
+            await this.addChunks(kept);
+        return { before, after: kept.length, duplicateIds, staleVersions };
+    }
     async getAllFileEntries() {
         const t = await this.openFiles();
         if (!t)
@@ -696,8 +1079,11 @@ export class Store {
             await conn.dropTable(CHUNKS_TABLE);
         if (tables.includes(FILES_TABLE))
             await conn.dropTable(FILES_TABLE);
+        if (tables.includes(VOCAB_TABLE))
+            await conn.dropTable(VOCAB_TABLE);
         this.chunksTable = undefined;
         this.filesTable = undefined;
+        this.vocabTable = undefined;
     }
     async compact() {
         const t = await this.openChunks();
@@ -706,6 +1092,9 @@ export class Store {
         const f = await this.openFiles();
         if (f)
             await f.optimize();
+        const v = await this.openVocab();
+        if (v)
+            await v.optimize();
     }
     /**
      * Import all chunks and file hashes from another Store's database.
@@ -732,6 +1121,7 @@ export class Store {
                     content: r.content,
                     context: r.context,
                     hash: r.hash,
+                    fileHash: r.fileHash ?? "",
                     vector: Array.from(r.vector),
                 }));
                 const conn = await this.connection();
@@ -819,6 +1209,7 @@ export class Store {
     async close() {
         this.chunksTable = undefined;
         this.filesTable = undefined;
+        this.vocabTable = undefined;
         this.db = undefined;
     }
 }