lmgrep 0.1.18 → 0.2.0

package/dist/infrastructure/lancedb/IndexMaintenance.js

@@ -0,0 +1,227 @@
+import { Index } from "@lancedb/lancedb";
+import { ChunkRepository } from "./ChunkRepository.js";
+import { TableName } from "./LanceTables.js";
+import { VectorIndexPolicy } from "./VectorIndexPolicy.js";
+/**
+ * Keeps the LanceDB tables compact and their ANN indexes current.
+ *
+ * Without a vector index LanceDB answers a search by brute force: it decodes
+ * every stored embedding to score it. At high dimensions that walks the whole
+ * table through memory per query, so peak RSS tracks index size roughly 1:1.
+ * IVF-PQ turns that into a handful of probes.
+ *
+ * Compaction matters for the same reason — small appends leave hundreds of
+ * fragments, each carrying its own decode buffers.
+ */
+export class IndexMaintenance {
+    tables;
+    manifest;
+    /** Tables holding embeddings, and therefore wanting a vector index. */
+    static VECTOR_TABLES = [TableName.Chunks];
+    constructor(tables, manifest) {
+        this.tables = tables;
+        this.manifest = manifest;
+    }
+    /**
+     * Cheap and idempotent when there is nothing to do, so it is safe to call
+     * after every build.
+     */
+    async optimize(options = {}) {
+        const report = { tables: [] };
+        for (const name of IndexMaintenance.VECTOR_TABLES) {
+            const table = await this.tables.table(name);
+            if (!table)
+                continue;
+            report.tables.push(await this.optimizeTable(name, table, options.force ?? false, options.create ?? false));
+        }
+        return report;
+    }
+    /**
+     * What `lmgrep compact` runs: the same work as {@link optimize} but
+     * unconditional, plus compaction of the files table (which holds no
+     * embeddings, so it never needs a vector index).
+     */
+    async compact() {
+        const report = await this.optimize({ force: true, create: true });
+        await this.dropLegacyTables(report);
+        const files = await this.tables.table(TableName.Files);
+        if (files) {
+            await files.optimize();
+            report.tables.push({
+                table: TableName.Files,
+                rows: await files.countRows(),
+                action: "optimized",
+            });
+        }
+        return report;
+    }
+    /**
+     * Remove tables no longer part of the schema.
+     *
+     * Databases built before faceting was removed still carry a `vocab` table
+     * holding one embedding per corpus term — nothing reads it now, and at
+     * full embedding width that is real disk. Dropping it here means users
+     * reclaim the space by running the command they already run.
+     */
+    async dropLegacyTables(report) {
+        const vocab = await this.tables.table(TableName.LegacyVocab);
+        if (!vocab)
+            return;
+        const rows = await vocab.countRows();
+        await this.tables.dropTable(TableName.LegacyVocab);
+        report.tables.push({
+            table: TableName.LegacyVocab,
+            rows,
+            action: "dropped",
+        });
+    }
+    async optimizeTable(name, table, force, create) {
+        const rows = await table.countRows();
+        // listIndices reports the columns each index covers, so this stays
+        // correct if a scalar index is ever added alongside the vector one.
+        const indices = await table.listIndices();
+        const vectorIndex = indices.find((i) => i.columns.includes("vector"));
+        if (!vectorIndex) {
+            // The size guard is not overridable. `force` means "don't wait for
+            // the tail to grow", not "cluster a table with too few vectors to
+            // cluster" — IVF-PQ on a small table trains empty partitions and
+            // degrades recall for no memory saving.
+            if (rows < VectorIndexPolicy.MIN_ROWS_FOR_ANN) {
+                return { table: name, rows, action: "skipped-small" };
+            }
+            if (!create) {
+                return { table: name, rows, action: "needs-index" };
+            }
+            // numPartitions and numSubVectors are left to LanceDB, which
+            // derives them from row count and dimensionality. Hard-coding them
+            // would only reproduce that rule for dimensions already seen.
+            await table.createIndex("vector", {
+                config: Index.ivfPq({
+                    distanceType: VectorIndexPolicy.DISTANCE_TYPE,
+                }),
+                replace: true,
+            });
+            await table.optimize();
+            return { table: name, rows, action: "created" };
+        }
+        const stats = await table.indexStats(vectorIndex.name);
+        const unindexed = stats?.numUnindexedRows ?? 0;
+        const indexed = stats?.numIndexedRows ?? 0;
+        const tolerated = Math.max(VectorIndexPolicy.UNINDEXED_REINDEX_FLOOR, Math.floor(indexed * VectorIndexPolicy.UNINDEXED_REINDEX_RATIO));
+        if (!force && unindexed <= tolerated) {
+            return { table: name, rows, action: "up-to-date", unindexed };
+        }
+        // optimize() both compacts fragments and folds the unindexed tail into
+        // the existing index — no retraining from scratch.
+        await table.optimize();
+        return { table: name, rows, action: "optimized", unindexed };
+    }
+    /**
+     * Remove rows that should not be there: exact id duplicates (from
+     * concurrent unlocked indexing) and chunks whose file version no branch
+     * references any more. Survivors are rewritten into a fresh table, which is
+     * also what reclaims the disk the dropped rows held.
+     */
+    async dedupe() {
+        const table = await this.tables.table(TableName.Chunks);
+        if (!table) {
+            return { before: 0, after: 0, duplicateIds: 0, staleVersions: 0 };
+        }
+        const before = await table.countRows();
+        // (filePath, fileHash) pairs any branch still references.
+        const referenced = new Map();
+        for (const entry of await this.manifest.allEntries()) {
+            const set = referenced.get(entry.filePath) ?? new Set();
+            set.add(entry.fileHash.toString());
+            referenced.set(entry.filePath, set);
+        }
+        const kept = [];
+        const seenIds = new Set();
+        let duplicateIds = 0;
+        let staleVersions = 0;
+        const total = before;
+        const BATCH = 2000;
+        for (let offset = 0; offset < total; offset += BATCH) {
+            const rows = await table.query().limit(BATCH).offset(offset).toArray();
+            if (rows.length === 0)
+                break;
+            for (const row of rows) {
+                const id = row.id;
+                if (seenIds.has(id)) {
+                    duplicateIds++;
+                    continue;
+                }
+                seenIds.add(id);
+                const filePath = row.filePath;
+                const fileHash = row.fileHash ?? "";
+                // "" is the legacy wildcard — never stale.
+                if (fileHash !== "" && !referenced.get(filePath)?.has(fileHash)) {
+                    staleVersions++;
+                    continue;
+                }
+                kept.push(this.toPlainRow(row));
+            }
+        }
+        if (duplicateIds + staleVersions === 0) {
+            return { before, after: before, duplicateIds, staleVersions };
+        }
+        await this.tables.dropTable(TableName.Chunks);
+        if (kept.length > 0) {
+            const { table: rebuilt, seeded } = await this.tables.tableOrCreate(TableName.Chunks, kept);
+            if (!seeded)
+                await rebuilt.add(kept);
+        }
+        return { before, after: kept.length, duplicateIds, staleVersions };
+    }
+    async vectorIndexState() {
+        const table = await this.tables.table(TableName.Chunks);
+        if (!table) {
+            return { rows: 0, built: false, unindexed: 0, worthBuilding: false };
+        }
+        const rows = await table.countRows();
+        const index = (await table.listIndices()).find((i) => i.columns.includes("vector"));
+        if (!index) {
+            return {
+                rows,
+                built: false,
+                unindexed: rows,
+                worthBuilding: rows >= VectorIndexPolicy.MIN_ROWS_FOR_ANN,
+            };
+        }
+        const stats = await table.indexStats(index.name);
+        return {
+            rows,
+            built: true,
+            unindexed: stats?.numUnindexedRows ?? 0,
+            worthBuilding: true,
+        };
+    }
+    async reset() {
+        for (const name of [
+            TableName.Chunks,
+            TableName.Files,
+            TableName.LegacyVocab,
+        ]) {
+            await this.tables.dropTable(name);
+        }
+        this.manifest.invalidate();
+    }
+    /** Arrow rows carry typed arrays; LanceDB needs plain values to re-infer a schema. */
+    toPlainRow(row) {
+        const chunk = ChunkRepository.rowToChunk(row);
+        return {
+            id: row.id,
+            filePath: chunk.location.filePath,
+            startLine: chunk.location.startLine,
+            endLine: chunk.location.endLine,
+            type: chunk.type,
+            name: chunk.name,
+            content: chunk.content,
+            context: chunk.context,
+            hash: chunk.hash.toString(),
+            fileHash: chunk.fileVersion.toStored(),
+            vector: Array.from(row.vector),
+        };
+    }
+}
+//# sourceMappingURL=IndexMaintenance.js.map

package/dist/infrastructure/lancedb/IndexMaintenance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexMaintenance.js","sourceRoot":"","sources":["../../../src/infrastructure/lancedb/IndexMaintenance.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAc,MAAM,kBAAkB,CAAC;AAUrD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAoB,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D;;;;;;;;;;GAUG;AACH,MAAM,OAAO,gBAAgB;IAKV;IACA;IALlB,uEAAuE;IAC/D,MAAM,CAAU,aAAa,GAAG,CAAC,SAAS,CAAC,MAAM,CAAU,CAAC;IAEpE,YACkB,MAAmB,EACnB,QAAoC;QADpC,WAAM,GAAN,MAAM,CAAa;QACnB,aAAQ,GAAR,QAAQ,CAA4B;IACnD,CAAC;IAEJ;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,UAA2B,EAAE;QAC3C,MAAM,MAAM,GAAmB,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAC9C,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,aAAa,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CACjB,MAAM,IAAI,CAAC,aAAa,CACvB,IAAI,EACJ,KAAK,EACL,OAAO,CAAC,KAAK,IAAI,KAAK,EACtB,OAAO,CAAC,MAAM,IAAI,KAAK,CACvB,CACD,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IACf,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAClE,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACpC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACvD,IAAI,KAAK,EAAE,CAAC;YACX,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;YACvB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;gBAClB,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,IAAI,EAAE,MAAM,KAAK,CAAC,SAAS,EAAE;gBAC7B,MAAM,EAAE,WAAW;aACnB,CAAC,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC;IACf,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,gBAAgB,CAAC,MAAsB;QACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAC7D,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QACrC,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;YAClB,KAAK,EAAE,SAAS,CAAC,WAAW;YAC5B,IAAI;YACJ,MAAM,EAAE,SAAS;SACjB,CAAC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAC1B,IAAY,EACZ,KAAY,EACZ,KAAc,EACd,MAAe;QAEf,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QAErC,mEAAmE;QACnE,oEAAoE;QACpE,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;QAEtE,IAAI,CAAC,WAAW,EAAE,CAAC;YAClB,mEAAmE;YACnE,kEAAkE;YAClE,iEAAiE;YACjE,wCAAwC;YACxC,IAAI,IAAI,GAAG,iBAAiB,CAAC,gBAAgB,EAAE,CAAC;gBAC/C,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;YACvD,CAAC;YACD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;YACrD,CAAC;YACD,6DAA6D;YAC7D,mEAAmE;YACnE,8DAA8D;YAC9D,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE;gBACjC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC;oBACnB,YAAY,EAAE,iBAAiB,CAAC,aAAa;iBAC7C,CAAC;gBACF,OAAO,EAAE,IAAI;aACb,CAAC,CAAC;YACH,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;YACvB,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QACjD,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,KAAK,EAAE,gBAAgB,IAAI,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,KAAK,EAAE,cAAc,IAAI,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACzB,iBAAiB,CAAC,uBAAuB,EACzC,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,iBAAiB,CAAC,uBAAuB,CAAC,CAC/D,CAAC;QAEF,IAAI,CAAC,KAAK,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;YACtC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC;QAC/D,CAAC;QAED,uEAAuE;QACvE,mDAAmD;QACnD,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;QACvB,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC;IAC9D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM;QACX,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;QACnE,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QAEvC,0DAA0D;QAC1D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAuB,CAAC;QAClD,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC;YACtD,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;YAChE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;YACnC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrC,CAAC;QAED,MAAM,IAAI,GAA8B,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,MAAM,KAAK,GAAG,MAAM,CAAC;QACrB,MAAM,KAAK,GAAG,IAAI,CAAC;QACnB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,EAAE,MAAM,IAAI,KAAK,EAAE,CAAC;YACtD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC;YACvE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM,EAAE,GAAG,GAAG,CAAC,EAAY,CAAC;gBAC5B,IAAI,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBACrB,YAAY,EAAE,CAAC;oBACf,SAAS;gBACV,CAAC;gBACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAEhB,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAkB,CAAC;gBACxC,MAAM,QAAQ,GAAI,GAAG,CAAC,QAAmB,IAAI,EAAE,CAAC;gBAChD,2CAA2C;gBAC3C,IAAI,QAAQ,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACjE,aAAa,EAAE,CAAC;oBAChB,SAAS;gBACV,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;YACjC,CAAC;QACF,CAAC;QAED,IAAI,YAAY,GAAG,aAAa,KAAK,CAAC,EAAE,CAAC;YACxC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC;QAC/D,CAAC;QAED,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CACjE,SAAS,CAAC,MAAM,EAChB,IAAI,CACJ,CAAC;YACF,IAAI,CAAC,MAAM;gBAAE,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,gBAAgB;QACrB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,CAAC,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CACpD,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAC5B,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO;gBACN,IAAI;gBACJ,KAAK,EAAE,KAAK;gBACZ,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI,IAAI,iBAAiB,CAAC,gBAAgB;aACzD,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjD,OAAO;YACN,IAAI;YACJ,KAAK,EAAE,IAAI;YACX,SAAS,EAAE,KAAK,EAAE,gBAAgB,IAAI,CAAC;YACvC,aAAa,EAAE,IAAI;SACnB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACV,KAAK,MAAM,IAAI,IAAI;YAClB,SAAS,CAAC,MAAM;YAChB,SAAS,CAAC,KAAK;YACf,SAAS,CAAC,WAAW;SACrB,EAAE,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;IAC5B,CAAC;IAED,sFAAsF;IAC9E,UAAU,CAAC,GAA4B;QAC9C,MAAM,KAAK,GAAG,eAAe,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC9C,OAAO;YACN,EAAE,EAAE,GAAG,CAAC,EAAY;YACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;YACjC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS;YACnC,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO;YAC/B,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE;YAC3B,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE;YACtC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAA0B,CAAC;SAClD,CAAC;IACH,CAAC"}

package/dist/infrastructure/lancedb/LanceTables.d.ts

@@ -0,0 +1,67 @@
+import { type Table } from "@lancedb/lancedb";
+import type { Branch } from "../../domain/project/Branch.js";
+/**
+ * Table names inside a LanceDB database directory.
+ *
+ * A const object rather than a `const enum`: const enums are erased at emit,
+ * so importing one across modules compiles but can vanish at runtime.
+ */
+export declare const TableName: {
+    readonly Chunks: "chunks";
+    readonly Files: "files";
+    /**
+     * Written by the removed faceting feature. Nothing reads it; it is listed
+     * so maintenance can drop it and give the disk back.
+     */
+    readonly LegacyVocab: "vocab";
+};
+export type TableName = (typeof TableName)[keyof typeof TableName];
+/**
+ * Shared access to one LanceDB database: connecting, opening tables lazily,
+ * and running the schema backfills legacy databases need.
+ *
+ * The repositories share an instance so they share a connection — LanceDB
+ * connections are per-directory and opening several would multiply the native
+ * runtime's memory.
+ */
+export declare class LanceTables {
+    readonly path: string;
+    private readonly branch;
+    /**
+     * Predicates are built by string interpolation, so batches are kept small
+     * enough that no single filter grows unreasonably long.
+     */
+    static readonly FILTER_BATCH_SIZE = 50;
+    private db;
+    private readonly open;
+    constructor(path: string, branch: Branch);
+    private connection;
+    /** Open a table, or undefined when the database has never held it. */
+    table(name: TableName): Promise<Table | undefined>;
+    /**
+     * Open a table, creating it from `seed` rows when it does not exist.
+     * `seeded` says whether creation consumed the rows, so the caller knows not
+     * to insert them a second time.
+     */
+    tableOrCreate(name: TableName, seed: Record<string, unknown>[]): Promise<{
+        table: Table;
+        seeded: boolean;
+    }>;
+    tableNames(): Promise<string[]>;
+    dropTable(name: TableName): Promise<void>;
+    /** Forget cached handles so the next access re-opens and re-migrates. */
+    forget(name?: TableName): void;
+    close(): void;
+    /** `column IN ('a', 'b')`, with values single-quote escaped. */
+    static inFilter(column: string, values: readonly string[]): string;
+    static quote(value: string): string;
+    /** Run `column IN (...)` deletes in batches small enough for one predicate. */
+    static deleteIn(table: Table, column: string, values: readonly string[]): Promise<void>;
+    /** Run a batched `column IN (...)` select, collecting one column's values. */
+    static selectIn(table: Table, column: string, values: readonly string[]): Promise<Set<string>>;
+    /**
+     * Backfill columns that legacy databases predate. Both are additive and
+     * idempotent — a table that already has the column is left alone.
+     */
+    private migrate;
+}

package/dist/infrastructure/lancedb/LanceTables.js

@@ -0,0 +1,159 @@
+import { mkdirSync } from "node:fs";
+import { connect } from "@lancedb/lancedb";
+/**
+ * Table names inside a LanceDB database directory.
+ *
+ * A const object rather than a `const enum`: const enums are erased at emit,
+ * so importing one across modules compiles but can vanish at runtime.
+ */
+export const TableName = {
+    Chunks: "chunks",
+    Files: "files",
+    /**
+     * Written by the removed faceting feature. Nothing reads it; it is listed
+     * so maintenance can drop it and give the disk back.
+     */
+    LegacyVocab: "vocab",
+};
+/**
+ * Shared access to one LanceDB database: connecting, opening tables lazily,
+ * and running the schema backfills legacy databases need.
+ *
+ * The repositories share an instance so they share a connection — LanceDB
+ * connections are per-directory and opening several would multiply the native
+ * runtime's memory.
+ */
+export class LanceTables {
+    path;
+    branch;
+    /**
+     * Predicates are built by string interpolation, so batches are kept small
+     * enough that no single filter grows unreasonably long.
+     */
+    static FILTER_BATCH_SIZE = 50;
+    db;
+    open = new Map();
+    constructor(path, branch) {
+        this.path = path;
+        this.branch = branch;
+    }
+    async connection() {
+        if (this.db)
+            return this.db;
+        mkdirSync(this.path, { recursive: true });
+        this.db = await connect(this.path);
+        return this.db;
+    }
+    /** Open a table, or undefined when the database has never held it. */
+    async table(name) {
+        const cached = this.open.get(name);
+        if (cached)
+            return cached;
+        const conn = await this.connection();
+        const names = await conn.tableNames();
+        if (!names.includes(name))
+            return undefined;
+        const table = await conn.openTable(name);
+        await this.migrate(name, table);
+        this.open.set(name, table);
+        return table;
+    }
+    /**
+     * Open a table, creating it from `seed` rows when it does not exist.
+     * `seeded` says whether creation consumed the rows, so the caller knows not
+     * to insert them a second time.
+     */
+    async tableOrCreate(name, seed) {
+        const existing = await this.table(name);
+        if (existing)
+            return { table: existing, seeded: false };
+        const conn = await this.connection();
+        const created = await conn.createTable(name, seed);
+        this.open.set(name, created);
+        return { table: created, seeded: true };
+    }
+    async tableNames() {
+        return (await this.connection()).tableNames();
+    }
+    async dropTable(name) {
+        const conn = await this.connection();
+        const names = await conn.tableNames();
+        if (names.includes(name))
+            await conn.dropTable(name);
+        this.open.delete(name);
+    }
+    /** Forget cached handles so the next access re-opens and re-migrates. */
+    forget(name) {
+        if (name)
+            this.open.delete(name);
+        else
+            this.open.clear();
+    }
+    close() {
+        this.open.clear();
+        this.db = undefined;
+    }
+    /** `column IN ('a', 'b')`, with values single-quote escaped. */
+    static inFilter(column, values) {
+        const escaped = values.map((v) => `'${v.replace(/'/g, "''")}'`);
+        return `${column} IN (${escaped.join(", ")})`;
+    }
+    static quote(value) {
+        return value.replace(/'/g, "''");
+    }
+    /** Run `column IN (...)` deletes in batches small enough for one predicate. */
+    static async deleteIn(table, column, values) {
+        for (let i = 0; i < values.length; i += LanceTables.FILTER_BATCH_SIZE) {
+            const batch = values.slice(i, i + LanceTables.FILTER_BATCH_SIZE);
+            await table.delete(LanceTables.inFilter(column, batch));
+        }
+    }
+    /** Run a batched `column IN (...)` select, collecting one column's values. */
+    static async selectIn(table, column, values) {
+        const found = new Set();
+        for (let i = 0; i < values.length; i += LanceTables.FILTER_BATCH_SIZE) {
+            const batch = values.slice(i, i + LanceTables.FILTER_BATCH_SIZE);
+            const rows = await table
+                .query()
+                .where(LanceTables.inFilter(column, batch))
+                .select([column])
+                .toArray();
+            for (const r of rows)
+                found.add(r[column]);
+        }
+        return found;
+    }
+    /**
+     * Backfill columns that legacy databases predate. Both are additive and
+     * idempotent — a table that already has the column is left alone.
+     */
+    async migrate(name, table) {
+        if (name === TableName.Files) {
+            // Pre branch-scoping `files` tables have no `branch` column. Fill it
+            // with the current branch so branch-filtered queries keep working.
+            const schema = await table.schema();
+            if (schema.fields.some((f) => f.name === "branch"))
+                return;
+            await table.addColumns([
+                {
+                    name: "branch",
+                    valueSql: `CAST('${LanceTables.quote(this.branch.toString())}' AS STRING)`,
+                },
+            ]);
+            return;
+        }
+        if (name === TableName.Chunks) {
+            // Pre version-scoping `chunks` tables have no `fileHash`. Fill it
+            // with "" — treated as a wildcard at search time, so legacy chunks
+            // keep appearing until their file is re-indexed and gets a real
+            // version hash.
+            const schema = await table.schema();
+            if (schema.fields.some((f) => f.name === "fileHash"))
+                return;
+            await table.addColumns([
+                { name: "fileHash", valueSql: `CAST('' AS STRING)` },
+            ]);
+        }
+    }
+}
+//# sourceMappingURL=LanceTables.js.map

package/dist/infrastructure/lancedb/LanceTables.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LanceTables.js","sourceRoot":"","sources":["../../../src/infrastructure/lancedb/LanceTables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAmB,OAAO,EAAc,MAAM,kBAAkB,CAAC;AAGxE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG;IACxB,MAAM,EAAE,QAAQ;IAChB,KAAK,EAAE,OAAO;IACd;;;OAGG;IACH,WAAW,EAAE,OAAO;CACX,CAAC;AAIX;;;;;;;GAOG;AACH,MAAM,OAAO,WAAW;IAWb;IACQ;IAXlB;;;OAGG;IACH,MAAM,CAAU,iBAAiB,GAAG,EAAE,CAAC;IAE/B,EAAE,CAAyB;IAClB,IAAI,GAAG,IAAI,GAAG,EAAoB,CAAC;IAEpD,YACU,IAAY,EACJ,MAAc;QADtB,SAAI,GAAJ,IAAI,CAAQ;QACJ,WAAM,GAAN,MAAM,CAAQ;IAC7B,CAAC;IAEI,KAAK,CAAC,UAAU;QACvB,IAAI,IAAI,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAC5B,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,OAAO,IAAI,CAAC,EAAE,CAAC;IAChB,CAAC;IAED,sEAAsE;IACtE,KAAK,CAAC,KAAK,CAAC,IAAe;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAE1B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3B,OAAO,KAAK,CAAC;IACd,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa,CAClB,IAAe,EACf,IAA+B;QAE/B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,QAAQ;YAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAExD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC7B,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,UAAU;QACf,OAAO,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAe;QAC9B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IAED,yEAAyE;IACzE,MAAM,CAAC,IAAgB;QACtB,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;;YAC5B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,KAAK;QACJ,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,EAAE,GAAG,SAAS,CAAC;IACrB,CAAC;IAED,gEAAgE;IAChE,MAAM,CAAC,QAAQ,CAAC,MAAc,EAAE,MAAyB;QACxD,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAChE,OAAO,GAAG,MAAM,QAAQ,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;IAC/C,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,KAAa;QACzB,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAClC,CAAC;IAED,+EAA+E;IAC/E,MAAM,CAAC,KAAK,CAAC,QAAQ,CACpB,KAAY,EACZ,MAAc,EACd,MAAyB;QAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,CAAC,iBAAiB,EAAE,CAAC;YACvE,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,iBAAiB,CAAC,CAAC;YACjE,MAAM,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QACzD,CAAC;IACF,CAAC;IAED,8EAA8E;IAC9E,MAAM,CAAC,KAAK,CAAC,QAAQ,CACpB,KAAY,EACZ,MAAc,EACd,MAAyB;QAEzB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,CAAC,iBAAiB,EAAE,CAAC;YACvE,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,iBAAiB,CAAC,CAAC;YACjE,MAAM,IAAI,GAAG,MAAM,KAAK;iBACtB,KAAK,EAAE;iBACP,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;iBAC1C,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC;iBAChB,OAAO,EAAE,CAAC;YACZ,KAAK,MAAM,CAAC,IAAI,IAAI;gBAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAW,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,KAAK,CAAC;IACd,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,OAAO,CAAC,IAAe,EAAE,KAAY;QAClD,IAAI,IAAI,KAAK,SAAS,CAAC,KAAK,EAAE,CAAC;YAC9B,qEAAqE;YACrE,mEAAmE;YACnE,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;YACpC,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC;gBAAE,OAAO;YAC3D,MAAM,KAAK,CAAC,UAAU,CAAC;gBACtB;oBACC,IAAI,EAAE,QAAQ;oBACd,QAAQ,EAAE,SAAS,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,cAAc;iBAC1E;aACD,CAAC,CAAC;YACH,OAAO;QACR,CAAC;QAED,IAAI,IAAI,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;YAC/B,kEAAkE;YAClE,mEAAmE;YACnE,gEAAgE;YAChE,gBAAgB;YAChB,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;YACpC,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC;gBAAE,OAAO;YAC7D,MAAM,KAAK,CAAC,UAAU,CAAC;gBACtB,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,oBAAoB,EAAE;aACpD,CAAC,CAAC;QACJ,CAAC;IACF,CAAC"}

package/dist/{lib/native-tuning.js → infrastructure/lancedb/NativeTuning.js}

@@ -6,4 +6,4 @@ process.env.UV_THREADPOOL_SIZE ??= "4";
 process.env.RAYON_NUM_THREADS ??= "4";
 process.env.MATMUL_NUM_THREADS ??= "4";
 export {};
-//# sourceMappingURL=native-tuning.js.map
+//# sourceMappingURL=NativeTuning.js.map

package/dist/infrastructure/lancedb/NativeTuning.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"NativeTuning.js","sourceRoot":"","sources":["../../../src/infrastructure/lancedb/NativeTuning.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,wEAAwE;AACxE,wDAAwD;AACxD,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,GAAG,CAAC;AACzC,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,GAAG,CAAC;AACvC,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,GAAG,CAAC;AACtC,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,GAAG,CAAC"}

package/dist/infrastructure/lancedb/RowReplication.d.ts

@@ -0,0 +1,27 @@
+import type { Branch } from "../../domain/project/Branch.js";
+import { type LanceTables } from "./LanceTables.js";
+/** A manifest row as it travels between peers. */
+export interface ManifestRow {
+    filePath: string;
+    fileHash: string;
+    branch: string;
+}
+/**
+ * Row-level access to the tables, for replication between databases.
+ *
+ * Peer sharing copies rows verbatim — embeddings included, since avoiding
+ * re-embedding is the entire point — so it works below the domain model rather
+ * than through it. Keeping that access behind this one class means the rest of
+ * the system never sees raw rows.
+ */
+export declare class RowReplication {
+    private readonly tables;
+    private readonly branch;
+    constructor(tables: LanceTables, branch: Branch);
+    chunkCount(): Promise<number>;
+    /** Stream chunk rows in pages, so a large index never loads at once. */
+    streamChunkRows(batchSize: number): AsyncGenerator<Record<string, unknown>[]>;
+    allManifestRows(): Promise<ManifestRow[]>;
+    addChunkRows(rows: Record<string, unknown>[]): Promise<void>;
+    addManifestRows(rows: ManifestRow[]): Promise<void>;
+}

package/dist/infrastructure/lancedb/RowReplication.js

@@ -0,0 +1,82 @@
+import { TableName } from "./LanceTables.js";
+/**
+ * Row-level access to the tables, for replication between databases.
+ *
+ * Peer sharing copies rows verbatim — embeddings included, since avoiding
+ * re-embedding is the entire point — so it works below the domain model rather
+ * than through it. Keeping that access behind this one class means the rest of
+ * the system never sees raw rows.
+ */
+export class RowReplication {
+    tables;
+    branch;
+    constructor(tables, branch) {
+        this.tables = tables;
+        this.branch = branch;
+    }
+    async chunkCount() {
+        const table = await this.tables.table(TableName.Chunks);
+        return table ? table.countRows() : 0;
+    }
+    /** Stream chunk rows in pages, so a large index never loads at once. */
+    async *streamChunkRows(batchSize) {
+        const table = await this.tables.table(TableName.Chunks);
+        if (!table)
+            return;
+        const total = await table.countRows();
+        for (let offset = 0; offset < total; offset += batchSize) {
+            const rows = await table
+                .query()
+                .limit(batchSize)
+                .offset(offset)
+                .toArray();
+            if (rows.length === 0)
+                break;
+            yield rows.map((r) => ({
+                id: r.id,
+                filePath: r.filePath,
+                startLine: r.startLine,
+                endLine: r.endLine,
+                type: r.type,
+                name: r.name,
+                content: r.content,
+                context: r.context,
+                hash: r.hash,
+                fileHash: r.fileHash ?? "",
+                // Arrow typed arrays do not survive JSON; send plain numbers.
+                vector: Array.from(r.vector),
+            }));
+        }
+    }
+    async allManifestRows() {
+        const table = await this.tables.table(TableName.Files);
+        if (!table)
+            return [];
+        const rows = await table.query().toArray();
+        return rows.map((r) => ({
+            filePath: r.filePath,
+            fileHash: r.fileHash,
+            branch: r.branch ?? this.branch.toString(),
+        }));
+    }
+    async addChunkRows(rows) {
+        if (rows.length === 0)
+            return;
+        const { table, seeded } = await this.tables.tableOrCreate(TableName.Chunks, rows);
+        if (!seeded)
+            await table.add(rows);
+    }
+    async addManifestRows(rows) {
+        if (rows.length === 0)
+            return;
+        const records = rows.map((r) => ({
+            filePath: r.filePath,
+            fileHash: r.fileHash,
+            branch: r.branch ?? this.branch.toString(),
+        }));
+        const { table, seeded } = await this.tables.tableOrCreate(TableName.Files, records);
+        if (!seeded)
+            await table.add(records);
+    }
+}
+//# sourceMappingURL=RowReplication.js.map

package/dist/infrastructure/lancedb/RowReplication.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RowReplication.js","sourceRoot":"","sources":["../../../src/infrastructure/lancedb/RowReplication.ts"],"names":[],"mappings":"AACA,OAAO,EAAoB,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAS/D;;;;;;;GAOG;AACH,MAAM,OAAO,cAAc;IAER;IACA;IAFlB,YACkB,MAAmB,EACnB,MAAc;QADd,WAAM,GAAN,MAAM,CAAa;QACnB,WAAM,GAAN,MAAM,CAAQ;IAC7B,CAAC;IAEJ,KAAK,CAAC,UAAU;QACf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACxD,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,CAAC,eAAe,CACrB,SAAiB;QAEjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC;YAC1D,MAAM,IAAI,GAAG,MAAM,KAAK;iBACtB,KAAK,EAAE;iBACP,KAAK,CAAC,SAAS,CAAC;iBAChB,MAAM,CAAC,MAAM,CAAC;iBACd,OAAO,EAAE,CAAC;YACZ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAC7B,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtB,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAG,CAAC,CAAC,QAAmB,IAAI,EAAE;gBACtC,8DAA8D;gBAC9D,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAA0B,CAAC;aAChD,CAAC,CAAC,CAAC;QACL,CAAC;IACF,CAAC;IAED,KAAK,CAAC,eAAe;QACpB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvB,QAAQ,EAAE,CAAC,CAAC,QAAkB;YAC9B,QAAQ,EAAE,CAAC,CAAC,QAAkB;YAC9B,MAAM,EAAG,CAAC,CAAC,MAAiB,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;SACtD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,IAA+B;QACjD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC9B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CACxD,SAAS,CAAC,MAAM,EAChB,IAAI,CACJ,CAAC;QACF,IAAI,CAAC,MAAM;YAAE,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAAmB;QACxC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChC,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;SAC1C,CAAC,CAAC,CAAC;QACJ,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CACxD,SAAS,CAAC,KAAK,EACf,OAAO,CACP,CAAC;QACF,IAAI,CAAC,MAAM;YAAE,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;CACD"}

package/dist/infrastructure/lancedb/VectorIndexPolicy.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Tuning for the LanceDB vector index. The numbers here were measured on a
+ * 43k-chunk / 3584-dimension index; the comments record what they bought so a
+ * later change can tell a tuned value from an arbitrary one.
+ */
+export declare const VectorIndexPolicy: {
+    /**
+     * Below this row count a brute-force scan is cheap enough that training an
+     * index costs more than it saves, and IVF-PQ has too few vectors to cluster
+     * well.
+     */
+    readonly MIN_ROWS_FOR_ANN: 5000;
+    /**
+     * Re-absorb new rows into the index once the unindexed tail passes this
+     * share of the indexed body. Rows outside the index are searched by flat
+     * scan, so the tail is exactly what needs bounding.
+     */
+    readonly UNINDEXED_REINDEX_RATIO: 0.2;
+    /** ...but always tolerate a small tail, so a 3-chunk edit never retrains. */
+    readonly UNINDEXED_REINDEX_FLOOR: 2000;
+    /**
+     * Vector searches run without an explicit distance type, which LanceDB
+     * resolves to L2. The index has its own default, so it is pinned: an index
+     * built on a different metric would silently reorder every result.
+     */
+    readonly DISTANCE_TYPE: "l2";
+    /**
+     * IVF-PQ stores compressed vectors, so its raw distances approximate the
+     * real ones. `refineFactor` makes LanceDB pull the uncompressed vectors for
+     * the top `limit * factor` candidates and re-score them exactly.
+     *
+     * It is not optional. Measured mean recall@20 against an exact scan:
+     *
+     *   off -> 53%   1 -> 75%   3 -> 86%   10 -> 91%   20 -> 94%   50 -> 94%
+     *
+     * and without it `_distance` is wrong by ~0.07, which matters because score
+     * is derived from it and users filter on `--min-score`. With refinement the
+     * scores match an exact scan to the digit.
+     *
+     * 20 sits at the knee: ~94% recall at ~11ms/query versus ~115ms for an
+     * exact scan. Beyond it recall flattens (the probed partitions run out of
+     * candidates) while latency keeps climbing. The refined set is bounded by
+     * the query limit, not by table size, so this costs a fixed few MB.
+     *
+     * Harmless on tables with no index: a flat scan already computes exact
+     * distances and returns identical rows.
+     */
+    readonly REFINE_FACTOR: 20;
+};
+/**
+ * Columns a search reads off a result row.
+ *
+ * Selecting explicitly keeps the embedding out of the result set — without it
+ * LanceDB ships every over-fetched row's full vector back into JS for nothing.
+ *
+ * `_distance` is listed on purpose: LanceDB currently auto-projects it into any
+ * scoring query but warns that it will stop, so naming it pins the behaviour
+ * rather than relying on a deprecated default.
+ */
+export declare const SEARCH_COLUMNS: readonly ["id", "filePath", "startLine", "endLine", "type", "name", "content", "context", "fileHash", "_distance"];