lmgrep 0.1.17 → 0.1.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/cli.d.ts +1 -1
  2. package/dist/cli.js +203 -3
  3. package/dist/cli.js.map +1 -1
  4. package/dist/index.d.ts +14 -2
  5. package/dist/index.js +355 -1
  6. package/dist/index.js.map +1 -1
  7. package/dist/lib/build.d.ts +14 -0
  8. package/dist/lib/build.js +126 -36
  9. package/dist/lib/build.js.map +1 -1
  10. package/dist/lib/chunker/context.d.ts +4 -4
  11. package/dist/lib/chunker/context.js.map +1 -1
  12. package/dist/lib/chunker/index.js +64 -5
  13. package/dist/lib/chunker/index.js.map +1 -1
  14. package/dist/lib/chunker/languages.d.ts +1 -1
  15. package/dist/lib/chunker/languages.js +36 -17
  16. package/dist/lib/chunker/languages.js.map +1 -1
  17. package/dist/lib/embedder.d.ts +11 -1
  18. package/dist/lib/embedder.js +16 -1
  19. package/dist/lib/embedder.js.map +1 -1
  20. package/dist/lib/facet-session.d.ts +44 -0
  21. package/dist/lib/facet-session.js +125 -0
  22. package/dist/lib/facet-session.js.map +1 -0
  23. package/dist/lib/native-tuning.d.ts +1 -0
  24. package/dist/lib/native-tuning.js +9 -0
  25. package/dist/lib/native-tuning.js.map +1 -0
  26. package/dist/lib/search-tool.d.ts +9 -1
  27. package/dist/lib/search-tool.js +92 -12
  28. package/dist/lib/search-tool.js.map +1 -1
  29. package/dist/lib/serve.js +17 -0
  30. package/dist/lib/serve.js.map +1 -1
  31. package/dist/lib/store.d.ts +78 -3
  32. package/dist/lib/store.js +407 -16
  33. package/dist/lib/store.js.map +1 -1
  34. package/dist/lib/types.d.ts +65 -0
  35. package/dist/lib/types.js.map +1 -1
  36. package/dist/lib/vocab.d.ts +20 -0
  37. package/dist/lib/vocab.js +275 -0
  38. package/dist/lib/vocab.js.map +1 -0
  39. package/dist/mcp.d.ts +1 -1
  40. package/dist/mcp.js +13 -1
  41. package/dist/mcp.js.map +1 -1
  42. package/package.json +29 -3
@@ -74,6 +74,17 @@ export declare function acquireDbLock(cwd: string): boolean;
74
74
  * Release the write lock for a project's DB.
75
75
  */
76
76
  export declare function releaseDbLock(cwd: string): void;
77
+ /**
78
+ * Run `fn` while holding the project's write mutex, so concurrent indexers
79
+ * (a watcher plus an ad-hoc `lmgrep index`) can't write at the same time and
80
+ * produce duplicate chunk rows. Waits up to `waitMs` for a busy lock, taking
81
+ * over a lock held by a dead process. Throws if the lock can't be acquired in
82
+ * time.
83
+ */
84
+ export declare function withWriteLock<T>(cwd: string, fn: () => Promise<T>, opts?: {
85
+ waitMs?: number;
86
+ pollMs?: number;
87
+ }): Promise<T>;
77
88
  /**
78
89
  * Check if a write lock is held by a live process.
79
90
  */
@@ -102,11 +113,33 @@ export declare class Store {
102
113
  private db;
103
114
  private chunksTable;
104
115
  private filesTable;
116
+ private vocabTable;
105
117
  constructor(dbPath: string, branch?: string);
106
118
  static forProject(cwd: string): Store;
107
119
  private connection;
108
120
  private openChunks;
109
121
  private openFiles;
122
+ private openVocab;
123
+ hasVocab(): Promise<boolean>;
124
+ /**
125
+ * Return the set of vocab terms already embedded in the vocab table.
126
+ * Used at index time to skip re-embedding.
127
+ */
128
+ getVocabTerms(): Promise<Set<string>>;
129
+ addVocab(entries: Array<{
130
+ term: string;
131
+ vector: number[];
132
+ }>): Promise<void>;
133
+ /**
134
+ * ANN search against the vocab table. Returns top-N terms closest to the
135
+ * given vector by cosine distance.
136
+ */
137
+ searchVocab(vector: number[], limit: number, excludeTerms?: Set<string>): Promise<Array<{
138
+ term: string;
139
+ score: number;
140
+ }>>;
141
+ vocabCount(): Promise<number>;
142
+ dropVocab(): Promise<void>;
110
143
  addChunks(chunks: IndexedChunk[]): Promise<void>;
111
144
  /**
112
145
  * Delete chunks for files that are no longer referenced by ANY branch.
@@ -114,16 +147,34 @@ export declare class Store {
114
147
  * the chunks are kept (they're shared via content-addressing).
115
148
  */
116
149
  deleteChunksByFiles(filePaths: string[]): Promise<void>;
117
- private branchFilesCache;
150
+ private branchVersionsCache;
118
151
  /**
119
- * Get the set of file paths indexed on the current branch (cached).
152
+ * Map of filePath -> file-version hash for the current branch (cached).
153
+ * Used to scope search to the exact file versions this branch references,
154
+ * so stale chunks from another version of the same path are excluded.
120
155
  */
121
- getBranchFiles(): Promise<Set<string> | undefined>;
156
+ getBranchFileVersions(): Promise<Map<string, string> | undefined>;
122
157
  /** Invalidate the branch files cache (call after index/import). */
123
158
  invalidateBranchFilesCache(): void;
124
159
  search(queryVector: number[], limit?: number, filePrefix?: string, typeFilter?: string[],
125
160
  /** Pass false to skip branch scoping (e.g. for cross-project search). */
126
161
  scopeToBranch?: boolean): Promise<SearchResult[]>;
162
+ /**
163
+ * Same as search() but also returns each chunk's vector — needed for
164
+ * client-side clustering (facet command).
165
+ */
166
+ searchWithVectors(queryVector: number[], limit?: number, filePrefix?: string, scopeToBranch?: boolean): Promise<Array<SearchResult & {
167
+ id: string;
168
+ vector: number[];
169
+ }>>;
170
+ /**
171
+ * Fetch chunks (with vectors) by id. Used to rehydrate faceting sessions.
172
+ * Missing ids are silently dropped.
173
+ */
174
+ getChunksByIds(ids: string[]): Promise<Array<SearchResult & {
175
+ id: string;
176
+ vector: number[];
177
+ }>>;
127
178
  getIndexedFiles(): Promise<Map<string, string[]>>;
128
179
  getIndexedHashes(): Promise<Set<string>>;
129
180
  /**
@@ -132,6 +183,14 @@ export declare class Store {
132
183
  */
133
184
  filterExistingChunkHashes(hashes: string[]): Promise<Set<string>>;
134
185
  chunkCount(): Promise<number>;
186
+ /**
187
+ * Stream chunk texts (name + content only) in batches. Used by vocab
188
+ * backfill to avoid loading all chunks into memory at once.
189
+ */
190
+ streamChunkTexts(batchSize?: number): AsyncIterable<Array<{
191
+ name: string;
192
+ content: string;
193
+ }>>;
135
194
  getFileHashes(): Promise<Map<string, string>>;
136
195
  /**
137
196
  * Given a set of file hashes, return those that already exist in the
@@ -145,6 +204,22 @@ export declare class Store {
145
204
  }>): Promise<void>;
146
205
  deleteFileHashes(filePaths: string[]): Promise<void>;
147
206
  streamAllChunks(batchSize: number): AsyncGenerator<Record<string, unknown>[]>;
207
+ /**
208
+ * Remove redundant rows from the chunks table:
209
+ * - exact duplicate ids (identical rows from concurrent unlocked indexing)
210
+ * - orphaned versions: chunks whose fileHash is no longer referenced by any
211
+ * branch's manifest (left behind when a file was edited on one branch
212
+ * while another branch still pointed at the old path). Legacy chunks with
213
+ * an empty fileHash are kept.
214
+ * Rewrites the table from the surviving rows. Loads chunks into memory in
215
+ * batches, so this is a maintenance operation, not a hot path.
216
+ */
217
+ dedupeChunks(): Promise<{
218
+ before: number;
219
+ after: number;
220
+ duplicateIds: number;
221
+ staleVersions: number;
222
+ }>;
148
223
  getAllFileEntries(): Promise<Array<{
149
224
  filePath: string;
150
225
  fileHash: string;
package/dist/lib/store.js CHANGED
@@ -6,6 +6,7 @@ import { homedir } from "node:os";
6
6
  import { join, resolve } from "node:path";
7
7
  const CHUNKS_TABLE = "chunks";
8
8
  const FILES_TABLE = "files";
9
+ const VOCAB_TABLE = "vocab";
9
10
  const DELETE_BATCH_SIZE = 50;
10
11
  function buildInFilter(column, values) {
11
12
  const escaped = values.map((v) => `'${v.replace(/'/g, "''")}'`);
@@ -17,6 +18,40 @@ async function batchDelete(table, column, values) {
17
18
  await table.delete(buildInFilter(column, batch));
18
19
  }
19
20
  }
21
+ /**
22
+ * Remove redundant search rows. Two passes, assuming `rows` is already in
23
+ * best-first order (ANN returns ascending distance = descending score):
24
+ *
25
+ * 1. Exact duplicates by chunk id — collapses identical rows produced by
26
+ * concurrent unlocked indexing (same filePath:row:contentHash).
27
+ * 2. Overlapping line ranges within the same file — collapses the
28
+ * fallback chunker's sliding-window overlap (and any parent/child or
29
+ * near-duplicate spans), keeping the highest-scoring chunk of each
30
+ * overlapping cluster. Tree-sitter chunks are node-bounded and don't
31
+ * overlap, so this only ever drops genuine near-duplicates.
32
+ */
33
+ function dedupeRows(rows) {
34
+ const seenIds = new Set();
35
+ const keptRanges = new Map();
36
+ const out = [];
37
+ for (const r of rows) {
38
+ if (seenIds.has(r.id))
39
+ continue;
40
+ seenIds.add(r.id);
41
+ const ranges = keptRanges.get(r.filePath);
42
+ if (ranges) {
43
+ const overlaps = ranges.some(([s, e]) => r.startLine <= e && s <= r.endLine);
44
+ if (overlaps)
45
+ continue;
46
+ ranges.push([r.startLine, r.endLine]);
47
+ }
48
+ else {
49
+ keptRanges.set(r.filePath, [[r.startLine, r.endLine]]);
50
+ }
51
+ out.push(r);
52
+ }
53
+ return out;
54
+ }
20
55
  /**
21
56
  * Legacy indexes (pre branch-scoping) have a `files` table without a `branch`
22
57
  * column. Detect and backfill it with the current branch so queries that filter
@@ -31,6 +66,20 @@ async function migrateBranchColumn(table, currentBranch) {
31
66
  { name: "branch", valueSql: `CAST('${escaped}' AS STRING)` },
32
67
  ]);
33
68
  }
69
+ /**
70
+ * Legacy chunk tables (pre version-scoping) have no `fileHash` column. Backfill
71
+ * it with "" — the empty string is treated as a wildcard at search time so
72
+ * legacy chunks keep appearing until the file is re-indexed and gets a real
73
+ * file-version hash.
74
+ */
75
+ async function migrateFileHashColumn(table) {
76
+ const schema = await table.schema();
77
+ if (schema.fields.some((f) => f.name === "fileHash"))
78
+ return;
79
+ await table.addColumns([
80
+ { name: "fileHash", valueSql: `CAST('' AS STRING)` },
81
+ ]);
82
+ }
34
83
  function git(cwd, ...args) {
35
84
  try {
36
85
  return execSync(`git ${args.join(" ")}`, {
@@ -267,6 +316,71 @@ export function releaseDbLock(cwd) {
267
316
  }
268
317
  catch { }
269
318
  }
319
+ // --- Per-build write mutex ---
320
+ //
321
+ // The `.lock` maintainer lock (above) is held for a watcher/serve process's
322
+ // whole lifetime and doubles as a liveness registry for `lmgrep status`. It
323
+ // cannot also serve as a write mutex, because then a one-shot `lmgrep index`
324
+ // could never run while a watcher is up. This separate `.writelock` is a
325
+ // short-lived mutex acquired around each build() so that the watcher and an
326
+ // ad-hoc `lmgrep index` serialize their writes instead of racing into
327
+ // duplicate rows. Named `.writelock` (not `.write.lock`) so it does not match
328
+ // the `.lock` suffix scan in discoverRunningProcesses.
329
+ function writeLockPath(cwd) {
330
+ return `${getDbPath(cwd)}.writelock`;
331
+ }
332
+ function tryAcquireWriteLock(cwd) {
333
+ const lockPath = writeLockPath(cwd);
334
+ if (existsSync(lockPath)) {
335
+ try {
336
+ const pid = Number.parseInt(readFileSync(lockPath, "utf-8").trim(), 10);
337
+ if (isProcessAlive(pid))
338
+ return false;
339
+ }
340
+ catch {
341
+ // stale/corrupt lock, take over
342
+ }
343
+ }
344
+ mkdirSync(getDbPath(cwd), { recursive: true });
345
+ writeFileSync(lockPath, `${process.pid}\n`);
346
+ return true;
347
+ }
348
+ function releaseWriteLock(cwd) {
349
+ const lockPath = writeLockPath(cwd);
350
+ try {
351
+ // Only remove the lock if we still own it.
352
+ const pid = Number.parseInt(readFileSync(lockPath, "utf-8").trim(), 10);
353
+ if (pid === process.pid)
354
+ unlinkSync(lockPath);
355
+ }
356
+ catch { }
357
+ }
358
+ /**
359
+ * Run `fn` while holding the project's write mutex, so concurrent indexers
360
+ * (a watcher plus an ad-hoc `lmgrep index`) can't write at the same time and
361
+ * produce duplicate chunk rows. Waits up to `waitMs` for a busy lock, taking
362
+ * over a lock held by a dead process. Throws if the lock can't be acquired in
363
+ * time.
364
+ */
365
+ export async function withWriteLock(cwd, fn, opts = {}) {
366
+ const waitMs = opts.waitMs ?? 120_000;
367
+ const pollMs = opts.pollMs ?? 200;
368
+ let waited = 0;
369
+ while (!tryAcquireWriteLock(cwd)) {
370
+ if (waited >= waitMs) {
371
+ throw new Error("Could not acquire the index write lock — another indexer is busy. " +
372
+ "Try again once it finishes.");
373
+ }
374
+ await new Promise((r) => setTimeout(r, pollMs));
375
+ waited += pollMs;
376
+ }
377
+ try {
378
+ return await fn();
379
+ }
380
+ finally {
381
+ releaseWriteLock(cwd);
382
+ }
383
+ }
270
384
  /**
271
385
  * Check if a write lock is held by a live process.
272
386
  */
@@ -351,6 +465,7 @@ export class Store {
351
465
  db;
352
466
  chunksTable;
353
467
  filesTable;
468
+ vocabTable;
354
469
  constructor(dbPath, branch = "_default") {
355
470
  this.dbPath = dbPath;
356
471
  this.branch = branch;
@@ -373,7 +488,9 @@ export class Store {
373
488
  const conn = await this.connection();
374
489
  const tables = await conn.tableNames();
375
490
  if (tables.includes(CHUNKS_TABLE)) {
376
- this.chunksTable = await conn.openTable(CHUNKS_TABLE);
491
+ const t = await conn.openTable(CHUNKS_TABLE);
492
+ await migrateFileHashColumn(t);
493
+ this.chunksTable = t;
377
494
  return this.chunksTable;
378
495
  }
379
496
  return undefined;
@@ -391,6 +508,100 @@ export class Store {
391
508
  }
392
509
  return undefined;
393
510
  }
511
+ // --- Vocab ---
512
+ async openVocab() {
513
+ if (this.vocabTable)
514
+ return this.vocabTable;
515
+ const conn = await this.connection();
516
+ const tables = await conn.tableNames();
517
+ if (tables.includes(VOCAB_TABLE)) {
518
+ this.vocabTable = await conn.openTable(VOCAB_TABLE);
519
+ return this.vocabTable;
520
+ }
521
+ return undefined;
522
+ }
523
+ async hasVocab() {
524
+ return (await this.openVocab()) !== undefined;
525
+ }
526
+ /**
527
+ * Return the set of vocab terms already embedded in the vocab table.
528
+ * Used at index time to skip re-embedding.
529
+ */
530
+ async getVocabTerms() {
531
+ const t = await this.openVocab();
532
+ if (!t)
533
+ return new Set();
534
+ const rows = await t.query().select(["term"]).toArray();
535
+ return new Set(rows.map((r) => r.term));
536
+ }
537
+ async addVocab(entries) {
538
+ if (entries.length === 0)
539
+ return;
540
+ // Dedup within the batch
541
+ const seen = new Set();
542
+ const batchUnique = [];
543
+ for (const e of entries) {
544
+ if (seen.has(e.term))
545
+ continue;
546
+ seen.add(e.term);
547
+ batchUnique.push(e);
548
+ }
549
+ // Skip terms already in the table
550
+ const known = await this.getVocabTerms();
551
+ const records = batchUnique
552
+ .filter((e) => !known.has(e.term))
553
+ .map((e) => ({ term: e.term, vector: e.vector }));
554
+ if (records.length === 0)
555
+ return;
556
+ const conn = await this.connection();
557
+ const tables = await conn.tableNames();
558
+ if (tables.includes(VOCAB_TABLE)) {
559
+ const t = await conn.openTable(VOCAB_TABLE);
560
+ this.vocabTable = t;
561
+ await t.add(records);
562
+ }
563
+ else {
564
+ this.vocabTable = await conn.createTable(VOCAB_TABLE, records);
565
+ }
566
+ }
567
+ /**
568
+ * ANN search against the vocab table. Returns top-N terms closest to the
569
+ * given vector by cosine distance.
570
+ */
571
+ async searchVocab(vector, limit, excludeTerms) {
572
+ const t = await this.openVocab();
573
+ if (!t)
574
+ return [];
575
+ const fetch = excludeTerms ? limit + excludeTerms.size : limit;
576
+ const rows = await t.search(vector).limit(fetch).toArray();
577
+ const out = [];
578
+ for (const r of rows) {
579
+ const term = r.term;
580
+ if (excludeTerms?.has(term))
581
+ continue;
582
+ out.push({
583
+ term,
584
+ score: r._distance != null ? 1 - r._distance : 0,
585
+ });
586
+ if (out.length >= limit)
587
+ break;
588
+ }
589
+ return out;
590
+ }
591
+ async vocabCount() {
592
+ const t = await this.openVocab();
593
+ if (!t)
594
+ return 0;
595
+ return t.countRows();
596
+ }
597
+ async dropVocab() {
598
+ const conn = await this.connection();
599
+ const tables = await conn.tableNames();
600
+ if (tables.includes(VOCAB_TABLE)) {
601
+ await conn.dropTable(VOCAB_TABLE);
602
+ }
603
+ this.vocabTable = undefined;
604
+ }
394
605
  // --- Chunks ---
395
606
  async addChunks(chunks) {
396
607
  if (chunks.length === 0)
@@ -406,6 +617,7 @@ export class Store {
406
617
  content: c.content,
407
618
  context: c.context,
408
619
  hash: c.hash,
620
+ fileHash: c.fileHash ?? "",
409
621
  vector: c.vector,
410
622
  }));
411
623
  const tables = await conn.tableNames();
@@ -454,13 +666,15 @@ export class Store {
454
666
  await batchDelete(t, "filePath", toDelete);
455
667
  }
456
668
  }
457
- branchFilesCache;
669
+ branchVersionsCache;
458
670
  /**
459
- * Get the set of file paths indexed on the current branch (cached).
671
+ * Map of filePath -> file-version hash for the current branch (cached).
672
+ * Used to scope search to the exact file versions this branch references,
673
+ * so stale chunks from another version of the same path are excluded.
460
674
  */
461
- async getBranchFiles() {
462
- if (this.branchFilesCache)
463
- return this.branchFilesCache;
675
+ async getBranchFileVersions() {
676
+ if (this.branchVersionsCache)
677
+ return this.branchVersionsCache;
464
678
  const t = await this.openFiles();
465
679
  if (!t)
466
680
  return undefined;
@@ -468,14 +682,18 @@ export class Store {
468
682
  const rows = await t
469
683
  .query()
470
684
  .where(`branch = '${escaped}'`)
471
- .select(["filePath"])
685
+ .select(["filePath", "fileHash"])
472
686
  .toArray();
473
- this.branchFilesCache = new Set(rows.map((r) => r.filePath));
474
- return this.branchFilesCache;
687
+ const map = new Map();
688
+ for (const r of rows) {
689
+ map.set(r.filePath, r.fileHash);
690
+ }
691
+ this.branchVersionsCache = map;
692
+ return this.branchVersionsCache;
475
693
  }
476
694
  /** Invalidate the branch files cache (call after index/import). */
477
695
  invalidateBranchFilesCache() {
478
- this.branchFilesCache = undefined;
696
+ this.branchVersionsCache = undefined;
479
697
  }
480
698
  async search(queryVector, limit = 25, filePrefix, typeFilter,
481
699
  /** Pass false to skip branch scoping (e.g. for cross-project search). */
@@ -484,9 +702,12 @@ export class Store {
484
702
  if (!t) {
485
703
  throw new Error("No index found. Run `lmgrep index` first.");
486
704
  }
487
- const branchFiles = scopeToBranch ? await this.getBranchFiles() : undefined;
488
- // Over-fetch when branch-filtering since some results will be discarded
489
- const fetchLimit = branchFiles ? limit * 3 : limit;
705
+ const branchVersions = scopeToBranch
706
+ ? await this.getBranchFileVersions()
707
+ : undefined;
708
+ // Over-fetch: branch/version filtering and dedup both discard rows, so
709
+ // pull extra to still return `limit` distinct results.
710
+ const fetchLimit = branchVersions ? limit * 3 : limit * 2;
490
711
  let query = t.search(queryVector).limit(fetchLimit);
491
712
  const conditions = [];
492
713
  if (filePrefix) {
@@ -501,6 +722,48 @@ export class Store {
501
722
  }
502
723
  const results = await query.toArray();
503
724
  let mapped = results.map((r) => ({
725
+ id: r.id,
726
+ filePath: r.filePath,
727
+ startLine: r.startLine,
728
+ endLine: r.endLine,
729
+ type: r.type,
730
+ name: r.name,
731
+ content: r.content,
732
+ context: r.context,
733
+ score: r._distance != null ? 1 - r._distance : 0,
734
+ fileHash: r.fileHash ?? "",
735
+ }));
736
+ if (branchVersions) {
737
+ mapped = mapped.filter((r) => {
738
+ const want = branchVersions.get(r.filePath);
739
+ // "" fileHash = legacy chunk, treated as wildcard.
740
+ return want !== undefined && (r.fileHash === "" || r.fileHash === want);
741
+ });
742
+ }
743
+ return dedupeRows(mapped)
744
+ .slice(0, limit)
745
+ .map(({ id, fileHash, ...rest }) => rest);
746
+ }
747
+ /**
748
+ * Same as search() but also returns each chunk's vector — needed for
749
+ * client-side clustering (facet command).
750
+ */
751
+ async searchWithVectors(queryVector, limit = 25, filePrefix, scopeToBranch = true) {
752
+ const t = await this.openChunks();
753
+ if (!t) {
754
+ throw new Error("No index found. Run `lmgrep index` first.");
755
+ }
756
+ const branchVersions = scopeToBranch
757
+ ? await this.getBranchFileVersions()
758
+ : undefined;
759
+ const fetchLimit = branchVersions ? limit * 3 : limit * 2;
760
+ let query = t.search(queryVector).limit(fetchLimit);
761
+ if (filePrefix) {
762
+ query = query.where(`filePath LIKE '${filePrefix.replace(/'/g, "''")}%'`);
763
+ }
764
+ const results = await query.toArray();
765
+ let mapped = results.map((r) => ({
766
+ id: r.id,
504
767
  filePath: r.filePath,
505
768
  startLine: r.startLine,
506
769
  endLine: r.endLine,
@@ -509,11 +772,43 @@ export class Store {
509
772
  content: r.content,
510
773
  context: r.context,
511
774
  score: r._distance != null ? 1 - r._distance : 0,
775
+ fileHash: r.fileHash ?? "",
776
+ vector: Array.from(r.vector),
512
777
  }));
513
- if (branchFiles) {
514
- mapped = mapped.filter((r) => branchFiles.has(r.filePath));
778
+ if (branchVersions) {
779
+ mapped = mapped.filter((r) => {
780
+ const want = branchVersions.get(r.filePath);
781
+ return want !== undefined && (r.fileHash === "" || r.fileHash === want);
782
+ });
515
783
  }
516
- return mapped.slice(0, limit);
784
+ return dedupeRows(mapped)
785
+ .slice(0, limit)
786
+ .map(({ fileHash, ...rest }) => rest);
787
+ }
788
+ /**
789
+ * Fetch chunks (with vectors) by id. Used to rehydrate faceting sessions.
790
+ * Missing ids are silently dropped.
791
+ */
792
+ async getChunksByIds(ids) {
793
+ if (ids.length === 0)
794
+ return [];
795
+ const t = await this.openChunks();
796
+ if (!t)
797
+ return [];
798
+ const escaped = ids.map((i) => `'${i.replace(/'/g, "''")}'`).join(",");
799
+ const rows = await t.query().where(`id IN (${escaped})`).toArray();
800
+ return rows.map((r) => ({
801
+ id: r.id,
802
+ filePath: r.filePath,
803
+ startLine: r.startLine,
804
+ endLine: r.endLine,
805
+ type: r.type,
806
+ name: r.name,
807
+ content: r.content,
808
+ context: r.context,
809
+ score: 0,
810
+ vector: Array.from(r.vector),
811
+ }));
517
812
  }
518
813
  async getIndexedFiles() {
519
814
  const t = await this.openChunks();
@@ -567,6 +862,25 @@ export class Store {
567
862
  return 0;
568
863
  return await t.countRows();
569
864
  }
865
+ /**
866
+ * Stream chunk texts (name + content only) in batches. Used by vocab
867
+ * backfill to avoid loading all chunks into memory at once.
868
+ */
869
+ async *streamChunkTexts(batchSize = 1000) {
870
+ const t = await this.openChunks();
871
+ if (!t)
872
+ return;
873
+ const stream = await t
874
+ .query()
875
+ .select(["name", "content"])
876
+ .toArray();
877
+ for (let i = 0; i < stream.length; i += batchSize) {
878
+ yield stream.slice(i, i + batchSize).map((r) => ({
879
+ name: r.name ?? "",
880
+ content: r.content ?? "",
881
+ }));
882
+ }
883
+ }
570
884
  // --- File hashes (change detection) ---
571
885
  async getFileHashes() {
572
886
  const t = await this.openFiles();
@@ -674,10 +988,79 @@ export class Store {
674
988
  content: r.content,
675
989
  context: r.context,
676
990
  hash: r.hash,
991
+ fileHash: r.fileHash ?? "",
677
992
  vector: Array.from(r.vector),
678
993
  }));
679
994
  }
680
995
  }
996
+ /**
997
+ * Remove redundant rows from the chunks table:
998
+ * - exact duplicate ids (identical rows from concurrent unlocked indexing)
999
+ * - orphaned versions: chunks whose fileHash is no longer referenced by any
1000
+ * branch's manifest (left behind when a file was edited on one branch
1001
+ * while another branch still pointed at the old path). Legacy chunks with
1002
+ * an empty fileHash are kept.
1003
+ * Rewrites the table from the surviving rows. Loads chunks into memory in
1004
+ * batches, so this is a maintenance operation, not a hot path.
1005
+ */
1006
+ async dedupeChunks() {
1007
+ const t = await this.openChunks();
1008
+ if (!t)
1009
+ return { before: 0, after: 0, duplicateIds: 0, staleVersions: 0 };
1010
+ const before = await t.countRows();
1011
+ // Which (filePath, fileHash) pairs any branch still references.
1012
+ const refs = new Map();
1013
+ for (const e of await this.getAllFileEntries()) {
1014
+ const set = refs.get(e.filePath) ?? new Set();
1015
+ set.add(e.fileHash);
1016
+ refs.set(e.filePath, set);
1017
+ }
1018
+ const kept = [];
1019
+ const seenIds = new Set();
1020
+ let duplicateIds = 0;
1021
+ let staleVersions = 0;
1022
+ for await (const batch of this.streamAllChunks(2000)) {
1023
+ for (const r of batch) {
1024
+ const id = r.id;
1025
+ if (seenIds.has(id)) {
1026
+ duplicateIds++;
1027
+ continue;
1028
+ }
1029
+ seenIds.add(id);
1030
+ const filePath = r.filePath;
1031
+ const fileHash = r.fileHash ?? "";
1032
+ if (fileHash !== "" && !refs.get(filePath)?.has(fileHash)) {
1033
+ staleVersions++;
1034
+ continue;
1035
+ }
1036
+ kept.push({
1037
+ id,
1038
+ filePath,
1039
+ startLine: r.startLine,
1040
+ endLine: r.endLine,
1041
+ type: r.type,
1042
+ name: r.name,
1043
+ content: r.content,
1044
+ context: r.context,
1045
+ hash: r.hash,
1046
+ fileHash,
1047
+ vector: r.vector,
1048
+ });
1049
+ }
1050
+ }
1051
+ if (duplicateIds + staleVersions === 0) {
1052
+ return { before, after: before, duplicateIds, staleVersions };
1053
+ }
1054
+ // Rewrite the table from the survivors.
1055
+ const conn = await this.connection();
1056
+ if ((await conn.tableNames()).includes(CHUNKS_TABLE)) {
1057
+ await conn.dropTable(CHUNKS_TABLE);
1058
+ }
1059
+ this.chunksTable = undefined;
1060
+ if (kept.length > 0)
1061
+ await this.addChunks(kept);
1062
+ return { before, after: kept.length, duplicateIds, staleVersions };
1063
+ }
681
1064
  async getAllFileEntries() {
682
1065
  const t = await this.openFiles();
683
1066
  if (!t)
@@ -696,8 +1079,11 @@ export class Store {
696
1079
  await conn.dropTable(CHUNKS_TABLE);
697
1080
  if (tables.includes(FILES_TABLE))
698
1081
  await conn.dropTable(FILES_TABLE);
1082
+ if (tables.includes(VOCAB_TABLE))
1083
+ await conn.dropTable(VOCAB_TABLE);
699
1084
  this.chunksTable = undefined;
700
1085
  this.filesTable = undefined;
1086
+ this.vocabTable = undefined;
701
1087
  }
702
1088
  async compact() {
703
1089
  const t = await this.openChunks();
@@ -706,6 +1092,9 @@ export class Store {
706
1092
  const f = await this.openFiles();
707
1093
  if (f)
708
1094
  await f.optimize();
1095
+ const v = await this.openVocab();
1096
+ if (v)
1097
+ await v.optimize();
709
1098
  }
710
1099
  /**
711
1100
  * Import all chunks and file hashes from another Store's database.
@@ -732,6 +1121,7 @@ export class Store {
732
1121
  content: r.content,
733
1122
  context: r.context,
734
1123
  hash: r.hash,
1124
+ fileHash: r.fileHash ?? "",
735
1125
  vector: Array.from(r.vector),
736
1126
  }));
737
1127
  const conn = await this.connection();
@@ -819,6 +1209,7 @@ export class Store {
819
1209
  async close() {
820
1210
  this.chunksTable = undefined;
821
1211
  this.filesTable = undefined;
1212
+ this.vocabTable = undefined;
822
1213
  this.db = undefined;
823
1214
  }
824
1215
  }