opencode-rag-plugin 2.12.2 → 2.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  import { normalizeFilePath } from "../core/manifest.js";
2
2
  import { normalizeFileExtensions, matchesFileExtension } from "../core/filters.js";
3
- const INDEX_VERSION = 3;
3
+ const INDEX_VERSION = 4;
4
4
  /**
5
5
  * Text indexed for keyword search: chunk content plus its LLM description when
6
6
  * one exists. Without this, keyword and vector search would see different text
@@ -75,31 +75,57 @@ export function indexPathFor(storePath) {
75
75
  /**
76
76
  * In-memory inverted keyword index with stemming, TF-IDF scoring, and optional JSON serialization.
77
77
  *
78
+ * Postings are stored CSR-style: token `t` owns the slice
79
+ * `[tokenStart[t], tokenStart[t+1])` of `postChunk`/`postFreq`, sorted by
80
+ * chunk index. Chunks have an internal numeric slot; the string id exists
81
+ * exactly once per chunk. Deletions are tombstones that a merge (run by
82
+ * `save()`) folds away by rebuilding the CSR without them.
83
+ *
78
84
  * @param storePath - Optional path to the vector store directory used for loading/saving the serialized index
79
85
  */
80
86
  export class KeywordIndex {
81
- invertedIndex = new Map();
87
+ // ---- chunk identity & metadata (chunkMap is the source of truth) ----
82
88
  chunkMap = new Map();
83
- storePath;
84
- /** filePath (normalized) → chunk IDs, for O(chunks-in-file) removal. */
89
+ /** chunk id -> numeric slot. Dead (deleted) ids are removed; slots are never reused. */
90
+ idToIdx = new Map();
91
+ /** slot -> chunk id (holes for dead slots are simply never consulted). */
92
+ idxToId = [];
93
+ /** filePath (normalized) -> slot set, for O(chunks-in-file) removal. */
85
94
  fileToIds = new Map();
86
- /** Indexed-token length per chunk, and the running total — drives BM25 length
87
- * normalization so long chunks can't dominate purely on term frequency. */
88
- docLengths = new Map();
95
+ /** Indexed-token length per slot — drives BM25 length normalization. */
96
+ docLengths = [];
89
97
  totalTokens = 0;
90
- /** The exact set of tokens indexed per chunk id, recorded at add time.
91
- * Removal deletes precisely these entries instead of re-tokenizing the
92
- * chunk: re-derivation would leave dangling inverted-index references
93
- * whenever the chunk's text drifted between indexing and removal (the
94
- * description stage mutates `chunk.description` after addChunks, or the
95
- * persisted index was written by a build whose tokenizer differed). */
96
- chunkTokens = new Map();
98
+ // ---- materialized CSR postings (snapshot as of the last merge) ----
99
+ tokNames = [];
100
+ tokIdx = new Map();
101
+ /** length tokNames.length + 1; slice bounds into postChunk/postFreq. */
102
+ tokenStart = new Uint32Array(1);
103
+ postChunk = new Uint32Array(0);
104
+ postFreq = new Uint32Array(0);
105
+ // ---- pending mutations folded into the CSR at the next merge ----
106
+ /** token -> slot -> term frequency, recorded at add time. Also plays the
107
+ * role the old chunkTokens Map played: removal and merging use these
108
+ * recorded postings rather than re-tokenizing, because the description
109
+ * stage mutates `chunk.description` AFTER addChunks ran and re-derivation
110
+ * would leak tokens that were never indexed. */
111
+ addPostings = new Map();
112
+ /** slots deleted since the last merge — filtered by search and save. */
113
+ dels = new Set();
114
+ /** Set by a v3-format load; the plugin re-saves once to migrate. */
115
+ migratedFromV3 = false;
97
116
  constructor(storePath) {
98
117
  this.storePath = storePath;
99
118
  }
119
+ storePath;
100
120
  addChunks(chunks) {
101
121
  for (const chunk of chunks) {
102
122
  const id = chunk.id;
123
+ let idx = this.idToIdx.get(id);
124
+ if (idx === undefined) {
125
+ idx = this.idxToId.length;
126
+ this.idxToId.push(id);
127
+ this.idToIdx.set(id, idx);
128
+ }
103
129
  this.chunkMap.set(id, chunk);
104
130
  const fileKey = normalizeFilePath(chunk.metadata.filePath);
105
131
  let ids = this.fileToIds.get(fileKey);
@@ -107,73 +133,88 @@ export class KeywordIndex {
107
133
  ids = new Set();
108
134
  this.fileToIds.set(fileKey, ids);
109
135
  }
110
- ids.add(id);
136
+ ids.add(idx);
111
137
  const tokens = tokenize(indexableText(chunk));
112
- this.docLengths.set(id, tokens.length);
138
+ this.docLengths[idx] = tokens.length;
113
139
  this.totalTokens += tokens.length;
114
- this.chunkTokens.set(id, new Set(tokens));
115
140
  for (const token of tokens) {
116
- let docs = this.invertedIndex.get(token);
117
- if (!docs) {
118
- docs = new Map();
119
- this.invertedIndex.set(token, docs);
141
+ let postings = this.addPostings.get(token);
142
+ if (!postings) {
143
+ postings = new Map();
144
+ this.addPostings.set(token, postings);
120
145
  }
121
- docs.set(id, (docs.get(id) ?? 0) + 1);
146
+ postings.set(idx, (postings.get(idx) ?? 0) + 1);
122
147
  }
123
148
  }
124
149
  }
125
150
  getMatchedTerms(query, chunkId) {
151
+ const idx = this.idToIdx.get(chunkId);
152
+ if (idx === undefined || this.dels.has(idx))
153
+ return [];
126
154
  const queryTokens = tokenize(query);
127
155
  const matched = [];
128
156
  for (const token of queryTokens) {
129
- const docs = this.invertedIndex.get(token);
130
- if (docs && docs.has(chunkId)) {
157
+ const ti = this.tokIdx.get(token);
158
+ if (ti !== undefined && this.csrContains(ti, idx)) {
159
+ matched.push(token);
160
+ continue;
161
+ }
162
+ if (this.addPostings.get(token)?.has(idx) === true) {
131
163
  matched.push(token);
132
164
  }
133
165
  }
134
166
  return matched;
135
167
  }
168
+ /** Binary search the sorted CSR slice of token `ti` for slot `idx`. */
169
+ csrContains(ti, idx) {
170
+ let lo = this.tokenStart[ti];
171
+ let hi = this.tokenStart[ti + 1];
172
+ while (lo < hi) {
173
+ const mid = (lo + hi) >>> 1;
174
+ const v = this.postChunk[mid];
175
+ if (v === idx)
176
+ return true;
177
+ if (v < idx)
178
+ lo = mid + 1;
179
+ else
180
+ hi = mid;
181
+ }
182
+ return false;
183
+ }
136
184
  removeByFilePath(filePath) {
137
185
  const fileKey = normalizeFilePath(filePath);
138
- // Fast path: per-file id index. Falls back to a full chunkMap scan when
139
- // the path is not tracked (entries loaded from an older persisted index).
140
- let idsToRemove;
186
+ // Fast path: per-file slot set. Falls back to a full chunkMap scan when
187
+ // the path is not tracked (should not happen for loaded indexes — load()
188
+ // rebuilds fileToIds — but kept for parity with the old behavior).
189
+ let slots;
141
190
  const tracked = this.fileToIds.get(fileKey);
142
191
  if (tracked) {
143
- idsToRemove = [...tracked];
192
+ slots = [...tracked];
144
193
  this.fileToIds.delete(fileKey);
145
194
  }
146
195
  else {
147
- idsToRemove = [];
196
+ slots = [];
148
197
  for (const [id, chunk] of this.chunkMap) {
149
198
  if (normalizeFilePath(chunk.metadata.filePath) === fileKey) {
150
- idsToRemove.push(id);
199
+ const idx = this.idToIdx.get(id);
200
+ if (idx !== undefined)
201
+ slots.push(idx);
151
202
  }
152
203
  }
153
204
  }
154
- for (const id of idsToRemove) {
155
- const chunk = this.chunkMap.get(id);
156
- if (chunk) {
157
- // Delete exactly the entries addChunks created. The recorded token set
158
- // is authoritative; re-derive only when the entry predates exact
159
- // tracking (loaded indexes always rebuild chunkTokens in load()).
160
- const tokens = this.chunkTokens.get(id) ?? new Set(tokenize(indexableText(chunk)));
161
- for (const token of tokens) {
162
- const docs = this.invertedIndex.get(token);
163
- if (docs) {
164
- docs.delete(id);
165
- if (docs.size === 0) {
166
- this.invertedIndex.delete(token);
167
- }
168
- }
169
- }
170
- this.chunkMap.delete(id);
171
- this.chunkTokens.delete(id);
172
- const len = this.docLengths.get(id);
173
- if (len !== undefined) {
174
- this.totalTokens -= len;
175
- this.docLengths.delete(id);
176
- }
205
+ for (const idx of slots) {
206
+ const id = this.idxToId[idx];
207
+ if (id === undefined)
208
+ continue;
209
+ if (!this.chunkMap.has(id))
210
+ continue;
211
+ this.chunkMap.delete(id);
212
+ this.idToIdx.delete(id);
213
+ this.dels.add(idx);
214
+ const len = this.docLengths[idx];
215
+ if (len !== undefined) {
216
+ this.totalTokens -= len;
217
+ delete this.docLengths[idx];
177
218
  }
178
219
  }
179
220
  }
@@ -190,89 +231,203 @@ export class KeywordIndex {
190
231
  // merely repeats a term (or is very long) can't outrank a focused match.
191
232
  const K1 = 1.2;
192
233
  const B = 0.75;
234
+ const hasPending = this.dels.size > 0 || this.addPostings.size > 0;
193
235
  for (const token of queryTokens) {
194
- const docs = this.invertedIndex.get(token);
195
- if (!docs)
236
+ const ti = this.tokIdx.get(token);
237
+ const start = ti !== undefined ? this.tokenStart[ti] : 0;
238
+ const end = ti !== undefined ? this.tokenStart[ti + 1] : 0;
239
+ const added = this.addPostings.get(token);
240
+ // Fast path: merged CSR only, no tombstones and no pending additions —
241
+ // score directly off the typed arrays.
242
+ if (!hasPending && ti !== undefined && !added) {
243
+ const df = end - start;
244
+ if (df === 0)
245
+ continue;
246
+ const idf = Math.log(1 + (totalChunks - df + 0.5) / (df + 0.5));
247
+ for (let p = start; p < end; p++) {
248
+ const idx = this.postChunk[p];
249
+ const freq = this.postFreq[p];
250
+ const docLength = this.docLengths[idx] ?? avgDocLength;
251
+ const lengthNorm = avgDocLength > 0 ? 1 - B + B * (docLength / avgDocLength) : 1;
252
+ const tf = (freq * (K1 + 1)) / (freq + K1 * lengthNorm);
253
+ scores.set(idx, (scores.get(idx) ?? 0) + idf * tf);
254
+ }
255
+ continue;
256
+ }
257
+ // Merged view of CSR slice + pending additions, deduplicated by slot and
258
+ // minus tombstones. df counts the same docs that are scored below.
259
+ const entries = new Map();
260
+ for (let p = start; p < end; p++) {
261
+ const idx = this.postChunk[p];
262
+ if (this.dels.has(idx))
263
+ continue;
264
+ entries.set(idx, this.postFreq[p]);
265
+ }
266
+ if (added) {
267
+ for (const [idx, freq] of added) {
268
+ if (this.dels.has(idx))
269
+ continue;
270
+ entries.set(idx, (entries.get(idx) ?? 0) + freq);
271
+ }
272
+ }
273
+ const df = entries.size;
274
+ if (df === 0)
196
275
  continue;
197
- const df = docs.size;
198
276
  const idf = Math.log(1 + (totalChunks - df + 0.5) / (df + 0.5));
199
- for (const [chunkId, freq] of docs) {
200
- const docLength = this.docLengths.get(chunkId) ?? avgDocLength;
201
- const lengthNorm = avgDocLength > 0
202
- ? 1 - B + B * (docLength / avgDocLength)
203
- : 1;
277
+ for (const [idx, freq] of entries) {
278
+ const docLength = this.docLengths[idx] ?? avgDocLength;
279
+ const lengthNorm = avgDocLength > 0 ? 1 - B + B * (docLength / avgDocLength) : 1;
204
280
  const tf = (freq * (K1 + 1)) / (freq + K1 * lengthNorm);
205
- scores.set(chunkId, (scores.get(chunkId) ?? 0) + idf * tf);
281
+ scores.set(idx, (scores.get(idx) ?? 0) + idf * tf);
206
282
  }
207
283
  }
208
284
  const dangling = [];
209
285
  const sorted = [...scores.entries()]
210
- .filter(([id]) => {
211
- const chunk = this.chunkMap.get(id);
286
+ .filter(([idx]) => {
287
+ const id = this.idxToId[idx];
288
+ const chunk = id !== undefined ? this.chunkMap.get(id) : undefined;
212
289
  if (!chunk) {
213
- // The inverted index references a chunk id that no longer exists.
214
- // This is the "undefined is not an object (evaluating
215
- // 'chunk.metadata')" crash site: a removal that missed tokens
216
- // (index persisted by an older build, interrupted re-index) leaves
217
- // phantom ids behind. Skip and prune them so a corrupt index
218
- // self-heals instead of failing every filtered search.
219
- dangling.push(id);
290
+ // The posting list references a slot whose chunk is gone. This is
291
+ // the "undefined is not an object (evaluating 'chunk.metadata')"
292
+ // crash site: a removal that raced a persisted index or an
293
+ // interrupted re-index leaves phantom slots behind. Skip and
294
+ // tombstone them so a corrupt index self-heals instead of failing
295
+ // every filtered search.
296
+ dangling.push(idx);
220
297
  return false;
221
298
  }
222
299
  return matchesFilter(chunk, filter);
223
300
  })
224
301
  .sort((a, b) => b[1] - a[1])
225
302
  .slice(0, topK);
226
- if (dangling.length > 0)
227
- this.pruneDangling(dangling);
228
- return sorted.map(([chunkId, score]) => {
229
- const chunk = this.chunkMap.get(chunkId);
230
- return { chunk, score };
231
- });
303
+ if (dangling.length > 0) {
304
+ for (const idx of dangling)
305
+ this.dels.add(idx);
306
+ }
307
+ return sorted.map(([idx, score]) => ({
308
+ chunk: this.chunkMap.get(this.idxToId[idx]),
309
+ score,
310
+ }));
232
311
  }
233
- /** Remove every inverted-index reference to chunk ids that no longer have a
234
- * chunkMap entry. Uses the recorded token set when available (fast path,
235
- * always taken for loaded indexes) and falls back to a full scan. */
236
- pruneDangling(ids) {
237
- for (const id of ids) {
238
- const tokens = this.chunkTokens.get(id);
239
- if (tokens) {
240
- for (const token of tokens) {
241
- const docs = this.invertedIndex.get(token);
242
- if (docs) {
243
- docs.delete(id);
244
- if (docs.size === 0)
245
- this.invertedIndex.delete(token);
246
- }
312
+ /** Fold pending additions and tombstones into the CSR. Empty token slices
313
+ * are dropped so the token table does not grow unboundedly across merges. */
314
+ merge() {
315
+ if (this.dels.size === 0 && this.addPostings.size === 0)
316
+ return;
317
+ const union = this.tokNames.slice();
318
+ for (const token of this.addPostings.keys()) {
319
+ if (!this.tokIdx.has(token))
320
+ union.push(token);
321
+ }
322
+ const keptTokens = [];
323
+ const perToken = [];
324
+ let total = 0;
325
+ for (const token of union) {
326
+ const tiOld = this.tokIdx.get(token);
327
+ const oldList = [];
328
+ if (tiOld !== undefined) {
329
+ const start = this.tokenStart[tiOld];
330
+ const end = this.tokenStart[tiOld + 1];
331
+ for (let p = start; p < end; p++) {
332
+ const idx = this.postChunk[p];
333
+ if (this.dels.has(idx))
334
+ continue;
335
+ oldList.push([idx, this.postFreq[p]]);
247
336
  }
248
337
  }
249
- else {
250
- for (const [token, docs] of this.invertedIndex) {
251
- if (docs.delete(id) && docs.size === 0)
252
- this.invertedIndex.delete(token);
338
+ const added = this.addPostings.get(token);
339
+ const addList = [];
340
+ if (added) {
341
+ for (const [idx, freq] of added) {
342
+ if (this.dels.has(idx))
343
+ continue;
344
+ addList.push([idx, freq]);
345
+ }
346
+ }
347
+ if (oldList.length === 0 && addList.length === 0)
348
+ continue;
349
+ // Both sources sorted by slot, then a linear merge; a slot present in
350
+ // both (re-added id without a removal) sums its frequencies, matching
351
+ // what search-time scoring computes.
352
+ oldList.sort((a, b) => a[0] - b[0]);
353
+ addList.sort((a, b) => a[0] - b[0]);
354
+ const chunks = [];
355
+ const freqs = [];
356
+ let i = 0;
357
+ let j = 0;
358
+ while (i < oldList.length || j < addList.length) {
359
+ if (j >= addList.length || (i < oldList.length && oldList[i][0] < addList[j][0])) {
360
+ chunks.push(oldList[i][0]);
361
+ freqs.push(oldList[i][1]);
362
+ i++;
363
+ }
364
+ else if (i >= oldList.length || addList[j][0] < oldList[i][0]) {
365
+ chunks.push(addList[j][0]);
366
+ freqs.push(addList[j][1]);
367
+ j++;
368
+ }
369
+ else {
370
+ chunks.push(oldList[i][0]);
371
+ freqs.push(oldList[i][1] + addList[j][1]);
372
+ i++;
373
+ j++;
253
374
  }
254
375
  }
255
- this.chunkTokens.delete(id);
256
- this.docLengths.delete(id);
376
+ keptTokens.push(token);
377
+ perToken.push({ chunks, freqs });
378
+ total += chunks.length;
379
+ }
380
+ this.tokNames = keptTokens;
381
+ this.tokIdx = new Map();
382
+ this.tokenStart = new Uint32Array(keptTokens.length + 1);
383
+ this.postChunk = new Uint32Array(total);
384
+ this.postFreq = new Uint32Array(total);
385
+ let cursor = 0;
386
+ for (let t = 0; t < keptTokens.length; t++) {
387
+ this.tokenStart[t] = cursor;
388
+ this.tokIdx.set(keptTokens[t], t);
389
+ const entry = perToken[t];
390
+ this.postChunk.set(entry.chunks, cursor);
391
+ this.postFreq.set(entry.freqs, cursor);
392
+ cursor += entry.chunks.length;
257
393
  }
394
+ this.tokenStart[keptTokens.length] = cursor;
395
+ this.dels.clear();
396
+ this.addPostings.clear();
258
397
  }
259
398
  close() {
260
- this.invertedIndex.clear();
399
+ this.tokNames = [];
400
+ this.tokIdx.clear();
401
+ this.tokenStart = new Uint32Array(1);
402
+ this.postChunk = new Uint32Array(0);
403
+ this.postFreq = new Uint32Array(0);
261
404
  this.chunkMap.clear();
405
+ this.idToIdx.clear();
262
406
  this.fileToIds.clear();
263
- this.chunkTokens.clear();
407
+ this.addPostings.clear();
408
+ this.dels.clear();
264
409
  }
265
410
  clear() {
266
- this.invertedIndex.clear();
267
- this.chunkMap.clear();
268
- this.fileToIds.clear();
269
- this.chunkTokens.clear();
270
- this.docLengths.clear();
411
+ this.close();
412
+ this.idxToId = [];
413
+ this.docLengths = [];
271
414
  this.totalTokens = 0;
272
415
  }
273
416
  count() {
274
417
  return this.chunkMap.size;
275
418
  }
419
+ /** Total postings (merged + pending). Diagnostic/logging only. */
420
+ postingCount() {
421
+ let n = this.postChunk.length;
422
+ for (const postings of this.addPostings.values())
423
+ n += postings.size;
424
+ return n;
425
+ }
426
+ /** True when this instance was loaded from the legacy v3 (Map-era) file
427
+ * format; the caller should persist once to migrate to v4. */
428
+ needsFormatMigration() {
429
+ return this.migratedFromV3;
430
+ }
276
431
  hasChunk(id) {
277
432
  return this.chunkMap.has(id);
278
433
  }
@@ -292,13 +447,41 @@ export class KeywordIndex {
292
447
  const effectiveStorePath = storePath ?? this.storePath;
293
448
  if (!effectiveStorePath)
294
449
  return;
450
+ this.merge();
295
451
  const targetPath = indexPathFor(effectiveStorePath);
296
452
  const { mkdir, writeFile } = await import("node:fs/promises");
297
453
  const path = await import("node:path");
298
454
  await mkdir(path.dirname(targetPath), { recursive: true });
455
+ // Alive slots, in ascending slot order, renumbered for the serialized
456
+ // posting arrays. idToIdx only holds live chunks.
457
+ const aliveSlots = [...this.idToIdx.values()].sort((a, b) => a - b);
458
+ const ids = new Array(aliveSlots.length);
459
+ const renumber = new Int32Array(this.idxToId.length).fill(-1);
460
+ for (let k = 0; k < aliveSlots.length; k++) {
461
+ const slot = aliveSlots[k];
462
+ ids[k] = this.idxToId[slot];
463
+ renumber[slot] = k;
464
+ }
465
+ const postings = [];
466
+ for (let t = 0; t < this.tokNames.length; t++) {
467
+ const start = this.tokenStart[t];
468
+ const end = this.tokenStart[t + 1];
469
+ if (start === end)
470
+ continue;
471
+ const flat = [];
472
+ for (let p = start; p < end; p++) {
473
+ const chunkIdx = renumber[this.postChunk[p]];
474
+ if (chunkIdx < 0)
475
+ continue; // dead slot — defensive, merge() already dropped them
476
+ flat.push(chunkIdx, this.postFreq[p]);
477
+ }
478
+ if (flat.length === 0)
479
+ continue;
480
+ postings.push([t, flat]);
481
+ }
299
482
  const serialized = {
300
483
  version: INDEX_VERSION,
301
- tokens: [...this.invertedIndex.entries()].map(([token, docs]) => [token, [...docs.entries()]]),
484
+ ids,
302
485
  chunkMap: Object.fromEntries([...this.chunkMap.entries()].map(([id, chunk]) => [
303
486
  id,
304
487
  {
@@ -315,8 +498,11 @@ export class KeywordIndex {
315
498
  tags: chunk.metadata.tags ? JSON.stringify(chunk.metadata.tags) : "",
316
499
  },
317
500
  ])),
501
+ tokens: this.tokNames,
502
+ postings,
318
503
  };
319
504
  await writeFile(targetPath, JSON.stringify(serialized), "utf-8");
505
+ this.migratedFromV3 = false;
320
506
  }
321
507
  static async load(storePath) {
322
508
  const targetPath = indexPathFor(storePath);
@@ -335,73 +521,158 @@ export class KeywordIndex {
335
521
  catch {
336
522
  return new KeywordIndex(storePath);
337
523
  }
524
+ if (parsed.version === 3) {
525
+ return KeywordIndex.loadV3(storePath, parsed);
526
+ }
338
527
  if (parsed.version !== INDEX_VERSION) {
339
528
  return new KeywordIndex(storePath);
340
529
  }
341
530
  const index = new KeywordIndex(storePath);
342
- for (const [token, docs] of parsed.tokens) {
343
- const docMap = new Map(docs);
344
- index.invertedIndex.set(token, docMap);
345
- }
346
- // Rebuild the exact per-chunk token sets from the persisted inverted
347
- // index, so removal is exact even for indexes written by older builds
348
- // (whose tokenizer or indexableText may have differed).
349
- for (const [token, docs] of index.invertedIndex) {
350
- for (const id of docs.keys()) {
351
- let tokens = index.chunkTokens.get(id);
352
- if (!tokens) {
353
- tokens = new Set();
354
- index.chunkTokens.set(id, tokens);
355
- }
356
- tokens.add(token);
357
- }
531
+ // Chunk slots follow the serialized ids array; ids missing from chunkMap
532
+ // stay a slot so dangling-posting self-healing behaves as it did with v3.
533
+ for (let idx = 0; idx < parsed.ids.length; idx++) {
534
+ const id = parsed.ids[idx];
535
+ index.idxToId.push(id);
536
+ index.idToIdx.set(id, idx);
358
537
  }
359
538
  for (const [id, data] of Object.entries(parsed.chunkMap)) {
360
- let tags;
361
- try {
362
- if (data.tags)
363
- tags = JSON.parse(data.tags);
539
+ const chunk = decodeChunk(id, data);
540
+ if (!chunk)
541
+ continue;
542
+ index.chunkMap.set(id, chunk);
543
+ const fileKey = normalizeFilePath(chunk.metadata.filePath);
544
+ let ids = index.fileToIds.get(fileKey);
545
+ if (!ids) {
546
+ ids = new Set();
547
+ index.fileToIds.set(fileKey, ids);
364
548
  }
365
- catch {
366
- tags = undefined;
549
+ const idx = index.idToIdx.get(id);
550
+ if (idx !== undefined)
551
+ ids.add(idx);
552
+ }
553
+ index.tokNames = parsed.tokens.slice();
554
+ index.tokIdx = new Map();
555
+ for (let t = 0; t < index.tokNames.length; t++) {
556
+ index.tokIdx.set(index.tokNames[t], t);
557
+ }
558
+ // Flatten [tokenIdx, flat] pairs into CSR typed arrays. save() writes
559
+ // token-ordered, deduplicated entries; sort + filter defensively anyway
560
+ // so a hand-edited or future-written file can never corrupt the CSR.
561
+ const posts = parsed.postings
562
+ .filter(([ti]) => ti >= 0 && ti < index.tokNames.length)
563
+ .sort((a, b) => a[0] - b[0])
564
+ .filter((e, i, arr) => i === 0 || e[0] !== arr[i - 1][0]);
565
+ let total = 0;
566
+ for (const [, flat] of posts)
567
+ total += Math.floor(flat.length / 2);
568
+ const tokenStart = new Uint32Array(index.tokNames.length + 1);
569
+ const postChunk = new Uint32Array(total);
570
+ const postFreq = new Uint32Array(total);
571
+ // Occurrence counters for BM25 docLengths (one occurrence per unique
572
+ // token per chunk — tokenize() dedupes per chunk, so counting slices is
573
+ // exactly the number of indexed tokens per chunk).
574
+ const docLen = new Array(index.idxToId.length).fill(0);
575
+ let t = 0;
576
+ let cursor = 0;
577
+ for (const [tokenIdx, flat] of posts) {
578
+ while (t < tokenIdx) {
579
+ tokenStart[t] = cursor; // token with no postings: empty slice
580
+ t++;
367
581
  }
368
- index.chunkMap.set(id, {
369
- id: data.id,
370
- content: data.content,
371
- description: data.description ?? "",
372
- metadata: {
373
- filePath: data.filePath,
374
- startLine: data.startLine,
375
- endLine: data.endLine,
376
- language: data.language,
377
- kind: data.kind || undefined,
378
- role: data.role === "source" || data.role === "test" || data.role === "doc" ? data.role : undefined,
379
- quirkType: data.quirkType || undefined,
380
- tags,
381
- },
382
- });
383
- }
384
- // Recompute BM25 length statistics from the loaded chunk map. The persisted
385
- // inverted index is authoritative for term frequencies; lengths are cheap
386
- // to derive and this keeps the serialized format lean.
387
- for (const [id, chunk] of index.chunkMap) {
388
- // Prefer the exact token count from the inverted index over
389
- // re-tokenizing — re-tokenization can drift from what was indexed.
390
- const recorded = index.chunkTokens.get(id);
391
- const len = recorded ? recorded.size : tokenize(indexableText(chunk)).length;
392
- index.docLengths.set(id, len);
393
- index.totalTokens += len;
394
- }
395
- // Rebuild the per-file id index from the loaded chunk map so removals
396
- // work even for entries persisted by older versions.
397
- for (const [id, chunk] of index.chunkMap) {
582
+ tokenStart[tokenIdx] = cursor;
583
+ for (let p = 0; p + 1 < flat.length; p += 2) {
584
+ const idx = flat[p];
585
+ postChunk[cursor] = idx;
586
+ postFreq[cursor] = flat[p + 1];
587
+ cursor++;
588
+ if (idx < docLen.length)
589
+ docLen[idx] = (docLen[idx] ?? 0) + 1;
590
+ }
591
+ t = tokenIdx + 1;
592
+ }
593
+ while (t <= index.tokNames.length) {
594
+ tokenStart[t] = cursor;
595
+ t++;
596
+ }
597
+ index.tokenStart = tokenStart;
598
+ index.postChunk = postChunk;
599
+ index.postFreq = postFreq;
600
+ for (let idx = 0; idx < docLen.length; idx++) {
601
+ const len = docLen[idx] ?? 0;
602
+ if (len > 0) {
603
+ index.docLengths[idx] = len;
604
+ index.totalTokens += len;
605
+ }
606
+ }
607
+ return index;
608
+ }
609
+ /** Read the legacy Map-era format and build the CSR representation.
610
+ * Transient parse peaks exist for this ONE read; the plugin migrates to v4
611
+ * immediately after (needsFormatMigration) so later boots are cheap. */
612
+ static loadV3(storePath, parsed) {
613
+ const index = new KeywordIndex(storePath);
614
+ index.migratedFromV3 = true;
615
+ for (const [id, data] of Object.entries(parsed.chunkMap)) {
616
+ const chunk = decodeChunk(id, data);
617
+ if (!chunk)
618
+ continue;
619
+ const idx = index.idxToId.length;
620
+ index.idxToId.push(id);
621
+ index.idToIdx.set(id, idx);
622
+ index.chunkMap.set(id, chunk);
398
623
  const fileKey = normalizeFilePath(chunk.metadata.filePath);
399
624
  let ids = index.fileToIds.get(fileKey);
400
625
  if (!ids) {
401
626
  ids = new Set();
402
627
  index.fileToIds.set(fileKey, ids);
403
628
  }
404
- ids.add(id);
629
+ ids.add(idx);
630
+ }
631
+ // Bucket postings by token, skipping ids that no longer exist (the v3
632
+ // ghost-entry case); each bucket is slot-sorted, then flattened.
633
+ const buckets = [];
634
+ const docLen = new Array(index.idxToId.length).fill(0);
635
+ for (const [token, docs] of parsed.tokens) {
636
+ const bucket = [];
637
+ for (const [id, freq] of docs) {
638
+ const idx = index.idToIdx.get(id);
639
+ if (idx === undefined)
640
+ continue;
641
+ bucket.push([idx, freq]);
642
+ docLen[idx] = (docLen[idx] ?? 0) + 1;
643
+ }
644
+ if (bucket.length === 0)
645
+ continue;
646
+ bucket.sort((a, b) => a[0] - b[0]);
647
+ buckets.push([token, bucket]);
648
+ }
649
+ index.tokNames = buckets.map(([token]) => token);
650
+ index.tokIdx = new Map();
651
+ for (let t = 0; t < index.tokNames.length; t++) {
652
+ index.tokIdx.set(index.tokNames[t], t);
653
+ }
654
+ let total = 0;
655
+ for (const [, bucket] of buckets)
656
+ total += bucket.length;
657
+ index.tokenStart = new Uint32Array(buckets.length + 1);
658
+ index.postChunk = new Uint32Array(total);
659
+ index.postFreq = new Uint32Array(total);
660
+ let cursor = 0;
661
+ for (let t = 0; t < buckets.length; t++) {
662
+ index.tokenStart[t] = cursor;
663
+ for (const [idx, freq] of buckets[t][1]) {
664
+ index.postChunk[cursor] = idx;
665
+ index.postFreq[cursor] = freq;
666
+ cursor++;
667
+ }
668
+ }
669
+ index.tokenStart[buckets.length] = cursor;
670
+ for (let idx = 0; idx < docLen.length; idx++) {
671
+ const len = docLen[idx] ?? 0;
672
+ if (len > 0) {
673
+ index.docLengths[idx] = len;
674
+ index.totalTokens += len;
675
+ }
405
676
  }
406
677
  return index;
407
678
  }
@@ -410,8 +681,35 @@ export class KeywordIndex {
410
681
  const { writeFile, mkdir } = await import("node:fs/promises");
411
682
  const path = await import("node:path");
412
683
  await mkdir(path.dirname(targetPath), { recursive: true });
413
- await writeFile(targetPath, JSON.stringify({ version: INDEX_VERSION, tokens: [], chunkMap: {} }), "utf-8");
684
+ await writeFile(targetPath, JSON.stringify({ version: INDEX_VERSION, ids: [], chunkMap: {}, tokens: [], postings: [] }), "utf-8");
685
+ }
686
+ }
687
+ function decodeChunk(id, data) {
688
+ if (!data || typeof data.filePath !== "string")
689
+ return undefined;
690
+ let tags;
691
+ try {
692
+ if (data.tags)
693
+ tags = JSON.parse(data.tags);
694
+ }
695
+ catch {
696
+ tags = undefined;
414
697
  }
698
+ return {
699
+ id: data.id ?? id,
700
+ content: data.content ?? "",
701
+ description: data.description ?? "",
702
+ metadata: {
703
+ filePath: data.filePath,
704
+ startLine: data.startLine,
705
+ endLine: data.endLine,
706
+ language: data.language,
707
+ kind: data.kind || undefined,
708
+ role: data.role === "source" || data.role === "test" || data.role === "doc" ? data.role : undefined,
709
+ quirkType: data.quirkType || undefined,
710
+ tags,
711
+ },
712
+ };
415
713
  }
416
714
  function globMatch(pattern, filePath) {
417
715
  const GLOBSTAR = "\x00GS\x00";