opencode-rag-plugin 2.12.2 → 2.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +4 -2
- package/dist/cli/browser.d.ts +20 -0
- package/dist/cli/browser.js +49 -0
- package/dist/cli/commands/setup.js +11 -0
- package/dist/cli/commands/ui.js +3 -11
- package/dist/cli/setup-ui.d.ts +38 -0
- package/dist/cli/setup-ui.js +133 -0
- package/dist/cli/types.d.ts +5 -0
- package/dist/core/config.d.ts +7 -0
- package/dist/core/config.js +4 -0
- package/dist/plugin.js +21 -1
- package/dist/retriever/keyword-index.d.ts +58 -16
- package/dist/retriever/keyword-index.js +457 -159
- package/dist/web/api.d.ts +8 -1
- package/dist/web/api.js +28 -14
- package/dist/web/config-api.d.ts +60 -0
- package/dist/web/config-api.js +614 -0
- package/dist/web/config-schema.d.ts +88 -0
- package/dist/web/config-schema.js +821 -0
- package/dist/web/server.js +14 -4
- package/dist/web/ui/assets/{ScatterPlot3D-XjXIwavM.js → ScatterPlot3D-CJpVHNv-.js} +1 -1
- package/dist/web/ui/assets/index-BitHPGOk.css +2 -0
- package/dist/web/ui/assets/index-DDZPYJ4D.js +4 -0
- package/dist/web/ui/index.html +20 -3
- package/package.json +1 -1
- package/dist/web/ui/assets/index-B-RH6fko.css +0 -2
- package/dist/web/ui/assets/index-FFPRbgLx.js +0 -4
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { normalizeFilePath } from "../core/manifest.js";
|
|
2
2
|
import { normalizeFileExtensions, matchesFileExtension } from "../core/filters.js";
|
|
3
|
-
const INDEX_VERSION =
|
|
3
|
+
const INDEX_VERSION = 4;
|
|
4
4
|
/**
|
|
5
5
|
* Text indexed for keyword search: chunk content plus its LLM description when
|
|
6
6
|
* one exists. Without this, keyword and vector search would see different text
|
|
@@ -75,31 +75,57 @@ export function indexPathFor(storePath) {
|
|
|
75
75
|
/**
|
|
76
76
|
* In-memory inverted keyword index with stemming, TF-IDF scoring, and optional JSON serialization.
|
|
77
77
|
*
|
|
78
|
+
* Postings are stored CSR-style: token `t` owns the slice
|
|
79
|
+
* `[tokenStart[t], tokenStart[t+1])` of `postChunk`/`postFreq`, sorted by
|
|
80
|
+
* chunk index. Chunks have an internal numeric slot; the string id exists
|
|
81
|
+
* exactly once per chunk. Deletions are tombstones that a merge (run by
|
|
82
|
+
* `save()`) folds away by rebuilding the CSR without them.
|
|
83
|
+
*
|
|
78
84
|
* @param storePath - Optional path to the vector store directory used for loading/saving the serialized index
|
|
79
85
|
*/
|
|
80
86
|
export class KeywordIndex {
|
|
81
|
-
|
|
87
|
+
// ---- chunk identity & metadata (chunkMap is the source of truth) ----
|
|
82
88
|
chunkMap = new Map();
|
|
83
|
-
|
|
84
|
-
|
|
89
|
+
/** chunk id -> numeric slot. Dead (deleted) ids are removed; slots are never reused. */
|
|
90
|
+
idToIdx = new Map();
|
|
91
|
+
/** slot -> chunk id (holes for dead slots are simply never consulted). */
|
|
92
|
+
idxToId = [];
|
|
93
|
+
/** filePath (normalized) -> slot set, for O(chunks-in-file) removal. */
|
|
85
94
|
fileToIds = new Map();
|
|
86
|
-
/** Indexed-token length per
|
|
87
|
-
|
|
88
|
-
docLengths = new Map();
|
|
95
|
+
/** Indexed-token length per slot — drives BM25 length normalization. */
|
|
96
|
+
docLengths = [];
|
|
89
97
|
totalTokens = 0;
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
98
|
+
// ---- materialized CSR postings (snapshot as of the last merge) ----
|
|
99
|
+
tokNames = [];
|
|
100
|
+
tokIdx = new Map();
|
|
101
|
+
/** length tokNames.length + 1; slice bounds into postChunk/postFreq. */
|
|
102
|
+
tokenStart = new Uint32Array(1);
|
|
103
|
+
postChunk = new Uint32Array(0);
|
|
104
|
+
postFreq = new Uint32Array(0);
|
|
105
|
+
// ---- pending mutations folded into the CSR at the next merge ----
|
|
106
|
+
/** token -> slot -> term frequency, recorded at add time. Also plays the
|
|
107
|
+
* role the old chunkTokens Map played: removal and merging use these
|
|
108
|
+
* recorded postings rather than re-tokenizing, because the description
|
|
109
|
+
* stage mutates `chunk.description` AFTER addChunks ran and re-derivation
|
|
110
|
+
* would leak tokens that were never indexed. */
|
|
111
|
+
addPostings = new Map();
|
|
112
|
+
/** slots deleted since the last merge — filtered by search and save. */
|
|
113
|
+
dels = new Set();
|
|
114
|
+
/** Set by a v3-format load; the plugin re-saves once to migrate. */
|
|
115
|
+
migratedFromV3 = false;
|
|
97
116
|
constructor(storePath) {
|
|
98
117
|
this.storePath = storePath;
|
|
99
118
|
}
|
|
119
|
+
storePath;
|
|
100
120
|
addChunks(chunks) {
|
|
101
121
|
for (const chunk of chunks) {
|
|
102
122
|
const id = chunk.id;
|
|
123
|
+
let idx = this.idToIdx.get(id);
|
|
124
|
+
if (idx === undefined) {
|
|
125
|
+
idx = this.idxToId.length;
|
|
126
|
+
this.idxToId.push(id);
|
|
127
|
+
this.idToIdx.set(id, idx);
|
|
128
|
+
}
|
|
103
129
|
this.chunkMap.set(id, chunk);
|
|
104
130
|
const fileKey = normalizeFilePath(chunk.metadata.filePath);
|
|
105
131
|
let ids = this.fileToIds.get(fileKey);
|
|
@@ -107,73 +133,88 @@ export class KeywordIndex {
|
|
|
107
133
|
ids = new Set();
|
|
108
134
|
this.fileToIds.set(fileKey, ids);
|
|
109
135
|
}
|
|
110
|
-
ids.add(
|
|
136
|
+
ids.add(idx);
|
|
111
137
|
const tokens = tokenize(indexableText(chunk));
|
|
112
|
-
this.docLengths
|
|
138
|
+
this.docLengths[idx] = tokens.length;
|
|
113
139
|
this.totalTokens += tokens.length;
|
|
114
|
-
this.chunkTokens.set(id, new Set(tokens));
|
|
115
140
|
for (const token of tokens) {
|
|
116
|
-
let
|
|
117
|
-
if (!
|
|
118
|
-
|
|
119
|
-
this.
|
|
141
|
+
let postings = this.addPostings.get(token);
|
|
142
|
+
if (!postings) {
|
|
143
|
+
postings = new Map();
|
|
144
|
+
this.addPostings.set(token, postings);
|
|
120
145
|
}
|
|
121
|
-
|
|
146
|
+
postings.set(idx, (postings.get(idx) ?? 0) + 1);
|
|
122
147
|
}
|
|
123
148
|
}
|
|
124
149
|
}
|
|
125
150
|
getMatchedTerms(query, chunkId) {
|
|
151
|
+
const idx = this.idToIdx.get(chunkId);
|
|
152
|
+
if (idx === undefined || this.dels.has(idx))
|
|
153
|
+
return [];
|
|
126
154
|
const queryTokens = tokenize(query);
|
|
127
155
|
const matched = [];
|
|
128
156
|
for (const token of queryTokens) {
|
|
129
|
-
const
|
|
130
|
-
if (
|
|
157
|
+
const ti = this.tokIdx.get(token);
|
|
158
|
+
if (ti !== undefined && this.csrContains(ti, idx)) {
|
|
159
|
+
matched.push(token);
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
if (this.addPostings.get(token)?.has(idx) === true) {
|
|
131
163
|
matched.push(token);
|
|
132
164
|
}
|
|
133
165
|
}
|
|
134
166
|
return matched;
|
|
135
167
|
}
|
|
168
|
+
/** Binary search the sorted CSR slice of token `ti` for slot `idx`. */
|
|
169
|
+
csrContains(ti, idx) {
|
|
170
|
+
let lo = this.tokenStart[ti];
|
|
171
|
+
let hi = this.tokenStart[ti + 1];
|
|
172
|
+
while (lo < hi) {
|
|
173
|
+
const mid = (lo + hi) >>> 1;
|
|
174
|
+
const v = this.postChunk[mid];
|
|
175
|
+
if (v === idx)
|
|
176
|
+
return true;
|
|
177
|
+
if (v < idx)
|
|
178
|
+
lo = mid + 1;
|
|
179
|
+
else
|
|
180
|
+
hi = mid;
|
|
181
|
+
}
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
136
184
|
removeByFilePath(filePath) {
|
|
137
185
|
const fileKey = normalizeFilePath(filePath);
|
|
138
|
-
// Fast path: per-file
|
|
139
|
-
// the path is not tracked (
|
|
140
|
-
|
|
186
|
+
// Fast path: per-file slot set. Falls back to a full chunkMap scan when
|
|
187
|
+
// the path is not tracked (should not happen for loaded indexes — load()
|
|
188
|
+
// rebuilds fileToIds — but kept for parity with the old behavior).
|
|
189
|
+
let slots;
|
|
141
190
|
const tracked = this.fileToIds.get(fileKey);
|
|
142
191
|
if (tracked) {
|
|
143
|
-
|
|
192
|
+
slots = [...tracked];
|
|
144
193
|
this.fileToIds.delete(fileKey);
|
|
145
194
|
}
|
|
146
195
|
else {
|
|
147
|
-
|
|
196
|
+
slots = [];
|
|
148
197
|
for (const [id, chunk] of this.chunkMap) {
|
|
149
198
|
if (normalizeFilePath(chunk.metadata.filePath) === fileKey) {
|
|
150
|
-
|
|
199
|
+
const idx = this.idToIdx.get(id);
|
|
200
|
+
if (idx !== undefined)
|
|
201
|
+
slots.push(idx);
|
|
151
202
|
}
|
|
152
203
|
}
|
|
153
204
|
}
|
|
154
|
-
for (const
|
|
155
|
-
const
|
|
156
|
-
if (
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
this.chunkMap.delete(id);
|
|
171
|
-
this.chunkTokens.delete(id);
|
|
172
|
-
const len = this.docLengths.get(id);
|
|
173
|
-
if (len !== undefined) {
|
|
174
|
-
this.totalTokens -= len;
|
|
175
|
-
this.docLengths.delete(id);
|
|
176
|
-
}
|
|
205
|
+
for (const idx of slots) {
|
|
206
|
+
const id = this.idxToId[idx];
|
|
207
|
+
if (id === undefined)
|
|
208
|
+
continue;
|
|
209
|
+
if (!this.chunkMap.has(id))
|
|
210
|
+
continue;
|
|
211
|
+
this.chunkMap.delete(id);
|
|
212
|
+
this.idToIdx.delete(id);
|
|
213
|
+
this.dels.add(idx);
|
|
214
|
+
const len = this.docLengths[idx];
|
|
215
|
+
if (len !== undefined) {
|
|
216
|
+
this.totalTokens -= len;
|
|
217
|
+
delete this.docLengths[idx];
|
|
177
218
|
}
|
|
178
219
|
}
|
|
179
220
|
}
|
|
@@ -190,89 +231,203 @@ export class KeywordIndex {
|
|
|
190
231
|
// merely repeats a term (or is very long) can't outrank a focused match.
|
|
191
232
|
const K1 = 1.2;
|
|
192
233
|
const B = 0.75;
|
|
234
|
+
const hasPending = this.dels.size > 0 || this.addPostings.size > 0;
|
|
193
235
|
for (const token of queryTokens) {
|
|
194
|
-
const
|
|
195
|
-
|
|
236
|
+
const ti = this.tokIdx.get(token);
|
|
237
|
+
const start = ti !== undefined ? this.tokenStart[ti] : 0;
|
|
238
|
+
const end = ti !== undefined ? this.tokenStart[ti + 1] : 0;
|
|
239
|
+
const added = this.addPostings.get(token);
|
|
240
|
+
// Fast path: merged CSR only, no tombstones and no pending additions —
|
|
241
|
+
// score directly off the typed arrays.
|
|
242
|
+
if (!hasPending && ti !== undefined && !added) {
|
|
243
|
+
const df = end - start;
|
|
244
|
+
if (df === 0)
|
|
245
|
+
continue;
|
|
246
|
+
const idf = Math.log(1 + (totalChunks - df + 0.5) / (df + 0.5));
|
|
247
|
+
for (let p = start; p < end; p++) {
|
|
248
|
+
const idx = this.postChunk[p];
|
|
249
|
+
const freq = this.postFreq[p];
|
|
250
|
+
const docLength = this.docLengths[idx] ?? avgDocLength;
|
|
251
|
+
const lengthNorm = avgDocLength > 0 ? 1 - B + B * (docLength / avgDocLength) : 1;
|
|
252
|
+
const tf = (freq * (K1 + 1)) / (freq + K1 * lengthNorm);
|
|
253
|
+
scores.set(idx, (scores.get(idx) ?? 0) + idf * tf);
|
|
254
|
+
}
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
// Merged view of CSR slice + pending additions, deduplicated by slot and
|
|
258
|
+
// minus tombstones. df counts the same docs that are scored below.
|
|
259
|
+
const entries = new Map();
|
|
260
|
+
for (let p = start; p < end; p++) {
|
|
261
|
+
const idx = this.postChunk[p];
|
|
262
|
+
if (this.dels.has(idx))
|
|
263
|
+
continue;
|
|
264
|
+
entries.set(idx, this.postFreq[p]);
|
|
265
|
+
}
|
|
266
|
+
if (added) {
|
|
267
|
+
for (const [idx, freq] of added) {
|
|
268
|
+
if (this.dels.has(idx))
|
|
269
|
+
continue;
|
|
270
|
+
entries.set(idx, (entries.get(idx) ?? 0) + freq);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
const df = entries.size;
|
|
274
|
+
if (df === 0)
|
|
196
275
|
continue;
|
|
197
|
-
const df = docs.size;
|
|
198
276
|
const idf = Math.log(1 + (totalChunks - df + 0.5) / (df + 0.5));
|
|
199
|
-
for (const [
|
|
200
|
-
const docLength = this.docLengths
|
|
201
|
-
const lengthNorm = avgDocLength > 0
|
|
202
|
-
? 1 - B + B * (docLength / avgDocLength)
|
|
203
|
-
: 1;
|
|
277
|
+
for (const [idx, freq] of entries) {
|
|
278
|
+
const docLength = this.docLengths[idx] ?? avgDocLength;
|
|
279
|
+
const lengthNorm = avgDocLength > 0 ? 1 - B + B * (docLength / avgDocLength) : 1;
|
|
204
280
|
const tf = (freq * (K1 + 1)) / (freq + K1 * lengthNorm);
|
|
205
|
-
scores.set(
|
|
281
|
+
scores.set(idx, (scores.get(idx) ?? 0) + idf * tf);
|
|
206
282
|
}
|
|
207
283
|
}
|
|
208
284
|
const dangling = [];
|
|
209
285
|
const sorted = [...scores.entries()]
|
|
210
|
-
.filter(([
|
|
211
|
-
const
|
|
286
|
+
.filter(([idx]) => {
|
|
287
|
+
const id = this.idxToId[idx];
|
|
288
|
+
const chunk = id !== undefined ? this.chunkMap.get(id) : undefined;
|
|
212
289
|
if (!chunk) {
|
|
213
|
-
// The
|
|
214
|
-
//
|
|
215
|
-
//
|
|
216
|
-
//
|
|
217
|
-
//
|
|
218
|
-
//
|
|
219
|
-
dangling.push(
|
|
290
|
+
// The posting list references a slot whose chunk is gone. This is
|
|
291
|
+
// the "undefined is not an object (evaluating 'chunk.metadata')"
|
|
292
|
+
// crash site: a removal that raced a persisted index or an
|
|
293
|
+
// interrupted re-index leaves phantom slots behind. Skip and
|
|
294
|
+
// tombstone them so a corrupt index self-heals instead of failing
|
|
295
|
+
// every filtered search.
|
|
296
|
+
dangling.push(idx);
|
|
220
297
|
return false;
|
|
221
298
|
}
|
|
222
299
|
return matchesFilter(chunk, filter);
|
|
223
300
|
})
|
|
224
301
|
.sort((a, b) => b[1] - a[1])
|
|
225
302
|
.slice(0, topK);
|
|
226
|
-
if (dangling.length > 0)
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
303
|
+
if (dangling.length > 0) {
|
|
304
|
+
for (const idx of dangling)
|
|
305
|
+
this.dels.add(idx);
|
|
306
|
+
}
|
|
307
|
+
return sorted.map(([idx, score]) => ({
|
|
308
|
+
chunk: this.chunkMap.get(this.idxToId[idx]),
|
|
309
|
+
score,
|
|
310
|
+
}));
|
|
232
311
|
}
|
|
233
|
-
/**
|
|
234
|
-
*
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
312
|
+
/** Fold pending additions and tombstones into the CSR. Empty token slices
|
|
313
|
+
* are dropped so the token table does not grow unboundedly across merges. */
|
|
314
|
+
merge() {
|
|
315
|
+
if (this.dels.size === 0 && this.addPostings.size === 0)
|
|
316
|
+
return;
|
|
317
|
+
const union = this.tokNames.slice();
|
|
318
|
+
for (const token of this.addPostings.keys()) {
|
|
319
|
+
if (!this.tokIdx.has(token))
|
|
320
|
+
union.push(token);
|
|
321
|
+
}
|
|
322
|
+
const keptTokens = [];
|
|
323
|
+
const perToken = [];
|
|
324
|
+
let total = 0;
|
|
325
|
+
for (const token of union) {
|
|
326
|
+
const tiOld = this.tokIdx.get(token);
|
|
327
|
+
const oldList = [];
|
|
328
|
+
if (tiOld !== undefined) {
|
|
329
|
+
const start = this.tokenStart[tiOld];
|
|
330
|
+
const end = this.tokenStart[tiOld + 1];
|
|
331
|
+
for (let p = start; p < end; p++) {
|
|
332
|
+
const idx = this.postChunk[p];
|
|
333
|
+
if (this.dels.has(idx))
|
|
334
|
+
continue;
|
|
335
|
+
oldList.push([idx, this.postFreq[p]]);
|
|
247
336
|
}
|
|
248
337
|
}
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
338
|
+
const added = this.addPostings.get(token);
|
|
339
|
+
const addList = [];
|
|
340
|
+
if (added) {
|
|
341
|
+
for (const [idx, freq] of added) {
|
|
342
|
+
if (this.dels.has(idx))
|
|
343
|
+
continue;
|
|
344
|
+
addList.push([idx, freq]);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
if (oldList.length === 0 && addList.length === 0)
|
|
348
|
+
continue;
|
|
349
|
+
// Both sources sorted by slot, then a linear merge; a slot present in
|
|
350
|
+
// both (re-added id without a removal) sums its frequencies, matching
|
|
351
|
+
// what search-time scoring computes.
|
|
352
|
+
oldList.sort((a, b) => a[0] - b[0]);
|
|
353
|
+
addList.sort((a, b) => a[0] - b[0]);
|
|
354
|
+
const chunks = [];
|
|
355
|
+
const freqs = [];
|
|
356
|
+
let i = 0;
|
|
357
|
+
let j = 0;
|
|
358
|
+
while (i < oldList.length || j < addList.length) {
|
|
359
|
+
if (j >= addList.length || (i < oldList.length && oldList[i][0] < addList[j][0])) {
|
|
360
|
+
chunks.push(oldList[i][0]);
|
|
361
|
+
freqs.push(oldList[i][1]);
|
|
362
|
+
i++;
|
|
363
|
+
}
|
|
364
|
+
else if (i >= oldList.length || addList[j][0] < oldList[i][0]) {
|
|
365
|
+
chunks.push(addList[j][0]);
|
|
366
|
+
freqs.push(addList[j][1]);
|
|
367
|
+
j++;
|
|
368
|
+
}
|
|
369
|
+
else {
|
|
370
|
+
chunks.push(oldList[i][0]);
|
|
371
|
+
freqs.push(oldList[i][1] + addList[j][1]);
|
|
372
|
+
i++;
|
|
373
|
+
j++;
|
|
253
374
|
}
|
|
254
375
|
}
|
|
255
|
-
|
|
256
|
-
|
|
376
|
+
keptTokens.push(token);
|
|
377
|
+
perToken.push({ chunks, freqs });
|
|
378
|
+
total += chunks.length;
|
|
379
|
+
}
|
|
380
|
+
this.tokNames = keptTokens;
|
|
381
|
+
this.tokIdx = new Map();
|
|
382
|
+
this.tokenStart = new Uint32Array(keptTokens.length + 1);
|
|
383
|
+
this.postChunk = new Uint32Array(total);
|
|
384
|
+
this.postFreq = new Uint32Array(total);
|
|
385
|
+
let cursor = 0;
|
|
386
|
+
for (let t = 0; t < keptTokens.length; t++) {
|
|
387
|
+
this.tokenStart[t] = cursor;
|
|
388
|
+
this.tokIdx.set(keptTokens[t], t);
|
|
389
|
+
const entry = perToken[t];
|
|
390
|
+
this.postChunk.set(entry.chunks, cursor);
|
|
391
|
+
this.postFreq.set(entry.freqs, cursor);
|
|
392
|
+
cursor += entry.chunks.length;
|
|
257
393
|
}
|
|
394
|
+
this.tokenStart[keptTokens.length] = cursor;
|
|
395
|
+
this.dels.clear();
|
|
396
|
+
this.addPostings.clear();
|
|
258
397
|
}
|
|
259
398
|
close() {
|
|
260
|
-
this.
|
|
399
|
+
this.tokNames = [];
|
|
400
|
+
this.tokIdx.clear();
|
|
401
|
+
this.tokenStart = new Uint32Array(1);
|
|
402
|
+
this.postChunk = new Uint32Array(0);
|
|
403
|
+
this.postFreq = new Uint32Array(0);
|
|
261
404
|
this.chunkMap.clear();
|
|
405
|
+
this.idToIdx.clear();
|
|
262
406
|
this.fileToIds.clear();
|
|
263
|
-
this.
|
|
407
|
+
this.addPostings.clear();
|
|
408
|
+
this.dels.clear();
|
|
264
409
|
}
|
|
265
410
|
clear() {
|
|
266
|
-
this.
|
|
267
|
-
this.
|
|
268
|
-
this.
|
|
269
|
-
this.chunkTokens.clear();
|
|
270
|
-
this.docLengths.clear();
|
|
411
|
+
this.close();
|
|
412
|
+
this.idxToId = [];
|
|
413
|
+
this.docLengths = [];
|
|
271
414
|
this.totalTokens = 0;
|
|
272
415
|
}
|
|
273
416
|
count() {
|
|
274
417
|
return this.chunkMap.size;
|
|
275
418
|
}
|
|
419
|
+
/** Total postings (merged + pending). Diagnostic/logging only. */
|
|
420
|
+
postingCount() {
|
|
421
|
+
let n = this.postChunk.length;
|
|
422
|
+
for (const postings of this.addPostings.values())
|
|
423
|
+
n += postings.size;
|
|
424
|
+
return n;
|
|
425
|
+
}
|
|
426
|
+
/** True when this instance was loaded from the legacy v3 (Map-era) file
|
|
427
|
+
* format; the caller should persist once to migrate to v4. */
|
|
428
|
+
needsFormatMigration() {
|
|
429
|
+
return this.migratedFromV3;
|
|
430
|
+
}
|
|
276
431
|
hasChunk(id) {
|
|
277
432
|
return this.chunkMap.has(id);
|
|
278
433
|
}
|
|
@@ -292,13 +447,41 @@ export class KeywordIndex {
|
|
|
292
447
|
const effectiveStorePath = storePath ?? this.storePath;
|
|
293
448
|
if (!effectiveStorePath)
|
|
294
449
|
return;
|
|
450
|
+
this.merge();
|
|
295
451
|
const targetPath = indexPathFor(effectiveStorePath);
|
|
296
452
|
const { mkdir, writeFile } = await import("node:fs/promises");
|
|
297
453
|
const path = await import("node:path");
|
|
298
454
|
await mkdir(path.dirname(targetPath), { recursive: true });
|
|
455
|
+
// Alive slots, in ascending slot order, renumbered for the serialized
|
|
456
|
+
// posting arrays. idToIdx only holds live chunks.
|
|
457
|
+
const aliveSlots = [...this.idToIdx.values()].sort((a, b) => a - b);
|
|
458
|
+
const ids = new Array(aliveSlots.length);
|
|
459
|
+
const renumber = new Int32Array(this.idxToId.length).fill(-1);
|
|
460
|
+
for (let k = 0; k < aliveSlots.length; k++) {
|
|
461
|
+
const slot = aliveSlots[k];
|
|
462
|
+
ids[k] = this.idxToId[slot];
|
|
463
|
+
renumber[slot] = k;
|
|
464
|
+
}
|
|
465
|
+
const postings = [];
|
|
466
|
+
for (let t = 0; t < this.tokNames.length; t++) {
|
|
467
|
+
const start = this.tokenStart[t];
|
|
468
|
+
const end = this.tokenStart[t + 1];
|
|
469
|
+
if (start === end)
|
|
470
|
+
continue;
|
|
471
|
+
const flat = [];
|
|
472
|
+
for (let p = start; p < end; p++) {
|
|
473
|
+
const chunkIdx = renumber[this.postChunk[p]];
|
|
474
|
+
if (chunkIdx < 0)
|
|
475
|
+
continue; // dead slot — defensive, merge() already dropped them
|
|
476
|
+
flat.push(chunkIdx, this.postFreq[p]);
|
|
477
|
+
}
|
|
478
|
+
if (flat.length === 0)
|
|
479
|
+
continue;
|
|
480
|
+
postings.push([t, flat]);
|
|
481
|
+
}
|
|
299
482
|
const serialized = {
|
|
300
483
|
version: INDEX_VERSION,
|
|
301
|
-
|
|
484
|
+
ids,
|
|
302
485
|
chunkMap: Object.fromEntries([...this.chunkMap.entries()].map(([id, chunk]) => [
|
|
303
486
|
id,
|
|
304
487
|
{
|
|
@@ -315,8 +498,11 @@ export class KeywordIndex {
|
|
|
315
498
|
tags: chunk.metadata.tags ? JSON.stringify(chunk.metadata.tags) : "",
|
|
316
499
|
},
|
|
317
500
|
])),
|
|
501
|
+
tokens: this.tokNames,
|
|
502
|
+
postings,
|
|
318
503
|
};
|
|
319
504
|
await writeFile(targetPath, JSON.stringify(serialized), "utf-8");
|
|
505
|
+
this.migratedFromV3 = false;
|
|
320
506
|
}
|
|
321
507
|
static async load(storePath) {
|
|
322
508
|
const targetPath = indexPathFor(storePath);
|
|
@@ -335,73 +521,158 @@ export class KeywordIndex {
|
|
|
335
521
|
catch {
|
|
336
522
|
return new KeywordIndex(storePath);
|
|
337
523
|
}
|
|
524
|
+
if (parsed.version === 3) {
|
|
525
|
+
return KeywordIndex.loadV3(storePath, parsed);
|
|
526
|
+
}
|
|
338
527
|
if (parsed.version !== INDEX_VERSION) {
|
|
339
528
|
return new KeywordIndex(storePath);
|
|
340
529
|
}
|
|
341
530
|
const index = new KeywordIndex(storePath);
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
// (whose tokenizer or indexableText may have differed).
|
|
349
|
-
for (const [token, docs] of index.invertedIndex) {
|
|
350
|
-
for (const id of docs.keys()) {
|
|
351
|
-
let tokens = index.chunkTokens.get(id);
|
|
352
|
-
if (!tokens) {
|
|
353
|
-
tokens = new Set();
|
|
354
|
-
index.chunkTokens.set(id, tokens);
|
|
355
|
-
}
|
|
356
|
-
tokens.add(token);
|
|
357
|
-
}
|
|
531
|
+
// Chunk slots follow the serialized ids array; ids missing from chunkMap
|
|
532
|
+
// stay a slot so dangling-posting self-healing behaves as it did with v3.
|
|
533
|
+
for (let idx = 0; idx < parsed.ids.length; idx++) {
|
|
534
|
+
const id = parsed.ids[idx];
|
|
535
|
+
index.idxToId.push(id);
|
|
536
|
+
index.idToIdx.set(id, idx);
|
|
358
537
|
}
|
|
359
538
|
for (const [id, data] of Object.entries(parsed.chunkMap)) {
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
539
|
+
const chunk = decodeChunk(id, data);
|
|
540
|
+
if (!chunk)
|
|
541
|
+
continue;
|
|
542
|
+
index.chunkMap.set(id, chunk);
|
|
543
|
+
const fileKey = normalizeFilePath(chunk.metadata.filePath);
|
|
544
|
+
let ids = index.fileToIds.get(fileKey);
|
|
545
|
+
if (!ids) {
|
|
546
|
+
ids = new Set();
|
|
547
|
+
index.fileToIds.set(fileKey, ids);
|
|
364
548
|
}
|
|
365
|
-
|
|
366
|
-
|
|
549
|
+
const idx = index.idToIdx.get(id);
|
|
550
|
+
if (idx !== undefined)
|
|
551
|
+
ids.add(idx);
|
|
552
|
+
}
|
|
553
|
+
index.tokNames = parsed.tokens.slice();
|
|
554
|
+
index.tokIdx = new Map();
|
|
555
|
+
for (let t = 0; t < index.tokNames.length; t++) {
|
|
556
|
+
index.tokIdx.set(index.tokNames[t], t);
|
|
557
|
+
}
|
|
558
|
+
// Flatten [tokenIdx, flat] pairs into CSR typed arrays. save() writes
|
|
559
|
+
// token-ordered, deduplicated entries; sort + filter defensively anyway
|
|
560
|
+
// so a hand-edited or future-written file can never corrupt the CSR.
|
|
561
|
+
const posts = parsed.postings
|
|
562
|
+
.filter(([ti]) => ti >= 0 && ti < index.tokNames.length)
|
|
563
|
+
.sort((a, b) => a[0] - b[0])
|
|
564
|
+
.filter((e, i, arr) => i === 0 || e[0] !== arr[i - 1][0]);
|
|
565
|
+
let total = 0;
|
|
566
|
+
for (const [, flat] of posts)
|
|
567
|
+
total += Math.floor(flat.length / 2);
|
|
568
|
+
const tokenStart = new Uint32Array(index.tokNames.length + 1);
|
|
569
|
+
const postChunk = new Uint32Array(total);
|
|
570
|
+
const postFreq = new Uint32Array(total);
|
|
571
|
+
// Occurrence counters for BM25 docLengths (one occurrence per unique
|
|
572
|
+
// token per chunk — tokenize() dedupes per chunk, so counting slices is
|
|
573
|
+
// exactly the number of indexed tokens per chunk).
|
|
574
|
+
const docLen = new Array(index.idxToId.length).fill(0);
|
|
575
|
+
let t = 0;
|
|
576
|
+
let cursor = 0;
|
|
577
|
+
for (const [tokenIdx, flat] of posts) {
|
|
578
|
+
while (t < tokenIdx) {
|
|
579
|
+
tokenStart[t] = cursor; // token with no postings: empty slice
|
|
580
|
+
t++;
|
|
367
581
|
}
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
582
|
+
tokenStart[tokenIdx] = cursor;
|
|
583
|
+
for (let p = 0; p + 1 < flat.length; p += 2) {
|
|
584
|
+
const idx = flat[p];
|
|
585
|
+
postChunk[cursor] = idx;
|
|
586
|
+
postFreq[cursor] = flat[p + 1];
|
|
587
|
+
cursor++;
|
|
588
|
+
if (idx < docLen.length)
|
|
589
|
+
docLen[idx] = (docLen[idx] ?? 0) + 1;
|
|
590
|
+
}
|
|
591
|
+
t = tokenIdx + 1;
|
|
592
|
+
}
|
|
593
|
+
while (t <= index.tokNames.length) {
|
|
594
|
+
tokenStart[t] = cursor;
|
|
595
|
+
t++;
|
|
596
|
+
}
|
|
597
|
+
index.tokenStart = tokenStart;
|
|
598
|
+
index.postChunk = postChunk;
|
|
599
|
+
index.postFreq = postFreq;
|
|
600
|
+
for (let idx = 0; idx < docLen.length; idx++) {
|
|
601
|
+
const len = docLen[idx] ?? 0;
|
|
602
|
+
if (len > 0) {
|
|
603
|
+
index.docLengths[idx] = len;
|
|
604
|
+
index.totalTokens += len;
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
return index;
|
|
608
|
+
}
|
|
609
|
+
/** Read the legacy Map-era format and build the CSR representation.
|
|
610
|
+
* Transient parse peaks exist for this ONE read; the plugin migrates to v4
|
|
611
|
+
* immediately after (needsFormatMigration) so later boots are cheap. */
|
|
612
|
+
static loadV3(storePath, parsed) {
|
|
613
|
+
const index = new KeywordIndex(storePath);
|
|
614
|
+
index.migratedFromV3 = true;
|
|
615
|
+
for (const [id, data] of Object.entries(parsed.chunkMap)) {
|
|
616
|
+
const chunk = decodeChunk(id, data);
|
|
617
|
+
if (!chunk)
|
|
618
|
+
continue;
|
|
619
|
+
const idx = index.idxToId.length;
|
|
620
|
+
index.idxToId.push(id);
|
|
621
|
+
index.idToIdx.set(id, idx);
|
|
622
|
+
index.chunkMap.set(id, chunk);
|
|
398
623
|
const fileKey = normalizeFilePath(chunk.metadata.filePath);
|
|
399
624
|
let ids = index.fileToIds.get(fileKey);
|
|
400
625
|
if (!ids) {
|
|
401
626
|
ids = new Set();
|
|
402
627
|
index.fileToIds.set(fileKey, ids);
|
|
403
628
|
}
|
|
404
|
-
ids.add(
|
|
629
|
+
ids.add(idx);
|
|
630
|
+
}
|
|
631
|
+
// Bucket postings by token, skipping ids that no longer exist (the v3
|
|
632
|
+
// ghost-entry case); each bucket is slot-sorted, then flattened.
|
|
633
|
+
const buckets = [];
|
|
634
|
+
const docLen = new Array(index.idxToId.length).fill(0);
|
|
635
|
+
for (const [token, docs] of parsed.tokens) {
|
|
636
|
+
const bucket = [];
|
|
637
|
+
for (const [id, freq] of docs) {
|
|
638
|
+
const idx = index.idToIdx.get(id);
|
|
639
|
+
if (idx === undefined)
|
|
640
|
+
continue;
|
|
641
|
+
bucket.push([idx, freq]);
|
|
642
|
+
docLen[idx] = (docLen[idx] ?? 0) + 1;
|
|
643
|
+
}
|
|
644
|
+
if (bucket.length === 0)
|
|
645
|
+
continue;
|
|
646
|
+
bucket.sort((a, b) => a[0] - b[0]);
|
|
647
|
+
buckets.push([token, bucket]);
|
|
648
|
+
}
|
|
649
|
+
index.tokNames = buckets.map(([token]) => token);
|
|
650
|
+
index.tokIdx = new Map();
|
|
651
|
+
for (let t = 0; t < index.tokNames.length; t++) {
|
|
652
|
+
index.tokIdx.set(index.tokNames[t], t);
|
|
653
|
+
}
|
|
654
|
+
let total = 0;
|
|
655
|
+
for (const [, bucket] of buckets)
|
|
656
|
+
total += bucket.length;
|
|
657
|
+
index.tokenStart = new Uint32Array(buckets.length + 1);
|
|
658
|
+
index.postChunk = new Uint32Array(total);
|
|
659
|
+
index.postFreq = new Uint32Array(total);
|
|
660
|
+
let cursor = 0;
|
|
661
|
+
for (let t = 0; t < buckets.length; t++) {
|
|
662
|
+
index.tokenStart[t] = cursor;
|
|
663
|
+
for (const [idx, freq] of buckets[t][1]) {
|
|
664
|
+
index.postChunk[cursor] = idx;
|
|
665
|
+
index.postFreq[cursor] = freq;
|
|
666
|
+
cursor++;
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
index.tokenStart[buckets.length] = cursor;
|
|
670
|
+
for (let idx = 0; idx < docLen.length; idx++) {
|
|
671
|
+
const len = docLen[idx] ?? 0;
|
|
672
|
+
if (len > 0) {
|
|
673
|
+
index.docLengths[idx] = len;
|
|
674
|
+
index.totalTokens += len;
|
|
675
|
+
}
|
|
405
676
|
}
|
|
406
677
|
return index;
|
|
407
678
|
}
|
|
@@ -410,8 +681,35 @@ export class KeywordIndex {
|
|
|
410
681
|
const { writeFile, mkdir } = await import("node:fs/promises");
|
|
411
682
|
const path = await import("node:path");
|
|
412
683
|
await mkdir(path.dirname(targetPath), { recursive: true });
|
|
413
|
-
await writeFile(targetPath, JSON.stringify({ version: INDEX_VERSION,
|
|
684
|
+
await writeFile(targetPath, JSON.stringify({ version: INDEX_VERSION, ids: [], chunkMap: {}, tokens: [], postings: [] }), "utf-8");
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
function decodeChunk(id, data) {
|
|
688
|
+
if (!data || typeof data.filePath !== "string")
|
|
689
|
+
return undefined;
|
|
690
|
+
let tags;
|
|
691
|
+
try {
|
|
692
|
+
if (data.tags)
|
|
693
|
+
tags = JSON.parse(data.tags);
|
|
694
|
+
}
|
|
695
|
+
catch {
|
|
696
|
+
tags = undefined;
|
|
414
697
|
}
|
|
698
|
+
return {
|
|
699
|
+
id: data.id ?? id,
|
|
700
|
+
content: data.content ?? "",
|
|
701
|
+
description: data.description ?? "",
|
|
702
|
+
metadata: {
|
|
703
|
+
filePath: data.filePath,
|
|
704
|
+
startLine: data.startLine,
|
|
705
|
+
endLine: data.endLine,
|
|
706
|
+
language: data.language,
|
|
707
|
+
kind: data.kind || undefined,
|
|
708
|
+
role: data.role === "source" || data.role === "test" || data.role === "doc" ? data.role : undefined,
|
|
709
|
+
quirkType: data.quirkType || undefined,
|
|
710
|
+
tags,
|
|
711
|
+
},
|
|
712
|
+
};
|
|
415
713
|
}
|
|
416
714
|
function globMatch(pattern, filePath) {
|
|
417
715
|
const GLOBSTAR = "\x00GS\x00";
|