opencode-rag-plugin 1.18.1 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +7 -4
- package/dist/cli/commands/query.js +1 -1
- package/dist/cli/commands/quirk.js +2 -2
- package/dist/content/reader.d.ts +2 -1
- package/dist/content/reader.js +9 -15
- package/dist/core/config.d.ts +34 -2
- package/dist/core/config.js +28 -0
- package/dist/core/exclude.d.ts +4 -0
- package/dist/core/exclude.js +45 -0
- package/dist/core/interfaces.d.ts +6 -0
- package/dist/core/runtime-overrides.d.ts +4 -1
- package/dist/core/runtime-overrides.js +6 -0
- package/dist/indexer/pipeline.js +21 -0
- package/dist/indexer/watch.js +4 -3
- package/dist/indexer.d.ts +1 -0
- package/dist/indexer.js +1 -0
- package/dist/plugin.js +113 -36
- package/dist/quirks/prompts.js +1 -1
- package/dist/quirks/quirk-store.d.ts +17 -0
- package/dist/quirks/quirk-store.js +32 -5
- package/dist/tui.js +134 -1
- package/dist/vectorstore/lancedb.d.ts +33 -0
- package/dist/vectorstore/lancedb.js +169 -80
- package/dist/web/api.d.ts +2 -1
- package/dist/web/api.js +219 -3
- package/dist/web/pca.d.ts +7 -0
- package/dist/web/pca.js +87 -0
- package/dist/web/server.js +23 -8
- package/dist/web/static.d.ts +7 -2
- package/dist/web/static.js +31 -7
- package/dist/web/ui/assets/index-CJBvt6e0.js +3 -0
- package/dist/web/ui/assets/index-CKdp79Tw.css +1 -0
- package/dist/web/ui/assets/vendor-Dy7HKFCY.js +1 -0
- package/dist/web/ui/index.html +9 -1632
- package/package.json +10 -4
- package/dist/web/ui/app.css +0 -1
- package/dist/web/ui/github-dark.css +0 -118
- package/dist/web/ui/highlight.min.js +0 -1213
|
@@ -20,8 +20,25 @@ export declare function recallQuirks(deps: QuirkStoreDeps, query: string, option
|
|
|
20
20
|
topK?: number;
|
|
21
21
|
quirkType?: string;
|
|
22
22
|
tags?: string[];
|
|
23
|
+
/** Override recallMinScore from config. Used by auto-inject for a lower threshold. */
|
|
24
|
+
minScore?: number;
|
|
23
25
|
}): Promise<SearchResult[]>;
|
|
24
26
|
/** Lint quirks for low confidence, staleness, duplicates, and orphan source refs. */
|
|
25
27
|
export declare function lintQuirks(deps: QuirkStoreDeps): Promise<string[]>;
|
|
26
28
|
/** Simple Jaccard-based lexical similarity (word overlap). */
|
|
27
29
|
export declare function lexicalSimilarity(a: string, b: string): number;
|
|
30
|
+
/**
|
|
31
|
+
* Count of meaningful word tokens shared between two texts (Jaccard numerator).
|
|
32
|
+
*
|
|
33
|
+
* Tokens are whitespace/punctuation-split, lowercased, and filtered to those
|
|
34
|
+
* with length ≥ `minTokenLen` (default 3 — skips short filler like "the").
|
|
35
|
+
*
|
|
36
|
+
* Used by the quirk auto-inject gate: candidate quirks that share no tokens
|
|
37
|
+
* with the user's *current* message (i.e. they matched only against the prior
|
|
38
|
+
* assistant text in the combined recall query) are filtered out. This prevents
|
|
39
|
+
* meta-quirks (quirks about quirks themselves) from being injected into
|
|
40
|
+
* unrelated tasks, e.g. when the agent previously explained how quirks work.
|
|
41
|
+
*
|
|
42
|
+
* Set `memory.autoInjectMinTokenOverlap` to `0` to disable the gate.
|
|
43
|
+
*/
|
|
44
|
+
export declare function sharedWords(a: string, b: string, minTokenLen?: number): number;
|
|
@@ -146,13 +146,13 @@ export async function recallQuirks(deps, query, options) {
|
|
|
146
146
|
if (options?.quirkType) {
|
|
147
147
|
filter.languages = [options.quirkType];
|
|
148
148
|
}
|
|
149
|
-
const recallMinScore = deps.cfg.memory?.recallMinScore ?? 0.72;
|
|
149
|
+
const recallMinScore = options?.minScore ?? deps.cfg.memory?.recallMinScore ?? 0.72;
|
|
150
150
|
const raw = await retrieve(query, deps.embedder, deps.store, {
|
|
151
151
|
topK: topK * 3,
|
|
152
152
|
minScore: recallMinScore,
|
|
153
153
|
keywordIndex: deps.keywordIndex,
|
|
154
154
|
keywordWeight: deps.cfg.retrieval.hybridSearch?.keywordWeight,
|
|
155
|
-
hybridEnabled:
|
|
155
|
+
hybridEnabled: true,
|
|
156
156
|
queryPrefix: deps.cfg.embedding.queryPrefix,
|
|
157
157
|
filter,
|
|
158
158
|
});
|
|
@@ -185,13 +185,13 @@ export async function lintQuirks(deps) {
|
|
|
185
185
|
const cfg = deps.cfg.memory;
|
|
186
186
|
for (const q of quirks) {
|
|
187
187
|
if (q.confidence < (cfg?.minConfidence ?? 0.5)) {
|
|
188
|
-
issues.push(`Low confidence (${q.confidence}): "${q.content
|
|
188
|
+
issues.push(`Low confidence (${q.confidence}): "${q.content}" [${q.id}]`);
|
|
189
189
|
}
|
|
190
190
|
if (cfg?.decay?.enabled && q.lastObserved) {
|
|
191
191
|
const ageDays = (Date.now() - new Date(q.lastObserved).getTime()) / 86_400_000;
|
|
192
192
|
const halfLife = cfg.decay.halfLifeDays;
|
|
193
193
|
if (halfLife > 0 && ageDays > halfLife * 2) {
|
|
194
|
-
issues.push(`Stale (${Math.round(ageDays)}d old): "${q.content
|
|
194
|
+
issues.push(`Stale (${Math.round(ageDays)}d old): "${q.content}" [${q.id}]`);
|
|
195
195
|
}
|
|
196
196
|
}
|
|
197
197
|
}
|
|
@@ -200,7 +200,7 @@ export async function lintQuirks(deps) {
|
|
|
200
200
|
for (let j = i + 1; j < quirks.length; j++) {
|
|
201
201
|
const sim = lexicalSimilarity(quirks[i].content, quirks[j].content);
|
|
202
202
|
if (sim > 0.85) {
|
|
203
|
-
issues.push(`Near-duplicate (${(sim * 100).toFixed(0)}% similar): "${quirks[i].content
|
|
203
|
+
issues.push(`Near-duplicate (${(sim * 100).toFixed(0)}% similar): "${quirks[i].content}" ↔ "${quirks[j].content}"`);
|
|
204
204
|
}
|
|
205
205
|
}
|
|
206
206
|
}
|
|
@@ -214,4 +214,31 @@ export function lexicalSimilarity(a, b) {
|
|
|
214
214
|
const union = new Set([...wordsA, ...wordsB]);
|
|
215
215
|
return union.size === 0 ? 0 : intersection.size / union.size;
|
|
216
216
|
}
|
|
217
|
+
/**
|
|
218
|
+
* Count of meaningful word tokens shared between two texts (Jaccard numerator).
|
|
219
|
+
*
|
|
220
|
+
* Tokens are whitespace/punctuation-split, lowercased, and filtered to those
|
|
221
|
+
* with length ≥ `minTokenLen` (default 3 — skips short filler like "the").
|
|
222
|
+
*
|
|
223
|
+
* Used by the quirk auto-inject gate: candidate quirks that share no tokens
|
|
224
|
+
* with the user's *current* message (i.e. they matched only against the prior
|
|
225
|
+
* assistant text in the combined recall query) are filtered out. This prevents
|
|
226
|
+
* meta-quirks (quirks about quirks themselves) from being injected into
|
|
227
|
+
* unrelated tasks, e.g. when the agent previously explained how quirks work.
|
|
228
|
+
*
|
|
229
|
+
* Set `memory.autoInjectMinTokenOverlap` to `0` to disable the gate.
|
|
230
|
+
*/
|
|
231
|
+
export function sharedWords(a, b, minTokenLen = 3) {
|
|
232
|
+
const tokensA = a.toLowerCase().split(/\W+/).filter((w) => w.length >= minTokenLen);
|
|
233
|
+
const wordsB = new Set(b.toLowerCase().split(/\W+/).filter((w) => w.length >= minTokenLen));
|
|
234
|
+
let count = 0;
|
|
235
|
+
const seen = new Set();
|
|
236
|
+
for (const tok of tokensA) {
|
|
237
|
+
if (wordsB.has(tok) && !seen.has(tok)) {
|
|
238
|
+
seen.add(tok);
|
|
239
|
+
count++;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
return count;
|
|
243
|
+
}
|
|
217
244
|
//# sourceMappingURL=quirk-store.js.map
|
package/dist/tui.js
CHANGED
|
@@ -345,10 +345,16 @@ function buildSettingCategories(cfg, ro, providers) {
|
|
|
345
345
|
const docModeRo = (ro.documentationMode ?? {});
|
|
346
346
|
const wikiModeCfg = (cfg.wikiMode ?? {});
|
|
347
347
|
const wikiModeRo = (ro.wikiMode ?? {});
|
|
348
|
+
const memoryCfg = (cfg.memory ?? {});
|
|
349
|
+
const memoryRo = (ro.memory ?? {});
|
|
348
350
|
const embeddingCfg = (cfg.embedding ?? {});
|
|
349
351
|
const embeddingRo = (ro.embedding ?? {});
|
|
350
352
|
const tuiCfg = (cfg.tui ?? {});
|
|
351
353
|
const tuiRo = (ro.tui ?? {});
|
|
354
|
+
const indexingCfg = (cfg.indexing ?? {});
|
|
355
|
+
const indexingRo = (ro.indexing ?? {});
|
|
356
|
+
const chunkingCfg = (cfg.chunking ?? {});
|
|
357
|
+
const chunkingRo = (ro.chunking ?? {});
|
|
352
358
|
const modelOptions = providers ? buildModelOptions(providers) : undefined;
|
|
353
359
|
function displayModel(roProvider, roModel, cfgProvider, cfgModel, defaultProvider, defaultModel) {
|
|
354
360
|
const p = roProvider ?? cfgProvider ?? defaultProvider;
|
|
@@ -416,6 +422,31 @@ function buildSettingCategories(cfg, ro, providers) {
|
|
|
416
422
|
},
|
|
417
423
|
],
|
|
418
424
|
},
|
|
425
|
+
{
|
|
426
|
+
id: "chunking",
|
|
427
|
+
label: "Chunking",
|
|
428
|
+
description: "Configure how files are split into chunks",
|
|
429
|
+
entries: [
|
|
430
|
+
{
|
|
431
|
+
path: ["indexing", "chunkOverlap"],
|
|
432
|
+
label: "Chunk overlap (lines)",
|
|
433
|
+
type: "number",
|
|
434
|
+
currentValue: indexingRo.chunkOverlap ?? indexingCfg.chunkOverlap ?? 0,
|
|
435
|
+
},
|
|
436
|
+
{
|
|
437
|
+
path: ["indexing", "maxSvgSizeBytes"],
|
|
438
|
+
label: "Max SVG size (bytes)",
|
|
439
|
+
type: "number",
|
|
440
|
+
currentValue: indexingRo.maxSvgSizeBytes ?? indexingCfg.maxSvgSizeBytes ?? 1_048_576,
|
|
441
|
+
},
|
|
442
|
+
{
|
|
443
|
+
path: ["chunking", "nodeTypes"],
|
|
444
|
+
label: "Node types (JSON)",
|
|
445
|
+
type: "json",
|
|
446
|
+
currentValue: chunkingRo.nodeTypes ?? chunkingCfg.nodeTypes ?? {},
|
|
447
|
+
},
|
|
448
|
+
],
|
|
449
|
+
},
|
|
419
450
|
{
|
|
420
451
|
id: "embedding",
|
|
421
452
|
label: "Embedding",
|
|
@@ -488,6 +519,67 @@ function buildSettingCategories(cfg, ro, providers) {
|
|
|
488
519
|
},
|
|
489
520
|
],
|
|
490
521
|
},
|
|
522
|
+
{
|
|
523
|
+
id: "memory",
|
|
524
|
+
label: "Quirk Memory",
|
|
525
|
+
description: "Configure experiential memory — gotchas, preferences, decisions recalled across sessions",
|
|
526
|
+
entries: [
|
|
527
|
+
{
|
|
528
|
+
path: ["memory", "enabled"],
|
|
529
|
+
label: "Quirk memory enabled",
|
|
530
|
+
type: "boolean",
|
|
531
|
+
currentValue: memoryRo.enabled ?? memoryCfg.enabled ?? true,
|
|
532
|
+
},
|
|
533
|
+
{
|
|
534
|
+
path: ["memory", "autoInject"],
|
|
535
|
+
label: "Auto-inject quirks",
|
|
536
|
+
type: "boolean",
|
|
537
|
+
currentValue: memoryRo.autoInject ?? memoryCfg.autoInject ?? false,
|
|
538
|
+
},
|
|
539
|
+
{
|
|
540
|
+
path: ["memory", "recallMinScore"],
|
|
541
|
+
label: "User prompt min score",
|
|
542
|
+
type: "number",
|
|
543
|
+
currentValue: memoryRo.recallMinScore ?? memoryCfg.recallMinScore ?? 0.72,
|
|
544
|
+
},
|
|
545
|
+
{
|
|
546
|
+
path: ["memory", "autoInjectMinScore"],
|
|
547
|
+
label: "System prompt min score",
|
|
548
|
+
type: "number",
|
|
549
|
+
currentValue: memoryRo.autoInjectMinScore ?? memoryCfg.autoInjectMinScore ?? 0.6,
|
|
550
|
+
},
|
|
551
|
+
{
|
|
552
|
+
path: ["memory", "autoInjectLatencyBudgetMs"],
|
|
553
|
+
label: "Latency budget (ms)",
|
|
554
|
+
type: "number",
|
|
555
|
+
currentValue: memoryRo.autoInjectLatencyBudgetMs ?? memoryCfg.autoInjectLatencyBudgetMs ?? 2000,
|
|
556
|
+
},
|
|
557
|
+
{
|
|
558
|
+
path: ["memory", "minConfidence"],
|
|
559
|
+
label: "Min confidence",
|
|
560
|
+
type: "number",
|
|
561
|
+
currentValue: memoryRo.minConfidence ?? memoryCfg.minConfidence ?? 0.5,
|
|
562
|
+
},
|
|
563
|
+
{
|
|
564
|
+
path: ["memory", "passiveCapture"],
|
|
565
|
+
label: "Passive capture",
|
|
566
|
+
type: "boolean",
|
|
567
|
+
currentValue: memoryRo.passiveCapture ?? memoryCfg.passiveCapture ?? false,
|
|
568
|
+
},
|
|
569
|
+
{
|
|
570
|
+
path: ["memory", "promptEnforcement"],
|
|
571
|
+
label: "Prompt enforcement",
|
|
572
|
+
type: "boolean",
|
|
573
|
+
currentValue: memoryRo.promptEnforcement ?? memoryCfg.promptEnforcement ?? true,
|
|
574
|
+
},
|
|
575
|
+
{
|
|
576
|
+
path: ["memory", "sessionEndExtraction"],
|
|
577
|
+
label: "Session-end extraction",
|
|
578
|
+
type: "boolean",
|
|
579
|
+
currentValue: memoryRo.sessionEndExtraction ?? memoryCfg.sessionEndExtraction ?? true,
|
|
580
|
+
},
|
|
581
|
+
],
|
|
582
|
+
},
|
|
491
583
|
{
|
|
492
584
|
id: "keybindings",
|
|
493
585
|
label: "Keybindings",
|
|
@@ -564,7 +656,7 @@ async function openSettingsDialog(api) {
|
|
|
564
656
|
function showSettingMenu(cat) {
|
|
565
657
|
const options = [
|
|
566
658
|
...cat.entries.map((s) => ({
|
|
567
|
-
title: `${s.label}: ${s.type === "boolean" ? (s.currentValue ? "Yes" : "No") : String(s.currentValue)}`,
|
|
659
|
+
title: `${s.label}: ${s.type === "boolean" ? (s.currentValue ? "Yes" : "No") : s.type === "json" ? JSON.stringify(s.currentValue) : String(s.currentValue)}`,
|
|
568
660
|
value: s.path.join("."),
|
|
569
661
|
description: s.options ? "Select to open model picker" : (s.type === "boolean" ? "Select to toggle" : "Select to edit"),
|
|
570
662
|
})),
|
|
@@ -594,6 +686,9 @@ async function openSettingsDialog(api) {
|
|
|
594
686
|
title: "Settings",
|
|
595
687
|
message: `${entry.label}: ${newVal ? "Yes" : "No"}`,
|
|
596
688
|
});
|
|
689
|
+
if (entry.path[0] === "indexing" || entry.path[0] === "chunking") {
|
|
690
|
+
api.ui.toast({ variant: "warning", title: "Settings", message: "Chunking changed. Re-index required." });
|
|
691
|
+
}
|
|
597
692
|
entry.currentValue = newVal;
|
|
598
693
|
showSettingMenu(cat);
|
|
599
694
|
}
|
|
@@ -612,6 +707,9 @@ async function openSettingsDialog(api) {
|
|
|
612
707
|
title: "Settings",
|
|
613
708
|
message: `${entry.label}: ${num}`,
|
|
614
709
|
});
|
|
710
|
+
if (entry.path[0] === "indexing" || entry.path[0] === "chunking") {
|
|
711
|
+
api.ui.toast({ variant: "warning", title: "Settings", message: "Chunking changed. Re-index required." });
|
|
712
|
+
}
|
|
615
713
|
entry.currentValue = num;
|
|
616
714
|
}
|
|
617
715
|
showSettingMenu(cat);
|
|
@@ -621,6 +719,41 @@ async function openSettingsDialog(api) {
|
|
|
621
719
|
},
|
|
622
720
|
}));
|
|
623
721
|
}
|
|
722
|
+
else if (entry.type === "json") {
|
|
723
|
+
api.ui.dialog.replace(() => api.ui.DialogPrompt({
|
|
724
|
+
title: `Edit ${entry.label}`,
|
|
725
|
+
placeholder: "JSON",
|
|
726
|
+
value: JSON.stringify(entry.currentValue, null, 2),
|
|
727
|
+
onConfirm: (input) => {
|
|
728
|
+
try {
|
|
729
|
+
const parsed = JSON.parse(input);
|
|
730
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
731
|
+
api.ui.toast({ variant: "error", title: "Settings", message: "Value must be a JSON object" });
|
|
732
|
+
showSettingMenu(cat);
|
|
733
|
+
return;
|
|
734
|
+
}
|
|
735
|
+
saveRuntimeOverride(storePath, entry.path, parsed);
|
|
736
|
+
saveConfigValue(configPath, entry.path, parsed);
|
|
737
|
+
api.ui.toast({
|
|
738
|
+
variant: "success",
|
|
739
|
+
title: "Settings",
|
|
740
|
+
message: `${entry.label}: updated`,
|
|
741
|
+
});
|
|
742
|
+
if (entry.path[0] === "indexing" || entry.path[0] === "chunking") {
|
|
743
|
+
api.ui.toast({ variant: "warning", title: "Settings", message: "Chunking changed. Re-index required." });
|
|
744
|
+
}
|
|
745
|
+
entry.currentValue = parsed;
|
|
746
|
+
}
|
|
747
|
+
catch {
|
|
748
|
+
api.ui.toast({ variant: "error", title: "Settings", message: "Invalid JSON" });
|
|
749
|
+
}
|
|
750
|
+
showSettingMenu(cat);
|
|
751
|
+
},
|
|
752
|
+
onCancel: () => {
|
|
753
|
+
showSettingMenu(cat);
|
|
754
|
+
},
|
|
755
|
+
}));
|
|
756
|
+
}
|
|
624
757
|
else {
|
|
625
758
|
api.ui.dialog.replace(() => api.ui.DialogPrompt({
|
|
626
759
|
title: `Edit ${entry.label}`,
|
|
@@ -90,6 +90,20 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
90
90
|
* @returns An array of chunk summaries.
|
|
91
91
|
*/
|
|
92
92
|
getChunks(offset: number, limit: number): Promise<ChunkSummary[]>;
|
|
93
|
+
/**
|
|
94
|
+
* Fetch chunks with their embedding vectors included (for embedding projection).
|
|
95
|
+
* @param limit - Maximum number of rows to return.
|
|
96
|
+
* @returns Array of { id, filePath, language, startLine, endLine, description, embedding }.
|
|
97
|
+
*/
|
|
98
|
+
getChunksWithEmbeddings(limit: number): Promise<{
|
|
99
|
+
id: string;
|
|
100
|
+
filePath: string;
|
|
101
|
+
language: string;
|
|
102
|
+
startLine: number;
|
|
103
|
+
endLine: number;
|
|
104
|
+
description: string;
|
|
105
|
+
embedding: number[];
|
|
106
|
+
}[]>;
|
|
93
107
|
/**
|
|
94
108
|
* Perform ANN search internally, returning results scored as cosine similarity (0-1).
|
|
95
109
|
* This method is called by `search()` and handles the actual query logic.
|
|
@@ -157,5 +171,24 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
157
171
|
*/
|
|
158
172
|
deleteByFilePath(filePath: string): Promise<void>;
|
|
159
173
|
private deleteByFilePathInternal;
|
|
174
|
+
/**
|
|
175
|
+
* Verify that data in the store is actually readable.
|
|
176
|
+
* Reads the `content` column (a large text field stored in data fragments,
|
|
177
|
+
* not in the version manifest) for the first few rows. If any of the
|
|
178
|
+
* underlying `.lance` data files are missing from disk, LanceDB throws a
|
|
179
|
+
* corruption error ("Not found: ... .lance") and we return false.
|
|
180
|
+
*
|
|
181
|
+
* This catches silent corruption where countRows() returns metadata from
|
|
182
|
+
* the version manifest while the actual row data on disk is gone.
|
|
183
|
+
*/
|
|
184
|
+
checkIntegrity(): Promise<boolean>;
|
|
185
|
+
/**
|
|
186
|
+
* Execute an async function with automatic corruption recovery.
|
|
187
|
+
* If the function throws a LanceDB corruption error, tryRepair() is run
|
|
188
|
+
* and the function is retried once. If repair fails, the original error
|
|
189
|
+
* is re-thrown so callers/higher layers can handle it (e.g. return
|
|
190
|
+
* fallback data, clear the manifest, or trigger a rebuild).
|
|
191
|
+
*/
|
|
192
|
+
private withCorruptionRecovery;
|
|
160
193
|
private tryRepair;
|
|
161
194
|
}
|
|
@@ -384,26 +384,28 @@ export class LanceDbStore {
|
|
|
384
384
|
* @returns An array of file summaries sorted by file path.
|
|
385
385
|
*/
|
|
386
386
|
async listFiles() {
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
const
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
existing
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
387
|
+
return this.withCorruptionRecovery(async () => {
|
|
388
|
+
const table = await this.getTable();
|
|
389
|
+
const count = await table.countRows();
|
|
390
|
+
if (count === 0)
|
|
391
|
+
return [];
|
|
392
|
+
const rows = await table.query().select(["filePath", "language"]).limit(count).toArray();
|
|
393
|
+
const fileMap = new Map();
|
|
394
|
+
for (const row of rows) {
|
|
395
|
+
const filePath = row.filePath;
|
|
396
|
+
const language = row.language;
|
|
397
|
+
const existing = fileMap.get(filePath);
|
|
398
|
+
if (existing) {
|
|
399
|
+
existing.chunkCount++;
|
|
400
|
+
}
|
|
401
|
+
else {
|
|
402
|
+
fileMap.set(filePath, { language, chunkCount: 1 });
|
|
403
|
+
}
|
|
402
404
|
}
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
405
|
+
return Array.from(fileMap.entries())
|
|
406
|
+
.map(([filePath, info]) => ({ filePath, ...info }))
|
|
407
|
+
.sort((a, b) => a.filePath.localeCompare(b.filePath));
|
|
408
|
+
});
|
|
407
409
|
}
|
|
408
410
|
/**
|
|
409
411
|
* Retrieve all chunks for a specific file path, sorted by start line.
|
|
@@ -411,39 +413,41 @@ export class LanceDbStore {
|
|
|
411
413
|
* @returns An array of chunks for that file.
|
|
412
414
|
*/
|
|
413
415
|
async getChunksByFilePath(filePath) {
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
.
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
416
|
+
return this.withCorruptionRecovery(async () => {
|
|
417
|
+
const table = await this.getTable();
|
|
418
|
+
const normalizedPath = normalizeFilePath(filePath).replace(/'/g, "''");
|
|
419
|
+
const rows = await table.query()
|
|
420
|
+
.select(QUERY_COLUMNS)
|
|
421
|
+
.where(`filePath = '${normalizedPath}'`)
|
|
422
|
+
.toArray();
|
|
423
|
+
return rows
|
|
424
|
+
.map((row) => {
|
|
425
|
+
let tags;
|
|
426
|
+
try {
|
|
427
|
+
const raw = row.tags;
|
|
428
|
+
if (raw)
|
|
429
|
+
tags = JSON.parse(raw);
|
|
430
|
+
}
|
|
431
|
+
catch {
|
|
432
|
+
tags = undefined;
|
|
433
|
+
}
|
|
434
|
+
return {
|
|
435
|
+
id: row.id,
|
|
436
|
+
content: row.content,
|
|
437
|
+
description: row.description ?? "",
|
|
438
|
+
metadata: {
|
|
439
|
+
filePath: row.filePath,
|
|
440
|
+
startLine: row.startLine,
|
|
441
|
+
endLine: row.endLine,
|
|
442
|
+
language: row.language,
|
|
443
|
+
kind: row.kind || undefined,
|
|
444
|
+
quirkType: row.quirkType || undefined,
|
|
445
|
+
tags,
|
|
446
|
+
},
|
|
447
|
+
};
|
|
448
|
+
})
|
|
449
|
+
.sort((a, b) => a.metadata.startLine - b.metadata.startLine);
|
|
450
|
+
});
|
|
447
451
|
}
|
|
448
452
|
/**
|
|
449
453
|
* Retrieve a paginated list of all chunks without embeddings.
|
|
@@ -452,24 +456,51 @@ export class LanceDbStore {
|
|
|
452
456
|
* @returns An array of chunk summaries.
|
|
453
457
|
*/
|
|
454
458
|
async getChunks(offset, limit) {
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
.
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
459
|
+
return this.withCorruptionRecovery(async () => {
|
|
460
|
+
const table = await this.getTable();
|
|
461
|
+
const rows = await table.query()
|
|
462
|
+
.select(QUERY_COLUMNS)
|
|
463
|
+
.offset(offset)
|
|
464
|
+
.limit(limit)
|
|
465
|
+
.toArray();
|
|
466
|
+
return rows.map((row) => ({
|
|
467
|
+
id: row.id,
|
|
468
|
+
filePath: row.filePath,
|
|
469
|
+
language: row.language,
|
|
470
|
+
startLine: row.startLine,
|
|
471
|
+
endLine: row.endLine,
|
|
472
|
+
content: row.content,
|
|
473
|
+
description: row.description ?? "",
|
|
474
|
+
kind: row.kind ?? "",
|
|
475
|
+
quirkType: row.quirkType ?? "",
|
|
476
|
+
tags: row.tags ?? "",
|
|
477
|
+
}));
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Fetch chunks with their embedding vectors included (for embedding projection).
|
|
482
|
+
* @param limit - Maximum number of rows to return.
|
|
483
|
+
* @returns Array of { id, filePath, language, startLine, endLine, description, embedding }.
|
|
484
|
+
*/
|
|
485
|
+
async getChunksWithEmbeddings(limit) {
|
|
486
|
+
return this.withCorruptionRecovery(async () => {
|
|
487
|
+
const table = await this.getTable();
|
|
488
|
+
const rows = await table.query()
|
|
489
|
+
.select([...QUERY_COLUMNS, "embedding"])
|
|
490
|
+
.limit(limit)
|
|
491
|
+
.toArray();
|
|
492
|
+
return rows
|
|
493
|
+
.filter((r) => r.embedding && typeof r.embedding === "object")
|
|
494
|
+
.map((row) => ({
|
|
495
|
+
id: row.id,
|
|
496
|
+
filePath: row.filePath,
|
|
497
|
+
language: row.language,
|
|
498
|
+
startLine: row.startLine,
|
|
499
|
+
endLine: row.endLine,
|
|
500
|
+
description: row.description ?? "",
|
|
501
|
+
embedding: Array.from(row.embedding),
|
|
502
|
+
}));
|
|
503
|
+
});
|
|
473
504
|
}
|
|
474
505
|
/**
|
|
475
506
|
* Perform ANN search internally, returning results scored as cosine similarity (0-1).
|
|
@@ -548,7 +579,13 @@ export class LanceDbStore {
|
|
|
548
579
|
async optimize() {
|
|
549
580
|
try {
|
|
550
581
|
const table = await this.getTable();
|
|
551
|
-
|
|
582
|
+
// Clean up versions older than 1 hour — not "right now" — so in-flight
|
|
583
|
+
// queries (e.g. Web UI search, background auto-index) can finish before
|
|
584
|
+
// their data files are reclaimed. Using new Date() here caused data-file
|
|
585
|
+
// race conditions where a reader got "Not found: … .lance" because the
|
|
586
|
+
// GC deleted fragments that the current version still referenced.
|
|
587
|
+
const threshold = new Date(Date.now() - 60 * 60 * 1000);
|
|
588
|
+
await table.optimize({ cleanupOlderThan: threshold, deleteUnverified: false });
|
|
552
589
|
}
|
|
553
590
|
catch {
|
|
554
591
|
// Optimize is best-effort — must not break indexing.
|
|
@@ -564,15 +601,17 @@ export class LanceDbStore {
|
|
|
564
601
|
const tableNames = await db.tableNames();
|
|
565
602
|
if (!tableNames.includes(TABLE_NAME))
|
|
566
603
|
return [];
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
const
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
604
|
+
return this.withCorruptionRecovery(async () => {
|
|
605
|
+
const table = await this.getTable();
|
|
606
|
+
const rows = await table.query().select(["filePath"]).toArray();
|
|
607
|
+
const paths = new Set();
|
|
608
|
+
for (const row of rows) {
|
|
609
|
+
const fp = row.filePath;
|
|
610
|
+
if (fp)
|
|
611
|
+
paths.add(fp);
|
|
612
|
+
}
|
|
613
|
+
return Array.from(paths);
|
|
614
|
+
});
|
|
576
615
|
}
|
|
577
616
|
catch {
|
|
578
617
|
return [];
|
|
@@ -708,6 +747,56 @@ export class LanceDbStore {
|
|
|
708
747
|
const normalizedPath = normalizeFilePath(filePath).replace(/'/g, "''");
|
|
709
748
|
await table.delete(`filePath = '${normalizedPath}'`);
|
|
710
749
|
}
|
|
750
|
+
/**
|
|
751
|
+
* Verify that data in the store is actually readable.
|
|
752
|
+
* Reads the `content` column (a large text field stored in data fragments,
|
|
753
|
+
* not in the version manifest) for the first few rows. If any of the
|
|
754
|
+
* underlying `.lance` data files are missing from disk, LanceDB throws a
|
|
755
|
+
* corruption error ("Not found: ... .lance") and we return false.
|
|
756
|
+
*
|
|
757
|
+
* This catches silent corruption where countRows() returns metadata from
|
|
758
|
+
* the version manifest while the actual row data on disk is gone.
|
|
759
|
+
*/
|
|
760
|
+
async checkIntegrity() {
|
|
761
|
+
try {
|
|
762
|
+
const db = await this.getDb();
|
|
763
|
+
const tableNames = await db.tableNames();
|
|
764
|
+
if (!tableNames.includes(TABLE_NAME))
|
|
765
|
+
return true;
|
|
766
|
+
const table = await this.getTable();
|
|
767
|
+
const count = await table.countRows();
|
|
768
|
+
if (count === 0)
|
|
769
|
+
return true;
|
|
770
|
+
// Read the `content` column (stored in data fragments, not in the
|
|
771
|
+
// version manifest) for the first few rows. If a fragment is
|
|
772
|
+
// missing, LanceDB will fail with a "Not found" corruption error.
|
|
773
|
+
const rows = await table.query().select(["id", "content"]).limit(10).toArray();
|
|
774
|
+
return rows.length > 0;
|
|
775
|
+
}
|
|
776
|
+
catch (err) {
|
|
777
|
+
if (isCorruptionError(err))
|
|
778
|
+
return false;
|
|
779
|
+
return true;
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
/**
|
|
783
|
+
* Execute an async function with automatic corruption recovery.
|
|
784
|
+
* If the function throws a LanceDB corruption error, tryRepair() is run
|
|
785
|
+
* and the function is retried once. If repair fails, the original error
|
|
786
|
+
* is re-thrown so callers/higher layers can handle it (e.g. return
|
|
787
|
+
* fallback data, clear the manifest, or trigger a rebuild).
|
|
788
|
+
*/
|
|
789
|
+
async withCorruptionRecovery(fn) {
|
|
790
|
+
try {
|
|
791
|
+
return await fn();
|
|
792
|
+
}
|
|
793
|
+
catch (err) {
|
|
794
|
+
if (isCorruptionError(err) && await this.tryRepair()) {
|
|
795
|
+
return fn();
|
|
796
|
+
}
|
|
797
|
+
throw err;
|
|
798
|
+
}
|
|
799
|
+
}
|
|
711
800
|
async tryRepair() {
|
|
712
801
|
try {
|
|
713
802
|
this.table = null;
|
package/dist/web/api.d.ts
CHANGED
|
@@ -5,6 +5,7 @@ import type { IncomingMessage, ServerResponse } from "node:http";
|
|
|
5
5
|
import { LanceDbStore } from "../vectorstore/lancedb.js";
|
|
6
6
|
import { KeywordIndex } from "../retriever/keyword-index.js";
|
|
7
7
|
import type { RagConfig } from "../core/config.js";
|
|
8
|
+
import type { EmbeddingProvider } from "../core/interfaces.js";
|
|
8
9
|
/** Internal shape for a JSON API response: an HTTP status code and a serialisable body. */
|
|
9
10
|
interface ApiResponse {
|
|
10
11
|
status: number;
|
|
@@ -27,7 +28,7 @@ interface ApiResponse {
|
|
|
27
28
|
* @param cfg - Active RAG configuration (used by quirk endpoints).
|
|
28
29
|
* @returns An async handler that returns `true` when a route matched or `false` otherwise.
|
|
29
30
|
*/
|
|
30
|
-
export declare function createApiHandler(store: LanceDbStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
|
|
31
|
+
export declare function createApiHandler(store: LanceDbStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig, getEmbedder?: () => Promise<EmbeddingProvider>): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
|
|
31
32
|
/**
|
|
32
33
|
* Perform token-usage analysis for a single evaluation session.
|
|
33
34
|
*
|