opencode-rag-plugin 1.18.1 → 1.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,8 +20,25 @@ export declare function recallQuirks(deps: QuirkStoreDeps, query: string, option
20
20
  topK?: number;
21
21
  quirkType?: string;
22
22
  tags?: string[];
23
+ /** Override recallMinScore from config. Used by auto-inject for a lower threshold. */
24
+ minScore?: number;
23
25
  }): Promise<SearchResult[]>;
24
26
  /** Lint quirks for low confidence, staleness, duplicates, and orphan source refs. */
25
27
  export declare function lintQuirks(deps: QuirkStoreDeps): Promise<string[]>;
26
28
  /** Simple Jaccard-based lexical similarity (word overlap). */
27
29
  export declare function lexicalSimilarity(a: string, b: string): number;
30
+ /**
31
+ * Count of meaningful word tokens shared between two texts (Jaccard numerator).
32
+ *
33
+ * Tokens are whitespace/punctuation-split, lowercased, and filtered to those
34
+ * with length ≥ `minTokenLen` (default 3 — skips short filler like "the").
35
+ *
36
+ * Used by the quirk auto-inject gate: candidate quirks that share no tokens
37
+ * with the user's *current* message (i.e. they matched only against the prior
38
+ * assistant text in the combined recall query) are filtered out. This prevents
39
+ * meta-quirks (quirks about quirks themselves) from being injected into
40
+ * unrelated tasks, e.g. when the agent previously explained how quirks work.
41
+ *
42
+ * Set `memory.autoInjectMinTokenOverlap` to `0` to disable the gate.
43
+ */
44
+ export declare function sharedWords(a: string, b: string, minTokenLen?: number): number;
@@ -146,13 +146,13 @@ export async function recallQuirks(deps, query, options) {
146
146
  if (options?.quirkType) {
147
147
  filter.languages = [options.quirkType];
148
148
  }
149
- const recallMinScore = deps.cfg.memory?.recallMinScore ?? 0.72;
149
+ const recallMinScore = options?.minScore ?? deps.cfg.memory?.recallMinScore ?? 0.72;
150
150
  const raw = await retrieve(query, deps.embedder, deps.store, {
151
151
  topK: topK * 3,
152
152
  minScore: recallMinScore,
153
153
  keywordIndex: deps.keywordIndex,
154
154
  keywordWeight: deps.cfg.retrieval.hybridSearch?.keywordWeight,
155
- hybridEnabled: false,
155
+ hybridEnabled: true,
156
156
  queryPrefix: deps.cfg.embedding.queryPrefix,
157
157
  filter,
158
158
  });
@@ -185,13 +185,13 @@ export async function lintQuirks(deps) {
185
185
  const cfg = deps.cfg.memory;
186
186
  for (const q of quirks) {
187
187
  if (q.confidence < (cfg?.minConfidence ?? 0.5)) {
188
- issues.push(`Low confidence (${q.confidence}): "${q.content.slice(0, 60)}..." [${q.id}]`);
188
+ issues.push(`Low confidence (${q.confidence}): "${q.content}" [${q.id}]`);
189
189
  }
190
190
  if (cfg?.decay?.enabled && q.lastObserved) {
191
191
  const ageDays = (Date.now() - new Date(q.lastObserved).getTime()) / 86_400_000;
192
192
  const halfLife = cfg.decay.halfLifeDays;
193
193
  if (halfLife > 0 && ageDays > halfLife * 2) {
194
- issues.push(`Stale (${Math.round(ageDays)}d old): "${q.content.slice(0, 60)}..." [${q.id}]`);
194
+ issues.push(`Stale (${Math.round(ageDays)}d old): "${q.content}" [${q.id}]`);
195
195
  }
196
196
  }
197
197
  }
@@ -200,7 +200,7 @@ export async function lintQuirks(deps) {
200
200
  for (let j = i + 1; j < quirks.length; j++) {
201
201
  const sim = lexicalSimilarity(quirks[i].content, quirks[j].content);
202
202
  if (sim > 0.85) {
203
- issues.push(`Near-duplicate (${(sim * 100).toFixed(0)}% similar): "${quirks[i].content.slice(0, 50)}..." ↔ "${quirks[j].content.slice(0, 50)}..."`);
203
+ issues.push(`Near-duplicate (${(sim * 100).toFixed(0)}% similar): "${quirks[i].content}" ↔ "${quirks[j].content}"`);
204
204
  }
205
205
  }
206
206
  }
@@ -214,4 +214,31 @@ export function lexicalSimilarity(a, b) {
214
214
  const union = new Set([...wordsA, ...wordsB]);
215
215
  return union.size === 0 ? 0 : intersection.size / union.size;
216
216
  }
217
+ /**
218
+ * Count of meaningful word tokens shared between two texts (Jaccard numerator).
219
+ *
220
+ * Tokens are whitespace/punctuation-split, lowercased, and filtered to those
221
+ * with length ≥ `minTokenLen` (default 3 — skips short filler like "the").
222
+ *
223
+ * Used by the quirk auto-inject gate: candidate quirks that share no tokens
224
+ * with the user's *current* message (i.e. they matched only against the prior
225
+ * assistant text in the combined recall query) are filtered out. This prevents
226
+ * meta-quirks (quirks about quirks themselves) from being injected into
227
+ * unrelated tasks, e.g. when the agent previously explained how quirks work.
228
+ *
229
+ * Set `memory.autoInjectMinTokenOverlap` to `0` to disable the gate.
230
+ */
231
+ export function sharedWords(a, b, minTokenLen = 3) {
232
+ const tokensA = a.toLowerCase().split(/\W+/).filter((w) => w.length >= minTokenLen);
233
+ const wordsB = new Set(b.toLowerCase().split(/\W+/).filter((w) => w.length >= minTokenLen));
234
+ let count = 0;
235
+ const seen = new Set();
236
+ for (const tok of tokensA) {
237
+ if (wordsB.has(tok) && !seen.has(tok)) {
238
+ seen.add(tok);
239
+ count++;
240
+ }
241
+ }
242
+ return count;
243
+ }
217
244
  //# sourceMappingURL=quirk-store.js.map
package/dist/tui.js CHANGED
@@ -345,10 +345,16 @@ function buildSettingCategories(cfg, ro, providers) {
345
345
  const docModeRo = (ro.documentationMode ?? {});
346
346
  const wikiModeCfg = (cfg.wikiMode ?? {});
347
347
  const wikiModeRo = (ro.wikiMode ?? {});
348
+ const memoryCfg = (cfg.memory ?? {});
349
+ const memoryRo = (ro.memory ?? {});
348
350
  const embeddingCfg = (cfg.embedding ?? {});
349
351
  const embeddingRo = (ro.embedding ?? {});
350
352
  const tuiCfg = (cfg.tui ?? {});
351
353
  const tuiRo = (ro.tui ?? {});
354
+ const indexingCfg = (cfg.indexing ?? {});
355
+ const indexingRo = (ro.indexing ?? {});
356
+ const chunkingCfg = (cfg.chunking ?? {});
357
+ const chunkingRo = (ro.chunking ?? {});
352
358
  const modelOptions = providers ? buildModelOptions(providers) : undefined;
353
359
  function displayModel(roProvider, roModel, cfgProvider, cfgModel, defaultProvider, defaultModel) {
354
360
  const p = roProvider ?? cfgProvider ?? defaultProvider;
@@ -416,6 +422,31 @@ function buildSettingCategories(cfg, ro, providers) {
416
422
  },
417
423
  ],
418
424
  },
425
+ {
426
+ id: "chunking",
427
+ label: "Chunking",
428
+ description: "Configure how files are split into chunks",
429
+ entries: [
430
+ {
431
+ path: ["indexing", "chunkOverlap"],
432
+ label: "Chunk overlap (lines)",
433
+ type: "number",
434
+ currentValue: indexingRo.chunkOverlap ?? indexingCfg.chunkOverlap ?? 0,
435
+ },
436
+ {
437
+ path: ["indexing", "maxSvgSizeBytes"],
438
+ label: "Max SVG size (bytes)",
439
+ type: "number",
440
+ currentValue: indexingRo.maxSvgSizeBytes ?? indexingCfg.maxSvgSizeBytes ?? 1_048_576,
441
+ },
442
+ {
443
+ path: ["chunking", "nodeTypes"],
444
+ label: "Node types (JSON)",
445
+ type: "json",
446
+ currentValue: chunkingRo.nodeTypes ?? chunkingCfg.nodeTypes ?? {},
447
+ },
448
+ ],
449
+ },
419
450
  {
420
451
  id: "embedding",
421
452
  label: "Embedding",
@@ -488,6 +519,67 @@ function buildSettingCategories(cfg, ro, providers) {
488
519
  },
489
520
  ],
490
521
  },
522
+ {
523
+ id: "memory",
524
+ label: "Quirk Memory",
525
+ description: "Configure experiential memory — gotchas, preferences, decisions recalled across sessions",
526
+ entries: [
527
+ {
528
+ path: ["memory", "enabled"],
529
+ label: "Quirk memory enabled",
530
+ type: "boolean",
531
+ currentValue: memoryRo.enabled ?? memoryCfg.enabled ?? true,
532
+ },
533
+ {
534
+ path: ["memory", "autoInject"],
535
+ label: "Auto-inject quirks",
536
+ type: "boolean",
537
+ currentValue: memoryRo.autoInject ?? memoryCfg.autoInject ?? false,
538
+ },
539
+ {
540
+ path: ["memory", "recallMinScore"],
541
+ label: "User prompt min score",
542
+ type: "number",
543
+ currentValue: memoryRo.recallMinScore ?? memoryCfg.recallMinScore ?? 0.72,
544
+ },
545
+ {
546
+ path: ["memory", "autoInjectMinScore"],
547
+ label: "System prompt min score",
548
+ type: "number",
549
+ currentValue: memoryRo.autoInjectMinScore ?? memoryCfg.autoInjectMinScore ?? 0.6,
550
+ },
551
+ {
552
+ path: ["memory", "autoInjectLatencyBudgetMs"],
553
+ label: "Latency budget (ms)",
554
+ type: "number",
555
+ currentValue: memoryRo.autoInjectLatencyBudgetMs ?? memoryCfg.autoInjectLatencyBudgetMs ?? 2000,
556
+ },
557
+ {
558
+ path: ["memory", "minConfidence"],
559
+ label: "Min confidence",
560
+ type: "number",
561
+ currentValue: memoryRo.minConfidence ?? memoryCfg.minConfidence ?? 0.5,
562
+ },
563
+ {
564
+ path: ["memory", "passiveCapture"],
565
+ label: "Passive capture",
566
+ type: "boolean",
567
+ currentValue: memoryRo.passiveCapture ?? memoryCfg.passiveCapture ?? false,
568
+ },
569
+ {
570
+ path: ["memory", "promptEnforcement"],
571
+ label: "Prompt enforcement",
572
+ type: "boolean",
573
+ currentValue: memoryRo.promptEnforcement ?? memoryCfg.promptEnforcement ?? true,
574
+ },
575
+ {
576
+ path: ["memory", "sessionEndExtraction"],
577
+ label: "Session-end extraction",
578
+ type: "boolean",
579
+ currentValue: memoryRo.sessionEndExtraction ?? memoryCfg.sessionEndExtraction ?? true,
580
+ },
581
+ ],
582
+ },
491
583
  {
492
584
  id: "keybindings",
493
585
  label: "Keybindings",
@@ -564,7 +656,7 @@ async function openSettingsDialog(api) {
564
656
  function showSettingMenu(cat) {
565
657
  const options = [
566
658
  ...cat.entries.map((s) => ({
567
- title: `${s.label}: ${s.type === "boolean" ? (s.currentValue ? "Yes" : "No") : String(s.currentValue)}`,
659
+ title: `${s.label}: ${s.type === "boolean" ? (s.currentValue ? "Yes" : "No") : s.type === "json" ? JSON.stringify(s.currentValue) : String(s.currentValue)}`,
568
660
  value: s.path.join("."),
569
661
  description: s.options ? "Select to open model picker" : (s.type === "boolean" ? "Select to toggle" : "Select to edit"),
570
662
  })),
@@ -594,6 +686,9 @@ async function openSettingsDialog(api) {
594
686
  title: "Settings",
595
687
  message: `${entry.label}: ${newVal ? "Yes" : "No"}`,
596
688
  });
689
+ if (entry.path[0] === "indexing" || entry.path[0] === "chunking") {
690
+ api.ui.toast({ variant: "warning", title: "Settings", message: "Chunking changed. Re-index required." });
691
+ }
597
692
  entry.currentValue = newVal;
598
693
  showSettingMenu(cat);
599
694
  }
@@ -612,6 +707,9 @@ async function openSettingsDialog(api) {
612
707
  title: "Settings",
613
708
  message: `${entry.label}: ${num}`,
614
709
  });
710
+ if (entry.path[0] === "indexing" || entry.path[0] === "chunking") {
711
+ api.ui.toast({ variant: "warning", title: "Settings", message: "Chunking changed. Re-index required." });
712
+ }
615
713
  entry.currentValue = num;
616
714
  }
617
715
  showSettingMenu(cat);
@@ -621,6 +719,41 @@ async function openSettingsDialog(api) {
621
719
  },
622
720
  }));
623
721
  }
722
+ else if (entry.type === "json") {
723
+ api.ui.dialog.replace(() => api.ui.DialogPrompt({
724
+ title: `Edit ${entry.label}`,
725
+ placeholder: "JSON",
726
+ value: JSON.stringify(entry.currentValue, null, 2),
727
+ onConfirm: (input) => {
728
+ try {
729
+ const parsed = JSON.parse(input);
730
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
731
+ api.ui.toast({ variant: "error", title: "Settings", message: "Value must be a JSON object" });
732
+ showSettingMenu(cat);
733
+ return;
734
+ }
735
+ saveRuntimeOverride(storePath, entry.path, parsed);
736
+ saveConfigValue(configPath, entry.path, parsed);
737
+ api.ui.toast({
738
+ variant: "success",
739
+ title: "Settings",
740
+ message: `${entry.label}: updated`,
741
+ });
742
+ if (entry.path[0] === "indexing" || entry.path[0] === "chunking") {
743
+ api.ui.toast({ variant: "warning", title: "Settings", message: "Chunking changed. Re-index required." });
744
+ }
745
+ entry.currentValue = parsed;
746
+ }
747
+ catch {
748
+ api.ui.toast({ variant: "error", title: "Settings", message: "Invalid JSON" });
749
+ }
750
+ showSettingMenu(cat);
751
+ },
752
+ onCancel: () => {
753
+ showSettingMenu(cat);
754
+ },
755
+ }));
756
+ }
624
757
  else {
625
758
  api.ui.dialog.replace(() => api.ui.DialogPrompt({
626
759
  title: `Edit ${entry.label}`,
@@ -90,6 +90,20 @@ export declare class LanceDbStore implements VectorStore {
90
90
  * @returns An array of chunk summaries.
91
91
  */
92
92
  getChunks(offset: number, limit: number): Promise<ChunkSummary[]>;
93
+ /**
94
+ * Fetch chunks with their embedding vectors included (for embedding projection).
95
+ * @param limit - Maximum number of rows to return.
96
+ * @returns Array of { id, filePath, language, startLine, endLine, description, embedding }.
97
+ */
98
+ getChunksWithEmbeddings(limit: number): Promise<{
99
+ id: string;
100
+ filePath: string;
101
+ language: string;
102
+ startLine: number;
103
+ endLine: number;
104
+ description: string;
105
+ embedding: number[];
106
+ }[]>;
93
107
  /**
94
108
  * Perform ANN search internally, returning results scored as cosine similarity (0-1).
95
109
  * This method is called by `search()` and handles the actual query logic.
@@ -157,5 +171,24 @@ export declare class LanceDbStore implements VectorStore {
157
171
  */
158
172
  deleteByFilePath(filePath: string): Promise<void>;
159
173
  private deleteByFilePathInternal;
174
+ /**
175
+ * Verify that data in the store is actually readable.
176
+ * Reads the `content` column (a large text field stored in data fragments,
177
+ * not in the version manifest) for the first few rows. If any of the
178
+ * underlying `.lance` data files are missing from disk, LanceDB throws a
179
+ * corruption error ("Not found: ... .lance") and we return false.
180
+ *
181
+ * This catches silent corruption where countRows() returns metadata from
182
+ * the version manifest while the actual row data on disk is gone.
183
+ */
184
+ checkIntegrity(): Promise<boolean>;
185
+ /**
186
+ * Execute an async function with automatic corruption recovery.
187
+ * If the function throws a LanceDB corruption error, tryRepair() is run
188
+ * and the function is retried once. If repair fails, the original error
189
+ * is re-thrown so callers/higher layers can handle it (e.g. return
190
+ * fallback data, clear the manifest, or trigger a rebuild).
191
+ */
192
+ private withCorruptionRecovery;
160
193
  private tryRepair;
161
194
  }
@@ -384,26 +384,28 @@ export class LanceDbStore {
384
384
  * @returns An array of file summaries sorted by file path.
385
385
  */
386
386
  async listFiles() {
387
- const table = await this.getTable();
388
- const count = await table.countRows();
389
- if (count === 0)
390
- return [];
391
- const rows = await table.query().select(["filePath", "language"]).limit(count).toArray();
392
- const fileMap = new Map();
393
- for (const row of rows) {
394
- const filePath = row.filePath;
395
- const language = row.language;
396
- const existing = fileMap.get(filePath);
397
- if (existing) {
398
- existing.chunkCount++;
399
- }
400
- else {
401
- fileMap.set(filePath, { language, chunkCount: 1 });
387
+ return this.withCorruptionRecovery(async () => {
388
+ const table = await this.getTable();
389
+ const count = await table.countRows();
390
+ if (count === 0)
391
+ return [];
392
+ const rows = await table.query().select(["filePath", "language"]).limit(count).toArray();
393
+ const fileMap = new Map();
394
+ for (const row of rows) {
395
+ const filePath = row.filePath;
396
+ const language = row.language;
397
+ const existing = fileMap.get(filePath);
398
+ if (existing) {
399
+ existing.chunkCount++;
400
+ }
401
+ else {
402
+ fileMap.set(filePath, { language, chunkCount: 1 });
403
+ }
402
404
  }
403
- }
404
- return Array.from(fileMap.entries())
405
- .map(([filePath, info]) => ({ filePath, ...info }))
406
- .sort((a, b) => a.filePath.localeCompare(b.filePath));
405
+ return Array.from(fileMap.entries())
406
+ .map(([filePath, info]) => ({ filePath, ...info }))
407
+ .sort((a, b) => a.filePath.localeCompare(b.filePath));
408
+ });
407
409
  }
408
410
  /**
409
411
  * Retrieve all chunks for a specific file path, sorted by start line.
@@ -411,39 +413,41 @@ export class LanceDbStore {
411
413
  * @returns An array of chunks for that file.
412
414
  */
413
415
  async getChunksByFilePath(filePath) {
414
- const table = await this.getTable();
415
- const normalizedPath = normalizeFilePath(filePath).replace(/'/g, "''");
416
- const rows = await table.query()
417
- .select(QUERY_COLUMNS)
418
- .where(`filePath = '${normalizedPath}'`)
419
- .toArray();
420
- return rows
421
- .map((row) => {
422
- let tags;
423
- try {
424
- const raw = row.tags;
425
- if (raw)
426
- tags = JSON.parse(raw);
427
- }
428
- catch {
429
- tags = undefined;
430
- }
431
- return {
432
- id: row.id,
433
- content: row.content,
434
- description: row.description ?? "",
435
- metadata: {
436
- filePath: row.filePath,
437
- startLine: row.startLine,
438
- endLine: row.endLine,
439
- language: row.language,
440
- kind: row.kind || undefined,
441
- quirkType: row.quirkType || undefined,
442
- tags,
443
- },
444
- };
445
- })
446
- .sort((a, b) => a.metadata.startLine - b.metadata.startLine);
416
+ return this.withCorruptionRecovery(async () => {
417
+ const table = await this.getTable();
418
+ const normalizedPath = normalizeFilePath(filePath).replace(/'/g, "''");
419
+ const rows = await table.query()
420
+ .select(QUERY_COLUMNS)
421
+ .where(`filePath = '${normalizedPath}'`)
422
+ .toArray();
423
+ return rows
424
+ .map((row) => {
425
+ let tags;
426
+ try {
427
+ const raw = row.tags;
428
+ if (raw)
429
+ tags = JSON.parse(raw);
430
+ }
431
+ catch {
432
+ tags = undefined;
433
+ }
434
+ return {
435
+ id: row.id,
436
+ content: row.content,
437
+ description: row.description ?? "",
438
+ metadata: {
439
+ filePath: row.filePath,
440
+ startLine: row.startLine,
441
+ endLine: row.endLine,
442
+ language: row.language,
443
+ kind: row.kind || undefined,
444
+ quirkType: row.quirkType || undefined,
445
+ tags,
446
+ },
447
+ };
448
+ })
449
+ .sort((a, b) => a.metadata.startLine - b.metadata.startLine);
450
+ });
447
451
  }
448
452
  /**
449
453
  * Retrieve a paginated list of all chunks without embeddings.
@@ -452,24 +456,51 @@ export class LanceDbStore {
452
456
  * @returns An array of chunk summaries.
453
457
  */
454
458
  async getChunks(offset, limit) {
455
- const table = await this.getTable();
456
- const rows = await table.query()
457
- .select(QUERY_COLUMNS)
458
- .offset(offset)
459
- .limit(limit)
460
- .toArray();
461
- return rows.map((row) => ({
462
- id: row.id,
463
- filePath: row.filePath,
464
- language: row.language,
465
- startLine: row.startLine,
466
- endLine: row.endLine,
467
- content: row.content,
468
- description: row.description ?? "",
469
- kind: row.kind ?? "",
470
- quirkType: row.quirkType ?? "",
471
- tags: row.tags ?? "",
472
- }));
459
+ return this.withCorruptionRecovery(async () => {
460
+ const table = await this.getTable();
461
+ const rows = await table.query()
462
+ .select(QUERY_COLUMNS)
463
+ .offset(offset)
464
+ .limit(limit)
465
+ .toArray();
466
+ return rows.map((row) => ({
467
+ id: row.id,
468
+ filePath: row.filePath,
469
+ language: row.language,
470
+ startLine: row.startLine,
471
+ endLine: row.endLine,
472
+ content: row.content,
473
+ description: row.description ?? "",
474
+ kind: row.kind ?? "",
475
+ quirkType: row.quirkType ?? "",
476
+ tags: row.tags ?? "",
477
+ }));
478
+ });
479
+ }
480
+ /**
481
+ * Fetch chunks with their embedding vectors included (for embedding projection).
482
+ * @param limit - Maximum number of rows to return.
483
+ * @returns Array of { id, filePath, language, startLine, endLine, description, embedding }.
484
+ */
485
+ async getChunksWithEmbeddings(limit) {
486
+ return this.withCorruptionRecovery(async () => {
487
+ const table = await this.getTable();
488
+ const rows = await table.query()
489
+ .select([...QUERY_COLUMNS, "embedding"])
490
+ .limit(limit)
491
+ .toArray();
492
+ return rows
493
+ .filter((r) => r.embedding && typeof r.embedding === "object")
494
+ .map((row) => ({
495
+ id: row.id,
496
+ filePath: row.filePath,
497
+ language: row.language,
498
+ startLine: row.startLine,
499
+ endLine: row.endLine,
500
+ description: row.description ?? "",
501
+ embedding: Array.from(row.embedding),
502
+ }));
503
+ });
473
504
  }
474
505
  /**
475
506
  * Perform ANN search internally, returning results scored as cosine similarity (0-1).
@@ -548,7 +579,13 @@ export class LanceDbStore {
548
579
  async optimize() {
549
580
  try {
550
581
  const table = await this.getTable();
551
- await table.optimize({ cleanupOlderThan: new Date(), deleteUnverified: false });
582
+ // Clean up versions older than 1 hour — not "right now" — so in-flight
583
+ // queries (e.g. Web UI search, background auto-index) can finish before
584
+ // their data files are reclaimed. Using new Date() here caused data-file
585
+ // race conditions where a reader got "Not found: … .lance" because the
586
+ // GC deleted fragments that the current version still referenced.
587
+ const threshold = new Date(Date.now() - 60 * 60 * 1000);
588
+ await table.optimize({ cleanupOlderThan: threshold, deleteUnverified: false });
552
589
  }
553
590
  catch {
554
591
  // Optimize is best-effort — must not break indexing.
@@ -564,15 +601,17 @@ export class LanceDbStore {
564
601
  const tableNames = await db.tableNames();
565
602
  if (!tableNames.includes(TABLE_NAME))
566
603
  return [];
567
- const table = await this.getTable();
568
- const rows = await table.query().select(["filePath"]).toArray();
569
- const paths = new Set();
570
- for (const row of rows) {
571
- const fp = row.filePath;
572
- if (fp)
573
- paths.add(fp);
574
- }
575
- return Array.from(paths);
604
+ return this.withCorruptionRecovery(async () => {
605
+ const table = await this.getTable();
606
+ const rows = await table.query().select(["filePath"]).toArray();
607
+ const paths = new Set();
608
+ for (const row of rows) {
609
+ const fp = row.filePath;
610
+ if (fp)
611
+ paths.add(fp);
612
+ }
613
+ return Array.from(paths);
614
+ });
576
615
  }
577
616
  catch {
578
617
  return [];
@@ -708,6 +747,56 @@ export class LanceDbStore {
708
747
  const normalizedPath = normalizeFilePath(filePath).replace(/'/g, "''");
709
748
  await table.delete(`filePath = '${normalizedPath}'`);
710
749
  }
750
+ /**
751
+ * Verify that data in the store is actually readable.
752
+ * Reads the `content` column (a large text field stored in data fragments,
753
+ * not in the version manifest) for the first few rows. If any of the
754
+ * underlying `.lance` data files are missing from disk, LanceDB throws a
755
+ * corruption error ("Not found: ... .lance") and we return false.
756
+ *
757
+ * This catches silent corruption where countRows() returns metadata from
758
+ * the version manifest while the actual row data on disk is gone.
759
+ */
760
+ async checkIntegrity() {
761
+ try {
762
+ const db = await this.getDb();
763
+ const tableNames = await db.tableNames();
764
+ if (!tableNames.includes(TABLE_NAME))
765
+ return true;
766
+ const table = await this.getTable();
767
+ const count = await table.countRows();
768
+ if (count === 0)
769
+ return true;
770
+ // Read the `content` column (stored in data fragments, not in the
771
+ // version manifest) for the first few rows. If a fragment is
772
+ // missing, LanceDB will fail with a "Not found" corruption error.
773
+ const rows = await table.query().select(["id", "content"]).limit(10).toArray();
774
+ return rows.length > 0;
775
+ }
776
+ catch (err) {
777
+ if (isCorruptionError(err))
778
+ return false;
779
+ return true;
780
+ }
781
+ }
782
+ /**
783
+ * Execute an async function with automatic corruption recovery.
784
+ * If the function throws a LanceDB corruption error, tryRepair() is run
785
+ * and the function is retried once. If repair fails, the original error
786
+ * is re-thrown so callers/higher layers can handle it (e.g. return
787
+ * fallback data, clear the manifest, or trigger a rebuild).
788
+ */
789
+ async withCorruptionRecovery(fn) {
790
+ try {
791
+ return await fn();
792
+ }
793
+ catch (err) {
794
+ if (isCorruptionError(err) && await this.tryRepair()) {
795
+ return fn();
796
+ }
797
+ throw err;
798
+ }
799
+ }
711
800
  async tryRepair() {
712
801
  try {
713
802
  this.table = null;
package/dist/web/api.d.ts CHANGED
@@ -5,6 +5,7 @@ import type { IncomingMessage, ServerResponse } from "node:http";
5
5
  import { LanceDbStore } from "../vectorstore/lancedb.js";
6
6
  import { KeywordIndex } from "../retriever/keyword-index.js";
7
7
  import type { RagConfig } from "../core/config.js";
8
+ import type { EmbeddingProvider } from "../core/interfaces.js";
8
9
  /** Internal shape for a JSON API response: an HTTP status code and a serialisable body. */
9
10
  interface ApiResponse {
10
11
  status: number;
@@ -27,7 +28,7 @@ interface ApiResponse {
27
28
  * @param cfg - Active RAG configuration (used by quirk endpoints).
28
29
  * @returns An async handler that returns `true` when a route matched or `false` otherwise.
29
30
  */
30
- export declare function createApiHandler(store: LanceDbStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
31
+ export declare function createApiHandler(store: LanceDbStore, keywordIndex: KeywordIndex, storePath: string, cwd?: string, cfg?: RagConfig, getEmbedder?: () => Promise<EmbeddingProvider>): (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
31
32
  /**
32
33
  * Perform token-usage analysis for a single evaluation session.
33
34
  *