opencode-rag-plugin 1.17.3 → 1.18.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/ReadMe.md +41 -1
  2. package/dist/cli/commands/index.d.ts +1 -0
  3. package/dist/cli/commands/index.js +1 -0
  4. package/dist/cli/commands/init-helpers.d.ts +5 -1
  5. package/dist/cli/commands/init-helpers.js +12 -19
  6. package/dist/cli/commands/init.js +42 -8
  7. package/dist/cli/commands/quirk.d.ts +10 -0
  8. package/dist/cli/commands/quirk.js +141 -0
  9. package/dist/cli/commands/setup.js +15 -6
  10. package/dist/cli/commands/ui.js +1 -1
  11. package/dist/cli/index.js +2 -1
  12. package/dist/core/config.d.ts +30 -0
  13. package/dist/core/config.js +26 -2
  14. package/dist/core/interfaces.d.ts +25 -1
  15. package/dist/core/runtime-overrides.d.ts +7 -0
  16. package/dist/core/runtime-overrides.js +15 -0
  17. package/dist/describer/anthropic.d.ts +4 -0
  18. package/dist/describer/anthropic.js +9 -2
  19. package/dist/describer/describer.d.ts +4 -0
  20. package/dist/describer/describer.js +8 -0
  21. package/dist/describer/gemini.d.ts +3 -0
  22. package/dist/describer/gemini.js +8 -2
  23. package/dist/indexer/pipeline.js +40 -0
  24. package/dist/opencode/system-guidance.d.ts +34 -0
  25. package/dist/opencode/system-guidance.js +127 -0
  26. package/dist/opencode/tools.d.ts +38 -0
  27. package/dist/opencode/tools.js +127 -0
  28. package/dist/plugin.js +204 -31
  29. package/dist/quirks/auto-capture.d.ts +14 -0
  30. package/dist/quirks/auto-capture.js +112 -0
  31. package/dist/quirks/monitor.d.ts +5 -0
  32. package/dist/quirks/monitor.js +23 -0
  33. package/dist/quirks/prompts.d.ts +1 -0
  34. package/dist/quirks/prompts.js +13 -0
  35. package/dist/quirks/quirk-store.d.ts +27 -0
  36. package/dist/quirks/quirk-store.js +217 -0
  37. package/dist/quirks/types.d.ts +20 -0
  38. package/dist/quirks/types.js +2 -0
  39. package/dist/retriever/keyword-index.js +16 -0
  40. package/dist/vectorstore/lancedb.d.ts +21 -11
  41. package/dist/vectorstore/lancedb.js +174 -18
  42. package/dist/vectorstore/memory.d.ts +2 -0
  43. package/dist/vectorstore/memory.js +6 -0
  44. package/dist/web/api.d.ts +3 -1
  45. package/dist/web/api.js +50 -1
  46. package/dist/web/server.d.ts +2 -1
  47. package/dist/web/server.js +3 -2
  48. package/dist/web/ui/index.html +163 -0
  49. package/package.json +1 -1
@@ -0,0 +1,217 @@
1
+ import { appendFileSync, existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
2
+ import { randomUUID } from "node:crypto";
3
+ import path from "node:path";
4
+ import { retrieve } from "../retriever/retriever.js";
5
+ import { isQuirkAllowed } from "./monitor.js";
6
+ const QUIRK_FILE_PREFIX = "quirk:";
7
+ const QUIKK_JSONL = "quirks.jsonl";
8
+ function jsonlPath(storePath) {
9
+ return path.join(storePath, QUIKK_JSONL);
10
+ }
11
+ function isMemoryStore(storePath) {
12
+ return storePath.startsWith("memory:");
13
+ }
14
+ /** In-memory backup for memory:// stores. */
15
+ const memQuirks = new Map();
16
+ function readJsonl(filePath) {
17
+ if (!existsSync(filePath))
18
+ return [];
19
+ const raw = readFileSync(filePath, "utf-8").trim();
20
+ if (!raw)
21
+ return [];
22
+ return raw.split("\n").map((l) => JSON.parse(l));
23
+ }
24
+ function appendJsonl(filePath, q) {
25
+ const dir = path.dirname(filePath);
26
+ if (!existsSync(dir)) {
27
+ mkdirSync(dir, { recursive: true });
28
+ }
29
+ appendFileSync(filePath, JSON.stringify(q) + "\n", "utf-8");
30
+ }
31
+ function rewriteJsonl(filePath, quirks) {
32
+ const dir = path.dirname(filePath);
33
+ if (!existsSync(dir)) {
34
+ mkdirSync(dir, { recursive: true });
35
+ }
36
+ writeFileSync(filePath, quirks.map((q) => JSON.stringify(q)).join("\n") + "\n", "utf-8");
37
+ }
38
+ function nowISO() {
39
+ return new Date().toISOString();
40
+ }
41
+ /** Add a new quirk to the vector store, keyword index, and audit log. */
42
+ export async function addQuirk(deps, input) {
43
+ const allowed = isQuirkAllowed(input.content);
44
+ if (!allowed.ok) {
45
+ throw new Error(`Quirk rejected by trust monitor: ${allowed.reason}`);
46
+ }
47
+ const id = `quirk:${randomUUID()}`;
48
+ const lastObserved = nowISO();
49
+ const confidence = input.confidence ?? 1;
50
+ const quirk = {
51
+ id,
52
+ content: input.content,
53
+ quirkType: input.quirkType,
54
+ tags: input.tags ?? [],
55
+ confidence,
56
+ lastObserved,
57
+ sourceRef: input.sourceRef,
58
+ };
59
+ const prefix = deps.cfg.embedding.documentPrefix ?? "";
60
+ const chunkContent = prefix + input.content;
61
+ const embeddings = await deps.embedder.embed([chunkContent], "document");
62
+ const embedding = embeddings[0];
63
+ if (!embedding || embedding.length === 0) {
64
+ throw new Error("Embedding returned empty vector for quirk content");
65
+ }
66
+ const chunk = {
67
+ id,
68
+ content: input.content,
69
+ description: "",
70
+ embedding,
71
+ metadata: {
72
+ filePath: QUIRK_FILE_PREFIX + id,
73
+ startLine: 0,
74
+ endLine: 0,
75
+ language: "quirk",
76
+ kind: "quirk",
77
+ quirkType: input.quirkType,
78
+ tags: input.tags ?? [],
79
+ confidence,
80
+ lastObserved,
81
+ },
82
+ };
83
+ await deps.store.addChunks([chunk]);
84
+ deps.keywordIndex.addChunks([chunk]);
85
+ if (!isMemoryStore(deps.storePath)) {
86
+ appendJsonl(jsonlPath(deps.storePath), quirk);
87
+ }
88
+ else {
89
+ memQuirks.set(id, quirk);
90
+ }
91
+ return quirk;
92
+ }
93
+ /** Remove a quirk by its ID. */
94
+ export async function removeQuirk(deps, id) {
95
+ const filePath = QUIRK_FILE_PREFIX + id;
96
+ await deps.store.deleteByFilePath(filePath);
97
+ deps.keywordIndex.removeByFilePath(filePath);
98
+ if (!isMemoryStore(deps.storePath)) {
99
+ const jp = jsonlPath(deps.storePath);
100
+ const all = readJsonl(jp).filter((q) => q.id !== id);
101
+ rewriteJsonl(jp, all);
102
+ }
103
+ else {
104
+ memQuirks.delete(id);
105
+ }
106
+ }
107
+ /** List all quirks sorted by lastObserved descending. */
108
+ export async function listQuirks(deps) {
109
+ if (!isMemoryStore(deps.storePath)) {
110
+ const jp = jsonlPath(deps.storePath);
111
+ if (existsSync(jp)) {
112
+ const all = readJsonl(jp);
113
+ all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
114
+ return all;
115
+ }
116
+ // Fallback: scan the store
117
+ const filePaths = await deps.store.getFilePaths();
118
+ const quirkPaths = filePaths.filter((fp) => fp.startsWith(QUIRK_FILE_PREFIX));
119
+ const result = [];
120
+ for (const fp of quirkPaths) {
121
+ const chunks = await deps.store.getChunksByFilePath(fp);
122
+ for (const c of chunks) {
123
+ result.push({
124
+ id: c.id,
125
+ content: c.content,
126
+ quirkType: c.metadata.quirkType,
127
+ tags: c.metadata.tags ?? [],
128
+ confidence: c.metadata.confidence ?? 1,
129
+ lastObserved: c.metadata.lastObserved ?? "",
130
+ sourceRef: undefined,
131
+ });
132
+ }
133
+ }
134
+ result.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
135
+ return result;
136
+ }
137
+ const all = [...memQuirks.values()];
138
+ all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
139
+ return all;
140
+ }
141
+ /** Recall quirks matching a query, with confidence re-weighting. */
142
+ export async function recallQuirks(deps, query, options) {
143
+ const topK = options?.topK ?? 10;
144
+ const minConfidence = deps.cfg.memory?.minConfidence ?? 0.5;
145
+ const filter = { kinds: ["quirk"] };
146
+ if (options?.quirkType) {
147
+ filter.languages = [options.quirkType];
148
+ }
149
+ const recallMinScore = deps.cfg.memory?.recallMinScore ?? 0.72;
150
+ const raw = await retrieve(query, deps.embedder, deps.store, {
151
+ topK: topK * 3,
152
+ minScore: recallMinScore,
153
+ keywordIndex: deps.keywordIndex,
154
+ keywordWeight: deps.cfg.retrieval.hybridSearch?.keywordWeight,
155
+ hybridEnabled: false,
156
+ queryPrefix: deps.cfg.embedding.queryPrefix,
157
+ filter,
158
+ });
159
+ // Confidence re-weighting + type/tag filtering
160
+ const filtered = raw.filter((r) => {
161
+ const conf = r.chunk.metadata.confidence ?? 1;
162
+ if (conf < minConfidence)
163
+ return false;
164
+ if (options?.quirkType && r.chunk.metadata.quirkType !== options.quirkType)
165
+ return false;
166
+ if (options?.tags?.length) {
167
+ const chunkTags = r.chunk.metadata.tags ?? [];
168
+ if (!options.tags.some((t) => chunkTags.includes(t)))
169
+ return false;
170
+ }
171
+ return true;
172
+ });
173
+ // Re-weight score by confidence, re-sort
174
+ for (const r of filtered) {
175
+ const conf = r.chunk.metadata.confidence ?? 1;
176
+ r.score = r.score * Math.min(1, Math.max(0.01, conf));
177
+ }
178
+ filtered.sort((a, b) => b.score - a.score);
179
+ return filtered.slice(0, topK);
180
+ }
181
+ /** Lint quirks for low confidence, staleness, duplicates, and orphan source refs. */
182
+ export async function lintQuirks(deps) {
183
+ const issues = [];
184
+ const quirks = await listQuirks(deps);
185
+ const cfg = deps.cfg.memory;
186
+ for (const q of quirks) {
187
+ if (q.confidence < (cfg?.minConfidence ?? 0.5)) {
188
+ issues.push(`Low confidence (${q.confidence}): "${q.content.slice(0, 60)}..." [${q.id}]`);
189
+ }
190
+ if (cfg?.decay?.enabled && q.lastObserved) {
191
+ const ageDays = (Date.now() - new Date(q.lastObserved).getTime()) / 86_400_000;
192
+ const halfLife = cfg.decay.halfLifeDays;
193
+ if (halfLife > 0 && ageDays > halfLife * 2) {
194
+ issues.push(`Stale (${Math.round(ageDays)}d old): "${q.content.slice(0, 60)}..." [${q.id}]`);
195
+ }
196
+ }
197
+ }
198
+ // Duplicate detection (lexically similar content)
199
+ for (let i = 0; i < quirks.length; i++) {
200
+ for (let j = i + 1; j < quirks.length; j++) {
201
+ const sim = lexicalSimilarity(quirks[i].content, quirks[j].content);
202
+ if (sim > 0.85) {
203
+ issues.push(`Near-duplicate (${(sim * 100).toFixed(0)}% similar): "${quirks[i].content.slice(0, 50)}..." ↔ "${quirks[j].content.slice(0, 50)}..."`);
204
+ }
205
+ }
206
+ }
207
+ return issues;
208
+ }
209
+ /** Simple Jaccard-based lexical similarity (word overlap). */
210
+ export function lexicalSimilarity(a, b) {
211
+ const wordsA = new Set(a.toLowerCase().split(/\W+/).filter(Boolean));
212
+ const wordsB = new Set(b.toLowerCase().split(/\W+/).filter(Boolean));
213
+ const intersection = new Set([...wordsA].filter((w) => wordsB.has(w)));
214
+ const union = new Set([...wordsA, ...wordsB]);
215
+ return union.size === 0 ? 0 : intersection.size / union.size;
216
+ }
217
+ //# sourceMappingURL=quirk-store.js.map
@@ -0,0 +1,20 @@
1
+ /** Quirk sub-type labels. */
2
+ export type QuirkType = "gotcha" | "preference" | "decision" | "environment-constraint";
3
+ /** Input for adding a new quirk. */
4
+ export interface QuirkInput {
5
+ content: string;
6
+ quirkType?: string;
7
+ tags?: string[];
8
+ confidence?: number;
9
+ sourceRef?: string;
10
+ }
11
+ /** A quirk entry stored in the vectordb and audit log. */
12
+ export interface Quirk {
13
+ id: string;
14
+ content: string;
15
+ quirkType?: string;
16
+ tags: string[];
17
+ confidence: number;
18
+ lastObserved: string;
19
+ sourceRef?: string;
20
+ }
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=types.js.map
@@ -184,6 +184,9 @@ export class KeywordIndex {
184
184
  startLine: chunk.metadata.startLine,
185
185
  endLine: chunk.metadata.endLine,
186
186
  language: chunk.metadata.language,
187
+ kind: chunk.metadata.kind ?? "",
188
+ quirkType: chunk.metadata.quirkType ?? "",
189
+ tags: chunk.metadata.tags ? JSON.stringify(chunk.metadata.tags) : "",
187
190
  },
188
191
  ])),
189
192
  };
@@ -215,6 +218,14 @@ export class KeywordIndex {
215
218
  index.invertedIndex.set(token, docMap);
216
219
  }
217
220
  for (const [id, data] of Object.entries(parsed.chunkMap)) {
221
+ let tags;
222
+ try {
223
+ if (data.tags)
224
+ tags = JSON.parse(data.tags);
225
+ }
226
+ catch {
227
+ tags = undefined;
228
+ }
218
229
  index.chunkMap.set(id, {
219
230
  id: data.id,
220
231
  content: data.content,
@@ -224,6 +235,9 @@ export class KeywordIndex {
224
235
  startLine: data.startLine,
225
236
  endLine: data.endLine,
226
237
  language: data.language,
238
+ kind: data.kind || undefined,
239
+ quirkType: data.quirkType || undefined,
240
+ tags,
227
241
  },
228
242
  });
229
243
  }
@@ -251,6 +265,8 @@ function matchesFilter(chunk, filter) {
251
265
  return true;
252
266
  if (filter.languages?.length && !filter.languages.includes(chunk.metadata.language))
253
267
  return false;
268
+ if (filter.kinds?.length && !filter.kinds.includes(chunk.metadata.kind ?? ""))
269
+ return false;
254
270
  if (filter.pathPatterns?.length) {
255
271
  return filter.pathPatterns.some((p) => globMatch(p, chunk.metadata.filePath));
256
272
  }
@@ -1,4 +1,4 @@
1
- import type { VectorStore, Chunk, SearchResult, MetadataFilter } from "../core/interfaces.js";
1
+ import type { VectorStore, Chunk, ChunkSummary, SearchResult, MetadataFilter } from "../core/interfaces.js";
2
2
  /**
3
3
  * L2-normalize a vector to unit length. Cosine models require unit vectors
4
4
  * for the dot product to equal cosine similarity.
@@ -40,6 +40,15 @@ export declare class LanceDbStore implements VectorStore {
40
40
  private getTable;
41
41
  private initTable;
42
42
  private tableHasDescriptionColumn;
43
+ private hasColumn;
44
+ /** Add kind/quirkType/tags columns if missing from an existing table. */
45
+ private migrateNewColumns;
46
+ /**
47
+ * Ensure that kind/quirkType/tags columns are nullable so that queries
48
+ * with WHERE clauses on these columns do not panic on old fragments
49
+ * written before the columns existed.
50
+ */
51
+ private ensureColumnsNullable;
43
52
  /**
44
53
  * Store chunks in the LanceDB table. New rows are inserted first, then
45
54
  * any old rows at the same (filePath, startLine) with different IDs are
@@ -80,15 +89,7 @@ export declare class LanceDbStore implements VectorStore {
80
89
  * @param limit - Maximum number of rows to return.
81
90
  * @returns An array of chunk summaries.
82
91
  */
83
- getChunks(offset: number, limit: number): Promise<{
84
- id: string;
85
- filePath: string;
86
- language: string;
87
- startLine: number;
88
- endLine: number;
89
- content: string;
90
- description: string;
91
- }[]>;
92
+ getChunks(offset: number, limit: number): Promise<ChunkSummary[]>;
92
93
  /**
93
94
  * Perform ANN search internally, returning results scored as cosine similarity (0-1).
94
95
  * This method is called by `search()` and handles the actual query logic.
@@ -99,9 +100,18 @@ export declare class LanceDbStore implements VectorStore {
99
100
  private searchInternal;
100
101
  /**
101
102
  * Return the total number of chunks stored in the table.
102
- * @returns The chunk count, or 0 if the table does not exist.
103
+ * Guarded by a 30s timeout — returns 0 if the store is unresponsive
104
+ * (e.g. corrupted version-manifest accumulation) so the pipeline can
105
+ * proceed with a fresh index instead of hanging forever.
106
+ * @returns The chunk count, or 0 if the table does not exist or times out.
103
107
  */
104
108
  count(): Promise<number>;
109
+ /**
110
+ * Compact fragments and prune old version manifests to prevent the
111
+ * version-manifest accumulation that causes countRows() to hang.
112
+ * Should be called at the end of a successful index pass.
113
+ */
114
+ optimize(): Promise<void>;
105
115
  /**
106
116
  * Return all unique file paths currently stored in the index.
107
117
  * @returns An array of normalized file paths.
@@ -6,7 +6,7 @@ import fs from "node:fs/promises";
6
6
  import path from "node:path";
7
7
  import { normalizeFilePath, manifestPathFor } from "../core/manifest.js";
8
8
  const TABLE_NAME = "chunks";
9
- const QUERY_COLUMNS = ["id", "content", "description", "filePath", "startLine", "endLine", "language"];
9
+ const QUERY_COLUMNS = ["id", "content", "description", "filePath", "startLine", "endLine", "language", "kind", "quirkType", "tags"];
10
10
  /**
11
11
  * L2-normalize a vector to unit length. Cosine models require unit vectors
12
12
  * for the dot product to equal cosine similarity.
@@ -109,12 +109,14 @@ export class LanceDbStore {
109
109
  if (tableNames.includes(TABLE_NAME)) {
110
110
  this.table = await db.openTable(TABLE_NAME);
111
111
  if (await this.tableHasDescriptionColumn()) {
112
+ await this.migrateNewColumns();
112
113
  return this.table;
113
114
  }
114
115
  // Schema missing 'description' column -- try to add it gracefully first.
115
116
  try {
116
117
  await this.table.addColumns([{ name: "description", valueSql: "''" }]);
117
118
  console.warn("[lancedb] Added missing 'description' column to existing table.");
119
+ await this.migrateNewColumns();
118
120
  return this.table;
119
121
  }
120
122
  catch {
@@ -150,6 +152,9 @@ export class LanceDbStore {
150
152
  startLine: 0,
151
153
  endLine: 0,
152
154
  language: "",
155
+ kind: "",
156
+ quirkType: "",
157
+ tags: "",
153
158
  };
154
159
  this.table = await db.createTable({
155
160
  name: TABLE_NAME,
@@ -175,6 +180,58 @@ export class LanceDbStore {
175
180
  return false;
176
181
  }
177
182
  }
183
+ async hasColumn(name) {
184
+ try {
185
+ const schema = await this.table.schema();
186
+ return schema.fields.some((f) => f.name === name);
187
+ }
188
+ catch {
189
+ return false;
190
+ }
191
+ }
192
+ /** Add kind/quirkType/tags columns if missing from an existing table. */
193
+ async migrateNewColumns() {
194
+ const missing = [];
195
+ for (const col of ["kind", "quirkType", "tags"]) {
196
+ if (!(await this.hasColumn(col))) {
197
+ missing.push({ name: col, valueSql: "''" });
198
+ }
199
+ }
200
+ if (missing.length > 0) {
201
+ try {
202
+ await this.table.addColumns(missing);
203
+ console.warn(`[lancedb] Added missing columns: ${missing.map((c) => c.name).join(", ")}`);
204
+ }
205
+ catch {
206
+ console.warn("[lancedb] Could not auto-add missing columns. " +
207
+ "Run 'opencode-rag index --force' to rebuild the index with the correct schema.");
208
+ return;
209
+ }
210
+ }
211
+ await this.ensureColumnsNullable(["kind", "quirkType", "tags"]);
212
+ }
213
+ /**
214
+ * Ensure that kind/quirkType/tags columns are nullable so that queries
215
+ * with WHERE clauses on these columns do not panic on old fragments
216
+ * written before the columns existed.
217
+ */
218
+ async ensureColumnsNullable(columns) {
219
+ for (const col of columns) {
220
+ if (!(await this.hasColumn(col)))
221
+ continue;
222
+ try {
223
+ const schema = await this.table.schema();
224
+ const field = schema.fields.find((f) => f.name === col);
225
+ if (field && field.nullable === false) {
226
+ await this.table.alterColumns([{ path: col, nullable: true }]);
227
+ console.warn(`[lancedb] Made column '${col}' nullable to avoid null-fragment panics.`);
228
+ }
229
+ }
230
+ catch {
231
+ console.warn(`[lancedb] Could not alter nullability of column '${col}'.`);
232
+ }
233
+ }
234
+ }
178
235
  /**
179
236
  * Store chunks in the LanceDB table. New rows are inserted first, then
180
237
  * any old rows at the same (filePath, startLine) with different IDs are
@@ -214,6 +271,9 @@ export class LanceDbStore {
214
271
  startLine: c.metadata.startLine,
215
272
  endLine: c.metadata.endLine,
216
273
  language: c.metadata.language,
274
+ kind: c.metadata.kind ?? "",
275
+ quirkType: c.metadata.quirkType ?? "",
276
+ tags: c.metadata.tags ? JSON.stringify(c.metadata.tags) : "",
217
277
  }));
218
278
  if (rows.length === 0)
219
279
  return;
@@ -292,6 +352,15 @@ export class LanceDbStore {
292
352
  }
293
353
  }
294
354
  rowToSearchResult(row) {
355
+ let tags;
356
+ try {
357
+ const raw = row.tags;
358
+ if (raw)
359
+ tags = JSON.parse(raw);
360
+ }
361
+ catch {
362
+ tags = undefined;
363
+ }
295
364
  return {
296
365
  score: Math.min(1, Math.max(0, 1 - (row._distance ?? 0) / 2)),
297
366
  chunk: {
@@ -303,6 +372,9 @@ export class LanceDbStore {
303
372
  startLine: row.startLine,
304
373
  endLine: row.endLine,
305
374
  language: row.language,
375
+ kind: row.kind || undefined,
376
+ quirkType: row.quirkType || undefined,
377
+ tags,
306
378
  },
307
379
  },
308
380
  };
@@ -346,17 +418,31 @@ export class LanceDbStore {
346
418
  .where(`filePath = '${normalizedPath}'`)
347
419
  .toArray();
348
420
  return rows
349
- .map((row) => ({
350
- id: row.id,
351
- content: row.content,
352
- description: row.description ?? "",
353
- metadata: {
354
- filePath: row.filePath,
355
- startLine: row.startLine,
356
- endLine: row.endLine,
357
- language: row.language,
358
- },
359
- }))
421
+ .map((row) => {
422
+ let tags;
423
+ try {
424
+ const raw = row.tags;
425
+ if (raw)
426
+ tags = JSON.parse(raw);
427
+ }
428
+ catch {
429
+ tags = undefined;
430
+ }
431
+ return {
432
+ id: row.id,
433
+ content: row.content,
434
+ description: row.description ?? "",
435
+ metadata: {
436
+ filePath: row.filePath,
437
+ startLine: row.startLine,
438
+ endLine: row.endLine,
439
+ language: row.language,
440
+ kind: row.kind || undefined,
441
+ quirkType: row.quirkType || undefined,
442
+ tags,
443
+ },
444
+ };
445
+ })
360
446
  .sort((a, b) => a.metadata.startLine - b.metadata.startLine);
361
447
  }
362
448
  /**
@@ -380,6 +466,9 @@ export class LanceDbStore {
380
466
  endLine: row.endLine,
381
467
  content: row.content,
382
468
  description: row.description ?? "",
469
+ kind: row.kind ?? "",
470
+ quirkType: row.quirkType ?? "",
471
+ tags: row.tags ?? "",
383
472
  }));
384
473
  }
385
474
  /**
@@ -398,18 +487,36 @@ export class LanceDbStore {
398
487
  const count = await table.countRows();
399
488
  if (count === 0)
400
489
  return [];
401
- let query = table.vectorSearch(l2Normalize(embedding)).distanceType("cosine");
402
490
  const whereClause = buildWhereClause(filter);
403
- if (whereClause)
404
- query = query.where(whereClause);
405
- const results = await query.limit(topK).toArray();
491
+ let results;
492
+ try {
493
+ let query = table.vectorSearch(l2Normalize(embedding)).distanceType("cosine");
494
+ if (whereClause)
495
+ query = query.where(whereClause);
496
+ results = await query.limit(topK).toArray();
497
+ }
498
+ catch (err) {
499
+ if (err instanceof Error && err.message.includes("non-nullable") && filter) {
500
+ const fallbackQuery = table.vectorSearch(l2Normalize(embedding)).distanceType("cosine");
501
+ const rawResults = await fallbackQuery.limit(topK * 5).toArray();
502
+ return rawResults
503
+ .map((row) => this.rowToSearchResult(row))
504
+ .filter((r) => r.chunk.id !== "__seed__")
505
+ .filter((r) => matchesFilterLocal(r.chunk, filter))
506
+ .slice(0, topK);
507
+ }
508
+ throw err;
509
+ }
406
510
  return results
407
511
  .map((row) => this.rowToSearchResult(row))
408
512
  .filter((r) => r.chunk.id !== "__seed__");
409
513
  }
410
514
  /**
411
515
  * Return the total number of chunks stored in the table.
412
- * @returns The chunk count, or 0 if the table does not exist.
516
+ * Guarded by a 30s timeout — returns 0 if the store is unresponsive
517
+ * (e.g. corrupted version-manifest accumulation) so the pipeline can
518
+ * proceed with a fresh index instead of hanging forever.
519
+ * @returns The chunk count, or 0 if the table does not exist or times out.
413
520
  */
414
521
  async count() {
415
522
  try {
@@ -418,12 +525,35 @@ export class LanceDbStore {
418
525
  if (!tableNames.includes(TABLE_NAME))
419
526
  return 0;
420
527
  const table = await this.getTable();
421
- return await table.countRows();
528
+ const COUNT_TIMEOUT_MS = 30_000;
529
+ const result = await Promise.race([
530
+ table.countRows(),
531
+ new Promise((resolve) => setTimeout(() => resolve(null), COUNT_TIMEOUT_MS)),
532
+ ]);
533
+ if (result === null) {
534
+ console.warn(`[lancedb] count() timed out after ${COUNT_TIMEOUT_MS / 1000}s — treating store as empty.`);
535
+ return 0;
536
+ }
537
+ return result;
422
538
  }
423
539
  catch {
424
540
  return 0;
425
541
  }
426
542
  }
543
+ /**
544
+ * Compact fragments and prune old version manifests to prevent the
545
+ * version-manifest accumulation that causes countRows() to hang.
546
+ * Should be called at the end of a successful index pass.
547
+ */
548
+ async optimize() {
549
+ try {
550
+ const table = await this.getTable();
551
+ await table.optimize({ cleanupOlderThan: new Date(), deleteUnverified: false });
552
+ }
553
+ catch {
554
+ // Optimize is best-effort — must not break indexing.
555
+ }
556
+ }
427
557
  /**
428
558
  * Return all unique file paths currently stored in the index.
429
559
  * @returns An array of normalized file paths.
@@ -641,6 +771,10 @@ function buildWhereClause(filter) {
641
771
  const langs = filter.languages.map((l) => `'${l.replace(/'/g, "''")}'`).join(",");
642
772
  parts.push(`language IN (${langs})`);
643
773
  }
774
+ if (filter.kinds?.length) {
775
+ const kinds = filter.kinds.map((k) => `'${k.replace(/'/g, "''")}'`).join(",");
776
+ parts.push(`kind IN (${kinds})`);
777
+ }
644
778
  if (filter.pathPatterns?.length) {
645
779
  const likes = filter.pathPatterns.map((p) => {
646
780
  const escaped = p.replace(/'/g, "''");
@@ -654,4 +788,26 @@ function buildWhereClause(filter) {
654
788
  }
655
789
  return parts.length ? parts.join(" AND ") : undefined;
656
790
  }
791
+ /** Client-side metadata filter for use as a fallback when LanceDB WHERE panics. */
792
+ function matchesFilterLocal(chunk, filter) {
793
+ if (!filter)
794
+ return true;
795
+ if (filter.languages?.length && !filter.languages.includes(chunk.metadata.language))
796
+ return false;
797
+ if (filter.kinds?.length && !filter.kinds.includes(chunk.metadata.kind ?? ""))
798
+ return false;
799
+ if (filter.pathPatterns?.length) {
800
+ return filter.pathPatterns.some((p) => globMatchLocal(p, chunk.metadata.filePath));
801
+ }
802
+ return true;
803
+ }
804
+ function globMatchLocal(pattern, filePath) {
805
+ const GLOBSTAR = "\x00GS\x00";
806
+ let re = pattern.replace(/\*\*/g, GLOBSTAR);
807
+ re = re.replace(/([.+^${}()|[\]\\])/g, "\\$1");
808
+ re = re.replace(new RegExp(GLOBSTAR, "g"), ".*");
809
+ re = re.replace(/\*/g, "[^/]*");
810
+ re = re.replace(/\?/g, ".");
811
+ return new RegExp("^" + re + "$").test(filePath);
812
+ }
657
813
  //# sourceMappingURL=lancedb.js.map
@@ -21,4 +21,6 @@ export declare class InMemoryVectorStore implements VectorStore {
21
21
  getChunksByFilePath(filePath: string): Promise<Chunk[]>;
22
22
  /** Release any held resources. No-op for the in-memory store. */
23
23
  close(): Promise<void>;
24
+ /** No-op for the in-memory store. */
25
+ optimize(): Promise<void>;
24
26
  }
@@ -45,6 +45,9 @@ export class InMemoryVectorStore {
45
45
  endLine: c.metadata.endLine,
46
46
  content: c.content,
47
47
  description: c.description ?? "",
48
+ kind: c.metadata.kind ?? "",
49
+ quirkType: c.metadata.quirkType ?? "",
50
+ tags: c.metadata.tags ? JSON.stringify(c.metadata.tags) : "",
48
51
  }));
49
52
  }
50
53
  async listFiles() {
@@ -70,6 +73,9 @@ export class InMemoryVectorStore {
70
73
  /** Release any held resources. No-op for the in-memory store. */
71
74
  async close() {
72
75
  }
76
+ /** No-op for the in-memory store. */
77
+ async optimize() {
78
+ }
73
79
  }
74
80
  /**
75
81
  * Compute the cosine similarity between two equal-length vectors.