opencode-rag-plugin 1.17.3 → 1.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +41 -1
- package/dist/cli/commands/index.d.ts +1 -0
- package/dist/cli/commands/index.js +1 -0
- package/dist/cli/commands/init-helpers.d.ts +5 -1
- package/dist/cli/commands/init-helpers.js +12 -19
- package/dist/cli/commands/init.js +42 -8
- package/dist/cli/commands/quirk.d.ts +10 -0
- package/dist/cli/commands/quirk.js +141 -0
- package/dist/cli/commands/setup.js +15 -6
- package/dist/cli/commands/ui.js +1 -1
- package/dist/cli/index.js +2 -1
- package/dist/core/config.d.ts +30 -0
- package/dist/core/config.js +26 -2
- package/dist/core/interfaces.d.ts +25 -1
- package/dist/core/runtime-overrides.d.ts +7 -0
- package/dist/core/runtime-overrides.js +15 -0
- package/dist/describer/anthropic.d.ts +4 -0
- package/dist/describer/anthropic.js +9 -2
- package/dist/describer/describer.d.ts +4 -0
- package/dist/describer/describer.js +8 -0
- package/dist/describer/gemini.d.ts +3 -0
- package/dist/describer/gemini.js +8 -2
- package/dist/indexer/pipeline.js +40 -0
- package/dist/opencode/system-guidance.d.ts +34 -0
- package/dist/opencode/system-guidance.js +127 -0
- package/dist/opencode/tools.d.ts +38 -0
- package/dist/opencode/tools.js +127 -0
- package/dist/plugin.js +204 -31
- package/dist/quirks/auto-capture.d.ts +14 -0
- package/dist/quirks/auto-capture.js +112 -0
- package/dist/quirks/monitor.d.ts +5 -0
- package/dist/quirks/monitor.js +23 -0
- package/dist/quirks/prompts.d.ts +1 -0
- package/dist/quirks/prompts.js +13 -0
- package/dist/quirks/quirk-store.d.ts +27 -0
- package/dist/quirks/quirk-store.js +217 -0
- package/dist/quirks/types.d.ts +20 -0
- package/dist/quirks/types.js +2 -0
- package/dist/retriever/keyword-index.js +16 -0
- package/dist/vectorstore/lancedb.d.ts +21 -11
- package/dist/vectorstore/lancedb.js +174 -18
- package/dist/vectorstore/memory.d.ts +2 -0
- package/dist/vectorstore/memory.js +6 -0
- package/dist/web/api.d.ts +3 -1
- package/dist/web/api.js +50 -1
- package/dist/web/server.d.ts +2 -1
- package/dist/web/server.js +3 -2
- package/dist/web/ui/index.html +163 -0
- package/package.json +1 -1
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import { appendFileSync, existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { retrieve } from "../retriever/retriever.js";
|
|
5
|
+
import { isQuirkAllowed } from "./monitor.js";
|
|
6
|
+
const QUIRK_FILE_PREFIX = "quirk:";
|
|
7
|
+
const QUIKK_JSONL = "quirks.jsonl";
|
|
8
|
+
function jsonlPath(storePath) {
|
|
9
|
+
return path.join(storePath, QUIKK_JSONL);
|
|
10
|
+
}
|
|
11
|
+
function isMemoryStore(storePath) {
|
|
12
|
+
return storePath.startsWith("memory:");
|
|
13
|
+
}
|
|
14
|
+
/** In-memory backup for memory:// stores. */
|
|
15
|
+
const memQuirks = new Map();
|
|
16
|
+
function readJsonl(filePath) {
|
|
17
|
+
if (!existsSync(filePath))
|
|
18
|
+
return [];
|
|
19
|
+
const raw = readFileSync(filePath, "utf-8").trim();
|
|
20
|
+
if (!raw)
|
|
21
|
+
return [];
|
|
22
|
+
return raw.split("\n").map((l) => JSON.parse(l));
|
|
23
|
+
}
|
|
24
|
+
function appendJsonl(filePath, q) {
|
|
25
|
+
const dir = path.dirname(filePath);
|
|
26
|
+
if (!existsSync(dir)) {
|
|
27
|
+
mkdirSync(dir, { recursive: true });
|
|
28
|
+
}
|
|
29
|
+
appendFileSync(filePath, JSON.stringify(q) + "\n", "utf-8");
|
|
30
|
+
}
|
|
31
|
+
function rewriteJsonl(filePath, quirks) {
|
|
32
|
+
const dir = path.dirname(filePath);
|
|
33
|
+
if (!existsSync(dir)) {
|
|
34
|
+
mkdirSync(dir, { recursive: true });
|
|
35
|
+
}
|
|
36
|
+
writeFileSync(filePath, quirks.map((q) => JSON.stringify(q)).join("\n") + "\n", "utf-8");
|
|
37
|
+
}
|
|
38
|
+
function nowISO() {
|
|
39
|
+
return new Date().toISOString();
|
|
40
|
+
}
|
|
41
|
+
/** Add a new quirk to the vector store, keyword index, and audit log. */
|
|
42
|
+
export async function addQuirk(deps, input) {
|
|
43
|
+
const allowed = isQuirkAllowed(input.content);
|
|
44
|
+
if (!allowed.ok) {
|
|
45
|
+
throw new Error(`Quirk rejected by trust monitor: ${allowed.reason}`);
|
|
46
|
+
}
|
|
47
|
+
const id = `quirk:${randomUUID()}`;
|
|
48
|
+
const lastObserved = nowISO();
|
|
49
|
+
const confidence = input.confidence ?? 1;
|
|
50
|
+
const quirk = {
|
|
51
|
+
id,
|
|
52
|
+
content: input.content,
|
|
53
|
+
quirkType: input.quirkType,
|
|
54
|
+
tags: input.tags ?? [],
|
|
55
|
+
confidence,
|
|
56
|
+
lastObserved,
|
|
57
|
+
sourceRef: input.sourceRef,
|
|
58
|
+
};
|
|
59
|
+
const prefix = deps.cfg.embedding.documentPrefix ?? "";
|
|
60
|
+
const chunkContent = prefix + input.content;
|
|
61
|
+
const embeddings = await deps.embedder.embed([chunkContent], "document");
|
|
62
|
+
const embedding = embeddings[0];
|
|
63
|
+
if (!embedding || embedding.length === 0) {
|
|
64
|
+
throw new Error("Embedding returned empty vector for quirk content");
|
|
65
|
+
}
|
|
66
|
+
const chunk = {
|
|
67
|
+
id,
|
|
68
|
+
content: input.content,
|
|
69
|
+
description: "",
|
|
70
|
+
embedding,
|
|
71
|
+
metadata: {
|
|
72
|
+
filePath: QUIRK_FILE_PREFIX + id,
|
|
73
|
+
startLine: 0,
|
|
74
|
+
endLine: 0,
|
|
75
|
+
language: "quirk",
|
|
76
|
+
kind: "quirk",
|
|
77
|
+
quirkType: input.quirkType,
|
|
78
|
+
tags: input.tags ?? [],
|
|
79
|
+
confidence,
|
|
80
|
+
lastObserved,
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
await deps.store.addChunks([chunk]);
|
|
84
|
+
deps.keywordIndex.addChunks([chunk]);
|
|
85
|
+
if (!isMemoryStore(deps.storePath)) {
|
|
86
|
+
appendJsonl(jsonlPath(deps.storePath), quirk);
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
memQuirks.set(id, quirk);
|
|
90
|
+
}
|
|
91
|
+
return quirk;
|
|
92
|
+
}
|
|
93
|
+
/** Remove a quirk by its ID. */
|
|
94
|
+
export async function removeQuirk(deps, id) {
|
|
95
|
+
const filePath = QUIRK_FILE_PREFIX + id;
|
|
96
|
+
await deps.store.deleteByFilePath(filePath);
|
|
97
|
+
deps.keywordIndex.removeByFilePath(filePath);
|
|
98
|
+
if (!isMemoryStore(deps.storePath)) {
|
|
99
|
+
const jp = jsonlPath(deps.storePath);
|
|
100
|
+
const all = readJsonl(jp).filter((q) => q.id !== id);
|
|
101
|
+
rewriteJsonl(jp, all);
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
memQuirks.delete(id);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/** List all quirks sorted by lastObserved descending. */
|
|
108
|
+
export async function listQuirks(deps) {
|
|
109
|
+
if (!isMemoryStore(deps.storePath)) {
|
|
110
|
+
const jp = jsonlPath(deps.storePath);
|
|
111
|
+
if (existsSync(jp)) {
|
|
112
|
+
const all = readJsonl(jp);
|
|
113
|
+
all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
|
|
114
|
+
return all;
|
|
115
|
+
}
|
|
116
|
+
// Fallback: scan the store
|
|
117
|
+
const filePaths = await deps.store.getFilePaths();
|
|
118
|
+
const quirkPaths = filePaths.filter((fp) => fp.startsWith(QUIRK_FILE_PREFIX));
|
|
119
|
+
const result = [];
|
|
120
|
+
for (const fp of quirkPaths) {
|
|
121
|
+
const chunks = await deps.store.getChunksByFilePath(fp);
|
|
122
|
+
for (const c of chunks) {
|
|
123
|
+
result.push({
|
|
124
|
+
id: c.id,
|
|
125
|
+
content: c.content,
|
|
126
|
+
quirkType: c.metadata.quirkType,
|
|
127
|
+
tags: c.metadata.tags ?? [],
|
|
128
|
+
confidence: c.metadata.confidence ?? 1,
|
|
129
|
+
lastObserved: c.metadata.lastObserved ?? "",
|
|
130
|
+
sourceRef: undefined,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
result.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
|
|
135
|
+
return result;
|
|
136
|
+
}
|
|
137
|
+
const all = [...memQuirks.values()];
|
|
138
|
+
all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
|
|
139
|
+
return all;
|
|
140
|
+
}
|
|
141
|
+
/** Recall quirks matching a query, with confidence re-weighting. */
|
|
142
|
+
export async function recallQuirks(deps, query, options) {
|
|
143
|
+
const topK = options?.topK ?? 10;
|
|
144
|
+
const minConfidence = deps.cfg.memory?.minConfidence ?? 0.5;
|
|
145
|
+
const filter = { kinds: ["quirk"] };
|
|
146
|
+
if (options?.quirkType) {
|
|
147
|
+
filter.languages = [options.quirkType];
|
|
148
|
+
}
|
|
149
|
+
const recallMinScore = deps.cfg.memory?.recallMinScore ?? 0.72;
|
|
150
|
+
const raw = await retrieve(query, deps.embedder, deps.store, {
|
|
151
|
+
topK: topK * 3,
|
|
152
|
+
minScore: recallMinScore,
|
|
153
|
+
keywordIndex: deps.keywordIndex,
|
|
154
|
+
keywordWeight: deps.cfg.retrieval.hybridSearch?.keywordWeight,
|
|
155
|
+
hybridEnabled: false,
|
|
156
|
+
queryPrefix: deps.cfg.embedding.queryPrefix,
|
|
157
|
+
filter,
|
|
158
|
+
});
|
|
159
|
+
// Confidence re-weighting + type/tag filtering
|
|
160
|
+
const filtered = raw.filter((r) => {
|
|
161
|
+
const conf = r.chunk.metadata.confidence ?? 1;
|
|
162
|
+
if (conf < minConfidence)
|
|
163
|
+
return false;
|
|
164
|
+
if (options?.quirkType && r.chunk.metadata.quirkType !== options.quirkType)
|
|
165
|
+
return false;
|
|
166
|
+
if (options?.tags?.length) {
|
|
167
|
+
const chunkTags = r.chunk.metadata.tags ?? [];
|
|
168
|
+
if (!options.tags.some((t) => chunkTags.includes(t)))
|
|
169
|
+
return false;
|
|
170
|
+
}
|
|
171
|
+
return true;
|
|
172
|
+
});
|
|
173
|
+
// Re-weight score by confidence, re-sort
|
|
174
|
+
for (const r of filtered) {
|
|
175
|
+
const conf = r.chunk.metadata.confidence ?? 1;
|
|
176
|
+
r.score = r.score * Math.min(1, Math.max(0.01, conf));
|
|
177
|
+
}
|
|
178
|
+
filtered.sort((a, b) => b.score - a.score);
|
|
179
|
+
return filtered.slice(0, topK);
|
|
180
|
+
}
|
|
181
|
+
/** Lint quirks for low confidence, staleness, duplicates, and orphan source refs. */
|
|
182
|
+
export async function lintQuirks(deps) {
|
|
183
|
+
const issues = [];
|
|
184
|
+
const quirks = await listQuirks(deps);
|
|
185
|
+
const cfg = deps.cfg.memory;
|
|
186
|
+
for (const q of quirks) {
|
|
187
|
+
if (q.confidence < (cfg?.minConfidence ?? 0.5)) {
|
|
188
|
+
issues.push(`Low confidence (${q.confidence}): "${q.content.slice(0, 60)}..." [${q.id}]`);
|
|
189
|
+
}
|
|
190
|
+
if (cfg?.decay?.enabled && q.lastObserved) {
|
|
191
|
+
const ageDays = (Date.now() - new Date(q.lastObserved).getTime()) / 86_400_000;
|
|
192
|
+
const halfLife = cfg.decay.halfLifeDays;
|
|
193
|
+
if (halfLife > 0 && ageDays > halfLife * 2) {
|
|
194
|
+
issues.push(`Stale (${Math.round(ageDays)}d old): "${q.content.slice(0, 60)}..." [${q.id}]`);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
// Duplicate detection (lexically similar content)
|
|
199
|
+
for (let i = 0; i < quirks.length; i++) {
|
|
200
|
+
for (let j = i + 1; j < quirks.length; j++) {
|
|
201
|
+
const sim = lexicalSimilarity(quirks[i].content, quirks[j].content);
|
|
202
|
+
if (sim > 0.85) {
|
|
203
|
+
issues.push(`Near-duplicate (${(sim * 100).toFixed(0)}% similar): "${quirks[i].content.slice(0, 50)}..." ↔ "${quirks[j].content.slice(0, 50)}..."`);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
return issues;
|
|
208
|
+
}
|
|
209
|
+
/** Simple Jaccard-based lexical similarity (word overlap). */
|
|
210
|
+
export function lexicalSimilarity(a, b) {
|
|
211
|
+
const wordsA = new Set(a.toLowerCase().split(/\W+/).filter(Boolean));
|
|
212
|
+
const wordsB = new Set(b.toLowerCase().split(/\W+/).filter(Boolean));
|
|
213
|
+
const intersection = new Set([...wordsA].filter((w) => wordsB.has(w)));
|
|
214
|
+
const union = new Set([...wordsA, ...wordsB]);
|
|
215
|
+
return union.size === 0 ? 0 : intersection.size / union.size;
|
|
216
|
+
}
|
|
217
|
+
//# sourceMappingURL=quirk-store.js.map
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/** Quirk sub-type labels. */
|
|
2
|
+
export type QuirkType = "gotcha" | "preference" | "decision" | "environment-constraint";
|
|
3
|
+
/** Input for adding a new quirk. */
|
|
4
|
+
export interface QuirkInput {
|
|
5
|
+
content: string;
|
|
6
|
+
quirkType?: string;
|
|
7
|
+
tags?: string[];
|
|
8
|
+
confidence?: number;
|
|
9
|
+
sourceRef?: string;
|
|
10
|
+
}
|
|
11
|
+
/** A quirk entry stored in the vectordb and audit log. */
|
|
12
|
+
export interface Quirk {
|
|
13
|
+
id: string;
|
|
14
|
+
content: string;
|
|
15
|
+
quirkType?: string;
|
|
16
|
+
tags: string[];
|
|
17
|
+
confidence: number;
|
|
18
|
+
lastObserved: string;
|
|
19
|
+
sourceRef?: string;
|
|
20
|
+
}
|
|
@@ -184,6 +184,9 @@ export class KeywordIndex {
|
|
|
184
184
|
startLine: chunk.metadata.startLine,
|
|
185
185
|
endLine: chunk.metadata.endLine,
|
|
186
186
|
language: chunk.metadata.language,
|
|
187
|
+
kind: chunk.metadata.kind ?? "",
|
|
188
|
+
quirkType: chunk.metadata.quirkType ?? "",
|
|
189
|
+
tags: chunk.metadata.tags ? JSON.stringify(chunk.metadata.tags) : "",
|
|
187
190
|
},
|
|
188
191
|
])),
|
|
189
192
|
};
|
|
@@ -215,6 +218,14 @@ export class KeywordIndex {
|
|
|
215
218
|
index.invertedIndex.set(token, docMap);
|
|
216
219
|
}
|
|
217
220
|
for (const [id, data] of Object.entries(parsed.chunkMap)) {
|
|
221
|
+
let tags;
|
|
222
|
+
try {
|
|
223
|
+
if (data.tags)
|
|
224
|
+
tags = JSON.parse(data.tags);
|
|
225
|
+
}
|
|
226
|
+
catch {
|
|
227
|
+
tags = undefined;
|
|
228
|
+
}
|
|
218
229
|
index.chunkMap.set(id, {
|
|
219
230
|
id: data.id,
|
|
220
231
|
content: data.content,
|
|
@@ -224,6 +235,9 @@ export class KeywordIndex {
|
|
|
224
235
|
startLine: data.startLine,
|
|
225
236
|
endLine: data.endLine,
|
|
226
237
|
language: data.language,
|
|
238
|
+
kind: data.kind || undefined,
|
|
239
|
+
quirkType: data.quirkType || undefined,
|
|
240
|
+
tags,
|
|
227
241
|
},
|
|
228
242
|
});
|
|
229
243
|
}
|
|
@@ -251,6 +265,8 @@ function matchesFilter(chunk, filter) {
|
|
|
251
265
|
return true;
|
|
252
266
|
if (filter.languages?.length && !filter.languages.includes(chunk.metadata.language))
|
|
253
267
|
return false;
|
|
268
|
+
if (filter.kinds?.length && !filter.kinds.includes(chunk.metadata.kind ?? ""))
|
|
269
|
+
return false;
|
|
254
270
|
if (filter.pathPatterns?.length) {
|
|
255
271
|
return filter.pathPatterns.some((p) => globMatch(p, chunk.metadata.filePath));
|
|
256
272
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { VectorStore, Chunk, SearchResult, MetadataFilter } from "../core/interfaces.js";
|
|
1
|
+
import type { VectorStore, Chunk, ChunkSummary, SearchResult, MetadataFilter } from "../core/interfaces.js";
|
|
2
2
|
/**
|
|
3
3
|
* L2-normalize a vector to unit length. Cosine models require unit vectors
|
|
4
4
|
* for the dot product to equal cosine similarity.
|
|
@@ -40,6 +40,15 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
40
40
|
private getTable;
|
|
41
41
|
private initTable;
|
|
42
42
|
private tableHasDescriptionColumn;
|
|
43
|
+
private hasColumn;
|
|
44
|
+
/** Add kind/quirkType/tags columns if missing from an existing table. */
|
|
45
|
+
private migrateNewColumns;
|
|
46
|
+
/**
|
|
47
|
+
* Ensure that kind/quirkType/tags columns are nullable so that queries
|
|
48
|
+
* with WHERE clauses on these columns do not panic on old fragments
|
|
49
|
+
* written before the columns existed.
|
|
50
|
+
*/
|
|
51
|
+
private ensureColumnsNullable;
|
|
43
52
|
/**
|
|
44
53
|
* Store chunks in the LanceDB table. New rows are inserted first, then
|
|
45
54
|
* any old rows at the same (filePath, startLine) with different IDs are
|
|
@@ -80,15 +89,7 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
80
89
|
* @param limit - Maximum number of rows to return.
|
|
81
90
|
* @returns An array of chunk summaries.
|
|
82
91
|
*/
|
|
83
|
-
getChunks(offset: number, limit: number): Promise<
|
|
84
|
-
id: string;
|
|
85
|
-
filePath: string;
|
|
86
|
-
language: string;
|
|
87
|
-
startLine: number;
|
|
88
|
-
endLine: number;
|
|
89
|
-
content: string;
|
|
90
|
-
description: string;
|
|
91
|
-
}[]>;
|
|
92
|
+
getChunks(offset: number, limit: number): Promise<ChunkSummary[]>;
|
|
92
93
|
/**
|
|
93
94
|
* Perform ANN search internally, returning results scored as cosine similarity (0-1).
|
|
94
95
|
* This method is called by `search()` and handles the actual query logic.
|
|
@@ -99,9 +100,18 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
99
100
|
private searchInternal;
|
|
100
101
|
/**
|
|
101
102
|
* Return the total number of chunks stored in the table.
|
|
102
|
-
*
|
|
103
|
+
* Guarded by a 30s timeout — returns 0 if the store is unresponsive
|
|
104
|
+
* (e.g. corrupted version-manifest accumulation) so the pipeline can
|
|
105
|
+
* proceed with a fresh index instead of hanging forever.
|
|
106
|
+
* @returns The chunk count, or 0 if the table does not exist or times out.
|
|
103
107
|
*/
|
|
104
108
|
count(): Promise<number>;
|
|
109
|
+
/**
|
|
110
|
+
* Compact fragments and prune old version manifests to prevent the
|
|
111
|
+
* version-manifest accumulation that causes countRows() to hang.
|
|
112
|
+
* Should be called at the end of a successful index pass.
|
|
113
|
+
*/
|
|
114
|
+
optimize(): Promise<void>;
|
|
105
115
|
/**
|
|
106
116
|
* Return all unique file paths currently stored in the index.
|
|
107
117
|
* @returns An array of normalized file paths.
|
|
@@ -6,7 +6,7 @@ import fs from "node:fs/promises";
|
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { normalizeFilePath, manifestPathFor } from "../core/manifest.js";
|
|
8
8
|
const TABLE_NAME = "chunks";
|
|
9
|
-
const QUERY_COLUMNS = ["id", "content", "description", "filePath", "startLine", "endLine", "language"];
|
|
9
|
+
const QUERY_COLUMNS = ["id", "content", "description", "filePath", "startLine", "endLine", "language", "kind", "quirkType", "tags"];
|
|
10
10
|
/**
|
|
11
11
|
* L2-normalize a vector to unit length. Cosine models require unit vectors
|
|
12
12
|
* for the dot product to equal cosine similarity.
|
|
@@ -109,12 +109,14 @@ export class LanceDbStore {
|
|
|
109
109
|
if (tableNames.includes(TABLE_NAME)) {
|
|
110
110
|
this.table = await db.openTable(TABLE_NAME);
|
|
111
111
|
if (await this.tableHasDescriptionColumn()) {
|
|
112
|
+
await this.migrateNewColumns();
|
|
112
113
|
return this.table;
|
|
113
114
|
}
|
|
114
115
|
// Schema missing 'description' column -- try to add it gracefully first.
|
|
115
116
|
try {
|
|
116
117
|
await this.table.addColumns([{ name: "description", valueSql: "''" }]);
|
|
117
118
|
console.warn("[lancedb] Added missing 'description' column to existing table.");
|
|
119
|
+
await this.migrateNewColumns();
|
|
118
120
|
return this.table;
|
|
119
121
|
}
|
|
120
122
|
catch {
|
|
@@ -150,6 +152,9 @@ export class LanceDbStore {
|
|
|
150
152
|
startLine: 0,
|
|
151
153
|
endLine: 0,
|
|
152
154
|
language: "",
|
|
155
|
+
kind: "",
|
|
156
|
+
quirkType: "",
|
|
157
|
+
tags: "",
|
|
153
158
|
};
|
|
154
159
|
this.table = await db.createTable({
|
|
155
160
|
name: TABLE_NAME,
|
|
@@ -175,6 +180,58 @@ export class LanceDbStore {
|
|
|
175
180
|
return false;
|
|
176
181
|
}
|
|
177
182
|
}
|
|
183
|
+
async hasColumn(name) {
|
|
184
|
+
try {
|
|
185
|
+
const schema = await this.table.schema();
|
|
186
|
+
return schema.fields.some((f) => f.name === name);
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
return false;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
/** Add kind/quirkType/tags columns if missing from an existing table. */
|
|
193
|
+
async migrateNewColumns() {
|
|
194
|
+
const missing = [];
|
|
195
|
+
for (const col of ["kind", "quirkType", "tags"]) {
|
|
196
|
+
if (!(await this.hasColumn(col))) {
|
|
197
|
+
missing.push({ name: col, valueSql: "''" });
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
if (missing.length > 0) {
|
|
201
|
+
try {
|
|
202
|
+
await this.table.addColumns(missing);
|
|
203
|
+
console.warn(`[lancedb] Added missing columns: ${missing.map((c) => c.name).join(", ")}`);
|
|
204
|
+
}
|
|
205
|
+
catch {
|
|
206
|
+
console.warn("[lancedb] Could not auto-add missing columns. " +
|
|
207
|
+
"Run 'opencode-rag index --force' to rebuild the index with the correct schema.");
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
await this.ensureColumnsNullable(["kind", "quirkType", "tags"]);
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Ensure that kind/quirkType/tags columns are nullable so that queries
|
|
215
|
+
* with WHERE clauses on these columns do not panic on old fragments
|
|
216
|
+
* written before the columns existed.
|
|
217
|
+
*/
|
|
218
|
+
async ensureColumnsNullable(columns) {
|
|
219
|
+
for (const col of columns) {
|
|
220
|
+
if (!(await this.hasColumn(col)))
|
|
221
|
+
continue;
|
|
222
|
+
try {
|
|
223
|
+
const schema = await this.table.schema();
|
|
224
|
+
const field = schema.fields.find((f) => f.name === col);
|
|
225
|
+
if (field && field.nullable === false) {
|
|
226
|
+
await this.table.alterColumns([{ path: col, nullable: true }]);
|
|
227
|
+
console.warn(`[lancedb] Made column '${col}' nullable to avoid null-fragment panics.`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
console.warn(`[lancedb] Could not alter nullability of column '${col}'.`);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
178
235
|
/**
|
|
179
236
|
* Store chunks in the LanceDB table. New rows are inserted first, then
|
|
180
237
|
* any old rows at the same (filePath, startLine) with different IDs are
|
|
@@ -214,6 +271,9 @@ export class LanceDbStore {
|
|
|
214
271
|
startLine: c.metadata.startLine,
|
|
215
272
|
endLine: c.metadata.endLine,
|
|
216
273
|
language: c.metadata.language,
|
|
274
|
+
kind: c.metadata.kind ?? "",
|
|
275
|
+
quirkType: c.metadata.quirkType ?? "",
|
|
276
|
+
tags: c.metadata.tags ? JSON.stringify(c.metadata.tags) : "",
|
|
217
277
|
}));
|
|
218
278
|
if (rows.length === 0)
|
|
219
279
|
return;
|
|
@@ -292,6 +352,15 @@ export class LanceDbStore {
|
|
|
292
352
|
}
|
|
293
353
|
}
|
|
294
354
|
rowToSearchResult(row) {
|
|
355
|
+
let tags;
|
|
356
|
+
try {
|
|
357
|
+
const raw = row.tags;
|
|
358
|
+
if (raw)
|
|
359
|
+
tags = JSON.parse(raw);
|
|
360
|
+
}
|
|
361
|
+
catch {
|
|
362
|
+
tags = undefined;
|
|
363
|
+
}
|
|
295
364
|
return {
|
|
296
365
|
score: Math.min(1, Math.max(0, 1 - (row._distance ?? 0) / 2)),
|
|
297
366
|
chunk: {
|
|
@@ -303,6 +372,9 @@ export class LanceDbStore {
|
|
|
303
372
|
startLine: row.startLine,
|
|
304
373
|
endLine: row.endLine,
|
|
305
374
|
language: row.language,
|
|
375
|
+
kind: row.kind || undefined,
|
|
376
|
+
quirkType: row.quirkType || undefined,
|
|
377
|
+
tags,
|
|
306
378
|
},
|
|
307
379
|
},
|
|
308
380
|
};
|
|
@@ -346,17 +418,31 @@ export class LanceDbStore {
|
|
|
346
418
|
.where(`filePath = '${normalizedPath}'`)
|
|
347
419
|
.toArray();
|
|
348
420
|
return rows
|
|
349
|
-
.map((row) =>
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
}
|
|
359
|
-
|
|
421
|
+
.map((row) => {
|
|
422
|
+
let tags;
|
|
423
|
+
try {
|
|
424
|
+
const raw = row.tags;
|
|
425
|
+
if (raw)
|
|
426
|
+
tags = JSON.parse(raw);
|
|
427
|
+
}
|
|
428
|
+
catch {
|
|
429
|
+
tags = undefined;
|
|
430
|
+
}
|
|
431
|
+
return {
|
|
432
|
+
id: row.id,
|
|
433
|
+
content: row.content,
|
|
434
|
+
description: row.description ?? "",
|
|
435
|
+
metadata: {
|
|
436
|
+
filePath: row.filePath,
|
|
437
|
+
startLine: row.startLine,
|
|
438
|
+
endLine: row.endLine,
|
|
439
|
+
language: row.language,
|
|
440
|
+
kind: row.kind || undefined,
|
|
441
|
+
quirkType: row.quirkType || undefined,
|
|
442
|
+
tags,
|
|
443
|
+
},
|
|
444
|
+
};
|
|
445
|
+
})
|
|
360
446
|
.sort((a, b) => a.metadata.startLine - b.metadata.startLine);
|
|
361
447
|
}
|
|
362
448
|
/**
|
|
@@ -380,6 +466,9 @@ export class LanceDbStore {
|
|
|
380
466
|
endLine: row.endLine,
|
|
381
467
|
content: row.content,
|
|
382
468
|
description: row.description ?? "",
|
|
469
|
+
kind: row.kind ?? "",
|
|
470
|
+
quirkType: row.quirkType ?? "",
|
|
471
|
+
tags: row.tags ?? "",
|
|
383
472
|
}));
|
|
384
473
|
}
|
|
385
474
|
/**
|
|
@@ -398,18 +487,36 @@ export class LanceDbStore {
|
|
|
398
487
|
const count = await table.countRows();
|
|
399
488
|
if (count === 0)
|
|
400
489
|
return [];
|
|
401
|
-
let query = table.vectorSearch(l2Normalize(embedding)).distanceType("cosine");
|
|
402
490
|
const whereClause = buildWhereClause(filter);
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
491
|
+
let results;
|
|
492
|
+
try {
|
|
493
|
+
let query = table.vectorSearch(l2Normalize(embedding)).distanceType("cosine");
|
|
494
|
+
if (whereClause)
|
|
495
|
+
query = query.where(whereClause);
|
|
496
|
+
results = await query.limit(topK).toArray();
|
|
497
|
+
}
|
|
498
|
+
catch (err) {
|
|
499
|
+
if (err instanceof Error && err.message.includes("non-nullable") && filter) {
|
|
500
|
+
const fallbackQuery = table.vectorSearch(l2Normalize(embedding)).distanceType("cosine");
|
|
501
|
+
const rawResults = await fallbackQuery.limit(topK * 5).toArray();
|
|
502
|
+
return rawResults
|
|
503
|
+
.map((row) => this.rowToSearchResult(row))
|
|
504
|
+
.filter((r) => r.chunk.id !== "__seed__")
|
|
505
|
+
.filter((r) => matchesFilterLocal(r.chunk, filter))
|
|
506
|
+
.slice(0, topK);
|
|
507
|
+
}
|
|
508
|
+
throw err;
|
|
509
|
+
}
|
|
406
510
|
return results
|
|
407
511
|
.map((row) => this.rowToSearchResult(row))
|
|
408
512
|
.filter((r) => r.chunk.id !== "__seed__");
|
|
409
513
|
}
|
|
410
514
|
/**
|
|
411
515
|
* Return the total number of chunks stored in the table.
|
|
412
|
-
*
|
|
516
|
+
* Guarded by a 30s timeout — returns 0 if the store is unresponsive
|
|
517
|
+
* (e.g. corrupted version-manifest accumulation) so the pipeline can
|
|
518
|
+
* proceed with a fresh index instead of hanging forever.
|
|
519
|
+
* @returns The chunk count, or 0 if the table does not exist or times out.
|
|
413
520
|
*/
|
|
414
521
|
async count() {
|
|
415
522
|
try {
|
|
@@ -418,12 +525,35 @@ export class LanceDbStore {
|
|
|
418
525
|
if (!tableNames.includes(TABLE_NAME))
|
|
419
526
|
return 0;
|
|
420
527
|
const table = await this.getTable();
|
|
421
|
-
|
|
528
|
+
const COUNT_TIMEOUT_MS = 30_000;
|
|
529
|
+
const result = await Promise.race([
|
|
530
|
+
table.countRows(),
|
|
531
|
+
new Promise((resolve) => setTimeout(() => resolve(null), COUNT_TIMEOUT_MS)),
|
|
532
|
+
]);
|
|
533
|
+
if (result === null) {
|
|
534
|
+
console.warn(`[lancedb] count() timed out after ${COUNT_TIMEOUT_MS / 1000}s — treating store as empty.`);
|
|
535
|
+
return 0;
|
|
536
|
+
}
|
|
537
|
+
return result;
|
|
422
538
|
}
|
|
423
539
|
catch {
|
|
424
540
|
return 0;
|
|
425
541
|
}
|
|
426
542
|
}
|
|
543
|
+
/**
|
|
544
|
+
* Compact fragments and prune old version manifests to prevent the
|
|
545
|
+
* version-manifest accumulation that causes countRows() to hang.
|
|
546
|
+
* Should be called at the end of a successful index pass.
|
|
547
|
+
*/
|
|
548
|
+
async optimize() {
|
|
549
|
+
try {
|
|
550
|
+
const table = await this.getTable();
|
|
551
|
+
await table.optimize({ cleanupOlderThan: new Date(), deleteUnverified: false });
|
|
552
|
+
}
|
|
553
|
+
catch {
|
|
554
|
+
// Optimize is best-effort — must not break indexing.
|
|
555
|
+
}
|
|
556
|
+
}
|
|
427
557
|
/**
|
|
428
558
|
* Return all unique file paths currently stored in the index.
|
|
429
559
|
* @returns An array of normalized file paths.
|
|
@@ -641,6 +771,10 @@ function buildWhereClause(filter) {
|
|
|
641
771
|
const langs = filter.languages.map((l) => `'${l.replace(/'/g, "''")}'`).join(",");
|
|
642
772
|
parts.push(`language IN (${langs})`);
|
|
643
773
|
}
|
|
774
|
+
if (filter.kinds?.length) {
|
|
775
|
+
const kinds = filter.kinds.map((k) => `'${k.replace(/'/g, "''")}'`).join(",");
|
|
776
|
+
parts.push(`kind IN (${kinds})`);
|
|
777
|
+
}
|
|
644
778
|
if (filter.pathPatterns?.length) {
|
|
645
779
|
const likes = filter.pathPatterns.map((p) => {
|
|
646
780
|
const escaped = p.replace(/'/g, "''");
|
|
@@ -654,4 +788,26 @@ function buildWhereClause(filter) {
|
|
|
654
788
|
}
|
|
655
789
|
return parts.length ? parts.join(" AND ") : undefined;
|
|
656
790
|
}
|
|
791
|
+
/** Client-side metadata filter for use as a fallback when LanceDB WHERE panics. */
|
|
792
|
+
function matchesFilterLocal(chunk, filter) {
|
|
793
|
+
if (!filter)
|
|
794
|
+
return true;
|
|
795
|
+
if (filter.languages?.length && !filter.languages.includes(chunk.metadata.language))
|
|
796
|
+
return false;
|
|
797
|
+
if (filter.kinds?.length && !filter.kinds.includes(chunk.metadata.kind ?? ""))
|
|
798
|
+
return false;
|
|
799
|
+
if (filter.pathPatterns?.length) {
|
|
800
|
+
return filter.pathPatterns.some((p) => globMatchLocal(p, chunk.metadata.filePath));
|
|
801
|
+
}
|
|
802
|
+
return true;
|
|
803
|
+
}
|
|
804
|
+
function globMatchLocal(pattern, filePath) {
|
|
805
|
+
const GLOBSTAR = "\x00GS\x00";
|
|
806
|
+
let re = pattern.replace(/\*\*/g, GLOBSTAR);
|
|
807
|
+
re = re.replace(/([.+^${}()|[\]\\])/g, "\\$1");
|
|
808
|
+
re = re.replace(new RegExp(GLOBSTAR, "g"), ".*");
|
|
809
|
+
re = re.replace(/\*/g, "[^/]*");
|
|
810
|
+
re = re.replace(/\?/g, ".");
|
|
811
|
+
return new RegExp("^" + re + "$").test(filePath);
|
|
812
|
+
}
|
|
657
813
|
//# sourceMappingURL=lancedb.js.map
|
|
@@ -21,4 +21,6 @@ export declare class InMemoryVectorStore implements VectorStore {
|
|
|
21
21
|
getChunksByFilePath(filePath: string): Promise<Chunk[]>;
|
|
22
22
|
/** Release any held resources. No-op for the in-memory store. */
|
|
23
23
|
close(): Promise<void>;
|
|
24
|
+
/** No-op for the in-memory store. */
|
|
25
|
+
optimize(): Promise<void>;
|
|
24
26
|
}
|
|
@@ -45,6 +45,9 @@ export class InMemoryVectorStore {
|
|
|
45
45
|
endLine: c.metadata.endLine,
|
|
46
46
|
content: c.content,
|
|
47
47
|
description: c.description ?? "",
|
|
48
|
+
kind: c.metadata.kind ?? "",
|
|
49
|
+
quirkType: c.metadata.quirkType ?? "",
|
|
50
|
+
tags: c.metadata.tags ? JSON.stringify(c.metadata.tags) : "",
|
|
48
51
|
}));
|
|
49
52
|
}
|
|
50
53
|
async listFiles() {
|
|
@@ -70,6 +73,9 @@ export class InMemoryVectorStore {
|
|
|
70
73
|
/** Release any held resources. No-op for the in-memory store. */
|
|
71
74
|
async close() {
|
|
72
75
|
}
|
|
76
|
+
/** No-op for the in-memory store. */
|
|
77
|
+
async optimize() {
|
|
78
|
+
}
|
|
73
79
|
}
|
|
74
80
|
/**
|
|
75
81
|
* Compute the cosine similarity between two equal-length vectors.
|