opencode-rag-plugin 1.19.4 → 1.19.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunker/base.js +19 -5
- package/dist/chunker/factory.js +27 -9
- package/dist/chunker/grammar.d.ts +18 -1
- package/dist/chunker/grammar.js +48 -10
- package/dist/chunker/image.js +5 -0
- package/dist/chunker/pdf.js +30 -14
- package/dist/cli/commands/backend-detect.d.ts +61 -0
- package/dist/cli/commands/backend-detect.js +119 -0
- package/dist/cli/commands/index-command.js +11 -0
- package/dist/cli/commands/init-helpers.d.ts +4 -1
- package/dist/cli/commands/init-helpers.js +21 -4
- package/dist/cli/commands/init.js +61 -24
- package/dist/cli/commands/quirk.js +9 -3
- package/dist/cli/commands/setup.js +12 -3
- package/dist/cli/commands/status.js +14 -5
- package/dist/cli/commands/ui.js +23 -9
- package/dist/cli/commands/update.js +4 -5
- package/dist/cli/format.d.ts +5 -2
- package/dist/cli/format.js +14 -5
- package/dist/content/image.js +33 -11
- package/dist/content/reader.js +74 -13
- package/dist/core/bootstrap.js +10 -3
- package/dist/core/config.d.ts +27 -1
- package/dist/core/config.js +41 -2
- package/dist/core/desc-cache.d.ts +8 -2
- package/dist/core/desc-cache.js +10 -3
- package/dist/core/doc-progress.js +5 -2
- package/dist/core/interfaces.d.ts +32 -4
- package/dist/core/manifest.js +1 -1
- package/dist/core/provider-defaults.d.ts +2 -0
- package/dist/core/provider-defaults.js +19 -4
- package/dist/core/runtime-overrides.d.ts +0 -6
- package/dist/core/version-check.d.ts +5 -0
- package/dist/core/version-check.js +8 -2
- package/dist/describer/anthropic.d.ts +2 -2
- package/dist/describer/anthropic.js +19 -5
- package/dist/describer/describer.d.ts +20 -0
- package/dist/describer/describer.js +132 -16
- package/dist/describer/gemini.js +25 -10
- package/dist/describer/shared.d.ts +28 -0
- package/dist/describer/shared.js +60 -0
- package/dist/embedder/factory.d.ts +5 -3
- package/dist/embedder/factory.js +42 -9
- package/dist/embedder/health.js +19 -19
- package/dist/embedder/http.d.ts +14 -1
- package/dist/embedder/http.js +60 -6
- package/dist/embedder/ollama.d.ts +3 -1
- package/dist/embedder/ollama.js +12 -2
- package/dist/eval/session-logger.js +7 -0
- package/dist/eval/storage.js +8 -0
- package/dist/indexer/git-diff.d.ts +1 -1
- package/dist/indexer/git-diff.js +5 -1
- package/dist/indexer/pipeline.js +511 -346
- package/dist/indexer/stats.d.ts +2 -0
- package/dist/indexer/stats.js +1 -0
- package/dist/indexer/watch.js +8 -1
- package/dist/indexer/worker.js +21 -0
- package/dist/mcp/cli.js +4 -0
- package/dist/mcp/handlers.js +16 -5
- package/dist/mcp/server.js +3 -0
- package/dist/opencode/create-read-tool.js +14 -3
- package/dist/opencode/tool-args.js +23 -1
- package/dist/plugin.js +66 -152
- package/dist/quirks/auto-capture.js +5 -0
- package/dist/quirks/quirk-store.d.ts +1 -1
- package/dist/quirks/quirk-store.js +56 -17
- package/dist/retriever/context-optimizer.js +18 -4
- package/dist/retriever/keyword-index.d.ts +2 -0
- package/dist/retriever/keyword-index.js +38 -4
- package/dist/retriever/retriever.js +6 -1
- package/dist/tui.js +41 -4
- package/dist/vectorstore/lancedb.d.ts +104 -8
- package/dist/vectorstore/lancedb.js +345 -71
- package/dist/vectorstore/memory.d.ts +8 -3
- package/dist/vectorstore/memory.js +27 -4
- package/dist/watcher.d.ts +8 -0
- package/dist/watcher.js +237 -85
- package/dist/web/api.d.ts +5 -1
- package/dist/web/api.js +195 -69
- package/dist/web/server.d.ts +2 -0
- package/dist/web/server.js +66 -28
- package/dist/web/static.d.ts +5 -2
- package/dist/web/static.js +9 -5
- package/dist/web/ui/assets/index-BDPYdtA1.js +3 -0
- package/dist/web/ui/index.html +1 -1
- package/package.json +1 -1
- package/dist/web/ui/assets/index-CJBvt6e0.js +0 -3
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { appendFileSync, existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
|
|
1
|
+
import { appendFileSync, existsSync, readFileSync, writeFileSync, mkdirSync, renameSync } from "node:fs";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { retrieve } from "../retriever/retriever.js";
|
|
@@ -11,15 +11,33 @@ function jsonlPath(storePath) {
|
|
|
11
11
|
function isMemoryStore(storePath) {
|
|
12
12
|
return storePath.startsWith("memory:");
|
|
13
13
|
}
|
|
14
|
-
/** In-memory backup for memory:// stores. */
|
|
15
|
-
const
|
|
14
|
+
/** In-memory backup for memory:// stores, keyed by store path. */
|
|
15
|
+
const memQuirksByStore = new Map();
|
|
16
|
+
function memQuirksFor(storePath) {
|
|
17
|
+
let store = memQuirksByStore.get(storePath);
|
|
18
|
+
if (!store) {
|
|
19
|
+
store = new Map();
|
|
20
|
+
memQuirksByStore.set(storePath, store);
|
|
21
|
+
}
|
|
22
|
+
return store;
|
|
23
|
+
}
|
|
16
24
|
function readJsonl(filePath) {
|
|
17
25
|
if (!existsSync(filePath))
|
|
18
26
|
return [];
|
|
19
27
|
const raw = readFileSync(filePath, "utf-8").trim();
|
|
20
28
|
if (!raw)
|
|
21
29
|
return [];
|
|
22
|
-
|
|
30
|
+
const quirks = [];
|
|
31
|
+
for (const line of raw.split("\n")) {
|
|
32
|
+
try {
|
|
33
|
+
quirks.push(JSON.parse(line));
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
// Skip corrupt lines — a single bad line must never break
|
|
37
|
+
// every quirk operation (readJsonl feeds get/update/remove/list).
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return quirks;
|
|
23
41
|
}
|
|
24
42
|
function appendJsonl(filePath, q) {
|
|
25
43
|
const dir = path.dirname(filePath);
|
|
@@ -33,7 +51,24 @@ function rewriteJsonl(filePath, quirks) {
|
|
|
33
51
|
if (!existsSync(dir)) {
|
|
34
52
|
mkdirSync(dir, { recursive: true });
|
|
35
53
|
}
|
|
36
|
-
|
|
54
|
+
// Atomic write: tmp file + rename, so a crash or concurrent process
|
|
55
|
+
// can never leave a truncated/empty quirks.jsonl behind.
|
|
56
|
+
const tmpPath = `${filePath}.tmp`;
|
|
57
|
+
writeFileSync(tmpPath, quirks.map((q) => JSON.stringify(q)).join("\n") + "\n", "utf-8");
|
|
58
|
+
try {
|
|
59
|
+
renameSync(tmpPath, filePath);
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
// Windows: rename can fail with EPERM when another process holds the file;
|
|
63
|
+
// fall back to a direct write (best-effort) rather than losing data.
|
|
64
|
+
writeFileSync(filePath, quirks.map((q) => JSON.stringify(q)).join("\n") + "\n", "utf-8");
|
|
65
|
+
try {
|
|
66
|
+
renameSync(tmpPath, `${filePath}.bak`);
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
// ignore
|
|
70
|
+
}
|
|
71
|
+
}
|
|
37
72
|
}
|
|
38
73
|
function nowISO() {
|
|
39
74
|
return new Date().toISOString();
|
|
@@ -62,7 +97,7 @@ export async function getQuirk(deps, id) {
|
|
|
62
97
|
}
|
|
63
98
|
return undefined;
|
|
64
99
|
}
|
|
65
|
-
return
|
|
100
|
+
return memQuirksFor(deps.storePath).get(id);
|
|
66
101
|
}
|
|
67
102
|
/**
|
|
68
103
|
* Update an existing quirk by ID. Fields in `patch` override the stored values.
|
|
@@ -100,8 +135,9 @@ export async function updateQuirk(deps, id, patch) {
|
|
|
100
135
|
sourceRef,
|
|
101
136
|
};
|
|
102
137
|
const filePath = QUIRK_FILE_PREFIX + id;
|
|
103
|
-
|
|
104
|
-
|
|
138
|
+
// Embed FIRST — if embedding fails, the old index entries are untouched
|
|
139
|
+
// (previously the delete happened before the embed, permanently removing
|
|
140
|
+
// the quirk from search while the JSONL still listed it).
|
|
105
141
|
const prefix = deps.cfg.embedding.documentPrefix ?? "";
|
|
106
142
|
const chunkContent = prefix + content;
|
|
107
143
|
const embeddings = await deps.embedder.embed([chunkContent], "document");
|
|
@@ -109,6 +145,8 @@ export async function updateQuirk(deps, id, patch) {
|
|
|
109
145
|
if (!embedding || embedding.length === 0) {
|
|
110
146
|
throw new Error("Embedding returned empty vector for quirk content");
|
|
111
147
|
}
|
|
148
|
+
await deps.store.deleteByFilePath(filePath);
|
|
149
|
+
deps.keywordIndex.removeByFilePath(filePath);
|
|
112
150
|
const chunk = {
|
|
113
151
|
id,
|
|
114
152
|
content,
|
|
@@ -134,7 +172,7 @@ export async function updateQuirk(deps, id, patch) {
|
|
|
134
172
|
rewriteJsonl(jp, all);
|
|
135
173
|
}
|
|
136
174
|
else {
|
|
137
|
-
|
|
175
|
+
memQuirksFor(deps.storePath).set(id, updated);
|
|
138
176
|
}
|
|
139
177
|
return updated;
|
|
140
178
|
}
|
|
@@ -186,7 +224,7 @@ export async function addQuirk(deps, input) {
|
|
|
186
224
|
appendJsonl(jsonlPath(deps.storePath), quirk);
|
|
187
225
|
}
|
|
188
226
|
else {
|
|
189
|
-
|
|
227
|
+
memQuirksFor(deps.storePath).set(id, quirk);
|
|
190
228
|
}
|
|
191
229
|
return quirk;
|
|
192
230
|
}
|
|
@@ -205,7 +243,7 @@ export async function removeQuirk(deps, id) {
|
|
|
205
243
|
rewriteJsonl(jp, all);
|
|
206
244
|
}
|
|
207
245
|
else {
|
|
208
|
-
|
|
246
|
+
memQuirksFor(deps.storePath).delete(id);
|
|
209
247
|
}
|
|
210
248
|
}
|
|
211
249
|
/** List all quirks sorted by lastObserved descending. */
|
|
@@ -238,7 +276,7 @@ export async function listQuirks(deps) {
|
|
|
238
276
|
result.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
|
|
239
277
|
return result;
|
|
240
278
|
}
|
|
241
|
-
const all = [...
|
|
279
|
+
const all = [...memQuirksFor(deps.storePath).values()];
|
|
242
280
|
all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
|
|
243
281
|
return all;
|
|
244
282
|
}
|
|
@@ -246,10 +284,11 @@ export async function listQuirks(deps) {
|
|
|
246
284
|
export async function recallQuirks(deps, query, options) {
|
|
247
285
|
const topK = options?.topK ?? 10;
|
|
248
286
|
const minConfidence = deps.cfg.memory?.minConfidence ?? 0.5;
|
|
287
|
+
// NOTE: do NOT put quirkType into `filter.languages` — quirk chunks store
|
|
288
|
+
// the type in metadata.quirkType, not metadata.language ("quirk").
|
|
289
|
+
// A languages filter would exclude every quirk chunk and return zero hits.
|
|
290
|
+
// Type filtering happens in the post-filter below instead.
|
|
249
291
|
const filter = { kinds: ["quirk"] };
|
|
250
|
-
if (options?.quirkType) {
|
|
251
|
-
filter.languages = [options.quirkType];
|
|
252
|
-
}
|
|
253
292
|
const recallMinScore = options?.minScore ?? deps.cfg.memory?.recallMinScore ?? 0.72;
|
|
254
293
|
const raw = await retrieve(query, deps.embedder, deps.store, {
|
|
255
294
|
topK: topK * 3,
|
|
@@ -304,7 +343,7 @@ export async function lintQuirks(deps) {
|
|
|
304
343
|
for (let j = i + 1; j < quirks.length; j++) {
|
|
305
344
|
const sim = lexicalSimilarity(quirks[i].content, quirks[j].content);
|
|
306
345
|
if (sim > 0.85) {
|
|
307
|
-
issues.push(`Near-duplicate (${(sim * 100).toFixed(0)}% similar): "${quirks[i].content}"
|
|
346
|
+
issues.push(`Near-duplicate (${(sim * 100).toFixed(0)}% similar): "${quirks[i].content}" ↔ "${quirks[j].content}"`);
|
|
308
347
|
}
|
|
309
348
|
}
|
|
310
349
|
}
|
|
@@ -322,7 +361,7 @@ export function lexicalSimilarity(a, b) {
|
|
|
322
361
|
* Count of meaningful word tokens shared between two texts (Jaccard numerator).
|
|
323
362
|
*
|
|
324
363
|
* Tokens are whitespace/punctuation-split, lowercased, and filtered to those
|
|
325
|
-
* with length
|
|
364
|
+
* with length ≥ `minTokenLen` (default 3 — skips short filler like "the").
|
|
326
365
|
*
|
|
327
366
|
* Used by the quirk auto-inject gate: candidate quirks that share no tokens
|
|
328
367
|
* with the user's *current* message (i.e. they matched only against the prior
|
|
@@ -26,10 +26,20 @@ function toOptimized(r) {
|
|
|
26
26
|
}
|
|
27
27
|
/**
|
|
28
28
|
* Compute Jaccard similarity between two strings based on their token sets.
|
|
29
|
+
* Accepts a per-call token cache so repeated comparisons (the dedup loop is
|
|
30
|
+
* O(n²) over same-file pairs) don't re-tokenize the same content every time.
|
|
29
31
|
*/
|
|
30
|
-
function jaccardSimilarity(a, b) {
|
|
31
|
-
|
|
32
|
-
|
|
32
|
+
function jaccardSimilarity(a, b, tokenCache) {
|
|
33
|
+
let tokensA = tokenCache.get(a);
|
|
34
|
+
if (!tokensA) {
|
|
35
|
+
tokensA = new Set(tokenize(a));
|
|
36
|
+
tokenCache.set(a, tokensA);
|
|
37
|
+
}
|
|
38
|
+
let tokensB = tokenCache.get(b);
|
|
39
|
+
if (!tokensB) {
|
|
40
|
+
tokensB = new Set(tokenize(b));
|
|
41
|
+
tokenCache.set(b, tokensB);
|
|
42
|
+
}
|
|
33
43
|
if (tokensA.size === 0 && tokensB.size === 0)
|
|
34
44
|
return 1;
|
|
35
45
|
let intersection = 0;
|
|
@@ -99,12 +109,16 @@ function dedupeSimilar(results, threshold) {
|
|
|
99
109
|
if (results.length <= 1)
|
|
100
110
|
return results;
|
|
101
111
|
const kept = [...results];
|
|
112
|
+
// Token cache shared across all pair comparisons of this group — the
|
|
113
|
+
// dedup loop is O(n²) over pairs and was re-tokenizing full chunk
|
|
114
|
+
// contents on every comparison.
|
|
115
|
+
const tokenCache = new Map();
|
|
102
116
|
let changed = true;
|
|
103
117
|
while (changed) {
|
|
104
118
|
changed = false;
|
|
105
119
|
for (let i = 0; i < kept.length; i++) {
|
|
106
120
|
for (let j = i + 1; j < kept.length; j++) {
|
|
107
|
-
const sim = jaccardSimilarity(kept[i].chunk.content, kept[j].chunk.content);
|
|
121
|
+
const sim = jaccardSimilarity(kept[i].chunk.content, kept[j].chunk.content, tokenCache);
|
|
108
122
|
if (sim > threshold) {
|
|
109
123
|
const [keepIdx, removeIdx] = kept[i].score >= kept[j].score ? [i, j] : [j, i];
|
|
110
124
|
const removedId = kept[removeIdx].chunk.id;
|
|
@@ -18,6 +18,8 @@ export declare class KeywordIndex {
|
|
|
18
18
|
private invertedIndex;
|
|
19
19
|
private chunkMap;
|
|
20
20
|
private readonly storePath?;
|
|
21
|
+
/** filePath (normalized) → chunk IDs, for O(chunks-in-file) removal. */
|
|
22
|
+
private fileToIds;
|
|
21
23
|
constructor(storePath?: string);
|
|
22
24
|
addChunks(chunks: Chunk[]): void;
|
|
23
25
|
getMatchedTerms(query: string, chunkId: string): string[];
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { normalizeFilePath } from "../core/manifest.js";
|
|
1
2
|
const INDEX_VERSION = 2;
|
|
2
3
|
/** Suffix-stripping stemmer. Only stems words >= 6 characters to reduce false positives. */
|
|
3
4
|
function stem(word) {
|
|
@@ -71,6 +72,8 @@ export class KeywordIndex {
|
|
|
71
72
|
invertedIndex = new Map();
|
|
72
73
|
chunkMap = new Map();
|
|
73
74
|
storePath;
|
|
75
|
+
/** filePath (normalized) → chunk IDs, for O(chunks-in-file) removal. */
|
|
76
|
+
fileToIds = new Map();
|
|
74
77
|
constructor(storePath) {
|
|
75
78
|
this.storePath = storePath;
|
|
76
79
|
}
|
|
@@ -78,6 +81,13 @@ export class KeywordIndex {
|
|
|
78
81
|
for (const chunk of chunks) {
|
|
79
82
|
const id = chunk.id;
|
|
80
83
|
this.chunkMap.set(id, chunk);
|
|
84
|
+
const fileKey = normalizeFilePath(chunk.metadata.filePath);
|
|
85
|
+
let ids = this.fileToIds.get(fileKey);
|
|
86
|
+
if (!ids) {
|
|
87
|
+
ids = new Set();
|
|
88
|
+
this.fileToIds.set(fileKey, ids);
|
|
89
|
+
}
|
|
90
|
+
ids.add(id);
|
|
81
91
|
const tokens = tokenize(chunk.content);
|
|
82
92
|
for (const token of tokens) {
|
|
83
93
|
let docs = this.invertedIndex.get(token);
|
|
@@ -101,10 +111,21 @@ export class KeywordIndex {
|
|
|
101
111
|
return matched;
|
|
102
112
|
}
|
|
103
113
|
removeByFilePath(filePath) {
|
|
104
|
-
const
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
114
|
+
const fileKey = normalizeFilePath(filePath);
|
|
115
|
+
// Fast path: per-file id index. Falls back to a full chunkMap scan when
|
|
116
|
+
// the path is not tracked (entries loaded from an older persisted index).
|
|
117
|
+
let idsToRemove;
|
|
118
|
+
const tracked = this.fileToIds.get(fileKey);
|
|
119
|
+
if (tracked) {
|
|
120
|
+
idsToRemove = [...tracked];
|
|
121
|
+
this.fileToIds.delete(fileKey);
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
idsToRemove = [];
|
|
125
|
+
for (const [id, chunk] of this.chunkMap) {
|
|
126
|
+
if (normalizeFilePath(chunk.metadata.filePath) === fileKey) {
|
|
127
|
+
idsToRemove.push(id);
|
|
128
|
+
}
|
|
108
129
|
}
|
|
109
130
|
}
|
|
110
131
|
for (const id of idsToRemove) {
|
|
@@ -155,10 +176,12 @@ export class KeywordIndex {
|
|
|
155
176
|
close() {
|
|
156
177
|
this.invertedIndex.clear();
|
|
157
178
|
this.chunkMap.clear();
|
|
179
|
+
this.fileToIds.clear();
|
|
158
180
|
}
|
|
159
181
|
clear() {
|
|
160
182
|
this.invertedIndex.clear();
|
|
161
183
|
this.chunkMap.clear();
|
|
184
|
+
this.fileToIds.clear();
|
|
162
185
|
}
|
|
163
186
|
count() {
|
|
164
187
|
return this.chunkMap.size;
|
|
@@ -241,6 +264,17 @@ export class KeywordIndex {
|
|
|
241
264
|
},
|
|
242
265
|
});
|
|
243
266
|
}
|
|
267
|
+
// Rebuild the per-file id index from the loaded chunk map so removals
|
|
268
|
+
// work even for entries persisted by older versions.
|
|
269
|
+
for (const [id, chunk] of index.chunkMap) {
|
|
270
|
+
const fileKey = normalizeFilePath(chunk.metadata.filePath);
|
|
271
|
+
let ids = index.fileToIds.get(fileKey);
|
|
272
|
+
if (!ids) {
|
|
273
|
+
ids = new Set();
|
|
274
|
+
index.fileToIds.set(fileKey, ids);
|
|
275
|
+
}
|
|
276
|
+
ids.add(id);
|
|
277
|
+
}
|
|
244
278
|
return index;
|
|
245
279
|
}
|
|
246
280
|
static async clearFile(storePath) {
|
|
@@ -96,7 +96,12 @@ export async function retrieve(query, embedder, store, options = {}) {
|
|
|
96
96
|
.slice(0, topK);
|
|
97
97
|
return combinedResults;
|
|
98
98
|
}
|
|
99
|
-
catch {
|
|
99
|
+
catch (err) {
|
|
100
|
+
// Never silently mask retrieval failures as "no results" — an embedder
|
|
101
|
+
// outage or store bug must be visible, not indistinguishable from an
|
|
102
|
+
// empty index.
|
|
103
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
104
|
+
console.warn(`[retriever] retrieve() failed (returning []): ${message}`);
|
|
100
105
|
return [];
|
|
101
106
|
}
|
|
102
107
|
}
|
package/dist/tui.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* settings dialog for editing config values, and model selection picker.
|
|
4
4
|
*/
|
|
5
5
|
import { createElement, insert, setProp } from "@opentui/solid";
|
|
6
|
-
import { readFileSync, existsSync, writeFileSync } from "node:fs";
|
|
6
|
+
import { readFileSync, existsSync, writeFileSync, unlinkSync } from "node:fs";
|
|
7
7
|
import { dirname, join, resolve } from "node:path";
|
|
8
8
|
import { fileURLToPath } from "node:url";
|
|
9
9
|
import { loadRuntimeOverrides, saveRuntimeOverride } from "./core/runtime-overrides.js";
|
|
@@ -277,7 +277,10 @@ function resolveProviderBaseUrl(provider) {
|
|
|
277
277
|
*/
|
|
278
278
|
function saveConfigValue(configPath, path, value) {
|
|
279
279
|
try {
|
|
280
|
-
|
|
280
|
+
let raw = readFileSync(configPath, "utf-8");
|
|
281
|
+
if (raw.charCodeAt(0) === 0xfeff)
|
|
282
|
+
raw = raw.slice(1);
|
|
283
|
+
const data = JSON.parse(raw);
|
|
281
284
|
let target = data;
|
|
282
285
|
for (let i = 0; i < path.length - 1; i++) {
|
|
283
286
|
const key = path[i];
|
|
@@ -292,6 +295,26 @@ function saveConfigValue(configPath, path, value) {
|
|
|
292
295
|
catch {
|
|
293
296
|
}
|
|
294
297
|
}
|
|
298
|
+
/**
|
|
299
|
+
* Mirror the watcher status file that the server plugin maintains, so the
|
|
300
|
+
* sidebar reflects a watcher toggle from the settings dialog immediately:
|
|
301
|
+
* - enabled → write the same initial status the background indexer writes on startup
|
|
302
|
+
* - disabled → remove the file (same cleanup the plugin does when auto-index is off)
|
|
303
|
+
*/
|
|
304
|
+
function syncWatcherStatusFile(storePath, enabled) {
|
|
305
|
+
const statusPath = join(storePath, "watcher-status.json");
|
|
306
|
+
try {
|
|
307
|
+
if (enabled) {
|
|
308
|
+
writeFileSync(statusPath, JSON.stringify({ running: false }, null, 2), "utf-8");
|
|
309
|
+
}
|
|
310
|
+
else if (existsSync(statusPath)) {
|
|
311
|
+
unlinkSync(statusPath);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
catch {
|
|
315
|
+
// best-effort mirror; the server plugin reconciles on its next load
|
|
316
|
+
}
|
|
317
|
+
}
|
|
295
318
|
/**
|
|
296
319
|
* Save a model selection to both runtime overrides and the config file.
|
|
297
320
|
* Resolves the RAG provider name and API base URL automatically.
|
|
@@ -320,8 +343,9 @@ function saveModelSelection(storePath, configPath, selectionValue, path, provide
|
|
|
320
343
|
}
|
|
321
344
|
const apiKey = provider?.options?.apiKey ?? "";
|
|
322
345
|
if (apiKey) {
|
|
346
|
+
// Persist the key only in the store-dir overrides, never in the workspace
|
|
347
|
+
// config file — `opencode-rag.json` is routinely committed to git.
|
|
323
348
|
saveRuntimeOverride(storePath, [section, "apiKey"], apiKey);
|
|
324
|
-
saveConfigValue(configPath, [section, "apiKey"], apiKey);
|
|
325
349
|
}
|
|
326
350
|
return selectionValue;
|
|
327
351
|
}
|
|
@@ -689,7 +713,12 @@ async function openSettingsDialog(api) {
|
|
|
689
713
|
if (entry.path[0] === "indexing" || entry.path[0] === "chunking") {
|
|
690
714
|
api.ui.toast({ variant: "warning", title: "Settings", message: "Chunking changed. Re-index required." });
|
|
691
715
|
}
|
|
716
|
+
if (entry.path.join(".") === "openCode.autoIndex.enabled") {
|
|
717
|
+
syncWatcherStatusFile(storePath, newVal);
|
|
718
|
+
api.ui.toast({ variant: "warning", title: "Settings", message: "Watcher changes take effect after an OpenCode restart" });
|
|
719
|
+
}
|
|
692
720
|
entry.currentValue = newVal;
|
|
721
|
+
api.onSettingsChanged?.();
|
|
693
722
|
showSettingMenu(cat);
|
|
694
723
|
}
|
|
695
724
|
else if (entry.type === "number") {
|
|
@@ -711,6 +740,7 @@ async function openSettingsDialog(api) {
|
|
|
711
740
|
api.ui.toast({ variant: "warning", title: "Settings", message: "Chunking changed. Re-index required." });
|
|
712
741
|
}
|
|
713
742
|
entry.currentValue = num;
|
|
743
|
+
api.onSettingsChanged?.();
|
|
714
744
|
}
|
|
715
745
|
showSettingMenu(cat);
|
|
716
746
|
},
|
|
@@ -743,6 +773,7 @@ async function openSettingsDialog(api) {
|
|
|
743
773
|
api.ui.toast({ variant: "warning", title: "Settings", message: "Chunking changed. Re-index required." });
|
|
744
774
|
}
|
|
745
775
|
entry.currentValue = parsed;
|
|
776
|
+
api.onSettingsChanged?.();
|
|
746
777
|
}
|
|
747
778
|
catch {
|
|
748
779
|
api.ui.toast({ variant: "error", title: "Settings", message: "Invalid JSON" });
|
|
@@ -768,6 +799,7 @@ async function openSettingsDialog(api) {
|
|
|
768
799
|
message: `${entry.label}: ${input}`,
|
|
769
800
|
});
|
|
770
801
|
entry.currentValue = input;
|
|
802
|
+
api.onSettingsChanged?.();
|
|
771
803
|
showSettingMenu(cat);
|
|
772
804
|
},
|
|
773
805
|
onCancel: () => {
|
|
@@ -799,6 +831,7 @@ async function openSettingsDialog(api) {
|
|
|
799
831
|
saveConfigValue(configPath, entry.path, input);
|
|
800
832
|
entry.currentValue = input;
|
|
801
833
|
}
|
|
834
|
+
api.onSettingsChanged?.();
|
|
802
835
|
showSettingMenu(cat);
|
|
803
836
|
},
|
|
804
837
|
onCancel: () => showSettingMenu(cat),
|
|
@@ -821,6 +854,7 @@ async function openSettingsDialog(api) {
|
|
|
821
854
|
message: "Embedding changed. Re-index may be required. Restart OpenCode for changes.",
|
|
822
855
|
});
|
|
823
856
|
}
|
|
857
|
+
api.onSettingsChanged?.();
|
|
824
858
|
}
|
|
825
859
|
showSettingMenu(cat);
|
|
826
860
|
},
|
|
@@ -840,7 +874,7 @@ const plugin = {
|
|
|
840
874
|
const version = meta.version ?? getVersion();
|
|
841
875
|
let cachedStatus = DEFAULT_STATUS;
|
|
842
876
|
let lastRefresh = 0;
|
|
843
|
-
const REFRESH_INTERVAL_MS = Number(process.env.OPENCODE_RAG_TUI_REFRESH_MS) ||
|
|
877
|
+
const REFRESH_INTERVAL_MS = Number(process.env.OPENCODE_RAG_TUI_REFRESH_MS) || 30000;
|
|
844
878
|
// Load tui config for keybinding display
|
|
845
879
|
let tuiConfig;
|
|
846
880
|
const worktree = api.state.path.worktree;
|
|
@@ -948,6 +982,9 @@ const plugin = {
|
|
|
948
982
|
openSettingsDialog({
|
|
949
983
|
ui: api.ui,
|
|
950
984
|
state: api.state,
|
|
985
|
+
onSettingsChanged: () => {
|
|
986
|
+
refreshStatus();
|
|
987
|
+
},
|
|
951
988
|
});
|
|
952
989
|
return undefined;
|
|
953
990
|
},
|
|
@@ -32,6 +32,16 @@ export declare function isTransientConflictError(err: unknown): boolean;
|
|
|
32
32
|
* @param realPath - Path to the current store (destination, will be replaced).
|
|
33
33
|
*/
|
|
34
34
|
export declare function swapStoreDirectories(tempPath: string, realPath: string): Promise<void>;
|
|
35
|
+
/**
|
|
36
|
+
* A single file's chunk payload for a bulk store write.
|
|
37
|
+
* `dedup: true` removes prior-revision rows for the same file path that are
|
|
38
|
+
* not part of this write; `dedup: false` appends only (safe when writing into
|
|
39
|
+
* a freshly-created store where no rows can collide).
|
|
40
|
+
*/
|
|
41
|
+
export interface BulkChunkWrite {
|
|
42
|
+
chunks: Chunk[];
|
|
43
|
+
dedup: boolean;
|
|
44
|
+
}
|
|
35
45
|
/**
|
|
36
46
|
* A LanceDB-backed vector store with persistent on-disk storage, vector search,
|
|
37
47
|
* and chunk metadata queries. Supports automatic corruption recovery by falling
|
|
@@ -44,6 +54,14 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
44
54
|
private table;
|
|
45
55
|
private tableInit;
|
|
46
56
|
private writeLock;
|
|
57
|
+
/**
|
|
58
|
+
* Memoized once-per-process index-metric repair. Stores built by versions
|
|
59
|
+
* before the cosine search switch carry an IVF index trained with the
|
|
60
|
+
* default L2 metric, which makes every cosine query log
|
|
61
|
+
* "Requested metric Cosine is incompatible with index metric L2" and fall
|
|
62
|
+
* back to brute-force. This repairs that stale index on first search.
|
|
63
|
+
*/
|
|
64
|
+
private indexRepairPromise;
|
|
47
65
|
/**
|
|
48
66
|
* Execute an async function under an exclusive write lock.
|
|
49
67
|
*
|
|
@@ -81,13 +99,36 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
81
99
|
private ensureColumnsNullable;
|
|
82
100
|
/**
|
|
83
101
|
* Store chunks in the LanceDB table. New rows are inserted first, then
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
* insert and cleanup. Automatically attempts repair on
|
|
102
|
+
* old rows for the same file that are not part of this write are removed
|
|
103
|
+
* in a single delete per file. This ensures no data is lost if the process
|
|
104
|
+
* aborts between insert and cleanup. Automatically attempts repair on
|
|
105
|
+
* corruption errors.
|
|
106
|
+
*
|
|
107
|
+
* When `options.dedup` is `false` the cleanup step is skipped entirely —
|
|
108
|
+
* a pure append. Use this when writing into a store that provably has no
|
|
109
|
+
* prior rows for these files (e.g. a freshly-created rebuild store).
|
|
110
|
+
*
|
|
87
111
|
* @param chunks - The chunks to add.
|
|
112
|
+
* @param options - Optional write options (`dedup`, default `true`).
|
|
88
113
|
*/
|
|
89
|
-
addChunks(chunks: Chunk[]
|
|
114
|
+
addChunks(chunks: Chunk[], options?: {
|
|
115
|
+
dedup?: boolean;
|
|
116
|
+
}): Promise<void>;
|
|
117
|
+
/**
|
|
118
|
+
* Store chunks for many files in a single transaction: one `table.add`
|
|
119
|
+
* across all items, then one `table.delete` per item that needs dedup.
|
|
120
|
+
* This collapses what used to be a per-file add + per-startLine deletes
|
|
121
|
+
* (K+2 LanceDB versions per file) into ~1 + M versions per batch.
|
|
122
|
+
*
|
|
123
|
+
* @param items - Per-file chunk payloads with their dedup flags.
|
|
124
|
+
*/
|
|
125
|
+
addChunksBulk(items: BulkChunkWrite[]): Promise<void>;
|
|
126
|
+
/** Map a chunk to its internal row shape, or null if it has no embedding. */
|
|
127
|
+
private chunkToRow;
|
|
128
|
+
/** Group new rows by file path for dedup deletes. */
|
|
129
|
+
private rowsByFilePath;
|
|
90
130
|
private addChunksInternal;
|
|
131
|
+
private addChunksBulkInternal;
|
|
91
132
|
/**
|
|
92
133
|
* Perform ANN (approximate nearest neighbor) search using LanceDB's native vector index.
|
|
93
134
|
* Returns results scored as cosine similarity (0-1). Falls back to repair on corruption.
|
|
@@ -120,6 +161,24 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
120
161
|
* @returns An array of chunk summaries.
|
|
121
162
|
*/
|
|
122
163
|
getChunks(offset: number, limit: number): Promise<ChunkSummary[]>;
|
|
164
|
+
/**
|
|
165
|
+
* Retrieve a paginated list of chunks without embeddings, with optional
|
|
166
|
+
* language and file-path filters pushed into the SQL WHERE clause.
|
|
167
|
+
*
|
|
168
|
+
* @param offset - Number of rows to skip (for pagination).
|
|
169
|
+
* @param limit - Maximum number of rows to return.
|
|
170
|
+
* @param lang - Optional exact language filter.
|
|
171
|
+
* @param filePrefix - Optional filePath prefix filter.
|
|
172
|
+
* @returns The page of chunks plus the total count of matching rows.
|
|
173
|
+
*/
|
|
174
|
+
getChunksFiltered(offset: number, limit: number, lang?: string, filePrefix?: string): Promise<{
|
|
175
|
+
chunks: ChunkSummary[];
|
|
176
|
+
total: number;
|
|
177
|
+
}>;
|
|
178
|
+
/** Look up a single chunk by its ID. Returns undefined when not found. */
|
|
179
|
+
getChunkById(id: string): Promise<ChunkSummary | undefined>;
|
|
180
|
+
/** Fetch chunks by their IDs (for the compare view). Bounded to 100 ids. */
|
|
181
|
+
getChunksByIds(ids: string[]): Promise<ChunkSummary[]>;
|
|
123
182
|
/**
|
|
124
183
|
* Fetch chunks with their embedding vectors included (for embedding projection).
|
|
125
184
|
* @param limit - Maximum number of rows to return.
|
|
@@ -150,12 +209,43 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
150
209
|
* @returns The chunk count, or 0 if the table does not exist or times out.
|
|
151
210
|
*/
|
|
152
211
|
count(): Promise<number>;
|
|
212
|
+
/**
|
|
213
|
+
* Ensure the ANN index on the `embedding` column uses the cosine metric,
|
|
214
|
+
* matching the `distanceType` requested by searchInternal.
|
|
215
|
+
*
|
|
216
|
+
* Older stores built the IVF index with the ivfFlat default (L2), so every
|
|
217
|
+
* cosine query hit "Requested metric Cosine is incompatible with index
|
|
218
|
+
* metric L2" and silently fell back to brute-force O(N) scans. This lazily
|
|
219
|
+
* replaces such an index with a cosine one — once per process.
|
|
220
|
+
*
|
|
221
|
+
* Callers must hold the write lock (searchInternal wraps the call in
|
|
222
|
+
* withWriteLock; optimize runs under it already).
|
|
223
|
+
*/
|
|
224
|
+
private ensureCosineIndex;
|
|
225
|
+
/**
|
|
226
|
+
* Perform a single index-metric repair pass. Skips stores that have no
|
|
227
|
+
* index and fewer than 1000 rows (brute-force is optimal there). Uses a
|
|
228
|
+
* single `createIndex` with `replace: true` — a dropIndex + createIndex
|
|
229
|
+
* sequence races in LanceDB ("Retryable commit conflict") and leaves the
|
|
230
|
+
* stale index in place. On failure the memo is cleared so the next
|
|
231
|
+
* search/optimize retries.
|
|
232
|
+
*/
|
|
233
|
+
private repairIndexMetricOnce;
|
|
153
234
|
/**
|
|
154
235
|
* Compact fragments and prune old version manifests to prevent the
|
|
155
|
-
* version-manifest accumulation that causes countRows() to hang
|
|
156
|
-
*
|
|
236
|
+
* version-manifest accumulation that causes countRows() to hang and the
|
|
237
|
+
* store phase to slow down as the index grows. Should be called at the
|
|
238
|
+
* end of a successful index pass, and periodically during long passes.
|
|
239
|
+
*
|
|
240
|
+
* @param options - `aggressive: true` prunes every version but the current
|
|
241
|
+
* one with `deleteUnverified`. Only safe for a private store that no other
|
|
242
|
+
* process reads (e.g. a temporary rebuild store). For the shared store,
|
|
243
|
+
* versions newer than 1 hour are retained so in-flight queries (Web UI,
|
|
244
|
+
* background auto-index) can finish before their data files are reclaimed.
|
|
157
245
|
*/
|
|
158
|
-
optimize(
|
|
246
|
+
optimize(options?: {
|
|
247
|
+
aggressive?: boolean;
|
|
248
|
+
}): Promise<void>;
|
|
159
249
|
/**
|
|
160
250
|
* Return all unique file paths currently stored in the index.
|
|
161
251
|
* @returns An array of normalized file paths.
|
|
@@ -169,7 +259,13 @@ export declare class LanceDbStore implements VectorStore {
|
|
|
169
259
|
* @param newPath - Optional new filesystem path for the LanceDB database.
|
|
170
260
|
*/
|
|
171
261
|
reopen(newPath?: string): Promise<void>;
|
|
172
|
-
/**
|
|
262
|
+
/**
|
|
263
|
+
* Close the database connection and release resources.
|
|
264
|
+
*
|
|
265
|
+
* The native LanceDB `close()` can hang indefinitely on Windows, so the
|
|
266
|
+
* whole close is raced against a timeout — callers (CLI commands, plugin
|
|
267
|
+
* reload, web server) must never block forever on shutdown.
|
|
268
|
+
*/
|
|
173
269
|
close(): Promise<void>;
|
|
174
270
|
/**
|
|
175
271
|
* Back up the chunks.lance directory before a destructive operation.
|