opencode-rag-plugin 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import { appendFileSync, existsSync, readFileSync, writeFileSync, mkdirSync, renameSync } from "node:fs";
1
+ import { appendFileSync, existsSync, readFileSync, writeFileSync, mkdirSync, renameSync, rmSync } from "node:fs";
2
2
  import { randomUUID } from "node:crypto";
3
3
  import path from "node:path";
4
4
  import { retrieve } from "../retriever/retriever.js";
@@ -201,23 +201,7 @@ export async function addQuirk(deps, input) {
201
201
  if (!embedding || embedding.length === 0) {
202
202
  throw new Error("Embedding returned empty vector for quirk content");
203
203
  }
204
- const chunk = {
205
- id,
206
- content: input.content,
207
- description: "",
208
- embedding,
209
- metadata: {
210
- filePath: QUIRK_FILE_PREFIX + id,
211
- startLine: 0,
212
- endLine: 0,
213
- language: "quirk",
214
- kind: "quirk",
215
- quirkType: input.quirkType,
216
- tags: input.tags ?? [],
217
- confidence,
218
- lastObserved,
219
- },
220
- };
204
+ const chunk = quirkChunk(quirk, embedding);
221
205
  await deps.store.addChunks([chunk]);
222
206
  deps.keywordIndex.addChunks([chunk]);
223
207
  if (!isMemoryStore(deps.storePath)) {
@@ -228,6 +212,33 @@ export async function addQuirk(deps, input) {
228
212
  }
229
213
  return quirk;
230
214
  }
215
+ /**
216
+ * Build the store/keyword-index chunk shape for a quirk. Shared by `addQuirk`
217
+ * and `reconcileQuirks` — a rebuild-restore must produce the exact same row
218
+ * shape (metadata.filePath in particular drives list/remove/reconcile id maps).
219
+ *
220
+ * `embedding` may be `[]` for keyword-index-only adds: `chunkToRow` skips rows
221
+ * with an empty embedding, and the keyword index only tokenizes content.
222
+ */
223
+ function quirkChunk(q, embedding) {
224
+ return {
225
+ id: q.id,
226
+ content: q.content,
227
+ description: "",
228
+ embedding,
229
+ metadata: {
230
+ filePath: QUIRK_FILE_PREFIX + q.id,
231
+ startLine: 0,
232
+ endLine: 0,
233
+ language: "quirk",
234
+ kind: "quirk",
235
+ quirkType: q.quirkType,
236
+ tags: q.tags ?? [],
237
+ confidence: q.confidence,
238
+ lastObserved: q.lastObserved,
239
+ },
240
+ };
241
+ }
231
242
  /** Remove a quirk by its ID. Throws if no quirk with the given ID exists. */
232
243
  export async function removeQuirk(deps, id) {
233
244
  const existing = await getQuirk(deps, id);
@@ -255,9 +266,10 @@ export async function listQuirks(deps) {
255
266
  all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
256
267
  return all;
257
268
  }
258
- // Fallback: scan the store
269
+ // Fallback: scan the store. Stored filePaths are workspace-absolute
270
+ // (chunkToRow normalizes), so match quirks on basename, not prefix.
259
271
  const filePaths = await deps.store.getFilePaths();
260
- const quirkPaths = filePaths.filter((fp) => fp.startsWith(QUIRK_FILE_PREFIX));
272
+ const quirkPaths = filePaths.filter((fp) => path.basename(fp).startsWith(QUIRK_FILE_PREFIX));
261
273
  const result = [];
262
274
  for (const fp of quirkPaths) {
263
275
  const chunks = await deps.store.getChunksByFilePath(fp);
@@ -280,6 +292,174 @@ export async function listQuirks(deps) {
280
292
  all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
281
293
  return all;
282
294
  }
295
+ /**
296
+ * Store-dir artifact recording when each orphaned store quirk (in the table but
297
+ * absent from quirks.jsonl) was FIRST observed as an orphan. Deletion requires
298
+ * the orphan to be seen again at least {@link QUIRK_ORPHAN_GRACE_MS} later.
299
+ *
300
+ * Two-sighting instead of a row timestamp because the table has no
301
+ * `lastObserved` column (ChunkRow never stored it), and because `addQuirk`
302
+ * writes the store BEFORE appending to quirks.jsonl — a quirk that is
303
+ * momentarily store-only is an in-flight add and must never be race-deleted.
304
+ * The file makes the grace window work across processes (plugin/CLI/watcher
305
+ * each run their own reconcile).
306
+ */
307
+ const QUIRK_ORPHAN_STATE_FILE = "quirk-orphans.json";
308
+ const QUIRK_ORPHAN_GRACE_MS = 60 * 60 * 1000;
309
+ /**
310
+ * Reconcile `quirks.jsonl` (the source of truth for quirk memory) into the
311
+ * vector store and the keyword index, in both directions.
312
+ *
313
+ * Why this exists: full store rebuilds rebuild the table from **workspace
314
+ * files only** — `swapStoreDirectories` carries `quirks.jsonl` across the swap,
315
+ * but nothing ever re-embeds those entries, so every quirk added before a
316
+ * rebuild silently vanishes from recall while `quirk list`/`lint` (which read
317
+ * the jsonl) still show it. The keyword index has the same gap: it only gains
318
+ * quirks via in-process `addQuirk`, and any process that saves its in-memory
319
+ * index clobbers entries added elsewhere.
320
+ *
321
+ * Behavior:
322
+ * - Restores jsonl quirks missing from the store (one batched embed call).
323
+ * - Adds jsonl quirks missing from the keyword index (no embedding needed).
324
+ * - Deletes store quirk chunks whose jsonl entry is gone — only when the jsonl
325
+ * exists and parses non-empty (a missing/unreadable file means "unknown",
326
+ * never "empty") and only after the orphan was observed twice across
327
+ * {@link QUIRK_ORPHAN_GRACE_MS} (see QUIRK_ORPHAN_STATE_FILE — the table has
328
+ * no row timestamps, and addQuirk writes the store before the jsonl append).
329
+ * - Persists the keyword index when it changed (best-effort).
330
+ *
331
+ * Idempotent and safe under concurrent runs: `store.addChunks` dedups by id and
332
+ * `KeywordIndex.addChunks` overwrites by id. Never applied to memory stores
333
+ * (per-process lifetime, nothing on disk to drift). Failures propagate to the
334
+ * caller, which is expected to treat reconcile as best-effort.
335
+ *
336
+ * @param deps - Embedder, store, optional keyword index, config, store path.
337
+ * @returns Counts of what changed (all zero when already consistent).
338
+ */
339
+ export async function reconcileQuirks(deps) {
340
+ const result = { restoredToStore: 0, addedToKeywordIndex: 0, removedOrphans: 0 };
341
+ if (isMemoryStore(deps.storePath))
342
+ return result;
343
+ const jsonlFile = jsonlPath(deps.storePath);
344
+ const jsonlExists = existsSync(jsonlFile);
345
+ const quirks = jsonlExists ? readJsonl(jsonlFile) : [];
346
+ const jsonlIds = new Set(quirks.map((q) => q.id));
347
+ // Exact enumeration of quirk chunks in the table. The store normalizes
348
+ // filePaths to absolute on write (chunkToRow → normalizeFilePath), so quirk
349
+ // rows read back as "<root>/quirk:<id>" — match on the basename, which is
350
+ // always QUIRK_FILE_PREFIX + id, and keep the stored path for lookups/deletes
351
+ // (deleteByFilePath normalizes its arg; passing the stored absolute path
352
+ // matches regardless of this process's cwd).
353
+ const quirkRows = [];
354
+ for (const p of await deps.store.getFilePaths()) {
355
+ const base = path.basename(p);
356
+ if (base.startsWith(QUIRK_FILE_PREFIX)) {
357
+ quirkRows.push({ filePath: p, id: base.slice(QUIRK_FILE_PREFIX.length) });
358
+ }
359
+ }
360
+ const storeIds = new Set(quirkRows.map((r) => r.id));
361
+ // 1. Restore jsonl quirks the table is missing (the rebuild-wipe direction).
362
+ const missing = quirks.filter((q) => !storeIds.has(q.id));
363
+ if (missing.length > 0) {
364
+ const prefix = deps.cfg.embedding.documentPrefix ?? "";
365
+ const embeddings = await deps.embedder.embed(missing.map((q) => prefix + q.content), "document");
366
+ if (embeddings.length !== missing.length || embeddings.some((e) => !e || e.length === 0)) {
367
+ throw new Error("Embedding returned empty vector(s) while reconciling quirks");
368
+ }
369
+ // Single batched write = single transaction; addChunks dedups by id, so a
370
+ // concurrent reconcile adding the same quirks converges instead of duplicating.
371
+ await deps.store.addChunks(missing.map((q, i) => quirkChunk(q, embeddings[i])));
372
+ result.restoredToStore = missing.length;
373
+ for (const q of missing)
374
+ storeIds.add(q.id); // keep orphan pass below consistent
375
+ }
376
+ // 2. Ensure every jsonl quirk is in the keyword index. Lexical only — the
377
+ // keyword index never looks at embeddings, so this needs no provider.
378
+ let kiDirty = false;
379
+ if (deps.keywordIndex) {
380
+ for (const q of quirks) {
381
+ if (deps.keywordIndex.hasChunk(q.id))
382
+ continue;
383
+ deps.keywordIndex.addChunks([quirkChunk(q, [])]);
384
+ result.addedToKeywordIndex++;
385
+ kiDirty = true;
386
+ }
387
+ }
388
+ // 3. Orphan direction: store quirks whose jsonl entry is gone. Runs only when
389
+ // the jsonl exists AND parses to at least one quirk (a missing or wholly
390
+ // unreadable jsonl means "unknown", never "empty" — never mass-delete on
391
+ // it). Each orphan must be observed twice, spanning the grace window,
392
+ // before deletion — see QUIRK_ORPHAN_STATE_FILE.
393
+ if (jsonlExists && quirks.length > 0) {
394
+ const suspectsPath = path.join(deps.storePath, QUIRK_ORPHAN_STATE_FILE);
395
+ const suspects = (() => {
396
+ try {
397
+ const parsed = JSON.parse(readFileSync(suspectsPath, "utf-8"));
398
+ return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
399
+ }
400
+ catch {
401
+ return {};
402
+ }
403
+ })();
404
+ let suspectsDirty = false;
405
+ const now = Date.now();
406
+ for (const { filePath, id } of quirkRows) {
407
+ if (jsonlIds.has(id)) {
408
+ if (id in suspects) {
409
+ delete suspects[id];
410
+ suspectsDirty = true;
411
+ } // regained its jsonl entry
412
+ continue;
413
+ }
414
+ const firstSeen = suspects[id];
415
+ if (firstSeen === undefined) {
416
+ suspects[id] = now; // first sighting — keep (may be an in-flight add)
417
+ suspectsDirty = true;
418
+ continue;
419
+ }
420
+ if (now - firstSeen < QUIRK_ORPHAN_GRACE_MS)
421
+ continue;
422
+ await deps.store.deleteByFilePath(filePath);
423
+ deps.keywordIndex?.removeByFilePath(filePath);
424
+ delete suspects[id];
425
+ result.removedOrphans++;
426
+ kiDirty = true;
427
+ suspectsDirty = true;
428
+ }
429
+ // Prune suspects that no longer exist in the store (jsonl regained them,
430
+ // a rebuild dropped them, or a previous reconcile deleted them).
431
+ for (const id of Object.keys(suspects)) {
432
+ if (!storeIds.has(id)) {
433
+ delete suspects[id];
434
+ suspectsDirty = true;
435
+ }
436
+ }
437
+ if (suspectsDirty) {
438
+ try {
439
+ if (Object.keys(suspects).length === 0)
440
+ rmSync(suspectsPath, { force: true });
441
+ else
442
+ writeFileSync(suspectsPath, JSON.stringify(suspects, null, 2));
443
+ }
444
+ catch {
445
+ // best-effort — worst case the grace restarts for unmarked orphans
446
+ }
447
+ }
448
+ }
449
+ // 4. Persist the keyword index so the NEXT process to load it inherits the
450
+ // reconciled state (this is what stops pass-end saves from clobbering
451
+ // quirks added by other processes). Best-effort: a failed save leaves the
452
+ // in-memory index correct; the next reconcile re-saves.
453
+ if (kiDirty && deps.keywordIndex) {
454
+ try {
455
+ await deps.keywordIndex.save(deps.storePath);
456
+ }
457
+ catch {
458
+ // best-effort
459
+ }
460
+ }
461
+ return result;
462
+ }
283
463
  /** Recall quirks matching a query, with confidence re-weighting. */
284
464
  export async function recallQuirks(deps, query, options) {
285
465
  const topK = options?.topK ?? 10;
@@ -43,6 +43,7 @@ export declare class KeywordIndex {
43
43
  close(): void;
44
44
  clear(): void;
45
45
  count(): number;
46
+ hasChunk(id: string): boolean;
46
47
  save(storePath?: string): Promise<void>;
47
48
  static load(storePath: string): Promise<KeywordIndex>;
48
49
  static clearFile(storePath: string): Promise<void>;
@@ -273,6 +273,9 @@ export class KeywordIndex {
273
273
  count() {
274
274
  return this.chunkMap.size;
275
275
  }
276
+ hasChunk(id) {
277
+ return this.chunkMap.has(id);
278
+ }
276
279
  async save(storePath) {
277
280
  const effectiveStorePath = storePath ?? this.storePath;
278
281
  if (!effectiveStorePath)
package/dist/tui.js CHANGED
@@ -458,6 +458,12 @@ function buildSettingCategories(cfg, ro, providers) {
458
458
  type: "json",
459
459
  currentValue: chunkingRo.nodeTypes ?? chunkingCfg.nodeTypes ?? {},
460
460
  },
461
+ {
462
+ path: ["chunking", "parsers"],
463
+ label: "Parser overrides — extension → language (JSON)",
464
+ type: "json",
465
+ currentValue: chunkingRo.parsers ?? chunkingCfg.parsers ?? {},
466
+ },
461
467
  ],
462
468
  },
463
469
  {
@@ -234,6 +234,7 @@ export async function readStoreDimension(storePath) {
234
234
  */
235
235
  const PRESERVED_STORE_ENTRIES = [
236
236
  "quirks.jsonl",
237
+ "quirk-orphans.json",
237
238
  "runtime-overrides.json",
238
239
  "watcher-status.json",
239
240
  ".desc-cache.json",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "opencode-rag-plugin",
3
- "version": "2.0.1",
3
+ "version": "2.1.0",
4
4
  "description": "OpenCode plugin for local-first RAG-based semantic code search",
5
5
  "type": "module",
6
6
  "main": "./dist/plugin-entry.js",