opencode-rag-plugin 2.10.1 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/ReadMe.md +40 -28
  2. package/dist/api.js +1 -0
  3. package/dist/cli/commands/clickhouse.d.ts +61 -0
  4. package/dist/cli/commands/clickhouse.js +335 -0
  5. package/dist/cli/commands/eval-tuner.js +1 -1
  6. package/dist/cli/commands/eval.js +1 -1
  7. package/dist/cli/commands/index-command.js +2 -1
  8. package/dist/cli/commands/index.d.ts +1 -0
  9. package/dist/cli/commands/index.js +1 -0
  10. package/dist/cli/commands/init-helpers.js +2 -2
  11. package/dist/cli/commands/init.js +49 -11
  12. package/dist/cli/commands/quirk.js +6 -6
  13. package/dist/cli/commands/status.js +70 -6
  14. package/dist/cli/commands/ui.js +2 -2
  15. package/dist/cli/index.js +2 -1
  16. package/dist/cli/types.d.ts +14 -0
  17. package/dist/core/bootstrap.d.ts +7 -0
  18. package/dist/core/bootstrap.js +40 -9
  19. package/dist/core/config.d.ts +128 -5
  20. package/dist/core/config.js +205 -8
  21. package/dist/core/interfaces.d.ts +54 -0
  22. package/dist/core/runtime-overrides.d.ts +9 -0
  23. package/dist/core/runtime-overrides.js +44 -0
  24. package/dist/core/state-sync.d.ts +91 -0
  25. package/dist/core/state-sync.js +273 -0
  26. package/dist/eval/fast-index.js +2 -2
  27. package/dist/eval/rerank-eval.js +2 -2
  28. package/dist/eval/run-branch-compare.js +2 -2
  29. package/dist/eval/run-token-test.js +2 -2
  30. package/dist/index.d.ts +5 -2
  31. package/dist/index.js +2 -1
  32. package/dist/indexer/pipeline.d.ts +6 -0
  33. package/dist/indexer/pipeline.js +173 -56
  34. package/dist/indexer/progress.d.ts +75 -0
  35. package/dist/indexer/progress.js +114 -0
  36. package/dist/indexer/stats.d.ts +14 -0
  37. package/dist/plugin.d.ts +6 -3
  38. package/dist/plugin.js +39 -20
  39. package/dist/quirks/quirk-store.d.ts +10 -0
  40. package/dist/quirks/quirk-store.js +110 -15
  41. package/dist/quirks/types.d.ts +7 -0
  42. package/dist/tui.js +128 -111
  43. package/dist/vectorstore/clickhouse.d.ts +150 -0
  44. package/dist/vectorstore/clickhouse.js +931 -0
  45. package/dist/vectorstore/dimension.d.ts +19 -0
  46. package/dist/vectorstore/dimension.js +36 -0
  47. package/dist/vectorstore/factory.d.ts +9 -3
  48. package/dist/vectorstore/factory.js +11 -2
  49. package/dist/vectorstore/lancedb.d.ts +2 -2
  50. package/dist/watcher.d.ts +29 -0
  51. package/dist/watcher.js +76 -2
  52. package/dist/web/api.d.ts +3 -4
  53. package/dist/web/api.js +18 -3
  54. package/dist/web/server.js +28 -2
  55. package/dist/web/ui/assets/index-B-RH6fko.css +2 -0
  56. package/dist/web/ui/index.html +2 -2
  57. package/package.json +2 -1
  58. package/dist/web/ui/assets/index-CWcp0jNf.css +0 -2
  59. /package/dist/web/ui/assets/{index-DbXFrn53.js → index-ChGi2bSp.js} +0 -0
@@ -70,6 +70,20 @@ export interface IndexStatusSummary {
70
70
  * reachable through any accounting. The next full index pass sweeps them.
71
71
  */
72
72
  orphanChunks: number;
73
+ /**
74
+ * Whether {@link upToDateFiles} and {@link pendingFiles} were actually
75
+ * computed from a workspace scan. When false the two counters are zero
76
+ * because no scan ran — never because every file is current.
77
+ */
78
+ scanned: boolean;
79
+ /**
80
+ * Stored chunks owned by quirks (`quirk:` file paths). quirks.jsonl owns
81
+ * them, so they belong to neither {@link manifestExpectedChunks} nor
82
+ * {@link orphanChunks}; without this line `Indexed chunks` does not
83
+ * reconcile against `Expected + Orphan`. Undefined when the store cannot
84
+ * enumerate its files.
85
+ */
86
+ quirkChunks?: number;
73
87
  }
74
88
  /**
75
89
  * Create a zeroed-out {@link IndexRunStats} instance initialised with the
package/dist/plugin.d.ts CHANGED
@@ -7,12 +7,13 @@ import type { Plugin, Hooks } from "@opencode-ai/plugin";
7
7
  import { type EmbeddingProvider, type DescriptionProvider, type KeywordIndex, type VectorStore } from "./core/interfaces.js";
8
8
  import { type RagConfig } from "./core/config.js";
9
9
  import { createEmbedder } from "./embedder/factory.js";
10
+ import { StateSync } from "./core/state-sync.js";
10
11
  import { retrieve } from "./retriever/retriever.js";
11
12
  /**
12
13
  * Load the RAG configuration for a directory: DEFAULT_CONFIG, then the global
13
14
  * layer (`~/.config/opencode/opencode-rag.json`), then the first workspace
14
- * config found (`opencode-rag.json`, `.opencode/opencode-rag.json`,
15
- * `.opencode/rag.json`).
15
+ * config found (`.opencode/opencode-rag.json`, then the legacy
16
+ * `opencode-rag.json` / `.opencode/rag.json`).
16
17
  *
17
18
  * The result is cached per directory and re-read whenever either config file
18
19
  * changes on disk, so an edited API key, baseUrl, or global `shellGuard` switch
@@ -57,7 +58,7 @@ export declare function createLiveEmbedder(getConfig: () => RagConfig, create: (
57
58
  /** Internal dependencies that can be overridden for testing. */
58
59
  type RagPluginDependencies = {
59
60
  createEmbedder: typeof createEmbedder;
60
- createStore: (storePath: string, dimension: number, config: RagConfig) => VectorStore;
61
+ createStore: (storePath: string, dimension: number, config: RagConfig, workspaceDir: string) => VectorStore;
61
62
  retrieve: typeof retrieve;
62
63
  };
63
64
  /** Options for constructing the RAG hooks object. */
@@ -82,6 +83,8 @@ type CreateRagHooksOptions = {
82
83
  keywordIndex?: KeywordIndex;
83
84
  /** Pre-created description provider for chunk descriptions. */
84
85
  descriptionProvider?: DescriptionProvider;
86
+ /** Durable-state synchronizer (ClickHouse); used to flush quirk writes. */
87
+ stateSync?: StateSync;
85
88
  };
86
89
  /**
87
90
  * Plugin tool names advertised by the mandatory guidance and granted to the
package/dist/plugin.js CHANGED
@@ -8,7 +8,8 @@ import { CODE_SEARCH_FILTER } from "./core/interfaces.js";
8
8
  import { normalizeFileExtensions } from "./core/filters.js";
9
9
  import { loadWorkspaceConfig, resolveConfigPaths, findConfigFile, DEFAULT_CONFIG, resolveLogConfig, persistProbedDimension } from "./core/config.js";
10
10
  import { createEmbedder, probeEmbeddingDimension } from "./embedder/factory.js";
11
- import { readStoreDimension } from "./vectorstore/lancedb.js";
11
+ import { readConfiguredStoreDimension } from "./vectorstore/dimension.js";
12
+ import { StateSync } from "./core/state-sync.js";
12
13
  import { createDescriptionProvider } from "./describer/factory.js";
13
14
  import { createVectorStore } from "./vectorstore/factory.js";
14
15
  import { retrieve } from "./retriever/retriever.js";
@@ -244,8 +245,8 @@ function configStamp(paths) {
244
245
  /**
245
246
  * Load the RAG configuration for a directory: DEFAULT_CONFIG, then the global
246
247
  * layer (`~/.config/opencode/opencode-rag.json`), then the first workspace
247
- * config found (`opencode-rag.json`, `.opencode/opencode-rag.json`,
248
- * `.opencode/rag.json`).
248
+ * config found (`.opencode/opencode-rag.json`, then the legacy
249
+ * `opencode-rag.json` / `.opencode/rag.json`).
249
250
  *
250
251
  * The result is cached per directory and re-read whenever either config file
251
252
  * changes on disk, so an edited API key, baseUrl, or global `shellGuard` switch
@@ -504,7 +505,7 @@ async function loadRetrievedResults(query, embedder, store, cfg, retrieveFn = re
504
505
  /** Default set of plugin dependencies using the real implementations. */
505
506
  const defaultDependencies = {
506
507
  createEmbedder,
507
- createStore: (storePath, dimension, config) => createVectorStore(config, storePath, dimension),
508
+ createStore: (storePath, dimension, config, workspaceDir) => createVectorStore(config, storePath, dimension, workspaceDir),
508
509
  retrieve,
509
510
  };
510
511
  /**
@@ -639,8 +640,11 @@ export function createRagHooks(options) {
639
640
  ...options.dependencies,
640
641
  };
641
642
  const configuredDimension = options.cfg.embedding.vectorDimension;
642
- const store = options.store ?? dependencies.createStore(options.storePath, configuredDimension && configuredDimension > 0 ? configuredDimension : 384, options.cfg);
643
+ const store = options.store ?? dependencies.createStore(options.storePath, configuredDimension && configuredDimension > 0 ? configuredDimension : 384, options.cfg, options.worktree);
643
644
  const keywordIndex = options.keywordIndex;
645
+ const flushState = options.stateSync
646
+ ? () => options.stateSync.flush(["quirks"])
647
+ : undefined;
644
648
  // Runtime overrides for live config editing from TUI
645
649
  let cachedOverrides = loadRuntimeOverrides(options.storePath);
646
650
  let overridesLastCheck = 0;
@@ -1029,7 +1033,7 @@ export function createRagHooks(options) {
1029
1033
  maxPerSession: memCfg.toolErrorMaxPerSession ?? 3,
1030
1034
  });
1031
1035
  boundedSet(sessionToolErrorState, sessionID, state, MAX_SESSION_MAP_SIZE);
1032
- const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
1036
+ const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
1033
1037
  // `explain: true` keeps the raw vector similarity on each result —
1034
1038
  // the confidence-discounted `score` is not comparable to a
1035
1039
  // similarity threshold, and the tiering below needs the raw value.
@@ -1275,7 +1279,7 @@ export function createRagHooks(options) {
1275
1279
  const effCfg = getEffectiveCfg();
1276
1280
  const memCfg = effCfg.memory;
1277
1281
  if (memCfg?.sessionEndExtraction && options.descriptionProvider) {
1278
- const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: effCfg, storePath: options.storePath };
1282
+ const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: effCfg, storePath: options.storePath, flushState: flushState };
1279
1283
  void autoCaptureQuirks(qDeps, options.descriptionProvider, transcript, effCfg, { captureAll: true }).catch(() => { });
1280
1284
  }
1281
1285
  boundedSet(sessionTranscript, sessionID, [], MAX_SESSION_MAP_SIZE);
@@ -1336,7 +1340,7 @@ export function createRagHooks(options) {
1336
1340
  const current = (sessionShellLongCount.get(counterKey) ?? 0);
1337
1341
  if (current < 3) {
1338
1342
  sessionShellLongCount.set(counterKey, current + 1);
1339
- const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
1343
+ const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
1340
1344
  void addQuirk(quirkDeps, {
1341
1345
  content: `A foreground shell command took ${durSec}s: \`${cmd}\`. Prefer \`background: true\` or an explicit \`timeout\` for long commands; press Ctrl+B to background a running call.`,
1342
1346
  quirkType: "gotcha",
@@ -1428,7 +1432,7 @@ export function createRagHooks(options) {
1428
1432
  if (toolNames.length > 0) {
1429
1433
  const memCfg = getEffectiveCfg().memory;
1430
1434
  if (memCfg?.enabled && memCfg.toolErrorProactiveInject) {
1431
- const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
1435
+ const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
1432
1436
  const gotchas = selectToolGotchas(await getToolErrorQuirks(qDeps), toolNames, memCfg.toolErrorProactiveMaxItems ?? TOOL_ERROR_CREATE_DEFAULTS.maxProactiveItems);
1433
1437
  if (gotchas.length > 0) {
1434
1438
  output.system.unshift(formatToolGotchaBlock(gotchas));
@@ -1490,7 +1494,7 @@ export function createRagHooks(options) {
1490
1494
  const userReq = sessionPrevUserReq.get(quirkSessionId) ?? "";
1491
1495
  const query = [assistantText, userReq].filter((t) => t.length > 0).join("\n");
1492
1496
  if (query.length > 0) {
1493
- const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
1497
+ const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
1494
1498
  const injectedSet = getOrCreateSessionSet(sessionInjectedQuirks, quirkSessionId, MAX_SESSION_MAP_SIZE);
1495
1499
  const selection = await selectAutoInjectQuirksWithinBudget((q, o) => recallQuirks(quirkDeps, q, { ...o, excludeTags: [TOOL_ERROR_QUIRK_TAG] }), {
1496
1500
  query,
@@ -1570,7 +1574,7 @@ export function createRagHooks(options) {
1570
1574
  const assistantText = sessionAssistantText.get(input.sessionID) ?? "";
1571
1575
  const toolResults = sessionToolResults.get(input.sessionID) ?? [];
1572
1576
  if (assistantText) {
1573
- const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: effCfg, storePath: options.storePath };
1577
+ const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: effCfg, storePath: options.storePath, flushState: flushState };
1574
1578
  const exchange = { userReq: prevUserReq, assistantText, toolResults };
1575
1579
  if (memCfg?.passiveCapture) {
1576
1580
  void autoCaptureQuirks(qDeps, options.descriptionProvider, [exchange], effCfg).catch(() => { });
@@ -1594,7 +1598,7 @@ export function createRagHooks(options) {
1594
1598
  if (Array.isArray(parts) && parts.length > 0) {
1595
1599
  const first = parts[0];
1596
1600
  if (typeof first.text === "string") {
1597
- parts[0] = { ...first, text: "Documentation mode is not enabled. Set `documentationMode.enabled` to `true` in opencode-rag.json." };
1601
+ parts[0] = { ...first, text: "Documentation mode is not enabled. Set `documentationMode.enabled` to `true` in .opencode/opencode-rag.json." };
1598
1602
  }
1599
1603
  }
1600
1604
  return;
@@ -1678,7 +1682,7 @@ export function createRagHooks(options) {
1678
1682
  if (Array.isArray(parts) && parts.length > 0) {
1679
1683
  const first = parts[0];
1680
1684
  if (typeof first.text === "string") {
1681
- parts[0] = { ...first, text: "Wiki mode is not enabled. Set `wikiMode.enabled` to `true` in opencode-rag.json." };
1685
+ parts[0] = { ...first, text: "Wiki mode is not enabled. Set `wikiMode.enabled` to `true` in .opencode/opencode-rag.json." };
1682
1686
  }
1683
1687
  }
1684
1688
  return;
@@ -1736,7 +1740,7 @@ export function createRagHooks(options) {
1736
1740
  if (Array.isArray(parts) && parts.length > 0) {
1737
1741
  const first = parts[0];
1738
1742
  if (typeof first.text === "string") {
1739
- parts[0] = { ...first, text: "Quirk memory is not enabled. Set `memory.enabled` to `true` in opencode-rag.json." };
1743
+ parts[0] = { ...first, text: "Quirk memory is not enabled. Set `memory.enabled` to `true` in .opencode/opencode-rag.json." };
1740
1744
  }
1741
1745
  }
1742
1746
  return;
@@ -1744,7 +1748,7 @@ export function createRagHooks(options) {
1744
1748
  const arg = text.slice(6).trim().toLowerCase();
1745
1749
  const lines = [];
1746
1750
  if (arg === "lint") {
1747
- const deps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
1751
+ const deps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
1748
1752
  const issues = await lintQuirks(deps);
1749
1753
  lines.push("## Quirk Lint", "");
1750
1754
  if (issues.length === 0) {
@@ -1763,7 +1767,7 @@ export function createRagHooks(options) {
1763
1767
  }
1764
1768
  else {
1765
1769
  // Status / list
1766
- const deps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
1770
+ const deps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
1767
1771
  const quirks = await listQuirks(deps);
1768
1772
  lines.push("## Quirk Memory", "");
1769
1773
  if (quirks.length === 0) {
@@ -1809,7 +1813,7 @@ export function createRagHooks(options) {
1809
1813
  const assistantText = sessionAssistantText.get(input.sessionID) ?? "";
1810
1814
  const quirkQuery = [assistantText, text].filter((t) => t.length > 0).join("\n");
1811
1815
  if (quirkQuery.length > 0) {
1812
- const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
1816
+ const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
1813
1817
  const injectedSet = getOrCreateSessionSet(sessionInjectedQuirks, input.sessionID, MAX_SESSION_MAP_SIZE);
1814
1818
  const selection = await selectAutoInjectQuirksWithinBudget((q, o) => recallQuirks(quirkDeps, q, { ...o, excludeTags: [TOOL_ERROR_QUIRK_TAG] }), {
1815
1819
  query: quirkQuery,
@@ -2089,7 +2093,7 @@ export const ragPlugin = async (input, _options) => {
2089
2093
  }, logLevel);
2090
2094
  }
2091
2095
  else {
2092
- vectorDimension = (await readStoreDimension(storePath)) ?? 384;
2096
+ vectorDimension = (await readConfiguredStoreDimension(effectiveCfg, storePath, input.directory)) ?? 384;
2093
2097
  appendDebugLog(logFilePath, {
2094
2098
  scope: "plugin",
2095
2099
  message: `Dimension probe failed, falling back to ${vectorDimension}`,
@@ -2097,8 +2101,21 @@ export const ragPlugin = async (input, _options) => {
2097
2101
  }, logLevel);
2098
2102
  }
2099
2103
  }
2100
- const store = createVectorStore(effectiveCfg, storePath, vectorDimension);
2104
+ const store = createVectorStore(effectiveCfg, storePath, vectorDimension, input.directory);
2101
2105
  ragStores.set(input.directory, store);
2106
+ // ClickHouse: pull the durable manifest/keyword index/quirks/overrides into
2107
+ // the disposable local cache before anything reads them.
2108
+ const stateSync = new StateSync({ store, storePath, config: effectiveCfg });
2109
+ try {
2110
+ await stateSync.hydrate();
2111
+ }
2112
+ catch (err) {
2113
+ appendDebugLog(logFilePath, {
2114
+ scope: "plugin",
2115
+ message: "State hydration failed",
2116
+ error: err,
2117
+ }, logLevel);
2118
+ }
2102
2119
  // Warn when the store was built by a different embedding model — vector
2103
2120
  // search fails (or silently degrades) until the index is rebuilt.
2104
2121
  try {
@@ -2120,7 +2137,7 @@ export const ragPlugin = async (input, _options) => {
2120
2137
  // or keyword index (full rebuilds drop quirk chunks — see reconcileQuirks).
2121
2138
  // Fire-and-forget: must never block plugin startup, and the embedder may be
2122
2139
  // offline. The background watcher re-runs this at every pass end.
2123
- void reconcileQuirks({ embedder: liveEmbedder, store, keywordIndex, cfg: effectiveCfg, storePath })
2140
+ void reconcileQuirks({ embedder: liveEmbedder, store, keywordIndex, cfg: effectiveCfg, storePath, flushState: () => stateSync.flush(["quirks"]) })
2124
2141
  .then((r) => {
2125
2142
  if (r.restoredToStore > 0 || r.addedToKeywordIndex > 0 || r.removedOrphans > 0) {
2126
2143
  appendDebugLog(logFilePath, {
@@ -2151,6 +2168,7 @@ export const ragPlugin = async (input, _options) => {
2151
2168
  store,
2152
2169
  keywordIndex,
2153
2170
  descriptionProvider,
2171
+ stateSync,
2154
2172
  });
2155
2173
  // Start background auto-indexer if enabled
2156
2174
  const autoIndexCfg = effectiveCfg.openCode.autoIndex ?? { enabled: false, debounceMs: 5000, intervalMs: 300000 };
@@ -2165,6 +2183,7 @@ export const ragPlugin = async (input, _options) => {
2165
2183
  logLevel,
2166
2184
  keywordIndex,
2167
2185
  descriptionProvider,
2186
+ stateSync,
2168
2187
  dimension: vectorDimension,
2169
2188
  });
2170
2189
  backgroundIndexers.set(input.directory, indexer);
@@ -8,8 +8,18 @@ export interface QuirkStoreDeps {
8
8
  keywordIndex: KeywordIndex;
9
9
  cfg: RagConfig;
10
10
  storePath: string;
11
+ /** Upload the durable quirk log after a write (ClickHouse state bundle). */
12
+ flushState?: () => Promise<void>;
11
13
  }
12
14
  export declare const QUIRK_FILE_PREFIX = "quirk:";
15
+ /**
16
+ * True when a JSONL record is a deletion tombstone rather than a live quirk.
17
+ *
18
+ * Tombstones exist so a deletion survives `mergeQuirkJsonl`, which is a union
19
+ * by id and would otherwise resurrect the record from the central store on the
20
+ * next hydrate/flush. Every read path must filter them out.
21
+ */
22
+ export declare function isTombstone(q: Quirk): boolean;
13
23
  /** Look up a single quirk by its ID, or `undefined` when not found. */
14
24
  export declare function getQuirk(deps: QuirkStoreDeps, id: string): Promise<Quirk | undefined>;
15
25
  /**
@@ -1,4 +1,4 @@
1
- import { appendFileSync, existsSync, readFileSync, writeFileSync, mkdirSync, renameSync, rmSync } from "node:fs";
1
+ import { appendFileSync, existsSync, readFileSync, statSync, writeFileSync, mkdirSync, renameSync, rmSync } from "node:fs";
2
2
  import { randomUUID } from "node:crypto";
3
3
  import path from "node:path";
4
4
  import { retrieve } from "../retriever/retriever.js";
@@ -40,6 +40,46 @@ function readJsonl(filePath) {
40
40
  }
41
41
  return quirks;
42
42
  }
43
+ /**
44
+ * True when a JSONL record is a deletion tombstone rather than a live quirk.
45
+ *
46
+ * Tombstones exist so a deletion survives `mergeQuirkJsonl`, which is a union
47
+ * by id and would otherwise resurrect the record from the central store on the
48
+ * next hydrate/flush. Every read path must filter them out.
49
+ */
50
+ export function isTombstone(q) {
51
+ return typeof q.deletedAt === "string" && q.deletedAt.length > 0;
52
+ }
53
+ const NO_TOMBSTONES = new Set();
54
+ /**
55
+ * Tombstone ids for a store, cached on `quirks.jsonl` mtime + size.
56
+ *
57
+ * `recallQuirks` runs on every turn (auto-inject) and on every tool error, so it
58
+ * must not parse the whole audit log per call. A missing or unreadable log means
59
+ * "no tombstone knowledge" — never an error and never filter-everything:
60
+ * `removeQuirk`'s atomic rewrite has a direct-write fallback, and a momentary
61
+ * unreadable file must not change recall behaviour.
62
+ */
63
+ const tombstoneIdsCache = new Map();
64
+ function tombstoneIds(storePath) {
65
+ if (isMemoryStore(storePath))
66
+ return NO_TOMBSTONES; // memory stores delete outright
67
+ const jp = jsonlPath(storePath);
68
+ try {
69
+ const st = statSync(jp);
70
+ const hit = tombstoneIdsCache.get(storePath);
71
+ if (hit && hit.mtimeMs === st.mtimeMs && hit.size === st.size)
72
+ return hit.ids;
73
+ const ids = new Set(readJsonl(jp).filter(isTombstone).map((q) => q.id));
74
+ if (tombstoneIdsCache.size > 16)
75
+ tombstoneIdsCache.clear(); // bounded: keyed by store path
76
+ tombstoneIdsCache.set(storePath, { mtimeMs: st.mtimeMs, size: st.size, ids });
77
+ return ids;
78
+ }
79
+ catch {
80
+ return NO_TOMBSTONES;
81
+ }
82
+ }
43
83
  function appendJsonl(filePath, q) {
44
84
  const dir = path.dirname(filePath);
45
85
  if (!existsSync(dir)) {
@@ -79,9 +119,16 @@ export async function getQuirk(deps, id) {
79
119
  if (!isMemoryStore(deps.storePath)) {
80
120
  const jp = jsonlPath(deps.storePath);
81
121
  if (existsSync(jp)) {
82
- const found = readJsonl(jp).find((q) => q.id === id);
83
- if (found)
84
- return found;
122
+ const records = readJsonl(jp).filter((q) => q.id === id);
123
+ const live = records.find((q) => !isTombstone(q));
124
+ if (live)
125
+ return live;
126
+ // A tombstone is authoritative: never fall through to a surviving store
127
+ // row (crash between the marker and the row delete, or a delete that
128
+ // threw) and report a deleted id as live — updateQuirk would rewrite the
129
+ // marker as a live record.
130
+ if (records.some((q) => isTombstone(q)))
131
+ return undefined;
85
132
  }
86
133
  const chunks = await deps.store.getChunksByFilePath(QUIRK_FILE_PREFIX + id);
87
134
  const c = chunks[0];
@@ -175,6 +222,10 @@ export async function updateQuirk(deps, id, patch) {
175
222
  else {
176
223
  memQuirksFor(deps.storePath).set(id, updated);
177
224
  }
225
+ try {
226
+ await deps.flushState?.();
227
+ }
228
+ catch { /* durable-state upload is best effort */ }
178
229
  return updated;
179
230
  }
180
231
  /** Add a new quirk to the vector store, keyword index, and audit log. */
@@ -211,6 +262,10 @@ export async function addQuirk(deps, input) {
211
262
  else {
212
263
  memQuirksFor(deps.storePath).set(id, quirk);
213
264
  }
265
+ try {
266
+ await deps.flushState?.();
267
+ }
268
+ catch { /* durable-state upload is best effort */ }
214
269
  return quirk;
215
270
  }
216
271
  /**
@@ -246,24 +301,50 @@ export async function removeQuirk(deps, id) {
246
301
  if (!existing) {
247
302
  throw new Error(`Quirk not found: ${id}`);
248
303
  }
249
- const filePath = QUIRK_FILE_PREFIX + id;
250
- await deps.store.deleteByFilePath(filePath);
251
- deps.keywordIndex.removeByFilePath(filePath);
304
+ // Persist the deletion intent BEFORE touching the index. The tombstone is the
305
+ // durable record that survives the union merge, so a crash between the two
306
+ // steps must leave the marker behind — the reverse order (delete rows first)
307
+ // lets reconcile re-embed the quirk and makes the deletion look undone.
308
+ const deletedAt = nowISO();
252
309
  if (!isMemoryStore(deps.storePath)) {
253
310
  const jp = jsonlPath(deps.storePath);
254
- const all = readJsonl(jp).filter((q) => q.id !== id);
311
+ const all = readJsonl(jp);
312
+ const idx = all.findIndex((q) => q.id === id);
313
+ if (idx >= 0) {
314
+ all[idx] = { ...all[idx], deletedAt, lastObserved: deletedAt };
315
+ }
316
+ else {
317
+ // The record was only in the vector store (legacy row): keep a tombstone
318
+ // so the deletion still propagates.
319
+ all.push({ ...existing, deletedAt, lastObserved: deletedAt });
320
+ }
255
321
  rewriteJsonl(jp, all);
256
322
  }
257
323
  else {
258
324
  memQuirksFor(deps.storePath).delete(id);
259
325
  }
326
+ const filePath = QUIRK_FILE_PREFIX + id;
327
+ // Best-effort: the marker is already durable and reconcileQuirks prunes a
328
+ // surviving row as an orphan, so a store error here must not report the
329
+ // deletion as failed — the record is gone from every read path regardless.
330
+ try {
331
+ await deps.store.deleteByFilePath(filePath);
332
+ }
333
+ catch {
334
+ /* reconcile prune */
335
+ }
336
+ deps.keywordIndex.removeByFilePath(filePath);
337
+ try {
338
+ await deps.flushState?.();
339
+ }
340
+ catch { /* durable-state upload is best effort */ }
260
341
  }
261
342
  /** List all quirks sorted by lastObserved descending. */
262
343
  export async function listQuirks(deps) {
263
344
  if (!isMemoryStore(deps.storePath)) {
264
345
  const jp = jsonlPath(deps.storePath);
265
346
  if (existsSync(jp)) {
266
- const all = readJsonl(jp);
347
+ const all = readJsonl(jp).filter((q) => !isTombstone(q));
267
348
  all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
268
349
  return all;
269
350
  }
@@ -294,7 +375,7 @@ export async function listQuirks(deps) {
294
375
  result.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
295
376
  return result;
296
377
  }
297
- const all = [...memQuirksFor(deps.storePath).values()];
378
+ const all = [...memQuirksFor(deps.storePath).values()].filter((q) => !isTombstone(q));
298
379
  all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
299
380
  return all;
300
381
  }
@@ -349,7 +430,11 @@ export async function reconcileQuirks(deps) {
349
430
  const jsonlFile = jsonlPath(deps.storePath);
350
431
  const jsonlExists = existsSync(jsonlFile);
351
432
  const quirks = jsonlExists ? readJsonl(jsonlFile) : [];
352
- const jsonlIds = new Set(quirks.map((q) => q.id));
433
+ // Tombstones are deletion markers, not live quirks: they must never be
434
+ // restored, re-embedded, or counted as "the jsonl still has this id" — their
435
+ // store rows (if any survived) belong on the orphan path below.
436
+ const liveQuirks = quirks.filter((q) => !isTombstone(q));
437
+ const jsonlIds = new Set(liveQuirks.map((q) => q.id));
353
438
  // Exact enumeration of quirk chunks in the table. The store normalizes
354
439
  // filePaths to absolute on write (chunkToRow → normalizeFilePath), so quirk
355
440
  // rows read back as "<root>/quirk:<id>" — match on the basename, which is
@@ -372,7 +457,7 @@ export async function reconcileQuirks(deps) {
372
457
  // per id with the variant-aware lookup. The jsonl content is authoritative:
373
458
  // delete every row, re-embed one raw row below.
374
459
  const dupIds = [];
375
- for (const q of quirks) {
460
+ for (const q of liveQuirks) {
376
461
  if (!storeIds.has(q.id))
377
462
  continue; // absent → the restore path below covers it
378
463
  const rows = await deps.store.getChunksByFilePath(QUIRK_FILE_PREFIX + q.id);
@@ -386,7 +471,7 @@ export async function reconcileQuirks(deps) {
386
471
  result.deduplicated++;
387
472
  }
388
473
  // 1. Restore jsonl quirks the table is missing (the rebuild-wipe direction).
389
- const missing = quirks.filter((q) => !storeIds.has(q.id));
474
+ const missing = liveQuirks.filter((q) => !storeIds.has(q.id));
390
475
  const restoredIds = new Set(missing.map((q) => q.id));
391
476
  if (missing.length > 0) {
392
477
  const prefix = deps.cfg.embedding.documentPrefix ?? "";
@@ -407,7 +492,7 @@ export async function reconcileQuirks(deps) {
407
492
  // content may predate the collapse.
408
493
  let kiDirty = false;
409
494
  if (deps.keywordIndex) {
410
- for (const q of quirks) {
495
+ for (const q of liveQuirks) {
411
496
  if (deps.keywordIndex.hasChunk(q.id) && !restoredIds.has(q.id))
412
497
  continue;
413
498
  deps.keywordIndex.addChunks([quirkChunk(q, [])]);
@@ -488,6 +573,10 @@ export async function reconcileQuirks(deps) {
488
573
  // best-effort
489
574
  }
490
575
  }
576
+ try {
577
+ await deps.flushState?.();
578
+ }
579
+ catch { /* durable-state upload is best effort */ }
491
580
  return result;
492
581
  }
493
582
  /**
@@ -527,7 +616,13 @@ export async function recallQuirks(deps, query, options) {
527
616
  // Confidence + semantic-similarity + type/tag filtering. The similarity gate
528
617
  // is `(cosine + 1) / 2` in [0,1] — the same domain the vector stores report.
529
618
  // A keyword-only candidate has rawVectorScore 0 and never passes minScore > 0.
619
+ // Deleted ids are excluded on the FINAL candidate list, so a surviving store
620
+ // row — or a keyword-index entry that removeByFilePath has not dropped yet —
621
+ // cannot be recalled or auto-injected during the orphan grace window.
622
+ const dead = tombstoneIds(deps.storePath);
530
623
  const filtered = raw.filter((r) => {
624
+ if (dead.has(r.chunk.id))
625
+ return false;
531
626
  const conf = r.chunk.metadata.confidence ?? 1;
532
627
  if (conf < minConfidence)
533
628
  return false;
@@ -576,7 +671,7 @@ export async function recallQuirks(deps, query, options) {
576
671
  * @returns The tool-error quirks, newest `lastObserved` first.
577
672
  */
578
673
  export async function listToolErrorQuirks(deps) {
579
- const tagged = (q) => (q.tags ?? []).includes(TOOL_ERROR_QUIRK_TAG);
674
+ const tagged = (q) => !isTombstone(q) && (q.tags ?? []).includes(TOOL_ERROR_QUIRK_TAG);
580
675
  if (isMemoryStore(deps.storePath)) {
581
676
  return [...memQuirksFor(deps.storePath).values()]
582
677
  .filter(tagged)
@@ -25,4 +25,11 @@ export interface Quirk {
25
25
  confidence: number;
26
26
  lastObserved: string;
27
27
  sourceRef?: string;
28
+ /**
29
+ * Set when the quirk was deleted. The record is kept in `quirks.jsonl` as a
30
+ * tombstone so the deletion survives `mergeQuirkJsonl` (which is a union by
31
+ * id with no negative records) and therefore propagates to the central store
32
+ * on both hydrate and flush. Tombstones are filtered out of every read path.
33
+ */
34
+ deletedAt?: string;
28
35
  }