opencode-rag-plugin 2.10.1 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +40 -28
- package/dist/api.js +1 -0
- package/dist/cli/commands/clickhouse.d.ts +61 -0
- package/dist/cli/commands/clickhouse.js +335 -0
- package/dist/cli/commands/eval-tuner.js +1 -1
- package/dist/cli/commands/eval.js +1 -1
- package/dist/cli/commands/index-command.js +2 -1
- package/dist/cli/commands/index.d.ts +1 -0
- package/dist/cli/commands/index.js +1 -0
- package/dist/cli/commands/init-helpers.js +2 -2
- package/dist/cli/commands/init.js +49 -11
- package/dist/cli/commands/quirk.js +6 -6
- package/dist/cli/commands/status.js +70 -6
- package/dist/cli/commands/ui.js +2 -2
- package/dist/cli/index.js +2 -1
- package/dist/cli/types.d.ts +14 -0
- package/dist/core/bootstrap.d.ts +7 -0
- package/dist/core/bootstrap.js +40 -9
- package/dist/core/config.d.ts +128 -5
- package/dist/core/config.js +205 -8
- package/dist/core/interfaces.d.ts +54 -0
- package/dist/core/runtime-overrides.d.ts +9 -0
- package/dist/core/runtime-overrides.js +44 -0
- package/dist/core/state-sync.d.ts +91 -0
- package/dist/core/state-sync.js +273 -0
- package/dist/eval/fast-index.js +2 -2
- package/dist/eval/rerank-eval.js +2 -2
- package/dist/eval/run-branch-compare.js +2 -2
- package/dist/eval/run-token-test.js +2 -2
- package/dist/index.d.ts +5 -2
- package/dist/index.js +2 -1
- package/dist/indexer/pipeline.d.ts +6 -0
- package/dist/indexer/pipeline.js +173 -56
- package/dist/indexer/progress.d.ts +75 -0
- package/dist/indexer/progress.js +114 -0
- package/dist/indexer/stats.d.ts +14 -0
- package/dist/plugin.d.ts +6 -3
- package/dist/plugin.js +39 -20
- package/dist/quirks/quirk-store.d.ts +10 -0
- package/dist/quirks/quirk-store.js +110 -15
- package/dist/quirks/types.d.ts +7 -0
- package/dist/tui.js +128 -111
- package/dist/vectorstore/clickhouse.d.ts +150 -0
- package/dist/vectorstore/clickhouse.js +931 -0
- package/dist/vectorstore/dimension.d.ts +19 -0
- package/dist/vectorstore/dimension.js +36 -0
- package/dist/vectorstore/factory.d.ts +9 -3
- package/dist/vectorstore/factory.js +11 -2
- package/dist/vectorstore/lancedb.d.ts +2 -2
- package/dist/watcher.d.ts +29 -0
- package/dist/watcher.js +76 -2
- package/dist/web/api.d.ts +3 -4
- package/dist/web/api.js +18 -3
- package/dist/web/server.js +28 -2
- package/dist/web/ui/assets/index-B-RH6fko.css +2 -0
- package/dist/web/ui/index.html +2 -2
- package/package.json +2 -1
- package/dist/web/ui/assets/index-CWcp0jNf.css +0 -2
- /package/dist/web/ui/assets/{index-DbXFrn53.js → index-ChGi2bSp.js} +0 -0
package/dist/indexer/stats.d.ts
CHANGED
|
@@ -70,6 +70,20 @@ export interface IndexStatusSummary {
|
|
|
70
70
|
* reachable through any accounting. The next full index pass sweeps them.
|
|
71
71
|
*/
|
|
72
72
|
orphanChunks: number;
|
|
73
|
+
/**
|
|
74
|
+
* Whether {@link upToDateFiles} and {@link pendingFiles} were actually
|
|
75
|
+
* computed from a workspace scan. When false the two counters are zero
|
|
76
|
+
* because no scan ran — never because every file is current.
|
|
77
|
+
*/
|
|
78
|
+
scanned: boolean;
|
|
79
|
+
/**
|
|
80
|
+
* Stored chunks owned by quirks (`quirk:` file paths). quirks.jsonl owns
|
|
81
|
+
* them, so they belong to neither {@link manifestExpectedChunks} nor
|
|
82
|
+
* {@link orphanChunks}; without this line `Indexed chunks` does not
|
|
83
|
+
* reconcile against `Expected + Orphan`. Undefined when the store cannot
|
|
84
|
+
* enumerate its files.
|
|
85
|
+
*/
|
|
86
|
+
quirkChunks?: number;
|
|
73
87
|
}
|
|
74
88
|
/**
|
|
75
89
|
* Create a zeroed-out {@link IndexRunStats} instance initialised with the
|
package/dist/plugin.d.ts
CHANGED
|
@@ -7,12 +7,13 @@ import type { Plugin, Hooks } from "@opencode-ai/plugin";
|
|
|
7
7
|
import { type EmbeddingProvider, type DescriptionProvider, type KeywordIndex, type VectorStore } from "./core/interfaces.js";
|
|
8
8
|
import { type RagConfig } from "./core/config.js";
|
|
9
9
|
import { createEmbedder } from "./embedder/factory.js";
|
|
10
|
+
import { StateSync } from "./core/state-sync.js";
|
|
10
11
|
import { retrieve } from "./retriever/retriever.js";
|
|
11
12
|
/**
|
|
12
13
|
* Load the RAG configuration for a directory: DEFAULT_CONFIG, then the global
|
|
13
14
|
* layer (`~/.config/opencode/opencode-rag.json`), then the first workspace
|
|
14
|
-
* config found (
|
|
15
|
-
* `.opencode/rag.json`).
|
|
15
|
+
* config found (`.opencode/opencode-rag.json`, then the legacy
|
|
16
|
+
* `opencode-rag.json` / `.opencode/rag.json`).
|
|
16
17
|
*
|
|
17
18
|
* The result is cached per directory and re-read whenever either config file
|
|
18
19
|
* changes on disk, so an edited API key, baseUrl, or global `shellGuard` switch
|
|
@@ -57,7 +58,7 @@ export declare function createLiveEmbedder(getConfig: () => RagConfig, create: (
|
|
|
57
58
|
/** Internal dependencies that can be overridden for testing. */
|
|
58
59
|
type RagPluginDependencies = {
|
|
59
60
|
createEmbedder: typeof createEmbedder;
|
|
60
|
-
createStore: (storePath: string, dimension: number, config: RagConfig) => VectorStore;
|
|
61
|
+
createStore: (storePath: string, dimension: number, config: RagConfig, workspaceDir: string) => VectorStore;
|
|
61
62
|
retrieve: typeof retrieve;
|
|
62
63
|
};
|
|
63
64
|
/** Options for constructing the RAG hooks object. */
|
|
@@ -82,6 +83,8 @@ type CreateRagHooksOptions = {
|
|
|
82
83
|
keywordIndex?: KeywordIndex;
|
|
83
84
|
/** Pre-created description provider for chunk descriptions. */
|
|
84
85
|
descriptionProvider?: DescriptionProvider;
|
|
86
|
+
/** Durable-state synchronizer (ClickHouse); used to flush quirk writes. */
|
|
87
|
+
stateSync?: StateSync;
|
|
85
88
|
};
|
|
86
89
|
/**
|
|
87
90
|
* Plugin tool names advertised by the mandatory guidance and granted to the
|
package/dist/plugin.js
CHANGED
|
@@ -8,7 +8,8 @@ import { CODE_SEARCH_FILTER } from "./core/interfaces.js";
|
|
|
8
8
|
import { normalizeFileExtensions } from "./core/filters.js";
|
|
9
9
|
import { loadWorkspaceConfig, resolveConfigPaths, findConfigFile, DEFAULT_CONFIG, resolveLogConfig, persistProbedDimension } from "./core/config.js";
|
|
10
10
|
import { createEmbedder, probeEmbeddingDimension } from "./embedder/factory.js";
|
|
11
|
-
import {
|
|
11
|
+
import { readConfiguredStoreDimension } from "./vectorstore/dimension.js";
|
|
12
|
+
import { StateSync } from "./core/state-sync.js";
|
|
12
13
|
import { createDescriptionProvider } from "./describer/factory.js";
|
|
13
14
|
import { createVectorStore } from "./vectorstore/factory.js";
|
|
14
15
|
import { retrieve } from "./retriever/retriever.js";
|
|
@@ -244,8 +245,8 @@ function configStamp(paths) {
|
|
|
244
245
|
/**
|
|
245
246
|
* Load the RAG configuration for a directory: DEFAULT_CONFIG, then the global
|
|
246
247
|
* layer (`~/.config/opencode/opencode-rag.json`), then the first workspace
|
|
247
|
-
* config found (
|
|
248
|
-
* `.opencode/rag.json`).
|
|
248
|
+
* config found (`.opencode/opencode-rag.json`, then the legacy
|
|
249
|
+
* `opencode-rag.json` / `.opencode/rag.json`).
|
|
249
250
|
*
|
|
250
251
|
* The result is cached per directory and re-read whenever either config file
|
|
251
252
|
* changes on disk, so an edited API key, baseUrl, or global `shellGuard` switch
|
|
@@ -504,7 +505,7 @@ async function loadRetrievedResults(query, embedder, store, cfg, retrieveFn = re
|
|
|
504
505
|
/** Default set of plugin dependencies using the real implementations. */
|
|
505
506
|
const defaultDependencies = {
|
|
506
507
|
createEmbedder,
|
|
507
|
-
createStore: (storePath, dimension, config) => createVectorStore(config, storePath, dimension),
|
|
508
|
+
createStore: (storePath, dimension, config, workspaceDir) => createVectorStore(config, storePath, dimension, workspaceDir),
|
|
508
509
|
retrieve,
|
|
509
510
|
};
|
|
510
511
|
/**
|
|
@@ -639,8 +640,11 @@ export function createRagHooks(options) {
|
|
|
639
640
|
...options.dependencies,
|
|
640
641
|
};
|
|
641
642
|
const configuredDimension = options.cfg.embedding.vectorDimension;
|
|
642
|
-
const store = options.store ?? dependencies.createStore(options.storePath, configuredDimension && configuredDimension > 0 ? configuredDimension : 384, options.cfg);
|
|
643
|
+
const store = options.store ?? dependencies.createStore(options.storePath, configuredDimension && configuredDimension > 0 ? configuredDimension : 384, options.cfg, options.worktree);
|
|
643
644
|
const keywordIndex = options.keywordIndex;
|
|
645
|
+
const flushState = options.stateSync
|
|
646
|
+
? () => options.stateSync.flush(["quirks"])
|
|
647
|
+
: undefined;
|
|
644
648
|
// Runtime overrides for live config editing from TUI
|
|
645
649
|
let cachedOverrides = loadRuntimeOverrides(options.storePath);
|
|
646
650
|
let overridesLastCheck = 0;
|
|
@@ -1029,7 +1033,7 @@ export function createRagHooks(options) {
|
|
|
1029
1033
|
maxPerSession: memCfg.toolErrorMaxPerSession ?? 3,
|
|
1030
1034
|
});
|
|
1031
1035
|
boundedSet(sessionToolErrorState, sessionID, state, MAX_SESSION_MAP_SIZE);
|
|
1032
|
-
const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
|
|
1036
|
+
const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
|
|
1033
1037
|
// `explain: true` keeps the raw vector similarity on each result —
|
|
1034
1038
|
// the confidence-discounted `score` is not comparable to a
|
|
1035
1039
|
// similarity threshold, and the tiering below needs the raw value.
|
|
@@ -1275,7 +1279,7 @@ export function createRagHooks(options) {
|
|
|
1275
1279
|
const effCfg = getEffectiveCfg();
|
|
1276
1280
|
const memCfg = effCfg.memory;
|
|
1277
1281
|
if (memCfg?.sessionEndExtraction && options.descriptionProvider) {
|
|
1278
|
-
const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: effCfg, storePath: options.storePath };
|
|
1282
|
+
const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: effCfg, storePath: options.storePath, flushState: flushState };
|
|
1279
1283
|
void autoCaptureQuirks(qDeps, options.descriptionProvider, transcript, effCfg, { captureAll: true }).catch(() => { });
|
|
1280
1284
|
}
|
|
1281
1285
|
boundedSet(sessionTranscript, sessionID, [], MAX_SESSION_MAP_SIZE);
|
|
@@ -1336,7 +1340,7 @@ export function createRagHooks(options) {
|
|
|
1336
1340
|
const current = (sessionShellLongCount.get(counterKey) ?? 0);
|
|
1337
1341
|
if (current < 3) {
|
|
1338
1342
|
sessionShellLongCount.set(counterKey, current + 1);
|
|
1339
|
-
const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
|
|
1343
|
+
const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
|
|
1340
1344
|
void addQuirk(quirkDeps, {
|
|
1341
1345
|
content: `A foreground shell command took ${durSec}s: \`${cmd}\`. Prefer \`background: true\` or an explicit \`timeout\` for long commands; press Ctrl+B to background a running call.`,
|
|
1342
1346
|
quirkType: "gotcha",
|
|
@@ -1428,7 +1432,7 @@ export function createRagHooks(options) {
|
|
|
1428
1432
|
if (toolNames.length > 0) {
|
|
1429
1433
|
const memCfg = getEffectiveCfg().memory;
|
|
1430
1434
|
if (memCfg?.enabled && memCfg.toolErrorProactiveInject) {
|
|
1431
|
-
const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
|
|
1435
|
+
const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
|
|
1432
1436
|
const gotchas = selectToolGotchas(await getToolErrorQuirks(qDeps), toolNames, memCfg.toolErrorProactiveMaxItems ?? TOOL_ERROR_CREATE_DEFAULTS.maxProactiveItems);
|
|
1433
1437
|
if (gotchas.length > 0) {
|
|
1434
1438
|
output.system.unshift(formatToolGotchaBlock(gotchas));
|
|
@@ -1490,7 +1494,7 @@ export function createRagHooks(options) {
|
|
|
1490
1494
|
const userReq = sessionPrevUserReq.get(quirkSessionId) ?? "";
|
|
1491
1495
|
const query = [assistantText, userReq].filter((t) => t.length > 0).join("\n");
|
|
1492
1496
|
if (query.length > 0) {
|
|
1493
|
-
const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
|
|
1497
|
+
const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
|
|
1494
1498
|
const injectedSet = getOrCreateSessionSet(sessionInjectedQuirks, quirkSessionId, MAX_SESSION_MAP_SIZE);
|
|
1495
1499
|
const selection = await selectAutoInjectQuirksWithinBudget((q, o) => recallQuirks(quirkDeps, q, { ...o, excludeTags: [TOOL_ERROR_QUIRK_TAG] }), {
|
|
1496
1500
|
query,
|
|
@@ -1570,7 +1574,7 @@ export function createRagHooks(options) {
|
|
|
1570
1574
|
const assistantText = sessionAssistantText.get(input.sessionID) ?? "";
|
|
1571
1575
|
const toolResults = sessionToolResults.get(input.sessionID) ?? [];
|
|
1572
1576
|
if (assistantText) {
|
|
1573
|
-
const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: effCfg, storePath: options.storePath };
|
|
1577
|
+
const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: effCfg, storePath: options.storePath, flushState: flushState };
|
|
1574
1578
|
const exchange = { userReq: prevUserReq, assistantText, toolResults };
|
|
1575
1579
|
if (memCfg?.passiveCapture) {
|
|
1576
1580
|
void autoCaptureQuirks(qDeps, options.descriptionProvider, [exchange], effCfg).catch(() => { });
|
|
@@ -1594,7 +1598,7 @@ export function createRagHooks(options) {
|
|
|
1594
1598
|
if (Array.isArray(parts) && parts.length > 0) {
|
|
1595
1599
|
const first = parts[0];
|
|
1596
1600
|
if (typeof first.text === "string") {
|
|
1597
|
-
parts[0] = { ...first, text: "Documentation mode is not enabled. Set `documentationMode.enabled` to `true` in opencode-rag.json." };
|
|
1601
|
+
parts[0] = { ...first, text: "Documentation mode is not enabled. Set `documentationMode.enabled` to `true` in .opencode/opencode-rag.json." };
|
|
1598
1602
|
}
|
|
1599
1603
|
}
|
|
1600
1604
|
return;
|
|
@@ -1678,7 +1682,7 @@ export function createRagHooks(options) {
|
|
|
1678
1682
|
if (Array.isArray(parts) && parts.length > 0) {
|
|
1679
1683
|
const first = parts[0];
|
|
1680
1684
|
if (typeof first.text === "string") {
|
|
1681
|
-
parts[0] = { ...first, text: "Wiki mode is not enabled. Set `wikiMode.enabled` to `true` in opencode-rag.json." };
|
|
1685
|
+
parts[0] = { ...first, text: "Wiki mode is not enabled. Set `wikiMode.enabled` to `true` in .opencode/opencode-rag.json." };
|
|
1682
1686
|
}
|
|
1683
1687
|
}
|
|
1684
1688
|
return;
|
|
@@ -1736,7 +1740,7 @@ export function createRagHooks(options) {
|
|
|
1736
1740
|
if (Array.isArray(parts) && parts.length > 0) {
|
|
1737
1741
|
const first = parts[0];
|
|
1738
1742
|
if (typeof first.text === "string") {
|
|
1739
|
-
parts[0] = { ...first, text: "Quirk memory is not enabled. Set `memory.enabled` to `true` in opencode-rag.json." };
|
|
1743
|
+
parts[0] = { ...first, text: "Quirk memory is not enabled. Set `memory.enabled` to `true` in .opencode/opencode-rag.json." };
|
|
1740
1744
|
}
|
|
1741
1745
|
}
|
|
1742
1746
|
return;
|
|
@@ -1744,7 +1748,7 @@ export function createRagHooks(options) {
|
|
|
1744
1748
|
const arg = text.slice(6).trim().toLowerCase();
|
|
1745
1749
|
const lines = [];
|
|
1746
1750
|
if (arg === "lint") {
|
|
1747
|
-
const deps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
|
|
1751
|
+
const deps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
|
|
1748
1752
|
const issues = await lintQuirks(deps);
|
|
1749
1753
|
lines.push("## Quirk Lint", "");
|
|
1750
1754
|
if (issues.length === 0) {
|
|
@@ -1763,7 +1767,7 @@ export function createRagHooks(options) {
|
|
|
1763
1767
|
}
|
|
1764
1768
|
else {
|
|
1765
1769
|
// Status / list
|
|
1766
|
-
const deps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
|
|
1770
|
+
const deps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
|
|
1767
1771
|
const quirks = await listQuirks(deps);
|
|
1768
1772
|
lines.push("## Quirk Memory", "");
|
|
1769
1773
|
if (quirks.length === 0) {
|
|
@@ -1809,7 +1813,7 @@ export function createRagHooks(options) {
|
|
|
1809
1813
|
const assistantText = sessionAssistantText.get(input.sessionID) ?? "";
|
|
1810
1814
|
const quirkQuery = [assistantText, text].filter((t) => t.length > 0).join("\n");
|
|
1811
1815
|
if (quirkQuery.length > 0) {
|
|
1812
|
-
const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
|
|
1816
|
+
const quirkDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath, flushState: flushState };
|
|
1813
1817
|
const injectedSet = getOrCreateSessionSet(sessionInjectedQuirks, input.sessionID, MAX_SESSION_MAP_SIZE);
|
|
1814
1818
|
const selection = await selectAutoInjectQuirksWithinBudget((q, o) => recallQuirks(quirkDeps, q, { ...o, excludeTags: [TOOL_ERROR_QUIRK_TAG] }), {
|
|
1815
1819
|
query: quirkQuery,
|
|
@@ -2089,7 +2093,7 @@ export const ragPlugin = async (input, _options) => {
|
|
|
2089
2093
|
}, logLevel);
|
|
2090
2094
|
}
|
|
2091
2095
|
else {
|
|
2092
|
-
vectorDimension = (await
|
|
2096
|
+
vectorDimension = (await readConfiguredStoreDimension(effectiveCfg, storePath, input.directory)) ?? 384;
|
|
2093
2097
|
appendDebugLog(logFilePath, {
|
|
2094
2098
|
scope: "plugin",
|
|
2095
2099
|
message: `Dimension probe failed, falling back to ${vectorDimension}`,
|
|
@@ -2097,8 +2101,21 @@ export const ragPlugin = async (input, _options) => {
|
|
|
2097
2101
|
}, logLevel);
|
|
2098
2102
|
}
|
|
2099
2103
|
}
|
|
2100
|
-
const store = createVectorStore(effectiveCfg, storePath, vectorDimension);
|
|
2104
|
+
const store = createVectorStore(effectiveCfg, storePath, vectorDimension, input.directory);
|
|
2101
2105
|
ragStores.set(input.directory, store);
|
|
2106
|
+
// ClickHouse: pull the durable manifest/keyword index/quirks/overrides into
|
|
2107
|
+
// the disposable local cache before anything reads them.
|
|
2108
|
+
const stateSync = new StateSync({ store, storePath, config: effectiveCfg });
|
|
2109
|
+
try {
|
|
2110
|
+
await stateSync.hydrate();
|
|
2111
|
+
}
|
|
2112
|
+
catch (err) {
|
|
2113
|
+
appendDebugLog(logFilePath, {
|
|
2114
|
+
scope: "plugin",
|
|
2115
|
+
message: "State hydration failed",
|
|
2116
|
+
error: err,
|
|
2117
|
+
}, logLevel);
|
|
2118
|
+
}
|
|
2102
2119
|
// Warn when the store was built by a different embedding model — vector
|
|
2103
2120
|
// search fails (or silently degrades) until the index is rebuilt.
|
|
2104
2121
|
try {
|
|
@@ -2120,7 +2137,7 @@ export const ragPlugin = async (input, _options) => {
|
|
|
2120
2137
|
// or keyword index (full rebuilds drop quirk chunks — see reconcileQuirks).
|
|
2121
2138
|
// Fire-and-forget: must never block plugin startup, and the embedder may be
|
|
2122
2139
|
// offline. The background watcher re-runs this at every pass end.
|
|
2123
|
-
void reconcileQuirks({ embedder: liveEmbedder, store, keywordIndex, cfg: effectiveCfg, storePath })
|
|
2140
|
+
void reconcileQuirks({ embedder: liveEmbedder, store, keywordIndex, cfg: effectiveCfg, storePath, flushState: () => stateSync.flush(["quirks"]) })
|
|
2124
2141
|
.then((r) => {
|
|
2125
2142
|
if (r.restoredToStore > 0 || r.addedToKeywordIndex > 0 || r.removedOrphans > 0) {
|
|
2126
2143
|
appendDebugLog(logFilePath, {
|
|
@@ -2151,6 +2168,7 @@ export const ragPlugin = async (input, _options) => {
|
|
|
2151
2168
|
store,
|
|
2152
2169
|
keywordIndex,
|
|
2153
2170
|
descriptionProvider,
|
|
2171
|
+
stateSync,
|
|
2154
2172
|
});
|
|
2155
2173
|
// Start background auto-indexer if enabled
|
|
2156
2174
|
const autoIndexCfg = effectiveCfg.openCode.autoIndex ?? { enabled: false, debounceMs: 5000, intervalMs: 300000 };
|
|
@@ -2165,6 +2183,7 @@ export const ragPlugin = async (input, _options) => {
|
|
|
2165
2183
|
logLevel,
|
|
2166
2184
|
keywordIndex,
|
|
2167
2185
|
descriptionProvider,
|
|
2186
|
+
stateSync,
|
|
2168
2187
|
dimension: vectorDimension,
|
|
2169
2188
|
});
|
|
2170
2189
|
backgroundIndexers.set(input.directory, indexer);
|
|
@@ -8,8 +8,18 @@ export interface QuirkStoreDeps {
|
|
|
8
8
|
keywordIndex: KeywordIndex;
|
|
9
9
|
cfg: RagConfig;
|
|
10
10
|
storePath: string;
|
|
11
|
+
/** Upload the durable quirk log after a write (ClickHouse state bundle). */
|
|
12
|
+
flushState?: () => Promise<void>;
|
|
11
13
|
}
|
|
12
14
|
export declare const QUIRK_FILE_PREFIX = "quirk:";
|
|
15
|
+
/**
|
|
16
|
+
* True when a JSONL record is a deletion tombstone rather than a live quirk.
|
|
17
|
+
*
|
|
18
|
+
* Tombstones exist so a deletion survives `mergeQuirkJsonl`, which is a union
|
|
19
|
+
* by id and would otherwise resurrect the record from the central store on the
|
|
20
|
+
* next hydrate/flush. Every read path must filter them out.
|
|
21
|
+
*/
|
|
22
|
+
export declare function isTombstone(q: Quirk): boolean;
|
|
13
23
|
/** Look up a single quirk by its ID, or `undefined` when not found. */
|
|
14
24
|
export declare function getQuirk(deps: QuirkStoreDeps, id: string): Promise<Quirk | undefined>;
|
|
15
25
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { appendFileSync, existsSync, readFileSync, writeFileSync, mkdirSync, renameSync, rmSync } from "node:fs";
|
|
1
|
+
import { appendFileSync, existsSync, readFileSync, statSync, writeFileSync, mkdirSync, renameSync, rmSync } from "node:fs";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { retrieve } from "../retriever/retriever.js";
|
|
@@ -40,6 +40,46 @@ function readJsonl(filePath) {
|
|
|
40
40
|
}
|
|
41
41
|
return quirks;
|
|
42
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* True when a JSONL record is a deletion tombstone rather than a live quirk.
|
|
45
|
+
*
|
|
46
|
+
* Tombstones exist so a deletion survives `mergeQuirkJsonl`, which is a union
|
|
47
|
+
* by id and would otherwise resurrect the record from the central store on the
|
|
48
|
+
* next hydrate/flush. Every read path must filter them out.
|
|
49
|
+
*/
|
|
50
|
+
export function isTombstone(q) {
|
|
51
|
+
return typeof q.deletedAt === "string" && q.deletedAt.length > 0;
|
|
52
|
+
}
|
|
53
|
+
const NO_TOMBSTONES = new Set();
|
|
54
|
+
/**
|
|
55
|
+
* Tombstone ids for a store, cached on `quirks.jsonl` mtime + size.
|
|
56
|
+
*
|
|
57
|
+
* `recallQuirks` runs on every turn (auto-inject) and on every tool error, so it
|
|
58
|
+
* must not parse the whole audit log per call. A missing or unreadable log means
|
|
59
|
+
* "no tombstone knowledge" — never an error and never filter-everything:
|
|
60
|
+
* `removeQuirk`'s atomic rewrite has a direct-write fallback, and a momentary
|
|
61
|
+
* unreadable file must not change recall behaviour.
|
|
62
|
+
*/
|
|
63
|
+
const tombstoneIdsCache = new Map();
|
|
64
|
+
function tombstoneIds(storePath) {
|
|
65
|
+
if (isMemoryStore(storePath))
|
|
66
|
+
return NO_TOMBSTONES; // memory stores delete outright
|
|
67
|
+
const jp = jsonlPath(storePath);
|
|
68
|
+
try {
|
|
69
|
+
const st = statSync(jp);
|
|
70
|
+
const hit = tombstoneIdsCache.get(storePath);
|
|
71
|
+
if (hit && hit.mtimeMs === st.mtimeMs && hit.size === st.size)
|
|
72
|
+
return hit.ids;
|
|
73
|
+
const ids = new Set(readJsonl(jp).filter(isTombstone).map((q) => q.id));
|
|
74
|
+
if (tombstoneIdsCache.size > 16)
|
|
75
|
+
tombstoneIdsCache.clear(); // bounded: keyed by store path
|
|
76
|
+
tombstoneIdsCache.set(storePath, { mtimeMs: st.mtimeMs, size: st.size, ids });
|
|
77
|
+
return ids;
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
return NO_TOMBSTONES;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
43
83
|
function appendJsonl(filePath, q) {
|
|
44
84
|
const dir = path.dirname(filePath);
|
|
45
85
|
if (!existsSync(dir)) {
|
|
@@ -79,9 +119,16 @@ export async function getQuirk(deps, id) {
|
|
|
79
119
|
if (!isMemoryStore(deps.storePath)) {
|
|
80
120
|
const jp = jsonlPath(deps.storePath);
|
|
81
121
|
if (existsSync(jp)) {
|
|
82
|
-
const
|
|
83
|
-
|
|
84
|
-
|
|
122
|
+
const records = readJsonl(jp).filter((q) => q.id === id);
|
|
123
|
+
const live = records.find((q) => !isTombstone(q));
|
|
124
|
+
if (live)
|
|
125
|
+
return live;
|
|
126
|
+
// A tombstone is authoritative: never fall through to a surviving store
|
|
127
|
+
// row (crash between the marker and the row delete, or a delete that
|
|
128
|
+
// threw) and report a deleted id as live — updateQuirk would rewrite the
|
|
129
|
+
// marker as a live record.
|
|
130
|
+
if (records.some((q) => isTombstone(q)))
|
|
131
|
+
return undefined;
|
|
85
132
|
}
|
|
86
133
|
const chunks = await deps.store.getChunksByFilePath(QUIRK_FILE_PREFIX + id);
|
|
87
134
|
const c = chunks[0];
|
|
@@ -175,6 +222,10 @@ export async function updateQuirk(deps, id, patch) {
|
|
|
175
222
|
else {
|
|
176
223
|
memQuirksFor(deps.storePath).set(id, updated);
|
|
177
224
|
}
|
|
225
|
+
try {
|
|
226
|
+
await deps.flushState?.();
|
|
227
|
+
}
|
|
228
|
+
catch { /* durable-state upload is best effort */ }
|
|
178
229
|
return updated;
|
|
179
230
|
}
|
|
180
231
|
/** Add a new quirk to the vector store, keyword index, and audit log. */
|
|
@@ -211,6 +262,10 @@ export async function addQuirk(deps, input) {
|
|
|
211
262
|
else {
|
|
212
263
|
memQuirksFor(deps.storePath).set(id, quirk);
|
|
213
264
|
}
|
|
265
|
+
try {
|
|
266
|
+
await deps.flushState?.();
|
|
267
|
+
}
|
|
268
|
+
catch { /* durable-state upload is best effort */ }
|
|
214
269
|
return quirk;
|
|
215
270
|
}
|
|
216
271
|
/**
|
|
@@ -246,24 +301,50 @@ export async function removeQuirk(deps, id) {
|
|
|
246
301
|
if (!existing) {
|
|
247
302
|
throw new Error(`Quirk not found: ${id}`);
|
|
248
303
|
}
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
304
|
+
// Persist the deletion intent BEFORE touching the index. The tombstone is the
|
|
305
|
+
// durable record that survives the union merge, so a crash between the two
|
|
306
|
+
// steps must leave the marker behind — the reverse order (delete rows first)
|
|
307
|
+
// lets reconcile re-embed the quirk and makes the deletion look undone.
|
|
308
|
+
const deletedAt = nowISO();
|
|
252
309
|
if (!isMemoryStore(deps.storePath)) {
|
|
253
310
|
const jp = jsonlPath(deps.storePath);
|
|
254
|
-
const all = readJsonl(jp)
|
|
311
|
+
const all = readJsonl(jp);
|
|
312
|
+
const idx = all.findIndex((q) => q.id === id);
|
|
313
|
+
if (idx >= 0) {
|
|
314
|
+
all[idx] = { ...all[idx], deletedAt, lastObserved: deletedAt };
|
|
315
|
+
}
|
|
316
|
+
else {
|
|
317
|
+
// The record was only in the vector store (legacy row): keep a tombstone
|
|
318
|
+
// so the deletion still propagates.
|
|
319
|
+
all.push({ ...existing, deletedAt, lastObserved: deletedAt });
|
|
320
|
+
}
|
|
255
321
|
rewriteJsonl(jp, all);
|
|
256
322
|
}
|
|
257
323
|
else {
|
|
258
324
|
memQuirksFor(deps.storePath).delete(id);
|
|
259
325
|
}
|
|
326
|
+
const filePath = QUIRK_FILE_PREFIX + id;
|
|
327
|
+
// Best-effort: the marker is already durable and reconcileQuirks prunes a
|
|
328
|
+
// surviving row as an orphan, so a store error here must not report the
|
|
329
|
+
// deletion as failed — the record is gone from every read path regardless.
|
|
330
|
+
try {
|
|
331
|
+
await deps.store.deleteByFilePath(filePath);
|
|
332
|
+
}
|
|
333
|
+
catch {
|
|
334
|
+
/* reconcile prune */
|
|
335
|
+
}
|
|
336
|
+
deps.keywordIndex.removeByFilePath(filePath);
|
|
337
|
+
try {
|
|
338
|
+
await deps.flushState?.();
|
|
339
|
+
}
|
|
340
|
+
catch { /* durable-state upload is best effort */ }
|
|
260
341
|
}
|
|
261
342
|
/** List all quirks sorted by lastObserved descending. */
|
|
262
343
|
export async function listQuirks(deps) {
|
|
263
344
|
if (!isMemoryStore(deps.storePath)) {
|
|
264
345
|
const jp = jsonlPath(deps.storePath);
|
|
265
346
|
if (existsSync(jp)) {
|
|
266
|
-
const all = readJsonl(jp);
|
|
347
|
+
const all = readJsonl(jp).filter((q) => !isTombstone(q));
|
|
267
348
|
all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
|
|
268
349
|
return all;
|
|
269
350
|
}
|
|
@@ -294,7 +375,7 @@ export async function listQuirks(deps) {
|
|
|
294
375
|
result.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
|
|
295
376
|
return result;
|
|
296
377
|
}
|
|
297
|
-
const all = [...memQuirksFor(deps.storePath).values()];
|
|
378
|
+
const all = [...memQuirksFor(deps.storePath).values()].filter((q) => !isTombstone(q));
|
|
298
379
|
all.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
|
|
299
380
|
return all;
|
|
300
381
|
}
|
|
@@ -349,7 +430,11 @@ export async function reconcileQuirks(deps) {
|
|
|
349
430
|
const jsonlFile = jsonlPath(deps.storePath);
|
|
350
431
|
const jsonlExists = existsSync(jsonlFile);
|
|
351
432
|
const quirks = jsonlExists ? readJsonl(jsonlFile) : [];
|
|
352
|
-
|
|
433
|
+
// Tombstones are deletion markers, not live quirks: they must never be
|
|
434
|
+
// restored, re-embedded, or counted as "the jsonl still has this id" — their
|
|
435
|
+
// store rows (if any survived) belong on the orphan path below.
|
|
436
|
+
const liveQuirks = quirks.filter((q) => !isTombstone(q));
|
|
437
|
+
const jsonlIds = new Set(liveQuirks.map((q) => q.id));
|
|
353
438
|
// Exact enumeration of quirk chunks in the table. The store normalizes
|
|
354
439
|
// filePaths to absolute on write (chunkToRow → normalizeFilePath), so quirk
|
|
355
440
|
// rows read back as "<root>/quirk:<id>" — match on the basename, which is
|
|
@@ -372,7 +457,7 @@ export async function reconcileQuirks(deps) {
|
|
|
372
457
|
// per id with the variant-aware lookup. The jsonl content is authoritative:
|
|
373
458
|
// delete every row, re-embed one raw row below.
|
|
374
459
|
const dupIds = [];
|
|
375
|
-
for (const q of
|
|
460
|
+
for (const q of liveQuirks) {
|
|
376
461
|
if (!storeIds.has(q.id))
|
|
377
462
|
continue; // absent → the restore path below covers it
|
|
378
463
|
const rows = await deps.store.getChunksByFilePath(QUIRK_FILE_PREFIX + q.id);
|
|
@@ -386,7 +471,7 @@ export async function reconcileQuirks(deps) {
|
|
|
386
471
|
result.deduplicated++;
|
|
387
472
|
}
|
|
388
473
|
// 1. Restore jsonl quirks the table is missing (the rebuild-wipe direction).
|
|
389
|
-
const missing =
|
|
474
|
+
const missing = liveQuirks.filter((q) => !storeIds.has(q.id));
|
|
390
475
|
const restoredIds = new Set(missing.map((q) => q.id));
|
|
391
476
|
if (missing.length > 0) {
|
|
392
477
|
const prefix = deps.cfg.embedding.documentPrefix ?? "";
|
|
@@ -407,7 +492,7 @@ export async function reconcileQuirks(deps) {
|
|
|
407
492
|
// content may predate the collapse.
|
|
408
493
|
let kiDirty = false;
|
|
409
494
|
if (deps.keywordIndex) {
|
|
410
|
-
for (const q of
|
|
495
|
+
for (const q of liveQuirks) {
|
|
411
496
|
if (deps.keywordIndex.hasChunk(q.id) && !restoredIds.has(q.id))
|
|
412
497
|
continue;
|
|
413
498
|
deps.keywordIndex.addChunks([quirkChunk(q, [])]);
|
|
@@ -488,6 +573,10 @@ export async function reconcileQuirks(deps) {
|
|
|
488
573
|
// best-effort
|
|
489
574
|
}
|
|
490
575
|
}
|
|
576
|
+
try {
|
|
577
|
+
await deps.flushState?.();
|
|
578
|
+
}
|
|
579
|
+
catch { /* durable-state upload is best effort */ }
|
|
491
580
|
return result;
|
|
492
581
|
}
|
|
493
582
|
/**
|
|
@@ -527,7 +616,13 @@ export async function recallQuirks(deps, query, options) {
|
|
|
527
616
|
// Confidence + semantic-similarity + type/tag filtering. The similarity gate
|
|
528
617
|
// is `(cosine + 1) / 2` in [0,1] — the same domain the vector stores report.
|
|
529
618
|
// A keyword-only candidate has rawVectorScore 0 and never passes minScore > 0.
|
|
619
|
+
// Deleted ids are excluded on the FINAL candidate list, so a surviving store
|
|
620
|
+
// row — or a keyword-index entry that removeByFilePath has not dropped yet —
|
|
621
|
+
// cannot be recalled or auto-injected during the orphan grace window.
|
|
622
|
+
const dead = tombstoneIds(deps.storePath);
|
|
530
623
|
const filtered = raw.filter((r) => {
|
|
624
|
+
if (dead.has(r.chunk.id))
|
|
625
|
+
return false;
|
|
531
626
|
const conf = r.chunk.metadata.confidence ?? 1;
|
|
532
627
|
if (conf < minConfidence)
|
|
533
628
|
return false;
|
|
@@ -576,7 +671,7 @@ export async function recallQuirks(deps, query, options) {
|
|
|
576
671
|
* @returns The tool-error quirks, newest `lastObserved` first.
|
|
577
672
|
*/
|
|
578
673
|
export async function listToolErrorQuirks(deps) {
|
|
579
|
-
const tagged = (q) => (q.tags ?? []).includes(TOOL_ERROR_QUIRK_TAG);
|
|
674
|
+
const tagged = (q) => !isTombstone(q) && (q.tags ?? []).includes(TOOL_ERROR_QUIRK_TAG);
|
|
580
675
|
if (isMemoryStore(deps.storePath)) {
|
|
581
676
|
return [...memQuirksFor(deps.storePath).values()]
|
|
582
677
|
.filter(tagged)
|
package/dist/quirks/types.d.ts
CHANGED
|
@@ -25,4 +25,11 @@ export interface Quirk {
|
|
|
25
25
|
confidence: number;
|
|
26
26
|
lastObserved: string;
|
|
27
27
|
sourceRef?: string;
|
|
28
|
+
/**
|
|
29
|
+
* Set when the quirk was deleted. The record is kept in `quirks.jsonl` as a
|
|
30
|
+
* tombstone so the deletion survives `mergeQuirkJsonl` (which is a union by
|
|
31
|
+
* id with no negative records) and therefore propagates to the central store
|
|
32
|
+
* on both hydrate and flush. Tombstones are filtered out of every read path.
|
|
33
|
+
*/
|
|
34
|
+
deletedAt?: string;
|
|
28
35
|
}
|