opencode-rag-plugin 2.6.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +7 -4
- package/dist/cli/commands/eval-tuner.d.ts +15 -0
- package/dist/cli/commands/eval-tuner.js +285 -0
- package/dist/cli/commands/index.d.ts +1 -0
- package/dist/cli/commands/index.js +1 -0
- package/dist/cli/commands/status.js +20 -0
- package/dist/cli/index.js +2 -1
- package/dist/core/config.d.ts +14 -2
- package/dist/core/config.js +6 -1
- package/dist/core/interfaces.d.ts +8 -0
- package/dist/eval/tuner.d.ts +219 -0
- package/dist/eval/tuner.js +293 -0
- package/dist/indexer/keyword-rehydration.d.ts +48 -0
- package/dist/indexer/keyword-rehydration.js +113 -0
- package/dist/indexer/pipeline.js +27 -0
- package/dist/plugin.js +90 -3
- package/dist/quirks/quirk-store.d.ts +14 -1
- package/dist/quirks/quirk-store.js +27 -0
- package/dist/quirks/tool-error-create.d.ts +79 -7
- package/dist/quirks/tool-error-create.js +135 -16
- package/dist/quirks/types.d.ts +8 -0
- package/dist/quirks/types.js +8 -1
- package/dist/retriever/keyword-index.d.ts +10 -0
- package/dist/retriever/keyword-index.js +12 -0
- package/package.json +1 -1
package/ReadMe.md
CHANGED
|
@@ -47,12 +47,13 @@ workspace, and `opencode-rag update` upgrades to a newer release.
|
|
|
47
47
|
| **Configurable parsers** | Remap file extensions to any parser via `chunking.parsers` (e.g. `.c` → `cpp`, or index a new `.cu` as C++) — applied to chunking, `get_file_skeleton`, and read labels ([docs](doc/chunking.md#parser-overrides-per-extension-chunkingparsers)) |
|
|
48
48
|
| **Document support** | Markdown, LaTeX, PDF, DOCX, DOC, Excel |
|
|
49
49
|
| **Image indexing** | Describe images via vision LLM and store descriptions as searchable vector chunks |
|
|
50
|
-
| **Hybrid search** | Vector similarity + TF×IDF keyword fusion |
|
|
50
|
+
| **Hybrid search** | Vector similarity + TF×IDF keyword fusion — the keyword index is repaired from the store whenever it drifts behind it, so the BM25 half never covers only part of the corpus |
|
|
51
|
+
| **Retrieval calibration** | `opencode-rag eval:tune` sweeps the fusion weights against the labelled golden set and reports (or writes) the best configuration for your embedder and corpus |
|
|
51
52
|
| **Decision model** | Local [tev1](https://ollama.com/library/tev1) classification — `make_decision` labels, routes, and scores short text with probabilities (40–150 ms warm); optional pre-ask routing so the agent asks fewer questions |
|
|
52
53
|
| **OpenCode plugin** | Auto-inject context, read-tool override, TUI settings, Ctrl+Enter to add RAG context, MCP registration on `init` |
|
|
53
54
|
| **Incremental indexing** | File-hash manifest, background watcher, auto-rebuild on corruption |
|
|
54
55
|
| **Privacy-first** | All processing stays local (when using Ollama) |
|
|
55
|
-
| **CLI Tools** | `init`, `index`, `query`, `status`, `list`, `show`, `dump`, `clear`, `describe-image`, `decide`, `ui`, `mcp`, `setup`, `quirk`, `eval:sessions`, `eval:analyze`, `eval:compare`, `eval:gate`, `eval:decide` |
|
|
56
|
+
| **CLI Tools** | `init`, `index`, `query`, `status`, `list`, `show`, `dump`, `clear`, `describe-image`, `decide`, `ui`, `mcp`, `setup`, `quirk`, `eval:sessions`, `eval:analyze`, `eval:compare`, `eval:gate`, `eval:decide`, `eval:tune` |
|
|
56
57
|
| **Proxy-aware** | Corporate proxy support with raw-socket localhost bypass |
|
|
57
58
|
| **OpenAI / Anthropic / Cohere** | Use alternate embedding providers with API key auto-resolution |
|
|
58
59
|
| **Evaluation** | Session-level token tracking, RAG-on vs RAG-off comparison, tiktoken BPE counting |
|
|
@@ -243,7 +244,9 @@ opencode-rag quirk test "npm needs --legacy-peer-deps"
|
|
|
243
244
|
|
|
244
245
|
When `memory.autoInject` is `true`, the plugin checks for relevant quirks on every user message using the combined agent-response + user-query as the search query. Quirks are only injected when their raw vector similarity to that query clears `autoInjectMinScore` (default 0.75) - the same threshold for the user-message and system-prompt injections; `recallMinScore` (default 0.72) is the bar for manual `recall_quirks` calls. A latency budget (`autoInjectLatencyBudgetMs`, default 2000ms) prevents slow embedders from blocking message processing. To avoid polluting the context window, each quirk is injected **at most once per session** — once recalled, it is filtered out from all subsequent auto-injections. Every `add_quirk` and every content-changing `update_quirk` is vetted by an immutable trust monitor that rejects destructive patterns (e.g. `rm -rf`, `force push`, `bypass security`). Outdated or fixed quirks should be corrected with `update_quirk` / `delete_quirk` rather than left to contradict newer memory. See [Plugin documentation](doc/plugin.md#9-quirk-memory-experiential-memory) and [CLI Reference: `quirk`](doc/cli.md#quirk).
|
|
245
246
|
|
|
246
|
-
**Failed tool calls become memory.** When a tool call fails, the plugin looks for a stored quirk that covers the failure and injects it into the next request (`memory.injectOnToolError`, default on). If **no** quirk covers it, it writes one — `memory.autoCreateOnToolError` (default on) — using a deterministic template built from the tool name and the error text, so no LLM is involved. The new quirk is tagged `tool-error` and recorded for *future* sessions; it is deliberately excluded from the generic auto-injection above, so a raw error dump never leaks into an unrelated turn. Creation is bounded by a per-session budget (2) and a per-error cooldown (10 min), and
|
|
247
|
+
**Failed tool calls become memory.** When a tool call fails, the plugin looks for a stored quirk that covers the failure and injects it into the next request (`memory.injectOnToolError`, default on). If **no** quirk covers it, it writes one — `memory.autoCreateOnToolError` (default on) — using a deterministic template built from the tool name and the error text, so no LLM is involved. The new quirk is tagged `tool-error` and recorded for *future* sessions; it is deliberately excluded from the generic auto-injection above, so a raw error dump never leaks into an unrelated turn. Creation is bounded by a per-session budget (2) and a per-error cooldown (10 min), and it is suppressed when a quirk for the same tool and cause class already exists.
|
|
248
|
+
|
|
249
|
+
**Prevention, not just learning.** The gotchas are also injected into the **system prompt before the model picks a tool** (`memory.toolErrorProactiveInject`, default on), scoped to the tools the session can actually call and to durable failures only — a rate limit is never warned about in advance, because the next call will not fail the same way. The first occurrence of a failure is unavoidable; every later session is warned up front.
|
|
247
250
|
|
|
248
251
|
## MCP Server (Optional)
|
|
249
252
|
|
|
@@ -317,7 +320,7 @@ opencode-rag eval:compare <A> <B> # side-by-side comparison
|
|
|
317
320
|
|
|
318
321
|
Token counting uses tiktoken BPE (cl100k_base) for accurate code tokenization. See [Evaluation documentation](doc/evaluation.md) for details.
|
|
319
322
|
|
|
320
|
-
Retrieval quality has its own harness: `opencode-rag eval:gate` runs the labelled golden set and fails when Hit@5 / MRR / nDCG@10 drop below the documented floors, and `opencode-rag eval:decide` measures tev1's accuracy, ECE, and Brier score on a labelled calibration set.
|
|
323
|
+
Retrieval quality has its own harness: `opencode-rag eval:gate` runs the labelled golden set and fails when Hit@5 / MRR / nDCG@10 drop below the documented floors, and `opencode-rag eval:decide` measures tev1's accuracy, ECE, and Brier score on a labelled calibration set. `opencode-rag eval:tune` sweeps the retrieval fusion weights against the same golden set and reports the best configuration for your setup — the shipped `keywordWeight=0.4` is a default, not an optimum (on a code embedder it costs ~8 Hit@5 points), so measure it instead of guessing.
|
|
321
324
|
|
|
322
325
|
---
|
|
323
326
|
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview `eval:tune` command — sweeps the retrieval fusion weights
|
|
3
|
+
* against the labelled golden set and reports the best configuration.
|
|
4
|
+
*/
|
|
5
|
+
import type { Command } from "commander";
|
|
6
|
+
/**
|
|
7
|
+
* Register the `eval:tune` command on the given Commander program.
|
|
8
|
+
*
|
|
9
|
+
* Measures the current retrieval configuration against the golden set, sweeps a
|
|
10
|
+
* grid of fusion weights, and reports the winner. Exits `0` when the sweep ran
|
|
11
|
+
* (even when the current configuration already wins), `2` when it could not run.
|
|
12
|
+
*
|
|
13
|
+
* @param program - The Commander `Command` instance to register on.
|
|
14
|
+
*/
|
|
15
|
+
export declare function registerEvalTuneCommand(program: Command): void;
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { c, resolveCliContext, cleanupContext, logCliError } from "../format.js";
|
|
3
|
+
/** Parse a comma-separated list of numbers. */
|
|
4
|
+
function parseNumberList(raw, option) {
|
|
5
|
+
if (raw === undefined)
|
|
6
|
+
return undefined;
|
|
7
|
+
const parts = raw
|
|
8
|
+
.split(",")
|
|
9
|
+
.map((p) => p.trim())
|
|
10
|
+
.filter((p) => p.length > 0);
|
|
11
|
+
if (parts.length === 0) {
|
|
12
|
+
throw new Error(`${option} needs at least one number (e.g. "${option} 0,0.2,0.4")`);
|
|
13
|
+
}
|
|
14
|
+
const values = parts.map((p) => Number(p));
|
|
15
|
+
for (const part of parts) {
|
|
16
|
+
const value = Number(part);
|
|
17
|
+
if (!Number.isFinite(value)) {
|
|
18
|
+
throw new Error(`${option} contains a non-numeric value: "${part}"`);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return values;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Register the `eval:tune` command on the given Commander program.
|
|
25
|
+
*
|
|
26
|
+
* Measures the current retrieval configuration against the golden set, sweeps a
|
|
27
|
+
* grid of fusion weights, and reports the winner. Exits `0` when the sweep ran
|
|
28
|
+
* (even when the current configuration already wins), `2` when it could not run.
|
|
29
|
+
*
|
|
30
|
+
* @param program - The Commander `Command` instance to register on.
|
|
31
|
+
*/
|
|
32
|
+
export function registerEvalTuneCommand(program) {
|
|
33
|
+
program
|
|
34
|
+
.command("eval:tune")
|
|
35
|
+
.description("Sweep the retrieval fusion weights against the golden set and report the best configuration")
|
|
36
|
+
.option("-c, --config <path>", "path to config file")
|
|
37
|
+
.option("--labels <path>", "golden-set label file (default: src/eval/rerank-labels.json)")
|
|
38
|
+
.option("--category <name>", "only run labels of this category")
|
|
39
|
+
.option("--limit <n>", "only run the first N labels")
|
|
40
|
+
.option("--metric <name>", "metric the winner is picked by (ndcg10 | hit5 | hit1 | mrr, default ndcg10)")
|
|
41
|
+
.option("--keyword-weights <list>", "comma-separated keyword weights to try (default 0,0.1,...,0.6)")
|
|
42
|
+
.option("--symbol-weights <list>", "comma-separated symbol keyword weights to try (default: same as keyword weights)")
|
|
43
|
+
.option("--min-scores <list>", "comma-separated minScore values to try (default: the configured one)")
|
|
44
|
+
.option("--topks <list>", "comma-separated topK values to try (default: the configured one)")
|
|
45
|
+
.option("--write", "write the winning values into the config file")
|
|
46
|
+
.option("--json", "print a machine-readable result as the last stdout line")
|
|
47
|
+
.action(async (options) => {
|
|
48
|
+
const cwd = process.cwd();
|
|
49
|
+
const logFilePath = path.resolve(cwd, ".opencode", "opencode-rag.log");
|
|
50
|
+
let ctx;
|
|
51
|
+
let exitCode = 0;
|
|
52
|
+
try {
|
|
53
|
+
const { loadGoldenLabels } = await import("../../eval/golden-set.js");
|
|
54
|
+
const { applyTuningToConfig, buildCandidateGrid, comparePaired, measureSignalQuality, runTuner, selectBestTuning, tuningDelta, } = await import("../../eval/tuner.js");
|
|
55
|
+
const { updateConfigValue } = await import("../../core/config.js");
|
|
56
|
+
const { retrieve } = await import("../../retriever/retriever.js");
|
|
57
|
+
const labelsPath = options.labels
|
|
58
|
+
? path.resolve(cwd, options.labels)
|
|
59
|
+
: path.join(cwd, "src", "eval", "rerank-labels.json");
|
|
60
|
+
let labels = loadGoldenLabels(labelsPath);
|
|
61
|
+
if (options.category)
|
|
62
|
+
labels = labels.filter((label) => label.category === options.category);
|
|
63
|
+
if (options.limit !== undefined) {
|
|
64
|
+
const limit = Number(options.limit);
|
|
65
|
+
if (!Number.isInteger(limit) || limit <= 0) {
|
|
66
|
+
throw new Error(`--limit must be a positive integer (got "${options.limit}")`);
|
|
67
|
+
}
|
|
68
|
+
labels = labels.slice(0, limit);
|
|
69
|
+
}
|
|
70
|
+
if (labels.length === 0) {
|
|
71
|
+
throw new Error("No labels selected (check --category/--limit).");
|
|
72
|
+
}
|
|
73
|
+
const metric = (options.metric ?? "ndcg10");
|
|
74
|
+
if (!["ndcg10", "hit5", "hit1", "mrr"].includes(metric)) {
|
|
75
|
+
throw new Error(`--metric must be one of ndcg10, hit5, hit1, mrr (got "${options.metric}")`);
|
|
76
|
+
}
|
|
77
|
+
ctx = await resolveCliContext(options, logFilePath);
|
|
78
|
+
const indexedCount = await ctx.store.count();
|
|
79
|
+
if (indexedCount === 0) {
|
|
80
|
+
console.error(c.error(`\nTuning cannot run: the index is empty. Run 'opencode-rag index' first.\n`));
|
|
81
|
+
await cleanupContext(ctx);
|
|
82
|
+
process.exit(2);
|
|
83
|
+
}
|
|
84
|
+
const hybrid = ctx.config.retrieval.hybridSearch;
|
|
85
|
+
const base = {
|
|
86
|
+
keywordWeight: hybrid?.keywordWeight ?? 0.4,
|
|
87
|
+
symbolKeywordWeight: hybrid?.symbolKeywordWeight,
|
|
88
|
+
minScore: ctx.config.retrieval.minScore,
|
|
89
|
+
topK: ctx.config.retrieval.topK,
|
|
90
|
+
};
|
|
91
|
+
const candidates = buildCandidateGrid(base, {
|
|
92
|
+
keywordWeights: parseNumberList(options.keywordWeights, "--keyword-weights"),
|
|
93
|
+
symbolKeywordWeights: parseNumberList(options.symbolWeights, "--symbol-weights"),
|
|
94
|
+
minScores: parseNumberList(options.minScores, "--min-scores"),
|
|
95
|
+
topKs: parseNumberList(options.topks, "--topks"),
|
|
96
|
+
});
|
|
97
|
+
console.log(`\n${c.heading("Retrieval Tuning")}\n`);
|
|
98
|
+
console.log(` Indexed chunks: ${c.num(indexedCount)}`);
|
|
99
|
+
console.log(` Embedder: ${c.value(`${ctx.config.embedding.provider}/${ctx.config.embedding.model}`)} (dim ${c.num(ctx.dimension)})`);
|
|
100
|
+
console.log(` Labels: ${c.num(labels.length)} from ${c.file(path.relative(cwd, labelsPath).replace(/\\/g, "/"))}`);
|
|
101
|
+
console.log(` Current config: keywordWeight=${c.num(base.keywordWeight)} symbolKeywordWeight=${c.num(base.symbolKeywordWeight ?? "unset")} minScore=${c.num(base.minScore)} topK=${c.num(base.topK)}`);
|
|
102
|
+
console.log(` Candidates: ${c.num(candidates.length)} · winner picked by ${c.value(metric)}\n`);
|
|
103
|
+
const results = await runTuner({
|
|
104
|
+
labels,
|
|
105
|
+
root: cwd,
|
|
106
|
+
candidates,
|
|
107
|
+
retrieve: (label, candidate) => retrieve(label.query, ctx.embedder, ctx.store, {
|
|
108
|
+
topK: candidate.topK,
|
|
109
|
+
minScore: candidate.minScore,
|
|
110
|
+
keywordIndex: ctx.keywordIndex,
|
|
111
|
+
keywordWeight: candidate.keywordWeight,
|
|
112
|
+
symbolKeywordWeight: candidate.symbolKeywordWeight,
|
|
113
|
+
docKeywordDemotion: hybrid?.docKeywordDemotion,
|
|
114
|
+
testKeywordDemotion: hybrid?.testKeywordDemotion,
|
|
115
|
+
hybridEnabled: hybrid?.enabled,
|
|
116
|
+
queryPrefix: ctx.config.embedding.queryPrefix,
|
|
117
|
+
}),
|
|
118
|
+
log: (line) => console.log(line),
|
|
119
|
+
});
|
|
120
|
+
const best = selectBestTuning(results, metric);
|
|
121
|
+
if (!best) {
|
|
122
|
+
throw new Error("The sweep produced no results.");
|
|
123
|
+
}
|
|
124
|
+
const current = results.find((r) => r.keywordWeight === base.keywordWeight &&
|
|
125
|
+
(r.symbolKeywordWeight ?? "unset") === (base.symbolKeywordWeight ?? "unset") &&
|
|
126
|
+
r.minScore === base.minScore &&
|
|
127
|
+
r.topK === base.topK);
|
|
128
|
+
const delta = tuningDelta(best, current);
|
|
129
|
+
console.log(`\n ${c.heading("Best configuration")}`);
|
|
130
|
+
console.log(` keywordWeight=${c.num(best.keywordWeight)} symbolKeywordWeight=${c.num(best.symbolKeywordWeight ?? "unset")} minScore=${c.num(best.minScore)} topK=${c.num(best.topK)}`);
|
|
131
|
+
console.log(` Hit@1 ${(best.summary.hit1 * 100).toFixed(1)}% · Hit@3 ${(best.summary.hit3 * 100).toFixed(1)}% · ` +
|
|
132
|
+
`Hit@5 ${(best.summary.hit5 * 100).toFixed(1)}% · Hit@10 ${(best.summary.hit10 * 100).toFixed(1)}% · ` +
|
|
133
|
+
`MRR ${best.summary.mrr.toFixed(3)} · nDCG@10 ${best.summary.ndcg10.toFixed(3)}`);
|
|
134
|
+
if (current) {
|
|
135
|
+
const sign = (v) => (v >= 0 ? "+" : "");
|
|
136
|
+
console.log(` ${c.dim("vs. current config:")} Hit@5 ${sign(delta.hit5)}${(delta.hit5 * 100).toFixed(1)} pts · ` +
|
|
137
|
+
`MRR ${sign(delta.mrr)}${delta.mrr.toFixed(3)} · nDCG@10 ${sign(delta.ndcg10)}${delta.ndcg10.toFixed(3)}`);
|
|
138
|
+
}
|
|
139
|
+
// The point estimate alone is not evidence on a small label set — show
|
|
140
|
+
// the confidence interval and refuse to recommend a change the data
|
|
141
|
+
// does not support.
|
|
142
|
+
const paired = comparePaired(best, current, metric);
|
|
143
|
+
if (paired) {
|
|
144
|
+
const sign = (v) => (v >= 0 ? "+" : "");
|
|
145
|
+
console.log(` ${c.dim("paired bootstrap:")} Δ${paired.metric} ${sign(paired.meanDelta)}${paired.meanDelta.toFixed(3)} ` +
|
|
146
|
+
`(95% CI ${paired.ciLow.toFixed(3)} … ${paired.ciHigh.toFixed(3)}) · ` +
|
|
147
|
+
`${paired.improved} better / ${paired.regressed} worse / ${paired.unchanged} equal · ` +
|
|
148
|
+
`sign test p=${paired.signTestP.toFixed(2)}`);
|
|
149
|
+
if (paired.signTestP > 0.05) {
|
|
150
|
+
console.log(` ${c.warn("Weak evidence:")} the sign test over the discordant labels is not significant — ` +
|
|
151
|
+
"more labels are needed before changing the shipped default.");
|
|
152
|
+
}
|
|
153
|
+
if (paired.withinNoise) {
|
|
154
|
+
console.log(` ${c.warn("Within noise:")} the 95% CI includes 0 for ${labels.length} label(s) — ` +
|
|
155
|
+
"treat the winner as a hint, not a measurement. Add labels or check the signal split below.");
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
// Per-signal split: a fused number is only interpretable next to the
|
|
159
|
+
// quality of the two halves it mixes.
|
|
160
|
+
const signals = await measureSignalQuality({
|
|
161
|
+
labels,
|
|
162
|
+
root: cwd,
|
|
163
|
+
retrieveKeyword: async (label) => ctx.keywordIndex.search(label.query, (base.topK ?? 20) * 3),
|
|
164
|
+
retrieveVector: async (label) => retrieve(label.query, ctx.embedder, ctx.store, {
|
|
165
|
+
topK: base.topK,
|
|
166
|
+
minScore: 0,
|
|
167
|
+
keywordIndex: undefined,
|
|
168
|
+
keywordWeight: 0,
|
|
169
|
+
queryPrefix: ctx.config.embedding.queryPrefix,
|
|
170
|
+
}),
|
|
171
|
+
retrieveFused: async (label) => retrieve(label.query, ctx.embedder, ctx.store, {
|
|
172
|
+
topK: base.topK,
|
|
173
|
+
minScore: base.minScore,
|
|
174
|
+
keywordIndex: ctx.keywordIndex,
|
|
175
|
+
keywordWeight: base.keywordWeight,
|
|
176
|
+
symbolKeywordWeight: base.symbolKeywordWeight,
|
|
177
|
+
docKeywordDemotion: hybrid?.docKeywordDemotion,
|
|
178
|
+
testKeywordDemotion: hybrid?.testKeywordDemotion,
|
|
179
|
+
hybridEnabled: hybrid?.enabled,
|
|
180
|
+
queryPrefix: ctx.config.embedding.queryPrefix,
|
|
181
|
+
}),
|
|
182
|
+
});
|
|
183
|
+
console.log(`\n ${c.heading("Signal split")} ${c.dim("(file level, current config, minScore=0 for the single signals)")}`);
|
|
184
|
+
const signalRow = (name, q) => ` ${name.padEnd(14)} Hit@1 ${(q.hit1 * 100).toFixed(1)}% · Hit@5 ${(q.hit5 * 100).toFixed(1)}% · ` +
|
|
185
|
+
`Hit@10 ${(q.hit10 * 100).toFixed(1)}% · ${q.misses} miss`;
|
|
186
|
+
console.log(signalRow("BM25 only", signals.keyword));
|
|
187
|
+
console.log(signalRow("vector only", signals.vector));
|
|
188
|
+
console.log(signalRow(`fused kw=${base.keywordWeight}`, signals.fused));
|
|
189
|
+
let written = false;
|
|
190
|
+
if (options.write) {
|
|
191
|
+
if (!ctx.configPath) {
|
|
192
|
+
throw new Error("No config file to write to — pass --config <path> or run this in a workspace with opencode-rag.json.");
|
|
193
|
+
}
|
|
194
|
+
const tuned = applyTuningToConfig(ctx.config, best);
|
|
195
|
+
const hybridWeights = tuned.retrieval.hybridSearch ?? { enabled: true, keywordWeight: best.keywordWeight };
|
|
196
|
+
const updates = [
|
|
197
|
+
[["retrieval", "hybridSearch", "keywordWeight"], hybridWeights.keywordWeight],
|
|
198
|
+
[["retrieval", "hybridSearch", "symbolKeywordWeight"], hybridWeights.symbolKeywordWeight],
|
|
199
|
+
[["retrieval", "minScore"], tuned.retrieval.minScore],
|
|
200
|
+
[["retrieval", "topK"], tuned.retrieval.topK],
|
|
201
|
+
];
|
|
202
|
+
for (const [keyPath, value] of updates) {
|
|
203
|
+
if (value === undefined)
|
|
204
|
+
continue;
|
|
205
|
+
if (!updateConfigValue(ctx.configPath, keyPath, value)) {
|
|
206
|
+
throw new Error(`Could not write ${keyPath.join(".")} to ${ctx.configPath}`);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
written = true;
|
|
210
|
+
console.log(` ${c.success(`Wrote the winning configuration to ${path.relative(cwd, ctx.configPath)}`)}`);
|
|
211
|
+
}
|
|
212
|
+
else {
|
|
213
|
+
console.log(` ${c.dim("Re-run with --write to persist these values.")}`);
|
|
214
|
+
}
|
|
215
|
+
if (options.json) {
|
|
216
|
+
console.log(JSON.stringify({
|
|
217
|
+
schema: 1,
|
|
218
|
+
status: "ok",
|
|
219
|
+
labels: labels.length,
|
|
220
|
+
labelsPath: path.relative(cwd, labelsPath).replace(/\\/g, "/"),
|
|
221
|
+
metric,
|
|
222
|
+
candidates: results.map((r) => ({
|
|
223
|
+
keywordWeight: r.keywordWeight,
|
|
224
|
+
symbolKeywordWeight: r.symbolKeywordWeight,
|
|
225
|
+
minScore: r.minScore,
|
|
226
|
+
topK: r.topK,
|
|
227
|
+
hit1: r.summary.hit1,
|
|
228
|
+
hit5: r.summary.hit5,
|
|
229
|
+
mrr: r.summary.mrr,
|
|
230
|
+
ndcg10: r.summary.ndcg10,
|
|
231
|
+
durationMs: r.durationMs,
|
|
232
|
+
})),
|
|
233
|
+
best: {
|
|
234
|
+
keywordWeight: best.keywordWeight,
|
|
235
|
+
symbolKeywordWeight: best.symbolKeywordWeight,
|
|
236
|
+
minScore: best.minScore,
|
|
237
|
+
topK: best.topK,
|
|
238
|
+
metrics: {
|
|
239
|
+
hit1: best.summary.hit1,
|
|
240
|
+
hit5: best.summary.hit5,
|
|
241
|
+
mrr: best.summary.mrr,
|
|
242
|
+
ndcg10: best.summary.ndcg10,
|
|
243
|
+
},
|
|
244
|
+
},
|
|
245
|
+
delta,
|
|
246
|
+
paired: paired
|
|
247
|
+
? {
|
|
248
|
+
metric: paired.metric,
|
|
249
|
+
meanDelta: paired.meanDelta,
|
|
250
|
+
ciLow: paired.ciLow,
|
|
251
|
+
ciHigh: paired.ciHigh,
|
|
252
|
+
pBetter: paired.pBetter,
|
|
253
|
+
improved: paired.improved,
|
|
254
|
+
regressed: paired.regressed,
|
|
255
|
+
unchanged: paired.unchanged,
|
|
256
|
+
withinNoise: paired.withinNoise,
|
|
257
|
+
signTestP: paired.signTestP,
|
|
258
|
+
}
|
|
259
|
+
: null,
|
|
260
|
+
signals: {
|
|
261
|
+
keyword: signals.keyword,
|
|
262
|
+
vector: signals.vector,
|
|
263
|
+
fused: signals.fused,
|
|
264
|
+
},
|
|
265
|
+
written,
|
|
266
|
+
}));
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
catch (err) {
|
|
270
|
+
const message = err.message || String(err);
|
|
271
|
+
console.error(c.error(`\nTuning could not run: ${message}\n`));
|
|
272
|
+
if (message.toLowerCase().includes("fetch") || message.toLowerCase().includes("econnrefused")) {
|
|
273
|
+
console.error(c.warn("Hint: Is your embedding provider running?\n"));
|
|
274
|
+
}
|
|
275
|
+
logCliError(logFilePath, "eval:tune", `Tuning failed: ${message}`, err);
|
|
276
|
+
exitCode = 2;
|
|
277
|
+
}
|
|
278
|
+
finally {
|
|
279
|
+
if (ctx)
|
|
280
|
+
await cleanupContext(ctx);
|
|
281
|
+
}
|
|
282
|
+
process.exit(exitCode);
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
//# sourceMappingURL=eval-tuner.js.map
|
|
@@ -14,6 +14,7 @@ export { registerMcpCommand } from "./mcp.js";
|
|
|
14
14
|
export { registerSetupCommand } from "./setup.js";
|
|
15
15
|
export { registerUpdateCommand } from "./update.js";
|
|
16
16
|
export { registerEvalSessionsCommand, registerEvalAnalyzeCommand, registerEvalCompareCommand, registerEvalGateCommand, registerEvalDecideCommand } from "./eval.js";
|
|
17
|
+
export { registerEvalTuneCommand } from "./eval-tuner.js";
|
|
17
18
|
export { registerDescribeImageCommand } from "./describe-image.js";
|
|
18
19
|
export { registerDecideCommand } from "./decide.js";
|
|
19
20
|
export { registerQuirkCommand } from "./quirk.js";
|
|
@@ -14,6 +14,7 @@ export { registerMcpCommand } from "./mcp.js";
|
|
|
14
14
|
export { registerSetupCommand } from "./setup.js";
|
|
15
15
|
export { registerUpdateCommand } from "./update.js";
|
|
16
16
|
export { registerEvalSessionsCommand, registerEvalAnalyzeCommand, registerEvalCompareCommand, registerEvalGateCommand, registerEvalDecideCommand } from "./eval.js";
|
|
17
|
+
export { registerEvalTuneCommand } from "./eval-tuner.js";
|
|
17
18
|
export { registerDescribeImageCommand } from "./describe-image.js";
|
|
18
19
|
export { registerDecideCommand } from "./decide.js";
|
|
19
20
|
export { registerQuirkCommand } from "./quirk.js";
|
|
@@ -9,6 +9,7 @@ import os from "node:os";
|
|
|
9
9
|
import fs from "node:fs";
|
|
10
10
|
import { c, resolveCliContext, logCliError, logCliInfo, formatTimestamp, formatRelativeAge } from "../format.js";
|
|
11
11
|
import { getIndexStatusSummary } from "../../indexer.js";
|
|
12
|
+
import { findKeywordIndexGaps } from "../../indexer/keyword-rehydration.js";
|
|
12
13
|
import { deriveWatcherHealth } from "../../watcher.js";
|
|
13
14
|
import { getPackageMetadata } from "../helpers.js";
|
|
14
15
|
import { checkForUpdate } from "../../core/version-check.js";
|
|
@@ -142,6 +143,25 @@ export function registerStatusCommand(program) {
|
|
|
142
143
|
? keywordIndex?.count() ?? 0
|
|
143
144
|
: 0;
|
|
144
145
|
logCliInfo(logFilePath, "status", `${c.label("Keyword index:")} ${config.retrieval.hybridSearch?.enabled ? c.enabled("enabled") : c.disabled("disabled")} (${c.num(kiCount)} chunks)`);
|
|
146
|
+
// The keyword index and the vector store are two independent stores of
|
|
147
|
+
// the same corpus. A file the store holds but the keyword index does
|
|
148
|
+
// not cover is unreachable through keyword search — report the gap
|
|
149
|
+
// instead of printing two unrelated chunk counts.
|
|
150
|
+
if (config.retrieval.hybridSearch?.enabled && keywordIndex) {
|
|
151
|
+
const gaps = await findKeywordIndexGaps(store, keywordIndex);
|
|
152
|
+
if (gaps.length > 0) {
|
|
153
|
+
logCliInfo(logFilePath, "status", `${c.label("Keyword coverage:")} ${c.warn(`${gaps.length} file(s) in the store are missing from the keyword index`)} ${c.dim("(run 'opencode-rag index' to rehydrate — no re-embedding needed)")}`);
|
|
154
|
+
for (const gap of gaps.slice(0, 5)) {
|
|
155
|
+
logCliInfo(logFilePath, "status", ` ${c.file(path.relative(cwd, gap) || gap)}`);
|
|
156
|
+
}
|
|
157
|
+
if (gaps.length > 5) {
|
|
158
|
+
logCliInfo(logFilePath, "status", ` ${c.dim(`… and ${gaps.length - 5} more`)}`);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
else {
|
|
162
|
+
logCliInfo(logFilePath, "status", `${c.label("Keyword coverage:")} ${c.success("complete")} ${c.dim("(every indexed file is searchable by keyword)")}`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
145
165
|
if (summary.rebuildRequired) {
|
|
146
166
|
logCliInfo(logFilePath, "status", `${c.label("Rebuild required:")} ${c.warn("yes")} (manifest missing/corrupt)`);
|
|
147
167
|
}
|
package/dist/cli/index.js
CHANGED
|
@@ -15,7 +15,7 @@ import { realpathSync } from "node:fs";
|
|
|
15
15
|
import { basename, dirname } from "node:path";
|
|
16
16
|
import { fileURLToPath } from "node:url";
|
|
17
17
|
import { getPackageMetadata } from "./helpers.js";
|
|
18
|
-
import { registerIndexCommand, registerQueryCommand, registerClearCommand, registerStatusCommand, registerListCommand, registerShowCommand, registerDumpCommand, registerInitCommand, registerUiCommand, registerMcpCommand, registerSetupCommand, registerUpdateCommand, registerEvalSessionsCommand, registerEvalAnalyzeCommand, registerEvalCompareCommand, registerEvalGateCommand, registerEvalDecideCommand, registerDescribeImageCommand, registerDecideCommand, registerQuirkCommand, } from "./commands/index.js";
|
|
18
|
+
import { registerIndexCommand, registerQueryCommand, registerClearCommand, registerStatusCommand, registerListCommand, registerShowCommand, registerDumpCommand, registerInitCommand, registerUiCommand, registerMcpCommand, registerSetupCommand, registerUpdateCommand, registerEvalSessionsCommand, registerEvalAnalyzeCommand, registerEvalCompareCommand, registerEvalGateCommand, registerEvalDecideCommand, registerEvalTuneCommand, registerDescribeImageCommand, registerDecideCommand, registerQuirkCommand, } from "./commands/index.js";
|
|
19
19
|
/**
|
|
20
20
|
* The top-level Commander program instance that defines the `opencode-rag` CLI.
|
|
21
21
|
*
|
|
@@ -56,6 +56,7 @@ registerEvalAnalyzeCommand(program);
|
|
|
56
56
|
registerEvalCompareCommand(program);
|
|
57
57
|
registerEvalGateCommand(program);
|
|
58
58
|
registerEvalDecideCommand(program);
|
|
59
|
+
registerEvalTuneCommand(program);
|
|
59
60
|
registerInitCommand(program);
|
|
60
61
|
registerQuirkCommand(program);
|
|
61
62
|
// ── Auto-run detection ──────────────────────────────────────────
|
package/dist/core/config.d.ts
CHANGED
|
@@ -294,8 +294,11 @@ export interface MemoryConfig {
|
|
|
294
294
|
/**
|
|
295
295
|
* Minimum raw similarity (0-1) for an existing quirk to count as "an
|
|
296
296
|
* equivalent quirk already exists" and therefore suppress creation
|
|
297
|
-
* (default 0.
|
|
298
|
-
*
|
|
297
|
+
* (default 0.7). Deliberately EQUAL to `toolErrorMinScore`: measured on a
|
|
298
|
+
* real workspace, an error query scores 0.55-0.63 against arbitrary stored
|
|
299
|
+
* quirks, so a lower floor sits inside the noise band and matches unrelated
|
|
300
|
+
* quirks — which suppressed creation on every real failure. The exact
|
|
301
|
+
* duplicate check (same tool + same cause class) is done by metadata.
|
|
299
302
|
*/
|
|
300
303
|
toolErrorCreateMinScore: number;
|
|
301
304
|
/** Latency budget (ms) for the tool-error creation path (embed + store write); default 2000. */
|
|
@@ -306,6 +309,15 @@ export interface MemoryConfig {
|
|
|
306
309
|
* created quirk carries no new information for that turn.
|
|
307
310
|
*/
|
|
308
311
|
toolErrorInjectCreated: boolean;
|
|
312
|
+
/**
|
|
313
|
+
* Surface known tool gotchas in the system prompt BEFORE the model picks a
|
|
314
|
+
* tool (default true). A failure that already happened in an earlier
|
|
315
|
+
* session is injected as a "known tool gotcha", so the model does not repeat
|
|
316
|
+
* it. Only gotchas whose tool is in the session's tool catalog are injected.
|
|
317
|
+
*/
|
|
318
|
+
toolErrorProactiveInject: boolean;
|
|
319
|
+
/** Max tool gotchas injected into the system prompt per request (default 5). */
|
|
320
|
+
toolErrorProactiveMaxItems: number;
|
|
309
321
|
/** Decay settings for aging quirks. */
|
|
310
322
|
decay: {
|
|
311
323
|
/** Whether confidence decays over time. */
|
package/dist/core/config.js
CHANGED
|
@@ -325,9 +325,11 @@ export const DEFAULT_CONFIG = {
|
|
|
325
325
|
autoCreateOnToolError: true,
|
|
326
326
|
toolErrorCreateMaxPerSession: 2,
|
|
327
327
|
toolErrorCreateCooldownMs: 600_000,
|
|
328
|
-
toolErrorCreateMinScore: 0.
|
|
328
|
+
toolErrorCreateMinScore: 0.7,
|
|
329
329
|
toolErrorCreateLatencyBudgetMs: 2000,
|
|
330
330
|
toolErrorInjectCreated: false,
|
|
331
|
+
toolErrorProactiveInject: true,
|
|
332
|
+
toolErrorProactiveMaxItems: 5,
|
|
331
333
|
decay: {
|
|
332
334
|
enabled: false,
|
|
333
335
|
halfLifeDays: 30,
|
|
@@ -487,6 +489,9 @@ export function validateConfig(config) {
|
|
|
487
489
|
if (config.memory?.toolErrorCreateLatencyBudgetMs != null && config.memory.toolErrorCreateLatencyBudgetMs < 0) {
|
|
488
490
|
warnings.push("memory.toolErrorCreateLatencyBudgetMs must be >= 0");
|
|
489
491
|
}
|
|
492
|
+
if (config.memory?.toolErrorProactiveMaxItems != null && config.memory.toolErrorProactiveMaxItems < 0) {
|
|
493
|
+
warnings.push("memory.toolErrorProactiveMaxItems must be >= 0");
|
|
494
|
+
}
|
|
490
495
|
if (config.memory?.decay && config.memory.decay.halfLifeDays <= 0) {
|
|
491
496
|
warnings.push("memory.decay.halfLifeDays must be > 0");
|
|
492
497
|
}
|
|
@@ -249,6 +249,14 @@ export interface KeywordIndex {
|
|
|
249
249
|
count(): number;
|
|
250
250
|
/** Whether a chunk with the given id is indexed. Used by quirk reconcile. */
|
|
251
251
|
hasChunk(id: string): boolean;
|
|
252
|
+
/**
|
|
253
|
+
* Normalized file paths that currently have at least one indexed chunk.
|
|
254
|
+
*
|
|
255
|
+
* Used to detect a keyword index that drifted behind the vector store: an
|
|
256
|
+
* incremental index pass only re-adds the files it reprocessed, so a
|
|
257
|
+
* once-partial index stays partial unless it is compared against the store.
|
|
258
|
+
*/
|
|
259
|
+
getIndexedFilePaths(): string[];
|
|
252
260
|
/** Persist the index to disk as JSON. */
|
|
253
261
|
save(filePath?: string): Promise<void>;
|
|
254
262
|
}
|