opencode-rag-plugin 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +4 -3
- package/dist/cli/commands/eval-tuner.d.ts +15 -0
- package/dist/cli/commands/eval-tuner.js +216 -0
- package/dist/cli/commands/index.d.ts +1 -0
- package/dist/cli/commands/index.js +1 -0
- package/dist/cli/commands/status.js +20 -0
- package/dist/cli/index.js +2 -1
- package/dist/core/interfaces.d.ts +8 -0
- package/dist/eval/tuner.d.ts +118 -0
- package/dist/eval/tuner.js +158 -0
- package/dist/indexer/keyword-rehydration.d.ts +48 -0
- package/dist/indexer/keyword-rehydration.js +113 -0
- package/dist/indexer/pipeline.js +27 -0
- package/dist/retriever/keyword-index.d.ts +10 -0
- package/dist/retriever/keyword-index.js +12 -0
- package/package.json +1 -1
package/ReadMe.md
CHANGED
|
@@ -47,12 +47,13 @@ workspace, and `opencode-rag update` upgrades to a newer release.
|
|
|
47
47
|
| **Configurable parsers** | Remap file extensions to any parser via `chunking.parsers` (e.g. `.c` → `cpp`, or index a new `.cu` as C++) — applied to chunking, `get_file_skeleton`, and read labels ([docs](doc/chunking.md#parser-overrides-per-extension-chunkingparsers)) |
|
|
48
48
|
| **Document support** | Markdown, LaTeX, PDF, DOCX, DOC, Excel |
|
|
49
49
|
| **Image indexing** | Describe images via vision LLM and store descriptions as searchable vector chunks |
|
|
50
|
-
| **Hybrid search** | Vector similarity + TF×IDF keyword fusion |
|
|
50
|
+
| **Hybrid search** | Vector similarity + TF×IDF keyword fusion — the keyword index is repaired from the store whenever it drifts behind it, so the BM25 half never covers only part of the corpus |
|
|
51
|
+
| **Retrieval calibration** | `opencode-rag eval:tune` sweeps the fusion weights against the labelled golden set and reports (or writes) the best configuration for your embedder and corpus |
|
|
51
52
|
| **Decision model** | Local [tev1](https://ollama.com/library/tev1) classification — `make_decision` labels, routes, and scores short text with probabilities (40–150 ms warm); optional pre-ask routing so the agent asks fewer questions |
|
|
52
53
|
| **OpenCode plugin** | Auto-inject context, read-tool override, TUI settings, Ctrl+Enter to add RAG context, MCP registration on `init` |
|
|
53
54
|
| **Incremental indexing** | File-hash manifest, background watcher, auto-rebuild on corruption |
|
|
54
55
|
| **Privacy-first** | All processing stays local (when using Ollama) |
|
|
55
|
-
| **CLI Tools** | `init`, `index`, `query`, `status`, `list`, `show`, `dump`, `clear`, `describe-image`, `decide`, `ui`, `mcp`, `setup`, `quirk`, `eval:sessions`, `eval:analyze`, `eval:compare`, `eval:gate`, `eval:decide` |
|
|
56
|
+
| **CLI Tools** | `init`, `index`, `query`, `status`, `list`, `show`, `dump`, `clear`, `describe-image`, `decide`, `ui`, `mcp`, `setup`, `quirk`, `eval:sessions`, `eval:analyze`, `eval:compare`, `eval:gate`, `eval:decide`, `eval:tune` |
|
|
56
57
|
| **Proxy-aware** | Corporate proxy support with raw-socket localhost bypass |
|
|
57
58
|
| **OpenAI / Anthropic / Cohere** | Use alternate embedding providers with API key auto-resolution |
|
|
58
59
|
| **Evaluation** | Session-level token tracking, RAG-on vs RAG-off comparison, tiktoken BPE counting |
|
|
@@ -317,7 +318,7 @@ opencode-rag eval:compare <A> <B> # side-by-side comparison
|
|
|
317
318
|
|
|
318
319
|
Token counting uses tiktoken BPE (cl100k_base) for accurate code tokenization. See [Evaluation documentation](doc/evaluation.md) for details.
|
|
319
320
|
|
|
320
|
-
Retrieval quality has its own harness: `opencode-rag eval:gate` runs the labelled golden set and fails when Hit@5 / MRR / nDCG@10 drop below the documented floors, and `opencode-rag eval:decide` measures tev1's accuracy, ECE, and Brier score on a labelled calibration set.
|
|
321
|
+
Retrieval quality has its own harness: `opencode-rag eval:gate` runs the labelled golden set and fails when Hit@5 / MRR / nDCG@10 drop below the documented floors, and `opencode-rag eval:decide` measures tev1's accuracy, ECE, and Brier score on a labelled calibration set. `opencode-rag eval:tune` sweeps the retrieval fusion weights against the same golden set and reports the best configuration for your setup — the shipped `keywordWeight=0.4` is a default, not an optimum (on a code embedder it costs ~8 Hit@5 points), so measure it instead of guessing.
|
|
321
322
|
|
|
322
323
|
---
|
|
323
324
|
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview `eval:tune` command — sweeps the retrieval fusion weights
|
|
3
|
+
* against the labelled golden set and reports the best configuration.
|
|
4
|
+
*/
|
|
5
|
+
import type { Command } from "commander";
|
|
6
|
+
/**
|
|
7
|
+
* Register the `eval:tune` command on the given Commander program.
|
|
8
|
+
*
|
|
9
|
+
* Measures the current retrieval configuration against the golden set, sweeps a
|
|
10
|
+
* grid of fusion weights, and reports the winner. Exits `0` when the sweep ran
|
|
11
|
+
* (even when the current configuration already wins), `2` when it could not run.
|
|
12
|
+
*
|
|
13
|
+
* @param program - The Commander `Command` instance to register on.
|
|
14
|
+
*/
|
|
15
|
+
export declare function registerEvalTuneCommand(program: Command): void;
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { c, resolveCliContext, cleanupContext, logCliError } from "../format.js";
|
|
3
|
+
/** Parse a comma-separated list of numbers. */
|
|
4
|
+
function parseNumberList(raw, option) {
|
|
5
|
+
if (raw === undefined)
|
|
6
|
+
return undefined;
|
|
7
|
+
const parts = raw
|
|
8
|
+
.split(",")
|
|
9
|
+
.map((p) => p.trim())
|
|
10
|
+
.filter((p) => p.length > 0);
|
|
11
|
+
if (parts.length === 0) {
|
|
12
|
+
throw new Error(`${option} needs at least one number (e.g. "${option} 0,0.2,0.4")`);
|
|
13
|
+
}
|
|
14
|
+
const values = parts.map((p) => Number(p));
|
|
15
|
+
for (const part of parts) {
|
|
16
|
+
const value = Number(part);
|
|
17
|
+
if (!Number.isFinite(value)) {
|
|
18
|
+
throw new Error(`${option} contains a non-numeric value: "${part}"`);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return values;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Register the `eval:tune` command on the given Commander program.
|
|
25
|
+
*
|
|
26
|
+
* Measures the current retrieval configuration against the golden set, sweeps a
|
|
27
|
+
* grid of fusion weights, and reports the winner. Exits `0` when the sweep ran
|
|
28
|
+
* (even when the current configuration already wins), `2` when it could not run.
|
|
29
|
+
*
|
|
30
|
+
* @param program - The Commander `Command` instance to register on.
|
|
31
|
+
*/
|
|
32
|
+
export function registerEvalTuneCommand(program) {
|
|
33
|
+
program
|
|
34
|
+
.command("eval:tune")
|
|
35
|
+
.description("Sweep the retrieval fusion weights against the golden set and report the best configuration")
|
|
36
|
+
.option("-c, --config <path>", "path to config file")
|
|
37
|
+
.option("--labels <path>", "golden-set label file (default: src/eval/rerank-labels.json)")
|
|
38
|
+
.option("--category <name>", "only run labels of this category")
|
|
39
|
+
.option("--limit <n>", "only run the first N labels")
|
|
40
|
+
.option("--metric <name>", "metric the winner is picked by (ndcg10 | hit5 | hit1 | mrr, default ndcg10)")
|
|
41
|
+
.option("--keyword-weights <list>", "comma-separated keyword weights to try (default 0,0.1,...,0.6)")
|
|
42
|
+
.option("--symbol-weights <list>", "comma-separated symbol keyword weights to try (default: same as keyword weights)")
|
|
43
|
+
.option("--min-scores <list>", "comma-separated minScore values to try (default: the configured one)")
|
|
44
|
+
.option("--topks <list>", "comma-separated topK values to try (default: the configured one)")
|
|
45
|
+
.option("--write", "write the winning values into the config file")
|
|
46
|
+
.option("--json", "print a machine-readable result as the last stdout line")
|
|
47
|
+
.action(async (options) => {
|
|
48
|
+
const cwd = process.cwd();
|
|
49
|
+
const logFilePath = path.resolve(cwd, ".opencode", "opencode-rag.log");
|
|
50
|
+
let ctx;
|
|
51
|
+
let exitCode = 0;
|
|
52
|
+
try {
|
|
53
|
+
const { loadGoldenLabels } = await import("../../eval/golden-set.js");
|
|
54
|
+
const { applyTuningToConfig, buildCandidateGrid, runTuner, selectBestTuning, tuningDelta, } = await import("../../eval/tuner.js");
|
|
55
|
+
const { updateConfigValue } = await import("../../core/config.js");
|
|
56
|
+
const { retrieve } = await import("../../retriever/retriever.js");
|
|
57
|
+
const labelsPath = options.labels
|
|
58
|
+
? path.resolve(cwd, options.labels)
|
|
59
|
+
: path.join(cwd, "src", "eval", "rerank-labels.json");
|
|
60
|
+
let labels = loadGoldenLabels(labelsPath);
|
|
61
|
+
if (options.category)
|
|
62
|
+
labels = labels.filter((label) => label.category === options.category);
|
|
63
|
+
if (options.limit !== undefined) {
|
|
64
|
+
const limit = Number(options.limit);
|
|
65
|
+
if (!Number.isInteger(limit) || limit <= 0) {
|
|
66
|
+
throw new Error(`--limit must be a positive integer (got "${options.limit}")`);
|
|
67
|
+
}
|
|
68
|
+
labels = labels.slice(0, limit);
|
|
69
|
+
}
|
|
70
|
+
if (labels.length === 0) {
|
|
71
|
+
throw new Error("No labels selected (check --category/--limit).");
|
|
72
|
+
}
|
|
73
|
+
const metric = (options.metric ?? "ndcg10");
|
|
74
|
+
if (!["ndcg10", "hit5", "hit1", "mrr"].includes(metric)) {
|
|
75
|
+
throw new Error(`--metric must be one of ndcg10, hit5, hit1, mrr (got "${options.metric}")`);
|
|
76
|
+
}
|
|
77
|
+
ctx = await resolveCliContext(options, logFilePath);
|
|
78
|
+
const indexedCount = await ctx.store.count();
|
|
79
|
+
if (indexedCount === 0) {
|
|
80
|
+
console.error(c.error(`\nTuning cannot run: the index is empty. Run 'opencode-rag index' first.\n`));
|
|
81
|
+
await cleanupContext(ctx);
|
|
82
|
+
process.exit(2);
|
|
83
|
+
}
|
|
84
|
+
const hybrid = ctx.config.retrieval.hybridSearch;
|
|
85
|
+
const base = {
|
|
86
|
+
keywordWeight: hybrid?.keywordWeight ?? 0.4,
|
|
87
|
+
symbolKeywordWeight: hybrid?.symbolKeywordWeight,
|
|
88
|
+
minScore: ctx.config.retrieval.minScore,
|
|
89
|
+
topK: ctx.config.retrieval.topK,
|
|
90
|
+
};
|
|
91
|
+
const candidates = buildCandidateGrid(base, {
|
|
92
|
+
keywordWeights: parseNumberList(options.keywordWeights, "--keyword-weights"),
|
|
93
|
+
symbolKeywordWeights: parseNumberList(options.symbolWeights, "--symbol-weights"),
|
|
94
|
+
minScores: parseNumberList(options.minScores, "--min-scores"),
|
|
95
|
+
topKs: parseNumberList(options.topks, "--topks"),
|
|
96
|
+
});
|
|
97
|
+
console.log(`\n${c.heading("Retrieval Tuning")}\n`);
|
|
98
|
+
console.log(` Indexed chunks: ${c.num(indexedCount)}`);
|
|
99
|
+
console.log(` Embedder: ${c.value(`${ctx.config.embedding.provider}/${ctx.config.embedding.model}`)} (dim ${c.num(ctx.dimension)})`);
|
|
100
|
+
console.log(` Labels: ${c.num(labels.length)} from ${c.file(path.relative(cwd, labelsPath).replace(/\\/g, "/"))}`);
|
|
101
|
+
console.log(` Current config: keywordWeight=${c.num(base.keywordWeight)} symbolKeywordWeight=${c.num(base.symbolKeywordWeight ?? "unset")} minScore=${c.num(base.minScore)} topK=${c.num(base.topK)}`);
|
|
102
|
+
console.log(` Candidates: ${c.num(candidates.length)} · winner picked by ${c.value(metric)}\n`);
|
|
103
|
+
const results = await runTuner({
|
|
104
|
+
labels,
|
|
105
|
+
root: cwd,
|
|
106
|
+
candidates,
|
|
107
|
+
retrieve: (label, candidate) => retrieve(label.query, ctx.embedder, ctx.store, {
|
|
108
|
+
topK: candidate.topK,
|
|
109
|
+
minScore: candidate.minScore,
|
|
110
|
+
keywordIndex: ctx.keywordIndex,
|
|
111
|
+
keywordWeight: candidate.keywordWeight,
|
|
112
|
+
symbolKeywordWeight: candidate.symbolKeywordWeight,
|
|
113
|
+
docKeywordDemotion: hybrid?.docKeywordDemotion,
|
|
114
|
+
testKeywordDemotion: hybrid?.testKeywordDemotion,
|
|
115
|
+
hybridEnabled: hybrid?.enabled,
|
|
116
|
+
queryPrefix: ctx.config.embedding.queryPrefix,
|
|
117
|
+
}),
|
|
118
|
+
log: (line) => console.log(line),
|
|
119
|
+
});
|
|
120
|
+
const best = selectBestTuning(results, metric);
|
|
121
|
+
if (!best) {
|
|
122
|
+
throw new Error("The sweep produced no results.");
|
|
123
|
+
}
|
|
124
|
+
const current = results.find((r) => r.keywordWeight === base.keywordWeight &&
|
|
125
|
+
(r.symbolKeywordWeight ?? "unset") === (base.symbolKeywordWeight ?? "unset") &&
|
|
126
|
+
r.minScore === base.minScore &&
|
|
127
|
+
r.topK === base.topK);
|
|
128
|
+
const delta = tuningDelta(best, current);
|
|
129
|
+
console.log(`\n ${c.heading("Best configuration")}`);
|
|
130
|
+
console.log(` keywordWeight=${c.num(best.keywordWeight)} symbolKeywordWeight=${c.num(best.symbolKeywordWeight ?? "unset")} minScore=${c.num(best.minScore)} topK=${c.num(best.topK)}`);
|
|
131
|
+
console.log(` Hit@1 ${(best.summary.hit1 * 100).toFixed(1)}% · Hit@3 ${(best.summary.hit3 * 100).toFixed(1)}% · ` +
|
|
132
|
+
`Hit@5 ${(best.summary.hit5 * 100).toFixed(1)}% · Hit@10 ${(best.summary.hit10 * 100).toFixed(1)}% · ` +
|
|
133
|
+
`MRR ${best.summary.mrr.toFixed(3)} · nDCG@10 ${best.summary.ndcg10.toFixed(3)}`);
|
|
134
|
+
if (current) {
|
|
135
|
+
const sign = (v) => (v >= 0 ? "+" : "");
|
|
136
|
+
console.log(` ${c.dim("vs. current config:")} Hit@5 ${sign(delta.hit5)}${(delta.hit5 * 100).toFixed(1)} pts · ` +
|
|
137
|
+
`MRR ${sign(delta.mrr)}${delta.mrr.toFixed(3)} · nDCG@10 ${sign(delta.ndcg10)}${delta.ndcg10.toFixed(3)}`);
|
|
138
|
+
}
|
|
139
|
+
let written = false;
|
|
140
|
+
if (options.write) {
|
|
141
|
+
if (!ctx.configPath) {
|
|
142
|
+
throw new Error("No config file to write to — pass --config <path> or run this in a workspace with opencode-rag.json.");
|
|
143
|
+
}
|
|
144
|
+
const tuned = applyTuningToConfig(ctx.config, best);
|
|
145
|
+
const hybridWeights = tuned.retrieval.hybridSearch ?? { enabled: true, keywordWeight: best.keywordWeight };
|
|
146
|
+
const updates = [
|
|
147
|
+
[["retrieval", "hybridSearch", "keywordWeight"], hybridWeights.keywordWeight],
|
|
148
|
+
[["retrieval", "hybridSearch", "symbolKeywordWeight"], hybridWeights.symbolKeywordWeight],
|
|
149
|
+
[["retrieval", "minScore"], tuned.retrieval.minScore],
|
|
150
|
+
[["retrieval", "topK"], tuned.retrieval.topK],
|
|
151
|
+
];
|
|
152
|
+
for (const [keyPath, value] of updates) {
|
|
153
|
+
if (value === undefined)
|
|
154
|
+
continue;
|
|
155
|
+
if (!updateConfigValue(ctx.configPath, keyPath, value)) {
|
|
156
|
+
throw new Error(`Could not write ${keyPath.join(".")} to ${ctx.configPath}`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
written = true;
|
|
160
|
+
console.log(` ${c.success(`Wrote the winning configuration to ${path.relative(cwd, ctx.configPath)}`)}`);
|
|
161
|
+
}
|
|
162
|
+
else {
|
|
163
|
+
console.log(` ${c.dim("Re-run with --write to persist these values.")}`);
|
|
164
|
+
}
|
|
165
|
+
if (options.json) {
|
|
166
|
+
console.log(JSON.stringify({
|
|
167
|
+
schema: 1,
|
|
168
|
+
status: "ok",
|
|
169
|
+
labels: labels.length,
|
|
170
|
+
labelsPath: path.relative(cwd, labelsPath).replace(/\\/g, "/"),
|
|
171
|
+
metric,
|
|
172
|
+
candidates: results.map((r) => ({
|
|
173
|
+
keywordWeight: r.keywordWeight,
|
|
174
|
+
symbolKeywordWeight: r.symbolKeywordWeight,
|
|
175
|
+
minScore: r.minScore,
|
|
176
|
+
topK: r.topK,
|
|
177
|
+
hit1: r.summary.hit1,
|
|
178
|
+
hit5: r.summary.hit5,
|
|
179
|
+
mrr: r.summary.mrr,
|
|
180
|
+
ndcg10: r.summary.ndcg10,
|
|
181
|
+
durationMs: r.durationMs,
|
|
182
|
+
})),
|
|
183
|
+
best: {
|
|
184
|
+
keywordWeight: best.keywordWeight,
|
|
185
|
+
symbolKeywordWeight: best.symbolKeywordWeight,
|
|
186
|
+
minScore: best.minScore,
|
|
187
|
+
topK: best.topK,
|
|
188
|
+
metrics: {
|
|
189
|
+
hit1: best.summary.hit1,
|
|
190
|
+
hit5: best.summary.hit5,
|
|
191
|
+
mrr: best.summary.mrr,
|
|
192
|
+
ndcg10: best.summary.ndcg10,
|
|
193
|
+
},
|
|
194
|
+
},
|
|
195
|
+
delta,
|
|
196
|
+
written,
|
|
197
|
+
}));
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
catch (err) {
|
|
201
|
+
const message = err.message || String(err);
|
|
202
|
+
console.error(c.error(`\nTuning could not run: ${message}\n`));
|
|
203
|
+
if (message.toLowerCase().includes("fetch") || message.toLowerCase().includes("econnrefused")) {
|
|
204
|
+
console.error(c.warn("Hint: Is your embedding provider running?\n"));
|
|
205
|
+
}
|
|
206
|
+
logCliError(logFilePath, "eval:tune", `Tuning failed: ${message}`, err);
|
|
207
|
+
exitCode = 2;
|
|
208
|
+
}
|
|
209
|
+
finally {
|
|
210
|
+
if (ctx)
|
|
211
|
+
await cleanupContext(ctx);
|
|
212
|
+
}
|
|
213
|
+
process.exit(exitCode);
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
//# sourceMappingURL=eval-tuner.js.map
|
|
@@ -14,6 +14,7 @@ export { registerMcpCommand } from "./mcp.js";
|
|
|
14
14
|
export { registerSetupCommand } from "./setup.js";
|
|
15
15
|
export { registerUpdateCommand } from "./update.js";
|
|
16
16
|
export { registerEvalSessionsCommand, registerEvalAnalyzeCommand, registerEvalCompareCommand, registerEvalGateCommand, registerEvalDecideCommand } from "./eval.js";
|
|
17
|
+
export { registerEvalTuneCommand } from "./eval-tuner.js";
|
|
17
18
|
export { registerDescribeImageCommand } from "./describe-image.js";
|
|
18
19
|
export { registerDecideCommand } from "./decide.js";
|
|
19
20
|
export { registerQuirkCommand } from "./quirk.js";
|
|
@@ -14,6 +14,7 @@ export { registerMcpCommand } from "./mcp.js";
|
|
|
14
14
|
export { registerSetupCommand } from "./setup.js";
|
|
15
15
|
export { registerUpdateCommand } from "./update.js";
|
|
16
16
|
export { registerEvalSessionsCommand, registerEvalAnalyzeCommand, registerEvalCompareCommand, registerEvalGateCommand, registerEvalDecideCommand } from "./eval.js";
|
|
17
|
+
export { registerEvalTuneCommand } from "./eval-tuner.js";
|
|
17
18
|
export { registerDescribeImageCommand } from "./describe-image.js";
|
|
18
19
|
export { registerDecideCommand } from "./decide.js";
|
|
19
20
|
export { registerQuirkCommand } from "./quirk.js";
|
|
@@ -9,6 +9,7 @@ import os from "node:os";
|
|
|
9
9
|
import fs from "node:fs";
|
|
10
10
|
import { c, resolveCliContext, logCliError, logCliInfo, formatTimestamp, formatRelativeAge } from "../format.js";
|
|
11
11
|
import { getIndexStatusSummary } from "../../indexer.js";
|
|
12
|
+
import { findKeywordIndexGaps } from "../../indexer/keyword-rehydration.js";
|
|
12
13
|
import { deriveWatcherHealth } from "../../watcher.js";
|
|
13
14
|
import { getPackageMetadata } from "../helpers.js";
|
|
14
15
|
import { checkForUpdate } from "../../core/version-check.js";
|
|
@@ -142,6 +143,25 @@ export function registerStatusCommand(program) {
|
|
|
142
143
|
? keywordIndex?.count() ?? 0
|
|
143
144
|
: 0;
|
|
144
145
|
logCliInfo(logFilePath, "status", `${c.label("Keyword index:")} ${config.retrieval.hybridSearch?.enabled ? c.enabled("enabled") : c.disabled("disabled")} (${c.num(kiCount)} chunks)`);
|
|
146
|
+
// The keyword index and the vector store are two independent stores of
|
|
147
|
+
// the same corpus. A file the store holds but the keyword index does
|
|
148
|
+
// not cover is unreachable through keyword search — report the gap
|
|
149
|
+
// instead of printing two unrelated chunk counts.
|
|
150
|
+
if (config.retrieval.hybridSearch?.enabled && keywordIndex) {
|
|
151
|
+
const gaps = await findKeywordIndexGaps(store, keywordIndex);
|
|
152
|
+
if (gaps.length > 0) {
|
|
153
|
+
logCliInfo(logFilePath, "status", `${c.label("Keyword coverage:")} ${c.warn(`${gaps.length} file(s) in the store are missing from the keyword index`)} ${c.dim("(run 'opencode-rag index' to rehydrate — no re-embedding needed)")}`);
|
|
154
|
+
for (const gap of gaps.slice(0, 5)) {
|
|
155
|
+
logCliInfo(logFilePath, "status", ` ${c.file(path.relative(cwd, gap) || gap)}`);
|
|
156
|
+
}
|
|
157
|
+
if (gaps.length > 5) {
|
|
158
|
+
logCliInfo(logFilePath, "status", ` ${c.dim(`… and ${gaps.length - 5} more`)}`);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
else {
|
|
162
|
+
logCliInfo(logFilePath, "status", `${c.label("Keyword coverage:")} ${c.success("complete")} ${c.dim("(every indexed file is searchable by keyword)")}`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
145
165
|
if (summary.rebuildRequired) {
|
|
146
166
|
logCliInfo(logFilePath, "status", `${c.label("Rebuild required:")} ${c.warn("yes")} (manifest missing/corrupt)`);
|
|
147
167
|
}
|
package/dist/cli/index.js
CHANGED
|
@@ -15,7 +15,7 @@ import { realpathSync } from "node:fs";
|
|
|
15
15
|
import { basename, dirname } from "node:path";
|
|
16
16
|
import { fileURLToPath } from "node:url";
|
|
17
17
|
import { getPackageMetadata } from "./helpers.js";
|
|
18
|
-
import { registerIndexCommand, registerQueryCommand, registerClearCommand, registerStatusCommand, registerListCommand, registerShowCommand, registerDumpCommand, registerInitCommand, registerUiCommand, registerMcpCommand, registerSetupCommand, registerUpdateCommand, registerEvalSessionsCommand, registerEvalAnalyzeCommand, registerEvalCompareCommand, registerEvalGateCommand, registerEvalDecideCommand, registerDescribeImageCommand, registerDecideCommand, registerQuirkCommand, } from "./commands/index.js";
|
|
18
|
+
import { registerIndexCommand, registerQueryCommand, registerClearCommand, registerStatusCommand, registerListCommand, registerShowCommand, registerDumpCommand, registerInitCommand, registerUiCommand, registerMcpCommand, registerSetupCommand, registerUpdateCommand, registerEvalSessionsCommand, registerEvalAnalyzeCommand, registerEvalCompareCommand, registerEvalGateCommand, registerEvalDecideCommand, registerEvalTuneCommand, registerDescribeImageCommand, registerDecideCommand, registerQuirkCommand, } from "./commands/index.js";
|
|
19
19
|
/**
|
|
20
20
|
* The top-level Commander program instance that defines the `opencode-rag` CLI.
|
|
21
21
|
*
|
|
@@ -56,6 +56,7 @@ registerEvalAnalyzeCommand(program);
|
|
|
56
56
|
registerEvalCompareCommand(program);
|
|
57
57
|
registerEvalGateCommand(program);
|
|
58
58
|
registerEvalDecideCommand(program);
|
|
59
|
+
registerEvalTuneCommand(program);
|
|
59
60
|
registerInitCommand(program);
|
|
60
61
|
registerQuirkCommand(program);
|
|
61
62
|
// ── Auto-run detection ──────────────────────────────────────────
|
|
@@ -249,6 +249,14 @@ export interface KeywordIndex {
|
|
|
249
249
|
count(): number;
|
|
250
250
|
/** Whether a chunk with the given id is indexed. Used by quirk reconcile. */
|
|
251
251
|
hasChunk(id: string): boolean;
|
|
252
|
+
/**
|
|
253
|
+
* Normalized file paths that currently have at least one indexed chunk.
|
|
254
|
+
*
|
|
255
|
+
* Used to detect a keyword index that drifted behind the vector store: an
|
|
256
|
+
* incremental index pass only re-adds the files it reprocessed, so a
|
|
257
|
+
* once-partial index stays partial unless it is compared against the store.
|
|
258
|
+
*/
|
|
259
|
+
getIndexedFilePaths(): string[];
|
|
252
260
|
/** Persist the index to disk as JSON. */
|
|
253
261
|
save(filePath?: string): Promise<void>;
|
|
254
262
|
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Golden-set-driven retrieval hyperparameter tuner.
|
|
3
|
+
*
|
|
4
|
+
* The retrieval fusion weights shipped in the config (`keywordWeight`,
|
|
5
|
+
* `symbolKeywordWeight`, …) are a single hardcoded default, but the optimum
|
|
6
|
+
* depends on the embedder, the corpus and the query mix — a code embedder
|
|
7
|
+
* ranks identifiers well through the vector half alone, while a general
|
|
8
|
+
* embedder leans on the keyword half. This module sweeps a small grid of
|
|
9
|
+
* candidates against the labelled golden set and reports which one wins.
|
|
10
|
+
*
|
|
11
|
+
* The metric implementation is shared with `eval:gate` (and therefore with
|
|
12
|
+
* the rerank A/B harness) so the tuner can never report different numbers
|
|
13
|
+
* than the gate for the same labels.
|
|
14
|
+
*/
|
|
15
|
+
import type { RagConfig } from "../core/config.js";
|
|
16
|
+
import type { SearchResult } from "../core/interfaces.js";
|
|
17
|
+
import type { GoldenLabel, GoldenOutcome, GoldenSummary } from "./golden-set.js";
|
|
18
|
+
/** One point in the retrieval search space. */
|
|
19
|
+
export interface TuningCandidate {
|
|
20
|
+
/** RRF keyword weight: 0 = pure vector ranking, 1 = pure keyword ranking. */
|
|
21
|
+
keywordWeight: number;
|
|
22
|
+
/** Keyword weight for bare-identifier queries. Defaults to `keywordWeight`. */
|
|
23
|
+
symbolKeywordWeight?: number;
|
|
24
|
+
/** Minimum fused score a result must reach. */
|
|
25
|
+
minScore: number;
|
|
26
|
+
/** Number of results requested from the pipeline. */
|
|
27
|
+
topK: number;
|
|
28
|
+
}
|
|
29
|
+
/** A candidate together with its measured quality. */
|
|
30
|
+
export interface TuningResult extends TuningCandidate {
|
|
31
|
+
/** Aggregate metrics over the whole label set. */
|
|
32
|
+
summary: GoldenSummary;
|
|
33
|
+
/** Per-label outcomes, for drill-down. */
|
|
34
|
+
outcomes: GoldenOutcome[];
|
|
35
|
+
/** Wall-clock duration of the candidate run in milliseconds. */
|
|
36
|
+
durationMs: number;
|
|
37
|
+
}
|
|
38
|
+
/** Options for {@link buildCandidateGrid}. */
|
|
39
|
+
export interface CandidateGridOptions {
|
|
40
|
+
/** Keyword weights to try. Defaults to `[0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6]`. */
|
|
41
|
+
keywordWeights?: number[];
|
|
42
|
+
/** Symbol keyword weights to try. Defaults to the keyword weights. */
|
|
43
|
+
symbolKeywordWeights?: number[];
|
|
44
|
+
/** Minimum scores to try. Defaults to `[base.minScore]`. */
|
|
45
|
+
minScores?: number[];
|
|
46
|
+
/** Top-K values to try. Defaults to `[base.topK]`. */
|
|
47
|
+
topKs?: number[];
|
|
48
|
+
/** Hard cap on the number of candidates (default 64). */
|
|
49
|
+
maxCandidates?: number;
|
|
50
|
+
}
|
|
51
|
+
/** Metrics a candidate can be ranked by. */
|
|
52
|
+
export type TuningMetric = "ndcg10" | "hit5" | "hit1" | "mrr";
|
|
53
|
+
/**
|
|
54
|
+
* Build the candidate grid.
|
|
55
|
+
*
|
|
56
|
+
* The base candidate is always included so the sweep can be compared against
|
|
57
|
+
* the configuration that is actually running. Duplicates (same weights, same
|
|
58
|
+
* minScore, same topK) are removed, and the grid is capped so a wide sweep
|
|
59
|
+
* cannot turn into an accidental benchmark.
|
|
60
|
+
*
|
|
61
|
+
* @param base - The candidate describing the current configuration.
|
|
62
|
+
* @param options - Grid overrides.
|
|
63
|
+
* @returns The candidate list, base candidate first.
|
|
64
|
+
*/
|
|
65
|
+
export declare function buildCandidateGrid(base: TuningCandidate, options?: CandidateGridOptions): TuningCandidate[];
|
|
66
|
+
/** Options for {@link runTuner}. */
|
|
67
|
+
export interface RunTunerOptions {
|
|
68
|
+
/** The labelled queries to measure. */
|
|
69
|
+
labels: GoldenLabel[];
|
|
70
|
+
/** Workspace root used to relativize result paths. */
|
|
71
|
+
root: string;
|
|
72
|
+
/**
|
|
73
|
+
* Retrieve closure for one candidate. The caller owns the embedder/store
|
|
74
|
+
* wiring and maps the candidate onto the production retrieval options.
|
|
75
|
+
*/
|
|
76
|
+
retrieve: (label: GoldenLabel, candidate: TuningCandidate) => Promise<SearchResult[]>;
|
|
77
|
+
/** The candidates to evaluate. */
|
|
78
|
+
candidates: TuningCandidate[];
|
|
79
|
+
/** Progress line callback (defaults to console.log). */
|
|
80
|
+
log?: (line: string) => void;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Evaluate every candidate against the golden set.
|
|
84
|
+
*
|
|
85
|
+
* @param opts - Labels, root, retrieve closure, candidates, logger.
|
|
86
|
+
* @returns One result per candidate, in the order the candidates were given.
|
|
87
|
+
*/
|
|
88
|
+
export declare function runTuner(opts: RunTunerOptions): Promise<TuningResult[]>;
|
|
89
|
+
/**
|
|
90
|
+
* Pick the best candidate.
|
|
91
|
+
*
|
|
92
|
+
* Ranked by the primary metric, then by Hit@5, MRR and Hit@1 so a tie on the
|
|
93
|
+
* primary metric still resolves to the more useful configuration.
|
|
94
|
+
*
|
|
95
|
+
* @param results - The measured candidates.
|
|
96
|
+
* @param metric - The primary metric (default `ndcg10`).
|
|
97
|
+
* @returns The winning result, or `undefined` when there are no results.
|
|
98
|
+
*/
|
|
99
|
+
export declare function selectBestTuning(results: TuningResult[], metric?: TuningMetric): TuningResult | undefined;
|
|
100
|
+
/**
|
|
101
|
+
* Compare the winner against the candidate that describes the current config.
|
|
102
|
+
*
|
|
103
|
+
* @param best - The winning candidate.
|
|
104
|
+
* @param current - The candidate describing the running configuration.
|
|
105
|
+
* @returns The per-metric deltas (winner minus current).
|
|
106
|
+
*/
|
|
107
|
+
export declare function tuningDelta(best: TuningResult, current: TuningResult | undefined): Record<string, number>;
|
|
108
|
+
/**
|
|
109
|
+
* Write the winning weights into a copy of the configuration.
|
|
110
|
+
*
|
|
111
|
+
* The input config is not mutated — the caller decides whether to persist the
|
|
112
|
+
* result. Only the retrieval section is touched.
|
|
113
|
+
*
|
|
114
|
+
* @param config - The configuration to base the result on.
|
|
115
|
+
* @param best - The winning candidate.
|
|
116
|
+
* @returns A new config with the winning fusion weights applied.
|
|
117
|
+
*/
|
|
118
|
+
export declare function applyTuningToConfig(config: RagConfig, best: TuningCandidate): RagConfig;
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import { runGoldenGate, } from "./golden-gate.js";
|
|
2
|
+
/** The metric used to pick the winner, and the tie-breakers after it. */
|
|
3
|
+
const METRIC_ORDER = {
|
|
4
|
+
ndcg10: (s) => s.ndcg10,
|
|
5
|
+
hit5: (s) => s.hit5,
|
|
6
|
+
hit1: (s) => s.hit1,
|
|
7
|
+
mrr: (s) => s.mrr,
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Build the candidate grid.
|
|
11
|
+
*
|
|
12
|
+
* The base candidate is always included so the sweep can be compared against
|
|
13
|
+
* the configuration that is actually running. Duplicates (same weights, same
|
|
14
|
+
* minScore, same topK) are removed, and the grid is capped so a wide sweep
|
|
15
|
+
* cannot turn into an accidental benchmark.
|
|
16
|
+
*
|
|
17
|
+
* @param base - The candidate describing the current configuration.
|
|
18
|
+
* @param options - Grid overrides.
|
|
19
|
+
* @returns The candidate list, base candidate first.
|
|
20
|
+
*/
|
|
21
|
+
export function buildCandidateGrid(base, options = {}) {
|
|
22
|
+
const keywordWeights = options.keywordWeights ?? [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6];
|
|
23
|
+
const symbolKeywordWeights = options.symbolKeywordWeights ?? keywordWeights;
|
|
24
|
+
const minScores = options.minScores ?? [base.minScore];
|
|
25
|
+
const topKs = options.topKs ?? [base.topK];
|
|
26
|
+
const maxCandidates = options.maxCandidates ?? 64;
|
|
27
|
+
const seen = new Set();
|
|
28
|
+
const candidates = [];
|
|
29
|
+
const push = (candidate) => {
|
|
30
|
+
const key = [
|
|
31
|
+
candidate.keywordWeight,
|
|
32
|
+
candidate.symbolKeywordWeight ?? "unset",
|
|
33
|
+
candidate.minScore,
|
|
34
|
+
candidate.topK,
|
|
35
|
+
].join("|");
|
|
36
|
+
if (seen.has(key))
|
|
37
|
+
return;
|
|
38
|
+
seen.add(key);
|
|
39
|
+
candidates.push(candidate);
|
|
40
|
+
};
|
|
41
|
+
push({ ...base });
|
|
42
|
+
for (const topK of topKs) {
|
|
43
|
+
for (const minScore of minScores) {
|
|
44
|
+
for (const keywordWeight of keywordWeights) {
|
|
45
|
+
for (const symbolKeywordWeight of symbolKeywordWeights) {
|
|
46
|
+
push({ keywordWeight, symbolKeywordWeight, minScore, topK });
|
|
47
|
+
if (candidates.length >= maxCandidates)
|
|
48
|
+
return candidates;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return candidates;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Evaluate every candidate against the golden set.
|
|
57
|
+
*
|
|
58
|
+
* @param opts - Labels, root, retrieve closure, candidates, logger.
|
|
59
|
+
* @returns One result per candidate, in the order the candidates were given.
|
|
60
|
+
*/
|
|
61
|
+
export async function runTuner(opts) {
|
|
62
|
+
const log = opts.log ?? (() => { });
|
|
63
|
+
const results = [];
|
|
64
|
+
for (const candidate of opts.candidates) {
|
|
65
|
+
const started = performance.now();
|
|
66
|
+
const gateOpts = {
|
|
67
|
+
labels: opts.labels,
|
|
68
|
+
root: opts.root,
|
|
69
|
+
retrieve: (label) => opts.retrieve(label, candidate),
|
|
70
|
+
log: () => { },
|
|
71
|
+
};
|
|
72
|
+
const { summary, outcomes } = await runGoldenGate(gateOpts);
|
|
73
|
+
const durationMs = performance.now() - started;
|
|
74
|
+
results.push({ ...candidate, summary, outcomes, durationMs });
|
|
75
|
+
log(` kw=${candidate.keywordWeight} skw=${candidate.symbolKeywordWeight ?? "-"} ` +
|
|
76
|
+
`minScore=${candidate.minScore} topK=${candidate.topK} ` +
|
|
77
|
+
`Hit@5 ${(summary.hit5 * 100).toFixed(1)}% MRR ${summary.mrr.toFixed(3)} ` +
|
|
78
|
+
`nDCG@10 ${summary.ndcg10.toFixed(3)} (${durationMs.toFixed(0)}ms)`);
|
|
79
|
+
}
|
|
80
|
+
return results;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Pick the best candidate.
|
|
84
|
+
*
|
|
85
|
+
* Ranked by the primary metric, then by Hit@5, MRR and Hit@1 so a tie on the
|
|
86
|
+
* primary metric still resolves to the more useful configuration.
|
|
87
|
+
*
|
|
88
|
+
* @param results - The measured candidates.
|
|
89
|
+
* @param metric - The primary metric (default `ndcg10`).
|
|
90
|
+
* @returns The winning result, or `undefined` when there are no results.
|
|
91
|
+
*/
|
|
92
|
+
export function selectBestTuning(results, metric = "ndcg10") {
|
|
93
|
+
if (results.length === 0)
|
|
94
|
+
return undefined;
|
|
95
|
+
const primary = METRIC_ORDER[metric];
|
|
96
|
+
const score = (r) => [
|
|
97
|
+
primary(r.summary),
|
|
98
|
+
r.summary.hit5,
|
|
99
|
+
r.summary.mrr,
|
|
100
|
+
r.summary.hit1,
|
|
101
|
+
];
|
|
102
|
+
return [...results].sort((a, b) => {
|
|
103
|
+
const sa = score(a);
|
|
104
|
+
const sb = score(b);
|
|
105
|
+
for (let i = 0; i < sa.length; i++) {
|
|
106
|
+
if (sb[i] !== sa[i])
|
|
107
|
+
return sb[i] - sa[i];
|
|
108
|
+
}
|
|
109
|
+
return 0;
|
|
110
|
+
})[0];
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Compare the winner against the candidate that describes the current config.
|
|
114
|
+
*
|
|
115
|
+
* @param best - The winning candidate.
|
|
116
|
+
* @param current - The candidate describing the running configuration.
|
|
117
|
+
* @returns The per-metric deltas (winner minus current).
|
|
118
|
+
*/
|
|
119
|
+
export function tuningDelta(best, current) {
|
|
120
|
+
if (!current)
|
|
121
|
+
return {};
|
|
122
|
+
return {
|
|
123
|
+
hit1: best.summary.hit1 - current.summary.hit1,
|
|
124
|
+
hit3: best.summary.hit3 - current.summary.hit3,
|
|
125
|
+
hit5: best.summary.hit5 - current.summary.hit5,
|
|
126
|
+
hit10: best.summary.hit10 - current.summary.hit10,
|
|
127
|
+
mrr: best.summary.mrr - current.summary.mrr,
|
|
128
|
+
ndcg10: best.summary.ndcg10 - current.summary.ndcg10,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Write the winning weights into a copy of the configuration.
|
|
133
|
+
*
|
|
134
|
+
* The input config is not mutated — the caller decides whether to persist the
|
|
135
|
+
* result. Only the retrieval section is touched.
|
|
136
|
+
*
|
|
137
|
+
* @param config - The configuration to base the result on.
|
|
138
|
+
* @param best - The winning candidate.
|
|
139
|
+
* @returns A new config with the winning fusion weights applied.
|
|
140
|
+
*/
|
|
141
|
+
export function applyTuningToConfig(config, best) {
|
|
142
|
+
const hybrid = config.retrieval.hybridSearch ?? { enabled: true, keywordWeight: best.keywordWeight };
|
|
143
|
+
return {
|
|
144
|
+
...config,
|
|
145
|
+
retrieval: {
|
|
146
|
+
...config.retrieval,
|
|
147
|
+
topK: best.topK,
|
|
148
|
+
minScore: best.minScore,
|
|
149
|
+
hybridSearch: {
|
|
150
|
+
...hybrid,
|
|
151
|
+
enabled: hybrid.enabled ?? true,
|
|
152
|
+
keywordWeight: best.keywordWeight,
|
|
153
|
+
symbolKeywordWeight: best.symbolKeywordWeight ?? hybrid.symbolKeywordWeight,
|
|
154
|
+
},
|
|
155
|
+
},
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
//# sourceMappingURL=tuner.js.map
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { KeywordIndex, VectorStore } from "../core/interfaces.js";
|
|
2
|
+
/** Options for {@link rehydrateKeywordIndex}. */
|
|
3
|
+
export interface KeywordRehydrationOptions {
|
|
4
|
+
/** The vector store holding the authoritative chunk rows. */
|
|
5
|
+
store: VectorStore;
|
|
6
|
+
/** The keyword index to repair. */
|
|
7
|
+
keywordIndex: KeywordIndex;
|
|
8
|
+
/** Optional logger (missing methods default to no-ops). */
|
|
9
|
+
logger?: {
|
|
10
|
+
info?(message: string): void;
|
|
11
|
+
warn?(message: string): void;
|
|
12
|
+
debug?(message: string): void;
|
|
13
|
+
};
|
|
14
|
+
/** Maximum number of files to rehydrate in one call (0/undefined = unlimited). */
|
|
15
|
+
maxFiles?: number;
|
|
16
|
+
}
|
|
17
|
+
/** Outcome of a rehydration attempt. */
|
|
18
|
+
export interface KeywordRehydrationResult {
|
|
19
|
+
/** Files present in the store with no keyword entries. */
|
|
20
|
+
missingFiles: number;
|
|
21
|
+
/** Files whose chunks were added to the keyword index. */
|
|
22
|
+
rehydratedFiles: number;
|
|
23
|
+
/** Chunks added to the keyword index. */
|
|
24
|
+
rehydratedChunks: number;
|
|
25
|
+
/** Chunks skipped because they were already indexed. */
|
|
26
|
+
skippedChunks: number;
|
|
27
|
+
/** Files that could not be rehydrated (one failure never aborts the rest). */
|
|
28
|
+
failedFiles: string[];
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Restore the keyword entries for every file the store holds but the keyword
|
|
32
|
+
* index does not.
|
|
33
|
+
*
|
|
34
|
+
* Quirk rows are skipped: `quirks.jsonl` owns them, and `reconcileQuirks()`
|
|
35
|
+
* adds and removes them under its own rules.
|
|
36
|
+
*
|
|
37
|
+
* @param options - The store, the keyword index, and optional logging.
|
|
38
|
+
* @returns What was missing and what was restored.
|
|
39
|
+
*/
|
|
40
|
+
export declare function rehydrateKeywordIndex(options: KeywordRehydrationOptions): Promise<KeywordRehydrationResult>;
|
|
41
|
+
/**
|
|
42
|
+
* Compare the keyword index against the vector store without changing either.
|
|
43
|
+
*
|
|
44
|
+
* @param store - The vector store holding the authoritative chunk rows.
|
|
45
|
+
* @param keywordIndex - The keyword index to compare.
|
|
46
|
+
* @returns The files the store has that the keyword index does not cover.
|
|
47
|
+
*/
|
|
48
|
+
export declare function findKeywordIndexGaps(store: VectorStore, keywordIndex: KeywordIndex): Promise<string[]>;
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Repairs a keyword index that drifted behind the vector store.
|
|
3
|
+
*
|
|
4
|
+
* The keyword index (BM25 side of hybrid search) and the vector store are two
|
|
5
|
+
* independent stores of the same corpus. Nothing keeps them in sync by
|
|
6
|
+
* construction: an incremental index pass only calls `addChunks()` for the
|
|
7
|
+
* files it reprocessed, so unchanged files keep whatever the *loaded* index
|
|
8
|
+
* had. Once the persisted index loses entries — a crashed pass that saved a
|
|
9
|
+
* cleared index, a partial indexer, a rebuild that swapped the store without
|
|
10
|
+
* rewriting the sidecar — every later pass re-saves the same partial state
|
|
11
|
+
* and the missing files stay unreachable through keyword search forever.
|
|
12
|
+
*
|
|
13
|
+
* The repair needs no re-embedding: the store already holds the chunk text
|
|
14
|
+
* and descriptions, so the missing keyword entries can be rebuilt from the
|
|
15
|
+
* store's own rows.
|
|
16
|
+
*/
|
|
17
|
+
import path from "node:path";
|
|
18
|
+
import { normalizeFilePath } from "../core/manifest.js";
|
|
19
|
+
import { QUIRK_FILE_PREFIX } from "../quirks/quirk-store.js";
|
|
20
|
+
/**
|
|
21
|
+
* Restore the keyword entries for every file the store holds but the keyword
|
|
22
|
+
* index does not.
|
|
23
|
+
*
|
|
24
|
+
* Quirk rows are skipped: `quirks.jsonl` owns them, and `reconcileQuirks()`
|
|
25
|
+
* adds and removes them under its own rules.
|
|
26
|
+
*
|
|
27
|
+
* @param options - The store, the keyword index, and optional logging.
|
|
28
|
+
* @returns What was missing and what was restored.
|
|
29
|
+
*/
|
|
30
|
+
export async function rehydrateKeywordIndex(options) {
|
|
31
|
+
const { store, keywordIndex, logger } = options;
|
|
32
|
+
const result = {
|
|
33
|
+
missingFiles: 0,
|
|
34
|
+
rehydratedFiles: 0,
|
|
35
|
+
rehydratedChunks: 0,
|
|
36
|
+
skippedChunks: 0,
|
|
37
|
+
failedFiles: [],
|
|
38
|
+
};
|
|
39
|
+
let storePaths;
|
|
40
|
+
try {
|
|
41
|
+
storePaths = await store.getFilePaths();
|
|
42
|
+
}
|
|
43
|
+
catch (err) {
|
|
44
|
+
logger?.warn?.(`Keyword rehydration skipped — could not list store files: ${err.message}`);
|
|
45
|
+
return result;
|
|
46
|
+
}
|
|
47
|
+
if (storePaths.length === 0)
|
|
48
|
+
return result;
|
|
49
|
+
const indexed = new Set(keywordIndex.getIndexedFilePaths());
|
|
50
|
+
const missing = storePaths.filter((p) => {
|
|
51
|
+
// Quirk rows belong to quirks.jsonl — reconcileQuirks() owns them.
|
|
52
|
+
if (path.basename(p).startsWith(QUIRK_FILE_PREFIX))
|
|
53
|
+
return false;
|
|
54
|
+
return !indexed.has(normalizeFilePath(p));
|
|
55
|
+
});
|
|
56
|
+
result.missingFiles = missing.length;
|
|
57
|
+
if (missing.length === 0)
|
|
58
|
+
return result;
|
|
59
|
+
const maxFiles = options.maxFiles && options.maxFiles > 0 ? options.maxFiles : missing.length;
|
|
60
|
+
const targets = missing.slice(0, maxFiles);
|
|
61
|
+
if (targets.length < missing.length) {
|
|
62
|
+
logger?.info?.(`Keyword rehydration: ${missing.length} file(s) missing from the keyword index — rehydrating the first ${targets.length}`);
|
|
63
|
+
}
|
|
64
|
+
for (const filePath of targets) {
|
|
65
|
+
try {
|
|
66
|
+
const chunks = await store.getChunksByFilePath(filePath);
|
|
67
|
+
if (chunks.length === 0)
|
|
68
|
+
continue;
|
|
69
|
+
// Guard against double counting: `addChunks()` is not idempotent for the
|
|
70
|
+
// BM25 length statistics and term frequencies, so a chunk that is
|
|
71
|
+
// already indexed must never be added a second time.
|
|
72
|
+
const fresh = chunks.filter((c) => !keywordIndex.hasChunk(c.id));
|
|
73
|
+
result.skippedChunks += chunks.length - fresh.length;
|
|
74
|
+
if (fresh.length === 0)
|
|
75
|
+
continue;
|
|
76
|
+
keywordIndex.addChunks(fresh);
|
|
77
|
+
result.rehydratedFiles++;
|
|
78
|
+
result.rehydratedChunks += fresh.length;
|
|
79
|
+
}
|
|
80
|
+
catch (err) {
|
|
81
|
+
result.failedFiles.push(filePath);
|
|
82
|
+
logger?.warn?.(`Keyword rehydration failed for ${filePath}: ${err.message}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (result.rehydratedFiles > 0) {
|
|
86
|
+
logger?.info?.(`Keyword rehydration: restored ${result.rehydratedChunks} chunk(s) from ` +
|
|
87
|
+
`${result.rehydratedFiles} file(s) (${result.missingFiles} file(s) were missing from the keyword index)`);
|
|
88
|
+
}
|
|
89
|
+
return result;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Compare the keyword index against the vector store without changing either.
|
|
93
|
+
*
|
|
94
|
+
* @param store - The vector store holding the authoritative chunk rows.
|
|
95
|
+
* @param keywordIndex - The keyword index to compare.
|
|
96
|
+
* @returns The files the store has that the keyword index does not cover.
|
|
97
|
+
*/
|
|
98
|
+
export async function findKeywordIndexGaps(store, keywordIndex) {
|
|
99
|
+
let storePaths;
|
|
100
|
+
try {
|
|
101
|
+
storePaths = await store.getFilePaths();
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
return [];
|
|
105
|
+
}
|
|
106
|
+
const indexed = new Set(keywordIndex.getIndexedFilePaths());
|
|
107
|
+
return storePaths.filter((p) => {
|
|
108
|
+
if (path.basename(p).startsWith(QUIRK_FILE_PREFIX))
|
|
109
|
+
return false;
|
|
110
|
+
return !indexed.has(normalizeFilePath(p));
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=keyword-rehydration.js.map
|
package/dist/indexer/pipeline.js
CHANGED
|
@@ -11,6 +11,7 @@ import { embedBatch, probeEmbeddingDimension } from "../embedder/factory.js";
|
|
|
11
11
|
import { createVectorStore } from "../vectorstore/factory.js";
|
|
12
12
|
import { swapStoreDirectories } from "../vectorstore/lancedb.js";
|
|
13
13
|
import { reconcileQuirks, QUIRK_FILE_PREFIX } from "../quirks/quirk-store.js";
|
|
14
|
+
import { rehydrateKeywordIndex } from "./keyword-rehydration.js";
|
|
14
15
|
import { createIndexStats } from "./stats.js";
|
|
15
16
|
import { prepareFile, buildTextsToEmbed } from "./worker.js";
|
|
16
17
|
import { tagChunksRole } from "../core/chunk-role.js";
|
|
@@ -309,6 +310,32 @@ async function runIndexPassInner(options, logger) {
|
|
|
309
310
|
}
|
|
310
311
|
let filterPaths;
|
|
311
312
|
let gitDeletedPaths = [];
|
|
313
|
+
// ── Keyword-index rehydration ──────────────────────────────────────────
|
|
314
|
+
// The keyword index (BM25 half of hybrid search) and the vector store are
|
|
315
|
+
// two independent stores of the same corpus, and nothing keeps them in sync
|
|
316
|
+
// by construction: an incremental pass only re-adds the files it reprocessed,
|
|
317
|
+
// so a once-partial index stays partial forever and the missing files are
|
|
318
|
+
// unreachable through keyword search. Rebuild the missing entries from the
|
|
319
|
+
// store's own rows — no re-embedding needed.
|
|
320
|
+
//
|
|
321
|
+
// Skipped during a rebuild: the rebuild re-indexes every file from source,
|
|
322
|
+
// so the keyword index is repopulated from scratch anyway.
|
|
323
|
+
if (options.keywordIndex && !rebuildPerformed) {
|
|
324
|
+
try {
|
|
325
|
+
await rehydrateKeywordIndex({
|
|
326
|
+
store: effectiveStore,
|
|
327
|
+
keywordIndex: options.keywordIndex,
|
|
328
|
+
logger: {
|
|
329
|
+
info: (message) => logger.info(message),
|
|
330
|
+
warn: (message) => logger.warn(message),
|
|
331
|
+
debug: (message) => logger.debug(message),
|
|
332
|
+
},
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
catch (err) {
|
|
336
|
+
logger.warn(`Keyword rehydration failed: ${err.message}`);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
312
339
|
if (!options.force && manifestStatus === "ok" && manifest.lastGitCommit) {
|
|
313
340
|
const repoRoot = getRepoRoot(options.cwd);
|
|
314
341
|
if (repoRoot) {
|
|
@@ -45,6 +45,16 @@ export declare class KeywordIndex {
|
|
|
45
45
|
clear(): void;
|
|
46
46
|
count(): number;
|
|
47
47
|
hasChunk(id: string): boolean;
|
|
48
|
+
/**
|
|
49
|
+
* Normalized file paths that currently have at least one indexed chunk.
|
|
50
|
+
*
|
|
51
|
+
* The keyword index and the vector store are two independent stores of the
|
|
52
|
+
* same corpus, and nothing keeps them in sync by construction: an
|
|
53
|
+
* incremental index pass only calls {@link addChunks} for the files it
|
|
54
|
+
* reprocessed, so a once-partial index stays partial forever. Comparing
|
|
55
|
+
* this list with the store's file list is how that drift is detected.
|
|
56
|
+
*/
|
|
57
|
+
getIndexedFilePaths(): string[];
|
|
48
58
|
save(storePath?: string): Promise<void>;
|
|
49
59
|
static load(storePath: string): Promise<KeywordIndex>;
|
|
50
60
|
static clearFile(storePath: string): Promise<void>;
|
|
@@ -276,6 +276,18 @@ export class KeywordIndex {
|
|
|
276
276
|
hasChunk(id) {
|
|
277
277
|
return this.chunkMap.has(id);
|
|
278
278
|
}
|
|
279
|
+
/**
|
|
280
|
+
* Normalized file paths that currently have at least one indexed chunk.
|
|
281
|
+
*
|
|
282
|
+
* The keyword index and the vector store are two independent stores of the
|
|
283
|
+
* same corpus, and nothing keeps them in sync by construction: an
|
|
284
|
+
* incremental index pass only calls {@link addChunks} for the files it
|
|
285
|
+
* reprocessed, so a once-partial index stays partial forever. Comparing
|
|
286
|
+
* this list with the store's file list is how that drift is detected.
|
|
287
|
+
*/
|
|
288
|
+
getIndexedFilePaths() {
|
|
289
|
+
return [...this.fileToIds.keys()];
|
|
290
|
+
}
|
|
279
291
|
async save(storePath) {
|
|
280
292
|
const effectiveStorePath = storePath ?? this.storePath;
|
|
281
293
|
if (!effectiveStorePath)
|