opencode-rag-plugin 2.2.2 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.js +2 -0
- package/dist/chunker/factory.js +48 -18
- package/dist/chunker/image.js +6 -0
- package/dist/cli/commands/clear.js +21 -1
- package/dist/cli/commands/query.js +6 -0
- package/dist/cli/commands/status.js +4 -0
- package/dist/core/bootstrap.d.ts +3 -1
- package/dist/core/bootstrap.js +5 -0
- package/dist/core/config.d.ts +69 -1
- package/dist/core/config.js +63 -1
- package/dist/core/interfaces.d.ts +34 -0
- package/dist/describer/describer.js +9 -1
- package/dist/embedder/factory.d.ts +5 -1
- package/dist/embedder/factory.js +76 -6
- package/dist/embedder/health.d.ts +1 -1
- package/dist/embedder/health.js +43 -0
- package/dist/eval/rerank-eval.d.ts +45 -0
- package/dist/eval/rerank-eval.js +581 -0
- package/dist/indexer/pipeline.d.ts +17 -0
- package/dist/indexer/pipeline.js +123 -3
- package/dist/indexer/stats.d.ts +7 -0
- package/dist/mcp/handlers.js +3 -0
- package/dist/opencode/create-read-tool.js +5 -2
- package/dist/plugin.js +67 -6
- package/dist/quirks/quirk-store.d.ts +1 -0
- package/dist/quirks/quirk-store.js +1 -1
- package/dist/quirks/tool-error-recall.d.ts +112 -0
- package/dist/quirks/tool-error-recall.js +144 -0
- package/dist/reranker/base.d.ts +47 -0
- package/dist/reranker/base.js +73 -0
- package/dist/reranker/factory.d.ts +16 -0
- package/dist/reranker/factory.js +47 -0
- package/dist/reranker/llama-server.d.ts +33 -0
- package/dist/reranker/llama-server.js +101 -0
- package/dist/retriever/keyword-index.d.ts +1 -0
- package/dist/retriever/keyword-index.js +1 -1
- package/dist/retriever/retriever.d.ts +24 -1
- package/dist/retriever/retriever.js +69 -5
- package/dist/types/opencode-plugin.d.ts +2 -2
- package/dist/v2/v2-adapter.d.ts +6 -1
- package/dist/v2/v2-adapter.js +52 -8
- package/dist/vectorstore/lancedb.d.ts +1 -1
- package/dist/vectorstore/lancedb.js +43 -7
- package/dist/web/api.js +5 -0
- package/package.json +1 -1
|
@@ -0,0 +1,581 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Rerank A/B evaluation harness: runs labeled ground-truth queries through
|
|
3
|
+
* the real retrieval pipeline (baseline vs. opt-in rerank stage) and reports
|
|
4
|
+
* Hit@k, Precision@5, Recall@k, MRR, nDCG@10, and warm latency (p50/p95).
|
|
5
|
+
*
|
|
6
|
+
* Usage: node --import tsx src/eval/rerank-eval.ts [options]
|
|
7
|
+
*
|
|
8
|
+
* Options:
|
|
9
|
+
* --arms=baseline,rerank20,rerank30,rerank-desc Arms to run (default: baseline,rerank20,rerank30)
|
|
10
|
+
* --topk=20 Override retrieval topK (default: config)
|
|
11
|
+
* --minScore=0.35 Override retrieval minScore (default: config)
|
|
12
|
+
* --category=nl Only run labels of one category (nl|gotcha|symbol|de)
|
|
13
|
+
* --limit=10 Only run the first N labels (after filtering)
|
|
14
|
+
* --warmup=2 Discarded warmup queries per arm before measuring (default 2;
|
|
15
|
+
* llama-swap cold starts measure 15-25s and would pollute p50/p95)
|
|
16
|
+
* --labels=path Label file (default: src/eval/rerank-labels.json)
|
|
17
|
+
* --report=path Report file (default: doc/rerank-eval-report.md)
|
|
18
|
+
* --rerank-base-url=URL Rerank endpoint (e.g. http://127.0.0.1:11437/v1). Passing any
|
|
19
|
+
* --rerank-* flag enables the rerank arms without touching the
|
|
20
|
+
* user's opencode-rag.json (synthesized enabled section).
|
|
21
|
+
* --rerank-model=NAME Reranker model name (e.g. Qwen3-Reranker-0.6B)
|
|
22
|
+
* --rerank-api-key=KEY Optional bearer token for the rerank endpoint
|
|
23
|
+
* --rerank-provider=ID Provider id (default: llama-server)
|
|
24
|
+
* --rerank-timeout=MS Override the rerank stage timeout for this run (default: provider
|
|
25
|
+
* config, 1500). Raise it when the server needs >1.5 s for large
|
|
26
|
+
* candidate pools — otherwise most queries degrade to fusion order.
|
|
27
|
+
* --embedding-model=ID Override embedding.model for this run (e.g. Qwen3-Embedding:8B).
|
|
28
|
+
* Useful when the shared config is broken or leased by another session.
|
|
29
|
+
* --embedding-base-url=… Override embedding.baseUrl for this run.
|
|
30
|
+
*
|
|
31
|
+
* Design notes:
|
|
32
|
+
* - Ground truth lives in `rerank-labels.json` as query -> file paths (+ optional
|
|
33
|
+
* symbol). Chunk IDs are deliberately NOT used: they do not survive re-indexing.
|
|
34
|
+
* - The reranker module is imported dynamically so this harness type-checks and runs
|
|
35
|
+
* while `src/reranker/**` is still under construction. Rerank arms are skipped with
|
|
36
|
+
* a diagnostic when the config has no enabled `reranking.*` section, the module is
|
|
37
|
+
* missing, or the provider cannot be created (e.g. llama-server without --reranking).
|
|
38
|
+
* - Latency is measured warm: each arm issues one discarded warmup query first, so
|
|
39
|
+
* embedding/rerank connection setup does not pollute p50/p95. Cold start is a
|
|
40
|
+
* separate, documented number — not part of this measurement.
|
|
41
|
+
* - `retrieve()` is called through a loose local signature on purpose: the rerank
|
|
42
|
+
* stage is being developed in parallel, and this harness must not depend on its
|
|
43
|
+
* exact RetrieveOptions type.
|
|
44
|
+
*/
|
|
45
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
46
|
+
import path from "node:path";
|
|
47
|
+
import { DEFAULT_CONFIG, loadConfig } from "../core/config.js";
|
|
48
|
+
import { createEmbedder } from "../embedder/factory.js";
|
|
49
|
+
import { createVectorStore } from "../vectorstore/factory.js";
|
|
50
|
+
import { readStoreDimension } from "../vectorstore/lancedb.js";
|
|
51
|
+
import { retrieve } from "../retriever/retriever.js";
|
|
52
|
+
import { KeywordIndex } from "../retriever/keyword-index.js";
|
|
53
|
+
import { applyRuntimeOverrides, loadRuntimeOverrides } from "../core/runtime-overrides.js";
|
|
54
|
+
import { resolveApiKey } from "../core/resolve-api-key.js";
|
|
55
|
+
const WORKTREE = process.cwd();
|
|
56
|
+
const CONFIG_PATH = path.join(WORKTREE, "opencode-rag.json");
|
|
57
|
+
const STORE_PATH = path.join(WORKTREE, ".opencode", "rag_db");
|
|
58
|
+
const DEFAULT_LABELS_PATH = path.join(WORKTREE, "src", "eval", "rerank-labels.json");
|
|
59
|
+
const DEFAULT_REPORT_PATH = path.join(WORKTREE, "doc", "rerank-eval-report.md");
|
|
60
|
+
const RERANK_FACTORY_SPECIFIER = "../reranker/factory.js";
|
|
61
|
+
const retrieveFn = retrieve;
|
|
62
|
+
const ARM_SPECS = [
|
|
63
|
+
{ name: "baseline", description: "current pipeline (reranking disabled)", reranked: false },
|
|
64
|
+
{ name: "rerank20", description: "rerank stage, candidates=20", reranked: true, overrides: { candidates: 20 } },
|
|
65
|
+
{ name: "rerank30", description: "rerank stage, candidates=30", reranked: true, overrides: { candidates: 30 } },
|
|
66
|
+
{ name: "rerank-desc", description: "rerank stage, docField=content+description", reranked: true, overrides: { candidates: 20, docField: "content+description" } },
|
|
67
|
+
];
|
|
68
|
+
function parseOptions(argv) {
|
|
69
|
+
const kv = new Map();
|
|
70
|
+
for (const arg of argv) {
|
|
71
|
+
if (!arg.startsWith("--"))
|
|
72
|
+
continue;
|
|
73
|
+
const eq = arg.indexOf("=");
|
|
74
|
+
if (eq === -1)
|
|
75
|
+
kv.set(arg.slice(2), "true");
|
|
76
|
+
else
|
|
77
|
+
kv.set(arg.slice(2, eq), arg.slice(eq + 1));
|
|
78
|
+
}
|
|
79
|
+
const num = (key) => {
|
|
80
|
+
const raw = kv.get(key);
|
|
81
|
+
if (raw === undefined)
|
|
82
|
+
return undefined;
|
|
83
|
+
const n = Number(raw);
|
|
84
|
+
return Number.isFinite(n) ? n : undefined;
|
|
85
|
+
};
|
|
86
|
+
const armsRaw = kv.get("arms");
|
|
87
|
+
return {
|
|
88
|
+
arms: armsRaw ? armsRaw.split(",").map((a) => a.trim()).filter(Boolean) : ["baseline", "rerank20", "rerank30"],
|
|
89
|
+
topK: num("topk"),
|
|
90
|
+
minScore: num("minScore"),
|
|
91
|
+
category: kv.get("category"),
|
|
92
|
+
limit: num("limit"),
|
|
93
|
+
warmup: num("warmup"),
|
|
94
|
+
labelsPath: kv.get("labels") ? path.resolve(WORKTREE, kv.get("labels")) : DEFAULT_LABELS_PATH,
|
|
95
|
+
reportPath: kv.get("report") ? path.resolve(WORKTREE, kv.get("report")) : DEFAULT_REPORT_PATH,
|
|
96
|
+
rerankBaseUrl: kv.get("rerank-base-url"),
|
|
97
|
+
rerankModel: kv.get("rerank-model"),
|
|
98
|
+
rerankApiKey: kv.get("rerank-api-key"),
|
|
99
|
+
rerankProvider: kv.get("rerank-provider"),
|
|
100
|
+
rerankTimeoutMs: num("rerank-timeout"),
|
|
101
|
+
embeddingModel: kv.get("embedding-model"),
|
|
102
|
+
embeddingBaseUrl: kv.get("embedding-base-url"),
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
function loadLabels(labelsPath) {
|
|
106
|
+
if (!existsSync(labelsPath)) {
|
|
107
|
+
throw new Error(`Label file not found: ${labelsPath}`);
|
|
108
|
+
}
|
|
109
|
+
const parsed = JSON.parse(readFileSync(labelsPath, "utf-8"));
|
|
110
|
+
if (!Array.isArray(parsed.labels) || parsed.labels.length === 0) {
|
|
111
|
+
throw new Error(`Label file has no labels: ${labelsPath}`);
|
|
112
|
+
}
|
|
113
|
+
for (const label of parsed.labels) {
|
|
114
|
+
if (!label.id || !label.query || !Array.isArray(label.files) || label.files.length === 0) {
|
|
115
|
+
throw new Error(`Malformed label (needs id, query, files[]): ${JSON.stringify(label)}`);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return parsed.labels;
|
|
119
|
+
}
|
|
120
|
+
async function bootstrap(opts) {
|
|
121
|
+
let cfg;
|
|
122
|
+
if (existsSync(CONFIG_PATH)) {
|
|
123
|
+
cfg = loadConfig(CONFIG_PATH);
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
cfg = structuredClone(DEFAULT_CONFIG);
|
|
127
|
+
}
|
|
128
|
+
const overrides = loadRuntimeOverrides(STORE_PATH);
|
|
129
|
+
cfg = applyRuntimeOverrides(cfg, overrides);
|
|
130
|
+
resolveApiKey(cfg, WORKTREE);
|
|
131
|
+
// CLI overrides for the embedding endpoint — never touch the shared config file
|
|
132
|
+
// (it may be leased/broken by another session).
|
|
133
|
+
if (opts.embeddingModel)
|
|
134
|
+
cfg.embedding.model = opts.embeddingModel;
|
|
135
|
+
if (opts.embeddingBaseUrl)
|
|
136
|
+
cfg.embedding.baseUrl = opts.embeddingBaseUrl;
|
|
137
|
+
const storePath = path.resolve(WORKTREE, cfg.vectorStore.path);
|
|
138
|
+
const dimension = (await readStoreDimension(storePath)) ??
|
|
139
|
+
cfg.embedding.vectorDimension ??
|
|
140
|
+
384;
|
|
141
|
+
const embedder = createEmbedder(cfg);
|
|
142
|
+
const store = createVectorStore(cfg, storePath, dimension);
|
|
143
|
+
let keywordIndex;
|
|
144
|
+
try {
|
|
145
|
+
keywordIndex = await KeywordIndex.load(storePath);
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
// optional — hybrid search degrades to vector-only, same as the plugin
|
|
149
|
+
}
|
|
150
|
+
const indexedCount = await store.count();
|
|
151
|
+
return { cfg, embedder, store, keywordIndex, storePath, dimension, indexedCount };
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Resolve a rerank provider from the parallel rerank implementation without a static
|
|
155
|
+
* import (the module may not exist yet). Returns a diagnostic instead of throwing.
|
|
156
|
+
*/
|
|
157
|
+
async function resolveRerankProvider(section) {
|
|
158
|
+
if (!section || section.enabled !== true) {
|
|
159
|
+
return { error: "no enabled reranking config — pass --rerank-base-url=… --rerank-model=… or set reranking.enabled=true" };
|
|
160
|
+
}
|
|
161
|
+
try {
|
|
162
|
+
const mod = (await import(RERANK_FACTORY_SPECIFIER));
|
|
163
|
+
if (typeof mod.getRerankerFor !== "function") {
|
|
164
|
+
return { error: "src/reranker/factory.ts exists but has no getRerankerFor export yet" };
|
|
165
|
+
}
|
|
166
|
+
const provider = await mod.getRerankerFor(section);
|
|
167
|
+
if (!provider) {
|
|
168
|
+
return { error: "getRerankerFor returned undefined (provider disabled/unavailable)" };
|
|
169
|
+
}
|
|
170
|
+
return { provider };
|
|
171
|
+
}
|
|
172
|
+
catch (err) {
|
|
173
|
+
return { error: `reranker module unavailable: ${err.message}` };
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
function toRepoRelative(filePath) {
|
|
177
|
+
let p = filePath.replace(/\\/g, "/");
|
|
178
|
+
const root = WORKTREE.replace(/\\/g, "/").toLowerCase();
|
|
179
|
+
const idx = p.toLowerCase().indexOf(root);
|
|
180
|
+
if (idx >= 0)
|
|
181
|
+
p = p.slice(idx + root.length);
|
|
182
|
+
return p.replace(/^\/+/, "").toLowerCase();
|
|
183
|
+
}
|
|
184
|
+
function evaluateResults(label, results) {
|
|
185
|
+
const expected = new Set(label.files.map((f) => f.toLowerCase()));
|
|
186
|
+
const fileRank = new Map();
|
|
187
|
+
for (const [i, r] of results.entries()) {
|
|
188
|
+
const file = toRepoRelative(r.chunk.metadata.filePath);
|
|
189
|
+
if (!fileRank.has(file))
|
|
190
|
+
fileRank.set(file, i + 1);
|
|
191
|
+
}
|
|
192
|
+
let firstHitRank = null;
|
|
193
|
+
for (const [file, rank] of fileRank) {
|
|
194
|
+
if (!expected.has(file))
|
|
195
|
+
continue;
|
|
196
|
+
firstHitRank = firstHitRank === null ? rank : Math.min(firstHitRank, rank);
|
|
197
|
+
}
|
|
198
|
+
const foundInK = (k) => {
|
|
199
|
+
let found = 0;
|
|
200
|
+
for (const file of expected) {
|
|
201
|
+
const rank = fileRank.get(file);
|
|
202
|
+
if (rank !== undefined && rank <= k)
|
|
203
|
+
found += 1;
|
|
204
|
+
}
|
|
205
|
+
return found;
|
|
206
|
+
};
|
|
207
|
+
const hitAt = (k) => (foundInK(k) > 0 ? 1 : 0);
|
|
208
|
+
// nDCG@10 with binary gains, deduplicated per expected file.
|
|
209
|
+
let dcg = 0;
|
|
210
|
+
const gained = new Set();
|
|
211
|
+
for (const [i, r] of results.slice(0, 10).entries()) {
|
|
212
|
+
const file = toRepoRelative(r.chunk.metadata.filePath);
|
|
213
|
+
if (expected.has(file) && !gained.has(file)) {
|
|
214
|
+
gained.add(file);
|
|
215
|
+
dcg += 1 / Math.log2(i + 2);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
const idealCount = Math.min(expected.size, 10);
|
|
219
|
+
let idcg = 0;
|
|
220
|
+
for (let i = 0; i < idealCount; i++)
|
|
221
|
+
idcg += 1 / Math.log2(i + 2);
|
|
222
|
+
const ndcg10 = idcg > 0 ? dcg / idcg : 0;
|
|
223
|
+
let symbolHitRank = null;
|
|
224
|
+
if (label.symbol) {
|
|
225
|
+
for (const [i, r] of results.entries()) {
|
|
226
|
+
const file = toRepoRelative(r.chunk.metadata.filePath);
|
|
227
|
+
if (expected.has(file) && r.chunk.content.includes(label.symbol)) {
|
|
228
|
+
symbolHitRank = i + 1;
|
|
229
|
+
break;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
// Did the rerank stage actually reorder this query? The stage sets
|
|
234
|
+
// explanation.scoreBreakdown.rerankScore only on a successful rerank; when it
|
|
235
|
+
// degrades (timeout/error/cooldown) the fusion order is kept and the field is absent.
|
|
236
|
+
const rerankRan = results.some((r) => {
|
|
237
|
+
const breakdown = r.explanation?.scoreBreakdown;
|
|
238
|
+
return typeof breakdown?.rerankScore === "number";
|
|
239
|
+
});
|
|
240
|
+
return {
|
|
241
|
+
firstHitRank,
|
|
242
|
+
hitAt1: hitAt(1),
|
|
243
|
+
hitAt3: hitAt(3),
|
|
244
|
+
hitAt5: hitAt(5),
|
|
245
|
+
hitAt10: hitAt(10),
|
|
246
|
+
precision5: foundInK(5) / 5,
|
|
247
|
+
recall10: foundInK(10) / Math.max(expected.size, 1),
|
|
248
|
+
rr: firstHitRank !== null ? 1 / firstHitRank : 0,
|
|
249
|
+
ndcg10,
|
|
250
|
+
symbolHitRank,
|
|
251
|
+
topFiles: results.slice(0, 5).map((r) => toRepoRelative(r.chunk.metadata.filePath)),
|
|
252
|
+
rerankRan,
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
function percentile(sorted, q) {
|
|
256
|
+
if (sorted.length === 0)
|
|
257
|
+
return 0;
|
|
258
|
+
const idx = Math.min(sorted.length - 1, Math.floor(q * (sorted.length - 1)));
|
|
259
|
+
return sorted[idx] ?? 0;
|
|
260
|
+
}
|
|
261
|
+
function mean(values) {
|
|
262
|
+
if (values.length === 0)
|
|
263
|
+
return 0;
|
|
264
|
+
return values.reduce((s, v) => s + v, 0) / values.length;
|
|
265
|
+
}
|
|
266
|
+
function summarize(outcomes) {
|
|
267
|
+
const latencies = outcomes.map((o) => o.latencyMs).sort((a, b) => a - b);
|
|
268
|
+
const withSymbol = outcomes.filter((o) => o.label.symbol);
|
|
269
|
+
return {
|
|
270
|
+
hit1: mean(outcomes.map((o) => o.hitAt1)),
|
|
271
|
+
hit3: mean(outcomes.map((o) => o.hitAt3)),
|
|
272
|
+
hit5: mean(outcomes.map((o) => o.hitAt5)),
|
|
273
|
+
hit10: mean(outcomes.map((o) => o.hitAt10)),
|
|
274
|
+
precision5: mean(outcomes.map((o) => o.precision5)),
|
|
275
|
+
recall10: mean(outcomes.map((o) => o.recall10)),
|
|
276
|
+
mrr: mean(outcomes.map((o) => o.rr)),
|
|
277
|
+
ndcg10: mean(outcomes.map((o) => o.ndcg10)),
|
|
278
|
+
symbolRate: withSymbol.length > 0 ? mean(withSymbol.map((o) => (o.symbolHitRank !== null ? 1 : 0))) : 0,
|
|
279
|
+
latencyAvg: mean(latencies),
|
|
280
|
+
latencyP50: percentile(latencies, 0.5),
|
|
281
|
+
latencyP95: percentile(latencies, 0.95),
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
// ────────────────────────────────────────────────────────────────────────────
|
|
285
|
+
// Run
|
|
286
|
+
// ────────────────────────────────────────────────────────────────────────────
|
|
287
|
+
async function runArm(spec, labels, runtime, baseOptions, reranker, rerankerError, warmupCount, rerankTimeoutMs) {
|
|
288
|
+
if (spec.reranked && reranker === undefined) {
|
|
289
|
+
return { spec, status: "skipped", skipReason: rerankerError ?? "no rerank provider", outcomes: [], rerankRanCount: 0 };
|
|
290
|
+
}
|
|
291
|
+
const options = { ...baseOptions };
|
|
292
|
+
if (spec.reranked) {
|
|
293
|
+
options.reranker = reranker;
|
|
294
|
+
options.reranking = {
|
|
295
|
+
...spec.overrides,
|
|
296
|
+
...(rerankTimeoutMs !== undefined ? { timeoutMs: rerankTimeoutMs } : {}),
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
// Discarded warmup queries so model loading/connection setup does not pollute
|
|
300
|
+
// p50/p95 — llama-swap cold starts measure 15-25s per model swap.
|
|
301
|
+
const warmupLabels = labels.slice(0, Math.max(0, Math.min(warmupCount, labels.length)));
|
|
302
|
+
for (const warmup of warmupLabels) {
|
|
303
|
+
await retrieveFn(warmup.query, runtime.embedder, runtime.store, options);
|
|
304
|
+
}
|
|
305
|
+
const outcomes = [];
|
|
306
|
+
for (const label of labels) {
|
|
307
|
+
const t0 = performance.now();
|
|
308
|
+
let results = [];
|
|
309
|
+
let failure;
|
|
310
|
+
try {
|
|
311
|
+
results = await retrieveFn(label.query, runtime.embedder, runtime.store, options);
|
|
312
|
+
}
|
|
313
|
+
catch (err) {
|
|
314
|
+
failure = err.message;
|
|
315
|
+
}
|
|
316
|
+
const latencyMs = performance.now() - t0;
|
|
317
|
+
const metrics = evaluateResults(label, results);
|
|
318
|
+
outcomes.push({ label, latencyMs, ...metrics });
|
|
319
|
+
const rankText = metrics.firstHitRank !== null ? `rank ${metrics.firstHitRank}` : "miss";
|
|
320
|
+
console.log(` ${armLabelPad(spec.name)} ${label.id.padEnd(10)} ${rankText.padEnd(8)} ${latencyMs.toFixed(0)}ms` +
|
|
321
|
+
(failure ? ` ERROR: ${failure}` : ""));
|
|
322
|
+
}
|
|
323
|
+
const rerankRanCount = outcomes.filter((o) => o.rerankRan).length;
|
|
324
|
+
return { spec, status: "ok", outcomes, rerankRanCount };
|
|
325
|
+
}
|
|
326
|
+
function armLabelPad(name) {
|
|
327
|
+
return name.padEnd(12);
|
|
328
|
+
}
|
|
329
|
+
function fmtPct(value) {
|
|
330
|
+
return `${(value * 100).toFixed(1)}%`;
|
|
331
|
+
}
|
|
332
|
+
/** Top-5 file overlap between two arms (Jaccard-ish: share of baseline files still present). */
|
|
333
|
+
function top5Overlap(baseline, other) {
|
|
334
|
+
if (baseline.topFiles.length === 0)
|
|
335
|
+
return 1;
|
|
336
|
+
const otherSet = new Set(other.topFiles);
|
|
337
|
+
let shared = 0;
|
|
338
|
+
for (const f of baseline.topFiles)
|
|
339
|
+
if (otherSet.has(f))
|
|
340
|
+
shared += 1;
|
|
341
|
+
return shared / baseline.topFiles.length;
|
|
342
|
+
}
|
|
343
|
+
async function main() {
|
|
344
|
+
const opts = parseOptions(process.argv.slice(2));
|
|
345
|
+
const labels = loadLabels(opts.labelsPath)
|
|
346
|
+
.filter((l) => (opts.category ? l.category === opts.category : true))
|
|
347
|
+
.slice(0, opts.limit ?? Number.MAX_SAFE_INTEGER);
|
|
348
|
+
if (labels.length === 0) {
|
|
349
|
+
throw new Error("No labels selected (check --category/--limit).");
|
|
350
|
+
}
|
|
351
|
+
const runtime = await bootstrap(opts);
|
|
352
|
+
const { cfg } = runtime;
|
|
353
|
+
const topK = opts.topK ?? cfg.retrieval.topK ?? 10;
|
|
354
|
+
const minScore = opts.minScore ?? cfg.retrieval.minScore ?? 0;
|
|
355
|
+
const hybrid = cfg.retrieval.hybridSearch;
|
|
356
|
+
const baseOptions = {
|
|
357
|
+
topK,
|
|
358
|
+
minScore,
|
|
359
|
+
keywordIndex: runtime.keywordIndex,
|
|
360
|
+
keywordWeight: hybrid?.keywordWeight,
|
|
361
|
+
symbolKeywordWeight: hybrid?.symbolKeywordWeight,
|
|
362
|
+
docKeywordDemotion: hybrid?.docKeywordDemotion,
|
|
363
|
+
testKeywordDemotion: hybrid?.testKeywordDemotion,
|
|
364
|
+
hybridEnabled: hybrid?.enabled,
|
|
365
|
+
queryPrefix: cfg.embedding.queryPrefix,
|
|
366
|
+
explain: true,
|
|
367
|
+
};
|
|
368
|
+
const requested = ARM_SPECS.filter((a) => opts.arms.includes(a.name));
|
|
369
|
+
const unknownArms = opts.arms.filter((a) => !ARM_SPECS.some((s) => s.name === a));
|
|
370
|
+
for (const u of unknownArms)
|
|
371
|
+
console.warn(` ! Unknown arm ignored: ${u}`);
|
|
372
|
+
const needsReranker = requested.some((a) => a.reranked);
|
|
373
|
+
// CLI --rerank-* flags synthesize an enabled section so the A/B runs without
|
|
374
|
+
// touching the user's opencode-rag.json (which stays reranking.enabled=false).
|
|
375
|
+
const cliRerankOverrides = {};
|
|
376
|
+
if (opts.rerankBaseUrl)
|
|
377
|
+
cliRerankOverrides.baseUrl = opts.rerankBaseUrl;
|
|
378
|
+
if (opts.rerankModel)
|
|
379
|
+
cliRerankOverrides.model = opts.rerankModel;
|
|
380
|
+
if (opts.rerankApiKey)
|
|
381
|
+
cliRerankOverrides.apiKey = opts.rerankApiKey;
|
|
382
|
+
if (opts.rerankProvider)
|
|
383
|
+
cliRerankOverrides.provider = opts.rerankProvider;
|
|
384
|
+
const cfgReranking = cfg.reranking;
|
|
385
|
+
const rerankSection = Object.keys(cliRerankOverrides).length > 0
|
|
386
|
+
? { ...(cfgReranking ?? {}), ...cliRerankOverrides, enabled: true }
|
|
387
|
+
: cfgReranking;
|
|
388
|
+
const resolved = needsReranker
|
|
389
|
+
? await resolveRerankProvider(rerankSection)
|
|
390
|
+
: { provider: undefined, error: undefined };
|
|
391
|
+
console.log(`\n OpenCodeRAG Rerank Evaluation`);
|
|
392
|
+
console.log(` ─────────────────────────────────────────`);
|
|
393
|
+
console.log(` Indexed chunks: ${runtime.indexedCount}`);
|
|
394
|
+
console.log(` Embedder: ${cfg.embedding.provider}/${cfg.embedding.model} (dim ${runtime.dimension})`);
|
|
395
|
+
console.log(` Retrieval: topK=${topK}, minScore=${minScore}, hybrid=${hybrid?.enabled ?? false}`);
|
|
396
|
+
console.log(` Labels: ${labels.length} (${opts.labelsPath})`);
|
|
397
|
+
console.log(` Arms: ${requested.map((a) => a.name).join(", ")}`);
|
|
398
|
+
if (needsReranker && resolved.error) {
|
|
399
|
+
console.log(` Reranker: UNAVAILABLE — ${resolved.error}`);
|
|
400
|
+
}
|
|
401
|
+
console.log();
|
|
402
|
+
const armResults = [];
|
|
403
|
+
const warmupCount = opts.warmup ?? 2;
|
|
404
|
+
for (const spec of requested) {
|
|
405
|
+
console.log(` Arm "${spec.name}" — ${spec.description}`);
|
|
406
|
+
// eslint-disable-next-line no-await-in-loop
|
|
407
|
+
const result = await runArm(spec, labels, runtime, baseOptions, resolved.provider, resolved.error, warmupCount, opts.rerankTimeoutMs);
|
|
408
|
+
if (result.status === "skipped") {
|
|
409
|
+
console.log(` SKIPPED: ${result.skipReason}\n`);
|
|
410
|
+
}
|
|
411
|
+
else {
|
|
412
|
+
console.log();
|
|
413
|
+
}
|
|
414
|
+
armResults.push(result);
|
|
415
|
+
}
|
|
416
|
+
const okResults = armResults.filter((r) => r.status === "ok");
|
|
417
|
+
const baselineResult = okResults.find((r) => r.spec.name === "baseline");
|
|
418
|
+
const summaries = new Map();
|
|
419
|
+
for (const result of okResults)
|
|
420
|
+
summaries.set(result.spec.name, summarize(result.outcomes));
|
|
421
|
+
// ── Report ──────────────────────────────────────────────────────────────
|
|
422
|
+
const report = [];
|
|
423
|
+
report.push("# Rerank A/B Evaluation");
|
|
424
|
+
report.push("");
|
|
425
|
+
report.push(`**Date:** ${new Date().toISOString()}`);
|
|
426
|
+
report.push(`**Worktree:** ${WORKTREE}`);
|
|
427
|
+
report.push(`**Indexed chunks:** ${runtime.indexedCount}`);
|
|
428
|
+
report.push(`**Embedder:** ${cfg.embedding.provider}/${cfg.embedding.model} (dimension ${runtime.dimension})`);
|
|
429
|
+
report.push(`**Retrieval:** topK=${topK}, minScore=${minScore}, hybrid=${hybrid?.enabled ?? false}, keywordWeight=${hybrid?.keywordWeight ?? "-"}`);
|
|
430
|
+
report.push(`**Labels:** ${labels.length} from \`${path.relative(WORKTREE, opts.labelsPath).replace(/\\/g, "/")}\``);
|
|
431
|
+
const byCategory = new Map();
|
|
432
|
+
for (const l of labels)
|
|
433
|
+
byCategory.set(l.category, (byCategory.get(l.category) ?? 0) + 1);
|
|
434
|
+
report.push(`**Categories:** ${[...byCategory.entries()].map(([c, n]) => `${c}=${n}`).join(", ")}`);
|
|
435
|
+
report.push(`**Warmup:** ${warmupCount} discarded queries per arm (p50/p95 are warm measurements)`);
|
|
436
|
+
report.push("");
|
|
437
|
+
report.push("## Arm status");
|
|
438
|
+
report.push("");
|
|
439
|
+
report.push("| Arm | Status | Note |");
|
|
440
|
+
report.push("|-----|--------|------|");
|
|
441
|
+
for (const result of armResults) {
|
|
442
|
+
const note = result.status === "ok" ? result.spec.description : (result.skipReason ?? "");
|
|
443
|
+
report.push(`| \`${result.spec.name}\` | ${result.status === "ok" ? "ok" : "SKIPPED"} | ${note} |`);
|
|
444
|
+
}
|
|
445
|
+
if (needsReranker && resolved.error) {
|
|
446
|
+
report.push("");
|
|
447
|
+
report.push(`> Rerank provider unavailable: ${resolved.error}`);
|
|
448
|
+
report.push(">");
|
|
449
|
+
report.push("> To enable the rerank arms: set `reranking.enabled=true` in the config and start a llama-server");
|
|
450
|
+
report.push("> with `--reranking --pooling rank` serving a verified Qwen3-Reranker GGUF (see AGENTS.md quirk notes).");
|
|
451
|
+
}
|
|
452
|
+
report.push("");
|
|
453
|
+
if (okResults.length > 0) {
|
|
454
|
+
report.push("## Aggregate metrics (warm)");
|
|
455
|
+
report.push("");
|
|
456
|
+
report.push("| Metric | " + okResults.map((r) => `\`${r.spec.name}\``).join(" | ") + " |");
|
|
457
|
+
report.push("|--------|" + okResults.map(() => "---").join("|") + "|");
|
|
458
|
+
const metricRows = [
|
|
459
|
+
["Hit@1", (s) => fmtPct(s.hit1)],
|
|
460
|
+
["Hit@3", (s) => fmtPct(s.hit3)],
|
|
461
|
+
["Hit@5", (s) => fmtPct(s.hit5)],
|
|
462
|
+
["Hit@10", (s) => fmtPct(s.hit10)],
|
|
463
|
+
["Precision@5", (s) => s.precision5.toFixed(3)],
|
|
464
|
+
["Recall@10", (s) => fmtPct(s.recall10)],
|
|
465
|
+
["MRR", (s) => s.mrr.toFixed(3)],
|
|
466
|
+
["nDCG@10", (s) => s.ndcg10.toFixed(3)],
|
|
467
|
+
["Symbol hit rate", (s) => (s.symbolRate > 0 ? fmtPct(s.symbolRate) : "-")],
|
|
468
|
+
["Latency avg", (s) => `${s.latencyAvg.toFixed(0)} ms`],
|
|
469
|
+
["Latency p50", (s) => `${s.latencyP50.toFixed(0)} ms`],
|
|
470
|
+
["Latency p95", (s) => `${s.latencyP95.toFixed(0)} ms`],
|
|
471
|
+
];
|
|
472
|
+
for (const [name, fn] of metricRows) {
|
|
473
|
+
report.push(`| ${name} | ${okResults.map((r) => fn(summaries.get(r.spec.name))).join(" | ")} |`);
|
|
474
|
+
}
|
|
475
|
+
report.push(`| Reranked queries | ${okResults.map((r) => `${r.rerankRanCount}/${r.outcomes.length}`).join(" | ")} |`);
|
|
476
|
+
report.push("");
|
|
477
|
+
if (baselineResult) {
|
|
478
|
+
const others = okResults.filter((r) => r !== baselineResult);
|
|
479
|
+
if (others.length > 0) {
|
|
480
|
+
const baselineSummary = summaries.get("baseline");
|
|
481
|
+
report.push("## Effect vs. baseline");
|
|
482
|
+
report.push("");
|
|
483
|
+
report.push("| Arm | Δ nDCG@10 | Δ MRR | Δ Hit@5 | Δ p50 latency | Top-5 overlap | Changed top-1 |");
|
|
484
|
+
report.push("|-----|-----------|-------|---------|---------------|---------------|---------------|");
|
|
485
|
+
for (const other of others) {
|
|
486
|
+
const s = summaries.get(other.spec.name);
|
|
487
|
+
let changedTop1 = 0;
|
|
488
|
+
let overlapSum = 0;
|
|
489
|
+
const baselineByLabel = new Map(baselineResult.outcomes.map((o) => [o.label.id, o]));
|
|
490
|
+
for (const o of other.outcomes) {
|
|
491
|
+
const b = baselineByLabel.get(o.label.id);
|
|
492
|
+
if (!b)
|
|
493
|
+
continue;
|
|
494
|
+
if (b.topFiles[0] !== o.topFiles[0])
|
|
495
|
+
changedTop1 += 1;
|
|
496
|
+
overlapSum += top5Overlap(b, o);
|
|
497
|
+
}
|
|
498
|
+
const overlap = other.outcomes.length > 0 ? overlapSum / other.outcomes.length : 0;
|
|
499
|
+
report.push(`| \`${other.spec.name}\` | ${(s.ndcg10 - baselineSummary.ndcg10 >= 0 ? "+" : "") + (s.ndcg10 - baselineSummary.ndcg10).toFixed(3)} | ` +
|
|
500
|
+
`${(s.mrr - baselineSummary.mrr >= 0 ? "+" : "") + (s.mrr - baselineSummary.mrr).toFixed(3)} | ` +
|
|
501
|
+
`${(s.hit5 - baselineSummary.hit5 >= 0 ? "+" : "") + (s.hit5 - baselineSummary.hit5).toFixed(3)} | ` +
|
|
502
|
+
`${(s.latencyP50 - baselineSummary.latencyP50 >= 0 ? "+" : "") + (s.latencyP50 - baselineSummary.latencyP50).toFixed(0)} ms | ` +
|
|
503
|
+
`${fmtPct(overlap)} | ${changedTop1}/${other.outcomes.length} |`);
|
|
504
|
+
}
|
|
505
|
+
report.push("");
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
report.push("## Metrics by category");
|
|
509
|
+
report.push("");
|
|
510
|
+
const categories = [...new Set(labels.map((l) => l.category))];
|
|
511
|
+
report.push("| Arm | Category | Hit@1 | Hit@5 | MRR | nDCG@10 |");
|
|
512
|
+
report.push("|-----|----------|-------|-------|-----|---------|");
|
|
513
|
+
for (const result of okResults) {
|
|
514
|
+
for (const category of categories) {
|
|
515
|
+
const catOutcomes = result.outcomes.filter((o) => o.label.category === category);
|
|
516
|
+
if (catOutcomes.length === 0)
|
|
517
|
+
continue;
|
|
518
|
+
const s = summarize(catOutcomes);
|
|
519
|
+
report.push(`| \`${result.spec.name}\` | ${category} | ${fmtPct(s.hit1)} | ${fmtPct(s.hit5)} | ${s.mrr.toFixed(3)} | ${s.ndcg10.toFixed(3)} |`);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
report.push("");
|
|
523
|
+
report.push("## Per-query results");
|
|
524
|
+
report.push("");
|
|
525
|
+
const armHeader = okResults.map((r) => `\`${r.spec.name}\``).join(" | ");
|
|
526
|
+
report.push(`| ID | Category | Query | Expected files | ${armHeader} |`);
|
|
527
|
+
report.push("|----|----------|-------|----------------|" + okResults.map(() => "---").join("|") + "|");
|
|
528
|
+
for (const label of labels) {
|
|
529
|
+
const cells = okResults.map((r) => {
|
|
530
|
+
const o = r.outcomes.find((x) => x.label.id === label.id);
|
|
531
|
+
if (!o)
|
|
532
|
+
return "-";
|
|
533
|
+
const hit = o.firstHitRank !== null ? `#${o.firstHitRank}` : "miss";
|
|
534
|
+
const sym = label.symbol ? (o.symbolHitRank !== null ? ` (sym #${o.symbolHitRank})` : " (sym miss)") : "";
|
|
535
|
+
return hit + sym;
|
|
536
|
+
});
|
|
537
|
+
const q = label.query.length > 52 ? label.query.substring(0, 49) + "..." : label.query;
|
|
538
|
+
report.push(`| ${label.id} | ${label.category} | ${q} | ${label.files.join(", ")} | ${cells.join(" | ")} |`);
|
|
539
|
+
}
|
|
540
|
+
report.push("");
|
|
541
|
+
report.push("## Interpretation notes");
|
|
542
|
+
report.push("");
|
|
543
|
+
report.push("- **Rank shown** is the position of the first expected file (1-based); `sym` marks the rank of a chunk");
|
|
544
|
+
report.push(" that also contains the expected symbol.");
|
|
545
|
+
report.push("- **Reranked queries** counts queries where the rerank stage actually reordered (`rerankScore` present in");
|
|
546
|
+
report.push(" the explanation). Fewer than N means the provider timed out/errored or the cooldown was active; those");
|
|
547
|
+
report.push(" queries measured the fusion order, so rerank deltas are conservative.");
|
|
548
|
+
report.push("- **Misses are not necessarily retrieval failures**: a label may map to multiple files and the metric");
|
|
549
|
+
report.push(" counts every expected file; check the per-query table before drawing conclusions.");
|
|
550
|
+
report.push("- **Rerank arms are reorder-only** (`reranking.minScore=0`): the candidate set is unchanged, only the order");
|
|
551
|
+
report.push(" within the top-K differs. A reranker that only churns ranks without improving nDCG@10/MRR is not worth");
|
|
552
|
+
report.push(" the latency cost — that is the decision this report informs.");
|
|
553
|
+
report.push("- Latency was measured warm on a local llama-swap tunnel; p95 includes network jitter. Cold start is not");
|
|
554
|
+
report.push(" included and must be measured separately before enabling the stage by default.");
|
|
555
|
+
report.push("");
|
|
556
|
+
}
|
|
557
|
+
writeFileSync(opts.reportPath, report.join("\n"), "utf-8");
|
|
558
|
+
// ── Console summary ─────────────────────────────────────────────────────
|
|
559
|
+
const sep = "─".repeat(78);
|
|
560
|
+
console.log(sep);
|
|
561
|
+
console.log(" SUMMARY (warm)");
|
|
562
|
+
console.log(sep);
|
|
563
|
+
console.log(" Arm Hit@1 Hit@5 Recall@10 MRR nDCG@10 P@5 p50 p95");
|
|
564
|
+
for (const result of okResults) {
|
|
565
|
+
const s = summaries.get(result.spec.name);
|
|
566
|
+
console.log(` ${result.spec.name.padEnd(12)} ${fmtPct(s.hit1).padStart(5)} ${fmtPct(s.hit5).padStart(5)} ` +
|
|
567
|
+
`${fmtPct(s.recall10).padStart(8)} ${s.mrr.toFixed(3)} ${s.ndcg10.toFixed(3).padStart(7)} ` +
|
|
568
|
+
`${s.precision5.toFixed(3)} ${s.latencyP50.toFixed(0).padStart(4)}ms ${s.latencyP95.toFixed(0).padStart(4)}ms`);
|
|
569
|
+
}
|
|
570
|
+
for (const result of armResults.filter((r) => r.status === "skipped")) {
|
|
571
|
+
console.log(` ${result.spec.name.padEnd(12)} SKIPPED — ${result.skipReason}`);
|
|
572
|
+
}
|
|
573
|
+
console.log(sep);
|
|
574
|
+
console.log(` Report written to: ${opts.reportPath}`);
|
|
575
|
+
console.log();
|
|
576
|
+
}
|
|
577
|
+
main().catch((err) => {
|
|
578
|
+
console.error("\n Rerank evaluation failed:", err.message);
|
|
579
|
+
process.exitCode = 1;
|
|
580
|
+
});
|
|
581
|
+
//# sourceMappingURL=rerank-eval.js.map
|
|
@@ -61,6 +61,23 @@ export interface Logger {
|
|
|
61
61
|
* @returns Aggregate statistics for the pass.
|
|
62
62
|
*/
|
|
63
63
|
export declare function runIndexPass(options: RunIndexPassOptions): Promise<IndexRunStats>;
|
|
64
|
+
/**
|
|
65
|
+
* Warning text for description settings that leave chain-of-thought enabled,
|
|
66
|
+
* or `null` when thinking is safely disabled.
|
|
67
|
+
*
|
|
68
|
+
* Ollama requests default to thinking off. OpenAI-compatible providers inherit
|
|
69
|
+
* whatever the served model does, and reasoning models (Qwen3.x etc.) then
|
|
70
|
+
* spend thousands of extra tokens per chunk, making indexing ~50x slower.
|
|
71
|
+
* `think: false` forwards `chat_template_kwargs.enable_thinking=false` for
|
|
72
|
+
* those providers (see LlmDescriptionProvider).
|
|
73
|
+
*
|
|
74
|
+
* @param description - Description config slice (`think`, `provider`).
|
|
75
|
+
* @returns Human-readable warning, or `null` when no warning is needed.
|
|
76
|
+
*/
|
|
77
|
+
export declare function getDescriptionThinkingWarning(description?: {
|
|
78
|
+
think?: boolean;
|
|
79
|
+
provider?: string;
|
|
80
|
+
}): string | null;
|
|
64
81
|
/**
|
|
65
82
|
* Build an overview of the current index health without running a full pass.
|
|
66
83
|
* Scans workspace files and compares their hashes against the manifest to
|