opencode-rag-plugin 2.3.1 → 2.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/ReadMe.md +71 -8
  2. package/dist/chunker/skeleton.js +1 -1
  3. package/dist/cli/commands/decide.d.ts +25 -0
  4. package/dist/cli/commands/decide.js +160 -0
  5. package/dist/cli/commands/eval.d.ts +23 -0
  6. package/dist/cli/commands/eval.js +331 -1
  7. package/dist/cli/commands/index-command.js +28 -10
  8. package/dist/cli/commands/index.d.ts +2 -1
  9. package/dist/cli/commands/index.js +2 -1
  10. package/dist/cli/commands/init-helpers.d.ts +16 -1
  11. package/dist/cli/commands/init-helpers.js +84 -2
  12. package/dist/cli/commands/init.js +29 -4
  13. package/dist/cli/commands/status.js +14 -8
  14. package/dist/cli/format.d.ts +50 -0
  15. package/dist/cli/format.js +122 -0
  16. package/dist/cli/index.js +4 -1
  17. package/dist/cli/types.d.ts +2 -0
  18. package/dist/core/bootstrap.js +1 -0
  19. package/dist/core/config.d.ts +47 -0
  20. package/dist/core/config.js +34 -1
  21. package/dist/core/interfaces.d.ts +69 -0
  22. package/dist/decision/factory.d.ts +17 -0
  23. package/dist/decision/factory.js +17 -0
  24. package/dist/decision/format.d.ts +14 -0
  25. package/dist/decision/format.js +53 -0
  26. package/dist/decision/systemone.d.ts +76 -0
  27. package/dist/decision/systemone.js +234 -0
  28. package/dist/decision/validate.d.ts +26 -0
  29. package/dist/decision/validate.js +74 -0
  30. package/dist/eval/decide-calibration.d.ts +112 -0
  31. package/dist/eval/decide-calibration.js +185 -0
  32. package/dist/eval/golden-gate.d.ts +65 -0
  33. package/dist/eval/golden-gate.js +61 -0
  34. package/dist/eval/golden-set.d.ts +81 -0
  35. package/dist/eval/golden-set.js +155 -0
  36. package/dist/eval/rerank-eval.js +10 -125
  37. package/dist/index.d.ts +6 -2
  38. package/dist/index.js +4 -0
  39. package/dist/indexer/pipeline.d.ts +0 -10
  40. package/dist/indexer/pipeline.js +1 -0
  41. package/dist/mcp/handlers.d.ts +17 -1
  42. package/dist/mcp/handlers.js +30 -0
  43. package/dist/mcp/server.d.ts +9 -1
  44. package/dist/mcp/server.js +47 -3
  45. package/dist/opencode/system-guidance.d.ts +11 -0
  46. package/dist/opencode/system-guidance.js +53 -0
  47. package/dist/opencode/tools.d.ts +24 -1
  48. package/dist/opencode/tools.js +101 -0
  49. package/dist/plugin.js +21 -3
  50. package/dist/quirks/quirk-store.d.ts +2 -0
  51. package/dist/quirks/quirk-store.js +33 -4
  52. package/dist/tui.js +21 -2
  53. package/dist/v2/v2-adapter.d.ts +20 -0
  54. package/dist/v2/v2-adapter.js +36 -8
  55. package/dist/vectorstore/lancedb.js +54 -5
  56. package/dist/watcher.d.ts +26 -1
  57. package/dist/watcher.js +71 -9
  58. package/dist/web/ui/assets/index-CWcp0jNf.css +2 -0
  59. package/dist/web/ui/index.html +2 -2
  60. package/package.json +5 -2
  61. package/dist/web/ui/assets/index-Dp009QCQ.css +0 -1
  62. /package/dist/web/ui/assets/{index-C0QXKA-b.js → index-DbXFrn53.js} +0 -0
@@ -0,0 +1,185 @@
1
+ /**
2
+ * @fileoverview Decision-model calibration harness (tev1 via Ollama `/v1/systemone`).
3
+ *
4
+ * Answers a small labelled set of `noul` / `choice` / `score` questions, compares the
5
+ * model's answers with the expected labels, and reports **accuracy, ECE (expected
6
+ * calibration error, 10 bins) and Brier score** over the predicted answer's confidence.
7
+ * Motivation (XERJ's model-card discipline): our docs say `confidence` is probability
8
+ * concentration, NOT the chance the answer is right — this harness measures how far off
9
+ * that distinction is in practice. `eval:decide` is the CLI entry.
10
+ */
11
+ import { existsSync, readFileSync } from "node:fs";
12
+ /**
13
+ * Load and validate a decision calibration label file.
14
+ *
15
+ * @param labelsPath - Absolute path to the JSON label file.
16
+ * @returns The validated items.
17
+ * @throws When the file is missing, has no labels, or an item lacks id/state/question/expected.
18
+ */
19
+ export function loadDecideCalibration(labelsPath) {
20
+ if (!existsSync(labelsPath)) {
21
+ throw new Error(`Decision label file not found: ${labelsPath}`);
22
+ }
23
+ const parsed = JSON.parse(readFileSync(labelsPath, "utf-8"));
24
+ if (!Array.isArray(parsed.labels) || parsed.labels.length === 0) {
25
+ throw new Error(`Decision label file has no labels: ${labelsPath}`);
26
+ }
27
+ for (const item of parsed.labels) {
28
+ if (!item.id || !item.state || !item.question?.type || !item.question?.instructions || !item.expected) {
29
+ throw new Error(`Malformed decision label (needs id, state, question, expected): ${JSON.stringify(item)}`);
30
+ }
31
+ }
32
+ return parsed.labels;
33
+ }
34
+ /** Argmax over a probability record (stable for ties: first key wins). */
35
+ function argmaxRecord(probs) {
36
+ let best;
37
+ let bestValue = Number.NEGATIVE_INFINITY;
38
+ for (const [key, value] of Object.entries(probs)) {
39
+ if (value > bestValue) {
40
+ best = key;
41
+ bestValue = value;
42
+ }
43
+ }
44
+ return best;
45
+ }
46
+ /**
47
+ * Score one model answer against the expected label.
48
+ *
49
+ * Pure function — no provider needed, which is what makes the math unit-testable.
50
+ *
51
+ * @param item - The labelled item.
52
+ * @param answer - The model's answer (may be undefined when the provider skipped it).
53
+ * @param latencyMs - Measured call latency.
54
+ * @returns Prediction, correctness, prediction confidence, expected-class probability.
55
+ */
56
+ export function scoreDecisionAnswer(item, answer, latencyMs) {
57
+ const base = { item, latencyMs };
58
+ if (!answer) {
59
+ return { ...base, predicted: undefined, correct: false, confidence: 0, pExpected: null };
60
+ }
61
+ if (item.question.type === "noul") {
62
+ const pTrue = answer.noul;
63
+ if (typeof pTrue !== "number" || !Number.isFinite(pTrue)) {
64
+ return { ...base, predicted: undefined, correct: false, confidence: 0, pExpected: null };
65
+ }
66
+ const predictedBool = pTrue >= 0.5;
67
+ const predicted = predictedBool ? "true" : "false";
68
+ const expectedBool = item.expected.trim().toLowerCase() === "true";
69
+ return {
70
+ ...base,
71
+ predicted,
72
+ correct: predictedBool === expectedBool,
73
+ confidence: predictedBool ? pTrue : 1 - pTrue,
74
+ pExpected: expectedBool ? pTrue : 1 - pTrue,
75
+ };
76
+ }
77
+ // choice / score: prefer the explicit choice, else argmax of the probability map.
78
+ const probs = answer.probabilities;
79
+ const probRecord = probs !== null && typeof probs === "object" && !Array.isArray(probs)
80
+ ? probs
81
+ : undefined;
82
+ let predicted = answer.choice;
83
+ if (item.question.type === "score") {
84
+ const levels = answer.legend ?? (Array.isArray(item.question.criteria) ? item.question.criteria : undefined);
85
+ if (typeof answer.score === "number" && levels && levels.length > 0) {
86
+ const idx = Math.min(levels.length - 1, Math.max(0, Math.round(answer.score)));
87
+ predicted = levels[idx];
88
+ }
89
+ }
90
+ if (!predicted && probRecord) {
91
+ predicted = argmaxRecord(probRecord);
92
+ }
93
+ const predictedProbability = predicted !== undefined && probRecord ? probRecord[predicted] : undefined;
94
+ const confidence = typeof predictedProbability === "number"
95
+ ? predictedProbability
96
+ : (typeof answer.confidence === "number" ? answer.confidence : 0.5);
97
+ const expectedProbability = probRecord ? probRecord[item.expected] : undefined;
98
+ const pExpected = typeof expectedProbability === "number" ? expectedProbability : null;
99
+ return {
100
+ ...base,
101
+ predicted,
102
+ correct: predicted !== undefined && predicted === item.expected,
103
+ confidence,
104
+ pExpected,
105
+ };
106
+ }
107
+ /**
108
+ * Aggregate scored items into accuracy / ECE / Brier.
109
+ *
110
+ * @param items - Scored items (unanswered ones count against accuracy but not into bins).
111
+ * @param binCount - Number of equal-width confidence bins (default 10).
112
+ * @returns The calibration summary including the reliability bins.
113
+ */
114
+ export function summarizeCalibration(items, binCount = 10) {
115
+ const n = items.length;
116
+ const answeredItems = items.filter((item) => item.predicted !== undefined);
117
+ const accuracy = n > 0 ? items.filter((item) => item.correct).length / n : 0;
118
+ const bins = [];
119
+ for (let i = 0; i < binCount; i++) {
120
+ const lo = i / binCount;
121
+ const hi = (i + 1) / binCount;
122
+ const inBin = answeredItems.filter((item) => i === binCount - 1 ? item.confidence >= lo && item.confidence <= hi : item.confidence >= lo && item.confidence < hi);
123
+ bins.push({
124
+ lo,
125
+ hi,
126
+ count: inBin.length,
127
+ confidence: inBin.length > 0 ? inBin.reduce((s, item) => s + item.confidence, 0) / inBin.length : 0,
128
+ accuracy: inBin.length > 0 ? inBin.filter((item) => item.correct).length / inBin.length : 0,
129
+ });
130
+ }
131
+ let ece = 0;
132
+ for (const bin of bins) {
133
+ if (bin.count === 0)
134
+ continue;
135
+ ece += (bin.count / Math.max(answeredItems.length, 1)) * Math.abs(bin.accuracy - bin.confidence);
136
+ }
137
+ const brier = answeredItems.length > 0
138
+ ? answeredItems.reduce((s, item) => s + (item.confidence - (item.correct ? 1 : 0)) ** 2, 0) / answeredItems.length
139
+ : 0;
140
+ const meanConfidence = answeredItems.length > 0
141
+ ? answeredItems.reduce((s, item) => s + item.confidence, 0) / answeredItems.length
142
+ : 0;
143
+ const expectedProbs = items.map((item) => item.pExpected).filter((p) => p !== null);
144
+ const meanPExpected = expectedProbs.length > 0
145
+ ? expectedProbs.reduce((s, p) => s + p, 0) / expectedProbs.length
146
+ : null;
147
+ return {
148
+ n,
149
+ answered: answeredItems.length,
150
+ accuracy,
151
+ ece,
152
+ brier,
153
+ meanConfidence,
154
+ meanPExpected,
155
+ bins,
156
+ };
157
+ }
158
+ /**
159
+ * Run every calibration item through the decision provider and score the answers.
160
+ *
161
+ * @param opts - Items, provider, logger.
162
+ * @returns Scored items and the aggregate summary.
163
+ * @throws When the provider fails for the very first item (e.g. Ollama down) — the
164
+ * caller should surface the provider error instead of a misleading 0% accuracy.
165
+ */
166
+ export async function runDecideCalibration(opts) {
167
+ const log = opts.log ?? ((line) => console.log(line));
168
+ const scored = [];
169
+ for (const item of opts.items) {
170
+ const t0 = performance.now();
171
+ const result = await opts.provider.decide({
172
+ state: item.state,
173
+ questions: [{ id: item.id, type: item.question.type, instructions: item.question.instructions, criteria: item.question.criteria }],
174
+ });
175
+ const latencyMs = performance.now() - t0;
176
+ const answer = result.answers.find((candidate) => candidate.id === item.id) ?? result.answers[0];
177
+ const outcome = scoreDecisionAnswer(item, answer, latencyMs);
178
+ scored.push(outcome);
179
+ const mark = outcome.predicted === undefined ? "NO ANSWER" : outcome.correct ? "ok" : "WRONG";
180
+ log(` ${item.id.padEnd(16)} ${String(outcome.predicted ?? "-").padEnd(10)} conf ${outcome.confidence.toFixed(2)} ` +
181
+ `(${mark}, ${latencyMs.toFixed(0)}ms)`);
182
+ }
183
+ return { scored, summary: summarizeCalibration(scored) };
184
+ }
185
+ //# sourceMappingURL=decide-calibration.js.map
@@ -0,0 +1,65 @@
1
+ /**
2
+ * @fileoverview Retrieval golden-set gate. Runs the labelled queries through the real
3
+ * retrieval pipeline (fusion only — the rerank stage stays out of the measurement) and
4
+ * fails when headline ranking metrics drop below documented floors. Meant as a release
5
+ * check with a live embedder and an indexed workspace; `eval:gate` is the CLI entry.
6
+ *
7
+ * Motivation (XERJ's "link the harness or drop the number"): a documented metric is only
8
+ * trustworthy when a command reproduces it and a gate notices when it regresses.
9
+ */
10
+ import type { SearchResult } from "../core/interfaces.js";
11
+ import { type GoldenLabel, type GoldenOutcome, type GoldenSummary } from "./golden-set.js";
12
+ /** Minimum metric floors the gate checks (values in [0,1]; undefined = not checked). */
13
+ export interface GateThresholds {
14
+ /** Minimum mean Hit@5. */
15
+ hit5?: number;
16
+ /** Minimum mean reciprocal rank. */
17
+ mrr?: number;
18
+ /** Minimum mean nDCG@10. */
19
+ ndcg10?: number;
20
+ }
21
+ /**
22
+ * Default floors, calibrated against the reference setup (Qwen3-Embedding:8B, 48 labels)
23
+ * with headroom below the measured fusion baseline (Hit@5 47.9%, MRR 0.339, nDCG@10 0.428
24
+ * on 2026-10-02) so a small wobble does not false-fail a release but a real regression does.
25
+ */
26
+ export declare const DEFAULT_GATE_THRESHOLDS: Required<GateThresholds>;
27
+ /** One checked metric row. */
28
+ export interface GateRow {
29
+ metric: string;
30
+ actual: number;
31
+ required: number;
32
+ pass: boolean;
33
+ }
34
+ /**
35
+ * Compare a summary against the thresholds.
36
+ *
37
+ * @param summary - Aggregated golden-set metrics.
38
+ * @param thresholds - Floors to check; omitted metrics are not gated.
39
+ * @returns `pass` plus one row per checked metric.
40
+ */
41
+ export declare function evaluateGate(summary: GoldenSummary, thresholds: GateThresholds): {
42
+ pass: boolean;
43
+ rows: GateRow[];
44
+ };
45
+ /** Options for {@link runGoldenGate}. */
46
+ export interface GoldenGateRunOptions {
47
+ /** Labels to run (already filtered). */
48
+ labels: GoldenLabel[];
49
+ /** Workspace root used to relativize result paths. */
50
+ root: string;
51
+ /** Retrieve closure built by the caller (owns embedder/store/config wiring). */
52
+ retrieve: (label: GoldenLabel) => Promise<SearchResult[]>;
53
+ /** Progress line callback (defaults to console.log). */
54
+ log?: (line: string) => void;
55
+ }
56
+ /**
57
+ * Run every label through the supplied retrieve closure and aggregate the metrics.
58
+ *
59
+ * @param opts - Labels, root, retrieve closure, logger.
60
+ * @returns The aggregate summary plus the per-query outcomes.
61
+ */
62
+ export declare function runGoldenGate(opts: GoldenGateRunOptions): Promise<{
63
+ summary: GoldenSummary;
64
+ outcomes: GoldenOutcome[];
65
+ }>;
@@ -0,0 +1,61 @@
1
+ import { evaluateGoldenResults, summarizeGoldenOutcomes, } from "./golden-set.js";
2
+ /**
3
+ * Default floors, calibrated against the reference setup (Qwen3-Embedding:8B, 48 labels)
4
+ * with headroom below the measured fusion baseline (Hit@5 47.9%, MRR 0.339, nDCG@10 0.428
5
+ * on 2026-10-02) so a small wobble does not false-fail a release but a real regression does.
6
+ */
7
+ export const DEFAULT_GATE_THRESHOLDS = {
8
+ hit5: 0.35,
9
+ mrr: 0.25,
10
+ ndcg10: 0.32,
11
+ };
12
+ /**
13
+ * Compare a summary against the thresholds.
14
+ *
15
+ * @param summary - Aggregated golden-set metrics.
16
+ * @param thresholds - Floors to check; omitted metrics are not gated.
17
+ * @returns `pass` plus one row per checked metric.
18
+ */
19
+ export function evaluateGate(summary, thresholds) {
20
+ const checks = [
21
+ ["Hit@5", summary.hit5, thresholds.hit5],
22
+ ["MRR", summary.mrr, thresholds.mrr],
23
+ ["nDCG@10", summary.ndcg10, thresholds.ndcg10],
24
+ ];
25
+ const rows = [];
26
+ for (const [metric, actual, required] of checks) {
27
+ if (required === undefined)
28
+ continue;
29
+ rows.push({ metric, actual, required, pass: actual + 1e-9 >= required });
30
+ }
31
+ return { pass: rows.every((row) => row.pass), rows };
32
+ }
33
+ /**
34
+ * Run every label through the supplied retrieve closure and aggregate the metrics.
35
+ *
36
+ * @param opts - Labels, root, retrieve closure, logger.
37
+ * @returns The aggregate summary plus the per-query outcomes.
38
+ */
39
+ export async function runGoldenGate(opts) {
40
+ const log = opts.log ?? ((line) => console.log(line));
41
+ const outcomes = [];
42
+ for (const label of opts.labels) {
43
+ const t0 = performance.now();
44
+ let results = [];
45
+ let failure;
46
+ try {
47
+ results = await opts.retrieve(label);
48
+ }
49
+ catch (err) {
50
+ failure = err.message;
51
+ }
52
+ const latencyMs = performance.now() - t0;
53
+ const metrics = evaluateGoldenResults(opts.root, label, results);
54
+ outcomes.push({ label, latencyMs, ...metrics });
55
+ const rankText = metrics.firstHitRank !== null ? `rank ${metrics.firstHitRank}` : "miss";
56
+ log(` ${label.id.padEnd(12)} ${rankText.padEnd(8)} ${latencyMs.toFixed(0)}ms` +
57
+ (failure ? ` ERROR: ${failure}` : ""));
58
+ }
59
+ return { summary: summarizeGoldenOutcomes(outcomes), outcomes };
60
+ }
61
+ //# sourceMappingURL=golden-gate.js.map
@@ -0,0 +1,81 @@
1
+ import type { SearchResult } from "../core/interfaces.js";
2
+ /** One labelled query in the golden set. */
3
+ export interface GoldenLabel {
4
+ id: string;
5
+ category: string;
6
+ query: string;
7
+ files: string[];
8
+ symbol?: string;
9
+ note?: string;
10
+ }
11
+ /** Serialized label file (`src/eval/rerank-labels.json`). */
12
+ export interface GoldenLabelFile {
13
+ version: number;
14
+ labels: GoldenLabel[];
15
+ }
16
+ /**
17
+ * Load and validate a golden-set label file.
18
+ *
19
+ * @param labelsPath - Absolute path to the JSON label file.
20
+ * @returns The validated labels.
21
+ * @throws When the file is missing, has no labels, or a label lacks id/query/files.
22
+ */
23
+ export declare function loadGoldenLabels(labelsPath: string): GoldenLabel[];
24
+ /** Make a result path relative to the workspace root, lower-cased for comparison. */
25
+ export declare function toRepoRelative(root: string, filePath: string): string;
26
+ /** Ranking metrics for one labelled query. */
27
+ export interface GoldenMetrics {
28
+ firstHitRank: number | null;
29
+ hitAt1: number;
30
+ hitAt3: number;
31
+ hitAt5: number;
32
+ hitAt10: number;
33
+ precision5: number;
34
+ recall10: number;
35
+ rr: number;
36
+ ndcg10: number;
37
+ symbolHitRank: number | null;
38
+ topFiles: string[];
39
+ /** True when the rerank stage actually reordered (scoreBreakdown.rerankScore present). */
40
+ rerankRan: boolean;
41
+ }
42
+ /**
43
+ * Score one label against a ranked result list.
44
+ *
45
+ * @param root - Workspace root used to relativize result paths.
46
+ * @param label - The labelled query (expected files + optional symbol).
47
+ * @param results - Ranked retrieval results.
48
+ * @returns Hit@1/3/5/10, Precision@5, Recall@10, reciprocal rank, nDCG@10, symbol rank.
49
+ */
50
+ export declare function evaluateGoldenResults(root: string, label: GoldenLabel, results: SearchResult[]): GoldenMetrics;
51
+ /** Percentile from a pre-sorted ascending array (`q` in [0,1]). */
52
+ export declare function percentile(sorted: number[], q: number): number;
53
+ /** Arithmetic mean; 0 for an empty list. */
54
+ export declare function mean(values: number[]): number;
55
+ /** One scored query: metrics plus the label and the measured latency. */
56
+ export interface GoldenOutcome extends GoldenMetrics {
57
+ label: GoldenLabel;
58
+ latencyMs: number;
59
+ }
60
+ /** Aggregated ranking metrics over a set of scored queries. */
61
+ export interface GoldenSummary {
62
+ hit1: number;
63
+ hit3: number;
64
+ hit5: number;
65
+ hit10: number;
66
+ precision5: number;
67
+ recall10: number;
68
+ mrr: number;
69
+ ndcg10: number;
70
+ symbolRate: number;
71
+ latencyAvg: number;
72
+ latencyP50: number;
73
+ latencyP95: number;
74
+ }
75
+ /**
76
+ * Aggregate per-query metrics into mean Hit@k, MRR, nDCG@10, symbol rate, and latency.
77
+ *
78
+ * @param outcomes - Scored queries.
79
+ * @returns The aggregate summary.
80
+ */
81
+ export declare function summarizeGoldenOutcomes(outcomes: GoldenOutcome[]): GoldenSummary;
@@ -0,0 +1,155 @@
1
+ /**
2
+ * @fileoverview Shared golden-set types and ranking metrics. One implementation feeds
3
+ * both the rerank A/B harness (`src/eval/rerank-eval.ts`) and the retrieval gate
4
+ * (`eval:gate`), so the two tools can never report different Hit@k / MRR / nDCG@10
5
+ * numbers for the same labels.
6
+ *
7
+ * Ground truth is a list of queries mapped to **file paths** (plus an optional symbol) —
8
+ * never chunk IDs, which do not survive re-indexing.
9
+ */
10
+ import { existsSync, readFileSync } from "node:fs";
11
+ /**
12
+ * Load and validate a golden-set label file.
13
+ *
14
+ * @param labelsPath - Absolute path to the JSON label file.
15
+ * @returns The validated labels.
16
+ * @throws When the file is missing, has no labels, or a label lacks id/query/files.
17
+ */
18
+ export function loadGoldenLabels(labelsPath) {
19
+ if (!existsSync(labelsPath)) {
20
+ throw new Error(`Label file not found: ${labelsPath}`);
21
+ }
22
+ const parsed = JSON.parse(readFileSync(labelsPath, "utf-8"));
23
+ if (!Array.isArray(parsed.labels) || parsed.labels.length === 0) {
24
+ throw new Error(`Label file has no labels: ${labelsPath}`);
25
+ }
26
+ for (const label of parsed.labels) {
27
+ if (!label.id || !label.query || !Array.isArray(label.files) || label.files.length === 0) {
28
+ throw new Error(`Malformed label (needs id, query, files[]): ${JSON.stringify(label)}`);
29
+ }
30
+ }
31
+ return parsed.labels;
32
+ }
33
+ /** Make a result path relative to the workspace root, lower-cased for comparison. */
34
+ export function toRepoRelative(root, filePath) {
35
+ let p = filePath.replace(/\\/g, "/");
36
+ const normalizedRoot = root.replace(/\\/g, "/").toLowerCase();
37
+ const idx = p.toLowerCase().indexOf(normalizedRoot);
38
+ if (idx >= 0)
39
+ p = p.slice(idx + normalizedRoot.length);
40
+ return p.replace(/^\/+/, "").toLowerCase();
41
+ }
42
+ /**
43
+ * Score one label against a ranked result list.
44
+ *
45
+ * @param root - Workspace root used to relativize result paths.
46
+ * @param label - The labelled query (expected files + optional symbol).
47
+ * @param results - Ranked retrieval results.
48
+ * @returns Hit@1/3/5/10, Precision@5, Recall@10, reciprocal rank, nDCG@10, symbol rank.
49
+ */
50
+ export function evaluateGoldenResults(root, label, results) {
51
+ const expected = new Set(label.files.map((f) => f.toLowerCase()));
52
+ const fileRank = new Map();
53
+ for (const [i, r] of results.entries()) {
54
+ const file = toRepoRelative(root, r.chunk.metadata.filePath);
55
+ if (!fileRank.has(file))
56
+ fileRank.set(file, i + 1);
57
+ }
58
+ let firstHitRank = null;
59
+ for (const [file, rank] of fileRank) {
60
+ if (!expected.has(file))
61
+ continue;
62
+ firstHitRank = firstHitRank === null ? rank : Math.min(firstHitRank, rank);
63
+ }
64
+ const foundInK = (k) => {
65
+ let found = 0;
66
+ for (const file of expected) {
67
+ const rank = fileRank.get(file);
68
+ if (rank !== undefined && rank <= k)
69
+ found += 1;
70
+ }
71
+ return found;
72
+ };
73
+ const hitAt = (k) => (foundInK(k) > 0 ? 1 : 0);
74
+ // nDCG@10 with binary gains, deduplicated per expected file.
75
+ let dcg = 0;
76
+ const gained = new Set();
77
+ for (const [i, r] of results.slice(0, 10).entries()) {
78
+ const file = toRepoRelative(root, r.chunk.metadata.filePath);
79
+ if (expected.has(file) && !gained.has(file)) {
80
+ gained.add(file);
81
+ dcg += 1 / Math.log2(i + 2);
82
+ }
83
+ }
84
+ const idealCount = Math.min(expected.size, 10);
85
+ let idcg = 0;
86
+ for (let i = 0; i < idealCount; i++)
87
+ idcg += 1 / Math.log2(i + 2);
88
+ const ndcg10 = idcg > 0 ? dcg / idcg : 0;
89
+ let symbolHitRank = null;
90
+ if (label.symbol) {
91
+ for (const [i, r] of results.entries()) {
92
+ const file = toRepoRelative(root, r.chunk.metadata.filePath);
93
+ if (expected.has(file) && r.chunk.content.includes(label.symbol)) {
94
+ symbolHitRank = i + 1;
95
+ break;
96
+ }
97
+ }
98
+ }
99
+ const rerankRan = results.some((r) => {
100
+ const breakdown = r.explanation?.scoreBreakdown;
101
+ return typeof breakdown?.rerankScore === "number";
102
+ });
103
+ return {
104
+ firstHitRank,
105
+ hitAt1: hitAt(1),
106
+ hitAt3: hitAt(3),
107
+ hitAt5: hitAt(5),
108
+ hitAt10: hitAt(10),
109
+ precision5: foundInK(5) / 5,
110
+ recall10: foundInK(10) / Math.max(expected.size, 1),
111
+ rr: firstHitRank !== null ? 1 / firstHitRank : 0,
112
+ ndcg10,
113
+ symbolHitRank,
114
+ topFiles: results.slice(0, 5).map((r) => toRepoRelative(root, r.chunk.metadata.filePath)),
115
+ rerankRan,
116
+ };
117
+ }
118
+ /** Percentile from a pre-sorted ascending array (`q` in [0,1]). */
119
+ export function percentile(sorted, q) {
120
+ if (sorted.length === 0)
121
+ return 0;
122
+ const idx = Math.min(sorted.length - 1, Math.floor(q * (sorted.length - 1)));
123
+ return sorted[idx] ?? 0;
124
+ }
125
+ /** Arithmetic mean; 0 for an empty list. */
126
+ export function mean(values) {
127
+ if (values.length === 0)
128
+ return 0;
129
+ return values.reduce((s, v) => s + v, 0) / values.length;
130
+ }
131
+ /**
132
+ * Aggregate per-query metrics into mean Hit@k, MRR, nDCG@10, symbol rate, and latency.
133
+ *
134
+ * @param outcomes - Scored queries.
135
+ * @returns The aggregate summary.
136
+ */
137
+ export function summarizeGoldenOutcomes(outcomes) {
138
+ const latencies = outcomes.map((o) => o.latencyMs).sort((a, b) => a - b);
139
+ const withSymbol = outcomes.filter((o) => o.label.symbol);
140
+ return {
141
+ hit1: mean(outcomes.map((o) => o.hitAt1)),
142
+ hit3: mean(outcomes.map((o) => o.hitAt3)),
143
+ hit5: mean(outcomes.map((o) => o.hitAt5)),
144
+ hit10: mean(outcomes.map((o) => o.hitAt10)),
145
+ precision5: mean(outcomes.map((o) => o.precision5)),
146
+ recall10: mean(outcomes.map((o) => o.recall10)),
147
+ mrr: mean(outcomes.map((o) => o.rr)),
148
+ ndcg10: mean(outcomes.map((o) => o.ndcg10)),
149
+ symbolRate: withSymbol.length > 0 ? mean(withSymbol.map((o) => (o.symbolHitRank !== null ? 1 : 0))) : 0,
150
+ latencyAvg: mean(latencies),
151
+ latencyP50: percentile(latencies, 0.5),
152
+ latencyP95: percentile(latencies, 0.95),
153
+ };
154
+ }
155
+ //# sourceMappingURL=golden-set.js.map