opencode-rag-plugin 2.6.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +7 -4
- package/dist/cli/commands/eval-tuner.d.ts +15 -0
- package/dist/cli/commands/eval-tuner.js +285 -0
- package/dist/cli/commands/index.d.ts +1 -0
- package/dist/cli/commands/index.js +1 -0
- package/dist/cli/commands/status.js +20 -0
- package/dist/cli/index.js +2 -1
- package/dist/core/config.d.ts +14 -2
- package/dist/core/config.js +6 -1
- package/dist/core/interfaces.d.ts +8 -0
- package/dist/eval/tuner.d.ts +219 -0
- package/dist/eval/tuner.js +293 -0
- package/dist/indexer/keyword-rehydration.d.ts +48 -0
- package/dist/indexer/keyword-rehydration.js +113 -0
- package/dist/indexer/pipeline.js +27 -0
- package/dist/plugin.js +90 -3
- package/dist/quirks/quirk-store.d.ts +14 -1
- package/dist/quirks/quirk-store.js +27 -0
- package/dist/quirks/tool-error-create.d.ts +79 -7
- package/dist/quirks/tool-error-create.js +135 -16
- package/dist/quirks/types.d.ts +8 -0
- package/dist/quirks/types.js +8 -1
- package/dist/retriever/keyword-index.d.ts +10 -0
- package/dist/retriever/keyword-index.js +12 -0
- package/package.json +1 -1
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Golden-set-driven retrieval hyperparameter tuner.
|
|
3
|
+
*
|
|
4
|
+
* The retrieval fusion weights shipped in the config (`keywordWeight`,
|
|
5
|
+
* `symbolKeywordWeight`, …) are a single hardcoded default, but the optimum
|
|
6
|
+
* depends on the embedder, the corpus and the query mix — a code embedder
|
|
7
|
+
* ranks identifiers well through the vector half alone, while a general
|
|
8
|
+
* embedder leans on the keyword half. This module sweeps a small grid of
|
|
9
|
+
* candidates against the labelled golden set and reports which one wins.
|
|
10
|
+
*
|
|
11
|
+
* The metric implementation is shared with `eval:gate` (and therefore with
|
|
12
|
+
* the rerank A/B harness) so the tuner can never report different numbers
|
|
13
|
+
* than the gate for the same labels.
|
|
14
|
+
*/
|
|
15
|
+
import type { RagConfig } from "../core/config.js";
|
|
16
|
+
import type { SearchResult } from "../core/interfaces.js";
|
|
17
|
+
import { type GoldenLabel, type GoldenOutcome, type GoldenSummary } from "./golden-set.js";
|
|
18
|
+
/** One point in the retrieval search space. */
|
|
19
|
+
export interface TuningCandidate {
|
|
20
|
+
/** RRF keyword weight: 0 = pure vector ranking, 1 = pure keyword ranking. */
|
|
21
|
+
keywordWeight: number;
|
|
22
|
+
/** Keyword weight for bare-identifier queries. Defaults to `keywordWeight`. */
|
|
23
|
+
symbolKeywordWeight?: number;
|
|
24
|
+
/** Minimum fused score a result must reach. */
|
|
25
|
+
minScore: number;
|
|
26
|
+
/** Number of results requested from the pipeline. */
|
|
27
|
+
topK: number;
|
|
28
|
+
}
|
|
29
|
+
/** A candidate together with its measured quality. */
|
|
30
|
+
export interface TuningResult extends TuningCandidate {
|
|
31
|
+
/** Aggregate metrics over the whole label set. */
|
|
32
|
+
summary: GoldenSummary;
|
|
33
|
+
/** Per-label outcomes, for drill-down. */
|
|
34
|
+
outcomes: GoldenOutcome[];
|
|
35
|
+
/** Wall-clock duration of the candidate run in milliseconds. */
|
|
36
|
+
durationMs: number;
|
|
37
|
+
}
|
|
38
|
+
/** Options for {@link buildCandidateGrid}. */
|
|
39
|
+
export interface CandidateGridOptions {
|
|
40
|
+
/** Keyword weights to try. Defaults to `[0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6]`. */
|
|
41
|
+
keywordWeights?: number[];
|
|
42
|
+
/** Symbol keyword weights to try. Defaults to the keyword weights. */
|
|
43
|
+
symbolKeywordWeights?: number[];
|
|
44
|
+
/** Minimum scores to try. Defaults to `[base.minScore]`. */
|
|
45
|
+
minScores?: number[];
|
|
46
|
+
/** Top-K values to try. Defaults to `[base.topK]`. */
|
|
47
|
+
topKs?: number[];
|
|
48
|
+
/** Hard cap on the number of candidates (default 64). */
|
|
49
|
+
maxCandidates?: number;
|
|
50
|
+
}
|
|
51
|
+
/** Metrics a candidate can be ranked by. */
|
|
52
|
+
export type TuningMetric = "ndcg10" | "hit5" | "hit1" | "mrr";
|
|
53
|
+
/**
|
|
54
|
+
* Build the candidate grid.
|
|
55
|
+
*
|
|
56
|
+
* The base candidate is always included so the sweep can be compared against
|
|
57
|
+
* the configuration that is actually running. Duplicates (same weights, same
|
|
58
|
+
* minScore, same topK) are removed, and the grid is capped so a wide sweep
|
|
59
|
+
* cannot turn into an accidental benchmark.
|
|
60
|
+
*
|
|
61
|
+
* @param base - The candidate describing the current configuration.
|
|
62
|
+
* @param options - Grid overrides.
|
|
63
|
+
* @returns The candidate list, base candidate first.
|
|
64
|
+
*/
|
|
65
|
+
export declare function buildCandidateGrid(base: TuningCandidate, options?: CandidateGridOptions): TuningCandidate[];
|
|
66
|
+
/** Options for {@link runTuner}. */
|
|
67
|
+
export interface RunTunerOptions {
|
|
68
|
+
/** The labelled queries to measure. */
|
|
69
|
+
labels: GoldenLabel[];
|
|
70
|
+
/** Workspace root used to relativize result paths. */
|
|
71
|
+
root: string;
|
|
72
|
+
/**
|
|
73
|
+
* Retrieve closure for one candidate. The caller owns the embedder/store
|
|
74
|
+
* wiring and maps the candidate onto the production retrieval options.
|
|
75
|
+
*/
|
|
76
|
+
retrieve: (label: GoldenLabel, candidate: TuningCandidate) => Promise<SearchResult[]>;
|
|
77
|
+
/** The candidates to evaluate. */
|
|
78
|
+
candidates: TuningCandidate[];
|
|
79
|
+
/** Progress line callback (defaults to console.log). */
|
|
80
|
+
log?: (line: string) => void;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Evaluate every candidate against the golden set.
|
|
84
|
+
*
|
|
85
|
+
* @param opts - Labels, root, retrieve closure, candidates, logger.
|
|
86
|
+
* @returns One result per candidate, in the order the candidates were given.
|
|
87
|
+
*/
|
|
88
|
+
export declare function runTuner(opts: RunTunerOptions): Promise<TuningResult[]>;
|
|
89
|
+
/**
|
|
90
|
+
* Pick the best candidate.
|
|
91
|
+
*
|
|
92
|
+
* Ranked by the primary metric, then by Hit@5, MRR and Hit@1 so a tie on the
|
|
93
|
+
* primary metric still resolves to the more useful configuration.
|
|
94
|
+
*
|
|
95
|
+
* The argmax alone is NOT evidence — on a small label set it is often within
|
|
96
|
+
* noise of the runner-up. Use {@link comparePaired} for the confidence
|
|
97
|
+
* interval before changing anything.
|
|
98
|
+
*
|
|
99
|
+
* @param results - The measured candidates.
|
|
100
|
+
* @param metric - The primary metric (default `ndcg10`).
|
|
101
|
+
* @returns The winning result, or `undefined` when there are no results.
|
|
102
|
+
*/
|
|
103
|
+
export declare function selectBestTuning(results: TuningResult[], metric?: TuningMetric): TuningResult | undefined;
|
|
104
|
+
/**
|
|
105
|
+
* Compare the winner against the candidate that describes the current config.
|
|
106
|
+
*
|
|
107
|
+
* @param best - The winning candidate.
|
|
108
|
+
* @param current - The candidate describing the running configuration.
|
|
109
|
+
* @returns The per-metric deltas (winner minus current).
|
|
110
|
+
*/
|
|
111
|
+
export declare function tuningDelta(best: TuningResult, current: TuningResult | undefined): Record<string, number>;
|
|
112
|
+
/** Paired comparison of two candidates measured on the same labels. */
|
|
113
|
+
export interface PairedComparison {
|
|
114
|
+
/** The metric the comparison is based on. */
|
|
115
|
+
metric: TuningMetric;
|
|
116
|
+
/** Mean per-label difference (first candidate minus second). */
|
|
117
|
+
meanDelta: number;
|
|
118
|
+
/** Lower bound of the 95% bootstrap confidence interval. */
|
|
119
|
+
ciLow: number;
|
|
120
|
+
/** Upper bound of the 95% bootstrap confidence interval. */
|
|
121
|
+
ciHigh: number;
|
|
122
|
+
/** Share of bootstrap resamples in which the first candidate wins. */
|
|
123
|
+
pBetter: number;
|
|
124
|
+
/** Labels where the first candidate scored higher. */
|
|
125
|
+
improved: number;
|
|
126
|
+
/** Labels where the second candidate scored higher. */
|
|
127
|
+
regressed: number;
|
|
128
|
+
/** Labels where both scored the same. */
|
|
129
|
+
unchanged: number;
|
|
130
|
+
/** True when the confidence interval includes 0 — the difference is within noise. */
|
|
131
|
+
withinNoise: boolean;
|
|
132
|
+
/**
|
|
133
|
+
* Two-sided exact sign-test p-value over the discordant labels.
|
|
134
|
+
*
|
|
135
|
+
* The bootstrap CI is the optimistic bound: it weights a label by how much it
|
|
136
|
+
* moved. The sign test only asks "how many labels flipped?", which is the
|
|
137
|
+
* conservative reading of a small label set — a 19-vs-11 split is still
|
|
138
|
+
* p ≈ 0.20. Report both.
|
|
139
|
+
*/
|
|
140
|
+
signTestP: number;
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Two-sided exact sign-test p-value for `wins` successes out of `wins + losses`
|
|
144
|
+
* trials under a 0.5 null.
|
|
145
|
+
*
|
|
146
|
+
* @param wins - Labels where the first candidate scored higher.
|
|
147
|
+
* @param losses - Labels where the second candidate scored higher.
|
|
148
|
+
* @returns The p-value in [0, 1]; 1 when there are no discordant labels.
|
|
149
|
+
*/
|
|
150
|
+
export declare function signTestP(wins: number, losses: number): number;
|
|
151
|
+
/**
|
|
152
|
+
* Compare two candidates with a paired bootstrap over the labels.
|
|
153
|
+
*
|
|
154
|
+
* The label set is small (48 by default), so a point estimate on its own is
|
|
155
|
+
* not evidence: a 10-point Hit@5 gap is a handful of labels, and a sign test
|
|
156
|
+
* on the discordant labels is the honest lower bound. This returns the
|
|
157
|
+
* confidence interval, the win/loss counts, and an explicit `withinNoise`
|
|
158
|
+
* flag so a caller can refuse to recommend a change the data does not support.
|
|
159
|
+
*
|
|
160
|
+
* @param better - The candidate to test.
|
|
161
|
+
* @param other - The candidate to compare against (usually the current config).
|
|
162
|
+
* @param metric - The metric to compare on (default `ndcg10`).
|
|
163
|
+
* @param iterations - Bootstrap resamples (default 20000).
|
|
164
|
+
* @param rng - Random source (injectable for deterministic tests).
|
|
165
|
+
* @returns The paired comparison, or `undefined` when there is nothing to compare.
|
|
166
|
+
*/
|
|
167
|
+
export declare function comparePaired(better: TuningResult, other: TuningResult | undefined, metric?: TuningMetric, iterations?: number, rng?: () => number): PairedComparison | undefined;
|
|
168
|
+
/** Aggregate file-level quality of one retrieval signal. */
|
|
169
|
+
export interface SignalQuality {
|
|
170
|
+
/** Mean Hit@1 over the labels. */
|
|
171
|
+
hit1: number;
|
|
172
|
+
/** Mean Hit@5 over the labels. */
|
|
173
|
+
hit5: number;
|
|
174
|
+
/** Mean Hit@10 over the labels. */
|
|
175
|
+
hit10: number;
|
|
176
|
+
/** Labels where the expected file was not found at all. */
|
|
177
|
+
misses: number;
|
|
178
|
+
}
|
|
179
|
+
/** Per-signal quality: the keyword half and the vector half measured apart. */
|
|
180
|
+
export interface SignalComparison {
|
|
181
|
+
/** Quality of the keyword (BM25) signal alone. */
|
|
182
|
+
keyword: SignalQuality;
|
|
183
|
+
/** Quality of the vector signal alone. */
|
|
184
|
+
vector: SignalQuality;
|
|
185
|
+
/** Quality of the fused pipeline. */
|
|
186
|
+
fused: SignalQuality;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Measure each retrieval signal on its own, so a fused number can be
|
|
190
|
+
* interpreted instead of just believed.
|
|
191
|
+
*
|
|
192
|
+
* A fused ranking can only be as good as the mix of its parts: when the
|
|
193
|
+
* keyword half answers far fewer queries than the vector half, any weight
|
|
194
|
+
* given to it dilutes the result. This measures the parts.
|
|
195
|
+
*
|
|
196
|
+
* @param opts - Labels, root, and one retrieve closure per signal.
|
|
197
|
+
* @returns The per-signal quality.
|
|
198
|
+
*/
|
|
199
|
+
export declare function measureSignalQuality(opts: {
|
|
200
|
+
labels: GoldenLabel[];
|
|
201
|
+
root: string;
|
|
202
|
+
/** Retrieve with the keyword signal only. */
|
|
203
|
+
retrieveKeyword: (label: GoldenLabel) => Promise<SearchResult[]>;
|
|
204
|
+
/** Retrieve with the vector signal only. */
|
|
205
|
+
retrieveVector: (label: GoldenLabel) => Promise<SearchResult[]>;
|
|
206
|
+
/** Retrieve with the full pipeline. */
|
|
207
|
+
retrieveFused: (label: GoldenLabel) => Promise<SearchResult[]>;
|
|
208
|
+
}): Promise<SignalComparison>;
|
|
209
|
+
/**
|
|
210
|
+
* Write the winning weights into a copy of the configuration.
|
|
211
|
+
*
|
|
212
|
+
* The input config is not mutated — the caller decides whether to persist the
|
|
213
|
+
* result. Only the retrieval section is touched.
|
|
214
|
+
*
|
|
215
|
+
* @param config - The configuration to base the result on.
|
|
216
|
+
* @param best - The winning candidate.
|
|
217
|
+
* @returns A new config with the winning fusion weights applied.
|
|
218
|
+
*/
|
|
219
|
+
export declare function applyTuningToConfig(config: RagConfig, best: TuningCandidate): RagConfig;
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
import { runGoldenGate, } from "./golden-gate.js";
|
|
2
|
+
import { evaluateGoldenResults, mean, } from "./golden-set.js";
|
|
3
|
+
/** The metric used to pick the winner, and the tie-breakers after it. */
|
|
4
|
+
const METRIC_ORDER = {
|
|
5
|
+
ndcg10: (s) => s.ndcg10,
|
|
6
|
+
hit5: (s) => s.hit5,
|
|
7
|
+
hit1: (s) => s.hit1,
|
|
8
|
+
mrr: (s) => s.mrr,
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Build the candidate grid.
|
|
12
|
+
*
|
|
13
|
+
* The base candidate is always included so the sweep can be compared against
|
|
14
|
+
* the configuration that is actually running. Duplicates (same weights, same
|
|
15
|
+
* minScore, same topK) are removed, and the grid is capped so a wide sweep
|
|
16
|
+
* cannot turn into an accidental benchmark.
|
|
17
|
+
*
|
|
18
|
+
* @param base - The candidate describing the current configuration.
|
|
19
|
+
* @param options - Grid overrides.
|
|
20
|
+
* @returns The candidate list, base candidate first.
|
|
21
|
+
*/
|
|
22
|
+
export function buildCandidateGrid(base, options = {}) {
|
|
23
|
+
const keywordWeights = options.keywordWeights ?? [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6];
|
|
24
|
+
const symbolKeywordWeights = options.symbolKeywordWeights ?? keywordWeights;
|
|
25
|
+
const minScores = options.minScores ?? [base.minScore];
|
|
26
|
+
const topKs = options.topKs ?? [base.topK];
|
|
27
|
+
const maxCandidates = options.maxCandidates ?? 64;
|
|
28
|
+
const seen = new Set();
|
|
29
|
+
const candidates = [];
|
|
30
|
+
const push = (candidate) => {
|
|
31
|
+
const key = [
|
|
32
|
+
candidate.keywordWeight,
|
|
33
|
+
candidate.symbolKeywordWeight ?? "unset",
|
|
34
|
+
candidate.minScore,
|
|
35
|
+
candidate.topK,
|
|
36
|
+
].join("|");
|
|
37
|
+
if (seen.has(key))
|
|
38
|
+
return;
|
|
39
|
+
seen.add(key);
|
|
40
|
+
candidates.push(candidate);
|
|
41
|
+
};
|
|
42
|
+
push({ ...base });
|
|
43
|
+
for (const topK of topKs) {
|
|
44
|
+
for (const minScore of minScores) {
|
|
45
|
+
for (const keywordWeight of keywordWeights) {
|
|
46
|
+
for (const symbolKeywordWeight of symbolKeywordWeights) {
|
|
47
|
+
push({ keywordWeight, symbolKeywordWeight, minScore, topK });
|
|
48
|
+
if (candidates.length >= maxCandidates)
|
|
49
|
+
return candidates;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return candidates;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Evaluate every candidate against the golden set.
|
|
58
|
+
*
|
|
59
|
+
* @param opts - Labels, root, retrieve closure, candidates, logger.
|
|
60
|
+
* @returns One result per candidate, in the order the candidates were given.
|
|
61
|
+
*/
|
|
62
|
+
export async function runTuner(opts) {
|
|
63
|
+
const log = opts.log ?? (() => { });
|
|
64
|
+
const results = [];
|
|
65
|
+
for (const candidate of opts.candidates) {
|
|
66
|
+
const started = performance.now();
|
|
67
|
+
const gateOpts = {
|
|
68
|
+
labels: opts.labels,
|
|
69
|
+
root: opts.root,
|
|
70
|
+
retrieve: (label) => opts.retrieve(label, candidate),
|
|
71
|
+
log: () => { },
|
|
72
|
+
};
|
|
73
|
+
const { summary, outcomes } = await runGoldenGate(gateOpts);
|
|
74
|
+
const durationMs = performance.now() - started;
|
|
75
|
+
results.push({ ...candidate, summary, outcomes, durationMs });
|
|
76
|
+
log(` kw=${candidate.keywordWeight} skw=${candidate.symbolKeywordWeight ?? "-"} ` +
|
|
77
|
+
`minScore=${candidate.minScore} topK=${candidate.topK} ` +
|
|
78
|
+
`Hit@5 ${(summary.hit5 * 100).toFixed(1)}% MRR ${summary.mrr.toFixed(3)} ` +
|
|
79
|
+
`nDCG@10 ${summary.ndcg10.toFixed(3)} (${durationMs.toFixed(0)}ms)`);
|
|
80
|
+
}
|
|
81
|
+
return results;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Pick the best candidate.
|
|
85
|
+
*
|
|
86
|
+
* Ranked by the primary metric, then by Hit@5, MRR and Hit@1 so a tie on the
|
|
87
|
+
* primary metric still resolves to the more useful configuration.
|
|
88
|
+
*
|
|
89
|
+
* The argmax alone is NOT evidence — on a small label set it is often within
|
|
90
|
+
* noise of the runner-up. Use {@link comparePaired} for the confidence
|
|
91
|
+
* interval before changing anything.
|
|
92
|
+
*
|
|
93
|
+
* @param results - The measured candidates.
|
|
94
|
+
* @param metric - The primary metric (default `ndcg10`).
|
|
95
|
+
* @returns The winning result, or `undefined` when there are no results.
|
|
96
|
+
*/
|
|
97
|
+
export function selectBestTuning(results, metric = "ndcg10") {
|
|
98
|
+
if (results.length === 0)
|
|
99
|
+
return undefined;
|
|
100
|
+
const primary = METRIC_ORDER[metric];
|
|
101
|
+
const score = (r) => [
|
|
102
|
+
primary(r.summary),
|
|
103
|
+
r.summary.hit5,
|
|
104
|
+
r.summary.mrr,
|
|
105
|
+
r.summary.hit1,
|
|
106
|
+
];
|
|
107
|
+
return [...results].sort((a, b) => {
|
|
108
|
+
const sa = score(a);
|
|
109
|
+
const sb = score(b);
|
|
110
|
+
for (let i = 0; i < sa.length; i++) {
|
|
111
|
+
if (sb[i] !== sa[i])
|
|
112
|
+
return sb[i] - sa[i];
|
|
113
|
+
}
|
|
114
|
+
return 0;
|
|
115
|
+
})[0];
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Compare the winner against the candidate that describes the current config.
|
|
119
|
+
*
|
|
120
|
+
* @param best - The winning candidate.
|
|
121
|
+
* @param current - The candidate describing the running configuration.
|
|
122
|
+
* @returns The per-metric deltas (winner minus current).
|
|
123
|
+
*/
|
|
124
|
+
export function tuningDelta(best, current) {
|
|
125
|
+
if (!current)
|
|
126
|
+
return {};
|
|
127
|
+
return {
|
|
128
|
+
hit1: best.summary.hit1 - current.summary.hit1,
|
|
129
|
+
hit3: best.summary.hit3 - current.summary.hit3,
|
|
130
|
+
hit5: best.summary.hit5 - current.summary.hit5,
|
|
131
|
+
hit10: best.summary.hit10 - current.summary.hit10,
|
|
132
|
+
mrr: best.summary.mrr - current.summary.mrr,
|
|
133
|
+
ndcg10: best.summary.ndcg10 - current.summary.ndcg10,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
/** Per-label value of a tuning metric. */
|
|
137
|
+
function metricValue(outcome, metric) {
|
|
138
|
+
switch (metric) {
|
|
139
|
+
case "hit5": return outcome.hitAt5;
|
|
140
|
+
case "hit1": return outcome.hitAt1;
|
|
141
|
+
case "mrr": return outcome.rr;
|
|
142
|
+
default: return outcome.ndcg10;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Two-sided exact sign-test p-value for `wins` successes out of `wins + losses`
|
|
147
|
+
* trials under a 0.5 null.
|
|
148
|
+
*
|
|
149
|
+
* @param wins - Labels where the first candidate scored higher.
|
|
150
|
+
* @param losses - Labels where the second candidate scored higher.
|
|
151
|
+
* @returns The p-value in [0, 1]; 1 when there are no discordant labels.
|
|
152
|
+
*/
|
|
153
|
+
export function signTestP(wins, losses) {
|
|
154
|
+
const n = wins + losses;
|
|
155
|
+
if (n === 0)
|
|
156
|
+
return 1;
|
|
157
|
+
const k = Math.max(wins, losses);
|
|
158
|
+
let tail = 0;
|
|
159
|
+
for (let i = k; i <= n; i++) {
|
|
160
|
+
tail += binomial(n, i);
|
|
161
|
+
}
|
|
162
|
+
const p = (tail * 2) / Math.pow(2, n);
|
|
163
|
+
return Math.min(1, p);
|
|
164
|
+
}
|
|
165
|
+
/** Binomial coefficient C(n, k). */
|
|
166
|
+
function binomial(n, k) {
|
|
167
|
+
let result = 1;
|
|
168
|
+
for (let i = 0; i < k; i++) {
|
|
169
|
+
result = (result * (n - i)) / (i + 1);
|
|
170
|
+
}
|
|
171
|
+
return Math.round(result);
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Compare two candidates with a paired bootstrap over the labels.
|
|
175
|
+
*
|
|
176
|
+
* The label set is small (48 by default), so a point estimate on its own is
|
|
177
|
+
* not evidence: a 10-point Hit@5 gap is a handful of labels, and a sign test
|
|
178
|
+
* on the discordant labels is the honest lower bound. This returns the
|
|
179
|
+
* confidence interval, the win/loss counts, and an explicit `withinNoise`
|
|
180
|
+
* flag so a caller can refuse to recommend a change the data does not support.
|
|
181
|
+
*
|
|
182
|
+
* @param better - The candidate to test.
|
|
183
|
+
* @param other - The candidate to compare against (usually the current config).
|
|
184
|
+
* @param metric - The metric to compare on (default `ndcg10`).
|
|
185
|
+
* @param iterations - Bootstrap resamples (default 20000).
|
|
186
|
+
* @param rng - Random source (injectable for deterministic tests).
|
|
187
|
+
* @returns The paired comparison, or `undefined` when there is nothing to compare.
|
|
188
|
+
*/
|
|
189
|
+
export function comparePaired(better, other, metric = "ndcg10", iterations = 20_000, rng = Math.random) {
|
|
190
|
+
if (!other)
|
|
191
|
+
return undefined;
|
|
192
|
+
const n = Math.min(better.outcomes.length, other.outcomes.length);
|
|
193
|
+
if (n === 0)
|
|
194
|
+
return undefined;
|
|
195
|
+
const diffs = [];
|
|
196
|
+
let improved = 0;
|
|
197
|
+
let regressed = 0;
|
|
198
|
+
let unchanged = 0;
|
|
199
|
+
for (let i = 0; i < n; i++) {
|
|
200
|
+
const d = metricValue(better.outcomes[i], metric) - metricValue(other.outcomes[i], metric);
|
|
201
|
+
diffs.push(d);
|
|
202
|
+
if (d > 1e-9)
|
|
203
|
+
improved++;
|
|
204
|
+
else if (d < -1e-9)
|
|
205
|
+
regressed++;
|
|
206
|
+
else
|
|
207
|
+
unchanged++;
|
|
208
|
+
}
|
|
209
|
+
const meanDelta = diffs.reduce((s, d) => s + d, 0) / n;
|
|
210
|
+
const boots = [];
|
|
211
|
+
for (let i = 0; i < iterations; i++) {
|
|
212
|
+
let sum = 0;
|
|
213
|
+
for (let j = 0; j < n; j++)
|
|
214
|
+
sum += diffs[Math.floor(rng() * n)];
|
|
215
|
+
boots.push(sum / n);
|
|
216
|
+
}
|
|
217
|
+
boots.sort((a, b) => a - b);
|
|
218
|
+
const at = (p) => boots[Math.min(boots.length - 1, Math.max(0, Math.floor(p * (boots.length - 1))))];
|
|
219
|
+
const ciLow = at(0.025);
|
|
220
|
+
const ciHigh = at(0.975);
|
|
221
|
+
const pBetter = boots.filter((b) => b > 0).length / boots.length;
|
|
222
|
+
return {
|
|
223
|
+
metric,
|
|
224
|
+
meanDelta,
|
|
225
|
+
ciLow,
|
|
226
|
+
ciHigh,
|
|
227
|
+
pBetter,
|
|
228
|
+
improved,
|
|
229
|
+
regressed,
|
|
230
|
+
unchanged,
|
|
231
|
+
withinNoise: ciLow <= 0 && ciHigh >= 0,
|
|
232
|
+
signTestP: signTestP(improved, regressed),
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Measure each retrieval signal on its own, so a fused number can be
|
|
237
|
+
* interpreted instead of just believed.
|
|
238
|
+
*
|
|
239
|
+
* A fused ranking can only be as good as the mix of its parts: when the
|
|
240
|
+
* keyword half answers far fewer queries than the vector half, any weight
|
|
241
|
+
* given to it dilutes the result. This measures the parts.
|
|
242
|
+
*
|
|
243
|
+
* @param opts - Labels, root, and one retrieve closure per signal.
|
|
244
|
+
* @returns The per-signal quality.
|
|
245
|
+
*/
|
|
246
|
+
export async function measureSignalQuality(opts) {
|
|
247
|
+
const summarize = async (retrieve) => {
|
|
248
|
+
const metrics = [];
|
|
249
|
+
for (const label of opts.labels) {
|
|
250
|
+
const results = await retrieve(label);
|
|
251
|
+
metrics.push(evaluateGoldenResults(opts.root, label, results));
|
|
252
|
+
}
|
|
253
|
+
return {
|
|
254
|
+
hit1: mean(metrics.map((m) => m.hitAt1)),
|
|
255
|
+
hit5: mean(metrics.map((m) => m.hitAt5)),
|
|
256
|
+
hit10: mean(metrics.map((m) => m.hitAt10)),
|
|
257
|
+
misses: metrics.filter((m) => m.firstHitRank === null).length,
|
|
258
|
+
};
|
|
259
|
+
};
|
|
260
|
+
return {
|
|
261
|
+
keyword: await summarize(opts.retrieveKeyword),
|
|
262
|
+
vector: await summarize(opts.retrieveVector),
|
|
263
|
+
fused: await summarize(opts.retrieveFused),
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Write the winning weights into a copy of the configuration.
|
|
268
|
+
*
|
|
269
|
+
* The input config is not mutated — the caller decides whether to persist the
|
|
270
|
+
* result. Only the retrieval section is touched.
|
|
271
|
+
*
|
|
272
|
+
* @param config - The configuration to base the result on.
|
|
273
|
+
* @param best - The winning candidate.
|
|
274
|
+
* @returns A new config with the winning fusion weights applied.
|
|
275
|
+
*/
|
|
276
|
+
export function applyTuningToConfig(config, best) {
|
|
277
|
+
const hybrid = config.retrieval.hybridSearch ?? { enabled: true, keywordWeight: best.keywordWeight };
|
|
278
|
+
return {
|
|
279
|
+
...config,
|
|
280
|
+
retrieval: {
|
|
281
|
+
...config.retrieval,
|
|
282
|
+
topK: best.topK,
|
|
283
|
+
minScore: best.minScore,
|
|
284
|
+
hybridSearch: {
|
|
285
|
+
...hybrid,
|
|
286
|
+
enabled: hybrid.enabled ?? true,
|
|
287
|
+
keywordWeight: best.keywordWeight,
|
|
288
|
+
symbolKeywordWeight: best.symbolKeywordWeight ?? hybrid.symbolKeywordWeight,
|
|
289
|
+
},
|
|
290
|
+
},
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
//# sourceMappingURL=tuner.js.map
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { KeywordIndex, VectorStore } from "../core/interfaces.js";
|
|
2
|
+
/** Options for {@link rehydrateKeywordIndex}. */
|
|
3
|
+
export interface KeywordRehydrationOptions {
|
|
4
|
+
/** The vector store holding the authoritative chunk rows. */
|
|
5
|
+
store: VectorStore;
|
|
6
|
+
/** The keyword index to repair. */
|
|
7
|
+
keywordIndex: KeywordIndex;
|
|
8
|
+
/** Optional logger (missing methods default to no-ops). */
|
|
9
|
+
logger?: {
|
|
10
|
+
info?(message: string): void;
|
|
11
|
+
warn?(message: string): void;
|
|
12
|
+
debug?(message: string): void;
|
|
13
|
+
};
|
|
14
|
+
/** Maximum number of files to rehydrate in one call (0/undefined = unlimited). */
|
|
15
|
+
maxFiles?: number;
|
|
16
|
+
}
|
|
17
|
+
/** Outcome of a rehydration attempt. */
|
|
18
|
+
export interface KeywordRehydrationResult {
|
|
19
|
+
/** Files present in the store with no keyword entries. */
|
|
20
|
+
missingFiles: number;
|
|
21
|
+
/** Files whose chunks were added to the keyword index. */
|
|
22
|
+
rehydratedFiles: number;
|
|
23
|
+
/** Chunks added to the keyword index. */
|
|
24
|
+
rehydratedChunks: number;
|
|
25
|
+
/** Chunks skipped because they were already indexed. */
|
|
26
|
+
skippedChunks: number;
|
|
27
|
+
/** Files that could not be rehydrated (one failure never aborts the rest). */
|
|
28
|
+
failedFiles: string[];
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Restore the keyword entries for every file the store holds but the keyword
|
|
32
|
+
* index does not.
|
|
33
|
+
*
|
|
34
|
+
* Quirk rows are skipped: `quirks.jsonl` owns them, and `reconcileQuirks()`
|
|
35
|
+
* adds and removes them under its own rules.
|
|
36
|
+
*
|
|
37
|
+
* @param options - The store, the keyword index, and optional logging.
|
|
38
|
+
* @returns What was missing and what was restored.
|
|
39
|
+
*/
|
|
40
|
+
export declare function rehydrateKeywordIndex(options: KeywordRehydrationOptions): Promise<KeywordRehydrationResult>;
|
|
41
|
+
/**
|
|
42
|
+
* Compare the keyword index against the vector store without changing either.
|
|
43
|
+
*
|
|
44
|
+
* @param store - The vector store holding the authoritative chunk rows.
|
|
45
|
+
* @param keywordIndex - The keyword index to compare.
|
|
46
|
+
* @returns The files the store has that the keyword index does not cover.
|
|
47
|
+
*/
|
|
48
|
+
export declare function findKeywordIndexGaps(store: VectorStore, keywordIndex: KeywordIndex): Promise<string[]>;
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Repairs a keyword index that drifted behind the vector store.
|
|
3
|
+
*
|
|
4
|
+
* The keyword index (BM25 side of hybrid search) and the vector store are two
|
|
5
|
+
* independent stores of the same corpus. Nothing keeps them in sync by
|
|
6
|
+
* construction: an incremental index pass only calls `addChunks()` for the
|
|
7
|
+
* files it reprocessed, so unchanged files keep whatever the *loaded* index
|
|
8
|
+
* had. Once the persisted index loses entries — a crashed pass that saved a
|
|
9
|
+
* cleared index, a partial indexer, a rebuild that swapped the store without
|
|
10
|
+
* rewriting the sidecar — every later pass re-saves the same partial state
|
|
11
|
+
* and the missing files stay unreachable through keyword search forever.
|
|
12
|
+
*
|
|
13
|
+
* The repair needs no re-embedding: the store already holds the chunk text
|
|
14
|
+
* and descriptions, so the missing keyword entries can be rebuilt from the
|
|
15
|
+
* store's own rows.
|
|
16
|
+
*/
|
|
17
|
+
import path from "node:path";
|
|
18
|
+
import { normalizeFilePath } from "../core/manifest.js";
|
|
19
|
+
import { QUIRK_FILE_PREFIX } from "../quirks/quirk-store.js";
|
|
20
|
+
/**
|
|
21
|
+
* Restore the keyword entries for every file the store holds but the keyword
|
|
22
|
+
* index does not.
|
|
23
|
+
*
|
|
24
|
+
* Quirk rows are skipped: `quirks.jsonl` owns them, and `reconcileQuirks()`
|
|
25
|
+
* adds and removes them under its own rules.
|
|
26
|
+
*
|
|
27
|
+
* @param options - The store, the keyword index, and optional logging.
|
|
28
|
+
* @returns What was missing and what was restored.
|
|
29
|
+
*/
|
|
30
|
+
export async function rehydrateKeywordIndex(options) {
|
|
31
|
+
const { store, keywordIndex, logger } = options;
|
|
32
|
+
const result = {
|
|
33
|
+
missingFiles: 0,
|
|
34
|
+
rehydratedFiles: 0,
|
|
35
|
+
rehydratedChunks: 0,
|
|
36
|
+
skippedChunks: 0,
|
|
37
|
+
failedFiles: [],
|
|
38
|
+
};
|
|
39
|
+
let storePaths;
|
|
40
|
+
try {
|
|
41
|
+
storePaths = await store.getFilePaths();
|
|
42
|
+
}
|
|
43
|
+
catch (err) {
|
|
44
|
+
logger?.warn?.(`Keyword rehydration skipped — could not list store files: ${err.message}`);
|
|
45
|
+
return result;
|
|
46
|
+
}
|
|
47
|
+
if (storePaths.length === 0)
|
|
48
|
+
return result;
|
|
49
|
+
const indexed = new Set(keywordIndex.getIndexedFilePaths());
|
|
50
|
+
const missing = storePaths.filter((p) => {
|
|
51
|
+
// Quirk rows belong to quirks.jsonl — reconcileQuirks() owns them.
|
|
52
|
+
if (path.basename(p).startsWith(QUIRK_FILE_PREFIX))
|
|
53
|
+
return false;
|
|
54
|
+
return !indexed.has(normalizeFilePath(p));
|
|
55
|
+
});
|
|
56
|
+
result.missingFiles = missing.length;
|
|
57
|
+
if (missing.length === 0)
|
|
58
|
+
return result;
|
|
59
|
+
const maxFiles = options.maxFiles && options.maxFiles > 0 ? options.maxFiles : missing.length;
|
|
60
|
+
const targets = missing.slice(0, maxFiles);
|
|
61
|
+
if (targets.length < missing.length) {
|
|
62
|
+
logger?.info?.(`Keyword rehydration: ${missing.length} file(s) missing from the keyword index — rehydrating the first ${targets.length}`);
|
|
63
|
+
}
|
|
64
|
+
for (const filePath of targets) {
|
|
65
|
+
try {
|
|
66
|
+
const chunks = await store.getChunksByFilePath(filePath);
|
|
67
|
+
if (chunks.length === 0)
|
|
68
|
+
continue;
|
|
69
|
+
// Guard against double counting: `addChunks()` is not idempotent for the
|
|
70
|
+
// BM25 length statistics and term frequencies, so a chunk that is
|
|
71
|
+
// already indexed must never be added a second time.
|
|
72
|
+
const fresh = chunks.filter((c) => !keywordIndex.hasChunk(c.id));
|
|
73
|
+
result.skippedChunks += chunks.length - fresh.length;
|
|
74
|
+
if (fresh.length === 0)
|
|
75
|
+
continue;
|
|
76
|
+
keywordIndex.addChunks(fresh);
|
|
77
|
+
result.rehydratedFiles++;
|
|
78
|
+
result.rehydratedChunks += fresh.length;
|
|
79
|
+
}
|
|
80
|
+
catch (err) {
|
|
81
|
+
result.failedFiles.push(filePath);
|
|
82
|
+
logger?.warn?.(`Keyword rehydration failed for ${filePath}: ${err.message}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (result.rehydratedFiles > 0) {
|
|
86
|
+
logger?.info?.(`Keyword rehydration: restored ${result.rehydratedChunks} chunk(s) from ` +
|
|
87
|
+
`${result.rehydratedFiles} file(s) (${result.missingFiles} file(s) were missing from the keyword index)`);
|
|
88
|
+
}
|
|
89
|
+
return result;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Compare the keyword index against the vector store without changing either.
|
|
93
|
+
*
|
|
94
|
+
* @param store - The vector store holding the authoritative chunk rows.
|
|
95
|
+
* @param keywordIndex - The keyword index to compare.
|
|
96
|
+
* @returns The files the store has that the keyword index does not cover.
|
|
97
|
+
*/
|
|
98
|
+
export async function findKeywordIndexGaps(store, keywordIndex) {
|
|
99
|
+
let storePaths;
|
|
100
|
+
try {
|
|
101
|
+
storePaths = await store.getFilePaths();
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
return [];
|
|
105
|
+
}
|
|
106
|
+
const indexed = new Set(keywordIndex.getIndexedFilePaths());
|
|
107
|
+
return storePaths.filter((p) => {
|
|
108
|
+
if (path.basename(p).startsWith(QUIRK_FILE_PREFIX))
|
|
109
|
+
return false;
|
|
110
|
+
return !indexed.has(normalizeFilePath(p));
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=keyword-rehydration.js.map
|