opencode-rag-plugin 2.7.0 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/ReadMe.md CHANGED
@@ -244,7 +244,9 @@ opencode-rag quirk test "npm needs --legacy-peer-deps"
244
244
 
245
245
  When `memory.autoInject` is `true`, the plugin checks for relevant quirks on every user message using the combined agent-response + user-query as the search query. Quirks are only injected when their raw vector similarity to that query clears `autoInjectMinScore` (default 0.75) - the same threshold for the user-message and system-prompt injections; `recallMinScore` (default 0.72) is the bar for manual `recall_quirks` calls. A latency budget (`autoInjectLatencyBudgetMs`, default 2000ms) prevents slow embedders from blocking message processing. To avoid polluting the context window, each quirk is injected **at most once per session** — once recalled, it is filtered out from all subsequent auto-injections. Every `add_quirk` and every content-changing `update_quirk` is vetted by an immutable trust monitor that rejects destructive patterns (e.g. `rm -rf`, `force push`, `bypass security`). Outdated or fixed quirks should be corrected with `update_quirk` / `delete_quirk` rather than left to contradict newer memory. See [Plugin documentation](doc/plugin.md#9-quirk-memory-experiential-memory) and [CLI Reference: `quirk`](doc/cli.md#quirk).
246
246
 
247
- **Failed tool calls become memory.** When a tool call fails, the plugin looks for a stored quirk that covers the failure and injects it into the next request (`memory.injectOnToolError`, default on). If **no** quirk covers it, it writes one — `memory.autoCreateOnToolError` (default on) — using a deterministic template built from the tool name and the error text, so no LLM is involved. The new quirk is tagged `tool-error` and recorded for *future* sessions; it is deliberately excluded from the generic auto-injection above, so a raw error dump never leaks into an unrelated turn. Creation is bounded by a per-session budget (2) and a per-error cooldown (10 min), and the quirk it would write is suppressed as soon as a similar one already exists.
247
+ **Failed tool calls become memory.** When a tool call fails, the plugin looks for a stored quirk that covers the failure and injects it into the next request (`memory.injectOnToolError`, default on). If **no** quirk covers it, it writes one — `memory.autoCreateOnToolError` (default on) — using a deterministic template built from the tool name and the error text, so no LLM is involved. The new quirk is tagged `tool-error` and recorded for *future* sessions; it is deliberately excluded from the generic auto-injection above, so a raw error dump never leaks into an unrelated turn. Creation is bounded by a per-session budget (2) and a per-error cooldown (10 min), and it is suppressed when a quirk for the same tool and cause class already exists.
248
+
249
+ **Prevention, not just learning.** The gotchas are also injected into the **system prompt before the model picks a tool** (`memory.toolErrorProactiveInject`, default on), scoped to the tools the session can actually call and to durable failures only — a rate limit is never warned about in advance, because the next call will not fail the same way. The first occurrence of a failure is unavoidable; every later session is warned up front.
248
250
 
249
251
  ## MCP Server (Optional)
250
252
 
package/dist/api.d.ts CHANGED
@@ -2,6 +2,7 @@
2
2
  * @fileoverview High-level public API for OpenCodeRAG: search, indexWorkspace, getContext,
3
3
  * scanWorkspace, and related types.
4
4
  */
5
+ import type { MinScoreMode } from "./retriever/retriever.js";
5
6
  import { type IndexRunStats } from "./indexer.js";
6
7
  import { type WorkspaceFile } from "./content/reader.js";
7
8
  import { type SearchResult } from "./core/interfaces.js";
@@ -15,6 +16,13 @@ export interface SearchOptions {
15
16
  topK?: number;
16
17
  /** Minimum relevance score threshold (0-1). */
17
18
  minScore?: number;
19
+ /**
20
+ * How the `minScore` gate reads the score: `"fused"` (default, today's
21
+ * behaviour) or `"normalized"` (scale-free — the score is divided by the
22
+ * maximum that result could reach, so the threshold keeps its meaning at
23
+ * every keyword weight).
24
+ */
25
+ minScoreMode?: MinScoreMode;
18
26
  /** Weight for hybrid keyword search (0-1). */
19
27
  keywordWeight?: number;
20
28
  /** Filter results to files matching these path patterns. */
package/dist/api.js CHANGED
@@ -48,6 +48,7 @@ export async function search(query, options = {}) {
48
48
  const rawResults = await retrieve(query, ctx.embedder, ctx.store, {
49
49
  topK,
50
50
  minScore: options.minScore ?? ctx.config.retrieval.minScore,
51
+ minScoreMode: options.minScoreMode ?? ctx.config.retrieval.minScoreMode,
51
52
  keywordIndex: ctx.keywordIndex,
52
53
  keywordWeight: options.keywordWeight ?? ctx.config.retrieval.hybridSearch?.keywordWeight ?? 0.4,
53
54
  symbolKeywordWeight: ctx.config.retrieval.hybridSearch?.symbolKeywordWeight,
@@ -51,7 +51,7 @@ export function registerEvalTuneCommand(program) {
51
51
  let exitCode = 0;
52
52
  try {
53
53
  const { loadGoldenLabels } = await import("../../eval/golden-set.js");
54
- const { applyTuningToConfig, buildCandidateGrid, runTuner, selectBestTuning, tuningDelta, } = await import("../../eval/tuner.js");
54
+ const { applyTuningToConfig, buildCandidateGrid, comparePaired, measureSignalQuality, runTuner, selectBestTuning, tuningDelta, } = await import("../../eval/tuner.js");
55
55
  const { updateConfigValue } = await import("../../core/config.js");
56
56
  const { retrieve } = await import("../../retriever/retriever.js");
57
57
  const labelsPath = options.labels
@@ -136,6 +136,56 @@ export function registerEvalTuneCommand(program) {
136
136
  console.log(` ${c.dim("vs. current config:")} Hit@5 ${sign(delta.hit5)}${(delta.hit5 * 100).toFixed(1)} pts · ` +
137
137
  `MRR ${sign(delta.mrr)}${delta.mrr.toFixed(3)} · nDCG@10 ${sign(delta.ndcg10)}${delta.ndcg10.toFixed(3)}`);
138
138
  }
139
+ // The point estimate alone is not evidence on a small label set — show
140
+ // the confidence interval and refuse to recommend a change the data
141
+ // does not support.
142
+ const paired = comparePaired(best, current, metric);
143
+ if (paired) {
144
+ const sign = (v) => (v >= 0 ? "+" : "");
145
+ console.log(` ${c.dim("paired bootstrap:")} Δ${paired.metric} ${sign(paired.meanDelta)}${paired.meanDelta.toFixed(3)} ` +
146
+ `(95% CI ${paired.ciLow.toFixed(3)} … ${paired.ciHigh.toFixed(3)}) · ` +
147
+ `${paired.improved} better / ${paired.regressed} worse / ${paired.unchanged} equal · ` +
148
+ `sign test p=${paired.signTestP.toFixed(2)}`);
149
+ if (paired.signTestP > 0.05) {
150
+ console.log(` ${c.warn("Weak evidence:")} the sign test over the discordant labels is not significant — ` +
151
+ "more labels are needed before changing the shipped default.");
152
+ }
153
+ if (paired.withinNoise) {
154
+ console.log(` ${c.warn("Within noise:")} the 95% CI includes 0 for ${labels.length} label(s) — ` +
155
+ "treat the winner as a hint, not a measurement. Add labels or check the signal split below.");
156
+ }
157
+ }
158
+ // Per-signal split: a fused number is only interpretable next to the
159
+ // quality of the two halves it mixes.
160
+ const signals = await measureSignalQuality({
161
+ labels,
162
+ root: cwd,
163
+ retrieveKeyword: async (label) => ctx.keywordIndex.search(label.query, (base.topK ?? 20) * 3),
164
+ retrieveVector: async (label) => retrieve(label.query, ctx.embedder, ctx.store, {
165
+ topK: base.topK,
166
+ minScore: 0,
167
+ keywordIndex: undefined,
168
+ keywordWeight: 0,
169
+ queryPrefix: ctx.config.embedding.queryPrefix,
170
+ }),
171
+ retrieveFused: async (label) => retrieve(label.query, ctx.embedder, ctx.store, {
172
+ topK: base.topK,
173
+ minScore: base.minScore,
174
+ keywordIndex: ctx.keywordIndex,
175
+ keywordWeight: base.keywordWeight,
176
+ symbolKeywordWeight: base.symbolKeywordWeight,
177
+ docKeywordDemotion: hybrid?.docKeywordDemotion,
178
+ testKeywordDemotion: hybrid?.testKeywordDemotion,
179
+ hybridEnabled: hybrid?.enabled,
180
+ queryPrefix: ctx.config.embedding.queryPrefix,
181
+ }),
182
+ });
183
+ console.log(`\n ${c.heading("Signal split")} ${c.dim("(file level, current config, minScore=0 for the single signals)")}`);
184
+ const signalRow = (name, q) => ` ${name.padEnd(14)} Hit@1 ${(q.hit1 * 100).toFixed(1)}% · Hit@5 ${(q.hit5 * 100).toFixed(1)}% · ` +
185
+ `Hit@10 ${(q.hit10 * 100).toFixed(1)}% · ${q.misses} miss`;
186
+ console.log(signalRow("BM25 only", signals.keyword));
187
+ console.log(signalRow("vector only", signals.vector));
188
+ console.log(signalRow(`fused kw=${base.keywordWeight}`, signals.fused));
139
189
  let written = false;
140
190
  if (options.write) {
141
191
  if (!ctx.configPath) {
@@ -193,6 +243,25 @@ export function registerEvalTuneCommand(program) {
193
243
  },
194
244
  },
195
245
  delta,
246
+ paired: paired
247
+ ? {
248
+ metric: paired.metric,
249
+ meanDelta: paired.meanDelta,
250
+ ciLow: paired.ciLow,
251
+ ciHigh: paired.ciHigh,
252
+ pBetter: paired.pBetter,
253
+ improved: paired.improved,
254
+ regressed: paired.regressed,
255
+ unchanged: paired.unchanged,
256
+ withinNoise: paired.withinNoise,
257
+ signTestP: paired.signTestP,
258
+ }
259
+ : null,
260
+ signals: {
261
+ keyword: signals.keyword,
262
+ vector: signals.vector,
263
+ fused: signals.fused,
264
+ },
196
265
  written,
197
266
  }));
198
267
  }
@@ -240,6 +240,7 @@ export function registerEvalGateCommand(program) {
240
240
  const retrieveOptions = {
241
241
  topK,
242
242
  minScore: ctx.config.retrieval.minScore,
243
+ minScoreMode: ctx.config.retrieval.minScoreMode,
243
244
  keywordIndex: ctx.keywordIndex,
244
245
  keywordWeight: hybrid?.keywordWeight,
245
246
  symbolKeywordWeight: hybrid?.symbolKeywordWeight,
@@ -51,6 +51,7 @@ export function registerQueryCommand(program) {
51
51
  const rawResults = await retrieve(query, embedder, store, {
52
52
  topK,
53
53
  minScore,
54
+ minScoreMode: config.retrieval.minScoreMode,
54
55
  keywordIndex,
55
56
  keywordWeight: hybridCfg?.keywordWeight,
56
57
  hybridEnabled: hybridCfg?.enabled,
@@ -294,8 +294,11 @@ export interface MemoryConfig {
294
294
  /**
295
295
  * Minimum raw similarity (0-1) for an existing quirk to count as "an
296
296
  * equivalent quirk already exists" and therefore suppress creation
297
- * (default 0.6). Deliberately below `toolErrorMinScore` (0.7): a probable
298
- * match still blocks a duplicate, it is just not injected on its own.
297
+ * (default 0.7). Deliberately EQUAL to `toolErrorMinScore`: measured on a
298
+ * real workspace, an error query scores 0.55-0.63 against arbitrary stored
299
+ * quirks, so a lower floor sits inside the noise band and matches unrelated
300
+ * quirks — which suppressed creation on every real failure. The exact
301
+ * duplicate check (same tool + same cause class) is done by metadata.
299
302
  */
300
303
  toolErrorCreateMinScore: number;
301
304
  /** Latency budget (ms) for the tool-error creation path (embed + store write); default 2000. */
@@ -306,6 +309,15 @@ export interface MemoryConfig {
306
309
  * created quirk carries no new information for that turn.
307
310
  */
308
311
  toolErrorInjectCreated: boolean;
312
+ /**
313
+ * Surface known tool gotchas in the system prompt BEFORE the model picks a
314
+ * tool (default true). A failure that already happened in an earlier
315
+ * session is injected as a "known tool gotcha", so the model does not repeat
316
+ * it. Only gotchas whose tool is in the session's tool catalog are injected.
317
+ */
318
+ toolErrorProactiveInject: boolean;
319
+ /** Max tool gotchas injected into the system prompt per request (default 5). */
320
+ toolErrorProactiveMaxItems: number;
309
321
  /** Decay settings for aging quirks. */
310
322
  decay: {
311
323
  /** Whether confidence decays over time. */
@@ -523,6 +535,18 @@ export interface RagConfig {
523
535
  topK: number;
524
536
  /** Minimum relevance score threshold (0-1). */
525
537
  minScore: number;
538
+ /**
539
+ * How `minScore` reads the score (default `"fused"`).
540
+ *
541
+ * The fused hybrid score's scale depends on `hybridSearch.keywordWeight`:
542
+ * a vector-only hit can never exceed `1 - kw`, so at `kw = 0.4` a fixed
543
+ * `minScore` of 0.65 silently demands a DUAL hit (13.5 of 20 results
544
+ * survive vs 17.7 at `kw = 0`). `"normalized"` divides the score by the
545
+ * maximum that result could reach, so the threshold keeps its meaning at
546
+ * every keyword weight. `"fused"` keeps today's behaviour — the default
547
+ * does not change silently; a default flip needs a bigger golden set.
548
+ */
549
+ minScoreMode?: "fused" | "normalized";
526
550
  /** Hybrid search (vector + keyword) configuration. */
527
551
  hybridSearch?: {
528
552
  /** Whether hybrid search is enabled. */
@@ -130,6 +130,7 @@ export const DEFAULT_CONFIG = {
130
130
  retrieval: {
131
131
  topK: 20,
132
132
  minScore: 0.35,
133
+ minScoreMode: "fused",
133
134
  hybridSearch: {
134
135
  enabled: true,
135
136
  keywordWeight: 0.4,
@@ -325,9 +326,11 @@ export const DEFAULT_CONFIG = {
325
326
  autoCreateOnToolError: true,
326
327
  toolErrorCreateMaxPerSession: 2,
327
328
  toolErrorCreateCooldownMs: 600_000,
328
- toolErrorCreateMinScore: 0.6,
329
+ toolErrorCreateMinScore: 0.7,
329
330
  toolErrorCreateLatencyBudgetMs: 2000,
330
331
  toolErrorInjectCreated: false,
332
+ toolErrorProactiveInject: true,
333
+ toolErrorProactiveMaxItems: 5,
331
334
  decay: {
332
335
  enabled: false,
333
336
  halfLifeDays: 30,
@@ -487,6 +490,12 @@ export function validateConfig(config) {
487
490
  if (config.memory?.toolErrorCreateLatencyBudgetMs != null && config.memory.toolErrorCreateLatencyBudgetMs < 0) {
488
491
  warnings.push("memory.toolErrorCreateLatencyBudgetMs must be >= 0");
489
492
  }
493
+ if (config.memory?.toolErrorProactiveMaxItems != null && config.memory.toolErrorProactiveMaxItems < 0) {
494
+ warnings.push("memory.toolErrorProactiveMaxItems must be >= 0");
495
+ }
496
+ if (config.retrieval?.minScoreMode != null && !["fused", "normalized"].includes(config.retrieval.minScoreMode)) {
497
+ warnings.push(`retrieval.minScoreMode "${config.retrieval.minScoreMode}" — expected "fused" or "normalized"`);
498
+ }
490
499
  if (config.memory?.decay && config.memory.decay.halfLifeDays <= 0) {
491
500
  warnings.push("memory.decay.halfLifeDays must be > 0");
492
501
  }
@@ -150,6 +150,12 @@ export interface SearchExplanation {
150
150
  vectorRank?: number;
151
151
  /** Rank (0-indexed) in the keyword index results. */
152
152
  keywordRank?: number;
153
+ /**
154
+ * The highest score a result with these legs could reach (rank 0 in
155
+ * each). Set only in `minScoreMode: "normalized"`, where the reported
156
+ * score is the fused score divided by this value.
157
+ */
158
+ maxReachableScore?: number;
153
159
  /** Cross-encoder relevance from the optional rerank stage. Higher means more
154
160
  * relevant. NOT a calibrated probability (provider-dependent, llama-server
155
161
  * semantics) and NOT comparable to the RRF-normalized fused components.
@@ -14,7 +14,7 @@
14
14
  */
15
15
  import type { RagConfig } from "../core/config.js";
16
16
  import type { SearchResult } from "../core/interfaces.js";
17
- import type { GoldenLabel, GoldenOutcome, GoldenSummary } from "./golden-set.js";
17
+ import { type GoldenLabel, type GoldenOutcome, type GoldenSummary } from "./golden-set.js";
18
18
  /** One point in the retrieval search space. */
19
19
  export interface TuningCandidate {
20
20
  /** RRF keyword weight: 0 = pure vector ranking, 1 = pure keyword ranking. */
@@ -92,6 +92,10 @@ export declare function runTuner(opts: RunTunerOptions): Promise<TuningResult[]>
92
92
  * Ranked by the primary metric, then by Hit@5, MRR and Hit@1 so a tie on the
93
93
  * primary metric still resolves to the more useful configuration.
94
94
  *
95
+ * The argmax alone is NOT evidence — on a small label set it is often within
96
+ * noise of the runner-up. Use {@link comparePaired} for the confidence
97
+ * interval before changing anything.
98
+ *
95
99
  * @param results - The measured candidates.
96
100
  * @param metric - The primary metric (default `ndcg10`).
97
101
  * @returns The winning result, or `undefined` when there are no results.
@@ -105,6 +109,103 @@ export declare function selectBestTuning(results: TuningResult[], metric?: Tunin
105
109
  * @returns The per-metric deltas (winner minus current).
106
110
  */
107
111
  export declare function tuningDelta(best: TuningResult, current: TuningResult | undefined): Record<string, number>;
112
+ /** Paired comparison of two candidates measured on the same labels. */
113
+ export interface PairedComparison {
114
+ /** The metric the comparison is based on. */
115
+ metric: TuningMetric;
116
+ /** Mean per-label difference (first candidate minus second). */
117
+ meanDelta: number;
118
+ /** Lower bound of the 95% bootstrap confidence interval. */
119
+ ciLow: number;
120
+ /** Upper bound of the 95% bootstrap confidence interval. */
121
+ ciHigh: number;
122
+ /** Share of bootstrap resamples in which the first candidate wins. */
123
+ pBetter: number;
124
+ /** Labels where the first candidate scored higher. */
125
+ improved: number;
126
+ /** Labels where the second candidate scored higher. */
127
+ regressed: number;
128
+ /** Labels where both scored the same. */
129
+ unchanged: number;
130
+ /** True when the confidence interval includes 0 — the difference is within noise. */
131
+ withinNoise: boolean;
132
+ /**
133
+ * Two-sided exact sign-test p-value over the discordant labels.
134
+ *
135
+ * The bootstrap CI is the optimistic bound: it weights a label by how much it
136
+ * moved. The sign test only asks "how many labels flipped?", which is the
137
+ * conservative reading of a small label set — a 19-vs-11 split is still
138
+ * p ≈ 0.20. Report both.
139
+ */
140
+ signTestP: number;
141
+ }
142
+ /**
143
+ * Two-sided exact sign-test p-value for `wins` successes out of `wins + losses`
144
+ * trials under a 0.5 null.
145
+ *
146
+ * @param wins - Labels where the first candidate scored higher.
147
+ * @param losses - Labels where the second candidate scored higher.
148
+ * @returns The p-value in [0, 1]; 1 when there are no discordant labels.
149
+ */
150
+ export declare function signTestP(wins: number, losses: number): number;
151
+ /**
152
+ * Compare two candidates with a paired bootstrap over the labels.
153
+ *
154
+ * The label set is small (48 by default), so a point estimate on its own is
155
+ * not evidence: a 10-point Hit@5 gap is a handful of labels, and a sign test
156
+ * on the discordant labels is the honest lower bound. This returns the
157
+ * confidence interval, the win/loss counts, and an explicit `withinNoise`
158
+ * flag so a caller can refuse to recommend a change the data does not support.
159
+ *
160
+ * @param better - The candidate to test.
161
+ * @param other - The candidate to compare against (usually the current config).
162
+ * @param metric - The metric to compare on (default `ndcg10`).
163
+ * @param iterations - Bootstrap resamples (default 20000).
164
+ * @param rng - Random source (injectable for deterministic tests).
165
+ * @returns The paired comparison, or `undefined` when there is nothing to compare.
166
+ */
167
+ export declare function comparePaired(better: TuningResult, other: TuningResult | undefined, metric?: TuningMetric, iterations?: number, rng?: () => number): PairedComparison | undefined;
168
+ /** Aggregate file-level quality of one retrieval signal. */
169
+ export interface SignalQuality {
170
+ /** Mean Hit@1 over the labels. */
171
+ hit1: number;
172
+ /** Mean Hit@5 over the labels. */
173
+ hit5: number;
174
+ /** Mean Hit@10 over the labels. */
175
+ hit10: number;
176
+ /** Labels where the expected file was not found at all. */
177
+ misses: number;
178
+ }
179
+ /** Per-signal quality: the keyword half and the vector half measured apart. */
180
+ export interface SignalComparison {
181
+ /** Quality of the keyword (BM25) signal alone. */
182
+ keyword: SignalQuality;
183
+ /** Quality of the vector signal alone. */
184
+ vector: SignalQuality;
185
+ /** Quality of the fused pipeline. */
186
+ fused: SignalQuality;
187
+ }
188
+ /**
189
+ * Measure each retrieval signal on its own, so a fused number can be
190
+ * interpreted instead of just believed.
191
+ *
192
+ * A fused ranking can only be as good as the mix of its parts: when the
193
+ * keyword half answers far fewer queries than the vector half, any weight
194
+ * given to it dilutes the result. This measures the parts.
195
+ *
196
+ * @param opts - Labels, root, and one retrieve closure per signal.
197
+ * @returns The per-signal quality.
198
+ */
199
+ export declare function measureSignalQuality(opts: {
200
+ labels: GoldenLabel[];
201
+ root: string;
202
+ /** Retrieve with the keyword signal only. */
203
+ retrieveKeyword: (label: GoldenLabel) => Promise<SearchResult[]>;
204
+ /** Retrieve with the vector signal only. */
205
+ retrieveVector: (label: GoldenLabel) => Promise<SearchResult[]>;
206
+ /** Retrieve with the full pipeline. */
207
+ retrieveFused: (label: GoldenLabel) => Promise<SearchResult[]>;
208
+ }): Promise<SignalComparison>;
108
209
  /**
109
210
  * Write the winning weights into a copy of the configuration.
110
211
  *
@@ -1,4 +1,5 @@
1
1
  import { runGoldenGate, } from "./golden-gate.js";
2
+ import { evaluateGoldenResults, mean, } from "./golden-set.js";
2
3
  /** The metric used to pick the winner, and the tie-breakers after it. */
3
4
  const METRIC_ORDER = {
4
5
  ndcg10: (s) => s.ndcg10,
@@ -85,6 +86,10 @@ export async function runTuner(opts) {
85
86
  * Ranked by the primary metric, then by Hit@5, MRR and Hit@1 so a tie on the
86
87
  * primary metric still resolves to the more useful configuration.
87
88
  *
89
+ * The argmax alone is NOT evidence — on a small label set it is often within
90
+ * noise of the runner-up. Use {@link comparePaired} for the confidence
91
+ * interval before changing anything.
92
+ *
88
93
  * @param results - The measured candidates.
89
94
  * @param metric - The primary metric (default `ndcg10`).
90
95
  * @returns The winning result, or `undefined` when there are no results.
@@ -128,6 +133,136 @@ export function tuningDelta(best, current) {
128
133
  ndcg10: best.summary.ndcg10 - current.summary.ndcg10,
129
134
  };
130
135
  }
136
+ /** Per-label value of a tuning metric. */
137
+ function metricValue(outcome, metric) {
138
+ switch (metric) {
139
+ case "hit5": return outcome.hitAt5;
140
+ case "hit1": return outcome.hitAt1;
141
+ case "mrr": return outcome.rr;
142
+ default: return outcome.ndcg10;
143
+ }
144
+ }
145
+ /**
146
+ * Two-sided exact sign-test p-value for `wins` successes out of `wins + losses`
147
+ * trials under a 0.5 null.
148
+ *
149
+ * @param wins - Labels where the first candidate scored higher.
150
+ * @param losses - Labels where the second candidate scored higher.
151
+ * @returns The p-value in [0, 1]; 1 when there are no discordant labels.
152
+ */
153
+ export function signTestP(wins, losses) {
154
+ const n = wins + losses;
155
+ if (n === 0)
156
+ return 1;
157
+ const k = Math.max(wins, losses);
158
+ let tail = 0;
159
+ for (let i = k; i <= n; i++) {
160
+ tail += binomial(n, i);
161
+ }
162
+ const p = (tail * 2) / Math.pow(2, n);
163
+ return Math.min(1, p);
164
+ }
165
+ /** Binomial coefficient C(n, k). */
166
+ function binomial(n, k) {
167
+ let result = 1;
168
+ for (let i = 0; i < k; i++) {
169
+ result = (result * (n - i)) / (i + 1);
170
+ }
171
+ return Math.round(result);
172
+ }
173
+ /**
174
+ * Compare two candidates with a paired bootstrap over the labels.
175
+ *
176
+ * The label set is small (48 by default), so a point estimate on its own is
177
+ * not evidence: a 10-point Hit@5 gap is a handful of labels, and a sign test
178
+ * on the discordant labels is the honest lower bound. This returns the
179
+ * confidence interval, the win/loss counts, and an explicit `withinNoise`
180
+ * flag so a caller can refuse to recommend a change the data does not support.
181
+ *
182
+ * @param better - The candidate to test.
183
+ * @param other - The candidate to compare against (usually the current config).
184
+ * @param metric - The metric to compare on (default `ndcg10`).
185
+ * @param iterations - Bootstrap resamples (default 20000).
186
+ * @param rng - Random source (injectable for deterministic tests).
187
+ * @returns The paired comparison, or `undefined` when there is nothing to compare.
188
+ */
189
+ export function comparePaired(better, other, metric = "ndcg10", iterations = 20_000, rng = Math.random) {
190
+ if (!other)
191
+ return undefined;
192
+ const n = Math.min(better.outcomes.length, other.outcomes.length);
193
+ if (n === 0)
194
+ return undefined;
195
+ const diffs = [];
196
+ let improved = 0;
197
+ let regressed = 0;
198
+ let unchanged = 0;
199
+ for (let i = 0; i < n; i++) {
200
+ const d = metricValue(better.outcomes[i], metric) - metricValue(other.outcomes[i], metric);
201
+ diffs.push(d);
202
+ if (d > 1e-9)
203
+ improved++;
204
+ else if (d < -1e-9)
205
+ regressed++;
206
+ else
207
+ unchanged++;
208
+ }
209
+ const meanDelta = diffs.reduce((s, d) => s + d, 0) / n;
210
+ const boots = [];
211
+ for (let i = 0; i < iterations; i++) {
212
+ let sum = 0;
213
+ for (let j = 0; j < n; j++)
214
+ sum += diffs[Math.floor(rng() * n)];
215
+ boots.push(sum / n);
216
+ }
217
+ boots.sort((a, b) => a - b);
218
+ const at = (p) => boots[Math.min(boots.length - 1, Math.max(0, Math.floor(p * (boots.length - 1))))];
219
+ const ciLow = at(0.025);
220
+ const ciHigh = at(0.975);
221
+ const pBetter = boots.filter((b) => b > 0).length / boots.length;
222
+ return {
223
+ metric,
224
+ meanDelta,
225
+ ciLow,
226
+ ciHigh,
227
+ pBetter,
228
+ improved,
229
+ regressed,
230
+ unchanged,
231
+ withinNoise: ciLow <= 0 && ciHigh >= 0,
232
+ signTestP: signTestP(improved, regressed),
233
+ };
234
+ }
235
+ /**
236
+ * Measure each retrieval signal on its own, so a fused number can be
237
+ * interpreted instead of just believed.
238
+ *
239
+ * A fused ranking can only be as good as the mix of its parts: when the
240
+ * keyword half answers far fewer queries than the vector half, any weight
241
+ * given to it dilutes the result. This measures the parts.
242
+ *
243
+ * @param opts - Labels, root, and one retrieve closure per signal.
244
+ * @returns The per-signal quality.
245
+ */
246
+ export async function measureSignalQuality(opts) {
247
+ const summarize = async (retrieve) => {
248
+ const metrics = [];
249
+ for (const label of opts.labels) {
250
+ const results = await retrieve(label);
251
+ metrics.push(evaluateGoldenResults(opts.root, label, results));
252
+ }
253
+ return {
254
+ hit1: mean(metrics.map((m) => m.hitAt1)),
255
+ hit5: mean(metrics.map((m) => m.hitAt5)),
256
+ hit10: mean(metrics.map((m) => m.hitAt10)),
257
+ misses: metrics.filter((m) => m.firstHitRank === null).length,
258
+ };
259
+ };
260
+ return {
261
+ keyword: await summarize(opts.retrieveKeyword),
262
+ vector: await summarize(opts.retrieveVector),
263
+ fused: await summarize(opts.retrieveFused),
264
+ };
265
+ }
131
266
  /**
132
267
  * Write the winning weights into a copy of the configuration.
133
268
  *
@@ -91,6 +91,7 @@ export async function handleSearchSemantic(params, embedder, store, cfg, keyword
91
91
  const retrieveOpts = {
92
92
  topK,
93
93
  minScore: cfg.retrieval.minScore,
94
+ minScoreMode: cfg.retrieval.minScoreMode,
94
95
  keywordIndex,
95
96
  keywordWeight: cfg.retrieval.hybridSearch?.keywordWeight,
96
97
  hybridEnabled: cfg.retrieval.hybridSearch?.enabled,
package/dist/plugin.js CHANGED
@@ -29,10 +29,10 @@ import { countTokens } from "./eval/token-counter.js";
29
29
  import { checkForUpdate, getCurrentVersion, installLatestUpdate } from "./core/version-check.js";
30
30
  import { loadAutoUpdateState, saveAutoUpdateState, shouldAttemptInstall } from "./core/auto-update-state.js";
31
31
  import { destroyAllPooledConnections } from "./embedder/http.js";
32
- import { listQuirks, lintQuirks, recallQuirks, sharedWords, reconcileQuirks, addQuirk } from "./quirks/quirk-store.js";
32
+ import { listQuirks, lintQuirks, recallQuirks, sharedWords, reconcileQuirks, addQuirk, listToolErrorQuirks } from "./quirks/quirk-store.js";
33
33
  import { autoCaptureQuirks } from "./quirks/auto-capture.js";
34
34
  import { createToolErrorInjectionState, recallToolErrorQuirks, formatToolErrorQuirkBlock, toToolErrorCaptureEntry, } from "./quirks/tool-error-recall.js";
35
- import { buildToolErrorQuirkContent, createToolErrorCreateState, createToolErrorQuirk, TOOL_ERROR_QUIRK_TAG, } from "./quirks/tool-error-create.js";
35
+ import { buildToolErrorQuirkContent, classifyToolErrorCause, createToolErrorCreateState, createToolErrorQuirk, formatToolGotchaBlock, hasEquivalentToolGotcha, selectToolGotchas, TOOL_ERROR_CREATE_DEFAULTS, TOOL_ERROR_QUIRK_TAG, } from "./quirks/tool-error-create.js";
36
36
  import { buildSystemGuidanceLines } from "./opencode/system-guidance.js";
37
37
  import { existsSync, readFileSync, readdirSync, statSync, unlinkSync } from "node:fs";
38
38
  import path from "node:path";
@@ -42,6 +42,8 @@ const MAX_SESSION_MAP_SIZE = 50;
42
42
  const MAX_SESSION_TOOL_RESULTS = 20;
43
43
  /** Pending tool-error quirk blocks older than this (ms) are dropped instead of injected. */
44
44
  const TOOL_ERROR_PENDING_TTL_MS = 120_000;
45
+ /** How long the tool-error quirk list is cached before it is re-read (ms). */
46
+ const TOOL_GOTCHA_CACHE_MS = 30_000;
45
47
  /** Cache of loaded RAG configurations keyed by workspace directory. */
46
48
  const configCache = new Map();
47
49
  /** Active background indexer instances keyed by workspace directory. */
@@ -431,10 +433,10 @@ function buildRetrievalQuery(hints) {
431
433
  * Perform a retrieval query against the vector store with the given parameters.
432
434
  * Returns an empty array for blank queries.
433
435
  */
434
- async function retrieveContext(query, embedder, store, topK, retrieveFn = retrieve, minScore = 0, keywordIndex, keywordWeight, queryPrefix, explain = false, hybridEnabled, filter, reranker, reranking) {
436
+ async function retrieveContext(query, embedder, store, topK, retrieveFn = retrieve, minScore = 0, keywordIndex, keywordWeight, queryPrefix, explain = false, hybridEnabled, filter, reranker, reranking, minScoreMode) {
435
437
  if (query.trim().length === 0)
436
438
  return [];
437
- return retrieveFn(query, embedder, store, { topK, minScore, keywordIndex, keywordWeight, hybridEnabled, queryPrefix, explain, filter, reranker, reranking });
439
+ return retrieveFn(query, embedder, store, { topK, minScore, keywordIndex, keywordWeight, hybridEnabled, queryPrefix, explain, filter, reranker, reranking, minScoreMode });
438
440
  }
439
441
  /**
440
442
  * Load results from one or two queries (primary + optional extra), optimize
@@ -443,11 +445,12 @@ async function retrieveContext(query, embedder, store, topK, retrieveFn = retrie
443
445
  */
444
446
  async function loadRetrievedResults(query, embedder, store, cfg, retrieveFn = retrieve, topK = cfg.retrieval.topK, extraQuery, keywordIndex, queryPrefix, explain = false, filter, reranker, reranking) {
445
447
  const minScore = cfg.retrieval.minScore;
448
+ const minScoreMode = cfg.retrieval.minScoreMode;
446
449
  const kw = cfg.retrieval.hybridSearch?.keywordWeight;
447
450
  const hybridEnabled = cfg.retrieval.hybridSearch?.enabled;
448
- const primaryResults = await retrieveContext(query, embedder, store, topK, retrieveFn, minScore, keywordIndex, kw, queryPrefix, explain, hybridEnabled, filter, reranker, reranking);
451
+ const primaryResults = await retrieveContext(query, embedder, store, topK, retrieveFn, minScore, keywordIndex, kw, queryPrefix, explain, hybridEnabled, filter, reranker, reranking, minScoreMode);
449
452
  const extraResults = extraQuery
450
- ? await retrieveContext(extraQuery, embedder, store, topK, retrieveFn, minScore, keywordIndex, kw, queryPrefix, explain, hybridEnabled, filter, reranker, reranking)
453
+ ? await retrieveContext(extraQuery, embedder, store, topK, retrieveFn, minScore, keywordIndex, kw, queryPrefix, explain, hybridEnabled, filter, reranker, reranking, minScoreMode)
451
454
  : [];
452
455
  const optCfg = cfg.retrieval.contextOptimization ?? DEFAULT_CONTEXT_OPTIMIZATION;
453
456
  return optimizeContext([...primaryResults, ...extraResults], { topK, config: optCfg })
@@ -646,6 +649,9 @@ export function createRagHooks(options) {
646
649
  // plugin writes itself when a failure has no matching memory.
647
650
  const sessionToolErrorCreateState = new Map();
648
651
  const sessionPendingToolErrorQuirk = new Map();
652
+ // Tool-gotcha cache: the tool-error quirks change rarely, so the audit log is
653
+ // read at most every TOOL_GOTCHA_CACHE_MS instead of on every model request.
654
+ const toolGotchaCache = new Map();
649
655
  // Evaluation session logger — captures OpenCode events for analysis
650
656
  const sessionLogger = createSessionLogger(options.storePath);
651
657
  appendDebugLog(options.logFilePath, {
@@ -653,6 +659,23 @@ export function createRagHooks(options) {
653
659
  message: "OpenCode plugin initialized",
654
660
  });
655
661
  const readOverride = getEffectiveCfg().openCode.readOverride === true;
662
+ /**
663
+ * Read the auto-created tool-error quirks, cached briefly.
664
+ *
665
+ * Runs on every model request (the proactive tool-gotcha injection), so it
666
+ * must not hit the disk every time. The cache is invalidated explicitly
667
+ * when a new tool-error quirk is created.
668
+ */
669
+ async function getToolErrorQuirks(deps) {
670
+ const key = options.storePath;
671
+ const now = Date.now();
672
+ const cached = toolGotchaCache.get(key);
673
+ if (cached && now - cached.fetchedAt < TOOL_GOTCHA_CACHE_MS)
674
+ return cached.quirks;
675
+ const quirks = await listToolErrorQuirks(deps);
676
+ toolGotchaCache.set(key, { quirks, fetchedAt: now });
677
+ return quirks;
678
+ }
656
679
  const retrievalTool = tool({
657
680
  description: "Search the indexed codebase by meaning, not just keywords. " +
658
681
  "ESSENTIAL before any code task — replaces blind reading with targeted retrieval. " +
@@ -997,6 +1020,11 @@ export function createRagHooks(options) {
997
1020
  // `output.error` is set by the V2 adapter's error branch; the status
998
1021
  // check keeps this working if a runtime reports errors via input only.
999
1022
  const errorText = output.error ?? (input.status === "error" ? output.output : undefined);
1023
+ appendDebugLog(options.logFilePath, {
1024
+ scope: "quirk.tool-error",
1025
+ message: `tool "${input.tool}" status=${input.status ?? "unset"} error=${errorText ? "yes" : "no"}`,
1026
+ severity: "debug",
1027
+ }, getEffectiveCfg().logging.level);
1000
1028
  if (errorText && errorText.length > 0) {
1001
1029
  const info = { tool: input.tool, error: errorText, sessionID: input.sessionID };
1002
1030
  results.push(toToolErrorCaptureEntry(info));
@@ -1057,8 +1085,17 @@ export function createRagHooks(options) {
1057
1085
  state.record(info, hits.map((h) => h.id));
1058
1086
  // No candidate at all → nothing in memory covers this failure.
1059
1087
  // Record one so a future occurrence can be answered from memory.
1088
+ // The duplicate check is EXACT (same tool + same cause class), not
1089
+ // a loose similarity floor: a floor low enough to catch a genuine
1090
+ // duplicate also catches unrelated quirks, which would suppress
1091
+ // creation on every real failure.
1060
1092
  let createdHit;
1061
- if (recallResult.candidates.length === 0 && memCfg.autoCreateOnToolError) {
1093
+ let createdQuirk = false;
1094
+ const causeClass = classifyToolErrorCause(info.error)?.cls;
1095
+ const equivalent = recallResult.candidates.length > 0 ||
1096
+ (causeClass !== undefined &&
1097
+ hasEquivalentToolGotcha(await getToolErrorQuirks(qDeps), info.tool, causeClass));
1098
+ if (!equivalent && memCfg.autoCreateOnToolError) {
1062
1099
  const createState = sessionToolErrorCreateState.get(input.sessionID) ?? createToolErrorCreateState({
1063
1100
  maxPerSession: memCfg.toolErrorCreateMaxPerSession ?? 2,
1064
1101
  cooldownMs: memCfg.toolErrorCreateCooldownMs ?? 600_000,
@@ -1078,6 +1115,10 @@ export function createRagHooks(options) {
1078
1115
  clearTimeout(createTimer);
1079
1116
  if (created?.created && created.id) {
1080
1117
  createState.record(info, true);
1118
+ createdQuirk = true;
1119
+ // The proactive gotcha list changed — drop the cache so
1120
+ // the next model request sees the new gotcha.
1121
+ toolGotchaCache.delete(options.storePath);
1081
1122
  appendDebugLog(options.logFilePath, {
1082
1123
  scope: "quirk.tool-error",
1083
1124
  message: `created quirk ${created.id} for tool "${input.tool}"`,
@@ -1114,6 +1155,22 @@ export function createRagHooks(options) {
1114
1155
  ts: Date.now(),
1115
1156
  }, MAX_SESSION_MAP_SIZE);
1116
1157
  }
1158
+ appendDebugLog(options.logFilePath, {
1159
+ scope: "quirk.tool-error",
1160
+ message: `tool error "${input.tool}": candidates=${recallResult.candidates.length} injected=${finalHits.length} created=${createdQuirk ? "yes" : "no"}`,
1161
+ });
1162
+ }
1163
+ else if (recallResult) {
1164
+ appendDebugLog(options.logFilePath, {
1165
+ scope: "quirk.tool-error",
1166
+ message: `tool error "${input.tool}": skipped (cooldown or budget)`,
1167
+ });
1168
+ }
1169
+ else {
1170
+ appendDebugLog(options.logFilePath, {
1171
+ scope: "quirk.tool-error",
1172
+ message: `tool error "${input.tool}": skipped (recall timed out after ${budgetMs}ms)`,
1173
+ });
1117
1174
  }
1118
1175
  }
1119
1176
  }
@@ -1137,6 +1194,37 @@ export function createRagHooks(options) {
1137
1194
  ? "system guidance injected"
1138
1195
  : `system guidance skipped (${guidanceEnabled ? `no RAG tools in session; agent=${input?.agent ?? "unknown"}` : "injectSystemPrompt=false"})`,
1139
1196
  });
1197
+ // Which tools can this session actually call? The catalog is the set of
1198
+ // tools the model may invoke, so it is the right filter for tool-scoped
1199
+ // memory (a gotcha about a tool that is not callable is not actionable).
1200
+ appendDebugLog(options.logFilePath, {
1201
+ scope: "experimental.chat.system.transform",
1202
+ message: `tool catalog (${Object.keys(input?.tools ?? {}).length}): ${Object.keys(input?.tools ?? {}).join(", ")}`,
1203
+ severity: "debug",
1204
+ }, getEffectiveCfg().logging.level);
1205
+ // Proactive tool gotchas: surface what earlier failures taught us BEFORE
1206
+ // the model picks a tool, so a known failure is not repeated in a later
1207
+ // session. Scoped to the tools this session can actually call.
1208
+ try {
1209
+ const toolNames = Object.keys(input?.tools ?? {});
1210
+ if (toolNames.length > 0) {
1211
+ const memCfg = getEffectiveCfg().memory;
1212
+ if (memCfg?.enabled && memCfg.toolErrorProactiveInject) {
1213
+ const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
1214
+ const gotchas = selectToolGotchas(await getToolErrorQuirks(qDeps), toolNames, memCfg.toolErrorProactiveMaxItems ?? TOOL_ERROR_CREATE_DEFAULTS.maxProactiveItems);
1215
+ if (gotchas.length > 0) {
1216
+ output.system.unshift(formatToolGotchaBlock(gotchas));
1217
+ appendDebugLog(options.logFilePath, {
1218
+ scope: "quirk.tool-error",
1219
+ message: `proactively injected ${gotchas.length} tool gotcha(s): ${gotchas.map((g) => g.tool).join(", ")}`,
1220
+ });
1221
+ }
1222
+ }
1223
+ }
1224
+ }
1225
+ catch {
1226
+ // Non-critical — must never throw
1227
+ }
1140
1228
  if (injectGuidance) {
1141
1229
  const guidance = buildSystemGuidanceLines({ promptEnforcement: !!cfg.memory?.promptEnforcement, decisionEnabled: !!cfg.decision?.enabled, routeBeforeAsking: !!cfg.decision?.routeBeforeAsking });
1142
1230
  output.system.unshift(guidance.join("\n"));
@@ -1,6 +1,6 @@
1
1
  import type { EmbeddingProvider, VectorStore, KeywordIndex, SearchResult } from "../core/interfaces.js";
2
2
  import type { RagConfig } from "../core/config.js";
3
- import type { Quirk, QuirkInput } from "./types.js";
3
+ import { type Quirk, type QuirkInput } from "./types.js";
4
4
  /** Dependencies required by all quirk-store operations. */
5
5
  export interface QuirkStoreDeps {
6
6
  embedder: EmbeddingProvider;
@@ -104,6 +104,19 @@ export declare function recallQuirks(deps: QuirkStoreDeps, query: string, option
104
104
  * similarity threshold. */
105
105
  explain?: boolean;
106
106
  }): Promise<SearchResult[]>;
107
+ /**
108
+ * List the auto-created tool-error quirks, newest first.
109
+ *
110
+ * Reads the audit log directly instead of going through `listQuirks`: that
111
+ * function falls back to scanning the whole vector store when
112
+ * `quirks.jsonl` is missing, which is far too expensive for a path that runs
113
+ * on every model request. A tool-error quirk is always appended to the audit
114
+ * log, so an absent log means there are none.
115
+ *
116
+ * @param deps - The quirk store dependencies.
117
+ * @returns The tool-error quirks, newest `lastObserved` first.
118
+ */
119
+ export declare function listToolErrorQuirks(deps: QuirkStoreDeps): Promise<Quirk[]>;
107
120
  /** Lint quirks for low confidence, staleness, duplicates, and orphan source refs. */
108
121
  export declare function lintQuirks(deps: QuirkStoreDeps): Promise<string[]>;
109
122
  /** Simple Jaccard-based lexical similarity (word overlap). */
@@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto";
3
3
  import path from "node:path";
4
4
  import { retrieve } from "../retriever/retriever.js";
5
5
  import { isQuirkAllowed } from "./monitor.js";
6
+ import { TOOL_ERROR_QUIRK_TAG } from "./types.js";
6
7
  export const QUIRK_FILE_PREFIX = "quirk:";
7
8
  const QUIKK_JSONL = "quirks.jsonl";
8
9
  function jsonlPath(storePath) {
@@ -562,6 +563,32 @@ export async function recallQuirks(deps, query, options) {
562
563
  filtered.sort((a, b) => b.score - a.score);
563
564
  return filtered.slice(0, topK);
564
565
  }
566
+ /**
567
+ * List the auto-created tool-error quirks, newest first.
568
+ *
569
+ * Reads the audit log directly instead of going through `listQuirks`: that
570
+ * function falls back to scanning the whole vector store when
571
+ * `quirks.jsonl` is missing, which is far too expensive for a path that runs
572
+ * on every model request. A tool-error quirk is always appended to the audit
573
+ * log, so an absent log means there are none.
574
+ *
575
+ * @param deps - The quirk store dependencies.
576
+ * @returns The tool-error quirks, newest `lastObserved` first.
577
+ */
578
+ export async function listToolErrorQuirks(deps) {
579
+ const tagged = (q) => (q.tags ?? []).includes(TOOL_ERROR_QUIRK_TAG);
580
+ if (isMemoryStore(deps.storePath)) {
581
+ return [...memQuirksFor(deps.storePath).values()]
582
+ .filter(tagged)
583
+ .sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
584
+ }
585
+ const jp = jsonlPath(deps.storePath);
586
+ if (!existsSync(jp))
587
+ return [];
588
+ return readJsonl(jp)
589
+ .filter(tagged)
590
+ .sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
591
+ }
565
592
  /** Lint quirks for low confidence, staleness, duplicates, and orphan source refs. */
566
593
  export async function lintQuirks(deps) {
567
594
  const issues = [];
@@ -21,19 +21,21 @@
21
21
  * caller logs it.
22
22
  */
23
23
  import type { Quirk, QuirkInput } from "./types.js";
24
+ import { TOOL_ERROR_AUTO_TAG, TOOL_ERROR_CAUSE_TAG_PREFIX, TOOL_ERROR_QUIRK_TAG, TOOL_ERROR_TOOL_TAG_PREFIX } from "./types.js";
24
25
  import { type ToolErrorInfo } from "./tool-error-recall.js";
25
- /** Tag every auto-created tool-error quirk carries. */
26
- export declare const TOOL_ERROR_QUIRK_TAG = "tool-error";
27
- /** Second tag marking the quirk as machine-created (not agent-authored). */
28
- export declare const TOOL_ERROR_AUTO_TAG = "auto-created";
26
+ export { TOOL_ERROR_AUTO_TAG, TOOL_ERROR_CAUSE_TAG_PREFIX, TOOL_ERROR_QUIRK_TAG, TOOL_ERROR_TOOL_TAG_PREFIX, };
29
27
  /** Defaults for the tool-error creation path. */
30
28
  export declare const TOOL_ERROR_CREATE_DEFAULTS: {
31
29
  /**
32
30
  * Minimum raw similarity for a candidate to count as "an equivalent quirk
33
- * already exists" (and therefore suppress creation). Default 0.6 — below
34
- * the 0.7 injection bar, so a probable match still blocks a duplicate.
31
+ * already exists" (and therefore suppress creation). Default 0.7 — the
32
+ * injection bar. Measured: an error query scores 0.55-0.63 against
33
+ * ARBITRARY stored quirks, so a lower floor sits inside the noise band and
34
+ * matches unrelated quirks, which suppresses creation on every real failure.
35
+ * The exact duplicate check is done by metadata instead — see
36
+ * `hasEquivalentToolGotcha`.
35
37
  */
36
- readonly minScore: 0.6;
38
+ readonly minScore: 0.7;
37
39
  /** Maximum quirks created per session. Default 2. */
38
40
  readonly maxPerSession: 2;
39
41
  /** Cooldown per normalized error signature in ms. Default 10 minutes. */
@@ -46,7 +48,33 @@ export declare const TOOL_ERROR_CREATE_DEFAULTS: {
46
48
  readonly confidence: 0.6;
47
49
  /** Errors shorter than this carry no information — nothing is recorded. */
48
50
  readonly minErrorChars: 15;
51
+ /** Max tool gotchas injected proactively into the system prompt. Default 5. */
52
+ readonly maxProactiveItems: 5;
49
53
  };
54
+ /**
55
+ * Cause classes that are worth surfacing BEFORE the tool is called.
56
+ *
57
+ * A transient failure (rate limit, timeout) must NOT be warned about in
58
+ * advance — the next call will not fail the same way, and the warning would
59
+ * be misleading. An unclassified error is excluded for the same reason: we do
60
+ * not know whether it is durable, and a raw error dump says what happened,
61
+ * not what to do.
62
+ */
63
+ export declare const PROACTIVE_CAUSE_CLASSES: ReadonlySet<string>;
64
+ /** A classified tool-error cause. */
65
+ export interface ToolErrorCause {
66
+ /** Stable class name — also used as the `cause:<class>` tag. */
67
+ cls: string;
68
+ /** Human-readable guidance appended to the quirk content. */
69
+ cause: string;
70
+ }
71
+ /**
72
+ * Classify an error message into a cause class plus its guidance.
73
+ *
74
+ * Returns `undefined` for unknown errors — the raw text is then the whole
75
+ * story.
76
+ */
77
+ export declare function classifyToolErrorCause(error: string): ToolErrorCause | undefined;
50
78
  /**
51
79
  * Classify an error message into a short, deterministic cause.
52
80
  *
@@ -109,3 +137,47 @@ export interface ToolErrorQuirkCreateResult {
109
137
  * `created: false` with a reason so the caller can log it and carry on.
110
138
  */
111
139
  export declare function createToolErrorQuirk(deps: ToolErrorQuirkDeps, info: ToolErrorInfo, content?: string): Promise<ToolErrorQuirkCreateResult>;
140
+ /** A stored tool gotcha, ready to be surfaced proactively. */
141
+ export interface ToolGotcha {
142
+ /** The tool the gotcha is about (lowercase). */
143
+ tool: string;
144
+ /** The quirk content — the lesson learned from the failure. */
145
+ content: string;
146
+ /** The quirk id, for `update_quirk` / `delete_quirk`. */
147
+ quirkId: string;
148
+ }
149
+ /**
150
+ * Pick the tool gotchas that are actionable for the tools a session can call.
151
+ *
152
+ * A gotcha about a tool the model cannot invoke is noise, so the tool name
153
+ * is matched against the session's tool catalog. This is a metadata scan, not
154
+ * a semantic recall: the question is "does a quirk exist for THIS tool", not
155
+ * "which quirk is most similar to the current request".
156
+ *
157
+ * @param quirks - Tool-error quirks, newest first.
158
+ * @param toolNames - The tool names the session may call.
159
+ * @param maxItems - Maximum number of gotchas to return.
160
+ */
161
+ export declare function selectToolGotchas(quirks: readonly Pick<Quirk, "id" | "content" | "tags">[], toolNames: readonly string[], maxItems?: number): ToolGotcha[];
162
+ /**
163
+ * Is there already a tool-error quirk for this tool AND cause class?
164
+ *
165
+ * The exact duplicate check. A semantic similarity floor cannot do this job:
166
+ * an error query scores 0.55-0.63 against arbitrary stored quirks, so any
167
+ * floor low enough to catch a genuine duplicate also catches noise — which
168
+ * silently suppressed creation on every real failure. The tags make the
169
+ * check exact: a quirk created for `write` + `schema` is the same lesson as
170
+ * the next `write` + `schema` failure, whatever the error text says.
171
+ *
172
+ * @param quirks - The stored tool-error quirks.
173
+ * @param tool - The failing tool name.
174
+ * @param causeClass - The classified cause class (e.g. "schema").
175
+ */
176
+ export declare function hasEquivalentToolGotcha(quirks: readonly Pick<Quirk, "tags">[], tool: string, causeClass: string): boolean;
177
+ /**
178
+ * Format the proactive tool-gotcha block for the system prompt.
179
+ *
180
+ * Injected BEFORE the model chooses a tool, so a known failure is not
181
+ * repeated in a later session. Empty when there are no gotchas.
182
+ */
183
+ export declare function formatToolGotchaBlock(gotchas: readonly ToolGotcha[]): string;
@@ -20,19 +20,21 @@
20
20
  * that contains `rm -rf`) is reported as "not created" with a reason; the
21
21
  * caller logs it.
22
22
  */
23
+ import { TOOL_ERROR_AUTO_TAG, TOOL_ERROR_CAUSE_TAG_PREFIX, TOOL_ERROR_QUIRK_TAG, TOOL_ERROR_TOOL_TAG_PREFIX, } from "./types.js";
23
24
  import { normalizeToolErrorSignature } from "./tool-error-recall.js";
24
- /** Tag every auto-created tool-error quirk carries. */
25
- export const TOOL_ERROR_QUIRK_TAG = "tool-error";
26
- /** Second tag marking the quirk as machine-created (not agent-authored). */
27
- export const TOOL_ERROR_AUTO_TAG = "auto-created";
25
+ export { TOOL_ERROR_AUTO_TAG, TOOL_ERROR_CAUSE_TAG_PREFIX, TOOL_ERROR_QUIRK_TAG, TOOL_ERROR_TOOL_TAG_PREFIX, };
28
26
  /** Defaults for the tool-error creation path. */
29
27
  export const TOOL_ERROR_CREATE_DEFAULTS = {
30
28
  /**
31
29
  * Minimum raw similarity for a candidate to count as "an equivalent quirk
32
- * already exists" (and therefore suppress creation). Default 0.6 — below
33
- * the 0.7 injection bar, so a probable match still blocks a duplicate.
30
+ * already exists" (and therefore suppress creation). Default 0.7 — the
31
+ * injection bar. Measured: an error query scores 0.55-0.63 against
32
+ * ARBITRARY stored quirks, so a lower floor sits inside the noise band and
33
+ * matches unrelated quirks, which suppresses creation on every real failure.
34
+ * The exact duplicate check is done by metadata instead — see
35
+ * `hasEquivalentToolGotcha`.
34
36
  */
35
- minScore: 0.6,
37
+ minScore: 0.7,
36
38
  /** Maximum quirks created per session. Default 2. */
37
39
  maxPerSession: 2,
38
40
  /** Cooldown per normalized error signature in ms. Default 10 minutes. */
@@ -45,47 +47,79 @@ export const TOOL_ERROR_CREATE_DEFAULTS = {
45
47
  confidence: 0.6,
46
48
  /** Errors shorter than this carry no information — nothing is recorded. */
47
49
  minErrorChars: 15,
50
+ /** Max tool gotchas injected proactively into the system prompt. Default 5. */
51
+ maxProactiveItems: 5,
48
52
  };
49
53
  /** Known error classes with a deterministic, actionable cause. */
50
54
  const ERROR_CAUSES = [
51
55
  {
56
+ cls: "unknown-tool",
52
57
  pattern: /unknown tool|no such tool|tool .*not found|not a registered tool/i,
53
58
  cause: "the tool name may be wrong, or the tool is not available in this session",
54
59
  },
55
60
  {
56
- pattern: /enoent|no such file or directory|cannot find module/i,
57
- cause: "the path or module name may be wrong",
61
+ cls: "schema",
62
+ pattern: /expected .*(received|but got)|invalid argument|invalid type|schema|must be (a|an|string|number|boolean|object|array)/i,
63
+ cause: "the arguments do not match the tool's schema",
58
64
  },
59
65
  {
66
+ cls: "permissions",
60
67
  pattern: /eacces|permission denied/i,
61
68
  cause: "the path may need different permissions",
62
69
  },
63
70
  {
64
- pattern: /expected .*(received|but got)|invalid argument|invalid type|schema|must be (a|an|string|number|boolean|object|array)/i,
65
- cause: "the arguments do not match the tool's schema",
71
+ cls: "path",
72
+ pattern: /enoent|no such file or directory|cannot find module/i,
73
+ cause: "the path or module name may be wrong",
66
74
  },
67
75
  {
76
+ cls: "rate-limit",
68
77
  pattern: /\b429\b|rate limit|too many requests/i,
69
78
  cause: "the request was rate limited — retry later",
70
79
  },
71
80
  {
81
+ cls: "timeout",
72
82
  pattern: /timed? ?out|timeout|etimedout/i,
73
83
  cause: "the operation timed out",
74
84
  },
75
85
  ];
76
86
  /**
77
- * Classify an error message into a short, deterministic cause.
87
+ * Cause classes that are worth surfacing BEFORE the tool is called.
88
+ *
89
+ * A transient failure (rate limit, timeout) must NOT be warned about in
90
+ * advance — the next call will not fail the same way, and the warning would
91
+ * be misleading. An unclassified error is excluded for the same reason: we do
92
+ * not know whether it is durable, and a raw error dump says what happened,
93
+ * not what to do.
94
+ */
95
+ export const PROACTIVE_CAUSE_CLASSES = new Set([
96
+ "unknown-tool",
97
+ "schema",
98
+ "permissions",
99
+ "path",
100
+ ]);
101
+ /**
102
+ * Classify an error message into a cause class plus its guidance.
78
103
  *
79
104
  * Returns `undefined` for unknown errors — the raw text is then the whole
80
105
  * story.
81
106
  */
82
- export function classifyToolError(error) {
83
- for (const { pattern, cause } of ERROR_CAUSES) {
107
+ export function classifyToolErrorCause(error) {
108
+ for (const { cls, pattern, cause } of ERROR_CAUSES) {
84
109
  if (pattern.test(error))
85
- return cause;
110
+ return { cls, cause };
86
111
  }
87
112
  return undefined;
88
113
  }
114
+ /**
115
+ * Classify an error message into a short, deterministic cause.
116
+ *
117
+ * Returns `undefined` for unknown errors — the raw text is then the whole
118
+ * story.
119
+ */
120
+ export function classifyToolError(error) {
121
+ return classifyToolErrorCause(error)?.cause;
122
+ }
89
123
  /**
90
124
  * Build the quirk content for a failed tool call.
91
125
  *
@@ -156,11 +190,17 @@ export async function createToolErrorQuirk(deps, info, content) {
156
190
  if (!text) {
157
191
  return { created: false, reason: "empty quirk content" };
158
192
  }
193
+ // The cause class is what makes a gotcha usable PROACTIVELY: a transient
194
+ // failure must not be warned about before the next call.
195
+ const cause = classifyToolErrorCause(info.error);
196
+ const tags = [TOOL_ERROR_QUIRK_TAG, TOOL_ERROR_AUTO_TAG, `tool:${info.tool}`];
197
+ if (cause)
198
+ tags.push(`${TOOL_ERROR_CAUSE_TAG_PREFIX}${cause.cls}`);
159
199
  try {
160
200
  const quirk = await deps.add({
161
201
  content: text,
162
202
  quirkType: "gotcha",
163
- tags: [TOOL_ERROR_QUIRK_TAG, TOOL_ERROR_AUTO_TAG, `tool:${info.tool}`],
203
+ tags,
164
204
  confidence: TOOL_ERROR_CREATE_DEFAULTS.confidence,
165
205
  sourceRef: `tool-error:${info.tool}`,
166
206
  });
@@ -170,4 +210,83 @@ export async function createToolErrorQuirk(deps, info, content) {
170
210
  return { created: false, reason: err instanceof Error ? err.message : String(err) };
171
211
  }
172
212
  }
213
+ /**
214
+ * Pick the tool gotchas that are actionable for the tools a session can call.
215
+ *
216
+ * A gotcha about a tool the model cannot invoke is noise, so the tool name
217
+ * is matched against the session's tool catalog. This is a metadata scan, not
218
+ * a semantic recall: the question is "does a quirk exist for THIS tool", not
219
+ * "which quirk is most similar to the current request".
220
+ *
221
+ * @param quirks - Tool-error quirks, newest first.
222
+ * @param toolNames - The tool names the session may call.
223
+ * @param maxItems - Maximum number of gotchas to return.
224
+ */
225
+ export function selectToolGotchas(quirks, toolNames, maxItems = TOOL_ERROR_CREATE_DEFAULTS.maxProactiveItems) {
226
+ if (toolNames.length === 0 || maxItems <= 0)
227
+ return [];
228
+ const callable = new Set(toolNames.map((t) => t.toLowerCase()));
229
+ const gotchas = [];
230
+ for (const q of quirks) {
231
+ const tags = q.tags ?? [];
232
+ if (!tags.includes(TOOL_ERROR_QUIRK_TAG))
233
+ continue;
234
+ const toolTag = tags.find((t) => t.startsWith(TOOL_ERROR_TOOL_TAG_PREFIX));
235
+ if (!toolTag)
236
+ continue;
237
+ const tool = toolTag.slice(TOOL_ERROR_TOOL_TAG_PREFIX.length).toLowerCase();
238
+ if (!callable.has(tool))
239
+ continue;
240
+ // Only durable, actionable causes are worth warning about in advance.
241
+ const causeTag = tags.find((t) => t.startsWith(TOOL_ERROR_CAUSE_TAG_PREFIX));
242
+ if (!causeTag)
243
+ continue;
244
+ const cls = causeTag.slice(TOOL_ERROR_CAUSE_TAG_PREFIX.length);
245
+ if (!PROACTIVE_CAUSE_CLASSES.has(cls))
246
+ continue;
247
+ gotchas.push({ tool, content: q.content, quirkId: q.id });
248
+ if (gotchas.length >= maxItems)
249
+ break;
250
+ }
251
+ return gotchas;
252
+ }
253
+ /**
254
+ * Is there already a tool-error quirk for this tool AND cause class?
255
+ *
256
+ * The exact duplicate check. A semantic similarity floor cannot do this job:
257
+ * an error query scores 0.55-0.63 against arbitrary stored quirks, so any
258
+ * floor low enough to catch a genuine duplicate also catches noise — which
259
+ * silently suppressed creation on every real failure. The tags make the
260
+ * check exact: a quirk created for `write` + `schema` is the same lesson as
261
+ * the next `write` + `schema` failure, whatever the error text says.
262
+ *
263
+ * @param quirks - The stored tool-error quirks.
264
+ * @param tool - The failing tool name.
265
+ * @param causeClass - The classified cause class (e.g. "schema").
266
+ */
267
+ export function hasEquivalentToolGotcha(quirks, tool, causeClass) {
268
+ const toolTag = `${TOOL_ERROR_TOOL_TAG_PREFIX}${tool.toLowerCase()}`;
269
+ const causeTag = `${TOOL_ERROR_CAUSE_TAG_PREFIX}${causeClass}`;
270
+ return quirks.some((q) => (q.tags ?? []).includes(toolTag) && (q.tags ?? []).includes(causeTag));
271
+ }
272
+ /**
273
+ * Format the proactive tool-gotcha block for the system prompt.
274
+ *
275
+ * Injected BEFORE the model chooses a tool, so a known failure is not
276
+ * repeated in a later session. Empty when there are no gotchas.
277
+ */
278
+ export function formatToolGotchaBlock(gotchas) {
279
+ if (gotchas.length === 0)
280
+ return "";
281
+ const lines = [
282
+ "",
283
+ "---",
284
+ "⚠ **Known tool gotchas** — these tools failed before. Check before calling them:",
285
+ "",
286
+ ];
287
+ for (const g of gotchas) {
288
+ lines.push(`- ${g.content}`);
289
+ }
290
+ return lines.join("\n");
291
+ }
173
292
  //# sourceMappingURL=tool-error-create.js.map
@@ -1,5 +1,13 @@
1
1
  /** Quirk sub-type labels. */
2
2
  export type QuirkType = "gotcha" | "preference" | "decision" | "environment-constraint";
3
+ /** Tag every auto-created tool-error quirk carries. */
4
+ export declare const TOOL_ERROR_QUIRK_TAG = "tool-error";
5
+ /** Second tag marking the quirk as machine-created (not agent-authored). */
6
+ export declare const TOOL_ERROR_AUTO_TAG = "auto-created";
7
+ /** Prefix of the per-tool tag: `tool:<name>`. */
8
+ export declare const TOOL_ERROR_TOOL_TAG_PREFIX = "tool:";
9
+ /** Prefix of the cause-class tag: `cause:<class>`. */
10
+ export declare const TOOL_ERROR_CAUSE_TAG_PREFIX = "cause:";
3
11
  /** Input for adding a new quirk. */
4
12
  export interface QuirkInput {
5
13
  content: string;
@@ -1,2 +1,9 @@
1
- export {};
1
+ /** Tag every auto-created tool-error quirk carries. */
2
+ export const TOOL_ERROR_QUIRK_TAG = "tool-error";
3
+ /** Second tag marking the quirk as machine-created (not agent-authored). */
4
+ export const TOOL_ERROR_AUTO_TAG = "auto-created";
5
+ /** Prefix of the per-tool tag: `tool:<name>`. */
6
+ export const TOOL_ERROR_TOOL_TAG_PREFIX = "tool:";
7
+ /** Prefix of the cause-class tag: `cause:<class>`. */
8
+ export const TOOL_ERROR_CAUSE_TAG_PREFIX = "cause:";
2
9
  //# sourceMappingURL=types.js.map
@@ -2,10 +2,35 @@
2
2
  * @fileoverview Performs hybrid vector-keyword retrieval with configurable scoring and explanation.
3
3
  */
4
4
  import type { EmbeddingProvider, KeywordIndex, RerankProvider, VectorStore, SearchResult, MetadataFilter } from "../core/interfaces.js";
5
+ /**
6
+ * How the `minScore` gate reads the score.
7
+ *
8
+ * - `"fused"` — compare against the raw fused RRF score. Its scale depends on
9
+ * `keywordWeight`, so the same threshold means different things at
10
+ * different weights.
11
+ * - `"normalized"` — compare against the score divided by the maximum that
12
+ * result could reach. Scale-free: the threshold keeps its meaning when
13
+ * `keywordWeight` changes.
14
+ */
15
+ export type MinScoreMode = "fused" | "normalized";
5
16
  /** Options controlling the retrieval behavior. */
6
17
  export interface RetrieveOptions {
7
18
  topK?: number;
8
19
  minScore?: number;
20
+ /**
21
+ * How the `minScore` gate reads the score.
22
+ *
23
+ * - `"fused"` (default): the gate compares against the raw fused RRF score.
24
+ * That score's scale DEPENDS on `keywordWeight` — a vector-only hit can
25
+ * never exceed `(1 - kw)`, so at `kw = 0.4` the fused score maxes out at
26
+ * 0.6 and a fixed `minScore` of 0.65 silently demands a DUAL hit.
27
+ * - `"normalized"`: the fused score is divided by the maximum that result
28
+ * could reach given the legs it has (a dual hit reaches the same maximum
29
+ * at every `keywordWeight`, a vector-only hit reaches `(1 - kw)` times
30
+ * that). The score becomes scale-free, so `minScore` means the same
31
+ * thing at every `keywordWeight`.
32
+ */
33
+ minScoreMode?: MinScoreMode;
9
34
  keywordIndex?: KeywordIndex;
10
35
  keywordWeight?: number;
11
36
  /** Keyword weight for symbol-style queries (a bare identifier like `cosineSimilarity`).
@@ -5,6 +5,18 @@ const FETCH_OVERFETCH_FACTOR = 3;
5
5
  const RRF_K = 60;
6
6
  /** Multiply raw RRF scores by (K+1) to normalize to ~[0,1]. */
7
7
  const RRF_NORMALIZE = RRF_K + 1;
8
+ /**
9
+ * The highest score a result with the given legs could reach (rank 0 in each).
10
+ *
11
+ * A dual hit reaches `(1 - kw) + kw` of the maximum — the same at every
12
+ * `keywordWeight`. A vector-only hit reaches `(1 - kw)` of it, a keyword-only
13
+ * hit `kw` of it. Dividing by this value is what makes the score scale-free.
14
+ */
15
+ function maxReachableScore(hasVector, hasKeyword, keywordWeight) {
16
+ const vector = hasVector ? ((1 - keywordWeight) * RRF_NORMALIZE) / (RRF_K + 1) : 0;
17
+ const keyword = hasKeyword ? (keywordWeight * RRF_NORMALIZE) / (RRF_K + 1) : 0;
18
+ return vector + keyword;
19
+ }
8
20
  const RERANK_DEFAULTS = {
9
21
  candidates: 20,
10
22
  maxDocChars: 1200,
@@ -122,6 +134,7 @@ export async function retrieve(query, embedder, store, options = {}) {
122
134
  try {
123
135
  const topK = options.topK ?? 10;
124
136
  const minScore = options.minScore ?? 0;
137
+ const minScoreMode = options.minScoreMode ?? "fused";
125
138
  const queryShape = detectQueryShape(query);
126
139
  const baseKw = options.keywordWeight ?? 0.4;
127
140
  // Symbol queries are exact-identifier lookups — keyword search is the strong
@@ -182,7 +195,12 @@ export async function retrieve(query, embedder, store, options = {}) {
182
195
  const kMultiplier = kR !== undefined ? keywordDemotion(chunk, docDemotion, testDemotion) : 1;
183
196
  const vContrib = vR !== undefined ? ((1 - kw) * RRF_NORMALIZE) / (RRF_K + vR + 1) : 0;
184
197
  const kContrib = kR !== undefined ? (kw * RRF_NORMALIZE) / (RRF_K + kR + 1) * kMultiplier : 0;
185
- const score = vContrib + kContrib;
198
+ const fusedScore = vContrib + kContrib;
199
+ // In "normalized" mode the score is divided by the maximum this result
200
+ // could reach, so the `minScore` gate keeps its meaning no matter what
201
+ // `keywordWeight` is set to.
202
+ const maxReachable = maxReachableScore(vR !== undefined, kR !== undefined, kw);
203
+ const score = minScoreMode === "normalized" && maxReachable > 0 ? fusedScore / maxReachable : fusedScore;
186
204
  const result = { chunk, score };
187
205
  if (options.explain) {
188
206
  result.explanation = {
@@ -194,6 +212,7 @@ export async function retrieve(query, embedder, store, options = {}) {
194
212
  keywordWeight: kw,
195
213
  vectorRank: vR,
196
214
  keywordRank: kR,
215
+ ...(minScoreMode === "normalized" ? { maxReachableScore: maxReachable } : {}),
197
216
  },
198
217
  queryShape,
199
218
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "opencode-rag-plugin",
3
- "version": "2.7.0",
3
+ "version": "2.9.0",
4
4
  "description": "OpenCode plugin for local-first RAG-based semantic code search",
5
5
  "type": "module",
6
6
  "main": "./dist/plugin-entry.js",