opencode-rag-plugin 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +3 -1
- package/dist/api.d.ts +8 -0
- package/dist/api.js +1 -0
- package/dist/cli/commands/eval-tuner.js +70 -1
- package/dist/cli/commands/eval.js +1 -0
- package/dist/cli/commands/query.js +1 -0
- package/dist/core/config.d.ts +26 -2
- package/dist/core/config.js +10 -1
- package/dist/core/interfaces.d.ts +6 -0
- package/dist/eval/tuner.d.ts +102 -1
- package/dist/eval/tuner.js +135 -0
- package/dist/mcp/handlers.js +1 -0
- package/dist/plugin.js +95 -7
- package/dist/quirks/quirk-store.d.ts +14 -1
- package/dist/quirks/quirk-store.js +27 -0
- package/dist/quirks/tool-error-create.d.ts +79 -7
- package/dist/quirks/tool-error-create.js +135 -16
- package/dist/quirks/types.d.ts +8 -0
- package/dist/quirks/types.js +8 -1
- package/dist/retriever/retriever.d.ts +25 -0
- package/dist/retriever/retriever.js +20 -1
- package/package.json +1 -1
package/ReadMe.md
CHANGED
|
@@ -244,7 +244,9 @@ opencode-rag quirk test "npm needs --legacy-peer-deps"
|
|
|
244
244
|
|
|
245
245
|
When `memory.autoInject` is `true`, the plugin checks for relevant quirks on every user message using the combined agent-response + user-query as the search query. Quirks are only injected when their raw vector similarity to that query clears `autoInjectMinScore` (default 0.75) - the same threshold for the user-message and system-prompt injections; `recallMinScore` (default 0.72) is the bar for manual `recall_quirks` calls. A latency budget (`autoInjectLatencyBudgetMs`, default 2000ms) prevents slow embedders from blocking message processing. To avoid polluting the context window, each quirk is injected **at most once per session** — once recalled, it is filtered out from all subsequent auto-injections. Every `add_quirk` and every content-changing `update_quirk` is vetted by an immutable trust monitor that rejects destructive patterns (e.g. `rm -rf`, `force push`, `bypass security`). Outdated or fixed quirks should be corrected with `update_quirk` / `delete_quirk` rather than left to contradict newer memory. See [Plugin documentation](doc/plugin.md#9-quirk-memory-experiential-memory) and [CLI Reference: `quirk`](doc/cli.md#quirk).
|
|
246
246
|
|
|
247
|
-
**Failed tool calls become memory.** When a tool call fails, the plugin looks for a stored quirk that covers the failure and injects it into the next request (`memory.injectOnToolError`, default on). If **no** quirk covers it, it writes one — `memory.autoCreateOnToolError` (default on) — using a deterministic template built from the tool name and the error text, so no LLM is involved. The new quirk is tagged `tool-error` and recorded for *future* sessions; it is deliberately excluded from the generic auto-injection above, so a raw error dump never leaks into an unrelated turn. Creation is bounded by a per-session budget (2) and a per-error cooldown (10 min), and
|
|
247
|
+
**Failed tool calls become memory.** When a tool call fails, the plugin looks for a stored quirk that covers the failure and injects it into the next request (`memory.injectOnToolError`, default on). If **no** quirk covers it, it writes one — `memory.autoCreateOnToolError` (default on) — using a deterministic template built from the tool name and the error text, so no LLM is involved. The new quirk is tagged `tool-error` and recorded for *future* sessions; it is deliberately excluded from the generic auto-injection above, so a raw error dump never leaks into an unrelated turn. Creation is bounded by a per-session budget (2) and a per-error cooldown (10 min), and it is suppressed when a quirk for the same tool and cause class already exists.
|
|
248
|
+
|
|
249
|
+
**Prevention, not just learning.** The gotchas are also injected into the **system prompt before the model picks a tool** (`memory.toolErrorProactiveInject`, default on), scoped to the tools the session can actually call and to durable failures only — a rate limit is never warned about in advance, because the next call will not fail the same way. The first occurrence of a failure is unavoidable; every later session is warned up front.
|
|
248
250
|
|
|
249
251
|
## MCP Server (Optional)
|
|
250
252
|
|
package/dist/api.d.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* @fileoverview High-level public API for OpenCodeRAG: search, indexWorkspace, getContext,
|
|
3
3
|
* scanWorkspace, and related types.
|
|
4
4
|
*/
|
|
5
|
+
import type { MinScoreMode } from "./retriever/retriever.js";
|
|
5
6
|
import { type IndexRunStats } from "./indexer.js";
|
|
6
7
|
import { type WorkspaceFile } from "./content/reader.js";
|
|
7
8
|
import { type SearchResult } from "./core/interfaces.js";
|
|
@@ -15,6 +16,13 @@ export interface SearchOptions {
|
|
|
15
16
|
topK?: number;
|
|
16
17
|
/** Minimum relevance score threshold (0-1). */
|
|
17
18
|
minScore?: number;
|
|
19
|
+
/**
|
|
20
|
+
* How the `minScore` gate reads the score: `"fused"` (default, today's
|
|
21
|
+
* behaviour) or `"normalized"` (scale-free — the score is divided by the
|
|
22
|
+
* maximum that result could reach, so the threshold keeps its meaning at
|
|
23
|
+
* every keyword weight).
|
|
24
|
+
*/
|
|
25
|
+
minScoreMode?: MinScoreMode;
|
|
18
26
|
/** Weight for hybrid keyword search (0-1). */
|
|
19
27
|
keywordWeight?: number;
|
|
20
28
|
/** Filter results to files matching these path patterns. */
|
package/dist/api.js
CHANGED
|
@@ -48,6 +48,7 @@ export async function search(query, options = {}) {
|
|
|
48
48
|
const rawResults = await retrieve(query, ctx.embedder, ctx.store, {
|
|
49
49
|
topK,
|
|
50
50
|
minScore: options.minScore ?? ctx.config.retrieval.minScore,
|
|
51
|
+
minScoreMode: options.minScoreMode ?? ctx.config.retrieval.minScoreMode,
|
|
51
52
|
keywordIndex: ctx.keywordIndex,
|
|
52
53
|
keywordWeight: options.keywordWeight ?? ctx.config.retrieval.hybridSearch?.keywordWeight ?? 0.4,
|
|
53
54
|
symbolKeywordWeight: ctx.config.retrieval.hybridSearch?.symbolKeywordWeight,
|
|
@@ -51,7 +51,7 @@ export function registerEvalTuneCommand(program) {
|
|
|
51
51
|
let exitCode = 0;
|
|
52
52
|
try {
|
|
53
53
|
const { loadGoldenLabels } = await import("../../eval/golden-set.js");
|
|
54
|
-
const { applyTuningToConfig, buildCandidateGrid, runTuner, selectBestTuning, tuningDelta, } = await import("../../eval/tuner.js");
|
|
54
|
+
const { applyTuningToConfig, buildCandidateGrid, comparePaired, measureSignalQuality, runTuner, selectBestTuning, tuningDelta, } = await import("../../eval/tuner.js");
|
|
55
55
|
const { updateConfigValue } = await import("../../core/config.js");
|
|
56
56
|
const { retrieve } = await import("../../retriever/retriever.js");
|
|
57
57
|
const labelsPath = options.labels
|
|
@@ -136,6 +136,56 @@ export function registerEvalTuneCommand(program) {
|
|
|
136
136
|
console.log(` ${c.dim("vs. current config:")} Hit@5 ${sign(delta.hit5)}${(delta.hit5 * 100).toFixed(1)} pts · ` +
|
|
137
137
|
`MRR ${sign(delta.mrr)}${delta.mrr.toFixed(3)} · nDCG@10 ${sign(delta.ndcg10)}${delta.ndcg10.toFixed(3)}`);
|
|
138
138
|
}
|
|
139
|
+
// The point estimate alone is not evidence on a small label set — show
|
|
140
|
+
// the confidence interval and refuse to recommend a change the data
|
|
141
|
+
// does not support.
|
|
142
|
+
const paired = comparePaired(best, current, metric);
|
|
143
|
+
if (paired) {
|
|
144
|
+
const sign = (v) => (v >= 0 ? "+" : "");
|
|
145
|
+
console.log(` ${c.dim("paired bootstrap:")} Δ${paired.metric} ${sign(paired.meanDelta)}${paired.meanDelta.toFixed(3)} ` +
|
|
146
|
+
`(95% CI ${paired.ciLow.toFixed(3)} … ${paired.ciHigh.toFixed(3)}) · ` +
|
|
147
|
+
`${paired.improved} better / ${paired.regressed} worse / ${paired.unchanged} equal · ` +
|
|
148
|
+
`sign test p=${paired.signTestP.toFixed(2)}`);
|
|
149
|
+
if (paired.signTestP > 0.05) {
|
|
150
|
+
console.log(` ${c.warn("Weak evidence:")} the sign test over the discordant labels is not significant — ` +
|
|
151
|
+
"more labels are needed before changing the shipped default.");
|
|
152
|
+
}
|
|
153
|
+
if (paired.withinNoise) {
|
|
154
|
+
console.log(` ${c.warn("Within noise:")} the 95% CI includes 0 for ${labels.length} label(s) — ` +
|
|
155
|
+
"treat the winner as a hint, not a measurement. Add labels or check the signal split below.");
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
// Per-signal split: a fused number is only interpretable next to the
|
|
159
|
+
// quality of the two halves it mixes.
|
|
160
|
+
const signals = await measureSignalQuality({
|
|
161
|
+
labels,
|
|
162
|
+
root: cwd,
|
|
163
|
+
retrieveKeyword: async (label) => ctx.keywordIndex.search(label.query, (base.topK ?? 20) * 3),
|
|
164
|
+
retrieveVector: async (label) => retrieve(label.query, ctx.embedder, ctx.store, {
|
|
165
|
+
topK: base.topK,
|
|
166
|
+
minScore: 0,
|
|
167
|
+
keywordIndex: undefined,
|
|
168
|
+
keywordWeight: 0,
|
|
169
|
+
queryPrefix: ctx.config.embedding.queryPrefix,
|
|
170
|
+
}),
|
|
171
|
+
retrieveFused: async (label) => retrieve(label.query, ctx.embedder, ctx.store, {
|
|
172
|
+
topK: base.topK,
|
|
173
|
+
minScore: base.minScore,
|
|
174
|
+
keywordIndex: ctx.keywordIndex,
|
|
175
|
+
keywordWeight: base.keywordWeight,
|
|
176
|
+
symbolKeywordWeight: base.symbolKeywordWeight,
|
|
177
|
+
docKeywordDemotion: hybrid?.docKeywordDemotion,
|
|
178
|
+
testKeywordDemotion: hybrid?.testKeywordDemotion,
|
|
179
|
+
hybridEnabled: hybrid?.enabled,
|
|
180
|
+
queryPrefix: ctx.config.embedding.queryPrefix,
|
|
181
|
+
}),
|
|
182
|
+
});
|
|
183
|
+
console.log(`\n ${c.heading("Signal split")} ${c.dim("(file level, current config, minScore=0 for the single signals)")}`);
|
|
184
|
+
const signalRow = (name, q) => ` ${name.padEnd(14)} Hit@1 ${(q.hit1 * 100).toFixed(1)}% · Hit@5 ${(q.hit5 * 100).toFixed(1)}% · ` +
|
|
185
|
+
`Hit@10 ${(q.hit10 * 100).toFixed(1)}% · ${q.misses} miss`;
|
|
186
|
+
console.log(signalRow("BM25 only", signals.keyword));
|
|
187
|
+
console.log(signalRow("vector only", signals.vector));
|
|
188
|
+
console.log(signalRow(`fused kw=${base.keywordWeight}`, signals.fused));
|
|
139
189
|
let written = false;
|
|
140
190
|
if (options.write) {
|
|
141
191
|
if (!ctx.configPath) {
|
|
@@ -193,6 +243,25 @@ export function registerEvalTuneCommand(program) {
|
|
|
193
243
|
},
|
|
194
244
|
},
|
|
195
245
|
delta,
|
|
246
|
+
paired: paired
|
|
247
|
+
? {
|
|
248
|
+
metric: paired.metric,
|
|
249
|
+
meanDelta: paired.meanDelta,
|
|
250
|
+
ciLow: paired.ciLow,
|
|
251
|
+
ciHigh: paired.ciHigh,
|
|
252
|
+
pBetter: paired.pBetter,
|
|
253
|
+
improved: paired.improved,
|
|
254
|
+
regressed: paired.regressed,
|
|
255
|
+
unchanged: paired.unchanged,
|
|
256
|
+
withinNoise: paired.withinNoise,
|
|
257
|
+
signTestP: paired.signTestP,
|
|
258
|
+
}
|
|
259
|
+
: null,
|
|
260
|
+
signals: {
|
|
261
|
+
keyword: signals.keyword,
|
|
262
|
+
vector: signals.vector,
|
|
263
|
+
fused: signals.fused,
|
|
264
|
+
},
|
|
196
265
|
written,
|
|
197
266
|
}));
|
|
198
267
|
}
|
|
@@ -240,6 +240,7 @@ export function registerEvalGateCommand(program) {
|
|
|
240
240
|
const retrieveOptions = {
|
|
241
241
|
topK,
|
|
242
242
|
minScore: ctx.config.retrieval.minScore,
|
|
243
|
+
minScoreMode: ctx.config.retrieval.minScoreMode,
|
|
243
244
|
keywordIndex: ctx.keywordIndex,
|
|
244
245
|
keywordWeight: hybrid?.keywordWeight,
|
|
245
246
|
symbolKeywordWeight: hybrid?.symbolKeywordWeight,
|
|
@@ -51,6 +51,7 @@ export function registerQueryCommand(program) {
|
|
|
51
51
|
const rawResults = await retrieve(query, embedder, store, {
|
|
52
52
|
topK,
|
|
53
53
|
minScore,
|
|
54
|
+
minScoreMode: config.retrieval.minScoreMode,
|
|
54
55
|
keywordIndex,
|
|
55
56
|
keywordWeight: hybridCfg?.keywordWeight,
|
|
56
57
|
hybridEnabled: hybridCfg?.enabled,
|
package/dist/core/config.d.ts
CHANGED
|
@@ -294,8 +294,11 @@ export interface MemoryConfig {
|
|
|
294
294
|
/**
|
|
295
295
|
* Minimum raw similarity (0-1) for an existing quirk to count as "an
|
|
296
296
|
* equivalent quirk already exists" and therefore suppress creation
|
|
297
|
-
* (default 0.
|
|
298
|
-
*
|
|
297
|
+
* (default 0.7). Deliberately EQUAL to `toolErrorMinScore`: measured on a
|
|
298
|
+
* real workspace, an error query scores 0.55-0.63 against arbitrary stored
|
|
299
|
+
* quirks, so a lower floor sits inside the noise band and matches unrelated
|
|
300
|
+
* quirks — which suppressed creation on every real failure. The exact
|
|
301
|
+
* duplicate check (same tool + same cause class) is done by metadata.
|
|
299
302
|
*/
|
|
300
303
|
toolErrorCreateMinScore: number;
|
|
301
304
|
/** Latency budget (ms) for the tool-error creation path (embed + store write); default 2000. */
|
|
@@ -306,6 +309,15 @@ export interface MemoryConfig {
|
|
|
306
309
|
* created quirk carries no new information for that turn.
|
|
307
310
|
*/
|
|
308
311
|
toolErrorInjectCreated: boolean;
|
|
312
|
+
/**
|
|
313
|
+
* Surface known tool gotchas in the system prompt BEFORE the model picks a
|
|
314
|
+
* tool (default true). A failure that already happened in an earlier
|
|
315
|
+
* session is injected as a "known tool gotcha", so the model does not repeat
|
|
316
|
+
* it. Only gotchas whose tool is in the session's tool catalog are injected.
|
|
317
|
+
*/
|
|
318
|
+
toolErrorProactiveInject: boolean;
|
|
319
|
+
/** Max tool gotchas injected into the system prompt per request (default 5). */
|
|
320
|
+
toolErrorProactiveMaxItems: number;
|
|
309
321
|
/** Decay settings for aging quirks. */
|
|
310
322
|
decay: {
|
|
311
323
|
/** Whether confidence decays over time. */
|
|
@@ -523,6 +535,18 @@ export interface RagConfig {
|
|
|
523
535
|
topK: number;
|
|
524
536
|
/** Minimum relevance score threshold (0-1). */
|
|
525
537
|
minScore: number;
|
|
538
|
+
/**
|
|
539
|
+
* How `minScore` reads the score (default `"fused"`).
|
|
540
|
+
*
|
|
541
|
+
* The fused hybrid score's scale depends on `hybridSearch.keywordWeight`:
|
|
542
|
+
* a vector-only hit can never exceed `1 - kw`, so at `kw = 0.4` a fixed
|
|
543
|
+
* `minScore` of 0.65 silently demands a DUAL hit (13.5 of 20 results
|
|
544
|
+
* survive vs 17.7 at `kw = 0`). `"normalized"` divides the score by the
|
|
545
|
+
* maximum that result could reach, so the threshold keeps its meaning at
|
|
546
|
+
* every keyword weight. `"fused"` keeps today's behaviour — the default
|
|
547
|
+
* does not change silently; a default flip needs a bigger golden set.
|
|
548
|
+
*/
|
|
549
|
+
minScoreMode?: "fused" | "normalized";
|
|
526
550
|
/** Hybrid search (vector + keyword) configuration. */
|
|
527
551
|
hybridSearch?: {
|
|
528
552
|
/** Whether hybrid search is enabled. */
|
package/dist/core/config.js
CHANGED
|
@@ -130,6 +130,7 @@ export const DEFAULT_CONFIG = {
|
|
|
130
130
|
retrieval: {
|
|
131
131
|
topK: 20,
|
|
132
132
|
minScore: 0.35,
|
|
133
|
+
minScoreMode: "fused",
|
|
133
134
|
hybridSearch: {
|
|
134
135
|
enabled: true,
|
|
135
136
|
keywordWeight: 0.4,
|
|
@@ -325,9 +326,11 @@ export const DEFAULT_CONFIG = {
|
|
|
325
326
|
autoCreateOnToolError: true,
|
|
326
327
|
toolErrorCreateMaxPerSession: 2,
|
|
327
328
|
toolErrorCreateCooldownMs: 600_000,
|
|
328
|
-
toolErrorCreateMinScore: 0.
|
|
329
|
+
toolErrorCreateMinScore: 0.7,
|
|
329
330
|
toolErrorCreateLatencyBudgetMs: 2000,
|
|
330
331
|
toolErrorInjectCreated: false,
|
|
332
|
+
toolErrorProactiveInject: true,
|
|
333
|
+
toolErrorProactiveMaxItems: 5,
|
|
331
334
|
decay: {
|
|
332
335
|
enabled: false,
|
|
333
336
|
halfLifeDays: 30,
|
|
@@ -487,6 +490,12 @@ export function validateConfig(config) {
|
|
|
487
490
|
if (config.memory?.toolErrorCreateLatencyBudgetMs != null && config.memory.toolErrorCreateLatencyBudgetMs < 0) {
|
|
488
491
|
warnings.push("memory.toolErrorCreateLatencyBudgetMs must be >= 0");
|
|
489
492
|
}
|
|
493
|
+
if (config.memory?.toolErrorProactiveMaxItems != null && config.memory.toolErrorProactiveMaxItems < 0) {
|
|
494
|
+
warnings.push("memory.toolErrorProactiveMaxItems must be >= 0");
|
|
495
|
+
}
|
|
496
|
+
if (config.retrieval?.minScoreMode != null && !["fused", "normalized"].includes(config.retrieval.minScoreMode)) {
|
|
497
|
+
warnings.push(`retrieval.minScoreMode "${config.retrieval.minScoreMode}" — expected "fused" or "normalized"`);
|
|
498
|
+
}
|
|
490
499
|
if (config.memory?.decay && config.memory.decay.halfLifeDays <= 0) {
|
|
491
500
|
warnings.push("memory.decay.halfLifeDays must be > 0");
|
|
492
501
|
}
|
|
@@ -150,6 +150,12 @@ export interface SearchExplanation {
|
|
|
150
150
|
vectorRank?: number;
|
|
151
151
|
/** Rank (0-indexed) in the keyword index results. */
|
|
152
152
|
keywordRank?: number;
|
|
153
|
+
/**
|
|
154
|
+
* The highest score a result with these legs could reach (rank 0 in
|
|
155
|
+
* each). Set only in `minScoreMode: "normalized"`, where the reported
|
|
156
|
+
* score is the fused score divided by this value.
|
|
157
|
+
*/
|
|
158
|
+
maxReachableScore?: number;
|
|
153
159
|
/** Cross-encoder relevance from the optional rerank stage. Higher means more
|
|
154
160
|
* relevant. NOT a calibrated probability (provider-dependent, llama-server
|
|
155
161
|
* semantics) and NOT comparable to the RRF-normalized fused components.
|
package/dist/eval/tuner.d.ts
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import type { RagConfig } from "../core/config.js";
|
|
16
16
|
import type { SearchResult } from "../core/interfaces.js";
|
|
17
|
-
import type
|
|
17
|
+
import { type GoldenLabel, type GoldenOutcome, type GoldenSummary } from "./golden-set.js";
|
|
18
18
|
/** One point in the retrieval search space. */
|
|
19
19
|
export interface TuningCandidate {
|
|
20
20
|
/** RRF keyword weight: 0 = pure vector ranking, 1 = pure keyword ranking. */
|
|
@@ -92,6 +92,10 @@ export declare function runTuner(opts: RunTunerOptions): Promise<TuningResult[]>
|
|
|
92
92
|
* Ranked by the primary metric, then by Hit@5, MRR and Hit@1 so a tie on the
|
|
93
93
|
* primary metric still resolves to the more useful configuration.
|
|
94
94
|
*
|
|
95
|
+
* The argmax alone is NOT evidence — on a small label set it is often within
|
|
96
|
+
* noise of the runner-up. Use {@link comparePaired} for the confidence
|
|
97
|
+
* interval before changing anything.
|
|
98
|
+
*
|
|
95
99
|
* @param results - The measured candidates.
|
|
96
100
|
* @param metric - The primary metric (default `ndcg10`).
|
|
97
101
|
* @returns The winning result, or `undefined` when there are no results.
|
|
@@ -105,6 +109,103 @@ export declare function selectBestTuning(results: TuningResult[], metric?: Tunin
|
|
|
105
109
|
* @returns The per-metric deltas (winner minus current).
|
|
106
110
|
*/
|
|
107
111
|
export declare function tuningDelta(best: TuningResult, current: TuningResult | undefined): Record<string, number>;
|
|
112
|
+
/** Paired comparison of two candidates measured on the same labels. */
|
|
113
|
+
export interface PairedComparison {
|
|
114
|
+
/** The metric the comparison is based on. */
|
|
115
|
+
metric: TuningMetric;
|
|
116
|
+
/** Mean per-label difference (first candidate minus second). */
|
|
117
|
+
meanDelta: number;
|
|
118
|
+
/** Lower bound of the 95% bootstrap confidence interval. */
|
|
119
|
+
ciLow: number;
|
|
120
|
+
/** Upper bound of the 95% bootstrap confidence interval. */
|
|
121
|
+
ciHigh: number;
|
|
122
|
+
/** Share of bootstrap resamples in which the first candidate wins. */
|
|
123
|
+
pBetter: number;
|
|
124
|
+
/** Labels where the first candidate scored higher. */
|
|
125
|
+
improved: number;
|
|
126
|
+
/** Labels where the second candidate scored higher. */
|
|
127
|
+
regressed: number;
|
|
128
|
+
/** Labels where both scored the same. */
|
|
129
|
+
unchanged: number;
|
|
130
|
+
/** True when the confidence interval includes 0 — the difference is within noise. */
|
|
131
|
+
withinNoise: boolean;
|
|
132
|
+
/**
|
|
133
|
+
* Two-sided exact sign-test p-value over the discordant labels.
|
|
134
|
+
*
|
|
135
|
+
* The bootstrap CI is the optimistic bound: it weights a label by how much it
|
|
136
|
+
* moved. The sign test only asks "how many labels flipped?", which is the
|
|
137
|
+
* conservative reading of a small label set — a 19-vs-11 split is still
|
|
138
|
+
* p ≈ 0.20. Report both.
|
|
139
|
+
*/
|
|
140
|
+
signTestP: number;
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Two-sided exact sign-test p-value for `wins` successes out of `wins + losses`
|
|
144
|
+
* trials under a 0.5 null.
|
|
145
|
+
*
|
|
146
|
+
* @param wins - Labels where the first candidate scored higher.
|
|
147
|
+
* @param losses - Labels where the second candidate scored higher.
|
|
148
|
+
* @returns The p-value in [0, 1]; 1 when there are no discordant labels.
|
|
149
|
+
*/
|
|
150
|
+
export declare function signTestP(wins: number, losses: number): number;
|
|
151
|
+
/**
|
|
152
|
+
* Compare two candidates with a paired bootstrap over the labels.
|
|
153
|
+
*
|
|
154
|
+
* The label set is small (48 by default), so a point estimate on its own is
|
|
155
|
+
* not evidence: a 10-point Hit@5 gap is a handful of labels, and a sign test
|
|
156
|
+
* on the discordant labels is the honest lower bound. This returns the
|
|
157
|
+
* confidence interval, the win/loss counts, and an explicit `withinNoise`
|
|
158
|
+
* flag so a caller can refuse to recommend a change the data does not support.
|
|
159
|
+
*
|
|
160
|
+
* @param better - The candidate to test.
|
|
161
|
+
* @param other - The candidate to compare against (usually the current config).
|
|
162
|
+
* @param metric - The metric to compare on (default `ndcg10`).
|
|
163
|
+
* @param iterations - Bootstrap resamples (default 20000).
|
|
164
|
+
* @param rng - Random source (injectable for deterministic tests).
|
|
165
|
+
* @returns The paired comparison, or `undefined` when there is nothing to compare.
|
|
166
|
+
*/
|
|
167
|
+
export declare function comparePaired(better: TuningResult, other: TuningResult | undefined, metric?: TuningMetric, iterations?: number, rng?: () => number): PairedComparison | undefined;
|
|
168
|
+
/** Aggregate file-level quality of one retrieval signal. */
|
|
169
|
+
export interface SignalQuality {
|
|
170
|
+
/** Mean Hit@1 over the labels. */
|
|
171
|
+
hit1: number;
|
|
172
|
+
/** Mean Hit@5 over the labels. */
|
|
173
|
+
hit5: number;
|
|
174
|
+
/** Mean Hit@10 over the labels. */
|
|
175
|
+
hit10: number;
|
|
176
|
+
/** Labels where the expected file was not found at all. */
|
|
177
|
+
misses: number;
|
|
178
|
+
}
|
|
179
|
+
/** Per-signal quality: the keyword half and the vector half measured apart. */
|
|
180
|
+
export interface SignalComparison {
|
|
181
|
+
/** Quality of the keyword (BM25) signal alone. */
|
|
182
|
+
keyword: SignalQuality;
|
|
183
|
+
/** Quality of the vector signal alone. */
|
|
184
|
+
vector: SignalQuality;
|
|
185
|
+
/** Quality of the fused pipeline. */
|
|
186
|
+
fused: SignalQuality;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Measure each retrieval signal on its own, so a fused number can be
|
|
190
|
+
* interpreted instead of just believed.
|
|
191
|
+
*
|
|
192
|
+
* A fused ranking can only be as good as the mix of its parts: when the
|
|
193
|
+
* keyword half answers far fewer queries than the vector half, any weight
|
|
194
|
+
* given to it dilutes the result. This measures the parts.
|
|
195
|
+
*
|
|
196
|
+
* @param opts - Labels, root, and one retrieve closure per signal.
|
|
197
|
+
* @returns The per-signal quality.
|
|
198
|
+
*/
|
|
199
|
+
export declare function measureSignalQuality(opts: {
|
|
200
|
+
labels: GoldenLabel[];
|
|
201
|
+
root: string;
|
|
202
|
+
/** Retrieve with the keyword signal only. */
|
|
203
|
+
retrieveKeyword: (label: GoldenLabel) => Promise<SearchResult[]>;
|
|
204
|
+
/** Retrieve with the vector signal only. */
|
|
205
|
+
retrieveVector: (label: GoldenLabel) => Promise<SearchResult[]>;
|
|
206
|
+
/** Retrieve with the full pipeline. */
|
|
207
|
+
retrieveFused: (label: GoldenLabel) => Promise<SearchResult[]>;
|
|
208
|
+
}): Promise<SignalComparison>;
|
|
108
209
|
/**
|
|
109
210
|
* Write the winning weights into a copy of the configuration.
|
|
110
211
|
*
|
package/dist/eval/tuner.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { runGoldenGate, } from "./golden-gate.js";
|
|
2
|
+
import { evaluateGoldenResults, mean, } from "./golden-set.js";
|
|
2
3
|
/** The metric used to pick the winner, and the tie-breakers after it. */
|
|
3
4
|
const METRIC_ORDER = {
|
|
4
5
|
ndcg10: (s) => s.ndcg10,
|
|
@@ -85,6 +86,10 @@ export async function runTuner(opts) {
|
|
|
85
86
|
* Ranked by the primary metric, then by Hit@5, MRR and Hit@1 so a tie on the
|
|
86
87
|
* primary metric still resolves to the more useful configuration.
|
|
87
88
|
*
|
|
89
|
+
* The argmax alone is NOT evidence — on a small label set it is often within
|
|
90
|
+
* noise of the runner-up. Use {@link comparePaired} for the confidence
|
|
91
|
+
* interval before changing anything.
|
|
92
|
+
*
|
|
88
93
|
* @param results - The measured candidates.
|
|
89
94
|
* @param metric - The primary metric (default `ndcg10`).
|
|
90
95
|
* @returns The winning result, or `undefined` when there are no results.
|
|
@@ -128,6 +133,136 @@ export function tuningDelta(best, current) {
|
|
|
128
133
|
ndcg10: best.summary.ndcg10 - current.summary.ndcg10,
|
|
129
134
|
};
|
|
130
135
|
}
|
|
136
|
+
/** Per-label value of a tuning metric. */
|
|
137
|
+
function metricValue(outcome, metric) {
|
|
138
|
+
switch (metric) {
|
|
139
|
+
case "hit5": return outcome.hitAt5;
|
|
140
|
+
case "hit1": return outcome.hitAt1;
|
|
141
|
+
case "mrr": return outcome.rr;
|
|
142
|
+
default: return outcome.ndcg10;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Two-sided exact sign-test p-value for `wins` successes out of `wins + losses`
|
|
147
|
+
* trials under a 0.5 null.
|
|
148
|
+
*
|
|
149
|
+
* @param wins - Labels where the first candidate scored higher.
|
|
150
|
+
* @param losses - Labels where the second candidate scored higher.
|
|
151
|
+
* @returns The p-value in [0, 1]; 1 when there are no discordant labels.
|
|
152
|
+
*/
|
|
153
|
+
export function signTestP(wins, losses) {
|
|
154
|
+
const n = wins + losses;
|
|
155
|
+
if (n === 0)
|
|
156
|
+
return 1;
|
|
157
|
+
const k = Math.max(wins, losses);
|
|
158
|
+
let tail = 0;
|
|
159
|
+
for (let i = k; i <= n; i++) {
|
|
160
|
+
tail += binomial(n, i);
|
|
161
|
+
}
|
|
162
|
+
const p = (tail * 2) / Math.pow(2, n);
|
|
163
|
+
return Math.min(1, p);
|
|
164
|
+
}
|
|
165
|
+
/** Binomial coefficient C(n, k). */
|
|
166
|
+
function binomial(n, k) {
|
|
167
|
+
let result = 1;
|
|
168
|
+
for (let i = 0; i < k; i++) {
|
|
169
|
+
result = (result * (n - i)) / (i + 1);
|
|
170
|
+
}
|
|
171
|
+
return Math.round(result);
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Compare two candidates with a paired bootstrap over the labels.
|
|
175
|
+
*
|
|
176
|
+
* The label set is small (48 by default), so a point estimate on its own is
|
|
177
|
+
* not evidence: a 10-point Hit@5 gap is a handful of labels, and a sign test
|
|
178
|
+
* on the discordant labels is the honest lower bound. This returns the
|
|
179
|
+
* confidence interval, the win/loss counts, and an explicit `withinNoise`
|
|
180
|
+
* flag so a caller can refuse to recommend a change the data does not support.
|
|
181
|
+
*
|
|
182
|
+
* @param better - The candidate to test.
|
|
183
|
+
* @param other - The candidate to compare against (usually the current config).
|
|
184
|
+
* @param metric - The metric to compare on (default `ndcg10`).
|
|
185
|
+
* @param iterations - Bootstrap resamples (default 20000).
|
|
186
|
+
* @param rng - Random source (injectable for deterministic tests).
|
|
187
|
+
* @returns The paired comparison, or `undefined` when there is nothing to compare.
|
|
188
|
+
*/
|
|
189
|
+
export function comparePaired(better, other, metric = "ndcg10", iterations = 20_000, rng = Math.random) {
|
|
190
|
+
if (!other)
|
|
191
|
+
return undefined;
|
|
192
|
+
const n = Math.min(better.outcomes.length, other.outcomes.length);
|
|
193
|
+
if (n === 0)
|
|
194
|
+
return undefined;
|
|
195
|
+
const diffs = [];
|
|
196
|
+
let improved = 0;
|
|
197
|
+
let regressed = 0;
|
|
198
|
+
let unchanged = 0;
|
|
199
|
+
for (let i = 0; i < n; i++) {
|
|
200
|
+
const d = metricValue(better.outcomes[i], metric) - metricValue(other.outcomes[i], metric);
|
|
201
|
+
diffs.push(d);
|
|
202
|
+
if (d > 1e-9)
|
|
203
|
+
improved++;
|
|
204
|
+
else if (d < -1e-9)
|
|
205
|
+
regressed++;
|
|
206
|
+
else
|
|
207
|
+
unchanged++;
|
|
208
|
+
}
|
|
209
|
+
const meanDelta = diffs.reduce((s, d) => s + d, 0) / n;
|
|
210
|
+
const boots = [];
|
|
211
|
+
for (let i = 0; i < iterations; i++) {
|
|
212
|
+
let sum = 0;
|
|
213
|
+
for (let j = 0; j < n; j++)
|
|
214
|
+
sum += diffs[Math.floor(rng() * n)];
|
|
215
|
+
boots.push(sum / n);
|
|
216
|
+
}
|
|
217
|
+
boots.sort((a, b) => a - b);
|
|
218
|
+
const at = (p) => boots[Math.min(boots.length - 1, Math.max(0, Math.floor(p * (boots.length - 1))))];
|
|
219
|
+
const ciLow = at(0.025);
|
|
220
|
+
const ciHigh = at(0.975);
|
|
221
|
+
const pBetter = boots.filter((b) => b > 0).length / boots.length;
|
|
222
|
+
return {
|
|
223
|
+
metric,
|
|
224
|
+
meanDelta,
|
|
225
|
+
ciLow,
|
|
226
|
+
ciHigh,
|
|
227
|
+
pBetter,
|
|
228
|
+
improved,
|
|
229
|
+
regressed,
|
|
230
|
+
unchanged,
|
|
231
|
+
withinNoise: ciLow <= 0 && ciHigh >= 0,
|
|
232
|
+
signTestP: signTestP(improved, regressed),
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Measure each retrieval signal on its own, so a fused number can be
|
|
237
|
+
* interpreted instead of just believed.
|
|
238
|
+
*
|
|
239
|
+
* A fused ranking can only be as good as the mix of its parts: when the
|
|
240
|
+
* keyword half answers far fewer queries than the vector half, any weight
|
|
241
|
+
* given to it dilutes the result. This measures the parts.
|
|
242
|
+
*
|
|
243
|
+
* @param opts - Labels, root, and one retrieve closure per signal.
|
|
244
|
+
* @returns The per-signal quality.
|
|
245
|
+
*/
|
|
246
|
+
export async function measureSignalQuality(opts) {
|
|
247
|
+
const summarize = async (retrieve) => {
|
|
248
|
+
const metrics = [];
|
|
249
|
+
for (const label of opts.labels) {
|
|
250
|
+
const results = await retrieve(label);
|
|
251
|
+
metrics.push(evaluateGoldenResults(opts.root, label, results));
|
|
252
|
+
}
|
|
253
|
+
return {
|
|
254
|
+
hit1: mean(metrics.map((m) => m.hitAt1)),
|
|
255
|
+
hit5: mean(metrics.map((m) => m.hitAt5)),
|
|
256
|
+
hit10: mean(metrics.map((m) => m.hitAt10)),
|
|
257
|
+
misses: metrics.filter((m) => m.firstHitRank === null).length,
|
|
258
|
+
};
|
|
259
|
+
};
|
|
260
|
+
return {
|
|
261
|
+
keyword: await summarize(opts.retrieveKeyword),
|
|
262
|
+
vector: await summarize(opts.retrieveVector),
|
|
263
|
+
fused: await summarize(opts.retrieveFused),
|
|
264
|
+
};
|
|
265
|
+
}
|
|
131
266
|
/**
|
|
132
267
|
* Write the winning weights into a copy of the configuration.
|
|
133
268
|
*
|
package/dist/mcp/handlers.js
CHANGED
|
@@ -91,6 +91,7 @@ export async function handleSearchSemantic(params, embedder, store, cfg, keyword
|
|
|
91
91
|
const retrieveOpts = {
|
|
92
92
|
topK,
|
|
93
93
|
minScore: cfg.retrieval.minScore,
|
|
94
|
+
minScoreMode: cfg.retrieval.minScoreMode,
|
|
94
95
|
keywordIndex,
|
|
95
96
|
keywordWeight: cfg.retrieval.hybridSearch?.keywordWeight,
|
|
96
97
|
hybridEnabled: cfg.retrieval.hybridSearch?.enabled,
|
package/dist/plugin.js
CHANGED
|
@@ -29,10 +29,10 @@ import { countTokens } from "./eval/token-counter.js";
|
|
|
29
29
|
import { checkForUpdate, getCurrentVersion, installLatestUpdate } from "./core/version-check.js";
|
|
30
30
|
import { loadAutoUpdateState, saveAutoUpdateState, shouldAttemptInstall } from "./core/auto-update-state.js";
|
|
31
31
|
import { destroyAllPooledConnections } from "./embedder/http.js";
|
|
32
|
-
import { listQuirks, lintQuirks, recallQuirks, sharedWords, reconcileQuirks, addQuirk } from "./quirks/quirk-store.js";
|
|
32
|
+
import { listQuirks, lintQuirks, recallQuirks, sharedWords, reconcileQuirks, addQuirk, listToolErrorQuirks } from "./quirks/quirk-store.js";
|
|
33
33
|
import { autoCaptureQuirks } from "./quirks/auto-capture.js";
|
|
34
34
|
import { createToolErrorInjectionState, recallToolErrorQuirks, formatToolErrorQuirkBlock, toToolErrorCaptureEntry, } from "./quirks/tool-error-recall.js";
|
|
35
|
-
import { buildToolErrorQuirkContent, createToolErrorCreateState, createToolErrorQuirk, TOOL_ERROR_QUIRK_TAG, } from "./quirks/tool-error-create.js";
|
|
35
|
+
import { buildToolErrorQuirkContent, classifyToolErrorCause, createToolErrorCreateState, createToolErrorQuirk, formatToolGotchaBlock, hasEquivalentToolGotcha, selectToolGotchas, TOOL_ERROR_CREATE_DEFAULTS, TOOL_ERROR_QUIRK_TAG, } from "./quirks/tool-error-create.js";
|
|
36
36
|
import { buildSystemGuidanceLines } from "./opencode/system-guidance.js";
|
|
37
37
|
import { existsSync, readFileSync, readdirSync, statSync, unlinkSync } from "node:fs";
|
|
38
38
|
import path from "node:path";
|
|
@@ -42,6 +42,8 @@ const MAX_SESSION_MAP_SIZE = 50;
|
|
|
42
42
|
const MAX_SESSION_TOOL_RESULTS = 20;
|
|
43
43
|
/** Pending tool-error quirk blocks older than this (ms) are dropped instead of injected. */
|
|
44
44
|
const TOOL_ERROR_PENDING_TTL_MS = 120_000;
|
|
45
|
+
/** How long the tool-error quirk list is cached before it is re-read (ms). */
|
|
46
|
+
const TOOL_GOTCHA_CACHE_MS = 30_000;
|
|
45
47
|
/** Cache of loaded RAG configurations keyed by workspace directory. */
|
|
46
48
|
const configCache = new Map();
|
|
47
49
|
/** Active background indexer instances keyed by workspace directory. */
|
|
@@ -431,10 +433,10 @@ function buildRetrievalQuery(hints) {
|
|
|
431
433
|
* Perform a retrieval query against the vector store with the given parameters.
|
|
432
434
|
* Returns an empty array for blank queries.
|
|
433
435
|
*/
|
|
434
|
-
async function retrieveContext(query, embedder, store, topK, retrieveFn = retrieve, minScore = 0, keywordIndex, keywordWeight, queryPrefix, explain = false, hybridEnabled, filter, reranker, reranking) {
|
|
436
|
+
async function retrieveContext(query, embedder, store, topK, retrieveFn = retrieve, minScore = 0, keywordIndex, keywordWeight, queryPrefix, explain = false, hybridEnabled, filter, reranker, reranking, minScoreMode) {
|
|
435
437
|
if (query.trim().length === 0)
|
|
436
438
|
return [];
|
|
437
|
-
return retrieveFn(query, embedder, store, { topK, minScore, keywordIndex, keywordWeight, hybridEnabled, queryPrefix, explain, filter, reranker, reranking });
|
|
439
|
+
return retrieveFn(query, embedder, store, { topK, minScore, keywordIndex, keywordWeight, hybridEnabled, queryPrefix, explain, filter, reranker, reranking, minScoreMode });
|
|
438
440
|
}
|
|
439
441
|
/**
|
|
440
442
|
* Load results from one or two queries (primary + optional extra), optimize
|
|
@@ -443,11 +445,12 @@ async function retrieveContext(query, embedder, store, topK, retrieveFn = retrie
|
|
|
443
445
|
*/
|
|
444
446
|
async function loadRetrievedResults(query, embedder, store, cfg, retrieveFn = retrieve, topK = cfg.retrieval.topK, extraQuery, keywordIndex, queryPrefix, explain = false, filter, reranker, reranking) {
|
|
445
447
|
const minScore = cfg.retrieval.minScore;
|
|
448
|
+
const minScoreMode = cfg.retrieval.minScoreMode;
|
|
446
449
|
const kw = cfg.retrieval.hybridSearch?.keywordWeight;
|
|
447
450
|
const hybridEnabled = cfg.retrieval.hybridSearch?.enabled;
|
|
448
|
-
const primaryResults = await retrieveContext(query, embedder, store, topK, retrieveFn, minScore, keywordIndex, kw, queryPrefix, explain, hybridEnabled, filter, reranker, reranking);
|
|
451
|
+
const primaryResults = await retrieveContext(query, embedder, store, topK, retrieveFn, minScore, keywordIndex, kw, queryPrefix, explain, hybridEnabled, filter, reranker, reranking, minScoreMode);
|
|
449
452
|
const extraResults = extraQuery
|
|
450
|
-
? await retrieveContext(extraQuery, embedder, store, topK, retrieveFn, minScore, keywordIndex, kw, queryPrefix, explain, hybridEnabled, filter, reranker, reranking)
|
|
453
|
+
? await retrieveContext(extraQuery, embedder, store, topK, retrieveFn, minScore, keywordIndex, kw, queryPrefix, explain, hybridEnabled, filter, reranker, reranking, minScoreMode)
|
|
451
454
|
: [];
|
|
452
455
|
const optCfg = cfg.retrieval.contextOptimization ?? DEFAULT_CONTEXT_OPTIMIZATION;
|
|
453
456
|
return optimizeContext([...primaryResults, ...extraResults], { topK, config: optCfg })
|
|
@@ -646,6 +649,9 @@ export function createRagHooks(options) {
|
|
|
646
649
|
// plugin writes itself when a failure has no matching memory.
|
|
647
650
|
const sessionToolErrorCreateState = new Map();
|
|
648
651
|
const sessionPendingToolErrorQuirk = new Map();
|
|
652
|
+
// Tool-gotcha cache: the tool-error quirks change rarely, so the audit log is
|
|
653
|
+
// read at most every TOOL_GOTCHA_CACHE_MS instead of on every model request.
|
|
654
|
+
const toolGotchaCache = new Map();
|
|
649
655
|
// Evaluation session logger — captures OpenCode events for analysis
|
|
650
656
|
const sessionLogger = createSessionLogger(options.storePath);
|
|
651
657
|
appendDebugLog(options.logFilePath, {
|
|
@@ -653,6 +659,23 @@ export function createRagHooks(options) {
|
|
|
653
659
|
message: "OpenCode plugin initialized",
|
|
654
660
|
});
|
|
655
661
|
const readOverride = getEffectiveCfg().openCode.readOverride === true;
|
|
662
|
+
/**
|
|
663
|
+
* Read the auto-created tool-error quirks, cached briefly.
|
|
664
|
+
*
|
|
665
|
+
* Runs on every model request (the proactive tool-gotcha injection), so it
|
|
666
|
+
* must not hit the disk every time. The cache is invalidated explicitly
|
|
667
|
+
* when a new tool-error quirk is created.
|
|
668
|
+
*/
|
|
669
|
+
async function getToolErrorQuirks(deps) {
|
|
670
|
+
const key = options.storePath;
|
|
671
|
+
const now = Date.now();
|
|
672
|
+
const cached = toolGotchaCache.get(key);
|
|
673
|
+
if (cached && now - cached.fetchedAt < TOOL_GOTCHA_CACHE_MS)
|
|
674
|
+
return cached.quirks;
|
|
675
|
+
const quirks = await listToolErrorQuirks(deps);
|
|
676
|
+
toolGotchaCache.set(key, { quirks, fetchedAt: now });
|
|
677
|
+
return quirks;
|
|
678
|
+
}
|
|
656
679
|
const retrievalTool = tool({
|
|
657
680
|
description: "Search the indexed codebase by meaning, not just keywords. " +
|
|
658
681
|
"ESSENTIAL before any code task — replaces blind reading with targeted retrieval. " +
|
|
@@ -997,6 +1020,11 @@ export function createRagHooks(options) {
|
|
|
997
1020
|
// `output.error` is set by the V2 adapter's error branch; the status
|
|
998
1021
|
// check keeps this working if a runtime reports errors via input only.
|
|
999
1022
|
const errorText = output.error ?? (input.status === "error" ? output.output : undefined);
|
|
1023
|
+
appendDebugLog(options.logFilePath, {
|
|
1024
|
+
scope: "quirk.tool-error",
|
|
1025
|
+
message: `tool "${input.tool}" status=${input.status ?? "unset"} error=${errorText ? "yes" : "no"}`,
|
|
1026
|
+
severity: "debug",
|
|
1027
|
+
}, getEffectiveCfg().logging.level);
|
|
1000
1028
|
if (errorText && errorText.length > 0) {
|
|
1001
1029
|
const info = { tool: input.tool, error: errorText, sessionID: input.sessionID };
|
|
1002
1030
|
results.push(toToolErrorCaptureEntry(info));
|
|
@@ -1057,8 +1085,17 @@ export function createRagHooks(options) {
|
|
|
1057
1085
|
state.record(info, hits.map((h) => h.id));
|
|
1058
1086
|
// No candidate at all → nothing in memory covers this failure.
|
|
1059
1087
|
// Record one so a future occurrence can be answered from memory.
|
|
1088
|
+
// The duplicate check is EXACT (same tool + same cause class), not
|
|
1089
|
+
// a loose similarity floor: a floor low enough to catch a genuine
|
|
1090
|
+
// duplicate also catches unrelated quirks, which would suppress
|
|
1091
|
+
// creation on every real failure.
|
|
1060
1092
|
let createdHit;
|
|
1061
|
-
|
|
1093
|
+
let createdQuirk = false;
|
|
1094
|
+
const causeClass = classifyToolErrorCause(info.error)?.cls;
|
|
1095
|
+
const equivalent = recallResult.candidates.length > 0 ||
|
|
1096
|
+
(causeClass !== undefined &&
|
|
1097
|
+
hasEquivalentToolGotcha(await getToolErrorQuirks(qDeps), info.tool, causeClass));
|
|
1098
|
+
if (!equivalent && memCfg.autoCreateOnToolError) {
|
|
1062
1099
|
const createState = sessionToolErrorCreateState.get(input.sessionID) ?? createToolErrorCreateState({
|
|
1063
1100
|
maxPerSession: memCfg.toolErrorCreateMaxPerSession ?? 2,
|
|
1064
1101
|
cooldownMs: memCfg.toolErrorCreateCooldownMs ?? 600_000,
|
|
@@ -1078,6 +1115,10 @@ export function createRagHooks(options) {
|
|
|
1078
1115
|
clearTimeout(createTimer);
|
|
1079
1116
|
if (created?.created && created.id) {
|
|
1080
1117
|
createState.record(info, true);
|
|
1118
|
+
createdQuirk = true;
|
|
1119
|
+
// The proactive gotcha list changed — drop the cache so
|
|
1120
|
+
// the next model request sees the new gotcha.
|
|
1121
|
+
toolGotchaCache.delete(options.storePath);
|
|
1081
1122
|
appendDebugLog(options.logFilePath, {
|
|
1082
1123
|
scope: "quirk.tool-error",
|
|
1083
1124
|
message: `created quirk ${created.id} for tool "${input.tool}"`,
|
|
@@ -1114,6 +1155,22 @@ export function createRagHooks(options) {
|
|
|
1114
1155
|
ts: Date.now(),
|
|
1115
1156
|
}, MAX_SESSION_MAP_SIZE);
|
|
1116
1157
|
}
|
|
1158
|
+
appendDebugLog(options.logFilePath, {
|
|
1159
|
+
scope: "quirk.tool-error",
|
|
1160
|
+
message: `tool error "${input.tool}": candidates=${recallResult.candidates.length} injected=${finalHits.length} created=${createdQuirk ? "yes" : "no"}`,
|
|
1161
|
+
});
|
|
1162
|
+
}
|
|
1163
|
+
else if (recallResult) {
|
|
1164
|
+
appendDebugLog(options.logFilePath, {
|
|
1165
|
+
scope: "quirk.tool-error",
|
|
1166
|
+
message: `tool error "${input.tool}": skipped (cooldown or budget)`,
|
|
1167
|
+
});
|
|
1168
|
+
}
|
|
1169
|
+
else {
|
|
1170
|
+
appendDebugLog(options.logFilePath, {
|
|
1171
|
+
scope: "quirk.tool-error",
|
|
1172
|
+
message: `tool error "${input.tool}": skipped (recall timed out after ${budgetMs}ms)`,
|
|
1173
|
+
});
|
|
1117
1174
|
}
|
|
1118
1175
|
}
|
|
1119
1176
|
}
|
|
@@ -1137,6 +1194,37 @@ export function createRagHooks(options) {
|
|
|
1137
1194
|
? "system guidance injected"
|
|
1138
1195
|
: `system guidance skipped (${guidanceEnabled ? `no RAG tools in session; agent=${input?.agent ?? "unknown"}` : "injectSystemPrompt=false"})`,
|
|
1139
1196
|
});
|
|
1197
|
+
// Which tools can this session actually call? The catalog is the set of
|
|
1198
|
+
// tools the model may invoke, so it is the right filter for tool-scoped
|
|
1199
|
+
// memory (a gotcha about a tool that is not callable is not actionable).
|
|
1200
|
+
appendDebugLog(options.logFilePath, {
|
|
1201
|
+
scope: "experimental.chat.system.transform",
|
|
1202
|
+
message: `tool catalog (${Object.keys(input?.tools ?? {}).length}): ${Object.keys(input?.tools ?? {}).join(", ")}`,
|
|
1203
|
+
severity: "debug",
|
|
1204
|
+
}, getEffectiveCfg().logging.level);
|
|
1205
|
+
// Proactive tool gotchas: surface what earlier failures taught us BEFORE
|
|
1206
|
+
// the model picks a tool, so a known failure is not repeated in a later
|
|
1207
|
+
// session. Scoped to the tools this session can actually call.
|
|
1208
|
+
try {
|
|
1209
|
+
const toolNames = Object.keys(input?.tools ?? {});
|
|
1210
|
+
if (toolNames.length > 0) {
|
|
1211
|
+
const memCfg = getEffectiveCfg().memory;
|
|
1212
|
+
if (memCfg?.enabled && memCfg.toolErrorProactiveInject) {
|
|
1213
|
+
const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
|
|
1214
|
+
const gotchas = selectToolGotchas(await getToolErrorQuirks(qDeps), toolNames, memCfg.toolErrorProactiveMaxItems ?? TOOL_ERROR_CREATE_DEFAULTS.maxProactiveItems);
|
|
1215
|
+
if (gotchas.length > 0) {
|
|
1216
|
+
output.system.unshift(formatToolGotchaBlock(gotchas));
|
|
1217
|
+
appendDebugLog(options.logFilePath, {
|
|
1218
|
+
scope: "quirk.tool-error",
|
|
1219
|
+
message: `proactively injected ${gotchas.length} tool gotcha(s): ${gotchas.map((g) => g.tool).join(", ")}`,
|
|
1220
|
+
});
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
catch {
|
|
1226
|
+
// Non-critical — must never throw
|
|
1227
|
+
}
|
|
1140
1228
|
if (injectGuidance) {
|
|
1141
1229
|
const guidance = buildSystemGuidanceLines({ promptEnforcement: !!cfg.memory?.promptEnforcement, decisionEnabled: !!cfg.decision?.enabled, routeBeforeAsking: !!cfg.decision?.routeBeforeAsking });
|
|
1142
1230
|
output.system.unshift(guidance.join("\n"));
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { EmbeddingProvider, VectorStore, KeywordIndex, SearchResult } from "../core/interfaces.js";
|
|
2
2
|
import type { RagConfig } from "../core/config.js";
|
|
3
|
-
import type
|
|
3
|
+
import { type Quirk, type QuirkInput } from "./types.js";
|
|
4
4
|
/** Dependencies required by all quirk-store operations. */
|
|
5
5
|
export interface QuirkStoreDeps {
|
|
6
6
|
embedder: EmbeddingProvider;
|
|
@@ -104,6 +104,19 @@ export declare function recallQuirks(deps: QuirkStoreDeps, query: string, option
|
|
|
104
104
|
* similarity threshold. */
|
|
105
105
|
explain?: boolean;
|
|
106
106
|
}): Promise<SearchResult[]>;
|
|
107
|
+
/**
|
|
108
|
+
* List the auto-created tool-error quirks, newest first.
|
|
109
|
+
*
|
|
110
|
+
* Reads the audit log directly instead of going through `listQuirks`: that
|
|
111
|
+
* function falls back to scanning the whole vector store when
|
|
112
|
+
* `quirks.jsonl` is missing, which is far too expensive for a path that runs
|
|
113
|
+
* on every model request. A tool-error quirk is always appended to the audit
|
|
114
|
+
* log, so an absent log means there are none.
|
|
115
|
+
*
|
|
116
|
+
* @param deps - The quirk store dependencies.
|
|
117
|
+
* @returns The tool-error quirks, newest `lastObserved` first.
|
|
118
|
+
*/
|
|
119
|
+
export declare function listToolErrorQuirks(deps: QuirkStoreDeps): Promise<Quirk[]>;
|
|
107
120
|
/** Lint quirks for low confidence, staleness, duplicates, and orphan source refs. */
|
|
108
121
|
export declare function lintQuirks(deps: QuirkStoreDeps): Promise<string[]>;
|
|
109
122
|
/** Simple Jaccard-based lexical similarity (word overlap). */
|
|
@@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { retrieve } from "../retriever/retriever.js";
|
|
5
5
|
import { isQuirkAllowed } from "./monitor.js";
|
|
6
|
+
import { TOOL_ERROR_QUIRK_TAG } from "./types.js";
|
|
6
7
|
export const QUIRK_FILE_PREFIX = "quirk:";
|
|
7
8
|
const QUIKK_JSONL = "quirks.jsonl";
|
|
8
9
|
function jsonlPath(storePath) {
|
|
@@ -562,6 +563,32 @@ export async function recallQuirks(deps, query, options) {
|
|
|
562
563
|
filtered.sort((a, b) => b.score - a.score);
|
|
563
564
|
return filtered.slice(0, topK);
|
|
564
565
|
}
|
|
566
|
+
/**
|
|
567
|
+
* List the auto-created tool-error quirks, newest first.
|
|
568
|
+
*
|
|
569
|
+
* Reads the audit log directly instead of going through `listQuirks`: that
|
|
570
|
+
* function falls back to scanning the whole vector store when
|
|
571
|
+
* `quirks.jsonl` is missing, which is far too expensive for a path that runs
|
|
572
|
+
* on every model request. A tool-error quirk is always appended to the audit
|
|
573
|
+
* log, so an absent log means there are none.
|
|
574
|
+
*
|
|
575
|
+
* @param deps - The quirk store dependencies.
|
|
576
|
+
* @returns The tool-error quirks, newest `lastObserved` first.
|
|
577
|
+
*/
|
|
578
|
+
export async function listToolErrorQuirks(deps) {
|
|
579
|
+
const tagged = (q) => (q.tags ?? []).includes(TOOL_ERROR_QUIRK_TAG);
|
|
580
|
+
if (isMemoryStore(deps.storePath)) {
|
|
581
|
+
return [...memQuirksFor(deps.storePath).values()]
|
|
582
|
+
.filter(tagged)
|
|
583
|
+
.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
|
|
584
|
+
}
|
|
585
|
+
const jp = jsonlPath(deps.storePath);
|
|
586
|
+
if (!existsSync(jp))
|
|
587
|
+
return [];
|
|
588
|
+
return readJsonl(jp)
|
|
589
|
+
.filter(tagged)
|
|
590
|
+
.sort((a, b) => b.lastObserved.localeCompare(a.lastObserved));
|
|
591
|
+
}
|
|
565
592
|
/** Lint quirks for low confidence, staleness, duplicates, and orphan source refs. */
|
|
566
593
|
export async function lintQuirks(deps) {
|
|
567
594
|
const issues = [];
|
|
@@ -21,19 +21,21 @@
|
|
|
21
21
|
* caller logs it.
|
|
22
22
|
*/
|
|
23
23
|
import type { Quirk, QuirkInput } from "./types.js";
|
|
24
|
+
import { TOOL_ERROR_AUTO_TAG, TOOL_ERROR_CAUSE_TAG_PREFIX, TOOL_ERROR_QUIRK_TAG, TOOL_ERROR_TOOL_TAG_PREFIX } from "./types.js";
|
|
24
25
|
import { type ToolErrorInfo } from "./tool-error-recall.js";
|
|
25
|
-
|
|
26
|
-
export declare const TOOL_ERROR_QUIRK_TAG = "tool-error";
|
|
27
|
-
/** Second tag marking the quirk as machine-created (not agent-authored). */
|
|
28
|
-
export declare const TOOL_ERROR_AUTO_TAG = "auto-created";
|
|
26
|
+
export { TOOL_ERROR_AUTO_TAG, TOOL_ERROR_CAUSE_TAG_PREFIX, TOOL_ERROR_QUIRK_TAG, TOOL_ERROR_TOOL_TAG_PREFIX, };
|
|
29
27
|
/** Defaults for the tool-error creation path. */
|
|
30
28
|
export declare const TOOL_ERROR_CREATE_DEFAULTS: {
|
|
31
29
|
/**
|
|
32
30
|
* Minimum raw similarity for a candidate to count as "an equivalent quirk
|
|
33
|
-
* already exists" (and therefore suppress creation). Default 0.
|
|
34
|
-
*
|
|
31
|
+
* already exists" (and therefore suppress creation). Default 0.7 — the
|
|
32
|
+
* injection bar. Measured: an error query scores 0.55-0.63 against
|
|
33
|
+
* ARBITRARY stored quirks, so a lower floor sits inside the noise band and
|
|
34
|
+
* matches unrelated quirks, which suppresses creation on every real failure.
|
|
35
|
+
* The exact duplicate check is done by metadata instead — see
|
|
36
|
+
* `hasEquivalentToolGotcha`.
|
|
35
37
|
*/
|
|
36
|
-
readonly minScore: 0.
|
|
38
|
+
readonly minScore: 0.7;
|
|
37
39
|
/** Maximum quirks created per session. Default 2. */
|
|
38
40
|
readonly maxPerSession: 2;
|
|
39
41
|
/** Cooldown per normalized error signature in ms. Default 10 minutes. */
|
|
@@ -46,7 +48,33 @@ export declare const TOOL_ERROR_CREATE_DEFAULTS: {
|
|
|
46
48
|
readonly confidence: 0.6;
|
|
47
49
|
/** Errors shorter than this carry no information — nothing is recorded. */
|
|
48
50
|
readonly minErrorChars: 15;
|
|
51
|
+
/** Max tool gotchas injected proactively into the system prompt. Default 5. */
|
|
52
|
+
readonly maxProactiveItems: 5;
|
|
49
53
|
};
|
|
54
|
+
/**
|
|
55
|
+
* Cause classes that are worth surfacing BEFORE the tool is called.
|
|
56
|
+
*
|
|
57
|
+
* A transient failure (rate limit, timeout) must NOT be warned about in
|
|
58
|
+
* advance — the next call will not fail the same way, and the warning would
|
|
59
|
+
* be misleading. An unclassified error is excluded for the same reason: we do
|
|
60
|
+
* not know whether it is durable, and a raw error dump says what happened,
|
|
61
|
+
* not what to do.
|
|
62
|
+
*/
|
|
63
|
+
export declare const PROACTIVE_CAUSE_CLASSES: ReadonlySet<string>;
|
|
64
|
+
/** A classified tool-error cause. */
|
|
65
|
+
export interface ToolErrorCause {
|
|
66
|
+
/** Stable class name — also used as the `cause:<class>` tag. */
|
|
67
|
+
cls: string;
|
|
68
|
+
/** Human-readable guidance appended to the quirk content. */
|
|
69
|
+
cause: string;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Classify an error message into a cause class plus its guidance.
|
|
73
|
+
*
|
|
74
|
+
* Returns `undefined` for unknown errors — the raw text is then the whole
|
|
75
|
+
* story.
|
|
76
|
+
*/
|
|
77
|
+
export declare function classifyToolErrorCause(error: string): ToolErrorCause | undefined;
|
|
50
78
|
/**
|
|
51
79
|
* Classify an error message into a short, deterministic cause.
|
|
52
80
|
*
|
|
@@ -109,3 +137,47 @@ export interface ToolErrorQuirkCreateResult {
|
|
|
109
137
|
* `created: false` with a reason so the caller can log it and carry on.
|
|
110
138
|
*/
|
|
111
139
|
export declare function createToolErrorQuirk(deps: ToolErrorQuirkDeps, info: ToolErrorInfo, content?: string): Promise<ToolErrorQuirkCreateResult>;
|
|
140
|
+
/** A stored tool gotcha, ready to be surfaced proactively. */
|
|
141
|
+
export interface ToolGotcha {
|
|
142
|
+
/** The tool the gotcha is about (lowercase). */
|
|
143
|
+
tool: string;
|
|
144
|
+
/** The quirk content — the lesson learned from the failure. */
|
|
145
|
+
content: string;
|
|
146
|
+
/** The quirk id, for `update_quirk` / `delete_quirk`. */
|
|
147
|
+
quirkId: string;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Pick the tool gotchas that are actionable for the tools a session can call.
|
|
151
|
+
*
|
|
152
|
+
* A gotcha about a tool the model cannot invoke is noise, so the tool name
|
|
153
|
+
* is matched against the session's tool catalog. This is a metadata scan, not
|
|
154
|
+
* a semantic recall: the question is "does a quirk exist for THIS tool", not
|
|
155
|
+
* "which quirk is most similar to the current request".
|
|
156
|
+
*
|
|
157
|
+
* @param quirks - Tool-error quirks, newest first.
|
|
158
|
+
* @param toolNames - The tool names the session may call.
|
|
159
|
+
* @param maxItems - Maximum number of gotchas to return.
|
|
160
|
+
*/
|
|
161
|
+
export declare function selectToolGotchas(quirks: readonly Pick<Quirk, "id" | "content" | "tags">[], toolNames: readonly string[], maxItems?: number): ToolGotcha[];
|
|
162
|
+
/**
|
|
163
|
+
* Is there already a tool-error quirk for this tool AND cause class?
|
|
164
|
+
*
|
|
165
|
+
* The exact duplicate check. A semantic similarity floor cannot do this job:
|
|
166
|
+
* an error query scores 0.55-0.63 against arbitrary stored quirks, so any
|
|
167
|
+
* floor low enough to catch a genuine duplicate also catches noise — which
|
|
168
|
+
* silently suppressed creation on every real failure. The tags make the
|
|
169
|
+
* check exact: a quirk created for `write` + `schema` is the same lesson as
|
|
170
|
+
* the next `write` + `schema` failure, whatever the error text says.
|
|
171
|
+
*
|
|
172
|
+
* @param quirks - The stored tool-error quirks.
|
|
173
|
+
* @param tool - The failing tool name.
|
|
174
|
+
* @param causeClass - The classified cause class (e.g. "schema").
|
|
175
|
+
*/
|
|
176
|
+
export declare function hasEquivalentToolGotcha(quirks: readonly Pick<Quirk, "tags">[], tool: string, causeClass: string): boolean;
|
|
177
|
+
/**
|
|
178
|
+
* Format the proactive tool-gotcha block for the system prompt.
|
|
179
|
+
*
|
|
180
|
+
* Injected BEFORE the model chooses a tool, so a known failure is not
|
|
181
|
+
* repeated in a later session. Empty when there are no gotchas.
|
|
182
|
+
*/
|
|
183
|
+
export declare function formatToolGotchaBlock(gotchas: readonly ToolGotcha[]): string;
|
|
@@ -20,19 +20,21 @@
|
|
|
20
20
|
* that contains `rm -rf`) is reported as "not created" with a reason; the
|
|
21
21
|
* caller logs it.
|
|
22
22
|
*/
|
|
23
|
+
import { TOOL_ERROR_AUTO_TAG, TOOL_ERROR_CAUSE_TAG_PREFIX, TOOL_ERROR_QUIRK_TAG, TOOL_ERROR_TOOL_TAG_PREFIX, } from "./types.js";
|
|
23
24
|
import { normalizeToolErrorSignature } from "./tool-error-recall.js";
|
|
24
|
-
|
|
25
|
-
export const TOOL_ERROR_QUIRK_TAG = "tool-error";
|
|
26
|
-
/** Second tag marking the quirk as machine-created (not agent-authored). */
|
|
27
|
-
export const TOOL_ERROR_AUTO_TAG = "auto-created";
|
|
25
|
+
export { TOOL_ERROR_AUTO_TAG, TOOL_ERROR_CAUSE_TAG_PREFIX, TOOL_ERROR_QUIRK_TAG, TOOL_ERROR_TOOL_TAG_PREFIX, };
|
|
28
26
|
/** Defaults for the tool-error creation path. */
|
|
29
27
|
export const TOOL_ERROR_CREATE_DEFAULTS = {
|
|
30
28
|
/**
|
|
31
29
|
* Minimum raw similarity for a candidate to count as "an equivalent quirk
|
|
32
|
-
* already exists" (and therefore suppress creation). Default 0.
|
|
33
|
-
*
|
|
30
|
+
* already exists" (and therefore suppress creation). Default 0.7 — the
|
|
31
|
+
* injection bar. Measured: an error query scores 0.55-0.63 against
|
|
32
|
+
* ARBITRARY stored quirks, so a lower floor sits inside the noise band and
|
|
33
|
+
* matches unrelated quirks, which suppresses creation on every real failure.
|
|
34
|
+
* The exact duplicate check is done by metadata instead — see
|
|
35
|
+
* `hasEquivalentToolGotcha`.
|
|
34
36
|
*/
|
|
35
|
-
minScore: 0.
|
|
37
|
+
minScore: 0.7,
|
|
36
38
|
/** Maximum quirks created per session. Default 2. */
|
|
37
39
|
maxPerSession: 2,
|
|
38
40
|
/** Cooldown per normalized error signature in ms. Default 10 minutes. */
|
|
@@ -45,47 +47,79 @@ export const TOOL_ERROR_CREATE_DEFAULTS = {
|
|
|
45
47
|
confidence: 0.6,
|
|
46
48
|
/** Errors shorter than this carry no information — nothing is recorded. */
|
|
47
49
|
minErrorChars: 15,
|
|
50
|
+
/** Max tool gotchas injected proactively into the system prompt. Default 5. */
|
|
51
|
+
maxProactiveItems: 5,
|
|
48
52
|
};
|
|
49
53
|
/** Known error classes with a deterministic, actionable cause. */
|
|
50
54
|
const ERROR_CAUSES = [
|
|
51
55
|
{
|
|
56
|
+
cls: "unknown-tool",
|
|
52
57
|
pattern: /unknown tool|no such tool|tool .*not found|not a registered tool/i,
|
|
53
58
|
cause: "the tool name may be wrong, or the tool is not available in this session",
|
|
54
59
|
},
|
|
55
60
|
{
|
|
56
|
-
|
|
57
|
-
|
|
61
|
+
cls: "schema",
|
|
62
|
+
pattern: /expected .*(received|but got)|invalid argument|invalid type|schema|must be (a|an|string|number|boolean|object|array)/i,
|
|
63
|
+
cause: "the arguments do not match the tool's schema",
|
|
58
64
|
},
|
|
59
65
|
{
|
|
66
|
+
cls: "permissions",
|
|
60
67
|
pattern: /eacces|permission denied/i,
|
|
61
68
|
cause: "the path may need different permissions",
|
|
62
69
|
},
|
|
63
70
|
{
|
|
64
|
-
|
|
65
|
-
|
|
71
|
+
cls: "path",
|
|
72
|
+
pattern: /enoent|no such file or directory|cannot find module/i,
|
|
73
|
+
cause: "the path or module name may be wrong",
|
|
66
74
|
},
|
|
67
75
|
{
|
|
76
|
+
cls: "rate-limit",
|
|
68
77
|
pattern: /\b429\b|rate limit|too many requests/i,
|
|
69
78
|
cause: "the request was rate limited — retry later",
|
|
70
79
|
},
|
|
71
80
|
{
|
|
81
|
+
cls: "timeout",
|
|
72
82
|
pattern: /timed? ?out|timeout|etimedout/i,
|
|
73
83
|
cause: "the operation timed out",
|
|
74
84
|
},
|
|
75
85
|
];
|
|
76
86
|
/**
|
|
77
|
-
*
|
|
87
|
+
* Cause classes that are worth surfacing BEFORE the tool is called.
|
|
88
|
+
*
|
|
89
|
+
* A transient failure (rate limit, timeout) must NOT be warned about in
|
|
90
|
+
* advance — the next call will not fail the same way, and the warning would
|
|
91
|
+
* be misleading. An unclassified error is excluded for the same reason: we do
|
|
92
|
+
* not know whether it is durable, and a raw error dump says what happened,
|
|
93
|
+
* not what to do.
|
|
94
|
+
*/
|
|
95
|
+
export const PROACTIVE_CAUSE_CLASSES = new Set([
|
|
96
|
+
"unknown-tool",
|
|
97
|
+
"schema",
|
|
98
|
+
"permissions",
|
|
99
|
+
"path",
|
|
100
|
+
]);
|
|
101
|
+
/**
|
|
102
|
+
* Classify an error message into a cause class plus its guidance.
|
|
78
103
|
*
|
|
79
104
|
* Returns `undefined` for unknown errors — the raw text is then the whole
|
|
80
105
|
* story.
|
|
81
106
|
*/
|
|
82
|
-
export function
|
|
83
|
-
for (const { pattern, cause } of ERROR_CAUSES) {
|
|
107
|
+
export function classifyToolErrorCause(error) {
|
|
108
|
+
for (const { cls, pattern, cause } of ERROR_CAUSES) {
|
|
84
109
|
if (pattern.test(error))
|
|
85
|
-
return cause;
|
|
110
|
+
return { cls, cause };
|
|
86
111
|
}
|
|
87
112
|
return undefined;
|
|
88
113
|
}
|
|
114
|
+
/**
|
|
115
|
+
* Classify an error message into a short, deterministic cause.
|
|
116
|
+
*
|
|
117
|
+
* Returns `undefined` for unknown errors — the raw text is then the whole
|
|
118
|
+
* story.
|
|
119
|
+
*/
|
|
120
|
+
export function classifyToolError(error) {
|
|
121
|
+
return classifyToolErrorCause(error)?.cause;
|
|
122
|
+
}
|
|
89
123
|
/**
|
|
90
124
|
* Build the quirk content for a failed tool call.
|
|
91
125
|
*
|
|
@@ -156,11 +190,17 @@ export async function createToolErrorQuirk(deps, info, content) {
|
|
|
156
190
|
if (!text) {
|
|
157
191
|
return { created: false, reason: "empty quirk content" };
|
|
158
192
|
}
|
|
193
|
+
// The cause class is what makes a gotcha usable PROACTIVELY: a transient
|
|
194
|
+
// failure must not be warned about before the next call.
|
|
195
|
+
const cause = classifyToolErrorCause(info.error);
|
|
196
|
+
const tags = [TOOL_ERROR_QUIRK_TAG, TOOL_ERROR_AUTO_TAG, `tool:${info.tool}`];
|
|
197
|
+
if (cause)
|
|
198
|
+
tags.push(`${TOOL_ERROR_CAUSE_TAG_PREFIX}${cause.cls}`);
|
|
159
199
|
try {
|
|
160
200
|
const quirk = await deps.add({
|
|
161
201
|
content: text,
|
|
162
202
|
quirkType: "gotcha",
|
|
163
|
-
tags
|
|
203
|
+
tags,
|
|
164
204
|
confidence: TOOL_ERROR_CREATE_DEFAULTS.confidence,
|
|
165
205
|
sourceRef: `tool-error:${info.tool}`,
|
|
166
206
|
});
|
|
@@ -170,4 +210,83 @@ export async function createToolErrorQuirk(deps, info, content) {
|
|
|
170
210
|
return { created: false, reason: err instanceof Error ? err.message : String(err) };
|
|
171
211
|
}
|
|
172
212
|
}
|
|
213
|
+
/**
|
|
214
|
+
* Pick the tool gotchas that are actionable for the tools a session can call.
|
|
215
|
+
*
|
|
216
|
+
* A gotcha about a tool the model cannot invoke is noise, so the tool name
|
|
217
|
+
* is matched against the session's tool catalog. This is a metadata scan, not
|
|
218
|
+
* a semantic recall: the question is "does a quirk exist for THIS tool", not
|
|
219
|
+
* "which quirk is most similar to the current request".
|
|
220
|
+
*
|
|
221
|
+
* @param quirks - Tool-error quirks, newest first.
|
|
222
|
+
* @param toolNames - The tool names the session may call.
|
|
223
|
+
* @param maxItems - Maximum number of gotchas to return.
|
|
224
|
+
*/
|
|
225
|
+
export function selectToolGotchas(quirks, toolNames, maxItems = TOOL_ERROR_CREATE_DEFAULTS.maxProactiveItems) {
|
|
226
|
+
if (toolNames.length === 0 || maxItems <= 0)
|
|
227
|
+
return [];
|
|
228
|
+
const callable = new Set(toolNames.map((t) => t.toLowerCase()));
|
|
229
|
+
const gotchas = [];
|
|
230
|
+
for (const q of quirks) {
|
|
231
|
+
const tags = q.tags ?? [];
|
|
232
|
+
if (!tags.includes(TOOL_ERROR_QUIRK_TAG))
|
|
233
|
+
continue;
|
|
234
|
+
const toolTag = tags.find((t) => t.startsWith(TOOL_ERROR_TOOL_TAG_PREFIX));
|
|
235
|
+
if (!toolTag)
|
|
236
|
+
continue;
|
|
237
|
+
const tool = toolTag.slice(TOOL_ERROR_TOOL_TAG_PREFIX.length).toLowerCase();
|
|
238
|
+
if (!callable.has(tool))
|
|
239
|
+
continue;
|
|
240
|
+
// Only durable, actionable causes are worth warning about in advance.
|
|
241
|
+
const causeTag = tags.find((t) => t.startsWith(TOOL_ERROR_CAUSE_TAG_PREFIX));
|
|
242
|
+
if (!causeTag)
|
|
243
|
+
continue;
|
|
244
|
+
const cls = causeTag.slice(TOOL_ERROR_CAUSE_TAG_PREFIX.length);
|
|
245
|
+
if (!PROACTIVE_CAUSE_CLASSES.has(cls))
|
|
246
|
+
continue;
|
|
247
|
+
gotchas.push({ tool, content: q.content, quirkId: q.id });
|
|
248
|
+
if (gotchas.length >= maxItems)
|
|
249
|
+
break;
|
|
250
|
+
}
|
|
251
|
+
return gotchas;
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Is there already a tool-error quirk for this tool AND cause class?
|
|
255
|
+
*
|
|
256
|
+
* The exact duplicate check. A semantic similarity floor cannot do this job:
|
|
257
|
+
* an error query scores 0.55-0.63 against arbitrary stored quirks, so any
|
|
258
|
+
* floor low enough to catch a genuine duplicate also catches noise — which
|
|
259
|
+
* silently suppressed creation on every real failure. The tags make the
|
|
260
|
+
* check exact: a quirk created for `write` + `schema` is the same lesson as
|
|
261
|
+
* the next `write` + `schema` failure, whatever the error text says.
|
|
262
|
+
*
|
|
263
|
+
* @param quirks - The stored tool-error quirks.
|
|
264
|
+
* @param tool - The failing tool name.
|
|
265
|
+
* @param causeClass - The classified cause class (e.g. "schema").
|
|
266
|
+
*/
|
|
267
|
+
export function hasEquivalentToolGotcha(quirks, tool, causeClass) {
|
|
268
|
+
const toolTag = `${TOOL_ERROR_TOOL_TAG_PREFIX}${tool.toLowerCase()}`;
|
|
269
|
+
const causeTag = `${TOOL_ERROR_CAUSE_TAG_PREFIX}${causeClass}`;
|
|
270
|
+
return quirks.some((q) => (q.tags ?? []).includes(toolTag) && (q.tags ?? []).includes(causeTag));
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* Format the proactive tool-gotcha block for the system prompt.
|
|
274
|
+
*
|
|
275
|
+
* Injected BEFORE the model chooses a tool, so a known failure is not
|
|
276
|
+
* repeated in a later session. Empty when there are no gotchas.
|
|
277
|
+
*/
|
|
278
|
+
export function formatToolGotchaBlock(gotchas) {
|
|
279
|
+
if (gotchas.length === 0)
|
|
280
|
+
return "";
|
|
281
|
+
const lines = [
|
|
282
|
+
"",
|
|
283
|
+
"---",
|
|
284
|
+
"⚠ **Known tool gotchas** — these tools failed before. Check before calling them:",
|
|
285
|
+
"",
|
|
286
|
+
];
|
|
287
|
+
for (const g of gotchas) {
|
|
288
|
+
lines.push(`- ${g.content}`);
|
|
289
|
+
}
|
|
290
|
+
return lines.join("\n");
|
|
291
|
+
}
|
|
173
292
|
//# sourceMappingURL=tool-error-create.js.map
|
package/dist/quirks/types.d.ts
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
/** Quirk sub-type labels. */
|
|
2
2
|
export type QuirkType = "gotcha" | "preference" | "decision" | "environment-constraint";
|
|
3
|
+
/** Tag every auto-created tool-error quirk carries. */
|
|
4
|
+
export declare const TOOL_ERROR_QUIRK_TAG = "tool-error";
|
|
5
|
+
/** Second tag marking the quirk as machine-created (not agent-authored). */
|
|
6
|
+
export declare const TOOL_ERROR_AUTO_TAG = "auto-created";
|
|
7
|
+
/** Prefix of the per-tool tag: `tool:<name>`. */
|
|
8
|
+
export declare const TOOL_ERROR_TOOL_TAG_PREFIX = "tool:";
|
|
9
|
+
/** Prefix of the cause-class tag: `cause:<class>`. */
|
|
10
|
+
export declare const TOOL_ERROR_CAUSE_TAG_PREFIX = "cause:";
|
|
3
11
|
/** Input for adding a new quirk. */
|
|
4
12
|
export interface QuirkInput {
|
|
5
13
|
content: string;
|
package/dist/quirks/types.js
CHANGED
|
@@ -1,2 +1,9 @@
|
|
|
1
|
-
|
|
1
|
+
/** Tag every auto-created tool-error quirk carries. */
|
|
2
|
+
export const TOOL_ERROR_QUIRK_TAG = "tool-error";
|
|
3
|
+
/** Second tag marking the quirk as machine-created (not agent-authored). */
|
|
4
|
+
export const TOOL_ERROR_AUTO_TAG = "auto-created";
|
|
5
|
+
/** Prefix of the per-tool tag: `tool:<name>`. */
|
|
6
|
+
export const TOOL_ERROR_TOOL_TAG_PREFIX = "tool:";
|
|
7
|
+
/** Prefix of the cause-class tag: `cause:<class>`. */
|
|
8
|
+
export const TOOL_ERROR_CAUSE_TAG_PREFIX = "cause:";
|
|
2
9
|
//# sourceMappingURL=types.js.map
|
|
@@ -2,10 +2,35 @@
|
|
|
2
2
|
* @fileoverview Performs hybrid vector-keyword retrieval with configurable scoring and explanation.
|
|
3
3
|
*/
|
|
4
4
|
import type { EmbeddingProvider, KeywordIndex, RerankProvider, VectorStore, SearchResult, MetadataFilter } from "../core/interfaces.js";
|
|
5
|
+
/**
|
|
6
|
+
* How the `minScore` gate reads the score.
|
|
7
|
+
*
|
|
8
|
+
* - `"fused"` — compare against the raw fused RRF score. Its scale depends on
|
|
9
|
+
* `keywordWeight`, so the same threshold means different things at
|
|
10
|
+
* different weights.
|
|
11
|
+
* - `"normalized"` — compare against the score divided by the maximum that
|
|
12
|
+
* result could reach. Scale-free: the threshold keeps its meaning when
|
|
13
|
+
* `keywordWeight` changes.
|
|
14
|
+
*/
|
|
15
|
+
export type MinScoreMode = "fused" | "normalized";
|
|
5
16
|
/** Options controlling the retrieval behavior. */
|
|
6
17
|
export interface RetrieveOptions {
|
|
7
18
|
topK?: number;
|
|
8
19
|
minScore?: number;
|
|
20
|
+
/**
|
|
21
|
+
* How the `minScore` gate reads the score.
|
|
22
|
+
*
|
|
23
|
+
* - `"fused"` (default): the gate compares against the raw fused RRF score.
|
|
24
|
+
* That score's scale DEPENDS on `keywordWeight` — a vector-only hit can
|
|
25
|
+
* never exceed `(1 - kw)`, so at `kw = 0.4` the fused score maxes out at
|
|
26
|
+
* 0.6 and a fixed `minScore` of 0.65 silently demands a DUAL hit.
|
|
27
|
+
* - `"normalized"`: the fused score is divided by the maximum that result
|
|
28
|
+
* could reach given the legs it has (a dual hit reaches the same maximum
|
|
29
|
+
* at every `keywordWeight`, a vector-only hit reaches `(1 - kw)` times
|
|
30
|
+
* that). The score becomes scale-free, so `minScore` means the same
|
|
31
|
+
* thing at every `keywordWeight`.
|
|
32
|
+
*/
|
|
33
|
+
minScoreMode?: MinScoreMode;
|
|
9
34
|
keywordIndex?: KeywordIndex;
|
|
10
35
|
keywordWeight?: number;
|
|
11
36
|
/** Keyword weight for symbol-style queries (a bare identifier like `cosineSimilarity`).
|
|
@@ -5,6 +5,18 @@ const FETCH_OVERFETCH_FACTOR = 3;
|
|
|
5
5
|
const RRF_K = 60;
|
|
6
6
|
/** Multiply raw RRF scores by (K+1) to normalize to ~[0,1]. */
|
|
7
7
|
const RRF_NORMALIZE = RRF_K + 1;
|
|
8
|
+
/**
|
|
9
|
+
* The highest score a result with the given legs could reach (rank 0 in each).
|
|
10
|
+
*
|
|
11
|
+
* A dual hit reaches `(1 - kw) + kw` of the maximum — the same at every
|
|
12
|
+
* `keywordWeight`. A vector-only hit reaches `(1 - kw)` of it, a keyword-only
|
|
13
|
+
* hit `kw` of it. Dividing by this value is what makes the score scale-free.
|
|
14
|
+
*/
|
|
15
|
+
function maxReachableScore(hasVector, hasKeyword, keywordWeight) {
|
|
16
|
+
const vector = hasVector ? ((1 - keywordWeight) * RRF_NORMALIZE) / (RRF_K + 1) : 0;
|
|
17
|
+
const keyword = hasKeyword ? (keywordWeight * RRF_NORMALIZE) / (RRF_K + 1) : 0;
|
|
18
|
+
return vector + keyword;
|
|
19
|
+
}
|
|
8
20
|
const RERANK_DEFAULTS = {
|
|
9
21
|
candidates: 20,
|
|
10
22
|
maxDocChars: 1200,
|
|
@@ -122,6 +134,7 @@ export async function retrieve(query, embedder, store, options = {}) {
|
|
|
122
134
|
try {
|
|
123
135
|
const topK = options.topK ?? 10;
|
|
124
136
|
const minScore = options.minScore ?? 0;
|
|
137
|
+
const minScoreMode = options.minScoreMode ?? "fused";
|
|
125
138
|
const queryShape = detectQueryShape(query);
|
|
126
139
|
const baseKw = options.keywordWeight ?? 0.4;
|
|
127
140
|
// Symbol queries are exact-identifier lookups — keyword search is the strong
|
|
@@ -182,7 +195,12 @@ export async function retrieve(query, embedder, store, options = {}) {
|
|
|
182
195
|
const kMultiplier = kR !== undefined ? keywordDemotion(chunk, docDemotion, testDemotion) : 1;
|
|
183
196
|
const vContrib = vR !== undefined ? ((1 - kw) * RRF_NORMALIZE) / (RRF_K + vR + 1) : 0;
|
|
184
197
|
const kContrib = kR !== undefined ? (kw * RRF_NORMALIZE) / (RRF_K + kR + 1) * kMultiplier : 0;
|
|
185
|
-
const
|
|
198
|
+
const fusedScore = vContrib + kContrib;
|
|
199
|
+
// In "normalized" mode the score is divided by the maximum this result
|
|
200
|
+
// could reach, so the `minScore` gate keeps its meaning no matter what
|
|
201
|
+
// `keywordWeight` is set to.
|
|
202
|
+
const maxReachable = maxReachableScore(vR !== undefined, kR !== undefined, kw);
|
|
203
|
+
const score = minScoreMode === "normalized" && maxReachable > 0 ? fusedScore / maxReachable : fusedScore;
|
|
186
204
|
const result = { chunk, score };
|
|
187
205
|
if (options.explain) {
|
|
188
206
|
result.explanation = {
|
|
@@ -194,6 +212,7 @@ export async function retrieve(query, embedder, store, options = {}) {
|
|
|
194
212
|
keywordWeight: kw,
|
|
195
213
|
vectorRank: vR,
|
|
196
214
|
keywordRank: kR,
|
|
215
|
+
...(minScoreMode === "normalized" ? { maxReachableScore: maxReachable } : {}),
|
|
197
216
|
},
|
|
198
217
|
queryShape,
|
|
199
218
|
};
|