opencode-rag-plugin 2.5.3 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +10 -4
- package/dist/cli/commands/eval-tuner.d.ts +15 -0
- package/dist/cli/commands/eval-tuner.js +216 -0
- package/dist/cli/commands/index.d.ts +1 -0
- package/dist/cli/commands/index.js +1 -0
- package/dist/cli/commands/status.js +20 -0
- package/dist/cli/index.js +2 -1
- package/dist/core/config.d.ts +27 -0
- package/dist/core/config.js +20 -0
- package/dist/core/interfaces.d.ts +8 -0
- package/dist/eval/tuner.d.ts +118 -0
- package/dist/eval/tuner.js +158 -0
- package/dist/indexer/keyword-rehydration.d.ts +48 -0
- package/dist/indexer/keyword-rehydration.js +113 -0
- package/dist/indexer/pipeline.js +27 -0
- package/dist/plugin.js +124 -15
- package/dist/quirks/quirk-store.d.ts +10 -0
- package/dist/quirks/quirk-store.js +9 -1
- package/dist/quirks/tool-error-create.d.ts +111 -0
- package/dist/quirks/tool-error-create.js +173 -0
- package/dist/quirks/tool-error-recall.d.ts +77 -2
- package/dist/quirks/tool-error-recall.js +58 -22
- package/dist/retriever/keyword-index.d.ts +10 -0
- package/dist/retriever/keyword-index.js +12 -0
- package/package.json +1 -1
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import { runGoldenGate, } from "./golden-gate.js";
|
|
2
|
+
/** The metric used to pick the winner, and the tie-breakers after it. */
|
|
3
|
+
const METRIC_ORDER = {
|
|
4
|
+
ndcg10: (s) => s.ndcg10,
|
|
5
|
+
hit5: (s) => s.hit5,
|
|
6
|
+
hit1: (s) => s.hit1,
|
|
7
|
+
mrr: (s) => s.mrr,
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Build the candidate grid.
|
|
11
|
+
*
|
|
12
|
+
* The base candidate is always included so the sweep can be compared against
|
|
13
|
+
* the configuration that is actually running. Duplicates (same weights, same
|
|
14
|
+
* minScore, same topK) are removed, and the grid is capped so a wide sweep
|
|
15
|
+
* cannot turn into an accidental benchmark.
|
|
16
|
+
*
|
|
17
|
+
* @param base - The candidate describing the current configuration.
|
|
18
|
+
* @param options - Grid overrides.
|
|
19
|
+
* @returns The candidate list, base candidate first.
|
|
20
|
+
*/
|
|
21
|
+
export function buildCandidateGrid(base, options = {}) {
|
|
22
|
+
const keywordWeights = options.keywordWeights ?? [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6];
|
|
23
|
+
const symbolKeywordWeights = options.symbolKeywordWeights ?? keywordWeights;
|
|
24
|
+
const minScores = options.minScores ?? [base.minScore];
|
|
25
|
+
const topKs = options.topKs ?? [base.topK];
|
|
26
|
+
const maxCandidates = options.maxCandidates ?? 64;
|
|
27
|
+
const seen = new Set();
|
|
28
|
+
const candidates = [];
|
|
29
|
+
const push = (candidate) => {
|
|
30
|
+
const key = [
|
|
31
|
+
candidate.keywordWeight,
|
|
32
|
+
candidate.symbolKeywordWeight ?? "unset",
|
|
33
|
+
candidate.minScore,
|
|
34
|
+
candidate.topK,
|
|
35
|
+
].join("|");
|
|
36
|
+
if (seen.has(key))
|
|
37
|
+
return;
|
|
38
|
+
seen.add(key);
|
|
39
|
+
candidates.push(candidate);
|
|
40
|
+
};
|
|
41
|
+
push({ ...base });
|
|
42
|
+
for (const topK of topKs) {
|
|
43
|
+
for (const minScore of minScores) {
|
|
44
|
+
for (const keywordWeight of keywordWeights) {
|
|
45
|
+
for (const symbolKeywordWeight of symbolKeywordWeights) {
|
|
46
|
+
push({ keywordWeight, symbolKeywordWeight, minScore, topK });
|
|
47
|
+
if (candidates.length >= maxCandidates)
|
|
48
|
+
return candidates;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return candidates;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Evaluate every candidate against the golden set.
|
|
57
|
+
*
|
|
58
|
+
* @param opts - Labels, root, retrieve closure, candidates, logger.
|
|
59
|
+
* @returns One result per candidate, in the order the candidates were given.
|
|
60
|
+
*/
|
|
61
|
+
export async function runTuner(opts) {
|
|
62
|
+
const log = opts.log ?? (() => { });
|
|
63
|
+
const results = [];
|
|
64
|
+
for (const candidate of opts.candidates) {
|
|
65
|
+
const started = performance.now();
|
|
66
|
+
const gateOpts = {
|
|
67
|
+
labels: opts.labels,
|
|
68
|
+
root: opts.root,
|
|
69
|
+
retrieve: (label) => opts.retrieve(label, candidate),
|
|
70
|
+
log: () => { },
|
|
71
|
+
};
|
|
72
|
+
const { summary, outcomes } = await runGoldenGate(gateOpts);
|
|
73
|
+
const durationMs = performance.now() - started;
|
|
74
|
+
results.push({ ...candidate, summary, outcomes, durationMs });
|
|
75
|
+
log(` kw=${candidate.keywordWeight} skw=${candidate.symbolKeywordWeight ?? "-"} ` +
|
|
76
|
+
`minScore=${candidate.minScore} topK=${candidate.topK} ` +
|
|
77
|
+
`Hit@5 ${(summary.hit5 * 100).toFixed(1)}% MRR ${summary.mrr.toFixed(3)} ` +
|
|
78
|
+
`nDCG@10 ${summary.ndcg10.toFixed(3)} (${durationMs.toFixed(0)}ms)`);
|
|
79
|
+
}
|
|
80
|
+
return results;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Pick the best candidate.
|
|
84
|
+
*
|
|
85
|
+
* Ranked by the primary metric, then by Hit@5, MRR and Hit@1 so a tie on the
|
|
86
|
+
* primary metric still resolves to the more useful configuration.
|
|
87
|
+
*
|
|
88
|
+
* @param results - The measured candidates.
|
|
89
|
+
* @param metric - The primary metric (default `ndcg10`).
|
|
90
|
+
* @returns The winning result, or `undefined` when there are no results.
|
|
91
|
+
*/
|
|
92
|
+
export function selectBestTuning(results, metric = "ndcg10") {
|
|
93
|
+
if (results.length === 0)
|
|
94
|
+
return undefined;
|
|
95
|
+
const primary = METRIC_ORDER[metric];
|
|
96
|
+
const score = (r) => [
|
|
97
|
+
primary(r.summary),
|
|
98
|
+
r.summary.hit5,
|
|
99
|
+
r.summary.mrr,
|
|
100
|
+
r.summary.hit1,
|
|
101
|
+
];
|
|
102
|
+
return [...results].sort((a, b) => {
|
|
103
|
+
const sa = score(a);
|
|
104
|
+
const sb = score(b);
|
|
105
|
+
for (let i = 0; i < sa.length; i++) {
|
|
106
|
+
if (sb[i] !== sa[i])
|
|
107
|
+
return sb[i] - sa[i];
|
|
108
|
+
}
|
|
109
|
+
return 0;
|
|
110
|
+
})[0];
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Compare the winner against the candidate that describes the current config.
|
|
114
|
+
*
|
|
115
|
+
* @param best - The winning candidate.
|
|
116
|
+
* @param current - The candidate describing the running configuration.
|
|
117
|
+
* @returns The per-metric deltas (winner minus current).
|
|
118
|
+
*/
|
|
119
|
+
export function tuningDelta(best, current) {
|
|
120
|
+
if (!current)
|
|
121
|
+
return {};
|
|
122
|
+
return {
|
|
123
|
+
hit1: best.summary.hit1 - current.summary.hit1,
|
|
124
|
+
hit3: best.summary.hit3 - current.summary.hit3,
|
|
125
|
+
hit5: best.summary.hit5 - current.summary.hit5,
|
|
126
|
+
hit10: best.summary.hit10 - current.summary.hit10,
|
|
127
|
+
mrr: best.summary.mrr - current.summary.mrr,
|
|
128
|
+
ndcg10: best.summary.ndcg10 - current.summary.ndcg10,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Write the winning weights into a copy of the configuration.
|
|
133
|
+
*
|
|
134
|
+
* The input config is not mutated — the caller decides whether to persist the
|
|
135
|
+
* result. Only the retrieval section is touched.
|
|
136
|
+
*
|
|
137
|
+
* @param config - The configuration to base the result on.
|
|
138
|
+
* @param best - The winning candidate.
|
|
139
|
+
* @returns A new config with the winning fusion weights applied.
|
|
140
|
+
*/
|
|
141
|
+
export function applyTuningToConfig(config, best) {
|
|
142
|
+
const hybrid = config.retrieval.hybridSearch ?? { enabled: true, keywordWeight: best.keywordWeight };
|
|
143
|
+
return {
|
|
144
|
+
...config,
|
|
145
|
+
retrieval: {
|
|
146
|
+
...config.retrieval,
|
|
147
|
+
topK: best.topK,
|
|
148
|
+
minScore: best.minScore,
|
|
149
|
+
hybridSearch: {
|
|
150
|
+
...hybrid,
|
|
151
|
+
enabled: hybrid.enabled ?? true,
|
|
152
|
+
keywordWeight: best.keywordWeight,
|
|
153
|
+
symbolKeywordWeight: best.symbolKeywordWeight ?? hybrid.symbolKeywordWeight,
|
|
154
|
+
},
|
|
155
|
+
},
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
//# sourceMappingURL=tuner.js.map
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { KeywordIndex, VectorStore } from "../core/interfaces.js";
|
|
2
|
+
/** Options for {@link rehydrateKeywordIndex}. */
|
|
3
|
+
export interface KeywordRehydrationOptions {
|
|
4
|
+
/** The vector store holding the authoritative chunk rows. */
|
|
5
|
+
store: VectorStore;
|
|
6
|
+
/** The keyword index to repair. */
|
|
7
|
+
keywordIndex: KeywordIndex;
|
|
8
|
+
/** Optional logger (missing methods default to no-ops). */
|
|
9
|
+
logger?: {
|
|
10
|
+
info?(message: string): void;
|
|
11
|
+
warn?(message: string): void;
|
|
12
|
+
debug?(message: string): void;
|
|
13
|
+
};
|
|
14
|
+
/** Maximum number of files to rehydrate in one call (0/undefined = unlimited). */
|
|
15
|
+
maxFiles?: number;
|
|
16
|
+
}
|
|
17
|
+
/** Outcome of a rehydration attempt. */
|
|
18
|
+
export interface KeywordRehydrationResult {
|
|
19
|
+
/** Files present in the store with no keyword entries. */
|
|
20
|
+
missingFiles: number;
|
|
21
|
+
/** Files whose chunks were added to the keyword index. */
|
|
22
|
+
rehydratedFiles: number;
|
|
23
|
+
/** Chunks added to the keyword index. */
|
|
24
|
+
rehydratedChunks: number;
|
|
25
|
+
/** Chunks skipped because they were already indexed. */
|
|
26
|
+
skippedChunks: number;
|
|
27
|
+
/** Files that could not be rehydrated (one failure never aborts the rest). */
|
|
28
|
+
failedFiles: string[];
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Restore the keyword entries for every file the store holds but the keyword
|
|
32
|
+
* index does not.
|
|
33
|
+
*
|
|
34
|
+
* Quirk rows are skipped: `quirks.jsonl` owns them, and `reconcileQuirks()`
|
|
35
|
+
* adds and removes them under its own rules.
|
|
36
|
+
*
|
|
37
|
+
* @param options - The store, the keyword index, and optional logging.
|
|
38
|
+
* @returns What was missing and what was restored.
|
|
39
|
+
*/
|
|
40
|
+
export declare function rehydrateKeywordIndex(options: KeywordRehydrationOptions): Promise<KeywordRehydrationResult>;
|
|
41
|
+
/**
|
|
42
|
+
* Compare the keyword index against the vector store without changing either.
|
|
43
|
+
*
|
|
44
|
+
* @param store - The vector store holding the authoritative chunk rows.
|
|
45
|
+
* @param keywordIndex - The keyword index to compare.
|
|
46
|
+
* @returns The files the store has that the keyword index does not cover.
|
|
47
|
+
*/
|
|
48
|
+
export declare function findKeywordIndexGaps(store: VectorStore, keywordIndex: KeywordIndex): Promise<string[]>;
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Repairs a keyword index that drifted behind the vector store.
|
|
3
|
+
*
|
|
4
|
+
* The keyword index (BM25 side of hybrid search) and the vector store are two
|
|
5
|
+
* independent stores of the same corpus. Nothing keeps them in sync by
|
|
6
|
+
* construction: an incremental index pass only calls `addChunks()` for the
|
|
7
|
+
* files it reprocessed, so unchanged files keep whatever the *loaded* index
|
|
8
|
+
* had. Once the persisted index loses entries — a crashed pass that saved a
|
|
9
|
+
* cleared index, a partial indexer, a rebuild that swapped the store without
|
|
10
|
+
* rewriting the sidecar — every later pass re-saves the same partial state
|
|
11
|
+
* and the missing files stay unreachable through keyword search forever.
|
|
12
|
+
*
|
|
13
|
+
* The repair needs no re-embedding: the store already holds the chunk text
|
|
14
|
+
* and descriptions, so the missing keyword entries can be rebuilt from the
|
|
15
|
+
* store's own rows.
|
|
16
|
+
*/
|
|
17
|
+
import path from "node:path";
|
|
18
|
+
import { normalizeFilePath } from "../core/manifest.js";
|
|
19
|
+
import { QUIRK_FILE_PREFIX } from "../quirks/quirk-store.js";
|
|
20
|
+
/**
|
|
21
|
+
* Restore the keyword entries for every file the store holds but the keyword
|
|
22
|
+
* index does not.
|
|
23
|
+
*
|
|
24
|
+
* Quirk rows are skipped: `quirks.jsonl` owns them, and `reconcileQuirks()`
|
|
25
|
+
* adds and removes them under its own rules.
|
|
26
|
+
*
|
|
27
|
+
* @param options - The store, the keyword index, and optional logging.
|
|
28
|
+
* @returns What was missing and what was restored.
|
|
29
|
+
*/
|
|
30
|
+
export async function rehydrateKeywordIndex(options) {
|
|
31
|
+
const { store, keywordIndex, logger } = options;
|
|
32
|
+
const result = {
|
|
33
|
+
missingFiles: 0,
|
|
34
|
+
rehydratedFiles: 0,
|
|
35
|
+
rehydratedChunks: 0,
|
|
36
|
+
skippedChunks: 0,
|
|
37
|
+
failedFiles: [],
|
|
38
|
+
};
|
|
39
|
+
let storePaths;
|
|
40
|
+
try {
|
|
41
|
+
storePaths = await store.getFilePaths();
|
|
42
|
+
}
|
|
43
|
+
catch (err) {
|
|
44
|
+
logger?.warn?.(`Keyword rehydration skipped — could not list store files: ${err.message}`);
|
|
45
|
+
return result;
|
|
46
|
+
}
|
|
47
|
+
if (storePaths.length === 0)
|
|
48
|
+
return result;
|
|
49
|
+
const indexed = new Set(keywordIndex.getIndexedFilePaths());
|
|
50
|
+
const missing = storePaths.filter((p) => {
|
|
51
|
+
// Quirk rows belong to quirks.jsonl — reconcileQuirks() owns them.
|
|
52
|
+
if (path.basename(p).startsWith(QUIRK_FILE_PREFIX))
|
|
53
|
+
return false;
|
|
54
|
+
return !indexed.has(normalizeFilePath(p));
|
|
55
|
+
});
|
|
56
|
+
result.missingFiles = missing.length;
|
|
57
|
+
if (missing.length === 0)
|
|
58
|
+
return result;
|
|
59
|
+
const maxFiles = options.maxFiles && options.maxFiles > 0 ? options.maxFiles : missing.length;
|
|
60
|
+
const targets = missing.slice(0, maxFiles);
|
|
61
|
+
if (targets.length < missing.length) {
|
|
62
|
+
logger?.info?.(`Keyword rehydration: ${missing.length} file(s) missing from the keyword index — rehydrating the first ${targets.length}`);
|
|
63
|
+
}
|
|
64
|
+
for (const filePath of targets) {
|
|
65
|
+
try {
|
|
66
|
+
const chunks = await store.getChunksByFilePath(filePath);
|
|
67
|
+
if (chunks.length === 0)
|
|
68
|
+
continue;
|
|
69
|
+
// Guard against double counting: `addChunks()` is not idempotent for the
|
|
70
|
+
// BM25 length statistics and term frequencies, so a chunk that is
|
|
71
|
+
// already indexed must never be added a second time.
|
|
72
|
+
const fresh = chunks.filter((c) => !keywordIndex.hasChunk(c.id));
|
|
73
|
+
result.skippedChunks += chunks.length - fresh.length;
|
|
74
|
+
if (fresh.length === 0)
|
|
75
|
+
continue;
|
|
76
|
+
keywordIndex.addChunks(fresh);
|
|
77
|
+
result.rehydratedFiles++;
|
|
78
|
+
result.rehydratedChunks += fresh.length;
|
|
79
|
+
}
|
|
80
|
+
catch (err) {
|
|
81
|
+
result.failedFiles.push(filePath);
|
|
82
|
+
logger?.warn?.(`Keyword rehydration failed for ${filePath}: ${err.message}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (result.rehydratedFiles > 0) {
|
|
86
|
+
logger?.info?.(`Keyword rehydration: restored ${result.rehydratedChunks} chunk(s) from ` +
|
|
87
|
+
`${result.rehydratedFiles} file(s) (${result.missingFiles} file(s) were missing from the keyword index)`);
|
|
88
|
+
}
|
|
89
|
+
return result;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Compare the keyword index against the vector store without changing either.
|
|
93
|
+
*
|
|
94
|
+
* @param store - The vector store holding the authoritative chunk rows.
|
|
95
|
+
* @param keywordIndex - The keyword index to compare.
|
|
96
|
+
* @returns The files the store has that the keyword index does not cover.
|
|
97
|
+
*/
|
|
98
|
+
export async function findKeywordIndexGaps(store, keywordIndex) {
|
|
99
|
+
let storePaths;
|
|
100
|
+
try {
|
|
101
|
+
storePaths = await store.getFilePaths();
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
return [];
|
|
105
|
+
}
|
|
106
|
+
const indexed = new Set(keywordIndex.getIndexedFilePaths());
|
|
107
|
+
return storePaths.filter((p) => {
|
|
108
|
+
if (path.basename(p).startsWith(QUIRK_FILE_PREFIX))
|
|
109
|
+
return false;
|
|
110
|
+
return !indexed.has(normalizeFilePath(p));
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=keyword-rehydration.js.map
|
package/dist/indexer/pipeline.js
CHANGED
|
@@ -11,6 +11,7 @@ import { embedBatch, probeEmbeddingDimension } from "../embedder/factory.js";
|
|
|
11
11
|
import { createVectorStore } from "../vectorstore/factory.js";
|
|
12
12
|
import { swapStoreDirectories } from "../vectorstore/lancedb.js";
|
|
13
13
|
import { reconcileQuirks, QUIRK_FILE_PREFIX } from "../quirks/quirk-store.js";
|
|
14
|
+
import { rehydrateKeywordIndex } from "./keyword-rehydration.js";
|
|
14
15
|
import { createIndexStats } from "./stats.js";
|
|
15
16
|
import { prepareFile, buildTextsToEmbed } from "./worker.js";
|
|
16
17
|
import { tagChunksRole } from "../core/chunk-role.js";
|
|
@@ -309,6 +310,32 @@ async function runIndexPassInner(options, logger) {
|
|
|
309
310
|
}
|
|
310
311
|
let filterPaths;
|
|
311
312
|
let gitDeletedPaths = [];
|
|
313
|
+
// ── Keyword-index rehydration ──────────────────────────────────────────
|
|
314
|
+
// The keyword index (BM25 half of hybrid search) and the vector store are
|
|
315
|
+
// two independent stores of the same corpus, and nothing keeps them in sync
|
|
316
|
+
// by construction: an incremental pass only re-adds the files it reprocessed,
|
|
317
|
+
// so a once-partial index stays partial forever and the missing files are
|
|
318
|
+
// unreachable through keyword search. Rebuild the missing entries from the
|
|
319
|
+
// store's own rows — no re-embedding needed.
|
|
320
|
+
//
|
|
321
|
+
// Skipped during a rebuild: the rebuild re-indexes every file from source,
|
|
322
|
+
// so the keyword index is repopulated from scratch anyway.
|
|
323
|
+
if (options.keywordIndex && !rebuildPerformed) {
|
|
324
|
+
try {
|
|
325
|
+
await rehydrateKeywordIndex({
|
|
326
|
+
store: effectiveStore,
|
|
327
|
+
keywordIndex: options.keywordIndex,
|
|
328
|
+
logger: {
|
|
329
|
+
info: (message) => logger.info(message),
|
|
330
|
+
warn: (message) => logger.warn(message),
|
|
331
|
+
debug: (message) => logger.debug(message),
|
|
332
|
+
},
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
catch (err) {
|
|
336
|
+
logger.warn(`Keyword rehydration failed: ${err.message}`);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
312
339
|
if (!options.force && manifestStatus === "ok" && manifest.lastGitCommit) {
|
|
313
340
|
const repoRoot = getRepoRoot(options.cwd);
|
|
314
341
|
if (repoRoot) {
|
package/dist/plugin.js
CHANGED
|
@@ -29,9 +29,10 @@ import { countTokens } from "./eval/token-counter.js";
|
|
|
29
29
|
import { checkForUpdate, getCurrentVersion, installLatestUpdate } from "./core/version-check.js";
|
|
30
30
|
import { loadAutoUpdateState, saveAutoUpdateState, shouldAttemptInstall } from "./core/auto-update-state.js";
|
|
31
31
|
import { destroyAllPooledConnections } from "./embedder/http.js";
|
|
32
|
-
import { listQuirks, lintQuirks, recallQuirks, sharedWords, reconcileQuirks } from "./quirks/quirk-store.js";
|
|
32
|
+
import { listQuirks, lintQuirks, recallQuirks, sharedWords, reconcileQuirks, addQuirk } from "./quirks/quirk-store.js";
|
|
33
33
|
import { autoCaptureQuirks } from "./quirks/auto-capture.js";
|
|
34
|
-
import { createToolErrorInjectionState,
|
|
34
|
+
import { createToolErrorInjectionState, recallToolErrorQuirks, formatToolErrorQuirkBlock, toToolErrorCaptureEntry, } from "./quirks/tool-error-recall.js";
|
|
35
|
+
import { buildToolErrorQuirkContent, createToolErrorCreateState, createToolErrorQuirk, TOOL_ERROR_QUIRK_TAG, } from "./quirks/tool-error-create.js";
|
|
35
36
|
import { buildSystemGuidanceLines } from "./opencode/system-guidance.js";
|
|
36
37
|
import { existsSync, readFileSync, readdirSync, statSync, unlinkSync } from "node:fs";
|
|
37
38
|
import path from "node:path";
|
|
@@ -641,6 +642,9 @@ export function createRagHooks(options) {
|
|
|
641
642
|
// Tool-error quirk path: per-session injection state (cooldown/budget) and the
|
|
642
643
|
// block pending for the next model request.
|
|
643
644
|
const sessionToolErrorState = new Map();
|
|
645
|
+
// Tool-error quirk creation: per-session budget/cooldown for the quirks the
|
|
646
|
+
// plugin writes itself when a failure has no matching memory.
|
|
647
|
+
const sessionToolErrorCreateState = new Map();
|
|
644
648
|
const sessionPendingToolErrorQuirk = new Map();
|
|
645
649
|
// Evaluation session logger — captures OpenCode events for analysis
|
|
646
650
|
const sessionLogger = createSessionLogger(options.storePath);
|
|
@@ -1004,22 +1008,113 @@ export function createRagHooks(options) {
|
|
|
1004
1008
|
});
|
|
1005
1009
|
boundedSet(sessionToolErrorState, input.sessionID, state, MAX_SESSION_MAP_SIZE);
|
|
1006
1010
|
const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
|
|
1007
|
-
|
|
1011
|
+
// `explain: true` keeps the raw vector similarity on each result —
|
|
1012
|
+
// the confidence-discounted `score` is not comparable to a
|
|
1013
|
+
// similarity threshold, and the tiering below needs the raw value.
|
|
1014
|
+
const recall = (query, recallOpts) => recallQuirks(qDeps, query, { ...recallOpts, explain: true }).then((rs) => rs.map((r) => ({
|
|
1015
|
+
chunk: {
|
|
1016
|
+
id: r.chunk.id,
|
|
1017
|
+
content: r.chunk.content,
|
|
1018
|
+
metadata: { quirkType: r.chunk.metadata.quirkType, tags: r.chunk.metadata.tags },
|
|
1019
|
+
},
|
|
1020
|
+
score: r.score,
|
|
1021
|
+
rawSimilarity: r.explanation?.scoreBreakdown?.rawVectorScore,
|
|
1022
|
+
})));
|
|
1008
1023
|
const budgetMs = memCfg.toolErrorLatencyBudgetMs ?? 1500;
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1024
|
+
// One recall, two bars: the floor decides which quirks are
|
|
1025
|
+
// candidates at all, the injection bar decides which are injected.
|
|
1026
|
+
// A candidate between the two is a "probable match": it suppresses
|
|
1027
|
+
// quirk creation but is only injected when no confident match
|
|
1028
|
+
// exists.
|
|
1029
|
+
const injectBar = memCfg.toolErrorMinScore ?? 0.7;
|
|
1030
|
+
const floorBar = memCfg.toolErrorCreateMinScore ?? 0.6;
|
|
1031
|
+
const topK = memCfg.toolErrorTopK ?? 2;
|
|
1032
|
+
let timer;
|
|
1033
|
+
const recallResult = await Promise.race([
|
|
1034
|
+
recallToolErrorQuirks(recall, info, state, {
|
|
1035
|
+
minScore: floorBar,
|
|
1036
|
+
injectMinScore: injectBar,
|
|
1037
|
+
topK,
|
|
1013
1038
|
minTokenOverlap: memCfg.toolErrorMinTokenOverlap ?? 1,
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1039
|
+
// The injection budget limits CONTEXT, not memory: when it is
|
|
1040
|
+
// spent we still need the candidate list to decide whether a
|
|
1041
|
+
// quirk already covers this failure (and whether to create
|
|
1042
|
+
// one). The cooldown still applies.
|
|
1043
|
+
enforceBudget: state.hasBudget(),
|
|
1018
1044
|
}),
|
|
1019
1045
|
new Promise((resolve) => {
|
|
1020
|
-
setTimeout(resolve, budgetMs);
|
|
1046
|
+
timer = setTimeout(() => resolve(undefined), budgetMs);
|
|
1021
1047
|
}),
|
|
1022
|
-
]).catch(() =>
|
|
1048
|
+
]).catch(() => undefined);
|
|
1049
|
+
if (timer)
|
|
1050
|
+
clearTimeout(timer);
|
|
1051
|
+
if (recallResult && !recallResult.blocked) {
|
|
1052
|
+
// Tier 1: confident matches (>= inject bar). Tier 2: probable
|
|
1053
|
+
// matches (>= floor bar) — used only when tier 1 is empty.
|
|
1054
|
+
const tier1 = recallResult.inject;
|
|
1055
|
+
const tier2 = recallResult.candidates.filter((c) => c.rawSimilarity < injectBar && !state.injectedQuirkIds.has(c.id));
|
|
1056
|
+
const hits = tier1.length > 0 ? tier1 : tier2.slice(0, topK);
|
|
1057
|
+
state.record(info, hits.map((h) => h.id));
|
|
1058
|
+
// No candidate at all → nothing in memory covers this failure.
|
|
1059
|
+
// Record one so a future occurrence can be answered from memory.
|
|
1060
|
+
let createdHit;
|
|
1061
|
+
if (recallResult.candidates.length === 0 && memCfg.autoCreateOnToolError) {
|
|
1062
|
+
const createState = sessionToolErrorCreateState.get(input.sessionID) ?? createToolErrorCreateState({
|
|
1063
|
+
maxPerSession: memCfg.toolErrorCreateMaxPerSession ?? 2,
|
|
1064
|
+
cooldownMs: memCfg.toolErrorCreateCooldownMs ?? 600_000,
|
|
1065
|
+
});
|
|
1066
|
+
boundedSet(sessionToolErrorCreateState, input.sessionID, createState, MAX_SESSION_MAP_SIZE);
|
|
1067
|
+
if (createState.shouldCreate(info)) {
|
|
1068
|
+
const content = buildToolErrorQuirkContent(info);
|
|
1069
|
+
const createBudgetMs = memCfg.toolErrorCreateLatencyBudgetMs ?? 2000;
|
|
1070
|
+
let createTimer;
|
|
1071
|
+
const created = await Promise.race([
|
|
1072
|
+
createToolErrorQuirk({ add: (q) => addQuirk(qDeps, q) }, info, content),
|
|
1073
|
+
new Promise((resolve) => {
|
|
1074
|
+
createTimer = setTimeout(() => resolve(undefined), createBudgetMs);
|
|
1075
|
+
}),
|
|
1076
|
+
]).catch(() => undefined);
|
|
1077
|
+
if (createTimer)
|
|
1078
|
+
clearTimeout(createTimer);
|
|
1079
|
+
if (created?.created && created.id) {
|
|
1080
|
+
createState.record(info, true);
|
|
1081
|
+
appendDebugLog(options.logFilePath, {
|
|
1082
|
+
scope: "quirk.tool-error",
|
|
1083
|
+
message: `created quirk ${created.id} for tool "${input.tool}"`,
|
|
1084
|
+
});
|
|
1085
|
+
if (memCfg.toolErrorInjectCreated) {
|
|
1086
|
+
createdHit = {
|
|
1087
|
+
id: created.id,
|
|
1088
|
+
content,
|
|
1089
|
+
quirkType: "gotcha",
|
|
1090
|
+
score: 1,
|
|
1091
|
+
rawSimilarity: 1,
|
|
1092
|
+
};
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
else {
|
|
1096
|
+
// A trust-monitor rejection or a store failure — log it and
|
|
1097
|
+
// carry on; the tool error must never break the session.
|
|
1098
|
+
createState.record(info, false);
|
|
1099
|
+
appendDebugLog(options.logFilePath, {
|
|
1100
|
+
scope: "quirk.tool-error",
|
|
1101
|
+
message: `quirk not created for tool "${input.tool}": ${created?.reason ?? "timeout"}`,
|
|
1102
|
+
});
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
const finalHits = createdHit ? [...hits, createdHit] : hits;
|
|
1107
|
+
if (finalHits.length > 0) {
|
|
1108
|
+
boundedSet(sessionPendingToolErrorQuirk, input.sessionID, {
|
|
1109
|
+
block: formatToolErrorQuirkBlock(info, finalHits, {
|
|
1110
|
+
heading: createdHit
|
|
1111
|
+
? `⚠ **Tool error** \`${info.tool}\` — quirk memory (incl. newly recorded):`
|
|
1112
|
+
: undefined,
|
|
1113
|
+
}),
|
|
1114
|
+
ts: Date.now(),
|
|
1115
|
+
}, MAX_SESSION_MAP_SIZE);
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1023
1118
|
}
|
|
1024
1119
|
}
|
|
1025
1120
|
// Cap the per-session list — long sessions accumulate full tool
|
|
@@ -1091,7 +1186,13 @@ export function createRagHooks(options) {
|
|
|
1091
1186
|
const budgetMs = memCfg.autoInjectLatencyBudgetMs ?? 2000;
|
|
1092
1187
|
let timer;
|
|
1093
1188
|
let quirkResults = await Promise.race([
|
|
1094
|
-
recallQuirks(quirkDeps, query, {
|
|
1189
|
+
recallQuirks(quirkDeps, query, {
|
|
1190
|
+
topK: memCfg.autoInjectTopK ?? 1,
|
|
1191
|
+
minScore: memCfg.autoInjectMinScore ?? 0.75,
|
|
1192
|
+
// Auto-created tool-error quirks are raw error dumps — they
|
|
1193
|
+
// are surfaced by the tool-error path, not on unrelated turns.
|
|
1194
|
+
excludeTags: [TOOL_ERROR_QUIRK_TAG],
|
|
1195
|
+
})
|
|
1095
1196
|
.then((r) => {
|
|
1096
1197
|
if (timer)
|
|
1097
1198
|
clearTimeout(timer);
|
|
@@ -1383,6 +1484,8 @@ export function createRagHooks(options) {
|
|
|
1383
1484
|
memCfg?.passiveCapture ? "passive-capture" : "",
|
|
1384
1485
|
memCfg?.promptEnforcement ? "prompt-enforce" : "",
|
|
1385
1486
|
memCfg?.sessionEndExtraction ? "session-end" : "",
|
|
1487
|
+
memCfg?.injectOnToolError ? "tool-error-inject" : "",
|
|
1488
|
+
memCfg?.autoCreateOnToolError ? "tool-error-create" : "",
|
|
1386
1489
|
].filter(Boolean);
|
|
1387
1490
|
if (flags.length > 0) {
|
|
1388
1491
|
lines.push(`Flags: ${flags.join(", ")}`);
|
|
@@ -1416,7 +1519,13 @@ export function createRagHooks(options) {
|
|
|
1416
1519
|
const minScore = quirkMemoryCfg.autoInjectMinScore ?? 0.75;
|
|
1417
1520
|
let timer;
|
|
1418
1521
|
let quirkResults = await Promise.race([
|
|
1419
|
-
recallQuirks(quirkDeps, quirkQuery, {
|
|
1522
|
+
recallQuirks(quirkDeps, quirkQuery, {
|
|
1523
|
+
topK: quirkMemoryCfg.autoInjectTopK ?? 1,
|
|
1524
|
+
minScore,
|
|
1525
|
+
// Auto-created tool-error quirks are raw error dumps — they
|
|
1526
|
+
// are surfaced by the tool-error path, not on unrelated turns.
|
|
1527
|
+
excludeTags: [TOOL_ERROR_QUIRK_TAG],
|
|
1528
|
+
})
|
|
1420
1529
|
.then((r) => {
|
|
1421
1530
|
if (timer)
|
|
1422
1531
|
clearTimeout(timer);
|
|
@@ -93,6 +93,16 @@ export declare function recallQuirks(deps: QuirkStoreDeps, query: string, option
|
|
|
93
93
|
* cosine mapped to [0,1] the same way the vector stores report raw scores).
|
|
94
94
|
* Auto-inject passes its own (usually higher) bar. */
|
|
95
95
|
minScore?: number;
|
|
96
|
+
/** Exclude quirks carrying any of these tags. Unlike `tags` (any-of
|
|
97
|
+
* inclusion), this is a negative filter — used to keep specialized quirk
|
|
98
|
+
* classes (e.g. `tool-error`) out of the generic auto-inject path. */
|
|
99
|
+
excludeTags?: string[];
|
|
100
|
+
/** Keep `explanation` on the returned results. The caller can then read the
|
|
101
|
+
* RAW vector similarity (`explanation.scoreBreakdown.rawVectorScore`),
|
|
102
|
+
* which is not the same as `score`: `score` is confidence-discounted
|
|
103
|
+
* (`similarity * confidence`), so it cannot be compared against a
|
|
104
|
+
* similarity threshold. */
|
|
105
|
+
explain?: boolean;
|
|
96
106
|
}): Promise<SearchResult[]>;
|
|
97
107
|
/** Lint quirks for low confidence, staleness, duplicates, and orphan source refs. */
|
|
98
108
|
export declare function lintQuirks(deps: QuirkStoreDeps): Promise<string[]>;
|
|
@@ -540,6 +540,11 @@ export async function recallQuirks(deps, query, options) {
|
|
|
540
540
|
if (!options.tags.some((t) => chunkTags.includes(t)))
|
|
541
541
|
return false;
|
|
542
542
|
}
|
|
543
|
+
if (options?.excludeTags?.length) {
|
|
544
|
+
const chunkTags = r.chunk.metadata.tags ?? [];
|
|
545
|
+
if (options.excludeTags.some((t) => chunkTags.includes(t)))
|
|
546
|
+
return false;
|
|
547
|
+
}
|
|
543
548
|
return true;
|
|
544
549
|
});
|
|
545
550
|
// Rank by semantic similarity (confidence-weighted) and report that same
|
|
@@ -549,7 +554,10 @@ export async function recallQuirks(deps, query, options) {
|
|
|
549
554
|
const conf = r.chunk.metadata.confidence ?? 1;
|
|
550
555
|
const similarity = r.explanation?.scoreBreakdown?.rawVectorScore ?? 0;
|
|
551
556
|
r.score = similarity * Math.min(1, Math.max(0.01, conf));
|
|
552
|
-
|
|
557
|
+
// `explain` keeps the breakdown so a caller can read the RAW similarity
|
|
558
|
+
// (unaffected by the confidence discount applied to `score` above).
|
|
559
|
+
if (!options?.explain)
|
|
560
|
+
delete r.explanation;
|
|
553
561
|
}
|
|
554
562
|
filtered.sort((a, b) => b.score - a.score);
|
|
555
563
|
return filtered.slice(0, topK);
|