opencode-rag-plugin 2.2.2 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +2 -0
- package/dist/api.js +2 -0
- package/dist/chunker/factory.js +48 -18
- package/dist/chunker/image.js +6 -0
- package/dist/cli/commands/clear.js +21 -1
- package/dist/cli/commands/init-helpers.d.ts +4 -1
- package/dist/cli/commands/init-helpers.js +55 -20
- package/dist/cli/commands/init.js +13 -1
- package/dist/cli/commands/query.js +6 -0
- package/dist/cli/commands/setup.js +43 -12
- package/dist/cli/commands/status.js +4 -0
- package/dist/cli/commands/update.d.ts +3 -2
- package/dist/cli/commands/update.js +7 -3
- package/dist/core/bootstrap.d.ts +3 -1
- package/dist/core/bootstrap.js +5 -0
- package/dist/core/config.d.ts +74 -1
- package/dist/core/config.js +63 -1
- package/dist/core/interfaces.d.ts +34 -0
- package/dist/core/package-manager.d.ts +69 -0
- package/dist/core/package-manager.js +250 -0
- package/dist/core/setup-runtime.d.ts +7 -4
- package/dist/core/setup-runtime.js +33 -31
- package/dist/core/version-check.d.ts +19 -11
- package/dist/core/version-check.js +30 -11
- package/dist/describer/describer.js +9 -1
- package/dist/embedder/factory.d.ts +5 -1
- package/dist/embedder/factory.js +76 -6
- package/dist/embedder/health.d.ts +1 -1
- package/dist/embedder/health.js +43 -0
- package/dist/eval/rerank-eval.d.ts +45 -0
- package/dist/eval/rerank-eval.js +581 -0
- package/dist/indexer/pipeline.d.ts +17 -0
- package/dist/indexer/pipeline.js +123 -3
- package/dist/indexer/stats.d.ts +7 -0
- package/dist/mcp/handlers.js +3 -0
- package/dist/opencode/create-read-tool.js +5 -2
- package/dist/plugin.js +67 -6
- package/dist/quirks/quirk-store.d.ts +1 -0
- package/dist/quirks/quirk-store.js +1 -1
- package/dist/quirks/tool-error-recall.d.ts +112 -0
- package/dist/quirks/tool-error-recall.js +144 -0
- package/dist/reranker/base.d.ts +47 -0
- package/dist/reranker/base.js +73 -0
- package/dist/reranker/factory.d.ts +16 -0
- package/dist/reranker/factory.js +47 -0
- package/dist/reranker/llama-server.d.ts +33 -0
- package/dist/reranker/llama-server.js +101 -0
- package/dist/retriever/keyword-index.d.ts +1 -0
- package/dist/retriever/keyword-index.js +1 -1
- package/dist/retriever/retriever.d.ts +24 -1
- package/dist/retriever/retriever.js +69 -5
- package/dist/types/opencode-plugin.d.ts +2 -2
- package/dist/v2/v2-adapter.d.ts +6 -1
- package/dist/v2/v2-adapter.js +52 -8
- package/dist/vectorstore/lancedb.d.ts +1 -1
- package/dist/vectorstore/lancedb.js +43 -7
- package/dist/web/api.js +5 -0
- package/package.json +1 -1
package/dist/indexer/pipeline.js
CHANGED
|
@@ -10,7 +10,7 @@ import { DescriptionCache } from "../core/desc-cache.js";
|
|
|
10
10
|
import { embedBatch, probeEmbeddingDimension } from "../embedder/factory.js";
|
|
11
11
|
import { createVectorStore } from "../vectorstore/factory.js";
|
|
12
12
|
import { swapStoreDirectories } from "../vectorstore/lancedb.js";
|
|
13
|
-
import { reconcileQuirks } from "../quirks/quirk-store.js";
|
|
13
|
+
import { reconcileQuirks, QUIRK_FILE_PREFIX } from "../quirks/quirk-store.js";
|
|
14
14
|
import { createIndexStats } from "./stats.js";
|
|
15
15
|
import { prepareFile, buildTextsToEmbed } from "./worker.js";
|
|
16
16
|
import { tagChunksRole } from "../core/chunk-role.js";
|
|
@@ -103,6 +103,30 @@ export async function runIndexPass(options) {
|
|
|
103
103
|
catch { }
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
|
+
/**
|
|
107
|
+
* Warning text for description settings that leave chain-of-thought enabled,
|
|
108
|
+
* or `null` when thinking is safely disabled.
|
|
109
|
+
*
|
|
110
|
+
* Ollama requests default to thinking off. OpenAI-compatible providers inherit
|
|
111
|
+
* whatever the served model does, and reasoning models (Qwen3.x etc.) then
|
|
112
|
+
* spend thousands of extra tokens per chunk, making indexing ~50x slower.
|
|
113
|
+
* `think: false` forwards `chat_template_kwargs.enable_thinking=false` for
|
|
114
|
+
* those providers (see LlmDescriptionProvider).
|
|
115
|
+
*
|
|
116
|
+
* @param description - Description config slice (`think`, `provider`).
|
|
117
|
+
* @returns Human-readable warning, or `null` when no warning is needed.
|
|
118
|
+
*/
|
|
119
|
+
export function getDescriptionThinkingWarning(description) {
|
|
120
|
+
if (!description)
|
|
121
|
+
return null;
|
|
122
|
+
const risky = description.think === true ||
|
|
123
|
+
(description.think === undefined && description.provider !== "ollama");
|
|
124
|
+
if (!risky)
|
|
125
|
+
return null;
|
|
126
|
+
return (`Description thinking is enabled (description.think=${description.think === true ? "true" : "unset"}, ` +
|
|
127
|
+
`provider "${description.provider}") - reasoning models can spend thousands of tokens per chunk ` +
|
|
128
|
+
"and make indexing ~50x slower. Set description.think=false to disable chain-of-thought.");
|
|
129
|
+
}
|
|
106
130
|
async function runIndexPassInner(options, logger) {
|
|
107
131
|
const loadResult = await loadManifest(options.storePath);
|
|
108
132
|
const manifest = loadResult.manifest;
|
|
@@ -113,6 +137,13 @@ async function runIndexPassInner(options, logger) {
|
|
|
113
137
|
manifestStatus = "missing";
|
|
114
138
|
logger.debug("Force mode: ignoring manifest");
|
|
115
139
|
}
|
|
140
|
+
// Advisory: chain-of-thought generation can make description latency ~50x
|
|
141
|
+
// higher on reasoning models - warn once per run when it is not disabled.
|
|
142
|
+
if (options.descriptionProvider) {
|
|
143
|
+
const warning = getDescriptionThinkingWarning(options.config.description);
|
|
144
|
+
if (warning)
|
|
145
|
+
logger.warn(warning);
|
|
146
|
+
}
|
|
116
147
|
// Compute a hash of the current description config — used to skip re-description
|
|
117
148
|
// when neither the file content nor the description config has changed.
|
|
118
149
|
const descHash = options.descriptionProvider
|
|
@@ -452,6 +483,12 @@ async function runIndexPassInner(options, logger) {
|
|
|
452
483
|
options.logger?.warn?.(`Failed to save manifest: ${err.message}`);
|
|
453
484
|
}));
|
|
454
485
|
}
|
|
486
|
+
/**
|
|
487
|
+
* Files that received rows during THIS pass. The end-of-pass orphan sweep
|
|
488
|
+
* leaves them alone on purpose: their rows are the current, honest state of
|
|
489
|
+
* a file that is still waiting for a complete embedding run.
|
|
490
|
+
*/
|
|
491
|
+
const writtenThisPass = new Set();
|
|
455
492
|
/**
|
|
456
493
|
* Store one window's prepared files into the vector store and update the
|
|
457
494
|
* manifest. Runs as a standalone async step so the caller can overlap it
|
|
@@ -469,6 +506,10 @@ async function runIndexPassInner(options, logger) {
|
|
|
469
506
|
const storePhaseStart = Date.now();
|
|
470
507
|
logger.info(`Store phase: storing ${filesToStore} file(s) into vector database...`);
|
|
471
508
|
let storedFiles = 0;
|
|
509
|
+
// Files whose chunks ALL failed to embed while no manifest entry exists.
|
|
510
|
+
// Rows an earlier pass wrote for them would otherwise never be removed:
|
|
511
|
+
// dedup only fires for files that insert new rows.
|
|
512
|
+
const staleUncommittedPaths = new Set();
|
|
472
513
|
// Compute per-file store payloads first (pure computation, no I/O).
|
|
473
514
|
const storePayloads = [];
|
|
474
515
|
for (const prep of prepared) {
|
|
@@ -486,8 +527,13 @@ async function runIndexPassInner(options, logger) {
|
|
|
486
527
|
if (!allEmbedded && (prep.chunks?.length ?? 0) > 0) {
|
|
487
528
|
options.logger?.warn?.(` ${prep.fileLabel}: ${validChunks.length}/${prep.chunks?.length} chunks embedded — marking for retry on next pass`);
|
|
488
529
|
}
|
|
489
|
-
if (validChunks.length > 0)
|
|
530
|
+
if (validChunks.length > 0) {
|
|
490
531
|
tempStoreWroteChunks = true;
|
|
532
|
+
writtenThisPass.add(prep.normalizedPath);
|
|
533
|
+
}
|
|
534
|
+
else if ((prep.chunks?.length ?? 0) > 0 && !manifest.files[prep.normalizedPath]) {
|
|
535
|
+
staleUncommittedPaths.add(prep.normalizedPath);
|
|
536
|
+
}
|
|
491
537
|
storePayloads.push({ prep, validChunks, allEmbedded });
|
|
492
538
|
}
|
|
493
539
|
// One bulk write per window. Dedup (per-modified-file deletes) is skipped
|
|
@@ -511,6 +557,16 @@ async function runIndexPassInner(options, logger) {
|
|
|
511
557
|
}
|
|
512
558
|
}
|
|
513
559
|
}
|
|
560
|
+
// Drop rows an EARLIER pass left behind for a file that produced no
|
|
561
|
+
// vectors at all here and has no manifest entry protecting them. Skipped
|
|
562
|
+
// whenever dedup is skipped too (empty/temp store cannot hold stale rows).
|
|
563
|
+
if (!skipDedup && staleUncommittedPaths.size > 0) {
|
|
564
|
+
for (const stalePath of staleUncommittedPaths) {
|
|
565
|
+
await effectiveStore.deleteByFilePath(stalePath);
|
|
566
|
+
options.keywordIndex?.removeByFilePath(stalePath);
|
|
567
|
+
}
|
|
568
|
+
logger.info(`Removed stored chunks of ${staleUncommittedPaths.size} file(s) that produced no vectors (stale partial writes)`);
|
|
569
|
+
}
|
|
514
570
|
// Manifest updates + progress reporting (per file, in-memory only).
|
|
515
571
|
const storeResults = [];
|
|
516
572
|
for (let fi = 0; fi < prepared.length; fi++) {
|
|
@@ -957,7 +1013,49 @@ async function runIndexPassInner(options, logger) {
|
|
|
957
1013
|
entry.size = size;
|
|
958
1014
|
}
|
|
959
1015
|
}
|
|
960
|
-
|
|
1016
|
+
// ── Orphan sweep: rows no manifest entry accounts for ───────────────────
|
|
1017
|
+
// A file whose chunks only partially embedded gets NO manifest entry, so
|
|
1018
|
+
// every later pass treats it as new again — but the rows an earlier pass
|
|
1019
|
+
// already wrote kept inflating `store.count()` forever, which is what made
|
|
1020
|
+
// "chunks stored" drift away from the manifest. Quirk chunks are owned by
|
|
1021
|
+
// quirks.jsonl (reconcileQuirks manages their lifecycle) and files written in
|
|
1022
|
+
// this pass hold the current state of a still-unfinished file — both skipped.
|
|
1023
|
+
const passTouchedIndex = stats.newFiles + stats.modifiedFiles + stats.deletedFiles + stats.removedFiles > 0 ||
|
|
1024
|
+
stats.embeddingFailures > 0;
|
|
1025
|
+
if (!aborted() && !tempStorePath && passTouchedIndex) {
|
|
1026
|
+
try {
|
|
1027
|
+
// Case-insensitive backstop: a path whose only difference is casing must
|
|
1028
|
+
// never be mistaken for an orphan (that would delete live data).
|
|
1029
|
+
const manifestKeysLower = new Set(Object.keys(manifest.files).map((k) => k.toLowerCase()));
|
|
1030
|
+
const orphans = [];
|
|
1031
|
+
for (const stored of await options.store.listFiles()) {
|
|
1032
|
+
const key = normalizeFilePath(stored.filePath);
|
|
1033
|
+
if (manifest.files[key])
|
|
1034
|
+
continue;
|
|
1035
|
+
if (manifestKeysLower.has(key.toLowerCase()))
|
|
1036
|
+
continue;
|
|
1037
|
+
if (writtenThisPass.has(key))
|
|
1038
|
+
continue;
|
|
1039
|
+
if (path.basename(key).startsWith(QUIRK_FILE_PREFIX))
|
|
1040
|
+
continue;
|
|
1041
|
+
orphans.push({ filePath: stored.filePath, chunkCount: stored.chunkCount });
|
|
1042
|
+
}
|
|
1043
|
+
if (orphans.length > 0) {
|
|
1044
|
+
let removedRows = 0;
|
|
1045
|
+
for (const orphan of orphans) {
|
|
1046
|
+
await options.store.deleteByFilePath(orphan.filePath);
|
|
1047
|
+
options.keywordIndex?.removeByFilePath(orphan.filePath);
|
|
1048
|
+
removedRows += orphan.chunkCount;
|
|
1049
|
+
}
|
|
1050
|
+
logger.info(`Orphan cleanup: removed ${removedRows} chunk(s) from ${orphans.length} file(s) without a manifest entry`);
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
catch (err) {
|
|
1054
|
+
// Best-effort: a failed sweep must never fail the pass.
|
|
1055
|
+
logger.warn(`Orphan cleanup failed: ${err.message}`);
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
logger.info(`Index pass complete: ${stats.totalChunks} chunks written this pass (${finalResults.length} files processed)`);
|
|
961
1059
|
// Update timestamps; advance lastGitCommit ONLY on a complete pass
|
|
962
1060
|
manifest.lastIndexedAt = Date.now();
|
|
963
1061
|
if (!aborted()) {
|
|
@@ -1151,6 +1249,7 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
|
|
|
1151
1249
|
rebuildRequired: storeCount > 0,
|
|
1152
1250
|
storeChunkCount: storeCount,
|
|
1153
1251
|
manifestExpectedChunks: 0,
|
|
1252
|
+
orphanChunks: 0,
|
|
1154
1253
|
};
|
|
1155
1254
|
}
|
|
1156
1255
|
let upToDateFiles = 0;
|
|
@@ -1183,6 +1282,26 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
|
|
|
1183
1282
|
else {
|
|
1184
1283
|
manifestExpectedChunks = Object.values(manifest.files).reduce((sum, entry) => sum + entry.chunkCount, 0);
|
|
1185
1284
|
}
|
|
1285
|
+
// Rows without a manifest entry (partial-embedding leftovers). Quirk chunks
|
|
1286
|
+
// live in quirks.jsonl and are deliberately not counted as orphans. A store
|
|
1287
|
+
// that cannot enumerate files simply reports 0.
|
|
1288
|
+
let orphanChunks = 0;
|
|
1289
|
+
try {
|
|
1290
|
+
const manifestKeysLower = new Set(Object.keys(manifest.files).map((k) => k.toLowerCase()));
|
|
1291
|
+
for (const file of await store.listFiles()) {
|
|
1292
|
+
const key = normalizeFilePath(file.filePath);
|
|
1293
|
+
if (manifest.files[key])
|
|
1294
|
+
continue;
|
|
1295
|
+
if (manifestKeysLower.has(key.toLowerCase()))
|
|
1296
|
+
continue;
|
|
1297
|
+
if (path.basename(key).startsWith(QUIRK_FILE_PREFIX))
|
|
1298
|
+
continue;
|
|
1299
|
+
orphanChunks += file.chunkCount;
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
catch {
|
|
1303
|
+
orphanChunks = 0;
|
|
1304
|
+
}
|
|
1186
1305
|
return {
|
|
1187
1306
|
manifestStatus: loadResult.status,
|
|
1188
1307
|
manifestEntries: Object.keys(manifest.files).length,
|
|
@@ -1192,6 +1311,7 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
|
|
|
1192
1311
|
rebuildRequired: false,
|
|
1193
1312
|
storeChunkCount: storeCount,
|
|
1194
1313
|
manifestExpectedChunks,
|
|
1314
|
+
orphanChunks,
|
|
1195
1315
|
};
|
|
1196
1316
|
}
|
|
1197
1317
|
//# sourceMappingURL=pipeline.js.map
|
package/dist/indexer/stats.d.ts
CHANGED
|
@@ -63,6 +63,13 @@ export interface IndexStatusSummary {
|
|
|
63
63
|
storeChunkCount: number;
|
|
64
64
|
/** Number of chunks the manifest expects. */
|
|
65
65
|
manifestExpectedChunks: number;
|
|
66
|
+
/**
|
|
67
|
+
* Stored chunks that NO manifest entry accounts for (excluding quirk chunks,
|
|
68
|
+
* which quirks.jsonl owns). They come from files whose chunks only partially
|
|
69
|
+
* embedded — such a file gets no manifest entry, so its rows are not
|
|
70
|
+
* reachable through any accounting. The next full index pass sweeps them.
|
|
71
|
+
*/
|
|
72
|
+
orphanChunks: number;
|
|
66
73
|
}
|
|
67
74
|
/**
|
|
68
75
|
* Create a zeroed-out {@link IndexRunStats} instance initialised with the
|
package/dist/mcp/handlers.js
CHANGED
|
@@ -5,6 +5,7 @@ import { CODE_SEARCH_FILTER } from "../core/interfaces.js";
|
|
|
5
5
|
import { normalizeFileExtensions } from "../core/filters.js";
|
|
6
6
|
import { SUPPORTED_IMAGE_EXTENSIONS } from "../chunker/image.js";
|
|
7
7
|
import { retrieve } from "../retriever/retriever.js";
|
|
8
|
+
import { getRerankerFor } from "../reranker/factory.js";
|
|
8
9
|
import { optimizeContext, DEFAULT_CONTEXT_OPTIMIZATION } from "../retriever/context-optimizer.js";
|
|
9
10
|
import { extractSkeleton, getExtension } from "../chunker/skeleton.js";
|
|
10
11
|
import { readFileSync } from "node:fs";
|
|
@@ -91,6 +92,8 @@ export async function handleSearchSemantic(params, embedder, store, cfg, keyword
|
|
|
91
92
|
keywordWeight: cfg.retrieval.hybridSearch?.keywordWeight,
|
|
92
93
|
hybridEnabled: cfg.retrieval.hybridSearch?.enabled,
|
|
93
94
|
queryPrefix: cfg.embedding.queryPrefix,
|
|
95
|
+
reranker: getRerankerFor(cfg.reranking) ?? undefined,
|
|
96
|
+
reranking: cfg.reranking,
|
|
94
97
|
filter,
|
|
95
98
|
};
|
|
96
99
|
const rawResults = await retrieveFn_(query, embedder, store, retrieveOpts);
|
|
@@ -5,6 +5,7 @@ import fs from "node:fs/promises";
|
|
|
5
5
|
import { tool } from "@opencode-ai/plugin/tool";
|
|
6
6
|
import { CODE_SEARCH_FILTER } from "../core/interfaces.js";
|
|
7
7
|
import { retrieve } from "../retriever/retriever.js";
|
|
8
|
+
import { getRerankerFor } from "../reranker/factory.js";
|
|
8
9
|
import { normalizeReadArgs, resolveWorkspacePath } from "./tool-args.js";
|
|
9
10
|
import { buildReadQuery } from "./read-query.js";
|
|
10
11
|
import { formatHybridReadOutput } from "./read-format.js";
|
|
@@ -23,6 +24,8 @@ export function createRagReadTool(options) {
|
|
|
23
24
|
// Over-fetch by 4x so post-filtering (line range, minScore) still yields enough results
|
|
24
25
|
const retrievalTopK = maxContextChunks * 4;
|
|
25
26
|
const readRelatedFilesMax = openCodeCfg.readRelatedFilesMax ?? 5;
|
|
27
|
+
const rerankProvider = getRerankerFor(config.reranking) ?? undefined;
|
|
28
|
+
const rerankOptions = config.reranking;
|
|
26
29
|
return tool({
|
|
27
30
|
description: "Read file contents from the workspace. Returns full file contents with " +
|
|
28
31
|
"relevant RAG context appended when available. " +
|
|
@@ -68,7 +71,7 @@ export function createRagReadTool(options) {
|
|
|
68
71
|
}
|
|
69
72
|
else {
|
|
70
73
|
const retrievalQuery = buildSessionQuery(messageText, resolvedPath, normalized);
|
|
71
|
-
rawResults = await retrieve(retrievalQuery, embedder, store, { topK: retrievalTopK, keywordIndex, hybridEnabled: config.retrieval.hybridSearch?.enabled, queryPrefix: config.embedding.queryPrefix, filter: CODE_SEARCH_FILTER });
|
|
74
|
+
rawResults = await retrieve(retrievalQuery, embedder, store, { topK: retrievalTopK, keywordIndex, hybridEnabled: config.retrieval.hybridSearch?.enabled, queryPrefix: config.embedding.queryPrefix, reranker: rerankProvider, reranking: rerankOptions, filter: CODE_SEARCH_FILTER });
|
|
72
75
|
const maxSize = options.maxSessionCacheSize ?? 50;
|
|
73
76
|
if (!sessionRetrievalCache.has(sessionID) && sessionRetrievalCache.size >= maxSize) {
|
|
74
77
|
const oldest = sessionRetrievalCache.keys().next().value;
|
|
@@ -85,7 +88,7 @@ export function createRagReadTool(options) {
|
|
|
85
88
|
startLine: normalized.startLine,
|
|
86
89
|
endLine: normalized.endLine,
|
|
87
90
|
});
|
|
88
|
-
rawResults = await retrieve(retrievalQuery, embedder, store, { topK: retrievalTopK, keywordIndex, hybridEnabled: config.retrieval.hybridSearch?.enabled, queryPrefix: config.embedding.queryPrefix, filter: CODE_SEARCH_FILTER });
|
|
91
|
+
rawResults = await retrieve(retrievalQuery, embedder, store, { topK: retrievalTopK, keywordIndex, hybridEnabled: config.retrieval.hybridSearch?.enabled, queryPrefix: config.embedding.queryPrefix, reranker: rerankProvider, reranking: rerankOptions, filter: CODE_SEARCH_FILTER });
|
|
89
92
|
}
|
|
90
93
|
// Collect related files from raw results (before filtering)
|
|
91
94
|
relatedFiles = collectRelatedFiles(rawResults, resolvedPath, readRelatedFilesMax);
|
package/dist/plugin.js
CHANGED
|
@@ -12,6 +12,7 @@ import { readStoreDimension } from "./vectorstore/lancedb.js";
|
|
|
12
12
|
import { createDescriptionProvider } from "./describer/factory.js";
|
|
13
13
|
import { createVectorStore } from "./vectorstore/factory.js";
|
|
14
14
|
import { retrieve } from "./retriever/retriever.js";
|
|
15
|
+
import { getRerankerFor } from "./reranker/factory.js";
|
|
15
16
|
import { optimizeContext, DEFAULT_CONTEXT_OPTIMIZATION } from "./retriever/context-optimizer.js";
|
|
16
17
|
import { loadChunkersFromConfig } from "./chunker/loader.js";
|
|
17
18
|
import { appendDebugLog } from "./core/fileLogger.js";
|
|
@@ -30,6 +31,7 @@ import { loadAutoUpdateState, saveAutoUpdateState, shouldAttemptInstall } from "
|
|
|
30
31
|
import { destroyAllPooledConnections } from "./embedder/http.js";
|
|
31
32
|
import { listQuirks, lintQuirks, recallQuirks, sharedWords, reconcileQuirks } from "./quirks/quirk-store.js";
|
|
32
33
|
import { autoCaptureQuirks } from "./quirks/auto-capture.js";
|
|
34
|
+
import { createToolErrorInjectionState, selectToolErrorQuirks, formatToolErrorQuirkBlock, toToolErrorCaptureEntry, } from "./quirks/tool-error-recall.js";
|
|
33
35
|
import { buildSystemGuidanceLines } from "./opencode/system-guidance.js";
|
|
34
36
|
import { existsSync, readFileSync, readdirSync, unlinkSync } from "node:fs";
|
|
35
37
|
import path from "node:path";
|
|
@@ -37,6 +39,8 @@ import path from "node:path";
|
|
|
37
39
|
const MAX_SESSION_MAP_SIZE = 50;
|
|
38
40
|
/** Maximum tool results kept per session (for quirk extraction). */
|
|
39
41
|
const MAX_SESSION_TOOL_RESULTS = 20;
|
|
42
|
+
/** Pending tool-error quirk blocks older than this (ms) are dropped instead of injected. */
|
|
43
|
+
const TOOL_ERROR_PENDING_TTL_MS = 120_000;
|
|
40
44
|
/** Cache of loaded RAG configurations keyed by workspace directory. */
|
|
41
45
|
const configCache = new Map();
|
|
42
46
|
/** Active background indexer instances keyed by workspace directory. */
|
|
@@ -300,23 +304,23 @@ function buildRetrievalQuery(hints) {
|
|
|
300
304
|
* Perform a retrieval query against the vector store with the given parameters.
|
|
301
305
|
* Returns an empty array for blank queries.
|
|
302
306
|
*/
|
|
303
|
-
async function retrieveContext(query, embedder, store, topK, retrieveFn = retrieve, minScore = 0, keywordIndex, keywordWeight, queryPrefix, explain = false, hybridEnabled, filter) {
|
|
307
|
+
async function retrieveContext(query, embedder, store, topK, retrieveFn = retrieve, minScore = 0, keywordIndex, keywordWeight, queryPrefix, explain = false, hybridEnabled, filter, reranker, reranking) {
|
|
304
308
|
if (query.trim().length === 0)
|
|
305
309
|
return [];
|
|
306
|
-
return retrieveFn(query, embedder, store, { topK, minScore, keywordIndex, keywordWeight, hybridEnabled, queryPrefix, explain, filter });
|
|
310
|
+
return retrieveFn(query, embedder, store, { topK, minScore, keywordIndex, keywordWeight, hybridEnabled, queryPrefix, explain, filter, reranker, reranking });
|
|
307
311
|
}
|
|
308
312
|
/**
|
|
309
313
|
* Load results from one or two queries (primary + optional extra), optimize
|
|
310
314
|
* them via context optimization (adjacent merge, similarity dedup, file cap),
|
|
311
315
|
* sort by descending score, and limit to maxContextChunks from config.
|
|
312
316
|
*/
|
|
313
|
-
async function loadRetrievedResults(query, embedder, store, cfg, retrieveFn = retrieve, topK = cfg.retrieval.topK, extraQuery, keywordIndex, queryPrefix, explain = false, filter) {
|
|
317
|
+
async function loadRetrievedResults(query, embedder, store, cfg, retrieveFn = retrieve, topK = cfg.retrieval.topK, extraQuery, keywordIndex, queryPrefix, explain = false, filter, reranker, reranking) {
|
|
314
318
|
const minScore = cfg.retrieval.minScore;
|
|
315
319
|
const kw = cfg.retrieval.hybridSearch?.keywordWeight;
|
|
316
320
|
const hybridEnabled = cfg.retrieval.hybridSearch?.enabled;
|
|
317
|
-
const primaryResults = await retrieveContext(query, embedder, store, topK, retrieveFn, minScore, keywordIndex, kw, queryPrefix, explain, hybridEnabled, filter);
|
|
321
|
+
const primaryResults = await retrieveContext(query, embedder, store, topK, retrieveFn, minScore, keywordIndex, kw, queryPrefix, explain, hybridEnabled, filter, reranker, reranking);
|
|
318
322
|
const extraResults = extraQuery
|
|
319
|
-
? await retrieveContext(extraQuery, embedder, store, topK, retrieveFn, minScore, keywordIndex, kw, queryPrefix, explain, hybridEnabled, filter)
|
|
323
|
+
? await retrieveContext(extraQuery, embedder, store, topK, retrieveFn, minScore, keywordIndex, kw, queryPrefix, explain, hybridEnabled, filter, reranker, reranking)
|
|
320
324
|
: [];
|
|
321
325
|
const optCfg = cfg.retrieval.contextOptimization ?? DEFAULT_CONTEXT_OPTIMIZATION;
|
|
322
326
|
return optimizeContext([...primaryResults, ...extraResults], { topK, config: optCfg })
|
|
@@ -454,6 +458,10 @@ export function createRagHooks(options) {
|
|
|
454
458
|
const sessionTranscript = new Map();
|
|
455
459
|
// Track quirk IDs already injected per session to avoid duplicate injection
|
|
456
460
|
const sessionInjectedQuirks = new Map();
|
|
461
|
+
// Tool-error quirk path: per-session injection state (cooldown/budget) and the
|
|
462
|
+
// block pending for the next model request.
|
|
463
|
+
const sessionToolErrorState = new Map();
|
|
464
|
+
const sessionPendingToolErrorQuirk = new Map();
|
|
457
465
|
// Evaluation session logger — captures OpenCode events for analysis
|
|
458
466
|
const sessionLogger = createSessionLogger(options.storePath);
|
|
459
467
|
appendDebugLog(options.logFilePath, {
|
|
@@ -505,7 +513,7 @@ export function createRagHooks(options) {
|
|
|
505
513
|
const explain = args.explain ?? false;
|
|
506
514
|
const fileExtensions = normalizeFileExtensions(args.fileExtensions);
|
|
507
515
|
const filter = fileExtensions.length > 0 ? { fileExtensions } : undefined;
|
|
508
|
-
const results = await loadRetrievedResults(query, embedder, store, effectiveCfg, dependencies.retrieve, topK, undefined, keywordIndex, effectiveCfg.embedding.queryPrefix, explain, filter);
|
|
516
|
+
const results = await loadRetrievedResults(query, embedder, store, effectiveCfg, dependencies.retrieve, topK, undefined, keywordIndex, effectiveCfg.embedding.queryPrefix, explain, filter, getRerankerFor(effectiveCfg.reranking) ?? undefined, effectiveCfg.reranking);
|
|
509
517
|
if (results.length === 0) {
|
|
510
518
|
appendVerboseLog(options.logFilePath, CONTEXT_TOOL_NAME, "retrieval completed with no matching chunks", {
|
|
511
519
|
query,
|
|
@@ -776,6 +784,40 @@ export function createRagHooks(options) {
|
|
|
776
784
|
try {
|
|
777
785
|
const results = sessionToolResults.get(input.sessionID) ?? [];
|
|
778
786
|
results.push({ tool: input.tool, output: output.output ?? "" });
|
|
787
|
+
// Failed tool calls: feed the auto-capture pipeline (learning loop) and
|
|
788
|
+
// recall matching quirks for injection into the next model request.
|
|
789
|
+
// `output.error` is set by the V2 adapter's error branch; the status
|
|
790
|
+
// check keeps this working if a runtime reports errors via input only.
|
|
791
|
+
const errorText = output.error ?? (input.status === "error" ? output.output : undefined);
|
|
792
|
+
if (errorText && errorText.length > 0) {
|
|
793
|
+
const info = { tool: input.tool, error: errorText, sessionID: input.sessionID };
|
|
794
|
+
results.push(toToolErrorCaptureEntry(info));
|
|
795
|
+
const memCfg = getEffectiveCfg().memory;
|
|
796
|
+
if (memCfg?.enabled && memCfg.injectOnToolError) {
|
|
797
|
+
const state = sessionToolErrorState.get(input.sessionID) ?? createToolErrorInjectionState({
|
|
798
|
+
cooldownMs: memCfg.toolErrorCooldownMs ?? 60_000,
|
|
799
|
+
maxPerSession: memCfg.toolErrorMaxPerSession ?? 3,
|
|
800
|
+
});
|
|
801
|
+
boundedSet(sessionToolErrorState, input.sessionID, state, MAX_SESSION_MAP_SIZE);
|
|
802
|
+
const qDeps = { embedder, store, keywordIndex: keywordIndex, cfg: getEffectiveCfg(), storePath: options.storePath };
|
|
803
|
+
const recall = (query, recallOpts) => recallQuirks(qDeps, query, recallOpts);
|
|
804
|
+
const budgetMs = memCfg.toolErrorLatencyBudgetMs ?? 1500;
|
|
805
|
+
await Promise.race([
|
|
806
|
+
selectToolErrorQuirks(recall, info, state, {
|
|
807
|
+
minScore: memCfg.toolErrorMinScore ?? 0.7,
|
|
808
|
+
topK: memCfg.toolErrorTopK ?? 2,
|
|
809
|
+
minTokenOverlap: memCfg.toolErrorMinTokenOverlap ?? 1,
|
|
810
|
+
}).then((hits) => {
|
|
811
|
+
if (hits.length > 0) {
|
|
812
|
+
boundedSet(sessionPendingToolErrorQuirk, input.sessionID, { block: formatToolErrorQuirkBlock(info, hits), ts: Date.now() }, MAX_SESSION_MAP_SIZE);
|
|
813
|
+
}
|
|
814
|
+
}),
|
|
815
|
+
new Promise((resolve) => {
|
|
816
|
+
setTimeout(resolve, budgetMs);
|
|
817
|
+
}),
|
|
818
|
+
]).catch(() => { });
|
|
819
|
+
}
|
|
820
|
+
}
|
|
779
821
|
// Cap the per-session list — long sessions accumulate full tool
|
|
780
822
|
// outputs; only the recent entries matter for quirk extraction.
|
|
781
823
|
const capped = results.length > MAX_SESSION_TOOL_RESULTS ? results.slice(-MAX_SESSION_TOOL_RESULTS) : results;
|
|
@@ -873,6 +915,23 @@ export function createRagHooks(options) {
|
|
|
873
915
|
}
|
|
874
916
|
}
|
|
875
917
|
}
|
|
918
|
+
// Tool-error quirks: flush the pending block captured in tool.execute.after.
|
|
919
|
+
// Freshness-bounded — a stale error must not leak into a later turn.
|
|
920
|
+
try {
|
|
921
|
+
const sessionId = _input?.sessionID;
|
|
922
|
+
if (sessionId) {
|
|
923
|
+
const pending = sessionPendingToolErrorQuirk.get(sessionId);
|
|
924
|
+
if (pending) {
|
|
925
|
+
if (Date.now() - pending.ts < TOOL_ERROR_PENDING_TTL_MS) {
|
|
926
|
+
output.system.unshift(pending.block);
|
|
927
|
+
}
|
|
928
|
+
sessionPendingToolErrorQuirk.delete(sessionId);
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
catch {
|
|
933
|
+
// Non-critical — must never throw
|
|
934
|
+
}
|
|
876
935
|
}
|
|
877
936
|
catch {
|
|
878
937
|
// Non-critical — must never throw
|
|
@@ -1225,6 +1284,8 @@ export function createRagHooks(options) {
|
|
|
1225
1284
|
keywordIndex,
|
|
1226
1285
|
keywordWeight: hybridCfg?.keywordWeight,
|
|
1227
1286
|
queryPrefix: effectiveCfg.embedding.queryPrefix,
|
|
1287
|
+
reranker: getRerankerFor(effectiveCfg.reranking) ?? undefined,
|
|
1288
|
+
reranking: effectiveCfg.reranking,
|
|
1228
1289
|
// Never surface quirk chunks in hotkey file lists / chunk injections.
|
|
1229
1290
|
filter: CODE_SEARCH_FILTER,
|
|
1230
1291
|
});
|
|
@@ -9,6 +9,7 @@ export interface QuirkStoreDeps {
|
|
|
9
9
|
cfg: RagConfig;
|
|
10
10
|
storePath: string;
|
|
11
11
|
}
|
|
12
|
+
export declare const QUIRK_FILE_PREFIX = "quirk:";
|
|
12
13
|
/** Look up a single quirk by its ID, or `undefined` when not found. */
|
|
13
14
|
export declare function getQuirk(deps: QuirkStoreDeps, id: string): Promise<Quirk | undefined>;
|
|
14
15
|
/**
|
|
@@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { retrieve } from "../retriever/retriever.js";
|
|
5
5
|
import { isQuirkAllowed } from "./monitor.js";
|
|
6
|
-
const QUIRK_FILE_PREFIX = "quirk:";
|
|
6
|
+
export const QUIRK_FILE_PREFIX = "quirk:";
|
|
7
7
|
const QUIKK_JSONL = "quirks.jsonl";
|
|
8
8
|
function jsonlPath(storePath) {
|
|
9
9
|
return path.join(storePath, QUIKK_JSONL);
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Tool-error → quirk recall: when a tool call fails, find matching quirk
|
|
3
|
+
* memory entries to inject into the next model request, and gate the existing
|
|
4
|
+
* auto-capture pipeline so recurring errors can become quirks.
|
|
5
|
+
*
|
|
6
|
+
* Design notes:
|
|
7
|
+
* - Pure and dependency-injected: the caller passes a `recall` function (the plugin
|
|
8
|
+
* wraps `recallQuirks`), so this module has no store/embedder dependencies and is
|
|
9
|
+
* cheap to unit-test.
|
|
10
|
+
* - Conservative by default: min score 0.7, max 2 hits per error, 3 injections per
|
|
11
|
+
* session, 60s cooldown per normalized error signature, plus a lexical gate so a
|
|
12
|
+
* semantically close but lexically unrelated quirk never gets injected.
|
|
13
|
+
* - A failed recall still counts as an attempt (cooldown), so a repeatedly failing
|
|
14
|
+
* tool does not hammer the embedder every call — but it does not consume the
|
|
15
|
+
* per-session budget; only actual injections do.
|
|
16
|
+
*/
|
|
17
|
+
/** A failed tool call as observed by the plugin. */
|
|
18
|
+
export interface ToolErrorInfo {
|
|
19
|
+
/** Tool name as reported by OpenCode (e.g. "search_semantic", "write"). */
|
|
20
|
+
tool: string;
|
|
21
|
+
/** Error text (ToolStateError.error or the V2 execute.after error payload). */
|
|
22
|
+
error: string;
|
|
23
|
+
/** Owning session — callers keep one state object per session. */
|
|
24
|
+
sessionID?: string;
|
|
25
|
+
}
|
|
26
|
+
/** Minimal recall-hit shape; structurally compatible with `SearchResult`. */
|
|
27
|
+
export interface ToolErrorRecallHit {
|
|
28
|
+
chunk: {
|
|
29
|
+
id: string;
|
|
30
|
+
content: string;
|
|
31
|
+
metadata?: {
|
|
32
|
+
quirkType?: string;
|
|
33
|
+
};
|
|
34
|
+
};
|
|
35
|
+
score: number;
|
|
36
|
+
}
|
|
37
|
+
/** Recall function the caller provides (usually a `recallQuirks` wrapper). */
|
|
38
|
+
export type ToolErrorRecallFn = (query: string, options: {
|
|
39
|
+
topK: number;
|
|
40
|
+
minScore: number;
|
|
41
|
+
}) => Promise<readonly ToolErrorRecallHit[]>;
|
|
42
|
+
/** A quirk selected for injection after a tool error. */
|
|
43
|
+
export interface ToolErrorQuirkHit {
|
|
44
|
+
id: string;
|
|
45
|
+
content: string;
|
|
46
|
+
quirkType?: string;
|
|
47
|
+
score: number;
|
|
48
|
+
}
|
|
49
|
+
/** Tunables for the tool-error quirk path. */
|
|
50
|
+
export interface ToolErrorRecallOptions {
|
|
51
|
+
/** Minimum recall score for a quirk to be injected. Default 0.7. */
|
|
52
|
+
minScore?: number;
|
|
53
|
+
/** Maximum quirks injected per error. Default 2. */
|
|
54
|
+
topK?: number;
|
|
55
|
+
/** Minimum shared meaningful words (>=3 chars) between quirk and error. 0 disables. Default 1. */
|
|
56
|
+
minTokenOverlap?: number;
|
|
57
|
+
}
|
|
58
|
+
/** Tunables for the per-session injection state. */
|
|
59
|
+
export interface ToolErrorInjectionOptions {
|
|
60
|
+
/** Cooldown per normalized error signature in ms. Default 60_000. */
|
|
61
|
+
cooldownMs?: number;
|
|
62
|
+
/** Maximum injections (errors that produced at least one quirk) per session. Default 3. */
|
|
63
|
+
maxPerSession?: number;
|
|
64
|
+
}
|
|
65
|
+
/** Per-session injection bookkeeping (cooldown, budget, already-injected quirk ids). */
|
|
66
|
+
export interface ToolErrorInjectionState {
|
|
67
|
+
/** True when this error may trigger a recall (budget left, signature not cooling down). */
|
|
68
|
+
shouldInject(info: ToolErrorInfo, now?: number): boolean;
|
|
69
|
+
/** Record an attempt. Always refreshes the cooldown; counts toward the budget only when quirkIds is non-empty. */
|
|
70
|
+
record(info: ToolErrorInfo, quirkIds: readonly string[], now?: number): void;
|
|
71
|
+
/** Quirk ids already injected in this session. */
|
|
72
|
+
readonly injectedQuirkIds: ReadonlySet<string>;
|
|
73
|
+
/** Number of injections that actually delivered at least one quirk. */
|
|
74
|
+
injectionCount(): number;
|
|
75
|
+
}
|
|
76
|
+
/** Defaults shared by the state factory and the selector. */
|
|
77
|
+
export declare const TOOL_ERROR_DEFAULTS: {
|
|
78
|
+
readonly minScore: 0.7;
|
|
79
|
+
readonly topK: 2;
|
|
80
|
+
readonly minTokenOverlap: 1;
|
|
81
|
+
readonly cooldownMs: 60000;
|
|
82
|
+
readonly maxPerSession: 3;
|
|
83
|
+
/** Error text is truncated before it becomes a recall query / capture payload. */
|
|
84
|
+
readonly maxErrorChars: 400;
|
|
85
|
+
/** Capture payloads are sliced to match the existing auto-capture tool-result cap. */
|
|
86
|
+
readonly maxCaptureChars: 500;
|
|
87
|
+
};
|
|
88
|
+
/** Collapse an error into a stable signature for cooldown/dedup purposes. */
|
|
89
|
+
export declare function normalizeToolErrorSignature(info: ToolErrorInfo): string;
|
|
90
|
+
/** Build the recall query for a failed tool call. */
|
|
91
|
+
export declare function buildToolErrorQuery(info: ToolErrorInfo): string;
|
|
92
|
+
/** Create the per-session injection state. */
|
|
93
|
+
export declare function createToolErrorInjectionState(options?: ToolErrorInjectionOptions): ToolErrorInjectionState;
|
|
94
|
+
/**
|
|
95
|
+
* Select quirks for a failed tool call.
|
|
96
|
+
*
|
|
97
|
+
* Returns `[]` when the state blocks the attempt (cooldown/budget) or nothing
|
|
98
|
+
* passes the score, dedup, and lexical gates. Records the attempt in the state —
|
|
99
|
+
* with cooldown always, with budget only when at least one quirk is returned.
|
|
100
|
+
*/
|
|
101
|
+
export declare function selectToolErrorQuirks(recall: ToolErrorRecallFn, info: ToolErrorInfo, state: ToolErrorInjectionState, options?: ToolErrorRecallOptions): Promise<ToolErrorQuirkHit[]>;
|
|
102
|
+
/** Format the injection block appended to the next model request. Empty when there are no hits. */
|
|
103
|
+
export declare function formatToolErrorQuirkBlock(info: ToolErrorInfo, hits: readonly ToolErrorQuirkHit[]): string;
|
|
104
|
+
/**
|
|
105
|
+
* Shape a tool error as an entry for the existing auto-capture pipeline
|
|
106
|
+
* (`CaptureExchange.toolResults`). Feed the returned entry into the session's
|
|
107
|
+
* tool-result list so recurring errors can become quirks at session end.
|
|
108
|
+
*/
|
|
109
|
+
export declare function toToolErrorCaptureEntry(info: ToolErrorInfo): {
|
|
110
|
+
tool: string;
|
|
111
|
+
output: string;
|
|
112
|
+
};
|