opencode-rag-plugin 2.3.1 → 2.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/ReadMe.md +71 -8
  2. package/dist/chunker/skeleton.js +1 -1
  3. package/dist/cli/commands/decide.d.ts +25 -0
  4. package/dist/cli/commands/decide.js +160 -0
  5. package/dist/cli/commands/eval.d.ts +23 -0
  6. package/dist/cli/commands/eval.js +331 -1
  7. package/dist/cli/commands/index-command.js +28 -10
  8. package/dist/cli/commands/index.d.ts +2 -1
  9. package/dist/cli/commands/index.js +2 -1
  10. package/dist/cli/commands/init-helpers.d.ts +16 -1
  11. package/dist/cli/commands/init-helpers.js +84 -2
  12. package/dist/cli/commands/init.js +29 -4
  13. package/dist/cli/commands/status.js +14 -8
  14. package/dist/cli/format.d.ts +50 -0
  15. package/dist/cli/format.js +122 -0
  16. package/dist/cli/index.js +4 -1
  17. package/dist/cli/types.d.ts +2 -0
  18. package/dist/core/bootstrap.js +1 -0
  19. package/dist/core/config.d.ts +47 -0
  20. package/dist/core/config.js +34 -1
  21. package/dist/core/interfaces.d.ts +69 -0
  22. package/dist/decision/factory.d.ts +17 -0
  23. package/dist/decision/factory.js +17 -0
  24. package/dist/decision/format.d.ts +14 -0
  25. package/dist/decision/format.js +53 -0
  26. package/dist/decision/systemone.d.ts +76 -0
  27. package/dist/decision/systemone.js +234 -0
  28. package/dist/decision/validate.d.ts +26 -0
  29. package/dist/decision/validate.js +74 -0
  30. package/dist/embedder/factory.d.ts +1 -1
  31. package/dist/embedder/factory.js +80 -3
  32. package/dist/eval/decide-calibration.d.ts +112 -0
  33. package/dist/eval/decide-calibration.js +185 -0
  34. package/dist/eval/golden-gate.d.ts +65 -0
  35. package/dist/eval/golden-gate.js +61 -0
  36. package/dist/eval/golden-set.d.ts +81 -0
  37. package/dist/eval/golden-set.js +155 -0
  38. package/dist/eval/rerank-eval.js +10 -125
  39. package/dist/index.d.ts +6 -2
  40. package/dist/index.js +4 -0
  41. package/dist/indexer/pipeline.d.ts +0 -10
  42. package/dist/indexer/pipeline.js +1 -0
  43. package/dist/mcp/handlers.d.ts +17 -1
  44. package/dist/mcp/handlers.js +30 -0
  45. package/dist/mcp/server.d.ts +9 -1
  46. package/dist/mcp/server.js +47 -3
  47. package/dist/opencode/system-guidance.d.ts +11 -0
  48. package/dist/opencode/system-guidance.js +53 -0
  49. package/dist/opencode/tools.d.ts +24 -1
  50. package/dist/opencode/tools.js +101 -0
  51. package/dist/plugin.js +21 -3
  52. package/dist/quirks/quirk-store.d.ts +2 -0
  53. package/dist/quirks/quirk-store.js +33 -4
  54. package/dist/tui.js +21 -2
  55. package/dist/v2/v2-adapter.d.ts +20 -0
  56. package/dist/v2/v2-adapter.js +36 -8
  57. package/dist/vectorstore/lancedb.js +54 -5
  58. package/dist/watcher.d.ts +26 -1
  59. package/dist/watcher.js +71 -9
  60. package/dist/web/ui/assets/index-CWcp0jNf.css +2 -0
  61. package/dist/web/ui/index.html +2 -2
  62. package/package.json +5 -2
  63. package/dist/web/ui/assets/index-Dp009QCQ.css +0 -1
  64. /package/dist/web/ui/assets/{index-C0QXKA-b.js → index-DbXFrn53.js} +0 -0
@@ -0,0 +1,76 @@
1
+ /**
2
+ * @fileoverview Ollama decision-model provider (tev1) using the
3
+ * `/v1/systemone` endpoint.
4
+ *
5
+ * Decision models are NOT chat models: `state` is scored against 1-64 named
6
+ * `questions` and the response carries probabilities per question. The
7
+ * endpoint is root-level (`/v1/systemone`), so the `/api` suffix used by the
8
+ * embedding/description sections is stripped before building the URL.
9
+ *
10
+ * Requires Ollama >= 0.35 and a pulled tev1 model (`ollama pull tev1` or
11
+ * `ollama pull tev1:0.8b`).
12
+ */
13
+ import type { DecisionProvider, DecisionRequest, DecisionResult } from "../core/interfaces.js";
14
+ import type { DecisionConfig } from "../core/config.js";
15
+ /** Request body size cap imposed by Ollama's decision endpoint (64 KiB). */
16
+ export declare const SYSTEMONE_MAX_REQUEST_BYTES: number;
17
+ /** Maximum number of questions per request (tev1 limit). */
18
+ export declare const SYSTEMONE_MAX_QUESTIONS = 64;
19
+ /** Minimum options/levels for `choice`/`score` questions. */
20
+ export declare const SYSTEMONE_MIN_OPTIONS = 2;
21
+ /** tev1 was trained on up to 24 options; 26 is the API maximum. */
22
+ export declare const SYSTEMONE_MAX_OPTIONS = 24;
23
+ /**
24
+ * Resolve the decision endpoint from a configured base URL.
25
+ *
26
+ * Accepts both root URLs (`http://127.0.0.1:11434`) and the `/api`-suffixed
27
+ * convention used by the embedding/description sections (`…:11434/api`) —
28
+ * `/api` is dropped because decision models are served at `/v1/systemone`.
29
+ *
30
+ * @param baseUrl - Configured Ollama base URL.
31
+ * @returns The absolute `/v1/systemone` URL.
32
+ */
33
+ export declare function resolveSystemOneUrl(baseUrl: string): string;
34
+ /**
35
+ * Decision provider for Ollama's `/v1/systemone` endpoint (tev1).
36
+ *
37
+ * Retries retryable HTTP statuses and network errors with exponential
38
+ * backoff. The first call may load the model, so configure a generous
39
+ * `timeoutMs` (default 120s) and optionally `keepAlive` to keep it resident.
40
+ */
41
+ export declare class OllamaDecisionProvider implements DecisionProvider {
42
+ readonly name = "ollama";
43
+ private readonly baseUrl;
44
+ private readonly model;
45
+ private readonly timeoutMs;
46
+ private readonly keepAlive?;
47
+ private readonly proxy?;
48
+ private readonly retryMax;
49
+ private readonly retryBaseDelayMs;
50
+ /**
51
+ * @param config - Decision-model configuration (endpoint, model, timeout, proxy, retries).
52
+ */
53
+ constructor(config: DecisionConfig);
54
+ /**
55
+ * Run a decision request. One answer is returned per question, in request
56
+ * order; answers missing from the response degrade to bare `{id, type}`.
57
+ *
58
+ * @param request - The state text and questions.
59
+ * @param abort - Optional abort signal.
60
+ * @throws If the endpoint is unreachable, rejects the request (e.g. Ollama
61
+ * < 0.35 or model not pulled), the body exceeds 64 KiB, or the response
62
+ * carries no answers.
63
+ */
64
+ decide(request: DecisionRequest, abort?: AbortSignal): Promise<DecisionResult>;
65
+ /**
66
+ * Map an HTTP failure to an actionable message.
67
+ *
68
+ * 404 is overloaded: Ollama < 0.35 returns the Go default "404 page not
69
+ * found" (route missing), while a server that has the decision route but
70
+ * not the model returns `model "X" not found, try pulling it first`. Split
71
+ * the two so the fix is obvious.
72
+ */
73
+ private describeHttpError;
74
+ /** Exponential backoff between retries. */
75
+ private sleep;
76
+ }
@@ -0,0 +1,234 @@
1
+ import { normalizeKeepAlive } from "../core/ollama.js";
2
+ import { postJson } from "../embedder/http.js";
3
+ /** HTTP status codes that are safe to retry on. */
4
+ const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
5
+ /** Request body size cap imposed by Ollama's decision endpoint (64 KiB). */
6
+ export const SYSTEMONE_MAX_REQUEST_BYTES = 64 * 1024;
7
+ /** Maximum number of questions per request (tev1 limit). */
8
+ export const SYSTEMONE_MAX_QUESTIONS = 64;
9
+ /** Minimum options/levels for `choice`/`score` questions. */
10
+ export const SYSTEMONE_MIN_OPTIONS = 2;
11
+ /** tev1 was trained on up to 24 options; 26 is the API maximum. */
12
+ export const SYSTEMONE_MAX_OPTIONS = 24;
13
+ /**
14
+ * Resolve the decision endpoint from a configured base URL.
15
+ *
16
+ * Accepts both root URLs (`http://127.0.0.1:11434`) and the `/api`-suffixed
17
+ * convention used by the embedding/description sections (`…:11434/api`) —
18
+ * `/api` is dropped because decision models are served at `/v1/systemone`.
19
+ *
20
+ * @param baseUrl - Configured Ollama base URL.
21
+ * @returns The absolute `/v1/systemone` URL.
22
+ */
23
+ export function resolveSystemOneUrl(baseUrl) {
24
+ const trimmed = baseUrl.replace(/\/+$/, "");
25
+ const root = trimmed.endsWith("/api") ? trimmed.slice(0, -"/api".length) : trimmed;
26
+ return `${root}/v1/systemone`;
27
+ }
28
+ /** Coerce a number-or-numeric-string to a finite number, else undefined. */
29
+ function toNumber(value) {
30
+ if (typeof value === "number" && Number.isFinite(value))
31
+ return value;
32
+ if (typeof value === "string" && value.trim() !== "") {
33
+ const n = Number(value);
34
+ if (Number.isFinite(n))
35
+ return n;
36
+ }
37
+ return undefined;
38
+ }
39
+ /** Parse one raw answer object into a typed {@link DecisionAnswer}. */
40
+ function parseAnswer(id, question, raw) {
41
+ const record = raw !== null && typeof raw === "object" ? raw : {};
42
+ const answer = { id, type: question.type };
43
+ if (typeof record.choice === "string" && record.choice.length > 0) {
44
+ answer.choice = record.choice;
45
+ }
46
+ const noul = record.noul;
47
+ if (noul !== null && typeof noul === "object") {
48
+ // Defensive: some embeddings of the API wrap the probability.
49
+ const wrapped = noul;
50
+ const p = toNumber(wrapped.probability) ?? toNumber(wrapped.noul);
51
+ if (p !== undefined)
52
+ answer.noul = p;
53
+ }
54
+ else {
55
+ const p = toNumber(noul);
56
+ if (p !== undefined)
57
+ answer.noul = p;
58
+ }
59
+ const score = toNumber(record.score);
60
+ if (score !== undefined)
61
+ answer.score = score;
62
+ if (Array.isArray(record.legend)) {
63
+ const legend = record.legend.filter((entry) => typeof entry === "string");
64
+ if (legend.length > 0)
65
+ answer.legend = legend;
66
+ }
67
+ const probabilities = record.probabilities;
68
+ if (Array.isArray(probabilities)) {
69
+ const values = probabilities
70
+ .map(toNumber)
71
+ .filter((value) => value !== undefined);
72
+ if (values.length > 0)
73
+ answer.probabilities = values;
74
+ }
75
+ else if (probabilities !== null && typeof probabilities === "object") {
76
+ const map = {};
77
+ for (const [key, value] of Object.entries(probabilities)) {
78
+ const n = toNumber(value);
79
+ if (n !== undefined)
80
+ map[key] = n;
81
+ }
82
+ if (Object.keys(map).length > 0)
83
+ answer.probabilities = map;
84
+ }
85
+ const confidence = toNumber(record.confidence);
86
+ if (confidence !== undefined)
87
+ answer.confidence = confidence;
88
+ return answer;
89
+ }
90
+ /**
91
+ * Build the `questions` entry for the request body. `criteria` is passed
92
+ * through as-is; validation happens before the provider is called.
93
+ */
94
+ function buildQuestionPayload(question) {
95
+ const payload = {
96
+ type: question.type,
97
+ instructions: question.instructions,
98
+ };
99
+ if (question.criteria !== undefined) {
100
+ payload.criteria = question.criteria;
101
+ }
102
+ return payload;
103
+ }
104
+ /** Truncate provider error bodies so logs stay readable. */
105
+ function truncate(text, max = 300) {
106
+ const trimmed = text.trim();
107
+ return trimmed.length > max ? `${trimmed.slice(0, max)}…` : trimmed;
108
+ }
109
+ /**
110
+ * Decision provider for Ollama's `/v1/systemone` endpoint (tev1).
111
+ *
112
+ * Retries retryable HTTP statuses and network errors with exponential
113
+ * backoff. The first call may load the model, so configure a generous
114
+ * `timeoutMs` (default 120s) and optionally `keepAlive` to keep it resident.
115
+ */
116
+ export class OllamaDecisionProvider {
117
+ name = "ollama";
118
+ baseUrl;
119
+ model;
120
+ timeoutMs;
121
+ keepAlive;
122
+ proxy;
123
+ retryMax;
124
+ retryBaseDelayMs;
125
+ /**
126
+ * @param config - Decision-model configuration (endpoint, model, timeout, proxy, retries).
127
+ */
128
+ constructor(config) {
129
+ this.baseUrl = config.baseUrl;
130
+ this.model = config.model;
131
+ this.timeoutMs = config.timeoutMs > 0 ? config.timeoutMs : 120000;
132
+ this.keepAlive = config.keepAlive;
133
+ this.proxy = config.proxy;
134
+ this.retryMax = config.retryMax ?? 2;
135
+ this.retryBaseDelayMs = config.retryBaseDelayMs ?? 500;
136
+ }
137
+ /**
138
+ * Run a decision request. One answer is returned per question, in request
139
+ * order; answers missing from the response degrade to bare `{id, type}`.
140
+ *
141
+ * @param request - The state text and questions.
142
+ * @param abort - Optional abort signal.
143
+ * @throws If the endpoint is unreachable, rejects the request (e.g. Ollama
144
+ * < 0.35 or model not pulled), the body exceeds 64 KiB, or the response
145
+ * carries no answers.
146
+ */
147
+ async decide(request, abort) {
148
+ const questions = {};
149
+ for (const question of request.questions) {
150
+ questions[question.id] = buildQuestionPayload(question);
151
+ }
152
+ const body = {
153
+ model: this.model,
154
+ state: request.state,
155
+ questions,
156
+ };
157
+ const keepAlive = normalizeKeepAlive(this.keepAlive);
158
+ if (keepAlive !== undefined)
159
+ body.keep_alive = keepAlive;
160
+ const serialized = JSON.stringify(body);
161
+ if (Buffer.byteLength(serialized, "utf-8") > SYSTEMONE_MAX_REQUEST_BYTES) {
162
+ throw new Error(`Decision request exceeds Ollama's 64 KiB body limit (${Buffer.byteLength(serialized, "utf-8")} bytes). ` +
163
+ `Shorten \`state\` or ask fewer questions.`);
164
+ }
165
+ const url = resolveSystemOneUrl(this.baseUrl);
166
+ let lastError;
167
+ for (let attempt = 0; attempt <= this.retryMax; attempt++) {
168
+ let response;
169
+ try {
170
+ response = await postJson(url, body, {}, this.timeoutMs, this.proxy, abort);
171
+ }
172
+ catch (err) {
173
+ lastError = err instanceof Error ? err : new Error(String(err));
174
+ if (attempt < this.retryMax) {
175
+ await this.sleep(attempt);
176
+ continue;
177
+ }
178
+ throw lastError;
179
+ }
180
+ if (!response.ok) {
181
+ const text = await response.text().catch(() => "");
182
+ const error = new Error(this.describeHttpError(response.status, text));
183
+ if (RETRYABLE_STATUSES.has(response.status) && attempt < this.retryMax) {
184
+ lastError = error;
185
+ await this.sleep(attempt);
186
+ continue;
187
+ }
188
+ throw error;
189
+ }
190
+ const json = (await response.json());
191
+ const rawAnswers = json?.answers;
192
+ if (rawAnswers === null || typeof rawAnswers !== "object" || Array.isArray(rawAnswers)) {
193
+ throw new Error(`Decision model returned no answers for model "${this.model}": ${truncate(JSON.stringify(json))}`);
194
+ }
195
+ const answerMap = rawAnswers;
196
+ const answers = [];
197
+ for (const question of request.questions) {
198
+ answers.push(parseAnswer(question.id, question, answerMap[question.id]));
199
+ }
200
+ return { answers, raw: json };
201
+ }
202
+ throw lastError ?? new Error("Decision request failed");
203
+ }
204
+ /**
205
+ * Map an HTTP failure to an actionable message.
206
+ *
207
+ * 404 is overloaded: Ollama < 0.35 returns the Go default "404 page not
208
+ * found" (route missing), while a server that has the decision route but
209
+ * not the model returns `model "X" not found, try pulling it first`. Split
210
+ * the two so the fix is obvious.
211
+ */
212
+ describeHttpError(status, text) {
213
+ const detail = truncate(text);
214
+ if (status === 404) {
215
+ const lower = text.toLowerCase();
216
+ const modelLower = this.model.toLowerCase();
217
+ if (lower.includes("try pulling") || (lower.includes("not found") && lower.includes(modelLower))) {
218
+ return `Decision model "${this.model}" is not pulled (404) — run \`ollama pull ${this.model}\`. ${detail}`;
219
+ }
220
+ if (lower.includes("page not found") || detail.length === 0) {
221
+ return `Decision endpoint not found (404) — tev1 requires Ollama >= 0.35 (\`POST /v1/systemone\`). ${detail}`;
222
+ }
223
+ return (`Decision request not found (404) — tev1 requires Ollama >= 0.35 and the model must be pulled ` +
224
+ `(\`ollama pull ${this.model}\`). ${detail}`);
225
+ }
226
+ return `Ollama decision request failed (${status}): ${detail}`;
227
+ }
228
+ /** Exponential backoff between retries. */
229
+ sleep(attempt) {
230
+ const delay = this.retryBaseDelayMs * Math.pow(2, attempt);
231
+ return new Promise((resolve) => setTimeout(resolve, delay));
232
+ }
233
+ }
234
+ //# sourceMappingURL=systemone.js.map
@@ -0,0 +1,26 @@
1
+ /**
2
+ * @fileoverview Input validation for decision requests, shared by the
3
+ * OpenCode plugin tool, the MCP handler, and the CLI command.
4
+ *
5
+ * The tev1 API is permissive about malformed questions (it answers with
6
+ * garbage rather than erroring), so validating up front keeps agent calls
7
+ * debuggable — especially option counts, where tev1 was trained on 2-24
8
+ * options.
9
+ */
10
+ import type { DecisionQuestion } from "../core/interfaces.js";
11
+ /** Default `state` character limit (~2k-token decision context). */
12
+ export declare const DEFAULT_MAX_STATE_CHARS = 8000;
13
+ /** A decision request as received from a tool/MCP/CLI caller. */
14
+ export interface DecisionValidationInput {
15
+ state: string;
16
+ questions: DecisionQuestion[];
17
+ /** Maximum characters accepted for `state`. */
18
+ maxStateChars?: number;
19
+ }
20
+ /**
21
+ * Validate a decision request before it is sent to the model.
22
+ *
23
+ * @param input - State text, questions, and the configured state limit.
24
+ * @returns An actionable error message, or `undefined` when the request is valid.
25
+ */
26
+ export declare function validateDecisionRequest(input: DecisionValidationInput): string | undefined;
@@ -0,0 +1,74 @@
1
+ import { SYSTEMONE_MAX_OPTIONS, SYSTEMONE_MAX_QUESTIONS, SYSTEMONE_MIN_OPTIONS } from "./systemone.js";
2
+ /** Default `state` character limit (~2k-token decision context). */
3
+ export const DEFAULT_MAX_STATE_CHARS = 8000;
4
+ /** Count the options of a `choice` question (object criteria or array fallback). */
5
+ function countChoiceOptions(question) {
6
+ if (Array.isArray(question.criteria))
7
+ return question.criteria.length;
8
+ if (question.criteria !== null && typeof question.criteria === "object") {
9
+ return Object.keys(question.criteria).length;
10
+ }
11
+ return 0;
12
+ }
13
+ /**
14
+ * Validate a decision request before it is sent to the model.
15
+ *
16
+ * @param input - State text, questions, and the configured state limit.
17
+ * @returns An actionable error message, or `undefined` when the request is valid.
18
+ */
19
+ export function validateDecisionRequest(input) {
20
+ const state = typeof input.state === "string" ? input.state : "";
21
+ if (state.trim().length === 0) {
22
+ return "`state` must not be empty.";
23
+ }
24
+ const maxStateChars = input.maxStateChars && input.maxStateChars > 0 ? input.maxStateChars : DEFAULT_MAX_STATE_CHARS;
25
+ if (state.length > maxStateChars) {
26
+ return (`\`state\` is ${state.length} characters; the configured limit is ${maxStateChars} ` +
27
+ "(decision models have a ~2k-token context). Shorten `state` to the relevant excerpt.");
28
+ }
29
+ const questions = Array.isArray(input.questions) ? input.questions : [];
30
+ if (questions.length < 1) {
31
+ return "At least one question is required.";
32
+ }
33
+ if (questions.length > SYSTEMONE_MAX_QUESTIONS) {
34
+ return `At most ${SYSTEMONE_MAX_QUESTIONS} questions per call (got ${questions.length}).`;
35
+ }
36
+ const seen = new Set();
37
+ for (const question of questions) {
38
+ if (typeof question.id !== "string" || question.id.trim().length === 0) {
39
+ return "Every question needs a non-empty `id`.";
40
+ }
41
+ if (seen.has(question.id)) {
42
+ return `Duplicate question id "${question.id}" — ids must be unique.`;
43
+ }
44
+ seen.add(question.id);
45
+ if (typeof question.instructions !== "string" || question.instructions.trim().length === 0) {
46
+ return `Question "${question.id}" needs non-empty \`instructions\`.`;
47
+ }
48
+ if (question.type === "choice") {
49
+ const optionCount = countChoiceOptions(question);
50
+ if (optionCount < SYSTEMONE_MIN_OPTIONS || optionCount > SYSTEMONE_MAX_OPTIONS) {
51
+ return (`Question "${question.id}": \`choice\` requires 2-24 options via \`criteria\` ` +
52
+ `(an object mapping option → description); got ${optionCount}. ` +
53
+ `Add a "none" option when no listed option may fit.`);
54
+ }
55
+ }
56
+ else if (question.type === "score") {
57
+ const levels = Array.isArray(question.criteria) ? question.criteria : [];
58
+ if (levels.length < SYSTEMONE_MIN_OPTIONS || levels.length > SYSTEMONE_MAX_OPTIONS) {
59
+ return (`Question "${question.id}": \`score\` requires \`criteria\` as an array of ` +
60
+ `2-24 level descriptions, lowest first (got ${levels.length}).`);
61
+ }
62
+ }
63
+ else if (question.type === "noul") {
64
+ if (Array.isArray(question.criteria)) {
65
+ return `Question "${question.id}": \`noul\` criteria must be an object like { "true": "...", "false": "..." }, not an array.`;
66
+ }
67
+ }
68
+ else {
69
+ return `Question "${question.id}": unknown type "${String(question.type)}" — expected "choice", "noul", or "score".`;
70
+ }
71
+ }
72
+ return undefined;
73
+ }
74
+ //# sourceMappingURL=validate.js.map
@@ -64,4 +64,4 @@ export declare function probeEmbeddingDimension(embedder: EmbeddingProvider): Pr
64
64
  * @returns A promise resolving to a flat array of embedding vectors (one per input text);
65
65
  * failed batches return empty arrays
66
66
  */
67
- export declare function embedBatch(embedder: EmbeddingProvider, texts: string[], batchSize?: number, purpose?: "query" | "document", concurrency?: number, onProgress?: (completed: number, total: number) => void, retryMax?: number, retryBaseDelayMs?: number): Promise<number[][]>;
67
+ export declare function embedBatch(embedder: EmbeddingProvider, texts: string[], batchSize?: number, purpose?: "query" | "document", concurrency?: number, onProgress?: (completed: number, total: number) => void, retryMax?: number, retryBaseDelayMs?: number, maxConsecutiveFailures?: number): Promise<number[][]>;
@@ -94,6 +94,29 @@ const TIMEOUT_RE = /timed out|timeout/i;
94
94
  function isTimeoutError(err) {
95
95
  return TIMEOUT_RE.test(err instanceof Error ? err.message : String(err));
96
96
  }
97
+ /**
98
+ * HTTP 502/503/504 — the request never got a usable answer from the model
99
+ * backend. A reverse proxy or model router (llama-swap, nginx, …) returns these
100
+ * when the backend is down, over capacity, or crashing on the input — most
101
+ * commonly an embedding server whose context window is smaller than the chunk
102
+ * we send (it accepts a short probe, so preflight passes, then 502s on every
103
+ * real chunk). Unlike a per-text context rejection, shrinking the batch cannot
104
+ * help: the server itself is the problem. So gateway errors are retried at the
105
+ * top level (a brief blip recovers) but never split — splitting only multiplies
106
+ * load on an unhealthy backend — and repeated ones trip the circuit breaker.
107
+ */
108
+ const GATEWAY_STATUS_RE = /\(?(502|503|504)\)?/;
109
+ function isGatewayError(err) {
110
+ return GATEWAY_STATUS_RE.test(err instanceof Error ? err.message : String(err));
111
+ }
112
+ /**
113
+ * How many CONSECUTIVELY failed batches stop all further embedding requests in a
114
+ * single `embedBatch` call (the counter is reset by any batch that succeeds).
115
+ * Bounds the damage when the provider is down or undersized so a run fails fast
116
+ * and reports the provider as unavailable, instead of silently grinding through
117
+ * every batch for minutes — which reads as a hang. `0` disables the breaker.
118
+ */
119
+ const DEFAULT_MAX_CONSECUTIVE_BATCH_FAILURES = 6;
97
120
  /** Resolve the workspace debug log file used for batch failure diagnostics. */
98
121
  function getBatchLogFilePath() {
99
122
  return path.resolve(process.cwd(), ".opencode", "opencode-rag.log");
@@ -127,7 +150,7 @@ function getBatchLogFilePath() {
127
150
  * @returns A promise resolving to a flat array of embedding vectors (one per input text);
128
151
  * failed batches return empty arrays
129
152
  */
130
- export async function embedBatch(embedder, texts, batchSize = 10, purpose, concurrency = 1, onProgress, retryMax = 3, retryBaseDelayMs = 1000) {
153
+ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concurrency = 1, onProgress, retryMax = 3, retryBaseDelayMs = 1000, maxConsecutiveFailures = DEFAULT_MAX_CONSECUTIVE_BATCH_FAILURES) {
131
154
  if (texts.length === 0)
132
155
  return [];
133
156
  const batches = [];
@@ -184,8 +207,16 @@ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concu
184
207
  // counted by the caller. A context-limit rejection carries a permanent 400
185
208
  // status, so it needs its own predicate; timeouts are already non-permanent
186
209
  // (belt and braces). Genuine permanent errors never split — halving the
187
- // batch would fail the same way, only slower.
188
- const splittable = !isPermanentError(lastError) || isContextLimitError(lastError) || isTimeoutError(lastError);
210
+ // batch would fail the same way, only slower. Neither do gateway errors
211
+ // (502/503/504): they come from the SERVER (down, overloaded, or a context
212
+ // smaller than our chunks), not a bad text, so a smaller batch fails
213
+ // identically. Top-level retry already handles a transient one; a persistent
214
+ // one is left to the circuit breaker below rather than split into an
215
+ // exponential retry tree that looks exactly like a hang.
216
+ const splittable = (!isPermanentError(lastError) ||
217
+ isContextLimitError(lastError) ||
218
+ isTimeoutError(lastError)) &&
219
+ !isGatewayError(lastError);
189
220
  if (batchTexts.length > 1 && splittable) {
190
221
  const mid = Math.ceil(batchTexts.length / 2);
191
222
  appendDebugLog(getBatchLogFilePath(), {
@@ -203,10 +234,50 @@ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concu
203
234
  }
204
235
  return null;
205
236
  }
237
+ // ── Circuit breaker ────────────────────────────────────────────────────
238
+ // A down / overloaded / undersized embedding server fails EVERY batch. Without
239
+ // a break we keep firing batch after batch at it for the whole run — a silent,
240
+ // minutes-long grind indistinguishable from a hang. After this many
241
+ // CONSECUTIVELY failed batches (reset by any success) we stop calling the
242
+ // provider entirely and short-circuit the rest to empty vectors, so the pass
243
+ // fails fast and the pipeline reports the provider as unavailable.
244
+ let consecutiveFailedBatches = 0;
245
+ let circuitTripped = false;
246
+ function recordBatchResult(succeeded) {
247
+ if (succeeded) {
248
+ consecutiveFailedBatches = 0;
249
+ return;
250
+ }
251
+ consecutiveFailedBatches++;
252
+ if (!circuitTripped &&
253
+ maxConsecutiveFailures > 0 &&
254
+ consecutiveFailedBatches >= maxConsecutiveFailures) {
255
+ circuitTripped = true;
256
+ appendDebugLog(getBatchLogFilePath(), {
257
+ scope: "embedder.batch",
258
+ severity: "warn",
259
+ message: `Embedding circuit breaker tripped after ${consecutiveFailedBatches} consecutive failed ` +
260
+ "batches — the embedding server is unavailable, overloaded, or rejecting the input (an " +
261
+ "over-large / too-small context is the usual cause). Stopping further requests; un-embedded " +
262
+ "chunks are kept for the next pass. Fix the provider (e.g. raise its context window) and " +
263
+ "run the index again.",
264
+ });
265
+ }
266
+ }
206
267
  if (concurrency <= 1 || batches.length <= 1) {
207
268
  const results = [];
208
269
  for (const batch of batches) {
270
+ if (circuitTripped) {
271
+ // Breaker open: emit empty vectors so caller index→vector alignment is
272
+ // preserved, advance progress, and never touch the provider again.
273
+ for (let i = 0; i < batch.texts.length; i++) {
274
+ results.push([]);
275
+ }
276
+ onProgress?.(results.length, texts.length);
277
+ continue;
278
+ }
209
279
  const embeddings = await embedWithRetry(batch.texts);
280
+ recordBatchResult(embeddings !== null);
210
281
  if (embeddings) {
211
282
  results.push(...embeddings);
212
283
  }
@@ -222,7 +293,13 @@ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concu
222
293
  let completedCount = 0;
223
294
  const limit = pLimit(concurrency);
224
295
  const batchResults = await Promise.all(batches.map((batch) => limit(async () => {
296
+ if (circuitTripped) {
297
+ completedCount += batch.texts.length;
298
+ onProgress?.(completedCount, texts.length);
299
+ return { index: batch.index, embeddings: batch.texts.map(() => []) };
300
+ }
225
301
  const embeddings = await embedWithRetry(batch.texts);
302
+ recordBatchResult(embeddings !== null);
226
303
  const flatResult = embeddings ?? batch.texts.map(() => []);
227
304
  // Count processed TEXTS (not returned vectors) so failed batches
228
305
  // still advance progress towards the total.
@@ -0,0 +1,112 @@
1
+ import type { DecisionAnswer, DecisionProvider } from "../core/interfaces.js";
2
+ /** One question in a calibration item. */
3
+ export interface DecideCalibrationQuestion {
4
+ type: "noul" | "choice" | "score";
5
+ instructions: string;
6
+ criteria?: Record<string, string> | string[] | {
7
+ true?: string;
8
+ false?: string;
9
+ };
10
+ }
11
+ /** One labelled decision task. */
12
+ export interface DecideCalibrationItem {
13
+ id: string;
14
+ category: string;
15
+ state: string;
16
+ question: DecideCalibrationQuestion;
17
+ /**
18
+ * Expected answer: `"true"`/`"false"` for `noul`, an option key for `choice`,
19
+ * a level label for `score`.
20
+ */
21
+ expected: string;
22
+ note?: string;
23
+ }
24
+ /** Serialized calibration file (`src/eval/decide-labels.json`). */
25
+ export interface DecideCalibrationFile {
26
+ version: number;
27
+ labels: DecideCalibrationItem[];
28
+ }
29
+ /**
30
+ * Load and validate a decision calibration label file.
31
+ *
32
+ * @param labelsPath - Absolute path to the JSON label file.
33
+ * @returns The validated items.
34
+ * @throws When the file is missing, has no labels, or an item lacks id/state/question/expected.
35
+ */
36
+ export declare function loadDecideCalibration(labelsPath: string): DecideCalibrationItem[];
37
+ /** One scored calibration item. */
38
+ export interface ScoredDecisionItem {
39
+ item: DecideCalibrationItem;
40
+ /** Normalized prediction (`"true"`/`"false"`, option key, or level label), or undefined. */
41
+ predicted: string | undefined;
42
+ correct: boolean;
43
+ /** Probability assigned to the predicted answer (0.5 fallback when the model reports none). */
44
+ confidence: number;
45
+ /** Probability assigned to the expected answer, when derivable (choice/noul). */
46
+ pExpected: number | null;
47
+ latencyMs: number;
48
+ }
49
+ /**
50
+ * Score one model answer against the expected label.
51
+ *
52
+ * Pure function — no provider needed, which is what makes the math unit-testable.
53
+ *
54
+ * @param item - The labelled item.
55
+ * @param answer - The model's answer (may be undefined when the provider skipped it).
56
+ * @param latencyMs - Measured call latency.
57
+ * @returns Prediction, correctness, prediction confidence, expected-class probability.
58
+ */
59
+ export declare function scoreDecisionAnswer(item: DecideCalibrationItem, answer: DecisionAnswer | undefined, latencyMs: number): ScoredDecisionItem;
60
+ /** One confidence bin of the reliability diagram. */
61
+ export interface CalibrationBin {
62
+ lo: number;
63
+ hi: number;
64
+ count: number;
65
+ /** Mean confidence of the predictions in this bin. */
66
+ confidence: number;
67
+ /** Share of correct predictions in this bin. */
68
+ accuracy: number;
69
+ }
70
+ /** Aggregate calibration metrics. */
71
+ export interface CalibrationSummary {
72
+ /** Total items. */
73
+ n: number;
74
+ /** Items the model actually answered (prediction present). */
75
+ answered: number;
76
+ accuracy: number;
77
+ /** Expected calibration error over `bins` equal-width confidence bins. */
78
+ ece: number;
79
+ /** Mean squared error of confidence vs. correctness. */
80
+ brier: number;
81
+ meanConfidence: number;
82
+ /** Mean probability assigned to the expected answer (noul/choice only, when derivable). */
83
+ meanPExpected: number | null;
84
+ bins: CalibrationBin[];
85
+ }
86
+ /**
87
+ * Aggregate scored items into accuracy / ECE / Brier.
88
+ *
89
+ * @param items - Scored items (unanswered ones count against accuracy but not into bins).
90
+ * @param binCount - Number of equal-width confidence bins (default 10).
91
+ * @returns The calibration summary including the reliability bins.
92
+ */
93
+ export declare function summarizeCalibration(items: ScoredDecisionItem[], binCount?: number): CalibrationSummary;
94
+ /** Options for {@link runDecideCalibration}. */
95
+ export interface DecideCalibrationRunOptions {
96
+ items: DecideCalibrationItem[];
97
+ provider: DecisionProvider;
98
+ /** Progress line callback (defaults to console.log). */
99
+ log?: (line: string) => void;
100
+ }
101
+ /**
102
+ * Run every calibration item through the decision provider and score the answers.
103
+ *
104
+ * @param opts - Items, provider, logger.
105
+ * @returns Scored items and the aggregate summary.
106
+ * @throws When the provider fails for the very first item (e.g. Ollama down) — the
107
+ * caller should surface the provider error instead of a misleading 0% accuracy.
108
+ */
109
+ export declare function runDecideCalibration(opts: DecideCalibrationRunOptions): Promise<{
110
+ scored: ScoredDecisionItem[];
111
+ summary: CalibrationSummary;
112
+ }>;