opencode-rag-plugin 2.3.1 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +71 -8
- package/dist/chunker/skeleton.js +1 -1
- package/dist/cli/commands/decide.d.ts +25 -0
- package/dist/cli/commands/decide.js +160 -0
- package/dist/cli/commands/eval.d.ts +23 -0
- package/dist/cli/commands/eval.js +331 -1
- package/dist/cli/commands/index-command.js +28 -10
- package/dist/cli/commands/index.d.ts +2 -1
- package/dist/cli/commands/index.js +2 -1
- package/dist/cli/commands/init-helpers.d.ts +16 -1
- package/dist/cli/commands/init-helpers.js +84 -2
- package/dist/cli/commands/init.js +29 -4
- package/dist/cli/commands/status.js +14 -8
- package/dist/cli/format.d.ts +50 -0
- package/dist/cli/format.js +122 -0
- package/dist/cli/index.js +4 -1
- package/dist/cli/types.d.ts +2 -0
- package/dist/core/bootstrap.js +1 -0
- package/dist/core/config.d.ts +47 -0
- package/dist/core/config.js +34 -1
- package/dist/core/interfaces.d.ts +69 -0
- package/dist/decision/factory.d.ts +17 -0
- package/dist/decision/factory.js +17 -0
- package/dist/decision/format.d.ts +14 -0
- package/dist/decision/format.js +53 -0
- package/dist/decision/systemone.d.ts +76 -0
- package/dist/decision/systemone.js +234 -0
- package/dist/decision/validate.d.ts +26 -0
- package/dist/decision/validate.js +74 -0
- package/dist/embedder/factory.d.ts +1 -1
- package/dist/embedder/factory.js +80 -3
- package/dist/eval/decide-calibration.d.ts +112 -0
- package/dist/eval/decide-calibration.js +185 -0
- package/dist/eval/golden-gate.d.ts +65 -0
- package/dist/eval/golden-gate.js +61 -0
- package/dist/eval/golden-set.d.ts +81 -0
- package/dist/eval/golden-set.js +155 -0
- package/dist/eval/rerank-eval.js +10 -125
- package/dist/index.d.ts +6 -2
- package/dist/index.js +4 -0
- package/dist/indexer/pipeline.d.ts +0 -10
- package/dist/indexer/pipeline.js +1 -0
- package/dist/mcp/handlers.d.ts +17 -1
- package/dist/mcp/handlers.js +30 -0
- package/dist/mcp/server.d.ts +9 -1
- package/dist/mcp/server.js +47 -3
- package/dist/opencode/system-guidance.d.ts +11 -0
- package/dist/opencode/system-guidance.js +53 -0
- package/dist/opencode/tools.d.ts +24 -1
- package/dist/opencode/tools.js +101 -0
- package/dist/plugin.js +21 -3
- package/dist/quirks/quirk-store.d.ts +2 -0
- package/dist/quirks/quirk-store.js +33 -4
- package/dist/tui.js +21 -2
- package/dist/v2/v2-adapter.d.ts +20 -0
- package/dist/v2/v2-adapter.js +36 -8
- package/dist/vectorstore/lancedb.js +54 -5
- package/dist/watcher.d.ts +26 -1
- package/dist/watcher.js +71 -9
- package/dist/web/ui/assets/index-CWcp0jNf.css +2 -0
- package/dist/web/ui/index.html +2 -2
- package/package.json +5 -2
- package/dist/web/ui/assets/index-Dp009QCQ.css +0 -1
- /package/dist/web/ui/assets/{index-C0QXKA-b.js → index-DbXFrn53.js} +0 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Ollama decision-model provider (tev1) using the
|
|
3
|
+
* `/v1/systemone` endpoint.
|
|
4
|
+
*
|
|
5
|
+
* Decision models are NOT chat models: `state` is scored against 1-64 named
|
|
6
|
+
* `questions` and the response carries probabilities per question. The
|
|
7
|
+
* endpoint is root-level (`/v1/systemone`), so the `/api` suffix used by the
|
|
8
|
+
* embedding/description sections is stripped before building the URL.
|
|
9
|
+
*
|
|
10
|
+
* Requires Ollama >= 0.35 and a pulled tev1 model (`ollama pull tev1` or
|
|
11
|
+
* `ollama pull tev1:0.8b`).
|
|
12
|
+
*/
|
|
13
|
+
import type { DecisionProvider, DecisionRequest, DecisionResult } from "../core/interfaces.js";
|
|
14
|
+
import type { DecisionConfig } from "../core/config.js";
|
|
15
|
+
/** Request body size cap imposed by Ollama's decision endpoint (64 KiB). */
|
|
16
|
+
export declare const SYSTEMONE_MAX_REQUEST_BYTES: number;
|
|
17
|
+
/** Maximum number of questions per request (tev1 limit). */
|
|
18
|
+
export declare const SYSTEMONE_MAX_QUESTIONS = 64;
|
|
19
|
+
/** Minimum options/levels for `choice`/`score` questions. */
|
|
20
|
+
export declare const SYSTEMONE_MIN_OPTIONS = 2;
|
|
21
|
+
/** tev1 was trained on up to 24 options; 26 is the API maximum. */
|
|
22
|
+
export declare const SYSTEMONE_MAX_OPTIONS = 24;
|
|
23
|
+
/**
|
|
24
|
+
* Resolve the decision endpoint from a configured base URL.
|
|
25
|
+
*
|
|
26
|
+
* Accepts both root URLs (`http://127.0.0.1:11434`) and the `/api`-suffixed
|
|
27
|
+
* convention used by the embedding/description sections (`…:11434/api`) —
|
|
28
|
+
* `/api` is dropped because decision models are served at `/v1/systemone`.
|
|
29
|
+
*
|
|
30
|
+
* @param baseUrl - Configured Ollama base URL.
|
|
31
|
+
* @returns The absolute `/v1/systemone` URL.
|
|
32
|
+
*/
|
|
33
|
+
export declare function resolveSystemOneUrl(baseUrl: string): string;
|
|
34
|
+
/**
|
|
35
|
+
* Decision provider for Ollama's `/v1/systemone` endpoint (tev1).
|
|
36
|
+
*
|
|
37
|
+
* Retries retryable HTTP statuses and network errors with exponential
|
|
38
|
+
* backoff. The first call may load the model, so configure a generous
|
|
39
|
+
* `timeoutMs` (default 120s) and optionally `keepAlive` to keep it resident.
|
|
40
|
+
*/
|
|
41
|
+
export declare class OllamaDecisionProvider implements DecisionProvider {
|
|
42
|
+
readonly name = "ollama";
|
|
43
|
+
private readonly baseUrl;
|
|
44
|
+
private readonly model;
|
|
45
|
+
private readonly timeoutMs;
|
|
46
|
+
private readonly keepAlive?;
|
|
47
|
+
private readonly proxy?;
|
|
48
|
+
private readonly retryMax;
|
|
49
|
+
private readonly retryBaseDelayMs;
|
|
50
|
+
/**
|
|
51
|
+
* @param config - Decision-model configuration (endpoint, model, timeout, proxy, retries).
|
|
52
|
+
*/
|
|
53
|
+
constructor(config: DecisionConfig);
|
|
54
|
+
/**
|
|
55
|
+
* Run a decision request. One answer is returned per question, in request
|
|
56
|
+
* order; answers missing from the response degrade to bare `{id, type}`.
|
|
57
|
+
*
|
|
58
|
+
* @param request - The state text and questions.
|
|
59
|
+
* @param abort - Optional abort signal.
|
|
60
|
+
* @throws If the endpoint is unreachable, rejects the request (e.g. Ollama
|
|
61
|
+
* < 0.35 or model not pulled), the body exceeds 64 KiB, or the response
|
|
62
|
+
* carries no answers.
|
|
63
|
+
*/
|
|
64
|
+
decide(request: DecisionRequest, abort?: AbortSignal): Promise<DecisionResult>;
|
|
65
|
+
/**
|
|
66
|
+
* Map an HTTP failure to an actionable message.
|
|
67
|
+
*
|
|
68
|
+
* 404 is overloaded: Ollama < 0.35 returns the Go default "404 page not
|
|
69
|
+
* found" (route missing), while a server that has the decision route but
|
|
70
|
+
* not the model returns `model "X" not found, try pulling it first`. Split
|
|
71
|
+
* the two so the fix is obvious.
|
|
72
|
+
*/
|
|
73
|
+
private describeHttpError;
|
|
74
|
+
/** Exponential backoff between retries. */
|
|
75
|
+
private sleep;
|
|
76
|
+
}
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
import { normalizeKeepAlive } from "../core/ollama.js";
|
|
2
|
+
import { postJson } from "../embedder/http.js";
|
|
3
|
+
/** HTTP status codes that are safe to retry on. */
|
|
4
|
+
const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
|
|
5
|
+
/** Request body size cap imposed by Ollama's decision endpoint (64 KiB). */
|
|
6
|
+
export const SYSTEMONE_MAX_REQUEST_BYTES = 64 * 1024;
|
|
7
|
+
/** Maximum number of questions per request (tev1 limit). */
|
|
8
|
+
export const SYSTEMONE_MAX_QUESTIONS = 64;
|
|
9
|
+
/** Minimum options/levels for `choice`/`score` questions. */
|
|
10
|
+
export const SYSTEMONE_MIN_OPTIONS = 2;
|
|
11
|
+
/** tev1 was trained on up to 24 options; 26 is the API maximum. */
|
|
12
|
+
export const SYSTEMONE_MAX_OPTIONS = 24;
|
|
13
|
+
/**
|
|
14
|
+
* Resolve the decision endpoint from a configured base URL.
|
|
15
|
+
*
|
|
16
|
+
* Accepts both root URLs (`http://127.0.0.1:11434`) and the `/api`-suffixed
|
|
17
|
+
* convention used by the embedding/description sections (`…:11434/api`) —
|
|
18
|
+
* `/api` is dropped because decision models are served at `/v1/systemone`.
|
|
19
|
+
*
|
|
20
|
+
* @param baseUrl - Configured Ollama base URL.
|
|
21
|
+
* @returns The absolute `/v1/systemone` URL.
|
|
22
|
+
*/
|
|
23
|
+
export function resolveSystemOneUrl(baseUrl) {
|
|
24
|
+
const trimmed = baseUrl.replace(/\/+$/, "");
|
|
25
|
+
const root = trimmed.endsWith("/api") ? trimmed.slice(0, -"/api".length) : trimmed;
|
|
26
|
+
return `${root}/v1/systemone`;
|
|
27
|
+
}
|
|
28
|
+
/** Coerce a number-or-numeric-string to a finite number, else undefined. */
|
|
29
|
+
function toNumber(value) {
|
|
30
|
+
if (typeof value === "number" && Number.isFinite(value))
|
|
31
|
+
return value;
|
|
32
|
+
if (typeof value === "string" && value.trim() !== "") {
|
|
33
|
+
const n = Number(value);
|
|
34
|
+
if (Number.isFinite(n))
|
|
35
|
+
return n;
|
|
36
|
+
}
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
39
|
+
/** Parse one raw answer object into a typed {@link DecisionAnswer}. */
|
|
40
|
+
function parseAnswer(id, question, raw) {
|
|
41
|
+
const record = raw !== null && typeof raw === "object" ? raw : {};
|
|
42
|
+
const answer = { id, type: question.type };
|
|
43
|
+
if (typeof record.choice === "string" && record.choice.length > 0) {
|
|
44
|
+
answer.choice = record.choice;
|
|
45
|
+
}
|
|
46
|
+
const noul = record.noul;
|
|
47
|
+
if (noul !== null && typeof noul === "object") {
|
|
48
|
+
// Defensive: some embeddings of the API wrap the probability.
|
|
49
|
+
const wrapped = noul;
|
|
50
|
+
const p = toNumber(wrapped.probability) ?? toNumber(wrapped.noul);
|
|
51
|
+
if (p !== undefined)
|
|
52
|
+
answer.noul = p;
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
const p = toNumber(noul);
|
|
56
|
+
if (p !== undefined)
|
|
57
|
+
answer.noul = p;
|
|
58
|
+
}
|
|
59
|
+
const score = toNumber(record.score);
|
|
60
|
+
if (score !== undefined)
|
|
61
|
+
answer.score = score;
|
|
62
|
+
if (Array.isArray(record.legend)) {
|
|
63
|
+
const legend = record.legend.filter((entry) => typeof entry === "string");
|
|
64
|
+
if (legend.length > 0)
|
|
65
|
+
answer.legend = legend;
|
|
66
|
+
}
|
|
67
|
+
const probabilities = record.probabilities;
|
|
68
|
+
if (Array.isArray(probabilities)) {
|
|
69
|
+
const values = probabilities
|
|
70
|
+
.map(toNumber)
|
|
71
|
+
.filter((value) => value !== undefined);
|
|
72
|
+
if (values.length > 0)
|
|
73
|
+
answer.probabilities = values;
|
|
74
|
+
}
|
|
75
|
+
else if (probabilities !== null && typeof probabilities === "object") {
|
|
76
|
+
const map = {};
|
|
77
|
+
for (const [key, value] of Object.entries(probabilities)) {
|
|
78
|
+
const n = toNumber(value);
|
|
79
|
+
if (n !== undefined)
|
|
80
|
+
map[key] = n;
|
|
81
|
+
}
|
|
82
|
+
if (Object.keys(map).length > 0)
|
|
83
|
+
answer.probabilities = map;
|
|
84
|
+
}
|
|
85
|
+
const confidence = toNumber(record.confidence);
|
|
86
|
+
if (confidence !== undefined)
|
|
87
|
+
answer.confidence = confidence;
|
|
88
|
+
return answer;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Build the `questions` entry for the request body. `criteria` is passed
|
|
92
|
+
* through as-is; validation happens before the provider is called.
|
|
93
|
+
*/
|
|
94
|
+
function buildQuestionPayload(question) {
|
|
95
|
+
const payload = {
|
|
96
|
+
type: question.type,
|
|
97
|
+
instructions: question.instructions,
|
|
98
|
+
};
|
|
99
|
+
if (question.criteria !== undefined) {
|
|
100
|
+
payload.criteria = question.criteria;
|
|
101
|
+
}
|
|
102
|
+
return payload;
|
|
103
|
+
}
|
|
104
|
+
/** Truncate provider error bodies so logs stay readable. */
|
|
105
|
+
function truncate(text, max = 300) {
|
|
106
|
+
const trimmed = text.trim();
|
|
107
|
+
return trimmed.length > max ? `${trimmed.slice(0, max)}…` : trimmed;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Decision provider for Ollama's `/v1/systemone` endpoint (tev1).
|
|
111
|
+
*
|
|
112
|
+
* Retries retryable HTTP statuses and network errors with exponential
|
|
113
|
+
* backoff. The first call may load the model, so configure a generous
|
|
114
|
+
* `timeoutMs` (default 120s) and optionally `keepAlive` to keep it resident.
|
|
115
|
+
*/
|
|
116
|
+
export class OllamaDecisionProvider {
|
|
117
|
+
name = "ollama";
|
|
118
|
+
baseUrl;
|
|
119
|
+
model;
|
|
120
|
+
timeoutMs;
|
|
121
|
+
keepAlive;
|
|
122
|
+
proxy;
|
|
123
|
+
retryMax;
|
|
124
|
+
retryBaseDelayMs;
|
|
125
|
+
/**
|
|
126
|
+
* @param config - Decision-model configuration (endpoint, model, timeout, proxy, retries).
|
|
127
|
+
*/
|
|
128
|
+
constructor(config) {
|
|
129
|
+
this.baseUrl = config.baseUrl;
|
|
130
|
+
this.model = config.model;
|
|
131
|
+
this.timeoutMs = config.timeoutMs > 0 ? config.timeoutMs : 120000;
|
|
132
|
+
this.keepAlive = config.keepAlive;
|
|
133
|
+
this.proxy = config.proxy;
|
|
134
|
+
this.retryMax = config.retryMax ?? 2;
|
|
135
|
+
this.retryBaseDelayMs = config.retryBaseDelayMs ?? 500;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Run a decision request. One answer is returned per question, in request
|
|
139
|
+
* order; answers missing from the response degrade to bare `{id, type}`.
|
|
140
|
+
*
|
|
141
|
+
* @param request - The state text and questions.
|
|
142
|
+
* @param abort - Optional abort signal.
|
|
143
|
+
* @throws If the endpoint is unreachable, rejects the request (e.g. Ollama
|
|
144
|
+
* < 0.35 or model not pulled), the body exceeds 64 KiB, or the response
|
|
145
|
+
* carries no answers.
|
|
146
|
+
*/
|
|
147
|
+
async decide(request, abort) {
|
|
148
|
+
const questions = {};
|
|
149
|
+
for (const question of request.questions) {
|
|
150
|
+
questions[question.id] = buildQuestionPayload(question);
|
|
151
|
+
}
|
|
152
|
+
const body = {
|
|
153
|
+
model: this.model,
|
|
154
|
+
state: request.state,
|
|
155
|
+
questions,
|
|
156
|
+
};
|
|
157
|
+
const keepAlive = normalizeKeepAlive(this.keepAlive);
|
|
158
|
+
if (keepAlive !== undefined)
|
|
159
|
+
body.keep_alive = keepAlive;
|
|
160
|
+
const serialized = JSON.stringify(body);
|
|
161
|
+
if (Buffer.byteLength(serialized, "utf-8") > SYSTEMONE_MAX_REQUEST_BYTES) {
|
|
162
|
+
throw new Error(`Decision request exceeds Ollama's 64 KiB body limit (${Buffer.byteLength(serialized, "utf-8")} bytes). ` +
|
|
163
|
+
`Shorten \`state\` or ask fewer questions.`);
|
|
164
|
+
}
|
|
165
|
+
const url = resolveSystemOneUrl(this.baseUrl);
|
|
166
|
+
let lastError;
|
|
167
|
+
for (let attempt = 0; attempt <= this.retryMax; attempt++) {
|
|
168
|
+
let response;
|
|
169
|
+
try {
|
|
170
|
+
response = await postJson(url, body, {}, this.timeoutMs, this.proxy, abort);
|
|
171
|
+
}
|
|
172
|
+
catch (err) {
|
|
173
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
174
|
+
if (attempt < this.retryMax) {
|
|
175
|
+
await this.sleep(attempt);
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
throw lastError;
|
|
179
|
+
}
|
|
180
|
+
if (!response.ok) {
|
|
181
|
+
const text = await response.text().catch(() => "");
|
|
182
|
+
const error = new Error(this.describeHttpError(response.status, text));
|
|
183
|
+
if (RETRYABLE_STATUSES.has(response.status) && attempt < this.retryMax) {
|
|
184
|
+
lastError = error;
|
|
185
|
+
await this.sleep(attempt);
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
throw error;
|
|
189
|
+
}
|
|
190
|
+
const json = (await response.json());
|
|
191
|
+
const rawAnswers = json?.answers;
|
|
192
|
+
if (rawAnswers === null || typeof rawAnswers !== "object" || Array.isArray(rawAnswers)) {
|
|
193
|
+
throw new Error(`Decision model returned no answers for model "${this.model}": ${truncate(JSON.stringify(json))}`);
|
|
194
|
+
}
|
|
195
|
+
const answerMap = rawAnswers;
|
|
196
|
+
const answers = [];
|
|
197
|
+
for (const question of request.questions) {
|
|
198
|
+
answers.push(parseAnswer(question.id, question, answerMap[question.id]));
|
|
199
|
+
}
|
|
200
|
+
return { answers, raw: json };
|
|
201
|
+
}
|
|
202
|
+
throw lastError ?? new Error("Decision request failed");
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Map an HTTP failure to an actionable message.
|
|
206
|
+
*
|
|
207
|
+
* 404 is overloaded: Ollama < 0.35 returns the Go default "404 page not
|
|
208
|
+
* found" (route missing), while a server that has the decision route but
|
|
209
|
+
* not the model returns `model "X" not found, try pulling it first`. Split
|
|
210
|
+
* the two so the fix is obvious.
|
|
211
|
+
*/
|
|
212
|
+
describeHttpError(status, text) {
|
|
213
|
+
const detail = truncate(text);
|
|
214
|
+
if (status === 404) {
|
|
215
|
+
const lower = text.toLowerCase();
|
|
216
|
+
const modelLower = this.model.toLowerCase();
|
|
217
|
+
if (lower.includes("try pulling") || (lower.includes("not found") && lower.includes(modelLower))) {
|
|
218
|
+
return `Decision model "${this.model}" is not pulled (404) — run \`ollama pull ${this.model}\`. ${detail}`;
|
|
219
|
+
}
|
|
220
|
+
if (lower.includes("page not found") || detail.length === 0) {
|
|
221
|
+
return `Decision endpoint not found (404) — tev1 requires Ollama >= 0.35 (\`POST /v1/systemone\`). ${detail}`;
|
|
222
|
+
}
|
|
223
|
+
return (`Decision request not found (404) — tev1 requires Ollama >= 0.35 and the model must be pulled ` +
|
|
224
|
+
`(\`ollama pull ${this.model}\`). ${detail}`);
|
|
225
|
+
}
|
|
226
|
+
return `Ollama decision request failed (${status}): ${detail}`;
|
|
227
|
+
}
|
|
228
|
+
/** Exponential backoff between retries. */
|
|
229
|
+
sleep(attempt) {
|
|
230
|
+
const delay = this.retryBaseDelayMs * Math.pow(2, attempt);
|
|
231
|
+
return new Promise((resolve) => setTimeout(resolve, delay));
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
//# sourceMappingURL=systemone.js.map
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Input validation for decision requests, shared by the
|
|
3
|
+
* OpenCode plugin tool, the MCP handler, and the CLI command.
|
|
4
|
+
*
|
|
5
|
+
* The tev1 API is permissive about malformed questions (it answers with
|
|
6
|
+
* garbage rather than erroring), so validating up front keeps agent calls
|
|
7
|
+
* debuggable — especially option counts, where tev1 was trained on 2-24
|
|
8
|
+
* options.
|
|
9
|
+
*/
|
|
10
|
+
import type { DecisionQuestion } from "../core/interfaces.js";
|
|
11
|
+
/** Default `state` character limit (~2k-token decision context). */
|
|
12
|
+
export declare const DEFAULT_MAX_STATE_CHARS = 8000;
|
|
13
|
+
/** A decision request as received from a tool/MCP/CLI caller. */
|
|
14
|
+
export interface DecisionValidationInput {
|
|
15
|
+
state: string;
|
|
16
|
+
questions: DecisionQuestion[];
|
|
17
|
+
/** Maximum characters accepted for `state`. */
|
|
18
|
+
maxStateChars?: number;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Validate a decision request before it is sent to the model.
|
|
22
|
+
*
|
|
23
|
+
* @param input - State text, questions, and the configured state limit.
|
|
24
|
+
* @returns An actionable error message, or `undefined` when the request is valid.
|
|
25
|
+
*/
|
|
26
|
+
export declare function validateDecisionRequest(input: DecisionValidationInput): string | undefined;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { SYSTEMONE_MAX_OPTIONS, SYSTEMONE_MAX_QUESTIONS, SYSTEMONE_MIN_OPTIONS } from "./systemone.js";
|
|
2
|
+
/** Default `state` character limit (~2k-token decision context). */
|
|
3
|
+
export const DEFAULT_MAX_STATE_CHARS = 8000;
|
|
4
|
+
/** Count the options of a `choice` question (object criteria or array fallback). */
|
|
5
|
+
function countChoiceOptions(question) {
|
|
6
|
+
if (Array.isArray(question.criteria))
|
|
7
|
+
return question.criteria.length;
|
|
8
|
+
if (question.criteria !== null && typeof question.criteria === "object") {
|
|
9
|
+
return Object.keys(question.criteria).length;
|
|
10
|
+
}
|
|
11
|
+
return 0;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Validate a decision request before it is sent to the model.
|
|
15
|
+
*
|
|
16
|
+
* @param input - State text, questions, and the configured state limit.
|
|
17
|
+
* @returns An actionable error message, or `undefined` when the request is valid.
|
|
18
|
+
*/
|
|
19
|
+
export function validateDecisionRequest(input) {
|
|
20
|
+
const state = typeof input.state === "string" ? input.state : "";
|
|
21
|
+
if (state.trim().length === 0) {
|
|
22
|
+
return "`state` must not be empty.";
|
|
23
|
+
}
|
|
24
|
+
const maxStateChars = input.maxStateChars && input.maxStateChars > 0 ? input.maxStateChars : DEFAULT_MAX_STATE_CHARS;
|
|
25
|
+
if (state.length > maxStateChars) {
|
|
26
|
+
return (`\`state\` is ${state.length} characters; the configured limit is ${maxStateChars} ` +
|
|
27
|
+
"(decision models have a ~2k-token context). Shorten `state` to the relevant excerpt.");
|
|
28
|
+
}
|
|
29
|
+
const questions = Array.isArray(input.questions) ? input.questions : [];
|
|
30
|
+
if (questions.length < 1) {
|
|
31
|
+
return "At least one question is required.";
|
|
32
|
+
}
|
|
33
|
+
if (questions.length > SYSTEMONE_MAX_QUESTIONS) {
|
|
34
|
+
return `At most ${SYSTEMONE_MAX_QUESTIONS} questions per call (got ${questions.length}).`;
|
|
35
|
+
}
|
|
36
|
+
const seen = new Set();
|
|
37
|
+
for (const question of questions) {
|
|
38
|
+
if (typeof question.id !== "string" || question.id.trim().length === 0) {
|
|
39
|
+
return "Every question needs a non-empty `id`.";
|
|
40
|
+
}
|
|
41
|
+
if (seen.has(question.id)) {
|
|
42
|
+
return `Duplicate question id "${question.id}" — ids must be unique.`;
|
|
43
|
+
}
|
|
44
|
+
seen.add(question.id);
|
|
45
|
+
if (typeof question.instructions !== "string" || question.instructions.trim().length === 0) {
|
|
46
|
+
return `Question "${question.id}" needs non-empty \`instructions\`.`;
|
|
47
|
+
}
|
|
48
|
+
if (question.type === "choice") {
|
|
49
|
+
const optionCount = countChoiceOptions(question);
|
|
50
|
+
if (optionCount < SYSTEMONE_MIN_OPTIONS || optionCount > SYSTEMONE_MAX_OPTIONS) {
|
|
51
|
+
return (`Question "${question.id}": \`choice\` requires 2-24 options via \`criteria\` ` +
|
|
52
|
+
`(an object mapping option → description); got ${optionCount}. ` +
|
|
53
|
+
`Add a "none" option when no listed option may fit.`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
else if (question.type === "score") {
|
|
57
|
+
const levels = Array.isArray(question.criteria) ? question.criteria : [];
|
|
58
|
+
if (levels.length < SYSTEMONE_MIN_OPTIONS || levels.length > SYSTEMONE_MAX_OPTIONS) {
|
|
59
|
+
return (`Question "${question.id}": \`score\` requires \`criteria\` as an array of ` +
|
|
60
|
+
`2-24 level descriptions, lowest first (got ${levels.length}).`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
else if (question.type === "noul") {
|
|
64
|
+
if (Array.isArray(question.criteria)) {
|
|
65
|
+
return `Question "${question.id}": \`noul\` criteria must be an object like { "true": "...", "false": "..." }, not an array.`;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
return `Question "${question.id}": unknown type "${String(question.type)}" — expected "choice", "noul", or "score".`;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return undefined;
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=validate.js.map
|
|
@@ -64,4 +64,4 @@ export declare function probeEmbeddingDimension(embedder: EmbeddingProvider): Pr
|
|
|
64
64
|
* @returns A promise resolving to a flat array of embedding vectors (one per input text);
|
|
65
65
|
* failed batches return empty arrays
|
|
66
66
|
*/
|
|
67
|
-
export declare function embedBatch(embedder: EmbeddingProvider, texts: string[], batchSize?: number, purpose?: "query" | "document", concurrency?: number, onProgress?: (completed: number, total: number) => void, retryMax?: number, retryBaseDelayMs?: number): Promise<number[][]>;
|
|
67
|
+
export declare function embedBatch(embedder: EmbeddingProvider, texts: string[], batchSize?: number, purpose?: "query" | "document", concurrency?: number, onProgress?: (completed: number, total: number) => void, retryMax?: number, retryBaseDelayMs?: number, maxConsecutiveFailures?: number): Promise<number[][]>;
|
package/dist/embedder/factory.js
CHANGED
|
@@ -94,6 +94,29 @@ const TIMEOUT_RE = /timed out|timeout/i;
|
|
|
94
94
|
function isTimeoutError(err) {
|
|
95
95
|
return TIMEOUT_RE.test(err instanceof Error ? err.message : String(err));
|
|
96
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* HTTP 502/503/504 — the request never got a usable answer from the model
|
|
99
|
+
* backend. A reverse proxy or model router (llama-swap, nginx, …) returns these
|
|
100
|
+
* when the backend is down, over capacity, or crashing on the input — most
|
|
101
|
+
* commonly an embedding server whose context window is smaller than the chunk
|
|
102
|
+
* we send (it accepts a short probe, so preflight passes, then 502s on every
|
|
103
|
+
* real chunk). Unlike a per-text context rejection, shrinking the batch cannot
|
|
104
|
+
* help: the server itself is the problem. So gateway errors are retried at the
|
|
105
|
+
* top level (a brief blip recovers) but never split — splitting only multiplies
|
|
106
|
+
* load on an unhealthy backend — and repeated ones trip the circuit breaker.
|
|
107
|
+
*/
|
|
108
|
+
const GATEWAY_STATUS_RE = /\(?(502|503|504)\)?/;
|
|
109
|
+
function isGatewayError(err) {
|
|
110
|
+
return GATEWAY_STATUS_RE.test(err instanceof Error ? err.message : String(err));
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* How many CONSECUTIVELY failed batches stop all further embedding requests in a
|
|
114
|
+
* single `embedBatch` call (the counter is reset by any batch that succeeds).
|
|
115
|
+
* Bounds the damage when the provider is down or undersized so a run fails fast
|
|
116
|
+
* and reports the provider as unavailable, instead of silently grinding through
|
|
117
|
+
* every batch for minutes — which reads as a hang. `0` disables the breaker.
|
|
118
|
+
*/
|
|
119
|
+
const DEFAULT_MAX_CONSECUTIVE_BATCH_FAILURES = 6;
|
|
97
120
|
/** Resolve the workspace debug log file used for batch failure diagnostics. */
|
|
98
121
|
function getBatchLogFilePath() {
|
|
99
122
|
return path.resolve(process.cwd(), ".opencode", "opencode-rag.log");
|
|
@@ -127,7 +150,7 @@ function getBatchLogFilePath() {
|
|
|
127
150
|
* @returns A promise resolving to a flat array of embedding vectors (one per input text);
|
|
128
151
|
* failed batches return empty arrays
|
|
129
152
|
*/
|
|
130
|
-
export async function embedBatch(embedder, texts, batchSize = 10, purpose, concurrency = 1, onProgress, retryMax = 3, retryBaseDelayMs = 1000) {
|
|
153
|
+
export async function embedBatch(embedder, texts, batchSize = 10, purpose, concurrency = 1, onProgress, retryMax = 3, retryBaseDelayMs = 1000, maxConsecutiveFailures = DEFAULT_MAX_CONSECUTIVE_BATCH_FAILURES) {
|
|
131
154
|
if (texts.length === 0)
|
|
132
155
|
return [];
|
|
133
156
|
const batches = [];
|
|
@@ -184,8 +207,16 @@ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concu
|
|
|
184
207
|
// counted by the caller. A context-limit rejection carries a permanent 400
|
|
185
208
|
// status, so it needs its own predicate; timeouts are already non-permanent
|
|
186
209
|
// (belt and braces). Genuine permanent errors never split — halving the
|
|
187
|
-
// batch would fail the same way, only slower.
|
|
188
|
-
|
|
210
|
+
// batch would fail the same way, only slower. Neither do gateway errors
|
|
211
|
+
// (502/503/504): they come from the SERVER (down, overloaded, or a context
|
|
212
|
+
// smaller than our chunks), not a bad text, so a smaller batch fails
|
|
213
|
+
// identically. Top-level retry already handles a transient one; a persistent
|
|
214
|
+
// one is left to the circuit breaker below rather than split into an
|
|
215
|
+
// exponential retry tree that looks exactly like a hang.
|
|
216
|
+
const splittable = (!isPermanentError(lastError) ||
|
|
217
|
+
isContextLimitError(lastError) ||
|
|
218
|
+
isTimeoutError(lastError)) &&
|
|
219
|
+
!isGatewayError(lastError);
|
|
189
220
|
if (batchTexts.length > 1 && splittable) {
|
|
190
221
|
const mid = Math.ceil(batchTexts.length / 2);
|
|
191
222
|
appendDebugLog(getBatchLogFilePath(), {
|
|
@@ -203,10 +234,50 @@ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concu
|
|
|
203
234
|
}
|
|
204
235
|
return null;
|
|
205
236
|
}
|
|
237
|
+
// ── Circuit breaker ────────────────────────────────────────────────────
|
|
238
|
+
// A down / overloaded / undersized embedding server fails EVERY batch. Without
|
|
239
|
+
// a break we keep firing batch after batch at it for the whole run — a silent,
|
|
240
|
+
// minutes-long grind indistinguishable from a hang. After this many
|
|
241
|
+
// CONSECUTIVELY failed batches (reset by any success) we stop calling the
|
|
242
|
+
// provider entirely and short-circuit the rest to empty vectors, so the pass
|
|
243
|
+
// fails fast and the pipeline reports the provider as unavailable.
|
|
244
|
+
let consecutiveFailedBatches = 0;
|
|
245
|
+
let circuitTripped = false;
|
|
246
|
+
function recordBatchResult(succeeded) {
|
|
247
|
+
if (succeeded) {
|
|
248
|
+
consecutiveFailedBatches = 0;
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
consecutiveFailedBatches++;
|
|
252
|
+
if (!circuitTripped &&
|
|
253
|
+
maxConsecutiveFailures > 0 &&
|
|
254
|
+
consecutiveFailedBatches >= maxConsecutiveFailures) {
|
|
255
|
+
circuitTripped = true;
|
|
256
|
+
appendDebugLog(getBatchLogFilePath(), {
|
|
257
|
+
scope: "embedder.batch",
|
|
258
|
+
severity: "warn",
|
|
259
|
+
message: `Embedding circuit breaker tripped after ${consecutiveFailedBatches} consecutive failed ` +
|
|
260
|
+
"batches — the embedding server is unavailable, overloaded, or rejecting the input (an " +
|
|
261
|
+
"over-large / too-small context is the usual cause). Stopping further requests; un-embedded " +
|
|
262
|
+
"chunks are kept for the next pass. Fix the provider (e.g. raise its context window) and " +
|
|
263
|
+
"run the index again.",
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
}
|
|
206
267
|
if (concurrency <= 1 || batches.length <= 1) {
|
|
207
268
|
const results = [];
|
|
208
269
|
for (const batch of batches) {
|
|
270
|
+
if (circuitTripped) {
|
|
271
|
+
// Breaker open: emit empty vectors so caller index→vector alignment is
|
|
272
|
+
// preserved, advance progress, and never touch the provider again.
|
|
273
|
+
for (let i = 0; i < batch.texts.length; i++) {
|
|
274
|
+
results.push([]);
|
|
275
|
+
}
|
|
276
|
+
onProgress?.(results.length, texts.length);
|
|
277
|
+
continue;
|
|
278
|
+
}
|
|
209
279
|
const embeddings = await embedWithRetry(batch.texts);
|
|
280
|
+
recordBatchResult(embeddings !== null);
|
|
210
281
|
if (embeddings) {
|
|
211
282
|
results.push(...embeddings);
|
|
212
283
|
}
|
|
@@ -222,7 +293,13 @@ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concu
|
|
|
222
293
|
let completedCount = 0;
|
|
223
294
|
const limit = pLimit(concurrency);
|
|
224
295
|
const batchResults = await Promise.all(batches.map((batch) => limit(async () => {
|
|
296
|
+
if (circuitTripped) {
|
|
297
|
+
completedCount += batch.texts.length;
|
|
298
|
+
onProgress?.(completedCount, texts.length);
|
|
299
|
+
return { index: batch.index, embeddings: batch.texts.map(() => []) };
|
|
300
|
+
}
|
|
225
301
|
const embeddings = await embedWithRetry(batch.texts);
|
|
302
|
+
recordBatchResult(embeddings !== null);
|
|
226
303
|
const flatResult = embeddings ?? batch.texts.map(() => []);
|
|
227
304
|
// Count processed TEXTS (not returned vectors) so failed batches
|
|
228
305
|
// still advance progress towards the total.
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import type { DecisionAnswer, DecisionProvider } from "../core/interfaces.js";
|
|
2
|
+
/** One question in a calibration item. */
|
|
3
|
+
export interface DecideCalibrationQuestion {
|
|
4
|
+
type: "noul" | "choice" | "score";
|
|
5
|
+
instructions: string;
|
|
6
|
+
criteria?: Record<string, string> | string[] | {
|
|
7
|
+
true?: string;
|
|
8
|
+
false?: string;
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
/** One labelled decision task. */
|
|
12
|
+
export interface DecideCalibrationItem {
|
|
13
|
+
id: string;
|
|
14
|
+
category: string;
|
|
15
|
+
state: string;
|
|
16
|
+
question: DecideCalibrationQuestion;
|
|
17
|
+
/**
|
|
18
|
+
* Expected answer: `"true"`/`"false"` for `noul`, an option key for `choice`,
|
|
19
|
+
* a level label for `score`.
|
|
20
|
+
*/
|
|
21
|
+
expected: string;
|
|
22
|
+
note?: string;
|
|
23
|
+
}
|
|
24
|
+
/** Serialized calibration file (`src/eval/decide-labels.json`). */
|
|
25
|
+
export interface DecideCalibrationFile {
|
|
26
|
+
version: number;
|
|
27
|
+
labels: DecideCalibrationItem[];
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Load and validate a decision calibration label file.
|
|
31
|
+
*
|
|
32
|
+
* @param labelsPath - Absolute path to the JSON label file.
|
|
33
|
+
* @returns The validated items.
|
|
34
|
+
* @throws When the file is missing, has no labels, or an item lacks id/state/question/expected.
|
|
35
|
+
*/
|
|
36
|
+
export declare function loadDecideCalibration(labelsPath: string): DecideCalibrationItem[];
|
|
37
|
+
/** One scored calibration item. */
|
|
38
|
+
export interface ScoredDecisionItem {
|
|
39
|
+
item: DecideCalibrationItem;
|
|
40
|
+
/** Normalized prediction (`"true"`/`"false"`, option key, or level label), or undefined. */
|
|
41
|
+
predicted: string | undefined;
|
|
42
|
+
correct: boolean;
|
|
43
|
+
/** Probability assigned to the predicted answer (0.5 fallback when the model reports none). */
|
|
44
|
+
confidence: number;
|
|
45
|
+
/** Probability assigned to the expected answer, when derivable (choice/noul). */
|
|
46
|
+
pExpected: number | null;
|
|
47
|
+
latencyMs: number;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Score one model answer against the expected label.
|
|
51
|
+
*
|
|
52
|
+
* Pure function — no provider needed, which is what makes the math unit-testable.
|
|
53
|
+
*
|
|
54
|
+
* @param item - The labelled item.
|
|
55
|
+
* @param answer - The model's answer (may be undefined when the provider skipped it).
|
|
56
|
+
* @param latencyMs - Measured call latency.
|
|
57
|
+
* @returns Prediction, correctness, prediction confidence, expected-class probability.
|
|
58
|
+
*/
|
|
59
|
+
export declare function scoreDecisionAnswer(item: DecideCalibrationItem, answer: DecisionAnswer | undefined, latencyMs: number): ScoredDecisionItem;
|
|
60
|
+
/** One confidence bin of the reliability diagram. */
|
|
61
|
+
export interface CalibrationBin {
|
|
62
|
+
lo: number;
|
|
63
|
+
hi: number;
|
|
64
|
+
count: number;
|
|
65
|
+
/** Mean confidence of the predictions in this bin. */
|
|
66
|
+
confidence: number;
|
|
67
|
+
/** Share of correct predictions in this bin. */
|
|
68
|
+
accuracy: number;
|
|
69
|
+
}
|
|
70
|
+
/** Aggregate calibration metrics. */
|
|
71
|
+
export interface CalibrationSummary {
|
|
72
|
+
/** Total items. */
|
|
73
|
+
n: number;
|
|
74
|
+
/** Items the model actually answered (prediction present). */
|
|
75
|
+
answered: number;
|
|
76
|
+
accuracy: number;
|
|
77
|
+
/** Expected calibration error over `bins` equal-width confidence bins. */
|
|
78
|
+
ece: number;
|
|
79
|
+
/** Mean squared error of confidence vs. correctness. */
|
|
80
|
+
brier: number;
|
|
81
|
+
meanConfidence: number;
|
|
82
|
+
/** Mean probability assigned to the expected answer (noul/choice only, when derivable). */
|
|
83
|
+
meanPExpected: number | null;
|
|
84
|
+
bins: CalibrationBin[];
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Aggregate scored items into accuracy / ECE / Brier.
|
|
88
|
+
*
|
|
89
|
+
* @param items - Scored items (unanswered ones count against accuracy but not into bins).
|
|
90
|
+
* @param binCount - Number of equal-width confidence bins (default 10).
|
|
91
|
+
* @returns The calibration summary including the reliability bins.
|
|
92
|
+
*/
|
|
93
|
+
export declare function summarizeCalibration(items: ScoredDecisionItem[], binCount?: number): CalibrationSummary;
|
|
94
|
+
/** Options for {@link runDecideCalibration}. */
|
|
95
|
+
export interface DecideCalibrationRunOptions {
|
|
96
|
+
items: DecideCalibrationItem[];
|
|
97
|
+
provider: DecisionProvider;
|
|
98
|
+
/** Progress line callback (defaults to console.log). */
|
|
99
|
+
log?: (line: string) => void;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Run every calibration item through the decision provider and score the answers.
|
|
103
|
+
*
|
|
104
|
+
* @param opts - Items, provider, logger.
|
|
105
|
+
* @returns Scored items and the aggregate summary.
|
|
106
|
+
* @throws When the provider fails for the very first item (e.g. Ollama down) — the
|
|
107
|
+
* caller should surface the provider error instead of a misleading 0% accuracy.
|
|
108
|
+
*/
|
|
109
|
+
export declare function runDecideCalibration(opts: DecideCalibrationRunOptions): Promise<{
|
|
110
|
+
scored: ScoredDecisionItem[];
|
|
111
|
+
summary: CalibrationSummary;
|
|
112
|
+
}>;
|