opencode-rag-plugin 2.3.1 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +71 -8
- package/dist/chunker/skeleton.js +1 -1
- package/dist/cli/commands/decide.d.ts +25 -0
- package/dist/cli/commands/decide.js +160 -0
- package/dist/cli/commands/eval.d.ts +23 -0
- package/dist/cli/commands/eval.js +331 -1
- package/dist/cli/commands/index-command.js +28 -10
- package/dist/cli/commands/index.d.ts +2 -1
- package/dist/cli/commands/index.js +2 -1
- package/dist/cli/commands/init-helpers.d.ts +16 -1
- package/dist/cli/commands/init-helpers.js +84 -2
- package/dist/cli/commands/init.js +29 -4
- package/dist/cli/commands/status.js +14 -8
- package/dist/cli/format.d.ts +50 -0
- package/dist/cli/format.js +122 -0
- package/dist/cli/index.js +4 -1
- package/dist/cli/types.d.ts +2 -0
- package/dist/core/bootstrap.js +1 -0
- package/dist/core/config.d.ts +47 -0
- package/dist/core/config.js +34 -1
- package/dist/core/interfaces.d.ts +69 -0
- package/dist/decision/factory.d.ts +17 -0
- package/dist/decision/factory.js +17 -0
- package/dist/decision/format.d.ts +14 -0
- package/dist/decision/format.js +53 -0
- package/dist/decision/systemone.d.ts +76 -0
- package/dist/decision/systemone.js +234 -0
- package/dist/decision/validate.d.ts +26 -0
- package/dist/decision/validate.js +74 -0
- package/dist/eval/decide-calibration.d.ts +112 -0
- package/dist/eval/decide-calibration.js +185 -0
- package/dist/eval/golden-gate.d.ts +65 -0
- package/dist/eval/golden-gate.js +61 -0
- package/dist/eval/golden-set.d.ts +81 -0
- package/dist/eval/golden-set.js +155 -0
- package/dist/eval/rerank-eval.js +10 -125
- package/dist/index.d.ts +6 -2
- package/dist/index.js +4 -0
- package/dist/indexer/pipeline.d.ts +0 -10
- package/dist/indexer/pipeline.js +1 -0
- package/dist/mcp/handlers.d.ts +17 -1
- package/dist/mcp/handlers.js +30 -0
- package/dist/mcp/server.d.ts +9 -1
- package/dist/mcp/server.js +47 -3
- package/dist/opencode/system-guidance.d.ts +11 -0
- package/dist/opencode/system-guidance.js +53 -0
- package/dist/opencode/tools.d.ts +24 -1
- package/dist/opencode/tools.js +101 -0
- package/dist/plugin.js +21 -3
- package/dist/quirks/quirk-store.d.ts +2 -0
- package/dist/quirks/quirk-store.js +33 -4
- package/dist/tui.js +21 -2
- package/dist/v2/v2-adapter.d.ts +20 -0
- package/dist/v2/v2-adapter.js +36 -8
- package/dist/vectorstore/lancedb.js +54 -5
- package/dist/watcher.d.ts +26 -1
- package/dist/watcher.js +71 -9
- package/dist/web/ui/assets/index-CWcp0jNf.css +2 -0
- package/dist/web/ui/index.html +2 -2
- package/package.json +5 -2
- package/dist/web/ui/assets/index-Dp009QCQ.css +0 -1
- /package/dist/web/ui/assets/{index-C0QXKA-b.js → index-DbXFrn53.js} +0 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Ollama decision-model provider (tev1) using the
|
|
3
|
+
* `/v1/systemone` endpoint.
|
|
4
|
+
*
|
|
5
|
+
* Decision models are NOT chat models: `state` is scored against 1-64 named
|
|
6
|
+
* `questions` and the response carries probabilities per question. The
|
|
7
|
+
* endpoint is root-level (`/v1/systemone`), so the `/api` suffix used by the
|
|
8
|
+
* embedding/description sections is stripped before building the URL.
|
|
9
|
+
*
|
|
10
|
+
* Requires Ollama >= 0.35 and a pulled tev1 model (`ollama pull tev1` or
|
|
11
|
+
* `ollama pull tev1:0.8b`).
|
|
12
|
+
*/
|
|
13
|
+
import type { DecisionProvider, DecisionRequest, DecisionResult } from "../core/interfaces.js";
|
|
14
|
+
import type { DecisionConfig } from "../core/config.js";
|
|
15
|
+
/** Request body size cap imposed by Ollama's decision endpoint (64 KiB). */
|
|
16
|
+
export declare const SYSTEMONE_MAX_REQUEST_BYTES: number;
|
|
17
|
+
/** Maximum number of questions per request (tev1 limit). */
|
|
18
|
+
export declare const SYSTEMONE_MAX_QUESTIONS = 64;
|
|
19
|
+
/** Minimum options/levels for `choice`/`score` questions. */
|
|
20
|
+
export declare const SYSTEMONE_MIN_OPTIONS = 2;
|
|
21
|
+
/** tev1 was trained on up to 24 options; 26 is the API maximum. */
|
|
22
|
+
export declare const SYSTEMONE_MAX_OPTIONS = 24;
|
|
23
|
+
/**
|
|
24
|
+
* Resolve the decision endpoint from a configured base URL.
|
|
25
|
+
*
|
|
26
|
+
* Accepts both root URLs (`http://127.0.0.1:11434`) and the `/api`-suffixed
|
|
27
|
+
* convention used by the embedding/description sections (`…:11434/api`) —
|
|
28
|
+
* `/api` is dropped because decision models are served at `/v1/systemone`.
|
|
29
|
+
*
|
|
30
|
+
* @param baseUrl - Configured Ollama base URL.
|
|
31
|
+
* @returns The absolute `/v1/systemone` URL.
|
|
32
|
+
*/
|
|
33
|
+
export declare function resolveSystemOneUrl(baseUrl: string): string;
|
|
34
|
+
/**
|
|
35
|
+
* Decision provider for Ollama's `/v1/systemone` endpoint (tev1).
|
|
36
|
+
*
|
|
37
|
+
* Retries retryable HTTP statuses and network errors with exponential
|
|
38
|
+
* backoff. The first call may load the model, so configure a generous
|
|
39
|
+
* `timeoutMs` (default 120s) and optionally `keepAlive` to keep it resident.
|
|
40
|
+
*/
|
|
41
|
+
export declare class OllamaDecisionProvider implements DecisionProvider {
|
|
42
|
+
readonly name = "ollama";
|
|
43
|
+
private readonly baseUrl;
|
|
44
|
+
private readonly model;
|
|
45
|
+
private readonly timeoutMs;
|
|
46
|
+
private readonly keepAlive?;
|
|
47
|
+
private readonly proxy?;
|
|
48
|
+
private readonly retryMax;
|
|
49
|
+
private readonly retryBaseDelayMs;
|
|
50
|
+
/**
|
|
51
|
+
* @param config - Decision-model configuration (endpoint, model, timeout, proxy, retries).
|
|
52
|
+
*/
|
|
53
|
+
constructor(config: DecisionConfig);
|
|
54
|
+
/**
|
|
55
|
+
* Run a decision request. One answer is returned per question, in request
|
|
56
|
+
* order; answers missing from the response degrade to bare `{id, type}`.
|
|
57
|
+
*
|
|
58
|
+
* @param request - The state text and questions.
|
|
59
|
+
* @param abort - Optional abort signal.
|
|
60
|
+
* @throws If the endpoint is unreachable, rejects the request (e.g. Ollama
|
|
61
|
+
* < 0.35 or model not pulled), the body exceeds 64 KiB, or the response
|
|
62
|
+
* carries no answers.
|
|
63
|
+
*/
|
|
64
|
+
decide(request: DecisionRequest, abort?: AbortSignal): Promise<DecisionResult>;
|
|
65
|
+
/**
|
|
66
|
+
* Map an HTTP failure to an actionable message.
|
|
67
|
+
*
|
|
68
|
+
* 404 is overloaded: Ollama < 0.35 returns the Go default "404 page not
|
|
69
|
+
* found" (route missing), while a server that has the decision route but
|
|
70
|
+
* not the model returns `model "X" not found, try pulling it first`. Split
|
|
71
|
+
* the two so the fix is obvious.
|
|
72
|
+
*/
|
|
73
|
+
private describeHttpError;
|
|
74
|
+
/** Exponential backoff between retries. */
|
|
75
|
+
private sleep;
|
|
76
|
+
}
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
import { normalizeKeepAlive } from "../core/ollama.js";
|
|
2
|
+
import { postJson } from "../embedder/http.js";
|
|
3
|
+
/** HTTP status codes that are safe to retry on. */
|
|
4
|
+
const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
|
|
5
|
+
/** Request body size cap imposed by Ollama's decision endpoint (64 KiB). */
|
|
6
|
+
export const SYSTEMONE_MAX_REQUEST_BYTES = 64 * 1024;
|
|
7
|
+
/** Maximum number of questions per request (tev1 limit). */
|
|
8
|
+
export const SYSTEMONE_MAX_QUESTIONS = 64;
|
|
9
|
+
/** Minimum options/levels for `choice`/`score` questions. */
|
|
10
|
+
export const SYSTEMONE_MIN_OPTIONS = 2;
|
|
11
|
+
/** tev1 was trained on up to 24 options; 26 is the API maximum. */
|
|
12
|
+
export const SYSTEMONE_MAX_OPTIONS = 24;
|
|
13
|
+
/**
|
|
14
|
+
* Resolve the decision endpoint from a configured base URL.
|
|
15
|
+
*
|
|
16
|
+
* Accepts both root URLs (`http://127.0.0.1:11434`) and the `/api`-suffixed
|
|
17
|
+
* convention used by the embedding/description sections (`…:11434/api`) —
|
|
18
|
+
* `/api` is dropped because decision models are served at `/v1/systemone`.
|
|
19
|
+
*
|
|
20
|
+
* @param baseUrl - Configured Ollama base URL.
|
|
21
|
+
* @returns The absolute `/v1/systemone` URL.
|
|
22
|
+
*/
|
|
23
|
+
export function resolveSystemOneUrl(baseUrl) {
|
|
24
|
+
const trimmed = baseUrl.replace(/\/+$/, "");
|
|
25
|
+
const root = trimmed.endsWith("/api") ? trimmed.slice(0, -"/api".length) : trimmed;
|
|
26
|
+
return `${root}/v1/systemone`;
|
|
27
|
+
}
|
|
28
|
+
/** Coerce a number-or-numeric-string to a finite number, else undefined. */
|
|
29
|
+
function toNumber(value) {
|
|
30
|
+
if (typeof value === "number" && Number.isFinite(value))
|
|
31
|
+
return value;
|
|
32
|
+
if (typeof value === "string" && value.trim() !== "") {
|
|
33
|
+
const n = Number(value);
|
|
34
|
+
if (Number.isFinite(n))
|
|
35
|
+
return n;
|
|
36
|
+
}
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
39
|
+
/** Parse one raw answer object into a typed {@link DecisionAnswer}. */
|
|
40
|
+
function parseAnswer(id, question, raw) {
|
|
41
|
+
const record = raw !== null && typeof raw === "object" ? raw : {};
|
|
42
|
+
const answer = { id, type: question.type };
|
|
43
|
+
if (typeof record.choice === "string" && record.choice.length > 0) {
|
|
44
|
+
answer.choice = record.choice;
|
|
45
|
+
}
|
|
46
|
+
const noul = record.noul;
|
|
47
|
+
if (noul !== null && typeof noul === "object") {
|
|
48
|
+
// Defensive: some embeddings of the API wrap the probability.
|
|
49
|
+
const wrapped = noul;
|
|
50
|
+
const p = toNumber(wrapped.probability) ?? toNumber(wrapped.noul);
|
|
51
|
+
if (p !== undefined)
|
|
52
|
+
answer.noul = p;
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
const p = toNumber(noul);
|
|
56
|
+
if (p !== undefined)
|
|
57
|
+
answer.noul = p;
|
|
58
|
+
}
|
|
59
|
+
const score = toNumber(record.score);
|
|
60
|
+
if (score !== undefined)
|
|
61
|
+
answer.score = score;
|
|
62
|
+
if (Array.isArray(record.legend)) {
|
|
63
|
+
const legend = record.legend.filter((entry) => typeof entry === "string");
|
|
64
|
+
if (legend.length > 0)
|
|
65
|
+
answer.legend = legend;
|
|
66
|
+
}
|
|
67
|
+
const probabilities = record.probabilities;
|
|
68
|
+
if (Array.isArray(probabilities)) {
|
|
69
|
+
const values = probabilities
|
|
70
|
+
.map(toNumber)
|
|
71
|
+
.filter((value) => value !== undefined);
|
|
72
|
+
if (values.length > 0)
|
|
73
|
+
answer.probabilities = values;
|
|
74
|
+
}
|
|
75
|
+
else if (probabilities !== null && typeof probabilities === "object") {
|
|
76
|
+
const map = {};
|
|
77
|
+
for (const [key, value] of Object.entries(probabilities)) {
|
|
78
|
+
const n = toNumber(value);
|
|
79
|
+
if (n !== undefined)
|
|
80
|
+
map[key] = n;
|
|
81
|
+
}
|
|
82
|
+
if (Object.keys(map).length > 0)
|
|
83
|
+
answer.probabilities = map;
|
|
84
|
+
}
|
|
85
|
+
const confidence = toNumber(record.confidence);
|
|
86
|
+
if (confidence !== undefined)
|
|
87
|
+
answer.confidence = confidence;
|
|
88
|
+
return answer;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Build the `questions` entry for the request body. `criteria` is passed
|
|
92
|
+
* through as-is; validation happens before the provider is called.
|
|
93
|
+
*/
|
|
94
|
+
function buildQuestionPayload(question) {
|
|
95
|
+
const payload = {
|
|
96
|
+
type: question.type,
|
|
97
|
+
instructions: question.instructions,
|
|
98
|
+
};
|
|
99
|
+
if (question.criteria !== undefined) {
|
|
100
|
+
payload.criteria = question.criteria;
|
|
101
|
+
}
|
|
102
|
+
return payload;
|
|
103
|
+
}
|
|
104
|
+
/** Truncate provider error bodies so logs stay readable. */
|
|
105
|
+
function truncate(text, max = 300) {
|
|
106
|
+
const trimmed = text.trim();
|
|
107
|
+
return trimmed.length > max ? `${trimmed.slice(0, max)}…` : trimmed;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Decision provider for Ollama's `/v1/systemone` endpoint (tev1).
|
|
111
|
+
*
|
|
112
|
+
* Retries retryable HTTP statuses and network errors with exponential
|
|
113
|
+
* backoff. The first call may load the model, so configure a generous
|
|
114
|
+
* `timeoutMs` (default 120s) and optionally `keepAlive` to keep it resident.
|
|
115
|
+
*/
|
|
116
|
+
export class OllamaDecisionProvider {
|
|
117
|
+
name = "ollama";
|
|
118
|
+
baseUrl;
|
|
119
|
+
model;
|
|
120
|
+
timeoutMs;
|
|
121
|
+
keepAlive;
|
|
122
|
+
proxy;
|
|
123
|
+
retryMax;
|
|
124
|
+
retryBaseDelayMs;
|
|
125
|
+
/**
|
|
126
|
+
* @param config - Decision-model configuration (endpoint, model, timeout, proxy, retries).
|
|
127
|
+
*/
|
|
128
|
+
constructor(config) {
|
|
129
|
+
this.baseUrl = config.baseUrl;
|
|
130
|
+
this.model = config.model;
|
|
131
|
+
this.timeoutMs = config.timeoutMs > 0 ? config.timeoutMs : 120000;
|
|
132
|
+
this.keepAlive = config.keepAlive;
|
|
133
|
+
this.proxy = config.proxy;
|
|
134
|
+
this.retryMax = config.retryMax ?? 2;
|
|
135
|
+
this.retryBaseDelayMs = config.retryBaseDelayMs ?? 500;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Run a decision request. One answer is returned per question, in request
|
|
139
|
+
* order; answers missing from the response degrade to bare `{id, type}`.
|
|
140
|
+
*
|
|
141
|
+
* @param request - The state text and questions.
|
|
142
|
+
* @param abort - Optional abort signal.
|
|
143
|
+
* @throws If the endpoint is unreachable, rejects the request (e.g. Ollama
|
|
144
|
+
* < 0.35 or model not pulled), the body exceeds 64 KiB, or the response
|
|
145
|
+
* carries no answers.
|
|
146
|
+
*/
|
|
147
|
+
async decide(request, abort) {
|
|
148
|
+
const questions = {};
|
|
149
|
+
for (const question of request.questions) {
|
|
150
|
+
questions[question.id] = buildQuestionPayload(question);
|
|
151
|
+
}
|
|
152
|
+
const body = {
|
|
153
|
+
model: this.model,
|
|
154
|
+
state: request.state,
|
|
155
|
+
questions,
|
|
156
|
+
};
|
|
157
|
+
const keepAlive = normalizeKeepAlive(this.keepAlive);
|
|
158
|
+
if (keepAlive !== undefined)
|
|
159
|
+
body.keep_alive = keepAlive;
|
|
160
|
+
const serialized = JSON.stringify(body);
|
|
161
|
+
if (Buffer.byteLength(serialized, "utf-8") > SYSTEMONE_MAX_REQUEST_BYTES) {
|
|
162
|
+
throw new Error(`Decision request exceeds Ollama's 64 KiB body limit (${Buffer.byteLength(serialized, "utf-8")} bytes). ` +
|
|
163
|
+
`Shorten \`state\` or ask fewer questions.`);
|
|
164
|
+
}
|
|
165
|
+
const url = resolveSystemOneUrl(this.baseUrl);
|
|
166
|
+
let lastError;
|
|
167
|
+
for (let attempt = 0; attempt <= this.retryMax; attempt++) {
|
|
168
|
+
let response;
|
|
169
|
+
try {
|
|
170
|
+
response = await postJson(url, body, {}, this.timeoutMs, this.proxy, abort);
|
|
171
|
+
}
|
|
172
|
+
catch (err) {
|
|
173
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
174
|
+
if (attempt < this.retryMax) {
|
|
175
|
+
await this.sleep(attempt);
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
throw lastError;
|
|
179
|
+
}
|
|
180
|
+
if (!response.ok) {
|
|
181
|
+
const text = await response.text().catch(() => "");
|
|
182
|
+
const error = new Error(this.describeHttpError(response.status, text));
|
|
183
|
+
if (RETRYABLE_STATUSES.has(response.status) && attempt < this.retryMax) {
|
|
184
|
+
lastError = error;
|
|
185
|
+
await this.sleep(attempt);
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
throw error;
|
|
189
|
+
}
|
|
190
|
+
const json = (await response.json());
|
|
191
|
+
const rawAnswers = json?.answers;
|
|
192
|
+
if (rawAnswers === null || typeof rawAnswers !== "object" || Array.isArray(rawAnswers)) {
|
|
193
|
+
throw new Error(`Decision model returned no answers for model "${this.model}": ${truncate(JSON.stringify(json))}`);
|
|
194
|
+
}
|
|
195
|
+
const answerMap = rawAnswers;
|
|
196
|
+
const answers = [];
|
|
197
|
+
for (const question of request.questions) {
|
|
198
|
+
answers.push(parseAnswer(question.id, question, answerMap[question.id]));
|
|
199
|
+
}
|
|
200
|
+
return { answers, raw: json };
|
|
201
|
+
}
|
|
202
|
+
throw lastError ?? new Error("Decision request failed");
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Map an HTTP failure to an actionable message.
|
|
206
|
+
*
|
|
207
|
+
* 404 is overloaded: Ollama < 0.35 returns the Go default "404 page not
|
|
208
|
+
* found" (route missing), while a server that has the decision route but
|
|
209
|
+
* not the model returns `model "X" not found, try pulling it first`. Split
|
|
210
|
+
* the two so the fix is obvious.
|
|
211
|
+
*/
|
|
212
|
+
describeHttpError(status, text) {
|
|
213
|
+
const detail = truncate(text);
|
|
214
|
+
if (status === 404) {
|
|
215
|
+
const lower = text.toLowerCase();
|
|
216
|
+
const modelLower = this.model.toLowerCase();
|
|
217
|
+
if (lower.includes("try pulling") || (lower.includes("not found") && lower.includes(modelLower))) {
|
|
218
|
+
return `Decision model "${this.model}" is not pulled (404) — run \`ollama pull ${this.model}\`. ${detail}`;
|
|
219
|
+
}
|
|
220
|
+
if (lower.includes("page not found") || detail.length === 0) {
|
|
221
|
+
return `Decision endpoint not found (404) — tev1 requires Ollama >= 0.35 (\`POST /v1/systemone\`). ${detail}`;
|
|
222
|
+
}
|
|
223
|
+
return (`Decision request not found (404) — tev1 requires Ollama >= 0.35 and the model must be pulled ` +
|
|
224
|
+
`(\`ollama pull ${this.model}\`). ${detail}`);
|
|
225
|
+
}
|
|
226
|
+
return `Ollama decision request failed (${status}): ${detail}`;
|
|
227
|
+
}
|
|
228
|
+
/** Exponential backoff between retries. */
|
|
229
|
+
sleep(attempt) {
|
|
230
|
+
const delay = this.retryBaseDelayMs * Math.pow(2, attempt);
|
|
231
|
+
return new Promise((resolve) => setTimeout(resolve, delay));
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
//# sourceMappingURL=systemone.js.map
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Input validation for decision requests, shared by the
|
|
3
|
+
* OpenCode plugin tool, the MCP handler, and the CLI command.
|
|
4
|
+
*
|
|
5
|
+
* The tev1 API is permissive about malformed questions (it answers with
|
|
6
|
+
* garbage rather than erroring), so validating up front keeps agent calls
|
|
7
|
+
* debuggable — especially option counts, where tev1 was trained on 2-24
|
|
8
|
+
* options.
|
|
9
|
+
*/
|
|
10
|
+
import type { DecisionQuestion } from "../core/interfaces.js";
|
|
11
|
+
/** Default `state` character limit (~2k-token decision context). */
|
|
12
|
+
export declare const DEFAULT_MAX_STATE_CHARS = 8000;
|
|
13
|
+
/** A decision request as received from a tool/MCP/CLI caller. */
|
|
14
|
+
export interface DecisionValidationInput {
|
|
15
|
+
state: string;
|
|
16
|
+
questions: DecisionQuestion[];
|
|
17
|
+
/** Maximum characters accepted for `state`. */
|
|
18
|
+
maxStateChars?: number;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Validate a decision request before it is sent to the model.
|
|
22
|
+
*
|
|
23
|
+
* @param input - State text, questions, and the configured state limit.
|
|
24
|
+
* @returns An actionable error message, or `undefined` when the request is valid.
|
|
25
|
+
*/
|
|
26
|
+
export declare function validateDecisionRequest(input: DecisionValidationInput): string | undefined;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { SYSTEMONE_MAX_OPTIONS, SYSTEMONE_MAX_QUESTIONS, SYSTEMONE_MIN_OPTIONS } from "./systemone.js";
|
|
2
|
+
/** Default `state` character limit (~2k-token decision context). */
|
|
3
|
+
export const DEFAULT_MAX_STATE_CHARS = 8000;
|
|
4
|
+
/** Count the options of a `choice` question (object criteria or array fallback). */
|
|
5
|
+
function countChoiceOptions(question) {
|
|
6
|
+
if (Array.isArray(question.criteria))
|
|
7
|
+
return question.criteria.length;
|
|
8
|
+
if (question.criteria !== null && typeof question.criteria === "object") {
|
|
9
|
+
return Object.keys(question.criteria).length;
|
|
10
|
+
}
|
|
11
|
+
return 0;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Validate a decision request before it is sent to the model.
|
|
15
|
+
*
|
|
16
|
+
* @param input - State text, questions, and the configured state limit.
|
|
17
|
+
* @returns An actionable error message, or `undefined` when the request is valid.
|
|
18
|
+
*/
|
|
19
|
+
export function validateDecisionRequest(input) {
|
|
20
|
+
const state = typeof input.state === "string" ? input.state : "";
|
|
21
|
+
if (state.trim().length === 0) {
|
|
22
|
+
return "`state` must not be empty.";
|
|
23
|
+
}
|
|
24
|
+
const maxStateChars = input.maxStateChars && input.maxStateChars > 0 ? input.maxStateChars : DEFAULT_MAX_STATE_CHARS;
|
|
25
|
+
if (state.length > maxStateChars) {
|
|
26
|
+
return (`\`state\` is ${state.length} characters; the configured limit is ${maxStateChars} ` +
|
|
27
|
+
"(decision models have a ~2k-token context). Shorten `state` to the relevant excerpt.");
|
|
28
|
+
}
|
|
29
|
+
const questions = Array.isArray(input.questions) ? input.questions : [];
|
|
30
|
+
if (questions.length < 1) {
|
|
31
|
+
return "At least one question is required.";
|
|
32
|
+
}
|
|
33
|
+
if (questions.length > SYSTEMONE_MAX_QUESTIONS) {
|
|
34
|
+
return `At most ${SYSTEMONE_MAX_QUESTIONS} questions per call (got ${questions.length}).`;
|
|
35
|
+
}
|
|
36
|
+
const seen = new Set();
|
|
37
|
+
for (const question of questions) {
|
|
38
|
+
if (typeof question.id !== "string" || question.id.trim().length === 0) {
|
|
39
|
+
return "Every question needs a non-empty `id`.";
|
|
40
|
+
}
|
|
41
|
+
if (seen.has(question.id)) {
|
|
42
|
+
return `Duplicate question id "${question.id}" — ids must be unique.`;
|
|
43
|
+
}
|
|
44
|
+
seen.add(question.id);
|
|
45
|
+
if (typeof question.instructions !== "string" || question.instructions.trim().length === 0) {
|
|
46
|
+
return `Question "${question.id}" needs non-empty \`instructions\`.`;
|
|
47
|
+
}
|
|
48
|
+
if (question.type === "choice") {
|
|
49
|
+
const optionCount = countChoiceOptions(question);
|
|
50
|
+
if (optionCount < SYSTEMONE_MIN_OPTIONS || optionCount > SYSTEMONE_MAX_OPTIONS) {
|
|
51
|
+
return (`Question "${question.id}": \`choice\` requires 2-24 options via \`criteria\` ` +
|
|
52
|
+
`(an object mapping option → description); got ${optionCount}. ` +
|
|
53
|
+
`Add a "none" option when no listed option may fit.`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
else if (question.type === "score") {
|
|
57
|
+
const levels = Array.isArray(question.criteria) ? question.criteria : [];
|
|
58
|
+
if (levels.length < SYSTEMONE_MIN_OPTIONS || levels.length > SYSTEMONE_MAX_OPTIONS) {
|
|
59
|
+
return (`Question "${question.id}": \`score\` requires \`criteria\` as an array of ` +
|
|
60
|
+
`2-24 level descriptions, lowest first (got ${levels.length}).`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
else if (question.type === "noul") {
|
|
64
|
+
if (Array.isArray(question.criteria)) {
|
|
65
|
+
return `Question "${question.id}": \`noul\` criteria must be an object like { "true": "...", "false": "..." }, not an array.`;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
return `Question "${question.id}": unknown type "${String(question.type)}" — expected "choice", "noul", or "score".`;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return undefined;
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=validate.js.map
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import type { DecisionAnswer, DecisionProvider } from "../core/interfaces.js";
|
|
2
|
+
/** One question in a calibration item. */
|
|
3
|
+
export interface DecideCalibrationQuestion {
|
|
4
|
+
type: "noul" | "choice" | "score";
|
|
5
|
+
instructions: string;
|
|
6
|
+
criteria?: Record<string, string> | string[] | {
|
|
7
|
+
true?: string;
|
|
8
|
+
false?: string;
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
/** One labelled decision task. */
|
|
12
|
+
export interface DecideCalibrationItem {
|
|
13
|
+
id: string;
|
|
14
|
+
category: string;
|
|
15
|
+
state: string;
|
|
16
|
+
question: DecideCalibrationQuestion;
|
|
17
|
+
/**
|
|
18
|
+
* Expected answer: `"true"`/`"false"` for `noul`, an option key for `choice`,
|
|
19
|
+
* a level label for `score`.
|
|
20
|
+
*/
|
|
21
|
+
expected: string;
|
|
22
|
+
note?: string;
|
|
23
|
+
}
|
|
24
|
+
/** Serialized calibration file (`src/eval/decide-labels.json`). */
|
|
25
|
+
export interface DecideCalibrationFile {
|
|
26
|
+
version: number;
|
|
27
|
+
labels: DecideCalibrationItem[];
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Load and validate a decision calibration label file.
|
|
31
|
+
*
|
|
32
|
+
* @param labelsPath - Absolute path to the JSON label file.
|
|
33
|
+
* @returns The validated items.
|
|
34
|
+
* @throws When the file is missing, has no labels, or an item lacks id/state/question/expected.
|
|
35
|
+
*/
|
|
36
|
+
export declare function loadDecideCalibration(labelsPath: string): DecideCalibrationItem[];
|
|
37
|
+
/** One scored calibration item. */
|
|
38
|
+
export interface ScoredDecisionItem {
|
|
39
|
+
item: DecideCalibrationItem;
|
|
40
|
+
/** Normalized prediction (`"true"`/`"false"`, option key, or level label), or undefined. */
|
|
41
|
+
predicted: string | undefined;
|
|
42
|
+
correct: boolean;
|
|
43
|
+
/** Probability assigned to the predicted answer (0.5 fallback when the model reports none). */
|
|
44
|
+
confidence: number;
|
|
45
|
+
/** Probability assigned to the expected answer, when derivable (choice/noul). */
|
|
46
|
+
pExpected: number | null;
|
|
47
|
+
latencyMs: number;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Score one model answer against the expected label.
|
|
51
|
+
*
|
|
52
|
+
* Pure function — no provider needed, which is what makes the math unit-testable.
|
|
53
|
+
*
|
|
54
|
+
* @param item - The labelled item.
|
|
55
|
+
* @param answer - The model's answer (may be undefined when the provider skipped it).
|
|
56
|
+
* @param latencyMs - Measured call latency.
|
|
57
|
+
* @returns Prediction, correctness, prediction confidence, expected-class probability.
|
|
58
|
+
*/
|
|
59
|
+
export declare function scoreDecisionAnswer(item: DecideCalibrationItem, answer: DecisionAnswer | undefined, latencyMs: number): ScoredDecisionItem;
|
|
60
|
+
/** One confidence bin of the reliability diagram. */
|
|
61
|
+
export interface CalibrationBin {
|
|
62
|
+
lo: number;
|
|
63
|
+
hi: number;
|
|
64
|
+
count: number;
|
|
65
|
+
/** Mean confidence of the predictions in this bin. */
|
|
66
|
+
confidence: number;
|
|
67
|
+
/** Share of correct predictions in this bin. */
|
|
68
|
+
accuracy: number;
|
|
69
|
+
}
|
|
70
|
+
/** Aggregate calibration metrics. */
|
|
71
|
+
export interface CalibrationSummary {
|
|
72
|
+
/** Total items. */
|
|
73
|
+
n: number;
|
|
74
|
+
/** Items the model actually answered (prediction present). */
|
|
75
|
+
answered: number;
|
|
76
|
+
accuracy: number;
|
|
77
|
+
/** Expected calibration error over `bins` equal-width confidence bins. */
|
|
78
|
+
ece: number;
|
|
79
|
+
/** Mean squared error of confidence vs. correctness. */
|
|
80
|
+
brier: number;
|
|
81
|
+
meanConfidence: number;
|
|
82
|
+
/** Mean probability assigned to the expected answer (noul/choice only, when derivable). */
|
|
83
|
+
meanPExpected: number | null;
|
|
84
|
+
bins: CalibrationBin[];
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Aggregate scored items into accuracy / ECE / Brier.
|
|
88
|
+
*
|
|
89
|
+
* @param items - Scored items (unanswered ones count against accuracy but not into bins).
|
|
90
|
+
* @param binCount - Number of equal-width confidence bins (default 10).
|
|
91
|
+
* @returns The calibration summary including the reliability bins.
|
|
92
|
+
*/
|
|
93
|
+
export declare function summarizeCalibration(items: ScoredDecisionItem[], binCount?: number): CalibrationSummary;
|
|
94
|
+
/** Options for {@link runDecideCalibration}. */
|
|
95
|
+
export interface DecideCalibrationRunOptions {
|
|
96
|
+
items: DecideCalibrationItem[];
|
|
97
|
+
provider: DecisionProvider;
|
|
98
|
+
/** Progress line callback (defaults to console.log). */
|
|
99
|
+
log?: (line: string) => void;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Run every calibration item through the decision provider and score the answers.
|
|
103
|
+
*
|
|
104
|
+
* @param opts - Items, provider, logger.
|
|
105
|
+
* @returns Scored items and the aggregate summary.
|
|
106
|
+
* @throws When the provider fails for the very first item (e.g. Ollama down) — the
|
|
107
|
+
* caller should surface the provider error instead of a misleading 0% accuracy.
|
|
108
|
+
*/
|
|
109
|
+
export declare function runDecideCalibration(opts: DecideCalibrationRunOptions): Promise<{
|
|
110
|
+
scored: ScoredDecisionItem[];
|
|
111
|
+
summary: CalibrationSummary;
|
|
112
|
+
}>;
|