modelrig 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +125 -0
- package/README.md +54 -4
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +36 -7
- package/dist/adapters/attempt-meta-map.d.ts +63 -0
- package/dist/adapters/attempt-meta-map.d.ts.map +1 -0
- package/dist/adapters/attempt-meta-map.js +154 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +25 -2
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +29 -2
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +25 -2
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +25 -2
- package/dist/adapters/openai-stream.d.ts +2 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -1
- package/dist/adapters/openai-stream.js +23 -3
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +36 -6
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +24 -5
- package/dist/artifacts/classification.d.ts +27 -0
- package/dist/artifacts/classification.d.ts.map +1 -0
- package/dist/artifacts/classification.js +38 -0
- package/dist/artifacts/index.d.ts +3 -1
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +10 -1
- package/dist/artifacts/run-context.d.ts +25 -3
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +61 -0
- package/dist/artifacts/scrub.d.ts +26 -0
- package/dist/artifacts/scrub.d.ts.map +1 -1
- package/dist/artifacts/scrub.js +53 -0
- package/dist/artifacts/step.d.ts +9 -0
- package/dist/artifacts/step.d.ts.map +1 -1
- package/dist/artifacts/step.js +9 -0
- package/dist/bakeoff/engine.d.ts +46 -4
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +269 -19
- package/dist/bakeoff/eval-cases.d.ts +5 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -1
- package/dist/bakeoff/eval-cases.js +3 -0
- package/dist/bakeoff/judge.d.ts +39 -0
- package/dist/bakeoff/judge.d.ts.map +1 -0
- package/dist/bakeoff/judge.js +102 -0
- package/dist/bakeoff/quality.d.ts +44 -0
- package/dist/bakeoff/quality.d.ts.map +1 -0
- package/dist/bakeoff/quality.js +126 -0
- package/dist/bakeoff/reference.d.ts +34 -0
- package/dist/bakeoff/reference.d.ts.map +1 -0
- package/dist/bakeoff/reference.js +68 -0
- package/dist/bakeoff/replay.d.ts +25 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +48 -1
- package/dist/bakeoff/report.d.ts +8 -1
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +83 -12
- package/dist/bakeoff/samples.d.ts +35 -2
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +67 -6
- package/dist/bakeoff/stats.d.ts +17 -1
- package/dist/bakeoff/stats.d.ts.map +1 -1
- package/dist/bakeoff/stats.js +67 -0
- package/dist/bakeoff/types.d.ts +76 -5
- package/dist/bakeoff/types.d.ts.map +1 -1
- package/dist/bakeoff/types.js +4 -3
- package/dist/cli-query.d.ts +14 -0
- package/dist/cli-query.d.ts.map +1 -0
- package/dist/cli-query.js +59 -0
- package/dist/cli.d.ts +2 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +26 -23
- package/dist/config.d.ts +6 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +14 -0
- package/dist/detect/detector.d.ts +82 -0
- package/dist/detect/detector.d.ts.map +1 -0
- package/dist/detect/detector.js +23 -0
- package/dist/index.d.ts +19 -14
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +38 -11
- package/dist/query/analytics.d.ts +84 -0
- package/dist/query/analytics.d.ts.map +1 -0
- package/dist/query/analytics.js +280 -0
- package/dist/query/columns.d.ts +59 -0
- package/dist/query/columns.d.ts.map +1 -0
- package/dist/query/columns.js +89 -0
- package/dist/query/grammar.d.ts +74 -7
- package/dist/query/grammar.d.ts.map +1 -1
- package/dist/query/grammar.js +179 -71
- package/dist/query/saved.d.ts +26 -0
- package/dist/query/saved.d.ts.map +1 -0
- package/dist/query/saved.js +51 -0
- package/dist/routes/loader.js +15 -10
- package/dist/routes/objective-loader.d.ts +35 -0
- package/dist/routes/objective-loader.d.ts.map +1 -0
- package/dist/routes/objective-loader.js +286 -0
- package/dist/run/accept.d.ts +43 -0
- package/dist/run/accept.d.ts.map +1 -0
- package/dist/run/accept.js +65 -0
- package/dist/run/cost.d.ts +13 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +27 -0
- package/dist/run/dispatch.d.ts +24 -1
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +19 -0
- package/dist/run/ladder-key.d.ts +4 -0
- package/dist/run/ladder-key.d.ts.map +1 -1
- package/dist/run/ladder-key.js +3 -0
- package/dist/run/quality-gate.d.ts +39 -0
- package/dist/run/quality-gate.d.ts.map +1 -0
- package/dist/run/quality-gate.js +70 -0
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +22 -3
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +22 -1
- package/dist/run/routing-docs.d.ts.map +1 -1
- package/dist/run/routing-docs.js +11 -4
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +46 -39
- package/dist/settings/index.d.ts +1 -1
- package/dist/settings/index.d.ts.map +1 -1
- package/dist/settings/index.js +2 -1
- package/dist/settings/task-settings.d.ts +10 -0
- package/dist/settings/task-settings.d.ts.map +1 -1
- package/dist/settings/task-settings.js +17 -0
- package/dist/telemetry/attempt-meta.d.ts +178 -0
- package/dist/telemetry/attempt-meta.d.ts.map +1 -0
- package/dist/telemetry/attempt-meta.js +319 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +6 -0
- package/dist/telemetry/ingest-protocol.d.ts +11 -0
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +3 -0
- package/dist/telemetry/otel-map.d.ts +10 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -1
- package/dist/telemetry/otel-map.js +64 -0
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +15 -0
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +25 -7
- package/dist/types/failures-teaching.d.ts.map +1 -1
- package/dist/types/failures-teaching.js +20 -0
- package/dist/types/failures.d.ts +1 -1
- package/dist/types/failures.d.ts.map +1 -1
- package/dist/types/failures.js +1 -0
- package/dist/types/provider.d.ts +9 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/quality.d.ts +41 -0
- package/dist/types/quality.d.ts.map +1 -0
- package/dist/types/quality.js +12 -0
- package/dist/types/route.d.ts +89 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +13 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +10 -0
- package/package.json +1 -1
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Route-bundle parsing for the RB-1 regret-bounded objective (near-optimal spec
|
|
4
|
+
* §3). Extracted from loader.ts (kept it under the 700-LOC cap). Each parser
|
|
5
|
+
* pushes teaching problems into the shared `problems` array — a non-empty array
|
|
6
|
+
* fails the bundle load, exactly like every other loader field. Pure functions,
|
|
7
|
+
* unit-tested directly.
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.parseRb1Fields = parseRb1Fields;
|
|
11
|
+
exports.parseObjective = parseObjective;
|
|
12
|
+
exports.parseBakeoffConfig = parseBakeoffConfig;
|
|
13
|
+
exports.parseValueChecks = parseValueChecks;
|
|
14
|
+
exports.missingBoundWarning = missingBoundWarning;
|
|
15
|
+
/** All three RB-1 route fields parsed in one call (keeps loader.ts under the
|
|
16
|
+
* 700-LOC cap): objective, bakeoff (reference arm + judge), value_checks. Pushes
|
|
17
|
+
* any problems into the shared array; emits the missing-bound advisory. */
|
|
18
|
+
function parseRb1Fields(doc, route, candidates, problems) {
|
|
19
|
+
const objective = parseObjective(doc.objective, problems);
|
|
20
|
+
const bakeoff = parseBakeoffConfig(doc.bakeoff, candidates, problems);
|
|
21
|
+
const valueChecks = parseValueChecks(doc.value_checks, problems);
|
|
22
|
+
// Only nag routes that OPT INTO RB-1 (declare objective/bakeoff) but omit a
|
|
23
|
+
// bound — a legacy route with neither block is not using regret bake-offs.
|
|
24
|
+
if (objective !== undefined || bakeoff !== undefined) {
|
|
25
|
+
const warning = missingBoundWarning(route, objective);
|
|
26
|
+
if (warning !== null)
|
|
27
|
+
console.warn(`modelrig: ${warning}`);
|
|
28
|
+
}
|
|
29
|
+
return {
|
|
30
|
+
...(objective !== undefined ? { objective } : {}),
|
|
31
|
+
...(bakeoff !== undefined ? { bakeoff } : {}),
|
|
32
|
+
...(valueChecks !== undefined ? { valueChecks } : {}),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
const OBJECTIVE_MODES = new Set([
|
|
36
|
+
"quality-first",
|
|
37
|
+
"balanced",
|
|
38
|
+
"cost-first",
|
|
39
|
+
]);
|
|
40
|
+
function isRecord(v) {
|
|
41
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
42
|
+
}
|
|
43
|
+
function parsePriceCeiling(raw, field, problems) {
|
|
44
|
+
if (raw === undefined || raw === null)
|
|
45
|
+
return undefined;
|
|
46
|
+
if (!isRecord(raw)) {
|
|
47
|
+
problems.push(`${field} must be an object { usd_per_1k } or { usd_per_run }`);
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
const per1k = raw.usd_per_1k;
|
|
51
|
+
const perRun = raw.usd_per_run;
|
|
52
|
+
if (per1k !== undefined && perRun !== undefined) {
|
|
53
|
+
problems.push(`${field} sets both usd_per_1k and usd_per_run — pick one unit`);
|
|
54
|
+
return undefined;
|
|
55
|
+
}
|
|
56
|
+
if (per1k === undefined && perRun === undefined) {
|
|
57
|
+
problems.push(`${field} must set usd_per_1k or usd_per_run`);
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
const value = per1k ?? perRun;
|
|
61
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
|
62
|
+
problems.push(`${field} value must be a positive number`);
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
return Object.freeze(per1k !== undefined ? { usdPer1k: value } : { usdPerRun: value });
|
|
66
|
+
}
|
|
67
|
+
/** Parse the `objective:` block. Absent ⇒ undefined (the engine defaults to
|
|
68
|
+
* quality-first with no bound: regret shown, nothing proposed). */
|
|
69
|
+
function parseObjective(raw, problems) {
|
|
70
|
+
if (raw === undefined || raw === null)
|
|
71
|
+
return undefined;
|
|
72
|
+
if (!isRecord(raw)) {
|
|
73
|
+
problems.push("objective must be an object");
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
const mode = raw.mode ?? "quality-first";
|
|
77
|
+
if (typeof mode !== "string" || !OBJECTIVE_MODES.has(mode)) {
|
|
78
|
+
problems.push(`objective.mode "${String(mode)}" is not one of [${[...OBJECTIVE_MODES].join(", ")}]`);
|
|
79
|
+
return undefined;
|
|
80
|
+
}
|
|
81
|
+
let maxRegret;
|
|
82
|
+
if (raw.max_regret !== undefined) {
|
|
83
|
+
if (typeof raw.max_regret !== "number" || !(raw.max_regret >= 0 && raw.max_regret <= 1)) {
|
|
84
|
+
problems.push("objective.max_regret must be a number in [0, 1] (quality points)");
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
maxRegret = raw.max_regret;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const priceCeiling = parsePriceCeiling(raw.price_ceiling, "objective.price_ceiling", problems);
|
|
91
|
+
let lambda;
|
|
92
|
+
if (raw.lambda !== undefined) {
|
|
93
|
+
if (typeof raw.lambda !== "number" || !Number.isFinite(raw.lambda) || raw.lambda < 0) {
|
|
94
|
+
problems.push("objective.lambda must be a non-negative number");
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
lambda = raw.lambda;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
if (mode !== "balanced" && lambda !== undefined) {
|
|
101
|
+
problems.push("objective.lambda only applies to mode: balanced");
|
|
102
|
+
}
|
|
103
|
+
if (mode === "balanced" && lambda === undefined) {
|
|
104
|
+
problems.push("objective.mode: balanced requires objective.lambda");
|
|
105
|
+
}
|
|
106
|
+
return Object.freeze({
|
|
107
|
+
mode: mode,
|
|
108
|
+
...(maxRegret !== undefined ? { maxRegret } : {}),
|
|
109
|
+
...(priceCeiling !== undefined ? { priceCeiling } : {}),
|
|
110
|
+
...(lambda !== undefined ? { lambda } : {}),
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
const PROVIDERS = new Set([
|
|
114
|
+
"gemini",
|
|
115
|
+
"openai",
|
|
116
|
+
"deepseek",
|
|
117
|
+
"anthropic",
|
|
118
|
+
"grok",
|
|
119
|
+
]);
|
|
120
|
+
/** "provider/model" or {provider, model} → CandidateSpec (no trial flag —
|
|
121
|
+
* a reference/judge is a concrete model, never a declare-first placeholder). */
|
|
122
|
+
function parseModelRef(raw, field, problems) {
|
|
123
|
+
let provider;
|
|
124
|
+
let model;
|
|
125
|
+
if (typeof raw === "string") {
|
|
126
|
+
const slash = raw.indexOf("/");
|
|
127
|
+
if (slash <= 0 || slash === raw.length - 1) {
|
|
128
|
+
problems.push(`${field} must be "provider/model" or {provider, model}`);
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
provider = raw.slice(0, slash);
|
|
132
|
+
model = raw.slice(slash + 1);
|
|
133
|
+
}
|
|
134
|
+
else if (isRecord(raw)) {
|
|
135
|
+
provider = raw.provider;
|
|
136
|
+
model = raw.model;
|
|
137
|
+
}
|
|
138
|
+
else {
|
|
139
|
+
problems.push(`${field} must be "provider/model" or {provider, model}`);
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
142
|
+
if (typeof provider !== "string" || !PROVIDERS.has(provider) || typeof model !== "string" || model === "") {
|
|
143
|
+
problems.push(`${field} has an unknown provider or empty model`);
|
|
144
|
+
return null;
|
|
145
|
+
}
|
|
146
|
+
return Object.freeze({ provider: provider, model });
|
|
147
|
+
}
|
|
148
|
+
/** Parse the `bakeoff:` block (reference arm + judge). `candidates` is the
|
|
149
|
+
* route's declared set — the candidate-set invariant is enforced here (#223 F3
|
|
150
|
+
* for the judge; the reference must be declared unless allow-external). */
|
|
151
|
+
function parseBakeoffConfig(raw, candidates, problems) {
|
|
152
|
+
if (raw === undefined || raw === null)
|
|
153
|
+
return undefined;
|
|
154
|
+
if (!isRecord(raw)) {
|
|
155
|
+
problems.push("bakeoff must be an object");
|
|
156
|
+
return undefined;
|
|
157
|
+
}
|
|
158
|
+
const declared = (c) => candidates.some((rc) => rc.provider === c.provider && rc.model === c.model);
|
|
159
|
+
let referenceAllowExternal = false;
|
|
160
|
+
if (raw.reference_allow_external !== undefined) {
|
|
161
|
+
if (typeof raw.reference_allow_external !== "boolean") {
|
|
162
|
+
problems.push("bakeoff.reference_allow_external must be a boolean");
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
referenceAllowExternal = raw.reference_allow_external;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
let reference;
|
|
169
|
+
if (raw.reference !== undefined && raw.reference !== null) {
|
|
170
|
+
if (raw.reference === "auto") {
|
|
171
|
+
reference = "auto";
|
|
172
|
+
}
|
|
173
|
+
else {
|
|
174
|
+
const ref = parseModelRef(raw.reference, "bakeoff.reference", problems);
|
|
175
|
+
if (ref !== null) {
|
|
176
|
+
if (!declared(ref) && !referenceAllowExternal) {
|
|
177
|
+
problems.push(`bakeoff.reference (${ref.provider}/${ref.model}) is not a declared candidate — ` +
|
|
178
|
+
"add it to candidates, or set bakeoff.reference_allow_external: true to record the exception");
|
|
179
|
+
}
|
|
180
|
+
else {
|
|
181
|
+
reference = { provider: ref.provider, model: ref.model };
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
let referenceEnvelopeUsd;
|
|
187
|
+
if (raw.reference_envelope_usd !== undefined) {
|
|
188
|
+
if (typeof raw.reference_envelope_usd !== "number" ||
|
|
189
|
+
!Number.isFinite(raw.reference_envelope_usd) ||
|
|
190
|
+
raw.reference_envelope_usd <= 0) {
|
|
191
|
+
problems.push("bakeoff.reference_envelope_usd must be a positive number");
|
|
192
|
+
}
|
|
193
|
+
else {
|
|
194
|
+
referenceEnvelopeUsd = raw.reference_envelope_usd;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
let judge;
|
|
198
|
+
if (raw.judge !== undefined && raw.judge !== null) {
|
|
199
|
+
if (!isRecord(raw.judge)) {
|
|
200
|
+
problems.push("bakeoff.judge must be an object { model, max_samples }");
|
|
201
|
+
}
|
|
202
|
+
else {
|
|
203
|
+
const model = parseModelRef(raw.judge.model, "bakeoff.judge.model", problems);
|
|
204
|
+
const maxSamples = raw.judge.max_samples;
|
|
205
|
+
if (typeof maxSamples !== "number" || !Number.isInteger(maxSamples) || maxSamples < 1) {
|
|
206
|
+
problems.push("bakeoff.judge.max_samples must be a positive integer");
|
|
207
|
+
}
|
|
208
|
+
if (model !== null && !declared(model)) {
|
|
209
|
+
// #223 F3: the judge model must be a declared candidate.
|
|
210
|
+
problems.push(`bakeoff.judge.model (${model.provider}/${model.model}) is not a declared candidate ` +
|
|
211
|
+
"(a judge model must be one of the route's declared candidates, #223 F3)");
|
|
212
|
+
}
|
|
213
|
+
else if (model !== null && typeof maxSamples === "number" && Number.isInteger(maxSamples) && maxSamples >= 1) {
|
|
214
|
+
judge = { model, maxSamples };
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
const out = {
|
|
219
|
+
...(reference !== undefined ? { reference } : {}),
|
|
220
|
+
...(referenceAllowExternal ? { referenceAllowExternal: true } : {}),
|
|
221
|
+
...(referenceEnvelopeUsd !== undefined ? { referenceEnvelopeUsd } : {}),
|
|
222
|
+
...(judge !== undefined ? { judge } : {}),
|
|
223
|
+
};
|
|
224
|
+
return Object.freeze(out);
|
|
225
|
+
}
|
|
226
|
+
/** Parse `value_checks:` — a list of {path, expect}. `path` is a JSON pointer;
|
|
227
|
+
* `expect` is a literal or {regex}. */
|
|
228
|
+
function parseValueChecks(raw, problems) {
|
|
229
|
+
if (raw === undefined || raw === null)
|
|
230
|
+
return undefined;
|
|
231
|
+
if (!Array.isArray(raw)) {
|
|
232
|
+
problems.push("value_checks must be a list");
|
|
233
|
+
return undefined;
|
|
234
|
+
}
|
|
235
|
+
const out = [];
|
|
236
|
+
for (const [i, c] of raw.entries()) {
|
|
237
|
+
const field = `value_checks[${i}]`;
|
|
238
|
+
if (!isRecord(c)) {
|
|
239
|
+
problems.push(`${field} must be an object { path, expect }`);
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
if (typeof c.path !== "string" || c.path === "" || !c.path.startsWith("/")) {
|
|
243
|
+
problems.push(`${field}.path must be a non-empty JSON pointer starting with "/"`);
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
if (c.expect === undefined) {
|
|
247
|
+
problems.push(`${field}.expect is required (a literal value or { regex })`);
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
250
|
+
let expect;
|
|
251
|
+
if (isRecord(c.expect) && typeof c.expect.regex === "string") {
|
|
252
|
+
try {
|
|
253
|
+
// eslint-disable-next-line no-new
|
|
254
|
+
new RegExp(c.expect.regex);
|
|
255
|
+
}
|
|
256
|
+
catch (err) {
|
|
257
|
+
problems.push(`${field}.expect.regex is not a valid regular expression: ${err.message}`);
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
expect = { regex: c.expect.regex };
|
|
261
|
+
}
|
|
262
|
+
else {
|
|
263
|
+
expect = { literal: c.expect };
|
|
264
|
+
}
|
|
265
|
+
out.push(Object.freeze({ path: c.path, expect: Object.freeze(expect) }));
|
|
266
|
+
}
|
|
267
|
+
return Object.freeze(out);
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Non-fatal advisory (CE-2 teaching-hint style): a route declares an objective
|
|
271
|
+
* with NEITHER a regret bound NOR a price ceiling, so a bake-off will REPORT
|
|
272
|
+
* regret but never propose anything. Warn so the missing bound is deliberate.
|
|
273
|
+
* Returns the warning line, or null. Only meaningful for quality-first/balanced
|
|
274
|
+
* (cost-first has its own gate and needs no bound).
|
|
275
|
+
*/
|
|
276
|
+
function missingBoundWarning(route, objective) {
|
|
277
|
+
const mode = objective?.mode ?? "quality-first";
|
|
278
|
+
if (mode === "cost-first")
|
|
279
|
+
return null;
|
|
280
|
+
const hasBound = objective?.maxRegret !== undefined || objective?.priceCeiling !== undefined;
|
|
281
|
+
if (hasBound)
|
|
282
|
+
return null;
|
|
283
|
+
return (`route "${route}" has no objective bound (neither max_regret nor price_ceiling): a bake-off ` +
|
|
284
|
+
"will REPORT regret vs the reference but propose nothing. Declare objective.max_regret and/or " +
|
|
285
|
+
"objective.price_ceiling to let a challenger be proposed under your ceiling.");
|
|
286
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Accept-path builders, extracted from run.ts (which sits at the 700-LOC cap so
|
|
3
|
+
* the Q1 quality-gate seam has to pull its shared shapes out). Two pure builders:
|
|
4
|
+
*
|
|
5
|
+
* - `successAttemptRow` — the AttemptTelemetry token/cost fields for a
|
|
6
|
+
* SUCCESSFUL adapter result, shared by the accept path and the Q1
|
|
7
|
+
* quality-REJECT path (which records the same billed row with a
|
|
8
|
+
* `quality_rejected` failure instead of `null`).
|
|
9
|
+
* - `buildSuccessMeta` — the validated `RunMeta` returned on the accept path.
|
|
10
|
+
*
|
|
11
|
+
* Behavior is byte-identical to the former inline literals; nothing new is
|
|
12
|
+
* computed here.
|
|
13
|
+
*/
|
|
14
|
+
import type { CandidateRef } from "../types/provider";
|
|
15
|
+
import type { AdapterUsage } from "../types/provider";
|
|
16
|
+
import type { FailureClass, RigFailure } from "../types/failures";
|
|
17
|
+
import type { RouteBundle, RunMeta } from "../types/route";
|
|
18
|
+
import type { AttemptTelemetry } from "./dispatch";
|
|
19
|
+
import type { AttemptMetaSource } from "../telemetry/attempt-meta";
|
|
20
|
+
/** The AttemptTelemetry for a successful adapter result. `failure` is null on the
|
|
21
|
+
* accept path and the `quality_rejected` failure on the Q1 reject path;
|
|
22
|
+
* `metaInputs` is the WI-1 attempt-meta envelope; `extraTags` carries the Q1
|
|
23
|
+
* quality tags (`quality` / `quality_score`). */
|
|
24
|
+
export declare function successAttemptRow(candidate: CandidateRef, usage: AdapterUsage, costUsd: number, latencyMs: number, pricingMissing: boolean, failure: RigFailure | null, metaInputs: AttemptMetaSource, extraTags?: Record<string, string>): AttemptTelemetry;
|
|
25
|
+
export interface SuccessMetaInput {
|
|
26
|
+
readonly inferenceId: string;
|
|
27
|
+
readonly route: RouteBundle;
|
|
28
|
+
readonly candidate: CandidateRef;
|
|
29
|
+
readonly usage: AdapterUsage;
|
|
30
|
+
readonly costUsd: number;
|
|
31
|
+
readonly latencyMs: number;
|
|
32
|
+
readonly attemptsByClass: Partial<Record<FailureClass, number>>;
|
|
33
|
+
readonly servedVariant: string | null;
|
|
34
|
+
readonly callId: string;
|
|
35
|
+
/** Grounding-inject citations, when the search rung served this run. */
|
|
36
|
+
readonly citations?: RunMeta["citations"];
|
|
37
|
+
/** Q1 quality verdict, when a gate ran on this run. */
|
|
38
|
+
readonly quality?: RunMeta["quality"];
|
|
39
|
+
}
|
|
40
|
+
/** The validated `RunMeta` for the accept path — `repairedBy` is null here (the
|
|
41
|
+
* repair path builds its own meta), `validated` is always true. */
|
|
42
|
+
export declare function buildSuccessMeta(input: SuccessMetaInput): RunMeta;
|
|
43
|
+
//# sourceMappingURL=accept.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"accept.d.ts","sourceRoot":"","sources":["../../src/run/accept.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAEnE;;;iDAGiD;AACjD,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,YAAY,EACvB,KAAK,EAAE,YAAY,EACnB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,cAAc,EAAE,OAAO,EACvB,OAAO,EAAE,UAAU,GAAG,IAAI,EAC1B,UAAU,EAAE,iBAAiB,EAC7B,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACjC,gBAAgB,CAiBlB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,wEAAwE;IACxE,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC1C,uDAAuD;IACvD,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;CACvC;AAED;mEACmE;AACnE,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAuBjE"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Accept-path builders, extracted from run.ts (which sits at the 700-LOC cap so
|
|
4
|
+
* the Q1 quality-gate seam has to pull its shared shapes out). Two pure builders:
|
|
5
|
+
*
|
|
6
|
+
* - `successAttemptRow` — the AttemptTelemetry token/cost fields for a
|
|
7
|
+
* SUCCESSFUL adapter result, shared by the accept path and the Q1
|
|
8
|
+
* quality-REJECT path (which records the same billed row with a
|
|
9
|
+
* `quality_rejected` failure instead of `null`).
|
|
10
|
+
* - `buildSuccessMeta` — the validated `RunMeta` returned on the accept path.
|
|
11
|
+
*
|
|
12
|
+
* Behavior is byte-identical to the former inline literals; nothing new is
|
|
13
|
+
* computed here.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.successAttemptRow = successAttemptRow;
|
|
17
|
+
exports.buildSuccessMeta = buildSuccessMeta;
|
|
18
|
+
/** The AttemptTelemetry for a successful adapter result. `failure` is null on the
|
|
19
|
+
* accept path and the `quality_rejected` failure on the Q1 reject path;
|
|
20
|
+
* `metaInputs` is the WI-1 attempt-meta envelope; `extraTags` carries the Q1
|
|
21
|
+
* quality tags (`quality` / `quality_score`). */
|
|
22
|
+
function successAttemptRow(candidate, usage, costUsd, latencyMs, pricingMissing, failure, metaInputs, extraTags) {
|
|
23
|
+
return {
|
|
24
|
+
candidate,
|
|
25
|
+
servedTier: usage.servedTier,
|
|
26
|
+
tokensIn: usage.tokensIn,
|
|
27
|
+
tokensOut: usage.tokensOut,
|
|
28
|
+
tokensCached: usage.tokensCached,
|
|
29
|
+
tokensCacheWrite: usage.tokensCacheWrite ?? 0,
|
|
30
|
+
tokensCacheWrite1h: usage.tokensCacheWrite1h ?? 0,
|
|
31
|
+
costUsd,
|
|
32
|
+
latencyMs,
|
|
33
|
+
...(usage.ttfbMs !== undefined ? { ttfbMs: usage.ttfbMs } : {}),
|
|
34
|
+
failure,
|
|
35
|
+
pricingMissing,
|
|
36
|
+
metaInputs,
|
|
37
|
+
...(extraTags !== undefined ? { extraTags } : {}),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
/** The validated `RunMeta` for the accept path — `repairedBy` is null here (the
|
|
41
|
+
* repair path builds its own meta), `validated` is always true. */
|
|
42
|
+
function buildSuccessMeta(input) {
|
|
43
|
+
const { route, candidate, usage } = input;
|
|
44
|
+
return {
|
|
45
|
+
inferenceId: input.inferenceId,
|
|
46
|
+
route: route.route,
|
|
47
|
+
routeVersion: route.version,
|
|
48
|
+
provider: candidate.provider,
|
|
49
|
+
model: candidate.model,
|
|
50
|
+
requestedTier: route.policy.tier,
|
|
51
|
+
servedTier: usage.servedTier,
|
|
52
|
+
tokensIn: usage.tokensIn,
|
|
53
|
+
tokensOut: usage.tokensOut,
|
|
54
|
+
tokensCached: usage.tokensCached,
|
|
55
|
+
costEstimateUsd: input.costUsd,
|
|
56
|
+
latencyMs: input.latencyMs,
|
|
57
|
+
validated: true,
|
|
58
|
+
attemptsByClass: input.attemptsByClass,
|
|
59
|
+
servedVariant: input.servedVariant,
|
|
60
|
+
repairedBy: null,
|
|
61
|
+
callId: input.callId,
|
|
62
|
+
...(input.citations !== undefined ? { citations: input.citations } : {}),
|
|
63
|
+
...(input.quality !== undefined ? { quality: input.quality } : {}),
|
|
64
|
+
};
|
|
65
|
+
}
|
package/dist/run/cost.d.ts
CHANGED
|
@@ -20,6 +20,19 @@ export interface AttemptCost {
|
|
|
20
20
|
readonly pricingMissing: boolean;
|
|
21
21
|
}
|
|
22
22
|
export declare function attemptCost(provider: ProviderId, model: string, usage: AdapterUsage, unpricedRatePerTok: number): AttemptCost;
|
|
23
|
+
/** USD cost broken into its components for the attempt envelope
|
|
24
|
+
* (`AttemptMeta.cost`, WI-1 spec §2). Returns null when the model has no
|
|
25
|
+
* pricing entry (absent = unknown, never an invented $0 — mirrors
|
|
26
|
+
* `attemptCost`'s `pricingMissing`). Every component uses the SAME per-token
|
|
27
|
+
* rates as {@link estimateCostUsd}, so the sum equals the recorded
|
|
28
|
+
* `cost_estimate_usd` for a priced model. `other` is reserved (0 today). */
|
|
29
|
+
export declare function costComponents(provider: ProviderId, model: string, usage: Pick<AdapterUsage, "tokensIn" | "tokensOut" | "tokensCached" | "tokensCacheWrite" | "tokensCacheWrite1h">): {
|
|
30
|
+
in: number;
|
|
31
|
+
out: number;
|
|
32
|
+
cache_read: number;
|
|
33
|
+
cache_write: number;
|
|
34
|
+
other: number;
|
|
35
|
+
} | null;
|
|
23
36
|
/** Pre-dispatch upper-bound estimate for the envelope precheck: prompt
|
|
24
37
|
* length + the full output bound, at real pricing or the conservative rate. */
|
|
25
38
|
export declare function dispatchEstimateUsd(provider: ProviderId, model: string, promptChars: number, unpricedRatePerTok: number): number;
|
package/dist/run/cost.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cost.d.ts","sourceRoot":"","sources":["../../src/run/cost.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAElE,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAC1C,eAAO,MAAM,0BAA0B,QAAS,CAAC;AAEjD;;yEAEyE;AACzE,eAAO,MAAM,kCAAkC,MAAM,CAAC;AAEtD,MAAM,WAAW,WAAW;IAC1B,kEAAkE;IAClE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,uEAAuE;IACvE,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;AAED,wBAAgB,WAAW,CACzB,QAAQ,EAAE,UAAU,EACpB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,YAAY,EACnB,kBAAkB,EAAE,MAAM,GACzB,WAAW,CAkBb;AAED;+EAC+E;AAC/E,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,UAAU,EACpB,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,MAAM,GACzB,MAAM,CAOR"}
|
|
1
|
+
{"version":3,"file":"cost.d.ts","sourceRoot":"","sources":["../../src/run/cost.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAElE,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAC1C,eAAO,MAAM,0BAA0B,QAAS,CAAC;AAEjD;;yEAEyE;AACzE,eAAO,MAAM,kCAAkC,MAAM,CAAC;AAEtD,MAAM,WAAW,WAAW;IAC1B,kEAAkE;IAClE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,uEAAuE;IACvE,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;AAED,wBAAgB,WAAW,CACzB,QAAQ,EAAE,UAAU,EACpB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,YAAY,EACnB,kBAAkB,EAAE,MAAM,GACzB,WAAW,CAkBb;AAED;;;;;4EAK4E;AAC5E,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,UAAU,EACpB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,IAAI,CACT,YAAY,EACZ,UAAU,GAAG,WAAW,GAAG,cAAc,GAAG,kBAAkB,GAAG,oBAAoB,CACtF,GACA;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAkB5F;AAED;+EAC+E;AAC/E,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,UAAU,EACpB,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,MAAM,GACzB,MAAM,CAOR"}
|
package/dist/run/cost.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
10
|
exports.DEFAULT_UNPRICED_RATE_USD_PER_MTOK = exports.ESTIMATE_MAX_OUTPUT_TOKENS = exports.ESTIMATE_CHARS_PER_TOKEN = void 0;
|
|
11
11
|
exports.attemptCost = attemptCost;
|
|
12
|
+
exports.costComponents = costComponents;
|
|
12
13
|
exports.dispatchEstimateUsd = dispatchEstimateUsd;
|
|
13
14
|
const pricing_1 = require("../registry/pricing");
|
|
14
15
|
exports.ESTIMATE_CHARS_PER_TOKEN = 4;
|
|
@@ -29,6 +30,32 @@ function attemptCost(provider, model, usage, unpricedRatePerTok) {
|
|
|
29
30
|
const costUsd = (0, pricing_1.estimateCostUsd)(pricing, usage.tokensIn, usage.tokensOut, usage.tokensCached, usage.tokensCacheWrite ?? 0, usage.tokensCacheWrite1h ?? 0);
|
|
30
31
|
return { costUsd, envelopeChargeUsd: costUsd, pricingMissing: false };
|
|
31
32
|
}
|
|
33
|
+
/** USD cost broken into its components for the attempt envelope
|
|
34
|
+
* (`AttemptMeta.cost`, WI-1 spec §2). Returns null when the model has no
|
|
35
|
+
* pricing entry (absent = unknown, never an invented $0 — mirrors
|
|
36
|
+
* `attemptCost`'s `pricingMissing`). Every component uses the SAME per-token
|
|
37
|
+
* rates as {@link estimateCostUsd}, so the sum equals the recorded
|
|
38
|
+
* `cost_estimate_usd` for a priced model. `other` is reserved (0 today). */
|
|
39
|
+
function costComponents(provider, model, usage) {
|
|
40
|
+
const pricing = (0, pricing_1.getPricing)(provider, model);
|
|
41
|
+
if (pricing === null)
|
|
42
|
+
return null;
|
|
43
|
+
const tokensIn = Math.max(usage.tokensIn, 0);
|
|
44
|
+
const cached = Math.min(Math.max(usage.tokensCached, 0), tokensIn);
|
|
45
|
+
const write = Math.max(usage.tokensCacheWrite ?? 0, 0);
|
|
46
|
+
const write1h = Math.min(Math.max(usage.tokensCacheWrite1h ?? 0, 0), write);
|
|
47
|
+
const uncachedIn = Math.max(tokensIn - cached - write, 0);
|
|
48
|
+
const cachedRate = pricing.cachedInputUsdPerTok ?? pricing.inputUsdPerTok;
|
|
49
|
+
const writeRate = pricing.cacheWriteUsdPerTok ?? pricing.inputUsdPerTok;
|
|
50
|
+
const write1hRate = pricing.cacheWrite1hUsdPerTok ?? writeRate;
|
|
51
|
+
return {
|
|
52
|
+
in: uncachedIn * pricing.inputUsdPerTok,
|
|
53
|
+
out: Math.max(usage.tokensOut, 0) * pricing.outputUsdPerTok,
|
|
54
|
+
cache_read: cached * cachedRate,
|
|
55
|
+
cache_write: (write - write1h) * writeRate + write1h * write1hRate,
|
|
56
|
+
other: 0,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
32
59
|
/** Pre-dispatch upper-bound estimate for the envelope precheck: prompt
|
|
33
60
|
* length + the full output bound, at real pricing or the conservative rate. */
|
|
34
61
|
function dispatchEstimateUsd(provider, model, promptChars, unpricedRatePerTok) {
|
package/dist/run/dispatch.d.ts
CHANGED
|
@@ -8,10 +8,11 @@
|
|
|
8
8
|
* Tool *execution* is never ours (spec §4.4); these helpers normalize the
|
|
9
9
|
* protocol and surface it — nothing here runs a tool.
|
|
10
10
|
*/
|
|
11
|
-
import type { AdapterRequest, AdapterResult, CandidateRef, ProviderAdapter } from "../types/provider";
|
|
11
|
+
import type { AdapterRequest, AdapterResult, CandidateRef, ProviderAdapter, ToolDefinition } from "../types/provider";
|
|
12
12
|
import type { RigFailure } from "../types/failures";
|
|
13
13
|
import type { RouteBundle, RunMeta, RunOptions, RunResult, RunTags } from "../types/route";
|
|
14
14
|
import type { TelemetryRung, TelemetryStore } from "../types/telemetry";
|
|
15
|
+
import type { AttemptMetaSource } from "../telemetry/attempt-meta";
|
|
15
16
|
import type { LadderKeyMinter } from "./ladder-key";
|
|
16
17
|
/** One telemetry attempt row's inputs — shared with run.ts's recordAttempt so
|
|
17
18
|
* the tool-call helper here can record through the same closure. */
|
|
@@ -39,7 +40,27 @@ export interface AttemptTelemetry {
|
|
|
39
40
|
* (`tool_call`); every other rung is derived in the recorder from the
|
|
40
41
|
* repaired_by / pre-dispatch / primary-vs-retry signals. */
|
|
41
42
|
readonly rung?: TelemetryRung;
|
|
43
|
+
/** WI-1 (spec §2): the adapter's native meta + rendered request / output /
|
|
44
|
+
* declared tool names. The recorder (record-attempt.ts) hands it to
|
|
45
|
+
* `summarizeAttemptMeta` with this row's provider/model/usage. Absent ⇒ the
|
|
46
|
+
* recorder builds a minimal envelope from usage + cost alone (never null-only). */
|
|
47
|
+
readonly metaInputs?: AttemptMetaSource;
|
|
48
|
+
/** SDK-owned extra telemetry tags merged onto the row (Q1 quality gate stamps
|
|
49
|
+
* `quality` / `quality_score` here). Never carries `rung` — the recorder
|
|
50
|
+
* stamps that last so a caller cannot override it. */
|
|
51
|
+
readonly extraTags?: Record<string, string>;
|
|
42
52
|
}
|
|
53
|
+
/** Assemble the recorder's `metaInputs` from a completed dispatch (WI-1) — the
|
|
54
|
+
* adapter's native meta plus the rendered request (system+user+schema, hashed)
|
|
55
|
+
* and the output text (hashed). One helper so the routed lane's several record
|
|
56
|
+
* sites stay one-liners and run.ts holds under the LOC cap. */
|
|
57
|
+
export declare function attemptMetaInputs(args: {
|
|
58
|
+
readonly systemPrompt: string;
|
|
59
|
+
readonly userDirective: string;
|
|
60
|
+
readonly outputSchema: object | null;
|
|
61
|
+
readonly result: AdapterResult;
|
|
62
|
+
readonly tools?: readonly ToolDefinition[];
|
|
63
|
+
}): AttemptMetaSource;
|
|
43
64
|
/** run.ts's recordAttempt, threaded in so the tool-call path records through
|
|
44
65
|
* the same ledger/tenancy closure the rest of the loop uses. */
|
|
45
66
|
export type RecordAttempt = (attempt: AttemptTelemetry) => string;
|
|
@@ -98,6 +119,8 @@ export interface ToolCallResultParts {
|
|
|
98
119
|
/** The call's ladder-key id (spec §2.1 R0) — surfaced on the tool-call meta so
|
|
99
120
|
* it matches the recorded row's `call_id`. */
|
|
100
121
|
readonly callId: string;
|
|
122
|
+
/** WI-1: the attempt-envelope inputs for the recorded row (spec §2). */
|
|
123
|
+
readonly metaInputs?: AttemptMetaSource;
|
|
101
124
|
}
|
|
102
125
|
/**
|
|
103
126
|
* Record a tool-call turn and build its RunResult (spec §4.4). A tool call is
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dispatch.d.ts","sourceRoot":"","sources":["../../src/run/dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EACV,cAAc,EACd,aAAa,EACb,YAAY,EACZ,eAAe,
|
|
1
|
+
{"version":3,"file":"dispatch.d.ts","sourceRoot":"","sources":["../../src/run/dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EACV,cAAc,EACd,aAAa,EACb,YAAY,EACZ,eAAe,EACf,cAAc,EACf,MAAM,mBAAmB,CAAC;AAE3B,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC3F,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACxE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAInE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEpD;oEACoE;AACpE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;6DACyD;IACzD,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;0BACsB;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,IAAI,CAAC;IACpC,6EAA6E;IAC7E,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAClC;;gEAE4D;IAC5D,QAAQ,CAAC,IAAI,CAAC,EAAE,aAAa,CAAC;IAC9B;;;uFAGmF;IACnF,QAAQ,CAAC,UAAU,CAAC,EAAE,iBAAiB,CAAC;IACxC;;0DAEsD;IACtD,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC7C;AAED;;;+DAG+D;AAC/D,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACtC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;CAC5C,GAAG,iBAAiB,CAWpB;AAED;gEACgE;AAChE,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,gBAAgB,KAAK,MAAM,CAAC;AAElE;;4EAE4E;AAC5E,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,cAAc,EACzB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,OAAO,EACtB,aAAa,EAAE,MAAM,GAAG,IAAI,EAC5B,gBAAgB,EAAE,MAAM,EACxB,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,eAAe,GACtB,IAAI,CAoCN;AAED;;8EAE8E;AAC9E,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAYvD;AAED;;0CAE0C;AAC1C,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,KAAK,IAAI,CAAC;AAEzE,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC;IACjC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,+EAA+E;IAC/E,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B;;oEAEgE;IAChE,QAAQ,CAAC,qBAAqB,CAAC,EAAE,OAAO,CAAC;CAC1C;AAED;uDACuD;AACvD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,mBAAmB,GAAG,cAAc,CAgC9E;AAED;4CAC4C;AAC5C,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,WAAW,EAClB,SAAS,EAAE,YAAY,EACvB,YAAY,EAAE,MAAM,EACpB,eAAe,EAAE,MAAM,GACtB,cAAc,CAmBhB;AAED;;;;;;GAMG;AACH,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,eAAe,EACxB,GAAG,EAAE,cAAc,EACnB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,aAAa,CAAC,CAuBxB;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC;IACjC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE;QAAE,EAAE,EAAE,IAAI,CAAA;KAAE,CAAC,CAAC;IACtD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACrD,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC;kDAC8C;IAC9C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,wEAAwE;IACxE,QAAQ,CAAC,UAAU,CAAC,EAAE,iBAAiB,CAAC;CACzC;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,mBAAmB,GAAG,SAAS,CAiD/F"}
|
package/dist/run/dispatch.js
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* protocol and surface it — nothing here runs a tool.
|
|
11
11
|
*/
|
|
12
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.attemptMetaInputs = attemptMetaInputs;
|
|
13
14
|
exports.recordSearchCostRow = recordSearchCostRow;
|
|
14
15
|
exports.jsonModeCoaching = jsonModeCoaching;
|
|
15
16
|
exports.buildAdapterRequest = buildAdapterRequest;
|
|
@@ -17,8 +18,25 @@ exports.buildRepairAdapterRequest = buildRepairAdapterRequest;
|
|
|
17
18
|
exports.dispatchCandidate = dispatchCandidate;
|
|
18
19
|
exports.emitToolCallResult = emitToolCallResult;
|
|
19
20
|
const node_crypto_1 = require("node:crypto");
|
|
21
|
+
const attempt_meta_1 = require("../telemetry/attempt-meta");
|
|
20
22
|
const cache_stamp_1 = require("./cache-stamp");
|
|
21
23
|
const cache_handle_1 = require("./cache-handle");
|
|
24
|
+
/** Assemble the recorder's `metaInputs` from a completed dispatch (WI-1) — the
|
|
25
|
+
* adapter's native meta plus the rendered request (system+user+schema, hashed)
|
|
26
|
+
* and the output text (hashed). One helper so the routed lane's several record
|
|
27
|
+
* sites stay one-liners and run.ts holds under the LOC cap. */
|
|
28
|
+
function attemptMetaInputs(args) {
|
|
29
|
+
return {
|
|
30
|
+
...(args.result.meta !== undefined ? { adapterMeta: args.result.meta } : {}),
|
|
31
|
+
renderedRequest: (0, attempt_meta_1.renderRequestForHash)({
|
|
32
|
+
system: args.systemPrompt,
|
|
33
|
+
user: args.userDirective,
|
|
34
|
+
schema: args.outputSchema,
|
|
35
|
+
}),
|
|
36
|
+
outputText: args.result.ok ? args.result.text : null,
|
|
37
|
+
...(args.tools !== undefined ? { declaredToolNames: args.tools.map((t) => t.function.name) } : {}),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
22
40
|
/** Record the grounding-inject search's own telemetry row (WS3): a $-only line
|
|
23
41
|
* under the synthetic "search" provider. Extracted from run.ts to keep it under
|
|
24
42
|
* the LOC cap; behaviour is byte-identical to the prior inline recorder. */
|
|
@@ -193,6 +211,7 @@ function emitToolCallResult(record, parts) {
|
|
|
193
211
|
// is stamped explicitly. The rollup counts it as a tool-call turn, never a
|
|
194
212
|
// dispatch or a serve (its cost still counts).
|
|
195
213
|
rung: "tool_call",
|
|
214
|
+
...(parts.metaInputs !== undefined ? { metaInputs: parts.metaInputs } : {}),
|
|
196
215
|
});
|
|
197
216
|
return {
|
|
198
217
|
output: null,
|
package/dist/run/ladder-key.d.ts
CHANGED
|
@@ -31,6 +31,10 @@ export interface LadderKeyMinter {
|
|
|
31
31
|
/** Stamp the next row: bump `attempt_seq` (1-based, monotonic) and read the
|
|
32
32
|
* current `candidate_index`. */
|
|
33
33
|
stamp(): LadderKey;
|
|
34
|
+
/** The `attempt_seq` the NEXT `stamp()` will assign — read WITHOUT consuming
|
|
35
|
+
* it, so the quality gate (Q1) can name the seq its judged candidate's row
|
|
36
|
+
* will carry before that row is recorded. */
|
|
37
|
+
peekNextSeq(): number;
|
|
34
38
|
}
|
|
35
39
|
/**
|
|
36
40
|
* Build a ladder-key minter. `callId` defaults to a fresh uuid; the raw lane and
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ladder-key.d.ts","sourceRoot":"","sources":["../../src/run/ladder-key.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,gFAAgF;AAChF,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CACxC;AAED,4EAA4E;AAC5E,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;6EACyE;IACzE,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAAC;IAC9C;oCACgC;IAChC,KAAK,IAAI,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"ladder-key.d.ts","sourceRoot":"","sources":["../../src/run/ladder-key.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,gFAAgF;AAChF,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CACxC;AAED,4EAA4E;AAC5E,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;6EACyE;IACzE,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAAC;IAC9C;oCACgC;IAChC,KAAK,IAAI,SAAS,CAAC;IACnB;;iDAE6C;IAC7C,WAAW,IAAI,MAAM,CAAC;CACvB;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,GAAE,MAAqB,GAAG,eAAe,CAgBpF"}
|
package/dist/run/ladder-key.js
CHANGED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The quality-gate seam (Q1, quality-gate spec §4.1). Extracted from run.ts —
|
|
3
|
+
* which sits at the 700-LOC cap — so the run loop gets ONE compact call site
|
|
4
|
+
* between `outcome.valid` and the accept branch.
|
|
5
|
+
*
|
|
6
|
+
* The gate evaluates a builder predicate on a schema-VALID output. It handles the
|
|
7
|
+
* awkward bits so the loop does not: awaiting an async verdict, normalizing an
|
|
8
|
+
* optional score/reason, and rejecting fail-closed when the predicate THROWS
|
|
9
|
+
* (spec §4.1 step 4, flag F2 — reject + warn ONCE per process). The loop keeps
|
|
10
|
+
* the control flow: a passing verdict takes the existing accept path (tagged),
|
|
11
|
+
* a failing one records `quality_rejected` and falls through on its own budget,
|
|
12
|
+
* never touching the repair rung.
|
|
13
|
+
*/
|
|
14
|
+
import type { QualityGate, QualityGateMeta } from "../types/quality";
|
|
15
|
+
/** The gate's answer, normalized for the run loop. `score`/`reason` are null
|
|
16
|
+
* rather than undefined so the row/meta stamping is total. */
|
|
17
|
+
export interface QualityGateOutcome {
|
|
18
|
+
/** True = accept (verdict.ok, or the predicate threw? no — a throw is a reject). */
|
|
19
|
+
readonly pass: boolean;
|
|
20
|
+
readonly score: number | null;
|
|
21
|
+
readonly reason: string | null;
|
|
22
|
+
}
|
|
23
|
+
/** Test-only: reset the once-per-process throw-warning latch so a test can
|
|
24
|
+
* assert the single-warning behavior deterministically. */
|
|
25
|
+
export declare function resetQualityGateWarningForTest(): void;
|
|
26
|
+
/**
|
|
27
|
+
* Evaluate a quality gate on a schema-valid output. Awaits an async verdict; a
|
|
28
|
+
* throwing predicate rejects fail-closed with a `qualityGate threw: …` reason
|
|
29
|
+
* and warns exactly once per process.
|
|
30
|
+
*/
|
|
31
|
+
export declare function evaluateQualityGate(gate: QualityGate, output: unknown, meta: QualityGateMeta): Promise<QualityGateOutcome>;
|
|
32
|
+
/** SDK-owned telemetry tags for the attempt row (spec §2): `quality` = pass|fail
|
|
33
|
+
* and, when a score exists, `quality_score` as a decimal string. Never carries
|
|
34
|
+
* `rung` — the recorder owns that and stamps it last. */
|
|
35
|
+
export declare function qualityTags(outcome: QualityGateOutcome): Record<string, string>;
|
|
36
|
+
/** The `quality_rejected` failure message — carries the reason and score so a
|
|
37
|
+
* stuck engineer (and the exhausted-ladder error) sees WHY it was rejected. */
|
|
38
|
+
export declare function qualityRejectedMessage(outcome: QualityGateOutcome): string;
|
|
39
|
+
//# sourceMappingURL=quality-gate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quality-gate.d.ts","sourceRoot":"","sources":["../../src/run/quality-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAErE;8DAC8D;AAC9D,MAAM,WAAW,kBAAkB;IACjC,oFAAoF;IACpF,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAMD;2DAC2D;AAC3D,wBAAgB,8BAA8B,IAAI,IAAI,CAErD;AAMD;;;;GAIG;AACH,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,WAAW,EACjB,MAAM,EAAE,OAAO,EACf,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC,kBAAkB,CAAC,CAe7B;AAOD;;yDAEyD;AACzD,wBAAgB,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAI/E;AAED;+EAC+E;AAC/E,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,kBAAkB,GAAG,MAAM,CAI1E"}
|