modelrig 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +179 -0
- package/README.md +51 -27
- package/data/PROVENANCE.md +14 -0
- package/data/pricing-map.json +26 -0
- package/dist/adapters/attempt-meta-map.d.ts +8 -1
- package/dist/adapters/attempt-meta-map.d.ts.map +1 -1
- package/dist/adapters/attempt-meta-map.js +4 -0
- package/dist/adapters/build-adapters.js +2 -2
- package/dist/adapters/gemini.d.ts +28 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +38 -9
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +53 -4
- package/dist/adapters/openai-compat.d.ts +8 -2
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +8 -1
- package/dist/adapters/openai-stream.d.ts +10 -2
- package/dist/adapters/openai-stream.d.ts.map +1 -1
- package/dist/adapters/openai-stream.js +5 -1
- package/dist/adapters/openai.d.ts +10 -1
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +57 -7
- package/dist/adapters/responses-api.d.ts +61 -0
- package/dist/adapters/responses-api.d.ts.map +1 -0
- package/dist/adapters/responses-api.js +124 -0
- package/dist/artifacts/artifact.d.ts +20 -0
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +207 -171
- package/dist/artifacts/index.d.ts +3 -1
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +8 -1
- package/dist/bakeoff/groundedness.d.ts +58 -0
- package/dist/bakeoff/groundedness.d.ts.map +1 -0
- package/dist/bakeoff/groundedness.js +165 -0
- package/dist/bakeoff/quality.d.ts +14 -1
- package/dist/bakeoff/quality.d.ts.map +1 -1
- package/dist/bakeoff/quality.js +10 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +4 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +8 -0
- package/dist/config.d.ts +31 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +23 -0
- package/dist/doctor-cli.d.ts +7 -0
- package/dist/doctor-cli.d.ts.map +1 -0
- package/dist/doctor-cli.js +67 -0
- package/dist/doctor-preflight.d.ts +52 -0
- package/dist/doctor-preflight.d.ts.map +1 -0
- package/dist/doctor-preflight.js +112 -0
- package/dist/feedback/grade.d.ts +8 -1
- package/dist/feedback/grade.d.ts.map +1 -1
- package/dist/feedback/grade.js +4 -0
- package/dist/gateway/http-protocol.d.ts +19 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -1
- package/dist/gateway/serve-client.d.ts +99 -0
- package/dist/gateway/serve-client.d.ts.map +1 -0
- package/dist/gateway/serve-client.js +360 -0
- package/dist/index.d.ts +25 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +62 -8
- package/dist/prompts/manifest.d.ts +128 -0
- package/dist/prompts/manifest.d.ts.map +1 -0
- package/dist/prompts/manifest.js +214 -0
- package/dist/prompts/prompt-fetch.d.ts +84 -0
- package/dist/prompts/prompt-fetch.d.ts.map +1 -0
- package/dist/prompts/prompt-fetch.js +120 -0
- package/dist/prompts/prompt-resolve.d.ts +76 -0
- package/dist/prompts/prompt-resolve.d.ts.map +1 -0
- package/dist/prompts/prompt-resolve.js +93 -0
- package/dist/prompts/slot-resolver.d.ts +91 -0
- package/dist/prompts/slot-resolver.d.ts.map +1 -0
- package/dist/prompts/slot-resolver.js +135 -0
- package/dist/prompts/tools-register.d.ts +60 -0
- package/dist/prompts/tools-register.d.ts.map +1 -0
- package/dist/prompts/tools-register.js +48 -0
- package/dist/prompts/version-hash.d.ts +29 -0
- package/dist/prompts/version-hash.d.ts.map +1 -0
- package/dist/prompts/version-hash.js +54 -0
- package/dist/prompts-export-cli.d.ts +21 -0
- package/dist/prompts-export-cli.d.ts.map +1 -0
- package/dist/prompts-export-cli.js +130 -0
- package/dist/registry/cache-policy.d.ts.map +1 -1
- package/dist/registry/cache-policy.js +22 -0
- package/dist/registry/pricing-shape.d.ts +38 -8
- package/dist/registry/pricing-shape.d.ts.map +1 -1
- package/dist/registry/pricing-shape.js +40 -12
- package/dist/registry/pricing.d.ts +6 -5
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +9 -5
- package/dist/routes/destination-loader.d.ts +16 -0
- package/dist/routes/destination-loader.d.ts.map +1 -0
- package/dist/routes/destination-loader.js +29 -0
- package/dist/routes/draft.d.ts +7 -0
- package/dist/routes/draft.d.ts.map +1 -1
- package/dist/routes/draft.js +12 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +27 -54
- package/dist/routes/output-loader.d.ts +42 -0
- package/dist/routes/output-loader.d.ts.map +1 -0
- package/dist/routes/output-loader.js +112 -0
- package/dist/routes/prompt-loader.d.ts +49 -0
- package/dist/routes/prompt-loader.d.ts.map +1 -0
- package/dist/routes/prompt-loader.js +113 -0
- package/dist/routes/resolved-bundle.d.ts +19 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -1
- package/dist/routes/resolved-bundle.js +24 -0
- package/dist/routes/screen-loader.d.ts +1 -1
- package/dist/routes/screen-loader.d.ts.map +1 -1
- package/dist/routes/screen-loader.js +8 -1
- package/dist/run/accept.d.ts +3 -0
- package/dist/run/accept.d.ts.map +1 -1
- package/dist/run/accept.js +1 -0
- package/dist/run/cost.d.ts +31 -2
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +57 -2
- package/dist/run/destination.d.ts +39 -0
- package/dist/run/destination.d.ts.map +1 -0
- package/dist/run/destination.js +45 -0
- package/dist/run/dispatch.d.ts +25 -3
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +26 -4
- package/dist/run/raw-defaults.d.ts.map +1 -1
- package/dist/run/raw-defaults.js +6 -1
- package/dist/run/raw-finalize.d.ts +50 -0
- package/dist/run/raw-finalize.d.ts.map +1 -0
- package/dist/run/raw-finalize.js +266 -0
- package/dist/run/raw-knobs.d.ts +36 -0
- package/dist/run/raw-knobs.d.ts.map +1 -0
- package/dist/run/raw-knobs.js +99 -0
- package/dist/run/raw-request.d.ts +24 -0
- package/dist/run/raw-request.d.ts.map +1 -0
- package/dist/run/raw-request.js +69 -0
- package/dist/run/raw-schema.d.ts +76 -0
- package/dist/run/raw-schema.d.ts.map +1 -0
- package/dist/run/raw-schema.js +166 -0
- package/dist/run/raw-search-cost.d.ts +48 -0
- package/dist/run/raw-search-cost.d.ts.map +1 -0
- package/dist/run/raw-search-cost.js +78 -0
- package/dist/run/raw-support.d.ts +64 -6
- package/dist/run/raw-support.d.ts.map +1 -1
- package/dist/run/raw-support.js +84 -11
- package/dist/run/raw.d.ts +25 -26
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +30 -351
- package/dist/run/record-attempt.d.ts +8 -0
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +26 -12
- package/dist/run/repair-attempt.d.ts +68 -0
- package/dist/run/repair-attempt.d.ts.map +1 -0
- package/dist/run/repair-attempt.js +180 -0
- package/dist/run/resolve-destination.d.ts +49 -0
- package/dist/run/resolve-destination.d.ts.map +1 -0
- package/dist/run/resolve-destination.js +76 -0
- package/dist/run/routing-docs.d.ts.map +1 -1
- package/dist/run/routing-docs.js +9 -4
- package/dist/run/run.d.ts +30 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +75 -148
- package/dist/run/telemetry-row.d.ts +92 -0
- package/dist/run/telemetry-row.d.ts.map +1 -0
- package/dist/run/telemetry-row.js +81 -0
- package/dist/run/validate.d.ts +12 -1
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +31 -0
- package/dist/run/web-search-pricing.d.ts +43 -0
- package/dist/run/web-search-pricing.d.ts.map +1 -0
- package/dist/run/web-search-pricing.js +56 -0
- package/dist/settings/grading-guide.d.ts +27 -1
- package/dist/settings/grading-guide.d.ts.map +1 -1
- package/dist/status.d.ts +6 -0
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +4 -1
- package/dist/telemetry/artifact-store.d.ts.map +1 -1
- package/dist/telemetry/artifact-store.js +11 -0
- package/dist/telemetry/attempt-meta.d.ts +55 -1
- package/dist/telemetry/attempt-meta.d.ts.map +1 -1
- package/dist/telemetry/attempt-meta.js +24 -1
- package/dist/telemetry/exporter-setup.d.ts +2 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -1
- package/dist/telemetry/exporter-setup.js +6 -1
- package/dist/telemetry/grade-mappers.d.ts +5 -2
- package/dist/telemetry/grade-mappers.d.ts.map +1 -1
- package/dist/telemetry/grade-mappers.js +8 -2
- package/dist/telemetry/grades-sql.d.ts +4 -1
- package/dist/telemetry/grades-sql.d.ts.map +1 -1
- package/dist/telemetry/grades-sql.js +26 -3
- package/dist/telemetry/ingest-protocol.d.ts +26 -2
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +32 -29
- package/dist/telemetry/org-identity.d.ts +44 -0
- package/dist/telemetry/org-identity.d.ts.map +1 -0
- package/dist/telemetry/org-identity.js +98 -0
- package/dist/telemetry/route-registration.d.ts +48 -0
- package/dist/telemetry/route-registration.d.ts.map +1 -0
- package/dist/telemetry/route-registration.js +77 -0
- package/dist/telemetry/sinks.d.ts +10 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +52 -6
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +13 -0
- package/dist/tenancy/api-key-format.d.ts +16 -1
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +17 -0
- package/dist/types/provider.d.ts +20 -3
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +1 -0
- package/dist/types/route.d.ts +76 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +30 -2
- package/dist/types/telemetry.d.ts.map +1 -1
- package/docs/agent-skill/SKILL.md +73 -5
- package/docs/instrumentation.md +81 -36
- package/docs/llms.txt +95 -16
- package/docs/managed-serving.md +150 -0
- package/docs/mcp-oracle.md +75 -2
- package/docs/migration-playbook.md +5 -4
- package/docs/quickstart.md +169 -83
- package/package.json +3 -2
- package/registry/registry.json +82 -0
- package/dist/feedback/feedback.d.ts +0 -35
- package/dist/feedback/feedback.d.ts.map +0 -1
- package/dist/feedback/feedback.js +0 -39
package/dist/run/run.js
CHANGED
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
* ruling 2 / finding F3).
|
|
9
9
|
*/
|
|
10
10
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.USER_DIRECTIVE = void 0;
|
|
12
|
+
exports.userTurnFor = userTurnFor;
|
|
11
13
|
exports.run = run;
|
|
12
14
|
const loader_1 = require("../routes/loader");
|
|
13
15
|
const serveability_1 = require("../routes/serveability");
|
|
@@ -18,9 +20,12 @@ const failures_1 = require("../types/failures");
|
|
|
18
20
|
const manifest_1 = require("../routes/manifest");
|
|
19
21
|
const resolve_1 = require("./resolve");
|
|
20
22
|
const variant_1 = require("./variant");
|
|
23
|
+
const resolve_destination_1 = require("./resolve-destination");
|
|
24
|
+
const serve_client_1 = require("../gateway/serve-client");
|
|
21
25
|
const envelope_1 = require("./envelope");
|
|
22
26
|
const retry_1 = require("./retry");
|
|
23
27
|
const repair_1 = require("./repair");
|
|
28
|
+
const repair_attempt_1 = require("./repair-attempt");
|
|
24
29
|
const grounding_1 = require("./grounding");
|
|
25
30
|
const validate_1 = require("./validate");
|
|
26
31
|
const dispatch_1 = require("./dispatch");
|
|
@@ -34,10 +39,18 @@ const capture_write_1 = require("./capture-write");
|
|
|
34
39
|
/** Phase 1 prompt convention: the bundle's system template carries the whole
|
|
35
40
|
* task (constitution, data, step instructions); the user turn is a terse
|
|
36
41
|
* trigger. Providers treat long system prompts + short user turns well, and
|
|
37
|
-
* it keeps the single-template bundle contract of spec §3.
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
42
|
+
* it keeps the single-template bundle contract of spec §3. Issue #5 (R1): a
|
|
43
|
+
* route that declares `prompt.user` replaces this trigger with its rendered
|
|
44
|
+
* user template — the per-call delta a cache handle cannot carry in the system
|
|
45
|
+
* slot (see `userTurnFor`). */
|
|
46
|
+
exports.USER_DIRECTIVE = "Produce the required output now, following the system instructions exactly.";
|
|
47
|
+
/** The user turn for one primary attempt (issue #5, R1): the route's rendered
|
|
48
|
+
* `prompt.user` template when declared — same variables, capability
|
|
49
|
+
* conditionals and `x-trust` removal as the system template — else the fixed
|
|
50
|
+
* trigger (byte-identical to pre-R1). Pure; exported for the run tests. */
|
|
51
|
+
function userTurnFor(route, ctx) {
|
|
52
|
+
return route.userTemplate === undefined ? exports.USER_DIRECTIVE : (0, template_1.renderTemplate)(route.userTemplate, ctx);
|
|
53
|
+
}
|
|
41
54
|
/** prefer:[cost] ordering metric — combined input+output USD per million
|
|
42
55
|
* tokens from the pricing snapshot; null (unknown) sorts last in resolve. */
|
|
43
56
|
function costPerMTok(provider, model) {
|
|
@@ -46,13 +59,14 @@ function costPerMTok(provider, model) {
|
|
|
46
59
|
return null;
|
|
47
60
|
return (pricing.inputUsdPerTok + pricing.outputUsdPerTok) * 1000000;
|
|
48
61
|
}
|
|
49
|
-
const DEFAULT_SLEEP = (ms) => ms <= 0 ? Promise.resolve() : new Promise((resolve) => setTimeout(resolve, ms));
|
|
62
|
+
const DEFAULT_SLEEP = (ms) => (ms <= 0 ? Promise.resolve() : new Promise((resolve) => setTimeout(resolve, ms)));
|
|
50
63
|
async function run(deps, routeName, opts) {
|
|
51
64
|
const mode = deps.mode ?? "live";
|
|
52
65
|
const baseRoute = deps.routes.get(routeName);
|
|
53
66
|
const variant = (0, variant_1.resolveVariant)(baseRoute, opts.variant, mode);
|
|
54
67
|
const servedVariant = variant?.name ?? null;
|
|
55
68
|
const route = (0, resolve_1.applyRunRequirements)((0, variant_1.effectiveRoute)(baseRoute, variant), opts.zeroRetention === true);
|
|
69
|
+
const destination = (0, resolve_destination_1.resolveDestination)(route.destination, opts.destination); // U2 (D-2): RunOptions > route > `direct`. Guard runs below, once the serving provider is resolved (U3).
|
|
56
70
|
// RC-1: inside a run context the SDK stamps `run_id` (episodeKey, else run id)
|
|
57
71
|
// when the caller passed none — the bridge between tags and the run graph.
|
|
58
72
|
const runCtx = (0, run_context_1.getActiveRunContext)();
|
|
@@ -96,7 +110,7 @@ async function run(deps, routeName, opts) {
|
|
|
96
110
|
autoStepKey,
|
|
97
111
|
cache: opts.cache,
|
|
98
112
|
droppedCacheHandle,
|
|
99
|
-
ladder, ...(opts.arm !== undefined ? { arm: opts.arm } : {}),
|
|
113
|
+
ladder, destination, ...(opts.arm !== undefined ? { arm: opts.arm } : {}),
|
|
100
114
|
});
|
|
101
115
|
function failRun(candidate, failure, latencyMs, screen) {
|
|
102
116
|
// N1 (spec §5, SC-1b req 2): a pre-dispatch enforce `block` must still record
|
|
@@ -112,135 +126,14 @@ async function run(deps, routeName, opts) {
|
|
|
112
126
|
// Repair rung (WS2): variant repair config wins over the route's; its own
|
|
113
127
|
// budget, never fungible with the content_invalid class budget.
|
|
114
128
|
const repairLedger = (0, repair_1.createRepairLedger)(variant?.repair ?? baseRoute.repair);
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
systemPrompt = originalSystemPrompt + (0, repair_1.repairFeedbackBlock)(errors);
|
|
124
|
-
}
|
|
125
|
-
else {
|
|
126
|
-
// Declare-first backstop (spec §4.4): the loader rejects a `trial` candidate
|
|
127
|
-
// as `repair_model`, but a programmatically-built bundle (runBundle) could
|
|
128
|
-
// bypass the loader — so refuse to mint a trial candidate for LIVE repair
|
|
129
|
-
// here too (the primary ladder's trial-drop has no repair-path equivalent).
|
|
130
|
-
if (mode === "live" &&
|
|
131
|
-
route.candidates.some((c) => c.provider === step.model.provider && c.model === step.model.model && c.trial === true)) {
|
|
132
|
-
return null;
|
|
133
|
-
}
|
|
134
|
-
// Brand stays in resolve.ts; loader validated repairModel ⊆ declared.
|
|
135
|
-
const minted = (0, resolve_1.resolveDeclaredCandidate)(route, step.model);
|
|
136
|
-
if (minted === null)
|
|
137
|
-
return null;
|
|
138
|
-
repairCandidate = minted;
|
|
139
|
-
systemPrompt = (0, repair_1.repairModelPrompt)(invalidText, errors, route.outputSchema);
|
|
140
|
-
}
|
|
141
|
-
const repairedBy = `${repairCandidate.provider}/${repairCandidate.model}`;
|
|
142
|
-
const adapter = deps.adapters.get(repairCandidate.provider);
|
|
143
|
-
if (adapter === undefined)
|
|
144
|
-
return null;
|
|
145
|
-
if (guard !== null) {
|
|
146
|
-
// A1: unpriced repair models fail closed exactly like primaries.
|
|
147
|
-
const breach = guard.precheck((0, cost_1.dispatchEstimateUsd)(repairCandidate.provider, repairCandidate.model, systemPrompt.length + REPAIR_DIRECTIVE.length, unpricedRatePerTok));
|
|
148
|
-
if (breach !== null) {
|
|
149
|
-
failRun(repairCandidate, breach, 0);
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
const started = Date.now();
|
|
153
|
-
const result = await adapter.call((0, dispatch_1.buildRepairAdapterRequest)(route, repairCandidate, systemPrompt, REPAIR_DIRECTIVE));
|
|
154
|
-
const latencyMs = Date.now() - started;
|
|
155
|
-
if (!result.ok) {
|
|
156
|
-
let cost = 0;
|
|
157
|
-
let repairFailurePricingMissing = false;
|
|
158
|
-
if (result.usage !== undefined) {
|
|
159
|
-
const settled = (0, cost_1.attemptCost)(repairCandidate.provider, repairCandidate.model, result.usage, unpricedRatePerTok);
|
|
160
|
-
cost = settled.costUsd;
|
|
161
|
-
repairFailurePricingMissing = settled.pricingMissing;
|
|
162
|
-
guard?.settle(settled.envelopeChargeUsd);
|
|
163
|
-
}
|
|
164
|
-
recordAttempt({
|
|
165
|
-
candidate: repairCandidate,
|
|
166
|
-
servedTier: result.usage?.servedTier ?? route.policy.tier,
|
|
167
|
-
tokensIn: result.usage?.tokensIn ?? 0,
|
|
168
|
-
tokensOut: result.usage?.tokensOut ?? 0,
|
|
169
|
-
tokensCached: result.usage?.tokensCached ?? 0,
|
|
170
|
-
tokensCacheWrite: result.usage?.tokensCacheWrite ?? 0,
|
|
171
|
-
tokensCacheWrite1h: result.usage?.tokensCacheWrite1h ?? 0,
|
|
172
|
-
costUsd: cost,
|
|
173
|
-
latencyMs,
|
|
174
|
-
...(result.usage?.ttfbMs !== undefined ? { ttfbMs: result.usage.ttfbMs } : {}),
|
|
175
|
-
failure: result.failure,
|
|
176
|
-
repairedBy,
|
|
177
|
-
pricingMissing: repairFailurePricingMissing,
|
|
178
|
-
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: REPAIR_DIRECTIVE, outputSchema: route.outputSchema, result }),
|
|
179
|
-
});
|
|
180
|
-
if (failures_1.TERMINAL_CLASSES.has(result.failure.class)) {
|
|
181
|
-
throw new failures_1.RigFailureError(result.failure);
|
|
182
|
-
}
|
|
183
|
-
return null;
|
|
184
|
-
}
|
|
185
|
-
const { costUsd, envelopeChargeUsd, pricingMissing: repairPricingMissing, } = (0, cost_1.attemptCost)(repairCandidate.provider, repairCandidate.model, result.usage, unpricedRatePerTok);
|
|
186
|
-
guard?.settle(envelopeChargeUsd);
|
|
187
|
-
const outcome = (0, validate_1.validateOutput)(route, result.text);
|
|
188
|
-
const failure = outcome.valid
|
|
189
|
-
? null
|
|
190
|
-
: {
|
|
191
|
-
class: "content_invalid",
|
|
192
|
-
message: `repair output failed schema validation: ${outcome.errors.join("; ")}`,
|
|
193
|
-
provider: repairCandidate.provider,
|
|
194
|
-
model: repairCandidate.model,
|
|
195
|
-
};
|
|
196
|
-
const inferenceId = recordAttempt({
|
|
197
|
-
candidate: repairCandidate,
|
|
198
|
-
servedTier: result.usage.servedTier,
|
|
199
|
-
tokensIn: result.usage.tokensIn,
|
|
200
|
-
tokensOut: result.usage.tokensOut,
|
|
201
|
-
tokensCached: result.usage.tokensCached,
|
|
202
|
-
tokensCacheWrite: result.usage.tokensCacheWrite ?? 0,
|
|
203
|
-
tokensCacheWrite1h: result.usage.tokensCacheWrite1h ?? 0,
|
|
204
|
-
costUsd,
|
|
205
|
-
latencyMs,
|
|
206
|
-
...(result.usage.ttfbMs !== undefined ? { ttfbMs: result.usage.ttfbMs } : {}),
|
|
207
|
-
failure,
|
|
208
|
-
repairedBy,
|
|
209
|
-
pricingMissing: repairPricingMissing,
|
|
210
|
-
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: REPAIR_DIRECTIVE, outputSchema: route.outputSchema, result }),
|
|
211
|
-
});
|
|
212
|
-
captureAttempt(inferenceId, result.text);
|
|
213
|
-
if (!outcome.valid) {
|
|
214
|
-
return { invalid: { text: result.text, errors: outcome.errors } };
|
|
215
|
-
}
|
|
216
|
-
return {
|
|
217
|
-
done: {
|
|
218
|
-
output: outcome.parsed,
|
|
219
|
-
meta: {
|
|
220
|
-
inferenceId,
|
|
221
|
-
route: route.route,
|
|
222
|
-
routeVersion: route.version,
|
|
223
|
-
provider: repairCandidate.provider,
|
|
224
|
-
model: repairCandidate.model,
|
|
225
|
-
requestedTier: route.policy.tier,
|
|
226
|
-
servedTier: result.usage.servedTier,
|
|
227
|
-
tokensIn: result.usage.tokensIn,
|
|
228
|
-
tokensOut: result.usage.tokensOut,
|
|
229
|
-
tokensCached: result.usage.tokensCached,
|
|
230
|
-
costEstimateUsd: costUsd,
|
|
231
|
-
latencyMs,
|
|
232
|
-
validated: true,
|
|
233
|
-
attemptsByClass: ledger.snapshot(),
|
|
234
|
-
servedVariant,
|
|
235
|
-
repairedBy,
|
|
236
|
-
callId: ladder.callId,
|
|
237
|
-
...(injectedSources !== null
|
|
238
|
-
? (0, grounding_1.groundingMeta)(result.text, injectedSources)
|
|
239
|
-
: {}),
|
|
240
|
-
},
|
|
241
|
-
},
|
|
242
|
-
};
|
|
243
|
-
}
|
|
129
|
+
// Repair dispatch (WS2), extracted to ./repair-attempt.ts (U4) to keep run.ts
|
|
130
|
+
// under the LOC cap — a factory over this run's per-call state, mirroring
|
|
131
|
+
// createRecordAttempt. The run loop's current `injectedSources` is passed in at
|
|
132
|
+
// the call site (below) rather than closed over.
|
|
133
|
+
const attemptRepair = (0, repair_attempt_1.createAttemptRepair)({
|
|
134
|
+
adapters: deps.adapters, guard, unpricedRatePerTok, route, mode,
|
|
135
|
+
recordAttempt, captureAttempt, ledger, servedVariant, ladder, destination, failRun,
|
|
136
|
+
});
|
|
244
137
|
const searchAvailable = deps.search !== undefined && deps.search !== null;
|
|
245
138
|
const iter = (0, resolve_1.resolveCandidates)(route, deps.capabilities, costPerMTok, searchAvailable, (0, resolve_1.candidateFilterFromOptions)(opts), undefined, mode);
|
|
246
139
|
// Advance the iterator AND publish the served candidate's index to the ladder,
|
|
@@ -255,6 +148,28 @@ async function run(deps, routeName, opts) {
|
|
|
255
148
|
throw new loader_1.RouteConfigError(`route "${route.route}" has no eligible candidate — none of the declared candidates ` +
|
|
256
149
|
`satisfy require=[${route.require.join(", ")}] with json=${route.policy.json}${(0, resolve_1.filterSummary)(opts)}`);
|
|
257
150
|
}
|
|
151
|
+
(0, resolve_destination_1.assertDestinationServable)(destination, deps.managedServingEnabled === true, candidate.provider); // U3 (D-2): serving guard, now the provider is resolved. `direct` passes (byte-identical); `modelrig-served` passes only with the flag ON AND a served-capable provider, else throws pre-dispatch (no provider call, no telemetry row).
|
|
152
|
+
// K1 (managed-serving-keyless §2/§3): the resolved destination is
|
|
153
|
+
// `modelrig-served` AND the flag is on (the guard above just confirmed a
|
|
154
|
+
// served-capable provider) ⇒ serve the run through the HOSTED gateway on
|
|
155
|
+
// ModelRig's managed key (serve-client.ts), NOT the local adapter loop. The
|
|
156
|
+
// gateway resolves the org's mirrored bundle BY SLUG and drives the SAME
|
|
157
|
+
// verified pipeline server-side; its provenance (provider/model/cost/
|
|
158
|
+
// inference-id/validated) is carried into `RunResult.meta`. No local telemetry
|
|
159
|
+
// row is written here — the authoritative inference row lives server-side and
|
|
160
|
+
// `meta.inferenceId` points at it (anti-cheat: the SDK never double-meters a
|
|
161
|
+
// server-measured call). Fail-closed-but-discoverable inside serve-client; no
|
|
162
|
+
// silent fall back to `direct`/local key. `direct` (and flag-off) NEVER reach
|
|
163
|
+
// this branch, so every other path is byte-identical to today.
|
|
164
|
+
if (destination === "modelrig-served" && deps.managedServingEnabled === true) {
|
|
165
|
+
return (0, serve_client_1.serveViaGateway)({
|
|
166
|
+
route,
|
|
167
|
+
opts,
|
|
168
|
+
servedVariant,
|
|
169
|
+
serve: deps.serve,
|
|
170
|
+
...(deps.serveFetch !== undefined ? { fetchFn: deps.serveFetch } : {}),
|
|
171
|
+
});
|
|
172
|
+
}
|
|
258
173
|
let lastFailure = null;
|
|
259
174
|
// Grounding-inject (WS3): one search per run, cached across candidates and
|
|
260
175
|
// retries; cost accrues to the envelope and its own telemetry row.
|
|
@@ -287,10 +202,15 @@ async function run(deps, routeName, opts) {
|
|
|
287
202
|
}
|
|
288
203
|
// Render against the SERVING candidate's capability flags.
|
|
289
204
|
const flags = deps.capabilities(candidate.provider, candidate.model);
|
|
290
|
-
|
|
205
|
+
const renderCtx = {
|
|
291
206
|
variables: opts.input,
|
|
292
207
|
capabilities: flags, variableTrust: (0, grounding_1.systemSlotRemoval)(route.variableTrust, opts.proxyTrust),
|
|
293
|
-
}
|
|
208
|
+
};
|
|
209
|
+
let systemPrompt = (0, template_1.renderTemplate)(route.systemTemplate, renderCtx);
|
|
210
|
+
// Issue #5 (R1): the user turn — the route's `prompt.user` when declared
|
|
211
|
+
// (rendered with the SAME context, so an untrusted variable is removed from
|
|
212
|
+
// it too and arrives via the spotlight block), else the fixed trigger.
|
|
213
|
+
const userPrompt = userTurnFor(route, renderCtx);
|
|
294
214
|
// Variant augmentation (WS1): appended after capability-conditionals,
|
|
295
215
|
// before json coaching — promptAppend first, then the reasoning scaffold.
|
|
296
216
|
if (variant?.promptAppend !== undefined) {
|
|
@@ -354,7 +274,7 @@ async function run(deps, routeName, opts) {
|
|
|
354
274
|
// pricing snapshot (prompt-length input + full output budget).
|
|
355
275
|
if (guard !== null) {
|
|
356
276
|
// A1: an unpriced model must never precheck at $0 (fail closed).
|
|
357
|
-
const breach = guard.precheck((0, cost_1.dispatchEstimateUsd)(candidate.provider, candidate.model, systemPrompt.length +
|
|
277
|
+
const breach = guard.precheck((0, cost_1.dispatchEstimateUsd)(candidate.provider, candidate.model, systemPrompt.length + userPrompt.length, unpricedRatePerTok));
|
|
358
278
|
if (breach !== null) {
|
|
359
279
|
failRun(candidate, breach, 0);
|
|
360
280
|
}
|
|
@@ -366,7 +286,7 @@ async function run(deps, routeName, opts) {
|
|
|
366
286
|
// + dispatch as ONE seam; both guards null ⇒ byte-identical. The untrusted
|
|
367
287
|
// appends + screening live in the seam so run.ts stays under the LOC cap.
|
|
368
288
|
const dispatched = await (0, dispatch_1.buildDispatchMaybeMasked)(adapter, {
|
|
369
|
-
route, candidate, systemPrompt, userDirective:
|
|
289
|
+
route, candidate, systemPrompt, userDirective: userPrompt,
|
|
370
290
|
nativeGrounded, opts, suppressCustomerCache: suppressed, injectedSources,
|
|
371
291
|
}, deps.onStreamDelta, guards);
|
|
372
292
|
if (!dispatched.result.ok && (dispatched.maskRefusal !== undefined || dispatched.screenRefusal !== undefined))
|
|
@@ -387,12 +307,12 @@ async function run(deps, routeName, opts) {
|
|
|
387
307
|
attemptsByClass: ledger.snapshot(),
|
|
388
308
|
servedVariant,
|
|
389
309
|
pricingMissing,
|
|
390
|
-
callId: ladder.callId,
|
|
391
|
-
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective:
|
|
310
|
+
callId: ladder.callId, destination,
|
|
311
|
+
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: userPrompt, outputSchema: route.outputSchema, outputKind: route.outputKind, result, tools: opts.tools }),
|
|
392
312
|
});
|
|
393
313
|
}
|
|
394
314
|
const outcome = (0, validate_1.validateOutput)(route, result.text);
|
|
395
|
-
const metaInputs = (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective:
|
|
315
|
+
const metaInputs = (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: userPrompt, outputSchema: route.outputSchema, outputKind: route.outputKind, result, tools: opts.tools });
|
|
396
316
|
if (outcome.valid) {
|
|
397
317
|
// Q1 quality gate (spec §4.1): a schema-valid output must still clear the
|
|
398
318
|
// builder's bar before it is accepted. Tool-call turns never reach here;
|
|
@@ -425,19 +345,23 @@ async function run(deps, routeName, opts) {
|
|
|
425
345
|
captureAttempt(inferenceId, result.text);
|
|
426
346
|
const meta = (0, accept_1.buildSuccessMeta)({
|
|
427
347
|
inferenceId, route, candidate, usage: result.usage, costUsd, latencyMs,
|
|
428
|
-
attemptsByClass: ledger.snapshot(), servedVariant, callId: ladder.callId,
|
|
348
|
+
attemptsByClass: ledger.snapshot(), servedVariant, callId: ladder.callId, destination,
|
|
429
349
|
...(injectedSources !== null ? (0, grounding_1.groundingMeta)(result.text, injectedSources) : {}),
|
|
430
350
|
...(gate !== null ? { quality: { score: gate.score, verdict: "pass", judgedBy: null } } : {}),
|
|
431
351
|
...(maskReceipt !== undefined ? { mask: maskReceipt } : {}), ...(screenReceipt !== undefined ? { screen: screenReceipt } : {}),
|
|
432
352
|
});
|
|
433
353
|
return { output, meta };
|
|
434
354
|
}
|
|
435
|
-
// Schema-invalid output — a real (billed) attempt that failed.
|
|
355
|
+
// Schema-invalid output — a real (billed) attempt that failed. On an
|
|
356
|
+
// `output: text` route this is ONLY a declared min_chars miss (issue #1 —
|
|
357
|
+
// prose is never JSON-parsed); `fixHint` teaches when the CONTRACT is the
|
|
358
|
+
// cause (prose arriving on a json route ⇒ "declare output: text").
|
|
436
359
|
const failure = {
|
|
437
360
|
class: "content_invalid",
|
|
438
361
|
message: `output failed schema validation: ${outcome.errors.join("; ")}`,
|
|
439
362
|
provider: candidate.provider,
|
|
440
363
|
model: candidate.model,
|
|
364
|
+
...(outcome.fixHint !== undefined ? { fixHint: outcome.fixHint } : {}),
|
|
441
365
|
};
|
|
442
366
|
const invalidId = recordAttempt({
|
|
443
367
|
candidate,
|
|
@@ -458,13 +382,16 @@ async function run(deps, routeName, opts) {
|
|
|
458
382
|
lastFailure = failure;
|
|
459
383
|
// Repair rung (WS2): consumed BEFORE any content_invalid retry, from
|
|
460
384
|
// its own budget. Chain: same-model error feedback, then the
|
|
461
|
-
// designated repair model on the newest invalid output.
|
|
462
|
-
|
|
385
|
+
// designated repair model on the newest invalid output. JSON-only by
|
|
386
|
+
// construction: an `output: text` route has no schema (loader-enforced),
|
|
387
|
+
// so prose is never sent for JSON repair (issue #1) — a min_chars miss
|
|
388
|
+
// falls straight through to the content_invalid retry below.
|
|
389
|
+
if (route.outputSchema !== null && route.outputKind !== "text") {
|
|
463
390
|
let latestText = result.text;
|
|
464
391
|
let latestErrors = outcome.errors;
|
|
465
392
|
let repairStep = repairLedger.next();
|
|
466
393
|
while (repairStep !== null) {
|
|
467
|
-
const repairOutcome = await attemptRepair(repairStep, candidate, systemPrompt, latestText, latestErrors);
|
|
394
|
+
const repairOutcome = await attemptRepair(repairStep, candidate, systemPrompt, latestText, latestErrors, injectedSources, userPrompt);
|
|
468
395
|
if (repairOutcome?.done !== undefined) {
|
|
469
396
|
return repairOutcome.done;
|
|
470
397
|
}
|
|
@@ -505,7 +432,7 @@ async function run(deps, routeName, opts) {
|
|
|
505
432
|
...(result.usage?.ttfbMs !== undefined ? { ttfbMs: result.usage.ttfbMs } : {}),
|
|
506
433
|
failure,
|
|
507
434
|
pricingMissing: failurePricingMissing,
|
|
508
|
-
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective:
|
|
435
|
+
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: userPrompt, outputSchema: route.outputSchema, outputKind: route.outputKind, result, tools: opts.tools }),
|
|
509
436
|
});
|
|
510
437
|
if (failures_1.TERMINAL_CLASSES.has(failure.class)) {
|
|
511
438
|
throw new failures_1.RigFailureError(failure);
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ONE telemetry row builder both dispatch lanes feed (Unified Route U4,
|
|
3
|
+
* spec §4 U4 / plan U4 Step 1). Before U4 the persisted `inferences` row was
|
|
4
|
+
* assembled — and its `tagsJson` serialized — in FOUR places (research §1c):
|
|
5
|
+
* the routed recorder (`record-attempt.ts`), the raw recorder (`raw-finalize.ts`
|
|
6
|
+
* `recordRawRow`), the grounding-inject search-cost recorder (`dispatch.ts`
|
|
7
|
+
* `recordSearchCostRow`), and the native-search cost recorder
|
|
8
|
+
* (`raw-search-cost.ts`). A new cross-lane tag therefore cost four edits and
|
|
9
|
+
* drifted. This module collapses them onto one builder: every field is placed
|
|
10
|
+
* ONCE, and `tagsJson` is serialized in exactly ONE place — {@link stampRowTags}.
|
|
11
|
+
*
|
|
12
|
+
* BEHAVIOR-PRESERVING: `buildTelemetryRow` takes every field the caller already
|
|
13
|
+
* resolved (raw's former hardcodes — `routeVersion: 0`, `attemptsJson: "{}"`,
|
|
14
|
+
* `servedVariant: null`, `attemptSeq: 1`, `candidateIndex: 0`, `lane: "raw"` —
|
|
15
|
+
* are now EXPLICIT inputs, not literals hidden in one lane's recorder), so the
|
|
16
|
+
* emitted row is byte-identical to what each recorder produced before. `meta`
|
|
17
|
+
* and `mask` are placed only when the caller passes them (conditional spread),
|
|
18
|
+
* preserving each lane's exact key-presence (the search-cost row omits `meta`;
|
|
19
|
+
* the raw search-cost row sets `meta: null`).
|
|
20
|
+
*
|
|
21
|
+
* No route/registry/capability-gate dependency — types only — so the raw lane's
|
|
22
|
+
* no-hard-hot-path-dependency invariant is preserved when raw-finalize.ts and
|
|
23
|
+
* raw-search-cost.ts feed this builder.
|
|
24
|
+
*/
|
|
25
|
+
import type { FailureClass } from "../types/failures";
|
|
26
|
+
import type { RunTags } from "../types/route";
|
|
27
|
+
import type { AttemptMetaV1 } from "../telemetry/attempt-meta";
|
|
28
|
+
import type { MaskReceipt } from "../mask/types";
|
|
29
|
+
import type { TelemetryRow, TelemetryRung } from "../types/telemetry";
|
|
30
|
+
/**
|
|
31
|
+
* THE single point where a telemetry row's `tags_json` is serialized (U4). Every
|
|
32
|
+
* row writer stamps its tags here, so a NEW cross-lane tag is a ONE-FILE edit:
|
|
33
|
+
* add it to the object assembled below and it appears on every lane's rows at
|
|
34
|
+
* once (proven by `telemetry-row.test.ts`). Key order is preserved exactly as
|
|
35
|
+
* the former inline sites had it — `{ ...baseTags, ...extraTags, rung }` — and
|
|
36
|
+
* `rung` is stamped LAST so a caller's `extraTags` can never override the
|
|
37
|
+
* recorder-owned rung.
|
|
38
|
+
*/
|
|
39
|
+
export declare function stampRowTags(baseTags: RunTags | Record<string, string>, rung: TelemetryRung, extraTags?: Record<string, string>): string;
|
|
40
|
+
/** Inputs to the one row builder. Mirrors {@link TelemetryRow} field-for-field,
|
|
41
|
+
* except `tagsJson` is assembled from (`baseTags`, `extraTags?`, `rung`) via
|
|
42
|
+
* {@link stampRowTags}. `meta` / `mask` are placed only when present (undefined
|
|
43
|
+
* ⇒ the key is omitted), matching each caller's former shape exactly. */
|
|
44
|
+
export interface TelemetryRowInput {
|
|
45
|
+
readonly id: string;
|
|
46
|
+
readonly route: string;
|
|
47
|
+
readonly routeVersion: number;
|
|
48
|
+
readonly provider: string;
|
|
49
|
+
readonly model: string;
|
|
50
|
+
readonly requestedTier: string;
|
|
51
|
+
readonly servedTier: string;
|
|
52
|
+
readonly tokensIn: number;
|
|
53
|
+
readonly tokensOut: number;
|
|
54
|
+
readonly tokensCached: number;
|
|
55
|
+
readonly tokensCacheWrite: number;
|
|
56
|
+
readonly tokensCacheWrite1h: number;
|
|
57
|
+
readonly costEstimateUsd: number;
|
|
58
|
+
readonly latencyMs: number;
|
|
59
|
+
readonly ttfbMs: number | null;
|
|
60
|
+
readonly failureClass: FailureClass | null;
|
|
61
|
+
readonly attemptsJson: string;
|
|
62
|
+
/** Base tags stamped on the row (routed: `{...step, ...effectiveTags}`; raw:
|
|
63
|
+
* the lane tags incl. `lane:"raw"`). Serialized via {@link stampRowTags}. */
|
|
64
|
+
readonly baseTags: RunTags | Record<string, string>;
|
|
65
|
+
/** SDK-owned extra tags merged after `baseTags`, before `rung` (Q1 quality
|
|
66
|
+
* tags on the routed accept path). Absent on every other site. */
|
|
67
|
+
readonly extraTags?: Record<string, string>;
|
|
68
|
+
readonly rung: TelemetryRung;
|
|
69
|
+
readonly fingerprintCell: string | null;
|
|
70
|
+
readonly servedVariant: string | null;
|
|
71
|
+
readonly repairedBy: string | null;
|
|
72
|
+
readonly pricingMissing: boolean;
|
|
73
|
+
readonly cacheKey: string | null;
|
|
74
|
+
readonly prefixFingerprint: string | null;
|
|
75
|
+
readonly callId: string | null;
|
|
76
|
+
readonly attemptSeq: number | null;
|
|
77
|
+
readonly candidateIndex: number | null;
|
|
78
|
+
/** WI-1 attempt envelope. Placed only when present — the grounding-inject
|
|
79
|
+
* search-cost row omits it entirely; the raw search-cost row sets it to null. */
|
|
80
|
+
readonly meta?: AttemptMetaV1 | null;
|
|
81
|
+
/** CS-1 mask receipt. Placed only when present (routed masked route only). */
|
|
82
|
+
readonly mask?: MaskReceipt;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Assemble the persisted `inferences` row (U4). One object literal, one
|
|
86
|
+
* `tags_json` serialization ({@link stampRowTags}). `ts` is stamped here (all
|
|
87
|
+
* former recorders stamped `new Date().toISOString()` at row-build time — this
|
|
88
|
+
* is the same instant). `meta` / `mask` are placed only when the caller passes
|
|
89
|
+
* them, preserving each lane's exact key-presence.
|
|
90
|
+
*/
|
|
91
|
+
export declare function buildTelemetryRow(input: TelemetryRowInput): TelemetryRow;
|
|
92
|
+
//# sourceMappingURL=telemetry-row.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"telemetry-row.d.ts","sourceRoot":"","sources":["../../src/run/telemetry-row.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEtE;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC1C,IAAI,EAAE,aAAa,EACnB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACjC,MAAM,CAER;AAED;;;yEAGyE;AACzE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;iFAC6E;IAC7E,QAAQ,CAAC,QAAQ,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpD;sEACkE;IAClE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC;qFACiF;IACjF,QAAQ,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACrC,8EAA8E;IAC9E,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,iBAAiB,GAAG,YAAY,CAiCxE"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The ONE telemetry row builder both dispatch lanes feed (Unified Route U4,
|
|
4
|
+
* spec §4 U4 / plan U4 Step 1). Before U4 the persisted `inferences` row was
|
|
5
|
+
* assembled — and its `tagsJson` serialized — in FOUR places (research §1c):
|
|
6
|
+
* the routed recorder (`record-attempt.ts`), the raw recorder (`raw-finalize.ts`
|
|
7
|
+
* `recordRawRow`), the grounding-inject search-cost recorder (`dispatch.ts`
|
|
8
|
+
* `recordSearchCostRow`), and the native-search cost recorder
|
|
9
|
+
* (`raw-search-cost.ts`). A new cross-lane tag therefore cost four edits and
|
|
10
|
+
* drifted. This module collapses them onto one builder: every field is placed
|
|
11
|
+
* ONCE, and `tagsJson` is serialized in exactly ONE place — {@link stampRowTags}.
|
|
12
|
+
*
|
|
13
|
+
* BEHAVIOR-PRESERVING: `buildTelemetryRow` takes every field the caller already
|
|
14
|
+
* resolved (raw's former hardcodes — `routeVersion: 0`, `attemptsJson: "{}"`,
|
|
15
|
+
* `servedVariant: null`, `attemptSeq: 1`, `candidateIndex: 0`, `lane: "raw"` —
|
|
16
|
+
* are now EXPLICIT inputs, not literals hidden in one lane's recorder), so the
|
|
17
|
+
* emitted row is byte-identical to what each recorder produced before. `meta`
|
|
18
|
+
* and `mask` are placed only when the caller passes them (conditional spread),
|
|
19
|
+
* preserving each lane's exact key-presence (the search-cost row omits `meta`;
|
|
20
|
+
* the raw search-cost row sets `meta: null`).
|
|
21
|
+
*
|
|
22
|
+
* No route/registry/capability-gate dependency — types only — so the raw lane's
|
|
23
|
+
* no-hard-hot-path-dependency invariant is preserved when raw-finalize.ts and
|
|
24
|
+
* raw-search-cost.ts feed this builder.
|
|
25
|
+
*/
|
|
26
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
27
|
+
exports.stampRowTags = stampRowTags;
|
|
28
|
+
exports.buildTelemetryRow = buildTelemetryRow;
|
|
29
|
+
/**
|
|
30
|
+
* THE single point where a telemetry row's `tags_json` is serialized (U4). Every
|
|
31
|
+
* row writer stamps its tags here, so a NEW cross-lane tag is a ONE-FILE edit:
|
|
32
|
+
* add it to the object assembled below and it appears on every lane's rows at
|
|
33
|
+
* once (proven by `telemetry-row.test.ts`). Key order is preserved exactly as
|
|
34
|
+
* the former inline sites had it — `{ ...baseTags, ...extraTags, rung }` — and
|
|
35
|
+
* `rung` is stamped LAST so a caller's `extraTags` can never override the
|
|
36
|
+
* recorder-owned rung.
|
|
37
|
+
*/
|
|
38
|
+
function stampRowTags(baseTags, rung, extraTags) {
|
|
39
|
+
return JSON.stringify({ ...baseTags, ...(extraTags ?? {}), rung });
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Assemble the persisted `inferences` row (U4). One object literal, one
|
|
43
|
+
* `tags_json` serialization ({@link stampRowTags}). `ts` is stamped here (all
|
|
44
|
+
* former recorders stamped `new Date().toISOString()` at row-build time — this
|
|
45
|
+
* is the same instant). `meta` / `mask` are placed only when the caller passes
|
|
46
|
+
* them, preserving each lane's exact key-presence.
|
|
47
|
+
*/
|
|
48
|
+
function buildTelemetryRow(input) {
|
|
49
|
+
return {
|
|
50
|
+
id: input.id,
|
|
51
|
+
ts: new Date().toISOString(),
|
|
52
|
+
route: input.route,
|
|
53
|
+
routeVersion: input.routeVersion,
|
|
54
|
+
provider: input.provider,
|
|
55
|
+
model: input.model,
|
|
56
|
+
requestedTier: input.requestedTier,
|
|
57
|
+
servedTier: input.servedTier,
|
|
58
|
+
tokensIn: input.tokensIn,
|
|
59
|
+
tokensOut: input.tokensOut,
|
|
60
|
+
tokensCached: input.tokensCached,
|
|
61
|
+
tokensCacheWrite: input.tokensCacheWrite,
|
|
62
|
+
tokensCacheWrite1h: input.tokensCacheWrite1h,
|
|
63
|
+
costEstimateUsd: input.costEstimateUsd,
|
|
64
|
+
latencyMs: input.latencyMs,
|
|
65
|
+
ttfbMs: input.ttfbMs,
|
|
66
|
+
failureClass: input.failureClass,
|
|
67
|
+
attemptsJson: input.attemptsJson,
|
|
68
|
+
tagsJson: stampRowTags(input.baseTags, input.rung, input.extraTags),
|
|
69
|
+
fingerprintCell: input.fingerprintCell,
|
|
70
|
+
servedVariant: input.servedVariant,
|
|
71
|
+
repairedBy: input.repairedBy,
|
|
72
|
+
pricingMissing: input.pricingMissing,
|
|
73
|
+
cacheKey: input.cacheKey,
|
|
74
|
+
prefixFingerprint: input.prefixFingerprint,
|
|
75
|
+
callId: input.callId,
|
|
76
|
+
attemptSeq: input.attemptSeq,
|
|
77
|
+
candidateIndex: input.candidateIndex,
|
|
78
|
+
...(input.meta !== undefined ? { meta: input.meta } : {}),
|
|
79
|
+
...(input.mask !== undefined ? { mask: input.mask } : {}),
|
|
80
|
+
};
|
|
81
|
+
}
|
package/dist/run/validate.d.ts
CHANGED
|
@@ -2,14 +2,25 @@
|
|
|
2
2
|
* Output validation — plan §4. Ajv compiled once per (route, version), cached.
|
|
3
3
|
* Phase 1 checks schema conformance; value-accuracy gates arrive with the
|
|
4
4
|
* bake-off in Phase 3.
|
|
5
|
+
*
|
|
6
|
+
* Two output kinds (issue #1, incident 2026-10-11):
|
|
7
|
+
* json (default) — JSON.parse, then ajv against `outputSchema` when declared.
|
|
8
|
+
* A schema-less json route still requires parseable JSON (unchanged).
|
|
9
|
+
* text — NO parse: `parsed` is the response string VERBATIM
|
|
10
|
+
* (untrimmed). The only check is the declared `min_chars` floor.
|
|
5
11
|
*/
|
|
6
12
|
import type { RouteBundle } from "../types/route";
|
|
7
13
|
export interface ValidationOutcome {
|
|
8
14
|
readonly valid: boolean;
|
|
9
15
|
/** ajv error summaries, capped, human- and agent-readable. */
|
|
10
16
|
readonly errors: readonly string[];
|
|
11
|
-
/** Parsed output when text was JSON-parseable (even if schema-invalid)
|
|
17
|
+
/** Parsed output when text was JSON-parseable (even if schema-invalid); the
|
|
18
|
+
* verbatim string on an `output: text` route. */
|
|
12
19
|
readonly parsed: unknown;
|
|
20
|
+
/** P4 (errors teach): the one-line fix for a failure whose cause is the
|
|
21
|
+
* route CONTRACT rather than the model — carried onto the `content_invalid`
|
|
22
|
+
* RigFailure as `fixHint`. */
|
|
23
|
+
readonly fixHint?: string;
|
|
13
24
|
}
|
|
14
25
|
export declare function validateOutput(route: RouteBundle, text: string): ValidationOutcome;
|
|
15
26
|
//# sourceMappingURL=validate.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../src/run/validate.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../src/run/validate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAIlD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,8DAA8D;IAC9D,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC;qDACiD;IACjD,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB;;kCAE8B;IAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAkDD,wBAAgB,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,GAAG,iBAAiB,CA+BlF"}
|
package/dist/run/validate.js
CHANGED
|
@@ -3,10 +3,38 @@
|
|
|
3
3
|
* Output validation — plan §4. Ajv compiled once per (route, version), cached.
|
|
4
4
|
* Phase 1 checks schema conformance; value-accuracy gates arrive with the
|
|
5
5
|
* bake-off in Phase 3.
|
|
6
|
+
*
|
|
7
|
+
* Two output kinds (issue #1, incident 2026-10-11):
|
|
8
|
+
* json (default) — JSON.parse, then ajv against `outputSchema` when declared.
|
|
9
|
+
* A schema-less json route still requires parseable JSON (unchanged).
|
|
10
|
+
* text — NO parse: `parsed` is the response string VERBATIM
|
|
11
|
+
* (untrimmed). The only check is the declared `min_chars` floor.
|
|
6
12
|
*/
|
|
7
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
14
|
exports.validateOutput = validateOutput;
|
|
15
|
+
const output_loader_1 = require("../routes/output-loader");
|
|
9
16
|
const ajv_1 = require("./ajv");
|
|
17
|
+
/** A prose-looking response on a json route: the incident shape (`# CoStar…`),
|
|
18
|
+
* or any body that does not even open like a JSON value. */
|
|
19
|
+
function looksLikeProse(text) {
|
|
20
|
+
const first = text.trimStart().charAt(0);
|
|
21
|
+
return first !== "{" && first !== "[";
|
|
22
|
+
}
|
|
23
|
+
const PROSE_ON_JSON_ROUTE_HINT = "this route expects JSON (the default output kind). If the task returns prose/markdown, declare " +
|
|
24
|
+
`\`output: text\` in the route bundle — the response is then returned verbatim and never JSON-parsed. See ${output_loader_1.PROSE_DOCS_ANCHOR}`;
|
|
25
|
+
function validateTextOutput(route, text) {
|
|
26
|
+
const floor = route.outputMinChars;
|
|
27
|
+
if (floor !== undefined && text.length < floor) {
|
|
28
|
+
return {
|
|
29
|
+
valid: false,
|
|
30
|
+
errors: [`output is ${text.length} characters, below the route's output.min_chars floor of ${floor}`],
|
|
31
|
+
parsed: text,
|
|
32
|
+
fixHint: "the model returned too little text for this prose route — the attempt is retried from the " +
|
|
33
|
+
"content_invalid budget (never JSON-repaired); lower output.min_chars if the floor is wrong",
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
return { valid: true, errors: [], parsed: text };
|
|
37
|
+
}
|
|
10
38
|
// Mirrors the production probe harness (packages/modelrig-probes/src/harness.ts):
|
|
11
39
|
// an Ajv-2020 instance with the draft-07 meta added for back-compat — the two
|
|
12
40
|
// must enforce identically, or a fixture would score differently than it serves.
|
|
@@ -26,6 +54,8 @@ function validatorFor(route) {
|
|
|
26
54
|
return compiled;
|
|
27
55
|
}
|
|
28
56
|
function validateOutput(route, text) {
|
|
57
|
+
if (route.outputKind === "text")
|
|
58
|
+
return validateTextOutput(route, text); // issue #1: prose is never parsed
|
|
29
59
|
let parsed;
|
|
30
60
|
try {
|
|
31
61
|
parsed = JSON.parse(text);
|
|
@@ -35,6 +65,7 @@ function validateOutput(route, text) {
|
|
|
35
65
|
valid: false,
|
|
36
66
|
errors: [`output is not valid JSON: ${err.message}`],
|
|
37
67
|
parsed: undefined,
|
|
68
|
+
...(looksLikeProse(text) ? { fixHint: PROSE_ON_JSON_ROUTE_HINT } : {}),
|
|
38
69
|
};
|
|
39
70
|
}
|
|
40
71
|
if (route.outputSchema === null) {
|