modelrig 0.8.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +324 -0
- package/README.md +55 -29
- package/data/PROVENANCE.md +14 -0
- package/data/pricing-map.json +26 -0
- package/dist/adapters/attempt-meta-map.d.ts +8 -1
- package/dist/adapters/attempt-meta-map.d.ts.map +1 -1
- package/dist/adapters/attempt-meta-map.js +4 -0
- package/dist/adapters/build-adapters.js +2 -2
- package/dist/adapters/gemini.d.ts +28 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +38 -9
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +53 -4
- package/dist/adapters/openai-compat.d.ts +8 -2
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +8 -1
- package/dist/adapters/openai-stream.d.ts +10 -2
- package/dist/adapters/openai-stream.d.ts.map +1 -1
- package/dist/adapters/openai-stream.js +5 -1
- package/dist/adapters/openai.d.ts +10 -1
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +57 -7
- package/dist/adapters/responses-api.d.ts +61 -0
- package/dist/adapters/responses-api.d.ts.map +1 -0
- package/dist/adapters/responses-api.js +124 -0
- package/dist/artifacts/artifact.d.ts +31 -12
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +218 -188
- package/dist/artifacts/index.d.ts +15 -5
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +19 -5
- package/dist/artifacts/run-context.d.ts +3 -24
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +3 -39
- package/dist/bakeoff/engine.d.ts +29 -0
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +47 -0
- package/dist/bakeoff/eval-cases-rig.d.ts +36 -0
- package/dist/bakeoff/eval-cases-rig.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases-rig.js +27 -0
- package/dist/bakeoff/eval-cases.d.ts +19 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -1
- package/dist/bakeoff/eval-cases.js +18 -1
- package/dist/bakeoff/golden-judge.d.ts +47 -0
- package/dist/bakeoff/golden-judge.d.ts.map +1 -0
- package/dist/bakeoff/golden-judge.js +49 -0
- package/dist/bakeoff/groundedness.d.ts +58 -0
- package/dist/bakeoff/groundedness.d.ts.map +1 -0
- package/dist/bakeoff/groundedness.js +165 -0
- package/dist/bakeoff/quality.d.ts +43 -0
- package/dist/bakeoff/quality.d.ts.map +1 -1
- package/dist/bakeoff/quality.js +31 -0
- package/dist/bakeoff/replay.d.ts +11 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +12 -0
- package/dist/bakeoff/report.d.ts +7 -0
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +26 -0
- package/dist/cli-judge.d.ts +12 -0
- package/dist/cli-judge.d.ts.map +1 -0
- package/dist/cli-judge.js +48 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +22 -42
- package/dist/config.d.ts +85 -29
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +109 -47
- package/dist/detect/detector.d.ts +4 -2
- package/dist/detect/detector.d.ts.map +1 -1
- package/dist/detect/safe-harbor.d.ts +57 -0
- package/dist/detect/safe-harbor.d.ts.map +1 -0
- package/dist/detect/safe-harbor.js +278 -0
- package/dist/doctor-cli.d.ts +7 -0
- package/dist/doctor-cli.d.ts.map +1 -0
- package/dist/doctor-cli.js +67 -0
- package/dist/doctor-preflight.d.ts +52 -0
- package/dist/doctor-preflight.d.ts.map +1 -0
- package/dist/doctor-preflight.js +112 -0
- package/dist/feedback/coverage.d.ts +11 -8
- package/dist/feedback/coverage.d.ts.map +1 -1
- package/dist/feedback/coverage.js +29 -16
- package/dist/feedback/grade.d.ts +58 -0
- package/dist/feedback/grade.d.ts.map +1 -0
- package/dist/feedback/grade.js +67 -0
- package/dist/gateway/http-protocol.d.ts +20 -1
- package/dist/gateway/http-protocol.d.ts.map +1 -1
- package/dist/gateway/http-protocol.js +1 -1
- package/dist/gateway/serve-client.d.ts +99 -0
- package/dist/gateway/serve-client.d.ts.map +1 -0
- package/dist/gateway/serve-client.js +360 -0
- package/dist/index.d.ts +68 -29
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +167 -46
- package/dist/init/observe-cli.js +1 -1
- package/dist/init/observe.d.ts +2 -2
- package/dist/init/observe.d.ts.map +1 -1
- package/dist/init/observe.js +11 -7
- package/dist/mask/arm.d.ts +112 -0
- package/dist/mask/arm.d.ts.map +1 -0
- package/dist/mask/arm.js +179 -0
- package/dist/mask/index.d.ts +2 -0
- package/dist/mask/index.d.ts.map +1 -1
- package/dist/mask/index.js +7 -1
- package/dist/mask/mask-policy-cli.d.ts +12 -4
- package/dist/mask/mask-policy-cli.d.ts.map +1 -1
- package/dist/mask/mask-policy-cli.js +34 -9
- package/dist/prompts/manifest.d.ts +128 -0
- package/dist/prompts/manifest.d.ts.map +1 -0
- package/dist/prompts/manifest.js +214 -0
- package/dist/prompts/prompt-fetch.d.ts +84 -0
- package/dist/prompts/prompt-fetch.d.ts.map +1 -0
- package/dist/prompts/prompt-fetch.js +120 -0
- package/dist/prompts/prompt-resolve.d.ts +76 -0
- package/dist/prompts/prompt-resolve.d.ts.map +1 -0
- package/dist/prompts/prompt-resolve.js +93 -0
- package/dist/prompts/slot-resolver.d.ts +91 -0
- package/dist/prompts/slot-resolver.d.ts.map +1 -0
- package/dist/prompts/slot-resolver.js +135 -0
- package/dist/prompts/tools-register.d.ts +60 -0
- package/dist/prompts/tools-register.d.ts.map +1 -0
- package/dist/prompts/tools-register.js +48 -0
- package/dist/prompts/version-hash.d.ts +29 -0
- package/dist/prompts/version-hash.d.ts.map +1 -0
- package/dist/prompts/version-hash.js +54 -0
- package/dist/prompts-export-cli.d.ts +21 -0
- package/dist/prompts-export-cli.d.ts.map +1 -0
- package/dist/prompts-export-cli.js +130 -0
- package/dist/query/grammar.d.ts +2 -2
- package/dist/query/grammar.d.ts.map +1 -1
- package/dist/query/grammar.js +3 -1
- package/dist/query/saved.d.ts +13 -0
- package/dist/query/saved.d.ts.map +1 -1
- package/dist/query/saved.js +67 -1
- package/dist/receipts/types.d.ts +1 -1
- package/dist/receipts/types.d.ts.map +1 -1
- package/dist/receipts/types.js +1 -0
- package/dist/registry/build.d.ts +43 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +52 -0
- package/dist/registry/cache-policy.d.ts.map +1 -1
- package/dist/registry/cache-policy.js +22 -0
- package/dist/registry/pricing-shape.d.ts +67 -8
- package/dist/registry/pricing-shape.d.ts.map +1 -1
- package/dist/registry/pricing-shape.js +50 -13
- package/dist/registry/pricing.d.ts +6 -5
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +9 -5
- package/dist/registry/rows.d.ts.map +1 -1
- package/dist/registry/rows.js +1 -0
- package/dist/registry/select.d.ts +6 -1
- package/dist/registry/select.d.ts.map +1 -1
- package/dist/registry/select.js +11 -0
- package/dist/routes/call-config.d.ts +199 -0
- package/dist/routes/call-config.d.ts.map +1 -0
- package/dist/routes/call-config.js +343 -0
- package/dist/routes/destination-loader.d.ts +16 -0
- package/dist/routes/destination-loader.d.ts.map +1 -0
- package/dist/routes/destination-loader.js +29 -0
- package/dist/routes/draft.d.ts +7 -0
- package/dist/routes/draft.d.ts.map +1 -1
- package/dist/routes/draft.js +12 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +31 -56
- package/dist/routes/objective-loader.d.ts +12 -0
- package/dist/routes/objective-loader.d.ts.map +1 -1
- package/dist/routes/objective-loader.js +82 -0
- package/dist/routes/output-loader.d.ts +42 -0
- package/dist/routes/output-loader.d.ts.map +1 -0
- package/dist/routes/output-loader.js +112 -0
- package/dist/routes/prompt-loader.d.ts +49 -0
- package/dist/routes/prompt-loader.d.ts.map +1 -0
- package/dist/routes/prompt-loader.js +113 -0
- package/dist/routes/resolved-bundle.d.ts +69 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -1
- package/dist/routes/resolved-bundle.js +76 -0
- package/dist/routes/screen-loader.d.ts +29 -1
- package/dist/routes/screen-loader.d.ts.map +1 -1
- package/dist/routes/screen-loader.js +59 -1
- package/dist/run/accept.d.ts +5 -0
- package/dist/run/accept.d.ts.map +1 -1
- package/dist/run/accept.js +2 -0
- package/dist/run/cost.d.ts +31 -2
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +57 -2
- package/dist/run/destination.d.ts +39 -0
- package/dist/run/destination.d.ts.map +1 -0
- package/dist/run/destination.js +45 -0
- package/dist/run/dispatch.d.ts +67 -16
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +133 -29
- package/dist/run/grounding.d.ts +28 -2
- package/dist/run/grounding.d.ts.map +1 -1
- package/dist/run/grounding.js +54 -10
- package/dist/run/raw-defaults.d.ts.map +1 -1
- package/dist/run/raw-defaults.js +6 -1
- package/dist/run/raw-finalize.d.ts +50 -0
- package/dist/run/raw-finalize.d.ts.map +1 -0
- package/dist/run/raw-finalize.js +266 -0
- package/dist/run/raw-knobs.d.ts +36 -0
- package/dist/run/raw-knobs.d.ts.map +1 -0
- package/dist/run/raw-knobs.js +99 -0
- package/dist/run/raw-request.d.ts +24 -0
- package/dist/run/raw-request.d.ts.map +1 -0
- package/dist/run/raw-request.js +69 -0
- package/dist/run/raw-schema.d.ts +76 -0
- package/dist/run/raw-schema.d.ts.map +1 -0
- package/dist/run/raw-schema.js +166 -0
- package/dist/run/raw-search-cost.d.ts +48 -0
- package/dist/run/raw-search-cost.d.ts.map +1 -0
- package/dist/run/raw-search-cost.js +78 -0
- package/dist/run/raw-support.d.ts +64 -6
- package/dist/run/raw-support.d.ts.map +1 -1
- package/dist/run/raw-support.js +84 -11
- package/dist/run/raw.d.ts +25 -26
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +30 -351
- package/dist/run/record-attempt.d.ts +12 -0
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +30 -12
- package/dist/run/repair-attempt.d.ts +68 -0
- package/dist/run/repair-attempt.d.ts.map +1 -0
- package/dist/run/repair-attempt.js +180 -0
- package/dist/run/resolve-destination.d.ts +49 -0
- package/dist/run/resolve-destination.d.ts.map +1 -0
- package/dist/run/resolve-destination.js +76 -0
- package/dist/run/routing-docs.d.ts.map +1 -1
- package/dist/run/routing-docs.js +10 -5
- package/dist/run/run.d.ts +30 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +94 -172
- package/dist/run/telemetry-row.d.ts +92 -0
- package/dist/run/telemetry-row.d.ts.map +1 -0
- package/dist/run/telemetry-row.js +81 -0
- package/dist/run/validate.d.ts +12 -1
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +31 -0
- package/dist/run/web-search-pricing.d.ts +43 -0
- package/dist/run/web-search-pricing.d.ts.map +1 -0
- package/dist/run/web-search-pricing.js +56 -0
- package/dist/savings/odometer.d.ts +4 -0
- package/dist/savings/odometer.d.ts.map +1 -1
- package/dist/savings/odometer.js +20 -1
- package/dist/savings/swap-execute.d.ts +7 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +30 -0
- package/dist/savings/swaps-client.d.ts +10 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/screen/fixtures/probe-corpora.d.ts +23 -0
- package/dist/screen/fixtures/probe-corpora.d.ts.map +1 -0
- package/dist/screen/fixtures/probe-corpora.js +69 -0
- package/dist/screen/index.d.ts +21 -0
- package/dist/screen/index.d.ts.map +1 -0
- package/dist/screen/index.js +64 -0
- package/dist/screen/model-screen.d.ts +111 -0
- package/dist/screen/model-screen.d.ts.map +1 -0
- package/dist/screen/model-screen.js +178 -0
- package/dist/screen/patterns.d.ts +51 -0
- package/dist/screen/patterns.d.ts.map +1 -0
- package/dist/screen/patterns.js +151 -0
- package/dist/screen/phi-gate.d.ts +52 -0
- package/dist/screen/phi-gate.d.ts.map +1 -0
- package/dist/screen/phi-gate.js +86 -0
- package/dist/screen/policy.d.ts +71 -0
- package/dist/screen/policy.d.ts.map +1 -0
- package/dist/screen/policy.js +334 -0
- package/dist/screen/probe.d.ts +118 -0
- package/dist/screen/probe.d.ts.map +1 -0
- package/dist/screen/probe.js +132 -0
- package/dist/screen/screen-cli.d.ts +16 -0
- package/dist/screen/screen-cli.d.ts.map +1 -0
- package/dist/screen/screen-cli.js +178 -0
- package/dist/screen/screen.d.ts +99 -0
- package/dist/screen/screen.d.ts.map +1 -0
- package/dist/screen/screen.js +0 -0
- package/dist/screen/sidecar.d.ts +69 -0
- package/dist/screen/sidecar.d.ts.map +1 -0
- package/dist/screen/sidecar.js +145 -0
- package/dist/screen/types.d.ts +43 -2
- package/dist/screen/types.d.ts.map +1 -1
- package/dist/settings/api-fetcher.d.ts +14 -3
- package/dist/settings/api-fetcher.d.ts.map +1 -1
- package/dist/settings/api-fetcher.js +61 -10
- package/dist/settings/grading-guide.d.ts +102 -0
- package/dist/settings/grading-guide.d.ts.map +1 -0
- package/dist/settings/grading-guide.js +23 -0
- package/dist/settings/index.d.ts +3 -2
- package/dist/settings/index.d.ts.map +1 -1
- package/dist/settings/index.js +2 -1
- package/dist/settings/task-settings.d.ts +44 -7
- package/dist/settings/task-settings.d.ts.map +1 -1
- package/dist/settings/task-settings.js +53 -12
- package/dist/status.d.ts +32 -20
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +51 -30
- package/dist/telemetry/artifact-store.d.ts +5 -6
- package/dist/telemetry/artifact-store.d.ts.map +1 -1
- package/dist/telemetry/artifact-store.js +21 -24
- package/dist/telemetry/attempt-meta.d.ts +76 -1
- package/dist/telemetry/attempt-meta.d.ts.map +1 -1
- package/dist/telemetry/attempt-meta.js +33 -1
- package/dist/telemetry/exporter-setup.d.ts +2 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -1
- package/dist/telemetry/exporter-setup.js +6 -1
- package/dist/telemetry/exporter.d.ts +2 -2
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +22 -41
- package/dist/telemetry/grade-mappers.d.ts +25 -0
- package/dist/telemetry/grade-mappers.d.ts.map +1 -0
- package/dist/telemetry/grade-mappers.js +64 -0
- package/dist/telemetry/grades-sql.d.ts +56 -0
- package/dist/telemetry/grades-sql.d.ts.map +1 -0
- package/dist/telemetry/grades-sql.js +156 -0
- package/dist/telemetry/ingest-protocol.d.ts +55 -48
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +48 -83
- package/dist/telemetry/org-identity.d.ts +44 -0
- package/dist/telemetry/org-identity.d.ts.map +1 -0
- package/dist/telemetry/org-identity.js +98 -0
- package/dist/telemetry/route-registration.d.ts +48 -0
- package/dist/telemetry/route-registration.d.ts.map +1 -0
- package/dist/telemetry/route-registration.js +77 -0
- package/dist/telemetry/sinks.d.ts +10 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +52 -6
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +64 -0
- package/dist/telemetry/store.d.ts +9 -6
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +8 -50
- package/dist/tenancy/api-key-format.d.ts +45 -1
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +50 -1
- package/dist/types/provider.d.ts +23 -3
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +26 -0
- package/dist/types/route.d.ts +154 -2
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +73 -48
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +32 -0
- package/dist/verify/verify.d.ts +5 -0
- package/dist/verify/verify.d.ts.map +1 -1
- package/dist/verify/verify.js +2 -0
- package/dist/watch/evaluate.d.ts +84 -1
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +28 -0
- package/dist/watch/monitor.d.ts +8 -2
- package/dist/watch/monitor.d.ts.map +1 -1
- package/dist/watch/monitor.js +13 -5
- package/dist/watch/regret-monitor-cli.d.ts +16 -0
- package/dist/watch/regret-monitor-cli.d.ts.map +1 -0
- package/dist/watch/regret-monitor-cli.js +137 -0
- package/dist/watch/regret-monitor.d.ts +168 -0
- package/dist/watch/regret-monitor.d.ts.map +1 -0
- package/dist/watch/regret-monitor.js +304 -0
- package/dist/watch/upgrade.d.ts +62 -0
- package/dist/watch/upgrade.d.ts.map +1 -0
- package/dist/watch/upgrade.js +57 -0
- package/docs/agent-skill/SKILL.md +181 -9
- package/docs/how-it-fits.md +3 -2
- package/docs/instrumentation.md +219 -46
- package/docs/llms.txt +158 -35
- package/docs/managed-serving.md +150 -0
- package/docs/mcp-oracle.md +322 -0
- package/docs/migration-playbook.md +5 -4
- package/docs/quickstart.md +263 -89
- package/package.json +18 -4
- package/registry/leaderboard.json +74 -0
- package/registry/registry.json +232 -112
- package/screen.d.ts +16 -0
- package/templates/call-disposition-qa.json +77 -0
- package/templates/catalog-cleansing.json +77 -0
- package/templates/compliance-review.json +76 -0
- package/templates/cs-next-action.json +76 -0
- package/templates/document-extraction.json +78 -0
- package/templates/financial-classification.json +75 -0
- package/templates/lead-qualification.json +76 -0
- package/templates/medical-classification.json +71 -0
- package/templates/ticket-triage.json +76 -0
- package/dist/feedback/feedback.d.ts +0 -35
- package/dist/feedback/feedback.d.ts.map +0 -1
- package/dist/feedback/feedback.js +0 -39
package/dist/run/run.js
CHANGED
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
* ruling 2 / finding F3).
|
|
9
9
|
*/
|
|
10
10
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.USER_DIRECTIVE = void 0;
|
|
12
|
+
exports.userTurnFor = userTurnFor;
|
|
11
13
|
exports.run = run;
|
|
12
14
|
const loader_1 = require("../routes/loader");
|
|
13
15
|
const serveability_1 = require("../routes/serveability");
|
|
@@ -18,9 +20,12 @@ const failures_1 = require("../types/failures");
|
|
|
18
20
|
const manifest_1 = require("../routes/manifest");
|
|
19
21
|
const resolve_1 = require("./resolve");
|
|
20
22
|
const variant_1 = require("./variant");
|
|
23
|
+
const resolve_destination_1 = require("./resolve-destination");
|
|
24
|
+
const serve_client_1 = require("../gateway/serve-client");
|
|
21
25
|
const envelope_1 = require("./envelope");
|
|
22
26
|
const retry_1 = require("./retry");
|
|
23
27
|
const repair_1 = require("./repair");
|
|
28
|
+
const repair_attempt_1 = require("./repair-attempt");
|
|
24
29
|
const grounding_1 = require("./grounding");
|
|
25
30
|
const validate_1 = require("./validate");
|
|
26
31
|
const dispatch_1 = require("./dispatch");
|
|
@@ -34,10 +39,18 @@ const capture_write_1 = require("./capture-write");
|
|
|
34
39
|
/** Phase 1 prompt convention: the bundle's system template carries the whole
|
|
35
40
|
* task (constitution, data, step instructions); the user turn is a terse
|
|
36
41
|
* trigger. Providers treat long system prompts + short user turns well, and
|
|
37
|
-
* it keeps the single-template bundle contract of spec §3.
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
42
|
+
* it keeps the single-template bundle contract of spec §3. Issue #5 (R1): a
|
|
43
|
+
* route that declares `prompt.user` replaces this trigger with its rendered
|
|
44
|
+
* user template — the per-call delta a cache handle cannot carry in the system
|
|
45
|
+
* slot (see `userTurnFor`). */
|
|
46
|
+
exports.USER_DIRECTIVE = "Produce the required output now, following the system instructions exactly.";
|
|
47
|
+
/** The user turn for one primary attempt (issue #5, R1): the route's rendered
|
|
48
|
+
* `prompt.user` template when declared — same variables, capability
|
|
49
|
+
* conditionals and `x-trust` removal as the system template — else the fixed
|
|
50
|
+
* trigger (byte-identical to pre-R1). Pure; exported for the run tests. */
|
|
51
|
+
function userTurnFor(route, ctx) {
|
|
52
|
+
return route.userTemplate === undefined ? exports.USER_DIRECTIVE : (0, template_1.renderTemplate)(route.userTemplate, ctx);
|
|
53
|
+
}
|
|
41
54
|
/** prefer:[cost] ordering metric — combined input+output USD per million
|
|
42
55
|
* tokens from the pricing snapshot; null (unknown) sorts last in resolve. */
|
|
43
56
|
function costPerMTok(provider, model) {
|
|
@@ -46,13 +59,14 @@ function costPerMTok(provider, model) {
|
|
|
46
59
|
return null;
|
|
47
60
|
return (pricing.inputUsdPerTok + pricing.outputUsdPerTok) * 1000000;
|
|
48
61
|
}
|
|
49
|
-
const DEFAULT_SLEEP = (ms) => ms <= 0 ? Promise.resolve() : new Promise((resolve) => setTimeout(resolve, ms));
|
|
62
|
+
const DEFAULT_SLEEP = (ms) => (ms <= 0 ? Promise.resolve() : new Promise((resolve) => setTimeout(resolve, ms)));
|
|
50
63
|
async function run(deps, routeName, opts) {
|
|
51
64
|
const mode = deps.mode ?? "live";
|
|
52
65
|
const baseRoute = deps.routes.get(routeName);
|
|
53
66
|
const variant = (0, variant_1.resolveVariant)(baseRoute, opts.variant, mode);
|
|
54
67
|
const servedVariant = variant?.name ?? null;
|
|
55
68
|
const route = (0, resolve_1.applyRunRequirements)((0, variant_1.effectiveRoute)(baseRoute, variant), opts.zeroRetention === true);
|
|
69
|
+
const destination = (0, resolve_destination_1.resolveDestination)(route.destination, opts.destination); // U2 (D-2): RunOptions > route > `direct`. Guard runs below, once the serving provider is resolved (U3).
|
|
56
70
|
// RC-1: inside a run context the SDK stamps `run_id` (episodeKey, else run id)
|
|
57
71
|
// when the caller passed none — the bridge between tags and the run graph.
|
|
58
72
|
const runCtx = (0, run_context_1.getActiveRunContext)();
|
|
@@ -96,153 +110,30 @@ async function run(deps, routeName, opts) {
|
|
|
96
110
|
autoStepKey,
|
|
97
111
|
cache: opts.cache,
|
|
98
112
|
droppedCacheHandle,
|
|
99
|
-
ladder,
|
|
113
|
+
ladder, destination, ...(opts.arm !== undefined ? { arm: opts.arm } : {}),
|
|
100
114
|
});
|
|
101
|
-
function failRun(candidate, failure, latencyMs) {
|
|
115
|
+
function failRun(candidate, failure, latencyMs, screen) {
|
|
116
|
+
// N1 (spec §5, SC-1b req 2): a pre-dispatch enforce `block` must still record
|
|
117
|
+
// its counts-only screen receipt on `meta.screen`, or screen-stats misses
|
|
118
|
+
// every pre-dispatch block. Absent ⇒ no key ⇒ byte-identical to a plain fail.
|
|
102
119
|
recordAttempt({
|
|
103
|
-
candidate,
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
tokensOut: 0,
|
|
107
|
-
tokensCached: 0,
|
|
108
|
-
costUsd: 0,
|
|
109
|
-
latencyMs,
|
|
110
|
-
failure,
|
|
120
|
+
candidate, servedTier: route.policy.tier, tokensIn: 0, tokensOut: 0,
|
|
121
|
+
tokensCached: 0, costUsd: 0, latencyMs, failure,
|
|
122
|
+
...(screen !== undefined ? { screen } : {}),
|
|
111
123
|
});
|
|
112
124
|
throw new failures_1.RigFailureError(failure);
|
|
113
125
|
}
|
|
114
126
|
// Repair rung (WS2): variant repair config wins over the route's; its own
|
|
115
127
|
// budget, never fungible with the content_invalid class budget.
|
|
116
128
|
const repairLedger = (0, repair_1.createRepairLedger)(variant?.repair ?? baseRoute.repair);
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
systemPrompt = originalSystemPrompt + (0, repair_1.repairFeedbackBlock)(errors);
|
|
126
|
-
}
|
|
127
|
-
else {
|
|
128
|
-
// Declare-first backstop (spec §4.4): the loader rejects a `trial` candidate
|
|
129
|
-
// as `repair_model`, but a programmatically-built bundle (runBundle) could
|
|
130
|
-
// bypass the loader — so refuse to mint a trial candidate for LIVE repair
|
|
131
|
-
// here too (the primary ladder's trial-drop has no repair-path equivalent).
|
|
132
|
-
if (mode === "live" &&
|
|
133
|
-
route.candidates.some((c) => c.provider === step.model.provider && c.model === step.model.model && c.trial === true)) {
|
|
134
|
-
return null;
|
|
135
|
-
}
|
|
136
|
-
// Brand stays in resolve.ts; loader validated repairModel ⊆ declared.
|
|
137
|
-
const minted = (0, resolve_1.resolveDeclaredCandidate)(route, step.model);
|
|
138
|
-
if (minted === null)
|
|
139
|
-
return null;
|
|
140
|
-
repairCandidate = minted;
|
|
141
|
-
systemPrompt = (0, repair_1.repairModelPrompt)(invalidText, errors, route.outputSchema);
|
|
142
|
-
}
|
|
143
|
-
const repairedBy = `${repairCandidate.provider}/${repairCandidate.model}`;
|
|
144
|
-
const adapter = deps.adapters.get(repairCandidate.provider);
|
|
145
|
-
if (adapter === undefined)
|
|
146
|
-
return null;
|
|
147
|
-
if (guard !== null) {
|
|
148
|
-
// A1: unpriced repair models fail closed exactly like primaries.
|
|
149
|
-
const breach = guard.precheck((0, cost_1.dispatchEstimateUsd)(repairCandidate.provider, repairCandidate.model, systemPrompt.length + REPAIR_DIRECTIVE.length, unpricedRatePerTok));
|
|
150
|
-
if (breach !== null) {
|
|
151
|
-
failRun(repairCandidate, breach, 0);
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
const started = Date.now();
|
|
155
|
-
const result = await adapter.call((0, dispatch_1.buildRepairAdapterRequest)(route, repairCandidate, systemPrompt, REPAIR_DIRECTIVE));
|
|
156
|
-
const latencyMs = Date.now() - started;
|
|
157
|
-
if (!result.ok) {
|
|
158
|
-
let cost = 0;
|
|
159
|
-
let repairFailurePricingMissing = false;
|
|
160
|
-
if (result.usage !== undefined) {
|
|
161
|
-
const settled = (0, cost_1.attemptCost)(repairCandidate.provider, repairCandidate.model, result.usage, unpricedRatePerTok);
|
|
162
|
-
cost = settled.costUsd;
|
|
163
|
-
repairFailurePricingMissing = settled.pricingMissing;
|
|
164
|
-
guard?.settle(settled.envelopeChargeUsd);
|
|
165
|
-
}
|
|
166
|
-
recordAttempt({
|
|
167
|
-
candidate: repairCandidate,
|
|
168
|
-
servedTier: result.usage?.servedTier ?? route.policy.tier,
|
|
169
|
-
tokensIn: result.usage?.tokensIn ?? 0,
|
|
170
|
-
tokensOut: result.usage?.tokensOut ?? 0,
|
|
171
|
-
tokensCached: result.usage?.tokensCached ?? 0,
|
|
172
|
-
tokensCacheWrite: result.usage?.tokensCacheWrite ?? 0,
|
|
173
|
-
tokensCacheWrite1h: result.usage?.tokensCacheWrite1h ?? 0,
|
|
174
|
-
costUsd: cost,
|
|
175
|
-
latencyMs,
|
|
176
|
-
...(result.usage?.ttfbMs !== undefined ? { ttfbMs: result.usage.ttfbMs } : {}),
|
|
177
|
-
failure: result.failure,
|
|
178
|
-
repairedBy,
|
|
179
|
-
pricingMissing: repairFailurePricingMissing,
|
|
180
|
-
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: REPAIR_DIRECTIVE, outputSchema: route.outputSchema, result }),
|
|
181
|
-
});
|
|
182
|
-
if (failures_1.TERMINAL_CLASSES.has(result.failure.class)) {
|
|
183
|
-
throw new failures_1.RigFailureError(result.failure);
|
|
184
|
-
}
|
|
185
|
-
return null;
|
|
186
|
-
}
|
|
187
|
-
const { costUsd, envelopeChargeUsd, pricingMissing: repairPricingMissing, } = (0, cost_1.attemptCost)(repairCandidate.provider, repairCandidate.model, result.usage, unpricedRatePerTok);
|
|
188
|
-
guard?.settle(envelopeChargeUsd);
|
|
189
|
-
const outcome = (0, validate_1.validateOutput)(route, result.text);
|
|
190
|
-
const failure = outcome.valid
|
|
191
|
-
? null
|
|
192
|
-
: {
|
|
193
|
-
class: "content_invalid",
|
|
194
|
-
message: `repair output failed schema validation: ${outcome.errors.join("; ")}`,
|
|
195
|
-
provider: repairCandidate.provider,
|
|
196
|
-
model: repairCandidate.model,
|
|
197
|
-
};
|
|
198
|
-
const inferenceId = recordAttempt({
|
|
199
|
-
candidate: repairCandidate,
|
|
200
|
-
servedTier: result.usage.servedTier,
|
|
201
|
-
tokensIn: result.usage.tokensIn,
|
|
202
|
-
tokensOut: result.usage.tokensOut,
|
|
203
|
-
tokensCached: result.usage.tokensCached,
|
|
204
|
-
tokensCacheWrite: result.usage.tokensCacheWrite ?? 0,
|
|
205
|
-
tokensCacheWrite1h: result.usage.tokensCacheWrite1h ?? 0,
|
|
206
|
-
costUsd,
|
|
207
|
-
latencyMs,
|
|
208
|
-
...(result.usage.ttfbMs !== undefined ? { ttfbMs: result.usage.ttfbMs } : {}),
|
|
209
|
-
failure,
|
|
210
|
-
repairedBy,
|
|
211
|
-
pricingMissing: repairPricingMissing,
|
|
212
|
-
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: REPAIR_DIRECTIVE, outputSchema: route.outputSchema, result }),
|
|
213
|
-
});
|
|
214
|
-
captureAttempt(inferenceId, result.text);
|
|
215
|
-
if (!outcome.valid) {
|
|
216
|
-
return { invalid: { text: result.text, errors: outcome.errors } };
|
|
217
|
-
}
|
|
218
|
-
return {
|
|
219
|
-
done: {
|
|
220
|
-
output: outcome.parsed,
|
|
221
|
-
meta: {
|
|
222
|
-
inferenceId,
|
|
223
|
-
route: route.route,
|
|
224
|
-
routeVersion: route.version,
|
|
225
|
-
provider: repairCandidate.provider,
|
|
226
|
-
model: repairCandidate.model,
|
|
227
|
-
requestedTier: route.policy.tier,
|
|
228
|
-
servedTier: result.usage.servedTier,
|
|
229
|
-
tokensIn: result.usage.tokensIn,
|
|
230
|
-
tokensOut: result.usage.tokensOut,
|
|
231
|
-
tokensCached: result.usage.tokensCached,
|
|
232
|
-
costEstimateUsd: costUsd,
|
|
233
|
-
latencyMs,
|
|
234
|
-
validated: true,
|
|
235
|
-
attemptsByClass: ledger.snapshot(),
|
|
236
|
-
servedVariant,
|
|
237
|
-
repairedBy,
|
|
238
|
-
callId: ladder.callId,
|
|
239
|
-
...(injectedSources !== null
|
|
240
|
-
? (0, grounding_1.groundingMeta)(result.text, injectedSources)
|
|
241
|
-
: {}),
|
|
242
|
-
},
|
|
243
|
-
},
|
|
244
|
-
};
|
|
245
|
-
}
|
|
129
|
+
// Repair dispatch (WS2), extracted to ./repair-attempt.ts (U4) to keep run.ts
|
|
130
|
+
// under the LOC cap — a factory over this run's per-call state, mirroring
|
|
131
|
+
// createRecordAttempt. The run loop's current `injectedSources` is passed in at
|
|
132
|
+
// the call site (below) rather than closed over.
|
|
133
|
+
const attemptRepair = (0, repair_attempt_1.createAttemptRepair)({
|
|
134
|
+
adapters: deps.adapters, guard, unpricedRatePerTok, route, mode,
|
|
135
|
+
recordAttempt, captureAttempt, ledger, servedVariant, ladder, destination, failRun,
|
|
136
|
+
});
|
|
246
137
|
const searchAvailable = deps.search !== undefined && deps.search !== null;
|
|
247
138
|
const iter = (0, resolve_1.resolveCandidates)(route, deps.capabilities, costPerMTok, searchAvailable, (0, resolve_1.candidateFilterFromOptions)(opts), undefined, mode);
|
|
248
139
|
// Advance the iterator AND publish the served candidate's index to the ladder,
|
|
@@ -257,12 +148,34 @@ async function run(deps, routeName, opts) {
|
|
|
257
148
|
throw new loader_1.RouteConfigError(`route "${route.route}" has no eligible candidate — none of the declared candidates ` +
|
|
258
149
|
`satisfy require=[${route.require.join(", ")}] with json=${route.policy.json}${(0, resolve_1.filterSummary)(opts)}`);
|
|
259
150
|
}
|
|
151
|
+
(0, resolve_destination_1.assertDestinationServable)(destination, deps.managedServingEnabled === true, candidate.provider); // U3 (D-2): serving guard, now the provider is resolved. `direct` passes (byte-identical); `modelrig-served` passes only with the flag ON AND a served-capable provider, else throws pre-dispatch (no provider call, no telemetry row).
|
|
152
|
+
// K1 (managed-serving-keyless §2/§3): the resolved destination is
|
|
153
|
+
// `modelrig-served` AND the flag is on (the guard above just confirmed a
|
|
154
|
+
// served-capable provider) ⇒ serve the run through the HOSTED gateway on
|
|
155
|
+
// ModelRig's managed key (serve-client.ts), NOT the local adapter loop. The
|
|
156
|
+
// gateway resolves the org's mirrored bundle BY SLUG and drives the SAME
|
|
157
|
+
// verified pipeline server-side; its provenance (provider/model/cost/
|
|
158
|
+
// inference-id/validated) is carried into `RunResult.meta`. No local telemetry
|
|
159
|
+
// row is written here — the authoritative inference row lives server-side and
|
|
160
|
+
// `meta.inferenceId` points at it (anti-cheat: the SDK never double-meters a
|
|
161
|
+
// server-measured call). Fail-closed-but-discoverable inside serve-client; no
|
|
162
|
+
// silent fall back to `direct`/local key. `direct` (and flag-off) NEVER reach
|
|
163
|
+
// this branch, so every other path is byte-identical to today.
|
|
164
|
+
if (destination === "modelrig-served" && deps.managedServingEnabled === true) {
|
|
165
|
+
return (0, serve_client_1.serveViaGateway)({
|
|
166
|
+
route,
|
|
167
|
+
opts,
|
|
168
|
+
servedVariant,
|
|
169
|
+
serve: deps.serve,
|
|
170
|
+
...(deps.serveFetch !== undefined ? { fetchFn: deps.serveFetch } : {}),
|
|
171
|
+
});
|
|
172
|
+
}
|
|
260
173
|
let lastFailure = null;
|
|
261
174
|
// Grounding-inject (WS3): one search per run, cached across candidates and
|
|
262
175
|
// retries; cost accrues to the envelope and its own telemetry row.
|
|
263
176
|
let searchResponse = null;
|
|
264
177
|
let injectedSources = null;
|
|
265
|
-
const
|
|
178
|
+
const guards = (0, dispatch_1.buildRunGuards)(route, vault_1.createRunVault); // CS-1 mask (run-scoped vault) + SC-1a screen; both null ⇒ byte-identical
|
|
266
179
|
while (candidate !== null) {
|
|
267
180
|
// Runtime layer of the candidate-set invariant: even a compromised
|
|
268
181
|
// iterator cannot reach an adapter with a pair outside the declared list.
|
|
@@ -289,10 +202,15 @@ async function run(deps, routeName, opts) {
|
|
|
289
202
|
}
|
|
290
203
|
// Render against the SERVING candidate's capability flags.
|
|
291
204
|
const flags = deps.capabilities(candidate.provider, candidate.model);
|
|
292
|
-
|
|
205
|
+
const renderCtx = {
|
|
293
206
|
variables: opts.input,
|
|
294
|
-
capabilities: flags, variableTrust: route.variableTrust,
|
|
295
|
-
}
|
|
207
|
+
capabilities: flags, variableTrust: (0, grounding_1.systemSlotRemoval)(route.variableTrust, opts.proxyTrust),
|
|
208
|
+
};
|
|
209
|
+
let systemPrompt = (0, template_1.renderTemplate)(route.systemTemplate, renderCtx);
|
|
210
|
+
// Issue #5 (R1): the user turn — the route's `prompt.user` when declared
|
|
211
|
+
// (rendered with the SAME context, so an untrusted variable is removed from
|
|
212
|
+
// it too and arrives via the spotlight block), else the fixed trigger.
|
|
213
|
+
const userPrompt = userTurnFor(route, renderCtx);
|
|
296
214
|
// Variant augmentation (WS1): appended after capability-conditionals,
|
|
297
215
|
// before json coaching — promptAppend first, then the reasoning scaffold.
|
|
298
216
|
if (variant?.promptAppend !== undefined) {
|
|
@@ -356,7 +274,7 @@ async function run(deps, routeName, opts) {
|
|
|
356
274
|
// pricing snapshot (prompt-length input + full output budget).
|
|
357
275
|
if (guard !== null) {
|
|
358
276
|
// A1: an unpriced model must never precheck at $0 (fail closed).
|
|
359
|
-
const breach = guard.precheck((0, cost_1.dispatchEstimateUsd)(candidate.provider, candidate.model, systemPrompt.length +
|
|
277
|
+
const breach = guard.precheck((0, cost_1.dispatchEstimateUsd)(candidate.provider, candidate.model, systemPrompt.length + userPrompt.length, unpricedRatePerTok));
|
|
360
278
|
if (breach !== null) {
|
|
361
279
|
failRun(candidate, breach, 0);
|
|
362
280
|
}
|
|
@@ -364,19 +282,16 @@ async function run(deps, routeName, opts) {
|
|
|
364
282
|
// CE-1 (spec §2.2, run/cache-handle.ts): `suppressed` drops the customer handle after a cache_invalid.
|
|
365
283
|
const candidateKey = `${candidate.provider} ${candidate.model}`;
|
|
366
284
|
const suppressed = droppedCacheHandle.has(candidateKey);
|
|
367
|
-
//
|
|
285
|
+
// Assemble untrusted block + screen (spec §4) + mask (spec §1.2 — user slot)
|
|
286
|
+
// + dispatch as ONE seam; both guards null ⇒ byte-identical. The untrusted
|
|
287
|
+
// appends + screening live in the seam so run.ts stays under the LOC cap.
|
|
368
288
|
const dispatched = await (0, dispatch_1.buildDispatchMaybeMasked)(adapter, {
|
|
369
|
-
route,
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
suppressCustomerCache: suppressed,
|
|
376
|
-
}, deps.onStreamDelta, maskCtx);
|
|
377
|
-
if (!dispatched.result.ok && dispatched.maskRefusal !== undefined)
|
|
378
|
-
failRun(candidate, dispatched.result.failure, 0);
|
|
379
|
-
const { result, latencyMs, seam: maskSeam } = dispatched;
|
|
289
|
+
route, candidate, systemPrompt, userDirective: userPrompt,
|
|
290
|
+
nativeGrounded, opts, suppressCustomerCache: suppressed, injectedSources,
|
|
291
|
+
}, deps.onStreamDelta, guards);
|
|
292
|
+
if (!dispatched.result.ok && (dispatched.maskRefusal !== undefined || dispatched.screenRefusal !== undefined))
|
|
293
|
+
failRun(candidate, dispatched.result.failure, 0, dispatched.screenReceipt);
|
|
294
|
+
const { result, latencyMs, seam: maskSeam, screenReceipt } = dispatched;
|
|
380
295
|
if (result.ok) {
|
|
381
296
|
const { costUsd, envelopeChargeUsd, pricingMissing } = (0, cost_1.attemptCost)(candidate.provider, candidate.model, result.usage, unpricedRatePerTok);
|
|
382
297
|
guard?.settle(envelopeChargeUsd);
|
|
@@ -392,12 +307,12 @@ async function run(deps, routeName, opts) {
|
|
|
392
307
|
attemptsByClass: ledger.snapshot(),
|
|
393
308
|
servedVariant,
|
|
394
309
|
pricingMissing,
|
|
395
|
-
callId: ladder.callId,
|
|
396
|
-
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective:
|
|
310
|
+
callId: ladder.callId, destination,
|
|
311
|
+
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: userPrompt, outputSchema: route.outputSchema, outputKind: route.outputKind, result, tools: opts.tools }),
|
|
397
312
|
});
|
|
398
313
|
}
|
|
399
314
|
const outcome = (0, validate_1.validateOutput)(route, result.text);
|
|
400
|
-
const metaInputs = (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective:
|
|
315
|
+
const metaInputs = (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: userPrompt, outputSchema: route.outputSchema, outputKind: route.outputKind, result, tools: opts.tools });
|
|
401
316
|
if (outcome.valid) {
|
|
402
317
|
// Q1 quality gate (spec §4.1): a schema-valid output must still clear the
|
|
403
318
|
// builder's bar before it is accepted. Tool-call turns never reach here;
|
|
@@ -426,23 +341,27 @@ async function run(deps, routeName, opts) {
|
|
|
426
341
|
const acceptTags = gate !== null ? (0, quality_gate_1.qualityTags)(gate) : undefined;
|
|
427
342
|
const output = maskSeam !== null ? maskSeam.rehydrateOutput(outcome.parsed) : outcome.parsed; // rehydrate post-validate (spec §5)
|
|
428
343
|
const maskReceipt = maskSeam?.receipt(); // read AFTER rehydrate so counts are final
|
|
429
|
-
const inferenceId = recordAttempt({ ...(0, accept_1.successAttemptRow)(candidate, result.usage, costUsd, latencyMs, pricingMissing, null, metaInputs, acceptTags), ...(maskReceipt !== undefined ? { mask: maskReceipt } : {}) });
|
|
344
|
+
const inferenceId = recordAttempt({ ...(0, accept_1.successAttemptRow)(candidate, result.usage, costUsd, latencyMs, pricingMissing, null, metaInputs, acceptTags), ...(maskReceipt !== undefined ? { mask: maskReceipt } : {}), ...(screenReceipt !== undefined ? { screen: screenReceipt } : {}) });
|
|
430
345
|
captureAttempt(inferenceId, result.text);
|
|
431
346
|
const meta = (0, accept_1.buildSuccessMeta)({
|
|
432
347
|
inferenceId, route, candidate, usage: result.usage, costUsd, latencyMs,
|
|
433
|
-
attemptsByClass: ledger.snapshot(), servedVariant, callId: ladder.callId,
|
|
348
|
+
attemptsByClass: ledger.snapshot(), servedVariant, callId: ladder.callId, destination,
|
|
434
349
|
...(injectedSources !== null ? (0, grounding_1.groundingMeta)(result.text, injectedSources) : {}),
|
|
435
350
|
...(gate !== null ? { quality: { score: gate.score, verdict: "pass", judgedBy: null } } : {}),
|
|
436
|
-
...(maskReceipt !== undefined ? { mask: maskReceipt } : {}),
|
|
351
|
+
...(maskReceipt !== undefined ? { mask: maskReceipt } : {}), ...(screenReceipt !== undefined ? { screen: screenReceipt } : {}),
|
|
437
352
|
});
|
|
438
353
|
return { output, meta };
|
|
439
354
|
}
|
|
440
|
-
// Schema-invalid output — a real (billed) attempt that failed.
|
|
355
|
+
// Schema-invalid output — a real (billed) attempt that failed. On an
|
|
356
|
+
// `output: text` route this is ONLY a declared min_chars miss (issue #1 —
|
|
357
|
+
// prose is never JSON-parsed); `fixHint` teaches when the CONTRACT is the
|
|
358
|
+
// cause (prose arriving on a json route ⇒ "declare output: text").
|
|
441
359
|
const failure = {
|
|
442
360
|
class: "content_invalid",
|
|
443
361
|
message: `output failed schema validation: ${outcome.errors.join("; ")}`,
|
|
444
362
|
provider: candidate.provider,
|
|
445
363
|
model: candidate.model,
|
|
364
|
+
...(outcome.fixHint !== undefined ? { fixHint: outcome.fixHint } : {}),
|
|
446
365
|
};
|
|
447
366
|
const invalidId = recordAttempt({
|
|
448
367
|
candidate,
|
|
@@ -463,13 +382,16 @@ async function run(deps, routeName, opts) {
|
|
|
463
382
|
lastFailure = failure;
|
|
464
383
|
// Repair rung (WS2): consumed BEFORE any content_invalid retry, from
|
|
465
384
|
// its own budget. Chain: same-model error feedback, then the
|
|
466
|
-
// designated repair model on the newest invalid output.
|
|
467
|
-
|
|
385
|
+
// designated repair model on the newest invalid output. JSON-only by
|
|
386
|
+
// construction: an `output: text` route has no schema (loader-enforced),
|
|
387
|
+
// so prose is never sent for JSON repair (issue #1) — a min_chars miss
|
|
388
|
+
// falls straight through to the content_invalid retry below.
|
|
389
|
+
if (route.outputSchema !== null && route.outputKind !== "text") {
|
|
468
390
|
let latestText = result.text;
|
|
469
391
|
let latestErrors = outcome.errors;
|
|
470
392
|
let repairStep = repairLedger.next();
|
|
471
393
|
while (repairStep !== null) {
|
|
472
|
-
const repairOutcome = await attemptRepair(repairStep, candidate, systemPrompt, latestText, latestErrors);
|
|
394
|
+
const repairOutcome = await attemptRepair(repairStep, candidate, systemPrompt, latestText, latestErrors, injectedSources, userPrompt);
|
|
473
395
|
if (repairOutcome?.done !== undefined) {
|
|
474
396
|
return repairOutcome.done;
|
|
475
397
|
}
|
|
@@ -510,7 +432,7 @@ async function run(deps, routeName, opts) {
|
|
|
510
432
|
...(result.usage?.ttfbMs !== undefined ? { ttfbMs: result.usage.ttfbMs } : {}),
|
|
511
433
|
failure,
|
|
512
434
|
pricingMissing: failurePricingMissing,
|
|
513
|
-
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective:
|
|
435
|
+
metaInputs: (0, dispatch_1.attemptMetaInputs)({ systemPrompt, userDirective: userPrompt, outputSchema: route.outputSchema, outputKind: route.outputKind, result, tools: opts.tools }),
|
|
514
436
|
});
|
|
515
437
|
if (failures_1.TERMINAL_CLASSES.has(failure.class)) {
|
|
516
438
|
throw new failures_1.RigFailureError(failure);
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ONE telemetry row builder both dispatch lanes feed (Unified Route U4,
|
|
3
|
+
* spec §4 U4 / plan U4 Step 1). Before U4 the persisted `inferences` row was
|
|
4
|
+
* assembled — and its `tagsJson` serialized — in FOUR places (research §1c):
|
|
5
|
+
* the routed recorder (`record-attempt.ts`), the raw recorder (`raw-finalize.ts`
|
|
6
|
+
* `recordRawRow`), the grounding-inject search-cost recorder (`dispatch.ts`
|
|
7
|
+
* `recordSearchCostRow`), and the native-search cost recorder
|
|
8
|
+
* (`raw-search-cost.ts`). A new cross-lane tag therefore cost four edits and
|
|
9
|
+
* drifted. This module collapses them onto one builder: every field is placed
|
|
10
|
+
* ONCE, and `tagsJson` is serialized in exactly ONE place — {@link stampRowTags}.
|
|
11
|
+
*
|
|
12
|
+
* BEHAVIOR-PRESERVING: `buildTelemetryRow` takes every field the caller already
|
|
13
|
+
* resolved (raw's former hardcodes — `routeVersion: 0`, `attemptsJson: "{}"`,
|
|
14
|
+
* `servedVariant: null`, `attemptSeq: 1`, `candidateIndex: 0`, `lane: "raw"` —
|
|
15
|
+
* are now EXPLICIT inputs, not literals hidden in one lane's recorder), so the
|
|
16
|
+
* emitted row is byte-identical to what each recorder produced before. `meta`
|
|
17
|
+
* and `mask` are placed only when the caller passes them (conditional spread),
|
|
18
|
+
* preserving each lane's exact key-presence (the search-cost row omits `meta`;
|
|
19
|
+
* the raw search-cost row sets `meta: null`).
|
|
20
|
+
*
|
|
21
|
+
* No route/registry/capability-gate dependency — types only — so the raw lane's
|
|
22
|
+
* no-hard-hot-path-dependency invariant is preserved when raw-finalize.ts and
|
|
23
|
+
* raw-search-cost.ts feed this builder.
|
|
24
|
+
*/
|
|
25
|
+
import type { FailureClass } from "../types/failures";
|
|
26
|
+
import type { RunTags } from "../types/route";
|
|
27
|
+
import type { AttemptMetaV1 } from "../telemetry/attempt-meta";
|
|
28
|
+
import type { MaskReceipt } from "../mask/types";
|
|
29
|
+
import type { TelemetryRow, TelemetryRung } from "../types/telemetry";
|
|
30
|
+
/**
|
|
31
|
+
* THE single point where a telemetry row's `tags_json` is serialized (U4). Every
|
|
32
|
+
* row writer stamps its tags here, so a NEW cross-lane tag is a ONE-FILE edit:
|
|
33
|
+
* add it to the object assembled below and it appears on every lane's rows at
|
|
34
|
+
* once (proven by `telemetry-row.test.ts`). Key order is preserved exactly as
|
|
35
|
+
* the former inline sites had it — `{ ...baseTags, ...extraTags, rung }` — and
|
|
36
|
+
* `rung` is stamped LAST so a caller's `extraTags` can never override the
|
|
37
|
+
* recorder-owned rung.
|
|
38
|
+
*/
|
|
39
|
+
export declare function stampRowTags(baseTags: RunTags | Record<string, string>, rung: TelemetryRung, extraTags?: Record<string, string>): string;
|
|
40
|
+
/** Inputs to the one row builder. Mirrors {@link TelemetryRow} field-for-field,
|
|
41
|
+
* except `tagsJson` is assembled from (`baseTags`, `extraTags?`, `rung`) via
|
|
42
|
+
* {@link stampRowTags}. `meta` / `mask` are placed only when present (undefined
|
|
43
|
+
* ⇒ the key is omitted), matching each caller's former shape exactly. */
|
|
44
|
+
export interface TelemetryRowInput {
|
|
45
|
+
readonly id: string;
|
|
46
|
+
readonly route: string;
|
|
47
|
+
readonly routeVersion: number;
|
|
48
|
+
readonly provider: string;
|
|
49
|
+
readonly model: string;
|
|
50
|
+
readonly requestedTier: string;
|
|
51
|
+
readonly servedTier: string;
|
|
52
|
+
readonly tokensIn: number;
|
|
53
|
+
readonly tokensOut: number;
|
|
54
|
+
readonly tokensCached: number;
|
|
55
|
+
readonly tokensCacheWrite: number;
|
|
56
|
+
readonly tokensCacheWrite1h: number;
|
|
57
|
+
readonly costEstimateUsd: number;
|
|
58
|
+
readonly latencyMs: number;
|
|
59
|
+
readonly ttfbMs: number | null;
|
|
60
|
+
readonly failureClass: FailureClass | null;
|
|
61
|
+
readonly attemptsJson: string;
|
|
62
|
+
/** Base tags stamped on the row (routed: `{...step, ...effectiveTags}`; raw:
|
|
63
|
+
* the lane tags incl. `lane:"raw"`). Serialized via {@link stampRowTags}. */
|
|
64
|
+
readonly baseTags: RunTags | Record<string, string>;
|
|
65
|
+
/** SDK-owned extra tags merged after `baseTags`, before `rung` (Q1 quality
|
|
66
|
+
* tags on the routed accept path). Absent on every other site. */
|
|
67
|
+
readonly extraTags?: Record<string, string>;
|
|
68
|
+
readonly rung: TelemetryRung;
|
|
69
|
+
readonly fingerprintCell: string | null;
|
|
70
|
+
readonly servedVariant: string | null;
|
|
71
|
+
readonly repairedBy: string | null;
|
|
72
|
+
readonly pricingMissing: boolean;
|
|
73
|
+
readonly cacheKey: string | null;
|
|
74
|
+
readonly prefixFingerprint: string | null;
|
|
75
|
+
readonly callId: string | null;
|
|
76
|
+
readonly attemptSeq: number | null;
|
|
77
|
+
readonly candidateIndex: number | null;
|
|
78
|
+
/** WI-1 attempt envelope. Placed only when present — the grounding-inject
|
|
79
|
+
* search-cost row omits it entirely; the raw search-cost row sets it to null. */
|
|
80
|
+
readonly meta?: AttemptMetaV1 | null;
|
|
81
|
+
/** CS-1 mask receipt. Placed only when present (routed masked route only). */
|
|
82
|
+
readonly mask?: MaskReceipt;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Assemble the persisted `inferences` row (U4). One object literal, one
|
|
86
|
+
* `tags_json` serialization ({@link stampRowTags}). `ts` is stamped here (all
|
|
87
|
+
* former recorders stamped `new Date().toISOString()` at row-build time — this
|
|
88
|
+
* is the same instant). `meta` / `mask` are placed only when the caller passes
|
|
89
|
+
* them, preserving each lane's exact key-presence.
|
|
90
|
+
*/
|
|
91
|
+
export declare function buildTelemetryRow(input: TelemetryRowInput): TelemetryRow;
|
|
92
|
+
//# sourceMappingURL=telemetry-row.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"telemetry-row.d.ts","sourceRoot":"","sources":["../../src/run/telemetry-row.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEtE;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC1C,IAAI,EAAE,aAAa,EACnB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACjC,MAAM,CAER;AAED;;;yEAGyE;AACzE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;iFAC6E;IAC7E,QAAQ,CAAC,QAAQ,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpD;sEACkE;IAClE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC;qFACiF;IACjF,QAAQ,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACrC,8EAA8E;IAC9E,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,iBAAiB,GAAG,YAAY,CAiCxE"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The ONE telemetry row builder both dispatch lanes feed (Unified Route U4,
|
|
4
|
+
* spec §4 U4 / plan U4 Step 1). Before U4 the persisted `inferences` row was
|
|
5
|
+
* assembled — and its `tagsJson` serialized — in FOUR places (research §1c):
|
|
6
|
+
* the routed recorder (`record-attempt.ts`), the raw recorder (`raw-finalize.ts`
|
|
7
|
+
* `recordRawRow`), the grounding-inject search-cost recorder (`dispatch.ts`
|
|
8
|
+
* `recordSearchCostRow`), and the native-search cost recorder
|
|
9
|
+
* (`raw-search-cost.ts`). A new cross-lane tag therefore cost four edits and
|
|
10
|
+
* drifted. This module collapses them onto one builder: every field is placed
|
|
11
|
+
* ONCE, and `tagsJson` is serialized in exactly ONE place — {@link stampRowTags}.
|
|
12
|
+
*
|
|
13
|
+
* BEHAVIOR-PRESERVING: `buildTelemetryRow` takes every field the caller already
|
|
14
|
+
* resolved (raw's former hardcodes — `routeVersion: 0`, `attemptsJson: "{}"`,
|
|
15
|
+
* `servedVariant: null`, `attemptSeq: 1`, `candidateIndex: 0`, `lane: "raw"` —
|
|
16
|
+
* are now EXPLICIT inputs, not literals hidden in one lane's recorder), so the
|
|
17
|
+
* emitted row is byte-identical to what each recorder produced before. `meta`
|
|
18
|
+
* and `mask` are placed only when the caller passes them (conditional spread),
|
|
19
|
+
* preserving each lane's exact key-presence (the search-cost row omits `meta`;
|
|
20
|
+
* the raw search-cost row sets `meta: null`).
|
|
21
|
+
*
|
|
22
|
+
* No route/registry/capability-gate dependency — types only — so the raw lane's
|
|
23
|
+
* no-hard-hot-path-dependency invariant is preserved when raw-finalize.ts and
|
|
24
|
+
* raw-search-cost.ts feed this builder.
|
|
25
|
+
*/
|
|
26
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
27
|
+
exports.stampRowTags = stampRowTags;
|
|
28
|
+
exports.buildTelemetryRow = buildTelemetryRow;
|
|
29
|
+
/**
|
|
30
|
+
* THE single point where a telemetry row's `tags_json` is serialized (U4). Every
|
|
31
|
+
* row writer stamps its tags here, so a NEW cross-lane tag is a ONE-FILE edit:
|
|
32
|
+
* add it to the object assembled below and it appears on every lane's rows at
|
|
33
|
+
* once (proven by `telemetry-row.test.ts`). Key order is preserved exactly as
|
|
34
|
+
* the former inline sites had it — `{ ...baseTags, ...extraTags, rung }` — and
|
|
35
|
+
* `rung` is stamped LAST so a caller's `extraTags` can never override the
|
|
36
|
+
* recorder-owned rung.
|
|
37
|
+
*/
|
|
38
|
+
function stampRowTags(baseTags, rung, extraTags) {
|
|
39
|
+
return JSON.stringify({ ...baseTags, ...(extraTags ?? {}), rung });
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Assemble the persisted `inferences` row (U4). One object literal, one
|
|
43
|
+
* `tags_json` serialization ({@link stampRowTags}). `ts` is stamped here (all
|
|
44
|
+
* former recorders stamped `new Date().toISOString()` at row-build time — this
|
|
45
|
+
* is the same instant). `meta` / `mask` are placed only when the caller passes
|
|
46
|
+
* them, preserving each lane's exact key-presence.
|
|
47
|
+
*/
|
|
48
|
+
function buildTelemetryRow(input) {
|
|
49
|
+
return {
|
|
50
|
+
id: input.id,
|
|
51
|
+
ts: new Date().toISOString(),
|
|
52
|
+
route: input.route,
|
|
53
|
+
routeVersion: input.routeVersion,
|
|
54
|
+
provider: input.provider,
|
|
55
|
+
model: input.model,
|
|
56
|
+
requestedTier: input.requestedTier,
|
|
57
|
+
servedTier: input.servedTier,
|
|
58
|
+
tokensIn: input.tokensIn,
|
|
59
|
+
tokensOut: input.tokensOut,
|
|
60
|
+
tokensCached: input.tokensCached,
|
|
61
|
+
tokensCacheWrite: input.tokensCacheWrite,
|
|
62
|
+
tokensCacheWrite1h: input.tokensCacheWrite1h,
|
|
63
|
+
costEstimateUsd: input.costEstimateUsd,
|
|
64
|
+
latencyMs: input.latencyMs,
|
|
65
|
+
ttfbMs: input.ttfbMs,
|
|
66
|
+
failureClass: input.failureClass,
|
|
67
|
+
attemptsJson: input.attemptsJson,
|
|
68
|
+
tagsJson: stampRowTags(input.baseTags, input.rung, input.extraTags),
|
|
69
|
+
fingerprintCell: input.fingerprintCell,
|
|
70
|
+
servedVariant: input.servedVariant,
|
|
71
|
+
repairedBy: input.repairedBy,
|
|
72
|
+
pricingMissing: input.pricingMissing,
|
|
73
|
+
cacheKey: input.cacheKey,
|
|
74
|
+
prefixFingerprint: input.prefixFingerprint,
|
|
75
|
+
callId: input.callId,
|
|
76
|
+
attemptSeq: input.attemptSeq,
|
|
77
|
+
candidateIndex: input.candidateIndex,
|
|
78
|
+
...(input.meta !== undefined ? { meta: input.meta } : {}),
|
|
79
|
+
...(input.mask !== undefined ? { mask: input.mask } : {}),
|
|
80
|
+
};
|
|
81
|
+
}
|
package/dist/run/validate.d.ts
CHANGED
|
@@ -2,14 +2,25 @@
|
|
|
2
2
|
* Output validation — plan §4. Ajv compiled once per (route, version), cached.
|
|
3
3
|
* Phase 1 checks schema conformance; value-accuracy gates arrive with the
|
|
4
4
|
* bake-off in Phase 3.
|
|
5
|
+
*
|
|
6
|
+
* Two output kinds (issue #1, incident 2026-10-11):
|
|
7
|
+
* json (default) — JSON.parse, then ajv against `outputSchema` when declared.
|
|
8
|
+
* A schema-less json route still requires parseable JSON (unchanged).
|
|
9
|
+
* text — NO parse: `parsed` is the response string VERBATIM
|
|
10
|
+
* (untrimmed). The only check is the declared `min_chars` floor.
|
|
5
11
|
*/
|
|
6
12
|
import type { RouteBundle } from "../types/route";
|
|
7
13
|
export interface ValidationOutcome {
|
|
8
14
|
readonly valid: boolean;
|
|
9
15
|
/** ajv error summaries, capped, human- and agent-readable. */
|
|
10
16
|
readonly errors: readonly string[];
|
|
11
|
-
/** Parsed output when text was JSON-parseable (even if schema-invalid)
|
|
17
|
+
/** Parsed output when text was JSON-parseable (even if schema-invalid); the
|
|
18
|
+
* verbatim string on an `output: text` route. */
|
|
12
19
|
readonly parsed: unknown;
|
|
20
|
+
/** P4 (errors teach): the one-line fix for a failure whose cause is the
|
|
21
|
+
* route CONTRACT rather than the model — carried onto the `content_invalid`
|
|
22
|
+
* RigFailure as `fixHint`. */
|
|
23
|
+
readonly fixHint?: string;
|
|
13
24
|
}
|
|
14
25
|
export declare function validateOutput(route: RouteBundle, text: string): ValidationOutcome;
|
|
15
26
|
//# sourceMappingURL=validate.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../src/run/validate.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../src/run/validate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAIlD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,8DAA8D;IAC9D,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC;qDACiD;IACjD,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB;;kCAE8B;IAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAkDD,wBAAgB,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,GAAG,iBAAiB,CA+BlF"}
|