modelrig 0.8.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +324 -0
- package/README.md +55 -29
- package/data/PROVENANCE.md +14 -0
- package/data/pricing-map.json +26 -0
- package/dist/adapters/attempt-meta-map.d.ts +8 -1
- package/dist/adapters/attempt-meta-map.d.ts.map +1 -1
- package/dist/adapters/attempt-meta-map.js +4 -0
- package/dist/adapters/build-adapters.js +2 -2
- package/dist/adapters/gemini.d.ts +28 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +38 -9
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +53 -4
- package/dist/adapters/openai-compat.d.ts +8 -2
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +8 -1
- package/dist/adapters/openai-stream.d.ts +10 -2
- package/dist/adapters/openai-stream.d.ts.map +1 -1
- package/dist/adapters/openai-stream.js +5 -1
- package/dist/adapters/openai.d.ts +10 -1
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +57 -7
- package/dist/adapters/responses-api.d.ts +61 -0
- package/dist/adapters/responses-api.d.ts.map +1 -0
- package/dist/adapters/responses-api.js +124 -0
- package/dist/artifacts/artifact.d.ts +31 -12
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +218 -188
- package/dist/artifacts/index.d.ts +15 -5
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +19 -5
- package/dist/artifacts/run-context.d.ts +3 -24
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +3 -39
- package/dist/bakeoff/engine.d.ts +29 -0
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +47 -0
- package/dist/bakeoff/eval-cases-rig.d.ts +36 -0
- package/dist/bakeoff/eval-cases-rig.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases-rig.js +27 -0
- package/dist/bakeoff/eval-cases.d.ts +19 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -1
- package/dist/bakeoff/eval-cases.js +18 -1
- package/dist/bakeoff/golden-judge.d.ts +47 -0
- package/dist/bakeoff/golden-judge.d.ts.map +1 -0
- package/dist/bakeoff/golden-judge.js +49 -0
- package/dist/bakeoff/groundedness.d.ts +58 -0
- package/dist/bakeoff/groundedness.d.ts.map +1 -0
- package/dist/bakeoff/groundedness.js +165 -0
- package/dist/bakeoff/quality.d.ts +43 -0
- package/dist/bakeoff/quality.d.ts.map +1 -1
- package/dist/bakeoff/quality.js +31 -0
- package/dist/bakeoff/replay.d.ts +11 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +12 -0
- package/dist/bakeoff/report.d.ts +7 -0
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +26 -0
- package/dist/cli-judge.d.ts +12 -0
- package/dist/cli-judge.d.ts.map +1 -0
- package/dist/cli-judge.js +48 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +22 -42
- package/dist/config.d.ts +85 -29
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +109 -47
- package/dist/detect/detector.d.ts +4 -2
- package/dist/detect/detector.d.ts.map +1 -1
- package/dist/detect/safe-harbor.d.ts +57 -0
- package/dist/detect/safe-harbor.d.ts.map +1 -0
- package/dist/detect/safe-harbor.js +278 -0
- package/dist/doctor-cli.d.ts +7 -0
- package/dist/doctor-cli.d.ts.map +1 -0
- package/dist/doctor-cli.js +67 -0
- package/dist/doctor-preflight.d.ts +52 -0
- package/dist/doctor-preflight.d.ts.map +1 -0
- package/dist/doctor-preflight.js +112 -0
- package/dist/feedback/coverage.d.ts +11 -8
- package/dist/feedback/coverage.d.ts.map +1 -1
- package/dist/feedback/coverage.js +29 -16
- package/dist/feedback/grade.d.ts +58 -0
- package/dist/feedback/grade.d.ts.map +1 -0
- package/dist/feedback/grade.js +67 -0
- package/dist/gateway/http-protocol.d.ts +20 -1
- package/dist/gateway/http-protocol.d.ts.map +1 -1
- package/dist/gateway/http-protocol.js +1 -1
- package/dist/gateway/serve-client.d.ts +99 -0
- package/dist/gateway/serve-client.d.ts.map +1 -0
- package/dist/gateway/serve-client.js +360 -0
- package/dist/index.d.ts +68 -29
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +167 -46
- package/dist/init/observe-cli.js +1 -1
- package/dist/init/observe.d.ts +2 -2
- package/dist/init/observe.d.ts.map +1 -1
- package/dist/init/observe.js +11 -7
- package/dist/mask/arm.d.ts +112 -0
- package/dist/mask/arm.d.ts.map +1 -0
- package/dist/mask/arm.js +179 -0
- package/dist/mask/index.d.ts +2 -0
- package/dist/mask/index.d.ts.map +1 -1
- package/dist/mask/index.js +7 -1
- package/dist/mask/mask-policy-cli.d.ts +12 -4
- package/dist/mask/mask-policy-cli.d.ts.map +1 -1
- package/dist/mask/mask-policy-cli.js +34 -9
- package/dist/prompts/manifest.d.ts +128 -0
- package/dist/prompts/manifest.d.ts.map +1 -0
- package/dist/prompts/manifest.js +214 -0
- package/dist/prompts/prompt-fetch.d.ts +84 -0
- package/dist/prompts/prompt-fetch.d.ts.map +1 -0
- package/dist/prompts/prompt-fetch.js +120 -0
- package/dist/prompts/prompt-resolve.d.ts +76 -0
- package/dist/prompts/prompt-resolve.d.ts.map +1 -0
- package/dist/prompts/prompt-resolve.js +93 -0
- package/dist/prompts/slot-resolver.d.ts +91 -0
- package/dist/prompts/slot-resolver.d.ts.map +1 -0
- package/dist/prompts/slot-resolver.js +135 -0
- package/dist/prompts/tools-register.d.ts +60 -0
- package/dist/prompts/tools-register.d.ts.map +1 -0
- package/dist/prompts/tools-register.js +48 -0
- package/dist/prompts/version-hash.d.ts +29 -0
- package/dist/prompts/version-hash.d.ts.map +1 -0
- package/dist/prompts/version-hash.js +54 -0
- package/dist/prompts-export-cli.d.ts +21 -0
- package/dist/prompts-export-cli.d.ts.map +1 -0
- package/dist/prompts-export-cli.js +130 -0
- package/dist/query/grammar.d.ts +2 -2
- package/dist/query/grammar.d.ts.map +1 -1
- package/dist/query/grammar.js +3 -1
- package/dist/query/saved.d.ts +13 -0
- package/dist/query/saved.d.ts.map +1 -1
- package/dist/query/saved.js +67 -1
- package/dist/receipts/types.d.ts +1 -1
- package/dist/receipts/types.d.ts.map +1 -1
- package/dist/receipts/types.js +1 -0
- package/dist/registry/build.d.ts +43 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +52 -0
- package/dist/registry/cache-policy.d.ts.map +1 -1
- package/dist/registry/cache-policy.js +22 -0
- package/dist/registry/pricing-shape.d.ts +67 -8
- package/dist/registry/pricing-shape.d.ts.map +1 -1
- package/dist/registry/pricing-shape.js +50 -13
- package/dist/registry/pricing.d.ts +6 -5
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +9 -5
- package/dist/registry/rows.d.ts.map +1 -1
- package/dist/registry/rows.js +1 -0
- package/dist/registry/select.d.ts +6 -1
- package/dist/registry/select.d.ts.map +1 -1
- package/dist/registry/select.js +11 -0
- package/dist/routes/call-config.d.ts +199 -0
- package/dist/routes/call-config.d.ts.map +1 -0
- package/dist/routes/call-config.js +343 -0
- package/dist/routes/destination-loader.d.ts +16 -0
- package/dist/routes/destination-loader.d.ts.map +1 -0
- package/dist/routes/destination-loader.js +29 -0
- package/dist/routes/draft.d.ts +7 -0
- package/dist/routes/draft.d.ts.map +1 -1
- package/dist/routes/draft.js +12 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +31 -56
- package/dist/routes/objective-loader.d.ts +12 -0
- package/dist/routes/objective-loader.d.ts.map +1 -1
- package/dist/routes/objective-loader.js +82 -0
- package/dist/routes/output-loader.d.ts +42 -0
- package/dist/routes/output-loader.d.ts.map +1 -0
- package/dist/routes/output-loader.js +112 -0
- package/dist/routes/prompt-loader.d.ts +49 -0
- package/dist/routes/prompt-loader.d.ts.map +1 -0
- package/dist/routes/prompt-loader.js +113 -0
- package/dist/routes/resolved-bundle.d.ts +69 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -1
- package/dist/routes/resolved-bundle.js +76 -0
- package/dist/routes/screen-loader.d.ts +29 -1
- package/dist/routes/screen-loader.d.ts.map +1 -1
- package/dist/routes/screen-loader.js +59 -1
- package/dist/run/accept.d.ts +5 -0
- package/dist/run/accept.d.ts.map +1 -1
- package/dist/run/accept.js +2 -0
- package/dist/run/cost.d.ts +31 -2
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +57 -2
- package/dist/run/destination.d.ts +39 -0
- package/dist/run/destination.d.ts.map +1 -0
- package/dist/run/destination.js +45 -0
- package/dist/run/dispatch.d.ts +67 -16
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +133 -29
- package/dist/run/grounding.d.ts +28 -2
- package/dist/run/grounding.d.ts.map +1 -1
- package/dist/run/grounding.js +54 -10
- package/dist/run/raw-defaults.d.ts.map +1 -1
- package/dist/run/raw-defaults.js +6 -1
- package/dist/run/raw-finalize.d.ts +50 -0
- package/dist/run/raw-finalize.d.ts.map +1 -0
- package/dist/run/raw-finalize.js +266 -0
- package/dist/run/raw-knobs.d.ts +36 -0
- package/dist/run/raw-knobs.d.ts.map +1 -0
- package/dist/run/raw-knobs.js +99 -0
- package/dist/run/raw-request.d.ts +24 -0
- package/dist/run/raw-request.d.ts.map +1 -0
- package/dist/run/raw-request.js +69 -0
- package/dist/run/raw-schema.d.ts +76 -0
- package/dist/run/raw-schema.d.ts.map +1 -0
- package/dist/run/raw-schema.js +166 -0
- package/dist/run/raw-search-cost.d.ts +48 -0
- package/dist/run/raw-search-cost.d.ts.map +1 -0
- package/dist/run/raw-search-cost.js +78 -0
- package/dist/run/raw-support.d.ts +64 -6
- package/dist/run/raw-support.d.ts.map +1 -1
- package/dist/run/raw-support.js +84 -11
- package/dist/run/raw.d.ts +25 -26
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +30 -351
- package/dist/run/record-attempt.d.ts +12 -0
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +30 -12
- package/dist/run/repair-attempt.d.ts +68 -0
- package/dist/run/repair-attempt.d.ts.map +1 -0
- package/dist/run/repair-attempt.js +180 -0
- package/dist/run/resolve-destination.d.ts +49 -0
- package/dist/run/resolve-destination.d.ts.map +1 -0
- package/dist/run/resolve-destination.js +76 -0
- package/dist/run/routing-docs.d.ts.map +1 -1
- package/dist/run/routing-docs.js +10 -5
- package/dist/run/run.d.ts +30 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +94 -172
- package/dist/run/telemetry-row.d.ts +92 -0
- package/dist/run/telemetry-row.d.ts.map +1 -0
- package/dist/run/telemetry-row.js +81 -0
- package/dist/run/validate.d.ts +12 -1
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +31 -0
- package/dist/run/web-search-pricing.d.ts +43 -0
- package/dist/run/web-search-pricing.d.ts.map +1 -0
- package/dist/run/web-search-pricing.js +56 -0
- package/dist/savings/odometer.d.ts +4 -0
- package/dist/savings/odometer.d.ts.map +1 -1
- package/dist/savings/odometer.js +20 -1
- package/dist/savings/swap-execute.d.ts +7 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +30 -0
- package/dist/savings/swaps-client.d.ts +10 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/screen/fixtures/probe-corpora.d.ts +23 -0
- package/dist/screen/fixtures/probe-corpora.d.ts.map +1 -0
- package/dist/screen/fixtures/probe-corpora.js +69 -0
- package/dist/screen/index.d.ts +21 -0
- package/dist/screen/index.d.ts.map +1 -0
- package/dist/screen/index.js +64 -0
- package/dist/screen/model-screen.d.ts +111 -0
- package/dist/screen/model-screen.d.ts.map +1 -0
- package/dist/screen/model-screen.js +178 -0
- package/dist/screen/patterns.d.ts +51 -0
- package/dist/screen/patterns.d.ts.map +1 -0
- package/dist/screen/patterns.js +151 -0
- package/dist/screen/phi-gate.d.ts +52 -0
- package/dist/screen/phi-gate.d.ts.map +1 -0
- package/dist/screen/phi-gate.js +86 -0
- package/dist/screen/policy.d.ts +71 -0
- package/dist/screen/policy.d.ts.map +1 -0
- package/dist/screen/policy.js +334 -0
- package/dist/screen/probe.d.ts +118 -0
- package/dist/screen/probe.d.ts.map +1 -0
- package/dist/screen/probe.js +132 -0
- package/dist/screen/screen-cli.d.ts +16 -0
- package/dist/screen/screen-cli.d.ts.map +1 -0
- package/dist/screen/screen-cli.js +178 -0
- package/dist/screen/screen.d.ts +99 -0
- package/dist/screen/screen.d.ts.map +1 -0
- package/dist/screen/screen.js +0 -0
- package/dist/screen/sidecar.d.ts +69 -0
- package/dist/screen/sidecar.d.ts.map +1 -0
- package/dist/screen/sidecar.js +145 -0
- package/dist/screen/types.d.ts +43 -2
- package/dist/screen/types.d.ts.map +1 -1
- package/dist/settings/api-fetcher.d.ts +14 -3
- package/dist/settings/api-fetcher.d.ts.map +1 -1
- package/dist/settings/api-fetcher.js +61 -10
- package/dist/settings/grading-guide.d.ts +102 -0
- package/dist/settings/grading-guide.d.ts.map +1 -0
- package/dist/settings/grading-guide.js +23 -0
- package/dist/settings/index.d.ts +3 -2
- package/dist/settings/index.d.ts.map +1 -1
- package/dist/settings/index.js +2 -1
- package/dist/settings/task-settings.d.ts +44 -7
- package/dist/settings/task-settings.d.ts.map +1 -1
- package/dist/settings/task-settings.js +53 -12
- package/dist/status.d.ts +32 -20
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +51 -30
- package/dist/telemetry/artifact-store.d.ts +5 -6
- package/dist/telemetry/artifact-store.d.ts.map +1 -1
- package/dist/telemetry/artifact-store.js +21 -24
- package/dist/telemetry/attempt-meta.d.ts +76 -1
- package/dist/telemetry/attempt-meta.d.ts.map +1 -1
- package/dist/telemetry/attempt-meta.js +33 -1
- package/dist/telemetry/exporter-setup.d.ts +2 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -1
- package/dist/telemetry/exporter-setup.js +6 -1
- package/dist/telemetry/exporter.d.ts +2 -2
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +22 -41
- package/dist/telemetry/grade-mappers.d.ts +25 -0
- package/dist/telemetry/grade-mappers.d.ts.map +1 -0
- package/dist/telemetry/grade-mappers.js +64 -0
- package/dist/telemetry/grades-sql.d.ts +56 -0
- package/dist/telemetry/grades-sql.d.ts.map +1 -0
- package/dist/telemetry/grades-sql.js +156 -0
- package/dist/telemetry/ingest-protocol.d.ts +55 -48
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +48 -83
- package/dist/telemetry/org-identity.d.ts +44 -0
- package/dist/telemetry/org-identity.d.ts.map +1 -0
- package/dist/telemetry/org-identity.js +98 -0
- package/dist/telemetry/route-registration.d.ts +48 -0
- package/dist/telemetry/route-registration.d.ts.map +1 -0
- package/dist/telemetry/route-registration.js +77 -0
- package/dist/telemetry/sinks.d.ts +10 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +52 -6
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +64 -0
- package/dist/telemetry/store.d.ts +9 -6
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +8 -50
- package/dist/tenancy/api-key-format.d.ts +45 -1
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +50 -1
- package/dist/types/provider.d.ts +23 -3
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +26 -0
- package/dist/types/route.d.ts +154 -2
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +73 -48
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +32 -0
- package/dist/verify/verify.d.ts +5 -0
- package/dist/verify/verify.d.ts.map +1 -1
- package/dist/verify/verify.js +2 -0
- package/dist/watch/evaluate.d.ts +84 -1
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +28 -0
- package/dist/watch/monitor.d.ts +8 -2
- package/dist/watch/monitor.d.ts.map +1 -1
- package/dist/watch/monitor.js +13 -5
- package/dist/watch/regret-monitor-cli.d.ts +16 -0
- package/dist/watch/regret-monitor-cli.d.ts.map +1 -0
- package/dist/watch/regret-monitor-cli.js +137 -0
- package/dist/watch/regret-monitor.d.ts +168 -0
- package/dist/watch/regret-monitor.d.ts.map +1 -0
- package/dist/watch/regret-monitor.js +304 -0
- package/dist/watch/upgrade.d.ts +62 -0
- package/dist/watch/upgrade.d.ts.map +1 -0
- package/dist/watch/upgrade.js +57 -0
- package/docs/agent-skill/SKILL.md +181 -9
- package/docs/how-it-fits.md +3 -2
- package/docs/instrumentation.md +219 -46
- package/docs/llms.txt +158 -35
- package/docs/managed-serving.md +150 -0
- package/docs/mcp-oracle.md +322 -0
- package/docs/migration-playbook.md +5 -4
- package/docs/quickstart.md +263 -89
- package/package.json +18 -4
- package/registry/leaderboard.json +74 -0
- package/registry/registry.json +232 -112
- package/screen.d.ts +16 -0
- package/templates/call-disposition-qa.json +77 -0
- package/templates/catalog-cleansing.json +77 -0
- package/templates/compliance-review.json +76 -0
- package/templates/cs-next-action.json +76 -0
- package/templates/document-extraction.json +78 -0
- package/templates/financial-classification.json +75 -0
- package/templates/lead-qualification.json +76 -0
- package/templates/medical-classification.json +71 -0
- package/templates/ticket-triage.json +76 -0
- package/dist/feedback/feedback.d.ts +0 -35
- package/dist/feedback/feedback.d.ts.map +0 -1
- package/dist/feedback/feedback.js +0 -39
|
@@ -8,18 +8,21 @@
|
|
|
8
8
|
* rung 0 — implicit signals only: conformance / repair / refusal rates
|
|
9
9
|
* from telemetry. Enough confidence for schema-mechanical swap
|
|
10
10
|
* proposals (the bake-off gate).
|
|
11
|
-
* rung 1 — ≥
|
|
11
|
+
* rung 1 — ≥ RUNG1_MIN_GRADES grades attached: prose
|
|
12
12
|
* routes (no schema to conform to) gain proposal confidence.
|
|
13
13
|
* rung 2+ — judges; Phase 5+, out of scope by decision.
|
|
14
14
|
*/
|
|
15
15
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
-
exports.
|
|
16
|
+
exports.RUNG1_MIN_GRADES = void 0;
|
|
17
17
|
exports.computeCoverage = computeCoverage;
|
|
18
|
-
exports.
|
|
19
|
-
/** Attribution: a
|
|
20
|
-
*
|
|
21
|
-
* to nothing (never guessed).
|
|
22
|
-
|
|
18
|
+
exports.RUNG1_MIN_GRADES = 10;
|
|
19
|
+
/** Attribution: a grade joins a route via its inference-subject id, or via its
|
|
20
|
+
* run-subject (episode) id matching a row's run_id tag. Unmatched grades attach
|
|
21
|
+
* to nothing (never guessed). Grade primitive (R6): every grade counts toward
|
|
22
|
+
* readiness — a wrapper-written feedback IS now a grade, so this is
|
|
23
|
+
* behaviour-preserving for the Rung-1 threshold. Artifact-subject grades never
|
|
24
|
+
* resolve to a route here (no artifact→route join) and simply do not count. */
|
|
25
|
+
function computeCoverage(rows, grades) {
|
|
23
26
|
const routeByInferenceId = new Map();
|
|
24
27
|
const routeByEpisode = new Map();
|
|
25
28
|
const byRoute = new Map();
|
|
@@ -42,7 +45,7 @@ function computeCoverage(rows, feedback) {
|
|
|
42
45
|
conform: 0,
|
|
43
46
|
repaired: 0,
|
|
44
47
|
refused: 0,
|
|
45
|
-
|
|
48
|
+
grades: 0,
|
|
46
49
|
};
|
|
47
50
|
agg.samples += 1;
|
|
48
51
|
if (row.failureClass === null)
|
|
@@ -63,22 +66,32 @@ function computeCoverage(rows, feedback) {
|
|
|
63
66
|
// Unparseable tags never break coverage — the row still counts above.
|
|
64
67
|
}
|
|
65
68
|
}
|
|
66
|
-
for (const
|
|
67
|
-
|
|
68
|
-
|
|
69
|
+
for (const g of grades) {
|
|
70
|
+
// R6 (grade primitive): EVERY grade counts toward Rung 1, not just the
|
|
71
|
+
// wrapper-written thumbs the pre-grade `feedback` table held. A grade counts
|
|
72
|
+
// only when its subject resolves to a route: an inference-subject id, or a
|
|
73
|
+
// run-subject (episode) id matching a row's run_id tag. Artifact-subject
|
|
74
|
+
// grades and native run verdicts (whose run-subject id is a run UUID, not an
|
|
75
|
+
// episode key) never resolve to a route here, so they do not over-count —
|
|
76
|
+
// behaviour-preserving for existing corpora while opening Rung 1 to
|
|
77
|
+
// model-judge/agent grades on inferences (the R6 intent).
|
|
78
|
+
const inferenceId = g.subjectKind === "inference" ? g.subjectId : null;
|
|
79
|
+
const episodeKey = g.subjectKind === "run" ? g.subjectId : null;
|
|
80
|
+
const route = (inferenceId !== null ? routeByInferenceId.get(inferenceId) : undefined) ??
|
|
81
|
+
(episodeKey !== null ? routeByEpisode.get(episodeKey) : undefined);
|
|
69
82
|
if (route === undefined)
|
|
70
83
|
continue;
|
|
71
84
|
const agg = byRoute.get(route);
|
|
72
85
|
if (agg !== undefined)
|
|
73
|
-
agg.
|
|
86
|
+
agg.grades += 1;
|
|
74
87
|
}
|
|
75
88
|
return [...byRoute.entries()]
|
|
76
89
|
.sort(([a], [b]) => a.localeCompare(b))
|
|
77
90
|
.map(([route, agg]) => {
|
|
78
|
-
const rung = agg.
|
|
91
|
+
const rung = agg.grades >= exports.RUNG1_MIN_GRADES ? 1 : 0;
|
|
79
92
|
const nextRungUnlock = rung === 0
|
|
80
|
-
? `Attach ${exports.
|
|
81
|
-
`(${agg.
|
|
93
|
+
? `Attach ${exports.RUNG1_MIN_GRADES - agg.grades} more rig.grade() grades ` +
|
|
94
|
+
`(${agg.grades}/${exports.RUNG1_MIN_GRADES}) to reach rung 1 — judged confidence ` +
|
|
82
95
|
"for routes without a schema to conform to."
|
|
83
96
|
: "Rung 2 (LLM judges — value accuracy beyond mechanical conformance) arrives in Phase 5.";
|
|
84
97
|
return {
|
|
@@ -87,7 +100,7 @@ function computeCoverage(rows, feedback) {
|
|
|
87
100
|
conformRate: agg.samples === 0 ? 0 : agg.conform / agg.samples,
|
|
88
101
|
repairRate: agg.samples === 0 ? 0 : agg.repaired / agg.samples,
|
|
89
102
|
refusalRate: agg.samples === 0 ? 0 : agg.refused / agg.samples,
|
|
90
|
-
|
|
103
|
+
gradeCount: agg.grades,
|
|
91
104
|
rung,
|
|
92
105
|
nextRungUnlock,
|
|
93
106
|
};
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* rig.grade() — the unified judgment primitive (grade primitive, R1).
|
|
3
|
+
*
|
|
4
|
+
* A grade attaches a judgment to a SUBJECT (an inference / run / artifact):
|
|
5
|
+
* `{ score: 0–1 (primitive), scale?, comment?, kind, grader?, source? }`. The
|
|
6
|
+
* pass/fail VERDICT is DERIVED at read time (`pass = score >= threshold`), never
|
|
7
|
+
* stored. `rig.grade` writes one local `grades` row (synchronous SQLite insert,
|
|
8
|
+
* fire-and-forget); the exporter mirrors it to the cloud.
|
|
9
|
+
*
|
|
10
|
+
* `rig.grade` is the SOLE judgment write API. The temporary GP-1 compat wrappers
|
|
11
|
+
* (`rig.feedback` / `artifact.evaluate` / `run.evaluate`) and their adapters were
|
|
12
|
+
* removed in GP-3 — every judgment now builds a grade directly.
|
|
13
|
+
*/
|
|
14
|
+
import type { GradeCriterion, GradeRow, GradeSubject } from "../types/telemetry";
|
|
15
|
+
/** Options for `rig.grade(subject, opts)`. `score` is the 0–1 primitive and is
|
|
16
|
+
* required; everything else is optional display / provenance. */
|
|
17
|
+
export interface GradeOpts {
|
|
18
|
+
/** 0–1 primitive (always present). up/pass = 1.0, down/fail = 0.0, else graded. */
|
|
19
|
+
readonly score: number;
|
|
20
|
+
/** Optional DISPLAY scale — `{ max }` (e.g. out of 100) or `{ label }` (e.g. "B+").
|
|
21
|
+
* Never changes the derived verdict, which is a threshold on `score`. */
|
|
22
|
+
readonly scale?: {
|
|
23
|
+
readonly max: number;
|
|
24
|
+
} | {
|
|
25
|
+
readonly label: string;
|
|
26
|
+
};
|
|
27
|
+
/** Written guidance saved for a later improvement path (was note/explanation). */
|
|
28
|
+
readonly comment?: string;
|
|
29
|
+
readonly kind: "human" | "model-judge" | "deterministic";
|
|
30
|
+
/** Optional specific grader name (e.g. "gpt-5-judge", "schema-check"). */
|
|
31
|
+
readonly grader?: string;
|
|
32
|
+
/** Provenance (e.g. "sdk", "console", a review-agent name). */
|
|
33
|
+
readonly source?: string;
|
|
34
|
+
/** OPTIONAL per-criterion breakdown (call-qa F4). When present on a human
|
|
35
|
+
* grade it rides the wire to the server, which persists it to `grade_criteria`
|
|
36
|
+
* (kind=human) so per-criterion agreement can pair the human vs model
|
|
37
|
+
* scorecards. Absent ⇒ today's single-score grade, unchanged. The server owns
|
|
38
|
+
* the scrub + dedup/drop-empty-id/clamp guard at persist; this is carried
|
|
39
|
+
* verbatim. */
|
|
40
|
+
readonly criteria?: readonly GradeCriterion[];
|
|
41
|
+
/** The grade's own org — server-supplied ONLY, already authorized (T3). Omit
|
|
42
|
+
* from application code (a rig has one org and the exporter knows it). */
|
|
43
|
+
readonly orgId?: string;
|
|
44
|
+
/** Authenticated user who recorded it — same provenance rule as `orgId`. */
|
|
45
|
+
readonly createdBy?: string | null;
|
|
46
|
+
}
|
|
47
|
+
/** Build a GradeRow from a subject + opts. Validates the score bound (the one
|
|
48
|
+
* primitive — a judgment with no numeric score has no place in the grade model). */
|
|
49
|
+
export declare function buildGradeRow(subject: GradeSubject, opts: GradeOpts): GradeRow;
|
|
50
|
+
/** Build the rig's `grade` surface over a grade writer (the telemetry store).
|
|
51
|
+
* Kept here (not inlined in index.ts) so the SDK entrypoint stays under the LOC
|
|
52
|
+
* cap and the grade write surface lives with the primitive. */
|
|
53
|
+
export declare function makeGradeApi(store: {
|
|
54
|
+
recordGrade(row: GradeRow): void;
|
|
55
|
+
}): {
|
|
56
|
+
grade(subject: GradeSubject, opts: GradeOpts): GradeRow;
|
|
57
|
+
};
|
|
58
|
+
//# sourceMappingURL=grade.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"grade.d.ts","sourceRoot":"","sources":["../../src/feedback/grade.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEjF;iEACiE;AACjE,MAAM,WAAW,SAAS;IACxB,mFAAmF;IACnF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;6EACyE;IACzE,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACvE,kFAAkF;IAClF,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,aAAa,GAAG,eAAe,CAAC;IACzD,0EAA0E;IAC1E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;mBAKe;IACf,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC9C;8EAC0E;IAC1E,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,4EAA4E;IAC5E,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAQD;oFACoF;AACpF,wBAAgB,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,GAAG,QAAQ,CA2B9E;AAED;;+DAE+D;AAC/D,wBAAgB,YAAY,CAAC,KAAK,EAAE;IAAE,WAAW,CAAC,GAAG,EAAE,QAAQ,GAAG,IAAI,CAAA;CAAE,GAAG;IACzE,KAAK,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,GAAG,QAAQ,CAAC;CACzD,CAQA"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* rig.grade() — the unified judgment primitive (grade primitive, R1).
|
|
4
|
+
*
|
|
5
|
+
* A grade attaches a judgment to a SUBJECT (an inference / run / artifact):
|
|
6
|
+
* `{ score: 0–1 (primitive), scale?, comment?, kind, grader?, source? }`. The
|
|
7
|
+
* pass/fail VERDICT is DERIVED at read time (`pass = score >= threshold`), never
|
|
8
|
+
* stored. `rig.grade` writes one local `grades` row (synchronous SQLite insert,
|
|
9
|
+
* fire-and-forget); the exporter mirrors it to the cloud.
|
|
10
|
+
*
|
|
11
|
+
* `rig.grade` is the SOLE judgment write API. The temporary GP-1 compat wrappers
|
|
12
|
+
* (`rig.feedback` / `artifact.evaluate` / `run.evaluate`) and their adapters were
|
|
13
|
+
* removed in GP-3 — every judgment now builds a grade directly.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.buildGradeRow = buildGradeRow;
|
|
17
|
+
exports.makeGradeApi = makeGradeApi;
|
|
18
|
+
const node_crypto_1 = require("node:crypto");
|
|
19
|
+
function scaleFields(scale) {
|
|
20
|
+
if (scale === undefined)
|
|
21
|
+
return { scaleMax: null, scaleLabel: null };
|
|
22
|
+
if ("max" in scale)
|
|
23
|
+
return { scaleMax: scale.max, scaleLabel: null };
|
|
24
|
+
return { scaleMax: null, scaleLabel: scale.label };
|
|
25
|
+
}
|
|
26
|
+
/** Build a GradeRow from a subject + opts. Validates the score bound (the one
|
|
27
|
+
* primitive — a judgment with no numeric score has no place in the grade model). */
|
|
28
|
+
function buildGradeRow(subject, opts) {
|
|
29
|
+
if (subject.id === "") {
|
|
30
|
+
throw new Error("grade subject must have a non-empty id");
|
|
31
|
+
}
|
|
32
|
+
if (!Number.isFinite(opts.score) || opts.score < 0 || opts.score > 1) {
|
|
33
|
+
throw new Error(`grade score must be in [0, 1], got ${opts.score}`);
|
|
34
|
+
}
|
|
35
|
+
const { scaleMax, scaleLabel } = scaleFields(opts.scale);
|
|
36
|
+
return {
|
|
37
|
+
id: (0, node_crypto_1.randomUUID)(),
|
|
38
|
+
subjectKind: subject.kind,
|
|
39
|
+
subjectId: subject.id,
|
|
40
|
+
score: opts.score,
|
|
41
|
+
scaleMax,
|
|
42
|
+
scaleLabel,
|
|
43
|
+
comment: opts.comment ?? null,
|
|
44
|
+
kind: opts.kind,
|
|
45
|
+
grader: opts.grader ?? null,
|
|
46
|
+
source: opts.source ?? null,
|
|
47
|
+
orgId: opts.orgId !== undefined && opts.orgId !== "" ? opts.orgId : null,
|
|
48
|
+
createdBy: opts.createdBy ?? null,
|
|
49
|
+
ts: new Date().toISOString(),
|
|
50
|
+
// F4: carry the per-criterion breakdown verbatim iff present (spread-iff so
|
|
51
|
+
// a single-score grade's row is byte-identical to pre-F4). Structural
|
|
52
|
+
// normalization (dedup/clamp/scrub) is the server's job at persist.
|
|
53
|
+
...(opts.criteria !== undefined ? { criteria: opts.criteria } : {}),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/** Build the rig's `grade` surface over a grade writer (the telemetry store).
|
|
57
|
+
* Kept here (not inlined in index.ts) so the SDK entrypoint stays under the LOC
|
|
58
|
+
* cap and the grade write surface lives with the primitive. */
|
|
59
|
+
function makeGradeApi(store) {
|
|
60
|
+
return {
|
|
61
|
+
grade(subject, opts) {
|
|
62
|
+
const row = buildGradeRow(subject, opts);
|
|
63
|
+
store.recordGrade(row);
|
|
64
|
+
return row;
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
}
|
|
@@ -42,7 +42,7 @@ export declare const MODELRIG_HEADERS: {
|
|
|
42
42
|
/** "provider/model" that repaired the output, when repair fired. */
|
|
43
43
|
readonly repairedBy: "x-modelrig-repaired-by";
|
|
44
44
|
readonly costUsd: "x-modelrig-cost-usd";
|
|
45
|
-
/** The `rig.
|
|
45
|
+
/** The `rig.grade()` inference handle (telemetry inference id). */
|
|
46
46
|
readonly inferenceId: "x-modelrig-inference-id";
|
|
47
47
|
};
|
|
48
48
|
export declare const MODELRIG_REQUEST_HEADERS: {
|
|
@@ -114,6 +114,25 @@ export interface ChatCompletionRequest {
|
|
|
114
114
|
readonly envelope?: string;
|
|
115
115
|
readonly variant?: string;
|
|
116
116
|
readonly stream?: "buffered";
|
|
117
|
+
/**
|
|
118
|
+
* K1b (managed-serving-keyless) — the FAITHFUL carrier for a TEMPLATED /
|
|
119
|
+
* NON-CHAT route's named template variables. A provider-neutral JSON object
|
|
120
|
+
* of the route's `{{variable}}` bindings, DISTINCT from OpenAI `messages`:
|
|
121
|
+
* `messages` is the chat-relay bridge (the server derives `message`/`messages`
|
|
122
|
+
* from it), whereas `input` carries the route's OWN declared variables
|
|
123
|
+
* verbatim (e.g. a report route's structured `{ topic, holdings, … }`), so the
|
|
124
|
+
* server renders the route template exactly as a local `rig.run` would — no
|
|
125
|
+
* lossy messages round-trip.
|
|
126
|
+
*
|
|
127
|
+
* PRECEDENCE (server bind): when present, `modelrig.input` WINS — the run's
|
|
128
|
+
* `input` is these named variables; the chat `messages` binding is the
|
|
129
|
+
* fallback used only when it is absent. The server validates the object shape
|
|
130
|
+
* fail-closed (a non-object, or a prototype-polluting key, is a teaching 422)
|
|
131
|
+
* — a request that cannot be faithfully served is refused, never served with
|
|
132
|
+
* a silently-wrong prompt. ADDITIVE: a chat request that omits it is
|
|
133
|
+
* byte-identical to the K1 wire.
|
|
134
|
+
*/
|
|
135
|
+
readonly input?: Readonly<Record<string, unknown>>;
|
|
117
136
|
};
|
|
118
137
|
}
|
|
119
138
|
export interface ChatCompletionUsage {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-protocol.d.ts","sourceRoot":"","sources":["../../src/gateway/http-protocol.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAE9E,uCAAuC;AACvC,eAAO,MAAM,qBAAqB,yBAAyB,CAAC;AAC5D,2EAA2E;AAC3E,eAAO,MAAM,WAAW,eAAe,CAAC;AAExC,6DAA6D;AAC7D,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAO1C,eAAO,MAAM,gBAAgB;IAC3B,0EAA0E;;IAE1E;;;;eAIW;;;;IAIX,yEAAyE;;IAEzE,oEAAoE;;;IAGpE,
|
|
1
|
+
{"version":3,"file":"http-protocol.d.ts","sourceRoot":"","sources":["../../src/gateway/http-protocol.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAE9E,uCAAuC;AACvC,eAAO,MAAM,qBAAqB,yBAAyB,CAAC;AAC5D,2EAA2E;AAC3E,eAAO,MAAM,WAAW,eAAe,CAAC;AAExC,6DAA6D;AAC7D,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAO1C,eAAO,MAAM,gBAAgB;IAC3B,0EAA0E;;IAE1E;;;;eAIW;;;;IAIX,yEAAyE;;IAEzE,oEAAoE;;;IAGpE,mEAAmE;;CAE3D,CAAC;AAGX,eAAO,MAAM,wBAAwB;IACnC,qCAAqC;;IAErC,oCAAoC;;IAEpC,8EAA8E;;CAEtE,CAAC;AAEX,MAAM,MAAM,IAAI,GAAG,MAAM,GAAG,KAAK,CAAC;AAElC,4EAA4E;AAC5E,MAAM,MAAM,cAAc,GACtB;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAChD;IAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAC3E;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1D;;;;;;;;;;GAUG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,MAAM,EACb,eAAe,GAAE,WAAW,CAAC,MAAM,CAAa,GAC/C,cAAc,CA6BhB;AAID,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,MAAM,CAAC;IACxD,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,0EAA0E;IAC1E,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;IAC1C,0DAA0D;IAC1D,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,CAAC;IAC1C,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC3C,QAAQ,CAAC,WAAW,CAAC,EAAE,UAAU,CAAC;IAClC,4EAA4E;IAC5E,QAAQ,CAAC,eAAe,CAAC,EACrB;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GACzB;QAAE,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAA;KAAE,GAChC;QACE,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;QAC7B,QAAQ,CAAC,WAAW,EAAE;YAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;SAAE,CAAC;KACrG,CAAC;IACN,wEAAwE;IACxE,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAClB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC;QAC7B;;;;;;;;;;;;;;;;;WAiBG;QACH,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;KACpD,CAAC;CACH;AAID,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE;QAChB,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;QAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;KAC3C,CAAC;IACF,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,QAAQ,GAAG,YAAY,GAAG,gBAAgB,GAAG,IAAI,CAAC;CACpF;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,SAAS,oBAAoB,EAAE,CAAC;IAClD,QAAQ,CAAC,KAAK,EAAE,mBAAmB,CAAC;CACrC;AAID,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC;QAC5B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACjC,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;KAC3C,CAAC;IACF,QAAQ,CAAC,aAAa,EAAE,oBAAoB,CAAC,eAAe,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAC;IACzC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,SAAS,yBAAyB,EAAE,CAAC;CACxD;AAED;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,MAAM,EAAE,qBAAqB,CAAC;IACvC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,2DAA2D;IAC3D,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACrC;AAED,gEAAgE;AAChE,eAAO,MAAM,QAAQ,WAAW,CAAC"}
|
|
@@ -51,7 +51,7 @@ exports.MODELRIG_HEADERS = {
|
|
|
51
51
|
/** "provider/model" that repaired the output, when repair fired. */
|
|
52
52
|
repairedBy: "x-modelrig-repaired-by",
|
|
53
53
|
costUsd: "x-modelrig-cost-usd",
|
|
54
|
-
/** The `rig.
|
|
54
|
+
/** The `rig.grade()` inference handle (telemetry inference id). */
|
|
55
55
|
inferenceId: "x-modelrig-inference-id",
|
|
56
56
|
};
|
|
57
57
|
// Request-side extension headers (spec §3.4).
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* K1 (managed-serving-keyless §2/§3) — the SDK-side KEYLESS serving transport.
|
|
3
|
+
*
|
|
4
|
+
* This is the client half of `destination: modelrig-served`: instead of the
|
|
5
|
+
* local adapter dispatch, a served run is POSTed to the HOSTED gateway's
|
|
6
|
+
* `POST /v1/chat/completions` (the org's mirrored bundle, resolved BY SLUG and
|
|
7
|
+
* driven server-side through `runBundle` — see modelrig-server
|
|
8
|
+
* `handleMirroredSlugLane`). The gateway spends ModelRig's OWN managed provider
|
|
9
|
+
* key; the SDK sends ONLY its `rig_sk_` control-plane key (Bearer), NEVER a
|
|
10
|
+
* provider key.
|
|
11
|
+
*
|
|
12
|
+
* ONE wire contract: everything here is expressed in the SHARED
|
|
13
|
+
* `gateway/http-protocol.ts` shapes (`CHAT_COMPLETIONS_PATH`,
|
|
14
|
+
* `ChatCompletionRequest`, `ChatCompletionResponse`, `MODELRIG_HEADERS`,
|
|
15
|
+
* `MODELRIG_REQUEST_HEADERS`) — the same module `modelrig-server` compiles
|
|
16
|
+
* against, so the request this produces is exactly the one the endpoint
|
|
17
|
+
* consumes and the response it parses is exactly the one the endpoint emits.
|
|
18
|
+
*
|
|
19
|
+
* FAIL-CLOSED-BUT-DISCOVERABLE (spec §3 K1, ruling R3): a transport error, a
|
|
20
|
+
* non-2xx, a 402 (no prepaid balance), or a 403 (no serve entitlement) throws a
|
|
21
|
+
* TEACHING `RigFailureError` (class `invariant_violation`, TERMINAL) whose
|
|
22
|
+
* `fixHint` names the cause and the fix — NEVER a silent fall back to `direct`
|
|
23
|
+
* or a local key. `fetch` is injected (`fetchFn`) so tests mock the transport
|
|
24
|
+
* with no live network and no real key.
|
|
25
|
+
*
|
|
26
|
+
* TRANSPORT + MAPPING ONLY. The flag/capability gate + the run.ts branch point
|
|
27
|
+
* live in `run/resolve-destination.ts` / `run/run.ts`; this module is reached
|
|
28
|
+
* only AFTER the guard has passed.
|
|
29
|
+
*/
|
|
30
|
+
import type { ChatMessage } from "./http-protocol";
|
|
31
|
+
import type { RouteBundle, RunOptions, RunResult } from "../types/route";
|
|
32
|
+
/** Injected transport — the global `fetch` in production, a mock in tests. */
|
|
33
|
+
export type ServeFetch = typeof fetch;
|
|
34
|
+
/**
|
|
35
|
+
* Where + how to reach the hosted gateway. Resolved (in `index.ts`) from the
|
|
36
|
+
* SAME `api`-mode export config the telemetry sink uses: `baseUrl` is the ingest
|
|
37
|
+
* host (`MODELRIG_INGEST_URL` ‖ `DEFAULT_INGEST_URL`) and `apiKey` is the
|
|
38
|
+
* `rig_sk_` key. ABSENT ⇒ no hosted control plane is configured ⇒ a served run
|
|
39
|
+
* fails closed (there is nowhere to serve it, and we never invent a key).
|
|
40
|
+
*/
|
|
41
|
+
export interface ServeTransportConfig {
|
|
42
|
+
/** Ingest host base, e.g. `https://api.modelrig.ai` (no path). */
|
|
43
|
+
readonly baseUrl: string;
|
|
44
|
+
/** `rig_sk_…` control-plane key. The ONLY credential sent — never a provider key. */
|
|
45
|
+
readonly apiKey: string;
|
|
46
|
+
}
|
|
47
|
+
export interface ServeRunInput {
|
|
48
|
+
readonly route: RouteBundle;
|
|
49
|
+
readonly opts: RunOptions;
|
|
50
|
+
/** The resolved variant name (Phase 3 WS1) — served via `x-modelrig-variant`. */
|
|
51
|
+
readonly servedVariant: string | null;
|
|
52
|
+
/** Resolved transport; `undefined` ⇒ fail closed (no hosted control plane). */
|
|
53
|
+
readonly serve: ServeTransportConfig | undefined;
|
|
54
|
+
/** Injected fetch (tests); defaults to the global `fetch`. */
|
|
55
|
+
readonly fetchFn?: ServeFetch;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Is this run's `input` a CHAT-SHAPED input — the shape the OpenAI `messages`
|
|
59
|
+
* bridge carries faithfully (`input.messages` a transcript, or `input.message` a
|
|
60
|
+
* single user turn)? A route authored for the OpenAI-compatible surface declares
|
|
61
|
+
* exactly these variables (PROXY_RELAYED_VARS, server-side). Anything else is a
|
|
62
|
+
* NAMED-variable input carried by `modelrig.input` (K1b) instead.
|
|
63
|
+
*/
|
|
64
|
+
export declare function isChatShapedInput(input: RunOptions["input"]): boolean;
|
|
65
|
+
/**
|
|
66
|
+
* Map the run's CHAT-shaped `input` into OpenAI `messages` — the chat-relay
|
|
67
|
+
* bridge (the server rebuilds `message`/`messages` from it). Faithful for a
|
|
68
|
+
* chat-shaped route:
|
|
69
|
+
* - `input.messages` (an array of `{role, content}`) → the transcript verbatim;
|
|
70
|
+
* - else `input.message` (a string) → a single user turn.
|
|
71
|
+
*
|
|
72
|
+
* K1b: a route that templates over OTHER named variables is NO LONGER fails-closed
|
|
73
|
+
* here — `serveViaGateway` carries those faithfully via `modelrig.input`
|
|
74
|
+
* (`buildServedNamedInput`). This function is reached only for a chat-shaped
|
|
75
|
+
* input, so the final fail-closed branch stays as a defensive teaching error for
|
|
76
|
+
* a caller who invokes it directly on a non-chat input.
|
|
77
|
+
*/
|
|
78
|
+
export declare function buildServedMessages(route: RouteBundle, input: RunOptions["input"]): readonly ChatMessage[];
|
|
79
|
+
/**
|
|
80
|
+
* K1b — carry a NAMED-variable route's `input` faithfully as `modelrig.input`,
|
|
81
|
+
* INSTEAD of the K1 fail-closed. A templated/non-chat route (an InferWealth
|
|
82
|
+
* report route, say) declares structured variables (`{ topic, holdings, … }`),
|
|
83
|
+
* not `message`/`messages`; this is the provider-neutral object the server binds
|
|
84
|
+
* so the route template renders exactly as a local `rig.run` would.
|
|
85
|
+
*
|
|
86
|
+
* Still fail-closed-but-discoverable (K1's guarantee preserved): the only reason
|
|
87
|
+
* to refuse now is a genuinely UN-SERIALIZABLE input — one JSON cannot carry
|
|
88
|
+
* losslessly, which would degrade to a silently-wrong prompt server-side.
|
|
89
|
+
* `assertJsonFaithful` names the exact offending value; we wrap it as a teaching
|
|
90
|
+
* error, never a silent drop.
|
|
91
|
+
*/
|
|
92
|
+
export declare function buildServedNamedInput(route: RouteBundle, input: RunOptions["input"]): Readonly<Record<string, unknown>>;
|
|
93
|
+
/**
|
|
94
|
+
* Serve a run through the hosted gateway and map the completion back into a
|
|
95
|
+
* normal `RunResult`. Reached only after `assertDestinationServable` has passed
|
|
96
|
+
* (flag on + served-capable provider). Fail-closed-but-discoverable throughout.
|
|
97
|
+
*/
|
|
98
|
+
export declare function serveViaGateway(input: ServeRunInput): Promise<RunResult>;
|
|
99
|
+
//# sourceMappingURL=serve-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"serve-client.d.ts","sourceRoot":"","sources":["../../src/gateway/serve-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAOH,OAAO,KAAK,EAGV,WAAW,EACZ,MAAM,iBAAiB,CAAC;AAIzB,OAAO,KAAK,EAAE,WAAW,EAAW,UAAU,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAElF,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,OAAO,KAAK,CAAC;AAEtC;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACnC,kEAAkE;IAClE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,qFAAqF;IACrF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,iFAAiF;IACjF,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACjD,8DAA8D;IAC9D,QAAQ,CAAC,OAAO,CAAC,EAAE,UAAU,CAAC;CAC/B;AAMD;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,OAAO,CAErE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,SAAS,WAAW,EAAE,CAoC1G;AAsDD;;;;;;;;;;;;GAYG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAYvH;AA2FD;;;;GAIG;AACH,wBAAsB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC,CA0F9E"}
|