modelrig 0.8.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +324 -0
- package/README.md +55 -29
- package/data/PROVENANCE.md +14 -0
- package/data/pricing-map.json +26 -0
- package/dist/adapters/attempt-meta-map.d.ts +8 -1
- package/dist/adapters/attempt-meta-map.d.ts.map +1 -1
- package/dist/adapters/attempt-meta-map.js +4 -0
- package/dist/adapters/build-adapters.js +2 -2
- package/dist/adapters/gemini.d.ts +28 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +38 -9
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +53 -4
- package/dist/adapters/openai-compat.d.ts +8 -2
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +8 -1
- package/dist/adapters/openai-stream.d.ts +10 -2
- package/dist/adapters/openai-stream.d.ts.map +1 -1
- package/dist/adapters/openai-stream.js +5 -1
- package/dist/adapters/openai.d.ts +10 -1
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +57 -7
- package/dist/adapters/responses-api.d.ts +61 -0
- package/dist/adapters/responses-api.d.ts.map +1 -0
- package/dist/adapters/responses-api.js +124 -0
- package/dist/artifacts/artifact.d.ts +31 -12
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +218 -188
- package/dist/artifacts/index.d.ts +15 -5
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +19 -5
- package/dist/artifacts/run-context.d.ts +3 -24
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +3 -39
- package/dist/bakeoff/engine.d.ts +29 -0
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +47 -0
- package/dist/bakeoff/eval-cases-rig.d.ts +36 -0
- package/dist/bakeoff/eval-cases-rig.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases-rig.js +27 -0
- package/dist/bakeoff/eval-cases.d.ts +19 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -1
- package/dist/bakeoff/eval-cases.js +18 -1
- package/dist/bakeoff/golden-judge.d.ts +47 -0
- package/dist/bakeoff/golden-judge.d.ts.map +1 -0
- package/dist/bakeoff/golden-judge.js +49 -0
- package/dist/bakeoff/groundedness.d.ts +58 -0
- package/dist/bakeoff/groundedness.d.ts.map +1 -0
- package/dist/bakeoff/groundedness.js +165 -0
- package/dist/bakeoff/quality.d.ts +43 -0
- package/dist/bakeoff/quality.d.ts.map +1 -1
- package/dist/bakeoff/quality.js +31 -0
- package/dist/bakeoff/replay.d.ts +11 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +12 -0
- package/dist/bakeoff/report.d.ts +7 -0
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +26 -0
- package/dist/cli-judge.d.ts +12 -0
- package/dist/cli-judge.d.ts.map +1 -0
- package/dist/cli-judge.js +48 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +22 -42
- package/dist/config.d.ts +85 -29
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +109 -47
- package/dist/detect/detector.d.ts +4 -2
- package/dist/detect/detector.d.ts.map +1 -1
- package/dist/detect/safe-harbor.d.ts +57 -0
- package/dist/detect/safe-harbor.d.ts.map +1 -0
- package/dist/detect/safe-harbor.js +278 -0
- package/dist/doctor-cli.d.ts +7 -0
- package/dist/doctor-cli.d.ts.map +1 -0
- package/dist/doctor-cli.js +67 -0
- package/dist/doctor-preflight.d.ts +52 -0
- package/dist/doctor-preflight.d.ts.map +1 -0
- package/dist/doctor-preflight.js +112 -0
- package/dist/feedback/coverage.d.ts +11 -8
- package/dist/feedback/coverage.d.ts.map +1 -1
- package/dist/feedback/coverage.js +29 -16
- package/dist/feedback/grade.d.ts +58 -0
- package/dist/feedback/grade.d.ts.map +1 -0
- package/dist/feedback/grade.js +67 -0
- package/dist/gateway/http-protocol.d.ts +20 -1
- package/dist/gateway/http-protocol.d.ts.map +1 -1
- package/dist/gateway/http-protocol.js +1 -1
- package/dist/gateway/serve-client.d.ts +99 -0
- package/dist/gateway/serve-client.d.ts.map +1 -0
- package/dist/gateway/serve-client.js +360 -0
- package/dist/index.d.ts +68 -29
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +167 -46
- package/dist/init/observe-cli.js +1 -1
- package/dist/init/observe.d.ts +2 -2
- package/dist/init/observe.d.ts.map +1 -1
- package/dist/init/observe.js +11 -7
- package/dist/mask/arm.d.ts +112 -0
- package/dist/mask/arm.d.ts.map +1 -0
- package/dist/mask/arm.js +179 -0
- package/dist/mask/index.d.ts +2 -0
- package/dist/mask/index.d.ts.map +1 -1
- package/dist/mask/index.js +7 -1
- package/dist/mask/mask-policy-cli.d.ts +12 -4
- package/dist/mask/mask-policy-cli.d.ts.map +1 -1
- package/dist/mask/mask-policy-cli.js +34 -9
- package/dist/prompts/manifest.d.ts +128 -0
- package/dist/prompts/manifest.d.ts.map +1 -0
- package/dist/prompts/manifest.js +214 -0
- package/dist/prompts/prompt-fetch.d.ts +84 -0
- package/dist/prompts/prompt-fetch.d.ts.map +1 -0
- package/dist/prompts/prompt-fetch.js +120 -0
- package/dist/prompts/prompt-resolve.d.ts +76 -0
- package/dist/prompts/prompt-resolve.d.ts.map +1 -0
- package/dist/prompts/prompt-resolve.js +93 -0
- package/dist/prompts/slot-resolver.d.ts +91 -0
- package/dist/prompts/slot-resolver.d.ts.map +1 -0
- package/dist/prompts/slot-resolver.js +135 -0
- package/dist/prompts/tools-register.d.ts +60 -0
- package/dist/prompts/tools-register.d.ts.map +1 -0
- package/dist/prompts/tools-register.js +48 -0
- package/dist/prompts/version-hash.d.ts +29 -0
- package/dist/prompts/version-hash.d.ts.map +1 -0
- package/dist/prompts/version-hash.js +54 -0
- package/dist/prompts-export-cli.d.ts +21 -0
- package/dist/prompts-export-cli.d.ts.map +1 -0
- package/dist/prompts-export-cli.js +130 -0
- package/dist/query/grammar.d.ts +2 -2
- package/dist/query/grammar.d.ts.map +1 -1
- package/dist/query/grammar.js +3 -1
- package/dist/query/saved.d.ts +13 -0
- package/dist/query/saved.d.ts.map +1 -1
- package/dist/query/saved.js +67 -1
- package/dist/receipts/types.d.ts +1 -1
- package/dist/receipts/types.d.ts.map +1 -1
- package/dist/receipts/types.js +1 -0
- package/dist/registry/build.d.ts +43 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +52 -0
- package/dist/registry/cache-policy.d.ts.map +1 -1
- package/dist/registry/cache-policy.js +22 -0
- package/dist/registry/pricing-shape.d.ts +67 -8
- package/dist/registry/pricing-shape.d.ts.map +1 -1
- package/dist/registry/pricing-shape.js +50 -13
- package/dist/registry/pricing.d.ts +6 -5
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +9 -5
- package/dist/registry/rows.d.ts.map +1 -1
- package/dist/registry/rows.js +1 -0
- package/dist/registry/select.d.ts +6 -1
- package/dist/registry/select.d.ts.map +1 -1
- package/dist/registry/select.js +11 -0
- package/dist/routes/call-config.d.ts +199 -0
- package/dist/routes/call-config.d.ts.map +1 -0
- package/dist/routes/call-config.js +343 -0
- package/dist/routes/destination-loader.d.ts +16 -0
- package/dist/routes/destination-loader.d.ts.map +1 -0
- package/dist/routes/destination-loader.js +29 -0
- package/dist/routes/draft.d.ts +7 -0
- package/dist/routes/draft.d.ts.map +1 -1
- package/dist/routes/draft.js +12 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +31 -56
- package/dist/routes/objective-loader.d.ts +12 -0
- package/dist/routes/objective-loader.d.ts.map +1 -1
- package/dist/routes/objective-loader.js +82 -0
- package/dist/routes/output-loader.d.ts +42 -0
- package/dist/routes/output-loader.d.ts.map +1 -0
- package/dist/routes/output-loader.js +112 -0
- package/dist/routes/prompt-loader.d.ts +49 -0
- package/dist/routes/prompt-loader.d.ts.map +1 -0
- package/dist/routes/prompt-loader.js +113 -0
- package/dist/routes/resolved-bundle.d.ts +69 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -1
- package/dist/routes/resolved-bundle.js +76 -0
- package/dist/routes/screen-loader.d.ts +29 -1
- package/dist/routes/screen-loader.d.ts.map +1 -1
- package/dist/routes/screen-loader.js +59 -1
- package/dist/run/accept.d.ts +5 -0
- package/dist/run/accept.d.ts.map +1 -1
- package/dist/run/accept.js +2 -0
- package/dist/run/cost.d.ts +31 -2
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +57 -2
- package/dist/run/destination.d.ts +39 -0
- package/dist/run/destination.d.ts.map +1 -0
- package/dist/run/destination.js +45 -0
- package/dist/run/dispatch.d.ts +67 -16
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +133 -29
- package/dist/run/grounding.d.ts +28 -2
- package/dist/run/grounding.d.ts.map +1 -1
- package/dist/run/grounding.js +54 -10
- package/dist/run/raw-defaults.d.ts.map +1 -1
- package/dist/run/raw-defaults.js +6 -1
- package/dist/run/raw-finalize.d.ts +50 -0
- package/dist/run/raw-finalize.d.ts.map +1 -0
- package/dist/run/raw-finalize.js +266 -0
- package/dist/run/raw-knobs.d.ts +36 -0
- package/dist/run/raw-knobs.d.ts.map +1 -0
- package/dist/run/raw-knobs.js +99 -0
- package/dist/run/raw-request.d.ts +24 -0
- package/dist/run/raw-request.d.ts.map +1 -0
- package/dist/run/raw-request.js +69 -0
- package/dist/run/raw-schema.d.ts +76 -0
- package/dist/run/raw-schema.d.ts.map +1 -0
- package/dist/run/raw-schema.js +166 -0
- package/dist/run/raw-search-cost.d.ts +48 -0
- package/dist/run/raw-search-cost.d.ts.map +1 -0
- package/dist/run/raw-search-cost.js +78 -0
- package/dist/run/raw-support.d.ts +64 -6
- package/dist/run/raw-support.d.ts.map +1 -1
- package/dist/run/raw-support.js +84 -11
- package/dist/run/raw.d.ts +25 -26
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +30 -351
- package/dist/run/record-attempt.d.ts +12 -0
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +30 -12
- package/dist/run/repair-attempt.d.ts +68 -0
- package/dist/run/repair-attempt.d.ts.map +1 -0
- package/dist/run/repair-attempt.js +180 -0
- package/dist/run/resolve-destination.d.ts +49 -0
- package/dist/run/resolve-destination.d.ts.map +1 -0
- package/dist/run/resolve-destination.js +76 -0
- package/dist/run/routing-docs.d.ts.map +1 -1
- package/dist/run/routing-docs.js +10 -5
- package/dist/run/run.d.ts +30 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +94 -172
- package/dist/run/telemetry-row.d.ts +92 -0
- package/dist/run/telemetry-row.d.ts.map +1 -0
- package/dist/run/telemetry-row.js +81 -0
- package/dist/run/validate.d.ts +12 -1
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +31 -0
- package/dist/run/web-search-pricing.d.ts +43 -0
- package/dist/run/web-search-pricing.d.ts.map +1 -0
- package/dist/run/web-search-pricing.js +56 -0
- package/dist/savings/odometer.d.ts +4 -0
- package/dist/savings/odometer.d.ts.map +1 -1
- package/dist/savings/odometer.js +20 -1
- package/dist/savings/swap-execute.d.ts +7 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +30 -0
- package/dist/savings/swaps-client.d.ts +10 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/screen/fixtures/probe-corpora.d.ts +23 -0
- package/dist/screen/fixtures/probe-corpora.d.ts.map +1 -0
- package/dist/screen/fixtures/probe-corpora.js +69 -0
- package/dist/screen/index.d.ts +21 -0
- package/dist/screen/index.d.ts.map +1 -0
- package/dist/screen/index.js +64 -0
- package/dist/screen/model-screen.d.ts +111 -0
- package/dist/screen/model-screen.d.ts.map +1 -0
- package/dist/screen/model-screen.js +178 -0
- package/dist/screen/patterns.d.ts +51 -0
- package/dist/screen/patterns.d.ts.map +1 -0
- package/dist/screen/patterns.js +151 -0
- package/dist/screen/phi-gate.d.ts +52 -0
- package/dist/screen/phi-gate.d.ts.map +1 -0
- package/dist/screen/phi-gate.js +86 -0
- package/dist/screen/policy.d.ts +71 -0
- package/dist/screen/policy.d.ts.map +1 -0
- package/dist/screen/policy.js +334 -0
- package/dist/screen/probe.d.ts +118 -0
- package/dist/screen/probe.d.ts.map +1 -0
- package/dist/screen/probe.js +132 -0
- package/dist/screen/screen-cli.d.ts +16 -0
- package/dist/screen/screen-cli.d.ts.map +1 -0
- package/dist/screen/screen-cli.js +178 -0
- package/dist/screen/screen.d.ts +99 -0
- package/dist/screen/screen.d.ts.map +1 -0
- package/dist/screen/screen.js +0 -0
- package/dist/screen/sidecar.d.ts +69 -0
- package/dist/screen/sidecar.d.ts.map +1 -0
- package/dist/screen/sidecar.js +145 -0
- package/dist/screen/types.d.ts +43 -2
- package/dist/screen/types.d.ts.map +1 -1
- package/dist/settings/api-fetcher.d.ts +14 -3
- package/dist/settings/api-fetcher.d.ts.map +1 -1
- package/dist/settings/api-fetcher.js +61 -10
- package/dist/settings/grading-guide.d.ts +102 -0
- package/dist/settings/grading-guide.d.ts.map +1 -0
- package/dist/settings/grading-guide.js +23 -0
- package/dist/settings/index.d.ts +3 -2
- package/dist/settings/index.d.ts.map +1 -1
- package/dist/settings/index.js +2 -1
- package/dist/settings/task-settings.d.ts +44 -7
- package/dist/settings/task-settings.d.ts.map +1 -1
- package/dist/settings/task-settings.js +53 -12
- package/dist/status.d.ts +32 -20
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +51 -30
- package/dist/telemetry/artifact-store.d.ts +5 -6
- package/dist/telemetry/artifact-store.d.ts.map +1 -1
- package/dist/telemetry/artifact-store.js +21 -24
- package/dist/telemetry/attempt-meta.d.ts +76 -1
- package/dist/telemetry/attempt-meta.d.ts.map +1 -1
- package/dist/telemetry/attempt-meta.js +33 -1
- package/dist/telemetry/exporter-setup.d.ts +2 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -1
- package/dist/telemetry/exporter-setup.js +6 -1
- package/dist/telemetry/exporter.d.ts +2 -2
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +22 -41
- package/dist/telemetry/grade-mappers.d.ts +25 -0
- package/dist/telemetry/grade-mappers.d.ts.map +1 -0
- package/dist/telemetry/grade-mappers.js +64 -0
- package/dist/telemetry/grades-sql.d.ts +56 -0
- package/dist/telemetry/grades-sql.d.ts.map +1 -0
- package/dist/telemetry/grades-sql.js +156 -0
- package/dist/telemetry/ingest-protocol.d.ts +55 -48
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +48 -83
- package/dist/telemetry/org-identity.d.ts +44 -0
- package/dist/telemetry/org-identity.d.ts.map +1 -0
- package/dist/telemetry/org-identity.js +98 -0
- package/dist/telemetry/route-registration.d.ts +48 -0
- package/dist/telemetry/route-registration.d.ts.map +1 -0
- package/dist/telemetry/route-registration.js +77 -0
- package/dist/telemetry/sinks.d.ts +10 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +52 -6
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +64 -0
- package/dist/telemetry/store.d.ts +9 -6
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +8 -50
- package/dist/tenancy/api-key-format.d.ts +45 -1
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +50 -1
- package/dist/types/provider.d.ts +23 -3
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +26 -0
- package/dist/types/route.d.ts +154 -2
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +73 -48
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +32 -0
- package/dist/verify/verify.d.ts +5 -0
- package/dist/verify/verify.d.ts.map +1 -1
- package/dist/verify/verify.js +2 -0
- package/dist/watch/evaluate.d.ts +84 -1
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +28 -0
- package/dist/watch/monitor.d.ts +8 -2
- package/dist/watch/monitor.d.ts.map +1 -1
- package/dist/watch/monitor.js +13 -5
- package/dist/watch/regret-monitor-cli.d.ts +16 -0
- package/dist/watch/regret-monitor-cli.d.ts.map +1 -0
- package/dist/watch/regret-monitor-cli.js +137 -0
- package/dist/watch/regret-monitor.d.ts +168 -0
- package/dist/watch/regret-monitor.d.ts.map +1 -0
- package/dist/watch/regret-monitor.js +304 -0
- package/dist/watch/upgrade.d.ts +62 -0
- package/dist/watch/upgrade.d.ts.map +1 -0
- package/dist/watch/upgrade.js +57 -0
- package/docs/agent-skill/SKILL.md +181 -9
- package/docs/how-it-fits.md +3 -2
- package/docs/instrumentation.md +219 -46
- package/docs/llms.txt +158 -35
- package/docs/managed-serving.md +150 -0
- package/docs/mcp-oracle.md +322 -0
- package/docs/migration-playbook.md +5 -4
- package/docs/quickstart.md +263 -89
- package/package.json +18 -4
- package/registry/leaderboard.json +74 -0
- package/registry/registry.json +232 -112
- package/screen.d.ts +16 -0
- package/templates/call-disposition-qa.json +77 -0
- package/templates/catalog-cleansing.json +77 -0
- package/templates/compliance-review.json +76 -0
- package/templates/cs-next-action.json +76 -0
- package/templates/document-extraction.json +78 -0
- package/templates/financial-classification.json +75 -0
- package/templates/lead-qualification.json +76 -0
- package/templates/medical-classification.json +71 -0
- package/templates/ticket-triage.json +76 -0
- package/dist/feedback/feedback.d.ts +0 -35
- package/dist/feedback/feedback.d.ts.map +0 -1
- package/dist/feedback/feedback.js +0 -39
package/dist/types/route.d.ts
CHANGED
|
@@ -3,9 +3,13 @@
|
|
|
3
3
|
* routes/loader.ts; frozen after load.
|
|
4
4
|
*/
|
|
5
5
|
import type { FailureClass } from "./failures";
|
|
6
|
+
import type { Destination } from "../run/destination";
|
|
6
7
|
import type { ProviderId, SamplingParams, ServiceTier, ToolCall, ToolChoice, ToolDefinition } from "./provider";
|
|
7
8
|
import type { QualityGate } from "./quality";
|
|
8
9
|
import type { MaskReceipt, ResolvedMaskPolicy } from "../mask/types";
|
|
10
|
+
import type { ResolvedScreenPolicy, ScreenReceipt } from "../screen/types";
|
|
11
|
+
import type { DataClassification } from "../artifacts/classification";
|
|
12
|
+
import type { Manifest } from "../prompts/manifest";
|
|
9
13
|
export interface CandidateSpec {
|
|
10
14
|
readonly provider: ProviderId;
|
|
11
15
|
readonly model: string;
|
|
@@ -166,17 +170,78 @@ export interface BakeoffConfig {
|
|
|
166
170
|
readonly maxSamples: number;
|
|
167
171
|
};
|
|
168
172
|
};
|
|
173
|
+
/**
|
|
174
|
+
* Q1 regret monitor (near-optimal spec §5) — additive & optional, DEFAULT OFF.
|
|
175
|
+
* When `enabled`, a scheduled cron replays a sample of recent captures against
|
|
176
|
+
* the route's reference arm and maintains a regret band, filing a `regret_breach`
|
|
177
|
+
* bake-off proposal when the band breaches ε / drifts up. Requires
|
|
178
|
+
* `bakeoff.reference` to be set (nothing to measure against otherwise — `validate`
|
|
179
|
+
* warns). All bounds default to the house constants when omitted. */
|
|
180
|
+
readonly regretMonitor?: {
|
|
181
|
+
readonly enabled: boolean;
|
|
182
|
+
/** Captures sampled per cycle (capped by the reference envelope). */
|
|
183
|
+
readonly sampleN?: number;
|
|
184
|
+
readonly windowDays?: number;
|
|
185
|
+
readonly baselineDays?: number;
|
|
186
|
+
};
|
|
187
|
+
/**
|
|
188
|
+
* N4 up-swap watch (near-optimal spec §4) — additive & optional. When true (and
|
|
189
|
+
* a reference resolves), the watch loop files an `upgrade_opportunity` bake-off
|
|
190
|
+
* when the route's ceiling rises. Absent ⇒ opt-in default: a route WITH a
|
|
191
|
+
* reference watches by default; a route with no reference can never fire it, and
|
|
192
|
+
* cost-first routes (no objective/bakeoff block) are always off. `validate` warns
|
|
193
|
+
* when set true on a route with no reference. */
|
|
194
|
+
readonly upgradeWatch?: boolean;
|
|
169
195
|
}
|
|
196
|
+
/**
|
|
197
|
+
* Route OUTPUT KIND (GitLab issue #1, incident 2026-10-11). `json` (the default)
|
|
198
|
+
* is today's contract: the response MUST parse as JSON, then conform to
|
|
199
|
+
* `outputSchema` when one is declared. `text` is the prose/markdown contract:
|
|
200
|
+
* the response is returned VERBATIM as `output` (a string) with no JSON parse,
|
|
201
|
+
* so a route serving prose is never rejected `content_invalid` for not being
|
|
202
|
+
* JSON. Declared in the bundle YAML as `output: text` (see route-bundles.md).
|
|
203
|
+
*/
|
|
204
|
+
export type OutputKind = "json" | "text";
|
|
170
205
|
export interface RouteBundle {
|
|
171
206
|
readonly route: string;
|
|
172
207
|
readonly version: number;
|
|
173
208
|
/** Resolved, parsed JSON Schema (null = unstructured route). */
|
|
174
209
|
readonly outputSchema: object | null;
|
|
210
|
+
/** Output kind (issue #1). ONLY ever carried as `"text"` — the loader OMITS
|
|
211
|
+
* the field for a JSON route (absent ⇒ `json` ⇒ BYTE-IDENTICAL behaviour and
|
|
212
|
+
* an unchanged `bundle_sha` for every existing route). A `text` route has a
|
|
213
|
+
* null `outputSchema` (the loader refuses `output: text` + `schema:`), skips
|
|
214
|
+
* JSON parsing in `validateOutput`, and never enters the JSON repair rung.
|
|
215
|
+
* Carried on the resolved (mirror) bundle so the hosted lane honours it.
|
|
216
|
+
* Additive & optional. */
|
|
217
|
+
readonly outputKind?: OutputKind;
|
|
218
|
+
/** `output: { kind: text, min_chars: N }` — the one declared, fail-closed check
|
|
219
|
+
* a prose route may carry: a response shorter than N characters (untrimmed)
|
|
220
|
+
* is `content_invalid` (retried per the class budget, never JSON-repaired).
|
|
221
|
+
* Absent ⇒ no floor (any string, including empty, is accepted verbatim).
|
|
222
|
+
* Text routes only (load error otherwise). */
|
|
223
|
+
readonly outputMinChars?: number;
|
|
175
224
|
readonly candidates: readonly CandidateSpec[];
|
|
176
225
|
readonly require: readonly Requirement[];
|
|
177
226
|
readonly prefer: readonly Preference[];
|
|
178
227
|
/** Raw template text with {{var}} and {{#if capability.X}} blocks. */
|
|
179
228
|
readonly systemTemplate: string;
|
|
229
|
+
/**
|
|
230
|
+
* Per-call USER-turn template (GitLab issue #5, R1) — `prompt.user:` in the
|
|
231
|
+
* bundle YAML, rendered with the SAME `variables` / capability conditionals /
|
|
232
|
+
* `x-trust` removal as `systemTemplate`. When present, the rendered text IS the
|
|
233
|
+
* user turn of every primary attempt (replacing the fixed trigger sentence),
|
|
234
|
+
* and a same-model repair re-ask carries it ahead of the repair directive.
|
|
235
|
+
* This is how a route carries the per-call DELTA next to a customer cache
|
|
236
|
+
* handle: on a Gemini dispatch that sets `cachedContent` the adapter cannot
|
|
237
|
+
* send `systemInstruction` (the API rejects both), so the cache holds the
|
|
238
|
+
* shared prefix and `prompt.user` holds what changes per call — nothing is
|
|
239
|
+
* dropped. Present ONLY when declared (absent ⇒ the fixed trigger ⇒
|
|
240
|
+
* BYTE-IDENTICAL behaviour and an unchanged `bundle_sha`). Carried on the
|
|
241
|
+
* resolved (mirror) bundle so the hosted lane renders the same turn.
|
|
242
|
+
* Additive & optional.
|
|
243
|
+
*/
|
|
244
|
+
readonly userTemplate?: string;
|
|
180
245
|
readonly variables: readonly string[];
|
|
181
246
|
readonly policy: RoutePolicy;
|
|
182
247
|
/** Base-config repair rung (Phase 3 WS2) — variants may override. */
|
|
@@ -225,6 +290,46 @@ export interface RouteBundle {
|
|
|
225
290
|
* route load (mask-loader.ts + policy.ts) so an unreadable dictionary or an
|
|
226
291
|
* unknown type is a load error, never a runtime surprise. Additive & optional. */
|
|
227
292
|
readonly mask?: ResolvedMaskPolicy;
|
|
293
|
+
/** Screening SC-1a (spec §4) — the resolved `screen:` block: mode, per-source
|
|
294
|
+
* tiers, built detectors (T0 always + any available T1 classifier), per-tier
|
|
295
|
+
* actions, threshold, output controls, and `policy_hash`. Absent ⇒ `mode: off`
|
|
296
|
+
* ⇒ screening off and BYTE-IDENTICAL behaviour (the golden test). Resolved at
|
|
297
|
+
* route load (screen-loader.ts + screen/policy.ts) so an unknown detector /
|
|
298
|
+
* action or a loosening `sources` key is a load error, never a runtime
|
|
299
|
+
* surprise. Additive & optional. */
|
|
300
|
+
readonly screen?: ResolvedScreenPolicy;
|
|
301
|
+
/** SC-2 (spec-sc2 §4) — the route-level data classification. A `phi` value
|
|
302
|
+
* gates the route posture at load (phi-gate.ts): the route must mask (≥ redact)
|
|
303
|
+
* AND (`require: zero_retention` OR a `sidecar` detector present), else it
|
|
304
|
+
* refuses to load with a teaching message. Rides the existing
|
|
305
|
+
* `data_classification` enum (00058) — zero-DDL. Additive & optional. */
|
|
306
|
+
readonly classification?: DataClassification;
|
|
307
|
+
/** Per-route grade pass threshold (grade primitive, R2) — the score at or above
|
|
308
|
+
* which a grade's DERIVED verdict is `pass`. Declared here (YAML: `grade: {
|
|
309
|
+
* pass_threshold: 0.7 }`) and mirrored, but UNREAD until GP-2 wires verdict
|
|
310
|
+
* derivation; a subject with no route (raw lane) uses 0.5. Absent ⇒ default 0.5.
|
|
311
|
+
* Additive & optional. */
|
|
312
|
+
readonly grade?: {
|
|
313
|
+
readonly passThreshold: number;
|
|
314
|
+
};
|
|
315
|
+
/** PSR-2 (spec §2.1/§3) — the served prompt's assembly manifest, structure-only.
|
|
316
|
+
* A promote writes the promoted `prompt_versions` row's `manifest` THROUGH into
|
|
317
|
+
* this bundle (+ recomputes `bundle_sha`) so `routes.bundle` is the served
|
|
318
|
+
* projection of the SoR. Carried on the wire so the SDK's client-fill engine
|
|
319
|
+
* (PSR-2b) can resolve each slot locally. Absent ⇒ no PSR manifest for this
|
|
320
|
+
* route (pre-PSR behavior, byte-identical). Structure only — no resolved values
|
|
321
|
+
* (D6). Additive & optional. */
|
|
322
|
+
readonly manifest?: Manifest;
|
|
323
|
+
/** Unified Route U2 (ruling D-2) — the serving `destination` for this route:
|
|
324
|
+
* `direct` (today's behaviour: the configured/BYOK key) or `modelrig-served`
|
|
325
|
+
* (ModelRig serves the model on its OWN managed key). Validated at load
|
|
326
|
+
* (routes/loader.ts) against the DESTINATIONS vocabulary. Absent ⇒ resolves to
|
|
327
|
+
* `direct` ⇒ BYTE-IDENTICAL (the top acceptance bar). A `RunOptions.destination`
|
|
328
|
+
* per-run override wins over this. In U2 a resolved `modelrig-served` FAILS
|
|
329
|
+
* CLOSED (managed serving is gated behind the `config.managedServing` flag, U3)
|
|
330
|
+
* rather than silently serving `direct` — fail-closed-but-discoverable. Additive
|
|
331
|
+
* & optional. */
|
|
332
|
+
readonly destination?: Destination;
|
|
228
333
|
}
|
|
229
334
|
export interface RunTags {
|
|
230
335
|
readonly run_id?: string;
|
|
@@ -252,6 +357,12 @@ export interface RunOptions {
|
|
|
252
357
|
/** Named variant to serve (Phase 3 WS1); omitted = the default variant
|
|
253
358
|
* (the route's base config). Unknown names are config errors. */
|
|
254
359
|
readonly variant?: string;
|
|
360
|
+
/** LRE-2 (live A/B): the experiment arm this call was drawn onto, stamped onto
|
|
361
|
+
* every recorded row's `inferences.meta.arm` so the console's A/B readout groups
|
|
362
|
+
* outcomes by arm. Set by the caller after `resolveCallConfigForRun` draws an
|
|
363
|
+
* arm locally; ABSENT ⇒ no `meta.arm` (byte-identical to a non-experiment run).
|
|
364
|
+
* A pure identifier — non-meta-safe values are dropped by the meta producer. */
|
|
365
|
+
readonly arm?: string;
|
|
255
366
|
/** Per-run data-governance opt-in (G-3). `true` demands zero-retention
|
|
256
367
|
* routing for THIS run — `zero_retention` is unioned into the run's effective
|
|
257
368
|
* `require`, so it routes only to ZDR-designated endpoints (empty set →
|
|
@@ -317,10 +428,34 @@ export interface RunOptions {
|
|
|
317
428
|
* hosted equivalent is the route-declared judge (Q2). See boundary B1.
|
|
318
429
|
*/
|
|
319
430
|
readonly qualityGate?: QualityGate;
|
|
431
|
+
/**
|
|
432
|
+
* S3 proxy-role validation (spec §11.3) — per-call input-variable names to
|
|
433
|
+
* treat as `proxy_user` (untrusted) for THIS run. Distinct from the route-level
|
|
434
|
+
* `variableTrust` (`x-trust`, a bundle annotation): `proxyTrust` is a per-call
|
|
435
|
+
* signal the SERVER owns, because the slug/proxy-lane default flip is a lane
|
|
436
|
+
* default (B3b), not a route annotation, and it rides on `RunOptions` (which the
|
|
437
|
+
* server builds and passes to `run`/`runBundle`) rather than through the mirror
|
|
438
|
+
* bundle. Each named variable present in `input` is:
|
|
439
|
+
* (1) removed from the trusted system slot — its `{{name}}` renders empty,
|
|
440
|
+
* reusing the exact x-trust removal path (routes/template.ts); and
|
|
441
|
+
* (2) emitted as a `{ source: "proxy_user", tier: "untrusted" }` segment,
|
|
442
|
+
* spotlighted into the untrusted user block by the provenance assembler.
|
|
443
|
+
* Additive & optional; ABSENT ⇒ unchanged behaviour (byte-identical). The seam
|
|
444
|
+
* is INERT until a caller sets it — S3 sets it NOWHERE (the request-edge trust
|
|
445
|
+
* rule ships as validation + tests; the structural flip is B3b).
|
|
446
|
+
*/
|
|
447
|
+
readonly proxyTrust?: readonly string[];
|
|
448
|
+
/** Unified Route U2 (ruling D-2) — per-run serving `destination` override. Wins
|
|
449
|
+
* over the route's declared `destination` when resolving the effective served
|
|
450
|
+
* destination (RunOptions > route > default `direct`). Absent ⇒ the route's
|
|
451
|
+
* declared `destination` stands (itself absent ⇒ `direct` ⇒ BYTE-IDENTICAL).
|
|
452
|
+
* The hosted gateway's YAML→RunOptions mapping of this key is a
|
|
453
|
+
* `modelrig-server` follow-up, out of this package's scope. Additive & optional. */
|
|
454
|
+
readonly destination?: Destination;
|
|
320
455
|
}
|
|
321
456
|
export interface RunMeta {
|
|
322
|
-
/** Telemetry row id of the serving attempt — the
|
|
323
|
-
*
|
|
457
|
+
/** Telemetry row id of the serving attempt — the id a grade attaches to via
|
|
458
|
+
* `rig.grade({ kind: "inference", id })`. */
|
|
324
459
|
readonly inferenceId: string;
|
|
325
460
|
readonly route: string;
|
|
326
461
|
readonly routeVersion: number;
|
|
@@ -377,6 +512,23 @@ export interface RunMeta {
|
|
|
377
512
|
* issued/rehydrated/unresolved). Present only when the route has an active
|
|
378
513
|
* `mask:` policy; NEVER carries a matched value. Additive & optional. */
|
|
379
514
|
readonly mask?: MaskReceipt;
|
|
515
|
+
/** Screening SC-1a receipt (spec §5) — the counts-only screen summary for this
|
|
516
|
+
* run's serving inference (policy hash, detectors, findings by type × source,
|
|
517
|
+
* risk bucket, action, and any quarantine/canary/citation/degraded counts).
|
|
518
|
+
* Present only when the route has an active `screen:` policy; NEVER carries a
|
|
519
|
+
* matched value. Rides `inferences.meta.screen` in telemetry (spec §5, §8 — no
|
|
520
|
+
* new column). Additive & optional. */
|
|
521
|
+
readonly screen?: ScreenReceipt;
|
|
522
|
+
/** Unified Route U2 (ruling D-2) — the RESOLVED served destination for this run
|
|
523
|
+
* (RunOptions override > route > default `direct`). Additive RESULT metadata,
|
|
524
|
+
* like `servedTier`: the routed lane populates it on every returned run so
|
|
525
|
+
* telemetry/console can read WHERE the call was served. This is NOT the wire
|
|
526
|
+
* telemetry row — stamping `destination` on the row is gated to a NON-default
|
|
527
|
+
* value (destinationTag), so an absent/`direct` route's row is byte-identical.
|
|
528
|
+
* The raw inline on-ramp omits it (its destination is always `direct`
|
|
529
|
+
* implicitly). In U2 a returned run's value is always `direct` (a resolved
|
|
530
|
+
* `modelrig-served` fails closed pre-dispatch). Additive & optional. */
|
|
531
|
+
readonly destination?: Destination;
|
|
380
532
|
}
|
|
381
533
|
export interface RunResult {
|
|
382
534
|
readonly output: unknown;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"route.d.ts","sourceRoot":"","sources":["../../src/types/route.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAChH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"route.d.ts","sourceRoot":"","sources":["../../src/types/route.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAChH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,KAAK,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC3E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAC;AACtE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAEpD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,UAAU,GAAG,gBAAgB,GAAG,eAAe,CAAC;AAChG,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,SAAS,CAAC;AAE5C,MAAM,WAAW,WAAW;IAC1B,0EAA0E;IAC1E,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,WAAW,CAAC;IACtC;;;;;;;mCAO+B;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,cAAc,CAAC;CACpC;AAED,uDAAuD;AACvD,MAAM,WAAW,YAAY;IAC3B;;;kCAG8B;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;oEACgE;IAChE,QAAQ,CAAC,WAAW,CAAC,EAAE,aAAa,CAAC;CACtC;AAED;6EAC6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;wBAEoB;IACpB,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IAC/C;6EACyE;IACzE,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC;gCAC4B;IAC5B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B;4DACwD;IACxD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,2CAA2C;IAC3C,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;IAC/B;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED;;;;+CAI+C;AAC/C,MAAM,MAAM,aAAa,GAAG,eAAe,GAAG,UAAU,GAAG,YAAY,CAAC;AAExE;;;8BAG8B;AAC9B,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,gFAAgF;IAChF,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY,CAAC;IACrC,kFAAkF;IAClF,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;OAOG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;CAClC;AAED;;;+EAG+E;AAC/E,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;KAAE,GAAG;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC7E;AAED;0DAC0D;AAC1D,MAAM,WAAW,aAAa;IAC5B;;wFAEoF;IACpF,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG;QAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,2EAA2E;IAC3E,QAAQ,CAAC,sBAAsB,CAAC,EAAE,OAAO,CAAC;IAC1C,qEAAqE;IACrE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IACvC;;;+BAG2B;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAChF;;;;;;;;;OASG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,CAAC,EAAE;YAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;YAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;IACxG;;;;;;yEAMqE;IACrE,QAAQ,CAAC,aAAa,CAAC,EAAE;QACvB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;QAC1B,qEAAqE;QACrE,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;KAChC,CAAC;IACF;;;;;;qDAMiD;IACjD,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;CACjC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;AAEzC,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,gEAAgE;IAChE,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC;;;;;;8BAM0B;IAC1B,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;;;;kDAI8C;IAC9C,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,CAAC;IAC9C,QAAQ,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,CAAC;IACvC,sEAAsE;IACtE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;;;;;;;;;;;;;;OAcG;IACH,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,qEAAqE;IACrE,QAAQ,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC;IAC/B,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C,sEAAsE;IACtE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B;;;;;;uCAMmC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB;;wEAEoE;IACpE,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC;kEAC8D;IAC9D,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;4BACwB;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,cAAc,CAAC;IACpC,sDAAsD;IACtD,QAAQ,CAAC,OAAO,CAAC,EAAE,aAAa,CAAC;IACjC,kEAAkE;IAClE,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C;;;;;0EAKsE;IACtE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;IACpD;;;iFAG6E;IAC7E,QAAQ,CAAC,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;IAC/D;;;;sFAIkF;IAClF,QAAQ,CAAC,IAAI,CAAC,EAAE,kBAAkB,CAAC;IACnC;;;;;;wCAMoC;IACpC,QAAQ,CAAC,MAAM,CAAC,EAAE,oBAAoB,CAAC;IACvC;;;;6EAIyE;IACzE,QAAQ,CAAC,cAAc,CAAC,EAAE,kBAAkB,CAAC;IAC7C;;;;8BAI0B;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;IACpD;;;;;;oCAMgC;IAChC,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAC7B;;;;;;;;qBAQiB;IACjB,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;CAC5C;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB;;;;mFAI+E;IAC/E,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE;QAChB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B;;;;kFAI0E;QAC1E,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF;qEACiE;IACjE,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;oFAIgF;IAChF,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;oCAMgC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;IACjC;;;;wEAIoE;IACpE,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,0EAA0E;IAC1E,QAAQ,CAAC,SAAS,CAAC,EAAE;QACnB,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;QACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;KACzC,CAAC;IACF;;;uDAGmD;IACnD,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE;QACf,iEAAiE;QACjE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QACrB;;;;;;iCAMyB;QACzB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC9B;2EACmE;QACnE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;KAC9B,CAAC;IACF;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;IACnC;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC;;;;;wFAKoF;IACpF,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,OAAO;IACtB;iDAC6C;IAC7C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,WAAW,CAAC;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE,2EAA2E;IAC3E,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;wBACoB;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;;;+EAG2E;IAC3E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB;gDAC4C;IAC5C,QAAQ,CAAC,SAAS,CAAC,EAAE,aAAa,CAAC;QACjC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;QAC9B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC,CAAC;IACH;;iFAE6E;IAC7E,QAAQ,CAAC,SAAS,CAAC,EAAE;QAAE,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5D;;gFAE4E;IAC5E,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;IACzC;;;;iFAI6E;IAC7E,QAAQ,CAAC,OAAO,CAAC,EAAE;QACjB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;QAClC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;KAClC,CAAC;IACF;;;6EAGyE;IACzE,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC;IAC5B;;;;;2CAKuC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC;IAChC;;;;;;;;4EAQwE;IACxE,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;CACxB"}
|
|
@@ -113,38 +113,75 @@ export interface EnvelopeRow {
|
|
|
113
113
|
readonly spentUsd: number;
|
|
114
114
|
readonly status: "open" | "exhausted";
|
|
115
115
|
}
|
|
116
|
-
/** One
|
|
117
|
-
*
|
|
118
|
-
|
|
119
|
-
|
|
116
|
+
/** One evidence citation on a per-criterion grade — a verbatim quote drawn from
|
|
117
|
+
* the subject, optionally anchored to a transcript turn (call-qa F2/F4). */
|
|
118
|
+
export interface GradeCriterionEvidence {
|
|
119
|
+
readonly quote: string;
|
|
120
|
+
readonly turn_id?: number;
|
|
121
|
+
}
|
|
122
|
+
/** One criterion's outcome on a HUMAN scorecard (call-qa F4 — the human twin of
|
|
123
|
+
* the server's model-emitted ScorecardCriterion). `score` is the normalized 0–1
|
|
124
|
+
* sub-score; `evidence`/`comment` are the analyst's grounded justification. A
|
|
125
|
+
* human grade may OPTIONALLY carry an array of these; when present the server
|
|
126
|
+
* persists them to `grade_criteria` (kind semantics = human) so per-criterion
|
|
127
|
+
* agreement (F4 calibration) has a human side to pair. Absent ⇒ today's
|
|
128
|
+
* single-score human grade, unchanged. */
|
|
129
|
+
export interface GradeCriterion {
|
|
120
130
|
readonly id: string;
|
|
121
|
-
readonly
|
|
122
|
-
readonly
|
|
123
|
-
readonly
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
131
|
+
readonly score: number;
|
|
132
|
+
readonly evidence?: readonly GradeCriterionEvidence[];
|
|
133
|
+
readonly comment?: string;
|
|
134
|
+
}
|
|
135
|
+
/** The subject a grade attaches to (grade primitive) — an inference, a run
|
|
136
|
+
* (episode), or an artifact. Exactly one, by construction. */
|
|
137
|
+
export type GradeSubject = {
|
|
138
|
+
readonly kind: "inference";
|
|
139
|
+
readonly id: string;
|
|
140
|
+
} | {
|
|
141
|
+
readonly kind: "run";
|
|
142
|
+
readonly id: string;
|
|
143
|
+
} | {
|
|
144
|
+
readonly kind: "artifact";
|
|
145
|
+
readonly id: string;
|
|
146
|
+
};
|
|
147
|
+
/** One unified judgment (grade primitive, R1) — the local mirror of `public.grades`.
|
|
148
|
+
* `score` is the 0–1 PRIMITIVE (always present; up/pass=1.0, down/fail=0.0, else
|
|
149
|
+
* the graded value). The pass/fail verdict is DERIVED at read time (`pass = score
|
|
150
|
+
* >= threshold`), never stored. `rig.grade` is the sole write API that builds one
|
|
151
|
+
* of these (the legacy feedback/evaluation shapes were removed in GP-3). */
|
|
152
|
+
export interface GradeRow {
|
|
153
|
+
readonly id: string;
|
|
154
|
+
readonly subjectKind: "inference" | "run" | "artifact";
|
|
155
|
+
readonly subjectId: string;
|
|
156
|
+
/** 0–1 primitive; ALWAYS present. A human thumbs-up is 1.0, thumbs-down 0.0
|
|
157
|
+
* (GP-2 dropped the nullable `legacyScore` shadow — score is the sole primitive). */
|
|
158
|
+
readonly score: number;
|
|
159
|
+
/** Optional display scale MAX (grade model `scale.max`) — e.g. out of 100. */
|
|
160
|
+
readonly scaleMax: number | null;
|
|
161
|
+
/** Optional display LABEL (grade model `scale.label`, e.g. "B+"). GP-2 also folds
|
|
162
|
+
* the legacy verdict `label` (a native run-outcome / evaluate summary) into this
|
|
163
|
+
* field, so it is the one display-label column readers read. */
|
|
164
|
+
readonly scaleLabel: string | null;
|
|
165
|
+
/** Written guidance (was feedback.note / evaluation.explanation). */
|
|
166
|
+
readonly comment: string | null;
|
|
167
|
+
readonly kind: "human" | "model-judge" | "deterministic";
|
|
168
|
+
/** The specific grader name (was evaluation.evaluator / the feedback source). */
|
|
169
|
+
readonly grader: string | null;
|
|
170
|
+
/** Provenance (was feedback.source) — "sdk", "console", a review-agent name. */
|
|
171
|
+
readonly source: string | null;
|
|
172
|
+
/** The grade's own org, when recorded through a server on behalf of another
|
|
173
|
+
* org (the feedback.org_id doctrine, QA-P6-010). Null = the exporter's org. */
|
|
143
174
|
readonly orgId: string | null;
|
|
144
|
-
/** Authenticated user who recorded it (00017's `feedback.created_by`), or
|
|
145
|
-
* null for an SDK-recorded verdict — which is a fact about the row, not
|
|
146
|
-
* missing data, and is never backfilled. */
|
|
147
175
|
readonly createdBy: string | null;
|
|
176
|
+
/** Judgment timestamp (feedback carried one; evaluations fall back to created_at). */
|
|
177
|
+
readonly ts: string | null;
|
|
178
|
+
/** OPTIONAL per-criterion scorecard (call-qa F4). Present only on a human grade
|
|
179
|
+
* carrying a breakdown; the exporter mirrors it on the wire (WireGrade.criteria)
|
|
180
|
+
* and the server persists it to `grade_criteria` (kind=human) after the parent
|
|
181
|
+
* grades row lands. Absent ⇒ today's single-score grade — no grade_criteria.
|
|
182
|
+
* Additive: stored locally as a JSON `criteria` TEXT column (v16), read back as
|
|
183
|
+
* null for any row written before F4. */
|
|
184
|
+
readonly criteria?: readonly GradeCriterion[];
|
|
148
185
|
}
|
|
149
186
|
/** One execution of a customer pipeline (Expansion E2 chunk A). Local SQLite is
|
|
150
187
|
* the write path (run.start/run.end); the exporter mirrors it to `public.runs`
|
|
@@ -215,24 +252,10 @@ export interface ArtifactEdgeRow {
|
|
|
215
252
|
readonly toArtifactId: string;
|
|
216
253
|
readonly kind: "consumed" | "derived_from";
|
|
217
254
|
}
|
|
218
|
-
/**
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
readonly artifactId: string | null;
|
|
223
|
-
/** The run subject (E5 chunk V) — null when this evaluates an artifact.
|
|
224
|
-
* Exactly one of artifactId / runId is set. */
|
|
225
|
-
readonly runId: string | null;
|
|
226
|
-
readonly evaluator: string;
|
|
227
|
-
readonly evaluatorKind: "deterministic" | "model-judge" | "human";
|
|
228
|
-
readonly score: number | null;
|
|
229
|
-
readonly passed: boolean | null;
|
|
230
|
-
readonly label: string | null;
|
|
231
|
-
readonly explanation: string | null;
|
|
232
|
-
readonly createdBy: string | null;
|
|
233
|
-
}
|
|
234
|
-
/** How a case entered the eval suite (E4 chunk A, migration 00040 CHECK). */
|
|
235
|
-
export type EvalCaseOrigin = "promoted-failure" | "human-corrected" | "manual";
|
|
255
|
+
/** How a case entered the eval suite (E4 chunk A, migration 00040 CHECK; the
|
|
256
|
+
* imported-* origins added by the dataset-source ingest thread, migration 00078
|
|
257
|
+
* CHECK). */
|
|
258
|
+
export type EvalCaseOrigin = "promoted-failure" | "human-corrected" | "manual" | "imported-ticket" | "imported-chat";
|
|
236
259
|
/** A per-org regression/golden eval case (E4 chunk A). `correctedArtifactId`
|
|
237
260
|
* null ⇒ a failure-case; set ⇒ a golden-corrected pair. A fact on ingest
|
|
238
261
|
* (created via promote); edits/deletes are server-side, not on this stream. */
|
|
@@ -247,7 +270,9 @@ export interface EvalCaseRow {
|
|
|
247
270
|
}
|
|
248
271
|
export interface TelemetryStore {
|
|
249
272
|
record(row: TelemetryRow): void;
|
|
250
|
-
|
|
273
|
+
/** Record a unified judgment (grade primitive, R1) — the canonical write path
|
|
274
|
+
* that `rig.grade` uses. */
|
|
275
|
+
recordGrade(row: GradeRow): void;
|
|
251
276
|
queryByTag(tag: keyof RunTags & string, value: string): TelemetryRow[];
|
|
252
277
|
getEnvelope(key: string): EnvelopeRow | null;
|
|
253
278
|
accrue(key: string, usd: number): EnvelopeRow;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/types/telemetry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC1E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,WAAW,GACX,kBAAkB,CAAC;AAEvB,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;;wCAEoC;IACpC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,sDAAsD;IACtD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;;;kFAM8E;IAC9E,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;wFACoF;IACpF,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;uDACmD;IACnD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;wEACoE;IACpE,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;oDAGgD;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;oFACgF;IAChF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C;;6DAEyD;IACzD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC;oDACgD;IAChD,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;uDACmD;IACnD,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;;;;;;+EAM2E;IAC3E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC;mEAC+D;IAC/D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;;sDAGkD;IAClD,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;;;uDAKmD;IACnD,QAAQ,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACrC;;;;uEAImE;IACnE,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC;CACvC;AAED
|
|
1
|
+
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/types/telemetry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC1E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,WAAW,GACX,kBAAkB,CAAC;AAEvB,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;;wCAEoC;IACpC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,sDAAsD;IACtD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;;;kFAM8E;IAC9E,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;wFACoF;IACpF,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;uDACmD;IACnD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;wEACoE;IACpE,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;oDAGgD;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;oFACgF;IAChF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C;;6DAEyD;IACzD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC;oDACgD;IAChD,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;uDACmD;IACnD,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;;;;;;+EAM2E;IAC3E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC;mEAC+D;IAC/D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;;sDAGkD;IAClD,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;;;uDAKmD;IACnD,QAAQ,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACrC;;;;uEAImE;IACnE,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC;CACvC;AAED;4EAC4E;AAC5E,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;0CAM0C;AAC1C,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,sBAAsB,EAAE,CAAC;IACtD,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;8DAC8D;AAC9D,MAAM,MAAM,YAAY,GACpB;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GACnD;IAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAC7C;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvD;;;;4EAI4E;AAC5E,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,EAAE,WAAW,GAAG,KAAK,GAAG,UAAU,CAAC;IACvD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;yFACqF;IACrF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,8EAA8E;IAC9E,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC;;oEAEgE;IAChE,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qEAAqE;IACrE,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,aAAa,GAAG,eAAe,CAAC;IACzD,iFAAiF;IACjF,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,gFAAgF;IAChF,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B;mFAC+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,sFAAsF;IACtF,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B;;;;;6CAKyC;IACzC,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;CAC/C;AAED;;;;cAIc;AACd,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;IAClE;;;6EAGyE;IACzE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;CAClC;AAED;;;gDAGgD;AAChD,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,cAAc,GAAG,UAAU,CAAC;IACjD,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED;;mFAEmF;AACnF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;;;kEAG8D;IAC9D,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,sDAAsD;AACtD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,cAAc,CAAC;CAC5C;AAED;;aAEa;AACb,MAAM,MAAM,cAAc,GACtB,kBAAkB,GAClB,iBAAiB,GACjB,QAAQ,GACR,iBAAiB,GACjB,eAAe,CAAC;AAEpB;;+EAE+E;AAC/E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,GAAG,EAAE,YAAY,GAAG,IAAI,CAAC;IAChC;gCAC4B;IAC5B,WAAW,CAAC,GAAG,EAAE,QAAQ,GAAG,IAAI,CAAC;IACjC,UAAU,CAAC,GAAG,EAAE,MAAM,OAAO,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAAC;IACvE,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAAC;IAC7C,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,WAAW,CAAC;IAC9C,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,WAAW,CAAC;IAC1D,KAAK,IAAI,IAAI,CAAC;CACf"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate-cli.d.ts","sourceRoot":"","sources":["../src/validate-cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;
|
|
1
|
+
{"version":3,"file":"validate-cli.d.ts","sourceRoot":"","sources":["../src/validate-cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAWH,wBAAgB,cAAc,IAAI,MAAM,CAwGvC"}
|
package/dist/validate-cli.js
CHANGED
|
@@ -18,6 +18,7 @@ const loader_1 = require("./routes/loader");
|
|
|
18
18
|
const manifest_1 = require("./routes/manifest");
|
|
19
19
|
const attempt_meta_1 = require("./telemetry/attempt-meta");
|
|
20
20
|
const status_runs_1 = require("./status-runs");
|
|
21
|
+
const phi_gate_1 = require("./screen/phi-gate");
|
|
21
22
|
function runValidateCli() {
|
|
22
23
|
const config = (0, config_1.loadConfigFromEnv)();
|
|
23
24
|
const routesDir = (0, rawonly_1.requireRoutesDir)(config, "modelrig validate");
|
|
@@ -67,6 +68,37 @@ function runValidateCli() {
|
|
|
67
68
|
console.error(`FAIL ${r.route ?? r.file}: ${r.problems.join("; ")}`);
|
|
68
69
|
}
|
|
69
70
|
}
|
|
71
|
+
// SC-1a (spec §9): print the EFFECTIVE `screen:` policy per route so an author
|
|
72
|
+
// sees exactly what will run (mode, detectors + versions, per-tier actions,
|
|
73
|
+
// threshold, and any DEGRADED optional detector). The `x-trust` warnings are
|
|
74
|
+
// already emitted at load time (loader.ts). Best-effort: only when every route
|
|
75
|
+
// loaded cleanly (a failing route has no bundle to read).
|
|
76
|
+
if (failures === 0) {
|
|
77
|
+
try {
|
|
78
|
+
for (const b of (0, loader_1.loadRoutes)(routesDir).list()) {
|
|
79
|
+
// SC-2 (spec-sc2 §4): print the resolved `phi` posture (mask/zdr/sidecar)
|
|
80
|
+
// and, if unmet, the specific requirement — even a phi route with no
|
|
81
|
+
// screen block (a mis-postured one already failed to load above, so this
|
|
82
|
+
// prints only for a route that passed the gate: the OK posture).
|
|
83
|
+
if (b.classification === "phi") {
|
|
84
|
+
const posture = (0, phi_gate_1.describePhiPosture)({ classification: b.classification, mask: b.mask, screen: b.screen, require: b.require });
|
|
85
|
+
console.log(` ${b.route}: ${posture}`);
|
|
86
|
+
}
|
|
87
|
+
if (b.screen === undefined)
|
|
88
|
+
continue;
|
|
89
|
+
const p = b.screen.policy;
|
|
90
|
+
console.log(` screen ${b.route}: mode=${p.mode} · detectors=[${b.screen.detectorMeta.map((d) => d.id).join(", ")}]` +
|
|
91
|
+
` · threshold=${p.threshold ?? 0.8}` +
|
|
92
|
+
(b.screen.degraded ? " · DEGRADED (an optional detector is unavailable — running T0 only)" : ""));
|
|
93
|
+
if (p.on !== undefined && Object.keys(p.on).length > 0) {
|
|
94
|
+
console.log(` actions: ${Object.entries(p.on).map(([t, a]) => `${t}→${a}`).join(", ")}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
/* a bundle failed to reload — already surfaced as a FAIL line above */
|
|
100
|
+
}
|
|
101
|
+
}
|
|
70
102
|
// GS-1 teaching hint: a rig that is USED (has recorded inferences) but never
|
|
71
103
|
// wrapped an execution in run.start / run.scope records no runs, so /runs stays
|
|
72
104
|
// empty and /projects shows "(no run)". A WARN, never a failure — the config is
|
package/dist/verify/verify.d.ts
CHANGED
|
@@ -24,6 +24,11 @@ export interface VerifySwapRequest {
|
|
|
24
24
|
readonly envelopeUsd?: number;
|
|
25
25
|
/** Injectable clock for a deterministic `ranAt` (tests). */
|
|
26
26
|
readonly now?: () => string;
|
|
27
|
+
/** CS-1 Chunk B (spec §7.4): what this equivalence run is FOR — `swap`
|
|
28
|
+
* (default), `mask-policy` (a Tier-2 migration PR that adds masking, so the
|
|
29
|
+
* PR carries equivalence semantics for the masked route), or `screen-assembly`
|
|
30
|
+
* (SC-0 B3). Additive & optional; stamped verbatim onto the report. */
|
|
31
|
+
readonly subject?: EquivalenceReport["subject"];
|
|
27
32
|
}
|
|
28
33
|
export declare function verifySwap(deps: RunDeps, req: VerifySwapRequest): Promise<EquivalenceReport>;
|
|
29
34
|
//# sourceMappingURL=verify.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"verify.d.ts","sourceRoot":"","sources":["../../src/verify/verify.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAK1C,OAAO,KAAK,EACV,iBAAiB,EAIjB,oBAAoB,EACrB,MAAM,SAAS,CAAC;AAWjB,MAAM,WAAW,iBAAiB;IAChC,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,2EAA2E;IAC3E,QAAQ,CAAC,OAAO,EAAE,oBAAoB,CAAC;IACvC,wDAAwD;IACxD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,4DAA4D;IAC5D,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"verify.d.ts","sourceRoot":"","sources":["../../src/verify/verify.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAK1C,OAAO,KAAK,EACV,iBAAiB,EAIjB,oBAAoB,EACrB,MAAM,SAAS,CAAC;AAWjB,MAAM,WAAW,iBAAiB;IAChC,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,2EAA2E;IAC3E,QAAQ,CAAC,OAAO,EAAE,oBAAoB,CAAC;IACvC,wDAAwD;IACxD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,4DAA4D;IAC5D,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IAC5B;;;2EAGuE;IACvE,QAAQ,CAAC,OAAO,CAAC,EAAE,iBAAiB,CAAC,SAAS,CAAC,CAAC;CACjD;AAkCD,wBAAsB,UAAU,CAC9B,IAAI,EAAE,OAAO,EACb,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,iBAAiB,CAAC,CAuH5B"}
|
package/dist/verify/verify.js
CHANGED
|
@@ -34,6 +34,7 @@ function refused(req, routeVersion, pinnedModel, reason, nowIso) {
|
|
|
34
34
|
return {
|
|
35
35
|
schemaVersion: 1,
|
|
36
36
|
kind: "shadow-equivalence",
|
|
37
|
+
...(req.subject !== undefined ? { subject: req.subject } : {}),
|
|
37
38
|
callSiteId: req.fixture.callSiteId,
|
|
38
39
|
route: req.route,
|
|
39
40
|
routeVersion,
|
|
@@ -132,6 +133,7 @@ async function verifySwap(deps, req) {
|
|
|
132
133
|
return {
|
|
133
134
|
schemaVersion: 1,
|
|
134
135
|
kind: "shadow-equivalence",
|
|
136
|
+
...(req.subject !== undefined ? { subject: req.subject } : {}),
|
|
135
137
|
callSiteId: req.fixture.callSiteId,
|
|
136
138
|
route: req.route,
|
|
137
139
|
routeVersion: bundle.version,
|
package/dist/watch/evaluate.d.ts
CHANGED
|
@@ -56,7 +56,28 @@ export interface WatchConfig {
|
|
|
56
56
|
* Analyst's triggers ride the widened kinds (`insight`/`promote-eval`) or reuse
|
|
57
57
|
* `bakeoff` for an Analyst-initiated bake-off — all through the SAME proposals
|
|
58
58
|
* machinery and the SAME `submitDrafts` seam (trajectory F1/F4, one emission path). */
|
|
59
|
-
export type ProposalTrigger = EconomicsTrigger | ConformDegradationTrigger | AnalystClusterTrigger | AnalystPromoteEvalTrigger | AnalystBakeoffTrigger | ManualChallengerTrigger;
|
|
59
|
+
export type ProposalTrigger = EconomicsTrigger | ConformDegradationTrigger | AnalystClusterTrigger | AnalystPromoteEvalTrigger | AnalystBakeoffTrigger | ManualChallengerTrigger | UpgradeOpportunityTrigger | RegretBreachTrigger | OptimizerImprovementTrigger;
|
|
60
|
+
/**
|
|
61
|
+
* IP-2a-2: an EXPLICIT optimizer run raised this (the /graders/[route] console
|
|
62
|
+
* action or the `run_optimizer` MCP tool — spec-ip2 §2; no cron, no sweep).
|
|
63
|
+
* `kind` stays `"swap"`: the winning arm is a variant swap in the wide sense
|
|
64
|
+
* (a model pin, a prompt override, or both), delivered through the SAME
|
|
65
|
+
* never-silent machinery — the change_spec on the evidence is the
|
|
66
|
+
* machine-readable half. `bakeoffId` (the eval run that PROVED the winner) is
|
|
67
|
+
* deliberately INSIDE the hash: each explicit run is its own decision record,
|
|
68
|
+
* never deduped against an earlier run's.
|
|
69
|
+
*/
|
|
70
|
+
export interface OptimizerImprovementTrigger {
|
|
71
|
+
readonly type: "optimizer_improvement";
|
|
72
|
+
readonly route: string;
|
|
73
|
+
/** Which inputs the engine ran on: prompt bytes (G1 open) or config-only. */
|
|
74
|
+
readonly mode: "prompt" | "config";
|
|
75
|
+
/** The bake-off (eval run) this run scored candidates through. */
|
|
76
|
+
readonly bakeoffId: string;
|
|
77
|
+
/** Honest input counts at trigger time. */
|
|
78
|
+
readonly evalCasesN: number;
|
|
79
|
+
readonly gradesN: number;
|
|
80
|
+
}
|
|
60
81
|
/**
|
|
61
82
|
* A human-initiated declare-first challenger (routing-console spec §2.4 / §3, R3)
|
|
62
83
|
* — the console picker (or the oracle) proposes trialing a specific model on a
|
|
@@ -149,6 +170,68 @@ export interface EconomicsTrigger {
|
|
|
149
170
|
readonly newRates?: TokenRates;
|
|
150
171
|
readonly pricing?: TokenRates | null;
|
|
151
172
|
}
|
|
173
|
+
/** An aggregate regret band over one window — the quality twin of the cost
|
|
174
|
+
* odometer's savings band (spec §2). Carries ONLY aggregate regret numbers +
|
|
175
|
+
* the paired n (content-custody: no capture content, ever). `regretCi95` is a
|
|
176
|
+
* seeded paired bootstrap; `nPaired` is the window's paired-sample count. */
|
|
177
|
+
export interface RegretBandSummary {
|
|
178
|
+
/** Window bounds — inclusive-from / exclusive-to ISO days. */
|
|
179
|
+
readonly from: string;
|
|
180
|
+
readonly to: string;
|
|
181
|
+
readonly regretMean: number;
|
|
182
|
+
readonly regretCi95: readonly [number, number];
|
|
183
|
+
readonly nPaired: number;
|
|
184
|
+
}
|
|
185
|
+
/** N4 upgrade trigger (spec §4.1): the route's reference/ceiling ROSE — a new
|
|
186
|
+
* registry model, a new snapshot of the reference, or a Q1 regret breach — so a
|
|
187
|
+
* bake-off is proposed to measure the opportunity. The mirror image of
|
|
188
|
+
* `price_change`/`new_model`. Fires ONLY for a route that resolves a reference
|
|
189
|
+
* (invariant `upswap-opt-in-only`). */
|
|
190
|
+
export interface UpgradeOpportunityTrigger {
|
|
191
|
+
readonly type: "upgrade_opportunity";
|
|
192
|
+
readonly route: string;
|
|
193
|
+
/** The model that raised the ceiling (new registry entry, or a new snapshot). */
|
|
194
|
+
readonly reference: CandidateEconomics;
|
|
195
|
+
/** Why it fired — console/audit context; the idempotency hash keys on the ref
|
|
196
|
+
* identity (+ snapshot) + cause, NOT the whole trigger. */
|
|
197
|
+
readonly cause: "new_reference" | "reference_snapshot" | "regret_breach";
|
|
198
|
+
/** Present when the fire came from Q1 (cause=regret_breach): the band that
|
|
199
|
+
* tripped ε — so the console shows WHY without a second read. */
|
|
200
|
+
readonly band?: RegretBandSummary;
|
|
201
|
+
}
|
|
202
|
+
/** Q1 regret-breach trigger (spec §5.2): the recent window's regret upper-CI
|
|
203
|
+
* exceeded ε, OR drifted up from baseline by ≥ the ruled margin, with min-n on
|
|
204
|
+
* BOTH sides. Emits `kind: "bakeoff"` — the sampled monitor number is the ALARM,
|
|
205
|
+
* a fresh bake-off is the EVIDENCE (and, on an opted-in route, where an up-swap
|
|
206
|
+
* winner is found → Q1 feeds N4). */
|
|
207
|
+
export interface RegretBreachTrigger {
|
|
208
|
+
readonly type: "regret_breach";
|
|
209
|
+
readonly route: string;
|
|
210
|
+
/** What we measured against. */
|
|
211
|
+
readonly reference: CandidateEconomics;
|
|
212
|
+
readonly recent: RegretBandSummary;
|
|
213
|
+
readonly baseline: RegretBandSummary;
|
|
214
|
+
/** The route's `max_regret` (null ⇒ drift-only breach). */
|
|
215
|
+
readonly epsilon: number | null;
|
|
216
|
+
readonly breachKind: "over_epsilon" | "drift_up";
|
|
217
|
+
readonly minN: number;
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Idempotency key for an `upgrade_opportunity` — per (route, reference, cause,
|
|
221
|
+
* snapshot) ONLY (spec §4.1), so the `band`/audit context never splits a dedupe.
|
|
222
|
+
* A re-run over the SAME ceiling is a no-op via the `(org_id, route,
|
|
223
|
+
* trigger_hash)` unique index; a NEWER snapshot (distinct `snapshotId`) is a
|
|
224
|
+
* genuinely new opportunity and a distinct hash.
|
|
225
|
+
*/
|
|
226
|
+
export declare function upgradeTriggerHash(route: string, reference: CandidateEconomics, cause: UpgradeOpportunityTrigger["cause"], snapshotId?: string): string;
|
|
227
|
+
/**
|
|
228
|
+
* Idempotency key for a `regret_breach` — per (route, reference, window bounds,
|
|
229
|
+
* breachKind) ONLY (spec §5.2), so the sampled band's noisy per-run regret
|
|
230
|
+
* numbers never split a dedupe. The window bounds are day-quantized off the
|
|
231
|
+
* injected `now`, so a re-run the SAME day is a no-op via the `(org_id, route,
|
|
232
|
+
* trigger_hash)` unique index.
|
|
233
|
+
*/
|
|
234
|
+
export declare function regretBreachTriggerHash(route: string, reference: CandidateEconomics, windowFrom: string, windowTo: string, breachKind: RegretBreachTrigger["breachKind"]): string;
|
|
152
235
|
/** One side of the degradation comparison (E4 chunk B) — a window's conform
|
|
153
236
|
* rate over its evaluations. Dates are inclusive-from / exclusive-to ISO days. */
|
|
154
237
|
export interface ConformWindow {
|