modelrig 0.0.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/LICENSE +201 -0
- package/README.md +145 -8
- package/data/PROVENANCE.md +19 -0
- package/data/pricing-manual.json +22 -0
- package/data/pricing-map.json +46362 -0
- package/data/pricing-overlay.json +419 -0
- package/dist/adapters/anthropic.d.ts +90 -0
- package/dist/adapters/anthropic.d.ts.map +1 -0
- package/dist/adapters/anthropic.js +296 -0
- package/dist/adapters/build-adapters.d.ts +11 -0
- package/dist/adapters/build-adapters.d.ts.map +1 -0
- package/dist/adapters/build-adapters.js +45 -0
- package/dist/adapters/deepseek.d.ts +47 -0
- package/dist/adapters/deepseek.d.ts.map +1 -0
- package/dist/adapters/deepseek.js +216 -0
- package/dist/adapters/gemini.d.ts +44 -0
- package/dist/adapters/gemini.d.ts.map +1 -0
- package/dist/adapters/gemini.js +262 -0
- package/dist/adapters/grok.d.ts +53 -0
- package/dist/adapters/grok.d.ts.map +1 -0
- package/dist/adapters/grok.js +232 -0
- package/dist/adapters/openai-compat.d.ts +80 -0
- package/dist/adapters/openai-compat.d.ts.map +1 -0
- package/dist/adapters/openai-compat.js +244 -0
- package/dist/adapters/openai-stream.d.ts +58 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -0
- package/dist/adapters/openai-stream.js +120 -0
- package/dist/adapters/openai.d.ts +55 -0
- package/dist/adapters/openai.d.ts.map +1 -0
- package/dist/adapters/openai.js +281 -0
- package/dist/adapters/sampling.d.ts +28 -0
- package/dist/adapters/sampling.d.ts.map +1 -0
- package/dist/adapters/sampling.js +34 -0
- package/dist/adapters/tool-mapping.d.ts +92 -0
- package/dist/adapters/tool-mapping.d.ts.map +1 -0
- package/dist/adapters/tool-mapping.js +143 -0
- package/dist/artifacts/artifact.d.ts +137 -0
- package/dist/artifacts/artifact.d.ts.map +1 -0
- package/dist/artifacts/artifact.js +377 -0
- package/dist/artifacts/content-uploader.d.ts +87 -0
- package/dist/artifacts/content-uploader.d.ts.map +1 -0
- package/dist/artifacts/content-uploader.js +124 -0
- package/dist/artifacts/index.d.ts +63 -0
- package/dist/artifacts/index.d.ts.map +1 -0
- package/dist/artifacts/index.js +80 -0
- package/dist/artifacts/run-context.d.ts +129 -0
- package/dist/artifacts/run-context.d.ts.map +1 -0
- package/dist/artifacts/run-context.js +208 -0
- package/dist/artifacts/scrub.d.ts +60 -0
- package/dist/artifacts/scrub.d.ts.map +1 -0
- package/dist/artifacts/scrub.js +139 -0
- package/dist/artifacts/step.d.ts +26 -0
- package/dist/artifacts/step.d.ts.map +1 -0
- package/dist/artifacts/step.js +37 -0
- package/dist/artifacts/taint.d.ts +63 -0
- package/dist/artifacts/taint.d.ts.map +1 -0
- package/dist/artifacts/taint.js +76 -0
- package/dist/bakeoff/cache-sim.d.ts +99 -0
- package/dist/bakeoff/cache-sim.d.ts.map +1 -0
- package/dist/bakeoff/cache-sim.js +126 -0
- package/dist/bakeoff/engine.d.ts +52 -0
- package/dist/bakeoff/engine.d.ts.map +1 -0
- package/dist/bakeoff/engine.js +163 -0
- package/dist/bakeoff/eval-cases.d.ts +96 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases.js +105 -0
- package/dist/bakeoff/replay.d.ts +52 -0
- package/dist/bakeoff/replay.d.ts.map +1 -0
- package/dist/bakeoff/replay.js +94 -0
- package/dist/bakeoff/report.d.ts +41 -0
- package/dist/bakeoff/report.d.ts.map +1 -0
- package/dist/bakeoff/report.js +142 -0
- package/dist/bakeoff/samples.d.ts +77 -0
- package/dist/bakeoff/samples.d.ts.map +1 -0
- package/dist/bakeoff/samples.js +148 -0
- package/dist/bakeoff/stats.d.ts +13 -0
- package/dist/bakeoff/stats.d.ts.map +1 -0
- package/dist/bakeoff/stats.js +48 -0
- package/dist/bakeoff/types.d.ts +37 -0
- package/dist/bakeoff/types.d.ts.map +1 -0
- package/dist/bakeoff/types.js +10 -0
- package/dist/cli.d.ts +25 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +590 -0
- package/dist/config.d.ts +204 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +252 -0
- package/dist/deprecations.d.ts +32 -0
- package/dist/deprecations.d.ts.map +1 -0
- package/dist/deprecations.js +59 -0
- package/dist/feedback/coverage.d.ts +32 -0
- package/dist/feedback/coverage.d.ts.map +1 -0
- package/dist/feedback/coverage.js +95 -0
- package/dist/feedback/feedback.d.ts +35 -0
- package/dist/feedback/feedback.d.ts.map +1 -0
- package/dist/feedback/feedback.js +39 -0
- package/dist/gateway/http-protocol.d.ts +175 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -0
- package/dist/gateway/http-protocol.js +105 -0
- package/dist/index.d.ts +242 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +576 -0
- package/dist/init/finder.d.ts +94 -0
- package/dist/init/finder.d.ts.map +1 -0
- package/dist/init/finder.js +343 -0
- package/dist/init/init-cli.d.ts +10 -0
- package/dist/init/init-cli.d.ts.map +1 -0
- package/dist/init/init-cli.js +86 -0
- package/dist/init/observe-cli.d.ts +19 -0
- package/dist/init/observe-cli.d.ts.map +1 -0
- package/dist/init/observe-cli.js +73 -0
- package/dist/init/observe.d.ts +81 -0
- package/dist/init/observe.d.ts.map +1 -0
- package/dist/init/observe.js +245 -0
- package/dist/init/run.d.ts +99 -0
- package/dist/init/run.d.ts.map +1 -0
- package/dist/init/run.js +281 -0
- package/dist/init/signatures.d.ts +39 -0
- package/dist/init/signatures.d.ts.map +1 -0
- package/dist/init/signatures.js +116 -0
- package/dist/matrix/aggregate.d.ts +89 -0
- package/dist/matrix/aggregate.d.ts.map +1 -0
- package/dist/matrix/aggregate.js +186 -0
- package/dist/query/grammar.d.ts +35 -0
- package/dist/query/grammar.d.ts.map +1 -0
- package/dist/query/grammar.js +279 -0
- package/dist/query/run.d.ts +8 -0
- package/dist/query/run.d.ts.map +1 -0
- package/dist/query/run.js +21 -0
- package/dist/registry/build.d.ts +219 -0
- package/dist/registry/build.d.ts.map +1 -0
- package/dist/registry/build.js +313 -0
- package/dist/registry/cache-policy.d.ts +94 -0
- package/dist/registry/cache-policy.d.ts.map +1 -0
- package/dist/registry/cache-policy.js +238 -0
- package/dist/registry/call-notes.d.ts +11 -0
- package/dist/registry/call-notes.d.ts.map +1 -0
- package/dist/registry/call-notes.js +110 -0
- package/dist/registry/claims.d.ts +63 -0
- package/dist/registry/claims.d.ts.map +1 -0
- package/dist/registry/claims.js +107 -0
- package/dist/registry/effective-rate.d.ts +151 -0
- package/dist/registry/effective-rate.d.ts.map +1 -0
- package/dist/registry/effective-rate.js +156 -0
- package/dist/registry/facts.d.ts +32 -0
- package/dist/registry/facts.d.ts.map +1 -0
- package/dist/registry/facts.js +136 -0
- package/dist/registry/fingerprint.d.ts +20 -0
- package/dist/registry/fingerprint.d.ts.map +1 -0
- package/dist/registry/fingerprint.js +150 -0
- package/dist/registry/parity.d.ts +52 -0
- package/dist/registry/parity.d.ts.map +1 -0
- package/dist/registry/parity.js +82 -0
- package/dist/registry/pricing-shape.d.ts +113 -0
- package/dist/registry/pricing-shape.d.ts.map +1 -0
- package/dist/registry/pricing-shape.js +123 -0
- package/dist/registry/pricing.d.ts +43 -0
- package/dist/registry/pricing.d.ts.map +1 -0
- package/dist/registry/pricing.js +127 -0
- package/dist/registry/rows.d.ts +16 -0
- package/dist/registry/rows.d.ts.map +1 -0
- package/dist/registry/rows.js +57 -0
- package/dist/registry/seed.d.ts +43 -0
- package/dist/registry/seed.d.ts.map +1 -0
- package/dist/registry/seed.js +97 -0
- package/dist/registry/select.d.ts +68 -0
- package/dist/registry/select.d.ts.map +1 -0
- package/dist/registry/select.js +141 -0
- package/dist/registry/staleness.d.ts +18 -0
- package/dist/registry/staleness.d.ts.map +1 -0
- package/dist/registry/staleness.js +39 -0
- package/dist/rigindex/bootstrap.d.ts +36 -0
- package/dist/rigindex/bootstrap.d.ts.map +1 -0
- package/dist/rigindex/bootstrap.js +97 -0
- package/dist/rigindex/fit.d.ts +24 -0
- package/dist/rigindex/fit.d.ts.map +1 -0
- package/dist/rigindex/fit.js +76 -0
- package/dist/rigindex/gates.d.ts +52 -0
- package/dist/rigindex/gates.d.ts.map +1 -0
- package/dist/rigindex/gates.js +109 -0
- package/dist/rigindex/graph.d.ts +31 -0
- package/dist/rigindex/graph.d.ts.map +1 -0
- package/dist/rigindex/graph.js +120 -0
- package/dist/rigindex/index.d.ts +23 -0
- package/dist/rigindex/index.d.ts.map +1 -0
- package/dist/rigindex/index.js +112 -0
- package/dist/rigindex/types.d.ts +76 -0
- package/dist/rigindex/types.d.ts.map +1 -0
- package/dist/rigindex/types.js +8 -0
- package/dist/routes/cache-lint.d.ts +41 -0
- package/dist/routes/cache-lint.d.ts.map +1 -0
- package/dist/routes/cache-lint.js +81 -0
- package/dist/routes/draft.d.ts +83 -0
- package/dist/routes/draft.d.ts.map +1 -0
- package/dist/routes/draft.js +135 -0
- package/dist/routes/loader.d.ts +63 -0
- package/dist/routes/loader.d.ts.map +1 -0
- package/dist/routes/loader.js +574 -0
- package/dist/routes/manifest.d.ts +24 -0
- package/dist/routes/manifest.d.ts.map +1 -0
- package/dist/routes/manifest.js +118 -0
- package/dist/routes/resolved-bundle.d.ts +79 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -0
- package/dist/routes/resolved-bundle.js +148 -0
- package/dist/routes/sampling-config.d.ts +32 -0
- package/dist/routes/sampling-config.d.ts.map +1 -0
- package/dist/routes/sampling-config.js +91 -0
- package/dist/routes/serveability.d.ts +52 -0
- package/dist/routes/serveability.d.ts.map +1 -0
- package/dist/routes/serveability.js +73 -0
- package/dist/routes/template.d.ts +18 -0
- package/dist/routes/template.d.ts.map +1 -0
- package/dist/routes/template.js +94 -0
- package/dist/run/ajv.d.ts +22 -0
- package/dist/run/ajv.d.ts.map +1 -0
- package/dist/run/ajv.js +32 -0
- package/dist/run/cache-handle.d.ts +58 -0
- package/dist/run/cache-handle.d.ts.map +1 -0
- package/dist/run/cache-handle.js +54 -0
- package/dist/run/cache-stamp.d.ts +65 -0
- package/dist/run/cache-stamp.d.ts.map +1 -0
- package/dist/run/cache-stamp.js +0 -0
- package/dist/run/capture-write.d.ts +33 -0
- package/dist/run/capture-write.d.ts.map +1 -0
- package/dist/run/capture-write.js +55 -0
- package/dist/run/cost.d.ts +26 -0
- package/dist/run/cost.d.ts.map +1 -0
- package/dist/run/cost.js +41 -0
- package/dist/run/dispatch.d.ts +99 -0
- package/dist/run/dispatch.d.ts.map +1 -0
- package/dist/run/dispatch.js +191 -0
- package/dist/run/envelope.d.ts +21 -0
- package/dist/run/envelope.d.ts.map +1 -0
- package/dist/run/envelope.js +45 -0
- package/dist/run/grounding.d.ts +56 -0
- package/dist/run/grounding.d.ts.map +1 -0
- package/dist/run/grounding.js +89 -0
- package/dist/run/raw.d.ts +88 -0
- package/dist/run/raw.d.ts.map +1 -0
- package/dist/run/raw.js +286 -0
- package/dist/run/record-attempt.d.ts +40 -0
- package/dist/run/record-attempt.d.ts.map +1 -0
- package/dist/run/record-attempt.js +99 -0
- package/dist/run/repair.d.ts +36 -0
- package/dist/run/repair.d.ts.map +1 -0
- package/dist/run/repair.js +61 -0
- package/dist/run/resolve.d.ts +99 -0
- package/dist/run/resolve.d.ts.map +1 -0
- package/dist/run/resolve.js +203 -0
- package/dist/run/retry.d.ts +19 -0
- package/dist/run/retry.d.ts.map +1 -0
- package/dist/run/retry.js +50 -0
- package/dist/run/run.d.ts +55 -0
- package/dist/run/run.d.ts.map +1 -0
- package/dist/run/run.js +540 -0
- package/dist/run/stream.d.ts +49 -0
- package/dist/run/stream.d.ts.map +1 -0
- package/dist/run/stream.js +87 -0
- package/dist/run/validate.d.ts +15 -0
- package/dist/run/validate.d.ts.map +1 -0
- package/dist/run/validate.js +55 -0
- package/dist/savings/cache-ledger.d.ts +108 -0
- package/dist/savings/cache-ledger.d.ts.map +1 -0
- package/dist/savings/cache-ledger.js +174 -0
- package/dist/savings/cache-reconcile.d.ts +64 -0
- package/dist/savings/cache-reconcile.d.ts.map +1 -0
- package/dist/savings/cache-reconcile.js +59 -0
- package/dist/savings/cache-stats.d.ts +63 -0
- package/dist/savings/cache-stats.d.ts.map +1 -0
- package/dist/savings/cache-stats.js +108 -0
- package/dist/savings/emit.d.ts +38 -0
- package/dist/savings/emit.d.ts.map +1 -0
- package/dist/savings/emit.js +100 -0
- package/dist/savings/odometer.d.ts +32 -0
- package/dist/savings/odometer.d.ts.map +1 -0
- package/dist/savings/odometer.js +70 -0
- package/dist/savings/swap-execute.d.ts +58 -0
- package/dist/savings/swap-execute.d.ts.map +1 -0
- package/dist/savings/swap-execute.js +130 -0
- package/dist/savings/swaps-client.d.ts +55 -0
- package/dist/savings/swaps-client.d.ts.map +1 -0
- package/dist/savings/swaps-client.js +95 -0
- package/dist/settings/api-fetcher.d.ts +39 -0
- package/dist/settings/api-fetcher.d.ts.map +1 -0
- package/dist/settings/api-fetcher.js +92 -0
- package/dist/settings/index.d.ts +10 -0
- package/dist/settings/index.d.ts.map +1 -0
- package/dist/settings/index.js +15 -0
- package/dist/settings/task-settings.d.ts +91 -0
- package/dist/settings/task-settings.d.ts.map +1 -0
- package/dist/settings/task-settings.js +116 -0
- package/dist/status.d.ts +77 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +113 -0
- package/dist/telemetry/artifact-store.d.ts +48 -0
- package/dist/telemetry/artifact-store.d.ts.map +1 -0
- package/dist/telemetry/artifact-store.js +189 -0
- package/dist/telemetry/cache-miss-streak.d.ts +79 -0
- package/dist/telemetry/cache-miss-streak.d.ts.map +1 -0
- package/dist/telemetry/cache-miss-streak.js +99 -0
- package/dist/telemetry/capture.d.ts +31 -0
- package/dist/telemetry/capture.d.ts.map +1 -0
- package/dist/telemetry/capture.js +81 -0
- package/dist/telemetry/exporter-setup.d.ts +28 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -0
- package/dist/telemetry/exporter-setup.js +34 -0
- package/dist/telemetry/exporter.d.ts +137 -0
- package/dist/telemetry/exporter.d.ts.map +1 -0
- package/dist/telemetry/exporter.js +510 -0
- package/dist/telemetry/health.d.ts +76 -0
- package/dist/telemetry/health.d.ts.map +1 -0
- package/dist/telemetry/health.js +92 -0
- package/dist/telemetry/ingest-protocol.d.ts +310 -0
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -0
- package/dist/telemetry/ingest-protocol.js +363 -0
- package/dist/telemetry/load-error-report.d.ts +24 -0
- package/dist/telemetry/load-error-report.d.ts.map +1 -0
- package/dist/telemetry/load-error-report.js +50 -0
- package/dist/telemetry/otel-map.d.ts +121 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -0
- package/dist/telemetry/otel-map.js +224 -0
- package/dist/telemetry/otel-normalize.d.ts +104 -0
- package/dist/telemetry/otel-normalize.d.ts.map +1 -0
- package/dist/telemetry/otel-normalize.js +530 -0
- package/dist/telemetry/otlp-emitter.d.ts +52 -0
- package/dist/telemetry/otlp-emitter.d.ts.map +1 -0
- package/dist/telemetry/otlp-emitter.js +144 -0
- package/dist/telemetry/otlp-proto.d.ts +29 -0
- package/dist/telemetry/otlp-proto.d.ts.map +1 -0
- package/dist/telemetry/otlp-proto.js +266 -0
- package/dist/telemetry/sinks.d.ts +90 -0
- package/dist/telemetry/sinks.d.ts.map +1 -0
- package/dist/telemetry/sinks.js +180 -0
- package/dist/telemetry/store.d.ts +65 -0
- package/dist/telemetry/store.d.ts.map +1 -0
- package/dist/telemetry/store.js +547 -0
- package/dist/tenancy/api-key-format.d.ts +77 -0
- package/dist/tenancy/api-key-format.d.ts.map +1 -0
- package/dist/tenancy/api-key-format.js +110 -0
- package/dist/tenancy/constants.d.ts +23 -0
- package/dist/tenancy/constants.d.ts.map +1 -0
- package/dist/tenancy/constants.js +25 -0
- package/dist/tenancy/scrub.d.ts +47 -0
- package/dist/tenancy/scrub.d.ts.map +1 -0
- package/dist/tenancy/scrub.js +85 -0
- package/dist/types/config-teaching.d.ts +62 -0
- package/dist/types/config-teaching.d.ts.map +1 -0
- package/dist/types/config-teaching.js +183 -0
- package/dist/types/failures-teaching.d.ts +56 -0
- package/dist/types/failures-teaching.d.ts.map +1 -0
- package/dist/types/failures-teaching.js +177 -0
- package/dist/types/failures.d.ts +27 -0
- package/dist/types/failures.d.ts.map +1 -0
- package/dist/types/failures.js +41 -0
- package/dist/types/provider.d.ts +200 -0
- package/dist/types/provider.d.ts.map +1 -0
- package/dist/types/provider.js +40 -0
- package/dist/types/route.d.ts +205 -0
- package/dist/types/route.d.ts.map +1 -0
- package/dist/types/route.js +6 -0
- package/dist/types/telemetry.d.ts +206 -0
- package/dist/types/telemetry.d.ts.map +1 -0
- package/dist/types/telemetry.js +6 -0
- package/dist/validate-cli.d.ts +12 -0
- package/dist/validate-cli.d.ts.map +1 -0
- package/dist/validate-cli.js +57 -0
- package/dist/verify/capture.d.ts +66 -0
- package/dist/verify/capture.d.ts.map +1 -0
- package/dist/verify/capture.js +180 -0
- package/dist/verify/cli-run.d.ts +23 -0
- package/dist/verify/cli-run.d.ts.map +1 -0
- package/dist/verify/cli-run.js +57 -0
- package/dist/verify/diff.d.ts +48 -0
- package/dist/verify/diff.d.ts.map +1 -0
- package/dist/verify/diff.js +157 -0
- package/dist/verify/index.d.ts +17 -0
- package/dist/verify/index.d.ts.map +1 -0
- package/dist/verify/index.js +24 -0
- package/dist/verify/report.d.ts +14 -0
- package/dist/verify/report.d.ts.map +1 -0
- package/dist/verify/report.js +88 -0
- package/dist/verify/types.d.ts +101 -0
- package/dist/verify/types.d.ts.map +1 -0
- package/dist/verify/types.js +17 -0
- package/dist/verify/verify.d.ts +29 -0
- package/dist/verify/verify.d.ts.map +1 -0
- package/dist/verify/verify.js +148 -0
- package/dist/watch/budget.d.ts +19 -0
- package/dist/watch/budget.d.ts.map +1 -0
- package/dist/watch/budget.js +35 -0
- package/dist/watch/evaluate.d.ts +225 -0
- package/dist/watch/evaluate.d.ts.map +1 -0
- package/dist/watch/evaluate.js +155 -0
- package/dist/watch/monitor.d.ts +116 -0
- package/dist/watch/monitor.d.ts.map +1 -0
- package/dist/watch/monitor.js +199 -0
- package/dist/watch/proposals.d.ts +89 -0
- package/dist/watch/proposals.d.ts.map +1 -0
- package/dist/watch/proposals.js +197 -0
- package/dist/watch/run-record.d.ts +64 -0
- package/dist/watch/run-record.d.ts.map +1 -0
- package/dist/watch/run-record.js +86 -0
- package/dist/watch/run-reported.d.ts +20 -0
- package/dist/watch/run-reported.d.ts.map +1 -0
- package/dist/watch/run-reported.js +53 -0
- package/dist/watch/run.d.ts +73 -0
- package/dist/watch/run.d.ts.map +1 -0
- package/dist/watch/run.js +83 -0
- package/dist/watch/state-store.d.ts +48 -0
- package/dist/watch/state-store.d.ts.map +1 -0
- package/dist/watch/state-store.js +86 -0
- package/dist/watch/state.d.ts +21 -0
- package/dist/watch/state.d.ts.map +1 -0
- package/dist/watch/state.js +93 -0
- package/package.json +56 -14
- package/registry/LICENSE +20 -0
- package/registry/leaderboard.json +3639 -0
- package/registry/registry.json +7263 -0
- package/index.js +0 -3
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Registry v0 build logic — spec WS3. Pure functions (script entrypoints in
|
|
3
|
+
* registry/ do the IO): declared from the vendored pricing map + adapter
|
|
4
|
+
* capability maps; probed from probe result files; observed from telemetry
|
|
5
|
+
* aggregates; merge + discrepancy detection.
|
|
6
|
+
*
|
|
7
|
+
* PRIVACY (spec WS6 / plan step-4 review): the observed layer publishes
|
|
8
|
+
* AGGREGATES ONLY — no tags, no payloads, no fingerprint joins.
|
|
9
|
+
*/
|
|
10
|
+
import type { ModelFacts } from "./claims";
|
|
11
|
+
import type { ModelPricingV2 } from "./pricing-shape";
|
|
12
|
+
export interface DeclaredLayer {
|
|
13
|
+
readonly as_of: string;
|
|
14
|
+
readonly source: string;
|
|
15
|
+
readonly pricing: {
|
|
16
|
+
readonly input_usd_per_mtok: number;
|
|
17
|
+
readonly output_usd_per_mtok: number;
|
|
18
|
+
readonly cached_input_usd_per_mtok: number | null;
|
|
19
|
+
} | null;
|
|
20
|
+
/**
|
|
21
|
+
* First-class structured pricing (pricing-first-class spec §2.2) — ADDITIVE
|
|
22
|
+
* and optional. Rides inside the existing `declared` jsonb; the flat `pricing`
|
|
23
|
+
* field above STAYS this pass (S3: retire it in a later change once every
|
|
24
|
+
* reader uses `pricing_v2`). `pricing_v2.base` always equals the flat field.
|
|
25
|
+
*/
|
|
26
|
+
readonly pricing_v2?: ModelPricingV2 | null;
|
|
27
|
+
readonly flags: {
|
|
28
|
+
readonly supports_response_schema: boolean | null;
|
|
29
|
+
readonly supports_web_search: boolean | null;
|
|
30
|
+
readonly supports_prompt_caching: boolean | null;
|
|
31
|
+
/** What OUR adapter actually drives for this model (static map v0). */
|
|
32
|
+
readonly adapter_structured_native: boolean | null;
|
|
33
|
+
readonly adapter_grounded_native: boolean | null;
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
/** Per-corpus (fixture-family) breakdown of a schema summary — demo-rig
|
|
37
|
+
* §11.3b: the leaderboard and registry surface which corpus produced a cell,
|
|
38
|
+
* so a model probed only by `demo-rig` fixtures is visibly that. The aggregate
|
|
39
|
+
* fields above stay the ranking basis (same whole corpus for every model);
|
|
40
|
+
* this splits them by provenance for honesty, not for per-model subsetting. */
|
|
41
|
+
export interface ProbedFamilySummary {
|
|
42
|
+
readonly samples: number;
|
|
43
|
+
readonly conform_rate: number | null;
|
|
44
|
+
readonly conform_ci95: readonly [number, number] | null;
|
|
45
|
+
readonly value_accuracy_mean: number | null;
|
|
46
|
+
/** Fraction of this subset's samples served via the NATIVE structured rung
|
|
47
|
+
* vs json_mode coaching (discriminating-fixtures follow-up 3a). Lets a hard
|
|
48
|
+
* cell read "coached, missed" rather than a flat fail: a model without probed
|
|
49
|
+
* structured_native is coached on every schema sample, so a strict-constraint
|
|
50
|
+
* miss is the coaching path's limit, not the model's. Null when no sample in
|
|
51
|
+
* the subset recorded a rung (pre-rung result files). */
|
|
52
|
+
readonly native_rung_rate: number | null;
|
|
53
|
+
/** Mean per-sample cost (USD) over this subset — the per-family analogue of
|
|
54
|
+
* the aggregate `mean_cost_usd` (task-type-leaderboards spec §2.1). Lets a
|
|
55
|
+
* per-family view compute its own effective-cost-of-conformance
|
|
56
|
+
* (`mean_cost_usd / conform_rate * 1000`) so the faceted "cheapest-conformant
|
|
57
|
+
* on <task type>" rank has a number to sort on. Optional/additive: absent on
|
|
58
|
+
* pre–task-type probed layers, so a consumer reads it behind a null guard. */
|
|
59
|
+
readonly mean_cost_usd?: number;
|
|
60
|
+
}
|
|
61
|
+
export interface ProbedSchemaSummary {
|
|
62
|
+
readonly samples: number;
|
|
63
|
+
readonly parse_rate: number;
|
|
64
|
+
readonly conform_rate: number | null;
|
|
65
|
+
readonly conform_ci95: readonly [number, number] | null;
|
|
66
|
+
readonly value_accuracy_mean: number | null;
|
|
67
|
+
readonly native_rung_rate: number | null;
|
|
68
|
+
readonly mean_cost_usd: number;
|
|
69
|
+
/** family → its own stats (demo-rig §11.3b). Absent on pre-family probed
|
|
70
|
+
* layers; present once a result file carries fixtureFamilies. */
|
|
71
|
+
readonly by_family?: Readonly<Record<string, ProbedFamilySummary>>;
|
|
72
|
+
/** difficulty → its own stats (discriminating-fixtures §3). Same shape as
|
|
73
|
+
* by_family; the "hard" bucket is the leaderboard's differentiating column.
|
|
74
|
+
* Absent on pre-difficulty probed layers; present once a result file carries
|
|
75
|
+
* fixtureDifficulties. */
|
|
76
|
+
readonly by_difficulty?: Readonly<Record<string, ProbedFamilySummary>>;
|
|
77
|
+
/** task subtype → its own stats (www-clarity §5.5 W16). Unlike family
|
|
78
|
+
* (provenance) these are REAL task types (extraction.tabular,
|
|
79
|
+
* classification, …), cross-corpus by design. No default bucket: an
|
|
80
|
+
* untagged fixture joins no subtype. Absent when nothing is tagged. */
|
|
81
|
+
readonly by_subtype?: Readonly<Record<string, ProbedFamilySummary>>;
|
|
82
|
+
}
|
|
83
|
+
export interface ProbedGroundingSummary {
|
|
84
|
+
readonly samples: number;
|
|
85
|
+
readonly grounded_rate: number;
|
|
86
|
+
readonly mean_citations: number;
|
|
87
|
+
}
|
|
88
|
+
export interface ProbedCachingSummary {
|
|
89
|
+
readonly samples: number;
|
|
90
|
+
readonly cache_hit_rate: number;
|
|
91
|
+
readonly mean_cached_tokens: number;
|
|
92
|
+
/** Samples that passed the documented-path eligibility gate (prefix ≥ the
|
|
93
|
+
* provider's minimum cacheable length; typed skips excluded). The
|
|
94
|
+
* caching discrepancy rule requires this > 0 — a rate of 0 over zero
|
|
95
|
+
* eligible samples is a probe-coverage fact, not a capability finding.
|
|
96
|
+
* Absent in pre-2026-08-18 result files → treated as `samples`. */
|
|
97
|
+
readonly eligible_samples?: number;
|
|
98
|
+
}
|
|
99
|
+
export interface ProbedLayer {
|
|
100
|
+
readonly as_of: string;
|
|
101
|
+
readonly source: string;
|
|
102
|
+
readonly schema: ProbedSchemaSummary | null;
|
|
103
|
+
readonly grounding: ProbedGroundingSummary | null;
|
|
104
|
+
readonly caching: ProbedCachingSummary | null;
|
|
105
|
+
}
|
|
106
|
+
export interface ObservedLayer {
|
|
107
|
+
readonly as_of: string;
|
|
108
|
+
readonly source: string;
|
|
109
|
+
readonly inferences: number;
|
|
110
|
+
readonly p50_latency_ms: number | null;
|
|
111
|
+
/** TTFB p50 (Phase 4 WS3) — optional: aggregators populate it where
|
|
112
|
+
* ttfb_ms rows exist; older observed layers lack the field. */
|
|
113
|
+
readonly p50_ttfb_ms?: number | null;
|
|
114
|
+
/**
|
|
115
|
+
* Observed-health percentiles + error-rate (G-4, observed-health/spec §4.2).
|
|
116
|
+
* Optional and additive — older observed layers lack these fields; they are
|
|
117
|
+
* min-n gated to `null` on thin endpoints. Operational health for the console
|
|
118
|
+
* + routing ONLY, NEVER a RigIndex input (spec §4.4).
|
|
119
|
+
*/
|
|
120
|
+
readonly p90_latency_ms?: number | null;
|
|
121
|
+
readonly p99_latency_ms?: number | null;
|
|
122
|
+
readonly p90_ttfb_ms?: number | null;
|
|
123
|
+
readonly p99_ttfb_ms?: number | null;
|
|
124
|
+
/** failure_class non-null / total; null under the min-n gate. */
|
|
125
|
+
readonly error_rate?: number | null;
|
|
126
|
+
/** Sample count behind the health stats (drives the min-n gate + freshness). */
|
|
127
|
+
readonly sample_n?: number;
|
|
128
|
+
readonly served_tier_rates: Readonly<Record<string, number>>;
|
|
129
|
+
readonly cache_hit_rate: number | null;
|
|
130
|
+
readonly failure_class_rates: Readonly<Record<string, number>>;
|
|
131
|
+
}
|
|
132
|
+
export interface Discrepancy {
|
|
133
|
+
readonly kind: string;
|
|
134
|
+
readonly message: string;
|
|
135
|
+
}
|
|
136
|
+
export interface RegistryEntry {
|
|
137
|
+
readonly model_key: string;
|
|
138
|
+
readonly declared: DeclaredLayer | null;
|
|
139
|
+
readonly probed: ProbedLayer | null;
|
|
140
|
+
readonly observed: ObservedLayer | null;
|
|
141
|
+
readonly discrepancies: readonly Discrepancy[];
|
|
142
|
+
/** Per-model call-shaping knowledge (addendum A1 / patch P1) — seeded from
|
|
143
|
+
* the adapters' documented constraints; see call-notes.ts. */
|
|
144
|
+
readonly callNotes: readonly string[];
|
|
145
|
+
/** Credential-scoped {value, scope, source, as_of} facts (Phase 4 WS3) —
|
|
146
|
+
* null when nothing is cited; absence IS the conservative claim. */
|
|
147
|
+
readonly facts: ModelFacts | null;
|
|
148
|
+
}
|
|
149
|
+
export interface Registry {
|
|
150
|
+
readonly generated_at: string;
|
|
151
|
+
readonly license: string;
|
|
152
|
+
readonly models: readonly RegistryEntry[];
|
|
153
|
+
}
|
|
154
|
+
/** Minimal shape of a probe result file this builder consumes (kept
|
|
155
|
+
* structural so the core package does not depend on modelrig-probes). */
|
|
156
|
+
export interface ProbeResultFile {
|
|
157
|
+
readonly modelKey: string;
|
|
158
|
+
readonly class: "schema" | "grounding" | "caching";
|
|
159
|
+
readonly date: string;
|
|
160
|
+
readonly harnessVersion: string;
|
|
161
|
+
/** fixtureId → family (demo-rig §11.3b). Absent on pre-family result files;
|
|
162
|
+
* those fixtures roll up under the default "probe-suite" family. */
|
|
163
|
+
readonly fixtureFamilies?: Readonly<Record<string, string>>;
|
|
164
|
+
/** fixtureId → difficulty (discriminating-fixtures §3). Absent on
|
|
165
|
+
* pre-difficulty result files; those fixtures roll up under "standard". */
|
|
166
|
+
readonly fixtureDifficulties?: Readonly<Record<string, string>>;
|
|
167
|
+
/** fixtureId → task subtype (www-clarity §5.5 W16). Absent on pre-subtype
|
|
168
|
+
* result files — the registry builder may inject the mapping from the
|
|
169
|
+
* current fixture corpus (subtype is hash-excluded retro-taggable
|
|
170
|
+
* metadata). Fixtures absent from the map join no subtype bucket. */
|
|
171
|
+
readonly fixtureSubtypes?: Readonly<Record<string, string>>;
|
|
172
|
+
readonly raw: Readonly<Record<string, ReadonlyArray<{
|
|
173
|
+
readonly rung?: "native" | "json_mode";
|
|
174
|
+
readonly parseOk: boolean;
|
|
175
|
+
readonly schemaConform: boolean | null;
|
|
176
|
+
readonly valueAccuracy: number | null;
|
|
177
|
+
readonly grounded: boolean | null;
|
|
178
|
+
readonly citationCount: number | null;
|
|
179
|
+
readonly cachedTokens: number | null;
|
|
180
|
+
readonly costUsd: number;
|
|
181
|
+
readonly failureClass: string | null;
|
|
182
|
+
}>>>;
|
|
183
|
+
}
|
|
184
|
+
/** Latest result per class wins (results are dated). */
|
|
185
|
+
export declare function latestPerClass(results: readonly ProbeResultFile[]): Partial<Record<ProbeResultFile["class"], ProbeResultFile>>;
|
|
186
|
+
export declare function summarizeSchemaResult(result: ProbeResultFile): ProbedSchemaSummary;
|
|
187
|
+
export declare function summarizeGroundingResult(result: ProbeResultFile): ProbedGroundingSummary;
|
|
188
|
+
export declare function summarizeCachingResult(result: ProbeResultFile): ProbedCachingSummary;
|
|
189
|
+
/** Compose a model's SCHEMA result files into one synthetic file, so a cheap
|
|
190
|
+
* SUPPLEMENTAL run (new fixtures only) ADDS to a prior full run instead of
|
|
191
|
+
* replacing it — the default latest-per-file rule would drop every fixture the
|
|
192
|
+
* newest file omits. The latest schema file is the base; older files fill in
|
|
193
|
+
* fixtures the base lacks, newest date winning on any per-fixture collision.
|
|
194
|
+
* ONLY fixtures in `currentFixtureIds` (the live corpus) are pulled forward, so
|
|
195
|
+
* a RETIRED id sitting in an old file is never resurrected (the trap a naive
|
|
196
|
+
* union hits, e.g. demo-support-summarize double-counting summarize). */
|
|
197
|
+
export declare function mergeSchemaResults(results: readonly ProbeResultFile[], currentFixtureIds: ReadonlySet<string>): ProbeResultFile | undefined;
|
|
198
|
+
/**
|
|
199
|
+
* Build a model's probed layer. When `currentSchemaFixtureIds` is supplied, the
|
|
200
|
+
* schema class COMPOSES across the model's result files (a supplemental run adds
|
|
201
|
+
* fixtures to a prior full run) via mergeSchemaResults; grounding/caching stay
|
|
202
|
+
* latest-per-file (one file per cycle). Omitting the ids keeps the pure
|
|
203
|
+
* latest-per-file behaviour (back-compat for hermetic tests).
|
|
204
|
+
*/
|
|
205
|
+
export declare function buildProbedLayer(results: readonly ProbeResultFile[], currentSchemaFixtureIds?: ReadonlySet<string>): ProbedLayer | null;
|
|
206
|
+
/** Declared-vs-probed disagreement detection — plan §4 (the rule pair) plus
|
|
207
|
+
* the rung gap that motivated the whole exercise (DeepSeek json_object). */
|
|
208
|
+
export declare function detectDiscrepancies(declared: DeclaredLayer | null, probed: ProbedLayer | null): Discrepancy[];
|
|
209
|
+
export declare function buildRegistry(entries: ReadonlyArray<{
|
|
210
|
+
readonly model_key: string;
|
|
211
|
+
readonly declared: DeclaredLayer | null;
|
|
212
|
+
readonly probed: ProbedLayer | null;
|
|
213
|
+
readonly observed: ObservedLayer | null;
|
|
214
|
+
}>, generatedAt: string): Registry;
|
|
215
|
+
/** Leaderboard ranking metric — spec WS3: "conformance per dollar".
|
|
216
|
+
* Effective $ per 1K conformant outputs: mean sample cost ÷ conform rate
|
|
217
|
+
* × 1000. Null when unjudged or never conformant. */
|
|
218
|
+
export declare function effectiveCostPer1kConformant(schema: ProbedSchemaSummary | null): number | null;
|
|
219
|
+
//# sourceMappingURL=build.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/registry/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE;QAChB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;QACpC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;QACrC,QAAQ,CAAC,yBAAyB,EAAE,MAAM,GAAG,IAAI,CAAC;KACnD,GAAG,IAAI,CAAC;IACT;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,wBAAwB,EAAE,OAAO,GAAG,IAAI,CAAC;QAClD,QAAQ,CAAC,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;QAC7C,QAAQ,CAAC,uBAAuB,EAAE,OAAO,GAAG,IAAI,CAAC;QACjD,uEAAuE;QACvE,QAAQ,CAAC,yBAAyB,EAAE,OAAO,GAAG,IAAI,CAAC;QACnD,QAAQ,CAAC,uBAAuB,EAAE,OAAO,GAAG,IAAI,CAAC;KAClD,CAAC;CACH;AAED;;;;+EAI+E;AAC/E,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACxD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C;;;;;6DAKyD;IACzD,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC;;;;;kFAK8E;IAC9E,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACxD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;qEACiE;IACjE,QAAQ,CAAC,SAAS,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACnE;;;8BAG0B;IAC1B,QAAQ,CAAC,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACvE;;;2EAGuE;IACvE,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;CACrE;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC;;;;uEAImE;IACnE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,SAAS,EAAE,sBAAsB,GAAG,IAAI,CAAC;IAClD,QAAQ,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC;mEAC+D;IAC/D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC;;;;;OAKG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,iEAAiE;IACjE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7D,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,mBAAmB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAChE;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,SAAS,WAAW,EAAE,CAAC;IAC/C;kEAC8D;IAC9D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC;wEACoE;IACpE,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,CAAC;CAC3C;AAED;yEACyE;AACzE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACnD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;wEACoE;IACpE,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5D;+EAC2E;IAC3E,QAAQ,CAAC,mBAAmB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE;;;yEAGqE;IACrE,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5D,QAAQ,CAAC,GAAG,EAAE,QAAQ,CACpB,MAAM,CACJ,MAAM,EACN,aAAa,CAAC;QACZ,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;QACvC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;QAC1B,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;QACvC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,QAAQ,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;QAClC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QACrC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KACtC,CAAC,CACH,CACF,CAAC;CACH;AAqBD,wDAAwD;AACxD,wBAAgB,cAAc,CAC5B,OAAO,EAAE,SAAS,eAAe,EAAE,GAClC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,eAAe,CAAC,CAAC,CAS5D;AAyDD,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,eAAe,GAAG,mBAAmB,CAmClF;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,eAAe,GAAG,sBAAsB,CAQxF;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,eAAe,GAAG,oBAAoB,CAYpF;AAED;;;;;;;yEAOyE;AACzE,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,SAAS,eAAe,EAAE,EACnC,iBAAiB,EAAE,WAAW,CAAC,MAAM,CAAC,GACrC,eAAe,GAAG,SAAS,CA4B7B;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,SAAS,eAAe,EAAE,EACnC,uBAAuB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GAC5C,WAAW,GAAG,IAAI,CAmBpB;AAED;4EAC4E;AAC5E,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,aAAa,GAAG,IAAI,EAC9B,MAAM,EAAE,WAAW,GAAG,IAAI,GACzB,WAAW,EAAE,CAmFf;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,aAAa,CAAC;IACrB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;CACzC,CAAC,EACF,WAAW,EAAE,MAAM,GAClB,QAAQ,CAgBV;AAED;;qDAEqD;AACrD,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAG9F"}
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Registry v0 build logic — spec WS3. Pure functions (script entrypoints in
|
|
4
|
+
* registry/ do the IO): declared from the vendored pricing map + adapter
|
|
5
|
+
* capability maps; probed from probe result files; observed from telemetry
|
|
6
|
+
* aggregates; merge + discrepancy detection.
|
|
7
|
+
*
|
|
8
|
+
* PRIVACY (spec WS6 / plan step-4 review): the observed layer publishes
|
|
9
|
+
* AGGREGATES ONLY — no tags, no payloads, no fingerprint joins.
|
|
10
|
+
*/
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.latestPerClass = latestPerClass;
|
|
13
|
+
exports.summarizeSchemaResult = summarizeSchemaResult;
|
|
14
|
+
exports.summarizeGroundingResult = summarizeGroundingResult;
|
|
15
|
+
exports.summarizeCachingResult = summarizeCachingResult;
|
|
16
|
+
exports.mergeSchemaResults = mergeSchemaResults;
|
|
17
|
+
exports.buildProbedLayer = buildProbedLayer;
|
|
18
|
+
exports.detectDiscrepancies = detectDiscrepancies;
|
|
19
|
+
exports.buildRegistry = buildRegistry;
|
|
20
|
+
exports.effectiveCostPer1kConformant = effectiveCostPer1kConformant;
|
|
21
|
+
const call_notes_1 = require("./call-notes");
|
|
22
|
+
const claims_1 = require("./claims");
|
|
23
|
+
const CONFORMANCE_BAR = 0.9;
|
|
24
|
+
const GROUNDED_BAR = 0.5;
|
|
25
|
+
function mean(values) {
|
|
26
|
+
if (values.length === 0)
|
|
27
|
+
return null;
|
|
28
|
+
return values.reduce((a, b) => a + b, 0) / values.length;
|
|
29
|
+
}
|
|
30
|
+
function wilson95(successes, n) {
|
|
31
|
+
if (n === 0)
|
|
32
|
+
return [0, 1];
|
|
33
|
+
const z = 1.959963984540054;
|
|
34
|
+
const p = successes / n;
|
|
35
|
+
const z2 = z * z;
|
|
36
|
+
const denominator = 1 + z2 / n;
|
|
37
|
+
const center = (p + z2 / (2 * n)) / denominator;
|
|
38
|
+
const half = (z * Math.sqrt((p * (1 - p)) / n + z2 / (4 * n * n))) / denominator;
|
|
39
|
+
return [Math.max(0, center - half), Math.min(1, center + half)];
|
|
40
|
+
}
|
|
41
|
+
/** Latest result per class wins (results are dated). */
|
|
42
|
+
function latestPerClass(results) {
|
|
43
|
+
const byClass = {};
|
|
44
|
+
for (const result of results) {
|
|
45
|
+
const current = byClass[result.class];
|
|
46
|
+
if (current === undefined || result.date > current.date) {
|
|
47
|
+
byClass[result.class] = result;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return byClass;
|
|
51
|
+
}
|
|
52
|
+
const DEFAULT_FIXTURE_FAMILY = "probe-suite";
|
|
53
|
+
const DEFAULT_FIXTURE_DIFFICULTY = "standard";
|
|
54
|
+
/** Sentinel for fixtures with no subtype tag — dropped from by_subtype. */
|
|
55
|
+
const UNTAGGED_SUBTYPE = "__untagged__";
|
|
56
|
+
/** Group a result's schema samples by a per-fixture classification map
|
|
57
|
+
* (family or difficulty), defaulting fixtures the map omits to `fallback`.
|
|
58
|
+
* Shared by the by_family and by_difficulty rollups so both are computed the
|
|
59
|
+
* same way as the headline (via schemaFamilyStats). */
|
|
60
|
+
function rollupBy(result, classification, fallback) {
|
|
61
|
+
const buckets = new Map();
|
|
62
|
+
for (const [fixtureId, fixtureSamples] of Object.entries(result.raw)) {
|
|
63
|
+
const key = classification?.[fixtureId] ?? fallback;
|
|
64
|
+
const bucket = buckets.get(key) ?? [];
|
|
65
|
+
bucket.push(...fixtureSamples);
|
|
66
|
+
buckets.set(key, bucket);
|
|
67
|
+
}
|
|
68
|
+
const out = {};
|
|
69
|
+
for (const [key, samples] of [...buckets.entries()].sort()) {
|
|
70
|
+
out[key] = schemaFamilyStats(samples);
|
|
71
|
+
}
|
|
72
|
+
return out;
|
|
73
|
+
}
|
|
74
|
+
/** Conformance + value-accuracy stats over a set of schema samples — shared
|
|
75
|
+
* by the aggregate summary and each per-family sub-summary so a family cell is
|
|
76
|
+
* computed the same way as the headline. */
|
|
77
|
+
function schemaFamilyStats(samples) {
|
|
78
|
+
const judged = samples.filter((s) => s.schemaConform !== null);
|
|
79
|
+
const conforming = judged.filter((s) => s.schemaConform === true).length;
|
|
80
|
+
const accuracies = samples.map((s) => s.valueAccuracy).filter((a) => a !== null);
|
|
81
|
+
// Same native-rung computation as the aggregate — the serving path a subset
|
|
82
|
+
// was served through (native strict vs json_mode coaching), so a hard cell
|
|
83
|
+
// can say which it was (follow-up 3a).
|
|
84
|
+
const rungs = samples.filter((s) => s.rung !== undefined);
|
|
85
|
+
return {
|
|
86
|
+
samples: samples.length,
|
|
87
|
+
conform_rate: judged.length === 0 ? null : conforming / judged.length,
|
|
88
|
+
conform_ci95: judged.length === 0 ? null : wilson95(conforming, judged.length),
|
|
89
|
+
value_accuracy_mean: mean(accuracies),
|
|
90
|
+
native_rung_rate: rungs.length === 0 ? null : rungs.filter((s) => s.rung === "native").length / rungs.length,
|
|
91
|
+
// Per-subset mean cost, same computation as the aggregate — so a per-family
|
|
92
|
+
// view can derive its own effective-cost-of-conformance (spec §2.1).
|
|
93
|
+
mean_cost_usd: samples.length === 0 ? 0 : samples.reduce((a, s) => a + s.costUsd, 0) / samples.length,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
function summarizeSchemaResult(result) {
|
|
97
|
+
const samples = Object.values(result.raw).flat();
|
|
98
|
+
const n = samples.length;
|
|
99
|
+
const judged = samples.filter((s) => s.schemaConform !== null);
|
|
100
|
+
const conforming = judged.filter((s) => s.schemaConform === true).length;
|
|
101
|
+
const rungs = samples.filter((s) => s.rung !== undefined);
|
|
102
|
+
const accuracies = samples
|
|
103
|
+
.map((s) => s.valueAccuracy)
|
|
104
|
+
.filter((a) => a !== null);
|
|
105
|
+
// Per-corpus rollup (demo-rig §11.3b) and per-difficulty rollup
|
|
106
|
+
// (discriminating-fixtures §3). Each groups the same samples by a different
|
|
107
|
+
// hash-excluded classification, so surfaces can say which corpus produced a
|
|
108
|
+
// cell and how a model does on the hard subset. The aggregate above stays the
|
|
109
|
+
// ranking basis (same whole corpus for every model).
|
|
110
|
+
const by_family = rollupBy(result, result.fixtureFamilies, DEFAULT_FIXTURE_FAMILY);
|
|
111
|
+
const by_difficulty = rollupBy(result, result.fixtureDifficulties, DEFAULT_FIXTURE_DIFFICULTY);
|
|
112
|
+
// Subtype rollup (W16): no default bucket — an untagged fixture belongs to
|
|
113
|
+
// no task type, so it rolls into a sentinel that is then dropped.
|
|
114
|
+
const by_subtype = rollupBy(result, result.fixtureSubtypes, UNTAGGED_SUBTYPE);
|
|
115
|
+
delete by_subtype[UNTAGGED_SUBTYPE];
|
|
116
|
+
return {
|
|
117
|
+
samples: n,
|
|
118
|
+
parse_rate: n === 0 ? 0 : samples.filter((s) => s.parseOk).length / n,
|
|
119
|
+
conform_rate: judged.length === 0 ? null : conforming / judged.length,
|
|
120
|
+
conform_ci95: judged.length === 0 ? null : wilson95(conforming, judged.length),
|
|
121
|
+
value_accuracy_mean: mean(accuracies),
|
|
122
|
+
native_rung_rate: rungs.length === 0 ? null : rungs.filter((s) => s.rung === "native").length / rungs.length,
|
|
123
|
+
mean_cost_usd: n === 0 ? 0 : samples.reduce((a, s) => a + s.costUsd, 0) / n,
|
|
124
|
+
...(Object.keys(by_family).length > 0 ? { by_family } : {}),
|
|
125
|
+
...(Object.keys(by_difficulty).length > 0 ? { by_difficulty } : {}),
|
|
126
|
+
...(Object.keys(by_subtype).length > 0 ? { by_subtype } : {}),
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
function summarizeGroundingResult(result) {
|
|
130
|
+
const samples = Object.values(result.raw).flat();
|
|
131
|
+
const n = samples.length;
|
|
132
|
+
return {
|
|
133
|
+
samples: n,
|
|
134
|
+
grounded_rate: n === 0 ? 0 : samples.filter((s) => s.grounded === true).length / n,
|
|
135
|
+
mean_citations: mean(samples.map((s) => s.citationCount ?? 0)) ?? 0,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
function summarizeCachingResult(result) {
|
|
139
|
+
const samples = Object.values(result.raw).flat();
|
|
140
|
+
// failureClass null = the paired call ran AND passed the documented-path
|
|
141
|
+
// eligibility gate (cache_prefix_below_minimum is a typed skip upstream).
|
|
142
|
+
const okSamples = samples.filter((s) => s.failureClass === null);
|
|
143
|
+
const n = okSamples.length;
|
|
144
|
+
return {
|
|
145
|
+
samples: samples.length,
|
|
146
|
+
cache_hit_rate: n === 0 ? 0 : okSamples.filter((s) => (s.cachedTokens ?? 0) > 0).length / n,
|
|
147
|
+
mean_cached_tokens: mean(okSamples.map((s) => s.cachedTokens ?? 0)) ?? 0,
|
|
148
|
+
eligible_samples: n,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
/** Compose a model's SCHEMA result files into one synthetic file, so a cheap
|
|
152
|
+
* SUPPLEMENTAL run (new fixtures only) ADDS to a prior full run instead of
|
|
153
|
+
* replacing it — the default latest-per-file rule would drop every fixture the
|
|
154
|
+
* newest file omits. The latest schema file is the base; older files fill in
|
|
155
|
+
* fixtures the base lacks, newest date winning on any per-fixture collision.
|
|
156
|
+
* ONLY fixtures in `currentFixtureIds` (the live corpus) are pulled forward, so
|
|
157
|
+
* a RETIRED id sitting in an old file is never resurrected (the trap a naive
|
|
158
|
+
* union hits, e.g. demo-support-summarize double-counting summarize). */
|
|
159
|
+
function mergeSchemaResults(results, currentFixtureIds) {
|
|
160
|
+
const schemaFiles = results
|
|
161
|
+
.filter((r) => r.class === "schema")
|
|
162
|
+
.sort((a, b) => (a.date > b.date ? -1 : a.date < b.date ? 1 : 0)); // newest first
|
|
163
|
+
const base = schemaFiles[0];
|
|
164
|
+
if (base === undefined)
|
|
165
|
+
return undefined;
|
|
166
|
+
const raw = { ...base.raw };
|
|
167
|
+
const fixtureSubtypes = { ...(base.fixtureSubtypes ?? {}) };
|
|
168
|
+
const fixtureFamilies = { ...(base.fixtureFamilies ?? {}) };
|
|
169
|
+
const fixtureDifficulties = { ...(base.fixtureDifficulties ?? {}) };
|
|
170
|
+
for (const older of schemaFiles.slice(1)) {
|
|
171
|
+
for (const [fid, samples] of Object.entries(older.raw)) {
|
|
172
|
+
if (fid in raw)
|
|
173
|
+
continue; // a newer file already carries it — newest wins
|
|
174
|
+
if (!currentFixtureIds.has(fid))
|
|
175
|
+
continue; // retired id — never resurrect
|
|
176
|
+
raw[fid] = samples;
|
|
177
|
+
if (older.fixtureSubtypes?.[fid] !== undefined)
|
|
178
|
+
fixtureSubtypes[fid] = older.fixtureSubtypes[fid];
|
|
179
|
+
if (older.fixtureFamilies?.[fid] !== undefined)
|
|
180
|
+
fixtureFamilies[fid] = older.fixtureFamilies[fid];
|
|
181
|
+
if (older.fixtureDifficulties?.[fid] !== undefined)
|
|
182
|
+
fixtureDifficulties[fid] = older.fixtureDifficulties[fid];
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return {
|
|
186
|
+
...base,
|
|
187
|
+
raw,
|
|
188
|
+
...(Object.keys(fixtureSubtypes).length > 0 ? { fixtureSubtypes } : {}),
|
|
189
|
+
...(Object.keys(fixtureFamilies).length > 0 ? { fixtureFamilies } : {}),
|
|
190
|
+
...(Object.keys(fixtureDifficulties).length > 0 ? { fixtureDifficulties } : {}),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Build a model's probed layer. When `currentSchemaFixtureIds` is supplied, the
|
|
195
|
+
* schema class COMPOSES across the model's result files (a supplemental run adds
|
|
196
|
+
* fixtures to a prior full run) via mergeSchemaResults; grounding/caching stay
|
|
197
|
+
* latest-per-file (one file per cycle). Omitting the ids keeps the pure
|
|
198
|
+
* latest-per-file behaviour (back-compat for hermetic tests).
|
|
199
|
+
*/
|
|
200
|
+
function buildProbedLayer(results, currentSchemaFixtureIds) {
|
|
201
|
+
if (results.length === 0)
|
|
202
|
+
return null;
|
|
203
|
+
const byClass = latestPerClass(results);
|
|
204
|
+
const schemaFile = currentSchemaFixtureIds !== undefined
|
|
205
|
+
? mergeSchemaResults(results, currentSchemaFixtureIds)
|
|
206
|
+
: byClass.schema;
|
|
207
|
+
const dates = results.map((r) => r.date).sort();
|
|
208
|
+
const versions = results.map((r) => r.harnessVersion).sort();
|
|
209
|
+
const newest = dates[dates.length - 1];
|
|
210
|
+
const version = versions[versions.length - 1];
|
|
211
|
+
return {
|
|
212
|
+
as_of: newest,
|
|
213
|
+
source: `modelrig-probes@${version}`,
|
|
214
|
+
schema: schemaFile !== undefined ? summarizeSchemaResult(schemaFile) : null,
|
|
215
|
+
grounding: byClass.grounding !== undefined ? summarizeGroundingResult(byClass.grounding) : null,
|
|
216
|
+
caching: byClass.caching !== undefined ? summarizeCachingResult(byClass.caching) : null,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
/** Declared-vs-probed disagreement detection — plan §4 (the rule pair) plus
|
|
220
|
+
* the rung gap that motivated the whole exercise (DeepSeek json_object). */
|
|
221
|
+
function detectDiscrepancies(declared, probed) {
|
|
222
|
+
if (declared === null || probed === null)
|
|
223
|
+
return [];
|
|
224
|
+
const out = [];
|
|
225
|
+
const schema = probed.schema;
|
|
226
|
+
if (schema !== null && schema.conform_rate !== null) {
|
|
227
|
+
if (declared.flags.supports_response_schema === true && schema.conform_rate < CONFORMANCE_BAR) {
|
|
228
|
+
out.push({
|
|
229
|
+
kind: "declared_schema_low_conformance",
|
|
230
|
+
message: `declared supports_response_schema, but probed conformance is ` +
|
|
231
|
+
`${(schema.conform_rate * 100).toFixed(0)}% (bar: ${CONFORMANCE_BAR * 100}%)`,
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
if (declared.flags.supports_response_schema !== true &&
|
|
235
|
+
schema.conform_rate >= CONFORMANCE_BAR) {
|
|
236
|
+
out.push({
|
|
237
|
+
kind: "undeclared_schema_capable",
|
|
238
|
+
message: `no declared schema support, but probed conformance is ` +
|
|
239
|
+
`${(schema.conform_rate * 100).toFixed(0)}% — capability exists undeclared`,
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
if (schema !== null &&
|
|
244
|
+
schema.native_rung_rate !== null &&
|
|
245
|
+
declared.flags.supports_response_schema === true &&
|
|
246
|
+
schema.native_rung_rate === 0) {
|
|
247
|
+
out.push({
|
|
248
|
+
kind: "schema_served_via_json_mode",
|
|
249
|
+
message: "declared supports_response_schema, but every probed sample was served via " +
|
|
250
|
+
"json_mode coaching — no native strict enforcement exists on this serving path " +
|
|
251
|
+
"(the json_object-vs-schema gap)",
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
const grounding = probed.grounding;
|
|
255
|
+
if (grounding !== null && grounding.samples > 0) {
|
|
256
|
+
if (declared.flags.supports_web_search === true && grounding.grounded_rate < GROUNDED_BAR) {
|
|
257
|
+
out.push({
|
|
258
|
+
kind: "declared_search_ungrounded",
|
|
259
|
+
message: `declared supports_web_search, but only ${(grounding.grounded_rate * 100).toFixed(0)}% ` +
|
|
260
|
+
`of probed answers carried citations (bar: ${GROUNDED_BAR * 100}%)`,
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
if (declared.flags.supports_web_search !== true && grounding.grounded_rate >= GROUNDED_BAR) {
|
|
264
|
+
out.push({
|
|
265
|
+
kind: "citations_without_declared_search",
|
|
266
|
+
message: `no declared web-search support, but ${(grounding.grounded_rate * 100).toFixed(0)}% of ` +
|
|
267
|
+
`probed answers cited URLs — verify whether these are real or fabricated sources`,
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
const caching = probed.caching;
|
|
272
|
+
if (caching !== null && caching.samples > 0) {
|
|
273
|
+
// Guarded (launch-runbook A1, 2026-08-18): fire only when at least one
|
|
274
|
+
// sample passed the documented-path eligibility gate — a zero rate over
|
|
275
|
+
// zero eligible samples means the probe couldn't measure, not that the
|
|
276
|
+
// capability is unrealized. Pre-gate result files lack the field and
|
|
277
|
+
// keep the old behavior via the ?? fallback.
|
|
278
|
+
if (declared.flags.supports_prompt_caching === true &&
|
|
279
|
+
caching.cache_hit_rate === 0 &&
|
|
280
|
+
(caching.eligible_samples ?? caching.samples) > 0) {
|
|
281
|
+
out.push({
|
|
282
|
+
kind: "declared_caching_unrealized",
|
|
283
|
+
message: "declared supports_prompt_caching, but no eligible probed repeat call on the documented caching path reported cached tokens",
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
return out;
|
|
288
|
+
}
|
|
289
|
+
function buildRegistry(entries, generatedAt) {
|
|
290
|
+
return {
|
|
291
|
+
generated_at: generatedAt,
|
|
292
|
+
license: "CC-BY-4.0",
|
|
293
|
+
models: entries
|
|
294
|
+
.map((e) => ({
|
|
295
|
+
model_key: e.model_key,
|
|
296
|
+
declared: e.declared,
|
|
297
|
+
probed: e.probed,
|
|
298
|
+
observed: e.observed,
|
|
299
|
+
discrepancies: detectDiscrepancies(e.declared, e.probed),
|
|
300
|
+
callNotes: (0, call_notes_1.callNotesFor)(e.model_key),
|
|
301
|
+
facts: (0, claims_1.factsFor)(e.model_key),
|
|
302
|
+
}))
|
|
303
|
+
.sort((a, b) => (a.model_key < b.model_key ? -1 : 1)),
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
/** Leaderboard ranking metric — spec WS3: "conformance per dollar".
|
|
307
|
+
* Effective $ per 1K conformant outputs: mean sample cost ÷ conform rate
|
|
308
|
+
* × 1000. Null when unjudged or never conformant. */
|
|
309
|
+
function effectiveCostPer1kConformant(schema) {
|
|
310
|
+
if (schema === null || schema.conform_rate === null || schema.conform_rate === 0)
|
|
311
|
+
return null;
|
|
312
|
+
return (schema.mean_cost_usd / schema.conform_rate) * 1000;
|
|
313
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cache Policy Registry — cache-aware-routing spec §3, [Briefing §7].
|
|
3
|
+
*
|
|
4
|
+
* Per-provider cache SEMANTICS as versioned DATA, never routing code: TTLs,
|
|
5
|
+
* refresh-on-read, minimum-cacheable prefix, key mechanism, telemetry dialect,
|
|
6
|
+
* endpoint affinity. A provider rule change (OpenAI shipping paid writes,
|
|
7
|
+
* Anthropic GA-ing the 1h TTL) becomes a data edit here with a new `as_of`, not
|
|
8
|
+
* a code change anywhere else — the whole point of the registry.
|
|
9
|
+
*
|
|
10
|
+
* TWO HOMES, both pre-existing (spec §3):
|
|
11
|
+
* • RATES ride pricing — `ModelPricingV2.base.{cache_read,cache_write,
|
|
12
|
+
* cache_write_1h}` and flat `ModelPricing.cacheWrite*UsdPerTok`, entered via
|
|
13
|
+
* the pricing sweep with source+as_of. NOT here.
|
|
14
|
+
* • POLICY (this file) is the non-price semantics.
|
|
15
|
+
*
|
|
16
|
+
* L5 (fail-closed): every field cites a first-party page (research §2.1, fetched
|
|
17
|
+
* 2026-08-18). A value we could not verify is `null` (or `cache_supported:
|
|
18
|
+
* false`) — never a guess. `getCachePolicy` returns a fail-closed
|
|
19
|
+
* `cache_supported: false` policy for any provider not seeded, identical posture
|
|
20
|
+
* to the zero-coverage gates.
|
|
21
|
+
*
|
|
22
|
+
* ── CROSS-REFERENCES (no import — the probes package is standalone since P2) ──
|
|
23
|
+
* • `min_cacheable_tokens` is reconciled by hand against the probes package's
|
|
24
|
+
* `MIN_CACHEABLE_TOKENS` (packages/modelrig-probes/src/caching.ts, PR #157),
|
|
25
|
+
* which cites the same first-party pages. See MIN_CACHEABLE_RECONCILIATION
|
|
26
|
+
* below for the value-by-value comparison and the two deliberate deltas
|
|
27
|
+
* (anthropic, gemini) flagged for the architect to converge if desired. A
|
|
28
|
+
* reciprocal comment lives in caching.ts.
|
|
29
|
+
* • OBSERVED cache facts (`cache_hit_rate`, `mean_cached_tokens`,
|
|
30
|
+
* `declared_caching_unrealized`) are materialized separately in
|
|
31
|
+
* `src/registry/build.ts` from probe/production telemetry — the DECLARED
|
|
32
|
+
* policy here is the counterpart truth, and the §5.6 streak alert (WS-C)
|
|
33
|
+
* cross-references `declared_caching_unrealized` (same smell, other plane).
|
|
34
|
+
*/
|
|
35
|
+
import type { ProviderId } from "../types/provider";
|
|
36
|
+
/**
|
|
37
|
+
* Per-provider cache semantics (spec §3). Additive to the registry's declared
|
|
38
|
+
* layer; every field is first-party-sourced or `null` (L5).
|
|
39
|
+
*/
|
|
40
|
+
export interface CachePolicy {
|
|
41
|
+
/** Whether the provider caches on the path ModelRig uses. false ⇒ every
|
|
42
|
+
* other field is moot (fail-closed). */
|
|
43
|
+
readonly cache_supported: boolean;
|
|
44
|
+
/** How caching is engaged. `explicit_breakpoints` = caller marks blocks
|
|
45
|
+
* (Anthropic cache_control); `automatic_prefix` = provider caches the longest
|
|
46
|
+
* matching prefix with no request change; `key_opt_in` = caller supplies a
|
|
47
|
+
* routing key (Mistral-style — no ModelRig provider uses it in v1). */
|
|
48
|
+
readonly mode: "explicit_breakpoints" | "automatic_prefix" | "key_opt_in";
|
|
49
|
+
/** Default cache lifetime, seconds. null ⇒ undocumented / best-effort. */
|
|
50
|
+
readonly default_ttl_seconds: number | null;
|
|
51
|
+
/** Extended-TTL option, seconds (Anthropic 1h write tier; OpenAI 24h
|
|
52
|
+
* retention). null ⇒ none published. */
|
|
53
|
+
readonly extended_ttl_seconds: number | null;
|
|
54
|
+
/** Whether a cache READ resets the TTL (Anthropic 5m: true). Conservative
|
|
55
|
+
* `false` when undocumented — under-claiming warmth never over-predicts a hit. */
|
|
56
|
+
readonly ttl_refresh_on_read: boolean;
|
|
57
|
+
/** Minimum prefix (tokens) below which caching silently does not engage.
|
|
58
|
+
* null ⇒ no documented minimum (L5). See MIN_CACHEABLE_RECONCILIATION. */
|
|
59
|
+
readonly min_cacheable_tokens: number | null;
|
|
60
|
+
/** The request parameter that engages/keys caching, or null for pure
|
|
61
|
+
* automatic caching with no caller control. */
|
|
62
|
+
readonly cache_key_param: "prompt_cache_key" | "cache_control" | "cachedContent" | null;
|
|
63
|
+
/** Whether cache-READ tokens are observable on the path ModelRig reads
|
|
64
|
+
* (usage JSON). false where they ride HTTP headers we do not yet lift. */
|
|
65
|
+
readonly reports_read_tokens: boolean;
|
|
66
|
+
/** Whether cache-WRITE (creation) tokens are separately reported. */
|
|
67
|
+
readonly reports_write_tokens: boolean;
|
|
68
|
+
/** Which usage shape the adapter normalizes from. */
|
|
69
|
+
readonly usage_dialect: "anthropic" | "openai_details" | "deepseek_partition" | "gemini_metadata" | "fireworks_headers";
|
|
70
|
+
/** true where the warm cache is endpoint-sticky and follow-ups must land on
|
|
71
|
+
* the same endpoint (research: OpenAI ~15 req/min/key; Fireworks
|
|
72
|
+
* x-session-affinity). */
|
|
73
|
+
readonly provider_affinity_required: boolean;
|
|
74
|
+
/** Provenance — same rule as pricing (first-party page). */
|
|
75
|
+
readonly source: string;
|
|
76
|
+
readonly as_of: string;
|
|
77
|
+
readonly policy_version: string;
|
|
78
|
+
}
|
|
79
|
+
/** The fail-closed policy for any provider not seeded (spec §3): caching is
|
|
80
|
+
* assumed OFF until a first-party source says otherwise — identical posture to
|
|
81
|
+
* the zero-coverage gates. `usage_dialect` defaults to the OpenAI shape only as
|
|
82
|
+
* a type-satisfying placeholder; `cache_supported: false` means no consumer
|
|
83
|
+
* reads it. */
|
|
84
|
+
export declare function failClosedCachePolicy(): CachePolicy;
|
|
85
|
+
/**
|
|
86
|
+
* The cache policy for a provider — the seeded first-party record, or a
|
|
87
|
+
* fail-closed `cache_supported: false` policy when the provider is not seeded
|
|
88
|
+
* (spec §3). Provider-level in v1; model-specific overrides land with the
|
|
89
|
+
* conversational-gateway phase.
|
|
90
|
+
*/
|
|
91
|
+
export declare function getCachePolicy(provider: ProviderId): CachePolicy;
|
|
92
|
+
/** Every seeded provider — for tests and the console policy surface (WS-D). */
|
|
93
|
+
export declare function seededCachePolicyProviders(): readonly ProviderId[];
|
|
94
|
+
//# sourceMappingURL=cache-policy.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache-policy.d.ts","sourceRoot":"","sources":["../../src/registry/cache-policy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B;4CACwC;IACxC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;IAClC;;;2EAGuE;IACvE,QAAQ,CAAC,IAAI,EAAE,sBAAsB,GAAG,kBAAkB,GAAG,YAAY,CAAC;IAC1E,0EAA0E;IAC1E,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C;4CACwC;IACxC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C;sFACkF;IAClF,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACtC;8EAC0E;IAC1E,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C;mDAC+C;IAC/C,QAAQ,CAAC,eAAe,EAAE,kBAAkB,GAAG,eAAe,GAAG,eAAe,GAAG,IAAI,CAAC;IACxF;8EAC0E;IAC1E,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACtC,qEAAqE;IACrE,QAAQ,CAAC,oBAAoB,EAAE,OAAO,CAAC;IACvC,qDAAqD;IACrD,QAAQ,CAAC,aAAa,EAClB,WAAW,GACX,gBAAgB,GAChB,oBAAoB,GACpB,iBAAiB,GACjB,mBAAmB,CAAC;IACxB;;8BAE0B;IAC1B,QAAQ,CAAC,0BAA0B,EAAE,OAAO,CAAC;IAC7C,4DAA4D;IAC5D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AA8KD;;;;eAIe;AACf,wBAAgB,qBAAqB,IAAI,WAAW,CAiBnD;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,UAAU,GAAG,WAAW,CAEhE;AAED,+EAA+E;AAC/E,wBAAgB,0BAA0B,IAAI,SAAS,UAAU,EAAE,CAElE"}
|