modelrig 0.0.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/LICENSE +201 -0
- package/README.md +145 -8
- package/data/PROVENANCE.md +19 -0
- package/data/pricing-manual.json +22 -0
- package/data/pricing-map.json +46362 -0
- package/data/pricing-overlay.json +419 -0
- package/dist/adapters/anthropic.d.ts +90 -0
- package/dist/adapters/anthropic.d.ts.map +1 -0
- package/dist/adapters/anthropic.js +296 -0
- package/dist/adapters/build-adapters.d.ts +11 -0
- package/dist/adapters/build-adapters.d.ts.map +1 -0
- package/dist/adapters/build-adapters.js +45 -0
- package/dist/adapters/deepseek.d.ts +47 -0
- package/dist/adapters/deepseek.d.ts.map +1 -0
- package/dist/adapters/deepseek.js +216 -0
- package/dist/adapters/gemini.d.ts +44 -0
- package/dist/adapters/gemini.d.ts.map +1 -0
- package/dist/adapters/gemini.js +262 -0
- package/dist/adapters/grok.d.ts +53 -0
- package/dist/adapters/grok.d.ts.map +1 -0
- package/dist/adapters/grok.js +232 -0
- package/dist/adapters/openai-compat.d.ts +80 -0
- package/dist/adapters/openai-compat.d.ts.map +1 -0
- package/dist/adapters/openai-compat.js +244 -0
- package/dist/adapters/openai-stream.d.ts +58 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -0
- package/dist/adapters/openai-stream.js +120 -0
- package/dist/adapters/openai.d.ts +55 -0
- package/dist/adapters/openai.d.ts.map +1 -0
- package/dist/adapters/openai.js +281 -0
- package/dist/adapters/sampling.d.ts +28 -0
- package/dist/adapters/sampling.d.ts.map +1 -0
- package/dist/adapters/sampling.js +34 -0
- package/dist/adapters/tool-mapping.d.ts +92 -0
- package/dist/adapters/tool-mapping.d.ts.map +1 -0
- package/dist/adapters/tool-mapping.js +143 -0
- package/dist/artifacts/artifact.d.ts +137 -0
- package/dist/artifacts/artifact.d.ts.map +1 -0
- package/dist/artifacts/artifact.js +377 -0
- package/dist/artifacts/content-uploader.d.ts +87 -0
- package/dist/artifacts/content-uploader.d.ts.map +1 -0
- package/dist/artifacts/content-uploader.js +124 -0
- package/dist/artifacts/index.d.ts +63 -0
- package/dist/artifacts/index.d.ts.map +1 -0
- package/dist/artifacts/index.js +80 -0
- package/dist/artifacts/run-context.d.ts +129 -0
- package/dist/artifacts/run-context.d.ts.map +1 -0
- package/dist/artifacts/run-context.js +208 -0
- package/dist/artifacts/scrub.d.ts +60 -0
- package/dist/artifacts/scrub.d.ts.map +1 -0
- package/dist/artifacts/scrub.js +139 -0
- package/dist/artifacts/step.d.ts +26 -0
- package/dist/artifacts/step.d.ts.map +1 -0
- package/dist/artifacts/step.js +37 -0
- package/dist/artifacts/taint.d.ts +63 -0
- package/dist/artifacts/taint.d.ts.map +1 -0
- package/dist/artifacts/taint.js +76 -0
- package/dist/bakeoff/cache-sim.d.ts +99 -0
- package/dist/bakeoff/cache-sim.d.ts.map +1 -0
- package/dist/bakeoff/cache-sim.js +126 -0
- package/dist/bakeoff/engine.d.ts +52 -0
- package/dist/bakeoff/engine.d.ts.map +1 -0
- package/dist/bakeoff/engine.js +163 -0
- package/dist/bakeoff/eval-cases.d.ts +96 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases.js +105 -0
- package/dist/bakeoff/replay.d.ts +52 -0
- package/dist/bakeoff/replay.d.ts.map +1 -0
- package/dist/bakeoff/replay.js +94 -0
- package/dist/bakeoff/report.d.ts +41 -0
- package/dist/bakeoff/report.d.ts.map +1 -0
- package/dist/bakeoff/report.js +142 -0
- package/dist/bakeoff/samples.d.ts +77 -0
- package/dist/bakeoff/samples.d.ts.map +1 -0
- package/dist/bakeoff/samples.js +148 -0
- package/dist/bakeoff/stats.d.ts +13 -0
- package/dist/bakeoff/stats.d.ts.map +1 -0
- package/dist/bakeoff/stats.js +48 -0
- package/dist/bakeoff/types.d.ts +37 -0
- package/dist/bakeoff/types.d.ts.map +1 -0
- package/dist/bakeoff/types.js +10 -0
- package/dist/cli.d.ts +25 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +590 -0
- package/dist/config.d.ts +204 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +252 -0
- package/dist/deprecations.d.ts +32 -0
- package/dist/deprecations.d.ts.map +1 -0
- package/dist/deprecations.js +59 -0
- package/dist/feedback/coverage.d.ts +32 -0
- package/dist/feedback/coverage.d.ts.map +1 -0
- package/dist/feedback/coverage.js +95 -0
- package/dist/feedback/feedback.d.ts +35 -0
- package/dist/feedback/feedback.d.ts.map +1 -0
- package/dist/feedback/feedback.js +39 -0
- package/dist/gateway/http-protocol.d.ts +175 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -0
- package/dist/gateway/http-protocol.js +105 -0
- package/dist/index.d.ts +242 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +576 -0
- package/dist/init/finder.d.ts +94 -0
- package/dist/init/finder.d.ts.map +1 -0
- package/dist/init/finder.js +343 -0
- package/dist/init/init-cli.d.ts +10 -0
- package/dist/init/init-cli.d.ts.map +1 -0
- package/dist/init/init-cli.js +86 -0
- package/dist/init/observe-cli.d.ts +19 -0
- package/dist/init/observe-cli.d.ts.map +1 -0
- package/dist/init/observe-cli.js +73 -0
- package/dist/init/observe.d.ts +81 -0
- package/dist/init/observe.d.ts.map +1 -0
- package/dist/init/observe.js +245 -0
- package/dist/init/run.d.ts +99 -0
- package/dist/init/run.d.ts.map +1 -0
- package/dist/init/run.js +281 -0
- package/dist/init/signatures.d.ts +39 -0
- package/dist/init/signatures.d.ts.map +1 -0
- package/dist/init/signatures.js +116 -0
- package/dist/matrix/aggregate.d.ts +89 -0
- package/dist/matrix/aggregate.d.ts.map +1 -0
- package/dist/matrix/aggregate.js +186 -0
- package/dist/query/grammar.d.ts +35 -0
- package/dist/query/grammar.d.ts.map +1 -0
- package/dist/query/grammar.js +279 -0
- package/dist/query/run.d.ts +8 -0
- package/dist/query/run.d.ts.map +1 -0
- package/dist/query/run.js +21 -0
- package/dist/registry/build.d.ts +219 -0
- package/dist/registry/build.d.ts.map +1 -0
- package/dist/registry/build.js +313 -0
- package/dist/registry/cache-policy.d.ts +94 -0
- package/dist/registry/cache-policy.d.ts.map +1 -0
- package/dist/registry/cache-policy.js +238 -0
- package/dist/registry/call-notes.d.ts +11 -0
- package/dist/registry/call-notes.d.ts.map +1 -0
- package/dist/registry/call-notes.js +110 -0
- package/dist/registry/claims.d.ts +63 -0
- package/dist/registry/claims.d.ts.map +1 -0
- package/dist/registry/claims.js +107 -0
- package/dist/registry/effective-rate.d.ts +151 -0
- package/dist/registry/effective-rate.d.ts.map +1 -0
- package/dist/registry/effective-rate.js +156 -0
- package/dist/registry/facts.d.ts +32 -0
- package/dist/registry/facts.d.ts.map +1 -0
- package/dist/registry/facts.js +136 -0
- package/dist/registry/fingerprint.d.ts +20 -0
- package/dist/registry/fingerprint.d.ts.map +1 -0
- package/dist/registry/fingerprint.js +150 -0
- package/dist/registry/parity.d.ts +52 -0
- package/dist/registry/parity.d.ts.map +1 -0
- package/dist/registry/parity.js +82 -0
- package/dist/registry/pricing-shape.d.ts +113 -0
- package/dist/registry/pricing-shape.d.ts.map +1 -0
- package/dist/registry/pricing-shape.js +123 -0
- package/dist/registry/pricing.d.ts +43 -0
- package/dist/registry/pricing.d.ts.map +1 -0
- package/dist/registry/pricing.js +127 -0
- package/dist/registry/rows.d.ts +16 -0
- package/dist/registry/rows.d.ts.map +1 -0
- package/dist/registry/rows.js +57 -0
- package/dist/registry/seed.d.ts +43 -0
- package/dist/registry/seed.d.ts.map +1 -0
- package/dist/registry/seed.js +97 -0
- package/dist/registry/select.d.ts +68 -0
- package/dist/registry/select.d.ts.map +1 -0
- package/dist/registry/select.js +141 -0
- package/dist/registry/staleness.d.ts +18 -0
- package/dist/registry/staleness.d.ts.map +1 -0
- package/dist/registry/staleness.js +39 -0
- package/dist/rigindex/bootstrap.d.ts +36 -0
- package/dist/rigindex/bootstrap.d.ts.map +1 -0
- package/dist/rigindex/bootstrap.js +97 -0
- package/dist/rigindex/fit.d.ts +24 -0
- package/dist/rigindex/fit.d.ts.map +1 -0
- package/dist/rigindex/fit.js +76 -0
- package/dist/rigindex/gates.d.ts +52 -0
- package/dist/rigindex/gates.d.ts.map +1 -0
- package/dist/rigindex/gates.js +109 -0
- package/dist/rigindex/graph.d.ts +31 -0
- package/dist/rigindex/graph.d.ts.map +1 -0
- package/dist/rigindex/graph.js +120 -0
- package/dist/rigindex/index.d.ts +23 -0
- package/dist/rigindex/index.d.ts.map +1 -0
- package/dist/rigindex/index.js +112 -0
- package/dist/rigindex/types.d.ts +76 -0
- package/dist/rigindex/types.d.ts.map +1 -0
- package/dist/rigindex/types.js +8 -0
- package/dist/routes/cache-lint.d.ts +41 -0
- package/dist/routes/cache-lint.d.ts.map +1 -0
- package/dist/routes/cache-lint.js +81 -0
- package/dist/routes/draft.d.ts +83 -0
- package/dist/routes/draft.d.ts.map +1 -0
- package/dist/routes/draft.js +135 -0
- package/dist/routes/loader.d.ts +63 -0
- package/dist/routes/loader.d.ts.map +1 -0
- package/dist/routes/loader.js +574 -0
- package/dist/routes/manifest.d.ts +24 -0
- package/dist/routes/manifest.d.ts.map +1 -0
- package/dist/routes/manifest.js +118 -0
- package/dist/routes/resolved-bundle.d.ts +79 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -0
- package/dist/routes/resolved-bundle.js +148 -0
- package/dist/routes/sampling-config.d.ts +32 -0
- package/dist/routes/sampling-config.d.ts.map +1 -0
- package/dist/routes/sampling-config.js +91 -0
- package/dist/routes/serveability.d.ts +52 -0
- package/dist/routes/serveability.d.ts.map +1 -0
- package/dist/routes/serveability.js +73 -0
- package/dist/routes/template.d.ts +18 -0
- package/dist/routes/template.d.ts.map +1 -0
- package/dist/routes/template.js +94 -0
- package/dist/run/ajv.d.ts +22 -0
- package/dist/run/ajv.d.ts.map +1 -0
- package/dist/run/ajv.js +32 -0
- package/dist/run/cache-handle.d.ts +58 -0
- package/dist/run/cache-handle.d.ts.map +1 -0
- package/dist/run/cache-handle.js +54 -0
- package/dist/run/cache-stamp.d.ts +65 -0
- package/dist/run/cache-stamp.d.ts.map +1 -0
- package/dist/run/cache-stamp.js +0 -0
- package/dist/run/capture-write.d.ts +33 -0
- package/dist/run/capture-write.d.ts.map +1 -0
- package/dist/run/capture-write.js +55 -0
- package/dist/run/cost.d.ts +26 -0
- package/dist/run/cost.d.ts.map +1 -0
- package/dist/run/cost.js +41 -0
- package/dist/run/dispatch.d.ts +99 -0
- package/dist/run/dispatch.d.ts.map +1 -0
- package/dist/run/dispatch.js +191 -0
- package/dist/run/envelope.d.ts +21 -0
- package/dist/run/envelope.d.ts.map +1 -0
- package/dist/run/envelope.js +45 -0
- package/dist/run/grounding.d.ts +56 -0
- package/dist/run/grounding.d.ts.map +1 -0
- package/dist/run/grounding.js +89 -0
- package/dist/run/raw.d.ts +88 -0
- package/dist/run/raw.d.ts.map +1 -0
- package/dist/run/raw.js +286 -0
- package/dist/run/record-attempt.d.ts +40 -0
- package/dist/run/record-attempt.d.ts.map +1 -0
- package/dist/run/record-attempt.js +99 -0
- package/dist/run/repair.d.ts +36 -0
- package/dist/run/repair.d.ts.map +1 -0
- package/dist/run/repair.js +61 -0
- package/dist/run/resolve.d.ts +99 -0
- package/dist/run/resolve.d.ts.map +1 -0
- package/dist/run/resolve.js +203 -0
- package/dist/run/retry.d.ts +19 -0
- package/dist/run/retry.d.ts.map +1 -0
- package/dist/run/retry.js +50 -0
- package/dist/run/run.d.ts +55 -0
- package/dist/run/run.d.ts.map +1 -0
- package/dist/run/run.js +540 -0
- package/dist/run/stream.d.ts +49 -0
- package/dist/run/stream.d.ts.map +1 -0
- package/dist/run/stream.js +87 -0
- package/dist/run/validate.d.ts +15 -0
- package/dist/run/validate.d.ts.map +1 -0
- package/dist/run/validate.js +55 -0
- package/dist/savings/cache-ledger.d.ts +108 -0
- package/dist/savings/cache-ledger.d.ts.map +1 -0
- package/dist/savings/cache-ledger.js +174 -0
- package/dist/savings/cache-reconcile.d.ts +64 -0
- package/dist/savings/cache-reconcile.d.ts.map +1 -0
- package/dist/savings/cache-reconcile.js +59 -0
- package/dist/savings/cache-stats.d.ts +63 -0
- package/dist/savings/cache-stats.d.ts.map +1 -0
- package/dist/savings/cache-stats.js +108 -0
- package/dist/savings/emit.d.ts +38 -0
- package/dist/savings/emit.d.ts.map +1 -0
- package/dist/savings/emit.js +100 -0
- package/dist/savings/odometer.d.ts +32 -0
- package/dist/savings/odometer.d.ts.map +1 -0
- package/dist/savings/odometer.js +70 -0
- package/dist/savings/swap-execute.d.ts +58 -0
- package/dist/savings/swap-execute.d.ts.map +1 -0
- package/dist/savings/swap-execute.js +130 -0
- package/dist/savings/swaps-client.d.ts +55 -0
- package/dist/savings/swaps-client.d.ts.map +1 -0
- package/dist/savings/swaps-client.js +95 -0
- package/dist/settings/api-fetcher.d.ts +39 -0
- package/dist/settings/api-fetcher.d.ts.map +1 -0
- package/dist/settings/api-fetcher.js +92 -0
- package/dist/settings/index.d.ts +10 -0
- package/dist/settings/index.d.ts.map +1 -0
- package/dist/settings/index.js +15 -0
- package/dist/settings/task-settings.d.ts +91 -0
- package/dist/settings/task-settings.d.ts.map +1 -0
- package/dist/settings/task-settings.js +116 -0
- package/dist/status.d.ts +77 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +113 -0
- package/dist/telemetry/artifact-store.d.ts +48 -0
- package/dist/telemetry/artifact-store.d.ts.map +1 -0
- package/dist/telemetry/artifact-store.js +189 -0
- package/dist/telemetry/cache-miss-streak.d.ts +79 -0
- package/dist/telemetry/cache-miss-streak.d.ts.map +1 -0
- package/dist/telemetry/cache-miss-streak.js +99 -0
- package/dist/telemetry/capture.d.ts +31 -0
- package/dist/telemetry/capture.d.ts.map +1 -0
- package/dist/telemetry/capture.js +81 -0
- package/dist/telemetry/exporter-setup.d.ts +28 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -0
- package/dist/telemetry/exporter-setup.js +34 -0
- package/dist/telemetry/exporter.d.ts +137 -0
- package/dist/telemetry/exporter.d.ts.map +1 -0
- package/dist/telemetry/exporter.js +510 -0
- package/dist/telemetry/health.d.ts +76 -0
- package/dist/telemetry/health.d.ts.map +1 -0
- package/dist/telemetry/health.js +92 -0
- package/dist/telemetry/ingest-protocol.d.ts +310 -0
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -0
- package/dist/telemetry/ingest-protocol.js +363 -0
- package/dist/telemetry/load-error-report.d.ts +24 -0
- package/dist/telemetry/load-error-report.d.ts.map +1 -0
- package/dist/telemetry/load-error-report.js +50 -0
- package/dist/telemetry/otel-map.d.ts +121 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -0
- package/dist/telemetry/otel-map.js +224 -0
- package/dist/telemetry/otel-normalize.d.ts +104 -0
- package/dist/telemetry/otel-normalize.d.ts.map +1 -0
- package/dist/telemetry/otel-normalize.js +530 -0
- package/dist/telemetry/otlp-emitter.d.ts +52 -0
- package/dist/telemetry/otlp-emitter.d.ts.map +1 -0
- package/dist/telemetry/otlp-emitter.js +144 -0
- package/dist/telemetry/otlp-proto.d.ts +29 -0
- package/dist/telemetry/otlp-proto.d.ts.map +1 -0
- package/dist/telemetry/otlp-proto.js +266 -0
- package/dist/telemetry/sinks.d.ts +90 -0
- package/dist/telemetry/sinks.d.ts.map +1 -0
- package/dist/telemetry/sinks.js +180 -0
- package/dist/telemetry/store.d.ts +65 -0
- package/dist/telemetry/store.d.ts.map +1 -0
- package/dist/telemetry/store.js +547 -0
- package/dist/tenancy/api-key-format.d.ts +77 -0
- package/dist/tenancy/api-key-format.d.ts.map +1 -0
- package/dist/tenancy/api-key-format.js +110 -0
- package/dist/tenancy/constants.d.ts +23 -0
- package/dist/tenancy/constants.d.ts.map +1 -0
- package/dist/tenancy/constants.js +25 -0
- package/dist/tenancy/scrub.d.ts +47 -0
- package/dist/tenancy/scrub.d.ts.map +1 -0
- package/dist/tenancy/scrub.js +85 -0
- package/dist/types/config-teaching.d.ts +62 -0
- package/dist/types/config-teaching.d.ts.map +1 -0
- package/dist/types/config-teaching.js +183 -0
- package/dist/types/failures-teaching.d.ts +56 -0
- package/dist/types/failures-teaching.d.ts.map +1 -0
- package/dist/types/failures-teaching.js +177 -0
- package/dist/types/failures.d.ts +27 -0
- package/dist/types/failures.d.ts.map +1 -0
- package/dist/types/failures.js +41 -0
- package/dist/types/provider.d.ts +200 -0
- package/dist/types/provider.d.ts.map +1 -0
- package/dist/types/provider.js +40 -0
- package/dist/types/route.d.ts +205 -0
- package/dist/types/route.d.ts.map +1 -0
- package/dist/types/route.js +6 -0
- package/dist/types/telemetry.d.ts +206 -0
- package/dist/types/telemetry.d.ts.map +1 -0
- package/dist/types/telemetry.js +6 -0
- package/dist/validate-cli.d.ts +12 -0
- package/dist/validate-cli.d.ts.map +1 -0
- package/dist/validate-cli.js +57 -0
- package/dist/verify/capture.d.ts +66 -0
- package/dist/verify/capture.d.ts.map +1 -0
- package/dist/verify/capture.js +180 -0
- package/dist/verify/cli-run.d.ts +23 -0
- package/dist/verify/cli-run.d.ts.map +1 -0
- package/dist/verify/cli-run.js +57 -0
- package/dist/verify/diff.d.ts +48 -0
- package/dist/verify/diff.d.ts.map +1 -0
- package/dist/verify/diff.js +157 -0
- package/dist/verify/index.d.ts +17 -0
- package/dist/verify/index.d.ts.map +1 -0
- package/dist/verify/index.js +24 -0
- package/dist/verify/report.d.ts +14 -0
- package/dist/verify/report.d.ts.map +1 -0
- package/dist/verify/report.js +88 -0
- package/dist/verify/types.d.ts +101 -0
- package/dist/verify/types.d.ts.map +1 -0
- package/dist/verify/types.js +17 -0
- package/dist/verify/verify.d.ts +29 -0
- package/dist/verify/verify.d.ts.map +1 -0
- package/dist/verify/verify.js +148 -0
- package/dist/watch/budget.d.ts +19 -0
- package/dist/watch/budget.d.ts.map +1 -0
- package/dist/watch/budget.js +35 -0
- package/dist/watch/evaluate.d.ts +225 -0
- package/dist/watch/evaluate.d.ts.map +1 -0
- package/dist/watch/evaluate.js +155 -0
- package/dist/watch/monitor.d.ts +116 -0
- package/dist/watch/monitor.d.ts.map +1 -0
- package/dist/watch/monitor.js +199 -0
- package/dist/watch/proposals.d.ts +89 -0
- package/dist/watch/proposals.d.ts.map +1 -0
- package/dist/watch/proposals.js +197 -0
- package/dist/watch/run-record.d.ts +64 -0
- package/dist/watch/run-record.d.ts.map +1 -0
- package/dist/watch/run-record.js +86 -0
- package/dist/watch/run-reported.d.ts +20 -0
- package/dist/watch/run-reported.d.ts.map +1 -0
- package/dist/watch/run-reported.js +53 -0
- package/dist/watch/run.d.ts +73 -0
- package/dist/watch/run.d.ts.map +1 -0
- package/dist/watch/run.js +83 -0
- package/dist/watch/state-store.d.ts +48 -0
- package/dist/watch/state-store.d.ts.map +1 -0
- package/dist/watch/state-store.js +86 -0
- package/dist/watch/state.d.ts +21 -0
- package/dist/watch/state.d.ts.map +1 -0
- package/dist/watch/state.js +93 -0
- package/package.json +56 -14
- package/registry/LICENSE +20 -0
- package/registry/leaderboard.json +3639 -0
- package/registry/registry.json +7263 -0
- package/index.js +0 -3
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Candidate seeding — the "what should serve this route?" answer, from the
|
|
3
|
+
* registry rather than from taste. Shared by `modelrig init` (WS2) and the
|
|
4
|
+
* oracle's `create_route` (WS1), so a drafted route looks the same however
|
|
5
|
+
* it was drafted.
|
|
6
|
+
*
|
|
7
|
+
* The rule: the INCUMBENT leads. The model you already call goes first, so
|
|
8
|
+
* the drafted route reproduces today's behavior on its first run; the
|
|
9
|
+
* cheapest probed models that satisfy the inferred capability needs follow
|
|
10
|
+
* as challengers a bake-off can put to the test. We never silently promote
|
|
11
|
+
* a cheaper model into first position — that decision belongs to evidence
|
|
12
|
+
* and a human.
|
|
13
|
+
*/
|
|
14
|
+
import type { CapabilityPredicate, ModelSelectRow } from "./select";
|
|
15
|
+
import type { DraftCandidate } from "../routes/draft";
|
|
16
|
+
export interface SeedCandidatesInput {
|
|
17
|
+
readonly capabilities: readonly CapabilityPredicate[];
|
|
18
|
+
/** "provider/model" the caller uses today, if known. */
|
|
19
|
+
readonly incumbent: string | null;
|
|
20
|
+
readonly maxInputUsdPerMtok: number | null;
|
|
21
|
+
/** Caller-supplied candidates short-circuit seeding entirely. */
|
|
22
|
+
readonly explicit: ReadonlyArray<{
|
|
23
|
+
readonly provider: string;
|
|
24
|
+
readonly model: string;
|
|
25
|
+
}> | null;
|
|
26
|
+
/** How many challengers to add behind the incumbent. */
|
|
27
|
+
readonly challengerCount?: number;
|
|
28
|
+
}
|
|
29
|
+
export interface SeededCandidates {
|
|
30
|
+
readonly candidates: readonly DraftCandidate[];
|
|
31
|
+
readonly incumbent: {
|
|
32
|
+
readonly provider: string;
|
|
33
|
+
readonly model: string;
|
|
34
|
+
} | null;
|
|
35
|
+
/** Human-readable account of why this list looks like it does. */
|
|
36
|
+
readonly reasoning: readonly string[];
|
|
37
|
+
}
|
|
38
|
+
export declare function splitModelKey(modelKey: string): {
|
|
39
|
+
provider: string;
|
|
40
|
+
model: string;
|
|
41
|
+
} | null;
|
|
42
|
+
export declare function seedCandidates(rows: readonly ModelSelectRow[], input: SeedCandidatesInput): SeededCandidates;
|
|
43
|
+
//# sourceMappingURL=seed.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"seed.d.ts","sourceRoot":"","sources":["../../src/registry/seed.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,KAAK,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,YAAY,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACtD,wDAAwD;IACxD,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,iEAAiE;IACjE,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;QAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,IAAI,CAAC;IAC/F,wDAAwD;IACxD,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,CAAC;IAC/C,QAAQ,CAAC,SAAS,EAAE;QAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACjF,kEAAkE;IAClE,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;CACvC;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAI1F;AAED,wBAAgB,cAAc,CAC5B,IAAI,EAAE,SAAS,cAAc,EAAE,EAC/B,KAAK,EAAE,mBAAmB,GACzB,gBAAgB,CAqFlB"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Candidate seeding — the "what should serve this route?" answer, from the
|
|
4
|
+
* registry rather than from taste. Shared by `modelrig init` (WS2) and the
|
|
5
|
+
* oracle's `create_route` (WS1), so a drafted route looks the same however
|
|
6
|
+
* it was drafted.
|
|
7
|
+
*
|
|
8
|
+
* The rule: the INCUMBENT leads. The model you already call goes first, so
|
|
9
|
+
* the drafted route reproduces today's behavior on its first run; the
|
|
10
|
+
* cheapest probed models that satisfy the inferred capability needs follow
|
|
11
|
+
* as challengers a bake-off can put to the test. We never silently promote
|
|
12
|
+
* a cheaper model into first position — that decision belongs to evidence
|
|
13
|
+
* and a human.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.splitModelKey = splitModelKey;
|
|
17
|
+
exports.seedCandidates = seedCandidates;
|
|
18
|
+
const select_1 = require("./select");
|
|
19
|
+
function splitModelKey(modelKey) {
|
|
20
|
+
const slash = modelKey.indexOf("/");
|
|
21
|
+
if (slash <= 0 || slash === modelKey.length - 1)
|
|
22
|
+
return null;
|
|
23
|
+
return { provider: modelKey.slice(0, slash), model: modelKey.slice(slash + 1) };
|
|
24
|
+
}
|
|
25
|
+
function seedCandidates(rows, input) {
|
|
26
|
+
if (input.explicit !== null && input.explicit.length > 0) {
|
|
27
|
+
return {
|
|
28
|
+
candidates: input.explicit.map((c) => ({ ...c })),
|
|
29
|
+
incumbent: input.explicit[0] ?? null,
|
|
30
|
+
reasoning: ["Candidates were given explicitly — the registry was not consulted."],
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
const reasoning = [];
|
|
34
|
+
const candidates = [];
|
|
35
|
+
let incumbent = null;
|
|
36
|
+
if (input.incumbent !== null && input.incumbent !== "") {
|
|
37
|
+
const parsed = splitModelKey(input.incumbent);
|
|
38
|
+
if (parsed === null) {
|
|
39
|
+
reasoning.push(`Could not read "${input.incumbent}" as an incumbent — model keys are "provider/model". ` +
|
|
40
|
+
"Drafted without one, so review the ordering before you run it.");
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
incumbent = parsed;
|
|
44
|
+
const known = rows.some((r) => r.key === input.incumbent);
|
|
45
|
+
candidates.push({
|
|
46
|
+
...parsed,
|
|
47
|
+
note: known
|
|
48
|
+
? "incumbent — your current model, listed first so behavior is unchanged on the first run"
|
|
49
|
+
: "incumbent — your current model (not in our probed registry, so unmeasured here)",
|
|
50
|
+
});
|
|
51
|
+
reasoning.push(`Incumbent ${input.incumbent} leads: the drafted route reproduces what you do today.`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
reasoning.push("No incumbent was identified, so the cheapest probed model that meets the requirements " +
|
|
56
|
+
"leads. If you already use a specific model, put it first before running this.");
|
|
57
|
+
}
|
|
58
|
+
const challengerCount = input.challengerCount ?? 2;
|
|
59
|
+
const wanted = challengerCount + (incumbent === null ? 0 : 1);
|
|
60
|
+
const hits = (0, select_1.selectModels)(rows, {
|
|
61
|
+
caps: input.capabilities,
|
|
62
|
+
maxInputUsdPerMtok: input.maxInputUsdPerMtok,
|
|
63
|
+
limit: challengerCount + 1,
|
|
64
|
+
});
|
|
65
|
+
for (const row of hits) {
|
|
66
|
+
if (candidates.length >= wanted)
|
|
67
|
+
break;
|
|
68
|
+
if (incumbent !== null && row.key === `${incumbent.provider}/${incumbent.model}`)
|
|
69
|
+
continue;
|
|
70
|
+
const parsed = splitModelKey(row.key);
|
|
71
|
+
if (parsed === null)
|
|
72
|
+
continue;
|
|
73
|
+
const effective = row.effectiveUsdPer1k;
|
|
74
|
+
candidates.push({
|
|
75
|
+
...parsed,
|
|
76
|
+
note: effective === null
|
|
77
|
+
? "probed, but no effective-cost figure yet"
|
|
78
|
+
: `probed: $${effective.toFixed(4)} per 1k conformant outputs`,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
if (input.capabilities.length > 0) {
|
|
82
|
+
reasoning.push(`Challengers had to satisfy every requirement asked for (${input.capabilities.join(", ")}) ` +
|
|
83
|
+
"in probing — declared support alone was not enough.");
|
|
84
|
+
}
|
|
85
|
+
if (input.maxInputUsdPerMtok !== null) {
|
|
86
|
+
reasoning.push(`Challengers were capped at $${input.maxInputUsdPerMtok}/M input tokens.`);
|
|
87
|
+
}
|
|
88
|
+
if (candidates.length === 0) {
|
|
89
|
+
reasoning.push("No probed model satisfied the requirements, so this draft has no candidates and will not " +
|
|
90
|
+
"load — name a model explicitly, or loosen the requirements.");
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
reasoning.push("Challengers are ranked by effective cost of conformance, not list price. Nothing switches " +
|
|
94
|
+
"to them without a bake-off and an explicit approval.");
|
|
95
|
+
}
|
|
96
|
+
return { candidates, incumbent, reasoning };
|
|
97
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Model selection — the capability-first question, answered once.
|
|
3
|
+
*
|
|
4
|
+
* "Which models can serve strict JSON under $1/M?" is asked in three places:
|
|
5
|
+
* the console's query box, the MCP oracle's `query_registry` tool, and the
|
|
6
|
+
* docs. Phase 5 plan §10 names the risk by name — oracle answers drifting
|
|
7
|
+
* from console answers — so the grammar and the predicates live HERE and
|
|
8
|
+
* both surfaces call in. If this file changes, every surface changes.
|
|
9
|
+
*
|
|
10
|
+
* Dependency-free on purpose: the console imports this source directly (its
|
|
11
|
+
* tsconfig path alias) and must not drag SQLite or the adapters into a
|
|
12
|
+
* browser bundle.
|
|
13
|
+
*
|
|
14
|
+
* Bars are the registry's own, not new judgments: 0.9 conformance (the
|
|
15
|
+
* discrepancy detector's bar) and 0.5 grounding (facts.ts's GROUNDED_BAR).
|
|
16
|
+
*/
|
|
17
|
+
/** The capability predicates a question can carry. Deliberately the probed
|
|
18
|
+
* dimensions only — we answer from measurement, not from provider claims. */
|
|
19
|
+
export type CapabilityPredicate = "structured-native" | "structured-coached" | "grounding" | "caching";
|
|
20
|
+
export declare const ALL_CAPABILITY_PREDICATES: readonly CapabilityPredicate[];
|
|
21
|
+
/** Plain-language gloss per predicate — the explainability layer travels
|
|
22
|
+
* WITH the answer (console-explainability.md §1), it is not a separate doc. */
|
|
23
|
+
export declare const PREDICATE_GLOSS: Readonly<Record<CapabilityPredicate, string>>;
|
|
24
|
+
/** The row shape selection needs. A projection, not the registry entry —
|
|
25
|
+
* console (Supabase mirror) and oracle (registry.json) both flatten into it. */
|
|
26
|
+
export interface ModelSelectRow {
|
|
27
|
+
readonly key: string;
|
|
28
|
+
readonly provider: string;
|
|
29
|
+
readonly inputUsdPerMtok: number | null;
|
|
30
|
+
readonly outputUsdPerMtok: number | null;
|
|
31
|
+
readonly conformRate: number | null;
|
|
32
|
+
readonly nativeRungRate: number | null;
|
|
33
|
+
readonly groundedRate: number | null;
|
|
34
|
+
readonly cacheHitRate: number | null;
|
|
35
|
+
readonly effectiveUsdPer1k: number | null;
|
|
36
|
+
readonly discrepancies: number;
|
|
37
|
+
readonly probedAsOf: string | null;
|
|
38
|
+
}
|
|
39
|
+
export declare function hasCapability(row: ModelSelectRow, cap: CapabilityPredicate): boolean;
|
|
40
|
+
export interface ParsedQuery {
|
|
41
|
+
readonly caps: readonly CapabilityPredicate[];
|
|
42
|
+
readonly maxInputUsdPerMtok: number | null;
|
|
43
|
+
/** Leftover words — substring-matched against the model key. */
|
|
44
|
+
readonly text: readonly string[];
|
|
45
|
+
}
|
|
46
|
+
/** The tiny grammar behind the query box and the oracle's free-text input:
|
|
47
|
+
* capability words + an "under $N/M" price cap; everything else is a
|
|
48
|
+
* substring match on the model key. Small on purpose — a question we cannot
|
|
49
|
+
* parse should fall through to text, not guess. */
|
|
50
|
+
export declare function parseCapabilityQuery(query: string): ParsedQuery;
|
|
51
|
+
export interface SelectCriteria {
|
|
52
|
+
/** Capability predicates ANDed together — every one must hold. */
|
|
53
|
+
readonly caps?: readonly CapabilityPredicate[];
|
|
54
|
+
readonly maxInputUsdPerMtok?: number | null;
|
|
55
|
+
readonly provider?: string | null;
|
|
56
|
+
readonly discrepanciesOnly?: boolean;
|
|
57
|
+
/** Substring terms ANDed against the model key. */
|
|
58
|
+
readonly text?: readonly string[];
|
|
59
|
+
/** Sort by effective cost of conformance ascending (default true) — the
|
|
60
|
+
* only ranking that answers "cheapest that actually works". */
|
|
61
|
+
readonly sortByEffectiveCost?: boolean;
|
|
62
|
+
readonly limit?: number;
|
|
63
|
+
}
|
|
64
|
+
export declare function selectModels(rows: readonly ModelSelectRow[], criteria?: SelectCriteria): ModelSelectRow[];
|
|
65
|
+
/** Why a question found nothing — the difference between "no such model" and
|
|
66
|
+
* "we have not probed one yet" is the whole point of the demand meter. */
|
|
67
|
+
export declare function explainEmptySelection(criteria: SelectCriteria): string;
|
|
68
|
+
//# sourceMappingURL=select.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"select.d.ts","sourceRoot":"","sources":["../../src/registry/select.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH;6EAC6E;AAC7E,MAAM,MAAM,mBAAmB,GAC3B,mBAAmB,GACnB,oBAAoB,GACpB,WAAW,GACX,SAAS,CAAC;AAEd,eAAO,MAAM,yBAAyB,EAAE,SAAS,mBAAmB,EAKnE,CAAC;AAEF;+EAC+E;AAC/E,eAAO,MAAM,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAQzE,CAAC;AAEF;gFACgF;AAChF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAKD,wBAAgB,aAAa,CAAC,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,mBAAmB,GAAG,OAAO,CAWpF;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC9C,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,gEAAgE;IAChE,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AA0BD;;;mDAGmD;AACnD,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,CAkB/D;AAED,MAAM,WAAW,cAAc;IAC7B,kEAAkE;IAClE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC/C,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IACrC,mDAAmD;IACnD,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC;mEAC+D;IAC/D,QAAQ,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IACvC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,SAAS,cAAc,EAAE,EAC/B,QAAQ,GAAE,cAAmB,GAC5B,cAAc,EAAE,CA6BlB;AAED;0EAC0E;AAC1E,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,CAatE"}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Model selection — the capability-first question, answered once.
|
|
4
|
+
*
|
|
5
|
+
* "Which models can serve strict JSON under $1/M?" is asked in three places:
|
|
6
|
+
* the console's query box, the MCP oracle's `query_registry` tool, and the
|
|
7
|
+
* docs. Phase 5 plan §10 names the risk by name — oracle answers drifting
|
|
8
|
+
* from console answers — so the grammar and the predicates live HERE and
|
|
9
|
+
* both surfaces call in. If this file changes, every surface changes.
|
|
10
|
+
*
|
|
11
|
+
* Dependency-free on purpose: the console imports this source directly (its
|
|
12
|
+
* tsconfig path alias) and must not drag SQLite or the adapters into a
|
|
13
|
+
* browser bundle.
|
|
14
|
+
*
|
|
15
|
+
* Bars are the registry's own, not new judgments: 0.9 conformance (the
|
|
16
|
+
* discrepancy detector's bar) and 0.5 grounding (facts.ts's GROUNDED_BAR).
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.PREDICATE_GLOSS = exports.ALL_CAPABILITY_PREDICATES = void 0;
|
|
20
|
+
exports.hasCapability = hasCapability;
|
|
21
|
+
exports.parseCapabilityQuery = parseCapabilityQuery;
|
|
22
|
+
exports.selectModels = selectModels;
|
|
23
|
+
exports.explainEmptySelection = explainEmptySelection;
|
|
24
|
+
exports.ALL_CAPABILITY_PREDICATES = [
|
|
25
|
+
"structured-native",
|
|
26
|
+
"structured-coached",
|
|
27
|
+
"grounding",
|
|
28
|
+
"caching",
|
|
29
|
+
];
|
|
30
|
+
/** Plain-language gloss per predicate — the explainability layer travels
|
|
31
|
+
* WITH the answer (console-explainability.md §1), it is not a separate doc. */
|
|
32
|
+
exports.PREDICATE_GLOSS = {
|
|
33
|
+
"structured-native": "the provider enforces your JSON Schema on its side — probed native rung rate above zero",
|
|
34
|
+
"structured-coached": "output matched the schema at least 90% of probed samples, whether or not the provider enforced it natively",
|
|
35
|
+
grounding: "at least half of probed answers cited real retrieved sources rather than answering from memory",
|
|
36
|
+
caching: "repeat calls actually reported cached input tokens — a measured discount, not a claim",
|
|
37
|
+
};
|
|
38
|
+
const CONFORMANCE_BAR = 0.9;
|
|
39
|
+
const GROUNDED_BAR = 0.5;
|
|
40
|
+
function hasCapability(row, cap) {
|
|
41
|
+
switch (cap) {
|
|
42
|
+
case "structured-native":
|
|
43
|
+
return (row.nativeRungRate ?? 0) > 0;
|
|
44
|
+
case "structured-coached":
|
|
45
|
+
return (row.conformRate ?? 0) >= CONFORMANCE_BAR;
|
|
46
|
+
case "grounding":
|
|
47
|
+
return (row.groundedRate ?? 0) >= GROUNDED_BAR;
|
|
48
|
+
case "caching":
|
|
49
|
+
return (row.cacheHitRate ?? 0) > 0;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/** Question scaffolding. A search box gets keywords; the MCP oracle gets a
|
|
53
|
+
* whole sentence ("which models can serve strict JSON under $1/M?"), and
|
|
54
|
+
* without this every one of those words would become a substring filter on
|
|
55
|
+
* the model key and match nothing. Dropping them is strictly safer than
|
|
56
|
+
* matching them: no model key contains "which". */
|
|
57
|
+
const STOP_WORDS = new Set([
|
|
58
|
+
"a", "about", "an", "and", "any", "are", "avail", "available", "be", "best", "can", "cheapest", "do",
|
|
59
|
+
"does", "find", "for", "give", "has", "have", "how", "i", "is", "it", "list", "me", "model",
|
|
60
|
+
"models", "need", "of", "or", "provider", "providers", "serve", "serves", "show", "some",
|
|
61
|
+
"support", "supports", "that", "the", "their", "them", "there", "these", "they", "this",
|
|
62
|
+
"those", "to", "use", "using", "want", "what", "whats", "which", "who", "will", "with",
|
|
63
|
+
]);
|
|
64
|
+
/** Punctuation a question drags in — stripped so "json?" is still "json". */
|
|
65
|
+
function normalizeWord(word) {
|
|
66
|
+
return word.replace(/^[^a-z0-9./_-]+|[^a-z0-9./_-]+$/g, "");
|
|
67
|
+
}
|
|
68
|
+
const CAP_WORDS = [
|
|
69
|
+
[["json", "strict", "structured", "schema"], "structured-native"],
|
|
70
|
+
[["grounded", "grounding", "search", "web"], "grounding"],
|
|
71
|
+
[["cache", "caching", "cached"], "caching"],
|
|
72
|
+
];
|
|
73
|
+
/** The tiny grammar behind the query box and the oracle's free-text input:
|
|
74
|
+
* capability words + an "under $N/M" price cap; everything else is a
|
|
75
|
+
* substring match on the model key. Small on purpose — a question we cannot
|
|
76
|
+
* parse should fall through to text, not guess. */
|
|
77
|
+
function parseCapabilityQuery(query) {
|
|
78
|
+
const caps = [];
|
|
79
|
+
const text = [];
|
|
80
|
+
const priceMatch = query.match(/under\s+\$?(\d+(?:\.\d+)?)\s*\/?\s*m/i);
|
|
81
|
+
const maxInputUsdPerMtok = priceMatch ? Number(priceMatch[1]) : null;
|
|
82
|
+
const cleaned = query.replace(/under\s+\$?\d+(?:\.\d+)?\s*\/?\s*m\w*/i, "");
|
|
83
|
+
for (const raw of cleaned.toLowerCase().split(/\s+/)) {
|
|
84
|
+
const word = normalizeWord(raw);
|
|
85
|
+
if (word === "" || STOP_WORDS.has(word))
|
|
86
|
+
continue;
|
|
87
|
+
const match = CAP_WORDS.find(([words]) => words.includes(word));
|
|
88
|
+
if (match === undefined) {
|
|
89
|
+
text.push(word);
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
if (!caps.includes(match[1]))
|
|
93
|
+
caps.push(match[1]);
|
|
94
|
+
}
|
|
95
|
+
return { caps, maxInputUsdPerMtok, text };
|
|
96
|
+
}
|
|
97
|
+
function selectModels(rows, criteria = {}) {
|
|
98
|
+
const caps = criteria.caps ?? [];
|
|
99
|
+
const text = criteria.text ?? [];
|
|
100
|
+
let out = rows.filter((row) => {
|
|
101
|
+
if (criteria.provider != null && criteria.provider !== "" && row.provider !== criteria.provider) {
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
104
|
+
if (criteria.discrepanciesOnly === true && row.discrepancies === 0)
|
|
105
|
+
return false;
|
|
106
|
+
if (caps.some((cap) => !hasCapability(row, cap)))
|
|
107
|
+
return false;
|
|
108
|
+
if (criteria.maxInputUsdPerMtok != null) {
|
|
109
|
+
// A model with no published input price cannot be proven under a cap —
|
|
110
|
+
// absence is not "cheap enough".
|
|
111
|
+
if (row.inputUsdPerMtok === null || row.inputUsdPerMtok > criteria.maxInputUsdPerMtok) {
|
|
112
|
+
return false;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
if (text.length > 0 && !text.every((w) => row.key.toLowerCase().includes(w)))
|
|
116
|
+
return false;
|
|
117
|
+
return true;
|
|
118
|
+
});
|
|
119
|
+
if (criteria.sortByEffectiveCost !== false) {
|
|
120
|
+
out = [...out].sort((a, b) => (a.effectiveUsdPer1k ?? Number.POSITIVE_INFINITY) -
|
|
121
|
+
(b.effectiveUsdPer1k ?? Number.POSITIVE_INFINITY));
|
|
122
|
+
}
|
|
123
|
+
if (criteria.limit !== undefined && criteria.limit > 0)
|
|
124
|
+
out = out.slice(0, criteria.limit);
|
|
125
|
+
return out;
|
|
126
|
+
}
|
|
127
|
+
/** Why a question found nothing — the difference between "no such model" and
|
|
128
|
+
* "we have not probed one yet" is the whole point of the demand meter. */
|
|
129
|
+
function explainEmptySelection(criteria) {
|
|
130
|
+
const parts = [];
|
|
131
|
+
for (const cap of criteria.caps ?? [])
|
|
132
|
+
parts.push(exports.PREDICATE_GLOSS[cap]);
|
|
133
|
+
if (criteria.maxInputUsdPerMtok != null) {
|
|
134
|
+
parts.push(`input price at or under $${criteria.maxInputUsdPerMtok}/M tokens`);
|
|
135
|
+
}
|
|
136
|
+
const asked = parts.length > 0 ? parts.join("; and ") : "the requested filters";
|
|
137
|
+
return (`No probed model in the registry satisfies ${asked}. ` +
|
|
138
|
+
"This means no model we have MEASURED qualifies — not that none exists. " +
|
|
139
|
+
"Unprobed models are absent from these answers by design; open a probe request " +
|
|
140
|
+
"if you need a specific model covered.");
|
|
141
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Probe-data freshness — addendum A2 (TrustedRouter pattern 3): never serve
|
|
3
|
+
* stale-as-fresh. The SLO is 7-day-on-change / monthly; without per-entry
|
|
4
|
+
* change-detection timestamps the enforceable bound is the monthly one, so
|
|
5
|
+
* staleness = newest probed as_of older than 31 days. Stale data stays
|
|
6
|
+
* visible — bannered and dated, never hidden.
|
|
7
|
+
*/
|
|
8
|
+
export declare const PROBE_FRESHNESS_SLO_DAYS = 31;
|
|
9
|
+
export interface ProbeFreshness {
|
|
10
|
+
readonly stale: boolean;
|
|
11
|
+
/** Newest probed as_of across the corpus — the banner's date. */
|
|
12
|
+
readonly newestAsOf: string | null;
|
|
13
|
+
readonly daysOld: number | null;
|
|
14
|
+
}
|
|
15
|
+
export declare function probeFreshness(asOfDates: ReadonlyArray<string | null | undefined>, now?: Date): ProbeFreshness;
|
|
16
|
+
/** The banner sentence (brand guide §7: plain, dated, never hidden). */
|
|
17
|
+
export declare function stalenessSentence(freshness: ProbeFreshness): string;
|
|
18
|
+
//# sourceMappingURL=staleness.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"staleness.d.ts","sourceRoot":"","sources":["../../src/registry/staleness.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,eAAO,MAAM,wBAAwB,KAAK,CAAC;AAE3C,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,iEAAiE;IACjE,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC;AAED,wBAAgB,cAAc,CAC5B,SAAS,EAAE,aAAa,CAAC,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,EACnD,GAAG,GAAE,IAAiB,GACrB,cAAc,CAgBhB;AAED,wEAAwE;AACxE,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,cAAc,GAAG,MAAM,CASnE"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Probe-data freshness — addendum A2 (TrustedRouter pattern 3): never serve
|
|
4
|
+
* stale-as-fresh. The SLO is 7-day-on-change / monthly; without per-entry
|
|
5
|
+
* change-detection timestamps the enforceable bound is the monthly one, so
|
|
6
|
+
* staleness = newest probed as_of older than 31 days. Stale data stays
|
|
7
|
+
* visible — bannered and dated, never hidden.
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.PROBE_FRESHNESS_SLO_DAYS = void 0;
|
|
11
|
+
exports.probeFreshness = probeFreshness;
|
|
12
|
+
exports.stalenessSentence = stalenessSentence;
|
|
13
|
+
exports.PROBE_FRESHNESS_SLO_DAYS = 31;
|
|
14
|
+
function probeFreshness(asOfDates, now = new Date()) {
|
|
15
|
+
const parsed = asOfDates
|
|
16
|
+
.filter((d) => typeof d === "string" && d !== "")
|
|
17
|
+
.map((d) => new Date(d).getTime())
|
|
18
|
+
.filter((t) => Number.isFinite(t));
|
|
19
|
+
if (parsed.length === 0) {
|
|
20
|
+
// No probe data at all is its own kind of stale — banner with no date.
|
|
21
|
+
return { stale: true, newestAsOf: null, daysOld: null };
|
|
22
|
+
}
|
|
23
|
+
const newest = Math.max(...parsed);
|
|
24
|
+
const daysOld = Math.floor((now.getTime() - newest) / 86400000);
|
|
25
|
+
return {
|
|
26
|
+
stale: daysOld > exports.PROBE_FRESHNESS_SLO_DAYS,
|
|
27
|
+
newestAsOf: new Date(newest).toISOString(),
|
|
28
|
+
daysOld,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
/** The banner sentence (brand guide §7: plain, dated, never hidden). */
|
|
32
|
+
function stalenessSentence(freshness) {
|
|
33
|
+
if (freshness.newestAsOf === null) {
|
|
34
|
+
return "No probe data yet — nothing on this page has been verified.";
|
|
35
|
+
}
|
|
36
|
+
return (`This probe data exceeded its freshness window (${exports.PROBE_FRESHNESS_SLO_DAYS} days): ` +
|
|
37
|
+
`newest result ${freshness.newestAsOf.slice(0, 10)}, ${freshness.daysOld} days old. ` +
|
|
38
|
+
"The data below is still shown, clearly dated — a rerun is due.");
|
|
39
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hierarchical bootstrap CI — RATIFIED PER THE ERRATUM (stop2-architect-review
|
|
3
|
+
* §1): per rep, (1) resample fixtures within the subtype with replacement,
|
|
4
|
+
* then (2) ONE resample of each (model, fixture)'s samples, used consistently
|
|
5
|
+
* across ALL of that model's comparisons in the rep (NOT per model-pair — the
|
|
6
|
+
* archived prototype's per-pair form injects extra noise and is superseded).
|
|
7
|
+
* ≥200 reps; 2.5/97.5 percentiles. RNG is seeded and injectable so a fixed
|
|
8
|
+
* seed reproduces scores and tiers exactly.
|
|
9
|
+
*
|
|
10
|
+
* Any change to the resampling scheme re-opens STOP-2.
|
|
11
|
+
*/
|
|
12
|
+
import type { SubtypeSamples } from "./types";
|
|
13
|
+
export declare const BOOTSTRAP_REPS = 200;
|
|
14
|
+
export declare const CI_LO = 0.025;
|
|
15
|
+
export declare const CI_HI = 0.975;
|
|
16
|
+
/** Deterministic PRNG (mulberry32) — seedable, dependency-free. */
|
|
17
|
+
export declare function mulberry32(seed: number): () => number;
|
|
18
|
+
/**
|
|
19
|
+
* The per-rep resampling plan, exposed for the determinism test family (f):
|
|
20
|
+
* one fixture multiset draw + exactly ONE per-(model, fixture) sample
|
|
21
|
+
* resample, keyed so the same array object serves every comparison the model
|
|
22
|
+
* enters that rep.
|
|
23
|
+
*/
|
|
24
|
+
export declare function drawRep(samples: SubtypeSamples, fixtures: readonly string[], rng: () => number): {
|
|
25
|
+
readonly fixtureDraws: readonly string[];
|
|
26
|
+
readonly resampled: SubtypeSamples;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* Bootstrap CIs for one subtype's anchored scores. Returns modelKey →
|
|
30
|
+
* [2.5th, 97.5th] percentile of the rep RigIndex distribution. A rep where the
|
|
31
|
+
* basket cannot anchor (all basket members drew empty) is skipped — with the
|
|
32
|
+
* probe corpus every model has every fixture, so this is a degenerate-input
|
|
33
|
+
* guard, not a data path.
|
|
34
|
+
*/
|
|
35
|
+
export declare function bootstrapCis(samples: SubtypeSamples, basket: readonly string[], reps: number, rng: () => number): Record<string, readonly [number, number]>;
|
|
36
|
+
//# sourceMappingURL=bootstrap.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bootstrap.d.ts","sourceRoot":"","sources":["../../src/rigindex/bootstrap.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,OAAO,KAAK,EAAgB,cAAc,EAAE,MAAM,SAAS,CAAC;AAE5D,eAAO,MAAM,cAAc,MAAM,CAAC;AAClC,eAAO,MAAM,KAAK,QAAQ,CAAC;AAC3B,eAAO,MAAM,KAAK,QAAQ,CAAC;AAE3B,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,MAAM,CASrD;AAOD;;;;;GAKG;AACH,wBAAgB,OAAO,CACrB,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,GAAG,EAAE,MAAM,MAAM,GAChB;IACD,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;CACpC,CAYA;AAWD;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,cAAc,EACvB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,MAAM,GAChB,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAsB3C"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Hierarchical bootstrap CI — RATIFIED PER THE ERRATUM (stop2-architect-review
|
|
4
|
+
* §1): per rep, (1) resample fixtures within the subtype with replacement,
|
|
5
|
+
* then (2) ONE resample of each (model, fixture)'s samples, used consistently
|
|
6
|
+
* across ALL of that model's comparisons in the rep (NOT per model-pair — the
|
|
7
|
+
* archived prototype's per-pair form injects extra noise and is superseded).
|
|
8
|
+
* ≥200 reps; 2.5/97.5 percentiles. RNG is seeded and injectable so a fixed
|
|
9
|
+
* seed reproduces scores and tiers exactly.
|
|
10
|
+
*
|
|
11
|
+
* Any change to the resampling scheme re-opens STOP-2.
|
|
12
|
+
*/
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.CI_HI = exports.CI_LO = exports.BOOTSTRAP_REPS = void 0;
|
|
15
|
+
exports.mulberry32 = mulberry32;
|
|
16
|
+
exports.drawRep = drawRep;
|
|
17
|
+
exports.bootstrapCis = bootstrapCis;
|
|
18
|
+
const graph_1 = require("./graph");
|
|
19
|
+
const fit_1 = require("./fit");
|
|
20
|
+
exports.BOOTSTRAP_REPS = 200;
|
|
21
|
+
exports.CI_LO = 0.025;
|
|
22
|
+
exports.CI_HI = 0.975;
|
|
23
|
+
/** Deterministic PRNG (mulberry32) — seedable, dependency-free. */
|
|
24
|
+
function mulberry32(seed) {
|
|
25
|
+
let a = seed >>> 0;
|
|
26
|
+
return () => {
|
|
27
|
+
a = (a + 0x6d2b79f5) >>> 0;
|
|
28
|
+
let t = a;
|
|
29
|
+
t = Math.imul(t ^ (t >>> 15), t | 1);
|
|
30
|
+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
|
31
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
/** One with-replacement resample of `list`, same length. */
|
|
35
|
+
function resample(list, rng) {
|
|
36
|
+
return list.map(() => list[Math.floor(rng() * list.length)]);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* The per-rep resampling plan, exposed for the determinism test family (f):
|
|
40
|
+
* one fixture multiset draw + exactly ONE per-(model, fixture) sample
|
|
41
|
+
* resample, keyed so the same array object serves every comparison the model
|
|
42
|
+
* enters that rep.
|
|
43
|
+
*/
|
|
44
|
+
function drawRep(samples, fixtures, rng) {
|
|
45
|
+
const fixtureDraws = resample(fixtures, rng);
|
|
46
|
+
// ONE resample per (model, fixture) — iterate in stable sorted order so a
|
|
47
|
+
// fixed seed reproduces the plan exactly.
|
|
48
|
+
const resampled = {};
|
|
49
|
+
for (const model of Object.keys(samples).sort()) {
|
|
50
|
+
resampled[model] = {};
|
|
51
|
+
for (const fixtureId of Object.keys(samples[model]).sort()) {
|
|
52
|
+
resampled[model][fixtureId] = resample(samples[model][fixtureId], rng);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return { fixtureDraws, resampled };
|
|
56
|
+
}
|
|
57
|
+
function percentile(sorted, q) {
|
|
58
|
+
if (sorted.length === 0)
|
|
59
|
+
return NaN;
|
|
60
|
+
const idx = q * (sorted.length - 1);
|
|
61
|
+
const lo = Math.floor(idx);
|
|
62
|
+
const hi = Math.ceil(idx);
|
|
63
|
+
if (lo === hi)
|
|
64
|
+
return sorted[lo];
|
|
65
|
+
return sorted[lo] + (sorted[hi] - sorted[lo]) * (idx - lo);
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Bootstrap CIs for one subtype's anchored scores. Returns modelKey →
|
|
69
|
+
* [2.5th, 97.5th] percentile of the rep RigIndex distribution. A rep where the
|
|
70
|
+
* basket cannot anchor (all basket members drew empty) is skipped — with the
|
|
71
|
+
* probe corpus every model has every fixture, so this is a degenerate-input
|
|
72
|
+
* guard, not a data path.
|
|
73
|
+
*/
|
|
74
|
+
function bootstrapCis(samples, basket, reps, rng) {
|
|
75
|
+
const models = Object.keys(samples).sort();
|
|
76
|
+
const fixtures = [...new Set(models.flatMap((m) => Object.keys(samples[m])))].sort();
|
|
77
|
+
const draws = Object.fromEntries(models.map((m) => [m, []]));
|
|
78
|
+
for (let rep = 0; rep < reps; rep += 1) {
|
|
79
|
+
const { fixtureDraws, resampled } = drawRep(samples, fixtures, rng);
|
|
80
|
+
const graph = (0, graph_1.buildSubtypeGraph)(resampled, fixtureDraws);
|
|
81
|
+
const scores = (0, fit_1.anchorScores)((0, fit_1.fitBradleyTerry)(graph), basket);
|
|
82
|
+
if (scores === null)
|
|
83
|
+
continue;
|
|
84
|
+
for (const m of models) {
|
|
85
|
+
if (scores[m] !== undefined)
|
|
86
|
+
draws[m].push(scores[m]);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
const out = {};
|
|
90
|
+
for (const m of models) {
|
|
91
|
+
const sorted = [...draws[m]].sort((a, b) => a - b);
|
|
92
|
+
if (sorted.length === 0)
|
|
93
|
+
continue;
|
|
94
|
+
out[m] = [percentile(sorted, exports.CI_LO), percentile(sorted, exports.CI_HI)];
|
|
95
|
+
}
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bradley-Terry fit + frontier anchor — RATIFIED (stop2-architect-review §1):
|
|
3
|
+
* Hunter's MM algorithm, 400 iterations or convergence, symmetric 0.01
|
|
4
|
+
* pseudo-wins per ordered pair (keeps saturated all-tie graphs finite and the
|
|
5
|
+
* graph connected), geometric-mean-normalized per iteration. Anchor:
|
|
6
|
+
* RigIndex_i = 100 × p_i / GM(basket strengths) — scores >100 are expected
|
|
7
|
+
* ("above the frontier average").
|
|
8
|
+
*
|
|
9
|
+
* Any change to the solver, regularization, or anchor re-opens STOP-2.
|
|
10
|
+
*/
|
|
11
|
+
import type { SubtypeGraph } from "./types";
|
|
12
|
+
export declare const MM_MAX_ITERS = 400;
|
|
13
|
+
export declare const MM_CONVERGENCE = 1e-10;
|
|
14
|
+
export declare const PSEUDO_WINS = 0.01;
|
|
15
|
+
/** Fit BT strengths over the graph's models. Returns modelKey → p, GM-normalized
|
|
16
|
+
* to 1. Deterministic — no randomness in the fit. */
|
|
17
|
+
export declare function fitBradleyTerry(graph: SubtypeGraph): Record<string, number>;
|
|
18
|
+
/** Geometric mean of the basket members' strengths, over the members present
|
|
19
|
+
* in the fit. Null when NO basket member is in the graph — the subtype cannot
|
|
20
|
+
* be anchored (and must not publish a score). */
|
|
21
|
+
export declare function basketGeometricMean(strengths: Readonly<Record<string, number>>, basket: readonly string[]): number | null;
|
|
22
|
+
/** RigIndex_i = 100 × p_i / GM(basket) (review §1). Null when unanchorable. */
|
|
23
|
+
export declare function anchorScores(strengths: Readonly<Record<string, number>>, basket: readonly string[]): Record<string, number> | null;
|
|
24
|
+
//# sourceMappingURL=fit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fit.d.ts","sourceRoot":"","sources":["../../src/rigindex/fit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE5C,eAAO,MAAM,YAAY,MAAM,CAAC;AAChC,eAAO,MAAM,cAAc,QAAQ,CAAC;AACpC,eAAO,MAAM,WAAW,OAAO,CAAC;AAEhC;qDACqD;AACrD,wBAAgB,eAAe,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAqC3E;AAED;;iDAEiD;AACjD,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,EAC3C,MAAM,EAAE,SAAS,MAAM,EAAE,GACxB,MAAM,GAAG,IAAI,CAIf;AAED,+EAA+E;AAC/E,wBAAgB,YAAY,CAC1B,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,EAC3C,MAAM,EAAE,SAAS,MAAM,EAAE,GACxB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAM/B"}
|