modelrig 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/README.md +41 -11
- package/data/PROVENANCE.md +19 -0
- package/data/pricing-manual.json +22 -0
- package/data/pricing-map.json +46362 -0
- package/data/pricing-overlay.json +419 -0
- package/dist/adapters/anthropic.d.ts +50 -6
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +123 -29
- package/dist/adapters/build-adapters.d.ts +11 -0
- package/dist/adapters/build-adapters.d.ts.map +1 -0
- package/dist/adapters/build-adapters.js +45 -0
- package/dist/adapters/deepseek.d.ts +6 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +54 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +46 -6
- package/dist/adapters/grok.d.ts +9 -0
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +67 -3
- package/dist/adapters/openai-compat.d.ts +6 -0
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +58 -1
- package/dist/adapters/openai-stream.d.ts +58 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -0
- package/dist/adapters/openai-stream.js +120 -0
- package/dist/adapters/openai.d.ts +6 -0
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +60 -1
- package/dist/adapters/sampling.d.ts +28 -0
- package/dist/adapters/sampling.d.ts.map +1 -0
- package/dist/adapters/sampling.js +34 -0
- package/dist/adapters/tool-mapping.d.ts +92 -0
- package/dist/adapters/tool-mapping.d.ts.map +1 -0
- package/dist/adapters/tool-mapping.js +143 -0
- package/dist/artifacts/artifact.d.ts +137 -0
- package/dist/artifacts/artifact.d.ts.map +1 -0
- package/dist/artifacts/artifact.js +377 -0
- package/dist/artifacts/content-uploader.d.ts +87 -0
- package/dist/artifacts/content-uploader.d.ts.map +1 -0
- package/dist/artifacts/content-uploader.js +124 -0
- package/dist/artifacts/index.d.ts +63 -0
- package/dist/artifacts/index.d.ts.map +1 -0
- package/dist/artifacts/index.js +80 -0
- package/dist/artifacts/run-context.d.ts +129 -0
- package/dist/artifacts/run-context.d.ts.map +1 -0
- package/dist/artifacts/run-context.js +208 -0
- package/dist/artifacts/scrub.d.ts +60 -0
- package/dist/artifacts/scrub.d.ts.map +1 -0
- package/dist/artifacts/scrub.js +139 -0
- package/dist/artifacts/step.d.ts +26 -0
- package/dist/artifacts/step.d.ts.map +1 -0
- package/dist/artifacts/step.js +37 -0
- package/dist/artifacts/taint.d.ts +63 -0
- package/dist/artifacts/taint.d.ts.map +1 -0
- package/dist/artifacts/taint.js +76 -0
- package/dist/bakeoff/cache-sim.d.ts +99 -0
- package/dist/bakeoff/cache-sim.d.ts.map +1 -0
- package/dist/bakeoff/cache-sim.js +126 -0
- package/dist/bakeoff/eval-cases.d.ts +96 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases.js +105 -0
- package/dist/bakeoff/report.d.ts +41 -0
- package/dist/bakeoff/report.d.ts.map +1 -0
- package/dist/bakeoff/report.js +142 -0
- package/dist/bakeoff/samples.d.ts +15 -0
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +20 -0
- package/dist/cli.d.ts +1 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +134 -116
- package/dist/config.d.ts +100 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +104 -0
- package/dist/deprecations.d.ts +32 -0
- package/dist/deprecations.d.ts.map +1 -0
- package/dist/deprecations.js +59 -0
- package/dist/gateway/http-protocol.d.ts +175 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -0
- package/dist/gateway/http-protocol.js +105 -0
- package/dist/index.d.ts +105 -18
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +266 -138
- package/dist/init/finder.d.ts +15 -0
- package/dist/init/finder.d.ts.map +1 -1
- package/dist/init/finder.js +46 -0
- package/dist/init/init-cli.d.ts +10 -0
- package/dist/init/init-cli.d.ts.map +1 -0
- package/dist/init/init-cli.js +86 -0
- package/dist/init/observe-cli.d.ts +19 -0
- package/dist/init/observe-cli.d.ts.map +1 -0
- package/dist/init/observe-cli.js +73 -0
- package/dist/init/observe.d.ts +81 -0
- package/dist/init/observe.d.ts.map +1 -0
- package/dist/init/observe.js +245 -0
- package/dist/registry/build.d.ts +94 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +125 -4
- package/dist/registry/cache-policy.d.ts +94 -0
- package/dist/registry/cache-policy.d.ts.map +1 -0
- package/dist/registry/cache-policy.js +238 -0
- package/dist/registry/call-notes.d.ts.map +1 -1
- package/dist/registry/call-notes.js +40 -2
- package/dist/registry/effective-rate.d.ts +151 -0
- package/dist/registry/effective-rate.d.ts.map +1 -0
- package/dist/registry/effective-rate.js +156 -0
- package/dist/registry/facts.d.ts.map +1 -1
- package/dist/registry/facts.js +16 -0
- package/dist/registry/parity.d.ts +26 -0
- package/dist/registry/parity.d.ts.map +1 -1
- package/dist/registry/parity.js +43 -2
- package/dist/registry/pricing-shape.d.ts +113 -0
- package/dist/registry/pricing-shape.d.ts.map +1 -0
- package/dist/registry/pricing-shape.js +123 -0
- package/dist/registry/pricing.d.ts +25 -1
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +64 -6
- package/dist/rigindex/bootstrap.d.ts +36 -0
- package/dist/rigindex/bootstrap.d.ts.map +1 -0
- package/dist/rigindex/bootstrap.js +97 -0
- package/dist/rigindex/fit.d.ts +24 -0
- package/dist/rigindex/fit.d.ts.map +1 -0
- package/dist/rigindex/fit.js +76 -0
- package/dist/rigindex/gates.d.ts +52 -0
- package/dist/rigindex/gates.d.ts.map +1 -0
- package/dist/rigindex/gates.js +109 -0
- package/dist/rigindex/graph.d.ts +31 -0
- package/dist/rigindex/graph.d.ts.map +1 -0
- package/dist/rigindex/graph.js +120 -0
- package/dist/rigindex/index.d.ts +23 -0
- package/dist/rigindex/index.d.ts.map +1 -0
- package/dist/rigindex/index.js +112 -0
- package/dist/rigindex/types.d.ts +76 -0
- package/dist/rigindex/types.d.ts.map +1 -0
- package/dist/rigindex/types.js +8 -0
- package/dist/routes/cache-lint.d.ts +41 -0
- package/dist/routes/cache-lint.d.ts.map +1 -0
- package/dist/routes/cache-lint.js +81 -0
- package/dist/routes/loader.d.ts +37 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +138 -0
- package/dist/routes/resolved-bundle.d.ts +79 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -0
- package/dist/routes/resolved-bundle.js +148 -0
- package/dist/routes/sampling-config.d.ts +32 -0
- package/dist/routes/sampling-config.d.ts.map +1 -0
- package/dist/routes/sampling-config.js +91 -0
- package/dist/run/ajv.d.ts +22 -0
- package/dist/run/ajv.d.ts.map +1 -0
- package/dist/run/ajv.js +32 -0
- package/dist/run/cache-handle.d.ts +58 -0
- package/dist/run/cache-handle.d.ts.map +1 -0
- package/dist/run/cache-handle.js +54 -0
- package/dist/run/cache-stamp.d.ts +65 -0
- package/dist/run/cache-stamp.d.ts.map +1 -0
- package/dist/run/cache-stamp.js +0 -0
- package/dist/run/capture-write.d.ts +33 -0
- package/dist/run/capture-write.d.ts.map +1 -0
- package/dist/run/capture-write.js +55 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +1 -1
- package/dist/run/dispatch.d.ts +99 -0
- package/dist/run/dispatch.d.ts.map +1 -0
- package/dist/run/dispatch.js +191 -0
- package/dist/run/raw.d.ts +88 -0
- package/dist/run/raw.d.ts.map +1 -0
- package/dist/run/raw.js +286 -0
- package/dist/run/record-attempt.d.ts +40 -0
- package/dist/run/record-attempt.d.ts.map +1 -0
- package/dist/run/record-attempt.js +99 -0
- package/dist/run/resolve.d.ts +75 -2
- package/dist/run/resolve.d.ts.map +1 -1
- package/dist/run/resolve.js +126 -4
- package/dist/run/run.d.ts +14 -0
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +0 -0
- package/dist/run/stream.d.ts +49 -0
- package/dist/run/stream.d.ts.map +1 -0
- package/dist/run/stream.js +87 -0
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +6 -7
- package/dist/savings/cache-ledger.d.ts +108 -0
- package/dist/savings/cache-ledger.d.ts.map +1 -0
- package/dist/savings/cache-ledger.js +174 -0
- package/dist/savings/cache-reconcile.d.ts +64 -0
- package/dist/savings/cache-reconcile.d.ts.map +1 -0
- package/dist/savings/cache-reconcile.js +59 -0
- package/dist/savings/cache-stats.d.ts +63 -0
- package/dist/savings/cache-stats.d.ts.map +1 -0
- package/dist/savings/cache-stats.js +108 -0
- package/dist/savings/emit.d.ts +13 -0
- package/dist/savings/emit.d.ts.map +1 -1
- package/dist/savings/emit.js +25 -0
- package/dist/savings/swap-execute.d.ts +14 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +44 -1
- package/dist/savings/swaps-client.d.ts +14 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/savings/swaps-client.js +16 -2
- package/dist/settings/api-fetcher.d.ts +39 -0
- package/dist/settings/api-fetcher.d.ts.map +1 -0
- package/dist/settings/api-fetcher.js +92 -0
- package/dist/settings/index.d.ts +10 -0
- package/dist/settings/index.d.ts.map +1 -0
- package/dist/settings/index.js +15 -0
- package/dist/settings/task-settings.d.ts +91 -0
- package/dist/settings/task-settings.d.ts.map +1 -0
- package/dist/settings/task-settings.js +116 -0
- package/dist/status.d.ts +77 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +113 -0
- package/dist/telemetry/artifact-store.d.ts +48 -0
- package/dist/telemetry/artifact-store.d.ts.map +1 -0
- package/dist/telemetry/artifact-store.js +189 -0
- package/dist/telemetry/cache-miss-streak.d.ts +79 -0
- package/dist/telemetry/cache-miss-streak.d.ts.map +1 -0
- package/dist/telemetry/cache-miss-streak.js +99 -0
- package/dist/telemetry/capture.d.ts +3 -0
- package/dist/telemetry/capture.d.ts.map +1 -1
- package/dist/telemetry/capture.js +15 -8
- package/dist/telemetry/exporter-setup.d.ts +28 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -0
- package/dist/telemetry/exporter-setup.js +34 -0
- package/dist/telemetry/exporter.d.ts +75 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +324 -47
- package/dist/telemetry/health.d.ts +76 -0
- package/dist/telemetry/health.d.ts.map +1 -0
- package/dist/telemetry/health.js +92 -0
- package/dist/telemetry/ingest-protocol.d.ts +152 -3
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +204 -4
- package/dist/telemetry/load-error-report.d.ts +24 -0
- package/dist/telemetry/load-error-report.d.ts.map +1 -0
- package/dist/telemetry/load-error-report.js +50 -0
- package/dist/telemetry/otel-map.d.ts +121 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -0
- package/dist/telemetry/otel-map.js +224 -0
- package/dist/telemetry/otel-normalize.d.ts +104 -0
- package/dist/telemetry/otel-normalize.d.ts.map +1 -0
- package/dist/telemetry/otel-normalize.js +530 -0
- package/dist/telemetry/otlp-emitter.d.ts +52 -0
- package/dist/telemetry/otlp-emitter.d.ts.map +1 -0
- package/dist/telemetry/otlp-emitter.js +144 -0
- package/dist/telemetry/otlp-proto.d.ts +29 -0
- package/dist/telemetry/otlp-proto.d.ts.map +1 -0
- package/dist/telemetry/otlp-proto.js +266 -0
- package/dist/telemetry/sinks.d.ts +37 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +121 -30
- package/dist/telemetry/store.d.ts +28 -1
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +218 -3
- package/dist/tenancy/api-key-format.d.ts +15 -5
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +24 -5
- package/dist/types/config-teaching.d.ts +62 -0
- package/dist/types/config-teaching.d.ts.map +1 -0
- package/dist/types/config-teaching.js +183 -0
- package/dist/types/provider.d.ts +122 -1
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +21 -1
- package/dist/types/route.d.ts +79 -1
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +117 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts +12 -0
- package/dist/validate-cli.d.ts.map +1 -0
- package/dist/validate-cli.js +57 -0
- package/dist/verify/capture.d.ts +66 -0
- package/dist/verify/capture.d.ts.map +1 -0
- package/dist/verify/capture.js +180 -0
- package/dist/verify/cli-run.d.ts +23 -0
- package/dist/verify/cli-run.d.ts.map +1 -0
- package/dist/verify/cli-run.js +57 -0
- package/dist/verify/diff.d.ts +48 -0
- package/dist/verify/diff.d.ts.map +1 -0
- package/dist/verify/diff.js +157 -0
- package/dist/verify/index.d.ts +17 -0
- package/dist/verify/index.d.ts.map +1 -0
- package/dist/verify/index.js +24 -0
- package/dist/verify/report.d.ts +14 -0
- package/dist/verify/report.d.ts.map +1 -0
- package/dist/verify/report.js +88 -0
- package/dist/verify/types.d.ts +101 -0
- package/dist/verify/types.d.ts.map +1 -0
- package/dist/verify/types.js +17 -0
- package/dist/verify/verify.d.ts +29 -0
- package/dist/verify/verify.d.ts.map +1 -0
- package/dist/verify/verify.js +148 -0
- package/dist/watch/evaluate.d.ts +162 -3
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +61 -0
- package/dist/watch/monitor.d.ts +116 -0
- package/dist/watch/monitor.d.ts.map +1 -0
- package/dist/watch/monitor.js +199 -0
- package/dist/watch/proposals.d.ts +20 -4
- package/dist/watch/proposals.d.ts.map +1 -1
- package/dist/watch/proposals.js +31 -3
- package/dist/watch/run.d.ts +33 -0
- package/dist/watch/run.d.ts.map +1 -1
- package/dist/watch/run.js +39 -0
- package/package.json +7 -1
- package/registry/LICENSE +20 -0
- package/registry/leaderboard.json +3639 -0
- package/registry/registry.json +7263 -0
- package/dist/telemetry/supabase-exporter.d.ts +0 -64
- package/dist/telemetry/supabase-exporter.d.ts.map +0 -1
- package/dist/telemetry/supabase-exporter.js +0 -211
package/dist/registry/build.d.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
* AGGREGATES ONLY — no tags, no payloads, no fingerprint joins.
|
|
9
9
|
*/
|
|
10
10
|
import type { ModelFacts } from "./claims";
|
|
11
|
+
import type { ModelPricingV2 } from "./pricing-shape";
|
|
11
12
|
export interface DeclaredLayer {
|
|
12
13
|
readonly as_of: string;
|
|
13
14
|
readonly source: string;
|
|
@@ -16,6 +17,13 @@ export interface DeclaredLayer {
|
|
|
16
17
|
readonly output_usd_per_mtok: number;
|
|
17
18
|
readonly cached_input_usd_per_mtok: number | null;
|
|
18
19
|
} | null;
|
|
20
|
+
/**
|
|
21
|
+
* First-class structured pricing (pricing-first-class spec §2.2) — ADDITIVE
|
|
22
|
+
* and optional. Rides inside the existing `declared` jsonb; the flat `pricing`
|
|
23
|
+
* field above STAYS this pass (S3: retire it in a later change once every
|
|
24
|
+
* reader uses `pricing_v2`). `pricing_v2.base` always equals the flat field.
|
|
25
|
+
*/
|
|
26
|
+
readonly pricing_v2?: ModelPricingV2 | null;
|
|
19
27
|
readonly flags: {
|
|
20
28
|
readonly supports_response_schema: boolean | null;
|
|
21
29
|
readonly supports_web_search: boolean | null;
|
|
@@ -25,6 +33,31 @@ export interface DeclaredLayer {
|
|
|
25
33
|
readonly adapter_grounded_native: boolean | null;
|
|
26
34
|
};
|
|
27
35
|
}
|
|
36
|
+
/** Per-corpus (fixture-family) breakdown of a schema summary — demo-rig
|
|
37
|
+
* §11.3b: the leaderboard and registry surface which corpus produced a cell,
|
|
38
|
+
* so a model probed only by `demo-rig` fixtures is visibly that. The aggregate
|
|
39
|
+
* fields above stay the ranking basis (same whole corpus for every model);
|
|
40
|
+
* this splits them by provenance for honesty, not for per-model subsetting. */
|
|
41
|
+
export interface ProbedFamilySummary {
|
|
42
|
+
readonly samples: number;
|
|
43
|
+
readonly conform_rate: number | null;
|
|
44
|
+
readonly conform_ci95: readonly [number, number] | null;
|
|
45
|
+
readonly value_accuracy_mean: number | null;
|
|
46
|
+
/** Fraction of this subset's samples served via the NATIVE structured rung
|
|
47
|
+
* vs json_mode coaching (discriminating-fixtures follow-up 3a). Lets a hard
|
|
48
|
+
* cell read "coached, missed" rather than a flat fail: a model without probed
|
|
49
|
+
* structured_native is coached on every schema sample, so a strict-constraint
|
|
50
|
+
* miss is the coaching path's limit, not the model's. Null when no sample in
|
|
51
|
+
* the subset recorded a rung (pre-rung result files). */
|
|
52
|
+
readonly native_rung_rate: number | null;
|
|
53
|
+
/** Mean per-sample cost (USD) over this subset — the per-family analogue of
|
|
54
|
+
* the aggregate `mean_cost_usd` (task-type-leaderboards spec §2.1). Lets a
|
|
55
|
+
* per-family view compute its own effective-cost-of-conformance
|
|
56
|
+
* (`mean_cost_usd / conform_rate * 1000`) so the faceted "cheapest-conformant
|
|
57
|
+
* on <task type>" rank has a number to sort on. Optional/additive: absent on
|
|
58
|
+
* pre–task-type probed layers, so a consumer reads it behind a null guard. */
|
|
59
|
+
readonly mean_cost_usd?: number;
|
|
60
|
+
}
|
|
28
61
|
export interface ProbedSchemaSummary {
|
|
29
62
|
readonly samples: number;
|
|
30
63
|
readonly parse_rate: number;
|
|
@@ -33,6 +66,19 @@ export interface ProbedSchemaSummary {
|
|
|
33
66
|
readonly value_accuracy_mean: number | null;
|
|
34
67
|
readonly native_rung_rate: number | null;
|
|
35
68
|
readonly mean_cost_usd: number;
|
|
69
|
+
/** family → its own stats (demo-rig §11.3b). Absent on pre-family probed
|
|
70
|
+
* layers; present once a result file carries fixtureFamilies. */
|
|
71
|
+
readonly by_family?: Readonly<Record<string, ProbedFamilySummary>>;
|
|
72
|
+
/** difficulty → its own stats (discriminating-fixtures §3). Same shape as
|
|
73
|
+
* by_family; the "hard" bucket is the leaderboard's differentiating column.
|
|
74
|
+
* Absent on pre-difficulty probed layers; present once a result file carries
|
|
75
|
+
* fixtureDifficulties. */
|
|
76
|
+
readonly by_difficulty?: Readonly<Record<string, ProbedFamilySummary>>;
|
|
77
|
+
/** task subtype → its own stats (www-clarity §5.5 W16). Unlike family
|
|
78
|
+
* (provenance) these are REAL task types (extraction.tabular,
|
|
79
|
+
* classification, …), cross-corpus by design. No default bucket: an
|
|
80
|
+
* untagged fixture joins no subtype. Absent when nothing is tagged. */
|
|
81
|
+
readonly by_subtype?: Readonly<Record<string, ProbedFamilySummary>>;
|
|
36
82
|
}
|
|
37
83
|
export interface ProbedGroundingSummary {
|
|
38
84
|
readonly samples: number;
|
|
@@ -43,6 +89,12 @@ export interface ProbedCachingSummary {
|
|
|
43
89
|
readonly samples: number;
|
|
44
90
|
readonly cache_hit_rate: number;
|
|
45
91
|
readonly mean_cached_tokens: number;
|
|
92
|
+
/** Samples that passed the documented-path eligibility gate (prefix ≥ the
|
|
93
|
+
* provider's minimum cacheable length; typed skips excluded). The
|
|
94
|
+
* caching discrepancy rule requires this > 0 — a rate of 0 over zero
|
|
95
|
+
* eligible samples is a probe-coverage fact, not a capability finding.
|
|
96
|
+
* Absent in pre-2026-08-18 result files → treated as `samples`. */
|
|
97
|
+
readonly eligible_samples?: number;
|
|
46
98
|
}
|
|
47
99
|
export interface ProbedLayer {
|
|
48
100
|
readonly as_of: string;
|
|
@@ -59,6 +111,20 @@ export interface ObservedLayer {
|
|
|
59
111
|
/** TTFB p50 (Phase 4 WS3) — optional: aggregators populate it where
|
|
60
112
|
* ttfb_ms rows exist; older observed layers lack the field. */
|
|
61
113
|
readonly p50_ttfb_ms?: number | null;
|
|
114
|
+
/**
|
|
115
|
+
* Observed-health percentiles + error-rate (G-4, observed-health/spec §4.2).
|
|
116
|
+
* Optional and additive — older observed layers lack these fields; they are
|
|
117
|
+
* min-n gated to `null` on thin endpoints. Operational health for the console
|
|
118
|
+
* + routing ONLY, NEVER a RigIndex input (spec §4.4).
|
|
119
|
+
*/
|
|
120
|
+
readonly p90_latency_ms?: number | null;
|
|
121
|
+
readonly p99_latency_ms?: number | null;
|
|
122
|
+
readonly p90_ttfb_ms?: number | null;
|
|
123
|
+
readonly p99_ttfb_ms?: number | null;
|
|
124
|
+
/** failure_class non-null / total; null under the min-n gate. */
|
|
125
|
+
readonly error_rate?: number | null;
|
|
126
|
+
/** Sample count behind the health stats (drives the min-n gate + freshness). */
|
|
127
|
+
readonly sample_n?: number;
|
|
62
128
|
readonly served_tier_rates: Readonly<Record<string, number>>;
|
|
63
129
|
readonly cache_hit_rate: number | null;
|
|
64
130
|
readonly failure_class_rates: Readonly<Record<string, number>>;
|
|
@@ -92,6 +158,17 @@ export interface ProbeResultFile {
|
|
|
92
158
|
readonly class: "schema" | "grounding" | "caching";
|
|
93
159
|
readonly date: string;
|
|
94
160
|
readonly harnessVersion: string;
|
|
161
|
+
/** fixtureId → family (demo-rig §11.3b). Absent on pre-family result files;
|
|
162
|
+
* those fixtures roll up under the default "probe-suite" family. */
|
|
163
|
+
readonly fixtureFamilies?: Readonly<Record<string, string>>;
|
|
164
|
+
/** fixtureId → difficulty (discriminating-fixtures §3). Absent on
|
|
165
|
+
* pre-difficulty result files; those fixtures roll up under "standard". */
|
|
166
|
+
readonly fixtureDifficulties?: Readonly<Record<string, string>>;
|
|
167
|
+
/** fixtureId → task subtype (www-clarity §5.5 W16). Absent on pre-subtype
|
|
168
|
+
* result files — the registry builder may inject the mapping from the
|
|
169
|
+
* current fixture corpus (subtype is hash-excluded retro-taggable
|
|
170
|
+
* metadata). Fixtures absent from the map join no subtype bucket. */
|
|
171
|
+
readonly fixtureSubtypes?: Readonly<Record<string, string>>;
|
|
95
172
|
readonly raw: Readonly<Record<string, ReadonlyArray<{
|
|
96
173
|
readonly rung?: "native" | "json_mode";
|
|
97
174
|
readonly parseOk: boolean;
|
|
@@ -109,7 +186,23 @@ export declare function latestPerClass(results: readonly ProbeResultFile[]): Par
|
|
|
109
186
|
export declare function summarizeSchemaResult(result: ProbeResultFile): ProbedSchemaSummary;
|
|
110
187
|
export declare function summarizeGroundingResult(result: ProbeResultFile): ProbedGroundingSummary;
|
|
111
188
|
export declare function summarizeCachingResult(result: ProbeResultFile): ProbedCachingSummary;
|
|
112
|
-
|
|
189
|
+
/** Compose a model's SCHEMA result files into one synthetic file, so a cheap
|
|
190
|
+
* SUPPLEMENTAL run (new fixtures only) ADDS to a prior full run instead of
|
|
191
|
+
* replacing it — the default latest-per-file rule would drop every fixture the
|
|
192
|
+
* newest file omits. The latest schema file is the base; older files fill in
|
|
193
|
+
* fixtures the base lacks, newest date winning on any per-fixture collision.
|
|
194
|
+
* ONLY fixtures in `currentFixtureIds` (the live corpus) are pulled forward, so
|
|
195
|
+
* a RETIRED id sitting in an old file is never resurrected (the trap a naive
|
|
196
|
+
* union hits, e.g. demo-support-summarize double-counting summarize). */
|
|
197
|
+
export declare function mergeSchemaResults(results: readonly ProbeResultFile[], currentFixtureIds: ReadonlySet<string>): ProbeResultFile | undefined;
|
|
198
|
+
/**
|
|
199
|
+
* Build a model's probed layer. When `currentSchemaFixtureIds` is supplied, the
|
|
200
|
+
* schema class COMPOSES across the model's result files (a supplemental run adds
|
|
201
|
+
* fixtures to a prior full run) via mergeSchemaResults; grounding/caching stay
|
|
202
|
+
* latest-per-file (one file per cycle). Omitting the ids keeps the pure
|
|
203
|
+
* latest-per-file behaviour (back-compat for hermetic tests).
|
|
204
|
+
*/
|
|
205
|
+
export declare function buildProbedLayer(results: readonly ProbeResultFile[], currentSchemaFixtureIds?: ReadonlySet<string>): ProbedLayer | null;
|
|
113
206
|
/** Declared-vs-probed disagreement detection — plan §4 (the rule pair) plus
|
|
114
207
|
* the rung gap that motivated the whole exercise (DeepSeek json_object). */
|
|
115
208
|
export declare function detectDiscrepancies(declared: DeclaredLayer | null, probed: ProbedLayer | null): Discrepancy[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/registry/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/registry/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE;QAChB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;QACpC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;QACrC,QAAQ,CAAC,yBAAyB,EAAE,MAAM,GAAG,IAAI,CAAC;KACnD,GAAG,IAAI,CAAC;IACT;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,wBAAwB,EAAE,OAAO,GAAG,IAAI,CAAC;QAClD,QAAQ,CAAC,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;QAC7C,QAAQ,CAAC,uBAAuB,EAAE,OAAO,GAAG,IAAI,CAAC;QACjD,uEAAuE;QACvE,QAAQ,CAAC,yBAAyB,EAAE,OAAO,GAAG,IAAI,CAAC;QACnD,QAAQ,CAAC,uBAAuB,EAAE,OAAO,GAAG,IAAI,CAAC;KAClD,CAAC;CACH;AAED;;;;+EAI+E;AAC/E,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACxD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C;;;;;6DAKyD;IACzD,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC;;;;;kFAK8E;IAC9E,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACxD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;qEACiE;IACjE,QAAQ,CAAC,SAAS,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACnE;;;8BAG0B;IAC1B,QAAQ,CAAC,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACvE;;;2EAGuE;IACvE,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;CACrE;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC;;;;uEAImE;IACnE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,SAAS,EAAE,sBAAsB,GAAG,IAAI,CAAC;IAClD,QAAQ,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC;mEAC+D;IAC/D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC;;;;;OAKG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,iEAAiE;IACjE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7D,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,mBAAmB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAChE;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,SAAS,WAAW,EAAE,CAAC;IAC/C;kEAC8D;IAC9D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC;wEACoE;IACpE,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,CAAC;CAC3C;AAED;yEACyE;AACzE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACnD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;wEACoE;IACpE,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5D;+EAC2E;IAC3E,QAAQ,CAAC,mBAAmB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE;;;yEAGqE;IACrE,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5D,QAAQ,CAAC,GAAG,EAAE,QAAQ,CACpB,MAAM,CACJ,MAAM,EACN,aAAa,CAAC;QACZ,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;QACvC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;QAC1B,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;QACvC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,QAAQ,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;QAClC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QACrC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KACtC,CAAC,CACH,CACF,CAAC;CACH;AAqBD,wDAAwD;AACxD,wBAAgB,cAAc,CAC5B,OAAO,EAAE,SAAS,eAAe,EAAE,GAClC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,eAAe,CAAC,CAAC,CAS5D;AAyDD,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,eAAe,GAAG,mBAAmB,CAmClF;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,eAAe,GAAG,sBAAsB,CAQxF;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,eAAe,GAAG,oBAAoB,CAYpF;AAED;;;;;;;yEAOyE;AACzE,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,SAAS,eAAe,EAAE,EACnC,iBAAiB,EAAE,WAAW,CAAC,MAAM,CAAC,GACrC,eAAe,GAAG,SAAS,CA4B7B;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,SAAS,eAAe,EAAE,EACnC,uBAAuB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GAC5C,WAAW,GAAG,IAAI,CAmBpB;AAED;4EAC4E;AAC5E,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,aAAa,GAAG,IAAI,EAC9B,MAAM,EAAE,WAAW,GAAG,IAAI,GACzB,WAAW,EAAE,CAmFf;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,aAAa,CAAC;IACrB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;CACzC,CAAC,EACF,WAAW,EAAE,MAAM,GAClB,QAAQ,CAgBV;AAED;;qDAEqD;AACrD,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAG9F"}
|
package/dist/registry/build.js
CHANGED
|
@@ -13,6 +13,7 @@ exports.latestPerClass = latestPerClass;
|
|
|
13
13
|
exports.summarizeSchemaResult = summarizeSchemaResult;
|
|
14
14
|
exports.summarizeGroundingResult = summarizeGroundingResult;
|
|
15
15
|
exports.summarizeCachingResult = summarizeCachingResult;
|
|
16
|
+
exports.mergeSchemaResults = mergeSchemaResults;
|
|
16
17
|
exports.buildProbedLayer = buildProbedLayer;
|
|
17
18
|
exports.detectDiscrepancies = detectDiscrepancies;
|
|
18
19
|
exports.buildRegistry = buildRegistry;
|
|
@@ -48,6 +49,50 @@ function latestPerClass(results) {
|
|
|
48
49
|
}
|
|
49
50
|
return byClass;
|
|
50
51
|
}
|
|
52
|
+
const DEFAULT_FIXTURE_FAMILY = "probe-suite";
|
|
53
|
+
const DEFAULT_FIXTURE_DIFFICULTY = "standard";
|
|
54
|
+
/** Sentinel for fixtures with no subtype tag — dropped from by_subtype. */
|
|
55
|
+
const UNTAGGED_SUBTYPE = "__untagged__";
|
|
56
|
+
/** Group a result's schema samples by a per-fixture classification map
|
|
57
|
+
* (family or difficulty), defaulting fixtures the map omits to `fallback`.
|
|
58
|
+
* Shared by the by_family and by_difficulty rollups so both are computed the
|
|
59
|
+
* same way as the headline (via schemaFamilyStats). */
|
|
60
|
+
function rollupBy(result, classification, fallback) {
|
|
61
|
+
const buckets = new Map();
|
|
62
|
+
for (const [fixtureId, fixtureSamples] of Object.entries(result.raw)) {
|
|
63
|
+
const key = classification?.[fixtureId] ?? fallback;
|
|
64
|
+
const bucket = buckets.get(key) ?? [];
|
|
65
|
+
bucket.push(...fixtureSamples);
|
|
66
|
+
buckets.set(key, bucket);
|
|
67
|
+
}
|
|
68
|
+
const out = {};
|
|
69
|
+
for (const [key, samples] of [...buckets.entries()].sort()) {
|
|
70
|
+
out[key] = schemaFamilyStats(samples);
|
|
71
|
+
}
|
|
72
|
+
return out;
|
|
73
|
+
}
|
|
74
|
+
/** Conformance + value-accuracy stats over a set of schema samples — shared
|
|
75
|
+
* by the aggregate summary and each per-family sub-summary so a family cell is
|
|
76
|
+
* computed the same way as the headline. */
|
|
77
|
+
function schemaFamilyStats(samples) {
|
|
78
|
+
const judged = samples.filter((s) => s.schemaConform !== null);
|
|
79
|
+
const conforming = judged.filter((s) => s.schemaConform === true).length;
|
|
80
|
+
const accuracies = samples.map((s) => s.valueAccuracy).filter((a) => a !== null);
|
|
81
|
+
// Same native-rung computation as the aggregate — the serving path a subset
|
|
82
|
+
// was served through (native strict vs json_mode coaching), so a hard cell
|
|
83
|
+
// can say which it was (follow-up 3a).
|
|
84
|
+
const rungs = samples.filter((s) => s.rung !== undefined);
|
|
85
|
+
return {
|
|
86
|
+
samples: samples.length,
|
|
87
|
+
conform_rate: judged.length === 0 ? null : conforming / judged.length,
|
|
88
|
+
conform_ci95: judged.length === 0 ? null : wilson95(conforming, judged.length),
|
|
89
|
+
value_accuracy_mean: mean(accuracies),
|
|
90
|
+
native_rung_rate: rungs.length === 0 ? null : rungs.filter((s) => s.rung === "native").length / rungs.length,
|
|
91
|
+
// Per-subset mean cost, same computation as the aggregate — so a per-family
|
|
92
|
+
// view can derive its own effective-cost-of-conformance (spec §2.1).
|
|
93
|
+
mean_cost_usd: samples.length === 0 ? 0 : samples.reduce((a, s) => a + s.costUsd, 0) / samples.length,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
51
96
|
function summarizeSchemaResult(result) {
|
|
52
97
|
const samples = Object.values(result.raw).flat();
|
|
53
98
|
const n = samples.length;
|
|
@@ -57,6 +102,17 @@ function summarizeSchemaResult(result) {
|
|
|
57
102
|
const accuracies = samples
|
|
58
103
|
.map((s) => s.valueAccuracy)
|
|
59
104
|
.filter((a) => a !== null);
|
|
105
|
+
// Per-corpus rollup (demo-rig §11.3b) and per-difficulty rollup
|
|
106
|
+
// (discriminating-fixtures §3). Each groups the same samples by a different
|
|
107
|
+
// hash-excluded classification, so surfaces can say which corpus produced a
|
|
108
|
+
// cell and how a model does on the hard subset. The aggregate above stays the
|
|
109
|
+
// ranking basis (same whole corpus for every model).
|
|
110
|
+
const by_family = rollupBy(result, result.fixtureFamilies, DEFAULT_FIXTURE_FAMILY);
|
|
111
|
+
const by_difficulty = rollupBy(result, result.fixtureDifficulties, DEFAULT_FIXTURE_DIFFICULTY);
|
|
112
|
+
// Subtype rollup (W16): no default bucket — an untagged fixture belongs to
|
|
113
|
+
// no task type, so it rolls into a sentinel that is then dropped.
|
|
114
|
+
const by_subtype = rollupBy(result, result.fixtureSubtypes, UNTAGGED_SUBTYPE);
|
|
115
|
+
delete by_subtype[UNTAGGED_SUBTYPE];
|
|
60
116
|
return {
|
|
61
117
|
samples: n,
|
|
62
118
|
parse_rate: n === 0 ? 0 : samples.filter((s) => s.parseOk).length / n,
|
|
@@ -65,6 +121,9 @@ function summarizeSchemaResult(result) {
|
|
|
65
121
|
value_accuracy_mean: mean(accuracies),
|
|
66
122
|
native_rung_rate: rungs.length === 0 ? null : rungs.filter((s) => s.rung === "native").length / rungs.length,
|
|
67
123
|
mean_cost_usd: n === 0 ? 0 : samples.reduce((a, s) => a + s.costUsd, 0) / n,
|
|
124
|
+
...(Object.keys(by_family).length > 0 ? { by_family } : {}),
|
|
125
|
+
...(Object.keys(by_difficulty).length > 0 ? { by_difficulty } : {}),
|
|
126
|
+
...(Object.keys(by_subtype).length > 0 ? { by_subtype } : {}),
|
|
68
127
|
};
|
|
69
128
|
}
|
|
70
129
|
function summarizeGroundingResult(result) {
|
|
@@ -78,18 +137,73 @@ function summarizeGroundingResult(result) {
|
|
|
78
137
|
}
|
|
79
138
|
function summarizeCachingResult(result) {
|
|
80
139
|
const samples = Object.values(result.raw).flat();
|
|
140
|
+
// failureClass null = the paired call ran AND passed the documented-path
|
|
141
|
+
// eligibility gate (cache_prefix_below_minimum is a typed skip upstream).
|
|
81
142
|
const okSamples = samples.filter((s) => s.failureClass === null);
|
|
82
143
|
const n = okSamples.length;
|
|
83
144
|
return {
|
|
84
145
|
samples: samples.length,
|
|
85
146
|
cache_hit_rate: n === 0 ? 0 : okSamples.filter((s) => (s.cachedTokens ?? 0) > 0).length / n,
|
|
86
147
|
mean_cached_tokens: mean(okSamples.map((s) => s.cachedTokens ?? 0)) ?? 0,
|
|
148
|
+
eligible_samples: n,
|
|
87
149
|
};
|
|
88
150
|
}
|
|
89
|
-
|
|
151
|
+
/** Compose a model's SCHEMA result files into one synthetic file, so a cheap
|
|
152
|
+
* SUPPLEMENTAL run (new fixtures only) ADDS to a prior full run instead of
|
|
153
|
+
* replacing it — the default latest-per-file rule would drop every fixture the
|
|
154
|
+
* newest file omits. The latest schema file is the base; older files fill in
|
|
155
|
+
* fixtures the base lacks, newest date winning on any per-fixture collision.
|
|
156
|
+
* ONLY fixtures in `currentFixtureIds` (the live corpus) are pulled forward, so
|
|
157
|
+
* a RETIRED id sitting in an old file is never resurrected (the trap a naive
|
|
158
|
+
* union hits, e.g. demo-support-summarize double-counting summarize). */
|
|
159
|
+
function mergeSchemaResults(results, currentFixtureIds) {
|
|
160
|
+
const schemaFiles = results
|
|
161
|
+
.filter((r) => r.class === "schema")
|
|
162
|
+
.sort((a, b) => (a.date > b.date ? -1 : a.date < b.date ? 1 : 0)); // newest first
|
|
163
|
+
const base = schemaFiles[0];
|
|
164
|
+
if (base === undefined)
|
|
165
|
+
return undefined;
|
|
166
|
+
const raw = { ...base.raw };
|
|
167
|
+
const fixtureSubtypes = { ...(base.fixtureSubtypes ?? {}) };
|
|
168
|
+
const fixtureFamilies = { ...(base.fixtureFamilies ?? {}) };
|
|
169
|
+
const fixtureDifficulties = { ...(base.fixtureDifficulties ?? {}) };
|
|
170
|
+
for (const older of schemaFiles.slice(1)) {
|
|
171
|
+
for (const [fid, samples] of Object.entries(older.raw)) {
|
|
172
|
+
if (fid in raw)
|
|
173
|
+
continue; // a newer file already carries it — newest wins
|
|
174
|
+
if (!currentFixtureIds.has(fid))
|
|
175
|
+
continue; // retired id — never resurrect
|
|
176
|
+
raw[fid] = samples;
|
|
177
|
+
if (older.fixtureSubtypes?.[fid] !== undefined)
|
|
178
|
+
fixtureSubtypes[fid] = older.fixtureSubtypes[fid];
|
|
179
|
+
if (older.fixtureFamilies?.[fid] !== undefined)
|
|
180
|
+
fixtureFamilies[fid] = older.fixtureFamilies[fid];
|
|
181
|
+
if (older.fixtureDifficulties?.[fid] !== undefined)
|
|
182
|
+
fixtureDifficulties[fid] = older.fixtureDifficulties[fid];
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return {
|
|
186
|
+
...base,
|
|
187
|
+
raw,
|
|
188
|
+
...(Object.keys(fixtureSubtypes).length > 0 ? { fixtureSubtypes } : {}),
|
|
189
|
+
...(Object.keys(fixtureFamilies).length > 0 ? { fixtureFamilies } : {}),
|
|
190
|
+
...(Object.keys(fixtureDifficulties).length > 0 ? { fixtureDifficulties } : {}),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Build a model's probed layer. When `currentSchemaFixtureIds` is supplied, the
|
|
195
|
+
* schema class COMPOSES across the model's result files (a supplemental run adds
|
|
196
|
+
* fixtures to a prior full run) via mergeSchemaResults; grounding/caching stay
|
|
197
|
+
* latest-per-file (one file per cycle). Omitting the ids keeps the pure
|
|
198
|
+
* latest-per-file behaviour (back-compat for hermetic tests).
|
|
199
|
+
*/
|
|
200
|
+
function buildProbedLayer(results, currentSchemaFixtureIds) {
|
|
90
201
|
if (results.length === 0)
|
|
91
202
|
return null;
|
|
92
203
|
const byClass = latestPerClass(results);
|
|
204
|
+
const schemaFile = currentSchemaFixtureIds !== undefined
|
|
205
|
+
? mergeSchemaResults(results, currentSchemaFixtureIds)
|
|
206
|
+
: byClass.schema;
|
|
93
207
|
const dates = results.map((r) => r.date).sort();
|
|
94
208
|
const versions = results.map((r) => r.harnessVersion).sort();
|
|
95
209
|
const newest = dates[dates.length - 1];
|
|
@@ -97,7 +211,7 @@ function buildProbedLayer(results) {
|
|
|
97
211
|
return {
|
|
98
212
|
as_of: newest,
|
|
99
213
|
source: `modelrig-probes@${version}`,
|
|
100
|
-
schema:
|
|
214
|
+
schema: schemaFile !== undefined ? summarizeSchemaResult(schemaFile) : null,
|
|
101
215
|
grounding: byClass.grounding !== undefined ? summarizeGroundingResult(byClass.grounding) : null,
|
|
102
216
|
caching: byClass.caching !== undefined ? summarizeCachingResult(byClass.caching) : null,
|
|
103
217
|
};
|
|
@@ -156,10 +270,17 @@ function detectDiscrepancies(declared, probed) {
|
|
|
156
270
|
}
|
|
157
271
|
const caching = probed.caching;
|
|
158
272
|
if (caching !== null && caching.samples > 0) {
|
|
159
|
-
|
|
273
|
+
// Guarded (launch-runbook A1, 2026-08-18): fire only when at least one
|
|
274
|
+
// sample passed the documented-path eligibility gate — a zero rate over
|
|
275
|
+
// zero eligible samples means the probe couldn't measure, not that the
|
|
276
|
+
// capability is unrealized. Pre-gate result files lack the field and
|
|
277
|
+
// keep the old behavior via the ?? fallback.
|
|
278
|
+
if (declared.flags.supports_prompt_caching === true &&
|
|
279
|
+
caching.cache_hit_rate === 0 &&
|
|
280
|
+
(caching.eligible_samples ?? caching.samples) > 0) {
|
|
160
281
|
out.push({
|
|
161
282
|
kind: "declared_caching_unrealized",
|
|
162
|
-
message: "declared supports_prompt_caching, but no probed repeat call reported cached tokens",
|
|
283
|
+
message: "declared supports_prompt_caching, but no eligible probed repeat call on the documented caching path reported cached tokens",
|
|
163
284
|
});
|
|
164
285
|
}
|
|
165
286
|
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cache Policy Registry — cache-aware-routing spec §3, [Briefing §7].
|
|
3
|
+
*
|
|
4
|
+
* Per-provider cache SEMANTICS as versioned DATA, never routing code: TTLs,
|
|
5
|
+
* refresh-on-read, minimum-cacheable prefix, key mechanism, telemetry dialect,
|
|
6
|
+
* endpoint affinity. A provider rule change (OpenAI shipping paid writes,
|
|
7
|
+
* Anthropic GA-ing the 1h TTL) becomes a data edit here with a new `as_of`, not
|
|
8
|
+
* a code change anywhere else — the whole point of the registry.
|
|
9
|
+
*
|
|
10
|
+
* TWO HOMES, both pre-existing (spec §3):
|
|
11
|
+
* • RATES ride pricing — `ModelPricingV2.base.{cache_read,cache_write,
|
|
12
|
+
* cache_write_1h}` and flat `ModelPricing.cacheWrite*UsdPerTok`, entered via
|
|
13
|
+
* the pricing sweep with source+as_of. NOT here.
|
|
14
|
+
* • POLICY (this file) is the non-price semantics.
|
|
15
|
+
*
|
|
16
|
+
* L5 (fail-closed): every field cites a first-party page (research §2.1, fetched
|
|
17
|
+
* 2026-08-18). A value we could not verify is `null` (or `cache_supported:
|
|
18
|
+
* false`) — never a guess. `getCachePolicy` returns a fail-closed
|
|
19
|
+
* `cache_supported: false` policy for any provider not seeded, identical posture
|
|
20
|
+
* to the zero-coverage gates.
|
|
21
|
+
*
|
|
22
|
+
* ── CROSS-REFERENCES (no import — the probes package is standalone since P2) ──
|
|
23
|
+
* • `min_cacheable_tokens` is reconciled by hand against the probes package's
|
|
24
|
+
* `MIN_CACHEABLE_TOKENS` (packages/modelrig-probes/src/caching.ts, PR #157),
|
|
25
|
+
* which cites the same first-party pages. See MIN_CACHEABLE_RECONCILIATION
|
|
26
|
+
* below for the value-by-value comparison and the two deliberate deltas
|
|
27
|
+
* (anthropic, gemini) flagged for the architect to converge if desired. A
|
|
28
|
+
* reciprocal comment lives in caching.ts.
|
|
29
|
+
* • OBSERVED cache facts (`cache_hit_rate`, `mean_cached_tokens`,
|
|
30
|
+
* `declared_caching_unrealized`) are materialized separately in
|
|
31
|
+
* `src/registry/build.ts` from probe/production telemetry — the DECLARED
|
|
32
|
+
* policy here is the counterpart truth, and the §5.6 streak alert (WS-C)
|
|
33
|
+
* cross-references `declared_caching_unrealized` (same smell, other plane).
|
|
34
|
+
*/
|
|
35
|
+
import type { ProviderId } from "../types/provider";
|
|
36
|
+
/**
|
|
37
|
+
* Per-provider cache semantics (spec §3). Additive to the registry's declared
|
|
38
|
+
* layer; every field is first-party-sourced or `null` (L5).
|
|
39
|
+
*/
|
|
40
|
+
export interface CachePolicy {
|
|
41
|
+
/** Whether the provider caches on the path ModelRig uses. false ⇒ every
|
|
42
|
+
* other field is moot (fail-closed). */
|
|
43
|
+
readonly cache_supported: boolean;
|
|
44
|
+
/** How caching is engaged. `explicit_breakpoints` = caller marks blocks
|
|
45
|
+
* (Anthropic cache_control); `automatic_prefix` = provider caches the longest
|
|
46
|
+
* matching prefix with no request change; `key_opt_in` = caller supplies a
|
|
47
|
+
* routing key (Mistral-style — no ModelRig provider uses it in v1). */
|
|
48
|
+
readonly mode: "explicit_breakpoints" | "automatic_prefix" | "key_opt_in";
|
|
49
|
+
/** Default cache lifetime, seconds. null ⇒ undocumented / best-effort. */
|
|
50
|
+
readonly default_ttl_seconds: number | null;
|
|
51
|
+
/** Extended-TTL option, seconds (Anthropic 1h write tier; OpenAI 24h
|
|
52
|
+
* retention). null ⇒ none published. */
|
|
53
|
+
readonly extended_ttl_seconds: number | null;
|
|
54
|
+
/** Whether a cache READ resets the TTL (Anthropic 5m: true). Conservative
|
|
55
|
+
* `false` when undocumented — under-claiming warmth never over-predicts a hit. */
|
|
56
|
+
readonly ttl_refresh_on_read: boolean;
|
|
57
|
+
/** Minimum prefix (tokens) below which caching silently does not engage.
|
|
58
|
+
* null ⇒ no documented minimum (L5). See MIN_CACHEABLE_RECONCILIATION. */
|
|
59
|
+
readonly min_cacheable_tokens: number | null;
|
|
60
|
+
/** The request parameter that engages/keys caching, or null for pure
|
|
61
|
+
* automatic caching with no caller control. */
|
|
62
|
+
readonly cache_key_param: "prompt_cache_key" | "cache_control" | "cachedContent" | null;
|
|
63
|
+
/** Whether cache-READ tokens are observable on the path ModelRig reads
|
|
64
|
+
* (usage JSON). false where they ride HTTP headers we do not yet lift. */
|
|
65
|
+
readonly reports_read_tokens: boolean;
|
|
66
|
+
/** Whether cache-WRITE (creation) tokens are separately reported. */
|
|
67
|
+
readonly reports_write_tokens: boolean;
|
|
68
|
+
/** Which usage shape the adapter normalizes from. */
|
|
69
|
+
readonly usage_dialect: "anthropic" | "openai_details" | "deepseek_partition" | "gemini_metadata" | "fireworks_headers";
|
|
70
|
+
/** true where the warm cache is endpoint-sticky and follow-ups must land on
|
|
71
|
+
* the same endpoint (research: OpenAI ~15 req/min/key; Fireworks
|
|
72
|
+
* x-session-affinity). */
|
|
73
|
+
readonly provider_affinity_required: boolean;
|
|
74
|
+
/** Provenance — same rule as pricing (first-party page). */
|
|
75
|
+
readonly source: string;
|
|
76
|
+
readonly as_of: string;
|
|
77
|
+
readonly policy_version: string;
|
|
78
|
+
}
|
|
79
|
+
/** The fail-closed policy for any provider not seeded (spec §3): caching is
|
|
80
|
+
* assumed OFF until a first-party source says otherwise — identical posture to
|
|
81
|
+
* the zero-coverage gates. `usage_dialect` defaults to the OpenAI shape only as
|
|
82
|
+
* a type-satisfying placeholder; `cache_supported: false` means no consumer
|
|
83
|
+
* reads it. */
|
|
84
|
+
export declare function failClosedCachePolicy(): CachePolicy;
|
|
85
|
+
/**
|
|
86
|
+
* The cache policy for a provider — the seeded first-party record, or a
|
|
87
|
+
* fail-closed `cache_supported: false` policy when the provider is not seeded
|
|
88
|
+
* (spec §3). Provider-level in v1; model-specific overrides land with the
|
|
89
|
+
* conversational-gateway phase.
|
|
90
|
+
*/
|
|
91
|
+
export declare function getCachePolicy(provider: ProviderId): CachePolicy;
|
|
92
|
+
/** Every seeded provider — for tests and the console policy surface (WS-D). */
|
|
93
|
+
export declare function seededCachePolicyProviders(): readonly ProviderId[];
|
|
94
|
+
//# sourceMappingURL=cache-policy.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache-policy.d.ts","sourceRoot":"","sources":["../../src/registry/cache-policy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B;4CACwC;IACxC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;IAClC;;;2EAGuE;IACvE,QAAQ,CAAC,IAAI,EAAE,sBAAsB,GAAG,kBAAkB,GAAG,YAAY,CAAC;IAC1E,0EAA0E;IAC1E,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C;4CACwC;IACxC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C;sFACkF;IAClF,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACtC;8EAC0E;IAC1E,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C;mDAC+C;IAC/C,QAAQ,CAAC,eAAe,EAAE,kBAAkB,GAAG,eAAe,GAAG,eAAe,GAAG,IAAI,CAAC;IACxF;8EAC0E;IAC1E,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACtC,qEAAqE;IACrE,QAAQ,CAAC,oBAAoB,EAAE,OAAO,CAAC;IACvC,qDAAqD;IACrD,QAAQ,CAAC,aAAa,EAClB,WAAW,GACX,gBAAgB,GAChB,oBAAoB,GACpB,iBAAiB,GACjB,mBAAmB,CAAC;IACxB;;8BAE0B;IAC1B,QAAQ,CAAC,0BAA0B,EAAE,OAAO,CAAC;IAC7C,4DAA4D;IAC5D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AA8KD;;;;eAIe;AACf,wBAAgB,qBAAqB,IAAI,WAAW,CAiBnD;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,UAAU,GAAG,WAAW,CAEhE;AAED,+EAA+E;AAC/E,wBAAgB,0BAA0B,IAAI,SAAS,UAAU,EAAE,CAElE"}
|