modelrig 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (310) hide show
  1. package/CHANGELOG.md +158 -0
  2. package/README.md +53 -11
  3. package/data/PROVENANCE.md +19 -0
  4. package/data/pricing-manual.json +22 -0
  5. package/data/pricing-map.json +46362 -0
  6. package/data/pricing-overlay.json +419 -0
  7. package/dist/adapters/anthropic.d.ts +50 -6
  8. package/dist/adapters/anthropic.d.ts.map +1 -1
  9. package/dist/adapters/anthropic.js +123 -29
  10. package/dist/adapters/build-adapters.d.ts +11 -0
  11. package/dist/adapters/build-adapters.d.ts.map +1 -0
  12. package/dist/adapters/build-adapters.js +45 -0
  13. package/dist/adapters/deepseek.d.ts +6 -0
  14. package/dist/adapters/deepseek.d.ts.map +1 -1
  15. package/dist/adapters/deepseek.js +54 -1
  16. package/dist/adapters/gemini.d.ts +10 -0
  17. package/dist/adapters/gemini.d.ts.map +1 -1
  18. package/dist/adapters/gemini.js +78 -7
  19. package/dist/adapters/grok.d.ts +9 -0
  20. package/dist/adapters/grok.d.ts.map +1 -1
  21. package/dist/adapters/grok.js +67 -3
  22. package/dist/adapters/openai-compat.d.ts +6 -0
  23. package/dist/adapters/openai-compat.d.ts.map +1 -1
  24. package/dist/adapters/openai-compat.js +58 -1
  25. package/dist/adapters/openai-stream.d.ts +58 -0
  26. package/dist/adapters/openai-stream.d.ts.map +1 -0
  27. package/dist/adapters/openai-stream.js +120 -0
  28. package/dist/adapters/openai.d.ts +6 -0
  29. package/dist/adapters/openai.d.ts.map +1 -1
  30. package/dist/adapters/openai.js +60 -1
  31. package/dist/adapters/sampling.d.ts +28 -0
  32. package/dist/adapters/sampling.d.ts.map +1 -0
  33. package/dist/adapters/sampling.js +34 -0
  34. package/dist/adapters/tool-mapping.d.ts +92 -0
  35. package/dist/adapters/tool-mapping.d.ts.map +1 -0
  36. package/dist/adapters/tool-mapping.js +143 -0
  37. package/dist/artifacts/artifact.d.ts +137 -0
  38. package/dist/artifacts/artifact.d.ts.map +1 -0
  39. package/dist/artifacts/artifact.js +377 -0
  40. package/dist/artifacts/content-uploader.d.ts +87 -0
  41. package/dist/artifacts/content-uploader.d.ts.map +1 -0
  42. package/dist/artifacts/content-uploader.js +124 -0
  43. package/dist/artifacts/index.d.ts +63 -0
  44. package/dist/artifacts/index.d.ts.map +1 -0
  45. package/dist/artifacts/index.js +80 -0
  46. package/dist/artifacts/run-context.d.ts +129 -0
  47. package/dist/artifacts/run-context.d.ts.map +1 -0
  48. package/dist/artifacts/run-context.js +208 -0
  49. package/dist/artifacts/scrub.d.ts +60 -0
  50. package/dist/artifacts/scrub.d.ts.map +1 -0
  51. package/dist/artifacts/scrub.js +139 -0
  52. package/dist/artifacts/step.d.ts +26 -0
  53. package/dist/artifacts/step.d.ts.map +1 -0
  54. package/dist/artifacts/step.js +37 -0
  55. package/dist/artifacts/taint.d.ts +63 -0
  56. package/dist/artifacts/taint.d.ts.map +1 -0
  57. package/dist/artifacts/taint.js +76 -0
  58. package/dist/bakeoff/cache-sim.d.ts +99 -0
  59. package/dist/bakeoff/cache-sim.d.ts.map +1 -0
  60. package/dist/bakeoff/cache-sim.js +126 -0
  61. package/dist/bakeoff/eval-cases.d.ts +96 -0
  62. package/dist/bakeoff/eval-cases.d.ts.map +1 -0
  63. package/dist/bakeoff/eval-cases.js +105 -0
  64. package/dist/bakeoff/report.d.ts +41 -0
  65. package/dist/bakeoff/report.d.ts.map +1 -0
  66. package/dist/bakeoff/report.js +142 -0
  67. package/dist/bakeoff/samples.d.ts +15 -0
  68. package/dist/bakeoff/samples.d.ts.map +1 -1
  69. package/dist/bakeoff/samples.js +20 -0
  70. package/dist/cli.d.ts +1 -3
  71. package/dist/cli.d.ts.map +1 -1
  72. package/dist/cli.js +134 -116
  73. package/dist/config.d.ts +100 -0
  74. package/dist/config.d.ts.map +1 -1
  75. package/dist/config.js +104 -0
  76. package/dist/deprecations.d.ts +32 -0
  77. package/dist/deprecations.d.ts.map +1 -0
  78. package/dist/deprecations.js +59 -0
  79. package/dist/gateway/http-protocol.d.ts +175 -0
  80. package/dist/gateway/http-protocol.d.ts.map +1 -0
  81. package/dist/gateway/http-protocol.js +105 -0
  82. package/dist/index.d.ts +106 -19
  83. package/dist/index.d.ts.map +1 -1
  84. package/dist/index.js +266 -138
  85. package/dist/init/finder.d.ts +15 -0
  86. package/dist/init/finder.d.ts.map +1 -1
  87. package/dist/init/finder.js +46 -0
  88. package/dist/init/init-cli.d.ts +10 -0
  89. package/dist/init/init-cli.d.ts.map +1 -0
  90. package/dist/init/init-cli.js +86 -0
  91. package/dist/init/observe-cli.d.ts +19 -0
  92. package/dist/init/observe-cli.d.ts.map +1 -0
  93. package/dist/init/observe-cli.js +73 -0
  94. package/dist/init/observe.d.ts +81 -0
  95. package/dist/init/observe.d.ts.map +1 -0
  96. package/dist/init/observe.js +245 -0
  97. package/dist/registry/build.d.ts +94 -1
  98. package/dist/registry/build.d.ts.map +1 -1
  99. package/dist/registry/build.js +125 -4
  100. package/dist/registry/cache-policy.d.ts +94 -0
  101. package/dist/registry/cache-policy.d.ts.map +1 -0
  102. package/dist/registry/cache-policy.js +238 -0
  103. package/dist/registry/call-notes.d.ts.map +1 -1
  104. package/dist/registry/call-notes.js +40 -2
  105. package/dist/registry/effective-rate.d.ts +151 -0
  106. package/dist/registry/effective-rate.d.ts.map +1 -0
  107. package/dist/registry/effective-rate.js +156 -0
  108. package/dist/registry/facts.d.ts.map +1 -1
  109. package/dist/registry/facts.js +16 -0
  110. package/dist/registry/parity.d.ts +26 -0
  111. package/dist/registry/parity.d.ts.map +1 -1
  112. package/dist/registry/parity.js +43 -2
  113. package/dist/registry/pricing-shape.d.ts +113 -0
  114. package/dist/registry/pricing-shape.d.ts.map +1 -0
  115. package/dist/registry/pricing-shape.js +123 -0
  116. package/dist/registry/pricing.d.ts +25 -1
  117. package/dist/registry/pricing.d.ts.map +1 -1
  118. package/dist/registry/pricing.js +64 -6
  119. package/dist/rigindex/bootstrap.d.ts +36 -0
  120. package/dist/rigindex/bootstrap.d.ts.map +1 -0
  121. package/dist/rigindex/bootstrap.js +97 -0
  122. package/dist/rigindex/fit.d.ts +24 -0
  123. package/dist/rigindex/fit.d.ts.map +1 -0
  124. package/dist/rigindex/fit.js +76 -0
  125. package/dist/rigindex/gates.d.ts +52 -0
  126. package/dist/rigindex/gates.d.ts.map +1 -0
  127. package/dist/rigindex/gates.js +109 -0
  128. package/dist/rigindex/graph.d.ts +31 -0
  129. package/dist/rigindex/graph.d.ts.map +1 -0
  130. package/dist/rigindex/graph.js +120 -0
  131. package/dist/rigindex/index.d.ts +23 -0
  132. package/dist/rigindex/index.d.ts.map +1 -0
  133. package/dist/rigindex/index.js +112 -0
  134. package/dist/rigindex/types.d.ts +76 -0
  135. package/dist/rigindex/types.d.ts.map +1 -0
  136. package/dist/rigindex/types.js +8 -0
  137. package/dist/routes/cache-lint.d.ts +41 -0
  138. package/dist/routes/cache-lint.d.ts.map +1 -0
  139. package/dist/routes/cache-lint.js +81 -0
  140. package/dist/routes/loader.d.ts +48 -0
  141. package/dist/routes/loader.d.ts.map +1 -1
  142. package/dist/routes/loader.js +167 -0
  143. package/dist/routes/resolved-bundle.d.ts +79 -0
  144. package/dist/routes/resolved-bundle.d.ts.map +1 -0
  145. package/dist/routes/resolved-bundle.js +148 -0
  146. package/dist/routes/sampling-config.d.ts +32 -0
  147. package/dist/routes/sampling-config.d.ts.map +1 -0
  148. package/dist/routes/sampling-config.js +91 -0
  149. package/dist/run/ajv.d.ts +22 -0
  150. package/dist/run/ajv.d.ts.map +1 -0
  151. package/dist/run/ajv.js +32 -0
  152. package/dist/run/cache-handle.d.ts +58 -0
  153. package/dist/run/cache-handle.d.ts.map +1 -0
  154. package/dist/run/cache-handle.js +54 -0
  155. package/dist/run/cache-stamp.d.ts +65 -0
  156. package/dist/run/cache-stamp.d.ts.map +1 -0
  157. package/dist/run/cache-stamp.js +0 -0
  158. package/dist/run/capture-write.d.ts +33 -0
  159. package/dist/run/capture-write.d.ts.map +1 -0
  160. package/dist/run/capture-write.js +55 -0
  161. package/dist/run/cost.d.ts.map +1 -1
  162. package/dist/run/cost.js +1 -1
  163. package/dist/run/dispatch.d.ts +99 -0
  164. package/dist/run/dispatch.d.ts.map +1 -0
  165. package/dist/run/dispatch.js +191 -0
  166. package/dist/run/raw.d.ts +130 -0
  167. package/dist/run/raw.d.ts.map +1 -0
  168. package/dist/run/raw.js +363 -0
  169. package/dist/run/record-attempt.d.ts +40 -0
  170. package/dist/run/record-attempt.d.ts.map +1 -0
  171. package/dist/run/record-attempt.js +99 -0
  172. package/dist/run/resolve.d.ts +75 -2
  173. package/dist/run/resolve.d.ts.map +1 -1
  174. package/dist/run/resolve.js +126 -4
  175. package/dist/run/run.d.ts +14 -0
  176. package/dist/run/run.d.ts.map +1 -1
  177. package/dist/run/run.js +0 -0
  178. package/dist/run/stream.d.ts +49 -0
  179. package/dist/run/stream.d.ts.map +1 -0
  180. package/dist/run/stream.js +87 -0
  181. package/dist/run/validate.d.ts.map +1 -1
  182. package/dist/run/validate.js +6 -7
  183. package/dist/savings/cache-ledger.d.ts +108 -0
  184. package/dist/savings/cache-ledger.d.ts.map +1 -0
  185. package/dist/savings/cache-ledger.js +174 -0
  186. package/dist/savings/cache-reconcile.d.ts +64 -0
  187. package/dist/savings/cache-reconcile.d.ts.map +1 -0
  188. package/dist/savings/cache-reconcile.js +59 -0
  189. package/dist/savings/cache-stats.d.ts +63 -0
  190. package/dist/savings/cache-stats.d.ts.map +1 -0
  191. package/dist/savings/cache-stats.js +108 -0
  192. package/dist/savings/emit.d.ts +13 -0
  193. package/dist/savings/emit.d.ts.map +1 -1
  194. package/dist/savings/emit.js +25 -0
  195. package/dist/savings/swap-execute.d.ts +14 -0
  196. package/dist/savings/swap-execute.d.ts.map +1 -1
  197. package/dist/savings/swap-execute.js +44 -1
  198. package/dist/savings/swaps-client.d.ts +14 -0
  199. package/dist/savings/swaps-client.d.ts.map +1 -1
  200. package/dist/savings/swaps-client.js +16 -2
  201. package/dist/settings/api-fetcher.d.ts +39 -0
  202. package/dist/settings/api-fetcher.d.ts.map +1 -0
  203. package/dist/settings/api-fetcher.js +92 -0
  204. package/dist/settings/index.d.ts +10 -0
  205. package/dist/settings/index.d.ts.map +1 -0
  206. package/dist/settings/index.js +15 -0
  207. package/dist/settings/task-settings.d.ts +91 -0
  208. package/dist/settings/task-settings.d.ts.map +1 -0
  209. package/dist/settings/task-settings.js +116 -0
  210. package/dist/status.d.ts +77 -0
  211. package/dist/status.d.ts.map +1 -0
  212. package/dist/status.js +113 -0
  213. package/dist/telemetry/artifact-store.d.ts +48 -0
  214. package/dist/telemetry/artifact-store.d.ts.map +1 -0
  215. package/dist/telemetry/artifact-store.js +189 -0
  216. package/dist/telemetry/cache-miss-streak.d.ts +79 -0
  217. package/dist/telemetry/cache-miss-streak.d.ts.map +1 -0
  218. package/dist/telemetry/cache-miss-streak.js +99 -0
  219. package/dist/telemetry/capture.d.ts +3 -0
  220. package/dist/telemetry/capture.d.ts.map +1 -1
  221. package/dist/telemetry/capture.js +15 -8
  222. package/dist/telemetry/exporter-setup.d.ts +28 -0
  223. package/dist/telemetry/exporter-setup.d.ts.map +1 -0
  224. package/dist/telemetry/exporter-setup.js +34 -0
  225. package/dist/telemetry/exporter.d.ts +75 -0
  226. package/dist/telemetry/exporter.d.ts.map +1 -1
  227. package/dist/telemetry/exporter.js +324 -47
  228. package/dist/telemetry/health.d.ts +76 -0
  229. package/dist/telemetry/health.d.ts.map +1 -0
  230. package/dist/telemetry/health.js +92 -0
  231. package/dist/telemetry/ingest-protocol.d.ts +152 -3
  232. package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
  233. package/dist/telemetry/ingest-protocol.js +204 -4
  234. package/dist/telemetry/load-error-report.d.ts +24 -0
  235. package/dist/telemetry/load-error-report.d.ts.map +1 -0
  236. package/dist/telemetry/load-error-report.js +50 -0
  237. package/dist/telemetry/otel-map.d.ts +121 -0
  238. package/dist/telemetry/otel-map.d.ts.map +1 -0
  239. package/dist/telemetry/otel-map.js +224 -0
  240. package/dist/telemetry/otel-normalize.d.ts +104 -0
  241. package/dist/telemetry/otel-normalize.d.ts.map +1 -0
  242. package/dist/telemetry/otel-normalize.js +530 -0
  243. package/dist/telemetry/otlp-emitter.d.ts +52 -0
  244. package/dist/telemetry/otlp-emitter.d.ts.map +1 -0
  245. package/dist/telemetry/otlp-emitter.js +144 -0
  246. package/dist/telemetry/otlp-proto.d.ts +29 -0
  247. package/dist/telemetry/otlp-proto.d.ts.map +1 -0
  248. package/dist/telemetry/otlp-proto.js +266 -0
  249. package/dist/telemetry/sinks.d.ts +37 -1
  250. package/dist/telemetry/sinks.d.ts.map +1 -1
  251. package/dist/telemetry/sinks.js +121 -30
  252. package/dist/telemetry/store.d.ts +28 -1
  253. package/dist/telemetry/store.d.ts.map +1 -1
  254. package/dist/telemetry/store.js +218 -3
  255. package/dist/tenancy/api-key-format.d.ts +15 -5
  256. package/dist/tenancy/api-key-format.d.ts.map +1 -1
  257. package/dist/tenancy/api-key-format.js +24 -5
  258. package/dist/types/config-teaching.d.ts +62 -0
  259. package/dist/types/config-teaching.d.ts.map +1 -0
  260. package/dist/types/config-teaching.js +183 -0
  261. package/dist/types/provider.d.ts +147 -1
  262. package/dist/types/provider.d.ts.map +1 -1
  263. package/dist/types/provider.js +21 -1
  264. package/dist/types/route.d.ts +79 -1
  265. package/dist/types/route.d.ts.map +1 -1
  266. package/dist/types/telemetry.d.ts +117 -0
  267. package/dist/types/telemetry.d.ts.map +1 -1
  268. package/dist/validate-cli.d.ts +12 -0
  269. package/dist/validate-cli.d.ts.map +1 -0
  270. package/dist/validate-cli.js +57 -0
  271. package/dist/verify/capture.d.ts +66 -0
  272. package/dist/verify/capture.d.ts.map +1 -0
  273. package/dist/verify/capture.js +180 -0
  274. package/dist/verify/cli-run.d.ts +23 -0
  275. package/dist/verify/cli-run.d.ts.map +1 -0
  276. package/dist/verify/cli-run.js +57 -0
  277. package/dist/verify/diff.d.ts +48 -0
  278. package/dist/verify/diff.d.ts.map +1 -0
  279. package/dist/verify/diff.js +157 -0
  280. package/dist/verify/index.d.ts +17 -0
  281. package/dist/verify/index.d.ts.map +1 -0
  282. package/dist/verify/index.js +24 -0
  283. package/dist/verify/report.d.ts +14 -0
  284. package/dist/verify/report.d.ts.map +1 -0
  285. package/dist/verify/report.js +88 -0
  286. package/dist/verify/types.d.ts +101 -0
  287. package/dist/verify/types.d.ts.map +1 -0
  288. package/dist/verify/types.js +17 -0
  289. package/dist/verify/verify.d.ts +29 -0
  290. package/dist/verify/verify.d.ts.map +1 -0
  291. package/dist/verify/verify.js +148 -0
  292. package/dist/watch/evaluate.d.ts +162 -3
  293. package/dist/watch/evaluate.d.ts.map +1 -1
  294. package/dist/watch/evaluate.js +61 -0
  295. package/dist/watch/monitor.d.ts +116 -0
  296. package/dist/watch/monitor.d.ts.map +1 -0
  297. package/dist/watch/monitor.js +199 -0
  298. package/dist/watch/proposals.d.ts +20 -4
  299. package/dist/watch/proposals.d.ts.map +1 -1
  300. package/dist/watch/proposals.js +31 -3
  301. package/dist/watch/run.d.ts +33 -0
  302. package/dist/watch/run.d.ts.map +1 -1
  303. package/dist/watch/run.js +39 -0
  304. package/package.json +7 -1
  305. package/registry/LICENSE +20 -0
  306. package/registry/leaderboard.json +3639 -0
  307. package/registry/registry.json +7263 -0
  308. package/dist/telemetry/supabase-exporter.d.ts +0 -64
  309. package/dist/telemetry/supabase-exporter.d.ts.map +0 -1
  310. package/dist/telemetry/supabase-exporter.js +0 -211
@@ -8,6 +8,7 @@
8
8
  * AGGREGATES ONLY — no tags, no payloads, no fingerprint joins.
9
9
  */
10
10
  import type { ModelFacts } from "./claims";
11
+ import type { ModelPricingV2 } from "./pricing-shape";
11
12
  export interface DeclaredLayer {
12
13
  readonly as_of: string;
13
14
  readonly source: string;
@@ -16,6 +17,13 @@ export interface DeclaredLayer {
16
17
  readonly output_usd_per_mtok: number;
17
18
  readonly cached_input_usd_per_mtok: number | null;
18
19
  } | null;
20
+ /**
21
+ * First-class structured pricing (pricing-first-class spec §2.2) — ADDITIVE
22
+ * and optional. Rides inside the existing `declared` jsonb; the flat `pricing`
23
+ * field above STAYS this pass (S3: retire it in a later change once every
24
+ * reader uses `pricing_v2`). `pricing_v2.base` always equals the flat field.
25
+ */
26
+ readonly pricing_v2?: ModelPricingV2 | null;
19
27
  readonly flags: {
20
28
  readonly supports_response_schema: boolean | null;
21
29
  readonly supports_web_search: boolean | null;
@@ -25,6 +33,31 @@ export interface DeclaredLayer {
25
33
  readonly adapter_grounded_native: boolean | null;
26
34
  };
27
35
  }
36
+ /** Per-corpus (fixture-family) breakdown of a schema summary — demo-rig
37
+ * §11.3b: the leaderboard and registry surface which corpus produced a cell,
38
+ * so a model probed only by `demo-rig` fixtures is visibly that. The aggregate
39
+ * fields above stay the ranking basis (same whole corpus for every model);
40
+ * this splits them by provenance for honesty, not for per-model subsetting. */
41
+ export interface ProbedFamilySummary {
42
+ readonly samples: number;
43
+ readonly conform_rate: number | null;
44
+ readonly conform_ci95: readonly [number, number] | null;
45
+ readonly value_accuracy_mean: number | null;
46
+ /** Fraction of this subset's samples served via the NATIVE structured rung
47
+ * vs json_mode coaching (discriminating-fixtures follow-up 3a). Lets a hard
48
+ * cell read "coached, missed" rather than a flat fail: a model without probed
49
+ * structured_native is coached on every schema sample, so a strict-constraint
50
+ * miss is the coaching path's limit, not the model's. Null when no sample in
51
+ * the subset recorded a rung (pre-rung result files). */
52
+ readonly native_rung_rate: number | null;
53
+ /** Mean per-sample cost (USD) over this subset — the per-family analogue of
54
+ * the aggregate `mean_cost_usd` (task-type-leaderboards spec §2.1). Lets a
55
+ * per-family view compute its own effective-cost-of-conformance
56
+ * (`mean_cost_usd / conform_rate * 1000`) so the faceted "cheapest-conformant
57
+ * on <task type>" rank has a number to sort on. Optional/additive: absent on
58
+ * pre–task-type probed layers, so a consumer reads it behind a null guard. */
59
+ readonly mean_cost_usd?: number;
60
+ }
28
61
  export interface ProbedSchemaSummary {
29
62
  readonly samples: number;
30
63
  readonly parse_rate: number;
@@ -33,6 +66,19 @@ export interface ProbedSchemaSummary {
33
66
  readonly value_accuracy_mean: number | null;
34
67
  readonly native_rung_rate: number | null;
35
68
  readonly mean_cost_usd: number;
69
+ /** family → its own stats (demo-rig §11.3b). Absent on pre-family probed
70
+ * layers; present once a result file carries fixtureFamilies. */
71
+ readonly by_family?: Readonly<Record<string, ProbedFamilySummary>>;
72
+ /** difficulty → its own stats (discriminating-fixtures §3). Same shape as
73
+ * by_family; the "hard" bucket is the leaderboard's differentiating column.
74
+ * Absent on pre-difficulty probed layers; present once a result file carries
75
+ * fixtureDifficulties. */
76
+ readonly by_difficulty?: Readonly<Record<string, ProbedFamilySummary>>;
77
+ /** task subtype → its own stats (www-clarity §5.5 W16). Unlike family
78
+ * (provenance) these are REAL task types (extraction.tabular,
79
+ * classification, …), cross-corpus by design. No default bucket: an
80
+ * untagged fixture joins no subtype. Absent when nothing is tagged. */
81
+ readonly by_subtype?: Readonly<Record<string, ProbedFamilySummary>>;
36
82
  }
37
83
  export interface ProbedGroundingSummary {
38
84
  readonly samples: number;
@@ -43,6 +89,12 @@ export interface ProbedCachingSummary {
43
89
  readonly samples: number;
44
90
  readonly cache_hit_rate: number;
45
91
  readonly mean_cached_tokens: number;
92
+ /** Samples that passed the documented-path eligibility gate (prefix ≥ the
93
+ * provider's minimum cacheable length; typed skips excluded). The
94
+ * caching discrepancy rule requires this > 0 — a rate of 0 over zero
95
+ * eligible samples is a probe-coverage fact, not a capability finding.
96
+ * Absent in pre-2026-08-18 result files → treated as `samples`. */
97
+ readonly eligible_samples?: number;
46
98
  }
47
99
  export interface ProbedLayer {
48
100
  readonly as_of: string;
@@ -59,6 +111,20 @@ export interface ObservedLayer {
59
111
  /** TTFB p50 (Phase 4 WS3) — optional: aggregators populate it where
60
112
  * ttfb_ms rows exist; older observed layers lack the field. */
61
113
  readonly p50_ttfb_ms?: number | null;
114
+ /**
115
+ * Observed-health percentiles + error-rate (G-4, observed-health/spec §4.2).
116
+ * Optional and additive — older observed layers lack these fields; they are
117
+ * min-n gated to `null` on thin endpoints. Operational health for the console
118
+ * + routing ONLY, NEVER a RigIndex input (spec §4.4).
119
+ */
120
+ readonly p90_latency_ms?: number | null;
121
+ readonly p99_latency_ms?: number | null;
122
+ readonly p90_ttfb_ms?: number | null;
123
+ readonly p99_ttfb_ms?: number | null;
124
+ /** failure_class non-null / total; null under the min-n gate. */
125
+ readonly error_rate?: number | null;
126
+ /** Sample count behind the health stats (drives the min-n gate + freshness). */
127
+ readonly sample_n?: number;
62
128
  readonly served_tier_rates: Readonly<Record<string, number>>;
63
129
  readonly cache_hit_rate: number | null;
64
130
  readonly failure_class_rates: Readonly<Record<string, number>>;
@@ -92,6 +158,17 @@ export interface ProbeResultFile {
92
158
  readonly class: "schema" | "grounding" | "caching";
93
159
  readonly date: string;
94
160
  readonly harnessVersion: string;
161
+ /** fixtureId → family (demo-rig §11.3b). Absent on pre-family result files;
162
+ * those fixtures roll up under the default "probe-suite" family. */
163
+ readonly fixtureFamilies?: Readonly<Record<string, string>>;
164
+ /** fixtureId → difficulty (discriminating-fixtures §3). Absent on
165
+ * pre-difficulty result files; those fixtures roll up under "standard". */
166
+ readonly fixtureDifficulties?: Readonly<Record<string, string>>;
167
+ /** fixtureId → task subtype (www-clarity §5.5 W16). Absent on pre-subtype
168
+ * result files — the registry builder may inject the mapping from the
169
+ * current fixture corpus (subtype is hash-excluded retro-taggable
170
+ * metadata). Fixtures absent from the map join no subtype bucket. */
171
+ readonly fixtureSubtypes?: Readonly<Record<string, string>>;
95
172
  readonly raw: Readonly<Record<string, ReadonlyArray<{
96
173
  readonly rung?: "native" | "json_mode";
97
174
  readonly parseOk: boolean;
@@ -109,7 +186,23 @@ export declare function latestPerClass(results: readonly ProbeResultFile[]): Par
109
186
  export declare function summarizeSchemaResult(result: ProbeResultFile): ProbedSchemaSummary;
110
187
  export declare function summarizeGroundingResult(result: ProbeResultFile): ProbedGroundingSummary;
111
188
  export declare function summarizeCachingResult(result: ProbeResultFile): ProbedCachingSummary;
112
- export declare function buildProbedLayer(results: readonly ProbeResultFile[]): ProbedLayer | null;
189
+ /** Compose a model's SCHEMA result files into one synthetic file, so a cheap
190
+ * SUPPLEMENTAL run (new fixtures only) ADDS to a prior full run instead of
191
+ * replacing it — the default latest-per-file rule would drop every fixture the
192
+ * newest file omits. The latest schema file is the base; older files fill in
193
+ * fixtures the base lacks, newest date winning on any per-fixture collision.
194
+ * ONLY fixtures in `currentFixtureIds` (the live corpus) are pulled forward, so
195
+ * a RETIRED id sitting in an old file is never resurrected (the trap a naive
196
+ * union hits, e.g. demo-support-summarize double-counting summarize). */
197
+ export declare function mergeSchemaResults(results: readonly ProbeResultFile[], currentFixtureIds: ReadonlySet<string>): ProbeResultFile | undefined;
198
+ /**
199
+ * Build a model's probed layer. When `currentSchemaFixtureIds` is supplied, the
200
+ * schema class COMPOSES across the model's result files (a supplemental run adds
201
+ * fixtures to a prior full run) via mergeSchemaResults; grounding/caching stay
202
+ * latest-per-file (one file per cycle). Omitting the ids keeps the pure
203
+ * latest-per-file behaviour (back-compat for hermetic tests).
204
+ */
205
+ export declare function buildProbedLayer(results: readonly ProbeResultFile[], currentSchemaFixtureIds?: ReadonlySet<string>): ProbedLayer | null;
113
206
  /** Declared-vs-probed disagreement detection — plan §4 (the rule pair) plus
114
207
  * the rung gap that motivated the whole exercise (DeepSeek json_object). */
115
208
  export declare function detectDiscrepancies(declared: DeclaredLayer | null, probed: ProbedLayer | null): Discrepancy[];
@@ -1 +1 @@
1
- {"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/registry/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE3C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE;QAChB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;QACpC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;QACrC,QAAQ,CAAC,yBAAyB,EAAE,MAAM,GAAG,IAAI,CAAC;KACnD,GAAG,IAAI,CAAC;IACT,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,wBAAwB,EAAE,OAAO,GAAG,IAAI,CAAC;QAClD,QAAQ,CAAC,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;QAC7C,QAAQ,CAAC,uBAAuB,EAAE,OAAO,GAAG,IAAI,CAAC;QACjD,uEAAuE;QACvE,QAAQ,CAAC,yBAAyB,EAAE,OAAO,GAAG,IAAI,CAAC;QACnD,QAAQ,CAAC,uBAAuB,EAAE,OAAO,GAAG,IAAI,CAAC;KAClD,CAAC;CACH;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACxD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,SAAS,EAAE,sBAAsB,GAAG,IAAI,CAAC;IAClD,QAAQ,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC;mEAC+D;IAC/D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7D,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,mBAAmB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAChE;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,SAAS,WAAW,EAAE,CAAC;IAC/C;kEAC8D;IAC9D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC;wEACoE;IACpE,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,CAAC;CAC3C;AAED;yEACyE;AACzE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACnD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CACpB,MAAM,CACJ,MAAM,EACN,aAAa,CAAC;QACZ,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;QACvC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;QAC1B,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;QACvC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,QAAQ,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;QAClC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QACrC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KACtC,CAAC,CACH,CACF,CAAC;CACH;AAqBD,wDAAwD;AACxD,wBAAgB,cAAc,CAC5B,OAAO,EAAE,SAAS,eAAe,EAAE,GAClC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,eAAe,CAAC,CAAC,CAS5D;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,eAAe,GAAG,mBAAmB,CAmBlF;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,eAAe,GAAG,sBAAsB,CAQxF;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,eAAe,GAAG,oBAAoB,CASpF;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,GAAG,WAAW,GAAG,IAAI,CAexF;AAED;4EAC4E;AAC5E,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,aAAa,GAAG,IAAI,EAC9B,MAAM,EAAE,WAAW,GAAG,IAAI,GACzB,WAAW,EAAE,CA0Ef;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,aAAa,CAAC;IACrB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;CACzC,CAAC,EACF,WAAW,EAAE,MAAM,GAClB,QAAQ,CAgBV;AAED;;qDAEqD;AACrD,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAG9F"}
1
+ {"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/registry/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE;QAChB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;QACpC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;QACrC,QAAQ,CAAC,yBAAyB,EAAE,MAAM,GAAG,IAAI,CAAC;KACnD,GAAG,IAAI,CAAC;IACT;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,wBAAwB,EAAE,OAAO,GAAG,IAAI,CAAC;QAClD,QAAQ,CAAC,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;QAC7C,QAAQ,CAAC,uBAAuB,EAAE,OAAO,GAAG,IAAI,CAAC;QACjD,uEAAuE;QACvE,QAAQ,CAAC,yBAAyB,EAAE,OAAO,GAAG,IAAI,CAAC;QACnD,QAAQ,CAAC,uBAAuB,EAAE,OAAO,GAAG,IAAI,CAAC;KAClD,CAAC;CACH;AAED;;;;+EAI+E;AAC/E,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACxD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C;;;;;6DAKyD;IACzD,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC;;;;;kFAK8E;IAC9E,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACxD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;qEACiE;IACjE,QAAQ,CAAC,SAAS,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACnE;;;8BAG0B;IAC1B,QAAQ,CAAC,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACvE;;;2EAGuE;IACvE,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;CACrE;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC;;;;uEAImE;IACnE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,SAAS,EAAE,sBAAsB,GAAG,IAAI,CAAC;IAClD,QAAQ,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC;mEAC+D;IAC/D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC;;;;;OAKG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,iEAAiE;IACjE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7D,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,mBAAmB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAChE;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,SAAS,WAAW,EAAE,CAAC;IAC/C;kEAC8D;IAC9D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC;wEACoE;IACpE,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,CAAC;CAC3C;AAED;yEACyE;AACzE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACnD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;wEACoE;IACpE,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5D;+EAC2E;IAC3E,QAAQ,CAAC,mBAAmB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE;;;yEAGqE;IACrE,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5D,QAAQ,CAAC,GAAG,EAAE,QAAQ,CACpB,MAAM,CACJ,MAAM,EACN,aAAa,CAAC;QACZ,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;QACvC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;QAC1B,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;QACvC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,QAAQ,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;QAClC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QACrC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KACtC,CAAC,CACH,CACF,CAAC;CACH;AAqBD,wDAAwD;AACxD,wBAAgB,cAAc,CAC5B,OAAO,EAAE,SAAS,eAAe,EAAE,GAClC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,eAAe,CAAC,CAAC,CAS5D;AAyDD,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,eAAe,GAAG,mBAAmB,CAmClF;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,eAAe,GAAG,sBAAsB,CAQxF;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,eAAe,GAAG,oBAAoB,CAYpF;AAED;;;;;;;yEAOyE;AACzE,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,SAAS,eAAe,EAAE,EACnC,iBAAiB,EAAE,WAAW,CAAC,MAAM,CAAC,GACrC,eAAe,GAAG,SAAS,CA4B7B;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,SAAS,eAAe,EAAE,EACnC,uBAAuB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GAC5C,WAAW,GAAG,IAAI,CAmBpB;AAED;4EAC4E;AAC5E,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,aAAa,GAAG,IAAI,EAC9B,MAAM,EAAE,WAAW,GAAG,IAAI,GACzB,WAAW,EAAE,CAmFf;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,aAAa,CAAC;IACrB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;CACzC,CAAC,EACF,WAAW,EAAE,MAAM,GAClB,QAAQ,CAgBV;AAED;;qDAEqD;AACrD,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAG9F"}
@@ -13,6 +13,7 @@ exports.latestPerClass = latestPerClass;
13
13
  exports.summarizeSchemaResult = summarizeSchemaResult;
14
14
  exports.summarizeGroundingResult = summarizeGroundingResult;
15
15
  exports.summarizeCachingResult = summarizeCachingResult;
16
+ exports.mergeSchemaResults = mergeSchemaResults;
16
17
  exports.buildProbedLayer = buildProbedLayer;
17
18
  exports.detectDiscrepancies = detectDiscrepancies;
18
19
  exports.buildRegistry = buildRegistry;
@@ -48,6 +49,50 @@ function latestPerClass(results) {
48
49
  }
49
50
  return byClass;
50
51
  }
52
+ const DEFAULT_FIXTURE_FAMILY = "probe-suite";
53
+ const DEFAULT_FIXTURE_DIFFICULTY = "standard";
54
+ /** Sentinel for fixtures with no subtype tag — dropped from by_subtype. */
55
+ const UNTAGGED_SUBTYPE = "__untagged__";
56
+ /** Group a result's schema samples by a per-fixture classification map
57
+ * (family or difficulty), defaulting fixtures the map omits to `fallback`.
58
+ * Shared by the by_family and by_difficulty rollups so both are computed the
59
+ * same way as the headline (via schemaFamilyStats). */
60
+ function rollupBy(result, classification, fallback) {
61
+ const buckets = new Map();
62
+ for (const [fixtureId, fixtureSamples] of Object.entries(result.raw)) {
63
+ const key = classification?.[fixtureId] ?? fallback;
64
+ const bucket = buckets.get(key) ?? [];
65
+ bucket.push(...fixtureSamples);
66
+ buckets.set(key, bucket);
67
+ }
68
+ const out = {};
69
+ for (const [key, samples] of [...buckets.entries()].sort()) {
70
+ out[key] = schemaFamilyStats(samples);
71
+ }
72
+ return out;
73
+ }
74
+ /** Conformance + value-accuracy stats over a set of schema samples — shared
75
+ * by the aggregate summary and each per-family sub-summary so a family cell is
76
+ * computed the same way as the headline. */
77
+ function schemaFamilyStats(samples) {
78
+ const judged = samples.filter((s) => s.schemaConform !== null);
79
+ const conforming = judged.filter((s) => s.schemaConform === true).length;
80
+ const accuracies = samples.map((s) => s.valueAccuracy).filter((a) => a !== null);
81
+ // Same native-rung computation as the aggregate — the serving path a subset
82
+ // was served through (native strict vs json_mode coaching), so a hard cell
83
+ // can say which it was (follow-up 3a).
84
+ const rungs = samples.filter((s) => s.rung !== undefined);
85
+ return {
86
+ samples: samples.length,
87
+ conform_rate: judged.length === 0 ? null : conforming / judged.length,
88
+ conform_ci95: judged.length === 0 ? null : wilson95(conforming, judged.length),
89
+ value_accuracy_mean: mean(accuracies),
90
+ native_rung_rate: rungs.length === 0 ? null : rungs.filter((s) => s.rung === "native").length / rungs.length,
91
+ // Per-subset mean cost, same computation as the aggregate — so a per-family
92
+ // view can derive its own effective-cost-of-conformance (spec §2.1).
93
+ mean_cost_usd: samples.length === 0 ? 0 : samples.reduce((a, s) => a + s.costUsd, 0) / samples.length,
94
+ };
95
+ }
51
96
  function summarizeSchemaResult(result) {
52
97
  const samples = Object.values(result.raw).flat();
53
98
  const n = samples.length;
@@ -57,6 +102,17 @@ function summarizeSchemaResult(result) {
57
102
  const accuracies = samples
58
103
  .map((s) => s.valueAccuracy)
59
104
  .filter((a) => a !== null);
105
+ // Per-corpus rollup (demo-rig §11.3b) and per-difficulty rollup
106
+ // (discriminating-fixtures §3). Each groups the same samples by a different
107
+ // hash-excluded classification, so surfaces can say which corpus produced a
108
+ // cell and how a model does on the hard subset. The aggregate above stays the
109
+ // ranking basis (same whole corpus for every model).
110
+ const by_family = rollupBy(result, result.fixtureFamilies, DEFAULT_FIXTURE_FAMILY);
111
+ const by_difficulty = rollupBy(result, result.fixtureDifficulties, DEFAULT_FIXTURE_DIFFICULTY);
112
+ // Subtype rollup (W16): no default bucket — an untagged fixture belongs to
113
+ // no task type, so it rolls into a sentinel that is then dropped.
114
+ const by_subtype = rollupBy(result, result.fixtureSubtypes, UNTAGGED_SUBTYPE);
115
+ delete by_subtype[UNTAGGED_SUBTYPE];
60
116
  return {
61
117
  samples: n,
62
118
  parse_rate: n === 0 ? 0 : samples.filter((s) => s.parseOk).length / n,
@@ -65,6 +121,9 @@ function summarizeSchemaResult(result) {
65
121
  value_accuracy_mean: mean(accuracies),
66
122
  native_rung_rate: rungs.length === 0 ? null : rungs.filter((s) => s.rung === "native").length / rungs.length,
67
123
  mean_cost_usd: n === 0 ? 0 : samples.reduce((a, s) => a + s.costUsd, 0) / n,
124
+ ...(Object.keys(by_family).length > 0 ? { by_family } : {}),
125
+ ...(Object.keys(by_difficulty).length > 0 ? { by_difficulty } : {}),
126
+ ...(Object.keys(by_subtype).length > 0 ? { by_subtype } : {}),
68
127
  };
69
128
  }
70
129
  function summarizeGroundingResult(result) {
@@ -78,18 +137,73 @@ function summarizeGroundingResult(result) {
78
137
  }
79
138
  function summarizeCachingResult(result) {
80
139
  const samples = Object.values(result.raw).flat();
140
+ // failureClass null = the paired call ran AND passed the documented-path
141
+ // eligibility gate (cache_prefix_below_minimum is a typed skip upstream).
81
142
  const okSamples = samples.filter((s) => s.failureClass === null);
82
143
  const n = okSamples.length;
83
144
  return {
84
145
  samples: samples.length,
85
146
  cache_hit_rate: n === 0 ? 0 : okSamples.filter((s) => (s.cachedTokens ?? 0) > 0).length / n,
86
147
  mean_cached_tokens: mean(okSamples.map((s) => s.cachedTokens ?? 0)) ?? 0,
148
+ eligible_samples: n,
87
149
  };
88
150
  }
89
- function buildProbedLayer(results) {
151
+ /** Compose a model's SCHEMA result files into one synthetic file, so a cheap
152
+ * SUPPLEMENTAL run (new fixtures only) ADDS to a prior full run instead of
153
+ * replacing it — the default latest-per-file rule would drop every fixture the
154
+ * newest file omits. The latest schema file is the base; older files fill in
155
+ * fixtures the base lacks, newest date winning on any per-fixture collision.
156
+ * ONLY fixtures in `currentFixtureIds` (the live corpus) are pulled forward, so
157
+ * a RETIRED id sitting in an old file is never resurrected (the trap a naive
158
+ * union hits, e.g. demo-support-summarize double-counting summarize). */
159
+ function mergeSchemaResults(results, currentFixtureIds) {
160
+ const schemaFiles = results
161
+ .filter((r) => r.class === "schema")
162
+ .sort((a, b) => (a.date > b.date ? -1 : a.date < b.date ? 1 : 0)); // newest first
163
+ const base = schemaFiles[0];
164
+ if (base === undefined)
165
+ return undefined;
166
+ const raw = { ...base.raw };
167
+ const fixtureSubtypes = { ...(base.fixtureSubtypes ?? {}) };
168
+ const fixtureFamilies = { ...(base.fixtureFamilies ?? {}) };
169
+ const fixtureDifficulties = { ...(base.fixtureDifficulties ?? {}) };
170
+ for (const older of schemaFiles.slice(1)) {
171
+ for (const [fid, samples] of Object.entries(older.raw)) {
172
+ if (fid in raw)
173
+ continue; // a newer file already carries it — newest wins
174
+ if (!currentFixtureIds.has(fid))
175
+ continue; // retired id — never resurrect
176
+ raw[fid] = samples;
177
+ if (older.fixtureSubtypes?.[fid] !== undefined)
178
+ fixtureSubtypes[fid] = older.fixtureSubtypes[fid];
179
+ if (older.fixtureFamilies?.[fid] !== undefined)
180
+ fixtureFamilies[fid] = older.fixtureFamilies[fid];
181
+ if (older.fixtureDifficulties?.[fid] !== undefined)
182
+ fixtureDifficulties[fid] = older.fixtureDifficulties[fid];
183
+ }
184
+ }
185
+ return {
186
+ ...base,
187
+ raw,
188
+ ...(Object.keys(fixtureSubtypes).length > 0 ? { fixtureSubtypes } : {}),
189
+ ...(Object.keys(fixtureFamilies).length > 0 ? { fixtureFamilies } : {}),
190
+ ...(Object.keys(fixtureDifficulties).length > 0 ? { fixtureDifficulties } : {}),
191
+ };
192
+ }
193
+ /**
194
+ * Build a model's probed layer. When `currentSchemaFixtureIds` is supplied, the
195
+ * schema class COMPOSES across the model's result files (a supplemental run adds
196
+ * fixtures to a prior full run) via mergeSchemaResults; grounding/caching stay
197
+ * latest-per-file (one file per cycle). Omitting the ids keeps the pure
198
+ * latest-per-file behaviour (back-compat for hermetic tests).
199
+ */
200
+ function buildProbedLayer(results, currentSchemaFixtureIds) {
90
201
  if (results.length === 0)
91
202
  return null;
92
203
  const byClass = latestPerClass(results);
204
+ const schemaFile = currentSchemaFixtureIds !== undefined
205
+ ? mergeSchemaResults(results, currentSchemaFixtureIds)
206
+ : byClass.schema;
93
207
  const dates = results.map((r) => r.date).sort();
94
208
  const versions = results.map((r) => r.harnessVersion).sort();
95
209
  const newest = dates[dates.length - 1];
@@ -97,7 +211,7 @@ function buildProbedLayer(results) {
97
211
  return {
98
212
  as_of: newest,
99
213
  source: `modelrig-probes@${version}`,
100
- schema: byClass.schema !== undefined ? summarizeSchemaResult(byClass.schema) : null,
214
+ schema: schemaFile !== undefined ? summarizeSchemaResult(schemaFile) : null,
101
215
  grounding: byClass.grounding !== undefined ? summarizeGroundingResult(byClass.grounding) : null,
102
216
  caching: byClass.caching !== undefined ? summarizeCachingResult(byClass.caching) : null,
103
217
  };
@@ -156,10 +270,17 @@ function detectDiscrepancies(declared, probed) {
156
270
  }
157
271
  const caching = probed.caching;
158
272
  if (caching !== null && caching.samples > 0) {
159
- if (declared.flags.supports_prompt_caching === true && caching.cache_hit_rate === 0) {
273
+ // Guarded (launch-runbook A1, 2026-08-18): fire only when at least one
274
+ // sample passed the documented-path eligibility gate — a zero rate over
275
+ // zero eligible samples means the probe couldn't measure, not that the
276
+ // capability is unrealized. Pre-gate result files lack the field and
277
+ // keep the old behavior via the ?? fallback.
278
+ if (declared.flags.supports_prompt_caching === true &&
279
+ caching.cache_hit_rate === 0 &&
280
+ (caching.eligible_samples ?? caching.samples) > 0) {
160
281
  out.push({
161
282
  kind: "declared_caching_unrealized",
162
- message: "declared supports_prompt_caching, but no probed repeat call reported cached tokens",
283
+ message: "declared supports_prompt_caching, but no eligible probed repeat call on the documented caching path reported cached tokens",
163
284
  });
164
285
  }
165
286
  }
@@ -0,0 +1,94 @@
1
+ /**
2
+ * Cache Policy Registry — cache-aware-routing spec §3, [Briefing §7].
3
+ *
4
+ * Per-provider cache SEMANTICS as versioned DATA, never routing code: TTLs,
5
+ * refresh-on-read, minimum-cacheable prefix, key mechanism, telemetry dialect,
6
+ * endpoint affinity. A provider rule change (OpenAI shipping paid writes,
7
+ * Anthropic GA-ing the 1h TTL) becomes a data edit here with a new `as_of`, not
8
+ * a code change anywhere else — the whole point of the registry.
9
+ *
10
+ * TWO HOMES, both pre-existing (spec §3):
11
+ * • RATES ride pricing — `ModelPricingV2.base.{cache_read,cache_write,
12
+ * cache_write_1h}` and flat `ModelPricing.cacheWrite*UsdPerTok`, entered via
13
+ * the pricing sweep with source+as_of. NOT here.
14
+ * • POLICY (this file) is the non-price semantics.
15
+ *
16
+ * L5 (fail-closed): every field cites a first-party page (research §2.1, fetched
17
+ * 2026-08-18). A value we could not verify is `null` (or `cache_supported:
18
+ * false`) — never a guess. `getCachePolicy` returns a fail-closed
19
+ * `cache_supported: false` policy for any provider not seeded, identical posture
20
+ * to the zero-coverage gates.
21
+ *
22
+ * ── CROSS-REFERENCES (no import — the probes package is standalone since P2) ──
23
+ * • `min_cacheable_tokens` is reconciled by hand against the probes package's
24
+ * `MIN_CACHEABLE_TOKENS` (packages/modelrig-probes/src/caching.ts, PR #157),
25
+ * which cites the same first-party pages. See MIN_CACHEABLE_RECONCILIATION
26
+ * below for the value-by-value comparison and the two deliberate deltas
27
+ * (anthropic, gemini) flagged for the architect to converge if desired. A
28
+ * reciprocal comment lives in caching.ts.
29
+ * • OBSERVED cache facts (`cache_hit_rate`, `mean_cached_tokens`,
30
+ * `declared_caching_unrealized`) are materialized separately in
31
+ * `src/registry/build.ts` from probe/production telemetry — the DECLARED
32
+ * policy here is the counterpart truth, and the §5.6 streak alert (WS-C)
33
+ * cross-references `declared_caching_unrealized` (same smell, other plane).
34
+ */
35
+ import type { ProviderId } from "../types/provider";
36
+ /**
37
+ * Per-provider cache semantics (spec §3). Additive to the registry's declared
38
+ * layer; every field is first-party-sourced or `null` (L5).
39
+ */
40
+ export interface CachePolicy {
41
+ /** Whether the provider caches on the path ModelRig uses. false ⇒ every
42
+ * other field is moot (fail-closed). */
43
+ readonly cache_supported: boolean;
44
+ /** How caching is engaged. `explicit_breakpoints` = caller marks blocks
45
+ * (Anthropic cache_control); `automatic_prefix` = provider caches the longest
46
+ * matching prefix with no request change; `key_opt_in` = caller supplies a
47
+ * routing key (Mistral-style — no ModelRig provider uses it in v1). */
48
+ readonly mode: "explicit_breakpoints" | "automatic_prefix" | "key_opt_in";
49
+ /** Default cache lifetime, seconds. null ⇒ undocumented / best-effort. */
50
+ readonly default_ttl_seconds: number | null;
51
+ /** Extended-TTL option, seconds (Anthropic 1h write tier; OpenAI 24h
52
+ * retention). null ⇒ none published. */
53
+ readonly extended_ttl_seconds: number | null;
54
+ /** Whether a cache READ resets the TTL (Anthropic 5m: true). Conservative
55
+ * `false` when undocumented — under-claiming warmth never over-predicts a hit. */
56
+ readonly ttl_refresh_on_read: boolean;
57
+ /** Minimum prefix (tokens) below which caching silently does not engage.
58
+ * null ⇒ no documented minimum (L5). See MIN_CACHEABLE_RECONCILIATION. */
59
+ readonly min_cacheable_tokens: number | null;
60
+ /** The request parameter that engages/keys caching, or null for pure
61
+ * automatic caching with no caller control. */
62
+ readonly cache_key_param: "prompt_cache_key" | "cache_control" | "cachedContent" | null;
63
+ /** Whether cache-READ tokens are observable on the path ModelRig reads
64
+ * (usage JSON). false where they ride HTTP headers we do not yet lift. */
65
+ readonly reports_read_tokens: boolean;
66
+ /** Whether cache-WRITE (creation) tokens are separately reported. */
67
+ readonly reports_write_tokens: boolean;
68
+ /** Which usage shape the adapter normalizes from. */
69
+ readonly usage_dialect: "anthropic" | "openai_details" | "deepseek_partition" | "gemini_metadata" | "fireworks_headers";
70
+ /** true where the warm cache is endpoint-sticky and follow-ups must land on
71
+ * the same endpoint (research: OpenAI ~15 req/min/key; Fireworks
72
+ * x-session-affinity). */
73
+ readonly provider_affinity_required: boolean;
74
+ /** Provenance — same rule as pricing (first-party page). */
75
+ readonly source: string;
76
+ readonly as_of: string;
77
+ readonly policy_version: string;
78
+ }
79
+ /** The fail-closed policy for any provider not seeded (spec §3): caching is
80
+ * assumed OFF until a first-party source says otherwise — identical posture to
81
+ * the zero-coverage gates. `usage_dialect` defaults to the OpenAI shape only as
82
+ * a type-satisfying placeholder; `cache_supported: false` means no consumer
83
+ * reads it. */
84
+ export declare function failClosedCachePolicy(): CachePolicy;
85
+ /**
86
+ * The cache policy for a provider — the seeded first-party record, or a
87
+ * fail-closed `cache_supported: false` policy when the provider is not seeded
88
+ * (spec §3). Provider-level in v1; model-specific overrides land with the
89
+ * conversational-gateway phase.
90
+ */
91
+ export declare function getCachePolicy(provider: ProviderId): CachePolicy;
92
+ /** Every seeded provider — for tests and the console policy surface (WS-D). */
93
+ export declare function seededCachePolicyProviders(): readonly ProviderId[];
94
+ //# sourceMappingURL=cache-policy.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cache-policy.d.ts","sourceRoot":"","sources":["../../src/registry/cache-policy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B;4CACwC;IACxC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;IAClC;;;2EAGuE;IACvE,QAAQ,CAAC,IAAI,EAAE,sBAAsB,GAAG,kBAAkB,GAAG,YAAY,CAAC;IAC1E,0EAA0E;IAC1E,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C;4CACwC;IACxC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C;sFACkF;IAClF,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACtC;8EAC0E;IAC1E,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C;mDAC+C;IAC/C,QAAQ,CAAC,eAAe,EAAE,kBAAkB,GAAG,eAAe,GAAG,eAAe,GAAG,IAAI,CAAC;IACxF;8EAC0E;IAC1E,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACtC,qEAAqE;IACrE,QAAQ,CAAC,oBAAoB,EAAE,OAAO,CAAC;IACvC,qDAAqD;IACrD,QAAQ,CAAC,aAAa,EAClB,WAAW,GACX,gBAAgB,GAChB,oBAAoB,GACpB,iBAAiB,GACjB,mBAAmB,CAAC;IACxB;;8BAE0B;IAC1B,QAAQ,CAAC,0BAA0B,EAAE,OAAO,CAAC;IAC7C,4DAA4D;IAC5D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AA8KD;;;;eAIe;AACf,wBAAgB,qBAAqB,IAAI,WAAW,CAiBnD;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,UAAU,GAAG,WAAW,CAEhE;AAED,+EAA+E;AAC/E,wBAAgB,0BAA0B,IAAI,SAAS,UAAU,EAAE,CAElE"}