modelrig 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +156 -0
- package/README.md +27 -2
- package/dist/artifacts/artifact.d.ts +12 -13
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +17 -23
- package/dist/artifacts/index.d.ts +13 -5
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +11 -4
- package/dist/artifacts/run-context.d.ts +3 -24
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +3 -39
- package/dist/bakeoff/engine.d.ts +29 -0
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +47 -0
- package/dist/bakeoff/eval-cases-rig.d.ts +36 -0
- package/dist/bakeoff/eval-cases-rig.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases-rig.js +27 -0
- package/dist/bakeoff/eval-cases.d.ts +19 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -1
- package/dist/bakeoff/eval-cases.js +18 -1
- package/dist/bakeoff/golden-judge.d.ts +47 -0
- package/dist/bakeoff/golden-judge.d.ts.map +1 -0
- package/dist/bakeoff/golden-judge.js +49 -0
- package/dist/bakeoff/quality.d.ts +30 -0
- package/dist/bakeoff/quality.d.ts.map +1 -1
- package/dist/bakeoff/quality.js +21 -0
- package/dist/bakeoff/replay.d.ts +11 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +8 -0
- package/dist/bakeoff/report.d.ts +7 -0
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +26 -0
- package/dist/cli-judge.d.ts +12 -0
- package/dist/cli-judge.d.ts.map +1 -0
- package/dist/cli-judge.js +48 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +14 -42
- package/dist/config.d.ts +54 -29
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +86 -47
- package/dist/detect/detector.d.ts +4 -2
- package/dist/detect/detector.d.ts.map +1 -1
- package/dist/detect/safe-harbor.d.ts +57 -0
- package/dist/detect/safe-harbor.d.ts.map +1 -0
- package/dist/detect/safe-harbor.js +278 -0
- package/dist/feedback/coverage.d.ts +11 -8
- package/dist/feedback/coverage.d.ts.map +1 -1
- package/dist/feedback/coverage.js +29 -16
- package/dist/feedback/grade.d.ts +51 -0
- package/dist/feedback/grade.d.ts.map +1 -0
- package/dist/feedback/grade.js +63 -0
- package/dist/gateway/http-protocol.d.ts +1 -1
- package/dist/gateway/http-protocol.d.ts.map +1 -1
- package/dist/gateway/http-protocol.js +1 -1
- package/dist/index.d.ts +44 -27
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +110 -43
- package/dist/init/observe-cli.js +1 -1
- package/dist/init/observe.d.ts +2 -2
- package/dist/init/observe.d.ts.map +1 -1
- package/dist/init/observe.js +11 -7
- package/dist/mask/arm.d.ts +112 -0
- package/dist/mask/arm.d.ts.map +1 -0
- package/dist/mask/arm.js +179 -0
- package/dist/mask/index.d.ts +2 -0
- package/dist/mask/index.d.ts.map +1 -1
- package/dist/mask/index.js +7 -1
- package/dist/mask/mask-policy-cli.d.ts +12 -4
- package/dist/mask/mask-policy-cli.d.ts.map +1 -1
- package/dist/mask/mask-policy-cli.js +34 -9
- package/dist/query/grammar.d.ts +2 -2
- package/dist/query/grammar.d.ts.map +1 -1
- package/dist/query/grammar.js +3 -1
- package/dist/query/saved.d.ts +13 -0
- package/dist/query/saved.d.ts.map +1 -1
- package/dist/query/saved.js +67 -1
- package/dist/receipts/types.d.ts +1 -1
- package/dist/receipts/types.d.ts.map +1 -1
- package/dist/receipts/types.js +1 -0
- package/dist/registry/build.d.ts +43 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +52 -0
- package/dist/registry/pricing-shape.d.ts +29 -0
- package/dist/registry/pricing-shape.d.ts.map +1 -1
- package/dist/registry/pricing-shape.js +10 -1
- package/dist/registry/rows.d.ts.map +1 -1
- package/dist/registry/rows.js +1 -0
- package/dist/registry/select.d.ts +6 -1
- package/dist/registry/select.d.ts.map +1 -1
- package/dist/registry/select.js +11 -0
- package/dist/routes/call-config.d.ts +199 -0
- package/dist/routes/call-config.d.ts.map +1 -0
- package/dist/routes/call-config.js +343 -0
- package/dist/routes/loader.js +5 -3
- package/dist/routes/objective-loader.d.ts +12 -0
- package/dist/routes/objective-loader.d.ts.map +1 -1
- package/dist/routes/objective-loader.js +82 -0
- package/dist/routes/resolved-bundle.d.ts +50 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -1
- package/dist/routes/resolved-bundle.js +52 -0
- package/dist/routes/screen-loader.d.ts +28 -0
- package/dist/routes/screen-loader.d.ts.map +1 -1
- package/dist/routes/screen-loader.js +51 -0
- package/dist/run/accept.d.ts +2 -0
- package/dist/run/accept.d.ts.map +1 -1
- package/dist/run/accept.js +1 -0
- package/dist/run/dispatch.d.ts +42 -13
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +107 -25
- package/dist/run/grounding.d.ts +28 -2
- package/dist/run/grounding.d.ts.map +1 -1
- package/dist/run/grounding.js +54 -10
- package/dist/run/record-attempt.d.ts +4 -0
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +4 -0
- package/dist/run/routing-docs.js +1 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +21 -26
- package/dist/savings/odometer.d.ts +4 -0
- package/dist/savings/odometer.d.ts.map +1 -1
- package/dist/savings/odometer.js +20 -1
- package/dist/savings/swap-execute.d.ts +7 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +30 -0
- package/dist/savings/swaps-client.d.ts +10 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/screen/fixtures/probe-corpora.d.ts +23 -0
- package/dist/screen/fixtures/probe-corpora.d.ts.map +1 -0
- package/dist/screen/fixtures/probe-corpora.js +69 -0
- package/dist/screen/index.d.ts +21 -0
- package/dist/screen/index.d.ts.map +1 -0
- package/dist/screen/index.js +64 -0
- package/dist/screen/model-screen.d.ts +111 -0
- package/dist/screen/model-screen.d.ts.map +1 -0
- package/dist/screen/model-screen.js +178 -0
- package/dist/screen/patterns.d.ts +51 -0
- package/dist/screen/patterns.d.ts.map +1 -0
- package/dist/screen/patterns.js +151 -0
- package/dist/screen/phi-gate.d.ts +52 -0
- package/dist/screen/phi-gate.d.ts.map +1 -0
- package/dist/screen/phi-gate.js +86 -0
- package/dist/screen/policy.d.ts +71 -0
- package/dist/screen/policy.d.ts.map +1 -0
- package/dist/screen/policy.js +334 -0
- package/dist/screen/probe.d.ts +118 -0
- package/dist/screen/probe.d.ts.map +1 -0
- package/dist/screen/probe.js +132 -0
- package/dist/screen/screen-cli.d.ts +16 -0
- package/dist/screen/screen-cli.d.ts.map +1 -0
- package/dist/screen/screen-cli.js +178 -0
- package/dist/screen/screen.d.ts +99 -0
- package/dist/screen/screen.d.ts.map +1 -0
- package/dist/screen/screen.js +0 -0
- package/dist/screen/sidecar.d.ts +69 -0
- package/dist/screen/sidecar.d.ts.map +1 -0
- package/dist/screen/sidecar.js +145 -0
- package/dist/screen/types.d.ts +43 -2
- package/dist/screen/types.d.ts.map +1 -1
- package/dist/settings/api-fetcher.d.ts +14 -3
- package/dist/settings/api-fetcher.d.ts.map +1 -1
- package/dist/settings/api-fetcher.js +61 -10
- package/dist/settings/grading-guide.d.ts +76 -0
- package/dist/settings/grading-guide.d.ts.map +1 -0
- package/dist/settings/grading-guide.js +23 -0
- package/dist/settings/index.d.ts +3 -2
- package/dist/settings/index.d.ts.map +1 -1
- package/dist/settings/index.js +2 -1
- package/dist/settings/task-settings.d.ts +44 -7
- package/dist/settings/task-settings.d.ts.map +1 -1
- package/dist/settings/task-settings.js +53 -12
- package/dist/status.d.ts +26 -20
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +47 -29
- package/dist/telemetry/artifact-store.d.ts +5 -6
- package/dist/telemetry/artifact-store.d.ts.map +1 -1
- package/dist/telemetry/artifact-store.js +10 -24
- package/dist/telemetry/attempt-meta.d.ts +21 -0
- package/dist/telemetry/attempt-meta.d.ts.map +1 -1
- package/dist/telemetry/attempt-meta.js +9 -0
- package/dist/telemetry/exporter.d.ts +2 -2
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +22 -41
- package/dist/telemetry/grade-mappers.d.ts +22 -0
- package/dist/telemetry/grade-mappers.d.ts.map +1 -0
- package/dist/telemetry/grade-mappers.js +58 -0
- package/dist/telemetry/grades-sql.d.ts +53 -0
- package/dist/telemetry/grades-sql.d.ts.map +1 -0
- package/dist/telemetry/grades-sql.js +133 -0
- package/dist/telemetry/ingest-protocol.d.ts +29 -46
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +16 -54
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +51 -0
- package/dist/telemetry/store.d.ts +9 -6
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +8 -50
- package/dist/tenancy/api-key-format.d.ts +30 -1
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +33 -1
- package/dist/types/provider.d.ts +3 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +25 -0
- package/dist/types/route.d.ts +78 -2
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +43 -46
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +32 -0
- package/dist/verify/verify.d.ts +5 -0
- package/dist/verify/verify.d.ts.map +1 -1
- package/dist/verify/verify.js +2 -0
- package/dist/watch/evaluate.d.ts +84 -1
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +28 -0
- package/dist/watch/monitor.d.ts +8 -2
- package/dist/watch/monitor.d.ts.map +1 -1
- package/dist/watch/monitor.js +13 -5
- package/dist/watch/regret-monitor-cli.d.ts +16 -0
- package/dist/watch/regret-monitor-cli.d.ts.map +1 -0
- package/dist/watch/regret-monitor-cli.js +137 -0
- package/dist/watch/regret-monitor.d.ts +168 -0
- package/dist/watch/regret-monitor.d.ts.map +1 -0
- package/dist/watch/regret-monitor.js +304 -0
- package/dist/watch/upgrade.d.ts +62 -0
- package/dist/watch/upgrade.d.ts.map +1 -0
- package/dist/watch/upgrade.js +57 -0
- package/docs/agent-skill/SKILL.md +109 -5
- package/docs/how-it-fits.md +3 -2
- package/docs/instrumentation.md +149 -21
- package/docs/llms.txt +64 -20
- package/docs/mcp-oracle.md +249 -0
- package/docs/quickstart.md +103 -9
- package/package.json +16 -3
- package/registry/leaderboard.json +74 -0
- package/registry/registry.json +150 -112
- package/screen.d.ts +16 -0
- package/templates/call-disposition-qa.json +77 -0
- package/templates/catalog-cleansing.json +77 -0
- package/templates/compliance-review.json +76 -0
- package/templates/cs-next-action.json +76 -0
- package/templates/document-extraction.json +78 -0
- package/templates/financial-classification.json +75 -0
- package/templates/lead-qualification.json +76 -0
- package/templates/medical-classification.json +71 -0
- package/templates/ticket-triage.json +76 -0
- package/dist/telemetry/supabase-exporter.d.ts +0 -64
- package/dist/telemetry/supabase-exporter.d.ts.map +0 -1
- package/dist/telemetry/supabase-exporter.js +0 -211
package/dist/query/saved.js
CHANGED
|
@@ -11,9 +11,10 @@
|
|
|
11
11
|
* editorializes (F5: measured-only).
|
|
12
12
|
*/
|
|
13
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
-
exports.SAVED_QUERIES = void 0;
|
|
14
|
+
exports.LAUNCH_COHORT_QUERIES = exports.SAVED_QUERIES = void 0;
|
|
15
15
|
exports.savedQueryById = savedQueryById;
|
|
16
16
|
exports.sinceIsoFor = sinceIsoFor;
|
|
17
|
+
exports.cohortQueryById = cohortQueryById;
|
|
17
18
|
exports.SAVED_QUERIES = [
|
|
18
19
|
{
|
|
19
20
|
id: "cost-per-subject",
|
|
@@ -49,3 +50,68 @@ function savedQueryById(id) {
|
|
|
49
50
|
function sinceIsoFor(windowDays, now) {
|
|
50
51
|
return new Date(now.getTime() - windowDays * 24 * 60 * 60 * 1000).toISOString();
|
|
51
52
|
}
|
|
53
|
+
exports.LAUNCH_COHORT_QUERIES = [
|
|
54
|
+
{
|
|
55
|
+
id: "cohort-orgs-active",
|
|
56
|
+
title: "Orgs created & active (≥1 run this week)",
|
|
57
|
+
description: "Orgs with at least one inference in the window — the active-org count the T+75 decision reads (spec §3.1).",
|
|
58
|
+
windowDays: 7,
|
|
59
|
+
sql: (since) => `SELECT count(DISTINCT org_id) AS active_orgs FROM inferences WHERE ts >= '${since}'`,
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
id: "cohort-byok-spend",
|
|
63
|
+
title: "BYOK list-price spend per org (p50 / p90 / total)",
|
|
64
|
+
description: "Per-org BYOK spend at list price over the window, then p50/p90/total across orgs — the launch value we are NOT charging during launch (spec §3.2).",
|
|
65
|
+
windowDays: 30,
|
|
66
|
+
sql: (since) => `WITH per_org AS (` +
|
|
67
|
+
`SELECT org_id, sum(cost_estimate_usd) AS byok_spend FROM inferences ` +
|
|
68
|
+
`WHERE ts >= '${since}' AND lane = 'byok' GROUP BY org_id) ` +
|
|
69
|
+
`SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY byok_spend) AS p50, ` +
|
|
70
|
+
`percentile_cont(0.9) WITHIN GROUP (ORDER BY byok_spend) AS p90, ` +
|
|
71
|
+
`sum(byok_spend) AS total FROM per_org`,
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
id: "cohort-card-adoption",
|
|
75
|
+
title: "Card-on-file adoption %",
|
|
76
|
+
description: "Share of billing orgs with a Stripe customer + verified card on file (the WS4 card path) — the cap-lifting signal (spec §3.3).",
|
|
77
|
+
windowDays: 30,
|
|
78
|
+
sql: () => `SELECT count(*) FILTER (WHERE stripe_customer IS NOT NULL) AS with_card, ` +
|
|
79
|
+
`count(*) AS billing_orgs FROM balances`,
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
id: "cohort-managed-adoption",
|
|
83
|
+
title: "Managed-key adoption % and managed spend",
|
|
84
|
+
description: "Orgs with any managed-lane usage and their managed list-price spend over the window (spec §3.4).",
|
|
85
|
+
windowDays: 30,
|
|
86
|
+
sql: (since) => `SELECT count(DISTINCT org_id) AS managed_orgs, sum(cost_estimate_usd) AS managed_spend ` +
|
|
87
|
+
`FROM inferences WHERE ts >= '${since}' AND lane = 'managed'`,
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
id: "cohort-bakeoffs-per-org",
|
|
91
|
+
title: "Bake-offs run per org",
|
|
92
|
+
description: "Bake-off artifacts published per org over the window — engagement with the proof surface (spec §3.6).",
|
|
93
|
+
windowDays: 30,
|
|
94
|
+
sql: (since) => `SELECT org_id, count(*) AS bakeoffs FROM artifacts ` +
|
|
95
|
+
`WHERE kind = 'bakeoff' AND created_at >= '${since}' GROUP BY org_id ORDER BY bakeoffs DESC`,
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
id: "cohort-receipts-per-org",
|
|
99
|
+
title: "Published receipts per org",
|
|
100
|
+
description: "Receipts published per org over the window (from the artifacts index) — a card-lifted, unlimited cap for card-on-file orgs (spec §3.7).",
|
|
101
|
+
windowDays: 30,
|
|
102
|
+
sql: (since) => `SELECT org_id, count(*) AS receipts FROM artifacts ` +
|
|
103
|
+
`WHERE kind = 'receipt' AND created_at >= '${since}' GROUP BY org_id ORDER BY receipts DESC`,
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
id: "cohort-cap-hits",
|
|
107
|
+
title: "Cap-hit counts by cap",
|
|
108
|
+
description: "How often each soft fair-use cap was exceeded over the window — the counted notices from the launch-terms cap counters (spec §3.9). Reads the counted cap-notice rows recorded in control_state.",
|
|
109
|
+
windowDays: 30,
|
|
110
|
+
sql: (since) => `SELECT key, value FROM control_state ` +
|
|
111
|
+
`WHERE key LIKE 'launch-terms:cap-hit:%' AND updated_at >= '${since}' ORDER BY key`,
|
|
112
|
+
},
|
|
113
|
+
];
|
|
114
|
+
/** Look up a launch cohort question by id, or null. */
|
|
115
|
+
function cohortQueryById(id) {
|
|
116
|
+
return exports.LAUNCH_COHORT_QUERIES.find((q) => q.id === id) ?? null;
|
|
117
|
+
}
|
package/dist/receipts/types.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type ReceiptKind = "bakeoff" | "regret" | "masking-proof" | "run" | "egress" | "disclosure" | "statement" | "fact-sheet" | "launch-report";
|
|
1
|
+
export type ReceiptKind = "bakeoff" | "regret" | "masking-proof" | "run" | "egress" | "disclosure" | "statement" | "fact-sheet" | "launch-report" | "detector-probe";
|
|
2
2
|
/** The kinds RP-1 ships producers for. Others land with their own missions. */
|
|
3
3
|
export declare const RECEIPT_KINDS: readonly ReceiptKind[];
|
|
4
4
|
export type MeasuredValue = number | string | boolean | null;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/receipts/types.ts"],"names":[],"mappings":"AAgBA,MAAM,MAAM,WAAW,GACnB,SAAS,GAAG,QAAQ,GAAG,eAAe,GAAG,KAAK,GAAG,QAAQ,GACzD,YAAY,GAAG,WAAW,GAAG,YAAY,GAAG,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/receipts/types.ts"],"names":[],"mappings":"AAgBA,MAAM,MAAM,WAAW,GACnB,SAAS,GAAG,QAAQ,GAAG,eAAe,GAAG,KAAK,GAAG,QAAQ,GACzD,YAAY,GAAG,WAAW,GAAG,YAAY,GAAG,eAAe,GAG3D,gBAAgB,CAAC;AAErB,+EAA+E;AAC/E,eAAO,MAAM,aAAa,EAAE,SAAS,WAAW,EAI/C,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;AAE7D,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC;IACd,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,QAAQ,CAAC,OAAO,EAAE;QAChB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAC9B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QACjC,QAAQ,CAAC,MAAM,CAAC,EAAE;YAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;SAAE,CAAC;KAClE,CAAC;IACF,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IAC3D,QAAQ,CAAC,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1D,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS;QAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC/G,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3C,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IACtD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE;QAAE,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;CAChG;AAED,2DAA2D;AAC3D,MAAM,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;AAwBhE;;;;;;;;;;;;GAYG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAiDxD;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,WAAW,GAAG,MAAM,CAE7D;AAED;mEACmE;AACnE,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,GAAG,MAAM,CAErD"}
|
package/dist/receipts/types.js
CHANGED
|
@@ -23,6 +23,7 @@ const diff_1 = require("../verify/diff");
|
|
|
23
23
|
exports.RECEIPT_KINDS = [
|
|
24
24
|
"bakeoff", "regret", "masking-proof", "run", "egress",
|
|
25
25
|
"disclosure", "statement", "fact-sheet", "launch-report",
|
|
26
|
+
"detector-probe",
|
|
26
27
|
];
|
|
27
28
|
/**
|
|
28
29
|
* Keys that would carry a tenant identifier and so must NEVER appear in
|
package/dist/registry/build.d.ts
CHANGED
|
@@ -31,6 +31,14 @@ export interface DeclaredLayer {
|
|
|
31
31
|
/** What OUR adapter actually drives for this model (static map v0). */
|
|
32
32
|
readonly adapter_structured_native: boolean | null;
|
|
33
33
|
readonly adapter_grounded_native: boolean | null;
|
|
34
|
+
/**
|
|
35
|
+
* DECLARED image-input claim (UC-3 M0) — the vendor's `supports_vision`
|
|
36
|
+
* flag from the pinned LiteLLM map. It SEEDS the image-probe candidate list
|
|
37
|
+
* and drives the declared-vs-probed discrepancy, but it can NEVER mark the
|
|
38
|
+
* probed `image_input` dimension true: probed-beats-declared, and a claim is
|
|
39
|
+
* not a measurement (dev-rule S9c / spec §3). `null` when the map is silent
|
|
40
|
+
* — silence is "unknown", never "no". */
|
|
41
|
+
readonly supports_image_input: boolean | null;
|
|
34
42
|
};
|
|
35
43
|
}
|
|
36
44
|
/** Per-corpus (fixture-family) breakdown of a schema summary — demo-rig
|
|
@@ -85,6 +93,27 @@ export interface ProbedGroundingSummary {
|
|
|
85
93
|
readonly grounded_rate: number;
|
|
86
94
|
readonly mean_citations: number;
|
|
87
95
|
}
|
|
96
|
+
/**
|
|
97
|
+
* Image-input probe summary (UC-3 M0) — the PROBED half of the `image_input`
|
|
98
|
+
* dimension. Every field is a MEASUREMENT over the image-fixture leg (a form
|
|
99
|
+
* image sent with a JSON-schema ask); nothing here is ever seeded from a
|
|
100
|
+
* declared claim. Absence of this block (probed.image === null) means UNPROBED,
|
|
101
|
+
* which is never rendered as "cannot" (spec §3, S9c honesty).
|
|
102
|
+
*/
|
|
103
|
+
export interface ProbedImageSummary {
|
|
104
|
+
readonly samples: number;
|
|
105
|
+
/** The probe's PASS rate: fraction of samples that returned schema-conformant
|
|
106
|
+
* JSON AND recovered every planted field (full value accuracy). A wrong-fields
|
|
107
|
+
* response — conformant shape, wrong values — is NOT an extraction, so it
|
|
108
|
+
* lowers this rate (the negative control the leg is tested against). `null`
|
|
109
|
+
* when no samples were taken. */
|
|
110
|
+
readonly extract_rate: number | null;
|
|
111
|
+
readonly extract_ci95: readonly [number, number] | null;
|
|
112
|
+
/** Mean fraction of planted fields recovered over parsed samples — the
|
|
113
|
+
* graded-partial signal behind `extract_rate`. `null` when nothing parsed. */
|
|
114
|
+
readonly value_accuracy_mean: number | null;
|
|
115
|
+
readonly mean_cost_usd: number;
|
|
116
|
+
}
|
|
88
117
|
export interface ProbedCachingSummary {
|
|
89
118
|
readonly samples: number;
|
|
90
119
|
readonly cache_hit_rate: number;
|
|
@@ -102,6 +131,10 @@ export interface ProbedLayer {
|
|
|
102
131
|
readonly schema: ProbedSchemaSummary | null;
|
|
103
132
|
readonly grounding: ProbedGroundingSummary | null;
|
|
104
133
|
readonly caching: ProbedCachingSummary | null;
|
|
134
|
+
/** Image-input leg (UC-3 M0) — null when the model has not been probed on the
|
|
135
|
+
* image class. Optional/additive: pre-image probed layers omit the key, so a
|
|
136
|
+
* reader guards on `?? null` (an absent leg is UNPROBED, never "no images"). */
|
|
137
|
+
readonly image?: ProbedImageSummary | null;
|
|
105
138
|
}
|
|
106
139
|
export interface ObservedLayer {
|
|
107
140
|
readonly as_of: string;
|
|
@@ -155,7 +188,7 @@ export interface Registry {
|
|
|
155
188
|
* structural so the core package does not depend on modelrig-probes). */
|
|
156
189
|
export interface ProbeResultFile {
|
|
157
190
|
readonly modelKey: string;
|
|
158
|
-
readonly class: "schema" | "grounding" | "caching";
|
|
191
|
+
readonly class: "schema" | "grounding" | "caching" | "image";
|
|
159
192
|
readonly date: string;
|
|
160
193
|
readonly harnessVersion: string;
|
|
161
194
|
/** fixtureId → family (demo-rig §11.3b). Absent on pre-family result files;
|
|
@@ -186,6 +219,15 @@ export declare function latestPerClass(results: readonly ProbeResultFile[]): Par
|
|
|
186
219
|
export declare function summarizeSchemaResult(result: ProbeResultFile): ProbedSchemaSummary;
|
|
187
220
|
export declare function summarizeGroundingResult(result: ProbeResultFile): ProbedGroundingSummary;
|
|
188
221
|
export declare function summarizeCachingResult(result: ProbeResultFile): ProbedCachingSummary;
|
|
222
|
+
/**
|
|
223
|
+
* Summarize an image-leg result (UC-3 M0). A sample is an EXTRACTION PASS iff it
|
|
224
|
+
* returned schema-conformant JSON (ajv) AND recovered every planted field
|
|
225
|
+
* (valueAccuracy 1.0) — a shape-right / values-wrong answer is the negative
|
|
226
|
+
* control and does NOT count. `extract_rate` is that pass rate over ALL samples
|
|
227
|
+
* (a failed call is a non-extraction), so a run of zero samples is `null`, never
|
|
228
|
+
* a confident 0%.
|
|
229
|
+
*/
|
|
230
|
+
export declare function summarizeImageResult(result: ProbeResultFile): ProbedImageSummary;
|
|
189
231
|
/** Compose a model's SCHEMA result files into one synthetic file, so a cheap
|
|
190
232
|
* SUPPLEMENTAL run (new fixtures only) ADDS to a prior full run instead of
|
|
191
233
|
* replacing it — the default latest-per-file rule would drop every fixture the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/registry/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE;QAChB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;QACpC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;QACrC,QAAQ,CAAC,yBAAyB,EAAE,MAAM,GAAG,IAAI,CAAC;KACnD,GAAG,IAAI,CAAC;IACT;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,wBAAwB,EAAE,OAAO,GAAG,IAAI,CAAC;QAClD,QAAQ,CAAC,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;QAC7C,QAAQ,CAAC,uBAAuB,EAAE,OAAO,GAAG,IAAI,CAAC;QACjD,uEAAuE;QACvE,QAAQ,CAAC,yBAAyB,EAAE,OAAO,GAAG,IAAI,CAAC;QACnD,QAAQ,CAAC,uBAAuB,EAAE,OAAO,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/registry/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE;QAChB,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;QACpC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;QACrC,QAAQ,CAAC,yBAAyB,EAAE,MAAM,GAAG,IAAI,CAAC;KACnD,GAAG,IAAI,CAAC;IACT;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,wBAAwB,EAAE,OAAO,GAAG,IAAI,CAAC;QAClD,QAAQ,CAAC,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;QAC7C,QAAQ,CAAC,uBAAuB,EAAE,OAAO,GAAG,IAAI,CAAC;QACjD,uEAAuE;QACvE,QAAQ,CAAC,yBAAyB,EAAE,OAAO,GAAG,IAAI,CAAC;QACnD,QAAQ,CAAC,uBAAuB,EAAE,OAAO,GAAG,IAAI,CAAC;QACjD;;;;;;iDAMyC;QACzC,QAAQ,CAAC,oBAAoB,EAAE,OAAO,GAAG,IAAI,CAAC;KAC/C,CAAC;CACH;AAED;;;;+EAI+E;AAC/E,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACxD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C;;;;;6DAKyD;IACzD,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC;;;;;kFAK8E;IAC9E,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACxD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;qEACiE;IACjE,QAAQ,CAAC,SAAS,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACnE;;;8BAG0B;IAC1B,QAAQ,CAAC,aAAa,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACvE;;;2EAGuE;IACvE,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;CACrE;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;;;;qCAIiC;IACjC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACxD;kFAC8E;IAC9E,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC;;;;uEAImE;IACnE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,SAAS,EAAE,sBAAsB,GAAG,IAAI,CAAC;IAClD,QAAQ,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAC9C;;oFAEgF;IAChF,QAAQ,CAAC,KAAK,CAAC,EAAE,kBAAkB,GAAG,IAAI,CAAC;CAC5C;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC;mEAC+D;IAC/D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC;;;;;OAKG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,iEAAiE;IACjE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7D,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,mBAAmB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAChE;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,SAAS,WAAW,EAAE,CAAC;IAC/C;kEAC8D;IAC9D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC;wEACoE;IACpE,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,CAAC;CAC3C;AAED;yEACyE;AACzE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,WAAW,GAAG,SAAS,GAAG,OAAO,CAAC;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;wEACoE;IACpE,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5D;+EAC2E;IAC3E,QAAQ,CAAC,mBAAmB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE;;;yEAGqE;IACrE,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5D,QAAQ,CAAC,GAAG,EAAE,QAAQ,CACpB,MAAM,CACJ,MAAM,EACN,aAAa,CAAC;QACZ,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;QACvC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;QAC1B,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;QACvC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,QAAQ,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;QAClC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QACrC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KACtC,CAAC,CACH,CACF,CAAC;CACH;AAyBD,wDAAwD;AACxD,wBAAgB,cAAc,CAC5B,OAAO,EAAE,SAAS,eAAe,EAAE,GAClC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,eAAe,CAAC,CAAC,CAS5D;AAyDD,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,eAAe,GAAG,mBAAmB,CAmClF;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,eAAe,GAAG,sBAAsB,CAQxF;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,eAAe,GAAG,oBAAoB,CAYpF;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,eAAe,GAAG,kBAAkB,CAchF;AAED;;;;;;;yEAOyE;AACzE,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,SAAS,eAAe,EAAE,EACnC,iBAAiB,EAAE,WAAW,CAAC,MAAM,CAAC,GACrC,eAAe,GAAG,SAAS,CA4B7B;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,SAAS,eAAe,EAAE,EACnC,uBAAuB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GAC5C,WAAW,GAAG,IAAI,CAuBpB;AAED;4EAC4E;AAC5E,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,aAAa,GAAG,IAAI,EAC9B,MAAM,EAAE,WAAW,GAAG,IAAI,GACzB,WAAW,EAAE,CA4Gf;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,aAAa,CAAC;IACrB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;CACzC,CAAC,EACF,WAAW,EAAE,MAAM,GAClB,QAAQ,CAgBV;AAED;;qDAEqD;AACrD,wBAAgB,4BAA4B,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAG9F"}
|
package/dist/registry/build.js
CHANGED
|
@@ -13,6 +13,7 @@ exports.latestPerClass = latestPerClass;
|
|
|
13
13
|
exports.summarizeSchemaResult = summarizeSchemaResult;
|
|
14
14
|
exports.summarizeGroundingResult = summarizeGroundingResult;
|
|
15
15
|
exports.summarizeCachingResult = summarizeCachingResult;
|
|
16
|
+
exports.summarizeImageResult = summarizeImageResult;
|
|
16
17
|
exports.mergeSchemaResults = mergeSchemaResults;
|
|
17
18
|
exports.buildProbedLayer = buildProbedLayer;
|
|
18
19
|
exports.detectDiscrepancies = detectDiscrepancies;
|
|
@@ -22,6 +23,10 @@ const call_notes_1 = require("./call-notes");
|
|
|
22
23
|
const claims_1 = require("./claims");
|
|
23
24
|
const CONFORMANCE_BAR = 0.9;
|
|
24
25
|
const GROUNDED_BAR = 0.5;
|
|
26
|
+
/** Schema-conformant-extraction bar for the image leg (UC-3 M0) — the same
|
|
27
|
+
* 0.9 the schema conformance detector uses; an image "capability" is a probed
|
|
28
|
+
* extract rate at or above it, never a declared claim. */
|
|
29
|
+
const IMAGE_BAR = 0.9;
|
|
25
30
|
function mean(values) {
|
|
26
31
|
if (values.length === 0)
|
|
27
32
|
return null;
|
|
@@ -148,6 +153,27 @@ function summarizeCachingResult(result) {
|
|
|
148
153
|
eligible_samples: n,
|
|
149
154
|
};
|
|
150
155
|
}
|
|
156
|
+
/**
|
|
157
|
+
* Summarize an image-leg result (UC-3 M0). A sample is an EXTRACTION PASS iff it
|
|
158
|
+
* returned schema-conformant JSON (ajv) AND recovered every planted field
|
|
159
|
+
* (valueAccuracy 1.0) — a shape-right / values-wrong answer is the negative
|
|
160
|
+
* control and does NOT count. `extract_rate` is that pass rate over ALL samples
|
|
161
|
+
* (a failed call is a non-extraction), so a run of zero samples is `null`, never
|
|
162
|
+
* a confident 0%.
|
|
163
|
+
*/
|
|
164
|
+
function summarizeImageResult(result) {
|
|
165
|
+
const samples = Object.values(result.raw).flat();
|
|
166
|
+
const n = samples.length;
|
|
167
|
+
const extracted = samples.filter((s) => s.schemaConform === true && s.valueAccuracy !== null && s.valueAccuracy >= 1).length;
|
|
168
|
+
const accuracies = samples.map((s) => s.valueAccuracy).filter((a) => a !== null);
|
|
169
|
+
return {
|
|
170
|
+
samples: n,
|
|
171
|
+
extract_rate: n === 0 ? null : extracted / n,
|
|
172
|
+
extract_ci95: n === 0 ? null : wilson95(extracted, n),
|
|
173
|
+
value_accuracy_mean: mean(accuracies),
|
|
174
|
+
mean_cost_usd: n === 0 ? 0 : samples.reduce((a, s) => a + s.costUsd, 0) / n,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
151
177
|
/** Compose a model's SCHEMA result files into one synthetic file, so a cheap
|
|
152
178
|
* SUPPLEMENTAL run (new fixtures only) ADDS to a prior full run instead of
|
|
153
179
|
* replacing it — the default latest-per-file rule would drop every fixture the
|
|
@@ -214,6 +240,10 @@ function buildProbedLayer(results, currentSchemaFixtureIds) {
|
|
|
214
240
|
schema: schemaFile !== undefined ? summarizeSchemaResult(schemaFile) : null,
|
|
215
241
|
grounding: byClass.grounding !== undefined ? summarizeGroundingResult(byClass.grounding) : null,
|
|
216
242
|
caching: byClass.caching !== undefined ? summarizeCachingResult(byClass.caching) : null,
|
|
243
|
+
// Image leg is latest-per-file like grounding/caching (one image result per
|
|
244
|
+
// cycle); the key is only emitted when the model was probed on it, so an
|
|
245
|
+
// unprobed model has no `image` block rather than a null-that-reads-as-no.
|
|
246
|
+
...(byClass.image !== undefined ? { image: summarizeImageResult(byClass.image) } : {}),
|
|
217
247
|
};
|
|
218
248
|
}
|
|
219
249
|
/** Declared-vs-probed disagreement detection — plan §4 (the rule pair) plus
|
|
@@ -284,6 +314,28 @@ function detectDiscrepancies(declared, probed) {
|
|
|
284
314
|
});
|
|
285
315
|
}
|
|
286
316
|
}
|
|
317
|
+
// Image leg (UC-3 M0). Fires only when the model was actually probed on the
|
|
318
|
+
// image class — an UNPROBED model produces no image discrepancy (unprobed ≠
|
|
319
|
+
// "cannot"; the whole detectDiscrepancies also early-returns when probed is
|
|
320
|
+
// null). A declared `supports_vision` claim can disagree with the measurement
|
|
321
|
+
// either way, and we surface the disagreement rather than pick a side.
|
|
322
|
+
const image = probed.image ?? null;
|
|
323
|
+
if (image !== null && image.extract_rate !== null && image.samples > 0) {
|
|
324
|
+
if (declared.flags.supports_image_input === true && image.extract_rate < IMAGE_BAR) {
|
|
325
|
+
out.push({
|
|
326
|
+
kind: "declared_image_low_extraction",
|
|
327
|
+
message: `declared image input (supports_vision), but probed schema-conformant extraction is ` +
|
|
328
|
+
`${(image.extract_rate * 100).toFixed(0)}% (bar: ${IMAGE_BAR * 100}%)`,
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
if (declared.flags.supports_image_input !== true && image.extract_rate >= IMAGE_BAR) {
|
|
332
|
+
out.push({
|
|
333
|
+
kind: "undeclared_image_capable",
|
|
334
|
+
message: `no declared image input support, but probed schema-conformant extraction is ` +
|
|
335
|
+
`${(image.extract_rate * 100).toFixed(0)}% — capability exists undeclared`,
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
}
|
|
287
339
|
return out;
|
|
288
340
|
}
|
|
289
341
|
function buildRegistry(entries, generatedAt) {
|
|
@@ -40,10 +40,39 @@ export interface ContextBand {
|
|
|
40
40
|
readonly above_tokens: number;
|
|
41
41
|
readonly rates: TokenRates;
|
|
42
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Typed per-provider image-input price (UC-3 M0). The BILLING MODEL varies by
|
|
45
|
+
* provider — some charge a flat fee per image, others tile the image into input
|
|
46
|
+
* tokens billed per-token — so it is a discriminated union, never averaged into
|
|
47
|
+
* one scalar. `source` + `as_of` are REQUIRED provenance (a sourced value is
|
|
48
|
+
* "declared" until an observed layer supersedes it). GOVERNING RULE (dev-rule
|
|
49
|
+
* L5 / spec §3): if a provider's image price cannot be sourced, this field is
|
|
50
|
+
* ABSENT — never guessed, never synthesized from the text rate.
|
|
51
|
+
*/
|
|
52
|
+
export type ImageInputPrice = {
|
|
53
|
+
/** A flat fee per image, USD (LiteLLM `input_cost_per_image`). */
|
|
54
|
+
readonly unit: "per_image";
|
|
55
|
+
readonly usd_per_image: number;
|
|
56
|
+
readonly source: string;
|
|
57
|
+
readonly as_of: string;
|
|
58
|
+
} | {
|
|
59
|
+
/** Per image-token, USD — the provider tiles the image into input tokens
|
|
60
|
+
* (LiteLLM `input_cost_per_image_token`). */
|
|
61
|
+
readonly unit: "per_token";
|
|
62
|
+
readonly usd_per_token: number;
|
|
63
|
+
readonly source: string;
|
|
64
|
+
readonly as_of: string;
|
|
65
|
+
};
|
|
43
66
|
/** Non-text input units, USD per unit. */
|
|
44
67
|
export interface ModalityRates {
|
|
45
68
|
readonly image_per_unit?: number | null;
|
|
46
69
|
readonly audio_per_second?: number | null;
|
|
70
|
+
/** Typed, provenance-bearing image-input price (UC-3 M0). Additive and
|
|
71
|
+
* optional; absent when the vendor's image price could not be sourced —
|
|
72
|
+
* absence is honest, never a zero or a guess. Supersedes the bare
|
|
73
|
+
* `image_per_unit` scalar for readers that understand the billing model (the
|
|
74
|
+
* scalar stays this pass — S3 order of destruction). */
|
|
75
|
+
readonly image_input?: ImageInputPrice | null;
|
|
47
76
|
}
|
|
48
77
|
export interface ModelPricingV2 {
|
|
49
78
|
/** standard-tier text — the only REQUIRED rates block. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pricing-shape.d.ts","sourceRoot":"","sources":["../../src/registry/pricing-shape.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,+BAA+B;IAC/B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC;kFAC8E;IAC9E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzC;AAED,8DAA8D;AAC9D,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,0DAA0D;AAC1D,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;CAC5B;AAED,0CAA0C;AAC1C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"pricing-shape.d.ts","sourceRoot":"","sources":["../../src/registry/pricing-shape.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,+BAA+B;IAC/B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC;kFAC8E;IAC9E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzC;AAED,8DAA8D;AAC9D,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,0DAA0D;AAC1D,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;CAC5B;AAED;;;;;;;;GAQG;AACH,MAAM,MAAM,eAAe,GACvB;IACE,kEAAkE;IAClE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,GACD;IACE;iDAC6C;IAC7C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAEN,0CAA0C;AAC1C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C;;;;4DAIwD;IACxD,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS,WAAW,EAAE,GAAG,IAAI,CAAC;IACvD,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACzC,iDAAiD;IACjD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED;iFACiF;AACjF,eAAO,MAAM,cAAc,0CAAgC,CAAC;AAE5D;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,GAAG,IAAI,EACzB,IAAI,EAAE,MAAM,EACZ,MAAM,SAAqB,GAC1B,cAAc,CAWhB;AAED,MAAM,MAAM,eAAe,GAAG,UAAU,GAAG,MAAM,GAAG,WAAW,CAAC;AAMhE;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,eAAe,GAAG,UAAU,GAAG,IAAI,CAU3F;AAED;gFACgF;AAChF,MAAM,WAAW,gBAAgB;IAC/B,wEAAwE;IACxE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,8EAA8E;IAC9E,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,wDAAwD;IACxD,QAAQ,CAAC,KAAK,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACrF,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,2EAA2E;IAC3E,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAqBD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,cAAc,GAAG,gBAAgB,CA+C3E"}
|
|
@@ -99,7 +99,16 @@ function describePricingV2(pricing) {
|
|
|
99
99
|
}
|
|
100
100
|
const contextBands = (pricing.context_bands ?? []).map((b) => `above ${b.above_tokens.toLocaleString()} tok: ${inOut(b.rates)}`);
|
|
101
101
|
const modality = [];
|
|
102
|
-
|
|
102
|
+
// Typed image-input price wins over the bare scalar when present (UC-3 M0) —
|
|
103
|
+
// it carries the billing model, so a reader is never left guessing per-image
|
|
104
|
+
// vs per-token. Absent ⇒ nothing rendered (honest absence, never a $0).
|
|
105
|
+
const imageInput = pricing.modality?.image_input;
|
|
106
|
+
if (imageInput != null) {
|
|
107
|
+
modality.push(imageInput.unit === "per_image"
|
|
108
|
+
? `image: $${imageInput.usd_per_image.toFixed(4)} per image`
|
|
109
|
+
: `image: ${perMtok(imageInput.usd_per_token)} per Mtok (tiled to tokens)`);
|
|
110
|
+
}
|
|
111
|
+
else if (pricing.modality?.image_per_unit != null) {
|
|
103
112
|
modality.push(`image: $${pricing.modality.image_per_unit.toFixed(4)} per unit`);
|
|
104
113
|
}
|
|
105
114
|
if (pricing.modality?.audio_per_second != null) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rows.d.ts","sourceRoot":"","sources":["../../src/registry/rows.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"rows.d.ts","sourceRoot":"","sources":["../../src/registry/rows.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AA0C/C;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,cAAc,EAAE,CA4BxE"}
|
package/dist/registry/rows.js
CHANGED
|
@@ -49,6 +49,7 @@ function loadRegistryRows(registryPath) {
|
|
|
49
49
|
nativeRungRate: entry.probed?.schema?.native_rung_rate ?? null,
|
|
50
50
|
groundedRate: entry.probed?.grounding?.grounded_rate ?? null,
|
|
51
51
|
cacheHitRate: entry.probed?.caching?.cache_hit_rate ?? null,
|
|
52
|
+
imageExtractRate: entry.probed?.image?.extract_rate ?? null,
|
|
52
53
|
effectiveUsdPer1k: effective.get(entry.model_key) ?? null,
|
|
53
54
|
discrepancies: entry.discrepancies?.length ?? 0,
|
|
54
55
|
probedAsOf: entry.probed?.as_of ?? null,
|
|
@@ -18,7 +18,7 @@ import type { CapabilityFlag } from "../types/provider";
|
|
|
18
18
|
import type { Requirement } from "../types/route";
|
|
19
19
|
/** The capability predicates a question can carry. Deliberately the probed
|
|
20
20
|
* dimensions only — we answer from measurement, not from provider claims. */
|
|
21
|
-
export type CapabilityPredicate = "structured-native" | "structured-coached" | "grounding" | "caching";
|
|
21
|
+
export type CapabilityPredicate = "structured-native" | "structured-coached" | "grounding" | "caching" | "image_input";
|
|
22
22
|
export declare const ALL_CAPABILITY_PREDICATES: readonly CapabilityPredicate[];
|
|
23
23
|
/** Plain-language gloss per predicate — the explainability layer travels
|
|
24
24
|
* WITH the answer (console-explainability.md §1), it is not a separate doc. */
|
|
@@ -34,6 +34,11 @@ export interface ModelSelectRow {
|
|
|
34
34
|
readonly nativeRungRate: number | null;
|
|
35
35
|
readonly groundedRate: number | null;
|
|
36
36
|
readonly cacheHitRate: number | null;
|
|
37
|
+
/** Probed image-input schema-conformant extraction rate (UC-3 M0). `null` when
|
|
38
|
+
* the model has NOT been probed on the image class — which the filter treats
|
|
39
|
+
* as "does not qualify" (a match needs measured evidence), NEVER as "cannot"
|
|
40
|
+
* (that distinction is preserved by the surfaces that render it). */
|
|
41
|
+
readonly imageExtractRate: number | null;
|
|
37
42
|
readonly effectiveUsdPer1k: number | null;
|
|
38
43
|
readonly discrepancies: number;
|
|
39
44
|
readonly probedAsOf: string | null;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"select.d.ts","sourceRoot":"","sources":["../../src/registry/select.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAElD;6EAC6E;AAC7E,MAAM,MAAM,mBAAmB,GAC3B,mBAAmB,GACnB,oBAAoB,GACpB,WAAW,GACX,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"select.d.ts","sourceRoot":"","sources":["../../src/registry/select.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAElD;6EAC6E;AAC7E,MAAM,MAAM,mBAAmB,GAC3B,mBAAmB,GACnB,oBAAoB,GACpB,WAAW,GACX,SAAS,GACT,aAAa,CAAC;AAElB,eAAO,MAAM,yBAAyB,EAAE,SAAS,mBAAmB,EAMnE,CAAC;AAEF;+EAC+E;AAC/E,eAAO,MAAM,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAUzE,CAAC;AAEF;gFACgF;AAChF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC;;;yEAGqE;IACrE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAOD,wBAAgB,aAAa,CAAC,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,mBAAmB,GAAG,OAAO,CAiBpF;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC9C,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,gEAAgE;IAChE,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AA2BD;;;mDAGmD;AACnD,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,CAkB/D;AAED,MAAM,WAAW,cAAc;IAC7B,kEAAkE;IAClE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC/C,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IACrC,mDAAmD;IACnD,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC;mEAC+D;IAC/D,QAAQ,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IACvC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,SAAS,cAAc,EAAE,EAC/B,QAAQ,GAAE,cAAmB,GAC5B,cAAc,EAAE,CA6BlB;AAED;;;;;;;;;;;;;;GAcG;AAEH;;uDAEuD;AACvD,MAAM,MAAM,oBAAoB,GAAG,SAAS,cAAc,EAAE,CAAC;AAE7D,MAAM,WAAW,gBAAgB;IAC/B,yEAAyE;IACzE,QAAQ,CAAC,UAAU,EAAE,SAAS,oBAAoB,EAAE,CAAC;IACrD;;;wEAGoE;IACpE,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,QAAQ,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,WAAW,CAAA;KAAE,CAAC;CAC5D,GAAG,gBAAgB,CAsBnB;AAeD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,+EAA+E;IAC/E,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;GAMG;AACH,wBAAgB,6BAA6B,CAC3C,GAAG,EAAE,gBAAgB,EACrB,KAAK,EAAE,WAAW,CAAC,cAAc,CAAC,EAClC,eAAe,UAAQ,GACtB,qBAAqB,CAYvB;AAED;0EAC0E;AAC1E,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,CAatE"}
|
package/dist/registry/select.js
CHANGED
|
@@ -28,6 +28,7 @@ exports.ALL_CAPABILITY_PREDICATES = [
|
|
|
28
28
|
"structured-coached",
|
|
29
29
|
"grounding",
|
|
30
30
|
"caching",
|
|
31
|
+
"image_input",
|
|
31
32
|
];
|
|
32
33
|
/** Plain-language gloss per predicate — the explainability layer travels
|
|
33
34
|
* WITH the answer (console-explainability.md §1), it is not a separate doc. */
|
|
@@ -36,9 +37,12 @@ exports.PREDICATE_GLOSS = {
|
|
|
36
37
|
"structured-coached": "output matched the schema at least 90% of probed samples, whether or not the provider enforced it natively",
|
|
37
38
|
grounding: "at least half of probed answers cited real retrieved sources rather than answering from memory",
|
|
38
39
|
caching: "repeat calls actually reported cached input tokens — a measured discount, not a claim",
|
|
40
|
+
image_input: "a probe sent a form image and the model returned schema-conformant JSON extracting the planted fields on at least 90% of samples — a MEASURED read of an image, never a provider's vision claim",
|
|
39
41
|
};
|
|
40
42
|
const CONFORMANCE_BAR = 0.9;
|
|
41
43
|
const GROUNDED_BAR = 0.5;
|
|
44
|
+
/** Image-input extraction bar (UC-3 M0) — the same 0.9 the schema bar uses. */
|
|
45
|
+
const IMAGE_BAR = 0.9;
|
|
42
46
|
function hasCapability(row, cap) {
|
|
43
47
|
switch (cap) {
|
|
44
48
|
case "structured-native":
|
|
@@ -49,6 +53,12 @@ function hasCapability(row, cap) {
|
|
|
49
53
|
return (row.groundedRate ?? 0) >= GROUNDED_BAR;
|
|
50
54
|
case "caching":
|
|
51
55
|
return (row.cacheHitRate ?? 0) > 0;
|
|
56
|
+
case "image_input":
|
|
57
|
+
// Probed-only: an unprobed model (null) is never returned as image-capable
|
|
58
|
+
// — absence is not a match. It is also not "cannot": the filter answers
|
|
59
|
+
// "which models we MEASURED can do this", and the empty-selection prose
|
|
60
|
+
// says exactly that.
|
|
61
|
+
return (row.imageExtractRate ?? 0) >= IMAGE_BAR;
|
|
52
62
|
}
|
|
53
63
|
}
|
|
54
64
|
/** Question scaffolding. A search box gets keywords; the MCP oracle gets a
|
|
@@ -71,6 +81,7 @@ const CAP_WORDS = [
|
|
|
71
81
|
[["json", "strict", "structured", "schema"], "structured-native"],
|
|
72
82
|
[["grounded", "grounding", "search", "web"], "grounding"],
|
|
73
83
|
[["cache", "caching", "cached"], "caching"],
|
|
84
|
+
[["image", "images", "vision", "multimodal", "ocr"], "image_input"],
|
|
74
85
|
];
|
|
75
86
|
/** The tiny grammar behind the query box and the oracle's free-text input:
|
|
76
87
|
* capability words + an "under $N/M" price cap; everything else is a
|