modelrig 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/README.md +41 -11
- package/data/PROVENANCE.md +19 -0
- package/data/pricing-manual.json +22 -0
- package/data/pricing-map.json +46362 -0
- package/data/pricing-overlay.json +419 -0
- package/dist/adapters/anthropic.d.ts +50 -6
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +123 -29
- package/dist/adapters/build-adapters.d.ts +11 -0
- package/dist/adapters/build-adapters.d.ts.map +1 -0
- package/dist/adapters/build-adapters.js +45 -0
- package/dist/adapters/deepseek.d.ts +6 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +54 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +46 -6
- package/dist/adapters/grok.d.ts +9 -0
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +67 -3
- package/dist/adapters/openai-compat.d.ts +6 -0
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +58 -1
- package/dist/adapters/openai-stream.d.ts +58 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -0
- package/dist/adapters/openai-stream.js +120 -0
- package/dist/adapters/openai.d.ts +6 -0
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +60 -1
- package/dist/adapters/sampling.d.ts +28 -0
- package/dist/adapters/sampling.d.ts.map +1 -0
- package/dist/adapters/sampling.js +34 -0
- package/dist/adapters/tool-mapping.d.ts +92 -0
- package/dist/adapters/tool-mapping.d.ts.map +1 -0
- package/dist/adapters/tool-mapping.js +143 -0
- package/dist/artifacts/artifact.d.ts +137 -0
- package/dist/artifacts/artifact.d.ts.map +1 -0
- package/dist/artifacts/artifact.js +377 -0
- package/dist/artifacts/content-uploader.d.ts +87 -0
- package/dist/artifacts/content-uploader.d.ts.map +1 -0
- package/dist/artifacts/content-uploader.js +124 -0
- package/dist/artifacts/index.d.ts +63 -0
- package/dist/artifacts/index.d.ts.map +1 -0
- package/dist/artifacts/index.js +80 -0
- package/dist/artifacts/run-context.d.ts +129 -0
- package/dist/artifacts/run-context.d.ts.map +1 -0
- package/dist/artifacts/run-context.js +208 -0
- package/dist/artifacts/scrub.d.ts +60 -0
- package/dist/artifacts/scrub.d.ts.map +1 -0
- package/dist/artifacts/scrub.js +139 -0
- package/dist/artifacts/step.d.ts +26 -0
- package/dist/artifacts/step.d.ts.map +1 -0
- package/dist/artifacts/step.js +37 -0
- package/dist/artifacts/taint.d.ts +63 -0
- package/dist/artifacts/taint.d.ts.map +1 -0
- package/dist/artifacts/taint.js +76 -0
- package/dist/bakeoff/cache-sim.d.ts +99 -0
- package/dist/bakeoff/cache-sim.d.ts.map +1 -0
- package/dist/bakeoff/cache-sim.js +126 -0
- package/dist/bakeoff/eval-cases.d.ts +96 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases.js +105 -0
- package/dist/bakeoff/report.d.ts +41 -0
- package/dist/bakeoff/report.d.ts.map +1 -0
- package/dist/bakeoff/report.js +142 -0
- package/dist/bakeoff/samples.d.ts +15 -0
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +20 -0
- package/dist/cli.d.ts +1 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +134 -116
- package/dist/config.d.ts +100 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +104 -0
- package/dist/deprecations.d.ts +32 -0
- package/dist/deprecations.d.ts.map +1 -0
- package/dist/deprecations.js +59 -0
- package/dist/gateway/http-protocol.d.ts +175 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -0
- package/dist/gateway/http-protocol.js +105 -0
- package/dist/index.d.ts +105 -18
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +266 -138
- package/dist/init/finder.d.ts +15 -0
- package/dist/init/finder.d.ts.map +1 -1
- package/dist/init/finder.js +46 -0
- package/dist/init/init-cli.d.ts +10 -0
- package/dist/init/init-cli.d.ts.map +1 -0
- package/dist/init/init-cli.js +86 -0
- package/dist/init/observe-cli.d.ts +19 -0
- package/dist/init/observe-cli.d.ts.map +1 -0
- package/dist/init/observe-cli.js +73 -0
- package/dist/init/observe.d.ts +81 -0
- package/dist/init/observe.d.ts.map +1 -0
- package/dist/init/observe.js +245 -0
- package/dist/registry/build.d.ts +94 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +125 -4
- package/dist/registry/cache-policy.d.ts +94 -0
- package/dist/registry/cache-policy.d.ts.map +1 -0
- package/dist/registry/cache-policy.js +238 -0
- package/dist/registry/call-notes.d.ts.map +1 -1
- package/dist/registry/call-notes.js +40 -2
- package/dist/registry/effective-rate.d.ts +151 -0
- package/dist/registry/effective-rate.d.ts.map +1 -0
- package/dist/registry/effective-rate.js +156 -0
- package/dist/registry/facts.d.ts.map +1 -1
- package/dist/registry/facts.js +16 -0
- package/dist/registry/parity.d.ts +26 -0
- package/dist/registry/parity.d.ts.map +1 -1
- package/dist/registry/parity.js +43 -2
- package/dist/registry/pricing-shape.d.ts +113 -0
- package/dist/registry/pricing-shape.d.ts.map +1 -0
- package/dist/registry/pricing-shape.js +123 -0
- package/dist/registry/pricing.d.ts +25 -1
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +64 -6
- package/dist/rigindex/bootstrap.d.ts +36 -0
- package/dist/rigindex/bootstrap.d.ts.map +1 -0
- package/dist/rigindex/bootstrap.js +97 -0
- package/dist/rigindex/fit.d.ts +24 -0
- package/dist/rigindex/fit.d.ts.map +1 -0
- package/dist/rigindex/fit.js +76 -0
- package/dist/rigindex/gates.d.ts +52 -0
- package/dist/rigindex/gates.d.ts.map +1 -0
- package/dist/rigindex/gates.js +109 -0
- package/dist/rigindex/graph.d.ts +31 -0
- package/dist/rigindex/graph.d.ts.map +1 -0
- package/dist/rigindex/graph.js +120 -0
- package/dist/rigindex/index.d.ts +23 -0
- package/dist/rigindex/index.d.ts.map +1 -0
- package/dist/rigindex/index.js +112 -0
- package/dist/rigindex/types.d.ts +76 -0
- package/dist/rigindex/types.d.ts.map +1 -0
- package/dist/rigindex/types.js +8 -0
- package/dist/routes/cache-lint.d.ts +41 -0
- package/dist/routes/cache-lint.d.ts.map +1 -0
- package/dist/routes/cache-lint.js +81 -0
- package/dist/routes/loader.d.ts +37 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +138 -0
- package/dist/routes/resolved-bundle.d.ts +79 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -0
- package/dist/routes/resolved-bundle.js +148 -0
- package/dist/routes/sampling-config.d.ts +32 -0
- package/dist/routes/sampling-config.d.ts.map +1 -0
- package/dist/routes/sampling-config.js +91 -0
- package/dist/run/ajv.d.ts +22 -0
- package/dist/run/ajv.d.ts.map +1 -0
- package/dist/run/ajv.js +32 -0
- package/dist/run/cache-handle.d.ts +58 -0
- package/dist/run/cache-handle.d.ts.map +1 -0
- package/dist/run/cache-handle.js +54 -0
- package/dist/run/cache-stamp.d.ts +65 -0
- package/dist/run/cache-stamp.d.ts.map +1 -0
- package/dist/run/cache-stamp.js +0 -0
- package/dist/run/capture-write.d.ts +33 -0
- package/dist/run/capture-write.d.ts.map +1 -0
- package/dist/run/capture-write.js +55 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +1 -1
- package/dist/run/dispatch.d.ts +99 -0
- package/dist/run/dispatch.d.ts.map +1 -0
- package/dist/run/dispatch.js +191 -0
- package/dist/run/raw.d.ts +88 -0
- package/dist/run/raw.d.ts.map +1 -0
- package/dist/run/raw.js +286 -0
- package/dist/run/record-attempt.d.ts +40 -0
- package/dist/run/record-attempt.d.ts.map +1 -0
- package/dist/run/record-attempt.js +99 -0
- package/dist/run/resolve.d.ts +75 -2
- package/dist/run/resolve.d.ts.map +1 -1
- package/dist/run/resolve.js +126 -4
- package/dist/run/run.d.ts +14 -0
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +0 -0
- package/dist/run/stream.d.ts +49 -0
- package/dist/run/stream.d.ts.map +1 -0
- package/dist/run/stream.js +87 -0
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +6 -7
- package/dist/savings/cache-ledger.d.ts +108 -0
- package/dist/savings/cache-ledger.d.ts.map +1 -0
- package/dist/savings/cache-ledger.js +174 -0
- package/dist/savings/cache-reconcile.d.ts +64 -0
- package/dist/savings/cache-reconcile.d.ts.map +1 -0
- package/dist/savings/cache-reconcile.js +59 -0
- package/dist/savings/cache-stats.d.ts +63 -0
- package/dist/savings/cache-stats.d.ts.map +1 -0
- package/dist/savings/cache-stats.js +108 -0
- package/dist/savings/emit.d.ts +13 -0
- package/dist/savings/emit.d.ts.map +1 -1
- package/dist/savings/emit.js +25 -0
- package/dist/savings/swap-execute.d.ts +14 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +44 -1
- package/dist/savings/swaps-client.d.ts +14 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/savings/swaps-client.js +16 -2
- package/dist/settings/api-fetcher.d.ts +39 -0
- package/dist/settings/api-fetcher.d.ts.map +1 -0
- package/dist/settings/api-fetcher.js +92 -0
- package/dist/settings/index.d.ts +10 -0
- package/dist/settings/index.d.ts.map +1 -0
- package/dist/settings/index.js +15 -0
- package/dist/settings/task-settings.d.ts +91 -0
- package/dist/settings/task-settings.d.ts.map +1 -0
- package/dist/settings/task-settings.js +116 -0
- package/dist/status.d.ts +77 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +113 -0
- package/dist/telemetry/artifact-store.d.ts +48 -0
- package/dist/telemetry/artifact-store.d.ts.map +1 -0
- package/dist/telemetry/artifact-store.js +189 -0
- package/dist/telemetry/cache-miss-streak.d.ts +79 -0
- package/dist/telemetry/cache-miss-streak.d.ts.map +1 -0
- package/dist/telemetry/cache-miss-streak.js +99 -0
- package/dist/telemetry/capture.d.ts +3 -0
- package/dist/telemetry/capture.d.ts.map +1 -1
- package/dist/telemetry/capture.js +15 -8
- package/dist/telemetry/exporter-setup.d.ts +28 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -0
- package/dist/telemetry/exporter-setup.js +34 -0
- package/dist/telemetry/exporter.d.ts +75 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +324 -47
- package/dist/telemetry/health.d.ts +76 -0
- package/dist/telemetry/health.d.ts.map +1 -0
- package/dist/telemetry/health.js +92 -0
- package/dist/telemetry/ingest-protocol.d.ts +152 -3
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +204 -4
- package/dist/telemetry/load-error-report.d.ts +24 -0
- package/dist/telemetry/load-error-report.d.ts.map +1 -0
- package/dist/telemetry/load-error-report.js +50 -0
- package/dist/telemetry/otel-map.d.ts +121 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -0
- package/dist/telemetry/otel-map.js +224 -0
- package/dist/telemetry/otel-normalize.d.ts +104 -0
- package/dist/telemetry/otel-normalize.d.ts.map +1 -0
- package/dist/telemetry/otel-normalize.js +530 -0
- package/dist/telemetry/otlp-emitter.d.ts +52 -0
- package/dist/telemetry/otlp-emitter.d.ts.map +1 -0
- package/dist/telemetry/otlp-emitter.js +144 -0
- package/dist/telemetry/otlp-proto.d.ts +29 -0
- package/dist/telemetry/otlp-proto.d.ts.map +1 -0
- package/dist/telemetry/otlp-proto.js +266 -0
- package/dist/telemetry/sinks.d.ts +37 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +121 -30
- package/dist/telemetry/store.d.ts +28 -1
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +218 -3
- package/dist/tenancy/api-key-format.d.ts +15 -5
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +24 -5
- package/dist/types/config-teaching.d.ts +62 -0
- package/dist/types/config-teaching.d.ts.map +1 -0
- package/dist/types/config-teaching.js +183 -0
- package/dist/types/provider.d.ts +122 -1
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +21 -1
- package/dist/types/route.d.ts +79 -1
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +117 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts +12 -0
- package/dist/validate-cli.d.ts.map +1 -0
- package/dist/validate-cli.js +57 -0
- package/dist/verify/capture.d.ts +66 -0
- package/dist/verify/capture.d.ts.map +1 -0
- package/dist/verify/capture.js +180 -0
- package/dist/verify/cli-run.d.ts +23 -0
- package/dist/verify/cli-run.d.ts.map +1 -0
- package/dist/verify/cli-run.js +57 -0
- package/dist/verify/diff.d.ts +48 -0
- package/dist/verify/diff.d.ts.map +1 -0
- package/dist/verify/diff.js +157 -0
- package/dist/verify/index.d.ts +17 -0
- package/dist/verify/index.d.ts.map +1 -0
- package/dist/verify/index.js +24 -0
- package/dist/verify/report.d.ts +14 -0
- package/dist/verify/report.d.ts.map +1 -0
- package/dist/verify/report.js +88 -0
- package/dist/verify/types.d.ts +101 -0
- package/dist/verify/types.d.ts.map +1 -0
- package/dist/verify/types.js +17 -0
- package/dist/verify/verify.d.ts +29 -0
- package/dist/verify/verify.d.ts.map +1 -0
- package/dist/verify/verify.js +148 -0
- package/dist/watch/evaluate.d.ts +162 -3
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +61 -0
- package/dist/watch/monitor.d.ts +116 -0
- package/dist/watch/monitor.d.ts.map +1 -0
- package/dist/watch/monitor.js +199 -0
- package/dist/watch/proposals.d.ts +20 -4
- package/dist/watch/proposals.d.ts.map +1 -1
- package/dist/watch/proposals.js +31 -3
- package/dist/watch/run.d.ts +33 -0
- package/dist/watch/run.d.ts.map +1 -1
- package/dist/watch/run.js +39 -0
- package/package.json +7 -1
- package/registry/LICENSE +20 -0
- package/registry/leaderboard.json +3639 -0
- package/registry/registry.json +7263 -0
- package/dist/telemetry/supabase-exporter.d.ts +0 -64
- package/dist/telemetry/supabase-exporter.d.ts.map +0 -1
- package/dist/telemetry/supabase-exporter.js +0 -211
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fit.d.ts","sourceRoot":"","sources":["../../src/rigindex/fit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE5C,eAAO,MAAM,YAAY,MAAM,CAAC;AAChC,eAAO,MAAM,cAAc,QAAQ,CAAC;AACpC,eAAO,MAAM,WAAW,OAAO,CAAC;AAEhC;qDACqD;AACrD,wBAAgB,eAAe,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAqC3E;AAED;;iDAEiD;AACjD,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,EAC3C,MAAM,EAAE,SAAS,MAAM,EAAE,GACxB,MAAM,GAAG,IAAI,CAIf;AAED,+EAA+E;AAC/E,wBAAgB,YAAY,CAC1B,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,EAC3C,MAAM,EAAE,SAAS,MAAM,EAAE,GACxB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAM/B"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Bradley-Terry fit + frontier anchor — RATIFIED (stop2-architect-review §1):
|
|
4
|
+
* Hunter's MM algorithm, 400 iterations or convergence, symmetric 0.01
|
|
5
|
+
* pseudo-wins per ordered pair (keeps saturated all-tie graphs finite and the
|
|
6
|
+
* graph connected), geometric-mean-normalized per iteration. Anchor:
|
|
7
|
+
* RigIndex_i = 100 × p_i / GM(basket strengths) — scores >100 are expected
|
|
8
|
+
* ("above the frontier average").
|
|
9
|
+
*
|
|
10
|
+
* Any change to the solver, regularization, or anchor re-opens STOP-2.
|
|
11
|
+
*/
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.PSEUDO_WINS = exports.MM_CONVERGENCE = exports.MM_MAX_ITERS = void 0;
|
|
14
|
+
exports.fitBradleyTerry = fitBradleyTerry;
|
|
15
|
+
exports.basketGeometricMean = basketGeometricMean;
|
|
16
|
+
exports.anchorScores = anchorScores;
|
|
17
|
+
exports.MM_MAX_ITERS = 400;
|
|
18
|
+
exports.MM_CONVERGENCE = 1e-10;
|
|
19
|
+
exports.PSEUDO_WINS = 0.01;
|
|
20
|
+
/** Fit BT strengths over the graph's models. Returns modelKey → p, GM-normalized
|
|
21
|
+
* to 1. Deterministic — no randomness in the fit. */
|
|
22
|
+
function fitBradleyTerry(graph) {
|
|
23
|
+
const models = graph.models;
|
|
24
|
+
const n = models.length;
|
|
25
|
+
if (n === 0)
|
|
26
|
+
return {};
|
|
27
|
+
if (n === 1)
|
|
28
|
+
return { [models[0]]: 1 };
|
|
29
|
+
// Effective wins with the symmetric pseudo-win regularization: every
|
|
30
|
+
// ordered pair (i, j), i ≠ j, gains PSEUDO_WINS wins of i over j.
|
|
31
|
+
const w = models.map((a) => models.map((b) => (a === b ? 0 : graph.wins[a][b] + exports.PSEUDO_WINS)));
|
|
32
|
+
// Total comparisons per pair (symmetric).
|
|
33
|
+
const t = models.map((_, i) => models.map((_, j) => w[i][j] + w[j][i]));
|
|
34
|
+
const totalWins = models.map((_, i) => w[i].reduce((s, x) => s + x, 0));
|
|
35
|
+
let p = new Array(n).fill(1);
|
|
36
|
+
for (let iter = 0; iter < exports.MM_MAX_ITERS; iter += 1) {
|
|
37
|
+
const next = new Array(n);
|
|
38
|
+
for (let i = 0; i < n; i += 1) {
|
|
39
|
+
let denom = 0;
|
|
40
|
+
for (let j = 0; j < n; j += 1) {
|
|
41
|
+
if (j === i)
|
|
42
|
+
continue;
|
|
43
|
+
denom += t[i][j] / (p[i] + p[j]);
|
|
44
|
+
}
|
|
45
|
+
next[i] = denom > 0 ? totalWins[i] / denom : p[i];
|
|
46
|
+
}
|
|
47
|
+
// Geometric-mean normalize per iteration (review §1).
|
|
48
|
+
const gm = Math.exp(next.reduce((s, x) => s + Math.log(x), 0) / n);
|
|
49
|
+
for (let i = 0; i < n; i += 1)
|
|
50
|
+
next[i] /= gm;
|
|
51
|
+
let maxDelta = 0;
|
|
52
|
+
for (let i = 0; i < n; i += 1) {
|
|
53
|
+
maxDelta = Math.max(maxDelta, Math.abs(Math.log(next[i] / p[i])));
|
|
54
|
+
}
|
|
55
|
+
p = next;
|
|
56
|
+
if (maxDelta < exports.MM_CONVERGENCE)
|
|
57
|
+
break;
|
|
58
|
+
}
|
|
59
|
+
return Object.fromEntries(models.map((m, i) => [m, p[i]]));
|
|
60
|
+
}
|
|
61
|
+
/** Geometric mean of the basket members' strengths, over the members present
|
|
62
|
+
* in the fit. Null when NO basket member is in the graph — the subtype cannot
|
|
63
|
+
* be anchored (and must not publish a score). */
|
|
64
|
+
function basketGeometricMean(strengths, basket) {
|
|
65
|
+
const present = basket.filter((m) => strengths[m] !== undefined);
|
|
66
|
+
if (present.length === 0)
|
|
67
|
+
return null;
|
|
68
|
+
return Math.exp(present.reduce((s, m) => s + Math.log(strengths[m]), 0) / present.length);
|
|
69
|
+
}
|
|
70
|
+
/** RigIndex_i = 100 × p_i / GM(basket) (review §1). Null when unanchorable. */
|
|
71
|
+
function anchorScores(strengths, basket) {
|
|
72
|
+
const gm = basketGeometricMean(strengths, basket);
|
|
73
|
+
if (gm === null)
|
|
74
|
+
return null;
|
|
75
|
+
return Object.fromEntries(Object.entries(strengths).map(([m, p]) => [m, (100 * p) / gm]));
|
|
76
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Believability gates — RATIFIED thresholds (stop2-architect-review §1):
|
|
3
|
+
* - min-n: ≥10 DISTINCT outcomes per model per subtype AND ≥2 fixtures.
|
|
4
|
+
* Never gate on raw pair counts (publish both; gate on distinct).
|
|
5
|
+
* - CI tie-tiers: overlapping CIs render as ONE tier, never a fake ordering.
|
|
6
|
+
* - SEPARATION GATE: a subtype publishes a rank view only when its measured
|
|
7
|
+
* models form ≥2 CI-distinct tiers; otherwise it renders the all-tie state.
|
|
8
|
+
* Threshold changes re-open STOP-2.
|
|
9
|
+
*/
|
|
10
|
+
export declare const MIN_DISTINCT_OUTCOMES = 10;
|
|
11
|
+
export declare const MIN_FIXTURES = 2;
|
|
12
|
+
export interface GateInput {
|
|
13
|
+
readonly model: string;
|
|
14
|
+
readonly score: number;
|
|
15
|
+
readonly ci95: readonly [number, number];
|
|
16
|
+
readonly distinctN: number;
|
|
17
|
+
readonly fixtures: number;
|
|
18
|
+
}
|
|
19
|
+
export interface GateResult {
|
|
20
|
+
readonly model: string;
|
|
21
|
+
readonly gated: boolean;
|
|
22
|
+
readonly gateReason: string | null;
|
|
23
|
+
/** 1-based tie-tier among ungated models; null when gated. */
|
|
24
|
+
readonly tier: number | null;
|
|
25
|
+
}
|
|
26
|
+
/** The min-n verdict for one model — null when it passes. */
|
|
27
|
+
export declare function minNReason(distinctN: number, fixtures: number): string | null;
|
|
28
|
+
/** Interval arithmetic tolerance, in RigIndex points. The MM fit + GM
|
|
29
|
+
* normalization leave ~1e-13 float dust on saturated scores; a genuine CI
|
|
30
|
+
* overlap is orders of magnitude wider. One millionth of a point separates
|
|
31
|
+
* the two regimes cleanly. */
|
|
32
|
+
export declare const TIER_EPSILON = 0.000001;
|
|
33
|
+
/**
|
|
34
|
+
* Assign tie-tiers to the ungated models: sort by score descending; walking
|
|
35
|
+
* down, a model joins the current tier iff its CI overlaps the CI of at least
|
|
36
|
+
* one model already in that tier; otherwise a new tier starts. Overlap chains
|
|
37
|
+
* deliberately — a bridge model with a wide CI merges the models it straddles
|
|
38
|
+
* into one tier rather than manufacturing an ordering the data can't support.
|
|
39
|
+
*/
|
|
40
|
+
export declare function assignTiers(inputs: readonly GateInput[]): GateResult[];
|
|
41
|
+
/** Number of CI-distinct tiers among ungated models. */
|
|
42
|
+
export declare function tierCount(results: readonly GateResult[]): number;
|
|
43
|
+
/**
|
|
44
|
+
* The separation-gate state sentence for a one-tier subtype — authored once
|
|
45
|
+
* here so the layer and the www render the SAME ratified words (review §1:
|
|
46
|
+
* "all measured models tie on this task type — no rank distinguishable").
|
|
47
|
+
* `n` is the total distinct outcomes across the tied models.
|
|
48
|
+
*/
|
|
49
|
+
export declare function allTieSentence(modelCount: number, n: number): string;
|
|
50
|
+
/** The insufficient-data state sentence (min-n gate, review §1). */
|
|
51
|
+
export declare function insufficientSentence(n: number): string;
|
|
52
|
+
//# sourceMappingURL=gates.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gates.d.ts","sourceRoot":"","sources":["../../src/rigindex/gates.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,eAAO,MAAM,qBAAqB,KAAK,CAAC;AACxC,eAAO,MAAM,YAAY,IAAI,CAAC;AAE9B,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,8DAA8D;IAC9D,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED,6DAA6D;AAC7D,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAS7E;AAED;;;8BAG8B;AAC9B,eAAO,MAAM,YAAY,WAAO,CAAC;AA8BjC;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,SAAS,SAAS,EAAE,GAAG,UAAU,EAAE,CA4BtE;AAED,wDAAwD;AACxD,wBAAgB,SAAS,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,MAAM,CAEhE;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAEpE;AAED,oEAAoE;AACpE,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAEtD"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Believability gates — RATIFIED thresholds (stop2-architect-review §1):
|
|
4
|
+
* - min-n: ≥10 DISTINCT outcomes per model per subtype AND ≥2 fixtures.
|
|
5
|
+
* Never gate on raw pair counts (publish both; gate on distinct).
|
|
6
|
+
* - CI tie-tiers: overlapping CIs render as ONE tier, never a fake ordering.
|
|
7
|
+
* - SEPARATION GATE: a subtype publishes a rank view only when its measured
|
|
8
|
+
* models form ≥2 CI-distinct tiers; otherwise it renders the all-tie state.
|
|
9
|
+
* Threshold changes re-open STOP-2.
|
|
10
|
+
*/
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.TIER_EPSILON = exports.MIN_FIXTURES = exports.MIN_DISTINCT_OUTCOMES = void 0;
|
|
13
|
+
exports.minNReason = minNReason;
|
|
14
|
+
exports.assignTiers = assignTiers;
|
|
15
|
+
exports.tierCount = tierCount;
|
|
16
|
+
exports.allTieSentence = allTieSentence;
|
|
17
|
+
exports.insufficientSentence = insufficientSentence;
|
|
18
|
+
exports.MIN_DISTINCT_OUTCOMES = 10;
|
|
19
|
+
exports.MIN_FIXTURES = 2;
|
|
20
|
+
/** The min-n verdict for one model — null when it passes. */
|
|
21
|
+
function minNReason(distinctN, fixtures) {
|
|
22
|
+
const reasons = [];
|
|
23
|
+
if (distinctN < exports.MIN_DISTINCT_OUTCOMES) {
|
|
24
|
+
reasons.push(`insufficient data (n=${distinctN}, need ≥${exports.MIN_DISTINCT_OUTCOMES} distinct outcomes)`);
|
|
25
|
+
}
|
|
26
|
+
if (fixtures < exports.MIN_FIXTURES) {
|
|
27
|
+
reasons.push(`insufficient fixtures (${fixtures}, need ≥${exports.MIN_FIXTURES})`);
|
|
28
|
+
}
|
|
29
|
+
return reasons.length === 0 ? null : reasons.join("; ");
|
|
30
|
+
}
|
|
31
|
+
/** Interval arithmetic tolerance, in RigIndex points. The MM fit + GM
|
|
32
|
+
* normalization leave ~1e-13 float dust on saturated scores; a genuine CI
|
|
33
|
+
* overlap is orders of magnitude wider. One millionth of a point separates
|
|
34
|
+
* the two regimes cleanly. */
|
|
35
|
+
exports.TIER_EPSILON = 1e-6;
|
|
36
|
+
/**
|
|
37
|
+
* CI overlap = the intersection has length > TIER_EPSILON; intervals equal
|
|
38
|
+
* within the epsilon (including degenerate zero-variance points, e.g. two
|
|
39
|
+
* saturated models both pegged at [100, 100] ± float dust) also tie. A
|
|
40
|
+
* zero-width touch at a boundary point is NOT overlap: the saturated top tier
|
|
41
|
+
* sits at exactly 100, and an unstable straggler whose 97.5th percentile
|
|
42
|
+
* grazes 100.0 must not chain the whole board into one tier through that
|
|
43
|
+
* point (the prototype's reasoning.numeric separation — review §2 — depends
|
|
44
|
+
* on exactly this).
|
|
45
|
+
*/
|
|
46
|
+
function ciOverlaps(a, b) {
|
|
47
|
+
if (Math.abs(a[0] - b[0]) <= exports.TIER_EPSILON && Math.abs(a[1] - b[1]) <= exports.TIER_EPSILON) {
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
if (Math.min(a[1], b[1]) - Math.max(a[0], b[0]) > exports.TIER_EPSILON)
|
|
51
|
+
return true;
|
|
52
|
+
// A zero-variance (degenerate) interval strictly INSIDE the other interval's
|
|
53
|
+
// interior ties — a model whose CI spans [98.5, 101.5] cannot be called
|
|
54
|
+
// distinct from one pegged at exactly 100. Only a boundary graze separates.
|
|
55
|
+
const inInterior = (point, range) => point[1] - point[0] <= exports.TIER_EPSILON &&
|
|
56
|
+
point[0] > range[0] + exports.TIER_EPSILON &&
|
|
57
|
+
point[1] < range[1] - exports.TIER_EPSILON;
|
|
58
|
+
return inInterior(a, b) || inInterior(b, a);
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Assign tie-tiers to the ungated models: sort by score descending; walking
|
|
62
|
+
* down, a model joins the current tier iff its CI overlaps the CI of at least
|
|
63
|
+
* one model already in that tier; otherwise a new tier starts. Overlap chains
|
|
64
|
+
* deliberately — a bridge model with a wide CI merges the models it straddles
|
|
65
|
+
* into one tier rather than manufacturing an ordering the data can't support.
|
|
66
|
+
*/
|
|
67
|
+
function assignTiers(inputs) {
|
|
68
|
+
const results = [];
|
|
69
|
+
const ungated = [];
|
|
70
|
+
for (const input of inputs) {
|
|
71
|
+
const reason = minNReason(input.distinctN, input.fixtures);
|
|
72
|
+
if (reason !== null) {
|
|
73
|
+
results.push({ model: input.model, gated: true, gateReason: reason, tier: null });
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
ungated.push(input);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const sorted = [...ungated].sort((a, b) => b.score - a.score || (a.model < b.model ? -1 : 1));
|
|
80
|
+
let tier = 0;
|
|
81
|
+
let currentTier = [];
|
|
82
|
+
for (const input of sorted) {
|
|
83
|
+
if (currentTier.length === 0 ||
|
|
84
|
+
!currentTier.some((member) => ciOverlaps(member.ci95, input.ci95))) {
|
|
85
|
+
tier += 1;
|
|
86
|
+
currentTier = [];
|
|
87
|
+
}
|
|
88
|
+
currentTier.push(input);
|
|
89
|
+
results.push({ model: input.model, gated: false, gateReason: null, tier });
|
|
90
|
+
}
|
|
91
|
+
return results;
|
|
92
|
+
}
|
|
93
|
+
/** Number of CI-distinct tiers among ungated models. */
|
|
94
|
+
function tierCount(results) {
|
|
95
|
+
return results.reduce((max, r) => Math.max(max, r.tier ?? 0), 0);
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* The separation-gate state sentence for a one-tier subtype — authored once
|
|
99
|
+
* here so the layer and the www render the SAME ratified words (review §1:
|
|
100
|
+
* "all measured models tie on this task type — no rank distinguishable").
|
|
101
|
+
* `n` is the total distinct outcomes across the tied models.
|
|
102
|
+
*/
|
|
103
|
+
function allTieSentence(modelCount, n) {
|
|
104
|
+
return `all measured models tie on this task type — no rank distinguishable (${modelCount} models, n=${n} distinct outcomes)`;
|
|
105
|
+
}
|
|
106
|
+
/** The insufficient-data state sentence (min-n gate, review §1). */
|
|
107
|
+
function insufficientSentence(n) {
|
|
108
|
+
return `insufficient data (n=${n} distinct outcomes)`;
|
|
109
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RigIndex pairwise graph — the RATIFIED pairing rule (stop2-architect-review
|
|
3
|
+
* §1, verbatim): outcome per (model, fixture, sample) = `valueAccuracy` if
|
|
4
|
+
* `schemaConform` else 0 (a non-conformant output failed the task — worst
|
|
5
|
+
* outcome). On each shared fixture, every cross-sample pair (A_i vs B_j) is a
|
|
6
|
+
* comparison: win if outcome differs beyond ε; ties count half. A model
|
|
7
|
+
* missing a fixture contributes NO edge for it (never a zero-outcome edge).
|
|
8
|
+
*
|
|
9
|
+
* Any change to this rule re-opens STOP-2. Do not edit without a new
|
|
10
|
+
* architect sign-off.
|
|
11
|
+
*/
|
|
12
|
+
import type { PairedSample, SubtypeGraph, SubtypeSamples } from "./types";
|
|
13
|
+
/** Float-noise guard only — sampled accuracies are exact fractions, so ε must
|
|
14
|
+
* never turn a real difference into a tie. */
|
|
15
|
+
export declare const OUTCOME_EPSILON = 1e-9;
|
|
16
|
+
/**
|
|
17
|
+
* The ratified outcome. A conformant sample carrying a null valueAccuracy
|
|
18
|
+
* (unscored — does not occur on deterministic-ground-truth fixtures) is
|
|
19
|
+
* conservatively 0, matching the rule's "failed the task" floor.
|
|
20
|
+
*/
|
|
21
|
+
export declare function outcomeOf(sample: PairedSample): number;
|
|
22
|
+
/**
|
|
23
|
+
* Build one subtype's pairwise graph from per-(model, fixture) sample lists.
|
|
24
|
+
* `fixtureDraws` (bootstrap use) lists the fixtures to tally, WITH
|
|
25
|
+
* multiplicity — a fixture drawn twice contributes its comparisons twice.
|
|
26
|
+
* Omitted: every fixture any model has, once each.
|
|
27
|
+
*/
|
|
28
|
+
export declare function buildSubtypeGraph(samples: SubtypeSamples, fixtureDraws?: readonly string[]): SubtypeGraph;
|
|
29
|
+
/** Total raw comparisons a model participates in across the graph. */
|
|
30
|
+
export declare function modelPairCount(graph: SubtypeGraph, model: string): number;
|
|
31
|
+
//# sourceMappingURL=graph.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graph.d.ts","sourceRoot":"","sources":["../../src/rigindex/graph.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE1E;8CAC8C;AAC9C,eAAO,MAAM,eAAe,OAAO,CAAC;AAEpC;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAGtD;AAOD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,cAAc,EACvB,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,GAC/B,YAAY,CAkEd;AAED,sEAAsE;AACtE,wBAAgB,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAOzE"}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* RigIndex pairwise graph — the RATIFIED pairing rule (stop2-architect-review
|
|
4
|
+
* §1, verbatim): outcome per (model, fixture, sample) = `valueAccuracy` if
|
|
5
|
+
* `schemaConform` else 0 (a non-conformant output failed the task — worst
|
|
6
|
+
* outcome). On each shared fixture, every cross-sample pair (A_i vs B_j) is a
|
|
7
|
+
* comparison: win if outcome differs beyond ε; ties count half. A model
|
|
8
|
+
* missing a fixture contributes NO edge for it (never a zero-outcome edge).
|
|
9
|
+
*
|
|
10
|
+
* Any change to this rule re-opens STOP-2. Do not edit without a new
|
|
11
|
+
* architect sign-off.
|
|
12
|
+
*/
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.OUTCOME_EPSILON = void 0;
|
|
15
|
+
exports.outcomeOf = outcomeOf;
|
|
16
|
+
exports.buildSubtypeGraph = buildSubtypeGraph;
|
|
17
|
+
exports.modelPairCount = modelPairCount;
|
|
18
|
+
/** Float-noise guard only — sampled accuracies are exact fractions, so ε must
|
|
19
|
+
* never turn a real difference into a tie. */
|
|
20
|
+
exports.OUTCOME_EPSILON = 1e-9;
|
|
21
|
+
/**
|
|
22
|
+
* The ratified outcome. A conformant sample carrying a null valueAccuracy
|
|
23
|
+
* (unscored — does not occur on deterministic-ground-truth fixtures) is
|
|
24
|
+
* conservatively 0, matching the rule's "failed the task" floor.
|
|
25
|
+
*/
|
|
26
|
+
function outcomeOf(sample) {
|
|
27
|
+
if (sample.schemaConform !== true)
|
|
28
|
+
return 0;
|
|
29
|
+
return sample.valueAccuracy ?? 0;
|
|
30
|
+
}
|
|
31
|
+
/** Unordered pair key, order-stable. */
|
|
32
|
+
function pairKey(a, b) {
|
|
33
|
+
return a < b ? `${a}|${b}` : `${b}|${a}`;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Build one subtype's pairwise graph from per-(model, fixture) sample lists.
|
|
37
|
+
* `fixtureDraws` (bootstrap use) lists the fixtures to tally, WITH
|
|
38
|
+
* multiplicity — a fixture drawn twice contributes its comparisons twice.
|
|
39
|
+
* Omitted: every fixture any model has, once each.
|
|
40
|
+
*/
|
|
41
|
+
function buildSubtypeGraph(samples, fixtureDraws) {
|
|
42
|
+
const models = Object.keys(samples).sort();
|
|
43
|
+
const fixtures = fixtureDraws ??
|
|
44
|
+
[...new Set(models.flatMap((m) => Object.keys(samples[m])))].sort();
|
|
45
|
+
const wins = {};
|
|
46
|
+
for (const m of models) {
|
|
47
|
+
wins[m] = {};
|
|
48
|
+
for (const other of models)
|
|
49
|
+
if (other !== m)
|
|
50
|
+
wins[m][other] = 0;
|
|
51
|
+
}
|
|
52
|
+
const pairCounts = {};
|
|
53
|
+
const distinctN = {};
|
|
54
|
+
const fixtureSet = {};
|
|
55
|
+
for (const m of models) {
|
|
56
|
+
distinctN[m] = 0;
|
|
57
|
+
fixtureSet[m] = new Set();
|
|
58
|
+
}
|
|
59
|
+
// Distinct outcomes / fixture coverage count each real (model, fixture)
|
|
60
|
+
// ONCE regardless of bootstrap multiplicity — n describes the data held,
|
|
61
|
+
// not the resampling plan.
|
|
62
|
+
for (const m of models) {
|
|
63
|
+
for (const [fixtureId, list] of Object.entries(samples[m])) {
|
|
64
|
+
if (list.length === 0)
|
|
65
|
+
continue;
|
|
66
|
+
distinctN[m] += list.length;
|
|
67
|
+
fixtureSet[m].add(fixtureId);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
for (const fixtureId of fixtures) {
|
|
71
|
+
for (let i = 0; i < models.length; i += 1) {
|
|
72
|
+
const a = models[i];
|
|
73
|
+
const aSamples = samples[a][fixtureId];
|
|
74
|
+
if (aSamples === undefined || aSamples.length === 0)
|
|
75
|
+
continue; // no edge
|
|
76
|
+
for (let j = i + 1; j < models.length; j += 1) {
|
|
77
|
+
const b = models[j];
|
|
78
|
+
const bSamples = samples[b][fixtureId];
|
|
79
|
+
if (bSamples === undefined || bSamples.length === 0)
|
|
80
|
+
continue; // no edge
|
|
81
|
+
let aWins = 0;
|
|
82
|
+
let bWins = 0;
|
|
83
|
+
let ties = 0;
|
|
84
|
+
for (const sa of aSamples) {
|
|
85
|
+
const oa = outcomeOf(sa);
|
|
86
|
+
for (const sb of bSamples) {
|
|
87
|
+
const ob = outcomeOf(sb);
|
|
88
|
+
if (oa > ob + exports.OUTCOME_EPSILON)
|
|
89
|
+
aWins += 1;
|
|
90
|
+
else if (ob > oa + exports.OUTCOME_EPSILON)
|
|
91
|
+
bWins += 1;
|
|
92
|
+
else
|
|
93
|
+
ties += 1;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
wins[a][b] += aWins + ties / 2;
|
|
97
|
+
wins[b][a] += bWins + ties / 2;
|
|
98
|
+
const key = pairKey(a, b);
|
|
99
|
+
pairCounts[key] = (pairCounts[key] ?? 0) + aWins + bWins + ties;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return {
|
|
104
|
+
models,
|
|
105
|
+
wins,
|
|
106
|
+
pairCounts,
|
|
107
|
+
distinctN,
|
|
108
|
+
fixtureCount: Object.fromEntries(models.map((m) => [m, fixtureSet[m].size])),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/** Total raw comparisons a model participates in across the graph. */
|
|
112
|
+
function modelPairCount(graph, model) {
|
|
113
|
+
let total = 0;
|
|
114
|
+
for (const [key, count] of Object.entries(graph.pairCounts)) {
|
|
115
|
+
const [a, b] = key.split("|");
|
|
116
|
+
if (a === model || b === model)
|
|
117
|
+
total += count;
|
|
118
|
+
}
|
|
119
|
+
return total;
|
|
120
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RigIndex orchestration — pure entry point over per-subtype paired samples.
|
|
3
|
+
* The whole pipeline per subtype: pairwise graph (ratified pairing rule) →
|
|
4
|
+
* Hunter MM Bradley-Terry fit → 2026-Q3 basket anchor → hierarchical
|
|
5
|
+
* bootstrap CIs → gates (min-n, tie-tiers, separation). All models in the
|
|
6
|
+
* graph shape the fit (pooled comparisons); the GATES decide what publishes —
|
|
7
|
+
* a gated model's score is null, never a confident number on thin evidence.
|
|
8
|
+
*/
|
|
9
|
+
import type { RigIndexLayer, RigIndexSubtype, SubtypeSamples } from "./types";
|
|
10
|
+
export interface ComputeOptions {
|
|
11
|
+
readonly reps?: number;
|
|
12
|
+
readonly seed?: number;
|
|
13
|
+
}
|
|
14
|
+
export declare const DEFAULT_SEED = 20260813;
|
|
15
|
+
export declare function computeSubtype(samples: SubtypeSamples, basket: readonly string[], options?: ComputeOptions): RigIndexSubtype;
|
|
16
|
+
export declare function computeRigIndexLayer(subtypeSamples: Readonly<Record<string, SubtypeSamples>>, basket: {
|
|
17
|
+
readonly version: string;
|
|
18
|
+
readonly members: readonly string[];
|
|
19
|
+
}, meta: {
|
|
20
|
+
readonly asOf: string;
|
|
21
|
+
readonly source: string;
|
|
22
|
+
}, options?: ComputeOptions): RigIndexLayer;
|
|
23
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/rigindex/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAYH,OAAO,KAAK,EACV,aAAa,EAEb,eAAe,EACf,cAAc,EACf,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,eAAO,MAAM,YAAY,WAAW,CAAC;AAErC,wBAAgB,cAAc,CAC5B,OAAO,EAAE,cAAc,EACvB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,OAAO,GAAE,cAAmB,GAC3B,eAAe,CAgFjB;AAED,wBAAgB,oBAAoB,CAClC,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,EACxD,MAAM,EAAE;IAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,EACzE,IAAI,EAAE;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EACxD,OAAO,GAAE,cAAmB,GAC3B,aAAa,CAiBf"}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* RigIndex orchestration — pure entry point over per-subtype paired samples.
|
|
4
|
+
* The whole pipeline per subtype: pairwise graph (ratified pairing rule) →
|
|
5
|
+
* Hunter MM Bradley-Terry fit → 2026-Q3 basket anchor → hierarchical
|
|
6
|
+
* bootstrap CIs → gates (min-n, tie-tiers, separation). All models in the
|
|
7
|
+
* graph shape the fit (pooled comparisons); the GATES decide what publishes —
|
|
8
|
+
* a gated model's score is null, never a confident number on thin evidence.
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.DEFAULT_SEED = void 0;
|
|
12
|
+
exports.computeSubtype = computeSubtype;
|
|
13
|
+
exports.computeRigIndexLayer = computeRigIndexLayer;
|
|
14
|
+
const graph_1 = require("./graph");
|
|
15
|
+
const fit_1 = require("./fit");
|
|
16
|
+
const bootstrap_1 = require("./bootstrap");
|
|
17
|
+
const gates_1 = require("./gates");
|
|
18
|
+
exports.DEFAULT_SEED = 20260813;
|
|
19
|
+
function computeSubtype(samples, basket, options = {}) {
|
|
20
|
+
const reps = options.reps ?? bootstrap_1.BOOTSTRAP_REPS;
|
|
21
|
+
const seed = options.seed ?? exports.DEFAULT_SEED;
|
|
22
|
+
const graph = (0, graph_1.buildSubtypeGraph)(samples);
|
|
23
|
+
const scores = (0, fit_1.anchorScores)((0, fit_1.fitBradleyTerry)(graph), basket);
|
|
24
|
+
const cis = scores === null
|
|
25
|
+
? {}
|
|
26
|
+
: (0, bootstrap_1.bootstrapCis)(samples, basket, reps, (0, bootstrap_1.mulberry32)(seed));
|
|
27
|
+
const gateInputs = [];
|
|
28
|
+
const noCi = [];
|
|
29
|
+
for (const model of graph.models) {
|
|
30
|
+
const score = scores?.[model];
|
|
31
|
+
const ci = cis[model];
|
|
32
|
+
if (score === undefined || ci === undefined) {
|
|
33
|
+
noCi.push(model);
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
gateInputs.push({
|
|
37
|
+
model,
|
|
38
|
+
score,
|
|
39
|
+
ci95: ci,
|
|
40
|
+
distinctN: graph.distinctN[model],
|
|
41
|
+
fixtures: graph.fixtureCount[model],
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
const gateResults = (0, gates_1.assignTiers)(gateInputs);
|
|
45
|
+
const byModel = new Map(gateResults.map((r) => [r.model, r]));
|
|
46
|
+
const scoreOf = new Map(gateInputs.map((g) => [g.model, g]));
|
|
47
|
+
const models = {};
|
|
48
|
+
let ungatedCount = 0;
|
|
49
|
+
let ungatedN = 0;
|
|
50
|
+
for (const model of graph.models) {
|
|
51
|
+
const gate = byModel.get(model);
|
|
52
|
+
const input = scoreOf.get(model);
|
|
53
|
+
const gated = gate === undefined ? true : gate.gated;
|
|
54
|
+
if (!gated) {
|
|
55
|
+
ungatedCount += 1;
|
|
56
|
+
ungatedN += graph.distinctN[model];
|
|
57
|
+
}
|
|
58
|
+
models[model] = {
|
|
59
|
+
score: gated ? null : (input?.score ?? null),
|
|
60
|
+
ci95: gated ? null : (input?.ci95 ?? null),
|
|
61
|
+
tier: gate?.tier ?? null,
|
|
62
|
+
distinct_n: graph.distinctN[model],
|
|
63
|
+
pair_count: (0, graph_1.modelPairCount)(graph, model),
|
|
64
|
+
fixtures: graph.fixtureCount[model],
|
|
65
|
+
// Probe-only launch (review §1): every comparison is probe-derived.
|
|
66
|
+
provenance: {
|
|
67
|
+
probe_derived_pairs: (0, graph_1.modelPairCount)(graph, model),
|
|
68
|
+
community_pairs: 0,
|
|
69
|
+
},
|
|
70
|
+
gated,
|
|
71
|
+
gate_reason: gate?.gateReason ??
|
|
72
|
+
(gate === undefined ? "unanchorable or no bootstrap distribution" : null),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
const tiers = (0, gates_1.tierCount)(gateResults);
|
|
76
|
+
const totalN = Object.values(graph.distinctN).reduce((s, n) => s + n, 0);
|
|
77
|
+
// The separation gate (review §1): <2 rankable models is an insufficiency,
|
|
78
|
+
// exactly one CI-distinct tier is the honest all-tie state.
|
|
79
|
+
let state;
|
|
80
|
+
let stateDetail;
|
|
81
|
+
if (ungatedCount < 2) {
|
|
82
|
+
state = "insufficient";
|
|
83
|
+
stateDetail = (0, gates_1.insufficientSentence)(totalN);
|
|
84
|
+
}
|
|
85
|
+
else if (tiers < 2) {
|
|
86
|
+
state = "all-tie";
|
|
87
|
+
stateDetail = (0, gates_1.allTieSentence)(ungatedCount, ungatedN);
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
state = "rank";
|
|
91
|
+
stateDetail = null;
|
|
92
|
+
}
|
|
93
|
+
return { state, state_detail: stateDetail, tiers, models };
|
|
94
|
+
}
|
|
95
|
+
function computeRigIndexLayer(subtypeSamples, basket, meta, options = {}) {
|
|
96
|
+
const subtypes = {};
|
|
97
|
+
for (const subtype of Object.keys(subtypeSamples).sort()) {
|
|
98
|
+
subtypes[subtype] = computeSubtype(subtypeSamples[subtype], basket.members, options);
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
as_of: meta.asOf,
|
|
102
|
+
source: meta.source,
|
|
103
|
+
basket_version: basket.version,
|
|
104
|
+
basket_members: [...basket.members],
|
|
105
|
+
bootstrap: {
|
|
106
|
+
reps: options.reps ?? bootstrap_1.BOOTSTRAP_REPS,
|
|
107
|
+
seed: options.seed ?? exports.DEFAULT_SEED,
|
|
108
|
+
method: "hierarchical-fixture-then-per-model-fixture-sample",
|
|
109
|
+
},
|
|
110
|
+
subtypes,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RigIndex shared types (Cycle 4 — rigindex-scoring/spec.md, math ratified in
|
|
3
|
+
* stop2-architect-review.md §1). The scoring layer is PURE: it consumes only
|
|
4
|
+
* per-sample paired outcomes from probe result files — never marginal
|
|
5
|
+
* statistics, telemetry, or observed health (the RigIndex wall, spec §6).
|
|
6
|
+
*/
|
|
7
|
+
/** The two per-sample fields the pairing rule reads — and the ONLY signal the
|
|
8
|
+
* scoring layer may consume (rigindex-wall.test.ts asserts cost/latency/parse
|
|
9
|
+
* fields never move a score). */
|
|
10
|
+
export interface PairedSample {
|
|
11
|
+
readonly schemaConform: boolean | null;
|
|
12
|
+
readonly valueAccuracy: number | null;
|
|
13
|
+
}
|
|
14
|
+
/** modelKey → fixtureId → that model's samples on the fixture. A model
|
|
15
|
+
* missing a fixture simply has no key for it (and contributes NO edge). */
|
|
16
|
+
export type SubtypeSamples = Readonly<Record<string, Readonly<Record<string, readonly PairedSample[]>>>>;
|
|
17
|
+
/** Pairwise tallies for one subtype. `wins[a][b]` counts a's effective wins
|
|
18
|
+
* over b — ties already folded in at half (review §1: ties count half). */
|
|
19
|
+
export interface SubtypeGraph {
|
|
20
|
+
readonly models: readonly string[];
|
|
21
|
+
/** wins[a][b] = wins of a over b + 0.5 per tie. */
|
|
22
|
+
readonly wins: Readonly<Record<string, Readonly<Record<string, number>>>>;
|
|
23
|
+
/** Raw cross-sample comparison count per unordered model pair, summed —
|
|
24
|
+
* published beside distinct_n, never gated on (review §1 min-n). */
|
|
25
|
+
readonly pairCounts: Readonly<Record<string, number>>;
|
|
26
|
+
/** Distinct per-(model,fixture,sample) outcomes per model — the min-n
|
|
27
|
+
* gate's n (review §1: gate on distinct outcomes, not pairs). */
|
|
28
|
+
readonly distinctN: Readonly<Record<string, number>>;
|
|
29
|
+
/** Fixtures each model contributed samples on (min-n needs ≥2). */
|
|
30
|
+
readonly fixtureCount: Readonly<Record<string, number>>;
|
|
31
|
+
}
|
|
32
|
+
/** One model's published entry in layers/rigindex.json. */
|
|
33
|
+
export interface RigIndexModelEntry {
|
|
34
|
+
readonly score: number | null;
|
|
35
|
+
readonly ci95: readonly [number, number] | null;
|
|
36
|
+
/** 1-based tie-tier (1 = top). Null when the model is min-n gated. */
|
|
37
|
+
readonly tier: number | null;
|
|
38
|
+
readonly distinct_n: number;
|
|
39
|
+
readonly pair_count: number;
|
|
40
|
+
readonly fixtures: number;
|
|
41
|
+
/** Probe-only launch: every edge is probe-derived (review §1 source
|
|
42
|
+
* diversity — labeled single source, community edges arrive later). */
|
|
43
|
+
readonly provenance: {
|
|
44
|
+
readonly probe_derived_pairs: number;
|
|
45
|
+
readonly community_pairs: number;
|
|
46
|
+
};
|
|
47
|
+
readonly gated: boolean;
|
|
48
|
+
readonly gate_reason: string | null;
|
|
49
|
+
}
|
|
50
|
+
/** A subtype's publish state after ALL gates (review §1):
|
|
51
|
+
* - "rank" — ≥2 CI-distinct tiers among ungated models → rank view
|
|
52
|
+
* - "all-tie" — one tier: "all measured models tie…" (separation gate)
|
|
53
|
+
* - "insufficient"— no (or one) model passes min-n; nothing rankable. */
|
|
54
|
+
export type SubtypeState = "rank" | "all-tie" | "insufficient";
|
|
55
|
+
export interface RigIndexSubtype {
|
|
56
|
+
readonly state: SubtypeState;
|
|
57
|
+
/** The honest display sentence for a non-rank state (gate rendering is
|
|
58
|
+
* authored ONCE here so the www renders the same words the layer states). */
|
|
59
|
+
readonly state_detail: string | null;
|
|
60
|
+
readonly tiers: number;
|
|
61
|
+
readonly models: Readonly<Record<string, RigIndexModelEntry>>;
|
|
62
|
+
}
|
|
63
|
+
export interface RigIndexLayer {
|
|
64
|
+
readonly as_of: string;
|
|
65
|
+
/** Single-source label (review §1): probe-derived, no community edges yet. */
|
|
66
|
+
readonly source: string;
|
|
67
|
+
readonly basket_version: string;
|
|
68
|
+
readonly basket_members: readonly string[];
|
|
69
|
+
readonly bootstrap: {
|
|
70
|
+
readonly reps: number;
|
|
71
|
+
readonly seed: number;
|
|
72
|
+
readonly method: "hierarchical-fixture-then-per-model-fixture-sample";
|
|
73
|
+
};
|
|
74
|
+
readonly subtypes: Readonly<Record<string, RigIndexSubtype>>;
|
|
75
|
+
}
|
|
76
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/rigindex/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;iCAEiC;AACjC,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CACvC;AAED;2EAC2E;AAC3E,MAAM,MAAM,cAAc,GAAG,QAAQ,CACnC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,CAAC,CAAC,CAAC,CAClE,CAAC;AAEF;2EAC2E;AAC3E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,mDAAmD;IACnD,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1E;wEACoE;IACpE,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACtD;qEACiE;IACjE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACrD,mEAAmE;IACnE,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACzD;AAED,2DAA2D;AAC3D,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IAChD,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;2EACuE;IACvE,QAAQ,CAAC,UAAU,EAAE;QACnB,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;QACrC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;KAClC,CAAC;IACF,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;yEAGyE;AACzE,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,SAAS,GAAG,cAAc,CAAC;AAE/D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B;iFAC6E;IAC7E,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC,CAAC;CAC/D;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE;QAClB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,MAAM,EAAE,oDAAoD,CAAC;KACvE,CAAC;IACF,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC;CAC9D"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* RigIndex shared types (Cycle 4 — rigindex-scoring/spec.md, math ratified in
|
|
4
|
+
* stop2-architect-review.md §1). The scoring layer is PURE: it consumes only
|
|
5
|
+
* per-sample paired outcomes from probe result files — never marginal
|
|
6
|
+
* statistics, telemetry, or observed health (the RigIndex wall, spec §6).
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|