modelrig 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +123 -0
- package/README.md +65 -1
- package/dist/artifacts/run-context.d.ts +10 -0
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +12 -0
- package/dist/bakeoff/disagreement.d.ts +93 -0
- package/dist/bakeoff/disagreement.d.ts.map +1 -0
- package/dist/bakeoff/disagreement.js +140 -0
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +58 -2
- package/dist/bakeoff/pairwise.d.ts +102 -0
- package/dist/bakeoff/pairwise.d.ts.map +1 -0
- package/dist/bakeoff/pairwise.js +214 -0
- package/dist/bakeoff/replay.d.ts +6 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +6 -2
- package/dist/bakeoff/report.d.ts +7 -0
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +46 -0
- package/dist/bakeoff/types.d.ts +24 -0
- package/dist/bakeoff/types.d.ts.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +9 -9
- package/dist/mask/apply.d.ts +80 -0
- package/dist/mask/apply.d.ts.map +1 -0
- package/dist/mask/apply.js +228 -0
- package/dist/mask/detect.d.ts +38 -0
- package/dist/mask/detect.d.ts.map +1 -0
- package/dist/mask/detect.js +167 -0
- package/dist/mask/index.d.ts +21 -0
- package/dist/mask/index.d.ts.map +1 -0
- package/dist/mask/index.js +54 -0
- package/dist/mask/mask-policy-cli.d.ts +11 -0
- package/dist/mask/mask-policy-cli.d.ts.map +1 -0
- package/dist/mask/mask-policy-cli.js +33 -0
- package/dist/mask/policy.d.ts +38 -0
- package/dist/mask/policy.d.ts.map +1 -0
- package/dist/mask/policy.js +187 -0
- package/dist/mask/proof.d.ts +54 -0
- package/dist/mask/proof.d.ts.map +1 -0
- package/dist/mask/proof.js +120 -0
- package/dist/mask/receipt.d.ts +30 -0
- package/dist/mask/receipt.d.ts.map +1 -0
- package/dist/mask/receipt.js +49 -0
- package/dist/mask/seam.d.ts +53 -0
- package/dist/mask/seam.d.ts.map +1 -0
- package/dist/mask/seam.js +120 -0
- package/dist/mask/types.d.ts +81 -0
- package/dist/mask/types.d.ts.map +1 -0
- package/dist/mask/types.js +10 -0
- package/dist/mask/vault.d.ts +50 -0
- package/dist/mask/vault.d.ts.map +1 -0
- package/dist/mask/vault.js +81 -0
- package/dist/receipts/types.d.ts +65 -0
- package/dist/receipts/types.d.ts.map +1 -0
- package/dist/receipts/types.js +134 -0
- package/dist/routes/loader-warnings.d.ts +8 -0
- package/dist/routes/loader-warnings.d.ts.map +1 -1
- package/dist/routes/loader-warnings.js +23 -0
- package/dist/routes/loader.d.ts +1 -1
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +24 -25
- package/dist/routes/mask-loader.d.ts +18 -0
- package/dist/routes/mask-loader.d.ts.map +1 -0
- package/dist/routes/mask-loader.js +39 -0
- package/dist/routes/objective-loader.d.ts.map +1 -1
- package/dist/routes/objective-loader.js +51 -17
- package/dist/routes/screen-loader.d.ts +23 -0
- package/dist/routes/screen-loader.d.ts.map +1 -0
- package/dist/routes/screen-loader.js +79 -0
- package/dist/routes/template.d.ts +10 -0
- package/dist/routes/template.d.ts.map +1 -1
- package/dist/routes/template.js +10 -3
- package/dist/run/accept.d.ts +4 -0
- package/dist/run/accept.d.ts.map +1 -1
- package/dist/run/accept.js +2 -0
- package/dist/run/dispatch.d.ts +35 -0
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +33 -0
- package/dist/run/grounding.d.ts +47 -2
- package/dist/run/grounding.d.ts.map +1 -1
- package/dist/run/grounding.js +113 -6
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +10 -3
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +12 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +25 -19
- package/dist/screen/provenance.d.ts +59 -0
- package/dist/screen/provenance.d.ts.map +1 -0
- package/dist/screen/provenance.js +104 -0
- package/dist/screen/types.d.ts +71 -0
- package/dist/screen/types.d.ts.map +1 -0
- package/dist/screen/types.js +9 -0
- package/dist/status-cli.d.ts +11 -0
- package/dist/status-cli.d.ts.map +1 -0
- package/dist/status-cli.js +21 -0
- package/dist/status-runs.d.ts +38 -0
- package/dist/status-runs.d.ts.map +1 -0
- package/dist/status-runs.js +80 -0
- package/dist/status.d.ts +7 -2
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +11 -2
- package/dist/telemetry/ingest-protocol.d.ts +8 -0
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +11 -0
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +12 -3
- package/dist/telemetry/supabase-exporter.d.ts +64 -0
- package/dist/telemetry/supabase-exporter.d.ts.map +1 -0
- package/dist/telemetry/supabase-exporter.js +211 -0
- package/dist/types/provider.d.ts +11 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +9 -0
- package/dist/types/route.d.ts +55 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +7 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +13 -0
- package/dist/verify/types.d.ts +6 -0
- package/dist/verify/types.d.ts.map +1 -1
- package/docs/agent-skill/SKILL.md +131 -0
- package/docs/how-it-fits.md +34 -0
- package/docs/instrumentation.md +221 -0
- package/docs/llms.txt +276 -0
- package/docs/migration-playbook.md +512 -0
- package/docs/quickstart.md +333 -0
- package/package.json +18 -2
- package/receipts.d.ts +12 -0
package/CHANGELOG.md
CHANGED
|
@@ -10,6 +10,129 @@ published package diverge.
|
|
|
10
10
|
This project adheres to [Semantic Versioning](https://semver.org/). Dates are
|
|
11
11
|
UTC.
|
|
12
12
|
|
|
13
|
+
## [0.8.0] — 2026-08-24
|
|
14
|
+
|
|
15
|
+
Packaging (GS-3) — the tarball now ships the customer-facing runs docs
|
|
16
|
+
(`docs/quickstart.md`, `docs/instrumentation.md`, `docs/how-it-fits.md`,
|
|
17
|
+
`docs/migration-playbook.md`, `docs/agent-skill/SKILL.md`, and the generated
|
|
18
|
+
`docs/llms.txt`) plus a self-contained **Runs — the standard path** section in
|
|
19
|
+
`README.md`, so `npm i modelrig` + read-the-package teaches run → step → artifact
|
|
20
|
+
offline; and every doc now states the `MODELRIG_ARTIFACTS` default consistently
|
|
21
|
+
(on by default once a control plane is configured; a local-only rig stays off;
|
|
22
|
+
`=0` opts out, `=1` forces on). Docs/packaging only — no SDK behaviour change.
|
|
23
|
+
|
|
24
|
+
Get-started v5 — runs are the standard path (GS-1). The front-door instructions
|
|
25
|
+
(www `/get-started`, the console setup prompt, `quickstart.md`, `how-it-fits.md`,
|
|
26
|
+
the agent `SKILL.md`) now teach **run → step → artifact** as the standard way to
|
|
27
|
+
instrument a pipeline, not an appendix; `instrumentation.md` stays the reference
|
|
28
|
+
they link to. SDK: inside a run context the SDK now also stamps **`tags.step`**
|
|
29
|
+
(the current step key: your explicit `stepKey`, else `tags.step`, else the auto
|
|
30
|
+
`route#n` / `provider/model#n` key) on every attempt row of both lanes when the
|
|
31
|
+
caller set none — the mirror of RC-1's `run_id` stamp; a caller value always wins,
|
|
32
|
+
and outside a run context rows are byte-identical. `modelrig status` prints
|
|
33
|
+
**`runs recorded: N (last: <ts>)`** from a best-effort readonly read of the local
|
|
34
|
+
buffer, and `modelrig validate` **warns when a rig has recorded calls but never
|
|
35
|
+
started a run** (a teaching hint — never a failure). ZERO DDL, no new deps.
|
|
36
|
+
|
|
37
|
+
Provenance-first assembly (screening B3) — additive, opt-in SDK seams for keeping
|
|
38
|
+
customer- and web-derived text out of the trusted system slot. **Default
|
|
39
|
+
behaviour is byte-identical**: a route with no `grounding.assembly` and no
|
|
40
|
+
`x-trust` takes the legacy path unchanged. A route may set
|
|
41
|
+
`grounding.assembly: spotlight` to route search results through the assembler as
|
|
42
|
+
an untrusted user block (search results are delivered as data, not instructions);
|
|
43
|
+
a template variable may be annotated `x-trust: untrusted` to move it out of the
|
|
44
|
+
system slot into that spotlighted block. Model-output citations are allowlisted to
|
|
45
|
+
the injected sources, with a `RunMeta.grounding.citations_dropped` count. `modelrig
|
|
46
|
+
validate` warns on legacy grounding and on an untrusted-shaped variable left
|
|
47
|
+
unannotated in the system slot. `EquivalenceReport.subject` gains
|
|
48
|
+
`"screen-assembly"`. `screen/provenance.ts` + `screen/types.ts` now ship in
|
|
49
|
+
`dist/`. ZERO DDL, no new deps.
|
|
50
|
+
|
|
51
|
+
Near-optimal pairwise + disagreement (N5/N6) — bake-offs gain a second read on
|
|
52
|
+
regret and a fixture-mining output, both additive and **byte-identical for
|
|
53
|
+
routes that don't opt in**. N5 scores each served arm's **pairwise standing vs
|
|
54
|
+
the reference arm** on the paired replay set — a `pairwise_win_rate`
|
|
55
|
+
(win=1/tie=0.5/loss=0) with a seeded-bootstrap CI and n — surfaced as
|
|
56
|
+
`VariantMetrics.pairwise` and a `pairwise-win` column in the report. It is
|
|
57
|
+
DETERMINISTIC-FIRST, exactly like the RB-1 regret tiers: when both an arm and the
|
|
58
|
+
reference carry a per-sample value-accuracy score the verdict is free (the sign
|
|
59
|
+
of their difference); only the samples nothing scored fall to an OPT-IN, CAPPED,
|
|
60
|
+
METERED pairwise judge, declared `bakeoff.pairwise.judge` (judge model ∈ declared
|
|
61
|
+
candidates, #223 F3; every call a metered inference tagged `bakeoff_pairwise`,
|
|
62
|
+
capped per arm). A new `objective.pairwiseAware` (default off) adds one
|
|
63
|
+
eligibility clause to the `quality-first`/`balanced` gate — a proposed arm must
|
|
64
|
+
not be pairwise-worse than the reference within CI — and never touches the
|
|
65
|
+
cost-first gate. N6 mines the same replay for **eval-case candidates**: ranked
|
|
66
|
+
POINTERS (by `captureId`) to the samples where the bake-off's own signals
|
|
67
|
+
disagree (conformance vs value score, the Q1 gate vs conformance, served arm vs
|
|
68
|
+
reference, a pairwise upset), surfaced on `BakeoffArtifact.disagreement` and a
|
|
69
|
+
report summary line. Candidates are **metadata only** (captureId + numbers /
|
|
70
|
+
booleans, asserted meta-safe — never input or output text) and are **never
|
|
71
|
+
auto-promoted**; the full pre-cap `found` count rides the truncated list so a
|
|
72
|
+
capped surface is never mistaken for all of them. No new root exports — additive
|
|
73
|
+
route YAML plus report/artifact fields. ZERO DDL, no new deps. See
|
|
74
|
+
`docs/bakeoffs.md`.
|
|
75
|
+
|
|
76
|
+
Published receipts (RP-1 / RP-1c) — a signed, login-free **Receipt@v1** an org
|
|
77
|
+
can publish from its own measured rows. **This is two halves; only the shape is
|
|
78
|
+
in the npm package.** The `modelrig` package exposes the receipt CONTRACT via the
|
|
79
|
+
new `modelrig/receipts` sub-path export ONLY (never re-exported from `index.ts`):
|
|
80
|
+
the `ReceiptV1` shape, `canonicalizeReceipt` / `hashReceipt` (JCS-style
|
|
81
|
+
canonical body, self-describing `sha256:` hash), and the pure `isReceiptSafe`
|
|
82
|
+
classification gate — `measured` carries numbers / booleans / hashes /
|
|
83
|
+
identifiers only, no content, no tenant ids (fail-closed). The additive
|
|
84
|
+
`exports` map (`.`, `./receipts`, `./data/*`, `./registry/*`) plus a
|
|
85
|
+
`receipts.d.ts` shim keep the historical bare surface and the runtime data /
|
|
86
|
+
registry files reachable now that a sub-path map turns resolution into an
|
|
87
|
+
allowlist; `dist/receipts/**` now ships. **Everything else lives in
|
|
88
|
+
`modelrig-server` (NOT the npm tag):** Ed25519 signing with a `key_id` (node
|
|
89
|
+
crypto, no new dep), authed publish / revoke, the store-only public family —
|
|
90
|
+
`GET /v1/r/:id`, `…/badge.svg`, `GET /v1/verify/:id` (re-canonicalize → re-hash →
|
|
91
|
+
verify signature), `GET /.well-known/modelrig-receipts.json` (key rotation) — a
|
|
92
|
+
handler constructed with NO database client, an org alias that never leaks
|
|
93
|
+
`org_id`, `recipient` visibility (a login-free capability URL never listed
|
|
94
|
+
publicly), and a revoke tombstone that answers 410. RP-1c adds the `run` and
|
|
95
|
+
`fact-sheet` producers + source readers and the `source_ref` lookup (X9 — the
|
|
96
|
+
PR-body "verify this" link). **Honest limit:** publish / revoke / signing degrade
|
|
97
|
+
to **501** until `MODELRIG_RECEIPT_SIGNING_KEY` + `MODELRIG_RECEIPT_KEY_ID` and
|
|
98
|
+
the content store are provisioned (a half-configured signer is refused, not
|
|
99
|
+
half-enabled); the public read / verify surface answers 404 until an object is
|
|
100
|
+
published. ZERO DDL, no new deps. See `docs/receipts.md`.
|
|
101
|
+
|
|
102
|
+
Tenant ledger (TB-1) — lazy per-customer identity and derived statements for a
|
|
103
|
+
platform that resells routed inference. **Server / console / docs only — the
|
|
104
|
+
`modelrig` npm surface is unchanged by TB-1.** TB-1a adds tenant identity as a
|
|
105
|
+
single column: migration `00059` puts a `kind` discriminator
|
|
106
|
+
(`'project'` | `'tenant'`) on `projects`, with a reserved per-org `__tenants__`
|
|
107
|
+
parent and tenant leaves slugged `t_<sha256(org_id:subject)[:24]>`, materialized
|
|
108
|
+
service-role at ingest from the principal's org (never the payload) — **zero new
|
|
109
|
+
tables**. TB-1b derives a per-tenant statement (`TenantStatementV1`) on read,
|
|
110
|
+
**never stored**: counts / cents / hashes only, where `verified` is DETERMINISTIC
|
|
111
|
+
conformance only (run-outcome@v1 / schema-conformant artifact; model judge and
|
|
112
|
+
human feedback are `quality_context` and never gate billable), unpriced models
|
|
113
|
+
fall to a separate "priced at list" line rather than into the total, and a new
|
|
114
|
+
Tenants console renders it. Rate cards and real price / margin are **TB-3, NOT in
|
|
115
|
+
this tag** — every TB-1 statement is "priced at list" with `billable = verified`
|
|
116
|
+
and null price / margin. The migration ships in the repo but is applied by a
|
|
117
|
+
Supabase-enabled session, not this build. ZERO DDL added by this release; no new
|
|
118
|
+
deps. See `docs/tenants.md`.
|
|
119
|
+
|
|
120
|
+
## [0.7.0] — 2026-08-23
|
|
121
|
+
|
|
122
|
+
Released with the docs/registry mirror in one R-A ceremony. Headline: wrap a
|
|
123
|
+
pipeline in `rig.artifacts.run.start({ pipeline, episodeKey })` and that is the
|
|
124
|
+
whole instrumentation story — the SDK stamps `tags.run_id` on every call of both
|
|
125
|
+
lanes, so `/runs` AND `/projects` fill with no per-call tag plumbing.
|
|
126
|
+
|
|
127
|
+
Run-context tags (RC-1) — inside `rig.artifacts.run.start` / `run.scope`, the SDK
|
|
128
|
+
stamps `tags.run_id` (the run's `episodeKey`, else its id) on every attempt row of
|
|
129
|
+
both lanes when the caller passed none; a caller-supplied `run_id` always wins,
|
|
130
|
+
and outside a run context rows are byte-identical. The console `/projects` tree
|
|
131
|
+
joins a uuid-valued `run_id` by `runs.id` as well as by `episode_key`, so a
|
|
132
|
+
pipeline that calls `run.start({ pipeline })` alone now fills `/runs` AND
|
|
133
|
+
`/projects`. `ActiveRunContext.episodeKey` is exposed; `withRunIdTag` is the one
|
|
134
|
+
rule. ZERO DDL.
|
|
135
|
+
|
|
13
136
|
## [0.6.0] — 2026-08-23
|
|
14
137
|
|
|
15
138
|
Released with the docs/registry mirror in one R-A ceremony. Headline for router
|
package/README.md
CHANGED
|
@@ -145,7 +145,8 @@ provider's list price (zero ModelRig margin) and throws a typed `RigFailureError
|
|
|
145
145
|
on failure, exactly like `rig.run`.
|
|
146
146
|
|
|
147
147
|
Called inside a `rig.artifacts.run.start` context, `runRaw`/`runRawStream` also
|
|
148
|
-
records a ground-truth **step** in the run's graph (`/runs/[id]`)
|
|
148
|
+
records a ground-truth **step** in the run's graph (`/runs/[id]`) and carries the
|
|
149
|
+
run's `run_id` tag — since
|
|
149
150
|
modelrig 0.5.0, on success and on failure. Name it with `stepKey` (the same field
|
|
150
151
|
`RunOptions` exposes on the routed lane; precedence `stepKey` > `tags.step` > an
|
|
151
152
|
auto `provider/model#n` key), so a raw call and the artifacts you `save` around it
|
|
@@ -228,6 +229,69 @@ lines up with the console's built-ins:
|
|
|
228
229
|
`customer`, `tenant`, `user_id`, `team`, … is nudged toward the reserved key it
|
|
229
230
|
resembles. They stay ordinary tags; nothing is required.
|
|
230
231
|
|
|
232
|
+
## Runs — the standard path
|
|
233
|
+
|
|
234
|
+
Every pipeline execution is a **run** — the standard way ModelRig records what
|
|
235
|
+
your workstream did, not an appendix. Wrap the execution in a run context, name
|
|
236
|
+
one **step** per model-call family, and **save the work products** as artifacts;
|
|
237
|
+
the console's **Runs** tab then shows the run → step → artifact chain you click
|
|
238
|
+
through, and `/projects` groups it by your tags.
|
|
239
|
+
|
|
240
|
+
```ts
|
|
241
|
+
import { createRig, loadConfigFromEnv } from "modelrig";
|
|
242
|
+
|
|
243
|
+
const rig = createRig(loadConfigFromEnv());
|
|
244
|
+
|
|
245
|
+
// One run per pipeline execution. run.scope isolates concurrent runs; use
|
|
246
|
+
// run.start()/run.end() for one run at a time on a request path.
|
|
247
|
+
await rig.artifacts.run.scope(
|
|
248
|
+
{ pipeline: "support", episodeKey: ticketId },
|
|
249
|
+
async () => {
|
|
250
|
+
const result = await rig.run("example.support_summarize", {
|
|
251
|
+
input: { ticket, product, priorContext },
|
|
252
|
+
tags: { subject: customer, feature: "summarize" }, // run_id auto-stamped
|
|
253
|
+
stepKey: "summarize", // names this step on /runs
|
|
254
|
+
});
|
|
255
|
+
// Save the work product — prompt / raw / parsed all save the same way.
|
|
256
|
+
rig.artifacts.artifact.save(result.output, { name: "summary", type: "step_output" });
|
|
257
|
+
},
|
|
258
|
+
);
|
|
259
|
+
rig.close();
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
- **Tags:** carry `subject` (what the run is *about* — an opaque id) and
|
|
263
|
+
`feature`. Inside a run context the SDK stamps `tags.run_id` (the run's
|
|
264
|
+
`episodeKey`, else its id) and `tags.step` (the current step key) on every
|
|
265
|
+
attempt row of both lanes — a value you pass always wins, and outside a run
|
|
266
|
+
context nothing is stamped, so the row is byte-identical.
|
|
267
|
+
- **Raw lane too:** a `rig.runRaw` call inside a run context records a
|
|
268
|
+
ground-truth step (since 0.5.0), so grounded/cached BYOK pipelines fill the
|
|
269
|
+
same graph.
|
|
270
|
+
- **Acceptance:** the run appears on `/runs` with its steps in order and the
|
|
271
|
+
artifacts you saved attached. `modelrig status` prints `runs recorded: N` as
|
|
272
|
+
the local mirror; `modelrig validate` warns if a rig records calls but never
|
|
273
|
+
starts a run.
|
|
274
|
+
|
|
275
|
+
**On by default under a control plane.** The run/artifact namespace is **on by
|
|
276
|
+
default once a control plane is configured** — set `MODELRIG_API_KEY=rig_sk_…`
|
|
277
|
+
(or the self-host Supabase pair) and runs record automatically, metadata and
|
|
278
|
+
hashes only. A **pure local-only rig** (no sink to ship to) stays off; force it
|
|
279
|
+
on there with `MODELRIG_ARTIFACTS=1`, or opt out anywhere with
|
|
280
|
+
`MODELRIG_ARTIFACTS=0`. With the namespace off it is inert — `run.start` is a
|
|
281
|
+
no-op, `artifact.save` returns null, zero rows are written, and your pipeline
|
|
282
|
+
behaves byte-for-byte as before. `modelrig status` prints the current posture.
|
|
283
|
+
|
|
284
|
+
> **Metadata and hashes only, this release.** Content custody (holding the
|
|
285
|
+
> serialized bytes) is early access; today the value you save is hashed (sha256)
|
|
286
|
+
> and the bytes are discarded — the metadata row and its integrity hash are what
|
|
287
|
+
> persist. A `zeroRetention` run refuses every artifact fail-closed at the SDK
|
|
288
|
+
> gate.
|
|
289
|
+
|
|
290
|
+
The full artifact API — lineage (`link`), evaluations (`evaluate`), the reusable
|
|
291
|
+
per-step seam, and `run.scope` vs `run.start` — is the
|
|
292
|
+
[instrumentation guide](./docs/instrumentation.md), shipped in this package so
|
|
293
|
+
you can read it offline.
|
|
294
|
+
|
|
231
295
|
## CLI
|
|
232
296
|
|
|
233
297
|
The package ships a `modelrig` binary:
|
|
@@ -65,6 +65,8 @@ export interface GroundTruthStepInput {
|
|
|
65
65
|
}
|
|
66
66
|
export interface ActiveRunContext {
|
|
67
67
|
readonly runId: string;
|
|
68
|
+
/** The `episodeKey` given to run.start/run.scope, or null (RC-1). */
|
|
69
|
+
readonly episodeKey: string | null;
|
|
68
70
|
/** Trace id adopted from a traceparent at run.start, if any (chunk B use). */
|
|
69
71
|
readonly traceId: string | null;
|
|
70
72
|
/** Zero-retention gate (chunk B): true ⇒ artifact.save refuses fail-closed. */
|
|
@@ -96,6 +98,14 @@ export interface ActiveRunContext {
|
|
|
96
98
|
* and never thrown into the run loop. Marks its key as the ambient step. */
|
|
97
99
|
emitGroundTruthStep(input: GroundTruthStepInput): void;
|
|
98
100
|
}
|
|
101
|
+
/** RC-1: the `run_id` tag value an active run context stamps on every attempt
|
|
102
|
+
* row of BOTH lanes when the caller supplied none — the run's `episodeKey`
|
|
103
|
+
* (what the `/projects` tree joins on) or, without one, the run's id. Never
|
|
104
|
+
* overrides a caller-supplied `run_id`; no context ⇒ tags unchanged. */
|
|
105
|
+
export declare function withRunIdTag<T extends {
|
|
106
|
+
readonly run_id?: string;
|
|
107
|
+
readonly [key: string]: string | undefined;
|
|
108
|
+
}>(tags: T, ctx: ActiveRunContext | null): T;
|
|
99
109
|
/** The active run context, or null when none is set (namespace disabled, or no
|
|
100
110
|
* run.start on this async path). Read by the run loop's auto-step seam. */
|
|
101
111
|
export declare function getActiveRunContext(): ActiveRunContext | null;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-context.d.ts","sourceRoot":"","sources":["../../src/artifacts/run-context.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAIH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAEjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAI3D;;sEAEsE;AACtE,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAC;CAChC;AAED,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;AAEzC,MAAM,WAAW,eAAe;IAC9B,8DAA8D;IAC9D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,mEAAmE;IACnE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;wEAEoE;IACpE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;IACjC;;;;mCAI+B;IAC/B,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAC;CAC3B;AA4CD,4EAA4E;AAC5E,wBAAgB,2BAA2B,IAAI,IAAI,CAElD;AAED;gFACgF;AAChF,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,8EAA8E;IAC9E,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,+EAA+E;IAC/E,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,+EAA+E;IAC/E,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;;uFAEmF;IACnF,aAAa,IAAI,MAAM,EAAE,CAAC;IAC1B;;;wEAGoE;IACpE,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;IACnC;;;;;;;;;+DAS2D;IAC3D,cAAc,IAAI,MAAM,GAAG,IAAI,CAAC;IAChC;;gFAE4E;IAC5E,mBAAmB,CAAC,KAAK,EAAE,oBAAoB,GAAG,IAAI,CAAC;CACxD;AAaD;2EAC2E;AAC3E,wBAAgB,mBAAmB,IAAI,gBAAgB,GAAG,IAAI,CAG7D;AAQD,qEAAqE;AACrE,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAEnD;AAID;;;;8EAI8E;AAC9E,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,IAAI,EAAE,MAAM,MAAM,GACjB,MAAM,CAYR;AAED,2EAA2E;AAC3E,wBAAgB,2BAA2B,IAAI,IAAI,CAElD;AAED;oFACoF;AACpF,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,eAAe,GAAG,aAAa,GAAG,OAAO,CAAC;IACzD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,uDAAuD;AACvD,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,GAAG,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;IAC9B;mFAC+E;IAC/E,QAAQ,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI,CAAC;CAC1C;AAED,+EAA+E;AAC/E,MAAM,WAAW,YAAY;IAC3B;;;;2EAIuE;IACvE,KAAK,CAAC,IAAI,EAAE,eAAe,GAAG,SAAS,CAAC;IACxC,uEAAuE;IACvE,GAAG,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;IAC9B;oFACgF;IAChF,QAAQ,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI,CAAC;IACzC;;;;qBAIiB;IACjB,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CACnE;AASD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE;IACvC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B;;qFAEiF;IACjF,QAAQ,CAAC,aAAa,CAAC,EAAE;QAAE,wBAAwB,CAAC,GAAG,EAAE,qBAAqB,GAAG,IAAI,CAAA;KAAE,CAAC;CACzF,GAAG,YAAY,
|
|
1
|
+
{"version":3,"file":"run-context.d.ts","sourceRoot":"","sources":["../../src/artifacts/run-context.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAIH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAEjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAI3D;;sEAEsE;AACtE,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAC;CAChC;AAED,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;AAEzC,MAAM,WAAW,eAAe;IAC9B,8DAA8D;IAC9D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,mEAAmE;IACnE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;wEAEoE;IACpE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;IACjC;;;;mCAI+B;IAC/B,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAC;CAC3B;AA4CD,4EAA4E;AAC5E,wBAAgB,2BAA2B,IAAI,IAAI,CAElD;AAED;gFACgF;AAChF,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,qEAAqE;IACrE,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,8EAA8E;IAC9E,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,+EAA+E;IAC/E,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,+EAA+E;IAC/E,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;;uFAEmF;IACnF,aAAa,IAAI,MAAM,EAAE,CAAC;IAC1B;;;wEAGoE;IACpE,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;IACnC;;;;;;;;;+DAS2D;IAC3D,cAAc,IAAI,MAAM,GAAG,IAAI,CAAC;IAChC;;gFAE4E;IAC5E,mBAAmB,CAAC,KAAK,EAAE,oBAAoB,GAAG,IAAI,CAAC;CACxD;AAaD;;;wEAGwE;AACxE,wBAAgB,YAAY,CAAC,CAAC,SAAS;IAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAA;CAAE,EAC7G,IAAI,EAAE,CAAC,EACP,GAAG,EAAE,gBAAgB,GAAG,IAAI,GAC3B,CAAC,CAGH;AAED;2EAC2E;AAC3E,wBAAgB,mBAAmB,IAAI,gBAAgB,GAAG,IAAI,CAG7D;AAQD,qEAAqE;AACrE,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAEnD;AAID;;;;8EAI8E;AAC9E,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,IAAI,EAAE,MAAM,MAAM,GACjB,MAAM,CAYR;AAED,2EAA2E;AAC3E,wBAAgB,2BAA2B,IAAI,IAAI,CAElD;AAED;oFACoF;AACpF,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,eAAe,GAAG,aAAa,GAAG,OAAO,CAAC;IACzD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,uDAAuD;AACvD,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,GAAG,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;IAC9B;mFAC+E;IAC/E,QAAQ,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI,CAAC;CAC1C;AAED,+EAA+E;AAC/E,MAAM,WAAW,YAAY;IAC3B;;;;2EAIuE;IACvE,KAAK,CAAC,IAAI,EAAE,eAAe,GAAG,SAAS,CAAC;IACxC,uEAAuE;IACvE,GAAG,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;IAC9B;oFACgF;IAChF,QAAQ,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI,CAAC;IACzC;;;;qBAIiB;IACjB,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CACnE;AASD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE;IACvC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B;;qFAEiF;IACjF,QAAQ,CAAC,aAAa,CAAC,EAAE;QAAE,wBAAwB,CAAC,GAAG,EAAE,qBAAqB,GAAG,IAAI,CAAA;KAAE,CAAC;CACzF,GAAG,YAAY,CAwLf"}
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
*/
|
|
22
22
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
23
|
exports.resetRunMetaWarningsForTest = resetRunMetaWarningsForTest;
|
|
24
|
+
exports.withRunIdTag = withRunIdTag;
|
|
24
25
|
exports.getActiveRunContext = getActiveRunContext;
|
|
25
26
|
exports.isValidStepKey = isValidStepKey;
|
|
26
27
|
exports.resolveStepKey = resolveStepKey;
|
|
@@ -75,6 +76,15 @@ function resetRunMetaWarningsForTest() {
|
|
|
75
76
|
warnedRunMetaFields.clear();
|
|
76
77
|
}
|
|
77
78
|
const storage = new node_async_hooks_1.AsyncLocalStorage();
|
|
79
|
+
/** RC-1: the `run_id` tag value an active run context stamps on every attempt
|
|
80
|
+
* row of BOTH lanes when the caller supplied none — the run's `episodeKey`
|
|
81
|
+
* (what the `/projects` tree joins on) or, without one, the run's id. Never
|
|
82
|
+
* overrides a caller-supplied `run_id`; no context ⇒ tags unchanged. */
|
|
83
|
+
function withRunIdTag(tags, ctx) {
|
|
84
|
+
if (ctx === null || (typeof tags.run_id === "string" && tags.run_id !== ""))
|
|
85
|
+
return tags;
|
|
86
|
+
return { ...tags, run_id: ctx.episodeKey ?? ctx.runId };
|
|
87
|
+
}
|
|
78
88
|
/** The active run context, or null when none is set (namespace disabled, or no
|
|
79
89
|
* run.start on this async path). Read by the run loop's auto-step seam. */
|
|
80
90
|
function getActiveRunContext() {
|
|
@@ -145,6 +155,7 @@ function createRunNamespace(deps) {
|
|
|
145
155
|
const inertHandle = { runId: "", end() { }, evaluate() { } };
|
|
146
156
|
function buildContext(startRow, traceId, zeroRetention) {
|
|
147
157
|
const runId = startRow.id;
|
|
158
|
+
const episodeKey = startRow.episodeKey;
|
|
148
159
|
const attemptByStep = new Map();
|
|
149
160
|
const loaded = new Set();
|
|
150
161
|
let stepSeq = 0;
|
|
@@ -153,6 +164,7 @@ function createRunNamespace(deps) {
|
|
|
153
164
|
let lastStepKey = null;
|
|
154
165
|
return {
|
|
155
166
|
runId,
|
|
167
|
+
episodeKey,
|
|
156
168
|
traceId,
|
|
157
169
|
zeroRetention,
|
|
158
170
|
ended: false,
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* N6 — disagreement mining into eval-case candidates (near-optimal brief §3.2 N6).
|
|
3
|
+
*
|
|
4
|
+
* The discriminating-fixtures gap (brief §2): a route's fixtures separate *valid*
|
|
5
|
+
* from *invalid* but rarely *good* from *great*. The richest place to look for the
|
|
6
|
+
* fixtures that WOULD discriminate is where the bake-off's own signals DISAGREE on
|
|
7
|
+
* a replayed sample:
|
|
8
|
+
* - deterministic conformance says one thing, the value score another
|
|
9
|
+
* (schema-valid-but-wrong, or invalid-but-scored-right);
|
|
10
|
+
* - the Q1 quality gate's verdict contradicts conformance;
|
|
11
|
+
* - the SERVED arm and the REFERENCE arm diverge on the same capture (the
|
|
12
|
+
* top-of-distribution signal the brief names);
|
|
13
|
+
* - the pairwise judge upsets the expected order (arm beats / loses to the ref).
|
|
14
|
+
*
|
|
15
|
+
* Each such sample becomes an eval-case CANDIDATE — a ranked POINTER (captureId)
|
|
16
|
+
* an operator may promote through the existing E4 path. NEVER auto-promoted.
|
|
17
|
+
*
|
|
18
|
+
* CONTENT CUSTODY (P6 / spec): a candidate carries ONLY already-captured
|
|
19
|
+
* metadata — the captureId (an identifier), booleans, and numeric scores. No
|
|
20
|
+
* output text, no input text. Every string field is asserted meta-safe
|
|
21
|
+
* (`isMetaSafeString`) before it is surfaced, the same predicate the WI-1
|
|
22
|
+
* envelope uses. This module is PURE and fixture-tested.
|
|
23
|
+
*/
|
|
24
|
+
import type { ReplaySampleResult } from "./replay";
|
|
25
|
+
import type { PairwiseOutcome } from "./pairwise";
|
|
26
|
+
export type DisagreementReason = "conform_vs_value" | "conform_vs_gate" | "arm_vs_reference" | "pairwise_upset";
|
|
27
|
+
/** A per-sample verdict divergence, surfaced as a ranked eval-case candidate. All
|
|
28
|
+
* fields are already-captured metadata — a POINTER to promote, never content. */
|
|
29
|
+
export interface DisagreementCandidate {
|
|
30
|
+
/** The captured input's inference id — the promote path rehydrates from it. */
|
|
31
|
+
readonly captureId: string;
|
|
32
|
+
/** Which served arm this divergence is on (never the reference arm). */
|
|
33
|
+
readonly arm: string;
|
|
34
|
+
/** 0..1 magnitude of the disagreement — the ranking key (desc). */
|
|
35
|
+
readonly score: number;
|
|
36
|
+
/** Every signal pair that fired (ranked candidates usually have one; the
|
|
37
|
+
* top ones several). */
|
|
38
|
+
readonly reasons: readonly DisagreementReason[];
|
|
39
|
+
/** The already-captured numbers/booleans behind the score (no content). */
|
|
40
|
+
readonly signals: {
|
|
41
|
+
readonly armConform: boolean;
|
|
42
|
+
readonly armValueAccuracy: number | null;
|
|
43
|
+
readonly armQualityPassed: boolean | null;
|
|
44
|
+
readonly refConform?: boolean;
|
|
45
|
+
readonly refValueAccuracy?: number | null;
|
|
46
|
+
readonly pairwise?: PairwiseOutcome;
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
export interface DisagreementReport {
|
|
50
|
+
/** Paired/served samples examined across all arms. */
|
|
51
|
+
readonly examined: number;
|
|
52
|
+
/** Ranked candidates (score desc, then captureId asc), truncated to the cap. */
|
|
53
|
+
readonly candidates: readonly DisagreementCandidate[];
|
|
54
|
+
/** How many candidates existed BEFORE the cap — surfaced so a truncated list is
|
|
55
|
+
* never mistaken for "all of them" (no silent caps). */
|
|
56
|
+
readonly found: number;
|
|
57
|
+
/** The cap applied when `found > candidates.length`, else null. */
|
|
58
|
+
readonly cappedAt: number | null;
|
|
59
|
+
/** Count of candidates each reason contributed to (a candidate with two
|
|
60
|
+
* reasons counts once under each). */
|
|
61
|
+
readonly counts: Readonly<Record<DisagreementReason, number>>;
|
|
62
|
+
}
|
|
63
|
+
/** Default cap on the surfaced candidate list — enough to act on, bounded so a
|
|
64
|
+
* noisy replay set does not flood the surface. The full `found` count rides
|
|
65
|
+
* alongside so the truncation is legible. */
|
|
66
|
+
export declare const DEFAULT_DISAGREEMENT_CAP = 50;
|
|
67
|
+
export interface MineInput {
|
|
68
|
+
/** One entry per SERVED arm (never the reference): its per-sample results. */
|
|
69
|
+
readonly arms: readonly {
|
|
70
|
+
readonly arm: string;
|
|
71
|
+
readonly samples: readonly ReplaySampleResult[];
|
|
72
|
+
}[];
|
|
73
|
+
/** The reference arm's per-sample results, keyed for the arm-vs-reference and
|
|
74
|
+
* pairwise signals. Absent ⇒ those two reasons never fire (only the
|
|
75
|
+
* within-arm signals do). */
|
|
76
|
+
readonly reference?: readonly ReplaySampleResult[];
|
|
77
|
+
/** Pairwise verdicts keyed `${arm}\0${captureId}` → outcome, when a pairwise
|
|
78
|
+
* signal was computed. */
|
|
79
|
+
readonly pairwise?: ReadonlyMap<string, PairwiseOutcome>;
|
|
80
|
+
/** Surfaced-list cap (default {@link DEFAULT_DISAGREEMENT_CAP}). */
|
|
81
|
+
readonly maxCandidates?: number;
|
|
82
|
+
}
|
|
83
|
+
/** The pairwise map key — arm + captureId, NUL-joined so neither can spoof the
|
|
84
|
+
* other's boundary. */
|
|
85
|
+
export declare function pairwiseKey(arm: string, captureId: string): string;
|
|
86
|
+
/**
|
|
87
|
+
* Mine a replayed bake-off's per-sample results for verdict disagreements and
|
|
88
|
+
* rank them as eval-case candidates (N6). Pure over its inputs; deterministic
|
|
89
|
+
* ordering (score desc, then captureId asc). A produced sample with NO
|
|
90
|
+
* divergence past {@link SIGNAL_EPS} yields no candidate.
|
|
91
|
+
*/
|
|
92
|
+
export declare function mineDisagreements(input: MineInput): DisagreementReport;
|
|
93
|
+
//# sourceMappingURL=disagreement.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"disagreement.d.ts","sourceRoot":"","sources":["../../src/bakeoff/disagreement.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,MAAM,MAAM,kBAAkB,GAC1B,kBAAkB,GAClB,iBAAiB,GACjB,kBAAkB,GAClB,gBAAgB,CAAC;AAErB;iFACiF;AACjF,MAAM,WAAW,qBAAqB;IACpC,+EAA+E;IAC/E,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,wEAAwE;IACxE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;4BACwB;IACxB,QAAQ,CAAC,OAAO,EAAE,SAAS,kBAAkB,EAAE,CAAC;IAChD,2EAA2E;IAC3E,QAAQ,CAAC,OAAO,EAAE;QAChB,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;QAC7B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;QACzC,QAAQ,CAAC,gBAAgB,EAAE,OAAO,GAAG,IAAI,CAAC;QAC1C,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAC9B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1C,QAAQ,CAAC,QAAQ,CAAC,EAAE,eAAe,CAAC;KACrC,CAAC;CACH;AAED,MAAM,WAAW,kBAAkB;IACjC,sDAAsD;IACtD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,gFAAgF;IAChF,QAAQ,CAAC,UAAU,EAAE,SAAS,qBAAqB,EAAE,CAAC;IACtD;4DACwD;IACxD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC;0CACsC;IACtC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC,CAAC;CAC/D;AAED;;6CAE6C;AAC7C,eAAO,MAAM,wBAAwB,KAAK,CAAC;AAY3C,MAAM,WAAW,SAAS;IACxB,8EAA8E;IAC9E,QAAQ,CAAC,IAAI,EAAE,SAAS;QAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,kBAAkB,EAAE,CAAA;KAAE,EAAE,CAAC;IACpG;;iCAE6B;IAC7B,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,kBAAkB,EAAE,CAAC;IACnD;8BAC0B;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACzD,oEAAoE;IACpE,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED;uBACuB;AACvB,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAElE;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,SAAS,GAAG,kBAAkB,CA0FtE"}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* N6 — disagreement mining into eval-case candidates (near-optimal brief §3.2 N6).
|
|
4
|
+
*
|
|
5
|
+
* The discriminating-fixtures gap (brief §2): a route's fixtures separate *valid*
|
|
6
|
+
* from *invalid* but rarely *good* from *great*. The richest place to look for the
|
|
7
|
+
* fixtures that WOULD discriminate is where the bake-off's own signals DISAGREE on
|
|
8
|
+
* a replayed sample:
|
|
9
|
+
* - deterministic conformance says one thing, the value score another
|
|
10
|
+
* (schema-valid-but-wrong, or invalid-but-scored-right);
|
|
11
|
+
* - the Q1 quality gate's verdict contradicts conformance;
|
|
12
|
+
* - the SERVED arm and the REFERENCE arm diverge on the same capture (the
|
|
13
|
+
* top-of-distribution signal the brief names);
|
|
14
|
+
* - the pairwise judge upsets the expected order (arm beats / loses to the ref).
|
|
15
|
+
*
|
|
16
|
+
* Each such sample becomes an eval-case CANDIDATE — a ranked POINTER (captureId)
|
|
17
|
+
* an operator may promote through the existing E4 path. NEVER auto-promoted.
|
|
18
|
+
*
|
|
19
|
+
* CONTENT CUSTODY (P6 / spec): a candidate carries ONLY already-captured
|
|
20
|
+
* metadata — the captureId (an identifier), booleans, and numeric scores. No
|
|
21
|
+
* output text, no input text. Every string field is asserted meta-safe
|
|
22
|
+
* (`isMetaSafeString`) before it is surfaced, the same predicate the WI-1
|
|
23
|
+
* envelope uses. This module is PURE and fixture-tested.
|
|
24
|
+
*/
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.DEFAULT_DISAGREEMENT_CAP = void 0;
|
|
27
|
+
exports.pairwiseKey = pairwiseKey;
|
|
28
|
+
exports.mineDisagreements = mineDisagreements;
|
|
29
|
+
const attempt_meta_1 = require("../telemetry/attempt-meta");
|
|
30
|
+
/** Default cap on the surfaced candidate list — enough to act on, bounded so a
|
|
31
|
+
* noisy replay set does not flood the surface. The full `found` count rides
|
|
32
|
+
* alongside so the truncation is legible. */
|
|
33
|
+
exports.DEFAULT_DISAGREEMENT_CAP = 50;
|
|
34
|
+
/** Per-sample quality on the SAME 0..1 scale regret uses: the value-accuracy
|
|
35
|
+
* score when present, else 1/0 conformance. */
|
|
36
|
+
function quality01(s) {
|
|
37
|
+
return s.valueAccuracy ?? (s.conform ? 1 : 0);
|
|
38
|
+
}
|
|
39
|
+
/** A signal is "meaningfully divergent" past this magnitude (below it is noise —
|
|
40
|
+
* e.g. a 0.98-vs-1.0 field-accuracy hair-split is not a fixture worth promoting). */
|
|
41
|
+
const SIGNAL_EPS = 0.05;
|
|
42
|
+
/** The pairwise map key — arm + captureId, NUL-joined so neither can spoof the
|
|
43
|
+
* other's boundary. */
|
|
44
|
+
function pairwiseKey(arm, captureId) {
|
|
45
|
+
return `${arm}\0${captureId}`;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Mine a replayed bake-off's per-sample results for verdict disagreements and
|
|
49
|
+
* rank them as eval-case candidates (N6). Pure over its inputs; deterministic
|
|
50
|
+
* ordering (score desc, then captureId asc). A produced sample with NO
|
|
51
|
+
* divergence past {@link SIGNAL_EPS} yields no candidate.
|
|
52
|
+
*/
|
|
53
|
+
function mineDisagreements(input) {
|
|
54
|
+
const cap = input.maxCandidates ?? exports.DEFAULT_DISAGREEMENT_CAP;
|
|
55
|
+
const refByCapture = new Map((input.reference ?? []).map((s) => [s.captureId, s]));
|
|
56
|
+
const counts = {
|
|
57
|
+
conform_vs_value: 0,
|
|
58
|
+
conform_vs_gate: 0,
|
|
59
|
+
arm_vs_reference: 0,
|
|
60
|
+
pairwise_upset: 0,
|
|
61
|
+
};
|
|
62
|
+
const candidates = [];
|
|
63
|
+
let examined = 0;
|
|
64
|
+
for (const { arm, samples } of input.arms) {
|
|
65
|
+
for (const s of samples) {
|
|
66
|
+
// Only PRODUCED samples carry verdicts to disagree about; a failed sample
|
|
67
|
+
// (no output) has nothing to mine. Guard the captureId is meta-safe before
|
|
68
|
+
// it is surfaced (captureIds are identifiers, always safe — asserted, never
|
|
69
|
+
// assumed, per the content-custody rule).
|
|
70
|
+
if (s.failureClass !== null)
|
|
71
|
+
continue;
|
|
72
|
+
if (!(0, attempt_meta_1.isMetaSafeString)(s.captureId))
|
|
73
|
+
continue;
|
|
74
|
+
examined += 1;
|
|
75
|
+
const reasons = [];
|
|
76
|
+
let score = 0;
|
|
77
|
+
const conform01 = s.conform ? 1 : 0;
|
|
78
|
+
// conform_vs_value: schema conformance disagrees with the value score.
|
|
79
|
+
if (s.valueAccuracy !== null) {
|
|
80
|
+
const gap = Math.abs(conform01 - s.valueAccuracy);
|
|
81
|
+
if (gap > SIGNAL_EPS) {
|
|
82
|
+
reasons.push("conform_vs_value");
|
|
83
|
+
score = Math.max(score, gap);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
// conform_vs_gate: the Q1 boolean verdict contradicts conformance.
|
|
87
|
+
if (s.qualityPassed !== null && s.qualityPassed !== s.conform) {
|
|
88
|
+
reasons.push("conform_vs_gate");
|
|
89
|
+
score = Math.max(score, 1);
|
|
90
|
+
}
|
|
91
|
+
const ref = refByCapture.get(s.captureId);
|
|
92
|
+
const signals = {
|
|
93
|
+
armConform: s.conform,
|
|
94
|
+
armValueAccuracy: s.valueAccuracy,
|
|
95
|
+
armQualityPassed: s.qualityPassed,
|
|
96
|
+
...(ref !== undefined ? { refConform: ref.conform, refValueAccuracy: ref.valueAccuracy } : {}),
|
|
97
|
+
};
|
|
98
|
+
// arm_vs_reference: the served arm and the reference diverge on this capture
|
|
99
|
+
// (the top-of-distribution signal — where the fixture corpus learns to tell
|
|
100
|
+
// good from great).
|
|
101
|
+
if (ref !== undefined) {
|
|
102
|
+
const gap = Math.abs(quality01(s) - quality01(ref));
|
|
103
|
+
if (gap > SIGNAL_EPS) {
|
|
104
|
+
reasons.push("arm_vs_reference");
|
|
105
|
+
score = Math.max(score, gap);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
// pairwise_upset: a non-tie pairwise verdict is itself a divergence worth a
|
|
109
|
+
// fixture (an arm beating OR losing to the reference by preference).
|
|
110
|
+
const pw = input.pairwise?.get(pairwiseKey(arm, s.captureId));
|
|
111
|
+
if (pw !== undefined && pw !== "tie") {
|
|
112
|
+
reasons.push("pairwise_upset");
|
|
113
|
+
score = Math.max(score, 1);
|
|
114
|
+
}
|
|
115
|
+
if (reasons.length === 0)
|
|
116
|
+
continue;
|
|
117
|
+
for (const r of reasons)
|
|
118
|
+
counts[r] += 1;
|
|
119
|
+
candidates.push({
|
|
120
|
+
captureId: s.captureId,
|
|
121
|
+
arm,
|
|
122
|
+
score,
|
|
123
|
+
reasons,
|
|
124
|
+
signals: pw !== undefined ? { ...signals, pairwise: pw } : signals,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
// Deterministic ranking: strongest disagreement first, captureId as a stable
|
|
129
|
+
// tiebreak so the same replay set always ranks identically.
|
|
130
|
+
candidates.sort((a, b) => b.score - a.score || a.captureId.localeCompare(b.captureId));
|
|
131
|
+
const found = candidates.length;
|
|
132
|
+
const surfaced = candidates.slice(0, cap);
|
|
133
|
+
return {
|
|
134
|
+
examined,
|
|
135
|
+
candidates: surfaced,
|
|
136
|
+
found,
|
|
137
|
+
cappedAt: found > surfaced.length ? cap : null,
|
|
138
|
+
counts,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/bakeoff/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAgB,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/bakeoff/engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAG1C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAgB,MAAM,gBAAgB,CAAC;AAS9F,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAmC,cAAc,EAAE,MAAM,SAAS,CAAC;AAElH;;+BAE+B;AAC/B,eAAO,MAAM,aAAa,kBAAkB,CAAC;AAE7C;;;gFAGgF;AAChF,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE;IACvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAC;CACtC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAgBpB,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,WAAW,CAAC,EAAE,eAAe,CAAC;IACvC;iEAC6D;IAC7D,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC;;wEAEoE;IACpE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,aAAa,GAAG,IAAI,CAAC;IAC7E;+BAC2B;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACrE;;;;;+EAK2E;IAC3E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;CACpC;AAsBD,wBAAsB,UAAU,CAC9B,IAAI,EAAE,OAAO,EACb,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,cAAc,EACnB,WAAW,CAAC,EAAE,kBAAkB,GAC/B,OAAO,CAAC,eAAe,CAAC,CA8K1B;AAuCD;iEACiE;AACjE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC;IACtC,mEAAmE;IACnE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AA8BD;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,OAAO,EAAE,SAAS,cAAc,EAAE,EAClC,GAAG,CAAC,EAAE,eAAe,GACpB,eAAe,CAAC,UAAU,CAAC,CA8G7B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAED;;uDAEuD;AACvD,wBAAsB,aAAa,CACjC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAuCpG"}
|