@skillstate/bench 3.0.0 → 3.0.1

package/dist/ab/usage.js

@@ -0,0 +1,98 @@
+/**
+ * Token accounting read from the HOST's own store.
+ *
+ * ── Why not parse the host's stdout ──────────────────────────────────────
+ *
+ * Three reasons, in order of how much they matter:
+ *
+ * 1. **The model does not report its own cost reliably.** A harness that
+ *    takes the number the subject of the experiment wrote down is trusting the
+ *    measurement to the thing being measured. The previous A/B took its
+ *    figures from the run's own reporting, and one of its two "arms" was
+ *    itself two different readings of a single session.
+ * 2. **OpenCode already records per-message token usage** in its local store
+ *    (`message.data.tokens` with `input`, `output`, `reasoning` and
+ *    `cache.{read,write}`). That is the host's own accounting, not a
+ *    reconstruction, and it is keyed by session so a run can be re-read
+ *    after the fact.
+ * 3. **It is available without a live model.** Reading the store works when
+ *    the provider quota is exhausted, which is exactly when one most wants to
+ *    re-analyse a previous run.
+ *
+ * Node's `node:sqlite` is used rather than a dependency: the store is a
+ * local file, the read is a single prepared statement, and adding a native
+ * dependency to a zero-deps package for this would be a poor trade. The
+ * reader is injected, so the query and its failure modes are testable
+ * without an OpenCode installation.
+ *
+ * @non-paper — measurement infrastructure, not the paper's evaluation.
+ */
+/** Zero usage, for a session whose messages carry no accounting. */
+export const NO_USAGE = {
+    input: 0,
+    cacheRead: 0,
+    cacheWrite: 0,
+    output: 0,
+};
+/** Sum one row's token fields, tolerating a partial or absent record. */
+function rowUsage(tokens) {
+    return {
+        input: tokens?.input ?? 0,
+        cacheRead: tokens?.cacheRead ?? 0,
+        cacheWrite: tokens?.cacheWrite ?? 0,
+        output: tokens?.output ?? 0,
+    };
+}
+/**
+ * Total token spend for a session.
+ *
+ * Only `assistant` rows count. A user message's `input` figure is the host
+ * describing the prompt it echoed, and adding it would double-count the
+ * conversation.
+ *
+ * `cacheRead` is summed, not treated as free: prompt caching changes the
+ * price of a token, not the fact that the model was shown it, and the
+ * paper's claim is about what the model is exposed to.
+ */
+export function sessionUsage(rows) {
+    return rows
+        .filter((row) => row.role === 'assistant')
+        .map((row) => rowUsage(row.tokens))
+        .reduce((sum, usage) => ({
+        input: sum.input + usage.input,
+        cacheRead: sum.cacheRead + usage.cacheRead,
+        cacheWrite: sum.cacheWrite + usage.cacheWrite,
+        output: sum.output + usage.output,
+    }), NO_USAGE);
+}
+/**
+ * Resolve a session's token spend, turning every failure into a value.
+ *
+ * A run whose accounting cannot be read is a run with unknown cost, which is
+ * not the same as a run that cost nothing. The distinction is the whole
+ * reason this returns a union instead of a number: `store_unavailable` must
+ * never be silently read as `NO_USAGE`.
+ */
+export async function resolveSessionUsage(reader, sessionID) {
+    let rows;
+    try {
+        rows = await reader.messagesFor(sessionID);
+    }
+    catch (error) {
+        return {
+            ok: false,
+            reason: 'store_unavailable',
+            detail: error instanceof Error ? error.message : String(error),
+        };
+    }
+    if (rows.length === 0) {
+        return {
+            ok: false,
+            reason: 'session_unknown',
+            detail: `no messages recorded for session ${sessionID}`,
+        };
+    }
+    const assistant = rows.filter((row) => row.role === 'assistant').length;
+    return { ok: true, usage: sessionUsage(rows), assistantMessages: assistant };
+}
+//# sourceMappingURL=usage.js.map

package/dist/ab/usage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"usage.js","sourceRoot":"","sources":["../../src/ab/usage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAmCH,oEAAoE;AACpE,MAAM,CAAC,MAAM,QAAQ,GAAe;IAClC,KAAK,EAAE,CAAC;IACR,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE,CAAC;IACb,MAAM,EAAE,CAAC;CACV,CAAC;AAEF,yEAAyE;AACzE,SAAS,QAAQ,CAAC,MAAkC;IAClD,OAAO;QACL,KAAK,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC;QACzB,SAAS,EAAE,MAAM,EAAE,SAAS,IAAI,CAAC;QACjC,UAAU,EAAE,MAAM,EAAE,UAAU,IAAI,CAAC;QACnC,MAAM,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,YAAY,CAAC,IAA+B;IAC1D,OAAO,IAAI;SACR,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,WAAW,CAAC;SACzC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SAClC,MAAM,CACL,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACf,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK;QAC9B,SAAS,EAAE,GAAG,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS;QAC1C,UAAU,EAAE,GAAG,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU;QAC7C,MAAM,EAAE,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM;KAClC,CAAC,EACF,QAAQ,CACT,CAAC;AACN,CAAC;AAcD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,MAAmB,EACnB,SAAiB;IAEjB,IAAI,IAA+B,CAAC;IACpC,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,mBAAmB;YAC3B,MAAM,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAC/D,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,iBAAiB;YACzB,MAAM,EAAE,oCAAoC,SAAS,EAAE;SACxD,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;IACxE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,iBAAiB,EAAE,SAAS,EAAE,CAAC;AAC/E,CAAC"}

package/dist/ab/verdict.d.ts

@@ -0,0 +1,88 @@
+/**
+ * The gates: what has to be true before a token comparison may be reported.
+ *
+ * ── What went wrong the first time ───────────────────────────────────────
+ *
+ * The previous A/B reported a 39% saving from a run in which the
+ * instrumented arm never once wrote the state file. Nothing flagged it,
+ * because the pipeline carried a number and a number has no opinion about
+ * whether the thing it measured was switched on.
+ *
+ * So the harness's primary output is a {@link Verdict}, and a bare percentage
+ * is only reachable by passing every gate. The gates are deliberately
+ * pessimistic and deliberately ordered: the most invalidating finding is
+ * computed first, so a later gate cannot refine a comparison that was never
+ * a comparison.
+ *
+ * @non-paper — measurement infrastructure for OUR host integration, not the
+ * paper's own evaluation. The paper's Limitations are cited here as the
+ * source of one gate, not as a claim being verified.
+ */
+import type { ArmId, ArmRecord } from './record.js';
+import type { Distribution, EffectSize } from './stats-core.js';
+/** Minimum trials per arm before a comparison may claim anything. */
+export declare const MIN_TRIALS = 2;
+/**
+ * What the harness concluded.
+ *
+ * The last four are refusals, and refusals are the point: a harness that can
+ * only produce numbers is a harness that will produce the wrong number.
+ */
+export type Verdict = 
+/** Engaged, comparable, and the instrumented arm spent significantly fewer prompt tokens. */
+'saving'
+/** Engaged, comparable, and the instrumented arm spent significantly more. */
+ | 'regression'
+/** A valid run whose effect sits inside the noise. */
+ | 'no-effect'
+/** The instrumented arm never wrote state. No number is reported. */
+ | 'inert'
+/** The arms did not do comparable work, so their tokens are not comparable. */
+ | 'not-comparable'
+/** A failed run, a mismatched task/model/host, or too few trials. */
+ | 'invalid';
+/** A single gate that refused to let a number be reported. */
+export interface GateFailure {
+    readonly gate: 'comparability' | 'completeness' | 'sample-size' | 'engagement' | 'task-equivalence' | 'outcome' | 'variance' | 'paper-compatibility';
+    /** Why the gate fired, phrased so a reader can act on it. */
+    readonly detail: string;
+}
+export interface VerdictResult {
+    readonly verdict: Verdict;
+    /** Every gate that fired, in gate order. */
+    readonly failures: readonly GateFailure[];
+    /** Per-arm prompt-token distributions; empty arms carry `n: 0`. */
+    readonly distributions: Readonly<Record<ArmId, Distribution>>;
+    /** The effect size, only when both arms have usable samples. */
+    readonly effect: EffectSize | null;
+    /** One line for printing. Never a bare percentage on a refusal. */
+    readonly summary: string;
+}
+/** Options for {@link runExperiment}. */
+export interface ExperimentOptions {
+    /**
+     * Set when the task's objective is defined over the historical trajectory
+     * (an audit, a review, a "what did I just do" task).
+     *
+     * Limitations, case (3): the paper's own assumption fails exactly there. The
+     * section is not numbered — `state.md` records it as `§ Limitations` with no
+     * number, so a number here would be a citation nobody can check. An
+     * unverifiable citation is worse than none: it looks verified.
+     * A flat result on such a task is the predicted outcome, so reporting it
+     * as a refutation would be a category error in the harness's own favour.
+     * The gate downgrades `no-effect` to a failure that says so.
+     */
+    readonly taskNeedsTranscript?: boolean;
+    /** Trials required per arm. Defaults to {@link MIN_TRIALS}. */
+    readonly minTrials?: number;
+}
+/**
+ * Run every gate and return the strongest conclusion the data supports.
+ *
+ * Pure and deterministic: no clock, no filesystem, no randomness. Identical
+ * input yields an identical result, which is what lets the gates be tested
+ * against the real historical runs and against inputs built to trip each
+ * branch.
+ */
+export declare function runExperiment(arms: ReadonlyMap<ArmId, ArmRecord>, options?: ExperimentOptions): VerdictResult;
+//# sourceMappingURL=verdict.d.ts.map

package/dist/ab/verdict.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict.d.ts","sourceRoot":"","sources":["../../src/ab/verdict.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAEpD,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAEhE,qEAAqE;AACrE,eAAO,MAAM,UAAU,IAAI,CAAC;AAE5B;;;;;GAKG;AACH,MAAM,MAAM,OAAO;AACjB,6FAA6F;AAC3F,QAAQ;AACV,8EAA8E;GAC5E,YAAY;AACd,sDAAsD;GACpD,WAAW;AACb,qEAAqE;GACnE,OAAO;AACT,+EAA+E;GAC7E,gBAAgB;AAClB,qEAAqE;GACnE,SAAS,CAAC;AAEd,8DAA8D;AAC9D,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EACT,eAAe,GACf,cAAc,GACd,aAAa,GACb,YAAY,GACZ,kBAAkB,GAClB,SAAS,GACT,UAAU,GACV,qBAAqB,CAAC;IAC1B,6DAA6D;IAC7D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,4CAA4C;IAC5C,QAAQ,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,CAAC;IAC1C,mEAAmE;IACnE,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC;IAC9D,gEAAgE;IAChE,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;IACnC,mEAAmE;IACnE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,yCAAyC;AACzC,MAAM,WAAW,iBAAiB;IAChC;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IACvC,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AA0BD;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,EACnC,OAAO,GAAE,iBAAsB,GAC9B,aAAa,CAiOf"}

package/dist/ab/verdict.js

@@ -0,0 +1,251 @@
+/**
+ * The gates: what has to be true before a token comparison may be reported.
+ *
+ * ── What went wrong the first time ───────────────────────────────────────
+ *
+ * The previous A/B reported a 39% saving from a run in which the
+ * instrumented arm never once wrote the state file. Nothing flagged it,
+ * because the pipeline carried a number and a number has no opinion about
+ * whether the thing it measured was switched on.
+ *
+ * So the harness's primary output is a {@link Verdict}, and a bare percentage
+ * is only reachable by passing every gate. The gates are deliberately
+ * pessimistic and deliberately ordered: the most invalidating finding is
+ * computed first, so a later gate cannot refine a comparison that was never
+ * a comparison.
+ *
+ * @non-paper — measurement infrastructure for OUR host integration, not the
+ * paper's own evaluation. The paper's Limitations are cited here as the
+ * source of one gate, not as a claim being verified.
+ */
+import { isInstrumented, promptTokens } from './record.js';
+import { describe, effectSize, MIN_EFFECT_IN_MADS } from './stats-core.js';
+/** Minimum trials per arm before a comparison may claim anything. */
+export const MIN_TRIALS = 2;
+const EMPTY_DISTRIBUTION = {
+    n: 0,
+    min: 0,
+    median: 0,
+    max: 0,
+    mad: 0,
+    relativeMad: 0,
+    sorted: [],
+};
+function distributionMap(arms) {
+    const out = {
+        plain: EMPTY_DISTRIBUTION,
+        notes: EMPTY_DISTRIBUTION,
+        paper: EMPTY_DISTRIBUTION,
+    };
+    for (const [arm, record] of arms) {
+        out[arm] = describe(record.runs.map((run) => promptTokens(run.usage)));
+    }
+    return out;
+}
+/**
+ * Run every gate and return the strongest conclusion the data supports.
+ *
+ * Pure and deterministic: no clock, no filesystem, no randomness. Identical
+ * input yields an identical result, which is what lets the gates be tested
+ * against the real historical runs and against inputs built to trip each
+ * branch.
+ */
+export function runExperiment(arms, options = {}) {
+    const minTrials = options.minTrials ?? MIN_TRIALS;
+    const failures = [];
+    const distributions = distributionMap(arms);
+    const control = arms.get('plain');
+    const instrumented = [...arms.entries()].filter(([arm]) => isInstrumented(arm));
+    // Every gate is evaluated before any early return, so a caller fixing a
+    // broken experiment is told everything that is wrong in one pass rather
+    // than discovering the next problem on the next run. Only gates that are
+    // undefined without a control arm (equivalence, effect) are skipped.
+    const hasBothArms = control !== undefined && instrumented.length > 0;
+    if (!hasBothArms) {
+        failures.push({
+            gate: 'comparability',
+            detail: 'need a `plain` control arm and at least one instrumented arm',
+        });
+    }
+    // ── Gate 1: comparability — same task, same model, same host ───────────
+    for (const [arm, record] of hasBothArms ? instrumented : []) {
+        const baseline = control;
+        if (record.task !== baseline.task ||
+            record.model !== baseline.model ||
+            record.hostVersion !== baseline.hostVersion) {
+            failures.push({
+                gate: 'comparability',
+                detail: `arm ${arm} ran ${JSON.stringify(record.task)} on ${record.model}@${record.hostVersion}` +
+                    ` but the control ran ${JSON.stringify(baseline.task)} on ${baseline.model}@${baseline.hostVersion}`,
+            });
+        }
+    }
+    // ── Gate 2: completeness — every run actually finished ────────────────
+    for (const [arm, record] of arms) {
+        for (const run of record.runs) {
+            if (!run.completed) {
+                failures.push({
+                    gate: 'completeness',
+                    detail: `arm ${arm} trial ${run.trial} did not complete: ${run.error ?? 'unknown error'}`,
+                });
+            }
+        }
+    }
+    // ── Gate 2b: the outcome — did the work get done? ───────────────────────
+    //
+    // The last gate, and the one that was missing. Every other gate asks whether
+    // the comparison is valid; none asks whether the result is worth anything.
+    // An arm that reads every file, writes state and never produces the answer
+    // passes engagement, comparability and variance, and would be reported as a
+    // saving. That is the failure this project has already paid for once: "paper
+    // 81,685 tokens vs control 203,801 = 60% cheaper, paper answered WRONG".
+    //
+    // Two refusals, and the second is the one that matters most. An experiment
+    // that measured cost and not the work has not earned the right to report the
+    // cost, whatever its token numbers say.
+    const measured = (record) => record.runs.some((run) => run.outcome !== undefined);
+    for (const [arm, record] of instrumented) {
+        if (!measured(record) || control === undefined || !measured(control)) {
+            failures.push({
+                gate: 'outcome',
+                detail: `arm ${arm} did not record whether the task was answered — a cost ` +
+                    'saving with no outcome is not a result, so none is reported. Pass ' +
+                    '`correct` in the trial files to measure it.',
+            });
+            continue;
+        }
+        const correct = (record) => record.runs.filter((run) => run.outcome?.correct === true).length;
+        // Strictly, not "at least as good". The n=7 experiment's honest reading
+        // was that the arms are comparably accurate and only the cost differs, and
+        // a gate that fired on a tie would have thrown that finding away.
+        if (correct(record) / record.runs.length < correct(control) / control.runs.length) {
+            failures.push({
+                gate: 'outcome',
+                detail: `arm ${arm} answered correctly ${correct(record)} of ${record.runs.length} trial(s) ` +
+                    `while the control answered ${correct(control)} of ${control.runs.length} — ` +
+                    'a cheaper run that did not finish the work is a regression, not a saving',
+            });
+        }
+    }
+    // ── Gate 3: sample size — one run cannot show variance ────────────────
+    for (const [arm, record] of arms) {
+        if (record.runs.length < minTrials) {
+            failures.push({
+                gate: 'sample-size',
+                detail: `arm ${arm} ran ${record.runs.length} trial(s); ${minTrials} are needed` +
+                    ' to separate an effect from run-to-run variance',
+            });
+        }
+    }
+    // ── Gate 4: engagement — was the integration switched on at all? ───────
+    //
+    // First in the list of reasons a run is unusable, because when it fires
+    // every token number in the experiment is a sample of model variance. The
+    // detail distinguishes "never engaged" from "engaged but every response
+    // was rejected", which are different bugs with different fixes.
+    for (const [arm, record] of instrumented) {
+        if (record.runs.every((run) => !run.engagement.engaged)) {
+            const rejections = record.runs.reduce((n, run) => n + run.engagement.rejections, 0);
+            const first = record.runs.find((run) => run.engagement.firstRejection !== undefined)?.engagement.firstRejection;
+            const detail = rejections > 0
+                ? `arm ${arm} never wrote state and rejected ${rejections} response(s)` +
+                    (first === undefined ? '' : ` (first: ${first})`) +
+                    ' — the model tried and the integration refused every patch, so its tokens measure the rejection, not the integration'
+                : `arm ${arm} never wrote the state file in ${record.runs.length} trial(s) — the integration was inert, so its token count is not a measurement of the integration`;
+            failures.push({ gate: 'engagement', detail });
+        }
+    }
+    // ── Gate 5: task equivalence — did both arms do the same work? ─────────
+    //
+    // Byte-identical artifacts are the strongest available evidence that both
+    // arms finished the same thing. A digest that appears in no control run
+    // means the arms are not comparable, whatever their token counts say.
+    const controlDigests = new Set(control === undefined ? [] : control.runs.map((run) => run.work.artifactDigest));
+    for (const [arm, record] of hasBothArms ? instrumented : []) {
+        for (const run of record.runs) {
+            if (!controlDigests.has(run.work.artifactDigest)) {
+                failures.push({
+                    gate: 'task-equivalence',
+                    detail: `arm ${arm} trial ${run.trial} produced artifact digest ${String(run.work.artifactDigest)},` +
+                        ` which is not among the control's (${[...controlDigests].map(String).join(', ')})`,
+                });
+            }
+        }
+    }
+    // Gates 1, 2, 3 and 5 all invalidate the comparison itself. Engagement (4)
+    // gets its own verdict, because "the integration did nothing" is a
+    // materially different finding from "the arms did different work".
+    const blocking = failures;
+    if (blocking.length > 0) {
+        const inert = blocking.some((failure) => failure.gate === 'engagement');
+        const invalid = blocking.some((failure) => failure.gate === 'comparability' ||
+            failure.gate === 'completeness' ||
+            failure.gate === 'sample-size');
+        return refuse(inert ? 'inert' : invalid ? 'invalid' : 'not-comparable', blocking, distributions);
+    }
+    // ── Gate 6: variance — is the effect bigger than the noise? ────────────
+    const [instrumentedArm, instrumentedRecord] = instrumented[0];
+    const effect = effectSize(control.runs.map((run) => promptTokens(run.usage)), instrumentedRecord.runs.map((run) => promptTokens(run.usage)));
+    if (effect.inMads !== null && Math.abs(effect.inMads) < MIN_EFFECT_IN_MADS) {
+        failures.push({
+            gate: 'variance',
+            detail: `effect is ${effect.inMads.toFixed(2)} MADs, under the ${MIN_EFFECT_IN_MADS} required to call it a signal;` +
+                ` the ${instrumentedArm} arm's own spread is ±${(distributions[instrumentedArm].relativeMad * 100).toFixed(0)}% of its median`,
+        });
+    }
+    // ── Gate 7: paper compatibility — a null on the wrong task is not a null ─
+    if (failures.length === 0 && options.taskNeedsTranscript === true) {
+        failures.push({
+            gate: 'paper-compatibility',
+            detail: 'this task is defined over the historical trajectory (audit-style), which is the case the paper’s Limitations call out' +
+                ' predicts will not benefit from a bounded prompt; a flat result here is consistent with the paper, not a' +
+                ' refutation of it — measure a task that needs cross-turn memory',
+        });
+    }
+    if (failures.length > 0) {
+        // A flat result on a transcript-shaped task is not a measurement of the
+        // integration at all, so it gets its own headline: a reader who sees
+        // "NO-EFFECT" would reasonably conclude skillstate does not help, when
+        // the honest statement is that this task was never a test of it.
+        const onTranscriptTask = options.taskNeedsTranscript === true &&
+            failures[0].gate === 'paper-compatibility';
+        return {
+            verdict: 'no-effect',
+            failures,
+            distributions,
+            effect,
+            summary: `${onTranscriptTask ? 'NOT-A-TEST' : 'NO-EFFECT'}: ${failures[0].detail} [${failures.length} gate(s) failed]`,
+        };
+    }
+    const direction = effect.difference > 0 ? 'fewer' : 'more';
+    const magnitude = effect.inMads === null
+        ? 'exact: both arms had zero spread'
+        : `${Math.abs(effect.inMads).toFixed(1)} MADs`;
+    return {
+        verdict: effect.difference > 0 ? 'saving' : 'regression',
+        failures,
+        distributions,
+        effect,
+        summary: `${effect.difference > 0 ? 'SAVING' : 'REGRESSION'}: arm ${instrumentedArm} spent ` +
+            `${Math.abs(effect.difference)} ${direction} prompt tokens at the median (${magnitude})`,
+    };
+}
+/** Build a refusal result. Every gate failure is reported, never just the first. */
+function refuse(verdict, failures, distributions) {
+    return {
+        verdict,
+        failures,
+        distributions,
+        effect: null,
+        summary: `${HEADLINE[verdict]}: ${failures[0].detail} [${failures.length} gate(s) failed]`,
+    };
+}
+const HEADLINE = {
+    saving: 'SAVING',
+    regression: 'REGRESSION',
+    'no-effect': 'NO-EFFECT',
+    inert: 'INERT',
+    'not-comparable': 'NOT-COMPARABLE',
+    invalid: 'INVALID',
+};
+//# sourceMappingURL=verdict.js.map

package/dist/ab/verdict.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict.js","sourceRoot":"","sources":["../../src/ab/verdict.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3D,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAG3E,qEAAqE;AACrE,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC;AAoE5B,MAAM,kBAAkB,GAAiB;IACvC,CAAC,EAAE,CAAC;IACJ,GAAG,EAAE,CAAC;IACN,MAAM,EAAE,CAAC;IACT,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,WAAW,EAAE,CAAC;IACd,MAAM,EAAE,EAAE;CACX,CAAC;AAEF,SAAS,eAAe,CACtB,IAAmC;IAEnC,MAAM,GAAG,GAAgC;QACvC,KAAK,EAAE,kBAAkB;QACzB,KAAK,EAAE,kBAAkB;QACzB,KAAK,EAAE,kBAAkB;KAC1B,CAAC;IACF,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACjC,GAAG,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAC3B,IAAmC,EACnC,OAAO,GAAsB,EAAE;IAE/B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,UAAU,CAAC;IAClD,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAE5C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;IAEhF,wEAAwE;IACxE,wEAAwE;IACxE,yEAAyE;IACzE,qEAAqE;IACrE,MAAM,WAAW,GAAG,OAAO,KAAK,SAAS,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;IACrE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE,8DAA8D;SACvE,CAAC,CAAC;IACL,CAAC;IAED,0EAA0E;IAC1E,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC5D,MAAM,QAAQ,GAAG,OAAQ,CAAC;QAC1B,IACE,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI;YAC7B,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK;YAC/B,MAAM,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW,EAC3C,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,eAAe;gBACrB,MAAM,EACJ,OAAO,GAAG,QAAQ,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,WAAW,EAAE;oBACxF,wBAAwB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,WAAW,EAAE;aACvG,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACjC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;gBACnB,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,cAAc;oBACpB,MAAM,EAAE,OAAO,GAAG,UAAU,GAAG,CAAC,KAAK,sBAAsB,GAAG,CAAC,KAAK,IAAI,eAAe,EAAE;iBAC1F,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,EAAE;IACF,6EAA6E;IAC7E,2EAA2E;IAC3E,2EAA2E;IAC3E,4EAA4E;IAC5E,6EAA6E;IAC7E,yEAAyE;IACzE,EAAE;IACF,2EAA2E;IAC3E,6EAA6E;IAC7E,wCAAwC;IACxC,MAAM,QAAQ,GAAG,CAAC,MAAiB,EAAW,EAAE,CAC9C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC;IACvD,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QACzC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,KAAK,SAAS,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACrE,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,SAAS;gBACf,MAAM,EACJ,OAAO,GAAG,yDAAyD;oBACnE,oEAAoE;oBACpE,6CAA6C;aAChD,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,MAAiB,EAAU,EAAE,CAC5C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;QACpE,wEAAwE;QACxE,2EAA2E;QAC3E,kEAAkE;QAClE,IAAI,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClF,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,SAAS;gBACf,MAAM,EACJ,OAAO,GAAG,uBAAuB,OAAO,CAAC,MAAM,CAAC,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,YAAY;oBACrF,8BAA8B,OAAO,CAAC,OAAO,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK;oBAC7E,0EAA0E;aAC7E,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACjC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YACnC,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,aAAa;gBACnB,MAAM,EACJ,OAAO,GAAG,QAAQ,MAAM,CAAC,IAAI,CAAC,MAAM,cAAc,SAAS,aAAa;oBACxE,iDAAiD;aACpD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,EAAE;IACF,wEAAwE;IACxE,0EAA0E;IAC1E,wEAAwE;IACxE,gEAAgE;IAChE,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QACzC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACxD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CACnC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,UAAU,EACzC,CAAC,CACF,CAAC;YACF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAC5B,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,KAAK,SAAS,CACrD,EAAE,UAAU,CAAC,cAAc,CAAC;YAC7B,MAAM,MAAM,GACV,UAAU,GAAG,CAAC;gBACZ,CAAC,CAAC,OAAO,GAAG,mCAAmC,UAAU,cAAc;oBACrE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,GAAG,CAAC;oBACjD,sHAAsH;gBACxH,CAAC,CAAC,OAAO,GAAG,kCAAkC,MAAM,CAAC,IAAI,CAAC,MAAM,mGAAmG,CAAC;YACxK,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,EAAE;IACF,0EAA0E;IAC1E,wEAAwE;IACxE,sEAAsE;IACtE,MAAM,cAAc,GAAG,IAAI,GAAG,CAC5B,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAChF,CAAC;IACF,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC5D,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;gBACjD,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,kBAAkB;oBACxB,MAAM,EACJ,OAAO,GAAG,UAAU,GAAG,CAAC,KAAK,6BAA6B,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG;wBAC5F,sCAAsC,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;iBACtF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,mEAAmE;IACnE,mEAAmE;IACnE,MAAM,QAAQ,GAAG,QAAQ,CAAC;IAC1B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC;QACxE,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAC3B,CAAC,OAAO,EAAE,EAAE,CACV,OAAO,CAAC,IAAI,KAAK,eAAe;YAChC,OAAO,CAAC,IAAI,KAAK,cAAc;YAC/B,OAAO,CAAC,IAAI,KAAK,aAAa,CACjC,CAAC;QACF,OAAO,MAAM,CACX,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,EACxD,QAAQ,EACR,aAAa,CACd,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,MAAM,CAAC,eAAe,EAAE,kBAAkB,CAAC,GAAG,YAAY,CAAC,CAAC,CAAE,CAAC;IAC/D,MAAM,MAAM,GAAG,UAAU,CACvB,OAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EACnD,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAC9D,CAAC;IAEF,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,kBAAkB,EAAE,CAAC;QAC3E,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,UAAU;YAChB,MAAM,EACJ,aAAa,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,kBAAkB,gCAAgC;gBAC3G,QAAQ,eAAe,yBAAyB,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB;SACjI,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,mBAAmB,KAAK,IAAI,EAAE,CAAC;QAClE,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,qBAAqB;YAC3B,MAAM,EACJ,uHAAuH;gBACvH,0GAA0G;gBAC1G,iEAAiE;SACpE,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,wEAAwE;QACxE,qEAAqE;QACrE,uEAAuE;QACvE,iEAAiE;QACjE,MAAM,gBAAgB,GACpB,OAAO,CAAC,mBAAmB,KAAK,IAAI;YACpC,QAAQ,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,qBAAqB,CAAC;QAC9C,OAAO;YACL,OAAO,EAAE,WAAW;YACpB,QAAQ;YACR,aAAa;YACb,MAAM;YACN,OAAO,EAAE,GAAG,gBAAgB,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,kBAAkB;SACxH,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,SAAS,GACb,MAAM,CAAC,MAAM,KAAK,IAAI;QACpB,CAAC,CAAC,kCAAkC;QACpC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;IACnD,OAAO;QACL,OAAO,EAAE,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY;QACxD,QAAQ;QACR,aAAa;QACb,MAAM;QACN,OAAO,EACL,GAAG,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,SAAS,eAAe,SAAS;YACnF,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,SAAS,iCAAiC,SAAS,GAAG;KAC3F,CAAC;AACJ,CAAC;AAED,oFAAoF;AACpF,SAAS,MAAM,CACb,OAAgB,EAChB,QAAgC,EAChC,aAAoD;IAEpD,OAAO;QACL,OAAO;QACP,QAAQ;QACR,aAAa;QACb,MAAM,EAAE,IAAI;QACZ,OAAO,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAE,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,kBAAkB;KAC5F,CAAC;AACJ,CAAC;AAED,MAAM,QAAQ,GAAsC;IAClD,MAAM,EAAE,QAAQ;IAChB,UAAU,EAAE,YAAY;IACxB,WAAW,EAAE,WAAW;IACxB,KAAK,EAAE,OAAO;IACd,gBAAgB,EAAE,gBAAgB;IAClC,OAAO,EAAE,SAAS;CACnB,CAAC"}

package/dist/ab-cli.d.ts

@@ -0,0 +1,35 @@
+/**
+ * `skillstate ab` — drive an A/B experiment and print a gated verdict.
+ *
+ * @non-paper — measurement infrastructure, not the paper's evaluation.
+ *
+ * Importing this module is side-effect free: it runs only when this file is
+ * the process entry (`node dist/ab-cli.js`), never on library import.
+ *
+ * ── The one rule this CLI enforces ───────────────────────────────────────
+ *
+ * It prints a percentage only when every gate passed. On any refusal it
+ * prints the gates instead. This is the whole point of the tool: the previous
+ * experiment's number was not wrong by arithmetic, it was wrong because
+ * nothing asked whether the instrumented arm had done anything.
+ */
+import type { ExperimentOptions } from './ab/verdict.js';
+/** Options for {@link main}. */
+export interface AbOptions extends ExperimentOptions {
+    /** Run files to read, in any order. */
+    readonly files: readonly string[];
+    /** Mark the task as defined over the historical trajectory. */
+    readonly taskNeedsTranscript?: boolean;
+    /** Overrides the value inferred from the run files. */
+    readonly minTrials?: number;
+}
+/**
+ * Read run files, run the gates, and print the report.
+ *
+ * Returns a process exit code. A refusal exits non-zero so a CI job wired to
+ * this cannot record an invalid experiment as a passing one.
+ */
+export declare function main(options: AbOptions): number;
+/** Parse argv and run. Exported so tests can drive it without a subprocess. */
+export declare function mainFromArgv(argv: readonly string[]): number;
+//# sourceMappingURL=ab-cli.d.ts.map

package/dist/ab-cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ab-cli.d.ts","sourceRoot":"","sources":["../src/ab-cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAWH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AA0EzD,gCAAgC;AAChC,MAAM,WAAW,SAAU,SAAQ,iBAAiB;IAClD,uCAAuC;IACvC,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,+DAA+D;IAC/D,QAAQ,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IACvC,uDAAuD;IACvD,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;GAKG;AACH,wBAAgB,IAAI,CAAC,OAAO,EAAE,SAAS,GAAG,MAAM,CAkC/C;AAED,+EAA+E;AAC/E,wBAAgB,YAAY,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAkD5D"}

package/dist/ab-cli.js

@@ -0,0 +1,161 @@
+/**
+ * `skillstate ab` — drive an A/B experiment and print a gated verdict.
+ *
+ * @non-paper — measurement infrastructure, not the paper's evaluation.
+ *
+ * Importing this module is side-effect free: it runs only when this file is
+ * the process entry (`node dist/ab-cli.js`), never on library import.
+ *
+ * ── The one rule this CLI enforces ───────────────────────────────────────
+ *
+ * It prints a percentage only when every gate passed. On any refusal it
+ * prints the gates instead. This is the whole point of the tool: the previous
+ * experiment's number was not wrong by arithmetic, it was wrong because
+ * nothing asked whether the instrumented arm had done anything.
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { assessEngagement, isWitnessed, withRejections } from './ab/engagement.js';
+import { buildArmRecord } from './ab/record.js';
+import { formatArmTable, formatVerdict } from './ab/report.js';
+import { runExperiment } from './ab/verdict.js';
+function parseRunFiles(files) {
+    const parsed = [];
+    for (const file of files) {
+        const raw = JSON.parse(fs.readFileSync(file, 'utf-8'));
+        for (const entry of Array.isArray(raw) ? raw : [raw]) {
+            parsed.push(entry);
+        }
+    }
+    return parsed;
+}
+function toRunRecord(trial) {
+    const report = assessEngagement(trial.stateSamples);
+    const witnessed = isWitnessed(trial.stateSamples);
+    // An unwitnessed trial is NOT recorded as inert. Nobody watched the file,
+    // and calling that "the integration did nothing" would be an accusation
+    // the harness cannot support. It is recorded as unwitnessed, which the
+    // sample-size and engagement gates then treat as a refusal.
+    const engagement = witnessed
+        ? withRejections(report.evidence, trial.rejections ?? 0, trial.firstRejection)
+        : {
+            ...report.evidence,
+            engaged: false,
+            writes: 0,
+            rejections: 0,
+        };
+    return {
+        arm: trial.arm,
+        trial: trial.trial,
+        task: trial.task,
+        model: trial.model,
+        hostVersion: trial.hostVersion,
+        sessionID: trial.sessionID,
+        usage: trial.usage,
+        engagement,
+        work: {
+            artifactDigest: trial.artifactDigest,
+            turns: trial.turns,
+            toolCalls: trial.toolCalls,
+        },
+        durationMs: trial.durationMs,
+        completed: trial.completed,
+        ...(trial.correct === undefined ? {} : { outcome: { correct: trial.correct } }),
+        ...(trial.error === undefined ? {} : { error: trial.error }),
+    };
+}
+/**
+ * Read run files, run the gates, and print the report.
+ *
+ * Returns a process exit code. A refusal exits non-zero so a CI job wired to
+ * this cannot record an invalid experiment as a passing one.
+ */
+export function main(options) {
+    const trials = parseRunFiles(options.files);
+    const byArm = new Map();
+    for (const trial of trials) {
+        const runs = byArm.get(trial.arm) ?? [];
+        runs.push(toRunRecord(trial));
+        byArm.set(trial.arm, runs);
+    }
+    const arms = new Map();
+    const problems = [];
+    for (const [armId, runs] of byArm) {
+        const built = buildArmRecord(armId, runs);
+        if (!built.ok) {
+            problems.push(built.reason);
+            continue;
+        }
+        arms.set(armId, built.record);
+    }
+    const result = runExperiment(arms, {
+        ...(options.taskNeedsTranscript === undefined
+            ? {}
+            : { taskNeedsTranscript: options.taskNeedsTranscript }),
+        ...(options.minTrials === undefined ? {} : { minTrials: options.minTrials }),
+    });
+    console.log(formatVerdict(result, formatArmTable(arms)));
+    for (const problem of problems) {
+        console.log(`  - [input] ${problem}`);
+    }
+    const measured = result.verdict === 'saving' || result.verdict === 'regression';
+    return measured && problems.length === 0 ? 0 : 1;
+}
+/** Parse argv and run. Exported so tests can drive it without a subprocess. */
+export function mainFromArgv(argv) {
+    const files = [];
+    let taskNeedsTranscript = false;
+    let minTrials;
+    for (let i = 0; i < argv.length; i += 1) {
+        const arg = argv[i];
+        if (arg === '--transcript-task') {
+            taskNeedsTranscript = true;
+        }
+        else if (arg === '--min-trials') {
+            const value = Number(argv[++i]);
+            if (!Number.isInteger(value) || value < 1) {
+                console.log('--min-trials needs a positive integer');
+                return 2;
+            }
+            minTrials = value;
+        }
+        else if (arg === '--help' || arg === '-h') {
+            console.log('usage: skillstate ab [--transcript-task] [--min-trials N] <run.json>...\n\n' +
+                'Each run file is a JSON object (or array of them) describing one trial:\n' +
+                '  arm, trial, task, model, hostVersion, sessionID, usage{input,cacheRead,\n' +
+                '  cacheWrite,output}, turns, toolCalls, artifactDigest, durationMs,\n' +
+                '  completed, and stateSamples[{trial,step,content,fromSink?}].\n\n' +
+                'Exits 0 only when every gate passed and a saving or regression was found.');
+            return 0;
+        }
+        else if (arg.startsWith('-')) {
+            console.log(`unknown flag: ${arg}`);
+            return 2;
+        }
+        else {
+            files.push(arg);
+        }
+    }
+    if (files.length === 0) {
+        console.log('no run files given — nothing to compare');
+        return 2;
+    }
+    for (const file of files) {
+        if (!fs.existsSync(path.resolve(file))) {
+            console.log(`no such run file: ${file}`);
+            return 2;
+        }
+    }
+    return main({
+        files,
+        ...(taskNeedsTranscript ? { taskNeedsTranscript: true } : {}),
+        ...(minTrials === undefined ? {} : { minTrials }),
+    });
+}
+const isEntry = process.argv[1] !== undefined &&
+    pathToFileURL(process.argv[1]).href === import.meta.url;
+if (isEntry) {
+    process.exitCode = mainFromArgv(process.argv.slice(2));
+}
+//# sourceMappingURL=ab-cli.js.map