@sema-ai/core 1.275.1

package/dist/bench/metrics.d.ts

@@ -0,0 +1,714 @@
+/**
+ * design/89 §2 — VALUE-JUDGMENT METRICS (the cost-per-truly-correct measurement tool).
+ *
+ * Pure, deterministic measurement logic for the value-judgment benchmark (design/89, milestone-gap
+ * ❓-未判项: "supervisor / multi-agent vs solo 到底值不值?"). The service three-arm harness feeds REAL
+ * runs in; this module computes truly-correct / three-class outcomes / cost four-components / Pareto.
+ *
+ * IRON LAWS (design/89 §2, blood-bought red lines — DO NOT relax):
+ *   1. The numerator counts ONLY objective oracles (exec-gate exit-code, build, property-harness, and
+ *      actual-delivery). LLM-judge is NEVER counted into truly-correct (it is a runtime GATE, not a
+ *      measurement; §2.2). It burns cost (C1/C2) but never the numerator.
+ *   2. NEVER synthesize a single scalar. Cost is FOUR separate components (§2.3); C1 (token µUSD) and
+ *      C2 (human seconds) are apples + oranges (§2.1) and are NEVER added. The verdict is a 3-D Pareto
+ *      frontier over (C1, C2.sec, P). A µUSD/tc ratio may be reported ONLY as a per-axis diagnostic.
+ *   3. THREE outcome classes (§2.2.1): DELIVERED-CORRECT / DELIVERED-WRONG / CORRECTLY-WITHHELD. A
+ *      correctly-withheld task (human/gate stopped a would-be DELIVERED-WRONG) is the supervisor's
+ *      avoided-loss value — it is NOT a failure and NOT in the numerator.
+ *   4. Deterministic + testable: this file has zero model/network dependency. Synthetic-run fixtures
+ *      validate the measurement logic without a real model (see metrics.test.ts).
+ *
+ * This is a SHADOW-LAB experiment tool (not the core engine). Run via `npx tsx`.
+ */
+/**
+ * The orchestration arms.
+ *   - "solo"   — no supervision profile (the cost floor / quality-risk baseline).
+ *   - "sup"    — SUP-per-step (S1a): the scripted human gate fires UNCONDITIONALLY at every plan_review +
+ *                risky tool ask (C2 paid every time). S1a proved this is `dominated` on coding tasks (same P
+ *                as solo, but pays C2 for nothing the model couldn't get right alone).
+ *   - "sup-vf" — SUP verifier-first (S1b): the SAME supervision profile, but the scripted human gate is only
+ *                ESCALATED (C2 paid) when ① a deterministic verifier (the mechanical hidden oracle / RepairOracle
+ *                / shellGate hazard list) is RED or ② the action hits the danger list (destructive-fs / secret /
+ *                pipe-to-shell). Otherwise the gate AUTO-ALLOWS with C2=0 (steals reflection/actor-critic + CRITIC:
+ *                an external deterministic verifier is the critic; the human is the escalation tier only). Hypothesis:
+ *                on coding tasks where the verifier goes green, SUP-vf collapses C2→~0 and stops being dominated by
+ *                solo, WHILE keeping the trap-safety CORRECTLY-WITHHELD (the danger list still escalates `rm -rf data`).
+ *   - "team"   — service-deferred (runLeaderTask).
+ */
+export type Arm = "solo" | "sup" | "sup-vf" | "team";
+/**
+ * The OBJECTIVE oracle verdicts for one run (design/89 §2.2). These are the four AND criteria.
+ * Every field is produced by an external, deterministic oracle run AFTER the agent finishes — never
+ * by an LLM-judge. Each is computed by the service harness (it owns exec-gate / build / git) and
+ * recorded verbatim; this module only ANDs them.
+ */
+export interface OracleVerdicts {
+    /** §2.2 criterion ① — hidden objective test suite all green: `runExecGate` exit==0 over a NON-EMPTY
+     *  suite. Encode "empty suite / null exit" as `false` (exec-gate never passes those). */
+    hiddenTestsGreen: boolean;
+    /** §2.2 criterion ② — build/compile passed (APK assembleOfflineDebug / tsc / gradle build). Guards
+     *  the "tests green but compile broke" false-positive. */
+    buildPassed: boolean;
+    /** §2.2 criterion ③ — mechanical invariants un-violated (property-harness). `true` when no violation
+     *  OR property-harness is not applicable to this task (the AND must not penalize an N/A task). The
+     *  harness sets this to `propertyHarnessApplicable ? noViolations : true`. */
+    invariantsOk: boolean;
+    /** §2.2 criterion ④ — ACTUALLY DELIVERED: a git commit exists AND the merge does not roll back.
+     *  "ran but didn't deliver" (run 019ec5d1 触顶蒸发: 104 lines all lost) MUST count 0, else solo is
+     *  systematically over-estimated. */
+    delivered: boolean;
+}
+/**
+ * Whether the human/gate WITHHELD this run's delivery (design/89 §2.2.1). When `true`, the agent
+ * (or a human at an approval gate) correctly DID NOT deliver — the §2.2.1 supervisor-fairness fix:
+ * a withheld run that WOULD have been DELIVERED-WRONG is avoided-loss value, not a failure, and is
+ * single-listed (not in the numerator, not a failure). Only the SUP/TEAM arms can withhold; for SOLO
+ * this is always `false` (no human/gate).
+ */
+export interface DeliveryDecision {
+    /** The human/gate stopped/aborted/refused-to-deliver this run. */
+    withheld: boolean;
+}
+/**
+ * design/95 §3.2 — the lifecycle status of a run, so INFRA noise never poisons the mode signal
+ * (codex BLOCKER B4 / design/89 §3.3 Beatsep red line). The Beatsep task#2b 9% pass-rate was infra
+ * death (OOM / K8S passthrough / nested-root), NOT a mode signal — feeding such runs into the
+ * numerator turns them into spurious DELIVERED-WRONG / zero datapoints and reproduces the very noise
+ * the design claims to have isolated. `buildReport` SCORES only `scored`; `infra-failed` / `excluded`
+ * runs are dropped from every cell/Pareto computation and surfaced as an exclusion ledger.
+ *
+ *   - "scored"        — a clean run whose oracle verdict is a real mode signal. The ONLY status scored.
+ *   - "infra-failed"  — OOM / sandbox / passthrough / nested-root death; the agent never got a fair
+ *                       attempt. Excluded; counted in the exclusion ledger with reason "infra-failed".
+ *   - "excluded"      — manually quarantined (e.g. corpusHash drift, approver-policy change mid-batch,
+ *                       a known-bad fixture). Excluded; counted with reason "excluded".
+ *
+ * ABSENT ⇒ treated as "scored" (back-compat: older RunRecord arrays predate this field).
+ */
+export type RunStatus = "scored" | "infra-failed" | "excluded";
+/** A single run fed in by the service three-arm harness. */
+export interface RunRecord {
+    arm: Arm;
+    /** Task id (for paired-seed grouping / cell folding). */
+    taskId: string;
+    /** Repeat/seed index within the cell (design/89 §3.4 N-repeat). */
+    seed?: number | string;
+    /**
+     * design/95 §3.2 / codex B4 — lifecycle status. Only "scored" runs reach the numerator / cost /
+     * Pareto. Absent ⇒ "scored" (back-compat). See {@link RunStatus}.
+     */
+    runStatus?: RunStatus;
+    /**
+     * design/95 §2 (B1 / Simpson's-paradox guard) — the heterogeneity coordinates a run belongs to. The
+     * suite is heterogeneous (different archetypes / value-dimensions / difficulties); folding everything
+     * into an arm mean can flip a per-cell conclusion (Simpson's paradox), making J1-J5 / JV1-JV4
+     * UN-evaluable. `buildReport` groups by `(suiteVersion, taskId, archetype, valueDimension, arm)` and
+     * keeps the seed-paired rows; the suite-level rollup is a WEIGHTED summary of scored cells only and is
+     * NOT a decision surface. All fields optional for back-compat (a run with none falls into a single
+     * "ungrouped" cell, preserving the old single-arm-mean behavior for legacy fixtures).
+     */
+    suiteVersion?: string;
+    archetype?: string;
+    valueDimension?: string;
+    /** The four objective oracle verdicts (§2.2). */
+    oracle: OracleVerdicts;
+    /** Whether delivery was correctly withheld (§2.2.1). */
+    delivery: DeliveryDecision;
+    /**
+     * The engine-produced `TaskResult.stats` (or the relevant subset). C1 is derived from
+     * `stats.costBreakdown` (design/80 D-E-core); C2 from `stats.humanReview` (design/91, 1.110.0 ship).
+     * Shape mirrors @sema-ai/core `TaskResult["stats"]` — see {@link CoreStatsSubset}.
+     */
+    stats: CoreStatsSubset;
+    /** Wall-clock seconds for the whole run (task start→done). The C3 source. */
+    wallTimeSec?: number;
+    /** Infra cost in µUSD (E2B vCPU-hr / egress / k8s) — owned & filled by the SERVICE (core never has
+     *  this). The C4 source. Absent for solo-local / when the service did not measure it. */
+    infraMicroUsd?: number;
+    /**
+     * §2.4#2 intercept-value counterfactual: for each human intervention on a SUP run, the harness
+     * re-runs the same brief + fixed seed with that intervention DISABLED and records whether the final
+     * truly-correct flipped. Absent for SOLO (no intervention) and when the counterfactual was not run.
+     *
+     * CRITICAL (design/89 §2.2.1 line 96): for a WITHHELD run this is REQUIRED to decide whether the
+     * withhold was CORRECT. A withhold is avoided-loss ONLY if disabling the intervention would have
+     * produced a DELIVERED-WRONG (`withoutInterventionTrulyCorrect===false`). When this is absent on a
+     * withheld run, the withhold is UNVERIFIED — it is NOT credited as avoided-loss.
+     */
+    counterfactual?: InterceptCounterfactual;
+    /**
+     * R2 (MINOR) — the HONESTY provenance of this run's judgment, stamped at the RECORD level (not only in the
+     * report/stamp layer). A SUP/TEAM run's value rides on a SCRIPTED approver (NOT a real human review), so it is
+     * an INITIAL judgment, never a firm verdict (design/95 §6.1.bis / §6.1). Carrying it on the record means a
+     * record-level filter / re-aggregation / persisted run cannot silently lose the "scripted-human, not firm"
+     * semantics (before this, filtering RunRecords dropped the label that lived only on `BatchStamp`). SOLO has no
+     * human/approver ⇒ undefined (a pure objective-oracle run — no scripted-human judgment to caveat).
+     */
+    judgeMode?: "scripted-human-initjudge";
+}
+/**
+ * The subset of @sema-ai/core `TaskResult["stats"]` this tool reads. Kept structurally
+ * compatible (all optional, same field names/shapes) so a real `TaskResult.stats` is assignable here
+ * with no adaptation — verified against src/core/types.ts (1.110.0).
+ */
+export interface CoreStatsSubset {
+    costMicroUsd?: number;
+    /** design/80 D-E-core finance taxonomy. C1 is the SUM of its LLM-derived lines. */
+    costBreakdown?: {
+        llmRootMicroUsd: number;
+        nestedSubagentMicroUsd: number;
+        memoryConsolidationMicroUsd: number;
+        compactionMicroUsd: number;
+    };
+    nested?: {
+        costMicroUsd?: number;
+    };
+    /** design/91 human-review burden (the C2 source). Absent ⇒ no approval gate was hit. */
+    humanReview?: {
+        count: number;
+        totalWaitMs: number;
+        gates: Array<{
+            kind: string;
+            waitMs: number;
+            decision?: string;
+        }>;
+    };
+}
+/**
+ * §2.2 — `truly-correct(run)` = AND of the four objective criteria (any false ⇒ 0). The numerator of
+ * P. By construction this takes ONLY {@link OracleVerdicts} (objective oracles); there is no parameter
+ * by which an LLM-judge could enter — that is the type-level enforcement of iron law #1.
+ */
+export declare function trulyCorrect(o: OracleVerdicts): boolean;
+export type Outcome = "DELIVERED-CORRECT" | "DELIVERED-WRONG" | "CORRECTLY-WITHHELD" | "INCORRECTLY-WITHHELD" | "UNVERIFIED-WITHHELD";
+/**
+ * §2.2.1 — classify a run into one of the outcome classes. The withheld bucket is SPLIT by the §2.1
+ * intercept-value counterfactual (design/89 §2.2.1 line 96 — a withhold is only avoided-loss if it
+ * stopped a would-be DELIVERED-WRONG):
+ *   - DELIVERED-CORRECT: not withheld AND truly-correct → the numerator.
+ *   - DELIVERED-WRONG: not withheld AND not truly-correct → the true negative (solo hard-committing a
+ *     buggy artifact, OR 触顶蒸发 with delivered=false). §2.2 criterion ④ folds "ran-but-didn't-
+ *     deliver" into DELIVERED-WRONG, not into a withheld bucket — withheld requires an EXPLICIT
+ *     human/gate decision, vaporized-on-overflow is a failure.
+ *   - CORRECTLY-WITHHELD: withheld AND the counterfactual proves the task WOULD have been DELIVERED-
+ *     WRONG (`withoutInterventionTrulyCorrect===false`). The supervisor's avoided-loss value — NOT in
+ *     the numerator, NOT a failure.
+ *   - INCORRECTLY-WITHHELD: withheld BUT the counterfactual shows the task would have been DELIVERED-
+ *     CORRECT (`withoutInterventionTrulyCorrect===true`). A value-DESTROYING over-cautious intervention
+ *     (a false-positive withhold). This is a COST/mistake, NEVER laundered into avoided-loss (the
+ *     focus-item-② failure mode the review caught).
+ *   - UNVERIFIED-WITHHELD: withheld but NO counterfactual was recorded → correctness unknown. Excluded
+ *     from the avoided-loss credit (NOT defaulted to CORRECTLY-WITHHELD) and flagged (design/89 §2.2.1
+ *     line 96: withheld MUST be counterfactual-verified).
+ */
+export declare function classifyOutcome(run: {
+    oracle: OracleVerdicts;
+    delivery: DeliveryDecision;
+    counterfactual?: InterceptCounterfactual;
+}): Outcome;
+/**
+ * §2.3 — the four cost components for one run. EACH single-listed; C1 and C2 are different units
+ * (µUSD vs seconds) and are NEVER added (that is the §2.1 apples+oranges trap). There is deliberately
+ * NO field that combines them.
+ */
+export interface CostComponents {
+    /** C1 — model (LLM-token) cost in µUSD. Derived from `stats.costBreakdown` (design/80 D-E-core):
+     *  the sum of the LLM-derived lines. */
+    c1ModelMicroUsd: number;
+    /** C2 — human-review burden: (count, seconds). From `stats.humanReview` (design/91). Seconds, not
+     *  µUSD — never folded into C1. */
+    c2HumanCount: number;
+    c2HumanSec: number;
+    /** C3 — wall-clock seconds (the speed axis; fan-out's true value場之一). */
+    c3WallSec: number;
+    /** C4 — infra µUSD (E2B vCPU-hr / egress / k8s), service-owned. 0 when not measured. */
+    c4InfraMicroUsd: number;
+}
+/**
+ * §2.3 — derive C1 from `stats.costBreakdown` (design/80 D-E-core). C1 is the LLM-token-derived total:
+ * the sum of the four breakdown lines. Falls back to `costMicroUsd` (+ nested) when `costBreakdown`
+ * is absent (older runs), then 0. This is the ONLY composition done — and it is WITHIN C1 (all µUSD,
+ * same unit); it never reaches across to C2.
+ */
+export declare function deriveC1(stats: CoreStatsSubset): number;
+/** §2.4 — derive C2 (count, seconds) from `stats.humanReview` (design/91). Absent ⇒ no gate ⇒ (0,0). */
+export declare function deriveC2(stats: CoreStatsSubset): {
+    count: number;
+    sec: number;
+};
+/** §2.3 — assemble all four cost components for a run. They stay separate by construction. */
+export declare function costComponents(run: RunRecord): CostComponents;
+/**
+ * §2.4#2 — the result of the intercept-value counterfactual for ONE human intervention: re-run the
+ * same brief + fixed seed with that intervention DISABLED and observe how the final OUTCOME CLASS
+ * changed. The intervention has intercept value when the class IMPROVED — including the avoided-loss
+ * case (CORRECTLY-WITHHELD vs would-be DELIVERED-WRONG), which truly-correct alone CANNOT see
+ * (a withheld run's `withInterventionTrulyCorrect` is trivially false — nothing was delivered).
+ * This is what gives J3's "human-review is quality not only cost" a measurement.
+ */
+export interface InterceptCounterfactual {
+    /** truly-correct WITH the intervention (the actual SUP run). For a WITHHELD run this is trivially
+     *  false (nothing delivered) — do NOT infer intercept value from it; use the class comparison. */
+    withInterventionTrulyCorrect: boolean;
+    /** truly-correct WITHOUT it (the re-run with the intervention disabled). For a withhold this is the
+     *  load-bearing field: `false` ⇒ the withhold prevented a DELIVERED-WRONG (avoided-loss). */
+    withoutInterventionTrulyCorrect: boolean;
+    /** Optional explicit override. When set, {@link interceptHadValue} uses it verbatim. Prefer leaving
+     *  it unset so the predicate derives value from the outcome classes (which handles withholds). */
+    changedOutcome?: boolean;
+}
+/**
+ * §2.4#2 — did a human intervention have intercept VALUE? Outcome-CLASS-aware (NOT truly-correct-binary
+ * -aware), so it correctly credits the supervisor's #1 value — an avoided-loss withhold — which a
+ * truly-correct XOR cannot see (design/89 §2.2.1: a withheld run's truly-correct is always false).
+ *
+ * Value exists when disabling the intervention would have WORSENED the outcome class:
+ *   - DELIVER-correcting intervention: WITH delivers correct, WITHOUT delivers wrong
+ *     (`withInterventionTrulyCorrect===true && withoutInterventionTrulyCorrect===false`).
+ *   - AVOIDED-LOSS withhold: the run was correctly withheld (CORRECTLY-WITHHELD) and WITHOUT the
+ *     intervention it would have been DELIVERED-WRONG (`withoutInterventionTrulyCorrect===false`).
+ *
+ * An INCORRECTLY-WITHHELD intervention (would-be correct, killed by an over-cautious human) has
+ * NEGATIVE value and returns `false` here — it is never counted as intercept value.
+ *
+ * Pass the run's outcome class so the predicate knows whether this was a withhold; the explicit
+ * `changedOutcome` override still wins when the harness sets it.
+ */
+export declare function interceptHadValue(cf: InterceptCounterfactual, outcome: Outcome): boolean;
+/**
+ * §2.4 — the intercept-value seam the service three-arm harness implements: given a SUP run, clone the
+ * brief + fix the seed, disable ONE human intervention, re-run, and return its truly-correct. This is
+ * a SEAM (not implemented here — it needs the real Runner/model). The metrics tool consumes its result
+ * via `RunRecord.counterfactual`; this type pins the contract so service implements it correctly.
+ */
+export type InterceptValueCounterfactualSeam = (args: {
+    /** The original SUP run whose intervention is being ablated. */
+    run: RunRecord;
+    /** Which intervention to disable (index into `stats.humanReview.gates`). */
+    interventionIndex: number;
+}) => Promise<InterceptCounterfactual>;
+/**
+ * §2.3 output-form — one arm's per-cell vector: `(C1, C2.sec, C3, C4, P=truly-correct rate, intercept
+ * value, CORRECTLY-WITHHELD count)` + the raw outcome tallies. Reported WITH n (and a CI when n large
+ * enough). NO single µUSD/tc headline (iron law #2).
+ */
+export interface ArmCell {
+    arm: Arm;
+    n: number;
+    /** Outcome tallies. The withheld bucket is SPLIT (§2.2.1 line 96): only counterfactual-verified
+     *  avoided-loss is `correctlyWithheld`; an over-cautious would-be-correct kill is
+     *  `incorrectlyWithheld` (a cost); a withhold with no counterfactual is `unverifiedWithheld`. */
+    deliveredCorrect: number;
+    deliveredWrong: number;
+    correctlyWithheld: number;
+    incorrectlyWithheld: number;
+    unverifiedWithheld: number;
+    /**
+     * P = truly-correct RATE. design/89 §2.2.1: CORRECTLY-WITHHELD is neither numerator nor failure, so
+     * it is EXCLUDED from the denominator (it is not an attempt that produced a delivered artifact). P =
+     * DELIVERED-CORRECT / (DELIVERED-CORRECT + DELIVERED-WRONG). When all runs were withheld the rate is
+     * undefined (no delivery attempt to score).
+     */
+    trulyCorrectRate: number | undefined;
+    /** Avoided-loss rate = CORRECTLY-WITHHELD / n. The supervisor's §2.2.1 value, single-listed. */
+    correctlyWithheldRate: number;
+    /**
+     * codex MINOR / design/95 M10 — withhold treated as a binary detector of would-be-wrong delivery
+     * (positive = withheld; ground truth = would-be-wrong, established by the §2.4 counterfactual). Reported
+     * as precision/recall — NOT just the raw correctly-withheld count — so an arm cannot look good by
+     * withholding indiscriminately (high count, low precision) or by rarely withholding (high precision,
+     * low recall). Both undefined when their denominator is 0.
+     *   withholdPrecision = correctlyWithheld / (correctlyWithheld + incorrectlyWithheld)
+     *     — of the VERIFIED withhold decisions, the fraction that avoided a real wrong delivery. UNVERIFIED
+     *       withholds are EXCLUDED (no counterfactual ⇒ TP/FP unknown; the honest, anti-over-claim choice).
+     *   withholdRecall = correctlyWithheld / (correctlyWithheld + deliveredWrong)
+     *     — of the wrong-delivery opportunities, the fraction the withhold caught (deliveredWrong = the FN:
+     *       a wrong artifact was delivered that a withhold would have avoided).
+     */
+    withholdPrecision: number | undefined;
+    withholdRecall: number | undefined;
+    /** Mean C1 (µUSD), C2 (count, sec), C3 (sec), C4 (µUSD) across the cell. Each single-listed. */
+    c1ModelMicroUsdMean: number;
+    c2HumanCountMean: number;
+    c2HumanSecMean: number;
+    c3WallSecMean: number;
+    c4InfraMicroUsdMean: number;
+    /** §2.4 intercept value: how many interventions changed the outcome / how many were measured. */
+    interceptValueChanged: number;
+    interceptValueMeasured: number;
+    /**
+     * Per-axis DIAGNOSTIC only (iron law #2): mean-C1 µUSD per DELIVERED-CORRECT. Undefined when 0
+     * delivered-correct. NEVER combines C2 — it is C1-only, labeled a diagnostic, and never the verdict.
+     */
+    c1PerTrulyCorrectDiag: number | undefined;
+}
+/** Fold a cell's runs (all same arm) into the per-arm vector. */
+export declare function foldCell(runs: RunRecord[]): ArmCell;
+/**
+ * §2.1 — the three Pareto axes for one arm. Lower is better for cost (C1, C2.sec); higher is better
+ * for correctness (P). NOTE: an arm with undefined P (all-withheld) cannot be placed on the frontier
+ * — it is excluded from domination judgment and flagged.
+ */
+export interface ParetoPoint {
+    arm: Arm;
+    c1ModelMicroUsd: number;
+    c2HumanSec: number;
+    trulyCorrectRate: number;
+}
+/**
+ * §2.1 — does point `a` Pareto-DOMINATE point `b`? `a` dominates `b` iff `a` is no worse on ALL three
+ * axes (≤ C1, ≤ C2.sec, ≥ P) AND strictly better on at least one. Domination means b is provably
+ * inferior. NOT dominated ⇒ both are on the frontier (a real trade-off — exactly the §2.1 outcome we
+ * refuse to collapse into one scalar).
+ */
+export declare function dominates(a: ParetoPoint, b: ParetoPoint): boolean;
+export interface ParetoResult {
+    /** The input points that could be placed (P defined). */
+    points: ParetoPoint[];
+    /** Arms on the Pareto frontier (NOT dominated by any other) — the real trade-offs. */
+    frontier: Arm[];
+    /** Arms dominated by ≥1 other arm, each with WHO dominates it. A dominated arm is provably inferior
+     *  (the only kind of verdict §2.1 permits — no scalar ranking of frontier members). */
+    dominated: Array<{
+        arm: Arm;
+        dominatedBy: Arm[];
+    }>;
+    /**
+     * Arms excluded from the frontier because P was undefined (all-withheld) — flagged, not silently
+     * dropped, and CARRYING their avoided-loss so a consumer that reads `frontier` cannot miss the
+     * withheld arm's value (design/89 §2.2.1: an all-withheld supervisor must not vanish, leaving a
+     * wrong-shipping solo as the sole "winner").
+     */
+    excludedUndefinedP: Array<{
+        arm: Arm;
+        correctlyWithheldRate: number;
+        correctlyWithheld: number;
+        deliveredWrong: number;
+    }>;
+    /**
+     * HARD COUNTERWEIGHT (design/89 §2.1/§2.2.1): set when the frontier is NOT a clean winner declaration
+     * — an arm with avoided-loss (correctlyWithheld>0) was excluded for undefined P WHILE a frontier arm
+     * ships real negatives (deliveredWrong>0). A downstream consumer keying on `frontier` MUST read this
+     * before concluding "X won": the excluded arm avoided loss the frontier arm did not.
+     */
+    frontierNotAWinnerDeclaration?: {
+        reason: string;
+        excludedAvoidedLossArms: Arm[];
+        frontierArmsShippingWrong: Arm[];
+    };
+}
+/**
+ * §2.1 — compute the 3-D Pareto frontier over the per-arm cells. Each arm contributes ONE point
+ * `(C1 mean, C2.sec mean, P)`. Arms with undefined P (all-withheld) are flagged and excluded from
+ * domination. The output is the frontier + dominated-by map — NEVER a single ranked scalar.
+ */
+export declare function paretoFrontier(cells: ArmCell[]): ParetoResult;
+/**
+ * §2.1 — the risk-transfer disclosure: solo having no C2 does NOT mean solo is cheaper — it transfers
+ * quality risk downstream (single-agent 3-runs-all-failed 0-commit = the cost of no human safety net;
+ * METR -19% / DORA more-code≠more-delivered / 31% PR zero-review). The report MUST emit this verbatim
+ * whenever a solo arm is present so a low solo C2 is never read as "solo cheaper". Returns the text +
+ * the supporting numbers from this run so it is not boilerplate but grounded.
+ */
+export declare function riskTransferDisclosure(cells: ArmCell[]): {
+    applies: boolean;
+    text: string;
+    evidence: {
+        soloC2Sec: number;
+        soloDeliveredWrong: number;
+        supCorrectlyWithheld: number;
+    };
+};
+/**
+ * codex B2 — a paired-binary comparison of two arms on the SAME tasks/seeds. The two arms' truly-
+ * correct flags are paired row-by-row (paired-seed, design/89 §3.4). For binary paired data the
+ * RIGHT tests are McNemar (discordant pairs) and a paired-bootstrap difference interval — NOT two
+ * independent proportions with "CI non-overlap" (that ignores the pairing and is under-powered, codex B2).
+ */
+export interface PairedBinaryComparison {
+    /** Arm A (e.g. "sup") vs arm B (e.g. "solo"). pDiff = P(A) − P(B). */
+    armA: Arm;
+    armB: Arm;
+    /** Number of PAIRED rows actually compared (only rows where both arms produced a delivery attempt). */
+    nPairs: number;
+    /** Discordant pair counts: b = A-correct & B-wrong; c = A-wrong & B-correct (McNemar's b,c). */
+    bAonly: number;
+    cBonly: number;
+    /** Point estimate of the paired proportion difference P(A) − P(B). */
+    pDiff: number;
+    /** McNemar exact-ish two-sided p-value over discordant pairs (binomial, continuity-corrected χ² when
+     *  b+c large). */
+    mcnemarP: number;
+    /** Bootstrap percentile 95% CI for the paired difference (lo, hi) — a percentile CI on the seeded
+     *  paired-bootstrap distribution, NOT Newcombe's analytic interval (codex MAJOR-B). Crosses 0 ⇒ not significant. */
+    ci95: [number, number];
+    /** TRUE iff the difference is statistically significant at α=0.05 (CI excludes 0 AND McNemar p<0.05). */
+    significant: boolean;
+    /**
+     * codex B2 — pre-registered Minimum Detectable Effect at the observed nPairs (the difference this
+     * comparison COULD have detected at 80% power). When |pDiff| is below this AND not significant, the
+     * verdict is "not powered" — NOT "no difference". This is the field that stops "N≥15 is an assertion".
+     */
+    mdeAt80Power: number;
+    /** "powered" iff nPairs ≥ the n needed to detect the pre-registered MDE; else "not-powered". */
+    power: "powered" | "not-powered";
+    /** Human-readable verdict that NEVER over-claims a null result from an under-powered comparison. */
+    verdict: "A-better" | "B-better" | "no-detectable-difference" | "not-powered";
+}
+/**
+ * codex B2 — compare two arms' delivered truly-correct as PAIRED binary. Pairs runs by the COMPOSITE
+ * `(taskId, seed)` key (codex MAJOR-A: a bare `seed` collides across tasks — the same repeat index
+ * recurs per task — so cross-task input would overwrite pairs and poison the McNemar sample);
+ * only rows where BOTH arms delivered (not withheld, both scored) form a pair — a
+ * withheld run has no delivered binary to pair (it is scored in the withhold/avoided-loss axis, not
+ * here). `mde` is the pre-registered minimum detectable effect (default 0.20 absolute = a 20pp swing).
+ */
+export declare function pairedBinaryCompare(scoredRuns: RunRecord[], armA: Arm, armB: Arm, opts?: {
+    mde?: number;
+    bootstrapSeed?: number;
+    bootstrapIters?: number;
+}): PairedBinaryComparison;
+/**
+ * codex B3 — the pre-registered C2 exchange rate(s): how many human-review SECONDS we are willing to
+ * pay to buy one unit of supervisor value. Without these, "higher P / more withholds ⇒ worth it" is
+ * unfalsifiable (the "helpful but too expensive" counter-thesis cannot be observed). Declared BEFORE
+ * the run (design/95 §9), not fit after.
+ */
+export interface C2Thresholds {
+    /** Max human-review seconds we will pay to avoid ONE DELIVERED-WRONG (per CORRECTLY-WITHHELD). */
+    secPerAvoidedWrong: number;
+    /** Max human-review seconds we will pay to save ONE repeated decision (V1). */
+    secPerSavedDecision: number;
+}
+export interface ParetoVerdict {
+    arm: Arm;
+    /** Non-dominated on (C1, C2.sec, P) — necessary condition to be considered at all (codex B3). */
+    nonDominated: boolean;
+    /** The pre-registered exchange-rate check: does the arm's extra C2 buy enough avoided-loss / saved
+     *  decisions to clear the declared threshold? Undefined when the arm has no extra C2 over baseline. */
+    clearsC2Threshold?: boolean;
+    /**
+     * codex B3 — the label. An arm may ONLY be "worth-it" when it is non-dominated AND clears the C2
+     * threshold. A non-dominated but threshold-failing arm is "quality-tradeoff" (higher quality, but the
+     * cost is not bought back) — NEVER "worth-it". A dominated arm is "dominated".
+     */
+    label: "worth-it" | "quality-tradeoff" | "dominated";
+}
+/**
+ * codex B3 — classify each arm against the SUP/TEAM-vs-baseline value question with a pre-registered
+ * exchange rate. `baselineArm` is the cost floor to compare extra C2 against (default "solo").
+ * `avoidedWrong` / `savedDecisions` per arm come from the cells / campaign report.
+ *
+ * 🔴 ADVERSARIAL-REVIEW DISCIPLINE (B1 partial false-close, design/95 §2.3): `cells` MUST be a
+ * HOMOGENEOUS cross-arm set — feed `report.comparisons[*].arms` (one `(suiteVersion,taskId,archetype,
+ * valueDimension)` comparison at a time), NEVER `report.cells` (the suite-wide ARM MEAN). The arm mean
+ * folds heterogeneous tasks → the value LABEL itself flips (Simpson): a cell that is "quality-tradeoff"
+ * (extra human seconds NOT bought back) gets averaged with a "worth-it" cell into a single misleading
+ * "worth-it", hiding the "helpful but too expensive" anti-thesis. The frontier folds too (§11 / B1).
+ * Regression: metrics.test §13b proves per-cell {worth-it, quality-tradeoff} ≠ arm-mean {worth-it}.
+ */
+export declare function paretoValueVerdict(cells: ArmCell[], thresholds: C2Thresholds, perArmValue: Partial<Record<Arm, {
+    avoidedWrong: number;
+    savedDecisions: number;
+}>>, baselineArm?: Arm): {
+    pareto: ParetoResult;
+    verdicts: ParetoVerdict[];
+};
+/**
+ * design/95 §7.1 — V1 (免重复劳动): campaign-level human-decision delta. The ONLY authoritative V1
+ * mechanism (design/95 reconcile, MAJOR#2): cross-task reuse of a decided strategy. Semantics:
+ *   - `soloHumanDecisions` is the V1 BASELINE — the operator's actual up-front decision count when
+ *     running solo across the campaign (codex M7: NOT 0, NOT synthetic — measured from real operator
+ *     prep across runs). `0` is only valid when the campaign genuinely needed no human decision.
+ *   - `repeatedDecisionsSaved = soloHumanDecisions − supHumanDecisions` (may be negative = SUP cost
+ *     MORE human decisions; reported honestly, not floored).
+ */
+export interface CampaignV1Saved {
+    decisionKind: string;
+    /** V1 baseline (codex M7): operator's measured up-front decisions when running SOLO. Provenance MUST
+     *  be recorded in the stamp (real prep, not 0/synthetic). */
+    soloHumanDecisions: number;
+    /** SUP arm's actual human-review decision count for this decision kind. */
+    supHumanDecisions: number;
+    /** = soloHumanDecisions − supHumanDecisions. Positive ⇒ SUP saved repeated labour (V1 evidence). */
+    repeatedDecisionsSaved: number;
+    /** Whether the baseline is a real measurement vs absent (codex M7 honesty gate). */
+    baselineProvenance: "measured-operator-baseline" | "absent-not-claimable";
+}
+/**
+ * design/95 §7.2 — V2 (蓝图清晰度) THREE-arm ablation (codex BLOCKER#1 + MAJOR#3 / M8). All three are
+ * paired on the SAME seeds (paired===true ⇒ the three pX arrays/values are over the same n seeds).
+ * INVARIANT: `n` is the paired-seed count common to all three sub-arms; the three deltas are computed
+ * over those n pairs.
+ *   - pSoloRaw            — only the original brief (no blueprint baseline).
+ *   - pSoloSupGenerated   — the SUP-arm-GENERATED blueprint (attributable to supervisor; gen cost charged).
+ *   - pSoloExperimenterIdeal — the experimenter "ideal" blueprint. codex M8: this is a CONSTRAINED upper
+ *     bound on "better-prompt help", NOT "supervisor's mechanistic ceiling", UNLESS the ideal blueprint
+ *     was generated under the {@link IdealBlueprintConstraints} (run-time, visible-brief-only, no oracle
+ *     access, same info budget, leak-reviewed). `idealConstraintsSatisfied` records which it is.
+ */
+export interface BlueprintAblationTriple {
+    subgoalId: string;
+    paired: boolean;
+    /** Paired-seed count common to all three sub-arms (array/Δ invariant base). */
+    n: number;
+    pSoloRaw: number;
+    pSoloSupGenerated: number;
+    pSoloExperimenterIdeal: number;
+    /** = pSoloSupGenerated − pSoloRaw (supervisor blueprint's real net value). */
+    deltaSupVsRaw: number;
+    /** = pSoloExperimenterIdeal − pSoloRaw. LABELLED per `idealConstraintsSatisfied` (codex M8). */
+    deltaIdealVsRaw: number;
+    /** codex M8 — when false, deltaIdealVsRaw is "manual-prompt upper bound", NOT "supervisor potential". */
+    idealConstraintsSatisfied: boolean;
+    /** codex M3 — blueprint generation cost charged to the sup-generated arm (budget-match red line). */
+    blueprintGenC1MicroUsd: number;
+    blueprintGenC2Count: number;
+    blueprintGenC2Sec: number;
+}
+/** codex M8 — the constraints under which an "ideal" blueprint may be read as a supervisor-mechanism
+ *  upper bound rather than a generic "better prompt helps" result. Recorded per ablation. */
+export interface IdealBlueprintConstraints {
+    generatedBeforeRun: boolean;
+    fromVisibleBriefOnly: boolean;
+    noOracleTestAccess: boolean;
+    sameInfoBudget: boolean;
+    solutionLeakReviewed: boolean;
+}
+/** True iff ALL ideal-blueprint constraints hold (codex M8 gate). */
+export declare function idealConstraintsSatisfied(c: IdealBlueprintConstraints): boolean;
+/**
+ * design/95 §7.3 — V3 (无关性隔离) contamination probe (canary, mechanical, zero LLM-judge). codex M9:
+ * the isolated-vs-shared P delta CONFOUNDS execution order / context size / worker count / budget; the
+ * PRIMARY signal must be per-subgoal oracle failure + explicit wrong-use of a sibling artifact (the
+ * canary leak), with the P delta kept as a DIAGNOSTIC only.
+ * INVARIANT: `isolatedSubgoalP.length === sharedSubgoalP.length === factorCount`.
+ */
+export interface IsolationContamination {
+    factorCount: number;
+    /** Per-factor P under isolation. length === factorCount. */
+    isolatedSubgoalP: number[];
+    /** Per-factor P under shared context. length === factorCount. */
+    sharedSubgoalP: number[];
+    /** PRIMARY mechanical signal (codex M9): cross-factor canary leak rate (explicit token bleed). */
+    canaryLeakRate: number;
+    /** PRIMARY mechanical signal (codex M9): count of factors that failed their oracle AND demonstrably
+     *  used a sibling factor's artifact (the controlled-ablation harm signal). */
+    wrongSiblingArtifactUses: number;
+    /** DIAGNOSTIC ONLY (codex M9 confound): mean(isolated P) − mean(shared P). Not the primary signal. */
+    contaminationRateDiag: number;
+    /** Extra C1 the isolation cost (multi-worker / repeated context load). */
+    isolationOverheadC1: number;
+}
+/**
+ * design/95 §7 — the CAMPAIGN-level wrapper (V1 is campaign-scoped, design/95 reconcile MAJOR#2). Built
+ * ON TOP of per-task {@link ValueJudgmentReport}s; does NOT re-derive their numerator/Pareto red lines.
+ * V2/V3 attach optionally per the §7.2/§7.3 structures. All §7 fields are populated by the harness
+ * (the producer), not synthesized in `buildReport` — `implementedAxes` declares which were filled.
+ */
+export interface CampaignReport {
+    perTask: ValueJudgmentReport[];
+    campaignV1?: CampaignV1Saved;
+    blueprintAblation?: BlueprintAblationTriple;
+    contaminationProbe?: IsolationContamination;
+    implementedAxes: ImplementedAxes;
+}
+/**
+ * design/95 §2 / codex B1 — one heterogeneity CELL: a `(suiteVersion, taskId, archetype, valueDimension,
+ * arm)` group with its folded {@link ArmCell}. The decision surface is the per-cell vector — NOT the
+ * arm mean (the Simpson's-paradox guard). `groupKey` is the stable join key.
+ */
+export interface GroupedCell {
+    groupKey: string;
+    suiteVersion?: string;
+    taskId: string;
+    archetype?: string;
+    valueDimension?: string;
+    cell: ArmCell;
+}
+/**
+ * 🔴 ADVERSARIAL-REVIEW FIX (B1 partial false-close): the cross-arm decision surface. A {@link
+ * GroupedCell} keys on `arm`, so each is SINGLE-ARM — you cannot run a cross-arm Pareto / value verdict
+ * (J2/J3/B3) from one. The prior `buildReport` exposed only `report.pareto`, computed on the suite-wide
+ * `cells` ARM MEAN — the very Simpson's-prone surface B1 demotes to "NOT a decision surface". A consumer
+ * reading the obvious `report.pareto` got the folded-across-heterogeneity frontier = exactly what B1
+ * claims to prevent. This is the CORRECT decision surface: one entry per `(suiteVersion, taskId,
+ * archetype, valueDimension)` (NO arm), carrying all arms' cells + their per-cell Pareto. J2/J3/B3 MUST
+ * read these, never `report.pareto`.
+ */
+export interface ComparisonCell {
+    comparisonKey: string;
+    suiteVersion?: string;
+    taskId: string;
+    archetype?: string;
+    valueDimension?: string;
+    /** All arms' folded cells WITHIN this homogeneous comparison (same task/archetype/dimension). */
+    arms: ArmCell[];
+    /** The per-comparison Pareto over `arms` — the decision surface, NOT the suite arm mean. */
+    pareto: ParetoResult;
+}
+/**
+ * codex MINOR11 / council Q1 — capability metadata: which of the four §7/Pareto value axes are actually
+ * IMPLEMENTED in this report. Consumers must read this before claiming a V1/V2/V3 result; an unset axis
+ * is NOT a null result, it is "not measured here". (The T1 structures above are types + pure helpers;
+ * the report-level V1/V2/V3 fields are populated by the harness, not synthesized in `buildReport`.)
+ */
+export interface ImplementedAxes {
+    pareto: boolean;
+    v1: boolean;
+    v2: boolean;
+    v3: boolean;
+}
+export interface ValueJudgmentReport {
+    /** codex B1 — per-arm cells grouped by full heterogeneity key (incl. arm). Single-arm; use for cell
+     *  inspection / `pairedBinaryCompare` inputs. NOT directly a cross-arm verdict surface (each is one arm). */
+    groupedCells: GroupedCell[];
+    /**
+     * 🔴 ADVERSARIAL-REVIEW FIX (B1): the CROSS-ARM decision surface — one per `(suiteVersion, taskId,
+     * archetype, valueDimension)` (NO arm), each carrying all arms + a per-comparison Pareto over a
+     * HOMOGENEOUS set of runs. J2/J3/B3 (`paretoValueVerdict`) MUST read `comparisons[*].pareto` /
+     * `comparisons[*].arms`, NEVER the suite-wide `pareto`/`cells` (which fold heterogeneous tasks → Simpson).
+     */
+    comparisons: ComparisonCell[];
+    /**
+     * codex B1 — suite-level rollup: per-arm cells folded over ALL scored runs. A WEIGHTED summary for
+     * dashboards ONLY — NOT a decision surface (folding heterogeneous tasks can flip a verdict, Simpson).
+     * Consumers MUST decide on `comparisons`; this is convenience aggregation.
+     */
+    cells: ArmCell[];
+    /**
+     * 🔴 DASHBOARD ONLY — the Pareto over the suite-wide ARM MEAN (`cells`). This folds heterogeneous
+     * tasks into one frontier (Simpson's paradox) and is NOT a verdict surface. The verdict surface is
+     * `comparisons[*].pareto`. Kept only for a single-number dashboard glance; flagged so a consumer
+     * cannot mistake it for the decision (the B1 false-close: prior code wired the verdict to this).
+     */
+    pareto: ParetoResult;
+    riskTransfer: ReturnType<typeof riskTransferDisclosure>;
+    /** design/89 §3.4: cells with n<8 are directional-only, not verdict-grade. Flagged, not dropped. */
+    directionalOnly: Arm[];
+    /**
+     * codex B4 — the exclusion ledger: runs dropped from scoring (infra-failed / excluded) with reasons.
+     * A regression-grade invariant: these never reach `cells` / `pareto` (no infra noise in the verdict).
+     */
+    excluded: Array<{
+        taskId: string;
+        arm: Arm;
+        seed?: number | string;
+        reason: RunStatus;
+    }>;
+    /** codex MINOR11 — which value axes this report actually measured. */
+    implementedAxes: ImplementedAxes;
+}
+/**
+ * Top-level (codex B1+B4): SCORE only `scored` runs (B4 — drop infra-failed/excluded with reasons),
+ * group by `(suiteVersion, taskId, archetype, valueDimension, arm)` (B1 — the per-cell decision surface,
+ * Simpson's-paradox guard), fold each group, AND provide a suite-level per-arm rollup for dashboards
+ * (explicitly NOT a decision surface). Deterministic; no model.
+ *
+ * `opts.implementedAxes` lets the harness declare which §7 axes it populated (codex MINOR11); default
+ * = only Pareto (the V1/V2/V3 structures are T1 net-new, not synthesized here).
+ */
+export declare function buildReport(runs: RunRecord[], opts?: {
+    implementedAxes?: Partial<ImplementedAxes>;
+}): ValueJudgmentReport;
+//# sourceMappingURL=metrics.d.ts.map