@mutagent/helix 0.1.0-alpha.14

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/flow-graph.ts

@@ -0,0 +1,194 @@
+/**
+ * scripts/contracts/flow-graph.ts — W3-OWN contracts (EV-032 / EV-037).
+ * ---------------------------------------------------------------------------
+ * The information-flow graph (EV-032) + the expected-flow profile (EV-037) the
+ * W3 context-flow audit reasons over. DELIBERATELY a SEPARATE contract file from
+ * `contracts/eval-types.ts` (the shared W1 surface) and `contracts/types.ts`
+ * (the v1 auditor surface): the flow-graph is a NEW W3 capability, kept disjoint
+ * so the shared types stay frozen while this grows (mirrors the eval-types.ts
+ * rationale).
+ *
+ * Design invariants (mirror the package's pure-core style):
+ *   - Pure data shapes + categorical constants (no magic strings).
+ *   - SUBJECT-AGNOSTIC: a flow-graph names producers/consumers by the trace's
+ *     OWN observation names; the expected-flow's dispatch/UI vocabulary is
+ *     SUPPLIED (auto-gen EV-049 or authored) — never a module constant.
+ *   - No clock / random / network: a flow-graph is a pure function of a trace.
+ */
+import { type Static, Type } from "@sinclair/typebox";
+import { Value } from "@sinclair/typebox/value";
+
+// ── Edge kind (no magic strings) ─────────────────────────────────────────────
+export const FlowEdgeKind = {
+  /** A producer's output value reaches a later consumer's input (EV-028). */
+  Threaded: "threaded",
+  /** The producer is a sub-agent DISPATCH; the edge is a context handoff (EV-029). */
+  Handoff: "handoff",
+} as const;
+export type FlowEdgeKindValue = (typeof FlowEdgeKind)[keyof typeof FlowEdgeKind];
+
+/**
+ * One node in the flow-graph: a single trace observation viewed as a
+ * producer/consumer of named data slots.
+ *   - `produces` / `consumes` are the top-level slot KEYS of the observation's
+ *     output / input (deterministic structural extraction). A primitive payload
+ *     surfaces as the sentinel slot `VALUE_SLOT` / `TEXT_SLOT`.
+ *   - `isHandoff` marks a sub-agent dispatch (name ∈ injected dispatchToolNames).
+ */
+export interface FlowNode {
+  id: string;
+  index: number;
+  type: string;
+  name: string;
+  produces: string[];
+  consumes: string[];
+  isHandoff: boolean;
+}
+
+/** A data-handoff edge: producer slot `slot` at `from` reached consumer `to`. */
+export interface FlowEdge {
+  from: string;
+  to: string;
+  slot: string;
+  kind: FlowEdgeKindValue;
+}
+
+/** A producer slot whose value NEVER reached a downstream consumer (EV-028 candidate). */
+export interface UnthreadedOutput {
+  node: string;
+  name: string;
+  slot: string;
+}
+
+/** The full information-flow graph for ONE trace. PURE function of the trace. */
+export interface FlowGraph {
+  traceId: string;
+  nodes: FlowNode[];
+  edges: FlowEdge[];
+  unthreadedOutputs: UnthreadedOutput[];
+}
+
+/** Sentinel slot names for primitive (non-object) payloads. NOT subject vocab. */
+export const VALUE_SLOT = "_value";
+export const TEXT_SLOT = "_text";
+
+// ── Expected-flow profile (EV-037) ───────────────────────────────────────────
+/**
+ * One expected producer→consumer threading the subject SHOULD exhibit. Names are
+ * the subject's OWN tool/step names (supplied, never hardcoded). `slot` is
+ * optional — when present the diff also requires that slot to thread.
+ */
+export interface ExpectedFlowEdge {
+  fromTool: string;
+  toTool: string;
+  slot?: string;
+}
+
+/**
+ * The agent-shaped expected-flow spec (EV-037). SUBJECT-AGNOSTIC: every field is
+ * supplied by the subject profile (auto-gen EV-049 or authored), so the audit
+ * diffs ACTUAL flow-graph vs EXPECTED without any subject constant in the engine.
+ *   - `dispatchToolNames` — which tool names are sub-agent dispatches (EV-029).
+ *   - `edges`             — producer→consumer threadings the subject should have.
+ *   - `expectedUiSlots`   — slots that SHOULD be rendered in the subject's HTML
+ *                            artifact (EV-039/040, the operator's missing-data case).
+ */
+export interface ExpectedFlow {
+  dispatchToolNames: string[];
+  edges: ExpectedFlowEdge[];
+  expectedUiSlots: string[];
+}
+
+/** Diff of an actual flow-graph against an expected-flow (deterministic). */
+export interface ExpectedFlowDiff {
+  satisfied: ExpectedFlowEdge[];
+  missing: ExpectedFlowEdge[];
+}
+
+// ── TypeBox guards (closed objects — auditable boundary) ─────────────────────
+export const FlowNodeSchema = Type.Object(
+  {
+    id: Type.String({ minLength: 1 }),
+    index: Type.Integer({ minimum: 0 }),
+    type: Type.String({ minLength: 1 }),
+    name: Type.String(),
+    produces: Type.Array(Type.String()),
+    consumes: Type.Array(Type.String()),
+    isHandoff: Type.Boolean(),
+  },
+  { additionalProperties: false },
+);
+
+export const FlowEdgeSchema = Type.Object(
+  {
+    from: Type.String({ minLength: 1 }),
+    to: Type.String({ minLength: 1 }),
+    slot: Type.String({ minLength: 1 }),
+    kind: Type.Union([
+      Type.Literal(FlowEdgeKind.Threaded),
+      Type.Literal(FlowEdgeKind.Handoff),
+    ]),
+  },
+  { additionalProperties: false },
+);
+
+export const UnthreadedOutputSchema = Type.Object(
+  {
+    node: Type.String({ minLength: 1 }),
+    name: Type.String(),
+    slot: Type.String({ minLength: 1 }),
+  },
+  { additionalProperties: false },
+);
+
+export const FlowGraphSchema = Type.Object(
+  {
+    traceId: Type.String(),
+    nodes: Type.Array(FlowNodeSchema),
+    edges: Type.Array(FlowEdgeSchema),
+    unthreadedOutputs: Type.Array(UnthreadedOutputSchema),
+  },
+  { additionalProperties: false },
+);
+export type FlowGraphStatic = Static<typeof FlowGraphSchema>;
+
+export const ExpectedFlowEdgeSchema = Type.Object(
+  {
+    fromTool: Type.String({ minLength: 1 }),
+    toTool: Type.String({ minLength: 1 }),
+    slot: Type.Optional(Type.String({ minLength: 1 })),
+  },
+  { additionalProperties: false },
+);
+
+export const ExpectedFlowSchema = Type.Object(
+  {
+    dispatchToolNames: Type.Array(Type.String({ minLength: 1 })),
+    edges: Type.Array(ExpectedFlowEdgeSchema),
+    expectedUiSlots: Type.Array(Type.String({ minLength: 1 })),
+  },
+  { additionalProperties: false },
+);
+export type ExpectedFlowStatic = Static<typeof ExpectedFlowSchema>;
+
+/**
+ * Validate + narrow an unknown value to `ExpectedFlow` (guarded-parse style).
+ * THROWS with the first schema error — a malformed authored/loaded expected-flow
+ * must never silently reach the audit.
+ */
+export function parseExpectedFlow(value: unknown): ExpectedFlow {
+  if (!Value.Check(ExpectedFlowSchema, value)) {
+    const first = [...Value.Errors(ExpectedFlowSchema, value)][0];
+    const where = first?.path ?? "(root)";
+    const msg = first?.message ?? "does not match ExpectedFlow";
+    throw new Error(`parseExpectedFlow: invalid expected-flow at ${where}: ${msg}`);
+  }
+  return value;
+}
+
+/** An empty expected-flow (no constraints) — the EV-049 pre-authoring default. */
+export const EMPTY_EXPECTED_FLOW: ExpectedFlow = {
+  dispatchToolNames: [],
+  edges: [],
+  expectedUiSlots: [],
+};

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/types.ts

@@ -0,0 +1,303 @@
+/**
+ * scripts/contracts/types.ts
+ * ---------------------------------------------------------------------------
+ * Canonical TypeBox contracts + exported TS types for the generic evaluator
+ * engine. These are the single source of truth that load-bundle / run-
+ * deterministic / assemble-scorecard / variance-compare / render-report all
+ * consume — no downstream guessing.
+ *
+ * Kept in lockstep with the human-readable shape contracts under schemas/*.yaml.
+ */
+import { type Static, Type } from "@sinclair/typebox";
+
+// ── Categorical constants (no magic strings — coding-rules) ─────────────────
+export const Dimension = {
+  OperationCorrectness: "operation-correctness",
+  DataCorrectness: "data-correctness",
+  OperationalDeviation: "operational-deviation",
+} as const;
+export type DimensionValue = (typeof Dimension)[keyof typeof Dimension];
+
+export const CheckMethod = {
+  DeterministicScript: "deterministic-script",
+  TypeboxSchema: "typebox-schema",
+  Gate: "gate",
+  TraceCrossRef: "trace-cross-ref",
+  TrajectoryDiff: "trajectory-diff",
+  // SF-2 (DEC-16) — additive generic content-judge checkMethod. Judge-track (it is
+  // NOT in DETERMINISTIC_CHECK_METHODS, so trackForCheckMethod routes it to Judge).
+  // Lets a v1 eval-matrix row be a generic LLM content judge, not only the
+  // diagnostics-specific trace-cross-ref / trajectory-diff. v2 MinedCriterion carries
+  // its own 3-value router (eval-types.ts check_method); the two coexist.
+  LlmJudge: "llm-judge",
+} as const;
+export type CheckMethodValue = (typeof CheckMethod)[keyof typeof CheckMethod];
+
+export const Severity = {
+  Crit: "CRIT",
+  High: "HIGH",
+  Med: "MED",
+  Low: "LOW",
+} as const;
+export type SeverityValue = (typeof Severity)[keyof typeof Severity];
+
+export const ExistingCoverage = {
+  Covered: "covered",
+  Partial: "partial",
+  None: "none",
+} as const;
+export type ExistingCoverageValue =
+  (typeof ExistingCoverage)[keyof typeof ExistingCoverage];
+
+export const Track = {
+  Deterministic: "deterministic",
+  Judge: "judge",
+} as const;
+export type TrackValue = (typeof Track)[keyof typeof Track];
+
+export const RowResult = {
+  Pass: "pass",
+  Fail: "fail",
+  Skip: "skip",
+  /** GA — a CRIT/HIGH criterion that adjudicated indeterminate (uncertain). It
+   *  does NOT fail the component but blocks certification ⇒ component/run
+   *  `incomplete`. Additive; the v1 audit path never emits it (back-compat). */
+  Incomplete: "incomplete",
+} as const;
+export type RowResultValue = (typeof RowResult)[keyof typeof RowResult];
+
+/** GA — the run/component ternary lattice `fail ▸ incomplete ▸ pass`. */
+export const RunVerdict = {
+  Pass: "pass",
+  Fail: "fail",
+  Incomplete: "incomplete",
+} as const;
+export type RunVerdictValue = (typeof RunVerdict)[keyof typeof RunVerdict];
+
+/**
+ * The deterministic-vs-judge split is a pure function of checkMethod
+ * (decision #4): deterministic-script | typebox-schema | gate run with NO
+ * model; trace-cross-ref | trajectory-diff run under the pinned judge.
+ */
+export const DETERMINISTIC_CHECK_METHODS: ReadonlySet<string> = new Set([
+  CheckMethod.DeterministicScript,
+  CheckMethod.TypeboxSchema,
+  CheckMethod.Gate,
+]);
+
+export function trackForCheckMethod(checkMethod: string): TrackValue {
+  return DETERMINISTIC_CHECK_METHODS.has(checkMethod)
+    ? Track.Deterministic
+    : Track.Judge;
+}
+
+// ── Eval-matrix (subject profile) ───────────────────────────────────────────
+export const CriterionSchema = Type.Object(
+  {
+    dimension: Type.Union([
+      Type.Literal(Dimension.OperationCorrectness),
+      Type.Literal(Dimension.DataCorrectness),
+      Type.Literal(Dimension.OperationalDeviation),
+    ]),
+    statement: Type.String({ minLength: 1 }),
+    checkMethod: Type.Union([
+      Type.Literal(CheckMethod.DeterministicScript),
+      Type.Literal(CheckMethod.TypeboxSchema),
+      Type.Literal(CheckMethod.Gate),
+      Type.Literal(CheckMethod.TraceCrossRef),
+      Type.Literal(CheckMethod.TrajectoryDiff),
+      Type.Literal(CheckMethod.LlmJudge), // SF-2 additive (DEC-16)
+    ]),
+    passCondition: Type.String({ minLength: 1 }),
+    severity: Type.Union([
+      Type.Literal(Severity.Crit),
+      Type.Literal(Severity.High),
+      Type.Literal(Severity.Med),
+      Type.Literal(Severity.Low),
+    ]),
+    existingCoverage: Type.Union([
+      Type.Literal(ExistingCoverage.Covered),
+      Type.Literal(ExistingCoverage.Partial),
+      Type.Literal(ExistingCoverage.None),
+    ]),
+    evidenceSource: Type.Optional(Type.String()),
+  },
+  { additionalProperties: false },
+);
+export type Criterion = Static<typeof CriterionSchema>;
+
+export const ComponentSchema = Type.Object(
+  {
+    componentId: Type.String({ minLength: 1 }),
+    componentName: Type.String({ minLength: 1 }),
+    componentType: Type.Union([
+      Type.Literal("agent"),
+      Type.Literal("skill-orchestrator-step"),
+      Type.Literal("star-command"),
+      Type.Literal("script"),
+      Type.Literal("gate"),
+      // SF-1 additive — product-feature / pipeline-stage component kinds (a
+      // multi-stage generation subject is a CHAIN of stages; §6a).
+      Type.Literal("feature"),
+      Type.Literal("pipeline-stage"),
+    ]),
+    criteria: Type.Array(CriterionSchema),
+  },
+  { additionalProperties: false },
+);
+export type Component = Static<typeof ComponentSchema>;
+
+export const EvalMatrixSchema = Type.Object(
+  {
+    subject: Type.String({ minLength: 1 }),
+    version: Type.String({ minLength: 1 }),
+    generatedAt: Type.String(),
+    designPrinciples: Type.Array(Type.String()),
+    components: Type.Array(ComponentSchema),
+  },
+  { additionalProperties: false },
+);
+export type EvalMatrix = Static<typeof EvalMatrixSchema>;
+
+// ── Scorecard (two-track) ───────────────────────────────────────────────────
+export const ScorecardCriterionSchema = Type.Object(
+  {
+    dimension: Type.String(),
+    severity: Type.String(),
+    checkMethod: Type.String(),
+    track: Type.Union([
+      Type.Literal(Track.Deterministic),
+      Type.Literal(Track.Judge),
+    ]),
+    result: Type.Union([
+      Type.Literal(RowResult.Pass),
+      Type.Literal(RowResult.Fail),
+      Type.Literal(RowResult.Skip),
+      Type.Literal(RowResult.Incomplete), // GA — additive (back-compat)
+    ]),
+    detail: Type.Optional(Type.String()),
+  },
+  { additionalProperties: false },
+);
+export type ScorecardCriterion = Static<typeof ScorecardCriterionSchema>;
+
+export const GateComponentSchema = Type.Object(
+  {
+    componentId: Type.String(),
+    pass: Type.Boolean(),
+    /** GA — the component ternary verdict (optional/back-compat; `pass` mirrors
+     *  `verdict === "pass"`). `incomplete` when a CRIT/HIGH row is indeterminate. */
+    verdict: Type.Optional(
+      Type.Union([
+        Type.Literal(RunVerdict.Pass),
+        Type.Literal(RunVerdict.Fail),
+        Type.Literal(RunVerdict.Incomplete),
+      ]),
+    ),
+    criteria: Type.Array(ScorecardCriterionSchema),
+  },
+  { additionalProperties: false },
+);
+export type GateComponent = Static<typeof GateComponentSchema>;
+
+export const TrendDimensionSchema = Type.Object(
+  {
+    name: Type.String(),
+    measure: Type.String(),
+    target: Type.String(),
+    divergence: Type.Union([
+      Type.Literal("identical"),
+      Type.Literal("within-target"),
+      Type.Literal("diverged"),
+      Type.Literal("not-evaluated"),
+    ]),
+  },
+  { additionalProperties: false },
+);
+export type TrendDimension = Static<typeof TrendDimensionSchema>;
+
+// Coverage honesty (EV-OUT-002): graded-vs-total + skip-rate warning. Optional
+// so older scorecards (without it) still validate; warning-only — never gates.
+export const CoverageSchema = Type.Object(
+  {
+    graded: Type.Integer(),
+    total: Type.Integer(),
+    skipped: Type.Integer(),
+    skipRate: Type.Number(),
+    skipRateWarnThreshold: Type.Number(),
+    coverageWarning: Type.Boolean(),
+  },
+  { additionalProperties: false },
+);
+export type Coverage = Static<typeof CoverageSchema>;
+
+export const ScorecardSchema = Type.Object(
+  {
+    subject: Type.String(),
+    runId: Type.String(),
+    generatedAt: Type.String(),
+    coverage: Type.Optional(CoverageSchema),
+    gate: Type.Object(
+      {
+        runPass: Type.Boolean(),
+        /** GA — the ternary run verdict `fail ▸ incomplete ▸ pass` (optional/
+         *  back-compat; `runPass` is `runVerdict === "pass"`). */
+        runVerdict: Type.Optional(
+          Type.Union([
+            Type.Literal(RunVerdict.Pass),
+            Type.Literal(RunVerdict.Fail),
+            Type.Literal(RunVerdict.Incomplete),
+          ]),
+        ),
+        components: Type.Array(GateComponentSchema),
+        totals: Type.Object(
+          {
+            pass: Type.Integer(),
+            fail: Type.Integer(),
+            skip: Type.Integer(),
+            critFail: Type.Integer(),
+            highFail: Type.Integer(),
+            /** GA — CRIT/HIGH indeterminate count (optional/back-compat). */
+            incomplete: Type.Optional(Type.Integer()),
+          },
+          { additionalProperties: false },
+        ),
+      },
+      { additionalProperties: false },
+    ),
+    trend: Type.Object(
+      {
+        runPair: Type.Optional(
+          Type.Object(
+            { a: Type.String(), b: Type.String() },
+            { additionalProperties: false },
+          ),
+        ),
+        dimensions: Type.Array(TrendDimensionSchema),
+        varianceScore: Type.Number(),
+      },
+      { additionalProperties: false },
+    ),
+  },
+  { additionalProperties: false },
+);
+export type Scorecard = Static<typeof ScorecardSchema>;
+
+// ── Run-bundle (the audited run's artifacts) ────────────────────────────────
+/**
+ * The validated audit input produced by load-bundle.ts. A run-bundle is a
+ * .mutagent/diagnostics/{runId}/ directory. The evaluator is subject-agnostic:
+ * it loads whatever artifacts the subject's run produced and exposes them as a
+ * flat, validated map. Only `runId` is structurally required; everything else
+ * is best-effort per the subject.
+ */
+export interface RunBundle {
+  runId: string;
+  bundleDir: string;
+  /** absolute paths to artifact files discovered in the bundle */
+  artifacts: Record<string, string>;
+  /** parsed JSON artifacts keyed by logical name (runMeta, tracesMetadata, ...) */
+  data: Record<string, unknown>;
+  /** non-fatal load warnings (missing-but-optional artifacts) */
+  warnings: string[];
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/validation.ts

@@ -0,0 +1,193 @@
+/**
+ * scripts/contracts/validation.ts — the W2 TRUST-layer data contracts.
+ * ---------------------------------------------------------------------------
+ * The executable (TypeBox) companion for the W2 trust track:
+ *   - `HumanLabel` — one ground-truth annotation produced by `*review`
+ *     (`scripts/build-review-ui.ts`) and consumed by `*validate`
+ *     (`scripts/validate-judge.ts`). This is the human-judgment a judge is
+ *     calibrated AGAINST (validate-evaluator.md TPR/TNR).
+ *
+ * Strict austerity (operator directive): this file is **Type A — data contract
+ * only**. It holds NO judge prompt, NO LLM-reasoning, makes NO pass/fail
+ * decision. It only declares + validates the label shapes the deterministic
+ * scripts read/write. PURE — no clock / random / network.
+ *
+ * Disjointness (W2-OWN): this is a NEW W2 contract file. It does NOT touch the
+ * shared `contracts/eval-types.ts` (W3 may touch shared types — kept out of it).
+ * The confusion-matrix / validation-result shapes (`*validate` numbers) are
+ * appended to this same file by the W2 *validate task.
+ */
+import { type Static, Type } from "@sinclair/typebox";
+import { Value } from "@sinclair/typebox/value";
+
+// ── HumanLabel verdict (the annotation the reviewer picks) ───────────────────
+//
+// Mirrors the determiner's OutcomeVerdict closed set (pass/fail) PLUS `defer`
+// — the review UI's third button for genuinely uncertain traces. A deferred
+// label is NOT a judgment: it is excluded from TPR/TNR (it provides no ground
+// truth), and it drives uncertainty resampling. We keep the set local to the
+// trust layer rather than importing OutcomeVerdict, because `defer` is a
+// review-UI concept (a human punting), not a determiner outcome.
+export const HumanVerdict = {
+  Pass: "pass",
+  Fail: "fail",
+  /** reviewer is uncertain — excluded from confusion-matrix math, never coerced. */
+  Defer: "defer",
+} as const;
+export type HumanVerdictValue = (typeof HumanVerdict)[keyof typeof HumanVerdict];
+
+// ── HumanLabel — one reviewer annotation for one trace ───────────────────────
+export const HumanLabelSchema = Type.Object(
+  {
+    /** the trace this label judges. */
+    traceId: Type.String({ minLength: 1 }),
+    /** the reviewer's binary verdict (or `defer`). */
+    label: Type.Union([
+      Type.Literal(HumanVerdict.Pass),
+      Type.Literal(HumanVerdict.Fail),
+      Type.Literal(HumanVerdict.Defer),
+    ]),
+    /** free-text note describing what went wrong/right (build-review-interface.md). */
+    notes: Type.Optional(Type.String()),
+    /**
+     * which data SPLIT this labeled trace belongs to (train/dev/test). Carried so
+     * `*validate` can enforce split-disjointness + the test-once guard. Optional —
+     * the review UI may collect labels before the split is assigned.
+     */
+    split: Type.Optional(
+      Type.Union([Type.Literal("train"), Type.Literal("dev"), Type.Literal("test")]),
+    ),
+    /** the annotator id (for two-annotator agreement). Optional. */
+    annotator: Type.Optional(Type.String()),
+    /**
+     * ISO timestamp stamped by the BROWSER on save (non-deterministic). The
+     * deterministic script core NEVER stamps this (mask discipline / C-PIN); it
+     * only round-trips + dedups by traceId. Optional so masked fixtures validate.
+     */
+    labeledAt: Type.Optional(Type.String()),
+  },
+  { additionalProperties: false },
+);
+export type HumanLabel = Static<typeof HumanLabelSchema>;
+
+/**
+ * Validate + parse a labels file (raw JSON text = `HumanLabel[]`). ENFORCES the
+ * schema (closed label set, traceId non-empty); THROWS on any violation — a
+ * malformed labels file is never silently accepted. PURE.
+ */
+export function parseHumanLabels(raw: string): HumanLabel[] {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    throw new Error(`parseHumanLabels: not valid JSON: ${raw.slice(0, 120)}`);
+  }
+  const schema = Type.Array(HumanLabelSchema);
+  if (!Value.Check(schema, parsed)) {
+    const first = [...Value.Errors(schema, parsed)][0];
+    throw new Error(
+      `parseHumanLabels: schema violation at '${first?.path ?? "?"}': ` +
+        `${first?.message ?? "invalid HumanLabel[]"}`,
+    );
+  }
+  return parsed;
+}
+
+/** Assert one HumanLabel conforms (used before persisting). THROWS on violation. */
+export function assertHumanLabel(label: unknown): asserts label is HumanLabel {
+  if (!Value.Check(HumanLabelSchema, label)) {
+    const first = [...Value.Errors(HumanLabelSchema, label)][0];
+    throw new Error(
+      `assertHumanLabel: schema violation at '${first?.path ?? "?"}': ` +
+        `${first?.message ?? "invalid HumanLabel"}`,
+    );
+  }
+}
+
+// ── *validate (EV-044) result shapes — the trust layer numbers ───────────────
+//
+// The output of calibrating a judge against human labels (validate-evaluator.md):
+// a 2×2 confusion matrix → TPR/TNR → Rogan-Gladen bias-corrected true rate →
+// bootstrap CI, with a `validated`/`unvalidated` status that degrades gracefully
+// when there are too few human labels (the loop NEVER blocks on a human).
+
+/** A judge's calibration status. */
+export const ValidationStatus = {
+  /** ≥ MIN_LABELS labels AND TPR/TNR clear the target — verdicts are trusted. */
+  Validated: "validated",
+  /**
+   * too few labels OR TPR/TNR below target — the judge's aggregate rate is
+   * reported BIAS-CORRECTED (not raw) and flagged; never blocks the loop.
+   */
+  Unvalidated: "unvalidated",
+} as const;
+export type ValidationStatusValue = (typeof ValidationStatus)[keyof typeof ValidationStatus];
+
+/** 2×2 confusion matrix (Pass = positive class). `defer`/`uncertain` excluded. */
+export const ConfusionMatrixSchema = Type.Object(
+  {
+    /** judge Pass AND human Pass. */
+    tp: Type.Integer({ minimum: 0 }),
+    /** judge Pass AND human Fail (false pass — judge too lenient). */
+    fp: Type.Integer({ minimum: 0 }),
+    /** judge Fail AND human Fail. */
+    tn: Type.Integer({ minimum: 0 }),
+    /** judge Fail AND human Pass (false fail — judge too strict). */
+    fn: Type.Integer({ minimum: 0 }),
+  },
+  { additionalProperties: false },
+);
+export type ConfusionMatrix = Static<typeof ConfusionMatrixSchema>;
+
+/** A bootstrap confidence interval for the corrected rate. */
+export const ConfidenceIntervalSchema = Type.Object(
+  {
+    lo: Type.Number({ minimum: 0, maximum: 1 }),
+    hi: Type.Number({ minimum: 0, maximum: 1 }),
+    /** the 2-sided level (e.g. 0.95). */
+    level: Type.Number({ minimum: 0, maximum: 1 }),
+  },
+  { additionalProperties: false },
+);
+export type ConfidenceInterval = Static<typeof ConfidenceIntervalSchema>;
+
+/** The full `*validate` result for ONE judge (one criterion). */
+export const ValidationResultSchema = Type.Object(
+  {
+    criterionId: Type.String({ minLength: 1 }),
+    /** the PINNED judge model the numbers belong to (C-PIN — re-validate on change). */
+    judgeModel: Type.String({ minLength: 1 }),
+    status: Type.Union([
+      Type.Literal(ValidationStatus.Validated),
+      Type.Literal(ValidationStatus.Unvalidated),
+    ]),
+    /** total non-deferred human labels used (the calibration sample size). */
+    labelCount: Type.Integer({ minimum: 0 }),
+    /** true-positive rate (judge agrees on Pass); null when no human Pass exists. */
+    tpr: Type.Union([Type.Number({ minimum: 0, maximum: 1 }), Type.Null()]),
+    /** true-negative rate (judge agrees on Fail); null when no human Fail exists. */
+    tnr: Type.Union([Type.Number({ minimum: 0, maximum: 1 }), Type.Null()]),
+    /** observed judge Pass-rate on the unlabeled production set (p_obs). */
+    observedPassRate: Type.Union([Type.Number({ minimum: 0, maximum: 1 }), Type.Null()]),
+    /** Rogan-Gladen bias-corrected true rate; null when invalid (judge ≈ random). */
+    correctedRate: Type.Union([Type.Number({ minimum: 0, maximum: 1 }), Type.Null()]),
+    /** whether the Rogan-Gladen correction was valid (TPR+TNR−1 not ≈ 0). */
+    correctionValid: Type.Boolean(),
+    ci: Type.Optional(ConfidenceIntervalSchema),
+    /** human-readable note on the status (why unvalidated, etc.). */
+    note: Type.Optional(Type.String()),
+  },
+  { additionalProperties: false },
+);
+export type ValidationResult = Static<typeof ValidationResultSchema>;
+
+/** Assert a ValidationResult conforms (used before stamping the scorecard). THROWS. */
+export function assertValidationResult(r: unknown): asserts r is ValidationResult {
+  if (!Value.Check(ValidationResultSchema, r)) {
+    const first = [...Value.Errors(ValidationResultSchema, r)][0];
+    throw new Error(
+      `assertValidationResult: schema violation at '${first?.path ?? "?"}': ` +
+        `${first?.message ?? "invalid ValidationResult"}`,
+    );
+  }
+}