@mutagent/evaluator 0.1.0-alpha.2

package/.claude/skills/mutagent-evaluator/scripts/cli/variance-check.ts

@@ -0,0 +1,105 @@
+/**
+ * scripts/cli/variance-check.ts — coordinator entrypoint.
+ * ---------------------------------------------------------------------------
+ * The COORDINATOR role (executor != reviewer): compares TWO run bundles on the
+ * fixed 15-dim determinism scorecard and emits the delta + trend. This is fully
+ * deterministic — no judge, no model — so it runs as a complete wired script.
+ *
+ *   variance-check.ts <bundleDirA> <bundleDirB> [--out <scorecard.json>]
+ *                     [--generated-at <iso>] [--emit-masked]
+ */
+import { writeFileSync } from "node:fs";
+import { loadBundle } from "../load-bundle.ts";
+import { varianceCompare } from "../variance-compare.ts";
+import { assembleScorecard } from "../assemble-scorecard.ts";
+import { maskedCanonicalJson } from "../mask.ts";
+import { DEFAULT_GENERATED_AT } from "./audit-run.ts";
+
+interface Args {
+  a: string;
+  b: string;
+  out?: string;
+  generatedAt: string;
+  emitMasked: boolean;
+}
+
+function parseArgs(argv: string[]): Args {
+  const positional: string[] = [];
+  const flags: Record<string, string | boolean> = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a.startsWith("--")) {
+      const key = a.slice(2);
+      const next = argv[i + 1];
+      if (next && !next.startsWith("--")) {
+        flags[key] = next;
+        i++;
+      } else flags[key] = true;
+    } else positional.push(a);
+  }
+  if (positional.length < 2) {
+    throw new Error(
+      "usage: variance-check.ts <bundleDirA> <bundleDirB> [--out <file>] [--generated-at <iso>] [--emit-masked]",
+    );
+  }
+  return {
+    a: positional[0],
+    b: positional[1],
+    out: (flags.out as string) || undefined,
+    generatedAt: (flags["generated-at"] as string) || DEFAULT_GENERATED_AT,
+    emitMasked: flags["emit-masked"] === true,
+  };
+}
+
+export function main(argvIn?: string[]): number {
+  const argv =
+    argvIn ??
+    (typeof Bun !== "undefined" ? Bun.argv.slice(2) : process.argv.slice(2));
+  const args = parseArgs(argv);
+
+  const { bundle: a } = loadBundle(args.a);
+  const { bundle: b } = loadBundle(args.b);
+  const trend = varianceCompare(a, b);
+
+  // Carry the trend into a TREND-only scorecard frame (GATE empty — this is the
+  // coordinator's variance comparison, not a conformance gate).
+  const scorecard = assembleScorecard({
+    subject: a.runId,
+    runId: `${a.runId}__vs__${b.runId}`,
+    generatedAt: args.generatedAt,
+    rows: [],
+    trendDimensions: trend.dimensions,
+    trendRunPair: trend.runPair,
+  });
+
+  const json = args.emitMasked
+    ? maskedCanonicalJson(scorecard)
+    : JSON.stringify(scorecard, null, 2);
+  if (args.out) writeFileSync(args.out, json + "\n");
+
+  console.info(
+    JSON.stringify(
+      {
+        runPair: trend.runPair,
+        varianceScore: trend.varianceScore,
+        dimensions: trend.dimensions.length,
+        diverged: trend.dimensions
+          .filter((d) => d.divergence === "diverged")
+          .map((d) => d.name),
+        out: args.out ?? "(stdout only)",
+      },
+      null,
+      2,
+    ),
+  );
+  // exit 0 when byte-identical (varianceScore 0), 1 otherwise.
+  return trend.varianceScore === 0 ? 0 : 1;
+}
+
+declare const Bun: { argv: string[] } | undefined;
+const isMain =
+  typeof import.meta !== "undefined" &&
+  (import.meta as unknown as { main?: boolean }).main === true;
+if (isMain) {
+  process.exit(main());
+}

package/.claude/skills/mutagent-evaluator/scripts/code-eval.ts

@@ -0,0 +1,248 @@
+/**
+ * scripts/code-eval.ts — P3 (EX-2): the v2 CODE-track primitive library.
+ * ---------------------------------------------------------------------------
+ * Code-class criteria (§5b `check_method: deterministic`, and the pre-filter
+ * half of `hybrid`) are evaluated by DETERMINISTIC code-eval scripts the
+ * evaluator agent runs via its Bash tool — NO LLM, ZERO judge tokens,
+ * byte-identical across reruns (restores full C-PIN on code rows; closes M1/M2).
+ *
+ * This is a generic, SUBJECT-AGNOSTIC primitive set over the `EvalTrace` shape:
+ *   - presence              — a field is present + non-empty
+ *   - string-equality       — exact (optionally case-insensitive) match
+ *   - format-validity       — a regex/format conformance check
+ *   - schema-conformance    — an object carries the required keys
+ *   - ref-integrity         — cross-stage: every value produced in stage A is
+ *                             present in stage B's output (referential integrity)
+ *   - recovery-after-failure — TEMPORAL: a failure marker on an observation MUST
+ *                             be followed by a recovery tool LATER in the trace; an
+ *                             un-recovered failure is a "silent drop" (FAIL).
+ *   - tool-output-failure   — a named tool's success flag is false on any call (FAIL).
+ * A per-subject extraction (check-method-router.ts) maps a code-class
+ * MinedCriterion → one of these specs; the primitives themselves hold NO subject
+ * literal — the field names, fail values, and tool names are all PARAMETERS, so
+ * the temporal/tool primitives are reusable on ANY agent's failure modes (the
+ * sample `send-failure-silent-drop` mode is just one parameterization).
+ *
+ * TWO-WORLDS BOUNDARY: this is the v2 code-track over `EvalTrace` +
+ * `MinedCriterion`. It deliberately does NOT import the v1 audit world
+ * (`run-deterministic.ts` / `contracts/types.ts` 6-value Track over RunBundle) —
+ * the two stay disjoint.
+ *
+ * PURE + deterministic: no clock / random / network. Reading a trace field is
+ * the only effect-free input; the verdict is a pure function of (spec, trace).
+ */
+import type { EvalTrace, TraceObservation } from "./contracts/eval-types.ts";
+
+/** A single code-eval's binary, deterministic outcome. `detail` → verdict critique. */
+export interface CodeEvalResult {
+  result: "pass" | "fail";
+  detail: string;
+}
+
+/**
+ * Read a dotted path off an `EvalTrace`. Supports:
+ *   - `output.response`, `input.prompt`        → navigate the trace object
+ *   - `obs:<name>.<path>`                       → the named GENERATION observation's
+ *                                                 output, then navigate (SV-1: per-stage
+ *                                                 outputs live in the observations)
+ * Returns undefined for any missing segment (tolerant — never throws). PURE.
+ */
+export function readTracePath(trace: EvalTrace, path: string): unknown {
+  if (path.startsWith("obs:")) {
+    const rest = path.slice(4);
+    const dot = rest.indexOf(".");
+    const name = dot === -1 ? rest : rest.slice(0, dot);
+    const sub = dot === -1 ? "" : rest.slice(dot + 1);
+    const obs = trace.observations.find(
+      (o: TraceObservation) => o.name === name && typeof o.type === "string" && o.type.toUpperCase() === "GENERATION",
+    );
+    if (obs === undefined) return undefined;
+    return sub.length === 0 ? obs.output : navigate(obs.output, sub);
+  }
+  return navigate(trace as unknown, path);
+}
+
+/** Navigate a dotted path through a plain object graph. Undefined on any gap. */
+function navigate(root: unknown, path: string): unknown {
+  let cur: unknown = root;
+  for (const seg of path.split(".")) {
+    if (cur === null || typeof cur !== "object") return undefined;
+    cur = (cur as Record<string, unknown>)[seg];
+    if (cur === undefined) return undefined;
+  }
+  return cur;
+}
+
+/** Coerce a value to a search "haystack" string for ref-integrity. */
+function haystack(value: unknown): string {
+  if (typeof value === "string") return value;
+  if (value === undefined || value === null) return "";
+  return JSON.stringify(value);
+}
+
+/** Coerce a producer value to its set of string tokens (array → items; scalar → [it]). */
+function tokens(value: unknown): string[] {
+  if (Array.isArray(value)) return value.map((v) => (typeof v === "string" ? v : JSON.stringify(v)));
+  if (value === undefined || value === null) return [];
+  return [typeof value === "string" ? value : JSON.stringify(value)];
+}
+
+/**
+ * Coerce a scalar trace value to a canonical string for EXACT equality compare
+ * (used by the temporal/tool primitives). Booleans/numbers stringify (`false` →
+ * "false", `0` → "0"); strings pass through; null/undefined → "" (so a missing
+ * field never spuriously equals a non-empty `failEquals`); objects JSON-serialize.
+ */
+function coerce(value: unknown): string {
+  if (typeof value === "string") return value;
+  if (typeof value === "boolean" || typeof value === "number" || typeof value === "bigint") {
+    return String(value);
+  }
+  if (value === undefined || value === null) return "";
+  return JSON.stringify(value);
+}
+
+/** The `name` of an observation, or "" when unnamed. */
+function obsName(o: TraceObservation): string {
+  return typeof o.name === "string" ? o.name : "";
+}
+
+// ── The CodeEvalSpec union (one per primitive) ──────────────────────────────
+
+export type CodeEvalSpec =
+  | { primitive: "presence"; field: string }
+  | { primitive: "string-equality"; field: string; expected: string; caseInsensitive?: boolean }
+  | { primitive: "format-validity"; field: string; pattern: string }
+  | { primitive: "schema-conformance"; field: string; requiredKeys: string[] }
+  | { primitive: "ref-integrity"; producer: string; consumer: string }
+  | {
+      primitive: "recovery-after-failure";
+      /** path read RELATIVE TO each observation (e.g. "output.status"). */
+      failField: string;
+      /** the coerced value at `failField` that marks an observation as a failure. */
+      failEquals: string;
+      /** observation `name`s that count as a recovery reaction AFTER the failure. */
+      recoveryTools: string[];
+    }
+  | {
+      primitive: "tool-output-failure";
+      /** the observation `name` of the tool to inspect. */
+      tool: string;
+      /** path read RELATIVE TO the tool observation; `false` (bool/string) ⇒ failure. */
+      successPath: string;
+    };
+
+function pass(detail: string): CodeEvalResult {
+  return { result: "pass", detail };
+}
+function fail(detail: string): CodeEvalResult {
+  return { result: "fail", detail };
+}
+
+/**
+ * Run one code-eval over a trace → a deterministic binary verdict. Exhaustive on
+ * the primitive union; an UNKNOWN primitive THROWS (no silent pass — a
+ * misconfigured code-eval is a fail-loud error, never a fabricated success). PURE.
+ */
+export function runCodeEval(spec: CodeEvalSpec, trace: EvalTrace): CodeEvalResult {
+  switch (spec.primitive) {
+    case "presence": {
+      const v = readTracePath(trace, spec.field);
+      const empty =
+        v === undefined ||
+        v === null ||
+        (typeof v === "string" && v.length === 0) ||
+        (Array.isArray(v) && v.length === 0);
+      return empty ? fail(`'${spec.field}' is absent/empty`) : pass(`'${spec.field}' is present`);
+    }
+    case "string-equality": {
+      const v = readTracePath(trace, spec.field);
+      const s = typeof v === "string" ? v : haystack(v);
+      const a = spec.caseInsensitive === true ? s.toLowerCase() : s;
+      const b = spec.caseInsensitive === true ? spec.expected.toLowerCase() : spec.expected;
+      return a === b
+        ? pass(`'${spec.field}' equals expected`)
+        : fail(`'${spec.field}'='${s}' != expected '${spec.expected}'`);
+    }
+    case "format-validity": {
+      const v = readTracePath(trace, spec.field);
+      const s = typeof v === "string" ? v : haystack(v);
+      return new RegExp(spec.pattern).test(s)
+        ? pass(`'${spec.field}' matches /${spec.pattern}/`)
+        : fail(`'${spec.field}'='${s}' does not match /${spec.pattern}/`);
+    }
+    case "schema-conformance": {
+      const v = readTracePath(trace, spec.field);
+      if (v === null || typeof v !== "object") {
+        return fail(`'${spec.field}' is not an object`);
+      }
+      const obj = v as Record<string, unknown>;
+      const missing = spec.requiredKeys.filter((k) => obj[k] === undefined);
+      return missing.length === 0
+        ? pass(`'${spec.field}' has all required keys`)
+        : fail(`'${spec.field}' missing keys: ${missing.join(", ")}`);
+    }
+    case "ref-integrity": {
+      const producerTokens = tokens(readTracePath(trace, spec.producer));
+      const hay = haystack(readTracePath(trace, spec.consumer));
+      const missing = producerTokens.filter((t) => !hay.includes(t));
+      return missing.length === 0
+        ? pass(`all ${producerTokens.length} '${spec.producer}' value(s) present in '${spec.consumer}'`)
+        : fail(`'${spec.consumer}' is missing producer value(s): ${missing.join(", ")}`);
+    }
+    case "recovery-after-failure": {
+      // TEMPORAL: find the FIRST observation whose `failField` marks a failure, then
+      // require a recovery-tool observation AFTER it. No failure ⇒ vacuously pass.
+      let failIdx = -1;
+      for (let i = 0; i < trace.observations.length; i++) {
+        const v = navigate(trace.observations[i], spec.failField);
+        if (v !== undefined && coerce(v) === spec.failEquals) {
+          failIdx = i;
+          break;
+        }
+      }
+      if (failIdx === -1) {
+        return pass(
+          `no observation has '${spec.failField}'=='${spec.failEquals}' — no failure to recover from`,
+        );
+      }
+      const recoverySet = new Set(spec.recoveryTools);
+      const after = trace.observations.slice(failIdx + 1);
+      const recovered = after.find((o) => recoverySet.has(obsName(o)));
+      return recovered !== undefined
+        ? pass(
+            `failure at obs#${failIdx} ('${spec.failField}'=='${spec.failEquals}') was followed by ` +
+              `recovery tool '${obsName(recovered)}'`,
+          )
+        : fail(
+            `failure at obs#${failIdx} ('${spec.failField}'=='${spec.failEquals}') with NO recovery ` +
+              `tool (${spec.recoveryTools.join(" | ")}) AFTER it — silent drop`,
+          );
+    }
+    case "tool-output-failure": {
+      const calls = trace.observations.filter((o) => obsName(o) === spec.tool);
+      if (calls.length === 0) {
+        return pass(`no '${spec.tool}' observation present — nothing to check`);
+      }
+      const failed = calls.filter((o) => {
+        const v = navigate(o, spec.successPath);
+        return v === false || coerce(v) === "false";
+      });
+      return failed.length === 0
+        ? pass(`all ${calls.length} '${spec.tool}' call(s) report '${spec.successPath}' != false`)
+        : fail(
+            `'${spec.tool}' reports failure ('${spec.successPath}'==false) in ` +
+              `${failed.length}/${calls.length} call(s)`,
+          );
+    }
+    default: {
+      // exhaustiveness: an unknown primitive is a fail-loud error.
+      const unknown = spec as { primitive?: unknown };
+      throw new Error(
+        `runCodeEval: unknown code-eval primitive '${String(unknown.primitive)}'. ` +
+          "Known: presence | string-equality | format-validity | schema-conformance | " +
+          "ref-integrity | recovery-after-failure | tool-output-failure.",
+      );
+    }
+  }
+}

package/.claude/skills/mutagent-evaluator/scripts/codegen-evals.ts

@@ -0,0 +1,173 @@
+/**
+ * scripts/codegen-evals.ts — Path B (F14): emit CODE-WRITTEN evals in the target's
+ * own language (Type A — PURE string codegen).
+ * ---------------------------------------------------------------------------
+ * Path B of the EVAL-stage engine fork (eval-engine.ts). For a code-FRAMEWORK
+ * target (Mastra/TS, runtime=bun) this emits a SELF-CONTAINED eval-suite source
+ * file the user runs DIRECTLY — no Claude-Code sub-agents anywhere (F14):
+ *
+ *   - deterministic CODE-CHECKS    (the `codeCheck` helper — objective, no LLM)
+ *   - an LLM-JUDGE call VIA SDK     (the `judgeViaSdk` seam — provider SDK, NOT a
+ *                                    CC sub-agent; the user wires their own key)
+ *   - eval-criteria checks → a per-criterion pass/fail + a gate (runVerdict)
+ *   - the scorecard JSON written to a DISCOVERABLE output sink.
+ *
+ * The emitted code reaches back into NOTHING in `.claude/skills/` (it ships INTO
+ * the target's repo). This module is PURE string assembly — same input →
+ * byte-identical source (no clock / random / network), so the generated suite is
+ * itself reproducible. Subject-agnostic: criteria are DATA.
+ *
+ * Austerity note: this is the Path-B analog of the EV-050 in-house/export seam —
+ * it EMITS a provider-calling artifact for the user to run; the DEFAULT
+ * agent-dispatch (Path A) path never imports it.
+ */
+
+/** A criterion to codegen — the minimal Path-B slice (maps from agentspec success_criteria). */
+export interface CodegenCriterion {
+  id: string;
+  statement: string;
+  /** `code-check` → a deterministic check; `llm-judge` → an SDK judge call. */
+  type: "code-check" | "llm-judge";
+  goal: string;
+}
+
+export interface CodegenInput {
+  subject: string;
+  /** the target runtime ("bun" | "node" | "deno" …) — drives the emitted module style. */
+  runtime: string;
+  targetFramework: string;
+  criteria: CodegenCriterion[];
+  /** the discoverable sink the suite writes its scorecard to. */
+  outputSink: string;
+}
+
+/** JSON-encode a value for safe embedding as a source literal. PURE. */
+function lit(value: unknown): string {
+  return JSON.stringify(value);
+}
+
+/**
+ * Emit a portable, self-contained eval-suite SOURCE file (TS) for the target.
+ * Deterministic. THROWS on an empty criteria set or an unknown criterion type
+ * (fail-loud — never silently emit a no-op suite or skip a criterion). PURE.
+ */
+export function codegenEvalSuite(input: CodegenInput): string {
+  if (input.criteria.length === 0) {
+    throw new Error(
+      "codegenEvalSuite: no criteria — there is nothing to write evals for. " +
+        "Materialize criteria from the agentspec.evals.success_criteria first.",
+    );
+  }
+  for (const c of input.criteria) {
+    if (c.type !== "code-check" && c.type !== "llm-judge") {
+      throw new Error(
+        `codegenEvalSuite: unknown criterion type '${String((c as { type: unknown }).type)}' ` +
+          `on criterion '${c.id}'. Expected 'code-check' | 'llm-judge'.`,
+      );
+    }
+  }
+
+  // The criteria are embedded as data; the runner dispatches by `type` at runtime.
+  const criteriaLiteral = lit(
+    input.criteria.map((c) => ({ id: c.id, statement: c.statement, type: c.type, goal: c.goal })),
+  );
+
+  // NOTE: keep this a single template literal so the codegen is PURE + deterministic.
+  // It deliberately contains NO Claude-Code dispatch — the only LLM path is the
+  // user-supplied `judgeViaSdk` SDK seam (Path B portability, F14).
+  return `/**
+ * code-evals.ts — PORTABLE code-written eval suite for ${input.subject}
+ * Target: ${input.targetFramework} (runtime: ${input.runtime})
+ * GENERATED by mutagent-evaluator (Path B — code-written). Runs WITHOUT Claude Code.
+ *
+ * Wire two seams to your target:
+ *   - runAgent(query): call YOUR built ${input.targetFramework} agent → its output.
+ *   - judgeViaSdk(prompt): call YOUR provider SDK (the LLM judge). NOT a Claude-Code
+ *     sub-agent — your own key, your own SDK. Returns {result:"pass"|"fail", critique}.
+ *
+ * Then: \`${input.runtime} run code-evals.ts <dataset.json>\` → writes the scorecard to
+ *   ${input.outputSink}scorecard.json  (the discoverable sink).
+ */
+
+export interface DatasetCase { id: string; query: string; tuple?: Record<string, string>; }
+export interface CriterionVerdict { criterionId: string; result: "pass" | "fail"; critique: string; }
+export interface Scorecard {
+  subject: string;
+  perCriterion: Record<string, { pass: number; fail: number }>;
+  runVerdict: "pass" | "fail";
+  rows: { caseId: string; verdicts: CriterionVerdict[] }[];
+}
+
+/** The agentspec-derived criteria (embedded as DATA — subject-agnostic). */
+const CRITERIA: { id: string; statement: string; type: "code-check" | "llm-judge"; goal: string }[] =
+  ${criteriaLiteral};
+
+// ── seam 1: run YOUR built agent (wire this to your ${input.targetFramework} entrypoint) ──
+export type RunAgent = (query: string) => Promise<{ output: string; trace?: unknown }>;
+
+// ── seam 2: your provider-SDK LLM judge (NOT a Claude-Code sub-agent — portable) ──
+export type JudgeViaSdk = (prompt: string) => Promise<{ result: "pass" | "fail"; critique: string }>;
+
+/** Deterministic CODE check for an objective criterion (no LLM). Extend per criterion id. */
+export function codeCheck(criterionId: string, output: string): CriterionVerdict {
+  // Default code-check: non-empty, well-formed output. Specialize by criterionId as needed.
+  const ok = typeof output === "string" && output.trim().length > 0;
+  return { criterionId, result: ok ? "pass" : "fail", critique: ok ? "output present" : "empty output" };
+}
+
+/** LLM-judge a subjective criterion VIA the user's SDK seam (critique-before-verdict, binary). */
+export async function judgeCriterion(
+  judgeViaSdk: JudgeViaSdk,
+  criterion: { id: string; statement: string },
+  query: string,
+  output: string,
+): Promise<CriterionVerdict> {
+  const prompt =
+    "Critique FIRST, then a BINARY verdict. Criterion: " + criterion.statement +
+    "\\nUser query: " + query + "\\nAgent output: " + output +
+    "\\nRespond {critique, result: 'pass'|'fail'}.";
+  const { result, critique } = await judgeViaSdk(prompt);
+  return { criterionId: criterion.id, result, critique };
+}
+
+/** Run the full suite over a dataset → a gate-bearing scorecard. Portable. */
+export async function runEvalSuite(
+  dataset: DatasetCase[],
+  runAgent: RunAgent,
+  judgeViaSdk: JudgeViaSdk,
+): Promise<Scorecard> {
+  const perCriterion: Record<string, { pass: number; fail: number }> = {};
+  for (const c of CRITERIA) perCriterion[c.id] = { pass: 0, fail: 0 };
+  const rows: { caseId: string; verdicts: CriterionVerdict[] }[] = [];
+  for (const dc of dataset) {
+    const { output } = await runAgent(dc.query);
+    const verdicts: CriterionVerdict[] = [];
+    for (const c of CRITERIA) {
+      const v =
+        c.type === "code-check"
+          ? codeCheck(c.id, output)
+          : await judgeCriterion(judgeViaSdk, c, dc.query, output);
+      perCriterion[c.id][v.result] += 1;
+      verdicts.push(v);
+    }
+    rows.push({ caseId: dc.id, verdicts });
+  }
+  const anyFail = Object.values(perCriterion).some((p) => p.fail > 0);
+  return { subject: ${lit(input.subject)}, perCriterion, runVerdict: anyFail ? "fail" : "pass", rows };
+}
+
+/** CLI: \`${input.runtime} run code-evals.ts <dataset.json>\` → write the scorecard to the sink. */
+export async function main(
+  datasetPath: string,
+  runAgent: RunAgent,
+  judgeViaSdk: JudgeViaSdk,
+  writeFile: (path: string, body: string) => Promise<void>,
+  readFile: (path: string) => Promise<string>,
+): Promise<Scorecard> {
+  const dataset = JSON.parse(await readFile(datasetPath)) as DatasetCase[];
+  const scorecard = await runEvalSuite(dataset, runAgent, judgeViaSdk);
+  await writeFile(${lit(input.outputSink)} + "scorecard.json", JSON.stringify(scorecard, null, 2));
+  return scorecard;
+}
+`;
+}

package/.claude/skills/mutagent-evaluator/scripts/cold-start-project.ts

@@ -0,0 +1,151 @@
+/**
+ * scripts/cold-start-project.ts — EV-5.3 the EvalTrace → ColdTrace projection.
+ * ---------------------------------------------------------------------------
+ * The COLD front-door (`cold-start-sampler.ts`) balances a bootstrap ✓/✗ suite
+ * from MECHANICAL labels alone (no LLM judge yet). Its input is a `ColdTrace`
+ * (id + `MechanicalSignals` + dedup keys). The determiner PREP stage, however,
+ * works over `EvalTrace[]` (the projection of the handed-over UniTF export). This
+ * module is the thin, PURE adapter between the two — and the SINGLE place the
+ * cold sampler is wired into the determiner/discover path.
+ *
+ * ── HONEST-NULL projection (why the mapping looks the way it does) ────────────
+ * The `EvalTrace` contract does NOT carry a generic `ext.signals` /
+ * `ext.classification` block — the UniTF→EvalTrace projection (`unitf-to-
+ * evaltrace.ts`) FLATTENS only the fields it can ground onto the top level
+ * (`errored` · `status` · `incomplete` · `scores`) and the UniTF source `ext`
+ * itself carries just `ext.eval` (the §9.4.2 fidelity marker). So the mechanical
+ * signals are read from what ACTUALLY survives to `EvalTrace`, and every signal
+ * with no grounded source stays UNKNOWN (absent), never a fabricated `false`:
+ *
+ *   MechanicalSignals.hasError       ← EvalTrace.errored            (ERROR family)
+ *   MechanicalSignals.hasApiErrors   ← EvalTrace.status === "error" (ERROR family)
+ *   MechanicalSignals.incomplete     ← EvalTrace.incomplete         (TERMINAL family)
+ *   MechanicalSignals.hasScore/minScore ← numeric EvalTrace.scores  (SCORE family)
+ *   MechanicalSignals.hasChatFeedback/negativeReaction ← UNKNOWN — the FEEDBACK
+ *       family is not carried on EvalTrace today; omitted (never coerced false).
+ *   ColdTrace.scenario   ← UNKNOWN — no classification field on EvalTrace, so a
+ *       trace has no dedup key and is kept (never a false collapse).
+ *   ColdTrace.worthiness ← UNKNOWN — no worthiness field on EvalTrace (dedup is a
+ *       no-op without a scenario key anyway).
+ *
+ * PURE + deterministic: no clock, no random, no network. A given `EvalTrace[]`
+ * always yields the identical selection (reproducible cold suites — C-PIN-adjacent).
+ */
+import {
+  coldStartSample,
+  type ColdStartSuite,
+  type ColdTrace,
+  type MechanicalSignals,
+} from "./cold-start-sampler.ts";
+import type { EvalTrace } from "./contracts/eval-types.ts";
+
+/**
+ * Read the SCORE family off `EvalTrace.scores`. Scores are carried verbatim
+ * (`unknown[]`, Langfuse-style `{ name, value }`). Only FINITE NUMERIC `value`s
+ * count toward the cold SCORE signal; a non-numeric or absent value contributes
+ * nothing. Returns `{}` (both UNKNOWN) when there is no numeric score — honest-
+ * null, never a fabricated `hasScore:false`.
+ */
+function scoreSignals(scores: unknown[] | undefined): Pick<MechanicalSignals, "hasScore" | "minScore"> {
+  if (scores === undefined || scores.length === 0) return {};
+  let min = Number.POSITIVE_INFINITY;
+  let seen = false;
+  for (const s of scores) {
+    if (s === null || typeof s !== "object") continue;
+    const value = (s as { value?: unknown }).value;
+    if (typeof value === "number" && Number.isFinite(value)) {
+      seen = true;
+      if (value < min) min = value;
+    }
+  }
+  return seen ? { hasScore: true, minScore: min } : {};
+}
+
+/**
+ * Project one `EvalTrace` → the cold sampler's `ColdTrace`. Honest-null: only
+ * grounded signals are set; everything else is left absent (UNKNOWN). Explicit
+ * about the ERROR/TERMINAL/SCORE families it can ground and the FEEDBACK family
+ * it cannot (see the module header).
+ */
+export function projectEvalTraceToCold(trace: EvalTrace): ColdTrace {
+  const signals: MechanicalSignals = {};
+  // ERROR family — `errored` is the always-set structured flag (false = a grounded
+  // not-errored). `status === "error"` is the raw carried tri-state; surface it as
+  // the API-error signal when the source supplied it.
+  if (trace.errored !== undefined) signals.hasError = trace.errored;
+  if (trace.status === "error") signals.hasApiErrors = true;
+  // TERMINAL family — the §9.4.2 truncation marker (absent ⇒ UNKNOWN, never false).
+  if (trace.incomplete !== undefined) signals.incomplete = trace.incomplete;
+  // SCORE family — derived from any numeric platform score.
+  Object.assign(signals, scoreSignals(trace.scores));
+  // FEEDBACK family (hasChatFeedback/negativeReaction), scenario, worthiness: NOT
+  // carried on EvalTrace — deliberately omitted so they read UNKNOWN downstream.
+  return { id: trace.id, signals };
+}
+
+/** Project a batch of `EvalTrace` → `ColdTrace[]`, order-preserving + PURE. */
+export function projectColdTraces(traces: EvalTrace[]): ColdTrace[] {
+  return traces.map(projectEvalTraceToCold);
+}
+
+/**
+ * The cold-start selection metadata surfaced into the discover/determiner report
+ * (the determiner-stage `manifest.json`). `lowConfidence` is the load-bearing
+ * flag: a thin-negative cold suite MUST have its first judge pass treated as
+ * LOW-CONFIDENCE and re-balanced once real verdicts exist.
+ */
+export interface ColdStartMeta {
+  /** the sampler's target suite size (default = incoming trace count → pass-through). */
+  targetSize: number;
+  /** how many traces the batch handed to the sampler. */
+  input: number;
+  /** how many traces the cold suite selected (≤ input; = input on pass-through). */
+  selected: number;
+  passCount: number;
+  failCount: number;
+  /** the ✗-pool was too thin — treat the first cold judge pass as LOW-CONFIDENCE. */
+  lowConfidence: boolean;
+  /** the ✗-pool was rarer than its balanced half → every available fail was taken. */
+  oversampledFailPool: boolean;
+  /** BOTH non-empty pools are represented (never a success-only bootstrap suite). */
+  minBothHeld: boolean;
+}
+
+/**
+ * Wire the cold-start sampler over an `EvalTrace[]` batch and map the balanced
+ * suite BACK to the concrete `EvalTrace[]` the determiner PREP consumes. This is
+ * the SAFE call-site helper: it subsamples FIRST, so the untouched
+ * `prepDeterminerTasks` still emits exactly one determiner task per SELECTED
+ * trace. `size` defaults to the incoming count — a pass-through that only
+ * subsamples when an explicit smaller `size` is given (or the batch's own
+ * ✓/✗ imbalance triggers oversample flagging). PURE + deterministic.
+ */
+export function selectColdStartSuite(
+  traces: EvalTrace[],
+  opts: { size?: number; minBoth?: number; prevalenceFloor?: number; scoreThreshold?: number } = {},
+): { selected: EvalTrace[]; meta: ColdStartMeta; suite: ColdStartSuite } {
+  const size = opts.size ?? traces.length;
+  const suite = coldStartSample(projectColdTraces(traces), {
+    size,
+    minBoth: opts.minBoth,
+    prevalenceFloor: opts.prevalenceFloor,
+    scoreThreshold: opts.scoreThreshold,
+  });
+  // Map the selected cold reps back to their source EvalTrace, preserving the
+  // suite's (✓-then-✗) order. `filter` guards the impossible missing-id case.
+  const byId = new Map(traces.map((t) => [t.id, t] as const));
+  const selected = suite.selected
+    .map((c) => byId.get(c.id))
+    .filter((t): t is EvalTrace => t !== undefined);
+  const meta: ColdStartMeta = {
+    targetSize: size,
+    input: traces.length,
+    selected: selected.length,
+    passCount: suite.passCount,
+    failCount: suite.failCount,
+    lowConfidence: suite.lowConfidence,
+    oversampledFailPool: suite.oversampledFailPool,
+    minBothHeld: suite.minBothHeld,
+  };
+  return { selected, meta, suite };
+}