@mutagent/helix 0.1.0-alpha.14

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/edd-types.ts

@@ -0,0 +1,210 @@
+/**
+ * scripts/edd/edd-types.ts
+ * ---------------------------------------------------------------------------
+ * Shared categorical constants + TypeBox contracts for the ADL IMPROVE stage —
+ * the Eval-Driven-Development (EDD) loop (F18 closure + F19 variance-first).
+ *
+ * EDD is the ③ IMPROVE stage of the ADL: after the initial *build, the loop
+ * drives spec ↔ impl ↔ eval to full green. Two findings live here:
+ *
+ *   F19 — VARIANCE-FIRST. The FIRST focus after the build is eliminating per-case
+ *         VARIANCE (run each case ~N times; the verdict must stop flapping) BEFORE
+ *         measuring accuracy over the full dataset. "Without stabilizing variance
+ *         first, accuracy over big samples is wasted." (variance-gate.ts gates this.)
+ *
+ *   F18 — EDD CLOSURE. The evaluator (judge-only, EV-051 — it NEVER fixes) REQUESTS
+ *         an AI-Engineer to amend the Agent / AgentSpec. The request carries the
+ *         failing cases + the proposed remedy target (agentspec vs impl); the engineer
+ *         amends; the suite re-runs. (change-request.ts is the structured contract.)
+ *
+ * This module is PURE — it holds only categorical constants + TypeBox shapes. No
+ * clock / Math.random / network (coding-rules · C-PIN). The two consumer modules
+ * (variance-gate.ts · change-request.ts) import from here.
+ *
+ * Kept in lockstep with schemas/edd-change-request.schema.yaml (the human shape).
+ */
+import { type Static, Type } from "@sinclair/typebox";
+
+// ── EDD phase ordering (F19 — variance precedes accuracy) ───────────────────
+/**
+ * The ordered EDD phases. The controller enforces this order: a build is
+ * followed by VARIANCE stabilization, and ACCURACY is only measured once
+ * variance is stable. DONE is the terminal full-green state; STOPPED is a
+ * bounded non-convergence exit (max-swings / wallclock / no-improvement).
+ */
+export const EddPhase = {
+  /** the initial *build landed; the loop has not stabilized variance yet. */
+  Build: "build",
+  /** F19 — repeat-N the SAME cases; drive per-case verdict spread below threshold. */
+  Variance: "variance",
+  /** accuracy over the full dataset — ONLY entered once variance is stable. */
+  Accuracy: "accuracy",
+  /** terminal — variance-stable AND accuracy-met (full green). */
+  Done: "done",
+  /** terminal — bounded non-convergence (report the convergence delta). */
+  Stopped: "stopped",
+} as const;
+export type EddPhaseValue = (typeof EddPhase)[keyof typeof EddPhase];
+
+// ── F18 — remedy target (which artifact the engineer amends) ────────────────
+/**
+ * The remedy target the evaluator PROPOSES on a change-request. The evaluator
+ * never fixes (EV-051) — it names WHERE the fix belongs and the engineer amends:
+ *   - agentspec : the DEFINITION (system_prompt / jobs / sop / criteria). The
+ *                 def→impl cascade means the engineer re-runs *build after editing.
+ *   - impl      : the IMPLEMENTATION scaffold only (a build-faithfulness / wiring
+ *                 defect that does NOT change the spec).
+ */
+export const RemedyTarget = {
+  AgentSpec: "agentspec",
+  Impl: "impl",
+} as const;
+export type RemedyTargetValue = (typeof RemedyTarget)[keyof typeof RemedyTarget];
+
+// ── F18 — change-request lifecycle status ───────────────────────────────────
+/**
+ * The lifecycle of one change-request as it crosses the SendMessage seam. The
+ * evaluator emits `requested`; the engineer transitions it to `amended`
+ * (re-eval pending) or `rejected` (with a reason — e.g. the remedy target is
+ * wrong, or the failing cases are not reproducible against the spec).
+ */
+export const ChangeRequestStatus = {
+  Requested: "requested",
+  Amended: "amended",
+  Rejected: "rejected",
+} as const;
+export type ChangeRequestStatusValue =
+  (typeof ChangeRequestStatus)[keyof typeof ChangeRequestStatus];
+
+// ── F18 — why the EDD loop stopped (the bounded terminator) ─────────────────
+/**
+ * The observable, bounded reasons the EDD loop terminates. `full-green` is the
+ * SUCCESS terminator (variance-stable AND accuracy-met). The other three are the
+ * afkloop-legal bounds that prevent an infinite loop — each reports a convergence
+ * delta. There is NO open-ended "keep going" terminator.
+ */
+export const EddStopReason = {
+  /** SUCCESS — variance-stable AND accuracy ≥ target. */
+  FullGreen: "full-green",
+  /** bound — hit the max number of evaluator→engineer→re-eval swings. */
+  MaxSwings: "max-swings",
+  /** bound — hit the wall-clock budget. */
+  MaxWallclock: "max-wallclock",
+  /** bound — N consecutive swings with no measurable improvement. */
+  NoImprovementStreak: "no-improvement-streak",
+} as const;
+export type EddStopReasonValue =
+  (typeof EddStopReason)[keyof typeof EddStopReason];
+
+// ── F19 — per-case variance observation (one case, repeat-N verdicts) ───────
+/**
+ * One eval case run repeat-N times. `verdicts` is the per-rerun OutcomeVerdict
+ * string ("pass" | "fail" | "uncertain") so this contract stays decoupled from
+ * the evaluate.ts numeric mapping (variance-gate.ts adapts it via evalScoreVariance).
+ */
+export const CaseVarianceObservationSchema = Type.Object(
+  {
+    caseId: Type.String({ minLength: 1 }),
+    /** the criterion this spread is measured for (one case may span criteria). */
+    criterionId: Type.String({ minLength: 1 }),
+    /** the per-rerun verdicts; length === repeat-N. */
+    verdicts: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
+    /** OPTIONAL per-rerun ordered tool trajectories (drives trajectory spread). */
+    trajectories: Type.Optional(Type.Array(Type.Array(Type.String()))),
+  },
+  { $id: "CaseVarianceObservation", additionalProperties: false },
+);
+export type CaseVarianceObservation = Static<typeof CaseVarianceObservationSchema>;
+
+// ── F18 — a failing case bundled into a change-request ──────────────────────
+/**
+ * One failing (or chronically-uncertain) case the evaluator hands to the engineer.
+ * Carries the criterion, the critique (verbatim from the verdict), and ≥1 structured
+ * grounding ref {obs,path,value} — the SAME machine-checkable grounding the judge
+ * already emitted (GA-1). A failing case with NO ref is a defect (ungrounded ask).
+ */
+export const FailingCaseRefSchema = Type.Object(
+  {
+    caseId: Type.String({ minLength: 1 }),
+    criterionId: Type.String({ minLength: 1 }),
+    /** verbatim critique from the verdict that flagged this case. */
+    critique: Type.String({ minLength: 1 }),
+    /** ≥1 structured grounding ref — the evidence the critique already cites. */
+    refs: Type.Array(
+      Type.Object(
+        {
+          obs: Type.String(),
+          path: Type.String(),
+          value: Type.String(),
+        },
+        { additionalProperties: false },
+      ),
+      { minItems: 1 },
+    ),
+  },
+  { $id: "FailingCaseRef", additionalProperties: false },
+);
+export type FailingCaseRef = Static<typeof FailingCaseRefSchema>;
+
+// ── F18 — the evaluator → ai-engineer change-request (the SendMessage payload) ─
+/**
+ * The structured request the evaluator emits to the AI-Engineer over SendMessage.
+ * It is a REQUEST, never a patch — the evaluator stays judge-only (EV-051). The
+ * engineer consumes it, amends the named `remedyTarget`, and replies with a
+ * ChangeRequestResponse. `swing` is the 1-based loop iteration (drives the bound).
+ */
+export const EddChangeRequestSchema = Type.Object(
+  {
+    requestId: Type.String({ minLength: 1 }),
+    /** 1-based EDD swing index (the Nth evaluator→engineer→re-eval cycle). */
+    swing: Type.Integer({ minimum: 1 }),
+    subject: Type.String({ minLength: 1 }),
+    /** WHERE the engineer should fix — the evaluator PROPOSES, never patches. */
+    remedyTarget: Type.Union([
+      Type.Literal(RemedyTarget.AgentSpec),
+      Type.Literal(RemedyTarget.Impl),
+    ]),
+    /** the failing cases driving this request (≥1; each grounded). */
+    failingCases: Type.Array(FailingCaseRefSchema, { minItems: 1 }),
+    /** the proposed remedy in NL — a hypothesis for the engineer, not a mandate. */
+    proposedRemedy: Type.String({ minLength: 1 }),
+    status: Type.Literal(ChangeRequestStatus.Requested),
+  },
+  { $id: "EddChangeRequest", additionalProperties: false },
+);
+export type EddChangeRequest = Static<typeof EddChangeRequestSchema>;
+/** Public alias used by change-request.ts (avoids shadowing the schema const). */
+export type EddChangeRequestType = Static<typeof EddChangeRequestSchema>;
+
+// ── F18 — the ai-engineer → evaluator response ──────────────────────────────
+/**
+ * The engineer's reply after consuming a request. `amended` ⇒ the named target was
+ * edited (and, for an agentspec target, *build was re-run — def→impl cascade) and a
+ * re-eval is now warranted; `rejected` ⇒ the engineer declined (wrong target, not
+ * reproducible) WITH a reason. The loop controller reads `status` to decide the next
+ * step. There is NO "fixed it, trust me, skip re-eval" path — re-eval always follows.
+ */
+export const ChangeRequestResponseSchema = Type.Object(
+  {
+    requestId: Type.String({ minLength: 1 }),
+    status: Type.Union([
+      Type.Literal(ChangeRequestStatus.Amended),
+      Type.Literal(ChangeRequestStatus.Rejected),
+    ]),
+    /** which artifact was actually amended (echoes the request's remedyTarget). */
+    amendedTarget: Type.Optional(
+      Type.Union([
+        Type.Literal(RemedyTarget.AgentSpec),
+        Type.Literal(RemedyTarget.Impl),
+      ]),
+    ),
+    /** for an agentspec amend: was *build re-run to cascade def→impl? */
+    rebuilt: Type.Optional(Type.Boolean()),
+    /** human summary of the amendment OR the rejection reason (always present). */
+    note: Type.String({ minLength: 1 }),
+  },
+  { $id: "ChangeRequestResponse", additionalProperties: false },
+);
+export type ChangeRequestResponse = Static<typeof ChangeRequestResponseSchema>;
+/** Public alias used by change-request.ts (avoids shadowing the schema const). */
+export type ChangeRequestResponseType = Static<typeof ChangeRequestResponseSchema>;

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/variance-gate.ts

@@ -0,0 +1,186 @@
+/**
+ * scripts/edd/variance-gate.ts — F19 VARIANCE-FIRST gate.
+ * ---------------------------------------------------------------------------
+ * The EDD loop's FIRST focus after the build: eliminate per-case VARIANCE before
+ * measuring accuracy. Run each case ~N times (default 5), report the spread, and
+ * GATE — accuracy over the full dataset is only worth measuring once the verdict
+ * has stopped flapping on the SAME cases. "Without stabilizing variance first,
+ * accuracy over big samples is wasted."
+ *
+ * This module is ADDITIVE and INTEGRATION-FRIENDLY. It does NOT re-implement the
+ * variance math — it consumes the already-shipped `evalScoreVariance` from
+ * evaluate.ts (EV-054). It adds (a) the per-case gate decision, (b) the run-level
+ * roll-up, and (c) the phase-ordering check that forbids entering ACCURACY until
+ * VARIANCE is stable.
+ *
+ * PURE — no clock / Math.random / network (coding-rules · C-PIN). Same inputs →
+ * same gate, always.
+ */
+import { evalScoreVariance, trajectoryVariance } from "../evaluate.ts";
+import { OutcomeVerdict, type OutcomeVerdictValue } from "../contracts/eval-types.ts";
+import {
+  EddPhase,
+  type CaseVarianceObservation,
+  type EddPhaseValue,
+} from "./edd-types.ts";
+
+/** F19 default repeats — run each case 5× before measuring accuracy. */
+export const DEFAULT_REPEAT_N = 5;
+
+/**
+ * Default per-case variance threshold. A case is variance-STABLE iff its eval-score
+ * population variance is ≤ this. 0 = byte-identical verdict across reruns (the ideal);
+ * the small default tolerance admits a single uncertain among passes without flapping
+ * the gate. Override per subject via `VarianceGateConfig.maxVariance`.
+ */
+export const DEFAULT_MAX_VARIANCE = 0;
+
+export interface VarianceGateConfig {
+  /** required repeats per case (the gate is UNDER-SAMPLED below this). */
+  repeatN: number;
+  /** the per-case eval-score variance ceiling (≤ ⇒ stable). */
+  maxVariance: number;
+}
+
+export const DEFAULT_VARIANCE_GATE_CONFIG: VarianceGateConfig = {
+  repeatN: DEFAULT_REPEAT_N,
+  maxVariance: DEFAULT_MAX_VARIANCE,
+};
+
+/** Map an OutcomeVerdict string to the canonical enum value (validates the set). */
+function asVerdict(s: string): OutcomeVerdictValue {
+  if (s === OutcomeVerdict.Pass || s === OutcomeVerdict.Fail || s === OutcomeVerdict.Uncertain) {
+    return s;
+  }
+  // Fail-loud: an out-of-set verdict is a contract violation, not a silent pass.
+  throw new Error(`variance-gate: out-of-set verdict "${s}" (expected pass|fail|uncertain)`);
+}
+
+export interface CaseVarianceResult {
+  caseId: string;
+  criterionId: string;
+  /** the eval-score population variance across the reruns (0 = no flap). */
+  variance: number;
+  mean: number;
+  /** distinct verdicts seen across reruns (1 = no flap). */
+  distinctVerdicts: number;
+  /** how many reruns were observed (vs the required repeatN). */
+  observed: number;
+  /** false ⇒ fewer reruns than repeatN: the gate is under-sampled for this case. */
+  sufficientlySampled: boolean;
+  /** distinct ordered tool trajectories (when trajectories supplied; else 0). */
+  distinctTrajectories: number;
+  /** true iff variance ≤ maxVariance AND sufficientlySampled. */
+  stable: boolean;
+}
+
+/**
+ * Evaluate ONE case's repeat-N spread against the threshold. PURE.
+ * A case is stable iff it was sampled ≥ repeatN times AND its variance is within
+ * the ceiling. Under-sampling NEVER reads as stable (you cannot certify a spread
+ * you did not measure).
+ */
+export function gradeCaseVariance(
+  obs: CaseVarianceObservation,
+  config: VarianceGateConfig = DEFAULT_VARIANCE_GATE_CONFIG,
+): CaseVarianceResult {
+  const verdicts = obs.verdicts.map(asVerdict);
+  const sv = evalScoreVariance(verdicts);
+  const distinctVerdicts = new Set(verdicts).size;
+  const observed = verdicts.length;
+  const sufficientlySampled = observed >= config.repeatN;
+  const distinctTrajectories =
+    obs.trajectories !== undefined ? trajectoryVariance(obs.trajectories).distinct : 0;
+  const withinThreshold = sv.variance <= config.maxVariance;
+  return {
+    caseId: obs.caseId,
+    criterionId: obs.criterionId,
+    variance: sv.variance,
+    mean: sv.mean,
+    distinctVerdicts,
+    observed,
+    sufficientlySampled,
+    distinctTrajectories,
+    stable: withinThreshold && sufficientlySampled,
+  };
+}
+
+export interface VarianceGateResult {
+  /** the per-case grades (sorted by caseId then criterionId for C-PIN stability). */
+  cases: CaseVarianceResult[];
+  /** count of cases whose variance exceeds the ceiling. */
+  flapping: number;
+  /** count of cases sampled below repeatN. */
+  underSampled: number;
+  /** the worst (highest) per-case variance observed. */
+  maxVariance: number;
+  /**
+   * THE GATE: true iff EVERY case is variance-stable (within threshold AND
+   * sufficiently sampled). The accuracy phase MUST NOT be entered until this
+   * is true (F19). A run with zero cases is NOT passed (nothing was measured).
+   */
+  passed: boolean;
+}
+
+/**
+ * Roll up the per-case grades into the run-level variance gate. PURE +
+ * deterministic: the case list is sorted so the artifact is byte-identical
+ * across reruns (C-PIN). An empty case set fails the gate (you cannot certify
+ * stability you never measured).
+ */
+export function evaluateVarianceGate(
+  observations: CaseVarianceObservation[],
+  config: VarianceGateConfig = DEFAULT_VARIANCE_GATE_CONFIG,
+): VarianceGateResult {
+  const cases = observations
+    .map((o) => gradeCaseVariance(o, config))
+    .sort((a, b) =>
+      a.caseId === b.caseId
+        ? a.criterionId.localeCompare(b.criterionId)
+        : a.caseId.localeCompare(b.caseId),
+    );
+  const flapping = cases.filter((c) => c.variance > config.maxVariance).length;
+  const underSampled = cases.filter((c) => !c.sufficientlySampled).length;
+  const maxVariance = cases.reduce((m, c) => Math.max(m, c.variance), 0);
+  const passed = cases.length > 0 && cases.every((c) => c.stable);
+  return { cases, flapping, underSampled, maxVariance, passed };
+}
+
+/**
+ * F19 phase-ordering guard. Given the current phase + the variance gate result,
+ * decide whether the loop MAY advance to the accuracy phase. This encodes the
+ * hard ordering: VARIANCE must pass BEFORE ACCURACY is entered.
+ *
+ * Returns the NEXT phase. PURE — a decision function, no side effects.
+ *   - from `build`/`variance`: advance to `accuracy` ONLY if the gate passed;
+ *     otherwise stay in `variance` (keep stabilizing).
+ *   - from any other phase: returned unchanged (this guard owns only the
+ *     variance→accuracy transition; DONE/STOPPED are owned by the loop controller).
+ */
+export function nextPhaseAfterVariance(
+  current: EddPhaseValue,
+  gate: VarianceGateResult,
+): EddPhaseValue {
+  if (current === EddPhase.Build || current === EddPhase.Variance) {
+    return gate.passed ? EddPhase.Accuracy : EddPhase.Variance;
+  }
+  return current;
+}
+
+/**
+ * Hard assertion for callers that must NOT proceed to accuracy with a flapping
+ * suite. THROWS when accuracy is attempted before variance is stable (F19 — the
+ * "accuracy over big samples is wasted" guard). Use at the accuracy-run entry.
+ */
+export function assertVarianceStableBeforeAccuracy(gate: VarianceGateResult): void {
+  if (!gate.passed) {
+    const detail =
+      gate.cases.length === 0
+        ? "no cases measured"
+        : `${gate.flapping} flapping, ${gate.underSampled} under-sampled (maxVariance=${gate.maxVariance})`;
+    throw new Error(
+      `variance-gate (F19): refusing to measure accuracy before variance is stable — ${detail}. ` +
+        `Stabilize per-case variance first; accuracy over big samples is wasted otherwise.`,
+    );
+  }
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/emit-completeness.ts

@@ -0,0 +1,111 @@
+/**
+ * scripts/emit-completeness.ts — WS-1: the judge EMIT-CONTRACT completeness gate.
+ * ---------------------------------------------------------------------------
+ * The §9.4 judge walk is OPTIONAL in the schema (`present iff the judge emitted
+ * it`), so a judge that drops the structured reasoning still produces a
+ * schema-valid verdict file. That silent gap is what starved the report's
+ * Trajectory (§2) + Self-Eval (§5) tabs: walk/understanding(M2)/expectedTrajectory(M3)
+ * were 0/500 and agentSteps only 200/500, while the verdicts themselves were
+ * fully reasoned (rich critiques). This module makes the gap MACHINE-VISIBLE.
+ *
+ * It assesses, over the COMPLETE-fidelity dispatched verdicts (an INCOMPLETE trace
+ * legitimately SKIPS the per-criterion walk via the deterministic node-1 gate, so
+ * it is EXEMPT — never counted as a missing emit), how many carry each required
+ * judge-walk field. PURE + ADVISORY: it NEVER throws and NEVER alters a verdict —
+ * it returns coverage counts + a loud `warning` when a required field is wholly
+ * absent. Two consumers read the SAME numbers: the engine's aggregate log (loud
+ * assert) and the report's §5 Self-Eval "Emit-Completeness" panel (honest coverage).
+ */
+import type { MatrixVerdictFile } from "./contracts/eval-matrix.ts";
+
+/** The judge-walk fields the emit-contract REQUIRES on a complete-fidelity verdict. */
+export const EMIT_FIELDS = ["agentSteps", "understanding", "expectedTrajectory", "judgeSteps"] as const;
+export type EmitField = (typeof EMIT_FIELDS)[number];
+
+/** Per-field coverage over the eligible (complete-fidelity) dispatched verdicts. */
+export interface EmitFieldCoverage {
+  field: EmitField;
+  /** how many eligible verdicts carry a NON-EMPTY value for this field. */
+  present: number;
+  /** trajectoryIds of the eligible verdicts MISSING this field. */
+  missing: string[];
+  /** present / eligible, as a 0..100 integer (100 when there are 0 eligible). */
+  pct: number;
+}
+
+export interface EmitCompleteness {
+  /** dispatched verdicts total (complete + incomplete). */
+  total: number;
+  /** the DENOMINATOR — complete-fidelity verdicts that SHOULD carry the walk. */
+  eligible: number;
+  /** INCOMPLETE-fidelity verdicts, exempt from the walk contract (node-1 short-circuit). */
+  exemptIncomplete: number;
+  /** per-field coverage. */
+  fields: EmitFieldCoverage[];
+  /** eligible verdicts carrying ALL required fields (a fully-complete emit). */
+  completeEmits: number;
+  /** completeEmits / eligible, 0..100 integer. */
+  completePct: number;
+  /** SET (loud) when ≥1 required field is wholly absent across all eligible verdicts. */
+  warning?: string;
+}
+
+/** A verdict file is eligible for the walk contract iff its fidelity is COMPLETE. */
+function isCompleteFidelity(f: MatrixVerdictFile): boolean {
+  // fidelity is required on the schema; `complete:false` is the node-1 INCOMPLETE gate.
+  return f.fidelity?.complete !== false;
+}
+
+/** Non-empty test for one emit field on a verdict file. */
+function fieldPresent(f: MatrixVerdictFile, field: EmitField): boolean {
+  const v = (f as Record<string, unknown>)[field];
+  if (Array.isArray(v)) return v.length > 0;
+  if (field === "understanding") {
+    // M2 is an object — present iff it carries a non-empty rephrase/understanding.
+    if (v === undefined || v === null || typeof v !== "object") return false;
+    return Object.keys(v as object).length > 0;
+  }
+  return v !== undefined && v !== null;
+}
+
+/**
+ * Assess the judge emit-contract completeness over the dispatched verdict files.
+ * PURE. NEVER throws. The denominator is the COMPLETE-fidelity verdicts (INCOMPLETE
+ * traces are exempt — they short-circuit the walk by design).
+ */
+export function assessEmitCompleteness(files: MatrixVerdictFile[]): EmitCompleteness {
+  const total = files.length;
+  const eligibleFiles = files.filter(isCompleteFidelity);
+  const exemptIncomplete = total - eligibleFiles.length;
+  const eligible = eligibleFiles.length;
+
+  const fields: EmitFieldCoverage[] = EMIT_FIELDS.map((field) => {
+    const missing = eligibleFiles.filter((f) => !fieldPresent(f, field)).map((f) => f.trajectoryId);
+    const present = eligible - missing.length;
+    return { field, present, missing, pct: eligible > 0 ? Math.round((100 * present) / eligible) : 100 };
+  });
+
+  const completeEmits = eligibleFiles.filter((f) => EMIT_FIELDS.every((field) => fieldPresent(f, field))).length;
+  const completePct = eligible > 0 ? Math.round((100 * completeEmits) / eligible) : 100;
+
+  const result: EmitCompleteness = {
+    total,
+    eligible,
+    exemptIncomplete,
+    fields,
+    completeEmits,
+    completePct,
+  };
+
+  const whollyAbsent = fields.filter((f) => eligible > 0 && f.present === 0).map((f) => f.field);
+  if (whollyAbsent.length > 0) {
+    result.warning =
+      `JUDGE EMIT-CONTRACT GAP: ${whollyAbsent.join(", ")} absent on ALL ${eligible} complete-fidelity ` +
+      `verdict(s) — the §9.4 judge walk was dropped. The Trajectory (§2) + Self-Eval (§5) tabs read ` +
+      `these as starved. The judge (#mode-judge-trajectory) MUST emit M2 understanding, M3 ` +
+      `expectedTrajectory, agentSteps and judgeSteps on every complete-fidelity trajectory ` +
+      `(assets/agents/evaluator.md §9.4 walk). agentSteps is factual trace data — reconstruct it ` +
+      `deterministically when the judge omits it; M2/M3/judgeSteps are judge reasoning — resend on absence.`;
+  }
+  return result;
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/eval-engine.ts

@@ -0,0 +1,160 @@
+/**
+ * scripts/eval-engine.ts — the ADL EVAL-stage ENGINE FORK resolver (Type A — PURE).
+ * ---------------------------------------------------------------------------
+ * F7 / F9 / F14. When the ADL `*build` stage hands the evaluator a freshly-built
+ * agent, `*build-evals` / `*evaluate` must ASK which eval engine to implement:
+ *
+ *   - Path A `native-matrix` — mutagent NATIVE eval-matrix + LLM-judge SUB-AGENTS.
+ *     HARD DEPENDENCY on a Claude-Code host (to spawn the judges). For a CODE
+ *     framework target (Mastra/TS) it ALSO needs the built agent to EMIT
+ *     logs/traces to a known sink for the judge to read (F9). The CC dependency +
+ *     sink requirement are SURFACED in the menu (F7) — never discovered late.
+ *   - Path B `code-written` — evals written in the TARGET'S OWN language
+ *     (bun/TS for Mastra): deterministic code-checks + an LLM-judge call VIA SDK +
+ *     eval-criteria checks. PORTABLE — no CC sub-agents, so a non-CC user is not
+ *     trapped in the matrix (F14).
+ *
+ * The fork is TARGET-CONDITIONAL (F9): a code FRAMEWORK target may pick EITHER
+ * engine; a `harness:*` target is native-only (there is no target language to
+ * write portable evals into). This module is PURE DATA + gates — the
+ * AskUserQuestion prompt and the actual codegen live elsewhere
+ * (codegen-evals.ts). Subject-agnostic; no clock / random / network.
+ */
+import {
+  EvalEngine,
+  type EvalEngineValue,
+  type EvalEngineOption,
+  type EvalEnginePlan,
+  type EngineTargetInput,
+} from "./contracts/eval-engine.ts";
+
+const HARNESS_PREFIX = "harness:";
+
+/**
+ * Is the target a coding FRAMEWORK (mastra · langgraph · pydantic-ai · …) rather
+ * than a `harness:*` runtime? A framework has a target LANGUAGE we can write
+ * portable code-written evals into; a harness does not. PURE.
+ */
+export function isFrameworkTarget(target: EngineTargetInput): boolean {
+  return !target.targetFramework.startsWith(HARNESS_PREFIX);
+}
+
+/**
+ * The discoverable OUTPUT SINK an engine writes (and, for native-on-framework,
+ * the sink the built agent must EMIT to). Always relative to the host project
+ * root — the discoverability success-gate ("outputs land in a discoverable
+ * sink"). PURE.
+ */
+export function defaultOutputSink(engine: EvalEngineValue, target: EngineTargetInput): string {
+  if (engine === EvalEngine.CodeWritten) {
+    // Path B writes the eval suite + its results in the target's own tree.
+    return ".mutagent/evaluator/code-evals/";
+  }
+  // Path A. On a CODE framework the built agent emits logs/traces HERE for the
+  // matrix judge to read; on a harness target the judge reads CC transcripts
+  // directly, but the verdicts + scorecard still land under the run dir.
+  return isFrameworkTarget(target)
+    ? ".mutagent/evaluator/agent-logs/"
+    : ".mutagent/evaluator/runs/";
+}
+
+/**
+ * The target-conditional ENGINE MENU (F9). A code framework offers BOTH paths
+ * (Path A + Path B); a `harness:*` target is native-only. The Claude-Code
+ * dependency + the log-sink requirement are SURFACED on each option (F7) so the
+ * picker never silently traps a user in the matrix. Deterministic order:
+ * code-written first (the recommended, portable default for a framework), then
+ * native-matrix. PURE.
+ */
+export function chooseEvalEngineOptions(target: EngineTargetInput): EvalEngineOption[] {
+  const framework = isFrameworkTarget(target);
+  const nativeNeedsSink = framework; // a code framework must emit to a sink (F9)
+  const nativeOption: EvalEngineOption = {
+    engine: EvalEngine.NativeMatrix,
+    label: "Path A — mutagent native eval-matrix (LLM-judge sub-agents)",
+    summary: framework
+      ? "Eval-matrix judged by Claude-Code sub-agents. Requires a Claude-Code host AND the " +
+        "built agent to emit logs/traces to a known sink for the judge to read."
+      : "Eval-matrix judged by Claude-Code sub-agents reading the harness session transcripts. " +
+        "Requires a Claude-Code host.",
+    requiresClaudeCode: true,
+    requiresLogSink: nativeNeedsSink,
+    portable: false,
+    recommended: !framework, // native is the only (thus default) choice for a harness
+  };
+  if (!framework) {
+    // harness:* — native-only (no target language for portable code-written evals).
+    return [nativeOption];
+  }
+  const codeOption: EvalEngineOption = {
+    engine: EvalEngine.CodeWritten,
+    label: `Path B — code-written evals in ${target.runtime}/${target.targetFramework}`,
+    summary:
+      "Evals written in the target's own language: deterministic code-checks + an LLM-judge " +
+      "call via SDK + eval-criteria checks. Portable — runs WITHOUT Claude Code.",
+    requiresClaudeCode: false,
+    requiresLogSink: false,
+    portable: true,
+    recommended: true, // portability-first default for a code framework
+  };
+  return [codeOption, nativeOption];
+}
+
+/**
+ * Gate: an engine choice must MATCH the target (F9). `code-written` needs a
+ * target LANGUAGE — it is rejected on a `harness:*` target. `native-matrix`
+ * matches any target. THROWS (fail-loud, no silent re-route). PURE.
+ */
+export function assertEngineMatchesTarget(
+  engine: EvalEngineValue,
+  target: EngineTargetInput,
+): void {
+  if (engine === EvalEngine.CodeWritten && !isFrameworkTarget(target)) {
+    throw new Error(
+      `assertEngineMatchesTarget: code-written evals require a code-FRAMEWORK target ` +
+        `with a writable language, but target_framework='${target.targetFramework}' is a ` +
+        `harness target. A harness target is native-matrix only (Path A). ` +
+        `Choose native-matrix or re-target to a code framework.`,
+    );
+  }
+}
+
+/**
+ * Resolve a user's engine choice into the RESOLVED plan `*evaluate` runs against.
+ * Surfaces the Path-A dependencies (Claude Code + log sink) explicitly. Unknown
+ * engine → THROW (no silent default). PURE.
+ */
+export function resolveEvalEngine(
+  choice: EvalEngineValue,
+  target: EngineTargetInput,
+): EvalEnginePlan {
+  if (choice !== EvalEngine.NativeMatrix && choice !== EvalEngine.CodeWritten) {
+    throw new Error(
+      `resolveEvalEngine: unknown eval engine '${String(choice)}'. ` +
+        `Choose one of {${EvalEngine.NativeMatrix}, ${EvalEngine.CodeWritten}}.`,
+    );
+  }
+  assertEngineMatchesTarget(choice, target);
+  const framework = isFrameworkTarget(target);
+  const requiresClaudeCode = choice === EvalEngine.NativeMatrix;
+  const requiresLogSink = choice === EvalEngine.NativeMatrix && framework;
+  const outputSink = defaultOutputSink(choice, target);
+  const rationale =
+    choice === EvalEngine.CodeWritten
+      ? `Code-written evals in ${target.runtime}/${target.targetFramework} — portable, ` +
+        `runs without Claude Code (F14). Results land in ${outputSink}.`
+      : framework
+        ? `Native eval-matrix judged by Claude-Code sub-agents. The built agent MUST emit ` +
+          `logs/traces to ${outputSink} for the judge to read (F9).`
+        : `Native eval-matrix judged by Claude-Code sub-agents reading the harness ` +
+          `session transcripts directly.`;
+  return {
+    engine: choice,
+    target,
+    targetIsFramework: framework,
+    requiresClaudeCode,
+    requiresLogSink,
+    outputSink,
+    rationale,
+  };
+}