@mutagent/evaluator 0.1.0-alpha.2

package/.claude/skills/mutagent-evaluator/scripts/config/schema.ts

@@ -0,0 +1,325 @@
+/**
+ * scripts/config/schema.ts — the evaluator's FIRST config contract (v0.2.0).
+ * ---------------------------------------------------------------------------
+ * PORTED IN, standalone. The evaluator reads the UNIFIED local
+ * `.mutagent/config.yaml` (operator decision 2026-06-29): ONE file drives the
+ * orchestrator + every skill. This module TypeBox-types the two blocks the
+ * evaluator reads:
+ *
+ *   global.sources[]   — the SOURCE catalog (where traces live). Bound BY ROLE
+ *                        (the evaluator is a source-CONSUMER → it binds
+ *                        `global.sources`). A single entry auto-binds; multiple
+ *                        ⇒ disambiguation is deferred (Fork A).
+ *   global.models      — { default, judge_model }. `judge_model` (renamed from
+ *                        `pinned_judge` in v0.2.0) is the C-PIN pinned judge.
+ *   lifecycle.evaluator — the evaluator's OWN section: `context[]` (stage-
+ *                        specific links) + `judge_runtime` (renamed from
+ *                        `substrate` in v0.2.0 — HOW judges run).
+ *
+ * SEALED-SIBLING (coding-rules): the source/target SHAPE is a PARITY PORT of
+ * mutagent-orchestrator's `scripts/config-schema.ts` GlobalSourceSchema /
+ * GlobalTargetSchema — NEVER cross-imported (like resolve-credential.ts). A
+ * parity test asserts the shapes stay in lockstep.
+ *
+ * SHAPE vs COMPLETENESS (mirrors the orchestrator): this schema enforces SHAPE
+ * only. Sub-fields of `global` are OPTIONAL so a partial, mid-onboarding config
+ * still validates structurally. The per-skill `lifecycle.evaluator` section is
+ * OPEN (additionalProperties:true) beyond the typed fields — the evaluator owns
+ * the rest of its knobs; the orchestrator types only the gate-relevant ones.
+ *
+ * Determinism: pure schema + a `Value.Check` validator. No clock, no random, no
+ * network. The loader (config/load.ts) reads an INJECTED path; the pure core
+ * never resolves a path on its own (tests run against committed fixtures).
+ *
+ * NO raw secrets: a `credential_ref` is an env-var NAME (or `{env,path}`),
+ * resolved at use-time via resolve-credential.ts (env → .env → .mutagentrc).
+ */
+import { type Static, Type } from "@sinclair/typebox";
+import { Value } from "@sinclair/typebox/value";
+
+/** The FROZEN config-contract version this loader understands. */
+export const CONFIG_VERSION = "0.2.0" as const;
+
+/** Legacy version(s) whose configs must NOT be parsed at runtime (hard-cut). */
+export const LEGACY_CONFIG_VERSIONS = ["0.1.0"] as const;
+
+// ── Categorical constants (no magic strings — coding-rules) ──────────────────
+
+/** Source platforms a `global.sources[]` entry can pull traces from. */
+export const SourcePlatform = {
+  Langfuse: "langfuse",
+  Otel: "otel",
+  LocalJsonl: "local-jsonl",
+  ClaudeCode: "claude-code",
+  Codex: "codex",
+} as const;
+export type SourcePlatformValue =
+  (typeof SourcePlatform)[keyof typeof SourcePlatform];
+
+/** Normalizer format hint for a source (how to parse the records). */
+export const SourceFormat = {
+  LangfuseExport: "langfuse-export",
+  ClaudeCode: "claude-code",
+  Codex: "codex",
+  Raw: "raw",
+} as const;
+export type SourceFormatValue = (typeof SourceFormat)[keyof typeof SourceFormat];
+
+/** Latency-unit override for a source (`auto` = infer from the record). */
+export const LatencyUnit = {
+  Auto: "auto",
+  Ms: "ms",
+  S: "s",
+} as const;
+export type LatencyUnitValue = (typeof LatencyUnit)[keyof typeof LatencyUnit];
+
+/** Target platforms a `global.targets[]` entry can write a fix TO. */
+export const TargetPlatform = {
+  LocalClaude: "local-claude",
+  LocalCodex: "local-codex",
+  LocalCursor: "local-cursor",
+  LocalOpencode: "local-opencode",
+  LocalMastra: "local-mastra",
+  LocalCloudAgentSdk: "local-cloud-agent-sdk",
+  CloudRest: "cloud-rest",
+  ReportOnly: "report-only",
+} as const;
+export type TargetPlatformValue =
+  (typeof TargetPlatform)[keyof typeof TargetPlatform];
+
+/** Whether a target's writes go through local files (worktree) or a remote API. */
+export const TargetMode = {
+  Local: "local",
+  Remote: "remote",
+} as const;
+export type TargetModeValue = (typeof TargetMode)[keyof typeof TargetMode];
+
+/** HOW a fix is applied to a target (`report-only` ⇒ no target write). */
+export const ApplyKind = {
+  CodePr: "code-pr",
+  Markdown: "markdown",
+  CloudDeploy: "cloud-deploy",
+  ReportOnly: "report-only",
+} as const;
+export type ApplyKindValue = (typeof ApplyKind)[keyof typeof ApplyKind];
+
+// ── Shared building-block schemas (PARITY PORT — orchestrator config-schema) ──
+
+/**
+ * A credential REFERENCE — an ENV-VAR NAME (string) or `{ env, path? }`. NEVER a
+ * raw secret value (mirrors resolve-credential.ts CredentialRef; no cross-import).
+ */
+export const CredentialRefSchema = Type.Union([
+  Type.String({ minLength: 1 }),
+  Type.Object(
+    {
+      env: Type.String({ minLength: 1 }),
+      path: Type.Optional(Type.String({ minLength: 1 })),
+    },
+    { additionalProperties: false },
+  ),
+]);
+export type CredentialRefValue = Static<typeof CredentialRefSchema>;
+
+/**
+ * A PROJECT-WIDE (or stage-specific) context link — a pointer to a supplementary
+ * doc with WHAT / WHY / WHEN. All four fields REQUIRED (a link with no rationale
+ * is noise).
+ */
+export const ContextLinkSchema = Type.Object(
+  {
+    path: Type.String({ minLength: 1 }),
+    what: Type.String({ minLength: 1 }),
+    why: Type.String({ minLength: 1 }),
+    when: Type.String({ minLength: 1 }),
+  },
+  { additionalProperties: false },
+);
+export type ContextLink = Static<typeof ContextLinkSchema>;
+
+/** The default model + the pinned judge model (`judge_model` — was `pinned_judge`). */
+export const ModelsSchema = Type.Object(
+  {
+    default: Type.Optional(Type.String({ minLength: 1 })),
+    judge_model: Type.Optional(Type.String({ minLength: 1 })),
+  },
+  { additionalProperties: false },
+);
+export type Models = Static<typeof ModelsSchema>;
+
+/**
+ * One SOURCE catalog entry — where traces live. PARITY PORT of the orchestrator's
+ * GlobalSourceSchema. `paths` (file sources) is mutually exclusive with `endpoint`
+ * (remote); the schema tolerates both — onboarding cares.
+ */
+export const GlobalSourceSchema = Type.Object(
+  {
+    name: Type.String({ minLength: 1 }),
+    platform: Type.Union([
+      Type.Literal(SourcePlatform.Langfuse),
+      Type.Literal(SourcePlatform.Otel),
+      Type.Literal(SourcePlatform.LocalJsonl),
+      Type.Literal(SourcePlatform.ClaudeCode),
+      Type.Literal(SourcePlatform.Codex),
+    ]),
+    project: Type.Optional(Type.String({ minLength: 1 })),
+    endpoint: Type.Optional(Type.String({ minLength: 1 })),
+    credential_ref: Type.Optional(CredentialRefSchema),
+    paths: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
+    format: Type.Optional(
+      Type.Union([
+        Type.Literal(SourceFormat.LangfuseExport),
+        Type.Literal(SourceFormat.ClaudeCode),
+        Type.Literal(SourceFormat.Codex),
+        Type.Literal(SourceFormat.Raw),
+      ]),
+    ),
+    agent_field: Type.Optional(Type.String({ minLength: 1 })),
+    latency_unit: Type.Optional(
+      Type.Union([
+        Type.Literal(LatencyUnit.Auto),
+        Type.Literal(LatencyUnit.Ms),
+        Type.Literal(LatencyUnit.S),
+      ]),
+    ),
+    // PARITY (orchestrator SourceSchema): the catalog's DEFAULT pick when >1 entry
+    // exists — the selection contract's silent-bind marker (exactly one `default:true`
+    // ⇒ auto-bind, skipping the disambiguation prompt). Optional: absent ⇒ no default.
+    default: Type.Optional(Type.Boolean()),
+  },
+  { additionalProperties: false },
+);
+export type GlobalSource = Static<typeof GlobalSourceSchema>;
+
+/** A precise link to an Agent / Tooling definition file in a code target. */
+export const CodeRefSchema = Type.Object(
+  {
+    path: Type.String({ minLength: 1 }),
+    why: Type.String({ minLength: 1 }),
+  },
+  { additionalProperties: false },
+);
+export type CodeRef = Static<typeof CodeRefSchema>;
+
+/** HOW a fix is applied for a target. `kind` gates the report-only branch. */
+export const ApplySchema = Type.Object(
+  {
+    kind: Type.Union([
+      Type.Literal(ApplyKind.CodePr),
+      Type.Literal(ApplyKind.Markdown),
+      Type.Literal(ApplyKind.CloudDeploy),
+      Type.Literal(ApplyKind.ReportOnly),
+    ]),
+    versioning: Type.Optional(Type.Boolean()),
+    pr: Type.Optional(Type.Boolean()),
+  },
+  { additionalProperties: false },
+);
+export type Apply = Static<typeof ApplySchema>;
+
+/**
+ * One TARGET catalog entry — where agent defs live + how a fix is applied. PARITY
+ * PORT of the orchestrator's GlobalTargetSchema. The evaluator is JUDGE-ONLY
+ * (EV-051) so it never WRITES a target; it ports the shape for parity + so the
+ * loader can validate a full `global` block without rejecting a present target.
+ */
+export const GlobalTargetSchema = Type.Object(
+  {
+    name: Type.String({ minLength: 1 }),
+    platform: Type.Union([
+      Type.Literal(TargetPlatform.LocalClaude),
+      Type.Literal(TargetPlatform.LocalCodex),
+      Type.Literal(TargetPlatform.LocalCursor),
+      Type.Literal(TargetPlatform.LocalOpencode),
+      Type.Literal(TargetPlatform.LocalMastra),
+      Type.Literal(TargetPlatform.LocalCloudAgentSdk),
+      Type.Literal(TargetPlatform.CloudRest),
+      Type.Literal(TargetPlatform.ReportOnly),
+    ]),
+    mode: Type.Union([
+      Type.Literal(TargetMode.Local),
+      Type.Literal(TargetMode.Remote),
+    ]),
+    root: Type.Optional(Type.String({ minLength: 1 })),
+    rest_base_url: Type.Optional(Type.String({ minLength: 1 })),
+    repo_url: Type.Optional(Type.String({ minLength: 1 })),
+    code_refs: Type.Optional(Type.Array(CodeRefSchema)),
+    credential_ref: Type.Optional(CredentialRefSchema),
+    apply: ApplySchema,
+    // PARITY (orchestrator TargetSchema): the catalog's DEFAULT pick when >1 entry
+    // exists. The evaluator is JUDGE-ONLY (EV-051) and never binds a target, but the
+    // shape stays in lockstep with the orchestrator (parity test). Optional.
+    default: Type.Optional(Type.Boolean()),
+  },
+  { additionalProperties: false },
+);
+export type GlobalTarget = Static<typeof GlobalTargetSchema>;
+
+/**
+ * The GLOBAL cross-cutting resources the evaluator reads. Sub-fields OPTIONAL (a
+ * partial mid-onboarding config still validates). The evaluator only READS
+ * `models` + `sources` (+ `targets` for shape parity); the orchestrator owns the
+ * rest of `global` (providers/workspace/brand/context) — passed through here as
+ * OPEN so a full unified config doesn't fail the evaluator's structural check.
+ */
+export const GlobalSchema = Type.Object(
+  {
+    models: Type.Optional(ModelsSchema),
+    context: Type.Optional(Type.Array(ContextLinkSchema)),
+    sources: Type.Optional(Type.Array(GlobalSourceSchema)),
+    targets: Type.Optional(Type.Array(GlobalTargetSchema)),
+  },
+  // OPEN: the orchestrator-owned global fields (providers/workspace/brand) pass
+  // through — the evaluator doesn't type them but must not reject them.
+  { additionalProperties: true },
+);
+export type Global = Static<typeof GlobalSchema>;
+
+// ── `lifecycle.evaluator` — the evaluator's OWN section ───────────────────────
+
+/**
+ * `lifecycle.evaluator` — the evaluator's config section. The typed fields:
+ *   context[]      — stage-specific supplementary context links (eval rubric).
+ *   judge_runtime  — HOW judges run (renamed from `substrate` in v0.2.0):
+ *                    agent-dispatch (DEFAULT) | in-house | code-based | user-framework.
+ * OPEN (additionalProperties:true): the evaluator owns any further knobs
+ * (subject profile path, dataset opts, …) — passthrough, validated elsewhere.
+ */
+export const LifecycleEvaluatorSchema = Type.Object(
+  {
+    context: Type.Optional(Type.Array(ContextLinkSchema)),
+    judge_runtime: Type.Optional(Type.String({ minLength: 1 })),
+  },
+  { additionalProperties: true },
+);
+export type LifecycleEvaluator = Static<typeof LifecycleEvaluatorSchema>;
+
+// ── Validation ────────────────────────────────────────────────────────────────
+
+export interface ValidationResult {
+  ok: boolean;
+  /** Human-readable error strings (path: message), empty when ok. */
+  errors: string[];
+}
+
+/** Validate a value against the evaluator's `global` block. Pure; never throws. */
+export function validateGlobal(obj: unknown): ValidationResult {
+  const errors: string[] = [];
+  if (!Value.Check(GlobalSchema, obj)) {
+    for (const e of Value.Errors(GlobalSchema, obj)) {
+      errors.push(`${e.path === "" ? "/" : e.path}: ${e.message}`);
+    }
+  }
+  return { ok: errors.length === 0, errors };
+}
+
+/** Validate a value against `lifecycle.evaluator`. Pure; never throws. */
+export function validateLifecycleEvaluator(obj: unknown): ValidationResult {
+  const errors: string[] = [];
+  if (!Value.Check(LifecycleEvaluatorSchema, obj)) {
+    for (const e of Value.Errors(LifecycleEvaluatorSchema, obj)) {
+      errors.push(`${e.path === "" ? "/" : e.path}: ${e.message}`);
+    }
+  }
+  return { ok: errors.length === 0, errors };
+}

package/.claude/skills/mutagent-evaluator/scripts/contracts/agentspec-evals.ts

@@ -0,0 +1,85 @@
+/**
+ * scripts/contracts/agentspec-evals.ts — the MINIMAL agentspec.evals slice the
+ * EVAL stage consumes (standalone — NEVER imports the agentspec skill's schema).
+ * ---------------------------------------------------------------------------
+ * The ADL `*build` stage emits an agentspec whose `definition.evals` block carries
+ * the SEED material the evaluator materializes into a real dataset (F8) + criteria:
+ *
+ *   - dataset_categories[] — `{ id, description, edge_cases[] }` — the golden
+ *     eval-suite slices + the explicit edge_cases each must exercise (the dataset
+ *     DEFINITION the spec hands forward; "seed, don't duplicate").
+ *   - scenarios[]          — `{ id, description, expected_behavior, category?,
+ *     edge_case? }` — representative situations + the correct behavior (extra seed
+ *     material, optionally tagged into a category).
+ *   - success_criteria[]   — `{ id, criterion, type, goal }` — binary-actionable
+ *     pass/fail criteria (type ∈ llm-judge | code-check).
+ *
+ * We DECLARE only the fields we read, with `additionalProperties: true` on the
+ * objects so a richer agentspec still parses (forward-compatible). The cross-skill
+ * import ban (coding-rules "Sealed-Sibling" + standalone discipline) is why this is
+ * a local re-declaration, not an import. PURE — no clock / random / network.
+ */
+import { type Static, Type } from "@sinclair/typebox";
+import { Value } from "@sinclair/typebox/value";
+
+/** A dataset CATEGORY slice + the edge-cases it must exercise. */
+export const AgentspecDatasetCategorySchema = Type.Object(
+  {
+    id: Type.String({ minLength: 1 }),
+    description: Type.String({ minLength: 1 }),
+    edge_cases: Type.Array(Type.String()),
+  },
+  { additionalProperties: true },
+);
+export type AgentspecDatasetCategory = Static<typeof AgentspecDatasetCategorySchema>;
+
+/** A representative SCENARIO (the situation + correct behavior). */
+export const AgentspecScenarioSchema = Type.Object(
+  {
+    id: Type.String({ minLength: 1 }),
+    description: Type.String({ minLength: 1 }),
+    expected_behavior: Type.String({ minLength: 1 }),
+    category: Type.Optional(Type.String()),
+    edge_case: Type.Optional(Type.Boolean()),
+  },
+  { additionalProperties: true },
+);
+export type AgentspecScenario = Static<typeof AgentspecScenarioSchema>;
+
+/** A binary-actionable SUCCESS CRITERION. */
+export const AgentspecSuccessCriterionSchema = Type.Object(
+  {
+    id: Type.String({ minLength: 1 }),
+    criterion: Type.String({ minLength: 1 }),
+    type: Type.Union([Type.Literal("llm-judge"), Type.Literal("code-check")]),
+    goal: Type.String({ minLength: 1 }),
+  },
+  { additionalProperties: true },
+);
+export type AgentspecSuccessCriterion = Static<typeof AgentspecSuccessCriterionSchema>;
+
+/** The `definition.evals` slice. Each array may be empty structurally. */
+export const AgentspecEvalsSchema = Type.Object(
+  {
+    success_criteria: Type.Array(AgentspecSuccessCriterionSchema),
+    scenarios: Type.Array(AgentspecScenarioSchema),
+    dataset_categories: Type.Array(AgentspecDatasetCategorySchema),
+  },
+  { additionalProperties: true },
+);
+export type AgentspecEvals = Static<typeof AgentspecEvalsSchema>;
+
+/**
+ * Parse + narrow the agentspec.evals slice (guarded). THROWS on schema violation —
+ * a malformed seed must never silently reach materialization. PURE.
+ */
+export function parseAgentspecEvals(value: unknown): AgentspecEvals {
+  if (!Value.Check(AgentspecEvalsSchema, value)) {
+    const first = [...Value.Errors(AgentspecEvalsSchema, value)][0];
+    throw new Error(
+      `parseAgentspecEvals: schema violation at '${first?.path ?? "(root)"}': ` +
+        `${first?.message ?? "invalid agentspec evals slice"}`,
+    );
+  }
+  return value;
+}

package/.claude/skills/mutagent-evaluator/scripts/contracts/dataset.ts

@@ -0,0 +1,149 @@
+/**
+ * scripts/contracts/dataset.ts — the W2 DATASET data contract.
+ * ---------------------------------------------------------------------------
+ * The executable (TypeBox) companion to `schemas/dataset.schema.yaml`. Declares
+ * the shapes for `*build-dataset` (EV-046, synthetic) + `*discover-dataset`
+ * (EV-047, distilled-from-traces) + the `living-suite` (EV-053) append target:
+ *
+ *   - `Dimension`     — one axis of variation (generate-synthetic-data Step 1).
+ *   - `DatasetTuple`  — one combination of dimension→value (Step 2).
+ *   - `DatasetCase`   — a tuple + its natural-language query + provenance.
+ *   - `Dataset`       — the subject's growing case set (append-only, EV-053).
+ *
+ * Strict austerity: Type A — data contract only. NO judge prompt, NO LLM
+ * reasoning, NO pass/fail decision. The LLM tuple/query GENERATION lives in
+ * `assets/agents/dataset-builder.md`; this file only declares + validates the
+ * DATA. PURE — no clock / random / network.
+ *
+ * Disjointness (W2-OWN): NEW W2 contract file — does NOT touch the shared
+ * `contracts/eval-types.ts`. Subject-agnostic (EV-002): dimensions + values are
+ * DATA carried here, never hard-coded engine constants.
+ */
+import { type Static, Type } from "@sinclair/typebox";
+import { Value } from "@sinclair/typebox/value";
+
+// ── Dimension — one axis of variation (target a failure-prone region) ────────
+export const DimensionSchema = Type.Object(
+  {
+    name: Type.String({ minLength: 1 }),
+    description: Type.Optional(Type.String()),
+    /** the discrete values this dimension can take (≥1). */
+    values: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
+  },
+  { additionalProperties: false },
+);
+export type Dimension = Static<typeof DimensionSchema>;
+
+// ── DatasetTuple — one combination of dimension→value ────────────────────────
+//
+// A flat map (dimension name → chosen value). Canonicalized for dedup by
+// `tupleKey` (sorted dimension names) so two tuples with the same assignments
+// in any insertion order collapse to one.
+export const DatasetTupleSchema = Type.Record(Type.String({ minLength: 1 }), Type.String(), {
+  minProperties: 1,
+});
+export type DatasetTuple = Static<typeof DatasetTupleSchema>;
+
+// ── Case provenance + source ─────────────────────────────────────────────────
+export const CaseSource = {
+  /** a user-confirmed seed tuple (the ~10 HITL interview). */
+  Seed: "seed",
+  /** LLM-generated by the dataset-builder agent (synthetic extension). */
+  Synthetic: "synthetic",
+  /** distilled from a real past trace (`*discover-dataset`, EV-047). */
+  Derived: "derived",
+} as const;
+export type CaseSourceValue = (typeof CaseSource)[keyof typeof CaseSource];
+
+// ── DatasetCase — a tuple + its NL query + provenance ────────────────────────
+export const DatasetCaseSchema = Type.Object(
+  {
+    /** deterministic stable id (content-derived; never a clock/random). */
+    id: Type.String({ minLength: 1 }),
+    /** the dimension assignment this case realizes. */
+    tuple: DatasetTupleSchema,
+    /** the natural-language query (Step 4 — from the dataset-builder agent). */
+    query: Type.String({ minLength: 1 }),
+    source: Type.Union([
+      Type.Literal(CaseSource.Seed),
+      Type.Literal(CaseSource.Synthetic),
+      Type.Literal(CaseSource.Derived),
+    ]),
+    /** optional pointer back to the originating trace (for `derived` cases). */
+    originTraceId: Type.Optional(Type.String()),
+    /**
+     * DETERMINISTIC provenance — which EV-052 regression selector nominated this
+     * case ("the data link"). One of: failure-driven (a confirmed ✗ catastrophe) ·
+     * outlier (an edge metric) · uncertainty (a brittle boundary call) · stratified
+     * (a known-✓ anchor). Code-derived, byte-identical on re-derive. Optional so
+     * non-derived (seed/synthetic) cases omit it.
+     */
+    selectedBy: Type.Optional(Type.String()),
+    /**
+     * The HUMAN-READABLE "why is this item high-value as a held-out test" — a
+     * JUDGMENT authored by whoever SELECTS the case (the discover/judge agent),
+     * not a code derivation. Explains what failure mode / uncertainty / coverage
+     * gap makes this trace worth keeping in the suite forever. Grounded in the
+     * originating trace. Optional (populated at selection time).
+     */
+    rationale: Type.Optional(Type.String()),
+  },
+  { additionalProperties: false },
+);
+export type DatasetCase = Static<typeof DatasetCaseSchema>;
+
+// ── Dataset — the subject's growing case set (append-only, EV-053) ───────────
+export const DatasetSchema = Type.Object(
+  {
+    /** subject display name (from the subject profile — subject-agnostic). */
+    subject: Type.String({ minLength: 1 }),
+    /** the dimensions this dataset varies over. */
+    dimensions: Type.Array(DimensionSchema),
+    /** the cases (append-only; monotonic-growth enforced by living-suite). */
+    cases: Type.Array(DatasetCaseSchema),
+    /** monotonically-increasing version (bumped on every append). */
+    version: Type.Integer({ minimum: 0 }),
+  },
+  { additionalProperties: false },
+);
+export type Dataset = Static<typeof DatasetSchema>;
+
+/** Parse a dataset file (raw JSON). THROWS on schema violation. PURE. */
+export function parseDataset(raw: string): Dataset {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    throw new Error(`parseDataset: not valid JSON: ${raw.slice(0, 120)}`);
+  }
+  if (!Value.Check(DatasetSchema, parsed)) {
+    const first = [...Value.Errors(DatasetSchema, parsed)][0];
+    throw new Error(
+      `parseDataset: schema violation at '${first?.path ?? "?"}': ` +
+        `${first?.message ?? "invalid Dataset"}`,
+    );
+  }
+  return parsed;
+}
+
+/** Assert a DatasetCase conforms (used before append). THROWS on violation. */
+export function assertDatasetCase(c: unknown): asserts c is DatasetCase {
+  if (!Value.Check(DatasetCaseSchema, c)) {
+    const first = [...Value.Errors(DatasetCaseSchema, c)][0];
+    throw new Error(
+      `assertDatasetCase: schema violation at '${first?.path ?? "?"}': ` +
+        `${first?.message ?? "invalid DatasetCase"}`,
+    );
+  }
+}
+
+/**
+ * Canonical dedup key for a tuple — dimension names sorted, so assignment order
+ * never matters. PURE + deterministic.
+ */
+export function tupleKey(tuple: DatasetTuple): string {
+  return Object.keys(tuple)
+    .sort()
+    .map((k) => `${k}=${tuple[k]}`)
+    .join("
");
+}

package/.claude/skills/mutagent-evaluator/scripts/contracts/eval-engine.ts

@@ -0,0 +1,118 @@
+/**
+ * scripts/contracts/eval-engine.ts — the ADL EVAL-stage ENGINE FORK contract.
+ * ---------------------------------------------------------------------------
+ * The TypeBox companion for the F7/F9/F14 eval-implementation fork: when the ADL
+ * `*build` stage hands a freshly-built agent to the evaluator, the user must
+ * choose HOW the eval suite is implemented. Two engines:
+ *
+ *   - Path A — `native-matrix`  (mutagent NATIVE evals): the eval-matrix format
+ *     judged by LLM-as-judge SUB-AGENTS. HARD DEPENDENCY on a Claude-Code host
+ *     to spawn the judge sub-agents. For a CODE-framework target (Mastra/TS) the
+ *     native matrix ALSO requires the built agent to EMIT logs/traces to a known
+ *     output SINK (CC session transcripts / a declared log file) for the judge to
+ *     read. Surfaced UP FRONT so a user is never silently "stuck with the matrix".
+ *   - Path B — `code-written`  (CODE-written evals): evals written in the target's
+ *     OWN language (bun/TS for Mastra) — deterministic code-checks + an LLM-judge
+ *     call VIA SDK + eval-criteria checks. PORTABLE — needs NO Claude-Code
+ *     sub-agents, so a non-CC user is NOT trapped in the matrix (F14).
+ *
+ * The fork is TARGET-CONDITIONAL (F9): a code-framework target (e.g. Mastra/TS)
+ * may pick EITHER engine; a harness target (harness:claude-code) is native-only.
+ * This file declares the DATA (the menu options + the resolved plan + the gates);
+ * the AskUserQuestion gate + the actual codegen live elsewhere. Subject-agnostic.
+ *
+ * Standalone: this skill NEVER imports the agentspec schema (cross-skill import is
+ * banned). It consumes a MINIMAL local slice (`EngineTargetInput`) the caller maps
+ * from the agentspec's `build` block. PURE — no clock / random / network.
+ */
+import { type Static, Type } from "@sinclair/typebox";
+import { Value } from "@sinclair/typebox/value";
+
+// ── EvalEngine — the two implementation paths (the fork) ─────────────────────
+export const EvalEngine = {
+  /** Path A — mutagent native eval-matrix + LLM-judge SUB-AGENTS (needs a CC host). */
+  NativeMatrix: "native-matrix",
+  /** Path B — code-written evals in the target's language (portable, no CC sub-agents). */
+  CodeWritten: "code-written",
+} as const;
+export type EvalEngineValue = (typeof EvalEngine)[keyof typeof EvalEngine];
+
+// ── EngineTargetInput — the MINIMAL slice the caller maps from agentspec.build ─
+//
+// We do NOT import the agentspec contract (standalone skill). The caller reads
+// `agentspec.build.{target_framework, runtime}` and hands us this slice.
+export const EngineTargetInputSchema = Type.Object(
+  {
+    /** e.g. "mastra" | "deepagents" | "langgraph" | "harness:claude-code" | "harness:codex". */
+    targetFramework: Type.String({ minLength: 1 }),
+    /** e.g. "bun" | "node" | "deno" | "python" | "shell". */
+    runtime: Type.String({ minLength: 1 }),
+  },
+  { additionalProperties: false },
+);
+export type EngineTargetInput = Static<typeof EngineTargetInputSchema>;
+
+// ── EvalEngineOption — one selectable menu entry (target-conditional) ─────────
+//
+// What the AskUserQuestion gate renders. `requiresClaudeCode` + `requiresLogSink`
+// are the F7/F9 SURFACED dependencies — shown UP FRONT, never discovered late.
+export const EvalEngineOptionSchema = Type.Object(
+  {
+    engine: Type.Union([
+      Type.Literal(EvalEngine.NativeMatrix),
+      Type.Literal(EvalEngine.CodeWritten),
+    ]),
+    /** short human label for the picker. */
+    label: Type.String({ minLength: 1 }),
+    /** one-line description of what this engine does for THIS target. */
+    summary: Type.String({ minLength: 1 }),
+    /** TRUE iff this engine needs a Claude-Code host to spawn judge sub-agents (Path A). */
+    requiresClaudeCode: Type.Boolean(),
+    /** TRUE iff this engine needs the built agent to emit logs/traces to a sink (F9). */
+    requiresLogSink: Type.Boolean(),
+    /** TRUE iff this engine is portable to a non-CC user (Path B). */
+    portable: Type.Boolean(),
+    /** TRUE iff this is the recommended default for the target. */
+    recommended: Type.Boolean(),
+  },
+  { additionalProperties: false },
+);
+export type EvalEngineOption = Static<typeof EvalEngineOptionSchema>;
+
+// ── EvalEnginePlan — the RESOLVED choice (what *evaluate runs against) ────────
+export const EvalEnginePlanSchema = Type.Object(
+  {
+    engine: Type.Union([
+      Type.Literal(EvalEngine.NativeMatrix),
+      Type.Literal(EvalEngine.CodeWritten),
+    ]),
+    target: EngineTargetInputSchema,
+    /** TRUE iff the target is a coding-FRAMEWORK (vs a `harness:*` target). */
+    targetIsFramework: Type.Boolean(),
+    /** the SURFACED dependency: needs a CC host (Path A). */
+    requiresClaudeCode: Type.Boolean(),
+    /**
+     * the SURFACED dependency: the built agent must emit logs/traces to THIS sink
+     * for the native matrix judge to read it. Present iff `requiresLogSink`.
+     */
+    requiresLogSink: Type.Boolean(),
+    /** the discoverable OUTPUT SINK the engine writes to (always set — discoverability gate). */
+    outputSink: Type.String({ minLength: 1 }),
+    /** human-readable rationale (echoed into the entity card + report). */
+    rationale: Type.String({ minLength: 1 }),
+  },
+  { additionalProperties: false },
+);
+export type EvalEnginePlan = Static<typeof EvalEnginePlanSchema>;
+
+/** Parse + narrow an EvalEnginePlan (guarded). THROWS on schema violation. PURE. */
+export function parseEvalEnginePlan(value: unknown): EvalEnginePlan {
+  if (!Value.Check(EvalEnginePlanSchema, value)) {
+    const first = [...Value.Errors(EvalEnginePlanSchema, value)][0];
+    throw new Error(
+      `parseEvalEnginePlan: schema violation at '${first?.path ?? "(root)"}': ` +
+        `${first?.message ?? "invalid EvalEnginePlan"}`,
+    );
+  }
+  return value;
+}