@mutagent/evaluator 0.1.0-alpha.6 → 0.2.0-alpha.1

package/.claude/skills/mutagent-evaluator/scripts/code-eval-library.ts

@@ -0,0 +1,201 @@
+/**
+ * code-eval-library — the L0 CODE-CHECKS layer of the v3 layered walk (W2 · D4=A).
+ *
+ * The growing library of code-detectable failure patterns, run FIRST as the cheap
+ * early filter before any trace is read by the LLM judge. Each entry is a STATIC
+ * check over the MatrixPacket's trajectory — NO LLM, no content semantics. A
+ * pattern FIRING flags a CANDIDATE for the deeper layers; it is NOT a verdict
+ * (code-detectable ≠ incorrect — the meta-judge fence). The judge records each
+ * hit as `codeEvalHits[{pattern, anchor, detail}]` (E-NEW-5) — evidence, never
+ * a layer verdict on its own.
+ *
+ * TS re-implementation of the Labs seeds (spec-first absorption, D4=A). The
+ * Python originals stay in Labs as measurement provenance (catch 1.0 / FP 0.0
+ * held-out on the aliased benchmark — a maturity fence, not a claim about any
+ * new subject). Origins:
+ *   CE-P1 fault-passthrough   ← M-tool-fault-passthrough   (fail at k → DIFFERENT tool at k+1)
+ *   CE-P2 error-output        ← M-tool-output-check        (completeness arm: any error output)
+ *   CE-P3 malformed-structure ← M-tool-output-check        (structural arm; schema-less here: truncation only)
+ *   CE-P4 unguarded-send      ← M-expected-outcome-map     (required-review-missing invariant)
+ *   CE-P5 out-of-order-gate   ← M-behavioral-tree-divergence (send precedes the present review)
+ *
+ * GENERIC fence: the send/review effect classifier is name-heuristic SEED logic
+ * and is INJECTABLE — a subject profile may carry better effect classes; nothing
+ * subject-specific is hard-coded. Deterministic — no clock, no random, no network.
+ *
+ * CLI (how the judge runs L0 per the mode doc):
+ *   bun scripts/code-eval-library.ts <packet.json>   → {"hits":[{pattern,anchor,detail}]} on stdout
+ */
+import { readFileSync } from "node:fs";
+
+/** One L0 hit — mirrors the CodeEvalHit contract (eval-matrix.ts). */
+export interface CodeEvalHit {
+  pattern: string;
+  anchor: string;
+  detail: string;
+}
+
+/** The minimal step shape the library reads (the packet's trajectory[N]). */
+export interface CodeEvalStep {
+  name: string;
+  input?: unknown;
+  output?: unknown;
+}
+
+/** send/review effect classes for the gate patterns (CE-P4/P5). */
+export type EffectClass = "send" | "review" | "other";
+
+/** Heuristic SEED classifier — injectable; a subject profile may override. */
+export function defaultEffectClassifier(step: CodeEvalStep): EffectClass {
+  const n = step.name.toLowerCase();
+  if (/send|dispatch|publish|reply|post/.test(n)) return "send";
+  if (/review|approv|guard|verify|check|confirm/.test(n)) return "review";
+  return "other";
+}
+
+/** Did this tool output fail? success:false OR an error-shaped payload. */
+function outputFailed(step: CodeEvalStep): boolean {
+  const out = step.output;
+  if (out === null || out === undefined || typeof out !== "object") return false;
+  const rec = out as Record<string, unknown>;
+  if (rec["success"] === false) return true;
+  if (typeof rec["error"] === "string" && rec["error"].length > 0) return true;
+  if (rec["status"] === "error" || rec["status"] === "failed") return true;
+  return false;
+}
+
+/** Was this output truncated? (the packet marks it; absent = false). */
+function outputTruncated(step: CodeEvalStep): boolean {
+  const out = step.output;
+  if (out === null || out === undefined || typeof out !== "object") return false;
+  return (out as Record<string, unknown>)["truncated"] === true;
+}
+
+/**
+ * CE-P1 fault-passthrough: a tool fails at step k and step k+1 is a DIFFERENT
+ * tool (not a same-tool retry, not run-end). Strict-local ⇒ an UPPER BOUND —
+ * handling that arrives >1 step later is under-credited (the origin's fence).
+ */
+function ceP1(steps: CodeEvalStep[]): CodeEvalHit | null {
+  for (let k = 0; k < steps.length; k++) {
+    const s = steps[k]!;
+    if (!outputFailed(s)) continue;
+    if (k + 1 >= steps.length) continue; // run ended at the fault → handled-by-termination
+    const nxt = steps[k + 1]!;
+    if (nxt.name === s.name) continue; // same-tool retry → handled
+    return {
+      pattern: "CE-P1 fault-passthrough",
+      anchor: `trajectory.${k}`,
+      detail: `${s.name} failed; the chronologically NEXT step is a DIFFERENT tool (${nxt.name}) — proceeded on a broken result (strict-local upper bound).`,
+    };
+  }
+  return null;
+}
+
+/** CE-P2 error-output: ANY error tool output (completeness arm; a later-handled
+ *  error still fires — separating handled is CE-P1's job, the declared cost). */
+function ceP2(steps: CodeEvalStep[]): CodeEvalHit | null {
+  const k = steps.findIndex(outputFailed);
+  if (k < 0) return null;
+  return {
+    pattern: "CE-P2 error-output",
+    anchor: `trajectory.${k}`,
+    detail: `${steps[k]!.name} returned an error-shaped output (success:false / error payload).`,
+  };
+}
+
+/** CE-P3 malformed-structure (schema-less form): a TRUNCATED payload is missing
+ *  its trailing fields — structural. Full schema conformance needs a derived
+ *  clean-corpus schema (not available generically) — declared scope limit. */
+function ceP3(steps: CodeEvalStep[]): CodeEvalHit | null {
+  for (let k = 0; k < steps.length; k++) {
+    if (outputFailed(steps[k]!)) continue; // error outputs are CE-P2's domain (disjoint scoping)
+    if (outputTruncated(steps[k]!)) {
+      return {
+        pattern: "CE-P3 malformed-structure",
+        anchor: `trajectory.${k}`,
+        detail: `${steps[k]!.name} output marked truncated — trailing fields missing (structural).`,
+      };
+    }
+  }
+  return null;
+}
+
+/** CE-P4 unguarded-send: a send effect with NO review effect anywhere. */
+function ceP4(seq: { cls: EffectClass; k: number; name: string }[]): CodeEvalHit | null {
+  const send = seq.find((e) => e.cls === "send");
+  if (send === undefined) return null;
+  if (seq.some((e) => e.cls === "review")) return null; // review present → CE-P5's domain
+  return {
+    pattern: "CE-P4 unguarded-send",
+    anchor: `trajectory.${send.k}`,
+    detail: `${send.name} (send effect) with NO review-class step anywhere in the trajectory.`,
+  };
+}
+
+/** CE-P5 out-of-order-gate: review EXISTS but a send precedes the first review. */
+function ceP5(seq: { cls: EffectClass; k: number; name: string }[]): CodeEvalHit | null {
+  const firstReview = seq.findIndex((e) => e.cls === "review");
+  if (firstReview < 0) return null; // no review at all → CE-P4's domain, not a reorder
+  const early = seq.find((e, i) => e.cls === "send" && i < firstReview);
+  if (early === undefined) return null;
+  return {
+    pattern: "CE-P5 out-of-order-gate",
+    anchor: `trajectory.${early.k}`,
+    detail: `${early.name} (send effect) chronologically precedes the first review-class step — gate present but out of order.`,
+  };
+}
+
+/** The step order of the input array. CE-P1/CE-P5 are ORDER-BEARING; a
+ *  MatrixPacket's `trajectory` preserves the trace's observations order, which
+ *  is REVERSE-chronological in this intake (the judge brief says the same:
+ *  "the observations array is reverse-chronological"). Default matches the
+ *  packet so `bun … <packet.json>` is correct out of the box. */
+export type StepOrder = "chronological" | "reverse-chronological";
+
+/**
+ * Run the whole library over a packet's trajectory steps. Returns every fired
+ * hit (empty = no pattern fired — which is NOT a clean-bill verdict, only "no
+ * code-detectable candidate"). Order-bearing checks (CE-P1/CE-P5) run over the
+ * CHRONOLOGICAL sequence; anchors always cite the ORIGINAL input index (so a
+ * `trajectory.N` anchor re-resolves against the packet as handed). PURE +
+ * deterministic.
+ */
+export function runCodeEvalLibrary(
+  steps: CodeEvalStep[],
+  classify: (s: CodeEvalStep) => EffectClass = defaultEffectClassifier,
+  order: StepOrder = "reverse-chronological",
+): CodeEvalHit[] {
+  // normalize to chronological, remembering each step's ORIGINAL packet index.
+  const indexed = steps.map((s, k) => ({ s, k }));
+  const chrono = order === "reverse-chronological" ? [...indexed].reverse() : indexed;
+  const chronoSteps = chrono.map((e) => e.s);
+  const seq = chrono.map((e, i) => ({ cls: classify(e.s), k: e.k, name: e.s.name, i }));
+  // re-anchor a hit produced against chrono positions back to the packet index.
+  const reanchor = (h: CodeEvalHit | null): CodeEvalHit | null => {
+    if (h === null) return h;
+    const m = /^trajectory\.(\d+)$/.exec(h.anchor);
+    if (m === null) return h;
+    const orig = chrono[Number.parseInt(m[1]!, 10)];
+    return orig === undefined ? h : { ...h, anchor: `trajectory.${orig.k}` };
+  };
+  return [
+    reanchor(ceP1(chronoSteps)),
+    reanchor(ceP2(chronoSteps)),
+    reanchor(ceP3(chronoSteps)),
+    ceP4(seq), // seq entries already carry the original index k
+    ceP5(seq),
+  ].filter((h): h is CodeEvalHit => h !== null);
+}
+
+// ── CLI: bun scripts/code-eval-library.ts <packet.json> ──────────────────────
+if (import.meta.main) {
+  const packetPath = process.argv[2];
+  if (packetPath === undefined) {
+    console.error("usage: bun scripts/code-eval-library.ts <packet.json>");
+    process.exit(2);
+  }
+  const packet = JSON.parse(readFileSync(packetPath, "utf8")) as { trajectory?: CodeEvalStep[] };
+  const hits = runCodeEvalLibrary(packet.trajectory ?? []);
+  console.info(JSON.stringify({ hits }, null, 2));
+}

package/.claude/skills/mutagent-evaluator/scripts/contracts/agentspec-evals.ts

@@ -1,84 +1,146 @@
 /**
- * scripts/contracts/agentspec-evals.ts — the MINIMAL agentspec.evals slice the
+ * scripts/contracts/agentspec-evals.ts — the MINIMAL `spec.evaluation` slice the
  * EVAL stage consumes (standalone — NEVER imports the agentspec skill's schema).
  * ---------------------------------------------------------------------------
- * The ADL `*build` stage emits an agentspec whose `definition.evals` block carries
- * the SEED material the evaluator materializes into a real dataset (F8) + criteria:
+ * AgentSpec 0.3.0 (D18/D19) replaces the flat 0.2 `definition.evals` shape
+ * (`dataset_categories[] × edge_cases[]`) with `spec.evaluation`, a universal
+ * closure of THREE arrays (each may be empty, F05):
  *
- *   - dataset_categories[] — `{ id, description, edge_cases[] }` — the golden
- *     eval-suite slices + the explicit edge_cases each must exercise (the dataset
- *     DEFINITION the spec hands forward; "seed, don't duplicate").
- *   - scenarios[]          — `{ id, description, expected_behavior, category?,
- *     edge_case? }` — representative situations + the correct behavior (extra seed
- *     material, optionally tagged into a category).
- *   - success_criteria[]   — `{ id, criterion, type, goal }` — binary-actionable
- *     pass/fail criteria (type ∈ llm-judge | code-check).
+ *   - criteria[]  — `{ id, description, type, goal }` — binary-actionable pass/fail
+ *     criteria (type ∈ llm-judge | code-check). (0.2 `success_criteria[]`, with
+ *     `criterion` → `description`.)
+ *   - scenarios[] — `{ id, description, expectedBehavior, edgeCase? }` — representative
+ *     situations + the correct behavior. Scenario↔dataset linkage now lives on the
+ *     dataset ITEM (`scenarioRef`), not on the scenario.
+ *   - datasets[]  — `{ id, description, mapsTo?, categories[], caseDimensions?,
+ *     items[]|itemsRef? }` — REAL, already-materialized dataset rows (D18):
+ *       · mapsTo        — direct job/scenario/criterion references this dataset exercises.
+ *       · categories[]  — DATASET-LOCAL taxonomy (`{ id, description, generationGuidance? }`),
+ *         no longer a top-level global list (D18).
+ *       · caseDimensions — a MAP `name → { description?, values[] }`; the independent
+ *         variation axes each item's `case` assigns over.
+ *       · items[]       — each `{ id, category?, scenarioRef?, case?, input?, expected? }`
+ *         carrying a KIND-SPECIFIC `input`/`expected` payload (opaque data — we read the
+ *         shape structurally, never interpret the kind).
  *
  * We DECLARE only the fields we read, with `additionalProperties: true` on the
- * objects so a richer agentspec still parses (forward-compatible). The cross-skill
+ * objects so a richer 0.3.x agentspec still parses (forward-compatible). The cross-skill
  * import ban (coding-rules "Sealed-Sibling" + standalone discipline) is why this is
  * a local re-declaration, not an import. PURE — no clock / random / network.
  */
 import { type Static, Type } from "@sinclair/typebox";
 import { Value } from "@sinclair/typebox/value";
 
-/** A dataset CATEGORY slice + the edge-cases it must exercise. */
-export const AgentspecDatasetCategorySchema = Type.Object(
+/** A binary-actionable SUCCESS CRITERION (0.3 `criteria[]`). */
+export const SpecEvalCriterionSchema = Type.Object(
   {
     id: Type.String({ minLength: 1 }),
     description: Type.String({ minLength: 1 }),
-    edge_cases: Type.Array(Type.String()),
+    type: Type.Union([Type.Literal("llm-judge"), Type.Literal("code-check")]),
+    goal: Type.String({ minLength: 1 }),
   },
   { additionalProperties: true },
 );
-export type AgentspecDatasetCategory = Static<typeof AgentspecDatasetCategorySchema>;
+export type SpecEvalCriterion = Static<typeof SpecEvalCriterionSchema>;
 
 /** A representative SCENARIO (the situation + correct behavior). */
-export const AgentspecScenarioSchema = Type.Object(
+export const SpecEvalScenarioSchema = Type.Object(
   {
     id: Type.String({ minLength: 1 }),
     description: Type.String({ minLength: 1 }),
-    expected_behavior: Type.String({ minLength: 1 }),
+    expectedBehavior: Type.String({ minLength: 1 }),
+    edgeCase: Type.Optional(Type.Boolean()),
+  },
+  { additionalProperties: true },
+);
+export type SpecEvalScenario = Static<typeof SpecEvalScenarioSchema>;
+
+/** A DATASET-LOCAL category slice (D18) — the taxonomy is owned by its dataset. */
+export const SpecEvalDatasetCategorySchema = Type.Object(
+  {
+    id: Type.String({ minLength: 1 }),
+    description: Type.String({ minLength: 1 }),
+    generationGuidance: Type.Optional(Type.String()),
+  },
+  { additionalProperties: true },
+);
+export type SpecEvalDatasetCategory = Static<typeof SpecEvalDatasetCategorySchema>;
+
+/** One independent variation AXIS in a dataset's `caseDimensions` map (the map VALUE). */
+export const SpecEvalCaseDimensionSchema = Type.Object(
+  {
+    description: Type.Optional(Type.String()),
+    values: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
+  },
+  { additionalProperties: true },
+);
+export type SpecEvalCaseDimension = Static<typeof SpecEvalCaseDimensionSchema>;
+
+/**
+ * A single, already-materialized dataset ITEM. `case` assigns a value per
+ * `caseDimensions` axis; `input`/`expected` are opaque kind-specific payloads we
+ * carry structurally but never interpret.
+ */
+export const SpecEvalDatasetItemSchema = Type.Object(
+  {
+    id: Type.String({ minLength: 1 }),
     category: Type.Optional(Type.String()),
-    edge_case: Type.Optional(Type.Boolean()),
+    scenarioRef: Type.Optional(Type.String()),
+    case: Type.Optional(Type.Record(Type.String({ minLength: 1 }), Type.String())),
+    input: Type.Optional(Type.Unknown()),
+    expected: Type.Optional(Type.Unknown()),
   },
   { additionalProperties: true },
 );
-export type AgentspecScenario = Static<typeof AgentspecScenarioSchema>;
+export type SpecEvalDatasetItem = Static<typeof SpecEvalDatasetItemSchema>;
 
-/** A binary-actionable SUCCESS CRITERION. */
-export const AgentspecSuccessCriterionSchema = Type.Object(
+/** Direct job/scenario/criterion references a dataset exercises (all optional). */
+export const SpecEvalDatasetMapsToSchema = Type.Object(
+  {
+    jobs: Type.Optional(Type.Array(Type.String())),
+    scenarios: Type.Optional(Type.Array(Type.String())),
+    criteria: Type.Optional(Type.Array(Type.String())),
+  },
+  { additionalProperties: true },
+);
+export type SpecEvalDatasetMapsTo = Static<typeof SpecEvalDatasetMapsToSchema>;
+
+/** A dataset: local categories + variation axes + the real materialized `items[]`. */
+export const SpecEvalDatasetSchema = Type.Object(
   {
     id: Type.String({ minLength: 1 }),
-    criterion: Type.String({ minLength: 1 }),
-    type: Type.Union([Type.Literal("llm-judge"), Type.Literal("code-check")]),
-    goal: Type.String({ minLength: 1 }),
+    description: Type.String({ minLength: 1 }),
+    mapsTo: Type.Optional(SpecEvalDatasetMapsToSchema),
+    categories: Type.Optional(Type.Array(SpecEvalDatasetCategorySchema)),
+    caseDimensions: Type.Optional(Type.Record(Type.String({ minLength: 1 }), SpecEvalCaseDimensionSchema)),
+    items: Type.Optional(Type.Array(SpecEvalDatasetItemSchema)),
+    itemsRef: Type.Optional(Type.String()),
   },
   { additionalProperties: true },
 );
-export type AgentspecSuccessCriterion = Static<typeof AgentspecSuccessCriterionSchema>;
+export type SpecEvalDataset = Static<typeof SpecEvalDatasetSchema>;
 
-/** The `definition.evals` slice. Each array may be empty structurally. */
-export const AgentspecEvalsSchema = Type.Object(
+/** The `spec.evaluation` closure. Each array is required but may be empty (F05). */
+export const SpecEvaluationSchema = Type.Object(
   {
-    success_criteria: Type.Array(AgentspecSuccessCriterionSchema),
-    scenarios: Type.Array(AgentspecScenarioSchema),
-    dataset_categories: Type.Array(AgentspecDatasetCategorySchema),
+    criteria: Type.Array(SpecEvalCriterionSchema),
+    scenarios: Type.Array(SpecEvalScenarioSchema),
+    datasets: Type.Array(SpecEvalDatasetSchema),
   },
   { additionalProperties: true },
 );
-export type AgentspecEvals = Static<typeof AgentspecEvalsSchema>;
+export type SpecEvaluation = Static<typeof SpecEvaluationSchema>;
 
 /**
- * Parse + narrow the agentspec.evals slice (guarded). THROWS on schema violation —
- * a malformed seed must never silently reach materialization. PURE.
+ * Parse + narrow the `spec.evaluation` slice (guarded). THROWS on schema violation —
+ * a malformed contract must never silently reach materialization. PURE.
  */
-export function parseAgentspecEvals(value: unknown): AgentspecEvals {
-  if (!Value.Check(AgentspecEvalsSchema, value)) {
-    const first = [...Value.Errors(AgentspecEvalsSchema, value)][0];
+export function parseSpecEvaluation(value: unknown): SpecEvaluation {
+  if (!Value.Check(SpecEvaluationSchema, value)) {
+    const first = [...Value.Errors(SpecEvaluationSchema, value)][0];
     throw new Error(
-      `parseAgentspecEvals: schema violation at '${first?.path ?? "(root)"}': ` +
-        `${first?.message ?? "invalid agentspec evals slice"}`,
+      `parseSpecEvaluation: schema violation at '${first?.path ?? "(root)"}': ` +
+        `${first?.message ?? "invalid spec.evaluation slice"}`,
     );
   }
   return value;