@mutagent/evaluator 0.1.0-alpha.5 → 0.2.0-alpha.1

package/.claude/skills/mutagent-evaluator/scripts/judge-prompt-template.ts

@@ -48,20 +48,94 @@ function promptOf(trace: EvalTrace): string {
 
 // ── Determiner (EV-042) prompt — MIRRORS assets/agents/error-analyst.md ──────
 
+/**
+ * EV-1 — the compact SUBJECT FRAME the determiner reads so it derives THIS subject's
+ * expected outcome (identity · purpose · tools + reversibility), instead of judging
+ * against a hardcoded email-agent signal block. Trajectory-LIGHT by design — it never
+ * carries the full step-by-step trajectory (that is the per-criterion judge's job).
+ */
+export interface SubjectFrame {
+  identity: string;
+  purpose: string;
+  tools: string[];
+  toolReversibility?: { name: string; reversibility: string }[];
+}
+
+/** The distinct tool names actually invoked in a trace, in first-seen order (compact summary). */
+function keyToolsUsed(trace: EvalTrace, limit = 8): string[] {
+  const seen: string[] = [];
+  for (const o of trace.observations) {
+    if (o.type === "TOOL" && typeof o.name === "string" && !seen.includes(o.name)) {
+      seen.push(o.name);
+      if (seen.length >= limit) break;
+    }
+  }
+  return seen;
+}
+
+/** Render the subject frame block for the determiner (EV-1). Trajectory-light. */
+function subjectFrameBlock(subject: SubjectFrame | undefined): string[] {
+  if (subject === undefined) {
+    return [
+      "SUBJECT: not supplied — RECONSTRUCT what the agent is from the input event + the",
+      "tools it called, and derive the expected outcome from that. Do NOT assume any",
+      "particular domain (e.g. email).",
+      "",
+    ];
+  }
+  const rev = new Map((subject.toolReversibility ?? []).map((t) => [t.name, t.reversibility]));
+  const toolLine =
+    subject.tools.length > 0
+      ? subject.tools
+          .map((n) => {
+            const r = rev.get(n);
+            return r !== undefined && r !== "unknown" ? `${n} [${r}]` : n;
+          })
+          .join(", ")
+      : "(none)";
+  const irreversible = (subject.toolReversibility ?? [])
+    .filter((t) => t.reversibility === "irreversible-external")
+    .map((t) => t.name);
+  return [
+    "SUBJECT (derive the EXPECTED outcome for THIS subject before you judge):",
+    `  identity: ${subject.identity}`,
+    `  purpose:  ${subject.purpose}`,
+    `  tools:    ${toolLine}`,
+    ...(irreversible.length > 0
+      ? [`  ⚠ irreversible-external tools: ${irreversible.join(", ")} — a failure that also fired one is materially worse.`]
+      : []),
+    "",
+  ];
+}
+
 /**
  * Render the determiner judge prompt for the in-house/export substrate. The
  * AUTHORITATIVE "inaction can be success" rubric is `error-analyst.md`; this is
  * its provider-callable mirror. No decision is made here.
+ *
+ * EV-1 — subject-aware: given a `subject` frame the determiner derives the expected
+ * outcome from the subject's identity/purpose/tools (not a fixed email-agent signal
+ * block), and reads a COMPACT outcome summary (event · did-it-act · key tools ·
+ * terminal state) — deliberately NOT the full trajectory (the per-criterion judge's
+ * job). The domain-specific signals (send / guard / recovery) render ONLY when the
+ * subject vocab actually defines them, so a non-email subject no longer sees dead lines.
  */
 export function buildOutcomePrompt(
   trace: EvalTrace,
   signals: OutcomeSignals,
   vocab: SubjectVocab,
+  subject?: SubjectFrame,
 ): { system: string; user: string } {
   const system = [
     "You are a success/failure determiner for an autonomous agent session.",
     "Decide whether the session REACHED THE GOAL implied by its input event.",
     "",
+    "Derive the EXPECTED outcome from the SUBJECT (its identity · purpose · tools) +",
+    "the input event — do NOT assume any particular domain (e.g. email). This is a",
+    "FAST input→output outcome check: read the goal (input), the result (output/self-",
+    "summary) and the compact summary below — NOT the full step-by-step trajectory",
+    "(that is the per-criterion judge's job).",
+    "",
     "CRITICAL RULE — inaction can be success. Holding (sending nothing, calling",
     "no tool) is the CORRECT outcome when the event is a restraint directive",
     "(e.g. a guard/hold directive) or when acting would be wrong. You MUST NOT",
@@ -74,23 +148,39 @@ export function buildOutcomePrompt(
     "Reason first in `critique`, then commit to `result`.",
   ].join("\n");
 
-  // Generic guard-counter label — the attribute NAME comes from the subject vocab.
-  const guardLabel = vocab.guardCounterAttr ?? "guard counter";
+  const responseText = typeof trace.output?.response === "string" ? trace.output.response : "";
+  const keyTools = keyToolsUsed(trace);
+
+  // EV-1 — domain signals render ONLY when the subject vocab defines them (else they
+  // were email-agent noise for every subject). honest-null preserved.
+  const conditionalSignals: string[] = [];
+  if (vocab.guardCounterAttr !== null) {
+    conditionalSignals.push(`  Guard ${vocab.guardCounterAttr}: ${signals.guardConsecutive ?? "n/a"}`);
+  }
+  if (vocab.sendTool.length > 0) {
+    conditionalSignals.push(
+      `  Sent a message: ${signals.sentMessage === null ? "unknown" : signals.sentMessage}`,
+      `  Send succeeded: ${signals.sendSucceeded ?? "n/a (no send)"}`,
+    );
+  }
+  if (vocab.recoveryTools.length > 0) {
+    conditionalSignals.push(`  Recovery tool present: ${signals.recoveryPresent}`);
+  }
+
   const user = [
-    `Event kind: ${signals.eventKind}`,
-    `Guard ${guardLabel}: ${signals.guardConsecutive ?? "n/a"}`,
-    `Tools called: ${signals.toolCount}`,
-    // EV-049 honest-null: render UNKNOWN as "unknown" (the subject's send tool is
-    // unset/uninferred) — never coerce it to "false", which would lie to the judge.
-    `Sent a message: ${signals.sentMessage === null ? "unknown" : signals.sentMessage}`,
-    `Send succeeded: ${signals.sendSucceeded ?? "n/a (no send)"}`,
-    `Recovery tool present: ${signals.recoveryPresent}`,
+    ...subjectFrameBlock(subject),
+    "OUTCOME SUMMARY (compact — NOT the full trajectory):",
+    `  Event kind: ${signals.eventKind}`,
+    `  Tools called: ${signals.toolCount}${keyTools.length > 0 ? ` · key tools: ${keyTools.join(", ")}` : ""}`,
+    `  Did the agent act: ${signals.toolCount > 0 ? "yes (called tools)" : "no (no tool calls)"}`,
+    `  Terminal state: ${responseText.length > 0 ? "produced a self-summary/response" : "no self-summary"}`,
+    ...conditionalSignals,
     "",
     "Input event prompt:",
     promptOf(trace),
     "",
     "Agent self-summary (output.response):",
-    typeof trace.output?.response === "string" ? trace.output.response : "(none)",
+    responseText.length > 0 ? responseText : "(none)",
   ].join("\n");
 
   return { system, user };
@@ -107,10 +197,13 @@ export async function determineOutcome(
   trace: EvalTrace,
   judge: JudgeInvoke,
   vocab: SubjectVocab,
+  subject?: SubjectFrame,
 ): Promise<OutcomeResult> {
   const signals = extractOutcomeSignals(trace, vocab);
-  const { system, user } = buildOutcomePrompt(trace, signals, vocab);
-  const raw = await judge(system, user);
+  const { system, user } = buildOutcomePrompt(trace, signals, vocab, subject);
+  // INF-3 — pass the trace id so the determiner verdict is keyed PER-TRACE (two
+  // sessions with an identical determiner prompt never share one verdict).
+  const raw = await judge(system, user, trace.id);
   const verdict = parseCritiqueVerdict(raw);
   return {
     traceId: trace.id,
@@ -153,7 +246,8 @@ function profilePreamble(profile?: SubjectProfile): string[] {
     `  identity: ${profile.identity}`,
     `  purpose:  ${profile.purpose}`,
     `  scope:    ${profile.scope}`,
-    `  tools:    ${(profile.tools ?? []).join(", ") || "(none observed)"}`,
+    `  tools:    ${renderToolLine(profile)}`,
+    ...reversibilityCallout(profile),
     profile.skill !== undefined ? `  skill:    ${profile.skill}` : "",
     `  harness:  ${profile.harness}`,
     `  provenance: ${profile.provenance}${profile.version !== undefined ? ` · version ${profile.version}` : ""}`,
@@ -161,6 +255,36 @@ function profilePreamble(profile?: SubjectProfile): string[] {
   ].filter((l) => l !== "");
 }
 
+/** Render the tools line, annotating each tool with its EV-5 reversibility when known. */
+function renderToolLine(profile: SubjectProfile): string {
+  const tools = profile.tools ?? [];
+  if (tools.length === 0) return "(none observed)";
+  const rev = new Map((profile.toolReversibility ?? []).map((t) => [t.name, t.reversibility]));
+  return tools
+    .map((name) => {
+      const r = rev.get(name);
+      return r !== undefined && r !== "unknown" ? `${name} [${r}]` : name;
+    })
+    .join(", ");
+}
+
+/**
+ * EV-5 — a one-line callout naming the tools that take IRREVERSIBLE EXTERNAL actions,
+ * so a failure that ALSO fired one weighs heavier than a read-only failure. Absent
+ * reversibility data ⇒ no callout (the judge still reads the bare tool list).
+ */
+function reversibilityCallout(profile: SubjectProfile): string[] {
+  const irreversible = (profile.toolReversibility ?? [])
+    .filter((t) => t.reversibility === "irreversible-external")
+    .map((t) => t.name);
+  if (irreversible.length === 0) return [];
+  return [
+    `  ⚠ irreversible-external tools: ${irreversible.join(", ")} — a failure that ALSO`,
+    "    fired one of these (an email sent, a live system mutated) is MATERIALLY WORSE",
+    "    than a read-only failure; weigh that in your critique.",
+  ];
+}
+
 /**
  * Render the 4-component per-criterion judge prompt for the in-house/export
  * substrate. The AUTHORITATIVE 4-component / binary / critique-before-verdict
@@ -227,6 +351,11 @@ export async function runJudge(
   subjectTrace: EvalTrace,
   judge: JudgeInvoke,
   pin: JudgePin,
+  // EV-5 CUT WIRE 1 — the M1 subject profile the judge reads in its preamble (tools +
+  // reversibility). Pre-fix runJudge called buildJudgePrompt WITHOUT it, so the export
+  // path always hit the "profile not supplied — reconstruct" branch and never saw the
+  // reversibility flags. The pipeline now builds it once (buildSubjectProfile) and passes it.
+  subjectProfile?: SubjectProfile,
 ): Promise<CriterionVerdict> {
   if (typeof pin.modelId !== "string" || pin.modelId.length === 0) {
     throw new Error(
@@ -240,7 +369,7 @@ export async function runJudge(
         "MODEL INTENT IS SACRED: reruns must be byte-identical (C-PIN); refusing.",
     );
   }
-  const { system, user } = buildJudgePrompt(spec, subjectTrace);
+  const { system, user } = buildJudgePrompt(spec, subjectTrace, subjectProfile);
   const raw = await judge(system, user);
   const verdict = parseCritiqueVerdict(raw);
   return {

package/.claude/skills/mutagent-evaluator/scripts/materialize-dataset.ts

@@ -1,24 +1,27 @@
 /**
- * scripts/materialize-dataset.ts — F8: MATERIALIZE real dataset items from the
- * agentspec (Type A — PURE). Seed → ACTUAL items, not just definitions.
+ * scripts/materialize-dataset.ts — F8: PROJECT the agentspec's real dataset items
+ * into runnable evaluator DatasetCases (Type A — PURE).
  * ---------------------------------------------------------------------------
- * The ADL `*build` stage emits an agentspec whose `definition.evals` carries the
- * dataset DEFINITION (dataset_categories[] × edge_cases[]) + scenarios[]. The old
- * `*build-dataset` only had the dimensions/definitions; F8 requires it to
- * MATERIALIZE real, runnable DatasetCases — ≥1 per category, plus one per declared
- * edge_case — BEFORE the dataset-builder agent expands them synthetically.
+ * AgentSpec 0.3.0 (D18/D19) moved the dataset from a DEFINITION (0.2's
+ * `dataset_categories[] × edge_cases[]`, which the evaluator had to synthesize
+ * base+edge queries from) to `spec.evaluation.datasets[].items[]` — REAL,
+ * already-materialized rows, each carrying `category` / `scenarioRef` / a `case`
+ * assignment over the dataset's `caseDimensions`, plus kind-specific `input` /
+ * `expected` payloads.
  *
- * The materialized cases are the SEED layer (source: "seed"):
- *   - one BASE item per category — from a category-tagged scenario when present,
- *     else synthesized from the category description ("seed, don't duplicate").
- *   - one EDGE item per declared edge_case — query = the edge_case phrase in the
- *     category's context; flagged `edge_case: "true"` in its tuple.
+ * So materialization is now a faithful PROJECTION, not a synthesis:
+ *   - one DatasetCase per spec item (the seed layer, source: "seed").
+ *   - the case TUPLE = the item's `category` + its `case` assignment (the axes the
+ *     dataset varies over — see `dimensionsFromAgentspec`).
+ *   - the case QUERY = the item's opaque `input` payload rendered deterministically
+ *     (raw string when `input` is a string; a stable key-sorted serialization
+ *     otherwise), so a re-projection is byte-identical.
  *
- * These are REAL queries (non-empty NL), not category definitions. The
- * dataset-builder agent then expands them; build-dataset.ts's deterministic
- * id/dedup/merge dedups any overlap. Subject-agnostic. DETERMINISTIC — stable
- * content-derived ids, scenarios/categories/edge_cases consumed in given order;
- * no clock / random / network → re-materializing is byte-identical (C-PIN).
+ * The dataset-builder agent then EXPANDS these seeds synthetically; build-dataset.ts's
+ * deterministic id/dedup/merge dedups any overlap. Subject-agnostic (EV-002): axes
+ * and values come from the agentspec, never hard-coded. DETERMINISTIC — items and
+ * axes consumed in given order; no clock / random / network → re-projecting is
+ * byte-identical (C-PIN).
  */
 import {
   type Dataset,
@@ -26,88 +29,147 @@ import {
   type DatasetTuple,
   type Dimension,
 } from "./contracts/dataset.ts";
-import type { AgentspecEvals, AgentspecScenario } from "./contracts/agentspec-evals.ts";
+import type {
+  SpecEvaluation,
+  SpecEvalDatasetItem,
+} from "./contracts/agentspec-evals.ts";
 import { buildCase, appendToDataset } from "./build-dataset.ts";
 import { CaseSource } from "./contracts/dataset.ts";
 
-/** The literal tuple values for the boolean edge_case dimension (DatasetTuple is string-valued). */
-const EDGE_TRUE = "true";
-const EDGE_FALSE = "false";
+/** The tuple key under which an item's dataset-local category is recorded. */
+const CATEGORY_DIM = "category";
 
 /**
- * The dimensions the materialized dataset varies over: `category` (values = the
- * category ids) + `edge_case` (boolean). Subject-agnostic — derived from the
- * agentspec, never hard-coded. PURE.
+ * Deterministic, key-sorted serialization — a stable rendering of an opaque item
+ * payload so two projections of the same payload are byte-identical regardless of
+ * key insertion order. PURE.
  */
-export function dimensionsFromAgentspec(evals: AgentspecEvals): Dimension[] {
-  const categories = evals.dataset_categories.map((c) => c.id);
+function stableStringify(value: unknown): string {
+  if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
+  const record = value as Record<string, unknown>;
+  const body = Object.keys(record)
+    .sort()
+    .map((k) => `${JSON.stringify(k)}:${stableStringify(record[k])}`)
+    .join(",");
+  return `{${body}}`;
+}
+
+/**
+ * The dimensions the projected dataset varies over: `category` (values = the
+ * dataset-local category ids, first-seen order) + one dimension per distinct
+ * `caseDimensions` axis (values merged across datasets, first-seen order). Both are
+ * derived from the agentspec, never hard-coded. Subject-agnostic. PURE.
+ */
+export function dimensionsFromAgentspec(evaluation: SpecEvaluation): Dimension[] {
   const dims: Dimension[] = [];
-  if (categories.length > 0) {
-    dims.push({ name: "category", description: "the dataset-category slice", values: categories });
+
+  // category dimension — the union of dataset-local category ids (first-seen, deduped).
+  const categoryIds: string[] = [];
+  const seenCategory = new Set<string>();
+  for (const dataset of evaluation.datasets) {
+    for (const category of dataset.categories ?? []) {
+      if (seenCategory.has(category.id)) continue;
+      seenCategory.add(category.id);
+      categoryIds.push(category.id);
+    }
+  }
+  if (categoryIds.length > 0) {
+    dims.push({ name: CATEGORY_DIM, description: "the dataset-local category slice", values: categoryIds });
   }
-  dims.push({ name: "edge_case", description: "is this an edge/adversarial case", values: [EDGE_FALSE, EDGE_TRUE] });
-  return dims;
-}
 
-/** The first scenario tagged into `categoryId` (given order), if any. PURE. */
-function baseScenarioFor(evals: AgentspecEvals, categoryId: string): AgentspecScenario | undefined {
-  return evals.scenarios.find((s) => s.category === categoryId && s.edge_case !== true);
+  // caseDimensions — each named axis, values merged across datasets (first-seen, deduped).
+  const axisOrder: string[] = [];
+  const axes = new Map<string, { description?: string; values: string[]; seen: Set<string> }>();
+  for (const dataset of evaluation.datasets) {
+    for (const [name, axis] of Object.entries(dataset.caseDimensions ?? {})) {
+      let entry = axes.get(name);
+      if (entry === undefined) {
+        entry = { values: [], seen: new Set<string>() };
+        if (axis.description !== undefined) entry.description = axis.description;
+        axes.set(name, entry);
+        axisOrder.push(name);
+      }
+      for (const value of axis.values) {
+        if (entry.seen.has(value)) continue;
+        entry.seen.add(value);
+        entry.values.push(value);
+      }
+    }
+  }
+  for (const name of axisOrder) {
+    const entry = axes.get(name);
+    if (entry === undefined) continue;
+    const dim: Dimension = { name, values: entry.values };
+    if (entry.description !== undefined) dim.description = entry.description;
+    dims.push(dim);
+  }
+
+  return dims;
 }
 
-/** The base query for a category — the scenario text when tagged, else the description. PURE. */
-function baseQueryFor(evals: AgentspecEvals, categoryId: string, description: string): string {
-  const scenario = baseScenarioFor(evals, categoryId);
-  return scenario !== undefined ? scenario.description : description;
+/**
+ * The variation tuple an item realizes — its dataset-local `category` (when present)
+ * plus its `case` axis assignments. Falls back to the item id when an item declares
+ * neither (a DatasetTuple must have ≥1 assignment). PURE.
+ */
+function tupleFromItem(item: SpecEvalDatasetItem): DatasetTuple {
+  const tuple: DatasetTuple = {};
+  if (item.category !== undefined && item.category.length > 0) tuple[CATEGORY_DIM] = item.category;
+  for (const [axis, value] of Object.entries(item.case ?? {})) tuple[axis] = value;
+  if (Object.keys(tuple).length === 0) tuple.item = item.id;
+  return tuple;
 }
 
-/** The edge query for an edge_case phrase in a category context. PURE. */
-function edgeQueryFor(categoryId: string, edgePhrase: string): string {
-  // A real NL query expressing the edge condition in the category's context.
-  return `[${categoryId}] handle the edge case: ${edgePhrase}`;
+/**
+ * The natural-language query an item realizes — its opaque `input` payload rendered
+ * deterministically. A string input is used verbatim; any other payload is
+ * stable-serialized; a missing payload falls back to a deterministic descriptor so
+ * the query is always non-empty (DatasetCase requires it). PURE.
+ */
+function queryFromItem(item: SpecEvalDatasetItem): string {
+  const input = item.input;
+  if (typeof input === "string" && input.length > 0) return input;
+  if (input !== undefined) {
+    const rendered = stableStringify(input);
+    if (rendered.length > 0) return rendered;
+  }
+  const context = item.category ?? item.scenarioRef ?? item.id;
+  return `[${context}] ${item.id}`;
 }
 
 /**
- * MATERIALIZE real DatasetCases from the agentspec.evals seed (F8). For EACH
- * category: one base item (≥1 per category — the success gate) + one item per
- * declared edge_case (flagged edge_case). DETERMINISTIC, deduped by content id.
- * Returns [] for an empty evals block (no categories → no items). PURE.
+ * PROJECT the agentspec's real dataset items (0.3.0 `spec.evaluation.datasets[].items[]`)
+ * into runnable DatasetCases (F8), one seed case per item. DETERMINISTIC, deduped by
+ * content id (a re-projection collides → drop). Returns [] when no dataset carries
+ * items. PURE.
  */
-export function materializeFromAgentspec(evals: AgentspecEvals): DatasetCase[] {
+export function materializeFromAgentspec(evaluation: SpecEvaluation): DatasetCase[] {
   const out: DatasetCase[] = [];
   const seen = new Set<string>();
-  const push = (tuple: DatasetTuple, query: string): void => {
-    const c = buildCase(tuple, query, CaseSource.Seed);
-    if (seen.has(c.id)) return; // content-id dedup (re-materialize collides → drop)
-    seen.add(c.id);
-    out.push(c);
-  };
-  for (const cat of evals.dataset_categories) {
-    // base item — one real query per category.
-    push(
-      { category: cat.id, edge_case: EDGE_FALSE },
-      baseQueryFor(evals, cat.id, cat.description),
-    );
-    // edge items — one real query per declared edge_case.
-    for (const edge of cat.edge_cases) {
-      if (edge.length === 0) continue;
-      push({ category: cat.id, edge_case: EDGE_TRUE }, edgeQueryFor(cat.id, edge));
+  for (const dataset of evaluation.datasets) {
+    for (const item of dataset.items ?? []) {
+      const c = buildCase(tupleFromItem(item), queryFromItem(item), CaseSource.Seed);
+      if (seen.has(c.id)) continue; // content-id dedup (re-project collides → drop)
+      seen.add(c.id);
+      out.push(c);
     }
   }
   return out;
 }
 
 /**
- * Assemble (or extend) a Dataset seeded with the materialized real items. When
- * `existing` is supplied, the materialized cases are merged MONOTONICALLY (no
+ * Assemble (or extend) a Dataset seeded with the projected real items. When
+ * `existing` is supplied, the projected cases are merged MONOTONICALLY (no
  * duplicates, version bumps). Subject-agnostic. DETERMINISTIC. PURE.
  */
 export function materializeToDataset(
   subject: string,
-  evals: AgentspecEvals,
+  evaluation: SpecEvaluation,
   existing?: Dataset,
 ): Dataset {
-  const dimensions = dimensionsFromAgentspec(evals);
-  const cases = materializeFromAgentspec(evals);
+  const dimensions = dimensionsFromAgentspec(evaluation);
+  const cases = materializeFromAgentspec(evaluation);
   const base: Dataset = existing ?? { subject, dimensions, cases: [], version: 0 };
   return appendToDataset(base, cases);
 }