@mutagent/evaluator 0.1.0-alpha.2

package/.claude/skills/mutagent-evaluator/scripts/memory/append.ts

@@ -0,0 +1,215 @@
+/**
+ * scripts/memory/append.ts — project-level AutoMemory: APPEND (dated + classified).
+ * ---------------------------------------------------------------------------
+ * Append an operator-feedback entry to the project-level AutoMemory store
+ * (`.mutagent/memory/`) in the Claude-Code AutoMemory format
+ * (`references/memory-format.md`). The SUBJECT is the TOOL + operator preferences
+ * — NOT the evaluated agent (subject findings go to the living-suite, never here).
+ *
+ * The store: one FACT per `<slug>.md` file + an index `MEMORY.md`
+ * (`- [Title](<slug>.md) — <hook>` one line/entry). Every entry is DATED
+ * (`metadata.created`) + lifecycle-tagged. DEDUPE on append: an entry with the
+ * same slug is UPDATED (body + refreshed date), never duplicated.
+ *
+ * Determinism (coding-rules): the CORES are PURE — `now` (the date) is INJECTED,
+ * never read from a clock; fs is done only in the thin writer at the bottom. Tests
+ * exercise the pure cores with a fixed `now`. Standalone: no cross-skill import
+ * (mirrors the diagnostics library-store APPEND DISCIPLINE, not its per-entity
+ * keying — this store is flat + tool-scoped).
+ */
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+// ── Categorical constants (no magic strings) ─────────────────────────────────
+
+/** Claude Code's four memory types (the classification target). */
+export const MemoryType = {
+  User: "user",
+  Feedback: "feedback",
+  Project: "project",
+  Reference: "reference",
+} as const;
+export type MemoryTypeValue = (typeof MemoryType)[keyof typeof MemoryType];
+
+/** The ADL lifecycle tag (the recall filter axis). */
+export const Lifecycle = {
+  Spec: "spec",
+  Build: "build",
+  Evaluate: "evaluate",
+  Diagnose: "diagnose",
+  Improve: "improve",
+  General: "general",
+} as const;
+export type LifecycleValue = (typeof Lifecycle)[keyof typeof Lifecycle];
+
+// ── Classification (the memory-format.md rubric — first match wins) ───────────
+
+/** The raw operator feedback to classify + persist. */
+export interface FeedbackInput {
+  /** the fact to store (verbatim operator text or a distilled summary). */
+  text: string;
+  /** optional caller hint (a URL / path) — strengthens the `reference` signal. */
+  resourceHint?: string;
+}
+
+/**
+ * Classify feedback into one of the four Claude-Code types by the memory-format.md
+ * decision order (FIRST match wins): reference → user → feedback → project. PURE,
+ * deterministic — regex + explicit hint, no I/O, no LLM. A caller may OVERRIDE the
+ * classification explicitly (an operator who knows the type); this is the default.
+ */
+export function classifyFeedback(fb: FeedbackInput): MemoryTypeValue {
+  const t = fb.text.trim();
+  // 1 — external resource (URL / dashboard / ticket / dataset path).
+  if (
+    fb.resourceHint !== undefined ||
+    /\bhttps?:\/\/\S+/i.test(t) ||
+    /\b(?:dashboard|ticket|dataset|\S+\.(?:ndjson|jsonl|csv|parquet)(?:\.gz)?)\b/i.test(t)
+  ) {
+    return MemoryType.Reference;
+  }
+  // 2 — who the operator is (identity / standing preference about themselves).
+  if (/\bI['’]?m\b|\bI am\b|\bmy (?:role|team|preference)\b|\bas the (?:lead|operator|owner)\b/i.test(t)) {
+    return MemoryType.User;
+  }
+  // 3 — how the TOOL should behave (behavioral steer).
+  if (/\b(?:stop|don['’]?t|do not|too verbose|next time|always|never|prefer)\b/i.test(t)) {
+    return MemoryType.Feedback;
+  }
+  // 4 — ongoing work / goals / constraints (default).
+  return MemoryType.Project;
+}
+
+// ── Slug + entry formatting ───────────────────────────────────────────────────
+
+/** Kebab-case a title into a stable slug (deterministic — no random). */
+export function slugify(title: string): string {
+  const s = title
+    .toLowerCase()
+    .replace(/['’"]/g, "")
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  return s === "" ? "untitled" : s;
+}
+
+export interface MemoryEntry {
+  /** kebab-case slug (the file name stem + `name:` frontmatter). */
+  slug: string;
+  /** one-line description (recall relevance). */
+  description: string;
+  type: MemoryTypeValue;
+  lifecycle: LifecycleValue;
+  /** YYYY-MM-DD (INJECTED — never a live clock). */
+  created: string;
+  /** the fact body (feedback/project carry **Why:** + **How to apply:** lines). */
+  body: string;
+}
+
+const DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
+
+/** Render a MemoryEntry to its `<slug>.md` file text (Claude-Code AutoMemory format). */
+export function renderEntryMarkdown(entry: MemoryEntry): string {
+  if (!DATE_RE.test(entry.created)) {
+    throw new Error(
+      `renderEntryMarkdown: created must be YYYY-MM-DD (got '${entry.created}'). Every memory is DATED — ` +
+        "inject `now`, never read a clock.",
+    );
+  }
+  return (
+    "---\n" +
+    `name: ${entry.slug}\n` +
+    `description: ${entry.description}\n` +
+    "metadata:\n" +
+    `  type: ${entry.type}\n` +
+    `  lifecycle: ${entry.lifecycle}\n` +
+    `  created: ${entry.created}\n` +
+    "---\n\n" +
+    entry.body.trimEnd() +
+    "\n"
+  );
+}
+
+/** The one index line for an entry (`- [Title](<slug>.md) — <hook>`). */
+export function indexLine(entry: MemoryEntry): string {
+  return `- [${entry.slug}](${entry.slug}.md) — ${entry.description}`;
+}
+
+/**
+ * Fold an entry's index line into an existing `MEMORY.md`, DEDUPED by slug (an
+ * existing line for the same slug is REPLACED in place; a new slug is appended).
+ * PURE — string in, string out. Preserves a leading title/header block.
+ */
+export function upsertIndex(existingIndex: string, entry: MemoryEntry): string {
+  const newLine = indexLine(entry);
+  const lineRe = new RegExp(`^- \\[${entry.slug}\\]\\(${entry.slug}\\.md\\).*$`);
+  const lines = existingIndex.split("\n");
+  let replaced = false;
+  const out = lines.map((l) => {
+    if (lineRe.test(l)) {
+      replaced = true;
+      return newLine;
+    }
+    return l;
+  });
+  if (replaced) return out.join("\n");
+  // Append (ensure a header exists + trailing newline hygiene).
+  const base =
+    existingIndex.trim() === ""
+      ? "# AutoMemory — index\n\n> Project-level operator-feedback + tool self-learning. One line per entry.\n"
+      : existingIndex.replace(/\n*$/, "\n");
+  return base + newLine + "\n";
+}
+
+// ── Filesystem writer (the ONLY impure edge) ──────────────────────────────────
+
+export interface AppendResult {
+  slug: string;
+  entryFile: string;
+  indexFile: string;
+  /** true when an existing entry was UPDATED (dedupe hit) rather than created. */
+  updated: boolean;
+}
+
+/**
+ * Append (or UPDATE) an AutoMemory entry under `<memoryDir>/`. Writes the
+ * `<slug>.md` file + upserts the `MEMORY.md` index line. DEDUPE: an existing
+ * `<slug>.md` is overwritten (not duplicated) and its index line replaced.
+ * `now` (YYYY-MM-DD) is INJECTED. The pure cores above do the formatting; this is
+ * the thin fs edge.
+ */
+export function appendMemory(
+  memoryDir: string,
+  input: { title: string; description: string; lifecycle: LifecycleValue; feedback: FeedbackInput; type?: MemoryTypeValue },
+  now: string,
+): AppendResult {
+  const slug = slugify(input.title);
+  const type = input.type ?? classifyFeedback(input.feedback);
+  const entry: MemoryEntry = {
+    slug,
+    description: input.description,
+    type,
+    lifecycle: input.lifecycle,
+    created: now,
+    body: input.feedback.text.trim(),
+  };
+
+  mkdirSync(memoryDir, { recursive: true });
+  const entryFile = join(memoryDir, `${slug}.md`);
+  const updated = existsSync(entryFile);
+  writeFileSync(entryFile, renderEntryMarkdown(entry));
+
+  const indexFile = join(memoryDir, "MEMORY.md");
+  const existingIndex = existsSync(indexFile) ? readFileSync(indexFile, "utf8") : "";
+  writeFileSync(indexFile, upsertIndex(existingIndex, entry));
+
+  return { slug, entryFile, indexFile, updated };
+}
+
+/** List the entry slugs currently in a memory dir (PURE-ish fs read; [] if absent). */
+export function listEntrySlugs(memoryDir: string): string[] {
+  if (!existsSync(memoryDir)) return [];
+  return readdirSync(memoryDir)
+    .filter((f) => f.endsWith(".md") && f !== "MEMORY.md")
+    .map((f) => f.replace(/\.md$/, ""))
+    .sort();
+}

package/.claude/skills/mutagent-evaluator/scripts/memory/read.ts

@@ -0,0 +1,168 @@
+/**
+ * scripts/memory/read.ts — project-level AutoMemory: READ (filter by lifecycle).
+ * ---------------------------------------------------------------------------
+ * Read the project-level AutoMemory store (`.mutagent/memory/`) at run START and
+ * return the entries relevant to THIS stage. The evaluator reads entries whose
+ * `lifecycle ∈ {evaluate, general}` (the run-start injection surface). The subject
+ * is the TOOL + operator preferences — NOT the evaluated agent.
+ *
+ * Determinism (coding-rules): the PARSE + FILTER cores are PURE (string in, data
+ * out); fs is confined to `readMemoryDir`. Tests exercise the pure cores against
+ * committed fixture text. Standalone — no cross-skill import.
+ *
+ * Format: one FACT per `<slug>.md`, Claude-Code AutoMemory frontmatter
+ * (`references/memory-format.md`). A malformed / partial entry is SKIPPED (never
+ * throws) so one bad file can't blind a run.
+ */
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+  Lifecycle,
+  MemoryType,
+  type LifecycleValue,
+  type MemoryTypeValue,
+} from "./append.ts";
+
+export interface ParsedMemoryEntry {
+  slug: string;
+  description: string;
+  type: MemoryTypeValue;
+  lifecycle: LifecycleValue;
+  created: string;
+  body: string;
+}
+
+const LIFECYCLE_VALUES = new Set<string>(Object.values(Lifecycle));
+const TYPE_VALUES = new Set<string>(Object.values(MemoryType));
+
+/**
+ * Parse ONE `<slug>.md` entry (frontmatter + body). Returns null on a malformed /
+ * partial entry (missing frontmatter, unknown type/lifecycle, absent date) — a bad
+ * file is SKIPPED, never fatal. PURE — string in, entry out. `fallbackSlug` is the
+ * file stem, used when the `name:` field is absent.
+ */
+export function parseMemoryEntry(text: string, fallbackSlug: string): ParsedMemoryEntry | null {
+  const m = /^---\n([\s\S]*?)\n---\n?([\s\S]*)$/.exec(text);
+  if (m === null) return null;
+  const front = m[1] ?? "";
+  const body = (m[2] ?? "").trim();
+
+  const scalar = (key: string): string | undefined => {
+    // top-level `key: value` OR nested `  key: value` (metadata block).
+    const re = new RegExp(`^\\s*${key}:\\s*(.+?)\\s*$`, "m");
+    const mm = re.exec(front);
+    return mm ? (mm[1] as string).trim() : undefined;
+  };
+
+  const slug = scalar("name") ?? fallbackSlug;
+  const description = scalar("description") ?? "";
+  const type = scalar("type");
+  const lifecycle = scalar("lifecycle");
+  const created = scalar("created");
+
+  if (type === undefined || !TYPE_VALUES.has(type)) return null;
+  if (lifecycle === undefined || !LIFECYCLE_VALUES.has(lifecycle)) return null;
+  if (created === undefined || !/^\d{4}-\d{2}-\d{2}$/.test(created)) return null;
+
+  return {
+    slug,
+    description,
+    type: type as MemoryTypeValue,
+    lifecycle: lifecycle as LifecycleValue,
+    created,
+    body,
+  };
+}
+
+/**
+ * Keep only entries whose `lifecycle` is in `stages` (the recall filter). PURE.
+ * For the evaluator run start: `stages = [Evaluate, General]`.
+ */
+export function filterByLifecycle(
+  entries: ParsedMemoryEntry[],
+  stages: LifecycleValue[],
+): ParsedMemoryEntry[] {
+  const want = new Set<string>(stages);
+  return entries.filter((e) => want.has(e.lifecycle));
+}
+
+/** The default recall filter for an EVALUATE run: {evaluate, general}. */
+export const EVALUATE_RECALL: LifecycleValue[] = [Lifecycle.Evaluate, Lifecycle.General];
+
+export interface ReadMemoryResult {
+  exists: boolean;
+  /** entries matching the lifecycle filter, sorted by slug (deterministic). */
+  entries: ParsedMemoryEntry[];
+  /** files that failed to parse (skipped) — surfaced for honesty, never fatal. */
+  skipped: string[];
+}
+
+/**
+ * Read `<memoryDir>/*.md` (excluding the `MEMORY.md` index), parse each, and
+ * return the entries matching `stages` (default = EVALUATE_RECALL). A missing dir
+ * → `{ exists:false, entries:[] }`. A malformed file is SKIPPED (listed in
+ * `skipped`). Deterministic: entries sorted by slug. The ONLY impure edge.
+ */
+export function readMemoryDir(
+  memoryDir: string,
+  stages: LifecycleValue[] = EVALUATE_RECALL,
+): ReadMemoryResult {
+  if (!existsSync(memoryDir)) return { exists: false, entries: [], skipped: [] };
+
+  const files = readdirSync(memoryDir)
+    .filter((f) => f.endsWith(".md") && f !== "MEMORY.md")
+    .sort();
+
+  const parsed: ParsedMemoryEntry[] = [];
+  const skipped: string[] = [];
+  for (const f of files) {
+    const stem = f.replace(/\.md$/, "");
+    let text: string;
+    try {
+      text = readFileSync(join(memoryDir, f), "utf8");
+    } catch {
+      skipped.push(f);
+      continue;
+    }
+    const entry = parseMemoryEntry(text, stem);
+    if (entry === null) {
+      skipped.push(f);
+      continue;
+    }
+    parsed.push(entry);
+  }
+
+  return { exists: true, entries: filterByLifecycle(parsed, stages), skipped };
+}
+
+/**
+ * Render the recalled entries as a compact operator-preference context block for
+ * injection at run start (one bullet per entry: type · slug · description). PURE.
+ * EMPTY string when nothing recalled (a run with no AutoMemory is byte-identical
+ * to the legacy no-memory path).
+ */
+export function renderRecallContext(entries: ParsedMemoryEntry[]): string {
+  if (entries.length === 0) return "";
+  const bullets = entries
+    .map((e) => `- [${e.type}] ${e.slug} — ${e.description}`)
+    .join("\n");
+  return `AutoMemory (operator preferences · evaluate + general):\n${bullets}`;
+}
+
+// ── CLI — thin wrapper (print the recalled entries for a project root) ─────────
+// bun scripts/memory/read.ts [project-root]
+
+function memoryDirFor(projectRoot: string): string {
+  return join(projectRoot, ".mutagent", "memory");
+}
+
+const isMain =
+  typeof import.meta !== "undefined" &&
+  (import.meta as unknown as { main?: boolean }).main === true;
+if (isMain) {
+  const argv = typeof Bun !== "undefined" ? Bun.argv : process.argv;
+  const projectRoot = argv[2] ?? process.cwd();
+  const result = readMemoryDir(memoryDirFor(projectRoot));
+  process.stdout.write(JSON.stringify(result, null, 2) + "\n");
+  process.exit(0);
+}

package/.claude/skills/mutagent-evaluator/scripts/prep-tasks.ts

@@ -0,0 +1,125 @@
+/**
+ * scripts/prep-tasks.ts — the deterministic PREP half of the agent-dispatch engine.
+ * ---------------------------------------------------------------------------
+ * The agent-dispatch substrate splits judging into PREP (this file) + DISPATCH
+ * (the parent session, per references/workflows/orchestrator-protocol.md) +
+ * AGGREGATE (run-pipeline.ts reading the verdict files). PREP emits one
+ * task-spec file per judging unit — the EXACT (system, user) prompt a dispatched
+ * leaf subagent must reason over, keyed by a content-hash so AGGREGATE re-derives
+ * the same key from the prompt alone.
+ *
+ * There is a HARD STAGE BARRIER (the same dependency diagnostics has between its
+ * deep-read and its analyzer fan-out): the determiner labels (EV-042) must be
+ * REAL before the *build-evals judge prompts can be built, because the judge
+ * few-shot block is drawn from the determiner-labeled TRAIN split. So PREP runs
+ * in two stages:
+ *
+ *   Stage A — prepDeterminerTasks  (no label dependency; pure per-trace prompt).
+ *             → dispatch error-analyst → collect determiner verdict files.
+ *   Stage B — prepJudgeTasks       (REQUIRES stage-A verdicts collected; replays
+ *             the pipeline with a capturing judge so the emitted judge prompts
+ *             are byte-identical to what AGGREGATE will build).
+ *
+ * PURE except for fs (writes task-spec files; reads stage-A verdict files). No
+ * clock / random / network. Model intent SACRED: the pinned envelope is carried
+ * on every task spec for the host runtime to honor.
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { extractOutcomeSignals, type JudgeInvoke } from "./determine-outcome.ts";
+import { buildOutcomePrompt } from "./judge-prompt-template.ts";
+import {
+  writeJudgeTask,
+  verdictFileName,
+  type JudgeTaskSpec,
+  type PinnedEnvelope,
+} from "./agent-dispatch.ts";
+import { runEvalPipeline } from "./run-pipeline.ts";
+import type { PipelineOptions } from "./run-pipeline.ts";
+import type { EvalTrace, SubjectVocab } from "./contracts/eval-types.ts";
+
+/** A PREP placeholder verdict — lets the pipeline complete during stage-B capture; never reported. */
+const PREP_PLACEHOLDER = JSON.stringify({
+  critique: "PREP placeholder — awaiting the dispatched subagent's verdict file",
+  result: "pass",
+  confidence: 0,
+});
+
+/**
+ * Stage A PREP — emit one determiner (EV-042) task-spec per trace. The
+ * determiner prompt has NO label dependency (it reads the trace's own event +
+ * trajectory + terminal state), so these can be emitted + dispatched first.
+ * The dispatched error-analyst writes each verdict file; AGGREGATE-A reads them.
+ */
+export function prepDeterminerTasks(
+  traces: EvalTrace[],
+  opts: { dir: string; pin: PinnedEnvelope; vocab: SubjectVocab },
+): JudgeTaskSpec[] {
+  return traces.map((trace) => {
+    const { system, user } = buildOutcomePrompt(
+      trace,
+      extractOutcomeSignals(trace, opts.vocab),
+      opts.vocab,
+    );
+    return writeJudgeTask(opts.dir, {
+      unit: { kind: "discover", traceId: trace.id },
+      system,
+      user,
+      pin: opts.pin,
+    });
+  });
+}
+
+/**
+ * A capturing JudgeInvoke: when a prompt's verdict file is already present
+ * (stage-A determiner verdicts), READ it; otherwise CAPTURE the prompt as a
+ * judge task-spec and return a deterministic placeholder so the pipeline runs to
+ * completion. Replaying the real pipeline with this judge emits the *build-evals
+ * judge prompts using the REAL determiner labels — guaranteeing they are
+ * byte-identical to the prompts AGGREGATE will build (same code, same labels).
+ */
+export function createCapturingJudge(opts: {
+  verdictDir: string;
+  taskDir: string;
+  pin: PinnedEnvelope;
+}): { judge: JudgeInvoke; captured: JudgeTaskSpec[] } {
+  const captured: JudgeTaskSpec[] = [];
+  const judge: JudgeInvoke = (system, user) => {
+    const vpath = join(opts.verdictDir, verdictFileName(system, user));
+    if (existsSync(vpath)) return Promise.resolve(readFileSync(vpath, "utf8"));
+    captured.push(
+      // D4 — the criterion judging axis (*build-evals / Step-2b). `kind` now matches the
+      // command (was mis-tagged "evaluate" pre-merge). ENVELOPE-ONLY retag: `unit` is not
+      // part of promptHash(system, user), so the rendered prompt + its key (8b58d9ca) are
+      // invariant — only the task.json envelope carries the new {kind, axis}.
+      writeJudgeTask(opts.taskDir, {
+        unit: { kind: "build-evals", axis: "criterion" },
+        system,
+        user,
+        pin: opts.pin,
+      }),
+    );
+    return Promise.resolve(PREP_PLACEHOLDER);
+  };
+  return { judge, captured };
+}
+
+/**
+ * Stage B PREP — emit the *build-evals / *evaluate judge (EV-043/048) task-specs.
+ * REQUIRES the stage-A determiner verdict files to already be collected in
+ * `verdictDir` (else the determiner prompts would themselves be captured with
+ * placeholder labels, corrupting the judge few-shot). Replays runEvalPipeline
+ * with the capturing judge and returns the captured judge tasks for dispatch.
+ */
+export async function prepJudgeTasks(
+  traces: EvalTrace[],
+  opts: { verdictDir: string; taskDir: string; pin: PinnedEnvelope; pipeline: PipelineOptions },
+): Promise<JudgeTaskSpec[]> {
+  const { judge, captured } = createCapturingJudge({
+    verdictDir: opts.verdictDir,
+    taskDir: opts.taskDir,
+    pin: opts.pin,
+  });
+  await runEvalPipeline(traces, judge, opts.pipeline);
+  return captured;
+}