@tangle-network/agent-eval 0.72.0 → 0.72.4

package/dist/campaign/index.js

@@ -1,4 +1,5 @@
 import {
+  buildEvidenceVector,
   composeGate,
   defaultProductionGate,
   detectScale,
@@ -6,13 +7,17 @@ import {
   evolutionaryDriver,
   heldoutSignificance,
   pairHoldout,
+  paretoPolicy,
+  paretoSignificanceGate,
   runEval
-} from "../chunk-UD6EF73X.js";
+} from "../chunk-XPILG2CA.js";
 import {
   agentProfileHash,
   estimateCost,
-  isModelPriced
-} from "../chunk-SL55X4VN.js";
+  extractProducedState,
+  isModelPriced,
+  verifyCompletion
+} from "../chunk-LB2UOI5F.js";
 import {
   buildLoopProvenanceRecord,
   campaignBreakdown,
@@ -29,11 +34,12 @@ import {
   openAutoPr,
   provenanceRecordPath,
   provenanceSpansPath,
+  renderAnalystEvidence,
   runImprovementLoop,
   runOptimization,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-4QJN7RDX.js";
+} from "../chunk-JYE3WOTE.js";
 import {
   assertRealBackend,
   fsCampaignStorage,
@@ -41,24 +47,441 @@ import {
   runCampaign,
   summarizeBackendIntegrity
 } from "../chunk-ZPSKPT3V.js";
+import {
+  AnalystRegistry,
+  DEFAULT_TRACE_ANALYST_KINDS,
+  createTraceAnalystKind
+} from "../chunk-WYIHD6EB.js";
 import "../chunk-YV7J7X5N.js";
 import {
-  validateRunRecord
-} from "../chunk-F3SRAAZO.js";
+  callLlm
+} from "../chunk-IHDHUN2X.js";
 import {
   pairedBootstrap
 } from "../chunk-ITBRCT73.js";
 import "../chunk-GGE4NNQT.js";
-import "../chunk-VSMTAMNK.js";
 import {
-  callLlm
-} from "../chunk-IHDHUN2X.js";
+  OtlpFileTraceStore,
+  analyzeTraces
+} from "../chunk-VUINJM5M.js";
 import "../chunk-PC4UYEBM.js";
+import {
+  validateRunRecord
+} from "../chunk-F3SRAAZO.js";
+import "../chunk-VSMTAMNK.js";
 import {
   AgentEvalError
 } from "../chunk-3BFEG2F6.js";
 import "../chunk-PZ5AY32C.js";
 
+// src/campaign/analyst-surface.ts
+function surfaceToText(surface) {
+  if (typeof surface === "string") return surface;
+  throw new Error(
+    `buildAnalystSurfaceDispatch: the analyst surface must be a string actorDescription, got a ${surface.kind}-tier surface (${surface.worktreeRef}). The analyst prompt is prompt-tier.`
+  );
+}
+function buildAnalystSurfaceDispatch(opts) {
+  const analyze = opts.analyze ?? analyzeTraces;
+  return async (surface, scenario, _ctx) => {
+    const actorDescription = surfaceToText(surface);
+    const res = await analyze(
+      { question: scenario.question },
+      { ...opts.analystOptions, actorDescription, source: scenario.source }
+    );
+    return {
+      answer: res.answer,
+      findings: res.findings,
+      actorPromptVersion: res.actorPromptVersion
+    };
+  };
+}
+function failureModeRecallJudge(opts = {}) {
+  const recallWeight = opts.recallWeight ?? 0.5;
+  return {
+    name: "failure-mode-recall",
+    dimensions: [
+      { key: "recall", description: "fraction of ground-truth failure modes the analyst surfaced" },
+      {
+        key: "precision",
+        description: "1 \u2212 share of findings that named a failure/tool/error absent from this corpus"
+      }
+    ],
+    appliesTo: (s) => s.kind === "analyst-surface",
+    score({ artifact, scenario }) {
+      const modes = scenario.expectedFailureModes;
+      if (modes.length === 0) {
+        throw new Error(
+          `failureModeRecallJudge: scenario '${scenario.id}' has no expectedFailureModes \u2014 refusing to score (a vacuous 1.0 would corrupt the comparison)`
+        );
+      }
+      const hay = artifact.findings.join("\n").toLowerCase();
+      const matched = modes.filter((m) => m.cues.some((c) => hay.includes(c.toLowerCase())));
+      const recall = matched.length / modes.length;
+      const forbidden = (scenario.forbiddenCues ?? []).map((c) => c.toLowerCase());
+      let precision = 1;
+      let hallucinated = 0;
+      if (forbidden.length > 0) {
+        const denom = Math.max(1, artifact.findings.length);
+        hallucinated = artifact.findings.filter(
+          (f) => forbidden.some((c) => f.toLowerCase().includes(c))
+        ).length;
+        precision = 1 - hallucinated / denom;
+      }
+      const composite = forbidden.length > 0 ? recallWeight * recall + (1 - recallWeight) * precision : recall;
+      const missed = modes.filter((m) => !matched.includes(m)).map((m) => m.id);
+      const notes = `matched ${matched.length}/${modes.length} failure modes` + (missed.length ? `; missed [${missed.join(", ")}]` : "") + (hallucinated ? `; ${hallucinated} out-of-corpus finding(s)` : "");
+      return { dimensions: { recall, precision }, composite, notes };
+    }
+  };
+}
+
+// src/campaign/drivers/_findings-text.ts
+function findingToLesson(f) {
+  if (typeof f === "string") return f.trim() || null;
+  if (f && typeof f === "object") {
+    const o = f;
+    const cand = o.recommended_action ?? o.claim ?? o.lesson ?? o.text ?? o.message;
+    if (typeof cand === "string" && cand.trim()) return cand.trim();
+  }
+  return null;
+}
+function normKey(s) {
+  return s.toLowerCase().replace(/\s+/g, " ").replace(/[.;:!?\s]+$/, "").trim();
+}
+function surfaceToText2(surface) {
+  if (typeof surface === "string") return surface;
+  throw new Error(
+    `curator driver: surface must be a string prompt, got a ${surface.kind}-tier surface (${surface.worktreeRef}) \u2014 curation is prompt-tier`
+  );
+}
+
+// src/campaign/drivers/ace.ts
+var BLOCK_START = "<!-- BEGIN ace-playbook (auto-managed by aceDriver) -->";
+var BLOCK_END = "<!-- END ace-playbook -->";
+var DEFAULT_HEADING = "## Playbook (accumulated lessons \u2014 append-only)";
+function parsePlaybook(surface) {
+  const start = surface.indexOf(BLOCK_START);
+  const end = surface.indexOf(BLOCK_END);
+  if (start === -1 || end === -1 || end < start) return [];
+  const body = surface.slice(start + BLOCK_START.length, end);
+  const out = [];
+  for (const raw of body.split("\n")) {
+    const line = raw.trim();
+    if (!line.startsWith("- ")) continue;
+    const item = line.slice(2).trim();
+    const tag = /^\[g(-?\d+)\]\s*(.*)$/.exec(item);
+    if (tag) out.push({ gen: Number(tag[1]), text: tag[2].trim() });
+    else out.push({ gen: -1, text: item });
+  }
+  return out;
+}
+function stripBlock(surface) {
+  const start = surface.indexOf(BLOCK_START);
+  const end = surface.indexOf(BLOCK_END);
+  if (start === -1 || end === -1 || end < start) return surface.trimEnd();
+  return (surface.slice(0, start) + surface.slice(end + BLOCK_END.length)).trimEnd();
+}
+function aceDriver(opts = {}) {
+  const maxEntries = opts.maxEntries ?? 50;
+  if (maxEntries < 1) throw new Error("aceDriver: maxEntries must be >= 1");
+  const heading = opts.sectionHeading ?? DEFAULT_HEADING;
+  return {
+    kind: "ace",
+    async propose(ctx) {
+      const parent = surfaceToText2(ctx.currentSurface);
+      const existing = parsePlaybook(parent);
+      const seen = new Set(existing.map((b) => normKey(b.text)));
+      const fresh = [];
+      for (const f of ctx.findings ?? []) {
+        const lesson = findingToLesson(f);
+        if (!lesson) continue;
+        const k = normKey(lesson);
+        if (!k || seen.has(k)) continue;
+        seen.add(k);
+        fresh.push({ gen: ctx.generation, text: lesson });
+      }
+      if (fresh.length === 0) return [];
+      const all = [...existing, ...fresh].slice(-maxEntries);
+      const block = [
+        BLOCK_START,
+        heading,
+        ...all.map((b) => `- [g${b.gen}] ${b.text}`),
+        BLOCK_END
+      ].join("\n");
+      const base = stripBlock(parent);
+      const surface = base ? `${base}
+
+${block}` : block;
+      return [
+        {
+          surface,
+          label: `ace-playbook +${fresh.length}`,
+          rationale: `appended ${fresh.length} new lesson(s) from gen ${ctx.generation} findings (playbook now ${all.length} bullet(s), append-only)`
+        }
+      ];
+    }
+  };
+}
+
+// src/campaign/drivers/guide.ts
+var DRIVER_GUIDE = {
+  gepa: {
+    summary: "Reflective full-surface rewrite: reflects on the best parent\u2019s weakest dimensions + per-scenario scores, proposes targeted rewrites, maintains a Pareto frontier across generations.",
+    surface: "prompt",
+    strategy: "reflective-rewrite",
+    whenUse: "The default for a prompt/instruction surface with headroom \u2014 broad rewrites plus Pareto-optimal exploration across scenarios.",
+    cost: "medium"
+  },
+  skillOpt: {
+    summary: "Patch-mode: bounded, anchored add/delete/replace edits to ONE skill document, so a good rule introduced earlier is not clobbered by a later sweeping rewrite.",
+    surface: "skill-doc",
+    strategy: "anchored-patch",
+    whenUse: 'Refining a skill document incrementally where accumulated rules must be preserved; the edit budget is the "textual learning rate".',
+    cost: "medium"
+  },
+  ace: {
+    summary: "Append-mostly playbook curator: grows the playbook with provenance-tagged delta bullets, never merging \u2014 guards against context collapse.",
+    surface: "playbook",
+    strategy: "append-only",
+    whenUse: "Accumulating many specific, hard-won lessons over time where dedup/rewrite would summarize away detail.",
+    cost: "low"
+  },
+  memoryCuration: {
+    summary: "Dedup-and-rank curator: builds a compact searchable memory and grafts the most relevant, most-recurrent lessons onto the surface.",
+    surface: "memory",
+    strategy: "dedup-curate",
+    whenUse: "Accumulating lessons while keeping the surface compact \u2014 the complement to ace when context size matters more than verbatim provenance.",
+    cost: "low"
+  },
+  halo: {
+    summary: "Wraps the real external HALO engine (Inference.net, `halo` CLI) and applies its findings to the prompt via one LLM edit.",
+    surface: "prompt",
+    strategy: "analysis-edit",
+    whenUse: "Benchmarking: compete HALO head-to-head against our own analysis on identical traces via compareDrivers.",
+    cost: "high",
+    external: true
+  },
+  traceAnalyst: {
+    summary: "Wraps agent-eval\u2019s own trace-analyst engine and applies its findings to the prompt via one identical LLM edit \u2014 the symmetric opponent to haloDriver.",
+    surface: "prompt",
+    strategy: "analysis-edit",
+    whenUse: "Benchmarking our trace-analyst\u2019s analysis quality against HALO (analysis-quality head-to-head), or improving from a real OTLP trace corpus.",
+    cost: "high"
+  },
+  evolutionary: {
+    summary: "Adapts a stateless Mutator (population mutate \u2192 measure \u2192 select); no generation memory beyond the current surface.",
+    surface: "any",
+    strategy: "population-mutate",
+    whenUse: "Blind population search when you have a Mutator and don\u2019t need reflective reasoning over findings.",
+    cost: "medium"
+  }
+};
+var GOAL_RANK = {
+  explore: ["gepa", "evolutionary"],
+  refine: ["skillOpt", "gepa"],
+  accumulate: ["ace", "memoryCuration"],
+  benchmark: ["traceAnalyst", "halo"]
+};
+function selectDriver(criteria) {
+  const ranked = GOAL_RANK[criteria.goal];
+  const out = [];
+  for (const name of ranked) {
+    const entry = DRIVER_GUIDE[name];
+    if (criteria.surface && criteria.surface !== "any" && entry.surface !== criteria.surface)
+      continue;
+    out.push({
+      name,
+      entry,
+      reason: `${criteria.goal}: ${entry.strategy} on the ${entry.surface} surface \u2014 ${entry.whenUse}`
+    });
+  }
+  if (out.length === 0 && criteria.surface) {
+    for (const name of Object.keys(DRIVER_GUIDE)) {
+      const entry = DRIVER_GUIDE[name];
+      if (entry.surface === criteria.surface || entry.surface === "any") {
+        out.push({ name, entry, reason: `surface match (${entry.surface}): ${entry.whenUse}` });
+      }
+    }
+  }
+  return out;
+}
+
+// src/campaign/drivers/halo.ts
+import { execFile } from "child_process";
+import { mkdtempSync, writeFileSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import { promisify } from "util";
+var execFileAsync = promisify(execFile);
+var DEFAULT_ANALYSIS_PROMPT = "Diagnose the failures in these agent execution traces \u2014 hallucinated tool calls, redundant tool arguments, refusal loops, and semantic-correctness errors \u2014 and suggest concrete, generalizable fixes to the agent instructions.";
+var APPLY_SYSTEM = "You apply a trace-analysis report to an agent instruction prompt. Output ONLY the full revised prompt \u2014 no preamble, no commentary, no code fences. Make the minimal edits that address the report findings; preserve everything else verbatim.";
+function haloDriver(opts) {
+  const haloBin = opts.haloBin ?? "halo";
+  const model = opts.model ?? "gpt-5.4-mini";
+  return {
+    kind: "halo",
+    async propose(ctx) {
+      const parent = typeof ctx.currentSurface === "string" ? ctx.currentSurface : JSON.stringify(ctx.currentSurface);
+      const traces = await opts.resolveTraces(ctx) ?? "";
+      if (!traces.trim()) {
+        throw new Error(
+          "haloDriver: resolveTraces returned no OTLP traces \u2014 the halo engine has nothing to analyze"
+        );
+      }
+      const dir = mkdtempSync(join(tmpdir(), "halo-driver-"));
+      const tracePath = join(dir, "traces.jsonl");
+      writeFileSync(tracePath, traces.endsWith("\n") ? traces : `${traces}
+`);
+      const args = [
+        tracePath,
+        "-p",
+        opts.analysisPrompt ?? DEFAULT_ANALYSIS_PROMPT,
+        "-m",
+        model,
+        ...opts.maxDepth !== void 0 ? ["--max-depth", String(opts.maxDepth)] : [],
+        ...opts.maxTurns !== void 0 ? ["--max-turns", String(opts.maxTurns)] : []
+      ];
+      let findings;
+      try {
+        const { stdout } = await execFileAsync(haloBin, args, {
+          maxBuffer: 64 * 1024 * 1024,
+          signal: ctx.signal,
+          env: {
+            ...process.env,
+            ...opts.apiKey ? { OPENAI_API_KEY: opts.apiKey } : {},
+            OPENAI_BASE_URL: opts.baseUrl
+          }
+        });
+        findings = stdout.trim();
+      } catch (e) {
+        throw new Error(
+          `haloDriver: halo-engine ('${haloBin}') failed \u2014 ${e instanceof Error ? e.message : String(e)}`
+        );
+      }
+      if (!findings) throw new Error("haloDriver: halo-engine produced no findings");
+      const applied = await callLlm(
+        {
+          model: opts.applyModel ?? model,
+          messages: [
+            { role: "system", content: APPLY_SYSTEM },
+            {
+              role: "user",
+              content: `CURRENT PROMPT:
+${parent}
+
+HALO TRACE-ANALYSIS REPORT:
+${findings}
+
+Return the full revised prompt.`
+            }
+          ]
+        },
+        { baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl }
+      );
+      const text = applied.content.trim();
+      if (!text || text === parent) return [];
+      return [
+        {
+          surface: text,
+          label: "halo",
+          rationale: `halo-engine findings:
+${findings.slice(0, 800)}`
+        }
+      ];
+    }
+  };
+}
+
+// src/campaign/drivers/memory.ts
+var BLOCK_START2 = "<!-- BEGIN curated-memory (auto-managed by memoryCurationDriver) -->";
+var BLOCK_END2 = "<!-- END curated-memory -->";
+var DEFAULT_HEADING2 = "## Learned from prior runs (curated memory)";
+var DISTILL_SYSTEM = 'You compress raw trace-analysis findings into crisp, generalizable agent guidance. Output ONLY a JSON array of strings, each one imperative lesson the agent should follow (e.g. "Always fetch a resource before mutating it"). No prose outside the JSON. Deduplicate; keep the most actionable and general; drop case-specific noise.';
+function extractExistingLessons(text) {
+  const start = text.indexOf(BLOCK_START2);
+  const end = text.indexOf(BLOCK_END2);
+  if (start === -1 || end === -1 || end < start) return [];
+  return text.slice(start + BLOCK_START2.length, end).split("\n").map((l) => l.replace(/^\s*-\s+/, "").trim()).filter((l) => l && !l.startsWith("#"));
+}
+function stripBlock2(text) {
+  const start = text.indexOf(BLOCK_START2);
+  const end = text.indexOf(BLOCK_END2);
+  if (start === -1 || end === -1 || end < start) return text.trimEnd();
+  return (text.slice(0, start) + text.slice(end + BLOCK_END2.length)).trimEnd();
+}
+async function distillLessons(raw, distill) {
+  const res = await callLlm(
+    {
+      model: distill.model,
+      messages: [
+        { role: "system", content: DISTILL_SYSTEM },
+        { role: "user", content: `Findings:
+${raw.map((r) => `- ${r}`).join("\n")}` }
+      ]
+    },
+    { baseUrl: distill.baseUrl, apiKey: distill.apiKey, fetch: distill.fetchImpl }
+  );
+  try {
+    const parsed = JSON.parse(res.content.trim());
+    if (Array.isArray(parsed)) {
+      const lessons = parsed.filter(
+        (x) => typeof x === "string" && x.trim().length > 0
+      );
+      if (lessons.length > 0) return lessons;
+    }
+  } catch {
+  }
+  return raw;
+}
+function memoryCurationDriver(opts = {}) {
+  const maxEntries = opts.maxEntries ?? 12;
+  const heading = opts.sectionHeading ?? DEFAULT_HEADING2;
+  return {
+    kind: "memory-curation",
+    async propose(ctx) {
+      const parent = surfaceToText2(ctx.currentSurface);
+      const fresh = [];
+      for (const f of ctx.findings ?? []) {
+        const l = findingToLesson(f);
+        if (l) fresh.push(l);
+      }
+      const carried = extractExistingLessons(parent);
+      if (fresh.length === 0 && carried.length === 0) return [];
+      const distilled = opts.distill && fresh.length > 0 ? await distillLessons(fresh, opts.distill) : fresh;
+      const byKey = /* @__PURE__ */ new Map();
+      for (const l of carried) {
+        const k = normKey(l);
+        if (k) byKey.set(k, { text: l, count: 1 });
+      }
+      for (const l of distilled) {
+        const k = normKey(l);
+        if (!k) continue;
+        const e = byKey.get(k);
+        if (e) e.count += 1;
+        else byKey.set(k, { text: l, count: 1 });
+      }
+      const ranked = [...byKey.values()].sort((a, b) => b.count - a.count || a.text.localeCompare(b.text)).slice(0, maxEntries);
+      if (ranked.length === 0) return [];
+      const block = [BLOCK_START2, heading, ...ranked.map((e) => `- ${e.text}`), BLOCK_END2].join(
+        "\n"
+      );
+      const next = `${stripBlock2(parent)}
+
+${block}
+`;
+      if (next === parent) return [];
+      return [
+        {
+          surface: next,
+          label: "memory-curation",
+          rationale: `curated ${ranked.length} lessons (from ${fresh.length} new finding(s) + ${carried.length} carried)`
+        }
+      ];
+    }
+  };
+}
+
 // src/campaign/skill-patch.ts
 function applySkillPatch(surface, patch) {
   let lines = surface.split("\n");
@@ -128,6 +551,7 @@ function skillOptDriver(opts) {
       editBudget: args.editBudget,
       rejectedBuffer: args.rejectedBuffer,
       metaNote: args.metaNote,
+      findingsNote: args.findingsNote,
       count: args.count
     });
     const result = await callLlm(
@@ -160,6 +584,7 @@ function skillOptDriver(opts) {
         evidence: evidenceFromHistory(ctx, evidenceK),
         editBudget: defaultBudget,
         rejectedBuffer: [],
+        findingsNote: renderAnalystEvidence(ctx.findings, ctx.report) ?? void 0,
         count: ctx.populationSize,
         signal: ctx.signal
       });
@@ -220,6 +645,9 @@ function buildPatchPrompt(args) {
       ...args.rejectedBuffer.map((e) => `- ${e.label}: ${e.rationale} \u2014 ${e.reason}`)
     );
   }
+  if (args.findingsNote) {
+    lines.push("", args.findingsNote);
+  }
   if (args.metaNote) {
     lines.push("", `Strategy note from prior epochs: ${args.metaNote}`);
   }
@@ -292,10 +720,105 @@ function snippet(s, max = 120) {
   return t.length <= max ? t : `${t.slice(0, max)}\u2026`;
 }
 
+// src/campaign/drivers/trace-analyst.ts
+import { mkdtempSync as mkdtempSync2, writeFileSync as writeFileSync2 } from "fs";
+import { tmpdir as tmpdir2 } from "os";
+import { join as join2 } from "path";
+import { ai } from "@ax-llm/ax";
+var APPLY_SYSTEM2 = "You apply a trace-analysis report to an agent instruction prompt. Output ONLY the full revised prompt \u2014 no preamble, no commentary, no code fences. Make the minimal edits that address the report findings; preserve everything else verbatim.";
+function renderFindings(findings) {
+  return findings.map((f, i) => {
+    const action = f.recommended_action ? `
+   FIX: ${f.recommended_action}` : "";
+    const subject = f.subject ? ` (${f.subject})` : "";
+    return `${i + 1}. [${f.severity}/${f.area}]${subject} ${f.claim}${action}`;
+  }).join("\n");
+}
+function traceAnalystDriver(opts) {
+  if (!opts.apiKey) throw new Error("traceAnalystDriver: apiKey is required");
+  if (!opts.model) throw new Error("traceAnalystDriver: model is required");
+  const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS;
+  return {
+    kind: "trace-analyst",
+    async propose(ctx) {
+      const parent = typeof ctx.currentSurface === "string" ? ctx.currentSurface : JSON.stringify(ctx.currentSurface);
+      const traces = await opts.resolveTraces(ctx) ?? "";
+      if (!traces.trim()) {
+        throw new Error(
+          "traceAnalystDriver: resolveTraces returned no OTLP traces \u2014 the analyst has nothing to read"
+        );
+      }
+      const dir = mkdtempSync2(join2(tmpdir2(), "trace-analyst-driver-"));
+      const tracePath = join2(dir, "traces.jsonl");
+      writeFileSync2(tracePath, traces.endsWith("\n") ? traces : `${traces}
+`);
+      const runAnalyze = opts.analyze ?? (async (path, c) => {
+        const aiService = ai({
+          name: opts.provider ?? "openai",
+          apiKey: opts.apiKey,
+          apiURL: opts.baseUrl,
+          config: { model: opts.model }
+        });
+        const registry = new AnalystRegistry();
+        for (const spec of kinds) {
+          registry.register(createTraceAnalystKind(spec, { ai: aiService, model: opts.model }));
+        }
+        const result = await registry.run(
+          `trace-analyst-gen-${c.generation}`,
+          { traceStore: new OtlpFileTraceStore({ path }) },
+          { signal: c.signal }
+        );
+        return result.findings;
+      });
+      let findings;
+      try {
+        findings = await runAnalyze(tracePath, ctx);
+      } catch (e) {
+        throw new Error(
+          `traceAnalystDriver: analyst engine failed \u2014 ${e instanceof Error ? e.message : String(e)}`
+        );
+      }
+      if (findings.length === 0) {
+        throw new Error("traceAnalystDriver: analyst engine produced no findings");
+      }
+      const report = renderFindings(findings);
+      const applied = await callLlm(
+        {
+          model: opts.applyModel ?? opts.model,
+          messages: [
+            { role: "system", content: APPLY_SYSTEM2 },
+            {
+              role: "user",
+              content: `CURRENT PROMPT:
+${parent}
+
+TRACE-ANALYSIS REPORT:
+${report}
+
+Return the full revised prompt.`
+            }
+          ]
+        },
+        { baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl }
+      );
+      const text = applied.content.trim();
+      if (!text || text === parent) return [];
+      return [
+        {
+          surface: text,
+          label: "trace-analyst",
+          rationale: `trace-analyst findings (${findings.length}):
+${report.slice(0, 800)}`
+        }
+      ];
+    }
+  };
+}
+
 // src/campaign/labeled-store/fs-adapter.ts
 import { createHash } from "crypto";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
-import { join } from "path";
+import { existsSync, mkdirSync, readFileSync, writeFileSync as writeFileSync3 } from "fs";
+import { join as join3 } from "path";
 var LabeledScenarioStoreError = class extends Error {
   constructor(code, message) {
     super(message);
@@ -455,7 +978,7 @@ var FsLabeledScenarioStore = class {
     };
   }
   pathForSource(source) {
-    return join(this.options.root, `${source}.jsonl`);
+    return join3(this.options.root, `${source}.jsonl`);
   }
 };
 var ALL_SOURCES = [
@@ -497,9 +1020,9 @@ function sha256(input) {
 function appendLine(path, line) {
   if (existsSync(path)) {
     const existing = readFileSync(path, "utf8");
-    writeFileSync(path, existing + line);
+    writeFileSync3(path, existing + line);
   } else {
-    writeFileSync(path, line);
+    writeFileSync3(path, line);
   }
 }
 
@@ -800,7 +1323,13 @@ function gepaEntry(config, combineParents, name) {
         }),
         autoOnPromote: "none",
         runDir: `${config.runDir}/${slug(name)}-loop`,
-        ...config.seed !== void 0 ? { seed: config.seed } : {}
+        ...config.seed !== void 0 ? { seed: config.seed } : {},
+        // EYES→HANDS: flow findings to the driver's propose(). These reach
+        // runOptimization unchanged (runImprovementLoop extends RunOptimizationOptions
+        // and forwards {...opts}); ctx.findings/report/analyzeGeneration are consumed there.
+        ...config.findings !== void 0 ? { findings: config.findings } : {},
+        ...config.analyzeGeneration ? { analyzeGeneration: config.analyzeGeneration } : {},
+        ...config.report !== void 0 ? { report: config.report } : {}
       });
       const costUsd = result.baselineCampaign.aggregates.totalCostUsd + result.generations.reduce(
         (sum, g) => sum + g.surfaces.reduce((s, sf) => s + sf.campaign.aggregates.totalCostUsd, 0),
@@ -836,9 +1365,114 @@ function skillOptEntry(config, name = "skill-opt") {
   };
 }
 
+// src/campaign/presets/playback.ts
+function makePlaybackDispatch(driver) {
+  return async (profile, scenario, ctx) => {
+    const events = await driver.run(scenario, { ...ctx, profile });
+    return extractProducedState(events);
+  };
+}
+async function scoreUserStory(story, state, checkCorrectness) {
+  const verdict = await verifyCompletion(
+    { taskId: story.id, requirements: story.requirements },
+    state,
+    checkCorrectness
+  );
+  return { ...verdict, title: story.title };
+}
+function userStoryScoreboard(verdicts) {
+  const rows = [];
+  for (const v of verdicts) {
+    for (const r of v.requirements) {
+      rows.push({
+        storyId: v.taskId,
+        storyTitle: v.title,
+        reqId: r.reqId,
+        reqTitle: r.title,
+        status: r.satisfied ? "PASS" : "FAIL",
+        evidence: r.evidence
+      });
+    }
+  }
+  return rows;
+}
+function scoreboardSummary(rows) {
+  const byStory = /* @__PURE__ */ new Map();
+  let passed = 0;
+  for (const r of rows) {
+    const s = byStory.get(r.storyId) ?? { total: 0, passed: 0 };
+    s.total++;
+    if (r.status === "PASS") {
+      s.passed++;
+      passed++;
+    }
+    byStory.set(r.storyId, s);
+  }
+  let storiesFullyComplete = 0;
+  for (const s of byStory.values()) if (s.total > 0 && s.passed === s.total) storiesFullyComplete++;
+  return {
+    stories: byStory.size,
+    storiesFullyComplete,
+    requirements: rows.length,
+    passed,
+    failed: rows.length - passed,
+    passRate: rows.length === 0 ? 0 : passed / rows.length
+  };
+}
+function escapeCell(s) {
+  return s.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
+}
+function truncate2(s, max) {
+  return s.length <= max ? s : `${s.slice(0, Math.max(0, max - 1))}\u2026`;
+}
+function renderScoreboardMarkdown(rows, opts = {}) {
+  const maxEv = opts.maxEvidenceChars ?? 160;
+  const sum = scoreboardSummary(rows);
+  const pct = (n) => `${Math.round(n * 100)}%`;
+  const ev = (e) => escapeCell(truncate2(e.join("; "), maxEv)) || "\u2014";
+  const out = [`# ${opts.title ?? "Product-flow playback scoreboard"}`, ""];
+  if (opts.meta) {
+    for (const [k, v] of Object.entries(opts.meta)) out.push(`- **${k}:** ${v}`);
+    out.push("");
+  }
+  out.push(
+    `**${sum.storiesFullyComplete}/${sum.stories}** user stories fully shipped \xB7 **${sum.passed}/${sum.requirements}** requirements passing (${pct(sum.passRate)}) \xB7 **${sum.failed}** open`,
+    ""
+  );
+  const fails = rows.filter((r) => r.status === "FAIL");
+  if (fails.length > 0) {
+    out.push("## Open tickets", "", "| Story | Requirement | Evidence |", "| --- | --- | --- |");
+    for (const r of fails) {
+      out.push(`| ${escapeCell(r.storyTitle)} | ${escapeCell(r.reqTitle)} | ${ev(r.evidence)} |`);
+    }
+    out.push("");
+  } else {
+    out.push("_All requirements passing \u2014 no open tickets._", "");
+  }
+  out.push("## Per-story tick-off", "");
+  for (const storyId of [...new Set(rows.map((r) => r.storyId))]) {
+    const storyRows = rows.filter((r) => r.storyId === storyId);
+    const passed = storyRows.filter((r) => r.status === "PASS").length;
+    const mark = passed === storyRows.length ? "\u2705" : "\u26A0\uFE0F";
+    out.push(
+      `### ${escapeCell(storyRows[0].storyTitle)} \u2014 ${passed}/${storyRows.length} ${mark}`,
+      "",
+      "| Requirement | Status | Evidence |",
+      "| --- | --- | --- |"
+    );
+    for (const r of storyRows) {
+      out.push(
+        `| ${escapeCell(r.reqTitle)} | ${r.status === "PASS" ? "\u2705 PASS" : "\u274C FAIL"} | ${ev(r.evidence)} |`
+      );
+    }
+    out.push("");
+  }
+  return out.join("\n");
+}
+
 // src/campaign/presets/run-profile-matrix.ts
 import { createHash as createHash2 } from "crypto";
-import { join as join2 } from "path";
+import { join as join4 } from "path";
 var ProfileMatrixError = class extends AgentEvalError {
   constructor(message) {
     super("profile_matrix", message);
@@ -989,7 +1623,7 @@ async function runProfileMatrix(opts) {
       captureSource: opts.captureSource,
       storage: opts.storage,
       now: opts.now,
-      runDir: join2(opts.runDir, sanitize(profile.id))
+      runDir: join4(opts.runDir, sanitize(profile.id))
     });
     const profileRecords = [];
     for (const cell of campaign.cells) {
@@ -1061,7 +1695,7 @@ function rollupByPersona(records, scenarios, personaOf) {
 // src/campaign/worktree/index.ts
 import { execFileSync } from "child_process";
 import { existsSync as existsSync2 } from "fs";
-import { basename, isAbsolute, join as join3 } from "path";
+import { basename, isAbsolute, join as join5 } from "path";
 var WorktreeAdapterError = class extends Error {
   constructor(message, cause) {
     super(message);
@@ -1083,13 +1717,13 @@ function slug2(label) {
 }
 function gitWorktreeAdapter(opts) {
   const git = opts.git ?? defaultGit;
-  const worktreeDir = opts.worktreeDir ?? join3(opts.repoRoot, ".worktrees");
+  const worktreeDir = opts.worktreeDir ?? join5(opts.repoRoot, ".worktrees");
   const branchPrefix = opts.branchPrefix ?? "improve";
   return {
     async create({ baseRef, label }) {
       const id = `${slug2(label)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
       const branch = `${branchPrefix}/${id}`;
-      const path = join3(worktreeDir, id);
+      const path = join5(worktreeDir, id);
       git(["worktree", "add", "-b", branch, path, baseRef], opts.repoRoot);
       return { path, branch, baseRef };
     },
@@ -1114,16 +1748,20 @@ function gitWorktreeAdapter(opts) {
 }
 function resolveWorktreePath(surface, worktreeDir) {
   if (isAbsolute(surface.worktreeRef) && existsSync2(surface.worktreeRef)) return surface.worktreeRef;
-  if (worktreeDir) return join3(worktreeDir, basename(surface.worktreeRef));
+  if (worktreeDir) return join5(worktreeDir, basename(surface.worktreeRef));
   return surface.worktreeRef;
 }
 export {
+  DRIVER_GUIDE,
   FsLabeledScenarioStore,
   LabeledScenarioStoreError,
   ProfileMatrixError,
   SkillPatchParseError,
   WorktreeAdapterError,
+  aceDriver,
   applySkillPatch,
+  buildAnalystSurfaceDispatch,
+  buildEvidenceVector,
   buildLoopProvenanceRecord,
   campaignBreakdown,
   campaignMeanComposite,
@@ -1137,23 +1775,30 @@ export {
   emitLoopProvenance,
   evolutionaryDriver,
   extractH2Sections,
+  failureModeRecallJudge,
   fsCampaignStorage,
   gepaDriver,
   gepaParetoEntry,
   gepaReflectionEntry,
   gitWorktreeAdapter,
+  haloDriver,
   heldOutGate,
   heldoutSignificance,
   inMemoryCampaignStorage,
   isProposedCandidate,
   labelTrustRank,
   loopProvenanceSpans,
+  makePlaybackDispatch,
+  memoryCurationDriver,
   openAutoPr,
   pairHoldout,
+  paretoPolicy,
+  paretoSignificanceGate,
   parseSkillPatchResponse,
   patchEditCount,
   provenanceRecordPath,
   provenanceSpansPath,
+  renderScoreboardMarkdown,
   resolveWorktreePath,
   runCampaign,
   runEval,
@@ -1161,9 +1806,14 @@ export {
   runOptimization,
   runProfileMatrix,
   runSkillOpt,
+  scoreUserStory,
+  scoreboardSummary,
+  selectDriver,
   skillOptDriver,
   skillOptEntry,
   surfaceContentHash,
-  surfaceHash
+  surfaceHash,
+  traceAnalystDriver,
+  userStoryScoreboard
 };
 //# sourceMappingURL=index.js.map