@tangle-network/agent-eval 0.79.0 → 0.81.0

package/dist/campaign/index.js

@@ -10,14 +10,12 @@ import {
   paretoPolicy,
   paretoSignificanceGate,
   runEval
-} from "../chunk-GXHLRXDI.js";
+} from "../chunk-OTYQPHPL.js";
 import {
   agentProfileHash,
-  estimateCost,
   extractProducedState,
-  isModelPriced,
   verifyCompletion
-} from "../chunk-LB2UOI5F.js";
+} from "../chunk-YGYXHNAQ.js";
 import {
   buildLoopProvenanceRecord,
   campaignBreakdown,
@@ -39,26 +37,30 @@ import {
   runOptimization,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-RPLZ4OIB.js";
+} from "../chunk-BABOZOSN.js";
 import {
   assertRealBackend,
   fsCampaignStorage,
   inMemoryCampaignStorage,
   runCampaign,
   summarizeBackendIntegrity
-} from "../chunk-ZPSKPT3V.js";
+} from "../chunk-ZZ2HOPME.js";
+import {
+  estimateCost,
+  isModelPriced
+} from "../chunk-VI2UW6B6.js";
 import {
   AnalystRegistry,
   DEFAULT_TRACE_ANALYST_KINDS,
   createTraceAnalystKind
-} from "../chunk-CF67I6QY.js";
+} from "../chunk-VIDQF3F5.js";
 import "../chunk-YV7J7X5N.js";
 import {
   callLlm
-} from "../chunk-IHDHUN2X.js";
+} from "../chunk-CVVHBFGN.js";
 import {
   pairedBootstrap
-} from "../chunk-ITBRCT73.js";
+} from "../chunk-IDVBLYCY.js";
 import "../chunk-GGE4NNQT.js";
 import {
   OtlpFileTraceStore,
@@ -155,16 +157,25 @@ function surfaceToText2(surface) {
     `curator driver: surface must be a string prompt, got a ${surface.kind}-tier surface (${surface.worktreeRef}) \u2014 curation is prompt-tier`
   );
 }
+function extractBlockBody(text, startMarker, endMarker) {
+  const start = text.indexOf(startMarker);
+  const end = text.indexOf(endMarker);
+  if (start === -1 || end === -1 || end < start) return "";
+  return text.slice(start + startMarker.length, end);
+}
+function stripBlock(text, startMarker, endMarker) {
+  const start = text.indexOf(startMarker);
+  const end = text.indexOf(endMarker);
+  if (start === -1 || end === -1 || end < start) return text.trimEnd();
+  return (text.slice(0, start) + text.slice(end + endMarker.length)).trimEnd();
+}
 
 // src/campaign/drivers/ace.ts
 var BLOCK_START = "<!-- BEGIN ace-playbook (auto-managed by aceDriver) -->";
 var BLOCK_END = "<!-- END ace-playbook -->";
 var DEFAULT_HEADING = "## Playbook (accumulated lessons \u2014 append-only)";
 function parsePlaybook(surface) {
-  const start = surface.indexOf(BLOCK_START);
-  const end = surface.indexOf(BLOCK_END);
-  if (start === -1 || end === -1 || end < start) return [];
-  const body = surface.slice(start + BLOCK_START.length, end);
+  const body = extractBlockBody(surface, BLOCK_START, BLOCK_END);
   const out = [];
   for (const raw of body.split("\n")) {
     const line = raw.trim();
@@ -176,12 +187,6 @@ function parsePlaybook(surface) {
   }
   return out;
 }
-function stripBlock(surface) {
-  const start = surface.indexOf(BLOCK_START);
-  const end = surface.indexOf(BLOCK_END);
-  if (start === -1 || end === -1 || end < start) return surface.trimEnd();
-  return (surface.slice(0, start) + surface.slice(end + BLOCK_END.length)).trimEnd();
-}
 function aceDriver(opts = {}) {
   const maxEntries = opts.maxEntries ?? 50;
   if (maxEntries < 1) throw new Error("aceDriver: maxEntries must be >= 1");
@@ -209,7 +214,7 @@ function aceDriver(opts = {}) {
         ...all.map((b) => `- [g${b.gen}] ${b.text}`),
         BLOCK_END
       ].join("\n");
-      const base = stripBlock(parent);
+      const base = stripBlock(parent, BLOCK_START, BLOCK_END);
       const surface = base ? `${base}
 
 ${block}` : block;
@@ -224,115 +229,75 @@ ${block}` : block;
   };
 }
 
-// src/campaign/drivers/guide.ts
-var DRIVER_GUIDE = {
-  gepa: {
-    summary: "Reflective full-surface rewrite: reflects on the best parent\u2019s weakest dimensions + per-scenario scores, proposes targeted rewrites, maintains a Pareto frontier across generations.",
-    surface: "prompt",
-    strategy: "reflective-rewrite",
-    whenUse: "The default for a prompt/instruction surface with headroom \u2014 broad rewrites plus Pareto-optimal exploration across scenarios.",
-    cost: "medium"
-  },
-  skillOpt: {
-    summary: "Patch-mode: bounded, anchored add/delete/replace edits to ONE skill document, so a good rule introduced earlier is not clobbered by a later sweeping rewrite.",
-    surface: "skill-doc",
-    strategy: "anchored-patch",
-    whenUse: 'Refining a skill document incrementally where accumulated rules must be preserved; the edit budget is the "textual learning rate".',
-    cost: "medium"
-  },
-  ace: {
-    summary: "Append-mostly playbook curator: grows the playbook with provenance-tagged delta bullets, never merging \u2014 guards against context collapse.",
-    surface: "playbook",
-    strategy: "append-only",
-    whenUse: "Accumulating many specific, hard-won lessons over time where dedup/rewrite would summarize away detail.",
-    cost: "low"
-  },
-  memoryCuration: {
-    summary: "Dedup-and-rank curator: builds a compact searchable memory and grafts the most relevant, most-recurrent lessons onto the surface.",
-    surface: "memory",
-    strategy: "dedup-curate",
-    whenUse: "Accumulating lessons while keeping the surface compact \u2014 the complement to ace when context size matters more than verbatim provenance.",
-    cost: "low"
-  },
-  halo: {
-    summary: "Wraps the real external HALO engine (Inference.net, `halo` CLI) and applies its findings to the prompt via one LLM edit.",
-    surface: "prompt",
-    strategy: "analysis-edit",
-    whenUse: "Benchmarking: compete HALO head-to-head against our own analysis on identical traces via compareDrivers.",
-    cost: "high",
-    external: true
-  },
-  traceAnalyst: {
-    summary: "Wraps agent-eval\u2019s own trace-analyst engine and applies its findings to the prompt via one identical LLM edit \u2014 the symmetric opponent to haloDriver.",
-    surface: "prompt",
-    strategy: "analysis-edit",
-    whenUse: "Benchmarking our trace-analyst\u2019s analysis quality against HALO (analysis-quality head-to-head), or improving from a real OTLP trace corpus.",
-    cost: "high"
-  },
-  evolutionary: {
-    summary: "Adapts a stateless Mutator (population mutate \u2192 measure \u2192 select); no generation memory beyond the current surface.",
-    surface: "any",
-    strategy: "population-mutate",
-    whenUse: "Blind population search when you have a Mutator and don\u2019t need reflective reasoning over findings.",
-    cost: "medium"
-  }
-};
-var GOAL_RANK = {
-  explore: ["gepa", "evolutionary"],
-  refine: ["skillOpt", "gepa"],
-  accumulate: ["ace", "memoryCuration"],
-  benchmark: ["traceAnalyst", "halo"]
-};
-function selectDriver(criteria) {
-  const ranked = GOAL_RANK[criteria.goal];
-  const out = [];
-  for (const name of ranked) {
-    const entry = DRIVER_GUIDE[name];
-    if (criteria.surface && criteria.surface !== "any" && entry.surface !== criteria.surface)
-      continue;
-    out.push({
-      name,
-      entry,
-      reason: `${criteria.goal}: ${entry.strategy} on the ${entry.surface} surface \u2014 ${entry.whenUse}`
-    });
-  }
-  if (out.length === 0 && criteria.surface) {
-    for (const name of Object.keys(DRIVER_GUIDE)) {
-      const entry = DRIVER_GUIDE[name];
-      if (entry.surface === criteria.surface || entry.surface === "any") {
-        out.push({ name, entry, reason: `surface match (${entry.surface}): ${entry.whenUse}` });
-      }
-    }
-  }
-  return out;
-}
-
 // src/campaign/drivers/halo.ts
 import { execFile } from "child_process";
+import { promisify } from "util";
+
+// src/campaign/drivers/analysis-edit.ts
 import { mkdtempSync, writeFileSync } from "fs";
 import { tmpdir } from "os";
 import { join } from "path";
-import { promisify } from "util";
-var execFileAsync = promisify(execFile);
-var DEFAULT_ANALYSIS_PROMPT = "Diagnose the failures in these agent execution traces \u2014 hallucinated tool calls, redundant tool arguments, refusal loops, and semantic-correctness errors \u2014 and suggest concrete, generalizable fixes to the agent instructions.";
 var APPLY_SYSTEM = "You apply a trace-analysis report to an agent instruction prompt. Output ONLY the full revised prompt \u2014 no preamble, no commentary, no code fences. Make the minimal edits that address the report findings; preserve everything else verbatim.";
-function haloDriver(opts) {
-  const haloBin = opts.haloBin ?? "halo";
-  const model = opts.model ?? "gpt-5.4-mini";
+function surfaceToPromptText(surface) {
+  return typeof surface === "string" ? surface : JSON.stringify(surface);
+}
+function analysisEditDriver(opts) {
   return {
-    kind: "halo",
+    kind: opts.kind,
     async propose(ctx) {
-      const parent = typeof ctx.currentSurface === "string" ? ctx.currentSurface : JSON.stringify(ctx.currentSurface);
+      const parent = surfaceToPromptText(ctx.currentSurface);
       const traces = await opts.resolveTraces(ctx) ?? "";
-      if (!traces.trim()) {
-        throw new Error(
-          "haloDriver: resolveTraces returned no OTLP traces \u2014 the halo engine has nothing to analyze"
-        );
-      }
-      const dir = mkdtempSync(join(tmpdir(), "halo-driver-"));
+      if (!traces.trim()) throw new Error(opts.noTracesError);
+      const dir = mkdtempSync(join(tmpdir(), `${opts.kind}-driver-`));
       const tracePath = join(dir, "traces.jsonl");
       writeFileSync(tracePath, traces.endsWith("\n") ? traces : `${traces}
 `);
+      const report = await opts.analyze(tracePath, ctx);
+      const applied = await callLlm(
+        {
+          model: opts.applyModel,
+          messages: [
+            { role: "system", content: APPLY_SYSTEM },
+            {
+              role: "user",
+              content: `CURRENT PROMPT:
+${parent}
+
+TRACE-ANALYSIS REPORT:
+${report}
+
+Return the full revised prompt.`
+            }
+          ]
+        },
+        { baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl }
+      );
+      const text = applied.content.trim();
+      if (!text || text === parent) return [];
+      return [{ surface: text, label: opts.label, rationale: opts.rationale(report) }];
+    }
+  };
+}
+
+// src/campaign/drivers/halo.ts
+var execFileAsync = promisify(execFile);
+var DEFAULT_ANALYSIS_PROMPT = "Diagnose the failures in these agent execution traces \u2014 hallucinated tool calls, redundant tool arguments, refusal loops, and semantic-correctness errors \u2014 and suggest concrete, generalizable fixes to the agent instructions.";
+function haloDriver(opts) {
+  const haloBin = opts.haloBin ?? "halo";
+  const model = opts.model ?? "gpt-5.4-mini";
+  return analysisEditDriver({
+    kind: "halo",
+    label: "halo",
+    baseUrl: opts.baseUrl,
+    apiKey: opts.apiKey,
+    applyModel: opts.applyModel ?? model,
+    fetchImpl: opts.fetchImpl,
+    resolveTraces: opts.resolveTraces,
+    noTracesError: "haloDriver: resolveTraces returned no OTLP traces \u2014 the halo engine has nothing to analyze",
+    // HALO's real findings are preserved verbatim in the rationale (attribution).
+    rationale: (findings) => `halo-engine findings:
+${findings.slice(0, 800)}`,
+    analyze: async (tracePath, ctx) => {
       const args = [
         tracePath,
         "-p",
@@ -360,37 +325,9 @@ function haloDriver(opts) {
         );
       }
       if (!findings) throw new Error("haloDriver: halo-engine produced no findings");
-      const applied = await callLlm(
-        {
-          model: opts.applyModel ?? model,
-          messages: [
-            { role: "system", content: APPLY_SYSTEM },
-            {
-              role: "user",
-              content: `CURRENT PROMPT:
-${parent}
-
-HALO TRACE-ANALYSIS REPORT:
-${findings}
-
-Return the full revised prompt.`
-            }
-          ]
-        },
-        { baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl }
-      );
-      const text = applied.content.trim();
-      if (!text || text === parent) return [];
-      return [
-        {
-          surface: text,
-          label: "halo",
-          rationale: `halo-engine findings:
-${findings.slice(0, 800)}`
-        }
-      ];
+      return findings;
     }
-  };
+  });
 }
 
 // src/campaign/drivers/memory.ts
@@ -399,16 +336,7 @@ var BLOCK_END2 = "<!-- END curated-memory -->";
 var DEFAULT_HEADING2 = "## Learned from prior runs (curated memory)";
 var DISTILL_SYSTEM = 'You compress raw trace-analysis findings into crisp, generalizable agent guidance. Output ONLY a JSON array of strings, each one imperative lesson the agent should follow (e.g. "Always fetch a resource before mutating it"). No prose outside the JSON. Deduplicate; keep the most actionable and general; drop case-specific noise.';
 function extractExistingLessons(text) {
-  const start = text.indexOf(BLOCK_START2);
-  const end = text.indexOf(BLOCK_END2);
-  if (start === -1 || end === -1 || end < start) return [];
-  return text.slice(start + BLOCK_START2.length, end).split("\n").map((l) => l.replace(/^\s*-\s+/, "").trim()).filter((l) => l && !l.startsWith("#"));
-}
-function stripBlock2(text) {
-  const start = text.indexOf(BLOCK_START2);
-  const end = text.indexOf(BLOCK_END2);
-  if (start === -1 || end === -1 || end < start) return text.trimEnd();
-  return (text.slice(0, start) + text.slice(end + BLOCK_END2.length)).trimEnd();
+  return extractBlockBody(text, BLOCK_START2, BLOCK_END2).split("\n").map((l) => l.replace(/^\s*-\s+/, "").trim()).filter((l) => l && !l.startsWith("#"));
 }
 async function distillLessons(raw, distill) {
   const res = await callLlm(
@@ -466,7 +394,7 @@ function memoryCurationDriver(opts = {}) {
       const block = [BLOCK_START2, heading, ...ranked.map((e) => `- ${e.text}`), BLOCK_END2].join(
         "\n"
       );
-      const next = `${stripBlock2(parent)}
+      const next = `${stripBlock(parent, BLOCK_START2, BLOCK_END2)}
 
 ${block}
 `;
@@ -721,11 +649,7 @@ function snippet(s, max = 120) {
 }
 
 // src/campaign/drivers/trace-analyst.ts
-import { mkdtempSync as mkdtempSync2, writeFileSync as writeFileSync2 } from "fs";
-import { tmpdir as tmpdir2 } from "os";
-import { join as join2 } from "path";
 import { ai } from "@ax-llm/ax";
-var APPLY_SYSTEM2 = "You apply a trace-analysis report to an agent instruction prompt. Output ONLY the full revised prompt \u2014 no preamble, no commentary, no code fences. Make the minimal edits that address the report findings; preserve everything else verbatim.";
 function renderFindings(findings) {
   return findings.map((f, i) => {
     const action = f.recommended_action ? `
@@ -738,41 +662,39 @@ function traceAnalystDriver(opts) {
   if (!opts.apiKey) throw new Error("traceAnalystDriver: apiKey is required");
   if (!opts.model) throw new Error("traceAnalystDriver: model is required");
   const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS;
-  return {
+  const produceFindings = opts.analyze ?? (async (path, c) => {
+    const aiService = ai({
+      name: opts.provider ?? "openai",
+      apiKey: opts.apiKey,
+      apiURL: opts.baseUrl,
+      config: { model: opts.model }
+    });
+    const registry = new AnalystRegistry();
+    for (const spec of kinds) {
+      registry.register(createTraceAnalystKind(spec, { ai: aiService, model: opts.model }));
+    }
+    const result = await registry.run(
+      `trace-analyst-gen-${c.generation}`,
+      { traceStore: new OtlpFileTraceStore({ path }) },
+      { signal: c.signal }
+    );
+    return result.findings;
+  });
+  return analysisEditDriver({
     kind: "trace-analyst",
-    async propose(ctx) {
-      const parent = typeof ctx.currentSurface === "string" ? ctx.currentSurface : JSON.stringify(ctx.currentSurface);
-      const traces = await opts.resolveTraces(ctx) ?? "";
-      if (!traces.trim()) {
-        throw new Error(
-          "traceAnalystDriver: resolveTraces returned no OTLP traces \u2014 the analyst has nothing to read"
-        );
-      }
-      const dir = mkdtempSync2(join2(tmpdir2(), "trace-analyst-driver-"));
-      const tracePath = join2(dir, "traces.jsonl");
-      writeFileSync2(tracePath, traces.endsWith("\n") ? traces : `${traces}
-`);
-      const runAnalyze = opts.analyze ?? (async (path, c) => {
-        const aiService = ai({
-          name: opts.provider ?? "openai",
-          apiKey: opts.apiKey,
-          apiURL: opts.baseUrl,
-          config: { model: opts.model }
-        });
-        const registry = new AnalystRegistry();
-        for (const spec of kinds) {
-          registry.register(createTraceAnalystKind(spec, { ai: aiService, model: opts.model }));
-        }
-        const result = await registry.run(
-          `trace-analyst-gen-${c.generation}`,
-          { traceStore: new OtlpFileTraceStore({ path }) },
-          { signal: c.signal }
-        );
-        return result.findings;
-      });
+    label: "trace-analyst",
+    baseUrl: opts.baseUrl,
+    apiKey: opts.apiKey,
+    applyModel: opts.applyModel ?? opts.model,
+    fetchImpl: opts.fetchImpl,
+    resolveTraces: opts.resolveTraces,
+    noTracesError: "traceAnalystDriver: resolveTraces returned no OTLP traces \u2014 the analyst has nothing to read",
+    rationale: (report) => `trace-analyst findings:
+${report.slice(0, 800)}`,
+    analyze: async (tracePath, ctx) => {
       let findings;
       try {
-        findings = await runAnalyze(tracePath, ctx);
+        findings = await produceFindings(tracePath, ctx);
       } catch (e) {
         throw new Error(
           `traceAnalystDriver: analyst engine failed \u2014 ${e instanceof Error ? e.message : String(e)}`
@@ -781,44 +703,15 @@ function traceAnalystDriver(opts) {
       if (findings.length === 0) {
         throw new Error("traceAnalystDriver: analyst engine produced no findings");
       }
-      const report = renderFindings(findings);
-      const applied = await callLlm(
-        {
-          model: opts.applyModel ?? opts.model,
-          messages: [
-            { role: "system", content: APPLY_SYSTEM2 },
-            {
-              role: "user",
-              content: `CURRENT PROMPT:
-${parent}
-
-TRACE-ANALYSIS REPORT:
-${report}
-
-Return the full revised prompt.`
-            }
-          ]
-        },
-        { baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl }
-      );
-      const text = applied.content.trim();
-      if (!text || text === parent) return [];
-      return [
-        {
-          surface: text,
-          label: "trace-analyst",
-          rationale: `trace-analyst findings (${findings.length}):
-${report.slice(0, 800)}`
-        }
-      ];
+      return renderFindings(findings);
     }
-  };
+  });
 }
 
 // src/campaign/labeled-store/fs-adapter.ts
 import { createHash } from "crypto";
-import { existsSync, mkdirSync, readFileSync, writeFileSync as writeFileSync3 } from "fs";
-import { join as join3 } from "path";
+import { existsSync, mkdirSync, readFileSync, writeFileSync as writeFileSync2 } from "fs";
+import { join as join2 } from "path";
 var LabeledScenarioStoreError = class extends Error {
   constructor(code, message) {
     super(message);
@@ -978,7 +871,7 @@ var FsLabeledScenarioStore = class {
     };
   }
   pathForSource(source) {
-    return join3(this.options.root, `${source}.jsonl`);
+    return join2(this.options.root, `${source}.jsonl`);
   }
 };
 var ALL_SOURCES = [
@@ -1020,9 +913,9 @@ function sha256(input) {
 function appendLine(path, line) {
   if (existsSync(path)) {
     const existing = readFileSync(path, "utf8");
-    writeFileSync3(path, existing + line);
+    writeFileSync2(path, existing + line);
   } else {
-    writeFileSync3(path, line);
+    writeFileSync2(path, line);
   }
 }
 
@@ -1472,7 +1365,7 @@ function renderScoreboardMarkdown(rows, opts = {}) {
 
 // src/campaign/presets/run-profile-matrix.ts
 import { createHash as createHash2 } from "crypto";
-import { join as join4 } from "path";
+import { join as join3 } from "path";
 var ProfileMatrixError = class extends AgentEvalError {
   constructor(message) {
     super("profile_matrix", message);
@@ -1623,7 +1516,7 @@ async function runProfileMatrix(opts) {
       captureSource: opts.captureSource,
       storage: opts.storage,
       now: opts.now,
-      runDir: join4(opts.runDir, sanitize(profile.id))
+      runDir: join3(opts.runDir, sanitize(profile.id))
     });
     const profileRecords = [];
     for (const cell of campaign.cells) {
@@ -1695,7 +1588,7 @@ function rollupByPersona(records, scenarios, personaOf) {
 // src/campaign/worktree/index.ts
 import { execFileSync } from "child_process";
 import { existsSync as existsSync2 } from "fs";
-import { basename, isAbsolute, join as join5 } from "path";
+import { basename, isAbsolute, join as join4 } from "path";
 var WorktreeAdapterError = class extends Error {
   constructor(message, cause) {
     super(message);
@@ -1717,13 +1610,13 @@ function slug2(label) {
 }
 function gitWorktreeAdapter(opts) {
   const git = opts.git ?? defaultGit;
-  const worktreeDir = opts.worktreeDir ?? join5(opts.repoRoot, ".worktrees");
+  const worktreeDir = opts.worktreeDir ?? join4(opts.repoRoot, ".worktrees");
   const branchPrefix = opts.branchPrefix ?? "improve";
   return {
     async create({ baseRef, label }) {
       const id = `${slug2(label)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
       const branch = `${branchPrefix}/${id}`;
-      const path = join5(worktreeDir, id);
+      const path = join4(worktreeDir, id);
       git(["worktree", "add", "-b", branch, path, baseRef], opts.repoRoot);
       return { path, branch, baseRef };
     },
@@ -1748,11 +1641,10 @@ function gitWorktreeAdapter(opts) {
 }
 function resolveWorktreePath(surface, worktreeDir) {
   if (isAbsolute(surface.worktreeRef) && existsSync2(surface.worktreeRef)) return surface.worktreeRef;
-  if (worktreeDir) return join5(worktreeDir, basename(surface.worktreeRef));
+  if (worktreeDir) return join4(worktreeDir, basename(surface.worktreeRef));
   return surface.worktreeRef;
 }
 export {
-  DRIVER_GUIDE,
   FsLabeledScenarioStore,
   LabeledScenarioStoreError,
   ProfileMatrixError,
@@ -1808,7 +1700,6 @@ export {
   runSkillOpt,
   scoreUserStory,
   scoreboardSummary,
-  selectDriver,
   skillOptDriver,
   skillOptEntry,
   surfaceContentHash,