@tangle-network/agent-runtime 0.79.3 → 0.80.0

package/dist/{chunk-AG335EXG.js → chunk-QSO2TVDS.js}

@@ -12,8 +12,9 @@ import {
   routerToolLoop,
   runLoop,
   settledToIteration,
+  supervise,
   withDriverExecutor
-} from "./chunk-KRULXIWS.js";
+} from "./chunk-TPII5AU7.js";
 import {
   addTokenUsage,
   isAbortError,
@@ -21,7 +22,7 @@ import {
   sleep,
   stringifySafe,
   zeroTokenUsage
-} from "./chunk-Z3RRRPRB.js";
+} from "./chunk-H7IBHAFT.js";
 import {
   AnalystError,
   PlannerError,
@@ -205,6 +206,495 @@ Audit the route: revealed intent, verdict, evidence, one recommendation.`
   };
 }
 
+// src/runtime/benchmark-report.ts
+import {
+  benjaminiHochberg,
+  confidenceInterval,
+  pairedBootstrap,
+  pairedTTest,
+  wilson
+} from "@tangle-network/agent-eval";
+var defaultScoreOf = (r) => {
+  const o = r.outcome;
+  if (typeof o.holdoutScore === "number") return o.holdoutScore;
+  if (typeof o.searchScore === "number") return o.searchScore;
+  const raw = o.raw ?? {};
+  for (const k of ["composite", "score", "passed", "resolved"]) {
+    if (typeof raw[k] === "number") return raw[k];
+  }
+  return void 0;
+};
+var defaultProfileKeyOf = (r) => {
+  const cell = r.agentProfile;
+  const harness = cell?.harness;
+  return harness ? `${harness}\xB7${r.model}` : r.model;
+};
+var defaultGroupOf = (r) => r.scenarioId ?? r.experimentId;
+function quantile(sorted, q) {
+  if (sorted.length === 0) return 0;
+  const idx = Math.min(sorted.length - 1, Math.floor(q * (sorted.length - 1)));
+  return sorted[idx] ?? 0;
+}
+function mean(xs) {
+  return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
+}
+function meanByScenario(records, scoreOf) {
+  const sums = /* @__PURE__ */ new Map();
+  for (const r of records) {
+    const s = scoreOf(r);
+    if (typeof s !== "number") continue;
+    const id = r.scenarioId;
+    if (!id) {
+      throw new Error(
+        `benchmark-report: RunRecord (candidate ${r.candidateId ?? "unknown"}) is missing scenarioId \u2014 cannot pair or interval it honestly. Pass opts.stats only on a scenario-tagged corpus.`
+      );
+    }
+    const acc = sums.get(id) ?? { total: 0, n: 0 };
+    acc.total += s;
+    acc.n += 1;
+    sums.set(id, acc);
+  }
+  const out = /* @__PURE__ */ new Map();
+  for (const [id, acc] of sums) out.set(id, acc.n ? acc.total / acc.n : 0);
+  return out;
+}
+function leaderboard(records, opts = {}) {
+  const scoreOf = opts.scoreOf ?? defaultScoreOf;
+  const profileKeyOf = opts.profileKeyOf ?? defaultProfileKeyOf;
+  const groupOf = opts.groupOf ?? defaultGroupOf;
+  const labelOf = opts.labelOf ?? ((k) => k);
+  const axisSet = /* @__PURE__ */ new Set();
+  if (opts.axisScoresOf) {
+    for (const r of records) for (const k of Object.keys(opts.axisScoresOf(r))) axisSet.add(k);
+  } else {
+    for (const r of records) axisSet.add(groupOf(r));
+  }
+  const axes = [...axisSet].sort();
+  const byProfile = /* @__PURE__ */ new Map();
+  for (const r of records) {
+    const key = profileKeyOf(r);
+    const bucket = byProfile.get(key);
+    if (bucket) bucket.push(r);
+    else byProfile.set(key, [r]);
+  }
+  const rows = [];
+  for (const [profileKey, recs] of byProfile) {
+    const scores = recs.map(scoreOf).filter((s) => typeof s === "number");
+    const axisBuckets = /* @__PURE__ */ new Map();
+    for (const r of recs) {
+      if (opts.axisScoresOf) {
+        for (const [axis, s] of Object.entries(opts.axisScoresOf(r))) {
+          const b = axisBuckets.get(axis) ?? [];
+          b.push(s);
+          axisBuckets.set(axis, b);
+        }
+      } else {
+        const s = scoreOf(r);
+        if (typeof s === "number") {
+          const axis = groupOf(r);
+          const b = axisBuckets.get(axis) ?? [];
+          b.push(s);
+          axisBuckets.set(axis, b);
+        }
+      }
+    }
+    const perAxis = {};
+    for (const [axis, b] of axisBuckets) perAxis[axis] = mean(b);
+    const latencies = recs.map((r) => r.wallMs).sort((a, b) => a - b);
+    const pass = opts.passThreshold ?? 0.999;
+    let scoreCi;
+    let passCi;
+    if (opts.stats) {
+      const collapsed = [...meanByScenario(recs, scoreOf).values()];
+      if (collapsed.length > 0) {
+        const ci2 = confidenceInterval(collapsed, 0.95, { seed: 7 });
+        scoreCi = { lower: ci2.lower, upper: ci2.upper };
+        const w = wilson(collapsed.filter((s) => s >= pass).length, collapsed.length, 0.95);
+        passCi = { lower: w.lower, upper: w.upper };
+      }
+    }
+    rows.push({
+      profileKey,
+      label: labelOf(profileKey),
+      model: recs[0]?.model ?? profileKey,
+      n: recs.length,
+      meanScore: mean(scores),
+      solveRate: scores.length === 0 ? 0 : scores.filter((s) => s >= pass).length / scores.length,
+      perAxis,
+      costUsd: recs.reduce((a, r) => a + r.costUsd, 0),
+      tokensIn: recs.reduce((a, r) => a + (r.tokenUsage?.input ?? 0), 0),
+      tokensOut: recs.reduce((a, r) => a + (r.tokenUsage?.output ?? 0), 0),
+      latencyP50Ms: quantile(latencies, 0.5),
+      latencyP90Ms: quantile(latencies, 0.9),
+      ...scoreCi ? { scoreCi } : {},
+      ...passCi ? { passCi } : {}
+    });
+  }
+  rows.sort(
+    (a, b) => b.meanScore - a.meanScore || a.costUsd - b.costUsd || a.label.localeCompare(b.label)
+  );
+  const models = [...new Set(records.map((r) => r.model))].sort();
+  return {
+    title: opts.title ?? "Benchmark report",
+    axes,
+    profiles: rows,
+    meta: opts.meta ?? {},
+    provenance: {
+      records: records.length,
+      profiles: rows.length,
+      axes: axes.length,
+      models,
+      totalCostUsd: records.reduce((a, r) => a + r.costUsd, 0)
+    }
+  };
+}
+function pairwiseSignificance(records, opts = {}) {
+  const scoreOf = opts.scoreOf ?? defaultScoreOf;
+  const profileKeyOf = opts.profileKeyOf ?? defaultProfileKeyOf;
+  const labelOf = opts.labelOf ?? ((k) => k);
+  const minPairs = opts.minPairs ?? 12;
+  const byProfile = /* @__PURE__ */ new Map();
+  for (const r of records) {
+    const k = profileKeyOf(r);
+    const b = byProfile.get(k);
+    if (b) b.push(r);
+    else byProfile.set(k, [r]);
+  }
+  const keys = [...byProfile.keys()].sort();
+  const collapsed = new Map(keys.map((k) => [k, meanByScenario(byProfile.get(k) ?? [], scoreOf)]));
+  const raw = [];
+  for (let i = 0; i < keys.length; i += 1) {
+    for (let j = i + 1; j < keys.length; j += 1) {
+      const ka = keys[i];
+      const kb = keys[j];
+      const am = collapsed.get(ka);
+      const bm = collapsed.get(kb);
+      const aScores = [];
+      const bScores = [];
+      for (const sid of [...am.keys()].sort()) {
+        const bv = bm.get(sid);
+        if (bv !== void 0) {
+          aScores.push(am.get(sid));
+          bScores.push(bv);
+        }
+      }
+      if (aScores.length === 0) continue;
+      const boot = pairedBootstrap(aScores, bScores, { seed: 7, statistic: "median" });
+      const p = pairedTTest(aScores, bScores).p;
+      raw.push({
+        a: labelOf(ka),
+        b: labelOf(kb),
+        pairs: aScores.length,
+        delta: boot.median,
+        ciLow: boot.low,
+        ciHigh: boot.high,
+        p
+      });
+    }
+  }
+  const { significant } = benjaminiHochberg(
+    raw.map((r) => r.p),
+    opts.fdr ?? 0.05
+  );
+  return raw.map((r, i) => ({
+    ...r,
+    significant: (significant[i] ?? false) && r.pairs >= minPairs
+  }));
+}
+var pct = (x) => `${(100 * x).toFixed(1)}%`;
+var ci = (iv) => iv ? ` [${pct(iv.lower)}, ${pct(iv.upper)}]` : "";
+function renderLeaderboardMarkdown(report) {
+  const lines = [];
+  lines.push(`# ${report.title}`, "");
+  const p = report.provenance;
+  lines.push(
+    `**${p.profiles} profiles \xD7 ${p.axes} axes**, ${p.records} runs, ${p.models.length} models \xB7 total $${p.totalCostUsd.toFixed(2)}`,
+    ""
+  );
+  for (const [k, v] of Object.entries(report.meta)) lines.push(`- **${k}:** ${v}`);
+  if (Object.keys(report.meta).length) lines.push("");
+  lines.push("## Leaderboard", "");
+  lines.push(
+    "| # | Profile | Score (95% CI) | Solved (95% CI) | Runs | Cost | Tok in/out | p50 | p90 |"
+  );
+  lines.push("|---|---|--:|--:|--:|--:|--:|--:|--:|");
+  report.profiles.forEach((r, i) => {
+    lines.push(
+      `| ${i + 1} | ${r.label} | ${pct(r.meanScore)}${ci(r.scoreCi)} | ${pct(r.solveRate)}${ci(r.passCi)} | ${r.n} | $${r.costUsd.toFixed(3)} | ${r.tokensIn}/${r.tokensOut} | ${(r.latencyP50Ms / 1e3).toFixed(1)}s | ${(r.latencyP90Ms / 1e3).toFixed(1)}s |`
+    );
+  });
+  lines.push("");
+  lines.push("## Score matrix \u2014 profile \xD7 axis", "");
+  lines.push(`| Profile | ${report.axes.join(" | ")} |`);
+  lines.push(`|---|${report.axes.map(() => "--:").join("|")}|`);
+  for (const r of report.profiles) {
+    const cells = report.axes.map((a) => {
+      const v = r.perAxis[a];
+      return v === void 0 ? "\xB7" : pct(v);
+    });
+    lines.push(`| ${r.label} | ${cells.join(" | ")} |`);
+  }
+  lines.push("");
+  lines.push("> `\xB7` = the profile never ran that axis (blank, never zero).");
+  return lines.join("\n");
+}
+function renderPairwiseMarkdown(verdicts, title = "Pairwise significance (paired, BH-corrected)") {
+  const lines = [`## ${title}`, ""];
+  if (verdicts.length === 0) return lines.concat("_no comparable pairs_").join("\n");
+  lines.push("| A vs B | \u0394(b\u2212a) median | 95% CI | pairs | p | verdict |");
+  lines.push("|---|--:|--:|--:|--:|---|");
+  for (const v of verdicts) {
+    const verdict = v.significant ? `**${v.delta >= 0 ? v.b : v.a} wins**` : "ns";
+    lines.push(
+      `| ${v.a} vs ${v.b} | ${v.delta >= 0 ? "+" : ""}${pct(v.delta)} | [${pct(v.ciLow)}, ${pct(v.ciHigh)}] | ${v.pairs} | ${v.p.toFixed(3)} | ${verdict} |`
+    );
+  }
+  lines.push(
+    "",
+    "> `ns` = not significant after Benjamini\u2013Hochberg (or below the paired-count floor)."
+  );
+  return lines.join("\n");
+}
+function esc(s) {
+  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+function ramp(score) {
+  const x = Math.max(0, Math.min(1, score));
+  const r = Math.round(x < 0.5 ? 220 : 220 - (x - 0.5) * 2 * 160);
+  const g = Math.round(x < 0.5 ? x * 2 * 170 : 170);
+  return `rgb(${r},${g},60)`;
+}
+function renderLeaderboardSvg(report) {
+  const rowH = 26;
+  const labelW = 220;
+  const cellW = 54;
+  const pad = 16;
+  const barAreaW = 360;
+  const profiles = report.profiles;
+  const barsH = profiles.length * rowH + pad;
+  const heatTop = barsH + 48;
+  const heatW = labelW + report.axes.length * cellW + pad;
+  const heatH = profiles.length * rowH + pad;
+  const width = Math.max(labelW + barAreaW + pad * 2, heatW);
+  const height = heatTop + heatH + pad;
+  const out = [];
+  out.push(
+    `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" font-family="ui-sans-serif,system-ui,sans-serif" font-size="12">`
+  );
+  out.push(`<rect width="${width}" height="${height}" fill="white"/>`);
+  out.push(
+    `<text x="${pad}" y="${pad + 4}" font-weight="700" font-size="14">${esc(report.title)} \u2014 score</text>`
+  );
+  profiles.forEach((r, i) => {
+    const y = pad + 12 + i * rowH;
+    out.push(`<text x="${pad}" y="${y + 14}">${esc(r.label)}</text>`);
+    const w = Math.round(r.meanScore * barAreaW);
+    out.push(`<rect x="${labelW}" y="${y + 4}" width="${barAreaW}" height="16" fill="#eef0f2"/>`);
+    out.push(
+      `<rect x="${labelW}" y="${y + 4}" width="${w}" height="16" fill="${ramp(r.meanScore)}"/>`
+    );
+    out.push(`<text x="${labelW + barAreaW + 6}" y="${y + 16}">${pct(r.meanScore)}</text>`);
+  });
+  out.push(
+    `<text x="${pad}" y="${heatTop - 12}" font-weight="700" font-size="14">profile \xD7 axis</text>`
+  );
+  report.axes.forEach((a, c) => {
+    const x = labelW + c * cellW;
+    out.push(
+      `<text x="${x + cellW / 2}" y="${heatTop}" text-anchor="middle" fill="#555">${esc(a.length > 8 ? `${a.slice(0, 7)}\u2026` : a)}</text>`
+    );
+  });
+  profiles.forEach((r, i) => {
+    const y = heatTop + 8 + i * rowH;
+    out.push(`<text x="${pad}" y="${y + 16}">${esc(r.label)}</text>`);
+    report.axes.forEach((a, c) => {
+      const x = labelW + c * cellW;
+      const s = r.perAxis[a];
+      if (s !== void 0) {
+        out.push(
+          `<rect x="${x}" y="${y}" width="${cellW - 3}" height="${rowH - 4}" fill="${ramp(s)}"/>`
+        );
+        out.push(
+          `<text x="${x + (cellW - 3) / 2}" y="${y + 15}" text-anchor="middle" fill="white" font-size="10">${Math.round(100 * s)}</text>`
+        );
+      } else {
+        out.push(
+          `<rect x="${x}" y="${y}" width="${cellW - 3}" height="${rowH - 4}" fill="#f3f4f6"/>`
+        );
+      }
+    });
+  });
+  out.push("</svg>");
+  return out.join("\n");
+}
+function renderLeaderboardHtml(report) {
+  const svg = renderLeaderboardSvg(report);
+  const p = report.provenance;
+  const rows = report.profiles.map(
+    (r, i) => `<tr><td>${i + 1}</td><td>${esc(r.label)}</td><td class="n">${pct(r.meanScore)}</td><td class="n">${pct(r.solveRate)}</td><td class="n">${r.n}</td><td class="n">$${r.costUsd.toFixed(3)}</td>${report.axes.map((a) => {
+      const v = r.perAxis[a];
+      return `<td class="n">${v === void 0 ? "\xB7" : pct(v)}</td>`;
+    }).join("")}</tr>`
+  ).join("\n");
+  const axisHead = report.axes.map((a) => `<th>${esc(a)}</th>`).join("");
+  return `<!doctype html><html><head><meta charset="utf-8"><title>${esc(report.title)}</title>
+<style>
+body{font:14px ui-sans-serif,system-ui,sans-serif;margin:2rem;color:#111}
+h1{font-size:1.4rem}.sub{color:#555;margin:.2rem 0 1.2rem}
+table{border-collapse:collapse;margin-top:1rem}td,th{border:1px solid #e5e7eb;padding:.35rem .6rem}
+th{background:#f9fafb;text-align:left}.n{text-align:right;font-variant-numeric:tabular-nums}
+tr:first-child td{font-weight:600}
+</style></head><body>
+<h1>${esc(report.title)}</h1>
+<div class="sub">${p.profiles} profiles \xD7 ${p.axes} axes \xB7 ${p.records} runs \xB7 ${p.models.length} models \xB7 total $${p.totalCostUsd.toFixed(2)}</div>
+${svg}
+<table><thead><tr><th>#</th><th>Profile</th><th>Score</th><th>Solved</th><th>Runs</th><th>Cost</th>${axisHead}</tr></thead>
+<tbody>
+${rows}
+</tbody></table>
+</body></html>`;
+}
+
+// src/runtime/inline-sandbox-client.ts
+function isAsyncIterable(v) {
+  return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
+}
+async function settle(exec, task, signal) {
+  const r = exec.execute(task, signal);
+  if (isAsyncIterable(r)) {
+    for await (const _ of r) {
+    }
+    return exec.resultArtifact();
+  }
+  return r;
+}
+function inlineSandboxClient(factory) {
+  let seq = 0;
+  return {
+    async create(options) {
+      const id = `inline-${seq++}`;
+      const createOptions = options;
+      return {
+        id,
+        async *streamPrompt(message) {
+          const controller = new AbortController();
+          const spec = { profile: { name: id }, harness: null };
+          const exec = factory(spec, { signal: controller.signal, seams: { createOptions } });
+          try {
+            const artifact = await settle(exec, message, controller.signal);
+            const out = artifact.out;
+            const tokensIn = artifact.spent.tokens.input;
+            const tokensOut = artifact.spent.tokens.output;
+            const costUsd = artifact.spent.usd;
+            if (tokensIn || tokensOut || costUsd) {
+              yield {
+                type: "llm_call",
+                data: { tokensIn, tokensOut, costUsd }
+              };
+            }
+            yield {
+              type: "result",
+              data: {
+                finalText: out?.content ?? "",
+                tokenUsage: {
+                  inputTokens: tokensIn,
+                  outputTokens: tokensOut
+                },
+                costUsd
+              }
+            };
+          } finally {
+            await exec.teardown("brutalKill").catch(() => {
+            });
+          }
+        },
+        async delete() {
+        }
+      };
+    }
+  };
+}
+
+// src/runtime/cli-bridge-sandbox-client.ts
+var DEFAULT_BRIDGE_URL = "http://127.0.0.1:3355";
+var BRIDGE_TIMEOUT_MS = 9e5;
+function bridgeModelId(harness, model) {
+  return model.startsWith(`${harness}/`) ? model : `${harness}/${model}`;
+}
+function resolveTarget(cfg, createOptions) {
+  const backend = createOptions?.backend;
+  const harness = backend?.type ?? cfg.harness;
+  const model = backend?.model?.model ?? cfg.model;
+  if (!harness)
+    throw new Error(
+      "cliBridgeSandboxClient: no harness (set cfg.harness or create({ backend: { type } }))"
+    );
+  if (!model)
+    throw new Error(
+      "cliBridgeSandboxClient: no model (set cfg.model or create({ backend: { model: { model } } }))"
+    );
+  return { harness, model };
+}
+async function bridgePost(url, bearer, bridgeModel, prompt, signal) {
+  const res = await fetch(`${url}/v1/chat/completions`, {
+    method: "POST",
+    headers: { "content-type": "application/json", authorization: `Bearer ${bearer}` },
+    body: JSON.stringify({ model: bridgeModel, messages: [{ role: "user", content: prompt }] }),
+    signal
+  });
+  if (!res.ok) {
+    throw new Error(
+      `cli-bridge ${bridgeModel}: HTTP ${res.status} ${(await res.text()).slice(0, 300)}`
+    );
+  }
+  const j = await res.json();
+  if (j.error) throw new Error(`cli-bridge ${bridgeModel}: ${j.error.message}`);
+  return {
+    content: j.choices?.[0]?.message?.content ?? "",
+    input: j.usage?.prompt_tokens ?? 0,
+    output: j.usage?.completion_tokens ?? 0
+  };
+}
+function cliBridgeSandboxClient(cfg) {
+  if (!cfg.bearer) throw new Error("cliBridgeSandboxClient: bearer is required");
+  const url = cfg.url ?? DEFAULT_BRIDGE_URL;
+  let seq = 0;
+  const factory = (_spec, ctx) => {
+    const id = `cli-bridge-${seq++}`;
+    const createOptions = ctx.seams.createOptions;
+    const { harness, model } = resolveTarget(cfg, createOptions);
+    const bridgeModel = bridgeModelId(harness, model);
+    let artifact;
+    return {
+      runtime: "cli",
+      async execute(task, signal) {
+        const started = Date.now();
+        const timeout = AbortSignal.timeout(BRIDGE_TIMEOUT_MS);
+        const composed = AbortSignal.any([signal, timeout]);
+        const r = await bridgePost(url, cfg.bearer, bridgeModel, String(task), composed);
+        artifact = {
+          outRef: `cli-bridge:${id}`,
+          out: { content: r.content },
+          spent: {
+            iterations: 1,
+            tokens: { input: r.input, output: r.output },
+            usd: 0,
+            ms: Date.now() - started
+          }
+        };
+        return artifact;
+      },
+      teardown: () => Promise.resolve({ destroyed: true }),
+      resultArtifact() {
+        if (!artifact)
+          throw new Error("cliBridgeSandboxClient: resultArtifact() read before execute()");
+        return artifact;
+      }
+    };
+  };
+  return inlineSandboxClient(factory);
+}
+
 // src/runtime/completion.ts
 function completionAuthorizes(v, policy) {
   if (!v?.done) return false;
@@ -466,7 +956,7 @@ import { mkdtempSync, rmSync } from "fs";
 import { mkdir, readFile, writeFile } from "fs/promises";
 import { tmpdir } from "os";
 import { dirname, join } from "path";
-function isAsyncIterable(v) {
+function isAsyncIterable2(v) {
   return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
 }
 function inProcessSandboxClient(options) {
@@ -517,7 +1007,7 @@ function inProcessSandboxClient(options) {
           const ctx = { round, workdir: boxWorkdir, signal };
           round += 1;
           const produced = await onPrompt(prompt, ctx);
-          if (isAsyncIterable(produced)) {
+          if (isAsyncIterable2(produced)) {
             for await (const ev of produced) yield ev;
           } else {
             for (const ev of produced) yield ev;
@@ -533,65 +1023,6 @@ function inProcessSandboxClient(options) {
   };
 }
 
-// src/runtime/inline-sandbox-client.ts
-function isAsyncIterable2(v) {
-  return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
-}
-async function settle(exec, task, signal) {
-  const r = exec.execute(task, signal);
-  if (isAsyncIterable2(r)) {
-    for await (const _ of r) {
-    }
-    return exec.resultArtifact();
-  }
-  return r;
-}
-function inlineSandboxClient(factory) {
-  let seq = 0;
-  return {
-    async create(_options) {
-      const id = `inline-${seq++}`;
-      return {
-        id,
-        async *streamPrompt(message) {
-          const controller = new AbortController();
-          const spec = { profile: { name: id }, harness: null };
-          const exec = factory(spec, { signal: controller.signal, seams: {} });
-          try {
-            const artifact = await settle(exec, message, controller.signal);
-            const out = artifact.out;
-            const tokensIn = artifact.spent.tokens.input;
-            const tokensOut = artifact.spent.tokens.output;
-            const costUsd = artifact.spent.usd;
-            if (tokensIn || tokensOut || costUsd) {
-              yield {
-                type: "llm_call",
-                data: { tokensIn, tokensOut, costUsd }
-              };
-            }
-            yield {
-              type: "result",
-              data: {
-                finalText: out?.content ?? "",
-                tokenUsage: {
-                  inputTokens: tokensIn,
-                  outputTokens: tokensOut
-                },
-                costUsd
-              }
-            };
-          } finally {
-            await exec.teardown("brutalKill").catch(() => {
-            });
-          }
-        },
-        async delete() {
-        }
-      };
-    }
-  };
-}
-
 // src/runtime/report-usage.ts
 function reportLoopUsage(cost, result, source = "loop") {
   cost.observe(result.costUsd, source);
@@ -608,6 +1039,9 @@ function campaignTraceToLoopEmitter(trace) {
 }
 async function runLoopForCell(opts, scenario, profile, ctx) {
   const loopOptions = opts.toLoopOptions(scenario, profile);
+  return runLoopWithCampaignContext(opts, loopOptions, ctx);
+}
+async function runLoopWithCampaignContext(opts, loopOptions, ctx) {
   const result = await runLoop({
     ...loopOptions,
     ctx: {
@@ -620,6 +1054,9 @@ async function runLoopForCell(opts, scenario, profile, ctx) {
   const toArtifact = opts.toArtifact ?? ((r) => r.winner?.output);
   return toArtifact(result);
 }
+function loopCampaignDispatch(opts) {
+  return (scenario, ctx) => runLoopWithCampaignContext(opts, opts.toLoopOptions(scenario), ctx);
+}
 function loopDispatch(opts) {
   return (profile, scenario, ctx) => runLoopForCell(opts, scenario, profile, ctx);
 }
@@ -1863,7 +2300,7 @@ function promotionGate(opts) {
 }
 
 // src/runtime/run-benchmark.ts
-import { pairedBootstrap, paretoFrontier } from "@tangle-network/agent-eval";
+import { pairedBootstrap as pairedBootstrap2, paretoFrontier } from "@tangle-network/agent-eval";
 
 // src/runtime/strategy.ts
 import { createChatClient, estimateCost, isModelPriced } from "@tangle-network/agent-eval";
@@ -2560,15 +2997,15 @@ async function runBenchmark(cfg) {
   const ok = perTask.filter(
     (r) => !!r.cells
   );
-  const mean = (xs) => xs.length ? xs.reduce((s, x) => s + x, 0) / xs.length : 0;
+  const mean2 = (xs) => xs.length ? xs.reduce((s, x) => s + x, 0) / xs.length : 0;
   const perStrategy = {};
   for (const s of strategies) {
     const cells = ok.map((r) => r.cells[s.name]).filter((c) => !!c);
     perStrategy[s.name] = {
-      score: mean(cells.map((c) => c.score)),
-      resolved: mean(cells.map((c) => c.resolved ? 1 : 0)),
-      usd: mean(cells.map((c) => c.usd)),
-      ms: mean(cells.map((c) => c.ms))
+      score: mean2(cells.map((c) => c.score)),
+      resolved: mean2(cells.map((c) => c.resolved ? 1 : 0)),
+      usd: mean2(cells.map((c) => c.usd)),
+      ms: mean2(cells.map((c) => c.ms))
     };
   }
   const frontier = paretoFrontier(
@@ -2587,7 +3024,7 @@ async function runBenchmark(cfg) {
   };
   const names = strategies.map((s) => s.name);
   if (names.includes("refine") && names.includes("sample") && ok.length >= 2) {
-    const b = pairedBootstrap(
+    const b = pairedBootstrap2(
       ok.map((r) => r.cells.sample?.score ?? 0),
       ok.map((r) => r.cells.refine?.score ?? 0)
     );
@@ -2596,7 +3033,7 @@ async function runBenchmark(cfg) {
   return report;
 }
 function printBenchmarkReport(report) {
-  const pct = (x) => `${(x * 100).toFixed(1)}%`;
+  const pct2 = (x) => `${(x * 100).toFixed(1)}%`;
   const pp = (x) => `${x >= 0 ? "+" : ""}${(x * 100).toFixed(1)}pp`;
   console.log(
     `
@@ -2607,7 +3044,7 @@ function printBenchmarkReport(report) {
   );
   for (const [s, v] of Object.entries(report.perStrategy))
     console.log(
-      `  ${(report.pareto.includes(s) ? `${s} *` : s).padEnd(16)} ${pct(v.score).padStart(7)} ${pct(v.resolved).padStart(9)} ${`$${v.usd.toFixed(3)}`.padStart(8)} ${(v.ms / 1e3).toFixed(0).padStart(6)}s`
+      `  ${(report.pareto.includes(s) ? `${s} *` : s).padEnd(16)} ${pct2(v.score).padStart(7)} ${pct2(v.resolved).padStart(9)} ${`$${v.usd.toFixed(3)}`.padStart(8)} ${(v.ms / 1e3).toFixed(0).padStart(6)}s`
     );
   if (report.pareto.length) console.log(`  * = on the (score, $) Pareto frontier`);
   for (const row of report.perTask)
@@ -3824,6 +4261,153 @@ function worktreeFanout(options) {
   });
 }
 
+// src/runtime/supervise-surface.ts
+function captureFailures(base) {
+  let lastReport = "";
+  const surface = {
+    name: base.name,
+    open: (t) => base.open(t),
+    tools: (t, h) => base.tools(t, h),
+    async call(h, name, args) {
+      const out = await base.call(h, name, args);
+      if (name === "run_tests") lastReport = out;
+      return out;
+    },
+    score: (t, h) => base.score(t, h),
+    close: (h) => base.close(h)
+  };
+  const failing = () => {
+    const body = /FAILING:\s*(.+)/i.exec(lastReport)?.[1];
+    return body ? body.split(",").map((s) => s.trim()).filter(Boolean) : [];
+  };
+  return { surface, failing };
+}
+function failuresAnalyst() {
+  return {
+    kinds: [
+      {
+        id: "failures",
+        description: "Surface the worker's still-failing tests so the driver targets them next.",
+        area: "progress"
+      }
+    ],
+    run: async (_kindId, trace) => {
+      const w = trace ?? {};
+      if (!(typeof w === "object" && w !== null && "resolved" in w))
+        return { summary: `worker produced: ${JSON.stringify(trace).slice(0, 300)}` };
+      if (w.resolved) return { summary: "worker RESOLVED \u2014 every check passed; stop." };
+      const failing = w.failing ?? [];
+      const head = `worker did NOT resolve \u2014 score ${(100 * (w.score ?? 0)).toFixed(0)}%, ${w.shots ?? "?"} shot(s)`;
+      return {
+        summary: failing.length ? `${head}. STILL FAILING (${failing.length}): ${failing.slice(0, 12).join(", ")}. Spawn the next worker to fix exactly these; if a test keeps failing across workers, give it concrete guidance about that case.` : `${head}. (no failing-test list available this round)`
+      };
+    }
+  };
+}
+function surfaceWorkerExecutor(surface, task, worker, strategy) {
+  let artifact;
+  return {
+    runtime: "surface-worker",
+    async execute(brief) {
+      const guidance = typeof brief === "string" ? brief.trim() : brief ? JSON.stringify(brief) : "";
+      const attemptTask = guidance ? {
+        ...task,
+        systemPrompt: `${task.systemPrompt ?? ""}
+
+\u2014 Supervisor guidance for THIS attempt (incorporate it; do not just repeat a prior approach) \u2014
+${guidance}`
+      } : task;
+      const cap = captureFailures(surface);
+      const r = await runAgentic({
+        surface: cap.surface,
+        task: attemptTask,
+        strategy,
+        budget: worker.budget ?? 1,
+        routerBaseUrl: worker.routerBaseUrl,
+        routerKey: worker.routerKey,
+        model: worker.model,
+        ...worker.maxTokens !== void 0 ? { maxTokens: worker.maxTokens } : {},
+        ...worker.innerTurns !== void 0 ? { innerTurns: worker.innerTurns } : {}
+      });
+      const out = {
+        resolved: r.resolved,
+        score: r.score,
+        shots: r.shots,
+        summary: `${strategy.name} ${r.shots} shot(s) \u2192 ${(100 * r.score).toFixed(0)}% (${r.resolved ? "resolved" : "unresolved"})`,
+        failing: r.resolved ? [] : cap.failing()
+      };
+      const spent = { iterations: r.completions, tokens: r.tokens, usd: r.usd, ms: r.ms };
+      artifact = {
+        outRef: `surface-worker:${task.id}:${r.shots}:${r.resolved ? "ok" : "no"}`,
+        out,
+        verdict: { valid: r.resolved, score: r.score },
+        spent
+      };
+      return artifact;
+    },
+    teardown: () => Promise.resolve({ destroyed: true }),
+    resultArtifact() {
+      if (!artifact) throw new Error("surfaceWorkerExecutor: resultArtifact before execute");
+      return artifact;
+    }
+  };
+}
+async function superviseSurface(profile, task, opts) {
+  const strategy = opts.strategy ?? refine;
+  const innerTurns = opts.worker.innerTurns ?? 6;
+  const router = opts.router ?? {
+    routerBaseUrl: opts.worker.routerBaseUrl,
+    routerKey: opts.worker.routerKey,
+    model: opts.worker.model
+  };
+  const budget = opts.budget ?? {
+    maxIterations: (innerTurns + 2) * 5 + 16,
+    maxTokens: (opts.worker.maxTokens ?? 4e3) * 8
+  };
+  const makeWorkerAgent = (rawProfile) => {
+    const p = rawProfile ?? {};
+    const name = typeof p.name === "string" && p.name.length > 0 ? p.name : "surface-worker";
+    const spec = {
+      profile: rawProfile,
+      harness: null,
+      executor: surfaceWorkerExecutor(
+        opts.surface,
+        task,
+        opts.worker,
+        strategy
+      )
+    };
+    return { name, act: async () => "", executorSpec: spec };
+  };
+  const deliverable = {
+    describe: `resolve the surface task ${task.id} (every required check passes)`,
+    check: (out2) => out2?.resolved === true
+  };
+  const analysts = opts.analysts === null ? void 0 : opts.analysts ?? failuresAnalyst();
+  const result = await supervise(profile, task, {
+    makeWorkerAgent,
+    deliverable,
+    budget,
+    maxLiveWorkers: opts.maxLiveWorkers ?? 1,
+    // A SMALL per-worker reservation so MULTIPLE workers fit the pool (the default reserves the whole pool
+    // per worker → only one ever spawns, defeating the spawn-a-targeted-worker steering).
+    perWorker: { maxIterations: innerTurns + 2, maxTokens: opts.worker.maxTokens ?? 4e3 },
+    router,
+    ...analysts ? { analysts, analyzeOnSettle: analysts.kinds.map((k) => k.id) } : {}
+  });
+  const out = result.kind === "winner" ? result.out : void 0;
+  const sp = result.spentTotal;
+  return {
+    resolved: out?.resolved ?? false,
+    score: out?.score ?? 0,
+    usd: sp.usd,
+    tokensIn: sp.tokens.input,
+    tokensOut: sp.tokens.output,
+    ms: sp.ms,
+    completions: sp.iterations
+  };
+}
+
 // src/runtime/verifier-environment.ts
 var submitTool = {
   type: "function",
@@ -4116,6 +4700,14 @@ export {
   renderAnytimeTable,
   defaultAuditorInstruction,
   auditIntent,
+  leaderboard,
+  pairwiseSignificance,
+  renderLeaderboardMarkdown,
+  renderPairwiseMarkdown,
+  renderLeaderboardSvg,
+  renderLeaderboardHtml,
+  inlineSandboxClient,
+  cliBridgeSandboxClient,
   completionAuthorizes,
   stopSentinel,
   sentinelCompletion,
@@ -4125,8 +4717,8 @@ export {
   renderReport,
   harvestCorpus,
   inProcessSandboxClient,
-  inlineSandboxClient,
   reportLoopUsage,
+  loopCampaignDispatch,
   loopDispatch,
   createMcpEnvironment,
   assertTraceDerivedFindings,
@@ -4182,6 +4774,8 @@ export {
   sandboxSessionTraceSource,
   analyzeTrace,
   worktreeFanout,
+  failuresAnalyst,
+  superviseSurface,
   createVerifierEnvironment,
   createWaterfallCollector,
   localShell,
@@ -4191,4 +4785,4 @@ export {
   computeFindingId,
   makeFinding2 as makeFinding
 };
-//# sourceMappingURL=chunk-AG335EXG.js.map
+//# sourceMappingURL=chunk-QSO2TVDS.js.map