@tangle-network/agent-runtime 0.79.2 → 0.79.4

package/dist/{chunk-T2HVQVB4.js → chunk-4J6RBI3K.js}

@@ -29,6 +29,19 @@ function extractLlmCallEvent(event, agentRunName) {
   }
   return void 0;
 }
+function sumSandboxUsage(events, agentRunName = "agent") {
+  let input = 0;
+  let output = 0;
+  let costUsd = 0;
+  for (const ev of events) {
+    const call = extractLlmCallEvent(ev, agentRunName);
+    if (!call) continue;
+    input += call.tokensIn ?? 0;
+    output += call.tokensOut ?? 0;
+    costUsd += call.costUsd ?? 0;
+  }
+  return { input, output, costUsd };
+}
 function buildLlmCall(data, agentRunName) {
   const tokensIn = pickFiniteNumber(data, ["tokensIn", "inputTokens", "prompt_tokens"]);
   const tokensOut = pickFiniteNumber(data, ["tokensOut", "outputTokens", "completion_tokens"]);
@@ -73,6 +86,7 @@ function mapSandboxEvent(event, opts = {}) {
 
 export {
   extractLlmCallEvent,
+  sumSandboxUsage,
   mapSandboxEvent
 };
-//# sourceMappingURL=chunk-T2HVQVB4.js.map
+//# sourceMappingURL=chunk-4J6RBI3K.js.map

package/dist/chunk-4J6RBI3K.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/runtime/sandbox-events.ts"],"sourcesContent":["/**\n * Sandbox-event → runtime-event mapping.\n *\n * The sandbox SDK emits a polymorphic `SandboxEvent = { type, data, id? }`\n * whose `type` vocabulary is backend-determined (opencode, etc.) rather than\n * enumerated by the SDK. Two consumers project it:\n *   - the loop kernel's cost ledger (`extractLlmCallEvent`) — sums usage off\n *     every cost-bearing event, regardless of stream shape;\n *   - the `AgentRuntime.act` streaming contract (`mapSandboxEvent`) — projects\n *     incremental events to the `RuntimeStreamEvent` chat-UX vocabulary.\n *\n * Both live here so the empirically-observed `type` vocabulary has one home.\n */\n\nimport type { SandboxEvent } from '@tangle-network/sandbox'\nimport type { RuntimeStreamEvent } from '../types'\n\n/**\n * Extract a `RuntimeStreamEvent`-shaped `llm_call` from a sandbox event when\n * the event carries usage/cost data. Returns `undefined` for non-cost events\n * so the kernel can iterate the full stream without branching.\n *\n * Canonical cost-carrying types observed in the wild:\n *   - `llm_call` — `data: { model, tokensIn, tokensOut, costUsd, ... }`\n *   - `message.completed` / `result` — `data: { usage: { inputTokens,\n *      outputTokens, totalCostUsd? } }`\n *   - `cost.usage` / `usage` — same shape under a dedicated type\n *\n * Numeric coercion is strict: `Number.isFinite` gates every accumulator write\n * so a sentinel `NaN` from a misbehaving backend cannot poison the ledger.\n */\nexport function extractLlmCallEvent(\n  event: SandboxEvent,\n  agentRunName: string,\n): (RuntimeStreamEvent & { type: 'llm_call' }) | undefined {\n  if (!event || typeof event !== 'object') return undefined\n  const type = String(event.type ?? '')\n  const data =\n    event.data && typeof event.data === 'object'\n      ? (event.data as Record<string, unknown>)\n      : ({} as Record<string, unknown>)\n\n  if (type === 'llm_call' || type === 'cost.usage' || type === 'usage') {\n    return buildLlmCall(data, agentRunName)\n  }\n  if (type === 'message.completed' || type === 'result' || type === 'final') {\n    const usage = data.usage as Record<string, unknown> | undefined\n    if (!usage || typeof usage !== 'object') return undefined\n    return buildLlmCall({ ...usage, model: data.model ?? usage.model }, agentRunName)\n  }\n  // sandbox 0.4.0 terminal event: `data = { tokenUsage: { inputTokens, outputTokens,\n  // reasoningTokens, cacheReadInputTokens }, totalCostUsd }`. Usage lives under\n  // `tokenUsage` (not `usage`) and the cost is top-level — neither matched the\n  // branches above, so an in-process loopDispatch run reported {0,0} and the\n  // backend-integrity guard misread a real run as a stub. Reasoning tokens are\n  // billed output (reasoning models), so they fold into the output count.\n  if (type === 'done') {\n    const usage = data.tokenUsage as Record<string, unknown> | undefined\n    if (!usage || typeof usage !== 'object') return undefined\n    const out = pickFiniteNumber(usage, ['outputTokens', 'completion_tokens', 'tokensOut'])\n    const reasoning = pickFiniteNumber(usage, ['reasoningTokens'])\n    const mergedOut =\n      out !== undefined || reasoning !== undefined ? (out ?? 0) + (reasoning ?? 0) : undefined\n    return buildLlmCall(\n      {\n        inputTokens: usage.inputTokens,\n        outputTokens: mergedOut,\n        totalCostUsd: data.totalCostUsd,\n        model: data.model ?? usage.model,\n      },\n      agentRunName,\n    )\n  }\n  return undefined\n}\n\n/**\n * Sum the token usage + USD cost of a sandbox turn's events — the one honest way to meter an\n * `openSandboxRun` cell. Folds `extractLlmCallEvent` over the stream (which reads usage off EVERY backend\n * event shape), so a `runProfileMatrix` dispatch can report it to `ctx.cost`:\n *\n *     const turn = await run.start(prompt)\n *     const u = sumSandboxUsage(turn.events)\n *     if (u.input || u.output) ctx.cost.observeTokens({ input: u.input, output: u.output })\n *     if (u.costUsd) ctx.cost.observe(u.costUsd, 'sandbox-cell')\n *\n * Without this a cell reads `{tokens:0, cost:0}` and the backend-integrity guard correctly aborts the\n * matrix as a stub. `agentRunName` is the fallback model label for cost-only events (default `'agent'`).\n */\nexport function sumSandboxUsage(\n  events: readonly SandboxEvent[],\n  agentRunName = 'agent',\n): { input: number; output: number; costUsd: number } {\n  let input = 0\n  let output = 0\n  let costUsd = 0\n  for (const ev of events) {\n    const call = extractLlmCallEvent(ev, agentRunName)\n    if (!call) continue\n    input += call.tokensIn ?? 0\n    output += call.tokensOut ?? 0\n    costUsd += call.costUsd ?? 0\n  }\n  return { input, output, costUsd }\n}\n\nfunction buildLlmCall(\n  data: Record<string, unknown>,\n  agentRunName: string,\n): (RuntimeStreamEvent & { type: 'llm_call' }) | undefined {\n  const tokensIn = pickFiniteNumber(data, ['tokensIn', 'inputTokens', 'prompt_tokens'])\n  const tokensOut = pickFiniteNumber(data, ['tokensOut', 'outputTokens', 'completion_tokens'])\n  const costUsd = pickFiniteNumber(data, ['costUsd', 'totalCostUsd', 'cost_usd', 'cost'])\n  if (tokensIn === undefined && tokensOut === undefined && costUsd === undefined) {\n    return undefined\n  }\n  const model = typeof data.model === 'string' && data.model.length > 0 ? data.model : agentRunName\n  const event: RuntimeStreamEvent & { type: 'llm_call' } = {\n    type: 'llm_call',\n    model,\n  }\n  if (tokensIn !== undefined) event.tokensIn = tokensIn\n  if (tokensOut !== undefined) event.tokensOut = tokensOut\n  if (costUsd !== undefined) event.costUsd = costUsd\n  return event\n}\n\nfunction pickFiniteNumber(data: Record<string, unknown>, keys: string[]): number | undefined {\n  for (const key of keys) {\n    const value = data[key]\n    if (typeof value === 'number' && Number.isFinite(value)) return value\n  }\n  return undefined\n}\n\n/**\n * Project one `SandboxEvent` onto the `RuntimeStreamEvent` chat-UX vocabulary,\n * for runtimes that bridge a sandbox `streamPrompt` into the\n * `AgentRuntime.act` streaming contract. Returns `undefined` for events that\n * have no faithful projection — the raw stream is preserved separately for the\n * `OutputAdapter`, so an unmapped event never loses data.\n *\n * Mapped (the task-optional incremental variants — no synthesized task\n * lifecycle, no guessed tool-part shapes):\n *   - `message.part.updated` text part → `text_delta`\n *   - `message.part.updated` reasoning/thinking part → `reasoning_delta`\n *   - cost-bearing events → `llm_call` (shared with the ledger extractor)\n *\n * The opencode backend emits incremental text as\n * `{ type: 'message.part.updated', data: { part: { type, text }, delta } }`;\n * `delta` is the increment, `part.text` the running accumulation.\n */\nexport function mapSandboxEvent(\n  event: SandboxEvent,\n  opts: { agentRunName?: string } = {},\n): RuntimeStreamEvent | undefined {\n  if (!event || typeof event !== 'object') return undefined\n  const type = String(event.type ?? '')\n  const data =\n    event.data && typeof event.data === 'object'\n      ? (event.data as Record<string, unknown>)\n      : ({} as Record<string, unknown>)\n\n  if (type === 'message.part.updated') {\n    const part =\n      data.part && typeof data.part === 'object' ? (data.part as Record<string, unknown>) : {}\n    const partType = String(part.type ?? '')\n    const delta = typeof data.delta === 'string' ? data.delta : undefined\n    const text = delta ?? (typeof part.text === 'string' ? part.text : undefined)\n    if (text === undefined) return undefined\n    if (partType === 'text') return { type: 'text_delta', text }\n    if (partType === 'reasoning' || partType === 'thinking')\n      return { type: 'reasoning_delta', text }\n    return undefined\n  }\n\n  return extractLlmCallEvent(event, opts.agentRunName ?? 'agent')\n}\n"],"mappings":";AA+BO,SAAS,oBACd,OACA,cACyD;AACzD,MAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAChD,QAAM,OAAO,OAAO,MAAM,QAAQ,EAAE;AACpC,QAAM,OACJ,MAAM,QAAQ,OAAO,MAAM,SAAS,WAC/B,MAAM,OACN,CAAC;AAER,MAAI,SAAS,cAAc,SAAS,gBAAgB,SAAS,SAAS;AACpE,WAAO,aAAa,MAAM,YAAY;AAAA,EACxC;AACA,MAAI,SAAS,uBAAuB,SAAS,YAAY,SAAS,SAAS;AACzE,UAAM,QAAQ,KAAK;AACnB,QAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAChD,WAAO,aAAa,EAAE,GAAG,OAAO,OAAO,KAAK,SAAS,MAAM,MAAM,GAAG,YAAY;AAAA,EAClF;AAOA,MAAI,SAAS,QAAQ;AACnB,UAAM,QAAQ,KAAK;AACnB,QAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAChD,UAAM,MAAM,iBAAiB,OAAO,CAAC,gBAAgB,qBAAqB,WAAW,CAAC;AACtF,UAAM,YAAY,iBAAiB,OAAO,CAAC,iBAAiB,CAAC;AAC7D,UAAM,YACJ,QAAQ,UAAa,cAAc,UAAa,OAAO,MAAM,aAAa,KAAK;AACjF,WAAO;AAAA,MACL;AAAA,QACE,aAAa,MAAM;AAAA,QACnB,cAAc;AAAA,QACd,cAAc,KAAK;AAAA,QACnB,OAAO,KAAK,SAAS,MAAM;AAAA,MAC7B;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAeO,SAAS,gBACd,QACA,eAAe,SACqC;AACpD,MAAI,QAAQ;AACZ,MAAI,SAAS;AACb,MAAI,UAAU;AACd,aAAW,MAAM,QAAQ;AACvB,UAAM,OAAO,oBAAoB,IAAI,YAAY;AACjD,QAAI,CAAC,KAAM;AACX,aAAS,KAAK,YAAY;AAC1B,cAAU,KAAK,aAAa;AAC5B,eAAW,KAAK,WAAW;AAAA,EAC7B;AACA,SAAO,EAAE,OAAO,QAAQ,QAAQ;AAClC;AAEA,SAAS,aACP,MACA,cACyD;AACzD,QAAM,WAAW,iBAAiB,MAAM,CAAC,YAAY,eAAe,eAAe,CAAC;AACpF,QAAM,YAAY,iBAAiB,MAAM,CAAC,aAAa,gBAAgB,mBAAmB,CAAC;AAC3F,QAAM,UAAU,iBAAiB,MAAM,CAAC,WAAW,gBAAgB,YAAY,MAAM,CAAC;AACtF,MAAI,aAAa,UAAa,cAAc,UAAa,YAAY,QAAW;AAC9E,WAAO;AAAA,EACT;AACA,QAAM,QAAQ,OAAO,KAAK,UAAU,YAAY,KAAK,MAAM,SAAS,IAAI,KAAK,QAAQ;AACrF,QAAM,QAAmD;AAAA,IACvD,MAAM;AAAA,IACN;AAAA,EACF;AACA,MAAI,aAAa,OAAW,OAAM,WAAW;AAC7C,MAAI,cAAc,OAAW,OAAM,YAAY;AAC/C,MAAI,YAAY,OAAW,OAAM,UAAU;AAC3C,SAAO;AACT;AAEA,SAAS,iBAAiB,MAA+B,MAAoC;AAC3F,aAAW,OAAO,MAAM;AACtB,UAAM,QAAQ,KAAK,GAAG;AACtB,QAAI,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,EAAG,QAAO;AAAA,EAClE;AACA,SAAO;AACT;AAmBO,SAAS,gBACd,OACA,OAAkC,CAAC,GACH;AAChC,MAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAChD,QAAM,OAAO,OAAO,MAAM,QAAQ,EAAE;AACpC,QAAM,OACJ,MAAM,QAAQ,OAAO,MAAM,SAAS,WAC/B,MAAM,OACN,CAAC;AAER,MAAI,SAAS,wBAAwB;AACnC,UAAM,OACJ,KAAK,QAAQ,OAAO,KAAK,SAAS,WAAY,KAAK,OAAmC,CAAC;AACzF,UAAM,WAAW,OAAO,KAAK,QAAQ,EAAE;AACvC,UAAM,QAAQ,OAAO,KAAK,UAAU,WAAW,KAAK,QAAQ;AAC5D,UAAM,OAAO,UAAU,OAAO,KAAK,SAAS,WAAW,KAAK,OAAO;AACnE,QAAI,SAAS,OAAW,QAAO;AAC/B,QAAI,aAAa,OAAQ,QAAO,EAAE,MAAM,cAAc,KAAK;AAC3D,QAAI,aAAa,eAAe,aAAa;AAC3C,aAAO,EAAE,MAAM,mBAAmB,KAAK;AACzC,WAAO;AAAA,EACT;AAEA,SAAO,oBAAoB,OAAO,KAAK,gBAAgB,OAAO;AAChE;","names":[]}

package/dist/{chunk-75V2XXYJ.js → chunk-AHZ3YBL6.js}

@@ -12,8 +12,9 @@ import {
   routerToolLoop,
   runLoop,
   settledToIteration,
+  supervise,
   withDriverExecutor
-} from "./chunk-TSDKBFZP.js";
+} from "./chunk-44CX5JU6.js";
 import {
   addTokenUsage,
   isAbortError,
@@ -21,7 +22,7 @@ import {
   sleep,
   stringifySafe,
   zeroTokenUsage
-} from "./chunk-Z3RRRPRB.js";
+} from "./chunk-H7IBHAFT.js";
 import {
   AnalystError,
   PlannerError,
@@ -205,6 +206,355 @@ Audit the route: revealed intent, verdict, evidence, one recommendation.`
   };
 }
 
+// src/runtime/benchmark-report.ts
+import {
+  benjaminiHochberg,
+  confidenceInterval,
+  pairedBootstrap,
+  pairedTTest,
+  wilson
+} from "@tangle-network/agent-eval";
+var defaultScoreOf = (r) => {
+  const o = r.outcome;
+  if (typeof o.holdoutScore === "number") return o.holdoutScore;
+  if (typeof o.searchScore === "number") return o.searchScore;
+  const raw = o.raw ?? {};
+  for (const k of ["composite", "score", "passed", "resolved"]) {
+    if (typeof raw[k] === "number") return raw[k];
+  }
+  return void 0;
+};
+var defaultProfileKeyOf = (r) => {
+  const cell = r.agentProfile;
+  const harness = cell?.harness;
+  return harness ? `${harness}\xB7${r.model}` : r.model;
+};
+var defaultGroupOf = (r) => r.scenarioId ?? r.experimentId;
+function quantile(sorted, q) {
+  if (sorted.length === 0) return 0;
+  const idx = Math.min(sorted.length - 1, Math.floor(q * (sorted.length - 1)));
+  return sorted[idx] ?? 0;
+}
+function mean(xs) {
+  return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
+}
+function meanByScenario(records, scoreOf) {
+  const sums = /* @__PURE__ */ new Map();
+  for (const r of records) {
+    const s = scoreOf(r);
+    if (typeof s !== "number") continue;
+    const id = r.scenarioId;
+    if (!id) {
+      throw new Error(
+        `benchmark-report: RunRecord (candidate ${r.candidateId ?? "unknown"}) is missing scenarioId \u2014 cannot pair or interval it honestly. Pass opts.stats only on a scenario-tagged corpus.`
+      );
+    }
+    const acc = sums.get(id) ?? { total: 0, n: 0 };
+    acc.total += s;
+    acc.n += 1;
+    sums.set(id, acc);
+  }
+  const out = /* @__PURE__ */ new Map();
+  for (const [id, acc] of sums) out.set(id, acc.n ? acc.total / acc.n : 0);
+  return out;
+}
+function leaderboard(records, opts = {}) {
+  const scoreOf = opts.scoreOf ?? defaultScoreOf;
+  const profileKeyOf = opts.profileKeyOf ?? defaultProfileKeyOf;
+  const groupOf = opts.groupOf ?? defaultGroupOf;
+  const labelOf = opts.labelOf ?? ((k) => k);
+  const axisSet = /* @__PURE__ */ new Set();
+  if (opts.axisScoresOf) {
+    for (const r of records) for (const k of Object.keys(opts.axisScoresOf(r))) axisSet.add(k);
+  } else {
+    for (const r of records) axisSet.add(groupOf(r));
+  }
+  const axes = [...axisSet].sort();
+  const byProfile = /* @__PURE__ */ new Map();
+  for (const r of records) {
+    const key = profileKeyOf(r);
+    const bucket = byProfile.get(key);
+    if (bucket) bucket.push(r);
+    else byProfile.set(key, [r]);
+  }
+  const rows = [];
+  for (const [profileKey, recs] of byProfile) {
+    const scores = recs.map(scoreOf).filter((s) => typeof s === "number");
+    const axisBuckets = /* @__PURE__ */ new Map();
+    for (const r of recs) {
+      if (opts.axisScoresOf) {
+        for (const [axis, s] of Object.entries(opts.axisScoresOf(r))) {
+          const b = axisBuckets.get(axis) ?? [];
+          b.push(s);
+          axisBuckets.set(axis, b);
+        }
+      } else {
+        const s = scoreOf(r);
+        if (typeof s === "number") {
+          const axis = groupOf(r);
+          const b = axisBuckets.get(axis) ?? [];
+          b.push(s);
+          axisBuckets.set(axis, b);
+        }
+      }
+    }
+    const perAxis = {};
+    for (const [axis, b] of axisBuckets) perAxis[axis] = mean(b);
+    const latencies = recs.map((r) => r.wallMs).sort((a, b) => a - b);
+    const pass = opts.passThreshold ?? 0.999;
+    let scoreCi;
+    let passCi;
+    if (opts.stats) {
+      const collapsed = [...meanByScenario(recs, scoreOf).values()];
+      if (collapsed.length > 0) {
+        const ci2 = confidenceInterval(collapsed, 0.95, { seed: 7 });
+        scoreCi = { lower: ci2.lower, upper: ci2.upper };
+        const w = wilson(collapsed.filter((s) => s >= pass).length, collapsed.length, 0.95);
+        passCi = { lower: w.lower, upper: w.upper };
+      }
+    }
+    rows.push({
+      profileKey,
+      label: labelOf(profileKey),
+      model: recs[0]?.model ?? profileKey,
+      n: recs.length,
+      meanScore: mean(scores),
+      solveRate: scores.length === 0 ? 0 : scores.filter((s) => s >= pass).length / scores.length,
+      perAxis,
+      costUsd: recs.reduce((a, r) => a + r.costUsd, 0),
+      tokensIn: recs.reduce((a, r) => a + (r.tokenUsage?.input ?? 0), 0),
+      tokensOut: recs.reduce((a, r) => a + (r.tokenUsage?.output ?? 0), 0),
+      latencyP50Ms: quantile(latencies, 0.5),
+      latencyP90Ms: quantile(latencies, 0.9),
+      ...scoreCi ? { scoreCi } : {},
+      ...passCi ? { passCi } : {}
+    });
+  }
+  rows.sort(
+    (a, b) => b.meanScore - a.meanScore || a.costUsd - b.costUsd || a.label.localeCompare(b.label)
+  );
+  const models = [...new Set(records.map((r) => r.model))].sort();
+  return {
+    title: opts.title ?? "Benchmark report",
+    axes,
+    profiles: rows,
+    meta: opts.meta ?? {},
+    provenance: {
+      records: records.length,
+      profiles: rows.length,
+      axes: axes.length,
+      models,
+      totalCostUsd: records.reduce((a, r) => a + r.costUsd, 0)
+    }
+  };
+}
+function pairwiseSignificance(records, opts = {}) {
+  const scoreOf = opts.scoreOf ?? defaultScoreOf;
+  const profileKeyOf = opts.profileKeyOf ?? defaultProfileKeyOf;
+  const labelOf = opts.labelOf ?? ((k) => k);
+  const minPairs = opts.minPairs ?? 12;
+  const byProfile = /* @__PURE__ */ new Map();
+  for (const r of records) {
+    const k = profileKeyOf(r);
+    const b = byProfile.get(k);
+    if (b) b.push(r);
+    else byProfile.set(k, [r]);
+  }
+  const keys = [...byProfile.keys()].sort();
+  const collapsed = new Map(keys.map((k) => [k, meanByScenario(byProfile.get(k) ?? [], scoreOf)]));
+  const raw = [];
+  for (let i = 0; i < keys.length; i += 1) {
+    for (let j = i + 1; j < keys.length; j += 1) {
+      const ka = keys[i];
+      const kb = keys[j];
+      const am = collapsed.get(ka);
+      const bm = collapsed.get(kb);
+      const aScores = [];
+      const bScores = [];
+      for (const sid of [...am.keys()].sort()) {
+        const bv = bm.get(sid);
+        if (bv !== void 0) {
+          aScores.push(am.get(sid));
+          bScores.push(bv);
+        }
+      }
+      if (aScores.length === 0) continue;
+      const boot = pairedBootstrap(aScores, bScores, { seed: 7, statistic: "median" });
+      const p = pairedTTest(aScores, bScores).p;
+      raw.push({
+        a: labelOf(ka),
+        b: labelOf(kb),
+        pairs: aScores.length,
+        delta: boot.median,
+        ciLow: boot.low,
+        ciHigh: boot.high,
+        p
+      });
+    }
+  }
+  const { significant } = benjaminiHochberg(
+    raw.map((r) => r.p),
+    opts.fdr ?? 0.05
+  );
+  return raw.map((r, i) => ({
+    ...r,
+    significant: (significant[i] ?? false) && r.pairs >= minPairs
+  }));
+}
+var pct = (x) => `${(100 * x).toFixed(1)}%`;
+var ci = (iv) => iv ? ` [${pct(iv.lower)}, ${pct(iv.upper)}]` : "";
+function renderLeaderboardMarkdown(report) {
+  const lines = [];
+  lines.push(`# ${report.title}`, "");
+  const p = report.provenance;
+  lines.push(
+    `**${p.profiles} profiles \xD7 ${p.axes} axes**, ${p.records} runs, ${p.models.length} models \xB7 total $${p.totalCostUsd.toFixed(2)}`,
+    ""
+  );
+  for (const [k, v] of Object.entries(report.meta)) lines.push(`- **${k}:** ${v}`);
+  if (Object.keys(report.meta).length) lines.push("");
+  lines.push("## Leaderboard", "");
+  lines.push(
+    "| # | Profile | Score (95% CI) | Solved (95% CI) | Runs | Cost | Tok in/out | p50 | p90 |"
+  );
+  lines.push("|---|---|--:|--:|--:|--:|--:|--:|--:|");
+  report.profiles.forEach((r, i) => {
+    lines.push(
+      `| ${i + 1} | ${r.label} | ${pct(r.meanScore)}${ci(r.scoreCi)} | ${pct(r.solveRate)}${ci(r.passCi)} | ${r.n} | $${r.costUsd.toFixed(3)} | ${r.tokensIn}/${r.tokensOut} | ${(r.latencyP50Ms / 1e3).toFixed(1)}s | ${(r.latencyP90Ms / 1e3).toFixed(1)}s |`
+    );
+  });
+  lines.push("");
+  lines.push("## Score matrix \u2014 profile \xD7 axis", "");
+  lines.push(`| Profile | ${report.axes.join(" | ")} |`);
+  lines.push(`|---|${report.axes.map(() => "--:").join("|")}|`);
+  for (const r of report.profiles) {
+    const cells = report.axes.map((a) => {
+      const v = r.perAxis[a];
+      return v === void 0 ? "\xB7" : pct(v);
+    });
+    lines.push(`| ${r.label} | ${cells.join(" | ")} |`);
+  }
+  lines.push("");
+  lines.push("> `\xB7` = the profile never ran that axis (blank, never zero).");
+  return lines.join("\n");
+}
+function renderPairwiseMarkdown(verdicts, title = "Pairwise significance (paired, BH-corrected)") {
+  const lines = [`## ${title}`, ""];
+  if (verdicts.length === 0) return lines.concat("_no comparable pairs_").join("\n");
+  lines.push("| A vs B | \u0394(b\u2212a) median | 95% CI | pairs | p | verdict |");
+  lines.push("|---|--:|--:|--:|--:|---|");
+  for (const v of verdicts) {
+    const verdict = v.significant ? `**${v.delta >= 0 ? v.b : v.a} wins**` : "ns";
+    lines.push(
+      `| ${v.a} vs ${v.b} | ${v.delta >= 0 ? "+" : ""}${pct(v.delta)} | [${pct(v.ciLow)}, ${pct(v.ciHigh)}] | ${v.pairs} | ${v.p.toFixed(3)} | ${verdict} |`
+    );
+  }
+  lines.push(
+    "",
+    "> `ns` = not significant after Benjamini\u2013Hochberg (or below the paired-count floor)."
+  );
+  return lines.join("\n");
+}
+function esc(s) {
+  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+function ramp(score) {
+  const x = Math.max(0, Math.min(1, score));
+  const r = Math.round(x < 0.5 ? 220 : 220 - (x - 0.5) * 2 * 160);
+  const g = Math.round(x < 0.5 ? x * 2 * 170 : 170);
+  return `rgb(${r},${g},60)`;
+}
+function renderLeaderboardSvg(report) {
+  const rowH = 26;
+  const labelW = 220;
+  const cellW = 54;
+  const pad = 16;
+  const barAreaW = 360;
+  const profiles = report.profiles;
+  const barsH = profiles.length * rowH + pad;
+  const heatTop = barsH + 48;
+  const heatW = labelW + report.axes.length * cellW + pad;
+  const heatH = profiles.length * rowH + pad;
+  const width = Math.max(labelW + barAreaW + pad * 2, heatW);
+  const height = heatTop + heatH + pad;
+  const out = [];
+  out.push(
+    `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" font-family="ui-sans-serif,system-ui,sans-serif" font-size="12">`
+  );
+  out.push(`<rect width="${width}" height="${height}" fill="white"/>`);
+  out.push(
+    `<text x="${pad}" y="${pad + 4}" font-weight="700" font-size="14">${esc(report.title)} \u2014 score</text>`
+  );
+  profiles.forEach((r, i) => {
+    const y = pad + 12 + i * rowH;
+    out.push(`<text x="${pad}" y="${y + 14}">${esc(r.label)}</text>`);
+    const w = Math.round(r.meanScore * barAreaW);
+    out.push(`<rect x="${labelW}" y="${y + 4}" width="${barAreaW}" height="16" fill="#eef0f2"/>`);
+    out.push(
+      `<rect x="${labelW}" y="${y + 4}" width="${w}" height="16" fill="${ramp(r.meanScore)}"/>`
+    );
+    out.push(`<text x="${labelW + barAreaW + 6}" y="${y + 16}">${pct(r.meanScore)}</text>`);
+  });
+  out.push(
+    `<text x="${pad}" y="${heatTop - 12}" font-weight="700" font-size="14">profile \xD7 axis</text>`
+  );
+  report.axes.forEach((a, c) => {
+    const x = labelW + c * cellW;
+    out.push(
+      `<text x="${x + cellW / 2}" y="${heatTop}" text-anchor="middle" fill="#555">${esc(a.length > 8 ? `${a.slice(0, 7)}\u2026` : a)}</text>`
+    );
+  });
+  profiles.forEach((r, i) => {
+    const y = heatTop + 8 + i * rowH;
+    out.push(`<text x="${pad}" y="${y + 16}">${esc(r.label)}</text>`);
+    report.axes.forEach((a, c) => {
+      const x = labelW + c * cellW;
+      const s = r.perAxis[a];
+      if (s !== void 0) {
+        out.push(
+          `<rect x="${x}" y="${y}" width="${cellW - 3}" height="${rowH - 4}" fill="${ramp(s)}"/>`
+        );
+        out.push(
+          `<text x="${x + (cellW - 3) / 2}" y="${y + 15}" text-anchor="middle" fill="white" font-size="10">${Math.round(100 * s)}</text>`
+        );
+      } else {
+        out.push(
+          `<rect x="${x}" y="${y}" width="${cellW - 3}" height="${rowH - 4}" fill="#f3f4f6"/>`
+        );
+      }
+    });
+  });
+  out.push("</svg>");
+  return out.join("\n");
+}
+function renderLeaderboardHtml(report) {
+  const svg = renderLeaderboardSvg(report);
+  const p = report.provenance;
+  const rows = report.profiles.map(
+    (r, i) => `<tr><td>${i + 1}</td><td>${esc(r.label)}</td><td class="n">${pct(r.meanScore)}</td><td class="n">${pct(r.solveRate)}</td><td class="n">${r.n}</td><td class="n">$${r.costUsd.toFixed(3)}</td>${report.axes.map((a) => {
+      const v = r.perAxis[a];
+      return `<td class="n">${v === void 0 ? "\xB7" : pct(v)}</td>`;
+    }).join("")}</tr>`
+  ).join("\n");
+  const axisHead = report.axes.map((a) => `<th>${esc(a)}</th>`).join("");
+  return `<!doctype html><html><head><meta charset="utf-8"><title>${esc(report.title)}</title>
+<style>
+body{font:14px ui-sans-serif,system-ui,sans-serif;margin:2rem;color:#111}
+h1{font-size:1.4rem}.sub{color:#555;margin:.2rem 0 1.2rem}
+table{border-collapse:collapse;margin-top:1rem}td,th{border:1px solid #e5e7eb;padding:.35rem .6rem}
+th{background:#f9fafb;text-align:left}.n{text-align:right;font-variant-numeric:tabular-nums}
+tr:first-child td{font-weight:600}
+</style></head><body>
+<h1>${esc(report.title)}</h1>
+<div class="sub">${p.profiles} profiles \xD7 ${p.axes} axes \xB7 ${p.records} runs \xB7 ${p.models.length} models \xB7 total $${p.totalCostUsd.toFixed(2)}</div>
+${svg}
+<table><thead><tr><th>#</th><th>Profile</th><th>Score</th><th>Solved</th><th>Runs</th><th>Cost</th>${axisHead}</tr></thead>
+<tbody>
+${rows}
+</tbody></table>
+</body></html>`;
+}
+
 // src/runtime/completion.ts
 function completionAuthorizes(v, policy) {
   if (!v?.done) return false;
@@ -608,6 +958,9 @@ function campaignTraceToLoopEmitter(trace) {
 }
 async function runLoopForCell(opts, scenario, profile, ctx) {
   const loopOptions = opts.toLoopOptions(scenario, profile);
+  return runLoopWithCampaignContext(opts, loopOptions, ctx);
+}
+async function runLoopWithCampaignContext(opts, loopOptions, ctx) {
   const result = await runLoop({
     ...loopOptions,
     ctx: {
@@ -620,6 +973,9 @@ async function runLoopForCell(opts, scenario, profile, ctx) {
   const toArtifact = opts.toArtifact ?? ((r) => r.winner?.output);
   return toArtifact(result);
 }
+function loopCampaignDispatch(opts) {
+  return (scenario, ctx) => runLoopWithCampaignContext(opts, opts.toLoopOptions(scenario), ctx);
+}
 function loopDispatch(opts) {
   return (profile, scenario, ctx) => runLoopForCell(opts, scenario, profile, ctx);
 }
@@ -1863,7 +2219,7 @@ function promotionGate(opts) {
 }
 
 // src/runtime/run-benchmark.ts
-import { pairedBootstrap, paretoFrontier } from "@tangle-network/agent-eval";
+import { pairedBootstrap as pairedBootstrap2, paretoFrontier } from "@tangle-network/agent-eval";
 
 // src/runtime/strategy.ts
 import { createChatClient, estimateCost, isModelPriced } from "@tangle-network/agent-eval";
@@ -2560,15 +2916,15 @@ async function runBenchmark(cfg) {
   const ok = perTask.filter(
     (r) => !!r.cells
   );
-  const mean = (xs) => xs.length ? xs.reduce((s, x) => s + x, 0) / xs.length : 0;
+  const mean2 = (xs) => xs.length ? xs.reduce((s, x) => s + x, 0) / xs.length : 0;
   const perStrategy = {};
   for (const s of strategies) {
     const cells = ok.map((r) => r.cells[s.name]).filter((c) => !!c);
     perStrategy[s.name] = {
-      score: mean(cells.map((c) => c.score)),
-      resolved: mean(cells.map((c) => c.resolved ? 1 : 0)),
-      usd: mean(cells.map((c) => c.usd)),
-      ms: mean(cells.map((c) => c.ms))
+      score: mean2(cells.map((c) => c.score)),
+      resolved: mean2(cells.map((c) => c.resolved ? 1 : 0)),
+      usd: mean2(cells.map((c) => c.usd)),
+      ms: mean2(cells.map((c) => c.ms))
     };
   }
   const frontier = paretoFrontier(
@@ -2587,7 +2943,7 @@ async function runBenchmark(cfg) {
   };
   const names = strategies.map((s) => s.name);
   if (names.includes("refine") && names.includes("sample") && ok.length >= 2) {
-    const b = pairedBootstrap(
+    const b = pairedBootstrap2(
       ok.map((r) => r.cells.sample?.score ?? 0),
       ok.map((r) => r.cells.refine?.score ?? 0)
     );
@@ -2596,7 +2952,7 @@ async function runBenchmark(cfg) {
   return report;
 }
 function printBenchmarkReport(report) {
-  const pct = (x) => `${(x * 100).toFixed(1)}%`;
+  const pct2 = (x) => `${(x * 100).toFixed(1)}%`;
   const pp = (x) => `${x >= 0 ? "+" : ""}${(x * 100).toFixed(1)}pp`;
   console.log(
     `
@@ -2607,7 +2963,7 @@ function printBenchmarkReport(report) {
   );
   for (const [s, v] of Object.entries(report.perStrategy))
     console.log(
-      `  ${(report.pareto.includes(s) ? `${s} *` : s).padEnd(16)} ${pct(v.score).padStart(7)} ${pct(v.resolved).padStart(9)} ${`$${v.usd.toFixed(3)}`.padStart(8)} ${(v.ms / 1e3).toFixed(0).padStart(6)}s`
+      `  ${(report.pareto.includes(s) ? `${s} *` : s).padEnd(16)} ${pct2(v.score).padStart(7)} ${pct2(v.resolved).padStart(9)} ${`$${v.usd.toFixed(3)}`.padStart(8)} ${(v.ms / 1e3).toFixed(0).padStart(6)}s`
     );
   if (report.pareto.length) console.log(`  * = on the (score, $) Pareto frontier`);
   for (const row of report.perTask)
@@ -3824,6 +4180,153 @@ function worktreeFanout(options) {
   });
 }
 
+// src/runtime/supervise-surface.ts
+function captureFailures(base) {
+  let lastReport = "";
+  const surface = {
+    name: base.name,
+    open: (t) => base.open(t),
+    tools: (t, h) => base.tools(t, h),
+    async call(h, name, args) {
+      const out = await base.call(h, name, args);
+      if (name === "run_tests") lastReport = out;
+      return out;
+    },
+    score: (t, h) => base.score(t, h),
+    close: (h) => base.close(h)
+  };
+  const failing = () => {
+    const body = /FAILING:\s*(.+)/i.exec(lastReport)?.[1];
+    return body ? body.split(",").map((s) => s.trim()).filter(Boolean) : [];
+  };
+  return { surface, failing };
+}
+function failuresAnalyst() {
+  return {
+    kinds: [
+      {
+        id: "failures",
+        description: "Surface the worker's still-failing tests so the driver targets them next.",
+        area: "progress"
+      }
+    ],
+    run: async (_kindId, trace) => {
+      const w = trace ?? {};
+      if (!(typeof w === "object" && w !== null && "resolved" in w))
+        return { summary: `worker produced: ${JSON.stringify(trace).slice(0, 300)}` };
+      if (w.resolved) return { summary: "worker RESOLVED \u2014 every check passed; stop." };
+      const failing = w.failing ?? [];
+      const head = `worker did NOT resolve \u2014 score ${(100 * (w.score ?? 0)).toFixed(0)}%, ${w.shots ?? "?"} shot(s)`;
+      return {
+        summary: failing.length ? `${head}. STILL FAILING (${failing.length}): ${failing.slice(0, 12).join(", ")}. Spawn the next worker to fix exactly these; if a test keeps failing across workers, give it concrete guidance about that case.` : `${head}. (no failing-test list available this round)`
+      };
+    }
+  };
+}
+function surfaceWorkerExecutor(surface, task, worker, strategy) {
+  let artifact;
+  return {
+    runtime: "surface-worker",
+    async execute(brief) {
+      const guidance = typeof brief === "string" ? brief.trim() : brief ? JSON.stringify(brief) : "";
+      const attemptTask = guidance ? {
+        ...task,
+        systemPrompt: `${task.systemPrompt ?? ""}
+
+\u2014 Supervisor guidance for THIS attempt (incorporate it; do not just repeat a prior approach) \u2014
+${guidance}`
+      } : task;
+      const cap = captureFailures(surface);
+      const r = await runAgentic({
+        surface: cap.surface,
+        task: attemptTask,
+        strategy,
+        budget: worker.budget ?? 1,
+        routerBaseUrl: worker.routerBaseUrl,
+        routerKey: worker.routerKey,
+        model: worker.model,
+        ...worker.maxTokens !== void 0 ? { maxTokens: worker.maxTokens } : {},
+        ...worker.innerTurns !== void 0 ? { innerTurns: worker.innerTurns } : {}
+      });
+      const out = {
+        resolved: r.resolved,
+        score: r.score,
+        shots: r.shots,
+        summary: `${strategy.name} ${r.shots} shot(s) \u2192 ${(100 * r.score).toFixed(0)}% (${r.resolved ? "resolved" : "unresolved"})`,
+        failing: r.resolved ? [] : cap.failing()
+      };
+      const spent = { iterations: r.completions, tokens: r.tokens, usd: r.usd, ms: r.ms };
+      artifact = {
+        outRef: `surface-worker:${task.id}:${r.shots}:${r.resolved ? "ok" : "no"}`,
+        out,
+        verdict: { valid: r.resolved, score: r.score },
+        spent
+      };
+      return artifact;
+    },
+    teardown: () => Promise.resolve({ destroyed: true }),
+    resultArtifact() {
+      if (!artifact) throw new Error("surfaceWorkerExecutor: resultArtifact before execute");
+      return artifact;
+    }
+  };
+}
+async function superviseSurface(profile, task, opts) {
+  const strategy = opts.strategy ?? refine;
+  const innerTurns = opts.worker.innerTurns ?? 6;
+  const router = opts.router ?? {
+    routerBaseUrl: opts.worker.routerBaseUrl,
+    routerKey: opts.worker.routerKey,
+    model: opts.worker.model
+  };
+  const budget = opts.budget ?? {
+    maxIterations: (innerTurns + 2) * 5 + 16,
+    maxTokens: (opts.worker.maxTokens ?? 4e3) * 8
+  };
+  const makeWorkerAgent = (rawProfile) => {
+    const p = rawProfile ?? {};
+    const name = typeof p.name === "string" && p.name.length > 0 ? p.name : "surface-worker";
+    const spec = {
+      profile: rawProfile,
+      harness: null,
+      executor: surfaceWorkerExecutor(
+        opts.surface,
+        task,
+        opts.worker,
+        strategy
+      )
+    };
+    return { name, act: async () => "", executorSpec: spec };
+  };
+  const deliverable = {
+    describe: `resolve the surface task ${task.id} (every required check passes)`,
+    check: (out2) => out2?.resolved === true
+  };
+  const analysts = opts.analysts === null ? void 0 : opts.analysts ?? failuresAnalyst();
+  const result = await supervise(profile, task, {
+    makeWorkerAgent,
+    deliverable,
+    budget,
+    maxLiveWorkers: opts.maxLiveWorkers ?? 1,
+    // A SMALL per-worker reservation so MULTIPLE workers fit the pool (the default reserves the whole pool
+    // per worker → only one ever spawns, defeating the spawn-a-targeted-worker steering).
+    perWorker: { maxIterations: innerTurns + 2, maxTokens: opts.worker.maxTokens ?? 4e3 },
+    router,
+    ...analysts ? { analysts, analyzeOnSettle: analysts.kinds.map((k) => k.id) } : {}
+  });
+  const out = result.kind === "winner" ? result.out : void 0;
+  const sp = result.spentTotal;
+  return {
+    resolved: out?.resolved ?? false,
+    score: out?.score ?? 0,
+    usd: sp.usd,
+    tokensIn: sp.tokens.input,
+    tokensOut: sp.tokens.output,
+    ms: sp.ms,
+    completions: sp.iterations
+  };
+}
+
 // src/runtime/verifier-environment.ts
 var submitTool = {
   type: "function",
@@ -4116,6 +4619,12 @@ export {
   renderAnytimeTable,
   defaultAuditorInstruction,
   auditIntent,
+  leaderboard,
+  pairwiseSignificance,
+  renderLeaderboardMarkdown,
+  renderPairwiseMarkdown,
+  renderLeaderboardSvg,
+  renderLeaderboardHtml,
   completionAuthorizes,
   stopSentinel,
   sentinelCompletion,
@@ -4127,6 +4636,7 @@ export {
   inProcessSandboxClient,
   inlineSandboxClient,
   reportLoopUsage,
+  loopCampaignDispatch,
   loopDispatch,
   createMcpEnvironment,
   assertTraceDerivedFindings,
@@ -4182,6 +4692,8 @@ export {
   sandboxSessionTraceSource,
   analyzeTrace,
   worktreeFanout,
+  failuresAnalyst,
+  superviseSurface,
   createVerifierEnvironment,
   createWaterfallCollector,
   localShell,
@@ -4191,4 +4703,4 @@ export {
   computeFindingId,
   makeFinding2 as makeFinding
 };
-//# sourceMappingURL=chunk-75V2XXYJ.js.map
+//# sourceMappingURL=chunk-AHZ3YBL6.js.map