@tangle-network/agent-eval 0.43.2 → 0.44.0

package/dist/chunk-H5BGRSN4.js

@@ -0,0 +1,642 @@
+import {
+  runCampaign
+} from "./chunk-RXK7FXLV.js";
+import {
+  buildReflectionPrompt,
+  parseReflectionResponse,
+  runCanaries,
+  scoreRedTeamOutput
+} from "./chunk-N4SBKEPJ.js";
+import {
+  detectRewardHacking
+} from "./chunk-YV7J7X5N.js";
+import {
+  callLlm
+} from "./chunk-VXNVVBZO.js";
+
+// src/campaign/auto-pr.ts
+import { execSync } from "child_process";
+import { writeFileSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+function openAutoPr(options) {
+  if (options.gate.decision !== "ship") {
+    return {
+      opened: false,
+      dryRun: false,
+      reason: `gate verdict was "${options.gate.decision}" \u2014 refusing to open PR`
+    };
+  }
+  const dryRun = options.dryRun ?? !process.env.GH_AUTO_PR_TOKEN;
+  const branch = options.branch ?? `auto/${options.result.manifestHash.slice(0, 12)}`;
+  const title = options.title ?? `auto: campaign ${options.result.manifestHash.slice(0, 8)} promoted by gate`;
+  const body = renderPrBody(options.result, options.gate, options.promotedDiff);
+  const bodyPath = join(tmpdir(), `auto-pr-body-${Date.now()}.md`);
+  writeFileSync(bodyPath, body);
+  if (dryRun) {
+    return {
+      opened: false,
+      dryRun: true,
+      reason: `dry-run (GH_AUTO_PR_TOKEN not set). Would create PR on ${options.ghOwner}/${options.ghRepo} branch ${branch}. Body at ${bodyPath}.`
+    };
+  }
+  const ghExec = options.ghExec ?? defaultGhExec;
+  const result = ghExec([
+    "pr",
+    "create",
+    "--repo",
+    `${options.ghOwner}/${options.ghRepo}`,
+    "--head",
+    branch,
+    "--title",
+    title,
+    "--body-file",
+    bodyPath
+  ]);
+  if (result.status !== 0) {
+    return {
+      opened: false,
+      dryRun: false,
+      reason: `gh pr create failed (exit ${result.status}): ${result.stderr.slice(0, 400)}`
+    };
+  }
+  const prUrl = result.stdout.trim();
+  return { opened: true, prUrl, dryRun: false, reason: "PR opened" };
+}
+function renderPrBody(result, gate, diff) {
+  const lines = [];
+  lines.push(`## Automated promotion by \`runImprovementLoop\``);
+  lines.push("");
+  lines.push(`**Manifest**: \`${result.manifestHash}\``);
+  lines.push(`**Seed**: ${result.seed}`);
+  lines.push(`**Duration**: ${Math.round(result.durationMs / 1e3)}s`);
+  lines.push(
+    `**Cells**: executed ${result.aggregates.cellsExecuted}, cached ${result.aggregates.cellsCached}, skipped ${result.aggregates.cellsSkipped}, failed ${result.aggregates.cellsFailed}`
+  );
+  lines.push(`**Total spend**: $${result.aggregates.totalCostUsd.toFixed(2)}`);
+  lines.push("");
+  lines.push(`### Gate verdict: \`${gate.decision}\``);
+  lines.push("");
+  for (const reason of gate.reasons) lines.push(`- ${reason}`);
+  if (gate.delta !== void 0) lines.push(`- delta: ${gate.delta.toFixed(3)}`);
+  lines.push("");
+  lines.push("### Contributing gates");
+  lines.push("");
+  lines.push("| gate | passed | detail |");
+  lines.push("|---|---|---|");
+  for (const c of gate.contributingGates) {
+    const detail = typeof c.detail === "object" ? JSON.stringify(c.detail).slice(0, 80) : String(c.detail).slice(0, 80);
+    lines.push(`| ${c.name} | ${c.passed ? "\u2713" : "\u2717"} | ${detail} |`);
+  }
+  lines.push("");
+  lines.push("### Promoted surface");
+  lines.push("");
+  lines.push("```diff");
+  lines.push(diff.slice(0, 8e3));
+  lines.push("```");
+  lines.push("");
+  lines.push("### By-judge aggregates");
+  lines.push("");
+  lines.push("| judge | mean | ci95 | n |");
+  lines.push("|---|---|---|---|");
+  for (const [name, agg] of Object.entries(result.aggregates.byJudge)) {
+    lines.push(
+      `| ${name} | ${agg.mean.toFixed(3)} | [${agg.ci95[0].toFixed(3)}, ${agg.ci95[1].toFixed(3)}] | ${agg.n} |`
+    );
+  }
+  return lines.join("\n");
+}
+function defaultGhExec(args) {
+  try {
+    const stdout = execSync(`gh ${args.map(quoteArg).join(" ")}`, {
+      env: { ...process.env, GH_TOKEN: process.env.GH_AUTO_PR_TOKEN ?? process.env.GH_TOKEN ?? "" },
+      stdio: ["ignore", "pipe", "pipe"]
+    }).toString("utf8");
+    return { stdout, stderr: "", status: 0 };
+  } catch (err) {
+    const e = err;
+    return {
+      stdout: e.stdout?.toString("utf8") ?? "",
+      stderr: e.stderr?.toString("utf8") ?? "",
+      status: e.status ?? 1
+    };
+  }
+}
+function quoteArg(arg) {
+  if (/^[a-zA-Z0-9_/\-:.@]+$/.test(arg)) return arg;
+  return `"${arg.replace(/"/g, '\\"')}"`;
+}
+
+// src/campaign/drivers/evolutionary.ts
+function evolutionaryDriver(opts) {
+  return {
+    kind: `evolutionary:${opts.mutator.kind}`,
+    async propose({ currentSurface, findings, populationSize, signal }) {
+      return opts.mutator.mutate({
+        findings: findings.length > 0 ? findings : opts.findings ?? [],
+        currentSurface,
+        populationSize,
+        signal
+      });
+    }
+  };
+}
+
+// src/campaign/drivers/gepa.ts
+var REFLECTION_SYSTEM = 'You are an expert prompt engineer. Output ONLY a JSON object of shape {"proposals":[{"label":string,"rationale":string,"payload":string}]} where each `payload` is the FULL improved surface text. No prose outside the JSON.';
+function gepaDriver(opts) {
+  const evidenceK = opts.evidenceK ?? 3;
+  return {
+    kind: "gepa",
+    async propose(ctx) {
+      const parent = typeof ctx.currentSurface === "string" ? ctx.currentSurface : JSON.stringify(ctx.currentSurface);
+      const { top, bottom, target } = buildEvidence(ctx, evidenceK, opts.target);
+      const userPrompt = buildReflectionPrompt({
+        target,
+        parentPayload: parent,
+        topTrials: top,
+        bottomTrials: bottom,
+        childCount: ctx.populationSize,
+        mutationPrimitives: opts.mutationPrimitives
+      });
+      const result = await callLlm(
+        {
+          model: opts.model,
+          messages: [
+            { role: "system", content: REFLECTION_SYSTEM },
+            { role: "user", content: userPrompt }
+          ],
+          jsonMode: true,
+          temperature: opts.temperature ?? 0.7,
+          maxTokens: opts.maxTokens ?? 6e3
+        },
+        opts.llm
+      );
+      const proposals = parseReflectionResponse(result.content, ctx.populationSize);
+      const out = [];
+      for (const proposal of proposals) {
+        const text = typeof proposal.payload === "string" ? proposal.payload.trim() : "";
+        if (text && text !== parent && !out.includes(text)) out.push(text);
+      }
+      return out;
+    }
+  };
+}
+function buildEvidence(ctx, evidenceK, baseTarget) {
+  const last = ctx.history.at(-1);
+  if (!last || last.candidates.length === 0) {
+    return { top: [], bottom: [], target: baseTarget };
+  }
+  const best = [...last.candidates].sort((a, b) => b.composite - a.composite)[0];
+  if (!best) return { top: [], bottom: [], target: baseTarget };
+  const byScore = [...best.scenarios].sort((a, b) => b.composite - a.composite);
+  const toTrace = (s) => ({
+    id: s.scenarioId,
+    score: s.composite
+  });
+  const top = byScore.slice(0, evidenceK).map(toTrace);
+  const bottom = byScore.slice(-evidenceK).reverse().map(toTrace);
+  const weakest = Object.entries(best.dimensions).sort((a, b) => a[1] - b[1]).slice(0, 3).map(([dim, value]) => `${dim} (${value.toFixed(2)})`);
+  const target = weakest.length > 0 ? `${baseTarget} \u2014 weakest dimensions: ${weakest.join(", ")}` : baseTarget;
+  return { top, bottom, target };
+}
+
+// src/campaign/gates/compose.ts
+function composeGate(...gates) {
+  if (gates.length === 0) {
+    throw new Error("composeGate requires at least one gate");
+  }
+  return {
+    name: `composed(${gates.map((g) => g.name).join(",")})`,
+    async decide(ctx) {
+      const results = [];
+      for (const gate of gates) {
+        const res = await gate.decide(ctx);
+        results.push({ gate, res });
+      }
+      const decisions = results.map((r) => r.res.decision);
+      const overall = decisions.every((d) => d === "ship") ? "ship" : decisions.includes("arch_ceiling") ? "arch_ceiling" : decisions.includes("model_ceiling") ? "model_ceiling" : decisions.includes("hold") ? "hold" : "need_more_work";
+      const contributing = results.flatMap(
+        (r) => r.res.contributingGates.length > 0 ? r.res.contributingGates : [{ name: r.gate.name, passed: r.res.decision === "ship", detail: r.res }]
+      );
+      const reasons = results.flatMap(
+        (r) => r.res.reasons.map((reason) => `[${r.gate.name}] ${reason}`)
+      );
+      return {
+        decision: overall,
+        reasons,
+        contributingGates: contributing,
+        delta: results[0]?.res.delta
+      };
+    }
+  };
+}
+
+// src/campaign/gates/default-production-gate.ts
+function defaultProductionGate(options) {
+  const deltaThreshold = options.deltaThreshold ?? 0.5;
+  const blockOnGaming = options.blockOnRewardHackingGaming ?? true;
+  return {
+    name: "defaultProductionGate",
+    async decide(ctx) {
+      const reasons = [];
+      const contributing = [];
+      const baselineComposite = meanComposite(
+        ctx.baselineArtifacts,
+        ctx.baselineJudgeScores ?? ctx.judgeScores,
+        options.holdoutScenarios
+      );
+      const candidateComposite = meanComposite(
+        ctx.candidateArtifacts,
+        ctx.judgeScores,
+        options.holdoutScenarios
+      );
+      const delta = candidateComposite - baselineComposite;
+      const heldoutPass = delta >= deltaThreshold;
+      contributing.push({
+        name: "heldout-delta",
+        passed: heldoutPass,
+        detail: { baselineComposite, candidateComposite, delta, deltaThreshold }
+      });
+      if (!heldoutPass) {
+        reasons.push(`heldout delta ${delta.toFixed(3)} < threshold ${deltaThreshold}`);
+      }
+      const budgetPass = options.budgetUsd === void 0 || ctx.cost.candidate + ctx.cost.baseline <= options.budgetUsd;
+      contributing.push({
+        name: "budget",
+        passed: budgetPass,
+        detail: {
+          candidateUsd: ctx.cost.candidate,
+          baselineUsd: ctx.cost.baseline,
+          budgetUsd: options.budgetUsd
+        }
+      });
+      if (!budgetPass) {
+        reasons.push(
+          `spend ${(ctx.cost.candidate + ctx.cost.baseline).toFixed(2)} > budget ${options.budgetUsd}`
+        );
+      }
+      const redTeamFindings = options.redTeamBattery ? probeRedTeam(ctx.candidateArtifacts, options.redTeamBattery) : { passed: true, findings: [] };
+      contributing.push({
+        name: "red-team",
+        passed: redTeamFindings.passed,
+        detail: {
+          failures: redTeamFindings.findings.length,
+          sample: redTeamFindings.findings.slice(0, 3)
+        }
+      });
+      if (!redTeamFindings.passed) {
+        reasons.push(`red-team probe failed (${redTeamFindings.findings.length} findings)`);
+      }
+      let rewardHackingReport = null;
+      if (options.recentRuns && options.recentRuns.length >= 10) {
+        rewardHackingReport = detectRewardHacking({ runs: options.recentRuns });
+      }
+      const gamingThreshold = 0.6;
+      const gamingFindings = (rewardHackingReport?.findings ?? []).filter(
+        (f) => f.severity >= gamingThreshold
+      );
+      const rewardHackingPass = !rewardHackingReport || !blockOnGaming || gamingFindings.length === 0 && rewardHackingReport.verdict !== "gaming";
+      contributing.push({
+        name: "reward-hacking",
+        passed: rewardHackingPass,
+        detail: { report: rewardHackingReport, gamingFindingCount: gamingFindings.length }
+      });
+      if (!rewardHackingPass) {
+        reasons.push(
+          `reward-hacking detector flagged ${gamingFindings.length} gaming-severity findings (verdict=${rewardHackingReport.verdict})`
+        );
+      }
+      let canaryReport = null;
+      if (options.recentRuns && options.recentRuns.length >= 10) {
+        canaryReport = runCanaries(options.recentRuns, {});
+      }
+      const errorAlerts = (canaryReport?.alerts ?? []).filter((a) => a.severity === "error");
+      const canaryPass = errorAlerts.length === 0;
+      contributing.push({
+        name: "canary",
+        passed: canaryPass,
+        detail: { totalAlerts: canaryReport?.alerts.length ?? 0, errorAlerts: errorAlerts.length }
+      });
+      if (!canaryPass) {
+        reasons.push(`canary error alerts: ${errorAlerts.length}`);
+      }
+      const allPassed = contributing.every((c) => c.passed);
+      const decision = allPassed ? "ship" : "hold";
+      return {
+        decision,
+        reasons: reasons.length > 0 ? reasons : ["all gates passed"],
+        contributingGates: contributing,
+        delta
+      };
+    }
+  };
+}
+function meanComposite(artifacts, judgeScoresByCell, scenarios) {
+  if (!artifacts || artifacts.size === 0) return 0;
+  const scenarioIds = new Set(scenarios.map((s) => s.id));
+  const composites = [];
+  for (const [cellId, scores] of judgeScoresByCell) {
+    const scenarioId = cellId.split(":")[0] ?? "";
+    if (!scenarioIds.has(scenarioId)) continue;
+    const cellComposites = Object.values(scores).map((s) => s.composite);
+    if (cellComposites.length === 0) continue;
+    composites.push(cellComposites.reduce((a, b) => a + b, 0) / cellComposites.length);
+  }
+  if (composites.length === 0) return 0;
+  return composites.reduce((a, b) => a + b, 0) / composites.length;
+}
+function probeRedTeam(artifacts, battery) {
+  const findings = [];
+  for (const [_cellId, artifact] of artifacts) {
+    const text = extractText(artifact);
+    if (text === void 0) continue;
+    for (const rtCase of battery) {
+      const finding = scoreRedTeamOutput(text, [], rtCase);
+      if (!finding.passed) {
+        findings.push({ scenarioId: rtCase.id, reason: finding.reason ?? "red-team probe failed" });
+      }
+    }
+  }
+  return { passed: findings.length === 0, findings };
+}
+function extractText(artifact) {
+  if (typeof artifact === "string") return artifact;
+  if (artifact && typeof artifact === "object") {
+    const rec = artifact;
+    if (typeof rec.text === "string") return rec.text;
+    if (typeof rec.output === "string") return rec.output;
+    if (typeof rec.content === "string") return rec.content;
+  }
+  return void 0;
+}
+
+// src/campaign/gates/heldout-gate.ts
+function heldOutGate(options) {
+  const deltaThreshold = options.deltaThreshold ?? 0.5;
+  return {
+    name: "heldOutGate",
+    async decide(ctx) {
+      const scenarioIds = new Set(options.scenarios.map((s) => s.id));
+      const baseline = meanForScenarios(ctx.baselineJudgeScores ?? ctx.judgeScores, scenarioIds);
+      const candidate = meanForScenarios(ctx.judgeScores, scenarioIds);
+      const delta = candidate - baseline;
+      const passed = delta >= deltaThreshold;
+      return {
+        decision: passed ? "ship" : "hold",
+        reasons: passed ? [`held-out delta ${delta.toFixed(3)} \u2265 ${deltaThreshold}`] : [`held-out delta ${delta.toFixed(3)} < ${deltaThreshold}`],
+        contributingGates: [
+          { name: "heldOutGate", passed, detail: { baseline, candidate, delta, deltaThreshold } }
+        ],
+        delta
+      };
+    }
+  };
+}
+function meanForScenarios(judgeScoresByCell, scenarioIds) {
+  const composites = [];
+  for (const [cellId, scores] of judgeScoresByCell) {
+    const scenarioId = cellId.split(":")[0] ?? "";
+    if (!scenarioIds.has(scenarioId)) continue;
+    const vals = Object.values(scores).map((s) => s.composite);
+    if (vals.length > 0) composites.push(vals.reduce((a, b) => a + b, 0) / vals.length);
+  }
+  return composites.length === 0 ? 0 : composites.reduce((a, b) => a + b, 0) / composites.length;
+}
+
+// src/campaign/presets/run-eval.ts
+async function runEval(opts) {
+  return runCampaign(opts);
+}
+
+// src/campaign/presets/run-optimization.ts
+import { createHash } from "crypto";
+async function runOptimization(opts) {
+  const promoteTopK = opts.promoteTopK ?? 2;
+  const baselineCampaign = await runCampaign({
+    ...opts,
+    dispatch: (scenario, ctx) => opts.dispatchWithSurface(opts.baselineSurface, scenario, ctx),
+    runDir: `${opts.runDir}/baseline`
+  });
+  const generations = [];
+  const history = [];
+  let currentSurfaces = [opts.baselineSurface];
+  let winnerSurface = opts.baselineSurface;
+  let winnerSurfaceHash = surfaceHash(opts.baselineSurface);
+  let winnerComposite = meanComposite2(baselineCampaign);
+  for (let gen = 0; gen < opts.maxGenerations; gen++) {
+    if (opts.driver.decide?.({ history }).stop) break;
+    const candidates = await opts.driver.propose({
+      currentSurface: currentSurfaces[0] ?? opts.baselineSurface,
+      history,
+      findings: [],
+      populationSize: opts.populationSize,
+      generation: gen,
+      signal: new AbortController().signal,
+      report: opts.report,
+      dataset: opts.labeledStore && opts.labeledStore !== "off" ? opts.labeledStore : void 0,
+      maxImprovementShots: opts.maxImprovementShots
+    });
+    const surfaceResults = [];
+    for (let i = 0; i < candidates.length; i++) {
+      const surface = candidates[i];
+      const hash = surfaceHash(surface);
+      const campaign = await runCampaign({
+        ...opts,
+        dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
+        runDir: `${opts.runDir}/gen-${gen}/candidate-${i}`
+      });
+      const composite = meanComposite2(campaign);
+      surfaceResults.push({ surfaceHash: hash, surface, campaign, composite });
+    }
+    surfaceResults.sort((a, b) => b.composite - a.composite);
+    const promoted = surfaceResults.slice(0, promoteTopK);
+    currentSurfaces = promoted.map((p) => p.surface);
+    const top = surfaceResults[0];
+    if (top && top.composite > winnerComposite) {
+      winnerSurface = top.surface;
+      winnerSurfaceHash = top.surfaceHash;
+      winnerComposite = top.composite;
+    }
+    const record = {
+      generationIndex: gen,
+      candidates: surfaceResults.map((s) => {
+        const breakdown = candidateBreakdown(s.campaign);
+        return {
+          surfaceHash: s.surfaceHash,
+          composite: s.composite,
+          ci95: [s.composite, s.composite],
+          dimensions: breakdown.dimensions,
+          scenarios: breakdown.scenarios
+        };
+      }),
+      promoted: promoted.map((p) => p.surfaceHash)
+    };
+    history.push(record);
+    generations.push({
+      record,
+      surfaces: surfaceResults.map((s) => ({
+        surfaceHash: s.surfaceHash,
+        surface: s.surface,
+        campaign: s.campaign
+      }))
+    });
+  }
+  return {
+    generations,
+    winnerSurface,
+    winnerSurfaceHash,
+    baselineCampaign
+  };
+}
+function surfaceHash(surface) {
+  const material = typeof surface === "string" ? surface : JSON.stringify({
+    kind: surface.kind,
+    worktreeRef: surface.worktreeRef,
+    baseRef: surface.baseRef ?? null
+  });
+  return createHash("sha256").update(material).digest("hex").slice(0, 16);
+}
+function meanComposite2(campaign) {
+  const composites = [];
+  for (const cell of campaign.cells) {
+    const cellComposites = Object.values(cell.judgeScores).map((s) => s.composite);
+    if (cellComposites.length > 0) {
+      composites.push(cellComposites.reduce((a, b) => a + b, 0) / cellComposites.length);
+    }
+  }
+  return composites.length === 0 ? 0 : composites.reduce((a, b) => a + b, 0) / composites.length;
+}
+function candidateBreakdown(campaign) {
+  const dimSums = {};
+  const dimCounts = {};
+  const byScenario = /* @__PURE__ */ new Map();
+  for (const cell of campaign.cells) {
+    const judgeScores = Object.values(cell.judgeScores);
+    if (judgeScores.length === 0) continue;
+    const cellComposite = judgeScores.reduce((a, s) => a + s.composite, 0) / judgeScores.length;
+    const arr = byScenario.get(cell.scenarioId) ?? [];
+    arr.push(cellComposite);
+    byScenario.set(cell.scenarioId, arr);
+    for (const score of judgeScores) {
+      for (const [key, value] of Object.entries(score.dimensions)) {
+        dimSums[key] = (dimSums[key] ?? 0) + value;
+        dimCounts[key] = (dimCounts[key] ?? 0) + 1;
+      }
+    }
+  }
+  const dimensions = {};
+  for (const key of Object.keys(dimSums)) {
+    const count = dimCounts[key] ?? 0;
+    dimensions[key] = count > 0 ? (dimSums[key] ?? 0) / count : 0;
+  }
+  const scenarios = [...byScenario.entries()].map(([scenarioId, comps]) => ({
+    scenarioId,
+    composite: comps.reduce((a, b) => a + b, 0) / comps.length
+  }));
+  return { dimensions, scenarios };
+}
+
+// src/campaign/presets/run-improvement-loop.ts
+async function runImprovementLoop(opts) {
+  if (opts.autoOnPromote === "config") {
+    throw new Error(
+      "runImprovementLoop: autoOnPromote='config' is deferred to Pass B (requires shadow deploy + rollback + ensemble judges). Use 'pr' or 'none' in v0.40."
+    );
+  }
+  if (opts.tracing === "off" && opts.driver) {
+    throw new Error(
+      "runImprovementLoop: tracing='off' is forbidden when a driver is wired. The improvement loop without traces is unattributable; candidate surfaces cannot be cited back to spans and the optimization dataset goes unfed."
+    );
+  }
+  if (opts.autoOnPromote === "pr" && (!opts.ghOwner || !opts.ghRepo)) {
+    throw new Error("runImprovementLoop: autoOnPromote='pr' requires ghOwner + ghRepo.");
+  }
+  const optimization = await runOptimization(opts);
+  const { runCampaign: runCampaign2 } = await import("./run-campaign-GNDO66B4.js");
+  const baselineOnHoldout = await runCampaign2({
+    ...opts,
+    scenarios: opts.holdoutScenarios,
+    dispatch: (scenario, ctx) => opts.dispatchWithSurface(opts.baselineSurface, scenario, ctx),
+    runDir: `${opts.runDir}/holdout-baseline`
+  });
+  const winnerOnHoldout = await runCampaign2({
+    ...opts,
+    scenarios: opts.holdoutScenarios,
+    dispatch: (scenario, ctx) => opts.dispatchWithSurface(optimization.winnerSurface, scenario, ctx),
+    runDir: `${opts.runDir}/holdout-winner`
+  });
+  const candidateArtifacts = /* @__PURE__ */ new Map();
+  const baselineArtifacts = /* @__PURE__ */ new Map();
+  const judgeScores = /* @__PURE__ */ new Map();
+  const baselineJudgeScores = /* @__PURE__ */ new Map();
+  for (const cell of winnerOnHoldout.cells) {
+    candidateArtifacts.set(cell.cellId, cell.artifact);
+    judgeScores.set(cell.cellId, cell.judgeScores);
+  }
+  for (const cell of baselineOnHoldout.cells) {
+    baselineArtifacts.set(cell.cellId, cell.artifact);
+    baselineJudgeScores.set(cell.cellId, cell.judgeScores);
+  }
+  const gateResult = await opts.gate.decide({
+    candidateArtifacts,
+    baselineArtifacts,
+    judgeScores,
+    baselineJudgeScores,
+    scenarios: opts.holdoutScenarios,
+    cost: {
+      candidate: winnerOnHoldout.aggregates.totalCostUsd,
+      baseline: baselineOnHoldout.aggregates.totalCostUsd
+    },
+    signal: new AbortController().signal
+  });
+  let prResult;
+  if (opts.autoOnPromote === "pr" && gateResult.decision === "ship") {
+    const render = opts.renderPromotedDiff ?? defaultRenderDiff;
+    const promotedDiff = render(optimization.winnerSurface, opts.baselineSurface);
+    prResult = openAutoPr({
+      result: winnerOnHoldout,
+      gate: gateResult,
+      promotedDiff,
+      ghOwner: opts.ghOwner,
+      ghRepo: opts.ghRepo
+    });
+  }
+  return {
+    ...optimization,
+    baselineOnHoldout,
+    winnerOnHoldout,
+    gateResult,
+    prResult
+  };
+}
+function defaultRenderDiff(winnerSurface, baselineSurface) {
+  if (typeof winnerSurface !== "string" || typeof baselineSurface !== "string") {
+    const fmt = (s) => typeof s === "string" ? "(prompt surface)" : `worktree=${s.worktreeRef}${s.baseRef ? ` base=${s.baseRef}` : ""}${s.summary ? `
+${s.summary}` : ""}`;
+    return `--- baseline
+${fmt(baselineSurface)}
++++ winner
+${fmt(winnerSurface)}`;
+  }
+  const lines = [];
+  lines.push("--- baseline");
+  lines.push("+++ winner");
+  for (const l of baselineSurface.split("\n")) lines.push(`- ${l}`);
+  for (const l of winnerSurface.split("\n")) lines.push(`+ ${l}`);
+  return lines.join("\n");
+}
+
+export {
+  openAutoPr,
+  evolutionaryDriver,
+  gepaDriver,
+  composeGate,
+  defaultProductionGate,
+  heldOutGate,
+  runEval,
+  runOptimization,
+  surfaceHash,
+  runImprovementLoop
+};
+//# sourceMappingURL=chunk-H5BGRSN4.js.map