@tangle-network/agent-eval 0.24.0 → 0.25.0

package/dist/index.js

@@ -8,9 +8,10 @@ import {
   classifyFailure,
   compareToBaseline,
   computeToolUseMetrics,
+  failureClusterView,
   iqr,
   welchsTTest
-} from "./chunk-OHEPNJQN.js";
+} from "./chunk-JLZQWFV3.js";
 import {
   exportTrainingData,
   toNdjson
@@ -95,7 +96,7 @@ import {
   summarizePreferenceMemory,
   trialTraceFromMultiShotTrial,
   withAssignedFeedbackSplit
-} from "./chunk-VRJVTXRV.js";
+} from "./chunk-EDUKQ5AM.js";
 import {
   RunRecordValidationError,
   isRunRecord,
@@ -220,6 +221,304 @@ import {
 } from "./chunk-NG236HPC.js";
 import "./chunk-PZ5AY32C.js";
 
+// src/auto-pr.ts
+async function proposeAutomatedPullRequest(client, input) {
+  validate(input);
+  return client.proposeChange(input);
+}
+function validate(input) {
+  if (!input.repo.owner.trim() || !input.repo.name.trim()) {
+    throw new ValidationError("proposeAutomatedPullRequest: repo.owner and repo.name required");
+  }
+  if (!input.branchName.trim() || /\s/.test(input.branchName)) {
+    throw new ValidationError(
+      "proposeAutomatedPullRequest: branchName must be non-empty and contain no whitespace"
+    );
+  }
+  if (input.branchName === (input.baseBranch ?? "main")) {
+    throw new ValidationError("proposeAutomatedPullRequest: branchName must differ from baseBranch");
+  }
+  if (input.fileChanges.length === 0) {
+    throw new ValidationError("proposeAutomatedPullRequest: fileChanges must not be empty");
+  }
+  const seenPaths = /* @__PURE__ */ new Set();
+  for (const change of input.fileChanges) {
+    if (!change.path.trim() || change.path.includes("..") || change.path.startsWith("/")) {
+      throw new ValidationError(
+        `proposeAutomatedPullRequest: invalid file path "${change.path}" (no '..' or leading '/')`
+      );
+    }
+    if (seenPaths.has(change.path)) {
+      throw new ValidationError(`proposeAutomatedPullRequest: duplicate file path "${change.path}"`);
+    }
+    seenPaths.add(change.path);
+  }
+  if (!input.title.trim()) {
+    throw new ValidationError("proposeAutomatedPullRequest: title must not be empty");
+  }
+}
+function httpGithubClient(opts) {
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const apiBase = (opts.apiBase ?? "https://api.github.com").replace(/\/+$/, "");
+  const now = opts.now ?? (() => /* @__PURE__ */ new Date());
+  async function api(method, path, body, accept404 = false) {
+    const res = await fetchImpl(`${apiBase}${path}`, {
+      method,
+      headers: {
+        accept: "application/vnd.github+json",
+        "content-type": "application/json",
+        authorization: `Bearer ${opts.token}`,
+        "x-github-api-version": "2022-11-28"
+      },
+      body: body === void 0 ? void 0 : JSON.stringify(body)
+    });
+    if (accept404 && res.status === 404) return null;
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new ConfigError(
+        `proposeAutomatedPullRequest: GitHub ${method} ${path} \u2192 ${res.status} ${text.slice(0, 400)}`
+      );
+    }
+    return await res.json();
+  }
+  return {
+    async proposeChange(input) {
+      const baseBranch = input.baseBranch ?? "main";
+      const repoPath = `/repos/${input.repo.owner}/${input.repo.name}`;
+      if (input.dryRun) {
+        return {
+          prUrl: `https://github.com/${input.repo.owner}/${input.repo.name}/compare/${baseBranch}...${input.branchName}`,
+          branchName: input.branchName,
+          headSha: "dry-run",
+          dryRun: true
+        };
+      }
+      const baseRef = await api("GET", `${repoPath}/git/ref/heads/${baseBranch}`);
+      if (!baseRef) {
+        throw new ConfigError(`proposeAutomatedPullRequest: base branch "${baseBranch}" not found`);
+      }
+      const baseSha = baseRef.object.sha;
+      const baseCommit = await api("GET", `${repoPath}/git/commits/${baseSha}`);
+      if (!baseCommit) {
+        throw new ConfigError(
+          `proposeAutomatedPullRequest: base commit ${baseSha} not found (race condition?)`
+        );
+      }
+      const treeEntries = [];
+      for (const change of input.fileChanges) {
+        const blob = await api("POST", `${repoPath}/git/blobs`, {
+          content: change.contents,
+          encoding: "utf-8"
+        });
+        if (!blob) throw new ConfigError("proposeAutomatedPullRequest: blob creation returned null");
+        treeEntries.push({
+          path: change.path,
+          mode: "100644",
+          type: "blob",
+          sha: blob.sha
+        });
+      }
+      const tree = await api("POST", `${repoPath}/git/trees`, {
+        base_tree: baseCommit.tree.sha,
+        tree: treeEntries
+      });
+      if (!tree) throw new ConfigError("proposeAutomatedPullRequest: tree creation returned null");
+      const author = input.authorName && input.authorEmail ? { name: input.authorName, email: input.authorEmail, date: now().toISOString() } : void 0;
+      const commitMessage = renderCommitMessage(input);
+      const commit = await api("POST", `${repoPath}/git/commits`, {
+        message: commitMessage,
+        tree: tree.sha,
+        parents: [baseSha],
+        ...author ? { author, committer: author } : {}
+      });
+      if (!commit)
+        throw new ConfigError("proposeAutomatedPullRequest: commit creation returned null");
+      const existing = await api(
+        "GET",
+        `${repoPath}/git/ref/heads/${input.branchName}`,
+        void 0,
+        true
+      );
+      if (!existing) {
+        await api("POST", `${repoPath}/git/refs`, {
+          ref: `refs/heads/${input.branchName}`,
+          sha: commit.sha
+        });
+      } else if (existing.object.sha !== commit.sha) {
+        await api("PATCH", `${repoPath}/git/refs/heads/${input.branchName}`, {
+          sha: commit.sha,
+          force: true
+        });
+      }
+      const openPrs = await api(
+        "GET",
+        `${repoPath}/pulls?state=open&head=${encodeURIComponent(`${input.repo.owner}:${input.branchName}`)}`
+      );
+      let pr;
+      if (openPrs && openPrs.length > 0) {
+        pr = openPrs[0];
+      } else {
+        const created = await api("POST", `${repoPath}/pulls`, {
+          title: input.title,
+          body: input.body,
+          head: input.branchName,
+          base: baseBranch
+        });
+        if (!created)
+          throw new ConfigError("proposeAutomatedPullRequest: PR creation returned null");
+        pr = created;
+      }
+      if (input.reviewers && input.reviewers.length > 0) {
+        await api(
+          "POST",
+          `${repoPath}/pulls/${pr.number}/requested_reviewers`,
+          { reviewers: input.reviewers },
+          true
+        ).catch(() => {
+        });
+      }
+      if (input.labels && input.labels.length > 0) {
+        await api(
+          "POST",
+          `${repoPath}/issues/${pr.number}/labels`,
+          { labels: input.labels },
+          true
+        ).catch(() => {
+        });
+      }
+      return {
+        prUrl: pr.html_url,
+        branchName: input.branchName,
+        headSha: commit.sha,
+        dryRun: false
+      };
+    }
+  };
+}
+function ghCliClient(opts = {}) {
+  const bin = opts.bin ?? "gh";
+  const cwd = opts.cwd ?? process.cwd();
+  const exec = opts.exec ?? defaultExec;
+  async function run(cmd, args, stdin) {
+    const r = await exec(cmd, args, { cwd, stdin });
+    if (r.exitCode !== 0) {
+      throw new ConfigError(
+        `proposeAutomatedPullRequest: ${cmd} ${args.join(" ")} failed (${r.exitCode}): ${r.stderr.trim() || r.stdout.trim()}`
+      );
+    }
+    return r;
+  }
+  return {
+    async proposeChange(input) {
+      const baseBranch = input.baseBranch ?? "main";
+      if (input.dryRun) {
+        return {
+          prUrl: `https://github.com/${input.repo.owner}/${input.repo.name}/compare/${baseBranch}...${input.branchName}`,
+          branchName: input.branchName,
+          headSha: "dry-run",
+          dryRun: true
+        };
+      }
+      await run("git", ["fetch", "origin", baseBranch]);
+      await run("git", ["checkout", baseBranch]);
+      await run("git", ["reset", "--hard", `origin/${baseBranch}`]);
+      await exec("git", ["branch", "-D", input.branchName], { cwd });
+      await run("git", ["checkout", "-b", input.branchName]);
+      const { mkdir, writeFile } = await import("fs/promises");
+      const { dirname: dirname5, join: join3, resolve } = await import("path");
+      for (const change of input.fileChanges) {
+        const abs = resolve(cwd, change.path);
+        await mkdir(dirname5(abs), { recursive: true });
+        await writeFile(abs, change.contents, "utf8");
+        await run("git", ["add", join3(change.path)]);
+      }
+      const env = {};
+      if (input.authorName) env.GIT_AUTHOR_NAME = input.authorName;
+      if (input.authorEmail) env.GIT_AUTHOR_EMAIL = input.authorEmail;
+      if (input.authorName) env.GIT_COMMITTER_NAME = input.authorName;
+      if (input.authorEmail) env.GIT_COMMITTER_EMAIL = input.authorEmail;
+      const message = renderCommitMessage(input);
+      await run("git", ["commit", "-m", message]);
+      const headRes = await run("git", ["rev-parse", "HEAD"]);
+      const headSha = headRes.stdout.trim();
+      await run("git", ["push", "-f", "origin", input.branchName]);
+      const existing = await exec(
+        bin,
+        [
+          "pr",
+          "list",
+          "--state",
+          "open",
+          "--head",
+          input.branchName,
+          "--json",
+          "url,number",
+          "--limit",
+          "1"
+        ],
+        { cwd }
+      );
+      let prUrl = "";
+      if (existing.exitCode === 0 && existing.stdout.trim()) {
+        const parsed = JSON.parse(existing.stdout);
+        if (parsed.length > 0 && parsed[0]) prUrl = parsed[0].url;
+      }
+      if (!prUrl) {
+        const args = [
+          "pr",
+          "create",
+          "--title",
+          input.title,
+          "--body",
+          input.body,
+          "--base",
+          baseBranch
+        ];
+        if (input.reviewers && input.reviewers.length > 0) {
+          args.push("--reviewer", input.reviewers.join(","));
+        }
+        if (input.labels && input.labels.length > 0) {
+          args.push("--label", input.labels.join(","));
+        }
+        const r = await run(bin, args);
+        const match = r.stdout.match(/https?:\/\/\S+/);
+        prUrl = match ? match[0] : r.stdout.trim();
+      }
+      return { prUrl, branchName: input.branchName, headSha, dryRun: false };
+    }
+  };
+}
+async function defaultExec(bin, args, opts) {
+  const { spawn } = await import("child_process");
+  return new Promise((resolveExec) => {
+    const child = spawn(bin, args, { cwd: opts.cwd });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (d) => {
+      stdout += d.toString();
+    });
+    child.stderr.on("data", (d) => {
+      stderr += d.toString();
+    });
+    if (opts.stdin) child.stdin.end(opts.stdin);
+    child.on("error", (err) => {
+      resolveExec({ stdout, stderr: `${stderr}${err.message}`, exitCode: 1 });
+    });
+    child.on("close", (code) => {
+      resolveExec({ stdout, stderr, exitCode: code ?? 1 });
+    });
+  });
+}
+function renderCommitMessage(input) {
+  const lines = [input.title, ""];
+  for (const change of input.fileChanges) {
+    if (change.rationale) lines.push(`- ${change.path}: ${change.rationale}`);
+  }
+  if (lines[lines.length - 1] !== "") lines.push("");
+  lines.push(input.body.trim());
+  return lines.join("\n").trim();
+}
+
 // src/executor.ts
 async function executeScenario(tc, scenario, config) {
   const startTime = Date.now();
@@ -1534,6 +1833,396 @@ function liveProofToReleaseTrace(config, trajectory, durationMs) {
   };
 }
 
+// src/production-loop.ts
+async function runProductionLoop(opts) {
+  validate2(opts);
+  const now = opts.now ?? (() => /* @__PURE__ */ new Date());
+  const startedAt = now().toISOString();
+  const observedRuns = await opts.traceStore.listRuns();
+  const observedFeedback = await opts.feedbackStore.list();
+  const clusterReport = await failureClusterView(opts.traceStore, {
+    minClusterSize: opts.cluster.minClusterSize ?? 1
+  });
+  const minSize = opts.cluster.minClusterSize ?? 5;
+  const minSeverity = opts.cluster.minSeverityRatio ?? 0.05;
+  const maxClusters = opts.cluster.maxClustersPerCycle ?? 1;
+  const totalRuns = clusterReport.totalRuns;
+  const actionable = clusterReport.clusters.filter((c) => c.runCount >= minSize).filter((c) => totalRuns === 0 || c.runCount / totalRuns >= minSeverity).slice(0, maxClusters);
+  if (actionable.length === 0) {
+    return finalize({
+      opts,
+      decision: "no_actionable_failures",
+      startedAt,
+      now,
+      observedRunCount: observedRuns.length,
+      observedFeedbackCount: observedFeedback.length,
+      clusters: clusterReport.clusters,
+      actedOnCluster: null,
+      evolution: null,
+      release: null,
+      gate: null,
+      promotedPrompt: opts.evolve.baselinePrompt,
+      pullRequest: null
+    });
+  }
+  const actedOn = actionable[0];
+  const baseline = {
+    id: opts.evolve.baselineId ?? "baseline",
+    label: opts.evolve.baselineId ?? "baseline",
+    generation: 0,
+    payload: opts.evolve.baselinePrompt
+  };
+  const holdoutIds = uniqueIds(opts.evolve.holdoutScenarios.map((s) => s.id));
+  const searchIds = uniqueIds(
+    (opts.evolve.searchScenarios ?? deriveSearchScenarios(opts.evolve.holdoutScenarios)).map(
+      (s) => s.id
+    )
+  );
+  if (searchIds.some((id) => holdoutIds.includes(id))) {
+    throw new ValidationError(
+      "runProductionLoop: searchScenarios and holdoutScenarios must be disjoint"
+    );
+  }
+  const reps = opts.evolve.reps ?? 3;
+  const generations = opts.evolve.generations ?? 3;
+  const populationSize = opts.evolve.populationSize ?? Math.max(2, opts.evolve.reps ?? 4);
+  const evolution = await runMultiShotOptimization({
+    runId: `${opts.runId}/evolve`,
+    target: opts.target,
+    seedVariants: [baseline],
+    searchScenarioIds: searchIds,
+    reps,
+    generations,
+    populationSize,
+    scoreConcurrency: opts.evolve.scoreConcurrency ?? 1,
+    runner: opts.evolve.runner,
+    scorer: opts.evolve.scorer,
+    mutateAdapter: opts.evolve.mutator,
+    gate: {
+      holdoutScenarioIds: holdoutIds,
+      reps,
+      gate: { ...opts.evolve.gate, baselineKey: baseline.id },
+      toRunRecord: opts.evolve.toRunRecord ?? (({ variant, scenarioId, rep, split, seed, trial }) => syntheticRunRecord({
+        runId: `${opts.runId}-${variant.id}-${scenarioId}-${rep}-${split}`,
+        variant,
+        scenarioId,
+        rep,
+        split,
+        seed,
+        trial,
+        target: opts.target
+      }))
+    }
+  });
+  const gate = evolution.gate?.decision ?? null;
+  const promotedVariant = evolution.promotedVariant;
+  const promoted = promotedVariant.payload;
+  const promotedChanged = promotedVariant.id !== baseline.id;
+  const allTrials = evolution.evolution.generations.flatMap(
+    (g) => g.trials
+  );
+  const traceEvidence = releaseTraceEvidenceFromMultiShotTrials(allTrials);
+  const releaseScenarios = [
+    ...(opts.evolve.searchScenarios ?? []).map((s) => ({
+      id: s.id,
+      payload: s,
+      split: "train",
+      tags: { persona: s.persona, label: s.label }
+    })),
+    ...opts.evolve.holdoutScenarios.map((s) => ({
+      id: s.id,
+      payload: s,
+      split: "holdout",
+      tags: { persona: s.persona, label: s.label }
+    }))
+  ];
+  const release = evaluateReleaseConfidence({
+    target: opts.target,
+    candidateId: promotedVariant.id,
+    baselineId: baseline.id,
+    scenarios: releaseScenarios,
+    traces: traceEvidence,
+    gateDecision: gate ?? void 0,
+    thresholds: opts.releaseThresholds,
+    runs: [...evolution.gate?.candidateRuns ?? [], ...evolution.gate?.baselineRuns ?? []]
+  });
+  if (!promotedChanged) {
+    return finalize({
+      opts,
+      decision: "evolve_yielded_no_improvement",
+      startedAt,
+      now,
+      observedRunCount: observedRuns.length,
+      observedFeedbackCount: observedFeedback.length,
+      clusters: clusterReport.clusters,
+      actedOnCluster: actedOn,
+      evolution,
+      release,
+      gate,
+      promotedPrompt: promoted,
+      pullRequest: null
+    });
+  }
+  if (release.status === "fail" || gate && !gate.promote) {
+    return finalize({
+      opts,
+      decision: "gate_failed",
+      startedAt,
+      now,
+      observedRunCount: observedRuns.length,
+      observedFeedbackCount: observedFeedback.length,
+      clusters: clusterReport.clusters,
+      actedOnCluster: actedOn,
+      evolution,
+      release,
+      gate,
+      promotedPrompt: promoted,
+      pullRequest: null
+    });
+  }
+  if (!opts.ship) {
+    return finalize({
+      opts,
+      decision: "proposed_change",
+      startedAt,
+      now,
+      observedRunCount: observedRuns.length,
+      observedFeedbackCount: observedFeedback.length,
+      clusters: clusterReport.clusters,
+      actedOnCluster: actedOn,
+      evolution,
+      release,
+      gate,
+      promotedPrompt: promoted,
+      pullRequest: null
+    });
+  }
+  const baselineStr = toPromptString(baseline.payload);
+  const promotedStr = toPromptString(promoted);
+  const ctx = {
+    runId: opts.runId,
+    target: opts.target,
+    decision: "pr_opened",
+    clusters: clusterReport.clusters,
+    actedOnCluster: actedOn,
+    observedRunCount: observedRuns.length,
+    observedFeedbackCount: observedFeedback.length,
+    evolution,
+    release,
+    gate,
+    baselinePromptString: baselineStr,
+    promotedPromptString: promotedStr
+  };
+  const renderBody = opts.ship.renderBody ?? defaultRenderBody;
+  const renderFile = opts.ship.renderPromptFile ?? ((next, _prev) => `${next}
+`);
+  const currentFile = opts.ship.readCurrentPromptFile ? await opts.ship.readCurrentPromptFile() : null;
+  const pr = await proposeAutomatedPullRequest(opts.ship.client, {
+    repo: opts.ship.repo,
+    baseBranch: opts.ship.baseBranch ?? "main",
+    branchName: `${opts.ship.branchPrefix.replace(/\/+$/, "")}/${opts.runId}`,
+    title: `${opts.target}: production-loop prompt update (${opts.runId})`,
+    body: renderBody(ctx),
+    reviewers: opts.ship.reviewers,
+    labels: opts.ship.labels,
+    fileChanges: [
+      {
+        path: opts.ship.promptFilePath,
+        contents: renderFile(promotedStr, currentFile),
+        rationale: `Auto-improved against cluster "${actedOn.failureClass}" (${actedOn.runCount} prod failures)`
+      }
+    ],
+    dryRun: opts.ship.dryRun
+  });
+  return finalize({
+    opts,
+    decision: "pr_opened",
+    startedAt,
+    now,
+    observedRunCount: observedRuns.length,
+    observedFeedbackCount: observedFeedback.length,
+    clusters: clusterReport.clusters,
+    actedOnCluster: actedOn,
+    evolution,
+    release,
+    gate,
+    promotedPrompt: promoted,
+    pullRequest: pr
+  });
+}
+function finalize(args) {
+  return {
+    runId: args.opts.runId,
+    target: args.opts.target,
+    decision: args.decision,
+    startedAt: args.startedAt,
+    finishedAt: args.now().toISOString(),
+    observedRunCount: args.observedRunCount,
+    observedFeedbackCount: args.observedFeedbackCount,
+    clusters: args.clusters,
+    actedOnCluster: args.actedOnCluster,
+    evolution: args.evolution,
+    release: args.release,
+    gate: args.gate,
+    baselinePrompt: args.opts.evolve.baselinePrompt,
+    promotedPrompt: args.promotedPrompt,
+    pullRequest: args.pullRequest,
+    cron: args.opts.cron ?? null
+  };
+}
+function validate2(opts) {
+  if (!opts.runId.trim()) throw new ValidationError("runProductionLoop: runId required");
+  if (!opts.target.trim()) throw new ValidationError("runProductionLoop: target required");
+  if (opts.evolve.holdoutScenarios.length === 0) {
+    throw new ValidationError("runProductionLoop: evolve.holdoutScenarios must not be empty");
+  }
+  if (opts.evolve.searchScenarios && opts.evolve.searchScenarios.length === 0) {
+    throw new ValidationError(
+      "runProductionLoop: evolve.searchScenarios must be omitted or non-empty"
+    );
+  }
+  if (!opts.evolve.gate.baselineKey && !opts.evolve.baselineId) {
+  }
+  if (opts.ship) {
+    if (!opts.ship.branchPrefix.trim()) {
+      throw new ValidationError("runProductionLoop: ship.branchPrefix required");
+    }
+    if (!opts.ship.promptFilePath.trim()) {
+      throw new ValidationError("runProductionLoop: ship.promptFilePath required");
+    }
+  }
+}
+function uniqueIds(ids) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const id of ids) {
+    if (seen.has(id)) continue;
+    seen.add(id);
+    out.push(id);
+  }
+  return out;
+}
+function deriveSearchScenarios(holdout) {
+  if (holdout.length < 4) {
+    return [
+      {
+        ...holdout[0],
+        id: `${holdout[0].id}__search`
+      }
+    ];
+  }
+  return holdout.filter((_, i) => i % 4 === 0).map((s) => ({ ...s, id: `${s.id}__search` }));
+}
+function syntheticRunRecord(input) {
+  const scoreKey = input.split === "holdout" ? "holdoutScore" : "searchScore";
+  return {
+    runId: input.runId,
+    experimentId: input.target,
+    candidateId: input.variant.id,
+    seed: input.seed,
+    model: "production-loop@synthetic",
+    promptHash: "0".repeat(64),
+    configHash: "0".repeat(64),
+    commitSha: "0".repeat(40),
+    wallMs: input.trial.durationMs ?? 1,
+    costUsd: input.trial.cost ?? 0,
+    tokenUsage: { input: 0, output: 0 },
+    outcome: {
+      [scoreKey]: input.trial.score,
+      raw: { score: input.trial.score, ok: input.trial.ok ? 1 : 0 }
+    },
+    splitTag: input.split,
+    scenarioId: input.scenarioId
+  };
+}
+function toPromptString(payload) {
+  if (typeof payload === "string") return payload;
+  if (payload == null) return "";
+  try {
+    return JSON.stringify(payload, null, 2);
+  } catch {
+    return String(payload);
+  }
+}
+function defaultRenderBody(ctx) {
+  const cluster = ctx.actedOnCluster;
+  const release = ctx.release;
+  const gate = ctx.gate;
+  const lines = [];
+  lines.push(`## Production-loop prompt update \u2014 \`${ctx.target}\``);
+  lines.push("");
+  lines.push(`Run id: \`${ctx.runId}\``);
+  lines.push(`Decision: \`${ctx.decision}\``);
+  lines.push(
+    `Observed in this cycle: ${ctx.observedRunCount} prod runs, ${ctx.observedFeedbackCount} feedback trajectories.`
+  );
+  lines.push("");
+  if (cluster) {
+    lines.push("### Triggering failure cluster");
+    lines.push("");
+    lines.push(`- **class**: \`${cluster.failureClass}\``);
+    lines.push(`- **runs in cluster**: ${cluster.runCount}`);
+    lines.push(`- **distinct scenarios**: ${cluster.scenarioIds.length}`);
+    if (cluster.toolName) lines.push(`- **tool**: \`${cluster.toolName}\``);
+    if (cluster.dimension) lines.push(`- **judge dimension**: \`${cluster.dimension}\``);
+    if (cluster.exampleError) {
+      lines.push(
+        `- **example error**: \`${cluster.exampleError.slice(0, 200).replace(/\n/g, " ")}\``
+      );
+    }
+    lines.push("");
+  }
+  if (gate) {
+    lines.push("### Held-out promotion gate");
+    lines.push("");
+    lines.push(`- **decision**: \`${gate.promote ? "PROMOTE" : "REJECT"}\``);
+    lines.push(`- **paired median delta**: ${gate.evidence.medianPairedDelta.toFixed(4)}`);
+    lines.push(
+      `- **paired 95% CI**: [${gate.evidence.pairedCI.low.toFixed(4)}, ${gate.evidence.pairedCI.high.toFixed(4)}]`
+    );
+    lines.push(`- **paired p-value**: ${gate.evidence.pairedPValue.toFixed(4)}`);
+    lines.push(
+      `- **search/holdout means**: ${gate.evidence.searchScore.toFixed(4)} / ${gate.evidence.holdoutScore.toFixed(4)}`
+    );
+    lines.push(`- **overfit gap**: ${gate.evidence.overfitGap.toFixed(4)}`);
+    lines.push("");
+  }
+  if (release) {
+    lines.push("### Release confidence");
+    lines.push("");
+    lines.push(`- **status**: \`${release.status}\``);
+    lines.push(`- **pass rate**: ${release.metrics.passRate.toFixed(4)}`);
+    lines.push(`- **mean score**: ${release.metrics.meanScore.toFixed(4)}`);
+    if (release.issues.length > 0) {
+      lines.push("- **issues**:");
+      for (const issue of release.issues) {
+        lines.push(`  - \`${issue.severity}\` ${issue.axis}: ${issue.detail}`);
+      }
+    }
+    lines.push("");
+  }
+  lines.push("### Prompt diff");
+  lines.push("");
+  lines.push("```diff");
+  lines.push(unifiedDiff(ctx.baselinePromptString, ctx.promotedPromptString));
+  lines.push("```");
+  return lines.join("\n");
+}
+function unifiedDiff(a, b) {
+  const aLines = a.split("\n");
+  const bLines = b.split("\n");
+  const out = [];
+  const max = Math.max(aLines.length, bLines.length);
+  for (let i = 0; i < max; i++) {
+    const al = aLines[i];
+    const bl = bLines[i];
+    if (al === bl) continue;
+    if (al !== void 0) out.push(`- ${al}`);
+    if (bl !== void 0) out.push(`+ ${bl}`);
+  }
+  return out.join("\n");
+}
+
 // src/registry.ts
 var ScenarioRegistry = class {
   scenarios = [];
@@ -8417,6 +9106,7 @@ export {
   formatDriverReport,
   formatFindings,
   gainHistogram,
+  ghCliClient,
   precision as goldenPrecision,
   gradeSemanticStatus,
   groupBy,
@@ -8424,6 +9114,7 @@ export {
   hashJson,
   hashScenarios,
   htmlContainsElement,
+  httpGithubClient,
   inMemoryReferenceReplayStore,
   inMemoryReviewStore,
   integrationAsi,
@@ -8484,6 +9175,7 @@ export {
   printDriverSummary,
   probeLlm,
   promptBisect,
+  proposeAutomatedPullRequest,
   proposeSynthesisTargets,
   providerFromBaseUrl,
   pytestTestParser,
@@ -8528,6 +9220,7 @@ export {
   runKeywordCoverageJudgeUrl,
   runLiveProof,
   runMultiShotOptimization,
+  runProductionLoop,
   runPromptEvolution,
   runProposeReview,
   runProposeReviewAsControlLoop,