@tangle-network/agent-eval 0.40.5 → 0.42.0

package/dist/index.js

@@ -1,14 +1,17 @@
 import {
+  DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
   Dataset,
   HoldoutLockedError,
+  buildReflectionPrompt,
   hashScenarios,
+  parseReflectionResponse,
   redTeamDataset,
   redTeamReport,
   runCanaries,
   scoreRedTeamOutput,
   toolNamesForRun
-} from "./chunk-5U2DOJU4.js";
+} from "./chunk-N4SBKEPJ.js";
 import {
   BENCHMARK_SPLIT_SEED,
   benchmarks_exports,
@@ -19,10 +22,9 @@ import {
   classifyFailure,
   compareToBaseline,
   computeToolUseMetrics,
-  failureClusterView,
   iqr,
   welchsTTest
-} from "./chunk-AU2JLNSZ.js";
+} from "./chunk-H4TOS272.js";
 import {
   exportTrainingData,
   toNdjson
@@ -74,63 +76,16 @@ import {
   stopOnRepeatedAction,
   subjectiveEval
 } from "./chunk-NCRFYPS3.js";
-import {
-  CallbackResearcher,
-  DEFAULT_MUTATION_PRIMITIVES,
-  FileSystemFeedbackTrajectoryStore,
-  HeldOutGate,
-  InMemoryFeedbackTrajectoryStore,
-  InMemoryTrialCache,
-  NoopResearcher,
-  assignFeedbackSplit,
-  buildReflectionPrompt,
-  controlRunToFeedbackTrajectory,
-  createFeedbackTrajectory,
-  crowdingDistance,
-  defaultMultiShotObjectives,
-  dominates,
-  feedbackTrajectoriesToDatasetScenarios,
-  feedbackTrajectoriesToOptimizerRows,
-  feedbackTrajectoryToDatasetScenario,
-  feedbackTrajectoryToOptimizerRow,
-  paretoFrontier,
-  paretoFrontierWithCrowding,
-  parseFeedbackTrajectoriesJsonl,
-  parseReflectionResponse,
-  renderPreferenceMemoryMarkdown,
-  replayFeedbackTrajectories,
-  replayFeedbackTrajectory,
-  runMultiShotOptimization,
-  runPromptEvolution,
-  scalarScore,
-  serializeFeedbackTrajectoriesJsonl,
-  summarizePreferenceMemory,
-  trialTraceFromMultiShotTrial,
-  withAssignedFeedbackSplit
-} from "./chunk-DMW5VENN.js";
 import {
   assertReleaseConfidence,
   bootstrapCi,
   evaluateReleaseConfidence,
   judgeReplayGate,
-  releaseTraceEvidenceFromMultiShotTrials,
   renderReleaseReport
-} from "./chunk-NKLGKF2Q.js";
+} from "./chunk-KQ26DYTQ.js";
 import {
   runEvalCampaign
-} from "./chunk-LCIDRYGP.js";
-import {
-  LlmCallError,
-  LlmClient,
-  LlmRouteAssertionError,
-  assertLlmRoute,
-  backoffMs,
-  callLlm,
-  callLlmJson,
-  isTransientLlmError,
-  probeLlm,
-  stripFencedJson
-} from "./chunk-VXNVVBZO.js";
+} from "./chunk-PD3MH6WU.js";
 import {
   AGENT_PROFILE_KINDS,
   AgentProfileCellValidationError,
@@ -150,17 +105,15 @@ import {
   validateRunRecord,
   verifyAgentProfileCell
 } from "./chunk-BWZEGTES.js";
-import {
-  evaluateInterimReleaseConfidence,
-  pairedEvalueSequence
-} from "./chunk-MAZ26DC7.js";
 import {
   RESEARCH_REPORT_HARD_PAIR_FLOOR,
+  evaluateInterimReleaseConfidence,
   gainHistogram,
+  pairedEvalueSequence,
   paretoChart,
   researchReport,
   summaryTable
-} from "./chunk-EGIPWXHL.js";
+} from "./chunk-MNL6LXGQ.js";
 import {
   benjaminiHochberg,
   bonferroni,
@@ -250,13 +203,6 @@ import {
   assertRunCaptured,
   throwIfRunIncomplete
 } from "./chunk-UBPIXOC4.js";
-import {
-  FileSystemRawProviderSink,
-  InMemoryRawProviderSink,
-  NoopRawProviderSink,
-  defaultProviderRedactor,
-  providerFromBaseUrl
-} from "./chunk-PC4UYEBM.js";
 import {
   TraceEmitter,
   llmSpanFromProvider
@@ -268,6 +214,25 @@ import {
   signManifest,
   verifyManifest
 } from "./chunk-VSMTAMNK.js";
+import {
+  LlmCallError,
+  LlmClient,
+  LlmRouteAssertionError,
+  assertLlmRoute,
+  backoffMs,
+  callLlm,
+  callLlmJson,
+  isTransientLlmError,
+  probeLlm,
+  stripFencedJson
+} from "./chunk-VXNVVBZO.js";
+import {
+  FileSystemRawProviderSink,
+  InMemoryRawProviderSink,
+  NoopRawProviderSink,
+  defaultProviderRedactor,
+  providerFromBaseUrl
+} from "./chunk-PC4UYEBM.js";
 import {
   AgentEvalError,
   CaptureIntegrityError,
@@ -853,8 +818,8 @@ function createJudgeAdapter(opts) {
     cost: opts.cost ?? { kind: "llm" },
     version: `judge-${ADAPTER_REV}`,
     async analyze(input) {
-      const scores = await opts.judge(opts.tcloud, input);
-      return scores.filter((s) => normalize10(s.score) < threshold).map((s) => liftJudgeScore(id, area, s));
+      const scores2 = await opts.judge(opts.tcloud, input);
+      return scores2.filter((s) => normalize10(s.score) < threshold).map((s) => liftJudgeScore(id, area, s));
     }
   };
 }
@@ -2203,10 +2168,10 @@ function ghCliClient(opts = {}) {
       await exec("git", ["branch", "-D", input.branchName], { cwd });
       await run("git", ["checkout", "-b", input.branchName]);
       const { mkdir, writeFile } = await import("fs/promises");
-      const { dirname: dirname6, join: join4, resolve } = await import("path");
+      const { dirname: dirname4, join: join4, resolve } = await import("path");
       for (const change of input.fileChanges) {
         const abs = resolve(cwd, change.path);
-        await mkdir(dirname6(abs), { recursive: true });
+        await mkdir(dirname4(abs), { recursive: true });
         await writeFile(abs, change.contents, "utf8");
         await run("git", ["add", join4(change.path)]);
       }
@@ -2404,8 +2369,8 @@ async function executeScenario(tc, scenario, config) {
           console.log(`    judge retry ${attempt}/2 (waiting ${wait / 1e3}s)`);
           await new Promise((r) => setTimeout(r, wait));
         }
-        const scores = await judge(tc, judgeInput);
-        judgeResults.push(scores);
+        const scores2 = await judge(tc, judgeInput);
+        judgeResults.push(scores2);
         await new Promise((r) => setTimeout(r, 3e3));
         break;
       } catch (err) {
@@ -3118,200 +3083,400 @@ ${lastResponse}` : "No conversation yet. Send your opening message \u2014 in cha
   return content.trim();
 }
 
-// src/integration-gates.ts
-function integrationManifestValidatedPayload(input) {
+// src/feedback-trajectory.ts
+var DEFAULT_SPLIT_POLICY = {
+  trainPct: 70,
+  devPct: 15,
+  testPct: 10,
+  holdoutPct: 5
+};
+var InMemoryFeedbackTrajectoryStore = class {
+  trajectories = /* @__PURE__ */ new Map();
+  async save(trajectory) {
+    this.trajectories.set(trajectory.id, cloneTrajectory(trajectory));
+  }
+  async get(id) {
+    const trajectory = this.trajectories.get(id);
+    return trajectory ? cloneTrajectory(trajectory) : null;
+  }
+  async list(filter = {}) {
+    return [...this.trajectories.values()].filter((trajectory) => matchesFilter(trajectory, filter)).map(cloneTrajectory);
+  }
+  async appendAttempt(id, attempt) {
+    const trajectory = this.trajectories.get(id);
+    if (!trajectory)
+      throw new Error(`FeedbackTrajectoryStore.appendAttempt: unknown trajectory "${id}"`);
+    const next = cloneTrajectory({
+      ...trajectory,
+      attempts: [...trajectory.attempts, attempt],
+      updatedAt: attempt.createdAt
+    });
+    this.trajectories.set(id, next);
+    return cloneTrajectory(next);
+  }
+  async appendLabel(id, label, attemptId) {
+    const trajectory = this.trajectories.get(id);
+    if (!trajectory)
+      throw new Error(`FeedbackTrajectoryStore.appendLabel: unknown trajectory "${id}"`);
+    const attempts = attemptId ? trajectory.attempts.map(
+      (attempt) => attempt.id === attemptId ? { ...attempt, feedback: [...attempt.feedback ?? [], label] } : attempt
+    ) : trajectory.attempts;
+    const next = cloneTrajectory({
+      ...trajectory,
+      attempts,
+      labels: attemptId ? trajectory.labels : [...trajectory.labels, label],
+      updatedAt: label.createdAt
+    });
+    this.trajectories.set(id, next);
+    return cloneTrajectory(next);
+  }
+};
+var FileSystemFeedbackTrajectoryStore = class {
+  dir;
+  memory = new InMemoryFeedbackTrajectoryStore();
+  loaded = false;
+  constructor(options) {
+    this.dir = options.dir;
+  }
+  async save(trajectory) {
+    await this.load();
+    await this.memory.save(trajectory);
+    await this.append({ op: "save", trajectory });
+  }
+  async get(id) {
+    await this.load();
+    return this.memory.get(id);
+  }
+  async list(filter = {}) {
+    await this.load();
+    return this.memory.list(filter);
+  }
+  async appendAttempt(id, attempt) {
+    await this.load();
+    const next = await this.memory.appendAttempt(id, attempt);
+    await this.append({ op: "appendAttempt", id, attempt });
+    return next;
+  }
+  async appendLabel(id, label, attemptId) {
+    await this.load();
+    const next = await this.memory.appendLabel(id, label, attemptId);
+    await this.append({ op: "appendLabel", id, label, attemptId });
+    return next;
+  }
+  async append(record) {
+    const { appendFile, mkdir } = await import("fs/promises");
+    const { join: join4 } = await import("path");
+    await mkdir(this.dir, { recursive: true });
+    await appendFile(
+      join4(this.dir, "feedback-trajectories.ndjson"),
+      `${JSON.stringify(record)}
+`,
+      "utf8"
+    );
+  }
+  async load() {
+    if (this.loaded) return;
+    const { readFile } = await import("fs/promises");
+    const { join: join4 } = await import("path");
+    const file = join4(this.dir, "feedback-trajectories.ndjson");
+    try {
+      const raw = await readFile(file, "utf8");
+      for (const line of raw.split("\n")) {
+        if (!line.trim()) continue;
+        try {
+          const record = JSON.parse(line);
+          if (record.op === "save") await this.memory.save(record.trajectory);
+          if (record.op === "appendAttempt")
+            await this.memory.appendAttempt(record.id, record.attempt);
+          if (record.op === "appendLabel")
+            await this.memory.appendLabel(record.id, record.label, record.attemptId);
+        } catch {
+        }
+      }
+    } catch {
+    }
+    this.loaded = true;
+  }
+};
+function createFeedbackTrajectory(input) {
+  const createdAt = input.createdAt ?? (/* @__PURE__ */ new Date()).toISOString();
+  const id = input.id ?? `ft_${stableHash(`${input.projectId ?? ""}|${input.scenarioId ?? ""}|${input.task.intent}|${createdAt}`).toString(16)}`;
   return {
-    kind: "integration_manifest_validated",
-    connectorId: input.connectorId,
-    ...input.actionId ? { actionId: input.actionId } : {},
-    valid: input.valid,
-    ...input.reason ? { reason: input.reason } : {},
-    ...input.metadata ? { metadata: input.metadata } : {}
+    id,
+    projectId: input.projectId,
+    scenarioId: input.scenarioId,
+    task: input.task,
+    attempts: input.attempts ?? [],
+    labels: input.labels ?? [],
+    outcome: input.outcome,
+    split: input.split,
+    tags: input.tags,
+    createdAt,
+    metadata: input.metadata
   };
 }
-function integrationManifestResolvedPayload(input) {
-  const missingConnections = input.missingConnections ?? [];
-  const missingScopes = input.missingScopes ?? [];
-  const requiredScopes = input.requiredScopes ?? [];
-  const status = input.status ?? statusForManifest(input);
+function assignFeedbackSplit(trajectory, policy = {}) {
+  const split = { ...DEFAULT_SPLIT_POLICY, ...policy };
+  const total = split.trainPct + split.devPct + split.testPct + split.holdoutPct;
+  if (total <= 0) throw new Error("assignFeedbackSplit: split percentages must sum above zero");
+  const bucket = stableHash(
+    `${trajectory.projectId ?? ""}|${trajectory.scenarioId ?? ""}|${trajectory.id}|${trajectory.task.intent}`
+  ) % total;
+  if (bucket < split.trainPct) return "train";
+  if (bucket < split.trainPct + split.devPct) return "dev";
+  if (bucket < split.trainPct + split.devPct + split.testPct) return "test";
+  return "holdout";
+}
+function withAssignedFeedbackSplit(trajectory, policy) {
   return {
-    kind: "integration_manifest_resolved",
-    connectorId: input.connectorId,
-    ...input.actionId ? { actionId: input.actionId } : {},
-    status,
-    missingConnections,
-    missingScopes,
-    requiredScopes,
-    missing: resolutionMissingItems(input, missingConnections, missingScopes, requiredScopes),
-    optionalMissing: [],
-    ready: status === "ready" ? [
-      {
-        status: "ready",
-        connectorId: input.connectorId,
-        ...input.actionId ? { actionId: input.actionId } : {},
-        requiredScopes
-      }
-    ] : [],
-    approvalRequired: input.approvalRequired ?? false,
-    ...input.reason ? { reason: input.reason } : {},
-    ...input.metadata ? { metadata: input.metadata } : {}
+    ...trajectory,
+    split: trajectory.split ?? assignFeedbackSplit(trajectory, policy)
   };
 }
-function integrationInvokeFailedPayload(input) {
+function feedbackTrajectoryToDatasetScenario(trajectory) {
+  const withSplit = withAssignedFeedbackSplit(trajectory);
   return {
-    kind: "integration_invoke_failed",
-    connectorId: input.connectorId,
-    actionId: input.actionId,
-    code: input.code,
-    message: input.message,
-    ...input.status !== void 0 ? { status: input.status } : {},
-    ...input.retryable !== void 0 ? { retryable: input.retryable } : {},
-    ...input.metadata ? { metadata: input.metadata } : {}
+    id: withSplit.scenarioId ?? withSplit.id,
+    split: withSplit.split,
+    payload: withSplit,
+    tags: {
+      ...withSplit.projectId ? { projectId: withSplit.projectId } : {},
+      ...withSplit.tags ?? {},
+      source: "feedback-trajectory"
+    }
   };
 }
-function integrationGateEvals(input) {
-  const evals = [];
-  evals.push(
-    objectiveEval({
-      id: `integration-manifest-valid:${input.connectorId}${input.actionId ? `:${input.actionId}` : ""}`,
-      passed: input.valid,
-      score: input.valid ? 1 : 0,
-      severity: input.valid ? "info" : "critical",
-      detail: input.valid ? "Integration manifest is valid." : input.reason ?? "Integration manifest is invalid.",
-      metadata: { integration: input }
-    })
-  );
-  const missingConnections = input.missingConnections ?? [];
-  evals.push(
-    objectiveEval({
-      id: `integration-connection-ready:${input.connectorId}`,
-      passed: missingConnections.length === 0,
-      score: missingConnections.length === 0 ? 1 : 0,
-      severity: missingConnections.length === 0 ? "info" : "critical",
-      detail: missingConnections.length === 0 ? "Required integration connections are present." : `Missing integration connection(s): ${missingConnections.join(", ")}`,
-      evidence: missingConnections.join(", ") || void 0,
-      metadata: { connectorId: input.connectorId, missingConnections }
-    })
-  );
-  const missingScopes = input.missingScopes ?? [];
-  evals.push(
-    objectiveEval({
-      id: `integration-scopes-ready:${input.connectorId}`,
-      passed: missingScopes.length === 0,
-      score: missingScopes.length === 0 ? 1 : 0,
-      severity: missingScopes.length === 0 ? "info" : "critical",
-      detail: missingScopes.length === 0 ? "Required integration scopes are granted." : `Missing integration scope(s): ${missingScopes.join(", ")}`,
-      evidence: missingScopes.join(", ") || void 0,
-      metadata: {
-        connectorId: input.connectorId,
-        missingScopes,
-        requiredScopes: input.requiredScopes ?? []
-      }
-    })
-  );
-  if (input.approvalRequired) {
-    evals.push(
-      objectiveEval({
-        id: `integration-approval-required:${input.connectorId}`,
-        passed: false,
-        score: 0,
-        severity: "warning",
-        detail: "Integration action requires approval before execution.",
-        metadata: { connectorId: input.connectorId, actionId: input.actionId }
-      })
-    );
-  }
-  return evals;
+function feedbackTrajectoriesToDatasetScenarios(trajectories) {
+  return trajectories.map(feedbackTrajectoryToDatasetScenario);
+}
+function feedbackTrajectoryToOptimizerRow(trajectory) {
+  const labels = allLabels(trajectory);
+  return {
+    scenarioId: trajectory.scenarioId ?? trajectory.id,
+    trajectoryId: trajectory.id,
+    labelKinds: [...new Set(labels.map((label) => label.kind))],
+    score: trajectory.outcome?.score ?? scoreFromLabels(labels),
+    metadata: {
+      projectId: trajectory.projectId,
+      split: trajectory.split,
+      intent: trajectory.task.intent,
+      attempts: trajectory.attempts.length,
+      outcome: trajectory.outcome,
+      labels
+    }
+  };
+}
+function feedbackTrajectoriesToOptimizerRows(trajectories) {
+  return trajectories.map(feedbackTrajectoryToOptimizerRow);
 }
-function integrationAsi(input) {
-  if ("code" in input) {
+async function replayFeedbackTrajectory(trajectory, adapter) {
+  try {
+    const result = await adapter.replay(trajectory);
     return {
-      expectationId: `integration-invoke:${input.connectorId}:${input.actionId}`,
-      message: input.message,
-      severity: severityForInvokeFailure(input.code),
-      responsibleSurface: surfaceForInvokeFailure(input.code),
-      suggestion: suggestionForInvokeFailure(input),
-      metadata: { integration: input }
+      trajectoryId: trajectory.id,
+      ...result
+    };
+  } catch (err) {
+    const createdAt = (/* @__PURE__ */ new Date()).toISOString();
+    const message = err instanceof Error ? err.message : String(err);
+    return {
+      trajectoryId: trajectory.id,
+      pass: false,
+      labels: [
+        {
+          source: "system",
+          kind: "reject",
+          value: false,
+          reason: message,
+          severity: "error",
+          createdAt
+        }
+      ],
+      outcome: {
+        success: false,
+        score: 0,
+        detail: message,
+        observedAt: createdAt
+      },
+      metadata: { replayError: true }
     };
   }
-  const missingConnections = input.missingConnections ?? [];
-  const missingScopes = input.missingScopes ?? [];
-  const surface = !input.valid ? "integration-manifest" : missingConnections.length > 0 ? "integration-connection" : missingScopes.length > 0 ? "integration-scope" : input.approvalRequired ? "integration-approval" : "integration-policy";
-  return {
-    expectationId: `integration-ready:${input.connectorId}${input.actionId ? `:${input.actionId}` : ""}`,
-    message: input.reason ?? messageForManifest(input),
-    severity: input.valid && missingConnections.length === 0 && missingScopes.length === 0 && !input.approvalRequired ? "info" : "error",
-    responsibleSurface: surface,
-    suggestion: suggestionForManifest(input),
-    metadata: { integration: input }
-  };
 }
-function statusForManifest(input) {
-  if (input.approvalRequired) return "approval_required";
-  if (!input.valid || (input.missingConnections?.length ?? 0) > 0 || (input.missingScopes?.length ?? 0) > 0)
-    return "blocked";
-  return "ready";
-}
-function resolutionMissingItems(input, missingConnections, missingScopes, requiredScopes) {
-  const connectionItems = missingConnections.map((connectorId) => ({
-    status: "missing_connection",
-    connectorId,
-    ...input.actionId ? { actionId: input.actionId } : {},
-    requiredScopes
-  }));
-  if (missingScopes.length === 0) return connectionItems;
-  return [
-    ...connectionItems,
-    {
-      status: "missing_scope",
-      connectorId: input.connectorId,
-      ...input.actionId ? { actionId: input.actionId } : {},
-      missingScopes,
-      requiredScopes
+async function replayFeedbackTrajectories(trajectories, adapter) {
+  const results = [];
+  for (const trajectory of trajectories) {
+    results.push(await replayFeedbackTrajectory(trajectory, adapter));
+  }
+  return results;
+}
+function summarizePreferenceMemory(trajectories, options = {}) {
+  const maxEntries = options.maxEntries ?? 20;
+  const entries = [];
+  for (const trajectory of trajectories) {
+    for (const label of allLabels(trajectory)) {
+      const instruction = instructionFromLabel(trajectory, label);
+      if (!instruction) continue;
+      entries.push({
+        instruction,
+        rationale: label.reason ?? `${label.kind} label from ${label.source}`,
+        weight: weightForLabel(label),
+        sourceTrajectoryId: trajectory.id,
+        sourceLabelId: label.id,
+        category: label.kind
+      });
+    }
+  }
+  const byInstruction = /* @__PURE__ */ new Map();
+  for (const entry of entries) {
+    const key = entry.instruction.toLowerCase().replace(/\s+/g, " ").trim();
+    const existing = byInstruction.get(key);
+    if (!existing || entry.weight > existing.weight) byInstruction.set(key, entry);
+  }
+  return [...byInstruction.values()].sort((a, b) => b.weight - a.weight).slice(0, maxEntries);
+}
+function renderPreferenceMemoryMarkdown(entries) {
+  const lines = ["# Preference Memory", ""];
+  for (const entry of entries) {
+    lines.push(`- ${entry.instruction}`);
+    lines.push(`  Rationale: ${entry.rationale}`);
+    lines.push(`  Source: ${entry.sourceTrajectoryId}`);
+    lines.push("");
+  }
+  return `${lines.join("\n").trim()}
+`;
+}
+function serializeFeedbackTrajectoriesJsonl(trajectories) {
+  return `${trajectories.slice().sort((a, b) => a.id.localeCompare(b.id)).map((trajectory) => JSON.stringify(canonicalize2(trajectory))).join("\n")}
+`;
+}
+function parseFeedbackTrajectoriesJsonl(jsonl) {
+  const trajectories = [];
+  for (const line of jsonl.split("\n")) {
+    if (!line.trim()) continue;
+    trajectories.push(JSON.parse(line));
+  }
+  return trajectories;
+}
+function controlRunToFeedbackTrajectory(run, options = {}) {
+  const createdAt = options.createdAt ?? (/* @__PURE__ */ new Date()).toISOString();
+  const trajectoryId = run.runId ?? `ft_control_${stableHash(`${run.intent}|${createdAt}`).toString(16)}`;
+  return createFeedbackTrajectory({
+    id: trajectoryId,
+    projectId: options.projectId,
+    scenarioId: options.scenarioId,
+    task: { intent: run.intent },
+    createdAt,
+    attempts: run.steps.map((step) => ({
+      id: `${trajectoryId}_step_${step.index}`,
+      stepIndex: step.index,
+      artifactType: options.artifactType ?? "action",
+      artifact: options.artifactFromStep?.(step) ?? step.actionOutcome?.result ?? step.decision,
+      proposedAction: options.proposedActionFromStep?.(step),
+      evals: step.evalsAfter,
+      createdAt: step.startedAt,
+      metadata: {
+        decision: step.decision,
+        actionOutcome: step.actionOutcome
+      }
+    })),
+    labels: [
+      {
+        source: "system",
+        kind: run.pass ? "approve" : "reject",
+        value: run.pass,
+        reason: run.reason,
+        severity: run.pass ? "info" : "error",
+        createdAt
+      }
+    ],
+    outcome: {
+      success: run.pass,
+      score: run.score,
+      costUsd: run.spentCostUsd,
+      detail: run.reason,
+      observedAt: createdAt,
+      metadata: {
+        stoppedBy: run.stoppedBy,
+        failureClass: run.failureClass
+      }
     }
+  });
+}
+function allLabels(trajectory) {
+  const labels = [
+    ...trajectory.labels,
+    ...trajectory.attempts.flatMap((attempt) => attempt.feedback ?? [])
   ];
+  const seen = /* @__PURE__ */ new Set();
+  return labels.filter((label) => {
+    const key = label.id ?? `${label.source}|${label.kind}|${label.createdAt}|${JSON.stringify(label.value)}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+function scoreFromLabels(labels) {
+  if (!labels.length) return void 0;
+  const scored = labels.map((label) => {
+    if (label.kind === "approve" || label.kind === "select") return 1;
+    if (label.kind === "reject" || label.kind === "policy_block") return 0;
+    if (label.kind === "rate" && typeof label.value === "number")
+      return Math.max(0, Math.min(1, label.value));
+    return void 0;
+  }).filter((value) => typeof value === "number");
+  if (!scored.length) return void 0;
+  return Math.round(scored.reduce((sum3, value) => sum3 + value, 0) / scored.length * 1e3) / 1e3;
+}
+function instructionFromLabel(trajectory, label) {
+  if (label.kind === "reject" && label.reason)
+    return `Avoid outputs like "${compact(trajectory.task.intent, 80)}" when: ${label.reason}`;
+  if (label.kind === "revision_request" && label.reason)
+    return `Revise similar work by applying: ${label.reason}`;
+  if (label.kind === "select" && label.reason)
+    return `Prefer selected options for "${compact(trajectory.task.intent, 80)}" because: ${label.reason}`;
+  if (label.kind === "approve" && label.reason)
+    return `Repeat the pattern approved for "${compact(trajectory.task.intent, 80)}": ${label.reason}`;
+  if (label.kind === "comment" && label.reason) return label.reason;
+  return void 0;
+}
+function weightForLabel(label) {
+  const severity = label.severity === "critical" ? 4 : label.severity === "error" ? 3 : label.severity === "warning" ? 2 : 1;
+  const source = label.source === "user" ? 3 : label.source === "metric" || label.source === "environment" ? 2 : 1;
+  return severity * source;
+}
+function matchesFilter(trajectory, filter) {
+  if (filter.projectId && trajectory.projectId !== filter.projectId) return false;
+  if (filter.scenarioId && trajectory.scenarioId !== filter.scenarioId) return false;
+  if (filter.split && trajectory.split !== filter.split) return false;
+  if (filter.tag) {
+    const [key, value] = filter.tag;
+    if (trajectory.tags?.[key] !== value) return false;
+  }
+  return true;
+}
+function cloneTrajectory(trajectory) {
+  return JSON.parse(JSON.stringify(trajectory));
 }
-function surfaceForInvokeFailure(code) {
-  if (code === "auth_expired") return "integration-auth";
-  if (code === "scope_denied") return "integration-scope";
-  if (code === "approval_required") return "integration-approval";
-  if (code === "unsafe_write_denied") return "integration-policy";
-  if (code === "manifest_invalid") return "integration-manifest";
-  return "integration-provider";
-}
-function severityForInvokeFailure(code) {
-  return code === "provider_failure" ? "warning" : "error";
-}
-function suggestionForInvokeFailure(input) {
-  if (input.code === "auth_expired") return `Reconnect ${input.connectorId} before retrying.`;
-  if (input.code === "scope_denied")
-    return `Request the missing scope for ${input.connectorId}.${input.actionId}.`;
-  if (input.code === "approval_required")
-    return `Ask the user to approve ${input.connectorId}.${input.actionId}.`;
-  if (input.code === "unsafe_write_denied")
-    return `Route ${input.connectorId}.${input.actionId} through the write-approval policy.`;
-  if (input.code === "manifest_invalid")
-    return `Fix the integration manifest for ${input.connectorId}.${input.actionId}.`;
-  return `Retry or degrade gracefully after ${input.connectorId} provider failure.`;
-}
-function messageForManifest(input) {
-  if (!input.valid) return `Integration manifest for ${input.connectorId} is invalid.`;
-  if ((input.missingConnections?.length ?? 0) > 0)
-    return `Missing connection for ${input.connectorId}.`;
-  if ((input.missingScopes?.length ?? 0) > 0)
-    return `Missing required scopes for ${input.connectorId}.`;
-  if (input.approvalRequired)
-    return `Approval required for ${input.connectorId}${input.actionId ? `.${input.actionId}` : ""}.`;
-  return `${input.connectorId} is ready.`;
-}
-function suggestionForManifest(input) {
-  if (!input.valid) return "Fix or regenerate the integration manifest before running the agent.";
-  if ((input.missingConnections?.length ?? 0) > 0)
-    return `Connect ${input.missingConnections.join(", ")} before replaying the workflow.`;
-  if ((input.missingScopes?.length ?? 0) > 0)
-    return `Request scopes: ${input.missingScopes.join(", ")}.`;
-  if (input.approvalRequired) return "Create an approval request and replay after approval.";
-  return "No action required.";
+function compact(value, max) {
+  const normalized = value.replace(/\s+/g, " ").trim();
+  return normalized.length > max ? `${normalized.slice(0, max).trim()}...` : normalized;
+}
+function stableHash(input) {
+  let hash = 2166136261;
+  for (let i = 0; i < input.length; i += 1) {
+    hash ^= input.charCodeAt(i);
+    hash = Math.imul(hash, 16777619);
+  }
+  return hash >>> 0;
+}
+function canonicalize2(value) {
+  if (value === null || typeof value !== "object") return value;
+  if (Array.isArray(value)) return value.map(canonicalize2);
+  const out = {};
+  for (const key of Object.keys(value).sort()) {
+    out[key] = canonicalize2(value[key]);
+  }
+  return out;
 }
 
 // src/integrity/backend-integrity.ts
@@ -3796,9 +3961,9 @@ function scorePrReviewComments(auditCase, comments, source, weights = {}) {
     })
   };
 }
-function summarizePrReviewBenchmark(scores) {
+function summarizePrReviewBenchmark(scores2) {
   const bySource = /* @__PURE__ */ new Map();
-  for (const score of scores) {
+  for (const score of scores2) {
     bySource.set(score.source, [...bySource.get(score.source) ?? [], score]);
   }
   return [...bySource.entries()].map(([source, sourceScores]) => ({
@@ -3913,396 +4078,6 @@ function sum(values) {
   return values.reduce((total, value) => total + value, 0);
 }
 
-// src/production-loop.ts
-async function runProductionLoop(opts) {
-  validate2(opts);
-  const now = opts.now ?? (() => /* @__PURE__ */ new Date());
-  const startedAt = now().toISOString();
-  const observedRuns = await opts.traceStore.listRuns();
-  const observedFeedback = await opts.feedbackStore.list();
-  const clusterReport = await failureClusterView(opts.traceStore, {
-    minClusterSize: opts.cluster.minClusterSize ?? 1
-  });
-  const minSize = opts.cluster.minClusterSize ?? 5;
-  const minSeverity = opts.cluster.minSeverityRatio ?? 0.05;
-  const maxClusters = opts.cluster.maxClustersPerCycle ?? 1;
-  const totalRuns = clusterReport.totalRuns;
-  const actionable = clusterReport.clusters.filter((c) => c.runCount >= minSize).filter((c) => totalRuns === 0 || c.runCount / totalRuns >= minSeverity).slice(0, maxClusters);
-  if (actionable.length === 0) {
-    return finalize({
-      opts,
-      decision: "no_actionable_failures",
-      startedAt,
-      now,
-      observedRunCount: observedRuns.length,
-      observedFeedbackCount: observedFeedback.length,
-      clusters: clusterReport.clusters,
-      actedOnCluster: null,
-      evolution: null,
-      release: null,
-      gate: null,
-      promotedPrompt: opts.evolve.baselinePrompt,
-      pullRequest: null
-    });
-  }
-  const actedOn = actionable[0];
-  const baseline = {
-    id: opts.evolve.baselineId ?? "baseline",
-    label: opts.evolve.baselineId ?? "baseline",
-    generation: 0,
-    payload: opts.evolve.baselinePrompt
-  };
-  const holdoutIds = uniqueIds(opts.evolve.holdoutScenarios.map((s) => s.id));
-  const searchIds = uniqueIds(
-    (opts.evolve.searchScenarios ?? deriveSearchScenarios(opts.evolve.holdoutScenarios)).map(
-      (s) => s.id
-    )
-  );
-  if (searchIds.some((id) => holdoutIds.includes(id))) {
-    throw new ValidationError(
-      "runProductionLoop: searchScenarios and holdoutScenarios must be disjoint"
-    );
-  }
-  const reps = opts.evolve.reps ?? 3;
-  const generations = opts.evolve.generations ?? 3;
-  const populationSize = opts.evolve.populationSize ?? Math.max(2, opts.evolve.reps ?? 4);
-  const evolution = await runMultiShotOptimization({
-    runId: `${opts.runId}/evolve`,
-    target: opts.target,
-    seedVariants: [baseline],
-    searchScenarioIds: searchIds,
-    reps,
-    generations,
-    populationSize,
-    scoreConcurrency: opts.evolve.scoreConcurrency ?? 1,
-    runner: opts.evolve.runner,
-    scorer: opts.evolve.scorer,
-    mutateAdapter: opts.evolve.mutator,
-    gate: {
-      holdoutScenarioIds: holdoutIds,
-      reps,
-      gate: { ...opts.evolve.gate, baselineKey: baseline.id },
-      toRunRecord: opts.evolve.toRunRecord ?? (({ variant, scenarioId, rep, split, seed, trial }) => syntheticRunRecord({
-        runId: `${opts.runId}-${variant.id}-${scenarioId}-${rep}-${split}`,
-        variant,
-        scenarioId,
-        rep,
-        split,
-        seed,
-        trial,
-        target: opts.target
-      }))
-    }
-  });
-  const gate = evolution.gate?.decision ?? null;
-  const promotedVariant = evolution.promotedVariant;
-  const promoted = promotedVariant.payload;
-  const promotedChanged = promotedVariant.id !== baseline.id;
-  const allTrials = evolution.evolution.generations.flatMap(
-    (g) => g.trials
-  );
-  const traceEvidence = releaseTraceEvidenceFromMultiShotTrials(allTrials);
-  const releaseScenarios = [
-    ...(opts.evolve.searchScenarios ?? []).map((s) => ({
-      id: s.id,
-      payload: s,
-      split: "train",
-      tags: { persona: s.persona, label: s.label }
-    })),
-    ...opts.evolve.holdoutScenarios.map((s) => ({
-      id: s.id,
-      payload: s,
-      split: "holdout",
-      tags: { persona: s.persona, label: s.label }
-    }))
-  ];
-  const release = evaluateReleaseConfidence({
-    target: opts.target,
-    candidateId: promotedVariant.id,
-    baselineId: baseline.id,
-    scenarios: releaseScenarios,
-    traces: traceEvidence,
-    gateDecision: gate ?? void 0,
-    thresholds: opts.releaseThresholds,
-    runs: [...evolution.gate?.candidateRuns ?? [], ...evolution.gate?.baselineRuns ?? []]
-  });
-  if (!promotedChanged) {
-    return finalize({
-      opts,
-      decision: "evolve_yielded_no_improvement",
-      startedAt,
-      now,
-      observedRunCount: observedRuns.length,
-      observedFeedbackCount: observedFeedback.length,
-      clusters: clusterReport.clusters,
-      actedOnCluster: actedOn,
-      evolution,
-      release,
-      gate,
-      promotedPrompt: promoted,
-      pullRequest: null
-    });
-  }
-  if (release.status === "fail" || gate && !gate.promote) {
-    return finalize({
-      opts,
-      decision: "gate_failed",
-      startedAt,
-      now,
-      observedRunCount: observedRuns.length,
-      observedFeedbackCount: observedFeedback.length,
-      clusters: clusterReport.clusters,
-      actedOnCluster: actedOn,
-      evolution,
-      release,
-      gate,
-      promotedPrompt: promoted,
-      pullRequest: null
-    });
-  }
-  if (!opts.ship) {
-    return finalize({
-      opts,
-      decision: "proposed_change",
-      startedAt,
-      now,
-      observedRunCount: observedRuns.length,
-      observedFeedbackCount: observedFeedback.length,
-      clusters: clusterReport.clusters,
-      actedOnCluster: actedOn,
-      evolution,
-      release,
-      gate,
-      promotedPrompt: promoted,
-      pullRequest: null
-    });
-  }
-  const baselineStr = toPromptString(baseline.payload);
-  const promotedStr = toPromptString(promoted);
-  const ctx = {
-    runId: opts.runId,
-    target: opts.target,
-    decision: "pr_opened",
-    clusters: clusterReport.clusters,
-    actedOnCluster: actedOn,
-    observedRunCount: observedRuns.length,
-    observedFeedbackCount: observedFeedback.length,
-    evolution,
-    release,
-    gate,
-    baselinePromptString: baselineStr,
-    promotedPromptString: promotedStr
-  };
-  const renderBody = opts.ship.renderBody ?? defaultRenderBody;
-  const renderFile = opts.ship.renderPromptFile ?? ((next, _prev) => `${next}
-`);
-  const currentFile = opts.ship.readCurrentPromptFile ? await opts.ship.readCurrentPromptFile() : null;
-  const pr = await proposeAutomatedPullRequest(opts.ship.client, {
-    repo: opts.ship.repo,
-    baseBranch: opts.ship.baseBranch ?? "main",
-    branchName: `${opts.ship.branchPrefix.replace(/\/+$/, "")}/${opts.runId}`,
-    title: `${opts.target}: production-loop prompt update (${opts.runId})`,
-    body: renderBody(ctx),
-    reviewers: opts.ship.reviewers,
-    labels: opts.ship.labels,
-    fileChanges: [
-      {
-        path: opts.ship.promptFilePath,
-        contents: renderFile(promotedStr, currentFile),
-        rationale: `Auto-improved against cluster "${actedOn.failureClass}" (${actedOn.runCount} prod failures)`
-      }
-    ],
-    dryRun: opts.ship.dryRun
-  });
-  return finalize({
-    opts,
-    decision: "pr_opened",
-    startedAt,
-    now,
-    observedRunCount: observedRuns.length,
-    observedFeedbackCount: observedFeedback.length,
-    clusters: clusterReport.clusters,
-    actedOnCluster: actedOn,
-    evolution,
-    release,
-    gate,
-    promotedPrompt: promoted,
-    pullRequest: pr
-  });
-}
-function finalize(args) {
-  return {
-    runId: args.opts.runId,
-    target: args.opts.target,
-    decision: args.decision,
-    startedAt: args.startedAt,
-    finishedAt: args.now().toISOString(),
-    observedRunCount: args.observedRunCount,
-    observedFeedbackCount: args.observedFeedbackCount,
-    clusters: args.clusters,
-    actedOnCluster: args.actedOnCluster,
-    evolution: args.evolution,
-    release: args.release,
-    gate: args.gate,
-    baselinePrompt: args.opts.evolve.baselinePrompt,
-    promotedPrompt: args.promotedPrompt,
-    pullRequest: args.pullRequest,
-    cron: args.opts.cron ?? null
-  };
-}
-function validate2(opts) {
-  if (!opts.runId.trim()) throw new ValidationError("runProductionLoop: runId required");
-  if (!opts.target.trim()) throw new ValidationError("runProductionLoop: target required");
-  if (opts.evolve.holdoutScenarios.length === 0) {
-    throw new ValidationError("runProductionLoop: evolve.holdoutScenarios must not be empty");
-  }
-  if (opts.evolve.searchScenarios && opts.evolve.searchScenarios.length === 0) {
-    throw new ValidationError(
-      "runProductionLoop: evolve.searchScenarios must be omitted or non-empty"
-    );
-  }
-  if (!opts.evolve.gate.baselineKey && !opts.evolve.baselineId) {
-  }
-  if (opts.ship) {
-    if (!opts.ship.branchPrefix.trim()) {
-      throw new ValidationError("runProductionLoop: ship.branchPrefix required");
-    }
-    if (!opts.ship.promptFilePath.trim()) {
-      throw new ValidationError("runProductionLoop: ship.promptFilePath required");
-    }
-  }
-}
-function uniqueIds(ids) {
-  const seen = /* @__PURE__ */ new Set();
-  const out = [];
-  for (const id of ids) {
-    if (seen.has(id)) continue;
-    seen.add(id);
-    out.push(id);
-  }
-  return out;
-}
-function deriveSearchScenarios(holdout) {
-  if (holdout.length < 4) {
-    return [
-      {
-        ...holdout[0],
-        id: `${holdout[0].id}__search`
-      }
-    ];
-  }
-  return holdout.filter((_, i) => i % 4 === 0).map((s) => ({ ...s, id: `${s.id}__search` }));
-}
-function syntheticRunRecord(input) {
-  const scoreKey = input.split === "holdout" ? "holdoutScore" : "searchScore";
-  return {
-    runId: input.runId,
-    experimentId: input.target,
-    candidateId: input.variant.id,
-    seed: input.seed,
-    model: "production-loop@synthetic",
-    promptHash: "0".repeat(64),
-    configHash: "0".repeat(64),
-    commitSha: "0".repeat(40),
-    wallMs: input.trial.durationMs ?? 1,
-    costUsd: input.trial.cost ?? 0,
-    tokenUsage: { input: 0, output: 0 },
-    outcome: {
-      [scoreKey]: input.trial.score,
-      raw: { score: input.trial.score, ok: input.trial.ok ? 1 : 0 }
-    },
-    splitTag: input.split,
-    scenarioId: input.scenarioId
-  };
-}
-function toPromptString(payload) {
-  if (typeof payload === "string") return payload;
-  if (payload == null) return "";
-  try {
-    return JSON.stringify(payload, null, 2);
-  } catch {
-    return String(payload);
-  }
-}
-function defaultRenderBody(ctx) {
-  const cluster = ctx.actedOnCluster;
-  const release = ctx.release;
-  const gate = ctx.gate;
-  const lines = [];
-  lines.push(`## Production-loop prompt update \u2014 \`${ctx.target}\``);
-  lines.push("");
-  lines.push(`Run id: \`${ctx.runId}\``);
-  lines.push(`Decision: \`${ctx.decision}\``);
-  lines.push(
-    `Observed in this cycle: ${ctx.observedRunCount} prod runs, ${ctx.observedFeedbackCount} feedback trajectories.`
-  );
-  lines.push("");
-  if (cluster) {
-    lines.push("### Triggering failure cluster");
-    lines.push("");
-    lines.push(`- **class**: \`${cluster.failureClass}\``);
-    lines.push(`- **runs in cluster**: ${cluster.runCount}`);
-    lines.push(`- **distinct scenarios**: ${cluster.scenarioIds.length}`);
-    if (cluster.toolName) lines.push(`- **tool**: \`${cluster.toolName}\``);
-    if (cluster.dimension) lines.push(`- **judge dimension**: \`${cluster.dimension}\``);
-    if (cluster.exampleError) {
-      lines.push(
-        `- **example error**: \`${cluster.exampleError.slice(0, 200).replace(/\n/g, " ")}\``
-      );
-    }
-    lines.push("");
-  }
-  if (gate) {
-    lines.push("### Held-out promotion gate");
-    lines.push("");
-    lines.push(`- **decision**: \`${gate.promote ? "PROMOTE" : "REJECT"}\``);
-    lines.push(`- **paired median delta**: ${gate.evidence.medianPairedDelta.toFixed(4)}`);
-    lines.push(
-      `- **paired 95% CI**: [${gate.evidence.pairedCI.low.toFixed(4)}, ${gate.evidence.pairedCI.high.toFixed(4)}]`
-    );
-    lines.push(`- **paired p-value**: ${gate.evidence.pairedPValue.toFixed(4)}`);
-    lines.push(
-      `- **search/holdout means**: ${gate.evidence.searchScore.toFixed(4)} / ${gate.evidence.holdoutScore.toFixed(4)}`
-    );
-    lines.push(`- **overfit gap**: ${gate.evidence.overfitGap.toFixed(4)}`);
-    lines.push("");
-  }
-  if (release) {
-    lines.push("### Release confidence");
-    lines.push("");
-    lines.push(`- **status**: \`${release.status}\``);
-    lines.push(`- **pass rate**: ${release.metrics.passRate.toFixed(4)}`);
-    lines.push(`- **mean score**: ${release.metrics.meanScore.toFixed(4)}`);
-    if (release.issues.length > 0) {
-      lines.push("- **issues**:");
-      for (const issue of release.issues) {
-        lines.push(`  - \`${issue.severity}\` ${issue.axis}: ${issue.detail}`);
-      }
-    }
-    lines.push("");
-  }
-  lines.push("### Prompt diff");
-  lines.push("");
-  lines.push("```diff");
-  lines.push(unifiedDiff(ctx.baselinePromptString, ctx.promotedPromptString));
-  lines.push("```");
-  return lines.join("\n");
-}
-function unifiedDiff(a, b) {
-  const aLines = a.split("\n");
-  const bLines = b.split("\n");
-  const out = [];
-  const max = Math.max(aLines.length, bLines.length);
-  for (let i = 0; i < max; i++) {
-    const al = aLines[i];
-    const bl = bLines[i];
-    if (al === bl) continue;
-    if (al !== void 0) out.push(`- ${al}`);
-    if (bl !== void 0) out.push(`+ ${bl}`);
-  }
-  return out.join("\n");
-}
-
 // src/registry.ts
 var ScenarioRegistry = class {
   scenarios = [];
@@ -5395,6 +5170,89 @@ var FileSystemExperimentStore = class {
   }
 };
 
+// src/pareto.ts
+function dominates(a, b, objectives) {
+  let strictlyBetter = false;
+  for (const obj of objectives) {
+    const av = obj.value(a);
+    const bv = obj.value(b);
+    if (!Number.isFinite(av) || !Number.isFinite(bv)) return false;
+    const aIsBetter = obj.direction === "maximize" ? av > bv : av < bv;
+    const aIsWorse = obj.direction === "maximize" ? av < bv : av > bv;
+    if (aIsWorse) return false;
+    if (aIsBetter) strictlyBetter = true;
+  }
+  return strictlyBetter;
+}
+function paretoFrontier(candidates, objectives) {
+  if (objectives.length === 0) {
+    throw new Error("paretoFrontier: at least 1 objective required");
+  }
+  const valid = candidates.filter((c) => objectives.every((o) => Number.isFinite(o.value(c))));
+  const frontier = [];
+  const dominated = [];
+  for (const c of valid) {
+    const isDominated = valid.some((other) => other !== c && dominates(other, c, objectives));
+    if (isDominated) dominated.push(c);
+    else frontier.push(c);
+  }
+  const dominanceMap = frontier.map((d) => ({
+    dominator: d,
+    dominated: dominated.filter((x) => dominates(d, x, objectives))
+  }));
+  return { frontier, dominated, dominanceMap };
+}
+function scalarScore(candidates, objectives, options = {}) {
+  if (candidates.length === 0) return [];
+  const weights = options.weights ?? {};
+  const totalWeight = objectives.reduce((s, o) => s + (weights[o.name] ?? 1), 0);
+  const ranges = objectives.map((obj) => {
+    const values = candidates.map((c) => obj.value(c)).filter((v) => Number.isFinite(v));
+    if (values.length === 0) return { min: 0, max: 1 };
+    const min = Math.min(...values);
+    const max = Math.max(...values);
+    return { min, max: max === min ? min + 1 : max };
+  });
+  return candidates.map((c) => {
+    let score = 0;
+    objectives.forEach((obj, i) => {
+      const v = obj.value(c);
+      if (!Number.isFinite(v)) return;
+      const { min, max } = ranges[i];
+      const normalised = (v - min) / (max - min);
+      const directional = obj.direction === "maximize" ? normalised : 1 - normalised;
+      const weight = (weights[obj.name] ?? 1) / totalWeight;
+      score += directional * weight;
+    });
+    return { candidate: c, score };
+  });
+}
+function crowdingDistance(candidates, objectives) {
+  const distances = new Map(candidates.map((c) => [c, 0]));
+  for (const obj of objectives) {
+    const sorted = [...candidates].sort((a, b) => obj.value(a) - obj.value(b));
+    const min = obj.value(sorted[0]);
+    const max = obj.value(sorted[sorted.length - 1]);
+    const range = max - min || 1;
+    distances.set(sorted[0], Infinity);
+    distances.set(sorted[sorted.length - 1], Infinity);
+    for (let i = 1; i < sorted.length - 1; i++) {
+      const prev = obj.value(sorted[i - 1]);
+      const next = obj.value(sorted[i + 1]);
+      const current = distances.get(sorted[i]);
+      if (current === Infinity) continue;
+      distances.set(sorted[i], current + (next - prev) / range);
+    }
+  }
+  return candidates.map((c) => ({ candidate: c, distance: distances.get(c) ?? 0 }));
+}
+function paretoFrontierWithCrowding(candidates, objectives) {
+  const { frontier } = paretoFrontier(candidates, objectives);
+  if (frontier.length === 0) return [];
+  const distances = crowdingDistance(frontier, objectives);
+  return distances.sort((a, b) => b.distance - a.distance);
+}
+
 // src/harness-optimizer.ts
 var DEFAULT_HARNESS_OBJECTIVES = [
   { name: "aggregate", direction: "maximize", value: (r) => r.aggregateMean },
@@ -5485,20 +5343,20 @@ async function mapLimit(items, limit, fn) {
 function mean2(values) {
   return values.length ? values.reduce((sum3, value) => sum3 + value, 0) / values.length : 0;
 }
-function meanRunScore(scores) {
+function meanRunScore(scores2) {
   return {
-    success: mean2(scores.map((s) => s.success)),
-    goalProgress: mean2(scores.map((s) => s.goalProgress)),
-    repoGroundedness: mean2(scores.map((s) => s.repoGroundedness)),
-    driftPenalty: mean2(scores.map((s) => s.driftPenalty)),
-    toolUseQuality: mean2(scores.map((s) => s.toolUseQuality)),
-    patchQuality: mean2(scores.map((s) => s.patchQuality)),
-    testReality: mean2(scores.map((s) => s.testReality)),
-    finalGate: mean2(scores.map((s) => s.finalGate)),
-    reviewerBlockers: mean2(scores.map((s) => s.reviewerBlockers)),
-    costUsd: mean2(scores.map((s) => s.costUsd)),
-    wallSeconds: mean2(scores.map((s) => s.wallSeconds)),
-    notes: scores.flatMap((s) => s.notes ?? [])
+    success: mean2(scores2.map((s) => s.success)),
+    goalProgress: mean2(scores2.map((s) => s.goalProgress)),
+    repoGroundedness: mean2(scores2.map((s) => s.repoGroundedness)),
+    driftPenalty: mean2(scores2.map((s) => s.driftPenalty)),
+    toolUseQuality: mean2(scores2.map((s) => s.toolUseQuality)),
+    patchQuality: mean2(scores2.map((s) => s.patchQuality)),
+    testReality: mean2(scores2.map((s) => s.testReality)),
+    finalGate: mean2(scores2.map((s) => s.finalGate)),
+    reviewerBlockers: mean2(scores2.map((s) => s.reviewerBlockers)),
+    costUsd: mean2(scores2.map((s) => s.costUsd)),
+    wallSeconds: mean2(scores2.map((s) => s.wallSeconds)),
+    notes: scores2.flatMap((s) => s.notes ?? [])
   };
 }
 
@@ -6473,12 +6331,12 @@ function recordRuns(runs, opts) {
   for (const [scenarioId, scenarioRuns] of byScenario) {
     const scored = scenarioRuns.map((run) => ({ run, score: runScore(run) })).filter((s) => s.score !== void 0);
     if (scored.length === 0) continue;
-    const scores = scored.map((s) => s.score);
+    const scores2 = scored.map((s) => s.score);
     const entry = {
       commitSha: opts.commitSha,
       timestamp,
-      scores,
-      composite: median(scores),
+      scores: scores2,
+      composite: median(scores2),
       runIds: scored.map((s) => s.run.runId)
     };
     const perDimension = aggregatePerDimension(scenarioRuns);
@@ -6600,17 +6458,17 @@ function formatScorecardDiff(diff) {
   lines.push(
     `Scorecard: ${summary.regressed} regressed \xB7 ${summary.improved} improved \xB7 ${summary.flat} flat \xB7 ${summary.new} new`
   );
-  const fmt = (n) => n.toFixed(3);
+  const fmt2 = (n) => n.toFixed(3);
   const noteworthy = diff.cells.filter((c) => c.verdict === "regressed" || c.verdict === "improved").sort((a, b) => {
     if (a.verdict !== b.verdict) return a.verdict === "regressed" ? -1 : 1;
     return Math.abs(b.delta ?? 0) - Math.abs(a.delta ?? 0);
   });
   for (const cell of noteworthy) {
     const mark = cell.verdict === "regressed" ? "REGRESSED" : "improved";
-    const deltaStr = cell.delta !== null ? cell.delta >= 0 ? `+${fmt(cell.delta)}` : fmt(cell.delta) : "\u2014";
+    const deltaStr = cell.delta !== null ? cell.delta >= 0 ? `+${fmt2(cell.delta)}` : fmt2(cell.delta) : "\u2014";
     const stat = cell.cohensD !== null ? ` (d=${cell.cohensD.toFixed(2)}${cell.pValue !== null ? `, p=${cell.pValue.toFixed(3)}` : ""})` : "";
     lines.push(
-      `  ${mark}  ${cell.scenarioId} \xB7 ${cell.model} \xB7 ${cell.profileHash.slice(0, 8)}  ${fmt(cell.baseline ?? 0)} \u2192 ${fmt(cell.current)}  ${deltaStr}${stat}`
+      `  ${mark}  ${cell.scenarioId} \xB7 ${cell.model} \xB7 ${cell.profileHash.slice(0, 8)}  ${fmt2(cell.baseline ?? 0)} \u2192 ${fmt2(cell.current)}  ${deltaStr}${stat}`
     );
   }
   return lines.join("\n");
@@ -6625,10 +6483,10 @@ function analyzeSeries(values, options = {}) {
     return { state: "insufficient-data", windowMean: 0, windowCv: 0, tailRun: 0, stable: false };
   }
   const tail = values.slice(-window);
-  const mean4 = tail.reduce((a, b) => a + b, 0) / tail.length;
-  const variance = tail.reduce((acc, v) => acc + (v - mean4) ** 2, 0) / tail.length;
+  const mean5 = tail.reduce((a, b) => a + b, 0) / tail.length;
+  const variance = tail.reduce((acc, v) => acc + (v - mean5) ** 2, 0) / tail.length;
   const stdDev = Math.sqrt(variance);
-  const refMean = Math.abs(mean4) > 1e-9 ? Math.abs(mean4) : 1;
+  const refMean = Math.abs(mean5) > 1e-9 ? Math.abs(mean5) : 1;
   const cv = stdDev / refMean;
   const stable = tail.length >= window && cv <= stableCv;
   let tailRun = 0;
@@ -6649,7 +6507,7 @@ function analyzeSeries(values, options = {}) {
   } else {
     state = "noisy";
   }
-  return { state, windowMean: mean4, windowCv: cv, tailRun, stable };
+  return { state, windowMean: mean5, windowCv: cv, tailRun, stable };
 }
 
 // src/slo.ts
@@ -7027,12 +6885,12 @@ function renderMarkdownReport(reports) {
 async function aggregateRunMetrics(runs, store) {
   if (runs.length === 0) return {};
   const durations = [];
-  const scores = [];
+  const scores2 = [];
   const passes = [];
   const costs = [];
   for (const r of runs) {
     if (r.endedAt) durations.push(r.endedAt - r.startedAt);
-    if (r.outcome?.score !== void 0) scores.push(r.outcome.score);
+    if (r.outcome?.score !== void 0) scores2.push(r.outcome.score);
     passes.push(r.outcome?.pass === true ? 1 : 0);
     const llm = await llmSpans(store, r.runId);
     costs.push(aggregateLlm(llm).costUsd);
@@ -7041,7 +6899,7 @@ async function aggregateRunMetrics(runs, store) {
     provisionMs: average(durations),
     firstTokenMs: average(durations),
     wallMs: average(durations),
-    overallScore: average(scores),
+    overallScore: average(scores2),
     passRate: average(passes),
     costUsd: average(costs)
   };
@@ -7205,7 +7063,7 @@ async function toLangfuseEnvelope(store, runId) {
     },
     metadata: { finishReason: s.finishReason, cachedTokens: s.cachedTokens }
   }));
-  const scores = judges.map((j) => ({
+  const scores2 = judges.map((j) => ({
     id: j.spanId,
     traceId: run.runId,
     observationId: j.targetSpanId,
@@ -7213,7 +7071,7 @@ async function toLangfuseEnvelope(store, runId) {
     value: j.score,
     comment: j.rationale
   }));
-  return { traceId: run.runId, generations, scores };
+  return { traceId: run.runId, generations, scores: scores2 };
 }
 async function toPrometheusText(store) {
   const runs = await store.listRuns();
@@ -7314,12 +7172,12 @@ async function paraphraseRobustness(prompt, mutators, scoreFn, options = {}) {
     variantScores.push({ mutator: id, score, mutated });
     all.push(score);
   }
-  const mean4 = all.reduce((a, b) => a + b, 0) / all.length;
-  const variance = all.reduce((a, v) => a + (v - mean4) ** 2, 0) / all.length;
+  const mean5 = all.reduce((a, b) => a + b, 0) / all.length;
+  const variance = all.reduce((a, v) => a + (v - mean5) ** 2, 0) / all.length;
   const stdDev = Math.sqrt(variance);
-  const ref = Math.abs(mean4) > 1e-9 ? Math.abs(mean4) : 1;
+  const ref = Math.abs(mean5) > 1e-9 ? Math.abs(mean5) : 1;
   const robustness = Math.max(0, 1 - stdDev / ref);
-  return { originalScore, variantScores, meanScore: mean4, stdDev, robustness };
+  return { originalScore, variantScores, meanScore: mean5, stdDev, robustness };
 }
 var lowercaseMutator = (p) => p.toLowerCase();
 var sentenceReorderMutator = (p, seed) => {
@@ -7376,18 +7234,18 @@ async function paraphraseRobustnessScenarios(args) {
     const deltas = {};
     const paraphrasedAll = [];
     for (const m of args.mutators) {
-      const scores = [];
+      const scores2 = [];
       for (let r = 0; r < reps; r++) {
         const mutatedTurns = scenario.userTurns.map((t) => m.mutator(t));
         const out = await args.runScenario({
           id: scenario.id,
           userTurns: mutatedTurns
         });
-        scores.push(out.score);
+        scores2.push(out.score);
       }
-      const mean4 = scores.reduce((a, b) => a + b, 0) / scores.length;
-      deltas[m.name] = mean4 - originalScore;
-      paraphrasedAll.push(...scores);
+      const mean5 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
+      deltas[m.name] = mean5 - originalScore;
+      paraphrasedAll.push(...scores2);
     }
     const paraphrasedMean = paraphrasedAll.length === 0 ? originalScore : paraphrasedAll.reduce((a, b) => a + b, 0) / paraphrasedAll.length;
     perScenario.push({ id: scenario.id, originalScore, paraphrasedMean, deltas });
@@ -7802,10 +7660,10 @@ async function proposeSynthesisTargets(dataset, traceStore, options = {}) {
   }
   for (const s of scenarios) {
     const sRuns = runs.filter((r) => r.scenarioId === s.id);
-    const scores = sRuns.map((r) => r.outcome?.score).filter((x) => typeof x === "number");
-    if (scores.length < 3) continue;
-    const mean4 = scores.reduce((a, b) => a + b, 0) / scores.length;
-    const variance = scores.reduce((a, b) => a + (b - mean4) ** 2, 0) / scores.length;
+    const scores2 = sRuns.map((r) => r.outcome?.score).filter((x) => typeof x === "number");
+    if (scores2.length < 3) continue;
+    const mean5 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
+    const variance = scores2.reduce((a, b) => a + (b - mean5) ** 2, 0) / scores2.length;
     if (variance > varianceThreshold) {
       targets.push({
         reason: "high-variance",
@@ -7979,15 +7837,15 @@ async function runSelfPlay(proposer, scorer, targets, options = {}) {
     const rejected = [];
     const surviving = [];
     for (const candidate of proposed) {
-      const scores = await scorer.scoreCandidate(candidate, targets);
-      if (scores.length < 2) {
+      const scores2 = await scorer.scoreCandidate(candidate, targets);
+      if (scores2.length < 2) {
         rejected.push({ candidate, reason: "scorer returned <2 results" });
         continue;
       }
-      const values = scores.map((s) => s.score);
+      const values = scores2.map((s) => s.score);
       const spread = Math.max(...values) - Math.min(...values);
       const maxScore = Math.max(...values);
-      scored.push({ candidate, scores, spread });
+      scored.push({ candidate, scores: scores2, spread });
       if (maxScore < floor) {
         rejected.push({
           candidate,
@@ -9138,15 +8996,15 @@ function scoreReferenceReplay(scenarios, options = {}) {
   const threshold = options.matchThreshold ?? DEFAULT_MATCH_THRESHOLD;
   const matchStrategy = options.matchStrategy ?? "reference-order";
   const allowedSplits = new Set(options.splits ?? ALL_SPLITS);
-  const scores = scenarios.filter((scenario) => {
+  const scores2 = scenarios.filter((scenario) => {
     const split = scenario.split ?? "train";
     if (split === "holdout" && !options.includeHoldout) return false;
     return allowedSplits.has(split);
   }).map((scenario) => scoreScenario(scenario, matcher, threshold, matchStrategy));
   return {
-    scenarios: scores,
-    aggregate: aggregateScenarioScores(scores),
-    bySplit: aggregateBySplit(scores)
+    scenarios: scores2,
+    aggregate: aggregateScenarioScores(scores2),
+    bySplit: aggregateBySplit(scores2)
   };
 }
 function compareReferenceReplay(baseline, candidate) {
@@ -9369,20 +9227,20 @@ function buildScenarioScore(scenario, matches, falsePositives) {
     matches
   };
 }
-function aggregateBySplit(scores) {
+function aggregateBySplit(scores2) {
   const out = {};
   for (const split of ALL_SPLITS) {
-    const scoped = scores.filter((score) => score.split === split);
+    const scoped = scores2.filter((score) => score.split === split);
     if (scoped.length > 0) out[split] = aggregateScenarioScores(scoped);
   }
   return out;
 }
-function aggregateScenarioScores(scores) {
-  const matched = sum2(scores.map((score) => score.matched));
-  const total = sum2(scores.map((score) => score.total));
-  const falsePositives = sum2(scores.map((score) => score.falsePositives));
-  const matchedWeight = sum2(scores.map((score) => score.matchedWeight));
-  const totalWeight = sum2(scores.map((score) => score.totalWeight));
+function aggregateScenarioScores(scores2) {
+  const matched = sum2(scores2.map((score) => score.matched));
+  const total = sum2(scores2.map((score) => score.total));
+  const falsePositives = sum2(scores2.map((score) => score.falsePositives));
+  const matchedWeight = sum2(scores2.map((score) => score.matchedWeight));
+  const totalWeight = sum2(scores2.map((score) => score.totalWeight));
   const precision2 = ratio(matched, matched + falsePositives);
   const recall = ratio(matched, total);
   return {
@@ -9625,154 +9483,6 @@ function createDefaultReviewer(options) {
   };
 }
 
-// src/code-mutator.ts
-function createSandboxCodeMutator(opts) {
-  const childIdFor = opts.childIdFor ?? ((parent, generation, index) => `${parent.id}.g${generation}.code.${index}`);
-  const labelFor = opts.labelFor ?? ((outcome, parent, _generation, index) => outcome.description?.slice(0, 80) ?? `${parent.label} \u2192 code.${index}`);
-  return {
-    async mutate(args) {
-      const { parent, parentAggregate, topTrials, bottomTrials, childCount, generation } = args;
-      const startedAt = Date.now();
-      const outcomes = await opts.pool.withSlot(async (slot) => {
-        try {
-          return await opts.runner({
-            slot,
-            parent,
-            parentAggregate,
-            topTrials,
-            bottomTrials,
-            childCount,
-            generation
-          });
-        } catch (err) {
-          return [
-            {
-              ok: false,
-              failureReason: "runner_error",
-              description: err instanceof Error ? err.message : String(err),
-              latencyMs: Date.now() - startedAt
-            }
-          ];
-        }
-      });
-      const variants = [];
-      let index = 0;
-      for (const outcome of outcomes) {
-        const childId = outcome.childId ?? childIdFor(parent, generation, index);
-        if (opts.mutationTelemetry) {
-          await opts.mutationTelemetry.record({
-            ts: Date.now(),
-            channel: "code",
-            generation,
-            parentId: parent.id,
-            childId: outcome.ok ? childId : null,
-            ok: outcome.ok,
-            failureReason: outcome.failureReason,
-            description: outcome.description,
-            latencyMs: outcome.latencyMs,
-            diffBytes: outcome.diffBytes,
-            filesTouched: outcome.filesTouched,
-            agentSteps: outcome.agentSteps,
-            costUsd: outcome.costUsd
-          });
-        }
-        if (opts.costLedger && outcome.costUsd !== void 0) {
-          await opts.costLedger.addMutation("code", outcome.costUsd, { generation });
-        }
-        if (outcome.ok) {
-          const variant = {
-            id: childId,
-            payload: opts.toVariantPayload(outcome, parent),
-            generation,
-            parentId: parent.id,
-            label: labelFor(outcome, parent, generation, index),
-            ...outcome.rationale ? { rationale: outcome.rationale } : {}
-          };
-          variants.push(variant);
-          if (opts.lineage) {
-            await opts.lineage.upsert({
-              id: variant.id,
-              parentId: variant.parentId ?? null,
-              generation: variant.generation,
-              kind: "code",
-              ...variant.rationale ? { rationale: variant.rationale } : {}
-            });
-          }
-        }
-        index++;
-      }
-      if (opts.costLedger) {
-        const u = opts.pool.utilization();
-        await opts.costLedger.setPoolUtilization(u.busyMs, u.totalMs);
-      }
-      return variants;
-    }
-  };
-}
-
-// src/composite-mutator.ts
-function createCompositeMutator(opts) {
-  const recentScores = [];
-  const plateauThreshold = opts.plateauThreshold ?? 0.02;
-  const plateauPatience = opts.plateauPatience ?? 2;
-  function pickMode(args) {
-    recentScores.push(args.parentAggregate.meanScore);
-    switch (opts.policy) {
-      case "primary-only":
-        return { mode: "primary", reason: "policy=primary-only" };
-      case "secondary-only":
-        if (!opts.secondary)
-          return {
-            mode: "primary",
-            reason: "secondary-only requested but no secondary mutator wired"
-          };
-        return { mode: "secondary", reason: "policy=secondary-only" };
-      case "alternate":
-        if (!opts.secondary)
-          return { mode: "primary", reason: "alternate requested but no secondary mutator wired" };
-        return args.generation % 2 === 1 ? { mode: "secondary", reason: `alternate: gen${args.generation} odd \u2192 secondary` } : { mode: "primary", reason: `alternate: gen${args.generation} even \u2192 primary` };
-      case "plateau": {
-        if (!opts.secondary)
-          return { mode: "primary", reason: "plateau requested but no secondary mutator wired" };
-        if (recentScores.length <= plateauPatience) {
-          return { mode: "primary", reason: "plateau: warming up with primary mutations" };
-        }
-        const window = recentScores.slice(-plateauPatience - 1);
-        const deltas = window.slice(1).map((v, i) => v - window[i]);
-        const stagnant = deltas.every((d) => d < plateauThreshold);
-        if (stagnant) {
-          return {
-            mode: "split",
-            reason: `plateau detected (${deltas.map((d) => d.toFixed(3)).join(", ")}) \u2192 split`
-          };
-        }
-        return {
-          mode: "primary",
-          reason: `plateau: still improving (${deltas[deltas.length - 1].toFixed(3)})`
-        };
-      }
-    }
-  }
-  return {
-    async mutate(args) {
-      const { mode, reason } = pickMode(args);
-      opts.onPolicyDecision?.({ generation: args.generation, chose: mode, reason });
-      if (mode === "primary") return opts.primary.mutate(args);
-      if (mode === "secondary" && opts.secondary) return opts.secondary.mutate(args);
-      if (mode === "split" && opts.secondary) {
-        const secondaryShare = Math.ceil(args.childCount / 2);
-        const primaryShare = args.childCount - secondaryShare;
-        const [primaryChildren, secondaryChildren] = await Promise.all([
-          opts.primary.mutate({ ...args, childCount: primaryShare }),
-          opts.secondary.mutate({ ...args, childCount: secondaryShare })
-        ]);
-        return [...primaryChildren, ...secondaryChildren];
-      }
-      return opts.primary.mutate(args);
-    }
-  };
-}
-
 // src/discover-personas.ts
 import { promises as fs } from "fs";
 import { basename, extname, join as join3 } from "path";
@@ -9819,238 +9529,6 @@ async function discoverPersonas(dir, opts = {}) {
   return results;
 }
 
-// src/evolution-telemetry.ts
-import { appendFileSync as appendFileSync4, existsSync as existsSync7, mkdirSync as mkdirSync4, readFileSync as readFileSync6, writeFileSync } from "fs";
-import { dirname as dirname4 } from "path";
-var MutationTelemetry = class {
-  appender;
-  constructor(path) {
-    this.appender = new LockedJsonlAppender(path);
-  }
-  async record(attempt) {
-    await this.appender.append(attempt);
-  }
-};
-var TrialTelemetry = class {
-  appender;
-  constructor(path) {
-    this.appender = new LockedJsonlAppender(path);
-  }
-  async record(attempt) {
-    await this.appender.append(attempt);
-  }
-};
-var LineageRecorder = class {
-  path;
-  snapshotPath;
-  mutex = new Mutex();
-  nodes = /* @__PURE__ */ new Map();
-  kindOf;
-  constructor(path, kindOf) {
-    this.path = path;
-    this.snapshotPath = `${path}.snapshot`;
-    this.kindOf = kindOf ?? defaultKindOf;
-    mkdirSync4(dirname4(path), { recursive: true });
-    if (existsSync7(this.snapshotPath)) {
-      try {
-        const parsed = JSON.parse(readFileSync6(this.snapshotPath, "utf-8"));
-        for (const n of parsed) this.nodes.set(n.id, n);
-      } catch {
-      }
-    }
-    if (existsSync7(path)) {
-      try {
-        for (const line of readFileSync6(path, "utf-8").split("\n")) {
-          if (!line.trim()) continue;
-          try {
-            const entry = JSON.parse(line);
-            const prev = this.nodes.get(entry.id);
-            this.nodes.set(entry.id, { ...prev, ...entry });
-          } catch {
-          }
-        }
-      } catch {
-      }
-    }
-    if (existsSync7(path) && this.nodes.size === 0) {
-      try {
-        const raw = readFileSync6(path, "utf-8").trim();
-        if (raw.startsWith("[")) {
-          const parsed = JSON.parse(raw);
-          for (const n of parsed) this.nodes.set(n.id, n);
-        }
-      } catch {
-      }
-    }
-  }
-  async upsert(node) {
-    await this.mutex.runExclusive(() => {
-      const prev = this.nodes.get(node.id);
-      this.nodes.set(node.id, { ...prev, ...node });
-      try {
-        if (existsSync7(this.path)) {
-          const head = readFileSync6(this.path, { encoding: "utf-8", flag: "r" }).slice(0, 1);
-          if (head === "[") {
-            writeFileSync(this.path, "");
-          }
-        }
-      } catch {
-      }
-      appendFileSync4(this.path, `${JSON.stringify(this.nodes.get(node.id))}
-`);
-    });
-  }
-  async upsertVariant(variant, opts = {}) {
-    await this.upsert({
-      id: variant.id,
-      parentId: variant.parentId ?? null,
-      generation: variant.generation,
-      kind: this.kindOf(variant),
-      ...variant.rationale ? { rationale: variant.rationale } : {},
-      ...opts.omitPayload || variant.payload === void 0 ? {} : { payload: variant.payload }
-    });
-  }
-  snapshot() {
-    return [...this.nodes.values()];
-  }
-  /**
-   * Write the current consolidated state to `<path>.snapshot` so external
-   * tools can read it without replaying the event log. Idempotent.
-   */
-  async compact() {
-    await this.mutex.runExclusive(() => {
-      writeFileSync(this.snapshotPath, JSON.stringify([...this.nodes.values()], null, 2));
-    });
-  }
-};
-function defaultKindOf(variant) {
-  if (variant.parentId === void 0) return "seed";
-  const payload = variant.payload;
-  if (payload && typeof payload === "object" && payload.codeMutation) return "code";
-  return "prompt";
-}
-function emptyGenBucket() {
-  return {
-    mutatorPromptUsd: 0,
-    mutatorCodeUsd: 0,
-    scorerPromptUsd: 0,
-    scorerCodeUsd: 0,
-    trialsCounted: 0,
-    cachedTrials: 0
-  };
-}
-var CostLedger = class {
-  totals = {
-    mutatorPromptUsd: 0,
-    mutatorCodeUsd: 0,
-    scorerPromptUsd: 0,
-    scorerCodeUsd: 0,
-    trialsCounted: 0,
-    cachedTrials: 0,
-    poolBusyMs: 0,
-    poolUtilizationPct: 0,
-    byGeneration: {}
-  };
-  path;
-  mutex = new Mutex();
-  constructor(path) {
-    this.path = path;
-    if (existsSync7(path)) {
-      try {
-        const loaded = JSON.parse(readFileSync6(path, "utf-8"));
-        for (const k of Object.keys(this.totals)) {
-          if (k === "byGeneration") {
-            if (loaded.byGeneration && typeof loaded.byGeneration === "object") {
-              this.totals.byGeneration = loaded.byGeneration;
-            }
-            continue;
-          }
-          const v = loaded[k];
-          if (typeof v === "number" && Number.isFinite(v)) {
-            ;
-            this.totals[k] = v;
-          }
-        }
-      } catch {
-      }
-    } else {
-      mkdirSync4(dirname4(path), { recursive: true });
-    }
-  }
-  genBucket(generation) {
-    if (generation === void 0) return null;
-    const key = String(generation);
-    if (!this.totals.byGeneration[key]) {
-      this.totals.byGeneration[key] = emptyGenBucket();
-    }
-    return this.totals.byGeneration[key];
-  }
-  async addMutation(channel, usd, opts = {}) {
-    await this.mutex.runExclusive(() => {
-      const bucket = this.genBucket(opts.generation);
-      if (channel === "prompt") {
-        this.totals.mutatorPromptUsd += usd;
-        if (bucket) bucket.mutatorPromptUsd += usd;
-      } else {
-        this.totals.mutatorCodeUsd += usd;
-        if (bucket) bucket.mutatorCodeUsd += usd;
-      }
-      this.persist();
-    });
-  }
-  async addTrial(channel, usd, cached, opts = {}) {
-    await this.mutex.runExclusive(() => {
-      const bucket = this.genBucket(opts.generation);
-      if (cached) {
-        this.totals.cachedTrials++;
-        this.totals.trialsCounted++;
-        if (bucket) {
-          bucket.cachedTrials++;
-          bucket.trialsCounted++;
-        }
-        this.persist();
-        return;
-      }
-      if (channel === "prompt") {
-        this.totals.scorerPromptUsd += usd;
-        if (bucket) bucket.scorerPromptUsd += usd;
-      } else {
-        this.totals.scorerCodeUsd += usd;
-        if (bucket) bucket.scorerCodeUsd += usd;
-      }
-      this.totals.trialsCounted++;
-      if (bucket) bucket.trialsCounted++;
-      this.persist();
-    });
-  }
-  async setPoolUtilization(busyMs, totalMs) {
-    await this.mutex.runExclusive(() => {
-      this.totals.poolBusyMs = busyMs;
-      this.totals.poolUtilizationPct = totalMs > 0 ? 100 * busyMs / totalMs : 0;
-      this.persist();
-    });
-  }
-  snapshot() {
-    const totalUsd = this.totals.mutatorPromptUsd + this.totals.mutatorCodeUsd + this.totals.scorerPromptUsd + this.totals.scorerCodeUsd;
-    const byGeneration = Object.entries(this.totals.byGeneration).map(([g, b]) => ({ generation: Number(g), ...b })).sort((a, b) => a.generation - b.generation);
-    return {
-      totalUsd,
-      mutatorPromptUsd: this.totals.mutatorPromptUsd,
-      mutatorCodeUsd: this.totals.mutatorCodeUsd,
-      scorerPromptUsd: this.totals.scorerPromptUsd,
-      scorerCodeUsd: this.totals.scorerCodeUsd,
-      trialsCounted: this.totals.trialsCounted,
-      cachedTrials: this.totals.cachedTrials,
-      poolBusyMs: this.totals.poolBusyMs,
-      poolUtilizationPct: this.totals.poolUtilizationPct,
-      byGeneration
-    };
-  }
-  persist() {
-    writeFileSync(this.path, JSON.stringify(this.totals, null, 2));
-  }
-};
-
 // src/golden-matcher.ts
 function matchGoldens(goldens, candidates, options = {}) {
   const extract = options.text ?? defaultExtract2;
@@ -10125,52 +9603,164 @@ function precision(goldens, candidates, options = {}) {
   return matched / candidates.length;
 }
 
-// src/jsonl-trial-cache.ts
-import { appendFileSync as appendFileSync5, existsSync as existsSync8, mkdirSync as mkdirSync5, readFileSync as readFileSync7 } from "fs";
-import { dirname as dirname5 } from "path";
-var JsonlTrialCache = class {
-  map = /* @__PURE__ */ new Map();
-  path;
-  appender;
-  constructor(path) {
-    this.path = path;
-    if (existsSync8(path)) {
-      for (const line of readFileSync7(path, "utf-8").split("\n")) {
-        if (!line.trim()) continue;
-        try {
-          const entry = JSON.parse(line);
-          this.map.set(entry.key, entry.result);
-        } catch {
-        }
-      }
-    } else {
-      mkdirSync5(dirname5(path), { recursive: true });
+// src/held-out-gate.ts
+var HeldOutGate = class {
+  minProductiveRuns;
+  pairedDeltaThreshold;
+  overfitGapThreshold;
+  baselineKey;
+  confidence;
+  resamples;
+  seed;
+  constructor(config) {
+    if (!config.baselineKey) {
+      throw new Error("HeldOutGate: baselineKey is required");
+    }
+    this.minProductiveRuns = config.minProductiveRuns ?? 3;
+    this.pairedDeltaThreshold = config.pairedDeltaThreshold ?? 0;
+    this.overfitGapThreshold = config.overfitGapThreshold ?? 0.15;
+    this.baselineKey = config.baselineKey;
+    this.confidence = config.confidence ?? 0.95;
+    this.resamples = config.bootstrapResamples ?? 2e3;
+    this.seed = config.seed;
+  }
+  /** Decide whether `candidate` should replace `baseline`. Pairing
+   *  is by (experimentId, seed) — identical experiment + seed pairs
+   *  the candidate run with the matching baseline run. Pairs without
+   *  a holdout score on both sides are dropped. */
+  evaluate(candidate, baseline) {
+    const candidateId = inferCandidateId(candidate, this.baselineKey);
+    const baselineId = this.baselineKey;
+    const baselineHoldoutByKey = indexHoldoutByKey(baseline);
+    const beforeHoldout = [];
+    const afterHoldout = [];
+    for (const run of candidate) {
+      if (run.splitTag !== "holdout") continue;
+      if (run.outcome.holdoutScore === void 0) continue;
+      const key = pairKey(run);
+      const counterpart = baselineHoldoutByKey.get(key);
+      if (counterpart === void 0) continue;
+      beforeHoldout.push(counterpart);
+      afterHoldout.push(run.outcome.holdoutScore);
+    }
+    const productiveRuns = beforeHoldout.length;
+    const candidateSearchMean = mean4(scores(candidate, "searchScore", "search"));
+    const candidateHoldoutMean = mean4(scores(candidate, "holdoutScore", "holdout"));
+    const baselineSearchMean = mean4(scores(baseline, "searchScore", "search"));
+    const baselineHoldoutMean = mean4(scores(baseline, "holdoutScore", "holdout"));
+    const overfitGap = safeDiff(candidateSearchMean, candidateHoldoutMean);
+    const baselineOverfitGap = safeDiff(baselineSearchMean, baselineHoldoutMean);
+    if (productiveRuns < this.minProductiveRuns) {
+      return {
+        promote: false,
+        candidateId,
+        baselineId,
+        evidence: {
+          productiveRuns,
+          medianPairedDelta: productiveRuns > 0 ? medianDelta(beforeHoldout, afterHoldout) : 0,
+          pairedCI: { low: 0, high: 0 },
+          pairedPValue: 1,
+          searchScore: candidateSearchMean,
+          holdoutScore: candidateHoldoutMean,
+          overfitGap,
+          baselineOverfitGap
+        },
+        reason: `few_runs: ${productiveRuns} paired holdout observation(s) < min ${this.minProductiveRuns}`,
+        rejectionCode: "few_runs"
+      };
     }
-    this.appender = new LockedJsonlAppender(path);
-  }
-  get(key) {
-    return this.map.get(key);
+    const ci = pairedBootstrap(beforeHoldout, afterHoldout, {
+      confidence: this.confidence,
+      resamples: this.resamples,
+      statistic: "median",
+      seed: this.seed
+    });
+    const wilcoxon = wilcoxonSignedRank(beforeHoldout, afterHoldout);
+    const evidence = {
+      productiveRuns,
+      medianPairedDelta: ci.median,
+      pairedCI: { low: ci.low, high: ci.high },
+      pairedPValue: wilcoxon.p,
+      searchScore: candidateSearchMean,
+      holdoutScore: candidateHoldoutMean,
+      overfitGap,
+      baselineOverfitGap
+    };
+    if (!(ci.low > this.pairedDeltaThreshold)) {
+      return {
+        promote: false,
+        candidateId,
+        baselineId,
+        evidence,
+        reason: `negative_delta: paired holdout median \u0394=${fmt(ci.median)} CI=[${fmt(ci.low)}, ${fmt(ci.high)}] does not clear threshold ${fmt(this.pairedDeltaThreshold)}`,
+        rejectionCode: "negative_delta"
+      };
+    }
+    if (Number.isFinite(overfitGap) && Number.isFinite(baselineOverfitGap) && overfitGap > baselineOverfitGap + this.overfitGapThreshold) {
+      return {
+        promote: false,
+        candidateId,
+        baselineId,
+        evidence,
+        reason: `overfit_gap: candidate gap=${fmt(overfitGap)} exceeds baseline gap=${fmt(baselineOverfitGap)} by more than ${fmt(this.overfitGapThreshold)}`,
+        rejectionCode: "overfit_gap"
+      };
+    }
+    return {
+      promote: true,
+      candidateId,
+      baselineId,
+      evidence,
+      reason: `promote: paired holdout median \u0394=${fmt(ci.median)} CI=[${fmt(ci.low)}, ${fmt(ci.high)}] over ${productiveRuns} pairs; overfit gap candidate=${fmt(overfitGap)} vs baseline=${fmt(baselineOverfitGap)}`,
+      rejectionCode: null
+    };
   }
-  set(key, value) {
-    this.map.set(key, value);
-    const line = { key, result: value, writtenAt: Date.now() };
-    void this.appender.append(line);
+};
+function inferCandidateId(candidate, baselineKey) {
+  for (const run of candidate) {
+    if (run.candidateId && run.candidateId !== baselineKey) return run.candidateId;
   }
-  size() {
-    return this.map.size;
+  return candidate[0]?.candidateId ?? "(unknown candidate)";
+}
+function indexHoldoutByKey(runs) {
+  const out = /* @__PURE__ */ new Map();
+  for (const r of runs) {
+    if (r.splitTag !== "holdout") continue;
+    if (r.outcome.holdoutScore === void 0) continue;
+    out.set(pairKey(r), r.outcome.holdoutScore);
   }
-  /**
-   * Synchronous fallback path for tests / CLI tools that want to be sure
-   * the line is on disk before returning. Bypasses the mutex (single-
-   * threaded callers only).
-   */
-  setSync(key, value) {
-    this.map.set(key, value);
-    const line = { key, result: value, writtenAt: Date.now() };
-    appendFileSync5(this.path, `${JSON.stringify(line)}
-`);
+  return out;
+}
+function pairKey(r) {
+  return `${r.experimentId}::${r.seed}`;
+}
+function scores(runs, field, splitFilter) {
+  const out = [];
+  for (const r of runs) {
+    if (r.splitTag !== splitFilter) continue;
+    const v = r.outcome[field];
+    if (typeof v === "number" && Number.isFinite(v)) out.push(v);
   }
-};
+  return out;
+}
+function mean4(xs) {
+  if (xs.length === 0) return Number.NaN;
+  return xs.reduce((s, x) => s + x, 0) / xs.length;
+}
+function safeDiff(a, b) {
+  if (!Number.isFinite(a) || !Number.isFinite(b)) return Number.NaN;
+  return a - b;
+}
+function medianDelta(before, after) {
+  const ds = before.map((b, i) => after[i] - b).sort((x, y) => x - y);
+  if (ds.length === 0) return 0;
+  const mid = Math.floor(ds.length / 2);
+  return ds.length % 2 === 0 ? (ds[mid - 1] + ds[mid]) / 2 : ds[mid];
+}
+function fmt(x) {
+  if (!Number.isFinite(x)) return String(x);
+  return x.toFixed(4);
+}
 
 // src/judge-retry.ts
 var DEFAULT_MAX_ATTEMPTS = 3;
@@ -10250,9 +9840,9 @@ function passOrthogonality(input) {
       sims.push(cosineSimilarity(vectors[i], vectors[j]));
     }
   }
-  const mean4 = sims.length === 0 ? 0 : sims.reduce((a, b) => a + b, 0) / sims.length;
+  const mean5 = sims.length === 0 ? 0 : sims.reduce((a, b) => a + b, 0) / sims.length;
   return {
-    orthogonality: Math.max(0, Math.min(1, 1 - mean4)),
+    orthogonality: Math.max(0, Math.min(1, 1 - mean5)),
     passCount: passes.length,
     similarities: sims
   };
@@ -10351,6 +9941,44 @@ function referenceReplayScenarioToRunScore(scenarioScore, durationMs = 0) {
   };
 }
 
+// src/researcher.ts
+var CallbackResearcher = class {
+  constructor(callbacks) {
+    this.callbacks = callbacks;
+  }
+  callbacks;
+  inspectFailures(runs) {
+    return this.callbacks.inspectFailures(runs);
+  }
+  proposeChange(failures) {
+    return this.callbacks.proposeChange(failures);
+  }
+  applyChange(changes, baseline) {
+    return this.callbacks.applyChange(changes, baseline);
+  }
+  evaluateChange(plan) {
+    return this.callbacks.evaluateChange(plan);
+  }
+};
+var NoopResearcher = class {
+  hint;
+  constructor(hint = "NoopResearcher: no implementation wired") {
+    this.hint = hint;
+  }
+  async inspectFailures(_runs) {
+    throw new Error(`${this.hint} (inspectFailures not implemented)`);
+  }
+  async proposeChange(_failures) {
+    throw new Error(`${this.hint} (proposeChange not implemented)`);
+  }
+  async applyChange(_changes, _baseline) {
+    throw new Error(`${this.hint} (applyChange not implemented)`);
+  }
+  async evaluateChange(_plan) {
+    throw new Error(`${this.hint} (evaluateChange not implemented)`);
+  }
+};
+
 // src/sandbox-pool.ts
 function createSandboxPool(opts) {
   if (opts.size < 1) throw new Error(`sandbox pool size must be >= 1 (got ${opts.size})`);
@@ -10453,55 +10081,6 @@ function createSandboxPool(opts) {
   };
 }
 
-// src/trial-aggregator.ts
-function meanOf(xs) {
-  if (xs.length === 0) return 0;
-  return xs.reduce((a, b) => a + b, 0) / xs.length;
-}
-function meanMetrics(rows) {
-  if (rows.length === 0) return {};
-  const keys = /* @__PURE__ */ new Set();
-  for (const row of rows) for (const k of Object.keys(row)) keys.add(k);
-  const out = {};
-  for (const k of keys) {
-    const xs = rows.map((r) => r[k]).filter((x) => typeof x === "number");
-    if (xs.length > 0) out[k] = meanOf(xs);
-  }
-  return out;
-}
-function aggregateTrialsByMode(trials, opts) {
-  const gradedTrials = trials.filter((t) => !t.error);
-  const judgeOk = gradedTrials.filter((t) => t.judgeSucceeded !== false);
-  const judgeFailed = gradedTrials.filter((t) => t.judgeSucceeded === false);
-  if (opts.mode === "strict-fail" && judgeFailed.length > 0) {
-    return {
-      meanScore: 0,
-      meanCost: 0,
-      meanDurationMs: 0,
-      okRate: 0,
-      countedTrials: 0,
-      excludedFailedTrials: judgeFailed.length,
-      totalTrials: trials.length,
-      metrics: {},
-      strictFailure: {
-        failedCount: judgeFailed.length,
-        firstError: judgeFailed.find((t) => t.judgeError)?.judgeError
-      }
-    };
-  }
-  const counted = opts.mode === "exclude-failed" ? judgeOk : gradedTrials;
-  return {
-    meanScore: meanOf(counted.map((t) => t.score)),
-    meanCost: meanOf(counted.map((t) => t.cost ?? 0)),
-    meanDurationMs: meanOf(counted.map((t) => t.durationMs ?? 0)),
-    okRate: gradedTrials.length === 0 ? 0 : gradedTrials.filter((t) => t.ok).length / gradedTrials.length,
-    countedTrials: counted.length,
-    excludedFailedTrials: judgeFailed.length,
-    totalTrials: trials.length,
-    metrics: meanMetrics(counted.map((t) => t.metrics ?? {}))
-  };
-}
-
 // src/otel-pipeline.ts
 function withOtelPipeline(opts) {
   const config = {
@@ -10594,17 +10173,17 @@ function traceJudge(judge, judgeName, opts) {
       }
     });
     try {
-      const scores = await judge(tc, input);
-      const composite = scores.length > 0 ? scores.reduce((sum3, s) => sum3 + s.score, 0) / scores.length : 0;
+      const scores2 = await judge(tc, input);
+      const composite = scores2.length > 0 ? scores2.reduce((sum3, s) => sum3 + s.score, 0) / scores2.length : 0;
       await span.end({
         attributes: {
           "judge.name": judgeName,
           "judge.composite_score": composite,
-          "judge.dimension_count": scores.length,
+          "judge.dimension_count": scores2.length,
           "eval.phase": "judge"
         }
       });
-      return scores;
+      return scores2;
     } catch (err) {
       await span.fail(err instanceof Error ? err : String(err));
       throw err;
@@ -10631,8 +10210,8 @@ function traceJudgeEnsemble(judges, judgeNames, opts) {
           emitter: opts.emitter,
           parentSpanId: ensembleSpan.span.spanId
         });
-        const scores = await tracedFn(tc, input);
-        allScores.push(...scores);
+        const scores2 = await tracedFn(tc, input);
+        allScores.push(...scores2);
       }
       const composite = allScores.length > 0 ? allScores.reduce((sum3, s) => sum3 + s.score, 0) / allScores.length : 0;
       await ensembleSpan.end({
@@ -10650,48 +10229,6 @@ function traceJudgeEnsemble(judges, judgeNames, opts) {
     }
   };
 }
-
-// src/traced-mutator.ts
-function traceMutator(adapter, opts) {
-  return {
-    async mutate(args) {
-      const span = await opts.emitter.span({
-        kind: "llm",
-        name: `mutator:gen-${args.generation}`,
-        parentSpanId: opts.parentSpanId,
-        attributes: {
-          "mutator.parent_id": args.parent.id,
-          "mutator.generation": args.generation,
-          "mutator.child_count": args.childCount,
-          "mutator.top_trials": args.topTrials.length,
-          "mutator.bottom_trials": args.bottomTrials.length,
-          "mutator.parent_score": args.parentAggregate.meanScore,
-          "eval.phase": "mutator"
-        }
-      });
-      try {
-        const children = await adapter.mutate(args);
-        await span.end({
-          attributes: {
-            "mutator.parent_id": args.parent.id,
-            "mutator.generation": args.generation,
-            "mutator.child_count": args.childCount,
-            "mutator.top_trials": args.topTrials.length,
-            "mutator.bottom_trials": args.bottomTrials.length,
-            "mutator.parent_score": args.parentAggregate.meanScore,
-            "mutator.produced_count": children.length,
-            "mutator.child_ids": children.map((c) => c.id).join(","),
-            "eval.phase": "mutator"
-          }
-        });
-        return children;
-      } catch (err) {
-        await span.fail(err instanceof Error ? err : String(err));
-        throw err;
-      }
-    }
-  };
-}
 export {
   AGENT_PROFILE_KINDS,
   ANALYST_SEVERITIES,
@@ -10709,7 +10246,6 @@ export {
   CaptureIntegrityError,
   ConfigError,
   ConvergenceTracker,
-  CostLedger,
   CostTracker,
   D1ExperimentStore,
   DEFAULT_AGENT_SLOS,
@@ -10750,15 +10286,12 @@ export {
   InMemoryFeedbackTrajectoryStore,
   InMemoryRawProviderSink,
   InMemoryTraceStore,
-  InMemoryTrialCache,
   InMemoryWorkspaceInspector,
-  JsonlTrialCache,
   JudgeError,
   JudgeRunner,
   KIND_EXPECTED_SUBJECTS,
   KNOWLEDGE_GAP_KIND_SPEC,
   KNOWLEDGE_POISONING_KIND_SPEC,
-  LineageRecorder,
   LlmCallError,
   LlmClient,
   LlmRouteAssertionError,
@@ -10766,7 +10299,6 @@ export {
   MODEL_PRICING,
   MetricsCollector,
   MultiLayerVerifier,
-  MutationTelemetry,
   Mutex,
   NoopRawProviderSink,
   NoopResearcher,
@@ -10800,7 +10332,6 @@ export {
   TraceEmitter,
   TraceFileMissingError,
   TraceNotFoundError,
-  TrialTelemetry,
   UNIVERSAL_FINDERS,
   ValidationError,
   VerificationError,
@@ -10812,7 +10343,6 @@ export {
   aggregateLlm,
   aggregatePrReviewScore,
   aggregateRunScore,
-  aggregateTrialsByMode,
   allCriticalPassed,
   analyzeAntiSlop,
   analyzeSeries,
@@ -10881,7 +10411,6 @@ export {
   corpusInterRaterAgreementFromJudgeScores,
   createAntiSlopJudge,
   createChatClient,
-  createCompositeMutator,
   createCustomJudge,
   createDefaultReviewer,
   createDomainExpertJudge,
@@ -10894,7 +10423,6 @@ export {
   createOtelTracingStore,
   createReplayFetch,
   createRunCriticAdapter,
-  createSandboxCodeMutator,
   createSandboxPool,
   createSemanticConceptJudge,
   createSemanticConceptJudgeAdapter,
@@ -10908,7 +10436,6 @@ export {
   decideReferenceReplayRunPromotion,
   defaultIsMaterial,
   defaultJudges,
-  defaultMultiShotObjectives,
   defaultProviderRedactor,
   defaultReferenceReplayMatcher,
   defaultTraceInsightPanel,
@@ -10966,11 +10493,6 @@ export {
   inMemoryReferenceReplayStore,
   inMemoryReviewStore,
   inferDomainKeywords,
-  integrationAsi,
-  integrationGateEvals,
-  integrationInvokeFailedPayload,
-  integrationManifestResolvedPayload,
-  integrationManifestValidatedPayload,
   interRaterReliability,
   iqr,
   isJudgeSpan,
@@ -11048,7 +10570,6 @@ export {
   referenceReplayScenarioToRunScore,
   regexMatch,
   regexMatches,
-  releaseTraceEvidenceFromMultiShotTrials,
   renderFindingSubject,
   renderMarkdown,
   renderMarkdownReport,
@@ -11083,9 +10604,6 @@ export {
   runKeywordCoverageJudge,
   runKeywordCoverageJudgeUrl,
   runLiveProof,
-  runMultiShotOptimization,
-  runProductionLoop,
-  runPromptEvolution,
   runProposeReview,
   runProposeReviewAsControlLoop,
   runReferenceReplay,
@@ -11134,9 +10652,7 @@ export {
   traceAnalystOnRunComplete,
   traceJudge,
   traceJudgeEnsemble,
-  traceMutator,
   tracedAnalyzeTraces,
-  trialTraceFromMultiShotTrial,
   typoMutator,
   urlContains,
   userQuestionsForKnowledgeGaps,