@tangle-network/agent-eval 0.17.3 → 0.19.0

package/dist/index.js

@@ -3473,174 +3473,6 @@ function rowToRun(row) {
   };
 }
 
-// src/power-analysis.ts
-function requiredSampleSize(opts) {
-  const effect = opts.effect;
-  if (!Number.isFinite(effect) || effect <= 0) return Infinity;
-  const alpha = opts.alpha ?? 0.05;
-  const power = opts.power ?? 0.8;
-  const twoSided = opts.twoSided ?? true;
-  const zAlpha = zQuantile(twoSided ? 1 - alpha / 2 : 1 - alpha);
-  const zBeta = zQuantile(power);
-  const n = 2 * Math.pow((zAlpha + zBeta) / effect, 2);
-  return Math.ceil(n);
-}
-function bonferroni(pValues, alpha = 0.05) {
-  const k = pValues.length;
-  const adjusted = pValues.map((p) => Math.min(1, p * k));
-  const significant = adjusted.map((p) => p < alpha);
-  return { adjusted, significant };
-}
-function benjaminiHochberg(pValues, fdr = 0.05) {
-  const n = pValues.length;
-  if (n === 0) return { qValues: [], significant: [] };
-  const indexed = pValues.map((p, i) => ({ p, i })).sort((a, b) => a.p - b.p);
-  const q = new Array(n);
-  let minRight = 1;
-  for (let k = n - 1; k >= 0; k--) {
-    const rank = k + 1;
-    const raw = indexed[k].p * n / rank;
-    const bounded = Math.min(minRight, raw);
-    minRight = bounded;
-    q[indexed[k].i] = Math.min(1, bounded);
-  }
-  const significant = q.map((v) => v < fdr);
-  return { qValues: q, significant };
-}
-function zQuantile(p) {
-  if (p <= 0 || p >= 1) {
-    if (p === 0) return -Infinity;
-    if (p === 1) return Infinity;
-    return NaN;
-  }
-  const a = [-39.69683028665376, 220.9460984245205, -275.9285104469687, 138.357751867269, -30.66479806614716, 2.506628277459239];
-  const b = [-54.47609879822406, 161.5858368580409, -155.6989798598866, 66.80131188771972, -13.28068155288572];
-  const c = [-0.007784894002430293, -0.3223964580411365, -2.400758277161838, -2.549732539343734, 4.374664141464968, 2.938163982698783];
-  const d = [0.007784695709041462, 0.3224671290700398, 2.445134137142996, 3.754408661907416];
-  const pLow = 0.02425;
-  const pHigh = 1 - pLow;
-  let q;
-  let r;
-  if (p < pLow) {
-    q = Math.sqrt(-2 * Math.log(p));
-    return (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
-  }
-  if (p <= pHigh) {
-    q = p - 0.5;
-    r = q * q;
-    return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q / (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1);
-  }
-  q = Math.sqrt(-2 * Math.log(1 - p));
-  return -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
-}
-
-// src/prompt-optimizer.ts
-var PromptOptimizer = class {
-  async run(config) {
-    const trials = config.trialsPerScenario ?? 3;
-    const alpha = config.significanceLevel ?? 0.05;
-    if (config.variants.length < 2) {
-      throw new Error("PromptOptimizer requires at least 2 variants");
-    }
-    if (config.scenarioIds.length === 0) {
-      throw new Error("PromptOptimizer requires at least 1 scenario");
-    }
-    const rawScores = /* @__PURE__ */ new Map();
-    for (const variant of config.variants) {
-      const scenarioMap = /* @__PURE__ */ new Map();
-      rawScores.set(variant.id, scenarioMap);
-      for (const scenarioId of config.scenarioIds) {
-        const samples = [];
-        for (let t = 0; t < trials; t++) {
-          const score = await config.scoreVariant({
-            variant,
-            scenarioId,
-            trialIndex: t
-          });
-          if (!Number.isFinite(score)) {
-            throw new Error(`scoreVariant returned non-finite: variant=${variant.id} scenario=${scenarioId} trial=${t}`);
-          }
-          samples.push(score);
-        }
-        scenarioMap.set(scenarioId, samples);
-        config.onScenarioComplete?.({
-          variantId: variant.id,
-          scenarioId,
-          scores: samples
-        });
-      }
-    }
-    const scores2 = config.variants.map((variant) => {
-      const scenarioMap = rawScores.get(variant.id);
-      const allSamples = [];
-      const perScenario = {};
-      for (const scenarioId of config.scenarioIds) {
-        const samples = scenarioMap.get(scenarioId) ?? [];
-        allSamples.push(...samples);
-        perScenario[scenarioId] = {
-          mean: samples.length ? samples.reduce((a, b) => a + b, 0) / samples.length : 0,
-          n: samples.length,
-          samples
-        };
-      }
-      const ci = confidenceInterval(allSamples, 0.95);
-      return {
-        variantId: variant.id,
-        mean: ci.mean,
-        ci95: { lower: ci.lower, upper: ci.upper },
-        n: allSamples.length,
-        perScenario
-      };
-    });
-    const rawPairs = [];
-    for (let i = 0; i < scores2.length; i++) {
-      for (let j = i + 1; j < scores2.length; j++) {
-        const a = scores2[i];
-        const b = scores2[j];
-        const { p } = mannWhitneyU(flatSamples(a), flatSamples(b));
-        rawPairs.push({ a, b, p });
-      }
-    }
-    const { qValues } = benjaminiHochberg(rawPairs.map((r) => r.p), alpha);
-    const pairwise2 = rawPairs.map((r, idx) => ({
-      variantA: r.a.variantId,
-      variantB: r.b.variantId,
-      pValue: r.p,
-      qValue: qValues[idx],
-      significant: qValues[idx] < alpha,
-      meanDelta: r.b.mean - r.a.mean
-    }));
-    const sorted = scores2.slice().sort((x, y) => y.mean - x.mean);
-    const winner = sorted[0];
-    const second = sorted[1];
-    const winnerComparisons = pairwise2.filter(
-      (c) => c.variantA === winner.variantId || c.variantB === winner.variantId
-    );
-    const significantOverAll = winnerComparisons.every((c) => c.significant);
-    const ciLowerBoundExceedsSecondMean = winner.ci95.lower > second.mean;
-    return {
-      winner: {
-        variantId: winner.variantId,
-        significant: significantOverAll,
-        ciLowerBoundExceedsSecondMean
-      },
-      scores: scores2,
-      pairwise: pairwise2,
-      config: {
-        trialsPerScenario: trials,
-        significanceLevel: alpha,
-        variants: config.variants.map((v) => v.id),
-        scenarios: config.scenarioIds
-      }
-    };
-  }
-};
-function flatSamples(score) {
-  const out = [];
-  for (const s of Object.values(score.perScenario)) out.push(...s.samples);
-  return out;
-}
-
 // src/steering.ts
 function mergeSteeringBundle(base, delta) {
   return {
@@ -3831,46 +3663,6 @@ function canonicalInstruction(value) {
   return normalized.length === 0 ? normalized : normalized[0].toUpperCase() + normalized.slice(1);
 }
 
-// src/optimization-loop.ts
-var OptimizationLoop = class {
-  optimizer;
-  constructor(optimizer = new PromptOptimizer()) {
-    this.optimizer = optimizer;
-  }
-  async run(config) {
-    const byId = new Map(config.variants.map((variant) => [variant.id, variant]));
-    const result = await this.optimizer.run({
-      variants: config.variants.map((variant) => ({
-        id: variant.id,
-        prompt: renderSteeringText(variant),
-        metadata: { bundle: variant }
-      })),
-      scenarioIds: config.examples.map((example) => example.scenarioId),
-      trialsPerScenario: config.trialsPerScenario,
-      scoreVariant: async ({ variant, scenarioId, trialIndex }) => {
-        const bundle = byId.get(variant.id);
-        if (!bundle) throw new Error(`unknown steering bundle ${variant.id}`);
-        const example = config.examples.find((item) => item.scenarioId === scenarioId);
-        if (!example) throw new Error(`unknown optimization example ${scenarioId}`);
-        const score = await config.evaluate({ variant: bundle, example, trialIndex });
-        return aggregateRunScore(score, config.scoreWeights);
-      }
-    });
-    return {
-      winner: byId.get(result.winner.variantId),
-      significant: result.winner.significant,
-      reports: result.scores.map((score) => ({
-        variantId: score.variantId,
-        bundle: byId.get(score.variantId),
-        mean: score.mean,
-        ci95: score.ci95,
-        scenarioScores: score.perScenario
-      })),
-      pairwise: result.pairwise
-    };
-  }
-};
-
 // src/steering-optimizer.ts
 var PairwiseSteeringOptimizer = class {
   optimize(rows, config = {}) {
@@ -7119,6 +6911,67 @@ function excerpt3(source, needle) {
   return (start > 0 ? "\u2026" : "") + source.slice(start, end) + (end < source.length ? "\u2026" : "");
 }
 
+// src/power-analysis.ts
+function requiredSampleSize(opts) {
+  const effect = opts.effect;
+  if (!Number.isFinite(effect) || effect <= 0) return Infinity;
+  const alpha = opts.alpha ?? 0.05;
+  const power = opts.power ?? 0.8;
+  const twoSided = opts.twoSided ?? true;
+  const zAlpha = zQuantile(twoSided ? 1 - alpha / 2 : 1 - alpha);
+  const zBeta = zQuantile(power);
+  const n = 2 * Math.pow((zAlpha + zBeta) / effect, 2);
+  return Math.ceil(n);
+}
+function bonferroni(pValues, alpha = 0.05) {
+  const k = pValues.length;
+  const adjusted = pValues.map((p) => Math.min(1, p * k));
+  const significant = adjusted.map((p) => p < alpha);
+  return { adjusted, significant };
+}
+function benjaminiHochberg(pValues, fdr = 0.05) {
+  const n = pValues.length;
+  if (n === 0) return { qValues: [], significant: [] };
+  const indexed = pValues.map((p, i) => ({ p, i })).sort((a, b) => a.p - b.p);
+  const q = new Array(n);
+  let minRight = 1;
+  for (let k = n - 1; k >= 0; k--) {
+    const rank = k + 1;
+    const raw = indexed[k].p * n / rank;
+    const bounded = Math.min(minRight, raw);
+    minRight = bounded;
+    q[indexed[k].i] = Math.min(1, bounded);
+  }
+  const significant = q.map((v) => v < fdr);
+  return { qValues: q, significant };
+}
+function zQuantile(p) {
+  if (p <= 0 || p >= 1) {
+    if (p === 0) return -Infinity;
+    if (p === 1) return Infinity;
+    return NaN;
+  }
+  const a = [-39.69683028665376, 220.9460984245205, -275.9285104469687, 138.357751867269, -30.66479806614716, 2.506628277459239];
+  const b = [-54.47609879822406, 161.5858368580409, -155.6989798598866, 66.80131188771972, -13.28068155288572];
+  const c = [-0.007784894002430293, -0.3223964580411365, -2.400758277161838, -2.549732539343734, 4.374664141464968, 2.938163982698783];
+  const d = [0.007784695709041462, 0.3224671290700398, 2.445134137142996, 3.754408661907416];
+  const pLow = 0.02425;
+  const pHigh = 1 - pLow;
+  let q;
+  let r;
+  if (p < pLow) {
+    q = Math.sqrt(-2 * Math.log(p));
+    return (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
+  }
+  if (p <= pHigh) {
+    q = p - 0.5;
+    r = q * q;
+    return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q / (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1);
+  }
+  q = Math.sqrt(-2 * Math.log(1 - p));
+  return -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
+}
+
 // src/behavior-dsl.ts
 var BehaviorAssertion = class {
   constructor(store, runId) {
@@ -12692,6 +12545,274 @@ function samePopulation(a, b) {
   return b.every((id) => setA.has(id));
 }
 
+// src/multi-shot-optimization.ts
+async function runMultiShotOptimization(config) {
+  validateConfig(config);
+  const scoreAdapter = {
+    score: (args) => scoreOne(config, args.variant, args.scenarioId, args.rep, "search")
+  };
+  const evolution = await runPromptEvolution({
+    runId: config.runId,
+    target: config.target,
+    seedVariants: config.seedVariants,
+    scenarioIds: config.searchScenarioIds,
+    reps: config.reps,
+    generations: config.generations,
+    populationSize: config.populationSize,
+    scoreConcurrency: config.scoreConcurrency ?? 1,
+    scoreAdapter,
+    mutateAdapter: {
+      mutate: (args) => config.mutateAdapter.mutate({
+        ...args,
+        topTrials: args.topTrials,
+        bottomTrials: args.bottomTrials
+      })
+    },
+    objectives: config.objectives ?? defaultMultiShotObjectives(),
+    scalarWeights: config.scalarWeights,
+    earlyStopOnNoImprovement: config.earlyStopOnNoImprovement,
+    cache: config.cache,
+    onProgress: config.onProgress
+  });
+  let gate = null;
+  const baseline = config.seedVariants[0];
+  let promotedVariant = evolution.bestVariant;
+  let promotedAggregate = evolution.bestAggregate;
+  if (config.gate && evolution.bestVariant.id !== baseline.id) {
+    gate = await evaluateMultiShotGate(config, baseline, evolution.bestVariant);
+    if (!gate.decision.promote) {
+      promotedVariant = baseline;
+      promotedAggregate = aggregateFor(evolution, baseline.id);
+    }
+  }
+  return {
+    evolution,
+    searchBestVariant: evolution.bestVariant,
+    searchBestAggregate: evolution.bestAggregate,
+    promotedVariant,
+    promotedAggregate,
+    gate
+  };
+}
+function defaultMultiShotObjectives() {
+  return [
+    { name: "score", direction: "maximize", value: (a) => a.meanScore },
+    { name: "cost", direction: "minimize", value: (a) => a.meanCost }
+  ];
+}
+function trialTraceFromMultiShotTrial(trial) {
+  return {
+    id: `${trial.variantId}/${trial.scenarioId}/r${trial.rep}`,
+    score: trial.score,
+    inputName: trial.scenarioId,
+    expectations: (trial.asi ?? []).map((item, i) => ({
+      id: item.expectationId ?? `asi-${i}`,
+      phrase: item.message,
+      matched: item.matched ?? false
+    })),
+    emitted: trial.emitted ?? traceExcerpt(trial.trace),
+    metrics: trial.metrics
+  };
+}
+async function evaluateMultiShotGate(config, baseline, candidate) {
+  const gateConfig = config.gate;
+  const reps = gateConfig.reps ?? config.reps;
+  const candidateRuns = [];
+  const baselineRuns = [];
+  const searchIds = gateConfig.searchScenarioIds ?? config.searchScenarioIds;
+  for (const scenarioId of searchIds) {
+    for (let rep = 0; rep < reps; rep++) {
+      const seed = seedFor(config, scenarioId, rep);
+      const baseTrial = await scoreOne(config, baseline, scenarioId, rep, "search");
+      const candTrial = await scoreOne(config, candidate, scenarioId, rep, "search");
+      baselineRuns.push(toValidatedRecord(config, baseline, scenarioId, rep, "search", seed, baseTrial));
+      candidateRuns.push(toValidatedRecord(config, candidate, scenarioId, rep, "search", seed, candTrial));
+    }
+  }
+  for (const scenarioId of gateConfig.holdoutScenarioIds) {
+    for (let rep = 0; rep < reps; rep++) {
+      const seed = seedFor(config, scenarioId, rep);
+      const baseTrial = await scoreOne(config, baseline, scenarioId, rep, "holdout");
+      const candTrial = await scoreOne(config, candidate, scenarioId, rep, "holdout");
+      baselineRuns.push(toValidatedRecord(config, baseline, scenarioId, rep, "holdout", seed, baseTrial));
+      candidateRuns.push(toValidatedRecord(config, candidate, scenarioId, rep, "holdout", seed, candTrial));
+    }
+  }
+  const decision = new HeldOutGate(gateConfig.gate).evaluate(candidateRuns, baselineRuns);
+  return { decision, candidateRuns, baselineRuns };
+}
+async function scoreOne(config, variant, scenarioId, rep, split) {
+  const seed = seedFor(config, scenarioId, rep);
+  const input = { variant, scenarioId, rep, split, seed };
+  try {
+    const run = await config.runner.run(input);
+    const scored = await config.scorer.score({ ...input, run });
+    const asi = scored.asi ?? [];
+    return {
+      variantId: variant.id,
+      scenarioId,
+      rep,
+      ok: scored.ok ?? true,
+      score: clamp013(scored.score),
+      cost: scored.costUsd ?? run.costUsd ?? 0,
+      durationMs: scored.durationMs ?? run.durationMs ?? 0,
+      metrics: {
+        ...numericMetrics(scored.metrics),
+        ...asiMetrics(asi)
+      },
+      split,
+      seed,
+      trace: run.trace,
+      asi,
+      emitted: scored.emitted ?? traceExcerpt(run.trace),
+      metadata: scored.metadata
+    };
+  } catch (err) {
+    return {
+      variantId: variant.id,
+      scenarioId,
+      rep,
+      ok: false,
+      score: 0,
+      cost: 0,
+      durationMs: 0,
+      metrics: { error: 1 },
+      error: err instanceof Error ? err.message : String(err),
+      split,
+      seed,
+      asi: [{
+        severity: "critical",
+        message: err instanceof Error ? err.message : String(err),
+        responsibleSurface: config.target
+      }],
+      emitted: ""
+    };
+  }
+}
+function toValidatedRecord(config, variant, scenarioId, rep, split, seed, trial) {
+  const record = config.gate.toRunRecord({ variant, scenarioId, rep, split, seed, trial });
+  return validateRunRecord(record);
+}
+function validateConfig(config) {
+  if (!config.runId.trim()) throw new Error("runMultiShotOptimization: runId must not be empty");
+  if (!config.target.trim()) throw new Error("runMultiShotOptimization: target must not be empty");
+  if (config.seedVariants.length === 0) {
+    throw new Error("runMultiShotOptimization: seedVariants must not be empty");
+  }
+  if (config.searchScenarioIds.length === 0) {
+    throw new Error("runMultiShotOptimization: searchScenarioIds must not be empty");
+  }
+  requirePositiveInteger(config.reps, "reps");
+  requirePositiveInteger(config.generations, "generations");
+  requirePositiveInteger(config.populationSize, "populationSize");
+  if (config.scoreConcurrency !== void 0) requirePositiveInteger(config.scoreConcurrency, "scoreConcurrency");
+  if (config.populationSize < config.seedVariants.length) {
+    throw new Error("runMultiShotOptimization: populationSize must be >= seedVariants.length");
+  }
+  assertUnique(config.seedVariants.map((v) => v.id), "seedVariants.id");
+  assertUnique(config.searchScenarioIds, "searchScenarioIds");
+  if (config.gate) {
+    if (config.gate.holdoutScenarioIds.length === 0) {
+      throw new Error("runMultiShotOptimization: gate.holdoutScenarioIds must not be empty");
+    }
+    if (config.gate.reps !== void 0) requirePositiveInteger(config.gate.reps, "gate.reps");
+    assertUnique(config.gate.holdoutScenarioIds, "gate.holdoutScenarioIds");
+    if (config.gate.searchScenarioIds) assertUnique(config.gate.searchScenarioIds, "gate.searchScenarioIds");
+    const searchIds = new Set(config.searchScenarioIds);
+    for (const id of config.gate.holdoutScenarioIds) {
+      if (searchIds.has(id)) {
+        throw new Error(`runMultiShotOptimization: holdout scenario "${id}" also appears in searchScenarioIds`);
+      }
+    }
+    const baselineId = config.seedVariants[0].id;
+    if (config.gate.gate.baselineKey !== baselineId) {
+      throw new Error(
+        `runMultiShotOptimization: gate.gate.baselineKey must match first seed variant id "${baselineId}"`
+      );
+    }
+  }
+}
+function requirePositiveInteger(value, name) {
+  if (!Number.isInteger(value) || value <= 0) {
+    throw new Error(`runMultiShotOptimization: ${name} must be a positive integer`);
+  }
+}
+function assertUnique(values, name) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const value of values) {
+    if (!value.trim()) throw new Error(`runMultiShotOptimization: ${name} must not contain empty values`);
+    if (seen.has(value)) throw new Error(`runMultiShotOptimization: duplicate ${name} "${value}"`);
+    seen.add(value);
+  }
+}
+function aggregateFor(evolution, variantId) {
+  const final = evolution.generations[evolution.generations.length - 1];
+  const aggregate2 = final?.aggregates.find((a) => a.variantId === variantId);
+  if (!aggregate2) {
+    throw new Error(`runMultiShotOptimization: missing aggregate for variant "${variantId}"`);
+  }
+  return aggregate2;
+}
+function seedFor(config, scenarioId, rep) {
+  const base = config.seedBase ?? 0;
+  return (base + stableHash2(`${scenarioId}${rep}`)) % Number.MAX_SAFE_INTEGER;
+}
+function stableHash2(input) {
+  let h = 2166136261;
+  for (let i = 0; i < input.length; i++) {
+    h ^= input.charCodeAt(i);
+    h = Math.imul(h, 16777619);
+  }
+  return h >>> 0;
+}
+function clamp013(n) {
+  if (!Number.isFinite(n)) return 0;
+  return Math.max(0, Math.min(1, n));
+}
+function numericMetrics(metrics) {
+  const out = {};
+  for (const [k, v] of Object.entries(metrics ?? {})) {
+    if (Number.isFinite(v)) out[k] = v;
+  }
+  return out;
+}
+function asiMetrics(asi) {
+  const out = { asi: asi.length };
+  for (const item of asi.slice(0, 1e3)) {
+    const sev = normalizeSeverity(item.severity);
+    out[`asi.${sev}`] = (out[`asi.${sev}`] ?? 0) + 1;
+    if (item.responsibleSurface) {
+      const key = `surface.${metricKeySegment(item.responsibleSurface)}`;
+      out[key] = (out[key] ?? 0) + 1;
+    }
+  }
+  return out;
+}
+function normalizeSeverity(severity) {
+  if (severity === "info" || severity === "warning" || severity === "error" || severity === "critical") {
+    return severity;
+  }
+  return "error";
+}
+function metricKeySegment(raw) {
+  return raw.trim().replace(/[^a-zA-Z0-9._-]+/g, "_").slice(0, 80) || "unknown";
+}
+function traceExcerpt(trace) {
+  if (!trace) return void 0;
+  if (typeof trace.output === "string") return trace.output;
+  if (trace.transcript) return trace.transcript;
+  if (trace.turns) {
+    try {
+      const clipped = trace.turns.slice(0, 20);
+      const suffix = trace.turns.length > clipped.length ? ` ... ${trace.turns.length - clipped.length} more turn(s)` : "";
+      return `${JSON.stringify(clipped).slice(0, 2e3)}${suffix}`;
+    } catch {
+      return "[unserializable trace turns]";
+    }
+  }
+  return void 0;
+}
+
 // src/jsonl-trial-cache.ts
 import { appendFileSync as appendFileSync4, existsSync as existsSync6, mkdirSync as mkdirSync4, readFileSync as readFileSync5 } from "fs";
 import { dirname as dirname4 } from "path";
@@ -13727,12 +13848,10 @@ export {
   Mutex,
   NoopResearcher,
   OTEL_AGENT_EVAL_SCOPE,
-  OptimizationLoop,
   PairwiseSteeringOptimizer,
   PrmGrader,
   ProductClient,
   ProjectRegistry,
-  PromptOptimizer,
   PromptRegistry,
   REDACTION_VERSION,
   RunCritic,
@@ -13811,6 +13930,7 @@ export {
   decideReferenceReplayPromotion,
   decideReferenceReplayRunPromotion,
   defaultJudges,
+  defaultMultiShotObjectives,
   defaultReferenceReplayMatcher,
   deployGateLayer,
   distillPlaybook,
@@ -13948,6 +14068,7 @@ export {
   runJudgeFleet,
   runKeywordCoverageJudge,
   runKeywordCoverageJudgeUrl,
+  runMultiShotOptimization,
   runPromptEvolution,
   runProposeReview,
   runProposeReviewAsControlLoop,
@@ -13991,6 +14112,7 @@ export {
   toolSpans,
   toolSuccessRubric,
   toolWasteView,
+  trialTraceFromMultiShotTrial,
   typoMutator,
   urlContains,
   validateRunRecord,