@tangle-network/agent-eval 0.7.0 → 0.7.2

package/dist/index.js

@@ -410,7 +410,7 @@ function confidenceInterval(scores, confidence = 0.95) {
   if (scores.length === 0) return { mean: 0, lower: 0, upper: 0 };
   if (scores.length === 1) return { mean: scores[0], lower: scores[0], upper: scores[0] };
   const n = scores.length;
-  const mean3 = scores.reduce((a, b) => a + b, 0) / n;
+  const mean4 = scores.reduce((a, b) => a + b, 0) / n;
   const B = 1e3;
   const bootstrapMeans = [];
   for (let i = 0; i < B; i++) {
@@ -425,7 +425,7 @@ function confidenceInterval(scores, confidence = 0.95) {
   const lowerIdx = Math.floor(alpha / 2 * B);
   const upperIdx = Math.floor((1 - alpha / 2) * B) - 1;
   return {
-    mean: mean3,
+    mean: mean4,
     lower: bootstrapMeans[lowerIdx],
     upper: bootstrapMeans[Math.min(upperIdx, B - 1)]
   };
@@ -513,11 +513,11 @@ function pairedTTest(before, after) {
   const n = before.length;
   if (n < 2) return { t: 0, df: 0, p: 1 };
   const diffs = before.map((b, i) => after[i] - b);
-  const mean3 = diffs.reduce((a, b) => a + b, 0) / n;
-  const variance2 = diffs.reduce((acc, d) => acc + (d - mean3) ** 2, 0) / (n - 1);
+  const mean4 = diffs.reduce((a, b) => a + b, 0) / n;
+  const variance2 = diffs.reduce((acc, d) => acc + (d - mean4) ** 2, 0) / (n - 1);
   const se = Math.sqrt(variance2 / n);
-  if (se === 0) return { t: mean3 === 0 ? 0 : Infinity, df: n - 1, p: mean3 === 0 ? 1 : 0 };
-  const t = mean3 / se;
+  if (se === 0) return { t: mean4 === 0 ? 0 : Infinity, df: n - 1, p: mean4 === 0 ? 1 : 0 };
+  const t = mean4 / se;
   const df = n - 1;
   const p = 2 * (1 - studentTCdf(Math.abs(t), df));
   return { t, df, p };
@@ -541,9 +541,9 @@ function wilcoxonSignedRank(before, after) {
   }
   let wPlus = 0;
   for (let k = 0; k < n; k++) if (diffs[k] > 0) wPlus += ranks3[k];
-  const mean3 = n * (n + 1) / 4;
+  const mean4 = n * (n + 1) / 4;
   const variance2 = n * (n + 1) * (2 * n + 1) / 24;
-  const z = (wPlus - mean3) / Math.sqrt(variance2);
+  const z = (wPlus - mean4) / Math.sqrt(variance2);
   const p = 2 * (1 - normalCdf(Math.abs(z)));
   return { w: wPlus, p };
 }
@@ -2135,12 +2135,14 @@ var DEFAULT_RUN_SCORE_WEIGHTS = {
   toolUseQuality: 1,
   patchQuality: 1.25,
   testReality: 1.5,
+  finalGate: 3,
+  reviewerBlockers: -2,
   costUsd: -0.2,
   wallSeconds: -0.1
 };
 function aggregateRunScore(score, weights = {}) {
   const w = { ...DEFAULT_RUN_SCORE_WEIGHTS, ...weights };
-  return w.success * clamp01(score.success) + w.goalProgress * clamp01(score.goalProgress) + w.repoGroundedness * clamp01(score.repoGroundedness) + w.driftPenalty * clamp01(score.driftPenalty) + w.toolUseQuality * clamp01(score.toolUseQuality) + w.patchQuality * clamp01(score.patchQuality) + w.testReality * clamp01(score.testReality) + w.costUsd * Math.max(0, score.costUsd) + w.wallSeconds * Math.max(0, score.wallSeconds / 60);
+  return w.success * clamp01(score.success) + w.goalProgress * clamp01(score.goalProgress) + w.repoGroundedness * clamp01(score.repoGroundedness) + w.driftPenalty * clamp01(score.driftPenalty) + w.toolUseQuality * clamp01(score.toolUseQuality) + w.patchQuality * clamp01(score.patchQuality) + w.testReality * clamp01(score.testReality) + w.finalGate * clamp01(score.finalGate) + w.reviewerBlockers * clamp01(score.reviewerBlockers) + w.costUsd * Math.max(0, score.costUsd) + w.wallSeconds * Math.max(0, score.wallSeconds / 60);
 }
 function clamp01(value) {
   if (!Number.isFinite(value)) return 0;
@@ -2180,6 +2182,9 @@ var RunCritic = class {
     const toolSpans2 = trace.spans.filter((s) => s.kind === "tool");
     const judgeSpans2 = trace.spans.filter((s) => s.kind === "judge");
     const sandboxSpans = trace.spans.filter((s) => s.kind === "sandbox");
+    const finalGateSpans = judgeSpans2.filter(
+      (span) => span.dimension === "final_gate" || span.attributes?.finalGate === true
+    );
     const success = trace.run.outcome?.pass === true ? 1 : trace.run.status === "completed" ? 0.5 : 0;
     if (!success) notes.push("run did not complete with pass=true");
     const judgeAverage = judgeSpans2.length ? judgeSpans2.reduce((sum, span) => sum + normalizeJudgeScore(span.score), 0) / judgeSpans2.length : void 0;
@@ -2194,6 +2199,15 @@ var RunCritic = class {
     const sandboxTests = sandboxSpans.filter((span) => typeof span.testsTotal === "number" && span.testsTotal > 0);
     const testReality = sandboxTests.length ? sandboxTests.reduce((sum, span) => sum + (span.testsPassed ?? 0) / Math.max(1, span.testsTotal ?? 1), 0) / sandboxTests.length : toolSpans2.some((span) => /\btest|vitest|pytest|jest|build|tsc\b/i.test(JSON.stringify(span.args))) ? 0.4 : 0;
     if (!testReality) notes.push("no real test/build evidence recorded");
+    const blockerSpans = judgeSpans2.filter(
+      (span) => isBlockingJudge(span)
+    );
+    const finalGateBlockers = finalGateSpans.filter((span) => isBlockingJudge(span));
+    const finalGate = finalGateSpans.length ? finalGateBlockers.length ? 0 : 1 : success;
+    if (finalGateBlockers.length) notes.push(`final gate blocked by ${finalGateBlockers.length} reviewer(s)`);
+    else if (!finalGateSpans.length) notes.push("no final gate judgment recorded");
+    const reviewerBlockers = judgeSpans2.length ? blockerSpans.length / judgeSpans2.length : 0;
+    if (reviewerBlockers) notes.push(`detected ${blockerSpans.length} blocking reviewer signal(s)`);
     const positiveGroundingSignals = patchEvidence + sandboxSpans.length + llmSpans2.filter((span) => looksRepoGrounded(span.output ?? "")).length;
     const driftSignals = llmSpans2.filter((span) => this.isDrift(span.output ?? "")).length + trace.events.filter((event) => this.isDrift(JSON.stringify(event.payload))).length;
     const repoGroundedness = positiveGroundingSignals + driftSignals === 0 ? 0 : positiveGroundingSignals / (positiveGroundingSignals + driftSignals);
@@ -2209,6 +2223,8 @@ var RunCritic = class {
       toolUseQuality,
       patchQuality,
       testReality,
+      finalGate,
+      reviewerBlockers,
       costUsd,
       wallSeconds,
       notes
@@ -2227,6 +2243,12 @@ function normalizeJudgeScore(score) {
 function looksRepoGrounded(text) {
   return /(?:src\/|tests?\/|package\.json|tsconfig|\.ts\b|\.tsx\b|git status|pnpm |npm |vitest|pytest|jest)/i.test(text);
 }
+function isBlockingJudge(span) {
+  return span.attributes?.blocking === true || span.attributes?.verdict === "BLOCKING" || positiveNumber(span.attributes?.blockingFindings) || positiveNumber(span.attributes?.highFindings) || span.score <= 2;
+}
+function positiveNumber(value) {
+  return typeof value === "number" && value > 0;
+}
 
 // src/playbook.ts
 function distillPlaybook(entries, options = {}) {
@@ -2430,6 +2452,144 @@ function createAxService(aiFactory, provider, apiKey, model) {
   });
 }
 
+// src/pareto.ts
+function dominates(a, b, objectives) {
+  let strictlyBetter = false;
+  for (const obj of objectives) {
+    const av = obj.value(a);
+    const bv = obj.value(b);
+    if (!Number.isFinite(av) || !Number.isFinite(bv)) return false;
+    const aIsBetter = obj.direction === "maximize" ? av > bv : av < bv;
+    const aIsWorse = obj.direction === "maximize" ? av < bv : av > bv;
+    if (aIsWorse) return false;
+    if (aIsBetter) strictlyBetter = true;
+  }
+  return strictlyBetter;
+}
+function paretoFrontier(candidates, objectives) {
+  if (objectives.length === 0) {
+    throw new Error("paretoFrontier: at least 1 objective required");
+  }
+  const valid = candidates.filter(
+    (c) => objectives.every((o) => Number.isFinite(o.value(c)))
+  );
+  const frontier = [];
+  const dominated = [];
+  for (const c of valid) {
+    const isDominated = valid.some((other) => other !== c && dominates(other, c, objectives));
+    if (isDominated) dominated.push(c);
+    else frontier.push(c);
+  }
+  const dominanceMap = frontier.map((d) => ({
+    dominator: d,
+    dominated: dominated.filter((x) => dominates(d, x, objectives))
+  }));
+  return { frontier, dominated, dominanceMap };
+}
+
+// src/harness-optimizer.ts
+var DEFAULT_HARNESS_OBJECTIVES = [
+  { name: "aggregate", direction: "maximize", value: (r) => r.aggregateMean },
+  { name: "pass_rate", direction: "maximize", value: (r) => r.passRate },
+  { name: "cost", direction: "minimize", value: (r) => r.costUsdMean },
+  { name: "wall", direction: "minimize", value: (r) => r.wallSecondsMean }
+];
+async function runHarnessExperiment(config) {
+  const jobs = buildJobs(config);
+  const critic = new RunCritic({ weights: config.weights });
+  const score = config.score ?? ((trace) => critic.scoreTrace(trace));
+  const results = await mapLimit(jobs, config.parallelism ?? 1, async (request) => {
+    const trace = await config.adapter.run(request);
+    const runScore = await score(trace, request);
+    const result = {
+      variant: request.variant,
+      scenario: request.scenario,
+      trialIndex: request.trialIndex,
+      trace,
+      score: runScore,
+      aggregate: aggregateRunScore(runScore, config.weights)
+    };
+    await config.onResult?.(result);
+    return result;
+  });
+  return { results, selection: selectHarnessVariant(results, config.objectives) };
+}
+function selectHarnessVariant(results, objectives = DEFAULT_HARNESS_OBJECTIVES) {
+  const reports = summarizeHarnessResults(results);
+  if (reports.length === 0) throw new Error("selectHarnessVariant: no results");
+  const frontier = paretoFrontier(reports, objectives);
+  const candidates = frontier.frontier.length ? frontier.frontier : reports;
+  const winner = [...candidates].sort((a, b) => b.aggregateMean - a.aggregateMean)[0];
+  if (!winner) throw new Error("selectHarnessVariant: no winner");
+  return { winner, frontier, reports };
+}
+function summarizeHarnessResults(results) {
+  const byVariant = /* @__PURE__ */ new Map();
+  for (const result of results) {
+    byVariant.set(result.variant.id, [...byVariant.get(result.variant.id) ?? [], result]);
+  }
+  return [...byVariant.values()].map((runs) => {
+    const variant = runs[0]?.variant;
+    if (!variant) throw new Error("summarizeHarnessResults: empty variant bucket");
+    return {
+      variant,
+      runs,
+      aggregateMean: mean(runs.map((r) => r.aggregate)),
+      passRate: mean(runs.map((r) => r.score.success)),
+      costUsdMean: mean(runs.map((r) => r.score.costUsd)),
+      wallSecondsMean: mean(runs.map((r) => r.score.wallSeconds)),
+      scoreMean: meanRunScore(runs.map((r) => r.score))
+    };
+  }).sort((a, b) => b.aggregateMean - a.aggregateMean);
+}
+function buildJobs(config) {
+  if (config.variants.length === 0) throw new Error("runHarnessExperiment: at least one variant required");
+  if (config.scenarios.length === 0) throw new Error("runHarnessExperiment: at least one scenario required");
+  const trials = Math.max(1, Math.floor(config.trialsPerScenario ?? 1));
+  const jobs = [];
+  for (const variant of config.variants) {
+    for (const scenario of config.scenarios) {
+      for (let trialIndex = 0; trialIndex < trials; trialIndex++) {
+        jobs.push({ variant, scenario, trialIndex });
+      }
+    }
+  }
+  return jobs;
+}
+async function mapLimit(items, limit, fn) {
+  const results = new Array(items.length);
+  let next = 0;
+  const workerCount = Math.max(1, Math.min(Math.floor(limit), items.length));
+  await Promise.all(Array.from({ length: workerCount }, async () => {
+    while (next < items.length) {
+      const index = next++;
+      const item = items[index];
+      if (item === void 0) continue;
+      results[index] = await fn(item);
+    }
+  }));
+  return results;
+}
+function mean(values) {
+  return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0;
+}
+function meanRunScore(scores) {
+  return {
+    success: mean(scores.map((s) => s.success)),
+    goalProgress: mean(scores.map((s) => s.goalProgress)),
+    repoGroundedness: mean(scores.map((s) => s.repoGroundedness)),
+    driftPenalty: mean(scores.map((s) => s.driftPenalty)),
+    toolUseQuality: mean(scores.map((s) => s.toolUseQuality)),
+    patchQuality: mean(scores.map((s) => s.patchQuality)),
+    testReality: mean(scores.map((s) => s.testReality)),
+    finalGate: mean(scores.map((s) => s.finalGate)),
+    reviewerBlockers: mean(scores.map((s) => s.reviewerBlockers)),
+    costUsd: mean(scores.map((s) => s.costUsd)),
+    wallSeconds: mean(scores.map((s) => s.wallSeconds)),
+    notes: scores.flatMap((s) => s.notes ?? [])
+  };
+}
+
 // src/trace/store.ts
 var InMemoryTraceStore = class {
   runs = /* @__PURE__ */ new Map();
@@ -2875,14 +3035,22 @@ function composeParsers(...parsers) {
 }
 var SubprocessSandboxDriver = class {
   id = "subprocess";
+  defaultCwd;
+  defaultEnv;
+  constructor(options = {}) {
+    this.defaultCwd = options.cwd;
+    this.defaultEnv = options.env;
+  }
   async exec(phase, command, config) {
     const { spawn } = await import("child_process");
     const start = Date.now();
+    const effectiveCwd = config.cwd ?? this.defaultCwd;
+    const effectiveEnv = { ...process.env, ...this.defaultEnv ?? {}, ...config.env ?? {} };
     return await new Promise((resolve) => {
       const child = spawn(command, {
         shell: true,
-        cwd: config.cwd,
-        env: { ...process.env, ...config.env ?? {} }
+        cwd: effectiveCwd,
+        env: effectiveEnv
       });
       let stdout = "";
       let stderr = "";
@@ -4308,8 +4476,8 @@ function compareToBaseline(samples, options = {}) {
     if (s.baseline.length < 2 || s.candidate.length < 2) {
       throw new Error(`compareToBaseline: need \u22652 samples per side for "${s.metric}"`);
     }
-    const bMean = mean(s.baseline);
-    const cMean = mean(s.candidate);
+    const bMean = mean2(s.baseline);
+    const cMean = mean2(s.candidate);
     const delta = cMean - bMean;
     const d = cohensD(s.baseline, s.candidate);
     const { t, df, p } = welchsTTest(s.baseline, s.candidate);
@@ -4348,7 +4516,7 @@ function compareToBaseline(samples, options = {}) {
     hasUnstable: metrics.some((m) => m.verdict === "unstable")
   };
 }
-function mean(xs) {
+function mean2(xs) {
   return xs.reduce((a, b) => a + b, 0) / xs.length;
 }
 function iqr(xs) {
@@ -4364,8 +4532,8 @@ function iqr(xs) {
 }
 function welchsTTest(a, b) {
   if (a.length < 2 || b.length < 2) return { t: 0, df: 0, p: 1 };
-  const mA = mean(a);
-  const mB = mean(b);
+  const mA = mean2(a);
+  const mB = mean2(b);
   const vA = variance(a, mA);
   const vB = variance(b, mB);
   const seSquared = vA / a.length + vB / b.length;
@@ -4685,6 +4853,30 @@ var CostTracker = class {
     if (!bucket) throw new Error(`CostTracker.markOutcome: unknown scenario "${scenarioId}"`);
     bucket.completed = completed;
   }
+  /**
+   * Convenience: record + markOutcome in one call from a
+   * `{ usage, verdict }`-shaped response (starter-foundry's
+   * `invokeMetaJudge` returns this shape; consumers that wrap any
+   * judge/critic can follow the same convention).
+   *
+   * `usage.model` must be present in `MODEL_PRICING` for cost math to
+   * populate; otherwise totalCostUsd stays at 0 for the entry but
+   * tokens still aggregate.
+   */
+  recordVerdict(verdict, scenarioId, tags) {
+    if (!verdict.usage) return null;
+    const entry = this.record({
+      scenarioId,
+      model: verdict.usage.model,
+      inputTokens: verdict.usage.inputTokens,
+      outputTokens: verdict.usage.outputTokens,
+      cachedTokens: verdict.usage.cachedTokens,
+      reasoningTokens: verdict.usage.reasoningTokens,
+      tags
+    });
+    this.markOutcome(scenarioId, verdict.verdict === "pass");
+    return entry;
+  }
   get(scenarioId) {
     return this.byScenario.get(scenarioId);
   }
@@ -4721,39 +4913,177 @@ function assertNonNegative(n, name) {
   }
 }
 
-// src/pareto.ts
-function dominates(a, b, objectives) {
-  let strictlyBetter = false;
-  for (const obj of objectives) {
-    const av = obj.value(a);
-    const bv = obj.value(b);
-    if (!Number.isFinite(av) || !Number.isFinite(bv)) return false;
-    const aIsBetter = obj.direction === "maximize" ? av > bv : av < bv;
-    const aIsWorse = obj.direction === "maximize" ? av < bv : av > bv;
-    if (aIsWorse) return false;
-    if (aIsBetter) strictlyBetter = true;
-  }
-  return strictlyBetter;
+// src/muffled-gate-scanner.ts
+import { readFileSync as readFileSync2, existsSync as existsSync2, readdirSync, statSync } from "fs";
+import { join } from "path";
+function codeOf(line) {
+  return line.replace(/\/\/.*$/, "").replace(/^\s*\*.*$/, "");
 }
-function paretoFrontier(candidates, objectives) {
-  if (objectives.length === 0) {
-    throw new Error("paretoFrontier: at least 1 objective required");
+function isMuffleOk(line) {
+  return line.includes("muffle-ok:");
+}
+var findFallbackToPass = (file, text) => {
+  const out = [];
+  const lines = text.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (isMuffleOk(line)) continue;
+    const code = codeOf(line);
+    if (!code.trim()) continue;
+    if (/\|\| true/.test(code) && /(testCommand|setupCommand|cmd|command)/.test(code)) {
+      out.push({ file, line: i + 1, lineText: line.trim(), pattern: "fallback-to-pass (|| true in command string)" });
+    }
   }
-  const valid = candidates.filter(
-    (c) => objectives.every((o) => Number.isFinite(o.value(c)))
-  );
-  const frontier = [];
-  const dominated = [];
-  for (const c of valid) {
-    const isDominated = valid.some((other) => other !== c && dominates(other, c, objectives));
-    if (isDominated) dominated.push(c);
-    else frontier.push(c);
+  return out;
+};
+var findLiteralTruePass = (file, text) => {
+  const out = [];
+  const lines = text.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (isMuffleOk(line)) continue;
+    const code = codeOf(line);
+    if (!code.trim()) continue;
+    if (/testCommand\s*:\s*['"]true['"]/.test(code)) {
+      out.push({ file, line: i + 1, lineText: line.trim(), pattern: 'literal-true-pass (testCommand: "true")' });
+    }
   }
-  const dominanceMap = frontier.map((d) => ({
-    dominator: d,
-    dominated: dominated.filter((x) => dominates(d, x, objectives))
-  }));
-  return { frontier, dominated, dominanceMap };
+  return out;
+};
+var findConstructorCwdDropped = (file, text) => {
+  const out = [];
+  const lines = text.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (isMuffleOk(line)) continue;
+    const code = codeOf(line);
+    if (!code.trim()) continue;
+    if (/new\s+SubprocessSandboxDriver\s*\(\s*\{[^}]*cwd\s*:/.test(code)) {
+      out.push({
+        file,
+        line: i + 1,
+        lineText: line.trim(),
+        pattern: "construct-vs-call cwd dropped (driver.exec reads config.cwd, not constructor.cwd)"
+      });
+    }
+  }
+  return out;
+};
+var findAutoMatchNoExpectation = (file, text) => {
+  const out = [];
+  const lines = text.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (isMuffleOk(line)) continue;
+    const code = codeOf(line);
+    if (!code.trim()) continue;
+    if (/if\s*\(\s*!expected\s*\)\s*return\s+true/.test(code)) {
+      out.push({
+        file,
+        line: i + 1,
+        lineText: line.trim(),
+        pattern: "auto-match-no-expectation (if (!expected) return true)"
+      });
+    }
+  }
+  return out;
+};
+var findSkipCountsAsPass = (file, text) => {
+  const out = [];
+  const lines = text.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (isMuffleOk(line)) continue;
+    const code = codeOf(line);
+    if (!code.trim()) continue;
+    if (/if\s*\(\s*\w+\.skipped\s*\)\s*return\s+true/.test(code)) {
+      out.push({
+        file,
+        line: i + 1,
+        lineText: line.trim(),
+        pattern: "skip-counts-as-pass (if (.skipped) return true)"
+      });
+    }
+  }
+  return out;
+};
+var DEFAULT_FINDERS = [
+  findFallbackToPass,
+  findLiteralTruePass,
+  findAutoMatchNoExpectation,
+  findSkipCountsAsPass
+];
+var UNIVERSAL_FINDERS = [
+  findConstructorCwdDropped
+];
+function autoDeriveImporters(repoRoot, roots, extensions, importsContain) {
+  const matches2 = [];
+  const walk = (rel) => {
+    const abs = join(repoRoot, rel);
+    if (!existsSync2(abs)) return;
+    for (const entry of readdirSync(abs)) {
+      const sub = join(rel, entry);
+      const subAbs = join(repoRoot, sub);
+      let st;
+      try {
+        st = statSync(subAbs);
+      } catch {
+        continue;
+      }
+      if (st.isDirectory()) {
+        if (entry === "node_modules" || entry === "dist" || entry === "dist-tests" || entry.startsWith(".")) continue;
+        walk(sub);
+      } else if (st.isFile() && extensions.test(entry)) {
+        if (entry.endsWith(".test.ts") || entry.endsWith(".test.mjs") || entry.endsWith(".test.js")) continue;
+        let text;
+        try {
+          text = readFileSync2(subAbs, "utf8");
+        } catch {
+          continue;
+        }
+        if (text.includes(importsContain)) matches2.push(sub);
+      }
+    }
+  };
+  for (const r of roots) walk(r);
+  return matches2;
+}
+function scanForMuffledGates(opts) {
+  const findings = [];
+  const scanned = /* @__PURE__ */ new Set();
+  for (const file of opts.scanFiles) {
+    const abs = join(opts.repoRoot, file);
+    if (!existsSync2(abs)) continue;
+    const text = readFileSync2(abs, "utf8");
+    for (const find of opts.finders) findings.push(...find(file, text));
+    scanned.add(file);
+  }
+  if (opts.autoDerive) {
+    const importers = autoDeriveImporters(
+      opts.repoRoot,
+      opts.autoDerive.roots,
+      opts.autoDerive.extensions,
+      opts.autoDerive.importsContain
+    );
+    for (const file of importers) {
+      if (scanned.has(file)) continue;
+      const abs = join(opts.repoRoot, file);
+      if (!existsSync2(abs)) continue;
+      const text = readFileSync2(abs, "utf8");
+      for (const find of opts.autoDerive.universalFinders) findings.push(...find(file, text));
+    }
+  }
+  return findings;
+}
+function formatFindings(findings) {
+  if (findings.length === 0) return "";
+  return [
+    `Found ${findings.length} muffled-gate pattern(s).`,
+    `Fix each or annotate the line with "// muffle-ok: <reason>".`,
+    "",
+    ...findings.map((f) => `  ${f.file}:${f.line} \u2014 ${f.pattern}
+    ${f.lineText}`)
+  ].join("\n");
 }
 
 // src/series-convergence.ts
@@ -4765,10 +5095,10 @@ function analyzeSeries(values, options = {}) {
     return { state: "insufficient-data", windowMean: 0, windowCv: 0, tailRun: 0, stable: false };
   }
   const tail = values.slice(-window);
-  const mean3 = tail.reduce((a, b) => a + b, 0) / tail.length;
-  const variance2 = tail.reduce((acc, v) => acc + (v - mean3) ** 2, 0) / tail.length;
+  const mean4 = tail.reduce((a, b) => a + b, 0) / tail.length;
+  const variance2 = tail.reduce((acc, v) => acc + (v - mean4) ** 2, 0) / tail.length;
   const stdDev = Math.sqrt(variance2);
-  const refMean = Math.abs(mean3) > 1e-9 ? Math.abs(mean3) : 1;
+  const refMean = Math.abs(mean4) > 1e-9 ? Math.abs(mean4) : 1;
   const cv = stdDev / refMean;
   const stable = tail.length >= window && cv <= stableCv;
   let tailRun = 0;
@@ -4789,7 +5119,7 @@ function analyzeSeries(values, options = {}) {
   } else {
     state = "noisy";
   }
-  return { state, windowMean: mean3, windowCv: cv, tailRun, stable };
+  return { state, windowMean: mean4, windowCv: cv, tailRun, stable };
 }
 
 // src/state-continuity.ts
@@ -5717,12 +6047,12 @@ async function paraphraseRobustness(prompt, mutators, scoreFn, options = {}) {
     variantScores.push({ mutator: id, score, mutated });
     all.push(score);
   }
-  const mean3 = all.reduce((a, b) => a + b, 0) / all.length;
-  const variance2 = all.reduce((a, v) => a + (v - mean3) ** 2, 0) / all.length;
+  const mean4 = all.reduce((a, b) => a + b, 0) / all.length;
+  const variance2 = all.reduce((a, v) => a + (v - mean4) ** 2, 0) / all.length;
   const stdDev = Math.sqrt(variance2);
-  const ref = Math.abs(mean3) > 1e-9 ? Math.abs(mean3) : 1;
+  const ref = Math.abs(mean4) > 1e-9 ? Math.abs(mean4) : 1;
   const robustness = Math.max(0, 1 - stdDev / ref);
-  return { originalScore, variantScores, meanScore: mean3, stdDev, robustness };
+  return { originalScore, variantScores, meanScore: mean4, stdDev, robustness };
 }
 var lowercaseMutator = (p) => p.toLowerCase();
 var sentenceReorderMutator = (p, seed) => {
@@ -6407,8 +6737,8 @@ async function calibrationCurve(traceStore, outcomeStore, evalMetric, outcomeMet
 function toBin(chunk, lower, upper) {
   const xs = chunk.map((c) => c.x);
   const ys = chunk.map((c) => c.y);
-  const evalMean = mean2(xs);
-  const outcomeMean = mean2(ys);
+  const evalMean = mean3(xs);
+  const outcomeMean = mean3(ys);
   return {
     lower: lower ?? Math.min(...xs),
     upper: upper ?? Math.max(...xs),
@@ -6418,7 +6748,7 @@ function toBin(chunk, lower, upper) {
     gap: Math.abs(outcomeMean - evalMean)
   };
 }
-function mean2(xs) {
+function mean3(xs) {
   return xs.reduce((a, b) => a + b, 0) / xs.length;
 }
 function defaultExtract4(metric) {
@@ -6643,8 +6973,8 @@ async function prmBestOfN(store, grader, runIds) {
   if (runIds.length === 0) throw new Error("prmBestOfN: at least 1 candidate required");
   const graded = await Promise.all(runIds.map((id) => grader.grade(store, id)));
   const ranked = [...graded].sort((a, b) => b.aggregateScore - a.aggregateScore);
-  const mean3 = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length;
-  const variance2 = graded.reduce((a, g) => a + (g.aggregateScore - mean3) ** 2, 0) / graded.length;
+  const mean4 = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length;
+  const variance2 = graded.reduce((a, g) => a + (g.aggregateScore - mean4) ** 2, 0) / graded.length;
   return { winner: ranked[0], ranked, stdDev: Math.sqrt(variance2) };
 }
 async function prmEnsembleBestOfN(store, graders, runIds) {
@@ -6666,8 +6996,8 @@ async function prmEnsembleBestOfN(store, graders, runIds) {
   const ranked = [...byRun.values()].sort(
     (a, b) => (bordaScores.get(b.runId) ?? 0) - (bordaScores.get(a.runId) ?? 0)
   );
-  const mean3 = ranked.reduce((a, g) => a + g.aggregateScore, 0) / ranked.length;
-  const variance2 = ranked.reduce((a, g) => a + (g.aggregateScore - mean3) ** 2, 0) / ranked.length;
+  const mean4 = ranked.reduce((a, g) => a + g.aggregateScore, 0) / ranked.length;
+  const variance2 = ranked.reduce((a, g) => a + (g.aggregateScore - mean4) ** 2, 0) / ranked.length;
   return { winner: ranked[0], ranked, stdDev: Math.sqrt(variance2) };
 }
 
@@ -6725,7 +7055,7 @@ async function commitBisect(options) {
 }
 async function promptBisect(options) {
   const split = options.paragraphSplitter ?? ((p) => p.split(/\n\s*\n/));
-  const join = (paragraphs) => paragraphs.join("\n\n");
+  const join2 = (paragraphs) => paragraphs.join("\n\n");
   const goodParas = split(options.good);
   const badParas = split(options.bad);
   if (goodParas.length !== badParas.length) {
@@ -6743,7 +7073,7 @@ async function promptBisect(options) {
   const result = await bisect({
     good: goodMask,
     bad: badMask,
-    runEval: (mask) => options.runEval(join(paragraphsFor(mask))),
+    runEval: (mask) => options.runEval(join2(paragraphsFor(mask))),
     maxIterations: options.maxIterations ?? n + 5,
     halfway: (g, b) => {
       for (let i = 0; i < g.length; i++) {
@@ -6774,12 +7104,12 @@ async function promptBisect(options) {
     }
   }
   const materializedPath = result.path.map((s) => ({
-    state: join(paragraphsFor(s.state)),
+    state: join2(paragraphsFor(s.state)),
     score: s.score,
     pass: s.pass
   }));
   return {
-    culprit: join(paragraphsFor(culprit)),
+    culprit: join2(paragraphsFor(culprit)),
     path: materializedPath,
     converged: result.converged,
     inputInconsistent: result.inputInconsistent,
@@ -7197,8 +7527,8 @@ async function proposeSynthesisTargets(dataset, traceStore, options = {}) {
     const sRuns = runs.filter((r) => r.scenarioId === s.id);
     const scores = sRuns.map((r) => r.outcome?.score).filter((x) => typeof x === "number");
     if (scores.length < 3) continue;
-    const mean3 = scores.reduce((a, b) => a + b, 0) / scores.length;
-    const variance2 = scores.reduce((a, b) => a + (b - mean3) ** 2, 0) / scores.length;
+    const mean4 = scores.reduce((a, b) => a + b, 0) / scores.length;
+    const variance2 = scores.reduce((a, b) => a + (b - mean4) ** 2, 0) / scores.length;
     if (variance2 > varianceThreshold) {
       targets.push({
         reason: "high-variance",
@@ -7688,6 +8018,8 @@ export {
   CostTracker,
   DEFAULT_AGENT_SLOS,
   DEFAULT_RULES as DEFAULT_FAILURE_RULES,
+  DEFAULT_FINDERS,
+  DEFAULT_HARNESS_OBJECTIVES,
   DEFAULT_MUTATORS,
   DEFAULT_REDACTION_RULES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -7724,6 +8056,7 @@ export {
   TRACE_SCHEMA_VERSION,
   TokenCounter,
   TraceEmitter,
+  UNIVERSAL_FINDERS,
   adversarialJudge,
   aggregateLlm,
   aggregateRunScore,
@@ -7782,9 +8115,15 @@ export {
   failureClusterView,
   fileContains,
   fileExists,
+  findAutoMatchNoExpectation,
+  findConstructorCwdDropped,
+  findFallbackToPass,
+  findLiteralTruePass,
+  findSkipCountsAsPass,
   firstDivergenceView,
   formatBenchmarkReport,
   formatDriverReport,
+  formatFindings,
   groupBy,
   hashContent,
   hashScenarios,
@@ -7851,16 +8190,19 @@ export {
   runE2EWorkflow,
   runExpectations,
   runFailureClass,
+  runHarnessExperiment,
   runJudgeFleet,
   runProposeReview,
   runSelfPlay,
   runTestGradedScenario,
   runsForScenario,
+  scanForMuffledGates,
   scoreAllProjects,
   scoreContinuity,
   scoreProject,
   scoreRedTeamOutput,
   securityJudge,
+  selectHarnessVariant,
   selfPreference,
   sentenceReorderMutator,
   signManifest,
@@ -7868,6 +8210,7 @@ export {
   statusAdvanced,
   stuckLoopView,
   summarize,
+  summarizeHarnessResults,
   testJudge,
   textInSnapshot,
   toLangfuseEnvelope,