@tangle-network/agent-eval 0.129.0 → 0.130.0

package/dist/chunk-M4YBQKIJ.js

@@ -1,1040 +0,0 @@
-import {
-  paretoChart
-} from "./chunk-TJVT4QFF.js";
-import {
-  pairRunRecords
-} from "./chunk-7FO3TNPI.js";
-import {
-  continuousAgreement,
-  pairedBootstrap,
-  pairedCohensDz,
-  pairedMde,
-  pairedTTest,
-  pearsonR,
-  requiredPairedSampleSize,
-  spearmanR
-} from "./chunk-ZHTZ4EYI.js";
-import {
-  llmSpans
-} from "./chunk-ZET2UAYW.js";
-import {
-  validateRunRecord
-} from "./chunk-56TAVBOK.js";
-import {
-  observedSplitScore
-} from "./chunk-OIUOT4QD.js";
-
-// src/contamination-guard.ts
-function checkCanaries(output, scenarios) {
-  const leaks = [];
-  for (const s of scenarios) {
-    if (!s.canary) continue;
-    if (output.includes(s.canary)) {
-      leaks.push({ scenarioId: s.id, canary: s.canary, evidence: excerpt(output, s.canary) });
-    }
-  }
-  return leaks;
-}
-function checkBehavioralCanary(output, scenario) {
-  const pattern = scenario.forbiddenPattern ?? scenario.canary;
-  if (!pattern) return null;
-  const hit = matchForbidden(output, pattern);
-  if (!hit) return null;
-  return {
-    scenarioId: scenario.id,
-    canary: pattern,
-    evidence: excerpt(output, hit)
-  };
-}
-function runBehavioralCanaries(cases) {
-  const leaks = [];
-  for (const c of cases) {
-    const leak = checkBehavioralCanary(c.output, c.scenario);
-    if (leak) leaks.push({ ...leak, runId: c.runId ?? leak.runId });
-  }
-  return leaks;
-}
-function matchForbidden(output, pattern) {
-  const re = tryParseRegex(pattern);
-  if (re) {
-    const m = output.match(re);
-    return m && m[0].length > 0 ? m[0] : null;
-  }
-  return output.includes(pattern) ? pattern : null;
-}
-function tryParseRegex(pattern) {
-  if (pattern.length < 2 || pattern[0] !== "/") return null;
-  const last = pattern.lastIndexOf("/");
-  if (last <= 0) return null;
-  const body = pattern.slice(1, last);
-  const flags = pattern.slice(last + 1);
-  if (!/^[gimsuy]*$/.test(flags)) return null;
-  try {
-    return new RegExp(body, flags);
-  } catch {
-    return null;
-  }
-}
-async function canaryLeakView(store, scenarios) {
-  const targets = scenarios.filter((s) => !!s.canary);
-  if (targets.length === 0) return [];
-  const spans = await llmSpans(store);
-  const leaks = [];
-  for (const span of spans) {
-    const output = span.output ?? "";
-    for (const s of targets) {
-      if (s.canary && output.includes(s.canary)) {
-        leaks.push({
-          scenarioId: s.id,
-          canary: s.canary,
-          runId: span.runId,
-          evidence: excerpt(output, s.canary)
-        });
-      }
-    }
-  }
-  return leaks;
-}
-var HoldoutAuditor = class {
-  scenarios;
-  accessLog = [];
-  constructor(scenarios) {
-    this.scenarios = scenarios;
-  }
-  /** Retrieve a holdout scenario for a declared purpose. Non-'evaluation' throws. */
-  get(scenarioId, purpose) {
-    if (purpose !== "evaluation" && purpose !== "debugging") {
-      throw new Error(
-        `HoldoutAuditor.get: purpose must be 'evaluation' or 'debugging', got ${purpose}`
-      );
-    }
-    const s = this.scenarios.find((x) => x.id === scenarioId);
-    if (!s) throw new Error(`holdout scenario "${scenarioId}" not found`);
-    this.accessLog.push({ scenarioId, purpose, at: Date.now() });
-    return s;
-  }
-  getAccessLog() {
-    return this.accessLog;
-  }
-};
-function excerpt(source, needle) {
-  const at = source.indexOf(needle);
-  if (at < 0) return "";
-  const start = Math.max(0, at - 30);
-  const end = Math.min(source.length, at + needle.length + 30);
-  return (start > 0 ? "\u2026" : "") + source.slice(start, end) + (end < source.length ? "\u2026" : "");
-}
-
-// src/contract/analyze-runs.ts
-function summarizeExecution(opts) {
-  const runs = opts.runs.map(validateRunRecord);
-  const bins = opts.histogramBins ?? 12;
-  return {
-    execution: computeExecutionInsight(runs, bins),
-    costProvenance: summarizeCostProvenance(runs)
-  };
-}
-async function analyzeRuns(opts) {
-  const runs = opts.runs.map(validateRunRecord);
-  const bins = opts.histogramBins ?? 12;
-  const threshold = opts.decisionThreshold ?? 0.02;
-  const split = resolveSplit(runs, opts.split ?? "auto");
-  const compositeWithIds = runs.map((r) => ({ runId: r.runId, score: compositeOf(r, split) })).filter((p) => Number.isFinite(p.score));
-  const composite = distributionOf(
-    compositeWithIds.map((p) => p.score),
-    bins,
-    compositeWithIds
-  );
-  const perDimension = computePerDimension(runs, bins);
-  const { execution, costProvenance: provenance } = summarizeExecution({
-    runs,
-    histogramBins: bins
-  });
-  const knownCostRuns = runs.filter((run) => run.costProvenance.kind !== "uncaptured");
-  const costs = knownCostRuns.map((r) => r.costUsd).filter(isFiniteNumber);
-  const costDist = distributionOf(costs, bins);
-  const pareto = paretoChart(knownCostRuns, { split });
-  const degraded = {};
-  if (provenance.uncaptured.n > 0) {
-    degraded.cost = diagnoseCostCoverage(runs, provenance);
-  } else if (costs.length === 0 || costs.every((c) => c === 0)) {
-    degraded.cost = `all ${runs.length} explicitly observed or estimated USD values are $0`;
-  }
-  if (pareto.points.length < 2) {
-    degraded.pareto = pareto.points.length === 0 ? "no candidates \u2014 Pareto unavailable" : "single candidate \u2014 Pareto is a single point, not a frontier";
-  }
-  const costQuality = {
-    cost: costDist,
-    pareto,
-    provenance,
-    ...degraded.cost || degraded.pareto ? { degraded } : {}
-  };
-  const judges = computeJudgeInsights(runs);
-  const interRater = opts.raterScores ? computeInterRater(opts.raterScores) : void 0;
-  const lift = computeLift(runs, opts.baselineCandidateId, opts.candidateCandidateId, split);
-  const failureClusters = opts.analyst ? await computeFailureClusters(runs, opts.analyst, split) : void 0;
-  const failureClasses = computeFailureClasses(runs, split);
-  const contamination = opts.canaryScenarios ? computeContamination(runs, opts.canaryScenarios) : void 0;
-  const outcomeCorrelation = opts.outcomeSignal ? computeOutcomeCorrelation(runs, opts.outcomeSignal, split) : void 0;
-  const release = buildReleaseScorecard(composite, lift, contamination);
-  const priorPeriodComparison = opts.baselineRuns ? computePriorPeriodComparison(runs, opts.baselineRuns, split, opts.baselineLabel) : void 0;
-  const recommendations = buildRecommendations({
-    composite,
-    judges,
-    interRater,
-    lift,
-    failureClusters,
-    failureClasses,
-    contamination,
-    outcomeCorrelation,
-    priorPeriodComparison,
-    threshold
-  });
-  return {
-    n: runs.length,
-    execution,
-    composite,
-    perDimension,
-    costQuality,
-    judges,
-    interRater,
-    lift,
-    failureClusters,
-    contamination,
-    outcomeCorrelation,
-    release,
-    ...failureClasses ? { failureClasses } : {},
-    ...priorPeriodComparison ? { priorPeriodComparison } : {},
-    recommendations
-  };
-}
-function computeExecutionInsight(runs, bins) {
-  const aggregateRows = runs.flatMap((run) => {
-    const usage = aggregateTokenUsage(run);
-    return usage ? [{ usage, costUsd: finiteRaw(run, "aggregate_cost_usd") }] : [];
-  });
-  const aggregateCosts = aggregateRows.flatMap(
-    (row) => row.costUsd !== void 0 ? [row.costUsd] : []
-  );
-  const modelCounts = /* @__PURE__ */ new Map();
-  let executionErrorRuns = 0;
-  let executionErrorEvents = 0;
-  let errorReportingRuns = 0;
-  let errorSpanEvents = 0;
-  let errorSpanReportingRuns = 0;
-  const terminalOutcomes = {
-    succeeded: 0,
-    failed: 0,
-    cancelled: 0,
-    incomplete: 0,
-    unknown: 0
-  };
-  const errorsByTerminalOutcome = {
-    succeeded: { withErrors: 0, withoutErrors: 0, unreported: 0 },
-    failed: { withErrors: 0, withoutErrors: 0, unreported: 0 },
-    cancelled: { withErrors: 0, withoutErrors: 0, unreported: 0 },
-    incomplete: { withErrors: 0, withoutErrors: 0, unreported: 0 },
-    unknown: { withErrors: 0, withoutErrors: 0, unreported: 0 }
-  };
-  let modelCallRuns = 0;
-  let modelCallEvents = 0;
-  let modelCallReportingRuns = 0;
-  for (const run of runs) {
-    modelCounts.set(run.model, (modelCounts.get(run.model) ?? 0) + 1);
-    const terminalOutcome = run.terminalOutcome;
-    terminalOutcomes[terminalOutcome] += 1;
-    const modelCalls = nonNegativeCountRaw(run, "llm_span_count");
-    if (modelCalls !== void 0) {
-      modelCallEvents += modelCalls;
-      modelCallReportingRuns += 1;
-    }
-    const usage = run.tokenUsage;
-    if ((modelCalls ?? 0) > 0 || usage.input > 0 || usage.output > 0 || (usage.cached ?? 0) > 0 || (usage.cacheWrite ?? 0) > 0) {
-      modelCallRuns += 1;
-    }
-    const errorEvents = reportedExecutionErrorEvents(run);
-    if (errorEvents !== void 0) {
-      executionErrorEvents += errorEvents;
-      errorReportingRuns += 1;
-      if (errorEvents > 0) {
-        executionErrorRuns += 1;
-        errorsByTerminalOutcome[terminalOutcome].withErrors += 1;
-      } else errorsByTerminalOutcome[terminalOutcome].withoutErrors += 1;
-    } else errorsByTerminalOutcome[terminalOutcome].unreported += 1;
-    const reportedErrorSpans = nonNegativeCountRaw(run, "error_span_count");
-    if (reportedErrorSpans !== void 0) {
-      errorSpanEvents += reportedErrorSpans;
-      errorSpanReportingRuns += 1;
-    }
-  }
-  return {
-    durationMs: distributionOf(
-      runs.map((run) => run.wallMs),
-      bins
-    ),
-    queueMs: distributionOf(
-      runs.filter((run) => run.queueMs !== void 0).map((run) => run.queueMs),
-      bins
-    ),
-    tokenUsage: summarizeTokenUsage(
-      runs.map((run) => run.tokenUsage),
-      bins
-    ),
-    aggregateUsage: {
-      runs: aggregateRows.length,
-      tokenUsage: summarizeTokenUsage(
-        aggregateRows.map((row) => row.usage),
-        bins
-      ),
-      costUsd: distributionOf(aggregateCosts, bins),
-      totalCostUsd: aggregateCosts.reduce((total, value) => total + value, 0)
-    },
-    models: [...modelCounts.entries()].map(([model, count]) => ({ model, runs: count })).sort((left, right) => right.runs - left.runs || left.model.localeCompare(right.model)),
-    modelCalls: {
-      runs: modelCallRuns,
-      events: modelCallEvents,
-      reportingRuns: modelCallReportingRuns
-    },
-    executionErrors: {
-      runs: executionErrorRuns,
-      fraction: errorReportingRuns > 0 ? executionErrorRuns / errorReportingRuns : null,
-      events: executionErrorEvents,
-      reportingRuns: errorReportingRuns,
-      errorSpanEvents,
-      errorSpanReportingRuns,
-      byTerminalOutcome: errorsByTerminalOutcome
-    },
-    terminalOutcomes
-  };
-}
-function reportedExecutionErrorEvents(run) {
-  return nonNegativeCountRaw(run, "execution_error_count");
-}
-function nonNegativeCountRaw(run, key) {
-  const value = finiteRaw(run, key);
-  return value !== void 0 && Number.isInteger(value) && value >= 0 ? value : void 0;
-}
-function summarizeTokenUsage(usages, bins) {
-  const reasoning = usages.flatMap(
-    (usage) => usage.reasoning !== void 0 ? [usage.reasoning] : []
-  );
-  const cached = usages.flatMap((usage) => usage.cached !== void 0 ? [usage.cached] : []);
-  const cacheWrite = usages.flatMap(
-    (usage) => usage.cacheWrite !== void 0 ? [usage.cacheWrite] : []
-  );
-  return {
-    input: distributionOf(
-      usages.map((usage) => usage.input),
-      bins
-    ),
-    output: distributionOf(
-      usages.map((usage) => usage.output),
-      bins
-    ),
-    reasoning: distributionOf(reasoning, bins),
-    cached: distributionOf(cached, bins),
-    cacheWrite: distributionOf(cacheWrite, bins),
-    totals: {
-      input: usages.reduce((total, usage) => total + usage.input, 0),
-      output: usages.reduce((total, usage) => total + usage.output, 0),
-      reasoning: reasoning.reduce((total, value) => total + value, 0),
-      cached: cached.reduce((total, value) => total + value, 0),
-      cacheWrite: cacheWrite.reduce((total, value) => total + value, 0)
-    }
-  };
-}
-function aggregateTokenUsage(run) {
-  const input = finiteRaw(run, "aggregate_prompt_tokens");
-  const output = finiteRaw(run, "aggregate_completion_tokens");
-  const reasoning = finiteRaw(run, "aggregate_reasoning_tokens");
-  const cached = finiteRaw(run, "aggregate_cached_tokens");
-  const cacheWrite = finiteRaw(run, "aggregate_cache_write_tokens");
-  if (input === void 0 && output === void 0 && reasoning === void 0 && cached === void 0 && cacheWrite === void 0)
-    return void 0;
-  return {
-    input: input ?? 0,
-    output: output ?? 0,
-    ...reasoning !== void 0 ? { reasoning } : {},
-    ...cached !== void 0 ? { cached } : {},
-    ...cacheWrite !== void 0 ? { cacheWrite } : {}
-  };
-}
-function finiteRaw(run, key) {
-  const value = run.outcome.raw[key];
-  return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
-}
-function summarizeCostProvenance(runs) {
-  const summary = {
-    observed: { n: 0, totalUsd: 0 },
-    estimated: { n: 0, totalUsd: 0 },
-    uncaptured: { n: 0 },
-    knownFraction: 0
-  };
-  for (const run of runs) {
-    const cost = run.costProvenance;
-    if (cost.kind === "uncaptured") {
-      summary.uncaptured.n += 1;
-    } else {
-      summary[cost.kind].n += 1;
-      summary[cost.kind].totalUsd += cost.usd;
-    }
-  }
-  const known = summary.observed.n + summary.estimated.n;
-  summary.knownFraction = runs.length > 0 ? known / runs.length : 0;
-  return summary;
-}
-function diagnoseCostCoverage(runs, provenance) {
-  const uncaptured = provenance.uncaptured.n;
-  const known = provenance.observed.n + provenance.estimated.n;
-  if (uncaptured === runs.length) {
-    return `USD cost uncaptured for all ${runs.length} runs \u2014 no observed or estimated USD values; token and wall-time metrics remain available.`;
-  }
-  return `USD cost uncaptured for ${uncaptured}/${runs.length} runs; excluded those rows from cost statistics (${known}/${runs.length} retained: ${provenance.observed.n} observed, ${provenance.estimated.n} estimated).`;
-}
-function computeFailureClasses(runs, split) {
-  const counts = /* @__PURE__ */ new Map();
-  for (const r of runs) {
-    if (!isTaskFailure(r, split)) continue;
-    const key = r.failureClass !== void 0 && r.failureClass !== "success" ? r.failureClass : "unknown";
-    counts.set(key, (counts.get(key) ?? 0) + 1);
-  }
-  if (counts.size === 0) return void 0;
-  const n = runs.length;
-  return [...counts.entries()].map(([failureClass, count]) => ({
-    failureClass,
-    count,
-    share: n > 0 ? count / n : 0
-  })).sort((a, b) => b.count - a.count || a.failureClass.localeCompare(b.failureClass));
-}
-function computePriorPeriodComparison(current, baseline, split, windowLabel) {
-  if (current.length === 0 || baseline.length === 0) return void 0;
-  const metrics = {};
-  const directions = {};
-  const compositeCurrent = current.map((r) => compositeOf(r, split)).filter(Number.isFinite);
-  const compositeBaseline = baseline.map((r) => compositeOf(r, split)).filter(Number.isFinite);
-  if (compositeCurrent.length > 0 && compositeBaseline.length > 0) {
-    metrics.composite = welchCompare(compositeBaseline, compositeCurrent);
-    directions.composite = "higher-is-better";
-  }
-  const costCurrent = knownCostValues(current);
-  const costBaseline = knownCostValues(baseline);
-  if (costCurrent.length > 0 && costBaseline.length > 0) {
-    metrics.cost = welchCompare(costBaseline, costCurrent);
-    directions.cost = "lower-is-better";
-  }
-  const durCurrent = current.map((r) => r.wallMs).filter(Number.isFinite);
-  const durBaseline = baseline.map((r) => r.wallMs).filter(Number.isFinite);
-  if (durCurrent.length > 0 && durBaseline.length > 0) {
-    metrics.duration = welchCompare(durBaseline, durCurrent);
-    directions.duration = "lower-is-better";
-  }
-  const tokCurrent = current.map((r) => (r.tokenUsage.input ?? 0) + (r.tokenUsage.output ?? 0)).filter(Number.isFinite);
-  const tokBaseline = baseline.map((r) => (r.tokenUsage.input ?? 0) + (r.tokenUsage.output ?? 0)).filter(Number.isFinite);
-  if (tokCurrent.length > 0 && tokBaseline.length > 0) {
-    metrics.tokenUsage = welchCompare(tokBaseline, tokCurrent);
-    directions.tokenUsage = "lower-is-better";
-  }
-  const dimsCurrent = collectPerDimension(current);
-  const dimsBaseline = collectPerDimension(baseline);
-  for (const dim of Object.keys(dimsCurrent)) {
-    const b = dimsBaseline[dim];
-    const c = dimsCurrent[dim];
-    if (!b || b.length === 0 || !c || c.length === 0) continue;
-    metrics[`dim.${dim}`] = welchCompare(b, c);
-    directions[`dim.${dim}`] = "higher-is-better";
-  }
-  const regressedMetrics = [];
-  const improvedMetrics = [];
-  for (const [name, delta] of Object.entries(metrics)) {
-    if (!delta.significant) continue;
-    const dir = directions[name] ?? "higher-is-better";
-    const better = dir === "higher-is-better" ? delta.delta > 0 : delta.delta < 0;
-    if (better) improvedMetrics.push(name);
-    else regressedMetrics.push(name);
-  }
-  return {
-    baselineN: baseline.length,
-    currentN: current.length,
-    ...windowLabel ? { windowLabel } : {},
-    metrics,
-    regressedMetrics,
-    improvedMetrics
-  };
-}
-function knownCostValues(runs) {
-  return runs.filter((run) => run.costProvenance.kind !== "uncaptured").map((run) => run.costUsd).filter(isFiniteNumber);
-}
-function isFiniteNumber(value) {
-  return typeof value === "number" && Number.isFinite(value);
-}
-function collectPerDimension(runs) {
-  const out = {};
-  for (const r of runs) {
-    const perDim = r.outcome.judgeScores?.perDimMean;
-    if (!perDim) continue;
-    for (const [dim, value] of Object.entries(perDim)) {
-      if (!Number.isFinite(value)) continue;
-      if (!out[dim]) out[dim] = [];
-      out[dim].push(value);
-    }
-  }
-  return out;
-}
-function welchCompare(baseline, current) {
-  const baselineMean = mean(baseline);
-  const currentMean = mean(current);
-  const baselineVar = sampleVariance(baseline, baselineMean);
-  const currentVar = sampleVariance(current, currentMean);
-  const baselineN = baseline.length;
-  const currentN = current.length;
-  const delta = currentMean - baselineMean;
-  const se = Math.sqrt(baselineVar / baselineN + currentVar / currentN);
-  const halfWidth = 1.96 * (se > 0 ? se : 0);
-  const ci95 = [delta - halfWidth, delta + halfWidth];
-  const t = se > 0 ? delta / se : 0;
-  const pValue = se > 0 ? 2 * (1 - standardNormalCdf(Math.abs(t))) : 1;
-  const pooledStddev = Math.sqrt(
-    ((baselineN - 1) * baselineVar + (currentN - 1) * currentVar) / Math.max(1, baselineN + currentN - 2)
-  );
-  const cohensD = pooledStddev > 0 ? delta / pooledStddev : 0;
-  const significant = pValue < 0.05 && Math.abs(cohensD) >= 0.2;
-  return {
-    current: currentMean,
-    baseline: baselineMean,
-    delta,
-    ci95,
-    pValue,
-    cohensD,
-    baselineN,
-    currentN,
-    significant
-  };
-}
-function sampleVariance(xs, xsMean) {
-  if (xs.length < 2) return 0;
-  let s = 0;
-  for (const x of xs) s += (x - xsMean) ** 2;
-  return s / (xs.length - 1);
-}
-function standardNormalCdf(z) {
-  const a1 = 0.254829592;
-  const a2 = -0.284496736;
-  const a3 = 1.421413741;
-  const a4 = -1.453152027;
-  const a5 = 1.061405429;
-  const p = 0.3275911;
-  const sign = z < 0 ? -1 : 1;
-  const x = Math.abs(z) / Math.SQRT2;
-  const t = 1 / (1 + p * x);
-  const y = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-x * x);
-  return 0.5 * (1 + sign * y);
-}
-function resolveSplit(runs, pref) {
-  if (pref !== "auto") return pref;
-  const hasHoldout = runs.some((r) => Number.isFinite(observedSplitScore(r, "holdout")));
-  return hasHoldout ? "holdout" : "search";
-}
-function compositeOf(run, split) {
-  const score = observedSplitScore(run, split);
-  return Number.isFinite(score) ? score : Number.NaN;
-}
-function distributionOf(values, bins, withIds) {
-  if (values.length === 0) {
-    return {
-      n: 0,
-      mean: null,
-      p50: null,
-      p95: null,
-      stddev: null,
-      min: null,
-      max: null,
-      histogram: []
-    };
-  }
-  const sorted = [...values].sort((a, b) => a - b);
-  const n = sorted.length;
-  const mean2 = sorted.reduce((s, v) => s + v, 0) / n;
-  const variance = sorted.reduce((s, v) => s + (v - mean2) ** 2, 0) / n;
-  const stddev = Math.sqrt(variance);
-  const tailRuns = withIds ? [...withIds].sort((a, b) => a.score - b.score).slice(0, Math.min(5, withIds.length)) : void 0;
-  return {
-    n,
-    mean: mean2,
-    p50: percentile(sorted, 0.5),
-    p95: percentile(sorted, 0.95),
-    stddev,
-    min: sorted[0],
-    max: sorted[n - 1],
-    histogram: histogram(sorted, bins),
-    ...tailRuns ? { tailRuns } : {}
-  };
-}
-function percentile(sorted, q) {
-  if (sorted.length === 0) return 0;
-  if (sorted.length === 1) return sorted[0];
-  const idx = (sorted.length - 1) * q;
-  const lo = Math.floor(idx);
-  const hi = Math.ceil(idx);
-  if (lo === hi) return sorted[lo];
-  const w = idx - lo;
-  return sorted[lo] * (1 - w) + sorted[hi] * w;
-}
-function histogram(sorted, bins) {
-  if (sorted.length === 0 || bins < 1) return [];
-  const min = sorted[0];
-  const max = sorted[sorted.length - 1];
-  if (min === max) return [{ lo: min, hi: max, count: sorted.length }];
-  const width = (max - min) / bins;
-  const out = [];
-  for (let i = 0; i < bins; i++) {
-    const lo = min + i * width;
-    const hi = i === bins - 1 ? max : lo + width;
-    out.push({ lo, hi, count: 0 });
-  }
-  for (const v of sorted) {
-    const idx = Math.min(bins - 1, Math.floor((v - min) / width));
-    out[idx].count++;
-  }
-  return out;
-}
-function computePerDimension(runs, bins) {
-  const byDim = /* @__PURE__ */ new Map();
-  for (const run of runs) {
-    const scores = run.outcome.judgeScores;
-    if (!scores) continue;
-    for (const [dim, value] of Object.entries(scores.perDimMean ?? {})) {
-      if (!Number.isFinite(value)) continue;
-      const arr = byDim.get(dim) ?? [];
-      arr.push(value);
-      byDim.set(dim, arr);
-    }
-  }
-  const out = {};
-  for (const [dim, values] of byDim) out[dim] = distributionOf(values, bins);
-  return out;
-}
-function computeJudgeInsights(runs) {
-  const out = {};
-  const byJudge = /* @__PURE__ */ new Map();
-  for (const run of runs) {
-    const scores = run.outcome.judgeScores;
-    if (!scores?.perJudge) continue;
-    for (const [judgeId, dims] of Object.entries(scores.perJudge)) {
-      const dimValues = Object.values(dims).filter(Number.isFinite);
-      if (dimValues.length === 0) continue;
-      const judgeMean = dimValues.reduce((s, v) => s + v, 0) / dimValues.length;
-      const arr = byJudge.get(judgeId) ?? [];
-      arr.push(judgeMean);
-      byJudge.set(judgeId, arr);
-    }
-  }
-  for (const [judgeId, values] of byJudge) {
-    out[judgeId] = {
-      n: values.length,
-      meanScore: values.reduce((s, v) => s + v, 0) / values.length
-    };
-  }
-  return out;
-}
-function computeInterRater(ratings) {
-  const byRun = /* @__PURE__ */ new Map();
-  for (const r of ratings) {
-    if (!Number.isFinite(r.score)) continue;
-    const list = byRun.get(r.runId) ?? [];
-    list.push({ rater: r.rater, score: r.score });
-    byRun.set(r.runId, list);
-  }
-  const raters = new Set(ratings.map((r) => r.rater));
-  const jointlyRated = [];
-  for (const [runId, ratersForRun] of byRun) {
-    const seen = new Set(ratersForRun.map((r) => r.rater));
-    let all = true;
-    for (const r of raters) if (!seen.has(r)) all = false;
-    if (all) jointlyRated.push(runId);
-  }
-  if (raters.size < 2 || jointlyRated.length === 0) return void 0;
-  const raterList = [...raters].sort();
-  const perPair = {};
-  for (let i = 0; i < raterList.length; i++) {
-    for (let j = i + 1; j < raterList.length; j++) {
-      const a = raterList[i];
-      const b = raterList[j];
-      const aScores = [];
-      const bScores = [];
-      for (const runId of jointlyRated) {
-        const ratersForRun = byRun.get(runId);
-        const sa = ratersForRun.find((r) => r.rater === a)?.score;
-        const sb = ratersForRun.find((r) => r.rater === b)?.score;
-        if (sa !== void 0 && sb !== void 0) {
-          aScores.push(sa);
-          bScores.push(sb);
-        }
-      }
-      const agreement2 = continuousAgreement(
-        aScores.map((score, index) => [score, bScores[index]]),
-        { bootstrap: 0 }
-      );
-      perPair[`${a}::${b}`] = agreement2.weightedKappa;
-    }
-  }
-  const matrix = jointlyRated.map((runId) => {
-    const ratingsByRater = new Map(byRun.get(runId).map((rating) => [rating.rater, rating.score]));
-    return raterList.map((rater) => ratingsByRater.get(rater));
-  });
-  const agreement = continuousAgreement(matrix, { bootstrap: 0 });
-  const disagreementCases = jointlyRated.map((runId) => {
-    const ratersForRun = byRun.get(runId);
-    const scores = ratersForRun.map((r) => r.score);
-    const range = Math.max(...scores) - Math.min(...scores);
-    return { runId, ratings: ratersForRun, range };
-  }).sort((a, b) => b.range - a.range).slice(0, 20);
-  return {
-    raters: raters.size,
-    jointlyRated: jointlyRated.length,
-    kappa: Number.isFinite(agreement.weightedKappa) ? agreement.weightedKappa : 0,
-    icc: agreement.icc,
-    pearson: agreement.pearson,
-    spearman: agreement.spearman,
-    perPair,
-    disagreementCases
-  };
-}
-function computeLift(runs, baselineId, candidateId, split) {
-  let bId = baselineId;
-  let cId = candidateId;
-  if (!bId || !cId) {
-    const ids = [...new Set(runs.map((r) => r.candidateId))];
-    if (ids.length !== 2) return void 0;
-    const [idA, idB] = ids;
-    const scoresA = finiteCompositeScores(
-      runs.filter((run) => run.candidateId === idA),
-      split
-    );
-    const scoresB = finiteCompositeScores(
-      runs.filter((run) => run.candidateId === idB),
-      split
-    );
-    if (scoresA.length === 0 || scoresB.length === 0) return void 0;
-    const meanA = mean(scoresA);
-    const meanB = mean(scoresB);
-    bId = meanA <= meanB ? idA : idB;
-    cId = meanA <= meanB ? idB : idA;
-  }
-  const baseline = runs.filter((r) => r.candidateId === bId);
-  const candidate = runs.filter((r) => r.candidateId === cId);
-  if (baseline.length === 0 || candidate.length === 0) return void 0;
-  const scoredBaseline = baseline.filter((run) => Number.isFinite(compositeOf(run, split)));
-  const scoredCandidate = candidate.filter((run) => Number.isFinite(compositeOf(run, split)));
-  const pairing = pairRunRecords(scoredBaseline, scoredCandidate);
-  const pairedBaseline = pairing.pairs.map((pair) => compositeOf(pair.baseline, split));
-  const pairedCandidate = pairing.pairs.map((pair) => compositeOf(pair.treatment, split));
-  if (pairedBaseline.length === 0) return void 0;
-  const baselineMean = mean(pairedBaseline);
-  const candidateMean = mean(pairedCandidate);
-  const delta = candidateMean - baselineMean;
-  const bootstrap = pairedBootstrap(pairedBaseline, pairedCandidate, {
-    confidence: 0.95,
-    resamples: 2e3,
-    statistic: "mean"
-  });
-  const tTest = pairedTTest(pairedBaseline, pairedCandidate);
-  const d = pairedCohensDz(pairedBaseline, pairedCandidate);
-  const mde = pairedMde({ nPaired: pairedBaseline.length, power: 0.8, alpha: 0.05 });
-  const requiredN = d === null || d === 0 ? null : requiredPairedSampleSize({
-    effect: Math.abs(d),
-    power: 0.8,
-    alpha: 0.05
-  });
-  return {
-    baselineMean,
-    candidateMean,
-    delta,
-    ci95: [bootstrap.low, bootstrap.high],
-    pValue: tTest.p,
-    n: pairedBaseline.length,
-    unpairedBaseline: pairing.unpairedBaseline.length,
-    unpairedCandidate: pairing.unpairedTreatment.length,
-    cohensD: d,
-    mde,
-    requiredN
-  };
-}
-function mean(arr) {
-  return arr.length === 0 ? 0 : arr.reduce((s, v) => s + v, 0) / arr.length;
-}
-async function computeFailureClusters(runs, analyst, split) {
-  const failed = runs.filter((run) => isTaskFailure(run, split));
-  if (failed.length === 0) return { clusters: [], totalFailures: 0 };
-  const clusters = /* @__PURE__ */ new Map();
-  for (const run of failed) {
-    try {
-      const result = await analyst.run(run.runId, { runRecord: run });
-      for (const finding of result.findings) {
-        const key = finding.area || finding.analyst_id || "unclassified";
-        const c = clusters.get(key) ?? { exemplars: [], share: 0 };
-        if (c.exemplars.length < 5) c.exemplars.push(run.runId);
-        clusters.set(key, c);
-      }
-    } catch {
-      const c = clusters.get("analyst-error") ?? { exemplars: [], share: 0 };
-      if (c.exemplars.length < 5) c.exemplars.push(run.runId);
-      clusters.set("analyst-error", c);
-    }
-  }
-  const clusterList = [...clusters.entries()].map(([id, c]) => ({
-    id,
-    name: id,
-    share: c.exemplars.length / failed.length,
-    exemplars: c.exemplars
-  }));
-  clusterList.sort((a, b) => b.share - a.share);
-  return { clusters: clusterList, totalFailures: failed.length };
-}
-function finiteCompositeScores(runs, split) {
-  return runs.map((run) => compositeOf(run, split)).filter(Number.isFinite);
-}
-function isTaskFailure(run, split) {
-  if (run.failureClass !== void 0 && run.failureClass !== "success") return true;
-  const score = compositeOf(run, split);
-  return Number.isFinite(score) && score < 0.5;
-}
-function computeContamination(runs, canaries) {
-  let leaks = 0;
-  const details = [];
-  for (const run of runs) {
-    const output = stringifyOutput(run);
-    if (!output) continue;
-    const leaksHere = checkCanaries(output, canaries);
-    for (const leak of leaksHere) {
-      leaks++;
-      details.push({ runId: run.runId, canary: leak.canary, matched: leak.evidence });
-    }
-  }
-  return { leaks, holdoutAuditPassed: leaks === 0, details };
-}
-function stringifyOutput(run) {
-  const metadata = run.metadata;
-  if (typeof metadata?.output === "string") return metadata.output;
-  if (typeof metadata?.text === "string") return metadata.text;
-  return void 0;
-}
-function computeOutcomeCorrelation(runs, outcome, split) {
-  const xs = [];
-  const ys = [];
-  for (const run of runs) {
-    const y = outcome.valueByRunId[run.runId];
-    if (y === void 0 || !Number.isFinite(y)) continue;
-    const x = compositeOf(run, split);
-    if (!Number.isFinite(x)) continue;
-    xs.push(x);
-    ys.push(y);
-  }
-  if (xs.length < 3) return void 0;
-  const p = pearsonR(xs, ys);
-  const s = spearmanR(xs, ys);
-  const meanX = mean(xs);
-  const meanY = mean(ys);
-  let num = 0;
-  let denom = 0;
-  for (let i = 0; i < xs.length; i++) {
-    num += (xs[i] - meanX) * (ys[i] - meanY);
-    denom += (xs[i] - meanX) ** 2;
-  }
-  const slope = denom === 0 ? 0 : num / denom;
-  const intercept = meanY - slope * meanX;
-  const ssTot = ys.reduce((a, y) => a + (y - meanY) ** 2, 0);
-  const ssRes = ys.reduce((a, y, i) => a + (y - (intercept + slope * xs[i])) ** 2, 0);
-  const r2 = ssTot === 0 ? 0 : 1 - ssRes / ssTot;
-  return {
-    metric: outcome.metric,
-    n: xs.length,
-    pearson: p,
-    spearman: s,
-    rewardModel: { intercept, slope, r2 }
-  };
-}
-function buildReleaseScorecard(composite, lift, contamination) {
-  const axes = [];
-  const liftPass = lift === void 0 ? "not_evaluated" : lift.ci95[0] > 0 ? "pass" : lift.delta > 0 ? "warn" : "fail";
-  axes.push({
-    name: "quality-lift",
-    status: liftPass,
-    detail: lift ? `delta=${lift.delta.toFixed(3)}, CI95=[${lift.ci95[0].toFixed(3)}, ${lift.ci95[1].toFixed(3)}], n=${lift.n}` : "no baseline/candidate pair available"
-  });
-  const contamPass = contamination === void 0 ? "not_evaluated" : contamination.leaks === 0 ? "pass" : "fail";
-  axes.push({
-    name: "contamination",
-    status: contamPass,
-    detail: contamination ? `${contamination.leaks} canary leak(s)` : "no canaries supplied"
-  });
-  axes.push(
-    composite.n === 0 ? {
-      name: "composite-distribution",
-      status: "not_evaluated",
-      detail: "no task-quality scores available"
-    } : {
-      name: "composite-distribution",
-      status: composite.mean !== null && composite.mean >= 0.5 ? "pass" : composite.mean !== null && composite.mean >= 0.3 ? "warn" : "fail",
-      detail: composite.mean === null || composite.p50 === null || composite.p95 === null ? "task-quality distribution is internally incomplete" : `mean=${composite.mean.toFixed(3)}, p50=${composite.p50.toFixed(3)}, p95=${composite.p95.toFixed(3)} over n=${composite.n}`
-    }
-  );
-  const status = axes.some((a) => a.status === "fail") ? "fail" : axes.some((a) => a.status === "warn" || a.status === "not_evaluated") ? "warn" : "pass";
-  return {
-    status,
-    axes,
-    issues: []
-  };
-}
-function buildRecommendations(ctx) {
-  const out = [];
-  if (ctx.priorPeriodComparison) {
-    const ppc = ctx.priorPeriodComparison;
-    const label = ppc.windowLabel ?? "baseline period";
-    for (const name of ppc.regressedMetrics) {
-      const d = ppc.metrics[name];
-      if (!d) continue;
-      out.push({
-        priority: "critical",
-        kind: "investigate",
-        title: `${name} regressed from ${d.baseline.toFixed(3)} \u2192 ${d.current.toFixed(3)} vs ${label}`,
-        detail: `Welch CI95 = [${d.ci95[0].toFixed(3)}, ${d.ci95[1].toFixed(3)}], p=${d.pValue.toFixed(4)}, Cohen's d=${d.cohensD.toFixed(2)} (n_current=${d.currentN}, n_baseline=${d.baselineN}). The regression is statistically significant at p<0.05 with at-least-small effect size.`,
-        evidencePath: `priorPeriodComparison.metrics.${name}`
-      });
-    }
-    for (const name of ppc.improvedMetrics) {
-      const d = ppc.metrics[name];
-      if (!d) continue;
-      out.push({
-        priority: "low",
-        kind: "ship",
-        title: `${name} improved from ${d.baseline.toFixed(3)} \u2192 ${d.current.toFixed(3)} vs ${label}`,
-        detail: `Welch CI95 = [${d.ci95[0].toFixed(3)}, ${d.ci95[1].toFixed(3)}], p=${d.pValue.toFixed(4)}, Cohen's d=${d.cohensD.toFixed(2)} (n_current=${d.currentN}, n_baseline=${d.baselineN}). Statistically significant improvement worth flagging.`,
-        evidencePath: `priorPeriodComparison.metrics.${name}`
-      });
-    }
-  }
-  if (ctx.composite.n > 0 && ctx.composite.mean !== null && ctx.composite.p50 !== null && ctx.composite.p95 !== null) {
-    if (ctx.composite.mean < 0.3) {
-      const tail = ctx.composite.tailRuns ?? [];
-      const names = tail.slice(0, 5).map((t) => `${t.runId}=${t.score.toFixed(3)}`).join(", ");
-      out.push({
-        priority: "critical",
-        kind: "investigate",
-        title: `Composite mean ${ctx.composite.mean.toFixed(3)} is below the 0.3 floor \u2014 the agent is broken on this corpus`,
-        detail: tail.length > 0 ? `Worst ${tail.length} run${tail.length === 1 ? "" : "s"} to inspect first: ${names}. Histogram p50=${ctx.composite.p50.toFixed(3)}, p95=${ctx.composite.p95.toFixed(3)}.` : `Histogram p50=${ctx.composite.p50.toFixed(3)}, p95=${ctx.composite.p95.toFixed(3)}.`,
-        evidencePath: "composite.tailRuns"
-      });
-    } else if (ctx.composite.mean < 0.5) {
-      const tail = ctx.composite.tailRuns ?? [];
-      const names = tail.slice(0, 3).map((t) => `${t.runId}=${t.score.toFixed(3)}`).join(", ");
-      out.push({
-        priority: "high",
-        kind: "investigate",
-        title: `Composite mean ${ctx.composite.mean.toFixed(3)} is below 0.5 \u2014 investigate the lower tail before claiming the agent is healthy`,
-        detail: tail.length > 0 ? `Worst ${tail.length} run${tail.length === 1 ? "" : "s"}: ${names}. Histogram p50=${ctx.composite.p50.toFixed(3)}, p95=${ctx.composite.p95.toFixed(3)}.` : `Histogram p50=${ctx.composite.p50.toFixed(3)}, p95=${ctx.composite.p95.toFixed(3)}.`,
-        evidencePath: "composite.tailRuns"
-      });
-    }
-  }
-  if (ctx.failureClasses && ctx.failureClasses.length > 0) {
-    const top = ctx.failureClasses[0];
-    if (top.count >= 3 && top.share >= 0.15) {
-      out.push({
-        priority: top.share >= 0.25 ? "high" : "medium",
-        kind: "investigate",
-        title: `'${top.failureClass}' is the dominant failure class \u2014 ${top.count} runs (${(top.share * 100).toFixed(0)}% of the corpus)`,
-        detail: `The mean composite can look acceptable while one failure class dominates the lower tail. ${top.count} of ${ctx.composite.n} runs failed with '${top.failureClass}'${ctx.failureClasses.length > 1 ? ` (next: '${ctx.failureClasses[1].failureClass}' \xD7${ctx.failureClasses[1].count})` : ""}. Fix this cause first.`,
-        evidencePath: "failureClasses"
-      });
-    }
-  }
-  if (Object.keys(ctx.judges).length === 0 && ctx.composite.n > 0) {
-    out.push({
-      priority: "medium",
-      kind: "expand-corpus",
-      title: "No judge scores recorded \u2014 per-dimension + calibration insights unavailable",
-      detail: "Records have no `outcome.judgeScores`. To unlock perDimension, judges, and calibration, attach a Judge run during your eval pass and populate `outcome.judgeScores.perJudge[judgeName][dimension] = score`. See `docs/insight-report.md` for the expected shape.",
-      evidencePath: "judges"
-    });
-  }
-  if (ctx.lift) {
-    const pairedEffect = ctx.lift.cohensD === null ? "undefined (zero delta variance)" : ctx.lift.cohensD.toFixed(2);
-    const requiredRuns = ctx.lift.requiredN === null ? "not estimable" : `~${ctx.lift.requiredN} paired runs`;
-    const decisive = ctx.lift.ci95[0] > ctx.threshold;
-    const inconclusive = ctx.lift.ci95[0] <= ctx.threshold && ctx.lift.ci95[1] > ctx.threshold;
-    if (decisive) {
-      out.push({
-        priority: "critical",
-        kind: "ship",
-        title: `Ship \u2014 lift ${ctx.lift.delta.toFixed(3)} (95% CI ${ctx.lift.ci95[0].toFixed(3)}..${ctx.lift.ci95[1].toFixed(3)})`,
-        detail: `Holdout lift exceeds threshold ${ctx.threshold} with 95% bootstrap confidence (n=${ctx.lift.n}, p=${ctx.lift.pValue.toFixed(4)}, paired d=${pairedEffect}).`,
-        evidencePath: "lift"
-      });
-    } else if (inconclusive) {
-      out.push({
-        priority: "high",
-        kind: "expand-corpus",
-        title: `Inconclusive \u2014 required sample is ${requiredRuns} (have ${ctx.lift.n}) at current effect size`,
-        detail: `CI straddles threshold. Current MDE at 80% power is ${ctx.lift.mde.toFixed(3)}; observed delta is ${ctx.lift.delta.toFixed(3)}.`,
-        evidencePath: "lift"
-      });
-    } else {
-      out.push({
-        priority: "critical",
-        kind: "hold",
-        title: `Hold \u2014 lift CI lower bound ${ctx.lift.ci95[0].toFixed(3)} is at or below threshold ${ctx.threshold}`,
-        detail: `Bootstrap CI provides no statistical evidence the candidate is better. Consider tightening the mutation or expanding the holdout.`,
-        evidencePath: "lift"
-      });
-    }
-  }
-  if (ctx.contamination && ctx.contamination.leaks > 0) {
-    out.push({
-      priority: "critical",
-      kind: "fix",
-      title: `${ctx.contamination.leaks} canary leak${ctx.contamination.leaks === 1 ? "" : "s"} detected`,
-      detail: `Holdout integrity is compromised. The lift number is unreliable until you investigate.`,
-      evidencePath: "contamination"
-    });
-  }
-  if (ctx.interRater && ctx.interRater.kappa < 0.5) {
-    out.push({
-      priority: "high",
-      kind: "recalibrate",
-      title: `Inter-rater weighted kappa ${ctx.interRater.kappa.toFixed(2)} is below 0.5`,
-      detail: "Raters disagree on what good looks like. Review the largest disagreement cases and refine the rubric before automating these decisions.",
-      evidencePath: "interRater"
-    });
-  }
-  if (ctx.failureClusters && ctx.failureClusters.clusters.length > 0) {
-    const top = ctx.failureClusters.clusters[0];
-    out.push({
-      priority: "high",
-      kind: "investigate",
-      title: `Top failure cluster: ${top.name} (${(top.share * 100).toFixed(0)}% of failures)`,
-      detail: `${ctx.failureClusters.totalFailures} runs failed. The largest cluster groups ${top.exemplars.length} exemplars under '${top.name}'.`,
-      evidencePath: "failureClusters.clusters[0]"
-    });
-  }
-  if (ctx.outcomeCorrelation && Math.abs(ctx.outcomeCorrelation.spearman) < 0.3) {
-    out.push({
-      priority: "medium",
-      kind: "recalibrate",
-      title: `Judge scores decoupled from ${ctx.outcomeCorrelation.metric} (Spearman \u03C1=${ctx.outcomeCorrelation.spearman.toFixed(2)})`,
-      detail: `Your judges score what they were trained to score, but it isn't predicting downstream ${ctx.outcomeCorrelation.metric}. Consider retraining the judge against ${ctx.outcomeCorrelation.metric} as the gold signal.`,
-      evidencePath: "outcomeCorrelation"
-    });
-  }
-  return out;
-}
-
-export {
-  checkCanaries,
-  checkBehavioralCanary,
-  runBehavioralCanaries,
-  canaryLeakView,
-  HoldoutAuditor,
-  summarizeExecution,
-  analyzeRuns
-};
-//# sourceMappingURL=chunk-M4YBQKIJ.js.map