@tangle-network/agent-eval 0.84.0 → 0.86.0

package/dist/index.js

@@ -230,6 +230,9 @@ import {
   describeTraceInsightScope,
   domainEvidencePattern,
   exportRunAsOtlp,
+  extractUsage,
+  extractUsageFromResponse,
+  extractUsageFromSse,
   flattenOtlpExportToNdjson,
   inferDomainKeywords,
   iterateRawCalls,
@@ -240,7 +243,7 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-XQL22JDG.js";
+} from "./chunk-P2J6SOXT.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -1751,6 +1754,107 @@ function canonicalize2(value) {
   return out;
 }
 
+// src/integrity/preflight.ts
+function stripSlash(url) {
+  return url.replace(/\/+$/, "");
+}
+function errorMessage(body) {
+  if (body == null || typeof body !== "object") return null;
+  const b = body;
+  if (b.error && typeof b.error.message === "string") return b.error.message;
+  if (typeof b.message === "string") return b.message;
+  return null;
+}
+async function preflightModels(opts) {
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const baseUrl = stripSlash(opts.baseUrl);
+  const authHeaders = { authorization: `Bearer ${opts.apiKey}` };
+  let served;
+  try {
+    const res = await fetchImpl(`${baseUrl}/models`, { method: "GET", headers: authHeaders });
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      return {
+        succeeded: false,
+        value: null,
+        error: `preflightModels: GET ${baseUrl}/models \u2192 ${res.status} ${text.slice(0, 400)}`
+      };
+    }
+    const body = await res.json();
+    const ids = Array.isArray(body.data) ? body.data : [];
+    served = new Set(ids.map((m) => m.id).filter((id) => typeof id === "string"));
+  } catch (err) {
+    return {
+      succeeded: false,
+      value: null,
+      error: `preflightModels: GET ${baseUrl}/models failed \u2014 ${err instanceof Error ? err.message : String(err)}`
+    };
+  }
+  const results = [];
+  for (const model of opts.models) {
+    const listed = served.has(model);
+    if (!opts.probe) {
+      results.push({ model, listed, served: null, status: null, detail: null });
+      continue;
+    }
+    try {
+      const res = await fetchImpl(`${baseUrl}/chat/completions`, {
+        method: "POST",
+        headers: { ...authHeaders, "content-type": "application/json" },
+        body: JSON.stringify({
+          model,
+          messages: [{ role: "user", content: "ping" }],
+          max_tokens: 5
+        })
+      });
+      let detail = null;
+      if (!res.ok) {
+        const body = await res.json().catch(() => null);
+        detail = errorMessage(body);
+      }
+      results.push({ model, listed, served: res.ok, status: res.status, detail });
+    } catch (err) {
+      return {
+        succeeded: false,
+        value: null,
+        error: `preflightModels: probe POST ${baseUrl}/chat/completions (model ${model}) failed \u2014 ${err instanceof Error ? err.message : String(err)}`
+      };
+    }
+  }
+  return { succeeded: true, value: results, error: null };
+}
+var ModelsUnreachableError = class extends AgentEvalError {
+  constructor(message, results) {
+    super("config", message);
+    this.results = results;
+    this.name = "ModelsUnreachableError";
+  }
+  results;
+};
+function describeFailure(r) {
+  if (!r.listed) {
+    const probeNote = r.served === false ? ` (probe ${r.status}${r.detail ? `: ${r.detail}` : ""})` : "";
+    return `${r.model}: not in /models${probeNote}`;
+  }
+  return `${r.model}: listed but probe ${r.status}${r.detail ? ` \u2014 ${r.detail}` : ""}`;
+}
+async function assertModelsServed(opts) {
+  const outcome = await preflightModels(opts);
+  if (!outcome.succeeded || outcome.value === null) {
+    throw new ConfigError(
+      outcome.error ?? "assertModelsServed: preflight failed without an error message"
+    );
+  }
+  const dead = outcome.value.filter((r) => !r.listed || r.served === false);
+  if (dead.length > 0) {
+    throw new ModelsUnreachableError(
+      `assertModelsServed: ${dead.length}/${outcome.value.length} model(s) unreachable on the router \u2014 ${dead.map(describeFailure).join("; ")}`,
+      outcome.value
+    );
+  }
+  return outcome.value;
+}
+
 // src/integrity/single-backend.ts
 var SingleBackendError = class extends AgentEvalError {
   constructor(message, report) {
@@ -1760,7 +1864,7 @@ var SingleBackendError = class extends AgentEvalError {
   }
   report;
 };
-function stripSlash(url) {
+function stripSlash2(url) {
   return url.replace(/\/+$/, "");
 }
 function assertSingleBackend(agent, judge, opts = {}) {
@@ -1768,7 +1872,7 @@ function assertSingleBackend(agent, judge, opts = {}) {
   if (agent.kind !== judge.kind) {
     divergences.push({ field: "kind", agent: agent.kind, judge: judge.kind });
   }
-  if (stripSlash(agent.baseUrl) !== stripSlash(judge.baseUrl)) {
+  if (stripSlash2(agent.baseUrl) !== stripSlash2(judge.baseUrl)) {
     divergences.push({ field: "baseUrl", agent: agent.baseUrl, judge: judge.baseUrl });
   }
   if (agent.model !== judge.model) {
@@ -2290,16 +2394,16 @@ function aggregatePrReviewScore(dimensions, weights = {}) {
   return (merged.recall * clamp01(dimensions.recall) + merged.precision * clamp01(dimensions.precision) + merged.actionability * clamp01(dimensions.actionability) + merged.severityCalibration * clamp01(dimensions.severityCalibration) + merged.lowNoise * clamp01(dimensions.lowNoise)) / weightSum;
 }
 function matchReferenceFindings(references, comments) {
-  const matches = [];
+  const matches2 = [];
   const usedCommentIds = /* @__PURE__ */ new Set();
   for (const reference of references) {
     const candidates = comments.filter((comment) => !usedCommentIds.has(comment.id)).map((comment) => ({ comment, score: matchScore(reference, comment) })).filter(({ score }) => score >= 0.55).sort((a, b) => b.score - a.score);
     const best = candidates[0];
     if (!best) continue;
     usedCommentIds.add(best.comment.id);
-    matches.push({ referenceId: reference.id, commentId: best.comment.id, score: best.score });
+    matches2.push({ referenceId: reference.id, commentId: best.comment.id, score: best.score });
   }
-  return matches;
+  return matches2;
 }
 function matchScore(reference, comment) {
   let score = 0;
@@ -2326,9 +2430,9 @@ function isActionableComment(comment) {
     body
   );
 }
-function isSeverityAligned(comment, references, matches) {
+function isSeverityAligned(comment, references, matches2) {
   if (!comment.severity) return false;
-  const match = matches.find((candidate) => candidate.commentId === comment.id);
+  const match = matches2.find((candidate) => candidate.commentId === comment.id);
   if (!match) return comment.severity === "nit" || comment.severity === "low";
   const reference = references.find((candidate) => candidate.id === match.referenceId);
   if (!reference) return false;
@@ -2624,28 +2728,28 @@ function analyzeAntiSlop(outputs, config) {
       }
     }
     for (const re of config.hedgingPatterns) {
-      const matches = output.match(
+      const matches2 = output.match(
         new RegExp(re, re.flags.includes("g") ? re.flags : `${re.flags}g`)
       );
-      if (matches) {
-        counts.hedging += matches.length;
+      if (matches2) {
+        counts.hedging += matches2.length;
         issues.push({
           category: "hedging",
-          detail: `${matches.length}x ${re.source}`,
-          example: matches[0]
+          detail: `${matches2.length}x ${re.source}`,
+          example: matches2[0]
         });
       }
     }
     for (const re of config.apologyPatterns) {
-      const matches = output.match(
+      const matches2 = output.match(
         new RegExp(re, re.flags.includes("g") ? re.flags : `${re.flags}g`)
       );
-      if (matches) {
-        counts.apology += matches.length;
+      if (matches2) {
+        counts.apology += matches2.length;
         issues.push({
           category: "apology",
-          detail: `${matches.length}x ${re.source}`,
-          example: matches[0]
+          detail: `${matches2.length}x ${re.source}`,
+          example: matches2[0]
         });
       }
     }
@@ -2915,14 +3019,14 @@ async function runHarnessExperiment(config) {
   const score = config.score ?? ((trace) => critic.scoreTrace(trace));
   const results = await mapLimit(jobs, config.parallelism ?? 1, async (request) => {
     const trace = await config.adapter.run(request);
-    const runScore3 = await score(trace, request);
+    const runScore4 = await score(trace, request);
     const result = {
       variant: request.variant,
       scenario: request.scenario,
       trialIndex: request.trialIndex,
       trace,
-      score: runScore3,
-      aggregate: aggregateRunScore(runScore3, config.weights)
+      score: runScore4,
+      aggregate: aggregateRunScore(runScore4, config.weights)
     };
     await config.onResult?.(result);
     return result;
@@ -3568,13 +3672,118 @@ var BudgetGuard = class {
   }
 };
 
+// src/cost-ledger.ts
+function modelPriceKey(model) {
+  return isModelPriced(model) ? model : null;
+}
+function costForUsage(model, usage) {
+  assertNonNegative(usage.inputTokens, "inputTokens");
+  assertNonNegative(usage.outputTokens, "outputTokens");
+  if (usage.cachedTokens !== void 0) assertNonNegative(usage.cachedTokens, "cachedTokens");
+  const pricing = resolveModelPricing(model);
+  if (!pricing) return { costUsd: 0, costUnknown: true };
+  const billedInput = usage.inputTokens + (usage.cachedTokens ?? 0);
+  return { costUsd: estimateCost(billedInput, usage.outputTokens, model), costUnknown: false };
+}
+var CostLedger = class {
+  entries = [];
+  completedTasks = 0;
+  /**
+   * Record one LLM call. The cost is computed from pricing unless
+   * `actualCostUsd` is supplied (a finite observed cost from the provider
+   * response), in which case `costUnknown` is false regardless of pricing.
+   */
+  record(input) {
+    const { costUsd, costUnknown } = costForUsage(input.model, input.usage);
+    const hasActual = typeof input.actualCostUsd === "number" && Number.isFinite(input.actualCostUsd);
+    if (hasActual) assertNonNegative(input.actualCostUsd, "actualCostUsd");
+    const entry = {
+      model: input.model,
+      channel: input.channel,
+      inputTokens: input.usage.inputTokens,
+      outputTokens: input.usage.outputTokens,
+      cachedTokens: input.usage.cachedTokens,
+      costUsd: hasActual ? input.actualCostUsd : costUsd,
+      costUnknown: hasActual ? false : costUnknown,
+      actualCostUsd: hasActual ? input.actualCostUsd : void 0,
+      tags: input.tags,
+      timestamp: input.timestamp ?? Date.now()
+    };
+    this.entries.push(entry);
+    return entry;
+  }
+  /** Increment the completed-task counter (used for cost-per-completed-task). */
+  markCompleted(count = 1) {
+    if (!Number.isInteger(count) || count < 0) {
+      throw new ValidationError(
+        `CostLedger.markCompleted: count must be a non-negative integer, got ${count}`
+      );
+    }
+    this.completedTasks += count;
+  }
+  list() {
+    return [...this.entries];
+  }
+  summary() {
+    const byChannel = /* @__PURE__ */ new Map();
+    const unpriced = /* @__PURE__ */ new Set();
+    let totalCost = 0;
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let cachedTokens = 0;
+    for (const e of this.entries) {
+      totalCost += e.costUsd;
+      inputTokens += e.inputTokens;
+      outputTokens += e.outputTokens;
+      cachedTokens += e.cachedTokens ?? 0;
+      if (e.costUnknown) unpriced.add(e.model);
+      const roll = byChannel.get(e.channel) ?? {
+        channel: e.channel,
+        calls: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        cachedTokens: 0,
+        costUsd: 0,
+        unpricedCalls: 0
+      };
+      roll.calls += 1;
+      roll.inputTokens += e.inputTokens;
+      roll.outputTokens += e.outputTokens;
+      roll.cachedTokens += e.cachedTokens ?? 0;
+      roll.costUsd += e.costUsd;
+      if (e.costUnknown) roll.unpricedCalls += 1;
+      byChannel.set(e.channel, roll);
+    }
+    return {
+      totalCalls: this.entries.length,
+      inputTokens,
+      outputTokens,
+      cachedTokens,
+      totalCostUsd: totalCost,
+      byChannel: [...byChannel.values()].sort((a, b) => a.channel.localeCompare(b.channel)),
+      unpricedModels: [...unpriced].sort(),
+      fullyPriced: unpriced.size === 0
+    };
+  }
+  /** Total spend divided by completed tasks; null when nothing completed. */
+  costPerCompletedTask() {
+    if (this.completedTasks === 0) return null;
+    return this.summary().totalCostUsd / this.completedTasks;
+  }
+};
+function assertNonNegative(n, name) {
+  if (!Number.isFinite(n) || n < 0) {
+    throw new ValidationError(`CostLedger: ${name} must be a non-negative finite number, got ${n}`);
+  }
+}
+
 // src/cost-tracker.ts
 var CostTracker = class {
   byScenario = /* @__PURE__ */ new Map();
   record(entry) {
     const full = { timestamp: entry.timestamp ?? Date.now(), ...entry };
-    assertNonNegative(full.inputTokens, "inputTokens");
-    assertNonNegative(full.outputTokens, "outputTokens");
+    assertNonNegative2(full.inputTokens, "inputTokens");
+    assertNonNegative2(full.outputTokens, "outputTokens");
     let bucket = this.byScenario.get(full.scenarioId);
     if (!bucket) {
       bucket = {
@@ -3653,12 +3862,433 @@ function costFor(entry) {
   }
   return estimateCost(entry.inputTokens, entry.outputTokens, entry.model);
 }
-function assertNonNegative(n, name) {
+function assertNonNegative2(n, name) {
   if (!Number.isFinite(n) || n < 0) {
     throw new Error(`CostTracker: ${name} must be a non-negative finite number, got ${n}`);
   }
 }
 
+// src/eval-trace-store.ts
+function runScore(record) {
+  const { holdoutScore, searchScore } = record.outcome;
+  if (typeof holdoutScore === "number") return holdoutScore;
+  if (typeof searchScore === "number") return searchScore;
+  throw new ValidationError(
+    `EvalTraceStore: run ${record.runId} has neither holdoutScore nor searchScore`
+  );
+}
+function matches(record, f) {
+  if (f.experimentId && record.experimentId !== f.experimentId) return false;
+  if (f.candidateId && record.candidateId !== f.candidateId) return false;
+  if (f.scenarioId && record.scenarioId !== f.scenarioId) return false;
+  if (f.model && record.model !== f.model) return false;
+  if (f.splitTag && record.splitTag !== f.splitTag) return false;
+  if (f.minScore !== void 0 && runScore(record) < f.minScore) return false;
+  if (f.maxScore !== void 0 && runScore(record) > f.maxScore) return false;
+  if (f.rawEquals && record.outcome.raw[f.rawEquals.key] !== f.rawEquals.value) return false;
+  if (f.where && !f.where(record)) return false;
+  return true;
+}
+function inMemoryRunRecordBackend(initial = []) {
+  const rows = initial.map((r) => validateRunRecord(r));
+  return {
+    async append(record) {
+      rows.push(record);
+    },
+    async load() {
+      return [...rows];
+    }
+  };
+}
+function jsonlRunRecordBackend(path, opts = {}) {
+  return {
+    async append(record) {
+      const fs2 = await import("fs/promises");
+      const pathMod = await import("path");
+      await fs2.mkdir(pathMod.dirname(path), { recursive: true });
+      await fs2.appendFile(path, `${JSON.stringify(record)}
+`, "utf8");
+    },
+    async load() {
+      const fs2 = await import("fs/promises");
+      let raw;
+      try {
+        raw = await fs2.readFile(path, "utf8");
+      } catch (err) {
+        if (err.code === "ENOENT") return [];
+        throw err;
+      }
+      const out = [];
+      let lineNo = 0;
+      for (const line of raw.split("\n")) {
+        lineNo++;
+        if (!line.trim()) continue;
+        let parsed;
+        try {
+          parsed = JSON.parse(line);
+        } catch (err) {
+          if (opts.skipInvalid) continue;
+          throw new ValidationError(`EvalTraceStore: ${path}:${lineNo} is not valid JSON`);
+        }
+        if (opts.skipInvalid) {
+          if (isRunRecord(parsed)) out.push(parsed);
+          continue;
+        }
+        try {
+          out.push(validateRunRecord(parsed));
+        } catch (err) {
+          throw new ValidationError(
+            `EvalTraceStore: ${path}:${lineNo} is not a valid RunRecord \u2014 ${err.message}`
+          );
+        }
+      }
+      return out;
+    }
+  };
+}
+var EvalTraceStore = class {
+  backend;
+  constructor(backend = inMemoryRunRecordBackend()) {
+    this.backend = backend;
+  }
+  /** Validate and append one run. Throws on an invalid record — the corpus
+   *  stays paper-grade. */
+  async append(record) {
+    await this.backend.append(validateRunRecord(record));
+  }
+  async all() {
+    return this.backend.load();
+  }
+  async query(filter = {}) {
+    const rows = await this.backend.load();
+    return rows.filter((r) => matches(r, filter));
+  }
+  /**
+   * Highest-scoring run for a scenario (optionally restricted to a candidate).
+   * Returns null when no run matches. Ties resolve to the earliest-appended run
+   * so the result is stable.
+   */
+  async getBest(scenarioId, opts = {}) {
+    const rows = await this.query({
+      scenarioId,
+      candidateId: opts.candidateId,
+      splitTag: opts.splitTag
+    });
+    if (rows.length === 0) return null;
+    let best = rows[0];
+    let bestScore = runScore(best);
+    for (let i = 1; i < rows.length; i++) {
+      const s = runScore(rows[i]);
+      if (s > bestScore) {
+        best = rows[i];
+        bestScore = s;
+      }
+    }
+    return best;
+  }
+  /**
+   * Compare two candidates on the scenarios they BOTH ran. When a candidate
+   * ran a scenario more than once, its best `runScore` for that scenario is
+   * used. Throws when there is no paired scenario — an unpaired "comparison" is
+   * not one.
+   */
+  async compareRuns(candidateA, candidateB) {
+    if (candidateA === candidateB) {
+      throw new ValidationError(
+        `EvalTraceStore.compareRuns: candidates must differ ("${candidateA}")`
+      );
+    }
+    const rows = await this.backend.load();
+    const bestByScenario = (candidate) => {
+      const m = /* @__PURE__ */ new Map();
+      for (const r of rows) {
+        if (r.candidateId !== candidate) continue;
+        const sid = r.scenarioId;
+        if (!sid) continue;
+        const s = runScore(r);
+        const prev = m.get(sid);
+        if (prev === void 0 || s > prev) m.set(sid, s);
+      }
+      return m;
+    };
+    const aScores = bestByScenario(candidateA);
+    const bScores = bestByScenario(candidateB);
+    const paired = [...aScores.keys()].filter((sid) => bScores.has(sid)).sort();
+    if (paired.length === 0) {
+      throw new ValidationError(
+        `EvalTraceStore.compareRuns: "${candidateA}" and "${candidateB}" share no scenario (need scenarioId on records)`
+      );
+    }
+    let sumA = 0;
+    let sumB = 0;
+    let bWins = 0;
+    let aWins = 0;
+    let ties = 0;
+    for (const sid of paired) {
+      const sa = aScores.get(sid);
+      const sb = bScores.get(sid);
+      sumA += sa;
+      sumB += sb;
+      if (sb > sa) bWins++;
+      else if (sb < sa) aWins++;
+      else ties++;
+    }
+    const meanA = sumA / paired.length;
+    const meanB = sumB / paired.length;
+    return {
+      a: candidateA,
+      b: candidateB,
+      pairedScenarioIds: paired,
+      meanA,
+      meanB,
+      meanDelta: meanB - meanA,
+      bWins,
+      ties,
+      aWins
+    };
+  }
+};
+
+// src/experiment-tracker.ts
+import { execSync } from "child_process";
+var DEFAULTS = {
+  keepThreshold: 5,
+  regressionThreshold: 5,
+  iqrUnstableAbove: 10,
+  stddevUnstableAbove: Number.POSITIVE_INFINITY,
+  minRepsForVerdict: 3
+};
+function resolveThresholds(t) {
+  const r = { ...DEFAULTS, ...t ?? {} };
+  if (r.keepThreshold < 0) {
+    throw new ValidationError(
+      `experiment-tracker: keepThreshold must be >= 0, got ${r.keepThreshold}`
+    );
+  }
+  if (r.regressionThreshold < 0) {
+    throw new ValidationError(
+      `experiment-tracker: regressionThreshold must be >= 0, got ${r.regressionThreshold}`
+    );
+  }
+  if (r.minRepsForVerdict < 1) {
+    throw new ValidationError(
+      `experiment-tracker: minRepsForVerdict must be >= 1, got ${r.minRepsForVerdict}`
+    );
+  }
+  return r;
+}
+function median(sorted) {
+  const n = sorted.length;
+  if (n === 0) return 0;
+  const mid = Math.floor(n / 2);
+  return n % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+}
+function stddev(values, mean5) {
+  if (values.length < 2) return 0;
+  const variance = values.reduce((acc, v) => acc + (v - mean5) ** 2, 0) / values.length;
+  return Math.sqrt(variance);
+}
+function computeExperimentStats(reps, thresholds) {
+  const t = resolveThresholds(thresholds);
+  const n = reps.length;
+  if (n === 0) {
+    return {
+      median: 0,
+      mean: 0,
+      min: 0,
+      max: 0,
+      iqr: 0,
+      stddev: 0,
+      passRate: null,
+      n: 0,
+      stable: false
+    };
+  }
+  const scores2 = reps.map((r) => {
+    if (!Number.isFinite(r.score)) {
+      throw new ValidationError(`experiment-tracker: rep ${r.rep} has non-finite score ${r.score}`);
+    }
+    return r.score;
+  });
+  const sorted = [...scores2].sort((a, b) => a - b);
+  const mean5 = scores2.reduce((s, v) => s + v, 0) / n;
+  const sd = stddev(scores2, mean5);
+  const spread = iqr(scores2);
+  const rated = reps.filter((r) => typeof r.passed === "boolean");
+  const passRate = rated.length === 0 ? null : rated.filter((r) => r.passed).length / rated.length;
+  const stable = spread < t.iqrUnstableAbove && sd < t.stddevUnstableAbove;
+  return {
+    median: median(sorted),
+    mean: mean5,
+    min: sorted[0],
+    max: sorted[n - 1],
+    iqr: spread,
+    stddev: sd,
+    passRate,
+    n,
+    stable
+  };
+}
+function improvementVerdict(candidate, parent, thresholds) {
+  const t = resolveThresholds(thresholds);
+  if (!parent) {
+    return {
+      verdict: "ITERATE",
+      medianDelta: null,
+      reason: "no parent experiment to compare against"
+    };
+  }
+  if (candidate.n < t.minRepsForVerdict || parent.n < t.minRepsForVerdict) {
+    return {
+      verdict: "ITERATE",
+      medianDelta: null,
+      reason: `need >= ${t.minRepsForVerdict} reps on both sides (candidate n=${candidate.n}, parent n=${parent.n})`
+    };
+  }
+  if (!candidate.stable) {
+    return {
+      verdict: "NOISE",
+      medianDelta: candidate.median - parent.median,
+      reason: `candidate unstable (iqr=${candidate.iqr}, stddev=${candidate.stddev.toFixed(2)})`
+    };
+  }
+  const medianDelta2 = candidate.median - parent.median;
+  if (medianDelta2 > t.keepThreshold) {
+    return { verdict: "KEEP", medianDelta: medianDelta2, reason: `median +${medianDelta2} > +${t.keepThreshold}` };
+  }
+  if (medianDelta2 < -t.regressionThreshold) {
+    return {
+      verdict: "REGRESSION",
+      medianDelta: medianDelta2,
+      reason: `median ${medianDelta2} < -${t.regressionThreshold}`
+    };
+  }
+  return {
+    verdict: "NOISE",
+    medianDelta: medianDelta2,
+    reason: `median delta ${medianDelta2} inside noise band [-${t.regressionThreshold}, +${t.keepThreshold}]`
+  };
+}
+var gitProvenanceReader = () => {
+  const run = (cmd) => execSync(cmd, { encoding: "utf8" }).trim();
+  const commit = run("git rev-parse --short HEAD");
+  const message = run("git log -1 --format=%s");
+  const changedRaw = run("git diff --name-only HEAD~1");
+  const changedFiles = changedRaw.length === 0 ? [] : changedRaw.split("\n").filter(Boolean);
+  return { commit, message, changedFiles };
+};
+function inMemoryExperimentStore(initial = []) {
+  let state = initial.map((e) => structuredClone(e));
+  return {
+    async load() {
+      return state.map((e) => structuredClone(e));
+    },
+    async save(experiments) {
+      state = experiments.map((e) => structuredClone(e));
+    }
+  };
+}
+function fileExperimentStore(path) {
+  return {
+    async load() {
+      const fs2 = await import("fs/promises");
+      try {
+        const raw = await fs2.readFile(path, "utf8");
+        const parsed = JSON.parse(raw);
+        if (!Array.isArray(parsed)) {
+          throw new ValidationError(`experiment-tracker: store at ${path} is not a JSON array`);
+        }
+        return parsed;
+      } catch (err) {
+        if (err.code === "ENOENT") return [];
+        throw err;
+      }
+    },
+    async save(experiments) {
+      const fs2 = await import("fs/promises");
+      const pathMod = await import("path");
+      await fs2.mkdir(pathMod.dirname(path), { recursive: true });
+      await fs2.writeFile(path, JSON.stringify(experiments, null, 2), "utf8");
+    }
+  };
+}
+var ExperimentTracker = class {
+  store;
+  provenanceReader;
+  thresholds;
+  now;
+  constructor(options = {}) {
+    this.store = options.store ?? inMemoryExperimentStore();
+    this.provenanceReader = options.provenanceReader ?? gitProvenanceReader;
+    this.thresholds = resolveThresholds(options.thresholds);
+    this.now = options.now ?? Date.now;
+  }
+  async create(input) {
+    const experiments = await this.store.load();
+    if (experiments.some((e) => e.id === input.id)) {
+      throw new ValidationError(`experiment-tracker: experiment id "${input.id}" already exists`);
+    }
+    if (input.parentId && !experiments.some((e) => e.id === input.parentId)) {
+      throw new ValidationError(
+        `experiment-tracker: parent experiment "${input.parentId}" not found`
+      );
+    }
+    const provenance = input.provenance ?? await this.provenanceReader();
+    const experiment = {
+      id: input.id,
+      label: input.label,
+      provenance,
+      parentId: input.parentId,
+      changeSummary: input.changeSummary,
+      reps: [],
+      stats: computeExperimentStats([], this.thresholds),
+      verdict: "ITERATE",
+      createdAt: new Date(this.now()).toISOString()
+    };
+    experiments.push(experiment);
+    await this.store.save(experiments);
+    return structuredClone(experiment);
+  }
+  /** Append a rep (its `rep` index defaults to the current rep count) and
+   *  recompute stats + verdict. Returns the updated experiment. */
+  async addRep(experimentId, rep) {
+    const experiments = await this.store.load();
+    const exp = experiments.find((e) => e.id === experimentId);
+    if (!exp)
+      throw new ValidationError(`experiment-tracker: experiment "${experimentId}" not found`);
+    const fullRep = {
+      rep: rep.rep ?? exp.reps.length,
+      score: rep.score,
+      passed: rep.passed,
+      metrics: rep.metrics,
+      timestamp: rep.timestamp ?? new Date(this.now()).toISOString()
+    };
+    exp.reps.push(fullRep);
+    exp.stats = computeExperimentStats(exp.reps, this.thresholds);
+    const parent = exp.parentId ? experiments.find((e) => e.id === exp.parentId) : void 0;
+    exp.verdict = improvementVerdict(exp.stats, parent?.stats ?? null, this.thresholds).verdict;
+    await this.store.save(experiments);
+    return structuredClone(exp);
+  }
+  async get(experimentId) {
+    const experiments = await this.store.load();
+    const found = experiments.find((e) => e.id === experimentId);
+    return found ? structuredClone(found) : void 0;
+  }
+  async list() {
+    return this.store.load();
+  }
+  /** Full verdict (not just the enum) for an experiment vs its parent. */
+  async verdictFor(experimentId) {
+    const experiments = await this.store.load();
+    const exp = experiments.find((e) => e.id === experimentId);
+    if (!exp)
+      throw new ValidationError(`experiment-tracker: experiment "${experimentId}" not found`);
+    const parent = exp.parentId ? experiments.find((e) => e.id === exp.parentId) : void 0;
+    return improvementVerdict(exp.stats, parent?.stats ?? null, this.thresholds);
+  }
+};
+
 // src/muffled-gate-scanner.ts
 import { existsSync, readdirSync, readFileSync, statSync } from "fs";
 import { join } from "path";
@@ -3771,7 +4401,7 @@ var DEFAULT_FINDERS = [
 ];
 var UNIVERSAL_FINDERS = [findConstructorCwdDropped];
 function autoDeriveImporters(repoRoot, roots, extensions, importsContain) {
-  const matches = [];
+  const matches2 = [];
   const walk = (rel) => {
     const abs = join(repoRoot, rel);
     if (!existsSync(abs)) return;
@@ -3797,12 +4427,12 @@ function autoDeriveImporters(repoRoot, roots, extensions, importsContain) {
         } catch {
           continue;
         }
-        if (text.includes(importsContain)) matches.push(sub);
+        if (text.includes(importsContain)) matches2.push(sub);
       }
     }
   };
   for (const r of roots) walk(r);
-  return matches;
+  return matches2;
 }
 function scanForMuffledGates(opts) {
   const findings = [];
@@ -3974,16 +4604,80 @@ function isObject(v) {
   return typeof v === "object" && v !== null && !Array.isArray(v);
 }
 
+// src/partition-held-out.ts
+function fnv1a32(input) {
+  let h = 2166136261;
+  for (let i = 0; i < input.length; i++) {
+    h ^= input.charCodeAt(i) & 255;
+    h = Math.imul(h, 16777619);
+  }
+  return h >>> 0;
+}
+function hashToUnit(id, seed) {
+  return fnv1a32(`${id}
+${seed}`) / 4294967296;
+}
+function assignHeldOutTag(id, options = {}) {
+  const seed = options.seed ?? "held-out-v1";
+  const holdoutFraction = options.holdoutFraction ?? 0.5;
+  assertFraction(holdoutFraction);
+  return hashToUnit(id, seed) < holdoutFraction ? "holdout" : "search";
+}
+function partitionHeldOut(ids, options = {}) {
+  const seed = options.seed ?? "held-out-v1";
+  const holdoutFraction = options.holdoutFraction ?? 0.5;
+  const minHoldout = options.minHoldout ?? 1;
+  const minSearch = options.minSearch ?? 1;
+  assertFraction(holdoutFraction);
+  if (ids.length === 0) {
+    throw new ValidationError("partitionHeldOut: no ids supplied");
+  }
+  const seen = /* @__PURE__ */ new Set();
+  const search = [];
+  const holdout = [];
+  for (const id of ids) {
+    if (typeof id !== "string" || id.length === 0) {
+      throw new ValidationError(
+        `partitionHeldOut: ids must be non-empty strings, got ${JSON.stringify(id)}`
+      );
+    }
+    if (seen.has(id)) {
+      throw new ValidationError(
+        `partitionHeldOut: duplicate id "${id}" \u2014 a paired comparison must not double-count`
+      );
+    }
+    seen.add(id);
+    if (hashToUnit(id, seed) < holdoutFraction) holdout.push(id);
+    else search.push(id);
+  }
+  if (holdout.length < minHoldout) {
+    throw new ValidationError(
+      `partitionHeldOut: holdout set has ${holdout.length} id(s), below the floor of ${minHoldout} (n=${ids.length}, holdoutFraction=${holdoutFraction}) \u2014 too few for a significant comparison`
+    );
+  }
+  if (search.length < minSearch) {
+    throw new ValidationError(
+      `partitionHeldOut: search set has ${search.length} id(s), below the floor of ${minSearch} (n=${ids.length}, holdoutFraction=${holdoutFraction})`
+    );
+  }
+  return { search, holdout, seed, holdoutFraction };
+}
+function assertFraction(f) {
+  if (!Number.isFinite(f) || f <= 0 || f >= 1) {
+    throw new ValidationError(`partitionHeldOut: holdoutFraction must be in (0, 1), got ${f}`);
+  }
+}
+
 // src/scorecard.ts
 import { appendFileSync, existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2 } from "fs";
 import { dirname } from "path";
-function median(xs) {
+function median2(xs) {
   if (xs.length === 0) return 0;
   const sorted = [...xs].sort((a, b) => a - b);
   const mid = Math.floor(sorted.length / 2);
   return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
 }
-function runScore(run) {
+function runScore2(run) {
   return run.outcome.holdoutScore ?? run.outcome.searchScore;
 }
 function aggregatePerDimension(runs) {
@@ -4017,14 +4711,14 @@ function recordRuns(runs, opts) {
   }
   const lines = [];
   for (const [scenarioId, scenarioRuns] of byScenario) {
-    const scored = scenarioRuns.map((run) => ({ run, score: runScore(run) })).filter((s) => s.score !== void 0);
+    const scored = scenarioRuns.map((run) => ({ run, score: runScore2(run) })).filter((s) => s.score !== void 0);
     if (scored.length === 0) continue;
     const scores2 = scored.map((s) => s.score);
     const entry = {
       commitSha: opts.commitSha,
       timestamp,
       scores: scores2,
-      composite: median(scores2),
+      composite: median2(scores2),
       runIds: scored.map((s) => s.run.runId)
     };
     const perDimension = aggregatePerDimension(scenarioRuns);
@@ -5772,13 +6466,13 @@ function extractErrorCount(text, opts = {}) {
     (p) => !opts.only || opts.only.includes(p.name)
   );
   for (const p of patterns) {
-    const matches = Array.from(text.matchAll(p.regex));
-    if (matches.length === 0) continue;
-    const count = p.transform ? matches.reduce((sum3, m) => sum3 + p.transform(m), 0) : matches.length;
+    const matches2 = Array.from(text.matchAll(p.regex));
+    if (matches2.length === 0) continue;
+    const count = p.transform ? matches2.reduce((sum3, m) => sum3 + p.transform(m), 0) : matches2.length;
     return {
       count,
       matched: p.name,
-      samples: matches.slice(0, 5).map((m) => m[0])
+      samples: matches2.slice(0, 5).map((m) => m[0])
     };
   }
   return { count: null, matched: null, samples: [] };
@@ -6719,7 +7413,7 @@ function scoreScenario(scenario, matcher, threshold, matchStrategy) {
 }
 function scoreScenarioReferenceOrder(scenario, matcher, threshold) {
   const candidatesLeft = scenario.candidates.map((candidate, index) => ({ candidate, index }));
-  const matches = [];
+  const matches2 = [];
   for (const reference of scenario.references) {
     let best = null;
     for (const item of candidatesLeft) {
@@ -6732,7 +7426,7 @@ function scoreScenarioReferenceOrder(scenario, matcher, threshold) {
     if (best && best.score >= threshold) {
       const matchIndex = candidatesLeft.findIndex((item) => item.index === best.index);
       if (matchIndex >= 0) candidatesLeft.splice(matchIndex, 1);
-      matches.push({
+      matches2.push({
         scenarioId: scenario.id,
         referenceId: reference.id,
         candidateId: best.candidate.id,
@@ -6742,7 +7436,7 @@ function scoreScenarioReferenceOrder(scenario, matcher, threshold) {
         reason: best.reason
       });
     } else {
-      matches.push({
+      matches2.push({
         scenarioId: scenario.id,
         referenceId: reference.id,
         candidateId: best?.candidate.id ?? null,
@@ -6753,7 +7447,7 @@ function scoreScenarioReferenceOrder(scenario, matcher, threshold) {
       });
     }
   }
-  return buildScenarioScore(scenario, matches, candidatesLeft.length);
+  return buildScenarioScore(scenario, matches2, candidatesLeft.length);
 }
 function scoreScenarioGlobalGreedy(scenario, matcher, threshold) {
   const pairs = [];
@@ -6780,7 +7474,7 @@ function scoreScenarioGlobalGreedy(scenario, matcher, threshold) {
     selectedByReference.set(pair.referenceIndex, pair);
     selectedCandidates.add(pair.candidateIndex);
   }
-  const matches = scenario.references.map((reference, referenceIndex) => {
+  const matches2 = scenario.references.map((reference, referenceIndex) => {
     const weight = reference.weight ?? 1;
     const selected = selectedByReference.get(referenceIndex);
     if (selected) {
@@ -6805,7 +7499,7 @@ function scoreScenarioGlobalGreedy(scenario, matcher, threshold) {
       reason: bestRejected?.reason ?? "no candidates"
     };
   });
-  return buildScenarioScore(scenario, matches, scenario.candidates.length - selectedCandidates.size);
+  return buildScenarioScore(scenario, matches2, scenario.candidates.length - selectedCandidates.size);
 }
 function scorePair(scenario, matcher, reference, candidate) {
   const result = matcher(reference, candidate, scenario);
@@ -6816,11 +7510,11 @@ function scorePair(scenario, matcher, reference, candidate) {
   }
   return { score: clamp012(result.score), reason: result.reason ?? "" };
 }
-function buildScenarioScore(scenario, matches, falsePositives) {
-  const matched = matches.filter((match) => match.matched).length;
+function buildScenarioScore(scenario, matches2, falsePositives) {
+  const matched = matches2.filter((match) => match.matched).length;
   const total = scenario.references.length;
-  const matchedWeight = matches.filter((match) => match.matched).reduce((sum3, match) => sum3 + match.weight, 0);
-  const totalWeight = matches.reduce((sum3, match) => sum3 + match.weight, 0);
+  const matchedWeight = matches2.filter((match) => match.matched).reduce((sum3, match) => sum3 + match.weight, 0);
+  const totalWeight = matches2.reduce((sum3, match) => sum3 + match.weight, 0);
   const precision2 = ratio(matched, matched + falsePositives);
   const recall = ratio(matched, total);
   return {
@@ -6834,7 +7528,7 @@ function buildScenarioScore(scenario, matches, falsePositives) {
     precision: precision2,
     recall,
     f1: f1(precision2, recall),
-    matches
+    matches: matches2
   };
 }
 function aggregateBySplit(scores2) {
@@ -7095,7 +7789,7 @@ function createDefaultReviewer(options) {
 
 // src/description-length-gate.ts
 import { gzipSync } from "zlib";
-function runScore2(run) {
+function runScore3(run) {
   const o = run.outcome;
   const s = o.holdoutScore ?? o.searchScore ?? o.raw?.score;
   return typeof s === "number" && Number.isFinite(s) ? s : void 0;
@@ -7106,7 +7800,7 @@ function taskKey(run) {
 function perTaskMeanScore(runs) {
   const acc = /* @__PURE__ */ new Map();
   for (const run of runs) {
-    const s = runScore2(run);
+    const s = runScore3(run);
     if (s === void 0) continue;
     const key = taskKey(run);
     const cur = acc.get(key) ?? { sum: 0, n: 0 };
@@ -7258,10 +7952,10 @@ async function discoverPersonas(dir, opts = {}) {
 function matchGoldens(goldens, candidates, options = {}) {
   const extract = options.text ?? defaultExtract2;
   const haystacks = candidates.map((c) => extract(c).toLowerCase());
-  const matches = goldens.map((golden) => goldenMatched(golden, haystacks));
+  const matches2 = goldens.map((golden) => goldenMatched(golden, haystacks));
   return {
-    matches,
-    hits: matches.filter(Boolean).length,
+    matches: matches2,
+    hits: matches2.filter(Boolean).length,
     total: goldens.length
   };
 }
@@ -8416,6 +9110,7 @@ export {
   CaptureIntegrityError,
   ConfigError,
   ConvergenceTracker,
+  CostLedger,
   CostTracker,
   CrossFamilyError,
   DEFAULT_AGENT_SLOS,
@@ -8437,6 +9132,8 @@ export {
   DockerSandboxDriver,
   DualAgentBench,
   ERROR_COUNT_PATTERNS,
+  EvalTraceStore,
+  ExperimentTracker,
   FAILURE_CLASSES,
   FAILURE_MODE_KIND_SPEC,
   FileSystemFeedbackTrajectoryStore,
@@ -8462,6 +9159,7 @@ export {
   LockedJsonlAppender,
   MODEL_PRICING,
   MetricsCollector,
+  ModelsUnreachableError,
   MultiLayerVerifier,
   Mutex,
   NoopRawProviderSink,
@@ -8521,12 +9219,14 @@ export {
   asString,
   assertCrossFamily,
   assertLlmRoute,
+  assertModelsServed,
   assertRealBackend,
   assertReleaseConfidence,
   assertRunAgentProfileCell,
   assertRunCaptured,
   assertSingleBackend,
   assignFeedbackSplit,
+  assignHeldOutTag,
   attributeCounterfactuals,
   backoffMs,
   deterministicSplit as benchmarkDeterministicSplit,
@@ -8574,6 +9274,7 @@ export {
   compilerJudge,
   composeParsers,
   composeValidators,
+  computeExperimentStats,
   computeFindingId,
   computeToolUseMetrics,
   computeTraceMetrics,
@@ -8586,6 +9287,7 @@ export {
   convertTraceStoresToOtlp,
   corpusInterRaterAgreement,
   corpusInterRaterAgreementFromJudgeScores,
+  costForUsage,
   createAnalystAi,
   createAntiSlopJudge,
   createChatClient,
@@ -8641,6 +9343,9 @@ export {
   extractErrorCount,
   extractOtlpAttributes,
   extractProducedState,
+  extractUsage,
+  extractUsageFromResponse,
+  extractUsageFromSse,
   feedbackTrajectoriesToDatasetScenarios,
   feedbackTrajectoriesToOptimizerRows,
   feedbackTrajectoryToDatasetScenario,
@@ -8648,6 +9353,7 @@ export {
   fieldAgreement,
   fileContains,
   fileExists,
+  fileExperimentStore,
   findAutoMatchNoExpectation,
   findConstructorCwdDropped,
   findFallbackToPass,
@@ -8657,12 +9363,14 @@ export {
   firstStringAttr,
   flattenOtlpExportToNdjson,
   flowLayer,
+  fnv1a32,
   formatBenchmarkReport,
   formatDriverReport,
   formatFindings,
   formatScorecardDiff,
   gainHistogram,
   ghCliClient,
+  gitProvenanceReader,
   precision as goldenPrecision,
   gradeSemanticStatus,
   groupBy,
@@ -8670,10 +9378,14 @@ export {
   hashContent,
   hashJson,
   hashScenarios,
+  hashToUnit,
   htmlContainsElement,
   httpGithubClient,
+  improvementVerdict,
+  inMemoryExperimentStore,
   inMemoryReferenceReplayStore,
   inMemoryReviewStore,
+  inMemoryRunRecordBackend,
   inferDomainKeywords,
   inferOtlpKind,
   interRaterReliability,
@@ -8694,6 +9406,7 @@ export {
   jsonShape,
   jsonlReferenceReplayStore,
   jsonlReviewStore,
+  jsonlRunRecordBackend,
   judgeFamily,
   judgeReplayGate,
   judgeSpans,
@@ -8713,6 +9426,7 @@ export {
   mergeLayerResults,
   mergeSteeringBundle,
   modelDescriptionBits,
+  modelPriceKey,
   multiToolchainLayer,
   nistAiRmfReport,
   normalizeScores,
@@ -8737,11 +9451,13 @@ export {
   parseRunRecordSafe,
   parseRuntimeTrajectoryHookEvent,
   partialCredit,
+  partitionHeldOut,
   passOrthogonality,
   pixelDeltaRatio,
   planTraceInsightQuestions,
   politenessPrefixMutator,
   positionalBias,
+  preflightModels,
   printDriverSummary,
   probeLlm,
   profile_exports as profile,
@@ -8801,6 +9517,7 @@ export {
   runProposeReview,
   runProposeReviewAsControlLoop,
   runReferenceReplay,
+  runScore,
   runSelfPlay,
   runSemanticConceptJudge,
   runTestGradedScenario,