@hawkeyexl/manni 4.2.0 → 4.3.0

package/dist/chunk-CI2GOU3A.js

@@ -0,0 +1,4447 @@
+#!/usr/bin/env node
+import {
+  NOT_GRANTED_REASON,
+  applyAvailabilityRecord,
+  declaredAt,
+  discoverConfig,
+  extractEvals,
+  findManifest,
+  foldBranchAvailability,
+  grantsFor,
+  isExecutionGrant,
+  makeRedactor,
+  newAvailability,
+  newReplay,
+  remapAvailability,
+  resolveArtifacts,
+  unknownGrantsMessage
+} from "./chunk-MOQXWXSG.js";
+import {
+  DocmetaError,
+  PATH_JOIN,
+  assertKnownProvider,
+  assertLocalFlag,
+  assertModelHasProvider,
+  classifyRef,
+  escapeWorkflowCommandMessage,
+  escapeWorkflowCommandProperty,
+  externalMetadataJoin,
+  extractFrontmatter,
+  familyMarks,
+  hasPagePlaceholder,
+  loadExternalMetadata,
+  localNotice,
+  memberOf,
+  mergeWithMarks,
+  noticeOnce,
+  ownsKey,
+  palette,
+  providerSpecFor,
+  resolveIdentity,
+  selectProvider,
+  warn,
+  xmlEscape
+} from "./chunk-DP2ZA3WH.js";
+import {
+  TracevalsError,
+  discoverTraces
+} from "./chunk-3R23C5DT.js";
+import {
+  errorMessage
+} from "./chunk-IOYXPBCL.js";
+
+// src/tracevals/commands/batch.ts
+import { writeFile as writeFile2 } from "node:fs/promises";
+
+// src/docevals/judge/budget.ts
+var TURN_BUDGET_SKIP = "turn budget";
+function turnBudgetSkipReason(maxTurns) {
+  return `judge ${TURN_BUDGET_SKIP} exhausted (${maxTurns})`;
+}
+function isTurnBudgetSkip(reason) {
+  return reason !== void 0 && reason.includes(TURN_BUDGET_SKIP);
+}
+
+// src/tracevals/aggregate.ts
+var zero = () => ({
+  pass: 0,
+  fail: 0,
+  error: 0,
+  needsReview: 0,
+  skipped: 0
+});
+function tally(counts2, outcome) {
+  if (outcome === "needs-review") counts2.needsReview += 1;
+  else counts2[outcome] += 1;
+}
+var byKey = (a, b) => a.key < b.key ? -1 : a.key > b.key ? 1 : 0;
+function accumulate(into, key, result, file, withEvalName) {
+  let row = into.get(key);
+  if (row === void 0) {
+    row = {
+      key,
+      artifactName: result.artifactName,
+      artifactType: result.artifactType,
+      ...withEvalName ? { evalName: result.evalName } : {},
+      artifacts: /* @__PURE__ */ new Set(),
+      graders: /* @__PURE__ */ new Set(),
+      traces: /* @__PURE__ */ new Set(),
+      counts: zero(),
+      failingTraces: [],
+      reviewTraces: [],
+      skipReasons: /* @__PURE__ */ new Set()
+    };
+    into.set(key, row);
+  }
+  row.artifacts.add(result.artifact);
+  row.graders.add(result.grader);
+  row.traces.add(file);
+  tally(row.counts, result.outcome);
+  if (result.outcome === "fail" || result.outcome === "error") {
+    if (!row.failingTraces.includes(file)) row.failingTraces.push(file);
+  }
+  if (result.outcome === "needs-review" && !row.reviewTraces.includes(file)) {
+    row.reviewTraces.push(file);
+  }
+  if (result.outcome === "skipped" && result.skipReason !== void 0) {
+    row.skipReasons.add(result.skipReason);
+  }
+}
+function finish(row, withGraders) {
+  const c = row.counts;
+  const total = c.pass + c.fail + c.error + c.needsReview + c.skipped;
+  const graded = total - c.skipped;
+  return {
+    key: row.key,
+    artifactName: row.artifactName,
+    artifactType: row.artifactType,
+    artifacts: [...row.artifacts].sort(),
+    ...row.evalName !== void 0 ? { evalName: row.evalName } : {},
+    ...withGraders ? { graders: [...row.graders].sort() } : {},
+    traces: row.traces.size,
+    total,
+    counts: { ...c },
+    graded,
+    passRate: graded > 0 ? c.pass / graded : null,
+    failingTraces: [...row.failingTraces],
+    reviewTraces: [...row.reviewTraces],
+    skipReasons: [...row.skipReasons].sort()
+  };
+}
+var artifactKey = (r) => `${r.artifactType}:${r.artifactName}`;
+function aggregate(outcomes, options) {
+  const artifactRows = /* @__PURE__ */ new Map();
+  const evalRows = /* @__PURE__ */ new Map();
+  const traces = [];
+  let budgetSkips = 0;
+  const budgetTraces = /* @__PURE__ */ new Set();
+  let budgetReason = "";
+  const summary = {
+    total: 0,
+    pass: 0,
+    fail: 0,
+    error: 0,
+    needsReview: 0,
+    skipped: 0,
+    passRate: 1,
+    traces: outcomes.length,
+    tracesPassed: 0,
+    tracesFailed: 0,
+    tracesErrored: 0
+  };
+  let turns = 0;
+  let gradedWeight = 0;
+  let passedWeight = 0;
+  for (const outcome of outcomes) {
+    if ("error" in outcome) {
+      traces.push({
+        file: outcome.file,
+        error: outcome.error,
+        warnings: [],
+        exitCode: 1,
+        turns: 0,
+        durationMs: outcome.durationMs
+      });
+      summary.tracesErrored += 1;
+      continue;
+    }
+    const { report } = outcome;
+    traces.push({
+      file: outcome.file,
+      ...report.trace.sessionId !== void 0 ? { sessionId: report.trace.sessionId } : {},
+      summary: report.summary,
+      warnings: report.warnings,
+      exitCode: report.exitCode,
+      turns: report.turns,
+      durationMs: report.durationMs
+    });
+    if (report.exitCode === 1) summary.tracesFailed += 1;
+    else summary.tracesPassed += 1;
+    turns += report.turns;
+    summary.total += report.summary.total;
+    summary.pass += report.summary.pass;
+    summary.fail += report.summary.fail;
+    summary.error += report.summary.error;
+    summary.needsReview += report.summary.needsReview;
+    summary.skipped += report.summary.skipped;
+    for (const result of report.evalResults) {
+      if (result.outcome === "skipped" && isTurnBudgetSkip(result.skipReason)) {
+        budgetSkips += 1;
+        budgetTraces.add(outcome.file);
+        budgetReason = result.skipReason;
+      }
+      if (result.outcome === "pass" || result.outcome === "fail" || result.outcome === "error") {
+        const weight = result.weight ?? 1;
+        gradedWeight += weight;
+        if (result.outcome === "pass") passedWeight += weight;
+      }
+      const base = artifactKey(result);
+      accumulate(artifactRows, base, result, outcome.file, false);
+      accumulate(
+        evalRows,
+        `${base}::${result.evalName}`,
+        result,
+        outcome.file,
+        true
+      );
+    }
+  }
+  summary.passRate = gradedWeight === 0 ? 1 : passedWeight / gradedWeight;
+  const budget = budgetSkips > 0 ? {
+    skippedEvals: budgetSkips,
+    traces: budgetTraces.size,
+    reason: budgetReason
+  } : void 0;
+  const warnings = [...options.warnings ?? []];
+  if (budget !== void 0) {
+    warnings.push(
+      `${budget.reason}: ${budget.skippedEvals} eval(s) across ${budget.traces} of ${outcomes.length} trace(s) were never judged \u2014 raise judge.maxTurns or narrow the corpus`
+    );
+  }
+  return {
+    traces,
+    artifacts: [...artifactRows.values()].map((r) => finish(r, false)).sort(byKey),
+    evals: [...evalRows.values()].map((r) => finish(r, true)).sort(byKey),
+    summary,
+    warnings,
+    // A batch that ran out of money mid-corpus did not measure the corpus, and
+    // "did not measure" must never render as "measured clean". Exit 1 rather
+    // than 2: unlike an empty selector, the report is real and worth keeping —
+    // the traces that were judged carry genuine verdicts, and throwing them
+    // away to raise an operational error would cost more than it says.
+    exitCode: summary.tracesFailed > 0 || summary.tracesErrored > 0 || budget !== void 0 ? 1 : 0,
+    ...budget !== void 0 ? { budget } : {},
+    turns,
+    durationMs: options.durationMs
+  };
+}
+
+// src/shared/duration.ts
+var UNITS = {
+  m: 6e4,
+  h: 60 * 6e4,
+  d: 24 * 60 * 6e4,
+  w: 7 * 24 * 60 * 6e4
+};
+function durationMessage(value) {
+  return `--newer-than must be a duration such as 30m, 24h, 7d or 2w, got "${value}"`;
+}
+function parseDuration(text, makeError) {
+  const match = /^(\d+(?:\.\d+)?)([mhdw])$/.exec(text.trim());
+  const count = match?.[1];
+  const unit = match?.[2];
+  const ms = unit === void 0 ? void 0 : UNITS[unit];
+  if (count === void 0 || ms === void 0) {
+    throw makeError(durationMessage(text));
+  }
+  return Number(count) * ms;
+}
+
+// src/tracevals/reporters/batch.ts
+function budgetLine(report) {
+  const b = report.budget;
+  if (b === void 0) return void 0;
+  return `${b.skippedEvals} eval(s) across ${b.traces} trace(s) were never judged \u2014 ${b.reason}`;
+}
+var cell = (value) => value.replace(/\|/g, "\\|");
+function rate(row) {
+  return row.passRate === null ? "\u2014" : `${Math.round(row.passRate * 1e3) / 10}%`;
+}
+function detail(row) {
+  const parts = [];
+  const c = row.counts;
+  parts.push(
+    `${c.pass} pass, ${c.fail} fail, ${c.error} error, ${c.needsReview} review, ${c.skipped} skip`
+  );
+  return parts.join(" \xB7 ");
+}
+function names(files, limit = 3) {
+  const shown = files.slice(0, limit).map((f) => basename(f));
+  return files.length > limit ? `${shown.join(", ")} +${files.length - limit} more` : shown.join(", ");
+}
+function basename(file) {
+  const parts = file.split(/[\\/]/);
+  return parts[parts.length - 1] ?? file;
+}
+function outliers(row) {
+  const parts = [];
+  if (row.failingTraces.length > 0) {
+    parts.push(`fail: ${names(row.failingTraces)}`);
+  }
+  if (row.reviewTraces.length > 0) {
+    parts.push(`review: ${names(row.reviewTraces)}`);
+  }
+  return parts.join("; ");
+}
+function renderBatchPretty(report, opts = {}) {
+  const pc = palette(opts.color === true);
+  const lines = [];
+  const s = report.summary;
+  lines.push(pc.bold(`manni tracevals \u2014 ${s.traces} trace(s)`));
+  lines.push(
+    pc.dim(
+      `${s.tracesPassed} passed \xB7 ${s.tracesFailed} failed \xB7 ${s.tracesErrored} unreadable`
+    )
+  );
+  const cutShort = budgetLine(report);
+  if (cutShort !== void 0) {
+    lines.push(pc.red(`${pc.bold("BUDGET EXHAUSTED")}  ${cutShort}`));
+  }
+  lines.push("");
+  lines.push(pc.bold("Eval pass rates"));
+  if (report.evals.length === 0) {
+    lines.push(pc.dim("  no evals were planned for any trace"));
+  }
+  for (const row of report.evals) {
+    const label = row.passRate === null ? pc.dim(rate(row).padStart(6)) : row.passRate === 1 ? pc.green(rate(row).padStart(6)) : pc.red(rate(row).padStart(6));
+    lines.push(
+      `  ${label}  ${row.artifactName} \u203A ${row.evalName ?? ""} ${pc.dim(`(${row.traces} trace(s))`)}`
+    );
+    lines.push(`          ${pc.dim(detail(row))}`);
+    if (row.failingTraces.length > 0) {
+      lines.push(`          ${pc.red(`failing: ${names(row.failingTraces)}`)}`);
+    }
+    if (row.reviewTraces.length > 0) {
+      lines.push(
+        `          ${pc.yellow(`review: ${names(row.reviewTraces)}`)}`
+      );
+    }
+    for (const reason of row.skipReasons) {
+      lines.push(`          ${pc.dim(`skipped: ${reason}`)}`);
+    }
+  }
+  lines.push("");
+  lines.push(pc.bold("Artifact pass rates"));
+  for (const row of report.artifacts) {
+    lines.push(
+      `  ${rate(row).padStart(6)}  ${row.artifactType}: ${row.artifactName} ${pc.dim(detail(row))}`
+    );
+  }
+  lines.push("");
+  lines.push(pc.bold("Traces"));
+  for (const t of report.traces) {
+    if (t.error !== void 0) {
+      lines.push(`  ${pc.red("ERROR ")} ${basename(t.file)}`);
+      lines.push(`          ${pc.red(t.error)}`);
+      continue;
+    }
+    const mark = t.exitCode === 0 ? pc.green("PASS  ") : pc.red("FAIL  ");
+    const ts = t.summary;
+    lines.push(
+      `  ${mark} ${basename(t.file)} ${pc.dim(
+        `${ts?.pass ?? 0} pass, ${ts?.fail ?? 0} fail, ${ts?.error ?? 0} error, ${ts?.needsReview ?? 0} review, ${ts?.skipped ?? 0} skip`
+      )}`
+    );
+  }
+  if (report.warnings.length > 0) {
+    lines.push("");
+    lines.push(pc.bold("Warnings"));
+    for (const warning of report.warnings) {
+      lines.push(`  ${pc.yellow("!")} ${warning}`);
+    }
+  }
+  lines.push("");
+  lines.push(
+    pc.bold(
+      `${s.total} eval(s) across ${s.traces} trace(s): ${pc.green(`${s.pass} pass`)}, ${s.fail > 0 ? pc.red(`${s.fail} fail`) : `${s.fail} fail`}, ${s.error} error, ${s.needsReview} needs-review, ${s.skipped} skipped`
+    )
+  );
+  if (report.turns > 0) {
+    lines.push(pc.dim(`judge turns ${report.turns}`));
+  }
+  return lines.join("\n");
+}
+function renderBatchMarkdown(report) {
+  const lines = [];
+  const s = report.summary;
+  lines.push(`# manni tracevals batch report`);
+  lines.push("");
+  lines.push(`- **Traces**: ${s.traces}`);
+  lines.push(
+    `- **Outcome**: ${s.tracesPassed} passed, ${s.tracesFailed} failed, ${s.tracesErrored} unreadable`
+  );
+  const cutShort = budgetLine(report);
+  if (cutShort !== void 0) lines.push(`- **Budget**: ${cutShort}`);
+  lines.push("");
+  lines.push(`## Eval pass rates`);
+  lines.push("");
+  lines.push(`| Rate | Artifact | Eval | Grader | Traces | Outcomes | Outliers |`);
+  lines.push(`|---|---|---|---|---|---|---|`);
+  for (const row of report.evals) {
+    lines.push(
+      `| ${rate(row)} | ${cell(row.artifactName)} | ${cell(row.evalName ?? "")} | ${cell(
+        (row.graders ?? []).join(", ")
+      )} | ${row.traces} | ${cell(detail(row))} | ${cell(outliers(row))} |`
+    );
+  }
+  lines.push("");
+  lines.push(`## Artifact pass rates`);
+  lines.push("");
+  lines.push(`| Rate | Kind | Artifact | Traces | Outcomes |`);
+  lines.push(`|---|---|---|---|---|`);
+  for (const row of report.artifacts) {
+    lines.push(
+      `| ${rate(row)} | ${cell(row.artifactType)} | ${cell(row.artifactName)} | ${row.traces} | ${cell(detail(row))} |`
+    );
+  }
+  lines.push("");
+  lines.push(`## Traces`);
+  lines.push("");
+  lines.push(`| Result | Trace | Evals | Detail |`);
+  lines.push(`|---|---|---|---|`);
+  for (const t of report.traces) {
+    const result = t.error !== void 0 ? "error" : t.exitCode === 0 ? "pass" : "fail";
+    const evals = t.summary === void 0 ? "" : `${t.summary.pass}/${t.summary.total} pass`;
+    lines.push(
+      `| ${result} | \`${cell(t.file)}\` | ${evals} | ${cell(t.error ?? "")} |`
+    );
+  }
+  lines.push("");
+  if (report.warnings.length > 0) {
+    lines.push(`## Warnings`);
+    lines.push("");
+    for (const warning of report.warnings) lines.push(`- ${warning}`);
+    lines.push("");
+  }
+  lines.push(
+    `**${s.total} eval(s)** across ${s.traces} trace(s) \u2014 ${s.pass} pass, ${s.fail} fail, ${s.error} error, ${s.needsReview} needs-review, ${s.skipped} skipped.`
+  );
+  return lines.join("\n");
+}
+
+// src/tracevals/reporters/calibration.ts
+import { isAbsolute, relative } from "node:path";
+var cell2 = (value) => value.replace(/\|/g, "\\|");
+function basename2(file) {
+  const parts = file.split(/[\\/]/);
+  return parts[parts.length - 1] ?? file;
+}
+function display(file) {
+  const rel = relative(process.cwd(), file);
+  return rel === "" || rel.startsWith("..") || isAbsolute(rel) ? file : rel;
+}
+var pct = (value) => value === null ? "\u2014" : `${Math.round(value * 1e3) / 10}%`;
+var settingOf = (c) => `runs=${c.setting.ensembleRuns} autoPass=${c.setting.zones.autoPass} autoFail=${c.setting.zones.autoFail}`;
+var SWEEP_COLUMNS = [
+  { label: "scored", width: 6, of: (c) => c.scored },
+  { label: "agree", width: 5, of: (c) => c.agree },
+  { label: "false-pass", width: 10, of: (c) => c.falsePass },
+  { label: "false-fail", width: 10, of: (c) => c.falseFail },
+  { label: "review", width: 6, of: (c) => c.review },
+  { label: "reviewVol", width: 9, of: (c) => c.reviewVolume },
+  { label: "insuff", width: 6, of: (c) => c.insufficient }
+];
+var sweepRow = (axis, setting, cells) => `  ${axis.padEnd(20)} ${setting.padEnd(38)} ${cells.join("  ")}`;
+function evidence(d) {
+  if (d.consensus === void 0) return "";
+  const v = d.consensus.votes;
+  return `${v.pass}p/${v.fail}f/${v.partial}?/${v.error}e over ${d.consensus.runs} run(s), confidence ${d.consensus.meanConfidence.toFixed(2)}`;
+}
+var KIND_LABEL = {
+  agree: "AGREE",
+  "false-pass": "FALSE-PASS",
+  "false-fail": "FALSE-FAIL",
+  review: "REVIEW",
+  "missed-review": "MISSED-REVIEW",
+  error: "ERROR",
+  skipped: "SKIPPED"
+};
+function renderCalibrationPretty(report, opts = {}) {
+  const pc = palette(opts.color === true);
+  const lines = [];
+  const c = report.counts;
+  lines.push(
+    pc.bold(
+      `manni tracevals calibrate \u2014 ${report.corpus.length} trace(s), ${c.labels} label(s)`
+    )
+  );
+  lines.push(
+    pc.dim(
+      `runs ${report.setting.ensembleRuns} \xB7 autoPass ${report.setting.zones.autoPass} \xB7 autoFail ${report.setting.zones.autoFail} \xB7 labels ${display(report.labelsFile)}`
+    )
+  );
+  lines.push("");
+  lines.push(
+    pc.bold(
+      `Agreement ${c.agree}/${c.scored} (${pct(c.agreement)})`
+    )
+  );
+  const headline = (count, label, gloss, colour) => {
+    const text = `  ${String(count).padStart(4)}  ${label.padEnd(14)} ${pc.dim(gloss)}`;
+    lines.push(count > 0 ? colour(text) : pc.dim(text));
+  };
+  headline(c.falsePass, "false passes", "labelled fail, judged pass \u2014 the expensive error", pc.red);
+  headline(c.falseFail, "false fails", "labelled pass, judged fail \u2014 erodes trust fastest", pc.red);
+  headline(c.reviewVolume, "needs-review", "across the whole corpus \u2014 the review volume", pc.yellow);
+  headline(c.review, "deferred", "labelled pass or fail, routed to a human instead", pc.yellow);
+  headline(c.missedReview, "missed reviews", "labelled ambiguous, decided anyway", pc.yellow);
+  headline(c.errored, "errors", "the eval could not be graded at all", pc.red);
+  headline(c.skipped, "unscored", "labelled, but the eval never armed", (s) => s);
+  if (report.disagreements.length > 0) {
+    lines.push("");
+    lines.push(pc.bold("Disagreements"));
+    for (const d of report.disagreements) {
+      const tag = KIND_LABEL[d.kind];
+      const colour = d.kind === "false-pass" || d.kind === "false-fail" || d.kind === "error" ? pc.red : pc.yellow;
+      lines.push(
+        `  ${colour(tag.padEnd(14))} ${basename2(d.trace)}  ${d.artifactName} \u203A ${d.evalName} ${pc.dim(`(${d.grader})`)}`
+      );
+      lines.push(
+        `                 ${pc.dim(`expected ${d.expected}, got ${d.actual}`)}${evidence(d) === "" ? "" : pc.dim(` \xB7 ${evidence(d)}`)}`
+      );
+      if (d.note !== void 0) {
+        lines.push(`                 ${pc.dim(`note: ${d.note}`)}`);
+      }
+    }
+  }
+  if (report.unscored.length > 0) {
+    lines.push("");
+    lines.push(pc.bold("Unscored (no evidence either way)"));
+    for (const d of report.unscored) {
+      lines.push(
+        `  ${pc.dim("SKIPPED".padEnd(14))} ${basename2(d.trace)}  ${d.artifactName} \u203A ${d.evalName}`
+      );
+      if (d.skipReason !== void 0) {
+        lines.push(`                 ${pc.dim(d.skipReason)}`);
+      }
+    }
+  }
+  if (report.sweep !== void 0) {
+    lines.push("");
+    lines.push(pc.bold("Sweep"));
+    lines.push(
+      pc.dim(
+        "  re-scored from the verdicts this run already produced \u2014 no further model calls"
+      )
+    );
+    lines.push(
+      pc.dim(
+        sweepRow(
+          "axis",
+          "setting",
+          SWEEP_COLUMNS.map((c2) => c2.label.padStart(c2.width))
+        )
+      )
+    );
+    for (const s of report.sweep) {
+      lines.push(
+        sweepRow(
+          s.axis,
+          settingOf(s),
+          SWEEP_COLUMNS.map((c2) => String(c2.of(s.counts)).padStart(c2.width))
+        )
+      );
+    }
+  }
+  if (report.gates.length > 0) {
+    lines.push("");
+    lines.push(pc.bold("Thresholds"));
+    for (const g of report.gates) {
+      const text = `  ${g.name}: ${g.actual} of at most ${g.limit}`;
+      lines.push(g.exceeded ? pc.red(`${text} \u2014 exceeded`) : pc.green(text));
+    }
+  }
+  if (report.warnings.length > 0) {
+    lines.push("");
+    lines.push(pc.bold("Warnings"));
+    for (const w of report.warnings) lines.push(`  ${pc.yellow("!")} ${w}`);
+  }
+  lines.push("");
+  lines.push(pc.dim(verdictLine(report)));
+  if (report.turns > 0) {
+    lines.push(pc.dim(`judge turns ${report.turns}`));
+  }
+  return lines.join("\n");
+}
+function countsRow(counts2) {
+  return `${counts2.scored} | ${counts2.agree} | ${counts2.falsePass} | ${counts2.falseFail} | ${counts2.review} | ${counts2.missedReview} | ${counts2.reviewVolume} | ${counts2.insufficient}`;
+}
+function verdictLine(report) {
+  if (report.counts.scored === 0) {
+    return "Exit 1: nothing was scored \u2014 every label joined to a skipped result, so agreement and every threshold above rest on an empty denominator.";
+  }
+  return report.exitCode === 0 ? "Measured cleanly. Disagreement is the finding, not a failure \u2014 exit 0." : "Exit 1: a threshold was exceeded, or the measurement is incomplete \u2014 a trace could not be evaluated, or the judge budget cut the corpus short.";
+}
+function renderCalibrationMarkdown(report) {
+  const lines = [];
+  const c = report.counts;
+  lines.push("# manni tracevals calibration report");
+  lines.push("");
+  lines.push(`- **Corpus**: ${report.corpus.length} trace(s), ${c.labels} label(s)`);
+  lines.push(
+    `- **Setting**: \`ensembleRuns\` ${report.setting.ensembleRuns}, \`zones.autoPass\` ${report.setting.zones.autoPass}, \`zones.autoFail\` ${report.setting.zones.autoFail}`
+  );
+  lines.push(`- **Agreement**: ${c.agree}/${c.scored} (${pct(c.agreement)})`);
+  lines.push(`- **Labels**: \`${cell2(display(report.labelsFile))}\``);
+  lines.push("");
+  lines.push("| Measure | Count | What it means |");
+  lines.push("|---|---|---|");
+  lines.push(`| False passes | ${c.falsePass} | Labelled fail, judged pass \u2014 the expensive error. |`);
+  lines.push(`| False fails | ${c.falseFail} | Labelled pass, judged fail \u2014 erodes trust fastest. |`);
+  lines.push(`| Review volume | ${c.reviewVolume} | \`needs-review\` across the whole corpus. |`);
+  lines.push(`| Deferred labels | ${c.review} | Labelled pass or fail, routed to a human instead. |`);
+  lines.push(`| Missed reviews | ${c.missedReview} | Labelled ambiguous, decided anyway. |`);
+  lines.push(`| Errors | ${c.errored} | The eval could not be graded at all. |`);
+  lines.push(`| Unscored | ${c.skipped} | Labelled, but the eval never armed. |`);
+  lines.push("");
+  lines.push("## Disagreements");
+  lines.push("");
+  if (report.disagreements.length === 0) {
+    lines.push("None \u2014 every scored label agreed.");
+  } else {
+    lines.push("| Kind | Trace | Artifact | Eval | Grader | Expected | Got | Evidence | Note |");
+    lines.push("|---|---|---|---|---|---|---|---|---|");
+    for (const d of report.disagreements) {
+      lines.push(
+        `| ${KIND_LABEL[d.kind]} | \`${cell2(basename2(d.trace))}\` | ${cell2(d.artifactName)} | ${cell2(
+          d.evalName
+        )} | ${cell2(d.grader)} | ${d.expected} | ${d.actual} | ${cell2(evidence(d))} | ${cell2(d.note ?? "")} |`
+      );
+    }
+  }
+  lines.push("");
+  if (report.unscored.length > 0) {
+    lines.push("## Unscored");
+    lines.push("");
+    lines.push("| Trace | Artifact | Eval | Reason |");
+    lines.push("|---|---|---|---|");
+    for (const d of report.unscored) {
+      lines.push(
+        `| \`${cell2(basename2(d.trace))}\` | ${cell2(d.artifactName)} | ${cell2(d.evalName)} | ${cell2(
+          d.skipReason ?? ""
+        )} |`
+      );
+    }
+    lines.push("");
+  }
+  if (report.sweep !== void 0) {
+    lines.push("## Sweep");
+    lines.push("");
+    lines.push(
+      "Re-scored from the verdicts this run already produced \u2014 no further model calls."
+    );
+    lines.push("");
+    lines.push(
+      "| Axis | Setting | Scored | Agree | False passes | False fails | Deferred | Missed reviews | Review volume | Insufficient |"
+    );
+    lines.push("|---|---|---|---|---|---|---|---|---|---|");
+    for (const s of report.sweep) {
+      lines.push(`| ${s.axis} | ${cell2(settingOf(s))} | ${countsRow(s.counts)} |`);
+    }
+    lines.push("");
+  }
+  if (report.gates.length > 0) {
+    lines.push("## Thresholds");
+    lines.push("");
+    lines.push("| Threshold | Limit | Actual | Result |");
+    lines.push("|---|---|---|---|");
+    for (const g of report.gates) {
+      lines.push(
+        `| ${g.name} | ${g.limit} | ${g.actual} | ${g.exceeded ? "exceeded" : "ok"} |`
+      );
+    }
+    lines.push("");
+  }
+  if (report.warnings.length > 0) {
+    lines.push("## Warnings");
+    lines.push("");
+    for (const w of report.warnings) lines.push(`- ${w}`);
+    lines.push("");
+  }
+  lines.push(
+    report.exitCode === 0 ? verdictLine(report) : `**${verdictLine(report)}**`
+  );
+  return lines.join("\n");
+}
+
+// src/shared/sarif-location.ts
+var URI_BASE_ID = "SRCROOT";
+var DRIVE = /^[A-Za-z]:\//;
+function toPosix(p) {
+  return p.replace(/\\/g, "/");
+}
+function isAbsolutePath(posix) {
+  return posix.startsWith("/") || DRIVE.test(posix);
+}
+function isWindowsPath(posix) {
+  return DRIVE.test(posix) || posix.startsWith("//");
+}
+function underRoot(posix, root) {
+  if (posix.startsWith(root)) return posix.slice(root.length);
+  if (isWindowsPath(posix) && isWindowsPath(root)) {
+    if (posix.toLowerCase().startsWith(root.toLowerCase())) {
+      return posix.slice(root.length);
+    }
+  }
+  return null;
+}
+function encodePath(p) {
+  return encodeURI(p).replace(/#/g, "%23").replace(/\?/g, "%3F");
+}
+function fileUri(posix) {
+  const encoded = encodePath(posix);
+  if (posix.startsWith("//")) return `file:${encoded}`;
+  if (posix.startsWith("/")) return `file://${encoded}`;
+  return `file:///${encoded}`;
+}
+function escapesRoot(posix) {
+  return posix === ".." || posix.startsWith("../");
+}
+function resolveFromRoot(root, rel) {
+  const parts = root.replace(/\/+$/, "").split("/");
+  for (const segment of rel.split("/")) {
+    if (segment === "" || segment === ".") continue;
+    if (segment === "..") {
+      if (parts.length > 1) parts.pop();
+      continue;
+    }
+    parts.push(segment);
+  }
+  const joined = parts.join("/");
+  return joined === "" ? "/" : joined;
+}
+function normalizeRoot(root) {
+  const posix = toPosix(root);
+  return posix.endsWith("/") ? posix : `${posix}/`;
+}
+function artifactLocation(file, root) {
+  const posix = toPosix(file);
+  if (!isAbsolutePath(posix)) {
+    if (escapesRoot(posix)) return { uri: fileUri(resolveFromRoot(root, posix)) };
+    return { uri: encodePath(posix), uriBaseId: URI_BASE_ID };
+  }
+  const rel = underRoot(posix, root);
+  if (rel !== null) return { uri: encodePath(rel), uriBaseId: URI_BASE_ID };
+  return { uri: fileUri(posix) };
+}
+
+// src/tracevals/reporters/ci.ts
+function ciInputFromRun(report, opts = {}) {
+  return {
+    runs: [report],
+    traceErrors: [],
+    warnings: [],
+    root: normalizeRoot(opts.root ?? process.cwd())
+  };
+}
+function ciInputFromBatch(batch, runs, opts = {}) {
+  const traceErrors = [];
+  for (const t of batch.traces) {
+    if (t.error !== void 0) {
+      traceErrors.push({ file: t.file, error: t.error, durationMs: t.durationMs });
+    }
+  }
+  return {
+    runs,
+    traceErrors,
+    warnings: batch.warnings,
+    root: normalizeRoot(opts.root ?? process.cwd())
+  };
+}
+function displayPath(file, root) {
+  const posix = toPosix(file);
+  return underRoot(posix, root) ?? posix;
+}
+var ABSOLUTE = /^(?:[A-Za-z]:)?\//;
+function locationPath(location, root) {
+  if (classifyRef(location.file).kind === "url") return location.file;
+  const posix = toPosix(location.file);
+  const abs = ABSOLUTE.test(posix) ? posix : `${normalizeRoot(process.cwd())}${posix}`;
+  return underRoot(abs, root) ?? abs;
+}
+function locationLabel(location) {
+  return location.line === void 0 ? location.file : `${location.file}:${String(location.line)}`;
+}
+function ruleIdFor(grader) {
+  return `tracevals/${grader}`;
+}
+var TRACE_RULE_ID = "tracevals/trace";
+function traceErrorText(error) {
+  return `trace could not be evaluated: ${error}`;
+}
+function judgeFailText(result) {
+  if (result.outcome !== "fail" || !result.consensus) return void 0;
+  const reasoning = result.consensus.runs.find((run) => run.verdict)?.verdict?.reasoning ?? "";
+  return `AI judge: fail (confidence ${result.consensus.meanConfidence.toFixed(2)}). ${reasoning}`.trim();
+}
+function errorText(result) {
+  return result.error ?? result.skipReason ?? "eval errored";
+}
+
+// src/tracevals/reporters/github.ts
+var TITLE = "manni tracevals";
+function annotation(level, message, props) {
+  const parts = [
+    props.file !== void 0 ? `file=${escapeWorkflowCommandProperty(props.file)}` : void 0,
+    props.line !== void 0 ? `line=${String(props.line)}` : void 0,
+    `title=${escapeWorkflowCommandProperty(props.title)}`
+  ].filter((p) => p !== void 0);
+  return `::${level} ${parts.join(",")}::${escapeWorkflowCommandMessage(message)}`;
+}
+function renderGithub(input, summary) {
+  const lines = [];
+  for (const run of input.runs) {
+    const trace = ` (trace: ${displayPath(run.trace.file, input.root)})`;
+    for (const r of run.evalResults) {
+      const at = {
+        file: locationPath(r.location, input.root),
+        ...r.location.line === void 0 ? {} : { line: r.location.line },
+        title: `${TITLE}: ${r.evalName}`
+      };
+      for (const f of r.findings ?? []) {
+        lines.push(annotation(f.severity, f.message + trace, at));
+      }
+      const judged = judgeFailText(r);
+      if (judged !== void 0) {
+        lines.push(annotation("error", judged + trace, at));
+      }
+      if (r.outcome === "error") {
+        lines.push(annotation("error", errorText(r) + trace, at));
+      }
+    }
+    for (const w of run.warnings) {
+      if (input.warnings.includes(w)) continue;
+      lines.push(annotation("warning", w + trace, { title: TITLE }));
+    }
+  }
+  for (const t of input.traceErrors) {
+    lines.push(
+      annotation("error", traceErrorText(t.error), {
+        file: displayPath(t.file, input.root),
+        title: TITLE
+      })
+    );
+  }
+  for (const w of input.warnings) {
+    lines.push(annotation("warning", w, { title: TITLE }));
+  }
+  lines.push("", summary);
+  return lines.join("\n");
+}
+
+// src/tracevals/reporters/junit.ts
+function attr(name, value) {
+  return `${name}="${xmlEscape(String(value))}"`;
+}
+var seconds = (ms) => (ms / 1e3).toFixed(3);
+function failureText(result) {
+  const findings = (result.findings ?? []).map((f) => f.message);
+  if (findings.length > 0) return findings.join("\n");
+  return judgeFailText(result) ?? `${result.evalName} failed`;
+}
+function testcase(result, root) {
+  const line = result.location.line;
+  const open = `    <testcase ${attr("classname", displayPath(result.artifact, root))} ${attr(
+    "name",
+    result.evalName
+  )} ${attr("file", locationPath(result.location, root))}${line === void 0 ? "" : ` ${attr("line", line)}`} ${attr("time", seconds(result.durationMs))}`;
+  const type = attr("type", ruleIdFor(result.grader));
+  switch (result.outcome) {
+    case "pass":
+      return `${open} />`;
+    case "skipped":
+      return `${open}>
+      <skipped ${attr(
+        "message",
+        result.skipReason ?? "skipped"
+      )} />
+    </testcase>`;
+    case "error":
+      return `${open}>
+      <error ${attr(
+        "message",
+        `${result.evalName} errored`
+      )} ${type}>${xmlEscape(errorText(result))}</error>
+    </testcase>`;
+    case "needs-review":
+      return `${open}>
+      <skipped ${attr(
+        "message",
+        "needs human review"
+      )} />
+    </testcase>`;
+    case "fail":
+      return `${open}>
+      <failure ${attr(
+        "message",
+        `${result.evalName} failed`
+      )} ${type}>${xmlEscape(failureText(result))}</failure>
+    </testcase>`;
+  }
+}
+function counts(run) {
+  const r = run.evalResults;
+  return {
+    tests: r.length,
+    failures: r.filter((x) => x.outcome === "fail").length,
+    errors: r.filter((x) => x.outcome === "error").length,
+    skipped: r.filter((x) => x.outcome === "skipped" || x.outcome === "needs-review").length
+  };
+}
+function suiteOpen(name, c, ms) {
+  return `  <testsuite ${attr("name", name)} ${attr("tests", c.tests)} ${attr(
+    "failures",
+    c.failures
+  )} ${attr("errors", c.errors)} ${attr("skipped", c.skipped)} ${attr(
+    "time",
+    seconds(ms)
+  )}>`;
+}
+function renderJunit(input) {
+  const total = { tests: 0, failures: 0, errors: 0, skipped: 0 };
+  const suites = [];
+  for (const run of input.runs) {
+    const c = counts(run);
+    total.tests += c.tests;
+    total.failures += c.failures;
+    total.errors += c.errors;
+    suites.push(
+      [
+        suiteOpen(displayPath(run.trace.file, input.root), c, run.durationMs),
+        ...run.evalResults.map((r) => testcase(r, input.root)),
+        "  </testsuite>"
+      ].join("\n")
+    );
+  }
+  for (const t of input.traceErrors) {
+    total.tests += 1;
+    total.errors += 1;
+    const name = displayPath(t.file, input.root);
+    suites.push(
+      [
+        suiteOpen(name, { tests: 1, failures: 0, errors: 1, skipped: 0 }, t.durationMs),
+        `    <testcase ${attr("classname", name)} ${attr("name", "trace")} ${attr(
+          "time",
+          seconds(t.durationMs)
+        )}>`,
+        `      <error ${attr("message", "trace could not be evaluated")} ${attr(
+          "type",
+          TRACE_RULE_ID
+        )}>${xmlEscape(t.error)}</error>`,
+        "    </testcase>",
+        "  </testsuite>"
+      ].join("\n")
+    );
+  }
+  return [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    `<testsuites ${attr("name", "manni-tracevals")} ${attr("tests", total.tests)} ${attr(
+      "failures",
+      total.failures
+    )} ${attr("errors", total.errors)}>`,
+    ...suites,
+    "</testsuites>",
+    ""
+  ].join("\n");
+}
+
+// src/tracevals/reporters/coverage.ts
+function coverageLocation(entry) {
+  if (entry.resolved) return entry.path ?? entry.note ?? "";
+  if (entry.availability === "offered-not-used") {
+    return entry.note ?? "offered, never used";
+  }
+  return entry.note ?? `not found (${entry.tried.length} location(s) tried)`;
+}
+function availabilityTag(entry) {
+  if (entry.availability === "not-offered") return "not offered";
+  if (entry.availability === "offered-not-used") return "offered, unused";
+  return void 0;
+}
+function availabilityLines(report) {
+  if (!report.recorded) {
+    return [
+      "no availability roster in this trace, so what the session was offered is unknown"
+    ];
+  }
+  const lines = [
+    `${report.skills.offered} skill(s) offered, ${report.skills.used} used, ${report.skills.unused} never used`,
+    `${report.agents.offered} agent(s) offered, ${report.agents.used} used, ${report.agents.unused} never used`
+  ];
+  if (!report.listed && report.skills.unused + report.agents.unused > 0) {
+    lines.push("pass --report-unused-artifacts to list them");
+  }
+  return lines;
+}
+function coverageStaleness(entry) {
+  if (entry.stale !== true) return "";
+  if (entry.contentCheck?.status === "mismatch") {
+    return "changed since the session started (session manifest sha256)";
+  }
+  return entry.modifiedAt !== void 0 ? `modified after the session ended (${entry.modifiedAt})` : "modified after the session ended";
+}
+function manifestLines(report) {
+  const lines = [
+    `${report.path} captured ${report.capturedAt}${report.gitSha !== void 0 ? ` at ${report.gitSha.slice(0, 12)}` : ""}`,
+    `${report.matched} artifact(s) unchanged, ${report.changed} changed, ${report.unrecorded} not recorded`
+  ];
+  if (report.unrecorded > 0) {
+    lines.push(
+      "artifacts the manifest did not record keep the mtime heuristic"
+    );
+  }
+  return lines;
+}
+
+// src/tracevals/reporters/pretty.ts
+var OUTCOME_LABEL = {
+  pass: "PASS",
+  fail: "FAIL",
+  error: "ERROR",
+  "needs-review": "REVIEW",
+  skipped: "SKIP"
+};
+function paintFor(pc, outcome) {
+  switch (outcome) {
+    case "pass":
+      return pc.green;
+    case "fail":
+    case "error":
+      return pc.red;
+    case "needs-review":
+      return pc.yellow;
+    default:
+      return pc.dim;
+  }
+}
+function coverageMark(pc, entry) {
+  if (entry.resolved) return pc.green("\u2713");
+  if (entry.availability === "offered-not-used") return pc.dim("\xB7");
+  return pc.yellow("\u25CB");
+}
+function renderPretty(report, opts = {}) {
+  const pc = palette(opts.color === true);
+  const lines = [];
+  const t = report.trace;
+  lines.push(pc.bold(`manni tracevals \u2014 ${t.file}`));
+  lines.push(
+    pc.dim(
+      `${t.source} \xB7 ${t.model ?? "unknown model"} \xB7 ${t.turnCount} turn(s) \xB7 ${t.cwd}`
+    )
+  );
+  lines.push("");
+  for (const result of report.evalResults) {
+    const paint = paintFor(pc, result.outcome);
+    const label = paint(OUTCOME_LABEL[result.outcome].padEnd(6));
+    const name = `${result.artifactName} \u203A ${result.evalName}`;
+    lines.push(
+      `  ${label} ${name}${result.implicit ? pc.dim(" (implicit)") : ""}  ${pc.dim(locationLabel(result.location))}`
+    );
+    for (const finding2 of result.findings ?? []) {
+      if (result.outcome !== "pass" || finding2.severity !== "error") {
+        lines.push(`         ${pc.dim(`[${finding2.severity}]`)} ${finding2.message}`);
+      }
+    }
+    if (result.error) lines.push(`         ${pc.red(result.error)}`);
+    if (result.skipReason) lines.push(`         ${pc.dim(result.skipReason)}`);
+    if (result.consensus && result.outcome !== "pass") {
+      const v = result.consensus.votes;
+      lines.push(
+        `         ${pc.dim(`votes pass:${v.pass} fail:${v.fail} partial:${v.partial} error:${v.error} \xB7 confidence ${result.consensus.meanConfidence.toFixed(2)}`)}`
+      );
+      const observed = result.consensus.runs.find((r) => r.verdict)?.verdict?.observed;
+      if (observed) lines.push(`         ${pc.dim(observed)}`);
+    }
+  }
+  lines.push("");
+  lines.push(pc.bold("Artifact coverage"));
+  for (const entry of report.coverage) {
+    const tag = availabilityTag(entry);
+    const location = coverageLocation(entry);
+    const stale = coverageStaleness(entry);
+    lines.push(
+      [
+        `  ${coverageMark(pc, entry)} ${entry.kind}: ${entry.ref}`,
+        tag === void 0 ? "" : pc.yellow(`[${tag}]`),
+        location ? pc.dim(location) : "",
+        stale ? pc.yellow(`\u26A0 ${stale}`) : ""
+      ].filter((part) => part !== "").join(" ")
+    );
+  }
+  if (report.manifest !== void 0) {
+    lines.push("");
+    lines.push(pc.bold("Session manifest"));
+    for (const line of manifestLines(report.manifest)) {
+      lines.push(`  ${pc.dim(line)}`);
+    }
+  }
+  lines.push("");
+  lines.push(pc.bold("Availability"));
+  for (const line of availabilityLines(report.availability)) {
+    lines.push(`  ${pc.dim(line)}`);
+  }
+  if (report.warnings.length > 0) {
+    lines.push("");
+    lines.push(pc.bold("Warnings"));
+    for (const warning of report.warnings) {
+      lines.push(`  ${pc.yellow("!")} ${warning}`);
+    }
+  }
+  const s = report.summary;
+  lines.push("");
+  lines.push(
+    pc.bold(
+      `${s.total} eval(s): ${pc.green(`${s.pass} pass`)}, ${s.fail > 0 ? pc.red(`${s.fail} fail`) : `${s.fail} fail`}, ${s.error} error, ${s.needsReview} needs-review, ${s.skipped} skipped`
+    )
+  );
+  const weighted = report.evalResults.some(
+    (r) => r.weight !== void 0 && r.weight !== 1
+  );
+  if (weighted) {
+    lines.push(
+      pc.dim(`weighted pass rate ${(s.passRate * 100).toFixed(0)}%`)
+    );
+  }
+  if (report.turns > 0) {
+    lines.push(pc.dim(`judge turns ${report.turns}`));
+  }
+  return lines.join("\n");
+}
+
+// src/tracevals/reporters/markdown.ts
+var cell3 = (value) => value.replace(/\|/g, "\\|");
+function renderMarkdown(report) {
+  const lines = [];
+  const t = report.trace;
+  lines.push(`# manni tracevals report`);
+  lines.push("");
+  lines.push(`- **Trace**: \`${t.file}\``);
+  lines.push(`- **Source**: ${t.source} (${t.model ?? "unknown model"})`);
+  lines.push(`- **Project**: \`${t.cwd}\``);
+  lines.push(`- **Turns**: ${t.turnCount}`);
+  lines.push("");
+  lines.push(`## Evals`);
+  lines.push("");
+  lines.push(`| Outcome | Artifact | Eval | Grader | Detail |`);
+  lines.push(`|---|---|---|---|---|`);
+  for (const r of report.evalResults) {
+    const detail2 = r.error ?? r.skipReason ?? r.findings?.map((f) => f.message).join("; ") ?? (r.consensus ? `votes p:${r.consensus.votes.pass} f:${r.consensus.votes.fail} e:${r.consensus.votes.error}` : "");
+    lines.push(
+      `| ${r.outcome} | ${cell3(r.artifactName)} | ${cell3(r.evalName)}${r.implicit ? " (implicit)" : ""} \`${cell3(locationLabel(r.location))}\` | ${cell3(r.grader)} | ${cell3(detail2)} |`
+    );
+  }
+  lines.push("");
+  lines.push(`## Artifact coverage`);
+  lines.push("");
+  lines.push(`| Resolved | Kind | Ref | Availability | Where |`);
+  lines.push(`|---|---|---|---|---|`);
+  for (const entry of report.coverage) {
+    const location = cell3(coverageLocation(entry));
+    const where = entry.resolved && entry.path !== void 0 ? `\`${location}\`` : location;
+    const resolved = entry.availability === "offered-not-used" ? "n/a" : entry.resolved ? "yes" : "no";
+    const stale = coverageStaleness(entry);
+    const cellText = stale ? `${where}${where ? " \u2014 " : ""}\u26A0 ${cell3(stale)}` : where;
+    lines.push(
+      `| ${resolved} | ${cell3(entry.kind)} | ${cell3(entry.ref)} | ${cell3(entry.availability ?? "")} | ${cellText} |`
+    );
+  }
+  lines.push("");
+  if (report.manifest !== void 0) {
+    lines.push(`## Session manifest`);
+    lines.push("");
+    for (const line of manifestLines(report.manifest)) {
+      lines.push(`- ${cell3(line)}`);
+    }
+    lines.push("");
+  }
+  lines.push(`## Availability`);
+  lines.push("");
+  for (const line of availabilityLines(report.availability)) {
+    lines.push(`- ${line}`);
+  }
+  lines.push("");
+  if (report.warnings.length > 0) {
+    lines.push(`## Warnings`);
+    lines.push("");
+    for (const warning of report.warnings) lines.push(`- ${warning}`);
+    lines.push("");
+  }
+  const s = report.summary;
+  lines.push(
+    `**${s.total} eval(s)** \u2014 ${s.pass} pass, ${s.fail} fail, ${s.error} error, ${s.needsReview} needs-review, ${s.skipped} skipped.`
+  );
+  return lines.join("\n");
+}
+
+// src/tracevals/reporters/sarif.ts
+var SARIF_SCHEMA = "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json";
+var LEVELS = {
+  error: "error",
+  warning: "warning",
+  notice: "note"
+};
+function declaringLocation(r, root) {
+  if (classifyRef(r.location.file).kind === "url") return { uri: r.location.file };
+  return artifactLocation(locationPath(r.location, root), root);
+}
+function renderSarif(input) {
+  const rules = /* @__PURE__ */ new Map();
+  const results = [];
+  const declare = (id, name, text, level) => {
+    if (!rules.has(id)) {
+      rules.set(id, {
+        id,
+        name,
+        shortDescription: { text },
+        defaultConfiguration: { level }
+      });
+    }
+  };
+  const push = (run, r, level, text) => {
+    const id = ruleIdFor(r.grader);
+    declare(
+      id,
+      r.grader,
+      r.grader === "ai" ? "manni tracevals AI judge" : `manni tracevals ${r.grader} grader`,
+      level
+    );
+    results.push({
+      ruleId: id,
+      level,
+      message: { text },
+      locations: [
+        {
+          physicalLocation: {
+            artifactLocation: declaringLocation(r, input.root),
+            // SARIF regions are 1-based and need `startLine`, so a location
+            // with no line has no region rather than an invented line 1.
+            ...r.location.line === void 0 ? {} : { region: { startLine: r.location.line } }
+          }
+        }
+      ],
+      properties: {
+        eval: r.evalName,
+        artifactName: r.artifactName,
+        artifactType: r.artifactType,
+        trace: displayPath(run.trace.file, input.root),
+        ...run.trace.sessionId !== void 0 ? { sessionId: run.trace.sessionId } : {}
+      }
+    });
+  };
+  for (const run of input.runs) {
+    for (const r of run.evalResults) {
+      for (const f of r.findings ?? []) {
+        push(run, r, LEVELS[f.severity], f.message);
+      }
+      const judged = judgeFailText(r);
+      if (judged !== void 0) push(run, r, "error", judged);
+      if (r.outcome === "error") push(run, r, "error", errorText(r));
+    }
+  }
+  for (const t of input.traceErrors) {
+    declare(TRACE_RULE_ID, "trace", "manni tracevals could not read the trace", "error");
+    results.push({
+      ruleId: TRACE_RULE_ID,
+      level: "error",
+      message: { text: traceErrorText(t.error) },
+      locations: [
+        {
+          physicalLocation: {
+            artifactLocation: artifactLocation(t.file, input.root)
+          }
+        }
+      ]
+    });
+  }
+  const log = {
+    $schema: SARIF_SCHEMA,
+    version: "2.1.0",
+    runs: [
+      {
+        tool: {
+          driver: {
+            name: "manni-tracevals",
+            informationUri: "https://hawkeyexl.github.io/manni/tracevals/",
+            rules: [...rules.values()]
+          }
+        },
+        originalUriBaseIds: { [URI_BASE_ID]: { uri: fileUri(input.root) } },
+        results
+      }
+    ]
+  };
+  return `${JSON.stringify(log, null, 2)}
+`;
+}
+
+// src/tracevals/reporters/format.ts
+var REPORT_FORMATS = [
+  "pretty",
+  "json",
+  "markdown",
+  "github",
+  "sarif",
+  "junit"
+];
+var CALIBRATE_FORMATS = ["pretty", "json", "markdown"];
+var SUMMARY_FORMATS = ["pretty", "json"];
+function parseFormat(value, allowed, flag) {
+  if (!allowed.includes(value)) {
+    throw new TracevalsError(
+      `${flag} must be one of ${allowed.join(" | ")}, got "${value}"`
+    );
+  }
+  return value;
+}
+
+// src/tracevals/reporters/index.ts
+function render(report, format, opts = {}) {
+  parseFormat(format, REPORT_FORMATS, "format");
+  switch (format) {
+    case "json":
+      return JSON.stringify(report, null, 2);
+    case "markdown":
+      return renderMarkdown(report);
+    case "pretty":
+      return renderPretty(report, opts);
+    case "github":
+      return renderGithub(ciInputFromRun(report), renderMarkdown(report));
+    case "sarif":
+      return renderSarif(ciInputFromRun(report));
+    case "junit":
+      return renderJunit(ciInputFromRun(report));
+  }
+}
+function renderBatch(report, format, runs, opts = {}) {
+  parseFormat(format, REPORT_FORMATS, "format");
+  const ci = () => {
+    if (runs === void 0) {
+      throw new TracevalsError(
+        `format "${format}" annotates findings, which live in the per-trace reports; pass them as renderBatch's third argument`
+      );
+    }
+    return ciInputFromBatch(report, runs);
+  };
+  switch (format) {
+    case "json":
+      return JSON.stringify(report, null, 2);
+    case "markdown":
+      return renderBatchMarkdown(report);
+    case "pretty":
+      return renderBatchPretty(report, opts);
+    case "github":
+      return renderGithub(ci(), renderBatchMarkdown(report));
+    case "sarif":
+      return renderSarif(ci());
+    case "junit":
+      return renderJunit(ci());
+  }
+}
+function renderCalibration(report, format, opts = {}) {
+  const narrowed = parseFormat(format, CALIBRATE_FORMATS, "format");
+  switch (narrowed) {
+    case "json":
+      return JSON.stringify(report, null, 2);
+    case "markdown":
+      return renderCalibrationMarkdown(report);
+    case "pretty":
+      return renderCalibrationPretty(report, opts);
+  }
+}
+
+// src/tracevals/commands/run.ts
+import { writeFile } from "node:fs/promises";
+import { resolve as resolve5 } from "node:path";
+
+// src/tracevals/trace/claude.ts
+import { readdir, readFile } from "node:fs/promises";
+import { basename as basename3, dirname, join, resolve } from "node:path";
+
+// src/tracevals/trace/detect.ts
+var SUPPORTED = "supported formats: Claude Code session files (~/.claude/projects/**.jsonl) and claude -p stream-json";
+function detectFormat(line) {
+  let rec;
+  try {
+    rec = JSON.parse(line);
+  } catch {
+    throw new TracevalsError(`trace is not JSONL (${SUPPORTED})`);
+  }
+  if (typeof rec !== "object" || rec === null) {
+    throw new TracevalsError(`unrecognized trace format (${SUPPORTED})`);
+  }
+  if ("parentUuid" in rec || "sessionId" in rec) return "claude-session";
+  if ("session_id" in rec) return "claude-stream";
+  throw new TracevalsError(`unrecognized trace format (${SUPPORTED})`);
+}
+function detectContentFormat(content) {
+  let sawParseable = false;
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    let rec;
+    try {
+      rec = JSON.parse(trimmed);
+    } catch {
+      continue;
+    }
+    if (typeof rec !== "object" || rec === null) continue;
+    sawParseable = true;
+    const r = rec;
+    if ("parentUuid" in r || "sessionId" in r) return "claude-session";
+    if ("session_id" in r) return "claude-stream";
+  }
+  throw new TracevalsError(
+    sawParseable ? `unrecognized trace format (${SUPPORTED})` : `empty or unparseable trace file (${SUPPORTED})`
+  );
+}
+
+// src/tracevals/trace/claude.ts
+async function parseTraceFile(filePath) {
+  const absolute = resolve(filePath);
+  const content = await readFile(absolute, "utf-8");
+  const trace = parseTraceContent(content, absolute);
+  await mergeSidecarBranches(trace, absolute);
+  return trace;
+}
+function parseTraceContent(content, filePath) {
+  const format = detectContentFormat(content);
+  const { records, unparseable } = parseLines(content);
+  const trace = format === "claude-session" ? parseSession(records, filePath) : parseStream(records, filePath);
+  if (unparseable > 0) {
+    trace.warnings.push(
+      `${unparseable} unparseable JSONL line(s) were skipped`
+    );
+  }
+  return trace;
+}
+function parseLines(content) {
+  const records = [];
+  let unparseable = 0;
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const rec = JSON.parse(trimmed);
+      if (typeof rec === "object" && rec !== null) records.push(rec);
+      else unparseable += 1;
+    } catch {
+      unparseable += 1;
+    }
+  }
+  return { records, unparseable };
+}
+var SESSION_META_TYPES = /* @__PURE__ */ new Set([
+  "queue-operation",
+  "attachment",
+  "last-prompt",
+  "custom-title",
+  "summary",
+  "mode",
+  "pr-link"
+]);
+var COMMAND_NAME_RE = /<command-name>([^<]+)<\/command-name>/;
+var COMMAND_ARGS_RE = /<command-args>([^<]*)<\/command-args>/;
+function newTrace(filePath) {
+  return {
+    source: "claude-code",
+    file: filePath,
+    cwd: "",
+    events: [],
+    toolCalls: [],
+    skillInvocations: [],
+    agentSpawns: [],
+    subagentBranches: [],
+    availability: newAvailability(),
+    fileAccesses: [],
+    userMessages: [],
+    assistantTexts: [],
+    turnCount: 0,
+    warnings: []
+  };
+}
+function pushEvent(trace, event) {
+  const index = trace.events.length;
+  trace.events.push({ ...event, index });
+  return index;
+}
+function plainMessageText(rec) {
+  const content = rec.message?.content;
+  return typeof content === "string" ? content : void 0;
+}
+function resolveBranch(branches, parentUuid, rootText) {
+  if (typeof parentUuid !== "string") return void 0;
+  const inherited = branches.resolved.get(parentUuid);
+  if (inherited !== void 0) return inherited;
+  const pending = branches.spawns.get(parentUuid);
+  if (pending === void 0 || pending.length === 0) return void 0;
+  const byPrompt = rootText !== void 0 ? pending.find((b) => !b.claimed && b.prompt === rootText) : void 0;
+  const branch = byPrompt ?? pending.find((b) => !b.claimed) ?? pending[0];
+  if (branch === void 0) return void 0;
+  branch.claimed = true;
+  return branch.toolUseId;
+}
+function parseSession(records, filePath) {
+  const trace = newTrace(filePath);
+  const branches = { spawns: /* @__PURE__ */ new Map(), resolved: /* @__PURE__ */ new Map() };
+  const availability = newReplay();
+  const uuids = /* @__PURE__ */ new Set();
+  for (const rec of records) {
+    if (typeof rec.uuid === "string") uuids.add(rec.uuid);
+  }
+  const continuations = [];
+  let sawMessage = false;
+  let compactions = 0;
+  let inputTokens = 0;
+  let outputTokens = 0;
+  let sawUsage = false;
+  for (const rec of records) {
+    const type = rec.type;
+    const timestamp = rec.timestamp;
+    if (timestamp) {
+      trace.startedAt ??= timestamp;
+      trace.endedAt = timestamp;
+    }
+    if (rec.sessionId) trace.sessionId ??= rec.sessionId;
+    if (rec.subtype === "compact_boundary") compactions += 1;
+    if (type !== void 0 && SESSION_META_TYPES.has(type)) {
+      if (type === "summary" && !sawMessage && typeof rec.leafUuid === "string" && !uuids.has(rec.leafUuid)) {
+        continuations.push(rec.leafUuid);
+      }
+      const index = pushEvent(trace, { kind: "meta", timestamp, raw: rec });
+      if (type === "attachment" && applyAvailabilityRecord(availability, rec.attachment, index)) {
+        availability.roster.recorded = true;
+      }
+      continue;
+    }
+    const sidechain = rec.isSidechain === true;
+    if (!sidechain && typeof rec.cwd === "string") trace.cwd = rec.cwd;
+    if (!sidechain && typeof rec.gitBranch === "string") {
+      trace.gitBranch = rec.gitBranch;
+    }
+    const context = { sidechain, timestamp, branches };
+    if (typeof rec.uuid === "string") context.uuid = rec.uuid;
+    if (sidechain) {
+      const branchId = resolveBranch(
+        branches,
+        rec.parentUuid,
+        plainMessageText(rec)
+      );
+      if (branchId !== void 0) {
+        context.branchId = branchId;
+        if (context.uuid) branches.resolved.set(context.uuid, branchId);
+      }
+    }
+    const message = rec.message;
+    if (type === "user" && message) {
+      sawMessage = true;
+      handleUserMessage(trace, message, context, rec);
+    } else if (type === "assistant" && message) {
+      sawMessage = true;
+      const usage = handleAssistantMessage(trace, message, context, rec);
+      if (usage && !sidechain) {
+        inputTokens += usage.input;
+        outputTokens += usage.output;
+        sawUsage = true;
+      }
+    } else {
+      pushEvent(trace, { kind: "system", timestamp, raw: rec });
+    }
+  }
+  if (sawUsage) trace.usage = { inputTokens, outputTokens };
+  for (const leaf of continuations) {
+    trace.warnings.push(
+      `this trace is a fragment of a longer session: it resumes an earlier one recorded elsewhere (summary leafUuid ${leaf} is not in this file), so evals see only the turns recorded here`
+    );
+  }
+  if (compactions > 0) {
+    trace.warnings.push(
+      `the conversation was compacted ${compactions} time(s): the agent stopped seeing the turns before each boundary, so early transcript content is not what it was working from`
+    );
+  }
+  trace.availability = availability.roster;
+  trace.subagentBranches = buildBranches(trace);
+  return trace;
+}
+function buildBranches(trace, sidecars = /* @__PURE__ */ new Map()) {
+  const ranges = /* @__PURE__ */ new Map();
+  for (const event of trace.events) {
+    const id = event.branchId;
+    if (id === void 0) continue;
+    const range = ranges.get(id);
+    if (range === void 0) ranges.set(id, { min: event.index, max: event.index });
+    else range.max = Math.max(range.max, event.index);
+  }
+  if (ranges.size === 0) return [];
+  const spawnFor = /* @__PURE__ */ new Map();
+  for (const spawn of trace.agentSpawns) {
+    if (spawn.toolUseId) spawnFor.set(spawn.toolUseId, spawn);
+  }
+  const parentOf = /* @__PURE__ */ new Map();
+  for (const id of ranges.keys()) {
+    const spawn = spawnFor.get(id);
+    parentOf.set(
+      id,
+      spawn ? trace.events[spawn.index]?.branchId : void 0
+    );
+  }
+  const depthOf = (id) => {
+    let depth = 1;
+    let cursor = parentOf.get(id);
+    const seen = /* @__PURE__ */ new Set([id]);
+    while (cursor !== void 0 && !seen.has(cursor)) {
+      seen.add(cursor);
+      depth += 1;
+      cursor = parentOf.get(cursor);
+    }
+    return depth;
+  };
+  const branches = [];
+  for (const [id, range] of ranges) {
+    const spawn = spawnFor.get(id);
+    const sidecar = sidecars.get(id);
+    const branch = {
+      branchId: id,
+      agentType: sidecar?.agentType ?? spawn?.subagentType ?? id,
+      origin: sidecar ? "sidecar" : "inline",
+      spawnDepth: sidecar?.spawnDepth ?? depthOf(id),
+      spawnIndex: spawn?.index ?? range.min,
+      startIndex: range.min,
+      endIndex: range.max + 1
+    };
+    const description = sidecar?.description ?? spawn?.description;
+    if (description !== void 0) branch.description = description;
+    if (sidecar?.agentId) branch.agentId = sidecar.agentId;
+    if (sidecar?.parentAgentId) branch.parentAgentId = sidecar.parentAgentId;
+    if (sidecar?.file) branch.file = sidecar.file;
+    branches.push(branch);
+  }
+  branches.sort((a, b) => b.spawnDepth - a.spawnDepth);
+  const byId = new Map(branches.map((b) => [b.branchId, b]));
+  for (const branch of branches) {
+    const parentId = parentOf.get(branch.branchId);
+    const parent = parentId === void 0 ? void 0 : byId.get(parentId);
+    if (parent === void 0) continue;
+    parent.startIndex = Math.min(parent.startIndex, branch.startIndex);
+    parent.endIndex = Math.max(parent.endIndex, branch.endIndex);
+  }
+  branches.sort((a, b) => a.startIndex - b.startIndex);
+  return branches;
+}
+function handleUserMessage(trace, message, context, raw) {
+  const { sidechain, timestamp, branchId } = context;
+  const content = message.content;
+  if (typeof content === "string") {
+    const index = pushEvent(trace, {
+      kind: "user",
+      timestamp,
+      text: content,
+      sidechain,
+      branchId,
+      raw
+    });
+    if (sidechain) return;
+    const command = COMMAND_NAME_RE.exec(content);
+    if (command?.[1]) {
+      const invocation = {
+        name: command[1].replace(/^\//, ""),
+        via: "command-injection",
+        index
+      };
+      const args = COMMAND_ARGS_RE.exec(content)?.[1]?.trim();
+      if (args) invocation.args = args;
+      trace.skillInvocations.push(invocation);
+    }
+    trace.userMessages.push(content);
+    trace.turnCount += 1;
+    return;
+  }
+  if (Array.isArray(content)) {
+    let hasToolResult = false;
+    const texts = [];
+    for (const block of content) {
+      if (block?.type === "tool_result") hasToolResult = true;
+      if (block?.type === "text" && typeof block.text === "string") {
+        texts.push(block.text);
+      }
+    }
+    if (hasToolResult) {
+      pushEvent(trace, {
+        kind: "tool_result",
+        timestamp,
+        sidechain,
+        branchId,
+        raw
+      });
+      return;
+    }
+    const text = texts.join("\n");
+    pushEvent(trace, {
+      kind: "user",
+      timestamp,
+      text,
+      sidechain,
+      branchId,
+      raw
+    });
+    if (!sidechain && text) {
+      trace.userMessages.push(text);
+      trace.turnCount += 1;
+    }
+  }
+}
+function handleAssistantMessage(trace, message, context, raw) {
+  const { sidechain, timestamp, branchId } = context;
+  if (!sidechain && typeof message.model === "string") {
+    trace.model ??= message.model;
+  }
+  const content = message.content;
+  if (Array.isArray(content)) {
+    for (const block of content) {
+      if (block?.type === "text" && typeof block.text === "string") {
+        pushEvent(trace, {
+          kind: "assistant",
+          timestamp,
+          text: block.text,
+          sidechain,
+          branchId,
+          raw
+        });
+        if (!sidechain) trace.assistantTexts.push(block.text);
+      } else if (block?.type === "tool_use") {
+        extractToolUse(trace, block, context);
+      }
+    }
+  }
+  const usage = message.usage;
+  if (usage) {
+    return {
+      input: typeof usage.input_tokens === "number" ? usage.input_tokens : 0,
+      output: typeof usage.output_tokens === "number" ? usage.output_tokens : 0
+    };
+  }
+  return void 0;
+}
+function extractToolUse(trace, block, context) {
+  const { sidechain, timestamp, branchId } = context;
+  const name = block.name;
+  const input = block.input ?? {};
+  const toolUseId = typeof block.id === "string" ? block.id : void 0;
+  const index = pushEvent(trace, {
+    kind: "tool_call",
+    timestamp,
+    toolName: name,
+    sidechain,
+    branchId,
+    raw: block
+  });
+  const call = { name, input, index, sidechain };
+  if (timestamp) call.timestamp = timestamp;
+  if (branchId) call.branchId = branchId;
+  trace.toolCalls.push(call);
+  switch (name) {
+    case "Skill": {
+      if (typeof input.skill === "string") {
+        const invocation = {
+          name: input.skill,
+          via: "skill-tool",
+          index
+        };
+        if (toolUseId) invocation.toolUseId = toolUseId;
+        if (typeof input.args === "string" && input.args) {
+          invocation.args = input.args;
+        }
+        trace.skillInvocations.push(invocation);
+      }
+      break;
+    }
+    case "Agent": {
+      if (typeof input.subagent_type === "string") {
+        const spawn = {
+          subagentType: input.subagent_type,
+          index
+        };
+        if (typeof input.description === "string") {
+          spawn.description = input.description;
+        }
+        if (toolUseId) spawn.toolUseId = toolUseId;
+        trace.agentSpawns.push(spawn);
+      }
+      if (context.branches && context.uuid && toolUseId) {
+        const pending = context.branches.spawns.get(context.uuid) ?? [];
+        pending.push({
+          toolUseId,
+          prompt: typeof input.prompt === "string" ? input.prompt : void 0,
+          claimed: false
+        });
+        context.branches.spawns.set(context.uuid, pending);
+      }
+      break;
+    }
+    case "Read":
+    case "Write":
+    case "Edit": {
+      if (typeof input.file_path === "string") {
+        const op = name === "Read" ? "read" : name === "Write" ? "write" : "edit";
+        trace.fileAccesses.push({ path: input.file_path, op, index });
+      }
+      break;
+    }
+  }
+}
+function parseStream(records, filePath) {
+  const trace = newTrace(filePath);
+  const context = { sidechain: false };
+  let inputTokens = 0;
+  let outputTokens = 0;
+  let sawMessageUsage = false;
+  for (const rec of records) {
+    const type = rec.type;
+    if (type === "system" && (rec.subtype === "init" || !rec.subtype)) {
+      if (typeof rec.cwd === "string") trace.cwd = rec.cwd;
+      if (typeof rec.model === "string") trace.model = rec.model;
+      if (typeof rec.session_id === "string") trace.sessionId = rec.session_id;
+      pushEvent(trace, { kind: "system", raw: rec });
+      continue;
+    }
+    if (type === "assistant" || type === "user") {
+      const message = rec.message ?? rec;
+      if (type === "assistant") {
+        const usage = handleAssistantMessage(trace, message, context, rec);
+        if (usage) {
+          inputTokens += usage.input;
+          outputTokens += usage.output;
+          sawMessageUsage = true;
+        }
+      } else {
+        handleUserMessage(trace, message, context, rec);
+      }
+      continue;
+    }
+    if (type === "result") {
+      trace.isError = rec.is_error === true;
+      if (typeof rec.num_turns === "number") trace.turnCount = rec.num_turns;
+      const usage = rec.usage;
+      const totalCost = typeof rec.total_cost_usd === "number" ? rec.total_cost_usd : void 0;
+      if (usage) {
+        trace.usage = {
+          inputTokens: typeof usage.input_tokens === "number" ? usage.input_tokens : 0,
+          outputTokens: typeof usage.output_tokens === "number" ? usage.output_tokens : 0
+        };
+        if (totalCost !== void 0) trace.usage.totalCostUsd = totalCost;
+      } else if (sawMessageUsage) {
+        trace.usage = { inputTokens, outputTokens };
+        if (totalCost !== void 0) trace.usage.totalCostUsd = totalCost;
+      }
+      pushEvent(trace, { kind: "meta", raw: rec });
+      continue;
+    }
+    pushEvent(trace, { kind: "meta", raw: rec });
+  }
+  if (!trace.usage && sawMessageUsage) {
+    trace.usage = { inputTokens, outputTokens };
+  }
+  return trace;
+}
+var AGENT_META_RE = /^agent-(.+)\.meta\.json$/;
+var AGENT_JSONL_RE = /^agent-(.+)\.jsonl$/;
+function sidecarDirFor(traceFile) {
+  const stem = basename3(traceFile).replace(/\.jsonl$/i, "");
+  return join(dirname(traceFile), stem, "subagents");
+}
+function asSource(id, trace) {
+  return {
+    id,
+    events: trace.events,
+    toolCalls: trace.toolCalls,
+    fileAccesses: trace.fileAccesses,
+    skillInvocations: trace.skillInvocations,
+    agentSpawns: trace.agentSpawns
+  };
+}
+function stampBranch(sub, branchId) {
+  for (const event of sub.events) {
+    event.sidechain = true;
+    event.branchId = branchId;
+  }
+  for (const call of sub.toolCalls) {
+    call.sidechain = true;
+    call.branchId = branchId;
+  }
+}
+async function loadSidecars(dir, trace) {
+  let names2;
+  try {
+    names2 = await readdir(dir);
+  } catch (error) {
+    const code = error.code;
+    if (code !== "ENOENT" && code !== "ENOTDIR") {
+      trace.warnings.push(
+        `could not read the subagent transcript directory ${dir} (${code ?? "unknown error"}), so subagent turns are missing here`
+      );
+    }
+    return [];
+  }
+  const metas = /* @__PURE__ */ new Set();
+  const transcripts = /* @__PURE__ */ new Set();
+  for (const name of names2) {
+    const meta = AGENT_META_RE.exec(name);
+    if (meta?.[1] !== void 0) {
+      metas.add(meta[1]);
+      continue;
+    }
+    const jsonl = AGENT_JSONL_RE.exec(name);
+    if (jsonl?.[1] !== void 0) transcripts.add(jsonl[1]);
+  }
+  for (const orphan of [...transcripts].sort()) {
+    if (!metas.has(orphan)) {
+      trace.warnings.push(
+        `subagent transcript agent-${orphan}.jsonl has no agent-${orphan}.meta.json, so it names no Agent spawn and was not merged`
+      );
+    }
+  }
+  const loaded = [];
+  for (const agentId of [...metas].sort()) {
+    let meta;
+    try {
+      const raw = await readFile(
+        join(dir, `agent-${agentId}.meta.json`),
+        "utf-8"
+      );
+      const parsed = JSON.parse(raw);
+      if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+        throw new Error("shape");
+      }
+      meta = parsed;
+    } catch {
+      trace.warnings.push(
+        `subagent metadata agent-${agentId}.meta.json is unreadable or not an object, so that branch was not merged`
+      );
+      continue;
+    }
+    const file = join(dir, `agent-${agentId}.jsonl`);
+    let sub;
+    try {
+      sub = parseTraceContent(await readFile(file, "utf-8"), file);
+    } catch {
+      trace.warnings.push(
+        `subagent transcript agent-${agentId}.jsonl is missing or unreadable, so that branch was not merged`
+      );
+      continue;
+    }
+    for (const warning of sub.warnings) {
+      trace.warnings.push(`subagent ${meta.agentType ?? agentId}: ${warning}`);
+    }
+    loaded.push({ agentId, file, meta, sub, source: asSource(agentId, sub) });
+  }
+  return loaded;
+}
+async function mergeSidecarBranches(trace, traceFile) {
+  const loaded = await loadSidecars(sidecarDirFor(traceFile), trace);
+  if (loaded.length === 0) return;
+  const base = asSource("", trace);
+  const sources = /* @__PURE__ */ new Map([["", base]]);
+  const spawnOwner = /* @__PURE__ */ new Map();
+  const registerSpawns = (source) => {
+    for (const spawn of source.agentSpawns) {
+      if (spawn.toolUseId !== void 0) {
+        spawnOwner.set(spawn.toolUseId, {
+          sourceId: source.id,
+          local: spawn.index
+        });
+      }
+    }
+  };
+  registerSpawns(base);
+  const children = /* @__PURE__ */ new Map();
+  const attached = [];
+  const pending = [...loaded].sort(
+    (a, b) => (a.meta.spawnDepth ?? 1) - (b.meta.spawnDepth ?? 1) || (a.agentId < b.agentId ? -1 : a.agentId > b.agentId ? 1 : 0)
+  );
+  let progress = true;
+  while (progress) {
+    progress = false;
+    for (let i = 0; i < pending.length; i += 1) {
+      const sidecar = pending[i];
+      const branchId = sidecar.meta.toolUseId;
+      const owner = branchId === void 0 ? void 0 : spawnOwner.get(branchId);
+      if (branchId === void 0 || owner === void 0) continue;
+      pending.splice(i, 1);
+      i -= 1;
+      progress = true;
+      stampBranch(sidecar.sub, branchId);
+      sources.set(sidecar.agentId, sidecar.source);
+      const key = `${owner.sourceId}\0${owner.local}`;
+      children.set(key, [...children.get(key) ?? [], sidecar]);
+      registerSpawns(sidecar.source);
+      attached.push({ sidecar, branchId });
+    }
+  }
+  for (const sidecar of pending) {
+    trace.warnings.push(
+      `subagent transcript agent-${sidecar.agentId}.jsonl names Agent call ${sidecar.meta.toolUseId ?? "(none recorded)"}, which is not in this trace, so that branch was not merged`
+    );
+  }
+  if (attached.length === 0) return;
+  const slots = [];
+  const emit = (sourceId) => {
+    const source = sources.get(sourceId);
+    if (source === void 0) return;
+    for (let local = 0; local < source.events.length; local += 1) {
+      slots.push({ sourceId, local });
+      for (const kid of children.get(`${sourceId}\0${local}`) ?? []) {
+        emit(kid.agentId);
+      }
+    }
+  };
+  emit("");
+  const remap = /* @__PURE__ */ new Map();
+  slots.forEach((slot, final) => {
+    const row = remap.get(slot.sourceId) ?? [];
+    row[slot.local] = final;
+    remap.set(slot.sourceId, row);
+  });
+  const finalIndex = (sourceId, local) => remap.get(sourceId)?.[local] ?? local;
+  trace.events = slots.map((slot, final) => {
+    const event = sources.get(slot.sourceId)?.events[slot.local];
+    return { ...event, index: final };
+  });
+  const gather = (pick) => [...sources.values()].flatMap(
+    (source) => pick(source).map((item) => ({
+      ...item,
+      index: finalIndex(source.id, item.index)
+    }))
+  ).sort((a, b) => a.index - b.index);
+  trace.toolCalls = gather((s) => s.toolCalls);
+  trace.fileAccesses = gather((s) => s.fileAccesses);
+  trace.skillInvocations = gather((s) => s.skillInvocations);
+  trace.agentSpawns = gather((s) => s.agentSpawns);
+  trace.availability = remapAvailability(
+    trace.availability,
+    (index) => finalIndex("", index)
+  );
+  for (const { sidecar, branchId } of attached) {
+    const owner = spawnOwner.get(branchId);
+    foldBranchAvailability(
+      trace.availability,
+      sidecar.sub.availability,
+      branchId,
+      owner === void 0 ? 0 : finalIndex(owner.sourceId, owner.local)
+    );
+  }
+  const described = /* @__PURE__ */ new Map();
+  for (const { sidecar, branchId } of attached) {
+    const info = {
+      agentId: sidecar.agentId,
+      file: sidecar.file
+    };
+    if (sidecar.meta.agentType) info.agentType = sidecar.meta.agentType;
+    if (sidecar.meta.description) info.description = sidecar.meta.description;
+    if (sidecar.meta.parentAgentId) {
+      info.parentAgentId = sidecar.meta.parentAgentId;
+    }
+    if (typeof sidecar.meta.spawnDepth === "number") {
+      info.spawnDepth = sidecar.meta.spawnDepth;
+    }
+    described.set(branchId, info);
+  }
+  trace.subagentBranches = buildBranches(trace, described);
+}
+
+// src/tracevals/core/plan.ts
+var IMPLICIT_EVAL_NAME = "adheres-to-artifact";
+function planEvals(artifacts, metadataFor) {
+  const plans = [];
+  for (const artifact of artifacts) {
+    const supplied = metadataFor?.(artifact);
+    const metadata = {
+      extracted: supplied?.extracted ?? extractFrontmatter(artifact.content, "markdown"),
+      ...supplied?.locate === void 0 ? {} : { locate: supplied.locate }
+    };
+    const extracted = extractEvals(artifact, metadata.extracted);
+    const at = (pointer) => declaredAt(artifact.path, pointer, metadata);
+    const whole = { file: artifact.path };
+    if (extracted.errors.length > 0) {
+      const detail2 = extracted.errors.map(
+        (e) => `${e.instancePath || "/"}${e.line !== void 0 ? ` (line ${e.line})` : ""}: ${e.message}`
+      ).join("; ");
+      plans.push({
+        artifact,
+        evalName: "evals-block-valid",
+        assertion: "The artifact's metadata.evals block matches the schema.",
+        grader: "ai",
+        severity: "error",
+        implicit: false,
+        location: at("/metadata/evals"),
+        error: `invalid metadata.evals block: ${detail2}`
+      });
+      continue;
+    }
+    if (extracted.skip) {
+      plans.push({
+        artifact,
+        evalName: IMPLICIT_EVAL_NAME,
+        assertion: "Artifact skipped via metadata.eval-skip.",
+        grader: "ai",
+        severity: "error",
+        implicit: true,
+        location: at("/metadata/eval-skip"),
+        skipped: true,
+        skipReason: "artifact skipped via metadata.eval-skip"
+      });
+      continue;
+    }
+    if (extracted.evals.length === 0) {
+      plans.push({
+        artifact,
+        evalName: IMPLICIT_EVAL_NAME,
+        assertion: `The session adhered to the instructions in this ${artifact.type} ("${artifact.name}"). Cite the specific instructions followed or violated.`,
+        grader: "ai",
+        severity: "error",
+        implicit: true,
+        location: whole
+      });
+      continue;
+    }
+    for (const [index, entry] of extracted.evals.entries()) {
+      const plan = {
+        artifact,
+        evalName: entry.id,
+        assertion: entry.assertion,
+        grader: entry.grader,
+        ...entry.model !== void 0 ? { model: entry.model } : {},
+        ...entry.runs !== void 0 ? { runs: entry.runs } : {},
+        ...entry.target !== void 0 ? { target: entry.target } : {},
+        ...entry.weight !== void 0 ? { weight: entry.weight } : {},
+        severity: entry.severity,
+        implicit: false,
+        // A string block is one entry at index 0, and the pointer walks up to
+        // the block's own line.
+        location: at(`/metadata/evals/${String(index)}`)
+      };
+      if (entry.options) plan.options = entry.options;
+      if (entry.evidence) plan.evidence = entry.evidence;
+      if (entry.examples) plan.examples = entry.examples;
+      if (entry.provider) plan.provider = entry.provider;
+      if (entry.command) plan.command = entry.command;
+      if (entry.successExitCodes) plan.successExitCodes = entry.successExitCodes;
+      if (entry.timeoutMs !== void 0) plan.timeoutMs = entry.timeoutMs;
+      if (entry.generatedAssertionHash !== void 0) {
+        plan.generatedAssertionHash = entry.generatedAssertionHash;
+      }
+      const proposedBy = extracted.proposedBy.get(entry.id);
+      if (proposedBy !== void 0 && proposedBy.length > 0) {
+        plan.proposedBy = proposedBy;
+      }
+      if (entry.skip) {
+        plan.skipped = true;
+        plan.skipReason = "eval skipped via its own skip: true";
+      }
+      plans.push(plan);
+    }
+  }
+  return plans;
+}
+
+// src/tracevals/judge/prompt.ts
+var PROMPT_VERSION = 4;
+var JUDGE_SYSTEM_PROMPT = `You are an adherence judge for AI agent sessions. You are given some evidence from an agent session and one assertion drawn from the instructions the session was operating under \u2014 a skill definition, an agent definition, or project rules.
+
+The evidence is whatever the eval asked to be graded, and its heading says which: usually the session transcript (messages, tool calls, skills used), but it may instead be the session's final message, the list of files it touched, the instruction artifact itself, or one named file. Judge the assertion against what you were actually given.
+
+Decide whether the session adhered to the assertion:
+- "pass" only when the session fully satisfied it.
+- "partial" when it partly satisfied it.
+- "fail" when it violated it or plainly did not do it.
+
+Ground every judgment in the graded content: quote or cite the specific part of it \u2014 a message, a tool call, a line, or a telling omission \u2014 that supports your verdict in the "observed" field. If the assertion concerns something the graded content cannot show (e.g. events outside what you were given), say so and lower your confidence.
+
+Respond with a single JSON object matching the provided schema. No prose outside the JSON.`;
+var MAX_ARTIFACT_CHARS = 8e3;
+function artifactWasTruncated(content) {
+  return content.length > MAX_ARTIFACT_CHARS;
+}
+function buildUserContent(plan, graded, targetLabel = "transcript") {
+  const parts = [];
+  parts.push(`# Assertion
+${plan.assertion ?? ""}`);
+  if (plan.evidence) {
+    parts.push(`# Where to look
+${plan.evidence}`);
+  }
+  if (plan.examples?.pass?.length) {
+    parts.push(`# Examples of passing behavior
+- ${plan.examples.pass.join("\n- ")}`);
+  }
+  if (plan.examples?.fail?.length) {
+    parts.push(`# Examples of failing behavior
+- ${plan.examples.fail.join("\n- ")}`);
+  }
+  let artifactBody = plan.artifact.content;
+  if (artifactBody.length > MAX_ARTIFACT_CHARS) {
+    artifactBody = `${artifactBody.slice(0, MAX_ARTIFACT_CHARS)}
+[... artifact truncated ...]`;
+  }
+  const gradingTheArtifact = targetLabel === "artifact";
+  const artifactHeading = `${plan.artifact.type} ("${plan.artifact.name}")`;
+  if (!gradingTheArtifact) {
+    parts.push(`# Source ${artifactHeading}
+${artifactBody}`);
+  }
+  parts.push(
+    targetLabel === "transcript" ? `# Session transcript
+${graded}` : gradingTheArtifact ? `# Graded content: the ${artifactHeading} itself
+${artifactBody}` : `# Graded content (${targetLabel})
+${graded}`
+  );
+  return parts.join("\n\n");
+}
+
+// src/tracevals/graders/glob.ts
+var literal = (ch) => ch.replace(/[.+^${}()|[\]\\]/g, "\\$&");
+var compiled = /* @__PURE__ */ new Map();
+function globToRegExp(glob) {
+  const cached = compiled.get(glob);
+  if (cached !== void 0) return cached;
+  const built = compile(glob);
+  compiled.set(glob, built);
+  return built;
+}
+function compile(glob) {
+  const spec = glob.replace(/\\/g, "/").toLowerCase().replace(/(?:\*\*\/)+/g, "**/");
+  let out = "";
+  for (let i = 0; i < spec.length; i += 1) {
+    const ch = spec[i];
+    if (ch === "?") {
+      out += "[^/]";
+      continue;
+    }
+    if (ch !== "*") {
+      out += literal(ch);
+      continue;
+    }
+    if (spec[i + 1] !== "*") {
+      out += "[^/]*";
+      continue;
+    }
+    i += 1;
+    if (spec[i + 1] === "/") {
+      i += 1;
+      if (out !== "") out += "(?:.*/)?";
+    } else if (out.endsWith("/")) {
+      out = `${out.slice(0, -1)}(?:/.*)?`;
+    } else {
+      out += ".*";
+    }
+  }
+  return new RegExp(`^(?:.*/)?${out}$`);
+}
+function normalizePath(path) {
+  return path.replace(/\\/g, "/").toLowerCase();
+}
+function matchesGlob(path, glob) {
+  return globToRegExp(glob).test(normalizePath(path));
+}
+
+// src/tracevals/graders/when.ts
+var stringValue = (kind) => (value) => typeof value === "string" && value.length > 0 ? void 0 : `options.when.${kind} must be a string`;
+var prompts = /* @__PURE__ */ new Map();
+function promptRe(pattern) {
+  const cached = prompts.get(pattern);
+  if (cached !== void 0) return cached;
+  const built = new RegExp(pattern);
+  prompts.set(pattern, built);
+  return built;
+}
+var CONDITIONS = {
+  "file-access": {
+    validate: stringValue("file-access"),
+    test: (value, window) => window.fileAccesses.some((a) => matchesGlob(a.path, value)),
+    reason: (value) => `no file matching "${value}" was accessed`
+  },
+  "tool-used": {
+    validate: stringValue("tool-used"),
+    test: (value, window) => window.toolCalls.some((c) => c.name === value),
+    reason: (value) => `the ${value} tool was never used`
+  },
+  "prompt-matches": {
+    validate: (value) => {
+      const shape = stringValue("prompt-matches")(value);
+      if (shape !== void 0) return shape;
+      try {
+        new RegExp(value);
+      } catch (err) {
+        return `options.when.prompt-matches is not a valid regular expression: ${err.message}`;
+      }
+      return void 0;
+    },
+    test: (value, window) => {
+      const re = promptRe(value);
+      return window.userMessages.some((text) => re.test(text));
+    },
+    reason: (value) => `no prompt matched /${value}/`
+  },
+  "turn-count-above": {
+    validate: (value) => {
+      if (typeof value !== "number" || !Number.isInteger(value)) {
+        return "options.when.turn-count-above must be a whole number";
+      }
+      if (value < 0) return "options.when.turn-count-above must be at least 0";
+      return void 0;
+    },
+    test: (value, window) => window.turnCount > value,
+    reason: (value, window) => `${window.turnCount} turn(s) is not more than ${value}`
+  }
+};
+var WHEN_CONDITIONS = Object.keys(
+  CONDITIONS
+);
+function validateWhen(options) {
+  const when = options.when;
+  if (when === void 0) return void 0;
+  if (typeof when !== "object" || when === null || Array.isArray(when)) {
+    return "options.when must be an object of conditions";
+  }
+  const conditions = when;
+  const keys = Object.keys(conditions);
+  if (keys.length === 0) {
+    return `options.when needs at least one of: ${WHEN_CONDITIONS.join(", ")}`;
+  }
+  for (const key of keys) {
+    const condition = CONDITIONS[key];
+    if (condition === void 0) {
+      return `options.when.${key} is not a known condition; expected one of: ${WHEN_CONDITIONS.join(", ")}`;
+    }
+    const invalid = condition.validate(conditions[key]);
+    if (invalid !== void 0) return invalid;
+  }
+  return void 0;
+}
+function evaluateWhen(options, window) {
+  const when = options.when;
+  if (when === void 0) return { armed: true };
+  for (const name of WHEN_CONDITIONS) {
+    const value = when[name];
+    if (value === void 0) continue;
+    const condition = CONDITIONS[name];
+    if (condition.test(value, window)) continue;
+    return {
+      armed: false,
+      reason: `trigger not met (${window.label}): ${condition.reason(value, window)}`
+    };
+  }
+  return { armed: true };
+}
+function rejectWhen(kind, options) {
+  if (options.when === void 0) return void 0;
+  return `options.when is not supported by the ${kind} grader, which grades the whole session rather than a window`;
+}
+function skippedTrigger(trigger) {
+  return { findings: [], skipped: trigger.reason ?? "trigger not met" };
+}
+
+// src/tracevals/graders/util.ts
+function finding(plan, message) {
+  return {
+    evalName: plan.evalName,
+    artifact: plan.artifact.path,
+    message,
+    severity: plan.severity
+  };
+}
+function fail(plan, message) {
+  return { findings: [finding(plan, message)] };
+}
+var pass = { findings: [] };
+function optionsError(kind, message) {
+  return { findings: [], error: `${kind}: ${message}` };
+}
+function firstError(...checks) {
+  return checks.find((check) => check !== void 0);
+}
+function requiredString(options, key) {
+  const value = options[key];
+  if (typeof value !== "string" || value.length === 0) {
+    return `options.${key} is required`;
+  }
+  return void 0;
+}
+function optionalString(options, key) {
+  const value = options[key];
+  if (value === void 0) return void 0;
+  if (typeof value !== "string" || value.length === 0) {
+    return `options.${key} must be a non-empty string`;
+  }
+  return void 0;
+}
+function optionalEnum(options, key, allowed) {
+  const value = options[key];
+  if (value === void 0) return void 0;
+  if (typeof value !== "string" || !allowed.includes(value)) {
+    return `options.${key} must be one of: ${allowed.join(", ")}`;
+  }
+  return void 0;
+}
+function optionalNumber(options, key, bounds = {}) {
+  const value = options[key];
+  if (value === void 0) return void 0;
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    return `options.${key} must be a finite number`;
+  }
+  if (bounds.integer === true && !Number.isInteger(value)) {
+    return `options.${key} must be a whole number`;
+  }
+  if (bounds.min !== void 0 && value < bounds.min) {
+    return `options.${key} must be at least ${bounds.min}`;
+  }
+  return void 0;
+}
+function optionalBoolean(options, key) {
+  if (options[key] !== void 0 && typeof options[key] !== "boolean") {
+    return `options.${key} must be a boolean`;
+  }
+  return void 0;
+}
+function orderedBounds(options, minKey, maxKey) {
+  const min = options[minKey];
+  const max = options[maxKey];
+  if (typeof min === "number" && typeof max === "number" && min > max) {
+    return `options.${maxKey} must be greater than or equal to options.${minKey}`;
+  }
+  return void 0;
+}
+function requireOneOf(options, keys) {
+  if (keys.some((key) => options[key] !== void 0)) return void 0;
+  return `at least one of ${keys.map((k) => `options.${k}`).join(" or ")} is required`;
+}
+function windowFor(trace, plan) {
+  const { name, type } = plan.artifact;
+  if (type === "skill") return skillWindow(trace, name);
+  if (type === "agent") return agentWindow(trace, name);
+  if (type === "slash-command") return slashCommandWindow(trace, name);
+  return {
+    scope: "session",
+    label: "whole session",
+    empty: false,
+    events: trace.events,
+    toolCalls: trace.toolCalls,
+    fileAccesses: trace.fileAccesses,
+    skillInvocations: trace.skillInvocations,
+    userMessages: trace.userMessages,
+    assistantTexts: trace.assistantTexts,
+    turnCount: trace.turnCount
+  };
+}
+function skippedWindow(window) {
+  return { findings: [], skipped: window.reason ?? "the artifact governed no turns" };
+}
+function skillWindow(trace, name) {
+  return injectionWindow(trace, "skill", `skill "${name}"`, (s) => s.name === name);
+}
+function slashCommandWindow(trace, name) {
+  return injectionWindow(
+    trace,
+    "slash-command",
+    `slash command "/${name}"`,
+    (s) => s.name === name && s.via === "command-injection"
+  );
+}
+function injectionWindow(trace, scope, label, opensHere) {
+  const opens = trace.skillInvocations.filter(opensHere).map((s) => s.index);
+  if (opens.length === 0) {
+    return emptyWindow(
+      scope,
+      label,
+      `${label} was never invoked in this trace, so it governed no turns`
+    );
+  }
+  const chainOf = (index) => trace.events[index]?.branchId;
+  const own = new Set(opens.map(chainOf));
+  const boundaries = trace.skillInvocations.filter((s) => own.has(chainOf(s.index))).map((s) => s.index).sort((a, b) => a - b);
+  const spans = opens.map((start) => ({
+    start,
+    end: boundaries.find((b) => b > start) ?? Number.POSITIVE_INFINITY
+  }));
+  return materialize(trace, scope, label, spans, void 0, own);
+}
+function agentWindow(trace, name) {
+  const label = `agent "${name}"`;
+  const mine = trace.subagentBranches.filter((b) => b.agentType === name);
+  if (mine.length === 0) {
+    const spawned = trace.agentSpawns.some((a) => a.subagentType === name);
+    return emptyWindow(
+      "agent",
+      label,
+      spawned ? `${label} was spawned but recorded no subagent turns (no inline sidechain records and no sidecar transcript), so it governed no turns` : `${label} was never spawned in this trace, so it governed no turns`
+    );
+  }
+  const branchIds = new Set(mine.map((b) => b.branchId));
+  for (const branch of trace.subagentBranches) {
+    if (branchIds.has(branch.branchId)) continue;
+    const nested = mine.some(
+      (m) => m.startIndex <= branch.startIndex && branch.endIndex <= m.endIndex
+    );
+    if (nested) branchIds.add(branch.branchId);
+  }
+  const spans = mine.map((b) => ({ start: b.startIndex, end: b.endIndex }));
+  return materialize(
+    trace,
+    "agent",
+    label,
+    spans,
+    branchIds,
+    new Set(mine.map((b) => b.branchId))
+  );
+}
+function emptyWindow(scope, label, reason) {
+  return {
+    scope,
+    label,
+    empty: true,
+    reason,
+    events: [],
+    toolCalls: [],
+    fileAccesses: [],
+    skillInvocations: [],
+    userMessages: [],
+    assistantTexts: [],
+    turnCount: 0
+  };
+}
+function materialize(trace, scope, label, spans, branchIds, own) {
+  const inSpan = (index) => spans.some((s) => index >= s.start && index < s.end);
+  const excluded = /* @__PURE__ */ new Set();
+  if (branchIds !== void 0) {
+    for (const event of trace.events) {
+      if (!inSpan(event.index)) continue;
+      if (event.branchId === void 0 || !branchIds.has(event.branchId)) {
+        excluded.add(event.index);
+      }
+    }
+  }
+  const inWindow = (index) => inSpan(index) && !excluded.has(index);
+  const events = trace.events.filter((e) => inWindow(e.index));
+  const textOf = (kind) => events.filter((e) => e.kind === kind && own.has(e.branchId) && e.text).map((e) => e.text);
+  const userMessages = textOf("user");
+  return {
+    scope,
+    label,
+    empty: false,
+    events,
+    toolCalls: trace.toolCalls.filter((c) => inWindow(c.index)),
+    fileAccesses: trace.fileAccesses.filter((a) => inWindow(a.index)),
+    skillInvocations: trace.skillInvocations.filter((s) => inWindow(s.index)),
+    userMessages,
+    assistantTexts: textOf("assistant"),
+    turnCount: userMessages.length
+  };
+}
+
+// src/tracevals/graders/tool-usage.ts
+function checkContradiction(options) {
+  const expect = options.expect ?? "used";
+  const { min, max } = options;
+  if (expect === "not-used" && typeof min === "number" && min > 0) {
+    return "options.min is unsatisfiable with expect: not-used";
+  }
+  if (expect === "used" && typeof max === "number" && max === 0) {
+    return "options.max of 0 is unsatisfiable with expect: used";
+  }
+  return void 0;
+}
+function validateOptions(options) {
+  return firstError(
+    requiredString(options, "tool"),
+    optionalEnum(options, "expect", ["used", "not-used"]),
+    optionalNumber(options, "min", { min: 0, integer: true }),
+    optionalNumber(options, "max", { min: 0, integer: true }),
+    orderedBounds(options, "min", "max"),
+    checkContradiction(options),
+    optionalBoolean(options, "includeSidechains"),
+    validateWhen(options)
+  );
+}
+var toolUsageGrader = {
+  kind: "tool-usage",
+  validateOptions,
+  grade({ trace, plan }) {
+    const options = plan.options ?? {};
+    const invalid = validateOptions(options);
+    if (invalid !== void 0) return optionsError("tool-usage", invalid);
+    const tool = options.tool;
+    const expect = options.expect ?? "used";
+    const includeSidechains = options.includeSidechains === true;
+    const window = windowFor(trace, plan);
+    if (window.empty) return skippedWindow(window);
+    const trigger = evaluateWhen(options, window);
+    if (!trigger.armed) return skippedTrigger(trigger);
+    const branchScoped = window.scope === "agent";
+    const count = window.toolCalls.filter(
+      (c) => c.name === tool && (branchScoped || includeSidechains || !c.sidechain)
+    ).length;
+    if (expect === "not-used" && count > 0) {
+      return fail(plan, `tool ${tool} was used ${count} time(s) but must not be`);
+    }
+    if (expect === "used" && count === 0 && options.min === void 0) {
+      return fail(plan, `tool ${tool} was never used`);
+    }
+    if (typeof options.min === "number" && count < options.min) {
+      return fail(
+        plan,
+        `tool ${tool} was used ${count} time(s); at least ${options.min} required`
+      );
+    }
+    if (typeof options.max === "number" && count > options.max) {
+      return fail(
+        plan,
+        `tool ${tool} was used ${count} time(s); at most ${options.max} allowed`
+      );
+    }
+    return pass;
+  }
+};
+
+// src/tracevals/graders/tool-order.ts
+function validateOptions2(options) {
+  return firstError(
+    requiredString(options, "before"),
+    requiredString(options, "after"),
+    optionalString(options, "beforeInputMatch"),
+    optionalString(options, "afterInputMatch"),
+    optionalBoolean(options, "includeSidechains"),
+    validateWhen(options),
+    ...["beforeInputMatch", "afterInputMatch"].map((key) => {
+      const value = options[key];
+      if (typeof value !== "string") return void 0;
+      try {
+        new RegExp(value);
+        return void 0;
+      } catch (err) {
+        return `options.${key} is not a valid regular expression (${err instanceof Error ? err.message : String(err)})`;
+      }
+    })
+  );
+}
+function ordinalsOf(calls, tool, inputMatch) {
+  const re = inputMatch === void 0 ? void 0 : new RegExp(inputMatch);
+  return calls.filter(
+    (c) => c.name === tool && (re === void 0 || re.test(JSON.stringify(c.input)))
+  ).map((c) => c.index);
+}
+function describe(tool, inputMatch) {
+  return inputMatch === void 0 ? tool : `${tool} matching /${inputMatch}/`;
+}
+var toolOrderGrader = {
+  kind: "tool-order",
+  validateOptions: validateOptions2,
+  grade({ trace, plan }) {
+    const options = plan.options ?? {};
+    const invalid = validateOptions2(options);
+    if (invalid !== void 0) return optionsError("tool-order", invalid);
+    const before = options.before;
+    const after = options.after;
+    const beforeMatch = options.beforeInputMatch;
+    const afterMatch = options.afterInputMatch;
+    const includeSidechains = options.includeSidechains === true;
+    const window = windowFor(trace, plan);
+    if (window.empty) return skippedWindow(window);
+    const trigger = evaluateWhen(options, window);
+    if (!trigger.armed) return skippedTrigger(trigger);
+    const branchScoped = window.scope === "agent";
+    const calls = window.toolCalls.filter(
+      (c) => branchScoped || includeSidechains || !c.sidechain
+    );
+    const beforeOrdinals = ordinalsOf(calls, before, beforeMatch);
+    const afterOrdinals = ordinalsOf(calls, after, afterMatch);
+    const firstBefore = beforeOrdinals.length > 0 ? Math.min(...beforeOrdinals) : void 0;
+    const lastAfter = afterOrdinals.length > 0 ? Math.max(...afterOrdinals) : void 0;
+    if (firstBefore === void 0 && lastAfter === void 0) return pass;
+    if (lastAfter === void 0) {
+      return fail(
+        plan,
+        `${describe(after, afterMatch)} was never used, so it could not follow ${describe(before, beforeMatch)}`
+      );
+    }
+    if (firstBefore === void 0) {
+      return fail(
+        plan,
+        `${describe(after, afterMatch)} was used without ${describe(before, beforeMatch)} ever being used first`
+      );
+    }
+    if (firstBefore > lastAfter) {
+      return fail(
+        plan,
+        `${describe(before, beforeMatch)} was only used after ${describe(after, afterMatch)}`
+      );
+    }
+    return pass;
+  }
+};
+
+// src/tracevals/graders/skill-invoked.ts
+function validateOptions3(options) {
+  return firstError(
+    requiredString(options, "skill"),
+    optionalEnum(options, "expect", ["used", "not-used"]),
+    validateWhen(options)
+  );
+}
+var skillInvokedGrader = {
+  kind: "skill-invoked",
+  validateOptions: validateOptions3,
+  grade({ trace, plan }) {
+    const options = plan.options ?? {};
+    const invalid = validateOptions3(options);
+    if (invalid !== void 0) return optionsError("skill-invoked", invalid);
+    const skill = options.skill;
+    const expect = options.expect ?? "used";
+    const window = windowFor(trace, plan);
+    if (window.empty) return skippedWindow(window);
+    const trigger = evaluateWhen(options, window);
+    if (!trigger.armed) return skippedTrigger(trigger);
+    const used = window.skillInvocations.some((s) => s.name === skill);
+    if (expect === "used" && !used) {
+      return fail(plan, `skill ${skill} was never invoked`);
+    }
+    if (expect === "not-used" && used) {
+      return fail(plan, `skill ${skill} was invoked but must not be`);
+    }
+    return pass;
+  }
+};
+
+// src/tracevals/graders/file-access.ts
+var OPS = ["read", "write", "edit"];
+function validateOptions4(options) {
+  return firstError(
+    requiredString(options, "path"),
+    optionalEnum(options, "op", OPS),
+    optionalEnum(options, "expect", ["accessed", "not-accessed"]),
+    validateWhen(options)
+  );
+}
+function matches(accessPath, spec) {
+  return matchesGlob(accessPath, spec);
+}
+var fileAccessGrader = {
+  kind: "file-access",
+  validateOptions: validateOptions4,
+  grade({ trace, plan }) {
+    const options = plan.options ?? {};
+    const invalid = validateOptions4(options);
+    if (invalid !== void 0) return optionsError("file-access", invalid);
+    const path = options.path;
+    const op = options.op;
+    const expect = options.expect ?? "accessed";
+    const window = windowFor(trace, plan);
+    if (window.empty) return skippedWindow(window);
+    const trigger = evaluateWhen(options, window);
+    if (!trigger.armed) return skippedTrigger(trigger);
+    const accessed = window.fileAccesses.some(
+      (a) => matches(a.path, path) && (op === void 0 || a.op === op)
+    );
+    const opLabel = op ?? "any";
+    if (expect === "accessed" && !accessed) {
+      return fail(plan, `file ${path} was never accessed (op: ${opLabel})`);
+    }
+    if (expect === "not-accessed" && accessed) {
+      return fail(plan, `file ${path} was accessed (op: ${opLabel}) but must not be`);
+    }
+    return pass;
+  }
+};
+
+// src/tracevals/graders/turn-count.ts
+function validateOptions5(options) {
+  return firstError(
+    requireOneOf(options, ["min", "max"]),
+    optionalNumber(options, "min", { min: 0, integer: true }),
+    optionalNumber(options, "max", { min: 0, integer: true }),
+    orderedBounds(options, "min", "max"),
+    validateWhen(options)
+  );
+}
+var turnCountGrader = {
+  kind: "turn-count",
+  validateOptions: validateOptions5,
+  grade({ trace, plan }) {
+    const options = plan.options ?? {};
+    const invalid = validateOptions5(options);
+    if (invalid !== void 0) return optionsError("turn-count", invalid);
+    const window = windowFor(trace, plan);
+    if (window.empty) return skippedWindow(window);
+    const trigger = evaluateWhen(options, window);
+    if (!trigger.armed) return skippedTrigger(trigger);
+    const { turnCount } = window;
+    if (typeof options.max === "number" && turnCount > options.max) {
+      return fail(plan, `${turnCount} turn(s); at most ${options.max} allowed`);
+    }
+    if (typeof options.min === "number" && turnCount < options.min) {
+      return fail(plan, `${turnCount} turn(s); at least ${options.min} required`);
+    }
+    return pass;
+  }
+};
+
+// src/tracevals/graders/cost.ts
+function validateOptions6(options) {
+  return firstError(
+    requireOneOf(options, ["maxUsd", "maxTokens"]),
+    optionalNumber(options, "maxUsd", { min: 0 }),
+    optionalNumber(options, "maxTokens", { min: 0 }),
+    rejectWhen("cost", options)
+  );
+}
+var costGrader = {
+  kind: "cost",
+  validateOptions: validateOptions6,
+  grade({ trace, plan }) {
+    const options = plan.options ?? {};
+    const invalid = validateOptions6(options);
+    if (invalid !== void 0) return optionsError("cost", invalid);
+    const usage = trace.usage;
+    const skips = [];
+    let checked = false;
+    if (typeof options.maxUsd === "number") {
+      if (usage?.totalCostUsd === void 0) {
+        skips.push("no cost data (maxUsd not checkable)");
+      } else {
+        checked = true;
+        if (usage.totalCostUsd > options.maxUsd) {
+          return fail(
+            plan,
+            `session cost $${usage.totalCostUsd.toFixed(4)}; at most $${options.maxUsd} allowed`
+          );
+        }
+      }
+    }
+    if (typeof options.maxTokens === "number") {
+      if (usage === void 0) {
+        skips.push("no usage data (maxTokens not checkable)");
+      } else {
+        checked = true;
+        const total = usage.inputTokens + usage.outputTokens;
+        if (total > options.maxTokens) {
+          return fail(
+            plan,
+            `session used ${total} tokens; at most ${options.maxTokens} allowed`
+          );
+        }
+      }
+    }
+    if (!checked && skips.length > 0) {
+      return {
+        findings: [],
+        skipped: `trace carries insufficient cost data: ${skips.join("; ")}`
+      };
+    }
+    return pass;
+  }
+};
+
+// src/tracevals/core/target.ts
+import { readFileSync } from "node:fs";
+import { isAbsolute as isAbsolute2, relative as relative2, resolve as resolve2, sep } from "node:path";
+var DEFAULT_TARGET = "transcript";
+function describeTarget(target) {
+  const t = target ?? DEFAULT_TARGET;
+  return typeof t === "string" ? t : `file ${t.path}`;
+}
+function readTarget(target, ctx) {
+  const t = target ?? DEFAULT_TARGET;
+  if (t === "transcript") {
+    return { ok: true, text: ctx.renderedTrace, label: "transcript" };
+  }
+  if (t === "last-message") {
+    const last = ctx.trace.assistantTexts.at(-1);
+    return { ok: true, text: last ?? "", label: "last-message" };
+  }
+  if (t === "files") {
+    const paths = [...new Set(ctx.trace.fileAccesses.map((f) => f.path))];
+    return { ok: true, text: paths.join("\n"), label: "files" };
+  }
+  if (t === "artifact") {
+    return { ok: true, text: ctx.artifactContent, label: "artifact" };
+  }
+  if (isAbsolute2(t.path)) {
+    return {
+      ok: false,
+      reason: `target file "${t.path}" is an absolute path; name it relative to the project root`
+    };
+  }
+  const abs = resolve2(ctx.root, t.path);
+  const rel = relative2(ctx.root, abs);
+  if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute2(rel)) {
+    return {
+      ok: false,
+      reason: `target file "${t.path}" resolves outside the project root`
+    };
+  }
+  try {
+    return { ok: true, text: readFileSync(abs, "utf8"), label: `file ${t.path}` };
+  } catch (err) {
+    const code = err.code ?? "unknown error";
+    return { ok: false, reason: `target file "${t.path}" could not be read (${code})` };
+  }
+}
+
+// src/tracevals/graders/regex.ts
+function checkFlags(options) {
+  const flags = options.flags;
+  if (flags === void 0) return void 0;
+  if (typeof flags !== "string") return "options.flags must be a string";
+  try {
+    new RegExp("", flags);
+  } catch (err) {
+    return `options.flags is invalid: ${err.message}`;
+  }
+  return void 0;
+}
+function checkPattern(options) {
+  const required = requiredString(options, "pattern");
+  if (required !== void 0) return required;
+  try {
+    new RegExp(
+      options.pattern,
+      options.flags ?? ""
+    );
+  } catch (err) {
+    return `options.pattern is not a valid regular expression: ${err.message}`;
+  }
+  return void 0;
+}
+function validateOptions7(options) {
+  return firstError(
+    checkFlags(options),
+    checkPattern(options),
+    optionalEnum(options, "on", ["assistant", "user", "all"]),
+    optionalEnum(options, "expect", ["match", "no-match"]),
+    validateWhen(options)
+  );
+}
+var regexGrader = {
+  kind: "regex",
+  validateOptions: validateOptions7,
+  grade({ trace, plan, projectRoot }) {
+    const options = plan.options ?? {};
+    const invalid = validateOptions7(options);
+    if (invalid !== void 0) return optionsError("regex", invalid);
+    const pattern = options.pattern;
+    const re = new RegExp(pattern, options.flags ?? "");
+    const on = options.on ?? "assistant";
+    const expect = options.expect ?? "match";
+    const window = windowFor(trace, plan);
+    if (window.empty) return skippedWindow(window);
+    const trigger = evaluateWhen(options, window);
+    if (!trigger.armed) return skippedTrigger(trigger);
+    const target = plan.target ?? "transcript";
+    let corpus;
+    let where;
+    if (target === "transcript") {
+      corpus = on === "user" ? window.userMessages : on === "all" ? [...window.userMessages, ...window.assistantTexts] : window.assistantTexts;
+      where = `${on} text`;
+    } else {
+      const selected = readTarget(target, {
+        trace,
+        // Never consulted: `transcript` took the branch above.
+        renderedTrace: "",
+        artifactContent: plan.artifact.content,
+        root: projectRoot ?? trace.cwd
+      });
+      if (!selected.ok) return optionsError("regex", selected.reason);
+      corpus = [selected.text];
+      where = selected.label;
+    }
+    const matched = corpus.some((text) => re.test(text));
+    if (expect === "match" && !matched) {
+      return fail(plan, `pattern /${pattern}/ never matched ${where}`);
+    }
+    if (expect === "no-match" && matched) {
+      return fail(plan, `pattern /${pattern}/ matched ${where} but must not`);
+    }
+    return pass;
+  }
+};
+
+// src/tracevals/graders/json-output.ts
+import { Ajv2020 } from "ajv/dist/2020.js";
+var compiled2 = /* @__PURE__ */ new Map();
+function compileSchema(schema) {
+  const key = JSON.stringify(schema);
+  const hit = compiled2.get(key);
+  if (hit !== void 0) return hit;
+  const built = new Ajv2020({ allErrors: true, strict: false }).compile(schema);
+  compiled2.set(key, built);
+  return built;
+}
+function validateOptions8(options) {
+  const schema = options.schema;
+  if (schema === void 0 || schema === null || typeof schema !== "object" || Array.isArray(schema)) {
+    return "options.schema is required and must be a JSON Schema object";
+  }
+  return rejectWhen("json-output", options);
+}
+var jsonOutputGrader = {
+  kind: "json-output",
+  validateOptions: validateOptions8,
+  grade({ trace, plan }) {
+    const options = plan.options ?? {};
+    const invalid = validateOptions8(options);
+    if (invalid !== void 0) return optionsError("json-output", invalid);
+    const schema = options.schema;
+    const last = trace.assistantTexts.at(-1);
+    if (last === void 0) {
+      return fail(plan, "session produced no assistant text to validate");
+    }
+    let data;
+    try {
+      data = JSON.parse(last);
+    } catch {
+      return fail(plan, "final assistant output is not valid JSON");
+    }
+    let validate;
+    try {
+      validate = compileSchema(schema);
+    } catch (err) {
+      return optionsError(
+        "json-output",
+        `schema failed to compile: ${err.message}`
+      );
+    }
+    if (validate(data)) return pass;
+    return {
+      findings: (validate.errors ?? []).map(
+        (e) => finding(plan, `${e.instancePath || "/"} ${e.message ?? "invalid"}`)
+      )
+    };
+  }
+};
+
+// src/tracevals/graders/command.ts
+import { createHash } from "node:crypto";
+
+// src/shared/exec.ts
+import { realExec } from "@hawkeyexl/inference";
+function outputTail(result, maxChars = 400) {
+  const text = (result.stderr.trim() || result.stdout.trim()).trim();
+  if (text.length <= maxChars) return text;
+  return `\u2026${text.slice(-maxChars)}`;
+}
+
+// src/tracevals/graders/command.ts
+var DEFAULT_TIMEOUT_MS = 3e4;
+var STDERR_LIMIT = 2e3;
+var commandGrader = {
+  kind: "command",
+  async grade(ctx) {
+    const { plan, trace } = ctx;
+    if (!plan.command || plan.command.length === 0) {
+      return {
+        findings: [],
+        error: "command-graded eval has no `command`; add one (or remove the eval) \u2014 manni tracevals does not generate check scripts"
+      };
+    }
+    if (plan.generatedAssertionHash !== void 0) {
+      if (plan.assertion === void 0) {
+        return {
+          findings: [],
+          error: "generated-assertion-hash is present but the eval has no assertion, so the hash cannot be checked against anything; remove the hash or restore the assertion"
+        };
+      }
+      const actual = createHash("sha256").update(plan.assertion).digest("hex");
+      if (actual !== plan.generatedAssertionHash) {
+        return {
+          findings: [],
+          error: `assertion has changed since the command was generated (generated-assertion-hash ${plan.generatedAssertionHash.slice(0, 12)}\u2026, assertion now hashes to ${actual.slice(0, 12)}\u2026); regenerate the command or update the hash`
+        };
+      }
+    }
+    const argv = plan.command.map((part) => part.replaceAll("{trace}", trace.file));
+    const timeoutMs = plan.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const successCodes = plan.successExitCodes ?? [0];
+    const run = await execute(argv, ctx.projectRoot ?? trace.cwd, timeoutMs);
+    if (run.timedOut) {
+      return {
+        findings: [],
+        error: `command timed out after ${timeoutMs}ms: ${argv.join(" ")}`
+      };
+    }
+    if (run.spawnError !== void 0) {
+      return { findings: [], error: `could not run command: ${run.spawnError}` };
+    }
+    if (run.code !== null && successCodes.includes(run.code)) {
+      return { findings: [] };
+    }
+    const detail2 = run.stderr.trim();
+    return {
+      findings: [
+        {
+          evalName: plan.evalName,
+          artifact: plan.artifact.path,
+          severity: plan.severity,
+          message: `command exited ${run.code ?? "on a signal"} (expected ${successCodes.join(" or ")})` + (detail2 ? `: ${detail2.slice(0, STDERR_LIMIT)}` : "")
+        }
+      ]
+    };
+  }
+};
+async function execute(argv, cwd, timeoutMs) {
+  const result = await realExec(argv, { cwd, timeoutMs });
+  return {
+    code: result.code,
+    // Capped here, not merely when printed: a check script can write far more
+    // to stderr than a report should carry.
+    stderr: result.stderr.slice(0, STDERR_LIMIT),
+    timedOut: result.timedOut,
+    ...result.spawnError === void 0 ? {} : { spawnError: result.spawnError }
+  };
+}
+
+// src/tracevals/graders/registry.ts
+var graders = new Map(
+  [
+    toolUsageGrader,
+    toolOrderGrader,
+    skillInvokedGrader,
+    fileAccessGrader,
+    turnCountGrader,
+    costGrader,
+    regexGrader,
+    jsonOutputGrader,
+    commandGrader
+  ].map((g) => [g.kind, g])
+);
+var BUILTIN_GRADER_KINDS = new Set(graders.keys());
+function registerGrader(grader) {
+  graders.set(grader.kind, grader);
+}
+function graderFor(kind) {
+  return graders.get(kind);
+}
+function listGraderKinds() {
+  return [...graders.keys()];
+}
+
+// src/tracevals/judge/render.ts
+var DEFAULTS = {
+  maxBlockChars: 2e3,
+  maxTotalChars: 15e4,
+  redact: []
+};
+var MIN_TIMELINE_CHARS = 200;
+var HEADER_TRUNCATED = "\n[... header truncated ...]";
+var TIMELINE_SHELL = "\n\n## Timeline\n";
+function renderTrace(trace, options = {}, plan) {
+  const { maxBlockChars, maxTotalChars, redact } = { ...DEFAULTS, ...options };
+  const scrub = makeRedactor(redact);
+  const clip = (text, max) => {
+    const safe = scrub(text);
+    if (safe.length <= max) return safe;
+    return `${safe.slice(0, max)} [... truncated ${safe.length - max} chars ...]`;
+  };
+  const window = plan === void 0 ? void 0 : windowFor(trace, plan);
+  const scoped = window !== void 0 && window.scope !== "session" ? window : void 0;
+  const events = scoped?.events ?? trace.events;
+  const fullHeader = scrub(
+    [
+      "# Session",
+      `source: ${trace.source}`,
+      trace.model ? `model: ${trace.model}` : void 0,
+      `cwd: ${trace.cwd}`,
+      trace.gitBranch ? `branch: ${trace.gitBranch}` : void 0,
+      scoped ? `scope: ${scoped.label} \u2014 ${events.length} of ${trace.events.length} session events` : void 0,
+      // Everything below describes what the judge can actually see. On a
+      // scoped render the timeline is the window, so a session-wide count here
+      // reads as evidence that the rest was truncated away — the judge would
+      // discount a digest that is complete for what it was asked about. The
+      // session totals stay, in parentheses, because "3 of 87" is context the
+      // window count alone does not carry.
+      scoped ? `turns: ${scoped.turnCount} (of ${trace.turnCount} in the session)` : `turns: ${trace.turnCount}`,
+      (() => {
+        const used = (scoped ?? trace).skillInvocations.map((s) => s.name);
+        return used.length > 0 ? `skills used: ${used.join(", ")}` : "skills used: none";
+      })(),
+      (() => {
+        const inWindow = new Set(events.map((e) => e.index));
+        const spawns = trace.agentSpawns.filter(
+          (a) => !scoped || inWindow.has(a.index)
+        );
+        return spawns.length > 0 ? `agents spawned: ${spawns.map((a) => a.subagentType).join(", ")}` : void 0;
+      })(),
+      (() => {
+        const inWindow = new Set(events.map((e) => e.index));
+        const branches = trace.subagentBranches.filter(
+          (b) => !scoped || inWindow.has(b.spawnIndex)
+        );
+        return branches.length > 0 ? `subagent transcripts: ${branches.map((b) => `${b.agentType} (depth ${b.spawnDepth})`).join(", ")}` : void 0;
+      })()
+    ].filter((line) => line !== void 0).join("\n")
+  );
+  const headerRoom = Math.max(
+    maxTotalChars - TIMELINE_SHELL.length - MIN_TIMELINE_CHARS,
+    0
+  );
+  const header = fullHeader.length <= headerRoom ? fullHeader : fullHeader.slice(
+    0,
+    Math.max(headerRoom - HEADER_TRUNCATED.length, 0)
+  ) + HEADER_TRUNCATED;
+  const branchTypes = /* @__PURE__ */ new Map();
+  for (const spawn of trace.agentSpawns) {
+    if (spawn.toolUseId) branchTypes.set(spawn.toolUseId, spawn.subagentType);
+  }
+  for (const branch2 of trace.subagentBranches) {
+    branchTypes.set(branch2.branchId, branch2.agentType);
+  }
+  const label = (branchId) => branchTypes.get(branchId) ?? branchId;
+  const callAt = new Map(trace.toolCalls.map((call) => [call.index, call]));
+  if (scoped?.empty === true) {
+    return `${header}${TIMELINE_SHELL}[no turns: ${scrub(scoped.reason ?? "")}]`;
+  }
+  const lines = [];
+  let branch;
+  for (const event of events) {
+    const tag = event.branchId ? `:${label(event.branchId)}` : event.sidechain ? ":sidechain" : "";
+    let line;
+    switch (event.kind) {
+      case "user":
+        if (event.text) line = `[user${tag}] ${clip(event.text, maxBlockChars)}`;
+        break;
+      case "assistant":
+        if (event.text) {
+          line = `[assistant${tag}] ${clip(event.text, maxBlockChars)}`;
+        }
+        break;
+      case "tool_call": {
+        const call = callAt.get(event.index);
+        const input = call ? JSON.stringify(call.input) : "";
+        line = `[tool${tag}] ${event.toolName ?? call?.name ?? "?"} ${clip(input, Math.min(maxBlockChars, 500))}`;
+        break;
+      }
+      default:
+        break;
+    }
+    if (line === void 0) continue;
+    if (event.branchId !== branch) {
+      if (branch !== void 0) lines.push(`[/subagent ${label(branch)}]`);
+      branch = event.branchId;
+      if (branch !== void 0) {
+        lines.push(`[subagent ${label(branch)} (${branch})]`);
+      }
+    }
+    lines.push(line);
+  }
+  if (branch !== void 0) lines.push(`[/subagent ${label(branch)}]`);
+  let timeline = scrub(lines.join("\n"));
+  const budget = Math.max(
+    maxTotalChars - header.length - TIMELINE_SHELL.length,
+    MIN_TIMELINE_CHARS
+  );
+  if (timeline.length > budget) {
+    const marker = (omitted2) => `
+[... truncated ${omitted2} chars of transcript ...]
+`;
+    const room = Math.max(budget - marker(timeline.length).length, 0);
+    const headLen = Math.floor(room * 0.6);
+    const tailLen = room - headLen;
+    const omitted = timeline.length - headLen - tailLen;
+    timeline = `${timeline.slice(0, headLen)}${marker(omitted)}${tailLen > 0 ? timeline.slice(-tailLen) : ""}`;
+  }
+  return `${header}${TIMELINE_SHELL}${timeline}`;
+}
+
+// src/tracevals/core/engine.ts
+async function runEvals(options) {
+  const start = Date.now();
+  const { config } = options;
+  const trace = await parseTraceFile(options.tracePath);
+  const refusals = [];
+  const found = await findManifest({
+    tracePath: options.tracePath,
+    ...trace.sessionId !== void 0 ? { sessionId: trace.sessionId } : {},
+    projectDir: options.projectDir ?? trace.cwd,
+    captureDir: config.capture.dir,
+    ...options.manifest !== void 0 ? { explicit: options.manifest } : {},
+    onRefused: (path, reason) => refusals.push({ path, reason })
+  });
+  if (found === null && options.manifest !== void 0) {
+    throw new TracevalsError(
+      `no usable session manifest at ${options.manifest} \u2014 ${refusals[0]?.reason ?? "it is missing, unreadable, or was recorded for a different session"}`
+    );
+  }
+  const manifestWarnings = found === null ? refusals.map(
+    (r) => `ignoring the session manifest at ${r.path} \u2014 ${r.reason}`
+  ) : [];
+  const resolved = await resolveArtifacts(trace, {
+    ...options.projectDir !== void 0 ? { projectDir: options.projectDir, projectRoot: options.projectDir } : {},
+    ...options.env !== void 0 ? { env: options.env } : {},
+    reportUnusedArtifacts: config.reportUnusedArtifacts,
+    ...found !== null ? { manifest: found.manifest } : {}
+  });
+  const loader = options.metadataFor;
+  const metadata = /* @__PURE__ */ new Map();
+  if (loader !== void 0) {
+    for (const artifact of resolved.artifacts) {
+      metadata.set(artifact, await loader(artifact));
+    }
+  }
+  const plans = planEvals(
+    resolved.artifacts,
+    loader === void 0 ? void 0 : (artifact) => metadata.get(artifact)
+  );
+  const results = [];
+  const aiPlans = [];
+  const root = normalizeRoot(process.cwd());
+  for (const plan of plans) {
+    const base = {
+      evalName: plan.evalName,
+      artifact: plan.artifact.path,
+      artifactName: plan.artifact.name,
+      artifactType: plan.artifact.type,
+      grader: plan.grader,
+      implicit: plan.implicit,
+      location: spelled(plan.location, root)
+    };
+    if (plan.error !== void 0) {
+      results.push({ ...base, outcome: "error", error: plan.error, durationMs: 0 });
+      continue;
+    }
+    if (plan.skipped) {
+      results.push({
+        ...base,
+        outcome: "skipped",
+        skipReason: plan.skipReason ?? "eval skipped",
+        durationMs: 0
+      });
+      continue;
+    }
+    if (plan.grader === "ai" || plan.grader === "human") {
+      const window = windowFor(trace, plan);
+      if (window.empty) {
+        results.push({
+          ...base,
+          outcome: "skipped",
+          skipReason: window.reason ?? "the artifact governed no turns",
+          durationMs: 0
+        });
+        continue;
+      }
+    }
+    if (plan.grader === "ai") {
+      aiPlans.push(plan);
+      continue;
+    }
+    if (plan.grader === "human") {
+      results.push({
+        ...base,
+        outcome: "needs-review",
+        skipReason: "awaiting human review",
+        durationMs: 0
+      });
+      continue;
+    }
+    if (plan.grader === "command" && !config.execution.allow.includes("frontmatter-commands")) {
+      results.push({
+        ...base,
+        outcome: "skipped",
+        skipReason: NOT_GRANTED_REASON,
+        durationMs: 0
+      });
+      continue;
+    }
+    const grader = graderFor(plan.grader);
+    if (!grader) {
+      results.push({
+        ...base,
+        outcome: "error",
+        error: `unknown grader kind "${plan.grader}"; the grader registry knows ${listGraderKinds().sort().join(", ")}, "ai", and "human"`,
+        durationMs: 0
+      });
+      continue;
+    }
+    const graded = Date.now();
+    let result;
+    try {
+      result = await grader.grade({
+        trace,
+        plan,
+        ...options.projectDir !== void 0 ? { projectRoot: options.projectDir } : {}
+      });
+    } catch (err) {
+      results.push({
+        ...base,
+        outcome: "error",
+        error: `${plan.grader}: ${err instanceof Error ? err.message : String(err)}`,
+        durationMs: Date.now() - graded
+      });
+      continue;
+    }
+    if (result.error !== void 0) {
+      results.push({
+        ...base,
+        outcome: "error",
+        error: result.error,
+        durationMs: Date.now() - graded
+      });
+      continue;
+    }
+    if (result.skipped !== void 0) {
+      results.push({
+        ...base,
+        outcome: "skipped",
+        skipReason: result.skipped,
+        durationMs: Date.now() - graded
+      });
+      continue;
+    }
+    const failing2 = result.findings.some((f) => f.severity === "error");
+    results.push({
+      ...base,
+      outcome: failing2 ? "fail" : "pass",
+      findings: result.findings,
+      durationMs: Date.now() - graded
+    });
+  }
+  if (aiPlans.length > 0) {
+    if (options.deterministicOnly || options.judge === void 0) {
+      const skipReason = options.deterministicOnly ? "ai evals skipped (deterministic-only run)" : "ai evals skipped (no judge provided)";
+      for (const plan of aiPlans) {
+        results.push({
+          evalName: plan.evalName,
+          artifact: plan.artifact.path,
+          artifactName: plan.artifact.name,
+          artifactType: plan.artifact.type,
+          grader: plan.grader,
+          implicit: plan.implicit,
+          location: spelled(plan.location, root),
+          outcome: "skipped",
+          skipReason,
+          durationMs: 0
+        });
+      }
+    } else {
+      const judged = await options.judge(
+        aiPlans,
+        (plan) => renderTrace(
+          trace,
+          { ...config.render, redact: config.judge.redact },
+          plan
+        ),
+        {
+          trace,
+          ...options.projectDir !== void 0 ? { projectRoot: options.projectDir } : {}
+        }
+      );
+      judged.forEach((j, i) => {
+        const plan = aiPlans[i];
+        results.push({
+          ...j.selfPreference ? { selfPreference: j.selfPreference } : {},
+          evalName: j.evalName,
+          artifact: j.artifact,
+          artifactName: j.artifactName,
+          artifactType: plan?.artifact.type ?? "skill",
+          grader: j.grader,
+          implicit: j.implicit,
+          location: spelled(plan?.location ?? { file: j.artifact }, root),
+          outcome: j.outcome,
+          ...j.consensus !== void 0 ? { consensus: j.consensus } : {},
+          ...j.skipReason !== void 0 ? { skipReason: j.skipReason } : {},
+          ...j.error !== void 0 ? { error: j.error } : {},
+          turns: j.turns,
+          durationMs: j.durationMs
+        });
+      });
+    }
+  }
+  const weightOf = /* @__PURE__ */ new Map();
+  for (const plan of plans) {
+    weightOf.set(weightKey(plan.artifact.path, plan.evalName), plan.weight ?? 1);
+  }
+  for (const r of results) {
+    r.weight = weightOf.get(weightKey(r.artifact, r.evalName)) ?? 1;
+  }
+  const gradedRows = results.filter(
+    (r) => r.outcome === "pass" || r.outcome === "fail" || r.outcome === "error"
+  );
+  const gradedWeight = gradedRows.reduce((n, r) => n + (r.weight ?? 1), 0);
+  const summary = {
+    total: results.length,
+    pass: results.filter((r) => r.outcome === "pass").length,
+    fail: results.filter((r) => r.outcome === "fail").length,
+    error: results.filter((r) => r.outcome === "error").length,
+    needsReview: results.filter((r) => r.outcome === "needs-review").length,
+    skipped: results.filter((r) => r.outcome === "skipped").length,
+    passRate: gradedWeight > 0 ? gradedRows.filter((r) => r.outcome === "pass").reduce((n, r) => n + (r.weight ?? 1), 0) / gradedWeight : 1
+  };
+  const failing = summary.fail > 0 || summary.error > 0 || config.failOnReview && summary.needsReview > 0;
+  return {
+    trace: {
+      file: trace.file,
+      source: trace.source,
+      ...trace.sessionId !== void 0 ? { sessionId: trace.sessionId } : {},
+      cwd: trace.cwd,
+      ...trace.model !== void 0 ? { model: trace.model } : {},
+      turnCount: trace.turnCount
+    },
+    warnings: [
+      ...options.warnings ?? [],
+      ...trace.warnings,
+      ...manifestWarnings,
+      ...resolved.warnings,
+      ...vanityMetricWarnings(plans),
+      ...truncatedArtifactWarnings(plans)
+    ],
+    coverage: resolved.coverage,
+    // An observation about the session's configuration, never a verdict: it is
+    // deliberately not part of `summary` and never moves `exitCode`.
+    availability: resolved.availability,
+    ...found !== null ? { manifest: summariseManifest(found, resolved.coverage) } : {},
+    evalResults: results,
+    summary,
+    exitCode: failing ? 1 : 0,
+    turns: results.reduce((sum, r) => sum + (r.turns ?? 0), 0),
+    durationMs: Date.now() - start
+  };
+}
+function spelled(location, root) {
+  if (classifyRef(location.file).kind === "url") return location;
+  const posix = toPosix(location.file);
+  return { ...location, file: underRoot(posix, root) ?? posix };
+}
+function weightKey(artifact, evalName) {
+  return `${artifact}\0${evalName}`;
+}
+function vanityMetricWarnings(plans) {
+  const byArtifact = /* @__PURE__ */ new Map();
+  for (const plan of plans) {
+    if (plan.implicit) continue;
+    const list = byArtifact.get(plan.artifact.path) ?? [];
+    list.push(plan);
+    byArtifact.set(plan.artifact.path, list);
+  }
+  const warnings = [];
+  for (const [path, group] of byArtifact) {
+    if (group.length > 0 && group.every((p) => p.grader === "skill-invoked")) {
+      warnings.push(
+        `${path}: every eval here is skill-invoked, which checks that the skill fired, not that the session followed it. Add at least one eval about what the session actually did.`
+      );
+    }
+  }
+  return warnings;
+}
+function truncatedArtifactWarnings(plans) {
+  const seen = /* @__PURE__ */ new Set();
+  const warnings = [];
+  for (const plan of plans) {
+    if (plan.grader !== "ai") continue;
+    const path = plan.artifact.path;
+    if (seen.has(path)) continue;
+    if (!artifactWasTruncated(plan.artifact.content)) continue;
+    seen.add(path);
+    warnings.push(
+      `${path}: only the first ${String(MAX_ARTIFACT_CHARS)} characters were shown to the judge (the artifact is ${String(plan.artifact.content.length)}). Verdicts about instructions beyond that point rest on text the judge never read.`
+    );
+  }
+  return warnings;
+}
+function summariseManifest(found, coverage) {
+  const checked = coverage.filter((c) => c.contentCheck !== void 0);
+  return {
+    path: found.path,
+    sessionId: found.manifest.sessionId,
+    capturedAt: found.manifest.capturedAt,
+    ...found.manifest.git !== void 0 ? { gitSha: found.manifest.git.sha } : {},
+    matched: checked.filter((c) => c.contentCheck?.status === "match").length,
+    changed: checked.filter((c) => c.contentCheck?.status === "mismatch").length,
+    unrecorded: checked.filter((c) => c.contentCheck?.status === "skipped").length
+  };
+}
+
+// src/tracevals/evals/external.ts
+import { relative as relative3, resolve as resolve3, sep as sep2 } from "node:path";
+var METADATA_KEY = "metadata";
+var MARKED = /* @__PURE__ */ new Set([METADATA_KEY]);
+var toPosix2 = (path) => path.split(sep2).join("/");
+function twoManifestsRefusal(label, a, b) {
+  return `${label} is in collections ${a} and ${b}, and both keep ${METADATA_KEY} in a manifest.`;
+}
+function urlManifestRefusal(label, file) {
+  return `${label}: ${METADATA_KEY} is owned by ${file}, and a URL manifest cannot be written. Vendor it to a path, or run fill with --dry-run.`;
+}
+function ownsMetadata(collection, manifest) {
+  return ownsKey(collection, manifest, METADATA_KEY, MARKED);
+}
+function loadExternalEvals(opts) {
+  const scoped = [];
+  for (const collection of opts.collections) {
+    const owning = collection.externalMetadata.filter((m) => ownsMetadata(collection, m));
+    if (owning.length === 0) continue;
+    scoped.push({ ...collection, externalMetadata: owning });
+  }
+  return Promise.resolve(scoped.length === 0 ? null : reader(scoped, opts));
+}
+function reader(scoped, opts) {
+  const { configDir } = opts;
+  const base = configDir;
+  const declared = opts.collections;
+  const marks = familyMarks({ configPath: opts.configPath, configDir, cwd: base });
+  const perPage = scoped.some(
+    (c) => c.externalMetadata.some((m) => hasPagePlaceholder(m.file))
+  );
+  const load = async (pages) => {
+    try {
+      return await loadExternalMetadata(scoped, {
+        configDir,
+        base,
+        ...opts.offline === void 0 ? {} : { offline: opts.offline },
+        ...pages === void 0 ? {} : { pages }
+      });
+    } catch (err) {
+      if (err instanceof DocmetaError) throw new TracevalsError(err.message);
+      throw err;
+    }
+  };
+  let shared;
+  const own = /* @__PURE__ */ new Map();
+  const indexFor = (abs) => {
+    if (!perPage) return shared ??= load();
+    let index = own.get(abs);
+    if (index === void 0) {
+      index = load([abs]);
+      own.set(abs, index);
+    }
+    return index;
+  };
+  const forArtifact = async (artifact) => {
+    const label = artifact.path;
+    const extracted = extractFrontmatter(artifact.content, "markdown");
+    const members = memberOf(declared, configDir, base, label);
+    const [first, second] = scoped.filter((c) => members.includes(c.name));
+    if (first === void 0) return { extracted };
+    if (second !== void 0) {
+      throw new TracevalsError(twoManifestsRefusal(label, first.name, second.name));
+    }
+    const merged = await mergeWithMarks(
+      label,
+      extracted,
+      await indexFor(resolve3(base, label)),
+      [first.name],
+      base,
+      { encryptionKey: () => opts.key, marks: await marks() }
+    );
+    const locate2 = (pointer) => {
+      const found = merged.locate(pointer);
+      if (found === void 0 || classifyRef(found.file).kind === "url") return found;
+      return { ...found, file: resolve3(base, found.file) };
+    };
+    const at = merged.locate(`/${METADATA_KEY}`);
+    const [declaration] = first.externalMetadata;
+    if (at === void 0 || declaration === void 0) {
+      return { extracted: merged.extracted, locate: locate2 };
+    }
+    const url = classifyRef(at.file).kind === "url";
+    const owner = {
+      collection: first.name,
+      path: url ? at.file : resolve3(base, at.file),
+      file: at.file,
+      join: externalMetadataJoin(declaration),
+      url
+    };
+    const out = { extracted: merged.extracted, locate: locate2, owner };
+    if (owner.join === PATH_JOIN) {
+      out.entry = toPosix2(relative3(configDir, resolve3(base, label)));
+    } else {
+      const value = merged.joins.find((j) => j.field === owner.join)?.value ?? extracted.data[owner.join];
+      if (typeof value === "string" && value !== "") out.entry = value;
+      else if (typeof value === "number" || typeof value === "boolean") {
+        out.entry = String(value);
+      }
+    }
+    return out;
+  };
+  return { configDir, forArtifact };
+}
+
+// src/tracevals/graders/plugins.ts
+import { createRequire } from "node:module";
+import { isAbsolute as isAbsolute3, join as join2, resolve as resolve4 } from "node:path";
+import { pathToFileURL } from "node:url";
+var alreadyLoaded = /* @__PURE__ */ new Set();
+var RELATIVE = /^\.\.?[/\\]/;
+function locate(specifier, configDir) {
+  if (specifier.startsWith("file:")) return { href: specifier };
+  if (RELATIVE.test(specifier) || isAbsolute3(specifier)) {
+    const path = resolve4(configDir, specifier);
+    return { href: pathToFileURL(path).href, shown: path };
+  }
+  try {
+    const path = createRequire(join2(resolve4(configDir), "package.json")).resolve(
+      specifier
+    );
+    return { href: pathToFileURL(path).href, shown: path };
+  } catch {
+    return { href: specifier };
+  }
+}
+function registrarIn(mod) {
+  if (typeof mod.register === "function") {
+    return mod.register;
+  }
+  if (typeof mod.default === "function") {
+    return mod.default;
+  }
+  return void 0;
+}
+function snapshot() {
+  const registered = /* @__PURE__ */ new Map();
+  for (const kind of listGraderKinds()) {
+    const grader = graderFor(kind);
+    if (grader !== void 0) registered.set(kind, grader);
+  }
+  return registered;
+}
+function changesSince(specifier, before) {
+  const added = [];
+  const replaced = [];
+  for (const kind of listGraderKinds()) {
+    const previous = before.get(kind);
+    if (previous === void 0) added.push(kind);
+    else if (previous !== graderFor(kind)) replaced.push(kind);
+  }
+  const warnings = [];
+  if (added.length === 0 && replaced.length === 0) {
+    warnings.push(
+      `grader plugin "${specifier}" registered no grader kinds \u2014 export a \`register({ registerGrader })\` function, or call registerGrader while the module is imported. Any eval naming a custom grader will report \`unknown grader kind\`.`
+    );
+  }
+  for (const kind of replaced) {
+    warnings.push(
+      BUILTIN_GRADER_KINDS.has(kind) ? `grader plugin "${specifier}" replaced the built-in grader "${kind}"; every eval declaring that kind now runs the plugin's implementation` : `grader plugin "${specifier}" replaced the grader "${kind}" registered by an earlier plugin`
+    );
+  }
+  return warnings;
+}
+async function loadGraderPlugins(options) {
+  const loaded = [];
+  const warnings = [];
+  for (const specifier of options.plugins) {
+    const { href, shown } = locate(specifier, options.configDir);
+    if (alreadyLoaded.has(href)) continue;
+    const before = snapshot();
+    let mod;
+    try {
+      mod = await import(href);
+    } catch (err) {
+      throw new TracevalsError(
+        `could not load grader plugin "${specifier}"` + (shown !== void 0 ? ` (resolved to ${shown})` : "") + `: ${err instanceof Error ? err.message : String(err)}`
+      );
+    }
+    const register = registrarIn(mod);
+    if (register !== void 0) {
+      try {
+        await register({ registerGrader });
+      } catch (err) {
+        throw new TracevalsError(
+          `grader plugin "${specifier}" threw while registering: ` + (err instanceof Error ? err.message : String(err))
+        );
+      }
+    }
+    alreadyLoaded.add(href);
+    warnings.push(...changesSince(specifier, before));
+    loaded.push(specifier);
+  }
+  return { loaded, warnings };
+}
+
+// src/tracevals/history.ts
+import { appendFile, mkdir, readFile as readFile2 } from "node:fs/promises";
+import { dirname as dirname2 } from "node:path";
+function entryFor(report) {
+  return {
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    traceFile: report.trace.file,
+    ...report.trace.sessionId !== void 0 ? { sessionId: report.trace.sessionId } : {},
+    project: report.trace.cwd,
+    summary: report.summary,
+    exitCode: report.exitCode,
+    turns: report.turns,
+    evals: report.evalResults.map((r) => ({
+      artifact: r.artifact,
+      evalName: r.evalName,
+      outcome: r.outcome
+    }))
+  };
+}
+async function appendHistory(file, report) {
+  try {
+    await mkdir(dirname2(file), { recursive: true });
+    await appendFile(file, `${JSON.stringify(entryFor(report))}
+`, "utf-8");
+  } catch (err) {
+    warn(
+      `could not write history at ${file} (${errorMessage(err)}). Continuing.`
+    );
+  }
+}
+async function loadHistory(file) {
+  let content;
+  try {
+    content = await readFile2(file, "utf-8");
+  } catch {
+    return [];
+  }
+  const entries = [];
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      entries.push(JSON.parse(trimmed));
+    } catch {
+      continue;
+    }
+  }
+  return entries;
+}
+var GOOD = /* @__PURE__ */ new Set(["pass"]);
+var BAD = /* @__PURE__ */ new Set(["fail", "error"]);
+function compareToLast(history, report) {
+  const key = report.trace.sessionId ?? report.trace.file;
+  const previous = [...history].reverse().find((e) => (e.sessionId ?? e.traceFile) === key);
+  if (!previous) return null;
+  const keyOf = (e) => `${e.artifact}::${e.evalName}`;
+  const before = new Map(previous.evals.map((e) => [keyOf(e), e]));
+  const current = entryFor(report).evals;
+  const currentKeys = new Set(current.map(keyOf));
+  const regressions = [];
+  const improvements = [];
+  const added = [];
+  for (const e of current) {
+    const prior = before.get(keyOf(e));
+    if (!prior) {
+      added.push(e.evalName);
+      continue;
+    }
+    if (GOOD.has(prior.outcome) && BAD.has(e.outcome)) regressions.push(e);
+    if (BAD.has(prior.outcome) && GOOD.has(e.outcome)) improvements.push(e);
+  }
+  const removed = previous.evals.filter((e) => !currentKeys.has(keyOf(e))).map((e) => e.evalName);
+  return {
+    previousTimestamp: previous.timestamp,
+    regressions,
+    improvements,
+    added,
+    removed
+  };
+}
+
+// src/tracevals/judge/provider.ts
+import {
+  makeProvider as makeInferenceProvider,
+  mockVerdict
+} from "@hawkeyexl/inference";
+var CONFIG_KEY = "tracevals.provider";
+var toTracevalsError = (message) => new TracevalsError(message);
+function selectProvider2(config, flags = {}, ev = {}) {
+  assertLocalFlag(flags, toTracevalsError);
+  return selectProvider(flags, [
+    {
+      ...ev.provider !== void 0 ? { provider: ev.provider } : {},
+      ...ev.model !== void 0 ? { model: ev.model } : {},
+      ...ev.origin !== void 0 ? { origin: ev.origin } : {}
+    },
+    {
+      ...config.provider !== null ? { provider: config.provider } : {},
+      ...config.model !== null ? { model: config.model } : {},
+      origin: CONFIG_KEY
+    },
+    {
+      ...config.providers.provider !== void 0 ? { provider: config.providers.provider } : {},
+      ...config.providers.model !== void 0 ? { model: config.providers.model } : {},
+      origin: "providers.provider"
+    }
+  ]);
+}
+function announceSelection(selection) {
+  const message = localNotice(selection);
+  if (message !== void 0) noticeOnce(message);
+}
+function assertProviderSelection(selection) {
+  assertKnownProvider(selection.provider, toTracevalsError);
+  assertModelHasProvider(selection.provider, selection.model, CONFIG_KEY, toTracevalsError);
+}
+function providerSpecFor2(config, selection, options = {}) {
+  const spec = providerSpecFor(config.providers, selection);
+  switch (selection.provider) {
+    case "anthropic":
+      return { ...spec, anthropic: { toolName: "record_verdict" } };
+    case "openai":
+      return { ...spec, openai: { schemaName: "verdict" } };
+    case "mock":
+      return {
+        ...spec,
+        mockResponses: options.mockResponses ?? [mockVerdict("pass", 0.95)]
+      };
+    default:
+      return spec;
+  }
+}
+async function resolveProviderIdentity(config, flags = {}, ev = {}) {
+  const selection = selectProvider2(config, flags, ev);
+  const { provider, model } = selection;
+  assertKnownProvider(provider, toTracevalsError);
+  assertModelHasProvider(provider, model, CONFIG_KEY, toTracevalsError);
+  announceSelection(selection);
+  return resolveIdentity(
+    providerSpecFor2(config, { provider, model: model ?? null }),
+    toTracevalsError
+  );
+}
+function constructProvider(config, identity, options = {}) {
+  try {
+    return makeInferenceProvider(providerSpecFor2(config, identity, options));
+  } catch (e) {
+    throw new TracevalsError(errorMessage(e));
+  }
+}
+async function makeJudgeProvider(config, flags = {}, ev = {}, options = {}) {
+  return constructProvider(
+    config,
+    await resolveProviderIdentity(config, flags, ev),
+    options
+  );
+}
+
+// src/tracevals/judge/trace-judge.ts
+import {
+  computeConsensus,
+  runEnsemble,
+  zoneFor
+} from "@hawkeyexl/inference";
+
+// src/tracevals/judge/verdict-schema.json
+var verdict_schema_default = {
+  $schema: "https://json-schema.org/draft/2020-12/schema",
+  $id: "manni:tracevals:verdict",
+  title: "manni tracevals judge verdict",
+  type: "object",
+  required: ["claim", "observed", "match", "confidence", "reasoning"],
+  properties: {
+    claim: {
+      type: "string",
+      description: "The specific instruction or assertion under evaluation."
+    },
+    observed: {
+      type: "string",
+      description: "What actually happened in the session, citing specific messages, tool calls, or instructions followed/violated."
+    },
+    match: {
+      enum: ["pass", "fail", "partial"],
+      description: "pass only if the session fully adhered; partial for partial adherence."
+    },
+    confidence: {
+      type: "number",
+      minimum: 0,
+      maximum: 1,
+      description: "Self-reported confidence in this verdict."
+    },
+    reasoning: {
+      type: "string",
+      description: "Why this conclusion was reached."
+    }
+  },
+  additionalProperties: false
+};
+
+// src/tracevals/judge/cache.ts
+import { JsonCache, buildCacheKey, sha256 } from "@hawkeyexl/inference";
+var VerdictCache = class _VerdictCache extends JsonCache {
+  /** An ensemble is cacheable only if every run produced a verdict. */
+  static usable(runs) {
+    return runs.every((r) => r.error === void 0 && r.verdict !== void 0);
+  }
+  set(key, value) {
+    if (!_VerdictCache.usable(value)) return;
+    super.set(key, value);
+  }
+  get(key) {
+    const hit = super.get(key);
+    if (hit === void 0) return void 0;
+    return _VerdictCache.usable(hit) ? hit : void 0;
+  }
+};
+function cacheKey(provider, model, runs, temperature, renderedTrace, plan, promptVersion = PROMPT_VERSION) {
+  const planFingerprint = JSON.stringify({
+    assertion: plan.assertion,
+    evidence: plan.evidence,
+    examples: plan.examples,
+    artifact: plan.artifact.content
+  });
+  return buildCacheKey([
+    provider,
+    model,
+    `v${promptVersion}`,
+    `r${runs}`,
+    `t${temperature}`,
+    // Pre-hashed: traces and artifacts are large, and key parts stay short.
+    sha256(renderedTrace),
+    sha256(planFingerprint)
+  ]);
+}
+
+// src/tracevals/judge/trace-judge.ts
+var verdictSchema = verdict_schema_default;
+function makeTraceJudge(options) {
+  const provider = options.provider;
+  const runsFor = (plan) => options.runs ?? plan.runs ?? 3;
+  const temperature = options.temperature ?? 0;
+  const zones = options.zones ?? { autoPass: 0.8, autoFail: 0.8 };
+  const cache = new VerdictCache(
+    options.cacheDir ?? ".manni/tracevals/cache",
+    options.noCache !== true && options.cacheDir !== void 0,
+    "manni-tracevals"
+  );
+  const maxTurns = options.maxTurns ?? null;
+  let turnsSpent = 0;
+  return async (plans, renderFor, context) => {
+    const trace = context?.trace;
+    const sessionModel = trace?.model;
+    const results = [];
+    for (const plan of plans) {
+      const start = Date.now();
+      const base = {
+        evalName: plan.evalName,
+        artifact: plan.artifact.path,
+        artifactName: plan.artifact.name,
+        grader: plan.grader,
+        implicit: plan.implicit
+      };
+      let evalProvider = provider;
+      if (plan.provider !== void 0 || plan.model !== void 0) {
+        const resolved = await resolveOverride(plan, options);
+        if ("error" in resolved) {
+          results.push({
+            ...base,
+            outcome: "error",
+            error: resolved.error,
+            turns: 0,
+            durationMs: Date.now() - start
+          });
+          continue;
+        }
+        evalProvider = resolved.provider;
+      }
+      const renderedTrace = renderFor(plan);
+      const selected = trace === void 0 ? plan.target === void 0 || plan.target === "transcript" || plan.target === "artifact" ? {
+        ok: true,
+        text: plan.target === "artifact" ? plan.artifact.content : renderedTrace,
+        label: plan.target === "artifact" ? "artifact" : "transcript"
+      } : {
+        ok: false,
+        reason: `target "${describeTarget(plan.target)}" needs the parsed session, which this run did not supply`
+      } : readTarget(plan.target, {
+        trace,
+        renderedTrace,
+        artifactContent: plan.artifact.content,
+        root: context?.projectRoot ?? trace.cwd
+      });
+      if (!selected.ok) {
+        results.push({
+          ...base,
+          outcome: "error",
+          error: selected.reason,
+          turns: 0,
+          durationMs: Date.now() - start
+        });
+        continue;
+      }
+      const runsPerEval = runsFor(plan);
+      const key = cacheKey(
+        evalProvider.provider(),
+        evalProvider.modelName(),
+        runsPerEval,
+        temperature,
+        // The selected bytes, not always the transcript: two targets on one
+        // session are two different questions and must not share a verdict.
+        selected.text,
+        plan
+      );
+      const cached = cache.get(key) !== void 0;
+      if (maxTurns !== null && !cached) {
+        if (turnsSpent + runsPerEval > maxTurns) {
+          results.push({
+            ...base,
+            outcome: "skipped",
+            skipReason: turnBudgetSkipReason(maxTurns),
+            turns: 0,
+            durationMs: 0
+          });
+          continue;
+        }
+        turnsSpent += runsPerEval;
+      }
+      const runs = await runEnsemble({
+        provider: evalProvider,
+        system: JUDGE_SYSTEM_PROMPT,
+        user: buildUserContent(plan, selected.text, selected.label),
+        runs: runsPerEval,
+        temperature,
+        schema: verdictSchema,
+        cache,
+        cacheKey: key,
+        label: "manni-tracevals"
+      });
+      const consensusBase = computeConsensus(runs);
+      const zone = zoneFor(consensusBase, zones);
+      const consensus = { ...consensusBase, zone };
+      const judgeModel = evalProvider.modelName();
+      const selfPreference = sessionModel !== void 0 && sessionModel === judgeModel ? { axis: "session", model: judgeModel } : plan.proposedBy?.includes(judgeModel) === true ? { axis: "criterion", model: judgeModel } : void 0;
+      results.push({
+        ...base,
+        outcome: zone === "auto-pass" ? "pass" : zone === "auto-fail" ? "fail" : "needs-review",
+        consensus,
+        turns: cached ? 0 : runsPerEval,
+        ...selfPreference ? { selfPreference } : {},
+        durationMs: Date.now() - start
+      });
+    }
+    return results;
+  };
+}
+async function resolveOverride(plan, options) {
+  const named = plan.provider !== void 0 ? `provider "${plan.provider}"` : `model "${plan.model ?? ""}"`;
+  if (options.providerFor === void 0) {
+    return {
+      error: `eval names ${named}, but this run cannot construct providers by name`
+    };
+  }
+  try {
+    return {
+      provider: await options.providerFor({
+        ...plan.provider !== void 0 ? { provider: plan.provider } : {},
+        ...plan.model !== void 0 ? { model: plan.model } : {},
+        origin: `eval "${plan.evalName}" in ${plan.artifact.path}`
+      })
+    };
+  } catch (err) {
+    return {
+      error: `could not construct ${named}${plan.provider === void 0 || plan.model === void 0 ? "" : ` at model "${plan.model}"`} for this eval: ${err instanceof Error ? err.message : String(err)}`
+    };
+  }
+}
+
+// src/tracevals/commands/run.ts
+function asGrants(values) {
+  const unknown = values.filter((v) => !isExecutionGrant(v));
+  if (unknown.length > 0) throw new TracevalsError(unknownGrantsMessage(unknown));
+  return values.filter(isExecutionGrant);
+}
+async function prepareRun(options) {
+  const {
+    config: loaded,
+    dir: configDir,
+    collections,
+    path: configPath
+  } = await discoverConfig(
+    options.configDir ?? process.cwd(),
+    {
+      ...options.config === void 0 ? {} : { configPath: options.config },
+      ...options.noConfig === void 0 ? {} : { noConfig: options.noConfig }
+    }
+  );
+  const config = {
+    ...loaded,
+    failOnReview: options.failOnReview ?? loaded.failOnReview,
+    execution: {
+      allow: [
+        ...grantsFor(loaded.execution.allow, {
+          ...options.allowExecution === void 0 ? {} : { allowExecution: asGrants(options.allowExecution) }
+        })
+      ]
+    },
+    // A set-valued knob, so `--require` *adds* instead of replacing. A one-off
+    // flag must not silently unregister the house graders a repo's config
+    // names — every eval declaring one would flip to `unknown grader kind`,
+    // which reads as a typo in an artifact nobody touched. Config entries load
+    // first, so a deliberate `--require` still wins a colliding kind
+    // (ADR 01017).
+    plugins: [...loaded.plugins, ...options.require ?? []],
+    reportUnusedArtifacts: options.reportUnusedArtifacts ?? loaded.reportUnusedArtifacts
+  };
+  const plugins = await loadGraderPlugins({
+    plugins: config.plugins,
+    configDir
+  });
+  const flags = {
+    ...options.provider !== void 0 ? { provider: options.provider } : {},
+    ...options.model !== void 0 ? { model: options.model } : {},
+    ...options.local !== void 0 ? { local: options.local } : {}
+  };
+  const selection = selectProvider2(config, flags);
+  assertProviderSelection(selection);
+  let judge = options.judge;
+  if (judge === void 0 && options.deterministicOnly !== true) {
+    const provider = constructProvider(
+      config,
+      await resolveProviderIdentity(config, flags)
+    );
+    const overridden = /* @__PURE__ */ new Map();
+    const keyOf = (s) => `${s.provider}:${s.model ?? ""}`;
+    const defaultKey = keyOf(selection);
+    judge = makeTraceJudge({
+      provider,
+      // An eval may name its own provider or model. It is selected by the same
+      // rule the run's own was — flag, then eval, then `tracevals.provider`,
+      // then the family's `providers:` — so an eval cannot quietly outrank a
+      // flag someone just typed.
+      providerFor: async (ev) => {
+        const chosen = selectProvider2(config, flags, ev);
+        announceSelection(chosen);
+        const key = keyOf(chosen);
+        if (key === defaultKey) return provider;
+        let built = overridden.get(key);
+        if (built === void 0) {
+          built = (async () => {
+            assertProviderSelection(chosen);
+            return constructProvider(
+              config,
+              await resolveProviderIdentity(config, flags, ev)
+            );
+          })();
+          overridden.set(key, built);
+        }
+        return built;
+      },
+      runs: options.runs ?? config.judge.ensembleRuns,
+      temperature: config.judge.temperature,
+      zones: config.judge.zones,
+      cacheDir: resolve5(configDir, config.judge.cacheDir),
+      ...options.noCache !== void 0 ? { noCache: options.noCache } : {},
+      maxTurns: options.maxTurns ?? config.judge.maxTurns
+    });
+  }
+  const external = await loadExternalEvals({
+    collections,
+    configDir,
+    ...configPath === void 0 ? {} : { configPath },
+    ...options.offline === void 0 ? {} : { offline: options.offline }
+  });
+  return {
+    config,
+    configDir,
+    warnings: plugins.warnings,
+    ...judge !== void 0 ? { judge } : {},
+    ...external === null ? {} : {
+      metadataFor: (artifact) => external.forArtifact(artifact)
+    }
+  };
+}
+async function runOne(options, context) {
+  const { config } = context;
+  const report = await runEvals({
+    tracePath: options.tracePath,
+    ...options.project !== void 0 ? { projectDir: options.project } : {},
+    ...options.env !== void 0 ? { env: options.env } : {},
+    config,
+    ...context.judge !== void 0 ? { judge: context.judge } : {},
+    ...options.deterministicOnly !== void 0 ? { deterministicOnly: options.deterministicOnly } : {},
+    ...options.manifest !== void 0 ? { manifest: options.manifest } : {},
+    ...context.warnings.length > 0 ? { warnings: context.warnings } : {},
+    ...context.metadataFor !== void 0 ? { metadataFor: context.metadataFor } : {}
+  });
+  let comparison;
+  if (options.history) {
+    const historyFile = resolve5(context.configDir, config.history.file);
+    comparison = compareToLast(await loadHistory(historyFile), report) ?? void 0;
+    await appendHistory(historyFile, report);
+  }
+  return { report, ...comparison ? { comparison } : {} };
+}
+async function runRun(options) {
+  const context = await prepareRun(options);
+  const { report, comparison } = await runOne(options, context);
+  const format = options.format ?? "pretty";
+  let rendered = render(report, format, { color: options.color === true });
+  if (comparison && (format === "pretty" || format === "markdown" || format === "github")) {
+    rendered += `
+
+${renderComparison(comparison)}`;
+  }
+  if (options.output) {
+    await writeFile(options.output, rendered, "utf-8");
+  }
+  return { report, rendered, ...comparison ? { comparison } : {} };
+}
+function renderComparison(comparison) {
+  const lines = [`History vs ${comparison.previousTimestamp}:`];
+  for (const r of comparison.regressions) {
+    lines.push(`  regression: ${r.evalName} (${r.outcome})`);
+  }
+  for (const i of comparison.improvements) {
+    lines.push(`  improvement: ${i.evalName} (${i.outcome})`);
+  }
+  if (comparison.added.length) lines.push(`  added: ${comparison.added.join(", ")}`);
+  if (comparison.removed.length) {
+    lines.push(`  removed: ${comparison.removed.join(", ")}`);
+  }
+  if (lines.length === 1) lines.push("  no changes");
+  return lines.join("\n");
+}
+
+// src/tracevals/commands/batch.ts
+function parseNewerThan(text) {
+  return parseDuration(text, (message) => new TracevalsError(message));
+}
+function keepNewerThan(listings, newerThan) {
+  if (newerThan === void 0) return listings;
+  const floor = Date.now() - parseNewerThan(newerThan);
+  return listings.filter((l) => l.mtimeMs >= floor);
+}
+function usesDiscovery(options) {
+  return options.allProjects === true || options.newerThan !== void 0 || options.limit !== void 0;
+}
+async function resolveBatchTraces(options) {
+  const named = options.traces ?? [];
+  if (named.length > 0 && usesDiscovery(options)) {
+    throw new TracevalsError(
+      "name traces or select them with --all-projects/--newer-than/--limit, not both"
+    );
+  }
+  if (named.length > 0) return named;
+  const listings = await discoverTraces({
+    ...options.allProjects !== void 0 ? { allProjects: options.allProjects } : {},
+    ...options.project !== void 0 ? { project: options.project } : {},
+    ...options.limit !== void 0 ? { limit: options.limit } : {},
+    ...options.env !== void 0 ? { env: options.env } : {}
+  });
+  const kept = keepNewerThan(listings, options.newerThan);
+  if (kept.length === 0) {
+    const how = [
+      options.allProjects === true ? "--all-projects" : `project ${options.project ?? process.cwd()}`,
+      options.newerThan !== void 0 ? `--newer-than ${options.newerThan}` : void 0,
+      options.limit !== void 0 ? `--limit ${options.limit}` : void 0
+    ].filter((s) => s !== void 0).join(", ");
+    throw new TracevalsError(
+      `no traces matched (${how}); run \`manni tracevals list\` to see what the session store holds`
+    );
+  }
+  return kept.map((l) => l.file);
+}
+async function runBatch(options) {
+  const start = Date.now();
+  const files = await resolveBatchTraces(options);
+  const context = await prepareRun(options);
+  const outcomes = [];
+  const reports = [];
+  const comparisons = [];
+  for (const file of files) {
+    const traceStart = Date.now();
+    try {
+      const { report: report2, comparison } = await runOne(
+        { ...options, tracePath: file },
+        context
+      );
+      outcomes.push({ file, report: report2 });
+      reports.push(report2);
+      if (comparison) comparisons.push(comparison);
+    } catch (err) {
+      outcomes.push({
+        file,
+        error: err instanceof Error ? err.message : String(err),
+        durationMs: Date.now() - traceStart
+      });
+    }
+  }
+  const report = aggregate(outcomes, {
+    warnings: context.warnings,
+    durationMs: Date.now() - start
+  });
+  const rendered = renderBatch(report, options.format ?? "pretty", reports, {
+    color: options.color === true
+  });
+  if (options.output) {
+    await writeFile2(options.output, rendered, "utf-8");
+  }
+  return { report, rendered, reports, outcomes, comparisons };
+}
+
+export {
+  outputTail,
+  realExec,
+  parseDuration,
+  TURN_BUDGET_SKIP,
+  turnBudgetSkipReason,
+  isTurnBudgetSkip,
+  URI_BASE_ID,
+  fileUri,
+  normalizeRoot,
+  artifactLocation,
+  aggregate,
+  renderBatchPretty,
+  renderBatchMarkdown,
+  renderCalibrationPretty,
+  renderCalibrationMarkdown,
+  REPORT_FORMATS,
+  CALIBRATE_FORMATS,
+  SUMMARY_FORMATS,
+  parseFormat,
+  render,
+  renderBatch,
+  renderCalibration,
+  detectFormat,
+  detectContentFormat,
+  parseTraceFile,
+  parseTraceContent,
+  IMPLICIT_EVAL_NAME,
+  planEvals,
+  PROMPT_VERSION,
+  JUDGE_SYSTEM_PROMPT,
+  buildUserContent,
+  globToRegExp,
+  matchesGlob,
+  WHEN_CONDITIONS,
+  validateWhen,
+  evaluateWhen,
+  windowFor,
+  skippedWindow,
+  BUILTIN_GRADER_KINDS,
+  registerGrader,
+  graderFor,
+  listGraderKinds,
+  renderTrace,
+  runEvals,
+  METADATA_KEY,
+  urlManifestRefusal,
+  loadExternalEvals,
+  loadGraderPlugins,
+  entryFor,
+  appendHistory,
+  loadHistory,
+  compareToLast,
+  selectProvider2 as selectProvider,
+  announceSelection,
+  assertProviderSelection,
+  providerSpecFor2 as providerSpecFor,
+  resolveProviderIdentity,
+  constructProvider,
+  makeJudgeProvider,
+  sha256,
+  VerdictCache,
+  cacheKey,
+  makeTraceJudge,
+  prepareRun,
+  runOne,
+  runRun,
+  parseNewerThan,
+  keepNewerThan,
+  resolveBatchTraces,
+  runBatch
+};
+//# sourceMappingURL=chunk-CI2GOU3A.js.map