agent-inspect 5.0.0 → 5.2.0

package/packages/core/dist/advanced.cjs

@@ -3279,7 +3279,7 @@ function buildRunSummary(events) {
   }
   let totalSteps = 0;
   let llmSteps = 0;
-  let toolSteps = 0;
+  let toolSteps2 = 0;
   let logicSteps = 0;
   let errorSteps = 0;
   let maxDepth = 0;
@@ -3309,7 +3309,7 @@ function buildRunSummary(events) {
   for (const [id, s] of steps.entries()) {
     totalSteps += 1;
     if (s.type === "llm") llmSteps += 1;
-    else if (s.type === "tool") toolSteps += 1;
+    else if (s.type === "tool") toolSteps2 += 1;
     else logicSteps += 1;
     if (s.status === "error") errorSteps += 1;
     const depth = computeDepth(id);
@@ -3343,7 +3343,7 @@ function buildRunSummary(events) {
     durationMs: durationMs2,
     totalSteps,
     llmSteps,
-    toolSteps,
+    toolSteps: toolSteps2,
     logicSteps,
     errorSteps,
     maxDepth,
@@ -4086,9 +4086,9 @@ async function searchTraces(metas, options) {
   }
   const limit = options.limit ?? 50;
   const sessionId = options.session?.trim();
-  const observationStatus = parseObservationFilter(options.observation);
+  const observationStatus2 = parseObservationFilter(options.observation);
   const hasContentFilter = Boolean(
-    options.status || stepTypeFilter || nameQuery || toolQuery || durationFilter || observationStatus
+    options.status || stepTypeFilter || nameQuery || toolQuery || durationFilter || observationStatus2
   );
   const results = [];
   const sessionLabel = sessionId && sessionId !== "" ? sessionId : void 0;
@@ -4133,9 +4133,9 @@ async function searchTraces(metas, options) {
       statusFilter: options.status
     });
     results.push(...stepMatches);
-    if (observationStatus) {
+    if (observationStatus2) {
       const outcomes = extractOutcomesFromTraceEvents(events);
-      const matched = outcomes.filter((outcome) => outcome.status === observationStatus);
+      const matched = outcomes.filter((outcome) => outcome.status === observationStatus2);
       for (const outcome of matched) {
         results.push({
           runId: m.runId,
@@ -4971,12 +4971,12 @@ function buildCriticalPath(runs, handoffs) {
     handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
   );
   const ordered = [...runs].sort(compareRuns);
-  const path11 = [];
+  const path12 = [];
   const visited = /* @__PURE__ */ new Set();
   const pushRun = (run, confidence, source) => {
     if (visited.has(run.runId)) return;
     visited.add(run.runId);
-    path11.push({
+    path12.push({
       runId: run.runId,
       name: run.name,
       startedAt: run.startedAt,
@@ -5001,7 +5001,7 @@ function buildCriticalPath(runs, handoffs) {
     const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
     pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
   }
-  return path11;
+  return path12;
 }
 function metaRunIdMatches(run, token, runById) {
   const meta = extractSessionWorkflowMetadata(run.metadata);
@@ -5824,7 +5824,7 @@ function stripPrefix(name, prefixes) {
   }
   return name;
 }
-function eventEvidence(event, path11) {
+function eventEvidence(event, path12) {
   return {
     runId: event.runId,
     eventId: event.eventId,
@@ -5834,7 +5834,7 @@ function eventEvidence(event, path11) {
     kind: event.kind,
     name: event.name,
     status: event.status,
-    ...path11 ? { path: path11 } : {}
+    ...path12 ? { path: path12 } : {}
   };
 }
 function runEvidence(run) {
@@ -8446,6 +8446,923 @@ function renderSuiteReport(result, options = {}) {
   return renderSuiteReportMarkdown(result);
 }
 
+// packages/core/src/cohort/types.ts
+var COHORT_METRIC_IDS = [
+  "errorRate",
+  "duration",
+  "toolChoice",
+  "toolOrdering",
+  "llmCallCount",
+  "tokenUsage",
+  "retryCount",
+  "observationFailure",
+  "guardrailFailure",
+  "circuitViolation",
+  "redactionWarning"
+];
+
+// packages/core/src/cohort/compare.ts
+function compareNumber(metric, label, baseline, candidate, higherIsWorse = true) {
+  if (baseline === void 0 && candidate === void 0) return void 0;
+  const delta = baseline !== void 0 && candidate !== void 0 ? candidate - baseline : void 0;
+  const regression = delta !== void 0 && (higherIsWorse && delta > 0 || !higherIsWorse && delta < 0);
+  return {
+    metric,
+    baseline,
+    candidate,
+    delta,
+    regression,
+    message: `${label}: baseline=${baseline ?? "n/a"} candidate=${candidate ?? "n/a"}${delta !== void 0 ? ` (delta ${delta})` : ""}`
+  };
+}
+function pickAggregate(groups, cohortLabel, groupKey) {
+  return groups.find(
+    (group) => group.cohortLabel === cohortLabel && (groupKey === void 0 || group.groupKey === groupKey)
+  );
+}
+function compareCohortAggregates(groups, options) {
+  const baselineAgg = pickAggregate(groups, options.baseline, options.groupKey);
+  const candidateAgg = pickAggregate(groups, options.candidate, options.groupKey);
+  const comparisons = [];
+  for (const metric of options.metrics) {
+    switch (metric) {
+      case "errorRate": {
+        const item = compareNumber(
+          metric,
+          "Error rate",
+          baselineAgg?.errorRate,
+          candidateAgg?.errorRate
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "duration": {
+        const item = compareNumber(
+          metric,
+          "Average duration (ms)",
+          baselineAgg?.avgDurationMs,
+          candidateAgg?.avgDurationMs
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "llmCallCount": {
+        const item = compareNumber(
+          metric,
+          "Average LLM calls",
+          baselineAgg?.avgLlmCallCount,
+          candidateAgg?.avgLlmCallCount
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "tokenUsage": {
+        const item = compareNumber(
+          metric,
+          "Average token usage",
+          baselineAgg?.avgTokenUsage,
+          candidateAgg?.avgTokenUsage
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "retryCount": {
+        const item = compareNumber(
+          metric,
+          "Average retries",
+          baselineAgg?.avgRetryCount,
+          candidateAgg?.avgRetryCount
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "observationFailure": {
+        const item = compareNumber(
+          metric,
+          "Observation failure rate",
+          baselineAgg?.observationFailureRate,
+          candidateAgg?.observationFailureRate
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "toolChoice": {
+        const baselineValue = baselineAgg?.dominantToolChoice;
+        const candidateValue = candidateAgg?.dominantToolChoice;
+        comparisons.push({
+          metric,
+          baseline: baselineValue,
+          candidate: candidateValue,
+          delta: baselineValue === candidateValue ? "same" : "changed",
+          regression: baselineValue !== candidateValue,
+          message: `Tool choice: baseline=${baselineValue ?? "n/a"} candidate=${candidateValue ?? "n/a"}`
+        });
+        break;
+      }
+      case "toolOrdering": {
+        const baselineValue = baselineAgg?.toolOrderingSignature;
+        const candidateValue = candidateAgg?.toolOrderingSignature;
+        comparisons.push({
+          metric,
+          baseline: baselineValue,
+          candidate: candidateValue,
+          delta: baselineValue === candidateValue ? "same" : "changed",
+          regression: baselineValue !== candidateValue,
+          message: `Tool ordering: baseline=${baselineValue ?? "n/a"} candidate=${candidateValue ?? "n/a"}`
+        });
+        break;
+      }
+      case "guardrailFailure": {
+        const item = compareNumber(
+          metric,
+          "Guardrail failures",
+          baselineAgg?.avgGuardrailFailures,
+          candidateAgg?.avgGuardrailFailures
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "circuitViolation": {
+        const item = compareNumber(
+          metric,
+          "Circuit violations",
+          baselineAgg?.avgCircuitViolations,
+          candidateAgg?.avgCircuitViolations
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "redactionWarning": {
+        const item = compareNumber(
+          metric,
+          "Redaction warnings",
+          baselineAgg?.avgRedactionWarnings,
+          candidateAgg?.avgRedactionWarnings
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+    }
+  }
+  return comparisons;
+}
+
+// packages/core/src/cohort/grouping.ts
+function parseCohortMetricList(value) {
+  if (value === void 0 || value.trim() === "") return [];
+  return value.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
+}
+function parseGroupBySpec(groupBy) {
+  const raw = (groupBy ?? "model").trim();
+  if (raw === "model") return { kind: "model" };
+  if (raw === "session") return { kind: "session" };
+  if (raw === "group") return { kind: "group" };
+  if (raw.startsWith("metadata.")) {
+    const metadataKey = raw.slice("metadata.".length).trim();
+    if (metadataKey === "") throw new Error("metadata group-by requires a key.");
+    return { kind: "metadata", metadataKey };
+  }
+  throw new Error(`Unsupported --group-by value: ${raw}`);
+}
+function metadataString(metadata, key) {
+  const value = metadata?.[key];
+  return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
+}
+function resolveRunGroupKey(run, groupBy) {
+  const metadata = run.metadata ?? {};
+  switch (groupBy.kind) {
+    case "model":
+      return metadataString(metadata, "model") ?? "unknown";
+    case "session":
+      return extractSessionWorkflowMetadata(metadata)?.sessionId ?? metadataString(metadata, "sessionId") ?? "__unscoped__";
+    case "group":
+      return extractSessionWorkflowMetadata(metadata)?.groupId ?? metadataString(metadata, "groupId") ?? "__unscoped__";
+    case "metadata":
+      return metadataString(metadata, groupBy.metadataKey) ?? "__missing__";
+    default:
+      return "unknown";
+  }
+}
+function resolveCohortLabel(run, cohortKey, baseline, candidate) {
+  const label = metadataString(run.metadata, cohortKey);
+  if (label === void 0) return void 0;
+  if (baseline !== void 0 && label === baseline) return baseline;
+  if (candidate !== void 0 && label === candidate) return candidate;
+  if (baseline === void 0 && candidate === void 0) return label;
+  return void 0;
+}
+function filterRunsForCohort(runs, options) {
+  const warnings = [];
+  if (options.baseline === void 0 && options.candidate === void 0) {
+    return { runs: [...runs], warnings };
+  }
+  const selected = [];
+  for (const run of runs) {
+    const label = resolveCohortLabel(
+      run,
+      options.cohortKey,
+      options.baseline,
+      options.candidate
+    );
+    if (label !== void 0) selected.push(run);
+  }
+  if (selected.length === 0) {
+    warnings.push(
+      `No runs matched baseline/candidate labels on metadata.${options.cohortKey}.`
+    );
+  }
+  return { runs: selected, warnings };
+}
+
+// packages/core/src/cohort/metrics.ts
+function asNumber(value) {
+  return typeof value === "number" && Number.isFinite(value) ? value : void 0;
+}
+function toolSteps(events) {
+  const ordering = [];
+  const choices = /* @__PURE__ */ new Set();
+  const sorted = [...events].filter((event) => event.event === "step_started").sort((a, b) => a.timestamp - b.timestamp);
+  for (const event of sorted) {
+    const step = event;
+    if (step.type !== "tool") continue;
+    const name = typeof step.metadata?.toolName === "string" ? step.metadata.toolName : step.name;
+    ordering.push(name);
+    choices.add(name);
+  }
+  return { choices: [...choices].sort(), ordering };
+}
+async function computeCohortRunMetrics(input) {
+  const events = await readTraceEventsFromFile(input.filePath);
+  const summary = buildRunSummary(events);
+  const tools = toolSteps(events);
+  const outcomes = extractOutcomesFromTraceEvents(events);
+  const observationFailures = outcomes.filter((item) => item.status === "failed").length;
+  const metadata = input.metadata ?? {};
+  const retryCount = asNumber(metadata.attempt) !== void 0 && asNumber(metadata.attempt) > 1 ? asNumber(metadata.attempt) - 1 : typeof metadata.retryOf === "string" ? 1 : 0;
+  return {
+    runId: input.runId,
+    ...input.cohortLabel !== void 0 ? { cohortLabel: input.cohortLabel } : {},
+    groupKey: input.groupKey,
+    status: summary.status,
+    error: summary.status === "error",
+    durationMs: summary.durationMs ?? input.durationMs,
+    llmCallCount: summary.llmSteps,
+    tokenUsageTotal: summary.totalTokens?.total,
+    retryCount,
+    observationFailures,
+    guardrailFailures: asNumber(metadata.guardrailFailures) ?? 0,
+    circuitViolations: asNumber(metadata.circuitViolations) ?? 0,
+    redactionWarnings: asNumber(metadata.redactionWarnings) ?? 0,
+    toolChoices: tools.choices,
+    toolOrdering: tools.ordering
+  };
+}
+function percentile2(values, p) {
+  if (values.length === 0) return void 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const idx = Math.min(
+    sorted.length - 1,
+    Math.max(0, Math.ceil(p / 100 * sorted.length) - 1)
+  );
+  return sorted[idx];
+}
+function dominantToolChoice(runs) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const run of runs) {
+    const signature = run.toolChoices.join(",");
+    if (signature === "") continue;
+    counts.set(signature, (counts.get(signature) ?? 0) + 1);
+  }
+  let best;
+  let bestCount = 0;
+  for (const [key, count] of counts) {
+    if (count > bestCount) {
+      best = key;
+      bestCount = count;
+    }
+  }
+  return best;
+}
+function orderingSignature(runs) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const run of runs) {
+    const signature = run.toolOrdering.join(">");
+    if (signature === "") continue;
+    counts.set(signature, (counts.get(signature) ?? 0) + 1);
+  }
+  let best;
+  let bestCount = 0;
+  for (const [key, count] of counts) {
+    if (count > bestCount) {
+      best = key;
+      bestCount = count;
+    }
+  }
+  return best;
+}
+function aggregateCohortMetrics(runs, groupKey, cohortLabel) {
+  const durations = runs.map((run) => run.durationMs).filter((value) => typeof value === "number");
+  const tokenValues = runs.map((run) => run.tokenUsageTotal).filter((value) => typeof value === "number");
+  const errors = runs.filter((run) => run.error).length;
+  const observationFailures = runs.reduce((sum, run) => sum + run.observationFailures, 0);
+  return {
+    groupKey,
+    ...cohortLabel !== void 0 ? { cohortLabel } : {},
+    runCount: runs.length,
+    errorRate: runs.length > 0 ? errors / runs.length : 0,
+    avgDurationMs: durations.length > 0 ? durations.reduce((sum, value) => sum + value, 0) / durations.length : void 0,
+    p95DurationMs: percentile2(durations, 95),
+    avgLlmCallCount: runs.length > 0 ? runs.reduce((sum, run) => sum + run.llmCallCount, 0) / runs.length : 0,
+    avgTokenUsage: tokenValues.length > 0 ? tokenValues.reduce((sum, value) => sum + value, 0) / tokenValues.length : void 0,
+    avgRetryCount: runs.length > 0 ? runs.reduce((sum, run) => sum + run.retryCount, 0) / runs.length : 0,
+    observationFailureRate: runs.length > 0 ? observationFailures / runs.length : 0,
+    avgGuardrailFailures: runs.length > 0 ? runs.reduce((sum, run) => sum + run.guardrailFailures, 0) / runs.length : 0,
+    avgCircuitViolations: runs.length > 0 ? runs.reduce((sum, run) => sum + run.circuitViolations, 0) / runs.length : 0,
+    avgRedactionWarnings: runs.length > 0 ? runs.reduce((sum, run) => sum + run.redactionWarnings, 0) / runs.length : 0,
+    dominantToolChoice: dominantToolChoice(runs),
+    toolOrderingSignature: orderingSignature(runs)
+  };
+}
+
+// packages/core/src/cohort/analyze.ts
+var DEFAULT_METRICS = [
+  "errorRate",
+  "duration",
+  "toolChoice",
+  "observationFailure"
+];
+function normalizeMetrics(metrics) {
+  if (metrics === void 0 || metrics.length === 0) return [...DEFAULT_METRICS];
+  const allowed = new Set(COHORT_METRIC_IDS);
+  return metrics.filter((metric) => allowed.has(metric));
+}
+async function analyzeCohort(runsInput, options) {
+  const cohortKey = options.cohortKey ?? "cohort";
+  const groupBySpec = parseGroupBySpec(options.groupBy);
+  const metrics = normalizeMetrics(options.metrics);
+  const { runs: filteredRuns, warnings } = filterRunsForCohort(runsInput, {
+    cohortKey,
+    baseline: options.baseline,
+    candidate: options.candidate
+  });
+  const runMetrics = [];
+  for (const run of filteredRuns) {
+    if (run.filePath === void 0) continue;
+    const cohortLabel = resolveCohortLabel(
+      run,
+      cohortKey,
+      options.baseline,
+      options.candidate
+    );
+    runMetrics.push(
+      await computeCohortRunMetrics({
+        runId: run.runId,
+        filePath: run.filePath,
+        metadata: run.metadata,
+        status: run.status,
+        durationMs: run.durationMs,
+        groupKey: resolveRunGroupKey(run, groupBySpec),
+        cohortLabel
+      })
+    );
+  }
+  const groupMap = /* @__PURE__ */ new Map();
+  for (const run of runMetrics) {
+    const key = `${run.cohortLabel ?? "*"}::${run.groupKey}`;
+    const bucket = groupMap.get(key) ?? [];
+    bucket.push(run);
+    groupMap.set(key, bucket);
+  }
+  const groups = [...groupMap.entries()].sort(([a], [b]) => a.localeCompare(b)).map(
+    ([, bucket]) => aggregateCohortMetrics(
+      bucket,
+      bucket[0].groupKey,
+      bucket[0]?.cohortLabel
+    )
+  );
+  const comparisons = options.baseline !== void 0 && options.candidate !== void 0 ? (() => {
+    const groupKeys = [
+      ...new Set(groups.map((group) => group.groupKey))
+    ].sort((a, b) => a.localeCompare(b));
+    const items = [];
+    for (const groupKey of groupKeys) {
+      items.push(
+        ...compareCohortAggregates(groups, {
+          baseline: options.baseline,
+          candidate: options.candidate,
+          metrics,
+          groupKey
+        })
+      );
+    }
+    return items;
+  })() : [];
+  const regression = comparisons.some((item) => item.regression);
+  return {
+    ok: !regression,
+    traceDir: options.traceDir,
+    ...options.baseline !== void 0 ? { baseline: options.baseline } : {},
+    ...options.candidate !== void 0 ? { candidate: options.candidate } : {},
+    cohortKey,
+    groupBy: options.groupBy ?? "model",
+    metrics,
+    groups,
+    comparisons,
+    runs: runMetrics,
+    warnings
+  };
+}
+
+// packages/core/src/exporters/helpers.ts
+function escapeHtml(value) {
+  return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
+}
+
+// packages/core/src/cohort/render.ts
+function formatRate(value) {
+  if (value === void 0) return "n/a";
+  return `${(value * 100).toFixed(1)}%`;
+}
+function renderCohortSummaryMarkdown(result) {
+  const lines = [];
+  lines.push("# Cohort analysis");
+  lines.push("");
+  lines.push(`Trace directory: \`${result.traceDir}\``);
+  lines.push(`Group by: \`${result.groupBy}\``);
+  if (result.baseline !== void 0 && result.candidate !== void 0) {
+    lines.push(
+      `Baseline/Candidate key: \`${result.cohortKey}\` (${result.baseline} vs ${result.candidate})`
+    );
+  }
+  lines.push(`Status: **${result.ok ? "PASS" : "REGRESSION"}**`);
+  lines.push("");
+  if (result.warnings.length > 0) {
+    lines.push("## Warnings");
+    for (const warning of result.warnings) lines.push(`- ${warning}`);
+    lines.push("");
+  }
+  lines.push("## Groups");
+  for (const group of result.groups) {
+    lines.push(
+      `### ${group.cohortLabel ?? "all"} / ${group.groupKey} (${group.runCount} runs)`
+    );
+    lines.push(`- Error rate: ${formatRate(group.errorRate)}`);
+    if (group.avgDurationMs !== void 0) {
+      lines.push(`- Avg duration: ${Math.round(group.avgDurationMs)} ms`);
+    }
+    if (group.dominantToolChoice !== void 0) {
+      lines.push(`- Dominant tools: ${group.dominantToolChoice}`);
+    }
+    lines.push(
+      `- Observation failure rate: ${formatRate(group.observationFailureRate)}`
+    );
+    lines.push("");
+  }
+  if (result.comparisons.length > 0) {
+    lines.push("## Comparisons");
+    for (const comparison of result.comparisons) {
+      const flag = comparison.regression ? " **REGRESSION**" : "";
+      lines.push(`- ${comparison.message}${flag}`);
+    }
+    lines.push("");
+  }
+  return lines.join("\n").trimEnd();
+}
+function renderCohortReportHtml(result) {
+  const rows = result.groups.map(
+    (group) => `<tr><td>${escapeHtml(group.cohortLabel ?? "all")}</td><td>${escapeHtml(group.groupKey)}</td><td>${group.runCount}</td><td>${escapeHtml(formatRate(group.errorRate))}</td><td>${group.avgDurationMs !== void 0 ? Math.round(group.avgDurationMs) : "n/a"}</td></tr>`
+  ).join("");
+  const comparisons = result.comparisons.map(
+    (item) => `<li>${escapeHtml(item.message)}${item.regression ? " <strong>REGRESSION</strong>" : ""}</li>`
+  ).join("");
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <title>Cohort report</title>
+  <style>
+    body { font-family: system-ui, sans-serif; margin: 2rem; }
+    table { border-collapse: collapse; width: 100%; }
+    th, td { border: 1px solid #ddd; padding: 0.5rem; text-align: left; }
+    th { background: #f6f6f6; }
+  </style>
+</head>
+<body>
+  <h1>Cohort analysis</h1>
+  <p>Status: <strong>${result.ok ? "PASS" : "REGRESSION"}</strong></p>
+  <p>Trace directory: <code>${escapeHtml(result.traceDir)}</code></p>
+  <h2>Groups</h2>
+  <table>
+    <thead><tr><th>Cohort</th><th>Group</th><th>Runs</th><th>Error rate</th><th>Avg duration (ms)</th></tr></thead>
+    <tbody>${rows}</tbody>
+  </table>
+  ${result.comparisons.length > 0 ? `<h2>Comparisons</h2><ul>${comparisons}</ul>` : ""}
+</body>
+</html>`;
+}
+function renderCohortReport(result, options = {}) {
+  const format = options.format ?? "markdown";
+  if (format === "json") return JSON.stringify(result, null, 2);
+  if (format === "html") return renderCohortReportHtml(result);
+  return renderCohortSummaryMarkdown(result);
+}
+
+// packages/core/src/gate/parse.ts
+function parseGateList(value) {
+  if (value === void 0 || value.trim() === "") return [];
+  return value.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
+}
+function parseGateNumber(value, label) {
+  if (value === void 0 || value.trim() === "") return void 0;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) {
+    throw new Error(`Invalid ${label}: ${value}`);
+  }
+  return parsed;
+}
+
+// packages/core/src/gate/evaluate.ts
+function percentile3(values, p) {
+  if (values.length === 0) return void 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const idx = Math.min(
+    sorted.length - 1,
+    Math.max(0, Math.ceil(p / 100 * sorted.length) - 1)
+  );
+  return sorted[idx];
+}
+function hasThresholds(options) {
+  return options.maxErrorRate !== void 0 || options.maxP95DurationMs !== void 0 || (options.forbidTools?.length ?? 0) > 0 || (options.requireObservations?.length ?? 0) > 0;
+}
+function gateHasThresholds(options) {
+  return hasThresholds(options);
+}
+async function loadRunMetrics(runs) {
+  const metrics = [];
+  for (const run of runs) {
+    if (run.filePath === void 0) continue;
+    metrics.push(
+      await computeCohortRunMetrics({
+        runId: run.runId,
+        filePath: run.filePath,
+        metadata: run.metadata,
+        status: run.status,
+        durationMs: run.durationMs,
+        groupKey: "all"
+      })
+    );
+  }
+  return metrics;
+}
+async function observationStatus(filePath, name) {
+  const events = await readTraceEventsFromFile(filePath);
+  const outcomes = extractOutcomesFromTraceEvents(events);
+  const match = outcomes.find((item) => item.name === name);
+  if (!match) return "missing";
+  return match.status === "passed" ? "passed" : "failed";
+}
+async function evaluateGateThresholds(runs, options) {
+  const checks = [];
+  const readErrors = [];
+  if (!hasThresholds(options)) {
+    return { checks, readErrors };
+  }
+  if (runs.length === 0) {
+    readErrors.push("No trace runs found in the gate directory.");
+    return { checks, readErrors };
+  }
+  let runMetrics;
+  try {
+    runMetrics = await loadRunMetrics(runs);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    readErrors.push(message);
+    return { checks, readErrors };
+  }
+  if (options.maxErrorRate !== void 0) {
+    const errors = runMetrics.filter((run) => run.error).length;
+    const actual = runMetrics.length > 0 ? errors / runMetrics.length * 100 : 0;
+    const ok = actual <= options.maxErrorRate;
+    checks.push({
+      id: "maxErrorRate",
+      name: "Max error rate",
+      ok,
+      expected: options.maxErrorRate,
+      actual: Math.round(actual * 10) / 10,
+      message: ok ? `Error rate ${actual.toFixed(1)}% within limit ${options.maxErrorRate}%` : `Error rate ${actual.toFixed(1)}% exceeds limit ${options.maxErrorRate}%`
+    });
+  }
+  if (options.maxP95DurationMs !== void 0) {
+    const durations = runMetrics.map((run) => run.durationMs).filter((value) => typeof value === "number");
+    const actual = percentile3(durations, 95);
+    const ok = actual !== void 0 && actual <= options.maxP95DurationMs;
+    checks.push({
+      id: "maxP95Duration",
+      name: "Max p95 duration (ms)",
+      ok,
+      expected: options.maxP95DurationMs,
+      actual: actual ?? "n/a",
+      message: actual === void 0 ? "No duration samples available for p95 check." : ok ? `P95 duration ${Math.round(actual)} ms within limit ${options.maxP95DurationMs} ms` : `P95 duration ${Math.round(actual)} ms exceeds limit ${options.maxP95DurationMs} ms`
+    });
+  }
+  for (const tool of options.forbidTools ?? []) {
+    let violated = false;
+    for (const run of runMetrics) {
+      const used = run.toolChoices.includes(tool) || run.toolOrdering.includes(tool);
+      if (used) {
+        violated = true;
+        checks.push({
+          id: "forbidTool",
+          name: `Forbid tool: ${tool}`,
+          ok: false,
+          expected: `not used`,
+          actual: "used",
+          runId: run.runId,
+          message: `Forbidden tool "${tool}" used in run ${run.runId}`
+        });
+      }
+    }
+    if (!violated) {
+      checks.push({
+        id: "forbidTool",
+        name: `Forbid tool: ${tool}`,
+        ok: true,
+        message: `Forbidden tool "${tool}" not used`
+      });
+    }
+  }
+  for (const observation of options.requireObservations ?? []) {
+    for (const run of runs) {
+      if (run.filePath === void 0) continue;
+      try {
+        const status = await observationStatus(run.filePath, observation);
+        const ok = status === "passed";
+        checks.push({
+          id: "requireObservation",
+          name: `Require observation: ${observation}`,
+          ok,
+          expected: "passed",
+          actual: status,
+          runId: run.runId,
+          message: ok ? `Observation "${observation}" passed in run ${run.runId}` : status === "missing" ? `Observation "${observation}" missing in run ${run.runId}` : `Observation "${observation}" failed in run ${run.runId}`
+        });
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        readErrors.push(`Run ${run.runId}: ${message}`);
+      }
+    }
+  }
+  return { checks, readErrors };
+}
+function checksFromSuiteResult(suiteResult) {
+  const checks = [
+    {
+      id: "suite",
+      name: `Suite: ${suiteResult.suiteName}`,
+      ok: suiteResult.ok,
+      message: suiteResult.ok ? `Suite passed (${suiteResult.summary.passed} cases)` : `Suite failed (${suiteResult.summary.failed} failed, ${suiteResult.summary.errors} errors)`
+    }
+  ];
+  for (const suiteCase of suiteResult.cases) {
+    if (suiteCase.status === "pass") continue;
+    checks.push({
+      id: "suite",
+      name: `Case: ${suiteCase.id}`,
+      ok: false,
+      message: suiteCase.message ?? `Case status: ${suiteCase.status}`
+    });
+  }
+  return checks;
+}
+function resolveExitCode(input) {
+  if (input.configError) return 2;
+  if (input.readError) return 3;
+  if (!input.ok) return 1;
+  return 0;
+}
+function validateOptions(options) {
+  const errors = [];
+  const hasSuite = options.suitePath !== void 0 && options.suitePath.trim() !== "";
+  const hasThresholds2 = gateHasThresholds(options);
+  if (!hasSuite && !hasThresholds2) {
+    errors.push(
+      "No gate rules specified. Pass --suite or at least one threshold flag."
+    );
+  }
+  if (hasThresholds2 && (options.traceDir === void 0 || options.traceDir.trim() === "")) {
+    if (!hasSuite) {
+      errors.push("Threshold flags require --dir <trace-directory>.");
+    }
+  }
+  if (options.maxErrorRate !== void 0 && options.maxErrorRate < 0) {
+    errors.push("--max-error-rate must be a non-negative percentage.");
+  }
+  if (options.maxP95DurationMs !== void 0 && options.maxP95DurationMs < 0) {
+    errors.push("--max-p95-duration must be a non-negative millisecond value.");
+  }
+  return errors;
+}
+function isConfigLoadError(error) {
+  if (!(error instanceof Error)) return false;
+  const ext = path__default.default.extname(error.message);
+  if (error.message.includes("Unsupported suite config extension")) return true;
+  if (error.message.includes("TypeScript suite configs require")) return true;
+  if (error.message.includes("No suite config found")) return true;
+  if (error.message.includes("AI_SUITE_CONFIG")) return true;
+  if (ext === ".ts" || ext === ".mts" || ext === ".cts") return true;
+  return "diagnostics" in error;
+}
+async function runGate(runs, options) {
+  const diagnostics = [];
+  const checks = [];
+  const validationErrors = validateOptions(options);
+  if (validationErrors.length > 0) {
+    return {
+      ok: false,
+      exitCode: 2,
+      runCount: 0,
+      checks,
+      diagnostics: validationErrors
+    };
+  }
+  let traceDir = options.traceDir?.trim();
+  let suiteResult;
+  if (options.suitePath !== void 0 && options.suitePath.trim() !== "") {
+    try {
+      suiteResult = await runSuite({
+        configPath: options.suitePath,
+        cwd: options.cwd
+      });
+      traceDir = traceDir ?? suiteResult.tracesDir;
+      checks.push(...checksFromSuiteResult(suiteResult));
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      diagnostics.push(message);
+      return {
+        ok: false,
+        exitCode: isConfigLoadError(error) ? 2 : 3,
+        traceDir,
+        suitePath: options.suitePath,
+        runCount: 0,
+        checks,
+        diagnostics
+      };
+    }
+  }
+  if (gateHasThresholds(options)) {
+    const thresholdDir = traceDir;
+    if (thresholdDir === void 0 || thresholdDir.trim() === "") {
+      return {
+        ok: false,
+        exitCode: 2,
+        traceDir,
+        suitePath: options.suitePath,
+        runCount: runs.length,
+        checks,
+        diagnostics: ["Threshold evaluation requires a trace directory."],
+        ...suiteResult !== void 0 ? { suiteResult } : {}
+      };
+    }
+    const thresholdRuns = runs.length > 0 ? runs : [];
+    const { checks: thresholdChecks, readErrors } = await evaluateGateThresholds(
+      thresholdRuns,
+      options
+    );
+    checks.push(...thresholdChecks);
+    diagnostics.push(...readErrors);
+    if (readErrors.length > 0) {
+      const ok2 = checks.length > 0 && checks.every((item) => item.ok);
+      return {
+        ok: ok2,
+        exitCode: resolveExitCode({
+          ok: ok2,
+          configError: false,
+          readError: true
+        }),
+        traceDir: thresholdDir,
+        suitePath: options.suitePath,
+        runCount: thresholdRuns.length,
+        checks,
+        diagnostics,
+        ...suiteResult !== void 0 ? { suiteResult } : {}
+      };
+    }
+  }
+  const ok = checks.length > 0 && checks.every((item) => item.ok);
+  return {
+    ok,
+    exitCode: resolveExitCode({ ok, configError: false, readError: false }),
+    traceDir,
+    suitePath: options.suitePath,
+    runCount: runs.length,
+    checks,
+    diagnostics,
+    ...suiteResult !== void 0 ? { suiteResult } : {}
+  };
+}
+
+// packages/core/src/gate/render.ts
+function escapeXml(value) {
+  return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+function renderGateSummaryMarkdown(result) {
+  const lines = [];
+  lines.push("# AgentInspect gate");
+  lines.push("");
+  lines.push(`Status: **${result.ok ? "PASS" : "FAIL"}** (exit ${result.exitCode})`);
+  if (result.traceDir !== void 0) {
+    lines.push(`Trace directory: \`${result.traceDir}\``);
+  }
+  if (result.suitePath !== void 0) {
+    lines.push(`Suite config: \`${result.suitePath}\``);
+  }
+  lines.push(`Runs evaluated: ${result.runCount}`);
+  lines.push("");
+  if (result.diagnostics.length > 0) {
+    lines.push("## Diagnostics");
+    for (const item of result.diagnostics) lines.push(`- ${item}`);
+    lines.push("");
+  }
+  lines.push("## Checks");
+  for (const check of result.checks) {
+    const flag = check.ok ? "PASS" : "FAIL";
+    lines.push(`- [${flag}] ${check.name}: ${check.message}`);
+  }
+  lines.push("");
+  return lines.join("\n").trimEnd();
+}
+function renderGateGithubStepSummary(result) {
+  const lines = [];
+  lines.push(`## AgentInspect gate: ${result.ok ? "PASS" : "FAIL"}`);
+  lines.push("");
+  lines.push("| Check | Status | Details |");
+  lines.push("| --- | --- | --- |");
+  for (const check of result.checks) {
+    lines.push(
+      `| ${check.name} | ${check.ok ? "pass" : "fail"} | ${check.message.replace(/\|/g, "/")} |`
+    );
+  }
+  if (result.diagnostics.length > 0) {
+    lines.push("");
+    lines.push("**Diagnostics**");
+    for (const item of result.diagnostics) lines.push(`- ${item}`);
+  }
+  return lines.join("\n").trimEnd();
+}
+function renderGateReportHtml(result) {
+  const rows = result.checks.map(
+    (check) => `<tr><td>${escapeHtml(check.name)}</td><td>${check.ok ? "PASS" : "FAIL"}</td><td>${escapeHtml(check.message)}</td></tr>`
+  ).join("");
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <title>Gate report</title>
+  <style>
+    body { font-family: system-ui, sans-serif; margin: 2rem; }
+    table { border-collapse: collapse; width: 100%; }
+    th, td { border: 1px solid #ddd; padding: 0.5rem; text-align: left; }
+    th { background: #f6f6f6; }
+  </style>
+</head>
+<body>
+  <h1>AgentInspect gate</h1>
+  <p>Status: <strong>${result.ok ? "PASS" : "FAIL"}</strong> (exit ${result.exitCode})</p>
+  <h2>Checks</h2>
+  <table>
+    <thead><tr><th>Check</th><th>Status</th><th>Details</th></tr></thead>
+    <tbody>${rows}</tbody>
+  </table>
+</body>
+</html>`;
+}
+function renderGateJUnit(result) {
+  const failures = result.checks.filter((check) => !check.ok).length;
+  const tests = result.checks.length;
+  const cases = result.checks.map((check) => {
+    if (check.ok) {
+      return `    <testcase name="${escapeXml(check.name)}" classname="gate" />`;
+    }
+    return `    <testcase name="${escapeXml(check.name)}" classname="gate">
+      <failure message="${escapeXml(check.message)}">${escapeXml(check.message)}</failure>
+    </testcase>`;
+  }).join("\n");
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="${tests}" failures="${failures}" errors="0" time="0">
+  <testsuite name="agent-inspect-gate" tests="${tests}" failures="${failures}" errors="0" time="0">
+${cases}
+  </testsuite>
+</testsuites>`;
+}
+function renderGateReport(result, options = {}) {
+  const format = options.format ?? "markdown";
+  if (format === "json") return JSON.stringify(result, null, 2);
+  if (format === "html") return renderGateReportHtml(result);
+  if (format === "junit") return renderGateJUnit(result);
+  if (format === "github") return renderGateGithubStepSummary(result);
+  return renderGateSummaryMarkdown(result);
+}
+
 // packages/core/src/inspect-run.ts
 function normalizeRunName(name) {
   if (typeof name !== "string" || name.trim() === "") {
@@ -8576,6 +9493,7 @@ async function maybeInspectRun(name, fn, options) {
   return inspectRun(name, fn, options);
 }
 
+exports.COHORT_METRIC_IDS = COHORT_METRIC_IDS;
 exports.DEFAULT_MAX_EVENT_BYTES = DEFAULT_MAX_EVENT_BYTES;
 exports.DEFAULT_MAX_METADATA_VALUE_LENGTH = DEFAULT_MAX_METADATA_VALUE_LENGTH;
 exports.DEFAULT_MAX_PREVIEW_LENGTH = DEFAULT_MAX_PREVIEW_LENGTH;
@@ -8592,6 +9510,7 @@ exports.TERMINAL_INDENT = TERMINAL_INDENT;
 exports.TraceDirectory = TraceDirectory;
 exports.aggregateBundleSafeStatus = aggregateBundleSafeStatus;
 exports.aggregateSessionCheckResults = aggregateSessionCheckResults;
+exports.analyzeCohort = analyzeCohort;
 exports.buildActivitySummary = buildActivitySummary;
 exports.buildBundleMetadata = buildBundleMetadata;
 exports.buildBundleSummaryMarkdown = buildBundleSummaryMarkdown;
@@ -8603,6 +9522,7 @@ exports.buildRunWhatSummary = buildRunWhatSummary;
 exports.buildSessionIndex = buildSessionIndex;
 exports.buildTraceStats = buildTraceStats;
 exports.bundleFailsOnSafety = bundleFailsOnSafety;
+exports.compareCohortAggregates = compareCohortAggregates;
 exports.createInspector = createInspector;
 exports.createInspectorRuntime = createInspectorRuntime;
 exports.createRunId = createRunId;
@@ -8621,6 +9541,7 @@ exports.formatDuration = formatDuration2;
 exports.formatError = formatError;
 exports.formatTerminalName = formatTerminalName;
 exports.formatTimestamp = formatTimestamp;
+exports.gateHasThresholds = gateHasThresholds;
 exports.getCurrentContext = getCurrentContext;
 exports.getCurrentCorrelationMetadata = getCurrentCorrelationMetadata;
 exports.getCurrentDepth = getCurrentDepth;
@@ -8650,8 +9571,12 @@ exports.loadTraceMetadataList = loadTraceMetadataList;
 exports.maybeInspectRun = maybeInspectRun;
 exports.normalizeBundleOutputPath = normalizeBundleOutputPath;
 exports.normalizeSuiteConfig = normalizeSuiteConfig;
+exports.parseCohortMetricList = parseCohortMetricList;
 exports.parseDuration = parseDuration;
 exports.parseDurationFilter = parseDurationFilter;
+exports.parseGateList = parseGateList;
+exports.parseGateNumber = parseGateNumber;
+exports.parseGroupBySpec = parseGroupBySpec;
 exports.parseTraceJsonl = parseTraceJsonl;
 exports.prepareMetadataForDisk = prepareMetadataForDisk;
 exports.prepareTraceEventForDisk = prepareTraceEventForDisk;
@@ -8664,7 +9589,13 @@ exports.printStepStart = printStepStart;
 exports.readTraceEvents = readTraceEvents;
 exports.readTraceFile = readTraceFile;
 exports.renderActivitySummaryHuman = renderActivitySummaryHuman;
+exports.renderCohortReport = renderCohortReport;
+exports.renderCohortSummaryMarkdown = renderCohortSummaryMarkdown;
 exports.renderErrorLine = renderErrorLine;
+exports.renderGateGithubStepSummary = renderGateGithubStepSummary;
+exports.renderGateJUnit = renderGateJUnit;
+exports.renderGateReport = renderGateReport;
+exports.renderGateSummaryMarkdown = renderGateSummaryMarkdown;
 exports.renderRunSummary = renderRunSummary;
 exports.renderRunWhat = renderRunWhat;
 exports.renderStepLine = renderStepLine;
@@ -8678,6 +9609,7 @@ exports.resolveSuiteCaseTrace = resolveSuiteCaseTrace;
 exports.resolveSuiteConfigPath = resolveSuiteConfigPath;
 exports.resolveTraceDir = resolveTraceDir;
 exports.resolveTraceSafetyOptions = resolveTraceSafetyOptions;
+exports.runGate = runGate;
 exports.runSuite = runSuite;
 exports.runWithContext = runWithContext;
 exports.runWithStepContext = runWithStepContext;