agent-inspect 5.1.0 → 5.2.0

package/packages/core/dist/advanced.cjs

@@ -4086,9 +4086,9 @@ async function searchTraces(metas, options) {
   }
   const limit = options.limit ?? 50;
   const sessionId = options.session?.trim();
-  const observationStatus = parseObservationFilter(options.observation);
+  const observationStatus2 = parseObservationFilter(options.observation);
   const hasContentFilter = Boolean(
-    options.status || stepTypeFilter || nameQuery || toolQuery || durationFilter || observationStatus
+    options.status || stepTypeFilter || nameQuery || toolQuery || durationFilter || observationStatus2
   );
   const results = [];
   const sessionLabel = sessionId && sessionId !== "" ? sessionId : void 0;
@@ -4133,9 +4133,9 @@ async function searchTraces(metas, options) {
       statusFilter: options.status
     });
     results.push(...stepMatches);
-    if (observationStatus) {
+    if (observationStatus2) {
       const outcomes = extractOutcomesFromTraceEvents(events);
-      const matched = outcomes.filter((outcome) => outcome.status === observationStatus);
+      const matched = outcomes.filter((outcome) => outcome.status === observationStatus2);
       for (const outcome of matched) {
         results.push({
           runId: m.runId,
@@ -4971,12 +4971,12 @@ function buildCriticalPath(runs, handoffs) {
     handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
   );
   const ordered = [...runs].sort(compareRuns);
-  const path11 = [];
+  const path12 = [];
   const visited = /* @__PURE__ */ new Set();
   const pushRun = (run, confidence, source) => {
     if (visited.has(run.runId)) return;
     visited.add(run.runId);
-    path11.push({
+    path12.push({
       runId: run.runId,
       name: run.name,
       startedAt: run.startedAt,
@@ -5001,7 +5001,7 @@ function buildCriticalPath(runs, handoffs) {
     const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
     pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
   }
-  return path11;
+  return path12;
 }
 function metaRunIdMatches(run, token, runById) {
   const meta = extractSessionWorkflowMetadata(run.metadata);
@@ -5824,7 +5824,7 @@ function stripPrefix(name, prefixes) {
   }
   return name;
 }
-function eventEvidence(event, path11) {
+function eventEvidence(event, path12) {
   return {
     runId: event.runId,
     eventId: event.eventId,
@@ -5834,7 +5834,7 @@ function eventEvidence(event, path11) {
     kind: event.kind,
     name: event.name,
     status: event.status,
-    ...path11 ? { path: path11 } : {}
+    ...path12 ? { path: path12 } : {}
   };
 }
 function runEvidence(run) {
@@ -8967,6 +8967,402 @@ function renderCohortReport(result, options = {}) {
   return renderCohortSummaryMarkdown(result);
 }
 
+// packages/core/src/gate/parse.ts
+function parseGateList(value) {
+  if (value === void 0 || value.trim() === "") return [];
+  return value.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
+}
+function parseGateNumber(value, label) {
+  if (value === void 0 || value.trim() === "") return void 0;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) {
+    throw new Error(`Invalid ${label}: ${value}`);
+  }
+  return parsed;
+}
+
+// packages/core/src/gate/evaluate.ts
+function percentile3(values, p) {
+  if (values.length === 0) return void 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const idx = Math.min(
+    sorted.length - 1,
+    Math.max(0, Math.ceil(p / 100 * sorted.length) - 1)
+  );
+  return sorted[idx];
+}
+function hasThresholds(options) {
+  return options.maxErrorRate !== void 0 || options.maxP95DurationMs !== void 0 || (options.forbidTools?.length ?? 0) > 0 || (options.requireObservations?.length ?? 0) > 0;
+}
+function gateHasThresholds(options) {
+  return hasThresholds(options);
+}
+async function loadRunMetrics(runs) {
+  const metrics = [];
+  for (const run of runs) {
+    if (run.filePath === void 0) continue;
+    metrics.push(
+      await computeCohortRunMetrics({
+        runId: run.runId,
+        filePath: run.filePath,
+        metadata: run.metadata,
+        status: run.status,
+        durationMs: run.durationMs,
+        groupKey: "all"
+      })
+    );
+  }
+  return metrics;
+}
+async function observationStatus(filePath, name) {
+  const events = await readTraceEventsFromFile(filePath);
+  const outcomes = extractOutcomesFromTraceEvents(events);
+  const match = outcomes.find((item) => item.name === name);
+  if (!match) return "missing";
+  return match.status === "passed" ? "passed" : "failed";
+}
+async function evaluateGateThresholds(runs, options) {
+  const checks = [];
+  const readErrors = [];
+  if (!hasThresholds(options)) {
+    return { checks, readErrors };
+  }
+  if (runs.length === 0) {
+    readErrors.push("No trace runs found in the gate directory.");
+    return { checks, readErrors };
+  }
+  let runMetrics;
+  try {
+    runMetrics = await loadRunMetrics(runs);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    readErrors.push(message);
+    return { checks, readErrors };
+  }
+  if (options.maxErrorRate !== void 0) {
+    const errors = runMetrics.filter((run) => run.error).length;
+    const actual = runMetrics.length > 0 ? errors / runMetrics.length * 100 : 0;
+    const ok = actual <= options.maxErrorRate;
+    checks.push({
+      id: "maxErrorRate",
+      name: "Max error rate",
+      ok,
+      expected: options.maxErrorRate,
+      actual: Math.round(actual * 10) / 10,
+      message: ok ? `Error rate ${actual.toFixed(1)}% within limit ${options.maxErrorRate}%` : `Error rate ${actual.toFixed(1)}% exceeds limit ${options.maxErrorRate}%`
+    });
+  }
+  if (options.maxP95DurationMs !== void 0) {
+    const durations = runMetrics.map((run) => run.durationMs).filter((value) => typeof value === "number");
+    const actual = percentile3(durations, 95);
+    const ok = actual !== void 0 && actual <= options.maxP95DurationMs;
+    checks.push({
+      id: "maxP95Duration",
+      name: "Max p95 duration (ms)",
+      ok,
+      expected: options.maxP95DurationMs,
+      actual: actual ?? "n/a",
+      message: actual === void 0 ? "No duration samples available for p95 check." : ok ? `P95 duration ${Math.round(actual)} ms within limit ${options.maxP95DurationMs} ms` : `P95 duration ${Math.round(actual)} ms exceeds limit ${options.maxP95DurationMs} ms`
+    });
+  }
+  for (const tool of options.forbidTools ?? []) {
+    let violated = false;
+    for (const run of runMetrics) {
+      const used = run.toolChoices.includes(tool) || run.toolOrdering.includes(tool);
+      if (used) {
+        violated = true;
+        checks.push({
+          id: "forbidTool",
+          name: `Forbid tool: ${tool}`,
+          ok: false,
+          expected: `not used`,
+          actual: "used",
+          runId: run.runId,
+          message: `Forbidden tool "${tool}" used in run ${run.runId}`
+        });
+      }
+    }
+    if (!violated) {
+      checks.push({
+        id: "forbidTool",
+        name: `Forbid tool: ${tool}`,
+        ok: true,
+        message: `Forbidden tool "${tool}" not used`
+      });
+    }
+  }
+  for (const observation of options.requireObservations ?? []) {
+    for (const run of runs) {
+      if (run.filePath === void 0) continue;
+      try {
+        const status = await observationStatus(run.filePath, observation);
+        const ok = status === "passed";
+        checks.push({
+          id: "requireObservation",
+          name: `Require observation: ${observation}`,
+          ok,
+          expected: "passed",
+          actual: status,
+          runId: run.runId,
+          message: ok ? `Observation "${observation}" passed in run ${run.runId}` : status === "missing" ? `Observation "${observation}" missing in run ${run.runId}` : `Observation "${observation}" failed in run ${run.runId}`
+        });
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        readErrors.push(`Run ${run.runId}: ${message}`);
+      }
+    }
+  }
+  return { checks, readErrors };
+}
+function checksFromSuiteResult(suiteResult) {
+  const checks = [
+    {
+      id: "suite",
+      name: `Suite: ${suiteResult.suiteName}`,
+      ok: suiteResult.ok,
+      message: suiteResult.ok ? `Suite passed (${suiteResult.summary.passed} cases)` : `Suite failed (${suiteResult.summary.failed} failed, ${suiteResult.summary.errors} errors)`
+    }
+  ];
+  for (const suiteCase of suiteResult.cases) {
+    if (suiteCase.status === "pass") continue;
+    checks.push({
+      id: "suite",
+      name: `Case: ${suiteCase.id}`,
+      ok: false,
+      message: suiteCase.message ?? `Case status: ${suiteCase.status}`
+    });
+  }
+  return checks;
+}
+function resolveExitCode(input) {
+  if (input.configError) return 2;
+  if (input.readError) return 3;
+  if (!input.ok) return 1;
+  return 0;
+}
+function validateOptions(options) {
+  const errors = [];
+  const hasSuite = options.suitePath !== void 0 && options.suitePath.trim() !== "";
+  const hasThresholds2 = gateHasThresholds(options);
+  if (!hasSuite && !hasThresholds2) {
+    errors.push(
+      "No gate rules specified. Pass --suite or at least one threshold flag."
+    );
+  }
+  if (hasThresholds2 && (options.traceDir === void 0 || options.traceDir.trim() === "")) {
+    if (!hasSuite) {
+      errors.push("Threshold flags require --dir <trace-directory>.");
+    }
+  }
+  if (options.maxErrorRate !== void 0 && options.maxErrorRate < 0) {
+    errors.push("--max-error-rate must be a non-negative percentage.");
+  }
+  if (options.maxP95DurationMs !== void 0 && options.maxP95DurationMs < 0) {
+    errors.push("--max-p95-duration must be a non-negative millisecond value.");
+  }
+  return errors;
+}
+function isConfigLoadError(error) {
+  if (!(error instanceof Error)) return false;
+  const ext = path__default.default.extname(error.message);
+  if (error.message.includes("Unsupported suite config extension")) return true;
+  if (error.message.includes("TypeScript suite configs require")) return true;
+  if (error.message.includes("No suite config found")) return true;
+  if (error.message.includes("AI_SUITE_CONFIG")) return true;
+  if (ext === ".ts" || ext === ".mts" || ext === ".cts") return true;
+  return "diagnostics" in error;
+}
+async function runGate(runs, options) {
+  const diagnostics = [];
+  const checks = [];
+  const validationErrors = validateOptions(options);
+  if (validationErrors.length > 0) {
+    return {
+      ok: false,
+      exitCode: 2,
+      runCount: 0,
+      checks,
+      diagnostics: validationErrors
+    };
+  }
+  let traceDir = options.traceDir?.trim();
+  let suiteResult;
+  if (options.suitePath !== void 0 && options.suitePath.trim() !== "") {
+    try {
+      suiteResult = await runSuite({
+        configPath: options.suitePath,
+        cwd: options.cwd
+      });
+      traceDir = traceDir ?? suiteResult.tracesDir;
+      checks.push(...checksFromSuiteResult(suiteResult));
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      diagnostics.push(message);
+      return {
+        ok: false,
+        exitCode: isConfigLoadError(error) ? 2 : 3,
+        traceDir,
+        suitePath: options.suitePath,
+        runCount: 0,
+        checks,
+        diagnostics
+      };
+    }
+  }
+  if (gateHasThresholds(options)) {
+    const thresholdDir = traceDir;
+    if (thresholdDir === void 0 || thresholdDir.trim() === "") {
+      return {
+        ok: false,
+        exitCode: 2,
+        traceDir,
+        suitePath: options.suitePath,
+        runCount: runs.length,
+        checks,
+        diagnostics: ["Threshold evaluation requires a trace directory."],
+        ...suiteResult !== void 0 ? { suiteResult } : {}
+      };
+    }
+    const thresholdRuns = runs.length > 0 ? runs : [];
+    const { checks: thresholdChecks, readErrors } = await evaluateGateThresholds(
+      thresholdRuns,
+      options
+    );
+    checks.push(...thresholdChecks);
+    diagnostics.push(...readErrors);
+    if (readErrors.length > 0) {
+      const ok2 = checks.length > 0 && checks.every((item) => item.ok);
+      return {
+        ok: ok2,
+        exitCode: resolveExitCode({
+          ok: ok2,
+          configError: false,
+          readError: true
+        }),
+        traceDir: thresholdDir,
+        suitePath: options.suitePath,
+        runCount: thresholdRuns.length,
+        checks,
+        diagnostics,
+        ...suiteResult !== void 0 ? { suiteResult } : {}
+      };
+    }
+  }
+  const ok = checks.length > 0 && checks.every((item) => item.ok);
+  return {
+    ok,
+    exitCode: resolveExitCode({ ok, configError: false, readError: false }),
+    traceDir,
+    suitePath: options.suitePath,
+    runCount: runs.length,
+    checks,
+    diagnostics,
+    ...suiteResult !== void 0 ? { suiteResult } : {}
+  };
+}
+
+// packages/core/src/gate/render.ts
+function escapeXml(value) {
+  return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+function renderGateSummaryMarkdown(result) {
+  const lines = [];
+  lines.push("# AgentInspect gate");
+  lines.push("");
+  lines.push(`Status: **${result.ok ? "PASS" : "FAIL"}** (exit ${result.exitCode})`);
+  if (result.traceDir !== void 0) {
+    lines.push(`Trace directory: \`${result.traceDir}\``);
+  }
+  if (result.suitePath !== void 0) {
+    lines.push(`Suite config: \`${result.suitePath}\``);
+  }
+  lines.push(`Runs evaluated: ${result.runCount}`);
+  lines.push("");
+  if (result.diagnostics.length > 0) {
+    lines.push("## Diagnostics");
+    for (const item of result.diagnostics) lines.push(`- ${item}`);
+    lines.push("");
+  }
+  lines.push("## Checks");
+  for (const check of result.checks) {
+    const flag = check.ok ? "PASS" : "FAIL";
+    lines.push(`- [${flag}] ${check.name}: ${check.message}`);
+  }
+  lines.push("");
+  return lines.join("\n").trimEnd();
+}
+function renderGateGithubStepSummary(result) {
+  const lines = [];
+  lines.push(`## AgentInspect gate: ${result.ok ? "PASS" : "FAIL"}`);
+  lines.push("");
+  lines.push("| Check | Status | Details |");
+  lines.push("| --- | --- | --- |");
+  for (const check of result.checks) {
+    lines.push(
+      `| ${check.name} | ${check.ok ? "pass" : "fail"} | ${check.message.replace(/\|/g, "/")} |`
+    );
+  }
+  if (result.diagnostics.length > 0) {
+    lines.push("");
+    lines.push("**Diagnostics**");
+    for (const item of result.diagnostics) lines.push(`- ${item}`);
+  }
+  return lines.join("\n").trimEnd();
+}
+function renderGateReportHtml(result) {
+  const rows = result.checks.map(
+    (check) => `<tr><td>${escapeHtml(check.name)}</td><td>${check.ok ? "PASS" : "FAIL"}</td><td>${escapeHtml(check.message)}</td></tr>`
+  ).join("");
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <title>Gate report</title>
+  <style>
+    body { font-family: system-ui, sans-serif; margin: 2rem; }
+    table { border-collapse: collapse; width: 100%; }
+    th, td { border: 1px solid #ddd; padding: 0.5rem; text-align: left; }
+    th { background: #f6f6f6; }
+  </style>
+</head>
+<body>
+  <h1>AgentInspect gate</h1>
+  <p>Status: <strong>${result.ok ? "PASS" : "FAIL"}</strong> (exit ${result.exitCode})</p>
+  <h2>Checks</h2>
+  <table>
+    <thead><tr><th>Check</th><th>Status</th><th>Details</th></tr></thead>
+    <tbody>${rows}</tbody>
+  </table>
+</body>
+</html>`;
+}
+function renderGateJUnit(result) {
+  const failures = result.checks.filter((check) => !check.ok).length;
+  const tests = result.checks.length;
+  const cases = result.checks.map((check) => {
+    if (check.ok) {
+      return `    <testcase name="${escapeXml(check.name)}" classname="gate" />`;
+    }
+    return `    <testcase name="${escapeXml(check.name)}" classname="gate">
+      <failure message="${escapeXml(check.message)}">${escapeXml(check.message)}</failure>
+    </testcase>`;
+  }).join("\n");
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="${tests}" failures="${failures}" errors="0" time="0">
+  <testsuite name="agent-inspect-gate" tests="${tests}" failures="${failures}" errors="0" time="0">
+${cases}
+  </testsuite>
+</testsuites>`;
+}
+function renderGateReport(result, options = {}) {
+  const format = options.format ?? "markdown";
+  if (format === "json") return JSON.stringify(result, null, 2);
+  if (format === "html") return renderGateReportHtml(result);
+  if (format === "junit") return renderGateJUnit(result);
+  if (format === "github") return renderGateGithubStepSummary(result);
+  return renderGateSummaryMarkdown(result);
+}
+
 // packages/core/src/inspect-run.ts
 function normalizeRunName(name) {
   if (typeof name !== "string" || name.trim() === "") {
@@ -9145,6 +9541,7 @@ exports.formatDuration = formatDuration2;
 exports.formatError = formatError;
 exports.formatTerminalName = formatTerminalName;
 exports.formatTimestamp = formatTimestamp;
+exports.gateHasThresholds = gateHasThresholds;
 exports.getCurrentContext = getCurrentContext;
 exports.getCurrentCorrelationMetadata = getCurrentCorrelationMetadata;
 exports.getCurrentDepth = getCurrentDepth;
@@ -9177,6 +9574,8 @@ exports.normalizeSuiteConfig = normalizeSuiteConfig;
 exports.parseCohortMetricList = parseCohortMetricList;
 exports.parseDuration = parseDuration;
 exports.parseDurationFilter = parseDurationFilter;
+exports.parseGateList = parseGateList;
+exports.parseGateNumber = parseGateNumber;
 exports.parseGroupBySpec = parseGroupBySpec;
 exports.parseTraceJsonl = parseTraceJsonl;
 exports.prepareMetadataForDisk = prepareMetadataForDisk;
@@ -9193,6 +9592,10 @@ exports.renderActivitySummaryHuman = renderActivitySummaryHuman;
 exports.renderCohortReport = renderCohortReport;
 exports.renderCohortSummaryMarkdown = renderCohortSummaryMarkdown;
 exports.renderErrorLine = renderErrorLine;
+exports.renderGateGithubStepSummary = renderGateGithubStepSummary;
+exports.renderGateJUnit = renderGateJUnit;
+exports.renderGateReport = renderGateReport;
+exports.renderGateSummaryMarkdown = renderGateSummaryMarkdown;
 exports.renderRunSummary = renderRunSummary;
 exports.renderRunWhat = renderRunWhat;
 exports.renderStepLine = renderStepLine;
@@ -9206,6 +9609,7 @@ exports.resolveSuiteCaseTrace = resolveSuiteCaseTrace;
 exports.resolveSuiteConfigPath = resolveSuiteConfigPath;
 exports.resolveTraceDir = resolveTraceDir;
 exports.resolveTraceSafetyOptions = resolveTraceSafetyOptions;
+exports.runGate = runGate;
 exports.runSuite = runSuite;
 exports.runWithContext = runWithContext;
 exports.runWithStepContext = runWithStepContext;