agent-inspect 5.1.0 → 5.2.0

package/packages/cli/dist/index.mjs

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createSafetyOversizedAttributeRule, resolveTraceDir, parseDuration, buildSessionIndex, enrichSessionRunRecord, TraceDirectory, loadTraceMetadataList, loadSessionRunRecords, extractMetadata, filterTraces, truncateName, getTraceFilePath, buildRunSummary, formatDuration, formatTimestamp, isAgentInspectTrace, openTrace, TraceReadError, filterMetasBySessionScope, runTraceChecks, aggregateSessionCheckResults, createBaselineRegressionRule, resolveBundleRunIds, aggregateBundleSafeStatus, bundleFailsOnSafety, buildPlaceholderArtifact, buildBundleMetadata, buildBundleSummaryMarkdown, buildRunTimeline, renderTimeline, buildTraceStats, renderTraceStats, parseDurationFilter, searchTraces, buildActivitySummary, renderActivitySummaryHuman, buildRunWhatSummary, renderRunWhat, buildLocalExplanation, defaultSuiteConfigTemplate, loadSuiteConfig, validateSuiteConfig, runSuite, renderSuiteReport, analyzeCohort, renderCohortReport, persistedInspectEventsToTraceEvents, renderStepLine, renderErrorLine, getIndent, Redactor, TreeBuilder, zeroKinds, validateEvent, traceEventToPersistedInspectEvent, isPersistedInspectEvent, createRunStatusRule, createStructureOrphanRule, createStructureCycleRule, createRunDurationRule, createMaxStepDurationRule, createRequireCompletedRule, createStallDetectionRule, createObservedOutcomeRule, createRunDepthRule, createToolUsageRule, createLlmUsageRule, createStructureRelationshipRule, createStructureParallelWidthRule, normalizeBundleOutputPath, defaultBundleOutputPath, summarizeObservedOutcomes, extractOutcomesFromTraceEvents, renderObservedOutcomesMarkdown, renderObservedOutcomesHtml, escapeMarkdown, escapeHtml, DEFAULT_SUITE_ARTIFACTS_DIR, parseCohortMetricList, COHORT_METRIC_IDS, nanoid, resolveRedactionProfile, applyProfileMetadataCaps, flattenTree, safeString, compactAttributes, stableJson, truncateStringForProfile } from './chunk-TSIQUIPF.mjs';
+import { createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createSafetyOversizedAttributeRule, resolveTraceDir, parseDuration, buildSessionIndex, enrichSessionRunRecord, TraceDirectory, loadTraceMetadataList, loadSessionRunRecords, extractMetadata, filterTraces, truncateName, getTraceFilePath, buildRunSummary, formatDuration, formatTimestamp, isAgentInspectTrace, openTrace, TraceReadError, filterMetasBySessionScope, runTraceChecks, aggregateSessionCheckResults, createBaselineRegressionRule, resolveBundleRunIds, aggregateBundleSafeStatus, bundleFailsOnSafety, buildPlaceholderArtifact, buildBundleMetadata, buildBundleSummaryMarkdown, buildRunTimeline, renderTimeline, buildTraceStats, renderTraceStats, parseDurationFilter, searchTraces, buildActivitySummary, renderActivitySummaryHuman, buildRunWhatSummary, renderRunWhat, buildLocalExplanation, defaultSuiteConfigTemplate, loadSuiteConfig, validateSuiteConfig, runSuite, renderSuiteReport, analyzeCohort, renderCohortReport, gateHasThresholds, runGate, renderGateReport, persistedInspectEventsToTraceEvents, renderStepLine, renderErrorLine, getIndent, Redactor, TreeBuilder, zeroKinds, validateEvent, traceEventToPersistedInspectEvent, isPersistedInspectEvent, createRunStatusRule, createStructureOrphanRule, createStructureCycleRule, createRunDurationRule, createMaxStepDurationRule, createRequireCompletedRule, createStallDetectionRule, createObservedOutcomeRule, createRunDepthRule, createToolUsageRule, createLlmUsageRule, createStructureRelationshipRule, createStructureParallelWidthRule, normalizeBundleOutputPath, defaultBundleOutputPath, summarizeObservedOutcomes, extractOutcomesFromTraceEvents, renderObservedOutcomesMarkdown, renderObservedOutcomesHtml, escapeMarkdown, escapeHtml, DEFAULT_SUITE_ARTIFACTS_DIR, parseCohortMetricList, COHORT_METRIC_IDS, parseGateList, nanoid, resolveRedactionProfile, applyProfileMetadataCaps, flattenTree, safeString, compactAttributes, stableJson, truncateStringForProfile } from './chunk-SJ5R2XBE.mjs';
 import { realpathSync, constants as constants$1 } from 'fs';
 import path12 from 'path';
 import { fileURLToPath, pathToFileURL } from 'url';
@@ -11,7 +11,7 @@ import { createServer } from 'http';
 import { createRequire } from 'module';
 
 // package.json
-var version = "5.1.0";
+var version = "5.2.0";
 
 // packages/cli/src/trace-dir-scale.ts
 var TRACE_COUNT_WARN = 1e3;
@@ -3129,13 +3129,13 @@ function pairSteps(left, right) {
   return pairs;
 }
 function compareLeafSteps(L, R, segments, opts, out) {
-  const path20 = buildPath(segments);
+  const path21 = buildPath(segments);
   if (L.name !== R.name) {
     out.push({
       kind: "structure",
       severity: "warning",
       message: "Step name differs",
-      path: path20,
+      path: path21,
       left: L.name,
       right: R.name
     });
@@ -3145,7 +3145,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
       kind: "step-type",
       severity: "warning",
       message: "Step type differs",
-      path: path20,
+      path: path21,
       left: L.type,
       right: R.type
     });
@@ -3155,7 +3155,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
       kind: "step-status",
       severity: "warning",
       message: "Step status differs",
-      path: path20,
+      path: path21,
       left: L.status,
       right: R.status
     });
@@ -3167,7 +3167,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
       kind: "error",
       severity: "error",
       message: "Step error message differs",
-      path: path20,
+      path: path21,
       left: le || void 0,
       right: re || void 0
     });
@@ -3185,7 +3185,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
         kind: "duration",
         severity: "info",
         message: "Step duration differs",
-        path: path20,
+        path: path21,
         left: ld,
         right: rd
       });
@@ -3198,7 +3198,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
       kind: "metadata",
       severity: "info",
       message: "Step metadata differs",
-      path: path20,
+      path: path21,
       left: L.metadata,
       right: R.metadata
     });
@@ -3210,7 +3210,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
       kind: "output",
       severity: "info",
       message: "Output preview differs",
-      path: path20,
+      path: path21,
       left: L.outputPreview,
       right: R.outputPreview
     });
@@ -3370,11 +3370,11 @@ function diffRuns(left, right, options) {
 }
 
 // packages/core/src/diff/renderer.ts
-function formatPath(path20) {
-  if (path20 === void 0 || path20.path.length === 0) {
+function formatPath(path21) {
+  if (path21 === void 0 || path21.path.length === 0) {
     return "(run)";
   }
-  return path20.path.map((s) => s.name).join(" > ");
+  return path21.path.map((s) => s.name).join(" > ");
 }
 function formatValue(v, verbose) {
   if (v === void 0) return "(undefined)";
@@ -3782,7 +3782,7 @@ async function newestTraceMtimeMs(traceDir) {
 }
 async function loadSessionRuns(traceDir) {
   try {
-    const mod = await import('./src-2SZWGKMM.mjs');
+    const mod = await import('./src-PK22BBMH.mjs');
     const dbPath = mod.resolveIndexDbPath(traceDir);
     const status = mod.indexStatus(dbPath);
     if (!status.healthy) {
@@ -4479,17 +4479,17 @@ function applyRule(rule, value, replacement) {
   }
   return value;
 }
-function childPath(path20, key) {
+function childPath(path21, key) {
   if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key)) {
-    return path20 ? `${path20}.${key}` : key;
+    return path21 ? `${path21}.${key}` : key;
   }
-  return `${path20 || "$"}[${JSON.stringify(key)}]`;
+  return `${path21 || "$"}[${JSON.stringify(key)}]`;
 }
-function indexPath(path20, index) {
-  return `${path20 || "$"}[${index}]`;
+function indexPath(path21, index) {
+  return `${path21 || "$"}[${index}]`;
 }
-function makeFinding(path20, detector, action, matchKind, severity = "warning", preview) {
-  return preview === void 0 ? { path: path20, detector, action, severity, matchKind } : { path: path20, detector, action, severity, matchKind, preview };
+function makeFinding(path21, detector, action, matchKind, severity = "warning", preview) {
+  return preview === void 0 ? { path: path21, detector, action, severity, matchKind } : { path: path21, detector, action, severity, matchKind, preview };
 }
 function createRedactionProfile(profile = "local") {
   switch (profile) {
@@ -4558,11 +4558,11 @@ var Redactor2 = class {
   #recordFinding(state, finding) {
     if (this.#collectFindings) state.findings.push(finding);
   }
-  #redactValue(value, key, path20, depth, state) {
+  #redactValue(value, key, path21, depth, state) {
     if (depth > this.#maxDepth) {
       this.#recordFinding(
         state,
-        makeFinding(path20, "structure.maxDepth", "truncate", "value", "warning")
+        makeFinding(path21, "structure.maxDepth", "truncate", "value", "warning")
       );
       return "[Truncated]";
     }
@@ -4571,19 +4571,19 @@ var Redactor2 = class {
       if (rule) {
         this.#recordFinding(
           state,
-          makeFinding(path20, `key.${rule.key}`, actionForRule(rule), "key", "warning")
+          makeFinding(path21, `key.${rule.key}`, actionForRule(rule), "key", "warning")
         );
         return applyRule(rule, value, this.#replacement);
       }
     }
     for (const detector of this.#detectors) {
-      const detections = detector.detect({ path: path20, key, value });
+      const detections = detector.detect({ path: path21, key, value });
       for (const detection of detections) {
         const action = detection.action ?? "replace";
         this.#recordFinding(
           state,
           makeFinding(
-            path20,
+            path21,
             detector.id,
             action,
             detection.matchKind ?? detector.matchKind ?? "custom",
@@ -4601,7 +4601,7 @@ var Redactor2 = class {
       const out = [];
       state.seen.set(value, out);
       value.forEach((item, index) => {
-        out[index] = this.#redactValue(item, void 0, indexPath(path20, index), depth + 1, state);
+        out[index] = this.#redactValue(item, void 0, indexPath(path21, index), depth + 1, state);
       });
       return out;
     }
@@ -4613,7 +4613,7 @@ var Redactor2 = class {
         out[entryKey] = this.#redactValue(
           entryValue,
           entryKey,
-          childPath(path20 === "$" ? "" : path20, entryKey),
+          childPath(path21 === "$" ? "" : path21, entryKey),
           depth + 1,
           state
         );
@@ -5540,23 +5540,23 @@ function evaluatePromptInjection(text, options = {}) {
   }
   return fail(ruleId, `Matched ${evidence.length} injection pattern(s).`, evidence, "warning");
 }
-function validateSchemaField(value, field, path20, evidence) {
+function validateSchemaField(value, field, path21, evidence) {
   const ruleId = "guardrail.structured-output";
   if (field.type) {
     const actual = value === null ? "null" : Array.isArray(value) ? "array" : typeof value;
     if (actual !== field.type) {
-      evidence.push({ ruleId, path: path20, preview: `expected ${field.type}, got ${actual}` });
+      evidence.push({ ruleId, path: path21, preview: `expected ${field.type}, got ${actual}` });
       return;
     }
   }
   if (field.enum && !field.enum.some((item) => Object.is(item, value))) {
-    evidence.push({ ruleId, path: path20, preview: "value not in enum" });
+    evidence.push({ ruleId, path: path21, preview: "value not in enum" });
   }
   if (field.type === "object" && field.required && value && typeof value === "object" && !Array.isArray(value)) {
     const record = value;
     for (const key of field.required) {
       if (!(key in record)) {
-        evidence.push({ ruleId, path: `${path20}.${key}`, preview: "missing required key" });
+        evidence.push({ ruleId, path: `${path21}.${key}`, preview: "missing required key" });
       }
     }
   }
@@ -6057,10 +6057,10 @@ function printHuman(result) {
     console.log(`- ${diagnostic3.code}: ${diagnostic3.message}`);
   }
   for (const finding of result.findings) {
-    const path20 = finding.evidence[0]?.path;
+    const path21 = finding.evidence[0]?.path;
     const run = finding.evidence[0]?.runId;
     const runPrefix = run ? `[${run}] ` : "";
-    console.log(`- ${runPrefix}${finding.ruleId}: ${finding.message}${path20 ? ` (${path20})` : ""}`);
+    console.log(`- ${runPrefix}${finding.ruleId}: ${finding.message}${path21 ? ` (${path21})` : ""}`);
   }
 }
 function readErrorResult(error) {
@@ -6598,10 +6598,10 @@ async function evalRun(input3, options = {}) {
     diagnostics: []
   };
 }
-function evidenceForRun(run, path20) {
-  return [{ runId: run.runId, ...path20 !== void 0 ? { path: path20 } : {} }];
+function evidenceForRun(run, path21) {
+  return [{ runId: run.runId, ...path21 !== void 0 ? { path: path21 } : {} }];
 }
-function evidenceForEvent(event, path20) {
+function evidenceForEvent(event, path21) {
   return [
     {
       runId: event.runId,
@@ -6609,7 +6609,7 @@ function evidenceForEvent(event, path20) {
       ...event.parentId !== void 0 ? { parentId: event.parentId } : {},
       kind: event.kind,
       name: event.name,
-      ...path20 !== void 0 ? { path: path20 } : {}
+      ...path21 !== void 0 ? { path: path21 } : {}
     }
   ];
 }
@@ -6767,9 +6767,9 @@ function collectTextFields(nodes, keys, preferredKinds = []) {
 function tokenize(text) {
   return [...text.toLowerCase().matchAll(/[a-z0-9][a-z0-9'-]{2,}/g)].map((match) => match[0].replace(/^['-]+|['-]+$/g, "")).filter((token) => token.length > 2 && !STOP_WORDS.has(token));
 }
-function firstEvidence(fields, run, path20) {
+function firstEvidence(fields, run, path21) {
   const first = fields[0];
-  return first === void 0 ? evidenceForRun(run, path20) : evidenceForEvent(first.node.event, first.path);
+  return first === void 0 ? evidenceForRun(run, path21) : evidenceForEvent(first.node.event, first.path);
 }
 function collectSourceIds(nodes, keys) {
   const wanted = keySet(keys);
@@ -7146,8 +7146,8 @@ function renderEvalMarkdown(result) {
   if (result.findings.length > 0) {
     lines.push("", "## Findings");
     for (const finding of result.findings) {
-      const path20 = finding.evidence[0]?.path;
-      lines.push(`- ${finding.ruleId}: ${finding.message}${path20 ? ` (${path20})` : ""}`);
+      const path21 = finding.evidence[0]?.path;
+      lines.push(`- ${finding.ruleId}: ${finding.message}${path21 ? ` (${path21})` : ""}`);
     }
   }
   return `${lines.join("\n")}
@@ -7340,8 +7340,8 @@ function printHuman2(result) {
     console.log(`- ${diagnostic3.code}: ${diagnostic3.message}`);
   }
   for (const finding of result.findings) {
-    const path20 = finding.evidence[0]?.path;
-    console.log(`- ${finding.ruleId}: ${finding.message}${path20 ? ` (${path20})` : ""}`);
+    const path21 = finding.evidence[0]?.path;
+    console.log(`- ${finding.ruleId}: ${finding.message}${path21 ? ` (${path21})` : ""}`);
   }
 }
 function readErrorResult2(error) {
@@ -7579,8 +7579,8 @@ function printHuman3(result) {
     console.log(`- ${diagnostic3.code}: ${diagnostic3.message}`);
   }
   for (const finding of result.findings) {
-    const path20 = finding.evidence[0]?.path;
-    console.log(`- ${finding.ruleId}: ${finding.message}${path20 ? ` (${path20})` : ""}`);
+    const path21 = finding.evidence[0]?.path;
+    console.log(`- ${finding.ruleId}: ${finding.message}${path21 ? ` (${path21})` : ""}`);
   }
   console.log(`Note: ${result.note}`);
 }
@@ -7731,8 +7731,8 @@ function renderCheckSection(result) {
     `Diagnostics: ${result.diagnostics.length}`
   ];
   for (const finding of result.findings.slice(0, 10)) {
-    const path20 = finding.evidence[0]?.path ?? "(run)";
-    lines.push(`- ${finding.ruleId}: ${finding.message} (${path20})`);
+    const path21 = finding.evidence[0]?.path ?? "(run)";
+    lines.push(`- ${finding.ruleId}: ${finding.message} (${path21})`);
   }
   for (const diagnostic3 of result.diagnostics.slice(0, 10)) {
     lines.push(`- ${diagnostic3.code}: ${diagnostic3.message}`);
@@ -9686,7 +9686,7 @@ function isModuleNotFound3(e) {
 }
 async function loadIndexSqlite() {
   try {
-    return await import('./src-2SZWGKMM.mjs');
+    return await import('./src-PK22BBMH.mjs');
   } catch (e) {
     if (isModuleNotFound3(e)) {
       console.error(
@@ -10249,6 +10249,132 @@ async function cohortCommand(options = {}) {
     process.exitCode = 1;
   }
 }
+var SUPPORTED_FORMATS = /* @__PURE__ */ new Set(["markdown", "json", "html", "junit", "github"]);
+function collectList(value) {
+  if (value === void 0) return [];
+  const items = Array.isArray(value) ? value : [value];
+  return items.flatMap((item) => parseGateList(item));
+}
+function parseRate(value) {
+  if (value === void 0 || value.trim() === "") return void 0;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) {
+    throw new Error(`Invalid --max-error-rate: ${value}`);
+  }
+  return parsed;
+}
+function parseDuration2(value) {
+  if (value === void 0 || value.trim() === "") return void 0;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) {
+    throw new Error(`Invalid --max-p95-duration: ${value}`);
+  }
+  return parsed;
+}
+async function writeArtifacts2(result, outputDir) {
+  await mkdir(outputDir, { recursive: true });
+  const paths = {
+    jsonPath: path12.join(outputDir, "gate-results.json"),
+    markdownPath: path12.join(outputDir, "gate-summary.md"),
+    htmlPath: path12.join(outputDir, "gate-report.html"),
+    junitPath: path12.join(outputDir, "junit.xml"),
+    githubPath: path12.join(outputDir, "github-step-summary.md")
+  };
+  await writeFile(
+    paths.jsonPath,
+    `${renderGateReport(result, { format: "json" })}
+`,
+    "utf-8"
+  );
+  await writeFile(
+    paths.markdownPath,
+    `${renderGateReport(result, { format: "markdown" })}
+`,
+    "utf-8"
+  );
+  await writeFile(
+    paths.htmlPath,
+    `${renderGateReport(result, { format: "html" })}
+`,
+    "utf-8"
+  );
+  await writeFile(
+    paths.junitPath,
+    `${renderGateReport(result, { format: "junit" })}
+`,
+    "utf-8"
+  );
+  await writeFile(
+    paths.githubPath,
+    `${renderGateReport(result, { format: "github" })}
+`,
+    "utf-8"
+  );
+  return paths;
+}
+async function gateCommand(options = {}) {
+  try {
+    const format = options.format ?? (options.json ? "json" : "markdown");
+    if (!SUPPORTED_FORMATS.has(format)) {
+      if (options.json) {
+        console.log(
+          JSON.stringify({ ok: false, exitCode: 4, error: `Unsupported format: ${format}` }, null, 2)
+        );
+      } else {
+        console.error(`[AgentInspect] Unsupported format: ${format}`);
+      }
+      process.exitCode = 4;
+      return;
+    }
+    const gateOptions = {
+      ...options.dir !== void 0 ? { traceDir: resolveTraceDir({ dir: options.dir }) } : {},
+      ...options.suite !== void 0 ? { suitePath: options.suite } : {},
+      ...options.cwd !== void 0 ? { cwd: options.cwd } : {},
+      ...parseRate(options.maxErrorRate) !== void 0 ? { maxErrorRate: parseRate(options.maxErrorRate) } : {},
+      ...parseDuration2(options.maxP95Duration) !== void 0 ? { maxP95DurationMs: parseDuration2(options.maxP95Duration) } : {},
+      forbidTools: collectList(options.forbidTool),
+      requireObservations: collectList(options.requireObservation)
+    };
+    let runs = [];
+    const needsRuns = gateHasThresholds(gateOptions) || gateOptions.traceDir !== void 0 && gateOptions.suitePath === void 0;
+    if (needsRuns) {
+      const traceDir = gateOptions.traceDir ?? resolveTraceDir({ dir: options.dir });
+      const loaded = await loadSessionRuns(traceDir);
+      runs = loaded.runs;
+      gateOptions.traceDir = traceDir;
+    }
+    const result = await runGate(runs, gateOptions);
+    let artifacts;
+    if (options.output !== void 0 && options.output.trim() !== "") {
+      artifacts = await writeArtifacts2(result, path12.resolve(options.output.trim()));
+    }
+    if (options.json || format === "json") {
+      console.log(JSON.stringify({ ...result, artifacts: artifacts ?? null }, null, 2));
+    } else if (format === "html") {
+      console.log(renderGateReport(result, { format: "html" }));
+    } else if (format === "junit") {
+      console.log(renderGateReport(result, { format: "junit" }));
+    } else if (format === "github") {
+      console.log(renderGateReport(result, { format: "github" }));
+    } else {
+      console.log(renderGateReport(result, { format: "markdown" }));
+      if (artifacts !== void 0) {
+        console.log("");
+        console.log("Artifacts:");
+        for (const value of Object.values(artifacts)) console.log(`- ${value}`);
+      }
+    }
+    process.exitCode = result.exitCode;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (options.json) {
+      console.log(JSON.stringify({ ok: false, exitCode: 2, error: message }, null, 2));
+    } else {
+      console.error(`[AgentInspect] gate failed: ${message}`);
+    }
+    process.exitCode = 2;
+  }
+}
 
 // packages/cli/src/index.ts
 function runCommand(action) {
@@ -10689,6 +10815,15 @@ function createCliProgram() {
   ).option("--format <format>", "markdown, json, or html", "markdown").option("-o, --output <dir>", "write cohort-results.json, cohort-summary.md, cohort-report.html").option("--json", "print deterministic JSON result").action((opts) => {
     runCommand(() => cohortCommand(opts));
   });
+  program.command("gate").description("Run deterministic CI quality gates over traces or suites (v5.2+)").option("--dir <path>", "trace directory for threshold checks").option("--suite <path>", "suite config path").option("--max-error-rate <percent>", "maximum allowed error rate percentage").option("--max-p95-duration <ms>", "maximum allowed p95 run duration in ms").option("--forbid-tool <name>", "forbidden tool name (repeatable or comma-separated)").option(
+    "--require-observation <name>",
+    "required passed observation (repeatable or comma-separated)"
+  ).option("--format <format>", "markdown, json, html, junit, or github", "markdown").option(
+    "-o, --output <dir>",
+    "write gate-results.json, gate-summary.md, gate-report.html, junit.xml, github-step-summary.md"
+  ).option("--json", "print deterministic JSON result").action((opts) => {
+    runCommand(() => gateCommand(opts));
+  });
   return program;
 }
 function isPrimaryModule() {