agent-inspect 4.3.0 → 4.4.0

package/packages/cli/dist/index.mjs

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { resolveTraceDir, parseDuration, buildSessionIndex, enrichSessionRunRecord, TraceDirectory, loadTraceMetadataList, loadSessionRunRecords, extractMetadata, filterTraces, truncateName, getTraceFilePath, buildRunSummary, formatDuration, formatTimestamp, isAgentInspectTrace, filterMetasBySessionScope, aggregateSessionCheckResults, resolveBundleRunIds, aggregateBundleSafeStatus, bundleFailsOnSafety, buildPlaceholderArtifact, buildBundleMetadata, buildBundleSummaryMarkdown, buildRunTimeline, renderTimeline, buildTraceStats, renderTraceStats, parseDurationFilter, searchTraces, buildActivitySummary, renderActivitySummaryHuman, buildRunWhatSummary, renderRunWhat, buildLocalExplanation, persistedInspectEventsToTraceEvents, renderStepLine, renderErrorLine, getIndent, Redactor, validateEvent, isPersistedInspectEvent, normalizeBundleOutputPath, defaultBundleOutputPath, nanoid, resolveRedactionProfile, applyProfileMetadataCaps, extractCorrelationMetadata, truncateStringForProfile, parseTraceJsonl } from './chunk-BS5LSKZ3.mjs';
+import { resolveTraceDir, parseDuration, buildSessionIndex, enrichSessionRunRecord, TraceDirectory, loadTraceMetadataList, loadSessionRunRecords, extractMetadata, filterTraces, truncateName, getTraceFilePath, buildRunSummary, formatDuration, formatTimestamp, isAgentInspectTrace, filterMetasBySessionScope, aggregateSessionCheckResults, resolveBundleRunIds, aggregateBundleSafeStatus, bundleFailsOnSafety, buildPlaceholderArtifact, buildBundleMetadata, buildBundleSummaryMarkdown, buildRunTimeline, renderTimeline, buildTraceStats, renderTraceStats, parseDurationFilter, searchTraces, buildActivitySummary, renderActivitySummaryHuman, buildRunWhatSummary, renderRunWhat, buildLocalExplanation, persistedInspectEventsToTraceEvents, renderStepLine, renderErrorLine, getIndent, Redactor, validateEvent, isPersistedInspectEvent, normalizeBundleOutputPath, defaultBundleOutputPath, summarizeObservedOutcomes, extractOutcomesFromTraceEvents, renderObservedOutcomesMarkdown, renderObservedOutcomesHtml, nanoid, resolveRedactionProfile, applyProfileMetadataCaps, extractCorrelationMetadata, extractOutcomesFromPersistedEvents, outcomesMatchingStatus, truncateStringForProfile, parseTraceJsonl } from './chunk-ORHDQUT6.mjs';
 import { realpathSync, constants as constants$1 } from 'fs';
 import path13 from 'path';
 import { fileURLToPath, pathToFileURL } from 'url';
@@ -11,7 +11,7 @@ import { createServer } from 'http';
 import { createRequire } from 'module';
 
 // package.json
-var version = "4.3.0";
+var version = "4.4.0";
 
 // packages/cli/src/trace-dir-scale.ts
 var TRACE_COUNT_WARN = 1e3;
@@ -353,6 +353,8 @@ function nodeIdForEvent(event) {
     case "step_started":
     case "step_completed":
       return event.stepId;
+    case "outcome_observed":
+      return event.outcomeId;
     default:
       return "unknown";
   }
@@ -536,6 +538,39 @@ function traceEventToPersistedInspectEvent(event, options) {
         error
       };
     }
+    case "outcome_observed": {
+      const tsObserved = toIsoTimestamp(event.observedAt);
+      const attributes = compactAttributes({
+        legacyEvent: "outcome_observed",
+        outcomeId: event.outcomeId,
+        outcomeStatus: event.status,
+        expectation: event.expectation,
+        method: event.method,
+        actual: event.actual,
+        evidence: event.evidence,
+        observedAt: tsObserved.iso,
+        invalidTimestamp: tsMain.invalidTimestamp || tsObserved.invalidTimestamp ? true : void 0
+      });
+      const out = {
+        schemaVersion: "0.2",
+        eventId,
+        runId: event.runId,
+        kind: "OUTCOME",
+        name: event.name,
+        status: event.status === "failed" ? "error" : "ok",
+        timestamp: tsMain.iso,
+        confidence: "explicit",
+        source,
+        attributes
+      };
+      if (event.parentId !== void 0) {
+        out.parentId = event.parentId;
+      }
+      if (event.actual !== void 0) {
+        out.outputSummary = event.actual;
+      }
+      return out;
+    }
     default: {
       const _exhaustive = event;
       throw new Error(`Unsupported trace event: ${_exhaustive.event}`);
@@ -1265,7 +1300,7 @@ function sanitizeOpenInferenceAttributes(attributes, pathPrefix) {
 function mapOpenInferenceKind(span, attributes, pathPrefix) {
   const warnings = [];
   const agentInspectKind = attributes["agent_inspect.kind"];
-  if (agentInspectKind === "RUN" || agentInspectKind === "AGENT" || agentInspectKind === "LLM" || agentInspectKind === "TOOL" || agentInspectKind === "CHAIN" || agentInspectKind === "RETRIEVER" || agentInspectKind === "DECISION" || agentInspectKind === "RESULT" || agentInspectKind === "ERROR" || agentInspectKind === "LOGIC" || agentInspectKind === "LOG") {
+  if (agentInspectKind === "RUN" || agentInspectKind === "AGENT" || agentInspectKind === "LLM" || agentInspectKind === "TOOL" || agentInspectKind === "CHAIN" || agentInspectKind === "RETRIEVER" || agentInspectKind === "DECISION" || agentInspectKind === "RESULT" || agentInspectKind === "ERROR" || agentInspectKind === "LOGIC" || agentInspectKind === "LOG" || agentInspectKind === "OUTCOME") {
     return { kind: agentInspectKind, warnings };
   }
   const rawKind = readStringField(span, ["kind", "span_kind", "spanKind"]) ?? (typeof attributes["openinference.span.kind"] === "string" ? attributes["openinference.span.kind"] : void 0);
@@ -1786,7 +1821,7 @@ function mapOtlpStatus(status) {
 function readOtlpKind(attributes, pathPrefix) {
   const warnings = [];
   const agentInspectKind = attributes["agent_inspect.kind"];
-  if (agentInspectKind === "RUN" || agentInspectKind === "AGENT" || agentInspectKind === "LLM" || agentInspectKind === "TOOL" || agentInspectKind === "CHAIN" || agentInspectKind === "RETRIEVER" || agentInspectKind === "DECISION" || agentInspectKind === "RESULT" || agentInspectKind === "ERROR" || agentInspectKind === "LOGIC" || agentInspectKind === "LOG") {
+  if (agentInspectKind === "RUN" || agentInspectKind === "AGENT" || agentInspectKind === "LLM" || agentInspectKind === "TOOL" || agentInspectKind === "CHAIN" || agentInspectKind === "RETRIEVER" || agentInspectKind === "DECISION" || agentInspectKind === "RESULT" || agentInspectKind === "ERROR" || agentInspectKind === "LOGIC" || agentInspectKind === "LOG" || agentInspectKind === "OUTCOME") {
     return { kind: agentInspectKind, warnings };
   }
   const operation = attributes["gen_ai.operation.name"];
@@ -3914,6 +3949,40 @@ function redactTraceEventsForReport(events, options) {
             )
           } : {}
         };
+      case "outcome_observed": {
+        const actualAttrs = event.actual !== void 0 ? redactEventAttributes(
+          { value: event.actual },
+          redactor,
+          maxMetadataValueLength,
+          maxPreviewLength
+        ) : void 0;
+        const redactedActual = actualAttrs !== void 0 && "value" in actualAttrs ? actualAttrs.value : void 0;
+        const redactedEvidence = event.evidence !== void 0 ? redactEventAttributes(
+          isRecord6(event.evidence) ? event.evidence : { value: event.evidence },
+          redactor,
+          maxMetadataValueLength,
+          maxPreviewLength
+        ) : void 0;
+        return {
+          ...event,
+          name: truncateStringForProfile(
+            event.name,
+            "name",
+            maxMetadataValueLength,
+            maxPreviewLength
+          ),
+          expectation: truncateStringForProfile(
+            event.expectation,
+            "expectation",
+            maxMetadataValueLength,
+            maxPreviewLength
+          ),
+          ...redactedActual !== void 0 ? { actual: redactedActual } : {},
+          ...redactedEvidence !== void 0 ? { evidence: redactedEvidence } : {}
+        };
+      }
+      default:
+        return event;
     }
   });
 }
@@ -4052,7 +4121,8 @@ function zeroKinds() {
     RESULT: 0,
     ERROR: 0,
     LOGIC: 0,
-    LOG: 0
+    LOG: 0,
+    OUTCOME: 0
   };
 }
 
@@ -4855,9 +4925,34 @@ footer{margin-top:2rem;font-size:0.85rem;color:#555}
 `.trim();
 function buildRunReport(events, options) {
   const profile = options.redactionProfile ?? "local";
+  const section = options.section ?? "all";
   const safeEvents = redactTraceEventsForReport(events, {
     redactionProfile: profile
   });
+  const outcomes = summarizeObservedOutcomes(
+    extractOutcomesFromTraceEvents(safeEvents)
+  );
+  const observationsMarkdown = renderObservedOutcomesMarkdown(outcomes);
+  const observationsHtml = renderObservedOutcomesHtml(outcomes);
+  if (section === "observations") {
+    if (options.format === "markdown") {
+      return {
+        format: "markdown",
+        content: `## Observed outcomes
+
+${observationsMarkdown}
+`,
+        contentType: "text/markdown",
+        fileExtension: ".md"
+      };
+    }
+    return {
+      format: "html",
+      content: `<!doctype html><html lang="en"><head><meta charset="utf-8"/><title>Observed outcomes</title><style>${REPORT_HTML_CSS}</style></head><body><h1>Observed outcomes</h1>${observationsHtml}</body></html>`,
+      contentType: "text/html",
+      fileExtension: ".html"
+    };
+  }
   const whatSummary = buildRunWhatSummary(safeEvents);
   const whatText = renderRunWhat(whatSummary, {
     correlation: options.correlation !== false
@@ -4891,6 +4986,10 @@ function buildRunReport(events, options) {
       "```text",
       timelineText,
       "```",
+      "",
+      "## Observed outcomes",
+      "",
+      observationsMarkdown,
       ""
     ];
     if (tail2) {
@@ -4934,6 +5033,7 @@ function buildRunReport(events, options) {
 <p class="note">Generated locally by AgentInspect. Review for sensitive data before sharing.</p>
 <section class="what"><h2>What happened</h2><pre>${escapeHtml(whatText)}</pre></section>
 <section class="timeline"><h2>Timeline</h2><pre>${escapeHtml(timelineText)}</pre></section>
+<section class="observations"><h2>Observed outcomes</h2>${observationsHtml}</section>
 ${treeSection || '<section class="tree"><h2>Execution tree</h2><p>No steps recorded.</p></section>'}
 ${errorsSection}
 ${attrsSection}
@@ -5785,7 +5885,8 @@ async function searchCommand(options = {}) {
       duration: options.duration,
       limit: parseLimit2(options.limit),
       ...sessionId ? { session: sessionId } : {},
-      ...options.correlateGroup ? { correlateGroup: true } : {}
+      ...options.correlateGroup ? { correlateGroup: true } : {},
+      ...options.observation ? { observation: options.observation } : {}
     });
     if (options.json) {
       console.log(JSON.stringify(results, null, 2));
@@ -5857,7 +5958,7 @@ async function newestTraceMtimeMs(traceDir) {
 }
 async function loadSessionRuns(traceDir) {
   try {
-    const mod = await import('./src-YFMPWEIS.mjs');
+    const mod = await import('./src-FJOHPC4X.mjs');
     const dbPath = mod.resolveIndexDbPath(traceDir);
     const status = mod.indexStatus(dbPath);
     if (!status.healthy) {
@@ -6302,7 +6403,8 @@ async function reportCommand(runId, options = {}) {
     includeAttributes: options.includeAttributes === true,
     includeErrors: options.noErrors !== true,
     redactionProfile,
-    correlation: !options.noCorrelation
+    correlation: !options.noCorrelation,
+    ...options.section ? { section: options.section } : {}
   });
   const outPath = options.output !== void 0 && options.output.trim() !== "" ? path13.resolve(options.output.trim()) : void 0;
   if (outPath !== void 0) {
@@ -8606,6 +8708,40 @@ function createBaselineRegressionRule(options) {
     }
   };
 }
+function createObservedOutcomeRule(options = {}) {
+  const failOn = options.failOn ?? ["failed"];
+  return {
+    id: "outcome.status",
+    category: "run",
+    defaultSeverity: "error",
+    evaluate(context) {
+      const outcomes = extractOutcomesFromPersistedEvents(context.events);
+      const matching = outcomesMatchingStatus(outcomes, failOn);
+      if (matching.length === 0) return [];
+      return [
+        failFinding(
+          "outcome.status",
+          `Observed outcome count ${matching.length} matched [${failOn.join(", ")}].`,
+          matching.map((outcome) => ({
+            runId: outcome.runId,
+            eventId: outcome.outcomeId,
+            ...outcome.parentId !== void 0 ? { parentId: outcome.parentId } : {},
+            kind: "OUTCOME",
+            name: outcome.name,
+            status: outcome.status,
+            path: `outcome.${outcome.name}`
+          })),
+          { failOn },
+          matching.map((outcome) => ({
+            name: outcome.name,
+            status: outcome.status,
+            expectation: outcome.expectation
+          }))
+        )
+      ];
+    }
+  };
+}
 function runTraceChecks(input3, options = {}) {
   const selected = resolveSelectedRun(input3, options.runId);
   if (selected.diagnostics.length > 0) {
@@ -9377,6 +9513,22 @@ function buildRules(config, options) {
   if (options.detectStalls) {
     rules.push(createStallDetectionRule({ requireEndedAt: true }));
   }
+  if (options.failOnObservation !== void 0 && options.failOnObservation.trim() !== "") {
+    try {
+      const statuses = options.failOnObservation.split(",").map((value) => value.trim()).filter((value) => value !== "").map((value) => {
+        if (value === "passed" || value === "failed" || value === "unknown" || value === "skipped") {
+          return value;
+        }
+        throw new Error(`unsupported status "${value}"`);
+      });
+      rules.push(createObservedOutcomeRule({ failOn: statuses }));
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      diagnostics.push(
+        diagnostic2("AI_CHECK_INVALID_ARGUMENTS", `--fail-on-observation: ${message}`)
+      );
+    }
+  }
   if (run.maxDepth !== void 0) {
     rules.push(createRunDepthRule({ maxDepth: run.maxDepth }));
   }
@@ -13100,7 +13252,7 @@ function isModuleNotFound3(e) {
 }
 async function loadIndexSqlite() {
   try {
-    return await import('./src-YFMPWEIS.mjs');
+    return await import('./src-FJOHPC4X.mjs');
   } catch (e) {
     if (isModuleNotFound3(e)) {
       console.error(
@@ -13505,7 +13657,10 @@ function createCliProgram() {
   ]).option("--max-total-tokens <number>", "add llm.usage with a max total-token budget").option(
     "--max-step-duration <duration>",
     "add run.maxStepDuration (e.g. 30s, 5m)"
-  ).option("--require-completed", "add run.requireCompleted").option("--detect-stalls", "add run.stall for running or incomplete events").option("--session <id>", "check all runs in a workflow session (requires --dir)").option("--group <id>", "check all runs sharing a groupId (requires --dir)").option(
+  ).option("--require-completed", "add run.requireCompleted").option("--detect-stalls", "add run.stall for running or incomplete events").option(
+    "--fail-on-observation <status>",
+    "fail when observed outcomes match status (passed, failed, unknown, skipped; comma-separated)"
+  ).option("--session <id>", "check all runs in a workflow session (requires --dir)").option("--group <id>", "check all runs sharing a groupId (requires --dir)").option(
     "--correlate-group",
     "when using --session, also match synthetic group: session keys"
   ).option(
@@ -13626,6 +13781,9 @@ function createCliProgram() {
       "unknown"
     ])
   ).option("--kind <kind>", "filter by step kind/type (llm, tool, logic, \u2026)").option("--type <type>", "alias for --kind on manual trace step type").option("--name <query>", "substring match on run or step name").option("--tool <query>", "substring match on tool step name or metadata.toolName").option(
+    "--observation <status>",
+    "filter by observed outcome status (passed, failed, unknown, skipped)"
+  ).option(
     "--duration <expr>",
     "duration filter on run or step (e.g. >5s, >=500ms)"
   ).option("--limit <number>", "max results (default 50)").option("--session <id>", "limit search to one workflow session").option(
@@ -13672,6 +13830,14 @@ function createCliProgram() {
       "html"
     ])
   ).option("-o, --output <path>", "write report to file (creates parent dirs)").option("--json", "emit JSON wrapper (includes content when writing to stdout)").option("--include-attributes", "include bounded step attributes in tree section").option("--no-errors", "omit error details from tree section").option("--no-correlation", "omit correlation metadata from what section").addOption(
+    new Option("--section <section>", "report section (default: all)").choices([
+      "all",
+      "observations",
+      "what",
+      "timeline",
+      "tree"
+    ])
+  ).addOption(
     new Option(
       "--redaction-profile <profile>",
       "redaction profile for entire report: local, share, strict (default: local)"