agent-inspect 4.3.0 → 4.4.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 4.4.0
+
+### Minor Changes
+
+- 42635d8: Add observed outcomes (`observeOutcome`) with OUTCOME events, report/check/search integration, and redaction-bounded evidence fields.
+
 ## 4.3.0
 
 ### Minor Changes

package/docs/API.md

@@ -84,6 +84,7 @@ import {
   - **`step.llm(model, fn)`**: convenience wrapper (`type: "llm"`, `metadata.model`).
   - **`step.tool(toolName, fn)`**: convenience wrapper (`type: "tool"`, `metadata.toolName`).
 - **`observe(agent, options?)`**: proxy wrapper that traces top-level `run` / `execute` / `invoke` methods via `inspectRun`.
+- **`observeOutcome(name, options)`** (v4.4.0+): records an observed outcome (`outcome_observed`) inside an active `inspectRun` context. Requires `expectation` and `status` (`passed` | `failed` | `unknown` | `skipped`); optional `method`, `actual`, and `evidence`. Outside a run → warn and no-op (never throws). `actual` / `evidence` are bounded and redacted before disk.
 - **`getCurrentCorrelationMetadata()`**: returns active run correlation fields (`correlationId`, `requestId`, `decisionId`, `groupId`) inside `inspectRun` / `maybeInspectRun`; `undefined` outside a traced run or when none were set.
 - **`RedactionProfile`**: `"local" | "share" | "strict"` — see `redactionProfile` on `InspectRunOptions` and `ExportOptions`.
 - **`resolveRedactionProfile(profile?)`**: advanced helper available from `agent-inspect/advanced`; resolves profile extra keys and metadata caps for integrations.
@@ -109,6 +110,7 @@ These APIs support local workflows like listing traces, extracting metadata/summ
   - `RunCompletedEvent` (`event: "run_completed"`)
   - `StepStartedEvent` (`event: "step_started"`)
   - `StepCompletedEvent` (`event: "step_completed"`)
+  - `OutcomeObservedEvent` (`event: "outcome_observed"`, v4.4.0+)
 - Related types: `StepType`, `StepStatus`, `RunStatus`, `ErrorInfo`, `StepMetadata`, `TokenMetadata`
 
 ### Log-derived normalized model (stable as a model, not necessarily the parsing APIs)
@@ -460,6 +462,7 @@ Public methods:
 - **`step(name, fn, options?)`**: writes nested step lifecycle events when called inside the same inspector's run context; outside a context it passes through.
 - **`tool(name, fn, options?)`** / **`llm(name, fn, options?)`**: convenience wrappers that set `type` and metadata.
 - **`observe(name, fn, options?)`**: returns an async wrapper that records the function call as an inspector step.
+- **`observeOutcome(name, options)`** (v4.4.0+): writes `outcome_observed` / `OUTCOME` events through the configured writer when inside an inspector run context.
 - **`getDiagnostics()`**: returns instrumentation error counts and writer stats without requiring direct runtime access.
 - **`flush()`** / **`close()`**: delegate to the configured writer through the runtime.
 

package/docs/CLI.md

@@ -317,6 +317,7 @@ Options:
 - `--correlate-group`: when using `--session`, also match synthetic `group:` session keys
 - `--guardrails <rule>`: optional deterministic guardrail rules (`banned-phrase`, `pii-leak`, `prompt-injection`, …); repeatable
 - `--circuit <rule>`: optional circuit analyzers (`same-tool-repetition`, `max-retries`, …); repeatable
+- `--fail-on-observation <status>`: add `outcome.status` rule; repeatable (`failed`, `passed`, `unknown`, `skipped`; default when flag present without value: `failed`)
 
 By default, `check` runs `run.status`. Additional built-in rules can be selected with `--rule` or config when their options are available.
 
@@ -667,6 +668,7 @@ Options:
 - `--limit <number>` — default 50
 - `--session <id>` — limit to runs in one workflow session
 - `--correlate-group` — when using `--session`, also match synthetic `group:` keys
+- `--observation <status>` — filter runs with observed outcomes matching status (`passed`, `failed`, `unknown`, `skipped`)
 - `--json`
 
 Examples:
@@ -779,6 +781,7 @@ Options:
 - `-o, --output <path>` — write to file
 - `--json` — JSON wrapper (includes `content` when writing to stdout)
 - `--include-attributes` — bounded attributes in the execution tree section
+- `--section <name>` — include report sections; repeatable (`what`, `timeline`, `tree`, `observations`; default: all except when narrowed)
 - `--no-errors` — omit error details from the execution tree section
 - `--no-correlation` — omit correlation ids from what section
 - `--redaction-profile <local|share|strict>` — key-based redaction profile applied to the complete report (default `local`); review output before sharing

package/docs/SCHEMA.md

@@ -36,6 +36,7 @@ The stable event names are:
 - `run_completed`
 - `step_started`
 - `step_completed`
+- `outcome_observed` (v4.4.0+)
 
 There is **no** `step_failed` event. Failures are represented by `step_completed` with `status: "error"`.
 
@@ -111,6 +112,31 @@ Every trace line contains:
 }
 ```
 
+### 3.5 `outcome_observed` (v4.4.0+)
+
+Records whether an external side effect matched expectations. Distinct from step/tool success.
+
+```ts
+{
+  schemaVersion: "0.1",
+  event: "outcome_observed",
+  timestamp: number,
+  runId: string,
+  outcomeId: string,
+  parentId?: string,
+  name: string,
+  expectation: string,
+  status: "passed" | "failed" | "unknown" | "skipped",
+  method?: "dom" | "accessibility" | "snapshot" | "network" | "storage"
+    | "filesystem" | "database" | "queue" | "custom",
+  actual?: unknown,
+  evidence?: Record<string, unknown>,
+  observedAt: number
+}
+```
+
+`actual` and `evidence` are bounded and redacted before disk like step metadata. See [OBSERVED-OUTCOMES-V4.4.md](./proposals/OBSERVED-OUTCOMES-V4.4.md).
+
 ## 4. Error representation
 
 - Errors are structured as `{ message, stack? }`.
@@ -173,7 +199,7 @@ interface InspectEvent {
   runId: string;
   parentId?: string;
   name: string;
-  kind: "RUN" | "AGENT" | "LLM" | "TOOL" | "CHAIN" | "RETRIEVER" | "DECISION" | "RESULT" | "ERROR" | "LOGIC" | "LOG";
+  kind: "RUN" | "AGENT" | "LLM" | "TOOL" | "CHAIN" | "RETRIEVER" | "DECISION" | "RESULT" | "ERROR" | "LOGIC" | "LOG" | "OUTCOME";
   timestamp: number;
   status?: "running" | "ok" | "error";
   durationMs?: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-inspect",
-  "version": "4.3.0",
+  "version": "4.4.0",
   "license": "MIT",
   "type": "module",
   "description": "Trace, check, and safely share TypeScript AI-agent runs locally — no account, no upload, metadata-only by default",

package/packages/cli/dist/{chunk-BS5LSKZ3.mjs → chunk-ORHDQUT6.mjs}

@@ -52,7 +52,8 @@ var INSPECT_KINDS = [
   "RESULT",
   "ERROR",
   "LOGIC",
-  "LOG"
+  "LOG",
+  "OUTCOME"
 ];
 var ATTRIBUTION_CONFIDENCES = [
   "explicit",
@@ -525,6 +526,32 @@ function fromLegacyStepCompleted(event) {
   if (error) out.error = error;
   return out;
 }
+function fromLegacyOutcomeObserved(event) {
+  const attrs = event.attributes ?? {};
+  const observedAtRaw = attrs.observedAt;
+  const observedAt = typeof observedAtRaw === "string" ? Date.parse(observedAtRaw) : typeof observedAtRaw === "number" && Number.isFinite(observedAtRaw) ? observedAtRaw : resolveTimes(event).timestamp;
+  const status = attrs.outcomeStatus;
+  const out = {
+    schemaVersion: "0.1",
+    event: "outcome_observed",
+    timestamp: observedAt,
+    runId: event.runId,
+    outcomeId: typeof attrs.outcomeId === "string" ? attrs.outcomeId : event.eventId,
+    name: event.name,
+    expectation: typeof attrs.expectation === "string" ? attrs.expectation : event.name,
+    status: status === "passed" || status === "failed" || status === "unknown" || status === "skipped" ? status : "unknown",
+    observedAt
+  };
+  if (event.parentId !== void 0) out.parentId = event.parentId;
+  if (typeof attrs.method === "string") out.method = attrs.method;
+  if (attrs.actual !== void 0) out.actual = attrs.actual;
+  if (event.outputSummary !== void 0) out.actual = event.outputSummary;
+  if (attrs.evidence !== void 0) out.evidence = attrs.evidence;
+  return out;
+}
+function fromNativeOutcome(event) {
+  return [fromLegacyOutcomeObserved(event)];
+}
 function fromNativeRun(event) {
   const { timestamp, startTime, endTime } = resolveTimes(event);
   const runStatus = mapPersistedStatusToRunStatus(event.status);
@@ -612,9 +639,13 @@ function persistedInspectEventToTraceEvents(event) {
   if (legacyEvent === "run_completed") return [fromLegacyRunCompleted(event)];
   if (legacyEvent === "step_started") return [fromLegacyStepStarted(event)];
   if (legacyEvent === "step_completed") return [fromLegacyStepCompleted(event)];
+  if (legacyEvent === "outcome_observed") return [fromLegacyOutcomeObserved(event)];
   if (event.kind === "RUN") {
     return fromNativeRun(event);
   }
+  if (event.kind === "OUTCOME") {
+    return fromNativeOutcome(event);
+  }
   return fromNativeStep(event);
 }
 function persistedInspectEventsToTraceEvents(events, options) {
@@ -663,6 +694,9 @@ function isTraceEvent(value) {
     case "step_completed": {
       return typeof value.runId === "string" && typeof value.stepId === "string" && (value.status === "success" || value.status === "error") && typeof value.endTime === "number" && typeof value.durationMs === "number";
     }
+    case "outcome_observed": {
+      return typeof value.runId === "string" && typeof value.outcomeId === "string" && typeof value.name === "string" && typeof value.expectation === "string" && (value.status === "passed" || value.status === "failed" || value.status === "unknown" || value.status === "skipped") && typeof value.observedAt === "number";
+    }
     default:
       return false;
   }
@@ -973,6 +1007,9 @@ function validateEvent(event) {
     case "step_completed": {
       return nonEmptyString(event.runId) && nonEmptyString(event.stepId) && (event.status === "success" || event.status === "error") && finiteNumber(event.endTime) && finiteNumber(event.durationMs) && optionalErrorInfo(event.error);
     }
+    case "outcome_observed": {
+      return nonEmptyString(event.runId) && nonEmptyString(event.outcomeId) && nonEmptyString(event.name) && nonEmptyString(event.expectation) && (event.status === "passed" || event.status === "failed" || event.status === "unknown" || event.status === "skipped") && finiteNumber(event.observedAt);
+    }
     default:
       return false;
   }
@@ -2512,6 +2549,142 @@ function renderTraceStats(stats) {
   return lines.join("\n");
 }
 
+// packages/core/src/outcomes/types.ts
+var OBSERVED_OUTCOME_STATUSES = [
+  "passed",
+  "failed",
+  "unknown",
+  "skipped"
+];
+var OUTCOME_ATTRIBUTE_STATUS_KEY = "outcomeStatus";
+var OUTCOME_ATTRIBUTE_EXPECTATION_KEY = "expectation";
+var OUTCOME_ATTRIBUTE_METHOD_KEY = "method";
+var OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY = "observedAt";
+var OUTCOME_LEGACY_EVENT = "outcome_observed";
+
+// packages/core/src/outcomes/validate.ts
+function parseObservedOutcomeStatus(value) {
+  const trimmed = value.trim().toLowerCase();
+  if (OBSERVED_OUTCOME_STATUSES.includes(trimmed)) {
+    return trimmed;
+  }
+  throw new Error(
+    `Unsupported observation status "${value}". Use passed, failed, unknown, or skipped.`
+  );
+}
+
+// packages/core/src/outcomes/extract.ts
+function isOutcomeStatus(value) {
+  return value === "passed" || value === "failed" || value === "unknown" || value === "skipped";
+}
+function parseMethod(value) {
+  if (typeof value !== "string" || value.trim() === "") return void 0;
+  return value;
+}
+function fromOutcomeObservedEvent(event) {
+  return {
+    outcomeId: event.outcomeId,
+    runId: event.runId,
+    ...event.parentId !== void 0 ? { parentId: event.parentId } : {},
+    name: event.name,
+    expectation: event.expectation,
+    status: event.status,
+    ...event.method !== void 0 ? { method: event.method } : {},
+    ...event.actual !== void 0 ? { actual: event.actual } : {},
+    ...event.evidence !== void 0 ? { evidence: event.evidence } : {},
+    observedAt: event.observedAt
+  };
+}
+function fromPersistedOutcome(event) {
+  if (event.kind !== "OUTCOME") return void 0;
+  const attrs = event.attributes ?? {};
+  const statusRaw = attrs[OUTCOME_ATTRIBUTE_STATUS_KEY];
+  if (!isOutcomeStatus(statusRaw)) return void 0;
+  const expectation = typeof attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] === "string" ? attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] : event.name;
+  const observedAtRaw = attrs[OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY];
+  const observedAt = typeof observedAtRaw === "string" ? Date.parse(observedAtRaw) : typeof observedAtRaw === "number" && Number.isFinite(observedAtRaw) ? observedAtRaw : Date.parse(event.timestamp);
+  return {
+    outcomeId: event.eventId,
+    runId: event.runId,
+    ...event.parentId !== void 0 ? { parentId: event.parentId } : {},
+    name: event.name,
+    expectation,
+    status: statusRaw,
+    ...parseMethod(attrs[OUTCOME_ATTRIBUTE_METHOD_KEY]) !== void 0 ? { method: parseMethod(attrs[OUTCOME_ATTRIBUTE_METHOD_KEY]) } : {},
+    ...event.outputSummary !== void 0 ? { actual: event.outputSummary } : {},
+    ...attrs.evidence !== void 0 ? { evidence: attrs.evidence } : {},
+    observedAt: Number.isFinite(observedAt) ? observedAt : Date.parse(event.timestamp)
+  };
+}
+function extractOutcomesFromTraceEvents(events) {
+  const out = [];
+  for (const event of events) {
+    if (event.event === OUTCOME_LEGACY_EVENT) {
+      out.push(fromOutcomeObservedEvent(event));
+    }
+  }
+  return out.sort((a, b) => a.observedAt - b.observedAt || a.name.localeCompare(b.name));
+}
+function extractOutcomesFromPersistedEvents(events) {
+  const out = [];
+  for (const event of events) {
+    const outcome = fromPersistedOutcome(event);
+    if (outcome) out.push(outcome);
+  }
+  return out.sort((a, b) => a.observedAt - b.observedAt || a.name.localeCompare(b.name));
+}
+function summarizeObservedOutcomes(outcomes) {
+  const summary = {
+    total: outcomes.length,
+    passed: 0,
+    failed: 0,
+    unknown: 0,
+    skipped: 0,
+    outcomes: [...outcomes]
+  };
+  for (const outcome of outcomes) {
+    summary[outcome.status] += 1;
+  }
+  return summary;
+}
+function outcomesMatchingStatus(outcomes, statuses) {
+  const set = new Set(statuses);
+  return outcomes.filter((outcome) => set.has(outcome.status));
+}
+function parseObservationFilter(value) {
+  if (value === void 0 || value.trim() === "") return void 0;
+  return parseObservedOutcomeStatus(value);
+}
+
+// packages/core/src/outcomes/render.ts
+function renderObservedOutcomesMarkdown(summary) {
+  if (summary.total === 0) {
+    return "No observed outcomes recorded for this run.";
+  }
+  const lines = [
+    `Total: ${summary.total} (passed ${summary.passed}, failed ${summary.failed}, unknown ${summary.unknown}, skipped ${summary.skipped})`,
+    "",
+    "| Name | Status | Expectation | Method |",
+    "| --- | --- | --- | --- |"
+  ];
+  for (const outcome of summary.outcomes) {
+    lines.push(
+      `| ${outcome.name} | ${outcome.status} | ${outcome.expectation.replace(/\|/g, "\\|")} | ${outcome.method ?? "-"} |`
+    );
+  }
+  return lines.join("\n");
+}
+function renderObservedOutcomesHtml(summary) {
+  if (summary.total === 0) {
+    return "<p>No observed outcomes recorded for this run.</p>";
+  }
+  const rows = summary.outcomes.map((outcome) => {
+    const esc = (v) => v.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
+    return `<tr><td>${esc(outcome.name)}</td><td>${esc(outcome.status)}</td><td>${esc(outcome.expectation)}</td><td>${esc(outcome.method ?? "-")}</td></tr>`;
+  }).join("");
+  return `<p>Total: ${summary.total} (passed ${summary.passed}, failed ${summary.failed}, unknown ${summary.unknown}, skipped ${summary.skipped})</p><table><thead><tr><th>Name</th><th>Status</th><th>Expectation</th><th>Method</th></tr></thead><tbody>${rows}</tbody></table>`;
+}
+
 // packages/core/src/search.ts
 function parseDurationFilter(expr) {
   const raw = expr.trim();
@@ -2558,8 +2731,9 @@ async function searchTraces(metas, options) {
   }
   const limit = options.limit ?? 50;
   const sessionId = options.session?.trim();
+  const observationStatus = parseObservationFilter(options.observation);
   const hasContentFilter = Boolean(
-    options.status || stepTypeFilter || nameQuery || toolQuery || durationFilter
+    options.status || stepTypeFilter || nameQuery || toolQuery || durationFilter || observationStatus
   );
   const results = [];
   const sessionLabel = sessionId && sessionId !== "" ? sessionId : void 0;
@@ -2604,6 +2778,22 @@ async function searchTraces(metas, options) {
       statusFilter: options.status
     });
     results.push(...stepMatches);
+    if (observationStatus) {
+      const outcomes = extractOutcomesFromTraceEvents(events);
+      const matched = outcomes.filter((outcome) => outcome.status === observationStatus);
+      for (const outcome of matched) {
+        results.push({
+          runId: m.runId,
+          runName: m.name,
+          runStatus: m.status,
+          stepName: outcome.name,
+          timestamp: outcome.observedAt,
+          matchReason: `observation status=${outcome.status}`,
+          matchedFields: ["outcome.status", "outcome.name"],
+          filePath: m.filePath
+        });
+      }
+    }
   }
   results.sort((a, b) => {
     const ta = a.timestamp ?? 0;
@@ -2780,9 +2970,10 @@ function buildActivitySummary(index, options = {}) {
     entries
   };
 }
-function renderActivitySummaryHuman(summary) {
+function renderActivitySummaryHuman(summary, options) {
   const lines = [];
-  const todayStart = /* @__PURE__ */ new Date();
+  const nowMs = options?.nowMs ?? Date.now();
+  const todayStart = new Date(nowMs);
   todayStart.setHours(0, 0, 0, 0);
   const todayMs = todayStart.getTime();
   const today = summary.entries.filter(
@@ -3666,6 +3857,6 @@ function defaultBundleOutputPath(runIds) {
   return path.resolve(`agent-inspect-bundle-${label}-${stamp}`);
 }
 
-export { Redactor, TraceDirectory, __commonJS, __require, __toESM, aggregateBundleSafeStatus, aggregateSessionCheckResults, applyProfileMetadataCaps, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, defaultBundleOutputPath, enrichSessionRunRecord, extractCorrelationMetadata, extractMetadata, filterMetasBySessionScope, filterTraces, formatDuration2 as formatDuration, formatTimestamp, getIndent, getTraceFilePath, isAgentInspectTrace, isPersistedInspectEvent, loadSessionRunRecords, loadTraceMetadataList, nanoid, normalizeBundleOutputPath, parseDuration, parseDurationFilter, parseTraceJsonl, persistedInspectEventsToTraceEvents, renderActivitySummaryHuman, renderErrorLine, renderRunWhat, renderStepLine, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveTraceDir, searchTraces, source_default, truncateName, truncateStringForProfile, validateEvent };
-//# sourceMappingURL=chunk-BS5LSKZ3.mjs.map
-//# sourceMappingURL=chunk-BS5LSKZ3.mjs.map
+export { Redactor, TraceDirectory, __commonJS, __require, __toESM, aggregateBundleSafeStatus, aggregateSessionCheckResults, applyProfileMetadataCaps, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, defaultBundleOutputPath, enrichSessionRunRecord, extractCorrelationMetadata, extractMetadata, extractOutcomesFromPersistedEvents, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, formatDuration2 as formatDuration, formatTimestamp, getIndent, getTraceFilePath, isAgentInspectTrace, isPersistedInspectEvent, loadSessionRunRecords, loadTraceMetadataList, nanoid, normalizeBundleOutputPath, outcomesMatchingStatus, parseDuration, parseDurationFilter, parseTraceJsonl, persistedInspectEventsToTraceEvents, renderActivitySummaryHuman, renderErrorLine, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunWhat, renderStepLine, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveTraceDir, searchTraces, source_default, summarizeObservedOutcomes, truncateName, truncateStringForProfile, validateEvent };
+//# sourceMappingURL=chunk-ORHDQUT6.mjs.map
+//# sourceMappingURL=chunk-ORHDQUT6.mjs.map