agent-inspect 4.3.0 → 4.4.0

package/packages/core/dist/checks.cjs

@@ -1,5 +1,69 @@
 'use strict';
 
+require('crypto');
+require('fs/promises');
+var path = require('path');
+var os = require('os');
+
+function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
+
+var path__default = /*#__PURE__*/_interopDefault(path);
+var os__default = /*#__PURE__*/_interopDefault(os);
+
+// packages/core/src/outcomes/types.ts
+var OUTCOME_ATTRIBUTE_STATUS_KEY = "outcomeStatus";
+var OUTCOME_ATTRIBUTE_EXPECTATION_KEY = "expectation";
+var OUTCOME_ATTRIBUTE_METHOD_KEY = "method";
+var OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY = "observedAt";
+var RUNS_DIR_NAME = "runs";
+path__default.default.join(
+  os__default.default.tmpdir(),
+  "agent-inspect",
+  RUNS_DIR_NAME
+);
+
+// packages/core/src/outcomes/extract.ts
+function isOutcomeStatus(value) {
+  return value === "passed" || value === "failed" || value === "unknown" || value === "skipped";
+}
+function parseMethod(value) {
+  if (typeof value !== "string" || value.trim() === "") return void 0;
+  return value;
+}
+function fromPersistedOutcome(event) {
+  if (event.kind !== "OUTCOME") return void 0;
+  const attrs = event.attributes ?? {};
+  const statusRaw = attrs[OUTCOME_ATTRIBUTE_STATUS_KEY];
+  if (!isOutcomeStatus(statusRaw)) return void 0;
+  const expectation = typeof attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] === "string" ? attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] : event.name;
+  const observedAtRaw = attrs[OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY];
+  const observedAt = typeof observedAtRaw === "string" ? Date.parse(observedAtRaw) : typeof observedAtRaw === "number" && Number.isFinite(observedAtRaw) ? observedAtRaw : Date.parse(event.timestamp);
+  return {
+    outcomeId: event.eventId,
+    runId: event.runId,
+    ...event.parentId !== void 0 ? { parentId: event.parentId } : {},
+    name: event.name,
+    expectation,
+    status: statusRaw,
+    ...parseMethod(attrs[OUTCOME_ATTRIBUTE_METHOD_KEY]) !== void 0 ? { method: parseMethod(attrs[OUTCOME_ATTRIBUTE_METHOD_KEY]) } : {},
+    ...event.outputSummary !== void 0 ? { actual: event.outputSummary } : {},
+    ...attrs.evidence !== void 0 ? { evidence: attrs.evidence } : {},
+    observedAt: Number.isFinite(observedAt) ? observedAt : Date.parse(event.timestamp)
+  };
+}
+function extractOutcomesFromPersistedEvents(events) {
+  const out = [];
+  for (const event of events) {
+    const outcome = fromPersistedOutcome(event);
+    if (outcome) out.push(outcome);
+  }
+  return out.sort((a, b) => a.observedAt - b.observedAt || a.name.localeCompare(b.name));
+}
+function outcomesMatchingStatus(outcomes, statuses) {
+  const set = new Set(statuses);
+  return outcomes.filter((outcome) => set.has(outcome.status));
+}
+
 // packages/core/src/checks/index.ts
 var SEVERITY_RANK = {
   error: 0,
@@ -267,7 +331,7 @@ function stripPrefix(name, prefixes) {
   }
   return name;
 }
-function eventEvidence(event, path) {
+function eventEvidence(event, path3) {
   return {
     runId: event.runId,
     eventId: event.eventId,
@@ -277,7 +341,7 @@ function eventEvidence(event, path) {
     kind: event.kind,
     name: event.name,
     status: event.status,
-    ...path ? { path } : {}
+    ...path3 ? { path: path3 } : {}
   };
 }
 function runEvidence(run) {
@@ -351,9 +415,9 @@ function sortedStrings(values) {
 function normalizedKey(value) {
   return value.toLowerCase().replace(/[^a-z0-9_]/g, "");
 }
-function lastPathSegment(path) {
-  const parts = path.split(".");
-  return parts[parts.length - 1] ?? path;
+function lastPathSegment(path3) {
+  const parts = path3.split(".");
+  return parts[parts.length - 1] ?? path3;
 }
 function valueType(value) {
   if (Array.isArray(value)) return "array";
@@ -367,12 +431,12 @@ function serializedByteLength(value) {
     return void 0;
   }
 }
-function pushValueEntries(entries, event, value, path, key, depth = 0) {
-  entries.push({ event, path, key, value });
+function pushValueEntries(entries, event, value, path3, key, depth = 0) {
+  entries.push({ event, path: path3, key, value });
   if (depth >= 8) return;
   if (Array.isArray(value)) {
     for (const [index, item] of value.entries()) {
-      pushValueEntries(entries, event, item, `${path}.${index}`, String(index), depth + 1);
+      pushValueEntries(entries, event, item, `${path3}.${index}`, String(index), depth + 1);
     }
     return;
   }
@@ -382,7 +446,7 @@ function pushValueEntries(entries, event, value, path, key, depth = 0) {
       entries,
       event,
       value[nestedKey],
-      `${path}.${nestedKey}`,
+      `${path3}.${nestedKey}`,
       nestedKey,
       depth + 1
     );
@@ -463,9 +527,9 @@ function eventDurationMs(event) {
 }
 function treeShape(nodes) {
   const lines = [];
-  const visit = (node, path) => {
-    lines.push(`${path}:${node.event.kind}:${node.event.name}:${node.event.status ?? "unknown"}`);
-    node.children.forEach((child, index) => visit(child, `${path}.${index}`));
+  const visit = (node, path3) => {
+    lines.push(`${path3}:${node.event.kind}:${node.event.name}:${node.event.status ?? "unknown"}`);
+    node.children.forEach((child, index) => visit(child, `${path3}.${index}`));
   };
   nodes.forEach((node, index) => visit(node, String(index)));
   return lines;
@@ -514,9 +578,9 @@ function retrievalShape(context) {
 function guardrailShape(context) {
   return guardrailEvents(context).map((event) => signalName(event, ["guardrailName", "guardrail", "guardrailId"], ["guardrail:"])).sort((a, b) => a.localeCompare(b));
 }
-function firstEvidenceForKind(context, kind, path) {
+function firstEvidenceForKind(context, kind, path3) {
   const event = context.events.find((candidate) => candidate.kind === kind);
-  return event ? [eventEvidence(event, path)] : runEvidence(context.selectedRun);
+  return event ? [eventEvidence(event, path3)] : runEvidence(context.selectedRun);
 }
 function baselineDiffFinding(message, evidence, expected, actual) {
   return failFinding("baseline.regression", message, evidence, expected, actual);
@@ -1007,13 +1071,13 @@ function createStructureCycleRule() {
       const seenCycles = /* @__PURE__ */ new Set();
       const findings = [];
       for (const event of [...context.events].sort((a, b) => a.eventId.localeCompare(b.eventId))) {
-        const path = [];
+        const path3 = [];
         const seenAt = /* @__PURE__ */ new Map();
         let current = event;
         while (current) {
           const existing = seenAt.get(current.eventId);
           if (existing !== void 0) {
-            const cycle = path.slice(existing);
+            const cycle = path3.slice(existing);
             const key = cycle.map((item) => item.eventId).sort().join("\0");
             if (!seenCycles.has(key)) {
               seenCycles.add(key);
@@ -1029,8 +1093,8 @@ function createStructureCycleRule() {
             }
             break;
           }
-          seenAt.set(current.eventId, path.length);
-          path.push(current);
+          seenAt.set(current.eventId, path3.length);
+          path3.push(current);
           current = current.parentId ? byId.get(current.parentId) : void 0;
         }
       }
@@ -1566,6 +1630,40 @@ function createBaselineRegressionRule(options) {
     }
   };
 }
+function createObservedOutcomeRule(options = {}) {
+  const failOn = options.failOn ?? ["failed"];
+  return {
+    id: "outcome.status",
+    category: "run",
+    defaultSeverity: "error",
+    evaluate(context) {
+      const outcomes = extractOutcomesFromPersistedEvents(context.events);
+      const matching = outcomesMatchingStatus(outcomes, failOn);
+      if (matching.length === 0) return [];
+      return [
+        failFinding(
+          "outcome.status",
+          `Observed outcome count ${matching.length} matched [${failOn.join(", ")}].`,
+          matching.map((outcome) => ({
+            runId: outcome.runId,
+            eventId: outcome.outcomeId,
+            ...outcome.parentId !== void 0 ? { parentId: outcome.parentId } : {},
+            kind: "OUTCOME",
+            name: outcome.name,
+            status: outcome.status,
+            path: `outcome.${outcome.name}`
+          })),
+          { failOn },
+          matching.map((outcome) => ({
+            name: outcome.name,
+            status: outcome.status,
+            expectation: outcome.expectation
+          }))
+        )
+      ];
+    }
+  };
+}
 function runTraceChecks(input, options = {}) {
   const selected = resolveSelectedRun(input, options.runId);
   if (selected.diagnostics.length > 0) {
@@ -1616,6 +1714,7 @@ exports.createDecisionRule = createDecisionRule;
 exports.createGuardrailRule = createGuardrailRule;
 exports.createLlmUsageRule = createLlmUsageRule;
 exports.createMaxStepDurationRule = createMaxStepDurationRule;
+exports.createObservedOutcomeRule = createObservedOutcomeRule;
 exports.createRequireCompletedRule = createRequireCompletedRule;
 exports.createRetrievalRule = createRetrievalRule;
 exports.createRunDepthRule = createRunDepthRule;