@agent-inspect/viewer 6.17.4 → 6.17.6

package/dist/index.mjs

@@ -2622,10 +2622,11 @@ function emptySummary() {
     passed: 0,
     failed: 0,
     warnings: 0,
-    errors: 0
+    errors: 0,
+    rulesEvaluated: 0
   };
 }
-function errorResult(input, diagnostics, selectedRun) {
+function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
   return {
     ok: false,
     status: "error",
@@ -2633,10 +2634,12 @@ function errorResult(input, diagnostics, selectedRun) {
     ...selectedRun ? { runId: selectedRun.runId } : {},
     summary: {
       ...emptySummary(),
-      errors: diagnostics.filter((item) => item.severity === "error").length
+      errors: diagnostics.filter((item) => item.severity === "error").length,
+      rulesEvaluated: ruleExecutions.length
     },
     findings: [],
-    diagnostics: [...diagnostics]
+    diagnostics: [...diagnostics],
+    ruleExecutions: [...ruleExecutions]
   };
 }
 function flattenNodes(nodes) {
@@ -2792,7 +2795,7 @@ function normalizeFinding(rule, finding) {
     ...finding.action !== void 0 ? { action: finding.action } : {}
   };
 }
-function summarize(findings, diagnostics) {
+function summarize(findings, diagnostics, rulesEvaluated) {
   return {
     passed: findings.filter((finding) => finding.status === "pass").length,
     failed: findings.filter(
@@ -2801,9 +2804,21 @@ function summarize(findings, diagnostics) {
     warnings: findings.filter(
       (finding) => finding.status === "warning" || finding.severity === "warning"
     ).length,
-    errors: diagnostics.filter((item) => item.severity === "error").length
+    errors: diagnostics.filter((item) => item.severity === "error").length,
+    rulesEvaluated
   };
 }
+function classifyRuleExecution(findings, threw) {
+  if (findings.some((finding) => finding.status === "fail" && finding.severity === "error")) {
+    return "fail";
+  }
+  if (findings.some(
+    (finding) => finding.status === "warning" || finding.severity === "warning"
+  )) {
+    return "warning";
+  }
+  return "pass";
+}
 function stringAttr(event, keys) {
   for (const key of keys) {
     const value = event.attributes?.[key];
@@ -2880,9 +2895,12 @@ function llmFinishReason(event) {
 }
 function finishedEvents(context, kind) {
   return semanticEvents(context).filter(
-    (event) => (kind === void 0 || event.kind === kind) && event.status !== "running"
+    (event) => (event.kind === kind) && event.status !== "running"
   );
 }
+function toolInvocationEvents(context) {
+  return semanticEvents(context).filter((event) => event.kind === "TOOL");
+}
 function createRunStatusRule(options = {}) {
   const expected = options.expected ?? "ok";
   const allowIncomplete = options.allowIncomplete === true;
@@ -2948,7 +2966,7 @@ function createToolUsageRule(options) {
     category: "tool",
     defaultSeverity: "error",
     evaluate(context) {
-      const tools = finishedEvents(context, "TOOL");
+      const tools = toolInvocationEvents(context);
       const names = tools.map(toolName);
       const nameSet = new Set(names);
       const findings = [];
@@ -3064,12 +3082,24 @@ function createLlmUsageRule(options) {
 }
 function createObservedOutcomeRule(options = {}) {
   const failOn = options.failOn ?? ["failed"];
+  const requireAny = options.requireAny === true;
   return {
     id: "outcome.status",
     category: "run",
     defaultSeverity: "error",
     evaluate(context) {
       const outcomes = extractOutcomesFromPersistedEvents(context.events);
+      if (requireAny && outcomes.length === 0) {
+        return [
+          failFinding(
+            "outcome.status",
+            "Expected at least one observed outcome.",
+            runEvidence(context.selectedRun),
+            { requireAny: true, expected: "at least one observed outcome" },
+            { code: "outcome.missing", actual: 0 }
+          )
+        ];
+      }
       const matching = outcomesMatchingStatus(outcomes, failOn);
       if (matching.length === 0) return [];
       return [
@@ -3105,6 +3135,18 @@ function runTraceChecks(input, options = {}) {
   if (rules.diagnostics.length > 0) {
     return errorResult(input, rules.diagnostics, selected.run);
   }
+  if (rules.rules.length === 0) {
+    return errorResult(
+      input,
+      [
+        diagnostic3(
+          "AI_CHECK_NO_RULES_EVALUATED",
+          "No trace check rules were evaluated. Configure at least one rule, contract, or CLI check option."
+        )
+      ],
+      selected.run
+    );
+  }
   const facts = buildFacts(input, selected.run);
   const context = {
     ...facts,
@@ -3113,22 +3155,38 @@ function runTraceChecks(input, options = {}) {
   };
   const diagnostics = [];
   const findings = [];
+  const ruleExecutions = [];
   for (const rule of rules.rules) {
     try {
-      findings.push(...rule.evaluate(context).map((finding) => normalizeFinding(rule, finding)));
+      const ruleFindings = rule.evaluate(context).map((finding) => normalizeFinding(rule, finding));
+      findings.push(...ruleFindings);
+      ruleExecutions.push({
+        ruleId: rule.id,
+        category: rule.category,
+        status: classifyRuleExecution(ruleFindings, false),
+        findingCount: ruleFindings.length,
+        ...selected.run ? { runId: selected.run.runId } : {}
+      });
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       diagnostics.push(
         diagnostic3("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
       );
+      ruleExecutions.push({
+        ruleId: rule.id,
+        category: rule.category,
+        status: "error",
+        findingCount: 0,
+        ...selected.run ? { runId: selected.run.runId } : {}
+      });
     }
   }
   if (diagnostics.length > 0) {
-    return errorResult(input, diagnostics, selected.run);
+    return errorResult(input, diagnostics, selected.run, ruleExecutions);
   }
   const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
   const sortedFindings = findings.sort(compareFindings(eventById));
-  const summary = summarize(sortedFindings, diagnostics);
+  const summary = summarize(sortedFindings, diagnostics, ruleExecutions.length);
   const status = summary.failed > 0 ? "fail" : "pass";
   return {
     ok: status === "pass",
@@ -3137,7 +3195,8 @@ function runTraceChecks(input, options = {}) {
     ...selected.run ? { runId: selected.run.runId } : {},
     summary,
     findings: sortedFindings,
-    diagnostics
+    diagnostics,
+    ruleExecutions
   };
 }
 
@@ -5464,8 +5523,7 @@ async function runSuiteCase(suiteCase, config, options) {
     status: "pass",
     format: read.format,
     findings: [],
-    diagnostics: []
-  };
+    diagnostics: []};
   const observationResult = validateExpectedObservations(suiteCase, read);
   const diagnostics = [
     ...checkResult.diagnostics.map(