agent-inspect 6.17.4 → 6.17.6

package/packages/core/dist/checks.cjs

@@ -673,9 +673,13 @@ function contractToRules(contract) {
     rules.push(createStructureIncompleteRule({ requireEndedAtForStarted: true }));
   }
   if (contract.tools) {
+    const order = contract.tools.requiredOrder ?? [];
     const required = [
-      ...contract.tools.required ?? [],
-      ...contract.tools.requiredTools ?? []
+      .../* @__PURE__ */ new Set([
+        ...contract.tools.required ?? [],
+        ...contract.tools.requiredTools ?? [],
+        ...order
+      ])
     ];
     const forbidden = [
       ...contract.tools.forbidden ?? [],
@@ -689,9 +693,14 @@ function contractToRules(contract) {
         ...contract.tools.maxCalls !== void 0 ? { maxCount: contract.tools.maxCalls } : {}
       })
     );
-    const order = contract.tools.requiredOrder ?? [];
     for (let i = 0; i < order.length - 1; i += 1) {
-      rules.push(createToolOrderingRule({ before: order[i], after: order[i + 1] }));
+      rules.push(
+        createToolOrderingRule({
+          before: order[i],
+          after: order[i + 1],
+          id: `contract.tool.order.${i}`
+        })
+      );
     }
   }
   if (contract.llm) {
@@ -851,10 +860,11 @@ function emptySummary() {
     passed: 0,
     failed: 0,
     warnings: 0,
-    errors: 0
+    errors: 0,
+    rulesEvaluated: 0
   };
 }
-function errorResult(input, diagnostics, selectedRun) {
+function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
   return {
     ok: false,
     status: "error",
@@ -862,10 +872,12 @@ function errorResult(input, diagnostics, selectedRun) {
     ...selectedRun ? { runId: selectedRun.runId } : {},
     summary: {
       ...emptySummary(),
-      errors: diagnostics.filter((item) => item.severity === "error").length
+      errors: diagnostics.filter((item) => item.severity === "error").length,
+      rulesEvaluated: ruleExecutions.length
     },
     findings: [],
-    diagnostics: [...diagnostics]
+    diagnostics: [...diagnostics],
+    ruleExecutions: [...ruleExecutions]
   };
 }
 function flattenNodes(nodes) {
@@ -1021,7 +1033,7 @@ function normalizeFinding(rule, finding) {
     ...finding.action !== void 0 ? { action: finding.action } : {}
   };
 }
-function summarize(findings, diagnostics) {
+function summarize(findings, diagnostics, rulesEvaluated) {
   return {
     passed: findings.filter((finding) => finding.status === "pass").length,
     failed: findings.filter(
@@ -1030,9 +1042,21 @@ function summarize(findings, diagnostics) {
     warnings: findings.filter(
       (finding) => finding.status === "warning" || finding.severity === "warning"
     ).length,
-    errors: diagnostics.filter((item) => item.severity === "error").length
+    errors: diagnostics.filter((item) => item.severity === "error").length,
+    rulesEvaluated
   };
 }
+function classifyRuleExecution(findings, threw) {
+  if (findings.some((finding) => finding.status === "fail" && finding.severity === "error")) {
+    return "fail";
+  }
+  if (findings.some(
+    (finding) => finding.status === "warning" || finding.severity === "warning"
+  )) {
+    return "warning";
+  }
+  return "pass";
+}
 function stringAttr(event, keys) {
   for (const key of keys) {
     const value = event.attributes?.[key];
@@ -1129,6 +1153,9 @@ function finishedEvents(context, kind) {
     (event) => (kind === void 0 || event.kind === kind) && event.status !== "running"
   );
 }
+function toolInvocationEvents(context) {
+  return semanticEvents(context).filter((event) => event.kind === "TOOL");
+}
 function firstIndexByName(events) {
   const index = /* @__PURE__ */ new Map();
   for (const [position, event] of events.entries()) {
@@ -1576,7 +1603,7 @@ function createToolUsageRule(options) {
     category: "tool",
     defaultSeverity: "error",
     evaluate(context) {
-      const tools = finishedEvents(context, "TOOL");
+      const tools = toolInvocationEvents(context);
       const names = tools.map(toolName);
       const nameSet = new Set(names);
       const findings = [];
@@ -1617,8 +1644,9 @@ function createToolUsageRule(options) {
   };
 }
 function createToolOrderingRule(options) {
+  const ruleId = options.id ?? "tool.order";
   return {
-    id: "tool.order",
+    id: ruleId,
     category: "tool",
     defaultSeverity: "error",
     evaluate(context) {
@@ -1626,18 +1654,42 @@ function createToolOrderingRule(options) {
       const index = firstIndexByName(tools);
       const beforeIndex = index.get(options.before);
       const afterIndex = index.get(options.after);
-      if (beforeIndex === void 0 || afterIndex === void 0 || beforeIndex < afterIndex) {
+      if (beforeIndex === void 0 || afterIndex === void 0) {
         return [];
       }
-      return [
-        failFinding(
-          "tool.order",
-          `Tool ${options.before} must appear before ${options.after}.`,
-          [eventEvidence(tools[beforeIndex]), eventEvidence(tools[afterIndex])],
-          { before: options.before, after: options.after },
-          tools.map(toolName)
-        )
-      ];
+      if (beforeIndex >= afterIndex) {
+        return [
+          failFinding(
+            ruleId,
+            `Tool ${options.before} must appear before ${options.after}.`,
+            [eventEvidence(tools[beforeIndex]), eventEvidence(tools[afterIndex])],
+            { before: options.before, after: options.after },
+            tools.map(toolName)
+          )
+        ];
+      }
+      const beforeEvent = tools[beforeIndex];
+      const afterEvent = tools[afterIndex];
+      const beforeEnd = eventEndMs(beforeEvent);
+      const afterStart = eventStartMs(afterEvent);
+      if (beforeEnd !== void 0 && afterStart !== void 0 && beforeEnd > afterStart) {
+        return [
+          {
+            ruleId,
+            severity: "warning",
+            status: "warning",
+            message: `Tool order ${options.before} before ${options.after} satisfied start/encounter order, but intervals overlap so causal completion before the next tool was not established.`,
+            expected: { before: options.before, after: options.after, mode: "first-occurrence" },
+            actual: {
+              code: "tool.order.overlap",
+              beforeEndedAt: beforeEvent.endedAt,
+              afterStartedAt: afterEvent.startedAt ?? afterEvent.timestamp
+            },
+            evidence: [eventEvidence(beforeEvent), eventEvidence(afterEvent)]
+          }
+        ];
+      }
+      return [];
     }
   };
 }
@@ -2434,12 +2486,24 @@ function createBaselineRegressionRule(options) {
 }
 function createObservedOutcomeRule(options = {}) {
   const failOn = options.failOn ?? ["failed"];
+  const requireAny = options.requireAny === true;
   return {
     id: "outcome.status",
     category: "run",
     defaultSeverity: "error",
     evaluate(context) {
       const outcomes = extractOutcomesFromPersistedEvents(context.events);
+      if (requireAny && outcomes.length === 0) {
+        return [
+          failFinding(
+            "outcome.status",
+            "Expected at least one observed outcome.",
+            runEvidence(context.selectedRun),
+            { requireAny: true, expected: "at least one observed outcome" },
+            { code: "outcome.missing", actual: 0 }
+          )
+        ];
+      }
       const matching = outcomesMatchingStatus(outcomes, failOn);
       if (matching.length === 0) return [];
       return [
@@ -2475,6 +2539,18 @@ function runTraceChecks(input, options = {}) {
   if (rules.diagnostics.length > 0) {
     return errorResult(input, rules.diagnostics, selected.run);
   }
+  if (rules.rules.length === 0) {
+    return errorResult(
+      input,
+      [
+        diagnostic(
+          "AI_CHECK_NO_RULES_EVALUATED",
+          "No trace check rules were evaluated. Configure at least one rule, contract, or CLI check option."
+        )
+      ],
+      selected.run
+    );
+  }
   const facts = buildFacts(input, selected.run);
   const context = {
     ...facts,
@@ -2483,22 +2559,38 @@ function runTraceChecks(input, options = {}) {
   };
   const diagnostics = [];
   const findings = [];
+  const ruleExecutions = [];
   for (const rule of rules.rules) {
     try {
-      findings.push(...rule.evaluate(context).map((finding) => normalizeFinding(rule, finding)));
+      const ruleFindings = rule.evaluate(context).map((finding) => normalizeFinding(rule, finding));
+      findings.push(...ruleFindings);
+      ruleExecutions.push({
+        ruleId: rule.id,
+        category: rule.category,
+        status: classifyRuleExecution(ruleFindings, false),
+        findingCount: ruleFindings.length,
+        ...selected.run ? { runId: selected.run.runId } : {}
+      });
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       diagnostics.push(
         diagnostic("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
       );
+      ruleExecutions.push({
+        ruleId: rule.id,
+        category: rule.category,
+        status: "error",
+        findingCount: 0,
+        ...selected.run ? { runId: selected.run.runId } : {}
+      });
     }
   }
   if (diagnostics.length > 0) {
-    return errorResult(input, diagnostics, selected.run);
+    return errorResult(input, diagnostics, selected.run, ruleExecutions);
   }
   const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
   const sortedFindings = findings.sort(compareFindings(eventById));
-  const summary = summarize(sortedFindings, diagnostics);
+  const summary = summarize(sortedFindings, diagnostics, ruleExecutions.length);
   const status = summary.failed > 0 ? "fail" : "pass";
   return {
     ok: status === "pass",
@@ -2507,7 +2599,8 @@ function runTraceChecks(input, options = {}) {
     ...selected.run ? { runId: selected.run.runId } : {},
     summary,
     findings: sortedFindings,
-    diagnostics
+    diagnostics,
+    ruleExecutions
   };
 }