agent-inspect 6.31.34 → 6.31.36

package/packages/cli/dist/index.cjs

@@ -5729,11 +5729,27 @@ function eventTimestamp(finding, eventById) {
   const eventId = finding.evidence[0]?.eventId;
   return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
 }
+function isObservedToolPolicyFinding(finding) {
+  return finding.ruleId === "tool.usage" && finding.evidence[0]?.eventId !== void 0;
+}
+function isMissingToolConsequenceFinding(finding) {
+  if (finding.evidence[0]?.eventId !== void 0) return false;
+  if (finding.ruleId === "tool.usage" && /did not appear/.test(finding.message)) {
+    return true;
+  }
+  return finding.ruleId.startsWith("contract.tool.arguments") && /^No finished tool named /.test(finding.message);
+}
 function compareFindings(eventById) {
   return (a, b) => {
     if (SEVERITY_RANK[a.severity] !== SEVERITY_RANK[b.severity]) {
       return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity];
     }
+    const aPolicy = isObservedToolPolicyFinding(a);
+    const bPolicy = isObservedToolPolicyFinding(b);
+    const aMissing = isMissingToolConsequenceFinding(a);
+    const bMissing = isMissingToolConsequenceFinding(b);
+    if (aPolicy && bMissing) return -1;
+    if (bPolicy && aMissing) return 1;
     const byRule = a.ruleId.localeCompare(b.ruleId);
     if (byRule !== 0) return byRule;
     if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
@@ -5869,20 +5885,42 @@ function failFinding(ruleId, message, evidence, expected, actual, meta2) {
 function toolName(event) {
   return resolveCanonicalToolName(event);
 }
+function isAsciiLetterCodePoint(codePoint) {
+  return codePoint >= 65 && codePoint <= 90 || codePoint >= 97 && codePoint <= 122;
+}
+function isRiskyToolNameCodePoint(codePoint) {
+  if (codePoint <= 31 || codePoint >= 127 && codePoint <= 159) return true;
+  return codePoint === 1564 || codePoint === 8203 || codePoint === 8204 || codePoint === 8205 || codePoint === 8206 || codePoint === 8207 || codePoint >= 8234 && codePoint <= 8238 || codePoint >= 8294 && codePoint <= 8297 || codePoint === 65279;
+}
+function formatCodePointDetail(codePoint, index) {
+  return `U+${codePoint.toString(16).toUpperCase().padStart(4, "0")} at index ${index}`;
+}
 function formatToolNameForMessage(name) {
+  let hasAsciiLetter = false;
+  let hasNonAscii = false;
+  let hasRisky = false;
   const details = [];
   for (let index = 0; index < name.length; ) {
     const codePoint = name.codePointAt(index);
     const width = codePoint > 65535 ? 2 : 1;
+    if (isAsciiLetterCodePoint(codePoint)) hasAsciiLetter = true;
+    const risky = isRiskyToolNameCodePoint(codePoint);
+    if (risky) hasRisky = true;
     if (codePoint > 127) {
-      details.push(
-        `U+${codePoint.toString(16).toUpperCase().padStart(4, "0")} at index ${index}`
-      );
+      hasNonAscii = true;
+      details.push(formatCodePointDetail(codePoint, index));
+    } else if (risky) {
+      details.push(formatCodePointDetail(codePoint, index));
     }
     index += width;
   }
-  if (details.length === 0) return name;
-  return `"${name}" (contains ${details.join("; ")})`;
+  if (!hasRisky && !(hasAsciiLetter && hasNonAscii) || details.length === 0) {
+    return name;
+  }
+  const shown = details.slice(0, MAX_TOOL_NAME_CODEPOINT_DETAILS);
+  const remaining = details.length - shown.length;
+  const detailText = remaining > 0 ? `${shown.join("; ")}; and ${remaining} more` : shown.join("; ");
+  return `"${name}" (contains ${detailText})`;
 }
 function describeRunningEvent(event) {
   if (event.kind === "TOOL") {
@@ -6827,6 +6865,9 @@ function createToolFailureRule(options) {
     }
   };
 }
+function recordsTokenUsage(event) {
+  return Object.values(event.tokenUsage ?? {}).some((count) => typeof count === "number");
+}
 function createLlmUsageRule(options) {
   return {
     id: "llm.usage",
@@ -6884,6 +6925,21 @@ function createLlmUsageRule(options) {
         ["total", options.maxTotalTokens],
         ["cached", options.maxCachedTokens]
       ];
+      const hasTokenLimit = tokenLimits.some(([, limit]) => limit !== void 0);
+      const callsWithoutUsage = llms.filter(
+        (event) => event.status !== "error" && !recordsTokenUsage(event)
+      );
+      if (hasTokenLimit && callsWithoutUsage.length > 0) {
+        findings.push(
+          failFinding(
+            "llm.usage",
+            `No token usage was read for ${callsWithoutUsage.length} of ${llms.length} LLM call(s), so the token limit cannot be verified.`,
+            callsWithoutUsage.map((event) => eventEvidence(event, "tokenUsage")),
+            "token usage on every completed LLM call",
+            { callsWithoutUsage: callsWithoutUsage.length, calls: llms.length }
+          )
+        );
+      }
       for (const [key, limit] of tokenLimits) {
         if (limit !== void 0 && tokenTotals[key] > limit) {
           findings.push(
@@ -7609,7 +7665,7 @@ function runTraceChecks(input3, options = {}) {
     ruleExecutions
   };
 }
-var SEVERITY_RANK, STATUS_RANK, CONFIDENCE_RANK, DEFAULT_SENSITIVE_KEYS, DEFAULT_RAW_CONTENT_KEYS, DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES, SAFE_USAGE_LEAF_KEYS, DEFAULT_SECRET_PATTERNS;
+var SEVERITY_RANK, STATUS_RANK, CONFIDENCE_RANK, DEFAULT_SENSITIVE_KEYS, DEFAULT_RAW_CONTENT_KEYS, DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES, SAFE_USAGE_LEAF_KEYS, DEFAULT_SECRET_PATTERNS, MAX_TOOL_NAME_CODEPOINT_DETAILS;
 var init_checks2 = __esm({
   "packages/core/src/checks/index.ts"() {
     init_inspect_event();
@@ -7713,6 +7769,7 @@ var init_checks2 = __esm({
         pattern: /\b(?:api[_-]?key|internal[_-]?token|access[_-]?token|auth[_-]?token|password|secret|token)=(?!\[(?:REDACTED|HASH:(?:[0-9a-f]{8}|unknown))\](?![^\s"'\\&#?]))([^\s"'\\]{8,})/i
       }
     ];
+    MAX_TOOL_NAME_CODEPOINT_DETAILS = 3;
   }
 });
 
@@ -17479,7 +17536,7 @@ var init_src = __esm({
 });
 
 // package.json
-var version = "6.31.34";
+var version = "6.31.36";
 
 // packages/cli/src/list.ts
 init_advanced();