agent-inspect 6.31.31 → 6.31.33

package/packages/cli/dist/index.cjs

@@ -268,6 +268,10 @@ var init_credential_value_patterns = __esm({
       /\b[a-z0-9_.-]+=[^;\s]+(?:;\s*[a-z0-9_.-]+=[^;\s]+)+/i,
       /\beyJ[a-zA-Z0-9_-]{8,}\.[a-zA-Z0-9_-]{8,}\.[a-zA-Z0-9_-]{8,}\b/,
       /\b(?:sk-(?:proj-)?[a-zA-Z0-9_-]{16,}|sk-ant-[a-zA-Z0-9_-]{16,}|AIza[0-9A-Za-z_-]{20,})\b/,
+      // Stripe secret / restricted keys use underscores (not sk-). Keep in sync with
+      // packages/redact/src/index.ts + packages/core/src/checks DEFAULT_SECRET_PATTERNS (#520).
+      /\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}\b/,
+      /\bxox[abposr]-[A-Za-z0-9-]{10,}\b/,
       /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{20,}\b/,
       /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/,
       /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*-----END [A-Z ]*PRIVATE KEY-----/,
@@ -5734,6 +5738,11 @@ function compareFindings(eventById) {
     if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
       return STATUS_RANK[a.status] - STATUS_RANK[b.status];
     }
+    if (a.ruleId === "tool.usage") {
+      const aHasEvent = a.evidence[0]?.eventId !== void 0;
+      const bHasEvent = b.evidence[0]?.eventId !== void 0;
+      if (aHasEvent !== bHasEvent) return aHasEvent ? -1 : 1;
+    }
     const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
     if (byRun !== 0) return byRun;
     const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
@@ -5859,6 +5868,37 @@ function failFinding(ruleId, message, evidence, expected, actual, meta2) {
 function toolName(event) {
   return resolveCanonicalToolName(event);
 }
+function formatToolNameForMessage(name) {
+  const details = [];
+  for (let index = 0; index < name.length; ) {
+    const codePoint = name.codePointAt(index);
+    const width = codePoint > 65535 ? 2 : 1;
+    if (codePoint > 127) {
+      details.push(
+        `U+${codePoint.toString(16).toUpperCase().padStart(4, "0")} at index ${index}`
+      );
+    }
+    index += width;
+  }
+  if (details.length === 0) return name;
+  return `"${name}" (contains ${details.join("; ")})`;
+}
+function describeRunningEvent(event) {
+  if (event.kind === "TOOL") {
+    return `tool ${formatToolNameForMessage(resolveCanonicalToolName(event))}`;
+  }
+  if (event.kind === "LLM") {
+    return `llm ${stepEndpointName(event, "LLM")}`;
+  }
+  return `${event.kind} ${event.name}`;
+}
+function incompleteRunningMessage(prefix, running) {
+  const labels = running.map(describeRunningEvent);
+  if (labels.length === 1) {
+    return `${prefix}: ${labels[0]} never completed.`;
+  }
+  return `${prefix}: ${labels.join("; ")}.`;
+}
 function stepEndpointName(event, kind) {
   if (kind === "TOOL") return resolveCanonicalToolName(event);
   for (const prefix of ["llm:", "generation:", "transcription:", "speech:"]) {
@@ -5933,6 +5973,56 @@ function eventEndMs(event) {
   }
   return void 0;
 }
+function rowIndexByEventId(events) {
+  const index = /* @__PURE__ */ new Map();
+  for (const [position, event] of events.entries()) {
+    if (!index.has(event.eventId)) index.set(event.eventId, position);
+  }
+  return index;
+}
+function recordedRowSpan(rowIndex, event) {
+  const sourceIds = "sourceEventIds" in event && Array.isArray(event.sourceEventIds) ? event.sourceEventIds : [event.eventId];
+  const rows = [];
+  for (const id of sourceIds) {
+    const row = typeof id === "string" ? rowIndex.get(id) : void 0;
+    if (row === void 0) return void 0;
+    rows.push(row);
+  }
+  if (rows.length === 0) return void 0;
+  return { first: Math.min(...rows), last: Math.max(...rows) };
+}
+function recordedTieOrder(context, tiedBefore, tiedAfter) {
+  const rowIndex = rowIndexByEventId(context.events);
+  let lastFinished;
+  let firstStarted;
+  for (const event of tiedBefore) {
+    const span = recordedRowSpan(rowIndex, event);
+    if (span === void 0) return { inOrder: void 0 };
+    if (lastFinished === void 0 || span.last > lastFinished.row) {
+      lastFinished = { event, row: span.last };
+    }
+  }
+  for (const event of tiedAfter) {
+    const span = recordedRowSpan(rowIndex, event);
+    if (span === void 0) return { inOrder: void 0 };
+    if (firstStarted === void 0 || span.first < firstStarted.row) {
+      firstStarted = { event, row: span.first };
+    }
+  }
+  if (lastFinished === void 0 || firstStarted === void 0) return { inOrder: void 0 };
+  return {
+    inOrder: lastFinished.row < firstStarted.row,
+    lastFinishedBefore: lastFinished.event,
+    firstStartedAfter: firstStarted.event
+  };
+}
+function tieResolution(order) {
+  return order.inOrder === void 0 ? "recorded-order-unavailable" : "recorded-order";
+}
+function tieMessage(order, beforeLabel, afterLabel) {
+  if (order === void 0 || order.inOrder === true) return "";
+  return order.inOrder === void 0 ? " Their timestamps tie at the same millisecond and the recorded event order could not be resolved." : ` Their timestamps tie at the same millisecond; the recorded event order shows ${afterLabel} started before ${beforeLabel} finished.`;
+}
 function normalizedKey(value) {
   return value.toLowerCase().replace(/[^a-z0-9_]/g, "");
 }
@@ -6150,10 +6240,13 @@ function createRunStatusRule(options = {}) {
           findings.push(
             failFinding(
               "run.status",
-              "Run contains incomplete running events.",
+              incompleteRunningMessage("Run contains incomplete running events", running),
               running.map((event) => eventEvidence(event)),
               "no running events",
-              running.length
+              {
+                count: running.length,
+                unfinished: running.map(describeRunningEvent)
+              }
             )
           );
         }
@@ -6323,6 +6416,34 @@ function createToolUsageRule(options) {
       const names = tools.map(toolName);
       const nameSet = new Set(names);
       const findings = [];
+      const forbidden = new Set(options.forbidden ?? []);
+      const allowed = options.allowed?.length ? new Set(options.allowed) : void 0;
+      for (const event of tools) {
+        const name = toolName(event);
+        const display = formatToolNameForMessage(name);
+        if (forbidden.has(name)) {
+          findings.push(
+            failFinding(
+              "tool.usage",
+              `Forbidden tool ${display} appeared.`,
+              [eventEvidence(event)],
+              "tool absent",
+              name
+            )
+          );
+        }
+        if (allowed && !allowed.has(name)) {
+          findings.push(
+            failFinding(
+              "tool.usage",
+              `Tool ${display} is not in the allowed tool set.`,
+              [eventEvidence(event)],
+              [...allowed].sort(),
+              name
+            )
+          );
+        }
+      }
       for (const required of options.required ?? []) {
         if (!nameSet.has(required)) {
           const otherKinds = [
@@ -6340,7 +6461,7 @@ function createToolUsageRule(options) {
           findings.push(
             failFinding(
               "tool.usage",
-              `Required tool ${required} did not appear.${hint}`,
+              `Required tool ${formatToolNameForMessage(required)} did not appear.${hint}`,
               runEvidence(context.selectedRun),
               required,
               names
@@ -6348,21 +6469,6 @@ function createToolUsageRule(options) {
           );
         }
       }
-      const forbidden = new Set(options.forbidden ?? []);
-      const allowed = options.allowed?.length ? new Set(options.allowed) : void 0;
-      for (const event of tools) {
-        const name = toolName(event);
-        if (forbidden.has(name)) {
-          findings.push(
-            failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence(event)], "tool absent", name)
-          );
-        }
-        if (allowed && !allowed.has(name)) {
-          findings.push(
-            failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence(event)], [...allowed].sort(), name)
-          );
-        }
-      }
       if (options.minCount !== void 0 && tools.length < options.minCount) {
         findings.push(
           failFinding("tool.usage", `Tool count ${tools.length} was below minimum ${options.minCount}.`, runEvidence(context.selectedRun), { minCount: options.minCount }, tools.length)
@@ -6396,6 +6502,28 @@ function createStepOrderingRule(options) {
       const afterEvents = finishedEvents(context, options.after.kind).filter(
         (event) => stepEndpointName(event, options.after.kind) === options.after.name
       );
+      const beforeRunning = semanticEvents(context).filter(
+        (event) => event.kind === options.before.kind && event.status === "running" && stepEndpointName(event, options.before.kind) === options.before.name
+      );
+      if (beforeEvents.length === 0 && afterEvents.length > 0 && beforeRunning.length > 0) {
+        const endpointExpected2 = bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after };
+        return [
+          failFinding(
+            ruleId,
+            `${subject} ${beforeLabel} started but never finished before ${afterLabel} ran.`,
+            [
+              ...beforeRunning.map((event) => eventEvidence(event)),
+              eventEvidence(afterEvents[0])
+            ],
+            { ...endpointExpected2, mode },
+            {
+              code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
+              unfinishedBefore: true,
+              beforeStatus: "running"
+            }
+          )
+        ];
+      }
       if (beforeEvents.length === 0 || afterEvents.length === 0) {
         if (!requireEndpoints) return [];
         const findings = [];
@@ -6512,16 +6640,18 @@ function createStepOrderingRule(options) {
             )
           ];
         }
-        if (beforeEnd > afterStart) {
+        const tie = beforeEnd === afterStart ? recordedTieOrder(context, [beforeEvent], [afterEvent]) : void 0;
+        if (beforeEnd > afterStart || tie !== void 0 && tie.inOrder !== true) {
           return [
             failFinding(
               ruleId,
-              `${subject} ${beforeLabel} must finish before ${afterLabel} starts.`,
+              `${subject} ${beforeLabel} must finish before ${afterLabel} starts.${tieMessage(tie, beforeLabel, afterLabel)}`,
               [eventEvidence(beforeEvent), eventEvidence(afterEvent)],
               expected,
               {
                 beforeEndedAt: new Date(beforeEnd).toISOString(),
-                afterStartedAt: new Date(afterStart).toISOString()
+                afterStartedAt: new Date(afterStart).toISOString(),
+                ...tie !== void 0 ? { resolvedBy: tieResolution(tie) } : {}
               }
             )
           ];
@@ -6584,16 +6714,27 @@ function createStepOrderingRule(options) {
             )
           ];
         }
-        if (latestBefore.end > earliestAfter.start) {
+        const tiedEnd = latestBefore.end;
+        const tiedStart = earliestAfter.start;
+        const tie = tiedEnd === tiedStart ? recordedTieOrder(
+          context,
+          beforeEvents.filter((event) => eventEndMs(event) === tiedEnd),
+          afterEvents.filter((event) => eventStartMs(event) === tiedStart)
+        ) : void 0;
+        if (latestBefore.end > earliestAfter.start || tie !== void 0 && tie.inOrder !== true) {
           return [
             failFinding(
               ruleId,
-              bothTools ? `Every ${options.before.name} tool call must finish before any ${options.after.name} tool call starts.` : `Every ${beforeLabel} must finish before any ${afterLabel} starts.`,
-              [eventEvidence(latestBefore.event), eventEvidence(earliestAfter.event)],
+              (bothTools ? `Every ${options.before.name} tool call must finish before any ${options.after.name} tool call starts.` : `Every ${beforeLabel} must finish before any ${afterLabel} starts.`) + tieMessage(tie, beforeLabel, afterLabel),
+              [
+                eventEvidence(tie?.lastFinishedBefore ?? latestBefore.event),
+                eventEvidence(tie?.firstStartedAfter ?? earliestAfter.event)
+              ],
               expected,
               {
                 latestBeforeEndedAt: new Date(latestBefore.end).toISOString(),
-                earliestAfterStartedAt: new Date(earliestAfter.start).toISOString()
+                earliestAfterStartedAt: new Date(earliestAfter.start).toISOString(),
+                ...tie !== void 0 ? { resolvedBy: tieResolution(tie) } : {}
               }
             )
           ];
@@ -6759,10 +6900,13 @@ function createStructureIncompleteRule(options = {}) {
           findings.push(
             failFinding(
               "structure.incomplete",
-              "Trace contains incomplete running events.",
+              incompleteRunningMessage("Trace contains incomplete running events", running),
               running.map((event) => eventEvidence(event, "status")),
               "no running events",
-              running.length
+              {
+                count: running.length,
+                unfinished: running.map(describeRunningEvent)
+              }
             )
           );
         }
@@ -7540,6 +7684,12 @@ var init_checks2 = __esm({
     DEFAULT_SECRET_PATTERNS = [
       { id: "bearer-token", pattern: /Bearer\s+[A-Za-z0-9._~+/-]{12,}=*/ },
       { id: "openai-key", pattern: /sk-[A-Za-z0-9_-]{16,}/ },
+      // Stripe + Slack: keep in sync with packages/redact + credential-value-patterns (#520).
+      {
+        id: "stripe-key",
+        pattern: /\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}\b/
+      },
+      { id: "slack-token", pattern: /\bxox[abposr]-[A-Za-z0-9-]{10,}\b/ },
       { id: "aws-access-key", pattern: /AKIA[0-9A-Z]{16}/ },
       { id: "github-token", pattern: /gh[opsu]_[A-Za-z0-9_]{20,}/ },
       // Keep in sync with packages/redact/src/key-value-secret.ts (KEY_VALUE_SECRET_PATTERN_SOURCE).
@@ -9491,10 +9641,17 @@ function contractToRules(contract) {
             (event) => event.status === "running"
           );
           if (running.length > 0) {
+            const labels = running.map((event) => {
+              if (event.kind === "TOOL") {
+                return `tool ${resolveCanonicalToolName(event)}`;
+              }
+              return `${event.kind} ${event.name}`;
+            });
+            const message = labels.length === 1 ? `Run contains incomplete running events: ${labels[0]} never completed.` : `Run contains incomplete running events: ${labels.join("; ")}.`;
             findings.push(
               contractFailFinding(
                 "contract.run.allowedStatuses",
-                "Run contains incomplete running events.",
+                message,
                 running.map((event) => ({
                   runId: event.runId,
                   eventId: event.eventId,
@@ -9503,7 +9660,10 @@ function contractToRules(contract) {
                   status: event.status
                 })),
                 "no running events",
-                running.length
+                {
+                  count: running.length,
+                  unfinished: labels
+                }
               )
             );
           }
@@ -17304,7 +17464,7 @@ var init_src = __esm({
 });
 
 // package.json
-var version = "6.31.31";
+var version = "6.31.33";
 
 // packages/cli/src/list.ts
 init_advanced();
@@ -22499,6 +22659,18 @@ var credentialDetectors = [
     pattern: /\b(?:sk-(?:proj-)?[a-zA-Z0-9_-]{16,}|sk-ant-[a-zA-Z0-9_-]{16,}|AIza[0-9A-Za-z_-]{20,})\b/,
     severity: "error"
   }),
+  // Stripe secret/restricted keys (underscore form). Keep in sync with
+  // packages/core/src/safety/credential-value-patterns.ts + checks DEFAULT_SECRET_PATTERNS (#520).
+  patternDetector({
+    id: "value.stripeSecretKey",
+    pattern: /\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}\b/,
+    severity: "error"
+  }),
+  patternDetector({
+    id: "value.slackToken",
+    pattern: /\bxox[abposr]-[A-Za-z0-9-]{10,}\b/,
+    severity: "error"
+  }),
   patternDetector({
     id: "value.githubToken",
     pattern: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{20,}\b/,
@@ -24264,8 +24436,9 @@ function parseEvidenceFormat(value) {
     `Unsupported --evidence-format "${value}". Use directory, html, or zip.`
   );
 }
-function toAssessmentStatus(failed) {
-  return failed ? "UNSAFE" : "SAFE";
+function residualStatusToEvidenceSafe(status) {
+  if (status === "SAFE_WITH_WARNINGS") return "SAFE WITH WARNINGS";
+  return status;
 }
 function stableJson2(value) {
   return `${JSON.stringify(value, null, 2)}
@@ -24305,7 +24478,10 @@ async function writeLocalEvidence(input3) {
     combined += redacted.content.endsWith("\n") ? redacted.content : `${redacted.content}
 `;
   }
-  const assessmentStatus = toAssessmentStatus(input3.failed);
+  const residual = await assessResidualFromContent(combined, {
+    format: "agent-inspect-jsonl"
+  });
+  const assessmentStatus = residualStatusToEvidenceSafe(residual.status);
   const evidencePackage = buildEvidenceCiPackage({
     generatorVersion: version,
     runIds: input3.runIds,