agent-inspect 6.25.0 → 6.26.0

package/packages/core/dist/checks.cjs

@@ -1500,6 +1500,99 @@ function eventEvidence(event) {
     status: event.status
   };
 }
+function eventTime(event) {
+  return event.startedAt ?? event.timestamp ?? "";
+}
+function sortByTime(events) {
+  return [...events].sort((a, b) => eventTime(a).localeCompare(eventTime(b)));
+}
+function attemptNumberOf(event) {
+  const workflow = workflowFor(event);
+  const value = workflow.attemptNumber ?? workflow.attempt;
+  return typeof value === "number" && Number.isFinite(value) ? value : void 0;
+}
+function attemptIdOf(event) {
+  const value = workflowFor(event).attemptId;
+  return typeof value === "string" && value.trim() !== "" ? value : void 0;
+}
+function countOperationAttempts(members) {
+  const attemptIds = new Set(
+    members.map((event) => attemptIdOf(event)).filter((value) => value !== void 0)
+  );
+  if (attemptIds.size > 0) {
+    return attemptIds.size;
+  }
+  const numbers = members.map((event) => attemptNumberOf(event)).filter((value) => value !== void 0).sort((a, b) => a - b);
+  if (numbers.length > 0) {
+    const unique = [...new Set(numbers)];
+    const min = unique[0];
+    const max = unique[unique.length - 1];
+    const contiguous = unique.length === max - min + 1 && unique.every((value, index) => value === min + index);
+    if (contiguous) {
+      return unique.length;
+    }
+    return unique.length;
+  }
+  return members.filter((event) => event.kind === "TOOL" || event.kind === "LLM").length;
+}
+function classifyRetryAttempt(event, priorOrdered, allMembers) {
+  const workflow = workflowFor(event);
+  const attemptNumber = attemptNumberOf(event);
+  const attemptId = attemptIdOf(event);
+  const retryOf = workflow.retryOf;
+  if (typeof retryOf === "string" && retryOf.trim() !== "") {
+    const target = allMembers.find(
+      (candidate) => candidate.eventId === retryOf || attemptIdOf(candidate) === retryOf || workflowFor(candidate).operationId === retryOf
+    );
+    if (!target) {
+      return { kind: "unknown", reason: "retryOf target missing" };
+    }
+    if (eventTime(target) > eventTime(event)) {
+      return { kind: "unknown", reason: "retryOf target does not precede retry" };
+    }
+    return { kind: "retry" };
+  }
+  if (attemptNumber !== void 0) {
+    if (attemptNumber > 1) {
+      return { kind: "retry" };
+    }
+    if (attemptNumber === 1) {
+      return { kind: "first" };
+    }
+  }
+  if (attemptId !== void 0) {
+    const priorDistinct = priorOrdered.some((candidate) => {
+      const priorId = attemptIdOf(candidate);
+      return priorId !== void 0 && priorId !== attemptId;
+    });
+    if (priorDistinct) {
+      return { kind: "retry" };
+    }
+    if (priorOrdered.length === 0) {
+      return { kind: "first" };
+    }
+  }
+  const priorFinished = priorOrdered.filter(
+    (candidate) => candidate.status === "ok" || candidate.status === "error"
+  );
+  if (priorFinished.length > 0) {
+    return { kind: "retry" };
+  }
+  if (priorOrdered.length === 0) {
+    return { kind: "first" };
+  }
+  return { kind: "unknown", reason: "ambiguous attempt identity" };
+}
+function hasDuplicateAttemptIds(members) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const event of members) {
+    const id = attemptIdOf(event);
+    if (!id) continue;
+    if (seen.has(id)) return true;
+    seen.add(id);
+  }
+  return false;
+}
 function evaluateRetrySafetyRules(events, rules, _runEvidence) {
   const findings = [];
   const byOperation = /* @__PURE__ */ new Map();
@@ -1513,11 +1606,18 @@ function evaluateRetrySafetyRules(events, rules, _runEvidence) {
   }
   if (rules.maxAttempts !== void 0) {
     for (const [operationId, members] of byOperation) {
-      const attemptIds = new Set(
-        members.map((event) => workflowFor(event).attemptId).filter((value) => typeof value === "string" && value.trim() !== "")
-      );
-      const attemptNumbers = members.map((event) => workflowFor(event).attemptNumber ?? workflowFor(event).attempt).filter((value) => typeof value === "number" && Number.isFinite(value));
-      const count = attemptIds.size > 0 ? attemptIds.size : attemptNumbers.length > 0 ? Math.max(...attemptNumbers) : members.filter((event) => event.kind === "TOOL" || event.kind === "LLM").length;
+      if (hasDuplicateAttemptIds(members)) {
+        findings.push(
+          fail2(
+            "contract.retry.duplicate-attempt-id",
+            `Operation ${operationId} has duplicate attemptId values.`,
+            members.slice(0, 4).map(eventEvidence),
+            "unique attemptId",
+            "duplicate"
+          )
+        );
+      }
+      const count = countOperationAttempts(members);
       if (count > rules.maxAttempts) {
         findings.push(
           fail2(
@@ -1552,18 +1652,18 @@ function evaluateRetrySafetyRules(events, rules, _runEvidence) {
       const workflow = workflowFor(event);
       const fallbackOf = workflow.fallbackOf;
       if (!fallbackOf) continue;
-      const prior = byOperation.get(fallbackOf) ?? events.filter((candidate) => {
-        const meta = workflowFor(candidate);
-        return meta.operationId === fallbackOf || candidate.runId === fallbackOf;
-      });
-      const priorFailure = prior.some((candidate) => candidate.status === "error");
-      if (!priorFailure) {
+      const prior = byOperation.get(fallbackOf) ?? [];
+      const fallbackStart = eventTime(event);
+      const earlierFailure = prior.some(
+        (candidate) => candidate.status === "error" && eventTime(candidate) !== "" && fallbackStart !== "" && eventTime(candidate) < fallbackStart
+      );
+      if (!earlierFailure) {
         findings.push(
           fail2(
             "contract.retry.fallback-after-failure",
-            `Fallback for ${fallbackOf} appeared without a prior failure.`,
+            `Fallback for ${fallbackOf} appeared without an earlier failure in the related operation.`,
             [eventEvidence(event)],
-            "prior error attempt",
+            "earlier error attempt",
             { fallbackOf }
           )
         );
@@ -1585,39 +1685,46 @@ function evaluateRetrySafetyRules(events, rules, _runEvidence) {
       byToolOp.set(key, list);
     }
     for (const [, members] of byToolOp) {
-      const ordered = [...members].sort((a, b) => {
-        const aTime = a.startedAt ?? a.timestamp ?? "";
-        const bTime = b.startedAt ?? b.timestamp ?? "";
-        return aTime.localeCompare(bTime);
-      });
+      const ordered = sortByTime(members);
       let sawOk = false;
       let sawSideEffectOk = false;
+      const prior = [];
       for (const event of ordered) {
         const name = resolveCanonicalToolName(event);
-        const isRetry = sawOk;
-        if (isRetry) {
-          if (rules.requireIdempotencyEvidenceForRetry && !hasIdempotencyEvidence(event)) {
-            findings.push(
-              fail2(
-                "contract.retry.idempotency-evidence",
-                `Retry of tool ${name} lacks idempotencyKey / noSideEffect evidence.`,
-                [eventEvidence(event)],
-                "idempotencyKey|noSideEffect",
-                { code: "AI_CHECK_RETRY_EVIDENCE_UNAVAILABLE" }
-              )
-            );
-          }
-          if (nonIdempotent.has(name) && sawSideEffectOk && !hasIdempotencyEvidence(event)) {
-            findings.push(
-              fail2(
-                "contract.retry.non-idempotent-side-effect",
-                `Retry of non-idempotent tool ${name} after a confirmed ok side effect.`,
-                [eventEvidence(event)],
-                "no retry after side effect",
-                name
-              )
-            );
-          }
+        const classification = classifyRetryAttempt(event, prior, members);
+        if (classification.kind === "unknown" && rules.requireIdempotencyEvidenceForRetry) {
+          findings.push(
+            fail2(
+              "contract.retry.ambiguous-identity",
+              `Ambiguous attempt identity for tool ${name}; strict retry evidence cannot pass.`,
+              [eventEvidence(event)],
+              "explicit attempt identity",
+              classification.reason
+            )
+          );
+        }
+        const isRetry = classification.kind === "retry";
+        if (isRetry && rules.requireIdempotencyEvidenceForRetry && !hasIdempotencyEvidence(event)) {
+          findings.push(
+            fail2(
+              "contract.retry.idempotency-evidence",
+              `Retry of tool ${name} lacks idempotencyKey / noSideEffect evidence.`,
+              [eventEvidence(event)],
+              "idempotencyKey|noSideEffect",
+              { code: "AI_CHECK_RETRY_EVIDENCE_UNAVAILABLE" }
+            )
+          );
+        }
+        if (sawOk && nonIdempotent.has(name) && sawSideEffectOk && !hasIdempotencyEvidence(event)) {
+          findings.push(
+            fail2(
+              "contract.retry.non-idempotent-side-effect",
+              `Retry of non-idempotent tool ${name} after a confirmed ok side effect.`,
+              [eventEvidence(event)],
+              "no retry after side effect",
+              name
+            )
+          );
         }
         if (event.status === "ok") {
           sawOk = true;
@@ -1625,25 +1732,43 @@ function evaluateRetrySafetyRules(events, rules, _runEvidence) {
             sawSideEffectOk = true;
           }
         }
+        prior.push(event);
       }
     }
   }
   if (rules.requireRecoveredFailureVisible) {
     for (const [operationId, members] of byOperation) {
-      const hasOk = members.some((event) => event.status === "ok");
-      const hasError = members.some((event) => event.status === "error");
-      const attemptish = members.some((event) => {
+      const ordered = sortByTime(members);
+      const toolish = ordered.filter(
+        (event) => (event.kind === "TOOL" || event.kind === "LLM") && (event.status === "ok" || event.status === "error")
+      );
+      if (toolish.length < 2) continue;
+      let sawEarlierError = false;
+      let sawLaterOkAfterError = false;
+      for (const event of toolish) {
+        if (event.status === "error") {
+          sawEarlierError = true;
+        } else if (event.status === "ok" && sawEarlierError) {
+          sawLaterOkAfterError = true;
+        }
+      }
+      const hasOk = toolish.some((event) => event.status === "ok");
+      const attemptish = toolish.some((event) => {
         const meta = workflowFor(event);
-        return meta.attemptId !== void 0 || meta.attemptNumber !== void 0 || meta.attempt !== void 0;
-      }) || members.length > 1;
-      if (hasOk && attemptish && !hasError) {
+        return meta.attemptId !== void 0 || meta.attemptNumber !== void 0 || meta.attempt !== void 0 || meta.retryOf !== void 0;
+      }) || toolish.length > 1;
+      if (!attemptish || !hasOk) continue;
+      if (!sawLaterOkAfterError) {
         findings.push(
           fail2(
             "contract.retry.recovered-failure-visible",
-            `Operation ${operationId} recovered without retaining a visible failure attempt.`,
+            `Operation ${operationId} does not show an earlier error attempt before a later success in the same chain.`,
             members.slice(0, 4).map(eventEvidence),
-            "error attempt retained",
-            { hasOk, hasError }
+            "earlier error then later ok",
+            {
+              sawEarlierError,
+              statuses: toolish.map((event) => event.status)
+            }
           )
         );
       }