agent-inspect 6.18.0 → 6.19.0

package/packages/core/dist/checks.cjs

@@ -488,6 +488,325 @@ function pickString(record, keys) {
   return void 0;
 }
 
+// packages/core/src/checks/derived-failure.ts
+function isRecord2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function pickString2(record, keys) {
+  if (!record) return void 0;
+  for (const key of keys) {
+    const value = record[key];
+    if (typeof value === "string" && value.trim() !== "") return value;
+  }
+  return void 0;
+}
+function pickNumber(record, keys) {
+  if (!record) return void 0;
+  for (const key of keys) {
+    const value = record[key];
+    if (typeof value === "number" && Number.isFinite(value) && value > 0) {
+      return value;
+    }
+  }
+  return void 0;
+}
+function eventMetadata(event) {
+  const attrs = isRecord2(event.attributes) ? event.attributes : void 0;
+  const nested = attrs !== void 0 && isRecord2(attrs.metadata) ? attrs.metadata : void 0;
+  return {
+    ...attrs ?? {},
+    ...nested ?? {}
+  };
+}
+function canonicalName(event) {
+  if (event.kind === "TOOL") return resolveCanonicalToolName(event);
+  return event.name;
+}
+function linkKeys(event) {
+  const meta = eventMetadata(event);
+  const keys = [];
+  for (const key of ["linkedStepId", "toolCallId", "mcpToolCallId"]) {
+    const value = pickString2(meta, [key]);
+    if (value !== void 0) keys.push(`${key}:${value}`);
+  }
+  const stepId = pickString2(meta, ["stepId"]);
+  if (stepId !== void 0) keys.push(`stepId:${stepId}`);
+  return keys;
+}
+function buildRunContexts(logicalEvents) {
+  const byRun = /* @__PURE__ */ new Map();
+  for (const event of logicalEvents) {
+    const existing = byRun.get(event.runId) ?? {
+      runId: event.runId,
+      name: event.name
+    };
+    const meta = eventMetadata(event);
+    if (event.kind === "RUN" || existing.name === event.runId) {
+      existing.name = event.name || existing.name;
+    }
+    if (event.kind === "RUN" && event.status !== void 0 && event.status !== "running") {
+      existing.status = event.status;
+    }
+    existing.retryOf ??= pickString2(meta, ["retryOf"]);
+    existing.attempt ??= pickNumber(meta, ["attempt", "retryAttempt", "retryCount"]);
+    existing.sessionId ??= pickString2(meta, ["sessionId", "conversationId"]);
+    existing.groupId ??= pickString2(meta, ["groupId"]);
+    existing.parentGroupId ??= pickString2(meta, ["parentGroupId"]);
+    existing.fallbackOf ??= pickString2(meta, ["fallbackOf", "fallbackFrom"]);
+    byRun.set(event.runId, existing);
+  }
+  return byRun;
+}
+function sameCorrelationScope(a, b) {
+  if (a.sessionId && b.sessionId && a.sessionId === b.sessionId) return true;
+  if (a.groupId && b.groupId && a.groupId === b.groupId) return true;
+  if (a.parentGroupId && b.parentGroupId && a.parentGroupId === b.parentGroupId) {
+    return true;
+  }
+  return false;
+}
+function isSuccessful(event) {
+  return event.status === "ok";
+}
+function isFailure(event) {
+  return event.status === "error";
+}
+function compareEventOrder(a, b) {
+  const byTime = a.timestamp.localeCompare(b.timestamp);
+  if (byTime !== 0) return byTime;
+  return a.eventId.localeCompare(b.eventId);
+}
+function collectCandidates(failure, logicalEvents, runs) {
+  const failureRun = runs.get(failure.runId);
+  const failureLinks = new Set(linkKeys(failure));
+  const failureName = canonicalName(failure);
+  const failureAttempt = pickNumber(eventMetadata(failure), ["attempt", "retryAttempt", "retryCount"]) ?? failureRun?.attempt;
+  const candidates = [];
+  for (const event of logicalEvents) {
+    if (event.eventId === failure.eventId) continue;
+    if (event.status === "running") continue;
+    const eventRun = runs.get(event.runId);
+    const eventMeta = eventMetadata(event);
+    const sameRun = event.runId === failure.runId;
+    if (eventRun?.retryOf === failure.runId) {
+      candidates.push({
+        event,
+        basis: "retryOf",
+        confidence: "explicit",
+        viaRunId: event.runId
+      });
+      continue;
+    }
+    if (eventRun?.fallbackOf === failure.runId || pickString2(eventMeta, ["fallbackOf", "fallbackFrom"]) === failure.runId) {
+      candidates.push({
+        event,
+        basis: "fallbackOf",
+        confidence: "explicit",
+        viaRunId: event.runId
+      });
+      continue;
+    }
+    if (sameRun && compareEventOrder(failure, event) >= 0) continue;
+    const eventLinks = linkKeys(event);
+    const sharedLink = eventLinks.find((key) => failureLinks.has(key));
+    if (sharedLink !== void 0) {
+      candidates.push({
+        event,
+        basis: sharedLink.split(":")[0] ?? "linkedId",
+        confidence: "explicit"
+      });
+      continue;
+    }
+    const eventAttempt = pickNumber(eventMeta, ["attempt", "retryAttempt", "retryCount"]) ?? eventRun?.attempt;
+    const sameParent = failure.parentId !== void 0 && event.parentId !== void 0 && failure.parentId === event.parentId;
+    const differentParent = failure.parentId !== void 0 && event.parentId !== void 0 && failure.parentId !== event.parentId;
+    const sessionScoped = failureRun !== void 0 && eventRun !== void 0 && sameCorrelationScope(failureRun, eventRun);
+    if (canonicalName(event) === failureName && failureAttempt !== void 0 && eventAttempt !== void 0 && eventAttempt > failureAttempt && !differentParent && (sameParent || sessionScoped || sameRun)) {
+      candidates.push({
+        event,
+        basis: "attempt-progression",
+        confidence: "correlated",
+        ...event.runId !== failure.runId ? { viaRunId: event.runId } : {}
+      });
+    }
+  }
+  const byId = /* @__PURE__ */ new Map();
+  for (const candidate of candidates) {
+    const prev = byId.get(candidate.event.eventId);
+    if (!prev || prev.confidence !== "explicit" && candidate.confidence === "explicit") {
+      byId.set(candidate.event.eventId, candidate);
+    }
+  }
+  return [...byId.values()].sort((a, b) => compareEventOrder(a.event, b.event));
+}
+function classifyFailure(failure, candidates, runs, logicalEvents) {
+  const successful = candidates.filter((c) => isSuccessful(c.event));
+  const unsuccessful = candidates.filter((c) => !isSuccessful(c.event));
+  const retryRunIds = Object.freeze(
+    [...new Set(candidates.map((c) => c.viaRunId).filter((id) => id !== void 0))].sort(
+      (a, b) => a.localeCompare(b)
+    )
+  );
+  if (successful.length > 1) {
+    const distinctRuns = new Set(successful.map((c) => c.event.runId));
+    const distinctParents = new Set(
+      successful.map((c) => c.event.parentId ?? "").filter((id) => id !== "")
+    );
+    if (distinctRuns.size > 1 || distinctParents.size > 1) {
+      return {
+        eventId: failure.eventId,
+        runId: failure.runId,
+        name: failure.name,
+        kind: failure.kind,
+        role: "unknown",
+        confidence: "unknown",
+        basis: Object.freeze(["ambiguous-recovery-candidates"]),
+        recoveryEventIds: Object.freeze(
+          successful.map((c) => c.event.eventId).sort((a, b) => a.localeCompare(b))
+        ),
+        retryRunIds
+      };
+    }
+  }
+  if (successful.length >= 1) {
+    const best = successful[0];
+    return {
+      eventId: failure.eventId,
+      runId: failure.runId,
+      name: failure.name,
+      kind: failure.kind,
+      role: "recovered",
+      confidence: best.confidence,
+      basis: Object.freeze([best.basis]),
+      recoveryEventIds: Object.freeze(
+        successful.map((c) => c.event.eventId).sort((a, b) => a.localeCompare(b))
+      ),
+      retryRunIds
+    };
+  }
+  if (candidates.length > 0) {
+    const best = candidates[0];
+    return {
+      eventId: failure.eventId,
+      runId: failure.runId,
+      name: failure.name,
+      kind: failure.kind,
+      role: "transient",
+      confidence: best.confidence,
+      basis: Object.freeze([
+        best.basis,
+        unsuccessful.some((c) => c.event.status === void 0) ? "retry-incomplete" : "retry-without-success"
+      ]),
+      recoveryEventIds: Object.freeze([]),
+      retryRunIds
+    };
+  }
+  const failureMeta = eventMetadata(failure);
+  const declaredSuccessor = pickString2(failureMeta, [
+    "retriedBy",
+    "nextRetryRunId",
+    "retryRunId"
+  ]);
+  if (declaredSuccessor !== void 0 && !runs.has(declaredSuccessor)) {
+    return {
+      eventId: failure.eventId,
+      runId: failure.runId,
+      name: failure.name,
+      kind: failure.kind,
+      role: "transient",
+      confidence: "explicit",
+      basis: Object.freeze(["retry-declared", "retry-run-missing"]),
+      recoveryEventIds: Object.freeze([]),
+      retryRunIds: Object.freeze([declaredSuccessor])
+    };
+  }
+  for (const run of runs.values()) {
+    if (run.retryOf === failure.runId) {
+      return {
+        eventId: failure.eventId,
+        runId: failure.runId,
+        name: failure.name,
+        kind: failure.kind,
+        role: "transient",
+        confidence: "explicit",
+        basis: Object.freeze(["retryOf", "retry-run-missing-or-empty"]),
+        recoveryEventIds: Object.freeze([]),
+        retryRunIds: Object.freeze([run.runId])
+      };
+    }
+  }
+  const failureRun = runs.get(failure.runId);
+  const hasSuccessorDeclared = [...runs.values()].some((run) => run.retryOf === failure.runId);
+  const isFinalInChain = failureRun !== void 0 && !hasSuccessorDeclared && (failureRun.retryOf !== void 0 || failureRun.attempt !== void 0 && failureRun.attempt > 1 || pickNumber(eventMetadata(failure), ["attempt"]) !== void 0);
+  if (isFinalInChain && failureRun?.status === "error" && !logicalEvents.some(
+    (event) => event.runId === failure.runId && event.eventId !== failure.eventId && isSuccessful(event) && canonicalName(event) === canonicalName(failure)
+  )) {
+    return {
+      eventId: failure.eventId,
+      runId: failure.runId,
+      name: failure.name,
+      kind: failure.kind,
+      role: "terminal",
+      confidence: failureRun.retryOf !== void 0 ? "explicit" : "correlated",
+      basis: Object.freeze(["final-retry-chain-member", "enclosing-run-error"]),
+      recoveryEventIds: Object.freeze([]),
+      retryRunIds: Object.freeze(
+        failureRun.retryOf !== void 0 ? [failureRun.retryOf] : []
+      )
+    };
+  }
+  return {
+    eventId: failure.eventId,
+    runId: failure.runId,
+    name: failure.name,
+    kind: failure.kind,
+    role: "unknown",
+    confidence: "unknown",
+    basis: Object.freeze(["no-explicit-or-correlated-recovery"]),
+    recoveryEventIds: Object.freeze([]),
+    retryRunIds: Object.freeze([])
+  };
+}
+function deriveFailureFacts(logicalEvents) {
+  const runs = buildRunContexts(logicalEvents);
+  const failures = logicalEvents.filter((event) => isFailure(event)).sort(compareEventOrder);
+  const failureFacts = failures.map(
+    (failure) => classifyFailure(failure, collectCandidates(failure, logicalEvents, runs), runs, logicalEvents)
+  );
+  const byRole = /* @__PURE__ */ new Map([
+    ["transient", []],
+    ["recovered", []],
+    ["terminal", []],
+    ["unknown", []]
+  ]);
+  for (const fact of failureFacts) {
+    byRole.get(fact.role).push(fact);
+  }
+  for (const [role, list] of byRole) {
+    byRole.set(
+      role,
+      Object.freeze(
+        [...list].sort((a, b) => {
+          const byRun = a.runId.localeCompare(b.runId);
+          if (byRun !== 0) return byRun;
+          return a.eventId.localeCompare(b.eventId);
+        })
+      )
+    );
+  }
+  const failureRoleCounts = {
+    transient: byRole.get("transient").length,
+    recovered: byRole.get("recovered").length,
+    terminal: byRole.get("terminal").length,
+    unknown: byRole.get("unknown").length
+  };
+  return {
+    failureFacts: Object.freeze(failureFacts),
+    failuresByRole: byRole,
+    failureRoleCounts
+  };
+}
+
 // packages/core/src/checks/trace-facts.ts
 function summarizeSemanticParity(events) {
   const projection = projectLogicalEvents(events);
@@ -498,6 +817,7 @@ function summarizeSemanticParity(events) {
   const finishedToolNames = Object.freeze(
     finishedTools.map((event) => resolveCanonicalToolName(event)).sort((a, b) => a.localeCompare(b))
   );
+  const derived = deriveFailureFacts(logical);
   return {
     rawEventCount: events.length,
     logicalEventCount: logical.length,
@@ -508,7 +828,8 @@ function summarizeSemanticParity(events) {
     parentRemapCount: projection.diagnostics.filter(
       (item) => item.code === "AI_LOGICAL_PARENT_REMAPPED"
     ).length,
-    diagnostics: projection.diagnostics
+    diagnostics: projection.diagnostics,
+    failureRoleCounts: derived.failureRoleCounts
   };
 }
 var TRACE_FACTS_INPUT_NOT_NORMALIZED = formatProgrammaticDiagnostic(
@@ -571,6 +892,8 @@ function buildTraceFacts(input) {
   for (const [name, list] of [...toolsByName.entries()]) {
     toolsByName.set(name, Object.freeze([...list]));
   }
+  const derived = deriveFailureFacts(projection.logicalEvents);
+  const summary = summarizeSemanticParity(events);
   return {
     rawEvents: Object.freeze([...events]),
     logicalEvents: projection.logicalEvents,
@@ -578,7 +901,12 @@ function buildTraceFacts(input) {
     toolsByName,
     llmEvents: Object.freeze(llmEvents),
     outcomeEvents: Object.freeze(outcomeEvents),
-    summary: summarizeSemanticParity(events)
+    summary: {
+      ...summary,
+      failureRoleCounts: derived.failureRoleCounts
+    },
+    failureFacts: derived.failureFacts,
+    failuresByRole: derived.failuresByRole
   };
 }
 
@@ -1168,7 +1496,7 @@ function firstIndexByName(events) {
   }
   return index;
 }
-function isRecord2(value) {
+function isRecord3(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 function eventMap(events) {
@@ -1222,7 +1550,7 @@ function pushValueEntries(entries, event, value, path3, key, depth = 0) {
     }
     return;
   }
-  if (!isRecord2(value)) return;
+  if (!isRecord3(value)) return;
   for (const nestedKey of Object.keys(value).sort((a, b) => a.localeCompare(b))) {
     pushValueEntries(
       entries,
@@ -2317,7 +2645,7 @@ function createSafetyOversizedAttributeRule(options) {
               )
             );
           }
-          if (isRecord2(entry.value) && options.maxObjectKeys !== void 0 && Object.keys(entry.value).length > options.maxObjectKeys) {
+          if (isRecord3(entry.value) && options.maxObjectKeys !== void 0 && Object.keys(entry.value).length > options.maxObjectKeys) {
             findings.push(
               failFinding(
                 "safety.oversizedAttribute",
@@ -2636,6 +2964,7 @@ exports.createToolFailureRule = createToolFailureRule;
 exports.createToolOrderingRule = createToolOrderingRule;
 exports.createToolUsageRule = createToolUsageRule;
 exports.defineTraceContract = defineTraceContract;
+exports.deriveFailureFacts = deriveFailureFacts;
 exports.evaluateTraceContract = evaluateTraceContract;
 exports.evaluateTraceContractRead = evaluateTraceContractRead;
 exports.formatProgrammaticDiagnostic = formatProgrammaticDiagnostic;