agent-workflow-inspector 1.0.0 → 2.0.0

package/dist/index.js

@@ -269,8 +269,8 @@ function otel(value) {
         if (++total > LIMITS.spans)
           throw new AwiError("Maximum span count exceeded");
         const s = record(raw);
-        const traceId = string(s.traceId), id = string(s.spanId);
-        if (!traceId || !/^[a-fA-F0-9]{32}$/.test(traceId) || /^0+$/.test(traceId) || !id || !/^[a-fA-F0-9]{16}$/.test(id) || /^0+$/.test(id))
+        const traceId2 = string(s.traceId), id = string(s.spanId);
+        if (!traceId2 || !/^[a-fA-F0-9]{32}$/.test(traceId2) || /^0+$/.test(traceId2) || !id || !/^[a-fA-F0-9]{16}$/.test(id) || /^0+$/.test(id))
           throw new AwiError("Invalid OTLP trace/span identity");
         const a = attributes(s.attributes);
         const kind = textAttribute(s.attributes, "gen_ai.operation.name");
@@ -345,7 +345,7 @@ function otel(value) {
           source: { format: "otel", index: total - 1 }
         };
         if (args !== void 0) validateValue(args);
-        const key = traceId.toLowerCase();
+        const key = traceId2.toLowerCase();
         if (!grouped.has(key))
           grouped.set(key, { version: 1, traceId: key, operations: [] });
         grouped.get(key).operations.push(op);
@@ -426,14 +426,20 @@ var configSchema = z2.strictObject({
       maxOperations: positive.max(5e4).default(100),
       maxDepth: positive.max(1e3).default(20)
     }).default({ enabled: true, maxOperations: 100, maxDepth: 20 }),
-    AWI006: z2.strictObject({ enabled: z2.boolean().default(true) }).default({ enabled: true })
+    AWI006: z2.strictObject({ enabled: z2.boolean().default(true) }).default({ enabled: true }),
+    AWI007: z2.strictObject({
+      enabled: z2.boolean().default(false),
+      threshold: positive.min(4).max(LIMITS.repetitionWindow).default(4),
+      minDistinctArguments: positive.min(3).max(LIMITS.repetitionWindow).default(3)
+    }).default({ enabled: false, threshold: 4, minDistinctArguments: 3 })
   }).default({
     AWI001: { enabled: true, threshold: 4 },
     AWI002: { enabled: true, threshold: 3 },
     AWI003: { enabled: true, minRepetitions: 3, maxCycleLength: 6 },
     AWI004: { enabled: true, threshold: 5, windowSeconds: 30 },
     AWI005: { enabled: true, maxOperations: 100, maxDepth: 20 },
-    AWI006: { enabled: true }
+    AWI006: { enabled: true },
+    AWI007: { enabled: false, threshold: 4, minDistinctArguments: 3 }
   }),
   fingerprint: z2.strictObject({
     ignorePaths: z2.array(
@@ -616,7 +622,8 @@ var titles = {
   AWI003: "Repeating execution cycle",
   AWI004: "Retry storm",
   AWI005: "Excessive execution steps or depth",
-  AWI006: "Recorded resource usage associated with suspicious execution"
+  AWI006: "Recorded resource usage associated with suspicious execution",
+  AWI007: "Repeated failure with argument drift"
 };
 function analyze(traces, configInput = {}, showContent = false) {
   const config = resolveConfig(configInput);
@@ -666,7 +673,7 @@ function analyze(traces, configInput = {}, showContent = false) {
     validated.push(result.data);
   }
   const report = {
-    version: 1,
+    version: 2,
     fingerprintVersion: "awi-sha256-v1",
     complete: true,
     summaries: [],
@@ -680,20 +687,20 @@ function analyze(traces, configInput = {}, showContent = false) {
     else grouped.set(t.traceId, { ...t, operations: [...t.operations] });
   }
   let capped = false;
-  function add(ruleId, traceId, ops, evidence, explanation, confidence = "moderate") {
+  function add(ruleId, traceId2, ops, evidence, explanation, confidence = "moderate") {
     if (report.findings.length >= LIMITS.findings) {
       capped = true;
       return;
     }
     report.findings.push({
       ruleId,
-      traceId,
+      traceId: traceId2,
       severity: ruleId === "AWI002" || ruleId === "AWI004" ? "high" : "medium",
       confidence,
       title: titles[ruleId],
       description: "Heuristic finding based on recorded execution evidence; intentional behavior may match.",
       explanation,
-      remediation: "Review the referenced operations and configured policy before changing the workflow.",
+      remediation: ruleId === "AWI007" ? "Review the shared error code and retry policy; consider bounded retries or escalation before varying arguments further." : "Review the referenced operations and configured policy before changing the workflow.",
       operationIds: ops.map((o) => o.id),
       evidence: {
         ...evidence,
@@ -704,9 +711,10 @@ function analyze(traces, configInput = {}, showContent = false) {
       }
     });
   }
-  for (const trace of [...grouped.values()].sort(
-    (a, b) => a.traceId < b.traceId ? -1 : a.traceId > b.traceId ? 1 : 0
-  )) {
+  const orderedTraces = [...grouped.values()].sort(
+    (a, b) => compare(a.traceId, b.traceId)
+  );
+  for (const trace of orderedTraces) {
     trace.operations.sort((a, b) => compare(a.id, b.id));
     const normalized = normalize(trace);
     report.warnings.push(...normalized.warnings);
@@ -773,6 +781,43 @@ function analyze(traces, configInput = {}, showContent = false) {
           "Overlapping sibling operations excluded from sequence rules."
         );
       for (const run of runs) {
+        if (config.rules.AWI007.enabled) {
+          const policy2 = config.rules.AWI007;
+          let failures = [];
+          const distinct = /* @__PURE__ */ new Set();
+          let tool;
+          let code;
+          const flush = () => {
+            if (failures.length >= policy2.threshold && distinct.size >= policy2.minDistinctArguments)
+              add(
+                "AWI007",
+                trace.traceId,
+                failures,
+                {
+                  failedCalls: failures.length,
+                  distinctArgumentFingerprints: distinct.size
+                },
+                "Consecutive recorded failures share a tool and explicit error code while argument fingerprints vary. This does not establish intent or lack of progress."
+              );
+            failures = [];
+            distinct.clear();
+          };
+          for (const op of run) {
+            if (op.type !== "tool") continue;
+            const fp = fps.get(op.id);
+            if (op.status !== "error" || !op.error?.code?.trim() || op.arguments === void 0 || !fp) {
+              flush();
+              continue;
+            }
+            const name = op.toolName ?? op.name;
+            if (name !== tool || op.error.code !== code) flush();
+            tool = name;
+            code = op.error.code;
+            failures.push(op);
+            distinct.add(fp);
+          }
+          flush();
+        }
         let segment = [];
         const segments = [];
         for (const op of run) {
@@ -927,7 +972,12 @@ function analyze(traces, configInput = {}, showContent = false) {
           recorded,
           {
             recordedUsage: tokens,
-            recordedCosts: costs,
+            recordedCosts: Object.fromEntries(
+              Object.keys(costs).sort(compare).map((currency, index) => [
+                `currency-${index + 1}`,
+                costs[currency]
+              ])
+            ),
             coverage: "Only implicated leaf operations with recorded usage or cost",
             relatedRules: [...new Set(related.map((f) => f.ruleId))]
           },
@@ -943,6 +993,28 @@ function analyze(traces, configInput = {}, showContent = false) {
   report.findings.sort(
     (a, b) => severityRank[b.severity] - severityRank[a.severity] || compare(a.traceId, b.traceId) || compare(a.ruleId, b.ruleId) || compare(a.operationIds.join("\0"), b.operationIds.join("\0"))
   );
+  const traceRefs = new Map(
+    orderedTraces.map((trace, index) => [
+      trace.traceId,
+      {
+        id: `trace-${index + 1}`,
+        operations: new Map(
+          trace.operations.map((op, index2) => [op.id, `op-${index2 + 1}`])
+        )
+      }
+    ])
+  );
+  for (const summary of report.summaries)
+    summary.traceId = traceRefs.get(summary.traceId).id;
+  for (const finding of report.findings) {
+    const refs = traceRefs.get(finding.traceId);
+    finding.traceId = refs.id;
+    finding.operationIds = finding.operationIds.map(
+      (id) => refs.operations.get(id)
+    );
+    for (const reference of finding.evidence.references)
+      reference.operationId = refs.operations.get(reference.operationId);
+  }
   report.warnings = [...new Set(report.warnings)].sort();
   try {
     validateValue(report);
@@ -952,8 +1024,9 @@ function analyze(traces, configInput = {}, showContent = false) {
     throw error;
   }
   if (showContent)
-    report.content = [...grouped.values()].sort((a, b) => compare(a.traceId, b.traceId)).map((t) => ({
-      traceId: t.traceId,
+    report.content = orderedTraces.map((t) => ({
+      traceId: traceRefs.get(t.traceId).id,
+      recordedTraceId: t.traceId,
       operations: redact(t.operations)
     }));
   return report;
@@ -981,7 +1054,8 @@ function humanReport(report, options = {}) {
 ${paint("[" + f.severity.toUpperCase() + "]", f.severity)} ${safeText(f.ruleId)} ${safeText(f.title)} (${safeText(f.confidence)} evidence)`,
       `  Trace: ${safeText(f.traceId)}; operations: ${f.operationIds.map(safeText).join(", ") || "trace-wide"}`,
       `  ${safeText(f.explanation)}`,
-      `  Evidence: ${safeText(JSON.stringify(f.evidence))}`
+      `  Evidence: ${safeText(JSON.stringify(f.evidence))}`,
+      `  Remediation: ${safeText(f.remediation)}`
     );
   for (const warning of report.warnings)
     lines.push("Warning: " + safeText(warning));
@@ -1009,13 +1083,15 @@ var usage = z4.strictObject({
   outputTokens: z4.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(),
   totalTokens: z4.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional()
 });
+var traceId = z4.string().regex(/^trace-[1-9][0-9]*$/);
+var operationId = z4.string().regex(/^op-[1-9][0-9]*$/);
 var reportSchema = z4.strictObject({
-  version: z4.literal(1),
+  version: z4.literal(2),
   fingerprintVersion: z4.literal("awi-sha256-v1"),
   complete: z4.boolean(),
   summaries: z4.array(
     z4.strictObject({
-      traceId: z4.string(),
+      traceId,
       operations: z4.number().int().nonnegative(),
       toolCalls: z4.number().int().nonnegative(),
       maxDepth: z4.number().int().nonnegative(),
@@ -1031,7 +1107,8 @@ var reportSchema = z4.strictObject({
         "AWI003",
         "AWI004",
         "AWI005",
-        "AWI006"
+        "AWI006",
+        "AWI007"
       ]),
       severity: z4.enum(["low", "medium", "high"]),
       confidence: z4.enum(["high", "moderate"]),
@@ -1039,15 +1116,26 @@ var reportSchema = z4.strictObject({
       description: z4.string(),
       explanation: z4.string(),
       remediation: z4.string(),
-      traceId: z4.string(),
-      operationIds: z4.array(z4.string()),
-      evidence: z4.record(z4.string(), z4.unknown())
+      traceId,
+      operationIds: z4.array(operationId),
+      evidence: z4.looseObject({
+        references: z4.array(
+          z4.strictObject({
+            operationId,
+            source: z4.strictObject({
+              format: z4.enum(["awi", "otel"]),
+              index: z4.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
+            }).optional()
+          })
+        )
+      })
     })
   ),
   warnings: z4.array(z4.string()),
   content: z4.array(
     z4.strictObject({
-      traceId: z4.string(),
+      traceId,
+      recordedTraceId: z4.string(),
       operations: z4.array(z4.unknown())
     })
   ).optional()