@agent-inspect/viewer 6.31.8 → 6.31.9

package/dist/index.d.cts

@@ -186,7 +186,7 @@ interface TimelineOptions {
 declare function buildRunTimeline(events: readonly TraceEvent[], options?: TimelineOptions): RunTimeline;
 
 type SuiteCaseStatus = "pass" | "fail" | "error" | "skipped";
-type SuiteDiagnosticCode = "AI_SUITE_CONFIG_INVALID" | "AI_SUITE_CONFIG_LOAD_FAILED" | "AI_SUITE_CASE_TRACE_MISSING" | "AI_SUITE_CASE_CHECK_FAILED" | "AI_SUITE_CASE_EVAL_FAILED" | "AI_SUITE_CASE_OBSERVATION_FAILED" | "AI_SUITE_TRACE_UNREADABLE";
+type SuiteDiagnosticCode = "AI_SUITE_CONFIG_INVALID" | "AI_SUITE_CONFIG_LOAD_FAILED" | "AI_SUITE_CASE_TRACE_MISSING" | "AI_SUITE_CASE_CHECK_FAILED" | "AI_SUITE_CASE_EVAL_FAILED" | "AI_SUITE_CASE_OBSERVATION_FAILED" | "AI_SUITE_NO_ASSERTIONS" | "AI_SUITE_UNKNOWN_SELECTOR" | "AI_SUITE_TRACE_UNREADABLE";
 interface SuiteDiagnostic {
     code: SuiteDiagnosticCode;
     severity: "error" | "warning" | "info";

package/dist/index.d.ts

@@ -186,7 +186,7 @@ interface TimelineOptions {
 declare function buildRunTimeline(events: readonly TraceEvent[], options?: TimelineOptions): RunTimeline;
 
 type SuiteCaseStatus = "pass" | "fail" | "error" | "skipped";
-type SuiteDiagnosticCode = "AI_SUITE_CONFIG_INVALID" | "AI_SUITE_CONFIG_LOAD_FAILED" | "AI_SUITE_CASE_TRACE_MISSING" | "AI_SUITE_CASE_CHECK_FAILED" | "AI_SUITE_CASE_EVAL_FAILED" | "AI_SUITE_CASE_OBSERVATION_FAILED" | "AI_SUITE_TRACE_UNREADABLE";
+type SuiteDiagnosticCode = "AI_SUITE_CONFIG_INVALID" | "AI_SUITE_CONFIG_LOAD_FAILED" | "AI_SUITE_CASE_TRACE_MISSING" | "AI_SUITE_CASE_CHECK_FAILED" | "AI_SUITE_CASE_EVAL_FAILED" | "AI_SUITE_CASE_OBSERVATION_FAILED" | "AI_SUITE_NO_ASSERTIONS" | "AI_SUITE_UNKNOWN_SELECTOR" | "AI_SUITE_TRACE_UNREADABLE";
 interface SuiteDiagnostic {
     code: SuiteDiagnosticCode;
     severity: "error" | "warning" | "info";

package/dist/index.mjs

@@ -764,7 +764,18 @@ async function readTraceEventsFromFile(filePath) {
 }
 
 // packages/core/src/context.ts
-new AsyncLocalStorage();
+var GLOBAL_CONTEXT_ALS_KEY = "agent-inspect:execution-context-als:v1";
+function getSharedContextStorage() {
+  const host = globalThis;
+  const existing = host[GLOBAL_CONTEXT_ALS_KEY];
+  if (existing instanceof AsyncLocalStorage) {
+    return existing;
+  }
+  const created = new AsyncLocalStorage();
+  host[GLOBAL_CONTEXT_ALS_KEY] = created;
+  return created;
+}
+getSharedContextStorage();
 
 // packages/core/src/outcomes/types.ts
 var OBSERVED_OUTCOME_METHODS = [
@@ -1367,7 +1378,7 @@ function enrichSessionSummary(summary, runs, options = {}) {
   }
   const lastMs = latestActivityMs(sessionRuns);
   const lastActivity = lastMs > 0 ? new Date(lastMs).toISOString() : (/* @__PURE__ */ new Date(0)).toISOString();
-  const retryCount = summary.retries.filter(
+  const retryCount2 = summary.retries.filter(
     (retry) => retry.retryOf !== void 0 || (retry.attempt ?? 0) > 1
   ).length;
   return {
@@ -1381,7 +1392,7 @@ function enrichSessionSummary(summary, runs, options = {}) {
     workflowId,
     lastError: deriveLastError(sessionRuns),
     lastActivity,
-    retryCount,
+    retryCount: retryCount2,
     observationSummary: deriveObservationSummary(sessionRuns),
     checkSummary: deriveCheckSummary(sessionRuns)
   };
@@ -2254,6 +2265,13 @@ function metadataStringAttr(event, keys) {
   }
   return void 0;
 }
+function numericAttr(event, keys) {
+  for (const key of keys) {
+    const value = event.attributes?.[key];
+    if (typeof value === "number" && Number.isFinite(value)) return value;
+  }
+  return void 0;
+}
 function stripPrefix(name, prefixes) {
   for (const prefix of prefixes) {
     if (name.startsWith(prefix)) return name.slice(prefix.length);
@@ -2309,9 +2327,12 @@ function llmFinishReason(event) {
   const keys = ["finishReason", "rawFinishReason", "finish_reason"];
   return stringAttr(event, keys) ?? metadataStringAttr(event, keys);
 }
+function retryCount(event) {
+  return numericAttr(event, ["retryCount", "retryAttempt", "retry_attempt", "attempt"]);
+}
 function finishedEvents(context, kind) {
   return semanticEvents(context).filter(
-    (event) => (event.kind === kind) && event.status !== "running"
+    (event) => (kind === void 0 || event.kind === kind) && event.status !== "running"
   );
 }
 function toolInvocationEvents(context) {
@@ -2376,6 +2397,35 @@ function createRunDurationRule(options) {
     }
   };
 }
+function createRunDepthRule(options) {
+  return {
+    id: "run.depth",
+    category: "run",
+    defaultSeverity: "error",
+    evaluate(context) {
+      const nodes = [...context.nodesByEventId.values()];
+      const maxDepth = nodes.reduce((max, node) => Math.max(max, node.depth), 0);
+      if (maxDepth <= options.maxDepth) return [];
+      const deepest = nodes.filter((node) => node.depth === maxDepth);
+      return [
+        failFinding(
+          "run.depth",
+          `Run depth ${maxDepth} exceeded ${options.maxDepth}.`,
+          deepest.map((node) => ({
+            runId: node.event.runId,
+            eventId: node.event.eventId,
+            parentId: node.event.parentId,
+            kind: node.event.kind,
+            name: node.event.name,
+            status: node.event.status
+          })),
+          { maxDepth: options.maxDepth },
+          maxDepth
+        )
+      ];
+    }
+  };
+}
 function createToolUsageRule(options) {
   return {
     id: "tool.usage",
@@ -2440,6 +2490,45 @@ function createToolUsageRule(options) {
     }
   };
 }
+function createToolFailureRule(options) {
+  return {
+    id: "tool.failures",
+    category: "tool",
+    defaultSeverity: "error",
+    evaluate(context) {
+      const tools = finishedEvents(context, "TOOL");
+      const failures = tools.filter((event) => event.status === "error");
+      const retries = tools.map((event) => ({ event, count: retryCount(event) })).filter((item) => item.count !== void 0);
+      const findings = [];
+      if (options.maxFailures !== void 0 && failures.length > options.maxFailures) {
+        findings.push(
+          failFinding(
+            "tool.failures",
+            `Tool failure count ${failures.length} exceeded ${options.maxFailures}.`,
+            failures.map((event) => eventEvidence(event)),
+            { maxFailures: options.maxFailures },
+            failures.length
+          )
+        );
+      }
+      if (options.maxRetries !== void 0) {
+        const excessiveRetries = retries.filter((item) => item.count > options.maxRetries);
+        if (excessiveRetries.length > 0) {
+          findings.push(
+            failFinding(
+              "tool.failures",
+              `Tool retry count exceeded ${options.maxRetries}.`,
+              excessiveRetries.map((item) => eventEvidence(item.event, "attributes.retryCount")),
+              { maxRetries: options.maxRetries },
+              excessiveRetries.map((item) => ({ tool: toolName(item.event), retries: item.count }))
+            )
+          );
+        }
+      }
+      return findings;
+    }
+  };
+}
 function createLlmUsageRule(options) {
   return {
     id: "llm.usage",
@@ -5543,21 +5632,48 @@ function openTrace(input, options = {}) {
 }
 
 // packages/core/src/suite/run.ts
+var KNOWN_SUITE_SELECT_IDS = /* @__PURE__ */ new Set([
+  "run.status",
+  "run.duration",
+  "run.depth",
+  "outcome.status",
+  "tool.usage",
+  "tool.failures",
+  "llm.usage"
+]);
 function diagnostic4(code, message, severity = "error", caseId) {
   return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
 }
-function buildCaseRules(suiteCase, config) {
+function buildCaseAssertions(suiteCase, config) {
   const rules = [];
   const select = new Set(config.checks?.select ?? []);
-  if (select.has("run.status")) {
+  const configDiagnostics = [];
+  const evalConfig = config.eval;
+  const declaredSelect = [...select];
+  for (const id of declaredSelect) {
+    if (!KNOWN_SUITE_SELECT_IDS.has(id)) {
+      configDiagnostics.push(
+        diagnostic4(
+          "AI_SUITE_UNKNOWN_SELECTOR",
+          `Unknown or unsupported suite check selector "${id}".`,
+          "error",
+          suiteCase.id
+        )
+      );
+    }
+  }
+  if (select.has("run.status") || evalConfig?.requireSuccess === true) {
     rules.push(createRunStatusRule());
+    select.add("run.status");
   }
   const requiredTools = [
     ...config.checks?.tool?.required ?? [],
+    ...evalConfig?.requiredTools ?? [],
     ...suiteCase.requireTools ?? []
   ];
   const forbiddenTools = [
     ...config.checks?.tool?.forbidden ?? [],
+    ...evalConfig?.forbiddenTools ?? [],
     ...suiteCase.forbidTools ?? []
   ];
   if (requiredTools.length > 0 || forbiddenTools.length > 0) {
@@ -5569,20 +5685,51 @@ function buildCaseRules(suiteCase, config) {
     );
     select.add("tool.usage");
   }
-  const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? config.eval?.maxDurationMs;
+  const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? evalConfig?.maxDurationMs;
   if (maxDurationMs !== void 0) {
     rules.push(createRunDurationRule({ maxDurationMs }));
     select.add("run.duration");
   }
-  const llm = config.checks?.llm;
-  if (llm?.allowedModels !== void 0 || llm?.maxTotalTokens !== void 0) {
-    rules.push(createLlmUsageRule(llm));
+  const maxDepth = config.checks?.run?.maxDepth ?? evalConfig?.maxDepth;
+  if (maxDepth !== void 0) {
+    rules.push(createRunDepthRule({ maxDepth }));
+    select.add("run.depth");
+  }
+  if (evalConfig?.maxRetries !== void 0) {
+    rules.push(createToolFailureRule({ maxRetries: evalConfig.maxRetries }));
+    select.add("tool.failures");
+  }
+  const allowedModels = config.checks?.llm?.allowedModels;
+  const maxTotalTokens = config.checks?.llm?.maxTotalTokens ?? evalConfig?.maxTotalTokens;
+  if (allowedModels !== void 0 || maxTotalTokens !== void 0) {
+    rules.push(
+      createLlmUsageRule({
+        ...allowedModels !== void 0 ? { allowedModels } : {},
+        ...maxTotalTokens !== void 0 ? { maxTotalTokens } : {}
+      })
+    );
     select.add("llm.usage");
   }
   if (select.has("outcome.status")) {
     rules.push(createObservedOutcomeRule({ failOn: ["failed"] }));
   }
-  return { rules, select: [...select] };
+  const observationCount = suiteCase.expectedObservations?.length ?? 0;
+  if (rules.length === 0 && observationCount === 0 && configDiagnostics.length === 0) {
+    configDiagnostics.push(
+      diagnostic4(
+        "AI_SUITE_NO_ASSERTIONS",
+        `Suite case "${suiteCase.id}" declares no effective checks, eval controls, or expected observations.`,
+        "error",
+        suiteCase.id
+      )
+    );
+  }
+  return {
+    rules,
+    select: [...select],
+    observationCount,
+    configDiagnostics
+  };
 }
 function outcomesFromRead(read) {
   const persistedOutcomes = extractOutcomesFromPersistedEvents(
@@ -5663,8 +5810,20 @@ async function runSuiteCase(suiteCase, config, options) {
       ]
     };
   }
-  const { rules, select } = buildCaseRules(suiteCase, config);
-  const checkResult = rules.length > 0 ? runTraceChecks({ read }, { rules, select }) : {
+  const compiled = buildCaseAssertions(suiteCase, config);
+  if (compiled.configDiagnostics.length > 0) {
+    return {
+      id: suiteCase.id,
+      status: "error",
+      tracePath: resolved.tracePath,
+      ...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
+      checkOk: false,
+      ...config.eval?.requireSuccess === true ? { evalOk: false } : {},
+      message: compiled.configDiagnostics.map((item) => item.message).join("; "),
+      diagnostics: compiled.configDiagnostics
+    };
+  }
+  const checkResult = compiled.rules.length > 0 ? runTraceChecks({ read }, { rules: compiled.rules, select: compiled.select }) : {
     ok: true,
     status: "pass",
     format: read.format,
@@ -5692,6 +5851,9 @@ async function runSuiteCase(suiteCase, config, options) {
   ];
   const checkOk = checkResult.ok;
   const observationsOk = observationResult.ok;
+  const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
+    (finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
+  ) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
   const ok = checkOk && observationsOk;
   const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
   return {
@@ -5700,6 +5862,7 @@ async function runSuiteCase(suiteCase, config, options) {
     tracePath: resolved.tracePath,
     ...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
     checkOk,
+    ...evalOk !== void 0 ? { evalOk } : {},
     observationsOk,
     diagnostics,
     ...ok ? {} : {