agent-inspect 6.31.8 → 6.31.9

package/packages/cli/dist/index.cjs

@@ -1252,11 +1252,23 @@ var init_trace_event_safety = __esm({
     init_persisted_inspect_event();
   }
 });
+function getSharedContextStorage() {
+  const host = globalThis;
+  const existing = host[GLOBAL_CONTEXT_ALS_KEY];
+  if (existing instanceof async_hooks.AsyncLocalStorage) {
+    return existing;
+  }
+  const created = new async_hooks.AsyncLocalStorage();
+  host[GLOBAL_CONTEXT_ALS_KEY] = created;
+  return created;
+}
+var GLOBAL_CONTEXT_ALS_KEY;
 var init_context = __esm({
   "packages/core/src/context.ts"() {
     init_correlation_metadata();
     init_trace_event_safety();
-    new async_hooks.AsyncLocalStorage();
+    GLOBAL_CONTEXT_ALS_KEY = "agent-inspect:execution-context-als:v1";
+    getSharedContextStorage();
   }
 });
 var init_inspector_runtime = __esm({
@@ -6524,6 +6536,45 @@ function createToolOrderingRule(options) {
     requireEndpoints: false
   });
 }
+function createToolFailureRule(options) {
+  return {
+    id: "tool.failures",
+    category: "tool",
+    defaultSeverity: "error",
+    evaluate(context) {
+      const tools = finishedEvents(context, "TOOL");
+      const failures = tools.filter((event) => event.status === "error");
+      const retries = tools.map((event) => ({ event, count: retryCount(event) })).filter((item) => item.count !== void 0);
+      const findings = [];
+      if (options.maxFailures !== void 0 && failures.length > options.maxFailures) {
+        findings.push(
+          failFinding(
+            "tool.failures",
+            `Tool failure count ${failures.length} exceeded ${options.maxFailures}.`,
+            failures.map((event) => eventEvidence(event)),
+            { maxFailures: options.maxFailures },
+            failures.length
+          )
+        );
+      }
+      if (options.maxRetries !== void 0) {
+        const excessiveRetries = retries.filter((item) => item.count > options.maxRetries);
+        if (excessiveRetries.length > 0) {
+          findings.push(
+            failFinding(
+              "tool.failures",
+              `Tool retry count exceeded ${options.maxRetries}.`,
+              excessiveRetries.map((item) => eventEvidence(item.event, "attributes.retryCount")),
+              { maxRetries: options.maxRetries },
+              excessiveRetries.map((item) => ({ tool: toolName(item.event), retries: item.count }))
+            )
+          );
+        }
+      }
+      return findings;
+    }
+  };
+}
 function createLlmUsageRule(options) {
   return {
     id: "llm.usage",
@@ -14297,18 +14348,36 @@ var init_readers = __esm({
 function diagnostic5(code, message, severity = "error", caseId) {
   return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
 }
-function buildCaseRules(suiteCase, config) {
+function buildCaseAssertions(suiteCase, config) {
   const rules = [];
   const select = new Set(config.checks?.select ?? []);
-  if (select.has("run.status")) {
+  const configDiagnostics = [];
+  const evalConfig = config.eval;
+  const declaredSelect = [...select];
+  for (const id of declaredSelect) {
+    if (!KNOWN_SUITE_SELECT_IDS.has(id)) {
+      configDiagnostics.push(
+        diagnostic5(
+          "AI_SUITE_UNKNOWN_SELECTOR",
+          `Unknown or unsupported suite check selector "${id}".`,
+          "error",
+          suiteCase.id
+        )
+      );
+    }
+  }
+  if (select.has("run.status") || evalConfig?.requireSuccess === true) {
     rules.push(createRunStatusRule());
+    select.add("run.status");
   }
   const requiredTools = [
     ...config.checks?.tool?.required ?? [],
+    ...evalConfig?.requiredTools ?? [],
     ...suiteCase.requireTools ?? []
   ];
   const forbiddenTools = [
     ...config.checks?.tool?.forbidden ?? [],
+    ...evalConfig?.forbiddenTools ?? [],
     ...suiteCase.forbidTools ?? []
   ];
   if (requiredTools.length > 0 || forbiddenTools.length > 0) {
@@ -14320,20 +14389,51 @@ function buildCaseRules(suiteCase, config) {
     );
     select.add("tool.usage");
   }
-  const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? config.eval?.maxDurationMs;
+  const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? evalConfig?.maxDurationMs;
   if (maxDurationMs !== void 0) {
     rules.push(createRunDurationRule({ maxDurationMs }));
     select.add("run.duration");
   }
-  const llm = config.checks?.llm;
-  if (llm?.allowedModels !== void 0 || llm?.maxTotalTokens !== void 0) {
-    rules.push(createLlmUsageRule(llm));
+  const maxDepth = config.checks?.run?.maxDepth ?? evalConfig?.maxDepth;
+  if (maxDepth !== void 0) {
+    rules.push(createRunDepthRule({ maxDepth }));
+    select.add("run.depth");
+  }
+  if (evalConfig?.maxRetries !== void 0) {
+    rules.push(createToolFailureRule({ maxRetries: evalConfig.maxRetries }));
+    select.add("tool.failures");
+  }
+  const allowedModels = config.checks?.llm?.allowedModels;
+  const maxTotalTokens = config.checks?.llm?.maxTotalTokens ?? evalConfig?.maxTotalTokens;
+  if (allowedModels !== void 0 || maxTotalTokens !== void 0) {
+    rules.push(
+      createLlmUsageRule({
+        ...allowedModels !== void 0 ? { allowedModels } : {},
+        ...maxTotalTokens !== void 0 ? { maxTotalTokens } : {}
+      })
+    );
     select.add("llm.usage");
   }
   if (select.has("outcome.status")) {
     rules.push(createObservedOutcomeRule({ failOn: ["failed"] }));
   }
-  return { rules, select: [...select] };
+  const observationCount = suiteCase.expectedObservations?.length ?? 0;
+  if (rules.length === 0 && observationCount === 0 && configDiagnostics.length === 0) {
+    configDiagnostics.push(
+      diagnostic5(
+        "AI_SUITE_NO_ASSERTIONS",
+        `Suite case "${suiteCase.id}" declares no effective checks, eval controls, or expected observations.`,
+        "error",
+        suiteCase.id
+      )
+    );
+  }
+  return {
+    rules,
+    select: [...select],
+    observationCount,
+    configDiagnostics
+  };
 }
 function outcomesFromRead(read) {
   const persistedOutcomes = extractOutcomesFromPersistedEvents(
@@ -14414,8 +14514,20 @@ async function runSuiteCase(suiteCase, config, options) {
       ]
     };
   }
-  const { rules, select } = buildCaseRules(suiteCase, config);
-  const checkResult = rules.length > 0 ? runTraceChecks({ read }, { rules, select }) : {
+  const compiled = buildCaseAssertions(suiteCase, config);
+  if (compiled.configDiagnostics.length > 0) {
+    return {
+      id: suiteCase.id,
+      status: "error",
+      tracePath: resolved.tracePath,
+      ...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
+      checkOk: false,
+      ...config.eval?.requireSuccess === true ? { evalOk: false } : {},
+      message: compiled.configDiagnostics.map((item) => item.message).join("; "),
+      diagnostics: compiled.configDiagnostics
+    };
+  }
+  const checkResult = compiled.rules.length > 0 ? runTraceChecks({ read }, { rules: compiled.rules, select: compiled.select }) : {
     ok: true,
     status: "pass",
     format: read.format,
@@ -14443,6 +14555,9 @@ async function runSuiteCase(suiteCase, config, options) {
   ];
   const checkOk = checkResult.ok;
   const observationsOk = observationResult.ok;
+  const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
+    (finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
+  ) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
   const ok = checkOk && observationsOk;
   const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
   return {
@@ -14451,6 +14566,7 @@ async function runSuiteCase(suiteCase, config, options) {
     tracePath: resolved.tracePath,
     ...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
     checkOk,
+    ...evalOk !== void 0 ? { evalOk } : {},
     observationsOk,
     diagnostics,
     ...ok ? {} : {
@@ -14495,6 +14611,7 @@ async function runSuite(options = {}) {
     diagnostics
   };
 }
+var KNOWN_SUITE_SELECT_IDS;
 var init_run = __esm({
   "packages/core/src/suite/run.ts"() {
     init_checks2();
@@ -14502,6 +14619,15 @@ var init_run = __esm({
     init_readers();
     init_load2();
     init_resolve2();
+    KNOWN_SUITE_SELECT_IDS = /* @__PURE__ */ new Set([
+      "run.status",
+      "run.duration",
+      "run.depth",
+      "outcome.status",
+      "tool.usage",
+      "tool.failures",
+      "llm.usage"
+    ]);
   }
 });
 
@@ -16249,7 +16375,7 @@ var init_src = __esm({
 });
 
 // package.json
-var version = "6.31.8";
+var version = "6.31.9";
 
 // packages/cli/src/list.ts
 init_advanced();