agent-inspect 6.31.8 → 6.31.9

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 6.31.9
+
+### Patch Changes
+
+- f71ec7e: Fail closed when suite cases declare no effective assertions or unknown selectors, wire eval.requireSuccess and related eval controls into real check rules, and share AsyncLocalStorage across packed CJS root and /advanced entrypoints so guarded wrappers see the active run context.
+
 ## 6.31.8
 
 ### Patch Changes

package/README.md

@@ -213,7 +213,7 @@ The root package is enough for custom capture, the CLI, checks, and Evidence wor
 
 ## Status and documentation
 
-**Current published baseline:** **6.31.8** · persisted schema `1.0` · Node.js `>=20` · MIT.
+**Current published baseline:** **6.31.9** · persisted schema `1.0` · Node.js `>=20` · MIT.
 
 Legacy v0.1 and v0.2 traces remain readable. Check the npm badge and [changelog](CHANGELOG.md) for the current published version.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-inspect",
-  "version": "6.31.8",
+  "version": "6.31.9",
   "license": "MIT",
   "type": "module",
   "description": "Local evidence debugger and trajectory-test toolkit for TypeScript AI agents — execution trees, TraceContract checks, Evidence v2, and read-only MCP",

package/packages/cli/dist/{chunk-6FE47XSF.mjs → chunk-SWUZRRDT.mjs}

@@ -1627,7 +1627,18 @@ function extractCorrelationMetadata(record) {
 }
 
 // packages/core/src/context.ts
-new AsyncLocalStorage();
+var GLOBAL_CONTEXT_ALS_KEY = "agent-inspect:execution-context-als:v1";
+function getSharedContextStorage() {
+  const host = globalThis;
+  const existing = host[GLOBAL_CONTEXT_ALS_KEY];
+  if (existing instanceof AsyncLocalStorage) {
+    return existing;
+  }
+  const created = new AsyncLocalStorage();
+  host[GLOBAL_CONTEXT_ALS_KEY] = created;
+  return created;
+}
+getSharedContextStorage();
 
 // packages/core/src/terminal.ts
 var TERMINAL_INDENT = "  ";
@@ -8407,6 +8418,45 @@ function createToolOrderingRule(options) {
     requireEndpoints: false
   });
 }
+function createToolFailureRule(options) {
+  return {
+    id: "tool.failures",
+    category: "tool",
+    defaultSeverity: "error",
+    evaluate(context) {
+      const tools = finishedEvents(context, "TOOL");
+      const failures = tools.filter((event) => event.status === "error");
+      const retries = tools.map((event) => ({ event, count: retryCount(event) })).filter((item) => item.count !== void 0);
+      const findings = [];
+      if (options.maxFailures !== void 0 && failures.length > options.maxFailures) {
+        findings.push(
+          failFinding(
+            "tool.failures",
+            `Tool failure count ${failures.length} exceeded ${options.maxFailures}.`,
+            failures.map((event) => eventEvidence3(event)),
+            { maxFailures: options.maxFailures },
+            failures.length
+          )
+        );
+      }
+      if (options.maxRetries !== void 0) {
+        const excessiveRetries = retries.filter((item) => item.count > options.maxRetries);
+        if (excessiveRetries.length > 0) {
+          findings.push(
+            failFinding(
+              "tool.failures",
+              `Tool retry count exceeded ${options.maxRetries}.`,
+              excessiveRetries.map((item) => eventEvidence3(item.event, "attributes.retryCount")),
+              { maxRetries: options.maxRetries },
+              excessiveRetries.map((item) => ({ tool: toolName(item.event), retries: item.count }))
+            )
+          );
+        }
+      }
+      return findings;
+    }
+  };
+}
 function createLlmUsageRule(options) {
   return {
     id: "llm.usage",
@@ -13649,21 +13699,48 @@ async function resolveSuiteCaseTrace(suiteCase, options) {
 }
 
 // packages/core/src/suite/run.ts
+var KNOWN_SUITE_SELECT_IDS = /* @__PURE__ */ new Set([
+  "run.status",
+  "run.duration",
+  "run.depth",
+  "outcome.status",
+  "tool.usage",
+  "tool.failures",
+  "llm.usage"
+]);
 function diagnostic5(code, message, severity = "error", caseId) {
   return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
 }
-function buildCaseRules(suiteCase, config) {
+function buildCaseAssertions(suiteCase, config) {
   const rules = [];
   const select = new Set(config.checks?.select ?? []);
-  if (select.has("run.status")) {
+  const configDiagnostics = [];
+  const evalConfig = config.eval;
+  const declaredSelect = [...select];
+  for (const id of declaredSelect) {
+    if (!KNOWN_SUITE_SELECT_IDS.has(id)) {
+      configDiagnostics.push(
+        diagnostic5(
+          "AI_SUITE_UNKNOWN_SELECTOR",
+          `Unknown or unsupported suite check selector "${id}".`,
+          "error",
+          suiteCase.id
+        )
+      );
+    }
+  }
+  if (select.has("run.status") || evalConfig?.requireSuccess === true) {
     rules.push(createRunStatusRule());
+    select.add("run.status");
   }
   const requiredTools = [
     ...config.checks?.tool?.required ?? [],
+    ...evalConfig?.requiredTools ?? [],
     ...suiteCase.requireTools ?? []
   ];
   const forbiddenTools = [
     ...config.checks?.tool?.forbidden ?? [],
+    ...evalConfig?.forbiddenTools ?? [],
     ...suiteCase.forbidTools ?? []
   ];
   if (requiredTools.length > 0 || forbiddenTools.length > 0) {
@@ -13675,20 +13752,51 @@ function buildCaseRules(suiteCase, config) {
     );
     select.add("tool.usage");
   }
-  const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? config.eval?.maxDurationMs;
+  const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? evalConfig?.maxDurationMs;
   if (maxDurationMs !== void 0) {
     rules.push(createRunDurationRule({ maxDurationMs }));
     select.add("run.duration");
   }
-  const llm = config.checks?.llm;
-  if (llm?.allowedModels !== void 0 || llm?.maxTotalTokens !== void 0) {
-    rules.push(createLlmUsageRule(llm));
+  const maxDepth = config.checks?.run?.maxDepth ?? evalConfig?.maxDepth;
+  if (maxDepth !== void 0) {
+    rules.push(createRunDepthRule({ maxDepth }));
+    select.add("run.depth");
+  }
+  if (evalConfig?.maxRetries !== void 0) {
+    rules.push(createToolFailureRule({ maxRetries: evalConfig.maxRetries }));
+    select.add("tool.failures");
+  }
+  const allowedModels = config.checks?.llm?.allowedModels;
+  const maxTotalTokens = config.checks?.llm?.maxTotalTokens ?? evalConfig?.maxTotalTokens;
+  if (allowedModels !== void 0 || maxTotalTokens !== void 0) {
+    rules.push(
+      createLlmUsageRule({
+        ...allowedModels !== void 0 ? { allowedModels } : {},
+        ...maxTotalTokens !== void 0 ? { maxTotalTokens } : {}
+      })
+    );
     select.add("llm.usage");
   }
   if (select.has("outcome.status")) {
     rules.push(createObservedOutcomeRule({ failOn: ["failed"] }));
   }
-  return { rules, select: [...select] };
+  const observationCount = suiteCase.expectedObservations?.length ?? 0;
+  if (rules.length === 0 && observationCount === 0 && configDiagnostics.length === 0) {
+    configDiagnostics.push(
+      diagnostic5(
+        "AI_SUITE_NO_ASSERTIONS",
+        `Suite case "${suiteCase.id}" declares no effective checks, eval controls, or expected observations.`,
+        "error",
+        suiteCase.id
+      )
+    );
+  }
+  return {
+    rules,
+    select: [...select],
+    observationCount,
+    configDiagnostics
+  };
 }
 function outcomesFromRead(read) {
   const persistedOutcomes = extractOutcomesFromPersistedEvents(
@@ -13769,8 +13877,20 @@ async function runSuiteCase(suiteCase, config, options) {
       ]
     };
   }
-  const { rules, select } = buildCaseRules(suiteCase, config);
-  const checkResult = rules.length > 0 ? runTraceChecks({ read }, { rules, select }) : {
+  const compiled = buildCaseAssertions(suiteCase, config);
+  if (compiled.configDiagnostics.length > 0) {
+    return {
+      id: suiteCase.id,
+      status: "error",
+      tracePath: resolved.tracePath,
+      ...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
+      checkOk: false,
+      ...config.eval?.requireSuccess === true ? { evalOk: false } : {},
+      message: compiled.configDiagnostics.map((item) => item.message).join("; "),
+      diagnostics: compiled.configDiagnostics
+    };
+  }
+  const checkResult = compiled.rules.length > 0 ? runTraceChecks({ read }, { rules: compiled.rules, select: compiled.select }) : {
     ok: true,
     status: "pass",
     format: read.format,
@@ -13798,6 +13918,9 @@ async function runSuiteCase(suiteCase, config, options) {
   ];
   const checkOk = checkResult.ok;
   const observationsOk = observationResult.ok;
+  const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
+    (finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
+  ) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
   const ok = checkOk && observationsOk;
   const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
   return {
@@ -13806,6 +13929,7 @@ async function runSuiteCase(suiteCase, config, options) {
     tracePath: resolved.tracePath,
     ...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
     checkOk,
+    ...evalOk !== void 0 ? { evalOk } : {},
     observationsOk,
     diagnostics,
     ...ok ? {} : {
@@ -14973,5 +15097,5 @@ function renderGateReport(result, options = {}) {
 }
 
 export { ATTRIBUTION_CONFIDENCES, COHORT_METRIC_IDS, DEFAULT_REDACT_KEYS, DEFAULT_SUITE_ARTIFACTS_DIR, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, Redactor, TraceDirectory, TraceReadError, TreeBuilder, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, assertEvidenceRelativePath, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceHtmlShell, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectTraceSchemaVersions, compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, defineTraceContract, diffRuns, diffTraceEvents, enrichSessionRunRecord, escapeHtml, escapeMarkdown, evaluateTraceContract, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatStepLabel, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, inferEvidenceFileRole, isAgentInspectTrace, isAttributionConfidence, isCredentialSensitiveKey, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, manualTraceEventsToComparableRun, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToRunTrees, persistedInspectEventsToTraceEvents, projectLogicalEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunDiff, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, serializeEvidenceManifest, sha256Hex, stableJson, stringContainsHighConfidenceCredential, summarizeObservedOutcomes, summarizeSemanticParity, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, verifyEvidenceDirectory, zeroKinds };
-//# sourceMappingURL=chunk-6FE47XSF.mjs.map
-//# sourceMappingURL=chunk-6FE47XSF.mjs.map
+//# sourceMappingURL=chunk-SWUZRRDT.mjs.map
+//# sourceMappingURL=chunk-SWUZRRDT.mjs.map