agent-inspect 6.31.7 → 6.31.9

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 6.31.9
+
+### Patch Changes
+
+- f71ec7e: Fail closed when suite cases declare no effective assertions or unknown selectors, wire eval.requireSuccess and related eval controls into real check rules, and share AsyncLocalStorage across packed CJS root and /advanced entrypoints so guarded wrappers see the active run context.
+
+## 6.31.8
+
+### Patch Changes
+
+- 5bbe8c0: Restore CLI export lifecycle coalescing for schema 0.1 nested tools, map adapter modelId/source/fractional duration on OTLP export, keep OTLP parent lookup trace-scoped, strengthen producer validation, and make Promptfoo/Collector/Elastic recipe verifiers honest.
+
 ## 6.31.7
 
 ### Patch Changes

package/README.md

@@ -213,7 +213,7 @@ The root package is enough for custom capture, the CLI, checks, and Evidence wor
 
 ## Status and documentation
 
-**Current published baseline:** **6.31.7** · persisted schema `1.0` · Node.js `>=20` · MIT.
+**Current published baseline:** **6.31.9** · persisted schema `1.0` · Node.js `>=20` · MIT.
 
 Legacy v0.1 and v0.2 traces remain readable. Check the npm badge and [changelog](CHANGELOG.md) for the current published version.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-inspect",
-  "version": "6.31.7",
+  "version": "6.31.9",
   "license": "MIT",
   "type": "module",
   "description": "Local evidence debugger and trajectory-test toolkit for TypeScript AI agents — execution trees, TraceContract checks, Evidence v2, and read-only MCP",

package/packages/cli/dist/{chunk-QV73NM25.mjs → chunk-SWUZRRDT.mjs}

@@ -1627,7 +1627,18 @@ function extractCorrelationMetadata(record) {
 }
 
 // packages/core/src/context.ts
-new AsyncLocalStorage();
+var GLOBAL_CONTEXT_ALS_KEY = "agent-inspect:execution-context-als:v1";
+function getSharedContextStorage() {
+  const host = globalThis;
+  const existing = host[GLOBAL_CONTEXT_ALS_KEY];
+  if (existing instanceof AsyncLocalStorage) {
+    return existing;
+  }
+  const created = new AsyncLocalStorage();
+  host[GLOBAL_CONTEXT_ALS_KEY] = created;
+  return created;
+}
+getSharedContextStorage();
 
 // packages/core/src/terminal.ts
 var TERMINAL_INDENT = "  ";
@@ -8407,6 +8418,45 @@ function createToolOrderingRule(options) {
     requireEndpoints: false
   });
 }
+function createToolFailureRule(options) {
+  return {
+    id: "tool.failures",
+    category: "tool",
+    defaultSeverity: "error",
+    evaluate(context) {
+      const tools = finishedEvents(context, "TOOL");
+      const failures = tools.filter((event) => event.status === "error");
+      const retries = tools.map((event) => ({ event, count: retryCount(event) })).filter((item) => item.count !== void 0);
+      const findings = [];
+      if (options.maxFailures !== void 0 && failures.length > options.maxFailures) {
+        findings.push(
+          failFinding(
+            "tool.failures",
+            `Tool failure count ${failures.length} exceeded ${options.maxFailures}.`,
+            failures.map((event) => eventEvidence3(event)),
+            { maxFailures: options.maxFailures },
+            failures.length
+          )
+        );
+      }
+      if (options.maxRetries !== void 0) {
+        const excessiveRetries = retries.filter((item) => item.count > options.maxRetries);
+        if (excessiveRetries.length > 0) {
+          findings.push(
+            failFinding(
+              "tool.failures",
+              `Tool retry count exceeded ${options.maxRetries}.`,
+              excessiveRetries.map((item) => eventEvidence3(item.event, "attributes.retryCount")),
+              { maxRetries: options.maxRetries },
+              excessiveRetries.map((item) => ({ tool: toolName(item.event), retries: item.count }))
+            )
+          );
+        }
+      }
+      return findings;
+    }
+  };
+}
 function createLlmUsageRule(options) {
   return {
     id: "llm.usage",
@@ -11954,8 +12004,13 @@ function persistedInspectEventsToInspectEvents(events, options) {
 }
 
 // packages/core/src/persisted/tree-bridge.ts
+function logicalToPersisted(event) {
+  const { sourceEventIds: _sourceEventIds, projection: _projection, ...rest } = event;
+  return rest;
+}
 function persistedInspectEventsToRunTrees(events, options) {
-  const inspectEvents = persistedInspectEventsToInspectEvents(events, {
+  const sourceEvents = options?.coalesceLifecycle === true ? projectLogicalEvents(events).logicalEvents.map(logicalToPersisted) : events;
+  const inspectEvents = persistedInspectEventsToInspectEvents(sourceEvents, {
     skipInvalid: options?.skipInvalid
   });
   return new TreeBuilder().build(inspectEvents);
@@ -12649,7 +12704,7 @@ function mapOpenInferenceSpan(span, index, version) {
   const errorMessage = isRecord11(span.status) && typeof span.status.message === "string" ? span.status.message : void 0;
   const event = {
     schemaVersion: "0.2",
-    eventId: typeof rawAttributes["agent_inspect.event_id"] === "string" ? rawAttributes["agent_inspect.event_id"] : spanId,
+    eventId: typeof rawAttributes["agent_inspect.event_id"] === "string" ? rawAttributes["agent_inspect.event_id"] : `${traceId}:${spanId}`,
     runId: typeof rawAttributes["agent_inspect.run_id"] === "string" ? rawAttributes["agent_inspect.run_id"] : traceId,
     kind,
     name,
@@ -12700,12 +12755,16 @@ function mapOpenInferenceEvents(document) {
   const mapped = document.spans.map(
     (span, index) => mapOpenInferenceSpan(span, index, document.version)
   );
-  const spanIdToEventId = new Map(
-    mapped.map((span) => [span.spanId, span.event.eventId])
+  const spanKeyToEventId = new Map(
+    mapped.map((span) => {
+      const traceId = typeof span.event.trace?.traceId === "string" ? span.event.trace.traceId : "";
+      return [`${traceId}:${span.spanId}`, span.event.eventId];
+    })
   );
   for (const span of mapped) {
     if (span.parentSpanId === void 0) continue;
-    span.event.parentId = spanIdToEventId.get(span.parentSpanId) ?? span.parentSpanId;
+    const traceId = typeof span.event.trace?.traceId === "string" ? span.event.trace.traceId : "";
+    span.event.parentId = spanKeyToEventId.get(`${traceId}:${span.parentSpanId}`) ?? span.parentSpanId;
   }
   return {
     events: mapped.map((span) => span.event),
@@ -13254,7 +13313,7 @@ function mapOtlpSpan(context) {
   const errorMessage = isRecord11(span.status) && typeof span.status.message === "string" ? span.status.message : void 0;
   const event = {
     schemaVersion: "0.2",
-    eventId: typeof parsedSpanAttributes.attributes["agent_inspect.event_id"] === "string" ? parsedSpanAttributes.attributes["agent_inspect.event_id"] : spanId,
+    eventId: typeof parsedSpanAttributes.attributes["agent_inspect.event_id"] === "string" ? parsedSpanAttributes.attributes["agent_inspect.event_id"] : `${traceId}:${spanId}`,
     runId: typeof parsedSpanAttributes.attributes["agent_inspect.run_id"] === "string" ? parsedSpanAttributes.attributes["agent_inspect.run_id"] : traceId,
     kind,
     name: readStringField(span, ["name"]) ?? spanId,
@@ -13303,12 +13362,16 @@ function mapOtlpSpan(context) {
 }
 function mapOtlpEventsToPersisted(document) {
   const mapped = document.spans.map((span) => mapOtlpSpan(span));
-  const spanIdToEventId = new Map(
-    mapped.map((span) => [span.spanId, span.event.eventId])
+  const spanKeyToEventId = new Map(
+    mapped.map((span) => {
+      const traceId = typeof span.event.trace?.traceId === "string" ? span.event.trace.traceId : "";
+      return [`${traceId}:${span.spanId}`, span.event.eventId];
+    })
   );
   for (const span of mapped) {
     if (span.parentSpanId === void 0) continue;
-    span.event.parentId = spanIdToEventId.get(span.parentSpanId) ?? span.parentSpanId;
+    const traceId = typeof span.event.trace?.traceId === "string" ? span.event.trace.traceId : "";
+    span.event.parentId = spanKeyToEventId.get(`${traceId}:${span.parentSpanId}`) ?? span.parentSpanId;
   }
   return {
     events: mapped.map((span) => span.event),
@@ -13636,21 +13699,48 @@ async function resolveSuiteCaseTrace(suiteCase, options) {
 }
 
 // packages/core/src/suite/run.ts
+var KNOWN_SUITE_SELECT_IDS = /* @__PURE__ */ new Set([
+  "run.status",
+  "run.duration",
+  "run.depth",
+  "outcome.status",
+  "tool.usage",
+  "tool.failures",
+  "llm.usage"
+]);
 function diagnostic5(code, message, severity = "error", caseId) {
   return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
 }
-function buildCaseRules(suiteCase, config) {
+function buildCaseAssertions(suiteCase, config) {
   const rules = [];
   const select = new Set(config.checks?.select ?? []);
-  if (select.has("run.status")) {
+  const configDiagnostics = [];
+  const evalConfig = config.eval;
+  const declaredSelect = [...select];
+  for (const id of declaredSelect) {
+    if (!KNOWN_SUITE_SELECT_IDS.has(id)) {
+      configDiagnostics.push(
+        diagnostic5(
+          "AI_SUITE_UNKNOWN_SELECTOR",
+          `Unknown or unsupported suite check selector "${id}".`,
+          "error",
+          suiteCase.id
+        )
+      );
+    }
+  }
+  if (select.has("run.status") || evalConfig?.requireSuccess === true) {
     rules.push(createRunStatusRule());
+    select.add("run.status");
   }
   const requiredTools = [
     ...config.checks?.tool?.required ?? [],
+    ...evalConfig?.requiredTools ?? [],
     ...suiteCase.requireTools ?? []
   ];
   const forbiddenTools = [
     ...config.checks?.tool?.forbidden ?? [],
+    ...evalConfig?.forbiddenTools ?? [],
     ...suiteCase.forbidTools ?? []
   ];
   if (requiredTools.length > 0 || forbiddenTools.length > 0) {
@@ -13662,20 +13752,51 @@ function buildCaseRules(suiteCase, config) {
     );
     select.add("tool.usage");
   }
-  const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? config.eval?.maxDurationMs;
+  const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? evalConfig?.maxDurationMs;
   if (maxDurationMs !== void 0) {
     rules.push(createRunDurationRule({ maxDurationMs }));
     select.add("run.duration");
   }
-  const llm = config.checks?.llm;
-  if (llm?.allowedModels !== void 0 || llm?.maxTotalTokens !== void 0) {
-    rules.push(createLlmUsageRule(llm));
+  const maxDepth = config.checks?.run?.maxDepth ?? evalConfig?.maxDepth;
+  if (maxDepth !== void 0) {
+    rules.push(createRunDepthRule({ maxDepth }));
+    select.add("run.depth");
+  }
+  if (evalConfig?.maxRetries !== void 0) {
+    rules.push(createToolFailureRule({ maxRetries: evalConfig.maxRetries }));
+    select.add("tool.failures");
+  }
+  const allowedModels = config.checks?.llm?.allowedModels;
+  const maxTotalTokens = config.checks?.llm?.maxTotalTokens ?? evalConfig?.maxTotalTokens;
+  if (allowedModels !== void 0 || maxTotalTokens !== void 0) {
+    rules.push(
+      createLlmUsageRule({
+        ...allowedModels !== void 0 ? { allowedModels } : {},
+        ...maxTotalTokens !== void 0 ? { maxTotalTokens } : {}
+      })
+    );
     select.add("llm.usage");
   }
   if (select.has("outcome.status")) {
     rules.push(createObservedOutcomeRule({ failOn: ["failed"] }));
   }
-  return { rules, select: [...select] };
+  const observationCount = suiteCase.expectedObservations?.length ?? 0;
+  if (rules.length === 0 && observationCount === 0 && configDiagnostics.length === 0) {
+    configDiagnostics.push(
+      diagnostic5(
+        "AI_SUITE_NO_ASSERTIONS",
+        `Suite case "${suiteCase.id}" declares no effective checks, eval controls, or expected observations.`,
+        "error",
+        suiteCase.id
+      )
+    );
+  }
+  return {
+    rules,
+    select: [...select],
+    observationCount,
+    configDiagnostics
+  };
 }
 function outcomesFromRead(read) {
   const persistedOutcomes = extractOutcomesFromPersistedEvents(
@@ -13756,8 +13877,20 @@ async function runSuiteCase(suiteCase, config, options) {
       ]
     };
   }
-  const { rules, select } = buildCaseRules(suiteCase, config);
-  const checkResult = rules.length > 0 ? runTraceChecks({ read }, { rules, select }) : {
+  const compiled = buildCaseAssertions(suiteCase, config);
+  if (compiled.configDiagnostics.length > 0) {
+    return {
+      id: suiteCase.id,
+      status: "error",
+      tracePath: resolved.tracePath,
+      ...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
+      checkOk: false,
+      ...config.eval?.requireSuccess === true ? { evalOk: false } : {},
+      message: compiled.configDiagnostics.map((item) => item.message).join("; "),
+      diagnostics: compiled.configDiagnostics
+    };
+  }
+  const checkResult = compiled.rules.length > 0 ? runTraceChecks({ read }, { rules: compiled.rules, select: compiled.select }) : {
     ok: true,
     status: "pass",
     format: read.format,
@@ -13785,6 +13918,9 @@ async function runSuiteCase(suiteCase, config, options) {
   ];
   const checkOk = checkResult.ok;
   const observationsOk = observationResult.ok;
+  const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
+    (finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
+  ) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
   const ok = checkOk && observationsOk;
   const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
   return {
@@ -13793,6 +13929,7 @@ async function runSuiteCase(suiteCase, config, options) {
     tracePath: resolved.tracePath,
     ...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
     checkOk,
+    ...evalOk !== void 0 ? { evalOk } : {},
     observationsOk,
     diagnostics,
     ...ok ? {} : {
@@ -14960,5 +15097,5 @@ function renderGateReport(result, options = {}) {
 }
 
 export { ATTRIBUTION_CONFIDENCES, COHORT_METRIC_IDS, DEFAULT_REDACT_KEYS, DEFAULT_SUITE_ARTIFACTS_DIR, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, Redactor, TraceDirectory, TraceReadError, TreeBuilder, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, assertEvidenceRelativePath, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceHtmlShell, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectTraceSchemaVersions, compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, defineTraceContract, diffRuns, diffTraceEvents, enrichSessionRunRecord, escapeHtml, escapeMarkdown, evaluateTraceContract, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatStepLabel, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, inferEvidenceFileRole, isAgentInspectTrace, isAttributionConfidence, isCredentialSensitiveKey, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, manualTraceEventsToComparableRun, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToRunTrees, persistedInspectEventsToTraceEvents, projectLogicalEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunDiff, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, serializeEvidenceManifest, sha256Hex, stableJson, stringContainsHighConfidenceCredential, summarizeObservedOutcomes, summarizeSemanticParity, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, verifyEvidenceDirectory, zeroKinds };
-//# sourceMappingURL=chunk-QV73NM25.mjs.map
-//# sourceMappingURL=chunk-QV73NM25.mjs.map
+//# sourceMappingURL=chunk-SWUZRRDT.mjs.map
+//# sourceMappingURL=chunk-SWUZRRDT.mjs.map