agent-inspect 6.31.8 → 6.31.10

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 6.31.10
+
+### Patch Changes
+
+- 1327356: Return actionable diagnostics for reversed evaluateTraceContract arguments and warn (without throwing) when inspectRun is passed an unsupported writer option, pointing callers at createInspector({ writer }).
+
+## 6.31.9
+
+### Patch Changes
+
+- f71ec7e: Fail closed when suite cases declare no effective assertions or unknown selectors, wire eval.requireSuccess and related eval controls into real check rules, and share AsyncLocalStorage across packed CJS root and /advanced entrypoints so guarded wrappers see the active run context.
+
 ## 6.31.8
 
 ### Patch Changes

package/README.md

@@ -213,7 +213,7 @@ The root package is enough for custom capture, the CLI, checks, and Evidence wor
 
 ## Status and documentation
 
-**Current published baseline:** **6.31.8** · persisted schema `1.0` · Node.js `>=20` · MIT.
+**Current published baseline:** **6.31.10** · persisted schema `1.0` · Node.js `>=20` · MIT.
 
 Legacy v0.1 and v0.2 traces remain readable. Check the npm badge and [changelog](CHANGELOG.md) for the current published version.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-inspect",
-  "version": "6.31.8",
+  "version": "6.31.10",
   "license": "MIT",
   "type": "module",
   "description": "Local evidence debugger and trajectory-test toolkit for TypeScript AI agents — execution trees, TraceContract checks, Evidence v2, and read-only MCP",

package/packages/cli/dist/{chunk-6FE47XSF.mjs → chunk-JCKC4ZRV.mjs}

@@ -1627,7 +1627,18 @@ function extractCorrelationMetadata(record) {
 }
 
 // packages/core/src/context.ts
-new AsyncLocalStorage();
+var GLOBAL_CONTEXT_ALS_KEY = "agent-inspect:execution-context-als:v1";
+function getSharedContextStorage() {
+  const host = globalThis;
+  const existing = host[GLOBAL_CONTEXT_ALS_KEY];
+  if (existing instanceof AsyncLocalStorage) {
+    return existing;
+  }
+  const created = new AsyncLocalStorage();
+  host[GLOBAL_CONTEXT_ALS_KEY] = created;
+  return created;
+}
+getSharedContextStorage();
 
 // packages/core/src/terminal.ts
 var TERMINAL_INDENT = "  ";
@@ -4392,6 +4403,17 @@ var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
     summary: "Trace contains a parentId cycle.",
     remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
     relatedCodes: ["structure.cycle"]
+  },
+  AI_TRACE_CONTRACT_INPUT_INVALID: {
+    code: "AI_TRACE_CONTRACT_INPUT_INVALID",
+    summary: "evaluateTraceContract expects ({ read }, contract) or use evaluateTraceContractRead(read, contract).",
+    remediation: "Pass TraceCheckInput as the first argument: evaluateTraceContract({ read }, contract).",
+    relatedCodes: ["AI_CHECK_INVALID_ARGUMENTS"]
+  },
+  AI_INSPECT_RUN_UNSUPPORTED_OPTION: {
+    code: "AI_INSPECT_RUN_UNSUPPORTED_OPTION",
+    summary: 'inspectRun does not accept a "writer" option.',
+    remediation: "Use createInspector({ writer }) then inspector.run(...). inspectRun persists via the global JSONL path (schema 0.1)."
   }
 });
 function formatProgrammaticDiagnostic(code, detail) {
@@ -7123,7 +7145,72 @@ function defineTraceContract(input) {
     ...alternatives ? { alternatives } : {}
   };
 }
+function isTraceReadLike(value) {
+  if (typeof value !== "object" || value === null) return false;
+  const candidate = value;
+  return typeof candidate.format === "string" && Array.isArray(candidate.runs) && Array.isArray(candidate.events);
+}
+function isTraceCheckInputLike(value) {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    return false;
+  }
+  return isTraceReadLike(value.read);
+}
+function looksLikeTraceContract(value) {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    return false;
+  }
+  if (isTraceReadLike(value) || isTraceCheckInputLike(value)) {
+    return false;
+  }
+  return true;
+}
+function invalidContractInputResult() {
+  return {
+    ok: false,
+    status: "error",
+    format: "unknown",
+    summary: {
+      passed: 0,
+      failed: 0,
+      warnings: 0,
+      errors: 1,
+      rulesEvaluated: 0
+    },
+    findings: [],
+    diagnostics: [
+      {
+        code: "AI_CHECK_INVALID_ARGUMENTS",
+        message: formatProgrammaticDiagnostic("AI_TRACE_CONTRACT_INPUT_INVALID"),
+        severity: "error"
+      }
+    ],
+    ruleExecutions: []
+  };
+}
+function validateEvaluateTraceContractArgs(input, contract) {
+  if (isTraceCheckInputLike(input) && looksLikeTraceContract(contract)) {
+    return void 0;
+  }
+  if (looksLikeTraceContract(input) && (isTraceCheckInputLike(contract) || isTraceReadLike(contract))) {
+    return invalidContractInputResult();
+  }
+  if (isTraceReadLike(input)) {
+    return invalidContractInputResult();
+  }
+  if (!isTraceCheckInputLike(input)) {
+    return invalidContractInputResult();
+  }
+  if (!looksLikeTraceContract(contract)) {
+    return invalidContractInputResult();
+  }
+  return void 0;
+}
 function evaluateTraceContract(input, contract, options = {}) {
+  const inputGuard = validateEvaluateTraceContractArgs(input, contract);
+  if (inputGuard !== void 0) {
+    return inputGuard;
+  }
   const shapeErrors = [
     ...validateAllowedStatusesShape(contract.run?.allowedStatuses),
     ...validateAlternativesShape(contract.alternatives),
@@ -8407,6 +8494,45 @@ function createToolOrderingRule(options) {
     requireEndpoints: false
   });
 }
+function createToolFailureRule(options) {
+  return {
+    id: "tool.failures",
+    category: "tool",
+    defaultSeverity: "error",
+    evaluate(context) {
+      const tools = finishedEvents(context, "TOOL");
+      const failures = tools.filter((event) => event.status === "error");
+      const retries = tools.map((event) => ({ event, count: retryCount(event) })).filter((item) => item.count !== void 0);
+      const findings = [];
+      if (options.maxFailures !== void 0 && failures.length > options.maxFailures) {
+        findings.push(
+          failFinding(
+            "tool.failures",
+            `Tool failure count ${failures.length} exceeded ${options.maxFailures}.`,
+            failures.map((event) => eventEvidence3(event)),
+            { maxFailures: options.maxFailures },
+            failures.length
+          )
+        );
+      }
+      if (options.maxRetries !== void 0) {
+        const excessiveRetries = retries.filter((item) => item.count > options.maxRetries);
+        if (excessiveRetries.length > 0) {
+          findings.push(
+            failFinding(
+              "tool.failures",
+              `Tool retry count exceeded ${options.maxRetries}.`,
+              excessiveRetries.map((item) => eventEvidence3(item.event, "attributes.retryCount")),
+              { maxRetries: options.maxRetries },
+              excessiveRetries.map((item) => ({ tool: toolName(item.event), retries: item.count }))
+            )
+          );
+        }
+      }
+      return findings;
+    }
+  };
+}
 function createLlmUsageRule(options) {
   return {
     id: "llm.usage",
@@ -13649,21 +13775,48 @@ async function resolveSuiteCaseTrace(suiteCase, options) {
 }
 
 // packages/core/src/suite/run.ts
+var KNOWN_SUITE_SELECT_IDS = /* @__PURE__ */ new Set([
+  "run.status",
+  "run.duration",
+  "run.depth",
+  "outcome.status",
+  "tool.usage",
+  "tool.failures",
+  "llm.usage"
+]);
 function diagnostic5(code, message, severity = "error", caseId) {
   return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
 }
-function buildCaseRules(suiteCase, config) {
+function buildCaseAssertions(suiteCase, config) {
   const rules = [];
   const select = new Set(config.checks?.select ?? []);
-  if (select.has("run.status")) {
+  const configDiagnostics = [];
+  const evalConfig = config.eval;
+  const declaredSelect = [...select];
+  for (const id of declaredSelect) {
+    if (!KNOWN_SUITE_SELECT_IDS.has(id)) {
+      configDiagnostics.push(
+        diagnostic5(
+          "AI_SUITE_UNKNOWN_SELECTOR",
+          `Unknown or unsupported suite check selector "${id}".`,
+          "error",
+          suiteCase.id
+        )
+      );
+    }
+  }
+  if (select.has("run.status") || evalConfig?.requireSuccess === true) {
     rules.push(createRunStatusRule());
+    select.add("run.status");
   }
   const requiredTools = [
     ...config.checks?.tool?.required ?? [],
+    ...evalConfig?.requiredTools ?? [],
     ...suiteCase.requireTools ?? []
   ];
   const forbiddenTools = [
     ...config.checks?.tool?.forbidden ?? [],
+    ...evalConfig?.forbiddenTools ?? [],
     ...suiteCase.forbidTools ?? []
   ];
   if (requiredTools.length > 0 || forbiddenTools.length > 0) {
@@ -13675,20 +13828,51 @@ function buildCaseRules(suiteCase, config) {
     );
     select.add("tool.usage");
   }
-  const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? config.eval?.maxDurationMs;
+  const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? evalConfig?.maxDurationMs;
   if (maxDurationMs !== void 0) {
     rules.push(createRunDurationRule({ maxDurationMs }));
     select.add("run.duration");
   }
-  const llm = config.checks?.llm;
-  if (llm?.allowedModels !== void 0 || llm?.maxTotalTokens !== void 0) {
-    rules.push(createLlmUsageRule(llm));
+  const maxDepth = config.checks?.run?.maxDepth ?? evalConfig?.maxDepth;
+  if (maxDepth !== void 0) {
+    rules.push(createRunDepthRule({ maxDepth }));
+    select.add("run.depth");
+  }
+  if (evalConfig?.maxRetries !== void 0) {
+    rules.push(createToolFailureRule({ maxRetries: evalConfig.maxRetries }));
+    select.add("tool.failures");
+  }
+  const allowedModels = config.checks?.llm?.allowedModels;
+  const maxTotalTokens = config.checks?.llm?.maxTotalTokens ?? evalConfig?.maxTotalTokens;
+  if (allowedModels !== void 0 || maxTotalTokens !== void 0) {
+    rules.push(
+      createLlmUsageRule({
+        ...allowedModels !== void 0 ? { allowedModels } : {},
+        ...maxTotalTokens !== void 0 ? { maxTotalTokens } : {}
+      })
+    );
     select.add("llm.usage");
   }
   if (select.has("outcome.status")) {
     rules.push(createObservedOutcomeRule({ failOn: ["failed"] }));
   }
-  return { rules, select: [...select] };
+  const observationCount = suiteCase.expectedObservations?.length ?? 0;
+  if (rules.length === 0 && observationCount === 0 && configDiagnostics.length === 0) {
+    configDiagnostics.push(
+      diagnostic5(
+        "AI_SUITE_NO_ASSERTIONS",
+        `Suite case "${suiteCase.id}" declares no effective checks, eval controls, or expected observations.`,
+        "error",
+        suiteCase.id
+      )
+    );
+  }
+  return {
+    rules,
+    select: [...select],
+    observationCount,
+    configDiagnostics
+  };
 }
 function outcomesFromRead(read) {
   const persistedOutcomes = extractOutcomesFromPersistedEvents(
@@ -13769,8 +13953,20 @@ async function runSuiteCase(suiteCase, config, options) {
       ]
     };
   }
-  const { rules, select } = buildCaseRules(suiteCase, config);
-  const checkResult = rules.length > 0 ? runTraceChecks({ read }, { rules, select }) : {
+  const compiled = buildCaseAssertions(suiteCase, config);
+  if (compiled.configDiagnostics.length > 0) {
+    return {
+      id: suiteCase.id,
+      status: "error",
+      tracePath: resolved.tracePath,
+      ...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
+      checkOk: false,
+      ...config.eval?.requireSuccess === true ? { evalOk: false } : {},
+      message: compiled.configDiagnostics.map((item) => item.message).join("; "),
+      diagnostics: compiled.configDiagnostics
+    };
+  }
+  const checkResult = compiled.rules.length > 0 ? runTraceChecks({ read }, { rules: compiled.rules, select: compiled.select }) : {
     ok: true,
     status: "pass",
     format: read.format,
@@ -13798,6 +13994,9 @@ async function runSuiteCase(suiteCase, config, options) {
   ];
   const checkOk = checkResult.ok;
   const observationsOk = observationResult.ok;
+  const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
+    (finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
+  ) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
   const ok = checkOk && observationsOk;
   const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
   return {
@@ -13806,6 +14005,7 @@ async function runSuiteCase(suiteCase, config, options) {
     tracePath: resolved.tracePath,
     ...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
     checkOk,
+    ...evalOk !== void 0 ? { evalOk } : {},
     observationsOk,
     diagnostics,
     ...ok ? {} : {
@@ -14973,5 +15173,5 @@ function renderGateReport(result, options = {}) {
 }
 
 export { ATTRIBUTION_CONFIDENCES, COHORT_METRIC_IDS, DEFAULT_REDACT_KEYS, DEFAULT_SUITE_ARTIFACTS_DIR, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, Redactor, TraceDirectory, TraceReadError, TreeBuilder, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, assertEvidenceRelativePath, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceHtmlShell, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectTraceSchemaVersions, compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, defineTraceContract, diffRuns, diffTraceEvents, enrichSessionRunRecord, escapeHtml, escapeMarkdown, evaluateTraceContract, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatStepLabel, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, inferEvidenceFileRole, isAgentInspectTrace, isAttributionConfidence, isCredentialSensitiveKey, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, manualTraceEventsToComparableRun, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToRunTrees, persistedInspectEventsToTraceEvents, projectLogicalEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunDiff, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, serializeEvidenceManifest, sha256Hex, stableJson, stringContainsHighConfidenceCredential, summarizeObservedOutcomes, summarizeSemanticParity, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, verifyEvidenceDirectory, zeroKinds };
-//# sourceMappingURL=chunk-6FE47XSF.mjs.map
-//# sourceMappingURL=chunk-6FE47XSF.mjs.map
+//# sourceMappingURL=chunk-JCKC4ZRV.mjs.map
+//# sourceMappingURL=chunk-JCKC4ZRV.mjs.map