agent-inspect 6.4.0 → 6.5.0

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 6.5.0
+
+### Minor Changes
+
+- e48a964: Add experimental TraceContract API in checks, fix all-skipped suite pass semantics, and cohort tolerance/sample diagnostics.
+
+## 6.4.1
+
+### Patch Changes
+
+- 7e832d7: Trust and security patch: MCP result boundary, real bundle safety assessment, path sanitization, viewer XSS hardening, strict plugin manifests, gate validation, Studio init fixes, and packed quickstart E2E.
+
 ## 6.4.0
 
 ### Minor Changes

package/README.md

@@ -45,7 +45,7 @@ node examples/agent-inspect-demo.mjs
 npx agent-inspect list --dir .agent-inspect
 npx agent-inspect view <run-id> --dir .agent-inspect
 npx agent-inspect report <run-id> --dir .agent-inspect
-npx agent-inspect verify-safe --dir .agent-inspect
+npx agent-inspect verify-safe <run-id> --dir .agent-inspect
 ```
 
 Install → one trace → one failure check → one share-safe artifact in under five minutes.
@@ -153,7 +153,7 @@ See [Compare](https://agentinspect.vercel.app/docs/compare/).
 
 ## Install details
 
-Current release: **3.5.5** (sixteen linked npm packages). Persisted trace schema **1.0**. Requires **Node.js >= 20**.
+Current release: **6.4.0** (sixteen linked npm packages). Persisted trace schema **1.0**. Requires **Node.js >= 20**.
 
 ```bash
 pnpm add agent-inspect

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-inspect",
-  "version": "6.4.0",
+  "version": "6.5.0",
   "license": "MIT",
   "type": "module",
   "description": "Trace, check, and safely share TypeScript AI-agent runs locally — no account, no upload, metadata-only by default",
@@ -215,7 +215,7 @@
     "test:all": "pnpm run typecheck && pnpm run test && pnpm run build && pnpm run size",
     "prepublish:checks": "pnpm run typecheck && pnpm run test && pnpm run test:coverage && pnpm run build && pnpm run fixtures:check && pnpm run recipes:check && pnpm run size && pnpm run pack:smoke",
     "pack:dry-run": "pnpm run build && npm pack --dry-run",
-    "pack:smoke": "pnpm run build && node scripts/package-smoke.mjs",
+    "pack:smoke": "pnpm run build && node scripts/package-smoke.mjs && node scripts/packed-quickstart-e2e.mjs",
     "compat:smoke": "node scripts/compat-smoke.mjs",
     "fixtures:check": "node scripts/validate-fixtures.mjs",
     "recipes:check": "node scripts/validate-recipes.mjs",

package/packages/cli/dist/{chunk-YS2I722C.mjs → chunk-M6PC5KUT.mjs}

@@ -1,6 +1,6 @@
 import { readdir, stat, readFile, access } from 'fs/promises';
 import os from 'os';
-import path7 from 'path';
+import path5 from 'path';
 import crypto2, { webcrypto } from 'crypto';
 import process2 from 'process';
 import tty from 'tty';
@@ -255,7 +255,7 @@ function nanoid(size = 21) {
 // packages/core/src/utils.ts
 var DEFAULT_TRACE_DIR_NAME = ".agent-inspect";
 var RUNS_DIR_NAME = "runs";
-var FALLBACK_TRACE_DIR = path7.join(
+var FALLBACK_TRACE_DIR = path5.join(
   os.tmpdir(),
   "agent-inspect",
   RUNS_DIR_NAME
@@ -290,7 +290,7 @@ function getDefaultTraceDir() {
     if (typeof home !== "string" || home.trim() === "") {
       return FALLBACK_TRACE_DIR;
     }
-    return path7.join(home, DEFAULT_TRACE_DIR_NAME, RUNS_DIR_NAME);
+    return path5.join(home, DEFAULT_TRACE_DIR_NAME, RUNS_DIR_NAME);
   } catch {
     return FALLBACK_TRACE_DIR;
   }
@@ -298,11 +298,11 @@ function getDefaultTraceDir() {
 function getTraceFilePath(runId, traceDir) {
   const baseDir = traceDir ?? getDefaultTraceDir();
   let safeId = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "run_unknown";
-  safeId = path7.basename(safeId);
+  safeId = path5.basename(safeId);
   if (safeId === "" || safeId === "." || safeId === "..") {
     safeId = "run_unknown";
   }
-  return path7.join(baseDir, `${safeId}.jsonl`);
+  return path5.join(baseDir, `${safeId}.jsonl`);
 }
 function formatError(error) {
   if (error instanceof Error) {
@@ -1619,7 +1619,7 @@ var TraceDirectory = class {
     this.#dir = resolveTraceDir(options);
   }
   getPath(filename) {
-    return filename ? path7.join(this.#dir, filename) : this.#dir;
+    return filename ? path5.join(this.#dir, filename) : this.#dir;
   }
   async list() {
     try {
@@ -1646,7 +1646,7 @@ function parseIsoToMs2(value) {
 }
 async function extractMetadata(filePath, _quickScan) {
   const stats = await stat(filePath);
-  let runIdFromFile = path7.basename(filePath);
+  let runIdFromFile = path5.basename(filePath);
   if (runIdFromFile.endsWith(".jsonl")) {
     runIdFromFile = runIdFromFile.slice(0, -".jsonl".length);
   }
@@ -3841,21 +3841,43 @@ function buildBundleSummaryMarkdown(parts) {
   );
   return lines.join("\n");
 }
+function sanitizeBundleRunId(runId) {
+  let safe = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "run_unknown";
+  safe = path5.basename(safe);
+  safe = safe.replace(/[^a-zA-Z0-9._-]+/g, "_");
+  if (safe === "" || safe === "." || safe === "..") {
+    safe = "run_unknown";
+  }
+  return safe;
+}
+function bundleRunAssetRelativePath(runId, extension) {
+  const safe = sanitizeBundleRunId(runId);
+  const ext = extension.startsWith(".") ? extension : `.${extension}`;
+  return path5.posix.join("assets", "runs", `${safe}${ext}`);
+}
+function assertBundlePathContained(outputDir, relativePath) {
+  const base = path5.resolve(outputDir);
+  const resolved = path5.resolve(base, relativePath);
+  if (resolved !== base && !resolved.startsWith(base + path5.sep)) {
+    throw new Error(`Bundle path escapes output directory: ${relativePath}`);
+  }
+  return resolved;
+}
 function normalizeBundleOutputPath(out) {
   const trimmed = out.trim();
   if (trimmed === "") {
     throw new Error("--out requires a non-empty path.");
   }
-  const resolved = path7.resolve(trimmed);
+  const resolved = path5.resolve(trimmed);
   if (resolved.toLowerCase().endsWith(".zip")) {
     return resolved.slice(0, -4);
   }
   return resolved;
 }
 function defaultBundleOutputPath(runIds) {
-  const label = runIds.length === 1 ? runIds[0] : `multi-${runIds.length}`;
+  const label = runIds.length === 1 ? sanitizeBundleRunId(runIds[0]) : `multi-${runIds.length}`;
   const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
-  return path7.resolve(`agent-inspect-bundle-${label}-${stamp}`);
+  return path5.resolve(`agent-inspect-bundle-${label}-${stamp}`);
 }
 
 // packages/core/src/suite/types.ts
@@ -4009,7 +4031,7 @@ function normalizeSuiteConfig(value) {
 }
 async function validateSuiteConfig(config, options) {
   const diagnostics = [];
-  const tracesDir = path7.resolve(options.configDir, config.traces);
+  const tracesDir = path5.resolve(options.configDir, config.traces);
   try {
     await access(tracesDir);
   } catch {
@@ -4019,7 +4041,7 @@ async function validateSuiteConfig(config, options) {
   }
   for (const suiteCase of config.cases) {
     if (suiteCase.input !== void 0) {
-      const inputPath = path7.resolve(options.configDir, suiteCase.input);
+      const inputPath = path5.resolve(options.configDir, suiteCase.input);
       try {
         await access(inputPath);
       } catch {
@@ -4050,12 +4072,12 @@ async function fileExists(filePath) {
   }
 }
 async function resolveSuiteConfigPath(options = {}) {
-  const cwd = path7.resolve(options.cwd ?? process.cwd());
+  const cwd = path5.resolve(options.cwd ?? process.cwd());
   if (options.configPath !== void 0 && options.configPath.trim() !== "") {
-    return path7.resolve(cwd, options.configPath.trim());
+    return path5.resolve(cwd, options.configPath.trim());
   }
   for (const name of DEFAULT_SUITE_CONFIG_NAMES) {
-    const candidate = path7.join(cwd, name);
+    const candidate = path5.join(cwd, name);
     if (await fileExists(candidate)) return candidate;
   }
   throw new Error(
@@ -4072,7 +4094,7 @@ async function loadSuiteConfig(options = {}) {
       diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
     });
   }
-  const extension = path7.extname(configPath);
+  const extension = path5.extname(configPath);
   if (TS_CONFIG_EXTENSIONS.has(extension)) {
     const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
     throw Object.assign(new Error(message), {
@@ -4097,7 +4119,7 @@ async function loadSuiteConfig(options = {}) {
     return {
       config,
       configPath,
-      configDir: path7.dirname(configPath)
+      configDir: path5.dirname(configPath)
     };
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
@@ -6136,7 +6158,7 @@ function findReaderByFormat(format, readers) {
 }
 async function jsonlFilesInDirectory(dirPath) {
   const entries = await readdir(dirPath, { withFileTypes: true });
-  return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => path7.join(dirPath, entry.name)).sort((a, b) => a.localeCompare(b));
+  return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => path5.join(dirPath, entry.name)).sort((a, b) => a.localeCompare(b));
 }
 async function resolveInput(input) {
   const cached = resolvedInputCache.get(input);
@@ -7596,7 +7618,7 @@ async function exists(filePath) {
 }
 async function resolveSuiteCaseTrace(suiteCase, options) {
   if (suiteCase.trace !== void 0) {
-    const tracePath = path7.resolve(options.configDir, suiteCase.trace);
+    const tracePath = path5.resolve(options.configDir, suiteCase.trace);
     if (await exists(tracePath)) {
       return { caseId: suiteCase.id, tracePath, missing: false };
     }
@@ -7612,7 +7634,7 @@ async function resolveSuiteCaseTrace(suiteCase, options) {
   if (await exists(directPath)) {
     return { caseId: suiteCase.id, tracePath: directPath, runId: runKey, missing: false };
   }
-  const nestedPath = path7.join(options.tracesDir, `${path7.basename(runKey)}.jsonl`);
+  const nestedPath = path5.join(options.tracesDir, `${path5.basename(runKey)}.jsonl`);
   if (await exists(nestedPath)) {
     return { caseId: suiteCase.id, tracePath: nestedPath, runId: runKey, missing: false };
   }
@@ -7793,7 +7815,7 @@ async function runSuiteCase(suiteCase, config, options) {
 async function runSuite(options = {}) {
   const startedAt = new Date(options.nowMs ?? Date.now()).toISOString();
   const { config, configPath, configDir } = await loadSuiteConfig(options);
-  const tracesDir = path7.resolve(configDir, config.traces);
+  const tracesDir = path5.resolve(configDir, config.traces);
   const cases = [];
   const diagnostics = [];
   for (const suiteCase of config.cases) {
@@ -7811,8 +7833,9 @@ async function runSuite(options = {}) {
     skipped: cases.filter((item) => item.status === "skipped").length
   };
   const finishedAt = new Date(options.nowMs ?? Date.now()).toISOString();
-  const ok = summary.failed === 0 && summary.errors === 0;
-  const status = summary.errors > 0 ? "error" : summary.failed > 0 || !ok ? "fail" : "pass";
+  const allSkipped = cases.length > 0 && summary.skipped === cases.length;
+  const ok = summary.failed === 0 && summary.errors === 0 && !allSkipped;
+  const status = summary.errors > 0 ? "error" : allSkipped ? "fail" : summary.failed > 0 || !ok ? "fail" : "pass";
   return {
     ok,
     status,
@@ -8068,10 +8091,14 @@ function filterRunsForCohort(runs, options) {
 }
 
 // packages/core/src/cohort/compare.ts
-function compareNumber(metric, label, baseline, candidate, higherIsWorse = true) {
+function compareNumber(metric, label, baseline, candidate, higherIsWorse = true, maxRelativeDelta) {
   if (baseline === void 0 && candidate === void 0) return void 0;
   const delta = baseline !== void 0 && candidate !== void 0 ? candidate - baseline : void 0;
-  const regression = delta !== void 0 && (higherIsWorse && delta > 0 || !higherIsWorse && delta < 0);
+  let regression = delta !== void 0 && (higherIsWorse && delta > 0 || !higherIsWorse && delta < 0);
+  if (regression && maxRelativeDelta !== void 0 && baseline !== void 0 && baseline !== 0 && delta !== void 0) {
+    const relative = Math.abs(delta / baseline);
+    if (relative <= maxRelativeDelta) regression = false;
+  }
   return {
     metric,
     baseline,
@@ -8090,6 +8117,7 @@ function compareCohortAggregates(groups, options) {
   const baselineAgg = pickAggregate(groups, options.baseline, options.groupKey);
   const candidateAgg = pickAggregate(groups, options.candidate, options.groupKey);
   const comparisons = [];
+  const tolerance = options.maxRelativeDelta;
   for (const metric of options.metrics) {
     switch (metric) {
       case "errorRate": {
@@ -8097,7 +8125,9 @@ function compareCohortAggregates(groups, options) {
           metric,
           "Error rate",
           baselineAgg?.errorRate,
-          candidateAgg?.errorRate
+          candidateAgg?.errorRate,
+          true,
+          tolerance
         );
         if (item) comparisons.push(item);
         break;
@@ -8384,13 +8414,27 @@ async function analyzeCohort(runsInput, options) {
       ...new Set(groups.map((group) => group.groupKey))
     ].sort((a, b) => a.localeCompare(b));
     const items = [];
+    const minSample = options.tolerance?.minSampleSize ?? 1;
     for (const groupKey of groupKeys) {
+      const baselineGroup = groups.find(
+        (group) => group.groupKey === groupKey && group.cohortLabel === options.baseline
+      );
+      const candidateGroup = groups.find(
+        (group) => group.groupKey === groupKey && group.cohortLabel === options.candidate
+      );
+      if (baselineGroup && baselineGroup.runCount < minSample || candidateGroup && candidateGroup.runCount < minSample) {
+        warnings.push(
+          `Insufficient sample for group ${groupKey}: need at least ${minSample} run(s) per cohort.`
+        );
+        continue;
+      }
       items.push(
         ...compareCohortAggregates(groups, {
           baseline: options.baseline,
           candidate: options.candidate,
           metrics,
-          groupKey
+          groupKey,
+          maxRelativeDelta: options.tolerance?.maxRelativeDelta
         })
       );
     }
@@ -8791,6 +8835,9 @@ function validateOptions(options) {
   if (options.maxErrorRate !== void 0 && options.maxErrorRate < 0) {
     errors.push("--max-error-rate must be a non-negative percentage.");
   }
+  if (options.maxErrorRate !== void 0 && options.maxErrorRate > 100) {
+    errors.push("--max-error-rate must be at most 100.");
+  }
   if (options.maxP95DurationMs !== void 0 && options.maxP95DurationMs < 0) {
     errors.push("--max-p95-duration must be a non-negative millisecond value.");
   }
@@ -8798,7 +8845,7 @@ function validateOptions(options) {
 }
 function isConfigLoadError(error) {
   if (!(error instanceof Error)) return false;
-  const ext = path7.extname(error.message);
+  const ext = path5.extname(error.message);
   if (error.message.includes("Unsupported suite config extension")) return true;
   if (error.message.includes("TypeScript suite configs require")) return true;
   if (error.message.includes("No suite config found")) return true;
@@ -8997,6 +9044,6 @@ function renderGateReport(result, options = {}) {
   return renderGateSummaryMarkdown(result);
 }
 
-export { COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, Redactor, TraceDirectory, TraceReadError, TreeBuilder, __commonJS, __require, __toESM, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, compactAttributes3 as compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, enrichSessionRunRecord, escapeHtml, escapeMarkdown, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, isAgentInspectTrace, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToTraceEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, searchTraces, source_default, stableJson, summarizeObservedOutcomes, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, zeroKinds };
-//# sourceMappingURL=chunk-YS2I722C.mjs.map
-//# sourceMappingURL=chunk-YS2I722C.mjs.map
+export { COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, Redactor, TraceDirectory, TraceReadError, TreeBuilder, __commonJS, __require, __toESM, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, bundleRunAssetRelativePath, compactAttributes3 as compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, enrichSessionRunRecord, escapeHtml, escapeMarkdown, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, isAgentInspectTrace, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToTraceEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, source_default, stableJson, summarizeObservedOutcomes, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, zeroKinds };
+//# sourceMappingURL=chunk-M6PC5KUT.mjs.map
+//# sourceMappingURL=chunk-M6PC5KUT.mjs.map