agent-inspect 4.4.0 → 5.1.0

package/packages/core/dist/advanced.mjs

@@ -1,19 +1,26 @@
+import { runTraceChecks, createRunStatusRule, createToolUsageRule, createRunDurationRule, createLlmUsageRule, createObservedOutcomeRule } from './chunk-QLYOCCEF.mjs';
 export { MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, TERMINAL_INDENT, TraceDirectory, createInspector, createInspectorRuntime, formatTerminalName, getCurrentContext, getCurrentCorrelationMetadata, getCurrentDepth, getCurrentRunId, getCurrentRunName, getCurrentStepId, getIndent, getParentStepId, getTraceDirFromContext, getTraceSafetyFromContext, hasActiveContext, isAgentInspectEnabled, isSilentContext, maybeInspectRun, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, renderErrorLine, renderRunSummary, renderStepLine, resolveTraceDir, runWithContext, runWithStepContext } from './chunk-ZWA5RWMV.mjs';
 import { buildRunSummary, extractMetadata } from './chunk-3USJVBLA.mjs';
 export { buildRunSummary, buildRunTimeline, buildRunWhatSummary, extractMetadata, renderRunWhat, renderTimeline } from './chunk-3USJVBLA.mjs';
-import { parseObservationFilter, extractOutcomesFromTraceEvents } from './chunk-VFO76UH3.mjs';
+import { parseObservationFilter, extractOutcomesFromTraceEvents, extractOutcomesFromPersistedEvents } from './chunk-VFO76UH3.mjs';
 import './chunk-BT7CATSD.mjs';
+import { escapeHtml } from './chunk-CAWPF22J.mjs';
 import { resolveRedactionProfile, readTraceEventsFromFile } from './chunk-ULCGIRTC.mjs';
 export { DEFAULT_MAX_EVENT_BYTES, DEFAULT_MAX_METADATA_VALUE_LENGTH, DEFAULT_MAX_PREVIEW_LENGTH, getRunIdFromTraceFileName, initializeTraceFile, listTraceFiles, prepareMetadataForDisk, prepareTraceEventForDisk, readTraceEvents, readTraceFile, resolveRedactionProfile, resolveTraceSafetyOptions, serializeEvent, validateEvent, writeTraceEvent } from './chunk-ULCGIRTC.mjs';
 import { Redactor } from './chunk-VU6O5QAH.mjs';
+import { openTrace } from './chunk-X2FLDF7M.mjs';
+import './chunk-KNYL56KZ.mjs';
 import './chunk-S4YWKV4G.mjs';
-import { parseDuration, formatDuration, isTraceEvent } from './chunk-WRE4CNPF.mjs';
+import './chunk-E5F2LQCX.mjs';
+import { parseDuration, formatDuration, isTraceEvent, getTraceFilePath } from './chunk-WRE4CNPF.mjs';
 export { DEFAULT_TRACE_DIR_NAME, FALLBACK_TRACE_DIR, MAX_NAME_LENGTH, RUNS_DIR_NAME, createRunId, createStepId, ensureTraceDir, formatDuration, formatError, formatTimestamp, getDefaultTraceDir, getTraceFilePath, isStepStatus, isStepType, isTraceEvent, parseDuration, parseTraceJsonl, truncateName, unknownTraceFormatMessage, warn } from './chunk-WRE4CNPF.mjs';
 import './chunk-P366CCL2.mjs';
 import './chunk-7TGZLWEE.mjs';
 import { createReadStream } from 'fs';
 import { createInterface } from 'readline';
-import path from 'path';
+import path3 from 'path';
+import { access, readFile } from 'fs/promises';
+import { pathToFileURL } from 'url';
 
 // packages/core/src/trace-filter.ts
 function toLower(s) {
@@ -1318,12 +1325,12 @@ function buildCriticalPath(runs, handoffs) {
     handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
   );
   const ordered = [...runs].sort(compareRuns);
-  const path2 = [];
+  const path6 = [];
   const visited = /* @__PURE__ */ new Set();
   const pushRun = (run, confidence, source) => {
     if (visited.has(run.runId)) return;
     visited.add(run.runId);
-    path2.push({
+    path6.push({
       runId: run.runId,
       name: run.name,
       startedAt: run.startedAt,
@@ -1348,7 +1355,7 @@ function buildCriticalPath(runs, handoffs) {
     const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
     pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
   }
-  return path2;
+  return path6;
 }
 function metaRunIdMatches(run, token, runById) {
   const meta = extractSessionWorkflowMetadata(run.metadata);
@@ -1640,7 +1647,7 @@ function normalizeBundleOutputPath(out) {
   if (trimmed === "") {
     throw new Error("--out requires a non-empty path.");
   }
-  const resolved = path.resolve(trimmed);
+  const resolved = path3.resolve(trimmed);
   if (resolved.toLowerCase().endsWith(".zip")) {
     return resolved.slice(0, -4);
   }
@@ -1649,9 +1656,1085 @@ function normalizeBundleOutputPath(out) {
 function defaultBundleOutputPath(runIds) {
   const label = runIds.length === 1 ? runIds[0] : `multi-${runIds.length}`;
   const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
-  return path.resolve(`agent-inspect-bundle-${label}-${stamp}`);
+  return path3.resolve(`agent-inspect-bundle-${label}-${stamp}`);
 }
 
-export { SESSION_WORKFLOW_KEYS, aggregateBundleSafeStatus, aggregateSessionCheckResults, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, defaultBundleOutputPath, deriveSessionStatus, enrichSessionRunRecord, enrichSessionSummary, extractSessionWorkflowMetadata, filterMetasBySessionScope, filterTraces, groupSessionCohorts, isAgentInspectTrace, loadSessionRunRecords, loadTraceMetadataList, normalizeBundleOutputPath, parseDurationFilter, renderActivitySummaryHuman, renderTraceStats, resolveBundleRunIds, searchTraces, sessionKeyForRun, toMetadataSafeStatus, traceMetasToSessionRunRecords };
+// packages/core/src/suite/types.ts
+var DEFAULT_SUITE_CONFIG_NAMES = [
+  "agent-inspect.suite.json",
+  "agent-inspect.suite.js",
+  "agent-inspect.suite.mjs",
+  "agent-inspect.suite.cjs"
+];
+var DEFAULT_SUITE_ARTIFACTS_DIR = ".agent-inspect/suite-runs";
+function diagnostic(code, message, severity = "error", caseId) {
+  return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
+}
+function asString(value, label) {
+  if (value === void 0) return void 0;
+  if (typeof value !== "string" || value.trim() === "") {
+    throw new Error(`${label} must be a non-empty string.`);
+  }
+  return value.trim();
+}
+function asStringArray(value, label) {
+  if (value === void 0) return void 0;
+  if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
+    throw new Error(`${label} must be an array of strings.`);
+  }
+  return value;
+}
+function asPositiveNumber(value, label) {
+  if (value === void 0) return void 0;
+  if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
+    throw new Error(`${label} must be a non-negative number.`);
+  }
+  return value;
+}
+function validateCaseConfig(value, index) {
+  const diagnostics = [];
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    diagnostics.push(
+      diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
+    );
+    return { diagnostics };
+  }
+  const raw = value;
+  try {
+    const id = asString(raw.id, `cases[${index}].id`);
+    if (id === void 0) {
+      diagnostics.push(
+        diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
+      );
+      return { diagnostics };
+    }
+    const trace = asString(raw.trace, `cases[${index}].trace`);
+    const runId = asString(raw.runId, `cases[${index}].runId`);
+    const input = asString(raw.input, `cases[${index}].input`);
+    return {
+      caseConfig: {
+        id,
+        ...trace !== void 0 ? { trace } : {},
+        ...runId !== void 0 ? { runId } : {},
+        ...input !== void 0 ? { input } : {},
+        ...asStringArray(raw.requireTools, `cases[${index}].requireTools`) !== void 0 ? { requireTools: asStringArray(raw.requireTools, `cases[${index}].requireTools`) } : {},
+        ...asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) !== void 0 ? { forbidTools: asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) } : {},
+        ...asPositiveNumber(raw.maxDurationMs, `cases[${index}].maxDurationMs`) !== void 0 ? {
+          maxDurationMs: asPositiveNumber(
+            raw.maxDurationMs,
+            `cases[${index}].maxDurationMs`
+          )
+        } : {},
+        ...asStringArray(
+          raw.expectedObservations,
+          `cases[${index}].expectedObservations`
+        ) !== void 0 ? {
+          expectedObservations: asStringArray(
+            raw.expectedObservations,
+            `cases[${index}].expectedObservations`
+          )
+        } : {}
+      },
+      diagnostics
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    diagnostics.push(diagnostic("AI_SUITE_CONFIG_INVALID", message));
+    return { diagnostics };
+  }
+}
+function normalizeSuiteConfig(value) {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new Error("Suite config must export an object.");
+  }
+  const raw = value;
+  const name = asString(raw.name, "name");
+  const traces = asString(raw.traces, "traces");
+  if (name === void 0) throw new Error("name is required.");
+  if (traces === void 0) throw new Error("traces is required.");
+  if (!Array.isArray(raw.cases) || raw.cases.length === 0) {
+    throw new Error("cases must be a non-empty array.");
+  }
+  const cases = [];
+  for (let index = 0; index < raw.cases.length; index += 1) {
+    const { caseConfig, diagnostics } = validateCaseConfig(raw.cases[index], index);
+    if (diagnostics.length > 0) {
+      throw new Error(diagnostics.map((item) => item.message).join("; "));
+    }
+    if (caseConfig !== void 0) cases.push(caseConfig);
+  }
+  const ids = /* @__PURE__ */ new Set();
+  for (const suiteCase of cases) {
+    if (ids.has(suiteCase.id)) {
+      throw new Error(`Duplicate case id "${suiteCase.id}".`);
+    }
+    ids.add(suiteCase.id);
+  }
+  const redactionProfile = raw.redactionProfile === "local" || raw.redactionProfile === "share" || raw.redactionProfile === "strict" ? raw.redactionProfile : void 0;
+  if (raw.redactionProfile !== void 0 && redactionProfile === void 0) {
+    throw new Error('redactionProfile must be "local", "share", or "strict".');
+  }
+  const config = { name, traces, cases };
+  if (raw.checks !== void 0) {
+    if (typeof raw.checks !== "object" || Array.isArray(raw.checks)) {
+      throw new Error("checks must be an object.");
+    }
+    config.checks = raw.checks;
+  }
+  if (raw.eval !== void 0) {
+    if (typeof raw.eval !== "object" || Array.isArray(raw.eval)) {
+      throw new Error("eval must be an object.");
+    }
+    config.eval = raw.eval;
+  }
+  if (raw.artifacts !== void 0) {
+    if (typeof raw.artifacts !== "object" || Array.isArray(raw.artifacts)) {
+      throw new Error("artifacts must be an object.");
+    }
+    const outputDir = asString(
+      raw.artifacts.outputDir,
+      "artifacts.outputDir"
+    );
+    config.artifacts = outputDir !== void 0 ? { outputDir } : {};
+  }
+  if (raw.baseline !== void 0) {
+    const baseline = asString(raw.baseline, "baseline");
+    if (baseline !== void 0) config.baseline = baseline;
+  }
+  if (raw.candidate !== void 0) {
+    const candidate = asString(raw.candidate, "candidate");
+    if (candidate !== void 0) config.candidate = candidate;
+  }
+  if (redactionProfile !== void 0) config.redactionProfile = redactionProfile;
+  return config;
+}
+async function validateSuiteConfig(config, options) {
+  const diagnostics = [];
+  const tracesDir = path3.resolve(options.configDir, config.traces);
+  try {
+    await access(tracesDir);
+  } catch {
+    diagnostics.push(
+      diagnostic("AI_SUITE_CONFIG_INVALID", `traces directory not found: ${tracesDir}`)
+    );
+  }
+  for (const suiteCase of config.cases) {
+    if (suiteCase.input !== void 0) {
+      const inputPath = path3.resolve(options.configDir, suiteCase.input);
+      try {
+        await access(inputPath);
+      } catch {
+        diagnostics.push(
+          diagnostic(
+            "AI_SUITE_CONFIG_INVALID",
+            `input fixture not found: ${suiteCase.input}`,
+            "warning",
+            suiteCase.id
+          )
+        );
+      }
+    }
+  }
+  return { ok: diagnostics.every((item) => item.severity !== "error"), diagnostics };
+}
+
+// packages/core/src/suite/load.ts
+var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
+var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
+function diagnostic2(code, message) {
+  return { code, message, severity: "error" };
+}
+async function fileExists(filePath) {
+  try {
+    await access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+async function resolveSuiteConfigPath(options = {}) {
+  const cwd = path3.resolve(options.cwd ?? process.cwd());
+  if (options.configPath !== void 0 && options.configPath.trim() !== "") {
+    return path3.resolve(cwd, options.configPath.trim());
+  }
+  for (const name of DEFAULT_SUITE_CONFIG_NAMES) {
+    const candidate = path3.join(cwd, name);
+    if (await fileExists(candidate)) return candidate;
+  }
+  throw new Error(
+    `No suite config found. Create one with \`agent-inspect suite init\` or pass --config.`
+  );
+}
+async function loadSuiteConfig(options = {}) {
+  let configPath;
+  try {
+    configPath = await resolveSuiteConfigPath(options);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw Object.assign(new Error(message), {
+      diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
+    });
+  }
+  const extension = path3.extname(configPath);
+  if (TS_CONFIG_EXTENSIONS.has(extension)) {
+    const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
+    throw Object.assign(new Error(message), {
+      diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
+    });
+  }
+  if (!CONFIG_EXTENSIONS.has(extension)) {
+    const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
+    throw Object.assign(new Error(message), {
+      diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
+    });
+  }
+  try {
+    let raw;
+    if (extension === ".json") {
+      raw = JSON.parse(await readFile(configPath, "utf-8"));
+    } else {
+      const mod = await import(pathToFileURL(configPath).href);
+      raw = "default" in mod ? mod.default : mod;
+    }
+    const config = normalizeSuiteConfig(raw);
+    return {
+      config,
+      configPath,
+      configDir: path3.dirname(configPath)
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw Object.assign(new Error(message), {
+      diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
+    });
+  }
+}
+function defaultSuiteConfigTemplate() {
+  return {
+    name: "my-agent-suite",
+    traces: "./.agent-inspect",
+    cases: [
+      {
+        id: "example-case",
+        runId: "example-run",
+        requireTools: ["searchDocs"],
+        maxDurationMs: 3e4,
+        expectedObservations: ["answerReady"]
+      }
+    ],
+    checks: {
+      select: ["run.status"]
+    },
+    redactionProfile: "local",
+    artifacts: {
+      outputDir: ".agent-inspect/suite-runs"
+    }
+  };
+}
+async function exists(filePath) {
+  try {
+    await access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+async function resolveSuiteCaseTrace(suiteCase, options) {
+  if (suiteCase.trace !== void 0) {
+    const tracePath = path3.resolve(options.configDir, suiteCase.trace);
+    if (await exists(tracePath)) {
+      return { caseId: suiteCase.id, tracePath, missing: false };
+    }
+    return {
+      caseId: suiteCase.id,
+      tracePath,
+      missing: true,
+      reason: `trace file not found: ${suiteCase.trace}`
+    };
+  }
+  const runKey = suiteCase.runId ?? suiteCase.id;
+  const directPath = getTraceFilePath(runKey, options.tracesDir);
+  if (await exists(directPath)) {
+    return { caseId: suiteCase.id, tracePath: directPath, runId: runKey, missing: false };
+  }
+  const nestedPath = path3.join(options.tracesDir, `${path3.basename(runKey)}.jsonl`);
+  if (await exists(nestedPath)) {
+    return { caseId: suiteCase.id, tracePath: nestedPath, runId: runKey, missing: false };
+  }
+  return {
+    caseId: suiteCase.id,
+    runId: runKey,
+    missing: true,
+    reason: `no trace found for run id "${runKey}" under ${options.tracesDir}`
+  };
+}
+function diagnostic3(code, message, severity = "error", caseId) {
+  return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
+}
+function buildCaseRules(suiteCase, config) {
+  const rules = [];
+  const select = new Set(config.checks?.select ?? []);
+  if (select.has("run.status")) {
+    rules.push(createRunStatusRule());
+  }
+  const requiredTools = [
+    ...config.checks?.tool?.required ?? [],
+    ...suiteCase.requireTools ?? []
+  ];
+  const forbiddenTools = [
+    ...config.checks?.tool?.forbidden ?? [],
+    ...suiteCase.forbidTools ?? []
+  ];
+  if (requiredTools.length > 0 || forbiddenTools.length > 0) {
+    rules.push(
+      createToolUsageRule({
+        required: requiredTools.length > 0 ? requiredTools : void 0,
+        forbidden: forbiddenTools.length > 0 ? forbiddenTools : void 0
+      })
+    );
+    select.add("tool.usage");
+  }
+  const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? config.eval?.maxDurationMs;
+  if (maxDurationMs !== void 0) {
+    rules.push(createRunDurationRule({ maxDurationMs }));
+    select.add("run.duration");
+  }
+  const llm = config.checks?.llm;
+  if (llm?.allowedModels !== void 0 || llm?.maxTotalTokens !== void 0) {
+    rules.push(createLlmUsageRule(llm));
+    select.add("llm.usage");
+  }
+  if (select.has("outcome.status")) {
+    rules.push(createObservedOutcomeRule({ failOn: ["failed"] }));
+  }
+  return { rules, select: [...select] };
+}
+function outcomesFromRead(read) {
+  const persistedOutcomes = extractOutcomesFromPersistedEvents(
+    read.events.filter((event) => event.kind === "OUTCOME")
+  );
+  if (persistedOutcomes.length > 0) return persistedOutcomes;
+  const traceEvents = [];
+  for (const event of read.events) {
+    const legacy = event;
+    if (typeof legacy === "object" && legacy !== null && "event" in legacy && legacy.event === "outcome_observed") {
+      traceEvents.push(legacy);
+    }
+  }
+  return traceEvents.length > 0 ? extractOutcomesFromTraceEvents(traceEvents) : [];
+}
+function validateExpectedObservations(suiteCase, read) {
+  const expected = suiteCase.expectedObservations ?? [];
+  if (expected.length === 0) return { ok: true, diagnostics: [] };
+  const outcomes = outcomesFromRead(read);
+  const diagnostics = [];
+  for (const name of expected) {
+    const match = outcomes.find((outcome) => outcome.name === name);
+    if (!match) {
+      diagnostics.push(
+        diagnostic3(
+          "AI_SUITE_CASE_OBSERVATION_FAILED",
+          `Expected observation "${name}" was not recorded.`,
+          "error",
+          suiteCase.id
+        )
+      );
+      continue;
+    }
+    if (match.status !== "passed") {
+      diagnostics.push(
+        diagnostic3(
+          "AI_SUITE_CASE_OBSERVATION_FAILED",
+          `Observation "${name}" has status "${match.status}", expected "passed".`,
+          "error",
+          suiteCase.id
+        )
+      );
+    }
+  }
+  return { ok: diagnostics.length === 0, diagnostics };
+}
+async function runSuiteCase(suiteCase, config, options) {
+  const resolved = await resolveSuiteCaseTrace(suiteCase, options);
+  if (resolved.missing || resolved.tracePath === void 0) {
+    return {
+      id: suiteCase.id,
+      status: "skipped",
+      ...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
+      message: resolved.reason,
+      diagnostics: [
+        diagnostic3(
+          "AI_SUITE_CASE_TRACE_MISSING",
+          resolved.reason ?? "Trace not found.",
+          "warning",
+          suiteCase.id
+        )
+      ]
+    };
+  }
+  let read;
+  try {
+    read = await openTrace({ type: "file", path: resolved.tracePath });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      id: suiteCase.id,
+      status: "error",
+      tracePath: resolved.tracePath,
+      ...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
+      message,
+      diagnostics: [
+        diagnostic3("AI_SUITE_TRACE_UNREADABLE", message, "error", suiteCase.id)
+      ]
+    };
+  }
+  const { rules, select } = buildCaseRules(suiteCase, config);
+  const checkResult = rules.length > 0 ? runTraceChecks({ read }, { rules, select }) : {
+    ok: true,
+    status: "pass",
+    format: read.format,
+    findings: [],
+    diagnostics: []
+  };
+  const observationResult = validateExpectedObservations(suiteCase, read);
+  const diagnostics = [
+    ...checkResult.diagnostics.map(
+      (item) => diagnostic3(
+        "AI_SUITE_CASE_CHECK_FAILED",
+        item.message,
+        item.severity,
+        suiteCase.id
+      )
+    ),
+    ...checkResult.findings.filter((finding) => finding.status === "fail").map(
+      (finding) => diagnostic3(
+        "AI_SUITE_CASE_CHECK_FAILED",
+        finding.message,
+        finding.severity,
+        suiteCase.id
+      )
+    ),
+    ...observationResult.diagnostics
+  ];
+  const checkOk = checkResult.ok;
+  const observationsOk = observationResult.ok;
+  const ok = checkOk && observationsOk;
+  const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
+  return {
+    id: suiteCase.id,
+    status,
+    tracePath: resolved.tracePath,
+    ...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
+    checkOk,
+    observationsOk,
+    diagnostics,
+    ...ok ? {} : {
+      message: diagnostics.map((item) => item.message).join("; ") || checkResult.findings.map((item) => item.message).join("; ")
+    }
+  };
+}
+async function runSuite(options = {}) {
+  const startedAt = new Date(options.nowMs ?? Date.now()).toISOString();
+  const { config, configPath, configDir } = await loadSuiteConfig(options);
+  const tracesDir = path3.resolve(configDir, config.traces);
+  const cases = [];
+  const diagnostics = [];
+  for (const suiteCase of config.cases) {
+    cases.push(
+      await runSuiteCase(suiteCase, config, {
+        configDir,
+        tracesDir
+      })
+    );
+  }
+  const summary = {
+    passed: cases.filter((item) => item.status === "pass").length,
+    failed: cases.filter((item) => item.status === "fail").length,
+    errors: cases.filter((item) => item.status === "error").length,
+    skipped: cases.filter((item) => item.status === "skipped").length
+  };
+  const finishedAt = new Date(options.nowMs ?? Date.now()).toISOString();
+  const ok = summary.failed === 0 && summary.errors === 0;
+  const status = summary.errors > 0 ? "error" : summary.failed > 0 || !ok ? "fail" : "pass";
+  return {
+    ok,
+    status,
+    suiteName: config.name,
+    configPath,
+    tracesDir,
+    startedAt,
+    finishedAt,
+    summary,
+    cases,
+    diagnostics
+  };
+}
+
+// packages/core/src/suite/report.ts
+function statusLabel(status) {
+  return status.toUpperCase();
+}
+function renderSuiteReportMarkdown(result) {
+  const lines = [];
+  lines.push(`# Suite: ${result.suiteName}`);
+  lines.push("");
+  lines.push(`Status: **${statusLabel(result.status)}**`);
+  lines.push(`Config: \`${result.configPath}\``);
+  lines.push(`Traces: \`${result.tracesDir}\``);
+  lines.push(`Started: ${result.startedAt}`);
+  lines.push(`Finished: ${result.finishedAt}`);
+  lines.push("");
+  lines.push("## Summary");
+  lines.push(`- Passed: ${result.summary.passed}`);
+  lines.push(`- Failed: ${result.summary.failed}`);
+  lines.push(`- Errors: ${result.summary.errors}`);
+  lines.push(`- Skipped: ${result.summary.skipped}`);
+  lines.push("");
+  lines.push("## Cases");
+  for (const suiteCase of result.cases) {
+    lines.push(`### ${suiteCase.id} \u2014 ${statusLabel(suiteCase.status)}`);
+    if (suiteCase.tracePath !== void 0) {
+      lines.push(`- Trace: \`${suiteCase.tracePath}\``);
+    }
+    if (suiteCase.runId !== void 0) {
+      lines.push(`- Run: \`${suiteCase.runId}\``);
+    }
+    if (suiteCase.message !== void 0 && suiteCase.message.trim() !== "") {
+      lines.push(`- ${suiteCase.message}`);
+    }
+    if (suiteCase.diagnostics.length > 0) {
+      for (const item of suiteCase.diagnostics) {
+        lines.push(`- [${item.severity}] ${item.message}`);
+      }
+    }
+    lines.push("");
+  }
+  return lines.join("\n").trimEnd();
+}
+function renderSuiteReport(result, options = {}) {
+  const format = options.format ?? "markdown";
+  if (format === "json") {
+    return JSON.stringify(result, null, 2);
+  }
+  return renderSuiteReportMarkdown(result);
+}
+
+// packages/core/src/cohort/types.ts
+var COHORT_METRIC_IDS = [
+  "errorRate",
+  "duration",
+  "toolChoice",
+  "toolOrdering",
+  "llmCallCount",
+  "tokenUsage",
+  "retryCount",
+  "observationFailure",
+  "guardrailFailure",
+  "circuitViolation",
+  "redactionWarning"
+];
+
+// packages/core/src/cohort/compare.ts
+function compareNumber(metric, label, baseline, candidate, higherIsWorse = true) {
+  if (baseline === void 0 && candidate === void 0) return void 0;
+  const delta = baseline !== void 0 && candidate !== void 0 ? candidate - baseline : void 0;
+  const regression = delta !== void 0 && (higherIsWorse && delta > 0 || !higherIsWorse && delta < 0);
+  return {
+    metric,
+    baseline,
+    candidate,
+    delta,
+    regression,
+    message: `${label}: baseline=${baseline ?? "n/a"} candidate=${candidate ?? "n/a"}${delta !== void 0 ? ` (delta ${delta})` : ""}`
+  };
+}
+function pickAggregate(groups, cohortLabel, groupKey) {
+  return groups.find(
+    (group) => group.cohortLabel === cohortLabel && (groupKey === void 0 || group.groupKey === groupKey)
+  );
+}
+function compareCohortAggregates(groups, options) {
+  const baselineAgg = pickAggregate(groups, options.baseline, options.groupKey);
+  const candidateAgg = pickAggregate(groups, options.candidate, options.groupKey);
+  const comparisons = [];
+  for (const metric of options.metrics) {
+    switch (metric) {
+      case "errorRate": {
+        const item = compareNumber(
+          metric,
+          "Error rate",
+          baselineAgg?.errorRate,
+          candidateAgg?.errorRate
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "duration": {
+        const item = compareNumber(
+          metric,
+          "Average duration (ms)",
+          baselineAgg?.avgDurationMs,
+          candidateAgg?.avgDurationMs
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "llmCallCount": {
+        const item = compareNumber(
+          metric,
+          "Average LLM calls",
+          baselineAgg?.avgLlmCallCount,
+          candidateAgg?.avgLlmCallCount
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "tokenUsage": {
+        const item = compareNumber(
+          metric,
+          "Average token usage",
+          baselineAgg?.avgTokenUsage,
+          candidateAgg?.avgTokenUsage
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "retryCount": {
+        const item = compareNumber(
+          metric,
+          "Average retries",
+          baselineAgg?.avgRetryCount,
+          candidateAgg?.avgRetryCount
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "observationFailure": {
+        const item = compareNumber(
+          metric,
+          "Observation failure rate",
+          baselineAgg?.observationFailureRate,
+          candidateAgg?.observationFailureRate
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "toolChoice": {
+        const baselineValue = baselineAgg?.dominantToolChoice;
+        const candidateValue = candidateAgg?.dominantToolChoice;
+        comparisons.push({
+          metric,
+          baseline: baselineValue,
+          candidate: candidateValue,
+          delta: baselineValue === candidateValue ? "same" : "changed",
+          regression: baselineValue !== candidateValue,
+          message: `Tool choice: baseline=${baselineValue ?? "n/a"} candidate=${candidateValue ?? "n/a"}`
+        });
+        break;
+      }
+      case "toolOrdering": {
+        const baselineValue = baselineAgg?.toolOrderingSignature;
+        const candidateValue = candidateAgg?.toolOrderingSignature;
+        comparisons.push({
+          metric,
+          baseline: baselineValue,
+          candidate: candidateValue,
+          delta: baselineValue === candidateValue ? "same" : "changed",
+          regression: baselineValue !== candidateValue,
+          message: `Tool ordering: baseline=${baselineValue ?? "n/a"} candidate=${candidateValue ?? "n/a"}`
+        });
+        break;
+      }
+      case "guardrailFailure": {
+        const item = compareNumber(
+          metric,
+          "Guardrail failures",
+          baselineAgg?.avgGuardrailFailures,
+          candidateAgg?.avgGuardrailFailures
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "circuitViolation": {
+        const item = compareNumber(
+          metric,
+          "Circuit violations",
+          baselineAgg?.avgCircuitViolations,
+          candidateAgg?.avgCircuitViolations
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+      case "redactionWarning": {
+        const item = compareNumber(
+          metric,
+          "Redaction warnings",
+          baselineAgg?.avgRedactionWarnings,
+          candidateAgg?.avgRedactionWarnings
+        );
+        if (item) comparisons.push(item);
+        break;
+      }
+    }
+  }
+  return comparisons;
+}
+
+// packages/core/src/cohort/grouping.ts
+function parseCohortMetricList(value) {
+  if (value === void 0 || value.trim() === "") return [];
+  return value.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
+}
+function parseGroupBySpec(groupBy) {
+  const raw = (groupBy ?? "model").trim();
+  if (raw === "model") return { kind: "model" };
+  if (raw === "session") return { kind: "session" };
+  if (raw === "group") return { kind: "group" };
+  if (raw.startsWith("metadata.")) {
+    const metadataKey = raw.slice("metadata.".length).trim();
+    if (metadataKey === "") throw new Error("metadata group-by requires a key.");
+    return { kind: "metadata", metadataKey };
+  }
+  throw new Error(`Unsupported --group-by value: ${raw}`);
+}
+function metadataString(metadata, key) {
+  const value = metadata?.[key];
+  return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
+}
+function resolveRunGroupKey(run, groupBy) {
+  const metadata = run.metadata ?? {};
+  switch (groupBy.kind) {
+    case "model":
+      return metadataString(metadata, "model") ?? "unknown";
+    case "session":
+      return extractSessionWorkflowMetadata(metadata)?.sessionId ?? metadataString(metadata, "sessionId") ?? "__unscoped__";
+    case "group":
+      return extractSessionWorkflowMetadata(metadata)?.groupId ?? metadataString(metadata, "groupId") ?? "__unscoped__";
+    case "metadata":
+      return metadataString(metadata, groupBy.metadataKey) ?? "__missing__";
+    default:
+      return "unknown";
+  }
+}
+function resolveCohortLabel(run, cohortKey, baseline, candidate) {
+  const label = metadataString(run.metadata, cohortKey);
+  if (label === void 0) return void 0;
+  if (baseline !== void 0 && label === baseline) return baseline;
+  if (candidate !== void 0 && label === candidate) return candidate;
+  if (baseline === void 0 && candidate === void 0) return label;
+  return void 0;
+}
+function filterRunsForCohort(runs, options) {
+  const warnings = [];
+  if (options.baseline === void 0 && options.candidate === void 0) {
+    return { runs: [...runs], warnings };
+  }
+  const selected = [];
+  for (const run of runs) {
+    const label = resolveCohortLabel(
+      run,
+      options.cohortKey,
+      options.baseline,
+      options.candidate
+    );
+    if (label !== void 0) selected.push(run);
+  }
+  if (selected.length === 0) {
+    warnings.push(
+      `No runs matched baseline/candidate labels on metadata.${options.cohortKey}.`
+    );
+  }
+  return { runs: selected, warnings };
+}
+
+// packages/core/src/cohort/metrics.ts
+function asNumber(value) {
+  return typeof value === "number" && Number.isFinite(value) ? value : void 0;
+}
+function toolSteps(events) {
+  const ordering = [];
+  const choices = /* @__PURE__ */ new Set();
+  const sorted = [...events].filter((event) => event.event === "step_started").sort((a, b) => a.timestamp - b.timestamp);
+  for (const event of sorted) {
+    const step = event;
+    if (step.type !== "tool") continue;
+    const name = typeof step.metadata?.toolName === "string" ? step.metadata.toolName : step.name;
+    ordering.push(name);
+    choices.add(name);
+  }
+  return { choices: [...choices].sort(), ordering };
+}
+async function computeCohortRunMetrics(input) {
+  const events = await readTraceEventsFromFile(input.filePath);
+  const summary = buildRunSummary(events);
+  const tools = toolSteps(events);
+  const outcomes = extractOutcomesFromTraceEvents(events);
+  const observationFailures = outcomes.filter((item) => item.status === "failed").length;
+  const metadata = input.metadata ?? {};
+  const retryCount = asNumber(metadata.attempt) !== void 0 && asNumber(metadata.attempt) > 1 ? asNumber(metadata.attempt) - 1 : typeof metadata.retryOf === "string" ? 1 : 0;
+  return {
+    runId: input.runId,
+    ...input.cohortLabel !== void 0 ? { cohortLabel: input.cohortLabel } : {},
+    groupKey: input.groupKey,
+    status: summary.status,
+    error: summary.status === "error",
+    durationMs: summary.durationMs ?? input.durationMs,
+    llmCallCount: summary.llmSteps,
+    tokenUsageTotal: summary.totalTokens?.total,
+    retryCount,
+    observationFailures,
+    guardrailFailures: asNumber(metadata.guardrailFailures) ?? 0,
+    circuitViolations: asNumber(metadata.circuitViolations) ?? 0,
+    redactionWarnings: asNumber(metadata.redactionWarnings) ?? 0,
+    toolChoices: tools.choices,
+    toolOrdering: tools.ordering
+  };
+}
+function percentile2(values, p) {
+  if (values.length === 0) return void 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const idx = Math.min(
+    sorted.length - 1,
+    Math.max(0, Math.ceil(p / 100 * sorted.length) - 1)
+  );
+  return sorted[idx];
+}
+function dominantToolChoice(runs) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const run of runs) {
+    const signature = run.toolChoices.join(",");
+    if (signature === "") continue;
+    counts.set(signature, (counts.get(signature) ?? 0) + 1);
+  }
+  let best;
+  let bestCount = 0;
+  for (const [key, count] of counts) {
+    if (count > bestCount) {
+      best = key;
+      bestCount = count;
+    }
+  }
+  return best;
+}
+function orderingSignature(runs) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const run of runs) {
+    const signature = run.toolOrdering.join(">");
+    if (signature === "") continue;
+    counts.set(signature, (counts.get(signature) ?? 0) + 1);
+  }
+  let best;
+  let bestCount = 0;
+  for (const [key, count] of counts) {
+    if (count > bestCount) {
+      best = key;
+      bestCount = count;
+    }
+  }
+  return best;
+}
+function aggregateCohortMetrics(runs, groupKey, cohortLabel) {
+  const durations = runs.map((run) => run.durationMs).filter((value) => typeof value === "number");
+  const tokenValues = runs.map((run) => run.tokenUsageTotal).filter((value) => typeof value === "number");
+  const errors = runs.filter((run) => run.error).length;
+  const observationFailures = runs.reduce((sum, run) => sum + run.observationFailures, 0);
+  return {
+    groupKey,
+    ...cohortLabel !== void 0 ? { cohortLabel } : {},
+    runCount: runs.length,
+    errorRate: runs.length > 0 ? errors / runs.length : 0,
+    avgDurationMs: durations.length > 0 ? durations.reduce((sum, value) => sum + value, 0) / durations.length : void 0,
+    p95DurationMs: percentile2(durations, 95),
+    avgLlmCallCount: runs.length > 0 ? runs.reduce((sum, run) => sum + run.llmCallCount, 0) / runs.length : 0,
+    avgTokenUsage: tokenValues.length > 0 ? tokenValues.reduce((sum, value) => sum + value, 0) / tokenValues.length : void 0,
+    avgRetryCount: runs.length > 0 ? runs.reduce((sum, run) => sum + run.retryCount, 0) / runs.length : 0,
+    observationFailureRate: runs.length > 0 ? observationFailures / runs.length : 0,
+    avgGuardrailFailures: runs.length > 0 ? runs.reduce((sum, run) => sum + run.guardrailFailures, 0) / runs.length : 0,
+    avgCircuitViolations: runs.length > 0 ? runs.reduce((sum, run) => sum + run.circuitViolations, 0) / runs.length : 0,
+    avgRedactionWarnings: runs.length > 0 ? runs.reduce((sum, run) => sum + run.redactionWarnings, 0) / runs.length : 0,
+    dominantToolChoice: dominantToolChoice(runs),
+    toolOrderingSignature: orderingSignature(runs)
+  };
+}
+
+// packages/core/src/cohort/analyze.ts
+var DEFAULT_METRICS = [
+  "errorRate",
+  "duration",
+  "toolChoice",
+  "observationFailure"
+];
+function normalizeMetrics(metrics) {
+  if (metrics === void 0 || metrics.length === 0) return [...DEFAULT_METRICS];
+  const allowed = new Set(COHORT_METRIC_IDS);
+  return metrics.filter((metric) => allowed.has(metric));
+}
+async function analyzeCohort(runsInput, options) {
+  const cohortKey = options.cohortKey ?? "cohort";
+  const groupBySpec = parseGroupBySpec(options.groupBy);
+  const metrics = normalizeMetrics(options.metrics);
+  const { runs: filteredRuns, warnings } = filterRunsForCohort(runsInput, {
+    cohortKey,
+    baseline: options.baseline,
+    candidate: options.candidate
+  });
+  const runMetrics = [];
+  for (const run of filteredRuns) {
+    if (run.filePath === void 0) continue;
+    const cohortLabel = resolveCohortLabel(
+      run,
+      cohortKey,
+      options.baseline,
+      options.candidate
+    );
+    runMetrics.push(
+      await computeCohortRunMetrics({
+        runId: run.runId,
+        filePath: run.filePath,
+        metadata: run.metadata,
+        status: run.status,
+        durationMs: run.durationMs,
+        groupKey: resolveRunGroupKey(run, groupBySpec),
+        cohortLabel
+      })
+    );
+  }
+  const groupMap = /* @__PURE__ */ new Map();
+  for (const run of runMetrics) {
+    const key = `${run.cohortLabel ?? "*"}::${run.groupKey}`;
+    const bucket = groupMap.get(key) ?? [];
+    bucket.push(run);
+    groupMap.set(key, bucket);
+  }
+  const groups = [...groupMap.entries()].sort(([a], [b]) => a.localeCompare(b)).map(
+    ([, bucket]) => aggregateCohortMetrics(
+      bucket,
+      bucket[0].groupKey,
+      bucket[0]?.cohortLabel
+    )
+  );
+  const comparisons = options.baseline !== void 0 && options.candidate !== void 0 ? (() => {
+    const groupKeys = [
+      ...new Set(groups.map((group) => group.groupKey))
+    ].sort((a, b) => a.localeCompare(b));
+    const items = [];
+    for (const groupKey of groupKeys) {
+      items.push(
+        ...compareCohortAggregates(groups, {
+          baseline: options.baseline,
+          candidate: options.candidate,
+          metrics,
+          groupKey
+        })
+      );
+    }
+    return items;
+  })() : [];
+  const regression = comparisons.some((item) => item.regression);
+  return {
+    ok: !regression,
+    traceDir: options.traceDir,
+    ...options.baseline !== void 0 ? { baseline: options.baseline } : {},
+    ...options.candidate !== void 0 ? { candidate: options.candidate } : {},
+    cohortKey,
+    groupBy: options.groupBy ?? "model",
+    metrics,
+    groups,
+    comparisons,
+    runs: runMetrics,
+    warnings
+  };
+}
+
+// packages/core/src/cohort/render.ts
+function formatRate(value) {
+  if (value === void 0) return "n/a";
+  return `${(value * 100).toFixed(1)}%`;
+}
+function renderCohortSummaryMarkdown(result) {
+  const lines = [];
+  lines.push("# Cohort analysis");
+  lines.push("");
+  lines.push(`Trace directory: \`${result.traceDir}\``);
+  lines.push(`Group by: \`${result.groupBy}\``);
+  if (result.baseline !== void 0 && result.candidate !== void 0) {
+    lines.push(
+      `Baseline/Candidate key: \`${result.cohortKey}\` (${result.baseline} vs ${result.candidate})`
+    );
+  }
+  lines.push(`Status: **${result.ok ? "PASS" : "REGRESSION"}**`);
+  lines.push("");
+  if (result.warnings.length > 0) {
+    lines.push("## Warnings");
+    for (const warning of result.warnings) lines.push(`- ${warning}`);
+    lines.push("");
+  }
+  lines.push("## Groups");
+  for (const group of result.groups) {
+    lines.push(
+      `### ${group.cohortLabel ?? "all"} / ${group.groupKey} (${group.runCount} runs)`
+    );
+    lines.push(`- Error rate: ${formatRate(group.errorRate)}`);
+    if (group.avgDurationMs !== void 0) {
+      lines.push(`- Avg duration: ${Math.round(group.avgDurationMs)} ms`);
+    }
+    if (group.dominantToolChoice !== void 0) {
+      lines.push(`- Dominant tools: ${group.dominantToolChoice}`);
+    }
+    lines.push(
+      `- Observation failure rate: ${formatRate(group.observationFailureRate)}`
+    );
+    lines.push("");
+  }
+  if (result.comparisons.length > 0) {
+    lines.push("## Comparisons");
+    for (const comparison of result.comparisons) {
+      const flag = comparison.regression ? " **REGRESSION**" : "";
+      lines.push(`- ${comparison.message}${flag}`);
+    }
+    lines.push("");
+  }
+  return lines.join("\n").trimEnd();
+}
+function renderCohortReportHtml(result) {
+  const rows = result.groups.map(
+    (group) => `<tr><td>${escapeHtml(group.cohortLabel ?? "all")}</td><td>${escapeHtml(group.groupKey)}</td><td>${group.runCount}</td><td>${escapeHtml(formatRate(group.errorRate))}</td><td>${group.avgDurationMs !== void 0 ? Math.round(group.avgDurationMs) : "n/a"}</td></tr>`
+  ).join("");
+  const comparisons = result.comparisons.map(
+    (item) => `<li>${escapeHtml(item.message)}${item.regression ? " <strong>REGRESSION</strong>" : ""}</li>`
+  ).join("");
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <title>Cohort report</title>
+  <style>
+    body { font-family: system-ui, sans-serif; margin: 2rem; }
+    table { border-collapse: collapse; width: 100%; }
+    th, td { border: 1px solid #ddd; padding: 0.5rem; text-align: left; }
+    th { background: #f6f6f6; }
+  </style>
+</head>
+<body>
+  <h1>Cohort analysis</h1>
+  <p>Status: <strong>${result.ok ? "PASS" : "REGRESSION"}</strong></p>
+  <p>Trace directory: <code>${escapeHtml(result.traceDir)}</code></p>
+  <h2>Groups</h2>
+  <table>
+    <thead><tr><th>Cohort</th><th>Group</th><th>Runs</th><th>Error rate</th><th>Avg duration (ms)</th></tr></thead>
+    <tbody>${rows}</tbody>
+  </table>
+  ${result.comparisons.length > 0 ? `<h2>Comparisons</h2><ul>${comparisons}</ul>` : ""}
+</body>
+</html>`;
+}
+function renderCohortReport(result, options = {}) {
+  const format = options.format ?? "markdown";
+  if (format === "json") return JSON.stringify(result, null, 2);
+  if (format === "html") return renderCohortReportHtml(result);
+  return renderCohortSummaryMarkdown(result);
+}
+
+export { COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, DEFAULT_SUITE_CONFIG_NAMES, SESSION_WORKFLOW_KEYS, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, compareCohortAggregates, defaultBundleOutputPath, defaultSuiteConfigTemplate, deriveSessionStatus, enrichSessionRunRecord, enrichSessionSummary, extractSessionWorkflowMetadata, filterMetasBySessionScope, filterTraces, groupSessionCohorts, isAgentInspectTrace, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, normalizeBundleOutputPath, normalizeSuiteConfig, parseCohortMetricList, parseDurationFilter, parseGroupBySpec, renderActivitySummaryHuman, renderCohortReport, renderCohortSummaryMarkdown, renderSuiteReport, renderSuiteReportMarkdown, renderTraceStats, resolveBundleRunIds, resolveSuiteCaseTrace, resolveSuiteConfigPath, runSuite, searchTraces, sessionKeyForRun, toMetadataSafeStatus, traceMetasToSessionRunRecords, validateSuiteConfig };
 //# sourceMappingURL=advanced.mjs.map
 //# sourceMappingURL=advanced.mjs.map