@agent-inspect/viewer 6.9.0 → 6.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -1,7 +1,7 @@
1
1
  'use strict';
2
2
 
3
3
  var http = require('http');
4
- var path13 = require('path');
4
+ var path15 = require('path');
5
5
  var async_hooks = require('async_hooks');
6
6
  require('crypto');
7
7
  var promises = require('fs/promises');
@@ -14,7 +14,7 @@ var url = require('url');
14
14
 
15
15
  function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
16
16
 
17
- var path13__default = /*#__PURE__*/_interopDefault(path13);
17
+ var path15__default = /*#__PURE__*/_interopDefault(path15);
18
18
  var os__default = /*#__PURE__*/_interopDefault(os);
19
19
 
20
20
  // packages/viewer/src/server.ts
@@ -525,7 +525,7 @@ function persistedInspectEventsToTraceEvents(events, options) {
525
525
  }
526
526
  var DEFAULT_TRACE_DIR_NAME = ".agent-inspect";
527
527
  var RUNS_DIR_NAME = "runs";
528
- var FALLBACK_TRACE_DIR = path13__default.default.join(
528
+ var FALLBACK_TRACE_DIR = path15__default.default.join(
529
529
  os__default.default.tmpdir(),
530
530
  "agent-inspect",
531
531
  RUNS_DIR_NAME
@@ -540,7 +540,7 @@ function getDefaultTraceDir() {
540
540
  if (typeof home !== "string" || home.trim() === "") {
541
541
  return FALLBACK_TRACE_DIR;
542
542
  }
543
- return path13__default.default.join(home, DEFAULT_TRACE_DIR_NAME, RUNS_DIR_NAME);
543
+ return path15__default.default.join(home, DEFAULT_TRACE_DIR_NAME, RUNS_DIR_NAME);
544
544
  } catch {
545
545
  return FALLBACK_TRACE_DIR;
546
546
  }
@@ -548,11 +548,11 @@ function getDefaultTraceDir() {
548
548
  function getTraceFilePath(runId, traceDir) {
549
549
  const baseDir = traceDir ?? getDefaultTraceDir();
550
550
  let safeId = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "run_unknown";
551
- safeId = path13__default.default.basename(safeId);
551
+ safeId = path15__default.default.basename(safeId);
552
552
  if (safeId === "" || safeId === "." || safeId === "..") {
553
553
  safeId = "run_unknown";
554
554
  }
555
- return path13__default.default.join(baseDir, `${safeId}.jsonl`);
555
+ return path15__default.default.join(baseDir, `${safeId}.jsonl`);
556
556
  }
557
557
  function formatError(error) {
558
558
  if (error instanceof Error) {
@@ -830,7 +830,7 @@ var TraceDirectory = class {
830
830
  this.#dir = resolveTraceDir(options);
831
831
  }
832
832
  getPath(filename) {
833
- return filename ? path13__default.default.join(this.#dir, filename) : this.#dir;
833
+ return filename ? path15__default.default.join(this.#dir, filename) : this.#dir;
834
834
  }
835
835
  async list() {
836
836
  try {
@@ -857,7 +857,7 @@ function parseIsoToMs2(value) {
857
857
  }
858
858
  async function extractMetadata(filePath, _quickScan) {
859
859
  const stats = await promises.stat(filePath);
860
- let runIdFromFile = path13__default.default.basename(filePath);
860
+ let runIdFromFile = path15__default.default.basename(filePath);
861
861
  if (runIdFromFile.endsWith(".jsonl")) {
862
862
  runIdFromFile = runIdFromFile.slice(0, -".jsonl".length);
863
863
  }
@@ -1467,12 +1467,12 @@ function buildCriticalPath(runs, handoffs) {
1467
1467
  handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
1468
1468
  );
1469
1469
  const ordered = [...runs].sort(compareRuns);
1470
- const path16 = [];
1470
+ const path18 = [];
1471
1471
  const visited = /* @__PURE__ */ new Set();
1472
1472
  const pushRun = (run, confidence, source) => {
1473
1473
  if (visited.has(run.runId)) return;
1474
1474
  visited.add(run.runId);
1475
- path16.push({
1475
+ path18.push({
1476
1476
  runId: run.runId,
1477
1477
  name: run.name,
1478
1478
  startedAt: run.startedAt,
@@ -1497,7 +1497,7 @@ function buildCriticalPath(runs, handoffs) {
1497
1497
  const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
1498
1498
  pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
1499
1499
  }
1500
- return path16;
1500
+ return path18;
1501
1501
  }
1502
1502
  function metaRunIdMatches(run, token, runById) {
1503
1503
  const meta = extractSessionWorkflowMetadata(run.metadata);
@@ -1574,524 +1574,915 @@ function buildSessionIndex(inputRuns, options = {}) {
1574
1574
  };
1575
1575
  }
1576
1576
 
1577
- // packages/core/src/suite/types.ts
1578
- var DEFAULT_SUITE_CONFIG_NAMES = [
1579
- "agent-inspect.suite.json",
1580
- "agent-inspect.suite.js",
1581
- "agent-inspect.suite.mjs",
1582
- "agent-inspect.suite.cjs"
1583
- ];
1584
- function diagnostic(code, message, severity = "error", caseId) {
1585
- return { code, message, severity, ...{} };
1586
- }
1587
- function asString(value, label) {
1588
- if (value === void 0) return void 0;
1589
- if (typeof value !== "string" || value.trim() === "") {
1590
- throw new Error(`${label} must be a non-empty string.`);
1577
+ // packages/core/src/exporters/helpers.ts
1578
+ function sortKeysDeep(input) {
1579
+ if (input === null || typeof input !== "object") return input;
1580
+ if (Array.isArray(input)) return input.map(sortKeysDeep);
1581
+ const o = input;
1582
+ const out = {};
1583
+ for (const k of Object.keys(o).sort()) {
1584
+ out[k] = sortKeysDeep(o[k]);
1591
1585
  }
1592
- return value.trim();
1586
+ return out;
1593
1587
  }
1594
- function asStringArray(value, label) {
1595
- if (value === void 0) return void 0;
1596
- if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
1597
- throw new Error(`${label} must be an array of strings.`);
1598
- }
1599
- return value;
1588
+ function stableJson(value, pretty) {
1589
+ const sorted = sortKeysDeep(value);
1590
+ return JSON.stringify(sorted);
1600
1591
  }
1601
- function asPositiveNumber(value, label) {
1602
- if (value === void 0) return void 0;
1603
- if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
1604
- throw new Error(`${label} must be a non-negative number.`);
1605
- }
1606
- return value;
1592
+
1593
+ // packages/core/src/diff/comparable.ts
1594
+ function extractOutputPreview(meta) {
1595
+ if (meta === void 0) return void 0;
1596
+ if ("outputPreview" in meta) return meta.outputPreview;
1597
+ if ("resultPreview" in meta) return meta.resultPreview;
1598
+ return void 0;
1607
1599
  }
1608
- function validateCaseConfig(value, index) {
1609
- const diagnostics = [];
1610
- if (typeof value !== "object" || value === null || Array.isArray(value)) {
1611
- diagnostics.push(
1612
- diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
1613
- );
1614
- return { diagnostics };
1615
- }
1616
- const raw = value;
1617
- try {
1618
- const id = asString(raw.id, `cases[${index}].id`);
1619
- if (id === void 0) {
1620
- diagnostics.push(
1621
- diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
1622
- );
1623
- return { diagnostics };
1624
- }
1625
- const trace = asString(raw.trace, `cases[${index}].trace`);
1626
- const runId = asString(raw.runId, `cases[${index}].runId`);
1627
- const input = asString(raw.input, `cases[${index}].input`);
1628
- return {
1629
- caseConfig: {
1630
- id,
1631
- ...trace !== void 0 ? { trace } : {},
1632
- ...runId !== void 0 ? { runId } : {},
1633
- ...input !== void 0 ? { input } : {},
1634
- ...asStringArray(raw.requireTools, `cases[${index}].requireTools`) !== void 0 ? { requireTools: asStringArray(raw.requireTools, `cases[${index}].requireTools`) } : {},
1635
- ...asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) !== void 0 ? { forbidTools: asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) } : {},
1636
- ...asPositiveNumber(raw.maxDurationMs, `cases[${index}].maxDurationMs`) !== void 0 ? {
1637
- maxDurationMs: asPositiveNumber(
1638
- raw.maxDurationMs,
1639
- `cases[${index}].maxDurationMs`
1640
- )
1641
- } : {},
1642
- ...asStringArray(
1643
- raw.expectedObservations,
1644
- `cases[${index}].expectedObservations`
1645
- ) !== void 0 ? {
1646
- expectedObservations: asStringArray(
1647
- raw.expectedObservations,
1648
- `cases[${index}].expectedObservations`
1649
- )
1650
- } : {}
1651
- },
1652
- diagnostics
1653
- };
1654
- } catch (error) {
1655
- const message = error instanceof Error ? error.message : String(error);
1656
- diagnostics.push(diagnostic("AI_SUITE_CONFIG_INVALID", message));
1657
- return { diagnostics };
1658
- }
1600
+ function mapStepStatus(s) {
1601
+ if (s === void 0) return "running";
1602
+ return s;
1659
1603
  }
1660
- function normalizeSuiteConfig(value) {
1661
- if (typeof value !== "object" || value === null || Array.isArray(value)) {
1662
- throw new Error("Suite config must export an object.");
1663
- }
1664
- const raw = value;
1665
- const name = asString(raw.name, "name");
1666
- const traces = asString(raw.traces, "traces");
1667
- if (name === void 0) throw new Error("name is required.");
1668
- if (traces === void 0) throw new Error("traces is required.");
1669
- if (!Array.isArray(raw.cases) || raw.cases.length === 0) {
1670
- throw new Error("cases must be a non-empty array.");
1604
+ function manualTraceEventsToComparableRun(events) {
1605
+ const started = events.find((e) => e.event === "run_started");
1606
+ if (!started || started.event !== "run_started") {
1607
+ throw new Error("Invalid trace: missing run_started");
1671
1608
  }
1672
- const cases = [];
1673
- for (let index = 0; index < raw.cases.length; index += 1) {
1674
- const { caseConfig, diagnostics } = validateCaseConfig(raw.cases[index], index);
1675
- if (diagnostics.length > 0) {
1676
- throw new Error(diagnostics.map((item) => item.message).join("; "));
1677
- }
1678
- if (caseConfig !== void 0) cases.push(caseConfig);
1609
+ const rs = started;
1610
+ const runId = rs.runId;
1611
+ const completedAll = events.filter((e) => e.event === "run_completed");
1612
+ const lastCompleted = completedAll[completedAll.length - 1];
1613
+ let runStatus;
1614
+ if (lastCompleted === void 0) runStatus = "running";
1615
+ else runStatus = lastCompleted.status;
1616
+ const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
1617
+ const steps = /* @__PURE__ */ new Map();
1618
+ let order = 0;
1619
+ for (const e of events) {
1620
+ if (e.event !== "step_started") continue;
1621
+ const s = e;
1622
+ const meta = s.metadata ? { ...s.metadata } : void 0;
1623
+ steps.set(s.stepId, {
1624
+ id: s.stepId,
1625
+ parentId: s.parentId,
1626
+ name: s.name,
1627
+ type: s.type,
1628
+ order: order++,
1629
+ timestamp: s.timestamp,
1630
+ metadata: meta
1631
+ });
1679
1632
  }
1680
- const ids = /* @__PURE__ */ new Set();
1681
- for (const suiteCase of cases) {
1682
- if (ids.has(suiteCase.id)) {
1683
- throw new Error(`Duplicate case id "${suiteCase.id}".`);
1633
+ for (const e of events) {
1634
+ if (e.event !== "step_completed") continue;
1635
+ const acc = steps.get(e.stepId);
1636
+ if (!acc) continue;
1637
+ acc.status = e.status;
1638
+ acc.durationMs = e.durationMs;
1639
+ if (e.error?.message) acc.errorMsg = e.error.message;
1640
+ const extra = e;
1641
+ if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
1642
+ acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
1684
1643
  }
1685
- ids.add(suiteCase.id);
1686
1644
  }
1687
- const redactionProfile = raw.redactionProfile === "local" || raw.redactionProfile === "share" || raw.redactionProfile === "strict" ? raw.redactionProfile : void 0;
1688
- if (raw.redactionProfile !== void 0 && redactionProfile === void 0) {
1689
- throw new Error('redactionProfile must be "local", "share", or "strict".');
1690
- }
1691
- const config = { name, traces, cases };
1692
- if (raw.checks !== void 0) {
1693
- if (typeof raw.checks !== "object" || Array.isArray(raw.checks)) {
1694
- throw new Error("checks must be an object.");
1645
+ const nodes = /* @__PURE__ */ new Map();
1646
+ for (const acc of steps.values()) {
1647
+ let meta = acc.metadata ? { ...acc.metadata } : void 0;
1648
+ if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
1649
+ meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
1695
1650
  }
1696
- config.checks = raw.checks;
1697
- }
1698
- if (raw.eval !== void 0) {
1699
- if (typeof raw.eval !== "object" || Array.isArray(raw.eval)) {
1700
- throw new Error("eval must be an object.");
1651
+ const outputPreview = extractOutputPreview(meta);
1652
+ if (meta !== void 0 && ("outputPreview" in meta || "resultPreview" in meta)) {
1653
+ delete meta.outputPreview;
1654
+ delete meta.resultPreview;
1701
1655
  }
1702
- config.eval = raw.eval;
1656
+ const sc = {
1657
+ id: acc.id,
1658
+ name: acc.name,
1659
+ type: acc.type,
1660
+ status: mapStepStatus(acc.status),
1661
+ durationMs: acc.durationMs,
1662
+ error: acc.errorMsg,
1663
+ metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
1664
+ outputPreview,
1665
+ children: []
1666
+ };
1667
+ nodes.set(acc.id, sc);
1703
1668
  }
1704
- if (raw.artifacts !== void 0) {
1705
- if (typeof raw.artifacts !== "object" || Array.isArray(raw.artifacts)) {
1706
- throw new Error("artifacts must be an object.");
1669
+ const roots = [];
1670
+ const sortByOrder = (a, b) => {
1671
+ const oa = steps.get(a.id)?.order ?? 0;
1672
+ const ob = steps.get(b.id)?.order ?? 0;
1673
+ return oa - ob;
1674
+ };
1675
+ for (const acc of steps.values()) {
1676
+ const node = nodes.get(acc.id);
1677
+ if (acc.parentId !== void 0 && nodes.has(acc.parentId)) {
1678
+ nodes.get(acc.parentId).children.push(node);
1679
+ } else {
1680
+ roots.push(node);
1707
1681
  }
1708
- const outputDir = asString(
1709
- raw.artifacts.outputDir,
1710
- "artifacts.outputDir"
1711
- );
1712
- config.artifacts = outputDir !== void 0 ? { outputDir } : {};
1713
- }
1714
- if (raw.baseline !== void 0) {
1715
- const baseline = asString(raw.baseline, "baseline");
1716
- if (baseline !== void 0) config.baseline = baseline;
1717
1682
  }
1718
- if (raw.candidate !== void 0) {
1719
- const candidate = asString(raw.candidate, "candidate");
1720
- if (candidate !== void 0) config.candidate = candidate;
1683
+ roots.sort(sortByOrder);
1684
+ for (const n of nodes.values()) {
1685
+ n.children.sort(sortByOrder);
1721
1686
  }
1722
- if (redactionProfile !== void 0) config.redactionProfile = redactionProfile;
1723
- return config;
1687
+ return {
1688
+ runId,
1689
+ name: rs.name,
1690
+ status: runStatus,
1691
+ durationMs,
1692
+ steps: roots
1693
+ };
1724
1694
  }
1725
1695
 
1726
- // packages/core/src/suite/load.ts
1727
- var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
1728
- var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
1729
- function diagnostic2(code, message) {
1730
- return { code, message, severity: "error" };
1696
+ // packages/core/src/diff/engine.ts
1697
+ var DEFAULT_THRESHOLD_MS = 0;
1698
+ function pathSeg(step, index) {
1699
+ return { index, name: step.name, stepId: step.id };
1731
1700
  }
1732
- async function fileExists(filePath) {
1733
- try {
1734
- await promises.access(filePath);
1735
- return true;
1736
- } catch {
1737
- return false;
1738
- }
1701
+ function buildPath(segments) {
1702
+ return { path: [...segments] };
1739
1703
  }
1740
- async function resolveSuiteConfigPath(options = {}) {
1741
- const cwd = path13__default.default.resolve(options.cwd ?? process.cwd());
1742
- if (options.configPath !== void 0 && options.configPath.trim() !== "") {
1743
- return path13__default.default.resolve(cwd, options.configPath.trim());
1744
- }
1745
- for (const name of DEFAULT_SUITE_CONFIG_NAMES) {
1746
- const candidate = path13__default.default.join(cwd, name);
1747
- if (await fileExists(candidate)) return candidate;
1704
+ function pairSteps(left, right) {
1705
+ const usedRight = /* @__PURE__ */ new Set();
1706
+ const pairs = [];
1707
+ for (let i = 0; i < left.length; i++) {
1708
+ const L = left[i];
1709
+ let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
1710
+ if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
1711
+ const cand = right[i];
1712
+ if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
1713
+ R = cand;
1714
+ }
1715
+ }
1716
+ if (R === void 0) {
1717
+ R = right.find(
1718
+ (r) => !usedRight.has(r.id) && r.name === L.name && (r.type ?? "") === (L.type ?? "")
1719
+ );
1720
+ }
1721
+ if (R !== void 0) {
1722
+ usedRight.add(R.id);
1723
+ pairs.push([L, R]);
1724
+ } else {
1725
+ pairs.push([L, void 0]);
1726
+ }
1748
1727
  }
1749
- throw new Error(
1750
- `No suite config found. Create one with \`agent-inspect suite init\` or pass --config.`
1751
- );
1728
+ for (const R of right) {
1729
+ if (!usedRight.has(R.id)) {
1730
+ pairs.push([void 0, R]);
1731
+ }
1732
+ }
1733
+ return pairs;
1752
1734
  }
1753
- async function loadSuiteConfig(options = {}) {
1754
- let configPath;
1755
- try {
1756
- configPath = await resolveSuiteConfigPath(options);
1757
- } catch (error) {
1758
- const message = error instanceof Error ? error.message : String(error);
1759
- throw Object.assign(new Error(message), {
1760
- diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
1735
+ function compareLeafSteps(L, R, segments, opts, out) {
1736
+ const path18 = buildPath(segments);
1737
+ if (L.name !== R.name) {
1738
+ out.push({
1739
+ kind: "structure",
1740
+ severity: "warning",
1741
+ message: "Step name differs",
1742
+ path: path18,
1743
+ left: L.name,
1744
+ right: R.name
1761
1745
  });
1762
1746
  }
1763
- const extension = path13__default.default.extname(configPath);
1764
- if (TS_CONFIG_EXTENSIONS.has(extension)) {
1765
- const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
1766
- throw Object.assign(new Error(message), {
1767
- diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
1747
+ if ((L.type ?? "") !== (R.type ?? "")) {
1748
+ out.push({
1749
+ kind: "step-type",
1750
+ severity: "warning",
1751
+ message: "Step type differs",
1752
+ path: path18,
1753
+ left: L.type,
1754
+ right: R.type
1768
1755
  });
1769
1756
  }
1770
- if (!CONFIG_EXTENSIONS.has(extension)) {
1771
- const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
1772
- throw Object.assign(new Error(message), {
1773
- diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
1757
+ if ((L.status ?? "") !== (R.status ?? "")) {
1758
+ out.push({
1759
+ kind: "step-status",
1760
+ severity: "warning",
1761
+ message: "Step status differs",
1762
+ path: path18,
1763
+ left: L.status,
1764
+ right: R.status
1774
1765
  });
1775
1766
  }
1776
- try {
1777
- let raw;
1778
- if (extension === ".json") {
1779
- raw = JSON.parse(await promises.readFile(configPath, "utf-8"));
1780
- } else {
1781
- const mod = await import(url.pathToFileURL(configPath).href);
1782
- raw = "default" in mod ? mod.default : mod;
1783
- }
1784
- const config = normalizeSuiteConfig(raw);
1785
- return {
1786
- config,
1787
- configPath,
1788
- configDir: path13__default.default.dirname(configPath)
1789
- };
1790
- } catch (error) {
1791
- const message = error instanceof Error ? error.message : String(error);
1792
- throw Object.assign(new Error(message), {
1793
- diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
1767
+ const le = L.error ?? "";
1768
+ const re = R.error ?? "";
1769
+ if (le !== re) {
1770
+ out.push({
1771
+ kind: "error",
1772
+ severity: "error",
1773
+ message: "Step error message differs",
1774
+ path: path18,
1775
+ left: le || void 0,
1776
+ right: re || void 0
1794
1777
  });
1795
1778
  }
1796
- }
1797
- async function exists(filePath) {
1798
- try {
1799
- await promises.access(filePath);
1800
- return true;
1801
- } catch {
1802
- return false;
1803
- }
1804
- }
1805
- async function resolveSuiteCaseTrace(suiteCase, options) {
1806
- if (suiteCase.trace !== void 0) {
1807
- const tracePath = path13__default.default.resolve(options.configDir, suiteCase.trace);
1808
- if (await exists(tracePath)) {
1809
- return { caseId: suiteCase.id, tracePath, missing: false };
1779
+ if (!opts.ignoreDuration) {
1780
+ const ld = L.durationMs;
1781
+ const rd = R.durationMs;
1782
+ const th = opts.durationThresholdMs;
1783
+ let differs = false;
1784
+ if (ld === void 0 && rd === void 0) differs = false;
1785
+ else if (ld === void 0 || rd === void 0) differs = true;
1786
+ else differs = Math.abs(ld - rd) > th;
1787
+ if (differs) {
1788
+ out.push({
1789
+ kind: "duration",
1790
+ severity: "info",
1791
+ message: "Step duration differs",
1792
+ path: path18,
1793
+ left: ld,
1794
+ right: rd
1795
+ });
1810
1796
  }
1811
- return {
1812
- caseId: suiteCase.id,
1813
- tracePath,
1814
- missing: true,
1815
- reason: `trace file not found: ${suiteCase.trace}`
1816
- };
1817
1797
  }
1818
- const runKey = suiteCase.runId ?? suiteCase.id;
1819
- const directPath = getTraceFilePath(runKey, options.tracesDir);
1820
- if (await exists(directPath)) {
1821
- return { caseId: suiteCase.id, tracePath: directPath, runId: runKey, missing: false };
1798
+ const lm = stableJson(L.metadata ?? {});
1799
+ const rm2 = stableJson(R.metadata ?? {});
1800
+ if (lm !== rm2) {
1801
+ out.push({
1802
+ kind: "metadata",
1803
+ severity: "info",
1804
+ message: "Step metadata differs",
1805
+ path: path18,
1806
+ left: L.metadata,
1807
+ right: R.metadata
1808
+ });
1822
1809
  }
1823
- const nestedPath = path13__default.default.join(options.tracesDir, `${path13__default.default.basename(runKey)}.jsonl`);
1824
- if (await exists(nestedPath)) {
1825
- return { caseId: suiteCase.id, tracePath: nestedPath, runId: runKey, missing: false };
1810
+ const lo = stableJson(L.outputPreview ?? null);
1811
+ const ro = stableJson(R.outputPreview ?? null);
1812
+ if (lo !== ro) {
1813
+ out.push({
1814
+ kind: "output",
1815
+ severity: "info",
1816
+ message: "Output preview differs",
1817
+ path: path18,
1818
+ left: L.outputPreview,
1819
+ right: R.outputPreview
1820
+ });
1826
1821
  }
1827
- return {
1828
- caseId: suiteCase.id,
1829
- runId: runKey,
1830
- missing: true,
1831
- reason: `no trace found for run id "${runKey}" under ${options.tracesDir}`
1832
- };
1833
1822
  }
1834
-
1835
- // packages/core/src/checks/index.ts
1836
- var SEVERITY_RANK = {
1837
- error: 0,
1838
- warning: 1,
1839
- info: 2
1840
- };
1841
- var STATUS_RANK = {
1842
- fail: 0,
1843
- warning: 1,
1844
- pass: 2
1845
- };
1846
- function compareStrings(a, b) {
1847
- return (a ?? "").localeCompare(b ?? "");
1823
+ function compareRecursive(L, R, segments, opts, out) {
1824
+ compareLeafSteps(L, R, segments, opts, out);
1825
+ const pairs = pairSteps(L.children, R.children);
1826
+ let ci = 0;
1827
+ for (const [lch, rch] of pairs) {
1828
+ if (lch !== void 0 && rch !== void 0) {
1829
+ compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
1830
+ } else if (lch !== void 0) {
1831
+ out.push({
1832
+ kind: "step-removed",
1833
+ severity: "warning",
1834
+ message: `Step only in left run: ${lch.name}`,
1835
+ path: buildPath([...segments, pathSeg(lch, ci)]),
1836
+ left: lch.id,
1837
+ right: void 0
1838
+ });
1839
+ } else if (rch !== void 0) {
1840
+ out.push({
1841
+ kind: "step-added",
1842
+ severity: "warning",
1843
+ message: `Step only in right run: ${rch.name}`,
1844
+ path: buildPath([...segments, pathSeg(rch, ci)]),
1845
+ left: void 0,
1846
+ right: rch.id
1847
+ });
1848
+ }
1849
+ ci += 1;
1850
+ }
1848
1851
  }
1849
- function diagnostic3(code, message, ruleId) {
1852
+ function mergeDiffDefaults(options) {
1850
1853
  return {
1851
- code,
1852
- message,
1853
- severity: "error",
1854
- ...ruleId ? { ruleId } : {}
1854
+ ignoreDuration: false,
1855
+ durationThresholdMs: DEFAULT_THRESHOLD_MS,
1856
+ focus: "all",
1857
+ check: "all"
1855
1858
  };
1856
1859
  }
1857
- function emptySummary() {
1858
- return {
1859
- passed: 0,
1860
- failed: 0,
1861
- warnings: 0,
1862
- errors: 0
1863
- };
1860
+ function kindMatchesFilter(kind, merged) {
1861
+ return true;
1864
1862
  }
1865
- function errorResult(input, diagnostics, selectedRun) {
1866
- return {
1867
- ok: false,
1868
- status: "error",
1869
- format: input.read.format,
1870
- ...selectedRun ? { runId: selectedRun.runId } : {},
1871
- summary: {
1872
- ...emptySummary(),
1873
- errors: diagnostics.filter((item) => item.severity === "error").length
1874
- },
1875
- findings: [],
1876
- diagnostics: [...diagnostics]
1863
+ function diffRuns(left, right, options) {
1864
+ const merged = mergeDiffDefaults();
1865
+ const opts = {
1866
+ ignoreDuration: merged.ignoreDuration,
1867
+ durationThresholdMs: merged.durationThresholdMs
1877
1868
  };
1878
- }
1879
- function flattenNodes(nodes) {
1880
- return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
1881
- }
1882
- function buildFacts(input, selectedRun) {
1883
- const scopedRuns = selectedRun ? [selectedRun] : input.read.runs;
1884
- const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
1885
- const scopedEvents = selectedRun === void 0 ? input.read.events : input.read.events.filter((event) => scopedRunIds.has(event.runId));
1886
- const nodes = flattenNodes(scopedRuns.flatMap((run) => run.children));
1887
- const nodesByEventId = /* @__PURE__ */ new Map();
1888
- const childrenByParentId = /* @__PURE__ */ new Map();
1889
- for (const node of nodes) {
1890
- nodesByEventId.set(node.event.eventId, node);
1891
- const parentId = node.event.parentId;
1892
- if (parentId) {
1893
- const children = childrenByParentId.get(parentId) ?? [];
1894
- children.push(node);
1895
- childrenByParentId.set(parentId, children);
1896
- }
1869
+ const raw = [];
1870
+ if ((left.status ?? "") !== (right.status ?? "")) {
1871
+ raw.push({
1872
+ kind: "run-status",
1873
+ severity: "warning",
1874
+ message: "Run completion status differs",
1875
+ left: left.status,
1876
+ right: right.status
1877
+ });
1897
1878
  }
1898
- return {
1899
- format: input.read.format,
1900
- runs: Object.freeze([...input.read.runs]),
1901
- events: Object.freeze([...scopedEvents]),
1902
- readerWarnings: Object.freeze([...input.read.warnings]),
1903
- unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
1904
- sourceFiles: Object.freeze([...input.read.sourceFiles]),
1905
- nodesByEventId,
1906
- childrenByParentId,
1907
- rootNodes: Object.freeze(scopedRuns.flatMap((run) => run.children))
1908
- };
1909
- }
1910
- function resolveSelectedRun(input, runId) {
1911
- if (input.selectedRun) {
1912
- if (runId && input.selectedRun.runId !== runId) {
1913
- return {
1914
- diagnostics: [
1915
- diagnostic3(
1916
- "AI_CHECK_INVALID_ARGUMENTS",
1917
- `Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
1918
- )
1919
- ]
1920
- };
1879
+ {
1880
+ const ld = left.durationMs;
1881
+ const rd = right.durationMs;
1882
+ const th = merged.durationThresholdMs;
1883
+ let differs = false;
1884
+ if (ld === void 0 && rd === void 0) differs = false;
1885
+ else if (ld === void 0 || rd === void 0) differs = true;
1886
+ else differs = Math.abs(ld - rd) > th;
1887
+ if (differs) {
1888
+ raw.push({
1889
+ kind: "duration",
1890
+ severity: "info",
1891
+ message: "Run duration differs",
1892
+ left: ld,
1893
+ right: rd
1894
+ });
1921
1895
  }
1922
- return { run: input.selectedRun, diagnostics: [] };
1923
1896
  }
1924
- if (runId) {
1925
- const run = input.read.runs.find((candidate) => candidate.runId === runId);
1926
- if (!run) {
1927
- return {
1928
- diagnostics: [
1929
- diagnostic3("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
1930
- ]
1931
- };
1897
+ const pairs = pairSteps(left.steps, right.steps);
1898
+ let idx = 0;
1899
+ for (const [ls, rs] of pairs) {
1900
+ if (ls !== void 0 && rs !== void 0) {
1901
+ compareRecursive(ls, rs, [pathSeg(ls, idx)], opts, raw);
1902
+ idx += 1;
1903
+ } else if (ls !== void 0) {
1904
+ raw.push({
1905
+ kind: "step-removed",
1906
+ severity: "warning",
1907
+ message: `Step only in left run: ${ls.name}`,
1908
+ path: buildPath([pathSeg(ls, idx)]),
1909
+ left: ls.id,
1910
+ right: void 0
1911
+ });
1912
+ idx += 1;
1913
+ } else if (rs !== void 0) {
1914
+ raw.push({
1915
+ kind: "step-added",
1916
+ severity: "warning",
1917
+ message: `Step only in right run: ${rs.name}`,
1918
+ path: buildPath([pathSeg(rs, idx)]),
1919
+ left: void 0,
1920
+ right: rs.id
1921
+ });
1922
+ idx += 1;
1932
1923
  }
1933
- return { run, diagnostics: [] };
1934
- }
1935
- if (input.read.runs.length === 1) {
1936
- return { run: input.read.runs[0], diagnostics: [] };
1937
1924
  }
1938
- if (input.read.runs.length === 0) {
1939
- return {
1940
- diagnostics: [
1941
- diagnostic3("AI_CHECK_RUN_SELECTION_REQUIRED", "No runs are available for checks.")
1942
- ]
1943
- };
1925
+ const differences = raw.filter((d) => kindMatchesFilter(d.kind));
1926
+ let errors = 0;
1927
+ let warnings = 0;
1928
+ let info = 0;
1929
+ for (const d of differences) {
1930
+ if (d.severity === "error") errors += 1;
1931
+ else if (d.severity === "warning") warnings += 1;
1932
+ else info += 1;
1944
1933
  }
1945
- return {
1946
- diagnostics: [
1947
- diagnostic3(
1948
- "AI_CHECK_RUN_SELECTION_REQUIRED",
1949
- "Multiple runs are available; select a run before executing checks."
1950
- )
1951
- ]
1934
+ const firstVisible = differences[0];
1935
+ const firstDivergence = firstVisible !== void 0 ? {
1936
+ kind: "first-divergence",
1937
+ severity: firstVisible.severity,
1938
+ message: `First divergence: ${firstVisible.message}`,
1939
+ path: firstVisible.path,
1940
+ left: firstVisible.left,
1941
+ right: firstVisible.right
1942
+ } : void 0;
1943
+ const summary = {
1944
+ leftRunId: left.runId,
1945
+ rightRunId: right.runId,
1946
+ totalDifferences: differences.length,
1947
+ errors,
1948
+ warnings,
1949
+ info,
1950
+ firstDivergence
1952
1951
  };
1952
+ return { summary, differences };
1953
1953
  }
1954
- function selectRules(rules, selectedIds) {
1954
+
1955
+ // packages/core/src/evidence/zip.ts
1956
+ (() => {
1957
+ const table = new Uint32Array(256);
1958
+ for (let n = 0; n < 256; n += 1) {
1959
+ let c = n;
1960
+ for (let k = 0; k < 8; k += 1) {
1961
+ c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
1962
+ }
1963
+ table[n] = c >>> 0;
1964
+ }
1965
+ return table;
1966
+ })();
1967
+
1968
+ // packages/core/src/suite/types.ts
1969
+ var DEFAULT_SUITE_CONFIG_NAMES = [
1970
+ "agent-inspect.suite.json",
1971
+ "agent-inspect.suite.js",
1972
+ "agent-inspect.suite.mjs",
1973
+ "agent-inspect.suite.cjs"
1974
+ ];
1975
+ function diagnostic(code, message, severity = "error", caseId) {
1976
+ return { code, message, severity, ...{} };
1977
+ }
1978
+ function asString(value, label) {
1979
+ if (value === void 0) return void 0;
1980
+ if (typeof value !== "string" || value.trim() === "") {
1981
+ throw new Error(`${label} must be a non-empty string.`);
1982
+ }
1983
+ return value.trim();
1984
+ }
1985
+ function asStringArray(value, label) {
1986
+ if (value === void 0) return void 0;
1987
+ if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
1988
+ throw new Error(`${label} must be an array of strings.`);
1989
+ }
1990
+ return value;
1991
+ }
1992
+ function asPositiveNumber(value, label) {
1993
+ if (value === void 0) return void 0;
1994
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
1995
+ throw new Error(`${label} must be a non-negative number.`);
1996
+ }
1997
+ return value;
1998
+ }
1999
+ function validateCaseConfig(value, index) {
1955
2000
  const diagnostics = [];
1956
- const byId = /* @__PURE__ */ new Map();
1957
- for (const rule of rules) {
1958
- if (byId.has(rule.id)) {
2001
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
2002
+ diagnostics.push(
2003
+ diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
2004
+ );
2005
+ return { diagnostics };
2006
+ }
2007
+ const raw = value;
2008
+ try {
2009
+ const id = asString(raw.id, `cases[${index}].id`);
2010
+ if (id === void 0) {
1959
2011
  diagnostics.push(
1960
- diagnostic3("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
2012
+ diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
1961
2013
  );
1962
- continue;
1963
- }
1964
- byId.set(rule.id, rule);
1965
- }
1966
- if (selectedIds && selectedIds.length > 0) {
1967
- const selected = new Set(selectedIds);
1968
- for (const id of selected) {
1969
- if (!byId.has(id)) {
1970
- diagnostics.push(
1971
- diagnostic3("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
1972
- );
1973
- }
2014
+ return { diagnostics };
1974
2015
  }
2016
+ const trace = asString(raw.trace, `cases[${index}].trace`);
2017
+ const runId = asString(raw.runId, `cases[${index}].runId`);
2018
+ const input = asString(raw.input, `cases[${index}].input`);
1975
2019
  return {
1976
- rules: [...byId.values()].filter((rule) => selected.has(rule.id)).sort(compareRules),
2020
+ caseConfig: {
2021
+ id,
2022
+ ...trace !== void 0 ? { trace } : {},
2023
+ ...runId !== void 0 ? { runId } : {},
2024
+ ...input !== void 0 ? { input } : {},
2025
+ ...asStringArray(raw.requireTools, `cases[${index}].requireTools`) !== void 0 ? { requireTools: asStringArray(raw.requireTools, `cases[${index}].requireTools`) } : {},
2026
+ ...asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) !== void 0 ? { forbidTools: asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) } : {},
2027
+ ...asPositiveNumber(raw.maxDurationMs, `cases[${index}].maxDurationMs`) !== void 0 ? {
2028
+ maxDurationMs: asPositiveNumber(
2029
+ raw.maxDurationMs,
2030
+ `cases[${index}].maxDurationMs`
2031
+ )
2032
+ } : {},
2033
+ ...asStringArray(
2034
+ raw.expectedObservations,
2035
+ `cases[${index}].expectedObservations`
2036
+ ) !== void 0 ? {
2037
+ expectedObservations: asStringArray(
2038
+ raw.expectedObservations,
2039
+ `cases[${index}].expectedObservations`
2040
+ )
2041
+ } : {}
2042
+ },
1977
2043
  diagnostics
1978
2044
  };
2045
+ } catch (error) {
2046
+ const message = error instanceof Error ? error.message : String(error);
2047
+ diagnostics.push(diagnostic("AI_SUITE_CONFIG_INVALID", message));
2048
+ return { diagnostics };
1979
2049
  }
1980
- return { rules: [...byId.values()].sort(compareRules), diagnostics };
1981
- }
1982
- function compareRules(a, b) {
1983
- return a.id.localeCompare(b.id);
1984
- }
1985
- function eventTimestamp(finding, eventById) {
1986
- const eventId = finding.evidence[0]?.eventId;
1987
- return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
1988
2050
  }
1989
- function compareFindings(eventById) {
1990
- return (a, b) => {
1991
- if (SEVERITY_RANK[a.severity] !== SEVERITY_RANK[b.severity]) {
1992
- return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity];
2051
+ function normalizeSuiteConfig(value) {
2052
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
2053
+ throw new Error("Suite config must export an object.");
2054
+ }
2055
+ const raw = value;
2056
+ const name = asString(raw.name, "name");
2057
+ const traces = asString(raw.traces, "traces");
2058
+ if (name === void 0) throw new Error("name is required.");
2059
+ if (traces === void 0) throw new Error("traces is required.");
2060
+ if (!Array.isArray(raw.cases) || raw.cases.length === 0) {
2061
+ throw new Error("cases must be a non-empty array.");
2062
+ }
2063
+ const cases = [];
2064
+ for (let index = 0; index < raw.cases.length; index += 1) {
2065
+ const { caseConfig, diagnostics } = validateCaseConfig(raw.cases[index], index);
2066
+ if (diagnostics.length > 0) {
2067
+ throw new Error(diagnostics.map((item) => item.message).join("; "));
1993
2068
  }
1994
- const byRule = a.ruleId.localeCompare(b.ruleId);
1995
- if (byRule !== 0) return byRule;
1996
- if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
1997
- return STATUS_RANK[a.status] - STATUS_RANK[b.status];
2069
+ if (caseConfig !== void 0) cases.push(caseConfig);
2070
+ }
2071
+ const ids = /* @__PURE__ */ new Set();
2072
+ for (const suiteCase of cases) {
2073
+ if (ids.has(suiteCase.id)) {
2074
+ throw new Error(`Duplicate case id "${suiteCase.id}".`);
1998
2075
  }
1999
- const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
2000
- if (byRun !== 0) return byRun;
2001
- const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
2002
- if (byTime !== 0) return byTime;
2003
- const byEvent = compareStrings(a.evidence[0]?.eventId, b.evidence[0]?.eventId);
2004
- if (byEvent !== 0) return byEvent;
2005
- return compareStrings(a.evidence[0]?.path, b.evidence[0]?.path);
2006
- };
2076
+ ids.add(suiteCase.id);
2077
+ }
2078
+ const redactionProfile = raw.redactionProfile === "local" || raw.redactionProfile === "share" || raw.redactionProfile === "strict" ? raw.redactionProfile : void 0;
2079
+ if (raw.redactionProfile !== void 0 && redactionProfile === void 0) {
2080
+ throw new Error('redactionProfile must be "local", "share", or "strict".');
2081
+ }
2082
+ const config = { name, traces, cases };
2083
+ if (raw.checks !== void 0) {
2084
+ if (typeof raw.checks !== "object" || Array.isArray(raw.checks)) {
2085
+ throw new Error("checks must be an object.");
2086
+ }
2087
+ config.checks = raw.checks;
2088
+ }
2089
+ if (raw.eval !== void 0) {
2090
+ if (typeof raw.eval !== "object" || Array.isArray(raw.eval)) {
2091
+ throw new Error("eval must be an object.");
2092
+ }
2093
+ config.eval = raw.eval;
2094
+ }
2095
+ if (raw.artifacts !== void 0) {
2096
+ if (typeof raw.artifacts !== "object" || Array.isArray(raw.artifacts)) {
2097
+ throw new Error("artifacts must be an object.");
2098
+ }
2099
+ const outputDir = asString(
2100
+ raw.artifacts.outputDir,
2101
+ "artifacts.outputDir"
2102
+ );
2103
+ config.artifacts = outputDir !== void 0 ? { outputDir } : {};
2104
+ }
2105
+ if (raw.baseline !== void 0) {
2106
+ const baseline = asString(raw.baseline, "baseline");
2107
+ if (baseline !== void 0) config.baseline = baseline;
2108
+ }
2109
+ if (raw.candidate !== void 0) {
2110
+ const candidate = asString(raw.candidate, "candidate");
2111
+ if (candidate !== void 0) config.candidate = candidate;
2112
+ }
2113
+ if (redactionProfile !== void 0) config.redactionProfile = redactionProfile;
2114
+ return config;
2007
2115
  }
2008
- function normalizeFinding(rule, finding) {
2009
- return {
2010
- ruleId: finding.ruleId || rule.id,
2011
- severity: finding.severity ?? rule.defaultSeverity,
2012
- status: finding.status,
2013
- message: finding.message,
2014
- ...finding.expected !== void 0 ? { expected: finding.expected } : {},
2015
- ...finding.actual !== void 0 ? { actual: finding.actual } : {},
2016
- evidence: [...finding.evidence ?? []],
2017
- ...finding.category !== void 0 ? { category: finding.category } : {},
2018
- ...finding.confidence !== void 0 ? { confidence: finding.confidence } : {},
2019
- ...finding.detector !== void 0 ? { detector: finding.detector } : {},
2020
- ...finding.action !== void 0 ? { action: finding.action } : {}
2021
- };
2116
+
2117
+ // packages/core/src/suite/load.ts
2118
+ var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
2119
+ var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
2120
+ function diagnostic2(code, message) {
2121
+ return { code, message, severity: "error" };
2022
2122
  }
2023
- function summarize(findings, diagnostics) {
2024
- return {
2025
- passed: findings.filter((finding) => finding.status === "pass").length,
2026
- failed: findings.filter(
2027
- (finding) => finding.status === "fail" && finding.severity === "error"
2028
- ).length,
2029
- warnings: findings.filter(
2030
- (finding) => finding.status === "warning" || finding.severity === "warning"
2031
- ).length,
2032
- errors: diagnostics.filter((item) => item.severity === "error").length
2033
- };
2123
+ async function fileExists(filePath) {
2124
+ try {
2125
+ await promises.access(filePath);
2126
+ return true;
2127
+ } catch {
2128
+ return false;
2129
+ }
2034
2130
  }
2035
- function stringAttr(event, keys) {
2036
- for (const key of keys) {
2037
- const value = event.attributes?.[key];
2038
- if (typeof value === "string" && value.trim() !== "") return value;
2131
+ async function resolveSuiteConfigPath(options = {}) {
2132
+ const cwd = path15__default.default.resolve(options.cwd ?? process.cwd());
2133
+ if (options.configPath !== void 0 && options.configPath.trim() !== "") {
2134
+ return path15__default.default.resolve(cwd, options.configPath.trim());
2039
2135
  }
2040
- return void 0;
2136
+ for (const name of DEFAULT_SUITE_CONFIG_NAMES) {
2137
+ const candidate = path15__default.default.join(cwd, name);
2138
+ if (await fileExists(candidate)) return candidate;
2139
+ }
2140
+ throw new Error(
2141
+ `No suite config found. Create one with \`agent-inspect suite init\` or pass --config.`
2142
+ );
2041
2143
  }
2042
- function stripPrefix(name, prefixes) {
2043
- for (const prefix of prefixes) {
2044
- if (name.startsWith(prefix)) return name.slice(prefix.length);
2144
+ async function loadSuiteConfig(options = {}) {
2145
+ let configPath;
2146
+ try {
2147
+ configPath = await resolveSuiteConfigPath(options);
2148
+ } catch (error) {
2149
+ const message = error instanceof Error ? error.message : String(error);
2150
+ throw Object.assign(new Error(message), {
2151
+ diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
2152
+ });
2153
+ }
2154
+ const extension = path15__default.default.extname(configPath);
2155
+ if (TS_CONFIG_EXTENSIONS.has(extension)) {
2156
+ const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
2157
+ throw Object.assign(new Error(message), {
2158
+ diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
2159
+ });
2160
+ }
2161
+ if (!CONFIG_EXTENSIONS.has(extension)) {
2162
+ const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
2163
+ throw Object.assign(new Error(message), {
2164
+ diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
2165
+ });
2166
+ }
2167
+ try {
2168
+ let raw;
2169
+ if (extension === ".json") {
2170
+ raw = JSON.parse(await promises.readFile(configPath, "utf-8"));
2171
+ } else {
2172
+ const mod = await import(url.pathToFileURL(configPath).href);
2173
+ raw = "default" in mod ? mod.default : mod;
2174
+ }
2175
+ const config = normalizeSuiteConfig(raw);
2176
+ return {
2177
+ config,
2178
+ configPath,
2179
+ configDir: path15__default.default.dirname(configPath)
2180
+ };
2181
+ } catch (error) {
2182
+ const message = error instanceof Error ? error.message : String(error);
2183
+ throw Object.assign(new Error(message), {
2184
+ diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
2185
+ });
2186
+ }
2187
+ }
2188
+ async function exists(filePath) {
2189
+ try {
2190
+ await promises.access(filePath);
2191
+ return true;
2192
+ } catch {
2193
+ return false;
2045
2194
  }
2046
- return name;
2047
2195
  }
2048
- function eventEvidence(event, path16) {
2196
+ async function resolveSuiteCaseTrace(suiteCase, options) {
2197
+ if (suiteCase.trace !== void 0) {
2198
+ const tracePath = path15__default.default.resolve(options.configDir, suiteCase.trace);
2199
+ if (await exists(tracePath)) {
2200
+ return { caseId: suiteCase.id, tracePath, missing: false };
2201
+ }
2202
+ return {
2203
+ caseId: suiteCase.id,
2204
+ tracePath,
2205
+ missing: true,
2206
+ reason: `trace file not found: ${suiteCase.trace}`
2207
+ };
2208
+ }
2209
+ const runKey = suiteCase.runId ?? suiteCase.id;
2210
+ const directPath = getTraceFilePath(runKey, options.tracesDir);
2211
+ if (await exists(directPath)) {
2212
+ return { caseId: suiteCase.id, tracePath: directPath, runId: runKey, missing: false };
2213
+ }
2214
+ const nestedPath = path15__default.default.join(options.tracesDir, `${path15__default.default.basename(runKey)}.jsonl`);
2215
+ if (await exists(nestedPath)) {
2216
+ return { caseId: suiteCase.id, tracePath: nestedPath, runId: runKey, missing: false };
2217
+ }
2049
2218
  return {
2050
- runId: event.runId,
2051
- eventId: event.eventId,
2052
- parentId: event.parentId,
2053
- traceId: event.trace?.traceId,
2054
- spanId: event.trace?.spanId,
2055
- kind: event.kind,
2056
- name: event.name,
2057
- status: event.status,
2058
- ...path16 ? { path: path16 } : {}
2219
+ caseId: suiteCase.id,
2220
+ runId: runKey,
2221
+ missing: true,
2222
+ reason: `no trace found for run id "${runKey}" under ${options.tracesDir}`
2059
2223
  };
2060
2224
  }
2061
- function runEvidence(run) {
2062
- return run ? [{ runId: run.runId, name: run.name, status: run.status }] : [];
2225
+
2226
+ // packages/core/src/checks/index.ts
2227
+ var SEVERITY_RANK = {
2228
+ error: 0,
2229
+ warning: 1,
2230
+ info: 2
2231
+ };
2232
+ var STATUS_RANK = {
2233
+ fail: 0,
2234
+ warning: 1,
2235
+ pass: 2
2236
+ };
2237
+ function compareStrings(a, b) {
2238
+ return (a ?? "").localeCompare(b ?? "");
2063
2239
  }
2064
- function failFinding(ruleId, message, evidence, expected, actual, meta) {
2240
+ function diagnostic3(code, message, ruleId) {
2065
2241
  return {
2066
- ruleId,
2067
- severity: meta?.severity ?? "error",
2068
- status: meta?.status ?? "fail",
2242
+ code,
2069
2243
  message,
2070
- ...expected !== void 0 ? { expected } : {},
2071
- ...actual !== void 0 ? { actual } : {},
2072
- evidence: [...evidence],
2073
- ...{},
2074
- ...{},
2075
- ...{},
2076
- ...{}
2244
+ severity: "error",
2245
+ ...ruleId ? { ruleId } : {}
2077
2246
  };
2078
2247
  }
2079
- function toolName(event) {
2080
- return stringAttr(event, ["toolName", "tool"]) ?? stripPrefix(event.name, ["tool:", "function:", "mcp-tools:"]);
2081
- }
2082
- function llmModel(event) {
2083
- return stringAttr(event, ["model", "modelId", "responseModelId", "modelName", "model_name"]) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
2248
+ function emptySummary() {
2249
+ return {
2250
+ passed: 0,
2251
+ failed: 0,
2252
+ warnings: 0,
2253
+ errors: 0
2254
+ };
2084
2255
  }
2085
- function llmProvider(event) {
2086
- return stringAttr(event, ["provider", "providerName", "provider_name"]);
2256
+ function errorResult(input, diagnostics, selectedRun) {
2257
+ return {
2258
+ ok: false,
2259
+ status: "error",
2260
+ format: input.read.format,
2261
+ ...selectedRun ? { runId: selectedRun.runId } : {},
2262
+ summary: {
2263
+ ...emptySummary(),
2264
+ errors: diagnostics.filter((item) => item.severity === "error").length
2265
+ },
2266
+ findings: [],
2267
+ diagnostics: [...diagnostics]
2268
+ };
2087
2269
  }
2088
- function llmFinishReason(event) {
2089
- return stringAttr(event, ["finishReason", "rawFinishReason", "finish_reason"]);
2270
+ function flattenNodes(nodes) {
2271
+ return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
2090
2272
  }
2091
- function finishedEvents(context, kind) {
2092
- return context.events.filter(
2093
- (event) => (kind === void 0 || event.kind === kind) && event.status !== "running"
2094
- );
2273
+ function buildFacts(input, selectedRun) {
2274
+ const scopedRuns = selectedRun ? [selectedRun] : input.read.runs;
2275
+ const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
2276
+ const scopedEvents = selectedRun === void 0 ? input.read.events : input.read.events.filter((event) => scopedRunIds.has(event.runId));
2277
+ const nodes = flattenNodes(scopedRuns.flatMap((run) => run.children));
2278
+ const nodesByEventId = /* @__PURE__ */ new Map();
2279
+ const childrenByParentId = /* @__PURE__ */ new Map();
2280
+ for (const node of nodes) {
2281
+ nodesByEventId.set(node.event.eventId, node);
2282
+ const parentId = node.event.parentId;
2283
+ if (parentId) {
2284
+ const children = childrenByParentId.get(parentId) ?? [];
2285
+ children.push(node);
2286
+ childrenByParentId.set(parentId, children);
2287
+ }
2288
+ }
2289
+ return {
2290
+ format: input.read.format,
2291
+ runs: Object.freeze([...input.read.runs]),
2292
+ events: Object.freeze([...scopedEvents]),
2293
+ readerWarnings: Object.freeze([...input.read.warnings]),
2294
+ unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
2295
+ sourceFiles: Object.freeze([...input.read.sourceFiles]),
2296
+ nodesByEventId,
2297
+ childrenByParentId,
2298
+ rootNodes: Object.freeze(scopedRuns.flatMap((run) => run.children))
2299
+ };
2300
+ }
2301
+ function resolveSelectedRun(input, runId) {
2302
+ if (input.selectedRun) {
2303
+ if (runId && input.selectedRun.runId !== runId) {
2304
+ return {
2305
+ diagnostics: [
2306
+ diagnostic3(
2307
+ "AI_CHECK_INVALID_ARGUMENTS",
2308
+ `Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
2309
+ )
2310
+ ]
2311
+ };
2312
+ }
2313
+ return { run: input.selectedRun, diagnostics: [] };
2314
+ }
2315
+ if (runId) {
2316
+ const run = input.read.runs.find((candidate) => candidate.runId === runId);
2317
+ if (!run) {
2318
+ return {
2319
+ diagnostics: [
2320
+ diagnostic3("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
2321
+ ]
2322
+ };
2323
+ }
2324
+ return { run, diagnostics: [] };
2325
+ }
2326
+ if (input.read.runs.length === 1) {
2327
+ return { run: input.read.runs[0], diagnostics: [] };
2328
+ }
2329
+ if (input.read.runs.length === 0) {
2330
+ return {
2331
+ diagnostics: [
2332
+ diagnostic3("AI_CHECK_RUN_SELECTION_REQUIRED", "No runs are available for checks.")
2333
+ ]
2334
+ };
2335
+ }
2336
+ return {
2337
+ diagnostics: [
2338
+ diagnostic3(
2339
+ "AI_CHECK_RUN_SELECTION_REQUIRED",
2340
+ "Multiple runs are available; select a run before executing checks."
2341
+ )
2342
+ ]
2343
+ };
2344
+ }
2345
+ function selectRules(rules, selectedIds) {
2346
+ const diagnostics = [];
2347
+ const byId = /* @__PURE__ */ new Map();
2348
+ for (const rule of rules) {
2349
+ if (byId.has(rule.id)) {
2350
+ diagnostics.push(
2351
+ diagnostic3("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
2352
+ );
2353
+ continue;
2354
+ }
2355
+ byId.set(rule.id, rule);
2356
+ }
2357
+ if (selectedIds && selectedIds.length > 0) {
2358
+ const selected = new Set(selectedIds);
2359
+ for (const id of selected) {
2360
+ if (!byId.has(id)) {
2361
+ diagnostics.push(
2362
+ diagnostic3("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
2363
+ );
2364
+ }
2365
+ }
2366
+ return {
2367
+ rules: [...byId.values()].filter((rule) => selected.has(rule.id)).sort(compareRules),
2368
+ diagnostics
2369
+ };
2370
+ }
2371
+ return { rules: [...byId.values()].sort(compareRules), diagnostics };
2372
+ }
2373
+ function compareRules(a, b) {
2374
+ return a.id.localeCompare(b.id);
2375
+ }
2376
+ function eventTimestamp(finding, eventById) {
2377
+ const eventId = finding.evidence[0]?.eventId;
2378
+ return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
2379
+ }
2380
+ function compareFindings(eventById) {
2381
+ return (a, b) => {
2382
+ if (SEVERITY_RANK[a.severity] !== SEVERITY_RANK[b.severity]) {
2383
+ return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity];
2384
+ }
2385
+ const byRule = a.ruleId.localeCompare(b.ruleId);
2386
+ if (byRule !== 0) return byRule;
2387
+ if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
2388
+ return STATUS_RANK[a.status] - STATUS_RANK[b.status];
2389
+ }
2390
+ const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
2391
+ if (byRun !== 0) return byRun;
2392
+ const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
2393
+ if (byTime !== 0) return byTime;
2394
+ const byEvent = compareStrings(a.evidence[0]?.eventId, b.evidence[0]?.eventId);
2395
+ if (byEvent !== 0) return byEvent;
2396
+ return compareStrings(a.evidence[0]?.path, b.evidence[0]?.path);
2397
+ };
2398
+ }
2399
+ function normalizeFinding(rule, finding) {
2400
+ return {
2401
+ ruleId: finding.ruleId || rule.id,
2402
+ severity: finding.severity ?? rule.defaultSeverity,
2403
+ status: finding.status,
2404
+ message: finding.message,
2405
+ ...finding.expected !== void 0 ? { expected: finding.expected } : {},
2406
+ ...finding.actual !== void 0 ? { actual: finding.actual } : {},
2407
+ evidence: [...finding.evidence ?? []],
2408
+ ...finding.category !== void 0 ? { category: finding.category } : {},
2409
+ ...finding.confidence !== void 0 ? { confidence: finding.confidence } : {},
2410
+ ...finding.detector !== void 0 ? { detector: finding.detector } : {},
2411
+ ...finding.action !== void 0 ? { action: finding.action } : {}
2412
+ };
2413
+ }
2414
+ function summarize(findings, diagnostics) {
2415
+ return {
2416
+ passed: findings.filter((finding) => finding.status === "pass").length,
2417
+ failed: findings.filter(
2418
+ (finding) => finding.status === "fail" && finding.severity === "error"
2419
+ ).length,
2420
+ warnings: findings.filter(
2421
+ (finding) => finding.status === "warning" || finding.severity === "warning"
2422
+ ).length,
2423
+ errors: diagnostics.filter((item) => item.severity === "error").length
2424
+ };
2425
+ }
2426
+ function stringAttr(event, keys) {
2427
+ for (const key of keys) {
2428
+ const value = event.attributes?.[key];
2429
+ if (typeof value === "string" && value.trim() !== "") return value;
2430
+ }
2431
+ return void 0;
2432
+ }
2433
+ function stripPrefix(name, prefixes) {
2434
+ for (const prefix of prefixes) {
2435
+ if (name.startsWith(prefix)) return name.slice(prefix.length);
2436
+ }
2437
+ return name;
2438
+ }
2439
+ function eventEvidence(event, path18) {
2440
+ return {
2441
+ runId: event.runId,
2442
+ eventId: event.eventId,
2443
+ parentId: event.parentId,
2444
+ traceId: event.trace?.traceId,
2445
+ spanId: event.trace?.spanId,
2446
+ kind: event.kind,
2447
+ name: event.name,
2448
+ status: event.status,
2449
+ ...path18 ? { path: path18 } : {}
2450
+ };
2451
+ }
2452
+ function runEvidence(run) {
2453
+ return run ? [{ runId: run.runId, name: run.name, status: run.status }] : [];
2454
+ }
2455
+ function failFinding(ruleId, message, evidence, expected, actual, meta) {
2456
+ return {
2457
+ ruleId,
2458
+ severity: meta?.severity ?? "error",
2459
+ status: meta?.status ?? "fail",
2460
+ message,
2461
+ ...expected !== void 0 ? { expected } : {},
2462
+ ...actual !== void 0 ? { actual } : {},
2463
+ evidence: [...evidence],
2464
+ ...{},
2465
+ ...{},
2466
+ ...{},
2467
+ ...{}
2468
+ };
2469
+ }
2470
+ function toolName(event) {
2471
+ return stringAttr(event, ["toolName", "tool"]) ?? stripPrefix(event.name, ["tool:", "function:", "mcp-tools:"]);
2472
+ }
2473
+ function llmModel(event) {
2474
+ return stringAttr(event, ["model", "modelId", "responseModelId", "modelName", "model_name"]) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
2475
+ }
2476
+ function llmProvider(event) {
2477
+ return stringAttr(event, ["provider", "providerName", "provider_name"]);
2478
+ }
2479
+ function llmFinishReason(event) {
2480
+ return stringAttr(event, ["finishReason", "rawFinishReason", "finish_reason"]);
2481
+ }
2482
+ function finishedEvents(context, kind) {
2483
+ return context.events.filter(
2484
+ (event) => (kind === void 0 || event.kind === kind) && event.status !== "running"
2485
+ );
2095
2486
  }
2096
2487
  function createRunStatusRule(options = {}) {
2097
2488
  const expected = options.expected ?? "ok";
@@ -2979,7 +3370,7 @@ function findReaderByFormat(format, readers) {
2979
3370
  }
2980
3371
  async function jsonlFilesInDirectory(dirPath) {
2981
3372
  const entries = await promises.readdir(dirPath, { withFileTypes: true });
2982
- return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => path13__default.default.join(dirPath, entry.name)).sort((a, b) => a.localeCompare(b));
3373
+ return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => path15__default.default.join(dirPath, entry.name)).sort((a, b) => a.localeCompare(b));
2983
3374
  }
2984
3375
  async function resolveInput(input) {
2985
3376
  const cached = resolvedInputCache.get(input);
@@ -4599,7 +4990,7 @@ async function runSuiteCase(suiteCase, config, options) {
4599
4990
  async function runSuite(options = {}) {
4600
4991
  const startedAt = new Date(options.nowMs ?? Date.now()).toISOString();
4601
4992
  const { config, configPath, configDir } = await loadSuiteConfig(options);
4602
- const tracesDir = path13__default.default.resolve(configDir, config.traces);
4993
+ const tracesDir = path15__default.default.resolve(configDir, config.traces);
4603
4994
  const cases = [];
4604
4995
  const diagnostics = [];
4605
4996
  for (const suiteCase of config.cases) {
@@ -4634,22 +5025,6 @@ async function runSuite(options = {}) {
4634
5025
  };
4635
5026
  }
4636
5027
 
4637
- // packages/core/src/exporters/helpers.ts
4638
- function sortKeysDeep(input) {
4639
- if (input === null || typeof input !== "object") return input;
4640
- if (Array.isArray(input)) return input.map(sortKeysDeep);
4641
- const o = input;
4642
- const out = {};
4643
- for (const k of Object.keys(o).sort()) {
4644
- out[k] = sortKeysDeep(o[k]);
4645
- }
4646
- return out;
4647
- }
4648
- function stableJson(value, pretty) {
4649
- const sorted = sortKeysDeep(value);
4650
- return JSON.stringify(sorted);
4651
- }
4652
-
4653
5028
  // packages/viewer/src/html.ts
4654
5029
  var viewerIndexHtml = `<!DOCTYPE html>
4655
5030
  <html lang="en">
@@ -4739,370 +5114,6 @@ var viewerIndexHtml = `<!DOCTYPE html>
4739
5114
  </body>
4740
5115
  </html>
4741
5116
  `;
4742
-
4743
- // packages/core/src/diff/comparable.ts
4744
- function extractOutputPreview(meta) {
4745
- if (meta === void 0) return void 0;
4746
- if ("outputPreview" in meta) return meta.outputPreview;
4747
- if ("resultPreview" in meta) return meta.resultPreview;
4748
- return void 0;
4749
- }
4750
- function mapStepStatus(s) {
4751
- if (s === void 0) return "running";
4752
- return s;
4753
- }
4754
- function manualTraceEventsToComparableRun(events) {
4755
- const started = events.find((e) => e.event === "run_started");
4756
- if (!started || started.event !== "run_started") {
4757
- throw new Error("Invalid trace: missing run_started");
4758
- }
4759
- const rs = started;
4760
- const runId = rs.runId;
4761
- const completedAll = events.filter((e) => e.event === "run_completed");
4762
- const lastCompleted = completedAll[completedAll.length - 1];
4763
- let runStatus;
4764
- if (lastCompleted === void 0) runStatus = "running";
4765
- else runStatus = lastCompleted.status;
4766
- const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
4767
- const steps = /* @__PURE__ */ new Map();
4768
- let order = 0;
4769
- for (const e of events) {
4770
- if (e.event !== "step_started") continue;
4771
- const s = e;
4772
- const meta = s.metadata ? { ...s.metadata } : void 0;
4773
- steps.set(s.stepId, {
4774
- id: s.stepId,
4775
- parentId: s.parentId,
4776
- name: s.name,
4777
- type: s.type,
4778
- order: order++,
4779
- timestamp: s.timestamp,
4780
- metadata: meta
4781
- });
4782
- }
4783
- for (const e of events) {
4784
- if (e.event !== "step_completed") continue;
4785
- const acc = steps.get(e.stepId);
4786
- if (!acc) continue;
4787
- acc.status = e.status;
4788
- acc.durationMs = e.durationMs;
4789
- if (e.error?.message) acc.errorMsg = e.error.message;
4790
- const extra = e;
4791
- if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
4792
- acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
4793
- }
4794
- }
4795
- const nodes = /* @__PURE__ */ new Map();
4796
- for (const acc of steps.values()) {
4797
- let meta = acc.metadata ? { ...acc.metadata } : void 0;
4798
- if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
4799
- meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
4800
- }
4801
- const outputPreview = extractOutputPreview(meta);
4802
- if (meta !== void 0 && ("outputPreview" in meta || "resultPreview" in meta)) {
4803
- delete meta.outputPreview;
4804
- delete meta.resultPreview;
4805
- }
4806
- const sc = {
4807
- id: acc.id,
4808
- name: acc.name,
4809
- type: acc.type,
4810
- status: mapStepStatus(acc.status),
4811
- durationMs: acc.durationMs,
4812
- error: acc.errorMsg,
4813
- metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
4814
- outputPreview,
4815
- children: []
4816
- };
4817
- nodes.set(acc.id, sc);
4818
- }
4819
- const roots = [];
4820
- const sortByOrder = (a, b) => {
4821
- const oa = steps.get(a.id)?.order ?? 0;
4822
- const ob = steps.get(b.id)?.order ?? 0;
4823
- return oa - ob;
4824
- };
4825
- for (const acc of steps.values()) {
4826
- const node = nodes.get(acc.id);
4827
- if (acc.parentId !== void 0 && nodes.has(acc.parentId)) {
4828
- nodes.get(acc.parentId).children.push(node);
4829
- } else {
4830
- roots.push(node);
4831
- }
4832
- }
4833
- roots.sort(sortByOrder);
4834
- for (const n of nodes.values()) {
4835
- n.children.sort(sortByOrder);
4836
- }
4837
- return {
4838
- runId,
4839
- name: rs.name,
4840
- status: runStatus,
4841
- durationMs,
4842
- steps: roots
4843
- };
4844
- }
4845
-
4846
- // packages/core/src/diff/engine.ts
4847
- var DEFAULT_THRESHOLD_MS = 0;
4848
- function pathSeg(step, index) {
4849
- return { index, name: step.name, stepId: step.id };
4850
- }
4851
- function buildPath(segments) {
4852
- return { path: [...segments] };
4853
- }
4854
- function pairSteps(left, right) {
4855
- const usedRight = /* @__PURE__ */ new Set();
4856
- const pairs = [];
4857
- for (let i = 0; i < left.length; i++) {
4858
- const L = left[i];
4859
- let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
4860
- if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
4861
- const cand = right[i];
4862
- if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
4863
- R = cand;
4864
- }
4865
- }
4866
- if (R === void 0) {
4867
- R = right.find(
4868
- (r) => !usedRight.has(r.id) && r.name === L.name && (r.type ?? "") === (L.type ?? "")
4869
- );
4870
- }
4871
- if (R !== void 0) {
4872
- usedRight.add(R.id);
4873
- pairs.push([L, R]);
4874
- } else {
4875
- pairs.push([L, void 0]);
4876
- }
4877
- }
4878
- for (const R of right) {
4879
- if (!usedRight.has(R.id)) {
4880
- pairs.push([void 0, R]);
4881
- }
4882
- }
4883
- return pairs;
4884
- }
4885
- function compareLeafSteps(L, R, segments, opts, out) {
4886
- const path16 = buildPath(segments);
4887
- if (L.name !== R.name) {
4888
- out.push({
4889
- kind: "structure",
4890
- severity: "warning",
4891
- message: "Step name differs",
4892
- path: path16,
4893
- left: L.name,
4894
- right: R.name
4895
- });
4896
- }
4897
- if ((L.type ?? "") !== (R.type ?? "")) {
4898
- out.push({
4899
- kind: "step-type",
4900
- severity: "warning",
4901
- message: "Step type differs",
4902
- path: path16,
4903
- left: L.type,
4904
- right: R.type
4905
- });
4906
- }
4907
- if ((L.status ?? "") !== (R.status ?? "")) {
4908
- out.push({
4909
- kind: "step-status",
4910
- severity: "warning",
4911
- message: "Step status differs",
4912
- path: path16,
4913
- left: L.status,
4914
- right: R.status
4915
- });
4916
- }
4917
- const le = L.error ?? "";
4918
- const re = R.error ?? "";
4919
- if (le !== re) {
4920
- out.push({
4921
- kind: "error",
4922
- severity: "error",
4923
- message: "Step error message differs",
4924
- path: path16,
4925
- left: le || void 0,
4926
- right: re || void 0
4927
- });
4928
- }
4929
- if (!opts.ignoreDuration) {
4930
- const ld = L.durationMs;
4931
- const rd = R.durationMs;
4932
- const th = opts.durationThresholdMs;
4933
- let differs = false;
4934
- if (ld === void 0 && rd === void 0) differs = false;
4935
- else if (ld === void 0 || rd === void 0) differs = true;
4936
- else differs = Math.abs(ld - rd) > th;
4937
- if (differs) {
4938
- out.push({
4939
- kind: "duration",
4940
- severity: "info",
4941
- message: "Step duration differs",
4942
- path: path16,
4943
- left: ld,
4944
- right: rd
4945
- });
4946
- }
4947
- }
4948
- const lm = stableJson(L.metadata ?? {});
4949
- const rm2 = stableJson(R.metadata ?? {});
4950
- if (lm !== rm2) {
4951
- out.push({
4952
- kind: "metadata",
4953
- severity: "info",
4954
- message: "Step metadata differs",
4955
- path: path16,
4956
- left: L.metadata,
4957
- right: R.metadata
4958
- });
4959
- }
4960
- const lo = stableJson(L.outputPreview ?? null);
4961
- const ro = stableJson(R.outputPreview ?? null);
4962
- if (lo !== ro) {
4963
- out.push({
4964
- kind: "output",
4965
- severity: "info",
4966
- message: "Output preview differs",
4967
- path: path16,
4968
- left: L.outputPreview,
4969
- right: R.outputPreview
4970
- });
4971
- }
4972
- }
4973
- function compareRecursive(L, R, segments, opts, out) {
4974
- compareLeafSteps(L, R, segments, opts, out);
4975
- const pairs = pairSteps(L.children, R.children);
4976
- let ci = 0;
4977
- for (const [lch, rch] of pairs) {
4978
- if (lch !== void 0 && rch !== void 0) {
4979
- compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
4980
- } else if (lch !== void 0) {
4981
- out.push({
4982
- kind: "step-removed",
4983
- severity: "warning",
4984
- message: `Step only in left run: ${lch.name}`,
4985
- path: buildPath([...segments, pathSeg(lch, ci)]),
4986
- left: lch.id,
4987
- right: void 0
4988
- });
4989
- } else if (rch !== void 0) {
4990
- out.push({
4991
- kind: "step-added",
4992
- severity: "warning",
4993
- message: `Step only in right run: ${rch.name}`,
4994
- path: buildPath([...segments, pathSeg(rch, ci)]),
4995
- left: void 0,
4996
- right: rch.id
4997
- });
4998
- }
4999
- ci += 1;
5000
- }
5001
- }
5002
- function mergeDiffDefaults(options) {
5003
- return {
5004
- ignoreDuration: false,
5005
- durationThresholdMs: DEFAULT_THRESHOLD_MS,
5006
- focus: "all",
5007
- check: "all"
5008
- };
5009
- }
5010
- function kindMatchesFilter(kind, merged) {
5011
- return true;
5012
- }
5013
- function diffRuns(left, right, options) {
5014
- const merged = mergeDiffDefaults();
5015
- const opts = {
5016
- ignoreDuration: merged.ignoreDuration,
5017
- durationThresholdMs: merged.durationThresholdMs
5018
- };
5019
- const raw = [];
5020
- if ((left.status ?? "") !== (right.status ?? "")) {
5021
- raw.push({
5022
- kind: "run-status",
5023
- severity: "warning",
5024
- message: "Run completion status differs",
5025
- left: left.status,
5026
- right: right.status
5027
- });
5028
- }
5029
- {
5030
- const ld = left.durationMs;
5031
- const rd = right.durationMs;
5032
- const th = merged.durationThresholdMs;
5033
- let differs = false;
5034
- if (ld === void 0 && rd === void 0) differs = false;
5035
- else if (ld === void 0 || rd === void 0) differs = true;
5036
- else differs = Math.abs(ld - rd) > th;
5037
- if (differs) {
5038
- raw.push({
5039
- kind: "duration",
5040
- severity: "info",
5041
- message: "Run duration differs",
5042
- left: ld,
5043
- right: rd
5044
- });
5045
- }
5046
- }
5047
- const pairs = pairSteps(left.steps, right.steps);
5048
- let idx = 0;
5049
- for (const [ls, rs] of pairs) {
5050
- if (ls !== void 0 && rs !== void 0) {
5051
- compareRecursive(ls, rs, [pathSeg(ls, idx)], opts, raw);
5052
- idx += 1;
5053
- } else if (ls !== void 0) {
5054
- raw.push({
5055
- kind: "step-removed",
5056
- severity: "warning",
5057
- message: `Step only in left run: ${ls.name}`,
5058
- path: buildPath([pathSeg(ls, idx)]),
5059
- left: ls.id,
5060
- right: void 0
5061
- });
5062
- idx += 1;
5063
- } else if (rs !== void 0) {
5064
- raw.push({
5065
- kind: "step-added",
5066
- severity: "warning",
5067
- message: `Step only in right run: ${rs.name}`,
5068
- path: buildPath([pathSeg(rs, idx)]),
5069
- left: void 0,
5070
- right: rs.id
5071
- });
5072
- idx += 1;
5073
- }
5074
- }
5075
- const differences = raw.filter((d) => kindMatchesFilter(d.kind));
5076
- let errors = 0;
5077
- let warnings = 0;
5078
- let info = 0;
5079
- for (const d of differences) {
5080
- if (d.severity === "error") errors += 1;
5081
- else if (d.severity === "warning") warnings += 1;
5082
- else info += 1;
5083
- }
5084
- const firstVisible = differences[0];
5085
- const firstDivergence = firstVisible !== void 0 ? {
5086
- kind: "first-divergence",
5087
- severity: firstVisible.severity,
5088
- message: `First divergence: ${firstVisible.message}`,
5089
- path: firstVisible.path,
5090
- left: firstVisible.left,
5091
- right: firstVisible.right
5092
- } : void 0;
5093
- const summary = {
5094
- leftRunId: left.runId,
5095
- rightRunId: right.runId,
5096
- totalDifferences: differences.length,
5097
- errors,
5098
- warnings,
5099
- info,
5100
- firstDivergence
5101
- };
5102
- return { summary, differences };
5103
- }
5104
-
5105
- // packages/viewer/src/suite-data.ts
5106
5117
  async function enrichCase(suiteCase, baselineTracePath) {
5107
5118
  const base = {
5108
5119
  id: suiteCase.id,
@@ -5162,7 +5173,7 @@ async function loadSuiteViewerData(options) {
5162
5173
  for (const suiteCase of result.cases) {
5163
5174
  cases.push(await enrichCase(suiteCase, baselinePath));
5164
5175
  }
5165
- const artifactsDir = path13__default.default.join(path13__default.default.dirname(result.configPath), ".agent-inspect/suite-runs");
5176
+ const artifactsDir = path15__default.default.join(path15__default.default.dirname(result.configPath), ".agent-inspect/suite-runs");
5166
5177
  return {
5167
5178
  suiteName: result.suiteName,
5168
5179
  configPath: result.configPath,
@@ -5323,19 +5334,19 @@ function parseWorkspaceManifest(json) {
5323
5334
  }
5324
5335
  var INDEX_DIR_NAME = "index";
5325
5336
  function resolveWorkspaceLocation(cwd = process.cwd()) {
5326
- const projectRoot = path13__default.default.resolve(cwd);
5327
- const workspaceDir = path13__default.default.join(projectRoot, WORKSPACE_DIR_NAME);
5337
+ const projectRoot = path15__default.default.resolve(cwd);
5338
+ const workspaceDir = path15__default.default.join(projectRoot, WORKSPACE_DIR_NAME);
5328
5339
  return {
5329
5340
  projectRoot,
5330
5341
  workspaceDir,
5331
- manifestPath: path13__default.default.join(workspaceDir, WORKSPACE_MANIFEST_FILENAME)
5342
+ manifestPath: path15__default.default.join(workspaceDir, WORKSPACE_MANIFEST_FILENAME)
5332
5343
  };
5333
5344
  }
5334
5345
  function resolveInsideWorkspace(workspaceDir, relative) {
5335
- const base = path13__default.default.resolve(workspaceDir);
5336
- const resolved = path13__default.default.resolve(base, relative);
5337
- const rel = path13__default.default.relative(base, resolved);
5338
- if (rel === "" || rel === "." || !rel.startsWith("..") && !path13__default.default.isAbsolute(rel)) {
5346
+ const base = path15__default.default.resolve(workspaceDir);
5347
+ const resolved = path15__default.default.resolve(base, relative);
5348
+ const rel = path15__default.default.relative(base, resolved);
5349
+ if (rel === "" || rel === "." || !rel.startsWith("..") && !path15__default.default.isAbsolute(rel)) {
5339
5350
  return resolved;
5340
5351
  }
5341
5352
  throw new Error(
@@ -5479,7 +5490,7 @@ async function doctorWorkspace(location) {
5479
5490
  const abs = resolveInsideWorkspace(location.workspaceDir, rel);
5480
5491
  for (const file of await listJsonl(abs)) {
5481
5492
  try {
5482
- const s = await promises.stat(path13__default.default.join(abs, file));
5493
+ const s = await promises.stat(path15__default.default.join(abs, file));
5483
5494
  newestTraceMtime = Math.max(newestTraceMtime, s.mtimeMs);
5484
5495
  } catch {
5485
5496
  checks.push({ id: "trace-readability", status: "warn", message: `cannot stat ${rel}/${file}` });
@@ -5509,7 +5520,7 @@ async function doctorWorkspace(location) {
5509
5520
 
5510
5521
  // packages/viewer/src/workspace-data.ts
5511
5522
  async function loadWorkspaceViewerData(options) {
5512
- const cwd = path13__default.default.resolve(options.cwd ?? process.cwd());
5523
+ const cwd = path15__default.default.resolve(options.cwd ?? process.cwd());
5513
5524
  const location = resolveWorkspaceLocation(cwd);
5514
5525
  const manifestRead = await readWorkspaceManifestFile(location);
5515
5526
  if (!manifestRead.ok || manifestRead.manifest === void 0) {
@@ -5522,7 +5533,7 @@ async function loadWorkspaceViewerData(options) {
5522
5533
  const runs = [];
5523
5534
  for (const rel of manifestRead.manifest.traceDirs) {
5524
5535
  const traceDir = resolveTraceDir({
5525
- dir: path13__default.default.join(location.workspaceDir, rel)
5536
+ dir: path15__default.default.join(location.workspaceDir, rel)
5526
5537
  });
5527
5538
  const td = new TraceDirectory({ dir: traceDir });
5528
5539
  const files = await td.list();
@@ -5536,7 +5547,7 @@ async function loadWorkspaceViewerData(options) {
5536
5547
  runId: meta.runId,
5537
5548
  ...meta.name !== void 0 ? { name: meta.name } : {},
5538
5549
  status: meta.status,
5539
- file: path13__default.default.basename(meta.filePath)
5550
+ file: path15__default.default.basename(meta.filePath)
5540
5551
  });
5541
5552
  }
5542
5553
  }
@@ -5547,8 +5558,8 @@ async function loadWorkspaceViewerData(options) {
5547
5558
  doctor,
5548
5559
  runs,
5549
5560
  bundleDirs: [
5550
- path13__default.default.join(location.workspaceDir, manifestRead.manifest.bundlesDir),
5551
- path13__default.default.join(location.workspaceDir, manifestRead.manifest.artifactsDir)
5561
+ path15__default.default.join(location.workspaceDir, manifestRead.manifest.bundlesDir),
5562
+ path15__default.default.join(location.workspaceDir, manifestRead.manifest.artifactsDir)
5552
5563
  ]
5553
5564
  };
5554
5565
  }
@@ -5611,7 +5622,7 @@ function createViewerServer(options = {}) {
5611
5622
  ok: true,
5612
5623
  readOnly: true,
5613
5624
  mode,
5614
- traceDir: path13__default.default.resolve(traceDir)
5625
+ traceDir: path15__default.default.resolve(traceDir)
5615
5626
  });
5616
5627
  }
5617
5628
  if (pathname === "/api/suite" && mode === "suite") {
@@ -5640,7 +5651,7 @@ function createViewerServer(options = {}) {
5640
5651
  runId: meta.runId,
5641
5652
  name: meta.name,
5642
5653
  status: meta.status,
5643
- file: path13__default.default.basename(meta.filePath),
5654
+ file: path15__default.default.basename(meta.filePath),
5644
5655
  startedAt: meta.startedAt,
5645
5656
  durationMs: meta.durationMs
5646
5657
  }))
@@ -5757,7 +5768,7 @@ function startViewerServer(options = {}) {
5757
5768
  resolve({
5758
5769
  host,
5759
5770
  port: resolvedPort,
5760
- traceDir: path13__default.default.resolve(traceDir),
5771
+ traceDir: path15__default.default.resolve(traceDir),
5761
5772
  url: `http://${host}:${resolvedPort}/${modeQuery}`,
5762
5773
  mode
5763
5774
  });