@agent-inspect/viewer 6.9.0 → 6.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +862 -851
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.mjs +861 -850
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var http = require('http');
|
|
4
|
-
var
|
|
4
|
+
var path15 = require('path');
|
|
5
5
|
var async_hooks = require('async_hooks');
|
|
6
6
|
require('crypto');
|
|
7
7
|
var promises = require('fs/promises');
|
|
@@ -14,7 +14,7 @@ var url = require('url');
|
|
|
14
14
|
|
|
15
15
|
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
16
16
|
|
|
17
|
-
var
|
|
17
|
+
var path15__default = /*#__PURE__*/_interopDefault(path15);
|
|
18
18
|
var os__default = /*#__PURE__*/_interopDefault(os);
|
|
19
19
|
|
|
20
20
|
// packages/viewer/src/server.ts
|
|
@@ -525,7 +525,7 @@ function persistedInspectEventsToTraceEvents(events, options) {
|
|
|
525
525
|
}
|
|
526
526
|
var DEFAULT_TRACE_DIR_NAME = ".agent-inspect";
|
|
527
527
|
var RUNS_DIR_NAME = "runs";
|
|
528
|
-
var FALLBACK_TRACE_DIR =
|
|
528
|
+
var FALLBACK_TRACE_DIR = path15__default.default.join(
|
|
529
529
|
os__default.default.tmpdir(),
|
|
530
530
|
"agent-inspect",
|
|
531
531
|
RUNS_DIR_NAME
|
|
@@ -540,7 +540,7 @@ function getDefaultTraceDir() {
|
|
|
540
540
|
if (typeof home !== "string" || home.trim() === "") {
|
|
541
541
|
return FALLBACK_TRACE_DIR;
|
|
542
542
|
}
|
|
543
|
-
return
|
|
543
|
+
return path15__default.default.join(home, DEFAULT_TRACE_DIR_NAME, RUNS_DIR_NAME);
|
|
544
544
|
} catch {
|
|
545
545
|
return FALLBACK_TRACE_DIR;
|
|
546
546
|
}
|
|
@@ -548,11 +548,11 @@ function getDefaultTraceDir() {
|
|
|
548
548
|
function getTraceFilePath(runId, traceDir) {
|
|
549
549
|
const baseDir = traceDir ?? getDefaultTraceDir();
|
|
550
550
|
let safeId = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "run_unknown";
|
|
551
|
-
safeId =
|
|
551
|
+
safeId = path15__default.default.basename(safeId);
|
|
552
552
|
if (safeId === "" || safeId === "." || safeId === "..") {
|
|
553
553
|
safeId = "run_unknown";
|
|
554
554
|
}
|
|
555
|
-
return
|
|
555
|
+
return path15__default.default.join(baseDir, `${safeId}.jsonl`);
|
|
556
556
|
}
|
|
557
557
|
function formatError(error) {
|
|
558
558
|
if (error instanceof Error) {
|
|
@@ -830,7 +830,7 @@ var TraceDirectory = class {
|
|
|
830
830
|
this.#dir = resolveTraceDir(options);
|
|
831
831
|
}
|
|
832
832
|
getPath(filename) {
|
|
833
|
-
return filename ?
|
|
833
|
+
return filename ? path15__default.default.join(this.#dir, filename) : this.#dir;
|
|
834
834
|
}
|
|
835
835
|
async list() {
|
|
836
836
|
try {
|
|
@@ -857,7 +857,7 @@ function parseIsoToMs2(value) {
|
|
|
857
857
|
}
|
|
858
858
|
async function extractMetadata(filePath, _quickScan) {
|
|
859
859
|
const stats = await promises.stat(filePath);
|
|
860
|
-
let runIdFromFile =
|
|
860
|
+
let runIdFromFile = path15__default.default.basename(filePath);
|
|
861
861
|
if (runIdFromFile.endsWith(".jsonl")) {
|
|
862
862
|
runIdFromFile = runIdFromFile.slice(0, -".jsonl".length);
|
|
863
863
|
}
|
|
@@ -1467,12 +1467,12 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
1467
1467
|
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
1468
1468
|
);
|
|
1469
1469
|
const ordered = [...runs].sort(compareRuns);
|
|
1470
|
-
const
|
|
1470
|
+
const path18 = [];
|
|
1471
1471
|
const visited = /* @__PURE__ */ new Set();
|
|
1472
1472
|
const pushRun = (run, confidence, source) => {
|
|
1473
1473
|
if (visited.has(run.runId)) return;
|
|
1474
1474
|
visited.add(run.runId);
|
|
1475
|
-
|
|
1475
|
+
path18.push({
|
|
1476
1476
|
runId: run.runId,
|
|
1477
1477
|
name: run.name,
|
|
1478
1478
|
startedAt: run.startedAt,
|
|
@@ -1497,7 +1497,7 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
1497
1497
|
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
1498
1498
|
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
1499
1499
|
}
|
|
1500
|
-
return
|
|
1500
|
+
return path18;
|
|
1501
1501
|
}
|
|
1502
1502
|
function metaRunIdMatches(run, token, runById) {
|
|
1503
1503
|
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
@@ -1574,524 +1574,915 @@ function buildSessionIndex(inputRuns, options = {}) {
|
|
|
1574
1574
|
};
|
|
1575
1575
|
}
|
|
1576
1576
|
|
|
1577
|
-
// packages/core/src/
|
|
1578
|
-
|
|
1579
|
-
"
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
return { code, message, severity, ...{} };
|
|
1586
|
-
}
|
|
1587
|
-
function asString(value, label) {
|
|
1588
|
-
if (value === void 0) return void 0;
|
|
1589
|
-
if (typeof value !== "string" || value.trim() === "") {
|
|
1590
|
-
throw new Error(`${label} must be a non-empty string.`);
|
|
1577
|
+
// packages/core/src/exporters/helpers.ts
|
|
1578
|
+
function sortKeysDeep(input) {
|
|
1579
|
+
if (input === null || typeof input !== "object") return input;
|
|
1580
|
+
if (Array.isArray(input)) return input.map(sortKeysDeep);
|
|
1581
|
+
const o = input;
|
|
1582
|
+
const out = {};
|
|
1583
|
+
for (const k of Object.keys(o).sort()) {
|
|
1584
|
+
out[k] = sortKeysDeep(o[k]);
|
|
1591
1585
|
}
|
|
1592
|
-
return
|
|
1586
|
+
return out;
|
|
1593
1587
|
}
|
|
1594
|
-
function
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
throw new Error(`${label} must be an array of strings.`);
|
|
1598
|
-
}
|
|
1599
|
-
return value;
|
|
1588
|
+
function stableJson(value, pretty) {
|
|
1589
|
+
const sorted = sortKeysDeep(value);
|
|
1590
|
+
return JSON.stringify(sorted);
|
|
1600
1591
|
}
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
return
|
|
1592
|
+
|
|
1593
|
+
// packages/core/src/diff/comparable.ts
|
|
1594
|
+
function extractOutputPreview(meta) {
|
|
1595
|
+
if (meta === void 0) return void 0;
|
|
1596
|
+
if ("outputPreview" in meta) return meta.outputPreview;
|
|
1597
|
+
if ("resultPreview" in meta) return meta.resultPreview;
|
|
1598
|
+
return void 0;
|
|
1607
1599
|
}
|
|
1608
|
-
function
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
diagnostics.push(
|
|
1612
|
-
diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
|
|
1613
|
-
);
|
|
1614
|
-
return { diagnostics };
|
|
1615
|
-
}
|
|
1616
|
-
const raw = value;
|
|
1617
|
-
try {
|
|
1618
|
-
const id = asString(raw.id, `cases[${index}].id`);
|
|
1619
|
-
if (id === void 0) {
|
|
1620
|
-
diagnostics.push(
|
|
1621
|
-
diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
|
|
1622
|
-
);
|
|
1623
|
-
return { diagnostics };
|
|
1624
|
-
}
|
|
1625
|
-
const trace = asString(raw.trace, `cases[${index}].trace`);
|
|
1626
|
-
const runId = asString(raw.runId, `cases[${index}].runId`);
|
|
1627
|
-
const input = asString(raw.input, `cases[${index}].input`);
|
|
1628
|
-
return {
|
|
1629
|
-
caseConfig: {
|
|
1630
|
-
id,
|
|
1631
|
-
...trace !== void 0 ? { trace } : {},
|
|
1632
|
-
...runId !== void 0 ? { runId } : {},
|
|
1633
|
-
...input !== void 0 ? { input } : {},
|
|
1634
|
-
...asStringArray(raw.requireTools, `cases[${index}].requireTools`) !== void 0 ? { requireTools: asStringArray(raw.requireTools, `cases[${index}].requireTools`) } : {},
|
|
1635
|
-
...asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) !== void 0 ? { forbidTools: asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) } : {},
|
|
1636
|
-
...asPositiveNumber(raw.maxDurationMs, `cases[${index}].maxDurationMs`) !== void 0 ? {
|
|
1637
|
-
maxDurationMs: asPositiveNumber(
|
|
1638
|
-
raw.maxDurationMs,
|
|
1639
|
-
`cases[${index}].maxDurationMs`
|
|
1640
|
-
)
|
|
1641
|
-
} : {},
|
|
1642
|
-
...asStringArray(
|
|
1643
|
-
raw.expectedObservations,
|
|
1644
|
-
`cases[${index}].expectedObservations`
|
|
1645
|
-
) !== void 0 ? {
|
|
1646
|
-
expectedObservations: asStringArray(
|
|
1647
|
-
raw.expectedObservations,
|
|
1648
|
-
`cases[${index}].expectedObservations`
|
|
1649
|
-
)
|
|
1650
|
-
} : {}
|
|
1651
|
-
},
|
|
1652
|
-
diagnostics
|
|
1653
|
-
};
|
|
1654
|
-
} catch (error) {
|
|
1655
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
1656
|
-
diagnostics.push(diagnostic("AI_SUITE_CONFIG_INVALID", message));
|
|
1657
|
-
return { diagnostics };
|
|
1658
|
-
}
|
|
1600
|
+
function mapStepStatus(s) {
|
|
1601
|
+
if (s === void 0) return "running";
|
|
1602
|
+
return s;
|
|
1659
1603
|
}
|
|
1660
|
-
function
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
const raw = value;
|
|
1665
|
-
const name = asString(raw.name, "name");
|
|
1666
|
-
const traces = asString(raw.traces, "traces");
|
|
1667
|
-
if (name === void 0) throw new Error("name is required.");
|
|
1668
|
-
if (traces === void 0) throw new Error("traces is required.");
|
|
1669
|
-
if (!Array.isArray(raw.cases) || raw.cases.length === 0) {
|
|
1670
|
-
throw new Error("cases must be a non-empty array.");
|
|
1604
|
+
function manualTraceEventsToComparableRun(events) {
|
|
1605
|
+
const started = events.find((e) => e.event === "run_started");
|
|
1606
|
+
if (!started || started.event !== "run_started") {
|
|
1607
|
+
throw new Error("Invalid trace: missing run_started");
|
|
1671
1608
|
}
|
|
1672
|
-
const
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1609
|
+
const rs = started;
|
|
1610
|
+
const runId = rs.runId;
|
|
1611
|
+
const completedAll = events.filter((e) => e.event === "run_completed");
|
|
1612
|
+
const lastCompleted = completedAll[completedAll.length - 1];
|
|
1613
|
+
let runStatus;
|
|
1614
|
+
if (lastCompleted === void 0) runStatus = "running";
|
|
1615
|
+
else runStatus = lastCompleted.status;
|
|
1616
|
+
const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
1617
|
+
const steps = /* @__PURE__ */ new Map();
|
|
1618
|
+
let order = 0;
|
|
1619
|
+
for (const e of events) {
|
|
1620
|
+
if (e.event !== "step_started") continue;
|
|
1621
|
+
const s = e;
|
|
1622
|
+
const meta = s.metadata ? { ...s.metadata } : void 0;
|
|
1623
|
+
steps.set(s.stepId, {
|
|
1624
|
+
id: s.stepId,
|
|
1625
|
+
parentId: s.parentId,
|
|
1626
|
+
name: s.name,
|
|
1627
|
+
type: s.type,
|
|
1628
|
+
order: order++,
|
|
1629
|
+
timestamp: s.timestamp,
|
|
1630
|
+
metadata: meta
|
|
1631
|
+
});
|
|
1679
1632
|
}
|
|
1680
|
-
const
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1633
|
+
for (const e of events) {
|
|
1634
|
+
if (e.event !== "step_completed") continue;
|
|
1635
|
+
const acc = steps.get(e.stepId);
|
|
1636
|
+
if (!acc) continue;
|
|
1637
|
+
acc.status = e.status;
|
|
1638
|
+
acc.durationMs = e.durationMs;
|
|
1639
|
+
if (e.error?.message) acc.errorMsg = e.error.message;
|
|
1640
|
+
const extra = e;
|
|
1641
|
+
if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
|
|
1642
|
+
acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
|
|
1684
1643
|
}
|
|
1685
|
-
ids.add(suiteCase.id);
|
|
1686
1644
|
}
|
|
1687
|
-
const
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
if (raw.checks !== void 0) {
|
|
1693
|
-
if (typeof raw.checks !== "object" || Array.isArray(raw.checks)) {
|
|
1694
|
-
throw new Error("checks must be an object.");
|
|
1645
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
1646
|
+
for (const acc of steps.values()) {
|
|
1647
|
+
let meta = acc.metadata ? { ...acc.metadata } : void 0;
|
|
1648
|
+
if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
|
|
1649
|
+
meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
|
|
1695
1650
|
}
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
throw new Error("eval must be an object.");
|
|
1651
|
+
const outputPreview = extractOutputPreview(meta);
|
|
1652
|
+
if (meta !== void 0 && ("outputPreview" in meta || "resultPreview" in meta)) {
|
|
1653
|
+
delete meta.outputPreview;
|
|
1654
|
+
delete meta.resultPreview;
|
|
1701
1655
|
}
|
|
1702
|
-
|
|
1656
|
+
const sc = {
|
|
1657
|
+
id: acc.id,
|
|
1658
|
+
name: acc.name,
|
|
1659
|
+
type: acc.type,
|
|
1660
|
+
status: mapStepStatus(acc.status),
|
|
1661
|
+
durationMs: acc.durationMs,
|
|
1662
|
+
error: acc.errorMsg,
|
|
1663
|
+
metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
|
|
1664
|
+
outputPreview,
|
|
1665
|
+
children: []
|
|
1666
|
+
};
|
|
1667
|
+
nodes.set(acc.id, sc);
|
|
1703
1668
|
}
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1669
|
+
const roots = [];
|
|
1670
|
+
const sortByOrder = (a, b) => {
|
|
1671
|
+
const oa = steps.get(a.id)?.order ?? 0;
|
|
1672
|
+
const ob = steps.get(b.id)?.order ?? 0;
|
|
1673
|
+
return oa - ob;
|
|
1674
|
+
};
|
|
1675
|
+
for (const acc of steps.values()) {
|
|
1676
|
+
const node = nodes.get(acc.id);
|
|
1677
|
+
if (acc.parentId !== void 0 && nodes.has(acc.parentId)) {
|
|
1678
|
+
nodes.get(acc.parentId).children.push(node);
|
|
1679
|
+
} else {
|
|
1680
|
+
roots.push(node);
|
|
1707
1681
|
}
|
|
1708
|
-
const outputDir = asString(
|
|
1709
|
-
raw.artifacts.outputDir,
|
|
1710
|
-
"artifacts.outputDir"
|
|
1711
|
-
);
|
|
1712
|
-
config.artifacts = outputDir !== void 0 ? { outputDir } : {};
|
|
1713
|
-
}
|
|
1714
|
-
if (raw.baseline !== void 0) {
|
|
1715
|
-
const baseline = asString(raw.baseline, "baseline");
|
|
1716
|
-
if (baseline !== void 0) config.baseline = baseline;
|
|
1717
1682
|
}
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1683
|
+
roots.sort(sortByOrder);
|
|
1684
|
+
for (const n of nodes.values()) {
|
|
1685
|
+
n.children.sort(sortByOrder);
|
|
1721
1686
|
}
|
|
1722
|
-
|
|
1723
|
-
|
|
1687
|
+
return {
|
|
1688
|
+
runId,
|
|
1689
|
+
name: rs.name,
|
|
1690
|
+
status: runStatus,
|
|
1691
|
+
durationMs,
|
|
1692
|
+
steps: roots
|
|
1693
|
+
};
|
|
1724
1694
|
}
|
|
1725
1695
|
|
|
1726
|
-
// packages/core/src/
|
|
1727
|
-
var
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
return { code, message, severity: "error" };
|
|
1696
|
+
// packages/core/src/diff/engine.ts
|
|
1697
|
+
var DEFAULT_THRESHOLD_MS = 0;
|
|
1698
|
+
function pathSeg(step, index) {
|
|
1699
|
+
return { index, name: step.name, stepId: step.id };
|
|
1731
1700
|
}
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
await promises.access(filePath);
|
|
1735
|
-
return true;
|
|
1736
|
-
} catch {
|
|
1737
|
-
return false;
|
|
1738
|
-
}
|
|
1701
|
+
function buildPath(segments) {
|
|
1702
|
+
return { path: [...segments] };
|
|
1739
1703
|
}
|
|
1740
|
-
|
|
1741
|
-
const
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1704
|
+
function pairSteps(left, right) {
|
|
1705
|
+
const usedRight = /* @__PURE__ */ new Set();
|
|
1706
|
+
const pairs = [];
|
|
1707
|
+
for (let i = 0; i < left.length; i++) {
|
|
1708
|
+
const L = left[i];
|
|
1709
|
+
let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
|
|
1710
|
+
if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
|
|
1711
|
+
const cand = right[i];
|
|
1712
|
+
if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
|
|
1713
|
+
R = cand;
|
|
1714
|
+
}
|
|
1715
|
+
}
|
|
1716
|
+
if (R === void 0) {
|
|
1717
|
+
R = right.find(
|
|
1718
|
+
(r) => !usedRight.has(r.id) && r.name === L.name && (r.type ?? "") === (L.type ?? "")
|
|
1719
|
+
);
|
|
1720
|
+
}
|
|
1721
|
+
if (R !== void 0) {
|
|
1722
|
+
usedRight.add(R.id);
|
|
1723
|
+
pairs.push([L, R]);
|
|
1724
|
+
} else {
|
|
1725
|
+
pairs.push([L, void 0]);
|
|
1726
|
+
}
|
|
1748
1727
|
}
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1728
|
+
for (const R of right) {
|
|
1729
|
+
if (!usedRight.has(R.id)) {
|
|
1730
|
+
pairs.push([void 0, R]);
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
return pairs;
|
|
1752
1734
|
}
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1735
|
+
function compareLeafSteps(L, R, segments, opts, out) {
|
|
1736
|
+
const path18 = buildPath(segments);
|
|
1737
|
+
if (L.name !== R.name) {
|
|
1738
|
+
out.push({
|
|
1739
|
+
kind: "structure",
|
|
1740
|
+
severity: "warning",
|
|
1741
|
+
message: "Step name differs",
|
|
1742
|
+
path: path18,
|
|
1743
|
+
left: L.name,
|
|
1744
|
+
right: R.name
|
|
1761
1745
|
});
|
|
1762
1746
|
}
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1747
|
+
if ((L.type ?? "") !== (R.type ?? "")) {
|
|
1748
|
+
out.push({
|
|
1749
|
+
kind: "step-type",
|
|
1750
|
+
severity: "warning",
|
|
1751
|
+
message: "Step type differs",
|
|
1752
|
+
path: path18,
|
|
1753
|
+
left: L.type,
|
|
1754
|
+
right: R.type
|
|
1768
1755
|
});
|
|
1769
1756
|
}
|
|
1770
|
-
if (
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1757
|
+
if ((L.status ?? "") !== (R.status ?? "")) {
|
|
1758
|
+
out.push({
|
|
1759
|
+
kind: "step-status",
|
|
1760
|
+
severity: "warning",
|
|
1761
|
+
message: "Step status differs",
|
|
1762
|
+
path: path18,
|
|
1763
|
+
left: L.status,
|
|
1764
|
+
right: R.status
|
|
1774
1765
|
});
|
|
1775
1766
|
}
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
config,
|
|
1787
|
-
configPath,
|
|
1788
|
-
configDir: path13__default.default.dirname(configPath)
|
|
1789
|
-
};
|
|
1790
|
-
} catch (error) {
|
|
1791
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
1792
|
-
throw Object.assign(new Error(message), {
|
|
1793
|
-
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
1767
|
+
const le = L.error ?? "";
|
|
1768
|
+
const re = R.error ?? "";
|
|
1769
|
+
if (le !== re) {
|
|
1770
|
+
out.push({
|
|
1771
|
+
kind: "error",
|
|
1772
|
+
severity: "error",
|
|
1773
|
+
message: "Step error message differs",
|
|
1774
|
+
path: path18,
|
|
1775
|
+
left: le || void 0,
|
|
1776
|
+
right: re || void 0
|
|
1794
1777
|
});
|
|
1795
1778
|
}
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1779
|
+
if (!opts.ignoreDuration) {
|
|
1780
|
+
const ld = L.durationMs;
|
|
1781
|
+
const rd = R.durationMs;
|
|
1782
|
+
const th = opts.durationThresholdMs;
|
|
1783
|
+
let differs = false;
|
|
1784
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
1785
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
1786
|
+
else differs = Math.abs(ld - rd) > th;
|
|
1787
|
+
if (differs) {
|
|
1788
|
+
out.push({
|
|
1789
|
+
kind: "duration",
|
|
1790
|
+
severity: "info",
|
|
1791
|
+
message: "Step duration differs",
|
|
1792
|
+
path: path18,
|
|
1793
|
+
left: ld,
|
|
1794
|
+
right: rd
|
|
1795
|
+
});
|
|
1810
1796
|
}
|
|
1811
|
-
return {
|
|
1812
|
-
caseId: suiteCase.id,
|
|
1813
|
-
tracePath,
|
|
1814
|
-
missing: true,
|
|
1815
|
-
reason: `trace file not found: ${suiteCase.trace}`
|
|
1816
|
-
};
|
|
1817
1797
|
}
|
|
1818
|
-
const
|
|
1819
|
-
const
|
|
1820
|
-
if (
|
|
1821
|
-
|
|
1798
|
+
const lm = stableJson(L.metadata ?? {});
|
|
1799
|
+
const rm2 = stableJson(R.metadata ?? {});
|
|
1800
|
+
if (lm !== rm2) {
|
|
1801
|
+
out.push({
|
|
1802
|
+
kind: "metadata",
|
|
1803
|
+
severity: "info",
|
|
1804
|
+
message: "Step metadata differs",
|
|
1805
|
+
path: path18,
|
|
1806
|
+
left: L.metadata,
|
|
1807
|
+
right: R.metadata
|
|
1808
|
+
});
|
|
1822
1809
|
}
|
|
1823
|
-
const
|
|
1824
|
-
|
|
1825
|
-
|
|
1810
|
+
const lo = stableJson(L.outputPreview ?? null);
|
|
1811
|
+
const ro = stableJson(R.outputPreview ?? null);
|
|
1812
|
+
if (lo !== ro) {
|
|
1813
|
+
out.push({
|
|
1814
|
+
kind: "output",
|
|
1815
|
+
severity: "info",
|
|
1816
|
+
message: "Output preview differs",
|
|
1817
|
+
path: path18,
|
|
1818
|
+
left: L.outputPreview,
|
|
1819
|
+
right: R.outputPreview
|
|
1820
|
+
});
|
|
1826
1821
|
}
|
|
1827
|
-
return {
|
|
1828
|
-
caseId: suiteCase.id,
|
|
1829
|
-
runId: runKey,
|
|
1830
|
-
missing: true,
|
|
1831
|
-
reason: `no trace found for run id "${runKey}" under ${options.tracesDir}`
|
|
1832
|
-
};
|
|
1833
1822
|
}
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
}
|
|
1846
|
-
|
|
1847
|
-
|
|
1823
|
+
function compareRecursive(L, R, segments, opts, out) {
|
|
1824
|
+
compareLeafSteps(L, R, segments, opts, out);
|
|
1825
|
+
const pairs = pairSteps(L.children, R.children);
|
|
1826
|
+
let ci = 0;
|
|
1827
|
+
for (const [lch, rch] of pairs) {
|
|
1828
|
+
if (lch !== void 0 && rch !== void 0) {
|
|
1829
|
+
compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
|
|
1830
|
+
} else if (lch !== void 0) {
|
|
1831
|
+
out.push({
|
|
1832
|
+
kind: "step-removed",
|
|
1833
|
+
severity: "warning",
|
|
1834
|
+
message: `Step only in left run: ${lch.name}`,
|
|
1835
|
+
path: buildPath([...segments, pathSeg(lch, ci)]),
|
|
1836
|
+
left: lch.id,
|
|
1837
|
+
right: void 0
|
|
1838
|
+
});
|
|
1839
|
+
} else if (rch !== void 0) {
|
|
1840
|
+
out.push({
|
|
1841
|
+
kind: "step-added",
|
|
1842
|
+
severity: "warning",
|
|
1843
|
+
message: `Step only in right run: ${rch.name}`,
|
|
1844
|
+
path: buildPath([...segments, pathSeg(rch, ci)]),
|
|
1845
|
+
left: void 0,
|
|
1846
|
+
right: rch.id
|
|
1847
|
+
});
|
|
1848
|
+
}
|
|
1849
|
+
ci += 1;
|
|
1850
|
+
}
|
|
1848
1851
|
}
|
|
1849
|
-
function
|
|
1852
|
+
function mergeDiffDefaults(options) {
|
|
1850
1853
|
return {
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1854
|
+
ignoreDuration: false,
|
|
1855
|
+
durationThresholdMs: DEFAULT_THRESHOLD_MS,
|
|
1856
|
+
focus: "all",
|
|
1857
|
+
check: "all"
|
|
1855
1858
|
};
|
|
1856
1859
|
}
|
|
1857
|
-
function
|
|
1858
|
-
return
|
|
1859
|
-
passed: 0,
|
|
1860
|
-
failed: 0,
|
|
1861
|
-
warnings: 0,
|
|
1862
|
-
errors: 0
|
|
1863
|
-
};
|
|
1860
|
+
function kindMatchesFilter(kind, merged) {
|
|
1861
|
+
return true;
|
|
1864
1862
|
}
|
|
1865
|
-
function
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
...selectedRun ? { runId: selectedRun.runId } : {},
|
|
1871
|
-
summary: {
|
|
1872
|
-
...emptySummary(),
|
|
1873
|
-
errors: diagnostics.filter((item) => item.severity === "error").length
|
|
1874
|
-
},
|
|
1875
|
-
findings: [],
|
|
1876
|
-
diagnostics: [...diagnostics]
|
|
1863
|
+
function diffRuns(left, right, options) {
|
|
1864
|
+
const merged = mergeDiffDefaults();
|
|
1865
|
+
const opts = {
|
|
1866
|
+
ignoreDuration: merged.ignoreDuration,
|
|
1867
|
+
durationThresholdMs: merged.durationThresholdMs
|
|
1877
1868
|
};
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
const nodesByEventId = /* @__PURE__ */ new Map();
|
|
1888
|
-
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
1889
|
-
for (const node of nodes) {
|
|
1890
|
-
nodesByEventId.set(node.event.eventId, node);
|
|
1891
|
-
const parentId = node.event.parentId;
|
|
1892
|
-
if (parentId) {
|
|
1893
|
-
const children = childrenByParentId.get(parentId) ?? [];
|
|
1894
|
-
children.push(node);
|
|
1895
|
-
childrenByParentId.set(parentId, children);
|
|
1896
|
-
}
|
|
1869
|
+
const raw = [];
|
|
1870
|
+
if ((left.status ?? "") !== (right.status ?? "")) {
|
|
1871
|
+
raw.push({
|
|
1872
|
+
kind: "run-status",
|
|
1873
|
+
severity: "warning",
|
|
1874
|
+
message: "Run completion status differs",
|
|
1875
|
+
left: left.status,
|
|
1876
|
+
right: right.status
|
|
1877
|
+
});
|
|
1897
1878
|
}
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
diagnostics: [
|
|
1915
|
-
diagnostic3(
|
|
1916
|
-
"AI_CHECK_INVALID_ARGUMENTS",
|
|
1917
|
-
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
1918
|
-
)
|
|
1919
|
-
]
|
|
1920
|
-
};
|
|
1879
|
+
{
|
|
1880
|
+
const ld = left.durationMs;
|
|
1881
|
+
const rd = right.durationMs;
|
|
1882
|
+
const th = merged.durationThresholdMs;
|
|
1883
|
+
let differs = false;
|
|
1884
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
1885
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
1886
|
+
else differs = Math.abs(ld - rd) > th;
|
|
1887
|
+
if (differs) {
|
|
1888
|
+
raw.push({
|
|
1889
|
+
kind: "duration",
|
|
1890
|
+
severity: "info",
|
|
1891
|
+
message: "Run duration differs",
|
|
1892
|
+
left: ld,
|
|
1893
|
+
right: rd
|
|
1894
|
+
});
|
|
1921
1895
|
}
|
|
1922
|
-
return { run: input.selectedRun, diagnostics: [] };
|
|
1923
1896
|
}
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1897
|
+
const pairs = pairSteps(left.steps, right.steps);
|
|
1898
|
+
let idx = 0;
|
|
1899
|
+
for (const [ls, rs] of pairs) {
|
|
1900
|
+
if (ls !== void 0 && rs !== void 0) {
|
|
1901
|
+
compareRecursive(ls, rs, [pathSeg(ls, idx)], opts, raw);
|
|
1902
|
+
idx += 1;
|
|
1903
|
+
} else if (ls !== void 0) {
|
|
1904
|
+
raw.push({
|
|
1905
|
+
kind: "step-removed",
|
|
1906
|
+
severity: "warning",
|
|
1907
|
+
message: `Step only in left run: ${ls.name}`,
|
|
1908
|
+
path: buildPath([pathSeg(ls, idx)]),
|
|
1909
|
+
left: ls.id,
|
|
1910
|
+
right: void 0
|
|
1911
|
+
});
|
|
1912
|
+
idx += 1;
|
|
1913
|
+
} else if (rs !== void 0) {
|
|
1914
|
+
raw.push({
|
|
1915
|
+
kind: "step-added",
|
|
1916
|
+
severity: "warning",
|
|
1917
|
+
message: `Step only in right run: ${rs.name}`,
|
|
1918
|
+
path: buildPath([pathSeg(rs, idx)]),
|
|
1919
|
+
left: void 0,
|
|
1920
|
+
right: rs.id
|
|
1921
|
+
});
|
|
1922
|
+
idx += 1;
|
|
1932
1923
|
}
|
|
1933
|
-
return { run, diagnostics: [] };
|
|
1934
|
-
}
|
|
1935
|
-
if (input.read.runs.length === 1) {
|
|
1936
|
-
return { run: input.read.runs[0], diagnostics: [] };
|
|
1937
1924
|
}
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1925
|
+
const differences = raw.filter((d) => kindMatchesFilter(d.kind));
|
|
1926
|
+
let errors = 0;
|
|
1927
|
+
let warnings = 0;
|
|
1928
|
+
let info = 0;
|
|
1929
|
+
for (const d of differences) {
|
|
1930
|
+
if (d.severity === "error") errors += 1;
|
|
1931
|
+
else if (d.severity === "warning") warnings += 1;
|
|
1932
|
+
else info += 1;
|
|
1944
1933
|
}
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1934
|
+
const firstVisible = differences[0];
|
|
1935
|
+
const firstDivergence = firstVisible !== void 0 ? {
|
|
1936
|
+
kind: "first-divergence",
|
|
1937
|
+
severity: firstVisible.severity,
|
|
1938
|
+
message: `First divergence: ${firstVisible.message}`,
|
|
1939
|
+
path: firstVisible.path,
|
|
1940
|
+
left: firstVisible.left,
|
|
1941
|
+
right: firstVisible.right
|
|
1942
|
+
} : void 0;
|
|
1943
|
+
const summary = {
|
|
1944
|
+
leftRunId: left.runId,
|
|
1945
|
+
rightRunId: right.runId,
|
|
1946
|
+
totalDifferences: differences.length,
|
|
1947
|
+
errors,
|
|
1948
|
+
warnings,
|
|
1949
|
+
info,
|
|
1950
|
+
firstDivergence
|
|
1952
1951
|
};
|
|
1952
|
+
return { summary, differences };
|
|
1953
1953
|
}
|
|
1954
|
-
|
|
1954
|
+
|
|
1955
|
+
// packages/core/src/evidence/zip.ts
|
|
1956
|
+
(() => {
|
|
1957
|
+
const table = new Uint32Array(256);
|
|
1958
|
+
for (let n = 0; n < 256; n += 1) {
|
|
1959
|
+
let c = n;
|
|
1960
|
+
for (let k = 0; k < 8; k += 1) {
|
|
1961
|
+
c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
|
|
1962
|
+
}
|
|
1963
|
+
table[n] = c >>> 0;
|
|
1964
|
+
}
|
|
1965
|
+
return table;
|
|
1966
|
+
})();
|
|
1967
|
+
|
|
1968
|
+
// packages/core/src/suite/types.ts
|
|
1969
|
+
var DEFAULT_SUITE_CONFIG_NAMES = [
|
|
1970
|
+
"agent-inspect.suite.json",
|
|
1971
|
+
"agent-inspect.suite.js",
|
|
1972
|
+
"agent-inspect.suite.mjs",
|
|
1973
|
+
"agent-inspect.suite.cjs"
|
|
1974
|
+
];
|
|
1975
|
+
function diagnostic(code, message, severity = "error", caseId) {
|
|
1976
|
+
return { code, message, severity, ...{} };
|
|
1977
|
+
}
|
|
1978
|
+
function asString(value, label) {
|
|
1979
|
+
if (value === void 0) return void 0;
|
|
1980
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
1981
|
+
throw new Error(`${label} must be a non-empty string.`);
|
|
1982
|
+
}
|
|
1983
|
+
return value.trim();
|
|
1984
|
+
}
|
|
1985
|
+
function asStringArray(value, label) {
|
|
1986
|
+
if (value === void 0) return void 0;
|
|
1987
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
1988
|
+
throw new Error(`${label} must be an array of strings.`);
|
|
1989
|
+
}
|
|
1990
|
+
return value;
|
|
1991
|
+
}
|
|
1992
|
+
function asPositiveNumber(value, label) {
|
|
1993
|
+
if (value === void 0) return void 0;
|
|
1994
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
1995
|
+
throw new Error(`${label} must be a non-negative number.`);
|
|
1996
|
+
}
|
|
1997
|
+
return value;
|
|
1998
|
+
}
|
|
1999
|
+
function validateCaseConfig(value, index) {
|
|
1955
2000
|
const diagnostics = [];
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
2001
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
2002
|
+
diagnostics.push(
|
|
2003
|
+
diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
|
|
2004
|
+
);
|
|
2005
|
+
return { diagnostics };
|
|
2006
|
+
}
|
|
2007
|
+
const raw = value;
|
|
2008
|
+
try {
|
|
2009
|
+
const id = asString(raw.id, `cases[${index}].id`);
|
|
2010
|
+
if (id === void 0) {
|
|
1959
2011
|
diagnostics.push(
|
|
1960
|
-
|
|
2012
|
+
diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
|
|
1961
2013
|
);
|
|
1962
|
-
|
|
1963
|
-
}
|
|
1964
|
-
byId.set(rule.id, rule);
|
|
1965
|
-
}
|
|
1966
|
-
if (selectedIds && selectedIds.length > 0) {
|
|
1967
|
-
const selected = new Set(selectedIds);
|
|
1968
|
-
for (const id of selected) {
|
|
1969
|
-
if (!byId.has(id)) {
|
|
1970
|
-
diagnostics.push(
|
|
1971
|
-
diagnostic3("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
1972
|
-
);
|
|
1973
|
-
}
|
|
2014
|
+
return { diagnostics };
|
|
1974
2015
|
}
|
|
2016
|
+
const trace = asString(raw.trace, `cases[${index}].trace`);
|
|
2017
|
+
const runId = asString(raw.runId, `cases[${index}].runId`);
|
|
2018
|
+
const input = asString(raw.input, `cases[${index}].input`);
|
|
1975
2019
|
return {
|
|
1976
|
-
|
|
2020
|
+
caseConfig: {
|
|
2021
|
+
id,
|
|
2022
|
+
...trace !== void 0 ? { trace } : {},
|
|
2023
|
+
...runId !== void 0 ? { runId } : {},
|
|
2024
|
+
...input !== void 0 ? { input } : {},
|
|
2025
|
+
...asStringArray(raw.requireTools, `cases[${index}].requireTools`) !== void 0 ? { requireTools: asStringArray(raw.requireTools, `cases[${index}].requireTools`) } : {},
|
|
2026
|
+
...asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) !== void 0 ? { forbidTools: asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) } : {},
|
|
2027
|
+
...asPositiveNumber(raw.maxDurationMs, `cases[${index}].maxDurationMs`) !== void 0 ? {
|
|
2028
|
+
maxDurationMs: asPositiveNumber(
|
|
2029
|
+
raw.maxDurationMs,
|
|
2030
|
+
`cases[${index}].maxDurationMs`
|
|
2031
|
+
)
|
|
2032
|
+
} : {},
|
|
2033
|
+
...asStringArray(
|
|
2034
|
+
raw.expectedObservations,
|
|
2035
|
+
`cases[${index}].expectedObservations`
|
|
2036
|
+
) !== void 0 ? {
|
|
2037
|
+
expectedObservations: asStringArray(
|
|
2038
|
+
raw.expectedObservations,
|
|
2039
|
+
`cases[${index}].expectedObservations`
|
|
2040
|
+
)
|
|
2041
|
+
} : {}
|
|
2042
|
+
},
|
|
1977
2043
|
diagnostics
|
|
1978
2044
|
};
|
|
2045
|
+
} catch (error) {
|
|
2046
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2047
|
+
diagnostics.push(diagnostic("AI_SUITE_CONFIG_INVALID", message));
|
|
2048
|
+
return { diagnostics };
|
|
1979
2049
|
}
|
|
1980
|
-
return { rules: [...byId.values()].sort(compareRules), diagnostics };
|
|
1981
|
-
}
|
|
1982
|
-
function compareRules(a, b) {
|
|
1983
|
-
return a.id.localeCompare(b.id);
|
|
1984
|
-
}
|
|
1985
|
-
function eventTimestamp(finding, eventById) {
|
|
1986
|
-
const eventId = finding.evidence[0]?.eventId;
|
|
1987
|
-
return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
|
|
1988
2050
|
}
|
|
1989
|
-
function
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
2051
|
+
function normalizeSuiteConfig(value) {
|
|
2052
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
2053
|
+
throw new Error("Suite config must export an object.");
|
|
2054
|
+
}
|
|
2055
|
+
const raw = value;
|
|
2056
|
+
const name = asString(raw.name, "name");
|
|
2057
|
+
const traces = asString(raw.traces, "traces");
|
|
2058
|
+
if (name === void 0) throw new Error("name is required.");
|
|
2059
|
+
if (traces === void 0) throw new Error("traces is required.");
|
|
2060
|
+
if (!Array.isArray(raw.cases) || raw.cases.length === 0) {
|
|
2061
|
+
throw new Error("cases must be a non-empty array.");
|
|
2062
|
+
}
|
|
2063
|
+
const cases = [];
|
|
2064
|
+
for (let index = 0; index < raw.cases.length; index += 1) {
|
|
2065
|
+
const { caseConfig, diagnostics } = validateCaseConfig(raw.cases[index], index);
|
|
2066
|
+
if (diagnostics.length > 0) {
|
|
2067
|
+
throw new Error(diagnostics.map((item) => item.message).join("; "));
|
|
1993
2068
|
}
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
2069
|
+
if (caseConfig !== void 0) cases.push(caseConfig);
|
|
2070
|
+
}
|
|
2071
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2072
|
+
for (const suiteCase of cases) {
|
|
2073
|
+
if (ids.has(suiteCase.id)) {
|
|
2074
|
+
throw new Error(`Duplicate case id "${suiteCase.id}".`);
|
|
1998
2075
|
}
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2076
|
+
ids.add(suiteCase.id);
|
|
2077
|
+
}
|
|
2078
|
+
const redactionProfile = raw.redactionProfile === "local" || raw.redactionProfile === "share" || raw.redactionProfile === "strict" ? raw.redactionProfile : void 0;
|
|
2079
|
+
if (raw.redactionProfile !== void 0 && redactionProfile === void 0) {
|
|
2080
|
+
throw new Error('redactionProfile must be "local", "share", or "strict".');
|
|
2081
|
+
}
|
|
2082
|
+
const config = { name, traces, cases };
|
|
2083
|
+
if (raw.checks !== void 0) {
|
|
2084
|
+
if (typeof raw.checks !== "object" || Array.isArray(raw.checks)) {
|
|
2085
|
+
throw new Error("checks must be an object.");
|
|
2086
|
+
}
|
|
2087
|
+
config.checks = raw.checks;
|
|
2088
|
+
}
|
|
2089
|
+
if (raw.eval !== void 0) {
|
|
2090
|
+
if (typeof raw.eval !== "object" || Array.isArray(raw.eval)) {
|
|
2091
|
+
throw new Error("eval must be an object.");
|
|
2092
|
+
}
|
|
2093
|
+
config.eval = raw.eval;
|
|
2094
|
+
}
|
|
2095
|
+
if (raw.artifacts !== void 0) {
|
|
2096
|
+
if (typeof raw.artifacts !== "object" || Array.isArray(raw.artifacts)) {
|
|
2097
|
+
throw new Error("artifacts must be an object.");
|
|
2098
|
+
}
|
|
2099
|
+
const outputDir = asString(
|
|
2100
|
+
raw.artifacts.outputDir,
|
|
2101
|
+
"artifacts.outputDir"
|
|
2102
|
+
);
|
|
2103
|
+
config.artifacts = outputDir !== void 0 ? { outputDir } : {};
|
|
2104
|
+
}
|
|
2105
|
+
if (raw.baseline !== void 0) {
|
|
2106
|
+
const baseline = asString(raw.baseline, "baseline");
|
|
2107
|
+
if (baseline !== void 0) config.baseline = baseline;
|
|
2108
|
+
}
|
|
2109
|
+
if (raw.candidate !== void 0) {
|
|
2110
|
+
const candidate = asString(raw.candidate, "candidate");
|
|
2111
|
+
if (candidate !== void 0) config.candidate = candidate;
|
|
2112
|
+
}
|
|
2113
|
+
if (redactionProfile !== void 0) config.redactionProfile = redactionProfile;
|
|
2114
|
+
return config;
|
|
2007
2115
|
}
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
...finding.expected !== void 0 ? { expected: finding.expected } : {},
|
|
2015
|
-
...finding.actual !== void 0 ? { actual: finding.actual } : {},
|
|
2016
|
-
evidence: [...finding.evidence ?? []],
|
|
2017
|
-
...finding.category !== void 0 ? { category: finding.category } : {},
|
|
2018
|
-
...finding.confidence !== void 0 ? { confidence: finding.confidence } : {},
|
|
2019
|
-
...finding.detector !== void 0 ? { detector: finding.detector } : {},
|
|
2020
|
-
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
2021
|
-
};
|
|
2116
|
+
|
|
2117
|
+
// packages/core/src/suite/load.ts
|
|
2118
|
+
var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
2119
|
+
var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
2120
|
+
function diagnostic2(code, message) {
|
|
2121
|
+
return { code, message, severity: "error" };
|
|
2022
2122
|
}
|
|
2023
|
-
function
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
2031
|
-
).length,
|
|
2032
|
-
errors: diagnostics.filter((item) => item.severity === "error").length
|
|
2033
|
-
};
|
|
2123
|
+
async function fileExists(filePath) {
|
|
2124
|
+
try {
|
|
2125
|
+
await promises.access(filePath);
|
|
2126
|
+
return true;
|
|
2127
|
+
} catch {
|
|
2128
|
+
return false;
|
|
2129
|
+
}
|
|
2034
2130
|
}
|
|
2035
|
-
function
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2131
|
+
async function resolveSuiteConfigPath(options = {}) {
|
|
2132
|
+
const cwd = path15__default.default.resolve(options.cwd ?? process.cwd());
|
|
2133
|
+
if (options.configPath !== void 0 && options.configPath.trim() !== "") {
|
|
2134
|
+
return path15__default.default.resolve(cwd, options.configPath.trim());
|
|
2039
2135
|
}
|
|
2040
|
-
|
|
2136
|
+
for (const name of DEFAULT_SUITE_CONFIG_NAMES) {
|
|
2137
|
+
const candidate = path15__default.default.join(cwd, name);
|
|
2138
|
+
if (await fileExists(candidate)) return candidate;
|
|
2139
|
+
}
|
|
2140
|
+
throw new Error(
|
|
2141
|
+
`No suite config found. Create one with \`agent-inspect suite init\` or pass --config.`
|
|
2142
|
+
);
|
|
2041
2143
|
}
|
|
2042
|
-
function
|
|
2043
|
-
|
|
2044
|
-
|
|
2144
|
+
async function loadSuiteConfig(options = {}) {
|
|
2145
|
+
let configPath;
|
|
2146
|
+
try {
|
|
2147
|
+
configPath = await resolveSuiteConfigPath(options);
|
|
2148
|
+
} catch (error) {
|
|
2149
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2150
|
+
throw Object.assign(new Error(message), {
|
|
2151
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
2152
|
+
});
|
|
2153
|
+
}
|
|
2154
|
+
const extension = path15__default.default.extname(configPath);
|
|
2155
|
+
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
2156
|
+
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
2157
|
+
throw Object.assign(new Error(message), {
|
|
2158
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
2159
|
+
});
|
|
2160
|
+
}
|
|
2161
|
+
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
2162
|
+
const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
|
|
2163
|
+
throw Object.assign(new Error(message), {
|
|
2164
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
2165
|
+
});
|
|
2166
|
+
}
|
|
2167
|
+
try {
|
|
2168
|
+
let raw;
|
|
2169
|
+
if (extension === ".json") {
|
|
2170
|
+
raw = JSON.parse(await promises.readFile(configPath, "utf-8"));
|
|
2171
|
+
} else {
|
|
2172
|
+
const mod = await import(url.pathToFileURL(configPath).href);
|
|
2173
|
+
raw = "default" in mod ? mod.default : mod;
|
|
2174
|
+
}
|
|
2175
|
+
const config = normalizeSuiteConfig(raw);
|
|
2176
|
+
return {
|
|
2177
|
+
config,
|
|
2178
|
+
configPath,
|
|
2179
|
+
configDir: path15__default.default.dirname(configPath)
|
|
2180
|
+
};
|
|
2181
|
+
} catch (error) {
|
|
2182
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2183
|
+
throw Object.assign(new Error(message), {
|
|
2184
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
2185
|
+
});
|
|
2186
|
+
}
|
|
2187
|
+
}
|
|
2188
|
+
async function exists(filePath) {
|
|
2189
|
+
try {
|
|
2190
|
+
await promises.access(filePath);
|
|
2191
|
+
return true;
|
|
2192
|
+
} catch {
|
|
2193
|
+
return false;
|
|
2045
2194
|
}
|
|
2046
|
-
return name;
|
|
2047
2195
|
}
|
|
2048
|
-
function
|
|
2196
|
+
async function resolveSuiteCaseTrace(suiteCase, options) {
|
|
2197
|
+
if (suiteCase.trace !== void 0) {
|
|
2198
|
+
const tracePath = path15__default.default.resolve(options.configDir, suiteCase.trace);
|
|
2199
|
+
if (await exists(tracePath)) {
|
|
2200
|
+
return { caseId: suiteCase.id, tracePath, missing: false };
|
|
2201
|
+
}
|
|
2202
|
+
return {
|
|
2203
|
+
caseId: suiteCase.id,
|
|
2204
|
+
tracePath,
|
|
2205
|
+
missing: true,
|
|
2206
|
+
reason: `trace file not found: ${suiteCase.trace}`
|
|
2207
|
+
};
|
|
2208
|
+
}
|
|
2209
|
+
const runKey = suiteCase.runId ?? suiteCase.id;
|
|
2210
|
+
const directPath = getTraceFilePath(runKey, options.tracesDir);
|
|
2211
|
+
if (await exists(directPath)) {
|
|
2212
|
+
return { caseId: suiteCase.id, tracePath: directPath, runId: runKey, missing: false };
|
|
2213
|
+
}
|
|
2214
|
+
const nestedPath = path15__default.default.join(options.tracesDir, `${path15__default.default.basename(runKey)}.jsonl`);
|
|
2215
|
+
if (await exists(nestedPath)) {
|
|
2216
|
+
return { caseId: suiteCase.id, tracePath: nestedPath, runId: runKey, missing: false };
|
|
2217
|
+
}
|
|
2049
2218
|
return {
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
spanId: event.trace?.spanId,
|
|
2055
|
-
kind: event.kind,
|
|
2056
|
-
name: event.name,
|
|
2057
|
-
status: event.status,
|
|
2058
|
-
...path16 ? { path: path16 } : {}
|
|
2219
|
+
caseId: suiteCase.id,
|
|
2220
|
+
runId: runKey,
|
|
2221
|
+
missing: true,
|
|
2222
|
+
reason: `no trace found for run id "${runKey}" under ${options.tracesDir}`
|
|
2059
2223
|
};
|
|
2060
2224
|
}
|
|
2061
|
-
|
|
2062
|
-
|
|
2225
|
+
|
|
2226
|
+
// packages/core/src/checks/index.ts
|
|
2227
|
+
var SEVERITY_RANK = {
|
|
2228
|
+
error: 0,
|
|
2229
|
+
warning: 1,
|
|
2230
|
+
info: 2
|
|
2231
|
+
};
|
|
2232
|
+
var STATUS_RANK = {
|
|
2233
|
+
fail: 0,
|
|
2234
|
+
warning: 1,
|
|
2235
|
+
pass: 2
|
|
2236
|
+
};
|
|
2237
|
+
function compareStrings(a, b) {
|
|
2238
|
+
return (a ?? "").localeCompare(b ?? "");
|
|
2063
2239
|
}
|
|
2064
|
-
function
|
|
2240
|
+
function diagnostic3(code, message, ruleId) {
|
|
2065
2241
|
return {
|
|
2066
|
-
|
|
2067
|
-
severity: meta?.severity ?? "error",
|
|
2068
|
-
status: meta?.status ?? "fail",
|
|
2242
|
+
code,
|
|
2069
2243
|
message,
|
|
2070
|
-
|
|
2071
|
-
...
|
|
2072
|
-
evidence: [...evidence],
|
|
2073
|
-
...{},
|
|
2074
|
-
...{},
|
|
2075
|
-
...{},
|
|
2076
|
-
...{}
|
|
2244
|
+
severity: "error",
|
|
2245
|
+
...ruleId ? { ruleId } : {}
|
|
2077
2246
|
};
|
|
2078
2247
|
}
|
|
2079
|
-
function
|
|
2080
|
-
return
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2248
|
+
function emptySummary() {
|
|
2249
|
+
return {
|
|
2250
|
+
passed: 0,
|
|
2251
|
+
failed: 0,
|
|
2252
|
+
warnings: 0,
|
|
2253
|
+
errors: 0
|
|
2254
|
+
};
|
|
2084
2255
|
}
|
|
2085
|
-
function
|
|
2086
|
-
return
|
|
2256
|
+
function errorResult(input, diagnostics, selectedRun) {
|
|
2257
|
+
return {
|
|
2258
|
+
ok: false,
|
|
2259
|
+
status: "error",
|
|
2260
|
+
format: input.read.format,
|
|
2261
|
+
...selectedRun ? { runId: selectedRun.runId } : {},
|
|
2262
|
+
summary: {
|
|
2263
|
+
...emptySummary(),
|
|
2264
|
+
errors: diagnostics.filter((item) => item.severity === "error").length
|
|
2265
|
+
},
|
|
2266
|
+
findings: [],
|
|
2267
|
+
diagnostics: [...diagnostics]
|
|
2268
|
+
};
|
|
2087
2269
|
}
|
|
2088
|
-
function
|
|
2089
|
-
return
|
|
2270
|
+
function flattenNodes(nodes) {
|
|
2271
|
+
return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
|
|
2090
2272
|
}
|
|
2091
|
-
function
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
);
|
|
2273
|
+
function buildFacts(input, selectedRun) {
|
|
2274
|
+
const scopedRuns = selectedRun ? [selectedRun] : input.read.runs;
|
|
2275
|
+
const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
|
|
2276
|
+
const scopedEvents = selectedRun === void 0 ? input.read.events : input.read.events.filter((event) => scopedRunIds.has(event.runId));
|
|
2277
|
+
const nodes = flattenNodes(scopedRuns.flatMap((run) => run.children));
|
|
2278
|
+
const nodesByEventId = /* @__PURE__ */ new Map();
|
|
2279
|
+
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
2280
|
+
for (const node of nodes) {
|
|
2281
|
+
nodesByEventId.set(node.event.eventId, node);
|
|
2282
|
+
const parentId = node.event.parentId;
|
|
2283
|
+
if (parentId) {
|
|
2284
|
+
const children = childrenByParentId.get(parentId) ?? [];
|
|
2285
|
+
children.push(node);
|
|
2286
|
+
childrenByParentId.set(parentId, children);
|
|
2287
|
+
}
|
|
2288
|
+
}
|
|
2289
|
+
return {
|
|
2290
|
+
format: input.read.format,
|
|
2291
|
+
runs: Object.freeze([...input.read.runs]),
|
|
2292
|
+
events: Object.freeze([...scopedEvents]),
|
|
2293
|
+
readerWarnings: Object.freeze([...input.read.warnings]),
|
|
2294
|
+
unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
|
|
2295
|
+
sourceFiles: Object.freeze([...input.read.sourceFiles]),
|
|
2296
|
+
nodesByEventId,
|
|
2297
|
+
childrenByParentId,
|
|
2298
|
+
rootNodes: Object.freeze(scopedRuns.flatMap((run) => run.children))
|
|
2299
|
+
};
|
|
2300
|
+
}
|
|
2301
|
+
function resolveSelectedRun(input, runId) {
|
|
2302
|
+
if (input.selectedRun) {
|
|
2303
|
+
if (runId && input.selectedRun.runId !== runId) {
|
|
2304
|
+
return {
|
|
2305
|
+
diagnostics: [
|
|
2306
|
+
diagnostic3(
|
|
2307
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
2308
|
+
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
2309
|
+
)
|
|
2310
|
+
]
|
|
2311
|
+
};
|
|
2312
|
+
}
|
|
2313
|
+
return { run: input.selectedRun, diagnostics: [] };
|
|
2314
|
+
}
|
|
2315
|
+
if (runId) {
|
|
2316
|
+
const run = input.read.runs.find((candidate) => candidate.runId === runId);
|
|
2317
|
+
if (!run) {
|
|
2318
|
+
return {
|
|
2319
|
+
diagnostics: [
|
|
2320
|
+
diagnostic3("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
2321
|
+
]
|
|
2322
|
+
};
|
|
2323
|
+
}
|
|
2324
|
+
return { run, diagnostics: [] };
|
|
2325
|
+
}
|
|
2326
|
+
if (input.read.runs.length === 1) {
|
|
2327
|
+
return { run: input.read.runs[0], diagnostics: [] };
|
|
2328
|
+
}
|
|
2329
|
+
if (input.read.runs.length === 0) {
|
|
2330
|
+
return {
|
|
2331
|
+
diagnostics: [
|
|
2332
|
+
diagnostic3("AI_CHECK_RUN_SELECTION_REQUIRED", "No runs are available for checks.")
|
|
2333
|
+
]
|
|
2334
|
+
};
|
|
2335
|
+
}
|
|
2336
|
+
return {
|
|
2337
|
+
diagnostics: [
|
|
2338
|
+
diagnostic3(
|
|
2339
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
2340
|
+
"Multiple runs are available; select a run before executing checks."
|
|
2341
|
+
)
|
|
2342
|
+
]
|
|
2343
|
+
};
|
|
2344
|
+
}
|
|
2345
|
+
function selectRules(rules, selectedIds) {
|
|
2346
|
+
const diagnostics = [];
|
|
2347
|
+
const byId = /* @__PURE__ */ new Map();
|
|
2348
|
+
for (const rule of rules) {
|
|
2349
|
+
if (byId.has(rule.id)) {
|
|
2350
|
+
diagnostics.push(
|
|
2351
|
+
diagnostic3("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
2352
|
+
);
|
|
2353
|
+
continue;
|
|
2354
|
+
}
|
|
2355
|
+
byId.set(rule.id, rule);
|
|
2356
|
+
}
|
|
2357
|
+
if (selectedIds && selectedIds.length > 0) {
|
|
2358
|
+
const selected = new Set(selectedIds);
|
|
2359
|
+
for (const id of selected) {
|
|
2360
|
+
if (!byId.has(id)) {
|
|
2361
|
+
diagnostics.push(
|
|
2362
|
+
diagnostic3("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
2363
|
+
);
|
|
2364
|
+
}
|
|
2365
|
+
}
|
|
2366
|
+
return {
|
|
2367
|
+
rules: [...byId.values()].filter((rule) => selected.has(rule.id)).sort(compareRules),
|
|
2368
|
+
diagnostics
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
return { rules: [...byId.values()].sort(compareRules), diagnostics };
|
|
2372
|
+
}
|
|
2373
|
+
function compareRules(a, b) {
|
|
2374
|
+
return a.id.localeCompare(b.id);
|
|
2375
|
+
}
|
|
2376
|
+
function eventTimestamp(finding, eventById) {
|
|
2377
|
+
const eventId = finding.evidence[0]?.eventId;
|
|
2378
|
+
return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
|
|
2379
|
+
}
|
|
2380
|
+
function compareFindings(eventById) {
|
|
2381
|
+
return (a, b) => {
|
|
2382
|
+
if (SEVERITY_RANK[a.severity] !== SEVERITY_RANK[b.severity]) {
|
|
2383
|
+
return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity];
|
|
2384
|
+
}
|
|
2385
|
+
const byRule = a.ruleId.localeCompare(b.ruleId);
|
|
2386
|
+
if (byRule !== 0) return byRule;
|
|
2387
|
+
if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
|
|
2388
|
+
return STATUS_RANK[a.status] - STATUS_RANK[b.status];
|
|
2389
|
+
}
|
|
2390
|
+
const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
|
|
2391
|
+
if (byRun !== 0) return byRun;
|
|
2392
|
+
const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
|
|
2393
|
+
if (byTime !== 0) return byTime;
|
|
2394
|
+
const byEvent = compareStrings(a.evidence[0]?.eventId, b.evidence[0]?.eventId);
|
|
2395
|
+
if (byEvent !== 0) return byEvent;
|
|
2396
|
+
return compareStrings(a.evidence[0]?.path, b.evidence[0]?.path);
|
|
2397
|
+
};
|
|
2398
|
+
}
|
|
2399
|
+
function normalizeFinding(rule, finding) {
|
|
2400
|
+
return {
|
|
2401
|
+
ruleId: finding.ruleId || rule.id,
|
|
2402
|
+
severity: finding.severity ?? rule.defaultSeverity,
|
|
2403
|
+
status: finding.status,
|
|
2404
|
+
message: finding.message,
|
|
2405
|
+
...finding.expected !== void 0 ? { expected: finding.expected } : {},
|
|
2406
|
+
...finding.actual !== void 0 ? { actual: finding.actual } : {},
|
|
2407
|
+
evidence: [...finding.evidence ?? []],
|
|
2408
|
+
...finding.category !== void 0 ? { category: finding.category } : {},
|
|
2409
|
+
...finding.confidence !== void 0 ? { confidence: finding.confidence } : {},
|
|
2410
|
+
...finding.detector !== void 0 ? { detector: finding.detector } : {},
|
|
2411
|
+
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
2412
|
+
};
|
|
2413
|
+
}
|
|
2414
|
+
function summarize(findings, diagnostics) {
|
|
2415
|
+
return {
|
|
2416
|
+
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
2417
|
+
failed: findings.filter(
|
|
2418
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
2419
|
+
).length,
|
|
2420
|
+
warnings: findings.filter(
|
|
2421
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
2422
|
+
).length,
|
|
2423
|
+
errors: diagnostics.filter((item) => item.severity === "error").length
|
|
2424
|
+
};
|
|
2425
|
+
}
|
|
2426
|
+
function stringAttr(event, keys) {
|
|
2427
|
+
for (const key of keys) {
|
|
2428
|
+
const value = event.attributes?.[key];
|
|
2429
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2430
|
+
}
|
|
2431
|
+
return void 0;
|
|
2432
|
+
}
|
|
2433
|
+
function stripPrefix(name, prefixes) {
|
|
2434
|
+
for (const prefix of prefixes) {
|
|
2435
|
+
if (name.startsWith(prefix)) return name.slice(prefix.length);
|
|
2436
|
+
}
|
|
2437
|
+
return name;
|
|
2438
|
+
}
|
|
2439
|
+
function eventEvidence(event, path18) {
|
|
2440
|
+
return {
|
|
2441
|
+
runId: event.runId,
|
|
2442
|
+
eventId: event.eventId,
|
|
2443
|
+
parentId: event.parentId,
|
|
2444
|
+
traceId: event.trace?.traceId,
|
|
2445
|
+
spanId: event.trace?.spanId,
|
|
2446
|
+
kind: event.kind,
|
|
2447
|
+
name: event.name,
|
|
2448
|
+
status: event.status,
|
|
2449
|
+
...path18 ? { path: path18 } : {}
|
|
2450
|
+
};
|
|
2451
|
+
}
|
|
2452
|
+
function runEvidence(run) {
|
|
2453
|
+
return run ? [{ runId: run.runId, name: run.name, status: run.status }] : [];
|
|
2454
|
+
}
|
|
2455
|
+
function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
2456
|
+
return {
|
|
2457
|
+
ruleId,
|
|
2458
|
+
severity: meta?.severity ?? "error",
|
|
2459
|
+
status: meta?.status ?? "fail",
|
|
2460
|
+
message,
|
|
2461
|
+
...expected !== void 0 ? { expected } : {},
|
|
2462
|
+
...actual !== void 0 ? { actual } : {},
|
|
2463
|
+
evidence: [...evidence],
|
|
2464
|
+
...{},
|
|
2465
|
+
...{},
|
|
2466
|
+
...{},
|
|
2467
|
+
...{}
|
|
2468
|
+
};
|
|
2469
|
+
}
|
|
2470
|
+
function toolName(event) {
|
|
2471
|
+
return stringAttr(event, ["toolName", "tool"]) ?? stripPrefix(event.name, ["tool:", "function:", "mcp-tools:"]);
|
|
2472
|
+
}
|
|
2473
|
+
function llmModel(event) {
|
|
2474
|
+
return stringAttr(event, ["model", "modelId", "responseModelId", "modelName", "model_name"]) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
2475
|
+
}
|
|
2476
|
+
function llmProvider(event) {
|
|
2477
|
+
return stringAttr(event, ["provider", "providerName", "provider_name"]);
|
|
2478
|
+
}
|
|
2479
|
+
function llmFinishReason(event) {
|
|
2480
|
+
return stringAttr(event, ["finishReason", "rawFinishReason", "finish_reason"]);
|
|
2481
|
+
}
|
|
2482
|
+
function finishedEvents(context, kind) {
|
|
2483
|
+
return context.events.filter(
|
|
2484
|
+
(event) => (kind === void 0 || event.kind === kind) && event.status !== "running"
|
|
2485
|
+
);
|
|
2095
2486
|
}
|
|
2096
2487
|
function createRunStatusRule(options = {}) {
|
|
2097
2488
|
const expected = options.expected ?? "ok";
|
|
@@ -2979,7 +3370,7 @@ function findReaderByFormat(format, readers) {
|
|
|
2979
3370
|
}
|
|
2980
3371
|
async function jsonlFilesInDirectory(dirPath) {
|
|
2981
3372
|
const entries = await promises.readdir(dirPath, { withFileTypes: true });
|
|
2982
|
-
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) =>
|
|
3373
|
+
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => path15__default.default.join(dirPath, entry.name)).sort((a, b) => a.localeCompare(b));
|
|
2983
3374
|
}
|
|
2984
3375
|
async function resolveInput(input) {
|
|
2985
3376
|
const cached = resolvedInputCache.get(input);
|
|
@@ -4599,7 +4990,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
4599
4990
|
async function runSuite(options = {}) {
|
|
4600
4991
|
const startedAt = new Date(options.nowMs ?? Date.now()).toISOString();
|
|
4601
4992
|
const { config, configPath, configDir } = await loadSuiteConfig(options);
|
|
4602
|
-
const tracesDir =
|
|
4993
|
+
const tracesDir = path15__default.default.resolve(configDir, config.traces);
|
|
4603
4994
|
const cases = [];
|
|
4604
4995
|
const diagnostics = [];
|
|
4605
4996
|
for (const suiteCase of config.cases) {
|
|
@@ -4634,22 +5025,6 @@ async function runSuite(options = {}) {
|
|
|
4634
5025
|
};
|
|
4635
5026
|
}
|
|
4636
5027
|
|
|
4637
|
-
// packages/core/src/exporters/helpers.ts
|
|
4638
|
-
function sortKeysDeep(input) {
|
|
4639
|
-
if (input === null || typeof input !== "object") return input;
|
|
4640
|
-
if (Array.isArray(input)) return input.map(sortKeysDeep);
|
|
4641
|
-
const o = input;
|
|
4642
|
-
const out = {};
|
|
4643
|
-
for (const k of Object.keys(o).sort()) {
|
|
4644
|
-
out[k] = sortKeysDeep(o[k]);
|
|
4645
|
-
}
|
|
4646
|
-
return out;
|
|
4647
|
-
}
|
|
4648
|
-
function stableJson(value, pretty) {
|
|
4649
|
-
const sorted = sortKeysDeep(value);
|
|
4650
|
-
return JSON.stringify(sorted);
|
|
4651
|
-
}
|
|
4652
|
-
|
|
4653
5028
|
// packages/viewer/src/html.ts
|
|
4654
5029
|
var viewerIndexHtml = `<!DOCTYPE html>
|
|
4655
5030
|
<html lang="en">
|
|
@@ -4739,370 +5114,6 @@ var viewerIndexHtml = `<!DOCTYPE html>
|
|
|
4739
5114
|
</body>
|
|
4740
5115
|
</html>
|
|
4741
5116
|
`;
|
|
4742
|
-
|
|
4743
|
-
// packages/core/src/diff/comparable.ts
|
|
4744
|
-
function extractOutputPreview(meta) {
|
|
4745
|
-
if (meta === void 0) return void 0;
|
|
4746
|
-
if ("outputPreview" in meta) return meta.outputPreview;
|
|
4747
|
-
if ("resultPreview" in meta) return meta.resultPreview;
|
|
4748
|
-
return void 0;
|
|
4749
|
-
}
|
|
4750
|
-
function mapStepStatus(s) {
|
|
4751
|
-
if (s === void 0) return "running";
|
|
4752
|
-
return s;
|
|
4753
|
-
}
|
|
4754
|
-
function manualTraceEventsToComparableRun(events) {
|
|
4755
|
-
const started = events.find((e) => e.event === "run_started");
|
|
4756
|
-
if (!started || started.event !== "run_started") {
|
|
4757
|
-
throw new Error("Invalid trace: missing run_started");
|
|
4758
|
-
}
|
|
4759
|
-
const rs = started;
|
|
4760
|
-
const runId = rs.runId;
|
|
4761
|
-
const completedAll = events.filter((e) => e.event === "run_completed");
|
|
4762
|
-
const lastCompleted = completedAll[completedAll.length - 1];
|
|
4763
|
-
let runStatus;
|
|
4764
|
-
if (lastCompleted === void 0) runStatus = "running";
|
|
4765
|
-
else runStatus = lastCompleted.status;
|
|
4766
|
-
const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
4767
|
-
const steps = /* @__PURE__ */ new Map();
|
|
4768
|
-
let order = 0;
|
|
4769
|
-
for (const e of events) {
|
|
4770
|
-
if (e.event !== "step_started") continue;
|
|
4771
|
-
const s = e;
|
|
4772
|
-
const meta = s.metadata ? { ...s.metadata } : void 0;
|
|
4773
|
-
steps.set(s.stepId, {
|
|
4774
|
-
id: s.stepId,
|
|
4775
|
-
parentId: s.parentId,
|
|
4776
|
-
name: s.name,
|
|
4777
|
-
type: s.type,
|
|
4778
|
-
order: order++,
|
|
4779
|
-
timestamp: s.timestamp,
|
|
4780
|
-
metadata: meta
|
|
4781
|
-
});
|
|
4782
|
-
}
|
|
4783
|
-
for (const e of events) {
|
|
4784
|
-
if (e.event !== "step_completed") continue;
|
|
4785
|
-
const acc = steps.get(e.stepId);
|
|
4786
|
-
if (!acc) continue;
|
|
4787
|
-
acc.status = e.status;
|
|
4788
|
-
acc.durationMs = e.durationMs;
|
|
4789
|
-
if (e.error?.message) acc.errorMsg = e.error.message;
|
|
4790
|
-
const extra = e;
|
|
4791
|
-
if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
|
|
4792
|
-
acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
|
|
4793
|
-
}
|
|
4794
|
-
}
|
|
4795
|
-
const nodes = /* @__PURE__ */ new Map();
|
|
4796
|
-
for (const acc of steps.values()) {
|
|
4797
|
-
let meta = acc.metadata ? { ...acc.metadata } : void 0;
|
|
4798
|
-
if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
|
|
4799
|
-
meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
|
|
4800
|
-
}
|
|
4801
|
-
const outputPreview = extractOutputPreview(meta);
|
|
4802
|
-
if (meta !== void 0 && ("outputPreview" in meta || "resultPreview" in meta)) {
|
|
4803
|
-
delete meta.outputPreview;
|
|
4804
|
-
delete meta.resultPreview;
|
|
4805
|
-
}
|
|
4806
|
-
const sc = {
|
|
4807
|
-
id: acc.id,
|
|
4808
|
-
name: acc.name,
|
|
4809
|
-
type: acc.type,
|
|
4810
|
-
status: mapStepStatus(acc.status),
|
|
4811
|
-
durationMs: acc.durationMs,
|
|
4812
|
-
error: acc.errorMsg,
|
|
4813
|
-
metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
|
|
4814
|
-
outputPreview,
|
|
4815
|
-
children: []
|
|
4816
|
-
};
|
|
4817
|
-
nodes.set(acc.id, sc);
|
|
4818
|
-
}
|
|
4819
|
-
const roots = [];
|
|
4820
|
-
const sortByOrder = (a, b) => {
|
|
4821
|
-
const oa = steps.get(a.id)?.order ?? 0;
|
|
4822
|
-
const ob = steps.get(b.id)?.order ?? 0;
|
|
4823
|
-
return oa - ob;
|
|
4824
|
-
};
|
|
4825
|
-
for (const acc of steps.values()) {
|
|
4826
|
-
const node = nodes.get(acc.id);
|
|
4827
|
-
if (acc.parentId !== void 0 && nodes.has(acc.parentId)) {
|
|
4828
|
-
nodes.get(acc.parentId).children.push(node);
|
|
4829
|
-
} else {
|
|
4830
|
-
roots.push(node);
|
|
4831
|
-
}
|
|
4832
|
-
}
|
|
4833
|
-
roots.sort(sortByOrder);
|
|
4834
|
-
for (const n of nodes.values()) {
|
|
4835
|
-
n.children.sort(sortByOrder);
|
|
4836
|
-
}
|
|
4837
|
-
return {
|
|
4838
|
-
runId,
|
|
4839
|
-
name: rs.name,
|
|
4840
|
-
status: runStatus,
|
|
4841
|
-
durationMs,
|
|
4842
|
-
steps: roots
|
|
4843
|
-
};
|
|
4844
|
-
}
|
|
4845
|
-
|
|
4846
|
-
// packages/core/src/diff/engine.ts
|
|
4847
|
-
var DEFAULT_THRESHOLD_MS = 0;
|
|
4848
|
-
function pathSeg(step, index) {
|
|
4849
|
-
return { index, name: step.name, stepId: step.id };
|
|
4850
|
-
}
|
|
4851
|
-
function buildPath(segments) {
|
|
4852
|
-
return { path: [...segments] };
|
|
4853
|
-
}
|
|
4854
|
-
function pairSteps(left, right) {
|
|
4855
|
-
const usedRight = /* @__PURE__ */ new Set();
|
|
4856
|
-
const pairs = [];
|
|
4857
|
-
for (let i = 0; i < left.length; i++) {
|
|
4858
|
-
const L = left[i];
|
|
4859
|
-
let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
|
|
4860
|
-
if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
|
|
4861
|
-
const cand = right[i];
|
|
4862
|
-
if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
|
|
4863
|
-
R = cand;
|
|
4864
|
-
}
|
|
4865
|
-
}
|
|
4866
|
-
if (R === void 0) {
|
|
4867
|
-
R = right.find(
|
|
4868
|
-
(r) => !usedRight.has(r.id) && r.name === L.name && (r.type ?? "") === (L.type ?? "")
|
|
4869
|
-
);
|
|
4870
|
-
}
|
|
4871
|
-
if (R !== void 0) {
|
|
4872
|
-
usedRight.add(R.id);
|
|
4873
|
-
pairs.push([L, R]);
|
|
4874
|
-
} else {
|
|
4875
|
-
pairs.push([L, void 0]);
|
|
4876
|
-
}
|
|
4877
|
-
}
|
|
4878
|
-
for (const R of right) {
|
|
4879
|
-
if (!usedRight.has(R.id)) {
|
|
4880
|
-
pairs.push([void 0, R]);
|
|
4881
|
-
}
|
|
4882
|
-
}
|
|
4883
|
-
return pairs;
|
|
4884
|
-
}
|
|
4885
|
-
function compareLeafSteps(L, R, segments, opts, out) {
|
|
4886
|
-
const path16 = buildPath(segments);
|
|
4887
|
-
if (L.name !== R.name) {
|
|
4888
|
-
out.push({
|
|
4889
|
-
kind: "structure",
|
|
4890
|
-
severity: "warning",
|
|
4891
|
-
message: "Step name differs",
|
|
4892
|
-
path: path16,
|
|
4893
|
-
left: L.name,
|
|
4894
|
-
right: R.name
|
|
4895
|
-
});
|
|
4896
|
-
}
|
|
4897
|
-
if ((L.type ?? "") !== (R.type ?? "")) {
|
|
4898
|
-
out.push({
|
|
4899
|
-
kind: "step-type",
|
|
4900
|
-
severity: "warning",
|
|
4901
|
-
message: "Step type differs",
|
|
4902
|
-
path: path16,
|
|
4903
|
-
left: L.type,
|
|
4904
|
-
right: R.type
|
|
4905
|
-
});
|
|
4906
|
-
}
|
|
4907
|
-
if ((L.status ?? "") !== (R.status ?? "")) {
|
|
4908
|
-
out.push({
|
|
4909
|
-
kind: "step-status",
|
|
4910
|
-
severity: "warning",
|
|
4911
|
-
message: "Step status differs",
|
|
4912
|
-
path: path16,
|
|
4913
|
-
left: L.status,
|
|
4914
|
-
right: R.status
|
|
4915
|
-
});
|
|
4916
|
-
}
|
|
4917
|
-
const le = L.error ?? "";
|
|
4918
|
-
const re = R.error ?? "";
|
|
4919
|
-
if (le !== re) {
|
|
4920
|
-
out.push({
|
|
4921
|
-
kind: "error",
|
|
4922
|
-
severity: "error",
|
|
4923
|
-
message: "Step error message differs",
|
|
4924
|
-
path: path16,
|
|
4925
|
-
left: le || void 0,
|
|
4926
|
-
right: re || void 0
|
|
4927
|
-
});
|
|
4928
|
-
}
|
|
4929
|
-
if (!opts.ignoreDuration) {
|
|
4930
|
-
const ld = L.durationMs;
|
|
4931
|
-
const rd = R.durationMs;
|
|
4932
|
-
const th = opts.durationThresholdMs;
|
|
4933
|
-
let differs = false;
|
|
4934
|
-
if (ld === void 0 && rd === void 0) differs = false;
|
|
4935
|
-
else if (ld === void 0 || rd === void 0) differs = true;
|
|
4936
|
-
else differs = Math.abs(ld - rd) > th;
|
|
4937
|
-
if (differs) {
|
|
4938
|
-
out.push({
|
|
4939
|
-
kind: "duration",
|
|
4940
|
-
severity: "info",
|
|
4941
|
-
message: "Step duration differs",
|
|
4942
|
-
path: path16,
|
|
4943
|
-
left: ld,
|
|
4944
|
-
right: rd
|
|
4945
|
-
});
|
|
4946
|
-
}
|
|
4947
|
-
}
|
|
4948
|
-
const lm = stableJson(L.metadata ?? {});
|
|
4949
|
-
const rm2 = stableJson(R.metadata ?? {});
|
|
4950
|
-
if (lm !== rm2) {
|
|
4951
|
-
out.push({
|
|
4952
|
-
kind: "metadata",
|
|
4953
|
-
severity: "info",
|
|
4954
|
-
message: "Step metadata differs",
|
|
4955
|
-
path: path16,
|
|
4956
|
-
left: L.metadata,
|
|
4957
|
-
right: R.metadata
|
|
4958
|
-
});
|
|
4959
|
-
}
|
|
4960
|
-
const lo = stableJson(L.outputPreview ?? null);
|
|
4961
|
-
const ro = stableJson(R.outputPreview ?? null);
|
|
4962
|
-
if (lo !== ro) {
|
|
4963
|
-
out.push({
|
|
4964
|
-
kind: "output",
|
|
4965
|
-
severity: "info",
|
|
4966
|
-
message: "Output preview differs",
|
|
4967
|
-
path: path16,
|
|
4968
|
-
left: L.outputPreview,
|
|
4969
|
-
right: R.outputPreview
|
|
4970
|
-
});
|
|
4971
|
-
}
|
|
4972
|
-
}
|
|
4973
|
-
function compareRecursive(L, R, segments, opts, out) {
|
|
4974
|
-
compareLeafSteps(L, R, segments, opts, out);
|
|
4975
|
-
const pairs = pairSteps(L.children, R.children);
|
|
4976
|
-
let ci = 0;
|
|
4977
|
-
for (const [lch, rch] of pairs) {
|
|
4978
|
-
if (lch !== void 0 && rch !== void 0) {
|
|
4979
|
-
compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
|
|
4980
|
-
} else if (lch !== void 0) {
|
|
4981
|
-
out.push({
|
|
4982
|
-
kind: "step-removed",
|
|
4983
|
-
severity: "warning",
|
|
4984
|
-
message: `Step only in left run: ${lch.name}`,
|
|
4985
|
-
path: buildPath([...segments, pathSeg(lch, ci)]),
|
|
4986
|
-
left: lch.id,
|
|
4987
|
-
right: void 0
|
|
4988
|
-
});
|
|
4989
|
-
} else if (rch !== void 0) {
|
|
4990
|
-
out.push({
|
|
4991
|
-
kind: "step-added",
|
|
4992
|
-
severity: "warning",
|
|
4993
|
-
message: `Step only in right run: ${rch.name}`,
|
|
4994
|
-
path: buildPath([...segments, pathSeg(rch, ci)]),
|
|
4995
|
-
left: void 0,
|
|
4996
|
-
right: rch.id
|
|
4997
|
-
});
|
|
4998
|
-
}
|
|
4999
|
-
ci += 1;
|
|
5000
|
-
}
|
|
5001
|
-
}
|
|
5002
|
-
function mergeDiffDefaults(options) {
|
|
5003
|
-
return {
|
|
5004
|
-
ignoreDuration: false,
|
|
5005
|
-
durationThresholdMs: DEFAULT_THRESHOLD_MS,
|
|
5006
|
-
focus: "all",
|
|
5007
|
-
check: "all"
|
|
5008
|
-
};
|
|
5009
|
-
}
|
|
5010
|
-
function kindMatchesFilter(kind, merged) {
|
|
5011
|
-
return true;
|
|
5012
|
-
}
|
|
5013
|
-
function diffRuns(left, right, options) {
|
|
5014
|
-
const merged = mergeDiffDefaults();
|
|
5015
|
-
const opts = {
|
|
5016
|
-
ignoreDuration: merged.ignoreDuration,
|
|
5017
|
-
durationThresholdMs: merged.durationThresholdMs
|
|
5018
|
-
};
|
|
5019
|
-
const raw = [];
|
|
5020
|
-
if ((left.status ?? "") !== (right.status ?? "")) {
|
|
5021
|
-
raw.push({
|
|
5022
|
-
kind: "run-status",
|
|
5023
|
-
severity: "warning",
|
|
5024
|
-
message: "Run completion status differs",
|
|
5025
|
-
left: left.status,
|
|
5026
|
-
right: right.status
|
|
5027
|
-
});
|
|
5028
|
-
}
|
|
5029
|
-
{
|
|
5030
|
-
const ld = left.durationMs;
|
|
5031
|
-
const rd = right.durationMs;
|
|
5032
|
-
const th = merged.durationThresholdMs;
|
|
5033
|
-
let differs = false;
|
|
5034
|
-
if (ld === void 0 && rd === void 0) differs = false;
|
|
5035
|
-
else if (ld === void 0 || rd === void 0) differs = true;
|
|
5036
|
-
else differs = Math.abs(ld - rd) > th;
|
|
5037
|
-
if (differs) {
|
|
5038
|
-
raw.push({
|
|
5039
|
-
kind: "duration",
|
|
5040
|
-
severity: "info",
|
|
5041
|
-
message: "Run duration differs",
|
|
5042
|
-
left: ld,
|
|
5043
|
-
right: rd
|
|
5044
|
-
});
|
|
5045
|
-
}
|
|
5046
|
-
}
|
|
5047
|
-
const pairs = pairSteps(left.steps, right.steps);
|
|
5048
|
-
let idx = 0;
|
|
5049
|
-
for (const [ls, rs] of pairs) {
|
|
5050
|
-
if (ls !== void 0 && rs !== void 0) {
|
|
5051
|
-
compareRecursive(ls, rs, [pathSeg(ls, idx)], opts, raw);
|
|
5052
|
-
idx += 1;
|
|
5053
|
-
} else if (ls !== void 0) {
|
|
5054
|
-
raw.push({
|
|
5055
|
-
kind: "step-removed",
|
|
5056
|
-
severity: "warning",
|
|
5057
|
-
message: `Step only in left run: ${ls.name}`,
|
|
5058
|
-
path: buildPath([pathSeg(ls, idx)]),
|
|
5059
|
-
left: ls.id,
|
|
5060
|
-
right: void 0
|
|
5061
|
-
});
|
|
5062
|
-
idx += 1;
|
|
5063
|
-
} else if (rs !== void 0) {
|
|
5064
|
-
raw.push({
|
|
5065
|
-
kind: "step-added",
|
|
5066
|
-
severity: "warning",
|
|
5067
|
-
message: `Step only in right run: ${rs.name}`,
|
|
5068
|
-
path: buildPath([pathSeg(rs, idx)]),
|
|
5069
|
-
left: void 0,
|
|
5070
|
-
right: rs.id
|
|
5071
|
-
});
|
|
5072
|
-
idx += 1;
|
|
5073
|
-
}
|
|
5074
|
-
}
|
|
5075
|
-
const differences = raw.filter((d) => kindMatchesFilter(d.kind));
|
|
5076
|
-
let errors = 0;
|
|
5077
|
-
let warnings = 0;
|
|
5078
|
-
let info = 0;
|
|
5079
|
-
for (const d of differences) {
|
|
5080
|
-
if (d.severity === "error") errors += 1;
|
|
5081
|
-
else if (d.severity === "warning") warnings += 1;
|
|
5082
|
-
else info += 1;
|
|
5083
|
-
}
|
|
5084
|
-
const firstVisible = differences[0];
|
|
5085
|
-
const firstDivergence = firstVisible !== void 0 ? {
|
|
5086
|
-
kind: "first-divergence",
|
|
5087
|
-
severity: firstVisible.severity,
|
|
5088
|
-
message: `First divergence: ${firstVisible.message}`,
|
|
5089
|
-
path: firstVisible.path,
|
|
5090
|
-
left: firstVisible.left,
|
|
5091
|
-
right: firstVisible.right
|
|
5092
|
-
} : void 0;
|
|
5093
|
-
const summary = {
|
|
5094
|
-
leftRunId: left.runId,
|
|
5095
|
-
rightRunId: right.runId,
|
|
5096
|
-
totalDifferences: differences.length,
|
|
5097
|
-
errors,
|
|
5098
|
-
warnings,
|
|
5099
|
-
info,
|
|
5100
|
-
firstDivergence
|
|
5101
|
-
};
|
|
5102
|
-
return { summary, differences };
|
|
5103
|
-
}
|
|
5104
|
-
|
|
5105
|
-
// packages/viewer/src/suite-data.ts
|
|
5106
5117
|
async function enrichCase(suiteCase, baselineTracePath) {
|
|
5107
5118
|
const base = {
|
|
5108
5119
|
id: suiteCase.id,
|
|
@@ -5162,7 +5173,7 @@ async function loadSuiteViewerData(options) {
|
|
|
5162
5173
|
for (const suiteCase of result.cases) {
|
|
5163
5174
|
cases.push(await enrichCase(suiteCase, baselinePath));
|
|
5164
5175
|
}
|
|
5165
|
-
const artifactsDir =
|
|
5176
|
+
const artifactsDir = path15__default.default.join(path15__default.default.dirname(result.configPath), ".agent-inspect/suite-runs");
|
|
5166
5177
|
return {
|
|
5167
5178
|
suiteName: result.suiteName,
|
|
5168
5179
|
configPath: result.configPath,
|
|
@@ -5323,19 +5334,19 @@ function parseWorkspaceManifest(json) {
|
|
|
5323
5334
|
}
|
|
5324
5335
|
var INDEX_DIR_NAME = "index";
|
|
5325
5336
|
function resolveWorkspaceLocation(cwd = process.cwd()) {
|
|
5326
|
-
const projectRoot =
|
|
5327
|
-
const workspaceDir =
|
|
5337
|
+
const projectRoot = path15__default.default.resolve(cwd);
|
|
5338
|
+
const workspaceDir = path15__default.default.join(projectRoot, WORKSPACE_DIR_NAME);
|
|
5328
5339
|
return {
|
|
5329
5340
|
projectRoot,
|
|
5330
5341
|
workspaceDir,
|
|
5331
|
-
manifestPath:
|
|
5342
|
+
manifestPath: path15__default.default.join(workspaceDir, WORKSPACE_MANIFEST_FILENAME)
|
|
5332
5343
|
};
|
|
5333
5344
|
}
|
|
5334
5345
|
function resolveInsideWorkspace(workspaceDir, relative) {
|
|
5335
|
-
const base =
|
|
5336
|
-
const resolved =
|
|
5337
|
-
const rel =
|
|
5338
|
-
if (rel === "" || rel === "." || !rel.startsWith("..") && !
|
|
5346
|
+
const base = path15__default.default.resolve(workspaceDir);
|
|
5347
|
+
const resolved = path15__default.default.resolve(base, relative);
|
|
5348
|
+
const rel = path15__default.default.relative(base, resolved);
|
|
5349
|
+
if (rel === "" || rel === "." || !rel.startsWith("..") && !path15__default.default.isAbsolute(rel)) {
|
|
5339
5350
|
return resolved;
|
|
5340
5351
|
}
|
|
5341
5352
|
throw new Error(
|
|
@@ -5479,7 +5490,7 @@ async function doctorWorkspace(location) {
|
|
|
5479
5490
|
const abs = resolveInsideWorkspace(location.workspaceDir, rel);
|
|
5480
5491
|
for (const file of await listJsonl(abs)) {
|
|
5481
5492
|
try {
|
|
5482
|
-
const s = await promises.stat(
|
|
5493
|
+
const s = await promises.stat(path15__default.default.join(abs, file));
|
|
5483
5494
|
newestTraceMtime = Math.max(newestTraceMtime, s.mtimeMs);
|
|
5484
5495
|
} catch {
|
|
5485
5496
|
checks.push({ id: "trace-readability", status: "warn", message: `cannot stat ${rel}/${file}` });
|
|
@@ -5509,7 +5520,7 @@ async function doctorWorkspace(location) {
|
|
|
5509
5520
|
|
|
5510
5521
|
// packages/viewer/src/workspace-data.ts
|
|
5511
5522
|
async function loadWorkspaceViewerData(options) {
|
|
5512
|
-
const cwd =
|
|
5523
|
+
const cwd = path15__default.default.resolve(options.cwd ?? process.cwd());
|
|
5513
5524
|
const location = resolveWorkspaceLocation(cwd);
|
|
5514
5525
|
const manifestRead = await readWorkspaceManifestFile(location);
|
|
5515
5526
|
if (!manifestRead.ok || manifestRead.manifest === void 0) {
|
|
@@ -5522,7 +5533,7 @@ async function loadWorkspaceViewerData(options) {
|
|
|
5522
5533
|
const runs = [];
|
|
5523
5534
|
for (const rel of manifestRead.manifest.traceDirs) {
|
|
5524
5535
|
const traceDir = resolveTraceDir({
|
|
5525
|
-
dir:
|
|
5536
|
+
dir: path15__default.default.join(location.workspaceDir, rel)
|
|
5526
5537
|
});
|
|
5527
5538
|
const td = new TraceDirectory({ dir: traceDir });
|
|
5528
5539
|
const files = await td.list();
|
|
@@ -5536,7 +5547,7 @@ async function loadWorkspaceViewerData(options) {
|
|
|
5536
5547
|
runId: meta.runId,
|
|
5537
5548
|
...meta.name !== void 0 ? { name: meta.name } : {},
|
|
5538
5549
|
status: meta.status,
|
|
5539
|
-
file:
|
|
5550
|
+
file: path15__default.default.basename(meta.filePath)
|
|
5540
5551
|
});
|
|
5541
5552
|
}
|
|
5542
5553
|
}
|
|
@@ -5547,8 +5558,8 @@ async function loadWorkspaceViewerData(options) {
|
|
|
5547
5558
|
doctor,
|
|
5548
5559
|
runs,
|
|
5549
5560
|
bundleDirs: [
|
|
5550
|
-
|
|
5551
|
-
|
|
5561
|
+
path15__default.default.join(location.workspaceDir, manifestRead.manifest.bundlesDir),
|
|
5562
|
+
path15__default.default.join(location.workspaceDir, manifestRead.manifest.artifactsDir)
|
|
5552
5563
|
]
|
|
5553
5564
|
};
|
|
5554
5565
|
}
|
|
@@ -5611,7 +5622,7 @@ function createViewerServer(options = {}) {
|
|
|
5611
5622
|
ok: true,
|
|
5612
5623
|
readOnly: true,
|
|
5613
5624
|
mode,
|
|
5614
|
-
traceDir:
|
|
5625
|
+
traceDir: path15__default.default.resolve(traceDir)
|
|
5615
5626
|
});
|
|
5616
5627
|
}
|
|
5617
5628
|
if (pathname === "/api/suite" && mode === "suite") {
|
|
@@ -5640,7 +5651,7 @@ function createViewerServer(options = {}) {
|
|
|
5640
5651
|
runId: meta.runId,
|
|
5641
5652
|
name: meta.name,
|
|
5642
5653
|
status: meta.status,
|
|
5643
|
-
file:
|
|
5654
|
+
file: path15__default.default.basename(meta.filePath),
|
|
5644
5655
|
startedAt: meta.startedAt,
|
|
5645
5656
|
durationMs: meta.durationMs
|
|
5646
5657
|
}))
|
|
@@ -5757,7 +5768,7 @@ function startViewerServer(options = {}) {
|
|
|
5757
5768
|
resolve({
|
|
5758
5769
|
host,
|
|
5759
5770
|
port: resolvedPort,
|
|
5760
|
-
traceDir:
|
|
5771
|
+
traceDir: path15__default.default.resolve(traceDir),
|
|
5761
5772
|
url: `http://${host}:${resolvedPort}/${modeQuery}`,
|
|
5762
5773
|
mode
|
|
5763
5774
|
});
|