@agent-inspect/viewer 5.1.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1672 -32
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +404 -2
- package/dist/index.d.ts +404 -2
- package/dist/index.mjs +1671 -33
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var http = require('http');
|
|
4
|
-
var
|
|
4
|
+
var path13 = require('path');
|
|
5
5
|
var async_hooks = require('async_hooks');
|
|
6
6
|
require('crypto');
|
|
7
7
|
var promises = require('fs/promises');
|
|
8
8
|
var os = require('os');
|
|
9
9
|
require('nanoid');
|
|
10
10
|
require('chalk');
|
|
11
|
-
require('fs');
|
|
11
|
+
var fs = require('fs');
|
|
12
12
|
require('readline');
|
|
13
|
-
require('url');
|
|
13
|
+
var url = require('url');
|
|
14
14
|
|
|
15
15
|
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
16
16
|
|
|
17
|
-
var
|
|
17
|
+
var path13__default = /*#__PURE__*/_interopDefault(path13);
|
|
18
18
|
var os__default = /*#__PURE__*/_interopDefault(os);
|
|
19
19
|
|
|
20
20
|
// packages/viewer/src/server.ts
|
|
@@ -525,7 +525,7 @@ function persistedInspectEventsToTraceEvents(events, options) {
|
|
|
525
525
|
}
|
|
526
526
|
var DEFAULT_TRACE_DIR_NAME = ".agent-inspect";
|
|
527
527
|
var RUNS_DIR_NAME = "runs";
|
|
528
|
-
var FALLBACK_TRACE_DIR =
|
|
528
|
+
var FALLBACK_TRACE_DIR = path13__default.default.join(
|
|
529
529
|
os__default.default.tmpdir(),
|
|
530
530
|
"agent-inspect",
|
|
531
531
|
RUNS_DIR_NAME
|
|
@@ -540,11 +540,20 @@ function getDefaultTraceDir() {
|
|
|
540
540
|
if (typeof home !== "string" || home.trim() === "") {
|
|
541
541
|
return FALLBACK_TRACE_DIR;
|
|
542
542
|
}
|
|
543
|
-
return
|
|
543
|
+
return path13__default.default.join(home, DEFAULT_TRACE_DIR_NAME, RUNS_DIR_NAME);
|
|
544
544
|
} catch {
|
|
545
545
|
return FALLBACK_TRACE_DIR;
|
|
546
546
|
}
|
|
547
547
|
}
|
|
548
|
+
function getTraceFilePath(runId, traceDir) {
|
|
549
|
+
const baseDir = traceDir ?? getDefaultTraceDir();
|
|
550
|
+
let safeId = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "run_unknown";
|
|
551
|
+
safeId = path13__default.default.basename(safeId);
|
|
552
|
+
if (safeId === "" || safeId === "." || safeId === "..") {
|
|
553
|
+
safeId = "run_unknown";
|
|
554
|
+
}
|
|
555
|
+
return path13__default.default.join(baseDir, `${safeId}.jsonl`);
|
|
556
|
+
}
|
|
548
557
|
function formatError(error) {
|
|
549
558
|
if (error instanceof Error) {
|
|
550
559
|
const out = { message: error.message };
|
|
@@ -733,6 +742,78 @@ async function readTraceEventsFromFile(filePath) {
|
|
|
733
742
|
|
|
734
743
|
// packages/core/src/context.ts
|
|
735
744
|
new async_hooks.AsyncLocalStorage();
|
|
745
|
+
|
|
746
|
+
// packages/core/src/outcomes/types.ts
|
|
747
|
+
var OUTCOME_ATTRIBUTE_STATUS_KEY = "outcomeStatus";
|
|
748
|
+
var OUTCOME_ATTRIBUTE_EXPECTATION_KEY = "expectation";
|
|
749
|
+
var OUTCOME_ATTRIBUTE_METHOD_KEY = "method";
|
|
750
|
+
var OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY = "observedAt";
|
|
751
|
+
var OUTCOME_LEGACY_EVENT = "outcome_observed";
|
|
752
|
+
|
|
753
|
+
// packages/core/src/outcomes/extract.ts
|
|
754
|
+
function isOutcomeStatus(value) {
|
|
755
|
+
return value === "passed" || value === "failed" || value === "unknown" || value === "skipped";
|
|
756
|
+
}
|
|
757
|
+
function parseMethod(value) {
|
|
758
|
+
if (typeof value !== "string" || value.trim() === "") return void 0;
|
|
759
|
+
return value;
|
|
760
|
+
}
|
|
761
|
+
function fromOutcomeObservedEvent(event) {
|
|
762
|
+
return {
|
|
763
|
+
outcomeId: event.outcomeId,
|
|
764
|
+
runId: event.runId,
|
|
765
|
+
...event.parentId !== void 0 ? { parentId: event.parentId } : {},
|
|
766
|
+
name: event.name,
|
|
767
|
+
expectation: event.expectation,
|
|
768
|
+
status: event.status,
|
|
769
|
+
...event.method !== void 0 ? { method: event.method } : {},
|
|
770
|
+
...event.actual !== void 0 ? { actual: event.actual } : {},
|
|
771
|
+
...event.evidence !== void 0 ? { evidence: event.evidence } : {},
|
|
772
|
+
observedAt: event.observedAt
|
|
773
|
+
};
|
|
774
|
+
}
|
|
775
|
+
function fromPersistedOutcome(event) {
|
|
776
|
+
if (event.kind !== "OUTCOME") return void 0;
|
|
777
|
+
const attrs = event.attributes ?? {};
|
|
778
|
+
const statusRaw = attrs[OUTCOME_ATTRIBUTE_STATUS_KEY];
|
|
779
|
+
if (!isOutcomeStatus(statusRaw)) return void 0;
|
|
780
|
+
const expectation = typeof attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] === "string" ? attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] : event.name;
|
|
781
|
+
const observedAtRaw = attrs[OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY];
|
|
782
|
+
const observedAt = typeof observedAtRaw === "string" ? Date.parse(observedAtRaw) : typeof observedAtRaw === "number" && Number.isFinite(observedAtRaw) ? observedAtRaw : Date.parse(event.timestamp);
|
|
783
|
+
return {
|
|
784
|
+
outcomeId: event.eventId,
|
|
785
|
+
runId: event.runId,
|
|
786
|
+
...event.parentId !== void 0 ? { parentId: event.parentId } : {},
|
|
787
|
+
name: event.name,
|
|
788
|
+
expectation,
|
|
789
|
+
status: statusRaw,
|
|
790
|
+
...parseMethod(attrs[OUTCOME_ATTRIBUTE_METHOD_KEY]) !== void 0 ? { method: parseMethod(attrs[OUTCOME_ATTRIBUTE_METHOD_KEY]) } : {},
|
|
791
|
+
...event.outputSummary !== void 0 ? { actual: event.outputSummary } : {},
|
|
792
|
+
...attrs.evidence !== void 0 ? { evidence: attrs.evidence } : {},
|
|
793
|
+
observedAt: Number.isFinite(observedAt) ? observedAt : Date.parse(event.timestamp)
|
|
794
|
+
};
|
|
795
|
+
}
|
|
796
|
+
function extractOutcomesFromTraceEvents(events) {
|
|
797
|
+
const out = [];
|
|
798
|
+
for (const event of events) {
|
|
799
|
+
if (event.event === OUTCOME_LEGACY_EVENT) {
|
|
800
|
+
out.push(fromOutcomeObservedEvent(event));
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
return out.sort((a, b) => a.observedAt - b.observedAt || a.name.localeCompare(b.name));
|
|
804
|
+
}
|
|
805
|
+
function extractOutcomesFromPersistedEvents(events) {
|
|
806
|
+
const out = [];
|
|
807
|
+
for (const event of events) {
|
|
808
|
+
const outcome = fromPersistedOutcome(event);
|
|
809
|
+
if (outcome) out.push(outcome);
|
|
810
|
+
}
|
|
811
|
+
return out.sort((a, b) => a.observedAt - b.observedAt || a.name.localeCompare(b.name));
|
|
812
|
+
}
|
|
813
|
+
function outcomesMatchingStatus(outcomes, statuses) {
|
|
814
|
+
const set = new Set(statuses);
|
|
815
|
+
return outcomes.filter((outcome) => set.has(outcome.status));
|
|
816
|
+
}
|
|
736
817
|
function resolveTraceDir(options = {}) {
|
|
737
818
|
if (typeof options.dir === "string" && options.dir.trim() !== "") {
|
|
738
819
|
return options.dir.trim();
|
|
@@ -749,7 +830,7 @@ var TraceDirectory = class {
|
|
|
749
830
|
this.#dir = resolveTraceDir(options);
|
|
750
831
|
}
|
|
751
832
|
getPath(filename) {
|
|
752
|
-
return filename ?
|
|
833
|
+
return filename ? path13__default.default.join(this.#dir, filename) : this.#dir;
|
|
753
834
|
}
|
|
754
835
|
async list() {
|
|
755
836
|
try {
|
|
@@ -776,7 +857,7 @@ function parseIsoToMs2(value) {
|
|
|
776
857
|
}
|
|
777
858
|
async function extractMetadata(filePath, _quickScan) {
|
|
778
859
|
const stats = await promises.stat(filePath);
|
|
779
|
-
let runIdFromFile =
|
|
860
|
+
let runIdFromFile = path13__default.default.basename(filePath);
|
|
780
861
|
if (runIdFromFile.endsWith(".jsonl")) {
|
|
781
862
|
runIdFromFile = runIdFromFile.slice(0, -".jsonl".length);
|
|
782
863
|
}
|
|
@@ -1386,12 +1467,12 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
1386
1467
|
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
1387
1468
|
);
|
|
1388
1469
|
const ordered = [...runs].sort(compareRuns);
|
|
1389
|
-
const
|
|
1470
|
+
const path16 = [];
|
|
1390
1471
|
const visited = /* @__PURE__ */ new Set();
|
|
1391
1472
|
const pushRun = (run, confidence, source) => {
|
|
1392
1473
|
if (visited.has(run.runId)) return;
|
|
1393
1474
|
visited.add(run.runId);
|
|
1394
|
-
|
|
1475
|
+
path16.push({
|
|
1395
1476
|
runId: run.runId,
|
|
1396
1477
|
name: run.name,
|
|
1397
1478
|
startedAt: run.startedAt,
|
|
@@ -1416,7 +1497,7 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
1416
1497
|
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
1417
1498
|
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
1418
1499
|
}
|
|
1419
|
-
return
|
|
1500
|
+
return path16;
|
|
1420
1501
|
}
|
|
1421
1502
|
function metaRunIdMatches(run, token, runById) {
|
|
1422
1503
|
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
@@ -1493,6 +1574,264 @@ function buildSessionIndex(inputRuns, options = {}) {
|
|
|
1493
1574
|
};
|
|
1494
1575
|
}
|
|
1495
1576
|
|
|
1577
|
+
// packages/core/src/suite/types.ts
|
|
1578
|
+
var DEFAULT_SUITE_CONFIG_NAMES = [
|
|
1579
|
+
"agent-inspect.suite.json",
|
|
1580
|
+
"agent-inspect.suite.js",
|
|
1581
|
+
"agent-inspect.suite.mjs",
|
|
1582
|
+
"agent-inspect.suite.cjs"
|
|
1583
|
+
];
|
|
1584
|
+
function diagnostic(code, message, severity = "error", caseId) {
|
|
1585
|
+
return { code, message, severity, ...{} };
|
|
1586
|
+
}
|
|
1587
|
+
function asString(value, label) {
|
|
1588
|
+
if (value === void 0) return void 0;
|
|
1589
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
1590
|
+
throw new Error(`${label} must be a non-empty string.`);
|
|
1591
|
+
}
|
|
1592
|
+
return value.trim();
|
|
1593
|
+
}
|
|
1594
|
+
function asStringArray(value, label) {
|
|
1595
|
+
if (value === void 0) return void 0;
|
|
1596
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
1597
|
+
throw new Error(`${label} must be an array of strings.`);
|
|
1598
|
+
}
|
|
1599
|
+
return value;
|
|
1600
|
+
}
|
|
1601
|
+
function asPositiveNumber(value, label) {
|
|
1602
|
+
if (value === void 0) return void 0;
|
|
1603
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
1604
|
+
throw new Error(`${label} must be a non-negative number.`);
|
|
1605
|
+
}
|
|
1606
|
+
return value;
|
|
1607
|
+
}
|
|
1608
|
+
function validateCaseConfig(value, index) {
|
|
1609
|
+
const diagnostics = [];
|
|
1610
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
1611
|
+
diagnostics.push(
|
|
1612
|
+
diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
|
|
1613
|
+
);
|
|
1614
|
+
return { diagnostics };
|
|
1615
|
+
}
|
|
1616
|
+
const raw = value;
|
|
1617
|
+
try {
|
|
1618
|
+
const id = asString(raw.id, `cases[${index}].id`);
|
|
1619
|
+
if (id === void 0) {
|
|
1620
|
+
diagnostics.push(
|
|
1621
|
+
diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
|
|
1622
|
+
);
|
|
1623
|
+
return { diagnostics };
|
|
1624
|
+
}
|
|
1625
|
+
const trace = asString(raw.trace, `cases[${index}].trace`);
|
|
1626
|
+
const runId = asString(raw.runId, `cases[${index}].runId`);
|
|
1627
|
+
const input = asString(raw.input, `cases[${index}].input`);
|
|
1628
|
+
return {
|
|
1629
|
+
caseConfig: {
|
|
1630
|
+
id,
|
|
1631
|
+
...trace !== void 0 ? { trace } : {},
|
|
1632
|
+
...runId !== void 0 ? { runId } : {},
|
|
1633
|
+
...input !== void 0 ? { input } : {},
|
|
1634
|
+
...asStringArray(raw.requireTools, `cases[${index}].requireTools`) !== void 0 ? { requireTools: asStringArray(raw.requireTools, `cases[${index}].requireTools`) } : {},
|
|
1635
|
+
...asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) !== void 0 ? { forbidTools: asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) } : {},
|
|
1636
|
+
...asPositiveNumber(raw.maxDurationMs, `cases[${index}].maxDurationMs`) !== void 0 ? {
|
|
1637
|
+
maxDurationMs: asPositiveNumber(
|
|
1638
|
+
raw.maxDurationMs,
|
|
1639
|
+
`cases[${index}].maxDurationMs`
|
|
1640
|
+
)
|
|
1641
|
+
} : {},
|
|
1642
|
+
...asStringArray(
|
|
1643
|
+
raw.expectedObservations,
|
|
1644
|
+
`cases[${index}].expectedObservations`
|
|
1645
|
+
) !== void 0 ? {
|
|
1646
|
+
expectedObservations: asStringArray(
|
|
1647
|
+
raw.expectedObservations,
|
|
1648
|
+
`cases[${index}].expectedObservations`
|
|
1649
|
+
)
|
|
1650
|
+
} : {}
|
|
1651
|
+
},
|
|
1652
|
+
diagnostics
|
|
1653
|
+
};
|
|
1654
|
+
} catch (error) {
|
|
1655
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1656
|
+
diagnostics.push(diagnostic("AI_SUITE_CONFIG_INVALID", message));
|
|
1657
|
+
return { diagnostics };
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
function normalizeSuiteConfig(value) {
|
|
1661
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
1662
|
+
throw new Error("Suite config must export an object.");
|
|
1663
|
+
}
|
|
1664
|
+
const raw = value;
|
|
1665
|
+
const name = asString(raw.name, "name");
|
|
1666
|
+
const traces = asString(raw.traces, "traces");
|
|
1667
|
+
if (name === void 0) throw new Error("name is required.");
|
|
1668
|
+
if (traces === void 0) throw new Error("traces is required.");
|
|
1669
|
+
if (!Array.isArray(raw.cases) || raw.cases.length === 0) {
|
|
1670
|
+
throw new Error("cases must be a non-empty array.");
|
|
1671
|
+
}
|
|
1672
|
+
const cases = [];
|
|
1673
|
+
for (let index = 0; index < raw.cases.length; index += 1) {
|
|
1674
|
+
const { caseConfig, diagnostics } = validateCaseConfig(raw.cases[index], index);
|
|
1675
|
+
if (diagnostics.length > 0) {
|
|
1676
|
+
throw new Error(diagnostics.map((item) => item.message).join("; "));
|
|
1677
|
+
}
|
|
1678
|
+
if (caseConfig !== void 0) cases.push(caseConfig);
|
|
1679
|
+
}
|
|
1680
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1681
|
+
for (const suiteCase of cases) {
|
|
1682
|
+
if (ids.has(suiteCase.id)) {
|
|
1683
|
+
throw new Error(`Duplicate case id "${suiteCase.id}".`);
|
|
1684
|
+
}
|
|
1685
|
+
ids.add(suiteCase.id);
|
|
1686
|
+
}
|
|
1687
|
+
const redactionProfile = raw.redactionProfile === "local" || raw.redactionProfile === "share" || raw.redactionProfile === "strict" ? raw.redactionProfile : void 0;
|
|
1688
|
+
if (raw.redactionProfile !== void 0 && redactionProfile === void 0) {
|
|
1689
|
+
throw new Error('redactionProfile must be "local", "share", or "strict".');
|
|
1690
|
+
}
|
|
1691
|
+
const config = { name, traces, cases };
|
|
1692
|
+
if (raw.checks !== void 0) {
|
|
1693
|
+
if (typeof raw.checks !== "object" || Array.isArray(raw.checks)) {
|
|
1694
|
+
throw new Error("checks must be an object.");
|
|
1695
|
+
}
|
|
1696
|
+
config.checks = raw.checks;
|
|
1697
|
+
}
|
|
1698
|
+
if (raw.eval !== void 0) {
|
|
1699
|
+
if (typeof raw.eval !== "object" || Array.isArray(raw.eval)) {
|
|
1700
|
+
throw new Error("eval must be an object.");
|
|
1701
|
+
}
|
|
1702
|
+
config.eval = raw.eval;
|
|
1703
|
+
}
|
|
1704
|
+
if (raw.artifacts !== void 0) {
|
|
1705
|
+
if (typeof raw.artifacts !== "object" || Array.isArray(raw.artifacts)) {
|
|
1706
|
+
throw new Error("artifacts must be an object.");
|
|
1707
|
+
}
|
|
1708
|
+
const outputDir = asString(
|
|
1709
|
+
raw.artifacts.outputDir,
|
|
1710
|
+
"artifacts.outputDir"
|
|
1711
|
+
);
|
|
1712
|
+
config.artifacts = outputDir !== void 0 ? { outputDir } : {};
|
|
1713
|
+
}
|
|
1714
|
+
if (raw.baseline !== void 0) {
|
|
1715
|
+
const baseline = asString(raw.baseline, "baseline");
|
|
1716
|
+
if (baseline !== void 0) config.baseline = baseline;
|
|
1717
|
+
}
|
|
1718
|
+
if (raw.candidate !== void 0) {
|
|
1719
|
+
const candidate = asString(raw.candidate, "candidate");
|
|
1720
|
+
if (candidate !== void 0) config.candidate = candidate;
|
|
1721
|
+
}
|
|
1722
|
+
if (redactionProfile !== void 0) config.redactionProfile = redactionProfile;
|
|
1723
|
+
return config;
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1726
|
+
// packages/core/src/suite/load.ts
|
|
1727
|
+
var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
1728
|
+
var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
1729
|
+
function diagnostic2(code, message) {
|
|
1730
|
+
return { code, message, severity: "error" };
|
|
1731
|
+
}
|
|
1732
|
+
async function fileExists(filePath) {
|
|
1733
|
+
try {
|
|
1734
|
+
await promises.access(filePath);
|
|
1735
|
+
return true;
|
|
1736
|
+
} catch {
|
|
1737
|
+
return false;
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
async function resolveSuiteConfigPath(options = {}) {
|
|
1741
|
+
const cwd = path13__default.default.resolve(options.cwd ?? process.cwd());
|
|
1742
|
+
if (options.configPath !== void 0 && options.configPath.trim() !== "") {
|
|
1743
|
+
return path13__default.default.resolve(cwd, options.configPath.trim());
|
|
1744
|
+
}
|
|
1745
|
+
for (const name of DEFAULT_SUITE_CONFIG_NAMES) {
|
|
1746
|
+
const candidate = path13__default.default.join(cwd, name);
|
|
1747
|
+
if (await fileExists(candidate)) return candidate;
|
|
1748
|
+
}
|
|
1749
|
+
throw new Error(
|
|
1750
|
+
`No suite config found. Create one with \`agent-inspect suite init\` or pass --config.`
|
|
1751
|
+
);
|
|
1752
|
+
}
|
|
1753
|
+
async function loadSuiteConfig(options = {}) {
|
|
1754
|
+
let configPath;
|
|
1755
|
+
try {
|
|
1756
|
+
configPath = await resolveSuiteConfigPath(options);
|
|
1757
|
+
} catch (error) {
|
|
1758
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1759
|
+
throw Object.assign(new Error(message), {
|
|
1760
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
1761
|
+
});
|
|
1762
|
+
}
|
|
1763
|
+
const extension = path13__default.default.extname(configPath);
|
|
1764
|
+
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
1765
|
+
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
1766
|
+
throw Object.assign(new Error(message), {
|
|
1767
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
1768
|
+
});
|
|
1769
|
+
}
|
|
1770
|
+
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
1771
|
+
const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
|
|
1772
|
+
throw Object.assign(new Error(message), {
|
|
1773
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
1774
|
+
});
|
|
1775
|
+
}
|
|
1776
|
+
try {
|
|
1777
|
+
let raw;
|
|
1778
|
+
if (extension === ".json") {
|
|
1779
|
+
raw = JSON.parse(await promises.readFile(configPath, "utf-8"));
|
|
1780
|
+
} else {
|
|
1781
|
+
const mod = await import(url.pathToFileURL(configPath).href);
|
|
1782
|
+
raw = "default" in mod ? mod.default : mod;
|
|
1783
|
+
}
|
|
1784
|
+
const config = normalizeSuiteConfig(raw);
|
|
1785
|
+
return {
|
|
1786
|
+
config,
|
|
1787
|
+
configPath,
|
|
1788
|
+
configDir: path13__default.default.dirname(configPath)
|
|
1789
|
+
};
|
|
1790
|
+
} catch (error) {
|
|
1791
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1792
|
+
throw Object.assign(new Error(message), {
|
|
1793
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
1794
|
+
});
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
async function exists(filePath) {
|
|
1798
|
+
try {
|
|
1799
|
+
await promises.access(filePath);
|
|
1800
|
+
return true;
|
|
1801
|
+
} catch {
|
|
1802
|
+
return false;
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
async function resolveSuiteCaseTrace(suiteCase, options) {
|
|
1806
|
+
if (suiteCase.trace !== void 0) {
|
|
1807
|
+
const tracePath = path13__default.default.resolve(options.configDir, suiteCase.trace);
|
|
1808
|
+
if (await exists(tracePath)) {
|
|
1809
|
+
return { caseId: suiteCase.id, tracePath, missing: false };
|
|
1810
|
+
}
|
|
1811
|
+
return {
|
|
1812
|
+
caseId: suiteCase.id,
|
|
1813
|
+
tracePath,
|
|
1814
|
+
missing: true,
|
|
1815
|
+
reason: `trace file not found: ${suiteCase.trace}`
|
|
1816
|
+
};
|
|
1817
|
+
}
|
|
1818
|
+
const runKey = suiteCase.runId ?? suiteCase.id;
|
|
1819
|
+
const directPath = getTraceFilePath(runKey, options.tracesDir);
|
|
1820
|
+
if (await exists(directPath)) {
|
|
1821
|
+
return { caseId: suiteCase.id, tracePath: directPath, runId: runKey, missing: false };
|
|
1822
|
+
}
|
|
1823
|
+
const nestedPath = path13__default.default.join(options.tracesDir, `${path13__default.default.basename(runKey)}.jsonl`);
|
|
1824
|
+
if (await exists(nestedPath)) {
|
|
1825
|
+
return { caseId: suiteCase.id, tracePath: nestedPath, runId: runKey, missing: false };
|
|
1826
|
+
}
|
|
1827
|
+
return {
|
|
1828
|
+
caseId: suiteCase.id,
|
|
1829
|
+
runId: runKey,
|
|
1830
|
+
missing: true,
|
|
1831
|
+
reason: `no trace found for run id "${runKey}" under ${options.tracesDir}`
|
|
1832
|
+
};
|
|
1833
|
+
}
|
|
1834
|
+
|
|
1496
1835
|
// packages/core/src/checks/index.ts
|
|
1497
1836
|
var SEVERITY_RANK = {
|
|
1498
1837
|
error: 0,
|
|
@@ -1507,7 +1846,7 @@ var STATUS_RANK = {
|
|
|
1507
1846
|
function compareStrings(a, b) {
|
|
1508
1847
|
return (a ?? "").localeCompare(b ?? "");
|
|
1509
1848
|
}
|
|
1510
|
-
function
|
|
1849
|
+
function diagnostic3(code, message, ruleId) {
|
|
1511
1850
|
return {
|
|
1512
1851
|
code,
|
|
1513
1852
|
message,
|
|
@@ -1573,7 +1912,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
1573
1912
|
if (runId && input.selectedRun.runId !== runId) {
|
|
1574
1913
|
return {
|
|
1575
1914
|
diagnostics: [
|
|
1576
|
-
|
|
1915
|
+
diagnostic3(
|
|
1577
1916
|
"AI_CHECK_INVALID_ARGUMENTS",
|
|
1578
1917
|
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
1579
1918
|
)
|
|
@@ -1587,7 +1926,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
1587
1926
|
if (!run) {
|
|
1588
1927
|
return {
|
|
1589
1928
|
diagnostics: [
|
|
1590
|
-
|
|
1929
|
+
diagnostic3("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
1591
1930
|
]
|
|
1592
1931
|
};
|
|
1593
1932
|
}
|
|
@@ -1599,13 +1938,13 @@ function resolveSelectedRun(input, runId) {
|
|
|
1599
1938
|
if (input.read.runs.length === 0) {
|
|
1600
1939
|
return {
|
|
1601
1940
|
diagnostics: [
|
|
1602
|
-
|
|
1941
|
+
diagnostic3("AI_CHECK_RUN_SELECTION_REQUIRED", "No runs are available for checks.")
|
|
1603
1942
|
]
|
|
1604
1943
|
};
|
|
1605
1944
|
}
|
|
1606
1945
|
return {
|
|
1607
1946
|
diagnostics: [
|
|
1608
|
-
|
|
1947
|
+
diagnostic3(
|
|
1609
1948
|
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
1610
1949
|
"Multiple runs are available; select a run before executing checks."
|
|
1611
1950
|
)
|
|
@@ -1618,7 +1957,7 @@ function selectRules(rules, selectedIds) {
|
|
|
1618
1957
|
for (const rule of rules) {
|
|
1619
1958
|
if (byId.has(rule.id)) {
|
|
1620
1959
|
diagnostics.push(
|
|
1621
|
-
|
|
1960
|
+
diagnostic3("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
1622
1961
|
);
|
|
1623
1962
|
continue;
|
|
1624
1963
|
}
|
|
@@ -1629,7 +1968,7 @@ function selectRules(rules, selectedIds) {
|
|
|
1629
1968
|
for (const id of selected) {
|
|
1630
1969
|
if (!byId.has(id)) {
|
|
1631
1970
|
diagnostics.push(
|
|
1632
|
-
|
|
1971
|
+
diagnostic3("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
1633
1972
|
);
|
|
1634
1973
|
}
|
|
1635
1974
|
}
|
|
@@ -1689,7 +2028,20 @@ function summarize(findings, diagnostics) {
|
|
|
1689
2028
|
errors: diagnostics.filter((item) => item.severity === "error").length
|
|
1690
2029
|
};
|
|
1691
2030
|
}
|
|
1692
|
-
function
|
|
2031
|
+
function stringAttr(event, keys) {
|
|
2032
|
+
for (const key of keys) {
|
|
2033
|
+
const value = event.attributes?.[key];
|
|
2034
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2035
|
+
}
|
|
2036
|
+
return void 0;
|
|
2037
|
+
}
|
|
2038
|
+
function stripPrefix(name, prefixes) {
|
|
2039
|
+
for (const prefix of prefixes) {
|
|
2040
|
+
if (name.startsWith(prefix)) return name.slice(prefix.length);
|
|
2041
|
+
}
|
|
2042
|
+
return name;
|
|
2043
|
+
}
|
|
2044
|
+
function eventEvidence(event, path16) {
|
|
1693
2045
|
return {
|
|
1694
2046
|
runId: event.runId,
|
|
1695
2047
|
eventId: event.eventId,
|
|
@@ -1699,7 +2051,7 @@ function eventEvidence(event, path12) {
|
|
|
1699
2051
|
kind: event.kind,
|
|
1700
2052
|
name: event.name,
|
|
1701
2053
|
status: event.status,
|
|
1702
|
-
...{}
|
|
2054
|
+
...path16 ? { path: path16 } : {}
|
|
1703
2055
|
};
|
|
1704
2056
|
}
|
|
1705
2057
|
function runEvidence(run) {
|
|
@@ -1716,6 +2068,23 @@ function failFinding(ruleId, message, evidence, expected, actual) {
|
|
|
1716
2068
|
evidence: [...evidence]
|
|
1717
2069
|
};
|
|
1718
2070
|
}
|
|
2071
|
+
function toolName(event) {
|
|
2072
|
+
return stringAttr(event, ["toolName", "tool"]) ?? stripPrefix(event.name, ["tool:", "function:", "mcp-tools:"]);
|
|
2073
|
+
}
|
|
2074
|
+
function llmModel(event) {
|
|
2075
|
+
return stringAttr(event, ["model", "modelId", "responseModelId", "modelName", "model_name"]) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
2076
|
+
}
|
|
2077
|
+
function llmProvider(event) {
|
|
2078
|
+
return stringAttr(event, ["provider", "providerName", "provider_name"]);
|
|
2079
|
+
}
|
|
2080
|
+
function llmFinishReason(event) {
|
|
2081
|
+
return stringAttr(event, ["finishReason", "rawFinishReason", "finish_reason"]);
|
|
2082
|
+
}
|
|
2083
|
+
function finishedEvents(context, kind) {
|
|
2084
|
+
return context.events.filter(
|
|
2085
|
+
(event) => (kind === void 0 || event.kind === kind) && event.status !== "running"
|
|
2086
|
+
);
|
|
2087
|
+
}
|
|
1719
2088
|
function createRunStatusRule(options = {}) {
|
|
1720
2089
|
const expected = options.expected ?? "ok";
|
|
1721
2090
|
const allowIncomplete = options.allowIncomplete === true;
|
|
@@ -1755,6 +2124,180 @@ function createRunStatusRule(options = {}) {
|
|
|
1755
2124
|
}
|
|
1756
2125
|
};
|
|
1757
2126
|
}
|
|
2127
|
+
function createRunDurationRule(options) {
|
|
2128
|
+
return {
|
|
2129
|
+
id: "run.duration",
|
|
2130
|
+
category: "run",
|
|
2131
|
+
defaultSeverity: "error",
|
|
2132
|
+
evaluate(context) {
|
|
2133
|
+
const actual = context.selectedRun?.durationMs;
|
|
2134
|
+
if (actual === void 0 || actual <= options.maxDurationMs) return [];
|
|
2135
|
+
return [
|
|
2136
|
+
failFinding(
|
|
2137
|
+
"run.duration",
|
|
2138
|
+
`Run duration ${actual}ms exceeded ${options.maxDurationMs}ms.`,
|
|
2139
|
+
runEvidence(context.selectedRun),
|
|
2140
|
+
{ maxDurationMs: options.maxDurationMs },
|
|
2141
|
+
actual
|
|
2142
|
+
)
|
|
2143
|
+
];
|
|
2144
|
+
}
|
|
2145
|
+
};
|
|
2146
|
+
}
|
|
2147
|
+
function createToolUsageRule(options) {
|
|
2148
|
+
return {
|
|
2149
|
+
id: "tool.usage",
|
|
2150
|
+
category: "tool",
|
|
2151
|
+
defaultSeverity: "error",
|
|
2152
|
+
evaluate(context) {
|
|
2153
|
+
const tools = finishedEvents(context, "TOOL");
|
|
2154
|
+
const names = tools.map(toolName);
|
|
2155
|
+
const nameSet = new Set(names);
|
|
2156
|
+
const findings = [];
|
|
2157
|
+
for (const required of options.required ?? []) {
|
|
2158
|
+
if (!nameSet.has(required)) {
|
|
2159
|
+
findings.push(
|
|
2160
|
+
failFinding("tool.usage", `Required tool ${required} did not appear.`, runEvidence(context.selectedRun), required, names)
|
|
2161
|
+
);
|
|
2162
|
+
}
|
|
2163
|
+
}
|
|
2164
|
+
const forbidden = new Set(options.forbidden ?? []);
|
|
2165
|
+
const allowed = options.allowed ? new Set(options.allowed) : void 0;
|
|
2166
|
+
for (const event of tools) {
|
|
2167
|
+
const name = toolName(event);
|
|
2168
|
+
if (forbidden.has(name)) {
|
|
2169
|
+
findings.push(
|
|
2170
|
+
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence(event)], "tool absent", name)
|
|
2171
|
+
);
|
|
2172
|
+
}
|
|
2173
|
+
if (allowed && !allowed.has(name)) {
|
|
2174
|
+
findings.push(
|
|
2175
|
+
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence(event)], [...allowed].sort(), name)
|
|
2176
|
+
);
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
if (options.minCount !== void 0 && tools.length < options.minCount) {
|
|
2180
|
+
findings.push(
|
|
2181
|
+
failFinding("tool.usage", `Tool count ${tools.length} was below minimum ${options.minCount}.`, runEvidence(context.selectedRun), { minCount: options.minCount }, tools.length)
|
|
2182
|
+
);
|
|
2183
|
+
}
|
|
2184
|
+
if (options.maxCount !== void 0 && tools.length > options.maxCount) {
|
|
2185
|
+
findings.push(
|
|
2186
|
+
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) => eventEvidence(event)), { maxCount: options.maxCount }, tools.length)
|
|
2187
|
+
);
|
|
2188
|
+
}
|
|
2189
|
+
return findings;
|
|
2190
|
+
}
|
|
2191
|
+
};
|
|
2192
|
+
}
|
|
2193
|
+
function createLlmUsageRule(options) {
|
|
2194
|
+
return {
|
|
2195
|
+
id: "llm.usage",
|
|
2196
|
+
category: "llm",
|
|
2197
|
+
defaultSeverity: "error",
|
|
2198
|
+
evaluate(context) {
|
|
2199
|
+
const llms = finishedEvents(context, "LLM");
|
|
2200
|
+
const findings = [];
|
|
2201
|
+
const allowedModels = options.allowedModels ? new Set(options.allowedModels) : void 0;
|
|
2202
|
+
const allowedProviders = options.allowedProviders ? new Set(options.allowedProviders) : void 0;
|
|
2203
|
+
const finishReasons = options.finishReasons ? new Set(options.finishReasons) : void 0;
|
|
2204
|
+
if (options.maxCalls !== void 0 && llms.length > options.maxCalls) {
|
|
2205
|
+
findings.push(
|
|
2206
|
+
failFinding(
|
|
2207
|
+
"llm.usage",
|
|
2208
|
+
`LLM call count ${llms.length} exceeded ${options.maxCalls}.`,
|
|
2209
|
+
llms.map((event) => eventEvidence(event)),
|
|
2210
|
+
{ maxCalls: options.maxCalls },
|
|
2211
|
+
llms.length
|
|
2212
|
+
)
|
|
2213
|
+
);
|
|
2214
|
+
}
|
|
2215
|
+
for (const event of llms) {
|
|
2216
|
+
const model = llmModel(event);
|
|
2217
|
+
const provider = llmProvider(event);
|
|
2218
|
+
const finishReason = llmFinishReason(event);
|
|
2219
|
+
if (allowedModels && (!model || !allowedModels.has(model))) {
|
|
2220
|
+
findings.push(
|
|
2221
|
+
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.model")], [...allowedModels].sort(), model ?? "unknown")
|
|
2222
|
+
);
|
|
2223
|
+
}
|
|
2224
|
+
if (allowedProviders && (!provider || !allowedProviders.has(provider))) {
|
|
2225
|
+
findings.push(
|
|
2226
|
+
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.provider")], [...allowedProviders].sort(), provider ?? "unknown")
|
|
2227
|
+
);
|
|
2228
|
+
}
|
|
2229
|
+
if (finishReasons && (!finishReason || !finishReasons.has(finishReason))) {
|
|
2230
|
+
findings.push(
|
|
2231
|
+
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.finishReason")], [...finishReasons].sort(), finishReason ?? "unknown")
|
|
2232
|
+
);
|
|
2233
|
+
}
|
|
2234
|
+
}
|
|
2235
|
+
const tokenTotals = llms.reduce(
|
|
2236
|
+
(totals, event) => ({
|
|
2237
|
+
input: totals.input + (event.tokenUsage?.input ?? 0),
|
|
2238
|
+
output: totals.output + (event.tokenUsage?.output ?? 0),
|
|
2239
|
+
total: totals.total + (event.tokenUsage?.total ?? 0),
|
|
2240
|
+
cached: totals.cached + (event.tokenUsage?.cached ?? 0)
|
|
2241
|
+
}),
|
|
2242
|
+
{ input: 0, output: 0, total: 0, cached: 0 }
|
|
2243
|
+
);
|
|
2244
|
+
const tokenLimits = [
|
|
2245
|
+
["input", options.maxInputTokens],
|
|
2246
|
+
["output", options.maxOutputTokens],
|
|
2247
|
+
["total", options.maxTotalTokens],
|
|
2248
|
+
["cached", options.maxCachedTokens]
|
|
2249
|
+
];
|
|
2250
|
+
for (const [key, limit] of tokenLimits) {
|
|
2251
|
+
if (limit !== void 0 && tokenTotals[key] > limit) {
|
|
2252
|
+
findings.push(
|
|
2253
|
+
failFinding(
|
|
2254
|
+
"llm.usage",
|
|
2255
|
+
`LLM ${key} token count ${tokenTotals[key]} exceeded ${limit}.`,
|
|
2256
|
+
llms.map((event) => eventEvidence(event, `tokenUsage.${key}`)),
|
|
2257
|
+
{ [`max${key[0].toUpperCase()}${key.slice(1)}Tokens`]: limit },
|
|
2258
|
+
tokenTotals[key]
|
|
2259
|
+
)
|
|
2260
|
+
);
|
|
2261
|
+
}
|
|
2262
|
+
}
|
|
2263
|
+
return findings;
|
|
2264
|
+
}
|
|
2265
|
+
};
|
|
2266
|
+
}
|
|
2267
|
+
function createObservedOutcomeRule(options = {}) {
|
|
2268
|
+
const failOn = options.failOn ?? ["failed"];
|
|
2269
|
+
return {
|
|
2270
|
+
id: "outcome.status",
|
|
2271
|
+
category: "run",
|
|
2272
|
+
defaultSeverity: "error",
|
|
2273
|
+
evaluate(context) {
|
|
2274
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
2275
|
+
const matching = outcomesMatchingStatus(outcomes, failOn);
|
|
2276
|
+
if (matching.length === 0) return [];
|
|
2277
|
+
return [
|
|
2278
|
+
failFinding(
|
|
2279
|
+
"outcome.status",
|
|
2280
|
+
`Observed outcome count ${matching.length} matched [${failOn.join(", ")}].`,
|
|
2281
|
+
matching.map((outcome) => ({
|
|
2282
|
+
runId: outcome.runId,
|
|
2283
|
+
eventId: outcome.outcomeId,
|
|
2284
|
+
...outcome.parentId !== void 0 ? { parentId: outcome.parentId } : {},
|
|
2285
|
+
kind: "OUTCOME",
|
|
2286
|
+
name: outcome.name,
|
|
2287
|
+
status: outcome.status,
|
|
2288
|
+
path: `outcome.${outcome.name}`
|
|
2289
|
+
})),
|
|
2290
|
+
{ failOn },
|
|
2291
|
+
matching.map((outcome) => ({
|
|
2292
|
+
name: outcome.name,
|
|
2293
|
+
status: outcome.status,
|
|
2294
|
+
expectation: outcome.expectation
|
|
2295
|
+
}))
|
|
2296
|
+
)
|
|
2297
|
+
];
|
|
2298
|
+
}
|
|
2299
|
+
};
|
|
2300
|
+
}
|
|
1758
2301
|
function runTraceChecks(input, options = {}) {
|
|
1759
2302
|
const selected = resolveSelectedRun(input, options.runId);
|
|
1760
2303
|
if (selected.diagnostics.length > 0) {
|
|
@@ -1778,7 +2321,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
1778
2321
|
} catch (error) {
|
|
1779
2322
|
const message = error instanceof Error ? error.message : String(error);
|
|
1780
2323
|
diagnostics.push(
|
|
1781
|
-
|
|
2324
|
+
diagnostic3("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
|
|
1782
2325
|
);
|
|
1783
2326
|
}
|
|
1784
2327
|
}
|
|
@@ -2417,7 +2960,7 @@ function findReaderByFormat(format, readers) {
|
|
|
2417
2960
|
}
|
|
2418
2961
|
async function jsonlFilesInDirectory(dirPath) {
|
|
2419
2962
|
const entries = await promises.readdir(dirPath, { withFileTypes: true });
|
|
2420
|
-
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) =>
|
|
2963
|
+
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => path13__default.default.join(dirPath, entry.name)).sort((a, b) => a.localeCompare(b));
|
|
2421
2964
|
}
|
|
2422
2965
|
async function resolveInput(input) {
|
|
2423
2966
|
const cached = resolvedInputCache.get(input);
|
|
@@ -3868,6 +4411,225 @@ function openTrace(input, options = {}) {
|
|
|
3868
4411
|
return readTrace(input, options);
|
|
3869
4412
|
}
|
|
3870
4413
|
|
|
4414
|
+
// packages/core/src/suite/run.ts
|
|
4415
|
+
function diagnostic4(code, message, severity = "error", caseId) {
|
|
4416
|
+
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
4417
|
+
}
|
|
4418
|
+
function buildCaseRules(suiteCase, config) {
|
|
4419
|
+
const rules = [];
|
|
4420
|
+
const select = new Set(config.checks?.select ?? []);
|
|
4421
|
+
if (select.has("run.status")) {
|
|
4422
|
+
rules.push(createRunStatusRule());
|
|
4423
|
+
}
|
|
4424
|
+
const requiredTools = [
|
|
4425
|
+
...config.checks?.tool?.required ?? [],
|
|
4426
|
+
...suiteCase.requireTools ?? []
|
|
4427
|
+
];
|
|
4428
|
+
const forbiddenTools = [
|
|
4429
|
+
...config.checks?.tool?.forbidden ?? [],
|
|
4430
|
+
...suiteCase.forbidTools ?? []
|
|
4431
|
+
];
|
|
4432
|
+
if (requiredTools.length > 0 || forbiddenTools.length > 0) {
|
|
4433
|
+
rules.push(
|
|
4434
|
+
createToolUsageRule({
|
|
4435
|
+
required: requiredTools.length > 0 ? requiredTools : void 0,
|
|
4436
|
+
forbidden: forbiddenTools.length > 0 ? forbiddenTools : void 0
|
|
4437
|
+
})
|
|
4438
|
+
);
|
|
4439
|
+
select.add("tool.usage");
|
|
4440
|
+
}
|
|
4441
|
+
const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? config.eval?.maxDurationMs;
|
|
4442
|
+
if (maxDurationMs !== void 0) {
|
|
4443
|
+
rules.push(createRunDurationRule({ maxDurationMs }));
|
|
4444
|
+
select.add("run.duration");
|
|
4445
|
+
}
|
|
4446
|
+
const llm = config.checks?.llm;
|
|
4447
|
+
if (llm?.allowedModels !== void 0 || llm?.maxTotalTokens !== void 0) {
|
|
4448
|
+
rules.push(createLlmUsageRule(llm));
|
|
4449
|
+
select.add("llm.usage");
|
|
4450
|
+
}
|
|
4451
|
+
if (select.has("outcome.status")) {
|
|
4452
|
+
rules.push(createObservedOutcomeRule({ failOn: ["failed"] }));
|
|
4453
|
+
}
|
|
4454
|
+
return { rules, select: [...select] };
|
|
4455
|
+
}
|
|
4456
|
+
function outcomesFromRead(read) {
|
|
4457
|
+
const persistedOutcomes = extractOutcomesFromPersistedEvents(
|
|
4458
|
+
read.events.filter((event) => event.kind === "OUTCOME")
|
|
4459
|
+
);
|
|
4460
|
+
if (persistedOutcomes.length > 0) return persistedOutcomes;
|
|
4461
|
+
const traceEvents = [];
|
|
4462
|
+
for (const event of read.events) {
|
|
4463
|
+
const legacy = event;
|
|
4464
|
+
if (typeof legacy === "object" && legacy !== null && "event" in legacy && legacy.event === "outcome_observed") {
|
|
4465
|
+
traceEvents.push(legacy);
|
|
4466
|
+
}
|
|
4467
|
+
}
|
|
4468
|
+
return traceEvents.length > 0 ? extractOutcomesFromTraceEvents(traceEvents) : [];
|
|
4469
|
+
}
|
|
4470
|
+
function validateExpectedObservations(suiteCase, read) {
|
|
4471
|
+
const expected = suiteCase.expectedObservations ?? [];
|
|
4472
|
+
if (expected.length === 0) return { ok: true, diagnostics: [] };
|
|
4473
|
+
const outcomes = outcomesFromRead(read);
|
|
4474
|
+
const diagnostics = [];
|
|
4475
|
+
for (const name of expected) {
|
|
4476
|
+
const match = outcomes.find((outcome) => outcome.name === name);
|
|
4477
|
+
if (!match) {
|
|
4478
|
+
diagnostics.push(
|
|
4479
|
+
diagnostic4(
|
|
4480
|
+
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
4481
|
+
`Expected observation "${name}" was not recorded.`,
|
|
4482
|
+
"error",
|
|
4483
|
+
suiteCase.id
|
|
4484
|
+
)
|
|
4485
|
+
);
|
|
4486
|
+
continue;
|
|
4487
|
+
}
|
|
4488
|
+
if (match.status !== "passed") {
|
|
4489
|
+
diagnostics.push(
|
|
4490
|
+
diagnostic4(
|
|
4491
|
+
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
4492
|
+
`Observation "${name}" has status "${match.status}", expected "passed".`,
|
|
4493
|
+
"error",
|
|
4494
|
+
suiteCase.id
|
|
4495
|
+
)
|
|
4496
|
+
);
|
|
4497
|
+
}
|
|
4498
|
+
}
|
|
4499
|
+
return { ok: diagnostics.length === 0, diagnostics };
|
|
4500
|
+
}
|
|
4501
|
+
async function runSuiteCase(suiteCase, config, options) {
|
|
4502
|
+
const resolved = await resolveSuiteCaseTrace(suiteCase, options);
|
|
4503
|
+
if (resolved.missing || resolved.tracePath === void 0) {
|
|
4504
|
+
return {
|
|
4505
|
+
id: suiteCase.id,
|
|
4506
|
+
status: "skipped",
|
|
4507
|
+
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
4508
|
+
message: resolved.reason,
|
|
4509
|
+
diagnostics: [
|
|
4510
|
+
diagnostic4(
|
|
4511
|
+
"AI_SUITE_CASE_TRACE_MISSING",
|
|
4512
|
+
resolved.reason ?? "Trace not found.",
|
|
4513
|
+
"warning",
|
|
4514
|
+
suiteCase.id
|
|
4515
|
+
)
|
|
4516
|
+
]
|
|
4517
|
+
};
|
|
4518
|
+
}
|
|
4519
|
+
let read;
|
|
4520
|
+
try {
|
|
4521
|
+
read = await openTrace({ type: "file", path: resolved.tracePath });
|
|
4522
|
+
} catch (error) {
|
|
4523
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4524
|
+
return {
|
|
4525
|
+
id: suiteCase.id,
|
|
4526
|
+
status: "error",
|
|
4527
|
+
tracePath: resolved.tracePath,
|
|
4528
|
+
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
4529
|
+
message,
|
|
4530
|
+
diagnostics: [
|
|
4531
|
+
diagnostic4("AI_SUITE_TRACE_UNREADABLE", message, "error", suiteCase.id)
|
|
4532
|
+
]
|
|
4533
|
+
};
|
|
4534
|
+
}
|
|
4535
|
+
const { rules, select } = buildCaseRules(suiteCase, config);
|
|
4536
|
+
const checkResult = rules.length > 0 ? runTraceChecks({ read }, { rules, select }) : {
|
|
4537
|
+
ok: true,
|
|
4538
|
+
status: "pass",
|
|
4539
|
+
format: read.format,
|
|
4540
|
+
findings: [],
|
|
4541
|
+
diagnostics: []
|
|
4542
|
+
};
|
|
4543
|
+
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
4544
|
+
const diagnostics = [
|
|
4545
|
+
...checkResult.diagnostics.map(
|
|
4546
|
+
(item) => diagnostic4(
|
|
4547
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
4548
|
+
item.message,
|
|
4549
|
+
item.severity,
|
|
4550
|
+
suiteCase.id
|
|
4551
|
+
)
|
|
4552
|
+
),
|
|
4553
|
+
...checkResult.findings.filter((finding) => finding.status === "fail").map(
|
|
4554
|
+
(finding) => diagnostic4(
|
|
4555
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
4556
|
+
finding.message,
|
|
4557
|
+
finding.severity,
|
|
4558
|
+
suiteCase.id
|
|
4559
|
+
)
|
|
4560
|
+
),
|
|
4561
|
+
...observationResult.diagnostics
|
|
4562
|
+
];
|
|
4563
|
+
const checkOk = checkResult.ok;
|
|
4564
|
+
const observationsOk = observationResult.ok;
|
|
4565
|
+
const ok = checkOk && observationsOk;
|
|
4566
|
+
const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
|
|
4567
|
+
return {
|
|
4568
|
+
id: suiteCase.id,
|
|
4569
|
+
status,
|
|
4570
|
+
tracePath: resolved.tracePath,
|
|
4571
|
+
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
4572
|
+
checkOk,
|
|
4573
|
+
observationsOk,
|
|
4574
|
+
diagnostics,
|
|
4575
|
+
...ok ? {} : {
|
|
4576
|
+
message: diagnostics.map((item) => item.message).join("; ") || checkResult.findings.map((item) => item.message).join("; ")
|
|
4577
|
+
}
|
|
4578
|
+
};
|
|
4579
|
+
}
|
|
4580
|
+
async function runSuite(options = {}) {
|
|
4581
|
+
const startedAt = new Date(options.nowMs ?? Date.now()).toISOString();
|
|
4582
|
+
const { config, configPath, configDir } = await loadSuiteConfig(options);
|
|
4583
|
+
const tracesDir = path13__default.default.resolve(configDir, config.traces);
|
|
4584
|
+
const cases = [];
|
|
4585
|
+
const diagnostics = [];
|
|
4586
|
+
for (const suiteCase of config.cases) {
|
|
4587
|
+
cases.push(
|
|
4588
|
+
await runSuiteCase(suiteCase, config, {
|
|
4589
|
+
configDir,
|
|
4590
|
+
tracesDir
|
|
4591
|
+
})
|
|
4592
|
+
);
|
|
4593
|
+
}
|
|
4594
|
+
const summary = {
|
|
4595
|
+
passed: cases.filter((item) => item.status === "pass").length,
|
|
4596
|
+
failed: cases.filter((item) => item.status === "fail").length,
|
|
4597
|
+
errors: cases.filter((item) => item.status === "error").length,
|
|
4598
|
+
skipped: cases.filter((item) => item.status === "skipped").length
|
|
4599
|
+
};
|
|
4600
|
+
const finishedAt = new Date(options.nowMs ?? Date.now()).toISOString();
|
|
4601
|
+
const ok = summary.failed === 0 && summary.errors === 0;
|
|
4602
|
+
const status = summary.errors > 0 ? "error" : summary.failed > 0 || !ok ? "fail" : "pass";
|
|
4603
|
+
return {
|
|
4604
|
+
ok,
|
|
4605
|
+
status,
|
|
4606
|
+
suiteName: config.name,
|
|
4607
|
+
configPath,
|
|
4608
|
+
tracesDir,
|
|
4609
|
+
startedAt,
|
|
4610
|
+
finishedAt,
|
|
4611
|
+
summary,
|
|
4612
|
+
cases,
|
|
4613
|
+
diagnostics
|
|
4614
|
+
};
|
|
4615
|
+
}
|
|
4616
|
+
|
|
4617
|
+
// packages/core/src/exporters/helpers.ts
|
|
4618
|
+
function sortKeysDeep(input) {
|
|
4619
|
+
if (input === null || typeof input !== "object") return input;
|
|
4620
|
+
if (Array.isArray(input)) return input.map(sortKeysDeep);
|
|
4621
|
+
const o = input;
|
|
4622
|
+
const out = {};
|
|
4623
|
+
for (const k of Object.keys(o).sort()) {
|
|
4624
|
+
out[k] = sortKeysDeep(o[k]);
|
|
4625
|
+
}
|
|
4626
|
+
return out;
|
|
4627
|
+
}
|
|
4628
|
+
function stableJson(value, pretty) {
|
|
4629
|
+
const sorted = sortKeysDeep(value);
|
|
4630
|
+
return JSON.stringify(sorted);
|
|
4631
|
+
}
|
|
4632
|
+
|
|
3871
4633
|
// packages/viewer/src/html.ts
|
|
3872
4634
|
var viewerIndexHtml = `<!DOCTYPE html>
|
|
3873
4635
|
<html lang="en">
|
|
@@ -3877,20 +4639,71 @@ var viewerIndexHtml = `<!DOCTYPE html>
|
|
|
3877
4639
|
<style>
|
|
3878
4640
|
body { font-family: system-ui, sans-serif; margin: 1.5rem; line-height: 1.4; }
|
|
3879
4641
|
h1 { font-size: 1.25rem; }
|
|
3880
|
-
pre { background: #f4f4f5; padding: 1rem; overflow: auto; max-height:
|
|
4642
|
+
pre { background: #f4f4f5; padding: 1rem; overflow: auto; max-height: 50vh; }
|
|
3881
4643
|
a { color: #0b57d0; }
|
|
3882
4644
|
.muted { color: #666; }
|
|
4645
|
+
table { border-collapse: collapse; width: 100%; margin: 1rem 0; }
|
|
4646
|
+
th, td { border: 1px solid #ddd; padding: 0.5rem; text-align: left; vertical-align: top; }
|
|
4647
|
+
th { background: #f6f6f6; }
|
|
4648
|
+
.fail { color: #b42318; font-weight: 600; }
|
|
4649
|
+
.pass { color: #027a48; font-weight: 600; }
|
|
4650
|
+
section { margin: 1.5rem 0; }
|
|
3883
4651
|
</style>
|
|
3884
4652
|
</head>
|
|
3885
4653
|
<body>
|
|
3886
4654
|
<h1>AgentInspect local viewer</h1>
|
|
3887
4655
|
<p class="muted">Read-only. JSONL on disk remains canonical.</p>
|
|
3888
|
-
<p
|
|
3889
|
-
<pre id="out">Loading
|
|
4656
|
+
<p id="nav"></p>
|
|
4657
|
+
<div id="content"><pre id="out">Loading\u2026</pre></div>
|
|
3890
4658
|
<script>
|
|
3891
|
-
|
|
3892
|
-
|
|
3893
|
-
|
|
4659
|
+
const params = new URLSearchParams(location.search);
|
|
4660
|
+
const mode = params.get("mode") || "traces";
|
|
4661
|
+
|
|
4662
|
+
function renderSuite(data) {
|
|
4663
|
+
const rows = data.cases.map((c) =>
|
|
4664
|
+
'<tr><td>' + c.id + '</td><td class="' + c.status + '">' + c.status +
|
|
4665
|
+
'</td><td>' + (c.message || '') + '</td><td>' + c.toolPath.join(' \u2192 ') +
|
|
4666
|
+
'</td><td>' + c.observations.map((o) => o.name + ':' + o.status).join(', ') + '</td></tr>'
|
|
4667
|
+
).join('');
|
|
4668
|
+
const failed = data.cases.filter((c) => c.status !== 'pass');
|
|
4669
|
+
const detail = failed.map((c) => {
|
|
4670
|
+
let html = '<h3>Case ' + c.id + '</h3><ul>';
|
|
4671
|
+
if (c.failureDiff) html += '<li>Diff errors: ' + c.failureDiff.summary.errors + '</li>';
|
|
4672
|
+
if (c.timeline) html += '<li>Timeline steps: ' + c.timeline.entries.length + '</li>';
|
|
4673
|
+
html += '<li>Diagnostics: ' + c.diagnostics.map((d) => d.message).join('; ') + '</li></ul>';
|
|
4674
|
+
return html;
|
|
4675
|
+
}).join('');
|
|
4676
|
+
return '<section><h2>Suite: ' + data.suiteName + ' (' + data.status + ')</h2>' +
|
|
4677
|
+
'<p>Passed ' + data.summary.passed + ', failed ' + data.summary.failed + '</p>' +
|
|
4678
|
+
'<table><thead><tr><th>Case</th><th>Status</th><th>Message</th><th>Tool path</th><th>Observations</th></tr></thead><tbody>' +
|
|
4679
|
+
rows + '</tbody></table>' +
|
|
4680
|
+
'<section><h2>Failure detail</h2>' + (detail || '<p class="pass">No failures</p>') + '</section>' +
|
|
4681
|
+
'<section><h2>CI artifacts</h2><p>' + (data.ciArtifactsDir || 'n/a') + '</p></section>' +
|
|
4682
|
+
'<section><h2>Bundle export</h2><p>' + data.bundleExportHint + '</p></section>';
|
|
4683
|
+
}
|
|
4684
|
+
|
|
4685
|
+
async function load() {
|
|
4686
|
+
const nav = document.getElementById("nav");
|
|
4687
|
+
const out = document.getElementById("out");
|
|
4688
|
+
const content = document.getElementById("content");
|
|
4689
|
+
if (mode === "suite") {
|
|
4690
|
+
nav.innerHTML = '<a href="/api/suite">/api/suite</a>';
|
|
4691
|
+
const data = await fetch("/api/suite").then((r) => r.json());
|
|
4692
|
+
content.innerHTML = renderSuite(data);
|
|
4693
|
+
return;
|
|
4694
|
+
}
|
|
4695
|
+
if (mode === "workspace") {
|
|
4696
|
+
nav.innerHTML = '<a href="/api/workspace">/api/workspace</a>';
|
|
4697
|
+
const data = await fetch("/api/workspace").then((r) => r.json());
|
|
4698
|
+
content.innerHTML = '<section><h2>Workspace: ' + (data.project || 'workspace') + '</h2>' +
|
|
4699
|
+
'<p>Runs: ' + data.runs.length + '</p><pre>' + JSON.stringify(data, null, 2) + '</pre></section>';
|
|
4700
|
+
return;
|
|
4701
|
+
}
|
|
4702
|
+
nav.innerHTML = '<a href="/api/traces">/api/traces</a> \xB7 <a href="/api/sessions">/api/sessions</a>';
|
|
4703
|
+
const data = await fetch("/api/traces").then((r) => r.json());
|
|
4704
|
+
out.textContent = JSON.stringify(data, null, 2);
|
|
4705
|
+
}
|
|
4706
|
+
load().catch((err) => {
|
|
3894
4707
|
document.getElementById("out").textContent = String(err);
|
|
3895
4708
|
});
|
|
3896
4709
|
</script>
|
|
@@ -3898,6 +4711,815 @@ var viewerIndexHtml = `<!DOCTYPE html>
|
|
|
3898
4711
|
</html>
|
|
3899
4712
|
`;
|
|
3900
4713
|
|
|
4714
|
+
// packages/core/src/diff/comparable.ts
|
|
4715
|
+
function extractOutputPreview(meta) {
|
|
4716
|
+
if (meta === void 0) return void 0;
|
|
4717
|
+
if ("outputPreview" in meta) return meta.outputPreview;
|
|
4718
|
+
if ("resultPreview" in meta) return meta.resultPreview;
|
|
4719
|
+
return void 0;
|
|
4720
|
+
}
|
|
4721
|
+
function mapStepStatus(s) {
|
|
4722
|
+
if (s === void 0) return "running";
|
|
4723
|
+
return s;
|
|
4724
|
+
}
|
|
4725
|
+
function manualTraceEventsToComparableRun(events) {
|
|
4726
|
+
const started = events.find((e) => e.event === "run_started");
|
|
4727
|
+
if (!started || started.event !== "run_started") {
|
|
4728
|
+
throw new Error("Invalid trace: missing run_started");
|
|
4729
|
+
}
|
|
4730
|
+
const rs = started;
|
|
4731
|
+
const runId = rs.runId;
|
|
4732
|
+
const completedAll = events.filter((e) => e.event === "run_completed");
|
|
4733
|
+
const lastCompleted = completedAll[completedAll.length - 1];
|
|
4734
|
+
let runStatus;
|
|
4735
|
+
if (lastCompleted === void 0) runStatus = "running";
|
|
4736
|
+
else runStatus = lastCompleted.status;
|
|
4737
|
+
const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
4738
|
+
const steps = /* @__PURE__ */ new Map();
|
|
4739
|
+
let order = 0;
|
|
4740
|
+
for (const e of events) {
|
|
4741
|
+
if (e.event !== "step_started") continue;
|
|
4742
|
+
const s = e;
|
|
4743
|
+
const meta = s.metadata ? { ...s.metadata } : void 0;
|
|
4744
|
+
steps.set(s.stepId, {
|
|
4745
|
+
id: s.stepId,
|
|
4746
|
+
parentId: s.parentId,
|
|
4747
|
+
name: s.name,
|
|
4748
|
+
type: s.type,
|
|
4749
|
+
order: order++,
|
|
4750
|
+
timestamp: s.timestamp,
|
|
4751
|
+
metadata: meta
|
|
4752
|
+
});
|
|
4753
|
+
}
|
|
4754
|
+
for (const e of events) {
|
|
4755
|
+
if (e.event !== "step_completed") continue;
|
|
4756
|
+
const acc = steps.get(e.stepId);
|
|
4757
|
+
if (!acc) continue;
|
|
4758
|
+
acc.status = e.status;
|
|
4759
|
+
acc.durationMs = e.durationMs;
|
|
4760
|
+
if (e.error?.message) acc.errorMsg = e.error.message;
|
|
4761
|
+
const extra = e;
|
|
4762
|
+
if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
|
|
4763
|
+
acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
|
|
4764
|
+
}
|
|
4765
|
+
}
|
|
4766
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
4767
|
+
for (const acc of steps.values()) {
|
|
4768
|
+
let meta = acc.metadata ? { ...acc.metadata } : void 0;
|
|
4769
|
+
if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
|
|
4770
|
+
meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
|
|
4771
|
+
}
|
|
4772
|
+
const outputPreview = extractOutputPreview(meta);
|
|
4773
|
+
const sc = {
|
|
4774
|
+
id: acc.id,
|
|
4775
|
+
name: acc.name,
|
|
4776
|
+
type: acc.type,
|
|
4777
|
+
status: mapStepStatus(acc.status),
|
|
4778
|
+
durationMs: acc.durationMs,
|
|
4779
|
+
error: acc.errorMsg,
|
|
4780
|
+
metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
|
|
4781
|
+
outputPreview,
|
|
4782
|
+
children: []
|
|
4783
|
+
};
|
|
4784
|
+
nodes.set(acc.id, sc);
|
|
4785
|
+
}
|
|
4786
|
+
const roots = [];
|
|
4787
|
+
const sortByOrder = (a, b) => {
|
|
4788
|
+
const oa = steps.get(a.id)?.order ?? 0;
|
|
4789
|
+
const ob = steps.get(b.id)?.order ?? 0;
|
|
4790
|
+
return oa - ob;
|
|
4791
|
+
};
|
|
4792
|
+
for (const acc of steps.values()) {
|
|
4793
|
+
const node = nodes.get(acc.id);
|
|
4794
|
+
if (acc.parentId !== void 0 && nodes.has(acc.parentId)) {
|
|
4795
|
+
nodes.get(acc.parentId).children.push(node);
|
|
4796
|
+
} else {
|
|
4797
|
+
roots.push(node);
|
|
4798
|
+
}
|
|
4799
|
+
}
|
|
4800
|
+
roots.sort(sortByOrder);
|
|
4801
|
+
for (const n of nodes.values()) {
|
|
4802
|
+
n.children.sort(sortByOrder);
|
|
4803
|
+
}
|
|
4804
|
+
return {
|
|
4805
|
+
runId,
|
|
4806
|
+
name: rs.name,
|
|
4807
|
+
status: runStatus,
|
|
4808
|
+
durationMs,
|
|
4809
|
+
steps: roots
|
|
4810
|
+
};
|
|
4811
|
+
}
|
|
4812
|
+
|
|
4813
|
+
// packages/core/src/diff/engine.ts
|
|
4814
|
+
var DEFAULT_THRESHOLD_MS = 0;
|
|
4815
|
+
function pathSeg(step, index) {
|
|
4816
|
+
return { index, name: step.name, stepId: step.id };
|
|
4817
|
+
}
|
|
4818
|
+
function buildPath(segments) {
|
|
4819
|
+
return { path: [...segments] };
|
|
4820
|
+
}
|
|
4821
|
+
function pairSteps(left, right) {
|
|
4822
|
+
const usedRight = /* @__PURE__ */ new Set();
|
|
4823
|
+
const pairs = [];
|
|
4824
|
+
for (let i = 0; i < left.length; i++) {
|
|
4825
|
+
const L = left[i];
|
|
4826
|
+
let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
|
|
4827
|
+
if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
|
|
4828
|
+
const cand = right[i];
|
|
4829
|
+
if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
|
|
4830
|
+
R = cand;
|
|
4831
|
+
}
|
|
4832
|
+
}
|
|
4833
|
+
if (R === void 0) {
|
|
4834
|
+
R = right.find(
|
|
4835
|
+
(r) => !usedRight.has(r.id) && r.name === L.name && (r.type ?? "") === (L.type ?? "")
|
|
4836
|
+
);
|
|
4837
|
+
}
|
|
4838
|
+
if (R !== void 0) {
|
|
4839
|
+
usedRight.add(R.id);
|
|
4840
|
+
pairs.push([L, R]);
|
|
4841
|
+
} else {
|
|
4842
|
+
pairs.push([L, void 0]);
|
|
4843
|
+
}
|
|
4844
|
+
}
|
|
4845
|
+
for (const R of right) {
|
|
4846
|
+
if (!usedRight.has(R.id)) {
|
|
4847
|
+
pairs.push([void 0, R]);
|
|
4848
|
+
}
|
|
4849
|
+
}
|
|
4850
|
+
return pairs;
|
|
4851
|
+
}
|
|
4852
|
+
function compareLeafSteps(L, R, segments, opts, out) {
|
|
4853
|
+
const path16 = buildPath(segments);
|
|
4854
|
+
if (L.name !== R.name) {
|
|
4855
|
+
out.push({
|
|
4856
|
+
kind: "structure",
|
|
4857
|
+
severity: "warning",
|
|
4858
|
+
message: "Step name differs",
|
|
4859
|
+
path: path16,
|
|
4860
|
+
left: L.name,
|
|
4861
|
+
right: R.name
|
|
4862
|
+
});
|
|
4863
|
+
}
|
|
4864
|
+
if ((L.type ?? "") !== (R.type ?? "")) {
|
|
4865
|
+
out.push({
|
|
4866
|
+
kind: "step-type",
|
|
4867
|
+
severity: "warning",
|
|
4868
|
+
message: "Step type differs",
|
|
4869
|
+
path: path16,
|
|
4870
|
+
left: L.type,
|
|
4871
|
+
right: R.type
|
|
4872
|
+
});
|
|
4873
|
+
}
|
|
4874
|
+
if ((L.status ?? "") !== (R.status ?? "")) {
|
|
4875
|
+
out.push({
|
|
4876
|
+
kind: "step-status",
|
|
4877
|
+
severity: "warning",
|
|
4878
|
+
message: "Step status differs",
|
|
4879
|
+
path: path16,
|
|
4880
|
+
left: L.status,
|
|
4881
|
+
right: R.status
|
|
4882
|
+
});
|
|
4883
|
+
}
|
|
4884
|
+
const le = L.error ?? "";
|
|
4885
|
+
const re = R.error ?? "";
|
|
4886
|
+
if (le !== re) {
|
|
4887
|
+
out.push({
|
|
4888
|
+
kind: "error",
|
|
4889
|
+
severity: "error",
|
|
4890
|
+
message: "Step error message differs",
|
|
4891
|
+
path: path16,
|
|
4892
|
+
left: le || void 0,
|
|
4893
|
+
right: re || void 0
|
|
4894
|
+
});
|
|
4895
|
+
}
|
|
4896
|
+
if (!opts.ignoreDuration) {
|
|
4897
|
+
const ld = L.durationMs;
|
|
4898
|
+
const rd = R.durationMs;
|
|
4899
|
+
const th = opts.durationThresholdMs;
|
|
4900
|
+
let differs = false;
|
|
4901
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
4902
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
4903
|
+
else differs = Math.abs(ld - rd) > th;
|
|
4904
|
+
if (differs) {
|
|
4905
|
+
out.push({
|
|
4906
|
+
kind: "duration",
|
|
4907
|
+
severity: "info",
|
|
4908
|
+
message: "Step duration differs",
|
|
4909
|
+
path: path16,
|
|
4910
|
+
left: ld,
|
|
4911
|
+
right: rd
|
|
4912
|
+
});
|
|
4913
|
+
}
|
|
4914
|
+
}
|
|
4915
|
+
const lm = stableJson(L.metadata ?? {});
|
|
4916
|
+
const rm2 = stableJson(R.metadata ?? {});
|
|
4917
|
+
if (lm !== rm2) {
|
|
4918
|
+
out.push({
|
|
4919
|
+
kind: "metadata",
|
|
4920
|
+
severity: "info",
|
|
4921
|
+
message: "Step metadata differs",
|
|
4922
|
+
path: path16,
|
|
4923
|
+
left: L.metadata,
|
|
4924
|
+
right: R.metadata
|
|
4925
|
+
});
|
|
4926
|
+
}
|
|
4927
|
+
const lo = stableJson(L.outputPreview ?? null);
|
|
4928
|
+
const ro = stableJson(R.outputPreview ?? null);
|
|
4929
|
+
if (lo !== ro) {
|
|
4930
|
+
out.push({
|
|
4931
|
+
kind: "output",
|
|
4932
|
+
severity: "info",
|
|
4933
|
+
message: "Output preview differs",
|
|
4934
|
+
path: path16,
|
|
4935
|
+
left: L.outputPreview,
|
|
4936
|
+
right: R.outputPreview
|
|
4937
|
+
});
|
|
4938
|
+
}
|
|
4939
|
+
}
|
|
4940
|
+
function compareRecursive(L, R, segments, opts, out) {
|
|
4941
|
+
compareLeafSteps(L, R, segments, opts, out);
|
|
4942
|
+
const pairs = pairSteps(L.children, R.children);
|
|
4943
|
+
let ci = 0;
|
|
4944
|
+
for (const [lch, rch] of pairs) {
|
|
4945
|
+
if (lch !== void 0 && rch !== void 0) {
|
|
4946
|
+
compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
|
|
4947
|
+
} else if (lch !== void 0) {
|
|
4948
|
+
out.push({
|
|
4949
|
+
kind: "step-removed",
|
|
4950
|
+
severity: "warning",
|
|
4951
|
+
message: `Step only in left run: ${lch.name}`,
|
|
4952
|
+
path: buildPath([...segments, pathSeg(lch, ci)]),
|
|
4953
|
+
left: lch.id,
|
|
4954
|
+
right: void 0
|
|
4955
|
+
});
|
|
4956
|
+
} else if (rch !== void 0) {
|
|
4957
|
+
out.push({
|
|
4958
|
+
kind: "step-added",
|
|
4959
|
+
severity: "warning",
|
|
4960
|
+
message: `Step only in right run: ${rch.name}`,
|
|
4961
|
+
path: buildPath([...segments, pathSeg(rch, ci)]),
|
|
4962
|
+
left: void 0,
|
|
4963
|
+
right: rch.id
|
|
4964
|
+
});
|
|
4965
|
+
}
|
|
4966
|
+
ci += 1;
|
|
4967
|
+
}
|
|
4968
|
+
}
|
|
4969
|
+
function mergeDiffDefaults(options) {
|
|
4970
|
+
return {
|
|
4971
|
+
ignoreDuration: false,
|
|
4972
|
+
durationThresholdMs: DEFAULT_THRESHOLD_MS,
|
|
4973
|
+
focus: "all",
|
|
4974
|
+
check: "all"
|
|
4975
|
+
};
|
|
4976
|
+
}
|
|
4977
|
+
function kindMatchesFilter(kind, merged) {
|
|
4978
|
+
return true;
|
|
4979
|
+
}
|
|
4980
|
+
function diffRuns(left, right, options) {
|
|
4981
|
+
const merged = mergeDiffDefaults();
|
|
4982
|
+
const opts = {
|
|
4983
|
+
ignoreDuration: merged.ignoreDuration,
|
|
4984
|
+
durationThresholdMs: merged.durationThresholdMs
|
|
4985
|
+
};
|
|
4986
|
+
const raw = [];
|
|
4987
|
+
if ((left.status ?? "") !== (right.status ?? "")) {
|
|
4988
|
+
raw.push({
|
|
4989
|
+
kind: "run-status",
|
|
4990
|
+
severity: "warning",
|
|
4991
|
+
message: "Run completion status differs",
|
|
4992
|
+
left: left.status,
|
|
4993
|
+
right: right.status
|
|
4994
|
+
});
|
|
4995
|
+
}
|
|
4996
|
+
{
|
|
4997
|
+
const ld = left.durationMs;
|
|
4998
|
+
const rd = right.durationMs;
|
|
4999
|
+
const th = merged.durationThresholdMs;
|
|
5000
|
+
let differs = false;
|
|
5001
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
5002
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
5003
|
+
else differs = Math.abs(ld - rd) > th;
|
|
5004
|
+
if (differs) {
|
|
5005
|
+
raw.push({
|
|
5006
|
+
kind: "duration",
|
|
5007
|
+
severity: "info",
|
|
5008
|
+
message: "Run duration differs",
|
|
5009
|
+
left: ld,
|
|
5010
|
+
right: rd
|
|
5011
|
+
});
|
|
5012
|
+
}
|
|
5013
|
+
}
|
|
5014
|
+
const pairs = pairSteps(left.steps, right.steps);
|
|
5015
|
+
let idx = 0;
|
|
5016
|
+
for (const [ls, rs] of pairs) {
|
|
5017
|
+
if (ls !== void 0 && rs !== void 0) {
|
|
5018
|
+
compareRecursive(ls, rs, [pathSeg(ls, idx)], opts, raw);
|
|
5019
|
+
idx += 1;
|
|
5020
|
+
} else if (ls !== void 0) {
|
|
5021
|
+
raw.push({
|
|
5022
|
+
kind: "step-removed",
|
|
5023
|
+
severity: "warning",
|
|
5024
|
+
message: `Step only in left run: ${ls.name}`,
|
|
5025
|
+
path: buildPath([pathSeg(ls, idx)]),
|
|
5026
|
+
left: ls.id,
|
|
5027
|
+
right: void 0
|
|
5028
|
+
});
|
|
5029
|
+
idx += 1;
|
|
5030
|
+
} else if (rs !== void 0) {
|
|
5031
|
+
raw.push({
|
|
5032
|
+
kind: "step-added",
|
|
5033
|
+
severity: "warning",
|
|
5034
|
+
message: `Step only in right run: ${rs.name}`,
|
|
5035
|
+
path: buildPath([pathSeg(rs, idx)]),
|
|
5036
|
+
left: void 0,
|
|
5037
|
+
right: rs.id
|
|
5038
|
+
});
|
|
5039
|
+
idx += 1;
|
|
5040
|
+
}
|
|
5041
|
+
}
|
|
5042
|
+
const differences = raw.filter((d) => kindMatchesFilter(d.kind));
|
|
5043
|
+
let errors = 0;
|
|
5044
|
+
let warnings = 0;
|
|
5045
|
+
let info = 0;
|
|
5046
|
+
for (const d of differences) {
|
|
5047
|
+
if (d.severity === "error") errors += 1;
|
|
5048
|
+
else if (d.severity === "warning") warnings += 1;
|
|
5049
|
+
else info += 1;
|
|
5050
|
+
}
|
|
5051
|
+
const firstVisible = differences[0];
|
|
5052
|
+
const firstDivergence = firstVisible !== void 0 ? {
|
|
5053
|
+
kind: "first-divergence",
|
|
5054
|
+
severity: firstVisible.severity,
|
|
5055
|
+
message: `First divergence: ${firstVisible.message}`,
|
|
5056
|
+
path: firstVisible.path,
|
|
5057
|
+
left: firstVisible.left,
|
|
5058
|
+
right: firstVisible.right
|
|
5059
|
+
} : void 0;
|
|
5060
|
+
const summary = {
|
|
5061
|
+
leftRunId: left.runId,
|
|
5062
|
+
rightRunId: right.runId,
|
|
5063
|
+
totalDifferences: differences.length,
|
|
5064
|
+
errors,
|
|
5065
|
+
warnings,
|
|
5066
|
+
info,
|
|
5067
|
+
firstDivergence
|
|
5068
|
+
};
|
|
5069
|
+
return { summary, differences };
|
|
5070
|
+
}
|
|
5071
|
+
|
|
5072
|
+
// packages/viewer/src/suite-data.ts
|
|
5073
|
+
async function enrichCase(suiteCase, baselineTracePath) {
|
|
5074
|
+
const base = {
|
|
5075
|
+
id: suiteCase.id,
|
|
5076
|
+
status: suiteCase.status,
|
|
5077
|
+
...suiteCase.tracePath !== void 0 ? { tracePath: suiteCase.tracePath } : {},
|
|
5078
|
+
...suiteCase.runId !== void 0 ? { runId: suiteCase.runId } : {},
|
|
5079
|
+
...suiteCase.message !== void 0 ? { message: suiteCase.message } : {},
|
|
5080
|
+
diagnostics: suiteCase.diagnostics,
|
|
5081
|
+
toolPath: [],
|
|
5082
|
+
observations: []
|
|
5083
|
+
};
|
|
5084
|
+
if (suiteCase.tracePath === void 0) return base;
|
|
5085
|
+
try {
|
|
5086
|
+
const read = await openTrace({ type: "file", path: suiteCase.tracePath });
|
|
5087
|
+
const legacy = persistedInspectEventsToTraceEvents(read.events);
|
|
5088
|
+
const timeline = buildRunTimeline(legacy, { focus: "all" });
|
|
5089
|
+
const observations = extractOutcomesFromTraceEvents(legacy);
|
|
5090
|
+
const toolPath = timeline.entries.filter((entry) => entry.type === "tool").map((entry) => entry.name);
|
|
5091
|
+
const detail = {
|
|
5092
|
+
...base,
|
|
5093
|
+
timeline,
|
|
5094
|
+
toolPath,
|
|
5095
|
+
observations
|
|
5096
|
+
};
|
|
5097
|
+
if (baselineTracePath !== void 0 && suiteCase.status !== "pass") {
|
|
5098
|
+
try {
|
|
5099
|
+
const baselineRead = await openTrace({ type: "file", path: baselineTracePath });
|
|
5100
|
+
const diff = diffRuns(
|
|
5101
|
+
manualTraceEventsToComparableRun(
|
|
5102
|
+
persistedInspectEventsToTraceEvents(baselineRead.events)
|
|
5103
|
+
),
|
|
5104
|
+
manualTraceEventsToComparableRun(legacy)
|
|
5105
|
+
);
|
|
5106
|
+
detail.failureDiff = {
|
|
5107
|
+
summary: diff.summary,
|
|
5108
|
+
differences: diff.differences.slice(0, 20).map((item) => ({
|
|
5109
|
+
kind: item.kind,
|
|
5110
|
+
message: item.message
|
|
5111
|
+
}))
|
|
5112
|
+
};
|
|
5113
|
+
} catch {
|
|
5114
|
+
}
|
|
5115
|
+
}
|
|
5116
|
+
return detail;
|
|
5117
|
+
} catch {
|
|
5118
|
+
return base;
|
|
5119
|
+
}
|
|
5120
|
+
}
|
|
5121
|
+
async function loadSuiteViewerData(options) {
|
|
5122
|
+
const result = await runSuite({
|
|
5123
|
+
configPath: options.suiteConfigPath,
|
|
5124
|
+
cwd: options.cwd
|
|
5125
|
+
});
|
|
5126
|
+
const baselineCase = result.cases.find((item) => item.status === "pass");
|
|
5127
|
+
const baselinePath = baselineCase?.tracePath;
|
|
5128
|
+
const cases = [];
|
|
5129
|
+
for (const suiteCase of result.cases) {
|
|
5130
|
+
cases.push(await enrichCase(suiteCase, baselinePath));
|
|
5131
|
+
}
|
|
5132
|
+
const artifactsDir = path13__default.default.join(path13__default.default.dirname(result.configPath), ".agent-inspect/suite-runs");
|
|
5133
|
+
return {
|
|
5134
|
+
suiteName: result.suiteName,
|
|
5135
|
+
configPath: result.configPath,
|
|
5136
|
+
tracesDir: result.tracesDir,
|
|
5137
|
+
ok: result.ok,
|
|
5138
|
+
status: result.status,
|
|
5139
|
+
summary: result.summary,
|
|
5140
|
+
cases,
|
|
5141
|
+
ciArtifactsDir: artifactsDir,
|
|
5142
|
+
bundleExportHint: "Export a share-safe bundle with: npx agent-inspect bundle <runId> --profile share"
|
|
5143
|
+
};
|
|
5144
|
+
}
|
|
5145
|
+
|
|
5146
|
+
// packages/core/src/workspace/types.ts
|
|
5147
|
+
var WORKSPACE_SCHEMA_VERSION = "1.0";
|
|
5148
|
+
var WORKSPACE_DIR_NAME = ".agent-inspect";
|
|
5149
|
+
var WORKSPACE_MANIFEST_FILENAME = "workspace.json";
|
|
5150
|
+
|
|
5151
|
+
// packages/core/src/workspace/manifest.ts
|
|
5152
|
+
var REDACTION_PROFILES = [
|
|
5153
|
+
"local",
|
|
5154
|
+
"share",
|
|
5155
|
+
"strict"
|
|
5156
|
+
];
|
|
5157
|
+
var INDEX_TYPES = ["none", "sqlite", "custom"];
|
|
5158
|
+
var MAX_WORKSPACE_MANIFEST_BYTES = 64 * 1024;
|
|
5159
|
+
function isPlainObject(value) {
|
|
5160
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
5161
|
+
}
|
|
5162
|
+
function isSafeRelativeWorkspacePath(p) {
|
|
5163
|
+
if (typeof p !== "string") return false;
|
|
5164
|
+
const trimmed = p.trim();
|
|
5165
|
+
if (trimmed === "") return false;
|
|
5166
|
+
if (trimmed.startsWith("/") || trimmed.startsWith("\\")) return false;
|
|
5167
|
+
if (/^[a-zA-Z]:/.test(trimmed)) return false;
|
|
5168
|
+
const segments = trimmed.split(/[/\\]+/);
|
|
5169
|
+
return !segments.some((seg) => seg === "..");
|
|
5170
|
+
}
|
|
5171
|
+
function validateDirField(value, field, errors) {
|
|
5172
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
5173
|
+
errors.push(`${field} must be a non-empty string`);
|
|
5174
|
+
return;
|
|
5175
|
+
}
|
|
5176
|
+
if (!isSafeRelativeWorkspacePath(value)) {
|
|
5177
|
+
errors.push(
|
|
5178
|
+
`${field} must be a relative path inside the workspace (no absolute paths or ".." traversal)`
|
|
5179
|
+
);
|
|
5180
|
+
}
|
|
5181
|
+
}
|
|
5182
|
+
function validateIndex(value, errors) {
|
|
5183
|
+
if (!isPlainObject(value)) {
|
|
5184
|
+
errors.push("index must be an object");
|
|
5185
|
+
return void 0;
|
|
5186
|
+
}
|
|
5187
|
+
if (typeof value.enabled !== "boolean") {
|
|
5188
|
+
errors.push("index.enabled must be a boolean");
|
|
5189
|
+
}
|
|
5190
|
+
if (!INDEX_TYPES.includes(value.type)) {
|
|
5191
|
+
errors.push(`index.type must be one of: ${INDEX_TYPES.join(", ")}`);
|
|
5192
|
+
}
|
|
5193
|
+
if (value.path !== void 0 && !isSafeRelativeWorkspacePath(value.path)) {
|
|
5194
|
+
errors.push(
|
|
5195
|
+
'index.path must be a relative path inside the workspace (no absolute paths or ".." traversal)'
|
|
5196
|
+
);
|
|
5197
|
+
}
|
|
5198
|
+
if (errors.length > 0) return void 0;
|
|
5199
|
+
return {
|
|
5200
|
+
enabled: value.enabled,
|
|
5201
|
+
type: value.type,
|
|
5202
|
+
...value.path !== void 0 ? { path: value.path } : {}
|
|
5203
|
+
};
|
|
5204
|
+
}
|
|
5205
|
+
function validateWorkspaceManifest(input) {
|
|
5206
|
+
const errors = [];
|
|
5207
|
+
const warnings = [];
|
|
5208
|
+
if (!isPlainObject(input)) {
|
|
5209
|
+
return { ok: false, errors: ["manifest must be an object"], warnings };
|
|
5210
|
+
}
|
|
5211
|
+
if (input.schemaVersion !== WORKSPACE_SCHEMA_VERSION) {
|
|
5212
|
+
errors.push(
|
|
5213
|
+
`schemaVersion must be "${WORKSPACE_SCHEMA_VERSION}" (received ${JSON.stringify(
|
|
5214
|
+
input.schemaVersion
|
|
5215
|
+
)})`
|
|
5216
|
+
);
|
|
5217
|
+
}
|
|
5218
|
+
if (typeof input.project !== "string" || input.project.trim() === "") {
|
|
5219
|
+
errors.push("project must be a non-empty string");
|
|
5220
|
+
}
|
|
5221
|
+
if (typeof input.createdAt !== "string" || input.createdAt.trim() === "") {
|
|
5222
|
+
errors.push("createdAt must be a non-empty ISO-8601 string");
|
|
5223
|
+
} else if (Number.isNaN(Date.parse(input.createdAt))) {
|
|
5224
|
+
errors.push("createdAt must be a valid ISO-8601 date string");
|
|
5225
|
+
}
|
|
5226
|
+
if (!Array.isArray(input.traceDirs) || input.traceDirs.length === 0) {
|
|
5227
|
+
errors.push("traceDirs must be a non-empty array");
|
|
5228
|
+
} else {
|
|
5229
|
+
input.traceDirs.forEach((dir, i) => {
|
|
5230
|
+
if (typeof dir !== "string" || dir.trim() === "") {
|
|
5231
|
+
errors.push(`traceDirs[${i}] must be a non-empty string`);
|
|
5232
|
+
} else if (!isSafeRelativeWorkspacePath(dir)) {
|
|
5233
|
+
errors.push(
|
|
5234
|
+
`traceDirs[${i}] must be a relative path inside the workspace (no absolute paths or ".." traversal)`
|
|
5235
|
+
);
|
|
5236
|
+
}
|
|
5237
|
+
});
|
|
5238
|
+
}
|
|
5239
|
+
validateDirField(input.reportsDir, "reportsDir", errors);
|
|
5240
|
+
validateDirField(input.artifactsDir, "artifactsDir", errors);
|
|
5241
|
+
validateDirField(input.bundlesDir, "bundlesDir", errors);
|
|
5242
|
+
validateDirField(input.notesDir, "notesDir", errors);
|
|
5243
|
+
if (!REDACTION_PROFILES.includes(input.redactionProfile)) {
|
|
5244
|
+
errors.push(`redactionProfile must be one of: ${REDACTION_PROFILES.join(", ")}`);
|
|
5245
|
+
}
|
|
5246
|
+
const indexErrors = [];
|
|
5247
|
+
const index = validateIndex(input.index, indexErrors);
|
|
5248
|
+
errors.push(...indexErrors);
|
|
5249
|
+
if (index && index.type !== "none" && !index.enabled) {
|
|
5250
|
+
warnings.push(`index.type is "${index.type}" but index.enabled is false`);
|
|
5251
|
+
}
|
|
5252
|
+
if (errors.length > 0 || index === void 0) {
|
|
5253
|
+
return { ok: false, errors, warnings };
|
|
5254
|
+
}
|
|
5255
|
+
const manifest = {
|
|
5256
|
+
schemaVersion: WORKSPACE_SCHEMA_VERSION,
|
|
5257
|
+
project: input.project.trim(),
|
|
5258
|
+
createdAt: input.createdAt,
|
|
5259
|
+
traceDirs: input.traceDirs.map((d) => d.trim()),
|
|
5260
|
+
reportsDir: input.reportsDir.trim(),
|
|
5261
|
+
artifactsDir: input.artifactsDir.trim(),
|
|
5262
|
+
bundlesDir: input.bundlesDir.trim(),
|
|
5263
|
+
notesDir: input.notesDir.trim(),
|
|
5264
|
+
redactionProfile: input.redactionProfile,
|
|
5265
|
+
index
|
|
5266
|
+
};
|
|
5267
|
+
return { ok: true, manifest, errors, warnings };
|
|
5268
|
+
}
|
|
5269
|
+
function parseWorkspaceManifest(json) {
|
|
5270
|
+
const warnings = [];
|
|
5271
|
+
if (typeof json !== "string") {
|
|
5272
|
+
return { ok: false, errors: ["manifest input must be a string"], warnings };
|
|
5273
|
+
}
|
|
5274
|
+
if (json.length > MAX_WORKSPACE_MANIFEST_BYTES) {
|
|
5275
|
+
return {
|
|
5276
|
+
ok: false,
|
|
5277
|
+
errors: [
|
|
5278
|
+
`manifest exceeds maximum size of ${MAX_WORKSPACE_MANIFEST_BYTES} bytes`
|
|
5279
|
+
],
|
|
5280
|
+
warnings
|
|
5281
|
+
};
|
|
5282
|
+
}
|
|
5283
|
+
let parsed;
|
|
5284
|
+
try {
|
|
5285
|
+
parsed = JSON.parse(json);
|
|
5286
|
+
} catch {
|
|
5287
|
+
return { ok: false, errors: ["manifest is not valid JSON"], warnings };
|
|
5288
|
+
}
|
|
5289
|
+
return validateWorkspaceManifest(parsed);
|
|
5290
|
+
}
|
|
5291
|
+
var INDEX_DIR_NAME = "index";
|
|
5292
|
+
function resolveWorkspaceLocation(cwd = process.cwd()) {
|
|
5293
|
+
const projectRoot = path13__default.default.resolve(cwd);
|
|
5294
|
+
const workspaceDir = path13__default.default.join(projectRoot, WORKSPACE_DIR_NAME);
|
|
5295
|
+
return {
|
|
5296
|
+
projectRoot,
|
|
5297
|
+
workspaceDir,
|
|
5298
|
+
manifestPath: path13__default.default.join(workspaceDir, WORKSPACE_MANIFEST_FILENAME)
|
|
5299
|
+
};
|
|
5300
|
+
}
|
|
5301
|
+
function resolveInsideWorkspace(workspaceDir, relative) {
|
|
5302
|
+
const base = path13__default.default.resolve(workspaceDir);
|
|
5303
|
+
const resolved = path13__default.default.resolve(base, relative);
|
|
5304
|
+
const rel = path13__default.default.relative(base, resolved);
|
|
5305
|
+
if (rel === "" || rel === "." || !rel.startsWith("..") && !path13__default.default.isAbsolute(rel)) {
|
|
5306
|
+
return resolved;
|
|
5307
|
+
}
|
|
5308
|
+
throw new Error(
|
|
5309
|
+
`Workspace path "${relative}" resolves outside the workspace directory`
|
|
5310
|
+
);
|
|
5311
|
+
}
|
|
5312
|
+
async function pathExists(p) {
|
|
5313
|
+
try {
|
|
5314
|
+
await promises.access(p);
|
|
5315
|
+
return true;
|
|
5316
|
+
} catch {
|
|
5317
|
+
return false;
|
|
5318
|
+
}
|
|
5319
|
+
}
|
|
5320
|
+
async function isWritable(p) {
|
|
5321
|
+
try {
|
|
5322
|
+
await promises.access(p, fs.constants.W_OK);
|
|
5323
|
+
return true;
|
|
5324
|
+
} catch {
|
|
5325
|
+
return false;
|
|
5326
|
+
}
|
|
5327
|
+
}
|
|
5328
|
+
async function listJsonl(dir) {
|
|
5329
|
+
try {
|
|
5330
|
+
const entries = await promises.readdir(dir);
|
|
5331
|
+
return entries.filter((f) => f.endsWith(".jsonl"));
|
|
5332
|
+
} catch {
|
|
5333
|
+
return [];
|
|
5334
|
+
}
|
|
5335
|
+
}
|
|
5336
|
+
async function countFiles(dir) {
|
|
5337
|
+
try {
|
|
5338
|
+
const entries = await promises.readdir(dir, { withFileTypes: true });
|
|
5339
|
+
return entries.filter((e) => e.isFile()).length;
|
|
5340
|
+
} catch {
|
|
5341
|
+
return 0;
|
|
5342
|
+
}
|
|
5343
|
+
}
|
|
5344
|
+
async function readWorkspaceManifestFile(location) {
|
|
5345
|
+
let raw;
|
|
5346
|
+
try {
|
|
5347
|
+
raw = await promises.readFile(location.manifestPath, "utf-8");
|
|
5348
|
+
} catch {
|
|
5349
|
+
return { exists: false, ok: false, errors: ["workspace.json not found"], warnings: [] };
|
|
5350
|
+
}
|
|
5351
|
+
const parsed = parseWorkspaceManifest(raw);
|
|
5352
|
+
return {
|
|
5353
|
+
exists: true,
|
|
5354
|
+
ok: parsed.ok,
|
|
5355
|
+
...parsed.manifest ? { manifest: parsed.manifest } : {},
|
|
5356
|
+
errors: parsed.errors,
|
|
5357
|
+
warnings: parsed.warnings
|
|
5358
|
+
};
|
|
5359
|
+
}
|
|
5360
|
+
async function getWorkspaceStatus(location, manifest) {
|
|
5361
|
+
let traceFiles = 0;
|
|
5362
|
+
for (const rel of manifest.traceDirs) {
|
|
5363
|
+
const abs = resolveInsideWorkspace(location.workspaceDir, rel);
|
|
5364
|
+
traceFiles += (await listJsonl(abs)).length;
|
|
5365
|
+
}
|
|
5366
|
+
const reports = await countFiles(
|
|
5367
|
+
resolveInsideWorkspace(location.workspaceDir, manifest.reportsDir)
|
|
5368
|
+
);
|
|
5369
|
+
const artifacts = await countFiles(
|
|
5370
|
+
resolveInsideWorkspace(location.workspaceDir, manifest.artifactsDir)
|
|
5371
|
+
);
|
|
5372
|
+
const bundles = await countFiles(
|
|
5373
|
+
resolveInsideWorkspace(location.workspaceDir, manifest.bundlesDir)
|
|
5374
|
+
);
|
|
5375
|
+
const notes = await countFiles(
|
|
5376
|
+
resolveInsideWorkspace(location.workspaceDir, manifest.notesDir)
|
|
5377
|
+
);
|
|
5378
|
+
const indexPath = manifest.index.path ? resolveInsideWorkspace(location.workspaceDir, manifest.index.path) : resolveInsideWorkspace(location.workspaceDir, INDEX_DIR_NAME);
|
|
5379
|
+
return {
|
|
5380
|
+
project: manifest.project,
|
|
5381
|
+
traceFiles,
|
|
5382
|
+
reports,
|
|
5383
|
+
artifacts,
|
|
5384
|
+
bundles,
|
|
5385
|
+
notes,
|
|
5386
|
+
index: {
|
|
5387
|
+
enabled: manifest.index.enabled,
|
|
5388
|
+
type: manifest.index.type,
|
|
5389
|
+
exists: await pathExists(indexPath)
|
|
5390
|
+
}
|
|
5391
|
+
};
|
|
5392
|
+
}
|
|
5393
|
+
async function doctorWorkspace(location) {
|
|
5394
|
+
const checks = [];
|
|
5395
|
+
const manifestResult = await readWorkspaceManifestFile(location);
|
|
5396
|
+
if (!manifestResult.exists) {
|
|
5397
|
+
checks.push({
|
|
5398
|
+
id: "manifest",
|
|
5399
|
+
status: "fail",
|
|
5400
|
+
message: "workspace.json not found (run `agent-inspect workspace init`)"
|
|
5401
|
+
});
|
|
5402
|
+
return { ok: false, checks };
|
|
5403
|
+
}
|
|
5404
|
+
if (!manifestResult.ok || !manifestResult.manifest) {
|
|
5405
|
+
checks.push({
|
|
5406
|
+
id: "manifest",
|
|
5407
|
+
status: "fail",
|
|
5408
|
+
message: `workspace.json is invalid: ${manifestResult.errors.join("; ")}`
|
|
5409
|
+
});
|
|
5410
|
+
return { ok: false, checks };
|
|
5411
|
+
}
|
|
5412
|
+
const manifest = manifestResult.manifest;
|
|
5413
|
+
checks.push({ id: "manifest", status: "pass", message: "workspace.json is valid" });
|
|
5414
|
+
for (const warning of manifestResult.warnings) {
|
|
5415
|
+
checks.push({ id: "manifest-warning", status: "warn", message: warning });
|
|
5416
|
+
}
|
|
5417
|
+
const dirFields = [
|
|
5418
|
+
...manifest.traceDirs.filter((d) => d !== ".").map((d, i) => [`traceDir[${i}]`, d]),
|
|
5419
|
+
["reportsDir", manifest.reportsDir],
|
|
5420
|
+
["artifactsDir", manifest.artifactsDir],
|
|
5421
|
+
["bundlesDir", manifest.bundlesDir],
|
|
5422
|
+
["notesDir", manifest.notesDir]
|
|
5423
|
+
];
|
|
5424
|
+
for (const [id, rel] of dirFields) {
|
|
5425
|
+
let abs;
|
|
5426
|
+
try {
|
|
5427
|
+
abs = resolveInsideWorkspace(location.workspaceDir, rel);
|
|
5428
|
+
} catch (error) {
|
|
5429
|
+
checks.push({
|
|
5430
|
+
id,
|
|
5431
|
+
status: "fail",
|
|
5432
|
+
message: error instanceof Error ? error.message : String(error)
|
|
5433
|
+
});
|
|
5434
|
+
continue;
|
|
5435
|
+
}
|
|
5436
|
+
if (!await pathExists(abs)) {
|
|
5437
|
+
checks.push({ id, status: "warn", message: `${rel}/ does not exist yet` });
|
|
5438
|
+
} else if (!await isWritable(abs)) {
|
|
5439
|
+
checks.push({ id, status: "fail", message: `${rel}/ is not writable` });
|
|
5440
|
+
} else {
|
|
5441
|
+
checks.push({ id, status: "pass", message: `${rel}/ is present and writable` });
|
|
5442
|
+
}
|
|
5443
|
+
}
|
|
5444
|
+
let newestTraceMtime = 0;
|
|
5445
|
+
for (const rel of manifest.traceDirs) {
|
|
5446
|
+
const abs = resolveInsideWorkspace(location.workspaceDir, rel);
|
|
5447
|
+
for (const file of await listJsonl(abs)) {
|
|
5448
|
+
try {
|
|
5449
|
+
const s = await promises.stat(path13__default.default.join(abs, file));
|
|
5450
|
+
newestTraceMtime = Math.max(newestTraceMtime, s.mtimeMs);
|
|
5451
|
+
} catch {
|
|
5452
|
+
checks.push({ id: "trace-readability", status: "warn", message: `cannot stat ${rel}/${file}` });
|
|
5453
|
+
}
|
|
5454
|
+
}
|
|
5455
|
+
}
|
|
5456
|
+
if (manifest.index.enabled) {
|
|
5457
|
+
const indexPath = manifest.index.path ? resolveInsideWorkspace(location.workspaceDir, manifest.index.path) : resolveInsideWorkspace(location.workspaceDir, INDEX_DIR_NAME);
|
|
5458
|
+
if (!await pathExists(indexPath)) {
|
|
5459
|
+
checks.push({ id: "index", status: "warn", message: "index enabled but not built" });
|
|
5460
|
+
} else {
|
|
5461
|
+
try {
|
|
5462
|
+
const s = await promises.stat(indexPath);
|
|
5463
|
+
if (newestTraceMtime > s.mtimeMs) {
|
|
5464
|
+
checks.push({ id: "index", status: "warn", message: "index is stale (traces are newer)" });
|
|
5465
|
+
} else {
|
|
5466
|
+
checks.push({ id: "index", status: "pass", message: "index is present" });
|
|
5467
|
+
}
|
|
5468
|
+
} catch {
|
|
5469
|
+
checks.push({ id: "index", status: "warn", message: "cannot stat index" });
|
|
5470
|
+
}
|
|
5471
|
+
}
|
|
5472
|
+
}
|
|
5473
|
+
const ok = !checks.some((c) => c.status === "fail");
|
|
5474
|
+
return { ok, checks };
|
|
5475
|
+
}
|
|
5476
|
+
|
|
5477
|
+
// packages/viewer/src/workspace-data.ts
|
|
5478
|
+
async function loadWorkspaceViewerData(options) {
|
|
5479
|
+
const cwd = path13__default.default.resolve(options.cwd ?? process.cwd());
|
|
5480
|
+
const location = resolveWorkspaceLocation(cwd);
|
|
5481
|
+
const manifestRead = await readWorkspaceManifestFile(location);
|
|
5482
|
+
if (!manifestRead.ok || manifestRead.manifest === void 0) {
|
|
5483
|
+
throw new Error(
|
|
5484
|
+
manifestRead.errors.join("; ") || "workspace.json not found (run workspace init)"
|
|
5485
|
+
);
|
|
5486
|
+
}
|
|
5487
|
+
const status = await getWorkspaceStatus(location, manifestRead.manifest);
|
|
5488
|
+
const doctor = await doctorWorkspace(location);
|
|
5489
|
+
const runs = [];
|
|
5490
|
+
for (const rel of manifestRead.manifest.traceDirs) {
|
|
5491
|
+
const traceDir = resolveTraceDir({
|
|
5492
|
+
dir: path13__default.default.join(location.workspaceDir, rel)
|
|
5493
|
+
});
|
|
5494
|
+
const td = new TraceDirectory({ dir: traceDir });
|
|
5495
|
+
const files = await td.list();
|
|
5496
|
+
const metas = await loadTraceMetadataList(
|
|
5497
|
+
traceDir,
|
|
5498
|
+
files,
|
|
5499
|
+
(fileName) => td.getPath(fileName)
|
|
5500
|
+
);
|
|
5501
|
+
for (const meta of metas) {
|
|
5502
|
+
runs.push({
|
|
5503
|
+
runId: meta.runId,
|
|
5504
|
+
...meta.name !== void 0 ? { name: meta.name } : {},
|
|
5505
|
+
status: meta.status,
|
|
5506
|
+
file: path13__default.default.basename(meta.filePath)
|
|
5507
|
+
});
|
|
5508
|
+
}
|
|
5509
|
+
}
|
|
5510
|
+
return {
|
|
5511
|
+
workspaceDir: location.workspaceDir,
|
|
5512
|
+
...status.project !== void 0 ? { project: status.project } : {},
|
|
5513
|
+
status,
|
|
5514
|
+
doctor,
|
|
5515
|
+
runs,
|
|
5516
|
+
bundleDirs: [
|
|
5517
|
+
path13__default.default.join(location.workspaceDir, manifestRead.manifest.bundlesDir),
|
|
5518
|
+
path13__default.default.join(location.workspaceDir, manifestRead.manifest.artifactsDir)
|
|
5519
|
+
]
|
|
5520
|
+
};
|
|
5521
|
+
}
|
|
5522
|
+
|
|
3901
5523
|
// packages/viewer/src/server.ts
|
|
3902
5524
|
var DEFAULT_HOST = "127.0.0.1";
|
|
3903
5525
|
var DEFAULT_PORT = 7337;
|
|
@@ -3933,6 +5555,7 @@ function createViewerServer(options = {}) {
|
|
|
3933
5555
|
const host = options.host ?? DEFAULT_HOST;
|
|
3934
5556
|
const port = options.port ?? DEFAULT_PORT;
|
|
3935
5557
|
const maxEvents = options.maxEvents ?? DEFAULT_MAX_EVENTS;
|
|
5558
|
+
const mode = options.mode ?? "traces";
|
|
3936
5559
|
if (host === "0.0.0.0") {
|
|
3937
5560
|
console.warn(
|
|
3938
5561
|
"[AgentInspect viewer] Binding to 0.0.0.0 exposes traces on the network. Use 127.0.0.1 unless you accept that risk."
|
|
@@ -3954,9 +5577,21 @@ function createViewerServer(options = {}) {
|
|
|
3954
5577
|
return sendJson(res, 200, {
|
|
3955
5578
|
ok: true,
|
|
3956
5579
|
readOnly: true,
|
|
3957
|
-
|
|
5580
|
+
mode,
|
|
5581
|
+
traceDir: path13__default.default.resolve(traceDir)
|
|
3958
5582
|
});
|
|
3959
5583
|
}
|
|
5584
|
+
if (pathname === "/api/suite" && mode === "suite") {
|
|
5585
|
+
const data = await loadSuiteViewerData({
|
|
5586
|
+
suiteConfigPath: options.suiteConfigPath,
|
|
5587
|
+
cwd: options.cwd
|
|
5588
|
+
});
|
|
5589
|
+
return sendJson(res, 200, data);
|
|
5590
|
+
}
|
|
5591
|
+
if (pathname === "/api/workspace" && mode === "workspace") {
|
|
5592
|
+
const data = await loadWorkspaceViewerData({ cwd: options.cwd });
|
|
5593
|
+
return sendJson(res, 200, data);
|
|
5594
|
+
}
|
|
3960
5595
|
const td = new TraceDirectory({ dir: traceDir });
|
|
3961
5596
|
if (pathname === "/api/traces") {
|
|
3962
5597
|
const files = await td.list();
|
|
@@ -3972,7 +5607,7 @@ function createViewerServer(options = {}) {
|
|
|
3972
5607
|
runId: meta.runId,
|
|
3973
5608
|
name: meta.name,
|
|
3974
5609
|
status: meta.status,
|
|
3975
|
-
file:
|
|
5610
|
+
file: path13__default.default.basename(meta.filePath),
|
|
3976
5611
|
startedAt: meta.startedAt,
|
|
3977
5612
|
durationMs: meta.durationMs
|
|
3978
5613
|
}))
|
|
@@ -4078,23 +5713,28 @@ function startViewerServer(options = {}) {
|
|
|
4078
5713
|
const host = options.host ?? DEFAULT_HOST;
|
|
4079
5714
|
const port = options.port ?? DEFAULT_PORT;
|
|
4080
5715
|
const traceDir = resolveTraceDir({ dir: options.traceDir });
|
|
5716
|
+
const mode = options.mode ?? "traces";
|
|
4081
5717
|
const server = createViewerServer(options);
|
|
4082
5718
|
return new Promise((resolve, reject) => {
|
|
4083
5719
|
server.once("error", reject);
|
|
4084
5720
|
server.listen(port, host, () => {
|
|
4085
5721
|
const address = server.address();
|
|
4086
5722
|
const resolvedPort = typeof address === "object" && address ? address.port : port;
|
|
5723
|
+
const modeQuery = mode === "traces" ? "" : `?mode=${encodeURIComponent(mode)}`;
|
|
4087
5724
|
resolve({
|
|
4088
5725
|
host,
|
|
4089
5726
|
port: resolvedPort,
|
|
4090
|
-
traceDir:
|
|
4091
|
-
url: `http://${host}:${resolvedPort}
|
|
5727
|
+
traceDir: path13__default.default.resolve(traceDir),
|
|
5728
|
+
url: `http://${host}:${resolvedPort}/${modeQuery}`,
|
|
5729
|
+
mode
|
|
4092
5730
|
});
|
|
4093
5731
|
});
|
|
4094
5732
|
});
|
|
4095
5733
|
}
|
|
4096
5734
|
|
|
4097
5735
|
exports.createViewerServer = createViewerServer;
|
|
5736
|
+
exports.loadSuiteViewerData = loadSuiteViewerData;
|
|
5737
|
+
exports.loadWorkspaceViewerData = loadWorkspaceViewerData;
|
|
4098
5738
|
exports.startViewerServer = startViewerServer;
|
|
4099
5739
|
exports.viewerIndexHtml = viewerIndexHtml;
|
|
4100
5740
|
//# sourceMappingURL=index.cjs.map
|