@agent-inspect/viewer 5.2.0 → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1672 -32
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +404 -2
- package/dist/index.d.ts +404 -2
- package/dist/index.mjs +1671 -33
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { createServer } from 'http';
|
|
2
|
-
import
|
|
2
|
+
import path13 from 'path';
|
|
3
3
|
import { AsyncLocalStorage } from 'async_hooks';
|
|
4
4
|
import 'crypto';
|
|
5
|
-
import {
|
|
5
|
+
import { readFile, stat, readdir, access } from 'fs/promises';
|
|
6
6
|
import os from 'os';
|
|
7
7
|
import 'nanoid';
|
|
8
8
|
import 'chalk';
|
|
9
|
-
import 'fs';
|
|
9
|
+
import { constants } from 'fs';
|
|
10
10
|
import 'readline';
|
|
11
|
-
import 'url';
|
|
11
|
+
import { pathToFileURL } from 'url';
|
|
12
12
|
|
|
13
13
|
// packages/viewer/src/server.ts
|
|
14
14
|
|
|
@@ -518,7 +518,7 @@ function persistedInspectEventsToTraceEvents(events, options) {
|
|
|
518
518
|
}
|
|
519
519
|
var DEFAULT_TRACE_DIR_NAME = ".agent-inspect";
|
|
520
520
|
var RUNS_DIR_NAME = "runs";
|
|
521
|
-
var FALLBACK_TRACE_DIR =
|
|
521
|
+
var FALLBACK_TRACE_DIR = path13.join(
|
|
522
522
|
os.tmpdir(),
|
|
523
523
|
"agent-inspect",
|
|
524
524
|
RUNS_DIR_NAME
|
|
@@ -533,11 +533,20 @@ function getDefaultTraceDir() {
|
|
|
533
533
|
if (typeof home !== "string" || home.trim() === "") {
|
|
534
534
|
return FALLBACK_TRACE_DIR;
|
|
535
535
|
}
|
|
536
|
-
return
|
|
536
|
+
return path13.join(home, DEFAULT_TRACE_DIR_NAME, RUNS_DIR_NAME);
|
|
537
537
|
} catch {
|
|
538
538
|
return FALLBACK_TRACE_DIR;
|
|
539
539
|
}
|
|
540
540
|
}
|
|
541
|
+
function getTraceFilePath(runId, traceDir) {
|
|
542
|
+
const baseDir = traceDir ?? getDefaultTraceDir();
|
|
543
|
+
let safeId = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "run_unknown";
|
|
544
|
+
safeId = path13.basename(safeId);
|
|
545
|
+
if (safeId === "" || safeId === "." || safeId === "..") {
|
|
546
|
+
safeId = "run_unknown";
|
|
547
|
+
}
|
|
548
|
+
return path13.join(baseDir, `${safeId}.jsonl`);
|
|
549
|
+
}
|
|
541
550
|
function formatError(error) {
|
|
542
551
|
if (error instanceof Error) {
|
|
543
552
|
const out = { message: error.message };
|
|
@@ -726,6 +735,78 @@ async function readTraceEventsFromFile(filePath) {
|
|
|
726
735
|
|
|
727
736
|
// packages/core/src/context.ts
|
|
728
737
|
new AsyncLocalStorage();
|
|
738
|
+
|
|
739
|
+
// packages/core/src/outcomes/types.ts
|
|
740
|
+
var OUTCOME_ATTRIBUTE_STATUS_KEY = "outcomeStatus";
|
|
741
|
+
var OUTCOME_ATTRIBUTE_EXPECTATION_KEY = "expectation";
|
|
742
|
+
var OUTCOME_ATTRIBUTE_METHOD_KEY = "method";
|
|
743
|
+
var OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY = "observedAt";
|
|
744
|
+
var OUTCOME_LEGACY_EVENT = "outcome_observed";
|
|
745
|
+
|
|
746
|
+
// packages/core/src/outcomes/extract.ts
|
|
747
|
+
function isOutcomeStatus(value) {
|
|
748
|
+
return value === "passed" || value === "failed" || value === "unknown" || value === "skipped";
|
|
749
|
+
}
|
|
750
|
+
function parseMethod(value) {
|
|
751
|
+
if (typeof value !== "string" || value.trim() === "") return void 0;
|
|
752
|
+
return value;
|
|
753
|
+
}
|
|
754
|
+
function fromOutcomeObservedEvent(event) {
|
|
755
|
+
return {
|
|
756
|
+
outcomeId: event.outcomeId,
|
|
757
|
+
runId: event.runId,
|
|
758
|
+
...event.parentId !== void 0 ? { parentId: event.parentId } : {},
|
|
759
|
+
name: event.name,
|
|
760
|
+
expectation: event.expectation,
|
|
761
|
+
status: event.status,
|
|
762
|
+
...event.method !== void 0 ? { method: event.method } : {},
|
|
763
|
+
...event.actual !== void 0 ? { actual: event.actual } : {},
|
|
764
|
+
...event.evidence !== void 0 ? { evidence: event.evidence } : {},
|
|
765
|
+
observedAt: event.observedAt
|
|
766
|
+
};
|
|
767
|
+
}
|
|
768
|
+
function fromPersistedOutcome(event) {
|
|
769
|
+
if (event.kind !== "OUTCOME") return void 0;
|
|
770
|
+
const attrs = event.attributes ?? {};
|
|
771
|
+
const statusRaw = attrs[OUTCOME_ATTRIBUTE_STATUS_KEY];
|
|
772
|
+
if (!isOutcomeStatus(statusRaw)) return void 0;
|
|
773
|
+
const expectation = typeof attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] === "string" ? attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] : event.name;
|
|
774
|
+
const observedAtRaw = attrs[OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY];
|
|
775
|
+
const observedAt = typeof observedAtRaw === "string" ? Date.parse(observedAtRaw) : typeof observedAtRaw === "number" && Number.isFinite(observedAtRaw) ? observedAtRaw : Date.parse(event.timestamp);
|
|
776
|
+
return {
|
|
777
|
+
outcomeId: event.eventId,
|
|
778
|
+
runId: event.runId,
|
|
779
|
+
...event.parentId !== void 0 ? { parentId: event.parentId } : {},
|
|
780
|
+
name: event.name,
|
|
781
|
+
expectation,
|
|
782
|
+
status: statusRaw,
|
|
783
|
+
...parseMethod(attrs[OUTCOME_ATTRIBUTE_METHOD_KEY]) !== void 0 ? { method: parseMethod(attrs[OUTCOME_ATTRIBUTE_METHOD_KEY]) } : {},
|
|
784
|
+
...event.outputSummary !== void 0 ? { actual: event.outputSummary } : {},
|
|
785
|
+
...attrs.evidence !== void 0 ? { evidence: attrs.evidence } : {},
|
|
786
|
+
observedAt: Number.isFinite(observedAt) ? observedAt : Date.parse(event.timestamp)
|
|
787
|
+
};
|
|
788
|
+
}
|
|
789
|
+
function extractOutcomesFromTraceEvents(events) {
|
|
790
|
+
const out = [];
|
|
791
|
+
for (const event of events) {
|
|
792
|
+
if (event.event === OUTCOME_LEGACY_EVENT) {
|
|
793
|
+
out.push(fromOutcomeObservedEvent(event));
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
return out.sort((a, b) => a.observedAt - b.observedAt || a.name.localeCompare(b.name));
|
|
797
|
+
}
|
|
798
|
+
function extractOutcomesFromPersistedEvents(events) {
|
|
799
|
+
const out = [];
|
|
800
|
+
for (const event of events) {
|
|
801
|
+
const outcome = fromPersistedOutcome(event);
|
|
802
|
+
if (outcome) out.push(outcome);
|
|
803
|
+
}
|
|
804
|
+
return out.sort((a, b) => a.observedAt - b.observedAt || a.name.localeCompare(b.name));
|
|
805
|
+
}
|
|
806
|
+
function outcomesMatchingStatus(outcomes, statuses) {
|
|
807
|
+
const set = new Set(statuses);
|
|
808
|
+
return outcomes.filter((outcome) => set.has(outcome.status));
|
|
809
|
+
}
|
|
729
810
|
function resolveTraceDir(options = {}) {
|
|
730
811
|
if (typeof options.dir === "string" && options.dir.trim() !== "") {
|
|
731
812
|
return options.dir.trim();
|
|
@@ -742,7 +823,7 @@ var TraceDirectory = class {
|
|
|
742
823
|
this.#dir = resolveTraceDir(options);
|
|
743
824
|
}
|
|
744
825
|
getPath(filename) {
|
|
745
|
-
return filename ?
|
|
826
|
+
return filename ? path13.join(this.#dir, filename) : this.#dir;
|
|
746
827
|
}
|
|
747
828
|
async list() {
|
|
748
829
|
try {
|
|
@@ -769,7 +850,7 @@ function parseIsoToMs2(value) {
|
|
|
769
850
|
}
|
|
770
851
|
async function extractMetadata(filePath, _quickScan) {
|
|
771
852
|
const stats = await stat(filePath);
|
|
772
|
-
let runIdFromFile =
|
|
853
|
+
let runIdFromFile = path13.basename(filePath);
|
|
773
854
|
if (runIdFromFile.endsWith(".jsonl")) {
|
|
774
855
|
runIdFromFile = runIdFromFile.slice(0, -".jsonl".length);
|
|
775
856
|
}
|
|
@@ -1379,12 +1460,12 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
1379
1460
|
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
1380
1461
|
);
|
|
1381
1462
|
const ordered = [...runs].sort(compareRuns);
|
|
1382
|
-
const
|
|
1463
|
+
const path16 = [];
|
|
1383
1464
|
const visited = /* @__PURE__ */ new Set();
|
|
1384
1465
|
const pushRun = (run, confidence, source) => {
|
|
1385
1466
|
if (visited.has(run.runId)) return;
|
|
1386
1467
|
visited.add(run.runId);
|
|
1387
|
-
|
|
1468
|
+
path16.push({
|
|
1388
1469
|
runId: run.runId,
|
|
1389
1470
|
name: run.name,
|
|
1390
1471
|
startedAt: run.startedAt,
|
|
@@ -1409,7 +1490,7 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
1409
1490
|
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
1410
1491
|
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
1411
1492
|
}
|
|
1412
|
-
return
|
|
1493
|
+
return path16;
|
|
1413
1494
|
}
|
|
1414
1495
|
function metaRunIdMatches(run, token, runById) {
|
|
1415
1496
|
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
@@ -1486,6 +1567,264 @@ function buildSessionIndex(inputRuns, options = {}) {
|
|
|
1486
1567
|
};
|
|
1487
1568
|
}
|
|
1488
1569
|
|
|
1570
|
+
// packages/core/src/suite/types.ts
|
|
1571
|
+
var DEFAULT_SUITE_CONFIG_NAMES = [
|
|
1572
|
+
"agent-inspect.suite.json",
|
|
1573
|
+
"agent-inspect.suite.js",
|
|
1574
|
+
"agent-inspect.suite.mjs",
|
|
1575
|
+
"agent-inspect.suite.cjs"
|
|
1576
|
+
];
|
|
1577
|
+
function diagnostic(code, message, severity = "error", caseId) {
|
|
1578
|
+
return { code, message, severity, ...{} };
|
|
1579
|
+
}
|
|
1580
|
+
function asString(value, label) {
|
|
1581
|
+
if (value === void 0) return void 0;
|
|
1582
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
1583
|
+
throw new Error(`${label} must be a non-empty string.`);
|
|
1584
|
+
}
|
|
1585
|
+
return value.trim();
|
|
1586
|
+
}
|
|
1587
|
+
function asStringArray(value, label) {
|
|
1588
|
+
if (value === void 0) return void 0;
|
|
1589
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
1590
|
+
throw new Error(`${label} must be an array of strings.`);
|
|
1591
|
+
}
|
|
1592
|
+
return value;
|
|
1593
|
+
}
|
|
1594
|
+
function asPositiveNumber(value, label) {
|
|
1595
|
+
if (value === void 0) return void 0;
|
|
1596
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
1597
|
+
throw new Error(`${label} must be a non-negative number.`);
|
|
1598
|
+
}
|
|
1599
|
+
return value;
|
|
1600
|
+
}
|
|
1601
|
+
function validateCaseConfig(value, index) {
|
|
1602
|
+
const diagnostics = [];
|
|
1603
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
1604
|
+
diagnostics.push(
|
|
1605
|
+
diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
|
|
1606
|
+
);
|
|
1607
|
+
return { diagnostics };
|
|
1608
|
+
}
|
|
1609
|
+
const raw = value;
|
|
1610
|
+
try {
|
|
1611
|
+
const id = asString(raw.id, `cases[${index}].id`);
|
|
1612
|
+
if (id === void 0) {
|
|
1613
|
+
diagnostics.push(
|
|
1614
|
+
diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
|
|
1615
|
+
);
|
|
1616
|
+
return { diagnostics };
|
|
1617
|
+
}
|
|
1618
|
+
const trace = asString(raw.trace, `cases[${index}].trace`);
|
|
1619
|
+
const runId = asString(raw.runId, `cases[${index}].runId`);
|
|
1620
|
+
const input = asString(raw.input, `cases[${index}].input`);
|
|
1621
|
+
return {
|
|
1622
|
+
caseConfig: {
|
|
1623
|
+
id,
|
|
1624
|
+
...trace !== void 0 ? { trace } : {},
|
|
1625
|
+
...runId !== void 0 ? { runId } : {},
|
|
1626
|
+
...input !== void 0 ? { input } : {},
|
|
1627
|
+
...asStringArray(raw.requireTools, `cases[${index}].requireTools`) !== void 0 ? { requireTools: asStringArray(raw.requireTools, `cases[${index}].requireTools`) } : {},
|
|
1628
|
+
...asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) !== void 0 ? { forbidTools: asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) } : {},
|
|
1629
|
+
...asPositiveNumber(raw.maxDurationMs, `cases[${index}].maxDurationMs`) !== void 0 ? {
|
|
1630
|
+
maxDurationMs: asPositiveNumber(
|
|
1631
|
+
raw.maxDurationMs,
|
|
1632
|
+
`cases[${index}].maxDurationMs`
|
|
1633
|
+
)
|
|
1634
|
+
} : {},
|
|
1635
|
+
...asStringArray(
|
|
1636
|
+
raw.expectedObservations,
|
|
1637
|
+
`cases[${index}].expectedObservations`
|
|
1638
|
+
) !== void 0 ? {
|
|
1639
|
+
expectedObservations: asStringArray(
|
|
1640
|
+
raw.expectedObservations,
|
|
1641
|
+
`cases[${index}].expectedObservations`
|
|
1642
|
+
)
|
|
1643
|
+
} : {}
|
|
1644
|
+
},
|
|
1645
|
+
diagnostics
|
|
1646
|
+
};
|
|
1647
|
+
} catch (error) {
|
|
1648
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1649
|
+
diagnostics.push(diagnostic("AI_SUITE_CONFIG_INVALID", message));
|
|
1650
|
+
return { diagnostics };
|
|
1651
|
+
}
|
|
1652
|
+
}
|
|
1653
|
+
function normalizeSuiteConfig(value) {
|
|
1654
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
1655
|
+
throw new Error("Suite config must export an object.");
|
|
1656
|
+
}
|
|
1657
|
+
const raw = value;
|
|
1658
|
+
const name = asString(raw.name, "name");
|
|
1659
|
+
const traces = asString(raw.traces, "traces");
|
|
1660
|
+
if (name === void 0) throw new Error("name is required.");
|
|
1661
|
+
if (traces === void 0) throw new Error("traces is required.");
|
|
1662
|
+
if (!Array.isArray(raw.cases) || raw.cases.length === 0) {
|
|
1663
|
+
throw new Error("cases must be a non-empty array.");
|
|
1664
|
+
}
|
|
1665
|
+
const cases = [];
|
|
1666
|
+
for (let index = 0; index < raw.cases.length; index += 1) {
|
|
1667
|
+
const { caseConfig, diagnostics } = validateCaseConfig(raw.cases[index], index);
|
|
1668
|
+
if (diagnostics.length > 0) {
|
|
1669
|
+
throw new Error(diagnostics.map((item) => item.message).join("; "));
|
|
1670
|
+
}
|
|
1671
|
+
if (caseConfig !== void 0) cases.push(caseConfig);
|
|
1672
|
+
}
|
|
1673
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1674
|
+
for (const suiteCase of cases) {
|
|
1675
|
+
if (ids.has(suiteCase.id)) {
|
|
1676
|
+
throw new Error(`Duplicate case id "${suiteCase.id}".`);
|
|
1677
|
+
}
|
|
1678
|
+
ids.add(suiteCase.id);
|
|
1679
|
+
}
|
|
1680
|
+
const redactionProfile = raw.redactionProfile === "local" || raw.redactionProfile === "share" || raw.redactionProfile === "strict" ? raw.redactionProfile : void 0;
|
|
1681
|
+
if (raw.redactionProfile !== void 0 && redactionProfile === void 0) {
|
|
1682
|
+
throw new Error('redactionProfile must be "local", "share", or "strict".');
|
|
1683
|
+
}
|
|
1684
|
+
const config = { name, traces, cases };
|
|
1685
|
+
if (raw.checks !== void 0) {
|
|
1686
|
+
if (typeof raw.checks !== "object" || Array.isArray(raw.checks)) {
|
|
1687
|
+
throw new Error("checks must be an object.");
|
|
1688
|
+
}
|
|
1689
|
+
config.checks = raw.checks;
|
|
1690
|
+
}
|
|
1691
|
+
if (raw.eval !== void 0) {
|
|
1692
|
+
if (typeof raw.eval !== "object" || Array.isArray(raw.eval)) {
|
|
1693
|
+
throw new Error("eval must be an object.");
|
|
1694
|
+
}
|
|
1695
|
+
config.eval = raw.eval;
|
|
1696
|
+
}
|
|
1697
|
+
if (raw.artifacts !== void 0) {
|
|
1698
|
+
if (typeof raw.artifacts !== "object" || Array.isArray(raw.artifacts)) {
|
|
1699
|
+
throw new Error("artifacts must be an object.");
|
|
1700
|
+
}
|
|
1701
|
+
const outputDir = asString(
|
|
1702
|
+
raw.artifacts.outputDir,
|
|
1703
|
+
"artifacts.outputDir"
|
|
1704
|
+
);
|
|
1705
|
+
config.artifacts = outputDir !== void 0 ? { outputDir } : {};
|
|
1706
|
+
}
|
|
1707
|
+
if (raw.baseline !== void 0) {
|
|
1708
|
+
const baseline = asString(raw.baseline, "baseline");
|
|
1709
|
+
if (baseline !== void 0) config.baseline = baseline;
|
|
1710
|
+
}
|
|
1711
|
+
if (raw.candidate !== void 0) {
|
|
1712
|
+
const candidate = asString(raw.candidate, "candidate");
|
|
1713
|
+
if (candidate !== void 0) config.candidate = candidate;
|
|
1714
|
+
}
|
|
1715
|
+
if (redactionProfile !== void 0) config.redactionProfile = redactionProfile;
|
|
1716
|
+
return config;
|
|
1717
|
+
}
|
|
1718
|
+
|
|
1719
|
+
// packages/core/src/suite/load.ts
|
|
1720
|
+
var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
1721
|
+
var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
1722
|
+
function diagnostic2(code, message) {
|
|
1723
|
+
return { code, message, severity: "error" };
|
|
1724
|
+
}
|
|
1725
|
+
async function fileExists(filePath) {
|
|
1726
|
+
try {
|
|
1727
|
+
await access(filePath);
|
|
1728
|
+
return true;
|
|
1729
|
+
} catch {
|
|
1730
|
+
return false;
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
async function resolveSuiteConfigPath(options = {}) {
|
|
1734
|
+
const cwd = path13.resolve(options.cwd ?? process.cwd());
|
|
1735
|
+
if (options.configPath !== void 0 && options.configPath.trim() !== "") {
|
|
1736
|
+
return path13.resolve(cwd, options.configPath.trim());
|
|
1737
|
+
}
|
|
1738
|
+
for (const name of DEFAULT_SUITE_CONFIG_NAMES) {
|
|
1739
|
+
const candidate = path13.join(cwd, name);
|
|
1740
|
+
if (await fileExists(candidate)) return candidate;
|
|
1741
|
+
}
|
|
1742
|
+
throw new Error(
|
|
1743
|
+
`No suite config found. Create one with \`agent-inspect suite init\` or pass --config.`
|
|
1744
|
+
);
|
|
1745
|
+
}
|
|
1746
|
+
async function loadSuiteConfig(options = {}) {
|
|
1747
|
+
let configPath;
|
|
1748
|
+
try {
|
|
1749
|
+
configPath = await resolveSuiteConfigPath(options);
|
|
1750
|
+
} catch (error) {
|
|
1751
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1752
|
+
throw Object.assign(new Error(message), {
|
|
1753
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
1754
|
+
});
|
|
1755
|
+
}
|
|
1756
|
+
const extension = path13.extname(configPath);
|
|
1757
|
+
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
1758
|
+
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
1759
|
+
throw Object.assign(new Error(message), {
|
|
1760
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
1761
|
+
});
|
|
1762
|
+
}
|
|
1763
|
+
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
1764
|
+
const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
|
|
1765
|
+
throw Object.assign(new Error(message), {
|
|
1766
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
1767
|
+
});
|
|
1768
|
+
}
|
|
1769
|
+
try {
|
|
1770
|
+
let raw;
|
|
1771
|
+
if (extension === ".json") {
|
|
1772
|
+
raw = JSON.parse(await readFile(configPath, "utf-8"));
|
|
1773
|
+
} else {
|
|
1774
|
+
const mod = await import(pathToFileURL(configPath).href);
|
|
1775
|
+
raw = "default" in mod ? mod.default : mod;
|
|
1776
|
+
}
|
|
1777
|
+
const config = normalizeSuiteConfig(raw);
|
|
1778
|
+
return {
|
|
1779
|
+
config,
|
|
1780
|
+
configPath,
|
|
1781
|
+
configDir: path13.dirname(configPath)
|
|
1782
|
+
};
|
|
1783
|
+
} catch (error) {
|
|
1784
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1785
|
+
throw Object.assign(new Error(message), {
|
|
1786
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
1787
|
+
});
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
async function exists(filePath) {
|
|
1791
|
+
try {
|
|
1792
|
+
await access(filePath);
|
|
1793
|
+
return true;
|
|
1794
|
+
} catch {
|
|
1795
|
+
return false;
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
async function resolveSuiteCaseTrace(suiteCase, options) {
|
|
1799
|
+
if (suiteCase.trace !== void 0) {
|
|
1800
|
+
const tracePath = path13.resolve(options.configDir, suiteCase.trace);
|
|
1801
|
+
if (await exists(tracePath)) {
|
|
1802
|
+
return { caseId: suiteCase.id, tracePath, missing: false };
|
|
1803
|
+
}
|
|
1804
|
+
return {
|
|
1805
|
+
caseId: suiteCase.id,
|
|
1806
|
+
tracePath,
|
|
1807
|
+
missing: true,
|
|
1808
|
+
reason: `trace file not found: ${suiteCase.trace}`
|
|
1809
|
+
};
|
|
1810
|
+
}
|
|
1811
|
+
const runKey = suiteCase.runId ?? suiteCase.id;
|
|
1812
|
+
const directPath = getTraceFilePath(runKey, options.tracesDir);
|
|
1813
|
+
if (await exists(directPath)) {
|
|
1814
|
+
return { caseId: suiteCase.id, tracePath: directPath, runId: runKey, missing: false };
|
|
1815
|
+
}
|
|
1816
|
+
const nestedPath = path13.join(options.tracesDir, `${path13.basename(runKey)}.jsonl`);
|
|
1817
|
+
if (await exists(nestedPath)) {
|
|
1818
|
+
return { caseId: suiteCase.id, tracePath: nestedPath, runId: runKey, missing: false };
|
|
1819
|
+
}
|
|
1820
|
+
return {
|
|
1821
|
+
caseId: suiteCase.id,
|
|
1822
|
+
runId: runKey,
|
|
1823
|
+
missing: true,
|
|
1824
|
+
reason: `no trace found for run id "${runKey}" under ${options.tracesDir}`
|
|
1825
|
+
};
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1489
1828
|
// packages/core/src/checks/index.ts
|
|
1490
1829
|
var SEVERITY_RANK = {
|
|
1491
1830
|
error: 0,
|
|
@@ -1500,7 +1839,7 @@ var STATUS_RANK = {
|
|
|
1500
1839
|
function compareStrings(a, b) {
|
|
1501
1840
|
return (a ?? "").localeCompare(b ?? "");
|
|
1502
1841
|
}
|
|
1503
|
-
function
|
|
1842
|
+
function diagnostic3(code, message, ruleId) {
|
|
1504
1843
|
return {
|
|
1505
1844
|
code,
|
|
1506
1845
|
message,
|
|
@@ -1566,7 +1905,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
1566
1905
|
if (runId && input.selectedRun.runId !== runId) {
|
|
1567
1906
|
return {
|
|
1568
1907
|
diagnostics: [
|
|
1569
|
-
|
|
1908
|
+
diagnostic3(
|
|
1570
1909
|
"AI_CHECK_INVALID_ARGUMENTS",
|
|
1571
1910
|
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
1572
1911
|
)
|
|
@@ -1580,7 +1919,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
1580
1919
|
if (!run) {
|
|
1581
1920
|
return {
|
|
1582
1921
|
diagnostics: [
|
|
1583
|
-
|
|
1922
|
+
diagnostic3("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
1584
1923
|
]
|
|
1585
1924
|
};
|
|
1586
1925
|
}
|
|
@@ -1592,13 +1931,13 @@ function resolveSelectedRun(input, runId) {
|
|
|
1592
1931
|
if (input.read.runs.length === 0) {
|
|
1593
1932
|
return {
|
|
1594
1933
|
diagnostics: [
|
|
1595
|
-
|
|
1934
|
+
diagnostic3("AI_CHECK_RUN_SELECTION_REQUIRED", "No runs are available for checks.")
|
|
1596
1935
|
]
|
|
1597
1936
|
};
|
|
1598
1937
|
}
|
|
1599
1938
|
return {
|
|
1600
1939
|
diagnostics: [
|
|
1601
|
-
|
|
1940
|
+
diagnostic3(
|
|
1602
1941
|
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
1603
1942
|
"Multiple runs are available; select a run before executing checks."
|
|
1604
1943
|
)
|
|
@@ -1611,7 +1950,7 @@ function selectRules(rules, selectedIds) {
|
|
|
1611
1950
|
for (const rule of rules) {
|
|
1612
1951
|
if (byId.has(rule.id)) {
|
|
1613
1952
|
diagnostics.push(
|
|
1614
|
-
|
|
1953
|
+
diagnostic3("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
1615
1954
|
);
|
|
1616
1955
|
continue;
|
|
1617
1956
|
}
|
|
@@ -1622,7 +1961,7 @@ function selectRules(rules, selectedIds) {
|
|
|
1622
1961
|
for (const id of selected) {
|
|
1623
1962
|
if (!byId.has(id)) {
|
|
1624
1963
|
diagnostics.push(
|
|
1625
|
-
|
|
1964
|
+
diagnostic3("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
1626
1965
|
);
|
|
1627
1966
|
}
|
|
1628
1967
|
}
|
|
@@ -1682,7 +2021,20 @@ function summarize(findings, diagnostics) {
|
|
|
1682
2021
|
errors: diagnostics.filter((item) => item.severity === "error").length
|
|
1683
2022
|
};
|
|
1684
2023
|
}
|
|
1685
|
-
function
|
|
2024
|
+
function stringAttr(event, keys) {
|
|
2025
|
+
for (const key of keys) {
|
|
2026
|
+
const value = event.attributes?.[key];
|
|
2027
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2028
|
+
}
|
|
2029
|
+
return void 0;
|
|
2030
|
+
}
|
|
2031
|
+
function stripPrefix(name, prefixes) {
|
|
2032
|
+
for (const prefix of prefixes) {
|
|
2033
|
+
if (name.startsWith(prefix)) return name.slice(prefix.length);
|
|
2034
|
+
}
|
|
2035
|
+
return name;
|
|
2036
|
+
}
|
|
2037
|
+
function eventEvidence(event, path16) {
|
|
1686
2038
|
return {
|
|
1687
2039
|
runId: event.runId,
|
|
1688
2040
|
eventId: event.eventId,
|
|
@@ -1692,7 +2044,7 @@ function eventEvidence(event, path13) {
|
|
|
1692
2044
|
kind: event.kind,
|
|
1693
2045
|
name: event.name,
|
|
1694
2046
|
status: event.status,
|
|
1695
|
-
...{}
|
|
2047
|
+
...path16 ? { path: path16 } : {}
|
|
1696
2048
|
};
|
|
1697
2049
|
}
|
|
1698
2050
|
function runEvidence(run) {
|
|
@@ -1709,6 +2061,23 @@ function failFinding(ruleId, message, evidence, expected, actual) {
|
|
|
1709
2061
|
evidence: [...evidence]
|
|
1710
2062
|
};
|
|
1711
2063
|
}
|
|
2064
|
+
function toolName(event) {
|
|
2065
|
+
return stringAttr(event, ["toolName", "tool"]) ?? stripPrefix(event.name, ["tool:", "function:", "mcp-tools:"]);
|
|
2066
|
+
}
|
|
2067
|
+
function llmModel(event) {
|
|
2068
|
+
return stringAttr(event, ["model", "modelId", "responseModelId", "modelName", "model_name"]) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
2069
|
+
}
|
|
2070
|
+
function llmProvider(event) {
|
|
2071
|
+
return stringAttr(event, ["provider", "providerName", "provider_name"]);
|
|
2072
|
+
}
|
|
2073
|
+
function llmFinishReason(event) {
|
|
2074
|
+
return stringAttr(event, ["finishReason", "rawFinishReason", "finish_reason"]);
|
|
2075
|
+
}
|
|
2076
|
+
function finishedEvents(context, kind) {
|
|
2077
|
+
return context.events.filter(
|
|
2078
|
+
(event) => (kind === void 0 || event.kind === kind) && event.status !== "running"
|
|
2079
|
+
);
|
|
2080
|
+
}
|
|
1712
2081
|
function createRunStatusRule(options = {}) {
|
|
1713
2082
|
const expected = options.expected ?? "ok";
|
|
1714
2083
|
const allowIncomplete = options.allowIncomplete === true;
|
|
@@ -1748,6 +2117,180 @@ function createRunStatusRule(options = {}) {
|
|
|
1748
2117
|
}
|
|
1749
2118
|
};
|
|
1750
2119
|
}
|
|
2120
|
+
function createRunDurationRule(options) {
|
|
2121
|
+
return {
|
|
2122
|
+
id: "run.duration",
|
|
2123
|
+
category: "run",
|
|
2124
|
+
defaultSeverity: "error",
|
|
2125
|
+
evaluate(context) {
|
|
2126
|
+
const actual = context.selectedRun?.durationMs;
|
|
2127
|
+
if (actual === void 0 || actual <= options.maxDurationMs) return [];
|
|
2128
|
+
return [
|
|
2129
|
+
failFinding(
|
|
2130
|
+
"run.duration",
|
|
2131
|
+
`Run duration ${actual}ms exceeded ${options.maxDurationMs}ms.`,
|
|
2132
|
+
runEvidence(context.selectedRun),
|
|
2133
|
+
{ maxDurationMs: options.maxDurationMs },
|
|
2134
|
+
actual
|
|
2135
|
+
)
|
|
2136
|
+
];
|
|
2137
|
+
}
|
|
2138
|
+
};
|
|
2139
|
+
}
|
|
2140
|
+
function createToolUsageRule(options) {
|
|
2141
|
+
return {
|
|
2142
|
+
id: "tool.usage",
|
|
2143
|
+
category: "tool",
|
|
2144
|
+
defaultSeverity: "error",
|
|
2145
|
+
evaluate(context) {
|
|
2146
|
+
const tools = finishedEvents(context, "TOOL");
|
|
2147
|
+
const names = tools.map(toolName);
|
|
2148
|
+
const nameSet = new Set(names);
|
|
2149
|
+
const findings = [];
|
|
2150
|
+
for (const required of options.required ?? []) {
|
|
2151
|
+
if (!nameSet.has(required)) {
|
|
2152
|
+
findings.push(
|
|
2153
|
+
failFinding("tool.usage", `Required tool ${required} did not appear.`, runEvidence(context.selectedRun), required, names)
|
|
2154
|
+
);
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
2157
|
+
const forbidden = new Set(options.forbidden ?? []);
|
|
2158
|
+
const allowed = options.allowed ? new Set(options.allowed) : void 0;
|
|
2159
|
+
for (const event of tools) {
|
|
2160
|
+
const name = toolName(event);
|
|
2161
|
+
if (forbidden.has(name)) {
|
|
2162
|
+
findings.push(
|
|
2163
|
+
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence(event)], "tool absent", name)
|
|
2164
|
+
);
|
|
2165
|
+
}
|
|
2166
|
+
if (allowed && !allowed.has(name)) {
|
|
2167
|
+
findings.push(
|
|
2168
|
+
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence(event)], [...allowed].sort(), name)
|
|
2169
|
+
);
|
|
2170
|
+
}
|
|
2171
|
+
}
|
|
2172
|
+
if (options.minCount !== void 0 && tools.length < options.minCount) {
|
|
2173
|
+
findings.push(
|
|
2174
|
+
failFinding("tool.usage", `Tool count ${tools.length} was below minimum ${options.minCount}.`, runEvidence(context.selectedRun), { minCount: options.minCount }, tools.length)
|
|
2175
|
+
);
|
|
2176
|
+
}
|
|
2177
|
+
if (options.maxCount !== void 0 && tools.length > options.maxCount) {
|
|
2178
|
+
findings.push(
|
|
2179
|
+
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) => eventEvidence(event)), { maxCount: options.maxCount }, tools.length)
|
|
2180
|
+
);
|
|
2181
|
+
}
|
|
2182
|
+
return findings;
|
|
2183
|
+
}
|
|
2184
|
+
};
|
|
2185
|
+
}
|
|
2186
|
+
function createLlmUsageRule(options) {
|
|
2187
|
+
return {
|
|
2188
|
+
id: "llm.usage",
|
|
2189
|
+
category: "llm",
|
|
2190
|
+
defaultSeverity: "error",
|
|
2191
|
+
evaluate(context) {
|
|
2192
|
+
const llms = finishedEvents(context, "LLM");
|
|
2193
|
+
const findings = [];
|
|
2194
|
+
const allowedModels = options.allowedModels ? new Set(options.allowedModels) : void 0;
|
|
2195
|
+
const allowedProviders = options.allowedProviders ? new Set(options.allowedProviders) : void 0;
|
|
2196
|
+
const finishReasons = options.finishReasons ? new Set(options.finishReasons) : void 0;
|
|
2197
|
+
if (options.maxCalls !== void 0 && llms.length > options.maxCalls) {
|
|
2198
|
+
findings.push(
|
|
2199
|
+
failFinding(
|
|
2200
|
+
"llm.usage",
|
|
2201
|
+
`LLM call count ${llms.length} exceeded ${options.maxCalls}.`,
|
|
2202
|
+
llms.map((event) => eventEvidence(event)),
|
|
2203
|
+
{ maxCalls: options.maxCalls },
|
|
2204
|
+
llms.length
|
|
2205
|
+
)
|
|
2206
|
+
);
|
|
2207
|
+
}
|
|
2208
|
+
for (const event of llms) {
|
|
2209
|
+
const model = llmModel(event);
|
|
2210
|
+
const provider = llmProvider(event);
|
|
2211
|
+
const finishReason = llmFinishReason(event);
|
|
2212
|
+
if (allowedModels && (!model || !allowedModels.has(model))) {
|
|
2213
|
+
findings.push(
|
|
2214
|
+
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.model")], [...allowedModels].sort(), model ?? "unknown")
|
|
2215
|
+
);
|
|
2216
|
+
}
|
|
2217
|
+
if (allowedProviders && (!provider || !allowedProviders.has(provider))) {
|
|
2218
|
+
findings.push(
|
|
2219
|
+
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.provider")], [...allowedProviders].sort(), provider ?? "unknown")
|
|
2220
|
+
);
|
|
2221
|
+
}
|
|
2222
|
+
if (finishReasons && (!finishReason || !finishReasons.has(finishReason))) {
|
|
2223
|
+
findings.push(
|
|
2224
|
+
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.finishReason")], [...finishReasons].sort(), finishReason ?? "unknown")
|
|
2225
|
+
);
|
|
2226
|
+
}
|
|
2227
|
+
}
|
|
2228
|
+
const tokenTotals = llms.reduce(
|
|
2229
|
+
(totals, event) => ({
|
|
2230
|
+
input: totals.input + (event.tokenUsage?.input ?? 0),
|
|
2231
|
+
output: totals.output + (event.tokenUsage?.output ?? 0),
|
|
2232
|
+
total: totals.total + (event.tokenUsage?.total ?? 0),
|
|
2233
|
+
cached: totals.cached + (event.tokenUsage?.cached ?? 0)
|
|
2234
|
+
}),
|
|
2235
|
+
{ input: 0, output: 0, total: 0, cached: 0 }
|
|
2236
|
+
);
|
|
2237
|
+
const tokenLimits = [
|
|
2238
|
+
["input", options.maxInputTokens],
|
|
2239
|
+
["output", options.maxOutputTokens],
|
|
2240
|
+
["total", options.maxTotalTokens],
|
|
2241
|
+
["cached", options.maxCachedTokens]
|
|
2242
|
+
];
|
|
2243
|
+
for (const [key, limit] of tokenLimits) {
|
|
2244
|
+
if (limit !== void 0 && tokenTotals[key] > limit) {
|
|
2245
|
+
findings.push(
|
|
2246
|
+
failFinding(
|
|
2247
|
+
"llm.usage",
|
|
2248
|
+
`LLM ${key} token count ${tokenTotals[key]} exceeded ${limit}.`,
|
|
2249
|
+
llms.map((event) => eventEvidence(event, `tokenUsage.${key}`)),
|
|
2250
|
+
{ [`max${key[0].toUpperCase()}${key.slice(1)}Tokens`]: limit },
|
|
2251
|
+
tokenTotals[key]
|
|
2252
|
+
)
|
|
2253
|
+
);
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
2256
|
+
return findings;
|
|
2257
|
+
}
|
|
2258
|
+
};
|
|
2259
|
+
}
|
|
2260
|
+
function createObservedOutcomeRule(options = {}) {
|
|
2261
|
+
const failOn = options.failOn ?? ["failed"];
|
|
2262
|
+
return {
|
|
2263
|
+
id: "outcome.status",
|
|
2264
|
+
category: "run",
|
|
2265
|
+
defaultSeverity: "error",
|
|
2266
|
+
evaluate(context) {
|
|
2267
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
2268
|
+
const matching = outcomesMatchingStatus(outcomes, failOn);
|
|
2269
|
+
if (matching.length === 0) return [];
|
|
2270
|
+
return [
|
|
2271
|
+
failFinding(
|
|
2272
|
+
"outcome.status",
|
|
2273
|
+
`Observed outcome count ${matching.length} matched [${failOn.join(", ")}].`,
|
|
2274
|
+
matching.map((outcome) => ({
|
|
2275
|
+
runId: outcome.runId,
|
|
2276
|
+
eventId: outcome.outcomeId,
|
|
2277
|
+
...outcome.parentId !== void 0 ? { parentId: outcome.parentId } : {},
|
|
2278
|
+
kind: "OUTCOME",
|
|
2279
|
+
name: outcome.name,
|
|
2280
|
+
status: outcome.status,
|
|
2281
|
+
path: `outcome.${outcome.name}`
|
|
2282
|
+
})),
|
|
2283
|
+
{ failOn },
|
|
2284
|
+
matching.map((outcome) => ({
|
|
2285
|
+
name: outcome.name,
|
|
2286
|
+
status: outcome.status,
|
|
2287
|
+
expectation: outcome.expectation
|
|
2288
|
+
}))
|
|
2289
|
+
)
|
|
2290
|
+
];
|
|
2291
|
+
}
|
|
2292
|
+
};
|
|
2293
|
+
}
|
|
1751
2294
|
function runTraceChecks(input, options = {}) {
|
|
1752
2295
|
const selected = resolveSelectedRun(input, options.runId);
|
|
1753
2296
|
if (selected.diagnostics.length > 0) {
|
|
@@ -1771,7 +2314,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
1771
2314
|
} catch (error) {
|
|
1772
2315
|
const message = error instanceof Error ? error.message : String(error);
|
|
1773
2316
|
diagnostics.push(
|
|
1774
|
-
|
|
2317
|
+
diagnostic3("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
|
|
1775
2318
|
);
|
|
1776
2319
|
}
|
|
1777
2320
|
}
|
|
@@ -2410,7 +2953,7 @@ function findReaderByFormat(format, readers) {
|
|
|
2410
2953
|
}
|
|
2411
2954
|
async function jsonlFilesInDirectory(dirPath) {
|
|
2412
2955
|
const entries = await readdir(dirPath, { withFileTypes: true });
|
|
2413
|
-
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) =>
|
|
2956
|
+
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => path13.join(dirPath, entry.name)).sort((a, b) => a.localeCompare(b));
|
|
2414
2957
|
}
|
|
2415
2958
|
async function resolveInput(input) {
|
|
2416
2959
|
const cached = resolvedInputCache.get(input);
|
|
@@ -3861,6 +4404,225 @@ function openTrace(input, options = {}) {
|
|
|
3861
4404
|
return readTrace(input, options);
|
|
3862
4405
|
}
|
|
3863
4406
|
|
|
4407
|
+
// packages/core/src/suite/run.ts
|
|
4408
|
+
function diagnostic4(code, message, severity = "error", caseId) {
|
|
4409
|
+
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
4410
|
+
}
|
|
4411
|
+
function buildCaseRules(suiteCase, config) {
|
|
4412
|
+
const rules = [];
|
|
4413
|
+
const select = new Set(config.checks?.select ?? []);
|
|
4414
|
+
if (select.has("run.status")) {
|
|
4415
|
+
rules.push(createRunStatusRule());
|
|
4416
|
+
}
|
|
4417
|
+
const requiredTools = [
|
|
4418
|
+
...config.checks?.tool?.required ?? [],
|
|
4419
|
+
...suiteCase.requireTools ?? []
|
|
4420
|
+
];
|
|
4421
|
+
const forbiddenTools = [
|
|
4422
|
+
...config.checks?.tool?.forbidden ?? [],
|
|
4423
|
+
...suiteCase.forbidTools ?? []
|
|
4424
|
+
];
|
|
4425
|
+
if (requiredTools.length > 0 || forbiddenTools.length > 0) {
|
|
4426
|
+
rules.push(
|
|
4427
|
+
createToolUsageRule({
|
|
4428
|
+
required: requiredTools.length > 0 ? requiredTools : void 0,
|
|
4429
|
+
forbidden: forbiddenTools.length > 0 ? forbiddenTools : void 0
|
|
4430
|
+
})
|
|
4431
|
+
);
|
|
4432
|
+
select.add("tool.usage");
|
|
4433
|
+
}
|
|
4434
|
+
const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? config.eval?.maxDurationMs;
|
|
4435
|
+
if (maxDurationMs !== void 0) {
|
|
4436
|
+
rules.push(createRunDurationRule({ maxDurationMs }));
|
|
4437
|
+
select.add("run.duration");
|
|
4438
|
+
}
|
|
4439
|
+
const llm = config.checks?.llm;
|
|
4440
|
+
if (llm?.allowedModels !== void 0 || llm?.maxTotalTokens !== void 0) {
|
|
4441
|
+
rules.push(createLlmUsageRule(llm));
|
|
4442
|
+
select.add("llm.usage");
|
|
4443
|
+
}
|
|
4444
|
+
if (select.has("outcome.status")) {
|
|
4445
|
+
rules.push(createObservedOutcomeRule({ failOn: ["failed"] }));
|
|
4446
|
+
}
|
|
4447
|
+
return { rules, select: [...select] };
|
|
4448
|
+
}
|
|
4449
|
+
function outcomesFromRead(read) {
|
|
4450
|
+
const persistedOutcomes = extractOutcomesFromPersistedEvents(
|
|
4451
|
+
read.events.filter((event) => event.kind === "OUTCOME")
|
|
4452
|
+
);
|
|
4453
|
+
if (persistedOutcomes.length > 0) return persistedOutcomes;
|
|
4454
|
+
const traceEvents = [];
|
|
4455
|
+
for (const event of read.events) {
|
|
4456
|
+
const legacy = event;
|
|
4457
|
+
if (typeof legacy === "object" && legacy !== null && "event" in legacy && legacy.event === "outcome_observed") {
|
|
4458
|
+
traceEvents.push(legacy);
|
|
4459
|
+
}
|
|
4460
|
+
}
|
|
4461
|
+
return traceEvents.length > 0 ? extractOutcomesFromTraceEvents(traceEvents) : [];
|
|
4462
|
+
}
|
|
4463
|
+
function validateExpectedObservations(suiteCase, read) {
|
|
4464
|
+
const expected = suiteCase.expectedObservations ?? [];
|
|
4465
|
+
if (expected.length === 0) return { ok: true, diagnostics: [] };
|
|
4466
|
+
const outcomes = outcomesFromRead(read);
|
|
4467
|
+
const diagnostics = [];
|
|
4468
|
+
for (const name of expected) {
|
|
4469
|
+
const match = outcomes.find((outcome) => outcome.name === name);
|
|
4470
|
+
if (!match) {
|
|
4471
|
+
diagnostics.push(
|
|
4472
|
+
diagnostic4(
|
|
4473
|
+
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
4474
|
+
`Expected observation "${name}" was not recorded.`,
|
|
4475
|
+
"error",
|
|
4476
|
+
suiteCase.id
|
|
4477
|
+
)
|
|
4478
|
+
);
|
|
4479
|
+
continue;
|
|
4480
|
+
}
|
|
4481
|
+
if (match.status !== "passed") {
|
|
4482
|
+
diagnostics.push(
|
|
4483
|
+
diagnostic4(
|
|
4484
|
+
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
4485
|
+
`Observation "${name}" has status "${match.status}", expected "passed".`,
|
|
4486
|
+
"error",
|
|
4487
|
+
suiteCase.id
|
|
4488
|
+
)
|
|
4489
|
+
);
|
|
4490
|
+
}
|
|
4491
|
+
}
|
|
4492
|
+
return { ok: diagnostics.length === 0, diagnostics };
|
|
4493
|
+
}
|
|
4494
|
+
async function runSuiteCase(suiteCase, config, options) {
|
|
4495
|
+
const resolved = await resolveSuiteCaseTrace(suiteCase, options);
|
|
4496
|
+
if (resolved.missing || resolved.tracePath === void 0) {
|
|
4497
|
+
return {
|
|
4498
|
+
id: suiteCase.id,
|
|
4499
|
+
status: "skipped",
|
|
4500
|
+
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
4501
|
+
message: resolved.reason,
|
|
4502
|
+
diagnostics: [
|
|
4503
|
+
diagnostic4(
|
|
4504
|
+
"AI_SUITE_CASE_TRACE_MISSING",
|
|
4505
|
+
resolved.reason ?? "Trace not found.",
|
|
4506
|
+
"warning",
|
|
4507
|
+
suiteCase.id
|
|
4508
|
+
)
|
|
4509
|
+
]
|
|
4510
|
+
};
|
|
4511
|
+
}
|
|
4512
|
+
let read;
|
|
4513
|
+
try {
|
|
4514
|
+
read = await openTrace({ type: "file", path: resolved.tracePath });
|
|
4515
|
+
} catch (error) {
|
|
4516
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4517
|
+
return {
|
|
4518
|
+
id: suiteCase.id,
|
|
4519
|
+
status: "error",
|
|
4520
|
+
tracePath: resolved.tracePath,
|
|
4521
|
+
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
4522
|
+
message,
|
|
4523
|
+
diagnostics: [
|
|
4524
|
+
diagnostic4("AI_SUITE_TRACE_UNREADABLE", message, "error", suiteCase.id)
|
|
4525
|
+
]
|
|
4526
|
+
};
|
|
4527
|
+
}
|
|
4528
|
+
const { rules, select } = buildCaseRules(suiteCase, config);
|
|
4529
|
+
const checkResult = rules.length > 0 ? runTraceChecks({ read }, { rules, select }) : {
|
|
4530
|
+
ok: true,
|
|
4531
|
+
status: "pass",
|
|
4532
|
+
format: read.format,
|
|
4533
|
+
findings: [],
|
|
4534
|
+
diagnostics: []
|
|
4535
|
+
};
|
|
4536
|
+
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
4537
|
+
const diagnostics = [
|
|
4538
|
+
...checkResult.diagnostics.map(
|
|
4539
|
+
(item) => diagnostic4(
|
|
4540
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
4541
|
+
item.message,
|
|
4542
|
+
item.severity,
|
|
4543
|
+
suiteCase.id
|
|
4544
|
+
)
|
|
4545
|
+
),
|
|
4546
|
+
...checkResult.findings.filter((finding) => finding.status === "fail").map(
|
|
4547
|
+
(finding) => diagnostic4(
|
|
4548
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
4549
|
+
finding.message,
|
|
4550
|
+
finding.severity,
|
|
4551
|
+
suiteCase.id
|
|
4552
|
+
)
|
|
4553
|
+
),
|
|
4554
|
+
...observationResult.diagnostics
|
|
4555
|
+
];
|
|
4556
|
+
const checkOk = checkResult.ok;
|
|
4557
|
+
const observationsOk = observationResult.ok;
|
|
4558
|
+
const ok = checkOk && observationsOk;
|
|
4559
|
+
const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
|
|
4560
|
+
return {
|
|
4561
|
+
id: suiteCase.id,
|
|
4562
|
+
status,
|
|
4563
|
+
tracePath: resolved.tracePath,
|
|
4564
|
+
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
4565
|
+
checkOk,
|
|
4566
|
+
observationsOk,
|
|
4567
|
+
diagnostics,
|
|
4568
|
+
...ok ? {} : {
|
|
4569
|
+
message: diagnostics.map((item) => item.message).join("; ") || checkResult.findings.map((item) => item.message).join("; ")
|
|
4570
|
+
}
|
|
4571
|
+
};
|
|
4572
|
+
}
|
|
4573
|
+
async function runSuite(options = {}) {
|
|
4574
|
+
const startedAt = new Date(options.nowMs ?? Date.now()).toISOString();
|
|
4575
|
+
const { config, configPath, configDir } = await loadSuiteConfig(options);
|
|
4576
|
+
const tracesDir = path13.resolve(configDir, config.traces);
|
|
4577
|
+
const cases = [];
|
|
4578
|
+
const diagnostics = [];
|
|
4579
|
+
for (const suiteCase of config.cases) {
|
|
4580
|
+
cases.push(
|
|
4581
|
+
await runSuiteCase(suiteCase, config, {
|
|
4582
|
+
configDir,
|
|
4583
|
+
tracesDir
|
|
4584
|
+
})
|
|
4585
|
+
);
|
|
4586
|
+
}
|
|
4587
|
+
const summary = {
|
|
4588
|
+
passed: cases.filter((item) => item.status === "pass").length,
|
|
4589
|
+
failed: cases.filter((item) => item.status === "fail").length,
|
|
4590
|
+
errors: cases.filter((item) => item.status === "error").length,
|
|
4591
|
+
skipped: cases.filter((item) => item.status === "skipped").length
|
|
4592
|
+
};
|
|
4593
|
+
const finishedAt = new Date(options.nowMs ?? Date.now()).toISOString();
|
|
4594
|
+
const ok = summary.failed === 0 && summary.errors === 0;
|
|
4595
|
+
const status = summary.errors > 0 ? "error" : summary.failed > 0 || !ok ? "fail" : "pass";
|
|
4596
|
+
return {
|
|
4597
|
+
ok,
|
|
4598
|
+
status,
|
|
4599
|
+
suiteName: config.name,
|
|
4600
|
+
configPath,
|
|
4601
|
+
tracesDir,
|
|
4602
|
+
startedAt,
|
|
4603
|
+
finishedAt,
|
|
4604
|
+
summary,
|
|
4605
|
+
cases,
|
|
4606
|
+
diagnostics
|
|
4607
|
+
};
|
|
4608
|
+
}
|
|
4609
|
+
|
|
4610
|
+
// packages/core/src/exporters/helpers.ts
|
|
4611
|
+
function sortKeysDeep(input) {
|
|
4612
|
+
if (input === null || typeof input !== "object") return input;
|
|
4613
|
+
if (Array.isArray(input)) return input.map(sortKeysDeep);
|
|
4614
|
+
const o = input;
|
|
4615
|
+
const out = {};
|
|
4616
|
+
for (const k of Object.keys(o).sort()) {
|
|
4617
|
+
out[k] = sortKeysDeep(o[k]);
|
|
4618
|
+
}
|
|
4619
|
+
return out;
|
|
4620
|
+
}
|
|
4621
|
+
function stableJson(value, pretty) {
|
|
4622
|
+
const sorted = sortKeysDeep(value);
|
|
4623
|
+
return JSON.stringify(sorted);
|
|
4624
|
+
}
|
|
4625
|
+
|
|
3864
4626
|
// packages/viewer/src/html.ts
|
|
3865
4627
|
var viewerIndexHtml = `<!DOCTYPE html>
|
|
3866
4628
|
<html lang="en">
|
|
@@ -3870,20 +4632,71 @@ var viewerIndexHtml = `<!DOCTYPE html>
|
|
|
3870
4632
|
<style>
|
|
3871
4633
|
body { font-family: system-ui, sans-serif; margin: 1.5rem; line-height: 1.4; }
|
|
3872
4634
|
h1 { font-size: 1.25rem; }
|
|
3873
|
-
pre { background: #f4f4f5; padding: 1rem; overflow: auto; max-height:
|
|
4635
|
+
pre { background: #f4f4f5; padding: 1rem; overflow: auto; max-height: 50vh; }
|
|
3874
4636
|
a { color: #0b57d0; }
|
|
3875
4637
|
.muted { color: #666; }
|
|
4638
|
+
table { border-collapse: collapse; width: 100%; margin: 1rem 0; }
|
|
4639
|
+
th, td { border: 1px solid #ddd; padding: 0.5rem; text-align: left; vertical-align: top; }
|
|
4640
|
+
th { background: #f6f6f6; }
|
|
4641
|
+
.fail { color: #b42318; font-weight: 600; }
|
|
4642
|
+
.pass { color: #027a48; font-weight: 600; }
|
|
4643
|
+
section { margin: 1.5rem 0; }
|
|
3876
4644
|
</style>
|
|
3877
4645
|
</head>
|
|
3878
4646
|
<body>
|
|
3879
4647
|
<h1>AgentInspect local viewer</h1>
|
|
3880
4648
|
<p class="muted">Read-only. JSONL on disk remains canonical.</p>
|
|
3881
|
-
<p
|
|
3882
|
-
<pre id="out">Loading
|
|
4649
|
+
<p id="nav"></p>
|
|
4650
|
+
<div id="content"><pre id="out">Loading\u2026</pre></div>
|
|
3883
4651
|
<script>
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
4652
|
+
const params = new URLSearchParams(location.search);
|
|
4653
|
+
const mode = params.get("mode") || "traces";
|
|
4654
|
+
|
|
4655
|
+
function renderSuite(data) {
|
|
4656
|
+
const rows = data.cases.map((c) =>
|
|
4657
|
+
'<tr><td>' + c.id + '</td><td class="' + c.status + '">' + c.status +
|
|
4658
|
+
'</td><td>' + (c.message || '') + '</td><td>' + c.toolPath.join(' \u2192 ') +
|
|
4659
|
+
'</td><td>' + c.observations.map((o) => o.name + ':' + o.status).join(', ') + '</td></tr>'
|
|
4660
|
+
).join('');
|
|
4661
|
+
const failed = data.cases.filter((c) => c.status !== 'pass');
|
|
4662
|
+
const detail = failed.map((c) => {
|
|
4663
|
+
let html = '<h3>Case ' + c.id + '</h3><ul>';
|
|
4664
|
+
if (c.failureDiff) html += '<li>Diff errors: ' + c.failureDiff.summary.errors + '</li>';
|
|
4665
|
+
if (c.timeline) html += '<li>Timeline steps: ' + c.timeline.entries.length + '</li>';
|
|
4666
|
+
html += '<li>Diagnostics: ' + c.diagnostics.map((d) => d.message).join('; ') + '</li></ul>';
|
|
4667
|
+
return html;
|
|
4668
|
+
}).join('');
|
|
4669
|
+
return '<section><h2>Suite: ' + data.suiteName + ' (' + data.status + ')</h2>' +
|
|
4670
|
+
'<p>Passed ' + data.summary.passed + ', failed ' + data.summary.failed + '</p>' +
|
|
4671
|
+
'<table><thead><tr><th>Case</th><th>Status</th><th>Message</th><th>Tool path</th><th>Observations</th></tr></thead><tbody>' +
|
|
4672
|
+
rows + '</tbody></table>' +
|
|
4673
|
+
'<section><h2>Failure detail</h2>' + (detail || '<p class="pass">No failures</p>') + '</section>' +
|
|
4674
|
+
'<section><h2>CI artifacts</h2><p>' + (data.ciArtifactsDir || 'n/a') + '</p></section>' +
|
|
4675
|
+
'<section><h2>Bundle export</h2><p>' + data.bundleExportHint + '</p></section>';
|
|
4676
|
+
}
|
|
4677
|
+
|
|
4678
|
+
async function load() {
|
|
4679
|
+
const nav = document.getElementById("nav");
|
|
4680
|
+
const out = document.getElementById("out");
|
|
4681
|
+
const content = document.getElementById("content");
|
|
4682
|
+
if (mode === "suite") {
|
|
4683
|
+
nav.innerHTML = '<a href="/api/suite">/api/suite</a>';
|
|
4684
|
+
const data = await fetch("/api/suite").then((r) => r.json());
|
|
4685
|
+
content.innerHTML = renderSuite(data);
|
|
4686
|
+
return;
|
|
4687
|
+
}
|
|
4688
|
+
if (mode === "workspace") {
|
|
4689
|
+
nav.innerHTML = '<a href="/api/workspace">/api/workspace</a>';
|
|
4690
|
+
const data = await fetch("/api/workspace").then((r) => r.json());
|
|
4691
|
+
content.innerHTML = '<section><h2>Workspace: ' + (data.project || 'workspace') + '</h2>' +
|
|
4692
|
+
'<p>Runs: ' + data.runs.length + '</p><pre>' + JSON.stringify(data, null, 2) + '</pre></section>';
|
|
4693
|
+
return;
|
|
4694
|
+
}
|
|
4695
|
+
nav.innerHTML = '<a href="/api/traces">/api/traces</a> \xB7 <a href="/api/sessions">/api/sessions</a>';
|
|
4696
|
+
const data = await fetch("/api/traces").then((r) => r.json());
|
|
4697
|
+
out.textContent = JSON.stringify(data, null, 2);
|
|
4698
|
+
}
|
|
4699
|
+
load().catch((err) => {
|
|
3887
4700
|
document.getElementById("out").textContent = String(err);
|
|
3888
4701
|
});
|
|
3889
4702
|
</script>
|
|
@@ -3891,6 +4704,815 @@ var viewerIndexHtml = `<!DOCTYPE html>
|
|
|
3891
4704
|
</html>
|
|
3892
4705
|
`;
|
|
3893
4706
|
|
|
4707
|
+
// packages/core/src/diff/comparable.ts
|
|
4708
|
+
function extractOutputPreview(meta) {
|
|
4709
|
+
if (meta === void 0) return void 0;
|
|
4710
|
+
if ("outputPreview" in meta) return meta.outputPreview;
|
|
4711
|
+
if ("resultPreview" in meta) return meta.resultPreview;
|
|
4712
|
+
return void 0;
|
|
4713
|
+
}
|
|
4714
|
+
function mapStepStatus(s) {
|
|
4715
|
+
if (s === void 0) return "running";
|
|
4716
|
+
return s;
|
|
4717
|
+
}
|
|
4718
|
+
function manualTraceEventsToComparableRun(events) {
|
|
4719
|
+
const started = events.find((e) => e.event === "run_started");
|
|
4720
|
+
if (!started || started.event !== "run_started") {
|
|
4721
|
+
throw new Error("Invalid trace: missing run_started");
|
|
4722
|
+
}
|
|
4723
|
+
const rs = started;
|
|
4724
|
+
const runId = rs.runId;
|
|
4725
|
+
const completedAll = events.filter((e) => e.event === "run_completed");
|
|
4726
|
+
const lastCompleted = completedAll[completedAll.length - 1];
|
|
4727
|
+
let runStatus;
|
|
4728
|
+
if (lastCompleted === void 0) runStatus = "running";
|
|
4729
|
+
else runStatus = lastCompleted.status;
|
|
4730
|
+
const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
4731
|
+
const steps = /* @__PURE__ */ new Map();
|
|
4732
|
+
let order = 0;
|
|
4733
|
+
for (const e of events) {
|
|
4734
|
+
if (e.event !== "step_started") continue;
|
|
4735
|
+
const s = e;
|
|
4736
|
+
const meta = s.metadata ? { ...s.metadata } : void 0;
|
|
4737
|
+
steps.set(s.stepId, {
|
|
4738
|
+
id: s.stepId,
|
|
4739
|
+
parentId: s.parentId,
|
|
4740
|
+
name: s.name,
|
|
4741
|
+
type: s.type,
|
|
4742
|
+
order: order++,
|
|
4743
|
+
timestamp: s.timestamp,
|
|
4744
|
+
metadata: meta
|
|
4745
|
+
});
|
|
4746
|
+
}
|
|
4747
|
+
for (const e of events) {
|
|
4748
|
+
if (e.event !== "step_completed") continue;
|
|
4749
|
+
const acc = steps.get(e.stepId);
|
|
4750
|
+
if (!acc) continue;
|
|
4751
|
+
acc.status = e.status;
|
|
4752
|
+
acc.durationMs = e.durationMs;
|
|
4753
|
+
if (e.error?.message) acc.errorMsg = e.error.message;
|
|
4754
|
+
const extra = e;
|
|
4755
|
+
if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
|
|
4756
|
+
acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
|
|
4757
|
+
}
|
|
4758
|
+
}
|
|
4759
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
4760
|
+
for (const acc of steps.values()) {
|
|
4761
|
+
let meta = acc.metadata ? { ...acc.metadata } : void 0;
|
|
4762
|
+
if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
|
|
4763
|
+
meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
|
|
4764
|
+
}
|
|
4765
|
+
const outputPreview = extractOutputPreview(meta);
|
|
4766
|
+
const sc = {
|
|
4767
|
+
id: acc.id,
|
|
4768
|
+
name: acc.name,
|
|
4769
|
+
type: acc.type,
|
|
4770
|
+
status: mapStepStatus(acc.status),
|
|
4771
|
+
durationMs: acc.durationMs,
|
|
4772
|
+
error: acc.errorMsg,
|
|
4773
|
+
metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
|
|
4774
|
+
outputPreview,
|
|
4775
|
+
children: []
|
|
4776
|
+
};
|
|
4777
|
+
nodes.set(acc.id, sc);
|
|
4778
|
+
}
|
|
4779
|
+
const roots = [];
|
|
4780
|
+
const sortByOrder = (a, b) => {
|
|
4781
|
+
const oa = steps.get(a.id)?.order ?? 0;
|
|
4782
|
+
const ob = steps.get(b.id)?.order ?? 0;
|
|
4783
|
+
return oa - ob;
|
|
4784
|
+
};
|
|
4785
|
+
for (const acc of steps.values()) {
|
|
4786
|
+
const node = nodes.get(acc.id);
|
|
4787
|
+
if (acc.parentId !== void 0 && nodes.has(acc.parentId)) {
|
|
4788
|
+
nodes.get(acc.parentId).children.push(node);
|
|
4789
|
+
} else {
|
|
4790
|
+
roots.push(node);
|
|
4791
|
+
}
|
|
4792
|
+
}
|
|
4793
|
+
roots.sort(sortByOrder);
|
|
4794
|
+
for (const n of nodes.values()) {
|
|
4795
|
+
n.children.sort(sortByOrder);
|
|
4796
|
+
}
|
|
4797
|
+
return {
|
|
4798
|
+
runId,
|
|
4799
|
+
name: rs.name,
|
|
4800
|
+
status: runStatus,
|
|
4801
|
+
durationMs,
|
|
4802
|
+
steps: roots
|
|
4803
|
+
};
|
|
4804
|
+
}
|
|
4805
|
+
|
|
4806
|
+
// packages/core/src/diff/engine.ts
|
|
4807
|
+
var DEFAULT_THRESHOLD_MS = 0;
|
|
4808
|
+
function pathSeg(step, index) {
|
|
4809
|
+
return { index, name: step.name, stepId: step.id };
|
|
4810
|
+
}
|
|
4811
|
+
function buildPath(segments) {
|
|
4812
|
+
return { path: [...segments] };
|
|
4813
|
+
}
|
|
4814
|
+
function pairSteps(left, right) {
|
|
4815
|
+
const usedRight = /* @__PURE__ */ new Set();
|
|
4816
|
+
const pairs = [];
|
|
4817
|
+
for (let i = 0; i < left.length; i++) {
|
|
4818
|
+
const L = left[i];
|
|
4819
|
+
let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
|
|
4820
|
+
if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
|
|
4821
|
+
const cand = right[i];
|
|
4822
|
+
if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
|
|
4823
|
+
R = cand;
|
|
4824
|
+
}
|
|
4825
|
+
}
|
|
4826
|
+
if (R === void 0) {
|
|
4827
|
+
R = right.find(
|
|
4828
|
+
(r) => !usedRight.has(r.id) && r.name === L.name && (r.type ?? "") === (L.type ?? "")
|
|
4829
|
+
);
|
|
4830
|
+
}
|
|
4831
|
+
if (R !== void 0) {
|
|
4832
|
+
usedRight.add(R.id);
|
|
4833
|
+
pairs.push([L, R]);
|
|
4834
|
+
} else {
|
|
4835
|
+
pairs.push([L, void 0]);
|
|
4836
|
+
}
|
|
4837
|
+
}
|
|
4838
|
+
for (const R of right) {
|
|
4839
|
+
if (!usedRight.has(R.id)) {
|
|
4840
|
+
pairs.push([void 0, R]);
|
|
4841
|
+
}
|
|
4842
|
+
}
|
|
4843
|
+
return pairs;
|
|
4844
|
+
}
|
|
4845
|
+
function compareLeafSteps(L, R, segments, opts, out) {
|
|
4846
|
+
const path16 = buildPath(segments);
|
|
4847
|
+
if (L.name !== R.name) {
|
|
4848
|
+
out.push({
|
|
4849
|
+
kind: "structure",
|
|
4850
|
+
severity: "warning",
|
|
4851
|
+
message: "Step name differs",
|
|
4852
|
+
path: path16,
|
|
4853
|
+
left: L.name,
|
|
4854
|
+
right: R.name
|
|
4855
|
+
});
|
|
4856
|
+
}
|
|
4857
|
+
if ((L.type ?? "") !== (R.type ?? "")) {
|
|
4858
|
+
out.push({
|
|
4859
|
+
kind: "step-type",
|
|
4860
|
+
severity: "warning",
|
|
4861
|
+
message: "Step type differs",
|
|
4862
|
+
path: path16,
|
|
4863
|
+
left: L.type,
|
|
4864
|
+
right: R.type
|
|
4865
|
+
});
|
|
4866
|
+
}
|
|
4867
|
+
if ((L.status ?? "") !== (R.status ?? "")) {
|
|
4868
|
+
out.push({
|
|
4869
|
+
kind: "step-status",
|
|
4870
|
+
severity: "warning",
|
|
4871
|
+
message: "Step status differs",
|
|
4872
|
+
path: path16,
|
|
4873
|
+
left: L.status,
|
|
4874
|
+
right: R.status
|
|
4875
|
+
});
|
|
4876
|
+
}
|
|
4877
|
+
const le = L.error ?? "";
|
|
4878
|
+
const re = R.error ?? "";
|
|
4879
|
+
if (le !== re) {
|
|
4880
|
+
out.push({
|
|
4881
|
+
kind: "error",
|
|
4882
|
+
severity: "error",
|
|
4883
|
+
message: "Step error message differs",
|
|
4884
|
+
path: path16,
|
|
4885
|
+
left: le || void 0,
|
|
4886
|
+
right: re || void 0
|
|
4887
|
+
});
|
|
4888
|
+
}
|
|
4889
|
+
if (!opts.ignoreDuration) {
|
|
4890
|
+
const ld = L.durationMs;
|
|
4891
|
+
const rd = R.durationMs;
|
|
4892
|
+
const th = opts.durationThresholdMs;
|
|
4893
|
+
let differs = false;
|
|
4894
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
4895
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
4896
|
+
else differs = Math.abs(ld - rd) > th;
|
|
4897
|
+
if (differs) {
|
|
4898
|
+
out.push({
|
|
4899
|
+
kind: "duration",
|
|
4900
|
+
severity: "info",
|
|
4901
|
+
message: "Step duration differs",
|
|
4902
|
+
path: path16,
|
|
4903
|
+
left: ld,
|
|
4904
|
+
right: rd
|
|
4905
|
+
});
|
|
4906
|
+
}
|
|
4907
|
+
}
|
|
4908
|
+
const lm = stableJson(L.metadata ?? {});
|
|
4909
|
+
const rm2 = stableJson(R.metadata ?? {});
|
|
4910
|
+
if (lm !== rm2) {
|
|
4911
|
+
out.push({
|
|
4912
|
+
kind: "metadata",
|
|
4913
|
+
severity: "info",
|
|
4914
|
+
message: "Step metadata differs",
|
|
4915
|
+
path: path16,
|
|
4916
|
+
left: L.metadata,
|
|
4917
|
+
right: R.metadata
|
|
4918
|
+
});
|
|
4919
|
+
}
|
|
4920
|
+
const lo = stableJson(L.outputPreview ?? null);
|
|
4921
|
+
const ro = stableJson(R.outputPreview ?? null);
|
|
4922
|
+
if (lo !== ro) {
|
|
4923
|
+
out.push({
|
|
4924
|
+
kind: "output",
|
|
4925
|
+
severity: "info",
|
|
4926
|
+
message: "Output preview differs",
|
|
4927
|
+
path: path16,
|
|
4928
|
+
left: L.outputPreview,
|
|
4929
|
+
right: R.outputPreview
|
|
4930
|
+
});
|
|
4931
|
+
}
|
|
4932
|
+
}
|
|
4933
|
+
function compareRecursive(L, R, segments, opts, out) {
|
|
4934
|
+
compareLeafSteps(L, R, segments, opts, out);
|
|
4935
|
+
const pairs = pairSteps(L.children, R.children);
|
|
4936
|
+
let ci = 0;
|
|
4937
|
+
for (const [lch, rch] of pairs) {
|
|
4938
|
+
if (lch !== void 0 && rch !== void 0) {
|
|
4939
|
+
compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
|
|
4940
|
+
} else if (lch !== void 0) {
|
|
4941
|
+
out.push({
|
|
4942
|
+
kind: "step-removed",
|
|
4943
|
+
severity: "warning",
|
|
4944
|
+
message: `Step only in left run: ${lch.name}`,
|
|
4945
|
+
path: buildPath([...segments, pathSeg(lch, ci)]),
|
|
4946
|
+
left: lch.id,
|
|
4947
|
+
right: void 0
|
|
4948
|
+
});
|
|
4949
|
+
} else if (rch !== void 0) {
|
|
4950
|
+
out.push({
|
|
4951
|
+
kind: "step-added",
|
|
4952
|
+
severity: "warning",
|
|
4953
|
+
message: `Step only in right run: ${rch.name}`,
|
|
4954
|
+
path: buildPath([...segments, pathSeg(rch, ci)]),
|
|
4955
|
+
left: void 0,
|
|
4956
|
+
right: rch.id
|
|
4957
|
+
});
|
|
4958
|
+
}
|
|
4959
|
+
ci += 1;
|
|
4960
|
+
}
|
|
4961
|
+
}
|
|
4962
|
+
function mergeDiffDefaults(options) {
|
|
4963
|
+
return {
|
|
4964
|
+
ignoreDuration: false,
|
|
4965
|
+
durationThresholdMs: DEFAULT_THRESHOLD_MS,
|
|
4966
|
+
focus: "all",
|
|
4967
|
+
check: "all"
|
|
4968
|
+
};
|
|
4969
|
+
}
|
|
4970
|
+
function kindMatchesFilter(kind, merged) {
|
|
4971
|
+
return true;
|
|
4972
|
+
}
|
|
4973
|
+
function diffRuns(left, right, options) {
|
|
4974
|
+
const merged = mergeDiffDefaults();
|
|
4975
|
+
const opts = {
|
|
4976
|
+
ignoreDuration: merged.ignoreDuration,
|
|
4977
|
+
durationThresholdMs: merged.durationThresholdMs
|
|
4978
|
+
};
|
|
4979
|
+
const raw = [];
|
|
4980
|
+
if ((left.status ?? "") !== (right.status ?? "")) {
|
|
4981
|
+
raw.push({
|
|
4982
|
+
kind: "run-status",
|
|
4983
|
+
severity: "warning",
|
|
4984
|
+
message: "Run completion status differs",
|
|
4985
|
+
left: left.status,
|
|
4986
|
+
right: right.status
|
|
4987
|
+
});
|
|
4988
|
+
}
|
|
4989
|
+
{
|
|
4990
|
+
const ld = left.durationMs;
|
|
4991
|
+
const rd = right.durationMs;
|
|
4992
|
+
const th = merged.durationThresholdMs;
|
|
4993
|
+
let differs = false;
|
|
4994
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
4995
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
4996
|
+
else differs = Math.abs(ld - rd) > th;
|
|
4997
|
+
if (differs) {
|
|
4998
|
+
raw.push({
|
|
4999
|
+
kind: "duration",
|
|
5000
|
+
severity: "info",
|
|
5001
|
+
message: "Run duration differs",
|
|
5002
|
+
left: ld,
|
|
5003
|
+
right: rd
|
|
5004
|
+
});
|
|
5005
|
+
}
|
|
5006
|
+
}
|
|
5007
|
+
const pairs = pairSteps(left.steps, right.steps);
|
|
5008
|
+
let idx = 0;
|
|
5009
|
+
for (const [ls, rs] of pairs) {
|
|
5010
|
+
if (ls !== void 0 && rs !== void 0) {
|
|
5011
|
+
compareRecursive(ls, rs, [pathSeg(ls, idx)], opts, raw);
|
|
5012
|
+
idx += 1;
|
|
5013
|
+
} else if (ls !== void 0) {
|
|
5014
|
+
raw.push({
|
|
5015
|
+
kind: "step-removed",
|
|
5016
|
+
severity: "warning",
|
|
5017
|
+
message: `Step only in left run: ${ls.name}`,
|
|
5018
|
+
path: buildPath([pathSeg(ls, idx)]),
|
|
5019
|
+
left: ls.id,
|
|
5020
|
+
right: void 0
|
|
5021
|
+
});
|
|
5022
|
+
idx += 1;
|
|
5023
|
+
} else if (rs !== void 0) {
|
|
5024
|
+
raw.push({
|
|
5025
|
+
kind: "step-added",
|
|
5026
|
+
severity: "warning",
|
|
5027
|
+
message: `Step only in right run: ${rs.name}`,
|
|
5028
|
+
path: buildPath([pathSeg(rs, idx)]),
|
|
5029
|
+
left: void 0,
|
|
5030
|
+
right: rs.id
|
|
5031
|
+
});
|
|
5032
|
+
idx += 1;
|
|
5033
|
+
}
|
|
5034
|
+
}
|
|
5035
|
+
const differences = raw.filter((d) => kindMatchesFilter(d.kind));
|
|
5036
|
+
let errors = 0;
|
|
5037
|
+
let warnings = 0;
|
|
5038
|
+
let info = 0;
|
|
5039
|
+
for (const d of differences) {
|
|
5040
|
+
if (d.severity === "error") errors += 1;
|
|
5041
|
+
else if (d.severity === "warning") warnings += 1;
|
|
5042
|
+
else info += 1;
|
|
5043
|
+
}
|
|
5044
|
+
const firstVisible = differences[0];
|
|
5045
|
+
const firstDivergence = firstVisible !== void 0 ? {
|
|
5046
|
+
kind: "first-divergence",
|
|
5047
|
+
severity: firstVisible.severity,
|
|
5048
|
+
message: `First divergence: ${firstVisible.message}`,
|
|
5049
|
+
path: firstVisible.path,
|
|
5050
|
+
left: firstVisible.left,
|
|
5051
|
+
right: firstVisible.right
|
|
5052
|
+
} : void 0;
|
|
5053
|
+
const summary = {
|
|
5054
|
+
leftRunId: left.runId,
|
|
5055
|
+
rightRunId: right.runId,
|
|
5056
|
+
totalDifferences: differences.length,
|
|
5057
|
+
errors,
|
|
5058
|
+
warnings,
|
|
5059
|
+
info,
|
|
5060
|
+
firstDivergence
|
|
5061
|
+
};
|
|
5062
|
+
return { summary, differences };
|
|
5063
|
+
}
|
|
5064
|
+
|
|
5065
|
+
// packages/viewer/src/suite-data.ts
|
|
5066
|
+
async function enrichCase(suiteCase, baselineTracePath) {
|
|
5067
|
+
const base = {
|
|
5068
|
+
id: suiteCase.id,
|
|
5069
|
+
status: suiteCase.status,
|
|
5070
|
+
...suiteCase.tracePath !== void 0 ? { tracePath: suiteCase.tracePath } : {},
|
|
5071
|
+
...suiteCase.runId !== void 0 ? { runId: suiteCase.runId } : {},
|
|
5072
|
+
...suiteCase.message !== void 0 ? { message: suiteCase.message } : {},
|
|
5073
|
+
diagnostics: suiteCase.diagnostics,
|
|
5074
|
+
toolPath: [],
|
|
5075
|
+
observations: []
|
|
5076
|
+
};
|
|
5077
|
+
if (suiteCase.tracePath === void 0) return base;
|
|
5078
|
+
try {
|
|
5079
|
+
const read = await openTrace({ type: "file", path: suiteCase.tracePath });
|
|
5080
|
+
const legacy = persistedInspectEventsToTraceEvents(read.events);
|
|
5081
|
+
const timeline = buildRunTimeline(legacy, { focus: "all" });
|
|
5082
|
+
const observations = extractOutcomesFromTraceEvents(legacy);
|
|
5083
|
+
const toolPath = timeline.entries.filter((entry) => entry.type === "tool").map((entry) => entry.name);
|
|
5084
|
+
const detail = {
|
|
5085
|
+
...base,
|
|
5086
|
+
timeline,
|
|
5087
|
+
toolPath,
|
|
5088
|
+
observations
|
|
5089
|
+
};
|
|
5090
|
+
if (baselineTracePath !== void 0 && suiteCase.status !== "pass") {
|
|
5091
|
+
try {
|
|
5092
|
+
const baselineRead = await openTrace({ type: "file", path: baselineTracePath });
|
|
5093
|
+
const diff = diffRuns(
|
|
5094
|
+
manualTraceEventsToComparableRun(
|
|
5095
|
+
persistedInspectEventsToTraceEvents(baselineRead.events)
|
|
5096
|
+
),
|
|
5097
|
+
manualTraceEventsToComparableRun(legacy)
|
|
5098
|
+
);
|
|
5099
|
+
detail.failureDiff = {
|
|
5100
|
+
summary: diff.summary,
|
|
5101
|
+
differences: diff.differences.slice(0, 20).map((item) => ({
|
|
5102
|
+
kind: item.kind,
|
|
5103
|
+
message: item.message
|
|
5104
|
+
}))
|
|
5105
|
+
};
|
|
5106
|
+
} catch {
|
|
5107
|
+
}
|
|
5108
|
+
}
|
|
5109
|
+
return detail;
|
|
5110
|
+
} catch {
|
|
5111
|
+
return base;
|
|
5112
|
+
}
|
|
5113
|
+
}
|
|
5114
|
+
async function loadSuiteViewerData(options) {
|
|
5115
|
+
const result = await runSuite({
|
|
5116
|
+
configPath: options.suiteConfigPath,
|
|
5117
|
+
cwd: options.cwd
|
|
5118
|
+
});
|
|
5119
|
+
const baselineCase = result.cases.find((item) => item.status === "pass");
|
|
5120
|
+
const baselinePath = baselineCase?.tracePath;
|
|
5121
|
+
const cases = [];
|
|
5122
|
+
for (const suiteCase of result.cases) {
|
|
5123
|
+
cases.push(await enrichCase(suiteCase, baselinePath));
|
|
5124
|
+
}
|
|
5125
|
+
const artifactsDir = path13.join(path13.dirname(result.configPath), ".agent-inspect/suite-runs");
|
|
5126
|
+
return {
|
|
5127
|
+
suiteName: result.suiteName,
|
|
5128
|
+
configPath: result.configPath,
|
|
5129
|
+
tracesDir: result.tracesDir,
|
|
5130
|
+
ok: result.ok,
|
|
5131
|
+
status: result.status,
|
|
5132
|
+
summary: result.summary,
|
|
5133
|
+
cases,
|
|
5134
|
+
ciArtifactsDir: artifactsDir,
|
|
5135
|
+
bundleExportHint: "Export a share-safe bundle with: npx agent-inspect bundle <runId> --profile share"
|
|
5136
|
+
};
|
|
5137
|
+
}
|
|
5138
|
+
|
|
5139
|
+
// packages/core/src/workspace/types.ts
|
|
5140
|
+
var WORKSPACE_SCHEMA_VERSION = "1.0";
|
|
5141
|
+
var WORKSPACE_DIR_NAME = ".agent-inspect";
|
|
5142
|
+
var WORKSPACE_MANIFEST_FILENAME = "workspace.json";
|
|
5143
|
+
|
|
5144
|
+
// packages/core/src/workspace/manifest.ts
|
|
5145
|
+
var REDACTION_PROFILES = [
|
|
5146
|
+
"local",
|
|
5147
|
+
"share",
|
|
5148
|
+
"strict"
|
|
5149
|
+
];
|
|
5150
|
+
var INDEX_TYPES = ["none", "sqlite", "custom"];
|
|
5151
|
+
var MAX_WORKSPACE_MANIFEST_BYTES = 64 * 1024;
|
|
5152
|
+
function isPlainObject(value) {
|
|
5153
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
5154
|
+
}
|
|
5155
|
+
function isSafeRelativeWorkspacePath(p) {
|
|
5156
|
+
if (typeof p !== "string") return false;
|
|
5157
|
+
const trimmed = p.trim();
|
|
5158
|
+
if (trimmed === "") return false;
|
|
5159
|
+
if (trimmed.startsWith("/") || trimmed.startsWith("\\")) return false;
|
|
5160
|
+
if (/^[a-zA-Z]:/.test(trimmed)) return false;
|
|
5161
|
+
const segments = trimmed.split(/[/\\]+/);
|
|
5162
|
+
return !segments.some((seg) => seg === "..");
|
|
5163
|
+
}
|
|
5164
|
+
function validateDirField(value, field, errors) {
|
|
5165
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
5166
|
+
errors.push(`${field} must be a non-empty string`);
|
|
5167
|
+
return;
|
|
5168
|
+
}
|
|
5169
|
+
if (!isSafeRelativeWorkspacePath(value)) {
|
|
5170
|
+
errors.push(
|
|
5171
|
+
`${field} must be a relative path inside the workspace (no absolute paths or ".." traversal)`
|
|
5172
|
+
);
|
|
5173
|
+
}
|
|
5174
|
+
}
|
|
5175
|
+
function validateIndex(value, errors) {
|
|
5176
|
+
if (!isPlainObject(value)) {
|
|
5177
|
+
errors.push("index must be an object");
|
|
5178
|
+
return void 0;
|
|
5179
|
+
}
|
|
5180
|
+
if (typeof value.enabled !== "boolean") {
|
|
5181
|
+
errors.push("index.enabled must be a boolean");
|
|
5182
|
+
}
|
|
5183
|
+
if (!INDEX_TYPES.includes(value.type)) {
|
|
5184
|
+
errors.push(`index.type must be one of: ${INDEX_TYPES.join(", ")}`);
|
|
5185
|
+
}
|
|
5186
|
+
if (value.path !== void 0 && !isSafeRelativeWorkspacePath(value.path)) {
|
|
5187
|
+
errors.push(
|
|
5188
|
+
'index.path must be a relative path inside the workspace (no absolute paths or ".." traversal)'
|
|
5189
|
+
);
|
|
5190
|
+
}
|
|
5191
|
+
if (errors.length > 0) return void 0;
|
|
5192
|
+
return {
|
|
5193
|
+
enabled: value.enabled,
|
|
5194
|
+
type: value.type,
|
|
5195
|
+
...value.path !== void 0 ? { path: value.path } : {}
|
|
5196
|
+
};
|
|
5197
|
+
}
|
|
5198
|
+
function validateWorkspaceManifest(input) {
|
|
5199
|
+
const errors = [];
|
|
5200
|
+
const warnings = [];
|
|
5201
|
+
if (!isPlainObject(input)) {
|
|
5202
|
+
return { ok: false, errors: ["manifest must be an object"], warnings };
|
|
5203
|
+
}
|
|
5204
|
+
if (input.schemaVersion !== WORKSPACE_SCHEMA_VERSION) {
|
|
5205
|
+
errors.push(
|
|
5206
|
+
`schemaVersion must be "${WORKSPACE_SCHEMA_VERSION}" (received ${JSON.stringify(
|
|
5207
|
+
input.schemaVersion
|
|
5208
|
+
)})`
|
|
5209
|
+
);
|
|
5210
|
+
}
|
|
5211
|
+
if (typeof input.project !== "string" || input.project.trim() === "") {
|
|
5212
|
+
errors.push("project must be a non-empty string");
|
|
5213
|
+
}
|
|
5214
|
+
if (typeof input.createdAt !== "string" || input.createdAt.trim() === "") {
|
|
5215
|
+
errors.push("createdAt must be a non-empty ISO-8601 string");
|
|
5216
|
+
} else if (Number.isNaN(Date.parse(input.createdAt))) {
|
|
5217
|
+
errors.push("createdAt must be a valid ISO-8601 date string");
|
|
5218
|
+
}
|
|
5219
|
+
if (!Array.isArray(input.traceDirs) || input.traceDirs.length === 0) {
|
|
5220
|
+
errors.push("traceDirs must be a non-empty array");
|
|
5221
|
+
} else {
|
|
5222
|
+
input.traceDirs.forEach((dir, i) => {
|
|
5223
|
+
if (typeof dir !== "string" || dir.trim() === "") {
|
|
5224
|
+
errors.push(`traceDirs[${i}] must be a non-empty string`);
|
|
5225
|
+
} else if (!isSafeRelativeWorkspacePath(dir)) {
|
|
5226
|
+
errors.push(
|
|
5227
|
+
`traceDirs[${i}] must be a relative path inside the workspace (no absolute paths or ".." traversal)`
|
|
5228
|
+
);
|
|
5229
|
+
}
|
|
5230
|
+
});
|
|
5231
|
+
}
|
|
5232
|
+
validateDirField(input.reportsDir, "reportsDir", errors);
|
|
5233
|
+
validateDirField(input.artifactsDir, "artifactsDir", errors);
|
|
5234
|
+
validateDirField(input.bundlesDir, "bundlesDir", errors);
|
|
5235
|
+
validateDirField(input.notesDir, "notesDir", errors);
|
|
5236
|
+
if (!REDACTION_PROFILES.includes(input.redactionProfile)) {
|
|
5237
|
+
errors.push(`redactionProfile must be one of: ${REDACTION_PROFILES.join(", ")}`);
|
|
5238
|
+
}
|
|
5239
|
+
const indexErrors = [];
|
|
5240
|
+
const index = validateIndex(input.index, indexErrors);
|
|
5241
|
+
errors.push(...indexErrors);
|
|
5242
|
+
if (index && index.type !== "none" && !index.enabled) {
|
|
5243
|
+
warnings.push(`index.type is "${index.type}" but index.enabled is false`);
|
|
5244
|
+
}
|
|
5245
|
+
if (errors.length > 0 || index === void 0) {
|
|
5246
|
+
return { ok: false, errors, warnings };
|
|
5247
|
+
}
|
|
5248
|
+
const manifest = {
|
|
5249
|
+
schemaVersion: WORKSPACE_SCHEMA_VERSION,
|
|
5250
|
+
project: input.project.trim(),
|
|
5251
|
+
createdAt: input.createdAt,
|
|
5252
|
+
traceDirs: input.traceDirs.map((d) => d.trim()),
|
|
5253
|
+
reportsDir: input.reportsDir.trim(),
|
|
5254
|
+
artifactsDir: input.artifactsDir.trim(),
|
|
5255
|
+
bundlesDir: input.bundlesDir.trim(),
|
|
5256
|
+
notesDir: input.notesDir.trim(),
|
|
5257
|
+
redactionProfile: input.redactionProfile,
|
|
5258
|
+
index
|
|
5259
|
+
};
|
|
5260
|
+
return { ok: true, manifest, errors, warnings };
|
|
5261
|
+
}
|
|
5262
|
+
function parseWorkspaceManifest(json) {
|
|
5263
|
+
const warnings = [];
|
|
5264
|
+
if (typeof json !== "string") {
|
|
5265
|
+
return { ok: false, errors: ["manifest input must be a string"], warnings };
|
|
5266
|
+
}
|
|
5267
|
+
if (json.length > MAX_WORKSPACE_MANIFEST_BYTES) {
|
|
5268
|
+
return {
|
|
5269
|
+
ok: false,
|
|
5270
|
+
errors: [
|
|
5271
|
+
`manifest exceeds maximum size of ${MAX_WORKSPACE_MANIFEST_BYTES} bytes`
|
|
5272
|
+
],
|
|
5273
|
+
warnings
|
|
5274
|
+
};
|
|
5275
|
+
}
|
|
5276
|
+
let parsed;
|
|
5277
|
+
try {
|
|
5278
|
+
parsed = JSON.parse(json);
|
|
5279
|
+
} catch {
|
|
5280
|
+
return { ok: false, errors: ["manifest is not valid JSON"], warnings };
|
|
5281
|
+
}
|
|
5282
|
+
return validateWorkspaceManifest(parsed);
|
|
5283
|
+
}
|
|
5284
|
+
var INDEX_DIR_NAME = "index";
|
|
5285
|
+
function resolveWorkspaceLocation(cwd = process.cwd()) {
|
|
5286
|
+
const projectRoot = path13.resolve(cwd);
|
|
5287
|
+
const workspaceDir = path13.join(projectRoot, WORKSPACE_DIR_NAME);
|
|
5288
|
+
return {
|
|
5289
|
+
projectRoot,
|
|
5290
|
+
workspaceDir,
|
|
5291
|
+
manifestPath: path13.join(workspaceDir, WORKSPACE_MANIFEST_FILENAME)
|
|
5292
|
+
};
|
|
5293
|
+
}
|
|
5294
|
+
function resolveInsideWorkspace(workspaceDir, relative) {
|
|
5295
|
+
const base = path13.resolve(workspaceDir);
|
|
5296
|
+
const resolved = path13.resolve(base, relative);
|
|
5297
|
+
const rel = path13.relative(base, resolved);
|
|
5298
|
+
if (rel === "" || rel === "." || !rel.startsWith("..") && !path13.isAbsolute(rel)) {
|
|
5299
|
+
return resolved;
|
|
5300
|
+
}
|
|
5301
|
+
throw new Error(
|
|
5302
|
+
`Workspace path "${relative}" resolves outside the workspace directory`
|
|
5303
|
+
);
|
|
5304
|
+
}
|
|
5305
|
+
async function pathExists(p) {
|
|
5306
|
+
try {
|
|
5307
|
+
await access(p);
|
|
5308
|
+
return true;
|
|
5309
|
+
} catch {
|
|
5310
|
+
return false;
|
|
5311
|
+
}
|
|
5312
|
+
}
|
|
5313
|
+
async function isWritable(p) {
|
|
5314
|
+
try {
|
|
5315
|
+
await access(p, constants.W_OK);
|
|
5316
|
+
return true;
|
|
5317
|
+
} catch {
|
|
5318
|
+
return false;
|
|
5319
|
+
}
|
|
5320
|
+
}
|
|
5321
|
+
async function listJsonl(dir) {
|
|
5322
|
+
try {
|
|
5323
|
+
const entries = await readdir(dir);
|
|
5324
|
+
return entries.filter((f) => f.endsWith(".jsonl"));
|
|
5325
|
+
} catch {
|
|
5326
|
+
return [];
|
|
5327
|
+
}
|
|
5328
|
+
}
|
|
5329
|
+
async function countFiles(dir) {
|
|
5330
|
+
try {
|
|
5331
|
+
const entries = await readdir(dir, { withFileTypes: true });
|
|
5332
|
+
return entries.filter((e) => e.isFile()).length;
|
|
5333
|
+
} catch {
|
|
5334
|
+
return 0;
|
|
5335
|
+
}
|
|
5336
|
+
}
|
|
5337
|
+
async function readWorkspaceManifestFile(location) {
|
|
5338
|
+
let raw;
|
|
5339
|
+
try {
|
|
5340
|
+
raw = await readFile(location.manifestPath, "utf-8");
|
|
5341
|
+
} catch {
|
|
5342
|
+
return { exists: false, ok: false, errors: ["workspace.json not found"], warnings: [] };
|
|
5343
|
+
}
|
|
5344
|
+
const parsed = parseWorkspaceManifest(raw);
|
|
5345
|
+
return {
|
|
5346
|
+
exists: true,
|
|
5347
|
+
ok: parsed.ok,
|
|
5348
|
+
...parsed.manifest ? { manifest: parsed.manifest } : {},
|
|
5349
|
+
errors: parsed.errors,
|
|
5350
|
+
warnings: parsed.warnings
|
|
5351
|
+
};
|
|
5352
|
+
}
|
|
5353
|
+
async function getWorkspaceStatus(location, manifest) {
|
|
5354
|
+
let traceFiles = 0;
|
|
5355
|
+
for (const rel of manifest.traceDirs) {
|
|
5356
|
+
const abs = resolveInsideWorkspace(location.workspaceDir, rel);
|
|
5357
|
+
traceFiles += (await listJsonl(abs)).length;
|
|
5358
|
+
}
|
|
5359
|
+
const reports = await countFiles(
|
|
5360
|
+
resolveInsideWorkspace(location.workspaceDir, manifest.reportsDir)
|
|
5361
|
+
);
|
|
5362
|
+
const artifacts = await countFiles(
|
|
5363
|
+
resolveInsideWorkspace(location.workspaceDir, manifest.artifactsDir)
|
|
5364
|
+
);
|
|
5365
|
+
const bundles = await countFiles(
|
|
5366
|
+
resolveInsideWorkspace(location.workspaceDir, manifest.bundlesDir)
|
|
5367
|
+
);
|
|
5368
|
+
const notes = await countFiles(
|
|
5369
|
+
resolveInsideWorkspace(location.workspaceDir, manifest.notesDir)
|
|
5370
|
+
);
|
|
5371
|
+
const indexPath = manifest.index.path ? resolveInsideWorkspace(location.workspaceDir, manifest.index.path) : resolveInsideWorkspace(location.workspaceDir, INDEX_DIR_NAME);
|
|
5372
|
+
return {
|
|
5373
|
+
project: manifest.project,
|
|
5374
|
+
traceFiles,
|
|
5375
|
+
reports,
|
|
5376
|
+
artifacts,
|
|
5377
|
+
bundles,
|
|
5378
|
+
notes,
|
|
5379
|
+
index: {
|
|
5380
|
+
enabled: manifest.index.enabled,
|
|
5381
|
+
type: manifest.index.type,
|
|
5382
|
+
exists: await pathExists(indexPath)
|
|
5383
|
+
}
|
|
5384
|
+
};
|
|
5385
|
+
}
|
|
5386
|
+
async function doctorWorkspace(location) {
|
|
5387
|
+
const checks = [];
|
|
5388
|
+
const manifestResult = await readWorkspaceManifestFile(location);
|
|
5389
|
+
if (!manifestResult.exists) {
|
|
5390
|
+
checks.push({
|
|
5391
|
+
id: "manifest",
|
|
5392
|
+
status: "fail",
|
|
5393
|
+
message: "workspace.json not found (run `agent-inspect workspace init`)"
|
|
5394
|
+
});
|
|
5395
|
+
return { ok: false, checks };
|
|
5396
|
+
}
|
|
5397
|
+
if (!manifestResult.ok || !manifestResult.manifest) {
|
|
5398
|
+
checks.push({
|
|
5399
|
+
id: "manifest",
|
|
5400
|
+
status: "fail",
|
|
5401
|
+
message: `workspace.json is invalid: ${manifestResult.errors.join("; ")}`
|
|
5402
|
+
});
|
|
5403
|
+
return { ok: false, checks };
|
|
5404
|
+
}
|
|
5405
|
+
const manifest = manifestResult.manifest;
|
|
5406
|
+
checks.push({ id: "manifest", status: "pass", message: "workspace.json is valid" });
|
|
5407
|
+
for (const warning of manifestResult.warnings) {
|
|
5408
|
+
checks.push({ id: "manifest-warning", status: "warn", message: warning });
|
|
5409
|
+
}
|
|
5410
|
+
const dirFields = [
|
|
5411
|
+
...manifest.traceDirs.filter((d) => d !== ".").map((d, i) => [`traceDir[${i}]`, d]),
|
|
5412
|
+
["reportsDir", manifest.reportsDir],
|
|
5413
|
+
["artifactsDir", manifest.artifactsDir],
|
|
5414
|
+
["bundlesDir", manifest.bundlesDir],
|
|
5415
|
+
["notesDir", manifest.notesDir]
|
|
5416
|
+
];
|
|
5417
|
+
for (const [id, rel] of dirFields) {
|
|
5418
|
+
let abs;
|
|
5419
|
+
try {
|
|
5420
|
+
abs = resolveInsideWorkspace(location.workspaceDir, rel);
|
|
5421
|
+
} catch (error) {
|
|
5422
|
+
checks.push({
|
|
5423
|
+
id,
|
|
5424
|
+
status: "fail",
|
|
5425
|
+
message: error instanceof Error ? error.message : String(error)
|
|
5426
|
+
});
|
|
5427
|
+
continue;
|
|
5428
|
+
}
|
|
5429
|
+
if (!await pathExists(abs)) {
|
|
5430
|
+
checks.push({ id, status: "warn", message: `${rel}/ does not exist yet` });
|
|
5431
|
+
} else if (!await isWritable(abs)) {
|
|
5432
|
+
checks.push({ id, status: "fail", message: `${rel}/ is not writable` });
|
|
5433
|
+
} else {
|
|
5434
|
+
checks.push({ id, status: "pass", message: `${rel}/ is present and writable` });
|
|
5435
|
+
}
|
|
5436
|
+
}
|
|
5437
|
+
let newestTraceMtime = 0;
|
|
5438
|
+
for (const rel of manifest.traceDirs) {
|
|
5439
|
+
const abs = resolveInsideWorkspace(location.workspaceDir, rel);
|
|
5440
|
+
for (const file of await listJsonl(abs)) {
|
|
5441
|
+
try {
|
|
5442
|
+
const s = await stat(path13.join(abs, file));
|
|
5443
|
+
newestTraceMtime = Math.max(newestTraceMtime, s.mtimeMs);
|
|
5444
|
+
} catch {
|
|
5445
|
+
checks.push({ id: "trace-readability", status: "warn", message: `cannot stat ${rel}/${file}` });
|
|
5446
|
+
}
|
|
5447
|
+
}
|
|
5448
|
+
}
|
|
5449
|
+
if (manifest.index.enabled) {
|
|
5450
|
+
const indexPath = manifest.index.path ? resolveInsideWorkspace(location.workspaceDir, manifest.index.path) : resolveInsideWorkspace(location.workspaceDir, INDEX_DIR_NAME);
|
|
5451
|
+
if (!await pathExists(indexPath)) {
|
|
5452
|
+
checks.push({ id: "index", status: "warn", message: "index enabled but not built" });
|
|
5453
|
+
} else {
|
|
5454
|
+
try {
|
|
5455
|
+
const s = await stat(indexPath);
|
|
5456
|
+
if (newestTraceMtime > s.mtimeMs) {
|
|
5457
|
+
checks.push({ id: "index", status: "warn", message: "index is stale (traces are newer)" });
|
|
5458
|
+
} else {
|
|
5459
|
+
checks.push({ id: "index", status: "pass", message: "index is present" });
|
|
5460
|
+
}
|
|
5461
|
+
} catch {
|
|
5462
|
+
checks.push({ id: "index", status: "warn", message: "cannot stat index" });
|
|
5463
|
+
}
|
|
5464
|
+
}
|
|
5465
|
+
}
|
|
5466
|
+
const ok = !checks.some((c) => c.status === "fail");
|
|
5467
|
+
return { ok, checks };
|
|
5468
|
+
}
|
|
5469
|
+
|
|
5470
|
+
// packages/viewer/src/workspace-data.ts
|
|
5471
|
+
async function loadWorkspaceViewerData(options) {
|
|
5472
|
+
const cwd = path13.resolve(options.cwd ?? process.cwd());
|
|
5473
|
+
const location = resolveWorkspaceLocation(cwd);
|
|
5474
|
+
const manifestRead = await readWorkspaceManifestFile(location);
|
|
5475
|
+
if (!manifestRead.ok || manifestRead.manifest === void 0) {
|
|
5476
|
+
throw new Error(
|
|
5477
|
+
manifestRead.errors.join("; ") || "workspace.json not found (run workspace init)"
|
|
5478
|
+
);
|
|
5479
|
+
}
|
|
5480
|
+
const status = await getWorkspaceStatus(location, manifestRead.manifest);
|
|
5481
|
+
const doctor = await doctorWorkspace(location);
|
|
5482
|
+
const runs = [];
|
|
5483
|
+
for (const rel of manifestRead.manifest.traceDirs) {
|
|
5484
|
+
const traceDir = resolveTraceDir({
|
|
5485
|
+
dir: path13.join(location.workspaceDir, rel)
|
|
5486
|
+
});
|
|
5487
|
+
const td = new TraceDirectory({ dir: traceDir });
|
|
5488
|
+
const files = await td.list();
|
|
5489
|
+
const metas = await loadTraceMetadataList(
|
|
5490
|
+
traceDir,
|
|
5491
|
+
files,
|
|
5492
|
+
(fileName) => td.getPath(fileName)
|
|
5493
|
+
);
|
|
5494
|
+
for (const meta of metas) {
|
|
5495
|
+
runs.push({
|
|
5496
|
+
runId: meta.runId,
|
|
5497
|
+
...meta.name !== void 0 ? { name: meta.name } : {},
|
|
5498
|
+
status: meta.status,
|
|
5499
|
+
file: path13.basename(meta.filePath)
|
|
5500
|
+
});
|
|
5501
|
+
}
|
|
5502
|
+
}
|
|
5503
|
+
return {
|
|
5504
|
+
workspaceDir: location.workspaceDir,
|
|
5505
|
+
...status.project !== void 0 ? { project: status.project } : {},
|
|
5506
|
+
status,
|
|
5507
|
+
doctor,
|
|
5508
|
+
runs,
|
|
5509
|
+
bundleDirs: [
|
|
5510
|
+
path13.join(location.workspaceDir, manifestRead.manifest.bundlesDir),
|
|
5511
|
+
path13.join(location.workspaceDir, manifestRead.manifest.artifactsDir)
|
|
5512
|
+
]
|
|
5513
|
+
};
|
|
5514
|
+
}
|
|
5515
|
+
|
|
3894
5516
|
// packages/viewer/src/server.ts
|
|
3895
5517
|
var DEFAULT_HOST = "127.0.0.1";
|
|
3896
5518
|
var DEFAULT_PORT = 7337;
|
|
@@ -3926,6 +5548,7 @@ function createViewerServer(options = {}) {
|
|
|
3926
5548
|
const host = options.host ?? DEFAULT_HOST;
|
|
3927
5549
|
const port = options.port ?? DEFAULT_PORT;
|
|
3928
5550
|
const maxEvents = options.maxEvents ?? DEFAULT_MAX_EVENTS;
|
|
5551
|
+
const mode = options.mode ?? "traces";
|
|
3929
5552
|
if (host === "0.0.0.0") {
|
|
3930
5553
|
console.warn(
|
|
3931
5554
|
"[AgentInspect viewer] Binding to 0.0.0.0 exposes traces on the network. Use 127.0.0.1 unless you accept that risk."
|
|
@@ -3947,9 +5570,21 @@ function createViewerServer(options = {}) {
|
|
|
3947
5570
|
return sendJson(res, 200, {
|
|
3948
5571
|
ok: true,
|
|
3949
5572
|
readOnly: true,
|
|
3950
|
-
|
|
5573
|
+
mode,
|
|
5574
|
+
traceDir: path13.resolve(traceDir)
|
|
3951
5575
|
});
|
|
3952
5576
|
}
|
|
5577
|
+
if (pathname === "/api/suite" && mode === "suite") {
|
|
5578
|
+
const data = await loadSuiteViewerData({
|
|
5579
|
+
suiteConfigPath: options.suiteConfigPath,
|
|
5580
|
+
cwd: options.cwd
|
|
5581
|
+
});
|
|
5582
|
+
return sendJson(res, 200, data);
|
|
5583
|
+
}
|
|
5584
|
+
if (pathname === "/api/workspace" && mode === "workspace") {
|
|
5585
|
+
const data = await loadWorkspaceViewerData({ cwd: options.cwd });
|
|
5586
|
+
return sendJson(res, 200, data);
|
|
5587
|
+
}
|
|
3953
5588
|
const td = new TraceDirectory({ dir: traceDir });
|
|
3954
5589
|
if (pathname === "/api/traces") {
|
|
3955
5590
|
const files = await td.list();
|
|
@@ -3965,7 +5600,7 @@ function createViewerServer(options = {}) {
|
|
|
3965
5600
|
runId: meta.runId,
|
|
3966
5601
|
name: meta.name,
|
|
3967
5602
|
status: meta.status,
|
|
3968
|
-
file:
|
|
5603
|
+
file: path13.basename(meta.filePath),
|
|
3969
5604
|
startedAt: meta.startedAt,
|
|
3970
5605
|
durationMs: meta.durationMs
|
|
3971
5606
|
}))
|
|
@@ -4071,22 +5706,25 @@ function startViewerServer(options = {}) {
|
|
|
4071
5706
|
const host = options.host ?? DEFAULT_HOST;
|
|
4072
5707
|
const port = options.port ?? DEFAULT_PORT;
|
|
4073
5708
|
const traceDir = resolveTraceDir({ dir: options.traceDir });
|
|
5709
|
+
const mode = options.mode ?? "traces";
|
|
4074
5710
|
const server = createViewerServer(options);
|
|
4075
5711
|
return new Promise((resolve, reject) => {
|
|
4076
5712
|
server.once("error", reject);
|
|
4077
5713
|
server.listen(port, host, () => {
|
|
4078
5714
|
const address = server.address();
|
|
4079
5715
|
const resolvedPort = typeof address === "object" && address ? address.port : port;
|
|
5716
|
+
const modeQuery = mode === "traces" ? "" : `?mode=${encodeURIComponent(mode)}`;
|
|
4080
5717
|
resolve({
|
|
4081
5718
|
host,
|
|
4082
5719
|
port: resolvedPort,
|
|
4083
|
-
traceDir:
|
|
4084
|
-
url: `http://${host}:${resolvedPort}
|
|
5720
|
+
traceDir: path13.resolve(traceDir),
|
|
5721
|
+
url: `http://${host}:${resolvedPort}/${modeQuery}`,
|
|
5722
|
+
mode
|
|
4085
5723
|
});
|
|
4086
5724
|
});
|
|
4087
5725
|
});
|
|
4088
5726
|
}
|
|
4089
5727
|
|
|
4090
|
-
export { createViewerServer, startViewerServer, viewerIndexHtml };
|
|
5728
|
+
export { createViewerServer, loadSuiteViewerData, loadWorkspaceViewerData, startViewerServer, viewerIndexHtml };
|
|
4091
5729
|
//# sourceMappingURL=index.mjs.map
|
|
4092
5730
|
//# sourceMappingURL=index.mjs.map
|