@agent-inspect/viewer 6.27.0 → 6.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1502 -1488
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +5 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.mjs +1502 -1488
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -172,6 +172,8 @@ function isPersistedTokenUsage(value) {
|
|
|
172
172
|
if (!isOptionalNonNegativeNumber(value.output)) return false;
|
|
173
173
|
if (!isOptionalNonNegativeNumber(value.total)) return false;
|
|
174
174
|
if (!isOptionalNonNegativeNumber(value.cached)) return false;
|
|
175
|
+
if (!isOptionalNonNegativeNumber(value.cacheWrite)) return false;
|
|
176
|
+
if (!isOptionalNonNegativeNumber(value.reasoning)) return false;
|
|
175
177
|
return true;
|
|
176
178
|
}
|
|
177
179
|
function isPersistedTraceContext(value) {
|
|
@@ -1594,1639 +1596,1639 @@ function buildSessionIndex(inputRuns, options = {}) {
|
|
|
1594
1596
|
};
|
|
1595
1597
|
}
|
|
1596
1598
|
|
|
1597
|
-
// packages/core/src/
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1599
|
+
// packages/core/src/diagnostics/programmatic.ts
|
|
1600
|
+
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
1601
|
+
AI_TRACE_INPUT_INVALID: {
|
|
1602
|
+
code: "AI_TRACE_INPUT_INVALID",
|
|
1603
|
+
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
1604
|
+
remediation: "For a file path, use openTraceFile(path).",
|
|
1605
|
+
relatedCodes: ["invalid_input"]
|
|
1606
|
+
},
|
|
1607
|
+
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
1608
|
+
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
1609
|
+
summary: "No trace reader could detect the input format.",
|
|
1610
|
+
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
1611
|
+
relatedCodes: ["unsupported_format"]
|
|
1612
|
+
},
|
|
1613
|
+
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
1614
|
+
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
1615
|
+
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
1616
|
+
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
1617
|
+
relatedCodes: ["ambiguous_format"]
|
|
1618
|
+
},
|
|
1619
|
+
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
1620
|
+
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
1621
|
+
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
1622
|
+
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
1623
|
+
},
|
|
1624
|
+
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
1625
|
+
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
1626
|
+
summary: "Multiple runs are available; select a run before executing checks.",
|
|
1627
|
+
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
1628
|
+
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
1629
|
+
},
|
|
1630
|
+
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
1631
|
+
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1632
|
+
summary: "A parentId equals its own event/step id.",
|
|
1633
|
+
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
1634
|
+
relatedCodes: [
|
|
1635
|
+
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
1636
|
+
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
1637
|
+
]
|
|
1638
|
+
},
|
|
1639
|
+
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
1640
|
+
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
1641
|
+
summary: "Trace contains a parentId cycle.",
|
|
1642
|
+
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
1643
|
+
relatedCodes: ["structure.cycle"]
|
|
1605
1644
|
}
|
|
1606
|
-
|
|
1645
|
+
});
|
|
1646
|
+
function formatProgrammaticDiagnostic(code, detail) {
|
|
1647
|
+
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
1648
|
+
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
1649
|
+
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
1607
1650
|
}
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1651
|
+
|
|
1652
|
+
// packages/core/src/safety/sensitive-key.ts
|
|
1653
|
+
function normalizeSensitiveKey(value) {
|
|
1654
|
+
return value.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
1611
1655
|
}
|
|
1656
|
+
new Set(
|
|
1657
|
+
[
|
|
1658
|
+
"tokens",
|
|
1659
|
+
"max_tokens",
|
|
1660
|
+
"min_tokens",
|
|
1661
|
+
"ls_max_tokens",
|
|
1662
|
+
"token_count",
|
|
1663
|
+
"token_limit",
|
|
1664
|
+
"token_budget",
|
|
1665
|
+
"input_tokens",
|
|
1666
|
+
"output_tokens",
|
|
1667
|
+
"total_tokens",
|
|
1668
|
+
"cached_tokens",
|
|
1669
|
+
"prompt_tokens",
|
|
1670
|
+
"completion_tokens"
|
|
1671
|
+
].map(normalizeSensitiveKey)
|
|
1672
|
+
);
|
|
1612
1673
|
|
|
1613
|
-
// packages/core/src/
|
|
1614
|
-
function
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1674
|
+
// packages/core/src/checks/logical-events.ts
|
|
1675
|
+
function isRecord5(value) {
|
|
1676
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1677
|
+
}
|
|
1678
|
+
function legacyEvent(event) {
|
|
1679
|
+
const value = event.attributes?.legacyEvent;
|
|
1680
|
+
return typeof value === "string" ? value : void 0;
|
|
1681
|
+
}
|
|
1682
|
+
function stepIdOf(event) {
|
|
1683
|
+
const value = event.attributes?.stepId;
|
|
1684
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
1618
1685
|
return void 0;
|
|
1619
1686
|
}
|
|
1620
|
-
function
|
|
1621
|
-
|
|
1622
|
-
|
|
1687
|
+
function cloneEvent(event) {
|
|
1688
|
+
return {
|
|
1689
|
+
...event,
|
|
1690
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
1691
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
1692
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
1693
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
1694
|
+
};
|
|
1623
1695
|
}
|
|
1624
|
-
function
|
|
1625
|
-
const
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
}
|
|
1629
|
-
const rs = started;
|
|
1630
|
-
const runId = rs.runId;
|
|
1631
|
-
const completedAll = events.filter((e) => e.event === "run_completed");
|
|
1632
|
-
const lastCompleted = completedAll[completedAll.length - 1];
|
|
1633
|
-
let runStatus;
|
|
1634
|
-
if (lastCompleted === void 0) runStatus = "running";
|
|
1635
|
-
else runStatus = lastCompleted.status;
|
|
1636
|
-
const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
1637
|
-
const steps = /* @__PURE__ */ new Map();
|
|
1638
|
-
let order = 0;
|
|
1639
|
-
for (const e of events) {
|
|
1640
|
-
if (e.event !== "step_started") continue;
|
|
1641
|
-
const s = e;
|
|
1642
|
-
const meta = s.metadata ? { ...s.metadata } : void 0;
|
|
1643
|
-
steps.set(s.stepId, {
|
|
1644
|
-
id: s.stepId,
|
|
1645
|
-
parentId: s.parentId,
|
|
1646
|
-
name: s.name,
|
|
1647
|
-
type: s.type,
|
|
1648
|
-
order: order++,
|
|
1649
|
-
timestamp: s.timestamp,
|
|
1650
|
-
metadata: meta
|
|
1651
|
-
});
|
|
1652
|
-
}
|
|
1653
|
-
for (const e of events) {
|
|
1654
|
-
if (e.event !== "step_completed") continue;
|
|
1655
|
-
const acc = steps.get(e.stepId);
|
|
1656
|
-
if (!acc) continue;
|
|
1657
|
-
acc.status = e.status;
|
|
1658
|
-
acc.durationMs = e.durationMs;
|
|
1659
|
-
if (e.error?.message) acc.errorMsg = e.error.message;
|
|
1660
|
-
const extra = e;
|
|
1661
|
-
if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
|
|
1662
|
-
acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
|
|
1663
|
-
}
|
|
1664
|
-
}
|
|
1665
|
-
const nodes = /* @__PURE__ */ new Map();
|
|
1666
|
-
for (const acc of steps.values()) {
|
|
1667
|
-
let meta = acc.metadata ? { ...acc.metadata } : void 0;
|
|
1668
|
-
if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
|
|
1669
|
-
meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
|
|
1670
|
-
}
|
|
1671
|
-
const outputPreview = extractOutputPreview(meta);
|
|
1672
|
-
if (meta !== void 0 && ("outputPreview" in meta || "resultPreview" in meta)) {
|
|
1673
|
-
delete meta.outputPreview;
|
|
1674
|
-
delete meta.resultPreview;
|
|
1675
|
-
}
|
|
1676
|
-
const sc = {
|
|
1677
|
-
id: acc.id,
|
|
1678
|
-
name: acc.name,
|
|
1679
|
-
type: acc.type,
|
|
1680
|
-
status: mapStepStatus(acc.status),
|
|
1681
|
-
durationMs: acc.durationMs,
|
|
1682
|
-
error: acc.errorMsg,
|
|
1683
|
-
metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
|
|
1684
|
-
outputPreview,
|
|
1685
|
-
children: []
|
|
1686
|
-
};
|
|
1687
|
-
nodes.set(acc.id, sc);
|
|
1688
|
-
}
|
|
1689
|
-
const roots = [];
|
|
1690
|
-
const sortByOrder = (a, b) => {
|
|
1691
|
-
const oa = steps.get(a.id)?.order ?? 0;
|
|
1692
|
-
const ob = steps.get(b.id)?.order ?? 0;
|
|
1693
|
-
return oa - ob;
|
|
1696
|
+
function mergeAttributes(start, complete) {
|
|
1697
|
+
const merged = {
|
|
1698
|
+
...isRecord5(complete.attributes) ? complete.attributes : {},
|
|
1699
|
+
...isRecord5(start.attributes) ? start.attributes : {}
|
|
1694
1700
|
};
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
} else {
|
|
1700
|
-
roots.push(node);
|
|
1701
|
-
}
|
|
1702
|
-
}
|
|
1703
|
-
roots.sort(sortByOrder);
|
|
1704
|
-
for (const n of nodes.values()) {
|
|
1705
|
-
n.children.sort(sortByOrder);
|
|
1701
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
1702
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
1703
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
1704
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
1706
1705
|
}
|
|
1706
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
1707
|
+
}
|
|
1708
|
+
function pairStartComplete(start, complete) {
|
|
1709
|
+
const attributes = mergeAttributes(start, complete);
|
|
1710
|
+
const paired = {
|
|
1711
|
+
...cloneEvent(start),
|
|
1712
|
+
status: complete.status,
|
|
1713
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
1714
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
1715
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
1716
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
1717
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
1718
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
1719
|
+
};
|
|
1707
1720
|
return {
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1721
|
+
...paired,
|
|
1722
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
1723
|
+
projection: {
|
|
1724
|
+
paired: true,
|
|
1725
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
1726
|
+
parentNormalized: false,
|
|
1727
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
1728
|
+
}
|
|
1713
1729
|
};
|
|
1714
1730
|
}
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
}
|
|
1724
|
-
function pairSteps(left, right) {
|
|
1725
|
-
const usedRight = /* @__PURE__ */ new Set();
|
|
1726
|
-
const pairs = [];
|
|
1727
|
-
for (let i = 0; i < left.length; i++) {
|
|
1728
|
-
const L = left[i];
|
|
1729
|
-
let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
|
|
1730
|
-
if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
|
|
1731
|
-
const cand = right[i];
|
|
1732
|
-
if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
|
|
1733
|
-
R = cand;
|
|
1734
|
-
}
|
|
1731
|
+
function asLogical(event, extras) {
|
|
1732
|
+
return {
|
|
1733
|
+
...cloneEvent(event),
|
|
1734
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
1735
|
+
projection: {
|
|
1736
|
+
paired: false,
|
|
1737
|
+
absorbedEventIds: Object.freeze([]),
|
|
1738
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
1739
|
+
...{}
|
|
1735
1740
|
}
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1741
|
+
};
|
|
1742
|
+
}
|
|
1743
|
+
function projectLogicalEvents(events) {
|
|
1744
|
+
const diagnostics = [];
|
|
1745
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
1746
|
+
for (const event of events) {
|
|
1747
|
+
const list = byRun.get(event.runId) ?? [];
|
|
1748
|
+
list.push(event);
|
|
1749
|
+
byRun.set(event.runId, list);
|
|
1750
|
+
}
|
|
1751
|
+
const absorbedIds = /* @__PURE__ */ new Set();
|
|
1752
|
+
const logicalByRawId = /* @__PURE__ */ new Map();
|
|
1753
|
+
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
1754
|
+
const logical = [];
|
|
1755
|
+
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
1756
|
+
for (const runId of orderedRuns) {
|
|
1757
|
+
const runEvents = byRun.get(runId) ?? [];
|
|
1758
|
+
const starts = [];
|
|
1759
|
+
const completes = [];
|
|
1760
|
+
const others = [];
|
|
1761
|
+
for (const event of runEvents) {
|
|
1762
|
+
const legacy = legacyEvent(event);
|
|
1763
|
+
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
1764
|
+
starts.push(event);
|
|
1765
|
+
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
1766
|
+
completes.push(event);
|
|
1767
|
+
} else {
|
|
1768
|
+
others.push(event);
|
|
1769
|
+
}
|
|
1740
1770
|
}
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1771
|
+
const usedCompletes = /* @__PURE__ */ new Set();
|
|
1772
|
+
for (const start of starts) {
|
|
1773
|
+
const stepId = stepIdOf(start);
|
|
1774
|
+
let match;
|
|
1775
|
+
if (legacyEvent(start) === "run_started") {
|
|
1776
|
+
const candidates = completes.filter(
|
|
1777
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
1778
|
+
);
|
|
1779
|
+
if (candidates.length > 1) {
|
|
1780
|
+
diagnostics.push({
|
|
1781
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1782
|
+
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
1783
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1784
|
+
});
|
|
1785
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1786
|
+
}
|
|
1787
|
+
match = candidates[0];
|
|
1788
|
+
} else if (stepId) {
|
|
1789
|
+
const candidates = completes.filter(
|
|
1790
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
1791
|
+
);
|
|
1792
|
+
if (candidates.length > 1) {
|
|
1793
|
+
diagnostics.push({
|
|
1794
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1795
|
+
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
1796
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1797
|
+
});
|
|
1798
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1799
|
+
}
|
|
1800
|
+
match = candidates[0];
|
|
1801
|
+
}
|
|
1802
|
+
if (match) {
|
|
1803
|
+
usedCompletes.add(match.eventId);
|
|
1804
|
+
absorbedIds.add(match.eventId);
|
|
1805
|
+
const paired = pairStartComplete(start, match);
|
|
1806
|
+
logical.push(paired);
|
|
1807
|
+
logicalByRawId.set(start.eventId, paired);
|
|
1808
|
+
logicalByRawId.set(match.eventId, paired);
|
|
1809
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
1810
|
+
} else {
|
|
1811
|
+
diagnostics.push({
|
|
1812
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
1813
|
+
message: `No matching complete for start ${start.eventId}.`,
|
|
1814
|
+
eventIds: [start.eventId]
|
|
1815
|
+
});
|
|
1816
|
+
const alone = asLogical(start);
|
|
1817
|
+
logical.push(alone);
|
|
1818
|
+
logicalByRawId.set(start.eventId, alone);
|
|
1819
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1820
|
+
}
|
|
1746
1821
|
}
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1822
|
+
for (const complete of completes) {
|
|
1823
|
+
if (usedCompletes.has(complete.eventId)) continue;
|
|
1824
|
+
diagnostics.push({
|
|
1825
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
1826
|
+
message: `No matching start for complete ${complete.eventId}.`,
|
|
1827
|
+
eventIds: [complete.eventId]
|
|
1828
|
+
});
|
|
1829
|
+
const alone = asLogical(complete);
|
|
1830
|
+
logical.push(alone);
|
|
1831
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
1832
|
+
const stepId = stepIdOf(complete);
|
|
1833
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1834
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1835
|
+
}
|
|
1836
|
+
}
|
|
1837
|
+
for (const event of others) {
|
|
1838
|
+
const alone = asLogical(event);
|
|
1839
|
+
logical.push(alone);
|
|
1840
|
+
logicalByRawId.set(event.eventId, alone);
|
|
1841
|
+
const stepId = stepIdOf(event);
|
|
1842
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1843
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1844
|
+
}
|
|
1751
1845
|
}
|
|
1752
1846
|
}
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1847
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
1848
|
+
const normalized = [];
|
|
1849
|
+
for (const event of logical) {
|
|
1850
|
+
const originalParentId = event.parentId;
|
|
1851
|
+
if (!originalParentId) {
|
|
1852
|
+
normalized.push(event);
|
|
1853
|
+
continue;
|
|
1854
|
+
}
|
|
1855
|
+
let nextParent = originalParentId;
|
|
1856
|
+
let remapped = false;
|
|
1857
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
1858
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
1859
|
+
nextParent = viaAbsorbed.eventId;
|
|
1860
|
+
remapped = true;
|
|
1861
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
1862
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
1863
|
+
if (viaStep) {
|
|
1864
|
+
nextParent = viaStep;
|
|
1865
|
+
remapped = true;
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1868
|
+
if (!remapped) {
|
|
1869
|
+
if (originalParentId === event.eventId) {
|
|
1870
|
+
diagnostics.push({
|
|
1871
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1872
|
+
message: formatProgrammaticDiagnostic(
|
|
1873
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1874
|
+
`Removed self-parent edge on ${event.eventId}.`
|
|
1875
|
+
),
|
|
1876
|
+
eventIds: [event.eventId]
|
|
1877
|
+
});
|
|
1878
|
+
const { parentId: _drop, ...rest } = event;
|
|
1879
|
+
normalized.push({
|
|
1880
|
+
...rest,
|
|
1881
|
+
projection: {
|
|
1882
|
+
...event.projection,
|
|
1883
|
+
parentNormalized: true,
|
|
1884
|
+
originalParentId
|
|
1885
|
+
}
|
|
1886
|
+
});
|
|
1887
|
+
continue;
|
|
1888
|
+
}
|
|
1889
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
1890
|
+
const mapping = event.attributes?.parentMapping;
|
|
1891
|
+
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
1892
|
+
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
1893
|
+
if (!unresolved) {
|
|
1894
|
+
diagnostics.push({
|
|
1895
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
1896
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
1897
|
+
eventIds: [event.eventId]
|
|
1898
|
+
});
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
normalized.push(event);
|
|
1902
|
+
continue;
|
|
1903
|
+
}
|
|
1904
|
+
if (nextParent === event.eventId) {
|
|
1905
|
+
diagnostics.push({
|
|
1906
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1907
|
+
message: formatProgrammaticDiagnostic(
|
|
1908
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1909
|
+
`Removed self-parent edge on ${event.eventId} (was ${originalParentId}).`
|
|
1910
|
+
),
|
|
1911
|
+
eventIds: [event.eventId]
|
|
1912
|
+
});
|
|
1913
|
+
const { parentId: _drop, ...rest } = event;
|
|
1914
|
+
normalized.push({
|
|
1915
|
+
...rest,
|
|
1916
|
+
projection: {
|
|
1917
|
+
...event.projection,
|
|
1918
|
+
parentNormalized: true,
|
|
1919
|
+
originalParentId
|
|
1920
|
+
}
|
|
1921
|
+
});
|
|
1922
|
+
continue;
|
|
1923
|
+
}
|
|
1924
|
+
diagnostics.push({
|
|
1925
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
1926
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
1927
|
+
eventIds: [event.eventId]
|
|
1765
1928
|
});
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
right: R.type
|
|
1929
|
+
normalized.push({
|
|
1930
|
+
...event,
|
|
1931
|
+
parentId: nextParent,
|
|
1932
|
+
projection: {
|
|
1933
|
+
...event.projection,
|
|
1934
|
+
parentNormalized: true,
|
|
1935
|
+
originalParentId
|
|
1936
|
+
}
|
|
1775
1937
|
});
|
|
1776
1938
|
}
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1939
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
1940
|
+
normalized.sort((a, b) => {
|
|
1941
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
1942
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
1943
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
1944
|
+
});
|
|
1945
|
+
return {
|
|
1946
|
+
logicalEvents: Object.freeze(normalized),
|
|
1947
|
+
diagnostics: Object.freeze(diagnostics)
|
|
1948
|
+
};
|
|
1949
|
+
}
|
|
1950
|
+
function resolveCanonicalToolName(event) {
|
|
1951
|
+
const attrs = event.attributes;
|
|
1952
|
+
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
1953
|
+
if (direct) return direct;
|
|
1954
|
+
const metadata = attrs?.metadata;
|
|
1955
|
+
if (isRecord5(metadata)) {
|
|
1956
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
1957
|
+
if (nested) return nested;
|
|
1786
1958
|
}
|
|
1787
|
-
const
|
|
1788
|
-
|
|
1789
|
-
if (le !== re) {
|
|
1790
|
-
out.push({
|
|
1791
|
-
kind: "error",
|
|
1792
|
-
severity: "error",
|
|
1793
|
-
message: "Step error message differs",
|
|
1794
|
-
path: path18,
|
|
1795
|
-
left: le || void 0,
|
|
1796
|
-
right: re || void 0
|
|
1797
|
-
});
|
|
1798
|
-
}
|
|
1799
|
-
if (!opts.ignoreDuration) {
|
|
1800
|
-
const ld = L.durationMs;
|
|
1801
|
-
const rd = R.durationMs;
|
|
1802
|
-
const th = opts.durationThresholdMs;
|
|
1803
|
-
let differs = false;
|
|
1804
|
-
if (ld === void 0 && rd === void 0) differs = false;
|
|
1805
|
-
else if (ld === void 0 || rd === void 0) differs = true;
|
|
1806
|
-
else differs = Math.abs(ld - rd) > th;
|
|
1807
|
-
if (differs) {
|
|
1808
|
-
out.push({
|
|
1809
|
-
kind: "duration",
|
|
1810
|
-
severity: "info",
|
|
1811
|
-
message: "Step duration differs",
|
|
1812
|
-
path: path18,
|
|
1813
|
-
left: ld,
|
|
1814
|
-
right: rd
|
|
1815
|
-
});
|
|
1816
|
-
}
|
|
1817
|
-
}
|
|
1818
|
-
const lm = stableJson(L.metadata ?? {});
|
|
1819
|
-
const rm2 = stableJson(R.metadata ?? {});
|
|
1820
|
-
if (lm !== rm2) {
|
|
1821
|
-
out.push({
|
|
1822
|
-
kind: "metadata",
|
|
1823
|
-
severity: "info",
|
|
1824
|
-
message: "Step metadata differs",
|
|
1825
|
-
path: path18,
|
|
1826
|
-
left: L.metadata,
|
|
1827
|
-
right: R.metadata
|
|
1828
|
-
});
|
|
1829
|
-
}
|
|
1830
|
-
const lo = stableJson(L.outputPreview ?? null);
|
|
1831
|
-
const ro = stableJson(R.outputPreview ?? null);
|
|
1832
|
-
if (lo !== ro) {
|
|
1833
|
-
out.push({
|
|
1834
|
-
kind: "output",
|
|
1835
|
-
severity: "info",
|
|
1836
|
-
message: "Output preview differs",
|
|
1837
|
-
path: path18,
|
|
1838
|
-
left: L.outputPreview,
|
|
1839
|
-
right: R.outputPreview
|
|
1840
|
-
});
|
|
1959
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
1960
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
1841
1961
|
}
|
|
1962
|
+
return event.name;
|
|
1842
1963
|
}
|
|
1843
|
-
function
|
|
1844
|
-
|
|
1845
|
-
const
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
if (lch !== void 0 && rch !== void 0) {
|
|
1849
|
-
compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
|
|
1850
|
-
} else if (lch !== void 0) {
|
|
1851
|
-
out.push({
|
|
1852
|
-
kind: "step-removed",
|
|
1853
|
-
severity: "warning",
|
|
1854
|
-
message: `Step only in left run: ${lch.name}`,
|
|
1855
|
-
path: buildPath([...segments, pathSeg(lch, ci)]),
|
|
1856
|
-
left: lch.id,
|
|
1857
|
-
right: void 0
|
|
1858
|
-
});
|
|
1859
|
-
} else if (rch !== void 0) {
|
|
1860
|
-
out.push({
|
|
1861
|
-
kind: "step-added",
|
|
1862
|
-
severity: "warning",
|
|
1863
|
-
message: `Step only in right run: ${rch.name}`,
|
|
1864
|
-
path: buildPath([...segments, pathSeg(rch, ci)]),
|
|
1865
|
-
left: void 0,
|
|
1866
|
-
right: rch.id
|
|
1867
|
-
});
|
|
1868
|
-
}
|
|
1869
|
-
ci += 1;
|
|
1964
|
+
function pickString(record, keys) {
|
|
1965
|
+
if (!record) return void 0;
|
|
1966
|
+
for (const key of keys) {
|
|
1967
|
+
const value = record[key];
|
|
1968
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
1870
1969
|
}
|
|
1970
|
+
return void 0;
|
|
1871
1971
|
}
|
|
1872
|
-
|
|
1972
|
+
|
|
1973
|
+
// packages/core/src/checks/trace-facts.ts
|
|
1974
|
+
formatProgrammaticDiagnostic(
|
|
1975
|
+
"AI_TRACE_FACTS_INPUT_NOT_NORMALIZED"
|
|
1976
|
+
);
|
|
1977
|
+
|
|
1978
|
+
// packages/core/src/checks/index.ts
|
|
1979
|
+
var SEVERITY_RANK = {
|
|
1980
|
+
error: 0,
|
|
1981
|
+
warning: 1,
|
|
1982
|
+
info: 2
|
|
1983
|
+
};
|
|
1984
|
+
var STATUS_RANK = {
|
|
1985
|
+
fail: 0,
|
|
1986
|
+
warning: 1,
|
|
1987
|
+
pass: 2
|
|
1988
|
+
};
|
|
1989
|
+
function compareStrings(a, b) {
|
|
1990
|
+
return (a ?? "").localeCompare(b ?? "");
|
|
1991
|
+
}
|
|
1992
|
+
function diagnostic(code, message, ruleId) {
|
|
1873
1993
|
return {
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1994
|
+
code,
|
|
1995
|
+
message,
|
|
1996
|
+
severity: "error",
|
|
1997
|
+
...ruleId ? { ruleId } : {}
|
|
1878
1998
|
};
|
|
1879
1999
|
}
|
|
1880
|
-
function
|
|
1881
|
-
return
|
|
2000
|
+
function emptySummary() {
|
|
2001
|
+
return {
|
|
2002
|
+
passed: 0,
|
|
2003
|
+
failed: 0,
|
|
2004
|
+
warnings: 0,
|
|
2005
|
+
errors: 0,
|
|
2006
|
+
rulesEvaluated: 0
|
|
2007
|
+
};
|
|
1882
2008
|
}
|
|
1883
|
-
function
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
2009
|
+
function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
2010
|
+
return {
|
|
2011
|
+
ok: false,
|
|
2012
|
+
status: "error",
|
|
2013
|
+
format: input.read.format,
|
|
2014
|
+
...selectedRun ? { runId: selectedRun.runId } : {},
|
|
2015
|
+
summary: {
|
|
2016
|
+
...emptySummary(),
|
|
2017
|
+
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2018
|
+
rulesEvaluated: ruleExecutions.length
|
|
2019
|
+
},
|
|
2020
|
+
findings: [],
|
|
2021
|
+
diagnostics: [...diagnostics],
|
|
2022
|
+
ruleExecutions: [...ruleExecutions]
|
|
1888
2023
|
};
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
2024
|
+
}
|
|
2025
|
+
function flattenNodes(nodes) {
|
|
2026
|
+
return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
|
|
2027
|
+
}
|
|
2028
|
+
function buildFacts(input, selectedRun) {
|
|
2029
|
+
const scopedRuns = selectedRun ? [selectedRun] : input.read.runs;
|
|
2030
|
+
const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
|
|
2031
|
+
const scopedEvents = selectedRun === void 0 ? input.read.events : input.read.events.filter((event) => scopedRunIds.has(event.runId));
|
|
2032
|
+
const nodes = flattenNodes(scopedRuns.flatMap((run) => run.children));
|
|
2033
|
+
const nodesByEventId = /* @__PURE__ */ new Map();
|
|
2034
|
+
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
2035
|
+
for (const node of nodes) {
|
|
2036
|
+
nodesByEventId.set(node.event.eventId, node);
|
|
2037
|
+
const parentId = node.event.parentId;
|
|
2038
|
+
if (parentId) {
|
|
2039
|
+
const children = childrenByParentId.get(parentId) ?? [];
|
|
2040
|
+
children.push(node);
|
|
2041
|
+
childrenByParentId.set(parentId, children);
|
|
2042
|
+
}
|
|
1898
2043
|
}
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
2044
|
+
const projection = projectLogicalEvents(scopedEvents);
|
|
2045
|
+
return {
|
|
2046
|
+
format: input.read.format,
|
|
2047
|
+
runs: Object.freeze([...input.read.runs]),
|
|
2048
|
+
events: Object.freeze([...scopedEvents]),
|
|
2049
|
+
logicalEvents: projection.logicalEvents,
|
|
2050
|
+
logicalProjectionDiagnostics: projection.diagnostics,
|
|
2051
|
+
readerWarnings: Object.freeze([...input.read.warnings]),
|
|
2052
|
+
unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
|
|
2053
|
+
sourceFiles: Object.freeze([...input.read.sourceFiles]),
|
|
2054
|
+
nodesByEventId,
|
|
2055
|
+
childrenByParentId,
|
|
2056
|
+
rootNodes: Object.freeze(scopedRuns.flatMap((run) => run.children))
|
|
2057
|
+
};
|
|
2058
|
+
}
|
|
2059
|
+
function resolveSelectedRun(input, runId) {
|
|
2060
|
+
if (input.selectedRun) {
|
|
2061
|
+
if (runId && input.selectedRun.runId !== runId) {
|
|
2062
|
+
return {
|
|
2063
|
+
diagnostics: [
|
|
2064
|
+
diagnostic(
|
|
2065
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
2066
|
+
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
2067
|
+
)
|
|
2068
|
+
]
|
|
2069
|
+
};
|
|
1915
2070
|
}
|
|
2071
|
+
return { run: input.selectedRun, diagnostics: [] };
|
|
1916
2072
|
}
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
kind: "step-removed",
|
|
1926
|
-
severity: "warning",
|
|
1927
|
-
message: `Step only in left run: ${ls.name}`,
|
|
1928
|
-
path: buildPath([pathSeg(ls, idx)]),
|
|
1929
|
-
left: ls.id,
|
|
1930
|
-
right: void 0
|
|
1931
|
-
});
|
|
1932
|
-
idx += 1;
|
|
1933
|
-
} else if (rs !== void 0) {
|
|
1934
|
-
raw.push({
|
|
1935
|
-
kind: "step-added",
|
|
1936
|
-
severity: "warning",
|
|
1937
|
-
message: `Step only in right run: ${rs.name}`,
|
|
1938
|
-
path: buildPath([pathSeg(rs, idx)]),
|
|
1939
|
-
left: void 0,
|
|
1940
|
-
right: rs.id
|
|
1941
|
-
});
|
|
1942
|
-
idx += 1;
|
|
2073
|
+
if (runId) {
|
|
2074
|
+
const run = input.read.runs.find((candidate) => candidate.runId === runId);
|
|
2075
|
+
if (!run) {
|
|
2076
|
+
return {
|
|
2077
|
+
diagnostics: [
|
|
2078
|
+
diagnostic("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
2079
|
+
]
|
|
2080
|
+
};
|
|
1943
2081
|
}
|
|
2082
|
+
return { run, diagnostics: [] };
|
|
1944
2083
|
}
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
let warnings = 0;
|
|
1948
|
-
let info = 0;
|
|
1949
|
-
for (const d of differences) {
|
|
1950
|
-
if (d.severity === "error") errors += 1;
|
|
1951
|
-
else if (d.severity === "warning") warnings += 1;
|
|
1952
|
-
else info += 1;
|
|
1953
|
-
}
|
|
1954
|
-
const firstVisible = differences[0];
|
|
1955
|
-
const firstDivergence = firstVisible !== void 0 ? {
|
|
1956
|
-
kind: "first-divergence",
|
|
1957
|
-
severity: firstVisible.severity,
|
|
1958
|
-
message: `First divergence: ${firstVisible.message}`,
|
|
1959
|
-
path: firstVisible.path,
|
|
1960
|
-
left: firstVisible.left,
|
|
1961
|
-
right: firstVisible.right
|
|
1962
|
-
} : void 0;
|
|
1963
|
-
const summary = {
|
|
1964
|
-
leftRunId: left.runId,
|
|
1965
|
-
rightRunId: right.runId,
|
|
1966
|
-
totalDifferences: differences.length,
|
|
1967
|
-
errors,
|
|
1968
|
-
warnings,
|
|
1969
|
-
info,
|
|
1970
|
-
firstDivergence
|
|
1971
|
-
};
|
|
1972
|
-
return { summary, differences };
|
|
1973
|
-
}
|
|
1974
|
-
|
|
1975
|
-
// packages/core/src/evidence/zip.ts
|
|
1976
|
-
(() => {
|
|
1977
|
-
const table = new Uint32Array(256);
|
|
1978
|
-
for (let n = 0; n < 256; n += 1) {
|
|
1979
|
-
let c = n;
|
|
1980
|
-
for (let k = 0; k < 8; k += 1) {
|
|
1981
|
-
c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
|
|
1982
|
-
}
|
|
1983
|
-
table[n] = c >>> 0;
|
|
1984
|
-
}
|
|
1985
|
-
return table;
|
|
1986
|
-
})();
|
|
1987
|
-
|
|
1988
|
-
// packages/core/src/suite/types.ts
|
|
1989
|
-
var DEFAULT_SUITE_CONFIG_NAMES = [
|
|
1990
|
-
"agent-inspect.suite.json",
|
|
1991
|
-
"agent-inspect.suite.js",
|
|
1992
|
-
"agent-inspect.suite.mjs",
|
|
1993
|
-
"agent-inspect.suite.cjs"
|
|
1994
|
-
];
|
|
1995
|
-
function diagnostic(code, message, severity = "error", caseId) {
|
|
1996
|
-
return { code, message, severity, ...{} };
|
|
1997
|
-
}
|
|
1998
|
-
function asString(value, label) {
|
|
1999
|
-
if (value === void 0) return void 0;
|
|
2000
|
-
if (typeof value !== "string" || value.trim() === "") {
|
|
2001
|
-
throw new Error(`${label} must be a non-empty string.`);
|
|
2002
|
-
}
|
|
2003
|
-
return value.trim();
|
|
2004
|
-
}
|
|
2005
|
-
function asStringArray(value, label) {
|
|
2006
|
-
if (value === void 0) return void 0;
|
|
2007
|
-
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
2008
|
-
throw new Error(`${label} must be an array of strings.`);
|
|
2084
|
+
if (input.read.runs.length === 1) {
|
|
2085
|
+
return { run: input.read.runs[0], diagnostics: [] };
|
|
2009
2086
|
}
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2087
|
+
if (input.read.runs.length === 0) {
|
|
2088
|
+
return {
|
|
2089
|
+
diagnostics: [
|
|
2090
|
+
diagnostic(
|
|
2091
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
2092
|
+
formatProgrammaticDiagnostic(
|
|
2093
|
+
"AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
2094
|
+
"No runs are available for checks."
|
|
2095
|
+
)
|
|
2096
|
+
)
|
|
2097
|
+
]
|
|
2098
|
+
};
|
|
2016
2099
|
}
|
|
2017
|
-
return
|
|
2100
|
+
return {
|
|
2101
|
+
diagnostics: [
|
|
2102
|
+
diagnostic(
|
|
2103
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
2104
|
+
formatProgrammaticDiagnostic("AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED")
|
|
2105
|
+
)
|
|
2106
|
+
]
|
|
2107
|
+
};
|
|
2018
2108
|
}
|
|
2019
|
-
function
|
|
2109
|
+
function selectRules(rules, selectedIds) {
|
|
2020
2110
|
const diagnostics = [];
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
);
|
|
2025
|
-
return { diagnostics };
|
|
2026
|
-
}
|
|
2027
|
-
const raw = value;
|
|
2028
|
-
try {
|
|
2029
|
-
const id = asString(raw.id, `cases[${index}].id`);
|
|
2030
|
-
if (id === void 0) {
|
|
2111
|
+
const byId = /* @__PURE__ */ new Map();
|
|
2112
|
+
for (const rule of rules) {
|
|
2113
|
+
if (byId.has(rule.id)) {
|
|
2031
2114
|
diagnostics.push(
|
|
2032
|
-
diagnostic("
|
|
2115
|
+
diagnostic("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
2033
2116
|
);
|
|
2034
|
-
|
|
2117
|
+
continue;
|
|
2118
|
+
}
|
|
2119
|
+
byId.set(rule.id, rule);
|
|
2120
|
+
}
|
|
2121
|
+
if (selectedIds && selectedIds.length > 0) {
|
|
2122
|
+
const selected = new Set(selectedIds);
|
|
2123
|
+
for (const id of selected) {
|
|
2124
|
+
if (!byId.has(id)) {
|
|
2125
|
+
diagnostics.push(
|
|
2126
|
+
diagnostic("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
2127
|
+
);
|
|
2128
|
+
}
|
|
2035
2129
|
}
|
|
2036
|
-
const trace = asString(raw.trace, `cases[${index}].trace`);
|
|
2037
|
-
const runId = asString(raw.runId, `cases[${index}].runId`);
|
|
2038
|
-
const input = asString(raw.input, `cases[${index}].input`);
|
|
2039
2130
|
return {
|
|
2040
|
-
|
|
2041
|
-
id,
|
|
2042
|
-
...trace !== void 0 ? { trace } : {},
|
|
2043
|
-
...runId !== void 0 ? { runId } : {},
|
|
2044
|
-
...input !== void 0 ? { input } : {},
|
|
2045
|
-
...asStringArray(raw.requireTools, `cases[${index}].requireTools`) !== void 0 ? { requireTools: asStringArray(raw.requireTools, `cases[${index}].requireTools`) } : {},
|
|
2046
|
-
...asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) !== void 0 ? { forbidTools: asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) } : {},
|
|
2047
|
-
...asPositiveNumber(raw.maxDurationMs, `cases[${index}].maxDurationMs`) !== void 0 ? {
|
|
2048
|
-
maxDurationMs: asPositiveNumber(
|
|
2049
|
-
raw.maxDurationMs,
|
|
2050
|
-
`cases[${index}].maxDurationMs`
|
|
2051
|
-
)
|
|
2052
|
-
} : {},
|
|
2053
|
-
...asStringArray(
|
|
2054
|
-
raw.expectedObservations,
|
|
2055
|
-
`cases[${index}].expectedObservations`
|
|
2056
|
-
) !== void 0 ? {
|
|
2057
|
-
expectedObservations: asStringArray(
|
|
2058
|
-
raw.expectedObservations,
|
|
2059
|
-
`cases[${index}].expectedObservations`
|
|
2060
|
-
)
|
|
2061
|
-
} : {}
|
|
2062
|
-
},
|
|
2131
|
+
rules: [...byId.values()].filter((rule) => selected.has(rule.id)).sort(compareRules),
|
|
2063
2132
|
diagnostics
|
|
2064
2133
|
};
|
|
2065
|
-
} catch (error) {
|
|
2066
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
2067
|
-
diagnostics.push(diagnostic("AI_SUITE_CONFIG_INVALID", message));
|
|
2068
|
-
return { diagnostics };
|
|
2069
2134
|
}
|
|
2135
|
+
return { rules: [...byId.values()].sort(compareRules), diagnostics };
|
|
2070
2136
|
}
|
|
2071
|
-
function
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
const
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
}
|
|
2083
|
-
const cases = [];
|
|
2084
|
-
for (let index = 0; index < raw.cases.length; index += 1) {
|
|
2085
|
-
const { caseConfig, diagnostics } = validateCaseConfig(raw.cases[index], index);
|
|
2086
|
-
if (diagnostics.length > 0) {
|
|
2087
|
-
throw new Error(diagnostics.map((item) => item.message).join("; "));
|
|
2137
|
+
function compareRules(a, b) {
|
|
2138
|
+
return a.id.localeCompare(b.id);
|
|
2139
|
+
}
|
|
2140
|
+
function eventTimestamp(finding, eventById) {
|
|
2141
|
+
const eventId = finding.evidence[0]?.eventId;
|
|
2142
|
+
return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
|
|
2143
|
+
}
|
|
2144
|
+
function compareFindings(eventById) {
|
|
2145
|
+
return (a, b) => {
|
|
2146
|
+
if (SEVERITY_RANK[a.severity] !== SEVERITY_RANK[b.severity]) {
|
|
2147
|
+
return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity];
|
|
2088
2148
|
}
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
if (ids.has(suiteCase.id)) {
|
|
2094
|
-
throw new Error(`Duplicate case id "${suiteCase.id}".`);
|
|
2149
|
+
const byRule = a.ruleId.localeCompare(b.ruleId);
|
|
2150
|
+
if (byRule !== 0) return byRule;
|
|
2151
|
+
if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
|
|
2152
|
+
return STATUS_RANK[a.status] - STATUS_RANK[b.status];
|
|
2095
2153
|
}
|
|
2096
|
-
|
|
2154
|
+
const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
|
|
2155
|
+
if (byRun !== 0) return byRun;
|
|
2156
|
+
const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
|
|
2157
|
+
if (byTime !== 0) return byTime;
|
|
2158
|
+
const byEvent = compareStrings(a.evidence[0]?.eventId, b.evidence[0]?.eventId);
|
|
2159
|
+
if (byEvent !== 0) return byEvent;
|
|
2160
|
+
return compareStrings(a.evidence[0]?.path, b.evidence[0]?.path);
|
|
2161
|
+
};
|
|
2162
|
+
}
|
|
2163
|
+
function normalizeFinding(rule, finding) {
|
|
2164
|
+
return {
|
|
2165
|
+
ruleId: finding.ruleId || rule.id,
|
|
2166
|
+
severity: finding.severity ?? rule.defaultSeverity,
|
|
2167
|
+
status: finding.status,
|
|
2168
|
+
message: finding.message,
|
|
2169
|
+
...finding.expected !== void 0 ? { expected: finding.expected } : {},
|
|
2170
|
+
...finding.actual !== void 0 ? { actual: finding.actual } : {},
|
|
2171
|
+
evidence: [...finding.evidence ?? []],
|
|
2172
|
+
...finding.category !== void 0 ? { category: finding.category } : {},
|
|
2173
|
+
...finding.confidence !== void 0 ? { confidence: finding.confidence } : {},
|
|
2174
|
+
...finding.detector !== void 0 ? { detector: finding.detector } : {},
|
|
2175
|
+
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
2176
|
+
};
|
|
2177
|
+
}
|
|
2178
|
+
function summarize(findings, diagnostics, rulesEvaluated) {
|
|
2179
|
+
return {
|
|
2180
|
+
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
2181
|
+
failed: findings.filter(
|
|
2182
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
2183
|
+
).length,
|
|
2184
|
+
warnings: findings.filter(
|
|
2185
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
2186
|
+
).length,
|
|
2187
|
+
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2188
|
+
rulesEvaluated
|
|
2189
|
+
};
|
|
2190
|
+
}
|
|
2191
|
+
function classifyRuleExecution(findings, threw) {
|
|
2192
|
+
if (findings.some((finding) => finding.status === "fail" && finding.severity === "error")) {
|
|
2193
|
+
return "fail";
|
|
2097
2194
|
}
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2195
|
+
if (findings.some(
|
|
2196
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
2197
|
+
)) {
|
|
2198
|
+
return "warning";
|
|
2101
2199
|
}
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2200
|
+
return "pass";
|
|
2201
|
+
}
|
|
2202
|
+
function stringAttr(event, keys) {
|
|
2203
|
+
for (const key of keys) {
|
|
2204
|
+
const value = event.attributes?.[key];
|
|
2205
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2108
2206
|
}
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2207
|
+
return void 0;
|
|
2208
|
+
}
|
|
2209
|
+
function metadataStringAttr(event, keys) {
|
|
2210
|
+
const metadata = event.attributes?.metadata;
|
|
2211
|
+
if (typeof metadata !== "object" || metadata === null || Array.isArray(metadata)) {
|
|
2212
|
+
return void 0;
|
|
2114
2213
|
}
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
const outputDir = asString(
|
|
2120
|
-
raw.artifacts.outputDir,
|
|
2121
|
-
"artifacts.outputDir"
|
|
2122
|
-
);
|
|
2123
|
-
config.artifacts = outputDir !== void 0 ? { outputDir } : {};
|
|
2124
|
-
}
|
|
2125
|
-
if (raw.baseline !== void 0) {
|
|
2126
|
-
const baseline = asString(raw.baseline, "baseline");
|
|
2127
|
-
if (baseline !== void 0) config.baseline = baseline;
|
|
2128
|
-
}
|
|
2129
|
-
if (raw.candidate !== void 0) {
|
|
2130
|
-
const candidate = asString(raw.candidate, "candidate");
|
|
2131
|
-
if (candidate !== void 0) config.candidate = candidate;
|
|
2132
|
-
}
|
|
2133
|
-
if (redactionProfile !== void 0) config.redactionProfile = redactionProfile;
|
|
2134
|
-
return config;
|
|
2135
|
-
}
|
|
2136
|
-
|
|
2137
|
-
// packages/core/src/suite/load.ts
|
|
2138
|
-
var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
2139
|
-
var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
2140
|
-
function diagnostic2(code, message) {
|
|
2141
|
-
return { code, message, severity: "error" };
|
|
2142
|
-
}
|
|
2143
|
-
async function fileExists(filePath) {
|
|
2144
|
-
try {
|
|
2145
|
-
await promises.access(filePath);
|
|
2146
|
-
return true;
|
|
2147
|
-
} catch {
|
|
2148
|
-
return false;
|
|
2214
|
+
const record = metadata;
|
|
2215
|
+
for (const key of keys) {
|
|
2216
|
+
const value = record[key];
|
|
2217
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2149
2218
|
}
|
|
2219
|
+
return void 0;
|
|
2150
2220
|
}
|
|
2151
|
-
|
|
2152
|
-
const
|
|
2153
|
-
|
|
2154
|
-
return path15__default.default.resolve(cwd, options.configPath.trim());
|
|
2155
|
-
}
|
|
2156
|
-
for (const name of DEFAULT_SUITE_CONFIG_NAMES) {
|
|
2157
|
-
const candidate = path15__default.default.join(cwd, name);
|
|
2158
|
-
if (await fileExists(candidate)) return candidate;
|
|
2221
|
+
function stripPrefix(name, prefixes) {
|
|
2222
|
+
for (const prefix of prefixes) {
|
|
2223
|
+
if (name.startsWith(prefix)) return name.slice(prefix.length);
|
|
2159
2224
|
}
|
|
2160
|
-
|
|
2161
|
-
`No suite config found. Create one with \`agent-inspect suite init\` or pass --config.`
|
|
2162
|
-
);
|
|
2225
|
+
return name;
|
|
2163
2226
|
}
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
2177
|
-
throw Object.assign(new Error(message), {
|
|
2178
|
-
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
2179
|
-
});
|
|
2180
|
-
}
|
|
2181
|
-
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
2182
|
-
const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
|
|
2183
|
-
throw Object.assign(new Error(message), {
|
|
2184
|
-
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
2185
|
-
});
|
|
2186
|
-
}
|
|
2187
|
-
try {
|
|
2188
|
-
let raw;
|
|
2189
|
-
if (extension === ".json") {
|
|
2190
|
-
raw = JSON.parse(await promises.readFile(configPath, "utf-8"));
|
|
2191
|
-
} else {
|
|
2192
|
-
const mod = await import(url.pathToFileURL(configPath).href);
|
|
2193
|
-
raw = "default" in mod ? mod.default : mod;
|
|
2194
|
-
}
|
|
2195
|
-
const config = normalizeSuiteConfig(raw);
|
|
2196
|
-
return {
|
|
2197
|
-
config,
|
|
2198
|
-
configPath,
|
|
2199
|
-
configDir: path15__default.default.dirname(configPath)
|
|
2200
|
-
};
|
|
2201
|
-
} catch (error) {
|
|
2202
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
2203
|
-
throw Object.assign(new Error(message), {
|
|
2204
|
-
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
2205
|
-
});
|
|
2206
|
-
}
|
|
2227
|
+
function eventEvidence(event, path18) {
|
|
2228
|
+
return {
|
|
2229
|
+
runId: event.runId,
|
|
2230
|
+
eventId: event.eventId,
|
|
2231
|
+
parentId: event.parentId,
|
|
2232
|
+
traceId: event.trace?.traceId,
|
|
2233
|
+
spanId: event.trace?.spanId,
|
|
2234
|
+
kind: event.kind,
|
|
2235
|
+
name: event.name,
|
|
2236
|
+
status: event.status,
|
|
2237
|
+
...path18 ? { path: path18 } : {}
|
|
2238
|
+
};
|
|
2207
2239
|
}
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
await promises.access(filePath);
|
|
2211
|
-
return true;
|
|
2212
|
-
} catch {
|
|
2213
|
-
return false;
|
|
2214
|
-
}
|
|
2240
|
+
function runEvidence(run) {
|
|
2241
|
+
return run ? [{ runId: run.runId, name: run.name, status: run.status }] : [];
|
|
2215
2242
|
}
|
|
2216
|
-
|
|
2217
|
-
if (suiteCase.trace !== void 0) {
|
|
2218
|
-
const tracePath = path15__default.default.resolve(options.configDir, suiteCase.trace);
|
|
2219
|
-
if (await exists(tracePath)) {
|
|
2220
|
-
return { caseId: suiteCase.id, tracePath, missing: false };
|
|
2221
|
-
}
|
|
2222
|
-
return {
|
|
2223
|
-
caseId: suiteCase.id,
|
|
2224
|
-
tracePath,
|
|
2225
|
-
missing: true,
|
|
2226
|
-
reason: `trace file not found: ${suiteCase.trace}`
|
|
2227
|
-
};
|
|
2228
|
-
}
|
|
2229
|
-
const runKey = suiteCase.runId ?? suiteCase.id;
|
|
2230
|
-
const directPath = getTraceFilePath(runKey, options.tracesDir);
|
|
2231
|
-
if (await exists(directPath)) {
|
|
2232
|
-
return { caseId: suiteCase.id, tracePath: directPath, runId: runKey, missing: false };
|
|
2233
|
-
}
|
|
2234
|
-
const nestedPath = path15__default.default.join(options.tracesDir, `${path15__default.default.basename(runKey)}.jsonl`);
|
|
2235
|
-
if (await exists(nestedPath)) {
|
|
2236
|
-
return { caseId: suiteCase.id, tracePath: nestedPath, runId: runKey, missing: false };
|
|
2237
|
-
}
|
|
2243
|
+
function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
2238
2244
|
return {
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2245
|
+
ruleId,
|
|
2246
|
+
severity: meta?.severity ?? "error",
|
|
2247
|
+
status: meta?.status ?? "fail",
|
|
2248
|
+
message,
|
|
2249
|
+
...expected !== void 0 ? { expected } : {},
|
|
2250
|
+
...actual !== void 0 ? { actual } : {},
|
|
2251
|
+
evidence: [...evidence],
|
|
2252
|
+
...{},
|
|
2253
|
+
...{},
|
|
2254
|
+
...{},
|
|
2255
|
+
...{}
|
|
2243
2256
|
};
|
|
2244
2257
|
}
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
2248
|
-
AI_TRACE_INPUT_INVALID: {
|
|
2249
|
-
code: "AI_TRACE_INPUT_INVALID",
|
|
2250
|
-
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
2251
|
-
remediation: "For a file path, use openTraceFile(path).",
|
|
2252
|
-
relatedCodes: ["invalid_input"]
|
|
2253
|
-
},
|
|
2254
|
-
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
2255
|
-
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
2256
|
-
summary: "No trace reader could detect the input format.",
|
|
2257
|
-
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
2258
|
-
relatedCodes: ["unsupported_format"]
|
|
2259
|
-
},
|
|
2260
|
-
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
2261
|
-
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
2262
|
-
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
2263
|
-
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
2264
|
-
relatedCodes: ["ambiguous_format"]
|
|
2265
|
-
},
|
|
2266
|
-
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
2267
|
-
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
2268
|
-
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
2269
|
-
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
2270
|
-
},
|
|
2271
|
-
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
2272
|
-
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
2273
|
-
summary: "Multiple runs are available; select a run before executing checks.",
|
|
2274
|
-
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
2275
|
-
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
2276
|
-
},
|
|
2277
|
-
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
2278
|
-
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2279
|
-
summary: "A parentId equals its own event/step id.",
|
|
2280
|
-
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
2281
|
-
relatedCodes: [
|
|
2282
|
-
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
2283
|
-
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
2284
|
-
]
|
|
2285
|
-
},
|
|
2286
|
-
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
2287
|
-
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
2288
|
-
summary: "Trace contains a parentId cycle.",
|
|
2289
|
-
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
2290
|
-
relatedCodes: ["structure.cycle"]
|
|
2291
|
-
}
|
|
2292
|
-
});
|
|
2293
|
-
function formatProgrammaticDiagnostic(code, detail) {
|
|
2294
|
-
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
2295
|
-
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
2296
|
-
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
2258
|
+
function toolName(event) {
|
|
2259
|
+
return resolveCanonicalToolName(event);
|
|
2297
2260
|
}
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
function normalizeSensitiveKey(value) {
|
|
2301
|
-
return value.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
2261
|
+
function semanticEvents(context) {
|
|
2262
|
+
return context.logicalEvents ?? context.events;
|
|
2302
2263
|
}
|
|
2303
|
-
|
|
2304
|
-
[
|
|
2305
|
-
|
|
2306
|
-
"max_tokens",
|
|
2307
|
-
"min_tokens",
|
|
2308
|
-
"ls_max_tokens",
|
|
2309
|
-
"token_count",
|
|
2310
|
-
"token_limit",
|
|
2311
|
-
"token_budget",
|
|
2312
|
-
"input_tokens",
|
|
2313
|
-
"output_tokens",
|
|
2314
|
-
"total_tokens",
|
|
2315
|
-
"cached_tokens",
|
|
2316
|
-
"prompt_tokens",
|
|
2317
|
-
"completion_tokens"
|
|
2318
|
-
].map(normalizeSensitiveKey)
|
|
2319
|
-
);
|
|
2320
|
-
|
|
2321
|
-
// packages/core/src/checks/logical-events.ts
|
|
2322
|
-
function isRecord5(value) {
|
|
2323
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2264
|
+
function llmModel(event) {
|
|
2265
|
+
const keys = ["model", "modelId", "responseModelId", "modelName", "model_name"];
|
|
2266
|
+
return stringAttr(event, keys) ?? metadataStringAttr(event, keys) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
2324
2267
|
}
|
|
2325
|
-
function
|
|
2326
|
-
const
|
|
2327
|
-
return
|
|
2268
|
+
function llmProvider(event) {
|
|
2269
|
+
const keys = ["provider", "providerName", "provider_name"];
|
|
2270
|
+
return stringAttr(event, keys) ?? metadataStringAttr(event, keys);
|
|
2328
2271
|
}
|
|
2329
|
-
function
|
|
2330
|
-
const
|
|
2331
|
-
|
|
2332
|
-
return void 0;
|
|
2272
|
+
function llmFinishReason(event) {
|
|
2273
|
+
const keys = ["finishReason", "rawFinishReason", "finish_reason"];
|
|
2274
|
+
return stringAttr(event, keys) ?? metadataStringAttr(event, keys);
|
|
2333
2275
|
}
|
|
2334
|
-
function
|
|
2335
|
-
return
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
2339
|
-
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
2340
|
-
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
2341
|
-
};
|
|
2276
|
+
function finishedEvents(context, kind) {
|
|
2277
|
+
return semanticEvents(context).filter(
|
|
2278
|
+
(event) => (event.kind === kind) && event.status !== "running"
|
|
2279
|
+
);
|
|
2342
2280
|
}
|
|
2343
|
-
function
|
|
2344
|
-
|
|
2345
|
-
...isRecord5(complete.attributes) ? complete.attributes : {},
|
|
2346
|
-
...isRecord5(start.attributes) ? start.attributes : {}
|
|
2347
|
-
};
|
|
2348
|
-
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
2349
|
-
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
2350
|
-
if (complete.attributes?.errorStack !== void 0) {
|
|
2351
|
-
merged.errorStack = complete.attributes.errorStack;
|
|
2352
|
-
}
|
|
2353
|
-
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
2281
|
+
function toolInvocationEvents(context) {
|
|
2282
|
+
return semanticEvents(context).filter((event) => event.kind === "TOOL");
|
|
2354
2283
|
}
|
|
2355
|
-
function
|
|
2356
|
-
const
|
|
2357
|
-
const
|
|
2358
|
-
...cloneEvent(start),
|
|
2359
|
-
status: complete.status,
|
|
2360
|
-
timestamp: complete.timestamp ?? start.timestamp,
|
|
2361
|
-
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
2362
|
-
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
2363
|
-
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
2364
|
-
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
2365
|
-
...attributes !== void 0 ? { attributes } : {}
|
|
2366
|
-
};
|
|
2284
|
+
function createRunStatusRule(options = {}) {
|
|
2285
|
+
const expected = options.expected ?? "ok";
|
|
2286
|
+
const allowIncomplete = options.allowIncomplete === true;
|
|
2367
2287
|
return {
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2288
|
+
id: "run.status",
|
|
2289
|
+
category: "run",
|
|
2290
|
+
defaultSeverity: "error",
|
|
2291
|
+
evaluate(context) {
|
|
2292
|
+
const findings = [];
|
|
2293
|
+
const actual = context.selectedRun?.status ?? "unknown";
|
|
2294
|
+
if (actual !== expected) {
|
|
2295
|
+
findings.push(
|
|
2296
|
+
failFinding(
|
|
2297
|
+
"run.status",
|
|
2298
|
+
`Run status ${actual} did not match expected ${expected}.`,
|
|
2299
|
+
runEvidence(context.selectedRun),
|
|
2300
|
+
expected,
|
|
2301
|
+
actual
|
|
2302
|
+
)
|
|
2303
|
+
);
|
|
2304
|
+
}
|
|
2305
|
+
if (!allowIncomplete) {
|
|
2306
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
2307
|
+
if (running.length > 0) {
|
|
2308
|
+
findings.push(
|
|
2309
|
+
failFinding(
|
|
2310
|
+
"run.status",
|
|
2311
|
+
"Run contains incomplete running events.",
|
|
2312
|
+
running.map((event) => eventEvidence(event)),
|
|
2313
|
+
"no running events",
|
|
2314
|
+
running.length
|
|
2315
|
+
)
|
|
2316
|
+
);
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
return findings;
|
|
2375
2320
|
}
|
|
2376
2321
|
};
|
|
2377
2322
|
}
|
|
2378
|
-
function
|
|
2323
|
+
function createRunDurationRule(options) {
|
|
2379
2324
|
return {
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2325
|
+
id: "run.duration",
|
|
2326
|
+
category: "run",
|
|
2327
|
+
defaultSeverity: "error",
|
|
2328
|
+
evaluate(context) {
|
|
2329
|
+
const actual = context.selectedRun?.durationMs;
|
|
2330
|
+
if (actual === void 0 || actual <= options.maxDurationMs) return [];
|
|
2331
|
+
return [
|
|
2332
|
+
failFinding(
|
|
2333
|
+
"run.duration",
|
|
2334
|
+
`Run duration ${actual}ms exceeded ${options.maxDurationMs}ms.`,
|
|
2335
|
+
runEvidence(context.selectedRun),
|
|
2336
|
+
{ maxDurationMs: options.maxDurationMs },
|
|
2337
|
+
actual
|
|
2338
|
+
)
|
|
2339
|
+
];
|
|
2387
2340
|
}
|
|
2388
2341
|
};
|
|
2389
2342
|
}
|
|
2390
|
-
function
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
const completes = [];
|
|
2407
|
-
const others = [];
|
|
2408
|
-
for (const event of runEvents) {
|
|
2409
|
-
const legacy = legacyEvent(event);
|
|
2410
|
-
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
2411
|
-
starts.push(event);
|
|
2412
|
-
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
2413
|
-
completes.push(event);
|
|
2414
|
-
} else {
|
|
2415
|
-
others.push(event);
|
|
2343
|
+
function createToolUsageRule(options) {
|
|
2344
|
+
return {
|
|
2345
|
+
id: "tool.usage",
|
|
2346
|
+
category: "tool",
|
|
2347
|
+
defaultSeverity: "error",
|
|
2348
|
+
evaluate(context) {
|
|
2349
|
+
const tools = toolInvocationEvents(context);
|
|
2350
|
+
const names = tools.map(toolName);
|
|
2351
|
+
const nameSet = new Set(names);
|
|
2352
|
+
const findings = [];
|
|
2353
|
+
for (const required of options.required ?? []) {
|
|
2354
|
+
if (!nameSet.has(required)) {
|
|
2355
|
+
findings.push(
|
|
2356
|
+
failFinding("tool.usage", `Required tool ${required} did not appear.`, runEvidence(context.selectedRun), required, names)
|
|
2357
|
+
);
|
|
2358
|
+
}
|
|
2416
2359
|
}
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
);
|
|
2426
|
-
if (candidates.length > 1) {
|
|
2427
|
-
diagnostics.push({
|
|
2428
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
2429
|
-
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
2430
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
2431
|
-
});
|
|
2432
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
2360
|
+
const forbidden = new Set(options.forbidden ?? []);
|
|
2361
|
+
const allowed = options.allowed?.length ? new Set(options.allowed) : void 0;
|
|
2362
|
+
for (const event of tools) {
|
|
2363
|
+
const name = toolName(event);
|
|
2364
|
+
if (forbidden.has(name)) {
|
|
2365
|
+
findings.push(
|
|
2366
|
+
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence(event)], "tool absent", name)
|
|
2367
|
+
);
|
|
2433
2368
|
}
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
);
|
|
2439
|
-
if (candidates.length > 1) {
|
|
2440
|
-
diagnostics.push({
|
|
2441
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
2442
|
-
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
2443
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
2444
|
-
});
|
|
2445
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
2369
|
+
if (allowed && !allowed.has(name)) {
|
|
2370
|
+
findings.push(
|
|
2371
|
+
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence(event)], [...allowed].sort(), name)
|
|
2372
|
+
);
|
|
2446
2373
|
}
|
|
2447
|
-
match = candidates[0];
|
|
2448
2374
|
}
|
|
2449
|
-
if (
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
logical.push(paired);
|
|
2454
|
-
logicalByRawId.set(start.eventId, paired);
|
|
2455
|
-
logicalByRawId.set(match.eventId, paired);
|
|
2456
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
2457
|
-
} else {
|
|
2458
|
-
diagnostics.push({
|
|
2459
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
2460
|
-
message: `No matching complete for start ${start.eventId}.`,
|
|
2461
|
-
eventIds: [start.eventId]
|
|
2462
|
-
});
|
|
2463
|
-
const alone = asLogical(start);
|
|
2464
|
-
logical.push(alone);
|
|
2465
|
-
logicalByRawId.set(start.eventId, alone);
|
|
2466
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2375
|
+
if (options.minCount !== void 0 && tools.length < options.minCount) {
|
|
2376
|
+
findings.push(
|
|
2377
|
+
failFinding("tool.usage", `Tool count ${tools.length} was below minimum ${options.minCount}.`, runEvidence(context.selectedRun), { minCount: options.minCount }, tools.length)
|
|
2378
|
+
);
|
|
2467
2379
|
}
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
2473
|
-
message: `No matching start for complete ${complete.eventId}.`,
|
|
2474
|
-
eventIds: [complete.eventId]
|
|
2475
|
-
});
|
|
2476
|
-
const alone = asLogical(complete);
|
|
2477
|
-
logical.push(alone);
|
|
2478
|
-
logicalByRawId.set(complete.eventId, alone);
|
|
2479
|
-
const stepId = stepIdOf(complete);
|
|
2480
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
2481
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2380
|
+
if (options.maxCount !== void 0 && tools.length > options.maxCount) {
|
|
2381
|
+
findings.push(
|
|
2382
|
+
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) => eventEvidence(event)), { maxCount: options.maxCount }, tools.length)
|
|
2383
|
+
);
|
|
2482
2384
|
}
|
|
2385
|
+
return findings;
|
|
2483
2386
|
}
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2387
|
+
};
|
|
2388
|
+
}
|
|
2389
|
+
function createLlmUsageRule(options) {
|
|
2390
|
+
return {
|
|
2391
|
+
id: "llm.usage",
|
|
2392
|
+
category: "llm",
|
|
2393
|
+
defaultSeverity: "error",
|
|
2394
|
+
evaluate(context) {
|
|
2395
|
+
const llms = finishedEvents(context, "LLM");
|
|
2396
|
+
const findings = [];
|
|
2397
|
+
const allowedModels = options.allowedModels?.length ? new Set(options.allowedModels) : void 0;
|
|
2398
|
+
const allowedProviders = options.allowedProviders?.length ? new Set(options.allowedProviders) : void 0;
|
|
2399
|
+
const finishReasons = options.finishReasons?.length ? new Set(options.finishReasons) : void 0;
|
|
2400
|
+
if (options.maxCalls !== void 0 && llms.length > options.maxCalls) {
|
|
2401
|
+
findings.push(
|
|
2402
|
+
failFinding(
|
|
2403
|
+
"llm.usage",
|
|
2404
|
+
`LLM call count ${llms.length} exceeded ${options.maxCalls}.`,
|
|
2405
|
+
llms.map((event) => eventEvidence(event)),
|
|
2406
|
+
{ maxCalls: options.maxCalls },
|
|
2407
|
+
llms.length
|
|
2408
|
+
)
|
|
2409
|
+
);
|
|
2491
2410
|
}
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
if (viaStep) {
|
|
2511
|
-
nextParent = viaStep;
|
|
2512
|
-
remapped = true;
|
|
2513
|
-
}
|
|
2514
|
-
}
|
|
2515
|
-
if (!remapped) {
|
|
2516
|
-
if (originalParentId === event.eventId) {
|
|
2517
|
-
diagnostics.push({
|
|
2518
|
-
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
2519
|
-
message: formatProgrammaticDiagnostic(
|
|
2520
|
-
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2521
|
-
`Removed self-parent edge on ${event.eventId}.`
|
|
2522
|
-
),
|
|
2523
|
-
eventIds: [event.eventId]
|
|
2524
|
-
});
|
|
2525
|
-
const { parentId: _drop, ...rest } = event;
|
|
2526
|
-
normalized.push({
|
|
2527
|
-
...rest,
|
|
2528
|
-
projection: {
|
|
2529
|
-
...event.projection,
|
|
2530
|
-
parentNormalized: true,
|
|
2531
|
-
originalParentId
|
|
2532
|
-
}
|
|
2533
|
-
});
|
|
2534
|
-
continue;
|
|
2535
|
-
}
|
|
2536
|
-
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
2537
|
-
const mapping = event.attributes?.parentMapping;
|
|
2538
|
-
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
2539
|
-
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
2540
|
-
if (!unresolved) {
|
|
2541
|
-
diagnostics.push({
|
|
2542
|
-
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
2543
|
-
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
2544
|
-
eventIds: [event.eventId]
|
|
2545
|
-
});
|
|
2411
|
+
for (const event of llms) {
|
|
2412
|
+
const model = llmModel(event);
|
|
2413
|
+
const provider = llmProvider(event);
|
|
2414
|
+
const finishReason = llmFinishReason(event);
|
|
2415
|
+
if (allowedModels && (!model || !allowedModels.has(model))) {
|
|
2416
|
+
findings.push(
|
|
2417
|
+
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.model")], [...allowedModels].sort(), model ?? "unknown")
|
|
2418
|
+
);
|
|
2419
|
+
}
|
|
2420
|
+
if (allowedProviders && (!provider || !allowedProviders.has(provider))) {
|
|
2421
|
+
findings.push(
|
|
2422
|
+
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.provider")], [...allowedProviders].sort(), provider ?? "unknown")
|
|
2423
|
+
);
|
|
2424
|
+
}
|
|
2425
|
+
if (finishReasons && (!finishReason || !finishReasons.has(finishReason))) {
|
|
2426
|
+
findings.push(
|
|
2427
|
+
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.finishReason")], [...finishReasons].sort(), finishReason ?? "unknown")
|
|
2428
|
+
);
|
|
2546
2429
|
}
|
|
2547
2430
|
}
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2431
|
+
const tokenTotals = llms.reduce(
|
|
2432
|
+
(totals, event) => ({
|
|
2433
|
+
input: totals.input + (event.tokenUsage?.input ?? 0),
|
|
2434
|
+
output: totals.output + (event.tokenUsage?.output ?? 0),
|
|
2435
|
+
total: totals.total + (event.tokenUsage?.total ?? 0),
|
|
2436
|
+
cached: totals.cached + (event.tokenUsage?.cached ?? 0)
|
|
2437
|
+
}),
|
|
2438
|
+
{ input: 0, output: 0, total: 0, cached: 0 }
|
|
2439
|
+
);
|
|
2440
|
+
const tokenLimits = [
|
|
2441
|
+
["input", options.maxInputTokens],
|
|
2442
|
+
["output", options.maxOutputTokens],
|
|
2443
|
+
["total", options.maxTotalTokens],
|
|
2444
|
+
["cached", options.maxCachedTokens]
|
|
2445
|
+
];
|
|
2446
|
+
for (const [key, limit] of tokenLimits) {
|
|
2447
|
+
if (limit !== void 0 && tokenTotals[key] > limit) {
|
|
2448
|
+
findings.push(
|
|
2449
|
+
failFinding(
|
|
2450
|
+
"llm.usage",
|
|
2451
|
+
`LLM ${key} token count ${tokenTotals[key]} exceeded ${limit}.`,
|
|
2452
|
+
llms.map((event) => eventEvidence(event, `tokenUsage.${key}`)),
|
|
2453
|
+
{ [`max${key[0].toUpperCase()}${key.slice(1)}Tokens`]: limit },
|
|
2454
|
+
tokenTotals[key]
|
|
2455
|
+
)
|
|
2456
|
+
);
|
|
2567
2457
|
}
|
|
2568
|
-
});
|
|
2569
|
-
continue;
|
|
2570
|
-
}
|
|
2571
|
-
diagnostics.push({
|
|
2572
|
-
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
2573
|
-
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
2574
|
-
eventIds: [event.eventId]
|
|
2575
|
-
});
|
|
2576
|
-
normalized.push({
|
|
2577
|
-
...event,
|
|
2578
|
-
parentId: nextParent,
|
|
2579
|
-
projection: {
|
|
2580
|
-
...event.projection,
|
|
2581
|
-
parentNormalized: true,
|
|
2582
|
-
originalParentId
|
|
2583
2458
|
}
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
});
|
|
2459
|
+
return findings;
|
|
2460
|
+
}
|
|
2461
|
+
};
|
|
2462
|
+
}
|
|
2463
|
+
function createObservedOutcomeRule(options = {}) {
|
|
2464
|
+
const failOn = options.failOn ?? ["failed"];
|
|
2465
|
+
const requireAny = options.requireAny === true;
|
|
2592
2466
|
return {
|
|
2593
|
-
|
|
2594
|
-
|
|
2467
|
+
id: "outcome.status",
|
|
2468
|
+
category: "run",
|
|
2469
|
+
defaultSeverity: "error",
|
|
2470
|
+
evaluate(context) {
|
|
2471
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
2472
|
+
if (requireAny && outcomes.length === 0) {
|
|
2473
|
+
return [
|
|
2474
|
+
failFinding(
|
|
2475
|
+
"outcome.status",
|
|
2476
|
+
"Expected at least one observed outcome.",
|
|
2477
|
+
runEvidence(context.selectedRun),
|
|
2478
|
+
{ requireAny: true, expected: "at least one observed outcome" },
|
|
2479
|
+
{ code: "outcome.missing", actual: 0 }
|
|
2480
|
+
)
|
|
2481
|
+
];
|
|
2482
|
+
}
|
|
2483
|
+
const matching = outcomesMatchingStatus(outcomes, failOn);
|
|
2484
|
+
if (matching.length === 0) return [];
|
|
2485
|
+
return [
|
|
2486
|
+
failFinding(
|
|
2487
|
+
"outcome.status",
|
|
2488
|
+
`Observed outcome count ${matching.length} matched [${failOn.join(", ")}].`,
|
|
2489
|
+
matching.map((outcome) => ({
|
|
2490
|
+
runId: outcome.runId,
|
|
2491
|
+
eventId: outcome.outcomeId,
|
|
2492
|
+
...outcome.parentId !== void 0 ? { parentId: outcome.parentId } : {},
|
|
2493
|
+
kind: "OUTCOME",
|
|
2494
|
+
name: outcome.name,
|
|
2495
|
+
status: outcome.status,
|
|
2496
|
+
path: `outcome.${outcome.name}`
|
|
2497
|
+
})),
|
|
2498
|
+
{ failOn },
|
|
2499
|
+
matching.map((outcome) => ({
|
|
2500
|
+
name: outcome.name,
|
|
2501
|
+
status: outcome.status,
|
|
2502
|
+
expectation: outcome.expectation
|
|
2503
|
+
}))
|
|
2504
|
+
)
|
|
2505
|
+
];
|
|
2506
|
+
}
|
|
2595
2507
|
};
|
|
2596
2508
|
}
|
|
2597
|
-
function
|
|
2598
|
-
const
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
const metadata = attrs?.metadata;
|
|
2602
|
-
if (isRecord5(metadata)) {
|
|
2603
|
-
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
2604
|
-
if (nested) return nested;
|
|
2509
|
+
function runTraceChecks(input, options = {}) {
|
|
2510
|
+
const selected = resolveSelectedRun(input, options.runId);
|
|
2511
|
+
if (selected.diagnostics.length > 0) {
|
|
2512
|
+
return errorResult(input, selected.diagnostics, selected.run);
|
|
2605
2513
|
}
|
|
2606
|
-
|
|
2607
|
-
|
|
2514
|
+
const rules = selectRules(options.rules ?? [], options.select);
|
|
2515
|
+
if (rules.diagnostics.length > 0) {
|
|
2516
|
+
return errorResult(input, rules.diagnostics, selected.run);
|
|
2608
2517
|
}
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2518
|
+
if (rules.rules.length === 0) {
|
|
2519
|
+
return errorResult(
|
|
2520
|
+
input,
|
|
2521
|
+
[
|
|
2522
|
+
diagnostic(
|
|
2523
|
+
"AI_CHECK_NO_RULES_EVALUATED",
|
|
2524
|
+
"No trace check rules were evaluated. Configure at least one rule, contract, or CLI check option."
|
|
2525
|
+
)
|
|
2526
|
+
],
|
|
2527
|
+
selected.run
|
|
2528
|
+
);
|
|
2616
2529
|
}
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
"AI_TRACE_FACTS_INPUT_NOT_NORMALIZED"
|
|
2623
|
-
);
|
|
2624
|
-
|
|
2625
|
-
// packages/core/src/checks/contract.ts
|
|
2626
|
-
new Set(OBSERVED_OUTCOME_METHODS);
|
|
2627
|
-
|
|
2628
|
-
// packages/core/src/checks/index.ts
|
|
2629
|
-
var SEVERITY_RANK = {
|
|
2630
|
-
error: 0,
|
|
2631
|
-
warning: 1,
|
|
2632
|
-
info: 2
|
|
2633
|
-
};
|
|
2634
|
-
var STATUS_RANK = {
|
|
2635
|
-
fail: 0,
|
|
2636
|
-
warning: 1,
|
|
2637
|
-
pass: 2
|
|
2638
|
-
};
|
|
2639
|
-
function compareStrings(a, b) {
|
|
2640
|
-
return (a ?? "").localeCompare(b ?? "");
|
|
2641
|
-
}
|
|
2642
|
-
function diagnostic3(code, message, ruleId) {
|
|
2643
|
-
return {
|
|
2644
|
-
code,
|
|
2645
|
-
message,
|
|
2646
|
-
severity: "error",
|
|
2647
|
-
...ruleId ? { ruleId } : {}
|
|
2648
|
-
};
|
|
2649
|
-
}
|
|
2650
|
-
function emptySummary() {
|
|
2651
|
-
return {
|
|
2652
|
-
passed: 0,
|
|
2653
|
-
failed: 0,
|
|
2654
|
-
warnings: 0,
|
|
2655
|
-
errors: 0,
|
|
2656
|
-
rulesEvaluated: 0
|
|
2530
|
+
const facts = buildFacts(input, selected.run);
|
|
2531
|
+
const context = {
|
|
2532
|
+
...facts,
|
|
2533
|
+
...selected.run ? { selectedRun: selected.run } : {},
|
|
2534
|
+
...input.sourceLabel ? { sourceLabel: input.sourceLabel } : {}
|
|
2657
2535
|
};
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
2685
|
-
for (const node of nodes) {
|
|
2686
|
-
nodesByEventId.set(node.event.eventId, node);
|
|
2687
|
-
const parentId = node.event.parentId;
|
|
2688
|
-
if (parentId) {
|
|
2689
|
-
const children = childrenByParentId.get(parentId) ?? [];
|
|
2690
|
-
children.push(node);
|
|
2691
|
-
childrenByParentId.set(parentId, children);
|
|
2536
|
+
const diagnostics = [];
|
|
2537
|
+
const findings = [];
|
|
2538
|
+
const ruleExecutions = [];
|
|
2539
|
+
for (const rule of rules.rules) {
|
|
2540
|
+
try {
|
|
2541
|
+
const ruleFindings = rule.evaluate(context).map((finding) => normalizeFinding(rule, finding));
|
|
2542
|
+
findings.push(...ruleFindings);
|
|
2543
|
+
ruleExecutions.push({
|
|
2544
|
+
ruleId: rule.id,
|
|
2545
|
+
category: rule.category,
|
|
2546
|
+
status: classifyRuleExecution(ruleFindings, false),
|
|
2547
|
+
findingCount: ruleFindings.length,
|
|
2548
|
+
...selected.run ? { runId: selected.run.runId } : {}
|
|
2549
|
+
});
|
|
2550
|
+
} catch (error) {
|
|
2551
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2552
|
+
diagnostics.push(
|
|
2553
|
+
diagnostic("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
|
|
2554
|
+
);
|
|
2555
|
+
ruleExecutions.push({
|
|
2556
|
+
ruleId: rule.id,
|
|
2557
|
+
category: rule.category,
|
|
2558
|
+
status: "error",
|
|
2559
|
+
findingCount: 0,
|
|
2560
|
+
...selected.run ? { runId: selected.run.runId } : {}
|
|
2561
|
+
});
|
|
2692
2562
|
}
|
|
2693
2563
|
}
|
|
2694
|
-
|
|
2564
|
+
if (diagnostics.length > 0) {
|
|
2565
|
+
return errorResult(input, diagnostics, selected.run, ruleExecutions);
|
|
2566
|
+
}
|
|
2567
|
+
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
2568
|
+
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
2569
|
+
const summary = summarize(sortedFindings, diagnostics, ruleExecutions.length);
|
|
2570
|
+
const status = summary.failed > 0 ? "fail" : "pass";
|
|
2695
2571
|
return {
|
|
2572
|
+
ok: status === "pass",
|
|
2573
|
+
status,
|
|
2696
2574
|
format: input.read.format,
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
|
|
2703
|
-
sourceFiles: Object.freeze([...input.read.sourceFiles]),
|
|
2704
|
-
nodesByEventId,
|
|
2705
|
-
childrenByParentId,
|
|
2706
|
-
rootNodes: Object.freeze(scopedRuns.flatMap((run) => run.children))
|
|
2575
|
+
...selected.run ? { runId: selected.run.runId } : {},
|
|
2576
|
+
summary,
|
|
2577
|
+
findings: sortedFindings,
|
|
2578
|
+
diagnostics,
|
|
2579
|
+
ruleExecutions
|
|
2707
2580
|
};
|
|
2708
2581
|
}
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
return { run: input.selectedRun, diagnostics: [] };
|
|
2582
|
+
|
|
2583
|
+
// packages/core/src/checks/contract.ts
|
|
2584
|
+
new Set(OBSERVED_OUTCOME_METHODS);
|
|
2585
|
+
|
|
2586
|
+
// packages/core/src/exporters/helpers.ts
|
|
2587
|
+
function sortKeysDeep(input) {
|
|
2588
|
+
if (input === null || typeof input !== "object") return input;
|
|
2589
|
+
if (Array.isArray(input)) return input.map(sortKeysDeep);
|
|
2590
|
+
const o = input;
|
|
2591
|
+
const out = {};
|
|
2592
|
+
for (const k of Object.keys(o).sort()) {
|
|
2593
|
+
out[k] = sortKeysDeep(o[k]);
|
|
2722
2594
|
}
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2595
|
+
return out;
|
|
2596
|
+
}
|
|
2597
|
+
function stableJson(value, pretty) {
|
|
2598
|
+
const sorted = sortKeysDeep(value);
|
|
2599
|
+
return JSON.stringify(sorted);
|
|
2600
|
+
}
|
|
2601
|
+
|
|
2602
|
+
// packages/core/src/diff/comparable.ts
|
|
2603
|
+
function extractOutputPreview(meta) {
|
|
2604
|
+
if (meta === void 0) return void 0;
|
|
2605
|
+
if ("outputPreview" in meta) return meta.outputPreview;
|
|
2606
|
+
if ("resultPreview" in meta) return meta.resultPreview;
|
|
2607
|
+
return void 0;
|
|
2608
|
+
}
|
|
2609
|
+
function mapStepStatus(s) {
|
|
2610
|
+
if (s === void 0) return "running";
|
|
2611
|
+
return s;
|
|
2612
|
+
}
|
|
2613
|
+
function manualTraceEventsToComparableRun(events) {
|
|
2614
|
+
const started = events.find((e) => e.event === "run_started");
|
|
2615
|
+
if (!started || started.event !== "run_started") {
|
|
2616
|
+
throw new Error("Invalid trace: missing run_started");
|
|
2733
2617
|
}
|
|
2734
|
-
|
|
2735
|
-
|
|
2618
|
+
const rs = started;
|
|
2619
|
+
const runId = rs.runId;
|
|
2620
|
+
const completedAll = events.filter((e) => e.event === "run_completed");
|
|
2621
|
+
const lastCompleted = completedAll[completedAll.length - 1];
|
|
2622
|
+
let runStatus;
|
|
2623
|
+
if (lastCompleted === void 0) runStatus = "running";
|
|
2624
|
+
else runStatus = lastCompleted.status;
|
|
2625
|
+
const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
2626
|
+
const steps = /* @__PURE__ */ new Map();
|
|
2627
|
+
let order = 0;
|
|
2628
|
+
for (const e of events) {
|
|
2629
|
+
if (e.event !== "step_started") continue;
|
|
2630
|
+
const s = e;
|
|
2631
|
+
const meta = s.metadata ? { ...s.metadata } : void 0;
|
|
2632
|
+
steps.set(s.stepId, {
|
|
2633
|
+
id: s.stepId,
|
|
2634
|
+
parentId: s.parentId,
|
|
2635
|
+
name: s.name,
|
|
2636
|
+
type: s.type,
|
|
2637
|
+
order: order++,
|
|
2638
|
+
timestamp: s.timestamp,
|
|
2639
|
+
metadata: meta
|
|
2640
|
+
});
|
|
2736
2641
|
}
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2642
|
+
for (const e of events) {
|
|
2643
|
+
if (e.event !== "step_completed") continue;
|
|
2644
|
+
const acc = steps.get(e.stepId);
|
|
2645
|
+
if (!acc) continue;
|
|
2646
|
+
acc.status = e.status;
|
|
2647
|
+
acc.durationMs = e.durationMs;
|
|
2648
|
+
if (e.error?.message) acc.errorMsg = e.error.message;
|
|
2649
|
+
const extra = e;
|
|
2650
|
+
if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
|
|
2651
|
+
acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
|
|
2652
|
+
}
|
|
2653
|
+
}
|
|
2654
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
2655
|
+
for (const acc of steps.values()) {
|
|
2656
|
+
let meta = acc.metadata ? { ...acc.metadata } : void 0;
|
|
2657
|
+
if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
|
|
2658
|
+
meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
|
|
2659
|
+
}
|
|
2660
|
+
const outputPreview = extractOutputPreview(meta);
|
|
2661
|
+
if (meta !== void 0 && ("outputPreview" in meta || "resultPreview" in meta)) {
|
|
2662
|
+
delete meta.outputPreview;
|
|
2663
|
+
delete meta.resultPreview;
|
|
2664
|
+
}
|
|
2665
|
+
const sc = {
|
|
2666
|
+
id: acc.id,
|
|
2667
|
+
name: acc.name,
|
|
2668
|
+
type: acc.type,
|
|
2669
|
+
status: mapStepStatus(acc.status),
|
|
2670
|
+
durationMs: acc.durationMs,
|
|
2671
|
+
error: acc.errorMsg,
|
|
2672
|
+
metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
|
|
2673
|
+
outputPreview,
|
|
2674
|
+
children: []
|
|
2748
2675
|
};
|
|
2676
|
+
nodes.set(acc.id, sc);
|
|
2749
2677
|
}
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
)
|
|
2756
|
-
]
|
|
2678
|
+
const roots = [];
|
|
2679
|
+
const sortByOrder = (a, b) => {
|
|
2680
|
+
const oa = steps.get(a.id)?.order ?? 0;
|
|
2681
|
+
const ob = steps.get(b.id)?.order ?? 0;
|
|
2682
|
+
return oa - ob;
|
|
2757
2683
|
};
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
diagnostics.push(
|
|
2765
|
-
diagnostic3("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
2766
|
-
);
|
|
2767
|
-
continue;
|
|
2684
|
+
for (const acc of steps.values()) {
|
|
2685
|
+
const node = nodes.get(acc.id);
|
|
2686
|
+
if (acc.parentId !== void 0 && nodes.has(acc.parentId)) {
|
|
2687
|
+
nodes.get(acc.parentId).children.push(node);
|
|
2688
|
+
} else {
|
|
2689
|
+
roots.push(node);
|
|
2768
2690
|
}
|
|
2769
|
-
byId.set(rule.id, rule);
|
|
2770
2691
|
}
|
|
2771
|
-
|
|
2772
|
-
|
|
2773
|
-
|
|
2774
|
-
if (!byId.has(id)) {
|
|
2775
|
-
diagnostics.push(
|
|
2776
|
-
diagnostic3("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
2777
|
-
);
|
|
2778
|
-
}
|
|
2779
|
-
}
|
|
2780
|
-
return {
|
|
2781
|
-
rules: [...byId.values()].filter((rule) => selected.has(rule.id)).sort(compareRules),
|
|
2782
|
-
diagnostics
|
|
2783
|
-
};
|
|
2692
|
+
roots.sort(sortByOrder);
|
|
2693
|
+
for (const n of nodes.values()) {
|
|
2694
|
+
n.children.sort(sortByOrder);
|
|
2784
2695
|
}
|
|
2785
|
-
return {
|
|
2696
|
+
return {
|
|
2697
|
+
runId,
|
|
2698
|
+
name: rs.name,
|
|
2699
|
+
status: runStatus,
|
|
2700
|
+
durationMs,
|
|
2701
|
+
steps: roots
|
|
2702
|
+
};
|
|
2786
2703
|
}
|
|
2787
|
-
|
|
2788
|
-
|
|
2704
|
+
|
|
2705
|
+
// packages/core/src/diff/engine.ts
|
|
2706
|
+
var DEFAULT_THRESHOLD_MS = 0;
|
|
2707
|
+
function pathSeg(step, index) {
|
|
2708
|
+
return { index, name: step.name, stepId: step.id };
|
|
2789
2709
|
}
|
|
2790
|
-
function
|
|
2791
|
-
|
|
2792
|
-
return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
|
|
2710
|
+
function buildPath(segments) {
|
|
2711
|
+
return { path: [...segments] };
|
|
2793
2712
|
}
|
|
2794
|
-
function
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2713
|
+
function pairSteps(left, right) {
|
|
2714
|
+
const usedRight = /* @__PURE__ */ new Set();
|
|
2715
|
+
const pairs = [];
|
|
2716
|
+
for (let i = 0; i < left.length; i++) {
|
|
2717
|
+
const L = left[i];
|
|
2718
|
+
let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
|
|
2719
|
+
if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
|
|
2720
|
+
const cand = right[i];
|
|
2721
|
+
if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
|
|
2722
|
+
R = cand;
|
|
2723
|
+
}
|
|
2798
2724
|
}
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2725
|
+
if (R === void 0) {
|
|
2726
|
+
R = right.find(
|
|
2727
|
+
(r) => !usedRight.has(r.id) && r.name === L.name && (r.type ?? "") === (L.type ?? "")
|
|
2728
|
+
);
|
|
2729
|
+
}
|
|
2730
|
+
if (R !== void 0) {
|
|
2731
|
+
usedRight.add(R.id);
|
|
2732
|
+
pairs.push([L, R]);
|
|
2733
|
+
} else {
|
|
2734
|
+
pairs.push([L, void 0]);
|
|
2803
2735
|
}
|
|
2804
|
-
const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
|
|
2805
|
-
if (byRun !== 0) return byRun;
|
|
2806
|
-
const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
|
|
2807
|
-
if (byTime !== 0) return byTime;
|
|
2808
|
-
const byEvent = compareStrings(a.evidence[0]?.eventId, b.evidence[0]?.eventId);
|
|
2809
|
-
if (byEvent !== 0) return byEvent;
|
|
2810
|
-
return compareStrings(a.evidence[0]?.path, b.evidence[0]?.path);
|
|
2811
|
-
};
|
|
2812
|
-
}
|
|
2813
|
-
function normalizeFinding(rule, finding) {
|
|
2814
|
-
return {
|
|
2815
|
-
ruleId: finding.ruleId || rule.id,
|
|
2816
|
-
severity: finding.severity ?? rule.defaultSeverity,
|
|
2817
|
-
status: finding.status,
|
|
2818
|
-
message: finding.message,
|
|
2819
|
-
...finding.expected !== void 0 ? { expected: finding.expected } : {},
|
|
2820
|
-
...finding.actual !== void 0 ? { actual: finding.actual } : {},
|
|
2821
|
-
evidence: [...finding.evidence ?? []],
|
|
2822
|
-
...finding.category !== void 0 ? { category: finding.category } : {},
|
|
2823
|
-
...finding.confidence !== void 0 ? { confidence: finding.confidence } : {},
|
|
2824
|
-
...finding.detector !== void 0 ? { detector: finding.detector } : {},
|
|
2825
|
-
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
2826
|
-
};
|
|
2827
|
-
}
|
|
2828
|
-
function summarize(findings, diagnostics, rulesEvaluated) {
|
|
2829
|
-
return {
|
|
2830
|
-
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
2831
|
-
failed: findings.filter(
|
|
2832
|
-
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
2833
|
-
).length,
|
|
2834
|
-
warnings: findings.filter(
|
|
2835
|
-
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
2836
|
-
).length,
|
|
2837
|
-
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2838
|
-
rulesEvaluated
|
|
2839
|
-
};
|
|
2840
|
-
}
|
|
2841
|
-
function classifyRuleExecution(findings, threw) {
|
|
2842
|
-
if (findings.some((finding) => finding.status === "fail" && finding.severity === "error")) {
|
|
2843
|
-
return "fail";
|
|
2844
2736
|
}
|
|
2845
|
-
|
|
2846
|
-
(
|
|
2847
|
-
|
|
2848
|
-
|
|
2737
|
+
for (const R of right) {
|
|
2738
|
+
if (!usedRight.has(R.id)) {
|
|
2739
|
+
pairs.push([void 0, R]);
|
|
2740
|
+
}
|
|
2849
2741
|
}
|
|
2850
|
-
return
|
|
2742
|
+
return pairs;
|
|
2851
2743
|
}
|
|
2852
|
-
function
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2744
|
+
function compareLeafSteps(L, R, segments, opts, out) {
|
|
2745
|
+
const path18 = buildPath(segments);
|
|
2746
|
+
if (L.name !== R.name) {
|
|
2747
|
+
out.push({
|
|
2748
|
+
kind: "structure",
|
|
2749
|
+
severity: "warning",
|
|
2750
|
+
message: "Step name differs",
|
|
2751
|
+
path: path18,
|
|
2752
|
+
left: L.name,
|
|
2753
|
+
right: R.name
|
|
2754
|
+
});
|
|
2856
2755
|
}
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2756
|
+
if ((L.type ?? "") !== (R.type ?? "")) {
|
|
2757
|
+
out.push({
|
|
2758
|
+
kind: "step-type",
|
|
2759
|
+
severity: "warning",
|
|
2760
|
+
message: "Step type differs",
|
|
2761
|
+
path: path18,
|
|
2762
|
+
left: L.type,
|
|
2763
|
+
right: R.type
|
|
2764
|
+
});
|
|
2863
2765
|
}
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2766
|
+
if ((L.status ?? "") !== (R.status ?? "")) {
|
|
2767
|
+
out.push({
|
|
2768
|
+
kind: "step-status",
|
|
2769
|
+
severity: "warning",
|
|
2770
|
+
message: "Step status differs",
|
|
2771
|
+
path: path18,
|
|
2772
|
+
left: L.status,
|
|
2773
|
+
right: R.status
|
|
2774
|
+
});
|
|
2775
|
+
}
|
|
2776
|
+
const le = L.error ?? "";
|
|
2777
|
+
const re = R.error ?? "";
|
|
2778
|
+
if (le !== re) {
|
|
2779
|
+
out.push({
|
|
2780
|
+
kind: "error",
|
|
2781
|
+
severity: "error",
|
|
2782
|
+
message: "Step error message differs",
|
|
2783
|
+
path: path18,
|
|
2784
|
+
left: le || void 0,
|
|
2785
|
+
right: re || void 0
|
|
2786
|
+
});
|
|
2787
|
+
}
|
|
2788
|
+
if (!opts.ignoreDuration) {
|
|
2789
|
+
const ld = L.durationMs;
|
|
2790
|
+
const rd = R.durationMs;
|
|
2791
|
+
const th = opts.durationThresholdMs;
|
|
2792
|
+
let differs = false;
|
|
2793
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
2794
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
2795
|
+
else differs = Math.abs(ld - rd) > th;
|
|
2796
|
+
if (differs) {
|
|
2797
|
+
out.push({
|
|
2798
|
+
kind: "duration",
|
|
2799
|
+
severity: "info",
|
|
2800
|
+
message: "Step duration differs",
|
|
2801
|
+
path: path18,
|
|
2802
|
+
left: ld,
|
|
2803
|
+
right: rd
|
|
2804
|
+
});
|
|
2805
|
+
}
|
|
2806
|
+
}
|
|
2807
|
+
const lm = stableJson(L.metadata ?? {});
|
|
2808
|
+
const rm2 = stableJson(R.metadata ?? {});
|
|
2809
|
+
if (lm !== rm2) {
|
|
2810
|
+
out.push({
|
|
2811
|
+
kind: "metadata",
|
|
2812
|
+
severity: "info",
|
|
2813
|
+
message: "Step metadata differs",
|
|
2814
|
+
path: path18,
|
|
2815
|
+
left: L.metadata,
|
|
2816
|
+
right: R.metadata
|
|
2817
|
+
});
|
|
2818
|
+
}
|
|
2819
|
+
const lo = stableJson(L.outputPreview ?? null);
|
|
2820
|
+
const ro = stableJson(R.outputPreview ?? null);
|
|
2821
|
+
if (lo !== ro) {
|
|
2822
|
+
out.push({
|
|
2823
|
+
kind: "output",
|
|
2824
|
+
severity: "info",
|
|
2825
|
+
message: "Output preview differs",
|
|
2826
|
+
path: path18,
|
|
2827
|
+
left: L.outputPreview,
|
|
2828
|
+
right: R.outputPreview
|
|
2829
|
+
});
|
|
2868
2830
|
}
|
|
2869
|
-
return void 0;
|
|
2870
2831
|
}
|
|
2871
|
-
function
|
|
2872
|
-
|
|
2873
|
-
|
|
2832
|
+
function compareRecursive(L, R, segments, opts, out) {
|
|
2833
|
+
compareLeafSteps(L, R, segments, opts, out);
|
|
2834
|
+
const pairs = pairSteps(L.children, R.children);
|
|
2835
|
+
let ci = 0;
|
|
2836
|
+
for (const [lch, rch] of pairs) {
|
|
2837
|
+
if (lch !== void 0 && rch !== void 0) {
|
|
2838
|
+
compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
|
|
2839
|
+
} else if (lch !== void 0) {
|
|
2840
|
+
out.push({
|
|
2841
|
+
kind: "step-removed",
|
|
2842
|
+
severity: "warning",
|
|
2843
|
+
message: `Step only in left run: ${lch.name}`,
|
|
2844
|
+
path: buildPath([...segments, pathSeg(lch, ci)]),
|
|
2845
|
+
left: lch.id,
|
|
2846
|
+
right: void 0
|
|
2847
|
+
});
|
|
2848
|
+
} else if (rch !== void 0) {
|
|
2849
|
+
out.push({
|
|
2850
|
+
kind: "step-added",
|
|
2851
|
+
severity: "warning",
|
|
2852
|
+
message: `Step only in right run: ${rch.name}`,
|
|
2853
|
+
path: buildPath([...segments, pathSeg(rch, ci)]),
|
|
2854
|
+
left: void 0,
|
|
2855
|
+
right: rch.id
|
|
2856
|
+
});
|
|
2857
|
+
}
|
|
2858
|
+
ci += 1;
|
|
2874
2859
|
}
|
|
2875
|
-
return name;
|
|
2876
2860
|
}
|
|
2877
|
-
function
|
|
2861
|
+
function mergeDiffDefaults(options) {
|
|
2878
2862
|
return {
|
|
2879
|
-
|
|
2880
|
-
|
|
2881
|
-
|
|
2882
|
-
|
|
2883
|
-
spanId: event.trace?.spanId,
|
|
2884
|
-
kind: event.kind,
|
|
2885
|
-
name: event.name,
|
|
2886
|
-
status: event.status,
|
|
2887
|
-
...path18 ? { path: path18 } : {}
|
|
2863
|
+
ignoreDuration: false,
|
|
2864
|
+
durationThresholdMs: DEFAULT_THRESHOLD_MS,
|
|
2865
|
+
focus: "all",
|
|
2866
|
+
check: "all"
|
|
2888
2867
|
};
|
|
2889
2868
|
}
|
|
2890
|
-
function
|
|
2891
|
-
return
|
|
2869
|
+
function kindMatchesFilter(kind, merged) {
|
|
2870
|
+
return true;
|
|
2892
2871
|
}
|
|
2893
|
-
function
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
message,
|
|
2899
|
-
...expected !== void 0 ? { expected } : {},
|
|
2900
|
-
...actual !== void 0 ? { actual } : {},
|
|
2901
|
-
evidence: [...evidence],
|
|
2902
|
-
...{},
|
|
2903
|
-
...{},
|
|
2904
|
-
...{},
|
|
2905
|
-
...{}
|
|
2872
|
+
function diffRuns(left, right, options) {
|
|
2873
|
+
const merged = mergeDiffDefaults();
|
|
2874
|
+
const opts = {
|
|
2875
|
+
ignoreDuration: merged.ignoreDuration,
|
|
2876
|
+
durationThresholdMs: merged.durationThresholdMs
|
|
2906
2877
|
};
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
}
|
|
2934
|
-
function createRunStatusRule(options = {}) {
|
|
2935
|
-
const expected = options.expected ?? "ok";
|
|
2936
|
-
const allowIncomplete = options.allowIncomplete === true;
|
|
2937
|
-
return {
|
|
2938
|
-
id: "run.status",
|
|
2939
|
-
category: "run",
|
|
2940
|
-
defaultSeverity: "error",
|
|
2941
|
-
evaluate(context) {
|
|
2942
|
-
const findings = [];
|
|
2943
|
-
const actual = context.selectedRun?.status ?? "unknown";
|
|
2944
|
-
if (actual !== expected) {
|
|
2945
|
-
findings.push(
|
|
2946
|
-
failFinding(
|
|
2947
|
-
"run.status",
|
|
2948
|
-
`Run status ${actual} did not match expected ${expected}.`,
|
|
2949
|
-
runEvidence(context.selectedRun),
|
|
2950
|
-
expected,
|
|
2951
|
-
actual
|
|
2952
|
-
)
|
|
2953
|
-
);
|
|
2954
|
-
}
|
|
2955
|
-
if (!allowIncomplete) {
|
|
2956
|
-
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
2957
|
-
if (running.length > 0) {
|
|
2958
|
-
findings.push(
|
|
2959
|
-
failFinding(
|
|
2960
|
-
"run.status",
|
|
2961
|
-
"Run contains incomplete running events.",
|
|
2962
|
-
running.map((event) => eventEvidence(event)),
|
|
2963
|
-
"no running events",
|
|
2964
|
-
running.length
|
|
2965
|
-
)
|
|
2966
|
-
);
|
|
2967
|
-
}
|
|
2968
|
-
}
|
|
2969
|
-
return findings;
|
|
2878
|
+
const raw = [];
|
|
2879
|
+
if ((left.status ?? "") !== (right.status ?? "")) {
|
|
2880
|
+
raw.push({
|
|
2881
|
+
kind: "run-status",
|
|
2882
|
+
severity: "warning",
|
|
2883
|
+
message: "Run completion status differs",
|
|
2884
|
+
left: left.status,
|
|
2885
|
+
right: right.status
|
|
2886
|
+
});
|
|
2887
|
+
}
|
|
2888
|
+
{
|
|
2889
|
+
const ld = left.durationMs;
|
|
2890
|
+
const rd = right.durationMs;
|
|
2891
|
+
const th = merged.durationThresholdMs;
|
|
2892
|
+
let differs = false;
|
|
2893
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
2894
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
2895
|
+
else differs = Math.abs(ld - rd) > th;
|
|
2896
|
+
if (differs) {
|
|
2897
|
+
raw.push({
|
|
2898
|
+
kind: "duration",
|
|
2899
|
+
severity: "info",
|
|
2900
|
+
message: "Run duration differs",
|
|
2901
|
+
left: ld,
|
|
2902
|
+
right: rd
|
|
2903
|
+
});
|
|
2970
2904
|
}
|
|
2971
|
-
}
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2905
|
+
}
|
|
2906
|
+
const pairs = pairSteps(left.steps, right.steps);
|
|
2907
|
+
let idx = 0;
|
|
2908
|
+
for (const [ls, rs] of pairs) {
|
|
2909
|
+
if (ls !== void 0 && rs !== void 0) {
|
|
2910
|
+
compareRecursive(ls, rs, [pathSeg(ls, idx)], opts, raw);
|
|
2911
|
+
idx += 1;
|
|
2912
|
+
} else if (ls !== void 0) {
|
|
2913
|
+
raw.push({
|
|
2914
|
+
kind: "step-removed",
|
|
2915
|
+
severity: "warning",
|
|
2916
|
+
message: `Step only in left run: ${ls.name}`,
|
|
2917
|
+
path: buildPath([pathSeg(ls, idx)]),
|
|
2918
|
+
left: ls.id,
|
|
2919
|
+
right: void 0
|
|
2920
|
+
});
|
|
2921
|
+
idx += 1;
|
|
2922
|
+
} else if (rs !== void 0) {
|
|
2923
|
+
raw.push({
|
|
2924
|
+
kind: "step-added",
|
|
2925
|
+
severity: "warning",
|
|
2926
|
+
message: `Step only in right run: ${rs.name}`,
|
|
2927
|
+
path: buildPath([pathSeg(rs, idx)]),
|
|
2928
|
+
left: void 0,
|
|
2929
|
+
right: rs.id
|
|
2930
|
+
});
|
|
2931
|
+
idx += 1;
|
|
2990
2932
|
}
|
|
2933
|
+
}
|
|
2934
|
+
const differences = raw.filter((d) => kindMatchesFilter(d.kind));
|
|
2935
|
+
let errors = 0;
|
|
2936
|
+
let warnings = 0;
|
|
2937
|
+
let info = 0;
|
|
2938
|
+
for (const d of differences) {
|
|
2939
|
+
if (d.severity === "error") errors += 1;
|
|
2940
|
+
else if (d.severity === "warning") warnings += 1;
|
|
2941
|
+
else info += 1;
|
|
2942
|
+
}
|
|
2943
|
+
const firstVisible = differences[0];
|
|
2944
|
+
const firstDivergence = firstVisible !== void 0 ? {
|
|
2945
|
+
kind: "first-divergence",
|
|
2946
|
+
severity: firstVisible.severity,
|
|
2947
|
+
message: `First divergence: ${firstVisible.message}`,
|
|
2948
|
+
path: firstVisible.path,
|
|
2949
|
+
left: firstVisible.left,
|
|
2950
|
+
right: firstVisible.right
|
|
2951
|
+
} : void 0;
|
|
2952
|
+
const summary = {
|
|
2953
|
+
leftRunId: left.runId,
|
|
2954
|
+
rightRunId: right.runId,
|
|
2955
|
+
totalDifferences: differences.length,
|
|
2956
|
+
errors,
|
|
2957
|
+
warnings,
|
|
2958
|
+
info,
|
|
2959
|
+
firstDivergence
|
|
2991
2960
|
};
|
|
2961
|
+
return { summary, differences };
|
|
2992
2962
|
}
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
const nameSet = new Set(names);
|
|
3002
|
-
const findings = [];
|
|
3003
|
-
for (const required of options.required ?? []) {
|
|
3004
|
-
if (!nameSet.has(required)) {
|
|
3005
|
-
findings.push(
|
|
3006
|
-
failFinding("tool.usage", `Required tool ${required} did not appear.`, runEvidence(context.selectedRun), required, names)
|
|
3007
|
-
);
|
|
3008
|
-
}
|
|
3009
|
-
}
|
|
3010
|
-
const forbidden = new Set(options.forbidden ?? []);
|
|
3011
|
-
const allowed = options.allowed?.length ? new Set(options.allowed) : void 0;
|
|
3012
|
-
for (const event of tools) {
|
|
3013
|
-
const name = toolName(event);
|
|
3014
|
-
if (forbidden.has(name)) {
|
|
3015
|
-
findings.push(
|
|
3016
|
-
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence(event)], "tool absent", name)
|
|
3017
|
-
);
|
|
3018
|
-
}
|
|
3019
|
-
if (allowed && !allowed.has(name)) {
|
|
3020
|
-
findings.push(
|
|
3021
|
-
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence(event)], [...allowed].sort(), name)
|
|
3022
|
-
);
|
|
3023
|
-
}
|
|
3024
|
-
}
|
|
3025
|
-
if (options.minCount !== void 0 && tools.length < options.minCount) {
|
|
3026
|
-
findings.push(
|
|
3027
|
-
failFinding("tool.usage", `Tool count ${tools.length} was below minimum ${options.minCount}.`, runEvidence(context.selectedRun), { minCount: options.minCount }, tools.length)
|
|
3028
|
-
);
|
|
3029
|
-
}
|
|
3030
|
-
if (options.maxCount !== void 0 && tools.length > options.maxCount) {
|
|
3031
|
-
findings.push(
|
|
3032
|
-
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) => eventEvidence(event)), { maxCount: options.maxCount }, tools.length)
|
|
3033
|
-
);
|
|
3034
|
-
}
|
|
3035
|
-
return findings;
|
|
2963
|
+
|
|
2964
|
+
// packages/core/src/evidence/zip.ts
|
|
2965
|
+
(() => {
|
|
2966
|
+
const table = new Uint32Array(256);
|
|
2967
|
+
for (let n = 0; n < 256; n += 1) {
|
|
2968
|
+
let c = n;
|
|
2969
|
+
for (let k = 0; k < 8; k += 1) {
|
|
2970
|
+
c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
|
|
3036
2971
|
}
|
|
3037
|
-
|
|
2972
|
+
table[n] = c >>> 0;
|
|
2973
|
+
}
|
|
2974
|
+
return table;
|
|
2975
|
+
})();
|
|
2976
|
+
|
|
2977
|
+
// packages/core/src/suite/types.ts
|
|
2978
|
+
var DEFAULT_SUITE_CONFIG_NAMES = [
|
|
2979
|
+
"agent-inspect.suite.json",
|
|
2980
|
+
"agent-inspect.suite.js",
|
|
2981
|
+
"agent-inspect.suite.mjs",
|
|
2982
|
+
"agent-inspect.suite.cjs"
|
|
2983
|
+
];
|
|
2984
|
+
function diagnostic2(code, message, severity = "error", caseId) {
|
|
2985
|
+
return { code, message, severity, ...{} };
|
|
3038
2986
|
}
|
|
3039
|
-
function
|
|
3040
|
-
return
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
const llms = finishedEvents(context, "LLM");
|
|
3046
|
-
const findings = [];
|
|
3047
|
-
const allowedModels = options.allowedModels?.length ? new Set(options.allowedModels) : void 0;
|
|
3048
|
-
const allowedProviders = options.allowedProviders?.length ? new Set(options.allowedProviders) : void 0;
|
|
3049
|
-
const finishReasons = options.finishReasons?.length ? new Set(options.finishReasons) : void 0;
|
|
3050
|
-
if (options.maxCalls !== void 0 && llms.length > options.maxCalls) {
|
|
3051
|
-
findings.push(
|
|
3052
|
-
failFinding(
|
|
3053
|
-
"llm.usage",
|
|
3054
|
-
`LLM call count ${llms.length} exceeded ${options.maxCalls}.`,
|
|
3055
|
-
llms.map((event) => eventEvidence(event)),
|
|
3056
|
-
{ maxCalls: options.maxCalls },
|
|
3057
|
-
llms.length
|
|
3058
|
-
)
|
|
3059
|
-
);
|
|
3060
|
-
}
|
|
3061
|
-
for (const event of llms) {
|
|
3062
|
-
const model = llmModel(event);
|
|
3063
|
-
const provider = llmProvider(event);
|
|
3064
|
-
const finishReason = llmFinishReason(event);
|
|
3065
|
-
if (allowedModels && (!model || !allowedModels.has(model))) {
|
|
3066
|
-
findings.push(
|
|
3067
|
-
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.model")], [...allowedModels].sort(), model ?? "unknown")
|
|
3068
|
-
);
|
|
3069
|
-
}
|
|
3070
|
-
if (allowedProviders && (!provider || !allowedProviders.has(provider))) {
|
|
3071
|
-
findings.push(
|
|
3072
|
-
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.provider")], [...allowedProviders].sort(), provider ?? "unknown")
|
|
3073
|
-
);
|
|
3074
|
-
}
|
|
3075
|
-
if (finishReasons && (!finishReason || !finishReasons.has(finishReason))) {
|
|
3076
|
-
findings.push(
|
|
3077
|
-
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.finishReason")], [...finishReasons].sort(), finishReason ?? "unknown")
|
|
3078
|
-
);
|
|
3079
|
-
}
|
|
3080
|
-
}
|
|
3081
|
-
const tokenTotals = llms.reduce(
|
|
3082
|
-
(totals, event) => ({
|
|
3083
|
-
input: totals.input + (event.tokenUsage?.input ?? 0),
|
|
3084
|
-
output: totals.output + (event.tokenUsage?.output ?? 0),
|
|
3085
|
-
total: totals.total + (event.tokenUsage?.total ?? 0),
|
|
3086
|
-
cached: totals.cached + (event.tokenUsage?.cached ?? 0)
|
|
3087
|
-
}),
|
|
3088
|
-
{ input: 0, output: 0, total: 0, cached: 0 }
|
|
3089
|
-
);
|
|
3090
|
-
const tokenLimits = [
|
|
3091
|
-
["input", options.maxInputTokens],
|
|
3092
|
-
["output", options.maxOutputTokens],
|
|
3093
|
-
["total", options.maxTotalTokens],
|
|
3094
|
-
["cached", options.maxCachedTokens]
|
|
3095
|
-
];
|
|
3096
|
-
for (const [key, limit] of tokenLimits) {
|
|
3097
|
-
if (limit !== void 0 && tokenTotals[key] > limit) {
|
|
3098
|
-
findings.push(
|
|
3099
|
-
failFinding(
|
|
3100
|
-
"llm.usage",
|
|
3101
|
-
`LLM ${key} token count ${tokenTotals[key]} exceeded ${limit}.`,
|
|
3102
|
-
llms.map((event) => eventEvidence(event, `tokenUsage.${key}`)),
|
|
3103
|
-
{ [`max${key[0].toUpperCase()}${key.slice(1)}Tokens`]: limit },
|
|
3104
|
-
tokenTotals[key]
|
|
3105
|
-
)
|
|
3106
|
-
);
|
|
3107
|
-
}
|
|
3108
|
-
}
|
|
3109
|
-
return findings;
|
|
3110
|
-
}
|
|
3111
|
-
};
|
|
2987
|
+
function asString(value, label) {
|
|
2988
|
+
if (value === void 0) return void 0;
|
|
2989
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
2990
|
+
throw new Error(`${label} must be a non-empty string.`);
|
|
2991
|
+
}
|
|
2992
|
+
return value.trim();
|
|
3112
2993
|
}
|
|
3113
|
-
function
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
kind: "OUTCOME",
|
|
3144
|
-
name: outcome.name,
|
|
3145
|
-
status: outcome.status,
|
|
3146
|
-
path: `outcome.${outcome.name}`
|
|
3147
|
-
})),
|
|
3148
|
-
{ failOn },
|
|
3149
|
-
matching.map((outcome) => ({
|
|
3150
|
-
name: outcome.name,
|
|
3151
|
-
status: outcome.status,
|
|
3152
|
-
expectation: outcome.expectation
|
|
3153
|
-
}))
|
|
3154
|
-
)
|
|
3155
|
-
];
|
|
2994
|
+
function asStringArray(value, label) {
|
|
2995
|
+
if (value === void 0) return void 0;
|
|
2996
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
2997
|
+
throw new Error(`${label} must be an array of strings.`);
|
|
2998
|
+
}
|
|
2999
|
+
return value;
|
|
3000
|
+
}
|
|
3001
|
+
function asPositiveNumber(value, label) {
|
|
3002
|
+
if (value === void 0) return void 0;
|
|
3003
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
3004
|
+
throw new Error(`${label} must be a non-negative number.`);
|
|
3005
|
+
}
|
|
3006
|
+
return value;
|
|
3007
|
+
}
|
|
3008
|
+
function validateCaseConfig(value, index) {
|
|
3009
|
+
const diagnostics = [];
|
|
3010
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
3011
|
+
diagnostics.push(
|
|
3012
|
+
diagnostic2("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
|
|
3013
|
+
);
|
|
3014
|
+
return { diagnostics };
|
|
3015
|
+
}
|
|
3016
|
+
const raw = value;
|
|
3017
|
+
try {
|
|
3018
|
+
const id = asString(raw.id, `cases[${index}].id`);
|
|
3019
|
+
if (id === void 0) {
|
|
3020
|
+
diagnostics.push(
|
|
3021
|
+
diagnostic2("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
|
|
3022
|
+
);
|
|
3023
|
+
return { diagnostics };
|
|
3156
3024
|
}
|
|
3157
|
-
|
|
3025
|
+
const trace = asString(raw.trace, `cases[${index}].trace`);
|
|
3026
|
+
const runId = asString(raw.runId, `cases[${index}].runId`);
|
|
3027
|
+
const input = asString(raw.input, `cases[${index}].input`);
|
|
3028
|
+
return {
|
|
3029
|
+
caseConfig: {
|
|
3030
|
+
id,
|
|
3031
|
+
...trace !== void 0 ? { trace } : {},
|
|
3032
|
+
...runId !== void 0 ? { runId } : {},
|
|
3033
|
+
...input !== void 0 ? { input } : {},
|
|
3034
|
+
...asStringArray(raw.requireTools, `cases[${index}].requireTools`) !== void 0 ? { requireTools: asStringArray(raw.requireTools, `cases[${index}].requireTools`) } : {},
|
|
3035
|
+
...asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) !== void 0 ? { forbidTools: asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) } : {},
|
|
3036
|
+
...asPositiveNumber(raw.maxDurationMs, `cases[${index}].maxDurationMs`) !== void 0 ? {
|
|
3037
|
+
maxDurationMs: asPositiveNumber(
|
|
3038
|
+
raw.maxDurationMs,
|
|
3039
|
+
`cases[${index}].maxDurationMs`
|
|
3040
|
+
)
|
|
3041
|
+
} : {},
|
|
3042
|
+
...asStringArray(
|
|
3043
|
+
raw.expectedObservations,
|
|
3044
|
+
`cases[${index}].expectedObservations`
|
|
3045
|
+
) !== void 0 ? {
|
|
3046
|
+
expectedObservations: asStringArray(
|
|
3047
|
+
raw.expectedObservations,
|
|
3048
|
+
`cases[${index}].expectedObservations`
|
|
3049
|
+
)
|
|
3050
|
+
} : {}
|
|
3051
|
+
},
|
|
3052
|
+
diagnostics
|
|
3053
|
+
};
|
|
3054
|
+
} catch (error) {
|
|
3055
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3056
|
+
diagnostics.push(diagnostic2("AI_SUITE_CONFIG_INVALID", message));
|
|
3057
|
+
return { diagnostics };
|
|
3058
|
+
}
|
|
3158
3059
|
}
|
|
3159
|
-
function
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
return errorResult(input, selected.diagnostics, selected.run);
|
|
3060
|
+
function normalizeSuiteConfig(value) {
|
|
3061
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
3062
|
+
throw new Error("Suite config must export an object.");
|
|
3163
3063
|
}
|
|
3164
|
-
const
|
|
3165
|
-
|
|
3166
|
-
|
|
3064
|
+
const raw = value;
|
|
3065
|
+
const name = asString(raw.name, "name");
|
|
3066
|
+
const traces = asString(raw.traces, "traces");
|
|
3067
|
+
if (name === void 0) throw new Error("name is required.");
|
|
3068
|
+
if (traces === void 0) throw new Error("traces is required.");
|
|
3069
|
+
if (!Array.isArray(raw.cases) || raw.cases.length === 0) {
|
|
3070
|
+
throw new Error("cases must be a non-empty array.");
|
|
3167
3071
|
}
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3072
|
+
const cases = [];
|
|
3073
|
+
for (let index = 0; index < raw.cases.length; index += 1) {
|
|
3074
|
+
const { caseConfig, diagnostics } = validateCaseConfig(raw.cases[index], index);
|
|
3075
|
+
if (diagnostics.length > 0) {
|
|
3076
|
+
throw new Error(diagnostics.map((item) => item.message).join("; "));
|
|
3077
|
+
}
|
|
3078
|
+
if (caseConfig !== void 0) cases.push(caseConfig);
|
|
3079
|
+
}
|
|
3080
|
+
const ids = /* @__PURE__ */ new Set();
|
|
3081
|
+
for (const suiteCase of cases) {
|
|
3082
|
+
if (ids.has(suiteCase.id)) {
|
|
3083
|
+
throw new Error(`Duplicate case id "${suiteCase.id}".`);
|
|
3084
|
+
}
|
|
3085
|
+
ids.add(suiteCase.id);
|
|
3086
|
+
}
|
|
3087
|
+
const redactionProfile = raw.redactionProfile === "local" || raw.redactionProfile === "share" || raw.redactionProfile === "strict" ? raw.redactionProfile : void 0;
|
|
3088
|
+
if (raw.redactionProfile !== void 0 && redactionProfile === void 0) {
|
|
3089
|
+
throw new Error('redactionProfile must be "local", "share", or "strict".');
|
|
3090
|
+
}
|
|
3091
|
+
const config = { name, traces, cases };
|
|
3092
|
+
if (raw.checks !== void 0) {
|
|
3093
|
+
if (typeof raw.checks !== "object" || Array.isArray(raw.checks)) {
|
|
3094
|
+
throw new Error("checks must be an object.");
|
|
3095
|
+
}
|
|
3096
|
+
config.checks = raw.checks;
|
|
3097
|
+
}
|
|
3098
|
+
if (raw.eval !== void 0) {
|
|
3099
|
+
if (typeof raw.eval !== "object" || Array.isArray(raw.eval)) {
|
|
3100
|
+
throw new Error("eval must be an object.");
|
|
3101
|
+
}
|
|
3102
|
+
config.eval = raw.eval;
|
|
3103
|
+
}
|
|
3104
|
+
if (raw.artifacts !== void 0) {
|
|
3105
|
+
if (typeof raw.artifacts !== "object" || Array.isArray(raw.artifacts)) {
|
|
3106
|
+
throw new Error("artifacts must be an object.");
|
|
3107
|
+
}
|
|
3108
|
+
const outputDir = asString(
|
|
3109
|
+
raw.artifacts.outputDir,
|
|
3110
|
+
"artifacts.outputDir"
|
|
3178
3111
|
);
|
|
3112
|
+
config.artifacts = outputDir !== void 0 ? { outputDir } : {};
|
|
3179
3113
|
}
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
|
|
3114
|
+
if (raw.baseline !== void 0) {
|
|
3115
|
+
const baseline = asString(raw.baseline, "baseline");
|
|
3116
|
+
if (baseline !== void 0) config.baseline = baseline;
|
|
3117
|
+
}
|
|
3118
|
+
if (raw.candidate !== void 0) {
|
|
3119
|
+
const candidate = asString(raw.candidate, "candidate");
|
|
3120
|
+
if (candidate !== void 0) config.candidate = candidate;
|
|
3121
|
+
}
|
|
3122
|
+
if (redactionProfile !== void 0) config.redactionProfile = redactionProfile;
|
|
3123
|
+
return config;
|
|
3124
|
+
}
|
|
3125
|
+
|
|
3126
|
+
// packages/core/src/suite/load.ts
|
|
3127
|
+
var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
3128
|
+
var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
3129
|
+
function diagnostic3(code, message) {
|
|
3130
|
+
return { code, message, severity: "error" };
|
|
3131
|
+
}
|
|
3132
|
+
async function fileExists(filePath) {
|
|
3133
|
+
try {
|
|
3134
|
+
await promises.access(filePath);
|
|
3135
|
+
return true;
|
|
3136
|
+
} catch {
|
|
3137
|
+
return false;
|
|
3138
|
+
}
|
|
3139
|
+
}
|
|
3140
|
+
async function resolveSuiteConfigPath(options = {}) {
|
|
3141
|
+
const cwd = path15__default.default.resolve(options.cwd ?? process.cwd());
|
|
3142
|
+
if (options.configPath !== void 0 && options.configPath.trim() !== "") {
|
|
3143
|
+
return path15__default.default.resolve(cwd, options.configPath.trim());
|
|
3144
|
+
}
|
|
3145
|
+
for (const name of DEFAULT_SUITE_CONFIG_NAMES) {
|
|
3146
|
+
const candidate = path15__default.default.join(cwd, name);
|
|
3147
|
+
if (await fileExists(candidate)) return candidate;
|
|
3148
|
+
}
|
|
3149
|
+
throw new Error(
|
|
3150
|
+
`No suite config found. Create one with \`agent-inspect suite init\` or pass --config.`
|
|
3151
|
+
);
|
|
3152
|
+
}
|
|
3153
|
+
async function loadSuiteConfig(options = {}) {
|
|
3154
|
+
let configPath;
|
|
3155
|
+
try {
|
|
3156
|
+
configPath = await resolveSuiteConfigPath(options);
|
|
3157
|
+
} catch (error) {
|
|
3158
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3159
|
+
throw Object.assign(new Error(message), {
|
|
3160
|
+
diagnostics: [diagnostic3("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
3161
|
+
});
|
|
3162
|
+
}
|
|
3163
|
+
const extension = path15__default.default.extname(configPath);
|
|
3164
|
+
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
3165
|
+
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
3166
|
+
throw Object.assign(new Error(message), {
|
|
3167
|
+
diagnostics: [diagnostic3("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
3168
|
+
});
|
|
3169
|
+
}
|
|
3170
|
+
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
3171
|
+
const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
|
|
3172
|
+
throw Object.assign(new Error(message), {
|
|
3173
|
+
diagnostics: [diagnostic3("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
3174
|
+
});
|
|
3175
|
+
}
|
|
3176
|
+
try {
|
|
3177
|
+
let raw;
|
|
3178
|
+
if (extension === ".json") {
|
|
3179
|
+
raw = JSON.parse(await promises.readFile(configPath, "utf-8"));
|
|
3180
|
+
} else {
|
|
3181
|
+
const mod = await import(url.pathToFileURL(configPath).href);
|
|
3182
|
+
raw = "default" in mod ? mod.default : mod;
|
|
3212
3183
|
}
|
|
3184
|
+
const config = normalizeSuiteConfig(raw);
|
|
3185
|
+
return {
|
|
3186
|
+
config,
|
|
3187
|
+
configPath,
|
|
3188
|
+
configDir: path15__default.default.dirname(configPath)
|
|
3189
|
+
};
|
|
3190
|
+
} catch (error) {
|
|
3191
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3192
|
+
throw Object.assign(new Error(message), {
|
|
3193
|
+
diagnostics: [diagnostic3("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
3194
|
+
});
|
|
3213
3195
|
}
|
|
3214
|
-
|
|
3215
|
-
|
|
3196
|
+
}
|
|
3197
|
+
async function exists(filePath) {
|
|
3198
|
+
try {
|
|
3199
|
+
await promises.access(filePath);
|
|
3200
|
+
return true;
|
|
3201
|
+
} catch {
|
|
3202
|
+
return false;
|
|
3203
|
+
}
|
|
3204
|
+
}
|
|
3205
|
+
async function resolveSuiteCaseTrace(suiteCase, options) {
|
|
3206
|
+
if (suiteCase.trace !== void 0) {
|
|
3207
|
+
const tracePath = path15__default.default.resolve(options.configDir, suiteCase.trace);
|
|
3208
|
+
if (await exists(tracePath)) {
|
|
3209
|
+
return { caseId: suiteCase.id, tracePath, missing: false };
|
|
3210
|
+
}
|
|
3211
|
+
return {
|
|
3212
|
+
caseId: suiteCase.id,
|
|
3213
|
+
tracePath,
|
|
3214
|
+
missing: true,
|
|
3215
|
+
reason: `trace file not found: ${suiteCase.trace}`
|
|
3216
|
+
};
|
|
3217
|
+
}
|
|
3218
|
+
const runKey = suiteCase.runId ?? suiteCase.id;
|
|
3219
|
+
const directPath = getTraceFilePath(runKey, options.tracesDir);
|
|
3220
|
+
if (await exists(directPath)) {
|
|
3221
|
+
return { caseId: suiteCase.id, tracePath: directPath, runId: runKey, missing: false };
|
|
3222
|
+
}
|
|
3223
|
+
const nestedPath = path15__default.default.join(options.tracesDir, `${path15__default.default.basename(runKey)}.jsonl`);
|
|
3224
|
+
if (await exists(nestedPath)) {
|
|
3225
|
+
return { caseId: suiteCase.id, tracePath: nestedPath, runId: runKey, missing: false };
|
|
3216
3226
|
}
|
|
3217
|
-
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
3218
|
-
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
3219
|
-
const summary = summarize(sortedFindings, diagnostics, ruleExecutions.length);
|
|
3220
|
-
const status = summary.failed > 0 ? "fail" : "pass";
|
|
3221
3227
|
return {
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
summary,
|
|
3227
|
-
findings: sortedFindings,
|
|
3228
|
-
diagnostics,
|
|
3229
|
-
ruleExecutions
|
|
3228
|
+
caseId: suiteCase.id,
|
|
3229
|
+
runId: runKey,
|
|
3230
|
+
missing: true,
|
|
3231
|
+
reason: `no trace found for run id "${runKey}" under ${options.tracesDir}`
|
|
3230
3232
|
};
|
|
3231
3233
|
}
|
|
3232
3234
|
|
|
@@ -3243,16 +3245,20 @@ function normalizeTokenUsage(value) {
|
|
|
3243
3245
|
const output = nonNegativeFinite(value.output);
|
|
3244
3246
|
const suppliedTotal = nonNegativeFinite(value.total);
|
|
3245
3247
|
const cached = nonNegativeFinite(value.cached);
|
|
3248
|
+
const cacheWrite = nonNegativeFinite(value.cacheWrite);
|
|
3249
|
+
const reasoning = nonNegativeFinite(value.reasoning);
|
|
3246
3250
|
const derivedTotal = input !== void 0 && output !== void 0 && Number.isFinite(input + output) ? input + output : void 0;
|
|
3247
3251
|
const total = suppliedTotal ?? derivedTotal;
|
|
3248
|
-
if (input === void 0 && output === void 0 && total === void 0 && cached === void 0) {
|
|
3252
|
+
if (input === void 0 && output === void 0 && total === void 0 && cached === void 0 && cacheWrite === void 0 && reasoning === void 0) {
|
|
3249
3253
|
return void 0;
|
|
3250
3254
|
}
|
|
3251
3255
|
return {
|
|
3252
3256
|
...input !== void 0 ? { input } : {},
|
|
3253
3257
|
...output !== void 0 ? { output } : {},
|
|
3254
3258
|
...total !== void 0 ? { total } : {},
|
|
3255
|
-
...cached !== void 0 ? { cached } : {}
|
|
3259
|
+
...cached !== void 0 ? { cached } : {},
|
|
3260
|
+
...cacheWrite !== void 0 ? { cacheWrite } : {},
|
|
3261
|
+
...reasoning !== void 0 ? { reasoning } : {}
|
|
3256
3262
|
};
|
|
3257
3263
|
}
|
|
3258
3264
|
|
|
@@ -4411,6 +4417,8 @@ function readOpenInferenceTokenUsage(attributes) {
|
|
|
4411
4417
|
const completion = attributes["llm.token_count.completion"];
|
|
4412
4418
|
const total = attributes["llm.token_count.total"];
|
|
4413
4419
|
const cached = attributes["llm.token_count.prompt_details.cache_read"];
|
|
4420
|
+
const cacheWrite = attributes["llm.token_count.prompt_details.cache_write"] ?? attributes["llm.token_count.prompt_details.cache_creation"];
|
|
4421
|
+
const reasoning = attributes["llm.token_count.completion_details.reasoning"] ?? attributes["llm.token_count.completion_details.reasoning_tokens"];
|
|
4414
4422
|
const usage = {};
|
|
4415
4423
|
if (typeof prompt === "number" && Number.isFinite(prompt) && prompt >= 0) {
|
|
4416
4424
|
usage.input = prompt;
|
|
@@ -4424,6 +4432,12 @@ function readOpenInferenceTokenUsage(attributes) {
|
|
|
4424
4432
|
if (typeof cached === "number" && Number.isFinite(cached) && cached >= 0) {
|
|
4425
4433
|
usage.cached = cached;
|
|
4426
4434
|
}
|
|
4435
|
+
if (typeof cacheWrite === "number" && Number.isFinite(cacheWrite) && cacheWrite >= 0) {
|
|
4436
|
+
usage.cacheWrite = cacheWrite;
|
|
4437
|
+
}
|
|
4438
|
+
if (typeof reasoning === "number" && Number.isFinite(reasoning) && reasoning >= 0) {
|
|
4439
|
+
usage.reasoning = reasoning;
|
|
4440
|
+
}
|
|
4427
4441
|
if (usage.total === void 0 && usage.input !== void 0 && usage.output !== void 0) {
|
|
4428
4442
|
usage.total = usage.input + usage.output;
|
|
4429
4443
|
}
|