@agent-inspect/viewer 6.27.0 → 6.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1502 -1488
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +5 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.mjs +1502 -1488
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -165,6 +165,8 @@ function isPersistedTokenUsage(value) {
|
|
|
165
165
|
if (!isOptionalNonNegativeNumber(value.output)) return false;
|
|
166
166
|
if (!isOptionalNonNegativeNumber(value.total)) return false;
|
|
167
167
|
if (!isOptionalNonNegativeNumber(value.cached)) return false;
|
|
168
|
+
if (!isOptionalNonNegativeNumber(value.cacheWrite)) return false;
|
|
169
|
+
if (!isOptionalNonNegativeNumber(value.reasoning)) return false;
|
|
168
170
|
return true;
|
|
169
171
|
}
|
|
170
172
|
function isPersistedTraceContext(value) {
|
|
@@ -1587,1639 +1589,1639 @@ function buildSessionIndex(inputRuns, options = {}) {
|
|
|
1587
1589
|
};
|
|
1588
1590
|
}
|
|
1589
1591
|
|
|
1590
|
-
// packages/core/src/
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1592
|
+
// packages/core/src/diagnostics/programmatic.ts
|
|
1593
|
+
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
1594
|
+
AI_TRACE_INPUT_INVALID: {
|
|
1595
|
+
code: "AI_TRACE_INPUT_INVALID",
|
|
1596
|
+
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
1597
|
+
remediation: "For a file path, use openTraceFile(path).",
|
|
1598
|
+
relatedCodes: ["invalid_input"]
|
|
1599
|
+
},
|
|
1600
|
+
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
1601
|
+
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
1602
|
+
summary: "No trace reader could detect the input format.",
|
|
1603
|
+
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
1604
|
+
relatedCodes: ["unsupported_format"]
|
|
1605
|
+
},
|
|
1606
|
+
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
1607
|
+
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
1608
|
+
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
1609
|
+
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
1610
|
+
relatedCodes: ["ambiguous_format"]
|
|
1611
|
+
},
|
|
1612
|
+
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
1613
|
+
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
1614
|
+
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
1615
|
+
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
1616
|
+
},
|
|
1617
|
+
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
1618
|
+
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
1619
|
+
summary: "Multiple runs are available; select a run before executing checks.",
|
|
1620
|
+
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
1621
|
+
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
1622
|
+
},
|
|
1623
|
+
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
1624
|
+
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1625
|
+
summary: "A parentId equals its own event/step id.",
|
|
1626
|
+
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
1627
|
+
relatedCodes: [
|
|
1628
|
+
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
1629
|
+
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
1630
|
+
]
|
|
1631
|
+
},
|
|
1632
|
+
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
1633
|
+
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
1634
|
+
summary: "Trace contains a parentId cycle.",
|
|
1635
|
+
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
1636
|
+
relatedCodes: ["structure.cycle"]
|
|
1598
1637
|
}
|
|
1599
|
-
|
|
1638
|
+
});
|
|
1639
|
+
function formatProgrammaticDiagnostic(code, detail) {
|
|
1640
|
+
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
1641
|
+
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
1642
|
+
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
1600
1643
|
}
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1644
|
+
|
|
1645
|
+
// packages/core/src/safety/sensitive-key.ts
|
|
1646
|
+
function normalizeSensitiveKey(value) {
|
|
1647
|
+
return value.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
1604
1648
|
}
|
|
1649
|
+
new Set(
|
|
1650
|
+
[
|
|
1651
|
+
"tokens",
|
|
1652
|
+
"max_tokens",
|
|
1653
|
+
"min_tokens",
|
|
1654
|
+
"ls_max_tokens",
|
|
1655
|
+
"token_count",
|
|
1656
|
+
"token_limit",
|
|
1657
|
+
"token_budget",
|
|
1658
|
+
"input_tokens",
|
|
1659
|
+
"output_tokens",
|
|
1660
|
+
"total_tokens",
|
|
1661
|
+
"cached_tokens",
|
|
1662
|
+
"prompt_tokens",
|
|
1663
|
+
"completion_tokens"
|
|
1664
|
+
].map(normalizeSensitiveKey)
|
|
1665
|
+
);
|
|
1605
1666
|
|
|
1606
|
-
// packages/core/src/
|
|
1607
|
-
function
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1667
|
+
// packages/core/src/checks/logical-events.ts
|
|
1668
|
+
function isRecord5(value) {
|
|
1669
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1670
|
+
}
|
|
1671
|
+
function legacyEvent(event) {
|
|
1672
|
+
const value = event.attributes?.legacyEvent;
|
|
1673
|
+
return typeof value === "string" ? value : void 0;
|
|
1674
|
+
}
|
|
1675
|
+
function stepIdOf(event) {
|
|
1676
|
+
const value = event.attributes?.stepId;
|
|
1677
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
1611
1678
|
return void 0;
|
|
1612
1679
|
}
|
|
1613
|
-
function
|
|
1614
|
-
|
|
1615
|
-
|
|
1680
|
+
function cloneEvent(event) {
|
|
1681
|
+
return {
|
|
1682
|
+
...event,
|
|
1683
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
1684
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
1685
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
1686
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
1687
|
+
};
|
|
1616
1688
|
}
|
|
1617
|
-
function
|
|
1618
|
-
const
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
}
|
|
1622
|
-
const rs = started;
|
|
1623
|
-
const runId = rs.runId;
|
|
1624
|
-
const completedAll = events.filter((e) => e.event === "run_completed");
|
|
1625
|
-
const lastCompleted = completedAll[completedAll.length - 1];
|
|
1626
|
-
let runStatus;
|
|
1627
|
-
if (lastCompleted === void 0) runStatus = "running";
|
|
1628
|
-
else runStatus = lastCompleted.status;
|
|
1629
|
-
const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
1630
|
-
const steps = /* @__PURE__ */ new Map();
|
|
1631
|
-
let order = 0;
|
|
1632
|
-
for (const e of events) {
|
|
1633
|
-
if (e.event !== "step_started") continue;
|
|
1634
|
-
const s = e;
|
|
1635
|
-
const meta = s.metadata ? { ...s.metadata } : void 0;
|
|
1636
|
-
steps.set(s.stepId, {
|
|
1637
|
-
id: s.stepId,
|
|
1638
|
-
parentId: s.parentId,
|
|
1639
|
-
name: s.name,
|
|
1640
|
-
type: s.type,
|
|
1641
|
-
order: order++,
|
|
1642
|
-
timestamp: s.timestamp,
|
|
1643
|
-
metadata: meta
|
|
1644
|
-
});
|
|
1645
|
-
}
|
|
1646
|
-
for (const e of events) {
|
|
1647
|
-
if (e.event !== "step_completed") continue;
|
|
1648
|
-
const acc = steps.get(e.stepId);
|
|
1649
|
-
if (!acc) continue;
|
|
1650
|
-
acc.status = e.status;
|
|
1651
|
-
acc.durationMs = e.durationMs;
|
|
1652
|
-
if (e.error?.message) acc.errorMsg = e.error.message;
|
|
1653
|
-
const extra = e;
|
|
1654
|
-
if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
|
|
1655
|
-
acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
|
|
1656
|
-
}
|
|
1657
|
-
}
|
|
1658
|
-
const nodes = /* @__PURE__ */ new Map();
|
|
1659
|
-
for (const acc of steps.values()) {
|
|
1660
|
-
let meta = acc.metadata ? { ...acc.metadata } : void 0;
|
|
1661
|
-
if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
|
|
1662
|
-
meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
|
|
1663
|
-
}
|
|
1664
|
-
const outputPreview = extractOutputPreview(meta);
|
|
1665
|
-
if (meta !== void 0 && ("outputPreview" in meta || "resultPreview" in meta)) {
|
|
1666
|
-
delete meta.outputPreview;
|
|
1667
|
-
delete meta.resultPreview;
|
|
1668
|
-
}
|
|
1669
|
-
const sc = {
|
|
1670
|
-
id: acc.id,
|
|
1671
|
-
name: acc.name,
|
|
1672
|
-
type: acc.type,
|
|
1673
|
-
status: mapStepStatus(acc.status),
|
|
1674
|
-
durationMs: acc.durationMs,
|
|
1675
|
-
error: acc.errorMsg,
|
|
1676
|
-
metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
|
|
1677
|
-
outputPreview,
|
|
1678
|
-
children: []
|
|
1679
|
-
};
|
|
1680
|
-
nodes.set(acc.id, sc);
|
|
1681
|
-
}
|
|
1682
|
-
const roots = [];
|
|
1683
|
-
const sortByOrder = (a, b) => {
|
|
1684
|
-
const oa = steps.get(a.id)?.order ?? 0;
|
|
1685
|
-
const ob = steps.get(b.id)?.order ?? 0;
|
|
1686
|
-
return oa - ob;
|
|
1689
|
+
function mergeAttributes(start, complete) {
|
|
1690
|
+
const merged = {
|
|
1691
|
+
...isRecord5(complete.attributes) ? complete.attributes : {},
|
|
1692
|
+
...isRecord5(start.attributes) ? start.attributes : {}
|
|
1687
1693
|
};
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
} else {
|
|
1693
|
-
roots.push(node);
|
|
1694
|
-
}
|
|
1695
|
-
}
|
|
1696
|
-
roots.sort(sortByOrder);
|
|
1697
|
-
for (const n of nodes.values()) {
|
|
1698
|
-
n.children.sort(sortByOrder);
|
|
1694
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
1695
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
1696
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
1697
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
1699
1698
|
}
|
|
1699
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
1700
|
+
}
|
|
1701
|
+
function pairStartComplete(start, complete) {
|
|
1702
|
+
const attributes = mergeAttributes(start, complete);
|
|
1703
|
+
const paired = {
|
|
1704
|
+
...cloneEvent(start),
|
|
1705
|
+
status: complete.status,
|
|
1706
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
1707
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
1708
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
1709
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
1710
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
1711
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
1712
|
+
};
|
|
1700
1713
|
return {
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1714
|
+
...paired,
|
|
1715
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
1716
|
+
projection: {
|
|
1717
|
+
paired: true,
|
|
1718
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
1719
|
+
parentNormalized: false,
|
|
1720
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
1721
|
+
}
|
|
1706
1722
|
};
|
|
1707
1723
|
}
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
}
|
|
1717
|
-
function pairSteps(left, right) {
|
|
1718
|
-
const usedRight = /* @__PURE__ */ new Set();
|
|
1719
|
-
const pairs = [];
|
|
1720
|
-
for (let i = 0; i < left.length; i++) {
|
|
1721
|
-
const L = left[i];
|
|
1722
|
-
let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
|
|
1723
|
-
if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
|
|
1724
|
-
const cand = right[i];
|
|
1725
|
-
if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
|
|
1726
|
-
R = cand;
|
|
1727
|
-
}
|
|
1724
|
+
function asLogical(event, extras) {
|
|
1725
|
+
return {
|
|
1726
|
+
...cloneEvent(event),
|
|
1727
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
1728
|
+
projection: {
|
|
1729
|
+
paired: false,
|
|
1730
|
+
absorbedEventIds: Object.freeze([]),
|
|
1731
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
1732
|
+
...{}
|
|
1728
1733
|
}
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1734
|
+
};
|
|
1735
|
+
}
|
|
1736
|
+
function projectLogicalEvents(events) {
|
|
1737
|
+
const diagnostics = [];
|
|
1738
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
1739
|
+
for (const event of events) {
|
|
1740
|
+
const list = byRun.get(event.runId) ?? [];
|
|
1741
|
+
list.push(event);
|
|
1742
|
+
byRun.set(event.runId, list);
|
|
1743
|
+
}
|
|
1744
|
+
const absorbedIds = /* @__PURE__ */ new Set();
|
|
1745
|
+
const logicalByRawId = /* @__PURE__ */ new Map();
|
|
1746
|
+
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
1747
|
+
const logical = [];
|
|
1748
|
+
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
1749
|
+
for (const runId of orderedRuns) {
|
|
1750
|
+
const runEvents = byRun.get(runId) ?? [];
|
|
1751
|
+
const starts = [];
|
|
1752
|
+
const completes = [];
|
|
1753
|
+
const others = [];
|
|
1754
|
+
for (const event of runEvents) {
|
|
1755
|
+
const legacy = legacyEvent(event);
|
|
1756
|
+
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
1757
|
+
starts.push(event);
|
|
1758
|
+
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
1759
|
+
completes.push(event);
|
|
1760
|
+
} else {
|
|
1761
|
+
others.push(event);
|
|
1762
|
+
}
|
|
1733
1763
|
}
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1764
|
+
const usedCompletes = /* @__PURE__ */ new Set();
|
|
1765
|
+
for (const start of starts) {
|
|
1766
|
+
const stepId = stepIdOf(start);
|
|
1767
|
+
let match;
|
|
1768
|
+
if (legacyEvent(start) === "run_started") {
|
|
1769
|
+
const candidates = completes.filter(
|
|
1770
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
1771
|
+
);
|
|
1772
|
+
if (candidates.length > 1) {
|
|
1773
|
+
diagnostics.push({
|
|
1774
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1775
|
+
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
1776
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1777
|
+
});
|
|
1778
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1779
|
+
}
|
|
1780
|
+
match = candidates[0];
|
|
1781
|
+
} else if (stepId) {
|
|
1782
|
+
const candidates = completes.filter(
|
|
1783
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
1784
|
+
);
|
|
1785
|
+
if (candidates.length > 1) {
|
|
1786
|
+
diagnostics.push({
|
|
1787
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1788
|
+
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
1789
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1790
|
+
});
|
|
1791
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1792
|
+
}
|
|
1793
|
+
match = candidates[0];
|
|
1794
|
+
}
|
|
1795
|
+
if (match) {
|
|
1796
|
+
usedCompletes.add(match.eventId);
|
|
1797
|
+
absorbedIds.add(match.eventId);
|
|
1798
|
+
const paired = pairStartComplete(start, match);
|
|
1799
|
+
logical.push(paired);
|
|
1800
|
+
logicalByRawId.set(start.eventId, paired);
|
|
1801
|
+
logicalByRawId.set(match.eventId, paired);
|
|
1802
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
1803
|
+
} else {
|
|
1804
|
+
diagnostics.push({
|
|
1805
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
1806
|
+
message: `No matching complete for start ${start.eventId}.`,
|
|
1807
|
+
eventIds: [start.eventId]
|
|
1808
|
+
});
|
|
1809
|
+
const alone = asLogical(start);
|
|
1810
|
+
logical.push(alone);
|
|
1811
|
+
logicalByRawId.set(start.eventId, alone);
|
|
1812
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1813
|
+
}
|
|
1739
1814
|
}
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1815
|
+
for (const complete of completes) {
|
|
1816
|
+
if (usedCompletes.has(complete.eventId)) continue;
|
|
1817
|
+
diagnostics.push({
|
|
1818
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
1819
|
+
message: `No matching start for complete ${complete.eventId}.`,
|
|
1820
|
+
eventIds: [complete.eventId]
|
|
1821
|
+
});
|
|
1822
|
+
const alone = asLogical(complete);
|
|
1823
|
+
logical.push(alone);
|
|
1824
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
1825
|
+
const stepId = stepIdOf(complete);
|
|
1826
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1827
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1828
|
+
}
|
|
1829
|
+
}
|
|
1830
|
+
for (const event of others) {
|
|
1831
|
+
const alone = asLogical(event);
|
|
1832
|
+
logical.push(alone);
|
|
1833
|
+
logicalByRawId.set(event.eventId, alone);
|
|
1834
|
+
const stepId = stepIdOf(event);
|
|
1835
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1836
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1837
|
+
}
|
|
1744
1838
|
}
|
|
1745
1839
|
}
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1840
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
1841
|
+
const normalized = [];
|
|
1842
|
+
for (const event of logical) {
|
|
1843
|
+
const originalParentId = event.parentId;
|
|
1844
|
+
if (!originalParentId) {
|
|
1845
|
+
normalized.push(event);
|
|
1846
|
+
continue;
|
|
1847
|
+
}
|
|
1848
|
+
let nextParent = originalParentId;
|
|
1849
|
+
let remapped = false;
|
|
1850
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
1851
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
1852
|
+
nextParent = viaAbsorbed.eventId;
|
|
1853
|
+
remapped = true;
|
|
1854
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
1855
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
1856
|
+
if (viaStep) {
|
|
1857
|
+
nextParent = viaStep;
|
|
1858
|
+
remapped = true;
|
|
1859
|
+
}
|
|
1860
|
+
}
|
|
1861
|
+
if (!remapped) {
|
|
1862
|
+
if (originalParentId === event.eventId) {
|
|
1863
|
+
diagnostics.push({
|
|
1864
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1865
|
+
message: formatProgrammaticDiagnostic(
|
|
1866
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1867
|
+
`Removed self-parent edge on ${event.eventId}.`
|
|
1868
|
+
),
|
|
1869
|
+
eventIds: [event.eventId]
|
|
1870
|
+
});
|
|
1871
|
+
const { parentId: _drop, ...rest } = event;
|
|
1872
|
+
normalized.push({
|
|
1873
|
+
...rest,
|
|
1874
|
+
projection: {
|
|
1875
|
+
...event.projection,
|
|
1876
|
+
parentNormalized: true,
|
|
1877
|
+
originalParentId
|
|
1878
|
+
}
|
|
1879
|
+
});
|
|
1880
|
+
continue;
|
|
1881
|
+
}
|
|
1882
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
1883
|
+
const mapping = event.attributes?.parentMapping;
|
|
1884
|
+
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
1885
|
+
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
1886
|
+
if (!unresolved) {
|
|
1887
|
+
diagnostics.push({
|
|
1888
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
1889
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
1890
|
+
eventIds: [event.eventId]
|
|
1891
|
+
});
|
|
1892
|
+
}
|
|
1893
|
+
}
|
|
1894
|
+
normalized.push(event);
|
|
1895
|
+
continue;
|
|
1896
|
+
}
|
|
1897
|
+
if (nextParent === event.eventId) {
|
|
1898
|
+
diagnostics.push({
|
|
1899
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1900
|
+
message: formatProgrammaticDiagnostic(
|
|
1901
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1902
|
+
`Removed self-parent edge on ${event.eventId} (was ${originalParentId}).`
|
|
1903
|
+
),
|
|
1904
|
+
eventIds: [event.eventId]
|
|
1905
|
+
});
|
|
1906
|
+
const { parentId: _drop, ...rest } = event;
|
|
1907
|
+
normalized.push({
|
|
1908
|
+
...rest,
|
|
1909
|
+
projection: {
|
|
1910
|
+
...event.projection,
|
|
1911
|
+
parentNormalized: true,
|
|
1912
|
+
originalParentId
|
|
1913
|
+
}
|
|
1914
|
+
});
|
|
1915
|
+
continue;
|
|
1916
|
+
}
|
|
1917
|
+
diagnostics.push({
|
|
1918
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
1919
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
1920
|
+
eventIds: [event.eventId]
|
|
1758
1921
|
});
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
right: R.type
|
|
1922
|
+
normalized.push({
|
|
1923
|
+
...event,
|
|
1924
|
+
parentId: nextParent,
|
|
1925
|
+
projection: {
|
|
1926
|
+
...event.projection,
|
|
1927
|
+
parentNormalized: true,
|
|
1928
|
+
originalParentId
|
|
1929
|
+
}
|
|
1768
1930
|
});
|
|
1769
1931
|
}
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1932
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
1933
|
+
normalized.sort((a, b) => {
|
|
1934
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
1935
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
1936
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
1937
|
+
});
|
|
1938
|
+
return {
|
|
1939
|
+
logicalEvents: Object.freeze(normalized),
|
|
1940
|
+
diagnostics: Object.freeze(diagnostics)
|
|
1941
|
+
};
|
|
1942
|
+
}
|
|
1943
|
+
function resolveCanonicalToolName(event) {
|
|
1944
|
+
const attrs = event.attributes;
|
|
1945
|
+
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
1946
|
+
if (direct) return direct;
|
|
1947
|
+
const metadata = attrs?.metadata;
|
|
1948
|
+
if (isRecord5(metadata)) {
|
|
1949
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
1950
|
+
if (nested) return nested;
|
|
1779
1951
|
}
|
|
1780
|
-
const
|
|
1781
|
-
|
|
1782
|
-
if (le !== re) {
|
|
1783
|
-
out.push({
|
|
1784
|
-
kind: "error",
|
|
1785
|
-
severity: "error",
|
|
1786
|
-
message: "Step error message differs",
|
|
1787
|
-
path: path18,
|
|
1788
|
-
left: le || void 0,
|
|
1789
|
-
right: re || void 0
|
|
1790
|
-
});
|
|
1791
|
-
}
|
|
1792
|
-
if (!opts.ignoreDuration) {
|
|
1793
|
-
const ld = L.durationMs;
|
|
1794
|
-
const rd = R.durationMs;
|
|
1795
|
-
const th = opts.durationThresholdMs;
|
|
1796
|
-
let differs = false;
|
|
1797
|
-
if (ld === void 0 && rd === void 0) differs = false;
|
|
1798
|
-
else if (ld === void 0 || rd === void 0) differs = true;
|
|
1799
|
-
else differs = Math.abs(ld - rd) > th;
|
|
1800
|
-
if (differs) {
|
|
1801
|
-
out.push({
|
|
1802
|
-
kind: "duration",
|
|
1803
|
-
severity: "info",
|
|
1804
|
-
message: "Step duration differs",
|
|
1805
|
-
path: path18,
|
|
1806
|
-
left: ld,
|
|
1807
|
-
right: rd
|
|
1808
|
-
});
|
|
1809
|
-
}
|
|
1810
|
-
}
|
|
1811
|
-
const lm = stableJson(L.metadata ?? {});
|
|
1812
|
-
const rm2 = stableJson(R.metadata ?? {});
|
|
1813
|
-
if (lm !== rm2) {
|
|
1814
|
-
out.push({
|
|
1815
|
-
kind: "metadata",
|
|
1816
|
-
severity: "info",
|
|
1817
|
-
message: "Step metadata differs",
|
|
1818
|
-
path: path18,
|
|
1819
|
-
left: L.metadata,
|
|
1820
|
-
right: R.metadata
|
|
1821
|
-
});
|
|
1822
|
-
}
|
|
1823
|
-
const lo = stableJson(L.outputPreview ?? null);
|
|
1824
|
-
const ro = stableJson(R.outputPreview ?? null);
|
|
1825
|
-
if (lo !== ro) {
|
|
1826
|
-
out.push({
|
|
1827
|
-
kind: "output",
|
|
1828
|
-
severity: "info",
|
|
1829
|
-
message: "Output preview differs",
|
|
1830
|
-
path: path18,
|
|
1831
|
-
left: L.outputPreview,
|
|
1832
|
-
right: R.outputPreview
|
|
1833
|
-
});
|
|
1952
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
1953
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
1834
1954
|
}
|
|
1955
|
+
return event.name;
|
|
1835
1956
|
}
|
|
1836
|
-
function
|
|
1837
|
-
|
|
1838
|
-
const
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
if (lch !== void 0 && rch !== void 0) {
|
|
1842
|
-
compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
|
|
1843
|
-
} else if (lch !== void 0) {
|
|
1844
|
-
out.push({
|
|
1845
|
-
kind: "step-removed",
|
|
1846
|
-
severity: "warning",
|
|
1847
|
-
message: `Step only in left run: ${lch.name}`,
|
|
1848
|
-
path: buildPath([...segments, pathSeg(lch, ci)]),
|
|
1849
|
-
left: lch.id,
|
|
1850
|
-
right: void 0
|
|
1851
|
-
});
|
|
1852
|
-
} else if (rch !== void 0) {
|
|
1853
|
-
out.push({
|
|
1854
|
-
kind: "step-added",
|
|
1855
|
-
severity: "warning",
|
|
1856
|
-
message: `Step only in right run: ${rch.name}`,
|
|
1857
|
-
path: buildPath([...segments, pathSeg(rch, ci)]),
|
|
1858
|
-
left: void 0,
|
|
1859
|
-
right: rch.id
|
|
1860
|
-
});
|
|
1861
|
-
}
|
|
1862
|
-
ci += 1;
|
|
1957
|
+
function pickString(record, keys) {
|
|
1958
|
+
if (!record) return void 0;
|
|
1959
|
+
for (const key of keys) {
|
|
1960
|
+
const value = record[key];
|
|
1961
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
1863
1962
|
}
|
|
1963
|
+
return void 0;
|
|
1864
1964
|
}
|
|
1865
|
-
|
|
1965
|
+
|
|
1966
|
+
// packages/core/src/checks/trace-facts.ts
|
|
1967
|
+
formatProgrammaticDiagnostic(
|
|
1968
|
+
"AI_TRACE_FACTS_INPUT_NOT_NORMALIZED"
|
|
1969
|
+
);
|
|
1970
|
+
|
|
1971
|
+
// packages/core/src/checks/index.ts
|
|
1972
|
+
var SEVERITY_RANK = {
|
|
1973
|
+
error: 0,
|
|
1974
|
+
warning: 1,
|
|
1975
|
+
info: 2
|
|
1976
|
+
};
|
|
1977
|
+
var STATUS_RANK = {
|
|
1978
|
+
fail: 0,
|
|
1979
|
+
warning: 1,
|
|
1980
|
+
pass: 2
|
|
1981
|
+
};
|
|
1982
|
+
function compareStrings(a, b) {
|
|
1983
|
+
return (a ?? "").localeCompare(b ?? "");
|
|
1984
|
+
}
|
|
1985
|
+
function diagnostic(code, message, ruleId) {
|
|
1866
1986
|
return {
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1987
|
+
code,
|
|
1988
|
+
message,
|
|
1989
|
+
severity: "error",
|
|
1990
|
+
...ruleId ? { ruleId } : {}
|
|
1871
1991
|
};
|
|
1872
1992
|
}
|
|
1873
|
-
function
|
|
1874
|
-
return
|
|
1993
|
+
function emptySummary() {
|
|
1994
|
+
return {
|
|
1995
|
+
passed: 0,
|
|
1996
|
+
failed: 0,
|
|
1997
|
+
warnings: 0,
|
|
1998
|
+
errors: 0,
|
|
1999
|
+
rulesEvaluated: 0
|
|
2000
|
+
};
|
|
1875
2001
|
}
|
|
1876
|
-
function
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
2002
|
+
function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
2003
|
+
return {
|
|
2004
|
+
ok: false,
|
|
2005
|
+
status: "error",
|
|
2006
|
+
format: input.read.format,
|
|
2007
|
+
...selectedRun ? { runId: selectedRun.runId } : {},
|
|
2008
|
+
summary: {
|
|
2009
|
+
...emptySummary(),
|
|
2010
|
+
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2011
|
+
rulesEvaluated: ruleExecutions.length
|
|
2012
|
+
},
|
|
2013
|
+
findings: [],
|
|
2014
|
+
diagnostics: [...diagnostics],
|
|
2015
|
+
ruleExecutions: [...ruleExecutions]
|
|
1881
2016
|
};
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
2017
|
+
}
|
|
2018
|
+
function flattenNodes(nodes) {
|
|
2019
|
+
return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
|
|
2020
|
+
}
|
|
2021
|
+
function buildFacts(input, selectedRun) {
|
|
2022
|
+
const scopedRuns = selectedRun ? [selectedRun] : input.read.runs;
|
|
2023
|
+
const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
|
|
2024
|
+
const scopedEvents = selectedRun === void 0 ? input.read.events : input.read.events.filter((event) => scopedRunIds.has(event.runId));
|
|
2025
|
+
const nodes = flattenNodes(scopedRuns.flatMap((run) => run.children));
|
|
2026
|
+
const nodesByEventId = /* @__PURE__ */ new Map();
|
|
2027
|
+
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
2028
|
+
for (const node of nodes) {
|
|
2029
|
+
nodesByEventId.set(node.event.eventId, node);
|
|
2030
|
+
const parentId = node.event.parentId;
|
|
2031
|
+
if (parentId) {
|
|
2032
|
+
const children = childrenByParentId.get(parentId) ?? [];
|
|
2033
|
+
children.push(node);
|
|
2034
|
+
childrenByParentId.set(parentId, children);
|
|
2035
|
+
}
|
|
1891
2036
|
}
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
2037
|
+
const projection = projectLogicalEvents(scopedEvents);
|
|
2038
|
+
return {
|
|
2039
|
+
format: input.read.format,
|
|
2040
|
+
runs: Object.freeze([...input.read.runs]),
|
|
2041
|
+
events: Object.freeze([...scopedEvents]),
|
|
2042
|
+
logicalEvents: projection.logicalEvents,
|
|
2043
|
+
logicalProjectionDiagnostics: projection.diagnostics,
|
|
2044
|
+
readerWarnings: Object.freeze([...input.read.warnings]),
|
|
2045
|
+
unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
|
|
2046
|
+
sourceFiles: Object.freeze([...input.read.sourceFiles]),
|
|
2047
|
+
nodesByEventId,
|
|
2048
|
+
childrenByParentId,
|
|
2049
|
+
rootNodes: Object.freeze(scopedRuns.flatMap((run) => run.children))
|
|
2050
|
+
};
|
|
2051
|
+
}
|
|
2052
|
+
function resolveSelectedRun(input, runId) {
|
|
2053
|
+
if (input.selectedRun) {
|
|
2054
|
+
if (runId && input.selectedRun.runId !== runId) {
|
|
2055
|
+
return {
|
|
2056
|
+
diagnostics: [
|
|
2057
|
+
diagnostic(
|
|
2058
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
2059
|
+
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
2060
|
+
)
|
|
2061
|
+
]
|
|
2062
|
+
};
|
|
1908
2063
|
}
|
|
2064
|
+
return { run: input.selectedRun, diagnostics: [] };
|
|
1909
2065
|
}
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
kind: "step-removed",
|
|
1919
|
-
severity: "warning",
|
|
1920
|
-
message: `Step only in left run: ${ls.name}`,
|
|
1921
|
-
path: buildPath([pathSeg(ls, idx)]),
|
|
1922
|
-
left: ls.id,
|
|
1923
|
-
right: void 0
|
|
1924
|
-
});
|
|
1925
|
-
idx += 1;
|
|
1926
|
-
} else if (rs !== void 0) {
|
|
1927
|
-
raw.push({
|
|
1928
|
-
kind: "step-added",
|
|
1929
|
-
severity: "warning",
|
|
1930
|
-
message: `Step only in right run: ${rs.name}`,
|
|
1931
|
-
path: buildPath([pathSeg(rs, idx)]),
|
|
1932
|
-
left: void 0,
|
|
1933
|
-
right: rs.id
|
|
1934
|
-
});
|
|
1935
|
-
idx += 1;
|
|
2066
|
+
if (runId) {
|
|
2067
|
+
const run = input.read.runs.find((candidate) => candidate.runId === runId);
|
|
2068
|
+
if (!run) {
|
|
2069
|
+
return {
|
|
2070
|
+
diagnostics: [
|
|
2071
|
+
diagnostic("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
2072
|
+
]
|
|
2073
|
+
};
|
|
1936
2074
|
}
|
|
2075
|
+
return { run, diagnostics: [] };
|
|
1937
2076
|
}
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
let warnings = 0;
|
|
1941
|
-
let info = 0;
|
|
1942
|
-
for (const d of differences) {
|
|
1943
|
-
if (d.severity === "error") errors += 1;
|
|
1944
|
-
else if (d.severity === "warning") warnings += 1;
|
|
1945
|
-
else info += 1;
|
|
1946
|
-
}
|
|
1947
|
-
const firstVisible = differences[0];
|
|
1948
|
-
const firstDivergence = firstVisible !== void 0 ? {
|
|
1949
|
-
kind: "first-divergence",
|
|
1950
|
-
severity: firstVisible.severity,
|
|
1951
|
-
message: `First divergence: ${firstVisible.message}`,
|
|
1952
|
-
path: firstVisible.path,
|
|
1953
|
-
left: firstVisible.left,
|
|
1954
|
-
right: firstVisible.right
|
|
1955
|
-
} : void 0;
|
|
1956
|
-
const summary = {
|
|
1957
|
-
leftRunId: left.runId,
|
|
1958
|
-
rightRunId: right.runId,
|
|
1959
|
-
totalDifferences: differences.length,
|
|
1960
|
-
errors,
|
|
1961
|
-
warnings,
|
|
1962
|
-
info,
|
|
1963
|
-
firstDivergence
|
|
1964
|
-
};
|
|
1965
|
-
return { summary, differences };
|
|
1966
|
-
}
|
|
1967
|
-
|
|
1968
|
-
// packages/core/src/evidence/zip.ts
|
|
1969
|
-
(() => {
|
|
1970
|
-
const table = new Uint32Array(256);
|
|
1971
|
-
for (let n = 0; n < 256; n += 1) {
|
|
1972
|
-
let c = n;
|
|
1973
|
-
for (let k = 0; k < 8; k += 1) {
|
|
1974
|
-
c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
|
|
1975
|
-
}
|
|
1976
|
-
table[n] = c >>> 0;
|
|
1977
|
-
}
|
|
1978
|
-
return table;
|
|
1979
|
-
})();
|
|
1980
|
-
|
|
1981
|
-
// packages/core/src/suite/types.ts
|
|
1982
|
-
var DEFAULT_SUITE_CONFIG_NAMES = [
|
|
1983
|
-
"agent-inspect.suite.json",
|
|
1984
|
-
"agent-inspect.suite.js",
|
|
1985
|
-
"agent-inspect.suite.mjs",
|
|
1986
|
-
"agent-inspect.suite.cjs"
|
|
1987
|
-
];
|
|
1988
|
-
function diagnostic(code, message, severity = "error", caseId) {
|
|
1989
|
-
return { code, message, severity, ...{} };
|
|
1990
|
-
}
|
|
1991
|
-
function asString(value, label) {
|
|
1992
|
-
if (value === void 0) return void 0;
|
|
1993
|
-
if (typeof value !== "string" || value.trim() === "") {
|
|
1994
|
-
throw new Error(`${label} must be a non-empty string.`);
|
|
1995
|
-
}
|
|
1996
|
-
return value.trim();
|
|
1997
|
-
}
|
|
1998
|
-
function asStringArray(value, label) {
|
|
1999
|
-
if (value === void 0) return void 0;
|
|
2000
|
-
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
2001
|
-
throw new Error(`${label} must be an array of strings.`);
|
|
2077
|
+
if (input.read.runs.length === 1) {
|
|
2078
|
+
return { run: input.read.runs[0], diagnostics: [] };
|
|
2002
2079
|
}
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2080
|
+
if (input.read.runs.length === 0) {
|
|
2081
|
+
return {
|
|
2082
|
+
diagnostics: [
|
|
2083
|
+
diagnostic(
|
|
2084
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
2085
|
+
formatProgrammaticDiagnostic(
|
|
2086
|
+
"AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
2087
|
+
"No runs are available for checks."
|
|
2088
|
+
)
|
|
2089
|
+
)
|
|
2090
|
+
]
|
|
2091
|
+
};
|
|
2009
2092
|
}
|
|
2010
|
-
return
|
|
2093
|
+
return {
|
|
2094
|
+
diagnostics: [
|
|
2095
|
+
diagnostic(
|
|
2096
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
2097
|
+
formatProgrammaticDiagnostic("AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED")
|
|
2098
|
+
)
|
|
2099
|
+
]
|
|
2100
|
+
};
|
|
2011
2101
|
}
|
|
2012
|
-
function
|
|
2102
|
+
function selectRules(rules, selectedIds) {
|
|
2013
2103
|
const diagnostics = [];
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
);
|
|
2018
|
-
return { diagnostics };
|
|
2019
|
-
}
|
|
2020
|
-
const raw = value;
|
|
2021
|
-
try {
|
|
2022
|
-
const id = asString(raw.id, `cases[${index}].id`);
|
|
2023
|
-
if (id === void 0) {
|
|
2104
|
+
const byId = /* @__PURE__ */ new Map();
|
|
2105
|
+
for (const rule of rules) {
|
|
2106
|
+
if (byId.has(rule.id)) {
|
|
2024
2107
|
diagnostics.push(
|
|
2025
|
-
diagnostic("
|
|
2108
|
+
diagnostic("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
2026
2109
|
);
|
|
2027
|
-
|
|
2110
|
+
continue;
|
|
2111
|
+
}
|
|
2112
|
+
byId.set(rule.id, rule);
|
|
2113
|
+
}
|
|
2114
|
+
if (selectedIds && selectedIds.length > 0) {
|
|
2115
|
+
const selected = new Set(selectedIds);
|
|
2116
|
+
for (const id of selected) {
|
|
2117
|
+
if (!byId.has(id)) {
|
|
2118
|
+
diagnostics.push(
|
|
2119
|
+
diagnostic("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
2120
|
+
);
|
|
2121
|
+
}
|
|
2028
2122
|
}
|
|
2029
|
-
const trace = asString(raw.trace, `cases[${index}].trace`);
|
|
2030
|
-
const runId = asString(raw.runId, `cases[${index}].runId`);
|
|
2031
|
-
const input = asString(raw.input, `cases[${index}].input`);
|
|
2032
2123
|
return {
|
|
2033
|
-
|
|
2034
|
-
id,
|
|
2035
|
-
...trace !== void 0 ? { trace } : {},
|
|
2036
|
-
...runId !== void 0 ? { runId } : {},
|
|
2037
|
-
...input !== void 0 ? { input } : {},
|
|
2038
|
-
...asStringArray(raw.requireTools, `cases[${index}].requireTools`) !== void 0 ? { requireTools: asStringArray(raw.requireTools, `cases[${index}].requireTools`) } : {},
|
|
2039
|
-
...asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) !== void 0 ? { forbidTools: asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) } : {},
|
|
2040
|
-
...asPositiveNumber(raw.maxDurationMs, `cases[${index}].maxDurationMs`) !== void 0 ? {
|
|
2041
|
-
maxDurationMs: asPositiveNumber(
|
|
2042
|
-
raw.maxDurationMs,
|
|
2043
|
-
`cases[${index}].maxDurationMs`
|
|
2044
|
-
)
|
|
2045
|
-
} : {},
|
|
2046
|
-
...asStringArray(
|
|
2047
|
-
raw.expectedObservations,
|
|
2048
|
-
`cases[${index}].expectedObservations`
|
|
2049
|
-
) !== void 0 ? {
|
|
2050
|
-
expectedObservations: asStringArray(
|
|
2051
|
-
raw.expectedObservations,
|
|
2052
|
-
`cases[${index}].expectedObservations`
|
|
2053
|
-
)
|
|
2054
|
-
} : {}
|
|
2055
|
-
},
|
|
2124
|
+
rules: [...byId.values()].filter((rule) => selected.has(rule.id)).sort(compareRules),
|
|
2056
2125
|
diagnostics
|
|
2057
2126
|
};
|
|
2058
|
-
} catch (error) {
|
|
2059
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
2060
|
-
diagnostics.push(diagnostic("AI_SUITE_CONFIG_INVALID", message));
|
|
2061
|
-
return { diagnostics };
|
|
2062
2127
|
}
|
|
2128
|
+
return { rules: [...byId.values()].sort(compareRules), diagnostics };
|
|
2063
2129
|
}
|
|
2064
|
-
function
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
const
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
}
|
|
2076
|
-
const cases = [];
|
|
2077
|
-
for (let index = 0; index < raw.cases.length; index += 1) {
|
|
2078
|
-
const { caseConfig, diagnostics } = validateCaseConfig(raw.cases[index], index);
|
|
2079
|
-
if (diagnostics.length > 0) {
|
|
2080
|
-
throw new Error(diagnostics.map((item) => item.message).join("; "));
|
|
2130
|
+
function compareRules(a, b) {
|
|
2131
|
+
return a.id.localeCompare(b.id);
|
|
2132
|
+
}
|
|
2133
|
+
function eventTimestamp(finding, eventById) {
|
|
2134
|
+
const eventId = finding.evidence[0]?.eventId;
|
|
2135
|
+
return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
|
|
2136
|
+
}
|
|
2137
|
+
function compareFindings(eventById) {
|
|
2138
|
+
return (a, b) => {
|
|
2139
|
+
if (SEVERITY_RANK[a.severity] !== SEVERITY_RANK[b.severity]) {
|
|
2140
|
+
return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity];
|
|
2081
2141
|
}
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
if (ids.has(suiteCase.id)) {
|
|
2087
|
-
throw new Error(`Duplicate case id "${suiteCase.id}".`);
|
|
2142
|
+
const byRule = a.ruleId.localeCompare(b.ruleId);
|
|
2143
|
+
if (byRule !== 0) return byRule;
|
|
2144
|
+
if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
|
|
2145
|
+
return STATUS_RANK[a.status] - STATUS_RANK[b.status];
|
|
2088
2146
|
}
|
|
2089
|
-
|
|
2147
|
+
const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
|
|
2148
|
+
if (byRun !== 0) return byRun;
|
|
2149
|
+
const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
|
|
2150
|
+
if (byTime !== 0) return byTime;
|
|
2151
|
+
const byEvent = compareStrings(a.evidence[0]?.eventId, b.evidence[0]?.eventId);
|
|
2152
|
+
if (byEvent !== 0) return byEvent;
|
|
2153
|
+
return compareStrings(a.evidence[0]?.path, b.evidence[0]?.path);
|
|
2154
|
+
};
|
|
2155
|
+
}
|
|
2156
|
+
function normalizeFinding(rule, finding) {
|
|
2157
|
+
return {
|
|
2158
|
+
ruleId: finding.ruleId || rule.id,
|
|
2159
|
+
severity: finding.severity ?? rule.defaultSeverity,
|
|
2160
|
+
status: finding.status,
|
|
2161
|
+
message: finding.message,
|
|
2162
|
+
...finding.expected !== void 0 ? { expected: finding.expected } : {},
|
|
2163
|
+
...finding.actual !== void 0 ? { actual: finding.actual } : {},
|
|
2164
|
+
evidence: [...finding.evidence ?? []],
|
|
2165
|
+
...finding.category !== void 0 ? { category: finding.category } : {},
|
|
2166
|
+
...finding.confidence !== void 0 ? { confidence: finding.confidence } : {},
|
|
2167
|
+
...finding.detector !== void 0 ? { detector: finding.detector } : {},
|
|
2168
|
+
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
2169
|
+
};
|
|
2170
|
+
}
|
|
2171
|
+
function summarize(findings, diagnostics, rulesEvaluated) {
|
|
2172
|
+
return {
|
|
2173
|
+
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
2174
|
+
failed: findings.filter(
|
|
2175
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
2176
|
+
).length,
|
|
2177
|
+
warnings: findings.filter(
|
|
2178
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
2179
|
+
).length,
|
|
2180
|
+
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2181
|
+
rulesEvaluated
|
|
2182
|
+
};
|
|
2183
|
+
}
|
|
2184
|
+
function classifyRuleExecution(findings, threw) {
|
|
2185
|
+
if (findings.some((finding) => finding.status === "fail" && finding.severity === "error")) {
|
|
2186
|
+
return "fail";
|
|
2090
2187
|
}
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2188
|
+
if (findings.some(
|
|
2189
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
2190
|
+
)) {
|
|
2191
|
+
return "warning";
|
|
2094
2192
|
}
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2193
|
+
return "pass";
|
|
2194
|
+
}
|
|
2195
|
+
function stringAttr(event, keys) {
|
|
2196
|
+
for (const key of keys) {
|
|
2197
|
+
const value = event.attributes?.[key];
|
|
2198
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2101
2199
|
}
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2200
|
+
return void 0;
|
|
2201
|
+
}
|
|
2202
|
+
function metadataStringAttr(event, keys) {
|
|
2203
|
+
const metadata = event.attributes?.metadata;
|
|
2204
|
+
if (typeof metadata !== "object" || metadata === null || Array.isArray(metadata)) {
|
|
2205
|
+
return void 0;
|
|
2107
2206
|
}
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
const outputDir = asString(
|
|
2113
|
-
raw.artifacts.outputDir,
|
|
2114
|
-
"artifacts.outputDir"
|
|
2115
|
-
);
|
|
2116
|
-
config.artifacts = outputDir !== void 0 ? { outputDir } : {};
|
|
2117
|
-
}
|
|
2118
|
-
if (raw.baseline !== void 0) {
|
|
2119
|
-
const baseline = asString(raw.baseline, "baseline");
|
|
2120
|
-
if (baseline !== void 0) config.baseline = baseline;
|
|
2121
|
-
}
|
|
2122
|
-
if (raw.candidate !== void 0) {
|
|
2123
|
-
const candidate = asString(raw.candidate, "candidate");
|
|
2124
|
-
if (candidate !== void 0) config.candidate = candidate;
|
|
2125
|
-
}
|
|
2126
|
-
if (redactionProfile !== void 0) config.redactionProfile = redactionProfile;
|
|
2127
|
-
return config;
|
|
2128
|
-
}
|
|
2129
|
-
|
|
2130
|
-
// packages/core/src/suite/load.ts
|
|
2131
|
-
var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
2132
|
-
var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
2133
|
-
function diagnostic2(code, message) {
|
|
2134
|
-
return { code, message, severity: "error" };
|
|
2135
|
-
}
|
|
2136
|
-
async function fileExists(filePath) {
|
|
2137
|
-
try {
|
|
2138
|
-
await access(filePath);
|
|
2139
|
-
return true;
|
|
2140
|
-
} catch {
|
|
2141
|
-
return false;
|
|
2207
|
+
const record = metadata;
|
|
2208
|
+
for (const key of keys) {
|
|
2209
|
+
const value = record[key];
|
|
2210
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2142
2211
|
}
|
|
2212
|
+
return void 0;
|
|
2143
2213
|
}
|
|
2144
|
-
|
|
2145
|
-
const
|
|
2146
|
-
|
|
2147
|
-
return path15.resolve(cwd, options.configPath.trim());
|
|
2148
|
-
}
|
|
2149
|
-
for (const name of DEFAULT_SUITE_CONFIG_NAMES) {
|
|
2150
|
-
const candidate = path15.join(cwd, name);
|
|
2151
|
-
if (await fileExists(candidate)) return candidate;
|
|
2214
|
+
function stripPrefix(name, prefixes) {
|
|
2215
|
+
for (const prefix of prefixes) {
|
|
2216
|
+
if (name.startsWith(prefix)) return name.slice(prefix.length);
|
|
2152
2217
|
}
|
|
2153
|
-
|
|
2154
|
-
`No suite config found. Create one with \`agent-inspect suite init\` or pass --config.`
|
|
2155
|
-
);
|
|
2218
|
+
return name;
|
|
2156
2219
|
}
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
2170
|
-
throw Object.assign(new Error(message), {
|
|
2171
|
-
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
2172
|
-
});
|
|
2173
|
-
}
|
|
2174
|
-
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
2175
|
-
const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
|
|
2176
|
-
throw Object.assign(new Error(message), {
|
|
2177
|
-
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
2178
|
-
});
|
|
2179
|
-
}
|
|
2180
|
-
try {
|
|
2181
|
-
let raw;
|
|
2182
|
-
if (extension === ".json") {
|
|
2183
|
-
raw = JSON.parse(await readFile(configPath, "utf-8"));
|
|
2184
|
-
} else {
|
|
2185
|
-
const mod = await import(pathToFileURL(configPath).href);
|
|
2186
|
-
raw = "default" in mod ? mod.default : mod;
|
|
2187
|
-
}
|
|
2188
|
-
const config = normalizeSuiteConfig(raw);
|
|
2189
|
-
return {
|
|
2190
|
-
config,
|
|
2191
|
-
configPath,
|
|
2192
|
-
configDir: path15.dirname(configPath)
|
|
2193
|
-
};
|
|
2194
|
-
} catch (error) {
|
|
2195
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
2196
|
-
throw Object.assign(new Error(message), {
|
|
2197
|
-
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
2198
|
-
});
|
|
2199
|
-
}
|
|
2220
|
+
function eventEvidence(event, path18) {
|
|
2221
|
+
return {
|
|
2222
|
+
runId: event.runId,
|
|
2223
|
+
eventId: event.eventId,
|
|
2224
|
+
parentId: event.parentId,
|
|
2225
|
+
traceId: event.trace?.traceId,
|
|
2226
|
+
spanId: event.trace?.spanId,
|
|
2227
|
+
kind: event.kind,
|
|
2228
|
+
name: event.name,
|
|
2229
|
+
status: event.status,
|
|
2230
|
+
...path18 ? { path: path18 } : {}
|
|
2231
|
+
};
|
|
2200
2232
|
}
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
await access(filePath);
|
|
2204
|
-
return true;
|
|
2205
|
-
} catch {
|
|
2206
|
-
return false;
|
|
2207
|
-
}
|
|
2233
|
+
function runEvidence(run) {
|
|
2234
|
+
return run ? [{ runId: run.runId, name: run.name, status: run.status }] : [];
|
|
2208
2235
|
}
|
|
2209
|
-
|
|
2210
|
-
if (suiteCase.trace !== void 0) {
|
|
2211
|
-
const tracePath = path15.resolve(options.configDir, suiteCase.trace);
|
|
2212
|
-
if (await exists(tracePath)) {
|
|
2213
|
-
return { caseId: suiteCase.id, tracePath, missing: false };
|
|
2214
|
-
}
|
|
2215
|
-
return {
|
|
2216
|
-
caseId: suiteCase.id,
|
|
2217
|
-
tracePath,
|
|
2218
|
-
missing: true,
|
|
2219
|
-
reason: `trace file not found: ${suiteCase.trace}`
|
|
2220
|
-
};
|
|
2221
|
-
}
|
|
2222
|
-
const runKey = suiteCase.runId ?? suiteCase.id;
|
|
2223
|
-
const directPath = getTraceFilePath(runKey, options.tracesDir);
|
|
2224
|
-
if (await exists(directPath)) {
|
|
2225
|
-
return { caseId: suiteCase.id, tracePath: directPath, runId: runKey, missing: false };
|
|
2226
|
-
}
|
|
2227
|
-
const nestedPath = path15.join(options.tracesDir, `${path15.basename(runKey)}.jsonl`);
|
|
2228
|
-
if (await exists(nestedPath)) {
|
|
2229
|
-
return { caseId: suiteCase.id, tracePath: nestedPath, runId: runKey, missing: false };
|
|
2230
|
-
}
|
|
2236
|
+
function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
2231
2237
|
return {
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2238
|
+
ruleId,
|
|
2239
|
+
severity: meta?.severity ?? "error",
|
|
2240
|
+
status: meta?.status ?? "fail",
|
|
2241
|
+
message,
|
|
2242
|
+
...expected !== void 0 ? { expected } : {},
|
|
2243
|
+
...actual !== void 0 ? { actual } : {},
|
|
2244
|
+
evidence: [...evidence],
|
|
2245
|
+
...{},
|
|
2246
|
+
...{},
|
|
2247
|
+
...{},
|
|
2248
|
+
...{}
|
|
2236
2249
|
};
|
|
2237
2250
|
}
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
2241
|
-
AI_TRACE_INPUT_INVALID: {
|
|
2242
|
-
code: "AI_TRACE_INPUT_INVALID",
|
|
2243
|
-
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
2244
|
-
remediation: "For a file path, use openTraceFile(path).",
|
|
2245
|
-
relatedCodes: ["invalid_input"]
|
|
2246
|
-
},
|
|
2247
|
-
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
2248
|
-
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
2249
|
-
summary: "No trace reader could detect the input format.",
|
|
2250
|
-
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
2251
|
-
relatedCodes: ["unsupported_format"]
|
|
2252
|
-
},
|
|
2253
|
-
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
2254
|
-
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
2255
|
-
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
2256
|
-
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
2257
|
-
relatedCodes: ["ambiguous_format"]
|
|
2258
|
-
},
|
|
2259
|
-
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
2260
|
-
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
2261
|
-
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
2262
|
-
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
2263
|
-
},
|
|
2264
|
-
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
2265
|
-
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
2266
|
-
summary: "Multiple runs are available; select a run before executing checks.",
|
|
2267
|
-
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
2268
|
-
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
2269
|
-
},
|
|
2270
|
-
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
2271
|
-
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2272
|
-
summary: "A parentId equals its own event/step id.",
|
|
2273
|
-
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
2274
|
-
relatedCodes: [
|
|
2275
|
-
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
2276
|
-
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
2277
|
-
]
|
|
2278
|
-
},
|
|
2279
|
-
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
2280
|
-
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
2281
|
-
summary: "Trace contains a parentId cycle.",
|
|
2282
|
-
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
2283
|
-
relatedCodes: ["structure.cycle"]
|
|
2284
|
-
}
|
|
2285
|
-
});
|
|
2286
|
-
function formatProgrammaticDiagnostic(code, detail) {
|
|
2287
|
-
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
2288
|
-
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
2289
|
-
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
2251
|
+
function toolName(event) {
|
|
2252
|
+
return resolveCanonicalToolName(event);
|
|
2290
2253
|
}
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
function normalizeSensitiveKey(value) {
|
|
2294
|
-
return value.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
2254
|
+
function semanticEvents(context) {
|
|
2255
|
+
return context.logicalEvents ?? context.events;
|
|
2295
2256
|
}
|
|
2296
|
-
|
|
2297
|
-
[
|
|
2298
|
-
|
|
2299
|
-
"max_tokens",
|
|
2300
|
-
"min_tokens",
|
|
2301
|
-
"ls_max_tokens",
|
|
2302
|
-
"token_count",
|
|
2303
|
-
"token_limit",
|
|
2304
|
-
"token_budget",
|
|
2305
|
-
"input_tokens",
|
|
2306
|
-
"output_tokens",
|
|
2307
|
-
"total_tokens",
|
|
2308
|
-
"cached_tokens",
|
|
2309
|
-
"prompt_tokens",
|
|
2310
|
-
"completion_tokens"
|
|
2311
|
-
].map(normalizeSensitiveKey)
|
|
2312
|
-
);
|
|
2313
|
-
|
|
2314
|
-
// packages/core/src/checks/logical-events.ts
|
|
2315
|
-
function isRecord5(value) {
|
|
2316
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2257
|
+
function llmModel(event) {
|
|
2258
|
+
const keys = ["model", "modelId", "responseModelId", "modelName", "model_name"];
|
|
2259
|
+
return stringAttr(event, keys) ?? metadataStringAttr(event, keys) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
2317
2260
|
}
|
|
2318
|
-
function
|
|
2319
|
-
const
|
|
2320
|
-
return
|
|
2261
|
+
function llmProvider(event) {
|
|
2262
|
+
const keys = ["provider", "providerName", "provider_name"];
|
|
2263
|
+
return stringAttr(event, keys) ?? metadataStringAttr(event, keys);
|
|
2321
2264
|
}
|
|
2322
|
-
function
|
|
2323
|
-
const
|
|
2324
|
-
|
|
2325
|
-
return void 0;
|
|
2265
|
+
function llmFinishReason(event) {
|
|
2266
|
+
const keys = ["finishReason", "rawFinishReason", "finish_reason"];
|
|
2267
|
+
return stringAttr(event, keys) ?? metadataStringAttr(event, keys);
|
|
2326
2268
|
}
|
|
2327
|
-
function
|
|
2328
|
-
return
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
2332
|
-
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
2333
|
-
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
2334
|
-
};
|
|
2269
|
+
function finishedEvents(context, kind) {
|
|
2270
|
+
return semanticEvents(context).filter(
|
|
2271
|
+
(event) => (event.kind === kind) && event.status !== "running"
|
|
2272
|
+
);
|
|
2335
2273
|
}
|
|
2336
|
-
function
|
|
2337
|
-
|
|
2338
|
-
...isRecord5(complete.attributes) ? complete.attributes : {},
|
|
2339
|
-
...isRecord5(start.attributes) ? start.attributes : {}
|
|
2340
|
-
};
|
|
2341
|
-
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
2342
|
-
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
2343
|
-
if (complete.attributes?.errorStack !== void 0) {
|
|
2344
|
-
merged.errorStack = complete.attributes.errorStack;
|
|
2345
|
-
}
|
|
2346
|
-
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
2274
|
+
function toolInvocationEvents(context) {
|
|
2275
|
+
return semanticEvents(context).filter((event) => event.kind === "TOOL");
|
|
2347
2276
|
}
|
|
2348
|
-
function
|
|
2349
|
-
const
|
|
2350
|
-
const
|
|
2351
|
-
...cloneEvent(start),
|
|
2352
|
-
status: complete.status,
|
|
2353
|
-
timestamp: complete.timestamp ?? start.timestamp,
|
|
2354
|
-
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
2355
|
-
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
2356
|
-
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
2357
|
-
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
2358
|
-
...attributes !== void 0 ? { attributes } : {}
|
|
2359
|
-
};
|
|
2277
|
+
function createRunStatusRule(options = {}) {
|
|
2278
|
+
const expected = options.expected ?? "ok";
|
|
2279
|
+
const allowIncomplete = options.allowIncomplete === true;
|
|
2360
2280
|
return {
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2281
|
+
id: "run.status",
|
|
2282
|
+
category: "run",
|
|
2283
|
+
defaultSeverity: "error",
|
|
2284
|
+
evaluate(context) {
|
|
2285
|
+
const findings = [];
|
|
2286
|
+
const actual = context.selectedRun?.status ?? "unknown";
|
|
2287
|
+
if (actual !== expected) {
|
|
2288
|
+
findings.push(
|
|
2289
|
+
failFinding(
|
|
2290
|
+
"run.status",
|
|
2291
|
+
`Run status ${actual} did not match expected ${expected}.`,
|
|
2292
|
+
runEvidence(context.selectedRun),
|
|
2293
|
+
expected,
|
|
2294
|
+
actual
|
|
2295
|
+
)
|
|
2296
|
+
);
|
|
2297
|
+
}
|
|
2298
|
+
if (!allowIncomplete) {
|
|
2299
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
2300
|
+
if (running.length > 0) {
|
|
2301
|
+
findings.push(
|
|
2302
|
+
failFinding(
|
|
2303
|
+
"run.status",
|
|
2304
|
+
"Run contains incomplete running events.",
|
|
2305
|
+
running.map((event) => eventEvidence(event)),
|
|
2306
|
+
"no running events",
|
|
2307
|
+
running.length
|
|
2308
|
+
)
|
|
2309
|
+
);
|
|
2310
|
+
}
|
|
2311
|
+
}
|
|
2312
|
+
return findings;
|
|
2368
2313
|
}
|
|
2369
2314
|
};
|
|
2370
2315
|
}
|
|
2371
|
-
function
|
|
2316
|
+
function createRunDurationRule(options) {
|
|
2372
2317
|
return {
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2318
|
+
id: "run.duration",
|
|
2319
|
+
category: "run",
|
|
2320
|
+
defaultSeverity: "error",
|
|
2321
|
+
evaluate(context) {
|
|
2322
|
+
const actual = context.selectedRun?.durationMs;
|
|
2323
|
+
if (actual === void 0 || actual <= options.maxDurationMs) return [];
|
|
2324
|
+
return [
|
|
2325
|
+
failFinding(
|
|
2326
|
+
"run.duration",
|
|
2327
|
+
`Run duration ${actual}ms exceeded ${options.maxDurationMs}ms.`,
|
|
2328
|
+
runEvidence(context.selectedRun),
|
|
2329
|
+
{ maxDurationMs: options.maxDurationMs },
|
|
2330
|
+
actual
|
|
2331
|
+
)
|
|
2332
|
+
];
|
|
2380
2333
|
}
|
|
2381
2334
|
};
|
|
2382
2335
|
}
|
|
2383
|
-
function
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
const completes = [];
|
|
2400
|
-
const others = [];
|
|
2401
|
-
for (const event of runEvents) {
|
|
2402
|
-
const legacy = legacyEvent(event);
|
|
2403
|
-
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
2404
|
-
starts.push(event);
|
|
2405
|
-
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
2406
|
-
completes.push(event);
|
|
2407
|
-
} else {
|
|
2408
|
-
others.push(event);
|
|
2336
|
+
function createToolUsageRule(options) {
|
|
2337
|
+
return {
|
|
2338
|
+
id: "tool.usage",
|
|
2339
|
+
category: "tool",
|
|
2340
|
+
defaultSeverity: "error",
|
|
2341
|
+
evaluate(context) {
|
|
2342
|
+
const tools = toolInvocationEvents(context);
|
|
2343
|
+
const names = tools.map(toolName);
|
|
2344
|
+
const nameSet = new Set(names);
|
|
2345
|
+
const findings = [];
|
|
2346
|
+
for (const required of options.required ?? []) {
|
|
2347
|
+
if (!nameSet.has(required)) {
|
|
2348
|
+
findings.push(
|
|
2349
|
+
failFinding("tool.usage", `Required tool ${required} did not appear.`, runEvidence(context.selectedRun), required, names)
|
|
2350
|
+
);
|
|
2351
|
+
}
|
|
2409
2352
|
}
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
);
|
|
2419
|
-
if (candidates.length > 1) {
|
|
2420
|
-
diagnostics.push({
|
|
2421
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
2422
|
-
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
2423
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
2424
|
-
});
|
|
2425
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
2353
|
+
const forbidden = new Set(options.forbidden ?? []);
|
|
2354
|
+
const allowed = options.allowed?.length ? new Set(options.allowed) : void 0;
|
|
2355
|
+
for (const event of tools) {
|
|
2356
|
+
const name = toolName(event);
|
|
2357
|
+
if (forbidden.has(name)) {
|
|
2358
|
+
findings.push(
|
|
2359
|
+
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence(event)], "tool absent", name)
|
|
2360
|
+
);
|
|
2426
2361
|
}
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
);
|
|
2432
|
-
if (candidates.length > 1) {
|
|
2433
|
-
diagnostics.push({
|
|
2434
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
2435
|
-
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
2436
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
2437
|
-
});
|
|
2438
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
2362
|
+
if (allowed && !allowed.has(name)) {
|
|
2363
|
+
findings.push(
|
|
2364
|
+
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence(event)], [...allowed].sort(), name)
|
|
2365
|
+
);
|
|
2439
2366
|
}
|
|
2440
|
-
match = candidates[0];
|
|
2441
2367
|
}
|
|
2442
|
-
if (
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
logical.push(paired);
|
|
2447
|
-
logicalByRawId.set(start.eventId, paired);
|
|
2448
|
-
logicalByRawId.set(match.eventId, paired);
|
|
2449
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
2450
|
-
} else {
|
|
2451
|
-
diagnostics.push({
|
|
2452
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
2453
|
-
message: `No matching complete for start ${start.eventId}.`,
|
|
2454
|
-
eventIds: [start.eventId]
|
|
2455
|
-
});
|
|
2456
|
-
const alone = asLogical(start);
|
|
2457
|
-
logical.push(alone);
|
|
2458
|
-
logicalByRawId.set(start.eventId, alone);
|
|
2459
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2368
|
+
if (options.minCount !== void 0 && tools.length < options.minCount) {
|
|
2369
|
+
findings.push(
|
|
2370
|
+
failFinding("tool.usage", `Tool count ${tools.length} was below minimum ${options.minCount}.`, runEvidence(context.selectedRun), { minCount: options.minCount }, tools.length)
|
|
2371
|
+
);
|
|
2460
2372
|
}
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
2466
|
-
message: `No matching start for complete ${complete.eventId}.`,
|
|
2467
|
-
eventIds: [complete.eventId]
|
|
2468
|
-
});
|
|
2469
|
-
const alone = asLogical(complete);
|
|
2470
|
-
logical.push(alone);
|
|
2471
|
-
logicalByRawId.set(complete.eventId, alone);
|
|
2472
|
-
const stepId = stepIdOf(complete);
|
|
2473
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
2474
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2373
|
+
if (options.maxCount !== void 0 && tools.length > options.maxCount) {
|
|
2374
|
+
findings.push(
|
|
2375
|
+
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) => eventEvidence(event)), { maxCount: options.maxCount }, tools.length)
|
|
2376
|
+
);
|
|
2475
2377
|
}
|
|
2378
|
+
return findings;
|
|
2476
2379
|
}
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2380
|
+
};
|
|
2381
|
+
}
|
|
2382
|
+
function createLlmUsageRule(options) {
|
|
2383
|
+
return {
|
|
2384
|
+
id: "llm.usage",
|
|
2385
|
+
category: "llm",
|
|
2386
|
+
defaultSeverity: "error",
|
|
2387
|
+
evaluate(context) {
|
|
2388
|
+
const llms = finishedEvents(context, "LLM");
|
|
2389
|
+
const findings = [];
|
|
2390
|
+
const allowedModels = options.allowedModels?.length ? new Set(options.allowedModels) : void 0;
|
|
2391
|
+
const allowedProviders = options.allowedProviders?.length ? new Set(options.allowedProviders) : void 0;
|
|
2392
|
+
const finishReasons = options.finishReasons?.length ? new Set(options.finishReasons) : void 0;
|
|
2393
|
+
if (options.maxCalls !== void 0 && llms.length > options.maxCalls) {
|
|
2394
|
+
findings.push(
|
|
2395
|
+
failFinding(
|
|
2396
|
+
"llm.usage",
|
|
2397
|
+
`LLM call count ${llms.length} exceeded ${options.maxCalls}.`,
|
|
2398
|
+
llms.map((event) => eventEvidence(event)),
|
|
2399
|
+
{ maxCalls: options.maxCalls },
|
|
2400
|
+
llms.length
|
|
2401
|
+
)
|
|
2402
|
+
);
|
|
2484
2403
|
}
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
if (viaStep) {
|
|
2504
|
-
nextParent = viaStep;
|
|
2505
|
-
remapped = true;
|
|
2506
|
-
}
|
|
2507
|
-
}
|
|
2508
|
-
if (!remapped) {
|
|
2509
|
-
if (originalParentId === event.eventId) {
|
|
2510
|
-
diagnostics.push({
|
|
2511
|
-
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
2512
|
-
message: formatProgrammaticDiagnostic(
|
|
2513
|
-
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2514
|
-
`Removed self-parent edge on ${event.eventId}.`
|
|
2515
|
-
),
|
|
2516
|
-
eventIds: [event.eventId]
|
|
2517
|
-
});
|
|
2518
|
-
const { parentId: _drop, ...rest } = event;
|
|
2519
|
-
normalized.push({
|
|
2520
|
-
...rest,
|
|
2521
|
-
projection: {
|
|
2522
|
-
...event.projection,
|
|
2523
|
-
parentNormalized: true,
|
|
2524
|
-
originalParentId
|
|
2525
|
-
}
|
|
2526
|
-
});
|
|
2527
|
-
continue;
|
|
2528
|
-
}
|
|
2529
|
-
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
2530
|
-
const mapping = event.attributes?.parentMapping;
|
|
2531
|
-
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
2532
|
-
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
2533
|
-
if (!unresolved) {
|
|
2534
|
-
diagnostics.push({
|
|
2535
|
-
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
2536
|
-
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
2537
|
-
eventIds: [event.eventId]
|
|
2538
|
-
});
|
|
2404
|
+
for (const event of llms) {
|
|
2405
|
+
const model = llmModel(event);
|
|
2406
|
+
const provider = llmProvider(event);
|
|
2407
|
+
const finishReason = llmFinishReason(event);
|
|
2408
|
+
if (allowedModels && (!model || !allowedModels.has(model))) {
|
|
2409
|
+
findings.push(
|
|
2410
|
+
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.model")], [...allowedModels].sort(), model ?? "unknown")
|
|
2411
|
+
);
|
|
2412
|
+
}
|
|
2413
|
+
if (allowedProviders && (!provider || !allowedProviders.has(provider))) {
|
|
2414
|
+
findings.push(
|
|
2415
|
+
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.provider")], [...allowedProviders].sort(), provider ?? "unknown")
|
|
2416
|
+
);
|
|
2417
|
+
}
|
|
2418
|
+
if (finishReasons && (!finishReason || !finishReasons.has(finishReason))) {
|
|
2419
|
+
findings.push(
|
|
2420
|
+
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.finishReason")], [...finishReasons].sort(), finishReason ?? "unknown")
|
|
2421
|
+
);
|
|
2539
2422
|
}
|
|
2540
2423
|
}
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2424
|
+
const tokenTotals = llms.reduce(
|
|
2425
|
+
(totals, event) => ({
|
|
2426
|
+
input: totals.input + (event.tokenUsage?.input ?? 0),
|
|
2427
|
+
output: totals.output + (event.tokenUsage?.output ?? 0),
|
|
2428
|
+
total: totals.total + (event.tokenUsage?.total ?? 0),
|
|
2429
|
+
cached: totals.cached + (event.tokenUsage?.cached ?? 0)
|
|
2430
|
+
}),
|
|
2431
|
+
{ input: 0, output: 0, total: 0, cached: 0 }
|
|
2432
|
+
);
|
|
2433
|
+
const tokenLimits = [
|
|
2434
|
+
["input", options.maxInputTokens],
|
|
2435
|
+
["output", options.maxOutputTokens],
|
|
2436
|
+
["total", options.maxTotalTokens],
|
|
2437
|
+
["cached", options.maxCachedTokens]
|
|
2438
|
+
];
|
|
2439
|
+
for (const [key, limit] of tokenLimits) {
|
|
2440
|
+
if (limit !== void 0 && tokenTotals[key] > limit) {
|
|
2441
|
+
findings.push(
|
|
2442
|
+
failFinding(
|
|
2443
|
+
"llm.usage",
|
|
2444
|
+
`LLM ${key} token count ${tokenTotals[key]} exceeded ${limit}.`,
|
|
2445
|
+
llms.map((event) => eventEvidence(event, `tokenUsage.${key}`)),
|
|
2446
|
+
{ [`max${key[0].toUpperCase()}${key.slice(1)}Tokens`]: limit },
|
|
2447
|
+
tokenTotals[key]
|
|
2448
|
+
)
|
|
2449
|
+
);
|
|
2560
2450
|
}
|
|
2561
|
-
});
|
|
2562
|
-
continue;
|
|
2563
|
-
}
|
|
2564
|
-
diagnostics.push({
|
|
2565
|
-
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
2566
|
-
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
2567
|
-
eventIds: [event.eventId]
|
|
2568
|
-
});
|
|
2569
|
-
normalized.push({
|
|
2570
|
-
...event,
|
|
2571
|
-
parentId: nextParent,
|
|
2572
|
-
projection: {
|
|
2573
|
-
...event.projection,
|
|
2574
|
-
parentNormalized: true,
|
|
2575
|
-
originalParentId
|
|
2576
2451
|
}
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
});
|
|
2452
|
+
return findings;
|
|
2453
|
+
}
|
|
2454
|
+
};
|
|
2455
|
+
}
|
|
2456
|
+
function createObservedOutcomeRule(options = {}) {
|
|
2457
|
+
const failOn = options.failOn ?? ["failed"];
|
|
2458
|
+
const requireAny = options.requireAny === true;
|
|
2585
2459
|
return {
|
|
2586
|
-
|
|
2587
|
-
|
|
2460
|
+
id: "outcome.status",
|
|
2461
|
+
category: "run",
|
|
2462
|
+
defaultSeverity: "error",
|
|
2463
|
+
evaluate(context) {
|
|
2464
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
2465
|
+
if (requireAny && outcomes.length === 0) {
|
|
2466
|
+
return [
|
|
2467
|
+
failFinding(
|
|
2468
|
+
"outcome.status",
|
|
2469
|
+
"Expected at least one observed outcome.",
|
|
2470
|
+
runEvidence(context.selectedRun),
|
|
2471
|
+
{ requireAny: true, expected: "at least one observed outcome" },
|
|
2472
|
+
{ code: "outcome.missing", actual: 0 }
|
|
2473
|
+
)
|
|
2474
|
+
];
|
|
2475
|
+
}
|
|
2476
|
+
const matching = outcomesMatchingStatus(outcomes, failOn);
|
|
2477
|
+
if (matching.length === 0) return [];
|
|
2478
|
+
return [
|
|
2479
|
+
failFinding(
|
|
2480
|
+
"outcome.status",
|
|
2481
|
+
`Observed outcome count ${matching.length} matched [${failOn.join(", ")}].`,
|
|
2482
|
+
matching.map((outcome) => ({
|
|
2483
|
+
runId: outcome.runId,
|
|
2484
|
+
eventId: outcome.outcomeId,
|
|
2485
|
+
...outcome.parentId !== void 0 ? { parentId: outcome.parentId } : {},
|
|
2486
|
+
kind: "OUTCOME",
|
|
2487
|
+
name: outcome.name,
|
|
2488
|
+
status: outcome.status,
|
|
2489
|
+
path: `outcome.${outcome.name}`
|
|
2490
|
+
})),
|
|
2491
|
+
{ failOn },
|
|
2492
|
+
matching.map((outcome) => ({
|
|
2493
|
+
name: outcome.name,
|
|
2494
|
+
status: outcome.status,
|
|
2495
|
+
expectation: outcome.expectation
|
|
2496
|
+
}))
|
|
2497
|
+
)
|
|
2498
|
+
];
|
|
2499
|
+
}
|
|
2588
2500
|
};
|
|
2589
2501
|
}
|
|
2590
|
-
function
|
|
2591
|
-
const
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
const metadata = attrs?.metadata;
|
|
2595
|
-
if (isRecord5(metadata)) {
|
|
2596
|
-
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
2597
|
-
if (nested) return nested;
|
|
2502
|
+
function runTraceChecks(input, options = {}) {
|
|
2503
|
+
const selected = resolveSelectedRun(input, options.runId);
|
|
2504
|
+
if (selected.diagnostics.length > 0) {
|
|
2505
|
+
return errorResult(input, selected.diagnostics, selected.run);
|
|
2598
2506
|
}
|
|
2599
|
-
|
|
2600
|
-
|
|
2507
|
+
const rules = selectRules(options.rules ?? [], options.select);
|
|
2508
|
+
if (rules.diagnostics.length > 0) {
|
|
2509
|
+
return errorResult(input, rules.diagnostics, selected.run);
|
|
2601
2510
|
}
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2511
|
+
if (rules.rules.length === 0) {
|
|
2512
|
+
return errorResult(
|
|
2513
|
+
input,
|
|
2514
|
+
[
|
|
2515
|
+
diagnostic(
|
|
2516
|
+
"AI_CHECK_NO_RULES_EVALUATED",
|
|
2517
|
+
"No trace check rules were evaluated. Configure at least one rule, contract, or CLI check option."
|
|
2518
|
+
)
|
|
2519
|
+
],
|
|
2520
|
+
selected.run
|
|
2521
|
+
);
|
|
2609
2522
|
}
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
"AI_TRACE_FACTS_INPUT_NOT_NORMALIZED"
|
|
2616
|
-
);
|
|
2617
|
-
|
|
2618
|
-
// packages/core/src/checks/contract.ts
|
|
2619
|
-
new Set(OBSERVED_OUTCOME_METHODS);
|
|
2620
|
-
|
|
2621
|
-
// packages/core/src/checks/index.ts
|
|
2622
|
-
var SEVERITY_RANK = {
|
|
2623
|
-
error: 0,
|
|
2624
|
-
warning: 1,
|
|
2625
|
-
info: 2
|
|
2626
|
-
};
|
|
2627
|
-
var STATUS_RANK = {
|
|
2628
|
-
fail: 0,
|
|
2629
|
-
warning: 1,
|
|
2630
|
-
pass: 2
|
|
2631
|
-
};
|
|
2632
|
-
function compareStrings(a, b) {
|
|
2633
|
-
return (a ?? "").localeCompare(b ?? "");
|
|
2634
|
-
}
|
|
2635
|
-
function diagnostic3(code, message, ruleId) {
|
|
2636
|
-
return {
|
|
2637
|
-
code,
|
|
2638
|
-
message,
|
|
2639
|
-
severity: "error",
|
|
2640
|
-
...ruleId ? { ruleId } : {}
|
|
2641
|
-
};
|
|
2642
|
-
}
|
|
2643
|
-
function emptySummary() {
|
|
2644
|
-
return {
|
|
2645
|
-
passed: 0,
|
|
2646
|
-
failed: 0,
|
|
2647
|
-
warnings: 0,
|
|
2648
|
-
errors: 0,
|
|
2649
|
-
rulesEvaluated: 0
|
|
2523
|
+
const facts = buildFacts(input, selected.run);
|
|
2524
|
+
const context = {
|
|
2525
|
+
...facts,
|
|
2526
|
+
...selected.run ? { selectedRun: selected.run } : {},
|
|
2527
|
+
...input.sourceLabel ? { sourceLabel: input.sourceLabel } : {}
|
|
2650
2528
|
};
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
2678
|
-
for (const node of nodes) {
|
|
2679
|
-
nodesByEventId.set(node.event.eventId, node);
|
|
2680
|
-
const parentId = node.event.parentId;
|
|
2681
|
-
if (parentId) {
|
|
2682
|
-
const children = childrenByParentId.get(parentId) ?? [];
|
|
2683
|
-
children.push(node);
|
|
2684
|
-
childrenByParentId.set(parentId, children);
|
|
2529
|
+
const diagnostics = [];
|
|
2530
|
+
const findings = [];
|
|
2531
|
+
const ruleExecutions = [];
|
|
2532
|
+
for (const rule of rules.rules) {
|
|
2533
|
+
try {
|
|
2534
|
+
const ruleFindings = rule.evaluate(context).map((finding) => normalizeFinding(rule, finding));
|
|
2535
|
+
findings.push(...ruleFindings);
|
|
2536
|
+
ruleExecutions.push({
|
|
2537
|
+
ruleId: rule.id,
|
|
2538
|
+
category: rule.category,
|
|
2539
|
+
status: classifyRuleExecution(ruleFindings, false),
|
|
2540
|
+
findingCount: ruleFindings.length,
|
|
2541
|
+
...selected.run ? { runId: selected.run.runId } : {}
|
|
2542
|
+
});
|
|
2543
|
+
} catch (error) {
|
|
2544
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2545
|
+
diagnostics.push(
|
|
2546
|
+
diagnostic("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
|
|
2547
|
+
);
|
|
2548
|
+
ruleExecutions.push({
|
|
2549
|
+
ruleId: rule.id,
|
|
2550
|
+
category: rule.category,
|
|
2551
|
+
status: "error",
|
|
2552
|
+
findingCount: 0,
|
|
2553
|
+
...selected.run ? { runId: selected.run.runId } : {}
|
|
2554
|
+
});
|
|
2685
2555
|
}
|
|
2686
2556
|
}
|
|
2687
|
-
|
|
2557
|
+
if (diagnostics.length > 0) {
|
|
2558
|
+
return errorResult(input, diagnostics, selected.run, ruleExecutions);
|
|
2559
|
+
}
|
|
2560
|
+
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
2561
|
+
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
2562
|
+
const summary = summarize(sortedFindings, diagnostics, ruleExecutions.length);
|
|
2563
|
+
const status = summary.failed > 0 ? "fail" : "pass";
|
|
2688
2564
|
return {
|
|
2565
|
+
ok: status === "pass",
|
|
2566
|
+
status,
|
|
2689
2567
|
format: input.read.format,
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
|
|
2696
|
-
sourceFiles: Object.freeze([...input.read.sourceFiles]),
|
|
2697
|
-
nodesByEventId,
|
|
2698
|
-
childrenByParentId,
|
|
2699
|
-
rootNodes: Object.freeze(scopedRuns.flatMap((run) => run.children))
|
|
2568
|
+
...selected.run ? { runId: selected.run.runId } : {},
|
|
2569
|
+
summary,
|
|
2570
|
+
findings: sortedFindings,
|
|
2571
|
+
diagnostics,
|
|
2572
|
+
ruleExecutions
|
|
2700
2573
|
};
|
|
2701
2574
|
}
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
return { run: input.selectedRun, diagnostics: [] };
|
|
2575
|
+
|
|
2576
|
+
// packages/core/src/checks/contract.ts
|
|
2577
|
+
new Set(OBSERVED_OUTCOME_METHODS);
|
|
2578
|
+
|
|
2579
|
+
// packages/core/src/exporters/helpers.ts
|
|
2580
|
+
function sortKeysDeep(input) {
|
|
2581
|
+
if (input === null || typeof input !== "object") return input;
|
|
2582
|
+
if (Array.isArray(input)) return input.map(sortKeysDeep);
|
|
2583
|
+
const o = input;
|
|
2584
|
+
const out = {};
|
|
2585
|
+
for (const k of Object.keys(o).sort()) {
|
|
2586
|
+
out[k] = sortKeysDeep(o[k]);
|
|
2715
2587
|
}
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2588
|
+
return out;
|
|
2589
|
+
}
|
|
2590
|
+
function stableJson(value, pretty) {
|
|
2591
|
+
const sorted = sortKeysDeep(value);
|
|
2592
|
+
return JSON.stringify(sorted);
|
|
2593
|
+
}
|
|
2594
|
+
|
|
2595
|
+
// packages/core/src/diff/comparable.ts
|
|
2596
|
+
function extractOutputPreview(meta) {
|
|
2597
|
+
if (meta === void 0) return void 0;
|
|
2598
|
+
if ("outputPreview" in meta) return meta.outputPreview;
|
|
2599
|
+
if ("resultPreview" in meta) return meta.resultPreview;
|
|
2600
|
+
return void 0;
|
|
2601
|
+
}
|
|
2602
|
+
function mapStepStatus(s) {
|
|
2603
|
+
if (s === void 0) return "running";
|
|
2604
|
+
return s;
|
|
2605
|
+
}
|
|
2606
|
+
function manualTraceEventsToComparableRun(events) {
|
|
2607
|
+
const started = events.find((e) => e.event === "run_started");
|
|
2608
|
+
if (!started || started.event !== "run_started") {
|
|
2609
|
+
throw new Error("Invalid trace: missing run_started");
|
|
2726
2610
|
}
|
|
2727
|
-
|
|
2728
|
-
|
|
2611
|
+
const rs = started;
|
|
2612
|
+
const runId = rs.runId;
|
|
2613
|
+
const completedAll = events.filter((e) => e.event === "run_completed");
|
|
2614
|
+
const lastCompleted = completedAll[completedAll.length - 1];
|
|
2615
|
+
let runStatus;
|
|
2616
|
+
if (lastCompleted === void 0) runStatus = "running";
|
|
2617
|
+
else runStatus = lastCompleted.status;
|
|
2618
|
+
const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
2619
|
+
const steps = /* @__PURE__ */ new Map();
|
|
2620
|
+
let order = 0;
|
|
2621
|
+
for (const e of events) {
|
|
2622
|
+
if (e.event !== "step_started") continue;
|
|
2623
|
+
const s = e;
|
|
2624
|
+
const meta = s.metadata ? { ...s.metadata } : void 0;
|
|
2625
|
+
steps.set(s.stepId, {
|
|
2626
|
+
id: s.stepId,
|
|
2627
|
+
parentId: s.parentId,
|
|
2628
|
+
name: s.name,
|
|
2629
|
+
type: s.type,
|
|
2630
|
+
order: order++,
|
|
2631
|
+
timestamp: s.timestamp,
|
|
2632
|
+
metadata: meta
|
|
2633
|
+
});
|
|
2729
2634
|
}
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2635
|
+
for (const e of events) {
|
|
2636
|
+
if (e.event !== "step_completed") continue;
|
|
2637
|
+
const acc = steps.get(e.stepId);
|
|
2638
|
+
if (!acc) continue;
|
|
2639
|
+
acc.status = e.status;
|
|
2640
|
+
acc.durationMs = e.durationMs;
|
|
2641
|
+
if (e.error?.message) acc.errorMsg = e.error.message;
|
|
2642
|
+
const extra = e;
|
|
2643
|
+
if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
|
|
2644
|
+
acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
|
|
2645
|
+
}
|
|
2646
|
+
}
|
|
2647
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
2648
|
+
for (const acc of steps.values()) {
|
|
2649
|
+
let meta = acc.metadata ? { ...acc.metadata } : void 0;
|
|
2650
|
+
if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
|
|
2651
|
+
meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
|
|
2652
|
+
}
|
|
2653
|
+
const outputPreview = extractOutputPreview(meta);
|
|
2654
|
+
if (meta !== void 0 && ("outputPreview" in meta || "resultPreview" in meta)) {
|
|
2655
|
+
delete meta.outputPreview;
|
|
2656
|
+
delete meta.resultPreview;
|
|
2657
|
+
}
|
|
2658
|
+
const sc = {
|
|
2659
|
+
id: acc.id,
|
|
2660
|
+
name: acc.name,
|
|
2661
|
+
type: acc.type,
|
|
2662
|
+
status: mapStepStatus(acc.status),
|
|
2663
|
+
durationMs: acc.durationMs,
|
|
2664
|
+
error: acc.errorMsg,
|
|
2665
|
+
metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
|
|
2666
|
+
outputPreview,
|
|
2667
|
+
children: []
|
|
2741
2668
|
};
|
|
2669
|
+
nodes.set(acc.id, sc);
|
|
2742
2670
|
}
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
)
|
|
2749
|
-
]
|
|
2671
|
+
const roots = [];
|
|
2672
|
+
const sortByOrder = (a, b) => {
|
|
2673
|
+
const oa = steps.get(a.id)?.order ?? 0;
|
|
2674
|
+
const ob = steps.get(b.id)?.order ?? 0;
|
|
2675
|
+
return oa - ob;
|
|
2750
2676
|
};
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
diagnostics.push(
|
|
2758
|
-
diagnostic3("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
2759
|
-
);
|
|
2760
|
-
continue;
|
|
2677
|
+
for (const acc of steps.values()) {
|
|
2678
|
+
const node = nodes.get(acc.id);
|
|
2679
|
+
if (acc.parentId !== void 0 && nodes.has(acc.parentId)) {
|
|
2680
|
+
nodes.get(acc.parentId).children.push(node);
|
|
2681
|
+
} else {
|
|
2682
|
+
roots.push(node);
|
|
2761
2683
|
}
|
|
2762
|
-
byId.set(rule.id, rule);
|
|
2763
2684
|
}
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
if (!byId.has(id)) {
|
|
2768
|
-
diagnostics.push(
|
|
2769
|
-
diagnostic3("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
2770
|
-
);
|
|
2771
|
-
}
|
|
2772
|
-
}
|
|
2773
|
-
return {
|
|
2774
|
-
rules: [...byId.values()].filter((rule) => selected.has(rule.id)).sort(compareRules),
|
|
2775
|
-
diagnostics
|
|
2776
|
-
};
|
|
2685
|
+
roots.sort(sortByOrder);
|
|
2686
|
+
for (const n of nodes.values()) {
|
|
2687
|
+
n.children.sort(sortByOrder);
|
|
2777
2688
|
}
|
|
2778
|
-
return {
|
|
2689
|
+
return {
|
|
2690
|
+
runId,
|
|
2691
|
+
name: rs.name,
|
|
2692
|
+
status: runStatus,
|
|
2693
|
+
durationMs,
|
|
2694
|
+
steps: roots
|
|
2695
|
+
};
|
|
2779
2696
|
}
|
|
2780
|
-
|
|
2781
|
-
|
|
2697
|
+
|
|
2698
|
+
// packages/core/src/diff/engine.ts
|
|
2699
|
+
var DEFAULT_THRESHOLD_MS = 0;
|
|
2700
|
+
function pathSeg(step, index) {
|
|
2701
|
+
return { index, name: step.name, stepId: step.id };
|
|
2782
2702
|
}
|
|
2783
|
-
function
|
|
2784
|
-
|
|
2785
|
-
return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
|
|
2703
|
+
function buildPath(segments) {
|
|
2704
|
+
return { path: [...segments] };
|
|
2786
2705
|
}
|
|
2787
|
-
function
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2706
|
+
function pairSteps(left, right) {
|
|
2707
|
+
const usedRight = /* @__PURE__ */ new Set();
|
|
2708
|
+
const pairs = [];
|
|
2709
|
+
for (let i = 0; i < left.length; i++) {
|
|
2710
|
+
const L = left[i];
|
|
2711
|
+
let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
|
|
2712
|
+
if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
|
|
2713
|
+
const cand = right[i];
|
|
2714
|
+
if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
|
|
2715
|
+
R = cand;
|
|
2716
|
+
}
|
|
2791
2717
|
}
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2718
|
+
if (R === void 0) {
|
|
2719
|
+
R = right.find(
|
|
2720
|
+
(r) => !usedRight.has(r.id) && r.name === L.name && (r.type ?? "") === (L.type ?? "")
|
|
2721
|
+
);
|
|
2722
|
+
}
|
|
2723
|
+
if (R !== void 0) {
|
|
2724
|
+
usedRight.add(R.id);
|
|
2725
|
+
pairs.push([L, R]);
|
|
2726
|
+
} else {
|
|
2727
|
+
pairs.push([L, void 0]);
|
|
2796
2728
|
}
|
|
2797
|
-
const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
|
|
2798
|
-
if (byRun !== 0) return byRun;
|
|
2799
|
-
const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
|
|
2800
|
-
if (byTime !== 0) return byTime;
|
|
2801
|
-
const byEvent = compareStrings(a.evidence[0]?.eventId, b.evidence[0]?.eventId);
|
|
2802
|
-
if (byEvent !== 0) return byEvent;
|
|
2803
|
-
return compareStrings(a.evidence[0]?.path, b.evidence[0]?.path);
|
|
2804
|
-
};
|
|
2805
|
-
}
|
|
2806
|
-
function normalizeFinding(rule, finding) {
|
|
2807
|
-
return {
|
|
2808
|
-
ruleId: finding.ruleId || rule.id,
|
|
2809
|
-
severity: finding.severity ?? rule.defaultSeverity,
|
|
2810
|
-
status: finding.status,
|
|
2811
|
-
message: finding.message,
|
|
2812
|
-
...finding.expected !== void 0 ? { expected: finding.expected } : {},
|
|
2813
|
-
...finding.actual !== void 0 ? { actual: finding.actual } : {},
|
|
2814
|
-
evidence: [...finding.evidence ?? []],
|
|
2815
|
-
...finding.category !== void 0 ? { category: finding.category } : {},
|
|
2816
|
-
...finding.confidence !== void 0 ? { confidence: finding.confidence } : {},
|
|
2817
|
-
...finding.detector !== void 0 ? { detector: finding.detector } : {},
|
|
2818
|
-
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
2819
|
-
};
|
|
2820
|
-
}
|
|
2821
|
-
function summarize(findings, diagnostics, rulesEvaluated) {
|
|
2822
|
-
return {
|
|
2823
|
-
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
2824
|
-
failed: findings.filter(
|
|
2825
|
-
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
2826
|
-
).length,
|
|
2827
|
-
warnings: findings.filter(
|
|
2828
|
-
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
2829
|
-
).length,
|
|
2830
|
-
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2831
|
-
rulesEvaluated
|
|
2832
|
-
};
|
|
2833
|
-
}
|
|
2834
|
-
function classifyRuleExecution(findings, threw) {
|
|
2835
|
-
if (findings.some((finding) => finding.status === "fail" && finding.severity === "error")) {
|
|
2836
|
-
return "fail";
|
|
2837
2729
|
}
|
|
2838
|
-
|
|
2839
|
-
(
|
|
2840
|
-
|
|
2841
|
-
|
|
2730
|
+
for (const R of right) {
|
|
2731
|
+
if (!usedRight.has(R.id)) {
|
|
2732
|
+
pairs.push([void 0, R]);
|
|
2733
|
+
}
|
|
2842
2734
|
}
|
|
2843
|
-
return
|
|
2735
|
+
return pairs;
|
|
2844
2736
|
}
|
|
2845
|
-
function
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2737
|
+
function compareLeafSteps(L, R, segments, opts, out) {
|
|
2738
|
+
const path18 = buildPath(segments);
|
|
2739
|
+
if (L.name !== R.name) {
|
|
2740
|
+
out.push({
|
|
2741
|
+
kind: "structure",
|
|
2742
|
+
severity: "warning",
|
|
2743
|
+
message: "Step name differs",
|
|
2744
|
+
path: path18,
|
|
2745
|
+
left: L.name,
|
|
2746
|
+
right: R.name
|
|
2747
|
+
});
|
|
2849
2748
|
}
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2749
|
+
if ((L.type ?? "") !== (R.type ?? "")) {
|
|
2750
|
+
out.push({
|
|
2751
|
+
kind: "step-type",
|
|
2752
|
+
severity: "warning",
|
|
2753
|
+
message: "Step type differs",
|
|
2754
|
+
path: path18,
|
|
2755
|
+
left: L.type,
|
|
2756
|
+
right: R.type
|
|
2757
|
+
});
|
|
2856
2758
|
}
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
2759
|
+
if ((L.status ?? "") !== (R.status ?? "")) {
|
|
2760
|
+
out.push({
|
|
2761
|
+
kind: "step-status",
|
|
2762
|
+
severity: "warning",
|
|
2763
|
+
message: "Step status differs",
|
|
2764
|
+
path: path18,
|
|
2765
|
+
left: L.status,
|
|
2766
|
+
right: R.status
|
|
2767
|
+
});
|
|
2768
|
+
}
|
|
2769
|
+
const le = L.error ?? "";
|
|
2770
|
+
const re = R.error ?? "";
|
|
2771
|
+
if (le !== re) {
|
|
2772
|
+
out.push({
|
|
2773
|
+
kind: "error",
|
|
2774
|
+
severity: "error",
|
|
2775
|
+
message: "Step error message differs",
|
|
2776
|
+
path: path18,
|
|
2777
|
+
left: le || void 0,
|
|
2778
|
+
right: re || void 0
|
|
2779
|
+
});
|
|
2780
|
+
}
|
|
2781
|
+
if (!opts.ignoreDuration) {
|
|
2782
|
+
const ld = L.durationMs;
|
|
2783
|
+
const rd = R.durationMs;
|
|
2784
|
+
const th = opts.durationThresholdMs;
|
|
2785
|
+
let differs = false;
|
|
2786
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
2787
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
2788
|
+
else differs = Math.abs(ld - rd) > th;
|
|
2789
|
+
if (differs) {
|
|
2790
|
+
out.push({
|
|
2791
|
+
kind: "duration",
|
|
2792
|
+
severity: "info",
|
|
2793
|
+
message: "Step duration differs",
|
|
2794
|
+
path: path18,
|
|
2795
|
+
left: ld,
|
|
2796
|
+
right: rd
|
|
2797
|
+
});
|
|
2798
|
+
}
|
|
2799
|
+
}
|
|
2800
|
+
const lm = stableJson(L.metadata ?? {});
|
|
2801
|
+
const rm2 = stableJson(R.metadata ?? {});
|
|
2802
|
+
if (lm !== rm2) {
|
|
2803
|
+
out.push({
|
|
2804
|
+
kind: "metadata",
|
|
2805
|
+
severity: "info",
|
|
2806
|
+
message: "Step metadata differs",
|
|
2807
|
+
path: path18,
|
|
2808
|
+
left: L.metadata,
|
|
2809
|
+
right: R.metadata
|
|
2810
|
+
});
|
|
2811
|
+
}
|
|
2812
|
+
const lo = stableJson(L.outputPreview ?? null);
|
|
2813
|
+
const ro = stableJson(R.outputPreview ?? null);
|
|
2814
|
+
if (lo !== ro) {
|
|
2815
|
+
out.push({
|
|
2816
|
+
kind: "output",
|
|
2817
|
+
severity: "info",
|
|
2818
|
+
message: "Output preview differs",
|
|
2819
|
+
path: path18,
|
|
2820
|
+
left: L.outputPreview,
|
|
2821
|
+
right: R.outputPreview
|
|
2822
|
+
});
|
|
2861
2823
|
}
|
|
2862
|
-
return void 0;
|
|
2863
2824
|
}
|
|
2864
|
-
function
|
|
2865
|
-
|
|
2866
|
-
|
|
2825
|
+
function compareRecursive(L, R, segments, opts, out) {
|
|
2826
|
+
compareLeafSteps(L, R, segments, opts, out);
|
|
2827
|
+
const pairs = pairSteps(L.children, R.children);
|
|
2828
|
+
let ci = 0;
|
|
2829
|
+
for (const [lch, rch] of pairs) {
|
|
2830
|
+
if (lch !== void 0 && rch !== void 0) {
|
|
2831
|
+
compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
|
|
2832
|
+
} else if (lch !== void 0) {
|
|
2833
|
+
out.push({
|
|
2834
|
+
kind: "step-removed",
|
|
2835
|
+
severity: "warning",
|
|
2836
|
+
message: `Step only in left run: ${lch.name}`,
|
|
2837
|
+
path: buildPath([...segments, pathSeg(lch, ci)]),
|
|
2838
|
+
left: lch.id,
|
|
2839
|
+
right: void 0
|
|
2840
|
+
});
|
|
2841
|
+
} else if (rch !== void 0) {
|
|
2842
|
+
out.push({
|
|
2843
|
+
kind: "step-added",
|
|
2844
|
+
severity: "warning",
|
|
2845
|
+
message: `Step only in right run: ${rch.name}`,
|
|
2846
|
+
path: buildPath([...segments, pathSeg(rch, ci)]),
|
|
2847
|
+
left: void 0,
|
|
2848
|
+
right: rch.id
|
|
2849
|
+
});
|
|
2850
|
+
}
|
|
2851
|
+
ci += 1;
|
|
2867
2852
|
}
|
|
2868
|
-
return name;
|
|
2869
2853
|
}
|
|
2870
|
-
function
|
|
2854
|
+
function mergeDiffDefaults(options) {
|
|
2871
2855
|
return {
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
|
|
2875
|
-
|
|
2876
|
-
spanId: event.trace?.spanId,
|
|
2877
|
-
kind: event.kind,
|
|
2878
|
-
name: event.name,
|
|
2879
|
-
status: event.status,
|
|
2880
|
-
...path18 ? { path: path18 } : {}
|
|
2856
|
+
ignoreDuration: false,
|
|
2857
|
+
durationThresholdMs: DEFAULT_THRESHOLD_MS,
|
|
2858
|
+
focus: "all",
|
|
2859
|
+
check: "all"
|
|
2881
2860
|
};
|
|
2882
2861
|
}
|
|
2883
|
-
function
|
|
2884
|
-
return
|
|
2862
|
+
function kindMatchesFilter(kind, merged) {
|
|
2863
|
+
return true;
|
|
2885
2864
|
}
|
|
2886
|
-
function
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
message,
|
|
2892
|
-
...expected !== void 0 ? { expected } : {},
|
|
2893
|
-
...actual !== void 0 ? { actual } : {},
|
|
2894
|
-
evidence: [...evidence],
|
|
2895
|
-
...{},
|
|
2896
|
-
...{},
|
|
2897
|
-
...{},
|
|
2898
|
-
...{}
|
|
2865
|
+
function diffRuns(left, right, options) {
|
|
2866
|
+
const merged = mergeDiffDefaults();
|
|
2867
|
+
const opts = {
|
|
2868
|
+
ignoreDuration: merged.ignoreDuration,
|
|
2869
|
+
durationThresholdMs: merged.durationThresholdMs
|
|
2899
2870
|
};
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
|
|
2903
|
-
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
}
|
|
2927
|
-
function createRunStatusRule(options = {}) {
|
|
2928
|
-
const expected = options.expected ?? "ok";
|
|
2929
|
-
const allowIncomplete = options.allowIncomplete === true;
|
|
2930
|
-
return {
|
|
2931
|
-
id: "run.status",
|
|
2932
|
-
category: "run",
|
|
2933
|
-
defaultSeverity: "error",
|
|
2934
|
-
evaluate(context) {
|
|
2935
|
-
const findings = [];
|
|
2936
|
-
const actual = context.selectedRun?.status ?? "unknown";
|
|
2937
|
-
if (actual !== expected) {
|
|
2938
|
-
findings.push(
|
|
2939
|
-
failFinding(
|
|
2940
|
-
"run.status",
|
|
2941
|
-
`Run status ${actual} did not match expected ${expected}.`,
|
|
2942
|
-
runEvidence(context.selectedRun),
|
|
2943
|
-
expected,
|
|
2944
|
-
actual
|
|
2945
|
-
)
|
|
2946
|
-
);
|
|
2947
|
-
}
|
|
2948
|
-
if (!allowIncomplete) {
|
|
2949
|
-
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
2950
|
-
if (running.length > 0) {
|
|
2951
|
-
findings.push(
|
|
2952
|
-
failFinding(
|
|
2953
|
-
"run.status",
|
|
2954
|
-
"Run contains incomplete running events.",
|
|
2955
|
-
running.map((event) => eventEvidence(event)),
|
|
2956
|
-
"no running events",
|
|
2957
|
-
running.length
|
|
2958
|
-
)
|
|
2959
|
-
);
|
|
2960
|
-
}
|
|
2961
|
-
}
|
|
2962
|
-
return findings;
|
|
2871
|
+
const raw = [];
|
|
2872
|
+
if ((left.status ?? "") !== (right.status ?? "")) {
|
|
2873
|
+
raw.push({
|
|
2874
|
+
kind: "run-status",
|
|
2875
|
+
severity: "warning",
|
|
2876
|
+
message: "Run completion status differs",
|
|
2877
|
+
left: left.status,
|
|
2878
|
+
right: right.status
|
|
2879
|
+
});
|
|
2880
|
+
}
|
|
2881
|
+
{
|
|
2882
|
+
const ld = left.durationMs;
|
|
2883
|
+
const rd = right.durationMs;
|
|
2884
|
+
const th = merged.durationThresholdMs;
|
|
2885
|
+
let differs = false;
|
|
2886
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
2887
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
2888
|
+
else differs = Math.abs(ld - rd) > th;
|
|
2889
|
+
if (differs) {
|
|
2890
|
+
raw.push({
|
|
2891
|
+
kind: "duration",
|
|
2892
|
+
severity: "info",
|
|
2893
|
+
message: "Run duration differs",
|
|
2894
|
+
left: ld,
|
|
2895
|
+
right: rd
|
|
2896
|
+
});
|
|
2963
2897
|
}
|
|
2964
|
-
}
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2898
|
+
}
|
|
2899
|
+
const pairs = pairSteps(left.steps, right.steps);
|
|
2900
|
+
let idx = 0;
|
|
2901
|
+
for (const [ls, rs] of pairs) {
|
|
2902
|
+
if (ls !== void 0 && rs !== void 0) {
|
|
2903
|
+
compareRecursive(ls, rs, [pathSeg(ls, idx)], opts, raw);
|
|
2904
|
+
idx += 1;
|
|
2905
|
+
} else if (ls !== void 0) {
|
|
2906
|
+
raw.push({
|
|
2907
|
+
kind: "step-removed",
|
|
2908
|
+
severity: "warning",
|
|
2909
|
+
message: `Step only in left run: ${ls.name}`,
|
|
2910
|
+
path: buildPath([pathSeg(ls, idx)]),
|
|
2911
|
+
left: ls.id,
|
|
2912
|
+
right: void 0
|
|
2913
|
+
});
|
|
2914
|
+
idx += 1;
|
|
2915
|
+
} else if (rs !== void 0) {
|
|
2916
|
+
raw.push({
|
|
2917
|
+
kind: "step-added",
|
|
2918
|
+
severity: "warning",
|
|
2919
|
+
message: `Step only in right run: ${rs.name}`,
|
|
2920
|
+
path: buildPath([pathSeg(rs, idx)]),
|
|
2921
|
+
left: void 0,
|
|
2922
|
+
right: rs.id
|
|
2923
|
+
});
|
|
2924
|
+
idx += 1;
|
|
2983
2925
|
}
|
|
2926
|
+
}
|
|
2927
|
+
const differences = raw.filter((d) => kindMatchesFilter(d.kind));
|
|
2928
|
+
let errors = 0;
|
|
2929
|
+
let warnings = 0;
|
|
2930
|
+
let info = 0;
|
|
2931
|
+
for (const d of differences) {
|
|
2932
|
+
if (d.severity === "error") errors += 1;
|
|
2933
|
+
else if (d.severity === "warning") warnings += 1;
|
|
2934
|
+
else info += 1;
|
|
2935
|
+
}
|
|
2936
|
+
const firstVisible = differences[0];
|
|
2937
|
+
const firstDivergence = firstVisible !== void 0 ? {
|
|
2938
|
+
kind: "first-divergence",
|
|
2939
|
+
severity: firstVisible.severity,
|
|
2940
|
+
message: `First divergence: ${firstVisible.message}`,
|
|
2941
|
+
path: firstVisible.path,
|
|
2942
|
+
left: firstVisible.left,
|
|
2943
|
+
right: firstVisible.right
|
|
2944
|
+
} : void 0;
|
|
2945
|
+
const summary = {
|
|
2946
|
+
leftRunId: left.runId,
|
|
2947
|
+
rightRunId: right.runId,
|
|
2948
|
+
totalDifferences: differences.length,
|
|
2949
|
+
errors,
|
|
2950
|
+
warnings,
|
|
2951
|
+
info,
|
|
2952
|
+
firstDivergence
|
|
2984
2953
|
};
|
|
2954
|
+
return { summary, differences };
|
|
2985
2955
|
}
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
const nameSet = new Set(names);
|
|
2995
|
-
const findings = [];
|
|
2996
|
-
for (const required of options.required ?? []) {
|
|
2997
|
-
if (!nameSet.has(required)) {
|
|
2998
|
-
findings.push(
|
|
2999
|
-
failFinding("tool.usage", `Required tool ${required} did not appear.`, runEvidence(context.selectedRun), required, names)
|
|
3000
|
-
);
|
|
3001
|
-
}
|
|
3002
|
-
}
|
|
3003
|
-
const forbidden = new Set(options.forbidden ?? []);
|
|
3004
|
-
const allowed = options.allowed?.length ? new Set(options.allowed) : void 0;
|
|
3005
|
-
for (const event of tools) {
|
|
3006
|
-
const name = toolName(event);
|
|
3007
|
-
if (forbidden.has(name)) {
|
|
3008
|
-
findings.push(
|
|
3009
|
-
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence(event)], "tool absent", name)
|
|
3010
|
-
);
|
|
3011
|
-
}
|
|
3012
|
-
if (allowed && !allowed.has(name)) {
|
|
3013
|
-
findings.push(
|
|
3014
|
-
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence(event)], [...allowed].sort(), name)
|
|
3015
|
-
);
|
|
3016
|
-
}
|
|
3017
|
-
}
|
|
3018
|
-
if (options.minCount !== void 0 && tools.length < options.minCount) {
|
|
3019
|
-
findings.push(
|
|
3020
|
-
failFinding("tool.usage", `Tool count ${tools.length} was below minimum ${options.minCount}.`, runEvidence(context.selectedRun), { minCount: options.minCount }, tools.length)
|
|
3021
|
-
);
|
|
3022
|
-
}
|
|
3023
|
-
if (options.maxCount !== void 0 && tools.length > options.maxCount) {
|
|
3024
|
-
findings.push(
|
|
3025
|
-
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) => eventEvidence(event)), { maxCount: options.maxCount }, tools.length)
|
|
3026
|
-
);
|
|
3027
|
-
}
|
|
3028
|
-
return findings;
|
|
2956
|
+
|
|
2957
|
+
// packages/core/src/evidence/zip.ts
|
|
2958
|
+
(() => {
|
|
2959
|
+
const table = new Uint32Array(256);
|
|
2960
|
+
for (let n = 0; n < 256; n += 1) {
|
|
2961
|
+
let c = n;
|
|
2962
|
+
for (let k = 0; k < 8; k += 1) {
|
|
2963
|
+
c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
|
|
3029
2964
|
}
|
|
3030
|
-
|
|
2965
|
+
table[n] = c >>> 0;
|
|
2966
|
+
}
|
|
2967
|
+
return table;
|
|
2968
|
+
})();
|
|
2969
|
+
|
|
2970
|
+
// packages/core/src/suite/types.ts
|
|
2971
|
+
var DEFAULT_SUITE_CONFIG_NAMES = [
|
|
2972
|
+
"agent-inspect.suite.json",
|
|
2973
|
+
"agent-inspect.suite.js",
|
|
2974
|
+
"agent-inspect.suite.mjs",
|
|
2975
|
+
"agent-inspect.suite.cjs"
|
|
2976
|
+
];
|
|
2977
|
+
function diagnostic2(code, message, severity = "error", caseId) {
|
|
2978
|
+
return { code, message, severity, ...{} };
|
|
3031
2979
|
}
|
|
3032
|
-
function
|
|
3033
|
-
return
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
const llms = finishedEvents(context, "LLM");
|
|
3039
|
-
const findings = [];
|
|
3040
|
-
const allowedModels = options.allowedModels?.length ? new Set(options.allowedModels) : void 0;
|
|
3041
|
-
const allowedProviders = options.allowedProviders?.length ? new Set(options.allowedProviders) : void 0;
|
|
3042
|
-
const finishReasons = options.finishReasons?.length ? new Set(options.finishReasons) : void 0;
|
|
3043
|
-
if (options.maxCalls !== void 0 && llms.length > options.maxCalls) {
|
|
3044
|
-
findings.push(
|
|
3045
|
-
failFinding(
|
|
3046
|
-
"llm.usage",
|
|
3047
|
-
`LLM call count ${llms.length} exceeded ${options.maxCalls}.`,
|
|
3048
|
-
llms.map((event) => eventEvidence(event)),
|
|
3049
|
-
{ maxCalls: options.maxCalls },
|
|
3050
|
-
llms.length
|
|
3051
|
-
)
|
|
3052
|
-
);
|
|
3053
|
-
}
|
|
3054
|
-
for (const event of llms) {
|
|
3055
|
-
const model = llmModel(event);
|
|
3056
|
-
const provider = llmProvider(event);
|
|
3057
|
-
const finishReason = llmFinishReason(event);
|
|
3058
|
-
if (allowedModels && (!model || !allowedModels.has(model))) {
|
|
3059
|
-
findings.push(
|
|
3060
|
-
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.model")], [...allowedModels].sort(), model ?? "unknown")
|
|
3061
|
-
);
|
|
3062
|
-
}
|
|
3063
|
-
if (allowedProviders && (!provider || !allowedProviders.has(provider))) {
|
|
3064
|
-
findings.push(
|
|
3065
|
-
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.provider")], [...allowedProviders].sort(), provider ?? "unknown")
|
|
3066
|
-
);
|
|
3067
|
-
}
|
|
3068
|
-
if (finishReasons && (!finishReason || !finishReasons.has(finishReason))) {
|
|
3069
|
-
findings.push(
|
|
3070
|
-
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.finishReason")], [...finishReasons].sort(), finishReason ?? "unknown")
|
|
3071
|
-
);
|
|
3072
|
-
}
|
|
3073
|
-
}
|
|
3074
|
-
const tokenTotals = llms.reduce(
|
|
3075
|
-
(totals, event) => ({
|
|
3076
|
-
input: totals.input + (event.tokenUsage?.input ?? 0),
|
|
3077
|
-
output: totals.output + (event.tokenUsage?.output ?? 0),
|
|
3078
|
-
total: totals.total + (event.tokenUsage?.total ?? 0),
|
|
3079
|
-
cached: totals.cached + (event.tokenUsage?.cached ?? 0)
|
|
3080
|
-
}),
|
|
3081
|
-
{ input: 0, output: 0, total: 0, cached: 0 }
|
|
3082
|
-
);
|
|
3083
|
-
const tokenLimits = [
|
|
3084
|
-
["input", options.maxInputTokens],
|
|
3085
|
-
["output", options.maxOutputTokens],
|
|
3086
|
-
["total", options.maxTotalTokens],
|
|
3087
|
-
["cached", options.maxCachedTokens]
|
|
3088
|
-
];
|
|
3089
|
-
for (const [key, limit] of tokenLimits) {
|
|
3090
|
-
if (limit !== void 0 && tokenTotals[key] > limit) {
|
|
3091
|
-
findings.push(
|
|
3092
|
-
failFinding(
|
|
3093
|
-
"llm.usage",
|
|
3094
|
-
`LLM ${key} token count ${tokenTotals[key]} exceeded ${limit}.`,
|
|
3095
|
-
llms.map((event) => eventEvidence(event, `tokenUsage.${key}`)),
|
|
3096
|
-
{ [`max${key[0].toUpperCase()}${key.slice(1)}Tokens`]: limit },
|
|
3097
|
-
tokenTotals[key]
|
|
3098
|
-
)
|
|
3099
|
-
);
|
|
3100
|
-
}
|
|
3101
|
-
}
|
|
3102
|
-
return findings;
|
|
3103
|
-
}
|
|
3104
|
-
};
|
|
2980
|
+
function asString(value, label) {
|
|
2981
|
+
if (value === void 0) return void 0;
|
|
2982
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
2983
|
+
throw new Error(`${label} must be a non-empty string.`);
|
|
2984
|
+
}
|
|
2985
|
+
return value.trim();
|
|
3105
2986
|
}
|
|
3106
|
-
function
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
kind: "OUTCOME",
|
|
3137
|
-
name: outcome.name,
|
|
3138
|
-
status: outcome.status,
|
|
3139
|
-
path: `outcome.${outcome.name}`
|
|
3140
|
-
})),
|
|
3141
|
-
{ failOn },
|
|
3142
|
-
matching.map((outcome) => ({
|
|
3143
|
-
name: outcome.name,
|
|
3144
|
-
status: outcome.status,
|
|
3145
|
-
expectation: outcome.expectation
|
|
3146
|
-
}))
|
|
3147
|
-
)
|
|
3148
|
-
];
|
|
2987
|
+
function asStringArray(value, label) {
|
|
2988
|
+
if (value === void 0) return void 0;
|
|
2989
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
2990
|
+
throw new Error(`${label} must be an array of strings.`);
|
|
2991
|
+
}
|
|
2992
|
+
return value;
|
|
2993
|
+
}
|
|
2994
|
+
function asPositiveNumber(value, label) {
|
|
2995
|
+
if (value === void 0) return void 0;
|
|
2996
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
2997
|
+
throw new Error(`${label} must be a non-negative number.`);
|
|
2998
|
+
}
|
|
2999
|
+
return value;
|
|
3000
|
+
}
|
|
3001
|
+
function validateCaseConfig(value, index) {
|
|
3002
|
+
const diagnostics = [];
|
|
3003
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
3004
|
+
diagnostics.push(
|
|
3005
|
+
diagnostic2("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
|
|
3006
|
+
);
|
|
3007
|
+
return { diagnostics };
|
|
3008
|
+
}
|
|
3009
|
+
const raw = value;
|
|
3010
|
+
try {
|
|
3011
|
+
const id = asString(raw.id, `cases[${index}].id`);
|
|
3012
|
+
if (id === void 0) {
|
|
3013
|
+
diagnostics.push(
|
|
3014
|
+
diagnostic2("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
|
|
3015
|
+
);
|
|
3016
|
+
return { diagnostics };
|
|
3149
3017
|
}
|
|
3150
|
-
|
|
3018
|
+
const trace = asString(raw.trace, `cases[${index}].trace`);
|
|
3019
|
+
const runId = asString(raw.runId, `cases[${index}].runId`);
|
|
3020
|
+
const input = asString(raw.input, `cases[${index}].input`);
|
|
3021
|
+
return {
|
|
3022
|
+
caseConfig: {
|
|
3023
|
+
id,
|
|
3024
|
+
...trace !== void 0 ? { trace } : {},
|
|
3025
|
+
...runId !== void 0 ? { runId } : {},
|
|
3026
|
+
...input !== void 0 ? { input } : {},
|
|
3027
|
+
...asStringArray(raw.requireTools, `cases[${index}].requireTools`) !== void 0 ? { requireTools: asStringArray(raw.requireTools, `cases[${index}].requireTools`) } : {},
|
|
3028
|
+
...asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) !== void 0 ? { forbidTools: asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) } : {},
|
|
3029
|
+
...asPositiveNumber(raw.maxDurationMs, `cases[${index}].maxDurationMs`) !== void 0 ? {
|
|
3030
|
+
maxDurationMs: asPositiveNumber(
|
|
3031
|
+
raw.maxDurationMs,
|
|
3032
|
+
`cases[${index}].maxDurationMs`
|
|
3033
|
+
)
|
|
3034
|
+
} : {},
|
|
3035
|
+
...asStringArray(
|
|
3036
|
+
raw.expectedObservations,
|
|
3037
|
+
`cases[${index}].expectedObservations`
|
|
3038
|
+
) !== void 0 ? {
|
|
3039
|
+
expectedObservations: asStringArray(
|
|
3040
|
+
raw.expectedObservations,
|
|
3041
|
+
`cases[${index}].expectedObservations`
|
|
3042
|
+
)
|
|
3043
|
+
} : {}
|
|
3044
|
+
},
|
|
3045
|
+
diagnostics
|
|
3046
|
+
};
|
|
3047
|
+
} catch (error) {
|
|
3048
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3049
|
+
diagnostics.push(diagnostic2("AI_SUITE_CONFIG_INVALID", message));
|
|
3050
|
+
return { diagnostics };
|
|
3051
|
+
}
|
|
3151
3052
|
}
|
|
3152
|
-
function
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
return errorResult(input, selected.diagnostics, selected.run);
|
|
3053
|
+
function normalizeSuiteConfig(value) {
|
|
3054
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
3055
|
+
throw new Error("Suite config must export an object.");
|
|
3156
3056
|
}
|
|
3157
|
-
const
|
|
3158
|
-
|
|
3159
|
-
|
|
3057
|
+
const raw = value;
|
|
3058
|
+
const name = asString(raw.name, "name");
|
|
3059
|
+
const traces = asString(raw.traces, "traces");
|
|
3060
|
+
if (name === void 0) throw new Error("name is required.");
|
|
3061
|
+
if (traces === void 0) throw new Error("traces is required.");
|
|
3062
|
+
if (!Array.isArray(raw.cases) || raw.cases.length === 0) {
|
|
3063
|
+
throw new Error("cases must be a non-empty array.");
|
|
3160
3064
|
}
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
|
|
3065
|
+
const cases = [];
|
|
3066
|
+
for (let index = 0; index < raw.cases.length; index += 1) {
|
|
3067
|
+
const { caseConfig, diagnostics } = validateCaseConfig(raw.cases[index], index);
|
|
3068
|
+
if (diagnostics.length > 0) {
|
|
3069
|
+
throw new Error(diagnostics.map((item) => item.message).join("; "));
|
|
3070
|
+
}
|
|
3071
|
+
if (caseConfig !== void 0) cases.push(caseConfig);
|
|
3072
|
+
}
|
|
3073
|
+
const ids = /* @__PURE__ */ new Set();
|
|
3074
|
+
for (const suiteCase of cases) {
|
|
3075
|
+
if (ids.has(suiteCase.id)) {
|
|
3076
|
+
throw new Error(`Duplicate case id "${suiteCase.id}".`);
|
|
3077
|
+
}
|
|
3078
|
+
ids.add(suiteCase.id);
|
|
3079
|
+
}
|
|
3080
|
+
const redactionProfile = raw.redactionProfile === "local" || raw.redactionProfile === "share" || raw.redactionProfile === "strict" ? raw.redactionProfile : void 0;
|
|
3081
|
+
if (raw.redactionProfile !== void 0 && redactionProfile === void 0) {
|
|
3082
|
+
throw new Error('redactionProfile must be "local", "share", or "strict".');
|
|
3083
|
+
}
|
|
3084
|
+
const config = { name, traces, cases };
|
|
3085
|
+
if (raw.checks !== void 0) {
|
|
3086
|
+
if (typeof raw.checks !== "object" || Array.isArray(raw.checks)) {
|
|
3087
|
+
throw new Error("checks must be an object.");
|
|
3088
|
+
}
|
|
3089
|
+
config.checks = raw.checks;
|
|
3090
|
+
}
|
|
3091
|
+
if (raw.eval !== void 0) {
|
|
3092
|
+
if (typeof raw.eval !== "object" || Array.isArray(raw.eval)) {
|
|
3093
|
+
throw new Error("eval must be an object.");
|
|
3094
|
+
}
|
|
3095
|
+
config.eval = raw.eval;
|
|
3096
|
+
}
|
|
3097
|
+
if (raw.artifacts !== void 0) {
|
|
3098
|
+
if (typeof raw.artifacts !== "object" || Array.isArray(raw.artifacts)) {
|
|
3099
|
+
throw new Error("artifacts must be an object.");
|
|
3100
|
+
}
|
|
3101
|
+
const outputDir = asString(
|
|
3102
|
+
raw.artifacts.outputDir,
|
|
3103
|
+
"artifacts.outputDir"
|
|
3171
3104
|
);
|
|
3105
|
+
config.artifacts = outputDir !== void 0 ? { outputDir } : {};
|
|
3172
3106
|
}
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3107
|
+
if (raw.baseline !== void 0) {
|
|
3108
|
+
const baseline = asString(raw.baseline, "baseline");
|
|
3109
|
+
if (baseline !== void 0) config.baseline = baseline;
|
|
3110
|
+
}
|
|
3111
|
+
if (raw.candidate !== void 0) {
|
|
3112
|
+
const candidate = asString(raw.candidate, "candidate");
|
|
3113
|
+
if (candidate !== void 0) config.candidate = candidate;
|
|
3114
|
+
}
|
|
3115
|
+
if (redactionProfile !== void 0) config.redactionProfile = redactionProfile;
|
|
3116
|
+
return config;
|
|
3117
|
+
}
|
|
3118
|
+
|
|
3119
|
+
// packages/core/src/suite/load.ts
|
|
3120
|
+
var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
3121
|
+
var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
3122
|
+
function diagnostic3(code, message) {
|
|
3123
|
+
return { code, message, severity: "error" };
|
|
3124
|
+
}
|
|
3125
|
+
async function fileExists(filePath) {
|
|
3126
|
+
try {
|
|
3127
|
+
await access(filePath);
|
|
3128
|
+
return true;
|
|
3129
|
+
} catch {
|
|
3130
|
+
return false;
|
|
3131
|
+
}
|
|
3132
|
+
}
|
|
3133
|
+
async function resolveSuiteConfigPath(options = {}) {
|
|
3134
|
+
const cwd = path15.resolve(options.cwd ?? process.cwd());
|
|
3135
|
+
if (options.configPath !== void 0 && options.configPath.trim() !== "") {
|
|
3136
|
+
return path15.resolve(cwd, options.configPath.trim());
|
|
3137
|
+
}
|
|
3138
|
+
for (const name of DEFAULT_SUITE_CONFIG_NAMES) {
|
|
3139
|
+
const candidate = path15.join(cwd, name);
|
|
3140
|
+
if (await fileExists(candidate)) return candidate;
|
|
3141
|
+
}
|
|
3142
|
+
throw new Error(
|
|
3143
|
+
`No suite config found. Create one with \`agent-inspect suite init\` or pass --config.`
|
|
3144
|
+
);
|
|
3145
|
+
}
|
|
3146
|
+
async function loadSuiteConfig(options = {}) {
|
|
3147
|
+
let configPath;
|
|
3148
|
+
try {
|
|
3149
|
+
configPath = await resolveSuiteConfigPath(options);
|
|
3150
|
+
} catch (error) {
|
|
3151
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3152
|
+
throw Object.assign(new Error(message), {
|
|
3153
|
+
diagnostics: [diagnostic3("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
3154
|
+
});
|
|
3155
|
+
}
|
|
3156
|
+
const extension = path15.extname(configPath);
|
|
3157
|
+
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
3158
|
+
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
3159
|
+
throw Object.assign(new Error(message), {
|
|
3160
|
+
diagnostics: [diagnostic3("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
3161
|
+
});
|
|
3162
|
+
}
|
|
3163
|
+
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
3164
|
+
const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
|
|
3165
|
+
throw Object.assign(new Error(message), {
|
|
3166
|
+
diagnostics: [diagnostic3("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
3167
|
+
});
|
|
3168
|
+
}
|
|
3169
|
+
try {
|
|
3170
|
+
let raw;
|
|
3171
|
+
if (extension === ".json") {
|
|
3172
|
+
raw = JSON.parse(await readFile(configPath, "utf-8"));
|
|
3173
|
+
} else {
|
|
3174
|
+
const mod = await import(pathToFileURL(configPath).href);
|
|
3175
|
+
raw = "default" in mod ? mod.default : mod;
|
|
3205
3176
|
}
|
|
3177
|
+
const config = normalizeSuiteConfig(raw);
|
|
3178
|
+
return {
|
|
3179
|
+
config,
|
|
3180
|
+
configPath,
|
|
3181
|
+
configDir: path15.dirname(configPath)
|
|
3182
|
+
};
|
|
3183
|
+
} catch (error) {
|
|
3184
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3185
|
+
throw Object.assign(new Error(message), {
|
|
3186
|
+
diagnostics: [diagnostic3("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
3187
|
+
});
|
|
3206
3188
|
}
|
|
3207
|
-
|
|
3208
|
-
|
|
3189
|
+
}
|
|
3190
|
+
async function exists(filePath) {
|
|
3191
|
+
try {
|
|
3192
|
+
await access(filePath);
|
|
3193
|
+
return true;
|
|
3194
|
+
} catch {
|
|
3195
|
+
return false;
|
|
3196
|
+
}
|
|
3197
|
+
}
|
|
3198
|
+
async function resolveSuiteCaseTrace(suiteCase, options) {
|
|
3199
|
+
if (suiteCase.trace !== void 0) {
|
|
3200
|
+
const tracePath = path15.resolve(options.configDir, suiteCase.trace);
|
|
3201
|
+
if (await exists(tracePath)) {
|
|
3202
|
+
return { caseId: suiteCase.id, tracePath, missing: false };
|
|
3203
|
+
}
|
|
3204
|
+
return {
|
|
3205
|
+
caseId: suiteCase.id,
|
|
3206
|
+
tracePath,
|
|
3207
|
+
missing: true,
|
|
3208
|
+
reason: `trace file not found: ${suiteCase.trace}`
|
|
3209
|
+
};
|
|
3210
|
+
}
|
|
3211
|
+
const runKey = suiteCase.runId ?? suiteCase.id;
|
|
3212
|
+
const directPath = getTraceFilePath(runKey, options.tracesDir);
|
|
3213
|
+
if (await exists(directPath)) {
|
|
3214
|
+
return { caseId: suiteCase.id, tracePath: directPath, runId: runKey, missing: false };
|
|
3215
|
+
}
|
|
3216
|
+
const nestedPath = path15.join(options.tracesDir, `${path15.basename(runKey)}.jsonl`);
|
|
3217
|
+
if (await exists(nestedPath)) {
|
|
3218
|
+
return { caseId: suiteCase.id, tracePath: nestedPath, runId: runKey, missing: false };
|
|
3209
3219
|
}
|
|
3210
|
-
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
3211
|
-
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
3212
|
-
const summary = summarize(sortedFindings, diagnostics, ruleExecutions.length);
|
|
3213
|
-
const status = summary.failed > 0 ? "fail" : "pass";
|
|
3214
3220
|
return {
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
summary,
|
|
3220
|
-
findings: sortedFindings,
|
|
3221
|
-
diagnostics,
|
|
3222
|
-
ruleExecutions
|
|
3221
|
+
caseId: suiteCase.id,
|
|
3222
|
+
runId: runKey,
|
|
3223
|
+
missing: true,
|
|
3224
|
+
reason: `no trace found for run id "${runKey}" under ${options.tracesDir}`
|
|
3223
3225
|
};
|
|
3224
3226
|
}
|
|
3225
3227
|
|
|
@@ -3236,16 +3238,20 @@ function normalizeTokenUsage(value) {
|
|
|
3236
3238
|
const output = nonNegativeFinite(value.output);
|
|
3237
3239
|
const suppliedTotal = nonNegativeFinite(value.total);
|
|
3238
3240
|
const cached = nonNegativeFinite(value.cached);
|
|
3241
|
+
const cacheWrite = nonNegativeFinite(value.cacheWrite);
|
|
3242
|
+
const reasoning = nonNegativeFinite(value.reasoning);
|
|
3239
3243
|
const derivedTotal = input !== void 0 && output !== void 0 && Number.isFinite(input + output) ? input + output : void 0;
|
|
3240
3244
|
const total = suppliedTotal ?? derivedTotal;
|
|
3241
|
-
if (input === void 0 && output === void 0 && total === void 0 && cached === void 0) {
|
|
3245
|
+
if (input === void 0 && output === void 0 && total === void 0 && cached === void 0 && cacheWrite === void 0 && reasoning === void 0) {
|
|
3242
3246
|
return void 0;
|
|
3243
3247
|
}
|
|
3244
3248
|
return {
|
|
3245
3249
|
...input !== void 0 ? { input } : {},
|
|
3246
3250
|
...output !== void 0 ? { output } : {},
|
|
3247
3251
|
...total !== void 0 ? { total } : {},
|
|
3248
|
-
...cached !== void 0 ? { cached } : {}
|
|
3252
|
+
...cached !== void 0 ? { cached } : {},
|
|
3253
|
+
...cacheWrite !== void 0 ? { cacheWrite } : {},
|
|
3254
|
+
...reasoning !== void 0 ? { reasoning } : {}
|
|
3249
3255
|
};
|
|
3250
3256
|
}
|
|
3251
3257
|
|
|
@@ -4404,6 +4410,8 @@ function readOpenInferenceTokenUsage(attributes) {
|
|
|
4404
4410
|
const completion = attributes["llm.token_count.completion"];
|
|
4405
4411
|
const total = attributes["llm.token_count.total"];
|
|
4406
4412
|
const cached = attributes["llm.token_count.prompt_details.cache_read"];
|
|
4413
|
+
const cacheWrite = attributes["llm.token_count.prompt_details.cache_write"] ?? attributes["llm.token_count.prompt_details.cache_creation"];
|
|
4414
|
+
const reasoning = attributes["llm.token_count.completion_details.reasoning"] ?? attributes["llm.token_count.completion_details.reasoning_tokens"];
|
|
4407
4415
|
const usage = {};
|
|
4408
4416
|
if (typeof prompt === "number" && Number.isFinite(prompt) && prompt >= 0) {
|
|
4409
4417
|
usage.input = prompt;
|
|
@@ -4417,6 +4425,12 @@ function readOpenInferenceTokenUsage(attributes) {
|
|
|
4417
4425
|
if (typeof cached === "number" && Number.isFinite(cached) && cached >= 0) {
|
|
4418
4426
|
usage.cached = cached;
|
|
4419
4427
|
}
|
|
4428
|
+
if (typeof cacheWrite === "number" && Number.isFinite(cacheWrite) && cacheWrite >= 0) {
|
|
4429
|
+
usage.cacheWrite = cacheWrite;
|
|
4430
|
+
}
|
|
4431
|
+
if (typeof reasoning === "number" && Number.isFinite(reasoning) && reasoning >= 0) {
|
|
4432
|
+
usage.reasoning = reasoning;
|
|
4433
|
+
}
|
|
4420
4434
|
if (usage.total === void 0 && usage.input !== void 0 && usage.output !== void 0) {
|
|
4421
4435
|
usage.total = usage.input + usage.output;
|
|
4422
4436
|
}
|