agent-inspect 6.12.1 → 6.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +1 -1
- package/docs/KNOWN-ISSUES.md +1 -0
- package/package.json +2 -2
- package/packages/cli/dist/{chunk-XTAA733P.mjs → chunk-QOKTMZAY.mjs} +321 -55
- package/packages/cli/dist/chunk-QOKTMZAY.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +350 -70
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +4 -4
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-G7PM24W2.mjs → src-MU4PS6GO.mjs} +3 -3
- package/packages/cli/dist/{src-G7PM24W2.mjs.map → src-MU4PS6GO.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +295 -30
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +1 -1
- package/packages/core/dist/advanced.d.ts +1 -1
- package/packages/core/dist/advanced.mjs +1 -1
- package/packages/core/dist/checks.cjs +368 -32
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +58 -4
- package/packages/core/dist/checks.d.ts +58 -4
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-QT6CQ2XA.mjs → chunk-BS2QCAJX.mjs} +367 -35
- package/packages/core/dist/chunk-BS2QCAJX.mjs.map +1 -0
- package/packages/core/dist/{index-BO7l0iAe.d.ts → index-Bu_vOjql.d.ts} +72 -1
- package/packages/core/dist/{index-mdFcxSOR.d.cts → index-CmXMS-MF.d.cts} +72 -1
- package/packages/cli/dist/chunk-XTAA733P.mjs.map +0 -1
- package/packages/core/dist/chunk-QT6CQ2XA.mjs.map +0 -1
|
@@ -1,5 +1,320 @@
|
|
|
1
1
|
import { extractOutcomesFromPersistedEvents, outcomesMatchingStatus } from './chunk-VFO76UH3.mjs';
|
|
2
2
|
|
|
3
|
+
// packages/core/src/checks/logical-events.ts
|
|
4
|
+
function isRecord(value) {
|
|
5
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
6
|
+
}
|
|
7
|
+
function legacyEvent(event) {
|
|
8
|
+
const value = event.attributes?.legacyEvent;
|
|
9
|
+
return typeof value === "string" ? value : void 0;
|
|
10
|
+
}
|
|
11
|
+
function stepIdOf(event) {
|
|
12
|
+
const value = event.attributes?.stepId;
|
|
13
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
14
|
+
return void 0;
|
|
15
|
+
}
|
|
16
|
+
function cloneEvent(event) {
|
|
17
|
+
return {
|
|
18
|
+
...event,
|
|
19
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
20
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
21
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
22
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
function mergeAttributes(start, complete) {
|
|
26
|
+
const merged = {
|
|
27
|
+
...isRecord(complete.attributes) ? complete.attributes : {},
|
|
28
|
+
...isRecord(start.attributes) ? start.attributes : {}
|
|
29
|
+
};
|
|
30
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
31
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
32
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
33
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
34
|
+
}
|
|
35
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
36
|
+
}
|
|
37
|
+
function pairStartComplete(start, complete) {
|
|
38
|
+
const attributes = mergeAttributes(start, complete);
|
|
39
|
+
const paired = {
|
|
40
|
+
...cloneEvent(start),
|
|
41
|
+
status: complete.status,
|
|
42
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
43
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
44
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
45
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
46
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
47
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
48
|
+
};
|
|
49
|
+
return {
|
|
50
|
+
...paired,
|
|
51
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
52
|
+
projection: {
|
|
53
|
+
paired: true,
|
|
54
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
55
|
+
parentNormalized: false,
|
|
56
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
function asLogical(event, extras) {
|
|
61
|
+
return {
|
|
62
|
+
...cloneEvent(event),
|
|
63
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
64
|
+
projection: {
|
|
65
|
+
paired: false,
|
|
66
|
+
absorbedEventIds: Object.freeze([]),
|
|
67
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
68
|
+
...{}
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
function projectLogicalEvents(events) {
|
|
73
|
+
const diagnostics = [];
|
|
74
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
75
|
+
for (const event of events) {
|
|
76
|
+
const list = byRun.get(event.runId) ?? [];
|
|
77
|
+
list.push(event);
|
|
78
|
+
byRun.set(event.runId, list);
|
|
79
|
+
}
|
|
80
|
+
const absorbedIds = /* @__PURE__ */ new Set();
|
|
81
|
+
const logicalByRawId = /* @__PURE__ */ new Map();
|
|
82
|
+
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
83
|
+
const logical = [];
|
|
84
|
+
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
85
|
+
for (const runId of orderedRuns) {
|
|
86
|
+
const runEvents = byRun.get(runId) ?? [];
|
|
87
|
+
const starts = [];
|
|
88
|
+
const completes = [];
|
|
89
|
+
const others = [];
|
|
90
|
+
for (const event of runEvents) {
|
|
91
|
+
const legacy = legacyEvent(event);
|
|
92
|
+
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
93
|
+
starts.push(event);
|
|
94
|
+
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
95
|
+
completes.push(event);
|
|
96
|
+
} else {
|
|
97
|
+
others.push(event);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const usedCompletes = /* @__PURE__ */ new Set();
|
|
101
|
+
for (const start of starts) {
|
|
102
|
+
const stepId = stepIdOf(start);
|
|
103
|
+
let match;
|
|
104
|
+
if (legacyEvent(start) === "run_started") {
|
|
105
|
+
const candidates = completes.filter(
|
|
106
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
107
|
+
);
|
|
108
|
+
if (candidates.length > 1) {
|
|
109
|
+
diagnostics.push({
|
|
110
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
111
|
+
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
112
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
113
|
+
});
|
|
114
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
115
|
+
}
|
|
116
|
+
match = candidates[0];
|
|
117
|
+
} else if (stepId) {
|
|
118
|
+
const candidates = completes.filter(
|
|
119
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
120
|
+
);
|
|
121
|
+
if (candidates.length > 1) {
|
|
122
|
+
diagnostics.push({
|
|
123
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
124
|
+
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
125
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
126
|
+
});
|
|
127
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
128
|
+
}
|
|
129
|
+
match = candidates[0];
|
|
130
|
+
}
|
|
131
|
+
if (match) {
|
|
132
|
+
usedCompletes.add(match.eventId);
|
|
133
|
+
absorbedIds.add(match.eventId);
|
|
134
|
+
const paired = pairStartComplete(start, match);
|
|
135
|
+
logical.push(paired);
|
|
136
|
+
logicalByRawId.set(start.eventId, paired);
|
|
137
|
+
logicalByRawId.set(match.eventId, paired);
|
|
138
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
139
|
+
} else {
|
|
140
|
+
diagnostics.push({
|
|
141
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
142
|
+
message: `No matching complete for start ${start.eventId}.`,
|
|
143
|
+
eventIds: [start.eventId]
|
|
144
|
+
});
|
|
145
|
+
const alone = asLogical(start);
|
|
146
|
+
logical.push(alone);
|
|
147
|
+
logicalByRawId.set(start.eventId, alone);
|
|
148
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
for (const complete of completes) {
|
|
152
|
+
if (usedCompletes.has(complete.eventId)) continue;
|
|
153
|
+
diagnostics.push({
|
|
154
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
155
|
+
message: `No matching start for complete ${complete.eventId}.`,
|
|
156
|
+
eventIds: [complete.eventId]
|
|
157
|
+
});
|
|
158
|
+
const alone = asLogical(complete);
|
|
159
|
+
logical.push(alone);
|
|
160
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
161
|
+
const stepId = stepIdOf(complete);
|
|
162
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
163
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
for (const event of others) {
|
|
167
|
+
const alone = asLogical(event);
|
|
168
|
+
logical.push(alone);
|
|
169
|
+
logicalByRawId.set(event.eventId, alone);
|
|
170
|
+
const stepId = stepIdOf(event);
|
|
171
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
172
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
177
|
+
const normalized = [];
|
|
178
|
+
for (const event of logical) {
|
|
179
|
+
const originalParentId = event.parentId;
|
|
180
|
+
if (!originalParentId) {
|
|
181
|
+
normalized.push(event);
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
let nextParent = originalParentId;
|
|
185
|
+
let remapped = false;
|
|
186
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
187
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
188
|
+
nextParent = viaAbsorbed.eventId;
|
|
189
|
+
remapped = true;
|
|
190
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
191
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
192
|
+
if (viaStep) {
|
|
193
|
+
nextParent = viaStep;
|
|
194
|
+
remapped = true;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
if (!remapped) {
|
|
198
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
199
|
+
const mapping = event.attributes?.parentMapping;
|
|
200
|
+
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
201
|
+
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
202
|
+
if (!unresolved) {
|
|
203
|
+
diagnostics.push({
|
|
204
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
205
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
206
|
+
eventIds: [event.eventId]
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
normalized.push(event);
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
diagnostics.push({
|
|
214
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
215
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
216
|
+
eventIds: [event.eventId]
|
|
217
|
+
});
|
|
218
|
+
normalized.push({
|
|
219
|
+
...event,
|
|
220
|
+
parentId: nextParent,
|
|
221
|
+
projection: {
|
|
222
|
+
...event.projection,
|
|
223
|
+
parentNormalized: true,
|
|
224
|
+
originalParentId
|
|
225
|
+
}
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
229
|
+
normalized.sort((a, b) => {
|
|
230
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
231
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
232
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
233
|
+
});
|
|
234
|
+
return {
|
|
235
|
+
logicalEvents: Object.freeze(normalized),
|
|
236
|
+
diagnostics: Object.freeze(diagnostics)
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
function resolveCanonicalToolName(event) {
|
|
240
|
+
const attrs = event.attributes;
|
|
241
|
+
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
242
|
+
if (direct) return direct;
|
|
243
|
+
const metadata = attrs?.metadata;
|
|
244
|
+
if (isRecord(metadata)) {
|
|
245
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
246
|
+
if (nested) return nested;
|
|
247
|
+
}
|
|
248
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
249
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
250
|
+
}
|
|
251
|
+
return event.name;
|
|
252
|
+
}
|
|
253
|
+
function pickString(record, keys) {
|
|
254
|
+
if (!record) return void 0;
|
|
255
|
+
for (const key of keys) {
|
|
256
|
+
const value = record[key];
|
|
257
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
258
|
+
}
|
|
259
|
+
return void 0;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// packages/core/src/checks/trace-facts.ts
|
|
263
|
+
function summarizeSemanticParity(events) {
|
|
264
|
+
const projection = projectLogicalEvents(events);
|
|
265
|
+
const logical = projection.logicalEvents;
|
|
266
|
+
const finishedTools = logical.filter(
|
|
267
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
268
|
+
);
|
|
269
|
+
const finishedToolNames = Object.freeze(
|
|
270
|
+
finishedTools.map((event) => resolveCanonicalToolName(event)).sort((a, b) => a.localeCompare(b))
|
|
271
|
+
);
|
|
272
|
+
return {
|
|
273
|
+
rawEventCount: events.length,
|
|
274
|
+
logicalEventCount: logical.length,
|
|
275
|
+
runningLogicalCount: logical.filter((event) => event.status === "running").length,
|
|
276
|
+
finishedToolNames,
|
|
277
|
+
finishedToolCount: finishedTools.length,
|
|
278
|
+
pairedCount: logical.filter((event) => event.projection.paired).length,
|
|
279
|
+
parentRemapCount: projection.diagnostics.filter(
|
|
280
|
+
(item) => item.code === "AI_LOGICAL_PARENT_REMAPPED"
|
|
281
|
+
).length,
|
|
282
|
+
diagnostics: projection.diagnostics
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
function buildTraceFacts(events) {
|
|
286
|
+
const projection = projectLogicalEvents(events);
|
|
287
|
+
const toolsByName = /* @__PURE__ */ new Map();
|
|
288
|
+
const llmEvents = [];
|
|
289
|
+
const outcomeEvents = [];
|
|
290
|
+
for (const event of projection.logicalEvents) {
|
|
291
|
+
if (event.kind === "TOOL" && event.status !== "running") {
|
|
292
|
+
const name = resolveCanonicalToolName(event);
|
|
293
|
+
const list = toolsByName.get(name) ?? [];
|
|
294
|
+
list.push(event);
|
|
295
|
+
toolsByName.set(name, list);
|
|
296
|
+
}
|
|
297
|
+
if (event.kind === "LLM" && event.status !== "running") {
|
|
298
|
+
llmEvents.push(event);
|
|
299
|
+
}
|
|
300
|
+
if (event.kind === "OUTCOME") {
|
|
301
|
+
outcomeEvents.push(event);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
for (const [name, list] of [...toolsByName.entries()]) {
|
|
305
|
+
toolsByName.set(name, Object.freeze([...list]));
|
|
306
|
+
}
|
|
307
|
+
return {
|
|
308
|
+
rawEvents: Object.freeze([...events]),
|
|
309
|
+
logicalEvents: projection.logicalEvents,
|
|
310
|
+
diagnostics: projection.diagnostics,
|
|
311
|
+
toolsByName,
|
|
312
|
+
llmEvents: Object.freeze(llmEvents),
|
|
313
|
+
outcomeEvents: Object.freeze(outcomeEvents),
|
|
314
|
+
summary: summarizeSemanticParity(events)
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
|
|
3
318
|
// packages/core/src/checks/index.ts
|
|
4
319
|
var SEVERITY_RANK = {
|
|
5
320
|
error: 0,
|
|
@@ -58,7 +373,11 @@ var DEFAULT_RAW_CONTENT_KEYS = [
|
|
|
58
373
|
"conversationtext",
|
|
59
374
|
"conversation_text"
|
|
60
375
|
];
|
|
61
|
-
var DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES = [
|
|
376
|
+
var DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES = [
|
|
377
|
+
"tokenUsage",
|
|
378
|
+
"usage",
|
|
379
|
+
"tokens"
|
|
380
|
+
];
|
|
62
381
|
var SAFE_USAGE_LEAF_KEYS = /* @__PURE__ */ new Set([
|
|
63
382
|
"input",
|
|
64
383
|
"output",
|
|
@@ -134,10 +453,13 @@ function buildFacts(input, selectedRun) {
|
|
|
134
453
|
childrenByParentId.set(parentId, children);
|
|
135
454
|
}
|
|
136
455
|
}
|
|
456
|
+
const projection = projectLogicalEvents(scopedEvents);
|
|
137
457
|
return {
|
|
138
458
|
format: input.read.format,
|
|
139
459
|
runs: Object.freeze([...input.read.runs]),
|
|
140
460
|
events: Object.freeze([...scopedEvents]),
|
|
461
|
+
logicalEvents: projection.logicalEvents,
|
|
462
|
+
logicalProjectionDiagnostics: projection.diagnostics,
|
|
141
463
|
readerWarnings: Object.freeze([...input.read.warnings]),
|
|
142
464
|
unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
|
|
143
465
|
sourceFiles: Object.freeze([...input.read.sourceFiles]),
|
|
@@ -330,7 +652,10 @@ function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
|
330
652
|
};
|
|
331
653
|
}
|
|
332
654
|
function toolName(event) {
|
|
333
|
-
return
|
|
655
|
+
return resolveCanonicalToolName(event);
|
|
656
|
+
}
|
|
657
|
+
function semanticEvents(context) {
|
|
658
|
+
return context.logicalEvents ?? context.events;
|
|
334
659
|
}
|
|
335
660
|
function llmModel(event) {
|
|
336
661
|
return stringAttr(event, ["model", "modelId", "responseModelId", "modelName", "model_name"]) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
@@ -345,7 +670,7 @@ function retryCount(event) {
|
|
|
345
670
|
return numericAttr(event, ["retryCount", "retryAttempt", "retry_attempt", "attempt"]);
|
|
346
671
|
}
|
|
347
672
|
function finishedEvents(context, kind) {
|
|
348
|
-
return context.
|
|
673
|
+
return semanticEvents(context).filter(
|
|
349
674
|
(event) => (kind === void 0 || event.kind === kind) && event.status !== "running"
|
|
350
675
|
);
|
|
351
676
|
}
|
|
@@ -357,7 +682,7 @@ function firstIndexByName(events) {
|
|
|
357
682
|
}
|
|
358
683
|
return index;
|
|
359
684
|
}
|
|
360
|
-
function
|
|
685
|
+
function isRecord2(value) {
|
|
361
686
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
362
687
|
}
|
|
363
688
|
function eventMap(events) {
|
|
@@ -411,7 +736,7 @@ function pushValueEntries(entries, event, value, path, key, depth = 0) {
|
|
|
411
736
|
}
|
|
412
737
|
return;
|
|
413
738
|
}
|
|
414
|
-
if (!
|
|
739
|
+
if (!isRecord2(value)) return;
|
|
415
740
|
for (const nestedKey of Object.keys(value).sort((a, b) => a.localeCompare(b))) {
|
|
416
741
|
pushValueEntries(
|
|
417
742
|
entries,
|
|
@@ -491,14 +816,11 @@ function signalName(event, attributeKeys, prefixes) {
|
|
|
491
816
|
return stringAttr(event, attributeKeys) ?? stripPrefix(event.name, prefixes);
|
|
492
817
|
}
|
|
493
818
|
function guardrailEvents(context) {
|
|
494
|
-
return
|
|
819
|
+
return finishedEvents(context).filter((event) => {
|
|
495
820
|
const name = event.name.toLowerCase();
|
|
496
821
|
if (name.startsWith("guardrail:") || name.includes(".guardrail.")) return true;
|
|
497
822
|
return stringAttr(event, ["guardrailName", "guardrail", "guardrailId"]) !== void 0;
|
|
498
823
|
});
|
|
499
|
-
function finishedEvents2() {
|
|
500
|
-
return context.events.filter((event) => event.status !== "running");
|
|
501
|
-
}
|
|
502
824
|
}
|
|
503
825
|
function retryValue(event) {
|
|
504
826
|
return retryCount(event) ?? 0;
|
|
@@ -519,7 +841,7 @@ function treeShape(nodes) {
|
|
|
519
841
|
return lines;
|
|
520
842
|
}
|
|
521
843
|
function statusShape(context) {
|
|
522
|
-
return context.
|
|
844
|
+
return semanticEvents(context).map((event) => `${event.kind}:${event.name}:${event.status ?? "unknown"}`).sort((a, b) => a.localeCompare(b));
|
|
523
845
|
}
|
|
524
846
|
function toolShape(context) {
|
|
525
847
|
return finishedEvents(context, "TOOL").map(
|
|
@@ -545,7 +867,7 @@ function llmShape(context) {
|
|
|
545
867
|
);
|
|
546
868
|
}
|
|
547
869
|
function errorShape(context) {
|
|
548
|
-
return context.
|
|
870
|
+
return semanticEvents(context).filter((event) => event.status === "error" || event.error !== void 0).map(
|
|
549
871
|
(event) => [
|
|
550
872
|
event.kind,
|
|
551
873
|
event.name,
|
|
@@ -563,7 +885,7 @@ function guardrailShape(context) {
|
|
|
563
885
|
return guardrailEvents(context).map((event) => signalName(event, ["guardrailName", "guardrail", "guardrailId"], ["guardrail:"])).sort((a, b) => a.localeCompare(b));
|
|
564
886
|
}
|
|
565
887
|
function firstEvidenceForKind(context, kind, path) {
|
|
566
|
-
const event = context.
|
|
888
|
+
const event = semanticEvents(context).find((candidate) => candidate.kind === kind);
|
|
567
889
|
return event ? [eventEvidence(event, path)] : runEvidence(context.selectedRun);
|
|
568
890
|
}
|
|
569
891
|
function baselineDiffFinding(message, evidence, expected, actual) {
|
|
@@ -591,7 +913,7 @@ function createRunStatusRule(options = {}) {
|
|
|
591
913
|
);
|
|
592
914
|
}
|
|
593
915
|
if (!allowIncomplete) {
|
|
594
|
-
const running = context.
|
|
916
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
595
917
|
if (running.length > 0) {
|
|
596
918
|
findings.push(
|
|
597
919
|
failFinding(
|
|
@@ -634,7 +956,7 @@ function createMaxStepDurationRule(options) {
|
|
|
634
956
|
category: "run",
|
|
635
957
|
defaultSeverity: "error",
|
|
636
958
|
evaluate(context) {
|
|
637
|
-
const over = context.
|
|
959
|
+
const over = semanticEvents(context).filter((event) => {
|
|
638
960
|
const duration = eventDurationMs(event);
|
|
639
961
|
return duration !== void 0 && duration > options.maxDurationMs;
|
|
640
962
|
});
|
|
@@ -663,7 +985,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
663
985
|
defaultSeverity: "warning",
|
|
664
986
|
evaluate(context) {
|
|
665
987
|
const findings = [];
|
|
666
|
-
const running = context.
|
|
988
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
667
989
|
if (running.length > 0) {
|
|
668
990
|
findings.push(
|
|
669
991
|
failFinding(
|
|
@@ -676,7 +998,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
676
998
|
);
|
|
677
999
|
}
|
|
678
1000
|
if (requireEndedAt) {
|
|
679
|
-
const incomplete = context.
|
|
1001
|
+
const incomplete = semanticEvents(context).filter(
|
|
680
1002
|
(event) => event.startedAt !== void 0 && event.endedAt === void 0 && event.status !== "running"
|
|
681
1003
|
);
|
|
682
1004
|
if (incomplete.length > 0) {
|
|
@@ -714,7 +1036,7 @@ function createRequireCompletedRule() {
|
|
|
714
1036
|
)
|
|
715
1037
|
);
|
|
716
1038
|
}
|
|
717
|
-
const running = context.
|
|
1039
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
718
1040
|
if (running.length > 0) {
|
|
719
1041
|
findings.push(
|
|
720
1042
|
failFinding(
|
|
@@ -736,7 +1058,7 @@ function createRunEventCountRule(options) {
|
|
|
736
1058
|
category: "run",
|
|
737
1059
|
defaultSeverity: "error",
|
|
738
1060
|
evaluate(context) {
|
|
739
|
-
const count = context.
|
|
1061
|
+
const count = semanticEvents(context).filter(
|
|
740
1062
|
(event) => options.kind === void 0 || event.kind === options.kind
|
|
741
1063
|
).length;
|
|
742
1064
|
const findings = [];
|
|
@@ -987,7 +1309,7 @@ function createStructureIncompleteRule(options = {}) {
|
|
|
987
1309
|
evaluate(context) {
|
|
988
1310
|
const findings = [];
|
|
989
1311
|
if (!options.allowRunning) {
|
|
990
|
-
const running = context.
|
|
1312
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
991
1313
|
if (running.length > 0) {
|
|
992
1314
|
findings.push(
|
|
993
1315
|
failFinding(
|
|
@@ -1001,7 +1323,7 @@ function createStructureIncompleteRule(options = {}) {
|
|
|
1001
1323
|
}
|
|
1002
1324
|
}
|
|
1003
1325
|
if (options.requireEndedAtForStarted) {
|
|
1004
|
-
const missingEndedAt = context.
|
|
1326
|
+
const missingEndedAt = semanticEvents(context).filter(
|
|
1005
1327
|
(event) => event.startedAt !== void 0 && event.endedAt === void 0 && event.status !== "running"
|
|
1006
1328
|
);
|
|
1007
1329
|
if (missingEndedAt.length > 0) {
|
|
@@ -1027,8 +1349,8 @@ function createStructureOrphanRule(options = {}) {
|
|
|
1027
1349
|
category: "structure",
|
|
1028
1350
|
defaultSeverity: "error",
|
|
1029
1351
|
evaluate(context) {
|
|
1030
|
-
const byId = eventMap(context
|
|
1031
|
-
const orphans = context.
|
|
1352
|
+
const byId = eventMap(semanticEvents(context));
|
|
1353
|
+
const orphans = semanticEvents(context).filter((event) => {
|
|
1032
1354
|
if (!event.parentId || byId.has(event.parentId)) return false;
|
|
1033
1355
|
return !(allowMarkedUnresolved && parentMarkedUnresolved(event));
|
|
1034
1356
|
});
|
|
@@ -1051,10 +1373,10 @@ function createStructureCycleRule() {
|
|
|
1051
1373
|
category: "structure",
|
|
1052
1374
|
defaultSeverity: "error",
|
|
1053
1375
|
evaluate(context) {
|
|
1054
|
-
const byId = eventMap(context
|
|
1376
|
+
const byId = eventMap(semanticEvents(context));
|
|
1055
1377
|
const seenCycles = /* @__PURE__ */ new Set();
|
|
1056
1378
|
const findings = [];
|
|
1057
|
-
for (const event of [...context
|
|
1379
|
+
for (const event of [...semanticEvents(context)].sort((a, b) => a.eventId.localeCompare(b.eventId))) {
|
|
1058
1380
|
const path = [];
|
|
1059
1381
|
const seenAt = /* @__PURE__ */ new Map();
|
|
1060
1382
|
let current = event;
|
|
@@ -1092,10 +1414,10 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
1092
1414
|
category: "structure",
|
|
1093
1415
|
defaultSeverity: "error",
|
|
1094
1416
|
evaluate(context) {
|
|
1095
|
-
const byId = eventMap(context
|
|
1417
|
+
const byId = eventMap(semanticEvents(context));
|
|
1096
1418
|
const findings = [];
|
|
1097
1419
|
const minConfidence = options.minConfidence;
|
|
1098
|
-
for (const event of context
|
|
1420
|
+
for (const event of semanticEvents(context)) {
|
|
1099
1421
|
if (minConfidence && CONFIDENCE_RANK[event.confidence] < CONFIDENCE_RANK[minConfidence]) {
|
|
1100
1422
|
findings.push(
|
|
1101
1423
|
failFinding(
|
|
@@ -1163,7 +1485,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
1163
1485
|
defaultSeverity: "error",
|
|
1164
1486
|
evaluate(context) {
|
|
1165
1487
|
const findings = [];
|
|
1166
|
-
const byId = eventMap(context
|
|
1488
|
+
const byId = eventMap(semanticEvents(context));
|
|
1167
1489
|
if (options.maxChildren !== void 0) {
|
|
1168
1490
|
for (const [parentId, children] of context.childrenByParentId.entries()) {
|
|
1169
1491
|
if (children.length <= options.maxChildren) continue;
|
|
@@ -1190,7 +1512,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
1190
1512
|
}
|
|
1191
1513
|
}
|
|
1192
1514
|
if (options.maxConcurrent !== void 0) {
|
|
1193
|
-
const intervals = context.
|
|
1515
|
+
const intervals = semanticEvents(context).map((event) => ({ event, start: eventStartMs(event), end: eventEndMs(event) })).filter(
|
|
1194
1516
|
(item) => item.start !== void 0 && item.end !== void 0 && item.end > item.start
|
|
1195
1517
|
);
|
|
1196
1518
|
const points = intervals.flatMap((item) => [
|
|
@@ -1486,7 +1808,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
1486
1808
|
)
|
|
1487
1809
|
);
|
|
1488
1810
|
}
|
|
1489
|
-
if (
|
|
1811
|
+
if (isRecord2(entry.value) && options.maxObjectKeys !== void 0 && Object.keys(entry.value).length > options.maxObjectKeys) {
|
|
1490
1812
|
findings.push(
|
|
1491
1813
|
failFinding(
|
|
1492
1814
|
"safety.oversizedAttribute",
|
|
@@ -1793,7 +2115,9 @@ function contractToRules(contract) {
|
|
|
1793
2115
|
);
|
|
1794
2116
|
}
|
|
1795
2117
|
if (!allowIncomplete) {
|
|
1796
|
-
const running = context.events.filter(
|
|
2118
|
+
const running = (context.logicalEvents ?? context.events).filter(
|
|
2119
|
+
(event) => event.status === "running"
|
|
2120
|
+
);
|
|
1797
2121
|
if (running.length > 0) {
|
|
1798
2122
|
findings.push(
|
|
1799
2123
|
contractFailFinding(
|
|
@@ -1825,10 +2149,18 @@ function contractToRules(contract) {
|
|
|
1825
2149
|
rules.push(createStructureIncompleteRule({ requireEndedAtForStarted: true }));
|
|
1826
2150
|
}
|
|
1827
2151
|
if (contract.tools) {
|
|
2152
|
+
const required = [
|
|
2153
|
+
...contract.tools.required ?? [],
|
|
2154
|
+
...contract.tools.requiredTools ?? []
|
|
2155
|
+
];
|
|
2156
|
+
const forbidden = [
|
|
2157
|
+
...contract.tools.forbidden ?? [],
|
|
2158
|
+
...contract.tools.forbiddenTools ?? []
|
|
2159
|
+
];
|
|
1828
2160
|
rules.push(
|
|
1829
2161
|
createToolUsageRule({
|
|
1830
|
-
...
|
|
1831
|
-
...
|
|
2162
|
+
...required.length > 0 ? { required } : {},
|
|
2163
|
+
...forbidden.length > 0 ? { forbidden } : {},
|
|
1832
2164
|
...contract.tools.allowed ? { allowed: contract.tools.allowed } : {},
|
|
1833
2165
|
...contract.tools.maxCalls !== void 0 ? { maxCount: contract.tools.maxCalls } : {}
|
|
1834
2166
|
})
|
|
@@ -1899,6 +2231,6 @@ function evaluateTraceContract(input, contract, options = {}) {
|
|
|
1899
2231
|
});
|
|
1900
2232
|
}
|
|
1901
2233
|
|
|
1902
|
-
export { createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, evaluateTraceContract, runTraceChecks };
|
|
1903
|
-
//# sourceMappingURL=chunk-
|
|
1904
|
-
//# sourceMappingURL=chunk-
|
|
2234
|
+
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, evaluateTraceContract, projectLogicalEvents, resolveCanonicalToolName, runTraceChecks, summarizeSemanticParity };
|
|
2235
|
+
//# sourceMappingURL=chunk-BS2QCAJX.mjs.map
|
|
2236
|
+
//# sourceMappingURL=chunk-BS2QCAJX.mjs.map
|