agent-inspect 6.12.1 → 6.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +1 -1
- package/docs/KNOWN-ISSUES.md +1 -0
- package/package.json +2 -2
- package/packages/cli/dist/{chunk-XTAA733P.mjs → chunk-QOKTMZAY.mjs} +321 -55
- package/packages/cli/dist/chunk-QOKTMZAY.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +350 -70
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +4 -4
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-G7PM24W2.mjs → src-MU4PS6GO.mjs} +3 -3
- package/packages/cli/dist/{src-G7PM24W2.mjs.map → src-MU4PS6GO.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +295 -30
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +1 -1
- package/packages/core/dist/advanced.d.ts +1 -1
- package/packages/core/dist/advanced.mjs +1 -1
- package/packages/core/dist/checks.cjs +368 -32
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +58 -4
- package/packages/core/dist/checks.d.ts +58 -4
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-QT6CQ2XA.mjs → chunk-BS2QCAJX.mjs} +367 -35
- package/packages/core/dist/chunk-BS2QCAJX.mjs.map +1 -0
- package/packages/core/dist/{index-BO7l0iAe.d.ts → index-Bu_vOjql.d.ts} +72 -1
- package/packages/core/dist/{index-mdFcxSOR.d.cts → index-CmXMS-MF.d.cts} +72 -1
- package/packages/cli/dist/chunk-XTAA733P.mjs.map +0 -1
- package/packages/core/dist/chunk-QT6CQ2XA.mjs.map +0 -1
|
@@ -7,7 +7,7 @@ import { P as PersistedInspectEvent } from './persisted-inspect-event-d-mSP8kd.c
|
|
|
7
7
|
import { Stats } from 'node:fs';
|
|
8
8
|
import { b as ObservedOutcome } from './types-B562HgN_.cjs';
|
|
9
9
|
export { O as ObservedOutcomeStatus } from './types-B562HgN_.cjs';
|
|
10
|
-
import {
|
|
10
|
+
import { b as TraceCheckResult, J as TraceCheckStatus } from './index-CmXMS-MF.cjs';
|
|
11
11
|
import './writers.cjs';
|
|
12
12
|
import './log-config-BG1WmWf2.cjs';
|
|
13
13
|
import './readers.cjs';
|
|
@@ -7,7 +7,7 @@ import { P as PersistedInspectEvent } from './persisted-inspect-event-ChE7pGb7.j
|
|
|
7
7
|
import { Stats } from 'node:fs';
|
|
8
8
|
import { b as ObservedOutcome } from './types-B562HgN_.js';
|
|
9
9
|
export { O as ObservedOutcomeStatus } from './types-B562HgN_.js';
|
|
10
|
-
import {
|
|
10
|
+
import { b as TraceCheckResult, J as TraceCheckStatus } from './index-Bu_vOjql.js';
|
|
11
11
|
import './writers.js';
|
|
12
12
|
import './log-config-Ck0aT_Ou.js';
|
|
13
13
|
import './readers.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { runTraceChecks, createRunStatusRule, createToolUsageRule, createRunDurationRule, createLlmUsageRule, createObservedOutcomeRule } from './chunk-
|
|
1
|
+
import { runTraceChecks, createRunStatusRule, createToolUsageRule, createRunDurationRule, createLlmUsageRule, createObservedOutcomeRule } from './chunk-BS2QCAJX.mjs';
|
|
2
2
|
export { MAX_TERMINAL_DEPTH, MAX_TERMINAL_NAME_LENGTH, TERMINAL_INDENT, TraceDirectory, createInspector, createInspectorRuntime, formatTerminalName, getCurrentContext, getCurrentCorrelationMetadata, getCurrentDepth, getCurrentRunId, getCurrentRunName, getCurrentStepId, getIndent, getParentStepId, getTraceDirFromContext, getTraceSafetyFromContext, hasActiveContext, isAgentInspectEnabled, isSilentContext, maybeInspectRun, printError, printFailedAt, printRunComplete, printRunStart, printStepComplete, printStepStart, renderErrorLine, renderRunSummary, renderStepLine, resolveTraceDir, runWithContext, runWithStepContext } from './chunk-ZWA5RWMV.mjs';
|
|
3
3
|
import { buildRunTimeline, filterTraces, extractMetadata, buildRunSummary } from './chunk-LFWZEO2L.mjs';
|
|
4
4
|
export { buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildTraceStats, extractMetadata, filterTraces, renderRunWhat, renderTimeline, renderTraceStats } from './chunk-LFWZEO2L.mjs';
|
|
@@ -64,6 +64,321 @@ function outcomesMatchingStatus(outcomes, statuses) {
|
|
|
64
64
|
return outcomes.filter((outcome) => set.has(outcome.status));
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
+
// packages/core/src/checks/logical-events.ts
|
|
68
|
+
function isRecord(value) {
|
|
69
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
70
|
+
}
|
|
71
|
+
function legacyEvent(event) {
|
|
72
|
+
const value = event.attributes?.legacyEvent;
|
|
73
|
+
return typeof value === "string" ? value : void 0;
|
|
74
|
+
}
|
|
75
|
+
function stepIdOf(event) {
|
|
76
|
+
const value = event.attributes?.stepId;
|
|
77
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
78
|
+
return void 0;
|
|
79
|
+
}
|
|
80
|
+
function cloneEvent(event) {
|
|
81
|
+
return {
|
|
82
|
+
...event,
|
|
83
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
84
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
85
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
86
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
function mergeAttributes(start, complete) {
|
|
90
|
+
const merged = {
|
|
91
|
+
...isRecord(complete.attributes) ? complete.attributes : {},
|
|
92
|
+
...isRecord(start.attributes) ? start.attributes : {}
|
|
93
|
+
};
|
|
94
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
95
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
96
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
97
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
98
|
+
}
|
|
99
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
100
|
+
}
|
|
101
|
+
function pairStartComplete(start, complete) {
|
|
102
|
+
const attributes = mergeAttributes(start, complete);
|
|
103
|
+
const paired = {
|
|
104
|
+
...cloneEvent(start),
|
|
105
|
+
status: complete.status,
|
|
106
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
107
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
108
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
109
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
110
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
111
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
112
|
+
};
|
|
113
|
+
return {
|
|
114
|
+
...paired,
|
|
115
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
116
|
+
projection: {
|
|
117
|
+
paired: true,
|
|
118
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
119
|
+
parentNormalized: false,
|
|
120
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
function asLogical(event, extras) {
|
|
125
|
+
return {
|
|
126
|
+
...cloneEvent(event),
|
|
127
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
128
|
+
projection: {
|
|
129
|
+
paired: false,
|
|
130
|
+
absorbedEventIds: Object.freeze([]),
|
|
131
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
132
|
+
...{}
|
|
133
|
+
}
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
function projectLogicalEvents(events) {
|
|
137
|
+
const diagnostics = [];
|
|
138
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
139
|
+
for (const event of events) {
|
|
140
|
+
const list = byRun.get(event.runId) ?? [];
|
|
141
|
+
list.push(event);
|
|
142
|
+
byRun.set(event.runId, list);
|
|
143
|
+
}
|
|
144
|
+
const absorbedIds = /* @__PURE__ */ new Set();
|
|
145
|
+
const logicalByRawId = /* @__PURE__ */ new Map();
|
|
146
|
+
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
147
|
+
const logical = [];
|
|
148
|
+
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
149
|
+
for (const runId of orderedRuns) {
|
|
150
|
+
const runEvents = byRun.get(runId) ?? [];
|
|
151
|
+
const starts = [];
|
|
152
|
+
const completes = [];
|
|
153
|
+
const others = [];
|
|
154
|
+
for (const event of runEvents) {
|
|
155
|
+
const legacy = legacyEvent(event);
|
|
156
|
+
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
157
|
+
starts.push(event);
|
|
158
|
+
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
159
|
+
completes.push(event);
|
|
160
|
+
} else {
|
|
161
|
+
others.push(event);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
const usedCompletes = /* @__PURE__ */ new Set();
|
|
165
|
+
for (const start of starts) {
|
|
166
|
+
const stepId = stepIdOf(start);
|
|
167
|
+
let match;
|
|
168
|
+
if (legacyEvent(start) === "run_started") {
|
|
169
|
+
const candidates = completes.filter(
|
|
170
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
171
|
+
);
|
|
172
|
+
if (candidates.length > 1) {
|
|
173
|
+
diagnostics.push({
|
|
174
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
175
|
+
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
176
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
177
|
+
});
|
|
178
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
179
|
+
}
|
|
180
|
+
match = candidates[0];
|
|
181
|
+
} else if (stepId) {
|
|
182
|
+
const candidates = completes.filter(
|
|
183
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
184
|
+
);
|
|
185
|
+
if (candidates.length > 1) {
|
|
186
|
+
diagnostics.push({
|
|
187
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
188
|
+
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
189
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
190
|
+
});
|
|
191
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
192
|
+
}
|
|
193
|
+
match = candidates[0];
|
|
194
|
+
}
|
|
195
|
+
if (match) {
|
|
196
|
+
usedCompletes.add(match.eventId);
|
|
197
|
+
absorbedIds.add(match.eventId);
|
|
198
|
+
const paired = pairStartComplete(start, match);
|
|
199
|
+
logical.push(paired);
|
|
200
|
+
logicalByRawId.set(start.eventId, paired);
|
|
201
|
+
logicalByRawId.set(match.eventId, paired);
|
|
202
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
203
|
+
} else {
|
|
204
|
+
diagnostics.push({
|
|
205
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
206
|
+
message: `No matching complete for start ${start.eventId}.`,
|
|
207
|
+
eventIds: [start.eventId]
|
|
208
|
+
});
|
|
209
|
+
const alone = asLogical(start);
|
|
210
|
+
logical.push(alone);
|
|
211
|
+
logicalByRawId.set(start.eventId, alone);
|
|
212
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
for (const complete of completes) {
|
|
216
|
+
if (usedCompletes.has(complete.eventId)) continue;
|
|
217
|
+
diagnostics.push({
|
|
218
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
219
|
+
message: `No matching start for complete ${complete.eventId}.`,
|
|
220
|
+
eventIds: [complete.eventId]
|
|
221
|
+
});
|
|
222
|
+
const alone = asLogical(complete);
|
|
223
|
+
logical.push(alone);
|
|
224
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
225
|
+
const stepId = stepIdOf(complete);
|
|
226
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
227
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
for (const event of others) {
|
|
231
|
+
const alone = asLogical(event);
|
|
232
|
+
logical.push(alone);
|
|
233
|
+
logicalByRawId.set(event.eventId, alone);
|
|
234
|
+
const stepId = stepIdOf(event);
|
|
235
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
236
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
241
|
+
const normalized = [];
|
|
242
|
+
for (const event of logical) {
|
|
243
|
+
const originalParentId = event.parentId;
|
|
244
|
+
if (!originalParentId) {
|
|
245
|
+
normalized.push(event);
|
|
246
|
+
continue;
|
|
247
|
+
}
|
|
248
|
+
let nextParent = originalParentId;
|
|
249
|
+
let remapped = false;
|
|
250
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
251
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
252
|
+
nextParent = viaAbsorbed.eventId;
|
|
253
|
+
remapped = true;
|
|
254
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
255
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
256
|
+
if (viaStep) {
|
|
257
|
+
nextParent = viaStep;
|
|
258
|
+
remapped = true;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
if (!remapped) {
|
|
262
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
263
|
+
const mapping = event.attributes?.parentMapping;
|
|
264
|
+
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
265
|
+
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
266
|
+
if (!unresolved) {
|
|
267
|
+
diagnostics.push({
|
|
268
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
269
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
270
|
+
eventIds: [event.eventId]
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
normalized.push(event);
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
diagnostics.push({
|
|
278
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
279
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
280
|
+
eventIds: [event.eventId]
|
|
281
|
+
});
|
|
282
|
+
normalized.push({
|
|
283
|
+
...event,
|
|
284
|
+
parentId: nextParent,
|
|
285
|
+
projection: {
|
|
286
|
+
...event.projection,
|
|
287
|
+
parentNormalized: true,
|
|
288
|
+
originalParentId
|
|
289
|
+
}
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
293
|
+
normalized.sort((a, b) => {
|
|
294
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
295
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
296
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
297
|
+
});
|
|
298
|
+
return {
|
|
299
|
+
logicalEvents: Object.freeze(normalized),
|
|
300
|
+
diagnostics: Object.freeze(diagnostics)
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
function resolveCanonicalToolName(event) {
|
|
304
|
+
const attrs = event.attributes;
|
|
305
|
+
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
306
|
+
if (direct) return direct;
|
|
307
|
+
const metadata = attrs?.metadata;
|
|
308
|
+
if (isRecord(metadata)) {
|
|
309
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
310
|
+
if (nested) return nested;
|
|
311
|
+
}
|
|
312
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
313
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
314
|
+
}
|
|
315
|
+
return event.name;
|
|
316
|
+
}
|
|
317
|
+
function pickString(record, keys) {
|
|
318
|
+
if (!record) return void 0;
|
|
319
|
+
for (const key of keys) {
|
|
320
|
+
const value = record[key];
|
|
321
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
322
|
+
}
|
|
323
|
+
return void 0;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// packages/core/src/checks/trace-facts.ts
|
|
327
|
+
function summarizeSemanticParity(events) {
|
|
328
|
+
const projection = projectLogicalEvents(events);
|
|
329
|
+
const logical = projection.logicalEvents;
|
|
330
|
+
const finishedTools = logical.filter(
|
|
331
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
332
|
+
);
|
|
333
|
+
const finishedToolNames = Object.freeze(
|
|
334
|
+
finishedTools.map((event) => resolveCanonicalToolName(event)).sort((a, b) => a.localeCompare(b))
|
|
335
|
+
);
|
|
336
|
+
return {
|
|
337
|
+
rawEventCount: events.length,
|
|
338
|
+
logicalEventCount: logical.length,
|
|
339
|
+
runningLogicalCount: logical.filter((event) => event.status === "running").length,
|
|
340
|
+
finishedToolNames,
|
|
341
|
+
finishedToolCount: finishedTools.length,
|
|
342
|
+
pairedCount: logical.filter((event) => event.projection.paired).length,
|
|
343
|
+
parentRemapCount: projection.diagnostics.filter(
|
|
344
|
+
(item) => item.code === "AI_LOGICAL_PARENT_REMAPPED"
|
|
345
|
+
).length,
|
|
346
|
+
diagnostics: projection.diagnostics
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
function buildTraceFacts(events) {
|
|
350
|
+
const projection = projectLogicalEvents(events);
|
|
351
|
+
const toolsByName = /* @__PURE__ */ new Map();
|
|
352
|
+
const llmEvents = [];
|
|
353
|
+
const outcomeEvents = [];
|
|
354
|
+
for (const event of projection.logicalEvents) {
|
|
355
|
+
if (event.kind === "TOOL" && event.status !== "running") {
|
|
356
|
+
const name = resolveCanonicalToolName(event);
|
|
357
|
+
const list = toolsByName.get(name) ?? [];
|
|
358
|
+
list.push(event);
|
|
359
|
+
toolsByName.set(name, list);
|
|
360
|
+
}
|
|
361
|
+
if (event.kind === "LLM" && event.status !== "running") {
|
|
362
|
+
llmEvents.push(event);
|
|
363
|
+
}
|
|
364
|
+
if (event.kind === "OUTCOME") {
|
|
365
|
+
outcomeEvents.push(event);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
for (const [name, list] of [...toolsByName.entries()]) {
|
|
369
|
+
toolsByName.set(name, Object.freeze([...list]));
|
|
370
|
+
}
|
|
371
|
+
return {
|
|
372
|
+
rawEvents: Object.freeze([...events]),
|
|
373
|
+
logicalEvents: projection.logicalEvents,
|
|
374
|
+
diagnostics: projection.diagnostics,
|
|
375
|
+
toolsByName,
|
|
376
|
+
llmEvents: Object.freeze(llmEvents),
|
|
377
|
+
outcomeEvents: Object.freeze(outcomeEvents),
|
|
378
|
+
summary: summarizeSemanticParity(events)
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
|
|
67
382
|
// packages/core/src/checks/contract.ts
|
|
68
383
|
function contractFailFinding(ruleId, message, evidence, expected, actual) {
|
|
69
384
|
return {
|
|
@@ -121,7 +436,9 @@ function contractToRules(contract) {
|
|
|
121
436
|
);
|
|
122
437
|
}
|
|
123
438
|
if (!allowIncomplete) {
|
|
124
|
-
const running = context.events.filter(
|
|
439
|
+
const running = (context.logicalEvents ?? context.events).filter(
|
|
440
|
+
(event) => event.status === "running"
|
|
441
|
+
);
|
|
125
442
|
if (running.length > 0) {
|
|
126
443
|
findings.push(
|
|
127
444
|
contractFailFinding(
|
|
@@ -153,10 +470,18 @@ function contractToRules(contract) {
|
|
|
153
470
|
rules.push(createStructureIncompleteRule({ requireEndedAtForStarted: true }));
|
|
154
471
|
}
|
|
155
472
|
if (contract.tools) {
|
|
473
|
+
const required = [
|
|
474
|
+
...contract.tools.required ?? [],
|
|
475
|
+
...contract.tools.requiredTools ?? []
|
|
476
|
+
];
|
|
477
|
+
const forbidden = [
|
|
478
|
+
...contract.tools.forbidden ?? [],
|
|
479
|
+
...contract.tools.forbiddenTools ?? []
|
|
480
|
+
];
|
|
156
481
|
rules.push(
|
|
157
482
|
createToolUsageRule({
|
|
158
|
-
...
|
|
159
|
-
...
|
|
483
|
+
...required.length > 0 ? { required } : {},
|
|
484
|
+
...forbidden.length > 0 ? { forbidden } : {},
|
|
160
485
|
...contract.tools.allowed ? { allowed: contract.tools.allowed } : {},
|
|
161
486
|
...contract.tools.maxCalls !== void 0 ? { maxCount: contract.tools.maxCalls } : {}
|
|
162
487
|
})
|
|
@@ -285,7 +610,11 @@ var DEFAULT_RAW_CONTENT_KEYS = [
|
|
|
285
610
|
"conversationtext",
|
|
286
611
|
"conversation_text"
|
|
287
612
|
];
|
|
288
|
-
var DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES = [
|
|
613
|
+
var DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES = [
|
|
614
|
+
"tokenUsage",
|
|
615
|
+
"usage",
|
|
616
|
+
"tokens"
|
|
617
|
+
];
|
|
289
618
|
var SAFE_USAGE_LEAF_KEYS = /* @__PURE__ */ new Set([
|
|
290
619
|
"input",
|
|
291
620
|
"output",
|
|
@@ -361,10 +690,13 @@ function buildFacts(input, selectedRun) {
|
|
|
361
690
|
childrenByParentId.set(parentId, children);
|
|
362
691
|
}
|
|
363
692
|
}
|
|
693
|
+
const projection = projectLogicalEvents(scopedEvents);
|
|
364
694
|
return {
|
|
365
695
|
format: input.read.format,
|
|
366
696
|
runs: Object.freeze([...input.read.runs]),
|
|
367
697
|
events: Object.freeze([...scopedEvents]),
|
|
698
|
+
logicalEvents: projection.logicalEvents,
|
|
699
|
+
logicalProjectionDiagnostics: projection.diagnostics,
|
|
368
700
|
readerWarnings: Object.freeze([...input.read.warnings]),
|
|
369
701
|
unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
|
|
370
702
|
sourceFiles: Object.freeze([...input.read.sourceFiles]),
|
|
@@ -557,7 +889,10 @@ function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
|
557
889
|
};
|
|
558
890
|
}
|
|
559
891
|
function toolName(event) {
|
|
560
|
-
return
|
|
892
|
+
return resolveCanonicalToolName(event);
|
|
893
|
+
}
|
|
894
|
+
function semanticEvents(context) {
|
|
895
|
+
return context.logicalEvents ?? context.events;
|
|
561
896
|
}
|
|
562
897
|
function llmModel(event) {
|
|
563
898
|
return stringAttr(event, ["model", "modelId", "responseModelId", "modelName", "model_name"]) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
@@ -572,7 +907,7 @@ function retryCount(event) {
|
|
|
572
907
|
return numericAttr(event, ["retryCount", "retryAttempt", "retry_attempt", "attempt"]);
|
|
573
908
|
}
|
|
574
909
|
function finishedEvents(context, kind) {
|
|
575
|
-
return context.
|
|
910
|
+
return semanticEvents(context).filter(
|
|
576
911
|
(event) => (kind === void 0 || event.kind === kind) && event.status !== "running"
|
|
577
912
|
);
|
|
578
913
|
}
|
|
@@ -584,7 +919,7 @@ function firstIndexByName(events) {
|
|
|
584
919
|
}
|
|
585
920
|
return index;
|
|
586
921
|
}
|
|
587
|
-
function
|
|
922
|
+
function isRecord2(value) {
|
|
588
923
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
589
924
|
}
|
|
590
925
|
function eventMap(events) {
|
|
@@ -638,7 +973,7 @@ function pushValueEntries(entries, event, value, path3, key, depth = 0) {
|
|
|
638
973
|
}
|
|
639
974
|
return;
|
|
640
975
|
}
|
|
641
|
-
if (!
|
|
976
|
+
if (!isRecord2(value)) return;
|
|
642
977
|
for (const nestedKey of Object.keys(value).sort((a, b) => a.localeCompare(b))) {
|
|
643
978
|
pushValueEntries(
|
|
644
979
|
entries,
|
|
@@ -718,14 +1053,11 @@ function signalName(event, attributeKeys, prefixes) {
|
|
|
718
1053
|
return stringAttr(event, attributeKeys) ?? stripPrefix(event.name, prefixes);
|
|
719
1054
|
}
|
|
720
1055
|
function guardrailEvents(context) {
|
|
721
|
-
return
|
|
1056
|
+
return finishedEvents(context).filter((event) => {
|
|
722
1057
|
const name = event.name.toLowerCase();
|
|
723
1058
|
if (name.startsWith("guardrail:") || name.includes(".guardrail.")) return true;
|
|
724
1059
|
return stringAttr(event, ["guardrailName", "guardrail", "guardrailId"]) !== void 0;
|
|
725
1060
|
});
|
|
726
|
-
function finishedEvents2() {
|
|
727
|
-
return context.events.filter((event) => event.status !== "running");
|
|
728
|
-
}
|
|
729
1061
|
}
|
|
730
1062
|
function retryValue(event) {
|
|
731
1063
|
return retryCount(event) ?? 0;
|
|
@@ -746,7 +1078,7 @@ function treeShape(nodes) {
|
|
|
746
1078
|
return lines;
|
|
747
1079
|
}
|
|
748
1080
|
function statusShape(context) {
|
|
749
|
-
return context.
|
|
1081
|
+
return semanticEvents(context).map((event) => `${event.kind}:${event.name}:${event.status ?? "unknown"}`).sort((a, b) => a.localeCompare(b));
|
|
750
1082
|
}
|
|
751
1083
|
function toolShape(context) {
|
|
752
1084
|
return finishedEvents(context, "TOOL").map(
|
|
@@ -772,7 +1104,7 @@ function llmShape(context) {
|
|
|
772
1104
|
);
|
|
773
1105
|
}
|
|
774
1106
|
function errorShape(context) {
|
|
775
|
-
return context.
|
|
1107
|
+
return semanticEvents(context).filter((event) => event.status === "error" || event.error !== void 0).map(
|
|
776
1108
|
(event) => [
|
|
777
1109
|
event.kind,
|
|
778
1110
|
event.name,
|
|
@@ -790,7 +1122,7 @@ function guardrailShape(context) {
|
|
|
790
1122
|
return guardrailEvents(context).map((event) => signalName(event, ["guardrailName", "guardrail", "guardrailId"], ["guardrail:"])).sort((a, b) => a.localeCompare(b));
|
|
791
1123
|
}
|
|
792
1124
|
function firstEvidenceForKind(context, kind, path3) {
|
|
793
|
-
const event = context.
|
|
1125
|
+
const event = semanticEvents(context).find((candidate) => candidate.kind === kind);
|
|
794
1126
|
return event ? [eventEvidence(event, path3)] : runEvidence(context.selectedRun);
|
|
795
1127
|
}
|
|
796
1128
|
function baselineDiffFinding(message, evidence, expected, actual) {
|
|
@@ -818,7 +1150,7 @@ function createRunStatusRule(options = {}) {
|
|
|
818
1150
|
);
|
|
819
1151
|
}
|
|
820
1152
|
if (!allowIncomplete) {
|
|
821
|
-
const running = context.
|
|
1153
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
822
1154
|
if (running.length > 0) {
|
|
823
1155
|
findings.push(
|
|
824
1156
|
failFinding(
|
|
@@ -861,7 +1193,7 @@ function createMaxStepDurationRule(options) {
|
|
|
861
1193
|
category: "run",
|
|
862
1194
|
defaultSeverity: "error",
|
|
863
1195
|
evaluate(context) {
|
|
864
|
-
const over = context.
|
|
1196
|
+
const over = semanticEvents(context).filter((event) => {
|
|
865
1197
|
const duration = eventDurationMs(event);
|
|
866
1198
|
return duration !== void 0 && duration > options.maxDurationMs;
|
|
867
1199
|
});
|
|
@@ -890,7 +1222,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
890
1222
|
defaultSeverity: "warning",
|
|
891
1223
|
evaluate(context) {
|
|
892
1224
|
const findings = [];
|
|
893
|
-
const running = context.
|
|
1225
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
894
1226
|
if (running.length > 0) {
|
|
895
1227
|
findings.push(
|
|
896
1228
|
failFinding(
|
|
@@ -903,7 +1235,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
903
1235
|
);
|
|
904
1236
|
}
|
|
905
1237
|
if (requireEndedAt) {
|
|
906
|
-
const incomplete = context.
|
|
1238
|
+
const incomplete = semanticEvents(context).filter(
|
|
907
1239
|
(event) => event.startedAt !== void 0 && event.endedAt === void 0 && event.status !== "running"
|
|
908
1240
|
);
|
|
909
1241
|
if (incomplete.length > 0) {
|
|
@@ -941,7 +1273,7 @@ function createRequireCompletedRule() {
|
|
|
941
1273
|
)
|
|
942
1274
|
);
|
|
943
1275
|
}
|
|
944
|
-
const running = context.
|
|
1276
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
945
1277
|
if (running.length > 0) {
|
|
946
1278
|
findings.push(
|
|
947
1279
|
failFinding(
|
|
@@ -963,7 +1295,7 @@ function createRunEventCountRule(options) {
|
|
|
963
1295
|
category: "run",
|
|
964
1296
|
defaultSeverity: "error",
|
|
965
1297
|
evaluate(context) {
|
|
966
|
-
const count = context.
|
|
1298
|
+
const count = semanticEvents(context).filter(
|
|
967
1299
|
(event) => options.kind === void 0 || event.kind === options.kind
|
|
968
1300
|
).length;
|
|
969
1301
|
const findings = [];
|
|
@@ -1214,7 +1546,7 @@ function createStructureIncompleteRule(options = {}) {
|
|
|
1214
1546
|
evaluate(context) {
|
|
1215
1547
|
const findings = [];
|
|
1216
1548
|
if (!options.allowRunning) {
|
|
1217
|
-
const running = context.
|
|
1549
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
1218
1550
|
if (running.length > 0) {
|
|
1219
1551
|
findings.push(
|
|
1220
1552
|
failFinding(
|
|
@@ -1228,7 +1560,7 @@ function createStructureIncompleteRule(options = {}) {
|
|
|
1228
1560
|
}
|
|
1229
1561
|
}
|
|
1230
1562
|
if (options.requireEndedAtForStarted) {
|
|
1231
|
-
const missingEndedAt = context.
|
|
1563
|
+
const missingEndedAt = semanticEvents(context).filter(
|
|
1232
1564
|
(event) => event.startedAt !== void 0 && event.endedAt === void 0 && event.status !== "running"
|
|
1233
1565
|
);
|
|
1234
1566
|
if (missingEndedAt.length > 0) {
|
|
@@ -1254,8 +1586,8 @@ function createStructureOrphanRule(options = {}) {
|
|
|
1254
1586
|
category: "structure",
|
|
1255
1587
|
defaultSeverity: "error",
|
|
1256
1588
|
evaluate(context) {
|
|
1257
|
-
const byId = eventMap(context
|
|
1258
|
-
const orphans = context.
|
|
1589
|
+
const byId = eventMap(semanticEvents(context));
|
|
1590
|
+
const orphans = semanticEvents(context).filter((event) => {
|
|
1259
1591
|
if (!event.parentId || byId.has(event.parentId)) return false;
|
|
1260
1592
|
return !(allowMarkedUnresolved && parentMarkedUnresolved(event));
|
|
1261
1593
|
});
|
|
@@ -1278,10 +1610,10 @@ function createStructureCycleRule() {
|
|
|
1278
1610
|
category: "structure",
|
|
1279
1611
|
defaultSeverity: "error",
|
|
1280
1612
|
evaluate(context) {
|
|
1281
|
-
const byId = eventMap(context
|
|
1613
|
+
const byId = eventMap(semanticEvents(context));
|
|
1282
1614
|
const seenCycles = /* @__PURE__ */ new Set();
|
|
1283
1615
|
const findings = [];
|
|
1284
|
-
for (const event of [...context
|
|
1616
|
+
for (const event of [...semanticEvents(context)].sort((a, b) => a.eventId.localeCompare(b.eventId))) {
|
|
1285
1617
|
const path3 = [];
|
|
1286
1618
|
const seenAt = /* @__PURE__ */ new Map();
|
|
1287
1619
|
let current = event;
|
|
@@ -1319,10 +1651,10 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
1319
1651
|
category: "structure",
|
|
1320
1652
|
defaultSeverity: "error",
|
|
1321
1653
|
evaluate(context) {
|
|
1322
|
-
const byId = eventMap(context
|
|
1654
|
+
const byId = eventMap(semanticEvents(context));
|
|
1323
1655
|
const findings = [];
|
|
1324
1656
|
const minConfidence = options.minConfidence;
|
|
1325
|
-
for (const event of context
|
|
1657
|
+
for (const event of semanticEvents(context)) {
|
|
1326
1658
|
if (minConfidence && CONFIDENCE_RANK[event.confidence] < CONFIDENCE_RANK[minConfidence]) {
|
|
1327
1659
|
findings.push(
|
|
1328
1660
|
failFinding(
|
|
@@ -1390,7 +1722,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
1390
1722
|
defaultSeverity: "error",
|
|
1391
1723
|
evaluate(context) {
|
|
1392
1724
|
const findings = [];
|
|
1393
|
-
const byId = eventMap(context
|
|
1725
|
+
const byId = eventMap(semanticEvents(context));
|
|
1394
1726
|
if (options.maxChildren !== void 0) {
|
|
1395
1727
|
for (const [parentId, children] of context.childrenByParentId.entries()) {
|
|
1396
1728
|
if (children.length <= options.maxChildren) continue;
|
|
@@ -1417,7 +1749,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
1417
1749
|
}
|
|
1418
1750
|
}
|
|
1419
1751
|
if (options.maxConcurrent !== void 0) {
|
|
1420
|
-
const intervals = context.
|
|
1752
|
+
const intervals = semanticEvents(context).map((event) => ({ event, start: eventStartMs(event), end: eventEndMs(event) })).filter(
|
|
1421
1753
|
(item) => item.start !== void 0 && item.end !== void 0 && item.end > item.start
|
|
1422
1754
|
);
|
|
1423
1755
|
const points = intervals.flatMap((item) => [
|
|
@@ -1713,7 +2045,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
1713
2045
|
)
|
|
1714
2046
|
);
|
|
1715
2047
|
}
|
|
1716
|
-
if (
|
|
2048
|
+
if (isRecord2(entry.value) && options.maxObjectKeys !== void 0 && Object.keys(entry.value).length > options.maxObjectKeys) {
|
|
1717
2049
|
findings.push(
|
|
1718
2050
|
failFinding(
|
|
1719
2051
|
"safety.oversizedAttribute",
|
|
@@ -1963,6 +2295,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
1963
2295
|
};
|
|
1964
2296
|
}
|
|
1965
2297
|
|
|
2298
|
+
exports.buildTraceFacts = buildTraceFacts;
|
|
1966
2299
|
exports.createBaselineRegressionRule = createBaselineRegressionRule;
|
|
1967
2300
|
exports.createDecisionRule = createDecisionRule;
|
|
1968
2301
|
exports.createGuardrailRule = createGuardrailRule;
|
|
@@ -1990,6 +2323,9 @@ exports.createToolOrderingRule = createToolOrderingRule;
|
|
|
1990
2323
|
exports.createToolUsageRule = createToolUsageRule;
|
|
1991
2324
|
exports.defineTraceContract = defineTraceContract;
|
|
1992
2325
|
exports.evaluateTraceContract = evaluateTraceContract;
|
|
2326
|
+
exports.projectLogicalEvents = projectLogicalEvents;
|
|
2327
|
+
exports.resolveCanonicalToolName = resolveCanonicalToolName;
|
|
1993
2328
|
exports.runTraceChecks = runTraceChecks;
|
|
2329
|
+
exports.summarizeSemanticParity = summarizeSemanticParity;
|
|
1994
2330
|
//# sourceMappingURL=checks.cjs.map
|
|
1995
2331
|
//# sourceMappingURL=checks.cjs.map
|