agent-inspect 6.12.0 → 6.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +27 -6
- package/docs/ADOPTION.md +1 -1
- package/docs/DEMO-SCRIPT.md +30 -25
- package/docs/DESIGN-PARTNER-GUIDE.md +23 -10
- package/docs/FIRST-TRACE-IN-5-MINUTES.md +5 -2
- package/docs/GOLDEN-PATH.md +2 -2
- package/docs/KNOWN-ISSUES.md +1 -0
- package/docs/PRE-V7-PILOT-KIT.md +41 -15
- package/docs/SUPPORT-LEVELS.md +1 -1
- package/docs/TEAM-WORKFLOWS.md +1 -1
- package/docs/TECHNICAL-GUIDE.md +2 -2
- package/docs/USE-CASES.md +34 -10
- package/package.json +2 -2
- package/packages/cli/dist/{chunk-XTAA733P.mjs → chunk-4WA7DQCM.mjs} +316 -55
- package/packages/cli/dist/chunk-4WA7DQCM.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +337 -70
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +4 -4
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-G7PM24W2.mjs → src-YAASTXGE.mjs} +3 -3
- package/packages/cli/dist/{src-G7PM24W2.mjs.map → src-YAASTXGE.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +290 -30
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +1 -1
- package/packages/core/dist/advanced.d.ts +1 -1
- package/packages/core/dist/advanced.mjs +1 -1
- package/packages/core/dist/checks.cjs +295 -30
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +2 -2
- package/packages/core/dist/checks.d.ts +2 -2
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-QT6CQ2XA.mjs → chunk-KJICERSD.mjs} +296 -33
- package/packages/core/dist/chunk-KJICERSD.mjs.map +1 -0
- package/packages/core/dist/{index-mdFcxSOR.d.cts → index-BOP33T4O.d.cts} +72 -1
- package/packages/core/dist/{index-BO7l0iAe.d.ts → index-CgCH8NE9.d.ts} +72 -1
- package/packages/cli/dist/chunk-XTAA733P.mjs.map +0 -1
- package/packages/core/dist/chunk-QT6CQ2XA.mjs.map +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { T as TraceCheckInput, a as TraceCheckResult } from './index-
|
|
2
|
-
export { B as BaselineRegressionRuleOptions, D as DecisionRuleOptions, G as GuardrailRuleOptions, L as LlmUsageRuleOptions, M as MaxStepDurationRuleOptions, O as ObservedOutcomeRuleOptions, R as RetrievalRuleOptions,
|
|
1
|
+
import { T as TraceCheckInput, a as TraceCheckResult } from './index-BOP33T4O.cjs';
|
|
2
|
+
export { B as BaselineRegressionRuleOptions, D as DecisionRuleOptions, G as GuardrailRuleOptions, L as LlmUsageRuleOptions, b as LogicalProjectionDiagnostic, c as LogicalTraceEvent, M as MaxStepDurationRuleOptions, O as ObservedOutcomeRuleOptions, R as RetrievalRuleOptions, d as RunDepthRuleOptions, e as RunDurationRuleOptions, f as RunEventCountRuleOptions, g as RunStatusRuleOptions, h as RunTraceChecksOptions, S as SafetyFindingCategory, i as SafetyFindingConfidence, j as SafetyOversizedAttributeRuleOptions, k as SafetyRawContentRuleOptions, l as SafetyRedactionRuleOptions, m as SafetySecretPattern, n as SafetySecretPatternRuleOptions, o as StallDetectionRuleOptions, p as StructureIncompleteRuleOptions, q as StructureOrphanRuleOptions, r as StructureParallelWidthRuleOptions, s as StructureRelationshipRuleOptions, t as ToolFailureRuleOptions, u as ToolOrderingRuleOptions, v as ToolUsageRuleOptions, w as TraceCheckContext, x as TraceCheckDiagnostic, y as TraceCheckDiagnosticCode, z as TraceCheckEvidence, A as TraceCheckFacts, C as TraceCheckFinding, E as TraceCheckFindingStatus, F as TraceCheckRule, H as TraceCheckRuleCategory, I as TraceCheckSeverity, J as TraceCheckStatus, K as TraceCheckSummary, N as TraceSignalRuleOptions, P as createBaselineRegressionRule, Q as createDecisionRule, U as createGuardrailRule, V as createLlmUsageRule, W as createMaxStepDurationRule, X as createObservedOutcomeRule, Y as createRequireCompletedRule, Z as createRetrievalRule, _ as createRunDepthRule, $ as createRunDurationRule, a0 as createRunEventCountRule, a1 as createRunStatusRule, a2 as createSafetyOversizedAttributeRule, a3 as createSafetyRawContentRule, a4 as createSafetyRedactionRule, a5 as createSafetySecretPatternRule, a6 as createStallDetectionRule, a7 as createStructureCycleRule, a8 as createStructureIncompleteRule, a9 as createStructureOrphanRule, aa as createStructureParallelWidthRule, ab as createStructureRelationshipRule, ac as createToolFailureRule, ad as createToolOrderingRule, ae as createToolUsageRule, af as projectLogicalEvents, ag as resolveCanonicalToolName, ah as runTraceChecks } from './index-BOP33T4O.cjs';
|
|
3
3
|
import './readers.cjs';
|
|
4
4
|
import './inspect-event-DVg84S4v.cjs';
|
|
5
5
|
import './persisted-inspect-event-d-mSP8kd.cjs';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { T as TraceCheckInput, a as TraceCheckResult } from './index-
|
|
2
|
-
export { B as BaselineRegressionRuleOptions, D as DecisionRuleOptions, G as GuardrailRuleOptions, L as LlmUsageRuleOptions, M as MaxStepDurationRuleOptions, O as ObservedOutcomeRuleOptions, R as RetrievalRuleOptions,
|
|
1
|
+
import { T as TraceCheckInput, a as TraceCheckResult } from './index-CgCH8NE9.js';
|
|
2
|
+
export { B as BaselineRegressionRuleOptions, D as DecisionRuleOptions, G as GuardrailRuleOptions, L as LlmUsageRuleOptions, b as LogicalProjectionDiagnostic, c as LogicalTraceEvent, M as MaxStepDurationRuleOptions, O as ObservedOutcomeRuleOptions, R as RetrievalRuleOptions, d as RunDepthRuleOptions, e as RunDurationRuleOptions, f as RunEventCountRuleOptions, g as RunStatusRuleOptions, h as RunTraceChecksOptions, S as SafetyFindingCategory, i as SafetyFindingConfidence, j as SafetyOversizedAttributeRuleOptions, k as SafetyRawContentRuleOptions, l as SafetyRedactionRuleOptions, m as SafetySecretPattern, n as SafetySecretPatternRuleOptions, o as StallDetectionRuleOptions, p as StructureIncompleteRuleOptions, q as StructureOrphanRuleOptions, r as StructureParallelWidthRuleOptions, s as StructureRelationshipRuleOptions, t as ToolFailureRuleOptions, u as ToolOrderingRuleOptions, v as ToolUsageRuleOptions, w as TraceCheckContext, x as TraceCheckDiagnostic, y as TraceCheckDiagnosticCode, z as TraceCheckEvidence, A as TraceCheckFacts, C as TraceCheckFinding, E as TraceCheckFindingStatus, F as TraceCheckRule, H as TraceCheckRuleCategory, I as TraceCheckSeverity, J as TraceCheckStatus, K as TraceCheckSummary, N as TraceSignalRuleOptions, P as createBaselineRegressionRule, Q as createDecisionRule, U as createGuardrailRule, V as createLlmUsageRule, W as createMaxStepDurationRule, X as createObservedOutcomeRule, Y as createRequireCompletedRule, Z as createRetrievalRule, _ as createRunDepthRule, $ as createRunDurationRule, a0 as createRunEventCountRule, a1 as createRunStatusRule, a2 as createSafetyOversizedAttributeRule, a3 as createSafetyRawContentRule, a4 as createSafetyRedactionRule, a5 as createSafetySecretPatternRule, a6 as createStallDetectionRule, a7 as createStructureCycleRule, a8 as createStructureIncompleteRule, a9 as createStructureOrphanRule, aa as createStructureParallelWidthRule, ab as createStructureRelationshipRule, ac as createToolFailureRule, ad as createToolOrderingRule, ae as createToolUsageRule, af as projectLogicalEvents, ag as resolveCanonicalToolName, ah as runTraceChecks } from './index-CgCH8NE9.js';
|
|
3
3
|
import './readers.js';
|
|
4
4
|
import './inspect-event-DVg84S4v.js';
|
|
5
5
|
import './persisted-inspect-event-ChE7pGb7.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, evaluateTraceContract, runTraceChecks } from './chunk-
|
|
1
|
+
export { createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, evaluateTraceContract, projectLogicalEvents, resolveCanonicalToolName, runTraceChecks } from './chunk-KJICERSD.mjs';
|
|
2
2
|
import './chunk-VFO76UH3.mjs';
|
|
3
3
|
import './chunk-ULCGIRTC.mjs';
|
|
4
4
|
import './chunk-VU6O5QAH.mjs';
|
|
@@ -1,5 +1,259 @@
|
|
|
1
1
|
import { extractOutcomesFromPersistedEvents, outcomesMatchingStatus } from './chunk-VFO76UH3.mjs';
|
|
2
2
|
|
|
3
|
+
// packages/core/src/checks/logical-events.ts
|
|
4
|
+
function isRecord(value) {
|
|
5
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
6
|
+
}
|
|
7
|
+
function legacyEvent(event) {
|
|
8
|
+
const value = event.attributes?.legacyEvent;
|
|
9
|
+
return typeof value === "string" ? value : void 0;
|
|
10
|
+
}
|
|
11
|
+
function stepIdOf(event) {
|
|
12
|
+
const value = event.attributes?.stepId;
|
|
13
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
14
|
+
return void 0;
|
|
15
|
+
}
|
|
16
|
+
function cloneEvent(event) {
|
|
17
|
+
return {
|
|
18
|
+
...event,
|
|
19
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
20
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
21
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
22
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
function mergeAttributes(start, complete) {
|
|
26
|
+
const merged = {
|
|
27
|
+
...isRecord(complete.attributes) ? complete.attributes : {},
|
|
28
|
+
...isRecord(start.attributes) ? start.attributes : {}
|
|
29
|
+
};
|
|
30
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
31
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
32
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
33
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
34
|
+
}
|
|
35
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
36
|
+
}
|
|
37
|
+
function pairStartComplete(start, complete) {
|
|
38
|
+
const attributes = mergeAttributes(start, complete);
|
|
39
|
+
const paired = {
|
|
40
|
+
...cloneEvent(start),
|
|
41
|
+
status: complete.status,
|
|
42
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
43
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
44
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
45
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
46
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
47
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
48
|
+
};
|
|
49
|
+
return {
|
|
50
|
+
...paired,
|
|
51
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
52
|
+
projection: {
|
|
53
|
+
paired: true,
|
|
54
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
55
|
+
parentNormalized: false,
|
|
56
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
function asLogical(event, extras) {
|
|
61
|
+
return {
|
|
62
|
+
...cloneEvent(event),
|
|
63
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
64
|
+
projection: {
|
|
65
|
+
paired: false,
|
|
66
|
+
absorbedEventIds: Object.freeze([]),
|
|
67
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
68
|
+
...{}
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
function projectLogicalEvents(events) {
|
|
73
|
+
const diagnostics = [];
|
|
74
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
75
|
+
for (const event of events) {
|
|
76
|
+
const list = byRun.get(event.runId) ?? [];
|
|
77
|
+
list.push(event);
|
|
78
|
+
byRun.set(event.runId, list);
|
|
79
|
+
}
|
|
80
|
+
const absorbedIds = /* @__PURE__ */ new Set();
|
|
81
|
+
const logicalByRawId = /* @__PURE__ */ new Map();
|
|
82
|
+
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
83
|
+
const logical = [];
|
|
84
|
+
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
85
|
+
for (const runId of orderedRuns) {
|
|
86
|
+
const runEvents = byRun.get(runId) ?? [];
|
|
87
|
+
const starts = [];
|
|
88
|
+
const completes = [];
|
|
89
|
+
const others = [];
|
|
90
|
+
for (const event of runEvents) {
|
|
91
|
+
const legacy = legacyEvent(event);
|
|
92
|
+
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
93
|
+
starts.push(event);
|
|
94
|
+
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
95
|
+
completes.push(event);
|
|
96
|
+
} else {
|
|
97
|
+
others.push(event);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const usedCompletes = /* @__PURE__ */ new Set();
|
|
101
|
+
for (const start of starts) {
|
|
102
|
+
const stepId = stepIdOf(start);
|
|
103
|
+
let match;
|
|
104
|
+
if (legacyEvent(start) === "run_started") {
|
|
105
|
+
const candidates = completes.filter(
|
|
106
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
107
|
+
);
|
|
108
|
+
if (candidates.length > 1) {
|
|
109
|
+
diagnostics.push({
|
|
110
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
111
|
+
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
112
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
113
|
+
});
|
|
114
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
115
|
+
}
|
|
116
|
+
match = candidates[0];
|
|
117
|
+
} else if (stepId) {
|
|
118
|
+
const candidates = completes.filter(
|
|
119
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
120
|
+
);
|
|
121
|
+
if (candidates.length > 1) {
|
|
122
|
+
diagnostics.push({
|
|
123
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
124
|
+
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
125
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
126
|
+
});
|
|
127
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
128
|
+
}
|
|
129
|
+
match = candidates[0];
|
|
130
|
+
}
|
|
131
|
+
if (match) {
|
|
132
|
+
usedCompletes.add(match.eventId);
|
|
133
|
+
absorbedIds.add(match.eventId);
|
|
134
|
+
const paired = pairStartComplete(start, match);
|
|
135
|
+
logical.push(paired);
|
|
136
|
+
logicalByRawId.set(start.eventId, paired);
|
|
137
|
+
logicalByRawId.set(match.eventId, paired);
|
|
138
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
139
|
+
} else {
|
|
140
|
+
diagnostics.push({
|
|
141
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
142
|
+
message: `No matching complete for start ${start.eventId}.`,
|
|
143
|
+
eventIds: [start.eventId]
|
|
144
|
+
});
|
|
145
|
+
const alone = asLogical(start);
|
|
146
|
+
logical.push(alone);
|
|
147
|
+
logicalByRawId.set(start.eventId, alone);
|
|
148
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
for (const complete of completes) {
|
|
152
|
+
if (usedCompletes.has(complete.eventId)) continue;
|
|
153
|
+
diagnostics.push({
|
|
154
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
155
|
+
message: `No matching start for complete ${complete.eventId}.`,
|
|
156
|
+
eventIds: [complete.eventId]
|
|
157
|
+
});
|
|
158
|
+
const alone = asLogical(complete);
|
|
159
|
+
logical.push(alone);
|
|
160
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
161
|
+
const stepId = stepIdOf(complete);
|
|
162
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
163
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
for (const event of others) {
|
|
167
|
+
const alone = asLogical(event);
|
|
168
|
+
logical.push(alone);
|
|
169
|
+
logicalByRawId.set(event.eventId, alone);
|
|
170
|
+
const stepId = stepIdOf(event);
|
|
171
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
172
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
177
|
+
const normalized = [];
|
|
178
|
+
for (const event of logical) {
|
|
179
|
+
const originalParentId = event.parentId;
|
|
180
|
+
if (!originalParentId) {
|
|
181
|
+
normalized.push(event);
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
let nextParent = originalParentId;
|
|
185
|
+
let remapped = false;
|
|
186
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
187
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
188
|
+
nextParent = viaAbsorbed.eventId;
|
|
189
|
+
remapped = true;
|
|
190
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
191
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
192
|
+
if (viaStep) {
|
|
193
|
+
nextParent = viaStep;
|
|
194
|
+
remapped = true;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
if (!remapped) {
|
|
198
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
199
|
+
diagnostics.push({
|
|
200
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
201
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
202
|
+
eventIds: [event.eventId]
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
normalized.push(event);
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
diagnostics.push({
|
|
209
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
210
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
211
|
+
eventIds: [event.eventId]
|
|
212
|
+
});
|
|
213
|
+
normalized.push({
|
|
214
|
+
...event,
|
|
215
|
+
parentId: nextParent,
|
|
216
|
+
projection: {
|
|
217
|
+
...event.projection,
|
|
218
|
+
parentNormalized: true,
|
|
219
|
+
originalParentId
|
|
220
|
+
}
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
224
|
+
normalized.sort((a, b) => {
|
|
225
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
226
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
227
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
228
|
+
});
|
|
229
|
+
return {
|
|
230
|
+
logicalEvents: Object.freeze(normalized),
|
|
231
|
+
diagnostics: Object.freeze(diagnostics)
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
function resolveCanonicalToolName(event) {
|
|
235
|
+
const attrs = event.attributes;
|
|
236
|
+
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
237
|
+
if (direct) return direct;
|
|
238
|
+
const metadata = attrs?.metadata;
|
|
239
|
+
if (isRecord(metadata)) {
|
|
240
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
241
|
+
if (nested) return nested;
|
|
242
|
+
}
|
|
243
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
244
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
245
|
+
}
|
|
246
|
+
return event.name;
|
|
247
|
+
}
|
|
248
|
+
function pickString(record, keys) {
|
|
249
|
+
if (!record) return void 0;
|
|
250
|
+
for (const key of keys) {
|
|
251
|
+
const value = record[key];
|
|
252
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
253
|
+
}
|
|
254
|
+
return void 0;
|
|
255
|
+
}
|
|
256
|
+
|
|
3
257
|
// packages/core/src/checks/index.ts
|
|
4
258
|
var SEVERITY_RANK = {
|
|
5
259
|
error: 0,
|
|
@@ -58,7 +312,11 @@ var DEFAULT_RAW_CONTENT_KEYS = [
|
|
|
58
312
|
"conversationtext",
|
|
59
313
|
"conversation_text"
|
|
60
314
|
];
|
|
61
|
-
var DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES = [
|
|
315
|
+
var DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES = [
|
|
316
|
+
"tokenUsage",
|
|
317
|
+
"usage",
|
|
318
|
+
"tokens"
|
|
319
|
+
];
|
|
62
320
|
var SAFE_USAGE_LEAF_KEYS = /* @__PURE__ */ new Set([
|
|
63
321
|
"input",
|
|
64
322
|
"output",
|
|
@@ -134,10 +392,13 @@ function buildFacts(input, selectedRun) {
|
|
|
134
392
|
childrenByParentId.set(parentId, children);
|
|
135
393
|
}
|
|
136
394
|
}
|
|
395
|
+
const projection = projectLogicalEvents(scopedEvents);
|
|
137
396
|
return {
|
|
138
397
|
format: input.read.format,
|
|
139
398
|
runs: Object.freeze([...input.read.runs]),
|
|
140
399
|
events: Object.freeze([...scopedEvents]),
|
|
400
|
+
logicalEvents: projection.logicalEvents,
|
|
401
|
+
logicalProjectionDiagnostics: projection.diagnostics,
|
|
141
402
|
readerWarnings: Object.freeze([...input.read.warnings]),
|
|
142
403
|
unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
|
|
143
404
|
sourceFiles: Object.freeze([...input.read.sourceFiles]),
|
|
@@ -330,7 +591,10 @@ function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
|
330
591
|
};
|
|
331
592
|
}
|
|
332
593
|
function toolName(event) {
|
|
333
|
-
return
|
|
594
|
+
return resolveCanonicalToolName(event);
|
|
595
|
+
}
|
|
596
|
+
function semanticEvents(context) {
|
|
597
|
+
return context.logicalEvents ?? context.events;
|
|
334
598
|
}
|
|
335
599
|
function llmModel(event) {
|
|
336
600
|
return stringAttr(event, ["model", "modelId", "responseModelId", "modelName", "model_name"]) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
@@ -345,7 +609,7 @@ function retryCount(event) {
|
|
|
345
609
|
return numericAttr(event, ["retryCount", "retryAttempt", "retry_attempt", "attempt"]);
|
|
346
610
|
}
|
|
347
611
|
function finishedEvents(context, kind) {
|
|
348
|
-
return context.
|
|
612
|
+
return semanticEvents(context).filter(
|
|
349
613
|
(event) => (kind === void 0 || event.kind === kind) && event.status !== "running"
|
|
350
614
|
);
|
|
351
615
|
}
|
|
@@ -357,7 +621,7 @@ function firstIndexByName(events) {
|
|
|
357
621
|
}
|
|
358
622
|
return index;
|
|
359
623
|
}
|
|
360
|
-
function
|
|
624
|
+
function isRecord2(value) {
|
|
361
625
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
362
626
|
}
|
|
363
627
|
function eventMap(events) {
|
|
@@ -411,7 +675,7 @@ function pushValueEntries(entries, event, value, path, key, depth = 0) {
|
|
|
411
675
|
}
|
|
412
676
|
return;
|
|
413
677
|
}
|
|
414
|
-
if (!
|
|
678
|
+
if (!isRecord2(value)) return;
|
|
415
679
|
for (const nestedKey of Object.keys(value).sort((a, b) => a.localeCompare(b))) {
|
|
416
680
|
pushValueEntries(
|
|
417
681
|
entries,
|
|
@@ -491,14 +755,11 @@ function signalName(event, attributeKeys, prefixes) {
|
|
|
491
755
|
return stringAttr(event, attributeKeys) ?? stripPrefix(event.name, prefixes);
|
|
492
756
|
}
|
|
493
757
|
function guardrailEvents(context) {
|
|
494
|
-
return
|
|
758
|
+
return finishedEvents(context).filter((event) => {
|
|
495
759
|
const name = event.name.toLowerCase();
|
|
496
760
|
if (name.startsWith("guardrail:") || name.includes(".guardrail.")) return true;
|
|
497
761
|
return stringAttr(event, ["guardrailName", "guardrail", "guardrailId"]) !== void 0;
|
|
498
762
|
});
|
|
499
|
-
function finishedEvents2() {
|
|
500
|
-
return context.events.filter((event) => event.status !== "running");
|
|
501
|
-
}
|
|
502
763
|
}
|
|
503
764
|
function retryValue(event) {
|
|
504
765
|
return retryCount(event) ?? 0;
|
|
@@ -519,7 +780,7 @@ function treeShape(nodes) {
|
|
|
519
780
|
return lines;
|
|
520
781
|
}
|
|
521
782
|
function statusShape(context) {
|
|
522
|
-
return context.
|
|
783
|
+
return semanticEvents(context).map((event) => `${event.kind}:${event.name}:${event.status ?? "unknown"}`).sort((a, b) => a.localeCompare(b));
|
|
523
784
|
}
|
|
524
785
|
function toolShape(context) {
|
|
525
786
|
return finishedEvents(context, "TOOL").map(
|
|
@@ -545,7 +806,7 @@ function llmShape(context) {
|
|
|
545
806
|
);
|
|
546
807
|
}
|
|
547
808
|
function errorShape(context) {
|
|
548
|
-
return context.
|
|
809
|
+
return semanticEvents(context).filter((event) => event.status === "error" || event.error !== void 0).map(
|
|
549
810
|
(event) => [
|
|
550
811
|
event.kind,
|
|
551
812
|
event.name,
|
|
@@ -563,7 +824,7 @@ function guardrailShape(context) {
|
|
|
563
824
|
return guardrailEvents(context).map((event) => signalName(event, ["guardrailName", "guardrail", "guardrailId"], ["guardrail:"])).sort((a, b) => a.localeCompare(b));
|
|
564
825
|
}
|
|
565
826
|
function firstEvidenceForKind(context, kind, path) {
|
|
566
|
-
const event = context.
|
|
827
|
+
const event = semanticEvents(context).find((candidate) => candidate.kind === kind);
|
|
567
828
|
return event ? [eventEvidence(event, path)] : runEvidence(context.selectedRun);
|
|
568
829
|
}
|
|
569
830
|
function baselineDiffFinding(message, evidence, expected, actual) {
|
|
@@ -591,7 +852,7 @@ function createRunStatusRule(options = {}) {
|
|
|
591
852
|
);
|
|
592
853
|
}
|
|
593
854
|
if (!allowIncomplete) {
|
|
594
|
-
const running = context.
|
|
855
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
595
856
|
if (running.length > 0) {
|
|
596
857
|
findings.push(
|
|
597
858
|
failFinding(
|
|
@@ -634,7 +895,7 @@ function createMaxStepDurationRule(options) {
|
|
|
634
895
|
category: "run",
|
|
635
896
|
defaultSeverity: "error",
|
|
636
897
|
evaluate(context) {
|
|
637
|
-
const over = context.
|
|
898
|
+
const over = semanticEvents(context).filter((event) => {
|
|
638
899
|
const duration = eventDurationMs(event);
|
|
639
900
|
return duration !== void 0 && duration > options.maxDurationMs;
|
|
640
901
|
});
|
|
@@ -663,7 +924,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
663
924
|
defaultSeverity: "warning",
|
|
664
925
|
evaluate(context) {
|
|
665
926
|
const findings = [];
|
|
666
|
-
const running = context.
|
|
927
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
667
928
|
if (running.length > 0) {
|
|
668
929
|
findings.push(
|
|
669
930
|
failFinding(
|
|
@@ -676,7 +937,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
676
937
|
);
|
|
677
938
|
}
|
|
678
939
|
if (requireEndedAt) {
|
|
679
|
-
const incomplete = context.
|
|
940
|
+
const incomplete = semanticEvents(context).filter(
|
|
680
941
|
(event) => event.startedAt !== void 0 && event.endedAt === void 0 && event.status !== "running"
|
|
681
942
|
);
|
|
682
943
|
if (incomplete.length > 0) {
|
|
@@ -714,7 +975,7 @@ function createRequireCompletedRule() {
|
|
|
714
975
|
)
|
|
715
976
|
);
|
|
716
977
|
}
|
|
717
|
-
const running = context.
|
|
978
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
718
979
|
if (running.length > 0) {
|
|
719
980
|
findings.push(
|
|
720
981
|
failFinding(
|
|
@@ -736,7 +997,7 @@ function createRunEventCountRule(options) {
|
|
|
736
997
|
category: "run",
|
|
737
998
|
defaultSeverity: "error",
|
|
738
999
|
evaluate(context) {
|
|
739
|
-
const count = context.
|
|
1000
|
+
const count = semanticEvents(context).filter(
|
|
740
1001
|
(event) => options.kind === void 0 || event.kind === options.kind
|
|
741
1002
|
).length;
|
|
742
1003
|
const findings = [];
|
|
@@ -987,7 +1248,7 @@ function createStructureIncompleteRule(options = {}) {
|
|
|
987
1248
|
evaluate(context) {
|
|
988
1249
|
const findings = [];
|
|
989
1250
|
if (!options.allowRunning) {
|
|
990
|
-
const running = context.
|
|
1251
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
991
1252
|
if (running.length > 0) {
|
|
992
1253
|
findings.push(
|
|
993
1254
|
failFinding(
|
|
@@ -1001,7 +1262,7 @@ function createStructureIncompleteRule(options = {}) {
|
|
|
1001
1262
|
}
|
|
1002
1263
|
}
|
|
1003
1264
|
if (options.requireEndedAtForStarted) {
|
|
1004
|
-
const missingEndedAt = context.
|
|
1265
|
+
const missingEndedAt = semanticEvents(context).filter(
|
|
1005
1266
|
(event) => event.startedAt !== void 0 && event.endedAt === void 0 && event.status !== "running"
|
|
1006
1267
|
);
|
|
1007
1268
|
if (missingEndedAt.length > 0) {
|
|
@@ -1027,8 +1288,8 @@ function createStructureOrphanRule(options = {}) {
|
|
|
1027
1288
|
category: "structure",
|
|
1028
1289
|
defaultSeverity: "error",
|
|
1029
1290
|
evaluate(context) {
|
|
1030
|
-
const byId = eventMap(context
|
|
1031
|
-
const orphans = context.
|
|
1291
|
+
const byId = eventMap(semanticEvents(context));
|
|
1292
|
+
const orphans = semanticEvents(context).filter((event) => {
|
|
1032
1293
|
if (!event.parentId || byId.has(event.parentId)) return false;
|
|
1033
1294
|
return !(allowMarkedUnresolved && parentMarkedUnresolved(event));
|
|
1034
1295
|
});
|
|
@@ -1051,10 +1312,10 @@ function createStructureCycleRule() {
|
|
|
1051
1312
|
category: "structure",
|
|
1052
1313
|
defaultSeverity: "error",
|
|
1053
1314
|
evaluate(context) {
|
|
1054
|
-
const byId = eventMap(context
|
|
1315
|
+
const byId = eventMap(semanticEvents(context));
|
|
1055
1316
|
const seenCycles = /* @__PURE__ */ new Set();
|
|
1056
1317
|
const findings = [];
|
|
1057
|
-
for (const event of [...context
|
|
1318
|
+
for (const event of [...semanticEvents(context)].sort((a, b) => a.eventId.localeCompare(b.eventId))) {
|
|
1058
1319
|
const path = [];
|
|
1059
1320
|
const seenAt = /* @__PURE__ */ new Map();
|
|
1060
1321
|
let current = event;
|
|
@@ -1092,10 +1353,10 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
1092
1353
|
category: "structure",
|
|
1093
1354
|
defaultSeverity: "error",
|
|
1094
1355
|
evaluate(context) {
|
|
1095
|
-
const byId = eventMap(context
|
|
1356
|
+
const byId = eventMap(semanticEvents(context));
|
|
1096
1357
|
const findings = [];
|
|
1097
1358
|
const minConfidence = options.minConfidence;
|
|
1098
|
-
for (const event of context
|
|
1359
|
+
for (const event of semanticEvents(context)) {
|
|
1099
1360
|
if (minConfidence && CONFIDENCE_RANK[event.confidence] < CONFIDENCE_RANK[minConfidence]) {
|
|
1100
1361
|
findings.push(
|
|
1101
1362
|
failFinding(
|
|
@@ -1163,7 +1424,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
1163
1424
|
defaultSeverity: "error",
|
|
1164
1425
|
evaluate(context) {
|
|
1165
1426
|
const findings = [];
|
|
1166
|
-
const byId = eventMap(context
|
|
1427
|
+
const byId = eventMap(semanticEvents(context));
|
|
1167
1428
|
if (options.maxChildren !== void 0) {
|
|
1168
1429
|
for (const [parentId, children] of context.childrenByParentId.entries()) {
|
|
1169
1430
|
if (children.length <= options.maxChildren) continue;
|
|
@@ -1190,7 +1451,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
1190
1451
|
}
|
|
1191
1452
|
}
|
|
1192
1453
|
if (options.maxConcurrent !== void 0) {
|
|
1193
|
-
const intervals = context.
|
|
1454
|
+
const intervals = semanticEvents(context).map((event) => ({ event, start: eventStartMs(event), end: eventEndMs(event) })).filter(
|
|
1194
1455
|
(item) => item.start !== void 0 && item.end !== void 0 && item.end > item.start
|
|
1195
1456
|
);
|
|
1196
1457
|
const points = intervals.flatMap((item) => [
|
|
@@ -1486,7 +1747,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
1486
1747
|
)
|
|
1487
1748
|
);
|
|
1488
1749
|
}
|
|
1489
|
-
if (
|
|
1750
|
+
if (isRecord2(entry.value) && options.maxObjectKeys !== void 0 && Object.keys(entry.value).length > options.maxObjectKeys) {
|
|
1490
1751
|
findings.push(
|
|
1491
1752
|
failFinding(
|
|
1492
1753
|
"safety.oversizedAttribute",
|
|
@@ -1793,7 +2054,9 @@ function contractToRules(contract) {
|
|
|
1793
2054
|
);
|
|
1794
2055
|
}
|
|
1795
2056
|
if (!allowIncomplete) {
|
|
1796
|
-
const running = context.events.filter(
|
|
2057
|
+
const running = (context.logicalEvents ?? context.events).filter(
|
|
2058
|
+
(event) => event.status === "running"
|
|
2059
|
+
);
|
|
1797
2060
|
if (running.length > 0) {
|
|
1798
2061
|
findings.push(
|
|
1799
2062
|
contractFailFinding(
|
|
@@ -1899,6 +2162,6 @@ function evaluateTraceContract(input, contract, options = {}) {
|
|
|
1899
2162
|
});
|
|
1900
2163
|
}
|
|
1901
2164
|
|
|
1902
|
-
export { createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, evaluateTraceContract, runTraceChecks };
|
|
1903
|
-
//# sourceMappingURL=chunk-
|
|
1904
|
-
//# sourceMappingURL=chunk-
|
|
2165
|
+
export { createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, evaluateTraceContract, projectLogicalEvents, resolveCanonicalToolName, runTraceChecks };
|
|
2166
|
+
//# sourceMappingURL=chunk-KJICERSD.mjs.map
|
|
2167
|
+
//# sourceMappingURL=chunk-KJICERSD.mjs.map
|