agent-inspect 6.3.0 → 6.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +2 -2
- package/package.json +2 -2
- package/packages/cli/dist/{chunk-YS2I722C.mjs → chunk-M6PC5KUT.mjs} +78 -31
- package/packages/cli/dist/chunk-M6PC5KUT.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +153 -42
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +88 -36
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-FVNE44UA.mjs → src-27LC2BB3.mjs} +8 -5
- package/packages/cli/dist/src-27LC2BB3.mjs.map +1 -0
- package/packages/core/dist/advanced.cjs +91 -35
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +36 -12
- package/packages/core/dist/advanced.d.ts +36 -12
- package/packages/core/dist/advanced.mjs +77 -24
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +113 -0
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +40 -599
- package/packages/core/dist/checks.d.ts +40 -599
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-QLYOCCEF.mjs → chunk-EWHN6NA4.mjs} +114 -3
- package/packages/core/dist/chunk-EWHN6NA4.mjs.map +1 -0
- package/packages/core/dist/{context-DyKbkmd4.d.cts → context-BHAgRjOe.d.cts} +4 -4
- package/packages/core/dist/{context-BJ5UQLDi.d.ts → context-Blyjb5my.d.ts} +4 -4
- package/packages/core/dist/diff.d.cts +3 -3
- package/packages/core/dist/diff.d.ts +3 -3
- package/packages/core/dist/exporters.cjs +75 -0
- package/packages/core/dist/exporters.cjs.map +1 -1
- package/packages/core/dist/exporters.d.cts +29 -4
- package/packages/core/dist/exporters.d.ts +29 -4
- package/packages/core/dist/exporters.mjs +71 -1
- package/packages/core/dist/exporters.mjs.map +1 -1
- package/packages/core/dist/index-BblJEdYZ.d.ts +618 -0
- package/packages/core/dist/index-DyKk7iR9.d.cts +618 -0
- package/packages/core/dist/index.d.cts +7 -7
- package/packages/core/dist/index.d.ts +7 -7
- package/packages/core/dist/{inspect-event-IqwOHbGo.d.cts → inspect-event-DVg84S4v.d.cts} +1 -1
- package/packages/core/dist/{inspect-event-IqwOHbGo.d.ts → inspect-event-DVg84S4v.d.ts} +1 -1
- package/packages/core/dist/{log-config-CW8w5YQ9.d.cts → log-config-BG1WmWf2.d.cts} +1 -1
- package/packages/core/dist/{log-config-SRA4xTp8.d.ts → log-config-Ck0aT_Ou.d.ts} +1 -1
- package/packages/core/dist/logs.d.cts +3 -3
- package/packages/core/dist/logs.d.ts +3 -3
- package/packages/core/dist/{persisted-inspect-event-DruwqARZ.d.ts → persisted-inspect-event-ChE7pGb7.d.ts} +1 -1
- package/packages/core/dist/{persisted-inspect-event-Gd-W8vB7.d.cts → persisted-inspect-event-d-mSP8kd.d.cts} +1 -1
- package/packages/core/dist/persisted.d.cts +5 -5
- package/packages/core/dist/persisted.d.ts +5 -5
- package/packages/core/dist/readers.d.cts +2 -2
- package/packages/core/dist/readers.d.ts +2 -2
- package/packages/core/dist/{types-J3Xtuf9x.d.cts → types-B562HgN_.d.cts} +1 -1
- package/packages/core/dist/{types-J3Xtuf9x.d.ts → types-B562HgN_.d.ts} +1 -1
- package/packages/core/dist/{types-B1X87dUW.d.cts → types-BLX3tiOW.d.cts} +1 -1
- package/packages/core/dist/{types-BVkjSd0q.d.ts → types-MuwXSJ4b.d.ts} +1 -1
- package/packages/core/dist/writers.d.cts +2 -2
- package/packages/core/dist/writers.d.ts +2 -2
- package/packages/cli/dist/chunk-YS2I722C.mjs.map +0 -1
- package/packages/cli/dist/src-FVNE44UA.mjs.map +0 -1
- package/packages/core/dist/chunk-QLYOCCEF.mjs.map +0 -1
|
@@ -1,618 +1,59 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
import
|
|
4
|
-
import
|
|
1
|
+
import { T as TraceCheckInput, a as TraceCheckResult } from './index-DyKk7iR9.cjs';
|
|
2
|
+
export { B as BaselineRegressionRuleOptions, D as DecisionRuleOptions, G as GuardrailRuleOptions, L as LlmUsageRuleOptions, M as MaxStepDurationRuleOptions, O as ObservedOutcomeRuleOptions, R as RetrievalRuleOptions, b as RunDepthRuleOptions, c as RunDurationRuleOptions, d as RunEventCountRuleOptions, e as RunStatusRuleOptions, f as RunTraceChecksOptions, S as SafetyOversizedAttributeRuleOptions, g as SafetyRawContentRuleOptions, h as SafetyRedactionRuleOptions, i as SafetySecretPattern, j as SafetySecretPatternRuleOptions, k as StallDetectionRuleOptions, l as StructureIncompleteRuleOptions, m as StructureOrphanRuleOptions, n as StructureParallelWidthRuleOptions, o as StructureRelationshipRuleOptions, p as ToolFailureRuleOptions, q as ToolOrderingRuleOptions, r as ToolUsageRuleOptions, s as TraceCheckContext, t as TraceCheckDiagnostic, u as TraceCheckDiagnosticCode, v as TraceCheckEvidence, w as TraceCheckFacts, x as TraceCheckFinding, y as TraceCheckFindingStatus, z as TraceCheckRule, A as TraceCheckRuleCategory, C as TraceCheckSeverity, E as TraceCheckStatus, F as TraceCheckSummary, H as TraceSignalRuleOptions, I as createBaselineRegressionRule, J as createDecisionRule, K as createGuardrailRule, N as createLlmUsageRule, P as createMaxStepDurationRule, Q as createObservedOutcomeRule, U as createRequireCompletedRule, V as createRetrievalRule, W as createRunDepthRule, X as createRunDurationRule, Y as createRunEventCountRule, Z as createRunStatusRule, _ as createSafetyOversizedAttributeRule, $ as createSafetyRawContentRule, a0 as createSafetyRedactionRule, a1 as createSafetySecretPatternRule, a2 as createStallDetectionRule, a3 as createStructureCycleRule, a4 as createStructureIncompleteRule, a5 as createStructureOrphanRule, a6 as createStructureParallelWidthRule, a7 as createStructureRelationshipRule, a8 as createToolFailureRule, a9 as createToolOrderingRule, aa as createToolUsageRule, ab as runTraceChecks } from './index-DyKk7iR9.cjs';
|
|
3
|
+
import './readers.cjs';
|
|
4
|
+
import './inspect-event-DVg84S4v.cjs';
|
|
5
|
+
import './persisted-inspect-event-d-mSP8kd.cjs';
|
|
6
|
+
import './types-B562HgN_.cjs';
|
|
5
7
|
|
|
6
8
|
/**
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
10
|
-
* evolve during the v1.x experimental period.
|
|
11
|
-
*/
|
|
12
|
-
type TraceCheckSeverity = "error" | "warning" | "info";
|
|
13
|
-
/**
|
|
14
|
-
* Experimental trace-check rule category.
|
|
15
|
-
*
|
|
16
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
17
|
-
* evolve during the v1.x experimental period.
|
|
18
|
-
*/
|
|
19
|
-
type TraceCheckRuleCategory = "run" | "tool" | "llm" | "structure" | "baseline" | "safety" | "reader";
|
|
20
|
-
/**
|
|
21
|
-
* Experimental trace-check finding status.
|
|
22
|
-
*
|
|
23
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
24
|
-
* evolve during the v1.x experimental period.
|
|
25
|
-
*/
|
|
26
|
-
type TraceCheckFindingStatus = "pass" | "fail" | "warning";
|
|
27
|
-
/**
|
|
28
|
-
* Experimental trace-check aggregate status.
|
|
29
|
-
*
|
|
30
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
31
|
-
* evolve during the v1.x experimental period.
|
|
32
|
-
*/
|
|
33
|
-
type TraceCheckStatus = "pass" | "fail" | "error";
|
|
34
|
-
/**
|
|
35
|
-
* Experimental stable diagnostic code for check execution and input errors.
|
|
36
|
-
*
|
|
37
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
38
|
-
* evolve during the v1.x experimental period.
|
|
39
|
-
*/
|
|
40
|
-
type TraceCheckDiagnosticCode = "AI_CHECK_INVALID_ARGUMENTS" | "AI_CHECK_INVALID_CONFIG" | "AI_CHECK_CONFIG_LOAD_FAILED" | "AI_CHECK_TRACE_UNREADABLE" | "AI_CHECK_UNSUPPORTED_FORMAT" | "AI_CHECK_AMBIGUOUS_FORMAT" | "AI_CHECK_RUN_SELECTION_REQUIRED" | "AI_CHECK_BASELINE_UNREADABLE" | "AI_CHECK_BASELINE_INCOMPATIBLE" | "AI_CHECK_RULE_FAILED" | "AI_CHECK_INTERNAL_ERROR";
|
|
41
|
-
/**
|
|
42
|
-
* Experimental evidence pointing at trace data relevant to a finding.
|
|
43
|
-
*
|
|
44
|
-
* Evidence intentionally identifies runs/events/spans and bounded paths rather
|
|
45
|
-
* than embedding raw prompts, outputs, request bodies, headers, or tool payloads.
|
|
46
|
-
*
|
|
47
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
48
|
-
* evolve during the v1.x experimental period.
|
|
49
|
-
*/
|
|
50
|
-
interface TraceCheckEvidence {
|
|
51
|
-
runId?: string;
|
|
52
|
-
eventId?: string;
|
|
53
|
-
parentId?: string;
|
|
54
|
-
traceId?: string;
|
|
55
|
-
spanId?: string;
|
|
56
|
-
kind?: string;
|
|
57
|
-
name?: string;
|
|
58
|
-
status?: string;
|
|
59
|
-
path?: string;
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Experimental finding emitted by a trace-check rule.
|
|
63
|
-
*
|
|
64
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
65
|
-
* evolve during the v1.x experimental period.
|
|
66
|
-
*/
|
|
67
|
-
interface TraceCheckFinding {
|
|
68
|
-
ruleId: string;
|
|
69
|
-
severity: TraceCheckSeverity;
|
|
70
|
-
status: TraceCheckFindingStatus;
|
|
71
|
-
message: string;
|
|
72
|
-
expected?: unknown;
|
|
73
|
-
actual?: unknown;
|
|
74
|
-
evidence: TraceCheckEvidence[];
|
|
75
|
-
}
|
|
76
|
-
/**
|
|
77
|
-
* Experimental diagnostic emitted when check execution cannot complete normally.
|
|
78
|
-
*
|
|
79
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
80
|
-
* evolve during the v1.x experimental period.
|
|
81
|
-
*/
|
|
82
|
-
interface TraceCheckDiagnostic {
|
|
83
|
-
code: TraceCheckDiagnosticCode;
|
|
84
|
-
message: string;
|
|
85
|
-
severity: TraceCheckSeverity;
|
|
86
|
-
ruleId?: string;
|
|
87
|
-
}
|
|
88
|
-
/**
|
|
89
|
-
* Experimental input projection for trace checks.
|
|
90
|
-
*
|
|
91
|
-
* The `read` value must come from `agent-inspect/readers` or an equivalent
|
|
92
|
-
* already-normalized `TraceReadResult`. The checks engine does not read files
|
|
93
|
-
* or reparse source payloads.
|
|
94
|
-
*
|
|
95
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
96
|
-
* evolve during the v1.x experimental period.
|
|
97
|
-
*/
|
|
98
|
-
interface TraceCheckInput {
|
|
99
|
-
read: TraceReadResult;
|
|
100
|
-
selectedRun?: InspectRunTree;
|
|
101
|
-
sourceLabel?: string;
|
|
102
|
-
}
|
|
103
|
-
/**
|
|
104
|
-
* Experimental normalized facts available to trace-check rules.
|
|
105
|
-
*
|
|
106
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
107
|
-
* evolve during the v1.x experimental period.
|
|
108
|
-
*/
|
|
109
|
-
interface TraceCheckFacts {
|
|
110
|
-
format: string;
|
|
111
|
-
runs: readonly InspectRunTree[];
|
|
112
|
-
events: readonly PersistedInspectEvent[];
|
|
113
|
-
readerWarnings: readonly TraceReadWarning[];
|
|
114
|
-
unsupportedFields: readonly string[];
|
|
115
|
-
sourceFiles: readonly string[];
|
|
116
|
-
nodesByEventId: ReadonlyMap<string, InspectNode>;
|
|
117
|
-
childrenByParentId: ReadonlyMap<string, readonly InspectNode[]>;
|
|
118
|
-
rootNodes: readonly InspectNode[];
|
|
119
|
-
}
|
|
120
|
-
/**
|
|
121
|
-
* Experimental rule evaluation context.
|
|
122
|
-
*
|
|
123
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
124
|
-
* evolve during the v1.x experimental period.
|
|
125
|
-
*/
|
|
126
|
-
interface TraceCheckContext extends TraceCheckFacts {
|
|
127
|
-
selectedRun?: InspectRunTree;
|
|
128
|
-
sourceLabel?: string;
|
|
129
|
-
}
|
|
130
|
-
/**
|
|
131
|
-
* Experimental trace-check rule definition.
|
|
132
|
-
*
|
|
133
|
-
* Rules are synchronous and pure: they receive normalized facts and return
|
|
134
|
-
* findings without reading files, mutating input, or performing network I/O.
|
|
135
|
-
*
|
|
136
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
137
|
-
* evolve during the v1.x experimental period.
|
|
138
|
-
*/
|
|
139
|
-
interface TraceCheckRule {
|
|
140
|
-
id: string;
|
|
141
|
-
category: TraceCheckRuleCategory;
|
|
142
|
-
defaultSeverity: TraceCheckSeverity;
|
|
143
|
-
evaluate(context: TraceCheckContext): readonly TraceCheckFinding[];
|
|
144
|
-
}
|
|
145
|
-
/**
|
|
146
|
-
* Experimental options for `runTraceChecks`.
|
|
147
|
-
*
|
|
148
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
149
|
-
* evolve during the v1.x experimental period.
|
|
150
|
-
*/
|
|
151
|
-
interface RunTraceChecksOptions {
|
|
152
|
-
rules?: readonly TraceCheckRule[];
|
|
153
|
-
select?: readonly string[];
|
|
154
|
-
runId?: string;
|
|
155
|
-
}
|
|
156
|
-
/**
|
|
157
|
-
* Experimental options for the built-in run status rule.
|
|
158
|
-
*
|
|
159
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
160
|
-
* evolve during the v1.x experimental period.
|
|
161
|
-
*/
|
|
162
|
-
interface RunStatusRuleOptions {
|
|
163
|
-
expected?: "ok" | "error" | "running";
|
|
164
|
-
allowIncomplete?: boolean;
|
|
165
|
-
}
|
|
166
|
-
/**
|
|
167
|
-
* Experimental options for the built-in run duration rule.
|
|
168
|
-
*
|
|
169
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
170
|
-
* evolve during the v1.x experimental period.
|
|
171
|
-
*/
|
|
172
|
-
interface RunDurationRuleOptions {
|
|
173
|
-
maxDurationMs: number;
|
|
174
|
-
}
|
|
175
|
-
/**
|
|
176
|
-
* Experimental options for the built-in max step duration rule.
|
|
177
|
-
*/
|
|
178
|
-
interface MaxStepDurationRuleOptions {
|
|
179
|
-
maxDurationMs: number;
|
|
180
|
-
}
|
|
181
|
-
/**
|
|
182
|
-
* Experimental options for stall detection (running / incomplete events).
|
|
183
|
-
*/
|
|
184
|
-
interface StallDetectionRuleOptions {
|
|
185
|
-
/** When true, events with startedAt but no endedAt also fail. */
|
|
186
|
-
requireEndedAt?: boolean;
|
|
187
|
-
}
|
|
188
|
-
/**
|
|
189
|
-
* Experimental options for the built-in event count rule.
|
|
190
|
-
*
|
|
191
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
192
|
-
* evolve during the v1.x experimental period.
|
|
193
|
-
*/
|
|
194
|
-
interface RunEventCountRuleOptions {
|
|
195
|
-
kind?: PersistedInspectEvent["kind"];
|
|
196
|
-
min?: number;
|
|
197
|
-
max?: number;
|
|
198
|
-
}
|
|
199
|
-
/**
|
|
200
|
-
* Experimental options for the built-in run depth rule.
|
|
201
|
-
*
|
|
202
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
203
|
-
* evolve during the v1.x experimental period.
|
|
204
|
-
*/
|
|
205
|
-
interface RunDepthRuleOptions {
|
|
206
|
-
maxDepth: number;
|
|
207
|
-
}
|
|
208
|
-
/**
|
|
209
|
-
* Experimental options for the built-in tool usage rule.
|
|
210
|
-
*
|
|
211
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
212
|
-
* evolve during the v1.x experimental period.
|
|
213
|
-
*/
|
|
214
|
-
interface ToolUsageRuleOptions {
|
|
215
|
-
required?: readonly string[];
|
|
216
|
-
forbidden?: readonly string[];
|
|
217
|
-
allowed?: readonly string[];
|
|
218
|
-
minCount?: number;
|
|
219
|
-
maxCount?: number;
|
|
220
|
-
}
|
|
221
|
-
/**
|
|
222
|
-
* Experimental options for the built-in tool ordering rule.
|
|
223
|
-
*
|
|
224
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
225
|
-
* evolve during the v1.x experimental period.
|
|
9
|
+
* @experimental Typed trace contract input. Evolves during v6.5.x.
|
|
226
10
|
*/
|
|
227
|
-
interface
|
|
228
|
-
|
|
229
|
-
|
|
11
|
+
interface TraceContractRunRules {
|
|
12
|
+
requireCompleted?: boolean;
|
|
13
|
+
allowedStatuses?: string[];
|
|
14
|
+
maxDurationMs?: number;
|
|
230
15
|
}
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
interface ToolFailureRuleOptions {
|
|
238
|
-
maxFailures?: number;
|
|
239
|
-
maxRetries?: number;
|
|
16
|
+
interface TraceContractToolRules {
|
|
17
|
+
required?: string[];
|
|
18
|
+
forbidden?: string[];
|
|
19
|
+
allowed?: string[];
|
|
20
|
+
maxCalls?: number;
|
|
21
|
+
requiredOrder?: string[];
|
|
240
22
|
}
|
|
241
|
-
|
|
242
|
-
* Experimental options for the built-in LLM usage rule.
|
|
243
|
-
*
|
|
244
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
245
|
-
* evolve during the v1.x experimental period.
|
|
246
|
-
*/
|
|
247
|
-
interface LlmUsageRuleOptions {
|
|
248
|
-
allowedModels?: readonly string[];
|
|
249
|
-
allowedProviders?: readonly string[];
|
|
23
|
+
interface TraceContractLlmRules {
|
|
250
24
|
maxCalls?: number;
|
|
251
|
-
maxInputTokens?: number;
|
|
252
|
-
maxOutputTokens?: number;
|
|
253
25
|
maxTotalTokens?: number;
|
|
254
|
-
|
|
255
|
-
finishReasons?: readonly string[];
|
|
256
|
-
}
|
|
257
|
-
/**
|
|
258
|
-
* Experimental options for the built-in structure incomplete rule.
|
|
259
|
-
*
|
|
260
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
261
|
-
* evolve during the v1.x experimental period.
|
|
262
|
-
*/
|
|
263
|
-
interface StructureIncompleteRuleOptions {
|
|
264
|
-
allowRunning?: boolean;
|
|
265
|
-
requireEndedAtForStarted?: boolean;
|
|
266
|
-
}
|
|
267
|
-
/**
|
|
268
|
-
* Experimental options for the built-in structure orphan rule.
|
|
269
|
-
*
|
|
270
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
271
|
-
* evolve during the v1.x experimental period.
|
|
272
|
-
*/
|
|
273
|
-
interface StructureOrphanRuleOptions {
|
|
274
|
-
allowMarkedUnresolved?: boolean;
|
|
275
|
-
}
|
|
276
|
-
/**
|
|
277
|
-
* Experimental options for the built-in structure relationship rule.
|
|
278
|
-
*
|
|
279
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
280
|
-
* evolve during the v1.x experimental period.
|
|
281
|
-
*/
|
|
282
|
-
interface StructureRelationshipRuleOptions {
|
|
283
|
-
minConfidence?: AttributionConfidence;
|
|
284
|
-
requireParentBeforeChild?: boolean;
|
|
285
|
-
requireTraceParentSpan?: boolean;
|
|
286
|
-
}
|
|
287
|
-
/**
|
|
288
|
-
* Experimental options for the built-in structure parallel-width rule.
|
|
289
|
-
*
|
|
290
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
291
|
-
* evolve during the v1.x experimental period.
|
|
292
|
-
*/
|
|
293
|
-
interface StructureParallelWidthRuleOptions {
|
|
294
|
-
maxChildren?: number;
|
|
295
|
-
maxConcurrent?: number;
|
|
296
|
-
}
|
|
297
|
-
/**
|
|
298
|
-
* Experimental options shared by built-in signal rules.
|
|
299
|
-
*
|
|
300
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
301
|
-
* evolve during the v1.x experimental period.
|
|
302
|
-
*/
|
|
303
|
-
interface TraceSignalRuleOptions {
|
|
304
|
-
required?: readonly string[];
|
|
305
|
-
forbidden?: readonly string[];
|
|
306
|
-
allowed?: readonly string[];
|
|
307
|
-
minCount?: number;
|
|
308
|
-
maxCount?: number;
|
|
309
|
-
}
|
|
310
|
-
/**
|
|
311
|
-
* Experimental options for the built-in retrieval signal rule.
|
|
312
|
-
*
|
|
313
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
314
|
-
* evolve during the v1.x experimental period.
|
|
315
|
-
*/
|
|
316
|
-
interface RetrievalRuleOptions extends TraceSignalRuleOptions {
|
|
26
|
+
allowedModels?: string[];
|
|
317
27
|
}
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
322
|
-
* evolve during the v1.x experimental period.
|
|
323
|
-
*/
|
|
324
|
-
interface GuardrailRuleOptions extends TraceSignalRuleOptions {
|
|
325
|
-
}
|
|
326
|
-
/**
|
|
327
|
-
* Experimental options for the built-in decision signal rule.
|
|
328
|
-
*
|
|
329
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
330
|
-
* evolve during the v1.x experimental period.
|
|
331
|
-
*/
|
|
332
|
-
interface DecisionRuleOptions extends TraceSignalRuleOptions {
|
|
333
|
-
}
|
|
334
|
-
/**
|
|
335
|
-
* Experimental options for the built-in safety redaction rule.
|
|
336
|
-
*
|
|
337
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
338
|
-
* evolve during the v1.x experimental period.
|
|
339
|
-
*/
|
|
340
|
-
interface SafetyRedactionRuleOptions {
|
|
341
|
-
sensitiveKeys?: readonly string[];
|
|
342
|
-
redactedMarkers?: readonly string[];
|
|
343
|
-
maxFindings?: number;
|
|
344
|
-
}
|
|
345
|
-
/**
|
|
346
|
-
* Experimental options for the built-in raw content path rule.
|
|
347
|
-
*
|
|
348
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
349
|
-
* evolve during the v1.x experimental period.
|
|
350
|
-
*/
|
|
351
|
-
interface SafetyRawContentRuleOptions {
|
|
352
|
-
forbiddenKeys?: readonly string[];
|
|
353
|
-
includeSummaries?: boolean;
|
|
354
|
-
maxFindings?: number;
|
|
28
|
+
interface TraceContractObservationRules {
|
|
29
|
+
required?: string[];
|
|
30
|
+
failOn?: Array<"failed" | "unknown" | "skipped">;
|
|
355
31
|
}
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
*/
|
|
362
|
-
interface SafetySecretPattern {
|
|
363
|
-
id: string;
|
|
364
|
-
pattern: RegExp;
|
|
365
|
-
}
|
|
366
|
-
/**
|
|
367
|
-
* Experimental options for the built-in secret pattern safety rule.
|
|
368
|
-
*
|
|
369
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
370
|
-
* evolve during the v1.x experimental period.
|
|
371
|
-
*/
|
|
372
|
-
interface SafetySecretPatternRuleOptions {
|
|
373
|
-
patterns?: readonly SafetySecretPattern[];
|
|
374
|
-
maxStringLength?: number;
|
|
375
|
-
maxFindings?: number;
|
|
32
|
+
interface TraceContractInput {
|
|
33
|
+
run?: TraceContractRunRules;
|
|
34
|
+
tools?: TraceContractToolRules;
|
|
35
|
+
llm?: TraceContractLlmRules;
|
|
36
|
+
observations?: TraceContractObservationRules;
|
|
376
37
|
}
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
*/
|
|
383
|
-
interface SafetyOversizedAttributeRuleOptions {
|
|
384
|
-
maxStringLength?: number;
|
|
385
|
-
maxArrayLength?: number;
|
|
386
|
-
maxObjectKeys?: number;
|
|
387
|
-
maxSerializedBytes?: number;
|
|
388
|
-
maxFindings?: number;
|
|
38
|
+
interface TraceContract {
|
|
39
|
+
run?: TraceContractRunRules;
|
|
40
|
+
tools?: TraceContractToolRules;
|
|
41
|
+
llm?: TraceContractLlmRules;
|
|
42
|
+
observations?: TraceContractObservationRules;
|
|
389
43
|
}
|
|
390
44
|
/**
|
|
391
|
-
*
|
|
45
|
+
* Define a normalized trace contract object.
|
|
392
46
|
*
|
|
393
|
-
*
|
|
394
|
-
* equivalent `TraceReadResult`; this rule does not read files or parse traces.
|
|
395
|
-
* `durationToleranceMs` defaults to `0`, meaning exact duration comparison.
|
|
396
|
-
*
|
|
397
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
398
|
-
* evolve during the v1.x experimental period.
|
|
47
|
+
* @experimental
|
|
399
48
|
*/
|
|
400
|
-
|
|
401
|
-
baseline: TraceCheckInput;
|
|
402
|
-
baselineRunId?: string;
|
|
403
|
-
durationToleranceMs?: number;
|
|
404
|
-
compareFormat?: boolean;
|
|
405
|
-
}
|
|
49
|
+
declare function defineTraceContract(input: TraceContractInput): TraceContract;
|
|
406
50
|
/**
|
|
407
|
-
*
|
|
51
|
+
* Evaluate a trace contract against an opened trace read result.
|
|
408
52
|
*
|
|
409
|
-
* @experimental
|
|
410
|
-
* evolve during the v1.x experimental period.
|
|
53
|
+
* @experimental
|
|
411
54
|
*/
|
|
412
|
-
|
|
413
|
-
passed: number;
|
|
414
|
-
failed: number;
|
|
415
|
-
warnings: number;
|
|
416
|
-
errors: number;
|
|
417
|
-
}
|
|
418
|
-
/**
|
|
419
|
-
* Experimental trace-check result.
|
|
420
|
-
*
|
|
421
|
-
* `status: "fail"` means rules ran and at least one error-severity finding
|
|
422
|
-
* failed. `status: "error"` means execution could not complete because of
|
|
423
|
-
* invalid input, invalid rule selection, or a thrown rule error.
|
|
424
|
-
*
|
|
425
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
426
|
-
* evolve during the v1.x experimental period.
|
|
427
|
-
*/
|
|
428
|
-
interface TraceCheckResult {
|
|
429
|
-
ok: boolean;
|
|
430
|
-
status: TraceCheckStatus;
|
|
431
|
-
format: string;
|
|
55
|
+
declare function evaluateTraceContract(input: TraceCheckInput, contract: TraceContract, options?: {
|
|
432
56
|
runId?: string;
|
|
433
|
-
|
|
434
|
-
findings: TraceCheckFinding[];
|
|
435
|
-
diagnostics: TraceCheckDiagnostic[];
|
|
436
|
-
}
|
|
437
|
-
/**
|
|
438
|
-
* Create the experimental built-in run status rule.
|
|
439
|
-
*
|
|
440
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
441
|
-
* evolve during the v1.x experimental period.
|
|
442
|
-
*/
|
|
443
|
-
declare function createRunStatusRule(options?: RunStatusRuleOptions): TraceCheckRule;
|
|
444
|
-
/**
|
|
445
|
-
* Create the experimental built-in run duration rule.
|
|
446
|
-
*
|
|
447
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
448
|
-
* evolve during the v1.x experimental period.
|
|
449
|
-
*/
|
|
450
|
-
declare function createRunDurationRule(options: RunDurationRuleOptions): TraceCheckRule;
|
|
451
|
-
/**
|
|
452
|
-
* Fail when any step duration exceeds the configured maximum.
|
|
453
|
-
*/
|
|
454
|
-
declare function createMaxStepDurationRule(options: MaxStepDurationRuleOptions): TraceCheckRule;
|
|
455
|
-
/**
|
|
456
|
-
* Detect stalled runs: running events and optionally started-but-unended events.
|
|
457
|
-
*/
|
|
458
|
-
declare function createStallDetectionRule(options?: StallDetectionRuleOptions): TraceCheckRule;
|
|
459
|
-
/**
|
|
460
|
-
* Require a completed run with no running events.
|
|
461
|
-
*/
|
|
462
|
-
declare function createRequireCompletedRule(): TraceCheckRule;
|
|
463
|
-
/**
|
|
464
|
-
* Create the experimental built-in event count rule.
|
|
465
|
-
*
|
|
466
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
467
|
-
* evolve during the v1.x experimental period.
|
|
468
|
-
*/
|
|
469
|
-
declare function createRunEventCountRule(options: RunEventCountRuleOptions): TraceCheckRule;
|
|
470
|
-
/**
|
|
471
|
-
* Create the experimental built-in run depth rule.
|
|
472
|
-
*
|
|
473
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
474
|
-
* evolve during the v1.x experimental period.
|
|
475
|
-
*/
|
|
476
|
-
declare function createRunDepthRule(options: RunDepthRuleOptions): TraceCheckRule;
|
|
477
|
-
/**
|
|
478
|
-
* Create the experimental built-in tool usage rule.
|
|
479
|
-
*
|
|
480
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
481
|
-
* evolve during the v1.x experimental period.
|
|
482
|
-
*/
|
|
483
|
-
declare function createToolUsageRule(options: ToolUsageRuleOptions): TraceCheckRule;
|
|
484
|
-
/**
|
|
485
|
-
* Create the experimental built-in tool ordering rule.
|
|
486
|
-
*
|
|
487
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
488
|
-
* evolve during the v1.x experimental period.
|
|
489
|
-
*/
|
|
490
|
-
declare function createToolOrderingRule(options: ToolOrderingRuleOptions): TraceCheckRule;
|
|
491
|
-
/**
|
|
492
|
-
* Create the experimental built-in tool failure/retry rule.
|
|
493
|
-
*
|
|
494
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
495
|
-
* evolve during the v1.x experimental period.
|
|
496
|
-
*/
|
|
497
|
-
declare function createToolFailureRule(options: ToolFailureRuleOptions): TraceCheckRule;
|
|
498
|
-
/**
|
|
499
|
-
* Create the experimental built-in LLM usage rule.
|
|
500
|
-
*
|
|
501
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
502
|
-
* evolve during the v1.x experimental period.
|
|
503
|
-
*/
|
|
504
|
-
declare function createLlmUsageRule(options: LlmUsageRuleOptions): TraceCheckRule;
|
|
505
|
-
/**
|
|
506
|
-
* Create the experimental built-in structure incomplete rule.
|
|
507
|
-
*
|
|
508
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
509
|
-
* evolve during the v1.x experimental period.
|
|
510
|
-
*/
|
|
511
|
-
declare function createStructureIncompleteRule(options?: StructureIncompleteRuleOptions): TraceCheckRule;
|
|
512
|
-
/**
|
|
513
|
-
* Create the experimental built-in structure orphan rule.
|
|
514
|
-
*
|
|
515
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
516
|
-
* evolve during the v1.x experimental period.
|
|
517
|
-
*/
|
|
518
|
-
declare function createStructureOrphanRule(options?: StructureOrphanRuleOptions): TraceCheckRule;
|
|
519
|
-
/**
|
|
520
|
-
* Create the experimental built-in structure cycle rule.
|
|
521
|
-
*
|
|
522
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
523
|
-
* evolve during the v1.x experimental period.
|
|
524
|
-
*/
|
|
525
|
-
declare function createStructureCycleRule(): TraceCheckRule;
|
|
526
|
-
/**
|
|
527
|
-
* Create the experimental built-in structure relationship rule.
|
|
528
|
-
*
|
|
529
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
530
|
-
* evolve during the v1.x experimental period.
|
|
531
|
-
*/
|
|
532
|
-
declare function createStructureRelationshipRule(options?: StructureRelationshipRuleOptions): TraceCheckRule;
|
|
533
|
-
/**
|
|
534
|
-
* Create the experimental built-in structure parallel-width rule.
|
|
535
|
-
*
|
|
536
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
537
|
-
* evolve during the v1.x experimental period.
|
|
538
|
-
*/
|
|
539
|
-
declare function createStructureParallelWidthRule(options: StructureParallelWidthRuleOptions): TraceCheckRule;
|
|
540
|
-
/**
|
|
541
|
-
* Create the experimental built-in retrieval signal rule.
|
|
542
|
-
*
|
|
543
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
544
|
-
* evolve during the v1.x experimental period.
|
|
545
|
-
*/
|
|
546
|
-
declare function createRetrievalRule(options: RetrievalRuleOptions): TraceCheckRule;
|
|
547
|
-
/**
|
|
548
|
-
* Create the experimental built-in guardrail signal rule.
|
|
549
|
-
*
|
|
550
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
551
|
-
* evolve during the v1.x experimental period.
|
|
552
|
-
*/
|
|
553
|
-
declare function createGuardrailRule(options: GuardrailRuleOptions): TraceCheckRule;
|
|
554
|
-
/**
|
|
555
|
-
* Create the experimental built-in decision signal rule.
|
|
556
|
-
*
|
|
557
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
558
|
-
* evolve during the v1.x experimental period.
|
|
559
|
-
*/
|
|
560
|
-
declare function createDecisionRule(options: DecisionRuleOptions): TraceCheckRule;
|
|
561
|
-
/**
|
|
562
|
-
* Create the experimental built-in safety redaction rule.
|
|
563
|
-
*
|
|
564
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
565
|
-
* evolve during the v1.x experimental period.
|
|
566
|
-
*/
|
|
567
|
-
declare function createSafetyRedactionRule(options?: SafetyRedactionRuleOptions): TraceCheckRule;
|
|
568
|
-
/**
|
|
569
|
-
* Create the experimental built-in raw content path safety rule.
|
|
570
|
-
*
|
|
571
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
572
|
-
* evolve during the v1.x experimental period.
|
|
573
|
-
*/
|
|
574
|
-
declare function createSafetyRawContentRule(options?: SafetyRawContentRuleOptions): TraceCheckRule;
|
|
575
|
-
/**
|
|
576
|
-
* Create the experimental built-in secret pattern safety rule.
|
|
577
|
-
*
|
|
578
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
579
|
-
* evolve during the v1.x experimental period.
|
|
580
|
-
*/
|
|
581
|
-
declare function createSafetySecretPatternRule(options?: SafetySecretPatternRuleOptions): TraceCheckRule;
|
|
582
|
-
/**
|
|
583
|
-
* Create the experimental built-in oversized attribute safety rule.
|
|
584
|
-
*
|
|
585
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
586
|
-
* evolve during the v1.x experimental period.
|
|
587
|
-
*/
|
|
588
|
-
declare function createSafetyOversizedAttributeRule(options: SafetyOversizedAttributeRuleOptions): TraceCheckRule;
|
|
589
|
-
/**
|
|
590
|
-
* Create the experimental built-in baseline regression rule.
|
|
591
|
-
*
|
|
592
|
-
* The rule compares safe structural summaries from a normalized baseline
|
|
593
|
-
* against the candidate context. It intentionally ignores raw prompt/output,
|
|
594
|
-
* request/response body, header, and tool payload text.
|
|
595
|
-
*
|
|
596
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
597
|
-
* evolve during the v1.x experimental period.
|
|
598
|
-
*/
|
|
599
|
-
declare function createBaselineRegressionRule(options: BaselineRegressionRuleOptions): TraceCheckRule;
|
|
600
|
-
/**
|
|
601
|
-
* Experimental rule for observed real-world outcomes (v4.4+).
|
|
602
|
-
*/
|
|
603
|
-
interface ObservedOutcomeRuleOptions {
|
|
604
|
-
failOn?: readonly ObservedOutcomeStatus[];
|
|
605
|
-
}
|
|
606
|
-
declare function createObservedOutcomeRule(options?: ObservedOutcomeRuleOptions): TraceCheckRule;
|
|
607
|
-
/**
|
|
608
|
-
* Run experimental deterministic trace checks against normalized reader output.
|
|
609
|
-
*
|
|
610
|
-
* The engine is local-only and pure: it does not discover config files, read
|
|
611
|
-
* trace files, call providers, perform network I/O, or mutate input objects.
|
|
612
|
-
*
|
|
613
|
-
* @experimental Available through `agent-inspect/checks`; the checks API may
|
|
614
|
-
* evolve during the v1.x experimental period.
|
|
615
|
-
*/
|
|
616
|
-
declare function runTraceChecks(input: TraceCheckInput, options?: RunTraceChecksOptions): TraceCheckResult;
|
|
57
|
+
}): TraceCheckResult;
|
|
617
58
|
|
|
618
|
-
export {
|
|
59
|
+
export { TraceCheckInput, TraceCheckResult, type TraceContract, type TraceContractInput, type TraceContractLlmRules, type TraceContractObservationRules, type TraceContractRunRules, type TraceContractToolRules, defineTraceContract, evaluateTraceContract };
|