agent-inspect 6.17.4 → 6.17.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +1 -1
- package/docs/ADAPTERS.md +3 -3
- package/docs/API.md +13 -7
- package/docs/CLI.md +2 -0
- package/docs/LIMITATIONS.md +1 -0
- package/docs/TRACE-CONTRACTS.md +90 -0
- package/package.json +5 -2
- package/packages/cli/dist/{chunk-YX6RFLV5.mjs → chunk-GOHYSE7W.mjs} +86 -17
- package/packages/cli/dist/chunk-GOHYSE7W.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +480 -80
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +399 -68
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-X6SLMBNH.mjs → src-OYGG7QAZ.mjs} +3 -3
- package/packages/cli/dist/{src-X6SLMBNH.mjs.map → src-OYGG7QAZ.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +85 -16
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +1 -1
- package/packages/core/dist/advanced.d.ts +1 -1
- package/packages/core/dist/advanced.mjs +15 -5
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +119 -26
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +28 -2
- package/packages/core/dist/checks.d.ts +28 -2
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-HN377P23.mjs → chunk-DIZPIPY2.mjs} +121 -28
- package/packages/core/dist/chunk-DIZPIPY2.mjs.map +1 -0
- package/packages/core/dist/{index-Xk9X-yjY.d.cts → index-DWu54Y28.d.cts} +107 -122
- package/packages/core/dist/{index-BsCcOKxy.d.ts → index-DlwbVqEs.d.ts} +107 -122
- package/packages/cli/dist/chunk-YX6RFLV5.mjs.map +0 -1
- package/packages/core/dist/chunk-HN377P23.mjs.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { a as TraceReadWarning, T as TraceReadResult } from './index-B9ZGvUVL.js';
|
|
2
2
|
import { a as InspectRunTree, b as InspectNode, A as AttributionConfidence } from './inspect-event-DRthdZlf.js';
|
|
3
3
|
import { P as PersistedInspectEvent } from './persisted-inspect-event-BVucFVrP.js';
|
|
4
4
|
import { O as ObservedOutcomeStatus } from './types-B562HgN_.js';
|
|
@@ -62,15 +62,13 @@ declare function resolveCanonicalToolName(event: PersistedInspectEvent): string;
|
|
|
62
62
|
/**
|
|
63
63
|
* Experimental trace-check finding severity.
|
|
64
64
|
*
|
|
65
|
-
* @experimental
|
|
66
|
-
* evolve during the v1.x experimental period.
|
|
65
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
67
66
|
*/
|
|
68
67
|
type TraceCheckSeverity = "error" | "warning" | "info";
|
|
69
68
|
/**
|
|
70
69
|
* Experimental trace-check rule category.
|
|
71
70
|
*
|
|
72
|
-
* @experimental
|
|
73
|
-
* evolve during the v1.x experimental period.
|
|
71
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
74
72
|
*/
|
|
75
73
|
type TraceCheckRuleCategory = "run" | "tool" | "llm" | "structure" | "baseline" | "safety" | "reader";
|
|
76
74
|
/**
|
|
@@ -90,32 +88,28 @@ type SafetyFindingConfidence = "high" | "medium" | "low";
|
|
|
90
88
|
/**
|
|
91
89
|
* Experimental trace-check finding status.
|
|
92
90
|
*
|
|
93
|
-
* @experimental
|
|
94
|
-
* evolve during the v1.x experimental period.
|
|
91
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
95
92
|
*/
|
|
96
93
|
type TraceCheckFindingStatus = "pass" | "fail" | "warning";
|
|
97
94
|
/**
|
|
98
95
|
* Experimental trace-check aggregate status.
|
|
99
96
|
*
|
|
100
|
-
* @experimental
|
|
101
|
-
* evolve during the v1.x experimental period.
|
|
97
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
102
98
|
*/
|
|
103
99
|
type TraceCheckStatus = "pass" | "fail" | "error";
|
|
104
100
|
/**
|
|
105
101
|
* Experimental stable diagnostic code for check execution and input errors.
|
|
106
102
|
*
|
|
107
|
-
* @experimental
|
|
108
|
-
* evolve during the v1.x experimental period.
|
|
103
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
109
104
|
*/
|
|
110
|
-
type TraceCheckDiagnosticCode = "AI_CHECK_INVALID_ARGUMENTS" | "AI_CHECK_INVALID_CONFIG" | "AI_CHECK_CONFIG_LOAD_FAILED" | "AI_CHECK_TRACE_UNREADABLE" | "AI_CHECK_UNSUPPORTED_FORMAT" | "AI_CHECK_AMBIGUOUS_FORMAT" | "AI_CHECK_RUN_SELECTION_REQUIRED" | "AI_CHECK_BASELINE_UNREADABLE" | "AI_CHECK_BASELINE_INCOMPATIBLE" | "AI_CHECK_RULE_FAILED" | "AI_CHECK_INTERNAL_ERROR";
|
|
105
|
+
type TraceCheckDiagnosticCode = "AI_CHECK_INVALID_ARGUMENTS" | "AI_CHECK_INVALID_CONFIG" | "AI_CHECK_CONFIG_LOAD_FAILED" | "AI_CHECK_CONFIG_UNKNOWN_KEY" | "AI_CHECK_CONFIG_INVALID_VALUE" | "AI_CHECK_CONFIG_NO_EFFECTIVE_RULES" | "AI_CHECK_NO_RULES_EVALUATED" | "AI_CHECK_TRACE_UNREADABLE" | "AI_CHECK_UNSUPPORTED_FORMAT" | "AI_CHECK_AMBIGUOUS_FORMAT" | "AI_CHECK_RUN_SELECTION_REQUIRED" | "AI_CHECK_BASELINE_UNREADABLE" | "AI_CHECK_BASELINE_INCOMPATIBLE" | "AI_CHECK_RULE_FAILED" | "AI_CHECK_INTERNAL_ERROR";
|
|
111
106
|
/**
|
|
112
107
|
* Experimental evidence pointing at trace data relevant to a finding.
|
|
113
108
|
*
|
|
114
109
|
* Evidence intentionally identifies runs/events/spans and bounded paths rather
|
|
115
110
|
* than embedding raw prompts, outputs, request bodies, headers, or tool payloads.
|
|
116
111
|
*
|
|
117
|
-
* @experimental
|
|
118
|
-
* evolve during the v1.x experimental period.
|
|
112
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
119
113
|
*/
|
|
120
114
|
interface TraceCheckEvidence {
|
|
121
115
|
runId?: string;
|
|
@@ -131,8 +125,7 @@ interface TraceCheckEvidence {
|
|
|
131
125
|
/**
|
|
132
126
|
* Experimental finding emitted by a trace-check rule.
|
|
133
127
|
*
|
|
134
|
-
* @experimental
|
|
135
|
-
* evolve during the v1.x experimental period.
|
|
128
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
136
129
|
*/
|
|
137
130
|
interface TraceCheckFinding {
|
|
138
131
|
ruleId: string;
|
|
@@ -170,8 +163,7 @@ interface TraceCheckFinding {
|
|
|
170
163
|
/**
|
|
171
164
|
* Experimental diagnostic emitted when check execution cannot complete normally.
|
|
172
165
|
*
|
|
173
|
-
* @experimental
|
|
174
|
-
* evolve during the v1.x experimental period.
|
|
166
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
175
167
|
*/
|
|
176
168
|
interface TraceCheckDiagnostic {
|
|
177
169
|
code: TraceCheckDiagnosticCode;
|
|
@@ -186,8 +178,7 @@ interface TraceCheckDiagnostic {
|
|
|
186
178
|
* already-normalized `TraceReadResult`. The checks engine does not read files
|
|
187
179
|
* or reparse source payloads.
|
|
188
180
|
*
|
|
189
|
-
* @experimental
|
|
190
|
-
* evolve during the v1.x experimental period.
|
|
181
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
191
182
|
*/
|
|
192
183
|
interface TraceCheckInput {
|
|
193
184
|
read: TraceReadResult;
|
|
@@ -197,8 +188,7 @@ interface TraceCheckInput {
|
|
|
197
188
|
/**
|
|
198
189
|
* Experimental normalized facts available to trace-check rules.
|
|
199
190
|
*
|
|
200
|
-
* @experimental
|
|
201
|
-
* evolve during the v1.x experimental period.
|
|
191
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
202
192
|
*/
|
|
203
193
|
interface TraceCheckFacts {
|
|
204
194
|
format: string;
|
|
@@ -229,8 +219,7 @@ interface TraceCheckFacts {
|
|
|
229
219
|
/**
|
|
230
220
|
* Experimental rule evaluation context.
|
|
231
221
|
*
|
|
232
|
-
* @experimental
|
|
233
|
-
* evolve during the v1.x experimental period.
|
|
222
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
234
223
|
*/
|
|
235
224
|
interface TraceCheckContext extends TraceCheckFacts {
|
|
236
225
|
selectedRun?: InspectRunTree;
|
|
@@ -242,8 +231,7 @@ interface TraceCheckContext extends TraceCheckFacts {
|
|
|
242
231
|
* Rules are synchronous and pure: they receive normalized facts and return
|
|
243
232
|
* findings without reading files, mutating input, or performing network I/O.
|
|
244
233
|
*
|
|
245
|
-
* @experimental
|
|
246
|
-
* evolve during the v1.x experimental period.
|
|
234
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
247
235
|
*/
|
|
248
236
|
interface TraceCheckRule {
|
|
249
237
|
id: string;
|
|
@@ -254,8 +242,7 @@ interface TraceCheckRule {
|
|
|
254
242
|
/**
|
|
255
243
|
* Experimental options for `runTraceChecks`.
|
|
256
244
|
*
|
|
257
|
-
* @experimental
|
|
258
|
-
* evolve during the v1.x experimental period.
|
|
245
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
259
246
|
*/
|
|
260
247
|
interface RunTraceChecksOptions {
|
|
261
248
|
rules?: readonly TraceCheckRule[];
|
|
@@ -265,8 +252,7 @@ interface RunTraceChecksOptions {
|
|
|
265
252
|
/**
|
|
266
253
|
* Experimental options for the built-in run status rule.
|
|
267
254
|
*
|
|
268
|
-
* @experimental
|
|
269
|
-
* evolve during the v1.x experimental period.
|
|
255
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
270
256
|
*/
|
|
271
257
|
interface RunStatusRuleOptions {
|
|
272
258
|
expected?: "ok" | "error" | "running";
|
|
@@ -275,8 +261,7 @@ interface RunStatusRuleOptions {
|
|
|
275
261
|
/**
|
|
276
262
|
* Experimental options for the built-in run duration rule.
|
|
277
263
|
*
|
|
278
|
-
* @experimental
|
|
279
|
-
* evolve during the v1.x experimental period.
|
|
264
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
280
265
|
*/
|
|
281
266
|
interface RunDurationRuleOptions {
|
|
282
267
|
maxDurationMs: number;
|
|
@@ -297,8 +282,7 @@ interface StallDetectionRuleOptions {
|
|
|
297
282
|
/**
|
|
298
283
|
* Experimental options for the built-in event count rule.
|
|
299
284
|
*
|
|
300
|
-
* @experimental
|
|
301
|
-
* evolve during the v1.x experimental period.
|
|
285
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
302
286
|
*/
|
|
303
287
|
interface RunEventCountRuleOptions {
|
|
304
288
|
kind?: PersistedInspectEvent["kind"];
|
|
@@ -308,8 +292,7 @@ interface RunEventCountRuleOptions {
|
|
|
308
292
|
/**
|
|
309
293
|
* Experimental options for the built-in run depth rule.
|
|
310
294
|
*
|
|
311
|
-
* @experimental
|
|
312
|
-
* evolve during the v1.x experimental period.
|
|
295
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
313
296
|
*/
|
|
314
297
|
interface RunDepthRuleOptions {
|
|
315
298
|
maxDepth: number;
|
|
@@ -317,8 +300,7 @@ interface RunDepthRuleOptions {
|
|
|
317
300
|
/**
|
|
318
301
|
* Experimental options for the built-in tool usage rule.
|
|
319
302
|
*
|
|
320
|
-
* @experimental
|
|
321
|
-
* evolve during the v1.x experimental period.
|
|
303
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
322
304
|
*/
|
|
323
305
|
interface ToolUsageRuleOptions {
|
|
324
306
|
required?: readonly string[];
|
|
@@ -330,18 +312,21 @@ interface ToolUsageRuleOptions {
|
|
|
330
312
|
/**
|
|
331
313
|
* Experimental options for the built-in tool ordering rule.
|
|
332
314
|
*
|
|
333
|
-
* @experimental
|
|
334
|
-
* evolve during the v1.x experimental period.
|
|
315
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
335
316
|
*/
|
|
336
317
|
interface ToolOrderingRuleOptions {
|
|
337
318
|
before: string;
|
|
338
319
|
after: string;
|
|
320
|
+
/**
|
|
321
|
+
* Explicit rule id. Defaults to `tool.order`. TraceContract-generated
|
|
322
|
+
* adjacent pairs use unique ids such as `contract.tool.order.0`.
|
|
323
|
+
*/
|
|
324
|
+
id?: string;
|
|
339
325
|
}
|
|
340
326
|
/**
|
|
341
327
|
* Experimental options for the built-in tool failure rule.
|
|
342
328
|
*
|
|
343
|
-
* @experimental
|
|
344
|
-
* evolve during the v1.x experimental period.
|
|
329
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
345
330
|
*/
|
|
346
331
|
interface ToolFailureRuleOptions {
|
|
347
332
|
maxFailures?: number;
|
|
@@ -350,8 +335,7 @@ interface ToolFailureRuleOptions {
|
|
|
350
335
|
/**
|
|
351
336
|
* Experimental options for the built-in LLM usage rule.
|
|
352
337
|
*
|
|
353
|
-
* @experimental
|
|
354
|
-
* evolve during the v1.x experimental period.
|
|
338
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
355
339
|
*/
|
|
356
340
|
interface LlmUsageRuleOptions {
|
|
357
341
|
allowedModels?: readonly string[];
|
|
@@ -366,8 +350,7 @@ interface LlmUsageRuleOptions {
|
|
|
366
350
|
/**
|
|
367
351
|
* Experimental options for the built-in structure incomplete rule.
|
|
368
352
|
*
|
|
369
|
-
* @experimental
|
|
370
|
-
* evolve during the v1.x experimental period.
|
|
353
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
371
354
|
*/
|
|
372
355
|
interface StructureIncompleteRuleOptions {
|
|
373
356
|
allowRunning?: boolean;
|
|
@@ -376,8 +359,7 @@ interface StructureIncompleteRuleOptions {
|
|
|
376
359
|
/**
|
|
377
360
|
* Experimental options for the built-in structure orphan rule.
|
|
378
361
|
*
|
|
379
|
-
* @experimental
|
|
380
|
-
* evolve during the v1.x experimental period.
|
|
362
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
381
363
|
*/
|
|
382
364
|
interface StructureOrphanRuleOptions {
|
|
383
365
|
allowMarkedUnresolved?: boolean;
|
|
@@ -385,8 +367,7 @@ interface StructureOrphanRuleOptions {
|
|
|
385
367
|
/**
|
|
386
368
|
* Experimental options for the built-in structure relationship rule.
|
|
387
369
|
*
|
|
388
|
-
* @experimental
|
|
389
|
-
* evolve during the v1.x experimental period.
|
|
370
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
390
371
|
*/
|
|
391
372
|
interface StructureRelationshipRuleOptions {
|
|
392
373
|
minConfidence?: AttributionConfidence;
|
|
@@ -396,8 +377,7 @@ interface StructureRelationshipRuleOptions {
|
|
|
396
377
|
/**
|
|
397
378
|
* Experimental options for the built-in structure parallel-width rule.
|
|
398
379
|
*
|
|
399
|
-
* @experimental
|
|
400
|
-
* evolve during the v1.x experimental period.
|
|
380
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
401
381
|
*/
|
|
402
382
|
interface StructureParallelWidthRuleOptions {
|
|
403
383
|
maxChildren?: number;
|
|
@@ -406,8 +386,7 @@ interface StructureParallelWidthRuleOptions {
|
|
|
406
386
|
/**
|
|
407
387
|
* Experimental options shared by built-in signal rules.
|
|
408
388
|
*
|
|
409
|
-
* @experimental
|
|
410
|
-
* evolve during the v1.x experimental period.
|
|
389
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
411
390
|
*/
|
|
412
391
|
interface TraceSignalRuleOptions {
|
|
413
392
|
required?: readonly string[];
|
|
@@ -419,32 +398,28 @@ interface TraceSignalRuleOptions {
|
|
|
419
398
|
/**
|
|
420
399
|
* Experimental options for the built-in retrieval signal rule.
|
|
421
400
|
*
|
|
422
|
-
* @experimental
|
|
423
|
-
* evolve during the v1.x experimental period.
|
|
401
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
424
402
|
*/
|
|
425
403
|
interface RetrievalRuleOptions extends TraceSignalRuleOptions {
|
|
426
404
|
}
|
|
427
405
|
/**
|
|
428
406
|
* Experimental options for the built-in guardrail signal rule.
|
|
429
407
|
*
|
|
430
|
-
* @experimental
|
|
431
|
-
* evolve during the v1.x experimental period.
|
|
408
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
432
409
|
*/
|
|
433
410
|
interface GuardrailRuleOptions extends TraceSignalRuleOptions {
|
|
434
411
|
}
|
|
435
412
|
/**
|
|
436
413
|
* Experimental options for the built-in decision signal rule.
|
|
437
414
|
*
|
|
438
|
-
* @experimental
|
|
439
|
-
* evolve during the v1.x experimental period.
|
|
415
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
440
416
|
*/
|
|
441
417
|
interface DecisionRuleOptions extends TraceSignalRuleOptions {
|
|
442
418
|
}
|
|
443
419
|
/**
|
|
444
420
|
* Experimental options for the built-in safety redaction rule.
|
|
445
421
|
*
|
|
446
|
-
* @experimental
|
|
447
|
-
* evolve during the v1.x experimental period.
|
|
422
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
448
423
|
*/
|
|
449
424
|
interface SafetyRedactionRuleOptions {
|
|
450
425
|
sensitiveKeys?: readonly string[];
|
|
@@ -454,8 +429,7 @@ interface SafetyRedactionRuleOptions {
|
|
|
454
429
|
/**
|
|
455
430
|
* Experimental options for the built-in raw content path rule.
|
|
456
431
|
*
|
|
457
|
-
* @experimental
|
|
458
|
-
* evolve during the v1.x experimental period.
|
|
432
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
459
433
|
*/
|
|
460
434
|
interface SafetyRawContentRuleOptions {
|
|
461
435
|
forbiddenKeys?: readonly string[];
|
|
@@ -470,8 +444,7 @@ interface SafetyRawContentRuleOptions {
|
|
|
470
444
|
/**
|
|
471
445
|
* Experimental secret pattern used by the built-in secret safety rule.
|
|
472
446
|
*
|
|
473
|
-
* @experimental
|
|
474
|
-
* evolve during the v1.x experimental period.
|
|
447
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
475
448
|
*/
|
|
476
449
|
interface SafetySecretPattern {
|
|
477
450
|
id: string;
|
|
@@ -480,8 +453,7 @@ interface SafetySecretPattern {
|
|
|
480
453
|
/**
|
|
481
454
|
* Experimental options for the built-in secret pattern safety rule.
|
|
482
455
|
*
|
|
483
|
-
* @experimental
|
|
484
|
-
* evolve during the v1.x experimental period.
|
|
456
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
485
457
|
*/
|
|
486
458
|
interface SafetySecretPatternRuleOptions {
|
|
487
459
|
patterns?: readonly SafetySecretPattern[];
|
|
@@ -491,8 +463,7 @@ interface SafetySecretPatternRuleOptions {
|
|
|
491
463
|
/**
|
|
492
464
|
* Experimental options for the built-in oversized attribute safety rule.
|
|
493
465
|
*
|
|
494
|
-
* @experimental
|
|
495
|
-
* evolve during the v1.x experimental period.
|
|
466
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
496
467
|
*/
|
|
497
468
|
interface SafetyOversizedAttributeRuleOptions {
|
|
498
469
|
maxStringLength?: number;
|
|
@@ -508,8 +479,7 @@ interface SafetyOversizedAttributeRuleOptions {
|
|
|
508
479
|
* equivalent `TraceReadResult`; this rule does not read files or parse traces.
|
|
509
480
|
* `durationToleranceMs` defaults to `0`, meaning exact duration comparison.
|
|
510
481
|
*
|
|
511
|
-
* @experimental
|
|
512
|
-
* evolve during the v1.x experimental period.
|
|
482
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
513
483
|
*/
|
|
514
484
|
interface BaselineRegressionRuleOptions {
|
|
515
485
|
baseline: TraceCheckInput;
|
|
@@ -517,27 +487,50 @@ interface BaselineRegressionRuleOptions {
|
|
|
517
487
|
durationToleranceMs?: number;
|
|
518
488
|
compareFormat?: boolean;
|
|
519
489
|
}
|
|
490
|
+
/**
|
|
491
|
+
* Per-rule execution status recorded on every check result.
|
|
492
|
+
*
|
|
493
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
494
|
+
*/
|
|
495
|
+
type TraceCheckRuleExecutionStatus = "pass" | "warning" | "fail" | "error";
|
|
496
|
+
/**
|
|
497
|
+
* Evidence that a specific rule was evaluated (or failed before evaluation).
|
|
498
|
+
*
|
|
499
|
+
* Distinguishes “all intended rules passed” from “no intended rules ran.”
|
|
500
|
+
*
|
|
501
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
502
|
+
*/
|
|
503
|
+
interface TraceCheckRuleExecution {
|
|
504
|
+
ruleId: string;
|
|
505
|
+
category: TraceCheckRule["category"];
|
|
506
|
+
status: TraceCheckRuleExecutionStatus;
|
|
507
|
+
findingCount: number;
|
|
508
|
+
runId?: string;
|
|
509
|
+
}
|
|
520
510
|
/**
|
|
521
511
|
* Experimental aggregate counts for trace-check results.
|
|
522
512
|
*
|
|
523
|
-
*
|
|
524
|
-
*
|
|
513
|
+
* `passed` / `failed` / `warnings` count findings. `rulesEvaluated` counts
|
|
514
|
+
* rules that were selected and executed (or attempted).
|
|
515
|
+
*
|
|
516
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
525
517
|
*/
|
|
526
518
|
interface TraceCheckSummary {
|
|
527
519
|
passed: number;
|
|
528
520
|
failed: number;
|
|
529
521
|
warnings: number;
|
|
530
522
|
errors: number;
|
|
523
|
+
rulesEvaluated: number;
|
|
531
524
|
}
|
|
532
525
|
/**
|
|
533
526
|
* Experimental trace-check result.
|
|
534
527
|
*
|
|
535
528
|
* `status: "fail"` means rules ran and at least one error-severity finding
|
|
536
529
|
* failed. `status: "error"` means execution could not complete because of
|
|
537
|
-
* invalid input, invalid rule selection, or a thrown
|
|
530
|
+
* invalid input, invalid rule selection, zero rules selected, or a thrown
|
|
531
|
+
* rule error.
|
|
538
532
|
*
|
|
539
|
-
* @experimental
|
|
540
|
-
* evolve during the v1.x experimental period.
|
|
533
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
541
534
|
*/
|
|
542
535
|
interface TraceCheckResult {
|
|
543
536
|
ok: boolean;
|
|
@@ -547,19 +540,18 @@ interface TraceCheckResult {
|
|
|
547
540
|
summary: TraceCheckSummary;
|
|
548
541
|
findings: TraceCheckFinding[];
|
|
549
542
|
diagnostics: TraceCheckDiagnostic[];
|
|
543
|
+
ruleExecutions: TraceCheckRuleExecution[];
|
|
550
544
|
}
|
|
551
545
|
/**
|
|
552
546
|
* Create the experimental built-in run status rule.
|
|
553
547
|
*
|
|
554
|
-
* @experimental
|
|
555
|
-
* evolve during the v1.x experimental period.
|
|
548
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
556
549
|
*/
|
|
557
550
|
declare function createRunStatusRule(options?: RunStatusRuleOptions): TraceCheckRule;
|
|
558
551
|
/**
|
|
559
552
|
* Create the experimental built-in run duration rule.
|
|
560
553
|
*
|
|
561
|
-
* @experimental
|
|
562
|
-
* evolve during the v1.x experimental period.
|
|
554
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
563
555
|
*/
|
|
564
556
|
declare function createRunDurationRule(options: RunDurationRuleOptions): TraceCheckRule;
|
|
565
557
|
/**
|
|
@@ -577,127 +569,116 @@ declare function createRequireCompletedRule(): TraceCheckRule;
|
|
|
577
569
|
/**
|
|
578
570
|
* Create the experimental built-in event count rule.
|
|
579
571
|
*
|
|
580
|
-
* @experimental
|
|
581
|
-
* evolve during the v1.x experimental period.
|
|
572
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
582
573
|
*/
|
|
583
574
|
declare function createRunEventCountRule(options: RunEventCountRuleOptions): TraceCheckRule;
|
|
584
575
|
/**
|
|
585
576
|
* Create the experimental built-in run depth rule.
|
|
586
577
|
*
|
|
587
|
-
* @experimental
|
|
588
|
-
* evolve during the v1.x experimental period.
|
|
578
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
589
579
|
*/
|
|
590
580
|
declare function createRunDepthRule(options: RunDepthRuleOptions): TraceCheckRule;
|
|
591
581
|
/**
|
|
592
582
|
* Create the experimental built-in tool usage rule.
|
|
593
583
|
*
|
|
594
|
-
* @experimental
|
|
595
|
-
* evolve during the v1.x experimental period.
|
|
584
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
596
585
|
*/
|
|
597
586
|
declare function createToolUsageRule(options: ToolUsageRuleOptions): TraceCheckRule;
|
|
598
587
|
/**
|
|
599
588
|
* Create the experimental built-in tool ordering rule.
|
|
600
589
|
*
|
|
601
|
-
*
|
|
602
|
-
*
|
|
590
|
+
* Low-level default: first-occurrence start/encounter order among finished
|
|
591
|
+
* tool events. Missing endpoints yield no finding (compositional). TraceContract
|
|
592
|
+
* `requiredOrder` additionally requires presence of listed tools.
|
|
593
|
+
*
|
|
594
|
+
* When order passes by encounter order but intervals overlap, emits a
|
|
595
|
+
* non-failing `tool.order.overlap` warning (not causal happens-before).
|
|
596
|
+
*
|
|
597
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
603
598
|
*/
|
|
604
599
|
declare function createToolOrderingRule(options: ToolOrderingRuleOptions): TraceCheckRule;
|
|
605
600
|
/**
|
|
606
601
|
* Create the experimental built-in tool failure/retry rule.
|
|
607
602
|
*
|
|
608
|
-
* @experimental
|
|
609
|
-
* evolve during the v1.x experimental period.
|
|
603
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
610
604
|
*/
|
|
611
605
|
declare function createToolFailureRule(options: ToolFailureRuleOptions): TraceCheckRule;
|
|
612
606
|
/**
|
|
613
607
|
* Create the experimental built-in LLM usage rule.
|
|
614
608
|
*
|
|
615
|
-
* @experimental
|
|
616
|
-
* evolve during the v1.x experimental period.
|
|
609
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
617
610
|
*/
|
|
618
611
|
declare function createLlmUsageRule(options: LlmUsageRuleOptions): TraceCheckRule;
|
|
619
612
|
/**
|
|
620
613
|
* Create the experimental built-in structure incomplete rule.
|
|
621
614
|
*
|
|
622
|
-
* @experimental
|
|
623
|
-
* evolve during the v1.x experimental period.
|
|
615
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
624
616
|
*/
|
|
625
617
|
declare function createStructureIncompleteRule(options?: StructureIncompleteRuleOptions): TraceCheckRule;
|
|
626
618
|
/**
|
|
627
619
|
* Create the experimental built-in structure orphan rule.
|
|
628
620
|
*
|
|
629
|
-
* @experimental
|
|
630
|
-
* evolve during the v1.x experimental period.
|
|
621
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
631
622
|
*/
|
|
632
623
|
declare function createStructureOrphanRule(options?: StructureOrphanRuleOptions): TraceCheckRule;
|
|
633
624
|
/**
|
|
634
625
|
* Create the experimental built-in structure cycle rule.
|
|
635
626
|
*
|
|
636
|
-
* @experimental
|
|
637
|
-
* evolve during the v1.x experimental period.
|
|
627
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
638
628
|
*/
|
|
639
629
|
declare function createStructureCycleRule(): TraceCheckRule;
|
|
640
630
|
/**
|
|
641
631
|
* Create the experimental built-in structure relationship rule.
|
|
642
632
|
*
|
|
643
|
-
* @experimental
|
|
644
|
-
* evolve during the v1.x experimental period.
|
|
633
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
645
634
|
*/
|
|
646
635
|
declare function createStructureRelationshipRule(options?: StructureRelationshipRuleOptions): TraceCheckRule;
|
|
647
636
|
/**
|
|
648
637
|
* Create the experimental built-in structure parallel-width rule.
|
|
649
638
|
*
|
|
650
|
-
* @experimental
|
|
651
|
-
* evolve during the v1.x experimental period.
|
|
639
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
652
640
|
*/
|
|
653
641
|
declare function createStructureParallelWidthRule(options: StructureParallelWidthRuleOptions): TraceCheckRule;
|
|
654
642
|
/**
|
|
655
643
|
* Create the experimental built-in retrieval signal rule.
|
|
656
644
|
*
|
|
657
|
-
* @experimental
|
|
658
|
-
* evolve during the v1.x experimental period.
|
|
645
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
659
646
|
*/
|
|
660
647
|
declare function createRetrievalRule(options: RetrievalRuleOptions): TraceCheckRule;
|
|
661
648
|
/**
|
|
662
649
|
* Create the experimental built-in guardrail signal rule.
|
|
663
650
|
*
|
|
664
|
-
* @experimental
|
|
665
|
-
* evolve during the v1.x experimental period.
|
|
651
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
666
652
|
*/
|
|
667
653
|
declare function createGuardrailRule(options: GuardrailRuleOptions): TraceCheckRule;
|
|
668
654
|
/**
|
|
669
655
|
* Create the experimental built-in decision signal rule.
|
|
670
656
|
*
|
|
671
|
-
* @experimental
|
|
672
|
-
* evolve during the v1.x experimental period.
|
|
657
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
673
658
|
*/
|
|
674
659
|
declare function createDecisionRule(options: DecisionRuleOptions): TraceCheckRule;
|
|
675
660
|
/**
|
|
676
661
|
* Create the experimental built-in safety redaction rule.
|
|
677
662
|
*
|
|
678
|
-
* @experimental
|
|
679
|
-
* evolve during the v1.x experimental period.
|
|
663
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
680
664
|
*/
|
|
681
665
|
declare function createSafetyRedactionRule(options?: SafetyRedactionRuleOptions): TraceCheckRule;
|
|
682
666
|
/**
|
|
683
667
|
* Create the experimental built-in raw content path safety rule.
|
|
684
668
|
*
|
|
685
|
-
* @experimental
|
|
686
|
-
* evolve during the v1.x experimental period.
|
|
669
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
687
670
|
*/
|
|
688
671
|
declare function createSafetyRawContentRule(options?: SafetyRawContentRuleOptions): TraceCheckRule;
|
|
689
672
|
/**
|
|
690
673
|
* Create the experimental built-in secret pattern safety rule.
|
|
691
674
|
*
|
|
692
|
-
* @experimental
|
|
693
|
-
* evolve during the v1.x experimental period.
|
|
675
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
694
676
|
*/
|
|
695
677
|
declare function createSafetySecretPatternRule(options?: SafetySecretPatternRuleOptions): TraceCheckRule;
|
|
696
678
|
/**
|
|
697
679
|
* Create the experimental built-in oversized attribute safety rule.
|
|
698
680
|
*
|
|
699
|
-
* @experimental
|
|
700
|
-
* evolve during the v1.x experimental period.
|
|
681
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
701
682
|
*/
|
|
702
683
|
declare function createSafetyOversizedAttributeRule(options: SafetyOversizedAttributeRuleOptions): TraceCheckRule;
|
|
703
684
|
/**
|
|
@@ -707,8 +688,7 @@ declare function createSafetyOversizedAttributeRule(options: SafetyOversizedAttr
|
|
|
707
688
|
* against the candidate context. It intentionally ignores raw prompt/output,
|
|
708
689
|
* request/response body, header, and tool payload text.
|
|
709
690
|
*
|
|
710
|
-
* @experimental
|
|
711
|
-
* evolve during the v1.x experimental period.
|
|
691
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
712
692
|
*/
|
|
713
693
|
declare function createBaselineRegressionRule(options: BaselineRegressionRuleOptions): TraceCheckRule;
|
|
714
694
|
/**
|
|
@@ -716,6 +696,12 @@ declare function createBaselineRegressionRule(options: BaselineRegressionRuleOpt
|
|
|
716
696
|
*/
|
|
717
697
|
interface ObservedOutcomeRuleOptions {
|
|
718
698
|
failOn?: readonly ObservedOutcomeStatus[];
|
|
699
|
+
/**
|
|
700
|
+
* When true, fail if the trace contains zero observed outcomes.
|
|
701
|
+
* Default false preserves low-level programmatic compatibility.
|
|
702
|
+
* CLI `--fail-on-observation` sets this to true.
|
|
703
|
+
*/
|
|
704
|
+
requireAny?: boolean;
|
|
719
705
|
}
|
|
720
706
|
declare function createObservedOutcomeRule(options?: ObservedOutcomeRuleOptions): TraceCheckRule;
|
|
721
707
|
/**
|
|
@@ -724,9 +710,8 @@ declare function createObservedOutcomeRule(options?: ObservedOutcomeRuleOptions)
|
|
|
724
710
|
* The engine is local-only and pure: it does not discover config files, read
|
|
725
711
|
* trace files, call providers, perform network I/O, or mutate input objects.
|
|
726
712
|
*
|
|
727
|
-
* @experimental
|
|
728
|
-
* evolve during the v1.x experimental period.
|
|
713
|
+
* @experimental Available through `agent-inspect/checks`. Additive changes may ship in minor releases; breaking changes require a future major.
|
|
729
714
|
*/
|
|
730
715
|
declare function runTraceChecks(input: TraceCheckInput, options?: RunTraceChecksOptions): TraceCheckResult;
|
|
731
716
|
|
|
732
|
-
export {
|
|
717
|
+
export { createRetrievalRule as $, type TraceCheckFacts as A, type BaselineRegressionRuleOptions as B, type TraceCheckFinding as C, type DecisionRuleOptions as D, type TraceCheckFindingStatus as E, type TraceCheckRule as F, type GuardrailRuleOptions as G, type TraceCheckRuleCategory as H, type TraceCheckRuleExecution as I, type TraceCheckRuleExecutionStatus as J, type TraceCheckSeverity as K, type LogicalProjectionDiagnostic as L, type MaxStepDurationRuleOptions as M, type TraceCheckStatus as N, type ObservedOutcomeRuleOptions as O, type TraceCheckSummary as P, type TraceSignalRuleOptions as Q, type RetrievalRuleOptions as R, type SafetyFindingCategory as S, type TraceCheckInput as T, createBaselineRegressionRule as U, createDecisionRule as V, createGuardrailRule as W, createLlmUsageRule as X, createMaxStepDurationRule as Y, createObservedOutcomeRule as Z, createRequireCompletedRule as _, type LogicalTraceEvent as a, createRunDepthRule as a0, createRunDurationRule as a1, createRunEventCountRule as a2, createRunStatusRule as a3, createSafetyOversizedAttributeRule as a4, createSafetyRawContentRule as a5, createSafetyRedactionRule as a6, createSafetySecretPatternRule as a7, createStallDetectionRule as a8, createStructureCycleRule as a9, createStructureIncompleteRule as aa, createStructureOrphanRule as ab, createStructureParallelWidthRule as ac, createStructureRelationshipRule as ad, createToolFailureRule as ae, createToolOrderingRule as af, createToolUsageRule as ag, projectLogicalEvents as ah, resolveCanonicalToolName as ai, runTraceChecks as aj, type TraceCheckResult as b, type LlmUsageRuleOptions as c, type RunDepthRuleOptions as d, type RunDurationRuleOptions as e, type RunEventCountRuleOptions as f, type RunStatusRuleOptions as g, type RunTraceChecksOptions as h, type SafetyFindingConfidence as i, type SafetyOversizedAttributeRuleOptions as j, type SafetyRawContentRuleOptions as k, type SafetyRedactionRuleOptions as l, type SafetySecretPattern as m, type SafetySecretPatternRuleOptions as n, type StallDetectionRuleOptions as o, type StructureIncompleteRuleOptions as p, type StructureOrphanRuleOptions as q, type StructureParallelWidthRuleOptions as r, type StructureRelationshipRuleOptions as s, type ToolFailureRuleOptions as t, type ToolOrderingRuleOptions as u, type ToolUsageRuleOptions as v, type TraceCheckContext as w, type TraceCheckDiagnostic as x, type TraceCheckDiagnosticCode as y, type TraceCheckEvidence as z };
|