agent-inspect 6.22.0 → 6.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +1 -1
- package/docs/TRACE-CONTRACTS.md +56 -0
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-CHBYF4LE.mjs → chunk-BVQO5CWA.mjs} +2 -2
- package/packages/cli/dist/{chunk-CHBYF4LE.mjs.map → chunk-BVQO5CWA.mjs.map} +1 -1
- package/packages/cli/dist/index.cjs +27 -1
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +4 -4
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-MY5WIBPF.mjs → src-Y4NXOO2K.mjs} +3 -3
- package/packages/cli/dist/{src-MY5WIBPF.mjs.map → src-Y4NXOO2K.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.mjs +2 -2
- package/packages/core/dist/checks.cjs +726 -50
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +174 -1
- package/packages/core/dist/checks.d.ts +174 -1
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-OG3GAWCX.mjs → chunk-VMDCDWBE.mjs} +680 -7
- package/packages/core/dist/chunk-VMDCDWBE.mjs.map +1 -0
- package/packages/core/dist/chunk-OG3GAWCX.mjs.map +0 -1
|
@@ -1128,6 +1128,133 @@ function resolveTraceContractScope(input, scope) {
|
|
|
1128
1128
|
};
|
|
1129
1129
|
}
|
|
1130
1130
|
|
|
1131
|
+
// packages/core/src/checks/tool-arguments.ts
|
|
1132
|
+
function decodeToken(token) {
|
|
1133
|
+
return token.replace(/~1/g, "/").replace(/~0/g, "~");
|
|
1134
|
+
}
|
|
1135
|
+
function resolveJsonPointer(document, pointer) {
|
|
1136
|
+
if (pointer === "") {
|
|
1137
|
+
return { found: true, value: document };
|
|
1138
|
+
}
|
|
1139
|
+
if (!pointer.startsWith("/")) {
|
|
1140
|
+
return { found: false };
|
|
1141
|
+
}
|
|
1142
|
+
const tokens = pointer.slice(1).split("/").map(decodeToken);
|
|
1143
|
+
let current = document;
|
|
1144
|
+
for (const token of tokens) {
|
|
1145
|
+
if (current === null || current === void 0) {
|
|
1146
|
+
return { found: false };
|
|
1147
|
+
}
|
|
1148
|
+
if (Array.isArray(current)) {
|
|
1149
|
+
if (!/^(0|[1-9][0-9]*)$/.test(token)) {
|
|
1150
|
+
return { found: false };
|
|
1151
|
+
}
|
|
1152
|
+
const index = Number(token);
|
|
1153
|
+
if (index >= current.length) {
|
|
1154
|
+
return { found: false };
|
|
1155
|
+
}
|
|
1156
|
+
current = current[index];
|
|
1157
|
+
continue;
|
|
1158
|
+
}
|
|
1159
|
+
if (typeof current !== "object") {
|
|
1160
|
+
return { found: false };
|
|
1161
|
+
}
|
|
1162
|
+
const record = current;
|
|
1163
|
+
if (!Object.prototype.hasOwnProperty.call(record, token)) {
|
|
1164
|
+
return { found: false };
|
|
1165
|
+
}
|
|
1166
|
+
current = record[token];
|
|
1167
|
+
}
|
|
1168
|
+
return { found: true, value: current };
|
|
1169
|
+
}
|
|
1170
|
+
function jsonType(value) {
|
|
1171
|
+
if (value === null) return "null";
|
|
1172
|
+
if (Array.isArray(value)) return "array";
|
|
1173
|
+
return typeof value;
|
|
1174
|
+
}
|
|
1175
|
+
function valuesEqual(left, right) {
|
|
1176
|
+
return JSON.stringify(left) === JSON.stringify(right);
|
|
1177
|
+
}
|
|
1178
|
+
function extractToolArgumentPayload(event) {
|
|
1179
|
+
const attrs2 = event.attributes ?? {};
|
|
1180
|
+
for (const key of ["arguments", "input", "toolArguments"]) {
|
|
1181
|
+
const candidate = attrs2[key];
|
|
1182
|
+
if (candidate !== void 0 && candidate !== null && typeof candidate === "object") {
|
|
1183
|
+
return { present: true, value: candidate };
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
if (event.inputSummary !== void 0 && event.inputSummary !== null && typeof event.inputSummary === "object") {
|
|
1187
|
+
return { present: true, value: event.inputSummary };
|
|
1188
|
+
}
|
|
1189
|
+
return { present: false, value: void 0 };
|
|
1190
|
+
}
|
|
1191
|
+
function evaluateToolArgumentValue(value, check, evidencePresent) {
|
|
1192
|
+
if (!evidencePresent) {
|
|
1193
|
+
return {
|
|
1194
|
+
status: "unavailable",
|
|
1195
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
1196
|
+
message: `Structured argument evidence unavailable for tool ${check.tool} at ${check.path}.`
|
|
1197
|
+
};
|
|
1198
|
+
}
|
|
1199
|
+
const resolved = resolveJsonPointer(value, check.path);
|
|
1200
|
+
if (check.operator === "exists") {
|
|
1201
|
+
return resolved.found ? { status: "pass" } : {
|
|
1202
|
+
status: "fail",
|
|
1203
|
+
code: "AI_CHECK_TOOL_ARGUMENT_MISSING",
|
|
1204
|
+
message: `Expected path ${check.path} to exist on tool ${check.tool}.`
|
|
1205
|
+
};
|
|
1206
|
+
}
|
|
1207
|
+
if (!resolved.found) {
|
|
1208
|
+
return {
|
|
1209
|
+
status: "unavailable",
|
|
1210
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
1211
|
+
message: `Path ${check.path} missing for tool ${check.tool}.`
|
|
1212
|
+
};
|
|
1213
|
+
}
|
|
1214
|
+
if (check.operator === "type") {
|
|
1215
|
+
const actual = jsonType(resolved.value);
|
|
1216
|
+
if (check.type === void 0) {
|
|
1217
|
+
return {
|
|
1218
|
+
status: "fail",
|
|
1219
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
1220
|
+
message: "type operator requires check.type"
|
|
1221
|
+
};
|
|
1222
|
+
}
|
|
1223
|
+
return actual === check.type ? { status: "pass" } : {
|
|
1224
|
+
status: "fail",
|
|
1225
|
+
code: "AI_CHECK_TOOL_ARGUMENT_TYPE",
|
|
1226
|
+
message: `Expected type ${check.type} at ${check.path} for tool ${check.tool}; found ${actual}.`
|
|
1227
|
+
};
|
|
1228
|
+
}
|
|
1229
|
+
if (check.operator === "equals") {
|
|
1230
|
+
return valuesEqual(resolved.value, check.expected) ? { status: "pass" } : {
|
|
1231
|
+
status: "fail",
|
|
1232
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EQUALS",
|
|
1233
|
+
message: `Value at ${check.path} for tool ${check.tool} did not equal expected (bounded comparison).`
|
|
1234
|
+
};
|
|
1235
|
+
}
|
|
1236
|
+
if (check.operator === "oneOf") {
|
|
1237
|
+
const options = check.oneOf ?? [];
|
|
1238
|
+
if (options.length === 0) {
|
|
1239
|
+
return {
|
|
1240
|
+
status: "fail",
|
|
1241
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
1242
|
+
message: "oneOf operator requires a non-empty oneOf array"
|
|
1243
|
+
};
|
|
1244
|
+
}
|
|
1245
|
+
return options.some((option) => valuesEqual(resolved.value, option)) ? { status: "pass" } : {
|
|
1246
|
+
status: "fail",
|
|
1247
|
+
code: "AI_CHECK_TOOL_ARGUMENT_ONE_OF",
|
|
1248
|
+
message: `Value at ${check.path} for tool ${check.tool} was not in the allowed oneOf set.`
|
|
1249
|
+
};
|
|
1250
|
+
}
|
|
1251
|
+
return {
|
|
1252
|
+
status: "fail",
|
|
1253
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
1254
|
+
message: `Unsupported operator.`
|
|
1255
|
+
};
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1131
1258
|
// packages/core/src/safety/sensitive-key.ts
|
|
1132
1259
|
function keyHasExplicitSeparator(value) {
|
|
1133
1260
|
return /[_\-.]/.test(value);
|
|
@@ -3166,6 +3293,338 @@ function runTraceChecks(input, options = {}) {
|
|
|
3166
3293
|
};
|
|
3167
3294
|
}
|
|
3168
3295
|
|
|
3296
|
+
// packages/core/src/checks/control-rules.ts
|
|
3297
|
+
function asStringList(value) {
|
|
3298
|
+
if (!Array.isArray(value)) return void 0;
|
|
3299
|
+
const out = [];
|
|
3300
|
+
for (const item of value) {
|
|
3301
|
+
if (typeof item !== "string" || item.trim() === "") return void 0;
|
|
3302
|
+
out.push(item.trim());
|
|
3303
|
+
}
|
|
3304
|
+
return out;
|
|
3305
|
+
}
|
|
3306
|
+
function attributeList(events, attribute) {
|
|
3307
|
+
if (!attribute) return void 0;
|
|
3308
|
+
for (const event of events) {
|
|
3309
|
+
const attrs2 = event.attributes;
|
|
3310
|
+
if (!attrs2 || typeof attrs2 !== "object") continue;
|
|
3311
|
+
const direct = asStringList(attrs2[attribute]);
|
|
3312
|
+
if (direct) return direct;
|
|
3313
|
+
const nested = attrs2.metadata && typeof attrs2.metadata === "object" ? asStringList(attrs2.metadata[attribute]) : void 0;
|
|
3314
|
+
if (nested) return nested;
|
|
3315
|
+
}
|
|
3316
|
+
return void 0;
|
|
3317
|
+
}
|
|
3318
|
+
function fail(ruleId, message, evidence, expected, actual) {
|
|
3319
|
+
return {
|
|
3320
|
+
ruleId,
|
|
3321
|
+
severity: "error",
|
|
3322
|
+
status: "fail",
|
|
3323
|
+
message,
|
|
3324
|
+
...expected !== void 0 ? { expected } : {},
|
|
3325
|
+
...actual !== void 0 ? { actual } : {},
|
|
3326
|
+
evidence: [...evidence]
|
|
3327
|
+
};
|
|
3328
|
+
}
|
|
3329
|
+
function sortedUnique(values) {
|
|
3330
|
+
return [...new Set(values)].sort((a, b) => a.localeCompare(b));
|
|
3331
|
+
}
|
|
3332
|
+
function setEqual(left, right) {
|
|
3333
|
+
const a = sortedUnique(left);
|
|
3334
|
+
const b = sortedUnique(right);
|
|
3335
|
+
return a.length === b.length && a.every((value, index) => value === b[index]);
|
|
3336
|
+
}
|
|
3337
|
+
function evaluateControlRules(events, rules, runEvidence2, observationNames) {
|
|
3338
|
+
const findings = [];
|
|
3339
|
+
const declared = rules.declaredTools ?? attributeList(events, rules.declaredToolsAttribute);
|
|
3340
|
+
const enforced = rules.enforcedTools ?? attributeList(events, rules.enforcedToolsAttribute);
|
|
3341
|
+
if (rules.requireDeclaredMatchesEnforced) {
|
|
3342
|
+
if (declared === void 0 || enforced === void 0) {
|
|
3343
|
+
findings.push(
|
|
3344
|
+
fail(
|
|
3345
|
+
"contract.controls.declared-matches-enforced",
|
|
3346
|
+
"Declared and enforced tool sets could not both be resolved.",
|
|
3347
|
+
runEvidence2,
|
|
3348
|
+
{ declaredDefined: declared !== void 0, enforcedDefined: enforced !== void 0 },
|
|
3349
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
3350
|
+
)
|
|
3351
|
+
);
|
|
3352
|
+
} else if (!setEqual(declared, enforced)) {
|
|
3353
|
+
findings.push(
|
|
3354
|
+
fail(
|
|
3355
|
+
"contract.controls.declared-matches-enforced",
|
|
3356
|
+
"Declared tools differ from enforced tools.",
|
|
3357
|
+
runEvidence2,
|
|
3358
|
+
sortedUnique(declared),
|
|
3359
|
+
sortedUnique(enforced)
|
|
3360
|
+
)
|
|
3361
|
+
);
|
|
3362
|
+
}
|
|
3363
|
+
}
|
|
3364
|
+
const observedTools = sortedUnique(
|
|
3365
|
+
events.filter((event) => event.kind === "TOOL" && event.status !== "running").map((event) => resolveCanonicalToolName(event))
|
|
3366
|
+
);
|
|
3367
|
+
if (rules.requireObservedWithinEnforced) {
|
|
3368
|
+
if (enforced === void 0) {
|
|
3369
|
+
findings.push(
|
|
3370
|
+
fail(
|
|
3371
|
+
"contract.controls.observed-within-enforced",
|
|
3372
|
+
"Enforced tool set unavailable for observed-within-enforced check.",
|
|
3373
|
+
runEvidence2,
|
|
3374
|
+
void 0,
|
|
3375
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
3376
|
+
)
|
|
3377
|
+
);
|
|
3378
|
+
} else {
|
|
3379
|
+
const enforcedSet = new Set(enforced);
|
|
3380
|
+
const outside = observedTools.filter((name) => !enforcedSet.has(name));
|
|
3381
|
+
if (outside.length > 0) {
|
|
3382
|
+
findings.push(
|
|
3383
|
+
fail(
|
|
3384
|
+
"contract.controls.observed-within-enforced",
|
|
3385
|
+
`Observed tools outside enforced allowlist: ${outside.join(", ")}.`,
|
|
3386
|
+
runEvidence2,
|
|
3387
|
+
sortedUnique(enforced),
|
|
3388
|
+
outside
|
|
3389
|
+
)
|
|
3390
|
+
);
|
|
3391
|
+
}
|
|
3392
|
+
}
|
|
3393
|
+
}
|
|
3394
|
+
if (rules.requireObservedWithinDeclared) {
|
|
3395
|
+
if (declared === void 0) {
|
|
3396
|
+
findings.push(
|
|
3397
|
+
fail(
|
|
3398
|
+
"contract.controls.observed-within-declared",
|
|
3399
|
+
"Declared tool set unavailable for observed-within-declared check.",
|
|
3400
|
+
runEvidence2,
|
|
3401
|
+
void 0,
|
|
3402
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
3403
|
+
)
|
|
3404
|
+
);
|
|
3405
|
+
} else {
|
|
3406
|
+
const declaredSet = new Set(declared);
|
|
3407
|
+
const outside = observedTools.filter((name) => !declaredSet.has(name));
|
|
3408
|
+
if (outside.length > 0) {
|
|
3409
|
+
findings.push(
|
|
3410
|
+
fail(
|
|
3411
|
+
"contract.controls.observed-within-declared",
|
|
3412
|
+
`Observed tools outside declared set: ${outside.join(", ")}.`,
|
|
3413
|
+
runEvidence2,
|
|
3414
|
+
sortedUnique(declared),
|
|
3415
|
+
outside
|
|
3416
|
+
)
|
|
3417
|
+
);
|
|
3418
|
+
}
|
|
3419
|
+
}
|
|
3420
|
+
}
|
|
3421
|
+
for (const stage of rules.requiredStages ?? []) {
|
|
3422
|
+
const observation = stage.observation ?? `control.${stage.stage}`;
|
|
3423
|
+
if (!observationNames.has(observation)) {
|
|
3424
|
+
findings.push(
|
|
3425
|
+
fail(
|
|
3426
|
+
`contract.controls.stage.${stage.stage}`,
|
|
3427
|
+
`Required control stage observation missing: ${observation}.`,
|
|
3428
|
+
runEvidence2,
|
|
3429
|
+
observation,
|
|
3430
|
+
[...observationNames]
|
|
3431
|
+
)
|
|
3432
|
+
);
|
|
3433
|
+
}
|
|
3434
|
+
}
|
|
3435
|
+
return findings;
|
|
3436
|
+
}
|
|
3437
|
+
|
|
3438
|
+
// packages/core/src/checks/retry-safety.ts
|
|
3439
|
+
function fail2(ruleId, message, evidence, expected, actual) {
|
|
3440
|
+
return {
|
|
3441
|
+
ruleId,
|
|
3442
|
+
severity: "error",
|
|
3443
|
+
status: "fail",
|
|
3444
|
+
message,
|
|
3445
|
+
...expected !== void 0 ? { expected } : {},
|
|
3446
|
+
...actual !== void 0 ? { actual } : {},
|
|
3447
|
+
evidence: [...evidence]
|
|
3448
|
+
};
|
|
3449
|
+
}
|
|
3450
|
+
function workflowFor(event) {
|
|
3451
|
+
const attrs2 = event.attributes;
|
|
3452
|
+
if (!attrs2 || typeof attrs2 !== "object") return {};
|
|
3453
|
+
const direct = extractSessionWorkflowMetadata(attrs2);
|
|
3454
|
+
const nested = attrs2.metadata && typeof attrs2.metadata === "object" ? extractSessionWorkflowMetadata(attrs2.metadata) : void 0;
|
|
3455
|
+
return { ...nested, ...direct };
|
|
3456
|
+
}
|
|
3457
|
+
function hasIdempotencyEvidence(event) {
|
|
3458
|
+
const workflow = workflowFor(event);
|
|
3459
|
+
if (typeof workflow.idempotencyKey === "string" && workflow.idempotencyKey.trim() !== "") {
|
|
3460
|
+
return true;
|
|
3461
|
+
}
|
|
3462
|
+
const attrs2 = event.attributes ?? {};
|
|
3463
|
+
if (attrs2.noSideEffect === true) return true;
|
|
3464
|
+
if (attrs2.sideEffect === false) return true;
|
|
3465
|
+
return false;
|
|
3466
|
+
}
|
|
3467
|
+
function eventEvidence2(event) {
|
|
3468
|
+
return {
|
|
3469
|
+
runId: event.runId,
|
|
3470
|
+
eventId: event.eventId,
|
|
3471
|
+
kind: event.kind,
|
|
3472
|
+
name: event.name,
|
|
3473
|
+
status: event.status
|
|
3474
|
+
};
|
|
3475
|
+
}
|
|
3476
|
+
function evaluateRetrySafetyRules(events, rules, _runEvidence) {
|
|
3477
|
+
const findings = [];
|
|
3478
|
+
const byOperation = /* @__PURE__ */ new Map();
|
|
3479
|
+
for (const event of events) {
|
|
3480
|
+
const workflow = workflowFor(event);
|
|
3481
|
+
const operationId = workflow.operationId;
|
|
3482
|
+
if (!operationId) continue;
|
|
3483
|
+
const list = byOperation.get(operationId) ?? [];
|
|
3484
|
+
list.push(event);
|
|
3485
|
+
byOperation.set(operationId, list);
|
|
3486
|
+
}
|
|
3487
|
+
if (rules.maxAttempts !== void 0) {
|
|
3488
|
+
for (const [operationId, members] of byOperation) {
|
|
3489
|
+
const attemptIds = new Set(
|
|
3490
|
+
members.map((event) => workflowFor(event).attemptId).filter((value) => typeof value === "string" && value.trim() !== "")
|
|
3491
|
+
);
|
|
3492
|
+
const attemptNumbers = members.map((event) => workflowFor(event).attemptNumber ?? workflowFor(event).attempt).filter((value) => typeof value === "number" && Number.isFinite(value));
|
|
3493
|
+
const count = attemptIds.size > 0 ? attemptIds.size : attemptNumbers.length > 0 ? Math.max(...attemptNumbers) : members.filter((event) => event.kind === "TOOL" || event.kind === "LLM").length;
|
|
3494
|
+
if (count > rules.maxAttempts) {
|
|
3495
|
+
findings.push(
|
|
3496
|
+
fail2(
|
|
3497
|
+
"contract.retry.max-attempts",
|
|
3498
|
+
`Operation ${operationId} exceeded maxAttempts ${rules.maxAttempts}.`,
|
|
3499
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
3500
|
+
rules.maxAttempts,
|
|
3501
|
+
count
|
|
3502
|
+
)
|
|
3503
|
+
);
|
|
3504
|
+
}
|
|
3505
|
+
}
|
|
3506
|
+
}
|
|
3507
|
+
if (rules.requireTerminalResult) {
|
|
3508
|
+
for (const [operationId, members] of byOperation) {
|
|
3509
|
+
const hasTerminal = members.some((event) => event.status === "ok" || event.status === "error");
|
|
3510
|
+
if (!hasTerminal) {
|
|
3511
|
+
findings.push(
|
|
3512
|
+
fail2(
|
|
3513
|
+
"contract.retry.terminal-result",
|
|
3514
|
+
`Operation ${operationId} has no terminal ok/error result.`,
|
|
3515
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
3516
|
+
"ok|error",
|
|
3517
|
+
members.map((event) => event.status)
|
|
3518
|
+
)
|
|
3519
|
+
);
|
|
3520
|
+
}
|
|
3521
|
+
}
|
|
3522
|
+
}
|
|
3523
|
+
if (rules.fallbackOnlyAfterFailure) {
|
|
3524
|
+
for (const event of events) {
|
|
3525
|
+
const workflow = workflowFor(event);
|
|
3526
|
+
const fallbackOf = workflow.fallbackOf;
|
|
3527
|
+
if (!fallbackOf) continue;
|
|
3528
|
+
const prior = byOperation.get(fallbackOf) ?? events.filter((candidate) => {
|
|
3529
|
+
const meta = workflowFor(candidate);
|
|
3530
|
+
return meta.operationId === fallbackOf || candidate.runId === fallbackOf;
|
|
3531
|
+
});
|
|
3532
|
+
const priorFailure = prior.some((candidate) => candidate.status === "error");
|
|
3533
|
+
if (!priorFailure) {
|
|
3534
|
+
findings.push(
|
|
3535
|
+
fail2(
|
|
3536
|
+
"contract.retry.fallback-after-failure",
|
|
3537
|
+
`Fallback for ${fallbackOf} appeared without a prior failure.`,
|
|
3538
|
+
[eventEvidence2(event)],
|
|
3539
|
+
"prior error attempt",
|
|
3540
|
+
{ fallbackOf }
|
|
3541
|
+
)
|
|
3542
|
+
);
|
|
3543
|
+
}
|
|
3544
|
+
}
|
|
3545
|
+
}
|
|
3546
|
+
const nonIdempotent = new Set(rules.nonIdempotentTools ?? []);
|
|
3547
|
+
if (nonIdempotent.size > 0 || rules.requireIdempotencyEvidenceForRetry) {
|
|
3548
|
+
const toolEvents = events.filter(
|
|
3549
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
3550
|
+
);
|
|
3551
|
+
const byToolOp = /* @__PURE__ */ new Map();
|
|
3552
|
+
for (const event of toolEvents) {
|
|
3553
|
+
const name = resolveCanonicalToolName(event);
|
|
3554
|
+
const workflow = workflowFor(event);
|
|
3555
|
+
const key = `${workflow.operationId ?? name}::${name}`;
|
|
3556
|
+
const list = byToolOp.get(key) ?? [];
|
|
3557
|
+
list.push(event);
|
|
3558
|
+
byToolOp.set(key, list);
|
|
3559
|
+
}
|
|
3560
|
+
for (const [, members] of byToolOp) {
|
|
3561
|
+
const ordered = [...members].sort((a, b) => {
|
|
3562
|
+
const aTime = a.startedAt ?? a.timestamp ?? "";
|
|
3563
|
+
const bTime = b.startedAt ?? b.timestamp ?? "";
|
|
3564
|
+
return aTime.localeCompare(bTime);
|
|
3565
|
+
});
|
|
3566
|
+
let sawOk = false;
|
|
3567
|
+
let sawSideEffectOk = false;
|
|
3568
|
+
for (const event of ordered) {
|
|
3569
|
+
const name = resolveCanonicalToolName(event);
|
|
3570
|
+
const isRetry = sawOk;
|
|
3571
|
+
if (isRetry) {
|
|
3572
|
+
if (rules.requireIdempotencyEvidenceForRetry && !hasIdempotencyEvidence(event)) {
|
|
3573
|
+
findings.push(
|
|
3574
|
+
fail2(
|
|
3575
|
+
"contract.retry.idempotency-evidence",
|
|
3576
|
+
`Retry of tool ${name} lacks idempotencyKey / noSideEffect evidence.`,
|
|
3577
|
+
[eventEvidence2(event)],
|
|
3578
|
+
"idempotencyKey|noSideEffect",
|
|
3579
|
+
{ code: "AI_CHECK_RETRY_EVIDENCE_UNAVAILABLE" }
|
|
3580
|
+
)
|
|
3581
|
+
);
|
|
3582
|
+
}
|
|
3583
|
+
if (nonIdempotent.has(name) && sawSideEffectOk && !hasIdempotencyEvidence(event)) {
|
|
3584
|
+
findings.push(
|
|
3585
|
+
fail2(
|
|
3586
|
+
"contract.retry.non-idempotent-side-effect",
|
|
3587
|
+
`Retry of non-idempotent tool ${name} after a confirmed ok side effect.`,
|
|
3588
|
+
[eventEvidence2(event)],
|
|
3589
|
+
"no retry after side effect",
|
|
3590
|
+
name
|
|
3591
|
+
)
|
|
3592
|
+
);
|
|
3593
|
+
}
|
|
3594
|
+
}
|
|
3595
|
+
if (event.status === "ok") {
|
|
3596
|
+
sawOk = true;
|
|
3597
|
+
if (nonIdempotent.has(name) && !hasIdempotencyEvidence(event)) {
|
|
3598
|
+
sawSideEffectOk = true;
|
|
3599
|
+
}
|
|
3600
|
+
}
|
|
3601
|
+
}
|
|
3602
|
+
}
|
|
3603
|
+
}
|
|
3604
|
+
if (rules.requireRecoveredFailureVisible) {
|
|
3605
|
+
for (const [operationId, members] of byOperation) {
|
|
3606
|
+
const hasOk = members.some((event) => event.status === "ok");
|
|
3607
|
+
const hasError = members.some((event) => event.status === "error");
|
|
3608
|
+
const attemptish = members.some((event) => {
|
|
3609
|
+
const meta = workflowFor(event);
|
|
3610
|
+
return meta.attemptId !== void 0 || meta.attemptNumber !== void 0 || meta.attempt !== void 0;
|
|
3611
|
+
}) || members.length > 1;
|
|
3612
|
+
if (hasOk && attemptish && !hasError) {
|
|
3613
|
+
findings.push(
|
|
3614
|
+
fail2(
|
|
3615
|
+
"contract.retry.recovered-failure-visible",
|
|
3616
|
+
`Operation ${operationId} recovered without retaining a visible failure attempt.`,
|
|
3617
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
3618
|
+
"error attempt retained",
|
|
3619
|
+
{ hasOk, hasError }
|
|
3620
|
+
)
|
|
3621
|
+
);
|
|
3622
|
+
}
|
|
3623
|
+
}
|
|
3624
|
+
}
|
|
3625
|
+
return findings;
|
|
3626
|
+
}
|
|
3627
|
+
|
|
3169
3628
|
// packages/core/src/checks/contract.ts
|
|
3170
3629
|
function contractFailFinding(ruleId, message, evidence, expected, actual) {
|
|
3171
3630
|
return {
|
|
@@ -3187,18 +3646,38 @@ function normalizeStatus(status) {
|
|
|
3187
3646
|
function cloneBody(body) {
|
|
3188
3647
|
return {
|
|
3189
3648
|
...body.run ? { run: { ...body.run } } : {},
|
|
3190
|
-
...body.tools ? {
|
|
3649
|
+
...body.tools ? {
|
|
3650
|
+
tools: {
|
|
3651
|
+
...body.tools,
|
|
3652
|
+
...body.tools.arguments ? { arguments: body.tools.arguments.map((item) => ({ ...item })) } : {},
|
|
3653
|
+
...body.tools.orderRules ? { orderRules: body.tools.orderRules.map((item) => ({ ...item })) } : {}
|
|
3654
|
+
}
|
|
3655
|
+
} : {},
|
|
3191
3656
|
...body.llm ? { llm: { ...body.llm } } : {},
|
|
3192
3657
|
...body.observations ? {
|
|
3193
3658
|
observations: {
|
|
3194
3659
|
...body.observations,
|
|
3195
3660
|
...body.observations.requireProvenance ? { requireProvenance: { ...body.observations.requireProvenance } } : {}
|
|
3196
3661
|
}
|
|
3662
|
+
} : {},
|
|
3663
|
+
...body.controls ? {
|
|
3664
|
+
controls: {
|
|
3665
|
+
...body.controls,
|
|
3666
|
+
...body.controls.declaredTools ? { declaredTools: [...body.controls.declaredTools] } : {},
|
|
3667
|
+
...body.controls.enforcedTools ? { enforcedTools: [...body.controls.enforcedTools] } : {},
|
|
3668
|
+
...body.controls.requiredStages ? { requiredStages: body.controls.requiredStages.map((item) => ({ ...item })) } : {}
|
|
3669
|
+
}
|
|
3670
|
+
} : {},
|
|
3671
|
+
...body.retry ? {
|
|
3672
|
+
retry: {
|
|
3673
|
+
...body.retry,
|
|
3674
|
+
...body.retry.nonIdempotentTools ? { nonIdempotentTools: [...body.retry.nonIdempotentTools] } : {}
|
|
3675
|
+
}
|
|
3197
3676
|
} : {}
|
|
3198
3677
|
};
|
|
3199
3678
|
}
|
|
3200
3679
|
function bodyHasRules(body) {
|
|
3201
|
-
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0;
|
|
3680
|
+
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
3202
3681
|
}
|
|
3203
3682
|
var MAX_EVIDENCE_EVENT_IDS = 16;
|
|
3204
3683
|
var METHOD_VOCABULARY = new Set(OBSERVED_OUTCOME_METHODS);
|
|
@@ -3399,11 +3878,14 @@ function contractToRules(contract) {
|
|
|
3399
3878
|
if (contract.tools) {
|
|
3400
3879
|
const order = contract.tools.requiredOrder ?? [];
|
|
3401
3880
|
const requiredOrderMode = contract.tools.requiredOrderMode ?? "first-occurrence";
|
|
3881
|
+
const orderRules = contract.tools.orderRules ?? [];
|
|
3882
|
+
const endpointRequired = orderRules.filter((rule) => rule.requireEndpoints !== false).flatMap((rule) => [rule.before, rule.after]);
|
|
3402
3883
|
const required = [
|
|
3403
3884
|
.../* @__PURE__ */ new Set([
|
|
3404
3885
|
...contract.tools.required ?? [],
|
|
3405
3886
|
...contract.tools.requiredTools ?? [],
|
|
3406
|
-
...order
|
|
3887
|
+
...order,
|
|
3888
|
+
...endpointRequired
|
|
3407
3889
|
])
|
|
3408
3890
|
];
|
|
3409
3891
|
const forbidden = [
|
|
@@ -3428,6 +3910,106 @@ function contractToRules(contract) {
|
|
|
3428
3910
|
})
|
|
3429
3911
|
);
|
|
3430
3912
|
}
|
|
3913
|
+
const defaultOccurrenceMode = contract.tools.defaultOccurrenceMode ?? requiredOrderMode;
|
|
3914
|
+
for (const [index, rule] of orderRules.entries()) {
|
|
3915
|
+
rules.push(
|
|
3916
|
+
createToolOrderingRule({
|
|
3917
|
+
before: rule.before,
|
|
3918
|
+
after: rule.after,
|
|
3919
|
+
id: `contract.tool.orderRule.${index}`,
|
|
3920
|
+
mode: rule.occurrenceMode ?? defaultOccurrenceMode
|
|
3921
|
+
})
|
|
3922
|
+
);
|
|
3923
|
+
}
|
|
3924
|
+
const argumentChecks = contract.tools.arguments ?? [];
|
|
3925
|
+
if (argumentChecks.length > 0) {
|
|
3926
|
+
rules.push({
|
|
3927
|
+
id: "contract.tool.arguments",
|
|
3928
|
+
category: "tool",
|
|
3929
|
+
defaultSeverity: "error",
|
|
3930
|
+
evaluate(context) {
|
|
3931
|
+
const tools = (context.logicalEvents ?? context.events).filter(
|
|
3932
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
3933
|
+
);
|
|
3934
|
+
const findings = [];
|
|
3935
|
+
for (const [index, check] of argumentChecks.entries()) {
|
|
3936
|
+
const matches = tools.filter(
|
|
3937
|
+
(event) => resolveCanonicalToolName(event) === check.tool
|
|
3938
|
+
);
|
|
3939
|
+
if (matches.length === 0) {
|
|
3940
|
+
findings.push(
|
|
3941
|
+
contractFailFinding(
|
|
3942
|
+
`contract.tool.arguments.${index}`,
|
|
3943
|
+
`No finished tool named ${check.tool} for argument check.`,
|
|
3944
|
+
context.selectedRun ? [
|
|
3945
|
+
{
|
|
3946
|
+
runId: context.selectedRun.runId,
|
|
3947
|
+
kind: "RUN",
|
|
3948
|
+
name: context.selectedRun.name
|
|
3949
|
+
}
|
|
3950
|
+
] : [],
|
|
3951
|
+
{ tool: check.tool, path: check.path },
|
|
3952
|
+
{ toolCount: 0 }
|
|
3953
|
+
)
|
|
3954
|
+
);
|
|
3955
|
+
continue;
|
|
3956
|
+
}
|
|
3957
|
+
const occurrence = check.occurrence ?? "all";
|
|
3958
|
+
const selected = occurrence === "first" ? [matches[0]] : occurrence === "last" ? [matches[matches.length - 1]] : matches;
|
|
3959
|
+
const results = selected.map((event) => {
|
|
3960
|
+
const payload = extractToolArgumentPayload(event);
|
|
3961
|
+
return evaluateToolArgumentValue(
|
|
3962
|
+
payload.value,
|
|
3963
|
+
check,
|
|
3964
|
+
payload.present
|
|
3965
|
+
);
|
|
3966
|
+
});
|
|
3967
|
+
if (occurrence === "any") {
|
|
3968
|
+
if (results.some((result) => result.status === "pass")) continue;
|
|
3969
|
+
const firstFail = results.find((result) => result.status !== "pass");
|
|
3970
|
+
findings.push(
|
|
3971
|
+
contractFailFinding(
|
|
3972
|
+
`contract.tool.arguments.${index}`,
|
|
3973
|
+
firstFail.message,
|
|
3974
|
+
selected.slice(0, 1).map((event) => ({
|
|
3975
|
+
runId: event.runId,
|
|
3976
|
+
eventId: event.eventId,
|
|
3977
|
+
kind: event.kind,
|
|
3978
|
+
name: event.name,
|
|
3979
|
+
path: `tool.${check.tool}${check.path}`
|
|
3980
|
+
})),
|
|
3981
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
3982
|
+
{ code: firstFail.code }
|
|
3983
|
+
)
|
|
3984
|
+
);
|
|
3985
|
+
continue;
|
|
3986
|
+
}
|
|
3987
|
+
for (const [selIndex, result] of results.entries()) {
|
|
3988
|
+
if (result.status === "pass") continue;
|
|
3989
|
+
const event = selected[selIndex];
|
|
3990
|
+
findings.push(
|
|
3991
|
+
contractFailFinding(
|
|
3992
|
+
`contract.tool.arguments.${index}`,
|
|
3993
|
+
result.message,
|
|
3994
|
+
[
|
|
3995
|
+
{
|
|
3996
|
+
runId: event.runId,
|
|
3997
|
+
eventId: event.eventId,
|
|
3998
|
+
kind: event.kind,
|
|
3999
|
+
name: event.name,
|
|
4000
|
+
path: `tool.${check.tool}${check.path}`
|
|
4001
|
+
}
|
|
4002
|
+
],
|
|
4003
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
4004
|
+
{ code: result.code }
|
|
4005
|
+
)
|
|
4006
|
+
);
|
|
4007
|
+
}
|
|
4008
|
+
}
|
|
4009
|
+
return findings;
|
|
4010
|
+
}
|
|
4011
|
+
});
|
|
4012
|
+
}
|
|
3431
4013
|
}
|
|
3432
4014
|
if (contract.llm) {
|
|
3433
4015
|
rules.push(
|
|
@@ -3488,6 +4070,46 @@ function contractToRules(contract) {
|
|
|
3488
4070
|
});
|
|
3489
4071
|
}
|
|
3490
4072
|
}
|
|
4073
|
+
if (contract.controls) {
|
|
4074
|
+
const controls = contract.controls;
|
|
4075
|
+
rules.push({
|
|
4076
|
+
id: "contract.controls",
|
|
4077
|
+
category: "run",
|
|
4078
|
+
defaultSeverity: "error",
|
|
4079
|
+
evaluate(context) {
|
|
4080
|
+
const events = context.logicalEvents ?? context.events;
|
|
4081
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
4082
|
+
const observationNames = new Set(outcomes.map((item) => item.name));
|
|
4083
|
+
const runEvidence2 = context.selectedRun ? [
|
|
4084
|
+
{
|
|
4085
|
+
runId: context.selectedRun.runId,
|
|
4086
|
+
kind: "RUN",
|
|
4087
|
+
name: context.selectedRun.name
|
|
4088
|
+
}
|
|
4089
|
+
] : [];
|
|
4090
|
+
return evaluateControlRules(events, controls, runEvidence2, observationNames);
|
|
4091
|
+
}
|
|
4092
|
+
});
|
|
4093
|
+
}
|
|
4094
|
+
if (contract.retry) {
|
|
4095
|
+
const retry = contract.retry;
|
|
4096
|
+
rules.push({
|
|
4097
|
+
id: "contract.retry",
|
|
4098
|
+
category: "run",
|
|
4099
|
+
defaultSeverity: "error",
|
|
4100
|
+
evaluate(context) {
|
|
4101
|
+
const events = context.logicalEvents ?? context.events;
|
|
4102
|
+
context.selectedRun ? [
|
|
4103
|
+
{
|
|
4104
|
+
runId: context.selectedRun.runId,
|
|
4105
|
+
kind: "RUN",
|
|
4106
|
+
name: context.selectedRun.name
|
|
4107
|
+
}
|
|
4108
|
+
] : [];
|
|
4109
|
+
return evaluateRetrySafetyRules(events, retry);
|
|
4110
|
+
}
|
|
4111
|
+
});
|
|
4112
|
+
}
|
|
3491
4113
|
return rules;
|
|
3492
4114
|
}
|
|
3493
4115
|
function validateScopeShape(scope) {
|
|
@@ -3574,7 +4196,7 @@ function validateAlternativesShape(alternatives) {
|
|
|
3574
4196
|
diagnostics.push({
|
|
3575
4197
|
code: "contract.alternatives.empty-branch",
|
|
3576
4198
|
severity: "error",
|
|
3577
|
-
message: `Branch ${branch.id} must declare at least one run/tools/llm/observations rule.`,
|
|
4199
|
+
message: `Branch ${branch.id} must declare at least one run/tools/llm/observations/controls/retry rule.`,
|
|
3578
4200
|
path: `${path}.contract`
|
|
3579
4201
|
});
|
|
3580
4202
|
}
|
|
@@ -3847,6 +4469,40 @@ function lintTraceContract(contract) {
|
|
|
3847
4469
|
path: "tools.requiredOrderMode"
|
|
3848
4470
|
});
|
|
3849
4471
|
}
|
|
4472
|
+
const orderRules = contract.tools?.orderRules ?? [];
|
|
4473
|
+
const seenPairs = /* @__PURE__ */ new Set();
|
|
4474
|
+
for (const [index, rule] of orderRules.entries()) {
|
|
4475
|
+
const key = `${rule.before}\0${rule.after}`;
|
|
4476
|
+
if (seenPairs.has(key)) {
|
|
4477
|
+
diagnostics.push({
|
|
4478
|
+
code: "contract.tools.orderRules.duplicate",
|
|
4479
|
+
severity: "warning",
|
|
4480
|
+
message: `Duplicate orderRules pair ${rule.before} \u2192 ${rule.after}.`,
|
|
4481
|
+
path: `tools.orderRules[${index}]`
|
|
4482
|
+
});
|
|
4483
|
+
}
|
|
4484
|
+
seenPairs.add(key);
|
|
4485
|
+
if (rule.before === rule.after) {
|
|
4486
|
+
diagnostics.push({
|
|
4487
|
+
code: "contract.tools.orderRules.self",
|
|
4488
|
+
severity: "error",
|
|
4489
|
+
message: "orderRules before and after must differ.",
|
|
4490
|
+
path: `tools.orderRules[${index}]`
|
|
4491
|
+
});
|
|
4492
|
+
}
|
|
4493
|
+
}
|
|
4494
|
+
if ((contract.tools?.arguments?.length ?? 0) > 0) {
|
|
4495
|
+
for (const [index, check] of (contract.tools?.arguments ?? []).entries()) {
|
|
4496
|
+
if (!check.path.startsWith("/") && check.path !== "") {
|
|
4497
|
+
diagnostics.push({
|
|
4498
|
+
code: "contract.tools.arguments.path",
|
|
4499
|
+
severity: "error",
|
|
4500
|
+
message: 'tools.arguments path must be a JSON Pointer ("" or start with "/").',
|
|
4501
|
+
path: `tools.arguments[${index}].path`
|
|
4502
|
+
});
|
|
4503
|
+
}
|
|
4504
|
+
}
|
|
4505
|
+
}
|
|
3850
4506
|
return diagnostics;
|
|
3851
4507
|
}
|
|
3852
4508
|
function explainTraceContract(contract) {
|
|
@@ -3890,6 +4546,17 @@ function explainTraceContract(contract) {
|
|
|
3890
4546
|
`Base: requiredOrder [${contract.tools.requiredOrder.join(" \u2192 ")}] mode=${mode}.`
|
|
3891
4547
|
);
|
|
3892
4548
|
}
|
|
4549
|
+
if ((contract.tools.orderRules?.length ?? 0) > 0) {
|
|
4550
|
+
const defaultMode = contract.tools.defaultOccurrenceMode ?? "first-occurrence";
|
|
4551
|
+
lines.push(
|
|
4552
|
+
`Base: ${contract.tools.orderRules.length} orderRules (defaultOccurrenceMode=${defaultMode}).`
|
|
4553
|
+
);
|
|
4554
|
+
}
|
|
4555
|
+
if ((contract.tools.arguments?.length ?? 0) > 0) {
|
|
4556
|
+
lines.push(
|
|
4557
|
+
`Base: ${contract.tools.arguments.length} structured tool-argument check(s).`
|
|
4558
|
+
);
|
|
4559
|
+
}
|
|
3893
4560
|
}
|
|
3894
4561
|
if (contract.llm) {
|
|
3895
4562
|
if (contract.llm.maxCalls !== void 0) {
|
|
@@ -3920,6 +4587,12 @@ function explainTraceContract(contract) {
|
|
|
3920
4587
|
);
|
|
3921
4588
|
}
|
|
3922
4589
|
}
|
|
4590
|
+
if (contract.controls) {
|
|
4591
|
+
lines.push("Base: declared-versus-enforced control checks enabled.");
|
|
4592
|
+
}
|
|
4593
|
+
if (contract.retry) {
|
|
4594
|
+
lines.push("Base: retry/side-effect safety checks enabled.");
|
|
4595
|
+
}
|
|
3923
4596
|
const branches = contract.alternatives?.anyOf ?? [];
|
|
3924
4597
|
if (branches.length > 0) {
|
|
3925
4598
|
lines.push(
|
|
@@ -3936,6 +4609,6 @@ function explainTraceContract(contract) {
|
|
|
3936
4609
|
return lines;
|
|
3937
4610
|
}
|
|
3938
4611
|
|
|
3939
|
-
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractSessionWorkflowMetadata, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveTraceContractScope, runTraceChecks, sessionKeyForRun, summarizeSemanticParity, workflowMetadataForRun };
|
|
3940
|
-
//# sourceMappingURL=chunk-
|
|
3941
|
-
//# sourceMappingURL=chunk-
|
|
4612
|
+
export { buildTraceFacts, createBaselineRegressionRule, createDecisionRule, createGuardrailRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRetrievalRule, createRunDepthRule, createRunDurationRule, createRunEventCountRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureIncompleteRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolFailureRule, createToolOrderingRule, createToolUsageRule, defineTraceContract, deriveFailureFacts, deriveRelationshipFacts, evaluateToolArgumentValue, evaluateTraceContract, evaluateTraceContractRead, explainTraceContract, extractSessionWorkflowMetadata, extractToolArgumentPayload, lintTraceContract, projectLogicalEvents, resolveCanonicalToolName, resolveJsonPointer, resolveTraceContractScope, runTraceChecks, sessionKeyForRun, summarizeSemanticParity, workflowMetadataForRun };
|
|
4613
|
+
//# sourceMappingURL=chunk-VMDCDWBE.mjs.map
|
|
4614
|
+
//# sourceMappingURL=chunk-VMDCDWBE.mjs.map
|