agent-inspect 6.29.5 → 6.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +1 -1
- package/docs/CLI.md +27 -5
- package/docs/TRACE-CONTRACTS.md +10 -7
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-XWQEHKGB.mjs → chunk-BDRJENLM.mjs} +2334 -102
- package/packages/cli/dist/chunk-BDRJENLM.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +3122 -168
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +793 -61
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-XNC4EUN3.mjs → src-ZUGAUXY6.mjs} +3 -3
- package/packages/cli/dist/{src-XNC4EUN3.mjs.map → src-ZUGAUXY6.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +19 -1
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +2 -2
- package/packages/core/dist/advanced.d.ts +2 -2
- package/packages/core/dist/advanced.mjs +2 -2
- package/packages/core/dist/checks.cjs +33 -1
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +2 -2
- package/packages/core/dist/checks.d.ts +2 -2
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-TU47EKKS.mjs → chunk-OOX25GTH.mjs} +35 -3
- package/packages/core/dist/chunk-OOX25GTH.mjs.map +1 -0
- package/packages/core/dist/{index-DvSbXNot.d.ts → index-7hG_qggp.d.ts} +5 -0
- package/packages/core/dist/{index-DKv7esIo.d.cts → index-CQJUNPPe.d.cts} +5 -0
- package/packages/cli/dist/chunk-XWQEHKGB.mjs.map +0 -1
- package/packages/core/dist/chunk-TU47EKKS.mjs.map +0 -1
|
@@ -3388,7 +3388,7 @@ function deriveLastError(runs) {
|
|
|
3388
3388
|
}
|
|
3389
3389
|
function deriveCheckSummary(runs) {
|
|
3390
3390
|
let pass2 = 0;
|
|
3391
|
-
let
|
|
3391
|
+
let fail7 = 0;
|
|
3392
3392
|
let warn2 = 0;
|
|
3393
3393
|
let found = false;
|
|
3394
3394
|
for (const run of runs) {
|
|
@@ -3400,7 +3400,7 @@ function deriveCheckSummary(runs) {
|
|
|
3400
3400
|
found = true;
|
|
3401
3401
|
}
|
|
3402
3402
|
if (typeof record.fail === "number") {
|
|
3403
|
-
|
|
3403
|
+
fail7 += record.fail;
|
|
3404
3404
|
found = true;
|
|
3405
3405
|
}
|
|
3406
3406
|
if (typeof record.warn === "number") {
|
|
@@ -3408,7 +3408,7 @@ function deriveCheckSummary(runs) {
|
|
|
3408
3408
|
found = true;
|
|
3409
3409
|
}
|
|
3410
3410
|
}
|
|
3411
|
-
return found ? { pass: pass2, fail:
|
|
3411
|
+
return found ? { pass: pass2, fail: fail7, warn: warn2 } : void 0;
|
|
3412
3412
|
}
|
|
3413
3413
|
function deriveObservationSummary(runs) {
|
|
3414
3414
|
for (const run of [...runs].sort((a, b) => activityMs(b) - activityMs(a))) {
|
|
@@ -5726,6 +5726,14 @@ function finishedEvents(context, kind) {
|
|
|
5726
5726
|
function toolInvocationEvents(context) {
|
|
5727
5727
|
return semanticEvents(context).filter((event) => event.kind === "TOOL");
|
|
5728
5728
|
}
|
|
5729
|
+
function firstIndexByName(events) {
|
|
5730
|
+
const index = /* @__PURE__ */ new Map();
|
|
5731
|
+
for (const [position, event] of events.entries()) {
|
|
5732
|
+
const name = toolName(event);
|
|
5733
|
+
if (!index.has(name)) index.set(name, position);
|
|
5734
|
+
}
|
|
5735
|
+
return index;
|
|
5736
|
+
}
|
|
5729
5737
|
function isRecord9(value) {
|
|
5730
5738
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
5731
5739
|
}
|
|
@@ -5814,6 +5822,18 @@ function limitFindings(findings, maxFindings) {
|
|
|
5814
5822
|
function hasRedactionMarker(value, markers) {
|
|
5815
5823
|
return markers.some((marker) => value.includes(marker)) || /^\[HASH:[A-Za-z0-9_-]+\]$/.test(value);
|
|
5816
5824
|
}
|
|
5825
|
+
function isFullyRedactedValue(value, markers = ["[REDACTED]", "[REDACTED:"]) {
|
|
5826
|
+
if (typeof value !== "string") return false;
|
|
5827
|
+
const trimmed = value.trim();
|
|
5828
|
+
if (trimmed.length === 0) return false;
|
|
5829
|
+
if (/^\[HASH:[A-Za-z0-9_-]+\]$/.test(trimmed)) return true;
|
|
5830
|
+
if (/^\[REDACTED:[^\]]*\]$/.test(trimmed)) return true;
|
|
5831
|
+
for (const marker of markers) {
|
|
5832
|
+
if (marker.endsWith(":")) continue;
|
|
5833
|
+
if (trimmed === marker) return true;
|
|
5834
|
+
}
|
|
5835
|
+
return false;
|
|
5836
|
+
}
|
|
5817
5837
|
function isSensitiveKey(key, sensitiveKeys) {
|
|
5818
5838
|
return isCredentialSensitiveKey(key, sensitiveKeys);
|
|
5819
5839
|
}
|
|
@@ -6132,8 +6152,26 @@ function createToolUsageRule(options) {
|
|
|
6132
6152
|
const findings = [];
|
|
6133
6153
|
for (const required of options.required ?? []) {
|
|
6134
6154
|
if (!nameSet.has(required)) {
|
|
6155
|
+
const otherKinds = [
|
|
6156
|
+
...new Set(
|
|
6157
|
+
context.events.filter((event) => {
|
|
6158
|
+
const kind = typeof event.kind === "string" ? event.kind.toUpperCase() : "";
|
|
6159
|
+
if (kind === "TOOL") return false;
|
|
6160
|
+
return resolveCanonicalToolName(event) === required || event.name === required || event.name === `tool:${required}` || event.name.endsWith(`:${required}`);
|
|
6161
|
+
}).map(
|
|
6162
|
+
(event) => typeof event.kind === "string" ? event.kind.toUpperCase() : "UNKNOWN"
|
|
6163
|
+
)
|
|
6164
|
+
)
|
|
6165
|
+
].sort();
|
|
6166
|
+
const hint = otherKinds.length > 0 ? ` Required name appears under non-TOOL kind(s): ${otherKinds.join(", ")}. tools.required / requiredOrder match TOOL events only.` : "";
|
|
6135
6167
|
findings.push(
|
|
6136
|
-
failFinding(
|
|
6168
|
+
failFinding(
|
|
6169
|
+
"tool.usage",
|
|
6170
|
+
`Required tool ${required} did not appear.${hint}`,
|
|
6171
|
+
runEvidence(context.selectedRun),
|
|
6172
|
+
required,
|
|
6173
|
+
names
|
|
6174
|
+
)
|
|
6137
6175
|
);
|
|
6138
6176
|
}
|
|
6139
6177
|
}
|
|
@@ -6166,6 +6204,181 @@ function createToolUsageRule(options) {
|
|
|
6166
6204
|
}
|
|
6167
6205
|
};
|
|
6168
6206
|
}
|
|
6207
|
+
function createToolOrderingRule(options) {
|
|
6208
|
+
const ruleId = options.id ?? "tool.order";
|
|
6209
|
+
const mode = options.mode ?? "first-occurrence";
|
|
6210
|
+
return {
|
|
6211
|
+
id: ruleId,
|
|
6212
|
+
category: "tool",
|
|
6213
|
+
defaultSeverity: "error",
|
|
6214
|
+
evaluate(context) {
|
|
6215
|
+
const tools = finishedEvents(context, "TOOL");
|
|
6216
|
+
const index = firstIndexByName(tools);
|
|
6217
|
+
const beforeIndex = index.get(options.before);
|
|
6218
|
+
const afterIndex = index.get(options.after);
|
|
6219
|
+
if (beforeIndex === void 0 || afterIndex === void 0) {
|
|
6220
|
+
return [];
|
|
6221
|
+
}
|
|
6222
|
+
if (mode === "first-occurrence" && beforeIndex >= afterIndex) {
|
|
6223
|
+
return [
|
|
6224
|
+
failFinding(
|
|
6225
|
+
ruleId,
|
|
6226
|
+
`Tool ${options.before} must appear before ${options.after}.`,
|
|
6227
|
+
[eventEvidence(tools[beforeIndex]), eventEvidence(tools[afterIndex])],
|
|
6228
|
+
{ before: options.before, after: options.after },
|
|
6229
|
+
tools.map(toolName)
|
|
6230
|
+
)
|
|
6231
|
+
];
|
|
6232
|
+
}
|
|
6233
|
+
const beforeEvent = tools[beforeIndex];
|
|
6234
|
+
const afterEvent = tools[afterIndex];
|
|
6235
|
+
const beforeEnd = eventEndMs(beforeEvent);
|
|
6236
|
+
const afterStart = eventStartMs(afterEvent);
|
|
6237
|
+
if (mode === "happens-before") {
|
|
6238
|
+
const expected = {
|
|
6239
|
+
before: options.before,
|
|
6240
|
+
after: options.after,
|
|
6241
|
+
mode,
|
|
6242
|
+
relation: "before.end <= after.start"
|
|
6243
|
+
};
|
|
6244
|
+
if (options.before === options.after) {
|
|
6245
|
+
return [
|
|
6246
|
+
failFinding(
|
|
6247
|
+
ruleId,
|
|
6248
|
+
`Tool ${options.before} cannot causally happen before itself.`,
|
|
6249
|
+
[eventEvidence(beforeEvent)],
|
|
6250
|
+
expected,
|
|
6251
|
+
{ code: "tool.order.same-tool" }
|
|
6252
|
+
)
|
|
6253
|
+
];
|
|
6254
|
+
}
|
|
6255
|
+
if (beforeEnd === void 0 || afterStart === void 0) {
|
|
6256
|
+
return [
|
|
6257
|
+
failFinding(
|
|
6258
|
+
ruleId,
|
|
6259
|
+
`Tool order ${options.before} before ${options.after} could not establish causal timing.`,
|
|
6260
|
+
[eventEvidence(beforeEvent), eventEvidence(afterEvent)],
|
|
6261
|
+
expected,
|
|
6262
|
+
{
|
|
6263
|
+
code: "tool.order.interval-unresolved",
|
|
6264
|
+
beforeEndResolved: beforeEnd !== void 0,
|
|
6265
|
+
afterStartResolved: afterStart !== void 0
|
|
6266
|
+
}
|
|
6267
|
+
)
|
|
6268
|
+
];
|
|
6269
|
+
}
|
|
6270
|
+
if (beforeEnd > afterStart) {
|
|
6271
|
+
return [
|
|
6272
|
+
failFinding(
|
|
6273
|
+
ruleId,
|
|
6274
|
+
`Tool ${options.before} must finish before ${options.after} starts.`,
|
|
6275
|
+
[eventEvidence(beforeEvent), eventEvidence(afterEvent)],
|
|
6276
|
+
expected,
|
|
6277
|
+
{
|
|
6278
|
+
beforeEndedAt: new Date(beforeEnd).toISOString(),
|
|
6279
|
+
afterStartedAt: new Date(afterStart).toISOString()
|
|
6280
|
+
}
|
|
6281
|
+
)
|
|
6282
|
+
];
|
|
6283
|
+
}
|
|
6284
|
+
return [];
|
|
6285
|
+
}
|
|
6286
|
+
if (mode === "all-occurrences") {
|
|
6287
|
+
const beforeEvents = tools.filter((event) => toolName(event) === options.before);
|
|
6288
|
+
const afterEvents = tools.filter((event) => toolName(event) === options.after);
|
|
6289
|
+
const expected = {
|
|
6290
|
+
before: options.before,
|
|
6291
|
+
after: options.after,
|
|
6292
|
+
mode,
|
|
6293
|
+
relation: "max(before.end) <= min(after.start)"
|
|
6294
|
+
};
|
|
6295
|
+
if (options.before === options.after) {
|
|
6296
|
+
return [
|
|
6297
|
+
failFinding(
|
|
6298
|
+
ruleId,
|
|
6299
|
+
`Tool ${options.before} cannot causally happen before itself.`,
|
|
6300
|
+
[eventEvidence(beforeEvent)],
|
|
6301
|
+
expected,
|
|
6302
|
+
{ code: "tool.order.same-tool" }
|
|
6303
|
+
)
|
|
6304
|
+
];
|
|
6305
|
+
}
|
|
6306
|
+
let latestBefore;
|
|
6307
|
+
let earliestAfter;
|
|
6308
|
+
let missingBeforeEnds = 0;
|
|
6309
|
+
let missingAfterStarts = 0;
|
|
6310
|
+
let firstMissingBefore;
|
|
6311
|
+
let firstMissingAfter;
|
|
6312
|
+
for (const event of beforeEvents) {
|
|
6313
|
+
const end = eventEndMs(event);
|
|
6314
|
+
if (end === void 0) {
|
|
6315
|
+
missingBeforeEnds += 1;
|
|
6316
|
+
firstMissingBefore ??= event;
|
|
6317
|
+
} else if (latestBefore === void 0 || end > latestBefore.end) {
|
|
6318
|
+
latestBefore = { event, end };
|
|
6319
|
+
}
|
|
6320
|
+
}
|
|
6321
|
+
for (const event of afterEvents) {
|
|
6322
|
+
const start = eventStartMs(event);
|
|
6323
|
+
if (start === void 0) {
|
|
6324
|
+
missingAfterStarts += 1;
|
|
6325
|
+
firstMissingAfter ??= event;
|
|
6326
|
+
} else if (earliestAfter === void 0 || start < earliestAfter.start) {
|
|
6327
|
+
earliestAfter = { event, start };
|
|
6328
|
+
}
|
|
6329
|
+
}
|
|
6330
|
+
if (missingBeforeEnds > 0 || missingAfterStarts > 0 || latestBefore === void 0 || earliestAfter === void 0) {
|
|
6331
|
+
return [
|
|
6332
|
+
failFinding(
|
|
6333
|
+
ruleId,
|
|
6334
|
+
`Tool order ${options.before} before ${options.after} could not establish all causal intervals.`,
|
|
6335
|
+
[firstMissingBefore, firstMissingAfter].filter((event) => event !== void 0).map((event) => eventEvidence(event)),
|
|
6336
|
+
expected,
|
|
6337
|
+
{
|
|
6338
|
+
code: "tool.order.interval-unresolved",
|
|
6339
|
+
missingBeforeEnds,
|
|
6340
|
+
missingAfterStarts
|
|
6341
|
+
}
|
|
6342
|
+
)
|
|
6343
|
+
];
|
|
6344
|
+
}
|
|
6345
|
+
if (latestBefore.end > earliestAfter.start) {
|
|
6346
|
+
return [
|
|
6347
|
+
failFinding(
|
|
6348
|
+
ruleId,
|
|
6349
|
+
`Every ${options.before} tool call must finish before any ${options.after} tool call starts.`,
|
|
6350
|
+
[eventEvidence(latestBefore.event), eventEvidence(earliestAfter.event)],
|
|
6351
|
+
expected,
|
|
6352
|
+
{
|
|
6353
|
+
latestBeforeEndedAt: new Date(latestBefore.end).toISOString(),
|
|
6354
|
+
earliestAfterStartedAt: new Date(earliestAfter.start).toISOString()
|
|
6355
|
+
}
|
|
6356
|
+
)
|
|
6357
|
+
];
|
|
6358
|
+
}
|
|
6359
|
+
return [];
|
|
6360
|
+
}
|
|
6361
|
+
if (beforeEnd !== void 0 && afterStart !== void 0 && beforeEnd > afterStart) {
|
|
6362
|
+
return [
|
|
6363
|
+
{
|
|
6364
|
+
ruleId,
|
|
6365
|
+
severity: "warning",
|
|
6366
|
+
status: "warning",
|
|
6367
|
+
message: `Tool order ${options.before} before ${options.after} satisfied start/encounter order, but intervals overlap so causal completion before the next tool was not established.`,
|
|
6368
|
+
expected: { before: options.before, after: options.after, mode: "first-occurrence" },
|
|
6369
|
+
actual: {
|
|
6370
|
+
code: "tool.order.overlap",
|
|
6371
|
+
beforeEndedAt: beforeEvent.endedAt,
|
|
6372
|
+
afterStartedAt: afterEvent.startedAt ?? afterEvent.timestamp
|
|
6373
|
+
},
|
|
6374
|
+
evidence: [eventEvidence(beforeEvent), eventEvidence(afterEvent)]
|
|
6375
|
+
}
|
|
6376
|
+
];
|
|
6377
|
+
}
|
|
6378
|
+
return [];
|
|
6379
|
+
}
|
|
6380
|
+
};
|
|
6381
|
+
}
|
|
6169
6382
|
function createLlmUsageRule(options) {
|
|
6170
6383
|
return {
|
|
6171
6384
|
id: "llm.usage",
|
|
@@ -6240,6 +6453,47 @@ function createLlmUsageRule(options) {
|
|
|
6240
6453
|
}
|
|
6241
6454
|
};
|
|
6242
6455
|
}
|
|
6456
|
+
function createStructureIncompleteRule(options = {}) {
|
|
6457
|
+
return {
|
|
6458
|
+
id: "structure.incomplete",
|
|
6459
|
+
category: "structure",
|
|
6460
|
+
defaultSeverity: "error",
|
|
6461
|
+
evaluate(context) {
|
|
6462
|
+
const findings = [];
|
|
6463
|
+
if (!options.allowRunning) {
|
|
6464
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
6465
|
+
if (running.length > 0) {
|
|
6466
|
+
findings.push(
|
|
6467
|
+
failFinding(
|
|
6468
|
+
"structure.incomplete",
|
|
6469
|
+
"Trace contains incomplete running events.",
|
|
6470
|
+
running.map((event) => eventEvidence(event, "status")),
|
|
6471
|
+
"no running events",
|
|
6472
|
+
running.length
|
|
6473
|
+
)
|
|
6474
|
+
);
|
|
6475
|
+
}
|
|
6476
|
+
}
|
|
6477
|
+
if (options.requireEndedAtForStarted) {
|
|
6478
|
+
const missingEndedAt = semanticEvents(context).filter(
|
|
6479
|
+
(event) => event.startedAt !== void 0 && event.endedAt === void 0 && event.status !== "running"
|
|
6480
|
+
);
|
|
6481
|
+
if (missingEndedAt.length > 0) {
|
|
6482
|
+
findings.push(
|
|
6483
|
+
failFinding(
|
|
6484
|
+
"structure.incomplete",
|
|
6485
|
+
"Trace contains events with startedAt but no endedAt.",
|
|
6486
|
+
missingEndedAt.map((event) => eventEvidence(event, "endedAt")),
|
|
6487
|
+
"endedAt for started events",
|
|
6488
|
+
missingEndedAt.length
|
|
6489
|
+
)
|
|
6490
|
+
);
|
|
6491
|
+
}
|
|
6492
|
+
}
|
|
6493
|
+
return findings;
|
|
6494
|
+
}
|
|
6495
|
+
};
|
|
6496
|
+
}
|
|
6243
6497
|
function createStructureOrphanRule(options = {}) {
|
|
6244
6498
|
const allowMarkedUnresolved = options.allowMarkedUnresolved ?? true;
|
|
6245
6499
|
return {
|
|
@@ -6498,6 +6752,7 @@ function createSafetyRedactionRule(options = {}) {
|
|
|
6498
6752
|
function createSafetyRawContentRule(options = {}) {
|
|
6499
6753
|
const forbiddenKeys = options.forbiddenKeys ?? DEFAULT_RAW_CONTENT_KEYS;
|
|
6500
6754
|
const safePathPrefixes = options.safePathPrefixes ?? DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES;
|
|
6755
|
+
const markers = options.redactedMarkers ?? ["[REDACTED]", "[REDACTED:"];
|
|
6501
6756
|
return {
|
|
6502
6757
|
id: "safety.rawPrompt",
|
|
6503
6758
|
category: "safety",
|
|
@@ -6508,6 +6763,7 @@ function createSafetyRawContentRule(options = {}) {
|
|
|
6508
6763
|
for (const entry of eventValueEntries(event, { includeSummaries: options.includeSummaries })) {
|
|
6509
6764
|
const key = entry.key ?? lastPathSegment(entry.path);
|
|
6510
6765
|
if (!isRawContentPath(entry.path, key, forbiddenKeys, safePathPrefixes)) continue;
|
|
6766
|
+
if (isFullyRedactedValue(entry.value, markers)) continue;
|
|
6511
6767
|
findings.push(
|
|
6512
6768
|
failFinding(
|
|
6513
6769
|
"safety.rawPrompt",
|
|
@@ -7003,6 +7259,198 @@ var init_checks2 = __esm({
|
|
|
7003
7259
|
});
|
|
7004
7260
|
|
|
7005
7261
|
// packages/core/src/checks/contract-scope.ts
|
|
7262
|
+
function diagnostic2(code, message, ruleId) {
|
|
7263
|
+
return {
|
|
7264
|
+
code,
|
|
7265
|
+
message,
|
|
7266
|
+
severity: "error",
|
|
7267
|
+
...ruleId !== void 0 ? { ruleId } : {}
|
|
7268
|
+
};
|
|
7269
|
+
}
|
|
7270
|
+
function isNonEmptyString4(value) {
|
|
7271
|
+
return typeof value === "string" && value.trim() !== "";
|
|
7272
|
+
}
|
|
7273
|
+
function scopeEntries(scope) {
|
|
7274
|
+
const out = [];
|
|
7275
|
+
if (isNonEmptyString4(scope.runId)) out.push(["runId", scope.runId.trim()]);
|
|
7276
|
+
if (isNonEmptyString4(scope.subAgentId)) out.push(["subAgentId", scope.subAgentId.trim()]);
|
|
7277
|
+
if (isNonEmptyString4(scope.groupId)) out.push(["groupId", scope.groupId.trim()]);
|
|
7278
|
+
if (isNonEmptyString4(scope.workflowStep)) {
|
|
7279
|
+
out.push(["workflowStep", scope.workflowStep.trim()]);
|
|
7280
|
+
}
|
|
7281
|
+
if (isNonEmptyString4(scope.rootEventId)) {
|
|
7282
|
+
out.push(["rootEventId", scope.rootEventId.trim()]);
|
|
7283
|
+
}
|
|
7284
|
+
return out;
|
|
7285
|
+
}
|
|
7286
|
+
function mergeWorkflowMeta(into, from) {
|
|
7287
|
+
if (!from) return;
|
|
7288
|
+
for (const [key, value] of Object.entries(from)) {
|
|
7289
|
+
if (value === void 0) continue;
|
|
7290
|
+
if (into[key] === void 0) {
|
|
7291
|
+
into[key] = value;
|
|
7292
|
+
}
|
|
7293
|
+
}
|
|
7294
|
+
}
|
|
7295
|
+
function workflowMetadataForRun(events, runId) {
|
|
7296
|
+
const preferred = {};
|
|
7297
|
+
const fallback = {};
|
|
7298
|
+
for (const event of events) {
|
|
7299
|
+
if (event.runId !== runId) continue;
|
|
7300
|
+
const attrs = event.attributes;
|
|
7301
|
+
if (!attrs || typeof attrs !== "object") continue;
|
|
7302
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
7303
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
7304
|
+
const bag = {};
|
|
7305
|
+
mergeWorkflowMeta(bag, direct);
|
|
7306
|
+
mergeWorkflowMeta(bag, nested);
|
|
7307
|
+
if (event.kind === "RUN") {
|
|
7308
|
+
mergeWorkflowMeta(preferred, bag);
|
|
7309
|
+
} else {
|
|
7310
|
+
mergeWorkflowMeta(fallback, bag);
|
|
7311
|
+
}
|
|
7312
|
+
}
|
|
7313
|
+
mergeWorkflowMeta(preferred, fallback);
|
|
7314
|
+
return preferred;
|
|
7315
|
+
}
|
|
7316
|
+
function flattenNodes2(nodes) {
|
|
7317
|
+
return nodes.flatMap((node) => [node, ...flattenNodes2(node.children)]);
|
|
7318
|
+
}
|
|
7319
|
+
function collectSubtreeEventIds(run, rootEventId) {
|
|
7320
|
+
const nodes = flattenNodes2(run.children);
|
|
7321
|
+
const byId = new Map(nodes.map((node) => [node.event.eventId, node]));
|
|
7322
|
+
const root = byId.get(rootEventId);
|
|
7323
|
+
if (!root) return void 0;
|
|
7324
|
+
const ids = /* @__PURE__ */ new Set();
|
|
7325
|
+
const visit = (node) => {
|
|
7326
|
+
ids.add(node.event.eventId);
|
|
7327
|
+
for (const child of node.children) visit(child);
|
|
7328
|
+
};
|
|
7329
|
+
visit(root);
|
|
7330
|
+
return ids;
|
|
7331
|
+
}
|
|
7332
|
+
function filterTree(nodes, keep) {
|
|
7333
|
+
const out = [];
|
|
7334
|
+
for (const node of nodes) {
|
|
7335
|
+
if (!keep.has(node.event.eventId)) continue;
|
|
7336
|
+
out.push({
|
|
7337
|
+
...node,
|
|
7338
|
+
children: filterTree(node.children, keep)
|
|
7339
|
+
});
|
|
7340
|
+
}
|
|
7341
|
+
return out;
|
|
7342
|
+
}
|
|
7343
|
+
function projectInput(input3, run, eventIds) {
|
|
7344
|
+
const runEvents = input3.read.events.filter((event) => event.runId === run.runId);
|
|
7345
|
+
const projectedEvents = eventIds === void 0 ? runEvents : runEvents.filter((event) => eventIds.has(event.eventId) || event.kind === "RUN");
|
|
7346
|
+
const projectedChildren = eventIds === void 0 ? run.children : filterTree(run.children, eventIds);
|
|
7347
|
+
const projectedRun = {
|
|
7348
|
+
...run,
|
|
7349
|
+
children: projectedChildren,
|
|
7350
|
+
metadata: {
|
|
7351
|
+
...run.metadata,
|
|
7352
|
+
totalEvents: flattenNodes2(projectedChildren).length
|
|
7353
|
+
}
|
|
7354
|
+
};
|
|
7355
|
+
return {
|
|
7356
|
+
evidenceEventCount: projectedEvents.length,
|
|
7357
|
+
input: {
|
|
7358
|
+
...input3,
|
|
7359
|
+
selectedRun: projectedRun,
|
|
7360
|
+
read: {
|
|
7361
|
+
...input3.read,
|
|
7362
|
+
runs: [projectedRun],
|
|
7363
|
+
events: projectedEvents
|
|
7364
|
+
}
|
|
7365
|
+
}
|
|
7366
|
+
};
|
|
7367
|
+
}
|
|
7368
|
+
function resolveTraceContractScope(input3, scope) {
|
|
7369
|
+
if (scope === void 0) {
|
|
7370
|
+
return { diagnostics: [] };
|
|
7371
|
+
}
|
|
7372
|
+
const entries = scopeEntries(scope);
|
|
7373
|
+
if (entries.length === 0) {
|
|
7374
|
+
return {
|
|
7375
|
+
diagnostics: [
|
|
7376
|
+
diagnostic2(
|
|
7377
|
+
"AI_CHECK_INVALID_CONFIG",
|
|
7378
|
+
"TraceContract scope requires at least one explicit selector (runId, subAgentId, groupId, workflowStep, or rootEventId).",
|
|
7379
|
+
"contract.scope.empty"
|
|
7380
|
+
)
|
|
7381
|
+
]
|
|
7382
|
+
};
|
|
7383
|
+
}
|
|
7384
|
+
const matchedSelectors = Object.fromEntries(entries);
|
|
7385
|
+
const actorKeys = entries.filter(([key]) => key !== "rootEventId");
|
|
7386
|
+
const rootEventId = matchedSelectors.rootEventId;
|
|
7387
|
+
let candidates = [...input3.read.runs];
|
|
7388
|
+
if (matchedSelectors.runId) {
|
|
7389
|
+
candidates = candidates.filter((run2) => run2.runId === matchedSelectors.runId);
|
|
7390
|
+
}
|
|
7391
|
+
for (const [key, value] of actorKeys) {
|
|
7392
|
+
if (key === "runId") continue;
|
|
7393
|
+
candidates = candidates.filter((run2) => {
|
|
7394
|
+
const meta2 = workflowMetadataForRun(input3.read.events, run2.runId);
|
|
7395
|
+
const actual = meta2[key];
|
|
7396
|
+
return actual === value;
|
|
7397
|
+
});
|
|
7398
|
+
}
|
|
7399
|
+
if (rootEventId && actorKeys.length === 0) {
|
|
7400
|
+
candidates = candidates.filter(
|
|
7401
|
+
(run2) => flattenNodes2(run2.children).some((node) => node.event.eventId === rootEventId)
|
|
7402
|
+
);
|
|
7403
|
+
}
|
|
7404
|
+
if (candidates.length === 0) {
|
|
7405
|
+
return {
|
|
7406
|
+
diagnostics: [
|
|
7407
|
+
diagnostic2(
|
|
7408
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
7409
|
+
`TraceContract scope matched zero actors for ${JSON.stringify(matchedSelectors)}. Missing actor metadata does not fall back to the whole session.`,
|
|
7410
|
+
"contract.scope.zero-match"
|
|
7411
|
+
)
|
|
7412
|
+
]
|
|
7413
|
+
};
|
|
7414
|
+
}
|
|
7415
|
+
if (candidates.length > 1) {
|
|
7416
|
+
return {
|
|
7417
|
+
diagnostics: [
|
|
7418
|
+
diagnostic2(
|
|
7419
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
7420
|
+
`TraceContract scope matched ${candidates.length} actors for ${JSON.stringify(matchedSelectors)} (runIds: ${candidates.map((run2) => run2.runId).sort().join(", ")}). Singular actor selectors must resolve to exactly one run.`,
|
|
7421
|
+
"contract.scope.ambiguous"
|
|
7422
|
+
)
|
|
7423
|
+
]
|
|
7424
|
+
};
|
|
7425
|
+
}
|
|
7426
|
+
const run = candidates[0];
|
|
7427
|
+
let eventIds;
|
|
7428
|
+
if (rootEventId) {
|
|
7429
|
+
eventIds = collectSubtreeEventIds(run, rootEventId);
|
|
7430
|
+
if (eventIds === void 0) {
|
|
7431
|
+
return {
|
|
7432
|
+
diagnostics: [
|
|
7433
|
+
diagnostic2(
|
|
7434
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
7435
|
+
`TraceContract scope rootEventId ${rootEventId} was not found in run ${run.runId}.`,
|
|
7436
|
+
"contract.scope.root-missing"
|
|
7437
|
+
)
|
|
7438
|
+
]
|
|
7439
|
+
};
|
|
7440
|
+
}
|
|
7441
|
+
}
|
|
7442
|
+
const projected = projectInput(input3, run, eventIds);
|
|
7443
|
+
return {
|
|
7444
|
+
resolved: {
|
|
7445
|
+
runId: run.runId,
|
|
7446
|
+
...rootEventId !== void 0 ? { rootEventId } : {},
|
|
7447
|
+
matchedSelectors,
|
|
7448
|
+
evidenceEventCount: projected.evidenceEventCount,
|
|
7449
|
+
input: projected.input
|
|
7450
|
+
},
|
|
7451
|
+
diagnostics: []
|
|
7452
|
+
};
|
|
7453
|
+
}
|
|
7006
7454
|
var init_contract_scope = __esm({
|
|
7007
7455
|
"packages/core/src/checks/contract-scope.ts"() {
|
|
7008
7456
|
init_metadata();
|
|
@@ -7010,6 +7458,146 @@ var init_contract_scope = __esm({
|
|
|
7010
7458
|
});
|
|
7011
7459
|
|
|
7012
7460
|
// packages/core/src/checks/control-rules.ts
|
|
7461
|
+
function asStringList(value) {
|
|
7462
|
+
if (!Array.isArray(value)) return void 0;
|
|
7463
|
+
const out = [];
|
|
7464
|
+
for (const item of value) {
|
|
7465
|
+
if (typeof item !== "string" || item.trim() === "") return void 0;
|
|
7466
|
+
out.push(item.trim());
|
|
7467
|
+
}
|
|
7468
|
+
return out;
|
|
7469
|
+
}
|
|
7470
|
+
function attributeList(events, attribute) {
|
|
7471
|
+
if (!attribute) return void 0;
|
|
7472
|
+
for (const event of events) {
|
|
7473
|
+
const attrs = event.attributes;
|
|
7474
|
+
if (!attrs || typeof attrs !== "object") continue;
|
|
7475
|
+
const direct = asStringList(attrs[attribute]);
|
|
7476
|
+
if (direct) return direct;
|
|
7477
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? asStringList(attrs.metadata[attribute]) : void 0;
|
|
7478
|
+
if (nested) return nested;
|
|
7479
|
+
}
|
|
7480
|
+
return void 0;
|
|
7481
|
+
}
|
|
7482
|
+
function fail(ruleId, message, evidence, expected, actual) {
|
|
7483
|
+
return {
|
|
7484
|
+
ruleId,
|
|
7485
|
+
severity: "error",
|
|
7486
|
+
status: "fail",
|
|
7487
|
+
message,
|
|
7488
|
+
...expected !== void 0 ? { expected } : {},
|
|
7489
|
+
...actual !== void 0 ? { actual } : {},
|
|
7490
|
+
evidence: [...evidence]
|
|
7491
|
+
};
|
|
7492
|
+
}
|
|
7493
|
+
function sortedUnique(values) {
|
|
7494
|
+
return [...new Set(values)].sort((a, b) => a.localeCompare(b));
|
|
7495
|
+
}
|
|
7496
|
+
function setEqual(left, right) {
|
|
7497
|
+
const a = sortedUnique(left);
|
|
7498
|
+
const b = sortedUnique(right);
|
|
7499
|
+
return a.length === b.length && a.every((value, index) => value === b[index]);
|
|
7500
|
+
}
|
|
7501
|
+
function evaluateControlRules(events, rules, runEvidence2, observationNames) {
|
|
7502
|
+
const findings = [];
|
|
7503
|
+
const declared = rules.declaredTools ?? attributeList(events, rules.declaredToolsAttribute);
|
|
7504
|
+
const enforced = rules.enforcedTools ?? attributeList(events, rules.enforcedToolsAttribute);
|
|
7505
|
+
if (rules.requireDeclaredMatchesEnforced) {
|
|
7506
|
+
if (declared === void 0 || enforced === void 0) {
|
|
7507
|
+
findings.push(
|
|
7508
|
+
fail(
|
|
7509
|
+
"contract.controls.declared-matches-enforced",
|
|
7510
|
+
"Declared and enforced tool sets could not both be resolved.",
|
|
7511
|
+
runEvidence2,
|
|
7512
|
+
{ declaredDefined: declared !== void 0, enforcedDefined: enforced !== void 0 },
|
|
7513
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
7514
|
+
)
|
|
7515
|
+
);
|
|
7516
|
+
} else if (!setEqual(declared, enforced)) {
|
|
7517
|
+
findings.push(
|
|
7518
|
+
fail(
|
|
7519
|
+
"contract.controls.declared-matches-enforced",
|
|
7520
|
+
"Declared tools differ from enforced tools.",
|
|
7521
|
+
runEvidence2,
|
|
7522
|
+
sortedUnique(declared),
|
|
7523
|
+
sortedUnique(enforced)
|
|
7524
|
+
)
|
|
7525
|
+
);
|
|
7526
|
+
}
|
|
7527
|
+
}
|
|
7528
|
+
const observedTools = sortedUnique(
|
|
7529
|
+
events.filter((event) => event.kind === "TOOL" && event.status !== "running").map((event) => resolveCanonicalToolName(event))
|
|
7530
|
+
);
|
|
7531
|
+
if (rules.requireObservedWithinEnforced) {
|
|
7532
|
+
if (enforced === void 0) {
|
|
7533
|
+
findings.push(
|
|
7534
|
+
fail(
|
|
7535
|
+
"contract.controls.observed-within-enforced",
|
|
7536
|
+
"Enforced tool set unavailable for observed-within-enforced check.",
|
|
7537
|
+
runEvidence2,
|
|
7538
|
+
void 0,
|
|
7539
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
7540
|
+
)
|
|
7541
|
+
);
|
|
7542
|
+
} else {
|
|
7543
|
+
const enforcedSet = new Set(enforced);
|
|
7544
|
+
const outside = observedTools.filter((name) => !enforcedSet.has(name));
|
|
7545
|
+
if (outside.length > 0) {
|
|
7546
|
+
findings.push(
|
|
7547
|
+
fail(
|
|
7548
|
+
"contract.controls.observed-within-enforced",
|
|
7549
|
+
`Observed tools outside enforced allowlist: ${outside.join(", ")}.`,
|
|
7550
|
+
runEvidence2,
|
|
7551
|
+
sortedUnique(enforced),
|
|
7552
|
+
outside
|
|
7553
|
+
)
|
|
7554
|
+
);
|
|
7555
|
+
}
|
|
7556
|
+
}
|
|
7557
|
+
}
|
|
7558
|
+
if (rules.requireObservedWithinDeclared) {
|
|
7559
|
+
if (declared === void 0) {
|
|
7560
|
+
findings.push(
|
|
7561
|
+
fail(
|
|
7562
|
+
"contract.controls.observed-within-declared",
|
|
7563
|
+
"Declared tool set unavailable for observed-within-declared check.",
|
|
7564
|
+
runEvidence2,
|
|
7565
|
+
void 0,
|
|
7566
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
7567
|
+
)
|
|
7568
|
+
);
|
|
7569
|
+
} else {
|
|
7570
|
+
const declaredSet = new Set(declared);
|
|
7571
|
+
const outside = observedTools.filter((name) => !declaredSet.has(name));
|
|
7572
|
+
if (outside.length > 0) {
|
|
7573
|
+
findings.push(
|
|
7574
|
+
fail(
|
|
7575
|
+
"contract.controls.observed-within-declared",
|
|
7576
|
+
`Observed tools outside declared set: ${outside.join(", ")}.`,
|
|
7577
|
+
runEvidence2,
|
|
7578
|
+
sortedUnique(declared),
|
|
7579
|
+
outside
|
|
7580
|
+
)
|
|
7581
|
+
);
|
|
7582
|
+
}
|
|
7583
|
+
}
|
|
7584
|
+
}
|
|
7585
|
+
for (const stage of rules.requiredStages ?? []) {
|
|
7586
|
+
const observation = stage.observation ?? `control.${stage.stage}`;
|
|
7587
|
+
if (!observationNames.has(observation)) {
|
|
7588
|
+
findings.push(
|
|
7589
|
+
fail(
|
|
7590
|
+
`contract.controls.stage.${stage.stage}`,
|
|
7591
|
+
`Required control stage observation missing: ${observation}.`,
|
|
7592
|
+
runEvidence2,
|
|
7593
|
+
observation,
|
|
7594
|
+
[...observationNames]
|
|
7595
|
+
)
|
|
7596
|
+
);
|
|
7597
|
+
}
|
|
7598
|
+
}
|
|
7599
|
+
return findings;
|
|
7600
|
+
}
|
|
7013
7601
|
var init_control_rules = __esm({
|
|
7014
7602
|
"packages/core/src/checks/control-rules.ts"() {
|
|
7015
7603
|
init_logical_events();
|
|
@@ -7017,38 +7605,1047 @@ var init_control_rules = __esm({
|
|
|
7017
7605
|
});
|
|
7018
7606
|
|
|
7019
7607
|
// packages/core/src/checks/canonical-equality.ts
|
|
7020
|
-
|
|
7021
|
-
|
|
7608
|
+
function sortKeysDeep(input3, depth, maxDepth) {
|
|
7609
|
+
if (depth > maxDepth) {
|
|
7610
|
+
throw new Error("AI_CHECK_CANONICAL_DEPTH_EXCEEDED");
|
|
7022
7611
|
}
|
|
7023
|
-
|
|
7024
|
-
|
|
7025
|
-
|
|
7026
|
-
var init_tool_arguments = __esm({
|
|
7027
|
-
"packages/core/src/checks/tool-arguments.ts"() {
|
|
7028
|
-
init_canonical_equality();
|
|
7612
|
+
if (input3 === null || typeof input3 !== "object") return input3;
|
|
7613
|
+
if (Array.isArray(input3)) {
|
|
7614
|
+
return input3.map((item) => sortKeysDeep(item, depth + 1, maxDepth));
|
|
7029
7615
|
}
|
|
7030
|
-
|
|
7031
|
-
|
|
7032
|
-
|
|
7033
|
-
|
|
7034
|
-
"packages/core/src/checks/retry-safety.ts"() {
|
|
7035
|
-
init_logical_events();
|
|
7036
|
-
init_metadata();
|
|
7616
|
+
const record = input3;
|
|
7617
|
+
const out = {};
|
|
7618
|
+
for (const key of Object.keys(record).sort()) {
|
|
7619
|
+
out[key] = sortKeysDeep(record[key], depth + 1, maxDepth);
|
|
7037
7620
|
}
|
|
7038
|
-
|
|
7039
|
-
|
|
7040
|
-
|
|
7041
|
-
|
|
7042
|
-
|
|
7043
|
-
|
|
7044
|
-
|
|
7045
|
-
|
|
7621
|
+
return out;
|
|
7622
|
+
}
|
|
7623
|
+
function encodeCanonicalStructured(value, limits) {
|
|
7624
|
+
const maxDepth = DEFAULT_MAX_DEPTH;
|
|
7625
|
+
const maxBytes = DEFAULT_MAX_BYTES;
|
|
7626
|
+
try {
|
|
7627
|
+
const sorted = sortKeysDeep(value, 0, maxDepth);
|
|
7628
|
+
const encoded = JSON.stringify(sorted);
|
|
7629
|
+
if (encoded === void 0) {
|
|
7630
|
+
return {
|
|
7631
|
+
ok: false,
|
|
7632
|
+
reason: "Value is not JSON-serializable for canonical comparison.",
|
|
7633
|
+
code: "AI_CHECK_CANONICAL_UNSERIALIZABLE"
|
|
7634
|
+
};
|
|
7635
|
+
}
|
|
7636
|
+
if (Buffer.byteLength(encoded, "utf8") > maxBytes) {
|
|
7637
|
+
return {
|
|
7638
|
+
ok: false,
|
|
7639
|
+
reason: "Canonical structured value exceeds bounded size limit.",
|
|
7640
|
+
code: "AI_CHECK_CANONICAL_SIZE_EXCEEDED"
|
|
7641
|
+
};
|
|
7642
|
+
}
|
|
7643
|
+
return { ok: true, value: encoded };
|
|
7644
|
+
} catch (error) {
|
|
7645
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
7646
|
+
if (message === "AI_CHECK_CANONICAL_DEPTH_EXCEEDED") {
|
|
7647
|
+
return {
|
|
7648
|
+
ok: false,
|
|
7649
|
+
reason: "Canonical structured value exceeds bounded depth limit.",
|
|
7650
|
+
code: "AI_CHECK_CANONICAL_DEPTH_EXCEEDED"
|
|
7651
|
+
};
|
|
7652
|
+
}
|
|
7653
|
+
return {
|
|
7654
|
+
ok: false,
|
|
7655
|
+
reason: "Canonical structured encoding failed.",
|
|
7656
|
+
code: "AI_CHECK_CANONICAL_UNSERIALIZABLE"
|
|
7657
|
+
};
|
|
7658
|
+
}
|
|
7659
|
+
}
|
|
7660
|
+
function canonicalStructuredEqual(left, right, limits) {
|
|
7661
|
+
const a = encodeCanonicalStructured(left);
|
|
7662
|
+
if (!a.ok) return { equal: false, reason: a.reason, code: a.code };
|
|
7663
|
+
const b = encodeCanonicalStructured(right);
|
|
7664
|
+
if (!b.ok) return { equal: false, reason: b.reason, code: b.code };
|
|
7665
|
+
if (a.value !== b.value) {
|
|
7666
|
+
return {
|
|
7667
|
+
equal: false,
|
|
7668
|
+
reason: "Canonical structured values differ.",
|
|
7669
|
+
code: "AI_CHECK_CANONICAL_NOT_EQUAL"
|
|
7670
|
+
};
|
|
7671
|
+
}
|
|
7672
|
+
return { equal: true };
|
|
7673
|
+
}
|
|
7674
|
+
var DEFAULT_MAX_DEPTH, DEFAULT_MAX_BYTES;
|
|
7675
|
+
var init_canonical_equality = __esm({
|
|
7676
|
+
"packages/core/src/checks/canonical-equality.ts"() {
|
|
7677
|
+
DEFAULT_MAX_DEPTH = 8;
|
|
7678
|
+
DEFAULT_MAX_BYTES = 16384;
|
|
7679
|
+
}
|
|
7680
|
+
});
|
|
7681
|
+
|
|
7682
|
+
// packages/core/src/checks/tool-arguments.ts
|
|
7683
|
+
function decodeToken(token) {
|
|
7684
|
+
return token.replace(/~1/g, "/").replace(/~0/g, "~");
|
|
7685
|
+
}
|
|
7686
|
+
function resolveJsonPointer(document, pointer) {
|
|
7687
|
+
if (pointer === "") {
|
|
7688
|
+
return { found: true, value: document };
|
|
7689
|
+
}
|
|
7690
|
+
if (!pointer.startsWith("/")) {
|
|
7691
|
+
return { found: false };
|
|
7692
|
+
}
|
|
7693
|
+
const tokens = pointer.slice(1).split("/").map(decodeToken);
|
|
7694
|
+
let current = document;
|
|
7695
|
+
for (const token of tokens) {
|
|
7696
|
+
if (current === null || current === void 0) {
|
|
7697
|
+
return { found: false };
|
|
7698
|
+
}
|
|
7699
|
+
if (Array.isArray(current)) {
|
|
7700
|
+
if (!/^(0|[1-9][0-9]*)$/.test(token)) {
|
|
7701
|
+
return { found: false };
|
|
7702
|
+
}
|
|
7703
|
+
const index = Number(token);
|
|
7704
|
+
if (index >= current.length) {
|
|
7705
|
+
return { found: false };
|
|
7706
|
+
}
|
|
7707
|
+
current = current[index];
|
|
7708
|
+
continue;
|
|
7709
|
+
}
|
|
7710
|
+
if (typeof current !== "object") {
|
|
7711
|
+
return { found: false };
|
|
7712
|
+
}
|
|
7713
|
+
const record = current;
|
|
7714
|
+
if (!Object.prototype.hasOwnProperty.call(record, token)) {
|
|
7715
|
+
return { found: false };
|
|
7716
|
+
}
|
|
7717
|
+
current = record[token];
|
|
7718
|
+
}
|
|
7719
|
+
return { found: true, value: current };
|
|
7720
|
+
}
|
|
7721
|
+
function jsonType(value) {
|
|
7722
|
+
if (value === null) return "null";
|
|
7723
|
+
if (Array.isArray(value)) return "array";
|
|
7724
|
+
return typeof value;
|
|
7725
|
+
}
|
|
7726
|
+
function valuesEqual(left, right) {
|
|
7727
|
+
return canonicalStructuredEqual(left, right).equal === true;
|
|
7728
|
+
}
|
|
7729
|
+
function isStructuredArgumentEvidence(value) {
|
|
7730
|
+
return value !== void 0 && value !== null && typeof value === "object";
|
|
7731
|
+
}
|
|
7732
|
+
function extractToolArgumentPayload(event) {
|
|
7733
|
+
const attrs = event.attributes ?? {};
|
|
7734
|
+
for (const key of ["arguments", "input", "toolArguments"]) {
|
|
7735
|
+
const candidate = attrs[key];
|
|
7736
|
+
if (isStructuredArgumentEvidence(candidate)) {
|
|
7737
|
+
return { present: true, value: candidate };
|
|
7738
|
+
}
|
|
7739
|
+
}
|
|
7740
|
+
const metadata = attrs.metadata;
|
|
7741
|
+
if (isStructuredArgumentEvidence(metadata) && !Array.isArray(metadata)) {
|
|
7742
|
+
const nested = metadata;
|
|
7743
|
+
for (const key of ["arguments", "input", "toolArguments"]) {
|
|
7744
|
+
const candidate = nested[key];
|
|
7745
|
+
if (isStructuredArgumentEvidence(candidate)) {
|
|
7746
|
+
return { present: true, value: candidate };
|
|
7747
|
+
}
|
|
7748
|
+
}
|
|
7749
|
+
}
|
|
7750
|
+
if (isStructuredArgumentEvidence(event.inputSummary)) {
|
|
7751
|
+
return { present: true, value: event.inputSummary };
|
|
7752
|
+
}
|
|
7753
|
+
return { present: false, value: void 0 };
|
|
7754
|
+
}
|
|
7755
|
+
function evaluateToolArgumentValue(value, check, evidencePresent) {
|
|
7756
|
+
if (!evidencePresent) {
|
|
7757
|
+
return {
|
|
7758
|
+
status: "unavailable",
|
|
7759
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
7760
|
+
message: `Structured argument evidence unavailable for tool ${check.tool} at ${check.path}.`
|
|
7761
|
+
};
|
|
7762
|
+
}
|
|
7763
|
+
const resolved = resolveJsonPointer(value, check.path);
|
|
7764
|
+
if (check.operator === "exists") {
|
|
7765
|
+
return resolved.found ? { status: "pass" } : {
|
|
7766
|
+
status: "fail",
|
|
7767
|
+
code: "AI_CHECK_TOOL_ARGUMENT_MISSING",
|
|
7768
|
+
message: `Expected path ${check.path} to exist on tool ${check.tool}.`
|
|
7769
|
+
};
|
|
7770
|
+
}
|
|
7771
|
+
if (!resolved.found) {
|
|
7772
|
+
return {
|
|
7773
|
+
status: "unavailable",
|
|
7774
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
7775
|
+
message: `Path ${check.path} missing for tool ${check.tool}.`
|
|
7776
|
+
};
|
|
7777
|
+
}
|
|
7778
|
+
if (check.operator === "type") {
|
|
7779
|
+
const actual = jsonType(resolved.value);
|
|
7780
|
+
if (check.type === void 0) {
|
|
7781
|
+
return {
|
|
7782
|
+
status: "fail",
|
|
7783
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
7784
|
+
message: "type operator requires check.type"
|
|
7785
|
+
};
|
|
7786
|
+
}
|
|
7787
|
+
return actual === check.type ? { status: "pass" } : {
|
|
7788
|
+
status: "fail",
|
|
7789
|
+
code: "AI_CHECK_TOOL_ARGUMENT_TYPE",
|
|
7790
|
+
message: `Expected type ${check.type} at ${check.path} for tool ${check.tool}; found ${actual}.`
|
|
7791
|
+
};
|
|
7792
|
+
}
|
|
7793
|
+
if (check.operator === "equals") {
|
|
7794
|
+
return valuesEqual(resolved.value, check.expected) ? { status: "pass" } : {
|
|
7795
|
+
status: "fail",
|
|
7796
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EQUALS",
|
|
7797
|
+
message: `Value at ${check.path} for tool ${check.tool} did not equal expected (bounded comparison).`
|
|
7798
|
+
};
|
|
7799
|
+
}
|
|
7800
|
+
if (check.operator === "oneOf") {
|
|
7801
|
+
const options = check.oneOf ?? [];
|
|
7802
|
+
if (options.length === 0) {
|
|
7803
|
+
return {
|
|
7804
|
+
status: "fail",
|
|
7805
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
7806
|
+
message: "oneOf operator requires a non-empty oneOf array"
|
|
7807
|
+
};
|
|
7808
|
+
}
|
|
7809
|
+
return options.some((option) => valuesEqual(resolved.value, option)) ? { status: "pass" } : {
|
|
7810
|
+
status: "fail",
|
|
7811
|
+
code: "AI_CHECK_TOOL_ARGUMENT_ONE_OF",
|
|
7812
|
+
message: `Value at ${check.path} for tool ${check.tool} was not in the allowed oneOf set.`
|
|
7813
|
+
};
|
|
7814
|
+
}
|
|
7815
|
+
return {
|
|
7816
|
+
status: "fail",
|
|
7817
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
7818
|
+
message: `Unsupported operator.`
|
|
7819
|
+
};
|
|
7820
|
+
}
|
|
7821
|
+
var init_tool_arguments = __esm({
|
|
7822
|
+
"packages/core/src/checks/tool-arguments.ts"() {
|
|
7823
|
+
init_canonical_equality();
|
|
7824
|
+
}
|
|
7825
|
+
});
|
|
7826
|
+
|
|
7827
|
+
// packages/core/src/checks/retry-safety.ts
|
|
7828
|
+
function fail2(ruleId, message, evidence, expected, actual) {
|
|
7829
|
+
return {
|
|
7830
|
+
ruleId,
|
|
7831
|
+
severity: "error",
|
|
7832
|
+
status: "fail",
|
|
7833
|
+
message,
|
|
7834
|
+
...expected !== void 0 ? { expected } : {},
|
|
7835
|
+
...actual !== void 0 ? { actual } : {},
|
|
7836
|
+
evidence: [...evidence]
|
|
7837
|
+
};
|
|
7838
|
+
}
|
|
7839
|
+
function workflowFor(event) {
|
|
7840
|
+
const attrs = event.attributes;
|
|
7841
|
+
if (!attrs || typeof attrs !== "object") return {};
|
|
7842
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
7843
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
7844
|
+
return { ...nested, ...direct };
|
|
7845
|
+
}
|
|
7846
|
+
function hasIdempotencyEvidence(event) {
|
|
7847
|
+
const workflow = workflowFor(event);
|
|
7848
|
+
if (typeof workflow.idempotencyKey === "string" && workflow.idempotencyKey.trim() !== "") {
|
|
7849
|
+
return true;
|
|
7850
|
+
}
|
|
7851
|
+
const attrs = event.attributes ?? {};
|
|
7852
|
+
if (attrs.noSideEffect === true) return true;
|
|
7853
|
+
if (attrs.sideEffect === false) return true;
|
|
7854
|
+
return false;
|
|
7855
|
+
}
|
|
7856
|
+
function eventEvidence2(event) {
|
|
7857
|
+
return {
|
|
7858
|
+
runId: event.runId,
|
|
7859
|
+
eventId: event.eventId,
|
|
7860
|
+
kind: event.kind,
|
|
7861
|
+
name: event.name,
|
|
7862
|
+
status: event.status
|
|
7863
|
+
};
|
|
7864
|
+
}
|
|
7865
|
+
function eventTime(event) {
|
|
7866
|
+
return event.startedAt ?? event.timestamp ?? "";
|
|
7867
|
+
}
|
|
7868
|
+
function sortByTime(events) {
|
|
7869
|
+
return [...events].sort((a, b) => eventTime(a).localeCompare(eventTime(b)));
|
|
7870
|
+
}
|
|
7871
|
+
function attemptNumberOf(event) {
|
|
7872
|
+
const workflow = workflowFor(event);
|
|
7873
|
+
const value = workflow.attemptNumber ?? workflow.attempt;
|
|
7874
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
7875
|
+
}
|
|
7876
|
+
function attemptIdOf(event) {
|
|
7877
|
+
const value = workflowFor(event).attemptId;
|
|
7878
|
+
return typeof value === "string" && value.trim() !== "" ? value : void 0;
|
|
7879
|
+
}
|
|
7880
|
+
function countOperationAttempts(members) {
|
|
7881
|
+
const attemptIds = new Set(
|
|
7882
|
+
members.map((event) => attemptIdOf(event)).filter((value) => value !== void 0)
|
|
7883
|
+
);
|
|
7884
|
+
if (attemptIds.size > 0) {
|
|
7885
|
+
return attemptIds.size;
|
|
7886
|
+
}
|
|
7887
|
+
const numbers = members.map((event) => attemptNumberOf(event)).filter((value) => value !== void 0).sort((a, b) => a - b);
|
|
7888
|
+
if (numbers.length > 0) {
|
|
7889
|
+
const unique = [...new Set(numbers)];
|
|
7890
|
+
const min = unique[0];
|
|
7891
|
+
const max = unique[unique.length - 1];
|
|
7892
|
+
const contiguous = unique.length === max - min + 1 && unique.every((value, index) => value === min + index);
|
|
7893
|
+
if (contiguous) {
|
|
7894
|
+
return unique.length;
|
|
7895
|
+
}
|
|
7896
|
+
return unique.length;
|
|
7897
|
+
}
|
|
7898
|
+
return members.filter((event) => event.kind === "TOOL" || event.kind === "LLM").length;
|
|
7899
|
+
}
|
|
7900
|
+
function classifyRetryAttempt(event, priorOrdered, allMembers) {
|
|
7901
|
+
const workflow = workflowFor(event);
|
|
7902
|
+
const attemptNumber2 = attemptNumberOf(event);
|
|
7903
|
+
const attemptId = attemptIdOf(event);
|
|
7904
|
+
const retryOf = workflow.retryOf;
|
|
7905
|
+
if (typeof retryOf === "string" && retryOf.trim() !== "") {
|
|
7906
|
+
const target = allMembers.find(
|
|
7907
|
+
(candidate) => candidate.eventId === retryOf || attemptIdOf(candidate) === retryOf || workflowFor(candidate).operationId === retryOf
|
|
7908
|
+
);
|
|
7909
|
+
if (!target) {
|
|
7910
|
+
return { kind: "unknown", reason: "retryOf target missing" };
|
|
7911
|
+
}
|
|
7912
|
+
if (eventTime(target) > eventTime(event)) {
|
|
7913
|
+
return { kind: "unknown", reason: "retryOf target does not precede retry" };
|
|
7914
|
+
}
|
|
7915
|
+
return { kind: "retry" };
|
|
7916
|
+
}
|
|
7917
|
+
if (attemptNumber2 !== void 0) {
|
|
7918
|
+
if (attemptNumber2 > 1) {
|
|
7919
|
+
return { kind: "retry" };
|
|
7920
|
+
}
|
|
7921
|
+
if (attemptNumber2 === 1) {
|
|
7922
|
+
return { kind: "first" };
|
|
7923
|
+
}
|
|
7924
|
+
}
|
|
7925
|
+
if (attemptId !== void 0) {
|
|
7926
|
+
const priorDistinct = priorOrdered.some((candidate) => {
|
|
7927
|
+
const priorId = attemptIdOf(candidate);
|
|
7928
|
+
return priorId !== void 0 && priorId !== attemptId;
|
|
7929
|
+
});
|
|
7930
|
+
if (priorDistinct) {
|
|
7931
|
+
return { kind: "retry" };
|
|
7932
|
+
}
|
|
7933
|
+
if (priorOrdered.length === 0) {
|
|
7934
|
+
return { kind: "first" };
|
|
7935
|
+
}
|
|
7936
|
+
}
|
|
7937
|
+
const priorFinished = priorOrdered.filter(
|
|
7938
|
+
(candidate) => candidate.status === "ok" || candidate.status === "error"
|
|
7939
|
+
);
|
|
7940
|
+
if (priorFinished.length > 0) {
|
|
7941
|
+
return { kind: "retry" };
|
|
7942
|
+
}
|
|
7943
|
+
if (priorOrdered.length === 0) {
|
|
7944
|
+
return { kind: "first" };
|
|
7945
|
+
}
|
|
7946
|
+
return { kind: "unknown", reason: "ambiguous attempt identity" };
|
|
7947
|
+
}
|
|
7948
|
+
function hasDuplicateAttemptIds(members) {
|
|
7949
|
+
const seen = /* @__PURE__ */ new Set();
|
|
7950
|
+
for (const event of members) {
|
|
7951
|
+
const id = attemptIdOf(event);
|
|
7952
|
+
if (!id) continue;
|
|
7953
|
+
if (seen.has(id)) return true;
|
|
7954
|
+
seen.add(id);
|
|
7955
|
+
}
|
|
7956
|
+
return false;
|
|
7957
|
+
}
|
|
7958
|
+
function evaluateRetrySafetyRules(events, rules, _runEvidence) {
|
|
7959
|
+
const findings = [];
|
|
7960
|
+
const byOperation = /* @__PURE__ */ new Map();
|
|
7961
|
+
for (const event of events) {
|
|
7962
|
+
const workflow = workflowFor(event);
|
|
7963
|
+
const operationId = workflow.operationId;
|
|
7964
|
+
if (!operationId) continue;
|
|
7965
|
+
const list2 = byOperation.get(operationId) ?? [];
|
|
7966
|
+
list2.push(event);
|
|
7967
|
+
byOperation.set(operationId, list2);
|
|
7968
|
+
}
|
|
7969
|
+
if (rules.maxAttempts !== void 0) {
|
|
7970
|
+
for (const [operationId, members] of byOperation) {
|
|
7971
|
+
if (hasDuplicateAttemptIds(members)) {
|
|
7972
|
+
findings.push(
|
|
7973
|
+
fail2(
|
|
7974
|
+
"contract.retry.duplicate-attempt-id",
|
|
7975
|
+
`Operation ${operationId} has duplicate attemptId values.`,
|
|
7976
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
7977
|
+
"unique attemptId",
|
|
7978
|
+
"duplicate"
|
|
7979
|
+
)
|
|
7980
|
+
);
|
|
7981
|
+
}
|
|
7982
|
+
const count = countOperationAttempts(members);
|
|
7983
|
+
if (count > rules.maxAttempts) {
|
|
7984
|
+
findings.push(
|
|
7985
|
+
fail2(
|
|
7986
|
+
"contract.retry.max-attempts",
|
|
7987
|
+
`Operation ${operationId} exceeded maxAttempts ${rules.maxAttempts}.`,
|
|
7988
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
7989
|
+
rules.maxAttempts,
|
|
7990
|
+
count
|
|
7991
|
+
)
|
|
7992
|
+
);
|
|
7993
|
+
}
|
|
7994
|
+
}
|
|
7995
|
+
}
|
|
7996
|
+
if (rules.requireTerminalResult) {
|
|
7997
|
+
for (const [operationId, members] of byOperation) {
|
|
7998
|
+
const hasTerminal = members.some((event) => event.status === "ok" || event.status === "error");
|
|
7999
|
+
if (!hasTerminal) {
|
|
8000
|
+
findings.push(
|
|
8001
|
+
fail2(
|
|
8002
|
+
"contract.retry.terminal-result",
|
|
8003
|
+
`Operation ${operationId} has no terminal ok/error result.`,
|
|
8004
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
8005
|
+
"ok|error",
|
|
8006
|
+
members.map((event) => event.status)
|
|
8007
|
+
)
|
|
8008
|
+
);
|
|
8009
|
+
}
|
|
8010
|
+
}
|
|
8011
|
+
}
|
|
8012
|
+
if (rules.fallbackOnlyAfterFailure) {
|
|
8013
|
+
for (const event of events) {
|
|
8014
|
+
const workflow = workflowFor(event);
|
|
8015
|
+
const fallbackOf = workflow.fallbackOf;
|
|
8016
|
+
if (!fallbackOf) continue;
|
|
8017
|
+
const prior = byOperation.get(fallbackOf) ?? [];
|
|
8018
|
+
const fallbackStart = eventTime(event);
|
|
8019
|
+
const earlierFailure = prior.some(
|
|
8020
|
+
(candidate) => candidate.status === "error" && eventTime(candidate) !== "" && fallbackStart !== "" && eventTime(candidate) < fallbackStart
|
|
8021
|
+
);
|
|
8022
|
+
if (!earlierFailure) {
|
|
8023
|
+
findings.push(
|
|
8024
|
+
fail2(
|
|
8025
|
+
"contract.retry.fallback-after-failure",
|
|
8026
|
+
`Fallback for ${fallbackOf} appeared without an earlier failure in the related operation.`,
|
|
8027
|
+
[eventEvidence2(event)],
|
|
8028
|
+
"earlier error attempt",
|
|
8029
|
+
{ fallbackOf }
|
|
8030
|
+
)
|
|
8031
|
+
);
|
|
8032
|
+
}
|
|
8033
|
+
}
|
|
8034
|
+
}
|
|
8035
|
+
const nonIdempotent = new Set(rules.nonIdempotentTools ?? []);
|
|
8036
|
+
if (nonIdempotent.size > 0 || rules.requireIdempotencyEvidenceForRetry) {
|
|
8037
|
+
const toolEvents = events.filter(
|
|
8038
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
8039
|
+
);
|
|
8040
|
+
const byToolOp = /* @__PURE__ */ new Map();
|
|
8041
|
+
for (const event of toolEvents) {
|
|
8042
|
+
const name = resolveCanonicalToolName(event);
|
|
8043
|
+
const workflow = workflowFor(event);
|
|
8044
|
+
const key = `${workflow.operationId ?? name}::${name}`;
|
|
8045
|
+
const list2 = byToolOp.get(key) ?? [];
|
|
8046
|
+
list2.push(event);
|
|
8047
|
+
byToolOp.set(key, list2);
|
|
8048
|
+
}
|
|
8049
|
+
for (const [, members] of byToolOp) {
|
|
8050
|
+
const ordered = sortByTime(members);
|
|
8051
|
+
let sawOk = false;
|
|
8052
|
+
let sawSideEffectOk = false;
|
|
8053
|
+
const prior = [];
|
|
8054
|
+
for (const event of ordered) {
|
|
8055
|
+
const name = resolveCanonicalToolName(event);
|
|
8056
|
+
const classification = classifyRetryAttempt(event, prior, members);
|
|
8057
|
+
if (classification.kind === "unknown" && rules.requireIdempotencyEvidenceForRetry) {
|
|
8058
|
+
findings.push(
|
|
8059
|
+
fail2(
|
|
8060
|
+
"contract.retry.ambiguous-identity",
|
|
8061
|
+
`Ambiguous attempt identity for tool ${name}; strict retry evidence cannot pass.`,
|
|
8062
|
+
[eventEvidence2(event)],
|
|
8063
|
+
"explicit attempt identity",
|
|
8064
|
+
classification.reason
|
|
8065
|
+
)
|
|
8066
|
+
);
|
|
8067
|
+
}
|
|
8068
|
+
const isRetry = classification.kind === "retry";
|
|
8069
|
+
if (isRetry && rules.requireIdempotencyEvidenceForRetry && !hasIdempotencyEvidence(event)) {
|
|
8070
|
+
findings.push(
|
|
8071
|
+
fail2(
|
|
8072
|
+
"contract.retry.idempotency-evidence",
|
|
8073
|
+
`Retry of tool ${name} lacks idempotencyKey / noSideEffect evidence.`,
|
|
8074
|
+
[eventEvidence2(event)],
|
|
8075
|
+
"idempotencyKey|noSideEffect",
|
|
8076
|
+
{ code: "AI_CHECK_RETRY_EVIDENCE_UNAVAILABLE" }
|
|
8077
|
+
)
|
|
8078
|
+
);
|
|
8079
|
+
}
|
|
8080
|
+
if (sawOk && nonIdempotent.has(name) && sawSideEffectOk && !hasIdempotencyEvidence(event)) {
|
|
8081
|
+
findings.push(
|
|
8082
|
+
fail2(
|
|
8083
|
+
"contract.retry.non-idempotent-side-effect",
|
|
8084
|
+
`Retry of non-idempotent tool ${name} after a confirmed ok side effect.`,
|
|
8085
|
+
[eventEvidence2(event)],
|
|
8086
|
+
"no retry after side effect",
|
|
8087
|
+
name
|
|
8088
|
+
)
|
|
8089
|
+
);
|
|
8090
|
+
}
|
|
8091
|
+
if (event.status === "ok") {
|
|
8092
|
+
sawOk = true;
|
|
8093
|
+
if (nonIdempotent.has(name) && !hasIdempotencyEvidence(event)) {
|
|
8094
|
+
sawSideEffectOk = true;
|
|
8095
|
+
}
|
|
8096
|
+
}
|
|
8097
|
+
prior.push(event);
|
|
8098
|
+
}
|
|
8099
|
+
}
|
|
8100
|
+
}
|
|
8101
|
+
if (rules.requireRecoveredFailureVisible) {
|
|
8102
|
+
for (const [operationId, members] of byOperation) {
|
|
8103
|
+
const ordered = sortByTime(members);
|
|
8104
|
+
const toolish = ordered.filter(
|
|
8105
|
+
(event) => (event.kind === "TOOL" || event.kind === "LLM") && (event.status === "ok" || event.status === "error")
|
|
8106
|
+
);
|
|
8107
|
+
if (toolish.length < 2) continue;
|
|
8108
|
+
let sawEarlierError = false;
|
|
8109
|
+
let sawLaterOkAfterError = false;
|
|
8110
|
+
for (const event of toolish) {
|
|
8111
|
+
if (event.status === "error") {
|
|
8112
|
+
sawEarlierError = true;
|
|
8113
|
+
} else if (event.status === "ok" && sawEarlierError) {
|
|
8114
|
+
sawLaterOkAfterError = true;
|
|
8115
|
+
}
|
|
8116
|
+
}
|
|
8117
|
+
const hasOk = toolish.some((event) => event.status === "ok");
|
|
8118
|
+
const attemptish = toolish.some((event) => {
|
|
8119
|
+
const meta2 = workflowFor(event);
|
|
8120
|
+
return meta2.attemptId !== void 0 || meta2.attemptNumber !== void 0 || meta2.attempt !== void 0 || meta2.retryOf !== void 0;
|
|
8121
|
+
}) || toolish.length > 1;
|
|
8122
|
+
if (!attemptish || !hasOk) continue;
|
|
8123
|
+
if (!sawLaterOkAfterError) {
|
|
8124
|
+
findings.push(
|
|
8125
|
+
fail2(
|
|
8126
|
+
"contract.retry.recovered-failure-visible",
|
|
8127
|
+
`Operation ${operationId} does not show an earlier error attempt before a later success in the same chain.`,
|
|
8128
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
8129
|
+
"earlier error then later ok",
|
|
8130
|
+
{
|
|
8131
|
+
sawEarlierError,
|
|
8132
|
+
statuses: toolish.map((event) => event.status)
|
|
8133
|
+
}
|
|
8134
|
+
)
|
|
8135
|
+
);
|
|
8136
|
+
}
|
|
8137
|
+
}
|
|
8138
|
+
}
|
|
8139
|
+
return findings;
|
|
8140
|
+
}
|
|
8141
|
+
var init_retry_safety = __esm({
|
|
8142
|
+
"packages/core/src/checks/retry-safety.ts"() {
|
|
8143
|
+
init_logical_events();
|
|
8144
|
+
init_metadata();
|
|
8145
|
+
}
|
|
8146
|
+
});
|
|
8147
|
+
|
|
8148
|
+
// packages/core/src/checks/recovery-operations.ts
|
|
8149
|
+
function fail3(ruleId, message, evidence, expected, actual) {
|
|
8150
|
+
return {
|
|
8151
|
+
ruleId,
|
|
8152
|
+
severity: "error",
|
|
8153
|
+
status: "fail",
|
|
8154
|
+
message,
|
|
8155
|
+
...expected !== void 0 ? { expected } : {},
|
|
8156
|
+
...actual !== void 0 ? { actual } : {},
|
|
8157
|
+
evidence: [...evidence]
|
|
8158
|
+
};
|
|
8159
|
+
}
|
|
8160
|
+
function workflowFor2(event) {
|
|
8161
|
+
const attrs = event.attributes;
|
|
8162
|
+
if (!attrs || typeof attrs !== "object") return {};
|
|
8163
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
8164
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
8165
|
+
return { ...nested, ...direct };
|
|
8166
|
+
}
|
|
8167
|
+
function eventEvidence3(event) {
|
|
8168
|
+
return {
|
|
8169
|
+
runId: event.runId,
|
|
8170
|
+
eventId: event.eventId,
|
|
8171
|
+
kind: event.kind,
|
|
8172
|
+
name: event.name,
|
|
8173
|
+
status: event.status
|
|
8174
|
+
};
|
|
8175
|
+
}
|
|
8176
|
+
function eventTime2(event) {
|
|
8177
|
+
return event.startedAt ?? event.timestamp ?? "";
|
|
8178
|
+
}
|
|
8179
|
+
function sortByTime2(events) {
|
|
8180
|
+
return [...events].sort((a, b) => eventTime2(a).localeCompare(eventTime2(b)));
|
|
8181
|
+
}
|
|
8182
|
+
function errorCodeOf(event) {
|
|
8183
|
+
if (typeof event.error?.code === "string" && event.error.code.trim() !== "") {
|
|
8184
|
+
return event.error.code.trim();
|
|
8185
|
+
}
|
|
8186
|
+
const attrs = event.attributes ?? {};
|
|
8187
|
+
for (const key of ["errorCode", "code"]) {
|
|
8188
|
+
const value = attrs[key];
|
|
8189
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
8190
|
+
}
|
|
8191
|
+
const nested = attrs.error;
|
|
8192
|
+
if (nested && typeof nested === "object") {
|
|
8193
|
+
const code = nested.code;
|
|
8194
|
+
if (typeof code === "string" && code.trim() !== "") return code.trim();
|
|
8195
|
+
}
|
|
8196
|
+
return void 0;
|
|
8197
|
+
}
|
|
8198
|
+
function argumentDigestOf(event) {
|
|
8199
|
+
const attrs = event.attributes ?? {};
|
|
8200
|
+
for (const key of ["argumentsDigest", "inputDigest", "toolArgumentsDigest"]) {
|
|
8201
|
+
const value = attrs[key];
|
|
8202
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
8203
|
+
}
|
|
8204
|
+
const commitment = attrs.omittedPayloadCommitment;
|
|
8205
|
+
if (commitment && typeof commitment === "object") {
|
|
8206
|
+
const digest = commitment.digest;
|
|
8207
|
+
if (typeof digest === "string" && digest.trim() !== "") return digest.trim();
|
|
8208
|
+
}
|
|
8209
|
+
return void 0;
|
|
8210
|
+
}
|
|
8211
|
+
function argumentFingerprint(event) {
|
|
8212
|
+
const payload = extractToolArgumentPayload(event);
|
|
8213
|
+
if (payload.present) {
|
|
8214
|
+
const encoded = encodeCanonicalStructured(payload.value);
|
|
8215
|
+
if (!encoded.ok) return { kind: "missing" };
|
|
8216
|
+
return { kind: "structured", value: encoded.value };
|
|
8217
|
+
}
|
|
8218
|
+
const digest = argumentDigestOf(event);
|
|
8219
|
+
if (digest) return { kind: "digest", value: digest };
|
|
8220
|
+
return { kind: "missing" };
|
|
8221
|
+
}
|
|
8222
|
+
function sameArguments(left, right) {
|
|
8223
|
+
const a = argumentFingerprint(left);
|
|
8224
|
+
const b = argumentFingerprint(right);
|
|
8225
|
+
if (a.kind === "missing" || b.kind === "missing") {
|
|
8226
|
+
return {
|
|
8227
|
+
ok: false,
|
|
8228
|
+
reason: "Structured or digest argument evidence unavailable for same-arguments check.",
|
|
8229
|
+
code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_UNAVAILABLE"
|
|
8230
|
+
};
|
|
8231
|
+
}
|
|
8232
|
+
if (a.kind !== b.kind) {
|
|
8233
|
+
return {
|
|
8234
|
+
ok: false,
|
|
8235
|
+
reason: "Argument evidence kinds differ across attempts (structured vs digest).",
|
|
8236
|
+
code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_MISMATCH"
|
|
8237
|
+
};
|
|
8238
|
+
}
|
|
8239
|
+
if (a.value !== b.value) {
|
|
8240
|
+
return {
|
|
8241
|
+
ok: false,
|
|
8242
|
+
reason: "Retry arguments do not match prior attempt.",
|
|
8243
|
+
code: "AI_CHECK_RECOVERY_ARGUMENTS_DIFFER"
|
|
8244
|
+
};
|
|
8245
|
+
}
|
|
8246
|
+
return { ok: true };
|
|
8247
|
+
}
|
|
8248
|
+
function isUnevaluableWriteCompletion(event) {
|
|
8249
|
+
if (event.status === "running" || event.status === "unknown") return true;
|
|
8250
|
+
const attrs = event.attributes ?? {};
|
|
8251
|
+
if (attrs.timeout === true) return true;
|
|
8252
|
+
if (attrs.completionState === "unknown" || attrs.completionState === "timeout") return true;
|
|
8253
|
+
const code = errorCodeOf(event)?.toUpperCase();
|
|
8254
|
+
if (code === "TIMEOUT" || code === "UNKNOWN_COMPLETION" || code === "DEADLINE_EXCEEDED") {
|
|
8255
|
+
return true;
|
|
8256
|
+
}
|
|
8257
|
+
return false;
|
|
8258
|
+
}
|
|
8259
|
+
function collectExplicitReferences(event) {
|
|
8260
|
+
const refs = /* @__PURE__ */ new Set();
|
|
8261
|
+
const attrs = event.attributes ?? {};
|
|
8262
|
+
const candidates = [
|
|
8263
|
+
attrs.referencedEventId,
|
|
8264
|
+
attrs.referencedEventIds,
|
|
8265
|
+
attrs.toolResultEventId,
|
|
8266
|
+
attrs.toolResultEventIds,
|
|
8267
|
+
attrs.inputEventIds,
|
|
8268
|
+
attrs.dependsOnEventIds,
|
|
8269
|
+
attrs.evidence,
|
|
8270
|
+
workflowFor2(event).toolCallId
|
|
8271
|
+
];
|
|
8272
|
+
for (const candidate of candidates) {
|
|
8273
|
+
if (typeof candidate === "string" && candidate.trim() !== "") {
|
|
8274
|
+
refs.add(candidate.trim());
|
|
8275
|
+
continue;
|
|
8276
|
+
}
|
|
8277
|
+
if (Array.isArray(candidate)) {
|
|
8278
|
+
for (const item of candidate) {
|
|
8279
|
+
if (typeof item === "string" && item.trim() !== "") refs.add(item.trim());
|
|
8280
|
+
if (item && typeof item === "object") {
|
|
8281
|
+
const id = item.eventId;
|
|
8282
|
+
if (typeof id === "string" && id.trim() !== "") refs.add(id.trim());
|
|
8283
|
+
}
|
|
8284
|
+
}
|
|
8285
|
+
continue;
|
|
8286
|
+
}
|
|
8287
|
+
if (candidate && typeof candidate === "object") {
|
|
8288
|
+
const obj = candidate;
|
|
8289
|
+
if (typeof obj.eventId === "string" && obj.eventId.trim() !== "") {
|
|
8290
|
+
refs.add(obj.eventId.trim());
|
|
8291
|
+
}
|
|
8292
|
+
if (Array.isArray(obj.eventIds)) {
|
|
8293
|
+
for (const id of obj.eventIds) {
|
|
8294
|
+
if (typeof id === "string" && id.trim() !== "") refs.add(id.trim());
|
|
8295
|
+
}
|
|
8296
|
+
}
|
|
8297
|
+
}
|
|
8298
|
+
}
|
|
8299
|
+
return refs;
|
|
8300
|
+
}
|
|
8301
|
+
function llmReferencesTool(llm, toolEvent, requireExplicit) {
|
|
8302
|
+
if (!requireExplicit) {
|
|
8303
|
+
return eventTime2(llm) >= eventTime2(toolEvent);
|
|
8304
|
+
}
|
|
8305
|
+
const refs = collectExplicitReferences(llm);
|
|
8306
|
+
if (refs.has(toolEvent.eventId)) return true;
|
|
8307
|
+
const toolCallId = workflowFor2(toolEvent).toolCallId;
|
|
8308
|
+
if (toolCallId && refs.has(toolCallId)) return true;
|
|
8309
|
+
return false;
|
|
8310
|
+
}
|
|
8311
|
+
function attemptIdOf2(event) {
|
|
8312
|
+
const value = workflowFor2(event).attemptId;
|
|
8313
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
8314
|
+
}
|
|
8315
|
+
function hasDuplicateAttemptIds2(members) {
|
|
8316
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8317
|
+
for (const event of members) {
|
|
8318
|
+
const id = attemptIdOf2(event);
|
|
8319
|
+
if (!id) continue;
|
|
8320
|
+
if (seen.has(id)) return true;
|
|
8321
|
+
seen.add(id);
|
|
8322
|
+
}
|
|
8323
|
+
return false;
|
|
8324
|
+
}
|
|
8325
|
+
function findRetryTarget(members, retryOf) {
|
|
8326
|
+
return members.find(
|
|
8327
|
+
(candidate) => candidate.eventId === retryOf || attemptIdOf2(candidate) === retryOf || workflowFor2(candidate).operationId === retryOf
|
|
8328
|
+
);
|
|
8329
|
+
}
|
|
8330
|
+
function groupToolAttempts(events, toolName2) {
|
|
8331
|
+
const members = events.filter(
|
|
8332
|
+
(event) => event.kind === "TOOL" && resolveCanonicalToolName(event) === toolName2
|
|
8333
|
+
);
|
|
8334
|
+
const parent = /* @__PURE__ */ new Map();
|
|
8335
|
+
const ensure = (id) => {
|
|
8336
|
+
if (!parent.has(id)) parent.set(id, id);
|
|
8337
|
+
return id;
|
|
8338
|
+
};
|
|
8339
|
+
const find = (id) => {
|
|
8340
|
+
ensure(id);
|
|
8341
|
+
let cur = id;
|
|
8342
|
+
while (parent.get(cur) !== cur) {
|
|
8343
|
+
cur = parent.get(cur);
|
|
8344
|
+
}
|
|
8345
|
+
let walk = id;
|
|
8346
|
+
while (parent.get(walk) !== cur) {
|
|
8347
|
+
const next = parent.get(walk);
|
|
8348
|
+
parent.set(walk, cur);
|
|
8349
|
+
walk = next;
|
|
8350
|
+
}
|
|
8351
|
+
return cur;
|
|
8352
|
+
};
|
|
8353
|
+
const union = (a, b) => {
|
|
8354
|
+
const ra = find(a);
|
|
8355
|
+
const rb = find(b);
|
|
8356
|
+
if (ra !== rb) parent.set(ra, rb);
|
|
8357
|
+
};
|
|
8358
|
+
const nodeId = (event) => {
|
|
8359
|
+
const operationId = workflowFor2(event).operationId;
|
|
8360
|
+
if (typeof operationId === "string" && operationId.trim() !== "") {
|
|
8361
|
+
return `op:${operationId.trim()}`;
|
|
8362
|
+
}
|
|
8363
|
+
return `event:${event.eventId}`;
|
|
8364
|
+
};
|
|
8365
|
+
for (const event of members) {
|
|
8366
|
+
ensure(nodeId(event));
|
|
8367
|
+
}
|
|
8368
|
+
for (const event of members) {
|
|
8369
|
+
const retryOf = workflowFor2(event).retryOf;
|
|
8370
|
+
if (typeof retryOf !== "string" || retryOf.trim() === "") continue;
|
|
8371
|
+
const target = findRetryTarget(members, retryOf.trim());
|
|
8372
|
+
if (target) {
|
|
8373
|
+
union(nodeId(event), nodeId(target));
|
|
8374
|
+
}
|
|
8375
|
+
}
|
|
8376
|
+
const groups = /* @__PURE__ */ new Map();
|
|
8377
|
+
for (const event of members) {
|
|
8378
|
+
const key = find(nodeId(event));
|
|
8379
|
+
const list2 = groups.get(key) ?? [];
|
|
8380
|
+
list2.push(event);
|
|
8381
|
+
groups.set(key, list2);
|
|
8382
|
+
}
|
|
8383
|
+
return groups;
|
|
8384
|
+
}
|
|
8385
|
+
function latestAttempt(ordered) {
|
|
8386
|
+
if (ordered.length === 0) return void 0;
|
|
8387
|
+
return ordered[ordered.length - 1];
|
|
8388
|
+
}
|
|
8389
|
+
function evaluateRecoveryOperations(events, operations) {
|
|
8390
|
+
const findings = [];
|
|
8391
|
+
const llmEvents = events.filter((event) => event.kind === "LLM");
|
|
8392
|
+
for (const operation of operations) {
|
|
8393
|
+
const sideEffectClass = operation.sideEffectClass ?? "read";
|
|
8394
|
+
const groups = groupToolAttempts(events, operation.tool);
|
|
8395
|
+
if (groups.size === 0) {
|
|
8396
|
+
if (operation.requireTerminalSuccess) {
|
|
8397
|
+
findings.push(
|
|
8398
|
+
fail3(
|
|
8399
|
+
"contract.retry.operations.terminal-success",
|
|
8400
|
+
`Recovery operation for tool ${operation.tool} requires a terminal success but no attempts were observed.`,
|
|
8401
|
+
[],
|
|
8402
|
+
"ok",
|
|
8403
|
+
"missing"
|
|
8404
|
+
)
|
|
8405
|
+
);
|
|
8406
|
+
}
|
|
8407
|
+
continue;
|
|
8408
|
+
}
|
|
8409
|
+
for (const [operationKey, members] of groups) {
|
|
8410
|
+
const ordered = sortByTime2(members);
|
|
8411
|
+
if (hasDuplicateAttemptIds2(members)) {
|
|
8412
|
+
findings.push(
|
|
8413
|
+
fail3(
|
|
8414
|
+
"contract.retry.operations.duplicate-attempt-id",
|
|
8415
|
+
`Tool ${operation.tool} operation ${operationKey} has duplicate attemptId values.`,
|
|
8416
|
+
members.slice(0, 4).map(eventEvidence3),
|
|
8417
|
+
"unique attemptId",
|
|
8418
|
+
"duplicate"
|
|
8419
|
+
)
|
|
8420
|
+
);
|
|
8421
|
+
}
|
|
8422
|
+
if (operation.maxAttempts !== void 0) {
|
|
8423
|
+
const count = countOperationAttempts(members);
|
|
8424
|
+
if (count > operation.maxAttempts) {
|
|
8425
|
+
findings.push(
|
|
8426
|
+
fail3(
|
|
8427
|
+
"contract.retry.operations.max-attempts",
|
|
8428
|
+
`Tool ${operation.tool} operation ${operationKey} exceeded maxAttempts ${operation.maxAttempts}.`,
|
|
8429
|
+
members.slice(0, 4).map(eventEvidence3),
|
|
8430
|
+
operation.maxAttempts,
|
|
8431
|
+
count
|
|
8432
|
+
)
|
|
8433
|
+
);
|
|
8434
|
+
}
|
|
8435
|
+
}
|
|
8436
|
+
if (sideEffectClass === "write") {
|
|
8437
|
+
for (const event of ordered) {
|
|
8438
|
+
if (isUnevaluableWriteCompletion(event)) {
|
|
8439
|
+
findings.push(
|
|
8440
|
+
fail3(
|
|
8441
|
+
"contract.retry.operations.write-completion-unevaluable",
|
|
8442
|
+
`Write tool ${operation.tool} has timeout/unknown/running completion that cannot be verified as an authoritative write result.`,
|
|
8443
|
+
[eventEvidence3(event)],
|
|
8444
|
+
"authoritative write completion observation",
|
|
8445
|
+
{
|
|
8446
|
+
code: "AI_CHECK_RECOVERY_WRITE_COMPLETION_UNAVAILABLE",
|
|
8447
|
+
status: event.status
|
|
8448
|
+
}
|
|
8449
|
+
)
|
|
8450
|
+
);
|
|
8451
|
+
}
|
|
8452
|
+
}
|
|
8453
|
+
}
|
|
8454
|
+
const prior = [];
|
|
8455
|
+
let sawEarlierError = false;
|
|
8456
|
+
let sawLaterOkAfterError = false;
|
|
8457
|
+
let lastOk;
|
|
8458
|
+
let identityUnavailable = false;
|
|
8459
|
+
for (const event of ordered) {
|
|
8460
|
+
const classification = classifyRetryAttempt(event, prior, members);
|
|
8461
|
+
if (classification.kind === "unknown") {
|
|
8462
|
+
identityUnavailable = true;
|
|
8463
|
+
findings.push(
|
|
8464
|
+
fail3(
|
|
8465
|
+
"contract.retry.operations.attempt-identity",
|
|
8466
|
+
`Tool ${operation.tool} operation ${operationKey} has ambiguous attempt identity (${classification.reason}).`,
|
|
8467
|
+
[eventEvidence3(event)],
|
|
8468
|
+
"explicit operationId/attemptId/retryOf chronology",
|
|
8469
|
+
{ code: "AI_CHECK_RECOVERY_ATTEMPT_IDENTITY_UNAVAILABLE", reason: classification.reason }
|
|
8470
|
+
)
|
|
8471
|
+
);
|
|
8472
|
+
}
|
|
8473
|
+
const isRetry = classification.kind === "retry";
|
|
8474
|
+
if (isRetry && operation.requireFailureBeforeRetry) {
|
|
8475
|
+
const earlierFailure = prior.some(
|
|
8476
|
+
(candidate) => candidate.status === "error" && eventTime2(candidate) !== "" && eventTime2(event) !== "" && eventTime2(candidate) < eventTime2(event)
|
|
8477
|
+
);
|
|
8478
|
+
if (!earlierFailure) {
|
|
8479
|
+
findings.push(
|
|
8480
|
+
fail3(
|
|
8481
|
+
"contract.retry.operations.failure-before-retry",
|
|
8482
|
+
`Retry of tool ${operation.tool} appeared without an earlier failure in the same operation.`,
|
|
8483
|
+
[eventEvidence3(event)],
|
|
8484
|
+
"earlier error attempt",
|
|
8485
|
+
{ operationKey }
|
|
8486
|
+
)
|
|
8487
|
+
);
|
|
8488
|
+
}
|
|
8489
|
+
}
|
|
8490
|
+
if (isRetry && operation.retryableErrors?.codes && operation.retryableErrors.codes.length > 0) {
|
|
8491
|
+
const allowed = new Set(operation.retryableErrors.codes);
|
|
8492
|
+
const priorErrors = prior.filter((candidate) => candidate.status === "error");
|
|
8493
|
+
const lastError = priorErrors[priorErrors.length - 1];
|
|
8494
|
+
if (lastError) {
|
|
8495
|
+
const code = errorCodeOf(lastError);
|
|
8496
|
+
if (!code || !allowed.has(code)) {
|
|
8497
|
+
findings.push(
|
|
8498
|
+
fail3(
|
|
8499
|
+
"contract.retry.operations.retryable-error",
|
|
8500
|
+
`Retry of tool ${operation.tool} followed a non-retryable or missing error code.`,
|
|
8501
|
+
[eventEvidence3(event), eventEvidence3(lastError)],
|
|
8502
|
+
[...allowed],
|
|
8503
|
+
code ?? null
|
|
8504
|
+
)
|
|
8505
|
+
);
|
|
8506
|
+
}
|
|
8507
|
+
} else {
|
|
8508
|
+
findings.push(
|
|
8509
|
+
fail3(
|
|
8510
|
+
"contract.retry.operations.retryable-error",
|
|
8511
|
+
`Retry of tool ${operation.tool} has no prior error code evidence.`,
|
|
8512
|
+
[eventEvidence3(event)],
|
|
8513
|
+
[...allowed],
|
|
8514
|
+
null
|
|
8515
|
+
)
|
|
8516
|
+
);
|
|
8517
|
+
}
|
|
8518
|
+
}
|
|
8519
|
+
if (isRetry && (operation.requireSameArguments === true || operation.requireSameArguments === "structured-or-digest")) {
|
|
8520
|
+
const baseline = prior[0] ?? prior[prior.length - 1];
|
|
8521
|
+
if (baseline) {
|
|
8522
|
+
const comparison = sameArguments(baseline, event);
|
|
8523
|
+
if (!comparison.ok) {
|
|
8524
|
+
findings.push(
|
|
8525
|
+
fail3(
|
|
8526
|
+
"contract.retry.operations.same-arguments",
|
|
8527
|
+
comparison.reason,
|
|
8528
|
+
[eventEvidence3(baseline), eventEvidence3(event)],
|
|
8529
|
+
"matching structured args or digests",
|
|
8530
|
+
{ code: comparison.code }
|
|
8531
|
+
)
|
|
8532
|
+
);
|
|
8533
|
+
}
|
|
8534
|
+
} else {
|
|
8535
|
+
findings.push(
|
|
8536
|
+
fail3(
|
|
8537
|
+
"contract.retry.operations.same-arguments",
|
|
8538
|
+
"Structured or digest argument evidence unavailable for same-arguments check.",
|
|
8539
|
+
[eventEvidence3(event)],
|
|
8540
|
+
"matching structured args or digests",
|
|
8541
|
+
{ code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_UNAVAILABLE" }
|
|
8542
|
+
)
|
|
8543
|
+
);
|
|
8544
|
+
}
|
|
8545
|
+
}
|
|
8546
|
+
if (event.status === "error") {
|
|
8547
|
+
sawEarlierError = true;
|
|
8548
|
+
} else if (event.status === "ok" && sawEarlierError) {
|
|
8549
|
+
sawLaterOkAfterError = true;
|
|
8550
|
+
}
|
|
8551
|
+
if (event.status === "ok") {
|
|
8552
|
+
lastOk = event;
|
|
8553
|
+
}
|
|
8554
|
+
prior.push(event);
|
|
8555
|
+
}
|
|
8556
|
+
const latest = latestAttempt(ordered);
|
|
8557
|
+
if (operation.requireTerminalSuccess) {
|
|
8558
|
+
if (!latest || latest.status !== "ok") {
|
|
8559
|
+
findings.push(
|
|
8560
|
+
fail3(
|
|
8561
|
+
"contract.retry.operations.terminal-success",
|
|
8562
|
+
`Tool ${operation.tool} operation ${operationKey} does not terminate with ok on the latest attempt.`,
|
|
8563
|
+
ordered.slice(0, 4).map(eventEvidence3),
|
|
8564
|
+
"latest attempt ok",
|
|
8565
|
+
latest?.status ?? "missing"
|
|
8566
|
+
)
|
|
8567
|
+
);
|
|
8568
|
+
}
|
|
8569
|
+
}
|
|
8570
|
+
if (operation.requireRecoveredFailureVisible) {
|
|
8571
|
+
const latestOk = latest?.status === "ok";
|
|
8572
|
+
if (ordered.length > 1 && latestOk && !sawLaterOkAfterError) {
|
|
8573
|
+
findings.push(
|
|
8574
|
+
fail3(
|
|
8575
|
+
"contract.retry.operations.recovered-failure-visible",
|
|
8576
|
+
`Tool ${operation.tool} operation ${operationKey} does not show an earlier error before later success.`,
|
|
8577
|
+
ordered.slice(0, 4).map(eventEvidence3),
|
|
8578
|
+
"earlier error then later ok",
|
|
8579
|
+
{ sawEarlierError }
|
|
8580
|
+
)
|
|
8581
|
+
);
|
|
8582
|
+
}
|
|
8583
|
+
}
|
|
8584
|
+
const dependency = operation.successfulResultDependency;
|
|
8585
|
+
if (dependency?.consumerKind === "LLM") {
|
|
8586
|
+
const successForDependency = latest?.status === "ok" ? latest : identityUnavailable ? void 0 : lastOk;
|
|
8587
|
+
if (successForDependency) {
|
|
8588
|
+
if (sideEffectClass === "write" && isUnevaluableWriteCompletion(successForDependency)) {
|
|
8589
|
+
findings.push(
|
|
8590
|
+
fail3(
|
|
8591
|
+
"contract.retry.operations.successful-result-dependency",
|
|
8592
|
+
`Successful ${operation.tool} result dependency is unevaluable because write completion evidence is unavailable.`,
|
|
8593
|
+
[eventEvidence3(successForDependency)],
|
|
8594
|
+
"LLM reference to successful tool event",
|
|
8595
|
+
{ code: "AI_CHECK_RECOVERY_WRITE_COMPLETION_UNAVAILABLE" }
|
|
8596
|
+
)
|
|
8597
|
+
);
|
|
8598
|
+
} else {
|
|
8599
|
+
const requireExplicit = dependency.requireExplicitReference === true;
|
|
8600
|
+
const referenced = llmEvents.some(
|
|
8601
|
+
(llmEvent) => llmReferencesTool(llmEvent, successForDependency, requireExplicit)
|
|
8602
|
+
);
|
|
8603
|
+
if (!referenced) {
|
|
8604
|
+
findings.push(
|
|
8605
|
+
fail3(
|
|
8606
|
+
"contract.retry.operations.successful-result-dependency",
|
|
8607
|
+
requireExplicit ? `Successful ${operation.tool} result is not explicitly referenced by a later LLM event.` : `Successful ${operation.tool} result has no later LLM consumer.`,
|
|
8608
|
+
[eventEvidence3(successForDependency)],
|
|
8609
|
+
"LLM reference to successful tool event",
|
|
8610
|
+
{ code: "AI_CHECK_RECOVERY_RESULT_DEPENDENCY_MISSING" }
|
|
8611
|
+
)
|
|
8612
|
+
);
|
|
8613
|
+
}
|
|
8614
|
+
}
|
|
8615
|
+
}
|
|
8616
|
+
}
|
|
8617
|
+
}
|
|
8618
|
+
}
|
|
8619
|
+
return findings;
|
|
8620
|
+
}
|
|
8621
|
+
var init_recovery_operations = __esm({
|
|
8622
|
+
"packages/core/src/checks/recovery-operations.ts"() {
|
|
8623
|
+
init_canonical_equality();
|
|
8624
|
+
init_logical_events();
|
|
8625
|
+
init_tool_arguments();
|
|
7046
8626
|
init_retry_safety();
|
|
7047
8627
|
init_metadata();
|
|
7048
8628
|
}
|
|
7049
8629
|
});
|
|
7050
8630
|
|
|
7051
8631
|
// packages/core/src/checks/contract.ts
|
|
8632
|
+
function contractFailFinding(ruleId, message, evidence, expected, actual) {
|
|
8633
|
+
return {
|
|
8634
|
+
ruleId,
|
|
8635
|
+
severity: "error",
|
|
8636
|
+
status: "fail",
|
|
8637
|
+
message,
|
|
8638
|
+
...expected !== void 0 ? { expected } : {},
|
|
8639
|
+
...actual !== void 0 ? { actual } : {},
|
|
8640
|
+
evidence: [...evidence]
|
|
8641
|
+
};
|
|
8642
|
+
}
|
|
8643
|
+
function normalizeStatus(status) {
|
|
8644
|
+
if (status === "ok" || status === "error" || status === "running") return status;
|
|
8645
|
+
if (status === "success") return "ok";
|
|
8646
|
+
if (status === "failed") return "error";
|
|
8647
|
+
return "error";
|
|
8648
|
+
}
|
|
7052
8649
|
function cloneBody(body) {
|
|
7053
8650
|
return {
|
|
7054
8651
|
...body.run ? { run: { ...body.run } } : {},
|
|
@@ -7101,6 +8698,441 @@ function cloneBody(body) {
|
|
|
7101
8698
|
function bodyHasRules(body) {
|
|
7102
8699
|
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
7103
8700
|
}
|
|
8701
|
+
function parseEvidenceEventIds(evidence) {
|
|
8702
|
+
if (evidence === void 0 || evidence === null) {
|
|
8703
|
+
return { status: "missing" };
|
|
8704
|
+
}
|
|
8705
|
+
if (typeof evidence === "string") {
|
|
8706
|
+
const trimmed = evidence.trim();
|
|
8707
|
+
if (trimmed === "") {
|
|
8708
|
+
return { status: "invalid", reason: "empty evidence string" };
|
|
8709
|
+
}
|
|
8710
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
8711
|
+
}
|
|
8712
|
+
if (typeof evidence !== "object" || Array.isArray(evidence)) {
|
|
8713
|
+
return {
|
|
8714
|
+
status: "invalid",
|
|
8715
|
+
reason: "evidence must be a string event id or { eventId } / { eventIds }"
|
|
8716
|
+
};
|
|
8717
|
+
}
|
|
8718
|
+
const record = evidence;
|
|
8719
|
+
if (typeof record.eventId === "string") {
|
|
8720
|
+
const trimmed = record.eventId.trim();
|
|
8721
|
+
if (trimmed === "") {
|
|
8722
|
+
return { status: "invalid", reason: "empty evidence.eventId" };
|
|
8723
|
+
}
|
|
8724
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
8725
|
+
}
|
|
8726
|
+
if (Array.isArray(record.eventIds)) {
|
|
8727
|
+
if (record.eventIds.length === 0) {
|
|
8728
|
+
return { status: "invalid", reason: "empty evidence.eventIds" };
|
|
8729
|
+
}
|
|
8730
|
+
if (record.eventIds.length > MAX_EVIDENCE_EVENT_IDS) {
|
|
8731
|
+
return {
|
|
8732
|
+
status: "invalid",
|
|
8733
|
+
reason: `evidence.eventIds exceeds bound of ${MAX_EVIDENCE_EVENT_IDS}`
|
|
8734
|
+
};
|
|
8735
|
+
}
|
|
8736
|
+
const eventIds = [];
|
|
8737
|
+
for (const item of record.eventIds) {
|
|
8738
|
+
if (typeof item !== "string" || item.trim() === "") {
|
|
8739
|
+
return { status: "invalid", reason: "evidence.eventIds contains a non-string id" };
|
|
8740
|
+
}
|
|
8741
|
+
eventIds.push(item.trim());
|
|
8742
|
+
}
|
|
8743
|
+
return { status: "ok", eventIds };
|
|
8744
|
+
}
|
|
8745
|
+
return {
|
|
8746
|
+
status: "invalid",
|
|
8747
|
+
reason: "evidence must include eventId or eventIds"
|
|
8748
|
+
};
|
|
8749
|
+
}
|
|
8750
|
+
function provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds) {
|
|
8751
|
+
const findings = [];
|
|
8752
|
+
const evidenceBase = [
|
|
8753
|
+
{
|
|
8754
|
+
runId: outcome.runId,
|
|
8755
|
+
eventId: outcome.outcomeId,
|
|
8756
|
+
kind: "OUTCOME",
|
|
8757
|
+
name: outcome.name,
|
|
8758
|
+
path: `outcome.${outcome.name}`
|
|
8759
|
+
}
|
|
8760
|
+
];
|
|
8761
|
+
if (requireProvenance.method) {
|
|
8762
|
+
if (outcome.method === void 0 || outcome.method.trim() === "") {
|
|
8763
|
+
findings.push(
|
|
8764
|
+
contractFailFinding(
|
|
8765
|
+
"contract.observation.provenance.method",
|
|
8766
|
+
`Observation ${outcome.name} is missing a structural method.`,
|
|
8767
|
+
evidenceBase,
|
|
8768
|
+
{ method: true },
|
|
8769
|
+
{ name: outcome.name, method: outcome.method ?? null }
|
|
8770
|
+
)
|
|
8771
|
+
);
|
|
8772
|
+
} else if (!METHOD_VOCABULARY.has(outcome.method)) {
|
|
8773
|
+
findings.push(
|
|
8774
|
+
contractFailFinding(
|
|
8775
|
+
"contract.observation.provenance.method",
|
|
8776
|
+
`Observation ${outcome.name} method ${outcome.method} is outside the bounded ObservedOutcomeMethod vocabulary.`,
|
|
8777
|
+
evidenceBase,
|
|
8778
|
+
{ methodVocabulary: [...OBSERVED_OUTCOME_METHODS] },
|
|
8779
|
+
{ name: outcome.name, method: outcome.method }
|
|
8780
|
+
)
|
|
8781
|
+
);
|
|
8782
|
+
}
|
|
8783
|
+
}
|
|
8784
|
+
if (requireProvenance.evidence || requireProvenance.sameRunEventReference) {
|
|
8785
|
+
const parsed = parseEvidenceEventIds(outcome.evidence);
|
|
8786
|
+
if (parsed.status === "missing") {
|
|
8787
|
+
findings.push(
|
|
8788
|
+
contractFailFinding(
|
|
8789
|
+
"contract.observation.provenance.evidence",
|
|
8790
|
+
`Observation ${outcome.name} is missing bounded evidence references.`,
|
|
8791
|
+
evidenceBase,
|
|
8792
|
+
{ evidence: true },
|
|
8793
|
+
{ name: outcome.name, evidence: null }
|
|
8794
|
+
)
|
|
8795
|
+
);
|
|
8796
|
+
} else if (parsed.status === "invalid") {
|
|
8797
|
+
findings.push(
|
|
8798
|
+
contractFailFinding(
|
|
8799
|
+
"contract.observation.provenance.evidence",
|
|
8800
|
+
`Observation ${outcome.name} evidence is not a bounded event reference (${parsed.reason}).`,
|
|
8801
|
+
evidenceBase,
|
|
8802
|
+
{ evidence: true },
|
|
8803
|
+
{ name: outcome.name, reason: parsed.reason }
|
|
8804
|
+
)
|
|
8805
|
+
);
|
|
8806
|
+
} else if (requireProvenance.sameRunEventReference) {
|
|
8807
|
+
const missing = parsed.eventIds.filter((eventId) => !knownEventIds.has(eventId));
|
|
8808
|
+
if (missing.length > 0) {
|
|
8809
|
+
findings.push(
|
|
8810
|
+
contractFailFinding(
|
|
8811
|
+
"contract.observation.provenance.same-run",
|
|
8812
|
+
`Observation ${outcome.name} references event ids missing from the same run: ${missing.join(", ")}.`,
|
|
8813
|
+
evidenceBase,
|
|
8814
|
+
{ sameRunEventReference: true, eventIds: parsed.eventIds },
|
|
8815
|
+
{ name: outcome.name, missing }
|
|
8816
|
+
)
|
|
8817
|
+
);
|
|
8818
|
+
}
|
|
8819
|
+
}
|
|
8820
|
+
}
|
|
8821
|
+
return findings;
|
|
8822
|
+
}
|
|
8823
|
+
function contractToRules(contract) {
|
|
8824
|
+
const rules = [];
|
|
8825
|
+
const allowedStatuses = contract.run?.allowedStatuses ?? [];
|
|
8826
|
+
if (allowedStatuses.length === 1) {
|
|
8827
|
+
rules.push(
|
|
8828
|
+
createRunStatusRule({
|
|
8829
|
+
expected: normalizeStatus(allowedStatuses[0]),
|
|
8830
|
+
allowIncomplete: contract.run?.requireCompleted === false
|
|
8831
|
+
})
|
|
8832
|
+
);
|
|
8833
|
+
} else if (allowedStatuses.length > 1) {
|
|
8834
|
+
const expected = [...new Set(allowedStatuses.map(normalizeStatus))];
|
|
8835
|
+
const allowIncomplete = contract.run?.requireCompleted === false;
|
|
8836
|
+
rules.push({
|
|
8837
|
+
id: "contract.run.allowedStatuses",
|
|
8838
|
+
category: "run",
|
|
8839
|
+
defaultSeverity: "error",
|
|
8840
|
+
evaluate(context) {
|
|
8841
|
+
const findings = [];
|
|
8842
|
+
const actual = context.selectedRun?.status ?? "unknown";
|
|
8843
|
+
if (!expected.includes(actual)) {
|
|
8844
|
+
findings.push(
|
|
8845
|
+
contractFailFinding(
|
|
8846
|
+
"contract.run.allowedStatuses",
|
|
8847
|
+
`Run status ${actual} is not one of the allowed statuses: ${expected.join(", ")}.`,
|
|
8848
|
+
context.selectedRun ? [
|
|
8849
|
+
{
|
|
8850
|
+
runId: context.selectedRun.runId,
|
|
8851
|
+
kind: "RUN",
|
|
8852
|
+
name: context.selectedRun.name,
|
|
8853
|
+
status: context.selectedRun.status
|
|
8854
|
+
}
|
|
8855
|
+
] : [],
|
|
8856
|
+
expected,
|
|
8857
|
+
actual
|
|
8858
|
+
)
|
|
8859
|
+
);
|
|
8860
|
+
}
|
|
8861
|
+
if (!allowIncomplete) {
|
|
8862
|
+
const running = (context.logicalEvents ?? context.events).filter(
|
|
8863
|
+
(event) => event.status === "running"
|
|
8864
|
+
);
|
|
8865
|
+
if (running.length > 0) {
|
|
8866
|
+
findings.push(
|
|
8867
|
+
contractFailFinding(
|
|
8868
|
+
"contract.run.allowedStatuses",
|
|
8869
|
+
"Run contains incomplete running events.",
|
|
8870
|
+
running.map((event) => ({
|
|
8871
|
+
runId: event.runId,
|
|
8872
|
+
eventId: event.eventId,
|
|
8873
|
+
kind: event.kind,
|
|
8874
|
+
name: event.name,
|
|
8875
|
+
status: event.status
|
|
8876
|
+
})),
|
|
8877
|
+
"no running events",
|
|
8878
|
+
running.length
|
|
8879
|
+
)
|
|
8880
|
+
);
|
|
8881
|
+
}
|
|
8882
|
+
}
|
|
8883
|
+
return findings;
|
|
8884
|
+
}
|
|
8885
|
+
});
|
|
8886
|
+
} else if (contract.run?.requireCompleted !== false) {
|
|
8887
|
+
rules.push(createRunStatusRule({ allowIncomplete: false }));
|
|
8888
|
+
}
|
|
8889
|
+
if (contract.run?.maxDurationMs !== void 0) {
|
|
8890
|
+
rules.push(createRunDurationRule({ maxDurationMs: contract.run.maxDurationMs }));
|
|
8891
|
+
}
|
|
8892
|
+
if (contract.run?.requireCompleted === true) {
|
|
8893
|
+
rules.push(createStructureIncompleteRule({ requireEndedAtForStarted: true }));
|
|
8894
|
+
}
|
|
8895
|
+
if (contract.tools) {
|
|
8896
|
+
const order = contract.tools.requiredOrder ?? [];
|
|
8897
|
+
const requiredOrderMode = contract.tools.requiredOrderMode ?? "first-occurrence";
|
|
8898
|
+
const orderRules = contract.tools.orderRules ?? [];
|
|
8899
|
+
const endpointRequired = orderRules.filter((rule) => rule.requireEndpoints !== false).flatMap((rule) => [rule.before, rule.after]);
|
|
8900
|
+
const required = [
|
|
8901
|
+
.../* @__PURE__ */ new Set([
|
|
8902
|
+
...contract.tools.required ?? [],
|
|
8903
|
+
...contract.tools.requiredTools ?? [],
|
|
8904
|
+
...order,
|
|
8905
|
+
...endpointRequired
|
|
8906
|
+
])
|
|
8907
|
+
];
|
|
8908
|
+
const forbidden = [
|
|
8909
|
+
...contract.tools.forbidden ?? [],
|
|
8910
|
+
...contract.tools.forbiddenTools ?? []
|
|
8911
|
+
];
|
|
8912
|
+
rules.push(
|
|
8913
|
+
createToolUsageRule({
|
|
8914
|
+
...required.length > 0 ? { required } : {},
|
|
8915
|
+
...forbidden.length > 0 ? { forbidden } : {},
|
|
8916
|
+
...contract.tools.allowed ? { allowed: contract.tools.allowed } : {},
|
|
8917
|
+
...contract.tools.maxCalls !== void 0 ? { maxCount: contract.tools.maxCalls } : {}
|
|
8918
|
+
})
|
|
8919
|
+
);
|
|
8920
|
+
for (let i = 0; i < order.length - 1; i += 1) {
|
|
8921
|
+
rules.push(
|
|
8922
|
+
createToolOrderingRule({
|
|
8923
|
+
before: order[i],
|
|
8924
|
+
after: order[i + 1],
|
|
8925
|
+
id: `contract.tool.order.${i}`,
|
|
8926
|
+
mode: requiredOrderMode
|
|
8927
|
+
})
|
|
8928
|
+
);
|
|
8929
|
+
}
|
|
8930
|
+
const defaultOccurrenceMode = contract.tools.defaultOccurrenceMode ?? requiredOrderMode;
|
|
8931
|
+
for (const [index, rule] of orderRules.entries()) {
|
|
8932
|
+
rules.push(
|
|
8933
|
+
createToolOrderingRule({
|
|
8934
|
+
before: rule.before,
|
|
8935
|
+
after: rule.after,
|
|
8936
|
+
id: `contract.tool.orderRule.${index}`,
|
|
8937
|
+
mode: rule.occurrenceMode ?? defaultOccurrenceMode
|
|
8938
|
+
})
|
|
8939
|
+
);
|
|
8940
|
+
}
|
|
8941
|
+
const argumentChecks = contract.tools.arguments ?? [];
|
|
8942
|
+
if (argumentChecks.length > 0) {
|
|
8943
|
+
rules.push({
|
|
8944
|
+
id: "contract.tool.arguments",
|
|
8945
|
+
category: "tool",
|
|
8946
|
+
defaultSeverity: "error",
|
|
8947
|
+
evaluate(context) {
|
|
8948
|
+
const tools = (context.logicalEvents ?? context.events).filter(
|
|
8949
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
8950
|
+
);
|
|
8951
|
+
const findings = [];
|
|
8952
|
+
for (const [index, check] of argumentChecks.entries()) {
|
|
8953
|
+
const matches = tools.filter(
|
|
8954
|
+
(event) => resolveCanonicalToolName(event) === check.tool
|
|
8955
|
+
);
|
|
8956
|
+
if (matches.length === 0) {
|
|
8957
|
+
findings.push(
|
|
8958
|
+
contractFailFinding(
|
|
8959
|
+
`contract.tool.arguments.${index}`,
|
|
8960
|
+
`No finished tool named ${check.tool} for argument check.`,
|
|
8961
|
+
context.selectedRun ? [
|
|
8962
|
+
{
|
|
8963
|
+
runId: context.selectedRun.runId,
|
|
8964
|
+
kind: "RUN",
|
|
8965
|
+
name: context.selectedRun.name
|
|
8966
|
+
}
|
|
8967
|
+
] : [],
|
|
8968
|
+
{ tool: check.tool, path: check.path },
|
|
8969
|
+
{ toolCount: 0 }
|
|
8970
|
+
)
|
|
8971
|
+
);
|
|
8972
|
+
continue;
|
|
8973
|
+
}
|
|
8974
|
+
const occurrence = check.occurrence ?? "all";
|
|
8975
|
+
const selected = occurrence === "first" ? [matches[0]] : occurrence === "last" ? [matches[matches.length - 1]] : matches;
|
|
8976
|
+
const results = selected.map((event) => {
|
|
8977
|
+
const payload = extractToolArgumentPayload(event);
|
|
8978
|
+
return evaluateToolArgumentValue(
|
|
8979
|
+
payload.value,
|
|
8980
|
+
check,
|
|
8981
|
+
payload.present
|
|
8982
|
+
);
|
|
8983
|
+
});
|
|
8984
|
+
if (occurrence === "any") {
|
|
8985
|
+
if (results.some((result) => result.status === "pass")) continue;
|
|
8986
|
+
const firstFail = results.find((result) => result.status !== "pass");
|
|
8987
|
+
findings.push(
|
|
8988
|
+
contractFailFinding(
|
|
8989
|
+
`contract.tool.arguments.${index}`,
|
|
8990
|
+
firstFail.message,
|
|
8991
|
+
selected.slice(0, 1).map((event) => ({
|
|
8992
|
+
runId: event.runId,
|
|
8993
|
+
eventId: event.eventId,
|
|
8994
|
+
kind: event.kind,
|
|
8995
|
+
name: event.name,
|
|
8996
|
+
path: `tool.${check.tool}${check.path}`
|
|
8997
|
+
})),
|
|
8998
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
8999
|
+
{ code: firstFail.code }
|
|
9000
|
+
)
|
|
9001
|
+
);
|
|
9002
|
+
continue;
|
|
9003
|
+
}
|
|
9004
|
+
for (const [selIndex, result] of results.entries()) {
|
|
9005
|
+
if (result.status === "pass") continue;
|
|
9006
|
+
const event = selected[selIndex];
|
|
9007
|
+
findings.push(
|
|
9008
|
+
contractFailFinding(
|
|
9009
|
+
`contract.tool.arguments.${index}`,
|
|
9010
|
+
result.message,
|
|
9011
|
+
[
|
|
9012
|
+
{
|
|
9013
|
+
runId: event.runId,
|
|
9014
|
+
eventId: event.eventId,
|
|
9015
|
+
kind: event.kind,
|
|
9016
|
+
name: event.name,
|
|
9017
|
+
path: `tool.${check.tool}${check.path}`
|
|
9018
|
+
}
|
|
9019
|
+
],
|
|
9020
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
9021
|
+
{ code: result.code }
|
|
9022
|
+
)
|
|
9023
|
+
);
|
|
9024
|
+
}
|
|
9025
|
+
}
|
|
9026
|
+
return findings;
|
|
9027
|
+
}
|
|
9028
|
+
});
|
|
9029
|
+
}
|
|
9030
|
+
}
|
|
9031
|
+
if (contract.llm) {
|
|
9032
|
+
rules.push(
|
|
9033
|
+
createLlmUsageRule({
|
|
9034
|
+
...contract.llm.maxCalls !== void 0 ? { maxCalls: contract.llm.maxCalls } : {},
|
|
9035
|
+
...contract.llm.maxTotalTokens !== void 0 ? { maxTotalTokens: contract.llm.maxTotalTokens } : {},
|
|
9036
|
+
...contract.llm.allowedModels ? { allowedModels: contract.llm.allowedModels } : {}
|
|
9037
|
+
})
|
|
9038
|
+
);
|
|
9039
|
+
}
|
|
9040
|
+
if (contract.observations) {
|
|
9041
|
+
const required = contract.observations.required ?? [];
|
|
9042
|
+
if (required.length > 0) {
|
|
9043
|
+
rules.push({
|
|
9044
|
+
id: "contract.observation.required",
|
|
9045
|
+
category: "run",
|
|
9046
|
+
defaultSeverity: "error",
|
|
9047
|
+
evaluate(context) {
|
|
9048
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
9049
|
+
const names = new Set(outcomes.map((item) => item.name));
|
|
9050
|
+
const missing = required.filter((name) => !names.has(name));
|
|
9051
|
+
if (missing.length === 0) return [];
|
|
9052
|
+
return [
|
|
9053
|
+
contractFailFinding(
|
|
9054
|
+
"contract.observation.required",
|
|
9055
|
+
`Required observations missing: ${missing.join(", ")}`,
|
|
9056
|
+
context.selectedRun ? [{ runId: context.selectedRun.runId, kind: "RUN", name: context.selectedRun.name }] : [],
|
|
9057
|
+
required,
|
|
9058
|
+
[...names]
|
|
9059
|
+
)
|
|
9060
|
+
];
|
|
9061
|
+
}
|
|
9062
|
+
});
|
|
9063
|
+
}
|
|
9064
|
+
if (contract.observations.failOn?.length) {
|
|
9065
|
+
rules.push(
|
|
9066
|
+
createObservedOutcomeRule({
|
|
9067
|
+
failOn: contract.observations.failOn.filter(
|
|
9068
|
+
(status) => status === "failed" || status === "unknown" || status === "skipped"
|
|
9069
|
+
)
|
|
9070
|
+
})
|
|
9071
|
+
);
|
|
9072
|
+
}
|
|
9073
|
+
const requireProvenance = contract.observations.requireProvenance;
|
|
9074
|
+
if (requireProvenance && (requireProvenance.method || requireProvenance.evidence || requireProvenance.sameRunEventReference)) {
|
|
9075
|
+
rules.push({
|
|
9076
|
+
id: "contract.observation.provenance",
|
|
9077
|
+
category: "run",
|
|
9078
|
+
defaultSeverity: "error",
|
|
9079
|
+
evaluate(context) {
|
|
9080
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
9081
|
+
const knownEventIds = new Set(context.events.map((event) => event.eventId));
|
|
9082
|
+
const targets = required.length > 0 ? outcomes.filter((outcome) => required.includes(outcome.name)) : outcomes;
|
|
9083
|
+
return targets.flatMap(
|
|
9084
|
+
(outcome) => provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds)
|
|
9085
|
+
);
|
|
9086
|
+
}
|
|
9087
|
+
});
|
|
9088
|
+
}
|
|
9089
|
+
}
|
|
9090
|
+
if (contract.controls) {
|
|
9091
|
+
const controls = contract.controls;
|
|
9092
|
+
rules.push({
|
|
9093
|
+
id: "contract.controls",
|
|
9094
|
+
category: "run",
|
|
9095
|
+
defaultSeverity: "error",
|
|
9096
|
+
evaluate(context) {
|
|
9097
|
+
const events = context.logicalEvents ?? context.events;
|
|
9098
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
9099
|
+
const observationNames = new Set(outcomes.map((item) => item.name));
|
|
9100
|
+
const runEvidence2 = context.selectedRun ? [
|
|
9101
|
+
{
|
|
9102
|
+
runId: context.selectedRun.runId,
|
|
9103
|
+
kind: "RUN",
|
|
9104
|
+
name: context.selectedRun.name
|
|
9105
|
+
}
|
|
9106
|
+
] : [];
|
|
9107
|
+
return evaluateControlRules(events, controls, runEvidence2, observationNames);
|
|
9108
|
+
}
|
|
9109
|
+
});
|
|
9110
|
+
}
|
|
9111
|
+
if (contract.retry) {
|
|
9112
|
+
const retry = contract.retry;
|
|
9113
|
+
rules.push({
|
|
9114
|
+
id: "contract.retry",
|
|
9115
|
+
category: "run",
|
|
9116
|
+
defaultSeverity: "error",
|
|
9117
|
+
evaluate(context) {
|
|
9118
|
+
const events = context.logicalEvents ?? context.events;
|
|
9119
|
+
context.selectedRun ? [
|
|
9120
|
+
{
|
|
9121
|
+
runId: context.selectedRun.runId,
|
|
9122
|
+
kind: "RUN",
|
|
9123
|
+
name: context.selectedRun.name
|
|
9124
|
+
}
|
|
9125
|
+
] : [];
|
|
9126
|
+
const base = evaluateRetrySafetyRules(events, retry);
|
|
9127
|
+
if (!retry.operations || retry.operations.length === 0) {
|
|
9128
|
+
return base;
|
|
9129
|
+
}
|
|
9130
|
+
return [...base, ...evaluateRecoveryOperations(events, retry.operations)];
|
|
9131
|
+
}
|
|
9132
|
+
});
|
|
9133
|
+
}
|
|
9134
|
+
return rules;
|
|
9135
|
+
}
|
|
7104
9136
|
function validateScopeShape(scope) {
|
|
7105
9137
|
if (scope === void 0) return [];
|
|
7106
9138
|
const hasSelector = typeof scope.runId === "string" && scope.runId.trim() !== "" || typeof scope.subAgentId === "string" && scope.subAgentId.trim() !== "" || typeof scope.groupId === "string" && scope.groupId.trim() !== "" || typeof scope.workflowStep === "string" && scope.workflowStep.trim() !== "" || typeof scope.rootEventId === "string" && scope.rootEventId.trim() !== "";
|
|
@@ -7201,6 +9233,57 @@ function validateAlternativesShape(alternatives) {
|
|
|
7201
9233
|
}
|
|
7202
9234
|
return diagnostics;
|
|
7203
9235
|
}
|
|
9236
|
+
function emptyPassResult(input3, runId) {
|
|
9237
|
+
return {
|
|
9238
|
+
ok: true,
|
|
9239
|
+
status: "pass",
|
|
9240
|
+
format: input3.read.format,
|
|
9241
|
+
...runId !== void 0 ? { runId } : {},
|
|
9242
|
+
summary: { passed: 0, failed: 0, warnings: 0, errors: 0, rulesEvaluated: 0 },
|
|
9243
|
+
findings: [],
|
|
9244
|
+
diagnostics: [],
|
|
9245
|
+
ruleExecutions: []
|
|
9246
|
+
};
|
|
9247
|
+
}
|
|
9248
|
+
function mergeContractResults(input3, parts, extraFindings = []) {
|
|
9249
|
+
const findings = [...parts.flatMap((part) => part.findings), ...extraFindings];
|
|
9250
|
+
const diagnostics = parts.flatMap((part) => part.diagnostics);
|
|
9251
|
+
const ruleExecutions = parts.flatMap((part) => part.ruleExecutions);
|
|
9252
|
+
const failed = findings.filter(
|
|
9253
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
9254
|
+
).length;
|
|
9255
|
+
const errors = diagnostics.filter((item) => item.severity === "error").length;
|
|
9256
|
+
const status = errors > 0 ? "error" : failed > 0 ? "fail" : "pass";
|
|
9257
|
+
const runId = parts.find((part) => part.runId !== void 0)?.runId;
|
|
9258
|
+
return {
|
|
9259
|
+
ok: status === "pass",
|
|
9260
|
+
status,
|
|
9261
|
+
format: input3.read.format,
|
|
9262
|
+
...runId !== void 0 ? { runId } : {},
|
|
9263
|
+
summary: {
|
|
9264
|
+
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
9265
|
+
failed,
|
|
9266
|
+
warnings: findings.filter(
|
|
9267
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
9268
|
+
).length,
|
|
9269
|
+
errors,
|
|
9270
|
+
rulesEvaluated: ruleExecutions.length
|
|
9271
|
+
},
|
|
9272
|
+
findings,
|
|
9273
|
+
diagnostics,
|
|
9274
|
+
ruleExecutions
|
|
9275
|
+
};
|
|
9276
|
+
}
|
|
9277
|
+
function evaluateBody(input3, body, options = {}) {
|
|
9278
|
+
const rules = contractToRules(body);
|
|
9279
|
+
if (rules.length === 0) {
|
|
9280
|
+
return emptyPassResult(input3, options.runId);
|
|
9281
|
+
}
|
|
9282
|
+
return runTraceChecks(input3, {
|
|
9283
|
+
rules,
|
|
9284
|
+
...options.runId !== void 0 ? { runId: options.runId } : {}
|
|
9285
|
+
});
|
|
9286
|
+
}
|
|
7204
9287
|
function defineTraceContract(input3) {
|
|
7205
9288
|
const shapeErrors = [
|
|
7206
9289
|
...validateAlternativesShape(input3.alternatives),
|
|
@@ -7231,6 +9314,144 @@ function defineTraceContract(input3) {
|
|
|
7231
9314
|
...alternatives ? { alternatives } : {}
|
|
7232
9315
|
};
|
|
7233
9316
|
}
|
|
9317
|
+
function evaluateTraceContract(input3, contract, options = {}) {
|
|
9318
|
+
const shapeErrors = [
|
|
9319
|
+
...validateAlternativesShape(contract.alternatives),
|
|
9320
|
+
...validateScopeShape(contract.scope),
|
|
9321
|
+
...validateProvenanceShape(contract.observations)
|
|
9322
|
+
].filter((item) => item.severity === "error");
|
|
9323
|
+
if (shapeErrors.length > 0) {
|
|
9324
|
+
return {
|
|
9325
|
+
ok: false,
|
|
9326
|
+
status: "error",
|
|
9327
|
+
format: input3.read.format,
|
|
9328
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
9329
|
+
summary: {
|
|
9330
|
+
passed: 0,
|
|
9331
|
+
failed: 0,
|
|
9332
|
+
warnings: 0,
|
|
9333
|
+
errors: shapeErrors.length,
|
|
9334
|
+
rulesEvaluated: 0
|
|
9335
|
+
},
|
|
9336
|
+
findings: [],
|
|
9337
|
+
diagnostics: shapeErrors.map((item) => ({
|
|
9338
|
+
code: "AI_CHECK_INVALID_CONFIG",
|
|
9339
|
+
message: item.message,
|
|
9340
|
+
severity: "error",
|
|
9341
|
+
ruleId: item.code
|
|
9342
|
+
})),
|
|
9343
|
+
ruleExecutions: []
|
|
9344
|
+
};
|
|
9345
|
+
}
|
|
9346
|
+
const scoped = resolveTraceContractScope(input3, contract.scope);
|
|
9347
|
+
if (scoped.diagnostics.length > 0) {
|
|
9348
|
+
return {
|
|
9349
|
+
ok: false,
|
|
9350
|
+
status: "error",
|
|
9351
|
+
format: input3.read.format,
|
|
9352
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
9353
|
+
...scoped.resolved?.runId !== void 0 ? { runId: scoped.resolved.runId } : {},
|
|
9354
|
+
summary: {
|
|
9355
|
+
passed: 0,
|
|
9356
|
+
failed: 0,
|
|
9357
|
+
warnings: 0,
|
|
9358
|
+
errors: scoped.diagnostics.length,
|
|
9359
|
+
rulesEvaluated: 0
|
|
9360
|
+
},
|
|
9361
|
+
findings: [],
|
|
9362
|
+
diagnostics: scoped.diagnostics,
|
|
9363
|
+
ruleExecutions: []
|
|
9364
|
+
};
|
|
9365
|
+
}
|
|
9366
|
+
const effectiveInput = scoped.resolved?.input ?? input3;
|
|
9367
|
+
const effectiveRunId = scoped.resolved?.runId ?? options.runId;
|
|
9368
|
+
const base = evaluateBody(effectiveInput, contract, {
|
|
9369
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
9370
|
+
});
|
|
9371
|
+
const scopeFinding = scoped.resolved ? {
|
|
9372
|
+
ruleId: "contract.scope.selected",
|
|
9373
|
+
severity: "info",
|
|
9374
|
+
status: "pass",
|
|
9375
|
+
message: `Selected actor ${scoped.resolved.runId} with ${scoped.resolved.evidenceEventCount} evidence event(s).`,
|
|
9376
|
+
expected: scoped.resolved.matchedSelectors,
|
|
9377
|
+
actual: {
|
|
9378
|
+
runId: scoped.resolved.runId,
|
|
9379
|
+
evidenceEventCount: scoped.resolved.evidenceEventCount,
|
|
9380
|
+
...scoped.resolved.rootEventId !== void 0 ? { rootEventId: scoped.resolved.rootEventId } : {}
|
|
9381
|
+
},
|
|
9382
|
+
evidence: [
|
|
9383
|
+
{
|
|
9384
|
+
runId: scoped.resolved.runId,
|
|
9385
|
+
kind: "RUN",
|
|
9386
|
+
path: "scope"
|
|
9387
|
+
}
|
|
9388
|
+
]
|
|
9389
|
+
} : void 0;
|
|
9390
|
+
const branches = contract.alternatives?.anyOf ?? [];
|
|
9391
|
+
if (branches.length === 0) {
|
|
9392
|
+
return scopeFinding ? mergeContractResults(effectiveInput, [base], [scopeFinding]) : base;
|
|
9393
|
+
}
|
|
9394
|
+
const branchEvaluations = branches.map((branch) => {
|
|
9395
|
+
const result = evaluateBody(effectiveInput, branch.contract, {
|
|
9396
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
9397
|
+
});
|
|
9398
|
+
return { branch, result };
|
|
9399
|
+
});
|
|
9400
|
+
const satisfied = branchEvaluations.filter((item) => item.result.ok && item.result.status === "pass").map((item) => item.branch.id);
|
|
9401
|
+
const failedBranches = branchEvaluations.filter(
|
|
9402
|
+
(item) => !(item.result.ok && item.result.status === "pass")
|
|
9403
|
+
);
|
|
9404
|
+
if (satisfied.length > 0) {
|
|
9405
|
+
const passFinding = {
|
|
9406
|
+
ruleId: "contract.alternatives.anyOf",
|
|
9407
|
+
severity: "info",
|
|
9408
|
+
status: "pass",
|
|
9409
|
+
message: `Satisfied alternative branch(es): ${satisfied.join(", ")}.`,
|
|
9410
|
+
expected: { anyOf: branches.map((branch) => branch.id) },
|
|
9411
|
+
actual: {
|
|
9412
|
+
satisfied,
|
|
9413
|
+
failed: failedBranches.map((item) => item.branch.id)
|
|
9414
|
+
},
|
|
9415
|
+
evidence: []
|
|
9416
|
+
};
|
|
9417
|
+
return mergeContractResults(
|
|
9418
|
+
effectiveInput,
|
|
9419
|
+
[base],
|
|
9420
|
+
[...scopeFinding ? [scopeFinding] : [], passFinding]
|
|
9421
|
+
);
|
|
9422
|
+
}
|
|
9423
|
+
const noneFinding = {
|
|
9424
|
+
ruleId: "contract.alternatives.none-satisfied",
|
|
9425
|
+
severity: "error",
|
|
9426
|
+
status: "fail",
|
|
9427
|
+
message: "No alternatives.anyOf branch fully satisfied its contract.",
|
|
9428
|
+
expected: { anyOf: branches.map((branch) => branch.id) },
|
|
9429
|
+
actual: {
|
|
9430
|
+
branchFailures: Object.fromEntries(
|
|
9431
|
+
failedBranches.map((item) => [
|
|
9432
|
+
item.branch.id,
|
|
9433
|
+
item.result.findings.filter((finding) => finding.status === "fail").map((finding) => ({
|
|
9434
|
+
ruleId: finding.ruleId,
|
|
9435
|
+
message: finding.message
|
|
9436
|
+
}))
|
|
9437
|
+
])
|
|
9438
|
+
)
|
|
9439
|
+
},
|
|
9440
|
+
evidence: []
|
|
9441
|
+
};
|
|
9442
|
+
const prefixedBranchFindings = failedBranches.flatMap(
|
|
9443
|
+
(item) => item.result.findings.filter((finding) => finding.status === "fail").map((finding) => ({
|
|
9444
|
+
...finding,
|
|
9445
|
+
ruleId: `contract.alternatives.${item.branch.id}.${finding.ruleId}`
|
|
9446
|
+
}))
|
|
9447
|
+
);
|
|
9448
|
+
return mergeContractResults(
|
|
9449
|
+
effectiveInput,
|
|
9450
|
+
[base],
|
|
9451
|
+
[...scopeFinding ? [scopeFinding] : [], noneFinding, ...prefixedBranchFindings]
|
|
9452
|
+
);
|
|
9453
|
+
}
|
|
9454
|
+
var MAX_EVIDENCE_EVENT_IDS, METHOD_VOCABULARY;
|
|
7234
9455
|
var init_contract = __esm({
|
|
7235
9456
|
"packages/core/src/checks/contract.ts"() {
|
|
7236
9457
|
init_checks2();
|
|
@@ -7243,7 +9464,8 @@ var init_contract = __esm({
|
|
|
7243
9464
|
init_outcomes();
|
|
7244
9465
|
init_contract_scope();
|
|
7245
9466
|
init_tool_arguments();
|
|
7246
|
-
|
|
9467
|
+
MAX_EVIDENCE_EVENT_IDS = 16;
|
|
9468
|
+
METHOD_VOCABULARY = new Set(OBSERVED_OUTCOME_METHODS);
|
|
7247
9469
|
}
|
|
7248
9470
|
});
|
|
7249
9471
|
|
|
@@ -7618,18 +9840,18 @@ function escapeMarkdown(value) {
|
|
|
7618
9840
|
function escapeHtml(value) {
|
|
7619
9841
|
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
7620
9842
|
}
|
|
7621
|
-
function
|
|
9843
|
+
function sortKeysDeep2(input3) {
|
|
7622
9844
|
if (input3 === null || typeof input3 !== "object") return input3;
|
|
7623
|
-
if (Array.isArray(input3)) return input3.map(
|
|
9845
|
+
if (Array.isArray(input3)) return input3.map(sortKeysDeep2);
|
|
7624
9846
|
const o = input3;
|
|
7625
9847
|
const out = {};
|
|
7626
9848
|
for (const k of Object.keys(o).sort()) {
|
|
7627
|
-
out[k] =
|
|
9849
|
+
out[k] = sortKeysDeep2(o[k]);
|
|
7628
9850
|
}
|
|
7629
9851
|
return out;
|
|
7630
9852
|
}
|
|
7631
9853
|
function stableJson(value, pretty) {
|
|
7632
|
-
const sorted =
|
|
9854
|
+
const sorted = sortKeysDeep2(value);
|
|
7633
9855
|
return pretty === true ? JSON.stringify(sorted, null, 2) : JSON.stringify(sorted);
|
|
7634
9856
|
}
|
|
7635
9857
|
function compactAttributes(attrs, options) {
|
|
@@ -9262,7 +11484,7 @@ var init_types5 = __esm({
|
|
|
9262
11484
|
DEFAULT_SUITE_ARTIFACTS_DIR = ".agent-inspect/suite-runs";
|
|
9263
11485
|
}
|
|
9264
11486
|
});
|
|
9265
|
-
function
|
|
11487
|
+
function diagnostic3(code, message, severity = "error", caseId) {
|
|
9266
11488
|
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
9267
11489
|
}
|
|
9268
11490
|
function asString(value, label) {
|
|
@@ -9290,7 +11512,7 @@ function validateCaseConfig(value, index) {
|
|
|
9290
11512
|
const diagnostics = [];
|
|
9291
11513
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
9292
11514
|
diagnostics.push(
|
|
9293
|
-
|
|
11515
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
|
|
9294
11516
|
);
|
|
9295
11517
|
return { diagnostics };
|
|
9296
11518
|
}
|
|
@@ -9299,7 +11521,7 @@ function validateCaseConfig(value, index) {
|
|
|
9299
11521
|
const id = asString(raw.id, `cases[${index}].id`);
|
|
9300
11522
|
if (id === void 0) {
|
|
9301
11523
|
diagnostics.push(
|
|
9302
|
-
|
|
11524
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
|
|
9303
11525
|
);
|
|
9304
11526
|
return { diagnostics };
|
|
9305
11527
|
}
|
|
@@ -9334,7 +11556,7 @@ function validateCaseConfig(value, index) {
|
|
|
9334
11556
|
};
|
|
9335
11557
|
} catch (error) {
|
|
9336
11558
|
const message = error instanceof Error ? error.message : String(error);
|
|
9337
|
-
diagnostics.push(
|
|
11559
|
+
diagnostics.push(diagnostic3("AI_SUITE_CONFIG_INVALID", message));
|
|
9338
11560
|
return { diagnostics };
|
|
9339
11561
|
}
|
|
9340
11562
|
}
|
|
@@ -9410,7 +11632,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
9410
11632
|
await promises.access(tracesDir);
|
|
9411
11633
|
} catch {
|
|
9412
11634
|
diagnostics.push(
|
|
9413
|
-
|
|
11635
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `traces directory not found: ${tracesDir}`)
|
|
9414
11636
|
);
|
|
9415
11637
|
}
|
|
9416
11638
|
for (const suiteCase of config.cases) {
|
|
@@ -9420,7 +11642,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
9420
11642
|
await promises.access(inputPath);
|
|
9421
11643
|
} catch {
|
|
9422
11644
|
diagnostics.push(
|
|
9423
|
-
|
|
11645
|
+
diagnostic3(
|
|
9424
11646
|
"AI_SUITE_CONFIG_INVALID",
|
|
9425
11647
|
`input fixture not found: ${suiteCase.input}`,
|
|
9426
11648
|
"warning",
|
|
@@ -9436,7 +11658,7 @@ var init_validate2 = __esm({
|
|
|
9436
11658
|
"packages/core/src/suite/validate.ts"() {
|
|
9437
11659
|
}
|
|
9438
11660
|
});
|
|
9439
|
-
function
|
|
11661
|
+
function diagnostic4(code, message) {
|
|
9440
11662
|
return { code, message, severity: "error" };
|
|
9441
11663
|
}
|
|
9442
11664
|
async function fileExists(filePath) {
|
|
@@ -9467,20 +11689,20 @@ async function loadSuiteConfig(options = {}) {
|
|
|
9467
11689
|
} catch (error) {
|
|
9468
11690
|
const message = error instanceof Error ? error.message : String(error);
|
|
9469
11691
|
throw Object.assign(new Error(message), {
|
|
9470
|
-
diagnostics: [
|
|
11692
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
9471
11693
|
});
|
|
9472
11694
|
}
|
|
9473
11695
|
const extension = path33__default.default.extname(configPath);
|
|
9474
11696
|
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
9475
11697
|
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
9476
11698
|
throw Object.assign(new Error(message), {
|
|
9477
|
-
diagnostics: [
|
|
11699
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
9478
11700
|
});
|
|
9479
11701
|
}
|
|
9480
11702
|
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
9481
11703
|
const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
|
|
9482
11704
|
throw Object.assign(new Error(message), {
|
|
9483
|
-
diagnostics: [
|
|
11705
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
9484
11706
|
});
|
|
9485
11707
|
}
|
|
9486
11708
|
try {
|
|
@@ -9500,7 +11722,7 @@ async function loadSuiteConfig(options = {}) {
|
|
|
9500
11722
|
} catch (error) {
|
|
9501
11723
|
const message = error instanceof Error ? error.message : String(error);
|
|
9502
11724
|
throw Object.assign(new Error(message), {
|
|
9503
|
-
diagnostics: [
|
|
11725
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
9504
11726
|
});
|
|
9505
11727
|
}
|
|
9506
11728
|
}
|
|
@@ -10427,13 +12649,13 @@ function persistedEventsForParsedTrace(parsed) {
|
|
|
10427
12649
|
function isRecord11(value) {
|
|
10428
12650
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
10429
12651
|
}
|
|
10430
|
-
function
|
|
12652
|
+
function isNonEmptyString5(value) {
|
|
10431
12653
|
return typeof value === "string" && value.trim() !== "";
|
|
10432
12654
|
}
|
|
10433
12655
|
function readStringField(record, keys) {
|
|
10434
12656
|
for (const key of keys) {
|
|
10435
12657
|
const value = record[key];
|
|
10436
|
-
if (
|
|
12658
|
+
if (isNonEmptyString5(value)) return value;
|
|
10437
12659
|
}
|
|
10438
12660
|
return void 0;
|
|
10439
12661
|
}
|
|
@@ -10567,7 +12789,7 @@ function parseUnixNanoToIso(value) {
|
|
|
10567
12789
|
return void 0;
|
|
10568
12790
|
}
|
|
10569
12791
|
function parseIsoTime(value) {
|
|
10570
|
-
if (!
|
|
12792
|
+
if (!isNonEmptyString5(value)) return void 0;
|
|
10571
12793
|
const ms = Date.parse(value);
|
|
10572
12794
|
if (!Number.isFinite(ms)) return void 0;
|
|
10573
12795
|
return new Date(ms).toISOString();
|
|
@@ -11824,7 +14046,7 @@ var init_readers = __esm({
|
|
|
11824
14046
|
];
|
|
11825
14047
|
}
|
|
11826
14048
|
});
|
|
11827
|
-
function
|
|
14049
|
+
function diagnostic5(code, message, severity = "error", caseId) {
|
|
11828
14050
|
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
11829
14051
|
}
|
|
11830
14052
|
function buildCaseRules(suiteCase, config) {
|
|
@@ -11888,7 +14110,7 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
11888
14110
|
const match = outcomes.find((outcome) => outcome.name === name);
|
|
11889
14111
|
if (!match) {
|
|
11890
14112
|
diagnostics.push(
|
|
11891
|
-
|
|
14113
|
+
diagnostic5(
|
|
11892
14114
|
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
11893
14115
|
`Expected observation "${name}" was not recorded.`,
|
|
11894
14116
|
"error",
|
|
@@ -11899,7 +14121,7 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
11899
14121
|
}
|
|
11900
14122
|
if (match.status !== "passed") {
|
|
11901
14123
|
diagnostics.push(
|
|
11902
|
-
|
|
14124
|
+
diagnostic5(
|
|
11903
14125
|
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
11904
14126
|
`Observation "${name}" has status "${match.status}", expected "passed".`,
|
|
11905
14127
|
"error",
|
|
@@ -11919,7 +14141,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11919
14141
|
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
11920
14142
|
message: resolved.reason,
|
|
11921
14143
|
diagnostics: [
|
|
11922
|
-
|
|
14144
|
+
diagnostic5(
|
|
11923
14145
|
"AI_SUITE_CASE_TRACE_MISSING",
|
|
11924
14146
|
resolved.reason ?? "Trace not found.",
|
|
11925
14147
|
"warning",
|
|
@@ -11940,7 +14162,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11940
14162
|
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
11941
14163
|
message,
|
|
11942
14164
|
diagnostics: [
|
|
11943
|
-
|
|
14165
|
+
diagnostic5("AI_SUITE_TRACE_UNREADABLE", message, "error", suiteCase.id)
|
|
11944
14166
|
]
|
|
11945
14167
|
};
|
|
11946
14168
|
}
|
|
@@ -11954,7 +14176,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11954
14176
|
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
11955
14177
|
const diagnostics = [
|
|
11956
14178
|
...checkResult.diagnostics.map(
|
|
11957
|
-
(item) =>
|
|
14179
|
+
(item) => diagnostic5(
|
|
11958
14180
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
11959
14181
|
item.message,
|
|
11960
14182
|
item.severity,
|
|
@@ -11962,7 +14184,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11962
14184
|
)
|
|
11963
14185
|
),
|
|
11964
14186
|
...checkResult.findings.filter((finding) => finding.status === "fail").map(
|
|
11965
|
-
(finding) =>
|
|
14187
|
+
(finding) => diagnostic5(
|
|
11966
14188
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
11967
14189
|
finding.message,
|
|
11968
14190
|
finding.severity,
|
|
@@ -13779,7 +16001,7 @@ var init_src = __esm({
|
|
|
13779
16001
|
});
|
|
13780
16002
|
|
|
13781
16003
|
// package.json
|
|
13782
|
-
var version = "6.
|
|
16004
|
+
var version = "6.30.0";
|
|
13783
16005
|
|
|
13784
16006
|
// packages/cli/src/list.ts
|
|
13785
16007
|
init_advanced();
|
|
@@ -18628,10 +20850,10 @@ function buildSafetyRules(options) {
|
|
|
18628
20850
|
})
|
|
18629
20851
|
];
|
|
18630
20852
|
}
|
|
18631
|
-
function
|
|
20853
|
+
function flattenNodes3(nodes) {
|
|
18632
20854
|
return nodes.flatMap((node) => [
|
|
18633
20855
|
node,
|
|
18634
|
-
...
|
|
20856
|
+
...flattenNodes3(
|
|
18635
20857
|
node.children
|
|
18636
20858
|
)
|
|
18637
20859
|
]);
|
|
@@ -18666,7 +20888,7 @@ function redactionDetectorFindings(read, runId, policy) {
|
|
|
18666
20888
|
const out = [];
|
|
18667
20889
|
const policyOptions = redactionOptionsFromPolicy(policy);
|
|
18668
20890
|
for (const run of runs) {
|
|
18669
|
-
for (const node of
|
|
20891
|
+
for (const node of flattenNodes3(run.children)) {
|
|
18670
20892
|
const attrs = node.event.attributes;
|
|
18671
20893
|
if (attrs === void 0) continue;
|
|
18672
20894
|
const result = createRedactor({
|
|
@@ -18806,8 +21028,8 @@ function printHuman(result, explain = false) {
|
|
|
18806
21028
|
console.log(
|
|
18807
21029
|
`Summary: ${result.summary.findings} finding(s), ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
18808
21030
|
);
|
|
18809
|
-
for (const
|
|
18810
|
-
console.log(`- ${
|
|
21031
|
+
for (const diagnostic8 of result.diagnostics) {
|
|
21032
|
+
console.log(`- ${diagnostic8.code}: ${diagnostic8.message}`);
|
|
18811
21033
|
}
|
|
18812
21034
|
for (const finding of result.findings) {
|
|
18813
21035
|
const path44 = finding.evidence[0]?.path;
|
|
@@ -19724,7 +21946,8 @@ async function writeLocalEvidence(input3) {
|
|
|
19724
21946
|
redactionProfile: profile,
|
|
19725
21947
|
assessmentStatus,
|
|
19726
21948
|
checkResultsJson: input3.checkResultsJson,
|
|
19727
|
-
...input3.summaryText !== void 0 ? { summaryText: input3.summaryText } : {}
|
|
21949
|
+
...input3.summaryText !== void 0 ? { summaryText: input3.summaryText } : {},
|
|
21950
|
+
...input3.contractPackage !== void 0 ? { contractPackage: input3.contractPackage } : {}
|
|
19728
21951
|
});
|
|
19729
21952
|
const files = [
|
|
19730
21953
|
["evidence.html", evidencePackage["evidence.html"]],
|
|
@@ -19732,6 +21955,10 @@ async function writeLocalEvidence(input3) {
|
|
|
19732
21955
|
["check-results.json", evidencePackage["check-results.json"]],
|
|
19733
21956
|
["trace.jsonl", evidencePackage["trace.jsonl"]]
|
|
19734
21957
|
];
|
|
21958
|
+
const resolvedContract = evidencePackage["contract.resolved.json"];
|
|
21959
|
+
if (resolvedContract !== void 0) {
|
|
21960
|
+
files.push(["contract.resolved.json", resolvedContract]);
|
|
21961
|
+
}
|
|
19735
21962
|
const expected = createEvidenceCiArtifacts({
|
|
19736
21963
|
redactionProfile: profile
|
|
19737
21964
|
}).map((a) => a.path);
|
|
@@ -20041,7 +22268,7 @@ var DEFAULT_INJECTION_PATTERNS = [
|
|
|
20041
22268
|
function pass(ruleId, message) {
|
|
20042
22269
|
return { ruleId, status: "pass", severity: "info", message, evidence: [] };
|
|
20043
22270
|
}
|
|
20044
|
-
function
|
|
22271
|
+
function fail4(ruleId, message, evidence, severity = "error") {
|
|
20045
22272
|
return { ruleId, status: severity === "warning" ? "warn" : "fail", severity, message, evidence };
|
|
20046
22273
|
}
|
|
20047
22274
|
function boundedPreview(value, max = 80) {
|
|
@@ -20061,7 +22288,7 @@ function evaluateBannedPhrase(text, options) {
|
|
|
20061
22288
|
if (evidence.length === 0) {
|
|
20062
22289
|
return pass(ruleId, "No banned phrases matched.");
|
|
20063
22290
|
}
|
|
20064
|
-
return
|
|
22291
|
+
return fail4(ruleId, `Matched ${evidence.length} banned phrase(s).`, evidence);
|
|
20065
22292
|
}
|
|
20066
22293
|
var SEVERITY_RANK2 = { info: 0, warning: 1, error: 2 };
|
|
20067
22294
|
function evaluatePiiLeak(value, options = {}) {
|
|
@@ -20080,7 +22307,7 @@ function evaluatePiiLeak(value, options = {}) {
|
|
|
20080
22307
|
detector: finding.detector,
|
|
20081
22308
|
preview: finding.preview
|
|
20082
22309
|
}));
|
|
20083
|
-
return
|
|
22310
|
+
return fail4(ruleId, `Detected ${findings.length} PII-style finding(s).`, evidence);
|
|
20084
22311
|
}
|
|
20085
22312
|
function measureDepth(value) {
|
|
20086
22313
|
if (value === null || typeof value !== "object") return 0;
|
|
@@ -20118,7 +22345,7 @@ function evaluateUnsafeToolArgs(toolName2, toolArgs2, options = {}) {
|
|
|
20118
22345
|
if (evidence.length === 0) {
|
|
20119
22346
|
return pass(ruleId, "Tool arguments within configured bounds.");
|
|
20120
22347
|
}
|
|
20121
|
-
return
|
|
22348
|
+
return fail4(ruleId, "Unsafe or oversized tool arguments detected.", evidence);
|
|
20122
22349
|
}
|
|
20123
22350
|
function evaluatePromptInjection(text, options = {}) {
|
|
20124
22351
|
const ruleId = "guardrail.prompt-injection";
|
|
@@ -20134,7 +22361,7 @@ function evaluatePromptInjection(text, options = {}) {
|
|
|
20134
22361
|
if (evidence.length === 0) {
|
|
20135
22362
|
return pass(ruleId, "No prompt-injection patterns matched.");
|
|
20136
22363
|
}
|
|
20137
|
-
return
|
|
22364
|
+
return fail4(ruleId, `Matched ${evidence.length} injection pattern(s).`, evidence, "warning");
|
|
20138
22365
|
}
|
|
20139
22366
|
function validateSchemaField(value, field, path44, evidence) {
|
|
20140
22367
|
const ruleId = "guardrail.structured-output";
|
|
@@ -20160,7 +22387,7 @@ function validateSchemaField(value, field, path44, evidence) {
|
|
|
20160
22387
|
function evaluateStructuredOutput(value, options) {
|
|
20161
22388
|
const ruleId = "guardrail.structured-output";
|
|
20162
22389
|
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
20163
|
-
return
|
|
22390
|
+
return fail4(ruleId, "Structured output must be an object.", [
|
|
20164
22391
|
{ ruleId, preview: typeof value }
|
|
20165
22392
|
]);
|
|
20166
22393
|
}
|
|
@@ -20172,7 +22399,7 @@ function evaluateStructuredOutput(value, options) {
|
|
|
20172
22399
|
if (evidence.length === 0) {
|
|
20173
22400
|
return pass(ruleId, "Structured output matches schema subset.");
|
|
20174
22401
|
}
|
|
20175
|
-
return
|
|
22402
|
+
return fail4(ruleId, "Structured output schema violation.", evidence);
|
|
20176
22403
|
}
|
|
20177
22404
|
function evaluateOversizeOutput(value, options = {}) {
|
|
20178
22405
|
const ruleId = "guardrail.oversize-output";
|
|
@@ -20181,7 +22408,7 @@ function evaluateOversizeOutput(value, options = {}) {
|
|
|
20181
22408
|
if (text.length <= maxLength) {
|
|
20182
22409
|
return pass(ruleId, "Output within size limits.");
|
|
20183
22410
|
}
|
|
20184
|
-
return
|
|
22411
|
+
return fail4(ruleId, `Output exceeds max length (${text.length} > ${maxLength}).`, [
|
|
20185
22412
|
{ ruleId, preview: `length=${text.length}` }
|
|
20186
22413
|
]);
|
|
20187
22414
|
}
|
|
@@ -20192,11 +22419,11 @@ function evaluateRequiredJsonShape(value, options) {
|
|
|
20192
22419
|
try {
|
|
20193
22420
|
parsed = JSON.parse(value);
|
|
20194
22421
|
} catch {
|
|
20195
|
-
return
|
|
22422
|
+
return fail4(ruleId, "Value is not valid JSON.", [{ ruleId, preview: boundedPreview(value) }]);
|
|
20196
22423
|
}
|
|
20197
22424
|
}
|
|
20198
22425
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
20199
|
-
return
|
|
22426
|
+
return fail4(ruleId, "JSON value must be an object.", [{ ruleId, preview: typeof parsed }]);
|
|
20200
22427
|
}
|
|
20201
22428
|
const record = parsed;
|
|
20202
22429
|
const evidence = [];
|
|
@@ -20208,7 +22435,7 @@ function evaluateRequiredJsonShape(value, options) {
|
|
|
20208
22435
|
if (evidence.length === 0) {
|
|
20209
22436
|
return pass(ruleId, "Required JSON keys present.");
|
|
20210
22437
|
}
|
|
20211
|
-
return
|
|
22438
|
+
return fail4(ruleId, "Missing required JSON keys.", evidence);
|
|
20212
22439
|
}
|
|
20213
22440
|
|
|
20214
22441
|
// packages/guardrails/src/run.ts
|
|
@@ -20656,11 +22883,11 @@ function resolvePreset(preset, context = {}) {
|
|
|
20656
22883
|
}
|
|
20657
22884
|
var CONFIG_EXTENSIONS2 = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
20658
22885
|
var TS_CONFIG_EXTENSIONS2 = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
20659
|
-
function
|
|
22886
|
+
function diagnostic6(code, message, severity = "error") {
|
|
20660
22887
|
return { code, message, severity };
|
|
20661
22888
|
}
|
|
20662
22889
|
function errorResult2(code, message, format = "unknown") {
|
|
20663
|
-
const diagnostics = [
|
|
22890
|
+
const diagnostics = [diagnostic6(code, message)];
|
|
20664
22891
|
return {
|
|
20665
22892
|
ok: false,
|
|
20666
22893
|
status: "error",
|
|
@@ -20693,6 +22920,131 @@ function asStringArray2(value) {
|
|
|
20693
22920
|
return value;
|
|
20694
22921
|
}
|
|
20695
22922
|
var CHECKS_KEYS = /* @__PURE__ */ new Set(["select", "run", "tool", "llm", "structure", "safety"]);
|
|
22923
|
+
var ROOT_CONFIG_KEYS = /* @__PURE__ */ new Set(["checks", "contract"]);
|
|
22924
|
+
var CONTRACT_KEYS = /* @__PURE__ */ new Set([
|
|
22925
|
+
"run",
|
|
22926
|
+
"tools",
|
|
22927
|
+
"llm",
|
|
22928
|
+
"observations",
|
|
22929
|
+
"scope",
|
|
22930
|
+
"alternatives",
|
|
22931
|
+
"controls",
|
|
22932
|
+
"retry"
|
|
22933
|
+
]);
|
|
22934
|
+
var CONTRACT_BODY_KEYS = /* @__PURE__ */ new Set([
|
|
22935
|
+
"run",
|
|
22936
|
+
"tools",
|
|
22937
|
+
"llm",
|
|
22938
|
+
"observations",
|
|
22939
|
+
"controls",
|
|
22940
|
+
"retry"
|
|
22941
|
+
]);
|
|
22942
|
+
var CONTRACT_SCOPE_KEYS = /* @__PURE__ */ new Set([
|
|
22943
|
+
"runId",
|
|
22944
|
+
"subAgentId",
|
|
22945
|
+
"groupId",
|
|
22946
|
+
"workflowStep",
|
|
22947
|
+
"rootEventId"
|
|
22948
|
+
]);
|
|
22949
|
+
var CONTRACT_RUN_KEYS = /* @__PURE__ */ new Set(["requireCompleted", "allowedStatuses", "maxDurationMs"]);
|
|
22950
|
+
var CONTRACT_TOOLS_KEYS = /* @__PURE__ */ new Set([
|
|
22951
|
+
"required",
|
|
22952
|
+
"requiredTools",
|
|
22953
|
+
"forbidden",
|
|
22954
|
+
"forbiddenTools",
|
|
22955
|
+
"allowed",
|
|
22956
|
+
"maxCalls",
|
|
22957
|
+
"requiredOrder",
|
|
22958
|
+
"requiredOrderMode",
|
|
22959
|
+
"arguments",
|
|
22960
|
+
"orderRules",
|
|
22961
|
+
"defaultOccurrenceMode"
|
|
22962
|
+
]);
|
|
22963
|
+
var CONTRACT_LLM_KEYS = /* @__PURE__ */ new Set(["maxCalls", "maxTotalTokens", "allowedModels"]);
|
|
22964
|
+
var CONTRACT_OBSERVATION_KEYS = /* @__PURE__ */ new Set(["required", "failOn", "requireProvenance"]);
|
|
22965
|
+
var CONTRACT_PROVENANCE_KEYS = /* @__PURE__ */ new Set(["method", "evidence", "sameRunEventReference"]);
|
|
22966
|
+
var CONTRACT_CONTROLS_KEYS = /* @__PURE__ */ new Set([
|
|
22967
|
+
"declaredTools",
|
|
22968
|
+
"enforcedTools",
|
|
22969
|
+
"declaredToolsAttribute",
|
|
22970
|
+
"enforcedToolsAttribute",
|
|
22971
|
+
"requireDeclaredMatchesEnforced",
|
|
22972
|
+
"requireObservedWithinEnforced",
|
|
22973
|
+
"requireObservedWithinDeclared",
|
|
22974
|
+
"requiredStages"
|
|
22975
|
+
]);
|
|
22976
|
+
var CONTRACT_CONTROL_STAGES = /* @__PURE__ */ new Set([
|
|
22977
|
+
"declared",
|
|
22978
|
+
"presented",
|
|
22979
|
+
"validated",
|
|
22980
|
+
"enforced",
|
|
22981
|
+
"observed",
|
|
22982
|
+
"accepted"
|
|
22983
|
+
]);
|
|
22984
|
+
var CONTRACT_CONTROL_STAGE_KEYS = /* @__PURE__ */ new Set(["stage", "observation"]);
|
|
22985
|
+
var CONTRACT_RETRY_KEYS = /* @__PURE__ */ new Set([
|
|
22986
|
+
"maxAttempts",
|
|
22987
|
+
"requireTerminalResult",
|
|
22988
|
+
"fallbackOnlyAfterFailure",
|
|
22989
|
+
"nonIdempotentTools",
|
|
22990
|
+
"requireIdempotencyEvidenceForRetry",
|
|
22991
|
+
"requireRecoveredFailureVisible",
|
|
22992
|
+
"operations"
|
|
22993
|
+
]);
|
|
22994
|
+
var CONTRACT_OPERATION_KEYS = /* @__PURE__ */ new Set([
|
|
22995
|
+
"tool",
|
|
22996
|
+
"maxAttempts",
|
|
22997
|
+
"retryableErrors",
|
|
22998
|
+
"requireFailureBeforeRetry",
|
|
22999
|
+
"requireSameArguments",
|
|
23000
|
+
"requireTerminalSuccess",
|
|
23001
|
+
"requireRecoveredFailureVisible",
|
|
23002
|
+
"successfulResultDependency",
|
|
23003
|
+
"sideEffectClass"
|
|
23004
|
+
]);
|
|
23005
|
+
var CONTRACT_RETRYABLE_ERRORS_KEYS = /* @__PURE__ */ new Set(["codes"]);
|
|
23006
|
+
var CONTRACT_RESULT_DEP_KEYS = /* @__PURE__ */ new Set(["consumerKind", "requireExplicitReference"]);
|
|
23007
|
+
var CONTRACT_ALT_KEYS = /* @__PURE__ */ new Set(["anyOf"]);
|
|
23008
|
+
var CONTRACT_ALT_BRANCH_KEYS = /* @__PURE__ */ new Set(["id", "description", "contract"]);
|
|
23009
|
+
var CONTRACT_ORDER_RULE_KEYS = /* @__PURE__ */ new Set([
|
|
23010
|
+
"before",
|
|
23011
|
+
"after",
|
|
23012
|
+
"occurrenceMode",
|
|
23013
|
+
"requireEndpoints"
|
|
23014
|
+
]);
|
|
23015
|
+
var CONTRACT_ARGUMENT_KEYS = /* @__PURE__ */ new Set([
|
|
23016
|
+
"tool",
|
|
23017
|
+
"path",
|
|
23018
|
+
"operator",
|
|
23019
|
+
"occurrence",
|
|
23020
|
+
"expected",
|
|
23021
|
+
"oneOf",
|
|
23022
|
+
"type"
|
|
23023
|
+
]);
|
|
23024
|
+
var CONTRACT_ARGUMENT_OPERATORS = /* @__PURE__ */ new Set(["exists", "type", "equals", "oneOf"]);
|
|
23025
|
+
var CONTRACT_ARGUMENT_OCCURRENCES = /* @__PURE__ */ new Set(["first", "last", "any", "all"]);
|
|
23026
|
+
var CONTRACT_ARGUMENT_TYPES = /* @__PURE__ */ new Set([
|
|
23027
|
+
"string",
|
|
23028
|
+
"number",
|
|
23029
|
+
"boolean",
|
|
23030
|
+
"object",
|
|
23031
|
+
"array",
|
|
23032
|
+
"null"
|
|
23033
|
+
]);
|
|
23034
|
+
var CONTRACT_ORDER_MODES = /* @__PURE__ */ new Set([
|
|
23035
|
+
"first-occurrence",
|
|
23036
|
+
"happens-before",
|
|
23037
|
+
"all-occurrences"
|
|
23038
|
+
]);
|
|
23039
|
+
var CONTRACT_FAIL_ON = /* @__PURE__ */ new Set(["failed", "unknown", "skipped"]);
|
|
23040
|
+
var CONTRACT_SIDE_EFFECT_CLASSES = /* @__PURE__ */ new Set(["read", "write"]);
|
|
23041
|
+
var CONTRACT_MAX_STRING_LIST = 64;
|
|
23042
|
+
var CONTRACT_MAX_ARGUMENTS = 32;
|
|
23043
|
+
var CONTRACT_MAX_ORDER_RULES = 32;
|
|
23044
|
+
var CONTRACT_MAX_ANY_OF = 16;
|
|
23045
|
+
var CONTRACT_MAX_OPERATIONS = 32;
|
|
23046
|
+
var CONTRACT_MAX_REQUIRED_STAGES = 16;
|
|
23047
|
+
var CONTRACT_MAX_ONE_OF = 32;
|
|
20696
23048
|
var RUN_KEYS = /* @__PURE__ */ new Set(["expected", "allowIncomplete", "maxDurationMs", "maxDepth"]);
|
|
20697
23049
|
var TOOL_KEYS = /* @__PURE__ */ new Set(["required", "forbidden", "allowed", "minCount", "maxCount"]);
|
|
20698
23050
|
var LLM_KEYS = /* @__PURE__ */ new Set([
|
|
@@ -20794,7 +23146,50 @@ function requireBoolean(value, pathPrefix) {
|
|
|
20794
23146
|
if (typeof value !== "boolean") {
|
|
20795
23147
|
throw new CheckConfigError(
|
|
20796
23148
|
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
20797
|
-
`${pathPrefix} must be a boolean.`
|
|
23149
|
+
`${pathPrefix} must be a boolean.`
|
|
23150
|
+
);
|
|
23151
|
+
}
|
|
23152
|
+
return value;
|
|
23153
|
+
}
|
|
23154
|
+
function requireNonEmptyString(value, pathPrefix) {
|
|
23155
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
23156
|
+
throw new CheckConfigError(
|
|
23157
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23158
|
+
`${pathPrefix} must be a non-empty string.`
|
|
23159
|
+
);
|
|
23160
|
+
}
|
|
23161
|
+
return value;
|
|
23162
|
+
}
|
|
23163
|
+
function requireBoundedArray(value, pathPrefix, max) {
|
|
23164
|
+
if (!Array.isArray(value)) {
|
|
23165
|
+
throw new CheckConfigError(
|
|
23166
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23167
|
+
`${pathPrefix} must be an array.`
|
|
23168
|
+
);
|
|
23169
|
+
}
|
|
23170
|
+
if (value.length > max) {
|
|
23171
|
+
throw new CheckConfigError(
|
|
23172
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23173
|
+
`${pathPrefix} exceeds bound of ${max} items.`
|
|
23174
|
+
);
|
|
23175
|
+
}
|
|
23176
|
+
return value;
|
|
23177
|
+
}
|
|
23178
|
+
function requireBoundedStringArray(value, pathPrefix, max = CONTRACT_MAX_STRING_LIST) {
|
|
23179
|
+
const items = requireBoundedArray(value, pathPrefix, max);
|
|
23180
|
+
if (items.some((item) => typeof item !== "string" || item.trim() === "")) {
|
|
23181
|
+
throw new CheckConfigError(
|
|
23182
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23183
|
+
`${pathPrefix} must be an array of non-empty strings.`
|
|
23184
|
+
);
|
|
23185
|
+
}
|
|
23186
|
+
return items;
|
|
23187
|
+
}
|
|
23188
|
+
function requireOrderMode(value, pathPrefix) {
|
|
23189
|
+
if (typeof value !== "string" || !CONTRACT_ORDER_MODES.has(value)) {
|
|
23190
|
+
throw new CheckConfigError(
|
|
23191
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23192
|
+
`${pathPrefix} must be one of: "first-occurrence", "happens-before", "all-occurrences".`
|
|
20798
23193
|
);
|
|
20799
23194
|
}
|
|
20800
23195
|
return value;
|
|
@@ -20931,6 +23326,7 @@ function sectionHasEffect(section) {
|
|
|
20931
23326
|
return Object.keys(section).length > 0;
|
|
20932
23327
|
}
|
|
20933
23328
|
function checkConfigHasEffect(config) {
|
|
23329
|
+
if (contractConfigHasEffect(config.contract)) return true;
|
|
20934
23330
|
const checks2 = config.checks;
|
|
20935
23331
|
if (checks2 === void 0) return false;
|
|
20936
23332
|
if ((checks2.select?.length ?? 0) > 0) return true;
|
|
@@ -20941,43 +23337,587 @@ function checkConfigHasEffect(config) {
|
|
|
20941
23337
|
if (sectionHasEffect(checks2.safety)) return true;
|
|
20942
23338
|
return false;
|
|
20943
23339
|
}
|
|
23340
|
+
function contractConfigHasEffect(contract) {
|
|
23341
|
+
if (contract === void 0) return false;
|
|
23342
|
+
if (sectionHasEffect(contract.run)) return true;
|
|
23343
|
+
if (sectionHasEffect(contract.tools)) return true;
|
|
23344
|
+
if (sectionHasEffect(contract.llm)) return true;
|
|
23345
|
+
if (sectionHasEffect(contract.observations)) {
|
|
23346
|
+
return true;
|
|
23347
|
+
}
|
|
23348
|
+
if (sectionHasEffect(contract.controls)) return true;
|
|
23349
|
+
if (sectionHasEffect(contract.retry)) return true;
|
|
23350
|
+
if ((contract.alternatives?.anyOf?.length ?? 0) > 0) return true;
|
|
23351
|
+
return false;
|
|
23352
|
+
}
|
|
23353
|
+
function parseContractRunSection(value, pathPrefix = "contract.run") {
|
|
23354
|
+
const record = requireObject(value, pathPrefix);
|
|
23355
|
+
rejectUnknownKeys(record, CONTRACT_RUN_KEYS, pathPrefix);
|
|
23356
|
+
const out = {};
|
|
23357
|
+
if (record.requireCompleted !== void 0) {
|
|
23358
|
+
out.requireCompleted = requireBoolean(
|
|
23359
|
+
record.requireCompleted,
|
|
23360
|
+
`${pathPrefix}.requireCompleted`
|
|
23361
|
+
);
|
|
23362
|
+
}
|
|
23363
|
+
if (record.allowedStatuses !== void 0) {
|
|
23364
|
+
out.allowedStatuses = requireBoundedStringArray(
|
|
23365
|
+
record.allowedStatuses,
|
|
23366
|
+
`${pathPrefix}.allowedStatuses`
|
|
23367
|
+
);
|
|
23368
|
+
}
|
|
23369
|
+
if (record.maxDurationMs !== void 0) {
|
|
23370
|
+
out.maxDurationMs = requireNonNegativeNumber(
|
|
23371
|
+
record.maxDurationMs,
|
|
23372
|
+
`${pathPrefix}.maxDurationMs`
|
|
23373
|
+
);
|
|
23374
|
+
}
|
|
23375
|
+
return out;
|
|
23376
|
+
}
|
|
23377
|
+
function parseToolArgumentCheck(value, pathPrefix) {
|
|
23378
|
+
const record = requireObject(value, pathPrefix);
|
|
23379
|
+
rejectUnknownKeys(record, CONTRACT_ARGUMENT_KEYS, pathPrefix);
|
|
23380
|
+
const tool = requireNonEmptyString(record.tool, `${pathPrefix}.tool`);
|
|
23381
|
+
if (typeof record.path !== "string") {
|
|
23382
|
+
throw new CheckConfigError(
|
|
23383
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23384
|
+
`${pathPrefix}.path must be a string JSON Pointer ("" or start with "/").`
|
|
23385
|
+
);
|
|
23386
|
+
}
|
|
23387
|
+
const pathValue = record.path;
|
|
23388
|
+
if (typeof record.operator !== "string" || !CONTRACT_ARGUMENT_OPERATORS.has(record.operator)) {
|
|
23389
|
+
throw new CheckConfigError(
|
|
23390
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23391
|
+
`${pathPrefix}.operator must be one of: "exists", "type", "equals", "oneOf".`
|
|
23392
|
+
);
|
|
23393
|
+
}
|
|
23394
|
+
const operator = record.operator;
|
|
23395
|
+
const out = { tool, path: pathValue, operator };
|
|
23396
|
+
if (record.occurrence !== void 0) {
|
|
23397
|
+
if (typeof record.occurrence !== "string" || !CONTRACT_ARGUMENT_OCCURRENCES.has(record.occurrence)) {
|
|
23398
|
+
throw new CheckConfigError(
|
|
23399
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23400
|
+
`${pathPrefix}.occurrence must be one of: "first", "last", "any", "all".`
|
|
23401
|
+
);
|
|
23402
|
+
}
|
|
23403
|
+
out.occurrence = record.occurrence;
|
|
23404
|
+
}
|
|
23405
|
+
if (record.expected !== void 0) {
|
|
23406
|
+
out.expected = record.expected;
|
|
23407
|
+
}
|
|
23408
|
+
if (record.oneOf !== void 0) {
|
|
23409
|
+
out.oneOf = requireBoundedArray(record.oneOf, `${pathPrefix}.oneOf`, CONTRACT_MAX_ONE_OF);
|
|
23410
|
+
}
|
|
23411
|
+
if (record.type !== void 0) {
|
|
23412
|
+
if (typeof record.type !== "string" || !CONTRACT_ARGUMENT_TYPES.has(record.type)) {
|
|
23413
|
+
throw new CheckConfigError(
|
|
23414
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23415
|
+
`${pathPrefix}.type must be one of: "string", "number", "boolean", "object", "array", "null".`
|
|
23416
|
+
);
|
|
23417
|
+
}
|
|
23418
|
+
out.type = record.type;
|
|
23419
|
+
}
|
|
23420
|
+
return out;
|
|
23421
|
+
}
|
|
23422
|
+
function parseOrderRule(value, pathPrefix) {
|
|
23423
|
+
const record = requireObject(value, pathPrefix);
|
|
23424
|
+
rejectUnknownKeys(record, CONTRACT_ORDER_RULE_KEYS, pathPrefix);
|
|
23425
|
+
const out = {
|
|
23426
|
+
before: requireNonEmptyString(record.before, `${pathPrefix}.before`),
|
|
23427
|
+
after: requireNonEmptyString(record.after, `${pathPrefix}.after`)
|
|
23428
|
+
};
|
|
23429
|
+
if (record.occurrenceMode !== void 0) {
|
|
23430
|
+
out.occurrenceMode = requireOrderMode(record.occurrenceMode, `${pathPrefix}.occurrenceMode`);
|
|
23431
|
+
}
|
|
23432
|
+
if (record.requireEndpoints !== void 0) {
|
|
23433
|
+
out.requireEndpoints = requireBoolean(
|
|
23434
|
+
record.requireEndpoints,
|
|
23435
|
+
`${pathPrefix}.requireEndpoints`
|
|
23436
|
+
);
|
|
23437
|
+
}
|
|
23438
|
+
return out;
|
|
23439
|
+
}
|
|
23440
|
+
function parseContractToolsSection(value, pathPrefix = "contract.tools") {
|
|
23441
|
+
const record = requireObject(value, pathPrefix);
|
|
23442
|
+
rejectUnknownKeys(record, CONTRACT_TOOLS_KEYS, pathPrefix);
|
|
23443
|
+
const out = {};
|
|
23444
|
+
if (record.required !== void 0) {
|
|
23445
|
+
out.required = requireBoundedStringArray(record.required, `${pathPrefix}.required`);
|
|
23446
|
+
}
|
|
23447
|
+
if (record.requiredTools !== void 0) {
|
|
23448
|
+
out.requiredTools = requireBoundedStringArray(
|
|
23449
|
+
record.requiredTools,
|
|
23450
|
+
`${pathPrefix}.requiredTools`
|
|
23451
|
+
);
|
|
23452
|
+
}
|
|
23453
|
+
if (record.forbidden !== void 0) {
|
|
23454
|
+
out.forbidden = requireBoundedStringArray(record.forbidden, `${pathPrefix}.forbidden`);
|
|
23455
|
+
}
|
|
23456
|
+
if (record.forbiddenTools !== void 0) {
|
|
23457
|
+
out.forbiddenTools = requireBoundedStringArray(
|
|
23458
|
+
record.forbiddenTools,
|
|
23459
|
+
`${pathPrefix}.forbiddenTools`
|
|
23460
|
+
);
|
|
23461
|
+
}
|
|
23462
|
+
if (record.allowed !== void 0) {
|
|
23463
|
+
out.allowed = requireBoundedStringArray(record.allowed, `${pathPrefix}.allowed`);
|
|
23464
|
+
}
|
|
23465
|
+
if (record.maxCalls !== void 0) {
|
|
23466
|
+
out.maxCalls = requireNonNegativeNumber(record.maxCalls, `${pathPrefix}.maxCalls`);
|
|
23467
|
+
}
|
|
23468
|
+
if (record.requiredOrder !== void 0) {
|
|
23469
|
+
out.requiredOrder = requireBoundedStringArray(
|
|
23470
|
+
record.requiredOrder,
|
|
23471
|
+
`${pathPrefix}.requiredOrder`
|
|
23472
|
+
);
|
|
23473
|
+
}
|
|
23474
|
+
if (record.requiredOrderMode !== void 0) {
|
|
23475
|
+
out.requiredOrderMode = requireOrderMode(
|
|
23476
|
+
record.requiredOrderMode,
|
|
23477
|
+
`${pathPrefix}.requiredOrderMode`
|
|
23478
|
+
);
|
|
23479
|
+
}
|
|
23480
|
+
if (record.defaultOccurrenceMode !== void 0) {
|
|
23481
|
+
out.defaultOccurrenceMode = requireOrderMode(
|
|
23482
|
+
record.defaultOccurrenceMode,
|
|
23483
|
+
`${pathPrefix}.defaultOccurrenceMode`
|
|
23484
|
+
);
|
|
23485
|
+
}
|
|
23486
|
+
if (record.arguments !== void 0) {
|
|
23487
|
+
const items = requireBoundedArray(
|
|
23488
|
+
record.arguments,
|
|
23489
|
+
`${pathPrefix}.arguments`,
|
|
23490
|
+
CONTRACT_MAX_ARGUMENTS
|
|
23491
|
+
);
|
|
23492
|
+
out.arguments = items.map(
|
|
23493
|
+
(item, index) => parseToolArgumentCheck(item, `${pathPrefix}.arguments[${index}]`)
|
|
23494
|
+
);
|
|
23495
|
+
}
|
|
23496
|
+
if (record.orderRules !== void 0) {
|
|
23497
|
+
const items = requireBoundedArray(
|
|
23498
|
+
record.orderRules,
|
|
23499
|
+
`${pathPrefix}.orderRules`,
|
|
23500
|
+
CONTRACT_MAX_ORDER_RULES
|
|
23501
|
+
);
|
|
23502
|
+
out.orderRules = items.map(
|
|
23503
|
+
(item, index) => parseOrderRule(item, `${pathPrefix}.orderRules[${index}]`)
|
|
23504
|
+
);
|
|
23505
|
+
}
|
|
23506
|
+
return out;
|
|
23507
|
+
}
|
|
23508
|
+
function parseContractLlmSection(value, pathPrefix = "contract.llm") {
|
|
23509
|
+
const record = requireObject(value, pathPrefix);
|
|
23510
|
+
rejectUnknownKeys(record, CONTRACT_LLM_KEYS, pathPrefix);
|
|
23511
|
+
const out = {};
|
|
23512
|
+
if (record.maxCalls !== void 0) {
|
|
23513
|
+
out.maxCalls = requireNonNegativeNumber(record.maxCalls, `${pathPrefix}.maxCalls`);
|
|
23514
|
+
}
|
|
23515
|
+
if (record.maxTotalTokens !== void 0) {
|
|
23516
|
+
out.maxTotalTokens = requireNonNegativeNumber(
|
|
23517
|
+
record.maxTotalTokens,
|
|
23518
|
+
`${pathPrefix}.maxTotalTokens`
|
|
23519
|
+
);
|
|
23520
|
+
}
|
|
23521
|
+
if (record.allowedModels !== void 0) {
|
|
23522
|
+
out.allowedModels = requireBoundedStringArray(
|
|
23523
|
+
record.allowedModels,
|
|
23524
|
+
`${pathPrefix}.allowedModels`
|
|
23525
|
+
);
|
|
23526
|
+
}
|
|
23527
|
+
return out;
|
|
23528
|
+
}
|
|
23529
|
+
function parseRequireProvenance(value, pathPrefix) {
|
|
23530
|
+
const record = requireObject(value, pathPrefix);
|
|
23531
|
+
rejectUnknownKeys(record, CONTRACT_PROVENANCE_KEYS, pathPrefix);
|
|
23532
|
+
const out = {};
|
|
23533
|
+
if (record.method !== void 0) {
|
|
23534
|
+
out.method = requireBoolean(record.method, `${pathPrefix}.method`);
|
|
23535
|
+
}
|
|
23536
|
+
if (record.evidence !== void 0) {
|
|
23537
|
+
out.evidence = requireBoolean(record.evidence, `${pathPrefix}.evidence`);
|
|
23538
|
+
}
|
|
23539
|
+
if (record.sameRunEventReference !== void 0) {
|
|
23540
|
+
out.sameRunEventReference = requireBoolean(
|
|
23541
|
+
record.sameRunEventReference,
|
|
23542
|
+
`${pathPrefix}.sameRunEventReference`
|
|
23543
|
+
);
|
|
23544
|
+
}
|
|
23545
|
+
return out;
|
|
23546
|
+
}
|
|
23547
|
+
function parseContractObservationsSection(value, pathPrefix = "contract.observations") {
|
|
23548
|
+
const record = requireObject(value, pathPrefix);
|
|
23549
|
+
rejectUnknownKeys(record, CONTRACT_OBSERVATION_KEYS, pathPrefix);
|
|
23550
|
+
const out = {};
|
|
23551
|
+
if (record.required !== void 0) {
|
|
23552
|
+
out.required = requireBoundedStringArray(record.required, `${pathPrefix}.required`);
|
|
23553
|
+
}
|
|
23554
|
+
if (record.failOn !== void 0) {
|
|
23555
|
+
const items = requireBoundedArray(record.failOn, `${pathPrefix}.failOn`, CONTRACT_MAX_STRING_LIST);
|
|
23556
|
+
for (const item of items) {
|
|
23557
|
+
if (typeof item !== "string" || !CONTRACT_FAIL_ON.has(item)) {
|
|
23558
|
+
throw new CheckConfigError(
|
|
23559
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23560
|
+
`${pathPrefix}.failOn values must be "failed", "unknown", or "skipped".`
|
|
23561
|
+
);
|
|
23562
|
+
}
|
|
23563
|
+
}
|
|
23564
|
+
out.failOn = items;
|
|
23565
|
+
}
|
|
23566
|
+
if (record.requireProvenance !== void 0) {
|
|
23567
|
+
out.requireProvenance = parseRequireProvenance(
|
|
23568
|
+
record.requireProvenance,
|
|
23569
|
+
`${pathPrefix}.requireProvenance`
|
|
23570
|
+
);
|
|
23571
|
+
}
|
|
23572
|
+
return out;
|
|
23573
|
+
}
|
|
23574
|
+
function parseControlRequiredStage(value, pathPrefix) {
|
|
23575
|
+
const record = requireObject(value, pathPrefix);
|
|
23576
|
+
rejectUnknownKeys(record, CONTRACT_CONTROL_STAGE_KEYS, pathPrefix);
|
|
23577
|
+
if (typeof record.stage !== "string" || !CONTRACT_CONTROL_STAGES.has(record.stage)) {
|
|
23578
|
+
throw new CheckConfigError(
|
|
23579
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23580
|
+
`${pathPrefix}.stage must be one of: declared, presented, validated, enforced, observed, accepted.`
|
|
23581
|
+
);
|
|
23582
|
+
}
|
|
23583
|
+
const out = {
|
|
23584
|
+
stage: record.stage
|
|
23585
|
+
};
|
|
23586
|
+
if (record.observation !== void 0) {
|
|
23587
|
+
out.observation = requireNonEmptyString(record.observation, `${pathPrefix}.observation`);
|
|
23588
|
+
}
|
|
23589
|
+
return out;
|
|
23590
|
+
}
|
|
23591
|
+
function parseContractControlsSection(value, pathPrefix = "contract.controls") {
|
|
23592
|
+
const record = requireObject(value, pathPrefix);
|
|
23593
|
+
rejectUnknownKeys(record, CONTRACT_CONTROLS_KEYS, pathPrefix);
|
|
23594
|
+
const out = {};
|
|
23595
|
+
if (record.declaredTools !== void 0) {
|
|
23596
|
+
out.declaredTools = requireBoundedStringArray(
|
|
23597
|
+
record.declaredTools,
|
|
23598
|
+
`${pathPrefix}.declaredTools`
|
|
23599
|
+
);
|
|
23600
|
+
}
|
|
23601
|
+
if (record.enforcedTools !== void 0) {
|
|
23602
|
+
out.enforcedTools = requireBoundedStringArray(
|
|
23603
|
+
record.enforcedTools,
|
|
23604
|
+
`${pathPrefix}.enforcedTools`
|
|
23605
|
+
);
|
|
23606
|
+
}
|
|
23607
|
+
if (record.declaredToolsAttribute !== void 0) {
|
|
23608
|
+
out.declaredToolsAttribute = requireNonEmptyString(
|
|
23609
|
+
record.declaredToolsAttribute,
|
|
23610
|
+
`${pathPrefix}.declaredToolsAttribute`
|
|
23611
|
+
);
|
|
23612
|
+
}
|
|
23613
|
+
if (record.enforcedToolsAttribute !== void 0) {
|
|
23614
|
+
out.enforcedToolsAttribute = requireNonEmptyString(
|
|
23615
|
+
record.enforcedToolsAttribute,
|
|
23616
|
+
`${pathPrefix}.enforcedToolsAttribute`
|
|
23617
|
+
);
|
|
23618
|
+
}
|
|
23619
|
+
if (record.requireDeclaredMatchesEnforced !== void 0) {
|
|
23620
|
+
out.requireDeclaredMatchesEnforced = requireBoolean(
|
|
23621
|
+
record.requireDeclaredMatchesEnforced,
|
|
23622
|
+
`${pathPrefix}.requireDeclaredMatchesEnforced`
|
|
23623
|
+
);
|
|
23624
|
+
}
|
|
23625
|
+
if (record.requireObservedWithinEnforced !== void 0) {
|
|
23626
|
+
out.requireObservedWithinEnforced = requireBoolean(
|
|
23627
|
+
record.requireObservedWithinEnforced,
|
|
23628
|
+
`${pathPrefix}.requireObservedWithinEnforced`
|
|
23629
|
+
);
|
|
23630
|
+
}
|
|
23631
|
+
if (record.requireObservedWithinDeclared !== void 0) {
|
|
23632
|
+
out.requireObservedWithinDeclared = requireBoolean(
|
|
23633
|
+
record.requireObservedWithinDeclared,
|
|
23634
|
+
`${pathPrefix}.requireObservedWithinDeclared`
|
|
23635
|
+
);
|
|
23636
|
+
}
|
|
23637
|
+
if (record.requiredStages !== void 0) {
|
|
23638
|
+
const items = requireBoundedArray(
|
|
23639
|
+
record.requiredStages,
|
|
23640
|
+
`${pathPrefix}.requiredStages`,
|
|
23641
|
+
CONTRACT_MAX_REQUIRED_STAGES
|
|
23642
|
+
);
|
|
23643
|
+
out.requiredStages = items.map(
|
|
23644
|
+
(item, index) => parseControlRequiredStage(item, `${pathPrefix}.requiredStages[${index}]`)
|
|
23645
|
+
);
|
|
23646
|
+
}
|
|
23647
|
+
return out;
|
|
23648
|
+
}
|
|
23649
|
+
function parseRecoveryOperation(value, pathPrefix) {
|
|
23650
|
+
const record = requireObject(value, pathPrefix);
|
|
23651
|
+
rejectUnknownKeys(record, CONTRACT_OPERATION_KEYS, pathPrefix);
|
|
23652
|
+
const out = {
|
|
23653
|
+
tool: requireNonEmptyString(record.tool, `${pathPrefix}.tool`)
|
|
23654
|
+
};
|
|
23655
|
+
if (record.maxAttempts !== void 0) {
|
|
23656
|
+
out.maxAttempts = requireNonNegativeNumber(record.maxAttempts, `${pathPrefix}.maxAttempts`);
|
|
23657
|
+
}
|
|
23658
|
+
if (record.retryableErrors !== void 0) {
|
|
23659
|
+
const errors = requireObject(record.retryableErrors, `${pathPrefix}.retryableErrors`);
|
|
23660
|
+
rejectUnknownKeys(errors, CONTRACT_RETRYABLE_ERRORS_KEYS, `${pathPrefix}.retryableErrors`);
|
|
23661
|
+
const retryableErrors = {};
|
|
23662
|
+
if (errors.codes !== void 0) {
|
|
23663
|
+
retryableErrors.codes = requireBoundedStringArray(
|
|
23664
|
+
errors.codes,
|
|
23665
|
+
`${pathPrefix}.retryableErrors.codes`
|
|
23666
|
+
);
|
|
23667
|
+
}
|
|
23668
|
+
out.retryableErrors = retryableErrors;
|
|
23669
|
+
}
|
|
23670
|
+
if (record.requireFailureBeforeRetry !== void 0) {
|
|
23671
|
+
out.requireFailureBeforeRetry = requireBoolean(
|
|
23672
|
+
record.requireFailureBeforeRetry,
|
|
23673
|
+
`${pathPrefix}.requireFailureBeforeRetry`
|
|
23674
|
+
);
|
|
23675
|
+
}
|
|
23676
|
+
if (record.requireSameArguments !== void 0) {
|
|
23677
|
+
if (record.requireSameArguments === true || record.requireSameArguments === false) {
|
|
23678
|
+
out.requireSameArguments = record.requireSameArguments;
|
|
23679
|
+
} else if (record.requireSameArguments === "structured-or-digest") {
|
|
23680
|
+
out.requireSameArguments = "structured-or-digest";
|
|
23681
|
+
} else {
|
|
23682
|
+
throw new CheckConfigError(
|
|
23683
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23684
|
+
`${pathPrefix}.requireSameArguments must be a boolean or "structured-or-digest".`
|
|
23685
|
+
);
|
|
23686
|
+
}
|
|
23687
|
+
}
|
|
23688
|
+
if (record.requireTerminalSuccess !== void 0) {
|
|
23689
|
+
out.requireTerminalSuccess = requireBoolean(
|
|
23690
|
+
record.requireTerminalSuccess,
|
|
23691
|
+
`${pathPrefix}.requireTerminalSuccess`
|
|
23692
|
+
);
|
|
23693
|
+
}
|
|
23694
|
+
if (record.requireRecoveredFailureVisible !== void 0) {
|
|
23695
|
+
out.requireRecoveredFailureVisible = requireBoolean(
|
|
23696
|
+
record.requireRecoveredFailureVisible,
|
|
23697
|
+
`${pathPrefix}.requireRecoveredFailureVisible`
|
|
23698
|
+
);
|
|
23699
|
+
}
|
|
23700
|
+
if (record.successfulResultDependency !== void 0) {
|
|
23701
|
+
const dep = requireObject(
|
|
23702
|
+
record.successfulResultDependency,
|
|
23703
|
+
`${pathPrefix}.successfulResultDependency`
|
|
23704
|
+
);
|
|
23705
|
+
rejectUnknownKeys(dep, CONTRACT_RESULT_DEP_KEYS, `${pathPrefix}.successfulResultDependency`);
|
|
23706
|
+
if (dep.consumerKind !== "LLM") {
|
|
23707
|
+
throw new CheckConfigError(
|
|
23708
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23709
|
+
`${pathPrefix}.successfulResultDependency.consumerKind must be "LLM".`
|
|
23710
|
+
);
|
|
23711
|
+
}
|
|
23712
|
+
const dependency = { consumerKind: "LLM" };
|
|
23713
|
+
if (dep.requireExplicitReference !== void 0) {
|
|
23714
|
+
dependency.requireExplicitReference = requireBoolean(
|
|
23715
|
+
dep.requireExplicitReference,
|
|
23716
|
+
`${pathPrefix}.successfulResultDependency.requireExplicitReference`
|
|
23717
|
+
);
|
|
23718
|
+
}
|
|
23719
|
+
out.successfulResultDependency = dependency;
|
|
23720
|
+
}
|
|
23721
|
+
if (record.sideEffectClass !== void 0) {
|
|
23722
|
+
if (typeof record.sideEffectClass !== "string" || !CONTRACT_SIDE_EFFECT_CLASSES.has(record.sideEffectClass)) {
|
|
23723
|
+
throw new CheckConfigError(
|
|
23724
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23725
|
+
`${pathPrefix}.sideEffectClass must be "read" or "write".`
|
|
23726
|
+
);
|
|
23727
|
+
}
|
|
23728
|
+
out.sideEffectClass = record.sideEffectClass;
|
|
23729
|
+
}
|
|
23730
|
+
return out;
|
|
23731
|
+
}
|
|
23732
|
+
function parseContractRetrySection(value, pathPrefix = "contract.retry") {
|
|
23733
|
+
const record = requireObject(value, pathPrefix);
|
|
23734
|
+
rejectUnknownKeys(record, CONTRACT_RETRY_KEYS, pathPrefix);
|
|
23735
|
+
const out = {};
|
|
23736
|
+
if (record.maxAttempts !== void 0) {
|
|
23737
|
+
out.maxAttempts = requireNonNegativeNumber(record.maxAttempts, `${pathPrefix}.maxAttempts`);
|
|
23738
|
+
}
|
|
23739
|
+
if (record.requireTerminalResult !== void 0) {
|
|
23740
|
+
out.requireTerminalResult = requireBoolean(
|
|
23741
|
+
record.requireTerminalResult,
|
|
23742
|
+
`${pathPrefix}.requireTerminalResult`
|
|
23743
|
+
);
|
|
23744
|
+
}
|
|
23745
|
+
if (record.fallbackOnlyAfterFailure !== void 0) {
|
|
23746
|
+
out.fallbackOnlyAfterFailure = requireBoolean(
|
|
23747
|
+
record.fallbackOnlyAfterFailure,
|
|
23748
|
+
`${pathPrefix}.fallbackOnlyAfterFailure`
|
|
23749
|
+
);
|
|
23750
|
+
}
|
|
23751
|
+
if (record.nonIdempotentTools !== void 0) {
|
|
23752
|
+
out.nonIdempotentTools = requireBoundedStringArray(
|
|
23753
|
+
record.nonIdempotentTools,
|
|
23754
|
+
`${pathPrefix}.nonIdempotentTools`
|
|
23755
|
+
);
|
|
23756
|
+
}
|
|
23757
|
+
if (record.requireIdempotencyEvidenceForRetry !== void 0) {
|
|
23758
|
+
out.requireIdempotencyEvidenceForRetry = requireBoolean(
|
|
23759
|
+
record.requireIdempotencyEvidenceForRetry,
|
|
23760
|
+
`${pathPrefix}.requireIdempotencyEvidenceForRetry`
|
|
23761
|
+
);
|
|
23762
|
+
}
|
|
23763
|
+
if (record.requireRecoveredFailureVisible !== void 0) {
|
|
23764
|
+
out.requireRecoveredFailureVisible = requireBoolean(
|
|
23765
|
+
record.requireRecoveredFailureVisible,
|
|
23766
|
+
`${pathPrefix}.requireRecoveredFailureVisible`
|
|
23767
|
+
);
|
|
23768
|
+
}
|
|
23769
|
+
if (record.operations !== void 0) {
|
|
23770
|
+
const items = requireBoundedArray(
|
|
23771
|
+
record.operations,
|
|
23772
|
+
`${pathPrefix}.operations`,
|
|
23773
|
+
CONTRACT_MAX_OPERATIONS
|
|
23774
|
+
);
|
|
23775
|
+
out.operations = items.map(
|
|
23776
|
+
(item, index) => parseRecoveryOperation(item, `${pathPrefix}.operations[${index}]`)
|
|
23777
|
+
);
|
|
23778
|
+
}
|
|
23779
|
+
return out;
|
|
23780
|
+
}
|
|
23781
|
+
function parseContractScopeSection(value) {
|
|
23782
|
+
const record = requireObject(value, "contract.scope");
|
|
23783
|
+
rejectUnknownKeys(record, CONTRACT_SCOPE_KEYS, "contract.scope");
|
|
23784
|
+
const out = {};
|
|
23785
|
+
if (record.runId !== void 0) {
|
|
23786
|
+
out.runId = requireNonEmptyString(record.runId, "contract.scope.runId");
|
|
23787
|
+
}
|
|
23788
|
+
if (record.subAgentId !== void 0) {
|
|
23789
|
+
out.subAgentId = requireNonEmptyString(record.subAgentId, "contract.scope.subAgentId");
|
|
23790
|
+
}
|
|
23791
|
+
if (record.groupId !== void 0) {
|
|
23792
|
+
out.groupId = requireNonEmptyString(record.groupId, "contract.scope.groupId");
|
|
23793
|
+
}
|
|
23794
|
+
if (record.workflowStep !== void 0) {
|
|
23795
|
+
out.workflowStep = requireNonEmptyString(
|
|
23796
|
+
record.workflowStep,
|
|
23797
|
+
"contract.scope.workflowStep"
|
|
23798
|
+
);
|
|
23799
|
+
}
|
|
23800
|
+
if (record.rootEventId !== void 0) {
|
|
23801
|
+
out.rootEventId = requireNonEmptyString(record.rootEventId, "contract.scope.rootEventId");
|
|
23802
|
+
}
|
|
23803
|
+
return out;
|
|
23804
|
+
}
|
|
23805
|
+
function parseContractBodySection(value, pathPrefix) {
|
|
23806
|
+
const record = requireObject(value, pathPrefix);
|
|
23807
|
+
rejectUnknownKeys(record, CONTRACT_BODY_KEYS, pathPrefix);
|
|
23808
|
+
const out = {};
|
|
23809
|
+
if (record.run !== void 0) out.run = parseContractRunSection(record.run, `${pathPrefix}.run`);
|
|
23810
|
+
if (record.tools !== void 0) {
|
|
23811
|
+
out.tools = parseContractToolsSection(record.tools, `${pathPrefix}.tools`);
|
|
23812
|
+
}
|
|
23813
|
+
if (record.llm !== void 0) out.llm = parseContractLlmSection(record.llm, `${pathPrefix}.llm`);
|
|
23814
|
+
if (record.observations !== void 0) {
|
|
23815
|
+
out.observations = parseContractObservationsSection(
|
|
23816
|
+
record.observations,
|
|
23817
|
+
`${pathPrefix}.observations`
|
|
23818
|
+
);
|
|
23819
|
+
}
|
|
23820
|
+
if (record.controls !== void 0) {
|
|
23821
|
+
out.controls = parseContractControlsSection(record.controls, `${pathPrefix}.controls`);
|
|
23822
|
+
}
|
|
23823
|
+
if (record.retry !== void 0) {
|
|
23824
|
+
out.retry = parseContractRetrySection(record.retry, `${pathPrefix}.retry`);
|
|
23825
|
+
}
|
|
23826
|
+
return out;
|
|
23827
|
+
}
|
|
23828
|
+
function parseContractAlternativesSection(value) {
|
|
23829
|
+
const record = requireObject(value, "contract.alternatives");
|
|
23830
|
+
rejectUnknownKeys(record, CONTRACT_ALT_KEYS, "contract.alternatives");
|
|
23831
|
+
if (record.anyOf === void 0) {
|
|
23832
|
+
throw new CheckConfigError(
|
|
23833
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23834
|
+
"contract.alternatives.anyOf is required."
|
|
23835
|
+
);
|
|
23836
|
+
}
|
|
23837
|
+
const items = requireBoundedArray(
|
|
23838
|
+
record.anyOf,
|
|
23839
|
+
"contract.alternatives.anyOf",
|
|
23840
|
+
CONTRACT_MAX_ANY_OF
|
|
23841
|
+
);
|
|
23842
|
+
const anyOf = items.map((item, index) => {
|
|
23843
|
+
const pathPrefix = `contract.alternatives.anyOf[${index}]`;
|
|
23844
|
+
const branch = requireObject(item, pathPrefix);
|
|
23845
|
+
rejectUnknownKeys(branch, CONTRACT_ALT_BRANCH_KEYS, pathPrefix);
|
|
23846
|
+
const id = requireNonEmptyString(branch.id, `${pathPrefix}.id`);
|
|
23847
|
+
if (branch.contract === void 0) {
|
|
23848
|
+
throw new CheckConfigError(
|
|
23849
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23850
|
+
`${pathPrefix}.contract is required.`
|
|
23851
|
+
);
|
|
23852
|
+
}
|
|
23853
|
+
const out = {
|
|
23854
|
+
id,
|
|
23855
|
+
contract: parseContractBodySection(branch.contract, `${pathPrefix}.contract`)
|
|
23856
|
+
};
|
|
23857
|
+
if (branch.description !== void 0) {
|
|
23858
|
+
out.description = requireNonEmptyString(branch.description, `${pathPrefix}.description`);
|
|
23859
|
+
}
|
|
23860
|
+
return out;
|
|
23861
|
+
});
|
|
23862
|
+
return { anyOf };
|
|
23863
|
+
}
|
|
23864
|
+
function parseContractSection(value) {
|
|
23865
|
+
const record = requireObject(value, "contract");
|
|
23866
|
+
rejectUnknownKeys(record, CONTRACT_KEYS, "contract");
|
|
23867
|
+
const bodyOnly = {};
|
|
23868
|
+
for (const key of Object.keys(record)) {
|
|
23869
|
+
if (CONTRACT_BODY_KEYS.has(key)) bodyOnly[key] = record[key];
|
|
23870
|
+
}
|
|
23871
|
+
const out = { ...parseContractBodySection(bodyOnly, "contract") };
|
|
23872
|
+
if (record.scope !== void 0) out.scope = parseContractScopeSection(record.scope);
|
|
23873
|
+
if (record.alternatives !== void 0) {
|
|
23874
|
+
out.alternatives = parseContractAlternativesSection(record.alternatives);
|
|
23875
|
+
}
|
|
23876
|
+
return out;
|
|
23877
|
+
}
|
|
20944
23878
|
function parseCheckConfig(value) {
|
|
20945
23879
|
if (value === void 0 || value === null) return {};
|
|
20946
23880
|
if (typeof value !== "object" || Array.isArray(value)) {
|
|
20947
23881
|
throw new CheckConfigError("AI_CHECK_CONFIG_INVALID_VALUE", "Config must export an object.");
|
|
20948
23882
|
}
|
|
20949
23883
|
const root = value;
|
|
20950
|
-
if ("contract" in root) {
|
|
20951
|
-
throw new CheckConfigError(
|
|
20952
|
-
"AI_CHECK_CONFIG_UNKNOWN_KEY",
|
|
20953
|
-
'Top-level "contract" is not supported by `agent-inspect check --config`. Use the TraceContract TypeScript API (`defineTraceContract` / `evaluateTraceContract` from `agent-inspect/checks`), or express tool/run/llm rules under `checks`.'
|
|
20954
|
-
);
|
|
20955
|
-
}
|
|
20956
23884
|
const rootKeys = Object.keys(root);
|
|
20957
23885
|
for (const key of rootKeys) {
|
|
20958
|
-
if (key
|
|
20959
|
-
const suggestion = closestKey(key, [
|
|
23886
|
+
if (ROOT_CONFIG_KEYS.has(key)) continue;
|
|
23887
|
+
const suggestion = closestKey(key, [...ROOT_CONFIG_KEYS]);
|
|
20960
23888
|
const hint = suggestion ? ` Did you mean "${suggestion}"?` : "";
|
|
20961
23889
|
throw new CheckConfigError(
|
|
20962
23890
|
"AI_CHECK_CONFIG_UNKNOWN_KEY",
|
|
20963
|
-
`Unknown top-level check config key "${key}". Allowed: checks.${hint}`
|
|
23891
|
+
`Unknown top-level check config key "${key}". Allowed: checks, contract.${hint}`
|
|
20964
23892
|
);
|
|
20965
23893
|
}
|
|
20966
|
-
|
|
20967
|
-
|
|
20968
|
-
|
|
20969
|
-
|
|
20970
|
-
|
|
20971
|
-
|
|
23894
|
+
const out = {};
|
|
23895
|
+
if ("checks" in root) {
|
|
23896
|
+
const checksRecord = requireObject(root.checks, "checks");
|
|
23897
|
+
rejectUnknownKeys(checksRecord, CHECKS_KEYS, "checks");
|
|
23898
|
+
const checks2 = {};
|
|
23899
|
+
if (checksRecord.select !== void 0) {
|
|
23900
|
+
checks2.select = requireStringArray(checksRecord.select, "checks.select");
|
|
23901
|
+
}
|
|
23902
|
+
if (checksRecord.run !== void 0) checks2.run = parseRunSection(checksRecord.run);
|
|
23903
|
+
if (checksRecord.tool !== void 0) checks2.tool = parseToolSection(checksRecord.tool);
|
|
23904
|
+
if (checksRecord.llm !== void 0) checks2.llm = parseLlmSection(checksRecord.llm);
|
|
23905
|
+
if (checksRecord.structure !== void 0) {
|
|
23906
|
+
checks2.structure = parseStructureSection(checksRecord.structure);
|
|
23907
|
+
}
|
|
23908
|
+
if (checksRecord.safety !== void 0) checks2.safety = parseSafetySection(checksRecord.safety);
|
|
23909
|
+
out.checks = checks2;
|
|
23910
|
+
}
|
|
23911
|
+
if ("contract" in root) {
|
|
23912
|
+
out.contract = parseContractSection(root.contract);
|
|
20972
23913
|
}
|
|
20973
|
-
if (
|
|
20974
|
-
|
|
20975
|
-
|
|
20976
|
-
|
|
20977
|
-
|
|
23914
|
+
if (out.contract !== void 0 && contractConfigHasEffect(out.contract) && checkConfigHasEffect({ checks: out.checks })) {
|
|
23915
|
+
throw new CheckConfigError(
|
|
23916
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23917
|
+
'Config cannot combine top-level "checks" and "contract". Use one vocabulary per config file.'
|
|
23918
|
+
);
|
|
20978
23919
|
}
|
|
20979
|
-
|
|
20980
|
-
return { checks: checks2 };
|
|
23920
|
+
return out;
|
|
20981
23921
|
}
|
|
20982
23922
|
async function loadConfig(configPath) {
|
|
20983
23923
|
if (configPath === void 0) return {};
|
|
@@ -21101,7 +24041,7 @@ function buildRules(config, options, presetSelect = []) {
|
|
|
21101
24041
|
} catch (error) {
|
|
21102
24042
|
const message = error instanceof Error ? error.message : String(error);
|
|
21103
24043
|
diagnostics.push(
|
|
21104
|
-
|
|
24044
|
+
diagnostic6("AI_CHECK_INVALID_ARGUMENTS", `--max-step-duration: ${message}`)
|
|
21105
24045
|
);
|
|
21106
24046
|
}
|
|
21107
24047
|
}
|
|
@@ -21136,7 +24076,7 @@ function buildRules(config, options, presetSelect = []) {
|
|
|
21136
24076
|
} catch (error) {
|
|
21137
24077
|
const message = error instanceof Error ? error.message : String(error);
|
|
21138
24078
|
diagnostics.push(
|
|
21139
|
-
|
|
24079
|
+
diagnostic6("AI_CHECK_INVALID_ARGUMENTS", `--fail-on-observation: ${message}`)
|
|
21140
24080
|
);
|
|
21141
24081
|
}
|
|
21142
24082
|
}
|
|
@@ -21278,8 +24218,8 @@ function printHuman2(result, options = {}) {
|
|
|
21278
24218
|
`Summary: ${result.summary.failed} failed, ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
21279
24219
|
);
|
|
21280
24220
|
console.log(`Rules evaluated: ${result.summary.rulesEvaluated ?? result.ruleExecutions?.length ?? 0}`);
|
|
21281
|
-
for (const
|
|
21282
|
-
console.log(`- ${
|
|
24221
|
+
for (const diagnostic8 of result.diagnostics) {
|
|
24222
|
+
console.log(`- ${diagnostic8.code}: ${diagnostic8.message}`);
|
|
21283
24223
|
}
|
|
21284
24224
|
for (const finding of result.findings) {
|
|
21285
24225
|
const path44 = finding.evidence[0]?.path;
|
|
@@ -21305,6 +24245,7 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21305
24245
|
let evidenceRead;
|
|
21306
24246
|
let evidenceRunIds = [];
|
|
21307
24247
|
let evidenceSourceContents;
|
|
24248
|
+
let evidenceContract;
|
|
21308
24249
|
const sessionId = options.session?.trim();
|
|
21309
24250
|
const groupId = options.group?.trim();
|
|
21310
24251
|
const useSessionScope = Boolean(sessionId || groupId);
|
|
@@ -21313,7 +24254,7 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21313
24254
|
if (options.config !== void 0 && !checkConfigHasEffect(config)) {
|
|
21314
24255
|
result = errorResult2(
|
|
21315
24256
|
"AI_CHECK_CONFIG_NO_EFFECTIVE_RULES",
|
|
21316
|
-
"Explicit --config has no effective check rules. Configure checks.select, checks.run, checks.tool, checks.llm, checks.structure,
|
|
24257
|
+
"Explicit --config has no effective check rules. Configure checks.select, checks.run, checks.tool, checks.llm, checks.structure, checks.safety, or contract."
|
|
21317
24258
|
);
|
|
21318
24259
|
if (options.json) printJson2(result);
|
|
21319
24260
|
else printHuman2(result, options);
|
|
@@ -21321,7 +24262,11 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21321
24262
|
return;
|
|
21322
24263
|
}
|
|
21323
24264
|
let effectiveOptions = options;
|
|
21324
|
-
const
|
|
24265
|
+
const useContract = config.contract !== void 0 && contractConfigHasEffect(config.contract);
|
|
24266
|
+
if (useContract) {
|
|
24267
|
+
evidenceContract = defineTraceContract(config.contract);
|
|
24268
|
+
}
|
|
24269
|
+
const resolved = useContract ? void 0 : resolvePreset(options.preset, {
|
|
21325
24270
|
hasToolRules: hasToolRulesConfigured(config, options)
|
|
21326
24271
|
});
|
|
21327
24272
|
if (resolved !== void 0) {
|
|
@@ -21329,7 +24274,7 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21329
24274
|
config = applied.config;
|
|
21330
24275
|
effectiveOptions = applied.options;
|
|
21331
24276
|
}
|
|
21332
|
-
const built = buildRules(config, effectiveOptions, resolved?.select ?? []);
|
|
24277
|
+
const built = useContract ? { rules: [], select: [], diagnostics: [] } : buildRules(config, effectiveOptions, resolved?.select ?? []);
|
|
21333
24278
|
if (built.diagnostics.some((item) => item.severity === "error")) {
|
|
21334
24279
|
result = errorResult2("AI_CHECK_INVALID_CONFIG", "Invalid check configuration.");
|
|
21335
24280
|
result.diagnostics = [...built.diagnostics];
|
|
@@ -21351,6 +24296,7 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21351
24296
|
});
|
|
21352
24297
|
const perRun = [];
|
|
21353
24298
|
const sourceContents = /* @__PURE__ */ new Map();
|
|
24299
|
+
const definedContract = evidenceContract;
|
|
21354
24300
|
for (const meta2 of scoped.metas) {
|
|
21355
24301
|
const read = await openTrace(
|
|
21356
24302
|
{ type: "file", path: meta2.filePath },
|
|
@@ -21367,23 +24313,20 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21367
24313
|
`
|
|
21368
24314
|
);
|
|
21369
24315
|
}
|
|
24316
|
+
const base = definedContract ? evaluateTraceContract({ read }, definedContract, { runId: meta2.runId }) : runTraceChecks(
|
|
24317
|
+
{ read },
|
|
24318
|
+
{
|
|
24319
|
+
rules: built.rules,
|
|
24320
|
+
select: built.select,
|
|
24321
|
+
runId: meta2.runId
|
|
24322
|
+
}
|
|
24323
|
+
);
|
|
21370
24324
|
perRun.push(
|
|
21371
|
-
mergeSafetyExtensions(
|
|
21372
|
-
|
|
21373
|
-
|
|
21374
|
-
|
|
21375
|
-
|
|
21376
|
-
select: built.select,
|
|
21377
|
-
runId: meta2.runId
|
|
21378
|
-
}
|
|
21379
|
-
),
|
|
21380
|
-
read,
|
|
21381
|
-
{
|
|
21382
|
-
...options.guardrails ? { guardrails: options.guardrails } : {},
|
|
21383
|
-
...options.circuit ? { circuits: options.circuit } : {},
|
|
21384
|
-
runId: meta2.runId
|
|
21385
|
-
}
|
|
21386
|
-
)
|
|
24325
|
+
mergeSafetyExtensions(base, read, {
|
|
24326
|
+
...options.guardrails ? { guardrails: options.guardrails } : {},
|
|
24327
|
+
...options.circuit ? { circuits: options.circuit } : {},
|
|
24328
|
+
runId: meta2.runId
|
|
24329
|
+
})
|
|
21387
24330
|
);
|
|
21388
24331
|
}
|
|
21389
24332
|
evidenceRunIds = scoped.runIds;
|
|
@@ -21422,22 +24365,25 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21422
24365
|
])
|
|
21423
24366
|
);
|
|
21424
24367
|
}
|
|
21425
|
-
|
|
21426
|
-
|
|
21427
|
-
|
|
21428
|
-
|
|
21429
|
-
|
|
21430
|
-
|
|
21431
|
-
|
|
21432
|
-
|
|
21433
|
-
),
|
|
21434
|
-
read,
|
|
24368
|
+
const base = useContract && evidenceContract !== void 0 ? evaluateTraceContract(
|
|
24369
|
+
{ read },
|
|
24370
|
+
evidenceContract,
|
|
24371
|
+
{
|
|
24372
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
24373
|
+
}
|
|
24374
|
+
) : runTraceChecks(
|
|
24375
|
+
{ read },
|
|
21435
24376
|
{
|
|
21436
|
-
|
|
21437
|
-
|
|
24377
|
+
rules: built.rules,
|
|
24378
|
+
select: built.select,
|
|
21438
24379
|
...options.run !== void 0 ? { runId: options.run } : {}
|
|
21439
24380
|
}
|
|
21440
24381
|
);
|
|
24382
|
+
result = mergeSafetyExtensions(base, read, {
|
|
24383
|
+
...options.guardrails ? { guardrails: options.guardrails } : {},
|
|
24384
|
+
...options.circuit ? { circuits: options.circuit } : {},
|
|
24385
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
24386
|
+
});
|
|
21441
24387
|
}
|
|
21442
24388
|
} catch (error) {
|
|
21443
24389
|
if (phase === "config") {
|
|
@@ -21481,7 +24427,15 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21481
24427
|
checkResultsJson: checkResultToEvidenceJson(result, runIds),
|
|
21482
24428
|
summaryText: `Check status: ${result.status}`,
|
|
21483
24429
|
redactionProfile,
|
|
21484
|
-
format: evidenceFormat
|
|
24430
|
+
format: evidenceFormat,
|
|
24431
|
+
...evidenceContract !== void 0 ? {
|
|
24432
|
+
contractPackage: {
|
|
24433
|
+
engineVersion: "",
|
|
24434
|
+
source: "file",
|
|
24435
|
+
contract: evidenceContract,
|
|
24436
|
+
...options.config !== void 0 ? { path: options.config } : {}
|
|
24437
|
+
}
|
|
24438
|
+
} : {}
|
|
21485
24439
|
});
|
|
21486
24440
|
if (!options.json) {
|
|
21487
24441
|
console.log(`Evidence: ${written}`);
|
|
@@ -22797,7 +25751,7 @@ function evidenceForEvent(event, path44) {
|
|
|
22797
25751
|
}
|
|
22798
25752
|
];
|
|
22799
25753
|
}
|
|
22800
|
-
function
|
|
25754
|
+
function fail5(ruleId, message, evidence, expected, actual) {
|
|
22801
25755
|
return {
|
|
22802
25756
|
ruleId,
|
|
22803
25757
|
status: "fail",
|
|
@@ -22993,7 +25947,7 @@ var checks = {
|
|
|
22993
25947
|
"eval.requireSuccess",
|
|
22994
25948
|
"run",
|
|
22995
25949
|
(context) => context.run.status === "ok" ? [] : [
|
|
22996
|
-
|
|
25950
|
+
fail5(
|
|
22997
25951
|
"eval.requireSuccess",
|
|
22998
25952
|
"Run did not complete successfully.",
|
|
22999
25953
|
evidenceForRun(context.run, "status"),
|
|
@@ -23008,7 +25962,7 @@ var checks = {
|
|
|
23008
25962
|
return createRule("eval.requiredTools", "tool", (context) => {
|
|
23009
25963
|
const tools = new Set(nodeNames(context.nodes, "TOOL"));
|
|
23010
25964
|
return expected.filter((name) => !tools.has(name)).map(
|
|
23011
|
-
(name) =>
|
|
25965
|
+
(name) => fail5(
|
|
23012
25966
|
"eval.requiredTools",
|
|
23013
25967
|
`Required tool ${name} did not appear.`,
|
|
23014
25968
|
evidenceForRun(context.run, "children"),
|
|
@@ -23024,7 +25978,7 @@ var checks = {
|
|
|
23024
25978
|
"eval.forbiddenTools",
|
|
23025
25979
|
"tool",
|
|
23026
25980
|
(context) => context.nodes.filter((node) => node.event.kind === "TOOL" && blocked.includes(node.event.name)).map(
|
|
23027
|
-
(node) =>
|
|
25981
|
+
(node) => fail5(
|
|
23028
25982
|
"eval.forbiddenTools",
|
|
23029
25983
|
`Forbidden tool ${node.event.name} appeared.`,
|
|
23030
25984
|
evidenceForEvent(node.event, "name"),
|
|
@@ -23039,7 +25993,7 @@ var checks = {
|
|
|
23039
25993
|
"eval.maxDurationMs",
|
|
23040
25994
|
"run",
|
|
23041
25995
|
(context) => context.run.durationMs !== void 0 && context.run.durationMs > maxDurationMs ? [
|
|
23042
|
-
|
|
25996
|
+
fail5(
|
|
23043
25997
|
"eval.maxDurationMs",
|
|
23044
25998
|
`Run duration exceeded ${maxDurationMs}ms.`,
|
|
23045
25999
|
evidenceForRun(context.run, "durationMs"),
|
|
@@ -23056,7 +26010,7 @@ var checks = {
|
|
|
23056
26010
|
void 0
|
|
23057
26011
|
);
|
|
23058
26012
|
return deepest !== void 0 && deepest.depth > maxDepth ? [
|
|
23059
|
-
|
|
26013
|
+
fail5(
|
|
23060
26014
|
"eval.maxDepth",
|
|
23061
26015
|
`Run tree depth exceeded ${maxDepth}.`,
|
|
23062
26016
|
evidenceForEvent(deepest.event, "depth"),
|
|
@@ -23073,7 +26027,7 @@ var checks = {
|
|
|
23073
26027
|
(context) => context.nodes.flatMap((node) => {
|
|
23074
26028
|
const retries = numericAttribute(node, ["retryCount", "retries", "attempt"]);
|
|
23075
26029
|
return retries !== void 0 && retries > maxRetries ? [
|
|
23076
|
-
|
|
26030
|
+
fail5(
|
|
23077
26031
|
"eval.maxRetries",
|
|
23078
26032
|
`Retry count exceeded ${maxRetries}.`,
|
|
23079
26033
|
evidenceForEvent(node.event, "attributes.retryCount"),
|
|
@@ -23088,7 +26042,7 @@ var checks = {
|
|
|
23088
26042
|
return createRule("eval.maxTotalTokens", "llm", (context) => {
|
|
23089
26043
|
const total = totalTokenCount(context.events);
|
|
23090
26044
|
return total > maxTotalTokens ? [
|
|
23091
|
-
|
|
26045
|
+
fail5(
|
|
23092
26046
|
"eval.maxTotalTokens",
|
|
23093
26047
|
`Total token usage exceeded ${maxTotalTokens}.`,
|
|
23094
26048
|
evidenceForRun(context.run, "tokenUsage.total"),
|
|
@@ -23103,7 +26057,7 @@ var checks = {
|
|
|
23103
26057
|
"eval.noFailedSteps",
|
|
23104
26058
|
"run",
|
|
23105
26059
|
(context) => context.nodes.filter((node) => node.event.status === "error" || node.event.kind === "ERROR").map(
|
|
23106
|
-
(node) =>
|
|
26060
|
+
(node) => fail5(
|
|
23107
26061
|
"eval.noFailedSteps",
|
|
23108
26062
|
"Run contains a failed step or error node.",
|
|
23109
26063
|
evidenceForEvent(node.event, "status"),
|
|
@@ -23121,7 +26075,7 @@ var checks = {
|
|
|
23121
26075
|
(node, index) => index < firstLlmIndex && node.event.kind === "RETRIEVER"
|
|
23122
26076
|
);
|
|
23123
26077
|
return retrievalIndex === -1 ? [
|
|
23124
|
-
|
|
26078
|
+
fail5(
|
|
23125
26079
|
"eval.requiredRetrievalBeforeGeneration",
|
|
23126
26080
|
"No retrieval step appeared before the first LLM generation.",
|
|
23127
26081
|
evidenceForEvent(context.nodes[firstLlmIndex].event, "kind"),
|
|
@@ -23137,7 +26091,7 @@ var checks = {
|
|
|
23137
26091
|
const decisions = context.nodes.filter((node) => node.event.kind === "DECISION");
|
|
23138
26092
|
if (decisions.length === 0) {
|
|
23139
26093
|
return [
|
|
23140
|
-
|
|
26094
|
+
fail5(
|
|
23141
26095
|
"eval.requiredDecisionMetadata",
|
|
23142
26096
|
"No decision node is available for required metadata.",
|
|
23143
26097
|
evidenceForRun(context.run, "children"),
|
|
@@ -23148,7 +26102,7 @@ var checks = {
|
|
|
23148
26102
|
}
|
|
23149
26103
|
return decisions.flatMap(
|
|
23150
26104
|
(node) => required.filter((key) => !hasAttribute(node, key)).map(
|
|
23151
|
-
(key) =>
|
|
26105
|
+
(key) => fail5(
|
|
23152
26106
|
"eval.requiredDecisionMetadata",
|
|
23153
26107
|
`Decision metadata ${key} is missing.`,
|
|
23154
26108
|
evidenceForEvent(node.event, `attributes.${key}`),
|
|
@@ -23169,7 +26123,7 @@ var checks = {
|
|
|
23169
26123
|
const contexts = collectTextFields(context.nodes, contextKeys, ["RETRIEVER", "TOOL"]);
|
|
23170
26124
|
if (answers.length === 0 || contexts.length === 0) {
|
|
23171
26125
|
return [
|
|
23172
|
-
|
|
26126
|
+
fail5(
|
|
23173
26127
|
"eval.contextOverlap",
|
|
23174
26128
|
"Answer and context text are required for overlap evaluation.",
|
|
23175
26129
|
firstEvidence(answers.length > 0 ? answers : contexts, context.run, "children"),
|
|
@@ -23183,7 +26137,7 @@ var checks = {
|
|
|
23183
26137
|
const sharedTerms = [...answerTerms].filter((term) => contextTerms.has(term)).length;
|
|
23184
26138
|
const overlap = answerTerms.size === 0 ? 0 : sharedTerms / answerTerms.size;
|
|
23185
26139
|
return sharedTerms < minSharedTerms || overlap < minOverlap ? [
|
|
23186
|
-
|
|
26140
|
+
fail5(
|
|
23187
26141
|
"eval.contextOverlap",
|
|
23188
26142
|
"Answer text did not sufficiently overlap retrieved context.",
|
|
23189
26143
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -23211,7 +26165,7 @@ var checks = {
|
|
|
23211
26165
|
const quotes = quotedSnippets(answerText, minQuoteLength);
|
|
23212
26166
|
if (quotes.length === 0) {
|
|
23213
26167
|
return requireQuote ? [
|
|
23214
|
-
|
|
26168
|
+
fail5(
|
|
23215
26169
|
"eval.quoteOverlap",
|
|
23216
26170
|
"Answer did not contain a quote for overlap evaluation.",
|
|
23217
26171
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -23222,7 +26176,7 @@ var checks = {
|
|
|
23222
26176
|
}
|
|
23223
26177
|
const missing = quotes.filter((quote) => !contextText.includes(quote.toLowerCase()));
|
|
23224
26178
|
return missing.length > 0 ? [
|
|
23225
|
-
|
|
26179
|
+
fail5(
|
|
23226
26180
|
"eval.quoteOverlap",
|
|
23227
26181
|
"Quoted answer text did not appear in retrieved context.",
|
|
23228
26182
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -23240,7 +26194,7 @@ var checks = {
|
|
|
23240
26194
|
const citations = collectTextFields(context.nodes, citationKeys);
|
|
23241
26195
|
const count = citationCount(answers.map((field) => field.text).join(" "), citations);
|
|
23242
26196
|
return count === 0 ? [
|
|
23243
|
-
|
|
26197
|
+
fail5(
|
|
23244
26198
|
"eval.citationPresence",
|
|
23245
26199
|
"Answer did not include citations or source references.",
|
|
23246
26200
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -23258,7 +26212,7 @@ var checks = {
|
|
|
23258
26212
|
const availableSet = new Set(available);
|
|
23259
26213
|
const missing = expected.filter((id) => !availableSet.has(id));
|
|
23260
26214
|
return missing.length > 0 ? [
|
|
23261
|
-
|
|
26215
|
+
fail5(
|
|
23262
26216
|
"eval.requiredSourceIds",
|
|
23263
26217
|
"Required source IDs were not present in trace context or citations.",
|
|
23264
26218
|
evidenceForRun(context.run, "children"),
|
|
@@ -23278,7 +26232,7 @@ var checks = {
|
|
|
23278
26232
|
const tooShort = options.minCharacters !== void 0 && characters < options.minCharacters || options.minWords !== void 0 && words < options.minWords;
|
|
23279
26233
|
const tooLong = options.maxCharacters !== void 0 && characters > options.maxCharacters || options.maxWords !== void 0 && words > options.maxWords;
|
|
23280
26234
|
return answer.length === 0 || tooShort || tooLong ? [
|
|
23281
|
-
|
|
26235
|
+
fail5(
|
|
23282
26236
|
"eval.answerLengthBounds",
|
|
23283
26237
|
"Answer length fell outside required bounds.",
|
|
23284
26238
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -23301,7 +26255,7 @@ var checks = {
|
|
|
23301
26255
|
const answer = answers.map((field) => field.text).join(" ").toLowerCase();
|
|
23302
26256
|
const matches = banned.filter((phrase) => answer.includes(phrase));
|
|
23303
26257
|
return matches.length > 0 ? [
|
|
23304
|
-
|
|
26258
|
+
fail5(
|
|
23305
26259
|
"eval.bannedUnsupportedPhrases",
|
|
23306
26260
|
"Answer contained banned unsupported-answer phrasing.",
|
|
23307
26261
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -23323,8 +26277,8 @@ function renderEvalMarkdown(result) {
|
|
|
23323
26277
|
];
|
|
23324
26278
|
if (result.diagnostics.length > 0) {
|
|
23325
26279
|
lines.push("", "## Diagnostics");
|
|
23326
|
-
for (const
|
|
23327
|
-
lines.push(`- ${
|
|
26280
|
+
for (const diagnostic8 of result.diagnostics) {
|
|
26281
|
+
lines.push(`- ${diagnostic8.code}: ${diagnostic8.message}`);
|
|
23328
26282
|
}
|
|
23329
26283
|
}
|
|
23330
26284
|
if (result.findings.length > 0) {
|
|
@@ -23341,7 +26295,7 @@ function renderEvalMarkdown(result) {
|
|
|
23341
26295
|
// packages/cli/src/eval.ts
|
|
23342
26296
|
var CONFIG_EXTENSIONS3 = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
23343
26297
|
var TS_CONFIG_EXTENSIONS3 = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
23344
|
-
function
|
|
26298
|
+
function diagnostic7(code, message, severity = "error") {
|
|
23345
26299
|
return { code, message, severity };
|
|
23346
26300
|
}
|
|
23347
26301
|
function errorResult4(code, message, format = "unknown") {
|
|
@@ -23351,7 +26305,7 @@ function errorResult4(code, message, format = "unknown") {
|
|
|
23351
26305
|
format,
|
|
23352
26306
|
summary: { passed: 0, failed: 0, warnings: 0, errors: 1 },
|
|
23353
26307
|
findings: [],
|
|
23354
|
-
diagnostics: [
|
|
26308
|
+
diagnostics: [diagnostic7(code, message)]
|
|
23355
26309
|
};
|
|
23356
26310
|
}
|
|
23357
26311
|
function parseNumber2(value, label) {
|
|
@@ -23520,8 +26474,8 @@ function printHuman3(result) {
|
|
|
23520
26474
|
console.log(
|
|
23521
26475
|
`Summary: ${result.summary.failed} failed, ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
23522
26476
|
);
|
|
23523
|
-
for (const
|
|
23524
|
-
console.log(`- ${
|
|
26477
|
+
for (const diagnostic8 of result.diagnostics) {
|
|
26478
|
+
console.log(`- ${diagnostic8.code}: ${diagnostic8.message}`);
|
|
23525
26479
|
}
|
|
23526
26480
|
for (const finding of result.findings) {
|
|
23527
26481
|
const path44 = finding.evidence[0]?.path;
|
|
@@ -23644,8 +26598,8 @@ function renderCheckSection(result) {
|
|
|
23644
26598
|
const path44 = finding.evidence[0]?.path ?? "(run)";
|
|
23645
26599
|
lines.push(`- ${finding.ruleId}: ${finding.message} (${path44})`);
|
|
23646
26600
|
}
|
|
23647
|
-
for (const
|
|
23648
|
-
lines.push(`- ${
|
|
26601
|
+
for (const diagnostic8 of result.diagnostics.slice(0, 10)) {
|
|
26602
|
+
lines.push(`- ${diagnostic8.code}: ${diagnostic8.message}`);
|
|
23649
26603
|
}
|
|
23650
26604
|
return lines.join("\n");
|
|
23651
26605
|
}
|
|
@@ -26200,7 +29154,7 @@ async function indexSqliteCleanCommand(options = {}) {
|
|
|
26200
29154
|
function printJson4(payload) {
|
|
26201
29155
|
console.log(JSON.stringify(payload, null, 2));
|
|
26202
29156
|
}
|
|
26203
|
-
function
|
|
29157
|
+
function fail6(json, message) {
|
|
26204
29158
|
if (json) {
|
|
26205
29159
|
printJson4({ ok: false, error: message });
|
|
26206
29160
|
} else {
|
|
@@ -26251,14 +29205,14 @@ async function workspaceInitCommand(options = {}) {
|
|
|
26251
29205
|
for (const dir of result.createdDirs) console.log(`Created ${dir}/`);
|
|
26252
29206
|
console.log("\nNext: `agent-inspect workspace status`");
|
|
26253
29207
|
} catch (error) {
|
|
26254
|
-
|
|
29208
|
+
fail6(options.json, error instanceof Error ? error.message : String(error));
|
|
26255
29209
|
}
|
|
26256
29210
|
}
|
|
26257
29211
|
async function workspaceStatusCommand(options = {}) {
|
|
26258
29212
|
const location = resolveWorkspaceLocation(options.cwd);
|
|
26259
29213
|
const manifestResult = await readWorkspaceManifestFile(location);
|
|
26260
29214
|
if (!manifestResult.ok || !manifestResult.manifest) {
|
|
26261
|
-
|
|
29215
|
+
fail6(
|
|
26262
29216
|
options.json,
|
|
26263
29217
|
manifestResult.exists ? `invalid workspace.json: ${manifestResult.errors.join("; ")}` : "no workspace found (run `agent-inspect workspace init`)"
|
|
26264
29218
|
);
|
|
@@ -26280,7 +29234,7 @@ async function workspaceStatusCommand(options = {}) {
|
|
|
26280
29234
|
`Index: ${status.index.enabled ? status.index.type : "disabled"}${status.index.exists ? " (present)" : ""}`
|
|
26281
29235
|
);
|
|
26282
29236
|
} catch (error) {
|
|
26283
|
-
|
|
29237
|
+
fail6(options.json, error instanceof Error ? error.message : String(error));
|
|
26284
29238
|
}
|
|
26285
29239
|
}
|
|
26286
29240
|
async function workspaceDoctorCommand(options = {}) {
|
|
@@ -26302,7 +29256,7 @@ async function workspaceCleanCommand(options = {}) {
|
|
|
26302
29256
|
const location = resolveWorkspaceLocation(options.cwd);
|
|
26303
29257
|
const manifestResult = await readWorkspaceManifestFile(location);
|
|
26304
29258
|
if (!manifestResult.ok || !manifestResult.manifest) {
|
|
26305
|
-
|
|
29259
|
+
fail6(
|
|
26306
29260
|
options.json,
|
|
26307
29261
|
manifestResult.exists ? `invalid workspace.json: ${manifestResult.errors.join("; ")}` : "no workspace found (run `agent-inspect workspace init`)"
|
|
26308
29262
|
);
|
|
@@ -26327,7 +29281,7 @@ async function workspaceCleanCommand(options = {}) {
|
|
|
26327
29281
|
if (result.removed.length === 0) console.log("- (nothing)");
|
|
26328
29282
|
for (const item of result.removed) console.log(`- ${item}`);
|
|
26329
29283
|
} catch (error) {
|
|
26330
|
-
|
|
29284
|
+
fail6(options.json, error instanceof Error ? error.message : String(error));
|
|
26331
29285
|
}
|
|
26332
29286
|
}
|
|
26333
29287
|
async function workspacePathCommand(options = {}) {
|
|
@@ -27025,7 +29979,7 @@ function createCliProgram() {
|
|
|
27025
29979
|
"add run.maxStepDuration (e.g. 30s, 5m)"
|
|
27026
29980
|
).option("--require-completed", "add run.requireCompleted").option("--detect-stalls", "add run.stall for running or incomplete events").option(
|
|
27027
29981
|
"--fail-on-observation <status>",
|
|
27028
|
-
"
|
|
29982
|
+
"outcome gate: require \u22651 OUTCOME and fail when status matches (passed,failed,unknown,skipped; comma-separated). Does not invent outcomes from tool/LLM/run success; use --preset trajectory for structural checks"
|
|
27029
29983
|
).option("--session <id>", "check all runs in a workflow session (requires --dir)").option("--group <id>", "check all runs sharing a groupId (requires --dir)").option(
|
|
27030
29984
|
"--correlate-group",
|
|
27031
29985
|
"when using --session, also match synthetic group: session keys"
|
|
@@ -27038,7 +29992,7 @@ function createCliProgram() {
|
|
|
27038
29992
|
"run optional circuit rules (repeatable): same-tool-repetition, max-retries, ...",
|
|
27039
29993
|
(value, previous = []) => [...previous, value]
|
|
27040
29994
|
).addOption(
|
|
27041
|
-
new commander.Option("--preset <name>", "additive check preset").choices(["trajectory", "safety", "comprehensive", "behavioral-session"])
|
|
29995
|
+
new commander.Option("--preset <name>", "additive check preset (trajectory=structure only; safety=share checks; not interchangeable)").choices(["trajectory", "safety", "comprehensive", "behavioral-session"])
|
|
27042
29996
|
).addOption(
|
|
27043
29997
|
new commander.Option(
|
|
27044
29998
|
"--evidence-on <mode>",
|