agent-inspect 6.29.6 → 6.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +1 -1
- package/docs/CLI.md +26 -4
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-DBRMI7TC.mjs → chunk-BDRJENLM.mjs} +2279 -79
- package/packages/cli/dist/chunk-BDRJENLM.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +3067 -145
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +791 -59
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-6CET2GW2.mjs → src-ZUGAUXY6.mjs} +3 -3
- package/packages/cli/dist/{src-6CET2GW2.mjs.map → src-ZUGAUXY6.mjs.map} +1 -1
- package/packages/cli/dist/chunk-DBRMI7TC.mjs.map +0 -1
|
@@ -3388,7 +3388,7 @@ function deriveLastError(runs) {
|
|
|
3388
3388
|
}
|
|
3389
3389
|
function deriveCheckSummary(runs) {
|
|
3390
3390
|
let pass2 = 0;
|
|
3391
|
-
let
|
|
3391
|
+
let fail7 = 0;
|
|
3392
3392
|
let warn2 = 0;
|
|
3393
3393
|
let found = false;
|
|
3394
3394
|
for (const run of runs) {
|
|
@@ -3400,7 +3400,7 @@ function deriveCheckSummary(runs) {
|
|
|
3400
3400
|
found = true;
|
|
3401
3401
|
}
|
|
3402
3402
|
if (typeof record.fail === "number") {
|
|
3403
|
-
|
|
3403
|
+
fail7 += record.fail;
|
|
3404
3404
|
found = true;
|
|
3405
3405
|
}
|
|
3406
3406
|
if (typeof record.warn === "number") {
|
|
@@ -3408,7 +3408,7 @@ function deriveCheckSummary(runs) {
|
|
|
3408
3408
|
found = true;
|
|
3409
3409
|
}
|
|
3410
3410
|
}
|
|
3411
|
-
return found ? { pass: pass2, fail:
|
|
3411
|
+
return found ? { pass: pass2, fail: fail7, warn: warn2 } : void 0;
|
|
3412
3412
|
}
|
|
3413
3413
|
function deriveObservationSummary(runs) {
|
|
3414
3414
|
for (const run of [...runs].sort((a, b) => activityMs(b) - activityMs(a))) {
|
|
@@ -5726,6 +5726,14 @@ function finishedEvents(context, kind) {
|
|
|
5726
5726
|
function toolInvocationEvents(context) {
|
|
5727
5727
|
return semanticEvents(context).filter((event) => event.kind === "TOOL");
|
|
5728
5728
|
}
|
|
5729
|
+
function firstIndexByName(events) {
|
|
5730
|
+
const index = /* @__PURE__ */ new Map();
|
|
5731
|
+
for (const [position, event] of events.entries()) {
|
|
5732
|
+
const name = toolName(event);
|
|
5733
|
+
if (!index.has(name)) index.set(name, position);
|
|
5734
|
+
}
|
|
5735
|
+
return index;
|
|
5736
|
+
}
|
|
5729
5737
|
function isRecord9(value) {
|
|
5730
5738
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
5731
5739
|
}
|
|
@@ -6196,6 +6204,181 @@ function createToolUsageRule(options) {
|
|
|
6196
6204
|
}
|
|
6197
6205
|
};
|
|
6198
6206
|
}
|
|
6207
|
+
function createToolOrderingRule(options) {
|
|
6208
|
+
const ruleId = options.id ?? "tool.order";
|
|
6209
|
+
const mode = options.mode ?? "first-occurrence";
|
|
6210
|
+
return {
|
|
6211
|
+
id: ruleId,
|
|
6212
|
+
category: "tool",
|
|
6213
|
+
defaultSeverity: "error",
|
|
6214
|
+
evaluate(context) {
|
|
6215
|
+
const tools = finishedEvents(context, "TOOL");
|
|
6216
|
+
const index = firstIndexByName(tools);
|
|
6217
|
+
const beforeIndex = index.get(options.before);
|
|
6218
|
+
const afterIndex = index.get(options.after);
|
|
6219
|
+
if (beforeIndex === void 0 || afterIndex === void 0) {
|
|
6220
|
+
return [];
|
|
6221
|
+
}
|
|
6222
|
+
if (mode === "first-occurrence" && beforeIndex >= afterIndex) {
|
|
6223
|
+
return [
|
|
6224
|
+
failFinding(
|
|
6225
|
+
ruleId,
|
|
6226
|
+
`Tool ${options.before} must appear before ${options.after}.`,
|
|
6227
|
+
[eventEvidence(tools[beforeIndex]), eventEvidence(tools[afterIndex])],
|
|
6228
|
+
{ before: options.before, after: options.after },
|
|
6229
|
+
tools.map(toolName)
|
|
6230
|
+
)
|
|
6231
|
+
];
|
|
6232
|
+
}
|
|
6233
|
+
const beforeEvent = tools[beforeIndex];
|
|
6234
|
+
const afterEvent = tools[afterIndex];
|
|
6235
|
+
const beforeEnd = eventEndMs(beforeEvent);
|
|
6236
|
+
const afterStart = eventStartMs(afterEvent);
|
|
6237
|
+
if (mode === "happens-before") {
|
|
6238
|
+
const expected = {
|
|
6239
|
+
before: options.before,
|
|
6240
|
+
after: options.after,
|
|
6241
|
+
mode,
|
|
6242
|
+
relation: "before.end <= after.start"
|
|
6243
|
+
};
|
|
6244
|
+
if (options.before === options.after) {
|
|
6245
|
+
return [
|
|
6246
|
+
failFinding(
|
|
6247
|
+
ruleId,
|
|
6248
|
+
`Tool ${options.before} cannot causally happen before itself.`,
|
|
6249
|
+
[eventEvidence(beforeEvent)],
|
|
6250
|
+
expected,
|
|
6251
|
+
{ code: "tool.order.same-tool" }
|
|
6252
|
+
)
|
|
6253
|
+
];
|
|
6254
|
+
}
|
|
6255
|
+
if (beforeEnd === void 0 || afterStart === void 0) {
|
|
6256
|
+
return [
|
|
6257
|
+
failFinding(
|
|
6258
|
+
ruleId,
|
|
6259
|
+
`Tool order ${options.before} before ${options.after} could not establish causal timing.`,
|
|
6260
|
+
[eventEvidence(beforeEvent), eventEvidence(afterEvent)],
|
|
6261
|
+
expected,
|
|
6262
|
+
{
|
|
6263
|
+
code: "tool.order.interval-unresolved",
|
|
6264
|
+
beforeEndResolved: beforeEnd !== void 0,
|
|
6265
|
+
afterStartResolved: afterStart !== void 0
|
|
6266
|
+
}
|
|
6267
|
+
)
|
|
6268
|
+
];
|
|
6269
|
+
}
|
|
6270
|
+
if (beforeEnd > afterStart) {
|
|
6271
|
+
return [
|
|
6272
|
+
failFinding(
|
|
6273
|
+
ruleId,
|
|
6274
|
+
`Tool ${options.before} must finish before ${options.after} starts.`,
|
|
6275
|
+
[eventEvidence(beforeEvent), eventEvidence(afterEvent)],
|
|
6276
|
+
expected,
|
|
6277
|
+
{
|
|
6278
|
+
beforeEndedAt: new Date(beforeEnd).toISOString(),
|
|
6279
|
+
afterStartedAt: new Date(afterStart).toISOString()
|
|
6280
|
+
}
|
|
6281
|
+
)
|
|
6282
|
+
];
|
|
6283
|
+
}
|
|
6284
|
+
return [];
|
|
6285
|
+
}
|
|
6286
|
+
if (mode === "all-occurrences") {
|
|
6287
|
+
const beforeEvents = tools.filter((event) => toolName(event) === options.before);
|
|
6288
|
+
const afterEvents = tools.filter((event) => toolName(event) === options.after);
|
|
6289
|
+
const expected = {
|
|
6290
|
+
before: options.before,
|
|
6291
|
+
after: options.after,
|
|
6292
|
+
mode,
|
|
6293
|
+
relation: "max(before.end) <= min(after.start)"
|
|
6294
|
+
};
|
|
6295
|
+
if (options.before === options.after) {
|
|
6296
|
+
return [
|
|
6297
|
+
failFinding(
|
|
6298
|
+
ruleId,
|
|
6299
|
+
`Tool ${options.before} cannot causally happen before itself.`,
|
|
6300
|
+
[eventEvidence(beforeEvent)],
|
|
6301
|
+
expected,
|
|
6302
|
+
{ code: "tool.order.same-tool" }
|
|
6303
|
+
)
|
|
6304
|
+
];
|
|
6305
|
+
}
|
|
6306
|
+
let latestBefore;
|
|
6307
|
+
let earliestAfter;
|
|
6308
|
+
let missingBeforeEnds = 0;
|
|
6309
|
+
let missingAfterStarts = 0;
|
|
6310
|
+
let firstMissingBefore;
|
|
6311
|
+
let firstMissingAfter;
|
|
6312
|
+
for (const event of beforeEvents) {
|
|
6313
|
+
const end = eventEndMs(event);
|
|
6314
|
+
if (end === void 0) {
|
|
6315
|
+
missingBeforeEnds += 1;
|
|
6316
|
+
firstMissingBefore ??= event;
|
|
6317
|
+
} else if (latestBefore === void 0 || end > latestBefore.end) {
|
|
6318
|
+
latestBefore = { event, end };
|
|
6319
|
+
}
|
|
6320
|
+
}
|
|
6321
|
+
for (const event of afterEvents) {
|
|
6322
|
+
const start = eventStartMs(event);
|
|
6323
|
+
if (start === void 0) {
|
|
6324
|
+
missingAfterStarts += 1;
|
|
6325
|
+
firstMissingAfter ??= event;
|
|
6326
|
+
} else if (earliestAfter === void 0 || start < earliestAfter.start) {
|
|
6327
|
+
earliestAfter = { event, start };
|
|
6328
|
+
}
|
|
6329
|
+
}
|
|
6330
|
+
if (missingBeforeEnds > 0 || missingAfterStarts > 0 || latestBefore === void 0 || earliestAfter === void 0) {
|
|
6331
|
+
return [
|
|
6332
|
+
failFinding(
|
|
6333
|
+
ruleId,
|
|
6334
|
+
`Tool order ${options.before} before ${options.after} could not establish all causal intervals.`,
|
|
6335
|
+
[firstMissingBefore, firstMissingAfter].filter((event) => event !== void 0).map((event) => eventEvidence(event)),
|
|
6336
|
+
expected,
|
|
6337
|
+
{
|
|
6338
|
+
code: "tool.order.interval-unresolved",
|
|
6339
|
+
missingBeforeEnds,
|
|
6340
|
+
missingAfterStarts
|
|
6341
|
+
}
|
|
6342
|
+
)
|
|
6343
|
+
];
|
|
6344
|
+
}
|
|
6345
|
+
if (latestBefore.end > earliestAfter.start) {
|
|
6346
|
+
return [
|
|
6347
|
+
failFinding(
|
|
6348
|
+
ruleId,
|
|
6349
|
+
`Every ${options.before} tool call must finish before any ${options.after} tool call starts.`,
|
|
6350
|
+
[eventEvidence(latestBefore.event), eventEvidence(earliestAfter.event)],
|
|
6351
|
+
expected,
|
|
6352
|
+
{
|
|
6353
|
+
latestBeforeEndedAt: new Date(latestBefore.end).toISOString(),
|
|
6354
|
+
earliestAfterStartedAt: new Date(earliestAfter.start).toISOString()
|
|
6355
|
+
}
|
|
6356
|
+
)
|
|
6357
|
+
];
|
|
6358
|
+
}
|
|
6359
|
+
return [];
|
|
6360
|
+
}
|
|
6361
|
+
if (beforeEnd !== void 0 && afterStart !== void 0 && beforeEnd > afterStart) {
|
|
6362
|
+
return [
|
|
6363
|
+
{
|
|
6364
|
+
ruleId,
|
|
6365
|
+
severity: "warning",
|
|
6366
|
+
status: "warning",
|
|
6367
|
+
message: `Tool order ${options.before} before ${options.after} satisfied start/encounter order, but intervals overlap so causal completion before the next tool was not established.`,
|
|
6368
|
+
expected: { before: options.before, after: options.after, mode: "first-occurrence" },
|
|
6369
|
+
actual: {
|
|
6370
|
+
code: "tool.order.overlap",
|
|
6371
|
+
beforeEndedAt: beforeEvent.endedAt,
|
|
6372
|
+
afterStartedAt: afterEvent.startedAt ?? afterEvent.timestamp
|
|
6373
|
+
},
|
|
6374
|
+
evidence: [eventEvidence(beforeEvent), eventEvidence(afterEvent)]
|
|
6375
|
+
}
|
|
6376
|
+
];
|
|
6377
|
+
}
|
|
6378
|
+
return [];
|
|
6379
|
+
}
|
|
6380
|
+
};
|
|
6381
|
+
}
|
|
6199
6382
|
function createLlmUsageRule(options) {
|
|
6200
6383
|
return {
|
|
6201
6384
|
id: "llm.usage",
|
|
@@ -6270,6 +6453,47 @@ function createLlmUsageRule(options) {
|
|
|
6270
6453
|
}
|
|
6271
6454
|
};
|
|
6272
6455
|
}
|
|
6456
|
+
function createStructureIncompleteRule(options = {}) {
|
|
6457
|
+
return {
|
|
6458
|
+
id: "structure.incomplete",
|
|
6459
|
+
category: "structure",
|
|
6460
|
+
defaultSeverity: "error",
|
|
6461
|
+
evaluate(context) {
|
|
6462
|
+
const findings = [];
|
|
6463
|
+
if (!options.allowRunning) {
|
|
6464
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
6465
|
+
if (running.length > 0) {
|
|
6466
|
+
findings.push(
|
|
6467
|
+
failFinding(
|
|
6468
|
+
"structure.incomplete",
|
|
6469
|
+
"Trace contains incomplete running events.",
|
|
6470
|
+
running.map((event) => eventEvidence(event, "status")),
|
|
6471
|
+
"no running events",
|
|
6472
|
+
running.length
|
|
6473
|
+
)
|
|
6474
|
+
);
|
|
6475
|
+
}
|
|
6476
|
+
}
|
|
6477
|
+
if (options.requireEndedAtForStarted) {
|
|
6478
|
+
const missingEndedAt = semanticEvents(context).filter(
|
|
6479
|
+
(event) => event.startedAt !== void 0 && event.endedAt === void 0 && event.status !== "running"
|
|
6480
|
+
);
|
|
6481
|
+
if (missingEndedAt.length > 0) {
|
|
6482
|
+
findings.push(
|
|
6483
|
+
failFinding(
|
|
6484
|
+
"structure.incomplete",
|
|
6485
|
+
"Trace contains events with startedAt but no endedAt.",
|
|
6486
|
+
missingEndedAt.map((event) => eventEvidence(event, "endedAt")),
|
|
6487
|
+
"endedAt for started events",
|
|
6488
|
+
missingEndedAt.length
|
|
6489
|
+
)
|
|
6490
|
+
);
|
|
6491
|
+
}
|
|
6492
|
+
}
|
|
6493
|
+
return findings;
|
|
6494
|
+
}
|
|
6495
|
+
};
|
|
6496
|
+
}
|
|
6273
6497
|
function createStructureOrphanRule(options = {}) {
|
|
6274
6498
|
const allowMarkedUnresolved = options.allowMarkedUnresolved ?? true;
|
|
6275
6499
|
return {
|
|
@@ -7035,6 +7259,198 @@ var init_checks2 = __esm({
|
|
|
7035
7259
|
});
|
|
7036
7260
|
|
|
7037
7261
|
// packages/core/src/checks/contract-scope.ts
|
|
7262
|
+
function diagnostic2(code, message, ruleId) {
|
|
7263
|
+
return {
|
|
7264
|
+
code,
|
|
7265
|
+
message,
|
|
7266
|
+
severity: "error",
|
|
7267
|
+
...ruleId !== void 0 ? { ruleId } : {}
|
|
7268
|
+
};
|
|
7269
|
+
}
|
|
7270
|
+
function isNonEmptyString4(value) {
|
|
7271
|
+
return typeof value === "string" && value.trim() !== "";
|
|
7272
|
+
}
|
|
7273
|
+
function scopeEntries(scope) {
|
|
7274
|
+
const out = [];
|
|
7275
|
+
if (isNonEmptyString4(scope.runId)) out.push(["runId", scope.runId.trim()]);
|
|
7276
|
+
if (isNonEmptyString4(scope.subAgentId)) out.push(["subAgentId", scope.subAgentId.trim()]);
|
|
7277
|
+
if (isNonEmptyString4(scope.groupId)) out.push(["groupId", scope.groupId.trim()]);
|
|
7278
|
+
if (isNonEmptyString4(scope.workflowStep)) {
|
|
7279
|
+
out.push(["workflowStep", scope.workflowStep.trim()]);
|
|
7280
|
+
}
|
|
7281
|
+
if (isNonEmptyString4(scope.rootEventId)) {
|
|
7282
|
+
out.push(["rootEventId", scope.rootEventId.trim()]);
|
|
7283
|
+
}
|
|
7284
|
+
return out;
|
|
7285
|
+
}
|
|
7286
|
+
function mergeWorkflowMeta(into, from) {
|
|
7287
|
+
if (!from) return;
|
|
7288
|
+
for (const [key, value] of Object.entries(from)) {
|
|
7289
|
+
if (value === void 0) continue;
|
|
7290
|
+
if (into[key] === void 0) {
|
|
7291
|
+
into[key] = value;
|
|
7292
|
+
}
|
|
7293
|
+
}
|
|
7294
|
+
}
|
|
7295
|
+
function workflowMetadataForRun(events, runId) {
|
|
7296
|
+
const preferred = {};
|
|
7297
|
+
const fallback = {};
|
|
7298
|
+
for (const event of events) {
|
|
7299
|
+
if (event.runId !== runId) continue;
|
|
7300
|
+
const attrs = event.attributes;
|
|
7301
|
+
if (!attrs || typeof attrs !== "object") continue;
|
|
7302
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
7303
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
7304
|
+
const bag = {};
|
|
7305
|
+
mergeWorkflowMeta(bag, direct);
|
|
7306
|
+
mergeWorkflowMeta(bag, nested);
|
|
7307
|
+
if (event.kind === "RUN") {
|
|
7308
|
+
mergeWorkflowMeta(preferred, bag);
|
|
7309
|
+
} else {
|
|
7310
|
+
mergeWorkflowMeta(fallback, bag);
|
|
7311
|
+
}
|
|
7312
|
+
}
|
|
7313
|
+
mergeWorkflowMeta(preferred, fallback);
|
|
7314
|
+
return preferred;
|
|
7315
|
+
}
|
|
7316
|
+
function flattenNodes2(nodes) {
|
|
7317
|
+
return nodes.flatMap((node) => [node, ...flattenNodes2(node.children)]);
|
|
7318
|
+
}
|
|
7319
|
+
function collectSubtreeEventIds(run, rootEventId) {
|
|
7320
|
+
const nodes = flattenNodes2(run.children);
|
|
7321
|
+
const byId = new Map(nodes.map((node) => [node.event.eventId, node]));
|
|
7322
|
+
const root = byId.get(rootEventId);
|
|
7323
|
+
if (!root) return void 0;
|
|
7324
|
+
const ids = /* @__PURE__ */ new Set();
|
|
7325
|
+
const visit = (node) => {
|
|
7326
|
+
ids.add(node.event.eventId);
|
|
7327
|
+
for (const child of node.children) visit(child);
|
|
7328
|
+
};
|
|
7329
|
+
visit(root);
|
|
7330
|
+
return ids;
|
|
7331
|
+
}
|
|
7332
|
+
function filterTree(nodes, keep) {
|
|
7333
|
+
const out = [];
|
|
7334
|
+
for (const node of nodes) {
|
|
7335
|
+
if (!keep.has(node.event.eventId)) continue;
|
|
7336
|
+
out.push({
|
|
7337
|
+
...node,
|
|
7338
|
+
children: filterTree(node.children, keep)
|
|
7339
|
+
});
|
|
7340
|
+
}
|
|
7341
|
+
return out;
|
|
7342
|
+
}
|
|
7343
|
+
function projectInput(input3, run, eventIds) {
|
|
7344
|
+
const runEvents = input3.read.events.filter((event) => event.runId === run.runId);
|
|
7345
|
+
const projectedEvents = eventIds === void 0 ? runEvents : runEvents.filter((event) => eventIds.has(event.eventId) || event.kind === "RUN");
|
|
7346
|
+
const projectedChildren = eventIds === void 0 ? run.children : filterTree(run.children, eventIds);
|
|
7347
|
+
const projectedRun = {
|
|
7348
|
+
...run,
|
|
7349
|
+
children: projectedChildren,
|
|
7350
|
+
metadata: {
|
|
7351
|
+
...run.metadata,
|
|
7352
|
+
totalEvents: flattenNodes2(projectedChildren).length
|
|
7353
|
+
}
|
|
7354
|
+
};
|
|
7355
|
+
return {
|
|
7356
|
+
evidenceEventCount: projectedEvents.length,
|
|
7357
|
+
input: {
|
|
7358
|
+
...input3,
|
|
7359
|
+
selectedRun: projectedRun,
|
|
7360
|
+
read: {
|
|
7361
|
+
...input3.read,
|
|
7362
|
+
runs: [projectedRun],
|
|
7363
|
+
events: projectedEvents
|
|
7364
|
+
}
|
|
7365
|
+
}
|
|
7366
|
+
};
|
|
7367
|
+
}
|
|
7368
|
+
function resolveTraceContractScope(input3, scope) {
|
|
7369
|
+
if (scope === void 0) {
|
|
7370
|
+
return { diagnostics: [] };
|
|
7371
|
+
}
|
|
7372
|
+
const entries = scopeEntries(scope);
|
|
7373
|
+
if (entries.length === 0) {
|
|
7374
|
+
return {
|
|
7375
|
+
diagnostics: [
|
|
7376
|
+
diagnostic2(
|
|
7377
|
+
"AI_CHECK_INVALID_CONFIG",
|
|
7378
|
+
"TraceContract scope requires at least one explicit selector (runId, subAgentId, groupId, workflowStep, or rootEventId).",
|
|
7379
|
+
"contract.scope.empty"
|
|
7380
|
+
)
|
|
7381
|
+
]
|
|
7382
|
+
};
|
|
7383
|
+
}
|
|
7384
|
+
const matchedSelectors = Object.fromEntries(entries);
|
|
7385
|
+
const actorKeys = entries.filter(([key]) => key !== "rootEventId");
|
|
7386
|
+
const rootEventId = matchedSelectors.rootEventId;
|
|
7387
|
+
let candidates = [...input3.read.runs];
|
|
7388
|
+
if (matchedSelectors.runId) {
|
|
7389
|
+
candidates = candidates.filter((run2) => run2.runId === matchedSelectors.runId);
|
|
7390
|
+
}
|
|
7391
|
+
for (const [key, value] of actorKeys) {
|
|
7392
|
+
if (key === "runId") continue;
|
|
7393
|
+
candidates = candidates.filter((run2) => {
|
|
7394
|
+
const meta2 = workflowMetadataForRun(input3.read.events, run2.runId);
|
|
7395
|
+
const actual = meta2[key];
|
|
7396
|
+
return actual === value;
|
|
7397
|
+
});
|
|
7398
|
+
}
|
|
7399
|
+
if (rootEventId && actorKeys.length === 0) {
|
|
7400
|
+
candidates = candidates.filter(
|
|
7401
|
+
(run2) => flattenNodes2(run2.children).some((node) => node.event.eventId === rootEventId)
|
|
7402
|
+
);
|
|
7403
|
+
}
|
|
7404
|
+
if (candidates.length === 0) {
|
|
7405
|
+
return {
|
|
7406
|
+
diagnostics: [
|
|
7407
|
+
diagnostic2(
|
|
7408
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
7409
|
+
`TraceContract scope matched zero actors for ${JSON.stringify(matchedSelectors)}. Missing actor metadata does not fall back to the whole session.`,
|
|
7410
|
+
"contract.scope.zero-match"
|
|
7411
|
+
)
|
|
7412
|
+
]
|
|
7413
|
+
};
|
|
7414
|
+
}
|
|
7415
|
+
if (candidates.length > 1) {
|
|
7416
|
+
return {
|
|
7417
|
+
diagnostics: [
|
|
7418
|
+
diagnostic2(
|
|
7419
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
7420
|
+
`TraceContract scope matched ${candidates.length} actors for ${JSON.stringify(matchedSelectors)} (runIds: ${candidates.map((run2) => run2.runId).sort().join(", ")}). Singular actor selectors must resolve to exactly one run.`,
|
|
7421
|
+
"contract.scope.ambiguous"
|
|
7422
|
+
)
|
|
7423
|
+
]
|
|
7424
|
+
};
|
|
7425
|
+
}
|
|
7426
|
+
const run = candidates[0];
|
|
7427
|
+
let eventIds;
|
|
7428
|
+
if (rootEventId) {
|
|
7429
|
+
eventIds = collectSubtreeEventIds(run, rootEventId);
|
|
7430
|
+
if (eventIds === void 0) {
|
|
7431
|
+
return {
|
|
7432
|
+
diagnostics: [
|
|
7433
|
+
diagnostic2(
|
|
7434
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
7435
|
+
`TraceContract scope rootEventId ${rootEventId} was not found in run ${run.runId}.`,
|
|
7436
|
+
"contract.scope.root-missing"
|
|
7437
|
+
)
|
|
7438
|
+
]
|
|
7439
|
+
};
|
|
7440
|
+
}
|
|
7441
|
+
}
|
|
7442
|
+
const projected = projectInput(input3, run, eventIds);
|
|
7443
|
+
return {
|
|
7444
|
+
resolved: {
|
|
7445
|
+
runId: run.runId,
|
|
7446
|
+
...rootEventId !== void 0 ? { rootEventId } : {},
|
|
7447
|
+
matchedSelectors,
|
|
7448
|
+
evidenceEventCount: projected.evidenceEventCount,
|
|
7449
|
+
input: projected.input
|
|
7450
|
+
},
|
|
7451
|
+
diagnostics: []
|
|
7452
|
+
};
|
|
7453
|
+
}
|
|
7038
7454
|
var init_contract_scope = __esm({
|
|
7039
7455
|
"packages/core/src/checks/contract-scope.ts"() {
|
|
7040
7456
|
init_metadata();
|
|
@@ -7042,6 +7458,146 @@ var init_contract_scope = __esm({
|
|
|
7042
7458
|
});
|
|
7043
7459
|
|
|
7044
7460
|
// packages/core/src/checks/control-rules.ts
|
|
7461
|
+
function asStringList(value) {
|
|
7462
|
+
if (!Array.isArray(value)) return void 0;
|
|
7463
|
+
const out = [];
|
|
7464
|
+
for (const item of value) {
|
|
7465
|
+
if (typeof item !== "string" || item.trim() === "") return void 0;
|
|
7466
|
+
out.push(item.trim());
|
|
7467
|
+
}
|
|
7468
|
+
return out;
|
|
7469
|
+
}
|
|
7470
|
+
function attributeList(events, attribute) {
|
|
7471
|
+
if (!attribute) return void 0;
|
|
7472
|
+
for (const event of events) {
|
|
7473
|
+
const attrs = event.attributes;
|
|
7474
|
+
if (!attrs || typeof attrs !== "object") continue;
|
|
7475
|
+
const direct = asStringList(attrs[attribute]);
|
|
7476
|
+
if (direct) return direct;
|
|
7477
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? asStringList(attrs.metadata[attribute]) : void 0;
|
|
7478
|
+
if (nested) return nested;
|
|
7479
|
+
}
|
|
7480
|
+
return void 0;
|
|
7481
|
+
}
|
|
7482
|
+
function fail(ruleId, message, evidence, expected, actual) {
|
|
7483
|
+
return {
|
|
7484
|
+
ruleId,
|
|
7485
|
+
severity: "error",
|
|
7486
|
+
status: "fail",
|
|
7487
|
+
message,
|
|
7488
|
+
...expected !== void 0 ? { expected } : {},
|
|
7489
|
+
...actual !== void 0 ? { actual } : {},
|
|
7490
|
+
evidence: [...evidence]
|
|
7491
|
+
};
|
|
7492
|
+
}
|
|
7493
|
+
function sortedUnique(values) {
|
|
7494
|
+
return [...new Set(values)].sort((a, b) => a.localeCompare(b));
|
|
7495
|
+
}
|
|
7496
|
+
function setEqual(left, right) {
|
|
7497
|
+
const a = sortedUnique(left);
|
|
7498
|
+
const b = sortedUnique(right);
|
|
7499
|
+
return a.length === b.length && a.every((value, index) => value === b[index]);
|
|
7500
|
+
}
|
|
7501
|
+
function evaluateControlRules(events, rules, runEvidence2, observationNames) {
|
|
7502
|
+
const findings = [];
|
|
7503
|
+
const declared = rules.declaredTools ?? attributeList(events, rules.declaredToolsAttribute);
|
|
7504
|
+
const enforced = rules.enforcedTools ?? attributeList(events, rules.enforcedToolsAttribute);
|
|
7505
|
+
if (rules.requireDeclaredMatchesEnforced) {
|
|
7506
|
+
if (declared === void 0 || enforced === void 0) {
|
|
7507
|
+
findings.push(
|
|
7508
|
+
fail(
|
|
7509
|
+
"contract.controls.declared-matches-enforced",
|
|
7510
|
+
"Declared and enforced tool sets could not both be resolved.",
|
|
7511
|
+
runEvidence2,
|
|
7512
|
+
{ declaredDefined: declared !== void 0, enforcedDefined: enforced !== void 0 },
|
|
7513
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
7514
|
+
)
|
|
7515
|
+
);
|
|
7516
|
+
} else if (!setEqual(declared, enforced)) {
|
|
7517
|
+
findings.push(
|
|
7518
|
+
fail(
|
|
7519
|
+
"contract.controls.declared-matches-enforced",
|
|
7520
|
+
"Declared tools differ from enforced tools.",
|
|
7521
|
+
runEvidence2,
|
|
7522
|
+
sortedUnique(declared),
|
|
7523
|
+
sortedUnique(enforced)
|
|
7524
|
+
)
|
|
7525
|
+
);
|
|
7526
|
+
}
|
|
7527
|
+
}
|
|
7528
|
+
const observedTools = sortedUnique(
|
|
7529
|
+
events.filter((event) => event.kind === "TOOL" && event.status !== "running").map((event) => resolveCanonicalToolName(event))
|
|
7530
|
+
);
|
|
7531
|
+
if (rules.requireObservedWithinEnforced) {
|
|
7532
|
+
if (enforced === void 0) {
|
|
7533
|
+
findings.push(
|
|
7534
|
+
fail(
|
|
7535
|
+
"contract.controls.observed-within-enforced",
|
|
7536
|
+
"Enforced tool set unavailable for observed-within-enforced check.",
|
|
7537
|
+
runEvidence2,
|
|
7538
|
+
void 0,
|
|
7539
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
7540
|
+
)
|
|
7541
|
+
);
|
|
7542
|
+
} else {
|
|
7543
|
+
const enforcedSet = new Set(enforced);
|
|
7544
|
+
const outside = observedTools.filter((name) => !enforcedSet.has(name));
|
|
7545
|
+
if (outside.length > 0) {
|
|
7546
|
+
findings.push(
|
|
7547
|
+
fail(
|
|
7548
|
+
"contract.controls.observed-within-enforced",
|
|
7549
|
+
`Observed tools outside enforced allowlist: ${outside.join(", ")}.`,
|
|
7550
|
+
runEvidence2,
|
|
7551
|
+
sortedUnique(enforced),
|
|
7552
|
+
outside
|
|
7553
|
+
)
|
|
7554
|
+
);
|
|
7555
|
+
}
|
|
7556
|
+
}
|
|
7557
|
+
}
|
|
7558
|
+
if (rules.requireObservedWithinDeclared) {
|
|
7559
|
+
if (declared === void 0) {
|
|
7560
|
+
findings.push(
|
|
7561
|
+
fail(
|
|
7562
|
+
"contract.controls.observed-within-declared",
|
|
7563
|
+
"Declared tool set unavailable for observed-within-declared check.",
|
|
7564
|
+
runEvidence2,
|
|
7565
|
+
void 0,
|
|
7566
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
7567
|
+
)
|
|
7568
|
+
);
|
|
7569
|
+
} else {
|
|
7570
|
+
const declaredSet = new Set(declared);
|
|
7571
|
+
const outside = observedTools.filter((name) => !declaredSet.has(name));
|
|
7572
|
+
if (outside.length > 0) {
|
|
7573
|
+
findings.push(
|
|
7574
|
+
fail(
|
|
7575
|
+
"contract.controls.observed-within-declared",
|
|
7576
|
+
`Observed tools outside declared set: ${outside.join(", ")}.`,
|
|
7577
|
+
runEvidence2,
|
|
7578
|
+
sortedUnique(declared),
|
|
7579
|
+
outside
|
|
7580
|
+
)
|
|
7581
|
+
);
|
|
7582
|
+
}
|
|
7583
|
+
}
|
|
7584
|
+
}
|
|
7585
|
+
for (const stage of rules.requiredStages ?? []) {
|
|
7586
|
+
const observation = stage.observation ?? `control.${stage.stage}`;
|
|
7587
|
+
if (!observationNames.has(observation)) {
|
|
7588
|
+
findings.push(
|
|
7589
|
+
fail(
|
|
7590
|
+
`contract.controls.stage.${stage.stage}`,
|
|
7591
|
+
`Required control stage observation missing: ${observation}.`,
|
|
7592
|
+
runEvidence2,
|
|
7593
|
+
observation,
|
|
7594
|
+
[...observationNames]
|
|
7595
|
+
)
|
|
7596
|
+
);
|
|
7597
|
+
}
|
|
7598
|
+
}
|
|
7599
|
+
return findings;
|
|
7600
|
+
}
|
|
7045
7601
|
var init_control_rules = __esm({
|
|
7046
7602
|
"packages/core/src/checks/control-rules.ts"() {
|
|
7047
7603
|
init_logical_events();
|
|
@@ -7049,19 +7605,539 @@ var init_control_rules = __esm({
|
|
|
7049
7605
|
});
|
|
7050
7606
|
|
|
7051
7607
|
// packages/core/src/checks/canonical-equality.ts
|
|
7608
|
+
function sortKeysDeep(input3, depth, maxDepth) {
|
|
7609
|
+
if (depth > maxDepth) {
|
|
7610
|
+
throw new Error("AI_CHECK_CANONICAL_DEPTH_EXCEEDED");
|
|
7611
|
+
}
|
|
7612
|
+
if (input3 === null || typeof input3 !== "object") return input3;
|
|
7613
|
+
if (Array.isArray(input3)) {
|
|
7614
|
+
return input3.map((item) => sortKeysDeep(item, depth + 1, maxDepth));
|
|
7615
|
+
}
|
|
7616
|
+
const record = input3;
|
|
7617
|
+
const out = {};
|
|
7618
|
+
for (const key of Object.keys(record).sort()) {
|
|
7619
|
+
out[key] = sortKeysDeep(record[key], depth + 1, maxDepth);
|
|
7620
|
+
}
|
|
7621
|
+
return out;
|
|
7622
|
+
}
|
|
7623
|
+
function encodeCanonicalStructured(value, limits) {
|
|
7624
|
+
const maxDepth = DEFAULT_MAX_DEPTH;
|
|
7625
|
+
const maxBytes = DEFAULT_MAX_BYTES;
|
|
7626
|
+
try {
|
|
7627
|
+
const sorted = sortKeysDeep(value, 0, maxDepth);
|
|
7628
|
+
const encoded = JSON.stringify(sorted);
|
|
7629
|
+
if (encoded === void 0) {
|
|
7630
|
+
return {
|
|
7631
|
+
ok: false,
|
|
7632
|
+
reason: "Value is not JSON-serializable for canonical comparison.",
|
|
7633
|
+
code: "AI_CHECK_CANONICAL_UNSERIALIZABLE"
|
|
7634
|
+
};
|
|
7635
|
+
}
|
|
7636
|
+
if (Buffer.byteLength(encoded, "utf8") > maxBytes) {
|
|
7637
|
+
return {
|
|
7638
|
+
ok: false,
|
|
7639
|
+
reason: "Canonical structured value exceeds bounded size limit.",
|
|
7640
|
+
code: "AI_CHECK_CANONICAL_SIZE_EXCEEDED"
|
|
7641
|
+
};
|
|
7642
|
+
}
|
|
7643
|
+
return { ok: true, value: encoded };
|
|
7644
|
+
} catch (error) {
|
|
7645
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
7646
|
+
if (message === "AI_CHECK_CANONICAL_DEPTH_EXCEEDED") {
|
|
7647
|
+
return {
|
|
7648
|
+
ok: false,
|
|
7649
|
+
reason: "Canonical structured value exceeds bounded depth limit.",
|
|
7650
|
+
code: "AI_CHECK_CANONICAL_DEPTH_EXCEEDED"
|
|
7651
|
+
};
|
|
7652
|
+
}
|
|
7653
|
+
return {
|
|
7654
|
+
ok: false,
|
|
7655
|
+
reason: "Canonical structured encoding failed.",
|
|
7656
|
+
code: "AI_CHECK_CANONICAL_UNSERIALIZABLE"
|
|
7657
|
+
};
|
|
7658
|
+
}
|
|
7659
|
+
}
|
|
7660
|
+
function canonicalStructuredEqual(left, right, limits) {
|
|
7661
|
+
const a = encodeCanonicalStructured(left);
|
|
7662
|
+
if (!a.ok) return { equal: false, reason: a.reason, code: a.code };
|
|
7663
|
+
const b = encodeCanonicalStructured(right);
|
|
7664
|
+
if (!b.ok) return { equal: false, reason: b.reason, code: b.code };
|
|
7665
|
+
if (a.value !== b.value) {
|
|
7666
|
+
return {
|
|
7667
|
+
equal: false,
|
|
7668
|
+
reason: "Canonical structured values differ.",
|
|
7669
|
+
code: "AI_CHECK_CANONICAL_NOT_EQUAL"
|
|
7670
|
+
};
|
|
7671
|
+
}
|
|
7672
|
+
return { equal: true };
|
|
7673
|
+
}
|
|
7674
|
+
var DEFAULT_MAX_DEPTH, DEFAULT_MAX_BYTES;
|
|
7052
7675
|
var init_canonical_equality = __esm({
|
|
7053
7676
|
"packages/core/src/checks/canonical-equality.ts"() {
|
|
7677
|
+
DEFAULT_MAX_DEPTH = 8;
|
|
7678
|
+
DEFAULT_MAX_BYTES = 16384;
|
|
7054
7679
|
}
|
|
7055
7680
|
});
|
|
7056
7681
|
|
|
7057
7682
|
// packages/core/src/checks/tool-arguments.ts
|
|
7058
|
-
|
|
7059
|
-
"
|
|
7060
|
-
|
|
7683
|
+
function decodeToken(token) {
|
|
7684
|
+
return token.replace(/~1/g, "/").replace(/~0/g, "~");
|
|
7685
|
+
}
|
|
7686
|
+
function resolveJsonPointer(document, pointer) {
|
|
7687
|
+
if (pointer === "") {
|
|
7688
|
+
return { found: true, value: document };
|
|
7689
|
+
}
|
|
7690
|
+
if (!pointer.startsWith("/")) {
|
|
7691
|
+
return { found: false };
|
|
7692
|
+
}
|
|
7693
|
+
const tokens = pointer.slice(1).split("/").map(decodeToken);
|
|
7694
|
+
let current = document;
|
|
7695
|
+
for (const token of tokens) {
|
|
7696
|
+
if (current === null || current === void 0) {
|
|
7697
|
+
return { found: false };
|
|
7698
|
+
}
|
|
7699
|
+
if (Array.isArray(current)) {
|
|
7700
|
+
if (!/^(0|[1-9][0-9]*)$/.test(token)) {
|
|
7701
|
+
return { found: false };
|
|
7702
|
+
}
|
|
7703
|
+
const index = Number(token);
|
|
7704
|
+
if (index >= current.length) {
|
|
7705
|
+
return { found: false };
|
|
7706
|
+
}
|
|
7707
|
+
current = current[index];
|
|
7708
|
+
continue;
|
|
7709
|
+
}
|
|
7710
|
+
if (typeof current !== "object") {
|
|
7711
|
+
return { found: false };
|
|
7712
|
+
}
|
|
7713
|
+
const record = current;
|
|
7714
|
+
if (!Object.prototype.hasOwnProperty.call(record, token)) {
|
|
7715
|
+
return { found: false };
|
|
7716
|
+
}
|
|
7717
|
+
current = record[token];
|
|
7718
|
+
}
|
|
7719
|
+
return { found: true, value: current };
|
|
7720
|
+
}
|
|
7721
|
+
function jsonType(value) {
|
|
7722
|
+
if (value === null) return "null";
|
|
7723
|
+
if (Array.isArray(value)) return "array";
|
|
7724
|
+
return typeof value;
|
|
7725
|
+
}
|
|
7726
|
+
function valuesEqual(left, right) {
|
|
7727
|
+
return canonicalStructuredEqual(left, right).equal === true;
|
|
7728
|
+
}
|
|
7729
|
+
function isStructuredArgumentEvidence(value) {
|
|
7730
|
+
return value !== void 0 && value !== null && typeof value === "object";
|
|
7731
|
+
}
|
|
7732
|
+
function extractToolArgumentPayload(event) {
|
|
7733
|
+
const attrs = event.attributes ?? {};
|
|
7734
|
+
for (const key of ["arguments", "input", "toolArguments"]) {
|
|
7735
|
+
const candidate = attrs[key];
|
|
7736
|
+
if (isStructuredArgumentEvidence(candidate)) {
|
|
7737
|
+
return { present: true, value: candidate };
|
|
7738
|
+
}
|
|
7739
|
+
}
|
|
7740
|
+
const metadata = attrs.metadata;
|
|
7741
|
+
if (isStructuredArgumentEvidence(metadata) && !Array.isArray(metadata)) {
|
|
7742
|
+
const nested = metadata;
|
|
7743
|
+
for (const key of ["arguments", "input", "toolArguments"]) {
|
|
7744
|
+
const candidate = nested[key];
|
|
7745
|
+
if (isStructuredArgumentEvidence(candidate)) {
|
|
7746
|
+
return { present: true, value: candidate };
|
|
7747
|
+
}
|
|
7748
|
+
}
|
|
7749
|
+
}
|
|
7750
|
+
if (isStructuredArgumentEvidence(event.inputSummary)) {
|
|
7751
|
+
return { present: true, value: event.inputSummary };
|
|
7752
|
+
}
|
|
7753
|
+
return { present: false, value: void 0 };
|
|
7754
|
+
}
|
|
7755
|
+
function evaluateToolArgumentValue(value, check, evidencePresent) {
|
|
7756
|
+
if (!evidencePresent) {
|
|
7757
|
+
return {
|
|
7758
|
+
status: "unavailable",
|
|
7759
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
7760
|
+
message: `Structured argument evidence unavailable for tool ${check.tool} at ${check.path}.`
|
|
7761
|
+
};
|
|
7762
|
+
}
|
|
7763
|
+
const resolved = resolveJsonPointer(value, check.path);
|
|
7764
|
+
if (check.operator === "exists") {
|
|
7765
|
+
return resolved.found ? { status: "pass" } : {
|
|
7766
|
+
status: "fail",
|
|
7767
|
+
code: "AI_CHECK_TOOL_ARGUMENT_MISSING",
|
|
7768
|
+
message: `Expected path ${check.path} to exist on tool ${check.tool}.`
|
|
7769
|
+
};
|
|
7770
|
+
}
|
|
7771
|
+
if (!resolved.found) {
|
|
7772
|
+
return {
|
|
7773
|
+
status: "unavailable",
|
|
7774
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
7775
|
+
message: `Path ${check.path} missing for tool ${check.tool}.`
|
|
7776
|
+
};
|
|
7777
|
+
}
|
|
7778
|
+
if (check.operator === "type") {
|
|
7779
|
+
const actual = jsonType(resolved.value);
|
|
7780
|
+
if (check.type === void 0) {
|
|
7781
|
+
return {
|
|
7782
|
+
status: "fail",
|
|
7783
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
7784
|
+
message: "type operator requires check.type"
|
|
7785
|
+
};
|
|
7786
|
+
}
|
|
7787
|
+
return actual === check.type ? { status: "pass" } : {
|
|
7788
|
+
status: "fail",
|
|
7789
|
+
code: "AI_CHECK_TOOL_ARGUMENT_TYPE",
|
|
7790
|
+
message: `Expected type ${check.type} at ${check.path} for tool ${check.tool}; found ${actual}.`
|
|
7791
|
+
};
|
|
7792
|
+
}
|
|
7793
|
+
if (check.operator === "equals") {
|
|
7794
|
+
return valuesEqual(resolved.value, check.expected) ? { status: "pass" } : {
|
|
7795
|
+
status: "fail",
|
|
7796
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EQUALS",
|
|
7797
|
+
message: `Value at ${check.path} for tool ${check.tool} did not equal expected (bounded comparison).`
|
|
7798
|
+
};
|
|
7799
|
+
}
|
|
7800
|
+
if (check.operator === "oneOf") {
|
|
7801
|
+
const options = check.oneOf ?? [];
|
|
7802
|
+
if (options.length === 0) {
|
|
7803
|
+
return {
|
|
7804
|
+
status: "fail",
|
|
7805
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
7806
|
+
message: "oneOf operator requires a non-empty oneOf array"
|
|
7807
|
+
};
|
|
7808
|
+
}
|
|
7809
|
+
return options.some((option) => valuesEqual(resolved.value, option)) ? { status: "pass" } : {
|
|
7810
|
+
status: "fail",
|
|
7811
|
+
code: "AI_CHECK_TOOL_ARGUMENT_ONE_OF",
|
|
7812
|
+
message: `Value at ${check.path} for tool ${check.tool} was not in the allowed oneOf set.`
|
|
7813
|
+
};
|
|
7814
|
+
}
|
|
7815
|
+
return {
|
|
7816
|
+
status: "fail",
|
|
7817
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
7818
|
+
message: `Unsupported operator.`
|
|
7819
|
+
};
|
|
7820
|
+
}
|
|
7821
|
+
var init_tool_arguments = __esm({
|
|
7822
|
+
"packages/core/src/checks/tool-arguments.ts"() {
|
|
7823
|
+
init_canonical_equality();
|
|
7061
7824
|
}
|
|
7062
7825
|
});
|
|
7063
7826
|
|
|
7064
7827
|
// packages/core/src/checks/retry-safety.ts
|
|
7828
|
+
function fail2(ruleId, message, evidence, expected, actual) {
|
|
7829
|
+
return {
|
|
7830
|
+
ruleId,
|
|
7831
|
+
severity: "error",
|
|
7832
|
+
status: "fail",
|
|
7833
|
+
message,
|
|
7834
|
+
...expected !== void 0 ? { expected } : {},
|
|
7835
|
+
...actual !== void 0 ? { actual } : {},
|
|
7836
|
+
evidence: [...evidence]
|
|
7837
|
+
};
|
|
7838
|
+
}
|
|
7839
|
+
function workflowFor(event) {
|
|
7840
|
+
const attrs = event.attributes;
|
|
7841
|
+
if (!attrs || typeof attrs !== "object") return {};
|
|
7842
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
7843
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
7844
|
+
return { ...nested, ...direct };
|
|
7845
|
+
}
|
|
7846
|
+
function hasIdempotencyEvidence(event) {
|
|
7847
|
+
const workflow = workflowFor(event);
|
|
7848
|
+
if (typeof workflow.idempotencyKey === "string" && workflow.idempotencyKey.trim() !== "") {
|
|
7849
|
+
return true;
|
|
7850
|
+
}
|
|
7851
|
+
const attrs = event.attributes ?? {};
|
|
7852
|
+
if (attrs.noSideEffect === true) return true;
|
|
7853
|
+
if (attrs.sideEffect === false) return true;
|
|
7854
|
+
return false;
|
|
7855
|
+
}
|
|
7856
|
+
function eventEvidence2(event) {
|
|
7857
|
+
return {
|
|
7858
|
+
runId: event.runId,
|
|
7859
|
+
eventId: event.eventId,
|
|
7860
|
+
kind: event.kind,
|
|
7861
|
+
name: event.name,
|
|
7862
|
+
status: event.status
|
|
7863
|
+
};
|
|
7864
|
+
}
|
|
7865
|
+
function eventTime(event) {
|
|
7866
|
+
return event.startedAt ?? event.timestamp ?? "";
|
|
7867
|
+
}
|
|
7868
|
+
function sortByTime(events) {
|
|
7869
|
+
return [...events].sort((a, b) => eventTime(a).localeCompare(eventTime(b)));
|
|
7870
|
+
}
|
|
7871
|
+
function attemptNumberOf(event) {
|
|
7872
|
+
const workflow = workflowFor(event);
|
|
7873
|
+
const value = workflow.attemptNumber ?? workflow.attempt;
|
|
7874
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
7875
|
+
}
|
|
7876
|
+
function attemptIdOf(event) {
|
|
7877
|
+
const value = workflowFor(event).attemptId;
|
|
7878
|
+
return typeof value === "string" && value.trim() !== "" ? value : void 0;
|
|
7879
|
+
}
|
|
7880
|
+
function countOperationAttempts(members) {
|
|
7881
|
+
const attemptIds = new Set(
|
|
7882
|
+
members.map((event) => attemptIdOf(event)).filter((value) => value !== void 0)
|
|
7883
|
+
);
|
|
7884
|
+
if (attemptIds.size > 0) {
|
|
7885
|
+
return attemptIds.size;
|
|
7886
|
+
}
|
|
7887
|
+
const numbers = members.map((event) => attemptNumberOf(event)).filter((value) => value !== void 0).sort((a, b) => a - b);
|
|
7888
|
+
if (numbers.length > 0) {
|
|
7889
|
+
const unique = [...new Set(numbers)];
|
|
7890
|
+
const min = unique[0];
|
|
7891
|
+
const max = unique[unique.length - 1];
|
|
7892
|
+
const contiguous = unique.length === max - min + 1 && unique.every((value, index) => value === min + index);
|
|
7893
|
+
if (contiguous) {
|
|
7894
|
+
return unique.length;
|
|
7895
|
+
}
|
|
7896
|
+
return unique.length;
|
|
7897
|
+
}
|
|
7898
|
+
return members.filter((event) => event.kind === "TOOL" || event.kind === "LLM").length;
|
|
7899
|
+
}
|
|
7900
|
+
function classifyRetryAttempt(event, priorOrdered, allMembers) {
|
|
7901
|
+
const workflow = workflowFor(event);
|
|
7902
|
+
const attemptNumber2 = attemptNumberOf(event);
|
|
7903
|
+
const attemptId = attemptIdOf(event);
|
|
7904
|
+
const retryOf = workflow.retryOf;
|
|
7905
|
+
if (typeof retryOf === "string" && retryOf.trim() !== "") {
|
|
7906
|
+
const target = allMembers.find(
|
|
7907
|
+
(candidate) => candidate.eventId === retryOf || attemptIdOf(candidate) === retryOf || workflowFor(candidate).operationId === retryOf
|
|
7908
|
+
);
|
|
7909
|
+
if (!target) {
|
|
7910
|
+
return { kind: "unknown", reason: "retryOf target missing" };
|
|
7911
|
+
}
|
|
7912
|
+
if (eventTime(target) > eventTime(event)) {
|
|
7913
|
+
return { kind: "unknown", reason: "retryOf target does not precede retry" };
|
|
7914
|
+
}
|
|
7915
|
+
return { kind: "retry" };
|
|
7916
|
+
}
|
|
7917
|
+
if (attemptNumber2 !== void 0) {
|
|
7918
|
+
if (attemptNumber2 > 1) {
|
|
7919
|
+
return { kind: "retry" };
|
|
7920
|
+
}
|
|
7921
|
+
if (attemptNumber2 === 1) {
|
|
7922
|
+
return { kind: "first" };
|
|
7923
|
+
}
|
|
7924
|
+
}
|
|
7925
|
+
if (attemptId !== void 0) {
|
|
7926
|
+
const priorDistinct = priorOrdered.some((candidate) => {
|
|
7927
|
+
const priorId = attemptIdOf(candidate);
|
|
7928
|
+
return priorId !== void 0 && priorId !== attemptId;
|
|
7929
|
+
});
|
|
7930
|
+
if (priorDistinct) {
|
|
7931
|
+
return { kind: "retry" };
|
|
7932
|
+
}
|
|
7933
|
+
if (priorOrdered.length === 0) {
|
|
7934
|
+
return { kind: "first" };
|
|
7935
|
+
}
|
|
7936
|
+
}
|
|
7937
|
+
const priorFinished = priorOrdered.filter(
|
|
7938
|
+
(candidate) => candidate.status === "ok" || candidate.status === "error"
|
|
7939
|
+
);
|
|
7940
|
+
if (priorFinished.length > 0) {
|
|
7941
|
+
return { kind: "retry" };
|
|
7942
|
+
}
|
|
7943
|
+
if (priorOrdered.length === 0) {
|
|
7944
|
+
return { kind: "first" };
|
|
7945
|
+
}
|
|
7946
|
+
return { kind: "unknown", reason: "ambiguous attempt identity" };
|
|
7947
|
+
}
|
|
7948
|
+
function hasDuplicateAttemptIds(members) {
|
|
7949
|
+
const seen = /* @__PURE__ */ new Set();
|
|
7950
|
+
for (const event of members) {
|
|
7951
|
+
const id = attemptIdOf(event);
|
|
7952
|
+
if (!id) continue;
|
|
7953
|
+
if (seen.has(id)) return true;
|
|
7954
|
+
seen.add(id);
|
|
7955
|
+
}
|
|
7956
|
+
return false;
|
|
7957
|
+
}
|
|
7958
|
+
function evaluateRetrySafetyRules(events, rules, _runEvidence) {
|
|
7959
|
+
const findings = [];
|
|
7960
|
+
const byOperation = /* @__PURE__ */ new Map();
|
|
7961
|
+
for (const event of events) {
|
|
7962
|
+
const workflow = workflowFor(event);
|
|
7963
|
+
const operationId = workflow.operationId;
|
|
7964
|
+
if (!operationId) continue;
|
|
7965
|
+
const list2 = byOperation.get(operationId) ?? [];
|
|
7966
|
+
list2.push(event);
|
|
7967
|
+
byOperation.set(operationId, list2);
|
|
7968
|
+
}
|
|
7969
|
+
if (rules.maxAttempts !== void 0) {
|
|
7970
|
+
for (const [operationId, members] of byOperation) {
|
|
7971
|
+
if (hasDuplicateAttemptIds(members)) {
|
|
7972
|
+
findings.push(
|
|
7973
|
+
fail2(
|
|
7974
|
+
"contract.retry.duplicate-attempt-id",
|
|
7975
|
+
`Operation ${operationId} has duplicate attemptId values.`,
|
|
7976
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
7977
|
+
"unique attemptId",
|
|
7978
|
+
"duplicate"
|
|
7979
|
+
)
|
|
7980
|
+
);
|
|
7981
|
+
}
|
|
7982
|
+
const count = countOperationAttempts(members);
|
|
7983
|
+
if (count > rules.maxAttempts) {
|
|
7984
|
+
findings.push(
|
|
7985
|
+
fail2(
|
|
7986
|
+
"contract.retry.max-attempts",
|
|
7987
|
+
`Operation ${operationId} exceeded maxAttempts ${rules.maxAttempts}.`,
|
|
7988
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
7989
|
+
rules.maxAttempts,
|
|
7990
|
+
count
|
|
7991
|
+
)
|
|
7992
|
+
);
|
|
7993
|
+
}
|
|
7994
|
+
}
|
|
7995
|
+
}
|
|
7996
|
+
if (rules.requireTerminalResult) {
|
|
7997
|
+
for (const [operationId, members] of byOperation) {
|
|
7998
|
+
const hasTerminal = members.some((event) => event.status === "ok" || event.status === "error");
|
|
7999
|
+
if (!hasTerminal) {
|
|
8000
|
+
findings.push(
|
|
8001
|
+
fail2(
|
|
8002
|
+
"contract.retry.terminal-result",
|
|
8003
|
+
`Operation ${operationId} has no terminal ok/error result.`,
|
|
8004
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
8005
|
+
"ok|error",
|
|
8006
|
+
members.map((event) => event.status)
|
|
8007
|
+
)
|
|
8008
|
+
);
|
|
8009
|
+
}
|
|
8010
|
+
}
|
|
8011
|
+
}
|
|
8012
|
+
if (rules.fallbackOnlyAfterFailure) {
|
|
8013
|
+
for (const event of events) {
|
|
8014
|
+
const workflow = workflowFor(event);
|
|
8015
|
+
const fallbackOf = workflow.fallbackOf;
|
|
8016
|
+
if (!fallbackOf) continue;
|
|
8017
|
+
const prior = byOperation.get(fallbackOf) ?? [];
|
|
8018
|
+
const fallbackStart = eventTime(event);
|
|
8019
|
+
const earlierFailure = prior.some(
|
|
8020
|
+
(candidate) => candidate.status === "error" && eventTime(candidate) !== "" && fallbackStart !== "" && eventTime(candidate) < fallbackStart
|
|
8021
|
+
);
|
|
8022
|
+
if (!earlierFailure) {
|
|
8023
|
+
findings.push(
|
|
8024
|
+
fail2(
|
|
8025
|
+
"contract.retry.fallback-after-failure",
|
|
8026
|
+
`Fallback for ${fallbackOf} appeared without an earlier failure in the related operation.`,
|
|
8027
|
+
[eventEvidence2(event)],
|
|
8028
|
+
"earlier error attempt",
|
|
8029
|
+
{ fallbackOf }
|
|
8030
|
+
)
|
|
8031
|
+
);
|
|
8032
|
+
}
|
|
8033
|
+
}
|
|
8034
|
+
}
|
|
8035
|
+
const nonIdempotent = new Set(rules.nonIdempotentTools ?? []);
|
|
8036
|
+
if (nonIdempotent.size > 0 || rules.requireIdempotencyEvidenceForRetry) {
|
|
8037
|
+
const toolEvents = events.filter(
|
|
8038
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
8039
|
+
);
|
|
8040
|
+
const byToolOp = /* @__PURE__ */ new Map();
|
|
8041
|
+
for (const event of toolEvents) {
|
|
8042
|
+
const name = resolveCanonicalToolName(event);
|
|
8043
|
+
const workflow = workflowFor(event);
|
|
8044
|
+
const key = `${workflow.operationId ?? name}::${name}`;
|
|
8045
|
+
const list2 = byToolOp.get(key) ?? [];
|
|
8046
|
+
list2.push(event);
|
|
8047
|
+
byToolOp.set(key, list2);
|
|
8048
|
+
}
|
|
8049
|
+
for (const [, members] of byToolOp) {
|
|
8050
|
+
const ordered = sortByTime(members);
|
|
8051
|
+
let sawOk = false;
|
|
8052
|
+
let sawSideEffectOk = false;
|
|
8053
|
+
const prior = [];
|
|
8054
|
+
for (const event of ordered) {
|
|
8055
|
+
const name = resolveCanonicalToolName(event);
|
|
8056
|
+
const classification = classifyRetryAttempt(event, prior, members);
|
|
8057
|
+
if (classification.kind === "unknown" && rules.requireIdempotencyEvidenceForRetry) {
|
|
8058
|
+
findings.push(
|
|
8059
|
+
fail2(
|
|
8060
|
+
"contract.retry.ambiguous-identity",
|
|
8061
|
+
`Ambiguous attempt identity for tool ${name}; strict retry evidence cannot pass.`,
|
|
8062
|
+
[eventEvidence2(event)],
|
|
8063
|
+
"explicit attempt identity",
|
|
8064
|
+
classification.reason
|
|
8065
|
+
)
|
|
8066
|
+
);
|
|
8067
|
+
}
|
|
8068
|
+
const isRetry = classification.kind === "retry";
|
|
8069
|
+
if (isRetry && rules.requireIdempotencyEvidenceForRetry && !hasIdempotencyEvidence(event)) {
|
|
8070
|
+
findings.push(
|
|
8071
|
+
fail2(
|
|
8072
|
+
"contract.retry.idempotency-evidence",
|
|
8073
|
+
`Retry of tool ${name} lacks idempotencyKey / noSideEffect evidence.`,
|
|
8074
|
+
[eventEvidence2(event)],
|
|
8075
|
+
"idempotencyKey|noSideEffect",
|
|
8076
|
+
{ code: "AI_CHECK_RETRY_EVIDENCE_UNAVAILABLE" }
|
|
8077
|
+
)
|
|
8078
|
+
);
|
|
8079
|
+
}
|
|
8080
|
+
if (sawOk && nonIdempotent.has(name) && sawSideEffectOk && !hasIdempotencyEvidence(event)) {
|
|
8081
|
+
findings.push(
|
|
8082
|
+
fail2(
|
|
8083
|
+
"contract.retry.non-idempotent-side-effect",
|
|
8084
|
+
`Retry of non-idempotent tool ${name} after a confirmed ok side effect.`,
|
|
8085
|
+
[eventEvidence2(event)],
|
|
8086
|
+
"no retry after side effect",
|
|
8087
|
+
name
|
|
8088
|
+
)
|
|
8089
|
+
);
|
|
8090
|
+
}
|
|
8091
|
+
if (event.status === "ok") {
|
|
8092
|
+
sawOk = true;
|
|
8093
|
+
if (nonIdempotent.has(name) && !hasIdempotencyEvidence(event)) {
|
|
8094
|
+
sawSideEffectOk = true;
|
|
8095
|
+
}
|
|
8096
|
+
}
|
|
8097
|
+
prior.push(event);
|
|
8098
|
+
}
|
|
8099
|
+
}
|
|
8100
|
+
}
|
|
8101
|
+
if (rules.requireRecoveredFailureVisible) {
|
|
8102
|
+
for (const [operationId, members] of byOperation) {
|
|
8103
|
+
const ordered = sortByTime(members);
|
|
8104
|
+
const toolish = ordered.filter(
|
|
8105
|
+
(event) => (event.kind === "TOOL" || event.kind === "LLM") && (event.status === "ok" || event.status === "error")
|
|
8106
|
+
);
|
|
8107
|
+
if (toolish.length < 2) continue;
|
|
8108
|
+
let sawEarlierError = false;
|
|
8109
|
+
let sawLaterOkAfterError = false;
|
|
8110
|
+
for (const event of toolish) {
|
|
8111
|
+
if (event.status === "error") {
|
|
8112
|
+
sawEarlierError = true;
|
|
8113
|
+
} else if (event.status === "ok" && sawEarlierError) {
|
|
8114
|
+
sawLaterOkAfterError = true;
|
|
8115
|
+
}
|
|
8116
|
+
}
|
|
8117
|
+
const hasOk = toolish.some((event) => event.status === "ok");
|
|
8118
|
+
const attemptish = toolish.some((event) => {
|
|
8119
|
+
const meta2 = workflowFor(event);
|
|
8120
|
+
return meta2.attemptId !== void 0 || meta2.attemptNumber !== void 0 || meta2.attempt !== void 0 || meta2.retryOf !== void 0;
|
|
8121
|
+
}) || toolish.length > 1;
|
|
8122
|
+
if (!attemptish || !hasOk) continue;
|
|
8123
|
+
if (!sawLaterOkAfterError) {
|
|
8124
|
+
findings.push(
|
|
8125
|
+
fail2(
|
|
8126
|
+
"contract.retry.recovered-failure-visible",
|
|
8127
|
+
`Operation ${operationId} does not show an earlier error attempt before a later success in the same chain.`,
|
|
8128
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
8129
|
+
"earlier error then later ok",
|
|
8130
|
+
{
|
|
8131
|
+
sawEarlierError,
|
|
8132
|
+
statuses: toolish.map((event) => event.status)
|
|
8133
|
+
}
|
|
8134
|
+
)
|
|
8135
|
+
);
|
|
8136
|
+
}
|
|
8137
|
+
}
|
|
8138
|
+
}
|
|
8139
|
+
return findings;
|
|
8140
|
+
}
|
|
7065
8141
|
var init_retry_safety = __esm({
|
|
7066
8142
|
"packages/core/src/checks/retry-safety.ts"() {
|
|
7067
8143
|
init_logical_events();
|
|
@@ -7070,6 +8146,478 @@ var init_retry_safety = __esm({
|
|
|
7070
8146
|
});
|
|
7071
8147
|
|
|
7072
8148
|
// packages/core/src/checks/recovery-operations.ts
|
|
8149
|
+
function fail3(ruleId, message, evidence, expected, actual) {
|
|
8150
|
+
return {
|
|
8151
|
+
ruleId,
|
|
8152
|
+
severity: "error",
|
|
8153
|
+
status: "fail",
|
|
8154
|
+
message,
|
|
8155
|
+
...expected !== void 0 ? { expected } : {},
|
|
8156
|
+
...actual !== void 0 ? { actual } : {},
|
|
8157
|
+
evidence: [...evidence]
|
|
8158
|
+
};
|
|
8159
|
+
}
|
|
8160
|
+
function workflowFor2(event) {
|
|
8161
|
+
const attrs = event.attributes;
|
|
8162
|
+
if (!attrs || typeof attrs !== "object") return {};
|
|
8163
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
8164
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
8165
|
+
return { ...nested, ...direct };
|
|
8166
|
+
}
|
|
8167
|
+
function eventEvidence3(event) {
|
|
8168
|
+
return {
|
|
8169
|
+
runId: event.runId,
|
|
8170
|
+
eventId: event.eventId,
|
|
8171
|
+
kind: event.kind,
|
|
8172
|
+
name: event.name,
|
|
8173
|
+
status: event.status
|
|
8174
|
+
};
|
|
8175
|
+
}
|
|
8176
|
+
function eventTime2(event) {
|
|
8177
|
+
return event.startedAt ?? event.timestamp ?? "";
|
|
8178
|
+
}
|
|
8179
|
+
function sortByTime2(events) {
|
|
8180
|
+
return [...events].sort((a, b) => eventTime2(a).localeCompare(eventTime2(b)));
|
|
8181
|
+
}
|
|
8182
|
+
function errorCodeOf(event) {
|
|
8183
|
+
if (typeof event.error?.code === "string" && event.error.code.trim() !== "") {
|
|
8184
|
+
return event.error.code.trim();
|
|
8185
|
+
}
|
|
8186
|
+
const attrs = event.attributes ?? {};
|
|
8187
|
+
for (const key of ["errorCode", "code"]) {
|
|
8188
|
+
const value = attrs[key];
|
|
8189
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
8190
|
+
}
|
|
8191
|
+
const nested = attrs.error;
|
|
8192
|
+
if (nested && typeof nested === "object") {
|
|
8193
|
+
const code = nested.code;
|
|
8194
|
+
if (typeof code === "string" && code.trim() !== "") return code.trim();
|
|
8195
|
+
}
|
|
8196
|
+
return void 0;
|
|
8197
|
+
}
|
|
8198
|
+
function argumentDigestOf(event) {
|
|
8199
|
+
const attrs = event.attributes ?? {};
|
|
8200
|
+
for (const key of ["argumentsDigest", "inputDigest", "toolArgumentsDigest"]) {
|
|
8201
|
+
const value = attrs[key];
|
|
8202
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
8203
|
+
}
|
|
8204
|
+
const commitment = attrs.omittedPayloadCommitment;
|
|
8205
|
+
if (commitment && typeof commitment === "object") {
|
|
8206
|
+
const digest = commitment.digest;
|
|
8207
|
+
if (typeof digest === "string" && digest.trim() !== "") return digest.trim();
|
|
8208
|
+
}
|
|
8209
|
+
return void 0;
|
|
8210
|
+
}
|
|
8211
|
+
function argumentFingerprint(event) {
|
|
8212
|
+
const payload = extractToolArgumentPayload(event);
|
|
8213
|
+
if (payload.present) {
|
|
8214
|
+
const encoded = encodeCanonicalStructured(payload.value);
|
|
8215
|
+
if (!encoded.ok) return { kind: "missing" };
|
|
8216
|
+
return { kind: "structured", value: encoded.value };
|
|
8217
|
+
}
|
|
8218
|
+
const digest = argumentDigestOf(event);
|
|
8219
|
+
if (digest) return { kind: "digest", value: digest };
|
|
8220
|
+
return { kind: "missing" };
|
|
8221
|
+
}
|
|
8222
|
+
function sameArguments(left, right) {
|
|
8223
|
+
const a = argumentFingerprint(left);
|
|
8224
|
+
const b = argumentFingerprint(right);
|
|
8225
|
+
if (a.kind === "missing" || b.kind === "missing") {
|
|
8226
|
+
return {
|
|
8227
|
+
ok: false,
|
|
8228
|
+
reason: "Structured or digest argument evidence unavailable for same-arguments check.",
|
|
8229
|
+
code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_UNAVAILABLE"
|
|
8230
|
+
};
|
|
8231
|
+
}
|
|
8232
|
+
if (a.kind !== b.kind) {
|
|
8233
|
+
return {
|
|
8234
|
+
ok: false,
|
|
8235
|
+
reason: "Argument evidence kinds differ across attempts (structured vs digest).",
|
|
8236
|
+
code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_MISMATCH"
|
|
8237
|
+
};
|
|
8238
|
+
}
|
|
8239
|
+
if (a.value !== b.value) {
|
|
8240
|
+
return {
|
|
8241
|
+
ok: false,
|
|
8242
|
+
reason: "Retry arguments do not match prior attempt.",
|
|
8243
|
+
code: "AI_CHECK_RECOVERY_ARGUMENTS_DIFFER"
|
|
8244
|
+
};
|
|
8245
|
+
}
|
|
8246
|
+
return { ok: true };
|
|
8247
|
+
}
|
|
8248
|
+
function isUnevaluableWriteCompletion(event) {
|
|
8249
|
+
if (event.status === "running" || event.status === "unknown") return true;
|
|
8250
|
+
const attrs = event.attributes ?? {};
|
|
8251
|
+
if (attrs.timeout === true) return true;
|
|
8252
|
+
if (attrs.completionState === "unknown" || attrs.completionState === "timeout") return true;
|
|
8253
|
+
const code = errorCodeOf(event)?.toUpperCase();
|
|
8254
|
+
if (code === "TIMEOUT" || code === "UNKNOWN_COMPLETION" || code === "DEADLINE_EXCEEDED") {
|
|
8255
|
+
return true;
|
|
8256
|
+
}
|
|
8257
|
+
return false;
|
|
8258
|
+
}
|
|
8259
|
+
function collectExplicitReferences(event) {
|
|
8260
|
+
const refs = /* @__PURE__ */ new Set();
|
|
8261
|
+
const attrs = event.attributes ?? {};
|
|
8262
|
+
const candidates = [
|
|
8263
|
+
attrs.referencedEventId,
|
|
8264
|
+
attrs.referencedEventIds,
|
|
8265
|
+
attrs.toolResultEventId,
|
|
8266
|
+
attrs.toolResultEventIds,
|
|
8267
|
+
attrs.inputEventIds,
|
|
8268
|
+
attrs.dependsOnEventIds,
|
|
8269
|
+
attrs.evidence,
|
|
8270
|
+
workflowFor2(event).toolCallId
|
|
8271
|
+
];
|
|
8272
|
+
for (const candidate of candidates) {
|
|
8273
|
+
if (typeof candidate === "string" && candidate.trim() !== "") {
|
|
8274
|
+
refs.add(candidate.trim());
|
|
8275
|
+
continue;
|
|
8276
|
+
}
|
|
8277
|
+
if (Array.isArray(candidate)) {
|
|
8278
|
+
for (const item of candidate) {
|
|
8279
|
+
if (typeof item === "string" && item.trim() !== "") refs.add(item.trim());
|
|
8280
|
+
if (item && typeof item === "object") {
|
|
8281
|
+
const id = item.eventId;
|
|
8282
|
+
if (typeof id === "string" && id.trim() !== "") refs.add(id.trim());
|
|
8283
|
+
}
|
|
8284
|
+
}
|
|
8285
|
+
continue;
|
|
8286
|
+
}
|
|
8287
|
+
if (candidate && typeof candidate === "object") {
|
|
8288
|
+
const obj = candidate;
|
|
8289
|
+
if (typeof obj.eventId === "string" && obj.eventId.trim() !== "") {
|
|
8290
|
+
refs.add(obj.eventId.trim());
|
|
8291
|
+
}
|
|
8292
|
+
if (Array.isArray(obj.eventIds)) {
|
|
8293
|
+
for (const id of obj.eventIds) {
|
|
8294
|
+
if (typeof id === "string" && id.trim() !== "") refs.add(id.trim());
|
|
8295
|
+
}
|
|
8296
|
+
}
|
|
8297
|
+
}
|
|
8298
|
+
}
|
|
8299
|
+
return refs;
|
|
8300
|
+
}
|
|
8301
|
+
function llmReferencesTool(llm, toolEvent, requireExplicit) {
|
|
8302
|
+
if (!requireExplicit) {
|
|
8303
|
+
return eventTime2(llm) >= eventTime2(toolEvent);
|
|
8304
|
+
}
|
|
8305
|
+
const refs = collectExplicitReferences(llm);
|
|
8306
|
+
if (refs.has(toolEvent.eventId)) return true;
|
|
8307
|
+
const toolCallId = workflowFor2(toolEvent).toolCallId;
|
|
8308
|
+
if (toolCallId && refs.has(toolCallId)) return true;
|
|
8309
|
+
return false;
|
|
8310
|
+
}
|
|
8311
|
+
function attemptIdOf2(event) {
|
|
8312
|
+
const value = workflowFor2(event).attemptId;
|
|
8313
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
8314
|
+
}
|
|
8315
|
+
function hasDuplicateAttemptIds2(members) {
|
|
8316
|
+
const seen = /* @__PURE__ */ new Set();
|
|
8317
|
+
for (const event of members) {
|
|
8318
|
+
const id = attemptIdOf2(event);
|
|
8319
|
+
if (!id) continue;
|
|
8320
|
+
if (seen.has(id)) return true;
|
|
8321
|
+
seen.add(id);
|
|
8322
|
+
}
|
|
8323
|
+
return false;
|
|
8324
|
+
}
|
|
8325
|
+
function findRetryTarget(members, retryOf) {
|
|
8326
|
+
return members.find(
|
|
8327
|
+
(candidate) => candidate.eventId === retryOf || attemptIdOf2(candidate) === retryOf || workflowFor2(candidate).operationId === retryOf
|
|
8328
|
+
);
|
|
8329
|
+
}
|
|
8330
|
+
function groupToolAttempts(events, toolName2) {
|
|
8331
|
+
const members = events.filter(
|
|
8332
|
+
(event) => event.kind === "TOOL" && resolveCanonicalToolName(event) === toolName2
|
|
8333
|
+
);
|
|
8334
|
+
const parent = /* @__PURE__ */ new Map();
|
|
8335
|
+
const ensure = (id) => {
|
|
8336
|
+
if (!parent.has(id)) parent.set(id, id);
|
|
8337
|
+
return id;
|
|
8338
|
+
};
|
|
8339
|
+
const find = (id) => {
|
|
8340
|
+
ensure(id);
|
|
8341
|
+
let cur = id;
|
|
8342
|
+
while (parent.get(cur) !== cur) {
|
|
8343
|
+
cur = parent.get(cur);
|
|
8344
|
+
}
|
|
8345
|
+
let walk = id;
|
|
8346
|
+
while (parent.get(walk) !== cur) {
|
|
8347
|
+
const next = parent.get(walk);
|
|
8348
|
+
parent.set(walk, cur);
|
|
8349
|
+
walk = next;
|
|
8350
|
+
}
|
|
8351
|
+
return cur;
|
|
8352
|
+
};
|
|
8353
|
+
const union = (a, b) => {
|
|
8354
|
+
const ra = find(a);
|
|
8355
|
+
const rb = find(b);
|
|
8356
|
+
if (ra !== rb) parent.set(ra, rb);
|
|
8357
|
+
};
|
|
8358
|
+
const nodeId = (event) => {
|
|
8359
|
+
const operationId = workflowFor2(event).operationId;
|
|
8360
|
+
if (typeof operationId === "string" && operationId.trim() !== "") {
|
|
8361
|
+
return `op:${operationId.trim()}`;
|
|
8362
|
+
}
|
|
8363
|
+
return `event:${event.eventId}`;
|
|
8364
|
+
};
|
|
8365
|
+
for (const event of members) {
|
|
8366
|
+
ensure(nodeId(event));
|
|
8367
|
+
}
|
|
8368
|
+
for (const event of members) {
|
|
8369
|
+
const retryOf = workflowFor2(event).retryOf;
|
|
8370
|
+
if (typeof retryOf !== "string" || retryOf.trim() === "") continue;
|
|
8371
|
+
const target = findRetryTarget(members, retryOf.trim());
|
|
8372
|
+
if (target) {
|
|
8373
|
+
union(nodeId(event), nodeId(target));
|
|
8374
|
+
}
|
|
8375
|
+
}
|
|
8376
|
+
const groups = /* @__PURE__ */ new Map();
|
|
8377
|
+
for (const event of members) {
|
|
8378
|
+
const key = find(nodeId(event));
|
|
8379
|
+
const list2 = groups.get(key) ?? [];
|
|
8380
|
+
list2.push(event);
|
|
8381
|
+
groups.set(key, list2);
|
|
8382
|
+
}
|
|
8383
|
+
return groups;
|
|
8384
|
+
}
|
|
8385
|
+
function latestAttempt(ordered) {
|
|
8386
|
+
if (ordered.length === 0) return void 0;
|
|
8387
|
+
return ordered[ordered.length - 1];
|
|
8388
|
+
}
|
|
8389
|
+
function evaluateRecoveryOperations(events, operations) {
|
|
8390
|
+
const findings = [];
|
|
8391
|
+
const llmEvents = events.filter((event) => event.kind === "LLM");
|
|
8392
|
+
for (const operation of operations) {
|
|
8393
|
+
const sideEffectClass = operation.sideEffectClass ?? "read";
|
|
8394
|
+
const groups = groupToolAttempts(events, operation.tool);
|
|
8395
|
+
if (groups.size === 0) {
|
|
8396
|
+
if (operation.requireTerminalSuccess) {
|
|
8397
|
+
findings.push(
|
|
8398
|
+
fail3(
|
|
8399
|
+
"contract.retry.operations.terminal-success",
|
|
8400
|
+
`Recovery operation for tool ${operation.tool} requires a terminal success but no attempts were observed.`,
|
|
8401
|
+
[],
|
|
8402
|
+
"ok",
|
|
8403
|
+
"missing"
|
|
8404
|
+
)
|
|
8405
|
+
);
|
|
8406
|
+
}
|
|
8407
|
+
continue;
|
|
8408
|
+
}
|
|
8409
|
+
for (const [operationKey, members] of groups) {
|
|
8410
|
+
const ordered = sortByTime2(members);
|
|
8411
|
+
if (hasDuplicateAttemptIds2(members)) {
|
|
8412
|
+
findings.push(
|
|
8413
|
+
fail3(
|
|
8414
|
+
"contract.retry.operations.duplicate-attempt-id",
|
|
8415
|
+
`Tool ${operation.tool} operation ${operationKey} has duplicate attemptId values.`,
|
|
8416
|
+
members.slice(0, 4).map(eventEvidence3),
|
|
8417
|
+
"unique attemptId",
|
|
8418
|
+
"duplicate"
|
|
8419
|
+
)
|
|
8420
|
+
);
|
|
8421
|
+
}
|
|
8422
|
+
if (operation.maxAttempts !== void 0) {
|
|
8423
|
+
const count = countOperationAttempts(members);
|
|
8424
|
+
if (count > operation.maxAttempts) {
|
|
8425
|
+
findings.push(
|
|
8426
|
+
fail3(
|
|
8427
|
+
"contract.retry.operations.max-attempts",
|
|
8428
|
+
`Tool ${operation.tool} operation ${operationKey} exceeded maxAttempts ${operation.maxAttempts}.`,
|
|
8429
|
+
members.slice(0, 4).map(eventEvidence3),
|
|
8430
|
+
operation.maxAttempts,
|
|
8431
|
+
count
|
|
8432
|
+
)
|
|
8433
|
+
);
|
|
8434
|
+
}
|
|
8435
|
+
}
|
|
8436
|
+
if (sideEffectClass === "write") {
|
|
8437
|
+
for (const event of ordered) {
|
|
8438
|
+
if (isUnevaluableWriteCompletion(event)) {
|
|
8439
|
+
findings.push(
|
|
8440
|
+
fail3(
|
|
8441
|
+
"contract.retry.operations.write-completion-unevaluable",
|
|
8442
|
+
`Write tool ${operation.tool} has timeout/unknown/running completion that cannot be verified as an authoritative write result.`,
|
|
8443
|
+
[eventEvidence3(event)],
|
|
8444
|
+
"authoritative write completion observation",
|
|
8445
|
+
{
|
|
8446
|
+
code: "AI_CHECK_RECOVERY_WRITE_COMPLETION_UNAVAILABLE",
|
|
8447
|
+
status: event.status
|
|
8448
|
+
}
|
|
8449
|
+
)
|
|
8450
|
+
);
|
|
8451
|
+
}
|
|
8452
|
+
}
|
|
8453
|
+
}
|
|
8454
|
+
const prior = [];
|
|
8455
|
+
let sawEarlierError = false;
|
|
8456
|
+
let sawLaterOkAfterError = false;
|
|
8457
|
+
let lastOk;
|
|
8458
|
+
let identityUnavailable = false;
|
|
8459
|
+
for (const event of ordered) {
|
|
8460
|
+
const classification = classifyRetryAttempt(event, prior, members);
|
|
8461
|
+
if (classification.kind === "unknown") {
|
|
8462
|
+
identityUnavailable = true;
|
|
8463
|
+
findings.push(
|
|
8464
|
+
fail3(
|
|
8465
|
+
"contract.retry.operations.attempt-identity",
|
|
8466
|
+
`Tool ${operation.tool} operation ${operationKey} has ambiguous attempt identity (${classification.reason}).`,
|
|
8467
|
+
[eventEvidence3(event)],
|
|
8468
|
+
"explicit operationId/attemptId/retryOf chronology",
|
|
8469
|
+
{ code: "AI_CHECK_RECOVERY_ATTEMPT_IDENTITY_UNAVAILABLE", reason: classification.reason }
|
|
8470
|
+
)
|
|
8471
|
+
);
|
|
8472
|
+
}
|
|
8473
|
+
const isRetry = classification.kind === "retry";
|
|
8474
|
+
if (isRetry && operation.requireFailureBeforeRetry) {
|
|
8475
|
+
const earlierFailure = prior.some(
|
|
8476
|
+
(candidate) => candidate.status === "error" && eventTime2(candidate) !== "" && eventTime2(event) !== "" && eventTime2(candidate) < eventTime2(event)
|
|
8477
|
+
);
|
|
8478
|
+
if (!earlierFailure) {
|
|
8479
|
+
findings.push(
|
|
8480
|
+
fail3(
|
|
8481
|
+
"contract.retry.operations.failure-before-retry",
|
|
8482
|
+
`Retry of tool ${operation.tool} appeared without an earlier failure in the same operation.`,
|
|
8483
|
+
[eventEvidence3(event)],
|
|
8484
|
+
"earlier error attempt",
|
|
8485
|
+
{ operationKey }
|
|
8486
|
+
)
|
|
8487
|
+
);
|
|
8488
|
+
}
|
|
8489
|
+
}
|
|
8490
|
+
if (isRetry && operation.retryableErrors?.codes && operation.retryableErrors.codes.length > 0) {
|
|
8491
|
+
const allowed = new Set(operation.retryableErrors.codes);
|
|
8492
|
+
const priorErrors = prior.filter((candidate) => candidate.status === "error");
|
|
8493
|
+
const lastError = priorErrors[priorErrors.length - 1];
|
|
8494
|
+
if (lastError) {
|
|
8495
|
+
const code = errorCodeOf(lastError);
|
|
8496
|
+
if (!code || !allowed.has(code)) {
|
|
8497
|
+
findings.push(
|
|
8498
|
+
fail3(
|
|
8499
|
+
"contract.retry.operations.retryable-error",
|
|
8500
|
+
`Retry of tool ${operation.tool} followed a non-retryable or missing error code.`,
|
|
8501
|
+
[eventEvidence3(event), eventEvidence3(lastError)],
|
|
8502
|
+
[...allowed],
|
|
8503
|
+
code ?? null
|
|
8504
|
+
)
|
|
8505
|
+
);
|
|
8506
|
+
}
|
|
8507
|
+
} else {
|
|
8508
|
+
findings.push(
|
|
8509
|
+
fail3(
|
|
8510
|
+
"contract.retry.operations.retryable-error",
|
|
8511
|
+
`Retry of tool ${operation.tool} has no prior error code evidence.`,
|
|
8512
|
+
[eventEvidence3(event)],
|
|
8513
|
+
[...allowed],
|
|
8514
|
+
null
|
|
8515
|
+
)
|
|
8516
|
+
);
|
|
8517
|
+
}
|
|
8518
|
+
}
|
|
8519
|
+
if (isRetry && (operation.requireSameArguments === true || operation.requireSameArguments === "structured-or-digest")) {
|
|
8520
|
+
const baseline = prior[0] ?? prior[prior.length - 1];
|
|
8521
|
+
if (baseline) {
|
|
8522
|
+
const comparison = sameArguments(baseline, event);
|
|
8523
|
+
if (!comparison.ok) {
|
|
8524
|
+
findings.push(
|
|
8525
|
+
fail3(
|
|
8526
|
+
"contract.retry.operations.same-arguments",
|
|
8527
|
+
comparison.reason,
|
|
8528
|
+
[eventEvidence3(baseline), eventEvidence3(event)],
|
|
8529
|
+
"matching structured args or digests",
|
|
8530
|
+
{ code: comparison.code }
|
|
8531
|
+
)
|
|
8532
|
+
);
|
|
8533
|
+
}
|
|
8534
|
+
} else {
|
|
8535
|
+
findings.push(
|
|
8536
|
+
fail3(
|
|
8537
|
+
"contract.retry.operations.same-arguments",
|
|
8538
|
+
"Structured or digest argument evidence unavailable for same-arguments check.",
|
|
8539
|
+
[eventEvidence3(event)],
|
|
8540
|
+
"matching structured args or digests",
|
|
8541
|
+
{ code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_UNAVAILABLE" }
|
|
8542
|
+
)
|
|
8543
|
+
);
|
|
8544
|
+
}
|
|
8545
|
+
}
|
|
8546
|
+
if (event.status === "error") {
|
|
8547
|
+
sawEarlierError = true;
|
|
8548
|
+
} else if (event.status === "ok" && sawEarlierError) {
|
|
8549
|
+
sawLaterOkAfterError = true;
|
|
8550
|
+
}
|
|
8551
|
+
if (event.status === "ok") {
|
|
8552
|
+
lastOk = event;
|
|
8553
|
+
}
|
|
8554
|
+
prior.push(event);
|
|
8555
|
+
}
|
|
8556
|
+
const latest = latestAttempt(ordered);
|
|
8557
|
+
if (operation.requireTerminalSuccess) {
|
|
8558
|
+
if (!latest || latest.status !== "ok") {
|
|
8559
|
+
findings.push(
|
|
8560
|
+
fail3(
|
|
8561
|
+
"contract.retry.operations.terminal-success",
|
|
8562
|
+
`Tool ${operation.tool} operation ${operationKey} does not terminate with ok on the latest attempt.`,
|
|
8563
|
+
ordered.slice(0, 4).map(eventEvidence3),
|
|
8564
|
+
"latest attempt ok",
|
|
8565
|
+
latest?.status ?? "missing"
|
|
8566
|
+
)
|
|
8567
|
+
);
|
|
8568
|
+
}
|
|
8569
|
+
}
|
|
8570
|
+
if (operation.requireRecoveredFailureVisible) {
|
|
8571
|
+
const latestOk = latest?.status === "ok";
|
|
8572
|
+
if (ordered.length > 1 && latestOk && !sawLaterOkAfterError) {
|
|
8573
|
+
findings.push(
|
|
8574
|
+
fail3(
|
|
8575
|
+
"contract.retry.operations.recovered-failure-visible",
|
|
8576
|
+
`Tool ${operation.tool} operation ${operationKey} does not show an earlier error before later success.`,
|
|
8577
|
+
ordered.slice(0, 4).map(eventEvidence3),
|
|
8578
|
+
"earlier error then later ok",
|
|
8579
|
+
{ sawEarlierError }
|
|
8580
|
+
)
|
|
8581
|
+
);
|
|
8582
|
+
}
|
|
8583
|
+
}
|
|
8584
|
+
const dependency = operation.successfulResultDependency;
|
|
8585
|
+
if (dependency?.consumerKind === "LLM") {
|
|
8586
|
+
const successForDependency = latest?.status === "ok" ? latest : identityUnavailable ? void 0 : lastOk;
|
|
8587
|
+
if (successForDependency) {
|
|
8588
|
+
if (sideEffectClass === "write" && isUnevaluableWriteCompletion(successForDependency)) {
|
|
8589
|
+
findings.push(
|
|
8590
|
+
fail3(
|
|
8591
|
+
"contract.retry.operations.successful-result-dependency",
|
|
8592
|
+
`Successful ${operation.tool} result dependency is unevaluable because write completion evidence is unavailable.`,
|
|
8593
|
+
[eventEvidence3(successForDependency)],
|
|
8594
|
+
"LLM reference to successful tool event",
|
|
8595
|
+
{ code: "AI_CHECK_RECOVERY_WRITE_COMPLETION_UNAVAILABLE" }
|
|
8596
|
+
)
|
|
8597
|
+
);
|
|
8598
|
+
} else {
|
|
8599
|
+
const requireExplicit = dependency.requireExplicitReference === true;
|
|
8600
|
+
const referenced = llmEvents.some(
|
|
8601
|
+
(llmEvent) => llmReferencesTool(llmEvent, successForDependency, requireExplicit)
|
|
8602
|
+
);
|
|
8603
|
+
if (!referenced) {
|
|
8604
|
+
findings.push(
|
|
8605
|
+
fail3(
|
|
8606
|
+
"contract.retry.operations.successful-result-dependency",
|
|
8607
|
+
requireExplicit ? `Successful ${operation.tool} result is not explicitly referenced by a later LLM event.` : `Successful ${operation.tool} result has no later LLM consumer.`,
|
|
8608
|
+
[eventEvidence3(successForDependency)],
|
|
8609
|
+
"LLM reference to successful tool event",
|
|
8610
|
+
{ code: "AI_CHECK_RECOVERY_RESULT_DEPENDENCY_MISSING" }
|
|
8611
|
+
)
|
|
8612
|
+
);
|
|
8613
|
+
}
|
|
8614
|
+
}
|
|
8615
|
+
}
|
|
8616
|
+
}
|
|
8617
|
+
}
|
|
8618
|
+
}
|
|
8619
|
+
return findings;
|
|
8620
|
+
}
|
|
7073
8621
|
var init_recovery_operations = __esm({
|
|
7074
8622
|
"packages/core/src/checks/recovery-operations.ts"() {
|
|
7075
8623
|
init_canonical_equality();
|
|
@@ -7081,6 +8629,23 @@ var init_recovery_operations = __esm({
|
|
|
7081
8629
|
});
|
|
7082
8630
|
|
|
7083
8631
|
// packages/core/src/checks/contract.ts
|
|
8632
|
+
function contractFailFinding(ruleId, message, evidence, expected, actual) {
|
|
8633
|
+
return {
|
|
8634
|
+
ruleId,
|
|
8635
|
+
severity: "error",
|
|
8636
|
+
status: "fail",
|
|
8637
|
+
message,
|
|
8638
|
+
...expected !== void 0 ? { expected } : {},
|
|
8639
|
+
...actual !== void 0 ? { actual } : {},
|
|
8640
|
+
evidence: [...evidence]
|
|
8641
|
+
};
|
|
8642
|
+
}
|
|
8643
|
+
function normalizeStatus(status) {
|
|
8644
|
+
if (status === "ok" || status === "error" || status === "running") return status;
|
|
8645
|
+
if (status === "success") return "ok";
|
|
8646
|
+
if (status === "failed") return "error";
|
|
8647
|
+
return "error";
|
|
8648
|
+
}
|
|
7084
8649
|
function cloneBody(body) {
|
|
7085
8650
|
return {
|
|
7086
8651
|
...body.run ? { run: { ...body.run } } : {},
|
|
@@ -7133,6 +8698,441 @@ function cloneBody(body) {
|
|
|
7133
8698
|
function bodyHasRules(body) {
|
|
7134
8699
|
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
7135
8700
|
}
|
|
8701
|
+
function parseEvidenceEventIds(evidence) {
|
|
8702
|
+
if (evidence === void 0 || evidence === null) {
|
|
8703
|
+
return { status: "missing" };
|
|
8704
|
+
}
|
|
8705
|
+
if (typeof evidence === "string") {
|
|
8706
|
+
const trimmed = evidence.trim();
|
|
8707
|
+
if (trimmed === "") {
|
|
8708
|
+
return { status: "invalid", reason: "empty evidence string" };
|
|
8709
|
+
}
|
|
8710
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
8711
|
+
}
|
|
8712
|
+
if (typeof evidence !== "object" || Array.isArray(evidence)) {
|
|
8713
|
+
return {
|
|
8714
|
+
status: "invalid",
|
|
8715
|
+
reason: "evidence must be a string event id or { eventId } / { eventIds }"
|
|
8716
|
+
};
|
|
8717
|
+
}
|
|
8718
|
+
const record = evidence;
|
|
8719
|
+
if (typeof record.eventId === "string") {
|
|
8720
|
+
const trimmed = record.eventId.trim();
|
|
8721
|
+
if (trimmed === "") {
|
|
8722
|
+
return { status: "invalid", reason: "empty evidence.eventId" };
|
|
8723
|
+
}
|
|
8724
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
8725
|
+
}
|
|
8726
|
+
if (Array.isArray(record.eventIds)) {
|
|
8727
|
+
if (record.eventIds.length === 0) {
|
|
8728
|
+
return { status: "invalid", reason: "empty evidence.eventIds" };
|
|
8729
|
+
}
|
|
8730
|
+
if (record.eventIds.length > MAX_EVIDENCE_EVENT_IDS) {
|
|
8731
|
+
return {
|
|
8732
|
+
status: "invalid",
|
|
8733
|
+
reason: `evidence.eventIds exceeds bound of ${MAX_EVIDENCE_EVENT_IDS}`
|
|
8734
|
+
};
|
|
8735
|
+
}
|
|
8736
|
+
const eventIds = [];
|
|
8737
|
+
for (const item of record.eventIds) {
|
|
8738
|
+
if (typeof item !== "string" || item.trim() === "") {
|
|
8739
|
+
return { status: "invalid", reason: "evidence.eventIds contains a non-string id" };
|
|
8740
|
+
}
|
|
8741
|
+
eventIds.push(item.trim());
|
|
8742
|
+
}
|
|
8743
|
+
return { status: "ok", eventIds };
|
|
8744
|
+
}
|
|
8745
|
+
return {
|
|
8746
|
+
status: "invalid",
|
|
8747
|
+
reason: "evidence must include eventId or eventIds"
|
|
8748
|
+
};
|
|
8749
|
+
}
|
|
8750
|
+
function provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds) {
|
|
8751
|
+
const findings = [];
|
|
8752
|
+
const evidenceBase = [
|
|
8753
|
+
{
|
|
8754
|
+
runId: outcome.runId,
|
|
8755
|
+
eventId: outcome.outcomeId,
|
|
8756
|
+
kind: "OUTCOME",
|
|
8757
|
+
name: outcome.name,
|
|
8758
|
+
path: `outcome.${outcome.name}`
|
|
8759
|
+
}
|
|
8760
|
+
];
|
|
8761
|
+
if (requireProvenance.method) {
|
|
8762
|
+
if (outcome.method === void 0 || outcome.method.trim() === "") {
|
|
8763
|
+
findings.push(
|
|
8764
|
+
contractFailFinding(
|
|
8765
|
+
"contract.observation.provenance.method",
|
|
8766
|
+
`Observation ${outcome.name} is missing a structural method.`,
|
|
8767
|
+
evidenceBase,
|
|
8768
|
+
{ method: true },
|
|
8769
|
+
{ name: outcome.name, method: outcome.method ?? null }
|
|
8770
|
+
)
|
|
8771
|
+
);
|
|
8772
|
+
} else if (!METHOD_VOCABULARY.has(outcome.method)) {
|
|
8773
|
+
findings.push(
|
|
8774
|
+
contractFailFinding(
|
|
8775
|
+
"contract.observation.provenance.method",
|
|
8776
|
+
`Observation ${outcome.name} method ${outcome.method} is outside the bounded ObservedOutcomeMethod vocabulary.`,
|
|
8777
|
+
evidenceBase,
|
|
8778
|
+
{ methodVocabulary: [...OBSERVED_OUTCOME_METHODS] },
|
|
8779
|
+
{ name: outcome.name, method: outcome.method }
|
|
8780
|
+
)
|
|
8781
|
+
);
|
|
8782
|
+
}
|
|
8783
|
+
}
|
|
8784
|
+
if (requireProvenance.evidence || requireProvenance.sameRunEventReference) {
|
|
8785
|
+
const parsed = parseEvidenceEventIds(outcome.evidence);
|
|
8786
|
+
if (parsed.status === "missing") {
|
|
8787
|
+
findings.push(
|
|
8788
|
+
contractFailFinding(
|
|
8789
|
+
"contract.observation.provenance.evidence",
|
|
8790
|
+
`Observation ${outcome.name} is missing bounded evidence references.`,
|
|
8791
|
+
evidenceBase,
|
|
8792
|
+
{ evidence: true },
|
|
8793
|
+
{ name: outcome.name, evidence: null }
|
|
8794
|
+
)
|
|
8795
|
+
);
|
|
8796
|
+
} else if (parsed.status === "invalid") {
|
|
8797
|
+
findings.push(
|
|
8798
|
+
contractFailFinding(
|
|
8799
|
+
"contract.observation.provenance.evidence",
|
|
8800
|
+
`Observation ${outcome.name} evidence is not a bounded event reference (${parsed.reason}).`,
|
|
8801
|
+
evidenceBase,
|
|
8802
|
+
{ evidence: true },
|
|
8803
|
+
{ name: outcome.name, reason: parsed.reason }
|
|
8804
|
+
)
|
|
8805
|
+
);
|
|
8806
|
+
} else if (requireProvenance.sameRunEventReference) {
|
|
8807
|
+
const missing = parsed.eventIds.filter((eventId) => !knownEventIds.has(eventId));
|
|
8808
|
+
if (missing.length > 0) {
|
|
8809
|
+
findings.push(
|
|
8810
|
+
contractFailFinding(
|
|
8811
|
+
"contract.observation.provenance.same-run",
|
|
8812
|
+
`Observation ${outcome.name} references event ids missing from the same run: ${missing.join(", ")}.`,
|
|
8813
|
+
evidenceBase,
|
|
8814
|
+
{ sameRunEventReference: true, eventIds: parsed.eventIds },
|
|
8815
|
+
{ name: outcome.name, missing }
|
|
8816
|
+
)
|
|
8817
|
+
);
|
|
8818
|
+
}
|
|
8819
|
+
}
|
|
8820
|
+
}
|
|
8821
|
+
return findings;
|
|
8822
|
+
}
|
|
8823
|
+
function contractToRules(contract) {
|
|
8824
|
+
const rules = [];
|
|
8825
|
+
const allowedStatuses = contract.run?.allowedStatuses ?? [];
|
|
8826
|
+
if (allowedStatuses.length === 1) {
|
|
8827
|
+
rules.push(
|
|
8828
|
+
createRunStatusRule({
|
|
8829
|
+
expected: normalizeStatus(allowedStatuses[0]),
|
|
8830
|
+
allowIncomplete: contract.run?.requireCompleted === false
|
|
8831
|
+
})
|
|
8832
|
+
);
|
|
8833
|
+
} else if (allowedStatuses.length > 1) {
|
|
8834
|
+
const expected = [...new Set(allowedStatuses.map(normalizeStatus))];
|
|
8835
|
+
const allowIncomplete = contract.run?.requireCompleted === false;
|
|
8836
|
+
rules.push({
|
|
8837
|
+
id: "contract.run.allowedStatuses",
|
|
8838
|
+
category: "run",
|
|
8839
|
+
defaultSeverity: "error",
|
|
8840
|
+
evaluate(context) {
|
|
8841
|
+
const findings = [];
|
|
8842
|
+
const actual = context.selectedRun?.status ?? "unknown";
|
|
8843
|
+
if (!expected.includes(actual)) {
|
|
8844
|
+
findings.push(
|
|
8845
|
+
contractFailFinding(
|
|
8846
|
+
"contract.run.allowedStatuses",
|
|
8847
|
+
`Run status ${actual} is not one of the allowed statuses: ${expected.join(", ")}.`,
|
|
8848
|
+
context.selectedRun ? [
|
|
8849
|
+
{
|
|
8850
|
+
runId: context.selectedRun.runId,
|
|
8851
|
+
kind: "RUN",
|
|
8852
|
+
name: context.selectedRun.name,
|
|
8853
|
+
status: context.selectedRun.status
|
|
8854
|
+
}
|
|
8855
|
+
] : [],
|
|
8856
|
+
expected,
|
|
8857
|
+
actual
|
|
8858
|
+
)
|
|
8859
|
+
);
|
|
8860
|
+
}
|
|
8861
|
+
if (!allowIncomplete) {
|
|
8862
|
+
const running = (context.logicalEvents ?? context.events).filter(
|
|
8863
|
+
(event) => event.status === "running"
|
|
8864
|
+
);
|
|
8865
|
+
if (running.length > 0) {
|
|
8866
|
+
findings.push(
|
|
8867
|
+
contractFailFinding(
|
|
8868
|
+
"contract.run.allowedStatuses",
|
|
8869
|
+
"Run contains incomplete running events.",
|
|
8870
|
+
running.map((event) => ({
|
|
8871
|
+
runId: event.runId,
|
|
8872
|
+
eventId: event.eventId,
|
|
8873
|
+
kind: event.kind,
|
|
8874
|
+
name: event.name,
|
|
8875
|
+
status: event.status
|
|
8876
|
+
})),
|
|
8877
|
+
"no running events",
|
|
8878
|
+
running.length
|
|
8879
|
+
)
|
|
8880
|
+
);
|
|
8881
|
+
}
|
|
8882
|
+
}
|
|
8883
|
+
return findings;
|
|
8884
|
+
}
|
|
8885
|
+
});
|
|
8886
|
+
} else if (contract.run?.requireCompleted !== false) {
|
|
8887
|
+
rules.push(createRunStatusRule({ allowIncomplete: false }));
|
|
8888
|
+
}
|
|
8889
|
+
if (contract.run?.maxDurationMs !== void 0) {
|
|
8890
|
+
rules.push(createRunDurationRule({ maxDurationMs: contract.run.maxDurationMs }));
|
|
8891
|
+
}
|
|
8892
|
+
if (contract.run?.requireCompleted === true) {
|
|
8893
|
+
rules.push(createStructureIncompleteRule({ requireEndedAtForStarted: true }));
|
|
8894
|
+
}
|
|
8895
|
+
if (contract.tools) {
|
|
8896
|
+
const order = contract.tools.requiredOrder ?? [];
|
|
8897
|
+
const requiredOrderMode = contract.tools.requiredOrderMode ?? "first-occurrence";
|
|
8898
|
+
const orderRules = contract.tools.orderRules ?? [];
|
|
8899
|
+
const endpointRequired = orderRules.filter((rule) => rule.requireEndpoints !== false).flatMap((rule) => [rule.before, rule.after]);
|
|
8900
|
+
const required = [
|
|
8901
|
+
.../* @__PURE__ */ new Set([
|
|
8902
|
+
...contract.tools.required ?? [],
|
|
8903
|
+
...contract.tools.requiredTools ?? [],
|
|
8904
|
+
...order,
|
|
8905
|
+
...endpointRequired
|
|
8906
|
+
])
|
|
8907
|
+
];
|
|
8908
|
+
const forbidden = [
|
|
8909
|
+
...contract.tools.forbidden ?? [],
|
|
8910
|
+
...contract.tools.forbiddenTools ?? []
|
|
8911
|
+
];
|
|
8912
|
+
rules.push(
|
|
8913
|
+
createToolUsageRule({
|
|
8914
|
+
...required.length > 0 ? { required } : {},
|
|
8915
|
+
...forbidden.length > 0 ? { forbidden } : {},
|
|
8916
|
+
...contract.tools.allowed ? { allowed: contract.tools.allowed } : {},
|
|
8917
|
+
...contract.tools.maxCalls !== void 0 ? { maxCount: contract.tools.maxCalls } : {}
|
|
8918
|
+
})
|
|
8919
|
+
);
|
|
8920
|
+
for (let i = 0; i < order.length - 1; i += 1) {
|
|
8921
|
+
rules.push(
|
|
8922
|
+
createToolOrderingRule({
|
|
8923
|
+
before: order[i],
|
|
8924
|
+
after: order[i + 1],
|
|
8925
|
+
id: `contract.tool.order.${i}`,
|
|
8926
|
+
mode: requiredOrderMode
|
|
8927
|
+
})
|
|
8928
|
+
);
|
|
8929
|
+
}
|
|
8930
|
+
const defaultOccurrenceMode = contract.tools.defaultOccurrenceMode ?? requiredOrderMode;
|
|
8931
|
+
for (const [index, rule] of orderRules.entries()) {
|
|
8932
|
+
rules.push(
|
|
8933
|
+
createToolOrderingRule({
|
|
8934
|
+
before: rule.before,
|
|
8935
|
+
after: rule.after,
|
|
8936
|
+
id: `contract.tool.orderRule.${index}`,
|
|
8937
|
+
mode: rule.occurrenceMode ?? defaultOccurrenceMode
|
|
8938
|
+
})
|
|
8939
|
+
);
|
|
8940
|
+
}
|
|
8941
|
+
const argumentChecks = contract.tools.arguments ?? [];
|
|
8942
|
+
if (argumentChecks.length > 0) {
|
|
8943
|
+
rules.push({
|
|
8944
|
+
id: "contract.tool.arguments",
|
|
8945
|
+
category: "tool",
|
|
8946
|
+
defaultSeverity: "error",
|
|
8947
|
+
evaluate(context) {
|
|
8948
|
+
const tools = (context.logicalEvents ?? context.events).filter(
|
|
8949
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
8950
|
+
);
|
|
8951
|
+
const findings = [];
|
|
8952
|
+
for (const [index, check] of argumentChecks.entries()) {
|
|
8953
|
+
const matches = tools.filter(
|
|
8954
|
+
(event) => resolveCanonicalToolName(event) === check.tool
|
|
8955
|
+
);
|
|
8956
|
+
if (matches.length === 0) {
|
|
8957
|
+
findings.push(
|
|
8958
|
+
contractFailFinding(
|
|
8959
|
+
`contract.tool.arguments.${index}`,
|
|
8960
|
+
`No finished tool named ${check.tool} for argument check.`,
|
|
8961
|
+
context.selectedRun ? [
|
|
8962
|
+
{
|
|
8963
|
+
runId: context.selectedRun.runId,
|
|
8964
|
+
kind: "RUN",
|
|
8965
|
+
name: context.selectedRun.name
|
|
8966
|
+
}
|
|
8967
|
+
] : [],
|
|
8968
|
+
{ tool: check.tool, path: check.path },
|
|
8969
|
+
{ toolCount: 0 }
|
|
8970
|
+
)
|
|
8971
|
+
);
|
|
8972
|
+
continue;
|
|
8973
|
+
}
|
|
8974
|
+
const occurrence = check.occurrence ?? "all";
|
|
8975
|
+
const selected = occurrence === "first" ? [matches[0]] : occurrence === "last" ? [matches[matches.length - 1]] : matches;
|
|
8976
|
+
const results = selected.map((event) => {
|
|
8977
|
+
const payload = extractToolArgumentPayload(event);
|
|
8978
|
+
return evaluateToolArgumentValue(
|
|
8979
|
+
payload.value,
|
|
8980
|
+
check,
|
|
8981
|
+
payload.present
|
|
8982
|
+
);
|
|
8983
|
+
});
|
|
8984
|
+
if (occurrence === "any") {
|
|
8985
|
+
if (results.some((result) => result.status === "pass")) continue;
|
|
8986
|
+
const firstFail = results.find((result) => result.status !== "pass");
|
|
8987
|
+
findings.push(
|
|
8988
|
+
contractFailFinding(
|
|
8989
|
+
`contract.tool.arguments.${index}`,
|
|
8990
|
+
firstFail.message,
|
|
8991
|
+
selected.slice(0, 1).map((event) => ({
|
|
8992
|
+
runId: event.runId,
|
|
8993
|
+
eventId: event.eventId,
|
|
8994
|
+
kind: event.kind,
|
|
8995
|
+
name: event.name,
|
|
8996
|
+
path: `tool.${check.tool}${check.path}`
|
|
8997
|
+
})),
|
|
8998
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
8999
|
+
{ code: firstFail.code }
|
|
9000
|
+
)
|
|
9001
|
+
);
|
|
9002
|
+
continue;
|
|
9003
|
+
}
|
|
9004
|
+
for (const [selIndex, result] of results.entries()) {
|
|
9005
|
+
if (result.status === "pass") continue;
|
|
9006
|
+
const event = selected[selIndex];
|
|
9007
|
+
findings.push(
|
|
9008
|
+
contractFailFinding(
|
|
9009
|
+
`contract.tool.arguments.${index}`,
|
|
9010
|
+
result.message,
|
|
9011
|
+
[
|
|
9012
|
+
{
|
|
9013
|
+
runId: event.runId,
|
|
9014
|
+
eventId: event.eventId,
|
|
9015
|
+
kind: event.kind,
|
|
9016
|
+
name: event.name,
|
|
9017
|
+
path: `tool.${check.tool}${check.path}`
|
|
9018
|
+
}
|
|
9019
|
+
],
|
|
9020
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
9021
|
+
{ code: result.code }
|
|
9022
|
+
)
|
|
9023
|
+
);
|
|
9024
|
+
}
|
|
9025
|
+
}
|
|
9026
|
+
return findings;
|
|
9027
|
+
}
|
|
9028
|
+
});
|
|
9029
|
+
}
|
|
9030
|
+
}
|
|
9031
|
+
if (contract.llm) {
|
|
9032
|
+
rules.push(
|
|
9033
|
+
createLlmUsageRule({
|
|
9034
|
+
...contract.llm.maxCalls !== void 0 ? { maxCalls: contract.llm.maxCalls } : {},
|
|
9035
|
+
...contract.llm.maxTotalTokens !== void 0 ? { maxTotalTokens: contract.llm.maxTotalTokens } : {},
|
|
9036
|
+
...contract.llm.allowedModels ? { allowedModels: contract.llm.allowedModels } : {}
|
|
9037
|
+
})
|
|
9038
|
+
);
|
|
9039
|
+
}
|
|
9040
|
+
if (contract.observations) {
|
|
9041
|
+
const required = contract.observations.required ?? [];
|
|
9042
|
+
if (required.length > 0) {
|
|
9043
|
+
rules.push({
|
|
9044
|
+
id: "contract.observation.required",
|
|
9045
|
+
category: "run",
|
|
9046
|
+
defaultSeverity: "error",
|
|
9047
|
+
evaluate(context) {
|
|
9048
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
9049
|
+
const names = new Set(outcomes.map((item) => item.name));
|
|
9050
|
+
const missing = required.filter((name) => !names.has(name));
|
|
9051
|
+
if (missing.length === 0) return [];
|
|
9052
|
+
return [
|
|
9053
|
+
contractFailFinding(
|
|
9054
|
+
"contract.observation.required",
|
|
9055
|
+
`Required observations missing: ${missing.join(", ")}`,
|
|
9056
|
+
context.selectedRun ? [{ runId: context.selectedRun.runId, kind: "RUN", name: context.selectedRun.name }] : [],
|
|
9057
|
+
required,
|
|
9058
|
+
[...names]
|
|
9059
|
+
)
|
|
9060
|
+
];
|
|
9061
|
+
}
|
|
9062
|
+
});
|
|
9063
|
+
}
|
|
9064
|
+
if (contract.observations.failOn?.length) {
|
|
9065
|
+
rules.push(
|
|
9066
|
+
createObservedOutcomeRule({
|
|
9067
|
+
failOn: contract.observations.failOn.filter(
|
|
9068
|
+
(status) => status === "failed" || status === "unknown" || status === "skipped"
|
|
9069
|
+
)
|
|
9070
|
+
})
|
|
9071
|
+
);
|
|
9072
|
+
}
|
|
9073
|
+
const requireProvenance = contract.observations.requireProvenance;
|
|
9074
|
+
if (requireProvenance && (requireProvenance.method || requireProvenance.evidence || requireProvenance.sameRunEventReference)) {
|
|
9075
|
+
rules.push({
|
|
9076
|
+
id: "contract.observation.provenance",
|
|
9077
|
+
category: "run",
|
|
9078
|
+
defaultSeverity: "error",
|
|
9079
|
+
evaluate(context) {
|
|
9080
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
9081
|
+
const knownEventIds = new Set(context.events.map((event) => event.eventId));
|
|
9082
|
+
const targets = required.length > 0 ? outcomes.filter((outcome) => required.includes(outcome.name)) : outcomes;
|
|
9083
|
+
return targets.flatMap(
|
|
9084
|
+
(outcome) => provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds)
|
|
9085
|
+
);
|
|
9086
|
+
}
|
|
9087
|
+
});
|
|
9088
|
+
}
|
|
9089
|
+
}
|
|
9090
|
+
if (contract.controls) {
|
|
9091
|
+
const controls = contract.controls;
|
|
9092
|
+
rules.push({
|
|
9093
|
+
id: "contract.controls",
|
|
9094
|
+
category: "run",
|
|
9095
|
+
defaultSeverity: "error",
|
|
9096
|
+
evaluate(context) {
|
|
9097
|
+
const events = context.logicalEvents ?? context.events;
|
|
9098
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
9099
|
+
const observationNames = new Set(outcomes.map((item) => item.name));
|
|
9100
|
+
const runEvidence2 = context.selectedRun ? [
|
|
9101
|
+
{
|
|
9102
|
+
runId: context.selectedRun.runId,
|
|
9103
|
+
kind: "RUN",
|
|
9104
|
+
name: context.selectedRun.name
|
|
9105
|
+
}
|
|
9106
|
+
] : [];
|
|
9107
|
+
return evaluateControlRules(events, controls, runEvidence2, observationNames);
|
|
9108
|
+
}
|
|
9109
|
+
});
|
|
9110
|
+
}
|
|
9111
|
+
if (contract.retry) {
|
|
9112
|
+
const retry = contract.retry;
|
|
9113
|
+
rules.push({
|
|
9114
|
+
id: "contract.retry",
|
|
9115
|
+
category: "run",
|
|
9116
|
+
defaultSeverity: "error",
|
|
9117
|
+
evaluate(context) {
|
|
9118
|
+
const events = context.logicalEvents ?? context.events;
|
|
9119
|
+
context.selectedRun ? [
|
|
9120
|
+
{
|
|
9121
|
+
runId: context.selectedRun.runId,
|
|
9122
|
+
kind: "RUN",
|
|
9123
|
+
name: context.selectedRun.name
|
|
9124
|
+
}
|
|
9125
|
+
] : [];
|
|
9126
|
+
const base = evaluateRetrySafetyRules(events, retry);
|
|
9127
|
+
if (!retry.operations || retry.operations.length === 0) {
|
|
9128
|
+
return base;
|
|
9129
|
+
}
|
|
9130
|
+
return [...base, ...evaluateRecoveryOperations(events, retry.operations)];
|
|
9131
|
+
}
|
|
9132
|
+
});
|
|
9133
|
+
}
|
|
9134
|
+
return rules;
|
|
9135
|
+
}
|
|
7136
9136
|
function validateScopeShape(scope) {
|
|
7137
9137
|
if (scope === void 0) return [];
|
|
7138
9138
|
const hasSelector = typeof scope.runId === "string" && scope.runId.trim() !== "" || typeof scope.subAgentId === "string" && scope.subAgentId.trim() !== "" || typeof scope.groupId === "string" && scope.groupId.trim() !== "" || typeof scope.workflowStep === "string" && scope.workflowStep.trim() !== "" || typeof scope.rootEventId === "string" && scope.rootEventId.trim() !== "";
|
|
@@ -7233,6 +9233,57 @@ function validateAlternativesShape(alternatives) {
|
|
|
7233
9233
|
}
|
|
7234
9234
|
return diagnostics;
|
|
7235
9235
|
}
|
|
9236
|
+
function emptyPassResult(input3, runId) {
|
|
9237
|
+
return {
|
|
9238
|
+
ok: true,
|
|
9239
|
+
status: "pass",
|
|
9240
|
+
format: input3.read.format,
|
|
9241
|
+
...runId !== void 0 ? { runId } : {},
|
|
9242
|
+
summary: { passed: 0, failed: 0, warnings: 0, errors: 0, rulesEvaluated: 0 },
|
|
9243
|
+
findings: [],
|
|
9244
|
+
diagnostics: [],
|
|
9245
|
+
ruleExecutions: []
|
|
9246
|
+
};
|
|
9247
|
+
}
|
|
9248
|
+
function mergeContractResults(input3, parts, extraFindings = []) {
|
|
9249
|
+
const findings = [...parts.flatMap((part) => part.findings), ...extraFindings];
|
|
9250
|
+
const diagnostics = parts.flatMap((part) => part.diagnostics);
|
|
9251
|
+
const ruleExecutions = parts.flatMap((part) => part.ruleExecutions);
|
|
9252
|
+
const failed = findings.filter(
|
|
9253
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
9254
|
+
).length;
|
|
9255
|
+
const errors = diagnostics.filter((item) => item.severity === "error").length;
|
|
9256
|
+
const status = errors > 0 ? "error" : failed > 0 ? "fail" : "pass";
|
|
9257
|
+
const runId = parts.find((part) => part.runId !== void 0)?.runId;
|
|
9258
|
+
return {
|
|
9259
|
+
ok: status === "pass",
|
|
9260
|
+
status,
|
|
9261
|
+
format: input3.read.format,
|
|
9262
|
+
...runId !== void 0 ? { runId } : {},
|
|
9263
|
+
summary: {
|
|
9264
|
+
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
9265
|
+
failed,
|
|
9266
|
+
warnings: findings.filter(
|
|
9267
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
9268
|
+
).length,
|
|
9269
|
+
errors,
|
|
9270
|
+
rulesEvaluated: ruleExecutions.length
|
|
9271
|
+
},
|
|
9272
|
+
findings,
|
|
9273
|
+
diagnostics,
|
|
9274
|
+
ruleExecutions
|
|
9275
|
+
};
|
|
9276
|
+
}
|
|
9277
|
+
function evaluateBody(input3, body, options = {}) {
|
|
9278
|
+
const rules = contractToRules(body);
|
|
9279
|
+
if (rules.length === 0) {
|
|
9280
|
+
return emptyPassResult(input3, options.runId);
|
|
9281
|
+
}
|
|
9282
|
+
return runTraceChecks(input3, {
|
|
9283
|
+
rules,
|
|
9284
|
+
...options.runId !== void 0 ? { runId: options.runId } : {}
|
|
9285
|
+
});
|
|
9286
|
+
}
|
|
7236
9287
|
function defineTraceContract(input3) {
|
|
7237
9288
|
const shapeErrors = [
|
|
7238
9289
|
...validateAlternativesShape(input3.alternatives),
|
|
@@ -7263,6 +9314,144 @@ function defineTraceContract(input3) {
|
|
|
7263
9314
|
...alternatives ? { alternatives } : {}
|
|
7264
9315
|
};
|
|
7265
9316
|
}
|
|
9317
|
+
function evaluateTraceContract(input3, contract, options = {}) {
|
|
9318
|
+
const shapeErrors = [
|
|
9319
|
+
...validateAlternativesShape(contract.alternatives),
|
|
9320
|
+
...validateScopeShape(contract.scope),
|
|
9321
|
+
...validateProvenanceShape(contract.observations)
|
|
9322
|
+
].filter((item) => item.severity === "error");
|
|
9323
|
+
if (shapeErrors.length > 0) {
|
|
9324
|
+
return {
|
|
9325
|
+
ok: false,
|
|
9326
|
+
status: "error",
|
|
9327
|
+
format: input3.read.format,
|
|
9328
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
9329
|
+
summary: {
|
|
9330
|
+
passed: 0,
|
|
9331
|
+
failed: 0,
|
|
9332
|
+
warnings: 0,
|
|
9333
|
+
errors: shapeErrors.length,
|
|
9334
|
+
rulesEvaluated: 0
|
|
9335
|
+
},
|
|
9336
|
+
findings: [],
|
|
9337
|
+
diagnostics: shapeErrors.map((item) => ({
|
|
9338
|
+
code: "AI_CHECK_INVALID_CONFIG",
|
|
9339
|
+
message: item.message,
|
|
9340
|
+
severity: "error",
|
|
9341
|
+
ruleId: item.code
|
|
9342
|
+
})),
|
|
9343
|
+
ruleExecutions: []
|
|
9344
|
+
};
|
|
9345
|
+
}
|
|
9346
|
+
const scoped = resolveTraceContractScope(input3, contract.scope);
|
|
9347
|
+
if (scoped.diagnostics.length > 0) {
|
|
9348
|
+
return {
|
|
9349
|
+
ok: false,
|
|
9350
|
+
status: "error",
|
|
9351
|
+
format: input3.read.format,
|
|
9352
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
9353
|
+
...scoped.resolved?.runId !== void 0 ? { runId: scoped.resolved.runId } : {},
|
|
9354
|
+
summary: {
|
|
9355
|
+
passed: 0,
|
|
9356
|
+
failed: 0,
|
|
9357
|
+
warnings: 0,
|
|
9358
|
+
errors: scoped.diagnostics.length,
|
|
9359
|
+
rulesEvaluated: 0
|
|
9360
|
+
},
|
|
9361
|
+
findings: [],
|
|
9362
|
+
diagnostics: scoped.diagnostics,
|
|
9363
|
+
ruleExecutions: []
|
|
9364
|
+
};
|
|
9365
|
+
}
|
|
9366
|
+
const effectiveInput = scoped.resolved?.input ?? input3;
|
|
9367
|
+
const effectiveRunId = scoped.resolved?.runId ?? options.runId;
|
|
9368
|
+
const base = evaluateBody(effectiveInput, contract, {
|
|
9369
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
9370
|
+
});
|
|
9371
|
+
const scopeFinding = scoped.resolved ? {
|
|
9372
|
+
ruleId: "contract.scope.selected",
|
|
9373
|
+
severity: "info",
|
|
9374
|
+
status: "pass",
|
|
9375
|
+
message: `Selected actor ${scoped.resolved.runId} with ${scoped.resolved.evidenceEventCount} evidence event(s).`,
|
|
9376
|
+
expected: scoped.resolved.matchedSelectors,
|
|
9377
|
+
actual: {
|
|
9378
|
+
runId: scoped.resolved.runId,
|
|
9379
|
+
evidenceEventCount: scoped.resolved.evidenceEventCount,
|
|
9380
|
+
...scoped.resolved.rootEventId !== void 0 ? { rootEventId: scoped.resolved.rootEventId } : {}
|
|
9381
|
+
},
|
|
9382
|
+
evidence: [
|
|
9383
|
+
{
|
|
9384
|
+
runId: scoped.resolved.runId,
|
|
9385
|
+
kind: "RUN",
|
|
9386
|
+
path: "scope"
|
|
9387
|
+
}
|
|
9388
|
+
]
|
|
9389
|
+
} : void 0;
|
|
9390
|
+
const branches = contract.alternatives?.anyOf ?? [];
|
|
9391
|
+
if (branches.length === 0) {
|
|
9392
|
+
return scopeFinding ? mergeContractResults(effectiveInput, [base], [scopeFinding]) : base;
|
|
9393
|
+
}
|
|
9394
|
+
const branchEvaluations = branches.map((branch) => {
|
|
9395
|
+
const result = evaluateBody(effectiveInput, branch.contract, {
|
|
9396
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
9397
|
+
});
|
|
9398
|
+
return { branch, result };
|
|
9399
|
+
});
|
|
9400
|
+
const satisfied = branchEvaluations.filter((item) => item.result.ok && item.result.status === "pass").map((item) => item.branch.id);
|
|
9401
|
+
const failedBranches = branchEvaluations.filter(
|
|
9402
|
+
(item) => !(item.result.ok && item.result.status === "pass")
|
|
9403
|
+
);
|
|
9404
|
+
if (satisfied.length > 0) {
|
|
9405
|
+
const passFinding = {
|
|
9406
|
+
ruleId: "contract.alternatives.anyOf",
|
|
9407
|
+
severity: "info",
|
|
9408
|
+
status: "pass",
|
|
9409
|
+
message: `Satisfied alternative branch(es): ${satisfied.join(", ")}.`,
|
|
9410
|
+
expected: { anyOf: branches.map((branch) => branch.id) },
|
|
9411
|
+
actual: {
|
|
9412
|
+
satisfied,
|
|
9413
|
+
failed: failedBranches.map((item) => item.branch.id)
|
|
9414
|
+
},
|
|
9415
|
+
evidence: []
|
|
9416
|
+
};
|
|
9417
|
+
return mergeContractResults(
|
|
9418
|
+
effectiveInput,
|
|
9419
|
+
[base],
|
|
9420
|
+
[...scopeFinding ? [scopeFinding] : [], passFinding]
|
|
9421
|
+
);
|
|
9422
|
+
}
|
|
9423
|
+
const noneFinding = {
|
|
9424
|
+
ruleId: "contract.alternatives.none-satisfied",
|
|
9425
|
+
severity: "error",
|
|
9426
|
+
status: "fail",
|
|
9427
|
+
message: "No alternatives.anyOf branch fully satisfied its contract.",
|
|
9428
|
+
expected: { anyOf: branches.map((branch) => branch.id) },
|
|
9429
|
+
actual: {
|
|
9430
|
+
branchFailures: Object.fromEntries(
|
|
9431
|
+
failedBranches.map((item) => [
|
|
9432
|
+
item.branch.id,
|
|
9433
|
+
item.result.findings.filter((finding) => finding.status === "fail").map((finding) => ({
|
|
9434
|
+
ruleId: finding.ruleId,
|
|
9435
|
+
message: finding.message
|
|
9436
|
+
}))
|
|
9437
|
+
])
|
|
9438
|
+
)
|
|
9439
|
+
},
|
|
9440
|
+
evidence: []
|
|
9441
|
+
};
|
|
9442
|
+
const prefixedBranchFindings = failedBranches.flatMap(
|
|
9443
|
+
(item) => item.result.findings.filter((finding) => finding.status === "fail").map((finding) => ({
|
|
9444
|
+
...finding,
|
|
9445
|
+
ruleId: `contract.alternatives.${item.branch.id}.${finding.ruleId}`
|
|
9446
|
+
}))
|
|
9447
|
+
);
|
|
9448
|
+
return mergeContractResults(
|
|
9449
|
+
effectiveInput,
|
|
9450
|
+
[base],
|
|
9451
|
+
[...scopeFinding ? [scopeFinding] : [], noneFinding, ...prefixedBranchFindings]
|
|
9452
|
+
);
|
|
9453
|
+
}
|
|
9454
|
+
var MAX_EVIDENCE_EVENT_IDS, METHOD_VOCABULARY;
|
|
7266
9455
|
var init_contract = __esm({
|
|
7267
9456
|
"packages/core/src/checks/contract.ts"() {
|
|
7268
9457
|
init_checks2();
|
|
@@ -7275,7 +9464,8 @@ var init_contract = __esm({
|
|
|
7275
9464
|
init_outcomes();
|
|
7276
9465
|
init_contract_scope();
|
|
7277
9466
|
init_tool_arguments();
|
|
7278
|
-
|
|
9467
|
+
MAX_EVIDENCE_EVENT_IDS = 16;
|
|
9468
|
+
METHOD_VOCABULARY = new Set(OBSERVED_OUTCOME_METHODS);
|
|
7279
9469
|
}
|
|
7280
9470
|
});
|
|
7281
9471
|
|
|
@@ -7650,18 +9840,18 @@ function escapeMarkdown(value) {
|
|
|
7650
9840
|
function escapeHtml(value) {
|
|
7651
9841
|
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
7652
9842
|
}
|
|
7653
|
-
function
|
|
9843
|
+
function sortKeysDeep2(input3) {
|
|
7654
9844
|
if (input3 === null || typeof input3 !== "object") return input3;
|
|
7655
|
-
if (Array.isArray(input3)) return input3.map(
|
|
9845
|
+
if (Array.isArray(input3)) return input3.map(sortKeysDeep2);
|
|
7656
9846
|
const o = input3;
|
|
7657
9847
|
const out = {};
|
|
7658
9848
|
for (const k of Object.keys(o).sort()) {
|
|
7659
|
-
out[k] =
|
|
9849
|
+
out[k] = sortKeysDeep2(o[k]);
|
|
7660
9850
|
}
|
|
7661
9851
|
return out;
|
|
7662
9852
|
}
|
|
7663
9853
|
function stableJson(value, pretty) {
|
|
7664
|
-
const sorted =
|
|
9854
|
+
const sorted = sortKeysDeep2(value);
|
|
7665
9855
|
return pretty === true ? JSON.stringify(sorted, null, 2) : JSON.stringify(sorted);
|
|
7666
9856
|
}
|
|
7667
9857
|
function compactAttributes(attrs, options) {
|
|
@@ -9294,7 +11484,7 @@ var init_types5 = __esm({
|
|
|
9294
11484
|
DEFAULT_SUITE_ARTIFACTS_DIR = ".agent-inspect/suite-runs";
|
|
9295
11485
|
}
|
|
9296
11486
|
});
|
|
9297
|
-
function
|
|
11487
|
+
function diagnostic3(code, message, severity = "error", caseId) {
|
|
9298
11488
|
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
9299
11489
|
}
|
|
9300
11490
|
function asString(value, label) {
|
|
@@ -9322,7 +11512,7 @@ function validateCaseConfig(value, index) {
|
|
|
9322
11512
|
const diagnostics = [];
|
|
9323
11513
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
9324
11514
|
diagnostics.push(
|
|
9325
|
-
|
|
11515
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
|
|
9326
11516
|
);
|
|
9327
11517
|
return { diagnostics };
|
|
9328
11518
|
}
|
|
@@ -9331,7 +11521,7 @@ function validateCaseConfig(value, index) {
|
|
|
9331
11521
|
const id = asString(raw.id, `cases[${index}].id`);
|
|
9332
11522
|
if (id === void 0) {
|
|
9333
11523
|
diagnostics.push(
|
|
9334
|
-
|
|
11524
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
|
|
9335
11525
|
);
|
|
9336
11526
|
return { diagnostics };
|
|
9337
11527
|
}
|
|
@@ -9366,7 +11556,7 @@ function validateCaseConfig(value, index) {
|
|
|
9366
11556
|
};
|
|
9367
11557
|
} catch (error) {
|
|
9368
11558
|
const message = error instanceof Error ? error.message : String(error);
|
|
9369
|
-
diagnostics.push(
|
|
11559
|
+
diagnostics.push(diagnostic3("AI_SUITE_CONFIG_INVALID", message));
|
|
9370
11560
|
return { diagnostics };
|
|
9371
11561
|
}
|
|
9372
11562
|
}
|
|
@@ -9442,7 +11632,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
9442
11632
|
await promises.access(tracesDir);
|
|
9443
11633
|
} catch {
|
|
9444
11634
|
diagnostics.push(
|
|
9445
|
-
|
|
11635
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `traces directory not found: ${tracesDir}`)
|
|
9446
11636
|
);
|
|
9447
11637
|
}
|
|
9448
11638
|
for (const suiteCase of config.cases) {
|
|
@@ -9452,7 +11642,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
9452
11642
|
await promises.access(inputPath);
|
|
9453
11643
|
} catch {
|
|
9454
11644
|
diagnostics.push(
|
|
9455
|
-
|
|
11645
|
+
diagnostic3(
|
|
9456
11646
|
"AI_SUITE_CONFIG_INVALID",
|
|
9457
11647
|
`input fixture not found: ${suiteCase.input}`,
|
|
9458
11648
|
"warning",
|
|
@@ -9468,7 +11658,7 @@ var init_validate2 = __esm({
|
|
|
9468
11658
|
"packages/core/src/suite/validate.ts"() {
|
|
9469
11659
|
}
|
|
9470
11660
|
});
|
|
9471
|
-
function
|
|
11661
|
+
function diagnostic4(code, message) {
|
|
9472
11662
|
return { code, message, severity: "error" };
|
|
9473
11663
|
}
|
|
9474
11664
|
async function fileExists(filePath) {
|
|
@@ -9499,20 +11689,20 @@ async function loadSuiteConfig(options = {}) {
|
|
|
9499
11689
|
} catch (error) {
|
|
9500
11690
|
const message = error instanceof Error ? error.message : String(error);
|
|
9501
11691
|
throw Object.assign(new Error(message), {
|
|
9502
|
-
diagnostics: [
|
|
11692
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
9503
11693
|
});
|
|
9504
11694
|
}
|
|
9505
11695
|
const extension = path33__default.default.extname(configPath);
|
|
9506
11696
|
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
9507
11697
|
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
9508
11698
|
throw Object.assign(new Error(message), {
|
|
9509
|
-
diagnostics: [
|
|
11699
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
9510
11700
|
});
|
|
9511
11701
|
}
|
|
9512
11702
|
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
9513
11703
|
const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
|
|
9514
11704
|
throw Object.assign(new Error(message), {
|
|
9515
|
-
diagnostics: [
|
|
11705
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
9516
11706
|
});
|
|
9517
11707
|
}
|
|
9518
11708
|
try {
|
|
@@ -9532,7 +11722,7 @@ async function loadSuiteConfig(options = {}) {
|
|
|
9532
11722
|
} catch (error) {
|
|
9533
11723
|
const message = error instanceof Error ? error.message : String(error);
|
|
9534
11724
|
throw Object.assign(new Error(message), {
|
|
9535
|
-
diagnostics: [
|
|
11725
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
9536
11726
|
});
|
|
9537
11727
|
}
|
|
9538
11728
|
}
|
|
@@ -10459,13 +12649,13 @@ function persistedEventsForParsedTrace(parsed) {
|
|
|
10459
12649
|
function isRecord11(value) {
|
|
10460
12650
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
10461
12651
|
}
|
|
10462
|
-
function
|
|
12652
|
+
function isNonEmptyString5(value) {
|
|
10463
12653
|
return typeof value === "string" && value.trim() !== "";
|
|
10464
12654
|
}
|
|
10465
12655
|
function readStringField(record, keys) {
|
|
10466
12656
|
for (const key of keys) {
|
|
10467
12657
|
const value = record[key];
|
|
10468
|
-
if (
|
|
12658
|
+
if (isNonEmptyString5(value)) return value;
|
|
10469
12659
|
}
|
|
10470
12660
|
return void 0;
|
|
10471
12661
|
}
|
|
@@ -10599,7 +12789,7 @@ function parseUnixNanoToIso(value) {
|
|
|
10599
12789
|
return void 0;
|
|
10600
12790
|
}
|
|
10601
12791
|
function parseIsoTime(value) {
|
|
10602
|
-
if (!
|
|
12792
|
+
if (!isNonEmptyString5(value)) return void 0;
|
|
10603
12793
|
const ms = Date.parse(value);
|
|
10604
12794
|
if (!Number.isFinite(ms)) return void 0;
|
|
10605
12795
|
return new Date(ms).toISOString();
|
|
@@ -11856,7 +14046,7 @@ var init_readers = __esm({
|
|
|
11856
14046
|
];
|
|
11857
14047
|
}
|
|
11858
14048
|
});
|
|
11859
|
-
function
|
|
14049
|
+
function diagnostic5(code, message, severity = "error", caseId) {
|
|
11860
14050
|
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
11861
14051
|
}
|
|
11862
14052
|
function buildCaseRules(suiteCase, config) {
|
|
@@ -11920,7 +14110,7 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
11920
14110
|
const match = outcomes.find((outcome) => outcome.name === name);
|
|
11921
14111
|
if (!match) {
|
|
11922
14112
|
diagnostics.push(
|
|
11923
|
-
|
|
14113
|
+
diagnostic5(
|
|
11924
14114
|
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
11925
14115
|
`Expected observation "${name}" was not recorded.`,
|
|
11926
14116
|
"error",
|
|
@@ -11931,7 +14121,7 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
11931
14121
|
}
|
|
11932
14122
|
if (match.status !== "passed") {
|
|
11933
14123
|
diagnostics.push(
|
|
11934
|
-
|
|
14124
|
+
diagnostic5(
|
|
11935
14125
|
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
11936
14126
|
`Observation "${name}" has status "${match.status}", expected "passed".`,
|
|
11937
14127
|
"error",
|
|
@@ -11951,7 +14141,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11951
14141
|
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
11952
14142
|
message: resolved.reason,
|
|
11953
14143
|
diagnostics: [
|
|
11954
|
-
|
|
14144
|
+
diagnostic5(
|
|
11955
14145
|
"AI_SUITE_CASE_TRACE_MISSING",
|
|
11956
14146
|
resolved.reason ?? "Trace not found.",
|
|
11957
14147
|
"warning",
|
|
@@ -11972,7 +14162,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11972
14162
|
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
11973
14163
|
message,
|
|
11974
14164
|
diagnostics: [
|
|
11975
|
-
|
|
14165
|
+
diagnostic5("AI_SUITE_TRACE_UNREADABLE", message, "error", suiteCase.id)
|
|
11976
14166
|
]
|
|
11977
14167
|
};
|
|
11978
14168
|
}
|
|
@@ -11986,7 +14176,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11986
14176
|
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
11987
14177
|
const diagnostics = [
|
|
11988
14178
|
...checkResult.diagnostics.map(
|
|
11989
|
-
(item) =>
|
|
14179
|
+
(item) => diagnostic5(
|
|
11990
14180
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
11991
14181
|
item.message,
|
|
11992
14182
|
item.severity,
|
|
@@ -11994,7 +14184,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11994
14184
|
)
|
|
11995
14185
|
),
|
|
11996
14186
|
...checkResult.findings.filter((finding) => finding.status === "fail").map(
|
|
11997
|
-
(finding) =>
|
|
14187
|
+
(finding) => diagnostic5(
|
|
11998
14188
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
11999
14189
|
finding.message,
|
|
12000
14190
|
finding.severity,
|
|
@@ -13811,7 +16001,7 @@ var init_src = __esm({
|
|
|
13811
16001
|
});
|
|
13812
16002
|
|
|
13813
16003
|
// package.json
|
|
13814
|
-
var version = "6.
|
|
16004
|
+
var version = "6.30.0";
|
|
13815
16005
|
|
|
13816
16006
|
// packages/cli/src/list.ts
|
|
13817
16007
|
init_advanced();
|
|
@@ -18660,10 +20850,10 @@ function buildSafetyRules(options) {
|
|
|
18660
20850
|
})
|
|
18661
20851
|
];
|
|
18662
20852
|
}
|
|
18663
|
-
function
|
|
20853
|
+
function flattenNodes3(nodes) {
|
|
18664
20854
|
return nodes.flatMap((node) => [
|
|
18665
20855
|
node,
|
|
18666
|
-
...
|
|
20856
|
+
...flattenNodes3(
|
|
18667
20857
|
node.children
|
|
18668
20858
|
)
|
|
18669
20859
|
]);
|
|
@@ -18698,7 +20888,7 @@ function redactionDetectorFindings(read, runId, policy) {
|
|
|
18698
20888
|
const out = [];
|
|
18699
20889
|
const policyOptions = redactionOptionsFromPolicy(policy);
|
|
18700
20890
|
for (const run of runs) {
|
|
18701
|
-
for (const node of
|
|
20891
|
+
for (const node of flattenNodes3(run.children)) {
|
|
18702
20892
|
const attrs = node.event.attributes;
|
|
18703
20893
|
if (attrs === void 0) continue;
|
|
18704
20894
|
const result = createRedactor({
|
|
@@ -18838,8 +21028,8 @@ function printHuman(result, explain = false) {
|
|
|
18838
21028
|
console.log(
|
|
18839
21029
|
`Summary: ${result.summary.findings} finding(s), ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
18840
21030
|
);
|
|
18841
|
-
for (const
|
|
18842
|
-
console.log(`- ${
|
|
21031
|
+
for (const diagnostic8 of result.diagnostics) {
|
|
21032
|
+
console.log(`- ${diagnostic8.code}: ${diagnostic8.message}`);
|
|
18843
21033
|
}
|
|
18844
21034
|
for (const finding of result.findings) {
|
|
18845
21035
|
const path44 = finding.evidence[0]?.path;
|
|
@@ -19756,7 +21946,8 @@ async function writeLocalEvidence(input3) {
|
|
|
19756
21946
|
redactionProfile: profile,
|
|
19757
21947
|
assessmentStatus,
|
|
19758
21948
|
checkResultsJson: input3.checkResultsJson,
|
|
19759
|
-
...input3.summaryText !== void 0 ? { summaryText: input3.summaryText } : {}
|
|
21949
|
+
...input3.summaryText !== void 0 ? { summaryText: input3.summaryText } : {},
|
|
21950
|
+
...input3.contractPackage !== void 0 ? { contractPackage: input3.contractPackage } : {}
|
|
19760
21951
|
});
|
|
19761
21952
|
const files = [
|
|
19762
21953
|
["evidence.html", evidencePackage["evidence.html"]],
|
|
@@ -19764,6 +21955,10 @@ async function writeLocalEvidence(input3) {
|
|
|
19764
21955
|
["check-results.json", evidencePackage["check-results.json"]],
|
|
19765
21956
|
["trace.jsonl", evidencePackage["trace.jsonl"]]
|
|
19766
21957
|
];
|
|
21958
|
+
const resolvedContract = evidencePackage["contract.resolved.json"];
|
|
21959
|
+
if (resolvedContract !== void 0) {
|
|
21960
|
+
files.push(["contract.resolved.json", resolvedContract]);
|
|
21961
|
+
}
|
|
19767
21962
|
const expected = createEvidenceCiArtifacts({
|
|
19768
21963
|
redactionProfile: profile
|
|
19769
21964
|
}).map((a) => a.path);
|
|
@@ -20073,7 +22268,7 @@ var DEFAULT_INJECTION_PATTERNS = [
|
|
|
20073
22268
|
function pass(ruleId, message) {
|
|
20074
22269
|
return { ruleId, status: "pass", severity: "info", message, evidence: [] };
|
|
20075
22270
|
}
|
|
20076
|
-
function
|
|
22271
|
+
function fail4(ruleId, message, evidence, severity = "error") {
|
|
20077
22272
|
return { ruleId, status: severity === "warning" ? "warn" : "fail", severity, message, evidence };
|
|
20078
22273
|
}
|
|
20079
22274
|
function boundedPreview(value, max = 80) {
|
|
@@ -20093,7 +22288,7 @@ function evaluateBannedPhrase(text, options) {
|
|
|
20093
22288
|
if (evidence.length === 0) {
|
|
20094
22289
|
return pass(ruleId, "No banned phrases matched.");
|
|
20095
22290
|
}
|
|
20096
|
-
return
|
|
22291
|
+
return fail4(ruleId, `Matched ${evidence.length} banned phrase(s).`, evidence);
|
|
20097
22292
|
}
|
|
20098
22293
|
var SEVERITY_RANK2 = { info: 0, warning: 1, error: 2 };
|
|
20099
22294
|
function evaluatePiiLeak(value, options = {}) {
|
|
@@ -20112,7 +22307,7 @@ function evaluatePiiLeak(value, options = {}) {
|
|
|
20112
22307
|
detector: finding.detector,
|
|
20113
22308
|
preview: finding.preview
|
|
20114
22309
|
}));
|
|
20115
|
-
return
|
|
22310
|
+
return fail4(ruleId, `Detected ${findings.length} PII-style finding(s).`, evidence);
|
|
20116
22311
|
}
|
|
20117
22312
|
function measureDepth(value) {
|
|
20118
22313
|
if (value === null || typeof value !== "object") return 0;
|
|
@@ -20150,7 +22345,7 @@ function evaluateUnsafeToolArgs(toolName2, toolArgs2, options = {}) {
|
|
|
20150
22345
|
if (evidence.length === 0) {
|
|
20151
22346
|
return pass(ruleId, "Tool arguments within configured bounds.");
|
|
20152
22347
|
}
|
|
20153
|
-
return
|
|
22348
|
+
return fail4(ruleId, "Unsafe or oversized tool arguments detected.", evidence);
|
|
20154
22349
|
}
|
|
20155
22350
|
function evaluatePromptInjection(text, options = {}) {
|
|
20156
22351
|
const ruleId = "guardrail.prompt-injection";
|
|
@@ -20166,7 +22361,7 @@ function evaluatePromptInjection(text, options = {}) {
|
|
|
20166
22361
|
if (evidence.length === 0) {
|
|
20167
22362
|
return pass(ruleId, "No prompt-injection patterns matched.");
|
|
20168
22363
|
}
|
|
20169
|
-
return
|
|
22364
|
+
return fail4(ruleId, `Matched ${evidence.length} injection pattern(s).`, evidence, "warning");
|
|
20170
22365
|
}
|
|
20171
22366
|
function validateSchemaField(value, field, path44, evidence) {
|
|
20172
22367
|
const ruleId = "guardrail.structured-output";
|
|
@@ -20192,7 +22387,7 @@ function validateSchemaField(value, field, path44, evidence) {
|
|
|
20192
22387
|
function evaluateStructuredOutput(value, options) {
|
|
20193
22388
|
const ruleId = "guardrail.structured-output";
|
|
20194
22389
|
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
20195
|
-
return
|
|
22390
|
+
return fail4(ruleId, "Structured output must be an object.", [
|
|
20196
22391
|
{ ruleId, preview: typeof value }
|
|
20197
22392
|
]);
|
|
20198
22393
|
}
|
|
@@ -20204,7 +22399,7 @@ function evaluateStructuredOutput(value, options) {
|
|
|
20204
22399
|
if (evidence.length === 0) {
|
|
20205
22400
|
return pass(ruleId, "Structured output matches schema subset.");
|
|
20206
22401
|
}
|
|
20207
|
-
return
|
|
22402
|
+
return fail4(ruleId, "Structured output schema violation.", evidence);
|
|
20208
22403
|
}
|
|
20209
22404
|
function evaluateOversizeOutput(value, options = {}) {
|
|
20210
22405
|
const ruleId = "guardrail.oversize-output";
|
|
@@ -20213,7 +22408,7 @@ function evaluateOversizeOutput(value, options = {}) {
|
|
|
20213
22408
|
if (text.length <= maxLength) {
|
|
20214
22409
|
return pass(ruleId, "Output within size limits.");
|
|
20215
22410
|
}
|
|
20216
|
-
return
|
|
22411
|
+
return fail4(ruleId, `Output exceeds max length (${text.length} > ${maxLength}).`, [
|
|
20217
22412
|
{ ruleId, preview: `length=${text.length}` }
|
|
20218
22413
|
]);
|
|
20219
22414
|
}
|
|
@@ -20224,11 +22419,11 @@ function evaluateRequiredJsonShape(value, options) {
|
|
|
20224
22419
|
try {
|
|
20225
22420
|
parsed = JSON.parse(value);
|
|
20226
22421
|
} catch {
|
|
20227
|
-
return
|
|
22422
|
+
return fail4(ruleId, "Value is not valid JSON.", [{ ruleId, preview: boundedPreview(value) }]);
|
|
20228
22423
|
}
|
|
20229
22424
|
}
|
|
20230
22425
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
20231
|
-
return
|
|
22426
|
+
return fail4(ruleId, "JSON value must be an object.", [{ ruleId, preview: typeof parsed }]);
|
|
20232
22427
|
}
|
|
20233
22428
|
const record = parsed;
|
|
20234
22429
|
const evidence = [];
|
|
@@ -20240,7 +22435,7 @@ function evaluateRequiredJsonShape(value, options) {
|
|
|
20240
22435
|
if (evidence.length === 0) {
|
|
20241
22436
|
return pass(ruleId, "Required JSON keys present.");
|
|
20242
22437
|
}
|
|
20243
|
-
return
|
|
22438
|
+
return fail4(ruleId, "Missing required JSON keys.", evidence);
|
|
20244
22439
|
}
|
|
20245
22440
|
|
|
20246
22441
|
// packages/guardrails/src/run.ts
|
|
@@ -20688,11 +22883,11 @@ function resolvePreset(preset, context = {}) {
|
|
|
20688
22883
|
}
|
|
20689
22884
|
var CONFIG_EXTENSIONS2 = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
20690
22885
|
var TS_CONFIG_EXTENSIONS2 = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
20691
|
-
function
|
|
22886
|
+
function diagnostic6(code, message, severity = "error") {
|
|
20692
22887
|
return { code, message, severity };
|
|
20693
22888
|
}
|
|
20694
22889
|
function errorResult2(code, message, format = "unknown") {
|
|
20695
|
-
const diagnostics = [
|
|
22890
|
+
const diagnostics = [diagnostic6(code, message)];
|
|
20696
22891
|
return {
|
|
20697
22892
|
ok: false,
|
|
20698
22893
|
status: "error",
|
|
@@ -20725,6 +22920,131 @@ function asStringArray2(value) {
|
|
|
20725
22920
|
return value;
|
|
20726
22921
|
}
|
|
20727
22922
|
var CHECKS_KEYS = /* @__PURE__ */ new Set(["select", "run", "tool", "llm", "structure", "safety"]);
|
|
22923
|
+
var ROOT_CONFIG_KEYS = /* @__PURE__ */ new Set(["checks", "contract"]);
|
|
22924
|
+
var CONTRACT_KEYS = /* @__PURE__ */ new Set([
|
|
22925
|
+
"run",
|
|
22926
|
+
"tools",
|
|
22927
|
+
"llm",
|
|
22928
|
+
"observations",
|
|
22929
|
+
"scope",
|
|
22930
|
+
"alternatives",
|
|
22931
|
+
"controls",
|
|
22932
|
+
"retry"
|
|
22933
|
+
]);
|
|
22934
|
+
var CONTRACT_BODY_KEYS = /* @__PURE__ */ new Set([
|
|
22935
|
+
"run",
|
|
22936
|
+
"tools",
|
|
22937
|
+
"llm",
|
|
22938
|
+
"observations",
|
|
22939
|
+
"controls",
|
|
22940
|
+
"retry"
|
|
22941
|
+
]);
|
|
22942
|
+
var CONTRACT_SCOPE_KEYS = /* @__PURE__ */ new Set([
|
|
22943
|
+
"runId",
|
|
22944
|
+
"subAgentId",
|
|
22945
|
+
"groupId",
|
|
22946
|
+
"workflowStep",
|
|
22947
|
+
"rootEventId"
|
|
22948
|
+
]);
|
|
22949
|
+
var CONTRACT_RUN_KEYS = /* @__PURE__ */ new Set(["requireCompleted", "allowedStatuses", "maxDurationMs"]);
|
|
22950
|
+
var CONTRACT_TOOLS_KEYS = /* @__PURE__ */ new Set([
|
|
22951
|
+
"required",
|
|
22952
|
+
"requiredTools",
|
|
22953
|
+
"forbidden",
|
|
22954
|
+
"forbiddenTools",
|
|
22955
|
+
"allowed",
|
|
22956
|
+
"maxCalls",
|
|
22957
|
+
"requiredOrder",
|
|
22958
|
+
"requiredOrderMode",
|
|
22959
|
+
"arguments",
|
|
22960
|
+
"orderRules",
|
|
22961
|
+
"defaultOccurrenceMode"
|
|
22962
|
+
]);
|
|
22963
|
+
var CONTRACT_LLM_KEYS = /* @__PURE__ */ new Set(["maxCalls", "maxTotalTokens", "allowedModels"]);
|
|
22964
|
+
var CONTRACT_OBSERVATION_KEYS = /* @__PURE__ */ new Set(["required", "failOn", "requireProvenance"]);
|
|
22965
|
+
var CONTRACT_PROVENANCE_KEYS = /* @__PURE__ */ new Set(["method", "evidence", "sameRunEventReference"]);
|
|
22966
|
+
var CONTRACT_CONTROLS_KEYS = /* @__PURE__ */ new Set([
|
|
22967
|
+
"declaredTools",
|
|
22968
|
+
"enforcedTools",
|
|
22969
|
+
"declaredToolsAttribute",
|
|
22970
|
+
"enforcedToolsAttribute",
|
|
22971
|
+
"requireDeclaredMatchesEnforced",
|
|
22972
|
+
"requireObservedWithinEnforced",
|
|
22973
|
+
"requireObservedWithinDeclared",
|
|
22974
|
+
"requiredStages"
|
|
22975
|
+
]);
|
|
22976
|
+
var CONTRACT_CONTROL_STAGES = /* @__PURE__ */ new Set([
|
|
22977
|
+
"declared",
|
|
22978
|
+
"presented",
|
|
22979
|
+
"validated",
|
|
22980
|
+
"enforced",
|
|
22981
|
+
"observed",
|
|
22982
|
+
"accepted"
|
|
22983
|
+
]);
|
|
22984
|
+
var CONTRACT_CONTROL_STAGE_KEYS = /* @__PURE__ */ new Set(["stage", "observation"]);
|
|
22985
|
+
var CONTRACT_RETRY_KEYS = /* @__PURE__ */ new Set([
|
|
22986
|
+
"maxAttempts",
|
|
22987
|
+
"requireTerminalResult",
|
|
22988
|
+
"fallbackOnlyAfterFailure",
|
|
22989
|
+
"nonIdempotentTools",
|
|
22990
|
+
"requireIdempotencyEvidenceForRetry",
|
|
22991
|
+
"requireRecoveredFailureVisible",
|
|
22992
|
+
"operations"
|
|
22993
|
+
]);
|
|
22994
|
+
var CONTRACT_OPERATION_KEYS = /* @__PURE__ */ new Set([
|
|
22995
|
+
"tool",
|
|
22996
|
+
"maxAttempts",
|
|
22997
|
+
"retryableErrors",
|
|
22998
|
+
"requireFailureBeforeRetry",
|
|
22999
|
+
"requireSameArguments",
|
|
23000
|
+
"requireTerminalSuccess",
|
|
23001
|
+
"requireRecoveredFailureVisible",
|
|
23002
|
+
"successfulResultDependency",
|
|
23003
|
+
"sideEffectClass"
|
|
23004
|
+
]);
|
|
23005
|
+
var CONTRACT_RETRYABLE_ERRORS_KEYS = /* @__PURE__ */ new Set(["codes"]);
|
|
23006
|
+
var CONTRACT_RESULT_DEP_KEYS = /* @__PURE__ */ new Set(["consumerKind", "requireExplicitReference"]);
|
|
23007
|
+
var CONTRACT_ALT_KEYS = /* @__PURE__ */ new Set(["anyOf"]);
|
|
23008
|
+
var CONTRACT_ALT_BRANCH_KEYS = /* @__PURE__ */ new Set(["id", "description", "contract"]);
|
|
23009
|
+
var CONTRACT_ORDER_RULE_KEYS = /* @__PURE__ */ new Set([
|
|
23010
|
+
"before",
|
|
23011
|
+
"after",
|
|
23012
|
+
"occurrenceMode",
|
|
23013
|
+
"requireEndpoints"
|
|
23014
|
+
]);
|
|
23015
|
+
var CONTRACT_ARGUMENT_KEYS = /* @__PURE__ */ new Set([
|
|
23016
|
+
"tool",
|
|
23017
|
+
"path",
|
|
23018
|
+
"operator",
|
|
23019
|
+
"occurrence",
|
|
23020
|
+
"expected",
|
|
23021
|
+
"oneOf",
|
|
23022
|
+
"type"
|
|
23023
|
+
]);
|
|
23024
|
+
var CONTRACT_ARGUMENT_OPERATORS = /* @__PURE__ */ new Set(["exists", "type", "equals", "oneOf"]);
|
|
23025
|
+
var CONTRACT_ARGUMENT_OCCURRENCES = /* @__PURE__ */ new Set(["first", "last", "any", "all"]);
|
|
23026
|
+
var CONTRACT_ARGUMENT_TYPES = /* @__PURE__ */ new Set([
|
|
23027
|
+
"string",
|
|
23028
|
+
"number",
|
|
23029
|
+
"boolean",
|
|
23030
|
+
"object",
|
|
23031
|
+
"array",
|
|
23032
|
+
"null"
|
|
23033
|
+
]);
|
|
23034
|
+
var CONTRACT_ORDER_MODES = /* @__PURE__ */ new Set([
|
|
23035
|
+
"first-occurrence",
|
|
23036
|
+
"happens-before",
|
|
23037
|
+
"all-occurrences"
|
|
23038
|
+
]);
|
|
23039
|
+
var CONTRACT_FAIL_ON = /* @__PURE__ */ new Set(["failed", "unknown", "skipped"]);
|
|
23040
|
+
var CONTRACT_SIDE_EFFECT_CLASSES = /* @__PURE__ */ new Set(["read", "write"]);
|
|
23041
|
+
var CONTRACT_MAX_STRING_LIST = 64;
|
|
23042
|
+
var CONTRACT_MAX_ARGUMENTS = 32;
|
|
23043
|
+
var CONTRACT_MAX_ORDER_RULES = 32;
|
|
23044
|
+
var CONTRACT_MAX_ANY_OF = 16;
|
|
23045
|
+
var CONTRACT_MAX_OPERATIONS = 32;
|
|
23046
|
+
var CONTRACT_MAX_REQUIRED_STAGES = 16;
|
|
23047
|
+
var CONTRACT_MAX_ONE_OF = 32;
|
|
20728
23048
|
var RUN_KEYS = /* @__PURE__ */ new Set(["expected", "allowIncomplete", "maxDurationMs", "maxDepth"]);
|
|
20729
23049
|
var TOOL_KEYS = /* @__PURE__ */ new Set(["required", "forbidden", "allowed", "minCount", "maxCount"]);
|
|
20730
23050
|
var LLM_KEYS = /* @__PURE__ */ new Set([
|
|
@@ -20826,7 +23146,50 @@ function requireBoolean(value, pathPrefix) {
|
|
|
20826
23146
|
if (typeof value !== "boolean") {
|
|
20827
23147
|
throw new CheckConfigError(
|
|
20828
23148
|
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
20829
|
-
`${pathPrefix} must be a boolean.`
|
|
23149
|
+
`${pathPrefix} must be a boolean.`
|
|
23150
|
+
);
|
|
23151
|
+
}
|
|
23152
|
+
return value;
|
|
23153
|
+
}
|
|
23154
|
+
function requireNonEmptyString(value, pathPrefix) {
|
|
23155
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
23156
|
+
throw new CheckConfigError(
|
|
23157
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23158
|
+
`${pathPrefix} must be a non-empty string.`
|
|
23159
|
+
);
|
|
23160
|
+
}
|
|
23161
|
+
return value;
|
|
23162
|
+
}
|
|
23163
|
+
function requireBoundedArray(value, pathPrefix, max) {
|
|
23164
|
+
if (!Array.isArray(value)) {
|
|
23165
|
+
throw new CheckConfigError(
|
|
23166
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23167
|
+
`${pathPrefix} must be an array.`
|
|
23168
|
+
);
|
|
23169
|
+
}
|
|
23170
|
+
if (value.length > max) {
|
|
23171
|
+
throw new CheckConfigError(
|
|
23172
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23173
|
+
`${pathPrefix} exceeds bound of ${max} items.`
|
|
23174
|
+
);
|
|
23175
|
+
}
|
|
23176
|
+
return value;
|
|
23177
|
+
}
|
|
23178
|
+
function requireBoundedStringArray(value, pathPrefix, max = CONTRACT_MAX_STRING_LIST) {
|
|
23179
|
+
const items = requireBoundedArray(value, pathPrefix, max);
|
|
23180
|
+
if (items.some((item) => typeof item !== "string" || item.trim() === "")) {
|
|
23181
|
+
throw new CheckConfigError(
|
|
23182
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23183
|
+
`${pathPrefix} must be an array of non-empty strings.`
|
|
23184
|
+
);
|
|
23185
|
+
}
|
|
23186
|
+
return items;
|
|
23187
|
+
}
|
|
23188
|
+
function requireOrderMode(value, pathPrefix) {
|
|
23189
|
+
if (typeof value !== "string" || !CONTRACT_ORDER_MODES.has(value)) {
|
|
23190
|
+
throw new CheckConfigError(
|
|
23191
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23192
|
+
`${pathPrefix} must be one of: "first-occurrence", "happens-before", "all-occurrences".`
|
|
20830
23193
|
);
|
|
20831
23194
|
}
|
|
20832
23195
|
return value;
|
|
@@ -20963,6 +23326,7 @@ function sectionHasEffect(section) {
|
|
|
20963
23326
|
return Object.keys(section).length > 0;
|
|
20964
23327
|
}
|
|
20965
23328
|
function checkConfigHasEffect(config) {
|
|
23329
|
+
if (contractConfigHasEffect(config.contract)) return true;
|
|
20966
23330
|
const checks2 = config.checks;
|
|
20967
23331
|
if (checks2 === void 0) return false;
|
|
20968
23332
|
if ((checks2.select?.length ?? 0) > 0) return true;
|
|
@@ -20973,43 +23337,587 @@ function checkConfigHasEffect(config) {
|
|
|
20973
23337
|
if (sectionHasEffect(checks2.safety)) return true;
|
|
20974
23338
|
return false;
|
|
20975
23339
|
}
|
|
23340
|
+
function contractConfigHasEffect(contract) {
|
|
23341
|
+
if (contract === void 0) return false;
|
|
23342
|
+
if (sectionHasEffect(contract.run)) return true;
|
|
23343
|
+
if (sectionHasEffect(contract.tools)) return true;
|
|
23344
|
+
if (sectionHasEffect(contract.llm)) return true;
|
|
23345
|
+
if (sectionHasEffect(contract.observations)) {
|
|
23346
|
+
return true;
|
|
23347
|
+
}
|
|
23348
|
+
if (sectionHasEffect(contract.controls)) return true;
|
|
23349
|
+
if (sectionHasEffect(contract.retry)) return true;
|
|
23350
|
+
if ((contract.alternatives?.anyOf?.length ?? 0) > 0) return true;
|
|
23351
|
+
return false;
|
|
23352
|
+
}
|
|
23353
|
+
function parseContractRunSection(value, pathPrefix = "contract.run") {
|
|
23354
|
+
const record = requireObject(value, pathPrefix);
|
|
23355
|
+
rejectUnknownKeys(record, CONTRACT_RUN_KEYS, pathPrefix);
|
|
23356
|
+
const out = {};
|
|
23357
|
+
if (record.requireCompleted !== void 0) {
|
|
23358
|
+
out.requireCompleted = requireBoolean(
|
|
23359
|
+
record.requireCompleted,
|
|
23360
|
+
`${pathPrefix}.requireCompleted`
|
|
23361
|
+
);
|
|
23362
|
+
}
|
|
23363
|
+
if (record.allowedStatuses !== void 0) {
|
|
23364
|
+
out.allowedStatuses = requireBoundedStringArray(
|
|
23365
|
+
record.allowedStatuses,
|
|
23366
|
+
`${pathPrefix}.allowedStatuses`
|
|
23367
|
+
);
|
|
23368
|
+
}
|
|
23369
|
+
if (record.maxDurationMs !== void 0) {
|
|
23370
|
+
out.maxDurationMs = requireNonNegativeNumber(
|
|
23371
|
+
record.maxDurationMs,
|
|
23372
|
+
`${pathPrefix}.maxDurationMs`
|
|
23373
|
+
);
|
|
23374
|
+
}
|
|
23375
|
+
return out;
|
|
23376
|
+
}
|
|
23377
|
+
function parseToolArgumentCheck(value, pathPrefix) {
|
|
23378
|
+
const record = requireObject(value, pathPrefix);
|
|
23379
|
+
rejectUnknownKeys(record, CONTRACT_ARGUMENT_KEYS, pathPrefix);
|
|
23380
|
+
const tool = requireNonEmptyString(record.tool, `${pathPrefix}.tool`);
|
|
23381
|
+
if (typeof record.path !== "string") {
|
|
23382
|
+
throw new CheckConfigError(
|
|
23383
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23384
|
+
`${pathPrefix}.path must be a string JSON Pointer ("" or start with "/").`
|
|
23385
|
+
);
|
|
23386
|
+
}
|
|
23387
|
+
const pathValue = record.path;
|
|
23388
|
+
if (typeof record.operator !== "string" || !CONTRACT_ARGUMENT_OPERATORS.has(record.operator)) {
|
|
23389
|
+
throw new CheckConfigError(
|
|
23390
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23391
|
+
`${pathPrefix}.operator must be one of: "exists", "type", "equals", "oneOf".`
|
|
23392
|
+
);
|
|
23393
|
+
}
|
|
23394
|
+
const operator = record.operator;
|
|
23395
|
+
const out = { tool, path: pathValue, operator };
|
|
23396
|
+
if (record.occurrence !== void 0) {
|
|
23397
|
+
if (typeof record.occurrence !== "string" || !CONTRACT_ARGUMENT_OCCURRENCES.has(record.occurrence)) {
|
|
23398
|
+
throw new CheckConfigError(
|
|
23399
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23400
|
+
`${pathPrefix}.occurrence must be one of: "first", "last", "any", "all".`
|
|
23401
|
+
);
|
|
23402
|
+
}
|
|
23403
|
+
out.occurrence = record.occurrence;
|
|
23404
|
+
}
|
|
23405
|
+
if (record.expected !== void 0) {
|
|
23406
|
+
out.expected = record.expected;
|
|
23407
|
+
}
|
|
23408
|
+
if (record.oneOf !== void 0) {
|
|
23409
|
+
out.oneOf = requireBoundedArray(record.oneOf, `${pathPrefix}.oneOf`, CONTRACT_MAX_ONE_OF);
|
|
23410
|
+
}
|
|
23411
|
+
if (record.type !== void 0) {
|
|
23412
|
+
if (typeof record.type !== "string" || !CONTRACT_ARGUMENT_TYPES.has(record.type)) {
|
|
23413
|
+
throw new CheckConfigError(
|
|
23414
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23415
|
+
`${pathPrefix}.type must be one of: "string", "number", "boolean", "object", "array", "null".`
|
|
23416
|
+
);
|
|
23417
|
+
}
|
|
23418
|
+
out.type = record.type;
|
|
23419
|
+
}
|
|
23420
|
+
return out;
|
|
23421
|
+
}
|
|
23422
|
+
function parseOrderRule(value, pathPrefix) {
|
|
23423
|
+
const record = requireObject(value, pathPrefix);
|
|
23424
|
+
rejectUnknownKeys(record, CONTRACT_ORDER_RULE_KEYS, pathPrefix);
|
|
23425
|
+
const out = {
|
|
23426
|
+
before: requireNonEmptyString(record.before, `${pathPrefix}.before`),
|
|
23427
|
+
after: requireNonEmptyString(record.after, `${pathPrefix}.after`)
|
|
23428
|
+
};
|
|
23429
|
+
if (record.occurrenceMode !== void 0) {
|
|
23430
|
+
out.occurrenceMode = requireOrderMode(record.occurrenceMode, `${pathPrefix}.occurrenceMode`);
|
|
23431
|
+
}
|
|
23432
|
+
if (record.requireEndpoints !== void 0) {
|
|
23433
|
+
out.requireEndpoints = requireBoolean(
|
|
23434
|
+
record.requireEndpoints,
|
|
23435
|
+
`${pathPrefix}.requireEndpoints`
|
|
23436
|
+
);
|
|
23437
|
+
}
|
|
23438
|
+
return out;
|
|
23439
|
+
}
|
|
23440
|
+
function parseContractToolsSection(value, pathPrefix = "contract.tools") {
|
|
23441
|
+
const record = requireObject(value, pathPrefix);
|
|
23442
|
+
rejectUnknownKeys(record, CONTRACT_TOOLS_KEYS, pathPrefix);
|
|
23443
|
+
const out = {};
|
|
23444
|
+
if (record.required !== void 0) {
|
|
23445
|
+
out.required = requireBoundedStringArray(record.required, `${pathPrefix}.required`);
|
|
23446
|
+
}
|
|
23447
|
+
if (record.requiredTools !== void 0) {
|
|
23448
|
+
out.requiredTools = requireBoundedStringArray(
|
|
23449
|
+
record.requiredTools,
|
|
23450
|
+
`${pathPrefix}.requiredTools`
|
|
23451
|
+
);
|
|
23452
|
+
}
|
|
23453
|
+
if (record.forbidden !== void 0) {
|
|
23454
|
+
out.forbidden = requireBoundedStringArray(record.forbidden, `${pathPrefix}.forbidden`);
|
|
23455
|
+
}
|
|
23456
|
+
if (record.forbiddenTools !== void 0) {
|
|
23457
|
+
out.forbiddenTools = requireBoundedStringArray(
|
|
23458
|
+
record.forbiddenTools,
|
|
23459
|
+
`${pathPrefix}.forbiddenTools`
|
|
23460
|
+
);
|
|
23461
|
+
}
|
|
23462
|
+
if (record.allowed !== void 0) {
|
|
23463
|
+
out.allowed = requireBoundedStringArray(record.allowed, `${pathPrefix}.allowed`);
|
|
23464
|
+
}
|
|
23465
|
+
if (record.maxCalls !== void 0) {
|
|
23466
|
+
out.maxCalls = requireNonNegativeNumber(record.maxCalls, `${pathPrefix}.maxCalls`);
|
|
23467
|
+
}
|
|
23468
|
+
if (record.requiredOrder !== void 0) {
|
|
23469
|
+
out.requiredOrder = requireBoundedStringArray(
|
|
23470
|
+
record.requiredOrder,
|
|
23471
|
+
`${pathPrefix}.requiredOrder`
|
|
23472
|
+
);
|
|
23473
|
+
}
|
|
23474
|
+
if (record.requiredOrderMode !== void 0) {
|
|
23475
|
+
out.requiredOrderMode = requireOrderMode(
|
|
23476
|
+
record.requiredOrderMode,
|
|
23477
|
+
`${pathPrefix}.requiredOrderMode`
|
|
23478
|
+
);
|
|
23479
|
+
}
|
|
23480
|
+
if (record.defaultOccurrenceMode !== void 0) {
|
|
23481
|
+
out.defaultOccurrenceMode = requireOrderMode(
|
|
23482
|
+
record.defaultOccurrenceMode,
|
|
23483
|
+
`${pathPrefix}.defaultOccurrenceMode`
|
|
23484
|
+
);
|
|
23485
|
+
}
|
|
23486
|
+
if (record.arguments !== void 0) {
|
|
23487
|
+
const items = requireBoundedArray(
|
|
23488
|
+
record.arguments,
|
|
23489
|
+
`${pathPrefix}.arguments`,
|
|
23490
|
+
CONTRACT_MAX_ARGUMENTS
|
|
23491
|
+
);
|
|
23492
|
+
out.arguments = items.map(
|
|
23493
|
+
(item, index) => parseToolArgumentCheck(item, `${pathPrefix}.arguments[${index}]`)
|
|
23494
|
+
);
|
|
23495
|
+
}
|
|
23496
|
+
if (record.orderRules !== void 0) {
|
|
23497
|
+
const items = requireBoundedArray(
|
|
23498
|
+
record.orderRules,
|
|
23499
|
+
`${pathPrefix}.orderRules`,
|
|
23500
|
+
CONTRACT_MAX_ORDER_RULES
|
|
23501
|
+
);
|
|
23502
|
+
out.orderRules = items.map(
|
|
23503
|
+
(item, index) => parseOrderRule(item, `${pathPrefix}.orderRules[${index}]`)
|
|
23504
|
+
);
|
|
23505
|
+
}
|
|
23506
|
+
return out;
|
|
23507
|
+
}
|
|
23508
|
+
function parseContractLlmSection(value, pathPrefix = "contract.llm") {
|
|
23509
|
+
const record = requireObject(value, pathPrefix);
|
|
23510
|
+
rejectUnknownKeys(record, CONTRACT_LLM_KEYS, pathPrefix);
|
|
23511
|
+
const out = {};
|
|
23512
|
+
if (record.maxCalls !== void 0) {
|
|
23513
|
+
out.maxCalls = requireNonNegativeNumber(record.maxCalls, `${pathPrefix}.maxCalls`);
|
|
23514
|
+
}
|
|
23515
|
+
if (record.maxTotalTokens !== void 0) {
|
|
23516
|
+
out.maxTotalTokens = requireNonNegativeNumber(
|
|
23517
|
+
record.maxTotalTokens,
|
|
23518
|
+
`${pathPrefix}.maxTotalTokens`
|
|
23519
|
+
);
|
|
23520
|
+
}
|
|
23521
|
+
if (record.allowedModels !== void 0) {
|
|
23522
|
+
out.allowedModels = requireBoundedStringArray(
|
|
23523
|
+
record.allowedModels,
|
|
23524
|
+
`${pathPrefix}.allowedModels`
|
|
23525
|
+
);
|
|
23526
|
+
}
|
|
23527
|
+
return out;
|
|
23528
|
+
}
|
|
23529
|
+
function parseRequireProvenance(value, pathPrefix) {
|
|
23530
|
+
const record = requireObject(value, pathPrefix);
|
|
23531
|
+
rejectUnknownKeys(record, CONTRACT_PROVENANCE_KEYS, pathPrefix);
|
|
23532
|
+
const out = {};
|
|
23533
|
+
if (record.method !== void 0) {
|
|
23534
|
+
out.method = requireBoolean(record.method, `${pathPrefix}.method`);
|
|
23535
|
+
}
|
|
23536
|
+
if (record.evidence !== void 0) {
|
|
23537
|
+
out.evidence = requireBoolean(record.evidence, `${pathPrefix}.evidence`);
|
|
23538
|
+
}
|
|
23539
|
+
if (record.sameRunEventReference !== void 0) {
|
|
23540
|
+
out.sameRunEventReference = requireBoolean(
|
|
23541
|
+
record.sameRunEventReference,
|
|
23542
|
+
`${pathPrefix}.sameRunEventReference`
|
|
23543
|
+
);
|
|
23544
|
+
}
|
|
23545
|
+
return out;
|
|
23546
|
+
}
|
|
23547
|
+
function parseContractObservationsSection(value, pathPrefix = "contract.observations") {
|
|
23548
|
+
const record = requireObject(value, pathPrefix);
|
|
23549
|
+
rejectUnknownKeys(record, CONTRACT_OBSERVATION_KEYS, pathPrefix);
|
|
23550
|
+
const out = {};
|
|
23551
|
+
if (record.required !== void 0) {
|
|
23552
|
+
out.required = requireBoundedStringArray(record.required, `${pathPrefix}.required`);
|
|
23553
|
+
}
|
|
23554
|
+
if (record.failOn !== void 0) {
|
|
23555
|
+
const items = requireBoundedArray(record.failOn, `${pathPrefix}.failOn`, CONTRACT_MAX_STRING_LIST);
|
|
23556
|
+
for (const item of items) {
|
|
23557
|
+
if (typeof item !== "string" || !CONTRACT_FAIL_ON.has(item)) {
|
|
23558
|
+
throw new CheckConfigError(
|
|
23559
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23560
|
+
`${pathPrefix}.failOn values must be "failed", "unknown", or "skipped".`
|
|
23561
|
+
);
|
|
23562
|
+
}
|
|
23563
|
+
}
|
|
23564
|
+
out.failOn = items;
|
|
23565
|
+
}
|
|
23566
|
+
if (record.requireProvenance !== void 0) {
|
|
23567
|
+
out.requireProvenance = parseRequireProvenance(
|
|
23568
|
+
record.requireProvenance,
|
|
23569
|
+
`${pathPrefix}.requireProvenance`
|
|
23570
|
+
);
|
|
23571
|
+
}
|
|
23572
|
+
return out;
|
|
23573
|
+
}
|
|
23574
|
+
function parseControlRequiredStage(value, pathPrefix) {
|
|
23575
|
+
const record = requireObject(value, pathPrefix);
|
|
23576
|
+
rejectUnknownKeys(record, CONTRACT_CONTROL_STAGE_KEYS, pathPrefix);
|
|
23577
|
+
if (typeof record.stage !== "string" || !CONTRACT_CONTROL_STAGES.has(record.stage)) {
|
|
23578
|
+
throw new CheckConfigError(
|
|
23579
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23580
|
+
`${pathPrefix}.stage must be one of: declared, presented, validated, enforced, observed, accepted.`
|
|
23581
|
+
);
|
|
23582
|
+
}
|
|
23583
|
+
const out = {
|
|
23584
|
+
stage: record.stage
|
|
23585
|
+
};
|
|
23586
|
+
if (record.observation !== void 0) {
|
|
23587
|
+
out.observation = requireNonEmptyString(record.observation, `${pathPrefix}.observation`);
|
|
23588
|
+
}
|
|
23589
|
+
return out;
|
|
23590
|
+
}
|
|
23591
|
+
function parseContractControlsSection(value, pathPrefix = "contract.controls") {
|
|
23592
|
+
const record = requireObject(value, pathPrefix);
|
|
23593
|
+
rejectUnknownKeys(record, CONTRACT_CONTROLS_KEYS, pathPrefix);
|
|
23594
|
+
const out = {};
|
|
23595
|
+
if (record.declaredTools !== void 0) {
|
|
23596
|
+
out.declaredTools = requireBoundedStringArray(
|
|
23597
|
+
record.declaredTools,
|
|
23598
|
+
`${pathPrefix}.declaredTools`
|
|
23599
|
+
);
|
|
23600
|
+
}
|
|
23601
|
+
if (record.enforcedTools !== void 0) {
|
|
23602
|
+
out.enforcedTools = requireBoundedStringArray(
|
|
23603
|
+
record.enforcedTools,
|
|
23604
|
+
`${pathPrefix}.enforcedTools`
|
|
23605
|
+
);
|
|
23606
|
+
}
|
|
23607
|
+
if (record.declaredToolsAttribute !== void 0) {
|
|
23608
|
+
out.declaredToolsAttribute = requireNonEmptyString(
|
|
23609
|
+
record.declaredToolsAttribute,
|
|
23610
|
+
`${pathPrefix}.declaredToolsAttribute`
|
|
23611
|
+
);
|
|
23612
|
+
}
|
|
23613
|
+
if (record.enforcedToolsAttribute !== void 0) {
|
|
23614
|
+
out.enforcedToolsAttribute = requireNonEmptyString(
|
|
23615
|
+
record.enforcedToolsAttribute,
|
|
23616
|
+
`${pathPrefix}.enforcedToolsAttribute`
|
|
23617
|
+
);
|
|
23618
|
+
}
|
|
23619
|
+
if (record.requireDeclaredMatchesEnforced !== void 0) {
|
|
23620
|
+
out.requireDeclaredMatchesEnforced = requireBoolean(
|
|
23621
|
+
record.requireDeclaredMatchesEnforced,
|
|
23622
|
+
`${pathPrefix}.requireDeclaredMatchesEnforced`
|
|
23623
|
+
);
|
|
23624
|
+
}
|
|
23625
|
+
if (record.requireObservedWithinEnforced !== void 0) {
|
|
23626
|
+
out.requireObservedWithinEnforced = requireBoolean(
|
|
23627
|
+
record.requireObservedWithinEnforced,
|
|
23628
|
+
`${pathPrefix}.requireObservedWithinEnforced`
|
|
23629
|
+
);
|
|
23630
|
+
}
|
|
23631
|
+
if (record.requireObservedWithinDeclared !== void 0) {
|
|
23632
|
+
out.requireObservedWithinDeclared = requireBoolean(
|
|
23633
|
+
record.requireObservedWithinDeclared,
|
|
23634
|
+
`${pathPrefix}.requireObservedWithinDeclared`
|
|
23635
|
+
);
|
|
23636
|
+
}
|
|
23637
|
+
if (record.requiredStages !== void 0) {
|
|
23638
|
+
const items = requireBoundedArray(
|
|
23639
|
+
record.requiredStages,
|
|
23640
|
+
`${pathPrefix}.requiredStages`,
|
|
23641
|
+
CONTRACT_MAX_REQUIRED_STAGES
|
|
23642
|
+
);
|
|
23643
|
+
out.requiredStages = items.map(
|
|
23644
|
+
(item, index) => parseControlRequiredStage(item, `${pathPrefix}.requiredStages[${index}]`)
|
|
23645
|
+
);
|
|
23646
|
+
}
|
|
23647
|
+
return out;
|
|
23648
|
+
}
|
|
23649
|
+
function parseRecoveryOperation(value, pathPrefix) {
|
|
23650
|
+
const record = requireObject(value, pathPrefix);
|
|
23651
|
+
rejectUnknownKeys(record, CONTRACT_OPERATION_KEYS, pathPrefix);
|
|
23652
|
+
const out = {
|
|
23653
|
+
tool: requireNonEmptyString(record.tool, `${pathPrefix}.tool`)
|
|
23654
|
+
};
|
|
23655
|
+
if (record.maxAttempts !== void 0) {
|
|
23656
|
+
out.maxAttempts = requireNonNegativeNumber(record.maxAttempts, `${pathPrefix}.maxAttempts`);
|
|
23657
|
+
}
|
|
23658
|
+
if (record.retryableErrors !== void 0) {
|
|
23659
|
+
const errors = requireObject(record.retryableErrors, `${pathPrefix}.retryableErrors`);
|
|
23660
|
+
rejectUnknownKeys(errors, CONTRACT_RETRYABLE_ERRORS_KEYS, `${pathPrefix}.retryableErrors`);
|
|
23661
|
+
const retryableErrors = {};
|
|
23662
|
+
if (errors.codes !== void 0) {
|
|
23663
|
+
retryableErrors.codes = requireBoundedStringArray(
|
|
23664
|
+
errors.codes,
|
|
23665
|
+
`${pathPrefix}.retryableErrors.codes`
|
|
23666
|
+
);
|
|
23667
|
+
}
|
|
23668
|
+
out.retryableErrors = retryableErrors;
|
|
23669
|
+
}
|
|
23670
|
+
if (record.requireFailureBeforeRetry !== void 0) {
|
|
23671
|
+
out.requireFailureBeforeRetry = requireBoolean(
|
|
23672
|
+
record.requireFailureBeforeRetry,
|
|
23673
|
+
`${pathPrefix}.requireFailureBeforeRetry`
|
|
23674
|
+
);
|
|
23675
|
+
}
|
|
23676
|
+
if (record.requireSameArguments !== void 0) {
|
|
23677
|
+
if (record.requireSameArguments === true || record.requireSameArguments === false) {
|
|
23678
|
+
out.requireSameArguments = record.requireSameArguments;
|
|
23679
|
+
} else if (record.requireSameArguments === "structured-or-digest") {
|
|
23680
|
+
out.requireSameArguments = "structured-or-digest";
|
|
23681
|
+
} else {
|
|
23682
|
+
throw new CheckConfigError(
|
|
23683
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23684
|
+
`${pathPrefix}.requireSameArguments must be a boolean or "structured-or-digest".`
|
|
23685
|
+
);
|
|
23686
|
+
}
|
|
23687
|
+
}
|
|
23688
|
+
if (record.requireTerminalSuccess !== void 0) {
|
|
23689
|
+
out.requireTerminalSuccess = requireBoolean(
|
|
23690
|
+
record.requireTerminalSuccess,
|
|
23691
|
+
`${pathPrefix}.requireTerminalSuccess`
|
|
23692
|
+
);
|
|
23693
|
+
}
|
|
23694
|
+
if (record.requireRecoveredFailureVisible !== void 0) {
|
|
23695
|
+
out.requireRecoveredFailureVisible = requireBoolean(
|
|
23696
|
+
record.requireRecoveredFailureVisible,
|
|
23697
|
+
`${pathPrefix}.requireRecoveredFailureVisible`
|
|
23698
|
+
);
|
|
23699
|
+
}
|
|
23700
|
+
if (record.successfulResultDependency !== void 0) {
|
|
23701
|
+
const dep = requireObject(
|
|
23702
|
+
record.successfulResultDependency,
|
|
23703
|
+
`${pathPrefix}.successfulResultDependency`
|
|
23704
|
+
);
|
|
23705
|
+
rejectUnknownKeys(dep, CONTRACT_RESULT_DEP_KEYS, `${pathPrefix}.successfulResultDependency`);
|
|
23706
|
+
if (dep.consumerKind !== "LLM") {
|
|
23707
|
+
throw new CheckConfigError(
|
|
23708
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23709
|
+
`${pathPrefix}.successfulResultDependency.consumerKind must be "LLM".`
|
|
23710
|
+
);
|
|
23711
|
+
}
|
|
23712
|
+
const dependency = { consumerKind: "LLM" };
|
|
23713
|
+
if (dep.requireExplicitReference !== void 0) {
|
|
23714
|
+
dependency.requireExplicitReference = requireBoolean(
|
|
23715
|
+
dep.requireExplicitReference,
|
|
23716
|
+
`${pathPrefix}.successfulResultDependency.requireExplicitReference`
|
|
23717
|
+
);
|
|
23718
|
+
}
|
|
23719
|
+
out.successfulResultDependency = dependency;
|
|
23720
|
+
}
|
|
23721
|
+
if (record.sideEffectClass !== void 0) {
|
|
23722
|
+
if (typeof record.sideEffectClass !== "string" || !CONTRACT_SIDE_EFFECT_CLASSES.has(record.sideEffectClass)) {
|
|
23723
|
+
throw new CheckConfigError(
|
|
23724
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23725
|
+
`${pathPrefix}.sideEffectClass must be "read" or "write".`
|
|
23726
|
+
);
|
|
23727
|
+
}
|
|
23728
|
+
out.sideEffectClass = record.sideEffectClass;
|
|
23729
|
+
}
|
|
23730
|
+
return out;
|
|
23731
|
+
}
|
|
23732
|
+
function parseContractRetrySection(value, pathPrefix = "contract.retry") {
|
|
23733
|
+
const record = requireObject(value, pathPrefix);
|
|
23734
|
+
rejectUnknownKeys(record, CONTRACT_RETRY_KEYS, pathPrefix);
|
|
23735
|
+
const out = {};
|
|
23736
|
+
if (record.maxAttempts !== void 0) {
|
|
23737
|
+
out.maxAttempts = requireNonNegativeNumber(record.maxAttempts, `${pathPrefix}.maxAttempts`);
|
|
23738
|
+
}
|
|
23739
|
+
if (record.requireTerminalResult !== void 0) {
|
|
23740
|
+
out.requireTerminalResult = requireBoolean(
|
|
23741
|
+
record.requireTerminalResult,
|
|
23742
|
+
`${pathPrefix}.requireTerminalResult`
|
|
23743
|
+
);
|
|
23744
|
+
}
|
|
23745
|
+
if (record.fallbackOnlyAfterFailure !== void 0) {
|
|
23746
|
+
out.fallbackOnlyAfterFailure = requireBoolean(
|
|
23747
|
+
record.fallbackOnlyAfterFailure,
|
|
23748
|
+
`${pathPrefix}.fallbackOnlyAfterFailure`
|
|
23749
|
+
);
|
|
23750
|
+
}
|
|
23751
|
+
if (record.nonIdempotentTools !== void 0) {
|
|
23752
|
+
out.nonIdempotentTools = requireBoundedStringArray(
|
|
23753
|
+
record.nonIdempotentTools,
|
|
23754
|
+
`${pathPrefix}.nonIdempotentTools`
|
|
23755
|
+
);
|
|
23756
|
+
}
|
|
23757
|
+
if (record.requireIdempotencyEvidenceForRetry !== void 0) {
|
|
23758
|
+
out.requireIdempotencyEvidenceForRetry = requireBoolean(
|
|
23759
|
+
record.requireIdempotencyEvidenceForRetry,
|
|
23760
|
+
`${pathPrefix}.requireIdempotencyEvidenceForRetry`
|
|
23761
|
+
);
|
|
23762
|
+
}
|
|
23763
|
+
if (record.requireRecoveredFailureVisible !== void 0) {
|
|
23764
|
+
out.requireRecoveredFailureVisible = requireBoolean(
|
|
23765
|
+
record.requireRecoveredFailureVisible,
|
|
23766
|
+
`${pathPrefix}.requireRecoveredFailureVisible`
|
|
23767
|
+
);
|
|
23768
|
+
}
|
|
23769
|
+
if (record.operations !== void 0) {
|
|
23770
|
+
const items = requireBoundedArray(
|
|
23771
|
+
record.operations,
|
|
23772
|
+
`${pathPrefix}.operations`,
|
|
23773
|
+
CONTRACT_MAX_OPERATIONS
|
|
23774
|
+
);
|
|
23775
|
+
out.operations = items.map(
|
|
23776
|
+
(item, index) => parseRecoveryOperation(item, `${pathPrefix}.operations[${index}]`)
|
|
23777
|
+
);
|
|
23778
|
+
}
|
|
23779
|
+
return out;
|
|
23780
|
+
}
|
|
23781
|
+
function parseContractScopeSection(value) {
|
|
23782
|
+
const record = requireObject(value, "contract.scope");
|
|
23783
|
+
rejectUnknownKeys(record, CONTRACT_SCOPE_KEYS, "contract.scope");
|
|
23784
|
+
const out = {};
|
|
23785
|
+
if (record.runId !== void 0) {
|
|
23786
|
+
out.runId = requireNonEmptyString(record.runId, "contract.scope.runId");
|
|
23787
|
+
}
|
|
23788
|
+
if (record.subAgentId !== void 0) {
|
|
23789
|
+
out.subAgentId = requireNonEmptyString(record.subAgentId, "contract.scope.subAgentId");
|
|
23790
|
+
}
|
|
23791
|
+
if (record.groupId !== void 0) {
|
|
23792
|
+
out.groupId = requireNonEmptyString(record.groupId, "contract.scope.groupId");
|
|
23793
|
+
}
|
|
23794
|
+
if (record.workflowStep !== void 0) {
|
|
23795
|
+
out.workflowStep = requireNonEmptyString(
|
|
23796
|
+
record.workflowStep,
|
|
23797
|
+
"contract.scope.workflowStep"
|
|
23798
|
+
);
|
|
23799
|
+
}
|
|
23800
|
+
if (record.rootEventId !== void 0) {
|
|
23801
|
+
out.rootEventId = requireNonEmptyString(record.rootEventId, "contract.scope.rootEventId");
|
|
23802
|
+
}
|
|
23803
|
+
return out;
|
|
23804
|
+
}
|
|
23805
|
+
function parseContractBodySection(value, pathPrefix) {
|
|
23806
|
+
const record = requireObject(value, pathPrefix);
|
|
23807
|
+
rejectUnknownKeys(record, CONTRACT_BODY_KEYS, pathPrefix);
|
|
23808
|
+
const out = {};
|
|
23809
|
+
if (record.run !== void 0) out.run = parseContractRunSection(record.run, `${pathPrefix}.run`);
|
|
23810
|
+
if (record.tools !== void 0) {
|
|
23811
|
+
out.tools = parseContractToolsSection(record.tools, `${pathPrefix}.tools`);
|
|
23812
|
+
}
|
|
23813
|
+
if (record.llm !== void 0) out.llm = parseContractLlmSection(record.llm, `${pathPrefix}.llm`);
|
|
23814
|
+
if (record.observations !== void 0) {
|
|
23815
|
+
out.observations = parseContractObservationsSection(
|
|
23816
|
+
record.observations,
|
|
23817
|
+
`${pathPrefix}.observations`
|
|
23818
|
+
);
|
|
23819
|
+
}
|
|
23820
|
+
if (record.controls !== void 0) {
|
|
23821
|
+
out.controls = parseContractControlsSection(record.controls, `${pathPrefix}.controls`);
|
|
23822
|
+
}
|
|
23823
|
+
if (record.retry !== void 0) {
|
|
23824
|
+
out.retry = parseContractRetrySection(record.retry, `${pathPrefix}.retry`);
|
|
23825
|
+
}
|
|
23826
|
+
return out;
|
|
23827
|
+
}
|
|
23828
|
+
function parseContractAlternativesSection(value) {
|
|
23829
|
+
const record = requireObject(value, "contract.alternatives");
|
|
23830
|
+
rejectUnknownKeys(record, CONTRACT_ALT_KEYS, "contract.alternatives");
|
|
23831
|
+
if (record.anyOf === void 0) {
|
|
23832
|
+
throw new CheckConfigError(
|
|
23833
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23834
|
+
"contract.alternatives.anyOf is required."
|
|
23835
|
+
);
|
|
23836
|
+
}
|
|
23837
|
+
const items = requireBoundedArray(
|
|
23838
|
+
record.anyOf,
|
|
23839
|
+
"contract.alternatives.anyOf",
|
|
23840
|
+
CONTRACT_MAX_ANY_OF
|
|
23841
|
+
);
|
|
23842
|
+
const anyOf = items.map((item, index) => {
|
|
23843
|
+
const pathPrefix = `contract.alternatives.anyOf[${index}]`;
|
|
23844
|
+
const branch = requireObject(item, pathPrefix);
|
|
23845
|
+
rejectUnknownKeys(branch, CONTRACT_ALT_BRANCH_KEYS, pathPrefix);
|
|
23846
|
+
const id = requireNonEmptyString(branch.id, `${pathPrefix}.id`);
|
|
23847
|
+
if (branch.contract === void 0) {
|
|
23848
|
+
throw new CheckConfigError(
|
|
23849
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23850
|
+
`${pathPrefix}.contract is required.`
|
|
23851
|
+
);
|
|
23852
|
+
}
|
|
23853
|
+
const out = {
|
|
23854
|
+
id,
|
|
23855
|
+
contract: parseContractBodySection(branch.contract, `${pathPrefix}.contract`)
|
|
23856
|
+
};
|
|
23857
|
+
if (branch.description !== void 0) {
|
|
23858
|
+
out.description = requireNonEmptyString(branch.description, `${pathPrefix}.description`);
|
|
23859
|
+
}
|
|
23860
|
+
return out;
|
|
23861
|
+
});
|
|
23862
|
+
return { anyOf };
|
|
23863
|
+
}
|
|
23864
|
+
function parseContractSection(value) {
|
|
23865
|
+
const record = requireObject(value, "contract");
|
|
23866
|
+
rejectUnknownKeys(record, CONTRACT_KEYS, "contract");
|
|
23867
|
+
const bodyOnly = {};
|
|
23868
|
+
for (const key of Object.keys(record)) {
|
|
23869
|
+
if (CONTRACT_BODY_KEYS.has(key)) bodyOnly[key] = record[key];
|
|
23870
|
+
}
|
|
23871
|
+
const out = { ...parseContractBodySection(bodyOnly, "contract") };
|
|
23872
|
+
if (record.scope !== void 0) out.scope = parseContractScopeSection(record.scope);
|
|
23873
|
+
if (record.alternatives !== void 0) {
|
|
23874
|
+
out.alternatives = parseContractAlternativesSection(record.alternatives);
|
|
23875
|
+
}
|
|
23876
|
+
return out;
|
|
23877
|
+
}
|
|
20976
23878
|
function parseCheckConfig(value) {
|
|
20977
23879
|
if (value === void 0 || value === null) return {};
|
|
20978
23880
|
if (typeof value !== "object" || Array.isArray(value)) {
|
|
20979
23881
|
throw new CheckConfigError("AI_CHECK_CONFIG_INVALID_VALUE", "Config must export an object.");
|
|
20980
23882
|
}
|
|
20981
23883
|
const root = value;
|
|
20982
|
-
if ("contract" in root) {
|
|
20983
|
-
throw new CheckConfigError(
|
|
20984
|
-
"AI_CHECK_CONFIG_UNKNOWN_KEY",
|
|
20985
|
-
'Top-level "contract" is not supported by `agent-inspect check --config`. Use the TraceContract TypeScript API (`defineTraceContract` / `evaluateTraceContract` from `agent-inspect/checks`), or express tool/run/llm rules under `checks`.'
|
|
20986
|
-
);
|
|
20987
|
-
}
|
|
20988
23884
|
const rootKeys = Object.keys(root);
|
|
20989
23885
|
for (const key of rootKeys) {
|
|
20990
|
-
if (key
|
|
20991
|
-
const suggestion = closestKey(key, [
|
|
23886
|
+
if (ROOT_CONFIG_KEYS.has(key)) continue;
|
|
23887
|
+
const suggestion = closestKey(key, [...ROOT_CONFIG_KEYS]);
|
|
20992
23888
|
const hint = suggestion ? ` Did you mean "${suggestion}"?` : "";
|
|
20993
23889
|
throw new CheckConfigError(
|
|
20994
23890
|
"AI_CHECK_CONFIG_UNKNOWN_KEY",
|
|
20995
|
-
`Unknown top-level check config key "${key}". Allowed: checks.${hint}`
|
|
23891
|
+
`Unknown top-level check config key "${key}". Allowed: checks, contract.${hint}`
|
|
20996
23892
|
);
|
|
20997
23893
|
}
|
|
20998
|
-
|
|
20999
|
-
|
|
21000
|
-
|
|
21001
|
-
|
|
21002
|
-
|
|
21003
|
-
|
|
23894
|
+
const out = {};
|
|
23895
|
+
if ("checks" in root) {
|
|
23896
|
+
const checksRecord = requireObject(root.checks, "checks");
|
|
23897
|
+
rejectUnknownKeys(checksRecord, CHECKS_KEYS, "checks");
|
|
23898
|
+
const checks2 = {};
|
|
23899
|
+
if (checksRecord.select !== void 0) {
|
|
23900
|
+
checks2.select = requireStringArray(checksRecord.select, "checks.select");
|
|
23901
|
+
}
|
|
23902
|
+
if (checksRecord.run !== void 0) checks2.run = parseRunSection(checksRecord.run);
|
|
23903
|
+
if (checksRecord.tool !== void 0) checks2.tool = parseToolSection(checksRecord.tool);
|
|
23904
|
+
if (checksRecord.llm !== void 0) checks2.llm = parseLlmSection(checksRecord.llm);
|
|
23905
|
+
if (checksRecord.structure !== void 0) {
|
|
23906
|
+
checks2.structure = parseStructureSection(checksRecord.structure);
|
|
23907
|
+
}
|
|
23908
|
+
if (checksRecord.safety !== void 0) checks2.safety = parseSafetySection(checksRecord.safety);
|
|
23909
|
+
out.checks = checks2;
|
|
23910
|
+
}
|
|
23911
|
+
if ("contract" in root) {
|
|
23912
|
+
out.contract = parseContractSection(root.contract);
|
|
21004
23913
|
}
|
|
21005
|
-
if (
|
|
21006
|
-
|
|
21007
|
-
|
|
21008
|
-
|
|
21009
|
-
|
|
23914
|
+
if (out.contract !== void 0 && contractConfigHasEffect(out.contract) && checkConfigHasEffect({ checks: out.checks })) {
|
|
23915
|
+
throw new CheckConfigError(
|
|
23916
|
+
"AI_CHECK_CONFIG_INVALID_VALUE",
|
|
23917
|
+
'Config cannot combine top-level "checks" and "contract". Use one vocabulary per config file.'
|
|
23918
|
+
);
|
|
21010
23919
|
}
|
|
21011
|
-
|
|
21012
|
-
return { checks: checks2 };
|
|
23920
|
+
return out;
|
|
21013
23921
|
}
|
|
21014
23922
|
async function loadConfig(configPath) {
|
|
21015
23923
|
if (configPath === void 0) return {};
|
|
@@ -21133,7 +24041,7 @@ function buildRules(config, options, presetSelect = []) {
|
|
|
21133
24041
|
} catch (error) {
|
|
21134
24042
|
const message = error instanceof Error ? error.message : String(error);
|
|
21135
24043
|
diagnostics.push(
|
|
21136
|
-
|
|
24044
|
+
diagnostic6("AI_CHECK_INVALID_ARGUMENTS", `--max-step-duration: ${message}`)
|
|
21137
24045
|
);
|
|
21138
24046
|
}
|
|
21139
24047
|
}
|
|
@@ -21168,7 +24076,7 @@ function buildRules(config, options, presetSelect = []) {
|
|
|
21168
24076
|
} catch (error) {
|
|
21169
24077
|
const message = error instanceof Error ? error.message : String(error);
|
|
21170
24078
|
diagnostics.push(
|
|
21171
|
-
|
|
24079
|
+
diagnostic6("AI_CHECK_INVALID_ARGUMENTS", `--fail-on-observation: ${message}`)
|
|
21172
24080
|
);
|
|
21173
24081
|
}
|
|
21174
24082
|
}
|
|
@@ -21310,8 +24218,8 @@ function printHuman2(result, options = {}) {
|
|
|
21310
24218
|
`Summary: ${result.summary.failed} failed, ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
21311
24219
|
);
|
|
21312
24220
|
console.log(`Rules evaluated: ${result.summary.rulesEvaluated ?? result.ruleExecutions?.length ?? 0}`);
|
|
21313
|
-
for (const
|
|
21314
|
-
console.log(`- ${
|
|
24221
|
+
for (const diagnostic8 of result.diagnostics) {
|
|
24222
|
+
console.log(`- ${diagnostic8.code}: ${diagnostic8.message}`);
|
|
21315
24223
|
}
|
|
21316
24224
|
for (const finding of result.findings) {
|
|
21317
24225
|
const path44 = finding.evidence[0]?.path;
|
|
@@ -21337,6 +24245,7 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21337
24245
|
let evidenceRead;
|
|
21338
24246
|
let evidenceRunIds = [];
|
|
21339
24247
|
let evidenceSourceContents;
|
|
24248
|
+
let evidenceContract;
|
|
21340
24249
|
const sessionId = options.session?.trim();
|
|
21341
24250
|
const groupId = options.group?.trim();
|
|
21342
24251
|
const useSessionScope = Boolean(sessionId || groupId);
|
|
@@ -21345,7 +24254,7 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21345
24254
|
if (options.config !== void 0 && !checkConfigHasEffect(config)) {
|
|
21346
24255
|
result = errorResult2(
|
|
21347
24256
|
"AI_CHECK_CONFIG_NO_EFFECTIVE_RULES",
|
|
21348
|
-
"Explicit --config has no effective check rules. Configure checks.select, checks.run, checks.tool, checks.llm, checks.structure,
|
|
24257
|
+
"Explicit --config has no effective check rules. Configure checks.select, checks.run, checks.tool, checks.llm, checks.structure, checks.safety, or contract."
|
|
21349
24258
|
);
|
|
21350
24259
|
if (options.json) printJson2(result);
|
|
21351
24260
|
else printHuman2(result, options);
|
|
@@ -21353,7 +24262,11 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21353
24262
|
return;
|
|
21354
24263
|
}
|
|
21355
24264
|
let effectiveOptions = options;
|
|
21356
|
-
const
|
|
24265
|
+
const useContract = config.contract !== void 0 && contractConfigHasEffect(config.contract);
|
|
24266
|
+
if (useContract) {
|
|
24267
|
+
evidenceContract = defineTraceContract(config.contract);
|
|
24268
|
+
}
|
|
24269
|
+
const resolved = useContract ? void 0 : resolvePreset(options.preset, {
|
|
21357
24270
|
hasToolRules: hasToolRulesConfigured(config, options)
|
|
21358
24271
|
});
|
|
21359
24272
|
if (resolved !== void 0) {
|
|
@@ -21361,7 +24274,7 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21361
24274
|
config = applied.config;
|
|
21362
24275
|
effectiveOptions = applied.options;
|
|
21363
24276
|
}
|
|
21364
|
-
const built = buildRules(config, effectiveOptions, resolved?.select ?? []);
|
|
24277
|
+
const built = useContract ? { rules: [], select: [], diagnostics: [] } : buildRules(config, effectiveOptions, resolved?.select ?? []);
|
|
21365
24278
|
if (built.diagnostics.some((item) => item.severity === "error")) {
|
|
21366
24279
|
result = errorResult2("AI_CHECK_INVALID_CONFIG", "Invalid check configuration.");
|
|
21367
24280
|
result.diagnostics = [...built.diagnostics];
|
|
@@ -21383,6 +24296,7 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21383
24296
|
});
|
|
21384
24297
|
const perRun = [];
|
|
21385
24298
|
const sourceContents = /* @__PURE__ */ new Map();
|
|
24299
|
+
const definedContract = evidenceContract;
|
|
21386
24300
|
for (const meta2 of scoped.metas) {
|
|
21387
24301
|
const read = await openTrace(
|
|
21388
24302
|
{ type: "file", path: meta2.filePath },
|
|
@@ -21399,23 +24313,20 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21399
24313
|
`
|
|
21400
24314
|
);
|
|
21401
24315
|
}
|
|
24316
|
+
const base = definedContract ? evaluateTraceContract({ read }, definedContract, { runId: meta2.runId }) : runTraceChecks(
|
|
24317
|
+
{ read },
|
|
24318
|
+
{
|
|
24319
|
+
rules: built.rules,
|
|
24320
|
+
select: built.select,
|
|
24321
|
+
runId: meta2.runId
|
|
24322
|
+
}
|
|
24323
|
+
);
|
|
21402
24324
|
perRun.push(
|
|
21403
|
-
mergeSafetyExtensions(
|
|
21404
|
-
|
|
21405
|
-
|
|
21406
|
-
|
|
21407
|
-
|
|
21408
|
-
select: built.select,
|
|
21409
|
-
runId: meta2.runId
|
|
21410
|
-
}
|
|
21411
|
-
),
|
|
21412
|
-
read,
|
|
21413
|
-
{
|
|
21414
|
-
...options.guardrails ? { guardrails: options.guardrails } : {},
|
|
21415
|
-
...options.circuit ? { circuits: options.circuit } : {},
|
|
21416
|
-
runId: meta2.runId
|
|
21417
|
-
}
|
|
21418
|
-
)
|
|
24325
|
+
mergeSafetyExtensions(base, read, {
|
|
24326
|
+
...options.guardrails ? { guardrails: options.guardrails } : {},
|
|
24327
|
+
...options.circuit ? { circuits: options.circuit } : {},
|
|
24328
|
+
runId: meta2.runId
|
|
24329
|
+
})
|
|
21419
24330
|
);
|
|
21420
24331
|
}
|
|
21421
24332
|
evidenceRunIds = scoped.runIds;
|
|
@@ -21454,22 +24365,25 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21454
24365
|
])
|
|
21455
24366
|
);
|
|
21456
24367
|
}
|
|
21457
|
-
|
|
21458
|
-
|
|
21459
|
-
|
|
21460
|
-
|
|
21461
|
-
|
|
21462
|
-
|
|
21463
|
-
|
|
21464
|
-
|
|
21465
|
-
),
|
|
21466
|
-
read,
|
|
24368
|
+
const base = useContract && evidenceContract !== void 0 ? evaluateTraceContract(
|
|
24369
|
+
{ read },
|
|
24370
|
+
evidenceContract,
|
|
24371
|
+
{
|
|
24372
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
24373
|
+
}
|
|
24374
|
+
) : runTraceChecks(
|
|
24375
|
+
{ read },
|
|
21467
24376
|
{
|
|
21468
|
-
|
|
21469
|
-
|
|
24377
|
+
rules: built.rules,
|
|
24378
|
+
select: built.select,
|
|
21470
24379
|
...options.run !== void 0 ? { runId: options.run } : {}
|
|
21471
24380
|
}
|
|
21472
24381
|
);
|
|
24382
|
+
result = mergeSafetyExtensions(base, read, {
|
|
24383
|
+
...options.guardrails ? { guardrails: options.guardrails } : {},
|
|
24384
|
+
...options.circuit ? { circuits: options.circuit } : {},
|
|
24385
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
24386
|
+
});
|
|
21473
24387
|
}
|
|
21474
24388
|
} catch (error) {
|
|
21475
24389
|
if (phase === "config") {
|
|
@@ -21513,7 +24427,15 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
21513
24427
|
checkResultsJson: checkResultToEvidenceJson(result, runIds),
|
|
21514
24428
|
summaryText: `Check status: ${result.status}`,
|
|
21515
24429
|
redactionProfile,
|
|
21516
|
-
format: evidenceFormat
|
|
24430
|
+
format: evidenceFormat,
|
|
24431
|
+
...evidenceContract !== void 0 ? {
|
|
24432
|
+
contractPackage: {
|
|
24433
|
+
engineVersion: "",
|
|
24434
|
+
source: "file",
|
|
24435
|
+
contract: evidenceContract,
|
|
24436
|
+
...options.config !== void 0 ? { path: options.config } : {}
|
|
24437
|
+
}
|
|
24438
|
+
} : {}
|
|
21517
24439
|
});
|
|
21518
24440
|
if (!options.json) {
|
|
21519
24441
|
console.log(`Evidence: ${written}`);
|
|
@@ -22829,7 +25751,7 @@ function evidenceForEvent(event, path44) {
|
|
|
22829
25751
|
}
|
|
22830
25752
|
];
|
|
22831
25753
|
}
|
|
22832
|
-
function
|
|
25754
|
+
function fail5(ruleId, message, evidence, expected, actual) {
|
|
22833
25755
|
return {
|
|
22834
25756
|
ruleId,
|
|
22835
25757
|
status: "fail",
|
|
@@ -23025,7 +25947,7 @@ var checks = {
|
|
|
23025
25947
|
"eval.requireSuccess",
|
|
23026
25948
|
"run",
|
|
23027
25949
|
(context) => context.run.status === "ok" ? [] : [
|
|
23028
|
-
|
|
25950
|
+
fail5(
|
|
23029
25951
|
"eval.requireSuccess",
|
|
23030
25952
|
"Run did not complete successfully.",
|
|
23031
25953
|
evidenceForRun(context.run, "status"),
|
|
@@ -23040,7 +25962,7 @@ var checks = {
|
|
|
23040
25962
|
return createRule("eval.requiredTools", "tool", (context) => {
|
|
23041
25963
|
const tools = new Set(nodeNames(context.nodes, "TOOL"));
|
|
23042
25964
|
return expected.filter((name) => !tools.has(name)).map(
|
|
23043
|
-
(name) =>
|
|
25965
|
+
(name) => fail5(
|
|
23044
25966
|
"eval.requiredTools",
|
|
23045
25967
|
`Required tool ${name} did not appear.`,
|
|
23046
25968
|
evidenceForRun(context.run, "children"),
|
|
@@ -23056,7 +25978,7 @@ var checks = {
|
|
|
23056
25978
|
"eval.forbiddenTools",
|
|
23057
25979
|
"tool",
|
|
23058
25980
|
(context) => context.nodes.filter((node) => node.event.kind === "TOOL" && blocked.includes(node.event.name)).map(
|
|
23059
|
-
(node) =>
|
|
25981
|
+
(node) => fail5(
|
|
23060
25982
|
"eval.forbiddenTools",
|
|
23061
25983
|
`Forbidden tool ${node.event.name} appeared.`,
|
|
23062
25984
|
evidenceForEvent(node.event, "name"),
|
|
@@ -23071,7 +25993,7 @@ var checks = {
|
|
|
23071
25993
|
"eval.maxDurationMs",
|
|
23072
25994
|
"run",
|
|
23073
25995
|
(context) => context.run.durationMs !== void 0 && context.run.durationMs > maxDurationMs ? [
|
|
23074
|
-
|
|
25996
|
+
fail5(
|
|
23075
25997
|
"eval.maxDurationMs",
|
|
23076
25998
|
`Run duration exceeded ${maxDurationMs}ms.`,
|
|
23077
25999
|
evidenceForRun(context.run, "durationMs"),
|
|
@@ -23088,7 +26010,7 @@ var checks = {
|
|
|
23088
26010
|
void 0
|
|
23089
26011
|
);
|
|
23090
26012
|
return deepest !== void 0 && deepest.depth > maxDepth ? [
|
|
23091
|
-
|
|
26013
|
+
fail5(
|
|
23092
26014
|
"eval.maxDepth",
|
|
23093
26015
|
`Run tree depth exceeded ${maxDepth}.`,
|
|
23094
26016
|
evidenceForEvent(deepest.event, "depth"),
|
|
@@ -23105,7 +26027,7 @@ var checks = {
|
|
|
23105
26027
|
(context) => context.nodes.flatMap((node) => {
|
|
23106
26028
|
const retries = numericAttribute(node, ["retryCount", "retries", "attempt"]);
|
|
23107
26029
|
return retries !== void 0 && retries > maxRetries ? [
|
|
23108
|
-
|
|
26030
|
+
fail5(
|
|
23109
26031
|
"eval.maxRetries",
|
|
23110
26032
|
`Retry count exceeded ${maxRetries}.`,
|
|
23111
26033
|
evidenceForEvent(node.event, "attributes.retryCount"),
|
|
@@ -23120,7 +26042,7 @@ var checks = {
|
|
|
23120
26042
|
return createRule("eval.maxTotalTokens", "llm", (context) => {
|
|
23121
26043
|
const total = totalTokenCount(context.events);
|
|
23122
26044
|
return total > maxTotalTokens ? [
|
|
23123
|
-
|
|
26045
|
+
fail5(
|
|
23124
26046
|
"eval.maxTotalTokens",
|
|
23125
26047
|
`Total token usage exceeded ${maxTotalTokens}.`,
|
|
23126
26048
|
evidenceForRun(context.run, "tokenUsage.total"),
|
|
@@ -23135,7 +26057,7 @@ var checks = {
|
|
|
23135
26057
|
"eval.noFailedSteps",
|
|
23136
26058
|
"run",
|
|
23137
26059
|
(context) => context.nodes.filter((node) => node.event.status === "error" || node.event.kind === "ERROR").map(
|
|
23138
|
-
(node) =>
|
|
26060
|
+
(node) => fail5(
|
|
23139
26061
|
"eval.noFailedSteps",
|
|
23140
26062
|
"Run contains a failed step or error node.",
|
|
23141
26063
|
evidenceForEvent(node.event, "status"),
|
|
@@ -23153,7 +26075,7 @@ var checks = {
|
|
|
23153
26075
|
(node, index) => index < firstLlmIndex && node.event.kind === "RETRIEVER"
|
|
23154
26076
|
);
|
|
23155
26077
|
return retrievalIndex === -1 ? [
|
|
23156
|
-
|
|
26078
|
+
fail5(
|
|
23157
26079
|
"eval.requiredRetrievalBeforeGeneration",
|
|
23158
26080
|
"No retrieval step appeared before the first LLM generation.",
|
|
23159
26081
|
evidenceForEvent(context.nodes[firstLlmIndex].event, "kind"),
|
|
@@ -23169,7 +26091,7 @@ var checks = {
|
|
|
23169
26091
|
const decisions = context.nodes.filter((node) => node.event.kind === "DECISION");
|
|
23170
26092
|
if (decisions.length === 0) {
|
|
23171
26093
|
return [
|
|
23172
|
-
|
|
26094
|
+
fail5(
|
|
23173
26095
|
"eval.requiredDecisionMetadata",
|
|
23174
26096
|
"No decision node is available for required metadata.",
|
|
23175
26097
|
evidenceForRun(context.run, "children"),
|
|
@@ -23180,7 +26102,7 @@ var checks = {
|
|
|
23180
26102
|
}
|
|
23181
26103
|
return decisions.flatMap(
|
|
23182
26104
|
(node) => required.filter((key) => !hasAttribute(node, key)).map(
|
|
23183
|
-
(key) =>
|
|
26105
|
+
(key) => fail5(
|
|
23184
26106
|
"eval.requiredDecisionMetadata",
|
|
23185
26107
|
`Decision metadata ${key} is missing.`,
|
|
23186
26108
|
evidenceForEvent(node.event, `attributes.${key}`),
|
|
@@ -23201,7 +26123,7 @@ var checks = {
|
|
|
23201
26123
|
const contexts = collectTextFields(context.nodes, contextKeys, ["RETRIEVER", "TOOL"]);
|
|
23202
26124
|
if (answers.length === 0 || contexts.length === 0) {
|
|
23203
26125
|
return [
|
|
23204
|
-
|
|
26126
|
+
fail5(
|
|
23205
26127
|
"eval.contextOverlap",
|
|
23206
26128
|
"Answer and context text are required for overlap evaluation.",
|
|
23207
26129
|
firstEvidence(answers.length > 0 ? answers : contexts, context.run, "children"),
|
|
@@ -23215,7 +26137,7 @@ var checks = {
|
|
|
23215
26137
|
const sharedTerms = [...answerTerms].filter((term) => contextTerms.has(term)).length;
|
|
23216
26138
|
const overlap = answerTerms.size === 0 ? 0 : sharedTerms / answerTerms.size;
|
|
23217
26139
|
return sharedTerms < minSharedTerms || overlap < minOverlap ? [
|
|
23218
|
-
|
|
26140
|
+
fail5(
|
|
23219
26141
|
"eval.contextOverlap",
|
|
23220
26142
|
"Answer text did not sufficiently overlap retrieved context.",
|
|
23221
26143
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -23243,7 +26165,7 @@ var checks = {
|
|
|
23243
26165
|
const quotes = quotedSnippets(answerText, minQuoteLength);
|
|
23244
26166
|
if (quotes.length === 0) {
|
|
23245
26167
|
return requireQuote ? [
|
|
23246
|
-
|
|
26168
|
+
fail5(
|
|
23247
26169
|
"eval.quoteOverlap",
|
|
23248
26170
|
"Answer did not contain a quote for overlap evaluation.",
|
|
23249
26171
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -23254,7 +26176,7 @@ var checks = {
|
|
|
23254
26176
|
}
|
|
23255
26177
|
const missing = quotes.filter((quote) => !contextText.includes(quote.toLowerCase()));
|
|
23256
26178
|
return missing.length > 0 ? [
|
|
23257
|
-
|
|
26179
|
+
fail5(
|
|
23258
26180
|
"eval.quoteOverlap",
|
|
23259
26181
|
"Quoted answer text did not appear in retrieved context.",
|
|
23260
26182
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -23272,7 +26194,7 @@ var checks = {
|
|
|
23272
26194
|
const citations = collectTextFields(context.nodes, citationKeys);
|
|
23273
26195
|
const count = citationCount(answers.map((field) => field.text).join(" "), citations);
|
|
23274
26196
|
return count === 0 ? [
|
|
23275
|
-
|
|
26197
|
+
fail5(
|
|
23276
26198
|
"eval.citationPresence",
|
|
23277
26199
|
"Answer did not include citations or source references.",
|
|
23278
26200
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -23290,7 +26212,7 @@ var checks = {
|
|
|
23290
26212
|
const availableSet = new Set(available);
|
|
23291
26213
|
const missing = expected.filter((id) => !availableSet.has(id));
|
|
23292
26214
|
return missing.length > 0 ? [
|
|
23293
|
-
|
|
26215
|
+
fail5(
|
|
23294
26216
|
"eval.requiredSourceIds",
|
|
23295
26217
|
"Required source IDs were not present in trace context or citations.",
|
|
23296
26218
|
evidenceForRun(context.run, "children"),
|
|
@@ -23310,7 +26232,7 @@ var checks = {
|
|
|
23310
26232
|
const tooShort = options.minCharacters !== void 0 && characters < options.minCharacters || options.minWords !== void 0 && words < options.minWords;
|
|
23311
26233
|
const tooLong = options.maxCharacters !== void 0 && characters > options.maxCharacters || options.maxWords !== void 0 && words > options.maxWords;
|
|
23312
26234
|
return answer.length === 0 || tooShort || tooLong ? [
|
|
23313
|
-
|
|
26235
|
+
fail5(
|
|
23314
26236
|
"eval.answerLengthBounds",
|
|
23315
26237
|
"Answer length fell outside required bounds.",
|
|
23316
26238
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -23333,7 +26255,7 @@ var checks = {
|
|
|
23333
26255
|
const answer = answers.map((field) => field.text).join(" ").toLowerCase();
|
|
23334
26256
|
const matches = banned.filter((phrase) => answer.includes(phrase));
|
|
23335
26257
|
return matches.length > 0 ? [
|
|
23336
|
-
|
|
26258
|
+
fail5(
|
|
23337
26259
|
"eval.bannedUnsupportedPhrases",
|
|
23338
26260
|
"Answer contained banned unsupported-answer phrasing.",
|
|
23339
26261
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -23355,8 +26277,8 @@ function renderEvalMarkdown(result) {
|
|
|
23355
26277
|
];
|
|
23356
26278
|
if (result.diagnostics.length > 0) {
|
|
23357
26279
|
lines.push("", "## Diagnostics");
|
|
23358
|
-
for (const
|
|
23359
|
-
lines.push(`- ${
|
|
26280
|
+
for (const diagnostic8 of result.diagnostics) {
|
|
26281
|
+
lines.push(`- ${diagnostic8.code}: ${diagnostic8.message}`);
|
|
23360
26282
|
}
|
|
23361
26283
|
}
|
|
23362
26284
|
if (result.findings.length > 0) {
|
|
@@ -23373,7 +26295,7 @@ function renderEvalMarkdown(result) {
|
|
|
23373
26295
|
// packages/cli/src/eval.ts
|
|
23374
26296
|
var CONFIG_EXTENSIONS3 = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
23375
26297
|
var TS_CONFIG_EXTENSIONS3 = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
23376
|
-
function
|
|
26298
|
+
function diagnostic7(code, message, severity = "error") {
|
|
23377
26299
|
return { code, message, severity };
|
|
23378
26300
|
}
|
|
23379
26301
|
function errorResult4(code, message, format = "unknown") {
|
|
@@ -23383,7 +26305,7 @@ function errorResult4(code, message, format = "unknown") {
|
|
|
23383
26305
|
format,
|
|
23384
26306
|
summary: { passed: 0, failed: 0, warnings: 0, errors: 1 },
|
|
23385
26307
|
findings: [],
|
|
23386
|
-
diagnostics: [
|
|
26308
|
+
diagnostics: [diagnostic7(code, message)]
|
|
23387
26309
|
};
|
|
23388
26310
|
}
|
|
23389
26311
|
function parseNumber2(value, label) {
|
|
@@ -23552,8 +26474,8 @@ function printHuman3(result) {
|
|
|
23552
26474
|
console.log(
|
|
23553
26475
|
`Summary: ${result.summary.failed} failed, ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
23554
26476
|
);
|
|
23555
|
-
for (const
|
|
23556
|
-
console.log(`- ${
|
|
26477
|
+
for (const diagnostic8 of result.diagnostics) {
|
|
26478
|
+
console.log(`- ${diagnostic8.code}: ${diagnostic8.message}`);
|
|
23557
26479
|
}
|
|
23558
26480
|
for (const finding of result.findings) {
|
|
23559
26481
|
const path44 = finding.evidence[0]?.path;
|
|
@@ -23676,8 +26598,8 @@ function renderCheckSection(result) {
|
|
|
23676
26598
|
const path44 = finding.evidence[0]?.path ?? "(run)";
|
|
23677
26599
|
lines.push(`- ${finding.ruleId}: ${finding.message} (${path44})`);
|
|
23678
26600
|
}
|
|
23679
|
-
for (const
|
|
23680
|
-
lines.push(`- ${
|
|
26601
|
+
for (const diagnostic8 of result.diagnostics.slice(0, 10)) {
|
|
26602
|
+
lines.push(`- ${diagnostic8.code}: ${diagnostic8.message}`);
|
|
23681
26603
|
}
|
|
23682
26604
|
return lines.join("\n");
|
|
23683
26605
|
}
|
|
@@ -26232,7 +29154,7 @@ async function indexSqliteCleanCommand(options = {}) {
|
|
|
26232
29154
|
function printJson4(payload) {
|
|
26233
29155
|
console.log(JSON.stringify(payload, null, 2));
|
|
26234
29156
|
}
|
|
26235
|
-
function
|
|
29157
|
+
function fail6(json, message) {
|
|
26236
29158
|
if (json) {
|
|
26237
29159
|
printJson4({ ok: false, error: message });
|
|
26238
29160
|
} else {
|
|
@@ -26283,14 +29205,14 @@ async function workspaceInitCommand(options = {}) {
|
|
|
26283
29205
|
for (const dir of result.createdDirs) console.log(`Created ${dir}/`);
|
|
26284
29206
|
console.log("\nNext: `agent-inspect workspace status`");
|
|
26285
29207
|
} catch (error) {
|
|
26286
|
-
|
|
29208
|
+
fail6(options.json, error instanceof Error ? error.message : String(error));
|
|
26287
29209
|
}
|
|
26288
29210
|
}
|
|
26289
29211
|
async function workspaceStatusCommand(options = {}) {
|
|
26290
29212
|
const location = resolveWorkspaceLocation(options.cwd);
|
|
26291
29213
|
const manifestResult = await readWorkspaceManifestFile(location);
|
|
26292
29214
|
if (!manifestResult.ok || !manifestResult.manifest) {
|
|
26293
|
-
|
|
29215
|
+
fail6(
|
|
26294
29216
|
options.json,
|
|
26295
29217
|
manifestResult.exists ? `invalid workspace.json: ${manifestResult.errors.join("; ")}` : "no workspace found (run `agent-inspect workspace init`)"
|
|
26296
29218
|
);
|
|
@@ -26312,7 +29234,7 @@ async function workspaceStatusCommand(options = {}) {
|
|
|
26312
29234
|
`Index: ${status.index.enabled ? status.index.type : "disabled"}${status.index.exists ? " (present)" : ""}`
|
|
26313
29235
|
);
|
|
26314
29236
|
} catch (error) {
|
|
26315
|
-
|
|
29237
|
+
fail6(options.json, error instanceof Error ? error.message : String(error));
|
|
26316
29238
|
}
|
|
26317
29239
|
}
|
|
26318
29240
|
async function workspaceDoctorCommand(options = {}) {
|
|
@@ -26334,7 +29256,7 @@ async function workspaceCleanCommand(options = {}) {
|
|
|
26334
29256
|
const location = resolveWorkspaceLocation(options.cwd);
|
|
26335
29257
|
const manifestResult = await readWorkspaceManifestFile(location);
|
|
26336
29258
|
if (!manifestResult.ok || !manifestResult.manifest) {
|
|
26337
|
-
|
|
29259
|
+
fail6(
|
|
26338
29260
|
options.json,
|
|
26339
29261
|
manifestResult.exists ? `invalid workspace.json: ${manifestResult.errors.join("; ")}` : "no workspace found (run `agent-inspect workspace init`)"
|
|
26340
29262
|
);
|
|
@@ -26359,7 +29281,7 @@ async function workspaceCleanCommand(options = {}) {
|
|
|
26359
29281
|
if (result.removed.length === 0) console.log("- (nothing)");
|
|
26360
29282
|
for (const item of result.removed) console.log(`- ${item}`);
|
|
26361
29283
|
} catch (error) {
|
|
26362
|
-
|
|
29284
|
+
fail6(options.json, error instanceof Error ? error.message : String(error));
|
|
26363
29285
|
}
|
|
26364
29286
|
}
|
|
26365
29287
|
async function workspacePathCommand(options = {}) {
|