agent-inspect 6.29.6 → 6.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +1 -1
- package/docs/CLI.md +26 -4
- package/docs/TRACE-CONTRACTS.md +25 -0
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-DBRMI7TC.mjs → chunk-Y5Z4BXUO.mjs} +2422 -101
- package/packages/cli/dist/chunk-Y5Z4BXUO.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +10313 -7195
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +866 -59
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-6CET2GW2.mjs → src-WTTH5G33.mjs} +3 -3
- package/packages/cli/dist/{src-6CET2GW2.mjs.map → src-WTTH5G33.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +12 -1
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +2 -2
- package/packages/core/dist/advanced.d.ts +2 -2
- package/packages/core/dist/advanced.mjs +2 -2
- package/packages/core/dist/checks.cjs +172 -43
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +2 -2
- package/packages/core/dist/checks.d.ts +2 -2
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-OOX25GTH.mjs → chunk-VEQKLWGJ.mjs} +174 -46
- package/packages/core/dist/chunk-VEQKLWGJ.mjs.map +1 -0
- package/packages/core/dist/{index-7hG_qggp.d.ts → index-CgOAGuM5.d.ts} +111 -1
- package/packages/core/dist/{index-CQJUNPPe.d.cts → index-EQihlEei.d.cts} +111 -1
- package/packages/cli/dist/chunk-DBRMI7TC.mjs.map +0 -1
- package/packages/core/dist/chunk-OOX25GTH.mjs.map +0 -1
|
@@ -3270,7 +3270,7 @@ function deriveLastError(runs) {
|
|
|
3270
3270
|
}
|
|
3271
3271
|
function deriveCheckSummary(runs) {
|
|
3272
3272
|
let pass = 0;
|
|
3273
|
-
let
|
|
3273
|
+
let fail4 = 0;
|
|
3274
3274
|
let warn2 = 0;
|
|
3275
3275
|
let found = false;
|
|
3276
3276
|
for (const run of runs) {
|
|
@@ -3282,7 +3282,7 @@ function deriveCheckSummary(runs) {
|
|
|
3282
3282
|
found = true;
|
|
3283
3283
|
}
|
|
3284
3284
|
if (typeof record.fail === "number") {
|
|
3285
|
-
|
|
3285
|
+
fail4 += record.fail;
|
|
3286
3286
|
found = true;
|
|
3287
3287
|
}
|
|
3288
3288
|
if (typeof record.warn === "number") {
|
|
@@ -3290,7 +3290,7 @@ function deriveCheckSummary(runs) {
|
|
|
3290
3290
|
found = true;
|
|
3291
3291
|
}
|
|
3292
3292
|
}
|
|
3293
|
-
return found ? { pass, fail, warn: warn2 } : void 0;
|
|
3293
|
+
return found ? { pass, fail: fail4, warn: warn2 } : void 0;
|
|
3294
3294
|
}
|
|
3295
3295
|
function deriveObservationSummary(runs) {
|
|
3296
3296
|
for (const run of [...runs].sort((a, b) => activityMs(b) - activityMs(a))) {
|
|
@@ -5004,7 +5004,1359 @@ function isCredentialSensitiveKey(key, sensitiveKeys = DEFAULT_CREDENTIAL_SENSIT
|
|
|
5004
5004
|
return false;
|
|
5005
5005
|
}
|
|
5006
5006
|
|
|
5007
|
+
// packages/core/src/checks/contract-scope.ts
|
|
5008
|
+
function diagnostic(code, message, ruleId) {
|
|
5009
|
+
return {
|
|
5010
|
+
code,
|
|
5011
|
+
message,
|
|
5012
|
+
severity: "error",
|
|
5013
|
+
...ruleId !== void 0 ? { ruleId } : {}
|
|
5014
|
+
};
|
|
5015
|
+
}
|
|
5016
|
+
function isNonEmptyString4(value) {
|
|
5017
|
+
return typeof value === "string" && value.trim() !== "";
|
|
5018
|
+
}
|
|
5019
|
+
function scopeEntries(scope) {
|
|
5020
|
+
const out = [];
|
|
5021
|
+
if (isNonEmptyString4(scope.runId)) out.push(["runId", scope.runId.trim()]);
|
|
5022
|
+
if (isNonEmptyString4(scope.subAgentId)) out.push(["subAgentId", scope.subAgentId.trim()]);
|
|
5023
|
+
if (isNonEmptyString4(scope.groupId)) out.push(["groupId", scope.groupId.trim()]);
|
|
5024
|
+
if (isNonEmptyString4(scope.workflowStep)) {
|
|
5025
|
+
out.push(["workflowStep", scope.workflowStep.trim()]);
|
|
5026
|
+
}
|
|
5027
|
+
if (isNonEmptyString4(scope.rootEventId)) {
|
|
5028
|
+
out.push(["rootEventId", scope.rootEventId.trim()]);
|
|
5029
|
+
}
|
|
5030
|
+
return out;
|
|
5031
|
+
}
|
|
5032
|
+
function mergeWorkflowMeta(into, from) {
|
|
5033
|
+
if (!from) return;
|
|
5034
|
+
for (const [key, value] of Object.entries(from)) {
|
|
5035
|
+
if (value === void 0) continue;
|
|
5036
|
+
if (into[key] === void 0) {
|
|
5037
|
+
into[key] = value;
|
|
5038
|
+
}
|
|
5039
|
+
}
|
|
5040
|
+
}
|
|
5041
|
+
function workflowMetadataForRun(events, runId) {
|
|
5042
|
+
const preferred = {};
|
|
5043
|
+
const fallback = {};
|
|
5044
|
+
for (const event of events) {
|
|
5045
|
+
if (event.runId !== runId) continue;
|
|
5046
|
+
const attrs = event.attributes;
|
|
5047
|
+
if (!attrs || typeof attrs !== "object") continue;
|
|
5048
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
5049
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
5050
|
+
const bag = {};
|
|
5051
|
+
mergeWorkflowMeta(bag, direct);
|
|
5052
|
+
mergeWorkflowMeta(bag, nested);
|
|
5053
|
+
if (event.kind === "RUN") {
|
|
5054
|
+
mergeWorkflowMeta(preferred, bag);
|
|
5055
|
+
} else {
|
|
5056
|
+
mergeWorkflowMeta(fallback, bag);
|
|
5057
|
+
}
|
|
5058
|
+
}
|
|
5059
|
+
mergeWorkflowMeta(preferred, fallback);
|
|
5060
|
+
return preferred;
|
|
5061
|
+
}
|
|
5062
|
+
function flattenNodes(nodes) {
|
|
5063
|
+
return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
|
|
5064
|
+
}
|
|
5065
|
+
function collectSubtreeEventIds(run, rootEventId) {
|
|
5066
|
+
const nodes = flattenNodes(run.children);
|
|
5067
|
+
const byId = new Map(nodes.map((node) => [node.event.eventId, node]));
|
|
5068
|
+
const root = byId.get(rootEventId);
|
|
5069
|
+
if (!root) return void 0;
|
|
5070
|
+
const ids = /* @__PURE__ */ new Set();
|
|
5071
|
+
const visit = (node) => {
|
|
5072
|
+
ids.add(node.event.eventId);
|
|
5073
|
+
for (const child of node.children) visit(child);
|
|
5074
|
+
};
|
|
5075
|
+
visit(root);
|
|
5076
|
+
return ids;
|
|
5077
|
+
}
|
|
5078
|
+
function filterTree(nodes, keep) {
|
|
5079
|
+
const out = [];
|
|
5080
|
+
for (const node of nodes) {
|
|
5081
|
+
if (!keep.has(node.event.eventId)) continue;
|
|
5082
|
+
out.push({
|
|
5083
|
+
...node,
|
|
5084
|
+
children: filterTree(node.children, keep)
|
|
5085
|
+
});
|
|
5086
|
+
}
|
|
5087
|
+
return out;
|
|
5088
|
+
}
|
|
5089
|
+
function projectInput(input, run, eventIds) {
|
|
5090
|
+
const runEvents = input.read.events.filter((event) => event.runId === run.runId);
|
|
5091
|
+
const projectedEvents = eventIds === void 0 ? runEvents : runEvents.filter((event) => eventIds.has(event.eventId) || event.kind === "RUN");
|
|
5092
|
+
const projectedChildren = eventIds === void 0 ? run.children : filterTree(run.children, eventIds);
|
|
5093
|
+
const projectedRun = {
|
|
5094
|
+
...run,
|
|
5095
|
+
children: projectedChildren,
|
|
5096
|
+
metadata: {
|
|
5097
|
+
...run.metadata,
|
|
5098
|
+
totalEvents: flattenNodes(projectedChildren).length
|
|
5099
|
+
}
|
|
5100
|
+
};
|
|
5101
|
+
return {
|
|
5102
|
+
evidenceEventCount: projectedEvents.length,
|
|
5103
|
+
input: {
|
|
5104
|
+
...input,
|
|
5105
|
+
selectedRun: projectedRun,
|
|
5106
|
+
read: {
|
|
5107
|
+
...input.read,
|
|
5108
|
+
runs: [projectedRun],
|
|
5109
|
+
events: projectedEvents
|
|
5110
|
+
}
|
|
5111
|
+
}
|
|
5112
|
+
};
|
|
5113
|
+
}
|
|
5114
|
+
function resolveTraceContractScope(input, scope) {
|
|
5115
|
+
if (scope === void 0) {
|
|
5116
|
+
return { diagnostics: [] };
|
|
5117
|
+
}
|
|
5118
|
+
const entries = scopeEntries(scope);
|
|
5119
|
+
if (entries.length === 0) {
|
|
5120
|
+
return {
|
|
5121
|
+
diagnostics: [
|
|
5122
|
+
diagnostic(
|
|
5123
|
+
"AI_CHECK_INVALID_CONFIG",
|
|
5124
|
+
"TraceContract scope requires at least one explicit selector (runId, subAgentId, groupId, workflowStep, or rootEventId).",
|
|
5125
|
+
"contract.scope.empty"
|
|
5126
|
+
)
|
|
5127
|
+
]
|
|
5128
|
+
};
|
|
5129
|
+
}
|
|
5130
|
+
const matchedSelectors = Object.fromEntries(entries);
|
|
5131
|
+
const actorKeys = entries.filter(([key]) => key !== "rootEventId");
|
|
5132
|
+
const rootEventId = matchedSelectors.rootEventId;
|
|
5133
|
+
let candidates = [...input.read.runs];
|
|
5134
|
+
if (matchedSelectors.runId) {
|
|
5135
|
+
candidates = candidates.filter((run2) => run2.runId === matchedSelectors.runId);
|
|
5136
|
+
}
|
|
5137
|
+
for (const [key, value] of actorKeys) {
|
|
5138
|
+
if (key === "runId") continue;
|
|
5139
|
+
candidates = candidates.filter((run2) => {
|
|
5140
|
+
const meta = workflowMetadataForRun(input.read.events, run2.runId);
|
|
5141
|
+
const actual = meta[key];
|
|
5142
|
+
return actual === value;
|
|
5143
|
+
});
|
|
5144
|
+
}
|
|
5145
|
+
if (rootEventId && actorKeys.length === 0) {
|
|
5146
|
+
candidates = candidates.filter(
|
|
5147
|
+
(run2) => flattenNodes(run2.children).some((node) => node.event.eventId === rootEventId)
|
|
5148
|
+
);
|
|
5149
|
+
}
|
|
5150
|
+
if (candidates.length === 0) {
|
|
5151
|
+
return {
|
|
5152
|
+
diagnostics: [
|
|
5153
|
+
diagnostic(
|
|
5154
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
5155
|
+
`TraceContract scope matched zero actors for ${JSON.stringify(matchedSelectors)}. Missing actor metadata does not fall back to the whole session.`,
|
|
5156
|
+
"contract.scope.zero-match"
|
|
5157
|
+
)
|
|
5158
|
+
]
|
|
5159
|
+
};
|
|
5160
|
+
}
|
|
5161
|
+
if (candidates.length > 1) {
|
|
5162
|
+
return {
|
|
5163
|
+
diagnostics: [
|
|
5164
|
+
diagnostic(
|
|
5165
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
5166
|
+
`TraceContract scope matched ${candidates.length} actors for ${JSON.stringify(matchedSelectors)} (runIds: ${candidates.map((run2) => run2.runId).sort().join(", ")}). Singular actor selectors must resolve to exactly one run.`,
|
|
5167
|
+
"contract.scope.ambiguous"
|
|
5168
|
+
)
|
|
5169
|
+
]
|
|
5170
|
+
};
|
|
5171
|
+
}
|
|
5172
|
+
const run = candidates[0];
|
|
5173
|
+
let eventIds;
|
|
5174
|
+
if (rootEventId) {
|
|
5175
|
+
eventIds = collectSubtreeEventIds(run, rootEventId);
|
|
5176
|
+
if (eventIds === void 0) {
|
|
5177
|
+
return {
|
|
5178
|
+
diagnostics: [
|
|
5179
|
+
diagnostic(
|
|
5180
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
5181
|
+
`TraceContract scope rootEventId ${rootEventId} was not found in run ${run.runId}.`,
|
|
5182
|
+
"contract.scope.root-missing"
|
|
5183
|
+
)
|
|
5184
|
+
]
|
|
5185
|
+
};
|
|
5186
|
+
}
|
|
5187
|
+
}
|
|
5188
|
+
const projected = projectInput(input, run, eventIds);
|
|
5189
|
+
return {
|
|
5190
|
+
resolved: {
|
|
5191
|
+
runId: run.runId,
|
|
5192
|
+
...rootEventId !== void 0 ? { rootEventId } : {},
|
|
5193
|
+
matchedSelectors,
|
|
5194
|
+
evidenceEventCount: projected.evidenceEventCount,
|
|
5195
|
+
input: projected.input
|
|
5196
|
+
},
|
|
5197
|
+
diagnostics: []
|
|
5198
|
+
};
|
|
5199
|
+
}
|
|
5200
|
+
|
|
5201
|
+
// packages/core/src/checks/control-rules.ts
|
|
5202
|
+
function asStringList(value) {
|
|
5203
|
+
if (!Array.isArray(value)) return void 0;
|
|
5204
|
+
const out = [];
|
|
5205
|
+
for (const item of value) {
|
|
5206
|
+
if (typeof item !== "string" || item.trim() === "") return void 0;
|
|
5207
|
+
out.push(item.trim());
|
|
5208
|
+
}
|
|
5209
|
+
return out;
|
|
5210
|
+
}
|
|
5211
|
+
function attributeList(events, attribute) {
|
|
5212
|
+
if (!attribute) return void 0;
|
|
5213
|
+
for (const event of events) {
|
|
5214
|
+
const attrs = event.attributes;
|
|
5215
|
+
if (!attrs || typeof attrs !== "object") continue;
|
|
5216
|
+
const direct = asStringList(attrs[attribute]);
|
|
5217
|
+
if (direct) return direct;
|
|
5218
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? asStringList(attrs.metadata[attribute]) : void 0;
|
|
5219
|
+
if (nested) return nested;
|
|
5220
|
+
}
|
|
5221
|
+
return void 0;
|
|
5222
|
+
}
|
|
5223
|
+
function fail(ruleId, message, evidence, expected, actual) {
|
|
5224
|
+
return {
|
|
5225
|
+
ruleId,
|
|
5226
|
+
severity: "error",
|
|
5227
|
+
status: "fail",
|
|
5228
|
+
message,
|
|
5229
|
+
...expected !== void 0 ? { expected } : {},
|
|
5230
|
+
...actual !== void 0 ? { actual } : {},
|
|
5231
|
+
evidence: [...evidence]
|
|
5232
|
+
};
|
|
5233
|
+
}
|
|
5234
|
+
function sortedUnique(values) {
|
|
5235
|
+
return [...new Set(values)].sort((a, b) => a.localeCompare(b));
|
|
5236
|
+
}
|
|
5237
|
+
function setEqual(left, right) {
|
|
5238
|
+
const a = sortedUnique(left);
|
|
5239
|
+
const b = sortedUnique(right);
|
|
5240
|
+
return a.length === b.length && a.every((value, index) => value === b[index]);
|
|
5241
|
+
}
|
|
5242
|
+
function evaluateControlRules(events, rules, runEvidence2, observationNames) {
|
|
5243
|
+
const findings = [];
|
|
5244
|
+
const declared = rules.declaredTools ?? attributeList(events, rules.declaredToolsAttribute);
|
|
5245
|
+
const enforced = rules.enforcedTools ?? attributeList(events, rules.enforcedToolsAttribute);
|
|
5246
|
+
if (rules.requireDeclaredMatchesEnforced) {
|
|
5247
|
+
if (declared === void 0 || enforced === void 0) {
|
|
5248
|
+
findings.push(
|
|
5249
|
+
fail(
|
|
5250
|
+
"contract.controls.declared-matches-enforced",
|
|
5251
|
+
"Declared and enforced tool sets could not both be resolved.",
|
|
5252
|
+
runEvidence2,
|
|
5253
|
+
{ declaredDefined: declared !== void 0, enforcedDefined: enforced !== void 0 },
|
|
5254
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
5255
|
+
)
|
|
5256
|
+
);
|
|
5257
|
+
} else if (!setEqual(declared, enforced)) {
|
|
5258
|
+
findings.push(
|
|
5259
|
+
fail(
|
|
5260
|
+
"contract.controls.declared-matches-enforced",
|
|
5261
|
+
"Declared tools differ from enforced tools.",
|
|
5262
|
+
runEvidence2,
|
|
5263
|
+
sortedUnique(declared),
|
|
5264
|
+
sortedUnique(enforced)
|
|
5265
|
+
)
|
|
5266
|
+
);
|
|
5267
|
+
}
|
|
5268
|
+
}
|
|
5269
|
+
const observedTools = sortedUnique(
|
|
5270
|
+
events.filter((event) => event.kind === "TOOL" && event.status !== "running").map((event) => resolveCanonicalToolName(event))
|
|
5271
|
+
);
|
|
5272
|
+
if (rules.requireObservedWithinEnforced) {
|
|
5273
|
+
if (enforced === void 0) {
|
|
5274
|
+
findings.push(
|
|
5275
|
+
fail(
|
|
5276
|
+
"contract.controls.observed-within-enforced",
|
|
5277
|
+
"Enforced tool set unavailable for observed-within-enforced check.",
|
|
5278
|
+
runEvidence2,
|
|
5279
|
+
void 0,
|
|
5280
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
5281
|
+
)
|
|
5282
|
+
);
|
|
5283
|
+
} else {
|
|
5284
|
+
const enforcedSet = new Set(enforced);
|
|
5285
|
+
const outside = observedTools.filter((name) => !enforcedSet.has(name));
|
|
5286
|
+
if (outside.length > 0) {
|
|
5287
|
+
findings.push(
|
|
5288
|
+
fail(
|
|
5289
|
+
"contract.controls.observed-within-enforced",
|
|
5290
|
+
`Observed tools outside enforced allowlist: ${outside.join(", ")}.`,
|
|
5291
|
+
runEvidence2,
|
|
5292
|
+
sortedUnique(enforced),
|
|
5293
|
+
outside
|
|
5294
|
+
)
|
|
5295
|
+
);
|
|
5296
|
+
}
|
|
5297
|
+
}
|
|
5298
|
+
}
|
|
5299
|
+
if (rules.requireObservedWithinDeclared) {
|
|
5300
|
+
if (declared === void 0) {
|
|
5301
|
+
findings.push(
|
|
5302
|
+
fail(
|
|
5303
|
+
"contract.controls.observed-within-declared",
|
|
5304
|
+
"Declared tool set unavailable for observed-within-declared check.",
|
|
5305
|
+
runEvidence2,
|
|
5306
|
+
void 0,
|
|
5307
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
5308
|
+
)
|
|
5309
|
+
);
|
|
5310
|
+
} else {
|
|
5311
|
+
const declaredSet = new Set(declared);
|
|
5312
|
+
const outside = observedTools.filter((name) => !declaredSet.has(name));
|
|
5313
|
+
if (outside.length > 0) {
|
|
5314
|
+
findings.push(
|
|
5315
|
+
fail(
|
|
5316
|
+
"contract.controls.observed-within-declared",
|
|
5317
|
+
`Observed tools outside declared set: ${outside.join(", ")}.`,
|
|
5318
|
+
runEvidence2,
|
|
5319
|
+
sortedUnique(declared),
|
|
5320
|
+
outside
|
|
5321
|
+
)
|
|
5322
|
+
);
|
|
5323
|
+
}
|
|
5324
|
+
}
|
|
5325
|
+
}
|
|
5326
|
+
for (const stage of rules.requiredStages ?? []) {
|
|
5327
|
+
const observation = stage.observation ?? `control.${stage.stage}`;
|
|
5328
|
+
if (!observationNames.has(observation)) {
|
|
5329
|
+
findings.push(
|
|
5330
|
+
fail(
|
|
5331
|
+
`contract.controls.stage.${stage.stage}`,
|
|
5332
|
+
`Required control stage observation missing: ${observation}.`,
|
|
5333
|
+
runEvidence2,
|
|
5334
|
+
observation,
|
|
5335
|
+
[...observationNames]
|
|
5336
|
+
)
|
|
5337
|
+
);
|
|
5338
|
+
}
|
|
5339
|
+
}
|
|
5340
|
+
return findings;
|
|
5341
|
+
}
|
|
5342
|
+
|
|
5343
|
+
// packages/core/src/checks/canonical-equality.ts
|
|
5344
|
+
var DEFAULT_MAX_DEPTH = 8;
|
|
5345
|
+
var DEFAULT_MAX_BYTES = 16384;
|
|
5346
|
+
function sortKeysDeep(input, depth, maxDepth) {
|
|
5347
|
+
if (depth > maxDepth) {
|
|
5348
|
+
throw new Error("AI_CHECK_CANONICAL_DEPTH_EXCEEDED");
|
|
5349
|
+
}
|
|
5350
|
+
if (input === null || typeof input !== "object") return input;
|
|
5351
|
+
if (Array.isArray(input)) {
|
|
5352
|
+
return input.map((item) => sortKeysDeep(item, depth + 1, maxDepth));
|
|
5353
|
+
}
|
|
5354
|
+
const record = input;
|
|
5355
|
+
const out = {};
|
|
5356
|
+
for (const key of Object.keys(record).sort()) {
|
|
5357
|
+
out[key] = sortKeysDeep(record[key], depth + 1, maxDepth);
|
|
5358
|
+
}
|
|
5359
|
+
return out;
|
|
5360
|
+
}
|
|
5361
|
+
function encodeCanonicalStructured(value, limits) {
|
|
5362
|
+
const maxDepth = DEFAULT_MAX_DEPTH;
|
|
5363
|
+
const maxBytes = DEFAULT_MAX_BYTES;
|
|
5364
|
+
try {
|
|
5365
|
+
const sorted = sortKeysDeep(value, 0, maxDepth);
|
|
5366
|
+
const encoded = JSON.stringify(sorted);
|
|
5367
|
+
if (encoded === void 0) {
|
|
5368
|
+
return {
|
|
5369
|
+
ok: false,
|
|
5370
|
+
reason: "Value is not JSON-serializable for canonical comparison.",
|
|
5371
|
+
code: "AI_CHECK_CANONICAL_UNSERIALIZABLE"
|
|
5372
|
+
};
|
|
5373
|
+
}
|
|
5374
|
+
if (Buffer.byteLength(encoded, "utf8") > maxBytes) {
|
|
5375
|
+
return {
|
|
5376
|
+
ok: false,
|
|
5377
|
+
reason: "Canonical structured value exceeds bounded size limit.",
|
|
5378
|
+
code: "AI_CHECK_CANONICAL_SIZE_EXCEEDED"
|
|
5379
|
+
};
|
|
5380
|
+
}
|
|
5381
|
+
return { ok: true, value: encoded };
|
|
5382
|
+
} catch (error) {
|
|
5383
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5384
|
+
if (message === "AI_CHECK_CANONICAL_DEPTH_EXCEEDED") {
|
|
5385
|
+
return {
|
|
5386
|
+
ok: false,
|
|
5387
|
+
reason: "Canonical structured value exceeds bounded depth limit.",
|
|
5388
|
+
code: "AI_CHECK_CANONICAL_DEPTH_EXCEEDED"
|
|
5389
|
+
};
|
|
5390
|
+
}
|
|
5391
|
+
return {
|
|
5392
|
+
ok: false,
|
|
5393
|
+
reason: "Canonical structured encoding failed.",
|
|
5394
|
+
code: "AI_CHECK_CANONICAL_UNSERIALIZABLE"
|
|
5395
|
+
};
|
|
5396
|
+
}
|
|
5397
|
+
}
|
|
5398
|
+
function canonicalStructuredEqual(left, right, limits) {
|
|
5399
|
+
const a = encodeCanonicalStructured(left);
|
|
5400
|
+
if (!a.ok) return { equal: false, reason: a.reason, code: a.code };
|
|
5401
|
+
const b = encodeCanonicalStructured(right);
|
|
5402
|
+
if (!b.ok) return { equal: false, reason: b.reason, code: b.code };
|
|
5403
|
+
if (a.value !== b.value) {
|
|
5404
|
+
return {
|
|
5405
|
+
equal: false,
|
|
5406
|
+
reason: "Canonical structured values differ.",
|
|
5407
|
+
code: "AI_CHECK_CANONICAL_NOT_EQUAL"
|
|
5408
|
+
};
|
|
5409
|
+
}
|
|
5410
|
+
return { equal: true };
|
|
5411
|
+
}
|
|
5412
|
+
|
|
5413
|
+
// packages/core/src/checks/tool-arguments.ts
|
|
5414
|
+
function decodeToken(token) {
|
|
5415
|
+
return token.replace(/~1/g, "/").replace(/~0/g, "~");
|
|
5416
|
+
}
|
|
5417
|
+
function resolveJsonPointer(document, pointer) {
|
|
5418
|
+
if (pointer === "") {
|
|
5419
|
+
return { found: true, value: document };
|
|
5420
|
+
}
|
|
5421
|
+
if (!pointer.startsWith("/")) {
|
|
5422
|
+
return { found: false };
|
|
5423
|
+
}
|
|
5424
|
+
const tokens = pointer.slice(1).split("/").map(decodeToken);
|
|
5425
|
+
let current = document;
|
|
5426
|
+
for (const token of tokens) {
|
|
5427
|
+
if (current === null || current === void 0) {
|
|
5428
|
+
return { found: false };
|
|
5429
|
+
}
|
|
5430
|
+
if (Array.isArray(current)) {
|
|
5431
|
+
if (!/^(0|[1-9][0-9]*)$/.test(token)) {
|
|
5432
|
+
return { found: false };
|
|
5433
|
+
}
|
|
5434
|
+
const index = Number(token);
|
|
5435
|
+
if (index >= current.length) {
|
|
5436
|
+
return { found: false };
|
|
5437
|
+
}
|
|
5438
|
+
current = current[index];
|
|
5439
|
+
continue;
|
|
5440
|
+
}
|
|
5441
|
+
if (typeof current !== "object") {
|
|
5442
|
+
return { found: false };
|
|
5443
|
+
}
|
|
5444
|
+
const record = current;
|
|
5445
|
+
if (!Object.prototype.hasOwnProperty.call(record, token)) {
|
|
5446
|
+
return { found: false };
|
|
5447
|
+
}
|
|
5448
|
+
current = record[token];
|
|
5449
|
+
}
|
|
5450
|
+
return { found: true, value: current };
|
|
5451
|
+
}
|
|
5452
|
+
function jsonType(value) {
|
|
5453
|
+
if (value === null) return "null";
|
|
5454
|
+
if (Array.isArray(value)) return "array";
|
|
5455
|
+
return typeof value;
|
|
5456
|
+
}
|
|
5457
|
+
function valuesEqual(left, right) {
|
|
5458
|
+
return canonicalStructuredEqual(left, right).equal === true;
|
|
5459
|
+
}
|
|
5460
|
+
function isStructuredArgumentEvidence(value) {
|
|
5461
|
+
return value !== void 0 && value !== null && typeof value === "object";
|
|
5462
|
+
}
|
|
5463
|
+
function extractToolArgumentPayload(event) {
|
|
5464
|
+
const attrs = event.attributes ?? {};
|
|
5465
|
+
for (const key of ["arguments", "input", "toolArguments"]) {
|
|
5466
|
+
const candidate = attrs[key];
|
|
5467
|
+
if (isStructuredArgumentEvidence(candidate)) {
|
|
5468
|
+
return { present: true, value: candidate };
|
|
5469
|
+
}
|
|
5470
|
+
}
|
|
5471
|
+
const metadata = attrs.metadata;
|
|
5472
|
+
if (isStructuredArgumentEvidence(metadata) && !Array.isArray(metadata)) {
|
|
5473
|
+
const nested = metadata;
|
|
5474
|
+
for (const key of ["arguments", "input", "toolArguments"]) {
|
|
5475
|
+
const candidate = nested[key];
|
|
5476
|
+
if (isStructuredArgumentEvidence(candidate)) {
|
|
5477
|
+
return { present: true, value: candidate };
|
|
5478
|
+
}
|
|
5479
|
+
}
|
|
5480
|
+
}
|
|
5481
|
+
if (isStructuredArgumentEvidence(event.inputSummary)) {
|
|
5482
|
+
return { present: true, value: event.inputSummary };
|
|
5483
|
+
}
|
|
5484
|
+
return { present: false, value: void 0 };
|
|
5485
|
+
}
|
|
5486
|
+
function evaluateToolArgumentValue(value, check, evidencePresent) {
|
|
5487
|
+
if (!evidencePresent) {
|
|
5488
|
+
return {
|
|
5489
|
+
status: "unavailable",
|
|
5490
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
5491
|
+
message: `Structured argument evidence unavailable for tool ${check.tool} at ${check.path}.`
|
|
5492
|
+
};
|
|
5493
|
+
}
|
|
5494
|
+
const resolved = resolveJsonPointer(value, check.path);
|
|
5495
|
+
if (check.operator === "exists") {
|
|
5496
|
+
return resolved.found ? { status: "pass" } : {
|
|
5497
|
+
status: "fail",
|
|
5498
|
+
code: "AI_CHECK_TOOL_ARGUMENT_MISSING",
|
|
5499
|
+
message: `Expected path ${check.path} to exist on tool ${check.tool}.`
|
|
5500
|
+
};
|
|
5501
|
+
}
|
|
5502
|
+
if (!resolved.found) {
|
|
5503
|
+
return {
|
|
5504
|
+
status: "unavailable",
|
|
5505
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
5506
|
+
message: `Path ${check.path} missing for tool ${check.tool}.`
|
|
5507
|
+
};
|
|
5508
|
+
}
|
|
5509
|
+
if (check.operator === "type") {
|
|
5510
|
+
const actual = jsonType(resolved.value);
|
|
5511
|
+
if (check.type === void 0) {
|
|
5512
|
+
return {
|
|
5513
|
+
status: "fail",
|
|
5514
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
5515
|
+
message: "type operator requires check.type"
|
|
5516
|
+
};
|
|
5517
|
+
}
|
|
5518
|
+
return actual === check.type ? { status: "pass" } : {
|
|
5519
|
+
status: "fail",
|
|
5520
|
+
code: "AI_CHECK_TOOL_ARGUMENT_TYPE",
|
|
5521
|
+
message: `Expected type ${check.type} at ${check.path} for tool ${check.tool}; found ${actual}.`
|
|
5522
|
+
};
|
|
5523
|
+
}
|
|
5524
|
+
if (check.operator === "equals") {
|
|
5525
|
+
return valuesEqual(resolved.value, check.expected) ? { status: "pass" } : {
|
|
5526
|
+
status: "fail",
|
|
5527
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EQUALS",
|
|
5528
|
+
message: `Value at ${check.path} for tool ${check.tool} did not equal expected (bounded comparison).`
|
|
5529
|
+
};
|
|
5530
|
+
}
|
|
5531
|
+
if (check.operator === "oneOf") {
|
|
5532
|
+
const options = check.oneOf ?? [];
|
|
5533
|
+
if (options.length === 0) {
|
|
5534
|
+
return {
|
|
5535
|
+
status: "fail",
|
|
5536
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
5537
|
+
message: "oneOf operator requires a non-empty oneOf array"
|
|
5538
|
+
};
|
|
5539
|
+
}
|
|
5540
|
+
return options.some((option) => valuesEqual(resolved.value, option)) ? { status: "pass" } : {
|
|
5541
|
+
status: "fail",
|
|
5542
|
+
code: "AI_CHECK_TOOL_ARGUMENT_ONE_OF",
|
|
5543
|
+
message: `Value at ${check.path} for tool ${check.tool} was not in the allowed oneOf set.`
|
|
5544
|
+
};
|
|
5545
|
+
}
|
|
5546
|
+
return {
|
|
5547
|
+
status: "fail",
|
|
5548
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
5549
|
+
message: `Unsupported operator.`
|
|
5550
|
+
};
|
|
5551
|
+
}
|
|
5552
|
+
|
|
5553
|
+
// packages/core/src/checks/retry-safety.ts
|
|
5554
|
+
function fail2(ruleId, message, evidence, expected, actual) {
|
|
5555
|
+
return {
|
|
5556
|
+
ruleId,
|
|
5557
|
+
severity: "error",
|
|
5558
|
+
status: "fail",
|
|
5559
|
+
message,
|
|
5560
|
+
...expected !== void 0 ? { expected } : {},
|
|
5561
|
+
...actual !== void 0 ? { actual } : {},
|
|
5562
|
+
evidence: [...evidence]
|
|
5563
|
+
};
|
|
5564
|
+
}
|
|
5565
|
+
function workflowFor(event) {
|
|
5566
|
+
const attrs = event.attributes;
|
|
5567
|
+
if (!attrs || typeof attrs !== "object") return {};
|
|
5568
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
5569
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
5570
|
+
return { ...nested, ...direct };
|
|
5571
|
+
}
|
|
5572
|
+
function hasIdempotencyEvidence(event) {
|
|
5573
|
+
const workflow = workflowFor(event);
|
|
5574
|
+
if (typeof workflow.idempotencyKey === "string" && workflow.idempotencyKey.trim() !== "") {
|
|
5575
|
+
return true;
|
|
5576
|
+
}
|
|
5577
|
+
const attrs = event.attributes ?? {};
|
|
5578
|
+
if (attrs.noSideEffect === true) return true;
|
|
5579
|
+
if (attrs.sideEffect === false) return true;
|
|
5580
|
+
return false;
|
|
5581
|
+
}
|
|
5582
|
+
function eventEvidence(event) {
|
|
5583
|
+
return {
|
|
5584
|
+
runId: event.runId,
|
|
5585
|
+
eventId: event.eventId,
|
|
5586
|
+
kind: event.kind,
|
|
5587
|
+
name: event.name,
|
|
5588
|
+
status: event.status
|
|
5589
|
+
};
|
|
5590
|
+
}
|
|
5591
|
+
function eventTime(event) {
|
|
5592
|
+
return event.startedAt ?? event.timestamp ?? "";
|
|
5593
|
+
}
|
|
5594
|
+
function sortByTime(events) {
|
|
5595
|
+
return [...events].sort((a, b) => eventTime(a).localeCompare(eventTime(b)));
|
|
5596
|
+
}
|
|
5597
|
+
function attemptNumberOf(event) {
|
|
5598
|
+
const workflow = workflowFor(event);
|
|
5599
|
+
const value = workflow.attemptNumber ?? workflow.attempt;
|
|
5600
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
5601
|
+
}
|
|
5602
|
+
function attemptIdOf(event) {
|
|
5603
|
+
const value = workflowFor(event).attemptId;
|
|
5604
|
+
return typeof value === "string" && value.trim() !== "" ? value : void 0;
|
|
5605
|
+
}
|
|
5606
|
+
function countOperationAttempts(members) {
|
|
5607
|
+
const attemptIds = new Set(
|
|
5608
|
+
members.map((event) => attemptIdOf(event)).filter((value) => value !== void 0)
|
|
5609
|
+
);
|
|
5610
|
+
if (attemptIds.size > 0) {
|
|
5611
|
+
return attemptIds.size;
|
|
5612
|
+
}
|
|
5613
|
+
const numbers = members.map((event) => attemptNumberOf(event)).filter((value) => value !== void 0).sort((a, b) => a - b);
|
|
5614
|
+
if (numbers.length > 0) {
|
|
5615
|
+
const unique = [...new Set(numbers)];
|
|
5616
|
+
const min = unique[0];
|
|
5617
|
+
const max = unique[unique.length - 1];
|
|
5618
|
+
const contiguous = unique.length === max - min + 1 && unique.every((value, index) => value === min + index);
|
|
5619
|
+
if (contiguous) {
|
|
5620
|
+
return unique.length;
|
|
5621
|
+
}
|
|
5622
|
+
return unique.length;
|
|
5623
|
+
}
|
|
5624
|
+
return members.filter((event) => event.kind === "TOOL" || event.kind === "LLM").length;
|
|
5625
|
+
}
|
|
5626
|
+
function classifyRetryAttempt(event, priorOrdered, allMembers) {
|
|
5627
|
+
const workflow = workflowFor(event);
|
|
5628
|
+
const attemptNumber = attemptNumberOf(event);
|
|
5629
|
+
const attemptId = attemptIdOf(event);
|
|
5630
|
+
const retryOf = workflow.retryOf;
|
|
5631
|
+
if (typeof retryOf === "string" && retryOf.trim() !== "") {
|
|
5632
|
+
const target = allMembers.find(
|
|
5633
|
+
(candidate) => candidate.eventId === retryOf || attemptIdOf(candidate) === retryOf || workflowFor(candidate).operationId === retryOf
|
|
5634
|
+
);
|
|
5635
|
+
if (!target) {
|
|
5636
|
+
return { kind: "unknown", reason: "retryOf target missing" };
|
|
5637
|
+
}
|
|
5638
|
+
if (eventTime(target) > eventTime(event)) {
|
|
5639
|
+
return { kind: "unknown", reason: "retryOf target does not precede retry" };
|
|
5640
|
+
}
|
|
5641
|
+
return { kind: "retry" };
|
|
5642
|
+
}
|
|
5643
|
+
if (attemptNumber !== void 0) {
|
|
5644
|
+
if (attemptNumber > 1) {
|
|
5645
|
+
return { kind: "retry" };
|
|
5646
|
+
}
|
|
5647
|
+
if (attemptNumber === 1) {
|
|
5648
|
+
return { kind: "first" };
|
|
5649
|
+
}
|
|
5650
|
+
}
|
|
5651
|
+
if (attemptId !== void 0) {
|
|
5652
|
+
const priorDistinct = priorOrdered.some((candidate) => {
|
|
5653
|
+
const priorId = attemptIdOf(candidate);
|
|
5654
|
+
return priorId !== void 0 && priorId !== attemptId;
|
|
5655
|
+
});
|
|
5656
|
+
if (priorDistinct) {
|
|
5657
|
+
return { kind: "retry" };
|
|
5658
|
+
}
|
|
5659
|
+
if (priorOrdered.length === 0) {
|
|
5660
|
+
return { kind: "first" };
|
|
5661
|
+
}
|
|
5662
|
+
}
|
|
5663
|
+
const priorFinished = priorOrdered.filter(
|
|
5664
|
+
(candidate) => candidate.status === "ok" || candidate.status === "error"
|
|
5665
|
+
);
|
|
5666
|
+
if (priorFinished.length > 0) {
|
|
5667
|
+
return { kind: "retry" };
|
|
5668
|
+
}
|
|
5669
|
+
if (priorOrdered.length === 0) {
|
|
5670
|
+
return { kind: "first" };
|
|
5671
|
+
}
|
|
5672
|
+
return { kind: "unknown", reason: "ambiguous attempt identity" };
|
|
5673
|
+
}
|
|
5674
|
+
function hasDuplicateAttemptIds(members) {
|
|
5675
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5676
|
+
for (const event of members) {
|
|
5677
|
+
const id = attemptIdOf(event);
|
|
5678
|
+
if (!id) continue;
|
|
5679
|
+
if (seen.has(id)) return true;
|
|
5680
|
+
seen.add(id);
|
|
5681
|
+
}
|
|
5682
|
+
return false;
|
|
5683
|
+
}
|
|
5684
|
+
function evaluateRetrySafetyRules(events, rules, _runEvidence) {
|
|
5685
|
+
const findings = [];
|
|
5686
|
+
const byOperation = /* @__PURE__ */ new Map();
|
|
5687
|
+
for (const event of events) {
|
|
5688
|
+
const workflow = workflowFor(event);
|
|
5689
|
+
const operationId = workflow.operationId;
|
|
5690
|
+
if (!operationId) continue;
|
|
5691
|
+
const list = byOperation.get(operationId) ?? [];
|
|
5692
|
+
list.push(event);
|
|
5693
|
+
byOperation.set(operationId, list);
|
|
5694
|
+
}
|
|
5695
|
+
if (rules.maxAttempts !== void 0) {
|
|
5696
|
+
for (const [operationId, members] of byOperation) {
|
|
5697
|
+
if (hasDuplicateAttemptIds(members)) {
|
|
5698
|
+
findings.push(
|
|
5699
|
+
fail2(
|
|
5700
|
+
"contract.retry.duplicate-attempt-id",
|
|
5701
|
+
`Operation ${operationId} has duplicate attemptId values.`,
|
|
5702
|
+
members.slice(0, 4).map(eventEvidence),
|
|
5703
|
+
"unique attemptId",
|
|
5704
|
+
"duplicate"
|
|
5705
|
+
)
|
|
5706
|
+
);
|
|
5707
|
+
}
|
|
5708
|
+
const count = countOperationAttempts(members);
|
|
5709
|
+
if (count > rules.maxAttempts) {
|
|
5710
|
+
findings.push(
|
|
5711
|
+
fail2(
|
|
5712
|
+
"contract.retry.max-attempts",
|
|
5713
|
+
`Operation ${operationId} exceeded maxAttempts ${rules.maxAttempts}.`,
|
|
5714
|
+
members.slice(0, 4).map(eventEvidence),
|
|
5715
|
+
rules.maxAttempts,
|
|
5716
|
+
count
|
|
5717
|
+
)
|
|
5718
|
+
);
|
|
5719
|
+
}
|
|
5720
|
+
}
|
|
5721
|
+
}
|
|
5722
|
+
if (rules.requireTerminalResult) {
|
|
5723
|
+
for (const [operationId, members] of byOperation) {
|
|
5724
|
+
const hasTerminal = members.some((event) => event.status === "ok" || event.status === "error");
|
|
5725
|
+
if (!hasTerminal) {
|
|
5726
|
+
findings.push(
|
|
5727
|
+
fail2(
|
|
5728
|
+
"contract.retry.terminal-result",
|
|
5729
|
+
`Operation ${operationId} has no terminal ok/error result.`,
|
|
5730
|
+
members.slice(0, 4).map(eventEvidence),
|
|
5731
|
+
"ok|error",
|
|
5732
|
+
members.map((event) => event.status)
|
|
5733
|
+
)
|
|
5734
|
+
);
|
|
5735
|
+
}
|
|
5736
|
+
}
|
|
5737
|
+
}
|
|
5738
|
+
if (rules.fallbackOnlyAfterFailure) {
|
|
5739
|
+
for (const event of events) {
|
|
5740
|
+
const workflow = workflowFor(event);
|
|
5741
|
+
const fallbackOf = workflow.fallbackOf;
|
|
5742
|
+
if (!fallbackOf) continue;
|
|
5743
|
+
const prior = byOperation.get(fallbackOf) ?? [];
|
|
5744
|
+
const fallbackStart = eventTime(event);
|
|
5745
|
+
const earlierFailure = prior.some(
|
|
5746
|
+
(candidate) => candidate.status === "error" && eventTime(candidate) !== "" && fallbackStart !== "" && eventTime(candidate) < fallbackStart
|
|
5747
|
+
);
|
|
5748
|
+
if (!earlierFailure) {
|
|
5749
|
+
findings.push(
|
|
5750
|
+
fail2(
|
|
5751
|
+
"contract.retry.fallback-after-failure",
|
|
5752
|
+
`Fallback for ${fallbackOf} appeared without an earlier failure in the related operation.`,
|
|
5753
|
+
[eventEvidence(event)],
|
|
5754
|
+
"earlier error attempt",
|
|
5755
|
+
{ fallbackOf }
|
|
5756
|
+
)
|
|
5757
|
+
);
|
|
5758
|
+
}
|
|
5759
|
+
}
|
|
5760
|
+
}
|
|
5761
|
+
const nonIdempotent = new Set(rules.nonIdempotentTools ?? []);
|
|
5762
|
+
if (nonIdempotent.size > 0 || rules.requireIdempotencyEvidenceForRetry) {
|
|
5763
|
+
const toolEvents = events.filter(
|
|
5764
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
5765
|
+
);
|
|
5766
|
+
const byToolOp = /* @__PURE__ */ new Map();
|
|
5767
|
+
for (const event of toolEvents) {
|
|
5768
|
+
const name = resolveCanonicalToolName(event);
|
|
5769
|
+
const workflow = workflowFor(event);
|
|
5770
|
+
const key = `${workflow.operationId ?? name}::${name}`;
|
|
5771
|
+
const list = byToolOp.get(key) ?? [];
|
|
5772
|
+
list.push(event);
|
|
5773
|
+
byToolOp.set(key, list);
|
|
5774
|
+
}
|
|
5775
|
+
for (const [, members] of byToolOp) {
|
|
5776
|
+
const ordered = sortByTime(members);
|
|
5777
|
+
let sawOk = false;
|
|
5778
|
+
let sawSideEffectOk = false;
|
|
5779
|
+
const prior = [];
|
|
5780
|
+
for (const event of ordered) {
|
|
5781
|
+
const name = resolveCanonicalToolName(event);
|
|
5782
|
+
const classification = classifyRetryAttempt(event, prior, members);
|
|
5783
|
+
if (classification.kind === "unknown" && rules.requireIdempotencyEvidenceForRetry) {
|
|
5784
|
+
findings.push(
|
|
5785
|
+
fail2(
|
|
5786
|
+
"contract.retry.ambiguous-identity",
|
|
5787
|
+
`Ambiguous attempt identity for tool ${name}; strict retry evidence cannot pass.`,
|
|
5788
|
+
[eventEvidence(event)],
|
|
5789
|
+
"explicit attempt identity",
|
|
5790
|
+
classification.reason
|
|
5791
|
+
)
|
|
5792
|
+
);
|
|
5793
|
+
}
|
|
5794
|
+
const isRetry = classification.kind === "retry";
|
|
5795
|
+
if (isRetry && rules.requireIdempotencyEvidenceForRetry && !hasIdempotencyEvidence(event)) {
|
|
5796
|
+
findings.push(
|
|
5797
|
+
fail2(
|
|
5798
|
+
"contract.retry.idempotency-evidence",
|
|
5799
|
+
`Retry of tool ${name} lacks idempotencyKey / noSideEffect evidence.`,
|
|
5800
|
+
[eventEvidence(event)],
|
|
5801
|
+
"idempotencyKey|noSideEffect",
|
|
5802
|
+
{ code: "AI_CHECK_RETRY_EVIDENCE_UNAVAILABLE" }
|
|
5803
|
+
)
|
|
5804
|
+
);
|
|
5805
|
+
}
|
|
5806
|
+
if (sawOk && nonIdempotent.has(name) && sawSideEffectOk && !hasIdempotencyEvidence(event)) {
|
|
5807
|
+
findings.push(
|
|
5808
|
+
fail2(
|
|
5809
|
+
"contract.retry.non-idempotent-side-effect",
|
|
5810
|
+
`Retry of non-idempotent tool ${name} after a confirmed ok side effect.`,
|
|
5811
|
+
[eventEvidence(event)],
|
|
5812
|
+
"no retry after side effect",
|
|
5813
|
+
name
|
|
5814
|
+
)
|
|
5815
|
+
);
|
|
5816
|
+
}
|
|
5817
|
+
if (event.status === "ok") {
|
|
5818
|
+
sawOk = true;
|
|
5819
|
+
if (nonIdempotent.has(name) && !hasIdempotencyEvidence(event)) {
|
|
5820
|
+
sawSideEffectOk = true;
|
|
5821
|
+
}
|
|
5822
|
+
}
|
|
5823
|
+
prior.push(event);
|
|
5824
|
+
}
|
|
5825
|
+
}
|
|
5826
|
+
}
|
|
5827
|
+
if (rules.requireRecoveredFailureVisible) {
|
|
5828
|
+
for (const [operationId, members] of byOperation) {
|
|
5829
|
+
const ordered = sortByTime(members);
|
|
5830
|
+
const toolish = ordered.filter(
|
|
5831
|
+
(event) => (event.kind === "TOOL" || event.kind === "LLM") && (event.status === "ok" || event.status === "error")
|
|
5832
|
+
);
|
|
5833
|
+
if (toolish.length < 2) continue;
|
|
5834
|
+
let sawEarlierError = false;
|
|
5835
|
+
let sawLaterOkAfterError = false;
|
|
5836
|
+
for (const event of toolish) {
|
|
5837
|
+
if (event.status === "error") {
|
|
5838
|
+
sawEarlierError = true;
|
|
5839
|
+
} else if (event.status === "ok" && sawEarlierError) {
|
|
5840
|
+
sawLaterOkAfterError = true;
|
|
5841
|
+
}
|
|
5842
|
+
}
|
|
5843
|
+
const hasOk = toolish.some((event) => event.status === "ok");
|
|
5844
|
+
const attemptish = toolish.some((event) => {
|
|
5845
|
+
const meta = workflowFor(event);
|
|
5846
|
+
return meta.attemptId !== void 0 || meta.attemptNumber !== void 0 || meta.attempt !== void 0 || meta.retryOf !== void 0;
|
|
5847
|
+
}) || toolish.length > 1;
|
|
5848
|
+
if (!attemptish || !hasOk) continue;
|
|
5849
|
+
if (!sawLaterOkAfterError) {
|
|
5850
|
+
findings.push(
|
|
5851
|
+
fail2(
|
|
5852
|
+
"contract.retry.recovered-failure-visible",
|
|
5853
|
+
`Operation ${operationId} does not show an earlier error attempt before a later success in the same chain.`,
|
|
5854
|
+
members.slice(0, 4).map(eventEvidence),
|
|
5855
|
+
"earlier error then later ok",
|
|
5856
|
+
{
|
|
5857
|
+
sawEarlierError,
|
|
5858
|
+
statuses: toolish.map((event) => event.status)
|
|
5859
|
+
}
|
|
5860
|
+
)
|
|
5861
|
+
);
|
|
5862
|
+
}
|
|
5863
|
+
}
|
|
5864
|
+
}
|
|
5865
|
+
return findings;
|
|
5866
|
+
}
|
|
5867
|
+
|
|
5868
|
+
// packages/core/src/checks/recovery-operations.ts
|
|
5869
|
+
function fail3(ruleId, message, evidence, expected, actual) {
|
|
5870
|
+
return {
|
|
5871
|
+
ruleId,
|
|
5872
|
+
severity: "error",
|
|
5873
|
+
status: "fail",
|
|
5874
|
+
message,
|
|
5875
|
+
...expected !== void 0 ? { expected } : {},
|
|
5876
|
+
...actual !== void 0 ? { actual } : {},
|
|
5877
|
+
evidence: [...evidence]
|
|
5878
|
+
};
|
|
5879
|
+
}
|
|
5880
|
+
function workflowFor2(event) {
|
|
5881
|
+
const attrs = event.attributes;
|
|
5882
|
+
if (!attrs || typeof attrs !== "object") return {};
|
|
5883
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
5884
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
5885
|
+
return { ...nested, ...direct };
|
|
5886
|
+
}
|
|
5887
|
+
function eventEvidence2(event) {
|
|
5888
|
+
return {
|
|
5889
|
+
runId: event.runId,
|
|
5890
|
+
eventId: event.eventId,
|
|
5891
|
+
kind: event.kind,
|
|
5892
|
+
name: event.name,
|
|
5893
|
+
status: event.status
|
|
5894
|
+
};
|
|
5895
|
+
}
|
|
5896
|
+
function eventTime2(event) {
|
|
5897
|
+
return event.startedAt ?? event.timestamp ?? "";
|
|
5898
|
+
}
|
|
5899
|
+
function sortByTime2(events) {
|
|
5900
|
+
return [...events].sort((a, b) => eventTime2(a).localeCompare(eventTime2(b)));
|
|
5901
|
+
}
|
|
5902
|
+
function errorCodeOf(event) {
|
|
5903
|
+
if (typeof event.error?.code === "string" && event.error.code.trim() !== "") {
|
|
5904
|
+
return event.error.code.trim();
|
|
5905
|
+
}
|
|
5906
|
+
const attrs = event.attributes ?? {};
|
|
5907
|
+
for (const key of ["errorCode", "code"]) {
|
|
5908
|
+
const value = attrs[key];
|
|
5909
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
5910
|
+
}
|
|
5911
|
+
const nested = attrs.error;
|
|
5912
|
+
if (nested && typeof nested === "object") {
|
|
5913
|
+
const code = nested.code;
|
|
5914
|
+
if (typeof code === "string" && code.trim() !== "") return code.trim();
|
|
5915
|
+
}
|
|
5916
|
+
return void 0;
|
|
5917
|
+
}
|
|
5918
|
+
function argumentDigestOf(event) {
|
|
5919
|
+
const attrs = event.attributes ?? {};
|
|
5920
|
+
for (const key of ["argumentsDigest", "inputDigest", "toolArgumentsDigest"]) {
|
|
5921
|
+
const value = attrs[key];
|
|
5922
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
5923
|
+
}
|
|
5924
|
+
const commitment = attrs.omittedPayloadCommitment;
|
|
5925
|
+
if (commitment && typeof commitment === "object") {
|
|
5926
|
+
const digest = commitment.digest;
|
|
5927
|
+
if (typeof digest === "string" && digest.trim() !== "") return digest.trim();
|
|
5928
|
+
}
|
|
5929
|
+
return void 0;
|
|
5930
|
+
}
|
|
5931
|
+
function argumentFingerprint(event) {
|
|
5932
|
+
const payload = extractToolArgumentPayload(event);
|
|
5933
|
+
if (payload.present) {
|
|
5934
|
+
const encoded = encodeCanonicalStructured(payload.value);
|
|
5935
|
+
if (!encoded.ok) return { kind: "missing" };
|
|
5936
|
+
return { kind: "structured", value: encoded.value };
|
|
5937
|
+
}
|
|
5938
|
+
const digest = argumentDigestOf(event);
|
|
5939
|
+
if (digest) return { kind: "digest", value: digest };
|
|
5940
|
+
return { kind: "missing" };
|
|
5941
|
+
}
|
|
5942
|
+
function sameArguments(left, right) {
|
|
5943
|
+
const a = argumentFingerprint(left);
|
|
5944
|
+
const b = argumentFingerprint(right);
|
|
5945
|
+
if (a.kind === "missing" || b.kind === "missing") {
|
|
5946
|
+
return {
|
|
5947
|
+
ok: false,
|
|
5948
|
+
reason: "Structured or digest argument evidence unavailable for same-arguments check.",
|
|
5949
|
+
code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_UNAVAILABLE"
|
|
5950
|
+
};
|
|
5951
|
+
}
|
|
5952
|
+
if (a.kind !== b.kind) {
|
|
5953
|
+
return {
|
|
5954
|
+
ok: false,
|
|
5955
|
+
reason: "Argument evidence kinds differ across attempts (structured vs digest).",
|
|
5956
|
+
code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_MISMATCH"
|
|
5957
|
+
};
|
|
5958
|
+
}
|
|
5959
|
+
if (a.value !== b.value) {
|
|
5960
|
+
return {
|
|
5961
|
+
ok: false,
|
|
5962
|
+
reason: "Retry arguments do not match prior attempt.",
|
|
5963
|
+
code: "AI_CHECK_RECOVERY_ARGUMENTS_DIFFER"
|
|
5964
|
+
};
|
|
5965
|
+
}
|
|
5966
|
+
return { ok: true };
|
|
5967
|
+
}
|
|
5968
|
+
function isUnevaluableWriteCompletion(event) {
|
|
5969
|
+
if (event.status === "running" || event.status === "unknown") return true;
|
|
5970
|
+
const attrs = event.attributes ?? {};
|
|
5971
|
+
if (attrs.timeout === true) return true;
|
|
5972
|
+
if (attrs.completionState === "unknown" || attrs.completionState === "timeout") return true;
|
|
5973
|
+
const code = errorCodeOf(event)?.toUpperCase();
|
|
5974
|
+
if (code === "TIMEOUT" || code === "UNKNOWN_COMPLETION" || code === "DEADLINE_EXCEEDED") {
|
|
5975
|
+
return true;
|
|
5976
|
+
}
|
|
5977
|
+
return false;
|
|
5978
|
+
}
|
|
5979
|
+
function collectExplicitReferences(event) {
|
|
5980
|
+
const refs = /* @__PURE__ */ new Set();
|
|
5981
|
+
const attrs = event.attributes ?? {};
|
|
5982
|
+
const candidates = [
|
|
5983
|
+
attrs.referencedEventId,
|
|
5984
|
+
attrs.referencedEventIds,
|
|
5985
|
+
attrs.toolResultEventId,
|
|
5986
|
+
attrs.toolResultEventIds,
|
|
5987
|
+
attrs.inputEventIds,
|
|
5988
|
+
attrs.dependsOnEventIds,
|
|
5989
|
+
attrs.evidence,
|
|
5990
|
+
workflowFor2(event).toolCallId
|
|
5991
|
+
];
|
|
5992
|
+
for (const candidate of candidates) {
|
|
5993
|
+
if (typeof candidate === "string" && candidate.trim() !== "") {
|
|
5994
|
+
refs.add(candidate.trim());
|
|
5995
|
+
continue;
|
|
5996
|
+
}
|
|
5997
|
+
if (Array.isArray(candidate)) {
|
|
5998
|
+
for (const item of candidate) {
|
|
5999
|
+
if (typeof item === "string" && item.trim() !== "") refs.add(item.trim());
|
|
6000
|
+
if (item && typeof item === "object") {
|
|
6001
|
+
const id = item.eventId;
|
|
6002
|
+
if (typeof id === "string" && id.trim() !== "") refs.add(id.trim());
|
|
6003
|
+
}
|
|
6004
|
+
}
|
|
6005
|
+
continue;
|
|
6006
|
+
}
|
|
6007
|
+
if (candidate && typeof candidate === "object") {
|
|
6008
|
+
const obj = candidate;
|
|
6009
|
+
if (typeof obj.eventId === "string" && obj.eventId.trim() !== "") {
|
|
6010
|
+
refs.add(obj.eventId.trim());
|
|
6011
|
+
}
|
|
6012
|
+
if (Array.isArray(obj.eventIds)) {
|
|
6013
|
+
for (const id of obj.eventIds) {
|
|
6014
|
+
if (typeof id === "string" && id.trim() !== "") refs.add(id.trim());
|
|
6015
|
+
}
|
|
6016
|
+
}
|
|
6017
|
+
}
|
|
6018
|
+
}
|
|
6019
|
+
return refs;
|
|
6020
|
+
}
|
|
6021
|
+
function llmReferencesTool(llm, toolEvent, requireExplicit) {
|
|
6022
|
+
if (!requireExplicit) {
|
|
6023
|
+
return eventTime2(llm) >= eventTime2(toolEvent);
|
|
6024
|
+
}
|
|
6025
|
+
const refs = collectExplicitReferences(llm);
|
|
6026
|
+
if (refs.has(toolEvent.eventId)) return true;
|
|
6027
|
+
const toolCallId = workflowFor2(toolEvent).toolCallId;
|
|
6028
|
+
if (toolCallId && refs.has(toolCallId)) return true;
|
|
6029
|
+
return false;
|
|
6030
|
+
}
|
|
6031
|
+
function attemptIdOf2(event) {
|
|
6032
|
+
const value = workflowFor2(event).attemptId;
|
|
6033
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
6034
|
+
}
|
|
6035
|
+
function hasDuplicateAttemptIds2(members) {
|
|
6036
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6037
|
+
for (const event of members) {
|
|
6038
|
+
const id = attemptIdOf2(event);
|
|
6039
|
+
if (!id) continue;
|
|
6040
|
+
if (seen.has(id)) return true;
|
|
6041
|
+
seen.add(id);
|
|
6042
|
+
}
|
|
6043
|
+
return false;
|
|
6044
|
+
}
|
|
6045
|
+
function findRetryTarget(members, retryOf) {
|
|
6046
|
+
return members.find(
|
|
6047
|
+
(candidate) => candidate.eventId === retryOf || attemptIdOf2(candidate) === retryOf || workflowFor2(candidate).operationId === retryOf
|
|
6048
|
+
);
|
|
6049
|
+
}
|
|
6050
|
+
function groupToolAttempts(events, toolName2) {
|
|
6051
|
+
const members = events.filter(
|
|
6052
|
+
(event) => event.kind === "TOOL" && resolveCanonicalToolName(event) === toolName2
|
|
6053
|
+
);
|
|
6054
|
+
const parent = /* @__PURE__ */ new Map();
|
|
6055
|
+
const ensure = (id) => {
|
|
6056
|
+
if (!parent.has(id)) parent.set(id, id);
|
|
6057
|
+
return id;
|
|
6058
|
+
};
|
|
6059
|
+
const find = (id) => {
|
|
6060
|
+
ensure(id);
|
|
6061
|
+
let cur = id;
|
|
6062
|
+
while (parent.get(cur) !== cur) {
|
|
6063
|
+
cur = parent.get(cur);
|
|
6064
|
+
}
|
|
6065
|
+
let walk = id;
|
|
6066
|
+
while (parent.get(walk) !== cur) {
|
|
6067
|
+
const next = parent.get(walk);
|
|
6068
|
+
parent.set(walk, cur);
|
|
6069
|
+
walk = next;
|
|
6070
|
+
}
|
|
6071
|
+
return cur;
|
|
6072
|
+
};
|
|
6073
|
+
const union = (a, b) => {
|
|
6074
|
+
const ra = find(a);
|
|
6075
|
+
const rb = find(b);
|
|
6076
|
+
if (ra !== rb) parent.set(ra, rb);
|
|
6077
|
+
};
|
|
6078
|
+
const nodeId = (event) => {
|
|
6079
|
+
const operationId = workflowFor2(event).operationId;
|
|
6080
|
+
if (typeof operationId === "string" && operationId.trim() !== "") {
|
|
6081
|
+
return `op:${operationId.trim()}`;
|
|
6082
|
+
}
|
|
6083
|
+
return `event:${event.eventId}`;
|
|
6084
|
+
};
|
|
6085
|
+
for (const event of members) {
|
|
6086
|
+
ensure(nodeId(event));
|
|
6087
|
+
}
|
|
6088
|
+
for (const event of members) {
|
|
6089
|
+
const retryOf = workflowFor2(event).retryOf;
|
|
6090
|
+
if (typeof retryOf !== "string" || retryOf.trim() === "") continue;
|
|
6091
|
+
const target = findRetryTarget(members, retryOf.trim());
|
|
6092
|
+
if (target) {
|
|
6093
|
+
union(nodeId(event), nodeId(target));
|
|
6094
|
+
}
|
|
6095
|
+
}
|
|
6096
|
+
const groups = /* @__PURE__ */ new Map();
|
|
6097
|
+
for (const event of members) {
|
|
6098
|
+
const key = find(nodeId(event));
|
|
6099
|
+
const list = groups.get(key) ?? [];
|
|
6100
|
+
list.push(event);
|
|
6101
|
+
groups.set(key, list);
|
|
6102
|
+
}
|
|
6103
|
+
return groups;
|
|
6104
|
+
}
|
|
6105
|
+
function latestAttempt(ordered) {
|
|
6106
|
+
if (ordered.length === 0) return void 0;
|
|
6107
|
+
return ordered[ordered.length - 1];
|
|
6108
|
+
}
|
|
6109
|
+
function evaluateRecoveryOperations(events, operations) {
|
|
6110
|
+
const findings = [];
|
|
6111
|
+
const llmEvents = events.filter((event) => event.kind === "LLM");
|
|
6112
|
+
for (const operation of operations) {
|
|
6113
|
+
const sideEffectClass = operation.sideEffectClass ?? "read";
|
|
6114
|
+
const groups = groupToolAttempts(events, operation.tool);
|
|
6115
|
+
if (groups.size === 0) {
|
|
6116
|
+
if (operation.requireTerminalSuccess) {
|
|
6117
|
+
findings.push(
|
|
6118
|
+
fail3(
|
|
6119
|
+
"contract.retry.operations.terminal-success",
|
|
6120
|
+
`Recovery operation for tool ${operation.tool} requires a terminal success but no attempts were observed.`,
|
|
6121
|
+
[],
|
|
6122
|
+
"ok",
|
|
6123
|
+
"missing"
|
|
6124
|
+
)
|
|
6125
|
+
);
|
|
6126
|
+
}
|
|
6127
|
+
continue;
|
|
6128
|
+
}
|
|
6129
|
+
for (const [operationKey, members] of groups) {
|
|
6130
|
+
const ordered = sortByTime2(members);
|
|
6131
|
+
if (hasDuplicateAttemptIds2(members)) {
|
|
6132
|
+
findings.push(
|
|
6133
|
+
fail3(
|
|
6134
|
+
"contract.retry.operations.duplicate-attempt-id",
|
|
6135
|
+
`Tool ${operation.tool} operation ${operationKey} has duplicate attemptId values.`,
|
|
6136
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
6137
|
+
"unique attemptId",
|
|
6138
|
+
"duplicate"
|
|
6139
|
+
)
|
|
6140
|
+
);
|
|
6141
|
+
}
|
|
6142
|
+
if (operation.maxAttempts !== void 0) {
|
|
6143
|
+
const count = countOperationAttempts(members);
|
|
6144
|
+
if (count > operation.maxAttempts) {
|
|
6145
|
+
findings.push(
|
|
6146
|
+
fail3(
|
|
6147
|
+
"contract.retry.operations.max-attempts",
|
|
6148
|
+
`Tool ${operation.tool} operation ${operationKey} exceeded maxAttempts ${operation.maxAttempts}.`,
|
|
6149
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
6150
|
+
operation.maxAttempts,
|
|
6151
|
+
count
|
|
6152
|
+
)
|
|
6153
|
+
);
|
|
6154
|
+
}
|
|
6155
|
+
}
|
|
6156
|
+
if (sideEffectClass === "write") {
|
|
6157
|
+
for (const event of ordered) {
|
|
6158
|
+
if (isUnevaluableWriteCompletion(event)) {
|
|
6159
|
+
findings.push(
|
|
6160
|
+
fail3(
|
|
6161
|
+
"contract.retry.operations.write-completion-unevaluable",
|
|
6162
|
+
`Write tool ${operation.tool} has timeout/unknown/running completion that cannot be verified as an authoritative write result.`,
|
|
6163
|
+
[eventEvidence2(event)],
|
|
6164
|
+
"authoritative write completion observation",
|
|
6165
|
+
{
|
|
6166
|
+
code: "AI_CHECK_RECOVERY_WRITE_COMPLETION_UNAVAILABLE",
|
|
6167
|
+
status: event.status
|
|
6168
|
+
}
|
|
6169
|
+
)
|
|
6170
|
+
);
|
|
6171
|
+
}
|
|
6172
|
+
}
|
|
6173
|
+
}
|
|
6174
|
+
const prior = [];
|
|
6175
|
+
let sawEarlierError = false;
|
|
6176
|
+
let sawLaterOkAfterError = false;
|
|
6177
|
+
let lastOk;
|
|
6178
|
+
let identityUnavailable = false;
|
|
6179
|
+
for (const event of ordered) {
|
|
6180
|
+
const classification = classifyRetryAttempt(event, prior, members);
|
|
6181
|
+
if (classification.kind === "unknown") {
|
|
6182
|
+
identityUnavailable = true;
|
|
6183
|
+
findings.push(
|
|
6184
|
+
fail3(
|
|
6185
|
+
"contract.retry.operations.attempt-identity",
|
|
6186
|
+
`Tool ${operation.tool} operation ${operationKey} has ambiguous attempt identity (${classification.reason}).`,
|
|
6187
|
+
[eventEvidence2(event)],
|
|
6188
|
+
"explicit operationId/attemptId/retryOf chronology",
|
|
6189
|
+
{ code: "AI_CHECK_RECOVERY_ATTEMPT_IDENTITY_UNAVAILABLE", reason: classification.reason }
|
|
6190
|
+
)
|
|
6191
|
+
);
|
|
6192
|
+
}
|
|
6193
|
+
const isRetry = classification.kind === "retry";
|
|
6194
|
+
if (isRetry && operation.requireFailureBeforeRetry) {
|
|
6195
|
+
const earlierFailure = prior.some(
|
|
6196
|
+
(candidate) => candidate.status === "error" && eventTime2(candidate) !== "" && eventTime2(event) !== "" && eventTime2(candidate) < eventTime2(event)
|
|
6197
|
+
);
|
|
6198
|
+
if (!earlierFailure) {
|
|
6199
|
+
findings.push(
|
|
6200
|
+
fail3(
|
|
6201
|
+
"contract.retry.operations.failure-before-retry",
|
|
6202
|
+
`Retry of tool ${operation.tool} appeared without an earlier failure in the same operation.`,
|
|
6203
|
+
[eventEvidence2(event)],
|
|
6204
|
+
"earlier error attempt",
|
|
6205
|
+
{ operationKey }
|
|
6206
|
+
)
|
|
6207
|
+
);
|
|
6208
|
+
}
|
|
6209
|
+
}
|
|
6210
|
+
if (isRetry && operation.retryableErrors?.codes && operation.retryableErrors.codes.length > 0) {
|
|
6211
|
+
const allowed = new Set(operation.retryableErrors.codes);
|
|
6212
|
+
const priorErrors = prior.filter((candidate) => candidate.status === "error");
|
|
6213
|
+
const lastError = priorErrors[priorErrors.length - 1];
|
|
6214
|
+
if (lastError) {
|
|
6215
|
+
const code = errorCodeOf(lastError);
|
|
6216
|
+
if (!code || !allowed.has(code)) {
|
|
6217
|
+
findings.push(
|
|
6218
|
+
fail3(
|
|
6219
|
+
"contract.retry.operations.retryable-error",
|
|
6220
|
+
`Retry of tool ${operation.tool} followed a non-retryable or missing error code.`,
|
|
6221
|
+
[eventEvidence2(event), eventEvidence2(lastError)],
|
|
6222
|
+
[...allowed],
|
|
6223
|
+
code ?? null
|
|
6224
|
+
)
|
|
6225
|
+
);
|
|
6226
|
+
}
|
|
6227
|
+
} else {
|
|
6228
|
+
findings.push(
|
|
6229
|
+
fail3(
|
|
6230
|
+
"contract.retry.operations.retryable-error",
|
|
6231
|
+
`Retry of tool ${operation.tool} has no prior error code evidence.`,
|
|
6232
|
+
[eventEvidence2(event)],
|
|
6233
|
+
[...allowed],
|
|
6234
|
+
null
|
|
6235
|
+
)
|
|
6236
|
+
);
|
|
6237
|
+
}
|
|
6238
|
+
}
|
|
6239
|
+
if (isRetry && (operation.requireSameArguments === true || operation.requireSameArguments === "structured-or-digest")) {
|
|
6240
|
+
const baseline = prior[0] ?? prior[prior.length - 1];
|
|
6241
|
+
if (baseline) {
|
|
6242
|
+
const comparison = sameArguments(baseline, event);
|
|
6243
|
+
if (!comparison.ok) {
|
|
6244
|
+
findings.push(
|
|
6245
|
+
fail3(
|
|
6246
|
+
"contract.retry.operations.same-arguments",
|
|
6247
|
+
comparison.reason,
|
|
6248
|
+
[eventEvidence2(baseline), eventEvidence2(event)],
|
|
6249
|
+
"matching structured args or digests",
|
|
6250
|
+
{ code: comparison.code }
|
|
6251
|
+
)
|
|
6252
|
+
);
|
|
6253
|
+
}
|
|
6254
|
+
} else {
|
|
6255
|
+
findings.push(
|
|
6256
|
+
fail3(
|
|
6257
|
+
"contract.retry.operations.same-arguments",
|
|
6258
|
+
"Structured or digest argument evidence unavailable for same-arguments check.",
|
|
6259
|
+
[eventEvidence2(event)],
|
|
6260
|
+
"matching structured args or digests",
|
|
6261
|
+
{ code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_UNAVAILABLE" }
|
|
6262
|
+
)
|
|
6263
|
+
);
|
|
6264
|
+
}
|
|
6265
|
+
}
|
|
6266
|
+
if (event.status === "error") {
|
|
6267
|
+
sawEarlierError = true;
|
|
6268
|
+
} else if (event.status === "ok" && sawEarlierError) {
|
|
6269
|
+
sawLaterOkAfterError = true;
|
|
6270
|
+
}
|
|
6271
|
+
if (event.status === "ok") {
|
|
6272
|
+
lastOk = event;
|
|
6273
|
+
}
|
|
6274
|
+
prior.push(event);
|
|
6275
|
+
}
|
|
6276
|
+
const latest = latestAttempt(ordered);
|
|
6277
|
+
if (operation.requireTerminalSuccess) {
|
|
6278
|
+
if (!latest || latest.status !== "ok") {
|
|
6279
|
+
findings.push(
|
|
6280
|
+
fail3(
|
|
6281
|
+
"contract.retry.operations.terminal-success",
|
|
6282
|
+
`Tool ${operation.tool} operation ${operationKey} does not terminate with ok on the latest attempt.`,
|
|
6283
|
+
ordered.slice(0, 4).map(eventEvidence2),
|
|
6284
|
+
"latest attempt ok",
|
|
6285
|
+
latest?.status ?? "missing"
|
|
6286
|
+
)
|
|
6287
|
+
);
|
|
6288
|
+
}
|
|
6289
|
+
}
|
|
6290
|
+
if (operation.requireRecoveredFailureVisible) {
|
|
6291
|
+
const latestOk = latest?.status === "ok";
|
|
6292
|
+
if (ordered.length > 1 && latestOk && !sawLaterOkAfterError) {
|
|
6293
|
+
findings.push(
|
|
6294
|
+
fail3(
|
|
6295
|
+
"contract.retry.operations.recovered-failure-visible",
|
|
6296
|
+
`Tool ${operation.tool} operation ${operationKey} does not show an earlier error before later success.`,
|
|
6297
|
+
ordered.slice(0, 4).map(eventEvidence2),
|
|
6298
|
+
"earlier error then later ok",
|
|
6299
|
+
{ sawEarlierError }
|
|
6300
|
+
)
|
|
6301
|
+
);
|
|
6302
|
+
}
|
|
6303
|
+
}
|
|
6304
|
+
const dependency = operation.successfulResultDependency;
|
|
6305
|
+
if (dependency?.consumerKind === "LLM") {
|
|
6306
|
+
const successForDependency = latest?.status === "ok" ? latest : identityUnavailable ? void 0 : lastOk;
|
|
6307
|
+
if (successForDependency) {
|
|
6308
|
+
if (sideEffectClass === "write" && isUnevaluableWriteCompletion(successForDependency)) {
|
|
6309
|
+
findings.push(
|
|
6310
|
+
fail3(
|
|
6311
|
+
"contract.retry.operations.successful-result-dependency",
|
|
6312
|
+
`Successful ${operation.tool} result dependency is unevaluable because write completion evidence is unavailable.`,
|
|
6313
|
+
[eventEvidence2(successForDependency)],
|
|
6314
|
+
"LLM reference to successful tool event",
|
|
6315
|
+
{ code: "AI_CHECK_RECOVERY_WRITE_COMPLETION_UNAVAILABLE" }
|
|
6316
|
+
)
|
|
6317
|
+
);
|
|
6318
|
+
} else {
|
|
6319
|
+
const requireExplicit = dependency.requireExplicitReference === true;
|
|
6320
|
+
const referenced = llmEvents.some(
|
|
6321
|
+
(llmEvent) => llmReferencesTool(llmEvent, successForDependency, requireExplicit)
|
|
6322
|
+
);
|
|
6323
|
+
if (!referenced) {
|
|
6324
|
+
findings.push(
|
|
6325
|
+
fail3(
|
|
6326
|
+
"contract.retry.operations.successful-result-dependency",
|
|
6327
|
+
requireExplicit ? `Successful ${operation.tool} result is not explicitly referenced by a later LLM event.` : `Successful ${operation.tool} result has no later LLM consumer.`,
|
|
6328
|
+
[eventEvidence2(successForDependency)],
|
|
6329
|
+
"LLM reference to successful tool event",
|
|
6330
|
+
{ code: "AI_CHECK_RECOVERY_RESULT_DEPENDENCY_MISSING" }
|
|
6331
|
+
)
|
|
6332
|
+
);
|
|
6333
|
+
}
|
|
6334
|
+
}
|
|
6335
|
+
}
|
|
6336
|
+
}
|
|
6337
|
+
}
|
|
6338
|
+
}
|
|
6339
|
+
return findings;
|
|
6340
|
+
}
|
|
6341
|
+
|
|
5007
6342
|
// packages/core/src/checks/contract.ts
|
|
6343
|
+
function contractFailFinding(ruleId, message, evidence, expected, actual) {
|
|
6344
|
+
return {
|
|
6345
|
+
ruleId,
|
|
6346
|
+
severity: "error",
|
|
6347
|
+
status: "fail",
|
|
6348
|
+
message,
|
|
6349
|
+
...expected !== void 0 ? { expected } : {},
|
|
6350
|
+
...actual !== void 0 ? { actual } : {},
|
|
6351
|
+
evidence: [...evidence]
|
|
6352
|
+
};
|
|
6353
|
+
}
|
|
6354
|
+
function normalizeStatus(status) {
|
|
6355
|
+
if (status === "ok" || status === "error" || status === "running") return status;
|
|
6356
|
+
if (status === "success") return "ok";
|
|
6357
|
+
if (status === "failed") return "error";
|
|
6358
|
+
return "error";
|
|
6359
|
+
}
|
|
5008
6360
|
function cloneBody(body) {
|
|
5009
6361
|
return {
|
|
5010
6362
|
...body.run ? { run: { ...body.run } } : {},
|
|
@@ -5016,6 +6368,17 @@ function cloneBody(body) {
|
|
|
5016
6368
|
}
|
|
5017
6369
|
} : {},
|
|
5018
6370
|
...body.llm ? { llm: { ...body.llm } } : {},
|
|
6371
|
+
...body.steps ? {
|
|
6372
|
+
steps: {
|
|
6373
|
+
...body.steps.orderRelations ? {
|
|
6374
|
+
orderRelations: body.steps.orderRelations.map((item) => ({
|
|
6375
|
+
...item,
|
|
6376
|
+
before: { ...item.before },
|
|
6377
|
+
after: { ...item.after }
|
|
6378
|
+
}))
|
|
6379
|
+
} : {}
|
|
6380
|
+
}
|
|
6381
|
+
} : {},
|
|
5019
6382
|
...body.observations ? {
|
|
5020
6383
|
observations: {
|
|
5021
6384
|
...body.observations,
|
|
@@ -5029,35 +6392,483 @@ function cloneBody(body) {
|
|
|
5029
6392
|
...body.controls.enforcedTools ? { enforcedTools: [...body.controls.enforcedTools] } : {},
|
|
5030
6393
|
...body.controls.requiredStages ? { requiredStages: body.controls.requiredStages.map((item) => ({ ...item })) } : {}
|
|
5031
6394
|
}
|
|
5032
|
-
} : {},
|
|
5033
|
-
...body.retry ? {
|
|
5034
|
-
retry: {
|
|
5035
|
-
...body.retry,
|
|
5036
|
-
...body.retry.nonIdempotentTools ? { nonIdempotentTools: [...body.retry.nonIdempotentTools] } : {},
|
|
5037
|
-
...body.retry.operations ? {
|
|
5038
|
-
operations: body.retry.operations.map((operation) => ({
|
|
5039
|
-
...operation,
|
|
5040
|
-
...operation.retryableErrors ? {
|
|
5041
|
-
retryableErrors: {
|
|
5042
|
-
...operation.retryableErrors,
|
|
5043
|
-
...operation.retryableErrors.codes ? { codes: [...operation.retryableErrors.codes] } : {}
|
|
5044
|
-
}
|
|
5045
|
-
} : {},
|
|
5046
|
-
...operation.successfulResultDependency ? {
|
|
5047
|
-
successfulResultDependency: {
|
|
5048
|
-
...operation.successfulResultDependency
|
|
5049
|
-
}
|
|
5050
|
-
} : {}
|
|
5051
|
-
}))
|
|
5052
|
-
} : {}
|
|
6395
|
+
} : {},
|
|
6396
|
+
...body.retry ? {
|
|
6397
|
+
retry: {
|
|
6398
|
+
...body.retry,
|
|
6399
|
+
...body.retry.nonIdempotentTools ? { nonIdempotentTools: [...body.retry.nonIdempotentTools] } : {},
|
|
6400
|
+
...body.retry.operations ? {
|
|
6401
|
+
operations: body.retry.operations.map((operation) => ({
|
|
6402
|
+
...operation,
|
|
6403
|
+
...operation.retryableErrors ? {
|
|
6404
|
+
retryableErrors: {
|
|
6405
|
+
...operation.retryableErrors,
|
|
6406
|
+
...operation.retryableErrors.codes ? { codes: [...operation.retryableErrors.codes] } : {}
|
|
6407
|
+
}
|
|
6408
|
+
} : {},
|
|
6409
|
+
...operation.successfulResultDependency ? {
|
|
6410
|
+
successfulResultDependency: {
|
|
6411
|
+
...operation.successfulResultDependency
|
|
6412
|
+
}
|
|
6413
|
+
} : {}
|
|
6414
|
+
}))
|
|
6415
|
+
} : {}
|
|
6416
|
+
}
|
|
6417
|
+
} : {}
|
|
6418
|
+
};
|
|
6419
|
+
}
|
|
6420
|
+
function bodyHasRules(body) {
|
|
6421
|
+
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.steps !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
6422
|
+
}
|
|
6423
|
+
var MAX_EVIDENCE_EVENT_IDS = 16;
|
|
6424
|
+
var METHOD_VOCABULARY = new Set(OBSERVED_OUTCOME_METHODS);
|
|
6425
|
+
function parseEvidenceEventIds(evidence) {
|
|
6426
|
+
if (evidence === void 0 || evidence === null) {
|
|
6427
|
+
return { status: "missing" };
|
|
6428
|
+
}
|
|
6429
|
+
if (typeof evidence === "string") {
|
|
6430
|
+
const trimmed = evidence.trim();
|
|
6431
|
+
if (trimmed === "") {
|
|
6432
|
+
return { status: "invalid", reason: "empty evidence string" };
|
|
6433
|
+
}
|
|
6434
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
6435
|
+
}
|
|
6436
|
+
if (typeof evidence !== "object" || Array.isArray(evidence)) {
|
|
6437
|
+
return {
|
|
6438
|
+
status: "invalid",
|
|
6439
|
+
reason: "evidence must be a string event id or { eventId } / { eventIds }"
|
|
6440
|
+
};
|
|
6441
|
+
}
|
|
6442
|
+
const record = evidence;
|
|
6443
|
+
if (typeof record.eventId === "string") {
|
|
6444
|
+
const trimmed = record.eventId.trim();
|
|
6445
|
+
if (trimmed === "") {
|
|
6446
|
+
return { status: "invalid", reason: "empty evidence.eventId" };
|
|
6447
|
+
}
|
|
6448
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
6449
|
+
}
|
|
6450
|
+
if (Array.isArray(record.eventIds)) {
|
|
6451
|
+
if (record.eventIds.length === 0) {
|
|
6452
|
+
return { status: "invalid", reason: "empty evidence.eventIds" };
|
|
6453
|
+
}
|
|
6454
|
+
if (record.eventIds.length > MAX_EVIDENCE_EVENT_IDS) {
|
|
6455
|
+
return {
|
|
6456
|
+
status: "invalid",
|
|
6457
|
+
reason: `evidence.eventIds exceeds bound of ${MAX_EVIDENCE_EVENT_IDS}`
|
|
6458
|
+
};
|
|
6459
|
+
}
|
|
6460
|
+
const eventIds = [];
|
|
6461
|
+
for (const item of record.eventIds) {
|
|
6462
|
+
if (typeof item !== "string" || item.trim() === "") {
|
|
6463
|
+
return { status: "invalid", reason: "evidence.eventIds contains a non-string id" };
|
|
6464
|
+
}
|
|
6465
|
+
eventIds.push(item.trim());
|
|
6466
|
+
}
|
|
6467
|
+
return { status: "ok", eventIds };
|
|
6468
|
+
}
|
|
6469
|
+
return {
|
|
6470
|
+
status: "invalid",
|
|
6471
|
+
reason: "evidence must include eventId or eventIds"
|
|
6472
|
+
};
|
|
6473
|
+
}
|
|
6474
|
+
function provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds) {
|
|
6475
|
+
const findings = [];
|
|
6476
|
+
const evidenceBase = [
|
|
6477
|
+
{
|
|
6478
|
+
runId: outcome.runId,
|
|
6479
|
+
eventId: outcome.outcomeId,
|
|
6480
|
+
kind: "OUTCOME",
|
|
6481
|
+
name: outcome.name,
|
|
6482
|
+
path: `outcome.${outcome.name}`
|
|
6483
|
+
}
|
|
6484
|
+
];
|
|
6485
|
+
if (requireProvenance.method) {
|
|
6486
|
+
if (outcome.method === void 0 || outcome.method.trim() === "") {
|
|
6487
|
+
findings.push(
|
|
6488
|
+
contractFailFinding(
|
|
6489
|
+
"contract.observation.provenance.method",
|
|
6490
|
+
`Observation ${outcome.name} is missing a structural method.`,
|
|
6491
|
+
evidenceBase,
|
|
6492
|
+
{ method: true },
|
|
6493
|
+
{ name: outcome.name, method: outcome.method ?? null }
|
|
6494
|
+
)
|
|
6495
|
+
);
|
|
6496
|
+
} else if (!METHOD_VOCABULARY.has(outcome.method)) {
|
|
6497
|
+
findings.push(
|
|
6498
|
+
contractFailFinding(
|
|
6499
|
+
"contract.observation.provenance.method",
|
|
6500
|
+
`Observation ${outcome.name} method ${outcome.method} is outside the bounded ObservedOutcomeMethod vocabulary.`,
|
|
6501
|
+
evidenceBase,
|
|
6502
|
+
{ methodVocabulary: [...OBSERVED_OUTCOME_METHODS] },
|
|
6503
|
+
{ name: outcome.name, method: outcome.method }
|
|
6504
|
+
)
|
|
6505
|
+
);
|
|
6506
|
+
}
|
|
6507
|
+
}
|
|
6508
|
+
if (requireProvenance.evidence || requireProvenance.sameRunEventReference) {
|
|
6509
|
+
const parsed = parseEvidenceEventIds(outcome.evidence);
|
|
6510
|
+
if (parsed.status === "missing") {
|
|
6511
|
+
findings.push(
|
|
6512
|
+
contractFailFinding(
|
|
6513
|
+
"contract.observation.provenance.evidence",
|
|
6514
|
+
`Observation ${outcome.name} is missing bounded evidence references.`,
|
|
6515
|
+
evidenceBase,
|
|
6516
|
+
{ evidence: true },
|
|
6517
|
+
{ name: outcome.name, evidence: null }
|
|
6518
|
+
)
|
|
6519
|
+
);
|
|
6520
|
+
} else if (parsed.status === "invalid") {
|
|
6521
|
+
findings.push(
|
|
6522
|
+
contractFailFinding(
|
|
6523
|
+
"contract.observation.provenance.evidence",
|
|
6524
|
+
`Observation ${outcome.name} evidence is not a bounded event reference (${parsed.reason}).`,
|
|
6525
|
+
evidenceBase,
|
|
6526
|
+
{ evidence: true },
|
|
6527
|
+
{ name: outcome.name, reason: parsed.reason }
|
|
6528
|
+
)
|
|
6529
|
+
);
|
|
6530
|
+
} else if (requireProvenance.sameRunEventReference) {
|
|
6531
|
+
const missing = parsed.eventIds.filter((eventId) => !knownEventIds.has(eventId));
|
|
6532
|
+
if (missing.length > 0) {
|
|
6533
|
+
findings.push(
|
|
6534
|
+
contractFailFinding(
|
|
6535
|
+
"contract.observation.provenance.same-run",
|
|
6536
|
+
`Observation ${outcome.name} references event ids missing from the same run: ${missing.join(", ")}.`,
|
|
6537
|
+
evidenceBase,
|
|
6538
|
+
{ sameRunEventReference: true, eventIds: parsed.eventIds },
|
|
6539
|
+
{ name: outcome.name, missing }
|
|
6540
|
+
)
|
|
6541
|
+
);
|
|
6542
|
+
}
|
|
6543
|
+
}
|
|
6544
|
+
}
|
|
6545
|
+
return findings;
|
|
6546
|
+
}
|
|
6547
|
+
function contractToRules(contract) {
|
|
6548
|
+
const rules = [];
|
|
6549
|
+
const allowedStatuses = contract.run?.allowedStatuses ?? [];
|
|
6550
|
+
if (allowedStatuses.length === 1) {
|
|
6551
|
+
rules.push(
|
|
6552
|
+
createRunStatusRule({
|
|
6553
|
+
expected: normalizeStatus(allowedStatuses[0]),
|
|
6554
|
+
allowIncomplete: contract.run?.requireCompleted === false
|
|
6555
|
+
})
|
|
6556
|
+
);
|
|
6557
|
+
} else if (allowedStatuses.length > 1) {
|
|
6558
|
+
const expected = [...new Set(allowedStatuses.map(normalizeStatus))];
|
|
6559
|
+
const allowIncomplete = contract.run?.requireCompleted === false;
|
|
6560
|
+
rules.push({
|
|
6561
|
+
id: "contract.run.allowedStatuses",
|
|
6562
|
+
category: "run",
|
|
6563
|
+
defaultSeverity: "error",
|
|
6564
|
+
evaluate(context) {
|
|
6565
|
+
const findings = [];
|
|
6566
|
+
const actual = context.selectedRun?.status ?? "unknown";
|
|
6567
|
+
if (!expected.includes(actual)) {
|
|
6568
|
+
findings.push(
|
|
6569
|
+
contractFailFinding(
|
|
6570
|
+
"contract.run.allowedStatuses",
|
|
6571
|
+
`Run status ${actual} is not one of the allowed statuses: ${expected.join(", ")}.`,
|
|
6572
|
+
context.selectedRun ? [
|
|
6573
|
+
{
|
|
6574
|
+
runId: context.selectedRun.runId,
|
|
6575
|
+
kind: "RUN",
|
|
6576
|
+
name: context.selectedRun.name,
|
|
6577
|
+
status: context.selectedRun.status
|
|
6578
|
+
}
|
|
6579
|
+
] : [],
|
|
6580
|
+
expected,
|
|
6581
|
+
actual
|
|
6582
|
+
)
|
|
6583
|
+
);
|
|
6584
|
+
}
|
|
6585
|
+
if (!allowIncomplete) {
|
|
6586
|
+
const running = (context.logicalEvents ?? context.events).filter(
|
|
6587
|
+
(event) => event.status === "running"
|
|
6588
|
+
);
|
|
6589
|
+
if (running.length > 0) {
|
|
6590
|
+
findings.push(
|
|
6591
|
+
contractFailFinding(
|
|
6592
|
+
"contract.run.allowedStatuses",
|
|
6593
|
+
"Run contains incomplete running events.",
|
|
6594
|
+
running.map((event) => ({
|
|
6595
|
+
runId: event.runId,
|
|
6596
|
+
eventId: event.eventId,
|
|
6597
|
+
kind: event.kind,
|
|
6598
|
+
name: event.name,
|
|
6599
|
+
status: event.status
|
|
6600
|
+
})),
|
|
6601
|
+
"no running events",
|
|
6602
|
+
running.length
|
|
6603
|
+
)
|
|
6604
|
+
);
|
|
6605
|
+
}
|
|
6606
|
+
}
|
|
6607
|
+
return findings;
|
|
6608
|
+
}
|
|
6609
|
+
});
|
|
6610
|
+
} else if (contract.run?.requireCompleted !== false) {
|
|
6611
|
+
rules.push(createRunStatusRule({ allowIncomplete: false }));
|
|
6612
|
+
}
|
|
6613
|
+
if (contract.run?.maxDurationMs !== void 0) {
|
|
6614
|
+
rules.push(createRunDurationRule({ maxDurationMs: contract.run.maxDurationMs }));
|
|
6615
|
+
}
|
|
6616
|
+
if (contract.run?.requireCompleted === true) {
|
|
6617
|
+
rules.push(createStructureIncompleteRule({ requireEndedAtForStarted: true }));
|
|
6618
|
+
}
|
|
6619
|
+
if (contract.tools) {
|
|
6620
|
+
const order = contract.tools.requiredOrder ?? [];
|
|
6621
|
+
const requiredOrderMode = contract.tools.requiredOrderMode ?? "first-occurrence";
|
|
6622
|
+
const orderRules = contract.tools.orderRules ?? [];
|
|
6623
|
+
const endpointRequired = orderRules.filter((rule) => rule.requireEndpoints !== false).flatMap((rule) => [rule.before, rule.after]);
|
|
6624
|
+
const required = [
|
|
6625
|
+
.../* @__PURE__ */ new Set([
|
|
6626
|
+
...contract.tools.required ?? [],
|
|
6627
|
+
...contract.tools.requiredTools ?? [],
|
|
6628
|
+
...order,
|
|
6629
|
+
...endpointRequired
|
|
6630
|
+
])
|
|
6631
|
+
];
|
|
6632
|
+
const forbidden = [
|
|
6633
|
+
...contract.tools.forbidden ?? [],
|
|
6634
|
+
...contract.tools.forbiddenTools ?? []
|
|
6635
|
+
];
|
|
6636
|
+
rules.push(
|
|
6637
|
+
createToolUsageRule({
|
|
6638
|
+
...required.length > 0 ? { required } : {},
|
|
6639
|
+
...forbidden.length > 0 ? { forbidden } : {},
|
|
6640
|
+
...contract.tools.allowed ? { allowed: contract.tools.allowed } : {},
|
|
6641
|
+
...contract.tools.maxCalls !== void 0 ? { maxCount: contract.tools.maxCalls } : {}
|
|
6642
|
+
})
|
|
6643
|
+
);
|
|
6644
|
+
for (let i = 0; i < order.length - 1; i += 1) {
|
|
6645
|
+
rules.push(
|
|
6646
|
+
createToolOrderingRule({
|
|
6647
|
+
before: order[i],
|
|
6648
|
+
after: order[i + 1],
|
|
6649
|
+
id: `contract.tool.order.${i}`,
|
|
6650
|
+
mode: requiredOrderMode
|
|
6651
|
+
})
|
|
6652
|
+
);
|
|
6653
|
+
}
|
|
6654
|
+
const defaultOccurrenceMode = contract.tools.defaultOccurrenceMode ?? requiredOrderMode;
|
|
6655
|
+
for (const [index, rule] of orderRules.entries()) {
|
|
6656
|
+
rules.push(
|
|
6657
|
+
createToolOrderingRule({
|
|
6658
|
+
before: rule.before,
|
|
6659
|
+
after: rule.after,
|
|
6660
|
+
id: `contract.tool.orderRule.${index}`,
|
|
6661
|
+
mode: rule.occurrenceMode ?? defaultOccurrenceMode
|
|
6662
|
+
})
|
|
6663
|
+
);
|
|
6664
|
+
}
|
|
6665
|
+
const argumentChecks = contract.tools.arguments ?? [];
|
|
6666
|
+
if (argumentChecks.length > 0) {
|
|
6667
|
+
rules.push({
|
|
6668
|
+
id: "contract.tool.arguments",
|
|
6669
|
+
category: "tool",
|
|
6670
|
+
defaultSeverity: "error",
|
|
6671
|
+
evaluate(context) {
|
|
6672
|
+
const tools = (context.logicalEvents ?? context.events).filter(
|
|
6673
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
6674
|
+
);
|
|
6675
|
+
const findings = [];
|
|
6676
|
+
for (const [index, check] of argumentChecks.entries()) {
|
|
6677
|
+
const matches = tools.filter(
|
|
6678
|
+
(event) => resolveCanonicalToolName(event) === check.tool
|
|
6679
|
+
);
|
|
6680
|
+
if (matches.length === 0) {
|
|
6681
|
+
findings.push(
|
|
6682
|
+
contractFailFinding(
|
|
6683
|
+
`contract.tool.arguments.${index}`,
|
|
6684
|
+
`No finished tool named ${check.tool} for argument check.`,
|
|
6685
|
+
context.selectedRun ? [
|
|
6686
|
+
{
|
|
6687
|
+
runId: context.selectedRun.runId,
|
|
6688
|
+
kind: "RUN",
|
|
6689
|
+
name: context.selectedRun.name
|
|
6690
|
+
}
|
|
6691
|
+
] : [],
|
|
6692
|
+
{ tool: check.tool, path: check.path },
|
|
6693
|
+
{ toolCount: 0 }
|
|
6694
|
+
)
|
|
6695
|
+
);
|
|
6696
|
+
continue;
|
|
6697
|
+
}
|
|
6698
|
+
const occurrence = check.occurrence ?? "all";
|
|
6699
|
+
const selected = occurrence === "first" ? [matches[0]] : occurrence === "last" ? [matches[matches.length - 1]] : matches;
|
|
6700
|
+
const results = selected.map((event) => {
|
|
6701
|
+
const payload = extractToolArgumentPayload(event);
|
|
6702
|
+
return evaluateToolArgumentValue(
|
|
6703
|
+
payload.value,
|
|
6704
|
+
check,
|
|
6705
|
+
payload.present
|
|
6706
|
+
);
|
|
6707
|
+
});
|
|
6708
|
+
if (occurrence === "any") {
|
|
6709
|
+
if (results.some((result) => result.status === "pass")) continue;
|
|
6710
|
+
const firstFail = results.find((result) => result.status !== "pass");
|
|
6711
|
+
findings.push(
|
|
6712
|
+
contractFailFinding(
|
|
6713
|
+
`contract.tool.arguments.${index}`,
|
|
6714
|
+
firstFail.message,
|
|
6715
|
+
selected.slice(0, 1).map((event) => ({
|
|
6716
|
+
runId: event.runId,
|
|
6717
|
+
eventId: event.eventId,
|
|
6718
|
+
kind: event.kind,
|
|
6719
|
+
name: event.name,
|
|
6720
|
+
path: `tool.${check.tool}${check.path}`
|
|
6721
|
+
})),
|
|
6722
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
6723
|
+
{ code: firstFail.code }
|
|
6724
|
+
)
|
|
6725
|
+
);
|
|
6726
|
+
continue;
|
|
6727
|
+
}
|
|
6728
|
+
for (const [selIndex, result] of results.entries()) {
|
|
6729
|
+
if (result.status === "pass") continue;
|
|
6730
|
+
const event = selected[selIndex];
|
|
6731
|
+
findings.push(
|
|
6732
|
+
contractFailFinding(
|
|
6733
|
+
`contract.tool.arguments.${index}`,
|
|
6734
|
+
result.message,
|
|
6735
|
+
[
|
|
6736
|
+
{
|
|
6737
|
+
runId: event.runId,
|
|
6738
|
+
eventId: event.eventId,
|
|
6739
|
+
kind: event.kind,
|
|
6740
|
+
name: event.name,
|
|
6741
|
+
path: `tool.${check.tool}${check.path}`
|
|
6742
|
+
}
|
|
6743
|
+
],
|
|
6744
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
6745
|
+
{ code: result.code }
|
|
6746
|
+
)
|
|
6747
|
+
);
|
|
6748
|
+
}
|
|
6749
|
+
}
|
|
6750
|
+
return findings;
|
|
6751
|
+
}
|
|
6752
|
+
});
|
|
6753
|
+
}
|
|
6754
|
+
}
|
|
6755
|
+
if (contract.llm) {
|
|
6756
|
+
rules.push(
|
|
6757
|
+
createLlmUsageRule({
|
|
6758
|
+
...contract.llm.maxCalls !== void 0 ? { maxCalls: contract.llm.maxCalls } : {},
|
|
6759
|
+
...contract.llm.maxTotalTokens !== void 0 ? { maxTotalTokens: contract.llm.maxTotalTokens } : {},
|
|
6760
|
+
...contract.llm.allowedModels ? { allowedModels: contract.llm.allowedModels } : {}
|
|
6761
|
+
})
|
|
6762
|
+
);
|
|
6763
|
+
}
|
|
6764
|
+
const orderRelations = contract.steps?.orderRelations ?? [];
|
|
6765
|
+
for (const [index, relation] of orderRelations.entries()) {
|
|
6766
|
+
rules.push(
|
|
6767
|
+
createStepOrderingRule({
|
|
6768
|
+
before: { kind: relation.before.kind, name: relation.before.name },
|
|
6769
|
+
after: { kind: relation.after.kind, name: relation.after.name },
|
|
6770
|
+
id: `contract.step.orderRelation.${index}`,
|
|
6771
|
+
mode: relation.mode ?? "first-occurrence",
|
|
6772
|
+
requireEndpoints: relation.requireEndpoints !== false
|
|
6773
|
+
})
|
|
6774
|
+
);
|
|
6775
|
+
}
|
|
6776
|
+
if (contract.observations) {
|
|
6777
|
+
const required = contract.observations.required ?? [];
|
|
6778
|
+
if (required.length > 0) {
|
|
6779
|
+
rules.push({
|
|
6780
|
+
id: "contract.observation.required",
|
|
6781
|
+
category: "run",
|
|
6782
|
+
defaultSeverity: "error",
|
|
6783
|
+
evaluate(context) {
|
|
6784
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
6785
|
+
const names = new Set(outcomes.map((item) => item.name));
|
|
6786
|
+
const missing = required.filter((name) => !names.has(name));
|
|
6787
|
+
if (missing.length === 0) return [];
|
|
6788
|
+
return [
|
|
6789
|
+
contractFailFinding(
|
|
6790
|
+
"contract.observation.required",
|
|
6791
|
+
`Required observations missing: ${missing.join(", ")}`,
|
|
6792
|
+
context.selectedRun ? [{ runId: context.selectedRun.runId, kind: "RUN", name: context.selectedRun.name }] : [],
|
|
6793
|
+
required,
|
|
6794
|
+
[...names]
|
|
6795
|
+
)
|
|
6796
|
+
];
|
|
6797
|
+
}
|
|
6798
|
+
});
|
|
6799
|
+
}
|
|
6800
|
+
if (contract.observations.failOn?.length) {
|
|
6801
|
+
rules.push(
|
|
6802
|
+
createObservedOutcomeRule({
|
|
6803
|
+
failOn: contract.observations.failOn.filter(
|
|
6804
|
+
(status) => status === "failed" || status === "unknown" || status === "skipped"
|
|
6805
|
+
)
|
|
6806
|
+
})
|
|
6807
|
+
);
|
|
6808
|
+
}
|
|
6809
|
+
const requireProvenance = contract.observations.requireProvenance;
|
|
6810
|
+
if (requireProvenance && (requireProvenance.method || requireProvenance.evidence || requireProvenance.sameRunEventReference)) {
|
|
6811
|
+
rules.push({
|
|
6812
|
+
id: "contract.observation.provenance",
|
|
6813
|
+
category: "run",
|
|
6814
|
+
defaultSeverity: "error",
|
|
6815
|
+
evaluate(context) {
|
|
6816
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
6817
|
+
const knownEventIds = new Set(context.events.map((event) => event.eventId));
|
|
6818
|
+
const targets = required.length > 0 ? outcomes.filter((outcome) => required.includes(outcome.name)) : outcomes;
|
|
6819
|
+
return targets.flatMap(
|
|
6820
|
+
(outcome) => provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds)
|
|
6821
|
+
);
|
|
6822
|
+
}
|
|
6823
|
+
});
|
|
6824
|
+
}
|
|
6825
|
+
}
|
|
6826
|
+
if (contract.controls) {
|
|
6827
|
+
const controls = contract.controls;
|
|
6828
|
+
rules.push({
|
|
6829
|
+
id: "contract.controls",
|
|
6830
|
+
category: "run",
|
|
6831
|
+
defaultSeverity: "error",
|
|
6832
|
+
evaluate(context) {
|
|
6833
|
+
const events = context.logicalEvents ?? context.events;
|
|
6834
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
6835
|
+
const observationNames = new Set(outcomes.map((item) => item.name));
|
|
6836
|
+
const runEvidence2 = context.selectedRun ? [
|
|
6837
|
+
{
|
|
6838
|
+
runId: context.selectedRun.runId,
|
|
6839
|
+
kind: "RUN",
|
|
6840
|
+
name: context.selectedRun.name
|
|
6841
|
+
}
|
|
6842
|
+
] : [];
|
|
6843
|
+
return evaluateControlRules(events, controls, runEvidence2, observationNames);
|
|
6844
|
+
}
|
|
6845
|
+
});
|
|
6846
|
+
}
|
|
6847
|
+
if (contract.retry) {
|
|
6848
|
+
const retry = contract.retry;
|
|
6849
|
+
rules.push({
|
|
6850
|
+
id: "contract.retry",
|
|
6851
|
+
category: "run",
|
|
6852
|
+
defaultSeverity: "error",
|
|
6853
|
+
evaluate(context) {
|
|
6854
|
+
const events = context.logicalEvents ?? context.events;
|
|
6855
|
+
context.selectedRun ? [
|
|
6856
|
+
{
|
|
6857
|
+
runId: context.selectedRun.runId,
|
|
6858
|
+
kind: "RUN",
|
|
6859
|
+
name: context.selectedRun.name
|
|
6860
|
+
}
|
|
6861
|
+
] : [];
|
|
6862
|
+
const base = evaluateRetrySafetyRules(events, retry);
|
|
6863
|
+
if (!retry.operations || retry.operations.length === 0) {
|
|
6864
|
+
return base;
|
|
6865
|
+
}
|
|
6866
|
+
return [...base, ...evaluateRecoveryOperations(events, retry.operations)];
|
|
5053
6867
|
}
|
|
5054
|
-
}
|
|
5055
|
-
}
|
|
5056
|
-
|
|
5057
|
-
function bodyHasRules(body) {
|
|
5058
|
-
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
6868
|
+
});
|
|
6869
|
+
}
|
|
6870
|
+
return rules;
|
|
5059
6871
|
}
|
|
5060
|
-
new Set(OBSERVED_OUTCOME_METHODS);
|
|
5061
6872
|
function validateScopeShape(scope) {
|
|
5062
6873
|
if (scope === void 0) return [];
|
|
5063
6874
|
const hasSelector = typeof scope.runId === "string" && scope.runId.trim() !== "" || typeof scope.subAgentId === "string" && scope.subAgentId.trim() !== "" || typeof scope.groupId === "string" && scope.groupId.trim() !== "" || typeof scope.workflowStep === "string" && scope.workflowStep.trim() !== "" || typeof scope.rootEventId === "string" && scope.rootEventId.trim() !== "";
|
|
@@ -5158,6 +6969,57 @@ function validateAlternativesShape(alternatives) {
|
|
|
5158
6969
|
}
|
|
5159
6970
|
return diagnostics;
|
|
5160
6971
|
}
|
|
6972
|
+
function emptyPassResult(input, runId) {
|
|
6973
|
+
return {
|
|
6974
|
+
ok: true,
|
|
6975
|
+
status: "pass",
|
|
6976
|
+
format: input.read.format,
|
|
6977
|
+
...runId !== void 0 ? { runId } : {},
|
|
6978
|
+
summary: { passed: 0, failed: 0, warnings: 0, errors: 0, rulesEvaluated: 0 },
|
|
6979
|
+
findings: [],
|
|
6980
|
+
diagnostics: [],
|
|
6981
|
+
ruleExecutions: []
|
|
6982
|
+
};
|
|
6983
|
+
}
|
|
6984
|
+
function mergeContractResults(input, parts, extraFindings = []) {
|
|
6985
|
+
const findings = [...parts.flatMap((part) => part.findings), ...extraFindings];
|
|
6986
|
+
const diagnostics = parts.flatMap((part) => part.diagnostics);
|
|
6987
|
+
const ruleExecutions = parts.flatMap((part) => part.ruleExecutions);
|
|
6988
|
+
const failed = findings.filter(
|
|
6989
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
6990
|
+
).length;
|
|
6991
|
+
const errors = diagnostics.filter((item) => item.severity === "error").length;
|
|
6992
|
+
const status = errors > 0 ? "error" : failed > 0 ? "fail" : "pass";
|
|
6993
|
+
const runId = parts.find((part) => part.runId !== void 0)?.runId;
|
|
6994
|
+
return {
|
|
6995
|
+
ok: status === "pass",
|
|
6996
|
+
status,
|
|
6997
|
+
format: input.read.format,
|
|
6998
|
+
...runId !== void 0 ? { runId } : {},
|
|
6999
|
+
summary: {
|
|
7000
|
+
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
7001
|
+
failed,
|
|
7002
|
+
warnings: findings.filter(
|
|
7003
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
7004
|
+
).length,
|
|
7005
|
+
errors,
|
|
7006
|
+
rulesEvaluated: ruleExecutions.length
|
|
7007
|
+
},
|
|
7008
|
+
findings,
|
|
7009
|
+
diagnostics,
|
|
7010
|
+
ruleExecutions
|
|
7011
|
+
};
|
|
7012
|
+
}
|
|
7013
|
+
function evaluateBody(input, body, options = {}) {
|
|
7014
|
+
const rules = contractToRules(body);
|
|
7015
|
+
if (rules.length === 0) {
|
|
7016
|
+
return emptyPassResult(input, options.runId);
|
|
7017
|
+
}
|
|
7018
|
+
return runTraceChecks(input, {
|
|
7019
|
+
rules,
|
|
7020
|
+
...options.runId !== void 0 ? { runId: options.runId } : {}
|
|
7021
|
+
});
|
|
7022
|
+
}
|
|
5161
7023
|
function defineTraceContract(input) {
|
|
5162
7024
|
const shapeErrors = [
|
|
5163
7025
|
...validateAlternativesShape(input.alternatives),
|
|
@@ -5188,6 +7050,143 @@ function defineTraceContract(input) {
|
|
|
5188
7050
|
...alternatives ? { alternatives } : {}
|
|
5189
7051
|
};
|
|
5190
7052
|
}
|
|
7053
|
+
function evaluateTraceContract(input, contract, options = {}) {
|
|
7054
|
+
const shapeErrors = [
|
|
7055
|
+
...validateAlternativesShape(contract.alternatives),
|
|
7056
|
+
...validateScopeShape(contract.scope),
|
|
7057
|
+
...validateProvenanceShape(contract.observations)
|
|
7058
|
+
].filter((item) => item.severity === "error");
|
|
7059
|
+
if (shapeErrors.length > 0) {
|
|
7060
|
+
return {
|
|
7061
|
+
ok: false,
|
|
7062
|
+
status: "error",
|
|
7063
|
+
format: input.read.format,
|
|
7064
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
7065
|
+
summary: {
|
|
7066
|
+
passed: 0,
|
|
7067
|
+
failed: 0,
|
|
7068
|
+
warnings: 0,
|
|
7069
|
+
errors: shapeErrors.length,
|
|
7070
|
+
rulesEvaluated: 0
|
|
7071
|
+
},
|
|
7072
|
+
findings: [],
|
|
7073
|
+
diagnostics: shapeErrors.map((item) => ({
|
|
7074
|
+
code: "AI_CHECK_INVALID_CONFIG",
|
|
7075
|
+
message: item.message,
|
|
7076
|
+
severity: "error",
|
|
7077
|
+
ruleId: item.code
|
|
7078
|
+
})),
|
|
7079
|
+
ruleExecutions: []
|
|
7080
|
+
};
|
|
7081
|
+
}
|
|
7082
|
+
const scoped = resolveTraceContractScope(input, contract.scope);
|
|
7083
|
+
if (scoped.diagnostics.length > 0) {
|
|
7084
|
+
return {
|
|
7085
|
+
ok: false,
|
|
7086
|
+
status: "error",
|
|
7087
|
+
format: input.read.format,
|
|
7088
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
7089
|
+
...scoped.resolved?.runId !== void 0 ? { runId: scoped.resolved.runId } : {},
|
|
7090
|
+
summary: {
|
|
7091
|
+
passed: 0,
|
|
7092
|
+
failed: 0,
|
|
7093
|
+
warnings: 0,
|
|
7094
|
+
errors: scoped.diagnostics.length,
|
|
7095
|
+
rulesEvaluated: 0
|
|
7096
|
+
},
|
|
7097
|
+
findings: [],
|
|
7098
|
+
diagnostics: scoped.diagnostics,
|
|
7099
|
+
ruleExecutions: []
|
|
7100
|
+
};
|
|
7101
|
+
}
|
|
7102
|
+
const effectiveInput = scoped.resolved?.input ?? input;
|
|
7103
|
+
const effectiveRunId = scoped.resolved?.runId ?? options.runId;
|
|
7104
|
+
const base = evaluateBody(effectiveInput, contract, {
|
|
7105
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
7106
|
+
});
|
|
7107
|
+
const scopeFinding = scoped.resolved ? {
|
|
7108
|
+
ruleId: "contract.scope.selected",
|
|
7109
|
+
severity: "info",
|
|
7110
|
+
status: "pass",
|
|
7111
|
+
message: `Selected actor ${scoped.resolved.runId} with ${scoped.resolved.evidenceEventCount} evidence event(s).`,
|
|
7112
|
+
expected: scoped.resolved.matchedSelectors,
|
|
7113
|
+
actual: {
|
|
7114
|
+
runId: scoped.resolved.runId,
|
|
7115
|
+
evidenceEventCount: scoped.resolved.evidenceEventCount,
|
|
7116
|
+
...scoped.resolved.rootEventId !== void 0 ? { rootEventId: scoped.resolved.rootEventId } : {}
|
|
7117
|
+
},
|
|
7118
|
+
evidence: [
|
|
7119
|
+
{
|
|
7120
|
+
runId: scoped.resolved.runId,
|
|
7121
|
+
kind: "RUN",
|
|
7122
|
+
path: "scope"
|
|
7123
|
+
}
|
|
7124
|
+
]
|
|
7125
|
+
} : void 0;
|
|
7126
|
+
const branches = contract.alternatives?.anyOf ?? [];
|
|
7127
|
+
if (branches.length === 0) {
|
|
7128
|
+
return scopeFinding ? mergeContractResults(effectiveInput, [base], [scopeFinding]) : base;
|
|
7129
|
+
}
|
|
7130
|
+
const branchEvaluations = branches.map((branch) => {
|
|
7131
|
+
const result = evaluateBody(effectiveInput, branch.contract, {
|
|
7132
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
7133
|
+
});
|
|
7134
|
+
return { branch, result };
|
|
7135
|
+
});
|
|
7136
|
+
const satisfied = branchEvaluations.filter((item) => item.result.ok && item.result.status === "pass").map((item) => item.branch.id);
|
|
7137
|
+
const failedBranches = branchEvaluations.filter(
|
|
7138
|
+
(item) => !(item.result.ok && item.result.status === "pass")
|
|
7139
|
+
);
|
|
7140
|
+
if (satisfied.length > 0) {
|
|
7141
|
+
const passFinding = {
|
|
7142
|
+
ruleId: "contract.alternatives.anyOf",
|
|
7143
|
+
severity: "info",
|
|
7144
|
+
status: "pass",
|
|
7145
|
+
message: `Satisfied alternative branch(es): ${satisfied.join(", ")}.`,
|
|
7146
|
+
expected: { anyOf: branches.map((branch) => branch.id) },
|
|
7147
|
+
actual: {
|
|
7148
|
+
satisfied,
|
|
7149
|
+
failed: failedBranches.map((item) => item.branch.id)
|
|
7150
|
+
},
|
|
7151
|
+
evidence: []
|
|
7152
|
+
};
|
|
7153
|
+
return mergeContractResults(
|
|
7154
|
+
effectiveInput,
|
|
7155
|
+
[base],
|
|
7156
|
+
[...scopeFinding ? [scopeFinding] : [], passFinding]
|
|
7157
|
+
);
|
|
7158
|
+
}
|
|
7159
|
+
const noneFinding = {
|
|
7160
|
+
ruleId: "contract.alternatives.none-satisfied",
|
|
7161
|
+
severity: "error",
|
|
7162
|
+
status: "fail",
|
|
7163
|
+
message: "No alternatives.anyOf branch fully satisfied its contract.",
|
|
7164
|
+
expected: { anyOf: branches.map((branch) => branch.id) },
|
|
7165
|
+
actual: {
|
|
7166
|
+
branchFailures: Object.fromEntries(
|
|
7167
|
+
failedBranches.map((item) => [
|
|
7168
|
+
item.branch.id,
|
|
7169
|
+
item.result.findings.filter((finding) => finding.status === "fail").map((finding) => ({
|
|
7170
|
+
ruleId: finding.ruleId,
|
|
7171
|
+
message: finding.message
|
|
7172
|
+
}))
|
|
7173
|
+
])
|
|
7174
|
+
)
|
|
7175
|
+
},
|
|
7176
|
+
evidence: []
|
|
7177
|
+
};
|
|
7178
|
+
const prefixedBranchFindings = failedBranches.flatMap(
|
|
7179
|
+
(item) => item.result.findings.filter((finding) => finding.status === "fail").map((finding) => ({
|
|
7180
|
+
...finding,
|
|
7181
|
+
ruleId: `contract.alternatives.${item.branch.id}.${finding.ruleId}`
|
|
7182
|
+
}))
|
|
7183
|
+
);
|
|
7184
|
+
return mergeContractResults(
|
|
7185
|
+
effectiveInput,
|
|
7186
|
+
[base],
|
|
7187
|
+
[...scopeFinding ? [scopeFinding] : [], noneFinding, ...prefixedBranchFindings]
|
|
7188
|
+
);
|
|
7189
|
+
}
|
|
5191
7190
|
|
|
5192
7191
|
// packages/core/src/checks/index.ts
|
|
5193
7192
|
var SEVERITY_RANK = {
|
|
@@ -5277,7 +7276,7 @@ var DEFAULT_SECRET_PATTERNS = [
|
|
|
5277
7276
|
function compareStrings(a, b) {
|
|
5278
7277
|
return (a ?? "").localeCompare(b ?? "");
|
|
5279
7278
|
}
|
|
5280
|
-
function
|
|
7279
|
+
function diagnostic2(code, message, ruleId) {
|
|
5281
7280
|
return {
|
|
5282
7281
|
code,
|
|
5283
7282
|
message,
|
|
@@ -5310,14 +7309,14 @@ function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
|
5310
7309
|
ruleExecutions: [...ruleExecutions]
|
|
5311
7310
|
};
|
|
5312
7311
|
}
|
|
5313
|
-
function
|
|
5314
|
-
return nodes.flatMap((node) => [node, ...
|
|
7312
|
+
function flattenNodes2(nodes) {
|
|
7313
|
+
return nodes.flatMap((node) => [node, ...flattenNodes2(node.children)]);
|
|
5315
7314
|
}
|
|
5316
7315
|
function buildFacts2(input, selectedRun) {
|
|
5317
7316
|
const scopedRuns = selectedRun ? [selectedRun] : input.read.runs;
|
|
5318
7317
|
const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
|
|
5319
7318
|
const scopedEvents = selectedRun === void 0 ? input.read.events : input.read.events.filter((event) => scopedRunIds.has(event.runId));
|
|
5320
|
-
const nodes =
|
|
7319
|
+
const nodes = flattenNodes2(scopedRuns.flatMap((run) => run.children));
|
|
5321
7320
|
const nodesByEventId = /* @__PURE__ */ new Map();
|
|
5322
7321
|
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
5323
7322
|
for (const node of nodes) {
|
|
@@ -5349,7 +7348,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
5349
7348
|
if (runId && input.selectedRun.runId !== runId) {
|
|
5350
7349
|
return {
|
|
5351
7350
|
diagnostics: [
|
|
5352
|
-
|
|
7351
|
+
diagnostic2(
|
|
5353
7352
|
"AI_CHECK_INVALID_ARGUMENTS",
|
|
5354
7353
|
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
5355
7354
|
)
|
|
@@ -5363,7 +7362,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
5363
7362
|
if (!run) {
|
|
5364
7363
|
return {
|
|
5365
7364
|
diagnostics: [
|
|
5366
|
-
|
|
7365
|
+
diagnostic2("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
5367
7366
|
]
|
|
5368
7367
|
};
|
|
5369
7368
|
}
|
|
@@ -5375,7 +7374,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
5375
7374
|
if (input.read.runs.length === 0) {
|
|
5376
7375
|
return {
|
|
5377
7376
|
diagnostics: [
|
|
5378
|
-
|
|
7377
|
+
diagnostic2(
|
|
5379
7378
|
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
5380
7379
|
formatProgrammaticDiagnostic(
|
|
5381
7380
|
"AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
@@ -5387,7 +7386,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
5387
7386
|
}
|
|
5388
7387
|
return {
|
|
5389
7388
|
diagnostics: [
|
|
5390
|
-
|
|
7389
|
+
diagnostic2(
|
|
5391
7390
|
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
5392
7391
|
formatProgrammaticDiagnostic("AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED")
|
|
5393
7392
|
)
|
|
@@ -5400,7 +7399,7 @@ function selectRules(rules, selectedIds) {
|
|
|
5400
7399
|
for (const rule of rules) {
|
|
5401
7400
|
if (byId.has(rule.id)) {
|
|
5402
7401
|
diagnostics.push(
|
|
5403
|
-
|
|
7402
|
+
diagnostic2("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
5404
7403
|
);
|
|
5405
7404
|
continue;
|
|
5406
7405
|
}
|
|
@@ -5411,7 +7410,7 @@ function selectRules(rules, selectedIds) {
|
|
|
5411
7410
|
for (const id of selected) {
|
|
5412
7411
|
if (!byId.has(id)) {
|
|
5413
7412
|
diagnostics.push(
|
|
5414
|
-
|
|
7413
|
+
diagnostic2("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
5415
7414
|
);
|
|
5416
7415
|
}
|
|
5417
7416
|
}
|
|
@@ -5526,7 +7525,7 @@ function stripPrefix(name, prefixes) {
|
|
|
5526
7525
|
}
|
|
5527
7526
|
return name;
|
|
5528
7527
|
}
|
|
5529
|
-
function
|
|
7528
|
+
function eventEvidence3(event, path14) {
|
|
5530
7529
|
return {
|
|
5531
7530
|
runId: event.runId,
|
|
5532
7531
|
eventId: event.eventId,
|
|
@@ -5560,6 +7559,31 @@ function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
|
5560
7559
|
function toolName(event) {
|
|
5561
7560
|
return resolveCanonicalToolName(event);
|
|
5562
7561
|
}
|
|
7562
|
+
function stepEndpointName(event, kind) {
|
|
7563
|
+
if (kind === "TOOL") return resolveCanonicalToolName(event);
|
|
7564
|
+
for (const prefix of ["llm:", "generation:", "transcription:", "speech:"]) {
|
|
7565
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
7566
|
+
}
|
|
7567
|
+
return event.name;
|
|
7568
|
+
}
|
|
7569
|
+
function formatStepEndpoint(endpoint) {
|
|
7570
|
+
return `${endpoint.kind} ${endpoint.name}`;
|
|
7571
|
+
}
|
|
7572
|
+
function otherKindsForStepName(context, expectedKind, name) {
|
|
7573
|
+
return [
|
|
7574
|
+
...new Set(
|
|
7575
|
+
semanticEvents(context).filter((event) => {
|
|
7576
|
+
const kind = typeof event.kind === "string" ? event.kind.toUpperCase() : "";
|
|
7577
|
+
if (kind === expectedKind) return false;
|
|
7578
|
+
if (kind === "TOOL") return resolveCanonicalToolName(event) === name;
|
|
7579
|
+
if (kind === "LLM") return stepEndpointName(event, "LLM") === name;
|
|
7580
|
+
return event.name === name || event.name === `tool:${name}` || event.name === `llm:${name}` || event.name.endsWith(`:${name}`);
|
|
7581
|
+
}).map(
|
|
7582
|
+
(event) => typeof event.kind === "string" ? event.kind.toUpperCase() : "UNKNOWN"
|
|
7583
|
+
)
|
|
7584
|
+
)
|
|
7585
|
+
].sort();
|
|
7586
|
+
}
|
|
5563
7587
|
function semanticEvents(context) {
|
|
5564
7588
|
return context.logicalEvents ?? context.events;
|
|
5565
7589
|
}
|
|
@@ -5797,7 +7821,7 @@ function guardrailShape(context) {
|
|
|
5797
7821
|
}
|
|
5798
7822
|
function firstEvidenceForKind(context, kind, path14) {
|
|
5799
7823
|
const event = semanticEvents(context).find((candidate) => candidate.kind === kind);
|
|
5800
|
-
return event ? [
|
|
7824
|
+
return event ? [eventEvidence3(event, path14)] : runEvidence(context.selectedRun);
|
|
5801
7825
|
}
|
|
5802
7826
|
function baselineDiffFinding(message, evidence, expected, actual) {
|
|
5803
7827
|
return failFinding("baseline.regression", message, evidence, expected, actual);
|
|
@@ -5830,7 +7854,7 @@ function createRunStatusRule(options = {}) {
|
|
|
5830
7854
|
failFinding(
|
|
5831
7855
|
"run.status",
|
|
5832
7856
|
"Run contains incomplete running events.",
|
|
5833
|
-
running.map((event) =>
|
|
7857
|
+
running.map((event) => eventEvidence3(event)),
|
|
5834
7858
|
"no running events",
|
|
5835
7859
|
running.length
|
|
5836
7860
|
)
|
|
@@ -5876,7 +7900,7 @@ function createMaxStepDurationRule(options) {
|
|
|
5876
7900
|
failFinding(
|
|
5877
7901
|
"run.maxStepDuration",
|
|
5878
7902
|
`${over.length} step(s) exceeded max duration ${options.maxDurationMs}ms.`,
|
|
5879
|
-
over.map((event) =>
|
|
7903
|
+
over.map((event) => eventEvidence3(event, "durationMs")),
|
|
5880
7904
|
{ maxDurationMs: options.maxDurationMs },
|
|
5881
7905
|
over.map((event) => ({
|
|
5882
7906
|
eventId: event.eventId,
|
|
@@ -5902,7 +7926,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
5902
7926
|
failFinding(
|
|
5903
7927
|
"run.stall",
|
|
5904
7928
|
`Found ${running.length} event(s) still running (possible stall).`,
|
|
5905
|
-
running.map((event) =>
|
|
7929
|
+
running.map((event) => eventEvidence3(event, "status")),
|
|
5906
7930
|
"no running events",
|
|
5907
7931
|
running.length
|
|
5908
7932
|
)
|
|
@@ -5917,7 +7941,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
5917
7941
|
failFinding(
|
|
5918
7942
|
"run.stall",
|
|
5919
7943
|
`Found ${incomplete.length} started event(s) without endedAt.`,
|
|
5920
|
-
incomplete.map((event) =>
|
|
7944
|
+
incomplete.map((event) => eventEvidence3(event, "endedAt")),
|
|
5921
7945
|
"endedAt for started events",
|
|
5922
7946
|
incomplete.length
|
|
5923
7947
|
)
|
|
@@ -5953,7 +7977,7 @@ function createRequireCompletedRule() {
|
|
|
5953
7977
|
failFinding(
|
|
5954
7978
|
"run.requireCompleted",
|
|
5955
7979
|
`Run has ${running.length} incomplete running event(s).`,
|
|
5956
|
-
running.map((event) =>
|
|
7980
|
+
running.map((event) => eventEvidence3(event, "status")),
|
|
5957
7981
|
"no running events",
|
|
5958
7982
|
running.length
|
|
5959
7983
|
)
|
|
@@ -6033,12 +8057,12 @@ function createToolUsageRule(options) {
|
|
|
6033
8057
|
const name = toolName(event);
|
|
6034
8058
|
if (forbidden.has(name)) {
|
|
6035
8059
|
findings.push(
|
|
6036
|
-
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [
|
|
8060
|
+
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence3(event)], "tool absent", name)
|
|
6037
8061
|
);
|
|
6038
8062
|
}
|
|
6039
8063
|
if (allowed && !allowed.has(name)) {
|
|
6040
8064
|
findings.push(
|
|
6041
|
-
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [
|
|
8065
|
+
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence3(event)], [...allowed].sort(), name)
|
|
6042
8066
|
);
|
|
6043
8067
|
}
|
|
6044
8068
|
}
|
|
@@ -6049,13 +8073,269 @@ function createToolUsageRule(options) {
|
|
|
6049
8073
|
}
|
|
6050
8074
|
if (options.maxCount !== void 0 && tools.length > options.maxCount) {
|
|
6051
8075
|
findings.push(
|
|
6052
|
-
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) =>
|
|
8076
|
+
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) => eventEvidence3(event)), { maxCount: options.maxCount }, tools.length)
|
|
6053
8077
|
);
|
|
6054
8078
|
}
|
|
6055
8079
|
return findings;
|
|
6056
8080
|
}
|
|
6057
8081
|
};
|
|
6058
8082
|
}
|
|
8083
|
+
function createStepOrderingRule(options) {
|
|
8084
|
+
const bothTools = options.before.kind === "TOOL" && options.after.kind === "TOOL";
|
|
8085
|
+
const ruleId = options.id ?? (bothTools ? "tool.order" : "step.order");
|
|
8086
|
+
const mode = options.mode ?? "first-occurrence";
|
|
8087
|
+
const requireEndpoints = options.requireEndpoints === true;
|
|
8088
|
+
const beforeLabel = bothTools ? options.before.name : formatStepEndpoint(options.before);
|
|
8089
|
+
const afterLabel = bothTools ? options.after.name : formatStepEndpoint(options.after);
|
|
8090
|
+
const subject = bothTools ? "Tool" : "Step";
|
|
8091
|
+
return {
|
|
8092
|
+
id: ruleId,
|
|
8093
|
+
category: bothTools ? "tool" : "structure",
|
|
8094
|
+
defaultSeverity: "error",
|
|
8095
|
+
evaluate(context) {
|
|
8096
|
+
const beforeEvents = finishedEvents(context, options.before.kind).filter(
|
|
8097
|
+
(event) => stepEndpointName(event, options.before.kind) === options.before.name
|
|
8098
|
+
);
|
|
8099
|
+
const afterEvents = finishedEvents(context, options.after.kind).filter(
|
|
8100
|
+
(event) => stepEndpointName(event, options.after.kind) === options.after.name
|
|
8101
|
+
);
|
|
8102
|
+
if (beforeEvents.length === 0 || afterEvents.length === 0) {
|
|
8103
|
+
if (!requireEndpoints) return [];
|
|
8104
|
+
const findings = [];
|
|
8105
|
+
if (beforeEvents.length === 0) {
|
|
8106
|
+
const otherKinds = otherKindsForStepName(
|
|
8107
|
+
context,
|
|
8108
|
+
options.before.kind,
|
|
8109
|
+
options.before.name
|
|
8110
|
+
);
|
|
8111
|
+
const hint = otherKinds.length > 0 ? ` Required name appears under non-${options.before.kind} kind(s): ${otherKinds.join(", ")}.` : "";
|
|
8112
|
+
findings.push(
|
|
8113
|
+
failFinding(
|
|
8114
|
+
ruleId,
|
|
8115
|
+
`Required ${formatStepEndpoint(options.before)} did not appear.${hint}`,
|
|
8116
|
+
runEvidence(context.selectedRun),
|
|
8117
|
+
options.before,
|
|
8118
|
+
{
|
|
8119
|
+
code: bothTools ? "tool.order.missing-endpoint" : "step.order.missing-endpoint",
|
|
8120
|
+
endpoint: "before",
|
|
8121
|
+
otherKinds
|
|
8122
|
+
}
|
|
8123
|
+
)
|
|
8124
|
+
);
|
|
8125
|
+
}
|
|
8126
|
+
if (afterEvents.length === 0) {
|
|
8127
|
+
const otherKinds = otherKindsForStepName(
|
|
8128
|
+
context,
|
|
8129
|
+
options.after.kind,
|
|
8130
|
+
options.after.name
|
|
8131
|
+
);
|
|
8132
|
+
const hint = otherKinds.length > 0 ? ` Required name appears under non-${options.after.kind} kind(s): ${otherKinds.join(", ")}.` : "";
|
|
8133
|
+
findings.push(
|
|
8134
|
+
failFinding(
|
|
8135
|
+
ruleId,
|
|
8136
|
+
`Required ${formatStepEndpoint(options.after)} did not appear.${hint}`,
|
|
8137
|
+
runEvidence(context.selectedRun),
|
|
8138
|
+
options.after,
|
|
8139
|
+
{
|
|
8140
|
+
code: bothTools ? "tool.order.missing-endpoint" : "step.order.missing-endpoint",
|
|
8141
|
+
endpoint: "after",
|
|
8142
|
+
otherKinds
|
|
8143
|
+
}
|
|
8144
|
+
)
|
|
8145
|
+
);
|
|
8146
|
+
}
|
|
8147
|
+
return findings;
|
|
8148
|
+
}
|
|
8149
|
+
const encounter = finishedEvents(context).filter(
|
|
8150
|
+
(event) => event.kind === options.before.kind && stepEndpointName(event, options.before.kind) === options.before.name || event.kind === options.after.kind && stepEndpointName(event, options.after.kind) === options.after.name
|
|
8151
|
+
);
|
|
8152
|
+
let beforeIndex;
|
|
8153
|
+
let afterIndex;
|
|
8154
|
+
for (const [position, event] of encounter.entries()) {
|
|
8155
|
+
if (beforeIndex === void 0 && event.kind === options.before.kind && stepEndpointName(event, options.before.kind) === options.before.name) {
|
|
8156
|
+
beforeIndex = position;
|
|
8157
|
+
}
|
|
8158
|
+
if (afterIndex === void 0 && event.kind === options.after.kind && stepEndpointName(event, options.after.kind) === options.after.name) {
|
|
8159
|
+
afterIndex = position;
|
|
8160
|
+
}
|
|
8161
|
+
}
|
|
8162
|
+
if (beforeIndex === void 0 || afterIndex === void 0) {
|
|
8163
|
+
return [];
|
|
8164
|
+
}
|
|
8165
|
+
if (mode === "first-occurrence" && beforeIndex >= afterIndex) {
|
|
8166
|
+
return [
|
|
8167
|
+
failFinding(
|
|
8168
|
+
ruleId,
|
|
8169
|
+
`${subject} ${beforeLabel} must appear before ${afterLabel}.`,
|
|
8170
|
+
[
|
|
8171
|
+
eventEvidence3(encounter[beforeIndex]),
|
|
8172
|
+
eventEvidence3(encounter[afterIndex])
|
|
8173
|
+
],
|
|
8174
|
+
bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after },
|
|
8175
|
+
bothTools ? finishedEvents(context, "TOOL").map(toolName) : encounter.map(
|
|
8176
|
+
(event) => event.kind === "TOOL" || event.kind === "LLM" ? `${event.kind}:${stepEndpointName(event, event.kind)}` : event.name
|
|
8177
|
+
)
|
|
8178
|
+
)
|
|
8179
|
+
];
|
|
8180
|
+
}
|
|
8181
|
+
const beforeEvent = beforeEvents[0];
|
|
8182
|
+
const afterEvent = afterEvents[0];
|
|
8183
|
+
const beforeEnd = eventEndMs(beforeEvent);
|
|
8184
|
+
const afterStart = eventStartMs(afterEvent);
|
|
8185
|
+
const endpointExpected = bothTools ? { before: options.before.name, after: options.after.name } : { before: options.before, after: options.after };
|
|
8186
|
+
if (mode === "happens-before") {
|
|
8187
|
+
const expected = {
|
|
8188
|
+
...endpointExpected,
|
|
8189
|
+
mode,
|
|
8190
|
+
relation: "before.end <= after.start"
|
|
8191
|
+
};
|
|
8192
|
+
if (options.before.kind === options.after.kind && options.before.name === options.after.name) {
|
|
8193
|
+
return [
|
|
8194
|
+
failFinding(
|
|
8195
|
+
ruleId,
|
|
8196
|
+
`${subject} ${beforeLabel} cannot causally happen before itself.`,
|
|
8197
|
+
[eventEvidence3(beforeEvent)],
|
|
8198
|
+
expected,
|
|
8199
|
+
{ code: bothTools ? "tool.order.same-tool" : "step.order.same-step" }
|
|
8200
|
+
)
|
|
8201
|
+
];
|
|
8202
|
+
}
|
|
8203
|
+
if (beforeEnd === void 0 || afterStart === void 0) {
|
|
8204
|
+
return [
|
|
8205
|
+
failFinding(
|
|
8206
|
+
ruleId,
|
|
8207
|
+
`${subject} order ${beforeLabel} before ${afterLabel} could not establish causal timing.`,
|
|
8208
|
+
[eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
|
|
8209
|
+
expected,
|
|
8210
|
+
{
|
|
8211
|
+
code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
|
|
8212
|
+
beforeEndResolved: beforeEnd !== void 0,
|
|
8213
|
+
afterStartResolved: afterStart !== void 0
|
|
8214
|
+
}
|
|
8215
|
+
)
|
|
8216
|
+
];
|
|
8217
|
+
}
|
|
8218
|
+
if (beforeEnd > afterStart) {
|
|
8219
|
+
return [
|
|
8220
|
+
failFinding(
|
|
8221
|
+
ruleId,
|
|
8222
|
+
`${subject} ${beforeLabel} must finish before ${afterLabel} starts.`,
|
|
8223
|
+
[eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
|
|
8224
|
+
expected,
|
|
8225
|
+
{
|
|
8226
|
+
beforeEndedAt: new Date(beforeEnd).toISOString(),
|
|
8227
|
+
afterStartedAt: new Date(afterStart).toISOString()
|
|
8228
|
+
}
|
|
8229
|
+
)
|
|
8230
|
+
];
|
|
8231
|
+
}
|
|
8232
|
+
return [];
|
|
8233
|
+
}
|
|
8234
|
+
if (mode === "all-occurrences") {
|
|
8235
|
+
const expected = {
|
|
8236
|
+
...endpointExpected,
|
|
8237
|
+
mode,
|
|
8238
|
+
relation: "max(before.end) <= min(after.start)"
|
|
8239
|
+
};
|
|
8240
|
+
if (options.before.kind === options.after.kind && options.before.name === options.after.name) {
|
|
8241
|
+
return [
|
|
8242
|
+
failFinding(
|
|
8243
|
+
ruleId,
|
|
8244
|
+
`${subject} ${beforeLabel} cannot causally happen before itself.`,
|
|
8245
|
+
[eventEvidence3(beforeEvent)],
|
|
8246
|
+
expected,
|
|
8247
|
+
{ code: bothTools ? "tool.order.same-tool" : "step.order.same-step" }
|
|
8248
|
+
)
|
|
8249
|
+
];
|
|
8250
|
+
}
|
|
8251
|
+
let latestBefore;
|
|
8252
|
+
let earliestAfter;
|
|
8253
|
+
let missingBeforeEnds = 0;
|
|
8254
|
+
let missingAfterStarts = 0;
|
|
8255
|
+
let firstMissingBefore;
|
|
8256
|
+
let firstMissingAfter;
|
|
8257
|
+
for (const event of beforeEvents) {
|
|
8258
|
+
const end = eventEndMs(event);
|
|
8259
|
+
if (end === void 0) {
|
|
8260
|
+
missingBeforeEnds += 1;
|
|
8261
|
+
firstMissingBefore ??= event;
|
|
8262
|
+
} else if (latestBefore === void 0 || end > latestBefore.end) {
|
|
8263
|
+
latestBefore = { event, end };
|
|
8264
|
+
}
|
|
8265
|
+
}
|
|
8266
|
+
for (const event of afterEvents) {
|
|
8267
|
+
const start = eventStartMs(event);
|
|
8268
|
+
if (start === void 0) {
|
|
8269
|
+
missingAfterStarts += 1;
|
|
8270
|
+
firstMissingAfter ??= event;
|
|
8271
|
+
} else if (earliestAfter === void 0 || start < earliestAfter.start) {
|
|
8272
|
+
earliestAfter = { event, start };
|
|
8273
|
+
}
|
|
8274
|
+
}
|
|
8275
|
+
if (missingBeforeEnds > 0 || missingAfterStarts > 0 || latestBefore === void 0 || earliestAfter === void 0) {
|
|
8276
|
+
return [
|
|
8277
|
+
failFinding(
|
|
8278
|
+
ruleId,
|
|
8279
|
+
`${subject} order ${beforeLabel} before ${afterLabel} could not establish all causal intervals.`,
|
|
8280
|
+
[firstMissingBefore, firstMissingAfter].filter((event) => event !== void 0).map((event) => eventEvidence3(event)),
|
|
8281
|
+
expected,
|
|
8282
|
+
{
|
|
8283
|
+
code: bothTools ? "tool.order.interval-unresolved" : "step.order.interval-unresolved",
|
|
8284
|
+
missingBeforeEnds,
|
|
8285
|
+
missingAfterStarts
|
|
8286
|
+
}
|
|
8287
|
+
)
|
|
8288
|
+
];
|
|
8289
|
+
}
|
|
8290
|
+
if (latestBefore.end > earliestAfter.start) {
|
|
8291
|
+
return [
|
|
8292
|
+
failFinding(
|
|
8293
|
+
ruleId,
|
|
8294
|
+
bothTools ? `Every ${options.before.name} tool call must finish before any ${options.after.name} tool call starts.` : `Every ${beforeLabel} must finish before any ${afterLabel} starts.`,
|
|
8295
|
+
[eventEvidence3(latestBefore.event), eventEvidence3(earliestAfter.event)],
|
|
8296
|
+
expected,
|
|
8297
|
+
{
|
|
8298
|
+
latestBeforeEndedAt: new Date(latestBefore.end).toISOString(),
|
|
8299
|
+
earliestAfterStartedAt: new Date(earliestAfter.start).toISOString()
|
|
8300
|
+
}
|
|
8301
|
+
)
|
|
8302
|
+
];
|
|
8303
|
+
}
|
|
8304
|
+
return [];
|
|
8305
|
+
}
|
|
8306
|
+
if (beforeEnd !== void 0 && afterStart !== void 0 && beforeEnd > afterStart) {
|
|
8307
|
+
return [
|
|
8308
|
+
{
|
|
8309
|
+
ruleId,
|
|
8310
|
+
severity: "warning",
|
|
8311
|
+
status: "warning",
|
|
8312
|
+
message: `${subject} order ${beforeLabel} before ${afterLabel} satisfied start/encounter order, but intervals overlap so causal completion before the next ${bothTools ? "tool" : "step"} was not established.`,
|
|
8313
|
+
expected: {
|
|
8314
|
+
...endpointExpected,
|
|
8315
|
+
mode: "first-occurrence"
|
|
8316
|
+
},
|
|
8317
|
+
actual: {
|
|
8318
|
+
code: bothTools ? "tool.order.overlap" : "step.order.overlap",
|
|
8319
|
+
beforeEndedAt: beforeEvent.endedAt,
|
|
8320
|
+
afterStartedAt: afterEvent.startedAt ?? afterEvent.timestamp
|
|
8321
|
+
},
|
|
8322
|
+
evidence: [eventEvidence3(beforeEvent), eventEvidence3(afterEvent)]
|
|
8323
|
+
}
|
|
8324
|
+
];
|
|
8325
|
+
}
|
|
8326
|
+
return [];
|
|
8327
|
+
}
|
|
8328
|
+
};
|
|
8329
|
+
}
|
|
8330
|
+
function createToolOrderingRule(options) {
|
|
8331
|
+
return createStepOrderingRule({
|
|
8332
|
+
before: { kind: "TOOL", name: options.before },
|
|
8333
|
+
after: { kind: "TOOL", name: options.after },
|
|
8334
|
+
...options.id !== void 0 ? { id: options.id } : {},
|
|
8335
|
+
...options.mode !== void 0 ? { mode: options.mode } : {},
|
|
8336
|
+
requireEndpoints: false
|
|
8337
|
+
});
|
|
8338
|
+
}
|
|
6059
8339
|
function createLlmUsageRule(options) {
|
|
6060
8340
|
return {
|
|
6061
8341
|
id: "llm.usage",
|
|
@@ -6072,7 +8352,7 @@ function createLlmUsageRule(options) {
|
|
|
6072
8352
|
failFinding(
|
|
6073
8353
|
"llm.usage",
|
|
6074
8354
|
`LLM call count ${llms.length} exceeded ${options.maxCalls}.`,
|
|
6075
|
-
llms.map((event) =>
|
|
8355
|
+
llms.map((event) => eventEvidence3(event)),
|
|
6076
8356
|
{ maxCalls: options.maxCalls },
|
|
6077
8357
|
llms.length
|
|
6078
8358
|
)
|
|
@@ -6084,17 +8364,17 @@ function createLlmUsageRule(options) {
|
|
|
6084
8364
|
const finishReason = llmFinishReason(event);
|
|
6085
8365
|
if (allowedModels && (!model || !allowedModels.has(model))) {
|
|
6086
8366
|
findings.push(
|
|
6087
|
-
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [
|
|
8367
|
+
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [eventEvidence3(event, "attributes.model")], [...allowedModels].sort(), model ?? "unknown")
|
|
6088
8368
|
);
|
|
6089
8369
|
}
|
|
6090
8370
|
if (allowedProviders && (!provider || !allowedProviders.has(provider))) {
|
|
6091
8371
|
findings.push(
|
|
6092
|
-
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [
|
|
8372
|
+
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [eventEvidence3(event, "attributes.provider")], [...allowedProviders].sort(), provider ?? "unknown")
|
|
6093
8373
|
);
|
|
6094
8374
|
}
|
|
6095
8375
|
if (finishReasons && (!finishReason || !finishReasons.has(finishReason))) {
|
|
6096
8376
|
findings.push(
|
|
6097
|
-
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [
|
|
8377
|
+
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [eventEvidence3(event, "attributes.finishReason")], [...finishReasons].sort(), finishReason ?? "unknown")
|
|
6098
8378
|
);
|
|
6099
8379
|
}
|
|
6100
8380
|
}
|
|
@@ -6119,7 +8399,7 @@ function createLlmUsageRule(options) {
|
|
|
6119
8399
|
failFinding(
|
|
6120
8400
|
"llm.usage",
|
|
6121
8401
|
`LLM ${key} token count ${tokenTotals[key]} exceeded ${limit}.`,
|
|
6122
|
-
llms.map((event) =>
|
|
8402
|
+
llms.map((event) => eventEvidence3(event, `tokenUsage.${key}`)),
|
|
6123
8403
|
{ [`max${key[0].toUpperCase()}${key.slice(1)}Tokens`]: limit },
|
|
6124
8404
|
tokenTotals[key]
|
|
6125
8405
|
)
|
|
@@ -6130,6 +8410,47 @@ function createLlmUsageRule(options) {
|
|
|
6130
8410
|
}
|
|
6131
8411
|
};
|
|
6132
8412
|
}
|
|
8413
|
+
function createStructureIncompleteRule(options = {}) {
|
|
8414
|
+
return {
|
|
8415
|
+
id: "structure.incomplete",
|
|
8416
|
+
category: "structure",
|
|
8417
|
+
defaultSeverity: "error",
|
|
8418
|
+
evaluate(context) {
|
|
8419
|
+
const findings = [];
|
|
8420
|
+
if (!options.allowRunning) {
|
|
8421
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
8422
|
+
if (running.length > 0) {
|
|
8423
|
+
findings.push(
|
|
8424
|
+
failFinding(
|
|
8425
|
+
"structure.incomplete",
|
|
8426
|
+
"Trace contains incomplete running events.",
|
|
8427
|
+
running.map((event) => eventEvidence3(event, "status")),
|
|
8428
|
+
"no running events",
|
|
8429
|
+
running.length
|
|
8430
|
+
)
|
|
8431
|
+
);
|
|
8432
|
+
}
|
|
8433
|
+
}
|
|
8434
|
+
if (options.requireEndedAtForStarted) {
|
|
8435
|
+
const missingEndedAt = semanticEvents(context).filter(
|
|
8436
|
+
(event) => event.startedAt !== void 0 && event.endedAt === void 0 && event.status !== "running"
|
|
8437
|
+
);
|
|
8438
|
+
if (missingEndedAt.length > 0) {
|
|
8439
|
+
findings.push(
|
|
8440
|
+
failFinding(
|
|
8441
|
+
"structure.incomplete",
|
|
8442
|
+
"Trace contains events with startedAt but no endedAt.",
|
|
8443
|
+
missingEndedAt.map((event) => eventEvidence3(event, "endedAt")),
|
|
8444
|
+
"endedAt for started events",
|
|
8445
|
+
missingEndedAt.length
|
|
8446
|
+
)
|
|
8447
|
+
);
|
|
8448
|
+
}
|
|
8449
|
+
}
|
|
8450
|
+
return findings;
|
|
8451
|
+
}
|
|
8452
|
+
};
|
|
8453
|
+
}
|
|
6133
8454
|
function createStructureOrphanRule(options = {}) {
|
|
6134
8455
|
const allowMarkedUnresolved = options.allowMarkedUnresolved ?? true;
|
|
6135
8456
|
return {
|
|
@@ -6147,7 +8468,7 @@ function createStructureOrphanRule(options = {}) {
|
|
|
6147
8468
|
failFinding(
|
|
6148
8469
|
"structure.orphan",
|
|
6149
8470
|
"Trace contains events whose parentId is not present in the selected run.",
|
|
6150
|
-
orphans.map((event) =>
|
|
8471
|
+
orphans.map((event) => eventEvidence3(event, "parentId")),
|
|
6151
8472
|
"parentId resolves to an event in the selected run",
|
|
6152
8473
|
orphans.length
|
|
6153
8474
|
)
|
|
@@ -6179,7 +8500,7 @@ function createStructureCycleRule() {
|
|
|
6179
8500
|
failFinding(
|
|
6180
8501
|
"structure.cycle",
|
|
6181
8502
|
formatProgrammaticDiagnostic("AI_TRACE_RELATIONSHIP_CYCLE"),
|
|
6182
|
-
cycle.map((item) =>
|
|
8503
|
+
cycle.map((item) => eventEvidence3(item, "parentId")),
|
|
6183
8504
|
"acyclic parentId graph",
|
|
6184
8505
|
cycle.map((item) => item.eventId).sort()
|
|
6185
8506
|
)
|
|
@@ -6219,7 +8540,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
6219
8540
|
failFinding(
|
|
6220
8541
|
"structure.relationship",
|
|
6221
8542
|
`Event confidence ${event.confidence} is below ${minConfidence}.`,
|
|
6222
|
-
[
|
|
8543
|
+
[eventEvidence3(event, "confidence")],
|
|
6223
8544
|
{ minConfidence },
|
|
6224
8545
|
event.confidence
|
|
6225
8546
|
)
|
|
@@ -6232,7 +8553,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
6232
8553
|
failFinding(
|
|
6233
8554
|
"structure.relationship",
|
|
6234
8555
|
"Event parentId points to itself.",
|
|
6235
|
-
[
|
|
8556
|
+
[eventEvidence3(event, "parentId")],
|
|
6236
8557
|
"parentId references a distinct event",
|
|
6237
8558
|
"self"
|
|
6238
8559
|
)
|
|
@@ -6249,7 +8570,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
6249
8570
|
failFinding(
|
|
6250
8571
|
"structure.relationship",
|
|
6251
8572
|
"Parent event starts after child event.",
|
|
6252
|
-
[
|
|
8573
|
+
[eventEvidence3(parent), eventEvidence3(event, "parentId")],
|
|
6253
8574
|
"parent start <= child start",
|
|
6254
8575
|
{ parentEventId: parent.eventId, childEventId: event.eventId }
|
|
6255
8576
|
)
|
|
@@ -6263,7 +8584,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
6263
8584
|
failFinding(
|
|
6264
8585
|
"structure.relationship",
|
|
6265
8586
|
"Trace parentSpanId does not match parent spanId.",
|
|
6266
|
-
[
|
|
8587
|
+
[eventEvidence3(event, "trace.parentSpanId")],
|
|
6267
8588
|
{ parentSpanId: parent.trace.spanId },
|
|
6268
8589
|
actual ?? "missing"
|
|
6269
8590
|
)
|
|
@@ -6292,7 +8613,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
6292
8613
|
"structure.parallelWidth",
|
|
6293
8614
|
`Parent ${parentId} has ${children.length} children, exceeding ${options.maxChildren}.`,
|
|
6294
8615
|
[
|
|
6295
|
-
...parent ? [
|
|
8616
|
+
...parent ? [eventEvidence3(parent)] : [{ runId: context.selectedRun?.runId, eventId: parentId }],
|
|
6296
8617
|
...children.map((child) => ({
|
|
6297
8618
|
runId: child.event.runId,
|
|
6298
8619
|
eventId: child.event.eventId,
|
|
@@ -6340,7 +8661,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
6340
8661
|
failFinding(
|
|
6341
8662
|
"structure.parallelWidth",
|
|
6342
8663
|
`Concurrent event width ${maxActive.length} exceeded ${options.maxConcurrent}.`,
|
|
6343
|
-
maxActive.map((event) =>
|
|
8664
|
+
maxActive.map((event) => eventEvidence3(event)),
|
|
6344
8665
|
{ maxConcurrent: options.maxConcurrent },
|
|
6345
8666
|
maxActive.length
|
|
6346
8667
|
)
|
|
@@ -6368,7 +8689,7 @@ function createSafetyRedactionRule(options = {}) {
|
|
|
6368
8689
|
failFinding(
|
|
6369
8690
|
"safety.redaction",
|
|
6370
8691
|
`Sensitive-looking field at ${entry.path} is not redacted.`,
|
|
6371
|
-
[
|
|
8692
|
+
[eventEvidence3(event, entry.path)],
|
|
6372
8693
|
"redaction marker",
|
|
6373
8694
|
{ path: entry.path, valueType: valueType(entry.value) },
|
|
6374
8695
|
{
|
|
@@ -6404,7 +8725,7 @@ function createSafetyRawContentRule(options = {}) {
|
|
|
6404
8725
|
failFinding(
|
|
6405
8726
|
"safety.rawPrompt",
|
|
6406
8727
|
`Raw content-like field ${entry.path} is present.`,
|
|
6407
|
-
[
|
|
8728
|
+
[eventEvidence3(event, entry.path)],
|
|
6408
8729
|
"metadata-only trace fields",
|
|
6409
8730
|
{ path: entry.path, valueType: valueType(entry.value) },
|
|
6410
8731
|
{
|
|
@@ -6442,7 +8763,7 @@ function createSafetySecretPatternRule(options = {}) {
|
|
|
6442
8763
|
failFinding(
|
|
6443
8764
|
"safety.secretPattern",
|
|
6444
8765
|
`Secret-like pattern ${pattern.id} matched at ${entry.path}.`,
|
|
6445
|
-
[
|
|
8766
|
+
[eventEvidence3(event, entry.path)],
|
|
6446
8767
|
"no secret-like strings",
|
|
6447
8768
|
{ pattern: pattern.id, path: entry.path },
|
|
6448
8769
|
{
|
|
@@ -6475,7 +8796,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
6475
8796
|
failFinding(
|
|
6476
8797
|
"safety.oversizedAttribute",
|
|
6477
8798
|
`String at ${entry.path} exceeds ${options.maxStringLength} characters.`,
|
|
6478
|
-
[
|
|
8799
|
+
[eventEvidence3(event, entry.path)],
|
|
6479
8800
|
{ maxStringLength: options.maxStringLength },
|
|
6480
8801
|
{ path: entry.path, length: entry.value.length },
|
|
6481
8802
|
{
|
|
@@ -6492,7 +8813,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
6492
8813
|
failFinding(
|
|
6493
8814
|
"safety.oversizedAttribute",
|
|
6494
8815
|
`Array at ${entry.path} exceeds ${options.maxArrayLength} items.`,
|
|
6495
|
-
[
|
|
8816
|
+
[eventEvidence3(event, entry.path)],
|
|
6496
8817
|
{ maxArrayLength: options.maxArrayLength },
|
|
6497
8818
|
{ path: entry.path, length: entry.value.length },
|
|
6498
8819
|
{
|
|
@@ -6509,7 +8830,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
6509
8830
|
failFinding(
|
|
6510
8831
|
"safety.oversizedAttribute",
|
|
6511
8832
|
`Object at ${entry.path} exceeds ${options.maxObjectKeys} keys.`,
|
|
6512
|
-
[
|
|
8833
|
+
[eventEvidence3(event, entry.path)],
|
|
6513
8834
|
{ maxObjectKeys: options.maxObjectKeys },
|
|
6514
8835
|
{ path: entry.path, keys: Object.keys(entry.value).length },
|
|
6515
8836
|
{
|
|
@@ -6528,7 +8849,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
6528
8849
|
failFinding(
|
|
6529
8850
|
"safety.oversizedAttribute",
|
|
6530
8851
|
`Value at ${entry.path} exceeds ${options.maxSerializedBytes} serialized bytes.`,
|
|
6531
|
-
[
|
|
8852
|
+
[eventEvidence3(event, entry.path)],
|
|
6532
8853
|
{ maxSerializedBytes: options.maxSerializedBytes },
|
|
6533
8854
|
{ path: entry.path, bytes },
|
|
6534
8855
|
{
|
|
@@ -6655,7 +8976,7 @@ function createBaselineRegressionRule(options) {
|
|
|
6655
8976
|
path: "guardrail",
|
|
6656
8977
|
expected: guardrailShape(baselineContext),
|
|
6657
8978
|
actual: guardrailShape(context),
|
|
6658
|
-
evidence: guardrailEvents(context)[0] ? [
|
|
8979
|
+
evidence: guardrailEvents(context)[0] ? [eventEvidence3(guardrailEvents(context)[0], "guardrail")] : runEvidence(context.selectedRun)
|
|
6659
8980
|
}
|
|
6660
8981
|
];
|
|
6661
8982
|
for (const comparison of comparisons) {
|
|
@@ -6734,7 +9055,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
6734
9055
|
return errorResult(
|
|
6735
9056
|
input,
|
|
6736
9057
|
[
|
|
6737
|
-
|
|
9058
|
+
diagnostic2(
|
|
6738
9059
|
"AI_CHECK_NO_RULES_EVALUATED",
|
|
6739
9060
|
"No trace check rules were evaluated. Configure at least one rule, contract, or CLI check option."
|
|
6740
9061
|
)
|
|
@@ -6765,7 +9086,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
6765
9086
|
} catch (error) {
|
|
6766
9087
|
const message = error instanceof Error ? error.message : String(error);
|
|
6767
9088
|
diagnostics.push(
|
|
6768
|
-
|
|
9089
|
+
diagnostic2("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
|
|
6769
9090
|
);
|
|
6770
9091
|
ruleExecutions.push({
|
|
6771
9092
|
ruleId: rule.id,
|
|
@@ -6829,18 +9150,18 @@ function escapeMarkdown(value) {
|
|
|
6829
9150
|
function escapeHtml(value) {
|
|
6830
9151
|
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
6831
9152
|
}
|
|
6832
|
-
function
|
|
9153
|
+
function sortKeysDeep2(input) {
|
|
6833
9154
|
if (input === null || typeof input !== "object") return input;
|
|
6834
|
-
if (Array.isArray(input)) return input.map(
|
|
9155
|
+
if (Array.isArray(input)) return input.map(sortKeysDeep2);
|
|
6835
9156
|
const o = input;
|
|
6836
9157
|
const out = {};
|
|
6837
9158
|
for (const k of Object.keys(o).sort()) {
|
|
6838
|
-
out[k] =
|
|
9159
|
+
out[k] = sortKeysDeep2(o[k]);
|
|
6839
9160
|
}
|
|
6840
9161
|
return out;
|
|
6841
9162
|
}
|
|
6842
9163
|
function stableJson(value, pretty) {
|
|
6843
|
-
const sorted =
|
|
9164
|
+
const sorted = sortKeysDeep2(value);
|
|
6844
9165
|
return pretty === true ? JSON.stringify(sorted, null, 2) : JSON.stringify(sorted);
|
|
6845
9166
|
}
|
|
6846
9167
|
function compactAttributes(attrs, options) {
|
|
@@ -8700,7 +11021,7 @@ var DEFAULT_SUITE_CONFIG_NAMES = [
|
|
|
8700
11021
|
"agent-inspect.suite.cjs"
|
|
8701
11022
|
];
|
|
8702
11023
|
var DEFAULT_SUITE_ARTIFACTS_DIR = ".agent-inspect/suite-runs";
|
|
8703
|
-
function
|
|
11024
|
+
function diagnostic3(code, message, severity = "error", caseId) {
|
|
8704
11025
|
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
8705
11026
|
}
|
|
8706
11027
|
function asString(value, label) {
|
|
@@ -8728,7 +11049,7 @@ function validateCaseConfig(value, index) {
|
|
|
8728
11049
|
const diagnostics = [];
|
|
8729
11050
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
8730
11051
|
diagnostics.push(
|
|
8731
|
-
|
|
11052
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
|
|
8732
11053
|
);
|
|
8733
11054
|
return { diagnostics };
|
|
8734
11055
|
}
|
|
@@ -8737,7 +11058,7 @@ function validateCaseConfig(value, index) {
|
|
|
8737
11058
|
const id = asString(raw.id, `cases[${index}].id`);
|
|
8738
11059
|
if (id === void 0) {
|
|
8739
11060
|
diagnostics.push(
|
|
8740
|
-
|
|
11061
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
|
|
8741
11062
|
);
|
|
8742
11063
|
return { diagnostics };
|
|
8743
11064
|
}
|
|
@@ -8772,7 +11093,7 @@ function validateCaseConfig(value, index) {
|
|
|
8772
11093
|
};
|
|
8773
11094
|
} catch (error) {
|
|
8774
11095
|
const message = error instanceof Error ? error.message : String(error);
|
|
8775
|
-
diagnostics.push(
|
|
11096
|
+
diagnostics.push(diagnostic3("AI_SUITE_CONFIG_INVALID", message));
|
|
8776
11097
|
return { diagnostics };
|
|
8777
11098
|
}
|
|
8778
11099
|
}
|
|
@@ -8848,7 +11169,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
8848
11169
|
await access(tracesDir);
|
|
8849
11170
|
} catch {
|
|
8850
11171
|
diagnostics.push(
|
|
8851
|
-
|
|
11172
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `traces directory not found: ${tracesDir}`)
|
|
8852
11173
|
);
|
|
8853
11174
|
}
|
|
8854
11175
|
for (const suiteCase of config.cases) {
|
|
@@ -8858,7 +11179,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
8858
11179
|
await access(inputPath);
|
|
8859
11180
|
} catch {
|
|
8860
11181
|
diagnostics.push(
|
|
8861
|
-
|
|
11182
|
+
diagnostic3(
|
|
8862
11183
|
"AI_SUITE_CONFIG_INVALID",
|
|
8863
11184
|
`input fixture not found: ${suiteCase.input}`,
|
|
8864
11185
|
"warning",
|
|
@@ -8872,7 +11193,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
8872
11193
|
}
|
|
8873
11194
|
var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
8874
11195
|
var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
8875
|
-
function
|
|
11196
|
+
function diagnostic4(code, message) {
|
|
8876
11197
|
return { code, message, severity: "error" };
|
|
8877
11198
|
}
|
|
8878
11199
|
async function fileExists(filePath) {
|
|
@@ -8903,20 +11224,20 @@ async function loadSuiteConfig(options = {}) {
|
|
|
8903
11224
|
} catch (error) {
|
|
8904
11225
|
const message = error instanceof Error ? error.message : String(error);
|
|
8905
11226
|
throw Object.assign(new Error(message), {
|
|
8906
|
-
diagnostics: [
|
|
11227
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
8907
11228
|
});
|
|
8908
11229
|
}
|
|
8909
11230
|
const extension = path7.extname(configPath);
|
|
8910
11231
|
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
8911
11232
|
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
8912
11233
|
throw Object.assign(new Error(message), {
|
|
8913
|
-
diagnostics: [
|
|
11234
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
8914
11235
|
});
|
|
8915
11236
|
}
|
|
8916
11237
|
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
8917
11238
|
const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
|
|
8918
11239
|
throw Object.assign(new Error(message), {
|
|
8919
|
-
diagnostics: [
|
|
11240
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
8920
11241
|
});
|
|
8921
11242
|
}
|
|
8922
11243
|
try {
|
|
@@ -8936,7 +11257,7 @@ async function loadSuiteConfig(options = {}) {
|
|
|
8936
11257
|
} catch (error) {
|
|
8937
11258
|
const message = error instanceof Error ? error.message : String(error);
|
|
8938
11259
|
throw Object.assign(new Error(message), {
|
|
8939
|
-
diagnostics: [
|
|
11260
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
8940
11261
|
});
|
|
8941
11262
|
}
|
|
8942
11263
|
}
|
|
@@ -9859,13 +12180,13 @@ function persistedEventsForParsedTrace(parsed) {
|
|
|
9859
12180
|
function isRecord11(value) {
|
|
9860
12181
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
9861
12182
|
}
|
|
9862
|
-
function
|
|
12183
|
+
function isNonEmptyString5(value) {
|
|
9863
12184
|
return typeof value === "string" && value.trim() !== "";
|
|
9864
12185
|
}
|
|
9865
12186
|
function readStringField(record, keys) {
|
|
9866
12187
|
for (const key of keys) {
|
|
9867
12188
|
const value = record[key];
|
|
9868
|
-
if (
|
|
12189
|
+
if (isNonEmptyString5(value)) return value;
|
|
9869
12190
|
}
|
|
9870
12191
|
return void 0;
|
|
9871
12192
|
}
|
|
@@ -9999,7 +12320,7 @@ function parseUnixNanoToIso(value) {
|
|
|
9999
12320
|
return void 0;
|
|
10000
12321
|
}
|
|
10001
12322
|
function parseIsoTime(value) {
|
|
10002
|
-
if (!
|
|
12323
|
+
if (!isNonEmptyString5(value)) return void 0;
|
|
10003
12324
|
const ms = Date.parse(value);
|
|
10004
12325
|
if (!Number.isFinite(ms)) return void 0;
|
|
10005
12326
|
return new Date(ms).toISOString();
|
|
@@ -11209,7 +13530,7 @@ async function resolveSuiteCaseTrace(suiteCase, options) {
|
|
|
11209
13530
|
}
|
|
11210
13531
|
|
|
11211
13532
|
// packages/core/src/suite/run.ts
|
|
11212
|
-
function
|
|
13533
|
+
function diagnostic5(code, message, severity = "error", caseId) {
|
|
11213
13534
|
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
11214
13535
|
}
|
|
11215
13536
|
function buildCaseRules(suiteCase, config) {
|
|
@@ -11273,7 +13594,7 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
11273
13594
|
const match = outcomes.find((outcome) => outcome.name === name);
|
|
11274
13595
|
if (!match) {
|
|
11275
13596
|
diagnostics.push(
|
|
11276
|
-
|
|
13597
|
+
diagnostic5(
|
|
11277
13598
|
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
11278
13599
|
`Expected observation "${name}" was not recorded.`,
|
|
11279
13600
|
"error",
|
|
@@ -11284,7 +13605,7 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
11284
13605
|
}
|
|
11285
13606
|
if (match.status !== "passed") {
|
|
11286
13607
|
diagnostics.push(
|
|
11287
|
-
|
|
13608
|
+
diagnostic5(
|
|
11288
13609
|
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
11289
13610
|
`Observation "${name}" has status "${match.status}", expected "passed".`,
|
|
11290
13611
|
"error",
|
|
@@ -11304,7 +13625,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11304
13625
|
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
11305
13626
|
message: resolved.reason,
|
|
11306
13627
|
diagnostics: [
|
|
11307
|
-
|
|
13628
|
+
diagnostic5(
|
|
11308
13629
|
"AI_SUITE_CASE_TRACE_MISSING",
|
|
11309
13630
|
resolved.reason ?? "Trace not found.",
|
|
11310
13631
|
"warning",
|
|
@@ -11325,7 +13646,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11325
13646
|
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
11326
13647
|
message,
|
|
11327
13648
|
diagnostics: [
|
|
11328
|
-
|
|
13649
|
+
diagnostic5("AI_SUITE_TRACE_UNREADABLE", message, "error", suiteCase.id)
|
|
11329
13650
|
]
|
|
11330
13651
|
};
|
|
11331
13652
|
}
|
|
@@ -11339,7 +13660,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11339
13660
|
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
11340
13661
|
const diagnostics = [
|
|
11341
13662
|
...checkResult.diagnostics.map(
|
|
11342
|
-
(item) =>
|
|
13663
|
+
(item) => diagnostic5(
|
|
11343
13664
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
11344
13665
|
item.message,
|
|
11345
13666
|
item.severity,
|
|
@@ -11347,7 +13668,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11347
13668
|
)
|
|
11348
13669
|
),
|
|
11349
13670
|
...checkResult.findings.filter((finding) => finding.status === "fail").map(
|
|
11350
|
-
(finding) =>
|
|
13671
|
+
(finding) => diagnostic5(
|
|
11351
13672
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
11352
13673
|
finding.message,
|
|
11353
13674
|
finding.severity,
|
|
@@ -12532,6 +14853,6 @@ function renderGateReport(result, options = {}) {
|
|
|
12532
14853
|
return renderGateSummaryMarkdown(result);
|
|
12533
14854
|
}
|
|
12534
14855
|
|
|
12535
|
-
export { ATTRIBUTION_CONFIDENCES, COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, Redactor, TraceDirectory, TraceReadError, TreeBuilder, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, assertEvidenceRelativePath, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceHtmlShell, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectTraceSchemaVersions, compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, diffRuns, diffTraceEvents, enrichSessionRunRecord, escapeHtml, escapeMarkdown, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatStepLabel, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, inferEvidenceFileRole, isAgentInspectTrace, isAttributionConfidence, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, manualTraceEventsToComparableRun, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToTraceEvents, projectLogicalEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunDiff, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, serializeEvidenceManifest, sha256Hex, stableJson, summarizeObservedOutcomes, summarizeSemanticParity, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, verifyEvidenceDirectory, zeroKinds };
|
|
12536
|
-
//# sourceMappingURL=chunk-
|
|
12537
|
-
//# sourceMappingURL=chunk-
|
|
14856
|
+
export { ATTRIBUTION_CONFIDENCES, COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, Redactor, TraceDirectory, TraceReadError, TreeBuilder, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, assertEvidenceRelativePath, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceHtmlShell, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectTraceSchemaVersions, compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, defineTraceContract, diffRuns, diffTraceEvents, enrichSessionRunRecord, escapeHtml, escapeMarkdown, evaluateTraceContract, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatStepLabel, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, inferEvidenceFileRole, isAgentInspectTrace, isAttributionConfidence, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, manualTraceEventsToComparableRun, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToTraceEvents, projectLogicalEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunDiff, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, serializeEvidenceManifest, sha256Hex, stableJson, summarizeObservedOutcomes, summarizeSemanticParity, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, verifyEvidenceDirectory, zeroKinds };
|
|
14857
|
+
//# sourceMappingURL=chunk-Y5Z4BXUO.mjs.map
|
|
14858
|
+
//# sourceMappingURL=chunk-Y5Z4BXUO.mjs.map
|