agent-inspect 6.29.5 → 6.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +1 -1
- package/docs/CLI.md +27 -5
- package/docs/TRACE-CONTRACTS.md +10 -7
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-XWQEHKGB.mjs → chunk-BDRJENLM.mjs} +2334 -102
- package/packages/cli/dist/chunk-BDRJENLM.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +3122 -168
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +793 -61
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-XNC4EUN3.mjs → src-ZUGAUXY6.mjs} +3 -3
- package/packages/cli/dist/{src-XNC4EUN3.mjs.map → src-ZUGAUXY6.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +19 -1
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +2 -2
- package/packages/core/dist/advanced.d.ts +2 -2
- package/packages/core/dist/advanced.mjs +2 -2
- package/packages/core/dist/checks.cjs +33 -1
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +2 -2
- package/packages/core/dist/checks.d.ts +2 -2
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-TU47EKKS.mjs → chunk-OOX25GTH.mjs} +35 -3
- package/packages/core/dist/chunk-OOX25GTH.mjs.map +1 -0
- package/packages/core/dist/{index-DvSbXNot.d.ts → index-7hG_qggp.d.ts} +5 -0
- package/packages/core/dist/{index-DKv7esIo.d.cts → index-CQJUNPPe.d.cts} +5 -0
- package/packages/cli/dist/chunk-XWQEHKGB.mjs.map +0 -1
- package/packages/core/dist/chunk-TU47EKKS.mjs.map +0 -1
|
@@ -3270,7 +3270,7 @@ function deriveLastError(runs) {
|
|
|
3270
3270
|
}
|
|
3271
3271
|
function deriveCheckSummary(runs) {
|
|
3272
3272
|
let pass = 0;
|
|
3273
|
-
let
|
|
3273
|
+
let fail4 = 0;
|
|
3274
3274
|
let warn2 = 0;
|
|
3275
3275
|
let found = false;
|
|
3276
3276
|
for (const run of runs) {
|
|
@@ -3282,7 +3282,7 @@ function deriveCheckSummary(runs) {
|
|
|
3282
3282
|
found = true;
|
|
3283
3283
|
}
|
|
3284
3284
|
if (typeof record.fail === "number") {
|
|
3285
|
-
|
|
3285
|
+
fail4 += record.fail;
|
|
3286
3286
|
found = true;
|
|
3287
3287
|
}
|
|
3288
3288
|
if (typeof record.warn === "number") {
|
|
@@ -3290,7 +3290,7 @@ function deriveCheckSummary(runs) {
|
|
|
3290
3290
|
found = true;
|
|
3291
3291
|
}
|
|
3292
3292
|
}
|
|
3293
|
-
return found ? { pass, fail, warn: warn2 } : void 0;
|
|
3293
|
+
return found ? { pass, fail: fail4, warn: warn2 } : void 0;
|
|
3294
3294
|
}
|
|
3295
3295
|
function deriveObservationSummary(runs) {
|
|
3296
3296
|
for (const run of [...runs].sort((a, b) => activityMs(b) - activityMs(a))) {
|
|
@@ -5004,7 +5004,1359 @@ function isCredentialSensitiveKey(key, sensitiveKeys = DEFAULT_CREDENTIAL_SENSIT
|
|
|
5004
5004
|
return false;
|
|
5005
5005
|
}
|
|
5006
5006
|
|
|
5007
|
+
// packages/core/src/checks/contract-scope.ts
|
|
5008
|
+
function diagnostic(code, message, ruleId) {
|
|
5009
|
+
return {
|
|
5010
|
+
code,
|
|
5011
|
+
message,
|
|
5012
|
+
severity: "error",
|
|
5013
|
+
...ruleId !== void 0 ? { ruleId } : {}
|
|
5014
|
+
};
|
|
5015
|
+
}
|
|
5016
|
+
function isNonEmptyString4(value) {
|
|
5017
|
+
return typeof value === "string" && value.trim() !== "";
|
|
5018
|
+
}
|
|
5019
|
+
function scopeEntries(scope) {
|
|
5020
|
+
const out = [];
|
|
5021
|
+
if (isNonEmptyString4(scope.runId)) out.push(["runId", scope.runId.trim()]);
|
|
5022
|
+
if (isNonEmptyString4(scope.subAgentId)) out.push(["subAgentId", scope.subAgentId.trim()]);
|
|
5023
|
+
if (isNonEmptyString4(scope.groupId)) out.push(["groupId", scope.groupId.trim()]);
|
|
5024
|
+
if (isNonEmptyString4(scope.workflowStep)) {
|
|
5025
|
+
out.push(["workflowStep", scope.workflowStep.trim()]);
|
|
5026
|
+
}
|
|
5027
|
+
if (isNonEmptyString4(scope.rootEventId)) {
|
|
5028
|
+
out.push(["rootEventId", scope.rootEventId.trim()]);
|
|
5029
|
+
}
|
|
5030
|
+
return out;
|
|
5031
|
+
}
|
|
5032
|
+
function mergeWorkflowMeta(into, from) {
|
|
5033
|
+
if (!from) return;
|
|
5034
|
+
for (const [key, value] of Object.entries(from)) {
|
|
5035
|
+
if (value === void 0) continue;
|
|
5036
|
+
if (into[key] === void 0) {
|
|
5037
|
+
into[key] = value;
|
|
5038
|
+
}
|
|
5039
|
+
}
|
|
5040
|
+
}
|
|
5041
|
+
function workflowMetadataForRun(events, runId) {
|
|
5042
|
+
const preferred = {};
|
|
5043
|
+
const fallback = {};
|
|
5044
|
+
for (const event of events) {
|
|
5045
|
+
if (event.runId !== runId) continue;
|
|
5046
|
+
const attrs = event.attributes;
|
|
5047
|
+
if (!attrs || typeof attrs !== "object") continue;
|
|
5048
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
5049
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
5050
|
+
const bag = {};
|
|
5051
|
+
mergeWorkflowMeta(bag, direct);
|
|
5052
|
+
mergeWorkflowMeta(bag, nested);
|
|
5053
|
+
if (event.kind === "RUN") {
|
|
5054
|
+
mergeWorkflowMeta(preferred, bag);
|
|
5055
|
+
} else {
|
|
5056
|
+
mergeWorkflowMeta(fallback, bag);
|
|
5057
|
+
}
|
|
5058
|
+
}
|
|
5059
|
+
mergeWorkflowMeta(preferred, fallback);
|
|
5060
|
+
return preferred;
|
|
5061
|
+
}
|
|
5062
|
+
function flattenNodes(nodes) {
|
|
5063
|
+
return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
|
|
5064
|
+
}
|
|
5065
|
+
function collectSubtreeEventIds(run, rootEventId) {
|
|
5066
|
+
const nodes = flattenNodes(run.children);
|
|
5067
|
+
const byId = new Map(nodes.map((node) => [node.event.eventId, node]));
|
|
5068
|
+
const root = byId.get(rootEventId);
|
|
5069
|
+
if (!root) return void 0;
|
|
5070
|
+
const ids = /* @__PURE__ */ new Set();
|
|
5071
|
+
const visit = (node) => {
|
|
5072
|
+
ids.add(node.event.eventId);
|
|
5073
|
+
for (const child of node.children) visit(child);
|
|
5074
|
+
};
|
|
5075
|
+
visit(root);
|
|
5076
|
+
return ids;
|
|
5077
|
+
}
|
|
5078
|
+
function filterTree(nodes, keep) {
|
|
5079
|
+
const out = [];
|
|
5080
|
+
for (const node of nodes) {
|
|
5081
|
+
if (!keep.has(node.event.eventId)) continue;
|
|
5082
|
+
out.push({
|
|
5083
|
+
...node,
|
|
5084
|
+
children: filterTree(node.children, keep)
|
|
5085
|
+
});
|
|
5086
|
+
}
|
|
5087
|
+
return out;
|
|
5088
|
+
}
|
|
5089
|
+
function projectInput(input, run, eventIds) {
|
|
5090
|
+
const runEvents = input.read.events.filter((event) => event.runId === run.runId);
|
|
5091
|
+
const projectedEvents = eventIds === void 0 ? runEvents : runEvents.filter((event) => eventIds.has(event.eventId) || event.kind === "RUN");
|
|
5092
|
+
const projectedChildren = eventIds === void 0 ? run.children : filterTree(run.children, eventIds);
|
|
5093
|
+
const projectedRun = {
|
|
5094
|
+
...run,
|
|
5095
|
+
children: projectedChildren,
|
|
5096
|
+
metadata: {
|
|
5097
|
+
...run.metadata,
|
|
5098
|
+
totalEvents: flattenNodes(projectedChildren).length
|
|
5099
|
+
}
|
|
5100
|
+
};
|
|
5101
|
+
return {
|
|
5102
|
+
evidenceEventCount: projectedEvents.length,
|
|
5103
|
+
input: {
|
|
5104
|
+
...input,
|
|
5105
|
+
selectedRun: projectedRun,
|
|
5106
|
+
read: {
|
|
5107
|
+
...input.read,
|
|
5108
|
+
runs: [projectedRun],
|
|
5109
|
+
events: projectedEvents
|
|
5110
|
+
}
|
|
5111
|
+
}
|
|
5112
|
+
};
|
|
5113
|
+
}
|
|
5114
|
+
function resolveTraceContractScope(input, scope) {
|
|
5115
|
+
if (scope === void 0) {
|
|
5116
|
+
return { diagnostics: [] };
|
|
5117
|
+
}
|
|
5118
|
+
const entries = scopeEntries(scope);
|
|
5119
|
+
if (entries.length === 0) {
|
|
5120
|
+
return {
|
|
5121
|
+
diagnostics: [
|
|
5122
|
+
diagnostic(
|
|
5123
|
+
"AI_CHECK_INVALID_CONFIG",
|
|
5124
|
+
"TraceContract scope requires at least one explicit selector (runId, subAgentId, groupId, workflowStep, or rootEventId).",
|
|
5125
|
+
"contract.scope.empty"
|
|
5126
|
+
)
|
|
5127
|
+
]
|
|
5128
|
+
};
|
|
5129
|
+
}
|
|
5130
|
+
const matchedSelectors = Object.fromEntries(entries);
|
|
5131
|
+
const actorKeys = entries.filter(([key]) => key !== "rootEventId");
|
|
5132
|
+
const rootEventId = matchedSelectors.rootEventId;
|
|
5133
|
+
let candidates = [...input.read.runs];
|
|
5134
|
+
if (matchedSelectors.runId) {
|
|
5135
|
+
candidates = candidates.filter((run2) => run2.runId === matchedSelectors.runId);
|
|
5136
|
+
}
|
|
5137
|
+
for (const [key, value] of actorKeys) {
|
|
5138
|
+
if (key === "runId") continue;
|
|
5139
|
+
candidates = candidates.filter((run2) => {
|
|
5140
|
+
const meta = workflowMetadataForRun(input.read.events, run2.runId);
|
|
5141
|
+
const actual = meta[key];
|
|
5142
|
+
return actual === value;
|
|
5143
|
+
});
|
|
5144
|
+
}
|
|
5145
|
+
if (rootEventId && actorKeys.length === 0) {
|
|
5146
|
+
candidates = candidates.filter(
|
|
5147
|
+
(run2) => flattenNodes(run2.children).some((node) => node.event.eventId === rootEventId)
|
|
5148
|
+
);
|
|
5149
|
+
}
|
|
5150
|
+
if (candidates.length === 0) {
|
|
5151
|
+
return {
|
|
5152
|
+
diagnostics: [
|
|
5153
|
+
diagnostic(
|
|
5154
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
5155
|
+
`TraceContract scope matched zero actors for ${JSON.stringify(matchedSelectors)}. Missing actor metadata does not fall back to the whole session.`,
|
|
5156
|
+
"contract.scope.zero-match"
|
|
5157
|
+
)
|
|
5158
|
+
]
|
|
5159
|
+
};
|
|
5160
|
+
}
|
|
5161
|
+
if (candidates.length > 1) {
|
|
5162
|
+
return {
|
|
5163
|
+
diagnostics: [
|
|
5164
|
+
diagnostic(
|
|
5165
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
5166
|
+
`TraceContract scope matched ${candidates.length} actors for ${JSON.stringify(matchedSelectors)} (runIds: ${candidates.map((run2) => run2.runId).sort().join(", ")}). Singular actor selectors must resolve to exactly one run.`,
|
|
5167
|
+
"contract.scope.ambiguous"
|
|
5168
|
+
)
|
|
5169
|
+
]
|
|
5170
|
+
};
|
|
5171
|
+
}
|
|
5172
|
+
const run = candidates[0];
|
|
5173
|
+
let eventIds;
|
|
5174
|
+
if (rootEventId) {
|
|
5175
|
+
eventIds = collectSubtreeEventIds(run, rootEventId);
|
|
5176
|
+
if (eventIds === void 0) {
|
|
5177
|
+
return {
|
|
5178
|
+
diagnostics: [
|
|
5179
|
+
diagnostic(
|
|
5180
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
5181
|
+
`TraceContract scope rootEventId ${rootEventId} was not found in run ${run.runId}.`,
|
|
5182
|
+
"contract.scope.root-missing"
|
|
5183
|
+
)
|
|
5184
|
+
]
|
|
5185
|
+
};
|
|
5186
|
+
}
|
|
5187
|
+
}
|
|
5188
|
+
const projected = projectInput(input, run, eventIds);
|
|
5189
|
+
return {
|
|
5190
|
+
resolved: {
|
|
5191
|
+
runId: run.runId,
|
|
5192
|
+
...rootEventId !== void 0 ? { rootEventId } : {},
|
|
5193
|
+
matchedSelectors,
|
|
5194
|
+
evidenceEventCount: projected.evidenceEventCount,
|
|
5195
|
+
input: projected.input
|
|
5196
|
+
},
|
|
5197
|
+
diagnostics: []
|
|
5198
|
+
};
|
|
5199
|
+
}
|
|
5200
|
+
|
|
5201
|
+
// packages/core/src/checks/control-rules.ts
|
|
5202
|
+
function asStringList(value) {
|
|
5203
|
+
if (!Array.isArray(value)) return void 0;
|
|
5204
|
+
const out = [];
|
|
5205
|
+
for (const item of value) {
|
|
5206
|
+
if (typeof item !== "string" || item.trim() === "") return void 0;
|
|
5207
|
+
out.push(item.trim());
|
|
5208
|
+
}
|
|
5209
|
+
return out;
|
|
5210
|
+
}
|
|
5211
|
+
function attributeList(events, attribute) {
|
|
5212
|
+
if (!attribute) return void 0;
|
|
5213
|
+
for (const event of events) {
|
|
5214
|
+
const attrs = event.attributes;
|
|
5215
|
+
if (!attrs || typeof attrs !== "object") continue;
|
|
5216
|
+
const direct = asStringList(attrs[attribute]);
|
|
5217
|
+
if (direct) return direct;
|
|
5218
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? asStringList(attrs.metadata[attribute]) : void 0;
|
|
5219
|
+
if (nested) return nested;
|
|
5220
|
+
}
|
|
5221
|
+
return void 0;
|
|
5222
|
+
}
|
|
5223
|
+
function fail(ruleId, message, evidence, expected, actual) {
|
|
5224
|
+
return {
|
|
5225
|
+
ruleId,
|
|
5226
|
+
severity: "error",
|
|
5227
|
+
status: "fail",
|
|
5228
|
+
message,
|
|
5229
|
+
...expected !== void 0 ? { expected } : {},
|
|
5230
|
+
...actual !== void 0 ? { actual } : {},
|
|
5231
|
+
evidence: [...evidence]
|
|
5232
|
+
};
|
|
5233
|
+
}
|
|
5234
|
+
function sortedUnique(values) {
|
|
5235
|
+
return [...new Set(values)].sort((a, b) => a.localeCompare(b));
|
|
5236
|
+
}
|
|
5237
|
+
function setEqual(left, right) {
|
|
5238
|
+
const a = sortedUnique(left);
|
|
5239
|
+
const b = sortedUnique(right);
|
|
5240
|
+
return a.length === b.length && a.every((value, index) => value === b[index]);
|
|
5241
|
+
}
|
|
5242
|
+
function evaluateControlRules(events, rules, runEvidence2, observationNames) {
|
|
5243
|
+
const findings = [];
|
|
5244
|
+
const declared = rules.declaredTools ?? attributeList(events, rules.declaredToolsAttribute);
|
|
5245
|
+
const enforced = rules.enforcedTools ?? attributeList(events, rules.enforcedToolsAttribute);
|
|
5246
|
+
if (rules.requireDeclaredMatchesEnforced) {
|
|
5247
|
+
if (declared === void 0 || enforced === void 0) {
|
|
5248
|
+
findings.push(
|
|
5249
|
+
fail(
|
|
5250
|
+
"contract.controls.declared-matches-enforced",
|
|
5251
|
+
"Declared and enforced tool sets could not both be resolved.",
|
|
5252
|
+
runEvidence2,
|
|
5253
|
+
{ declaredDefined: declared !== void 0, enforcedDefined: enforced !== void 0 },
|
|
5254
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
5255
|
+
)
|
|
5256
|
+
);
|
|
5257
|
+
} else if (!setEqual(declared, enforced)) {
|
|
5258
|
+
findings.push(
|
|
5259
|
+
fail(
|
|
5260
|
+
"contract.controls.declared-matches-enforced",
|
|
5261
|
+
"Declared tools differ from enforced tools.",
|
|
5262
|
+
runEvidence2,
|
|
5263
|
+
sortedUnique(declared),
|
|
5264
|
+
sortedUnique(enforced)
|
|
5265
|
+
)
|
|
5266
|
+
);
|
|
5267
|
+
}
|
|
5268
|
+
}
|
|
5269
|
+
const observedTools = sortedUnique(
|
|
5270
|
+
events.filter((event) => event.kind === "TOOL" && event.status !== "running").map((event) => resolveCanonicalToolName(event))
|
|
5271
|
+
);
|
|
5272
|
+
if (rules.requireObservedWithinEnforced) {
|
|
5273
|
+
if (enforced === void 0) {
|
|
5274
|
+
findings.push(
|
|
5275
|
+
fail(
|
|
5276
|
+
"contract.controls.observed-within-enforced",
|
|
5277
|
+
"Enforced tool set unavailable for observed-within-enforced check.",
|
|
5278
|
+
runEvidence2,
|
|
5279
|
+
void 0,
|
|
5280
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
5281
|
+
)
|
|
5282
|
+
);
|
|
5283
|
+
} else {
|
|
5284
|
+
const enforcedSet = new Set(enforced);
|
|
5285
|
+
const outside = observedTools.filter((name) => !enforcedSet.has(name));
|
|
5286
|
+
if (outside.length > 0) {
|
|
5287
|
+
findings.push(
|
|
5288
|
+
fail(
|
|
5289
|
+
"contract.controls.observed-within-enforced",
|
|
5290
|
+
`Observed tools outside enforced allowlist: ${outside.join(", ")}.`,
|
|
5291
|
+
runEvidence2,
|
|
5292
|
+
sortedUnique(enforced),
|
|
5293
|
+
outside
|
|
5294
|
+
)
|
|
5295
|
+
);
|
|
5296
|
+
}
|
|
5297
|
+
}
|
|
5298
|
+
}
|
|
5299
|
+
if (rules.requireObservedWithinDeclared) {
|
|
5300
|
+
if (declared === void 0) {
|
|
5301
|
+
findings.push(
|
|
5302
|
+
fail(
|
|
5303
|
+
"contract.controls.observed-within-declared",
|
|
5304
|
+
"Declared tool set unavailable for observed-within-declared check.",
|
|
5305
|
+
runEvidence2,
|
|
5306
|
+
void 0,
|
|
5307
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
5308
|
+
)
|
|
5309
|
+
);
|
|
5310
|
+
} else {
|
|
5311
|
+
const declaredSet = new Set(declared);
|
|
5312
|
+
const outside = observedTools.filter((name) => !declaredSet.has(name));
|
|
5313
|
+
if (outside.length > 0) {
|
|
5314
|
+
findings.push(
|
|
5315
|
+
fail(
|
|
5316
|
+
"contract.controls.observed-within-declared",
|
|
5317
|
+
`Observed tools outside declared set: ${outside.join(", ")}.`,
|
|
5318
|
+
runEvidence2,
|
|
5319
|
+
sortedUnique(declared),
|
|
5320
|
+
outside
|
|
5321
|
+
)
|
|
5322
|
+
);
|
|
5323
|
+
}
|
|
5324
|
+
}
|
|
5325
|
+
}
|
|
5326
|
+
for (const stage of rules.requiredStages ?? []) {
|
|
5327
|
+
const observation = stage.observation ?? `control.${stage.stage}`;
|
|
5328
|
+
if (!observationNames.has(observation)) {
|
|
5329
|
+
findings.push(
|
|
5330
|
+
fail(
|
|
5331
|
+
`contract.controls.stage.${stage.stage}`,
|
|
5332
|
+
`Required control stage observation missing: ${observation}.`,
|
|
5333
|
+
runEvidence2,
|
|
5334
|
+
observation,
|
|
5335
|
+
[...observationNames]
|
|
5336
|
+
)
|
|
5337
|
+
);
|
|
5338
|
+
}
|
|
5339
|
+
}
|
|
5340
|
+
return findings;
|
|
5341
|
+
}
|
|
5342
|
+
|
|
5343
|
+
// packages/core/src/checks/canonical-equality.ts
|
|
5344
|
+
var DEFAULT_MAX_DEPTH = 8;
|
|
5345
|
+
var DEFAULT_MAX_BYTES = 16384;
|
|
5346
|
+
function sortKeysDeep(input, depth, maxDepth) {
|
|
5347
|
+
if (depth > maxDepth) {
|
|
5348
|
+
throw new Error("AI_CHECK_CANONICAL_DEPTH_EXCEEDED");
|
|
5349
|
+
}
|
|
5350
|
+
if (input === null || typeof input !== "object") return input;
|
|
5351
|
+
if (Array.isArray(input)) {
|
|
5352
|
+
return input.map((item) => sortKeysDeep(item, depth + 1, maxDepth));
|
|
5353
|
+
}
|
|
5354
|
+
const record = input;
|
|
5355
|
+
const out = {};
|
|
5356
|
+
for (const key of Object.keys(record).sort()) {
|
|
5357
|
+
out[key] = sortKeysDeep(record[key], depth + 1, maxDepth);
|
|
5358
|
+
}
|
|
5359
|
+
return out;
|
|
5360
|
+
}
|
|
5361
|
+
function encodeCanonicalStructured(value, limits) {
|
|
5362
|
+
const maxDepth = DEFAULT_MAX_DEPTH;
|
|
5363
|
+
const maxBytes = DEFAULT_MAX_BYTES;
|
|
5364
|
+
try {
|
|
5365
|
+
const sorted = sortKeysDeep(value, 0, maxDepth);
|
|
5366
|
+
const encoded = JSON.stringify(sorted);
|
|
5367
|
+
if (encoded === void 0) {
|
|
5368
|
+
return {
|
|
5369
|
+
ok: false,
|
|
5370
|
+
reason: "Value is not JSON-serializable for canonical comparison.",
|
|
5371
|
+
code: "AI_CHECK_CANONICAL_UNSERIALIZABLE"
|
|
5372
|
+
};
|
|
5373
|
+
}
|
|
5374
|
+
if (Buffer.byteLength(encoded, "utf8") > maxBytes) {
|
|
5375
|
+
return {
|
|
5376
|
+
ok: false,
|
|
5377
|
+
reason: "Canonical structured value exceeds bounded size limit.",
|
|
5378
|
+
code: "AI_CHECK_CANONICAL_SIZE_EXCEEDED"
|
|
5379
|
+
};
|
|
5380
|
+
}
|
|
5381
|
+
return { ok: true, value: encoded };
|
|
5382
|
+
} catch (error) {
|
|
5383
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5384
|
+
if (message === "AI_CHECK_CANONICAL_DEPTH_EXCEEDED") {
|
|
5385
|
+
return {
|
|
5386
|
+
ok: false,
|
|
5387
|
+
reason: "Canonical structured value exceeds bounded depth limit.",
|
|
5388
|
+
code: "AI_CHECK_CANONICAL_DEPTH_EXCEEDED"
|
|
5389
|
+
};
|
|
5390
|
+
}
|
|
5391
|
+
return {
|
|
5392
|
+
ok: false,
|
|
5393
|
+
reason: "Canonical structured encoding failed.",
|
|
5394
|
+
code: "AI_CHECK_CANONICAL_UNSERIALIZABLE"
|
|
5395
|
+
};
|
|
5396
|
+
}
|
|
5397
|
+
}
|
|
5398
|
+
function canonicalStructuredEqual(left, right, limits) {
|
|
5399
|
+
const a = encodeCanonicalStructured(left);
|
|
5400
|
+
if (!a.ok) return { equal: false, reason: a.reason, code: a.code };
|
|
5401
|
+
const b = encodeCanonicalStructured(right);
|
|
5402
|
+
if (!b.ok) return { equal: false, reason: b.reason, code: b.code };
|
|
5403
|
+
if (a.value !== b.value) {
|
|
5404
|
+
return {
|
|
5405
|
+
equal: false,
|
|
5406
|
+
reason: "Canonical structured values differ.",
|
|
5407
|
+
code: "AI_CHECK_CANONICAL_NOT_EQUAL"
|
|
5408
|
+
};
|
|
5409
|
+
}
|
|
5410
|
+
return { equal: true };
|
|
5411
|
+
}
|
|
5412
|
+
|
|
5413
|
+
// packages/core/src/checks/tool-arguments.ts
|
|
5414
|
+
function decodeToken(token) {
|
|
5415
|
+
return token.replace(/~1/g, "/").replace(/~0/g, "~");
|
|
5416
|
+
}
|
|
5417
|
+
function resolveJsonPointer(document, pointer) {
|
|
5418
|
+
if (pointer === "") {
|
|
5419
|
+
return { found: true, value: document };
|
|
5420
|
+
}
|
|
5421
|
+
if (!pointer.startsWith("/")) {
|
|
5422
|
+
return { found: false };
|
|
5423
|
+
}
|
|
5424
|
+
const tokens = pointer.slice(1).split("/").map(decodeToken);
|
|
5425
|
+
let current = document;
|
|
5426
|
+
for (const token of tokens) {
|
|
5427
|
+
if (current === null || current === void 0) {
|
|
5428
|
+
return { found: false };
|
|
5429
|
+
}
|
|
5430
|
+
if (Array.isArray(current)) {
|
|
5431
|
+
if (!/^(0|[1-9][0-9]*)$/.test(token)) {
|
|
5432
|
+
return { found: false };
|
|
5433
|
+
}
|
|
5434
|
+
const index = Number(token);
|
|
5435
|
+
if (index >= current.length) {
|
|
5436
|
+
return { found: false };
|
|
5437
|
+
}
|
|
5438
|
+
current = current[index];
|
|
5439
|
+
continue;
|
|
5440
|
+
}
|
|
5441
|
+
if (typeof current !== "object") {
|
|
5442
|
+
return { found: false };
|
|
5443
|
+
}
|
|
5444
|
+
const record = current;
|
|
5445
|
+
if (!Object.prototype.hasOwnProperty.call(record, token)) {
|
|
5446
|
+
return { found: false };
|
|
5447
|
+
}
|
|
5448
|
+
current = record[token];
|
|
5449
|
+
}
|
|
5450
|
+
return { found: true, value: current };
|
|
5451
|
+
}
|
|
5452
|
+
function jsonType(value) {
|
|
5453
|
+
if (value === null) return "null";
|
|
5454
|
+
if (Array.isArray(value)) return "array";
|
|
5455
|
+
return typeof value;
|
|
5456
|
+
}
|
|
5457
|
+
function valuesEqual(left, right) {
|
|
5458
|
+
return canonicalStructuredEqual(left, right).equal === true;
|
|
5459
|
+
}
|
|
5460
|
+
function isStructuredArgumentEvidence(value) {
|
|
5461
|
+
return value !== void 0 && value !== null && typeof value === "object";
|
|
5462
|
+
}
|
|
5463
|
+
function extractToolArgumentPayload(event) {
|
|
5464
|
+
const attrs = event.attributes ?? {};
|
|
5465
|
+
for (const key of ["arguments", "input", "toolArguments"]) {
|
|
5466
|
+
const candidate = attrs[key];
|
|
5467
|
+
if (isStructuredArgumentEvidence(candidate)) {
|
|
5468
|
+
return { present: true, value: candidate };
|
|
5469
|
+
}
|
|
5470
|
+
}
|
|
5471
|
+
const metadata = attrs.metadata;
|
|
5472
|
+
if (isStructuredArgumentEvidence(metadata) && !Array.isArray(metadata)) {
|
|
5473
|
+
const nested = metadata;
|
|
5474
|
+
for (const key of ["arguments", "input", "toolArguments"]) {
|
|
5475
|
+
const candidate = nested[key];
|
|
5476
|
+
if (isStructuredArgumentEvidence(candidate)) {
|
|
5477
|
+
return { present: true, value: candidate };
|
|
5478
|
+
}
|
|
5479
|
+
}
|
|
5480
|
+
}
|
|
5481
|
+
if (isStructuredArgumentEvidence(event.inputSummary)) {
|
|
5482
|
+
return { present: true, value: event.inputSummary };
|
|
5483
|
+
}
|
|
5484
|
+
return { present: false, value: void 0 };
|
|
5485
|
+
}
|
|
5486
|
+
function evaluateToolArgumentValue(value, check, evidencePresent) {
|
|
5487
|
+
if (!evidencePresent) {
|
|
5488
|
+
return {
|
|
5489
|
+
status: "unavailable",
|
|
5490
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
5491
|
+
message: `Structured argument evidence unavailable for tool ${check.tool} at ${check.path}.`
|
|
5492
|
+
};
|
|
5493
|
+
}
|
|
5494
|
+
const resolved = resolveJsonPointer(value, check.path);
|
|
5495
|
+
if (check.operator === "exists") {
|
|
5496
|
+
return resolved.found ? { status: "pass" } : {
|
|
5497
|
+
status: "fail",
|
|
5498
|
+
code: "AI_CHECK_TOOL_ARGUMENT_MISSING",
|
|
5499
|
+
message: `Expected path ${check.path} to exist on tool ${check.tool}.`
|
|
5500
|
+
};
|
|
5501
|
+
}
|
|
5502
|
+
if (!resolved.found) {
|
|
5503
|
+
return {
|
|
5504
|
+
status: "unavailable",
|
|
5505
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
5506
|
+
message: `Path ${check.path} missing for tool ${check.tool}.`
|
|
5507
|
+
};
|
|
5508
|
+
}
|
|
5509
|
+
if (check.operator === "type") {
|
|
5510
|
+
const actual = jsonType(resolved.value);
|
|
5511
|
+
if (check.type === void 0) {
|
|
5512
|
+
return {
|
|
5513
|
+
status: "fail",
|
|
5514
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
5515
|
+
message: "type operator requires check.type"
|
|
5516
|
+
};
|
|
5517
|
+
}
|
|
5518
|
+
return actual === check.type ? { status: "pass" } : {
|
|
5519
|
+
status: "fail",
|
|
5520
|
+
code: "AI_CHECK_TOOL_ARGUMENT_TYPE",
|
|
5521
|
+
message: `Expected type ${check.type} at ${check.path} for tool ${check.tool}; found ${actual}.`
|
|
5522
|
+
};
|
|
5523
|
+
}
|
|
5524
|
+
if (check.operator === "equals") {
|
|
5525
|
+
return valuesEqual(resolved.value, check.expected) ? { status: "pass" } : {
|
|
5526
|
+
status: "fail",
|
|
5527
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EQUALS",
|
|
5528
|
+
message: `Value at ${check.path} for tool ${check.tool} did not equal expected (bounded comparison).`
|
|
5529
|
+
};
|
|
5530
|
+
}
|
|
5531
|
+
if (check.operator === "oneOf") {
|
|
5532
|
+
const options = check.oneOf ?? [];
|
|
5533
|
+
if (options.length === 0) {
|
|
5534
|
+
return {
|
|
5535
|
+
status: "fail",
|
|
5536
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
5537
|
+
message: "oneOf operator requires a non-empty oneOf array"
|
|
5538
|
+
};
|
|
5539
|
+
}
|
|
5540
|
+
return options.some((option) => valuesEqual(resolved.value, option)) ? { status: "pass" } : {
|
|
5541
|
+
status: "fail",
|
|
5542
|
+
code: "AI_CHECK_TOOL_ARGUMENT_ONE_OF",
|
|
5543
|
+
message: `Value at ${check.path} for tool ${check.tool} was not in the allowed oneOf set.`
|
|
5544
|
+
};
|
|
5545
|
+
}
|
|
5546
|
+
return {
|
|
5547
|
+
status: "fail",
|
|
5548
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
5549
|
+
message: `Unsupported operator.`
|
|
5550
|
+
};
|
|
5551
|
+
}
|
|
5552
|
+
|
|
5553
|
+
// packages/core/src/checks/retry-safety.ts
|
|
5554
|
+
function fail2(ruleId, message, evidence, expected, actual) {
|
|
5555
|
+
return {
|
|
5556
|
+
ruleId,
|
|
5557
|
+
severity: "error",
|
|
5558
|
+
status: "fail",
|
|
5559
|
+
message,
|
|
5560
|
+
...expected !== void 0 ? { expected } : {},
|
|
5561
|
+
...actual !== void 0 ? { actual } : {},
|
|
5562
|
+
evidence: [...evidence]
|
|
5563
|
+
};
|
|
5564
|
+
}
|
|
5565
|
+
function workflowFor(event) {
|
|
5566
|
+
const attrs = event.attributes;
|
|
5567
|
+
if (!attrs || typeof attrs !== "object") return {};
|
|
5568
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
5569
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
5570
|
+
return { ...nested, ...direct };
|
|
5571
|
+
}
|
|
5572
|
+
function hasIdempotencyEvidence(event) {
|
|
5573
|
+
const workflow = workflowFor(event);
|
|
5574
|
+
if (typeof workflow.idempotencyKey === "string" && workflow.idempotencyKey.trim() !== "") {
|
|
5575
|
+
return true;
|
|
5576
|
+
}
|
|
5577
|
+
const attrs = event.attributes ?? {};
|
|
5578
|
+
if (attrs.noSideEffect === true) return true;
|
|
5579
|
+
if (attrs.sideEffect === false) return true;
|
|
5580
|
+
return false;
|
|
5581
|
+
}
|
|
5582
|
+
function eventEvidence(event) {
|
|
5583
|
+
return {
|
|
5584
|
+
runId: event.runId,
|
|
5585
|
+
eventId: event.eventId,
|
|
5586
|
+
kind: event.kind,
|
|
5587
|
+
name: event.name,
|
|
5588
|
+
status: event.status
|
|
5589
|
+
};
|
|
5590
|
+
}
|
|
5591
|
+
function eventTime(event) {
|
|
5592
|
+
return event.startedAt ?? event.timestamp ?? "";
|
|
5593
|
+
}
|
|
5594
|
+
function sortByTime(events) {
|
|
5595
|
+
return [...events].sort((a, b) => eventTime(a).localeCompare(eventTime(b)));
|
|
5596
|
+
}
|
|
5597
|
+
function attemptNumberOf(event) {
|
|
5598
|
+
const workflow = workflowFor(event);
|
|
5599
|
+
const value = workflow.attemptNumber ?? workflow.attempt;
|
|
5600
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
5601
|
+
}
|
|
5602
|
+
function attemptIdOf(event) {
|
|
5603
|
+
const value = workflowFor(event).attemptId;
|
|
5604
|
+
return typeof value === "string" && value.trim() !== "" ? value : void 0;
|
|
5605
|
+
}
|
|
5606
|
+
function countOperationAttempts(members) {
|
|
5607
|
+
const attemptIds = new Set(
|
|
5608
|
+
members.map((event) => attemptIdOf(event)).filter((value) => value !== void 0)
|
|
5609
|
+
);
|
|
5610
|
+
if (attemptIds.size > 0) {
|
|
5611
|
+
return attemptIds.size;
|
|
5612
|
+
}
|
|
5613
|
+
const numbers = members.map((event) => attemptNumberOf(event)).filter((value) => value !== void 0).sort((a, b) => a - b);
|
|
5614
|
+
if (numbers.length > 0) {
|
|
5615
|
+
const unique = [...new Set(numbers)];
|
|
5616
|
+
const min = unique[0];
|
|
5617
|
+
const max = unique[unique.length - 1];
|
|
5618
|
+
const contiguous = unique.length === max - min + 1 && unique.every((value, index) => value === min + index);
|
|
5619
|
+
if (contiguous) {
|
|
5620
|
+
return unique.length;
|
|
5621
|
+
}
|
|
5622
|
+
return unique.length;
|
|
5623
|
+
}
|
|
5624
|
+
return members.filter((event) => event.kind === "TOOL" || event.kind === "LLM").length;
|
|
5625
|
+
}
|
|
5626
|
+
function classifyRetryAttempt(event, priorOrdered, allMembers) {
|
|
5627
|
+
const workflow = workflowFor(event);
|
|
5628
|
+
const attemptNumber = attemptNumberOf(event);
|
|
5629
|
+
const attemptId = attemptIdOf(event);
|
|
5630
|
+
const retryOf = workflow.retryOf;
|
|
5631
|
+
if (typeof retryOf === "string" && retryOf.trim() !== "") {
|
|
5632
|
+
const target = allMembers.find(
|
|
5633
|
+
(candidate) => candidate.eventId === retryOf || attemptIdOf(candidate) === retryOf || workflowFor(candidate).operationId === retryOf
|
|
5634
|
+
);
|
|
5635
|
+
if (!target) {
|
|
5636
|
+
return { kind: "unknown", reason: "retryOf target missing" };
|
|
5637
|
+
}
|
|
5638
|
+
if (eventTime(target) > eventTime(event)) {
|
|
5639
|
+
return { kind: "unknown", reason: "retryOf target does not precede retry" };
|
|
5640
|
+
}
|
|
5641
|
+
return { kind: "retry" };
|
|
5642
|
+
}
|
|
5643
|
+
if (attemptNumber !== void 0) {
|
|
5644
|
+
if (attemptNumber > 1) {
|
|
5645
|
+
return { kind: "retry" };
|
|
5646
|
+
}
|
|
5647
|
+
if (attemptNumber === 1) {
|
|
5648
|
+
return { kind: "first" };
|
|
5649
|
+
}
|
|
5650
|
+
}
|
|
5651
|
+
if (attemptId !== void 0) {
|
|
5652
|
+
const priorDistinct = priorOrdered.some((candidate) => {
|
|
5653
|
+
const priorId = attemptIdOf(candidate);
|
|
5654
|
+
return priorId !== void 0 && priorId !== attemptId;
|
|
5655
|
+
});
|
|
5656
|
+
if (priorDistinct) {
|
|
5657
|
+
return { kind: "retry" };
|
|
5658
|
+
}
|
|
5659
|
+
if (priorOrdered.length === 0) {
|
|
5660
|
+
return { kind: "first" };
|
|
5661
|
+
}
|
|
5662
|
+
}
|
|
5663
|
+
const priorFinished = priorOrdered.filter(
|
|
5664
|
+
(candidate) => candidate.status === "ok" || candidate.status === "error"
|
|
5665
|
+
);
|
|
5666
|
+
if (priorFinished.length > 0) {
|
|
5667
|
+
return { kind: "retry" };
|
|
5668
|
+
}
|
|
5669
|
+
if (priorOrdered.length === 0) {
|
|
5670
|
+
return { kind: "first" };
|
|
5671
|
+
}
|
|
5672
|
+
return { kind: "unknown", reason: "ambiguous attempt identity" };
|
|
5673
|
+
}
|
|
5674
|
+
function hasDuplicateAttemptIds(members) {
|
|
5675
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5676
|
+
for (const event of members) {
|
|
5677
|
+
const id = attemptIdOf(event);
|
|
5678
|
+
if (!id) continue;
|
|
5679
|
+
if (seen.has(id)) return true;
|
|
5680
|
+
seen.add(id);
|
|
5681
|
+
}
|
|
5682
|
+
return false;
|
|
5683
|
+
}
|
|
5684
|
+
function evaluateRetrySafetyRules(events, rules, _runEvidence) {
|
|
5685
|
+
const findings = [];
|
|
5686
|
+
const byOperation = /* @__PURE__ */ new Map();
|
|
5687
|
+
for (const event of events) {
|
|
5688
|
+
const workflow = workflowFor(event);
|
|
5689
|
+
const operationId = workflow.operationId;
|
|
5690
|
+
if (!operationId) continue;
|
|
5691
|
+
const list = byOperation.get(operationId) ?? [];
|
|
5692
|
+
list.push(event);
|
|
5693
|
+
byOperation.set(operationId, list);
|
|
5694
|
+
}
|
|
5695
|
+
if (rules.maxAttempts !== void 0) {
|
|
5696
|
+
for (const [operationId, members] of byOperation) {
|
|
5697
|
+
if (hasDuplicateAttemptIds(members)) {
|
|
5698
|
+
findings.push(
|
|
5699
|
+
fail2(
|
|
5700
|
+
"contract.retry.duplicate-attempt-id",
|
|
5701
|
+
`Operation ${operationId} has duplicate attemptId values.`,
|
|
5702
|
+
members.slice(0, 4).map(eventEvidence),
|
|
5703
|
+
"unique attemptId",
|
|
5704
|
+
"duplicate"
|
|
5705
|
+
)
|
|
5706
|
+
);
|
|
5707
|
+
}
|
|
5708
|
+
const count = countOperationAttempts(members);
|
|
5709
|
+
if (count > rules.maxAttempts) {
|
|
5710
|
+
findings.push(
|
|
5711
|
+
fail2(
|
|
5712
|
+
"contract.retry.max-attempts",
|
|
5713
|
+
`Operation ${operationId} exceeded maxAttempts ${rules.maxAttempts}.`,
|
|
5714
|
+
members.slice(0, 4).map(eventEvidence),
|
|
5715
|
+
rules.maxAttempts,
|
|
5716
|
+
count
|
|
5717
|
+
)
|
|
5718
|
+
);
|
|
5719
|
+
}
|
|
5720
|
+
}
|
|
5721
|
+
}
|
|
5722
|
+
if (rules.requireTerminalResult) {
|
|
5723
|
+
for (const [operationId, members] of byOperation) {
|
|
5724
|
+
const hasTerminal = members.some((event) => event.status === "ok" || event.status === "error");
|
|
5725
|
+
if (!hasTerminal) {
|
|
5726
|
+
findings.push(
|
|
5727
|
+
fail2(
|
|
5728
|
+
"contract.retry.terminal-result",
|
|
5729
|
+
`Operation ${operationId} has no terminal ok/error result.`,
|
|
5730
|
+
members.slice(0, 4).map(eventEvidence),
|
|
5731
|
+
"ok|error",
|
|
5732
|
+
members.map((event) => event.status)
|
|
5733
|
+
)
|
|
5734
|
+
);
|
|
5735
|
+
}
|
|
5736
|
+
}
|
|
5737
|
+
}
|
|
5738
|
+
if (rules.fallbackOnlyAfterFailure) {
|
|
5739
|
+
for (const event of events) {
|
|
5740
|
+
const workflow = workflowFor(event);
|
|
5741
|
+
const fallbackOf = workflow.fallbackOf;
|
|
5742
|
+
if (!fallbackOf) continue;
|
|
5743
|
+
const prior = byOperation.get(fallbackOf) ?? [];
|
|
5744
|
+
const fallbackStart = eventTime(event);
|
|
5745
|
+
const earlierFailure = prior.some(
|
|
5746
|
+
(candidate) => candidate.status === "error" && eventTime(candidate) !== "" && fallbackStart !== "" && eventTime(candidate) < fallbackStart
|
|
5747
|
+
);
|
|
5748
|
+
if (!earlierFailure) {
|
|
5749
|
+
findings.push(
|
|
5750
|
+
fail2(
|
|
5751
|
+
"contract.retry.fallback-after-failure",
|
|
5752
|
+
`Fallback for ${fallbackOf} appeared without an earlier failure in the related operation.`,
|
|
5753
|
+
[eventEvidence(event)],
|
|
5754
|
+
"earlier error attempt",
|
|
5755
|
+
{ fallbackOf }
|
|
5756
|
+
)
|
|
5757
|
+
);
|
|
5758
|
+
}
|
|
5759
|
+
}
|
|
5760
|
+
}
|
|
5761
|
+
const nonIdempotent = new Set(rules.nonIdempotentTools ?? []);
|
|
5762
|
+
if (nonIdempotent.size > 0 || rules.requireIdempotencyEvidenceForRetry) {
|
|
5763
|
+
const toolEvents = events.filter(
|
|
5764
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
5765
|
+
);
|
|
5766
|
+
const byToolOp = /* @__PURE__ */ new Map();
|
|
5767
|
+
for (const event of toolEvents) {
|
|
5768
|
+
const name = resolveCanonicalToolName(event);
|
|
5769
|
+
const workflow = workflowFor(event);
|
|
5770
|
+
const key = `${workflow.operationId ?? name}::${name}`;
|
|
5771
|
+
const list = byToolOp.get(key) ?? [];
|
|
5772
|
+
list.push(event);
|
|
5773
|
+
byToolOp.set(key, list);
|
|
5774
|
+
}
|
|
5775
|
+
for (const [, members] of byToolOp) {
|
|
5776
|
+
const ordered = sortByTime(members);
|
|
5777
|
+
let sawOk = false;
|
|
5778
|
+
let sawSideEffectOk = false;
|
|
5779
|
+
const prior = [];
|
|
5780
|
+
for (const event of ordered) {
|
|
5781
|
+
const name = resolveCanonicalToolName(event);
|
|
5782
|
+
const classification = classifyRetryAttempt(event, prior, members);
|
|
5783
|
+
if (classification.kind === "unknown" && rules.requireIdempotencyEvidenceForRetry) {
|
|
5784
|
+
findings.push(
|
|
5785
|
+
fail2(
|
|
5786
|
+
"contract.retry.ambiguous-identity",
|
|
5787
|
+
`Ambiguous attempt identity for tool ${name}; strict retry evidence cannot pass.`,
|
|
5788
|
+
[eventEvidence(event)],
|
|
5789
|
+
"explicit attempt identity",
|
|
5790
|
+
classification.reason
|
|
5791
|
+
)
|
|
5792
|
+
);
|
|
5793
|
+
}
|
|
5794
|
+
const isRetry = classification.kind === "retry";
|
|
5795
|
+
if (isRetry && rules.requireIdempotencyEvidenceForRetry && !hasIdempotencyEvidence(event)) {
|
|
5796
|
+
findings.push(
|
|
5797
|
+
fail2(
|
|
5798
|
+
"contract.retry.idempotency-evidence",
|
|
5799
|
+
`Retry of tool ${name} lacks idempotencyKey / noSideEffect evidence.`,
|
|
5800
|
+
[eventEvidence(event)],
|
|
5801
|
+
"idempotencyKey|noSideEffect",
|
|
5802
|
+
{ code: "AI_CHECK_RETRY_EVIDENCE_UNAVAILABLE" }
|
|
5803
|
+
)
|
|
5804
|
+
);
|
|
5805
|
+
}
|
|
5806
|
+
if (sawOk && nonIdempotent.has(name) && sawSideEffectOk && !hasIdempotencyEvidence(event)) {
|
|
5807
|
+
findings.push(
|
|
5808
|
+
fail2(
|
|
5809
|
+
"contract.retry.non-idempotent-side-effect",
|
|
5810
|
+
`Retry of non-idempotent tool ${name} after a confirmed ok side effect.`,
|
|
5811
|
+
[eventEvidence(event)],
|
|
5812
|
+
"no retry after side effect",
|
|
5813
|
+
name
|
|
5814
|
+
)
|
|
5815
|
+
);
|
|
5816
|
+
}
|
|
5817
|
+
if (event.status === "ok") {
|
|
5818
|
+
sawOk = true;
|
|
5819
|
+
if (nonIdempotent.has(name) && !hasIdempotencyEvidence(event)) {
|
|
5820
|
+
sawSideEffectOk = true;
|
|
5821
|
+
}
|
|
5822
|
+
}
|
|
5823
|
+
prior.push(event);
|
|
5824
|
+
}
|
|
5825
|
+
}
|
|
5826
|
+
}
|
|
5827
|
+
if (rules.requireRecoveredFailureVisible) {
|
|
5828
|
+
for (const [operationId, members] of byOperation) {
|
|
5829
|
+
const ordered = sortByTime(members);
|
|
5830
|
+
const toolish = ordered.filter(
|
|
5831
|
+
(event) => (event.kind === "TOOL" || event.kind === "LLM") && (event.status === "ok" || event.status === "error")
|
|
5832
|
+
);
|
|
5833
|
+
if (toolish.length < 2) continue;
|
|
5834
|
+
let sawEarlierError = false;
|
|
5835
|
+
let sawLaterOkAfterError = false;
|
|
5836
|
+
for (const event of toolish) {
|
|
5837
|
+
if (event.status === "error") {
|
|
5838
|
+
sawEarlierError = true;
|
|
5839
|
+
} else if (event.status === "ok" && sawEarlierError) {
|
|
5840
|
+
sawLaterOkAfterError = true;
|
|
5841
|
+
}
|
|
5842
|
+
}
|
|
5843
|
+
const hasOk = toolish.some((event) => event.status === "ok");
|
|
5844
|
+
const attemptish = toolish.some((event) => {
|
|
5845
|
+
const meta = workflowFor(event);
|
|
5846
|
+
return meta.attemptId !== void 0 || meta.attemptNumber !== void 0 || meta.attempt !== void 0 || meta.retryOf !== void 0;
|
|
5847
|
+
}) || toolish.length > 1;
|
|
5848
|
+
if (!attemptish || !hasOk) continue;
|
|
5849
|
+
if (!sawLaterOkAfterError) {
|
|
5850
|
+
findings.push(
|
|
5851
|
+
fail2(
|
|
5852
|
+
"contract.retry.recovered-failure-visible",
|
|
5853
|
+
`Operation ${operationId} does not show an earlier error attempt before a later success in the same chain.`,
|
|
5854
|
+
members.slice(0, 4).map(eventEvidence),
|
|
5855
|
+
"earlier error then later ok",
|
|
5856
|
+
{
|
|
5857
|
+
sawEarlierError,
|
|
5858
|
+
statuses: toolish.map((event) => event.status)
|
|
5859
|
+
}
|
|
5860
|
+
)
|
|
5861
|
+
);
|
|
5862
|
+
}
|
|
5863
|
+
}
|
|
5864
|
+
}
|
|
5865
|
+
return findings;
|
|
5866
|
+
}
|
|
5867
|
+
|
|
5868
|
+
// packages/core/src/checks/recovery-operations.ts
|
|
5869
|
+
function fail3(ruleId, message, evidence, expected, actual) {
|
|
5870
|
+
return {
|
|
5871
|
+
ruleId,
|
|
5872
|
+
severity: "error",
|
|
5873
|
+
status: "fail",
|
|
5874
|
+
message,
|
|
5875
|
+
...expected !== void 0 ? { expected } : {},
|
|
5876
|
+
...actual !== void 0 ? { actual } : {},
|
|
5877
|
+
evidence: [...evidence]
|
|
5878
|
+
};
|
|
5879
|
+
}
|
|
5880
|
+
function workflowFor2(event) {
|
|
5881
|
+
const attrs = event.attributes;
|
|
5882
|
+
if (!attrs || typeof attrs !== "object") return {};
|
|
5883
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
5884
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
5885
|
+
return { ...nested, ...direct };
|
|
5886
|
+
}
|
|
5887
|
+
function eventEvidence2(event) {
|
|
5888
|
+
return {
|
|
5889
|
+
runId: event.runId,
|
|
5890
|
+
eventId: event.eventId,
|
|
5891
|
+
kind: event.kind,
|
|
5892
|
+
name: event.name,
|
|
5893
|
+
status: event.status
|
|
5894
|
+
};
|
|
5895
|
+
}
|
|
5896
|
+
function eventTime2(event) {
|
|
5897
|
+
return event.startedAt ?? event.timestamp ?? "";
|
|
5898
|
+
}
|
|
5899
|
+
function sortByTime2(events) {
|
|
5900
|
+
return [...events].sort((a, b) => eventTime2(a).localeCompare(eventTime2(b)));
|
|
5901
|
+
}
|
|
5902
|
+
function errorCodeOf(event) {
|
|
5903
|
+
if (typeof event.error?.code === "string" && event.error.code.trim() !== "") {
|
|
5904
|
+
return event.error.code.trim();
|
|
5905
|
+
}
|
|
5906
|
+
const attrs = event.attributes ?? {};
|
|
5907
|
+
for (const key of ["errorCode", "code"]) {
|
|
5908
|
+
const value = attrs[key];
|
|
5909
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
5910
|
+
}
|
|
5911
|
+
const nested = attrs.error;
|
|
5912
|
+
if (nested && typeof nested === "object") {
|
|
5913
|
+
const code = nested.code;
|
|
5914
|
+
if (typeof code === "string" && code.trim() !== "") return code.trim();
|
|
5915
|
+
}
|
|
5916
|
+
return void 0;
|
|
5917
|
+
}
|
|
5918
|
+
function argumentDigestOf(event) {
|
|
5919
|
+
const attrs = event.attributes ?? {};
|
|
5920
|
+
for (const key of ["argumentsDigest", "inputDigest", "toolArgumentsDigest"]) {
|
|
5921
|
+
const value = attrs[key];
|
|
5922
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
5923
|
+
}
|
|
5924
|
+
const commitment = attrs.omittedPayloadCommitment;
|
|
5925
|
+
if (commitment && typeof commitment === "object") {
|
|
5926
|
+
const digest = commitment.digest;
|
|
5927
|
+
if (typeof digest === "string" && digest.trim() !== "") return digest.trim();
|
|
5928
|
+
}
|
|
5929
|
+
return void 0;
|
|
5930
|
+
}
|
|
5931
|
+
function argumentFingerprint(event) {
|
|
5932
|
+
const payload = extractToolArgumentPayload(event);
|
|
5933
|
+
if (payload.present) {
|
|
5934
|
+
const encoded = encodeCanonicalStructured(payload.value);
|
|
5935
|
+
if (!encoded.ok) return { kind: "missing" };
|
|
5936
|
+
return { kind: "structured", value: encoded.value };
|
|
5937
|
+
}
|
|
5938
|
+
const digest = argumentDigestOf(event);
|
|
5939
|
+
if (digest) return { kind: "digest", value: digest };
|
|
5940
|
+
return { kind: "missing" };
|
|
5941
|
+
}
|
|
5942
|
+
function sameArguments(left, right) {
|
|
5943
|
+
const a = argumentFingerprint(left);
|
|
5944
|
+
const b = argumentFingerprint(right);
|
|
5945
|
+
if (a.kind === "missing" || b.kind === "missing") {
|
|
5946
|
+
return {
|
|
5947
|
+
ok: false,
|
|
5948
|
+
reason: "Structured or digest argument evidence unavailable for same-arguments check.",
|
|
5949
|
+
code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_UNAVAILABLE"
|
|
5950
|
+
};
|
|
5951
|
+
}
|
|
5952
|
+
if (a.kind !== b.kind) {
|
|
5953
|
+
return {
|
|
5954
|
+
ok: false,
|
|
5955
|
+
reason: "Argument evidence kinds differ across attempts (structured vs digest).",
|
|
5956
|
+
code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_MISMATCH"
|
|
5957
|
+
};
|
|
5958
|
+
}
|
|
5959
|
+
if (a.value !== b.value) {
|
|
5960
|
+
return {
|
|
5961
|
+
ok: false,
|
|
5962
|
+
reason: "Retry arguments do not match prior attempt.",
|
|
5963
|
+
code: "AI_CHECK_RECOVERY_ARGUMENTS_DIFFER"
|
|
5964
|
+
};
|
|
5965
|
+
}
|
|
5966
|
+
return { ok: true };
|
|
5967
|
+
}
|
|
5968
|
+
function isUnevaluableWriteCompletion(event) {
|
|
5969
|
+
if (event.status === "running" || event.status === "unknown") return true;
|
|
5970
|
+
const attrs = event.attributes ?? {};
|
|
5971
|
+
if (attrs.timeout === true) return true;
|
|
5972
|
+
if (attrs.completionState === "unknown" || attrs.completionState === "timeout") return true;
|
|
5973
|
+
const code = errorCodeOf(event)?.toUpperCase();
|
|
5974
|
+
if (code === "TIMEOUT" || code === "UNKNOWN_COMPLETION" || code === "DEADLINE_EXCEEDED") {
|
|
5975
|
+
return true;
|
|
5976
|
+
}
|
|
5977
|
+
return false;
|
|
5978
|
+
}
|
|
5979
|
+
function collectExplicitReferences(event) {
|
|
5980
|
+
const refs = /* @__PURE__ */ new Set();
|
|
5981
|
+
const attrs = event.attributes ?? {};
|
|
5982
|
+
const candidates = [
|
|
5983
|
+
attrs.referencedEventId,
|
|
5984
|
+
attrs.referencedEventIds,
|
|
5985
|
+
attrs.toolResultEventId,
|
|
5986
|
+
attrs.toolResultEventIds,
|
|
5987
|
+
attrs.inputEventIds,
|
|
5988
|
+
attrs.dependsOnEventIds,
|
|
5989
|
+
attrs.evidence,
|
|
5990
|
+
workflowFor2(event).toolCallId
|
|
5991
|
+
];
|
|
5992
|
+
for (const candidate of candidates) {
|
|
5993
|
+
if (typeof candidate === "string" && candidate.trim() !== "") {
|
|
5994
|
+
refs.add(candidate.trim());
|
|
5995
|
+
continue;
|
|
5996
|
+
}
|
|
5997
|
+
if (Array.isArray(candidate)) {
|
|
5998
|
+
for (const item of candidate) {
|
|
5999
|
+
if (typeof item === "string" && item.trim() !== "") refs.add(item.trim());
|
|
6000
|
+
if (item && typeof item === "object") {
|
|
6001
|
+
const id = item.eventId;
|
|
6002
|
+
if (typeof id === "string" && id.trim() !== "") refs.add(id.trim());
|
|
6003
|
+
}
|
|
6004
|
+
}
|
|
6005
|
+
continue;
|
|
6006
|
+
}
|
|
6007
|
+
if (candidate && typeof candidate === "object") {
|
|
6008
|
+
const obj = candidate;
|
|
6009
|
+
if (typeof obj.eventId === "string" && obj.eventId.trim() !== "") {
|
|
6010
|
+
refs.add(obj.eventId.trim());
|
|
6011
|
+
}
|
|
6012
|
+
if (Array.isArray(obj.eventIds)) {
|
|
6013
|
+
for (const id of obj.eventIds) {
|
|
6014
|
+
if (typeof id === "string" && id.trim() !== "") refs.add(id.trim());
|
|
6015
|
+
}
|
|
6016
|
+
}
|
|
6017
|
+
}
|
|
6018
|
+
}
|
|
6019
|
+
return refs;
|
|
6020
|
+
}
|
|
6021
|
+
function llmReferencesTool(llm, toolEvent, requireExplicit) {
|
|
6022
|
+
if (!requireExplicit) {
|
|
6023
|
+
return eventTime2(llm) >= eventTime2(toolEvent);
|
|
6024
|
+
}
|
|
6025
|
+
const refs = collectExplicitReferences(llm);
|
|
6026
|
+
if (refs.has(toolEvent.eventId)) return true;
|
|
6027
|
+
const toolCallId = workflowFor2(toolEvent).toolCallId;
|
|
6028
|
+
if (toolCallId && refs.has(toolCallId)) return true;
|
|
6029
|
+
return false;
|
|
6030
|
+
}
|
|
6031
|
+
function attemptIdOf2(event) {
|
|
6032
|
+
const value = workflowFor2(event).attemptId;
|
|
6033
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
6034
|
+
}
|
|
6035
|
+
function hasDuplicateAttemptIds2(members) {
|
|
6036
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6037
|
+
for (const event of members) {
|
|
6038
|
+
const id = attemptIdOf2(event);
|
|
6039
|
+
if (!id) continue;
|
|
6040
|
+
if (seen.has(id)) return true;
|
|
6041
|
+
seen.add(id);
|
|
6042
|
+
}
|
|
6043
|
+
return false;
|
|
6044
|
+
}
|
|
6045
|
+
function findRetryTarget(members, retryOf) {
|
|
6046
|
+
return members.find(
|
|
6047
|
+
(candidate) => candidate.eventId === retryOf || attemptIdOf2(candidate) === retryOf || workflowFor2(candidate).operationId === retryOf
|
|
6048
|
+
);
|
|
6049
|
+
}
|
|
6050
|
+
function groupToolAttempts(events, toolName2) {
|
|
6051
|
+
const members = events.filter(
|
|
6052
|
+
(event) => event.kind === "TOOL" && resolveCanonicalToolName(event) === toolName2
|
|
6053
|
+
);
|
|
6054
|
+
const parent = /* @__PURE__ */ new Map();
|
|
6055
|
+
const ensure = (id) => {
|
|
6056
|
+
if (!parent.has(id)) parent.set(id, id);
|
|
6057
|
+
return id;
|
|
6058
|
+
};
|
|
6059
|
+
const find = (id) => {
|
|
6060
|
+
ensure(id);
|
|
6061
|
+
let cur = id;
|
|
6062
|
+
while (parent.get(cur) !== cur) {
|
|
6063
|
+
cur = parent.get(cur);
|
|
6064
|
+
}
|
|
6065
|
+
let walk = id;
|
|
6066
|
+
while (parent.get(walk) !== cur) {
|
|
6067
|
+
const next = parent.get(walk);
|
|
6068
|
+
parent.set(walk, cur);
|
|
6069
|
+
walk = next;
|
|
6070
|
+
}
|
|
6071
|
+
return cur;
|
|
6072
|
+
};
|
|
6073
|
+
const union = (a, b) => {
|
|
6074
|
+
const ra = find(a);
|
|
6075
|
+
const rb = find(b);
|
|
6076
|
+
if (ra !== rb) parent.set(ra, rb);
|
|
6077
|
+
};
|
|
6078
|
+
const nodeId = (event) => {
|
|
6079
|
+
const operationId = workflowFor2(event).operationId;
|
|
6080
|
+
if (typeof operationId === "string" && operationId.trim() !== "") {
|
|
6081
|
+
return `op:${operationId.trim()}`;
|
|
6082
|
+
}
|
|
6083
|
+
return `event:${event.eventId}`;
|
|
6084
|
+
};
|
|
6085
|
+
for (const event of members) {
|
|
6086
|
+
ensure(nodeId(event));
|
|
6087
|
+
}
|
|
6088
|
+
for (const event of members) {
|
|
6089
|
+
const retryOf = workflowFor2(event).retryOf;
|
|
6090
|
+
if (typeof retryOf !== "string" || retryOf.trim() === "") continue;
|
|
6091
|
+
const target = findRetryTarget(members, retryOf.trim());
|
|
6092
|
+
if (target) {
|
|
6093
|
+
union(nodeId(event), nodeId(target));
|
|
6094
|
+
}
|
|
6095
|
+
}
|
|
6096
|
+
const groups = /* @__PURE__ */ new Map();
|
|
6097
|
+
for (const event of members) {
|
|
6098
|
+
const key = find(nodeId(event));
|
|
6099
|
+
const list = groups.get(key) ?? [];
|
|
6100
|
+
list.push(event);
|
|
6101
|
+
groups.set(key, list);
|
|
6102
|
+
}
|
|
6103
|
+
return groups;
|
|
6104
|
+
}
|
|
6105
|
+
function latestAttempt(ordered) {
|
|
6106
|
+
if (ordered.length === 0) return void 0;
|
|
6107
|
+
return ordered[ordered.length - 1];
|
|
6108
|
+
}
|
|
6109
|
+
function evaluateRecoveryOperations(events, operations) {
|
|
6110
|
+
const findings = [];
|
|
6111
|
+
const llmEvents = events.filter((event) => event.kind === "LLM");
|
|
6112
|
+
for (const operation of operations) {
|
|
6113
|
+
const sideEffectClass = operation.sideEffectClass ?? "read";
|
|
6114
|
+
const groups = groupToolAttempts(events, operation.tool);
|
|
6115
|
+
if (groups.size === 0) {
|
|
6116
|
+
if (operation.requireTerminalSuccess) {
|
|
6117
|
+
findings.push(
|
|
6118
|
+
fail3(
|
|
6119
|
+
"contract.retry.operations.terminal-success",
|
|
6120
|
+
`Recovery operation for tool ${operation.tool} requires a terminal success but no attempts were observed.`,
|
|
6121
|
+
[],
|
|
6122
|
+
"ok",
|
|
6123
|
+
"missing"
|
|
6124
|
+
)
|
|
6125
|
+
);
|
|
6126
|
+
}
|
|
6127
|
+
continue;
|
|
6128
|
+
}
|
|
6129
|
+
for (const [operationKey, members] of groups) {
|
|
6130
|
+
const ordered = sortByTime2(members);
|
|
6131
|
+
if (hasDuplicateAttemptIds2(members)) {
|
|
6132
|
+
findings.push(
|
|
6133
|
+
fail3(
|
|
6134
|
+
"contract.retry.operations.duplicate-attempt-id",
|
|
6135
|
+
`Tool ${operation.tool} operation ${operationKey} has duplicate attemptId values.`,
|
|
6136
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
6137
|
+
"unique attemptId",
|
|
6138
|
+
"duplicate"
|
|
6139
|
+
)
|
|
6140
|
+
);
|
|
6141
|
+
}
|
|
6142
|
+
if (operation.maxAttempts !== void 0) {
|
|
6143
|
+
const count = countOperationAttempts(members);
|
|
6144
|
+
if (count > operation.maxAttempts) {
|
|
6145
|
+
findings.push(
|
|
6146
|
+
fail3(
|
|
6147
|
+
"contract.retry.operations.max-attempts",
|
|
6148
|
+
`Tool ${operation.tool} operation ${operationKey} exceeded maxAttempts ${operation.maxAttempts}.`,
|
|
6149
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
6150
|
+
operation.maxAttempts,
|
|
6151
|
+
count
|
|
6152
|
+
)
|
|
6153
|
+
);
|
|
6154
|
+
}
|
|
6155
|
+
}
|
|
6156
|
+
if (sideEffectClass === "write") {
|
|
6157
|
+
for (const event of ordered) {
|
|
6158
|
+
if (isUnevaluableWriteCompletion(event)) {
|
|
6159
|
+
findings.push(
|
|
6160
|
+
fail3(
|
|
6161
|
+
"contract.retry.operations.write-completion-unevaluable",
|
|
6162
|
+
`Write tool ${operation.tool} has timeout/unknown/running completion that cannot be verified as an authoritative write result.`,
|
|
6163
|
+
[eventEvidence2(event)],
|
|
6164
|
+
"authoritative write completion observation",
|
|
6165
|
+
{
|
|
6166
|
+
code: "AI_CHECK_RECOVERY_WRITE_COMPLETION_UNAVAILABLE",
|
|
6167
|
+
status: event.status
|
|
6168
|
+
}
|
|
6169
|
+
)
|
|
6170
|
+
);
|
|
6171
|
+
}
|
|
6172
|
+
}
|
|
6173
|
+
}
|
|
6174
|
+
const prior = [];
|
|
6175
|
+
let sawEarlierError = false;
|
|
6176
|
+
let sawLaterOkAfterError = false;
|
|
6177
|
+
let lastOk;
|
|
6178
|
+
let identityUnavailable = false;
|
|
6179
|
+
for (const event of ordered) {
|
|
6180
|
+
const classification = classifyRetryAttempt(event, prior, members);
|
|
6181
|
+
if (classification.kind === "unknown") {
|
|
6182
|
+
identityUnavailable = true;
|
|
6183
|
+
findings.push(
|
|
6184
|
+
fail3(
|
|
6185
|
+
"contract.retry.operations.attempt-identity",
|
|
6186
|
+
`Tool ${operation.tool} operation ${operationKey} has ambiguous attempt identity (${classification.reason}).`,
|
|
6187
|
+
[eventEvidence2(event)],
|
|
6188
|
+
"explicit operationId/attemptId/retryOf chronology",
|
|
6189
|
+
{ code: "AI_CHECK_RECOVERY_ATTEMPT_IDENTITY_UNAVAILABLE", reason: classification.reason }
|
|
6190
|
+
)
|
|
6191
|
+
);
|
|
6192
|
+
}
|
|
6193
|
+
const isRetry = classification.kind === "retry";
|
|
6194
|
+
if (isRetry && operation.requireFailureBeforeRetry) {
|
|
6195
|
+
const earlierFailure = prior.some(
|
|
6196
|
+
(candidate) => candidate.status === "error" && eventTime2(candidate) !== "" && eventTime2(event) !== "" && eventTime2(candidate) < eventTime2(event)
|
|
6197
|
+
);
|
|
6198
|
+
if (!earlierFailure) {
|
|
6199
|
+
findings.push(
|
|
6200
|
+
fail3(
|
|
6201
|
+
"contract.retry.operations.failure-before-retry",
|
|
6202
|
+
`Retry of tool ${operation.tool} appeared without an earlier failure in the same operation.`,
|
|
6203
|
+
[eventEvidence2(event)],
|
|
6204
|
+
"earlier error attempt",
|
|
6205
|
+
{ operationKey }
|
|
6206
|
+
)
|
|
6207
|
+
);
|
|
6208
|
+
}
|
|
6209
|
+
}
|
|
6210
|
+
if (isRetry && operation.retryableErrors?.codes && operation.retryableErrors.codes.length > 0) {
|
|
6211
|
+
const allowed = new Set(operation.retryableErrors.codes);
|
|
6212
|
+
const priorErrors = prior.filter((candidate) => candidate.status === "error");
|
|
6213
|
+
const lastError = priorErrors[priorErrors.length - 1];
|
|
6214
|
+
if (lastError) {
|
|
6215
|
+
const code = errorCodeOf(lastError);
|
|
6216
|
+
if (!code || !allowed.has(code)) {
|
|
6217
|
+
findings.push(
|
|
6218
|
+
fail3(
|
|
6219
|
+
"contract.retry.operations.retryable-error",
|
|
6220
|
+
`Retry of tool ${operation.tool} followed a non-retryable or missing error code.`,
|
|
6221
|
+
[eventEvidence2(event), eventEvidence2(lastError)],
|
|
6222
|
+
[...allowed],
|
|
6223
|
+
code ?? null
|
|
6224
|
+
)
|
|
6225
|
+
);
|
|
6226
|
+
}
|
|
6227
|
+
} else {
|
|
6228
|
+
findings.push(
|
|
6229
|
+
fail3(
|
|
6230
|
+
"contract.retry.operations.retryable-error",
|
|
6231
|
+
`Retry of tool ${operation.tool} has no prior error code evidence.`,
|
|
6232
|
+
[eventEvidence2(event)],
|
|
6233
|
+
[...allowed],
|
|
6234
|
+
null
|
|
6235
|
+
)
|
|
6236
|
+
);
|
|
6237
|
+
}
|
|
6238
|
+
}
|
|
6239
|
+
if (isRetry && (operation.requireSameArguments === true || operation.requireSameArguments === "structured-or-digest")) {
|
|
6240
|
+
const baseline = prior[0] ?? prior[prior.length - 1];
|
|
6241
|
+
if (baseline) {
|
|
6242
|
+
const comparison = sameArguments(baseline, event);
|
|
6243
|
+
if (!comparison.ok) {
|
|
6244
|
+
findings.push(
|
|
6245
|
+
fail3(
|
|
6246
|
+
"contract.retry.operations.same-arguments",
|
|
6247
|
+
comparison.reason,
|
|
6248
|
+
[eventEvidence2(baseline), eventEvidence2(event)],
|
|
6249
|
+
"matching structured args or digests",
|
|
6250
|
+
{ code: comparison.code }
|
|
6251
|
+
)
|
|
6252
|
+
);
|
|
6253
|
+
}
|
|
6254
|
+
} else {
|
|
6255
|
+
findings.push(
|
|
6256
|
+
fail3(
|
|
6257
|
+
"contract.retry.operations.same-arguments",
|
|
6258
|
+
"Structured or digest argument evidence unavailable for same-arguments check.",
|
|
6259
|
+
[eventEvidence2(event)],
|
|
6260
|
+
"matching structured args or digests",
|
|
6261
|
+
{ code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_UNAVAILABLE" }
|
|
6262
|
+
)
|
|
6263
|
+
);
|
|
6264
|
+
}
|
|
6265
|
+
}
|
|
6266
|
+
if (event.status === "error") {
|
|
6267
|
+
sawEarlierError = true;
|
|
6268
|
+
} else if (event.status === "ok" && sawEarlierError) {
|
|
6269
|
+
sawLaterOkAfterError = true;
|
|
6270
|
+
}
|
|
6271
|
+
if (event.status === "ok") {
|
|
6272
|
+
lastOk = event;
|
|
6273
|
+
}
|
|
6274
|
+
prior.push(event);
|
|
6275
|
+
}
|
|
6276
|
+
const latest = latestAttempt(ordered);
|
|
6277
|
+
if (operation.requireTerminalSuccess) {
|
|
6278
|
+
if (!latest || latest.status !== "ok") {
|
|
6279
|
+
findings.push(
|
|
6280
|
+
fail3(
|
|
6281
|
+
"contract.retry.operations.terminal-success",
|
|
6282
|
+
`Tool ${operation.tool} operation ${operationKey} does not terminate with ok on the latest attempt.`,
|
|
6283
|
+
ordered.slice(0, 4).map(eventEvidence2),
|
|
6284
|
+
"latest attempt ok",
|
|
6285
|
+
latest?.status ?? "missing"
|
|
6286
|
+
)
|
|
6287
|
+
);
|
|
6288
|
+
}
|
|
6289
|
+
}
|
|
6290
|
+
if (operation.requireRecoveredFailureVisible) {
|
|
6291
|
+
const latestOk = latest?.status === "ok";
|
|
6292
|
+
if (ordered.length > 1 && latestOk && !sawLaterOkAfterError) {
|
|
6293
|
+
findings.push(
|
|
6294
|
+
fail3(
|
|
6295
|
+
"contract.retry.operations.recovered-failure-visible",
|
|
6296
|
+
`Tool ${operation.tool} operation ${operationKey} does not show an earlier error before later success.`,
|
|
6297
|
+
ordered.slice(0, 4).map(eventEvidence2),
|
|
6298
|
+
"earlier error then later ok",
|
|
6299
|
+
{ sawEarlierError }
|
|
6300
|
+
)
|
|
6301
|
+
);
|
|
6302
|
+
}
|
|
6303
|
+
}
|
|
6304
|
+
const dependency = operation.successfulResultDependency;
|
|
6305
|
+
if (dependency?.consumerKind === "LLM") {
|
|
6306
|
+
const successForDependency = latest?.status === "ok" ? latest : identityUnavailable ? void 0 : lastOk;
|
|
6307
|
+
if (successForDependency) {
|
|
6308
|
+
if (sideEffectClass === "write" && isUnevaluableWriteCompletion(successForDependency)) {
|
|
6309
|
+
findings.push(
|
|
6310
|
+
fail3(
|
|
6311
|
+
"contract.retry.operations.successful-result-dependency",
|
|
6312
|
+
`Successful ${operation.tool} result dependency is unevaluable because write completion evidence is unavailable.`,
|
|
6313
|
+
[eventEvidence2(successForDependency)],
|
|
6314
|
+
"LLM reference to successful tool event",
|
|
6315
|
+
{ code: "AI_CHECK_RECOVERY_WRITE_COMPLETION_UNAVAILABLE" }
|
|
6316
|
+
)
|
|
6317
|
+
);
|
|
6318
|
+
} else {
|
|
6319
|
+
const requireExplicit = dependency.requireExplicitReference === true;
|
|
6320
|
+
const referenced = llmEvents.some(
|
|
6321
|
+
(llmEvent) => llmReferencesTool(llmEvent, successForDependency, requireExplicit)
|
|
6322
|
+
);
|
|
6323
|
+
if (!referenced) {
|
|
6324
|
+
findings.push(
|
|
6325
|
+
fail3(
|
|
6326
|
+
"contract.retry.operations.successful-result-dependency",
|
|
6327
|
+
requireExplicit ? `Successful ${operation.tool} result is not explicitly referenced by a later LLM event.` : `Successful ${operation.tool} result has no later LLM consumer.`,
|
|
6328
|
+
[eventEvidence2(successForDependency)],
|
|
6329
|
+
"LLM reference to successful tool event",
|
|
6330
|
+
{ code: "AI_CHECK_RECOVERY_RESULT_DEPENDENCY_MISSING" }
|
|
6331
|
+
)
|
|
6332
|
+
);
|
|
6333
|
+
}
|
|
6334
|
+
}
|
|
6335
|
+
}
|
|
6336
|
+
}
|
|
6337
|
+
}
|
|
6338
|
+
}
|
|
6339
|
+
return findings;
|
|
6340
|
+
}
|
|
6341
|
+
|
|
5007
6342
|
// packages/core/src/checks/contract.ts
|
|
6343
|
+
function contractFailFinding(ruleId, message, evidence, expected, actual) {
|
|
6344
|
+
return {
|
|
6345
|
+
ruleId,
|
|
6346
|
+
severity: "error",
|
|
6347
|
+
status: "fail",
|
|
6348
|
+
message,
|
|
6349
|
+
...expected !== void 0 ? { expected } : {},
|
|
6350
|
+
...actual !== void 0 ? { actual } : {},
|
|
6351
|
+
evidence: [...evidence]
|
|
6352
|
+
};
|
|
6353
|
+
}
|
|
6354
|
+
function normalizeStatus(status) {
|
|
6355
|
+
if (status === "ok" || status === "error" || status === "running") return status;
|
|
6356
|
+
if (status === "success") return "ok";
|
|
6357
|
+
if (status === "failed") return "error";
|
|
6358
|
+
return "error";
|
|
6359
|
+
}
|
|
5008
6360
|
function cloneBody(body) {
|
|
5009
6361
|
return {
|
|
5010
6362
|
...body.run ? { run: { ...body.run } } : {},
|
|
@@ -5029,35 +6381,471 @@ function cloneBody(body) {
|
|
|
5029
6381
|
...body.controls.enforcedTools ? { enforcedTools: [...body.controls.enforcedTools] } : {},
|
|
5030
6382
|
...body.controls.requiredStages ? { requiredStages: body.controls.requiredStages.map((item) => ({ ...item })) } : {}
|
|
5031
6383
|
}
|
|
5032
|
-
} : {},
|
|
5033
|
-
...body.retry ? {
|
|
5034
|
-
retry: {
|
|
5035
|
-
...body.retry,
|
|
5036
|
-
...body.retry.nonIdempotentTools ? { nonIdempotentTools: [...body.retry.nonIdempotentTools] } : {},
|
|
5037
|
-
...body.retry.operations ? {
|
|
5038
|
-
operations: body.retry.operations.map((operation) => ({
|
|
5039
|
-
...operation,
|
|
5040
|
-
...operation.retryableErrors ? {
|
|
5041
|
-
retryableErrors: {
|
|
5042
|
-
...operation.retryableErrors,
|
|
5043
|
-
...operation.retryableErrors.codes ? { codes: [...operation.retryableErrors.codes] } : {}
|
|
5044
|
-
}
|
|
5045
|
-
} : {},
|
|
5046
|
-
...operation.successfulResultDependency ? {
|
|
5047
|
-
successfulResultDependency: {
|
|
5048
|
-
...operation.successfulResultDependency
|
|
5049
|
-
}
|
|
5050
|
-
} : {}
|
|
5051
|
-
}))
|
|
5052
|
-
} : {}
|
|
6384
|
+
} : {},
|
|
6385
|
+
...body.retry ? {
|
|
6386
|
+
retry: {
|
|
6387
|
+
...body.retry,
|
|
6388
|
+
...body.retry.nonIdempotentTools ? { nonIdempotentTools: [...body.retry.nonIdempotentTools] } : {},
|
|
6389
|
+
...body.retry.operations ? {
|
|
6390
|
+
operations: body.retry.operations.map((operation) => ({
|
|
6391
|
+
...operation,
|
|
6392
|
+
...operation.retryableErrors ? {
|
|
6393
|
+
retryableErrors: {
|
|
6394
|
+
...operation.retryableErrors,
|
|
6395
|
+
...operation.retryableErrors.codes ? { codes: [...operation.retryableErrors.codes] } : {}
|
|
6396
|
+
}
|
|
6397
|
+
} : {},
|
|
6398
|
+
...operation.successfulResultDependency ? {
|
|
6399
|
+
successfulResultDependency: {
|
|
6400
|
+
...operation.successfulResultDependency
|
|
6401
|
+
}
|
|
6402
|
+
} : {}
|
|
6403
|
+
}))
|
|
6404
|
+
} : {}
|
|
6405
|
+
}
|
|
6406
|
+
} : {}
|
|
6407
|
+
};
|
|
6408
|
+
}
|
|
6409
|
+
function bodyHasRules(body) {
|
|
6410
|
+
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
6411
|
+
}
|
|
6412
|
+
var MAX_EVIDENCE_EVENT_IDS = 16;
|
|
6413
|
+
var METHOD_VOCABULARY = new Set(OBSERVED_OUTCOME_METHODS);
|
|
6414
|
+
function parseEvidenceEventIds(evidence) {
|
|
6415
|
+
if (evidence === void 0 || evidence === null) {
|
|
6416
|
+
return { status: "missing" };
|
|
6417
|
+
}
|
|
6418
|
+
if (typeof evidence === "string") {
|
|
6419
|
+
const trimmed = evidence.trim();
|
|
6420
|
+
if (trimmed === "") {
|
|
6421
|
+
return { status: "invalid", reason: "empty evidence string" };
|
|
6422
|
+
}
|
|
6423
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
6424
|
+
}
|
|
6425
|
+
if (typeof evidence !== "object" || Array.isArray(evidence)) {
|
|
6426
|
+
return {
|
|
6427
|
+
status: "invalid",
|
|
6428
|
+
reason: "evidence must be a string event id or { eventId } / { eventIds }"
|
|
6429
|
+
};
|
|
6430
|
+
}
|
|
6431
|
+
const record = evidence;
|
|
6432
|
+
if (typeof record.eventId === "string") {
|
|
6433
|
+
const trimmed = record.eventId.trim();
|
|
6434
|
+
if (trimmed === "") {
|
|
6435
|
+
return { status: "invalid", reason: "empty evidence.eventId" };
|
|
6436
|
+
}
|
|
6437
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
6438
|
+
}
|
|
6439
|
+
if (Array.isArray(record.eventIds)) {
|
|
6440
|
+
if (record.eventIds.length === 0) {
|
|
6441
|
+
return { status: "invalid", reason: "empty evidence.eventIds" };
|
|
6442
|
+
}
|
|
6443
|
+
if (record.eventIds.length > MAX_EVIDENCE_EVENT_IDS) {
|
|
6444
|
+
return {
|
|
6445
|
+
status: "invalid",
|
|
6446
|
+
reason: `evidence.eventIds exceeds bound of ${MAX_EVIDENCE_EVENT_IDS}`
|
|
6447
|
+
};
|
|
6448
|
+
}
|
|
6449
|
+
const eventIds = [];
|
|
6450
|
+
for (const item of record.eventIds) {
|
|
6451
|
+
if (typeof item !== "string" || item.trim() === "") {
|
|
6452
|
+
return { status: "invalid", reason: "evidence.eventIds contains a non-string id" };
|
|
6453
|
+
}
|
|
6454
|
+
eventIds.push(item.trim());
|
|
6455
|
+
}
|
|
6456
|
+
return { status: "ok", eventIds };
|
|
6457
|
+
}
|
|
6458
|
+
return {
|
|
6459
|
+
status: "invalid",
|
|
6460
|
+
reason: "evidence must include eventId or eventIds"
|
|
6461
|
+
};
|
|
6462
|
+
}
|
|
6463
|
+
function provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds) {
|
|
6464
|
+
const findings = [];
|
|
6465
|
+
const evidenceBase = [
|
|
6466
|
+
{
|
|
6467
|
+
runId: outcome.runId,
|
|
6468
|
+
eventId: outcome.outcomeId,
|
|
6469
|
+
kind: "OUTCOME",
|
|
6470
|
+
name: outcome.name,
|
|
6471
|
+
path: `outcome.${outcome.name}`
|
|
6472
|
+
}
|
|
6473
|
+
];
|
|
6474
|
+
if (requireProvenance.method) {
|
|
6475
|
+
if (outcome.method === void 0 || outcome.method.trim() === "") {
|
|
6476
|
+
findings.push(
|
|
6477
|
+
contractFailFinding(
|
|
6478
|
+
"contract.observation.provenance.method",
|
|
6479
|
+
`Observation ${outcome.name} is missing a structural method.`,
|
|
6480
|
+
evidenceBase,
|
|
6481
|
+
{ method: true },
|
|
6482
|
+
{ name: outcome.name, method: outcome.method ?? null }
|
|
6483
|
+
)
|
|
6484
|
+
);
|
|
6485
|
+
} else if (!METHOD_VOCABULARY.has(outcome.method)) {
|
|
6486
|
+
findings.push(
|
|
6487
|
+
contractFailFinding(
|
|
6488
|
+
"contract.observation.provenance.method",
|
|
6489
|
+
`Observation ${outcome.name} method ${outcome.method} is outside the bounded ObservedOutcomeMethod vocabulary.`,
|
|
6490
|
+
evidenceBase,
|
|
6491
|
+
{ methodVocabulary: [...OBSERVED_OUTCOME_METHODS] },
|
|
6492
|
+
{ name: outcome.name, method: outcome.method }
|
|
6493
|
+
)
|
|
6494
|
+
);
|
|
6495
|
+
}
|
|
6496
|
+
}
|
|
6497
|
+
if (requireProvenance.evidence || requireProvenance.sameRunEventReference) {
|
|
6498
|
+
const parsed = parseEvidenceEventIds(outcome.evidence);
|
|
6499
|
+
if (parsed.status === "missing") {
|
|
6500
|
+
findings.push(
|
|
6501
|
+
contractFailFinding(
|
|
6502
|
+
"contract.observation.provenance.evidence",
|
|
6503
|
+
`Observation ${outcome.name} is missing bounded evidence references.`,
|
|
6504
|
+
evidenceBase,
|
|
6505
|
+
{ evidence: true },
|
|
6506
|
+
{ name: outcome.name, evidence: null }
|
|
6507
|
+
)
|
|
6508
|
+
);
|
|
6509
|
+
} else if (parsed.status === "invalid") {
|
|
6510
|
+
findings.push(
|
|
6511
|
+
contractFailFinding(
|
|
6512
|
+
"contract.observation.provenance.evidence",
|
|
6513
|
+
`Observation ${outcome.name} evidence is not a bounded event reference (${parsed.reason}).`,
|
|
6514
|
+
evidenceBase,
|
|
6515
|
+
{ evidence: true },
|
|
6516
|
+
{ name: outcome.name, reason: parsed.reason }
|
|
6517
|
+
)
|
|
6518
|
+
);
|
|
6519
|
+
} else if (requireProvenance.sameRunEventReference) {
|
|
6520
|
+
const missing = parsed.eventIds.filter((eventId) => !knownEventIds.has(eventId));
|
|
6521
|
+
if (missing.length > 0) {
|
|
6522
|
+
findings.push(
|
|
6523
|
+
contractFailFinding(
|
|
6524
|
+
"contract.observation.provenance.same-run",
|
|
6525
|
+
`Observation ${outcome.name} references event ids missing from the same run: ${missing.join(", ")}.`,
|
|
6526
|
+
evidenceBase,
|
|
6527
|
+
{ sameRunEventReference: true, eventIds: parsed.eventIds },
|
|
6528
|
+
{ name: outcome.name, missing }
|
|
6529
|
+
)
|
|
6530
|
+
);
|
|
6531
|
+
}
|
|
6532
|
+
}
|
|
6533
|
+
}
|
|
6534
|
+
return findings;
|
|
6535
|
+
}
|
|
6536
|
+
function contractToRules(contract) {
|
|
6537
|
+
const rules = [];
|
|
6538
|
+
const allowedStatuses = contract.run?.allowedStatuses ?? [];
|
|
6539
|
+
if (allowedStatuses.length === 1) {
|
|
6540
|
+
rules.push(
|
|
6541
|
+
createRunStatusRule({
|
|
6542
|
+
expected: normalizeStatus(allowedStatuses[0]),
|
|
6543
|
+
allowIncomplete: contract.run?.requireCompleted === false
|
|
6544
|
+
})
|
|
6545
|
+
);
|
|
6546
|
+
} else if (allowedStatuses.length > 1) {
|
|
6547
|
+
const expected = [...new Set(allowedStatuses.map(normalizeStatus))];
|
|
6548
|
+
const allowIncomplete = contract.run?.requireCompleted === false;
|
|
6549
|
+
rules.push({
|
|
6550
|
+
id: "contract.run.allowedStatuses",
|
|
6551
|
+
category: "run",
|
|
6552
|
+
defaultSeverity: "error",
|
|
6553
|
+
evaluate(context) {
|
|
6554
|
+
const findings = [];
|
|
6555
|
+
const actual = context.selectedRun?.status ?? "unknown";
|
|
6556
|
+
if (!expected.includes(actual)) {
|
|
6557
|
+
findings.push(
|
|
6558
|
+
contractFailFinding(
|
|
6559
|
+
"contract.run.allowedStatuses",
|
|
6560
|
+
`Run status ${actual} is not one of the allowed statuses: ${expected.join(", ")}.`,
|
|
6561
|
+
context.selectedRun ? [
|
|
6562
|
+
{
|
|
6563
|
+
runId: context.selectedRun.runId,
|
|
6564
|
+
kind: "RUN",
|
|
6565
|
+
name: context.selectedRun.name,
|
|
6566
|
+
status: context.selectedRun.status
|
|
6567
|
+
}
|
|
6568
|
+
] : [],
|
|
6569
|
+
expected,
|
|
6570
|
+
actual
|
|
6571
|
+
)
|
|
6572
|
+
);
|
|
6573
|
+
}
|
|
6574
|
+
if (!allowIncomplete) {
|
|
6575
|
+
const running = (context.logicalEvents ?? context.events).filter(
|
|
6576
|
+
(event) => event.status === "running"
|
|
6577
|
+
);
|
|
6578
|
+
if (running.length > 0) {
|
|
6579
|
+
findings.push(
|
|
6580
|
+
contractFailFinding(
|
|
6581
|
+
"contract.run.allowedStatuses",
|
|
6582
|
+
"Run contains incomplete running events.",
|
|
6583
|
+
running.map((event) => ({
|
|
6584
|
+
runId: event.runId,
|
|
6585
|
+
eventId: event.eventId,
|
|
6586
|
+
kind: event.kind,
|
|
6587
|
+
name: event.name,
|
|
6588
|
+
status: event.status
|
|
6589
|
+
})),
|
|
6590
|
+
"no running events",
|
|
6591
|
+
running.length
|
|
6592
|
+
)
|
|
6593
|
+
);
|
|
6594
|
+
}
|
|
6595
|
+
}
|
|
6596
|
+
return findings;
|
|
6597
|
+
}
|
|
6598
|
+
});
|
|
6599
|
+
} else if (contract.run?.requireCompleted !== false) {
|
|
6600
|
+
rules.push(createRunStatusRule({ allowIncomplete: false }));
|
|
6601
|
+
}
|
|
6602
|
+
if (contract.run?.maxDurationMs !== void 0) {
|
|
6603
|
+
rules.push(createRunDurationRule({ maxDurationMs: contract.run.maxDurationMs }));
|
|
6604
|
+
}
|
|
6605
|
+
if (contract.run?.requireCompleted === true) {
|
|
6606
|
+
rules.push(createStructureIncompleteRule({ requireEndedAtForStarted: true }));
|
|
6607
|
+
}
|
|
6608
|
+
if (contract.tools) {
|
|
6609
|
+
const order = contract.tools.requiredOrder ?? [];
|
|
6610
|
+
const requiredOrderMode = contract.tools.requiredOrderMode ?? "first-occurrence";
|
|
6611
|
+
const orderRules = contract.tools.orderRules ?? [];
|
|
6612
|
+
const endpointRequired = orderRules.filter((rule) => rule.requireEndpoints !== false).flatMap((rule) => [rule.before, rule.after]);
|
|
6613
|
+
const required = [
|
|
6614
|
+
.../* @__PURE__ */ new Set([
|
|
6615
|
+
...contract.tools.required ?? [],
|
|
6616
|
+
...contract.tools.requiredTools ?? [],
|
|
6617
|
+
...order,
|
|
6618
|
+
...endpointRequired
|
|
6619
|
+
])
|
|
6620
|
+
];
|
|
6621
|
+
const forbidden = [
|
|
6622
|
+
...contract.tools.forbidden ?? [],
|
|
6623
|
+
...contract.tools.forbiddenTools ?? []
|
|
6624
|
+
];
|
|
6625
|
+
rules.push(
|
|
6626
|
+
createToolUsageRule({
|
|
6627
|
+
...required.length > 0 ? { required } : {},
|
|
6628
|
+
...forbidden.length > 0 ? { forbidden } : {},
|
|
6629
|
+
...contract.tools.allowed ? { allowed: contract.tools.allowed } : {},
|
|
6630
|
+
...contract.tools.maxCalls !== void 0 ? { maxCount: contract.tools.maxCalls } : {}
|
|
6631
|
+
})
|
|
6632
|
+
);
|
|
6633
|
+
for (let i = 0; i < order.length - 1; i += 1) {
|
|
6634
|
+
rules.push(
|
|
6635
|
+
createToolOrderingRule({
|
|
6636
|
+
before: order[i],
|
|
6637
|
+
after: order[i + 1],
|
|
6638
|
+
id: `contract.tool.order.${i}`,
|
|
6639
|
+
mode: requiredOrderMode
|
|
6640
|
+
})
|
|
6641
|
+
);
|
|
6642
|
+
}
|
|
6643
|
+
const defaultOccurrenceMode = contract.tools.defaultOccurrenceMode ?? requiredOrderMode;
|
|
6644
|
+
for (const [index, rule] of orderRules.entries()) {
|
|
6645
|
+
rules.push(
|
|
6646
|
+
createToolOrderingRule({
|
|
6647
|
+
before: rule.before,
|
|
6648
|
+
after: rule.after,
|
|
6649
|
+
id: `contract.tool.orderRule.${index}`,
|
|
6650
|
+
mode: rule.occurrenceMode ?? defaultOccurrenceMode
|
|
6651
|
+
})
|
|
6652
|
+
);
|
|
6653
|
+
}
|
|
6654
|
+
const argumentChecks = contract.tools.arguments ?? [];
|
|
6655
|
+
if (argumentChecks.length > 0) {
|
|
6656
|
+
rules.push({
|
|
6657
|
+
id: "contract.tool.arguments",
|
|
6658
|
+
category: "tool",
|
|
6659
|
+
defaultSeverity: "error",
|
|
6660
|
+
evaluate(context) {
|
|
6661
|
+
const tools = (context.logicalEvents ?? context.events).filter(
|
|
6662
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
6663
|
+
);
|
|
6664
|
+
const findings = [];
|
|
6665
|
+
for (const [index, check] of argumentChecks.entries()) {
|
|
6666
|
+
const matches = tools.filter(
|
|
6667
|
+
(event) => resolveCanonicalToolName(event) === check.tool
|
|
6668
|
+
);
|
|
6669
|
+
if (matches.length === 0) {
|
|
6670
|
+
findings.push(
|
|
6671
|
+
contractFailFinding(
|
|
6672
|
+
`contract.tool.arguments.${index}`,
|
|
6673
|
+
`No finished tool named ${check.tool} for argument check.`,
|
|
6674
|
+
context.selectedRun ? [
|
|
6675
|
+
{
|
|
6676
|
+
runId: context.selectedRun.runId,
|
|
6677
|
+
kind: "RUN",
|
|
6678
|
+
name: context.selectedRun.name
|
|
6679
|
+
}
|
|
6680
|
+
] : [],
|
|
6681
|
+
{ tool: check.tool, path: check.path },
|
|
6682
|
+
{ toolCount: 0 }
|
|
6683
|
+
)
|
|
6684
|
+
);
|
|
6685
|
+
continue;
|
|
6686
|
+
}
|
|
6687
|
+
const occurrence = check.occurrence ?? "all";
|
|
6688
|
+
const selected = occurrence === "first" ? [matches[0]] : occurrence === "last" ? [matches[matches.length - 1]] : matches;
|
|
6689
|
+
const results = selected.map((event) => {
|
|
6690
|
+
const payload = extractToolArgumentPayload(event);
|
|
6691
|
+
return evaluateToolArgumentValue(
|
|
6692
|
+
payload.value,
|
|
6693
|
+
check,
|
|
6694
|
+
payload.present
|
|
6695
|
+
);
|
|
6696
|
+
});
|
|
6697
|
+
if (occurrence === "any") {
|
|
6698
|
+
if (results.some((result) => result.status === "pass")) continue;
|
|
6699
|
+
const firstFail = results.find((result) => result.status !== "pass");
|
|
6700
|
+
findings.push(
|
|
6701
|
+
contractFailFinding(
|
|
6702
|
+
`contract.tool.arguments.${index}`,
|
|
6703
|
+
firstFail.message,
|
|
6704
|
+
selected.slice(0, 1).map((event) => ({
|
|
6705
|
+
runId: event.runId,
|
|
6706
|
+
eventId: event.eventId,
|
|
6707
|
+
kind: event.kind,
|
|
6708
|
+
name: event.name,
|
|
6709
|
+
path: `tool.${check.tool}${check.path}`
|
|
6710
|
+
})),
|
|
6711
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
6712
|
+
{ code: firstFail.code }
|
|
6713
|
+
)
|
|
6714
|
+
);
|
|
6715
|
+
continue;
|
|
6716
|
+
}
|
|
6717
|
+
for (const [selIndex, result] of results.entries()) {
|
|
6718
|
+
if (result.status === "pass") continue;
|
|
6719
|
+
const event = selected[selIndex];
|
|
6720
|
+
findings.push(
|
|
6721
|
+
contractFailFinding(
|
|
6722
|
+
`contract.tool.arguments.${index}`,
|
|
6723
|
+
result.message,
|
|
6724
|
+
[
|
|
6725
|
+
{
|
|
6726
|
+
runId: event.runId,
|
|
6727
|
+
eventId: event.eventId,
|
|
6728
|
+
kind: event.kind,
|
|
6729
|
+
name: event.name,
|
|
6730
|
+
path: `tool.${check.tool}${check.path}`
|
|
6731
|
+
}
|
|
6732
|
+
],
|
|
6733
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
6734
|
+
{ code: result.code }
|
|
6735
|
+
)
|
|
6736
|
+
);
|
|
6737
|
+
}
|
|
6738
|
+
}
|
|
6739
|
+
return findings;
|
|
6740
|
+
}
|
|
6741
|
+
});
|
|
6742
|
+
}
|
|
6743
|
+
}
|
|
6744
|
+
if (contract.llm) {
|
|
6745
|
+
rules.push(
|
|
6746
|
+
createLlmUsageRule({
|
|
6747
|
+
...contract.llm.maxCalls !== void 0 ? { maxCalls: contract.llm.maxCalls } : {},
|
|
6748
|
+
...contract.llm.maxTotalTokens !== void 0 ? { maxTotalTokens: contract.llm.maxTotalTokens } : {},
|
|
6749
|
+
...contract.llm.allowedModels ? { allowedModels: contract.llm.allowedModels } : {}
|
|
6750
|
+
})
|
|
6751
|
+
);
|
|
6752
|
+
}
|
|
6753
|
+
if (contract.observations) {
|
|
6754
|
+
const required = contract.observations.required ?? [];
|
|
6755
|
+
if (required.length > 0) {
|
|
6756
|
+
rules.push({
|
|
6757
|
+
id: "contract.observation.required",
|
|
6758
|
+
category: "run",
|
|
6759
|
+
defaultSeverity: "error",
|
|
6760
|
+
evaluate(context) {
|
|
6761
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
6762
|
+
const names = new Set(outcomes.map((item) => item.name));
|
|
6763
|
+
const missing = required.filter((name) => !names.has(name));
|
|
6764
|
+
if (missing.length === 0) return [];
|
|
6765
|
+
return [
|
|
6766
|
+
contractFailFinding(
|
|
6767
|
+
"contract.observation.required",
|
|
6768
|
+
`Required observations missing: ${missing.join(", ")}`,
|
|
6769
|
+
context.selectedRun ? [{ runId: context.selectedRun.runId, kind: "RUN", name: context.selectedRun.name }] : [],
|
|
6770
|
+
required,
|
|
6771
|
+
[...names]
|
|
6772
|
+
)
|
|
6773
|
+
];
|
|
6774
|
+
}
|
|
6775
|
+
});
|
|
6776
|
+
}
|
|
6777
|
+
if (contract.observations.failOn?.length) {
|
|
6778
|
+
rules.push(
|
|
6779
|
+
createObservedOutcomeRule({
|
|
6780
|
+
failOn: contract.observations.failOn.filter(
|
|
6781
|
+
(status) => status === "failed" || status === "unknown" || status === "skipped"
|
|
6782
|
+
)
|
|
6783
|
+
})
|
|
6784
|
+
);
|
|
6785
|
+
}
|
|
6786
|
+
const requireProvenance = contract.observations.requireProvenance;
|
|
6787
|
+
if (requireProvenance && (requireProvenance.method || requireProvenance.evidence || requireProvenance.sameRunEventReference)) {
|
|
6788
|
+
rules.push({
|
|
6789
|
+
id: "contract.observation.provenance",
|
|
6790
|
+
category: "run",
|
|
6791
|
+
defaultSeverity: "error",
|
|
6792
|
+
evaluate(context) {
|
|
6793
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
6794
|
+
const knownEventIds = new Set(context.events.map((event) => event.eventId));
|
|
6795
|
+
const targets = required.length > 0 ? outcomes.filter((outcome) => required.includes(outcome.name)) : outcomes;
|
|
6796
|
+
return targets.flatMap(
|
|
6797
|
+
(outcome) => provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds)
|
|
6798
|
+
);
|
|
6799
|
+
}
|
|
6800
|
+
});
|
|
6801
|
+
}
|
|
6802
|
+
}
|
|
6803
|
+
if (contract.controls) {
|
|
6804
|
+
const controls = contract.controls;
|
|
6805
|
+
rules.push({
|
|
6806
|
+
id: "contract.controls",
|
|
6807
|
+
category: "run",
|
|
6808
|
+
defaultSeverity: "error",
|
|
6809
|
+
evaluate(context) {
|
|
6810
|
+
const events = context.logicalEvents ?? context.events;
|
|
6811
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
6812
|
+
const observationNames = new Set(outcomes.map((item) => item.name));
|
|
6813
|
+
const runEvidence2 = context.selectedRun ? [
|
|
6814
|
+
{
|
|
6815
|
+
runId: context.selectedRun.runId,
|
|
6816
|
+
kind: "RUN",
|
|
6817
|
+
name: context.selectedRun.name
|
|
6818
|
+
}
|
|
6819
|
+
] : [];
|
|
6820
|
+
return evaluateControlRules(events, controls, runEvidence2, observationNames);
|
|
6821
|
+
}
|
|
6822
|
+
});
|
|
6823
|
+
}
|
|
6824
|
+
if (contract.retry) {
|
|
6825
|
+
const retry = contract.retry;
|
|
6826
|
+
rules.push({
|
|
6827
|
+
id: "contract.retry",
|
|
6828
|
+
category: "run",
|
|
6829
|
+
defaultSeverity: "error",
|
|
6830
|
+
evaluate(context) {
|
|
6831
|
+
const events = context.logicalEvents ?? context.events;
|
|
6832
|
+
context.selectedRun ? [
|
|
6833
|
+
{
|
|
6834
|
+
runId: context.selectedRun.runId,
|
|
6835
|
+
kind: "RUN",
|
|
6836
|
+
name: context.selectedRun.name
|
|
6837
|
+
}
|
|
6838
|
+
] : [];
|
|
6839
|
+
const base = evaluateRetrySafetyRules(events, retry);
|
|
6840
|
+
if (!retry.operations || retry.operations.length === 0) {
|
|
6841
|
+
return base;
|
|
6842
|
+
}
|
|
6843
|
+
return [...base, ...evaluateRecoveryOperations(events, retry.operations)];
|
|
5053
6844
|
}
|
|
5054
|
-
}
|
|
5055
|
-
}
|
|
5056
|
-
|
|
5057
|
-
function bodyHasRules(body) {
|
|
5058
|
-
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
6845
|
+
});
|
|
6846
|
+
}
|
|
6847
|
+
return rules;
|
|
5059
6848
|
}
|
|
5060
|
-
new Set(OBSERVED_OUTCOME_METHODS);
|
|
5061
6849
|
function validateScopeShape(scope) {
|
|
5062
6850
|
if (scope === void 0) return [];
|
|
5063
6851
|
const hasSelector = typeof scope.runId === "string" && scope.runId.trim() !== "" || typeof scope.subAgentId === "string" && scope.subAgentId.trim() !== "" || typeof scope.groupId === "string" && scope.groupId.trim() !== "" || typeof scope.workflowStep === "string" && scope.workflowStep.trim() !== "" || typeof scope.rootEventId === "string" && scope.rootEventId.trim() !== "";
|
|
@@ -5158,6 +6946,57 @@ function validateAlternativesShape(alternatives) {
|
|
|
5158
6946
|
}
|
|
5159
6947
|
return diagnostics;
|
|
5160
6948
|
}
|
|
6949
|
+
function emptyPassResult(input, runId) {
|
|
6950
|
+
return {
|
|
6951
|
+
ok: true,
|
|
6952
|
+
status: "pass",
|
|
6953
|
+
format: input.read.format,
|
|
6954
|
+
...runId !== void 0 ? { runId } : {},
|
|
6955
|
+
summary: { passed: 0, failed: 0, warnings: 0, errors: 0, rulesEvaluated: 0 },
|
|
6956
|
+
findings: [],
|
|
6957
|
+
diagnostics: [],
|
|
6958
|
+
ruleExecutions: []
|
|
6959
|
+
};
|
|
6960
|
+
}
|
|
6961
|
+
function mergeContractResults(input, parts, extraFindings = []) {
|
|
6962
|
+
const findings = [...parts.flatMap((part) => part.findings), ...extraFindings];
|
|
6963
|
+
const diagnostics = parts.flatMap((part) => part.diagnostics);
|
|
6964
|
+
const ruleExecutions = parts.flatMap((part) => part.ruleExecutions);
|
|
6965
|
+
const failed = findings.filter(
|
|
6966
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
6967
|
+
).length;
|
|
6968
|
+
const errors = diagnostics.filter((item) => item.severity === "error").length;
|
|
6969
|
+
const status = errors > 0 ? "error" : failed > 0 ? "fail" : "pass";
|
|
6970
|
+
const runId = parts.find((part) => part.runId !== void 0)?.runId;
|
|
6971
|
+
return {
|
|
6972
|
+
ok: status === "pass",
|
|
6973
|
+
status,
|
|
6974
|
+
format: input.read.format,
|
|
6975
|
+
...runId !== void 0 ? { runId } : {},
|
|
6976
|
+
summary: {
|
|
6977
|
+
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
6978
|
+
failed,
|
|
6979
|
+
warnings: findings.filter(
|
|
6980
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
6981
|
+
).length,
|
|
6982
|
+
errors,
|
|
6983
|
+
rulesEvaluated: ruleExecutions.length
|
|
6984
|
+
},
|
|
6985
|
+
findings,
|
|
6986
|
+
diagnostics,
|
|
6987
|
+
ruleExecutions
|
|
6988
|
+
};
|
|
6989
|
+
}
|
|
6990
|
+
function evaluateBody(input, body, options = {}) {
|
|
6991
|
+
const rules = contractToRules(body);
|
|
6992
|
+
if (rules.length === 0) {
|
|
6993
|
+
return emptyPassResult(input, options.runId);
|
|
6994
|
+
}
|
|
6995
|
+
return runTraceChecks(input, {
|
|
6996
|
+
rules,
|
|
6997
|
+
...options.runId !== void 0 ? { runId: options.runId } : {}
|
|
6998
|
+
});
|
|
6999
|
+
}
|
|
5161
7000
|
function defineTraceContract(input) {
|
|
5162
7001
|
const shapeErrors = [
|
|
5163
7002
|
...validateAlternativesShape(input.alternatives),
|
|
@@ -5188,6 +7027,143 @@ function defineTraceContract(input) {
|
|
|
5188
7027
|
...alternatives ? { alternatives } : {}
|
|
5189
7028
|
};
|
|
5190
7029
|
}
|
|
7030
|
+
function evaluateTraceContract(input, contract, options = {}) {
|
|
7031
|
+
const shapeErrors = [
|
|
7032
|
+
...validateAlternativesShape(contract.alternatives),
|
|
7033
|
+
...validateScopeShape(contract.scope),
|
|
7034
|
+
...validateProvenanceShape(contract.observations)
|
|
7035
|
+
].filter((item) => item.severity === "error");
|
|
7036
|
+
if (shapeErrors.length > 0) {
|
|
7037
|
+
return {
|
|
7038
|
+
ok: false,
|
|
7039
|
+
status: "error",
|
|
7040
|
+
format: input.read.format,
|
|
7041
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
7042
|
+
summary: {
|
|
7043
|
+
passed: 0,
|
|
7044
|
+
failed: 0,
|
|
7045
|
+
warnings: 0,
|
|
7046
|
+
errors: shapeErrors.length,
|
|
7047
|
+
rulesEvaluated: 0
|
|
7048
|
+
},
|
|
7049
|
+
findings: [],
|
|
7050
|
+
diagnostics: shapeErrors.map((item) => ({
|
|
7051
|
+
code: "AI_CHECK_INVALID_CONFIG",
|
|
7052
|
+
message: item.message,
|
|
7053
|
+
severity: "error",
|
|
7054
|
+
ruleId: item.code
|
|
7055
|
+
})),
|
|
7056
|
+
ruleExecutions: []
|
|
7057
|
+
};
|
|
7058
|
+
}
|
|
7059
|
+
const scoped = resolveTraceContractScope(input, contract.scope);
|
|
7060
|
+
if (scoped.diagnostics.length > 0) {
|
|
7061
|
+
return {
|
|
7062
|
+
ok: false,
|
|
7063
|
+
status: "error",
|
|
7064
|
+
format: input.read.format,
|
|
7065
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
7066
|
+
...scoped.resolved?.runId !== void 0 ? { runId: scoped.resolved.runId } : {},
|
|
7067
|
+
summary: {
|
|
7068
|
+
passed: 0,
|
|
7069
|
+
failed: 0,
|
|
7070
|
+
warnings: 0,
|
|
7071
|
+
errors: scoped.diagnostics.length,
|
|
7072
|
+
rulesEvaluated: 0
|
|
7073
|
+
},
|
|
7074
|
+
findings: [],
|
|
7075
|
+
diagnostics: scoped.diagnostics,
|
|
7076
|
+
ruleExecutions: []
|
|
7077
|
+
};
|
|
7078
|
+
}
|
|
7079
|
+
const effectiveInput = scoped.resolved?.input ?? input;
|
|
7080
|
+
const effectiveRunId = scoped.resolved?.runId ?? options.runId;
|
|
7081
|
+
const base = evaluateBody(effectiveInput, contract, {
|
|
7082
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
7083
|
+
});
|
|
7084
|
+
const scopeFinding = scoped.resolved ? {
|
|
7085
|
+
ruleId: "contract.scope.selected",
|
|
7086
|
+
severity: "info",
|
|
7087
|
+
status: "pass",
|
|
7088
|
+
message: `Selected actor ${scoped.resolved.runId} with ${scoped.resolved.evidenceEventCount} evidence event(s).`,
|
|
7089
|
+
expected: scoped.resolved.matchedSelectors,
|
|
7090
|
+
actual: {
|
|
7091
|
+
runId: scoped.resolved.runId,
|
|
7092
|
+
evidenceEventCount: scoped.resolved.evidenceEventCount,
|
|
7093
|
+
...scoped.resolved.rootEventId !== void 0 ? { rootEventId: scoped.resolved.rootEventId } : {}
|
|
7094
|
+
},
|
|
7095
|
+
evidence: [
|
|
7096
|
+
{
|
|
7097
|
+
runId: scoped.resolved.runId,
|
|
7098
|
+
kind: "RUN",
|
|
7099
|
+
path: "scope"
|
|
7100
|
+
}
|
|
7101
|
+
]
|
|
7102
|
+
} : void 0;
|
|
7103
|
+
const branches = contract.alternatives?.anyOf ?? [];
|
|
7104
|
+
if (branches.length === 0) {
|
|
7105
|
+
return scopeFinding ? mergeContractResults(effectiveInput, [base], [scopeFinding]) : base;
|
|
7106
|
+
}
|
|
7107
|
+
const branchEvaluations = branches.map((branch) => {
|
|
7108
|
+
const result = evaluateBody(effectiveInput, branch.contract, {
|
|
7109
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
7110
|
+
});
|
|
7111
|
+
return { branch, result };
|
|
7112
|
+
});
|
|
7113
|
+
const satisfied = branchEvaluations.filter((item) => item.result.ok && item.result.status === "pass").map((item) => item.branch.id);
|
|
7114
|
+
const failedBranches = branchEvaluations.filter(
|
|
7115
|
+
(item) => !(item.result.ok && item.result.status === "pass")
|
|
7116
|
+
);
|
|
7117
|
+
if (satisfied.length > 0) {
|
|
7118
|
+
const passFinding = {
|
|
7119
|
+
ruleId: "contract.alternatives.anyOf",
|
|
7120
|
+
severity: "info",
|
|
7121
|
+
status: "pass",
|
|
7122
|
+
message: `Satisfied alternative branch(es): ${satisfied.join(", ")}.`,
|
|
7123
|
+
expected: { anyOf: branches.map((branch) => branch.id) },
|
|
7124
|
+
actual: {
|
|
7125
|
+
satisfied,
|
|
7126
|
+
failed: failedBranches.map((item) => item.branch.id)
|
|
7127
|
+
},
|
|
7128
|
+
evidence: []
|
|
7129
|
+
};
|
|
7130
|
+
return mergeContractResults(
|
|
7131
|
+
effectiveInput,
|
|
7132
|
+
[base],
|
|
7133
|
+
[...scopeFinding ? [scopeFinding] : [], passFinding]
|
|
7134
|
+
);
|
|
7135
|
+
}
|
|
7136
|
+
const noneFinding = {
|
|
7137
|
+
ruleId: "contract.alternatives.none-satisfied",
|
|
7138
|
+
severity: "error",
|
|
7139
|
+
status: "fail",
|
|
7140
|
+
message: "No alternatives.anyOf branch fully satisfied its contract.",
|
|
7141
|
+
expected: { anyOf: branches.map((branch) => branch.id) },
|
|
7142
|
+
actual: {
|
|
7143
|
+
branchFailures: Object.fromEntries(
|
|
7144
|
+
failedBranches.map((item) => [
|
|
7145
|
+
item.branch.id,
|
|
7146
|
+
item.result.findings.filter((finding) => finding.status === "fail").map((finding) => ({
|
|
7147
|
+
ruleId: finding.ruleId,
|
|
7148
|
+
message: finding.message
|
|
7149
|
+
}))
|
|
7150
|
+
])
|
|
7151
|
+
)
|
|
7152
|
+
},
|
|
7153
|
+
evidence: []
|
|
7154
|
+
};
|
|
7155
|
+
const prefixedBranchFindings = failedBranches.flatMap(
|
|
7156
|
+
(item) => item.result.findings.filter((finding) => finding.status === "fail").map((finding) => ({
|
|
7157
|
+
...finding,
|
|
7158
|
+
ruleId: `contract.alternatives.${item.branch.id}.${finding.ruleId}`
|
|
7159
|
+
}))
|
|
7160
|
+
);
|
|
7161
|
+
return mergeContractResults(
|
|
7162
|
+
effectiveInput,
|
|
7163
|
+
[base],
|
|
7164
|
+
[...scopeFinding ? [scopeFinding] : [], noneFinding, ...prefixedBranchFindings]
|
|
7165
|
+
);
|
|
7166
|
+
}
|
|
5191
7167
|
|
|
5192
7168
|
// packages/core/src/checks/index.ts
|
|
5193
7169
|
var SEVERITY_RANK = {
|
|
@@ -5277,7 +7253,7 @@ var DEFAULT_SECRET_PATTERNS = [
|
|
|
5277
7253
|
function compareStrings(a, b) {
|
|
5278
7254
|
return (a ?? "").localeCompare(b ?? "");
|
|
5279
7255
|
}
|
|
5280
|
-
function
|
|
7256
|
+
function diagnostic2(code, message, ruleId) {
|
|
5281
7257
|
return {
|
|
5282
7258
|
code,
|
|
5283
7259
|
message,
|
|
@@ -5310,14 +7286,14 @@ function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
|
5310
7286
|
ruleExecutions: [...ruleExecutions]
|
|
5311
7287
|
};
|
|
5312
7288
|
}
|
|
5313
|
-
function
|
|
5314
|
-
return nodes.flatMap((node) => [node, ...
|
|
7289
|
+
function flattenNodes2(nodes) {
|
|
7290
|
+
return nodes.flatMap((node) => [node, ...flattenNodes2(node.children)]);
|
|
5315
7291
|
}
|
|
5316
7292
|
function buildFacts2(input, selectedRun) {
|
|
5317
7293
|
const scopedRuns = selectedRun ? [selectedRun] : input.read.runs;
|
|
5318
7294
|
const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
|
|
5319
7295
|
const scopedEvents = selectedRun === void 0 ? input.read.events : input.read.events.filter((event) => scopedRunIds.has(event.runId));
|
|
5320
|
-
const nodes =
|
|
7296
|
+
const nodes = flattenNodes2(scopedRuns.flatMap((run) => run.children));
|
|
5321
7297
|
const nodesByEventId = /* @__PURE__ */ new Map();
|
|
5322
7298
|
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
5323
7299
|
for (const node of nodes) {
|
|
@@ -5349,7 +7325,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
5349
7325
|
if (runId && input.selectedRun.runId !== runId) {
|
|
5350
7326
|
return {
|
|
5351
7327
|
diagnostics: [
|
|
5352
|
-
|
|
7328
|
+
diagnostic2(
|
|
5353
7329
|
"AI_CHECK_INVALID_ARGUMENTS",
|
|
5354
7330
|
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
5355
7331
|
)
|
|
@@ -5363,7 +7339,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
5363
7339
|
if (!run) {
|
|
5364
7340
|
return {
|
|
5365
7341
|
diagnostics: [
|
|
5366
|
-
|
|
7342
|
+
diagnostic2("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
5367
7343
|
]
|
|
5368
7344
|
};
|
|
5369
7345
|
}
|
|
@@ -5375,7 +7351,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
5375
7351
|
if (input.read.runs.length === 0) {
|
|
5376
7352
|
return {
|
|
5377
7353
|
diagnostics: [
|
|
5378
|
-
|
|
7354
|
+
diagnostic2(
|
|
5379
7355
|
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
5380
7356
|
formatProgrammaticDiagnostic(
|
|
5381
7357
|
"AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
@@ -5387,7 +7363,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
5387
7363
|
}
|
|
5388
7364
|
return {
|
|
5389
7365
|
diagnostics: [
|
|
5390
|
-
|
|
7366
|
+
diagnostic2(
|
|
5391
7367
|
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
5392
7368
|
formatProgrammaticDiagnostic("AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED")
|
|
5393
7369
|
)
|
|
@@ -5400,7 +7376,7 @@ function selectRules(rules, selectedIds) {
|
|
|
5400
7376
|
for (const rule of rules) {
|
|
5401
7377
|
if (byId.has(rule.id)) {
|
|
5402
7378
|
diagnostics.push(
|
|
5403
|
-
|
|
7379
|
+
diagnostic2("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
5404
7380
|
);
|
|
5405
7381
|
continue;
|
|
5406
7382
|
}
|
|
@@ -5411,7 +7387,7 @@ function selectRules(rules, selectedIds) {
|
|
|
5411
7387
|
for (const id of selected) {
|
|
5412
7388
|
if (!byId.has(id)) {
|
|
5413
7389
|
diagnostics.push(
|
|
5414
|
-
|
|
7390
|
+
diagnostic2("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
5415
7391
|
);
|
|
5416
7392
|
}
|
|
5417
7393
|
}
|
|
@@ -5526,7 +7502,7 @@ function stripPrefix(name, prefixes) {
|
|
|
5526
7502
|
}
|
|
5527
7503
|
return name;
|
|
5528
7504
|
}
|
|
5529
|
-
function
|
|
7505
|
+
function eventEvidence3(event, path14) {
|
|
5530
7506
|
return {
|
|
5531
7507
|
runId: event.runId,
|
|
5532
7508
|
eventId: event.eventId,
|
|
@@ -5586,6 +7562,14 @@ function finishedEvents(context, kind) {
|
|
|
5586
7562
|
function toolInvocationEvents(context) {
|
|
5587
7563
|
return semanticEvents(context).filter((event) => event.kind === "TOOL");
|
|
5588
7564
|
}
|
|
7565
|
+
function firstIndexByName(events) {
|
|
7566
|
+
const index = /* @__PURE__ */ new Map();
|
|
7567
|
+
for (const [position, event] of events.entries()) {
|
|
7568
|
+
const name = toolName(event);
|
|
7569
|
+
if (!index.has(name)) index.set(name, position);
|
|
7570
|
+
}
|
|
7571
|
+
return index;
|
|
7572
|
+
}
|
|
5589
7573
|
function isRecord9(value) {
|
|
5590
7574
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
5591
7575
|
}
|
|
@@ -5674,6 +7658,18 @@ function limitFindings(findings, maxFindings) {
|
|
|
5674
7658
|
function hasRedactionMarker(value, markers) {
|
|
5675
7659
|
return markers.some((marker) => value.includes(marker)) || /^\[HASH:[A-Za-z0-9_-]+\]$/.test(value);
|
|
5676
7660
|
}
|
|
7661
|
+
function isFullyRedactedValue(value, markers = ["[REDACTED]", "[REDACTED:"]) {
|
|
7662
|
+
if (typeof value !== "string") return false;
|
|
7663
|
+
const trimmed = value.trim();
|
|
7664
|
+
if (trimmed.length === 0) return false;
|
|
7665
|
+
if (/^\[HASH:[A-Za-z0-9_-]+\]$/.test(trimmed)) return true;
|
|
7666
|
+
if (/^\[REDACTED:[^\]]*\]$/.test(trimmed)) return true;
|
|
7667
|
+
for (const marker of markers) {
|
|
7668
|
+
if (marker.endsWith(":")) continue;
|
|
7669
|
+
if (trimmed === marker) return true;
|
|
7670
|
+
}
|
|
7671
|
+
return false;
|
|
7672
|
+
}
|
|
5677
7673
|
function isSensitiveKey(key, sensitiveKeys) {
|
|
5678
7674
|
return isCredentialSensitiveKey(key, sensitiveKeys);
|
|
5679
7675
|
}
|
|
@@ -5785,7 +7781,7 @@ function guardrailShape(context) {
|
|
|
5785
7781
|
}
|
|
5786
7782
|
function firstEvidenceForKind(context, kind, path14) {
|
|
5787
7783
|
const event = semanticEvents(context).find((candidate) => candidate.kind === kind);
|
|
5788
|
-
return event ? [
|
|
7784
|
+
return event ? [eventEvidence3(event, path14)] : runEvidence(context.selectedRun);
|
|
5789
7785
|
}
|
|
5790
7786
|
function baselineDiffFinding(message, evidence, expected, actual) {
|
|
5791
7787
|
return failFinding("baseline.regression", message, evidence, expected, actual);
|
|
@@ -5818,7 +7814,7 @@ function createRunStatusRule(options = {}) {
|
|
|
5818
7814
|
failFinding(
|
|
5819
7815
|
"run.status",
|
|
5820
7816
|
"Run contains incomplete running events.",
|
|
5821
|
-
running.map((event) =>
|
|
7817
|
+
running.map((event) => eventEvidence3(event)),
|
|
5822
7818
|
"no running events",
|
|
5823
7819
|
running.length
|
|
5824
7820
|
)
|
|
@@ -5864,7 +7860,7 @@ function createMaxStepDurationRule(options) {
|
|
|
5864
7860
|
failFinding(
|
|
5865
7861
|
"run.maxStepDuration",
|
|
5866
7862
|
`${over.length} step(s) exceeded max duration ${options.maxDurationMs}ms.`,
|
|
5867
|
-
over.map((event) =>
|
|
7863
|
+
over.map((event) => eventEvidence3(event, "durationMs")),
|
|
5868
7864
|
{ maxDurationMs: options.maxDurationMs },
|
|
5869
7865
|
over.map((event) => ({
|
|
5870
7866
|
eventId: event.eventId,
|
|
@@ -5890,7 +7886,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
5890
7886
|
failFinding(
|
|
5891
7887
|
"run.stall",
|
|
5892
7888
|
`Found ${running.length} event(s) still running (possible stall).`,
|
|
5893
|
-
running.map((event) =>
|
|
7889
|
+
running.map((event) => eventEvidence3(event, "status")),
|
|
5894
7890
|
"no running events",
|
|
5895
7891
|
running.length
|
|
5896
7892
|
)
|
|
@@ -5905,7 +7901,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
5905
7901
|
failFinding(
|
|
5906
7902
|
"run.stall",
|
|
5907
7903
|
`Found ${incomplete.length} started event(s) without endedAt.`,
|
|
5908
|
-
incomplete.map((event) =>
|
|
7904
|
+
incomplete.map((event) => eventEvidence3(event, "endedAt")),
|
|
5909
7905
|
"endedAt for started events",
|
|
5910
7906
|
incomplete.length
|
|
5911
7907
|
)
|
|
@@ -5941,7 +7937,7 @@ function createRequireCompletedRule() {
|
|
|
5941
7937
|
failFinding(
|
|
5942
7938
|
"run.requireCompleted",
|
|
5943
7939
|
`Run has ${running.length} incomplete running event(s).`,
|
|
5944
|
-
running.map((event) =>
|
|
7940
|
+
running.map((event) => eventEvidence3(event, "status")),
|
|
5945
7941
|
"no running events",
|
|
5946
7942
|
running.length
|
|
5947
7943
|
)
|
|
@@ -5992,8 +7988,26 @@ function createToolUsageRule(options) {
|
|
|
5992
7988
|
const findings = [];
|
|
5993
7989
|
for (const required of options.required ?? []) {
|
|
5994
7990
|
if (!nameSet.has(required)) {
|
|
7991
|
+
const otherKinds = [
|
|
7992
|
+
...new Set(
|
|
7993
|
+
context.events.filter((event) => {
|
|
7994
|
+
const kind = typeof event.kind === "string" ? event.kind.toUpperCase() : "";
|
|
7995
|
+
if (kind === "TOOL") return false;
|
|
7996
|
+
return resolveCanonicalToolName(event) === required || event.name === required || event.name === `tool:${required}` || event.name.endsWith(`:${required}`);
|
|
7997
|
+
}).map(
|
|
7998
|
+
(event) => typeof event.kind === "string" ? event.kind.toUpperCase() : "UNKNOWN"
|
|
7999
|
+
)
|
|
8000
|
+
)
|
|
8001
|
+
].sort();
|
|
8002
|
+
const hint = otherKinds.length > 0 ? ` Required name appears under non-TOOL kind(s): ${otherKinds.join(", ")}. tools.required / requiredOrder match TOOL events only.` : "";
|
|
5995
8003
|
findings.push(
|
|
5996
|
-
failFinding(
|
|
8004
|
+
failFinding(
|
|
8005
|
+
"tool.usage",
|
|
8006
|
+
`Required tool ${required} did not appear.${hint}`,
|
|
8007
|
+
runEvidence(context.selectedRun),
|
|
8008
|
+
required,
|
|
8009
|
+
names
|
|
8010
|
+
)
|
|
5997
8011
|
);
|
|
5998
8012
|
}
|
|
5999
8013
|
}
|
|
@@ -6003,12 +8017,12 @@ function createToolUsageRule(options) {
|
|
|
6003
8017
|
const name = toolName(event);
|
|
6004
8018
|
if (forbidden.has(name)) {
|
|
6005
8019
|
findings.push(
|
|
6006
|
-
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [
|
|
8020
|
+
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence3(event)], "tool absent", name)
|
|
6007
8021
|
);
|
|
6008
8022
|
}
|
|
6009
8023
|
if (allowed && !allowed.has(name)) {
|
|
6010
8024
|
findings.push(
|
|
6011
|
-
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [
|
|
8025
|
+
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence3(event)], [...allowed].sort(), name)
|
|
6012
8026
|
);
|
|
6013
8027
|
}
|
|
6014
8028
|
}
|
|
@@ -6019,13 +8033,188 @@ function createToolUsageRule(options) {
|
|
|
6019
8033
|
}
|
|
6020
8034
|
if (options.maxCount !== void 0 && tools.length > options.maxCount) {
|
|
6021
8035
|
findings.push(
|
|
6022
|
-
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) =>
|
|
8036
|
+
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) => eventEvidence3(event)), { maxCount: options.maxCount }, tools.length)
|
|
6023
8037
|
);
|
|
6024
8038
|
}
|
|
6025
8039
|
return findings;
|
|
6026
8040
|
}
|
|
6027
8041
|
};
|
|
6028
8042
|
}
|
|
8043
|
+
function createToolOrderingRule(options) {
|
|
8044
|
+
const ruleId = options.id ?? "tool.order";
|
|
8045
|
+
const mode = options.mode ?? "first-occurrence";
|
|
8046
|
+
return {
|
|
8047
|
+
id: ruleId,
|
|
8048
|
+
category: "tool",
|
|
8049
|
+
defaultSeverity: "error",
|
|
8050
|
+
evaluate(context) {
|
|
8051
|
+
const tools = finishedEvents(context, "TOOL");
|
|
8052
|
+
const index = firstIndexByName(tools);
|
|
8053
|
+
const beforeIndex = index.get(options.before);
|
|
8054
|
+
const afterIndex = index.get(options.after);
|
|
8055
|
+
if (beforeIndex === void 0 || afterIndex === void 0) {
|
|
8056
|
+
return [];
|
|
8057
|
+
}
|
|
8058
|
+
if (mode === "first-occurrence" && beforeIndex >= afterIndex) {
|
|
8059
|
+
return [
|
|
8060
|
+
failFinding(
|
|
8061
|
+
ruleId,
|
|
8062
|
+
`Tool ${options.before} must appear before ${options.after}.`,
|
|
8063
|
+
[eventEvidence3(tools[beforeIndex]), eventEvidence3(tools[afterIndex])],
|
|
8064
|
+
{ before: options.before, after: options.after },
|
|
8065
|
+
tools.map(toolName)
|
|
8066
|
+
)
|
|
8067
|
+
];
|
|
8068
|
+
}
|
|
8069
|
+
const beforeEvent = tools[beforeIndex];
|
|
8070
|
+
const afterEvent = tools[afterIndex];
|
|
8071
|
+
const beforeEnd = eventEndMs(beforeEvent);
|
|
8072
|
+
const afterStart = eventStartMs(afterEvent);
|
|
8073
|
+
if (mode === "happens-before") {
|
|
8074
|
+
const expected = {
|
|
8075
|
+
before: options.before,
|
|
8076
|
+
after: options.after,
|
|
8077
|
+
mode,
|
|
8078
|
+
relation: "before.end <= after.start"
|
|
8079
|
+
};
|
|
8080
|
+
if (options.before === options.after) {
|
|
8081
|
+
return [
|
|
8082
|
+
failFinding(
|
|
8083
|
+
ruleId,
|
|
8084
|
+
`Tool ${options.before} cannot causally happen before itself.`,
|
|
8085
|
+
[eventEvidence3(beforeEvent)],
|
|
8086
|
+
expected,
|
|
8087
|
+
{ code: "tool.order.same-tool" }
|
|
8088
|
+
)
|
|
8089
|
+
];
|
|
8090
|
+
}
|
|
8091
|
+
if (beforeEnd === void 0 || afterStart === void 0) {
|
|
8092
|
+
return [
|
|
8093
|
+
failFinding(
|
|
8094
|
+
ruleId,
|
|
8095
|
+
`Tool order ${options.before} before ${options.after} could not establish causal timing.`,
|
|
8096
|
+
[eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
|
|
8097
|
+
expected,
|
|
8098
|
+
{
|
|
8099
|
+
code: "tool.order.interval-unresolved",
|
|
8100
|
+
beforeEndResolved: beforeEnd !== void 0,
|
|
8101
|
+
afterStartResolved: afterStart !== void 0
|
|
8102
|
+
}
|
|
8103
|
+
)
|
|
8104
|
+
];
|
|
8105
|
+
}
|
|
8106
|
+
if (beforeEnd > afterStart) {
|
|
8107
|
+
return [
|
|
8108
|
+
failFinding(
|
|
8109
|
+
ruleId,
|
|
8110
|
+
`Tool ${options.before} must finish before ${options.after} starts.`,
|
|
8111
|
+
[eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
|
|
8112
|
+
expected,
|
|
8113
|
+
{
|
|
8114
|
+
beforeEndedAt: new Date(beforeEnd).toISOString(),
|
|
8115
|
+
afterStartedAt: new Date(afterStart).toISOString()
|
|
8116
|
+
}
|
|
8117
|
+
)
|
|
8118
|
+
];
|
|
8119
|
+
}
|
|
8120
|
+
return [];
|
|
8121
|
+
}
|
|
8122
|
+
if (mode === "all-occurrences") {
|
|
8123
|
+
const beforeEvents = tools.filter((event) => toolName(event) === options.before);
|
|
8124
|
+
const afterEvents = tools.filter((event) => toolName(event) === options.after);
|
|
8125
|
+
const expected = {
|
|
8126
|
+
before: options.before,
|
|
8127
|
+
after: options.after,
|
|
8128
|
+
mode,
|
|
8129
|
+
relation: "max(before.end) <= min(after.start)"
|
|
8130
|
+
};
|
|
8131
|
+
if (options.before === options.after) {
|
|
8132
|
+
return [
|
|
8133
|
+
failFinding(
|
|
8134
|
+
ruleId,
|
|
8135
|
+
`Tool ${options.before} cannot causally happen before itself.`,
|
|
8136
|
+
[eventEvidence3(beforeEvent)],
|
|
8137
|
+
expected,
|
|
8138
|
+
{ code: "tool.order.same-tool" }
|
|
8139
|
+
)
|
|
8140
|
+
];
|
|
8141
|
+
}
|
|
8142
|
+
let latestBefore;
|
|
8143
|
+
let earliestAfter;
|
|
8144
|
+
let missingBeforeEnds = 0;
|
|
8145
|
+
let missingAfterStarts = 0;
|
|
8146
|
+
let firstMissingBefore;
|
|
8147
|
+
let firstMissingAfter;
|
|
8148
|
+
for (const event of beforeEvents) {
|
|
8149
|
+
const end = eventEndMs(event);
|
|
8150
|
+
if (end === void 0) {
|
|
8151
|
+
missingBeforeEnds += 1;
|
|
8152
|
+
firstMissingBefore ??= event;
|
|
8153
|
+
} else if (latestBefore === void 0 || end > latestBefore.end) {
|
|
8154
|
+
latestBefore = { event, end };
|
|
8155
|
+
}
|
|
8156
|
+
}
|
|
8157
|
+
for (const event of afterEvents) {
|
|
8158
|
+
const start = eventStartMs(event);
|
|
8159
|
+
if (start === void 0) {
|
|
8160
|
+
missingAfterStarts += 1;
|
|
8161
|
+
firstMissingAfter ??= event;
|
|
8162
|
+
} else if (earliestAfter === void 0 || start < earliestAfter.start) {
|
|
8163
|
+
earliestAfter = { event, start };
|
|
8164
|
+
}
|
|
8165
|
+
}
|
|
8166
|
+
if (missingBeforeEnds > 0 || missingAfterStarts > 0 || latestBefore === void 0 || earliestAfter === void 0) {
|
|
8167
|
+
return [
|
|
8168
|
+
failFinding(
|
|
8169
|
+
ruleId,
|
|
8170
|
+
`Tool order ${options.before} before ${options.after} could not establish all causal intervals.`,
|
|
8171
|
+
[firstMissingBefore, firstMissingAfter].filter((event) => event !== void 0).map((event) => eventEvidence3(event)),
|
|
8172
|
+
expected,
|
|
8173
|
+
{
|
|
8174
|
+
code: "tool.order.interval-unresolved",
|
|
8175
|
+
missingBeforeEnds,
|
|
8176
|
+
missingAfterStarts
|
|
8177
|
+
}
|
|
8178
|
+
)
|
|
8179
|
+
];
|
|
8180
|
+
}
|
|
8181
|
+
if (latestBefore.end > earliestAfter.start) {
|
|
8182
|
+
return [
|
|
8183
|
+
failFinding(
|
|
8184
|
+
ruleId,
|
|
8185
|
+
`Every ${options.before} tool call must finish before any ${options.after} tool call starts.`,
|
|
8186
|
+
[eventEvidence3(latestBefore.event), eventEvidence3(earliestAfter.event)],
|
|
8187
|
+
expected,
|
|
8188
|
+
{
|
|
8189
|
+
latestBeforeEndedAt: new Date(latestBefore.end).toISOString(),
|
|
8190
|
+
earliestAfterStartedAt: new Date(earliestAfter.start).toISOString()
|
|
8191
|
+
}
|
|
8192
|
+
)
|
|
8193
|
+
];
|
|
8194
|
+
}
|
|
8195
|
+
return [];
|
|
8196
|
+
}
|
|
8197
|
+
if (beforeEnd !== void 0 && afterStart !== void 0 && beforeEnd > afterStart) {
|
|
8198
|
+
return [
|
|
8199
|
+
{
|
|
8200
|
+
ruleId,
|
|
8201
|
+
severity: "warning",
|
|
8202
|
+
status: "warning",
|
|
8203
|
+
message: `Tool order ${options.before} before ${options.after} satisfied start/encounter order, but intervals overlap so causal completion before the next tool was not established.`,
|
|
8204
|
+
expected: { before: options.before, after: options.after, mode: "first-occurrence" },
|
|
8205
|
+
actual: {
|
|
8206
|
+
code: "tool.order.overlap",
|
|
8207
|
+
beforeEndedAt: beforeEvent.endedAt,
|
|
8208
|
+
afterStartedAt: afterEvent.startedAt ?? afterEvent.timestamp
|
|
8209
|
+
},
|
|
8210
|
+
evidence: [eventEvidence3(beforeEvent), eventEvidence3(afterEvent)]
|
|
8211
|
+
}
|
|
8212
|
+
];
|
|
8213
|
+
}
|
|
8214
|
+
return [];
|
|
8215
|
+
}
|
|
8216
|
+
};
|
|
8217
|
+
}
|
|
6029
8218
|
function createLlmUsageRule(options) {
|
|
6030
8219
|
return {
|
|
6031
8220
|
id: "llm.usage",
|
|
@@ -6042,7 +8231,7 @@ function createLlmUsageRule(options) {
|
|
|
6042
8231
|
failFinding(
|
|
6043
8232
|
"llm.usage",
|
|
6044
8233
|
`LLM call count ${llms.length} exceeded ${options.maxCalls}.`,
|
|
6045
|
-
llms.map((event) =>
|
|
8234
|
+
llms.map((event) => eventEvidence3(event)),
|
|
6046
8235
|
{ maxCalls: options.maxCalls },
|
|
6047
8236
|
llms.length
|
|
6048
8237
|
)
|
|
@@ -6054,17 +8243,17 @@ function createLlmUsageRule(options) {
|
|
|
6054
8243
|
const finishReason = llmFinishReason(event);
|
|
6055
8244
|
if (allowedModels && (!model || !allowedModels.has(model))) {
|
|
6056
8245
|
findings.push(
|
|
6057
|
-
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [
|
|
8246
|
+
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [eventEvidence3(event, "attributes.model")], [...allowedModels].sort(), model ?? "unknown")
|
|
6058
8247
|
);
|
|
6059
8248
|
}
|
|
6060
8249
|
if (allowedProviders && (!provider || !allowedProviders.has(provider))) {
|
|
6061
8250
|
findings.push(
|
|
6062
|
-
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [
|
|
8251
|
+
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [eventEvidence3(event, "attributes.provider")], [...allowedProviders].sort(), provider ?? "unknown")
|
|
6063
8252
|
);
|
|
6064
8253
|
}
|
|
6065
8254
|
if (finishReasons && (!finishReason || !finishReasons.has(finishReason))) {
|
|
6066
8255
|
findings.push(
|
|
6067
|
-
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [
|
|
8256
|
+
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [eventEvidence3(event, "attributes.finishReason")], [...finishReasons].sort(), finishReason ?? "unknown")
|
|
6068
8257
|
);
|
|
6069
8258
|
}
|
|
6070
8259
|
}
|
|
@@ -6089,7 +8278,7 @@ function createLlmUsageRule(options) {
|
|
|
6089
8278
|
failFinding(
|
|
6090
8279
|
"llm.usage",
|
|
6091
8280
|
`LLM ${key} token count ${tokenTotals[key]} exceeded ${limit}.`,
|
|
6092
|
-
llms.map((event) =>
|
|
8281
|
+
llms.map((event) => eventEvidence3(event, `tokenUsage.${key}`)),
|
|
6093
8282
|
{ [`max${key[0].toUpperCase()}${key.slice(1)}Tokens`]: limit },
|
|
6094
8283
|
tokenTotals[key]
|
|
6095
8284
|
)
|
|
@@ -6100,6 +8289,47 @@ function createLlmUsageRule(options) {
|
|
|
6100
8289
|
}
|
|
6101
8290
|
};
|
|
6102
8291
|
}
|
|
8292
|
+
function createStructureIncompleteRule(options = {}) {
|
|
8293
|
+
return {
|
|
8294
|
+
id: "structure.incomplete",
|
|
8295
|
+
category: "structure",
|
|
8296
|
+
defaultSeverity: "error",
|
|
8297
|
+
evaluate(context) {
|
|
8298
|
+
const findings = [];
|
|
8299
|
+
if (!options.allowRunning) {
|
|
8300
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
8301
|
+
if (running.length > 0) {
|
|
8302
|
+
findings.push(
|
|
8303
|
+
failFinding(
|
|
8304
|
+
"structure.incomplete",
|
|
8305
|
+
"Trace contains incomplete running events.",
|
|
8306
|
+
running.map((event) => eventEvidence3(event, "status")),
|
|
8307
|
+
"no running events",
|
|
8308
|
+
running.length
|
|
8309
|
+
)
|
|
8310
|
+
);
|
|
8311
|
+
}
|
|
8312
|
+
}
|
|
8313
|
+
if (options.requireEndedAtForStarted) {
|
|
8314
|
+
const missingEndedAt = semanticEvents(context).filter(
|
|
8315
|
+
(event) => event.startedAt !== void 0 && event.endedAt === void 0 && event.status !== "running"
|
|
8316
|
+
);
|
|
8317
|
+
if (missingEndedAt.length > 0) {
|
|
8318
|
+
findings.push(
|
|
8319
|
+
failFinding(
|
|
8320
|
+
"structure.incomplete",
|
|
8321
|
+
"Trace contains events with startedAt but no endedAt.",
|
|
8322
|
+
missingEndedAt.map((event) => eventEvidence3(event, "endedAt")),
|
|
8323
|
+
"endedAt for started events",
|
|
8324
|
+
missingEndedAt.length
|
|
8325
|
+
)
|
|
8326
|
+
);
|
|
8327
|
+
}
|
|
8328
|
+
}
|
|
8329
|
+
return findings;
|
|
8330
|
+
}
|
|
8331
|
+
};
|
|
8332
|
+
}
|
|
6103
8333
|
function createStructureOrphanRule(options = {}) {
|
|
6104
8334
|
const allowMarkedUnresolved = options.allowMarkedUnresolved ?? true;
|
|
6105
8335
|
return {
|
|
@@ -6117,7 +8347,7 @@ function createStructureOrphanRule(options = {}) {
|
|
|
6117
8347
|
failFinding(
|
|
6118
8348
|
"structure.orphan",
|
|
6119
8349
|
"Trace contains events whose parentId is not present in the selected run.",
|
|
6120
|
-
orphans.map((event) =>
|
|
8350
|
+
orphans.map((event) => eventEvidence3(event, "parentId")),
|
|
6121
8351
|
"parentId resolves to an event in the selected run",
|
|
6122
8352
|
orphans.length
|
|
6123
8353
|
)
|
|
@@ -6149,7 +8379,7 @@ function createStructureCycleRule() {
|
|
|
6149
8379
|
failFinding(
|
|
6150
8380
|
"structure.cycle",
|
|
6151
8381
|
formatProgrammaticDiagnostic("AI_TRACE_RELATIONSHIP_CYCLE"),
|
|
6152
|
-
cycle.map((item) =>
|
|
8382
|
+
cycle.map((item) => eventEvidence3(item, "parentId")),
|
|
6153
8383
|
"acyclic parentId graph",
|
|
6154
8384
|
cycle.map((item) => item.eventId).sort()
|
|
6155
8385
|
)
|
|
@@ -6189,7 +8419,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
6189
8419
|
failFinding(
|
|
6190
8420
|
"structure.relationship",
|
|
6191
8421
|
`Event confidence ${event.confidence} is below ${minConfidence}.`,
|
|
6192
|
-
[
|
|
8422
|
+
[eventEvidence3(event, "confidence")],
|
|
6193
8423
|
{ minConfidence },
|
|
6194
8424
|
event.confidence
|
|
6195
8425
|
)
|
|
@@ -6202,7 +8432,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
6202
8432
|
failFinding(
|
|
6203
8433
|
"structure.relationship",
|
|
6204
8434
|
"Event parentId points to itself.",
|
|
6205
|
-
[
|
|
8435
|
+
[eventEvidence3(event, "parentId")],
|
|
6206
8436
|
"parentId references a distinct event",
|
|
6207
8437
|
"self"
|
|
6208
8438
|
)
|
|
@@ -6219,7 +8449,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
6219
8449
|
failFinding(
|
|
6220
8450
|
"structure.relationship",
|
|
6221
8451
|
"Parent event starts after child event.",
|
|
6222
|
-
[
|
|
8452
|
+
[eventEvidence3(parent), eventEvidence3(event, "parentId")],
|
|
6223
8453
|
"parent start <= child start",
|
|
6224
8454
|
{ parentEventId: parent.eventId, childEventId: event.eventId }
|
|
6225
8455
|
)
|
|
@@ -6233,7 +8463,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
6233
8463
|
failFinding(
|
|
6234
8464
|
"structure.relationship",
|
|
6235
8465
|
"Trace parentSpanId does not match parent spanId.",
|
|
6236
|
-
[
|
|
8466
|
+
[eventEvidence3(event, "trace.parentSpanId")],
|
|
6237
8467
|
{ parentSpanId: parent.trace.spanId },
|
|
6238
8468
|
actual ?? "missing"
|
|
6239
8469
|
)
|
|
@@ -6262,7 +8492,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
6262
8492
|
"structure.parallelWidth",
|
|
6263
8493
|
`Parent ${parentId} has ${children.length} children, exceeding ${options.maxChildren}.`,
|
|
6264
8494
|
[
|
|
6265
|
-
...parent ? [
|
|
8495
|
+
...parent ? [eventEvidence3(parent)] : [{ runId: context.selectedRun?.runId, eventId: parentId }],
|
|
6266
8496
|
...children.map((child) => ({
|
|
6267
8497
|
runId: child.event.runId,
|
|
6268
8498
|
eventId: child.event.eventId,
|
|
@@ -6310,7 +8540,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
6310
8540
|
failFinding(
|
|
6311
8541
|
"structure.parallelWidth",
|
|
6312
8542
|
`Concurrent event width ${maxActive.length} exceeded ${options.maxConcurrent}.`,
|
|
6313
|
-
maxActive.map((event) =>
|
|
8543
|
+
maxActive.map((event) => eventEvidence3(event)),
|
|
6314
8544
|
{ maxConcurrent: options.maxConcurrent },
|
|
6315
8545
|
maxActive.length
|
|
6316
8546
|
)
|
|
@@ -6338,7 +8568,7 @@ function createSafetyRedactionRule(options = {}) {
|
|
|
6338
8568
|
failFinding(
|
|
6339
8569
|
"safety.redaction",
|
|
6340
8570
|
`Sensitive-looking field at ${entry.path} is not redacted.`,
|
|
6341
|
-
[
|
|
8571
|
+
[eventEvidence3(event, entry.path)],
|
|
6342
8572
|
"redaction marker",
|
|
6343
8573
|
{ path: entry.path, valueType: valueType(entry.value) },
|
|
6344
8574
|
{
|
|
@@ -6358,6 +8588,7 @@ function createSafetyRedactionRule(options = {}) {
|
|
|
6358
8588
|
function createSafetyRawContentRule(options = {}) {
|
|
6359
8589
|
const forbiddenKeys = options.forbiddenKeys ?? DEFAULT_RAW_CONTENT_KEYS;
|
|
6360
8590
|
const safePathPrefixes = options.safePathPrefixes ?? DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES;
|
|
8591
|
+
const markers = options.redactedMarkers ?? ["[REDACTED]", "[REDACTED:"];
|
|
6361
8592
|
return {
|
|
6362
8593
|
id: "safety.rawPrompt",
|
|
6363
8594
|
category: "safety",
|
|
@@ -6368,11 +8599,12 @@ function createSafetyRawContentRule(options = {}) {
|
|
|
6368
8599
|
for (const entry of eventValueEntries(event, { includeSummaries: options.includeSummaries })) {
|
|
6369
8600
|
const key = entry.key ?? lastPathSegment(entry.path);
|
|
6370
8601
|
if (!isRawContentPath(entry.path, key, forbiddenKeys, safePathPrefixes)) continue;
|
|
8602
|
+
if (isFullyRedactedValue(entry.value, markers)) continue;
|
|
6371
8603
|
findings.push(
|
|
6372
8604
|
failFinding(
|
|
6373
8605
|
"safety.rawPrompt",
|
|
6374
8606
|
`Raw content-like field ${entry.path} is present.`,
|
|
6375
|
-
[
|
|
8607
|
+
[eventEvidence3(event, entry.path)],
|
|
6376
8608
|
"metadata-only trace fields",
|
|
6377
8609
|
{ path: entry.path, valueType: valueType(entry.value) },
|
|
6378
8610
|
{
|
|
@@ -6410,7 +8642,7 @@ function createSafetySecretPatternRule(options = {}) {
|
|
|
6410
8642
|
failFinding(
|
|
6411
8643
|
"safety.secretPattern",
|
|
6412
8644
|
`Secret-like pattern ${pattern.id} matched at ${entry.path}.`,
|
|
6413
|
-
[
|
|
8645
|
+
[eventEvidence3(event, entry.path)],
|
|
6414
8646
|
"no secret-like strings",
|
|
6415
8647
|
{ pattern: pattern.id, path: entry.path },
|
|
6416
8648
|
{
|
|
@@ -6443,7 +8675,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
6443
8675
|
failFinding(
|
|
6444
8676
|
"safety.oversizedAttribute",
|
|
6445
8677
|
`String at ${entry.path} exceeds ${options.maxStringLength} characters.`,
|
|
6446
|
-
[
|
|
8678
|
+
[eventEvidence3(event, entry.path)],
|
|
6447
8679
|
{ maxStringLength: options.maxStringLength },
|
|
6448
8680
|
{ path: entry.path, length: entry.value.length },
|
|
6449
8681
|
{
|
|
@@ -6460,7 +8692,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
6460
8692
|
failFinding(
|
|
6461
8693
|
"safety.oversizedAttribute",
|
|
6462
8694
|
`Array at ${entry.path} exceeds ${options.maxArrayLength} items.`,
|
|
6463
|
-
[
|
|
8695
|
+
[eventEvidence3(event, entry.path)],
|
|
6464
8696
|
{ maxArrayLength: options.maxArrayLength },
|
|
6465
8697
|
{ path: entry.path, length: entry.value.length },
|
|
6466
8698
|
{
|
|
@@ -6477,7 +8709,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
6477
8709
|
failFinding(
|
|
6478
8710
|
"safety.oversizedAttribute",
|
|
6479
8711
|
`Object at ${entry.path} exceeds ${options.maxObjectKeys} keys.`,
|
|
6480
|
-
[
|
|
8712
|
+
[eventEvidence3(event, entry.path)],
|
|
6481
8713
|
{ maxObjectKeys: options.maxObjectKeys },
|
|
6482
8714
|
{ path: entry.path, keys: Object.keys(entry.value).length },
|
|
6483
8715
|
{
|
|
@@ -6496,7 +8728,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
6496
8728
|
failFinding(
|
|
6497
8729
|
"safety.oversizedAttribute",
|
|
6498
8730
|
`Value at ${entry.path} exceeds ${options.maxSerializedBytes} serialized bytes.`,
|
|
6499
|
-
[
|
|
8731
|
+
[eventEvidence3(event, entry.path)],
|
|
6500
8732
|
{ maxSerializedBytes: options.maxSerializedBytes },
|
|
6501
8733
|
{ path: entry.path, bytes },
|
|
6502
8734
|
{
|
|
@@ -6623,7 +8855,7 @@ function createBaselineRegressionRule(options) {
|
|
|
6623
8855
|
path: "guardrail",
|
|
6624
8856
|
expected: guardrailShape(baselineContext),
|
|
6625
8857
|
actual: guardrailShape(context),
|
|
6626
|
-
evidence: guardrailEvents(context)[0] ? [
|
|
8858
|
+
evidence: guardrailEvents(context)[0] ? [eventEvidence3(guardrailEvents(context)[0], "guardrail")] : runEvidence(context.selectedRun)
|
|
6627
8859
|
}
|
|
6628
8860
|
];
|
|
6629
8861
|
for (const comparison of comparisons) {
|
|
@@ -6702,7 +8934,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
6702
8934
|
return errorResult(
|
|
6703
8935
|
input,
|
|
6704
8936
|
[
|
|
6705
|
-
|
|
8937
|
+
diagnostic2(
|
|
6706
8938
|
"AI_CHECK_NO_RULES_EVALUATED",
|
|
6707
8939
|
"No trace check rules were evaluated. Configure at least one rule, contract, or CLI check option."
|
|
6708
8940
|
)
|
|
@@ -6733,7 +8965,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
6733
8965
|
} catch (error) {
|
|
6734
8966
|
const message = error instanceof Error ? error.message : String(error);
|
|
6735
8967
|
diagnostics.push(
|
|
6736
|
-
|
|
8968
|
+
diagnostic2("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
|
|
6737
8969
|
);
|
|
6738
8970
|
ruleExecutions.push({
|
|
6739
8971
|
ruleId: rule.id,
|
|
@@ -6797,18 +9029,18 @@ function escapeMarkdown(value) {
|
|
|
6797
9029
|
function escapeHtml(value) {
|
|
6798
9030
|
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
6799
9031
|
}
|
|
6800
|
-
function
|
|
9032
|
+
function sortKeysDeep2(input) {
|
|
6801
9033
|
if (input === null || typeof input !== "object") return input;
|
|
6802
|
-
if (Array.isArray(input)) return input.map(
|
|
9034
|
+
if (Array.isArray(input)) return input.map(sortKeysDeep2);
|
|
6803
9035
|
const o = input;
|
|
6804
9036
|
const out = {};
|
|
6805
9037
|
for (const k of Object.keys(o).sort()) {
|
|
6806
|
-
out[k] =
|
|
9038
|
+
out[k] = sortKeysDeep2(o[k]);
|
|
6807
9039
|
}
|
|
6808
9040
|
return out;
|
|
6809
9041
|
}
|
|
6810
9042
|
function stableJson(value, pretty) {
|
|
6811
|
-
const sorted =
|
|
9043
|
+
const sorted = sortKeysDeep2(value);
|
|
6812
9044
|
return pretty === true ? JSON.stringify(sorted, null, 2) : JSON.stringify(sorted);
|
|
6813
9045
|
}
|
|
6814
9046
|
function compactAttributes(attrs, options) {
|
|
@@ -8668,7 +10900,7 @@ var DEFAULT_SUITE_CONFIG_NAMES = [
|
|
|
8668
10900
|
"agent-inspect.suite.cjs"
|
|
8669
10901
|
];
|
|
8670
10902
|
var DEFAULT_SUITE_ARTIFACTS_DIR = ".agent-inspect/suite-runs";
|
|
8671
|
-
function
|
|
10903
|
+
function diagnostic3(code, message, severity = "error", caseId) {
|
|
8672
10904
|
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
8673
10905
|
}
|
|
8674
10906
|
function asString(value, label) {
|
|
@@ -8696,7 +10928,7 @@ function validateCaseConfig(value, index) {
|
|
|
8696
10928
|
const diagnostics = [];
|
|
8697
10929
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
8698
10930
|
diagnostics.push(
|
|
8699
|
-
|
|
10931
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
|
|
8700
10932
|
);
|
|
8701
10933
|
return { diagnostics };
|
|
8702
10934
|
}
|
|
@@ -8705,7 +10937,7 @@ function validateCaseConfig(value, index) {
|
|
|
8705
10937
|
const id = asString(raw.id, `cases[${index}].id`);
|
|
8706
10938
|
if (id === void 0) {
|
|
8707
10939
|
diagnostics.push(
|
|
8708
|
-
|
|
10940
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
|
|
8709
10941
|
);
|
|
8710
10942
|
return { diagnostics };
|
|
8711
10943
|
}
|
|
@@ -8740,7 +10972,7 @@ function validateCaseConfig(value, index) {
|
|
|
8740
10972
|
};
|
|
8741
10973
|
} catch (error) {
|
|
8742
10974
|
const message = error instanceof Error ? error.message : String(error);
|
|
8743
|
-
diagnostics.push(
|
|
10975
|
+
diagnostics.push(diagnostic3("AI_SUITE_CONFIG_INVALID", message));
|
|
8744
10976
|
return { diagnostics };
|
|
8745
10977
|
}
|
|
8746
10978
|
}
|
|
@@ -8816,7 +11048,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
8816
11048
|
await access(tracesDir);
|
|
8817
11049
|
} catch {
|
|
8818
11050
|
diagnostics.push(
|
|
8819
|
-
|
|
11051
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `traces directory not found: ${tracesDir}`)
|
|
8820
11052
|
);
|
|
8821
11053
|
}
|
|
8822
11054
|
for (const suiteCase of config.cases) {
|
|
@@ -8826,7 +11058,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
8826
11058
|
await access(inputPath);
|
|
8827
11059
|
} catch {
|
|
8828
11060
|
diagnostics.push(
|
|
8829
|
-
|
|
11061
|
+
diagnostic3(
|
|
8830
11062
|
"AI_SUITE_CONFIG_INVALID",
|
|
8831
11063
|
`input fixture not found: ${suiteCase.input}`,
|
|
8832
11064
|
"warning",
|
|
@@ -8840,7 +11072,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
8840
11072
|
}
|
|
8841
11073
|
var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
8842
11074
|
var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
8843
|
-
function
|
|
11075
|
+
function diagnostic4(code, message) {
|
|
8844
11076
|
return { code, message, severity: "error" };
|
|
8845
11077
|
}
|
|
8846
11078
|
async function fileExists(filePath) {
|
|
@@ -8871,20 +11103,20 @@ async function loadSuiteConfig(options = {}) {
|
|
|
8871
11103
|
} catch (error) {
|
|
8872
11104
|
const message = error instanceof Error ? error.message : String(error);
|
|
8873
11105
|
throw Object.assign(new Error(message), {
|
|
8874
|
-
diagnostics: [
|
|
11106
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
8875
11107
|
});
|
|
8876
11108
|
}
|
|
8877
11109
|
const extension = path7.extname(configPath);
|
|
8878
11110
|
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
8879
11111
|
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
8880
11112
|
throw Object.assign(new Error(message), {
|
|
8881
|
-
diagnostics: [
|
|
11113
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
8882
11114
|
});
|
|
8883
11115
|
}
|
|
8884
11116
|
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
8885
11117
|
const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
|
|
8886
11118
|
throw Object.assign(new Error(message), {
|
|
8887
|
-
diagnostics: [
|
|
11119
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
8888
11120
|
});
|
|
8889
11121
|
}
|
|
8890
11122
|
try {
|
|
@@ -8904,7 +11136,7 @@ async function loadSuiteConfig(options = {}) {
|
|
|
8904
11136
|
} catch (error) {
|
|
8905
11137
|
const message = error instanceof Error ? error.message : String(error);
|
|
8906
11138
|
throw Object.assign(new Error(message), {
|
|
8907
|
-
diagnostics: [
|
|
11139
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
8908
11140
|
});
|
|
8909
11141
|
}
|
|
8910
11142
|
}
|
|
@@ -9827,13 +12059,13 @@ function persistedEventsForParsedTrace(parsed) {
|
|
|
9827
12059
|
function isRecord11(value) {
|
|
9828
12060
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
9829
12061
|
}
|
|
9830
|
-
function
|
|
12062
|
+
function isNonEmptyString5(value) {
|
|
9831
12063
|
return typeof value === "string" && value.trim() !== "";
|
|
9832
12064
|
}
|
|
9833
12065
|
function readStringField(record, keys) {
|
|
9834
12066
|
for (const key of keys) {
|
|
9835
12067
|
const value = record[key];
|
|
9836
|
-
if (
|
|
12068
|
+
if (isNonEmptyString5(value)) return value;
|
|
9837
12069
|
}
|
|
9838
12070
|
return void 0;
|
|
9839
12071
|
}
|
|
@@ -9967,7 +12199,7 @@ function parseUnixNanoToIso(value) {
|
|
|
9967
12199
|
return void 0;
|
|
9968
12200
|
}
|
|
9969
12201
|
function parseIsoTime(value) {
|
|
9970
|
-
if (!
|
|
12202
|
+
if (!isNonEmptyString5(value)) return void 0;
|
|
9971
12203
|
const ms = Date.parse(value);
|
|
9972
12204
|
if (!Number.isFinite(ms)) return void 0;
|
|
9973
12205
|
return new Date(ms).toISOString();
|
|
@@ -11177,7 +13409,7 @@ async function resolveSuiteCaseTrace(suiteCase, options) {
|
|
|
11177
13409
|
}
|
|
11178
13410
|
|
|
11179
13411
|
// packages/core/src/suite/run.ts
|
|
11180
|
-
function
|
|
13412
|
+
function diagnostic5(code, message, severity = "error", caseId) {
|
|
11181
13413
|
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
11182
13414
|
}
|
|
11183
13415
|
function buildCaseRules(suiteCase, config) {
|
|
@@ -11241,7 +13473,7 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
11241
13473
|
const match = outcomes.find((outcome) => outcome.name === name);
|
|
11242
13474
|
if (!match) {
|
|
11243
13475
|
diagnostics.push(
|
|
11244
|
-
|
|
13476
|
+
diagnostic5(
|
|
11245
13477
|
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
11246
13478
|
`Expected observation "${name}" was not recorded.`,
|
|
11247
13479
|
"error",
|
|
@@ -11252,7 +13484,7 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
11252
13484
|
}
|
|
11253
13485
|
if (match.status !== "passed") {
|
|
11254
13486
|
diagnostics.push(
|
|
11255
|
-
|
|
13487
|
+
diagnostic5(
|
|
11256
13488
|
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
11257
13489
|
`Observation "${name}" has status "${match.status}", expected "passed".`,
|
|
11258
13490
|
"error",
|
|
@@ -11272,7 +13504,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11272
13504
|
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
11273
13505
|
message: resolved.reason,
|
|
11274
13506
|
diagnostics: [
|
|
11275
|
-
|
|
13507
|
+
diagnostic5(
|
|
11276
13508
|
"AI_SUITE_CASE_TRACE_MISSING",
|
|
11277
13509
|
resolved.reason ?? "Trace not found.",
|
|
11278
13510
|
"warning",
|
|
@@ -11293,7 +13525,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11293
13525
|
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
11294
13526
|
message,
|
|
11295
13527
|
diagnostics: [
|
|
11296
|
-
|
|
13528
|
+
diagnostic5("AI_SUITE_TRACE_UNREADABLE", message, "error", suiteCase.id)
|
|
11297
13529
|
]
|
|
11298
13530
|
};
|
|
11299
13531
|
}
|
|
@@ -11307,7 +13539,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11307
13539
|
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
11308
13540
|
const diagnostics = [
|
|
11309
13541
|
...checkResult.diagnostics.map(
|
|
11310
|
-
(item) =>
|
|
13542
|
+
(item) => diagnostic5(
|
|
11311
13543
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
11312
13544
|
item.message,
|
|
11313
13545
|
item.severity,
|
|
@@ -11315,7 +13547,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11315
13547
|
)
|
|
11316
13548
|
),
|
|
11317
13549
|
...checkResult.findings.filter((finding) => finding.status === "fail").map(
|
|
11318
|
-
(finding) =>
|
|
13550
|
+
(finding) => diagnostic5(
|
|
11319
13551
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
11320
13552
|
finding.message,
|
|
11321
13553
|
finding.severity,
|
|
@@ -12500,6 +14732,6 @@ function renderGateReport(result, options = {}) {
|
|
|
12500
14732
|
return renderGateSummaryMarkdown(result);
|
|
12501
14733
|
}
|
|
12502
14734
|
|
|
12503
|
-
export { ATTRIBUTION_CONFIDENCES, COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, Redactor, TraceDirectory, TraceReadError, TreeBuilder, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, assertEvidenceRelativePath, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceHtmlShell, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectTraceSchemaVersions, compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, diffRuns, diffTraceEvents, enrichSessionRunRecord, escapeHtml, escapeMarkdown, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatStepLabel, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, inferEvidenceFileRole, isAgentInspectTrace, isAttributionConfidence, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, manualTraceEventsToComparableRun, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToTraceEvents, projectLogicalEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunDiff, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, serializeEvidenceManifest, sha256Hex, stableJson, summarizeObservedOutcomes, summarizeSemanticParity, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, verifyEvidenceDirectory, zeroKinds };
|
|
12504
|
-
//# sourceMappingURL=chunk-
|
|
12505
|
-
//# sourceMappingURL=chunk-
|
|
14735
|
+
export { ATTRIBUTION_CONFIDENCES, COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, Redactor, TraceDirectory, TraceReadError, TreeBuilder, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, assertEvidenceRelativePath, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceHtmlShell, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectTraceSchemaVersions, compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, defineTraceContract, diffRuns, diffTraceEvents, enrichSessionRunRecord, escapeHtml, escapeMarkdown, evaluateTraceContract, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatStepLabel, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, inferEvidenceFileRole, isAgentInspectTrace, isAttributionConfidence, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, manualTraceEventsToComparableRun, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToTraceEvents, projectLogicalEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunDiff, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, serializeEvidenceManifest, sha256Hex, stableJson, summarizeObservedOutcomes, summarizeSemanticParity, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, verifyEvidenceDirectory, zeroKinds };
|
|
14736
|
+
//# sourceMappingURL=chunk-BDRJENLM.mjs.map
|
|
14737
|
+
//# sourceMappingURL=chunk-BDRJENLM.mjs.map
|