agent-inspect 6.29.6 → 6.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +1 -1
- package/docs/CLI.md +26 -4
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-DBRMI7TC.mjs → chunk-BDRJENLM.mjs} +2279 -79
- package/packages/cli/dist/chunk-BDRJENLM.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +3067 -145
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +791 -59
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-6CET2GW2.mjs → src-ZUGAUXY6.mjs} +3 -3
- package/packages/cli/dist/{src-6CET2GW2.mjs.map → src-ZUGAUXY6.mjs.map} +1 -1
- package/packages/cli/dist/chunk-DBRMI7TC.mjs.map +0 -1
|
@@ -3270,7 +3270,7 @@ function deriveLastError(runs) {
|
|
|
3270
3270
|
}
|
|
3271
3271
|
function deriveCheckSummary(runs) {
|
|
3272
3272
|
let pass = 0;
|
|
3273
|
-
let
|
|
3273
|
+
let fail4 = 0;
|
|
3274
3274
|
let warn2 = 0;
|
|
3275
3275
|
let found = false;
|
|
3276
3276
|
for (const run of runs) {
|
|
@@ -3282,7 +3282,7 @@ function deriveCheckSummary(runs) {
|
|
|
3282
3282
|
found = true;
|
|
3283
3283
|
}
|
|
3284
3284
|
if (typeof record.fail === "number") {
|
|
3285
|
-
|
|
3285
|
+
fail4 += record.fail;
|
|
3286
3286
|
found = true;
|
|
3287
3287
|
}
|
|
3288
3288
|
if (typeof record.warn === "number") {
|
|
@@ -3290,7 +3290,7 @@ function deriveCheckSummary(runs) {
|
|
|
3290
3290
|
found = true;
|
|
3291
3291
|
}
|
|
3292
3292
|
}
|
|
3293
|
-
return found ? { pass, fail, warn: warn2 } : void 0;
|
|
3293
|
+
return found ? { pass, fail: fail4, warn: warn2 } : void 0;
|
|
3294
3294
|
}
|
|
3295
3295
|
function deriveObservationSummary(runs) {
|
|
3296
3296
|
for (const run of [...runs].sort((a, b) => activityMs(b) - activityMs(a))) {
|
|
@@ -5004,7 +5004,1359 @@ function isCredentialSensitiveKey(key, sensitiveKeys = DEFAULT_CREDENTIAL_SENSIT
|
|
|
5004
5004
|
return false;
|
|
5005
5005
|
}
|
|
5006
5006
|
|
|
5007
|
+
// packages/core/src/checks/contract-scope.ts
|
|
5008
|
+
function diagnostic(code, message, ruleId) {
|
|
5009
|
+
return {
|
|
5010
|
+
code,
|
|
5011
|
+
message,
|
|
5012
|
+
severity: "error",
|
|
5013
|
+
...ruleId !== void 0 ? { ruleId } : {}
|
|
5014
|
+
};
|
|
5015
|
+
}
|
|
5016
|
+
function isNonEmptyString4(value) {
|
|
5017
|
+
return typeof value === "string" && value.trim() !== "";
|
|
5018
|
+
}
|
|
5019
|
+
function scopeEntries(scope) {
|
|
5020
|
+
const out = [];
|
|
5021
|
+
if (isNonEmptyString4(scope.runId)) out.push(["runId", scope.runId.trim()]);
|
|
5022
|
+
if (isNonEmptyString4(scope.subAgentId)) out.push(["subAgentId", scope.subAgentId.trim()]);
|
|
5023
|
+
if (isNonEmptyString4(scope.groupId)) out.push(["groupId", scope.groupId.trim()]);
|
|
5024
|
+
if (isNonEmptyString4(scope.workflowStep)) {
|
|
5025
|
+
out.push(["workflowStep", scope.workflowStep.trim()]);
|
|
5026
|
+
}
|
|
5027
|
+
if (isNonEmptyString4(scope.rootEventId)) {
|
|
5028
|
+
out.push(["rootEventId", scope.rootEventId.trim()]);
|
|
5029
|
+
}
|
|
5030
|
+
return out;
|
|
5031
|
+
}
|
|
5032
|
+
function mergeWorkflowMeta(into, from) {
|
|
5033
|
+
if (!from) return;
|
|
5034
|
+
for (const [key, value] of Object.entries(from)) {
|
|
5035
|
+
if (value === void 0) continue;
|
|
5036
|
+
if (into[key] === void 0) {
|
|
5037
|
+
into[key] = value;
|
|
5038
|
+
}
|
|
5039
|
+
}
|
|
5040
|
+
}
|
|
5041
|
+
function workflowMetadataForRun(events, runId) {
|
|
5042
|
+
const preferred = {};
|
|
5043
|
+
const fallback = {};
|
|
5044
|
+
for (const event of events) {
|
|
5045
|
+
if (event.runId !== runId) continue;
|
|
5046
|
+
const attrs = event.attributes;
|
|
5047
|
+
if (!attrs || typeof attrs !== "object") continue;
|
|
5048
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
5049
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
5050
|
+
const bag = {};
|
|
5051
|
+
mergeWorkflowMeta(bag, direct);
|
|
5052
|
+
mergeWorkflowMeta(bag, nested);
|
|
5053
|
+
if (event.kind === "RUN") {
|
|
5054
|
+
mergeWorkflowMeta(preferred, bag);
|
|
5055
|
+
} else {
|
|
5056
|
+
mergeWorkflowMeta(fallback, bag);
|
|
5057
|
+
}
|
|
5058
|
+
}
|
|
5059
|
+
mergeWorkflowMeta(preferred, fallback);
|
|
5060
|
+
return preferred;
|
|
5061
|
+
}
|
|
5062
|
+
function flattenNodes(nodes) {
|
|
5063
|
+
return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
|
|
5064
|
+
}
|
|
5065
|
+
function collectSubtreeEventIds(run, rootEventId) {
|
|
5066
|
+
const nodes = flattenNodes(run.children);
|
|
5067
|
+
const byId = new Map(nodes.map((node) => [node.event.eventId, node]));
|
|
5068
|
+
const root = byId.get(rootEventId);
|
|
5069
|
+
if (!root) return void 0;
|
|
5070
|
+
const ids = /* @__PURE__ */ new Set();
|
|
5071
|
+
const visit = (node) => {
|
|
5072
|
+
ids.add(node.event.eventId);
|
|
5073
|
+
for (const child of node.children) visit(child);
|
|
5074
|
+
};
|
|
5075
|
+
visit(root);
|
|
5076
|
+
return ids;
|
|
5077
|
+
}
|
|
5078
|
+
function filterTree(nodes, keep) {
|
|
5079
|
+
const out = [];
|
|
5080
|
+
for (const node of nodes) {
|
|
5081
|
+
if (!keep.has(node.event.eventId)) continue;
|
|
5082
|
+
out.push({
|
|
5083
|
+
...node,
|
|
5084
|
+
children: filterTree(node.children, keep)
|
|
5085
|
+
});
|
|
5086
|
+
}
|
|
5087
|
+
return out;
|
|
5088
|
+
}
|
|
5089
|
+
function projectInput(input, run, eventIds) {
|
|
5090
|
+
const runEvents = input.read.events.filter((event) => event.runId === run.runId);
|
|
5091
|
+
const projectedEvents = eventIds === void 0 ? runEvents : runEvents.filter((event) => eventIds.has(event.eventId) || event.kind === "RUN");
|
|
5092
|
+
const projectedChildren = eventIds === void 0 ? run.children : filterTree(run.children, eventIds);
|
|
5093
|
+
const projectedRun = {
|
|
5094
|
+
...run,
|
|
5095
|
+
children: projectedChildren,
|
|
5096
|
+
metadata: {
|
|
5097
|
+
...run.metadata,
|
|
5098
|
+
totalEvents: flattenNodes(projectedChildren).length
|
|
5099
|
+
}
|
|
5100
|
+
};
|
|
5101
|
+
return {
|
|
5102
|
+
evidenceEventCount: projectedEvents.length,
|
|
5103
|
+
input: {
|
|
5104
|
+
...input,
|
|
5105
|
+
selectedRun: projectedRun,
|
|
5106
|
+
read: {
|
|
5107
|
+
...input.read,
|
|
5108
|
+
runs: [projectedRun],
|
|
5109
|
+
events: projectedEvents
|
|
5110
|
+
}
|
|
5111
|
+
}
|
|
5112
|
+
};
|
|
5113
|
+
}
|
|
5114
|
+
function resolveTraceContractScope(input, scope) {
|
|
5115
|
+
if (scope === void 0) {
|
|
5116
|
+
return { diagnostics: [] };
|
|
5117
|
+
}
|
|
5118
|
+
const entries = scopeEntries(scope);
|
|
5119
|
+
if (entries.length === 0) {
|
|
5120
|
+
return {
|
|
5121
|
+
diagnostics: [
|
|
5122
|
+
diagnostic(
|
|
5123
|
+
"AI_CHECK_INVALID_CONFIG",
|
|
5124
|
+
"TraceContract scope requires at least one explicit selector (runId, subAgentId, groupId, workflowStep, or rootEventId).",
|
|
5125
|
+
"contract.scope.empty"
|
|
5126
|
+
)
|
|
5127
|
+
]
|
|
5128
|
+
};
|
|
5129
|
+
}
|
|
5130
|
+
const matchedSelectors = Object.fromEntries(entries);
|
|
5131
|
+
const actorKeys = entries.filter(([key]) => key !== "rootEventId");
|
|
5132
|
+
const rootEventId = matchedSelectors.rootEventId;
|
|
5133
|
+
let candidates = [...input.read.runs];
|
|
5134
|
+
if (matchedSelectors.runId) {
|
|
5135
|
+
candidates = candidates.filter((run2) => run2.runId === matchedSelectors.runId);
|
|
5136
|
+
}
|
|
5137
|
+
for (const [key, value] of actorKeys) {
|
|
5138
|
+
if (key === "runId") continue;
|
|
5139
|
+
candidates = candidates.filter((run2) => {
|
|
5140
|
+
const meta = workflowMetadataForRun(input.read.events, run2.runId);
|
|
5141
|
+
const actual = meta[key];
|
|
5142
|
+
return actual === value;
|
|
5143
|
+
});
|
|
5144
|
+
}
|
|
5145
|
+
if (rootEventId && actorKeys.length === 0) {
|
|
5146
|
+
candidates = candidates.filter(
|
|
5147
|
+
(run2) => flattenNodes(run2.children).some((node) => node.event.eventId === rootEventId)
|
|
5148
|
+
);
|
|
5149
|
+
}
|
|
5150
|
+
if (candidates.length === 0) {
|
|
5151
|
+
return {
|
|
5152
|
+
diagnostics: [
|
|
5153
|
+
diagnostic(
|
|
5154
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
5155
|
+
`TraceContract scope matched zero actors for ${JSON.stringify(matchedSelectors)}. Missing actor metadata does not fall back to the whole session.`,
|
|
5156
|
+
"contract.scope.zero-match"
|
|
5157
|
+
)
|
|
5158
|
+
]
|
|
5159
|
+
};
|
|
5160
|
+
}
|
|
5161
|
+
if (candidates.length > 1) {
|
|
5162
|
+
return {
|
|
5163
|
+
diagnostics: [
|
|
5164
|
+
diagnostic(
|
|
5165
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
5166
|
+
`TraceContract scope matched ${candidates.length} actors for ${JSON.stringify(matchedSelectors)} (runIds: ${candidates.map((run2) => run2.runId).sort().join(", ")}). Singular actor selectors must resolve to exactly one run.`,
|
|
5167
|
+
"contract.scope.ambiguous"
|
|
5168
|
+
)
|
|
5169
|
+
]
|
|
5170
|
+
};
|
|
5171
|
+
}
|
|
5172
|
+
const run = candidates[0];
|
|
5173
|
+
let eventIds;
|
|
5174
|
+
if (rootEventId) {
|
|
5175
|
+
eventIds = collectSubtreeEventIds(run, rootEventId);
|
|
5176
|
+
if (eventIds === void 0) {
|
|
5177
|
+
return {
|
|
5178
|
+
diagnostics: [
|
|
5179
|
+
diagnostic(
|
|
5180
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
5181
|
+
`TraceContract scope rootEventId ${rootEventId} was not found in run ${run.runId}.`,
|
|
5182
|
+
"contract.scope.root-missing"
|
|
5183
|
+
)
|
|
5184
|
+
]
|
|
5185
|
+
};
|
|
5186
|
+
}
|
|
5187
|
+
}
|
|
5188
|
+
const projected = projectInput(input, run, eventIds);
|
|
5189
|
+
return {
|
|
5190
|
+
resolved: {
|
|
5191
|
+
runId: run.runId,
|
|
5192
|
+
...rootEventId !== void 0 ? { rootEventId } : {},
|
|
5193
|
+
matchedSelectors,
|
|
5194
|
+
evidenceEventCount: projected.evidenceEventCount,
|
|
5195
|
+
input: projected.input
|
|
5196
|
+
},
|
|
5197
|
+
diagnostics: []
|
|
5198
|
+
};
|
|
5199
|
+
}
|
|
5200
|
+
|
|
5201
|
+
// packages/core/src/checks/control-rules.ts
|
|
5202
|
+
function asStringList(value) {
|
|
5203
|
+
if (!Array.isArray(value)) return void 0;
|
|
5204
|
+
const out = [];
|
|
5205
|
+
for (const item of value) {
|
|
5206
|
+
if (typeof item !== "string" || item.trim() === "") return void 0;
|
|
5207
|
+
out.push(item.trim());
|
|
5208
|
+
}
|
|
5209
|
+
return out;
|
|
5210
|
+
}
|
|
5211
|
+
function attributeList(events, attribute) {
|
|
5212
|
+
if (!attribute) return void 0;
|
|
5213
|
+
for (const event of events) {
|
|
5214
|
+
const attrs = event.attributes;
|
|
5215
|
+
if (!attrs || typeof attrs !== "object") continue;
|
|
5216
|
+
const direct = asStringList(attrs[attribute]);
|
|
5217
|
+
if (direct) return direct;
|
|
5218
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? asStringList(attrs.metadata[attribute]) : void 0;
|
|
5219
|
+
if (nested) return nested;
|
|
5220
|
+
}
|
|
5221
|
+
return void 0;
|
|
5222
|
+
}
|
|
5223
|
+
function fail(ruleId, message, evidence, expected, actual) {
|
|
5224
|
+
return {
|
|
5225
|
+
ruleId,
|
|
5226
|
+
severity: "error",
|
|
5227
|
+
status: "fail",
|
|
5228
|
+
message,
|
|
5229
|
+
...expected !== void 0 ? { expected } : {},
|
|
5230
|
+
...actual !== void 0 ? { actual } : {},
|
|
5231
|
+
evidence: [...evidence]
|
|
5232
|
+
};
|
|
5233
|
+
}
|
|
5234
|
+
function sortedUnique(values) {
|
|
5235
|
+
return [...new Set(values)].sort((a, b) => a.localeCompare(b));
|
|
5236
|
+
}
|
|
5237
|
+
function setEqual(left, right) {
|
|
5238
|
+
const a = sortedUnique(left);
|
|
5239
|
+
const b = sortedUnique(right);
|
|
5240
|
+
return a.length === b.length && a.every((value, index) => value === b[index]);
|
|
5241
|
+
}
|
|
5242
|
+
function evaluateControlRules(events, rules, runEvidence2, observationNames) {
|
|
5243
|
+
const findings = [];
|
|
5244
|
+
const declared = rules.declaredTools ?? attributeList(events, rules.declaredToolsAttribute);
|
|
5245
|
+
const enforced = rules.enforcedTools ?? attributeList(events, rules.enforcedToolsAttribute);
|
|
5246
|
+
if (rules.requireDeclaredMatchesEnforced) {
|
|
5247
|
+
if (declared === void 0 || enforced === void 0) {
|
|
5248
|
+
findings.push(
|
|
5249
|
+
fail(
|
|
5250
|
+
"contract.controls.declared-matches-enforced",
|
|
5251
|
+
"Declared and enforced tool sets could not both be resolved.",
|
|
5252
|
+
runEvidence2,
|
|
5253
|
+
{ declaredDefined: declared !== void 0, enforcedDefined: enforced !== void 0 },
|
|
5254
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
5255
|
+
)
|
|
5256
|
+
);
|
|
5257
|
+
} else if (!setEqual(declared, enforced)) {
|
|
5258
|
+
findings.push(
|
|
5259
|
+
fail(
|
|
5260
|
+
"contract.controls.declared-matches-enforced",
|
|
5261
|
+
"Declared tools differ from enforced tools.",
|
|
5262
|
+
runEvidence2,
|
|
5263
|
+
sortedUnique(declared),
|
|
5264
|
+
sortedUnique(enforced)
|
|
5265
|
+
)
|
|
5266
|
+
);
|
|
5267
|
+
}
|
|
5268
|
+
}
|
|
5269
|
+
const observedTools = sortedUnique(
|
|
5270
|
+
events.filter((event) => event.kind === "TOOL" && event.status !== "running").map((event) => resolveCanonicalToolName(event))
|
|
5271
|
+
);
|
|
5272
|
+
if (rules.requireObservedWithinEnforced) {
|
|
5273
|
+
if (enforced === void 0) {
|
|
5274
|
+
findings.push(
|
|
5275
|
+
fail(
|
|
5276
|
+
"contract.controls.observed-within-enforced",
|
|
5277
|
+
"Enforced tool set unavailable for observed-within-enforced check.",
|
|
5278
|
+
runEvidence2,
|
|
5279
|
+
void 0,
|
|
5280
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
5281
|
+
)
|
|
5282
|
+
);
|
|
5283
|
+
} else {
|
|
5284
|
+
const enforcedSet = new Set(enforced);
|
|
5285
|
+
const outside = observedTools.filter((name) => !enforcedSet.has(name));
|
|
5286
|
+
if (outside.length > 0) {
|
|
5287
|
+
findings.push(
|
|
5288
|
+
fail(
|
|
5289
|
+
"contract.controls.observed-within-enforced",
|
|
5290
|
+
`Observed tools outside enforced allowlist: ${outside.join(", ")}.`,
|
|
5291
|
+
runEvidence2,
|
|
5292
|
+
sortedUnique(enforced),
|
|
5293
|
+
outside
|
|
5294
|
+
)
|
|
5295
|
+
);
|
|
5296
|
+
}
|
|
5297
|
+
}
|
|
5298
|
+
}
|
|
5299
|
+
if (rules.requireObservedWithinDeclared) {
|
|
5300
|
+
if (declared === void 0) {
|
|
5301
|
+
findings.push(
|
|
5302
|
+
fail(
|
|
5303
|
+
"contract.controls.observed-within-declared",
|
|
5304
|
+
"Declared tool set unavailable for observed-within-declared check.",
|
|
5305
|
+
runEvidence2,
|
|
5306
|
+
void 0,
|
|
5307
|
+
{ code: "AI_CHECK_CONTROL_EVIDENCE_UNAVAILABLE" }
|
|
5308
|
+
)
|
|
5309
|
+
);
|
|
5310
|
+
} else {
|
|
5311
|
+
const declaredSet = new Set(declared);
|
|
5312
|
+
const outside = observedTools.filter((name) => !declaredSet.has(name));
|
|
5313
|
+
if (outside.length > 0) {
|
|
5314
|
+
findings.push(
|
|
5315
|
+
fail(
|
|
5316
|
+
"contract.controls.observed-within-declared",
|
|
5317
|
+
`Observed tools outside declared set: ${outside.join(", ")}.`,
|
|
5318
|
+
runEvidence2,
|
|
5319
|
+
sortedUnique(declared),
|
|
5320
|
+
outside
|
|
5321
|
+
)
|
|
5322
|
+
);
|
|
5323
|
+
}
|
|
5324
|
+
}
|
|
5325
|
+
}
|
|
5326
|
+
for (const stage of rules.requiredStages ?? []) {
|
|
5327
|
+
const observation = stage.observation ?? `control.${stage.stage}`;
|
|
5328
|
+
if (!observationNames.has(observation)) {
|
|
5329
|
+
findings.push(
|
|
5330
|
+
fail(
|
|
5331
|
+
`contract.controls.stage.${stage.stage}`,
|
|
5332
|
+
`Required control stage observation missing: ${observation}.`,
|
|
5333
|
+
runEvidence2,
|
|
5334
|
+
observation,
|
|
5335
|
+
[...observationNames]
|
|
5336
|
+
)
|
|
5337
|
+
);
|
|
5338
|
+
}
|
|
5339
|
+
}
|
|
5340
|
+
return findings;
|
|
5341
|
+
}
|
|
5342
|
+
|
|
5343
|
+
// packages/core/src/checks/canonical-equality.ts
|
|
5344
|
+
var DEFAULT_MAX_DEPTH = 8;
|
|
5345
|
+
var DEFAULT_MAX_BYTES = 16384;
|
|
5346
|
+
function sortKeysDeep(input, depth, maxDepth) {
|
|
5347
|
+
if (depth > maxDepth) {
|
|
5348
|
+
throw new Error("AI_CHECK_CANONICAL_DEPTH_EXCEEDED");
|
|
5349
|
+
}
|
|
5350
|
+
if (input === null || typeof input !== "object") return input;
|
|
5351
|
+
if (Array.isArray(input)) {
|
|
5352
|
+
return input.map((item) => sortKeysDeep(item, depth + 1, maxDepth));
|
|
5353
|
+
}
|
|
5354
|
+
const record = input;
|
|
5355
|
+
const out = {};
|
|
5356
|
+
for (const key of Object.keys(record).sort()) {
|
|
5357
|
+
out[key] = sortKeysDeep(record[key], depth + 1, maxDepth);
|
|
5358
|
+
}
|
|
5359
|
+
return out;
|
|
5360
|
+
}
|
|
5361
|
+
function encodeCanonicalStructured(value, limits) {
|
|
5362
|
+
const maxDepth = DEFAULT_MAX_DEPTH;
|
|
5363
|
+
const maxBytes = DEFAULT_MAX_BYTES;
|
|
5364
|
+
try {
|
|
5365
|
+
const sorted = sortKeysDeep(value, 0, maxDepth);
|
|
5366
|
+
const encoded = JSON.stringify(sorted);
|
|
5367
|
+
if (encoded === void 0) {
|
|
5368
|
+
return {
|
|
5369
|
+
ok: false,
|
|
5370
|
+
reason: "Value is not JSON-serializable for canonical comparison.",
|
|
5371
|
+
code: "AI_CHECK_CANONICAL_UNSERIALIZABLE"
|
|
5372
|
+
};
|
|
5373
|
+
}
|
|
5374
|
+
if (Buffer.byteLength(encoded, "utf8") > maxBytes) {
|
|
5375
|
+
return {
|
|
5376
|
+
ok: false,
|
|
5377
|
+
reason: "Canonical structured value exceeds bounded size limit.",
|
|
5378
|
+
code: "AI_CHECK_CANONICAL_SIZE_EXCEEDED"
|
|
5379
|
+
};
|
|
5380
|
+
}
|
|
5381
|
+
return { ok: true, value: encoded };
|
|
5382
|
+
} catch (error) {
|
|
5383
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5384
|
+
if (message === "AI_CHECK_CANONICAL_DEPTH_EXCEEDED") {
|
|
5385
|
+
return {
|
|
5386
|
+
ok: false,
|
|
5387
|
+
reason: "Canonical structured value exceeds bounded depth limit.",
|
|
5388
|
+
code: "AI_CHECK_CANONICAL_DEPTH_EXCEEDED"
|
|
5389
|
+
};
|
|
5390
|
+
}
|
|
5391
|
+
return {
|
|
5392
|
+
ok: false,
|
|
5393
|
+
reason: "Canonical structured encoding failed.",
|
|
5394
|
+
code: "AI_CHECK_CANONICAL_UNSERIALIZABLE"
|
|
5395
|
+
};
|
|
5396
|
+
}
|
|
5397
|
+
}
|
|
5398
|
+
function canonicalStructuredEqual(left, right, limits) {
|
|
5399
|
+
const a = encodeCanonicalStructured(left);
|
|
5400
|
+
if (!a.ok) return { equal: false, reason: a.reason, code: a.code };
|
|
5401
|
+
const b = encodeCanonicalStructured(right);
|
|
5402
|
+
if (!b.ok) return { equal: false, reason: b.reason, code: b.code };
|
|
5403
|
+
if (a.value !== b.value) {
|
|
5404
|
+
return {
|
|
5405
|
+
equal: false,
|
|
5406
|
+
reason: "Canonical structured values differ.",
|
|
5407
|
+
code: "AI_CHECK_CANONICAL_NOT_EQUAL"
|
|
5408
|
+
};
|
|
5409
|
+
}
|
|
5410
|
+
return { equal: true };
|
|
5411
|
+
}
|
|
5412
|
+
|
|
5413
|
+
// packages/core/src/checks/tool-arguments.ts
|
|
5414
|
+
function decodeToken(token) {
|
|
5415
|
+
return token.replace(/~1/g, "/").replace(/~0/g, "~");
|
|
5416
|
+
}
|
|
5417
|
+
function resolveJsonPointer(document, pointer) {
|
|
5418
|
+
if (pointer === "") {
|
|
5419
|
+
return { found: true, value: document };
|
|
5420
|
+
}
|
|
5421
|
+
if (!pointer.startsWith("/")) {
|
|
5422
|
+
return { found: false };
|
|
5423
|
+
}
|
|
5424
|
+
const tokens = pointer.slice(1).split("/").map(decodeToken);
|
|
5425
|
+
let current = document;
|
|
5426
|
+
for (const token of tokens) {
|
|
5427
|
+
if (current === null || current === void 0) {
|
|
5428
|
+
return { found: false };
|
|
5429
|
+
}
|
|
5430
|
+
if (Array.isArray(current)) {
|
|
5431
|
+
if (!/^(0|[1-9][0-9]*)$/.test(token)) {
|
|
5432
|
+
return { found: false };
|
|
5433
|
+
}
|
|
5434
|
+
const index = Number(token);
|
|
5435
|
+
if (index >= current.length) {
|
|
5436
|
+
return { found: false };
|
|
5437
|
+
}
|
|
5438
|
+
current = current[index];
|
|
5439
|
+
continue;
|
|
5440
|
+
}
|
|
5441
|
+
if (typeof current !== "object") {
|
|
5442
|
+
return { found: false };
|
|
5443
|
+
}
|
|
5444
|
+
const record = current;
|
|
5445
|
+
if (!Object.prototype.hasOwnProperty.call(record, token)) {
|
|
5446
|
+
return { found: false };
|
|
5447
|
+
}
|
|
5448
|
+
current = record[token];
|
|
5449
|
+
}
|
|
5450
|
+
return { found: true, value: current };
|
|
5451
|
+
}
|
|
5452
|
+
function jsonType(value) {
|
|
5453
|
+
if (value === null) return "null";
|
|
5454
|
+
if (Array.isArray(value)) return "array";
|
|
5455
|
+
return typeof value;
|
|
5456
|
+
}
|
|
5457
|
+
function valuesEqual(left, right) {
|
|
5458
|
+
return canonicalStructuredEqual(left, right).equal === true;
|
|
5459
|
+
}
|
|
5460
|
+
function isStructuredArgumentEvidence(value) {
|
|
5461
|
+
return value !== void 0 && value !== null && typeof value === "object";
|
|
5462
|
+
}
|
|
5463
|
+
function extractToolArgumentPayload(event) {
|
|
5464
|
+
const attrs = event.attributes ?? {};
|
|
5465
|
+
for (const key of ["arguments", "input", "toolArguments"]) {
|
|
5466
|
+
const candidate = attrs[key];
|
|
5467
|
+
if (isStructuredArgumentEvidence(candidate)) {
|
|
5468
|
+
return { present: true, value: candidate };
|
|
5469
|
+
}
|
|
5470
|
+
}
|
|
5471
|
+
const metadata = attrs.metadata;
|
|
5472
|
+
if (isStructuredArgumentEvidence(metadata) && !Array.isArray(metadata)) {
|
|
5473
|
+
const nested = metadata;
|
|
5474
|
+
for (const key of ["arguments", "input", "toolArguments"]) {
|
|
5475
|
+
const candidate = nested[key];
|
|
5476
|
+
if (isStructuredArgumentEvidence(candidate)) {
|
|
5477
|
+
return { present: true, value: candidate };
|
|
5478
|
+
}
|
|
5479
|
+
}
|
|
5480
|
+
}
|
|
5481
|
+
if (isStructuredArgumentEvidence(event.inputSummary)) {
|
|
5482
|
+
return { present: true, value: event.inputSummary };
|
|
5483
|
+
}
|
|
5484
|
+
return { present: false, value: void 0 };
|
|
5485
|
+
}
|
|
5486
|
+
function evaluateToolArgumentValue(value, check, evidencePresent) {
|
|
5487
|
+
if (!evidencePresent) {
|
|
5488
|
+
return {
|
|
5489
|
+
status: "unavailable",
|
|
5490
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
5491
|
+
message: `Structured argument evidence unavailable for tool ${check.tool} at ${check.path}.`
|
|
5492
|
+
};
|
|
5493
|
+
}
|
|
5494
|
+
const resolved = resolveJsonPointer(value, check.path);
|
|
5495
|
+
if (check.operator === "exists") {
|
|
5496
|
+
return resolved.found ? { status: "pass" } : {
|
|
5497
|
+
status: "fail",
|
|
5498
|
+
code: "AI_CHECK_TOOL_ARGUMENT_MISSING",
|
|
5499
|
+
message: `Expected path ${check.path} to exist on tool ${check.tool}.`
|
|
5500
|
+
};
|
|
5501
|
+
}
|
|
5502
|
+
if (!resolved.found) {
|
|
5503
|
+
return {
|
|
5504
|
+
status: "unavailable",
|
|
5505
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EVIDENCE_UNAVAILABLE",
|
|
5506
|
+
message: `Path ${check.path} missing for tool ${check.tool}.`
|
|
5507
|
+
};
|
|
5508
|
+
}
|
|
5509
|
+
if (check.operator === "type") {
|
|
5510
|
+
const actual = jsonType(resolved.value);
|
|
5511
|
+
if (check.type === void 0) {
|
|
5512
|
+
return {
|
|
5513
|
+
status: "fail",
|
|
5514
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
5515
|
+
message: "type operator requires check.type"
|
|
5516
|
+
};
|
|
5517
|
+
}
|
|
5518
|
+
return actual === check.type ? { status: "pass" } : {
|
|
5519
|
+
status: "fail",
|
|
5520
|
+
code: "AI_CHECK_TOOL_ARGUMENT_TYPE",
|
|
5521
|
+
message: `Expected type ${check.type} at ${check.path} for tool ${check.tool}; found ${actual}.`
|
|
5522
|
+
};
|
|
5523
|
+
}
|
|
5524
|
+
if (check.operator === "equals") {
|
|
5525
|
+
return valuesEqual(resolved.value, check.expected) ? { status: "pass" } : {
|
|
5526
|
+
status: "fail",
|
|
5527
|
+
code: "AI_CHECK_TOOL_ARGUMENT_EQUALS",
|
|
5528
|
+
message: `Value at ${check.path} for tool ${check.tool} did not equal expected (bounded comparison).`
|
|
5529
|
+
};
|
|
5530
|
+
}
|
|
5531
|
+
if (check.operator === "oneOf") {
|
|
5532
|
+
const options = check.oneOf ?? [];
|
|
5533
|
+
if (options.length === 0) {
|
|
5534
|
+
return {
|
|
5535
|
+
status: "fail",
|
|
5536
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
5537
|
+
message: "oneOf operator requires a non-empty oneOf array"
|
|
5538
|
+
};
|
|
5539
|
+
}
|
|
5540
|
+
return options.some((option) => valuesEqual(resolved.value, option)) ? { status: "pass" } : {
|
|
5541
|
+
status: "fail",
|
|
5542
|
+
code: "AI_CHECK_TOOL_ARGUMENT_ONE_OF",
|
|
5543
|
+
message: `Value at ${check.path} for tool ${check.tool} was not in the allowed oneOf set.`
|
|
5544
|
+
};
|
|
5545
|
+
}
|
|
5546
|
+
return {
|
|
5547
|
+
status: "fail",
|
|
5548
|
+
code: "AI_CHECK_TOOL_ARGUMENT_INVALID_CONFIG",
|
|
5549
|
+
message: `Unsupported operator.`
|
|
5550
|
+
};
|
|
5551
|
+
}
|
|
5552
|
+
|
|
5553
|
+
// packages/core/src/checks/retry-safety.ts
|
|
5554
|
+
function fail2(ruleId, message, evidence, expected, actual) {
|
|
5555
|
+
return {
|
|
5556
|
+
ruleId,
|
|
5557
|
+
severity: "error",
|
|
5558
|
+
status: "fail",
|
|
5559
|
+
message,
|
|
5560
|
+
...expected !== void 0 ? { expected } : {},
|
|
5561
|
+
...actual !== void 0 ? { actual } : {},
|
|
5562
|
+
evidence: [...evidence]
|
|
5563
|
+
};
|
|
5564
|
+
}
|
|
5565
|
+
function workflowFor(event) {
|
|
5566
|
+
const attrs = event.attributes;
|
|
5567
|
+
if (!attrs || typeof attrs !== "object") return {};
|
|
5568
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
5569
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
5570
|
+
return { ...nested, ...direct };
|
|
5571
|
+
}
|
|
5572
|
+
function hasIdempotencyEvidence(event) {
|
|
5573
|
+
const workflow = workflowFor(event);
|
|
5574
|
+
if (typeof workflow.idempotencyKey === "string" && workflow.idempotencyKey.trim() !== "") {
|
|
5575
|
+
return true;
|
|
5576
|
+
}
|
|
5577
|
+
const attrs = event.attributes ?? {};
|
|
5578
|
+
if (attrs.noSideEffect === true) return true;
|
|
5579
|
+
if (attrs.sideEffect === false) return true;
|
|
5580
|
+
return false;
|
|
5581
|
+
}
|
|
5582
|
+
function eventEvidence(event) {
|
|
5583
|
+
return {
|
|
5584
|
+
runId: event.runId,
|
|
5585
|
+
eventId: event.eventId,
|
|
5586
|
+
kind: event.kind,
|
|
5587
|
+
name: event.name,
|
|
5588
|
+
status: event.status
|
|
5589
|
+
};
|
|
5590
|
+
}
|
|
5591
|
+
function eventTime(event) {
|
|
5592
|
+
return event.startedAt ?? event.timestamp ?? "";
|
|
5593
|
+
}
|
|
5594
|
+
function sortByTime(events) {
|
|
5595
|
+
return [...events].sort((a, b) => eventTime(a).localeCompare(eventTime(b)));
|
|
5596
|
+
}
|
|
5597
|
+
function attemptNumberOf(event) {
|
|
5598
|
+
const workflow = workflowFor(event);
|
|
5599
|
+
const value = workflow.attemptNumber ?? workflow.attempt;
|
|
5600
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
5601
|
+
}
|
|
5602
|
+
function attemptIdOf(event) {
|
|
5603
|
+
const value = workflowFor(event).attemptId;
|
|
5604
|
+
return typeof value === "string" && value.trim() !== "" ? value : void 0;
|
|
5605
|
+
}
|
|
5606
|
+
function countOperationAttempts(members) {
|
|
5607
|
+
const attemptIds = new Set(
|
|
5608
|
+
members.map((event) => attemptIdOf(event)).filter((value) => value !== void 0)
|
|
5609
|
+
);
|
|
5610
|
+
if (attemptIds.size > 0) {
|
|
5611
|
+
return attemptIds.size;
|
|
5612
|
+
}
|
|
5613
|
+
const numbers = members.map((event) => attemptNumberOf(event)).filter((value) => value !== void 0).sort((a, b) => a - b);
|
|
5614
|
+
if (numbers.length > 0) {
|
|
5615
|
+
const unique = [...new Set(numbers)];
|
|
5616
|
+
const min = unique[0];
|
|
5617
|
+
const max = unique[unique.length - 1];
|
|
5618
|
+
const contiguous = unique.length === max - min + 1 && unique.every((value, index) => value === min + index);
|
|
5619
|
+
if (contiguous) {
|
|
5620
|
+
return unique.length;
|
|
5621
|
+
}
|
|
5622
|
+
return unique.length;
|
|
5623
|
+
}
|
|
5624
|
+
return members.filter((event) => event.kind === "TOOL" || event.kind === "LLM").length;
|
|
5625
|
+
}
|
|
5626
|
+
function classifyRetryAttempt(event, priorOrdered, allMembers) {
|
|
5627
|
+
const workflow = workflowFor(event);
|
|
5628
|
+
const attemptNumber = attemptNumberOf(event);
|
|
5629
|
+
const attemptId = attemptIdOf(event);
|
|
5630
|
+
const retryOf = workflow.retryOf;
|
|
5631
|
+
if (typeof retryOf === "string" && retryOf.trim() !== "") {
|
|
5632
|
+
const target = allMembers.find(
|
|
5633
|
+
(candidate) => candidate.eventId === retryOf || attemptIdOf(candidate) === retryOf || workflowFor(candidate).operationId === retryOf
|
|
5634
|
+
);
|
|
5635
|
+
if (!target) {
|
|
5636
|
+
return { kind: "unknown", reason: "retryOf target missing" };
|
|
5637
|
+
}
|
|
5638
|
+
if (eventTime(target) > eventTime(event)) {
|
|
5639
|
+
return { kind: "unknown", reason: "retryOf target does not precede retry" };
|
|
5640
|
+
}
|
|
5641
|
+
return { kind: "retry" };
|
|
5642
|
+
}
|
|
5643
|
+
if (attemptNumber !== void 0) {
|
|
5644
|
+
if (attemptNumber > 1) {
|
|
5645
|
+
return { kind: "retry" };
|
|
5646
|
+
}
|
|
5647
|
+
if (attemptNumber === 1) {
|
|
5648
|
+
return { kind: "first" };
|
|
5649
|
+
}
|
|
5650
|
+
}
|
|
5651
|
+
if (attemptId !== void 0) {
|
|
5652
|
+
const priorDistinct = priorOrdered.some((candidate) => {
|
|
5653
|
+
const priorId = attemptIdOf(candidate);
|
|
5654
|
+
return priorId !== void 0 && priorId !== attemptId;
|
|
5655
|
+
});
|
|
5656
|
+
if (priorDistinct) {
|
|
5657
|
+
return { kind: "retry" };
|
|
5658
|
+
}
|
|
5659
|
+
if (priorOrdered.length === 0) {
|
|
5660
|
+
return { kind: "first" };
|
|
5661
|
+
}
|
|
5662
|
+
}
|
|
5663
|
+
const priorFinished = priorOrdered.filter(
|
|
5664
|
+
(candidate) => candidate.status === "ok" || candidate.status === "error"
|
|
5665
|
+
);
|
|
5666
|
+
if (priorFinished.length > 0) {
|
|
5667
|
+
return { kind: "retry" };
|
|
5668
|
+
}
|
|
5669
|
+
if (priorOrdered.length === 0) {
|
|
5670
|
+
return { kind: "first" };
|
|
5671
|
+
}
|
|
5672
|
+
return { kind: "unknown", reason: "ambiguous attempt identity" };
|
|
5673
|
+
}
|
|
5674
|
+
function hasDuplicateAttemptIds(members) {
|
|
5675
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5676
|
+
for (const event of members) {
|
|
5677
|
+
const id = attemptIdOf(event);
|
|
5678
|
+
if (!id) continue;
|
|
5679
|
+
if (seen.has(id)) return true;
|
|
5680
|
+
seen.add(id);
|
|
5681
|
+
}
|
|
5682
|
+
return false;
|
|
5683
|
+
}
|
|
5684
|
+
function evaluateRetrySafetyRules(events, rules, _runEvidence) {
|
|
5685
|
+
const findings = [];
|
|
5686
|
+
const byOperation = /* @__PURE__ */ new Map();
|
|
5687
|
+
for (const event of events) {
|
|
5688
|
+
const workflow = workflowFor(event);
|
|
5689
|
+
const operationId = workflow.operationId;
|
|
5690
|
+
if (!operationId) continue;
|
|
5691
|
+
const list = byOperation.get(operationId) ?? [];
|
|
5692
|
+
list.push(event);
|
|
5693
|
+
byOperation.set(operationId, list);
|
|
5694
|
+
}
|
|
5695
|
+
if (rules.maxAttempts !== void 0) {
|
|
5696
|
+
for (const [operationId, members] of byOperation) {
|
|
5697
|
+
if (hasDuplicateAttemptIds(members)) {
|
|
5698
|
+
findings.push(
|
|
5699
|
+
fail2(
|
|
5700
|
+
"contract.retry.duplicate-attempt-id",
|
|
5701
|
+
`Operation ${operationId} has duplicate attemptId values.`,
|
|
5702
|
+
members.slice(0, 4).map(eventEvidence),
|
|
5703
|
+
"unique attemptId",
|
|
5704
|
+
"duplicate"
|
|
5705
|
+
)
|
|
5706
|
+
);
|
|
5707
|
+
}
|
|
5708
|
+
const count = countOperationAttempts(members);
|
|
5709
|
+
if (count > rules.maxAttempts) {
|
|
5710
|
+
findings.push(
|
|
5711
|
+
fail2(
|
|
5712
|
+
"contract.retry.max-attempts",
|
|
5713
|
+
`Operation ${operationId} exceeded maxAttempts ${rules.maxAttempts}.`,
|
|
5714
|
+
members.slice(0, 4).map(eventEvidence),
|
|
5715
|
+
rules.maxAttempts,
|
|
5716
|
+
count
|
|
5717
|
+
)
|
|
5718
|
+
);
|
|
5719
|
+
}
|
|
5720
|
+
}
|
|
5721
|
+
}
|
|
5722
|
+
if (rules.requireTerminalResult) {
|
|
5723
|
+
for (const [operationId, members] of byOperation) {
|
|
5724
|
+
const hasTerminal = members.some((event) => event.status === "ok" || event.status === "error");
|
|
5725
|
+
if (!hasTerminal) {
|
|
5726
|
+
findings.push(
|
|
5727
|
+
fail2(
|
|
5728
|
+
"contract.retry.terminal-result",
|
|
5729
|
+
`Operation ${operationId} has no terminal ok/error result.`,
|
|
5730
|
+
members.slice(0, 4).map(eventEvidence),
|
|
5731
|
+
"ok|error",
|
|
5732
|
+
members.map((event) => event.status)
|
|
5733
|
+
)
|
|
5734
|
+
);
|
|
5735
|
+
}
|
|
5736
|
+
}
|
|
5737
|
+
}
|
|
5738
|
+
if (rules.fallbackOnlyAfterFailure) {
|
|
5739
|
+
for (const event of events) {
|
|
5740
|
+
const workflow = workflowFor(event);
|
|
5741
|
+
const fallbackOf = workflow.fallbackOf;
|
|
5742
|
+
if (!fallbackOf) continue;
|
|
5743
|
+
const prior = byOperation.get(fallbackOf) ?? [];
|
|
5744
|
+
const fallbackStart = eventTime(event);
|
|
5745
|
+
const earlierFailure = prior.some(
|
|
5746
|
+
(candidate) => candidate.status === "error" && eventTime(candidate) !== "" && fallbackStart !== "" && eventTime(candidate) < fallbackStart
|
|
5747
|
+
);
|
|
5748
|
+
if (!earlierFailure) {
|
|
5749
|
+
findings.push(
|
|
5750
|
+
fail2(
|
|
5751
|
+
"contract.retry.fallback-after-failure",
|
|
5752
|
+
`Fallback for ${fallbackOf} appeared without an earlier failure in the related operation.`,
|
|
5753
|
+
[eventEvidence(event)],
|
|
5754
|
+
"earlier error attempt",
|
|
5755
|
+
{ fallbackOf }
|
|
5756
|
+
)
|
|
5757
|
+
);
|
|
5758
|
+
}
|
|
5759
|
+
}
|
|
5760
|
+
}
|
|
5761
|
+
const nonIdempotent = new Set(rules.nonIdempotentTools ?? []);
|
|
5762
|
+
if (nonIdempotent.size > 0 || rules.requireIdempotencyEvidenceForRetry) {
|
|
5763
|
+
const toolEvents = events.filter(
|
|
5764
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
5765
|
+
);
|
|
5766
|
+
const byToolOp = /* @__PURE__ */ new Map();
|
|
5767
|
+
for (const event of toolEvents) {
|
|
5768
|
+
const name = resolveCanonicalToolName(event);
|
|
5769
|
+
const workflow = workflowFor(event);
|
|
5770
|
+
const key = `${workflow.operationId ?? name}::${name}`;
|
|
5771
|
+
const list = byToolOp.get(key) ?? [];
|
|
5772
|
+
list.push(event);
|
|
5773
|
+
byToolOp.set(key, list);
|
|
5774
|
+
}
|
|
5775
|
+
for (const [, members] of byToolOp) {
|
|
5776
|
+
const ordered = sortByTime(members);
|
|
5777
|
+
let sawOk = false;
|
|
5778
|
+
let sawSideEffectOk = false;
|
|
5779
|
+
const prior = [];
|
|
5780
|
+
for (const event of ordered) {
|
|
5781
|
+
const name = resolveCanonicalToolName(event);
|
|
5782
|
+
const classification = classifyRetryAttempt(event, prior, members);
|
|
5783
|
+
if (classification.kind === "unknown" && rules.requireIdempotencyEvidenceForRetry) {
|
|
5784
|
+
findings.push(
|
|
5785
|
+
fail2(
|
|
5786
|
+
"contract.retry.ambiguous-identity",
|
|
5787
|
+
`Ambiguous attempt identity for tool ${name}; strict retry evidence cannot pass.`,
|
|
5788
|
+
[eventEvidence(event)],
|
|
5789
|
+
"explicit attempt identity",
|
|
5790
|
+
classification.reason
|
|
5791
|
+
)
|
|
5792
|
+
);
|
|
5793
|
+
}
|
|
5794
|
+
const isRetry = classification.kind === "retry";
|
|
5795
|
+
if (isRetry && rules.requireIdempotencyEvidenceForRetry && !hasIdempotencyEvidence(event)) {
|
|
5796
|
+
findings.push(
|
|
5797
|
+
fail2(
|
|
5798
|
+
"contract.retry.idempotency-evidence",
|
|
5799
|
+
`Retry of tool ${name} lacks idempotencyKey / noSideEffect evidence.`,
|
|
5800
|
+
[eventEvidence(event)],
|
|
5801
|
+
"idempotencyKey|noSideEffect",
|
|
5802
|
+
{ code: "AI_CHECK_RETRY_EVIDENCE_UNAVAILABLE" }
|
|
5803
|
+
)
|
|
5804
|
+
);
|
|
5805
|
+
}
|
|
5806
|
+
if (sawOk && nonIdempotent.has(name) && sawSideEffectOk && !hasIdempotencyEvidence(event)) {
|
|
5807
|
+
findings.push(
|
|
5808
|
+
fail2(
|
|
5809
|
+
"contract.retry.non-idempotent-side-effect",
|
|
5810
|
+
`Retry of non-idempotent tool ${name} after a confirmed ok side effect.`,
|
|
5811
|
+
[eventEvidence(event)],
|
|
5812
|
+
"no retry after side effect",
|
|
5813
|
+
name
|
|
5814
|
+
)
|
|
5815
|
+
);
|
|
5816
|
+
}
|
|
5817
|
+
if (event.status === "ok") {
|
|
5818
|
+
sawOk = true;
|
|
5819
|
+
if (nonIdempotent.has(name) && !hasIdempotencyEvidence(event)) {
|
|
5820
|
+
sawSideEffectOk = true;
|
|
5821
|
+
}
|
|
5822
|
+
}
|
|
5823
|
+
prior.push(event);
|
|
5824
|
+
}
|
|
5825
|
+
}
|
|
5826
|
+
}
|
|
5827
|
+
if (rules.requireRecoveredFailureVisible) {
|
|
5828
|
+
for (const [operationId, members] of byOperation) {
|
|
5829
|
+
const ordered = sortByTime(members);
|
|
5830
|
+
const toolish = ordered.filter(
|
|
5831
|
+
(event) => (event.kind === "TOOL" || event.kind === "LLM") && (event.status === "ok" || event.status === "error")
|
|
5832
|
+
);
|
|
5833
|
+
if (toolish.length < 2) continue;
|
|
5834
|
+
let sawEarlierError = false;
|
|
5835
|
+
let sawLaterOkAfterError = false;
|
|
5836
|
+
for (const event of toolish) {
|
|
5837
|
+
if (event.status === "error") {
|
|
5838
|
+
sawEarlierError = true;
|
|
5839
|
+
} else if (event.status === "ok" && sawEarlierError) {
|
|
5840
|
+
sawLaterOkAfterError = true;
|
|
5841
|
+
}
|
|
5842
|
+
}
|
|
5843
|
+
const hasOk = toolish.some((event) => event.status === "ok");
|
|
5844
|
+
const attemptish = toolish.some((event) => {
|
|
5845
|
+
const meta = workflowFor(event);
|
|
5846
|
+
return meta.attemptId !== void 0 || meta.attemptNumber !== void 0 || meta.attempt !== void 0 || meta.retryOf !== void 0;
|
|
5847
|
+
}) || toolish.length > 1;
|
|
5848
|
+
if (!attemptish || !hasOk) continue;
|
|
5849
|
+
if (!sawLaterOkAfterError) {
|
|
5850
|
+
findings.push(
|
|
5851
|
+
fail2(
|
|
5852
|
+
"contract.retry.recovered-failure-visible",
|
|
5853
|
+
`Operation ${operationId} does not show an earlier error attempt before a later success in the same chain.`,
|
|
5854
|
+
members.slice(0, 4).map(eventEvidence),
|
|
5855
|
+
"earlier error then later ok",
|
|
5856
|
+
{
|
|
5857
|
+
sawEarlierError,
|
|
5858
|
+
statuses: toolish.map((event) => event.status)
|
|
5859
|
+
}
|
|
5860
|
+
)
|
|
5861
|
+
);
|
|
5862
|
+
}
|
|
5863
|
+
}
|
|
5864
|
+
}
|
|
5865
|
+
return findings;
|
|
5866
|
+
}
|
|
5867
|
+
|
|
5868
|
+
// packages/core/src/checks/recovery-operations.ts
|
|
5869
|
+
function fail3(ruleId, message, evidence, expected, actual) {
|
|
5870
|
+
return {
|
|
5871
|
+
ruleId,
|
|
5872
|
+
severity: "error",
|
|
5873
|
+
status: "fail",
|
|
5874
|
+
message,
|
|
5875
|
+
...expected !== void 0 ? { expected } : {},
|
|
5876
|
+
...actual !== void 0 ? { actual } : {},
|
|
5877
|
+
evidence: [...evidence]
|
|
5878
|
+
};
|
|
5879
|
+
}
|
|
5880
|
+
function workflowFor2(event) {
|
|
5881
|
+
const attrs = event.attributes;
|
|
5882
|
+
if (!attrs || typeof attrs !== "object") return {};
|
|
5883
|
+
const direct = extractSessionWorkflowMetadata(attrs);
|
|
5884
|
+
const nested = attrs.metadata && typeof attrs.metadata === "object" ? extractSessionWorkflowMetadata(attrs.metadata) : void 0;
|
|
5885
|
+
return { ...nested, ...direct };
|
|
5886
|
+
}
|
|
5887
|
+
function eventEvidence2(event) {
|
|
5888
|
+
return {
|
|
5889
|
+
runId: event.runId,
|
|
5890
|
+
eventId: event.eventId,
|
|
5891
|
+
kind: event.kind,
|
|
5892
|
+
name: event.name,
|
|
5893
|
+
status: event.status
|
|
5894
|
+
};
|
|
5895
|
+
}
|
|
5896
|
+
function eventTime2(event) {
|
|
5897
|
+
return event.startedAt ?? event.timestamp ?? "";
|
|
5898
|
+
}
|
|
5899
|
+
function sortByTime2(events) {
|
|
5900
|
+
return [...events].sort((a, b) => eventTime2(a).localeCompare(eventTime2(b)));
|
|
5901
|
+
}
|
|
5902
|
+
function errorCodeOf(event) {
|
|
5903
|
+
if (typeof event.error?.code === "string" && event.error.code.trim() !== "") {
|
|
5904
|
+
return event.error.code.trim();
|
|
5905
|
+
}
|
|
5906
|
+
const attrs = event.attributes ?? {};
|
|
5907
|
+
for (const key of ["errorCode", "code"]) {
|
|
5908
|
+
const value = attrs[key];
|
|
5909
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
5910
|
+
}
|
|
5911
|
+
const nested = attrs.error;
|
|
5912
|
+
if (nested && typeof nested === "object") {
|
|
5913
|
+
const code = nested.code;
|
|
5914
|
+
if (typeof code === "string" && code.trim() !== "") return code.trim();
|
|
5915
|
+
}
|
|
5916
|
+
return void 0;
|
|
5917
|
+
}
|
|
5918
|
+
function argumentDigestOf(event) {
|
|
5919
|
+
const attrs = event.attributes ?? {};
|
|
5920
|
+
for (const key of ["argumentsDigest", "inputDigest", "toolArgumentsDigest"]) {
|
|
5921
|
+
const value = attrs[key];
|
|
5922
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
5923
|
+
}
|
|
5924
|
+
const commitment = attrs.omittedPayloadCommitment;
|
|
5925
|
+
if (commitment && typeof commitment === "object") {
|
|
5926
|
+
const digest = commitment.digest;
|
|
5927
|
+
if (typeof digest === "string" && digest.trim() !== "") return digest.trim();
|
|
5928
|
+
}
|
|
5929
|
+
return void 0;
|
|
5930
|
+
}
|
|
5931
|
+
function argumentFingerprint(event) {
|
|
5932
|
+
const payload = extractToolArgumentPayload(event);
|
|
5933
|
+
if (payload.present) {
|
|
5934
|
+
const encoded = encodeCanonicalStructured(payload.value);
|
|
5935
|
+
if (!encoded.ok) return { kind: "missing" };
|
|
5936
|
+
return { kind: "structured", value: encoded.value };
|
|
5937
|
+
}
|
|
5938
|
+
const digest = argumentDigestOf(event);
|
|
5939
|
+
if (digest) return { kind: "digest", value: digest };
|
|
5940
|
+
return { kind: "missing" };
|
|
5941
|
+
}
|
|
5942
|
+
function sameArguments(left, right) {
|
|
5943
|
+
const a = argumentFingerprint(left);
|
|
5944
|
+
const b = argumentFingerprint(right);
|
|
5945
|
+
if (a.kind === "missing" || b.kind === "missing") {
|
|
5946
|
+
return {
|
|
5947
|
+
ok: false,
|
|
5948
|
+
reason: "Structured or digest argument evidence unavailable for same-arguments check.",
|
|
5949
|
+
code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_UNAVAILABLE"
|
|
5950
|
+
};
|
|
5951
|
+
}
|
|
5952
|
+
if (a.kind !== b.kind) {
|
|
5953
|
+
return {
|
|
5954
|
+
ok: false,
|
|
5955
|
+
reason: "Argument evidence kinds differ across attempts (structured vs digest).",
|
|
5956
|
+
code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_MISMATCH"
|
|
5957
|
+
};
|
|
5958
|
+
}
|
|
5959
|
+
if (a.value !== b.value) {
|
|
5960
|
+
return {
|
|
5961
|
+
ok: false,
|
|
5962
|
+
reason: "Retry arguments do not match prior attempt.",
|
|
5963
|
+
code: "AI_CHECK_RECOVERY_ARGUMENTS_DIFFER"
|
|
5964
|
+
};
|
|
5965
|
+
}
|
|
5966
|
+
return { ok: true };
|
|
5967
|
+
}
|
|
5968
|
+
function isUnevaluableWriteCompletion(event) {
|
|
5969
|
+
if (event.status === "running" || event.status === "unknown") return true;
|
|
5970
|
+
const attrs = event.attributes ?? {};
|
|
5971
|
+
if (attrs.timeout === true) return true;
|
|
5972
|
+
if (attrs.completionState === "unknown" || attrs.completionState === "timeout") return true;
|
|
5973
|
+
const code = errorCodeOf(event)?.toUpperCase();
|
|
5974
|
+
if (code === "TIMEOUT" || code === "UNKNOWN_COMPLETION" || code === "DEADLINE_EXCEEDED") {
|
|
5975
|
+
return true;
|
|
5976
|
+
}
|
|
5977
|
+
return false;
|
|
5978
|
+
}
|
|
5979
|
+
function collectExplicitReferences(event) {
|
|
5980
|
+
const refs = /* @__PURE__ */ new Set();
|
|
5981
|
+
const attrs = event.attributes ?? {};
|
|
5982
|
+
const candidates = [
|
|
5983
|
+
attrs.referencedEventId,
|
|
5984
|
+
attrs.referencedEventIds,
|
|
5985
|
+
attrs.toolResultEventId,
|
|
5986
|
+
attrs.toolResultEventIds,
|
|
5987
|
+
attrs.inputEventIds,
|
|
5988
|
+
attrs.dependsOnEventIds,
|
|
5989
|
+
attrs.evidence,
|
|
5990
|
+
workflowFor2(event).toolCallId
|
|
5991
|
+
];
|
|
5992
|
+
for (const candidate of candidates) {
|
|
5993
|
+
if (typeof candidate === "string" && candidate.trim() !== "") {
|
|
5994
|
+
refs.add(candidate.trim());
|
|
5995
|
+
continue;
|
|
5996
|
+
}
|
|
5997
|
+
if (Array.isArray(candidate)) {
|
|
5998
|
+
for (const item of candidate) {
|
|
5999
|
+
if (typeof item === "string" && item.trim() !== "") refs.add(item.trim());
|
|
6000
|
+
if (item && typeof item === "object") {
|
|
6001
|
+
const id = item.eventId;
|
|
6002
|
+
if (typeof id === "string" && id.trim() !== "") refs.add(id.trim());
|
|
6003
|
+
}
|
|
6004
|
+
}
|
|
6005
|
+
continue;
|
|
6006
|
+
}
|
|
6007
|
+
if (candidate && typeof candidate === "object") {
|
|
6008
|
+
const obj = candidate;
|
|
6009
|
+
if (typeof obj.eventId === "string" && obj.eventId.trim() !== "") {
|
|
6010
|
+
refs.add(obj.eventId.trim());
|
|
6011
|
+
}
|
|
6012
|
+
if (Array.isArray(obj.eventIds)) {
|
|
6013
|
+
for (const id of obj.eventIds) {
|
|
6014
|
+
if (typeof id === "string" && id.trim() !== "") refs.add(id.trim());
|
|
6015
|
+
}
|
|
6016
|
+
}
|
|
6017
|
+
}
|
|
6018
|
+
}
|
|
6019
|
+
return refs;
|
|
6020
|
+
}
|
|
6021
|
+
function llmReferencesTool(llm, toolEvent, requireExplicit) {
|
|
6022
|
+
if (!requireExplicit) {
|
|
6023
|
+
return eventTime2(llm) >= eventTime2(toolEvent);
|
|
6024
|
+
}
|
|
6025
|
+
const refs = collectExplicitReferences(llm);
|
|
6026
|
+
if (refs.has(toolEvent.eventId)) return true;
|
|
6027
|
+
const toolCallId = workflowFor2(toolEvent).toolCallId;
|
|
6028
|
+
if (toolCallId && refs.has(toolCallId)) return true;
|
|
6029
|
+
return false;
|
|
6030
|
+
}
|
|
6031
|
+
function attemptIdOf2(event) {
|
|
6032
|
+
const value = workflowFor2(event).attemptId;
|
|
6033
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
6034
|
+
}
|
|
6035
|
+
function hasDuplicateAttemptIds2(members) {
|
|
6036
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6037
|
+
for (const event of members) {
|
|
6038
|
+
const id = attemptIdOf2(event);
|
|
6039
|
+
if (!id) continue;
|
|
6040
|
+
if (seen.has(id)) return true;
|
|
6041
|
+
seen.add(id);
|
|
6042
|
+
}
|
|
6043
|
+
return false;
|
|
6044
|
+
}
|
|
6045
|
+
function findRetryTarget(members, retryOf) {
|
|
6046
|
+
return members.find(
|
|
6047
|
+
(candidate) => candidate.eventId === retryOf || attemptIdOf2(candidate) === retryOf || workflowFor2(candidate).operationId === retryOf
|
|
6048
|
+
);
|
|
6049
|
+
}
|
|
6050
|
+
function groupToolAttempts(events, toolName2) {
|
|
6051
|
+
const members = events.filter(
|
|
6052
|
+
(event) => event.kind === "TOOL" && resolveCanonicalToolName(event) === toolName2
|
|
6053
|
+
);
|
|
6054
|
+
const parent = /* @__PURE__ */ new Map();
|
|
6055
|
+
const ensure = (id) => {
|
|
6056
|
+
if (!parent.has(id)) parent.set(id, id);
|
|
6057
|
+
return id;
|
|
6058
|
+
};
|
|
6059
|
+
const find = (id) => {
|
|
6060
|
+
ensure(id);
|
|
6061
|
+
let cur = id;
|
|
6062
|
+
while (parent.get(cur) !== cur) {
|
|
6063
|
+
cur = parent.get(cur);
|
|
6064
|
+
}
|
|
6065
|
+
let walk = id;
|
|
6066
|
+
while (parent.get(walk) !== cur) {
|
|
6067
|
+
const next = parent.get(walk);
|
|
6068
|
+
parent.set(walk, cur);
|
|
6069
|
+
walk = next;
|
|
6070
|
+
}
|
|
6071
|
+
return cur;
|
|
6072
|
+
};
|
|
6073
|
+
const union = (a, b) => {
|
|
6074
|
+
const ra = find(a);
|
|
6075
|
+
const rb = find(b);
|
|
6076
|
+
if (ra !== rb) parent.set(ra, rb);
|
|
6077
|
+
};
|
|
6078
|
+
const nodeId = (event) => {
|
|
6079
|
+
const operationId = workflowFor2(event).operationId;
|
|
6080
|
+
if (typeof operationId === "string" && operationId.trim() !== "") {
|
|
6081
|
+
return `op:${operationId.trim()}`;
|
|
6082
|
+
}
|
|
6083
|
+
return `event:${event.eventId}`;
|
|
6084
|
+
};
|
|
6085
|
+
for (const event of members) {
|
|
6086
|
+
ensure(nodeId(event));
|
|
6087
|
+
}
|
|
6088
|
+
for (const event of members) {
|
|
6089
|
+
const retryOf = workflowFor2(event).retryOf;
|
|
6090
|
+
if (typeof retryOf !== "string" || retryOf.trim() === "") continue;
|
|
6091
|
+
const target = findRetryTarget(members, retryOf.trim());
|
|
6092
|
+
if (target) {
|
|
6093
|
+
union(nodeId(event), nodeId(target));
|
|
6094
|
+
}
|
|
6095
|
+
}
|
|
6096
|
+
const groups = /* @__PURE__ */ new Map();
|
|
6097
|
+
for (const event of members) {
|
|
6098
|
+
const key = find(nodeId(event));
|
|
6099
|
+
const list = groups.get(key) ?? [];
|
|
6100
|
+
list.push(event);
|
|
6101
|
+
groups.set(key, list);
|
|
6102
|
+
}
|
|
6103
|
+
return groups;
|
|
6104
|
+
}
|
|
6105
|
+
function latestAttempt(ordered) {
|
|
6106
|
+
if (ordered.length === 0) return void 0;
|
|
6107
|
+
return ordered[ordered.length - 1];
|
|
6108
|
+
}
|
|
6109
|
+
function evaluateRecoveryOperations(events, operations) {
|
|
6110
|
+
const findings = [];
|
|
6111
|
+
const llmEvents = events.filter((event) => event.kind === "LLM");
|
|
6112
|
+
for (const operation of operations) {
|
|
6113
|
+
const sideEffectClass = operation.sideEffectClass ?? "read";
|
|
6114
|
+
const groups = groupToolAttempts(events, operation.tool);
|
|
6115
|
+
if (groups.size === 0) {
|
|
6116
|
+
if (operation.requireTerminalSuccess) {
|
|
6117
|
+
findings.push(
|
|
6118
|
+
fail3(
|
|
6119
|
+
"contract.retry.operations.terminal-success",
|
|
6120
|
+
`Recovery operation for tool ${operation.tool} requires a terminal success but no attempts were observed.`,
|
|
6121
|
+
[],
|
|
6122
|
+
"ok",
|
|
6123
|
+
"missing"
|
|
6124
|
+
)
|
|
6125
|
+
);
|
|
6126
|
+
}
|
|
6127
|
+
continue;
|
|
6128
|
+
}
|
|
6129
|
+
for (const [operationKey, members] of groups) {
|
|
6130
|
+
const ordered = sortByTime2(members);
|
|
6131
|
+
if (hasDuplicateAttemptIds2(members)) {
|
|
6132
|
+
findings.push(
|
|
6133
|
+
fail3(
|
|
6134
|
+
"contract.retry.operations.duplicate-attempt-id",
|
|
6135
|
+
`Tool ${operation.tool} operation ${operationKey} has duplicate attemptId values.`,
|
|
6136
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
6137
|
+
"unique attemptId",
|
|
6138
|
+
"duplicate"
|
|
6139
|
+
)
|
|
6140
|
+
);
|
|
6141
|
+
}
|
|
6142
|
+
if (operation.maxAttempts !== void 0) {
|
|
6143
|
+
const count = countOperationAttempts(members);
|
|
6144
|
+
if (count > operation.maxAttempts) {
|
|
6145
|
+
findings.push(
|
|
6146
|
+
fail3(
|
|
6147
|
+
"contract.retry.operations.max-attempts",
|
|
6148
|
+
`Tool ${operation.tool} operation ${operationKey} exceeded maxAttempts ${operation.maxAttempts}.`,
|
|
6149
|
+
members.slice(0, 4).map(eventEvidence2),
|
|
6150
|
+
operation.maxAttempts,
|
|
6151
|
+
count
|
|
6152
|
+
)
|
|
6153
|
+
);
|
|
6154
|
+
}
|
|
6155
|
+
}
|
|
6156
|
+
if (sideEffectClass === "write") {
|
|
6157
|
+
for (const event of ordered) {
|
|
6158
|
+
if (isUnevaluableWriteCompletion(event)) {
|
|
6159
|
+
findings.push(
|
|
6160
|
+
fail3(
|
|
6161
|
+
"contract.retry.operations.write-completion-unevaluable",
|
|
6162
|
+
`Write tool ${operation.tool} has timeout/unknown/running completion that cannot be verified as an authoritative write result.`,
|
|
6163
|
+
[eventEvidence2(event)],
|
|
6164
|
+
"authoritative write completion observation",
|
|
6165
|
+
{
|
|
6166
|
+
code: "AI_CHECK_RECOVERY_WRITE_COMPLETION_UNAVAILABLE",
|
|
6167
|
+
status: event.status
|
|
6168
|
+
}
|
|
6169
|
+
)
|
|
6170
|
+
);
|
|
6171
|
+
}
|
|
6172
|
+
}
|
|
6173
|
+
}
|
|
6174
|
+
const prior = [];
|
|
6175
|
+
let sawEarlierError = false;
|
|
6176
|
+
let sawLaterOkAfterError = false;
|
|
6177
|
+
let lastOk;
|
|
6178
|
+
let identityUnavailable = false;
|
|
6179
|
+
for (const event of ordered) {
|
|
6180
|
+
const classification = classifyRetryAttempt(event, prior, members);
|
|
6181
|
+
if (classification.kind === "unknown") {
|
|
6182
|
+
identityUnavailable = true;
|
|
6183
|
+
findings.push(
|
|
6184
|
+
fail3(
|
|
6185
|
+
"contract.retry.operations.attempt-identity",
|
|
6186
|
+
`Tool ${operation.tool} operation ${operationKey} has ambiguous attempt identity (${classification.reason}).`,
|
|
6187
|
+
[eventEvidence2(event)],
|
|
6188
|
+
"explicit operationId/attemptId/retryOf chronology",
|
|
6189
|
+
{ code: "AI_CHECK_RECOVERY_ATTEMPT_IDENTITY_UNAVAILABLE", reason: classification.reason }
|
|
6190
|
+
)
|
|
6191
|
+
);
|
|
6192
|
+
}
|
|
6193
|
+
const isRetry = classification.kind === "retry";
|
|
6194
|
+
if (isRetry && operation.requireFailureBeforeRetry) {
|
|
6195
|
+
const earlierFailure = prior.some(
|
|
6196
|
+
(candidate) => candidate.status === "error" && eventTime2(candidate) !== "" && eventTime2(event) !== "" && eventTime2(candidate) < eventTime2(event)
|
|
6197
|
+
);
|
|
6198
|
+
if (!earlierFailure) {
|
|
6199
|
+
findings.push(
|
|
6200
|
+
fail3(
|
|
6201
|
+
"contract.retry.operations.failure-before-retry",
|
|
6202
|
+
`Retry of tool ${operation.tool} appeared without an earlier failure in the same operation.`,
|
|
6203
|
+
[eventEvidence2(event)],
|
|
6204
|
+
"earlier error attempt",
|
|
6205
|
+
{ operationKey }
|
|
6206
|
+
)
|
|
6207
|
+
);
|
|
6208
|
+
}
|
|
6209
|
+
}
|
|
6210
|
+
if (isRetry && operation.retryableErrors?.codes && operation.retryableErrors.codes.length > 0) {
|
|
6211
|
+
const allowed = new Set(operation.retryableErrors.codes);
|
|
6212
|
+
const priorErrors = prior.filter((candidate) => candidate.status === "error");
|
|
6213
|
+
const lastError = priorErrors[priorErrors.length - 1];
|
|
6214
|
+
if (lastError) {
|
|
6215
|
+
const code = errorCodeOf(lastError);
|
|
6216
|
+
if (!code || !allowed.has(code)) {
|
|
6217
|
+
findings.push(
|
|
6218
|
+
fail3(
|
|
6219
|
+
"contract.retry.operations.retryable-error",
|
|
6220
|
+
`Retry of tool ${operation.tool} followed a non-retryable or missing error code.`,
|
|
6221
|
+
[eventEvidence2(event), eventEvidence2(lastError)],
|
|
6222
|
+
[...allowed],
|
|
6223
|
+
code ?? null
|
|
6224
|
+
)
|
|
6225
|
+
);
|
|
6226
|
+
}
|
|
6227
|
+
} else {
|
|
6228
|
+
findings.push(
|
|
6229
|
+
fail3(
|
|
6230
|
+
"contract.retry.operations.retryable-error",
|
|
6231
|
+
`Retry of tool ${operation.tool} has no prior error code evidence.`,
|
|
6232
|
+
[eventEvidence2(event)],
|
|
6233
|
+
[...allowed],
|
|
6234
|
+
null
|
|
6235
|
+
)
|
|
6236
|
+
);
|
|
6237
|
+
}
|
|
6238
|
+
}
|
|
6239
|
+
if (isRetry && (operation.requireSameArguments === true || operation.requireSameArguments === "structured-or-digest")) {
|
|
6240
|
+
const baseline = prior[0] ?? prior[prior.length - 1];
|
|
6241
|
+
if (baseline) {
|
|
6242
|
+
const comparison = sameArguments(baseline, event);
|
|
6243
|
+
if (!comparison.ok) {
|
|
6244
|
+
findings.push(
|
|
6245
|
+
fail3(
|
|
6246
|
+
"contract.retry.operations.same-arguments",
|
|
6247
|
+
comparison.reason,
|
|
6248
|
+
[eventEvidence2(baseline), eventEvidence2(event)],
|
|
6249
|
+
"matching structured args or digests",
|
|
6250
|
+
{ code: comparison.code }
|
|
6251
|
+
)
|
|
6252
|
+
);
|
|
6253
|
+
}
|
|
6254
|
+
} else {
|
|
6255
|
+
findings.push(
|
|
6256
|
+
fail3(
|
|
6257
|
+
"contract.retry.operations.same-arguments",
|
|
6258
|
+
"Structured or digest argument evidence unavailable for same-arguments check.",
|
|
6259
|
+
[eventEvidence2(event)],
|
|
6260
|
+
"matching structured args or digests",
|
|
6261
|
+
{ code: "AI_CHECK_RECOVERY_ARGUMENT_EVIDENCE_UNAVAILABLE" }
|
|
6262
|
+
)
|
|
6263
|
+
);
|
|
6264
|
+
}
|
|
6265
|
+
}
|
|
6266
|
+
if (event.status === "error") {
|
|
6267
|
+
sawEarlierError = true;
|
|
6268
|
+
} else if (event.status === "ok" && sawEarlierError) {
|
|
6269
|
+
sawLaterOkAfterError = true;
|
|
6270
|
+
}
|
|
6271
|
+
if (event.status === "ok") {
|
|
6272
|
+
lastOk = event;
|
|
6273
|
+
}
|
|
6274
|
+
prior.push(event);
|
|
6275
|
+
}
|
|
6276
|
+
const latest = latestAttempt(ordered);
|
|
6277
|
+
if (operation.requireTerminalSuccess) {
|
|
6278
|
+
if (!latest || latest.status !== "ok") {
|
|
6279
|
+
findings.push(
|
|
6280
|
+
fail3(
|
|
6281
|
+
"contract.retry.operations.terminal-success",
|
|
6282
|
+
`Tool ${operation.tool} operation ${operationKey} does not terminate with ok on the latest attempt.`,
|
|
6283
|
+
ordered.slice(0, 4).map(eventEvidence2),
|
|
6284
|
+
"latest attempt ok",
|
|
6285
|
+
latest?.status ?? "missing"
|
|
6286
|
+
)
|
|
6287
|
+
);
|
|
6288
|
+
}
|
|
6289
|
+
}
|
|
6290
|
+
if (operation.requireRecoveredFailureVisible) {
|
|
6291
|
+
const latestOk = latest?.status === "ok";
|
|
6292
|
+
if (ordered.length > 1 && latestOk && !sawLaterOkAfterError) {
|
|
6293
|
+
findings.push(
|
|
6294
|
+
fail3(
|
|
6295
|
+
"contract.retry.operations.recovered-failure-visible",
|
|
6296
|
+
`Tool ${operation.tool} operation ${operationKey} does not show an earlier error before later success.`,
|
|
6297
|
+
ordered.slice(0, 4).map(eventEvidence2),
|
|
6298
|
+
"earlier error then later ok",
|
|
6299
|
+
{ sawEarlierError }
|
|
6300
|
+
)
|
|
6301
|
+
);
|
|
6302
|
+
}
|
|
6303
|
+
}
|
|
6304
|
+
const dependency = operation.successfulResultDependency;
|
|
6305
|
+
if (dependency?.consumerKind === "LLM") {
|
|
6306
|
+
const successForDependency = latest?.status === "ok" ? latest : identityUnavailable ? void 0 : lastOk;
|
|
6307
|
+
if (successForDependency) {
|
|
6308
|
+
if (sideEffectClass === "write" && isUnevaluableWriteCompletion(successForDependency)) {
|
|
6309
|
+
findings.push(
|
|
6310
|
+
fail3(
|
|
6311
|
+
"contract.retry.operations.successful-result-dependency",
|
|
6312
|
+
`Successful ${operation.tool} result dependency is unevaluable because write completion evidence is unavailable.`,
|
|
6313
|
+
[eventEvidence2(successForDependency)],
|
|
6314
|
+
"LLM reference to successful tool event",
|
|
6315
|
+
{ code: "AI_CHECK_RECOVERY_WRITE_COMPLETION_UNAVAILABLE" }
|
|
6316
|
+
)
|
|
6317
|
+
);
|
|
6318
|
+
} else {
|
|
6319
|
+
const requireExplicit = dependency.requireExplicitReference === true;
|
|
6320
|
+
const referenced = llmEvents.some(
|
|
6321
|
+
(llmEvent) => llmReferencesTool(llmEvent, successForDependency, requireExplicit)
|
|
6322
|
+
);
|
|
6323
|
+
if (!referenced) {
|
|
6324
|
+
findings.push(
|
|
6325
|
+
fail3(
|
|
6326
|
+
"contract.retry.operations.successful-result-dependency",
|
|
6327
|
+
requireExplicit ? `Successful ${operation.tool} result is not explicitly referenced by a later LLM event.` : `Successful ${operation.tool} result has no later LLM consumer.`,
|
|
6328
|
+
[eventEvidence2(successForDependency)],
|
|
6329
|
+
"LLM reference to successful tool event",
|
|
6330
|
+
{ code: "AI_CHECK_RECOVERY_RESULT_DEPENDENCY_MISSING" }
|
|
6331
|
+
)
|
|
6332
|
+
);
|
|
6333
|
+
}
|
|
6334
|
+
}
|
|
6335
|
+
}
|
|
6336
|
+
}
|
|
6337
|
+
}
|
|
6338
|
+
}
|
|
6339
|
+
return findings;
|
|
6340
|
+
}
|
|
6341
|
+
|
|
5007
6342
|
// packages/core/src/checks/contract.ts
|
|
6343
|
+
function contractFailFinding(ruleId, message, evidence, expected, actual) {
|
|
6344
|
+
return {
|
|
6345
|
+
ruleId,
|
|
6346
|
+
severity: "error",
|
|
6347
|
+
status: "fail",
|
|
6348
|
+
message,
|
|
6349
|
+
...expected !== void 0 ? { expected } : {},
|
|
6350
|
+
...actual !== void 0 ? { actual } : {},
|
|
6351
|
+
evidence: [...evidence]
|
|
6352
|
+
};
|
|
6353
|
+
}
|
|
6354
|
+
function normalizeStatus(status) {
|
|
6355
|
+
if (status === "ok" || status === "error" || status === "running") return status;
|
|
6356
|
+
if (status === "success") return "ok";
|
|
6357
|
+
if (status === "failed") return "error";
|
|
6358
|
+
return "error";
|
|
6359
|
+
}
|
|
5008
6360
|
function cloneBody(body) {
|
|
5009
6361
|
return {
|
|
5010
6362
|
...body.run ? { run: { ...body.run } } : {},
|
|
@@ -5051,13 +6403,449 @@ function cloneBody(body) {
|
|
|
5051
6403
|
}))
|
|
5052
6404
|
} : {}
|
|
5053
6405
|
}
|
|
5054
|
-
} : {}
|
|
5055
|
-
};
|
|
6406
|
+
} : {}
|
|
6407
|
+
};
|
|
6408
|
+
}
|
|
6409
|
+
function bodyHasRules(body) {
|
|
6410
|
+
return body.run !== void 0 || body.tools !== void 0 || body.llm !== void 0 || body.observations !== void 0 || body.controls !== void 0 || body.retry !== void 0;
|
|
6411
|
+
}
|
|
6412
|
+
var MAX_EVIDENCE_EVENT_IDS = 16;
|
|
6413
|
+
var METHOD_VOCABULARY = new Set(OBSERVED_OUTCOME_METHODS);
|
|
6414
|
+
function parseEvidenceEventIds(evidence) {
|
|
6415
|
+
if (evidence === void 0 || evidence === null) {
|
|
6416
|
+
return { status: "missing" };
|
|
6417
|
+
}
|
|
6418
|
+
if (typeof evidence === "string") {
|
|
6419
|
+
const trimmed = evidence.trim();
|
|
6420
|
+
if (trimmed === "") {
|
|
6421
|
+
return { status: "invalid", reason: "empty evidence string" };
|
|
6422
|
+
}
|
|
6423
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
6424
|
+
}
|
|
6425
|
+
if (typeof evidence !== "object" || Array.isArray(evidence)) {
|
|
6426
|
+
return {
|
|
6427
|
+
status: "invalid",
|
|
6428
|
+
reason: "evidence must be a string event id or { eventId } / { eventIds }"
|
|
6429
|
+
};
|
|
6430
|
+
}
|
|
6431
|
+
const record = evidence;
|
|
6432
|
+
if (typeof record.eventId === "string") {
|
|
6433
|
+
const trimmed = record.eventId.trim();
|
|
6434
|
+
if (trimmed === "") {
|
|
6435
|
+
return { status: "invalid", reason: "empty evidence.eventId" };
|
|
6436
|
+
}
|
|
6437
|
+
return { status: "ok", eventIds: [trimmed] };
|
|
6438
|
+
}
|
|
6439
|
+
if (Array.isArray(record.eventIds)) {
|
|
6440
|
+
if (record.eventIds.length === 0) {
|
|
6441
|
+
return { status: "invalid", reason: "empty evidence.eventIds" };
|
|
6442
|
+
}
|
|
6443
|
+
if (record.eventIds.length > MAX_EVIDENCE_EVENT_IDS) {
|
|
6444
|
+
return {
|
|
6445
|
+
status: "invalid",
|
|
6446
|
+
reason: `evidence.eventIds exceeds bound of ${MAX_EVIDENCE_EVENT_IDS}`
|
|
6447
|
+
};
|
|
6448
|
+
}
|
|
6449
|
+
const eventIds = [];
|
|
6450
|
+
for (const item of record.eventIds) {
|
|
6451
|
+
if (typeof item !== "string" || item.trim() === "") {
|
|
6452
|
+
return { status: "invalid", reason: "evidence.eventIds contains a non-string id" };
|
|
6453
|
+
}
|
|
6454
|
+
eventIds.push(item.trim());
|
|
6455
|
+
}
|
|
6456
|
+
return { status: "ok", eventIds };
|
|
6457
|
+
}
|
|
6458
|
+
return {
|
|
6459
|
+
status: "invalid",
|
|
6460
|
+
reason: "evidence must include eventId or eventIds"
|
|
6461
|
+
};
|
|
6462
|
+
}
|
|
6463
|
+
function provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds) {
|
|
6464
|
+
const findings = [];
|
|
6465
|
+
const evidenceBase = [
|
|
6466
|
+
{
|
|
6467
|
+
runId: outcome.runId,
|
|
6468
|
+
eventId: outcome.outcomeId,
|
|
6469
|
+
kind: "OUTCOME",
|
|
6470
|
+
name: outcome.name,
|
|
6471
|
+
path: `outcome.${outcome.name}`
|
|
6472
|
+
}
|
|
6473
|
+
];
|
|
6474
|
+
if (requireProvenance.method) {
|
|
6475
|
+
if (outcome.method === void 0 || outcome.method.trim() === "") {
|
|
6476
|
+
findings.push(
|
|
6477
|
+
contractFailFinding(
|
|
6478
|
+
"contract.observation.provenance.method",
|
|
6479
|
+
`Observation ${outcome.name} is missing a structural method.`,
|
|
6480
|
+
evidenceBase,
|
|
6481
|
+
{ method: true },
|
|
6482
|
+
{ name: outcome.name, method: outcome.method ?? null }
|
|
6483
|
+
)
|
|
6484
|
+
);
|
|
6485
|
+
} else if (!METHOD_VOCABULARY.has(outcome.method)) {
|
|
6486
|
+
findings.push(
|
|
6487
|
+
contractFailFinding(
|
|
6488
|
+
"contract.observation.provenance.method",
|
|
6489
|
+
`Observation ${outcome.name} method ${outcome.method} is outside the bounded ObservedOutcomeMethod vocabulary.`,
|
|
6490
|
+
evidenceBase,
|
|
6491
|
+
{ methodVocabulary: [...OBSERVED_OUTCOME_METHODS] },
|
|
6492
|
+
{ name: outcome.name, method: outcome.method }
|
|
6493
|
+
)
|
|
6494
|
+
);
|
|
6495
|
+
}
|
|
6496
|
+
}
|
|
6497
|
+
if (requireProvenance.evidence || requireProvenance.sameRunEventReference) {
|
|
6498
|
+
const parsed = parseEvidenceEventIds(outcome.evidence);
|
|
6499
|
+
if (parsed.status === "missing") {
|
|
6500
|
+
findings.push(
|
|
6501
|
+
contractFailFinding(
|
|
6502
|
+
"contract.observation.provenance.evidence",
|
|
6503
|
+
`Observation ${outcome.name} is missing bounded evidence references.`,
|
|
6504
|
+
evidenceBase,
|
|
6505
|
+
{ evidence: true },
|
|
6506
|
+
{ name: outcome.name, evidence: null }
|
|
6507
|
+
)
|
|
6508
|
+
);
|
|
6509
|
+
} else if (parsed.status === "invalid") {
|
|
6510
|
+
findings.push(
|
|
6511
|
+
contractFailFinding(
|
|
6512
|
+
"contract.observation.provenance.evidence",
|
|
6513
|
+
`Observation ${outcome.name} evidence is not a bounded event reference (${parsed.reason}).`,
|
|
6514
|
+
evidenceBase,
|
|
6515
|
+
{ evidence: true },
|
|
6516
|
+
{ name: outcome.name, reason: parsed.reason }
|
|
6517
|
+
)
|
|
6518
|
+
);
|
|
6519
|
+
} else if (requireProvenance.sameRunEventReference) {
|
|
6520
|
+
const missing = parsed.eventIds.filter((eventId) => !knownEventIds.has(eventId));
|
|
6521
|
+
if (missing.length > 0) {
|
|
6522
|
+
findings.push(
|
|
6523
|
+
contractFailFinding(
|
|
6524
|
+
"contract.observation.provenance.same-run",
|
|
6525
|
+
`Observation ${outcome.name} references event ids missing from the same run: ${missing.join(", ")}.`,
|
|
6526
|
+
evidenceBase,
|
|
6527
|
+
{ sameRunEventReference: true, eventIds: parsed.eventIds },
|
|
6528
|
+
{ name: outcome.name, missing }
|
|
6529
|
+
)
|
|
6530
|
+
);
|
|
6531
|
+
}
|
|
6532
|
+
}
|
|
6533
|
+
}
|
|
6534
|
+
return findings;
|
|
5056
6535
|
}
|
|
5057
|
-
function
|
|
5058
|
-
|
|
6536
|
+
function contractToRules(contract) {
|
|
6537
|
+
const rules = [];
|
|
6538
|
+
const allowedStatuses = contract.run?.allowedStatuses ?? [];
|
|
6539
|
+
if (allowedStatuses.length === 1) {
|
|
6540
|
+
rules.push(
|
|
6541
|
+
createRunStatusRule({
|
|
6542
|
+
expected: normalizeStatus(allowedStatuses[0]),
|
|
6543
|
+
allowIncomplete: contract.run?.requireCompleted === false
|
|
6544
|
+
})
|
|
6545
|
+
);
|
|
6546
|
+
} else if (allowedStatuses.length > 1) {
|
|
6547
|
+
const expected = [...new Set(allowedStatuses.map(normalizeStatus))];
|
|
6548
|
+
const allowIncomplete = contract.run?.requireCompleted === false;
|
|
6549
|
+
rules.push({
|
|
6550
|
+
id: "contract.run.allowedStatuses",
|
|
6551
|
+
category: "run",
|
|
6552
|
+
defaultSeverity: "error",
|
|
6553
|
+
evaluate(context) {
|
|
6554
|
+
const findings = [];
|
|
6555
|
+
const actual = context.selectedRun?.status ?? "unknown";
|
|
6556
|
+
if (!expected.includes(actual)) {
|
|
6557
|
+
findings.push(
|
|
6558
|
+
contractFailFinding(
|
|
6559
|
+
"contract.run.allowedStatuses",
|
|
6560
|
+
`Run status ${actual} is not one of the allowed statuses: ${expected.join(", ")}.`,
|
|
6561
|
+
context.selectedRun ? [
|
|
6562
|
+
{
|
|
6563
|
+
runId: context.selectedRun.runId,
|
|
6564
|
+
kind: "RUN",
|
|
6565
|
+
name: context.selectedRun.name,
|
|
6566
|
+
status: context.selectedRun.status
|
|
6567
|
+
}
|
|
6568
|
+
] : [],
|
|
6569
|
+
expected,
|
|
6570
|
+
actual
|
|
6571
|
+
)
|
|
6572
|
+
);
|
|
6573
|
+
}
|
|
6574
|
+
if (!allowIncomplete) {
|
|
6575
|
+
const running = (context.logicalEvents ?? context.events).filter(
|
|
6576
|
+
(event) => event.status === "running"
|
|
6577
|
+
);
|
|
6578
|
+
if (running.length > 0) {
|
|
6579
|
+
findings.push(
|
|
6580
|
+
contractFailFinding(
|
|
6581
|
+
"contract.run.allowedStatuses",
|
|
6582
|
+
"Run contains incomplete running events.",
|
|
6583
|
+
running.map((event) => ({
|
|
6584
|
+
runId: event.runId,
|
|
6585
|
+
eventId: event.eventId,
|
|
6586
|
+
kind: event.kind,
|
|
6587
|
+
name: event.name,
|
|
6588
|
+
status: event.status
|
|
6589
|
+
})),
|
|
6590
|
+
"no running events",
|
|
6591
|
+
running.length
|
|
6592
|
+
)
|
|
6593
|
+
);
|
|
6594
|
+
}
|
|
6595
|
+
}
|
|
6596
|
+
return findings;
|
|
6597
|
+
}
|
|
6598
|
+
});
|
|
6599
|
+
} else if (contract.run?.requireCompleted !== false) {
|
|
6600
|
+
rules.push(createRunStatusRule({ allowIncomplete: false }));
|
|
6601
|
+
}
|
|
6602
|
+
if (contract.run?.maxDurationMs !== void 0) {
|
|
6603
|
+
rules.push(createRunDurationRule({ maxDurationMs: contract.run.maxDurationMs }));
|
|
6604
|
+
}
|
|
6605
|
+
if (contract.run?.requireCompleted === true) {
|
|
6606
|
+
rules.push(createStructureIncompleteRule({ requireEndedAtForStarted: true }));
|
|
6607
|
+
}
|
|
6608
|
+
if (contract.tools) {
|
|
6609
|
+
const order = contract.tools.requiredOrder ?? [];
|
|
6610
|
+
const requiredOrderMode = contract.tools.requiredOrderMode ?? "first-occurrence";
|
|
6611
|
+
const orderRules = contract.tools.orderRules ?? [];
|
|
6612
|
+
const endpointRequired = orderRules.filter((rule) => rule.requireEndpoints !== false).flatMap((rule) => [rule.before, rule.after]);
|
|
6613
|
+
const required = [
|
|
6614
|
+
.../* @__PURE__ */ new Set([
|
|
6615
|
+
...contract.tools.required ?? [],
|
|
6616
|
+
...contract.tools.requiredTools ?? [],
|
|
6617
|
+
...order,
|
|
6618
|
+
...endpointRequired
|
|
6619
|
+
])
|
|
6620
|
+
];
|
|
6621
|
+
const forbidden = [
|
|
6622
|
+
...contract.tools.forbidden ?? [],
|
|
6623
|
+
...contract.tools.forbiddenTools ?? []
|
|
6624
|
+
];
|
|
6625
|
+
rules.push(
|
|
6626
|
+
createToolUsageRule({
|
|
6627
|
+
...required.length > 0 ? { required } : {},
|
|
6628
|
+
...forbidden.length > 0 ? { forbidden } : {},
|
|
6629
|
+
...contract.tools.allowed ? { allowed: contract.tools.allowed } : {},
|
|
6630
|
+
...contract.tools.maxCalls !== void 0 ? { maxCount: contract.tools.maxCalls } : {}
|
|
6631
|
+
})
|
|
6632
|
+
);
|
|
6633
|
+
for (let i = 0; i < order.length - 1; i += 1) {
|
|
6634
|
+
rules.push(
|
|
6635
|
+
createToolOrderingRule({
|
|
6636
|
+
before: order[i],
|
|
6637
|
+
after: order[i + 1],
|
|
6638
|
+
id: `contract.tool.order.${i}`,
|
|
6639
|
+
mode: requiredOrderMode
|
|
6640
|
+
})
|
|
6641
|
+
);
|
|
6642
|
+
}
|
|
6643
|
+
const defaultOccurrenceMode = contract.tools.defaultOccurrenceMode ?? requiredOrderMode;
|
|
6644
|
+
for (const [index, rule] of orderRules.entries()) {
|
|
6645
|
+
rules.push(
|
|
6646
|
+
createToolOrderingRule({
|
|
6647
|
+
before: rule.before,
|
|
6648
|
+
after: rule.after,
|
|
6649
|
+
id: `contract.tool.orderRule.${index}`,
|
|
6650
|
+
mode: rule.occurrenceMode ?? defaultOccurrenceMode
|
|
6651
|
+
})
|
|
6652
|
+
);
|
|
6653
|
+
}
|
|
6654
|
+
const argumentChecks = contract.tools.arguments ?? [];
|
|
6655
|
+
if (argumentChecks.length > 0) {
|
|
6656
|
+
rules.push({
|
|
6657
|
+
id: "contract.tool.arguments",
|
|
6658
|
+
category: "tool",
|
|
6659
|
+
defaultSeverity: "error",
|
|
6660
|
+
evaluate(context) {
|
|
6661
|
+
const tools = (context.logicalEvents ?? context.events).filter(
|
|
6662
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
6663
|
+
);
|
|
6664
|
+
const findings = [];
|
|
6665
|
+
for (const [index, check] of argumentChecks.entries()) {
|
|
6666
|
+
const matches = tools.filter(
|
|
6667
|
+
(event) => resolveCanonicalToolName(event) === check.tool
|
|
6668
|
+
);
|
|
6669
|
+
if (matches.length === 0) {
|
|
6670
|
+
findings.push(
|
|
6671
|
+
contractFailFinding(
|
|
6672
|
+
`contract.tool.arguments.${index}`,
|
|
6673
|
+
`No finished tool named ${check.tool} for argument check.`,
|
|
6674
|
+
context.selectedRun ? [
|
|
6675
|
+
{
|
|
6676
|
+
runId: context.selectedRun.runId,
|
|
6677
|
+
kind: "RUN",
|
|
6678
|
+
name: context.selectedRun.name
|
|
6679
|
+
}
|
|
6680
|
+
] : [],
|
|
6681
|
+
{ tool: check.tool, path: check.path },
|
|
6682
|
+
{ toolCount: 0 }
|
|
6683
|
+
)
|
|
6684
|
+
);
|
|
6685
|
+
continue;
|
|
6686
|
+
}
|
|
6687
|
+
const occurrence = check.occurrence ?? "all";
|
|
6688
|
+
const selected = occurrence === "first" ? [matches[0]] : occurrence === "last" ? [matches[matches.length - 1]] : matches;
|
|
6689
|
+
const results = selected.map((event) => {
|
|
6690
|
+
const payload = extractToolArgumentPayload(event);
|
|
6691
|
+
return evaluateToolArgumentValue(
|
|
6692
|
+
payload.value,
|
|
6693
|
+
check,
|
|
6694
|
+
payload.present
|
|
6695
|
+
);
|
|
6696
|
+
});
|
|
6697
|
+
if (occurrence === "any") {
|
|
6698
|
+
if (results.some((result) => result.status === "pass")) continue;
|
|
6699
|
+
const firstFail = results.find((result) => result.status !== "pass");
|
|
6700
|
+
findings.push(
|
|
6701
|
+
contractFailFinding(
|
|
6702
|
+
`contract.tool.arguments.${index}`,
|
|
6703
|
+
firstFail.message,
|
|
6704
|
+
selected.slice(0, 1).map((event) => ({
|
|
6705
|
+
runId: event.runId,
|
|
6706
|
+
eventId: event.eventId,
|
|
6707
|
+
kind: event.kind,
|
|
6708
|
+
name: event.name,
|
|
6709
|
+
path: `tool.${check.tool}${check.path}`
|
|
6710
|
+
})),
|
|
6711
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
6712
|
+
{ code: firstFail.code }
|
|
6713
|
+
)
|
|
6714
|
+
);
|
|
6715
|
+
continue;
|
|
6716
|
+
}
|
|
6717
|
+
for (const [selIndex, result] of results.entries()) {
|
|
6718
|
+
if (result.status === "pass") continue;
|
|
6719
|
+
const event = selected[selIndex];
|
|
6720
|
+
findings.push(
|
|
6721
|
+
contractFailFinding(
|
|
6722
|
+
`contract.tool.arguments.${index}`,
|
|
6723
|
+
result.message,
|
|
6724
|
+
[
|
|
6725
|
+
{
|
|
6726
|
+
runId: event.runId,
|
|
6727
|
+
eventId: event.eventId,
|
|
6728
|
+
kind: event.kind,
|
|
6729
|
+
name: event.name,
|
|
6730
|
+
path: `tool.${check.tool}${check.path}`
|
|
6731
|
+
}
|
|
6732
|
+
],
|
|
6733
|
+
{ tool: check.tool, path: check.path, operator: check.operator },
|
|
6734
|
+
{ code: result.code }
|
|
6735
|
+
)
|
|
6736
|
+
);
|
|
6737
|
+
}
|
|
6738
|
+
}
|
|
6739
|
+
return findings;
|
|
6740
|
+
}
|
|
6741
|
+
});
|
|
6742
|
+
}
|
|
6743
|
+
}
|
|
6744
|
+
if (contract.llm) {
|
|
6745
|
+
rules.push(
|
|
6746
|
+
createLlmUsageRule({
|
|
6747
|
+
...contract.llm.maxCalls !== void 0 ? { maxCalls: contract.llm.maxCalls } : {},
|
|
6748
|
+
...contract.llm.maxTotalTokens !== void 0 ? { maxTotalTokens: contract.llm.maxTotalTokens } : {},
|
|
6749
|
+
...contract.llm.allowedModels ? { allowedModels: contract.llm.allowedModels } : {}
|
|
6750
|
+
})
|
|
6751
|
+
);
|
|
6752
|
+
}
|
|
6753
|
+
if (contract.observations) {
|
|
6754
|
+
const required = contract.observations.required ?? [];
|
|
6755
|
+
if (required.length > 0) {
|
|
6756
|
+
rules.push({
|
|
6757
|
+
id: "contract.observation.required",
|
|
6758
|
+
category: "run",
|
|
6759
|
+
defaultSeverity: "error",
|
|
6760
|
+
evaluate(context) {
|
|
6761
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
6762
|
+
const names = new Set(outcomes.map((item) => item.name));
|
|
6763
|
+
const missing = required.filter((name) => !names.has(name));
|
|
6764
|
+
if (missing.length === 0) return [];
|
|
6765
|
+
return [
|
|
6766
|
+
contractFailFinding(
|
|
6767
|
+
"contract.observation.required",
|
|
6768
|
+
`Required observations missing: ${missing.join(", ")}`,
|
|
6769
|
+
context.selectedRun ? [{ runId: context.selectedRun.runId, kind: "RUN", name: context.selectedRun.name }] : [],
|
|
6770
|
+
required,
|
|
6771
|
+
[...names]
|
|
6772
|
+
)
|
|
6773
|
+
];
|
|
6774
|
+
}
|
|
6775
|
+
});
|
|
6776
|
+
}
|
|
6777
|
+
if (contract.observations.failOn?.length) {
|
|
6778
|
+
rules.push(
|
|
6779
|
+
createObservedOutcomeRule({
|
|
6780
|
+
failOn: contract.observations.failOn.filter(
|
|
6781
|
+
(status) => status === "failed" || status === "unknown" || status === "skipped"
|
|
6782
|
+
)
|
|
6783
|
+
})
|
|
6784
|
+
);
|
|
6785
|
+
}
|
|
6786
|
+
const requireProvenance = contract.observations.requireProvenance;
|
|
6787
|
+
if (requireProvenance && (requireProvenance.method || requireProvenance.evidence || requireProvenance.sameRunEventReference)) {
|
|
6788
|
+
rules.push({
|
|
6789
|
+
id: "contract.observation.provenance",
|
|
6790
|
+
category: "run",
|
|
6791
|
+
defaultSeverity: "error",
|
|
6792
|
+
evaluate(context) {
|
|
6793
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
6794
|
+
const knownEventIds = new Set(context.events.map((event) => event.eventId));
|
|
6795
|
+
const targets = required.length > 0 ? outcomes.filter((outcome) => required.includes(outcome.name)) : outcomes;
|
|
6796
|
+
return targets.flatMap(
|
|
6797
|
+
(outcome) => provenanceFindingsForOutcome(outcome, requireProvenance, knownEventIds)
|
|
6798
|
+
);
|
|
6799
|
+
}
|
|
6800
|
+
});
|
|
6801
|
+
}
|
|
6802
|
+
}
|
|
6803
|
+
if (contract.controls) {
|
|
6804
|
+
const controls = contract.controls;
|
|
6805
|
+
rules.push({
|
|
6806
|
+
id: "contract.controls",
|
|
6807
|
+
category: "run",
|
|
6808
|
+
defaultSeverity: "error",
|
|
6809
|
+
evaluate(context) {
|
|
6810
|
+
const events = context.logicalEvents ?? context.events;
|
|
6811
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
6812
|
+
const observationNames = new Set(outcomes.map((item) => item.name));
|
|
6813
|
+
const runEvidence2 = context.selectedRun ? [
|
|
6814
|
+
{
|
|
6815
|
+
runId: context.selectedRun.runId,
|
|
6816
|
+
kind: "RUN",
|
|
6817
|
+
name: context.selectedRun.name
|
|
6818
|
+
}
|
|
6819
|
+
] : [];
|
|
6820
|
+
return evaluateControlRules(events, controls, runEvidence2, observationNames);
|
|
6821
|
+
}
|
|
6822
|
+
});
|
|
6823
|
+
}
|
|
6824
|
+
if (contract.retry) {
|
|
6825
|
+
const retry = contract.retry;
|
|
6826
|
+
rules.push({
|
|
6827
|
+
id: "contract.retry",
|
|
6828
|
+
category: "run",
|
|
6829
|
+
defaultSeverity: "error",
|
|
6830
|
+
evaluate(context) {
|
|
6831
|
+
const events = context.logicalEvents ?? context.events;
|
|
6832
|
+
context.selectedRun ? [
|
|
6833
|
+
{
|
|
6834
|
+
runId: context.selectedRun.runId,
|
|
6835
|
+
kind: "RUN",
|
|
6836
|
+
name: context.selectedRun.name
|
|
6837
|
+
}
|
|
6838
|
+
] : [];
|
|
6839
|
+
const base = evaluateRetrySafetyRules(events, retry);
|
|
6840
|
+
if (!retry.operations || retry.operations.length === 0) {
|
|
6841
|
+
return base;
|
|
6842
|
+
}
|
|
6843
|
+
return [...base, ...evaluateRecoveryOperations(events, retry.operations)];
|
|
6844
|
+
}
|
|
6845
|
+
});
|
|
6846
|
+
}
|
|
6847
|
+
return rules;
|
|
5059
6848
|
}
|
|
5060
|
-
new Set(OBSERVED_OUTCOME_METHODS);
|
|
5061
6849
|
function validateScopeShape(scope) {
|
|
5062
6850
|
if (scope === void 0) return [];
|
|
5063
6851
|
const hasSelector = typeof scope.runId === "string" && scope.runId.trim() !== "" || typeof scope.subAgentId === "string" && scope.subAgentId.trim() !== "" || typeof scope.groupId === "string" && scope.groupId.trim() !== "" || typeof scope.workflowStep === "string" && scope.workflowStep.trim() !== "" || typeof scope.rootEventId === "string" && scope.rootEventId.trim() !== "";
|
|
@@ -5158,6 +6946,57 @@ function validateAlternativesShape(alternatives) {
|
|
|
5158
6946
|
}
|
|
5159
6947
|
return diagnostics;
|
|
5160
6948
|
}
|
|
6949
|
+
function emptyPassResult(input, runId) {
|
|
6950
|
+
return {
|
|
6951
|
+
ok: true,
|
|
6952
|
+
status: "pass",
|
|
6953
|
+
format: input.read.format,
|
|
6954
|
+
...runId !== void 0 ? { runId } : {},
|
|
6955
|
+
summary: { passed: 0, failed: 0, warnings: 0, errors: 0, rulesEvaluated: 0 },
|
|
6956
|
+
findings: [],
|
|
6957
|
+
diagnostics: [],
|
|
6958
|
+
ruleExecutions: []
|
|
6959
|
+
};
|
|
6960
|
+
}
|
|
6961
|
+
function mergeContractResults(input, parts, extraFindings = []) {
|
|
6962
|
+
const findings = [...parts.flatMap((part) => part.findings), ...extraFindings];
|
|
6963
|
+
const diagnostics = parts.flatMap((part) => part.diagnostics);
|
|
6964
|
+
const ruleExecutions = parts.flatMap((part) => part.ruleExecutions);
|
|
6965
|
+
const failed = findings.filter(
|
|
6966
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
6967
|
+
).length;
|
|
6968
|
+
const errors = diagnostics.filter((item) => item.severity === "error").length;
|
|
6969
|
+
const status = errors > 0 ? "error" : failed > 0 ? "fail" : "pass";
|
|
6970
|
+
const runId = parts.find((part) => part.runId !== void 0)?.runId;
|
|
6971
|
+
return {
|
|
6972
|
+
ok: status === "pass",
|
|
6973
|
+
status,
|
|
6974
|
+
format: input.read.format,
|
|
6975
|
+
...runId !== void 0 ? { runId } : {},
|
|
6976
|
+
summary: {
|
|
6977
|
+
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
6978
|
+
failed,
|
|
6979
|
+
warnings: findings.filter(
|
|
6980
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
6981
|
+
).length,
|
|
6982
|
+
errors,
|
|
6983
|
+
rulesEvaluated: ruleExecutions.length
|
|
6984
|
+
},
|
|
6985
|
+
findings,
|
|
6986
|
+
diagnostics,
|
|
6987
|
+
ruleExecutions
|
|
6988
|
+
};
|
|
6989
|
+
}
|
|
6990
|
+
function evaluateBody(input, body, options = {}) {
|
|
6991
|
+
const rules = contractToRules(body);
|
|
6992
|
+
if (rules.length === 0) {
|
|
6993
|
+
return emptyPassResult(input, options.runId);
|
|
6994
|
+
}
|
|
6995
|
+
return runTraceChecks(input, {
|
|
6996
|
+
rules,
|
|
6997
|
+
...options.runId !== void 0 ? { runId: options.runId } : {}
|
|
6998
|
+
});
|
|
6999
|
+
}
|
|
5161
7000
|
function defineTraceContract(input) {
|
|
5162
7001
|
const shapeErrors = [
|
|
5163
7002
|
...validateAlternativesShape(input.alternatives),
|
|
@@ -5188,6 +7027,143 @@ function defineTraceContract(input) {
|
|
|
5188
7027
|
...alternatives ? { alternatives } : {}
|
|
5189
7028
|
};
|
|
5190
7029
|
}
|
|
7030
|
+
function evaluateTraceContract(input, contract, options = {}) {
|
|
7031
|
+
const shapeErrors = [
|
|
7032
|
+
...validateAlternativesShape(contract.alternatives),
|
|
7033
|
+
...validateScopeShape(contract.scope),
|
|
7034
|
+
...validateProvenanceShape(contract.observations)
|
|
7035
|
+
].filter((item) => item.severity === "error");
|
|
7036
|
+
if (shapeErrors.length > 0) {
|
|
7037
|
+
return {
|
|
7038
|
+
ok: false,
|
|
7039
|
+
status: "error",
|
|
7040
|
+
format: input.read.format,
|
|
7041
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
7042
|
+
summary: {
|
|
7043
|
+
passed: 0,
|
|
7044
|
+
failed: 0,
|
|
7045
|
+
warnings: 0,
|
|
7046
|
+
errors: shapeErrors.length,
|
|
7047
|
+
rulesEvaluated: 0
|
|
7048
|
+
},
|
|
7049
|
+
findings: [],
|
|
7050
|
+
diagnostics: shapeErrors.map((item) => ({
|
|
7051
|
+
code: "AI_CHECK_INVALID_CONFIG",
|
|
7052
|
+
message: item.message,
|
|
7053
|
+
severity: "error",
|
|
7054
|
+
ruleId: item.code
|
|
7055
|
+
})),
|
|
7056
|
+
ruleExecutions: []
|
|
7057
|
+
};
|
|
7058
|
+
}
|
|
7059
|
+
const scoped = resolveTraceContractScope(input, contract.scope);
|
|
7060
|
+
if (scoped.diagnostics.length > 0) {
|
|
7061
|
+
return {
|
|
7062
|
+
ok: false,
|
|
7063
|
+
status: "error",
|
|
7064
|
+
format: input.read.format,
|
|
7065
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
7066
|
+
...scoped.resolved?.runId !== void 0 ? { runId: scoped.resolved.runId } : {},
|
|
7067
|
+
summary: {
|
|
7068
|
+
passed: 0,
|
|
7069
|
+
failed: 0,
|
|
7070
|
+
warnings: 0,
|
|
7071
|
+
errors: scoped.diagnostics.length,
|
|
7072
|
+
rulesEvaluated: 0
|
|
7073
|
+
},
|
|
7074
|
+
findings: [],
|
|
7075
|
+
diagnostics: scoped.diagnostics,
|
|
7076
|
+
ruleExecutions: []
|
|
7077
|
+
};
|
|
7078
|
+
}
|
|
7079
|
+
const effectiveInput = scoped.resolved?.input ?? input;
|
|
7080
|
+
const effectiveRunId = scoped.resolved?.runId ?? options.runId;
|
|
7081
|
+
const base = evaluateBody(effectiveInput, contract, {
|
|
7082
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
7083
|
+
});
|
|
7084
|
+
const scopeFinding = scoped.resolved ? {
|
|
7085
|
+
ruleId: "contract.scope.selected",
|
|
7086
|
+
severity: "info",
|
|
7087
|
+
status: "pass",
|
|
7088
|
+
message: `Selected actor ${scoped.resolved.runId} with ${scoped.resolved.evidenceEventCount} evidence event(s).`,
|
|
7089
|
+
expected: scoped.resolved.matchedSelectors,
|
|
7090
|
+
actual: {
|
|
7091
|
+
runId: scoped.resolved.runId,
|
|
7092
|
+
evidenceEventCount: scoped.resolved.evidenceEventCount,
|
|
7093
|
+
...scoped.resolved.rootEventId !== void 0 ? { rootEventId: scoped.resolved.rootEventId } : {}
|
|
7094
|
+
},
|
|
7095
|
+
evidence: [
|
|
7096
|
+
{
|
|
7097
|
+
runId: scoped.resolved.runId,
|
|
7098
|
+
kind: "RUN",
|
|
7099
|
+
path: "scope"
|
|
7100
|
+
}
|
|
7101
|
+
]
|
|
7102
|
+
} : void 0;
|
|
7103
|
+
const branches = contract.alternatives?.anyOf ?? [];
|
|
7104
|
+
if (branches.length === 0) {
|
|
7105
|
+
return scopeFinding ? mergeContractResults(effectiveInput, [base], [scopeFinding]) : base;
|
|
7106
|
+
}
|
|
7107
|
+
const branchEvaluations = branches.map((branch) => {
|
|
7108
|
+
const result = evaluateBody(effectiveInput, branch.contract, {
|
|
7109
|
+
...effectiveRunId !== void 0 ? { runId: effectiveRunId } : {}
|
|
7110
|
+
});
|
|
7111
|
+
return { branch, result };
|
|
7112
|
+
});
|
|
7113
|
+
const satisfied = branchEvaluations.filter((item) => item.result.ok && item.result.status === "pass").map((item) => item.branch.id);
|
|
7114
|
+
const failedBranches = branchEvaluations.filter(
|
|
7115
|
+
(item) => !(item.result.ok && item.result.status === "pass")
|
|
7116
|
+
);
|
|
7117
|
+
if (satisfied.length > 0) {
|
|
7118
|
+
const passFinding = {
|
|
7119
|
+
ruleId: "contract.alternatives.anyOf",
|
|
7120
|
+
severity: "info",
|
|
7121
|
+
status: "pass",
|
|
7122
|
+
message: `Satisfied alternative branch(es): ${satisfied.join(", ")}.`,
|
|
7123
|
+
expected: { anyOf: branches.map((branch) => branch.id) },
|
|
7124
|
+
actual: {
|
|
7125
|
+
satisfied,
|
|
7126
|
+
failed: failedBranches.map((item) => item.branch.id)
|
|
7127
|
+
},
|
|
7128
|
+
evidence: []
|
|
7129
|
+
};
|
|
7130
|
+
return mergeContractResults(
|
|
7131
|
+
effectiveInput,
|
|
7132
|
+
[base],
|
|
7133
|
+
[...scopeFinding ? [scopeFinding] : [], passFinding]
|
|
7134
|
+
);
|
|
7135
|
+
}
|
|
7136
|
+
const noneFinding = {
|
|
7137
|
+
ruleId: "contract.alternatives.none-satisfied",
|
|
7138
|
+
severity: "error",
|
|
7139
|
+
status: "fail",
|
|
7140
|
+
message: "No alternatives.anyOf branch fully satisfied its contract.",
|
|
7141
|
+
expected: { anyOf: branches.map((branch) => branch.id) },
|
|
7142
|
+
actual: {
|
|
7143
|
+
branchFailures: Object.fromEntries(
|
|
7144
|
+
failedBranches.map((item) => [
|
|
7145
|
+
item.branch.id,
|
|
7146
|
+
item.result.findings.filter((finding) => finding.status === "fail").map((finding) => ({
|
|
7147
|
+
ruleId: finding.ruleId,
|
|
7148
|
+
message: finding.message
|
|
7149
|
+
}))
|
|
7150
|
+
])
|
|
7151
|
+
)
|
|
7152
|
+
},
|
|
7153
|
+
evidence: []
|
|
7154
|
+
};
|
|
7155
|
+
const prefixedBranchFindings = failedBranches.flatMap(
|
|
7156
|
+
(item) => item.result.findings.filter((finding) => finding.status === "fail").map((finding) => ({
|
|
7157
|
+
...finding,
|
|
7158
|
+
ruleId: `contract.alternatives.${item.branch.id}.${finding.ruleId}`
|
|
7159
|
+
}))
|
|
7160
|
+
);
|
|
7161
|
+
return mergeContractResults(
|
|
7162
|
+
effectiveInput,
|
|
7163
|
+
[base],
|
|
7164
|
+
[...scopeFinding ? [scopeFinding] : [], noneFinding, ...prefixedBranchFindings]
|
|
7165
|
+
);
|
|
7166
|
+
}
|
|
5191
7167
|
|
|
5192
7168
|
// packages/core/src/checks/index.ts
|
|
5193
7169
|
var SEVERITY_RANK = {
|
|
@@ -5277,7 +7253,7 @@ var DEFAULT_SECRET_PATTERNS = [
|
|
|
5277
7253
|
function compareStrings(a, b) {
|
|
5278
7254
|
return (a ?? "").localeCompare(b ?? "");
|
|
5279
7255
|
}
|
|
5280
|
-
function
|
|
7256
|
+
function diagnostic2(code, message, ruleId) {
|
|
5281
7257
|
return {
|
|
5282
7258
|
code,
|
|
5283
7259
|
message,
|
|
@@ -5310,14 +7286,14 @@ function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
|
5310
7286
|
ruleExecutions: [...ruleExecutions]
|
|
5311
7287
|
};
|
|
5312
7288
|
}
|
|
5313
|
-
function
|
|
5314
|
-
return nodes.flatMap((node) => [node, ...
|
|
7289
|
+
function flattenNodes2(nodes) {
|
|
7290
|
+
return nodes.flatMap((node) => [node, ...flattenNodes2(node.children)]);
|
|
5315
7291
|
}
|
|
5316
7292
|
function buildFacts2(input, selectedRun) {
|
|
5317
7293
|
const scopedRuns = selectedRun ? [selectedRun] : input.read.runs;
|
|
5318
7294
|
const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
|
|
5319
7295
|
const scopedEvents = selectedRun === void 0 ? input.read.events : input.read.events.filter((event) => scopedRunIds.has(event.runId));
|
|
5320
|
-
const nodes =
|
|
7296
|
+
const nodes = flattenNodes2(scopedRuns.flatMap((run) => run.children));
|
|
5321
7297
|
const nodesByEventId = /* @__PURE__ */ new Map();
|
|
5322
7298
|
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
5323
7299
|
for (const node of nodes) {
|
|
@@ -5349,7 +7325,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
5349
7325
|
if (runId && input.selectedRun.runId !== runId) {
|
|
5350
7326
|
return {
|
|
5351
7327
|
diagnostics: [
|
|
5352
|
-
|
|
7328
|
+
diagnostic2(
|
|
5353
7329
|
"AI_CHECK_INVALID_ARGUMENTS",
|
|
5354
7330
|
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
5355
7331
|
)
|
|
@@ -5363,7 +7339,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
5363
7339
|
if (!run) {
|
|
5364
7340
|
return {
|
|
5365
7341
|
diagnostics: [
|
|
5366
|
-
|
|
7342
|
+
diagnostic2("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
5367
7343
|
]
|
|
5368
7344
|
};
|
|
5369
7345
|
}
|
|
@@ -5375,7 +7351,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
5375
7351
|
if (input.read.runs.length === 0) {
|
|
5376
7352
|
return {
|
|
5377
7353
|
diagnostics: [
|
|
5378
|
-
|
|
7354
|
+
diagnostic2(
|
|
5379
7355
|
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
5380
7356
|
formatProgrammaticDiagnostic(
|
|
5381
7357
|
"AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
@@ -5387,7 +7363,7 @@ function resolveSelectedRun(input, runId) {
|
|
|
5387
7363
|
}
|
|
5388
7364
|
return {
|
|
5389
7365
|
diagnostics: [
|
|
5390
|
-
|
|
7366
|
+
diagnostic2(
|
|
5391
7367
|
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
5392
7368
|
formatProgrammaticDiagnostic("AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED")
|
|
5393
7369
|
)
|
|
@@ -5400,7 +7376,7 @@ function selectRules(rules, selectedIds) {
|
|
|
5400
7376
|
for (const rule of rules) {
|
|
5401
7377
|
if (byId.has(rule.id)) {
|
|
5402
7378
|
diagnostics.push(
|
|
5403
|
-
|
|
7379
|
+
diagnostic2("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
5404
7380
|
);
|
|
5405
7381
|
continue;
|
|
5406
7382
|
}
|
|
@@ -5411,7 +7387,7 @@ function selectRules(rules, selectedIds) {
|
|
|
5411
7387
|
for (const id of selected) {
|
|
5412
7388
|
if (!byId.has(id)) {
|
|
5413
7389
|
diagnostics.push(
|
|
5414
|
-
|
|
7390
|
+
diagnostic2("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
5415
7391
|
);
|
|
5416
7392
|
}
|
|
5417
7393
|
}
|
|
@@ -5526,7 +7502,7 @@ function stripPrefix(name, prefixes) {
|
|
|
5526
7502
|
}
|
|
5527
7503
|
return name;
|
|
5528
7504
|
}
|
|
5529
|
-
function
|
|
7505
|
+
function eventEvidence3(event, path14) {
|
|
5530
7506
|
return {
|
|
5531
7507
|
runId: event.runId,
|
|
5532
7508
|
eventId: event.eventId,
|
|
@@ -5586,6 +7562,14 @@ function finishedEvents(context, kind) {
|
|
|
5586
7562
|
function toolInvocationEvents(context) {
|
|
5587
7563
|
return semanticEvents(context).filter((event) => event.kind === "TOOL");
|
|
5588
7564
|
}
|
|
7565
|
+
function firstIndexByName(events) {
|
|
7566
|
+
const index = /* @__PURE__ */ new Map();
|
|
7567
|
+
for (const [position, event] of events.entries()) {
|
|
7568
|
+
const name = toolName(event);
|
|
7569
|
+
if (!index.has(name)) index.set(name, position);
|
|
7570
|
+
}
|
|
7571
|
+
return index;
|
|
7572
|
+
}
|
|
5589
7573
|
function isRecord9(value) {
|
|
5590
7574
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
5591
7575
|
}
|
|
@@ -5797,7 +7781,7 @@ function guardrailShape(context) {
|
|
|
5797
7781
|
}
|
|
5798
7782
|
function firstEvidenceForKind(context, kind, path14) {
|
|
5799
7783
|
const event = semanticEvents(context).find((candidate) => candidate.kind === kind);
|
|
5800
|
-
return event ? [
|
|
7784
|
+
return event ? [eventEvidence3(event, path14)] : runEvidence(context.selectedRun);
|
|
5801
7785
|
}
|
|
5802
7786
|
function baselineDiffFinding(message, evidence, expected, actual) {
|
|
5803
7787
|
return failFinding("baseline.regression", message, evidence, expected, actual);
|
|
@@ -5830,7 +7814,7 @@ function createRunStatusRule(options = {}) {
|
|
|
5830
7814
|
failFinding(
|
|
5831
7815
|
"run.status",
|
|
5832
7816
|
"Run contains incomplete running events.",
|
|
5833
|
-
running.map((event) =>
|
|
7817
|
+
running.map((event) => eventEvidence3(event)),
|
|
5834
7818
|
"no running events",
|
|
5835
7819
|
running.length
|
|
5836
7820
|
)
|
|
@@ -5876,7 +7860,7 @@ function createMaxStepDurationRule(options) {
|
|
|
5876
7860
|
failFinding(
|
|
5877
7861
|
"run.maxStepDuration",
|
|
5878
7862
|
`${over.length} step(s) exceeded max duration ${options.maxDurationMs}ms.`,
|
|
5879
|
-
over.map((event) =>
|
|
7863
|
+
over.map((event) => eventEvidence3(event, "durationMs")),
|
|
5880
7864
|
{ maxDurationMs: options.maxDurationMs },
|
|
5881
7865
|
over.map((event) => ({
|
|
5882
7866
|
eventId: event.eventId,
|
|
@@ -5902,7 +7886,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
5902
7886
|
failFinding(
|
|
5903
7887
|
"run.stall",
|
|
5904
7888
|
`Found ${running.length} event(s) still running (possible stall).`,
|
|
5905
|
-
running.map((event) =>
|
|
7889
|
+
running.map((event) => eventEvidence3(event, "status")),
|
|
5906
7890
|
"no running events",
|
|
5907
7891
|
running.length
|
|
5908
7892
|
)
|
|
@@ -5917,7 +7901,7 @@ function createStallDetectionRule(options = {}) {
|
|
|
5917
7901
|
failFinding(
|
|
5918
7902
|
"run.stall",
|
|
5919
7903
|
`Found ${incomplete.length} started event(s) without endedAt.`,
|
|
5920
|
-
incomplete.map((event) =>
|
|
7904
|
+
incomplete.map((event) => eventEvidence3(event, "endedAt")),
|
|
5921
7905
|
"endedAt for started events",
|
|
5922
7906
|
incomplete.length
|
|
5923
7907
|
)
|
|
@@ -5953,7 +7937,7 @@ function createRequireCompletedRule() {
|
|
|
5953
7937
|
failFinding(
|
|
5954
7938
|
"run.requireCompleted",
|
|
5955
7939
|
`Run has ${running.length} incomplete running event(s).`,
|
|
5956
|
-
running.map((event) =>
|
|
7940
|
+
running.map((event) => eventEvidence3(event, "status")),
|
|
5957
7941
|
"no running events",
|
|
5958
7942
|
running.length
|
|
5959
7943
|
)
|
|
@@ -6033,12 +8017,12 @@ function createToolUsageRule(options) {
|
|
|
6033
8017
|
const name = toolName(event);
|
|
6034
8018
|
if (forbidden.has(name)) {
|
|
6035
8019
|
findings.push(
|
|
6036
|
-
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [
|
|
8020
|
+
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence3(event)], "tool absent", name)
|
|
6037
8021
|
);
|
|
6038
8022
|
}
|
|
6039
8023
|
if (allowed && !allowed.has(name)) {
|
|
6040
8024
|
findings.push(
|
|
6041
|
-
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [
|
|
8025
|
+
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence3(event)], [...allowed].sort(), name)
|
|
6042
8026
|
);
|
|
6043
8027
|
}
|
|
6044
8028
|
}
|
|
@@ -6049,13 +8033,188 @@ function createToolUsageRule(options) {
|
|
|
6049
8033
|
}
|
|
6050
8034
|
if (options.maxCount !== void 0 && tools.length > options.maxCount) {
|
|
6051
8035
|
findings.push(
|
|
6052
|
-
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) =>
|
|
8036
|
+
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) => eventEvidence3(event)), { maxCount: options.maxCount }, tools.length)
|
|
6053
8037
|
);
|
|
6054
8038
|
}
|
|
6055
8039
|
return findings;
|
|
6056
8040
|
}
|
|
6057
8041
|
};
|
|
6058
8042
|
}
|
|
8043
|
+
function createToolOrderingRule(options) {
|
|
8044
|
+
const ruleId = options.id ?? "tool.order";
|
|
8045
|
+
const mode = options.mode ?? "first-occurrence";
|
|
8046
|
+
return {
|
|
8047
|
+
id: ruleId,
|
|
8048
|
+
category: "tool",
|
|
8049
|
+
defaultSeverity: "error",
|
|
8050
|
+
evaluate(context) {
|
|
8051
|
+
const tools = finishedEvents(context, "TOOL");
|
|
8052
|
+
const index = firstIndexByName(tools);
|
|
8053
|
+
const beforeIndex = index.get(options.before);
|
|
8054
|
+
const afterIndex = index.get(options.after);
|
|
8055
|
+
if (beforeIndex === void 0 || afterIndex === void 0) {
|
|
8056
|
+
return [];
|
|
8057
|
+
}
|
|
8058
|
+
if (mode === "first-occurrence" && beforeIndex >= afterIndex) {
|
|
8059
|
+
return [
|
|
8060
|
+
failFinding(
|
|
8061
|
+
ruleId,
|
|
8062
|
+
`Tool ${options.before} must appear before ${options.after}.`,
|
|
8063
|
+
[eventEvidence3(tools[beforeIndex]), eventEvidence3(tools[afterIndex])],
|
|
8064
|
+
{ before: options.before, after: options.after },
|
|
8065
|
+
tools.map(toolName)
|
|
8066
|
+
)
|
|
8067
|
+
];
|
|
8068
|
+
}
|
|
8069
|
+
const beforeEvent = tools[beforeIndex];
|
|
8070
|
+
const afterEvent = tools[afterIndex];
|
|
8071
|
+
const beforeEnd = eventEndMs(beforeEvent);
|
|
8072
|
+
const afterStart = eventStartMs(afterEvent);
|
|
8073
|
+
if (mode === "happens-before") {
|
|
8074
|
+
const expected = {
|
|
8075
|
+
before: options.before,
|
|
8076
|
+
after: options.after,
|
|
8077
|
+
mode,
|
|
8078
|
+
relation: "before.end <= after.start"
|
|
8079
|
+
};
|
|
8080
|
+
if (options.before === options.after) {
|
|
8081
|
+
return [
|
|
8082
|
+
failFinding(
|
|
8083
|
+
ruleId,
|
|
8084
|
+
`Tool ${options.before} cannot causally happen before itself.`,
|
|
8085
|
+
[eventEvidence3(beforeEvent)],
|
|
8086
|
+
expected,
|
|
8087
|
+
{ code: "tool.order.same-tool" }
|
|
8088
|
+
)
|
|
8089
|
+
];
|
|
8090
|
+
}
|
|
8091
|
+
if (beforeEnd === void 0 || afterStart === void 0) {
|
|
8092
|
+
return [
|
|
8093
|
+
failFinding(
|
|
8094
|
+
ruleId,
|
|
8095
|
+
`Tool order ${options.before} before ${options.after} could not establish causal timing.`,
|
|
8096
|
+
[eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
|
|
8097
|
+
expected,
|
|
8098
|
+
{
|
|
8099
|
+
code: "tool.order.interval-unresolved",
|
|
8100
|
+
beforeEndResolved: beforeEnd !== void 0,
|
|
8101
|
+
afterStartResolved: afterStart !== void 0
|
|
8102
|
+
}
|
|
8103
|
+
)
|
|
8104
|
+
];
|
|
8105
|
+
}
|
|
8106
|
+
if (beforeEnd > afterStart) {
|
|
8107
|
+
return [
|
|
8108
|
+
failFinding(
|
|
8109
|
+
ruleId,
|
|
8110
|
+
`Tool ${options.before} must finish before ${options.after} starts.`,
|
|
8111
|
+
[eventEvidence3(beforeEvent), eventEvidence3(afterEvent)],
|
|
8112
|
+
expected,
|
|
8113
|
+
{
|
|
8114
|
+
beforeEndedAt: new Date(beforeEnd).toISOString(),
|
|
8115
|
+
afterStartedAt: new Date(afterStart).toISOString()
|
|
8116
|
+
}
|
|
8117
|
+
)
|
|
8118
|
+
];
|
|
8119
|
+
}
|
|
8120
|
+
return [];
|
|
8121
|
+
}
|
|
8122
|
+
if (mode === "all-occurrences") {
|
|
8123
|
+
const beforeEvents = tools.filter((event) => toolName(event) === options.before);
|
|
8124
|
+
const afterEvents = tools.filter((event) => toolName(event) === options.after);
|
|
8125
|
+
const expected = {
|
|
8126
|
+
before: options.before,
|
|
8127
|
+
after: options.after,
|
|
8128
|
+
mode,
|
|
8129
|
+
relation: "max(before.end) <= min(after.start)"
|
|
8130
|
+
};
|
|
8131
|
+
if (options.before === options.after) {
|
|
8132
|
+
return [
|
|
8133
|
+
failFinding(
|
|
8134
|
+
ruleId,
|
|
8135
|
+
`Tool ${options.before} cannot causally happen before itself.`,
|
|
8136
|
+
[eventEvidence3(beforeEvent)],
|
|
8137
|
+
expected,
|
|
8138
|
+
{ code: "tool.order.same-tool" }
|
|
8139
|
+
)
|
|
8140
|
+
];
|
|
8141
|
+
}
|
|
8142
|
+
let latestBefore;
|
|
8143
|
+
let earliestAfter;
|
|
8144
|
+
let missingBeforeEnds = 0;
|
|
8145
|
+
let missingAfterStarts = 0;
|
|
8146
|
+
let firstMissingBefore;
|
|
8147
|
+
let firstMissingAfter;
|
|
8148
|
+
for (const event of beforeEvents) {
|
|
8149
|
+
const end = eventEndMs(event);
|
|
8150
|
+
if (end === void 0) {
|
|
8151
|
+
missingBeforeEnds += 1;
|
|
8152
|
+
firstMissingBefore ??= event;
|
|
8153
|
+
} else if (latestBefore === void 0 || end > latestBefore.end) {
|
|
8154
|
+
latestBefore = { event, end };
|
|
8155
|
+
}
|
|
8156
|
+
}
|
|
8157
|
+
for (const event of afterEvents) {
|
|
8158
|
+
const start = eventStartMs(event);
|
|
8159
|
+
if (start === void 0) {
|
|
8160
|
+
missingAfterStarts += 1;
|
|
8161
|
+
firstMissingAfter ??= event;
|
|
8162
|
+
} else if (earliestAfter === void 0 || start < earliestAfter.start) {
|
|
8163
|
+
earliestAfter = { event, start };
|
|
8164
|
+
}
|
|
8165
|
+
}
|
|
8166
|
+
if (missingBeforeEnds > 0 || missingAfterStarts > 0 || latestBefore === void 0 || earliestAfter === void 0) {
|
|
8167
|
+
return [
|
|
8168
|
+
failFinding(
|
|
8169
|
+
ruleId,
|
|
8170
|
+
`Tool order ${options.before} before ${options.after} could not establish all causal intervals.`,
|
|
8171
|
+
[firstMissingBefore, firstMissingAfter].filter((event) => event !== void 0).map((event) => eventEvidence3(event)),
|
|
8172
|
+
expected,
|
|
8173
|
+
{
|
|
8174
|
+
code: "tool.order.interval-unresolved",
|
|
8175
|
+
missingBeforeEnds,
|
|
8176
|
+
missingAfterStarts
|
|
8177
|
+
}
|
|
8178
|
+
)
|
|
8179
|
+
];
|
|
8180
|
+
}
|
|
8181
|
+
if (latestBefore.end > earliestAfter.start) {
|
|
8182
|
+
return [
|
|
8183
|
+
failFinding(
|
|
8184
|
+
ruleId,
|
|
8185
|
+
`Every ${options.before} tool call must finish before any ${options.after} tool call starts.`,
|
|
8186
|
+
[eventEvidence3(latestBefore.event), eventEvidence3(earliestAfter.event)],
|
|
8187
|
+
expected,
|
|
8188
|
+
{
|
|
8189
|
+
latestBeforeEndedAt: new Date(latestBefore.end).toISOString(),
|
|
8190
|
+
earliestAfterStartedAt: new Date(earliestAfter.start).toISOString()
|
|
8191
|
+
}
|
|
8192
|
+
)
|
|
8193
|
+
];
|
|
8194
|
+
}
|
|
8195
|
+
return [];
|
|
8196
|
+
}
|
|
8197
|
+
if (beforeEnd !== void 0 && afterStart !== void 0 && beforeEnd > afterStart) {
|
|
8198
|
+
return [
|
|
8199
|
+
{
|
|
8200
|
+
ruleId,
|
|
8201
|
+
severity: "warning",
|
|
8202
|
+
status: "warning",
|
|
8203
|
+
message: `Tool order ${options.before} before ${options.after} satisfied start/encounter order, but intervals overlap so causal completion before the next tool was not established.`,
|
|
8204
|
+
expected: { before: options.before, after: options.after, mode: "first-occurrence" },
|
|
8205
|
+
actual: {
|
|
8206
|
+
code: "tool.order.overlap",
|
|
8207
|
+
beforeEndedAt: beforeEvent.endedAt,
|
|
8208
|
+
afterStartedAt: afterEvent.startedAt ?? afterEvent.timestamp
|
|
8209
|
+
},
|
|
8210
|
+
evidence: [eventEvidence3(beforeEvent), eventEvidence3(afterEvent)]
|
|
8211
|
+
}
|
|
8212
|
+
];
|
|
8213
|
+
}
|
|
8214
|
+
return [];
|
|
8215
|
+
}
|
|
8216
|
+
};
|
|
8217
|
+
}
|
|
6059
8218
|
function createLlmUsageRule(options) {
|
|
6060
8219
|
return {
|
|
6061
8220
|
id: "llm.usage",
|
|
@@ -6072,7 +8231,7 @@ function createLlmUsageRule(options) {
|
|
|
6072
8231
|
failFinding(
|
|
6073
8232
|
"llm.usage",
|
|
6074
8233
|
`LLM call count ${llms.length} exceeded ${options.maxCalls}.`,
|
|
6075
|
-
llms.map((event) =>
|
|
8234
|
+
llms.map((event) => eventEvidence3(event)),
|
|
6076
8235
|
{ maxCalls: options.maxCalls },
|
|
6077
8236
|
llms.length
|
|
6078
8237
|
)
|
|
@@ -6084,17 +8243,17 @@ function createLlmUsageRule(options) {
|
|
|
6084
8243
|
const finishReason = llmFinishReason(event);
|
|
6085
8244
|
if (allowedModels && (!model || !allowedModels.has(model))) {
|
|
6086
8245
|
findings.push(
|
|
6087
|
-
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [
|
|
8246
|
+
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [eventEvidence3(event, "attributes.model")], [...allowedModels].sort(), model ?? "unknown")
|
|
6088
8247
|
);
|
|
6089
8248
|
}
|
|
6090
8249
|
if (allowedProviders && (!provider || !allowedProviders.has(provider))) {
|
|
6091
8250
|
findings.push(
|
|
6092
|
-
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [
|
|
8251
|
+
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [eventEvidence3(event, "attributes.provider")], [...allowedProviders].sort(), provider ?? "unknown")
|
|
6093
8252
|
);
|
|
6094
8253
|
}
|
|
6095
8254
|
if (finishReasons && (!finishReason || !finishReasons.has(finishReason))) {
|
|
6096
8255
|
findings.push(
|
|
6097
|
-
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [
|
|
8256
|
+
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [eventEvidence3(event, "attributes.finishReason")], [...finishReasons].sort(), finishReason ?? "unknown")
|
|
6098
8257
|
);
|
|
6099
8258
|
}
|
|
6100
8259
|
}
|
|
@@ -6119,7 +8278,7 @@ function createLlmUsageRule(options) {
|
|
|
6119
8278
|
failFinding(
|
|
6120
8279
|
"llm.usage",
|
|
6121
8280
|
`LLM ${key} token count ${tokenTotals[key]} exceeded ${limit}.`,
|
|
6122
|
-
llms.map((event) =>
|
|
8281
|
+
llms.map((event) => eventEvidence3(event, `tokenUsage.${key}`)),
|
|
6123
8282
|
{ [`max${key[0].toUpperCase()}${key.slice(1)}Tokens`]: limit },
|
|
6124
8283
|
tokenTotals[key]
|
|
6125
8284
|
)
|
|
@@ -6130,6 +8289,47 @@ function createLlmUsageRule(options) {
|
|
|
6130
8289
|
}
|
|
6131
8290
|
};
|
|
6132
8291
|
}
|
|
8292
|
+
function createStructureIncompleteRule(options = {}) {
|
|
8293
|
+
return {
|
|
8294
|
+
id: "structure.incomplete",
|
|
8295
|
+
category: "structure",
|
|
8296
|
+
defaultSeverity: "error",
|
|
8297
|
+
evaluate(context) {
|
|
8298
|
+
const findings = [];
|
|
8299
|
+
if (!options.allowRunning) {
|
|
8300
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
8301
|
+
if (running.length > 0) {
|
|
8302
|
+
findings.push(
|
|
8303
|
+
failFinding(
|
|
8304
|
+
"structure.incomplete",
|
|
8305
|
+
"Trace contains incomplete running events.",
|
|
8306
|
+
running.map((event) => eventEvidence3(event, "status")),
|
|
8307
|
+
"no running events",
|
|
8308
|
+
running.length
|
|
8309
|
+
)
|
|
8310
|
+
);
|
|
8311
|
+
}
|
|
8312
|
+
}
|
|
8313
|
+
if (options.requireEndedAtForStarted) {
|
|
8314
|
+
const missingEndedAt = semanticEvents(context).filter(
|
|
8315
|
+
(event) => event.startedAt !== void 0 && event.endedAt === void 0 && event.status !== "running"
|
|
8316
|
+
);
|
|
8317
|
+
if (missingEndedAt.length > 0) {
|
|
8318
|
+
findings.push(
|
|
8319
|
+
failFinding(
|
|
8320
|
+
"structure.incomplete",
|
|
8321
|
+
"Trace contains events with startedAt but no endedAt.",
|
|
8322
|
+
missingEndedAt.map((event) => eventEvidence3(event, "endedAt")),
|
|
8323
|
+
"endedAt for started events",
|
|
8324
|
+
missingEndedAt.length
|
|
8325
|
+
)
|
|
8326
|
+
);
|
|
8327
|
+
}
|
|
8328
|
+
}
|
|
8329
|
+
return findings;
|
|
8330
|
+
}
|
|
8331
|
+
};
|
|
8332
|
+
}
|
|
6133
8333
|
function createStructureOrphanRule(options = {}) {
|
|
6134
8334
|
const allowMarkedUnresolved = options.allowMarkedUnresolved ?? true;
|
|
6135
8335
|
return {
|
|
@@ -6147,7 +8347,7 @@ function createStructureOrphanRule(options = {}) {
|
|
|
6147
8347
|
failFinding(
|
|
6148
8348
|
"structure.orphan",
|
|
6149
8349
|
"Trace contains events whose parentId is not present in the selected run.",
|
|
6150
|
-
orphans.map((event) =>
|
|
8350
|
+
orphans.map((event) => eventEvidence3(event, "parentId")),
|
|
6151
8351
|
"parentId resolves to an event in the selected run",
|
|
6152
8352
|
orphans.length
|
|
6153
8353
|
)
|
|
@@ -6179,7 +8379,7 @@ function createStructureCycleRule() {
|
|
|
6179
8379
|
failFinding(
|
|
6180
8380
|
"structure.cycle",
|
|
6181
8381
|
formatProgrammaticDiagnostic("AI_TRACE_RELATIONSHIP_CYCLE"),
|
|
6182
|
-
cycle.map((item) =>
|
|
8382
|
+
cycle.map((item) => eventEvidence3(item, "parentId")),
|
|
6183
8383
|
"acyclic parentId graph",
|
|
6184
8384
|
cycle.map((item) => item.eventId).sort()
|
|
6185
8385
|
)
|
|
@@ -6219,7 +8419,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
6219
8419
|
failFinding(
|
|
6220
8420
|
"structure.relationship",
|
|
6221
8421
|
`Event confidence ${event.confidence} is below ${minConfidence}.`,
|
|
6222
|
-
[
|
|
8422
|
+
[eventEvidence3(event, "confidence")],
|
|
6223
8423
|
{ minConfidence },
|
|
6224
8424
|
event.confidence
|
|
6225
8425
|
)
|
|
@@ -6232,7 +8432,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
6232
8432
|
failFinding(
|
|
6233
8433
|
"structure.relationship",
|
|
6234
8434
|
"Event parentId points to itself.",
|
|
6235
|
-
[
|
|
8435
|
+
[eventEvidence3(event, "parentId")],
|
|
6236
8436
|
"parentId references a distinct event",
|
|
6237
8437
|
"self"
|
|
6238
8438
|
)
|
|
@@ -6249,7 +8449,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
6249
8449
|
failFinding(
|
|
6250
8450
|
"structure.relationship",
|
|
6251
8451
|
"Parent event starts after child event.",
|
|
6252
|
-
[
|
|
8452
|
+
[eventEvidence3(parent), eventEvidence3(event, "parentId")],
|
|
6253
8453
|
"parent start <= child start",
|
|
6254
8454
|
{ parentEventId: parent.eventId, childEventId: event.eventId }
|
|
6255
8455
|
)
|
|
@@ -6263,7 +8463,7 @@ function createStructureRelationshipRule(options = {}) {
|
|
|
6263
8463
|
failFinding(
|
|
6264
8464
|
"structure.relationship",
|
|
6265
8465
|
"Trace parentSpanId does not match parent spanId.",
|
|
6266
|
-
[
|
|
8466
|
+
[eventEvidence3(event, "trace.parentSpanId")],
|
|
6267
8467
|
{ parentSpanId: parent.trace.spanId },
|
|
6268
8468
|
actual ?? "missing"
|
|
6269
8469
|
)
|
|
@@ -6292,7 +8492,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
6292
8492
|
"structure.parallelWidth",
|
|
6293
8493
|
`Parent ${parentId} has ${children.length} children, exceeding ${options.maxChildren}.`,
|
|
6294
8494
|
[
|
|
6295
|
-
...parent ? [
|
|
8495
|
+
...parent ? [eventEvidence3(parent)] : [{ runId: context.selectedRun?.runId, eventId: parentId }],
|
|
6296
8496
|
...children.map((child) => ({
|
|
6297
8497
|
runId: child.event.runId,
|
|
6298
8498
|
eventId: child.event.eventId,
|
|
@@ -6340,7 +8540,7 @@ function createStructureParallelWidthRule(options) {
|
|
|
6340
8540
|
failFinding(
|
|
6341
8541
|
"structure.parallelWidth",
|
|
6342
8542
|
`Concurrent event width ${maxActive.length} exceeded ${options.maxConcurrent}.`,
|
|
6343
|
-
maxActive.map((event) =>
|
|
8543
|
+
maxActive.map((event) => eventEvidence3(event)),
|
|
6344
8544
|
{ maxConcurrent: options.maxConcurrent },
|
|
6345
8545
|
maxActive.length
|
|
6346
8546
|
)
|
|
@@ -6368,7 +8568,7 @@ function createSafetyRedactionRule(options = {}) {
|
|
|
6368
8568
|
failFinding(
|
|
6369
8569
|
"safety.redaction",
|
|
6370
8570
|
`Sensitive-looking field at ${entry.path} is not redacted.`,
|
|
6371
|
-
[
|
|
8571
|
+
[eventEvidence3(event, entry.path)],
|
|
6372
8572
|
"redaction marker",
|
|
6373
8573
|
{ path: entry.path, valueType: valueType(entry.value) },
|
|
6374
8574
|
{
|
|
@@ -6404,7 +8604,7 @@ function createSafetyRawContentRule(options = {}) {
|
|
|
6404
8604
|
failFinding(
|
|
6405
8605
|
"safety.rawPrompt",
|
|
6406
8606
|
`Raw content-like field ${entry.path} is present.`,
|
|
6407
|
-
[
|
|
8607
|
+
[eventEvidence3(event, entry.path)],
|
|
6408
8608
|
"metadata-only trace fields",
|
|
6409
8609
|
{ path: entry.path, valueType: valueType(entry.value) },
|
|
6410
8610
|
{
|
|
@@ -6442,7 +8642,7 @@ function createSafetySecretPatternRule(options = {}) {
|
|
|
6442
8642
|
failFinding(
|
|
6443
8643
|
"safety.secretPattern",
|
|
6444
8644
|
`Secret-like pattern ${pattern.id} matched at ${entry.path}.`,
|
|
6445
|
-
[
|
|
8645
|
+
[eventEvidence3(event, entry.path)],
|
|
6446
8646
|
"no secret-like strings",
|
|
6447
8647
|
{ pattern: pattern.id, path: entry.path },
|
|
6448
8648
|
{
|
|
@@ -6475,7 +8675,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
6475
8675
|
failFinding(
|
|
6476
8676
|
"safety.oversizedAttribute",
|
|
6477
8677
|
`String at ${entry.path} exceeds ${options.maxStringLength} characters.`,
|
|
6478
|
-
[
|
|
8678
|
+
[eventEvidence3(event, entry.path)],
|
|
6479
8679
|
{ maxStringLength: options.maxStringLength },
|
|
6480
8680
|
{ path: entry.path, length: entry.value.length },
|
|
6481
8681
|
{
|
|
@@ -6492,7 +8692,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
6492
8692
|
failFinding(
|
|
6493
8693
|
"safety.oversizedAttribute",
|
|
6494
8694
|
`Array at ${entry.path} exceeds ${options.maxArrayLength} items.`,
|
|
6495
|
-
[
|
|
8695
|
+
[eventEvidence3(event, entry.path)],
|
|
6496
8696
|
{ maxArrayLength: options.maxArrayLength },
|
|
6497
8697
|
{ path: entry.path, length: entry.value.length },
|
|
6498
8698
|
{
|
|
@@ -6509,7 +8709,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
6509
8709
|
failFinding(
|
|
6510
8710
|
"safety.oversizedAttribute",
|
|
6511
8711
|
`Object at ${entry.path} exceeds ${options.maxObjectKeys} keys.`,
|
|
6512
|
-
[
|
|
8712
|
+
[eventEvidence3(event, entry.path)],
|
|
6513
8713
|
{ maxObjectKeys: options.maxObjectKeys },
|
|
6514
8714
|
{ path: entry.path, keys: Object.keys(entry.value).length },
|
|
6515
8715
|
{
|
|
@@ -6528,7 +8728,7 @@ function createSafetyOversizedAttributeRule(options) {
|
|
|
6528
8728
|
failFinding(
|
|
6529
8729
|
"safety.oversizedAttribute",
|
|
6530
8730
|
`Value at ${entry.path} exceeds ${options.maxSerializedBytes} serialized bytes.`,
|
|
6531
|
-
[
|
|
8731
|
+
[eventEvidence3(event, entry.path)],
|
|
6532
8732
|
{ maxSerializedBytes: options.maxSerializedBytes },
|
|
6533
8733
|
{ path: entry.path, bytes },
|
|
6534
8734
|
{
|
|
@@ -6655,7 +8855,7 @@ function createBaselineRegressionRule(options) {
|
|
|
6655
8855
|
path: "guardrail",
|
|
6656
8856
|
expected: guardrailShape(baselineContext),
|
|
6657
8857
|
actual: guardrailShape(context),
|
|
6658
|
-
evidence: guardrailEvents(context)[0] ? [
|
|
8858
|
+
evidence: guardrailEvents(context)[0] ? [eventEvidence3(guardrailEvents(context)[0], "guardrail")] : runEvidence(context.selectedRun)
|
|
6659
8859
|
}
|
|
6660
8860
|
];
|
|
6661
8861
|
for (const comparison of comparisons) {
|
|
@@ -6734,7 +8934,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
6734
8934
|
return errorResult(
|
|
6735
8935
|
input,
|
|
6736
8936
|
[
|
|
6737
|
-
|
|
8937
|
+
diagnostic2(
|
|
6738
8938
|
"AI_CHECK_NO_RULES_EVALUATED",
|
|
6739
8939
|
"No trace check rules were evaluated. Configure at least one rule, contract, or CLI check option."
|
|
6740
8940
|
)
|
|
@@ -6765,7 +8965,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
6765
8965
|
} catch (error) {
|
|
6766
8966
|
const message = error instanceof Error ? error.message : String(error);
|
|
6767
8967
|
diagnostics.push(
|
|
6768
|
-
|
|
8968
|
+
diagnostic2("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
|
|
6769
8969
|
);
|
|
6770
8970
|
ruleExecutions.push({
|
|
6771
8971
|
ruleId: rule.id,
|
|
@@ -6829,18 +9029,18 @@ function escapeMarkdown(value) {
|
|
|
6829
9029
|
function escapeHtml(value) {
|
|
6830
9030
|
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
6831
9031
|
}
|
|
6832
|
-
function
|
|
9032
|
+
function sortKeysDeep2(input) {
|
|
6833
9033
|
if (input === null || typeof input !== "object") return input;
|
|
6834
|
-
if (Array.isArray(input)) return input.map(
|
|
9034
|
+
if (Array.isArray(input)) return input.map(sortKeysDeep2);
|
|
6835
9035
|
const o = input;
|
|
6836
9036
|
const out = {};
|
|
6837
9037
|
for (const k of Object.keys(o).sort()) {
|
|
6838
|
-
out[k] =
|
|
9038
|
+
out[k] = sortKeysDeep2(o[k]);
|
|
6839
9039
|
}
|
|
6840
9040
|
return out;
|
|
6841
9041
|
}
|
|
6842
9042
|
function stableJson(value, pretty) {
|
|
6843
|
-
const sorted =
|
|
9043
|
+
const sorted = sortKeysDeep2(value);
|
|
6844
9044
|
return pretty === true ? JSON.stringify(sorted, null, 2) : JSON.stringify(sorted);
|
|
6845
9045
|
}
|
|
6846
9046
|
function compactAttributes(attrs, options) {
|
|
@@ -8700,7 +10900,7 @@ var DEFAULT_SUITE_CONFIG_NAMES = [
|
|
|
8700
10900
|
"agent-inspect.suite.cjs"
|
|
8701
10901
|
];
|
|
8702
10902
|
var DEFAULT_SUITE_ARTIFACTS_DIR = ".agent-inspect/suite-runs";
|
|
8703
|
-
function
|
|
10903
|
+
function diagnostic3(code, message, severity = "error", caseId) {
|
|
8704
10904
|
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
8705
10905
|
}
|
|
8706
10906
|
function asString(value, label) {
|
|
@@ -8728,7 +10928,7 @@ function validateCaseConfig(value, index) {
|
|
|
8728
10928
|
const diagnostics = [];
|
|
8729
10929
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
8730
10930
|
diagnostics.push(
|
|
8731
|
-
|
|
10931
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
|
|
8732
10932
|
);
|
|
8733
10933
|
return { diagnostics };
|
|
8734
10934
|
}
|
|
@@ -8737,7 +10937,7 @@ function validateCaseConfig(value, index) {
|
|
|
8737
10937
|
const id = asString(raw.id, `cases[${index}].id`);
|
|
8738
10938
|
if (id === void 0) {
|
|
8739
10939
|
diagnostics.push(
|
|
8740
|
-
|
|
10940
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
|
|
8741
10941
|
);
|
|
8742
10942
|
return { diagnostics };
|
|
8743
10943
|
}
|
|
@@ -8772,7 +10972,7 @@ function validateCaseConfig(value, index) {
|
|
|
8772
10972
|
};
|
|
8773
10973
|
} catch (error) {
|
|
8774
10974
|
const message = error instanceof Error ? error.message : String(error);
|
|
8775
|
-
diagnostics.push(
|
|
10975
|
+
diagnostics.push(diagnostic3("AI_SUITE_CONFIG_INVALID", message));
|
|
8776
10976
|
return { diagnostics };
|
|
8777
10977
|
}
|
|
8778
10978
|
}
|
|
@@ -8848,7 +11048,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
8848
11048
|
await access(tracesDir);
|
|
8849
11049
|
} catch {
|
|
8850
11050
|
diagnostics.push(
|
|
8851
|
-
|
|
11051
|
+
diagnostic3("AI_SUITE_CONFIG_INVALID", `traces directory not found: ${tracesDir}`)
|
|
8852
11052
|
);
|
|
8853
11053
|
}
|
|
8854
11054
|
for (const suiteCase of config.cases) {
|
|
@@ -8858,7 +11058,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
8858
11058
|
await access(inputPath);
|
|
8859
11059
|
} catch {
|
|
8860
11060
|
diagnostics.push(
|
|
8861
|
-
|
|
11061
|
+
diagnostic3(
|
|
8862
11062
|
"AI_SUITE_CONFIG_INVALID",
|
|
8863
11063
|
`input fixture not found: ${suiteCase.input}`,
|
|
8864
11064
|
"warning",
|
|
@@ -8872,7 +11072,7 @@ async function validateSuiteConfig(config, options) {
|
|
|
8872
11072
|
}
|
|
8873
11073
|
var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
8874
11074
|
var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
8875
|
-
function
|
|
11075
|
+
function diagnostic4(code, message) {
|
|
8876
11076
|
return { code, message, severity: "error" };
|
|
8877
11077
|
}
|
|
8878
11078
|
async function fileExists(filePath) {
|
|
@@ -8903,20 +11103,20 @@ async function loadSuiteConfig(options = {}) {
|
|
|
8903
11103
|
} catch (error) {
|
|
8904
11104
|
const message = error instanceof Error ? error.message : String(error);
|
|
8905
11105
|
throw Object.assign(new Error(message), {
|
|
8906
|
-
diagnostics: [
|
|
11106
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
8907
11107
|
});
|
|
8908
11108
|
}
|
|
8909
11109
|
const extension = path7.extname(configPath);
|
|
8910
11110
|
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
8911
11111
|
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
8912
11112
|
throw Object.assign(new Error(message), {
|
|
8913
|
-
diagnostics: [
|
|
11113
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
8914
11114
|
});
|
|
8915
11115
|
}
|
|
8916
11116
|
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
8917
11117
|
const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
|
|
8918
11118
|
throw Object.assign(new Error(message), {
|
|
8919
|
-
diagnostics: [
|
|
11119
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
8920
11120
|
});
|
|
8921
11121
|
}
|
|
8922
11122
|
try {
|
|
@@ -8936,7 +11136,7 @@ async function loadSuiteConfig(options = {}) {
|
|
|
8936
11136
|
} catch (error) {
|
|
8937
11137
|
const message = error instanceof Error ? error.message : String(error);
|
|
8938
11138
|
throw Object.assign(new Error(message), {
|
|
8939
|
-
diagnostics: [
|
|
11139
|
+
diagnostics: [diagnostic4("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
8940
11140
|
});
|
|
8941
11141
|
}
|
|
8942
11142
|
}
|
|
@@ -9859,13 +12059,13 @@ function persistedEventsForParsedTrace(parsed) {
|
|
|
9859
12059
|
function isRecord11(value) {
|
|
9860
12060
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
9861
12061
|
}
|
|
9862
|
-
function
|
|
12062
|
+
function isNonEmptyString5(value) {
|
|
9863
12063
|
return typeof value === "string" && value.trim() !== "";
|
|
9864
12064
|
}
|
|
9865
12065
|
function readStringField(record, keys) {
|
|
9866
12066
|
for (const key of keys) {
|
|
9867
12067
|
const value = record[key];
|
|
9868
|
-
if (
|
|
12068
|
+
if (isNonEmptyString5(value)) return value;
|
|
9869
12069
|
}
|
|
9870
12070
|
return void 0;
|
|
9871
12071
|
}
|
|
@@ -9999,7 +12199,7 @@ function parseUnixNanoToIso(value) {
|
|
|
9999
12199
|
return void 0;
|
|
10000
12200
|
}
|
|
10001
12201
|
function parseIsoTime(value) {
|
|
10002
|
-
if (!
|
|
12202
|
+
if (!isNonEmptyString5(value)) return void 0;
|
|
10003
12203
|
const ms = Date.parse(value);
|
|
10004
12204
|
if (!Number.isFinite(ms)) return void 0;
|
|
10005
12205
|
return new Date(ms).toISOString();
|
|
@@ -11209,7 +13409,7 @@ async function resolveSuiteCaseTrace(suiteCase, options) {
|
|
|
11209
13409
|
}
|
|
11210
13410
|
|
|
11211
13411
|
// packages/core/src/suite/run.ts
|
|
11212
|
-
function
|
|
13412
|
+
function diagnostic5(code, message, severity = "error", caseId) {
|
|
11213
13413
|
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
11214
13414
|
}
|
|
11215
13415
|
function buildCaseRules(suiteCase, config) {
|
|
@@ -11273,7 +13473,7 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
11273
13473
|
const match = outcomes.find((outcome) => outcome.name === name);
|
|
11274
13474
|
if (!match) {
|
|
11275
13475
|
diagnostics.push(
|
|
11276
|
-
|
|
13476
|
+
diagnostic5(
|
|
11277
13477
|
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
11278
13478
|
`Expected observation "${name}" was not recorded.`,
|
|
11279
13479
|
"error",
|
|
@@ -11284,7 +13484,7 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
11284
13484
|
}
|
|
11285
13485
|
if (match.status !== "passed") {
|
|
11286
13486
|
diagnostics.push(
|
|
11287
|
-
|
|
13487
|
+
diagnostic5(
|
|
11288
13488
|
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
11289
13489
|
`Observation "${name}" has status "${match.status}", expected "passed".`,
|
|
11290
13490
|
"error",
|
|
@@ -11304,7 +13504,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11304
13504
|
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
11305
13505
|
message: resolved.reason,
|
|
11306
13506
|
diagnostics: [
|
|
11307
|
-
|
|
13507
|
+
diagnostic5(
|
|
11308
13508
|
"AI_SUITE_CASE_TRACE_MISSING",
|
|
11309
13509
|
resolved.reason ?? "Trace not found.",
|
|
11310
13510
|
"warning",
|
|
@@ -11325,7 +13525,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11325
13525
|
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
11326
13526
|
message,
|
|
11327
13527
|
diagnostics: [
|
|
11328
|
-
|
|
13528
|
+
diagnostic5("AI_SUITE_TRACE_UNREADABLE", message, "error", suiteCase.id)
|
|
11329
13529
|
]
|
|
11330
13530
|
};
|
|
11331
13531
|
}
|
|
@@ -11339,7 +13539,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11339
13539
|
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
11340
13540
|
const diagnostics = [
|
|
11341
13541
|
...checkResult.diagnostics.map(
|
|
11342
|
-
(item) =>
|
|
13542
|
+
(item) => diagnostic5(
|
|
11343
13543
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
11344
13544
|
item.message,
|
|
11345
13545
|
item.severity,
|
|
@@ -11347,7 +13547,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
11347
13547
|
)
|
|
11348
13548
|
),
|
|
11349
13549
|
...checkResult.findings.filter((finding) => finding.status === "fail").map(
|
|
11350
|
-
(finding) =>
|
|
13550
|
+
(finding) => diagnostic5(
|
|
11351
13551
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
11352
13552
|
finding.message,
|
|
11353
13553
|
finding.severity,
|
|
@@ -12532,6 +14732,6 @@ function renderGateReport(result, options = {}) {
|
|
|
12532
14732
|
return renderGateSummaryMarkdown(result);
|
|
12533
14733
|
}
|
|
12534
14734
|
|
|
12535
|
-
export { ATTRIBUTION_CONFIDENCES, COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, Redactor, TraceDirectory, TraceReadError, TreeBuilder, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, assertEvidenceRelativePath, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceHtmlShell, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectTraceSchemaVersions, compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, diffRuns, diffTraceEvents, enrichSessionRunRecord, escapeHtml, escapeMarkdown, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatStepLabel, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, inferEvidenceFileRole, isAgentInspectTrace, isAttributionConfidence, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, manualTraceEventsToComparableRun, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToTraceEvents, projectLogicalEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunDiff, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, serializeEvidenceManifest, sha256Hex, stableJson, summarizeObservedOutcomes, summarizeSemanticParity, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, verifyEvidenceDirectory, zeroKinds };
|
|
12536
|
-
//# sourceMappingURL=chunk-
|
|
12537
|
-
//# sourceMappingURL=chunk-
|
|
14735
|
+
export { ATTRIBUTION_CONFIDENCES, COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, EVIDENCE_FORMAT_VERSION, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, Redactor, TraceDirectory, TraceReadError, TreeBuilder, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, assertEvidenceRelativePath, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceCausalFailureViewHtml, buildEvidenceCiPackage, buildEvidenceCircuitViewHtml, buildEvidenceContractsViewHtml, buildEvidenceDiffViewHtml, buildEvidenceHtmlShell, buildEvidenceManifest, buildEvidenceOutcomesViewHtml, buildEvidenceProvenanceViewHtml, buildEvidenceSafetyViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceTreeViewHtml, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, buildZipArchive, bundleFailsOnSafety, bundleRunAssetRelativePath, collectTraceSchemaVersions, compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, defineTraceContract, diffRuns, diffTraceEvents, enrichSessionRunRecord, escapeHtml, escapeMarkdown, evaluateTraceContract, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatStepLabel, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, inferEvidenceFileRole, isAgentInspectTrace, isAttributionConfidence, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, manualTraceEventsToComparableRun, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToTraceEvents, projectLogicalEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunDiff, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, serializeEvidenceManifest, sha256Hex, stableJson, summarizeObservedOutcomes, summarizeSemanticParity, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, verifyEvidenceDirectory, zeroKinds };
|
|
14736
|
+
//# sourceMappingURL=chunk-BDRJENLM.mjs.map
|
|
14737
|
+
//# sourceMappingURL=chunk-BDRJENLM.mjs.map
|