@agent-inspect/mcp-server 6.26.0 → 6.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-DSUDP7FB.mjs → chunk-V6J5UBTW.mjs} +1942 -1938
- package/dist/chunk-V6J5UBTW.mjs.map +1 -0
- package/dist/cli.cjs +1940 -1936
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.mjs +1 -1
- package/dist/index.cjs +1940 -1936
- package/dist/index.cjs.map +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +3 -3
- package/dist/chunk-DSUDP7FB.mjs.map +0 -1
package/dist/cli.cjs
CHANGED
|
@@ -2150,6 +2150,9 @@ function inferEvidenceFileRole(relativePath) {
|
|
|
2150
2150
|
if (base === "check-results.json") {
|
|
2151
2151
|
return "checks";
|
|
2152
2152
|
}
|
|
2153
|
+
if (base === "contract.resolved.json") {
|
|
2154
|
+
return "contract";
|
|
2155
|
+
}
|
|
2153
2156
|
if (base === "redaction-report.json") {
|
|
2154
2157
|
return "redaction-report";
|
|
2155
2158
|
}
|
|
@@ -2216,670 +2219,204 @@ function buildEvidenceManifest(parts) {
|
|
|
2216
2219
|
},
|
|
2217
2220
|
assessment,
|
|
2218
2221
|
...parts.semantics !== void 0 ? { semantics: { ...parts.semantics } } : {},
|
|
2222
|
+
...parts.contract !== void 0 ? { contract: { ...parts.contract } } : {},
|
|
2219
2223
|
files: buildEvidenceFileEntries(parts.files)
|
|
2220
2224
|
};
|
|
2221
2225
|
}
|
|
2222
2226
|
|
|
2223
|
-
// packages/core/src/
|
|
2224
|
-
var
|
|
2227
|
+
// packages/core/src/diagnostics/programmatic.ts
|
|
2228
|
+
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
2229
|
+
AI_TRACE_INPUT_INVALID: {
|
|
2230
|
+
code: "AI_TRACE_INPUT_INVALID",
|
|
2231
|
+
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
2232
|
+
remediation: "For a file path, use openTraceFile(path).",
|
|
2233
|
+
relatedCodes: ["invalid_input"]
|
|
2234
|
+
},
|
|
2235
|
+
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
2236
|
+
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
2237
|
+
summary: "No trace reader could detect the input format.",
|
|
2238
|
+
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
2239
|
+
relatedCodes: ["unsupported_format"]
|
|
2240
|
+
},
|
|
2241
|
+
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
2242
|
+
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
2243
|
+
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
2244
|
+
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
2245
|
+
relatedCodes: ["ambiguous_format"]
|
|
2246
|
+
},
|
|
2247
|
+
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
2248
|
+
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
2249
|
+
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
2250
|
+
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
2251
|
+
},
|
|
2252
|
+
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
2253
|
+
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
2254
|
+
summary: "Multiple runs are available; select a run before executing checks.",
|
|
2255
|
+
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
2256
|
+
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
2257
|
+
},
|
|
2258
|
+
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
2259
|
+
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2260
|
+
summary: "A parentId equals its own event/step id.",
|
|
2261
|
+
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
2262
|
+
relatedCodes: [
|
|
2263
|
+
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
2264
|
+
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
2265
|
+
]
|
|
2266
|
+
},
|
|
2267
|
+
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
2268
|
+
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
2269
|
+
summary: "Trace contains a parentId cycle.",
|
|
2270
|
+
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
2271
|
+
relatedCodes: ["structure.cycle"]
|
|
2272
|
+
}
|
|
2273
|
+
});
|
|
2274
|
+
function formatProgrammaticDiagnostic(code, detail) {
|
|
2275
|
+
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
2276
|
+
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
2277
|
+
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
2278
|
+
}
|
|
2279
|
+
|
|
2280
|
+
// packages/core/src/safety/sensitive-key.ts
|
|
2281
|
+
function keyHasExplicitSeparator(value) {
|
|
2282
|
+
return /[_\-.]/.test(value);
|
|
2283
|
+
}
|
|
2284
|
+
function normalizeSensitiveKey(value) {
|
|
2285
|
+
return value.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
2286
|
+
}
|
|
2287
|
+
var NON_CREDENTIAL_TOKEN_CONFIG_KEYS = new Set(
|
|
2288
|
+
[
|
|
2289
|
+
"tokens",
|
|
2290
|
+
"max_tokens",
|
|
2291
|
+
"min_tokens",
|
|
2292
|
+
"ls_max_tokens",
|
|
2293
|
+
"token_count",
|
|
2294
|
+
"token_limit",
|
|
2295
|
+
"token_budget",
|
|
2296
|
+
"input_tokens",
|
|
2297
|
+
"output_tokens",
|
|
2298
|
+
"total_tokens",
|
|
2299
|
+
"cached_tokens",
|
|
2300
|
+
"prompt_tokens",
|
|
2301
|
+
"completion_tokens"
|
|
2302
|
+
].map(normalizeSensitiveKey)
|
|
2303
|
+
);
|
|
2304
|
+
var DEFAULT_CREDENTIAL_SENSITIVE_KEYS = [
|
|
2225
2305
|
"authorization",
|
|
2226
2306
|
"cookie",
|
|
2227
2307
|
"token",
|
|
2308
|
+
"access_token",
|
|
2309
|
+
"accesstoken",
|
|
2310
|
+
"auth_token",
|
|
2311
|
+
"authtoken",
|
|
2312
|
+
"refresh_token",
|
|
2313
|
+
"refreshtoken",
|
|
2314
|
+
"id_token",
|
|
2315
|
+
"idtoken",
|
|
2316
|
+
"bearer_token",
|
|
2317
|
+
"bearertoken",
|
|
2318
|
+
"api_token",
|
|
2319
|
+
"apitoken",
|
|
2228
2320
|
"apikey",
|
|
2321
|
+
"api_key",
|
|
2229
2322
|
"password",
|
|
2230
2323
|
"secret",
|
|
2231
2324
|
"email"
|
|
2232
2325
|
];
|
|
2233
|
-
function
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
return false;
|
|
2326
|
+
function isTokenCredentialKey(normalized) {
|
|
2327
|
+
if (NON_CREDENTIAL_TOKEN_CONFIG_KEYS.has(normalized)) return false;
|
|
2328
|
+
if (normalized === "token") return true;
|
|
2329
|
+
if (normalized.endsWith("tokens")) return false;
|
|
2330
|
+
return normalized.endsWith("token");
|
|
2239
2331
|
}
|
|
2240
|
-
function
|
|
2241
|
-
if (
|
|
2242
|
-
|
|
2243
|
-
if (
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2332
|
+
function isCredentialSensitiveKey(key, sensitiveKeys = DEFAULT_CREDENTIAL_SENSITIVE_KEYS) {
|
|
2333
|
+
if (!key) return false;
|
|
2334
|
+
const normalized = normalizeSensitiveKey(key);
|
|
2335
|
+
if (!normalized) return false;
|
|
2336
|
+
if (NON_CREDENTIAL_TOKEN_CONFIG_KEYS.has(normalized)) return false;
|
|
2337
|
+
const allowPrefixCompound = keyHasExplicitSeparator(key);
|
|
2338
|
+
for (const sensitive of sensitiveKeys) {
|
|
2339
|
+
const s = normalizeSensitiveKey(sensitive);
|
|
2340
|
+
if (!s) continue;
|
|
2341
|
+
if (s === "token") {
|
|
2342
|
+
if (isTokenCredentialKey(normalized)) return true;
|
|
2343
|
+
continue;
|
|
2344
|
+
}
|
|
2345
|
+
if (normalized === s) return true;
|
|
2346
|
+
if (normalized.endsWith(`_${s}`)) return true;
|
|
2347
|
+
if (allowPrefixCompound && normalized.startsWith(`${s}_`)) return true;
|
|
2248
2348
|
}
|
|
2249
|
-
return
|
|
2250
|
-
}
|
|
2251
|
-
function escapeMarkdown(value) {
|
|
2252
|
-
return value.replace(/\|/g, "\\|").replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(/\n/g, " ");
|
|
2349
|
+
return false;
|
|
2253
2350
|
}
|
|
2254
|
-
|
|
2255
|
-
|
|
2351
|
+
|
|
2352
|
+
// packages/core/src/checks/logical-events.ts
|
|
2353
|
+
function isRecord6(value) {
|
|
2354
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2256
2355
|
}
|
|
2257
|
-
function
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
const o = input;
|
|
2261
|
-
const out = {};
|
|
2262
|
-
for (const k of Object.keys(o).sort()) {
|
|
2263
|
-
out[k] = sortKeysDeep(o[k]);
|
|
2264
|
-
}
|
|
2265
|
-
return out;
|
|
2356
|
+
function legacyEvent(event) {
|
|
2357
|
+
const value = event.attributes?.legacyEvent;
|
|
2358
|
+
return typeof value === "string" ? value : void 0;
|
|
2266
2359
|
}
|
|
2267
|
-
function
|
|
2268
|
-
const
|
|
2269
|
-
|
|
2360
|
+
function stepIdOf(event) {
|
|
2361
|
+
const value = event.attributes?.stepId;
|
|
2362
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2363
|
+
return void 0;
|
|
2270
2364
|
}
|
|
2271
|
-
function
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
continue;
|
|
2280
|
-
}
|
|
2281
|
-
const v = attrs2[key];
|
|
2282
|
-
out[key] = compactValue(v, maxLen, redacted);
|
|
2283
|
-
}
|
|
2284
|
-
return out;
|
|
2365
|
+
function cloneEvent(event) {
|
|
2366
|
+
return {
|
|
2367
|
+
...event,
|
|
2368
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
2369
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
2370
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
2371
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
2372
|
+
};
|
|
2285
2373
|
}
|
|
2286
|
-
function
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
const o = value;
|
|
2296
|
-
const inner = {};
|
|
2297
|
-
for (const k of Object.keys(o)) {
|
|
2298
|
-
if (redacted && shouldRedactKey(k)) inner[k] = "[REDACTED]";
|
|
2299
|
-
else inner[k] = compactValue(o[k], maxLen, redacted);
|
|
2374
|
+
function mergeAttributes(start, complete) {
|
|
2375
|
+
const merged = {
|
|
2376
|
+
...isRecord6(complete.attributes) ? complete.attributes : {},
|
|
2377
|
+
...isRecord6(start.attributes) ? start.attributes : {}
|
|
2378
|
+
};
|
|
2379
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
2380
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
2381
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
2382
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
2300
2383
|
}
|
|
2301
|
-
return
|
|
2384
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
2302
2385
|
}
|
|
2303
|
-
function
|
|
2304
|
-
const
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2386
|
+
function pairStartComplete(start, complete) {
|
|
2387
|
+
const attributes = mergeAttributes(start, complete);
|
|
2388
|
+
const paired = {
|
|
2389
|
+
...cloneEvent(start),
|
|
2390
|
+
status: complete.status,
|
|
2391
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
2392
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
2393
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
2394
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
2395
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
2396
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
2397
|
+
};
|
|
2398
|
+
return {
|
|
2399
|
+
...paired,
|
|
2400
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
2401
|
+
projection: {
|
|
2402
|
+
paired: true,
|
|
2403
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
2404
|
+
parentNormalized: false,
|
|
2405
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
2309
2406
|
}
|
|
2310
|
-
}
|
|
2311
|
-
walk(tree.children);
|
|
2312
|
-
return out;
|
|
2407
|
+
};
|
|
2313
2408
|
}
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
}
|
|
2326
|
-
function manualTraceEventsToComparableRun(events) {
|
|
2327
|
-
const started = events.find((e) => e.event === "run_started");
|
|
2328
|
-
if (!started || started.event !== "run_started") {
|
|
2329
|
-
throw new Error("Invalid trace: missing run_started");
|
|
2330
|
-
}
|
|
2331
|
-
const rs = started;
|
|
2332
|
-
const runId = rs.runId;
|
|
2333
|
-
const completedAll = events.filter((e) => e.event === "run_completed");
|
|
2334
|
-
const lastCompleted = completedAll[completedAll.length - 1];
|
|
2335
|
-
let runStatus;
|
|
2336
|
-
if (lastCompleted === void 0) runStatus = "running";
|
|
2337
|
-
else runStatus = lastCompleted.status;
|
|
2338
|
-
const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
2339
|
-
const steps = /* @__PURE__ */ new Map();
|
|
2340
|
-
let order = 0;
|
|
2341
|
-
for (const e of events) {
|
|
2342
|
-
if (e.event !== "step_started") continue;
|
|
2343
|
-
const s = e;
|
|
2344
|
-
const meta = s.metadata ? { ...s.metadata } : void 0;
|
|
2345
|
-
steps.set(s.stepId, {
|
|
2346
|
-
id: s.stepId,
|
|
2347
|
-
parentId: s.parentId,
|
|
2348
|
-
name: s.name,
|
|
2349
|
-
type: s.type,
|
|
2350
|
-
order: order++,
|
|
2351
|
-
timestamp: s.timestamp,
|
|
2352
|
-
metadata: meta
|
|
2353
|
-
});
|
|
2354
|
-
}
|
|
2355
|
-
for (const e of events) {
|
|
2356
|
-
if (e.event !== "step_completed") continue;
|
|
2357
|
-
const acc = steps.get(e.stepId);
|
|
2358
|
-
if (!acc) continue;
|
|
2359
|
-
acc.status = e.status;
|
|
2360
|
-
acc.durationMs = e.durationMs;
|
|
2361
|
-
if (e.error?.message) acc.errorMsg = e.error.message;
|
|
2362
|
-
const extra = e;
|
|
2363
|
-
if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
|
|
2364
|
-
acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
|
|
2365
|
-
}
|
|
2366
|
-
}
|
|
2367
|
-
const nodes = /* @__PURE__ */ new Map();
|
|
2368
|
-
for (const acc of steps.values()) {
|
|
2369
|
-
let meta = acc.metadata ? { ...acc.metadata } : void 0;
|
|
2370
|
-
if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
|
|
2371
|
-
meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
|
|
2372
|
-
}
|
|
2373
|
-
const outputPreview = extractOutputPreview(meta);
|
|
2374
|
-
if (meta !== void 0 && ("outputPreview" in meta || "resultPreview" in meta)) {
|
|
2375
|
-
delete meta.outputPreview;
|
|
2376
|
-
delete meta.resultPreview;
|
|
2377
|
-
}
|
|
2378
|
-
const sc = {
|
|
2379
|
-
id: acc.id,
|
|
2380
|
-
name: acc.name,
|
|
2381
|
-
type: acc.type,
|
|
2382
|
-
status: mapStepStatus(acc.status),
|
|
2383
|
-
durationMs: acc.durationMs,
|
|
2384
|
-
error: acc.errorMsg,
|
|
2385
|
-
metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
|
|
2386
|
-
outputPreview,
|
|
2387
|
-
children: []
|
|
2388
|
-
};
|
|
2389
|
-
nodes.set(acc.id, sc);
|
|
2390
|
-
}
|
|
2391
|
-
const roots = [];
|
|
2392
|
-
const sortByOrder = (a, b) => {
|
|
2393
|
-
const oa = steps.get(a.id)?.order ?? 0;
|
|
2394
|
-
const ob = steps.get(b.id)?.order ?? 0;
|
|
2395
|
-
return oa - ob;
|
|
2396
|
-
};
|
|
2397
|
-
for (const acc of steps.values()) {
|
|
2398
|
-
const node = nodes.get(acc.id);
|
|
2399
|
-
if (acc.parentId !== void 0 && nodes.has(acc.parentId)) {
|
|
2400
|
-
nodes.get(acc.parentId).children.push(node);
|
|
2401
|
-
} else {
|
|
2402
|
-
roots.push(node);
|
|
2403
|
-
}
|
|
2404
|
-
}
|
|
2405
|
-
roots.sort(sortByOrder);
|
|
2406
|
-
for (const n of nodes.values()) {
|
|
2407
|
-
n.children.sort(sortByOrder);
|
|
2408
|
-
}
|
|
2409
|
-
return {
|
|
2410
|
-
runId,
|
|
2411
|
-
name: rs.name,
|
|
2412
|
-
status: runStatus,
|
|
2413
|
-
durationMs,
|
|
2414
|
-
steps: roots
|
|
2415
|
-
};
|
|
2416
|
-
}
|
|
2417
|
-
|
|
2418
|
-
// packages/core/src/diff/engine.ts
|
|
2419
|
-
var DEFAULT_THRESHOLD_MS = 0;
|
|
2420
|
-
function pathSeg(step, index) {
|
|
2421
|
-
return { index, name: step.name, stepId: step.id };
|
|
2422
|
-
}
|
|
2423
|
-
function buildPath(segments) {
|
|
2424
|
-
return { path: [...segments] };
|
|
2425
|
-
}
|
|
2426
|
-
function pairSteps(left, right) {
|
|
2427
|
-
const usedRight = /* @__PURE__ */ new Set();
|
|
2428
|
-
const pairs = [];
|
|
2429
|
-
for (let i = 0; i < left.length; i++) {
|
|
2430
|
-
const L = left[i];
|
|
2431
|
-
let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
|
|
2432
|
-
if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
|
|
2433
|
-
const cand = right[i];
|
|
2434
|
-
if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
|
|
2435
|
-
R = cand;
|
|
2436
|
-
}
|
|
2437
|
-
}
|
|
2438
|
-
if (R === void 0) {
|
|
2439
|
-
R = right.find(
|
|
2440
|
-
(r) => !usedRight.has(r.id) && r.name === L.name && (r.type ?? "") === (L.type ?? "")
|
|
2441
|
-
);
|
|
2442
|
-
}
|
|
2443
|
-
if (R !== void 0) {
|
|
2444
|
-
usedRight.add(R.id);
|
|
2445
|
-
pairs.push([L, R]);
|
|
2446
|
-
} else {
|
|
2447
|
-
pairs.push([L, void 0]);
|
|
2448
|
-
}
|
|
2449
|
-
}
|
|
2450
|
-
for (const R of right) {
|
|
2451
|
-
if (!usedRight.has(R.id)) {
|
|
2452
|
-
pairs.push([void 0, R]);
|
|
2453
|
-
}
|
|
2454
|
-
}
|
|
2455
|
-
return pairs;
|
|
2456
|
-
}
|
|
2457
|
-
function compareLeafSteps(L, R, segments, opts, out) {
|
|
2458
|
-
const path17 = buildPath(segments);
|
|
2459
|
-
if (L.name !== R.name) {
|
|
2460
|
-
out.push({
|
|
2461
|
-
kind: "structure",
|
|
2462
|
-
severity: "warning",
|
|
2463
|
-
message: "Step name differs",
|
|
2464
|
-
path: path17,
|
|
2465
|
-
left: L.name,
|
|
2466
|
-
right: R.name
|
|
2467
|
-
});
|
|
2468
|
-
}
|
|
2469
|
-
if ((L.type ?? "") !== (R.type ?? "")) {
|
|
2470
|
-
out.push({
|
|
2471
|
-
kind: "step-type",
|
|
2472
|
-
severity: "warning",
|
|
2473
|
-
message: "Step type differs",
|
|
2474
|
-
path: path17,
|
|
2475
|
-
left: L.type,
|
|
2476
|
-
right: R.type
|
|
2477
|
-
});
|
|
2478
|
-
}
|
|
2479
|
-
if ((L.status ?? "") !== (R.status ?? "")) {
|
|
2480
|
-
out.push({
|
|
2481
|
-
kind: "step-status",
|
|
2482
|
-
severity: "warning",
|
|
2483
|
-
message: "Step status differs",
|
|
2484
|
-
path: path17,
|
|
2485
|
-
left: L.status,
|
|
2486
|
-
right: R.status
|
|
2487
|
-
});
|
|
2488
|
-
}
|
|
2489
|
-
const le = L.error ?? "";
|
|
2490
|
-
const re = R.error ?? "";
|
|
2491
|
-
if (le !== re) {
|
|
2492
|
-
out.push({
|
|
2493
|
-
kind: "error",
|
|
2494
|
-
severity: "error",
|
|
2495
|
-
message: "Step error message differs",
|
|
2496
|
-
path: path17,
|
|
2497
|
-
left: le || void 0,
|
|
2498
|
-
right: re || void 0
|
|
2499
|
-
});
|
|
2500
|
-
}
|
|
2501
|
-
if (!opts.ignoreDuration) {
|
|
2502
|
-
const ld = L.durationMs;
|
|
2503
|
-
const rd = R.durationMs;
|
|
2504
|
-
const th = opts.durationThresholdMs;
|
|
2505
|
-
let differs = false;
|
|
2506
|
-
if (ld === void 0 && rd === void 0) differs = false;
|
|
2507
|
-
else if (ld === void 0 || rd === void 0) differs = true;
|
|
2508
|
-
else differs = Math.abs(ld - rd) > th;
|
|
2509
|
-
if (differs) {
|
|
2510
|
-
out.push({
|
|
2511
|
-
kind: "duration",
|
|
2512
|
-
severity: "info",
|
|
2513
|
-
message: "Step duration differs",
|
|
2514
|
-
path: path17,
|
|
2515
|
-
left: ld,
|
|
2516
|
-
right: rd
|
|
2517
|
-
});
|
|
2518
|
-
}
|
|
2519
|
-
}
|
|
2520
|
-
const lm = stableJson(L.metadata ?? {});
|
|
2521
|
-
const rm = stableJson(R.metadata ?? {});
|
|
2522
|
-
if (lm !== rm) {
|
|
2523
|
-
out.push({
|
|
2524
|
-
kind: "metadata",
|
|
2525
|
-
severity: "info",
|
|
2526
|
-
message: "Step metadata differs",
|
|
2527
|
-
path: path17,
|
|
2528
|
-
left: L.metadata,
|
|
2529
|
-
right: R.metadata
|
|
2530
|
-
});
|
|
2531
|
-
}
|
|
2532
|
-
const lo = stableJson(L.outputPreview ?? null);
|
|
2533
|
-
const ro = stableJson(R.outputPreview ?? null);
|
|
2534
|
-
if (lo !== ro) {
|
|
2535
|
-
out.push({
|
|
2536
|
-
kind: "output",
|
|
2537
|
-
severity: "info",
|
|
2538
|
-
message: "Output preview differs",
|
|
2539
|
-
path: path17,
|
|
2540
|
-
left: L.outputPreview,
|
|
2541
|
-
right: R.outputPreview
|
|
2542
|
-
});
|
|
2543
|
-
}
|
|
2544
|
-
}
|
|
2545
|
-
function compareRecursive(L, R, segments, opts, out) {
|
|
2546
|
-
compareLeafSteps(L, R, segments, opts, out);
|
|
2547
|
-
const pairs = pairSteps(L.children, R.children);
|
|
2548
|
-
let ci = 0;
|
|
2549
|
-
for (const [lch, rch] of pairs) {
|
|
2550
|
-
if (lch !== void 0 && rch !== void 0) {
|
|
2551
|
-
compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
|
|
2552
|
-
} else if (lch !== void 0) {
|
|
2553
|
-
out.push({
|
|
2554
|
-
kind: "step-removed",
|
|
2555
|
-
severity: "warning",
|
|
2556
|
-
message: `Step only in left run: ${lch.name}`,
|
|
2557
|
-
path: buildPath([...segments, pathSeg(lch, ci)]),
|
|
2558
|
-
left: lch.id,
|
|
2559
|
-
right: void 0
|
|
2560
|
-
});
|
|
2561
|
-
} else if (rch !== void 0) {
|
|
2562
|
-
out.push({
|
|
2563
|
-
kind: "step-added",
|
|
2564
|
-
severity: "warning",
|
|
2565
|
-
message: `Step only in right run: ${rch.name}`,
|
|
2566
|
-
path: buildPath([...segments, pathSeg(rch, ci)]),
|
|
2567
|
-
left: void 0,
|
|
2568
|
-
right: rch.id
|
|
2569
|
-
});
|
|
2570
|
-
}
|
|
2571
|
-
ci += 1;
|
|
2572
|
-
}
|
|
2573
|
-
}
|
|
2574
|
-
function mergeDiffDefaults(options) {
|
|
2575
|
-
return {
|
|
2576
|
-
ignoreDuration: false,
|
|
2577
|
-
durationThresholdMs: DEFAULT_THRESHOLD_MS,
|
|
2578
|
-
focus: "all",
|
|
2579
|
-
check: "all"
|
|
2580
|
-
};
|
|
2581
|
-
}
|
|
2582
|
-
function kindMatchesFilter(kind, merged) {
|
|
2583
|
-
return true;
|
|
2584
|
-
}
|
|
2585
|
-
function diffRuns(left, right, options) {
|
|
2586
|
-
const merged = mergeDiffDefaults();
|
|
2587
|
-
const opts = {
|
|
2588
|
-
ignoreDuration: merged.ignoreDuration,
|
|
2589
|
-
durationThresholdMs: merged.durationThresholdMs
|
|
2590
|
-
};
|
|
2591
|
-
const raw = [];
|
|
2592
|
-
if ((left.status ?? "") !== (right.status ?? "")) {
|
|
2593
|
-
raw.push({
|
|
2594
|
-
kind: "run-status",
|
|
2595
|
-
severity: "warning",
|
|
2596
|
-
message: "Run completion status differs",
|
|
2597
|
-
left: left.status,
|
|
2598
|
-
right: right.status
|
|
2599
|
-
});
|
|
2600
|
-
}
|
|
2601
|
-
{
|
|
2602
|
-
const ld = left.durationMs;
|
|
2603
|
-
const rd = right.durationMs;
|
|
2604
|
-
const th = merged.durationThresholdMs;
|
|
2605
|
-
let differs = false;
|
|
2606
|
-
if (ld === void 0 && rd === void 0) differs = false;
|
|
2607
|
-
else if (ld === void 0 || rd === void 0) differs = true;
|
|
2608
|
-
else differs = Math.abs(ld - rd) > th;
|
|
2609
|
-
if (differs) {
|
|
2610
|
-
raw.push({
|
|
2611
|
-
kind: "duration",
|
|
2612
|
-
severity: "info",
|
|
2613
|
-
message: "Run duration differs",
|
|
2614
|
-
left: ld,
|
|
2615
|
-
right: rd
|
|
2616
|
-
});
|
|
2617
|
-
}
|
|
2618
|
-
}
|
|
2619
|
-
const pairs = pairSteps(left.steps, right.steps);
|
|
2620
|
-
let idx = 0;
|
|
2621
|
-
for (const [ls, rs] of pairs) {
|
|
2622
|
-
if (ls !== void 0 && rs !== void 0) {
|
|
2623
|
-
compareRecursive(ls, rs, [pathSeg(ls, idx)], opts, raw);
|
|
2624
|
-
idx += 1;
|
|
2625
|
-
} else if (ls !== void 0) {
|
|
2626
|
-
raw.push({
|
|
2627
|
-
kind: "step-removed",
|
|
2628
|
-
severity: "warning",
|
|
2629
|
-
message: `Step only in left run: ${ls.name}`,
|
|
2630
|
-
path: buildPath([pathSeg(ls, idx)]),
|
|
2631
|
-
left: ls.id,
|
|
2632
|
-
right: void 0
|
|
2633
|
-
});
|
|
2634
|
-
idx += 1;
|
|
2635
|
-
} else if (rs !== void 0) {
|
|
2636
|
-
raw.push({
|
|
2637
|
-
kind: "step-added",
|
|
2638
|
-
severity: "warning",
|
|
2639
|
-
message: `Step only in right run: ${rs.name}`,
|
|
2640
|
-
path: buildPath([pathSeg(rs, idx)]),
|
|
2641
|
-
left: void 0,
|
|
2642
|
-
right: rs.id
|
|
2643
|
-
});
|
|
2644
|
-
idx += 1;
|
|
2645
|
-
}
|
|
2646
|
-
}
|
|
2647
|
-
const differences = raw.filter((d) => kindMatchesFilter(d.kind));
|
|
2648
|
-
let errors = 0;
|
|
2649
|
-
let warnings = 0;
|
|
2650
|
-
let info = 0;
|
|
2651
|
-
for (const d of differences) {
|
|
2652
|
-
if (d.severity === "error") errors += 1;
|
|
2653
|
-
else if (d.severity === "warning") warnings += 1;
|
|
2654
|
-
else info += 1;
|
|
2655
|
-
}
|
|
2656
|
-
const firstVisible = differences[0];
|
|
2657
|
-
const firstDivergence = firstVisible !== void 0 ? {
|
|
2658
|
-
kind: "first-divergence",
|
|
2659
|
-
severity: firstVisible.severity,
|
|
2660
|
-
message: `First divergence: ${firstVisible.message}`,
|
|
2661
|
-
path: firstVisible.path,
|
|
2662
|
-
left: firstVisible.left,
|
|
2663
|
-
right: firstVisible.right
|
|
2664
|
-
} : void 0;
|
|
2665
|
-
const summary = {
|
|
2666
|
-
leftRunId: left.runId,
|
|
2667
|
-
rightRunId: right.runId,
|
|
2668
|
-
totalDifferences: differences.length,
|
|
2669
|
-
errors,
|
|
2670
|
-
warnings,
|
|
2671
|
-
info,
|
|
2672
|
-
firstDivergence
|
|
2673
|
-
};
|
|
2674
|
-
return { summary, differences };
|
|
2675
|
-
}
|
|
2676
|
-
|
|
2677
|
-
// packages/core/src/evidence/zip.ts
|
|
2678
|
-
(() => {
|
|
2679
|
-
const table = new Uint32Array(256);
|
|
2680
|
-
for (let n = 0; n < 256; n += 1) {
|
|
2681
|
-
let c = n;
|
|
2682
|
-
for (let k = 0; k < 8; k += 1) {
|
|
2683
|
-
c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
|
|
2684
|
-
}
|
|
2685
|
-
table[n] = c >>> 0;
|
|
2686
|
-
}
|
|
2687
|
-
return table;
|
|
2688
|
-
})();
|
|
2689
|
-
|
|
2690
|
-
// packages/core/src/diagnostics/programmatic.ts
|
|
2691
|
-
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
2692
|
-
AI_TRACE_INPUT_INVALID: {
|
|
2693
|
-
code: "AI_TRACE_INPUT_INVALID",
|
|
2694
|
-
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
2695
|
-
remediation: "For a file path, use openTraceFile(path).",
|
|
2696
|
-
relatedCodes: ["invalid_input"]
|
|
2697
|
-
},
|
|
2698
|
-
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
2699
|
-
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
2700
|
-
summary: "No trace reader could detect the input format.",
|
|
2701
|
-
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
2702
|
-
relatedCodes: ["unsupported_format"]
|
|
2703
|
-
},
|
|
2704
|
-
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
2705
|
-
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
2706
|
-
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
2707
|
-
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
2708
|
-
relatedCodes: ["ambiguous_format"]
|
|
2709
|
-
},
|
|
2710
|
-
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
2711
|
-
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
2712
|
-
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
2713
|
-
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
2714
|
-
},
|
|
2715
|
-
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
2716
|
-
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
2717
|
-
summary: "Multiple runs are available; select a run before executing checks.",
|
|
2718
|
-
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
2719
|
-
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
2720
|
-
},
|
|
2721
|
-
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
2722
|
-
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2723
|
-
summary: "A parentId equals its own event/step id.",
|
|
2724
|
-
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
2725
|
-
relatedCodes: [
|
|
2726
|
-
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
2727
|
-
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
2728
|
-
]
|
|
2729
|
-
},
|
|
2730
|
-
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
2731
|
-
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
2732
|
-
summary: "Trace contains a parentId cycle.",
|
|
2733
|
-
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
2734
|
-
relatedCodes: ["structure.cycle"]
|
|
2735
|
-
}
|
|
2736
|
-
});
|
|
2737
|
-
function formatProgrammaticDiagnostic(code, detail) {
|
|
2738
|
-
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
2739
|
-
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
2740
|
-
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
2741
|
-
}
|
|
2742
|
-
|
|
2743
|
-
// packages/core/src/safety/sensitive-key.ts
|
|
2744
|
-
function keyHasExplicitSeparator(value) {
|
|
2745
|
-
return /[_\-.]/.test(value);
|
|
2746
|
-
}
|
|
2747
|
-
function normalizeSensitiveKey(value) {
|
|
2748
|
-
return value.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
2749
|
-
}
|
|
2750
|
-
var NON_CREDENTIAL_TOKEN_CONFIG_KEYS = new Set(
|
|
2751
|
-
[
|
|
2752
|
-
"tokens",
|
|
2753
|
-
"max_tokens",
|
|
2754
|
-
"min_tokens",
|
|
2755
|
-
"ls_max_tokens",
|
|
2756
|
-
"token_count",
|
|
2757
|
-
"token_limit",
|
|
2758
|
-
"token_budget",
|
|
2759
|
-
"input_tokens",
|
|
2760
|
-
"output_tokens",
|
|
2761
|
-
"total_tokens",
|
|
2762
|
-
"cached_tokens",
|
|
2763
|
-
"prompt_tokens",
|
|
2764
|
-
"completion_tokens"
|
|
2765
|
-
].map(normalizeSensitiveKey)
|
|
2766
|
-
);
|
|
2767
|
-
var DEFAULT_CREDENTIAL_SENSITIVE_KEYS = [
|
|
2768
|
-
"authorization",
|
|
2769
|
-
"cookie",
|
|
2770
|
-
"token",
|
|
2771
|
-
"access_token",
|
|
2772
|
-
"accesstoken",
|
|
2773
|
-
"auth_token",
|
|
2774
|
-
"authtoken",
|
|
2775
|
-
"refresh_token",
|
|
2776
|
-
"refreshtoken",
|
|
2777
|
-
"id_token",
|
|
2778
|
-
"idtoken",
|
|
2779
|
-
"bearer_token",
|
|
2780
|
-
"bearertoken",
|
|
2781
|
-
"api_token",
|
|
2782
|
-
"apitoken",
|
|
2783
|
-
"apikey",
|
|
2784
|
-
"api_key",
|
|
2785
|
-
"password",
|
|
2786
|
-
"secret",
|
|
2787
|
-
"email"
|
|
2788
|
-
];
|
|
2789
|
-
function isTokenCredentialKey(normalized) {
|
|
2790
|
-
if (NON_CREDENTIAL_TOKEN_CONFIG_KEYS.has(normalized)) return false;
|
|
2791
|
-
if (normalized === "token") return true;
|
|
2792
|
-
if (normalized.endsWith("tokens")) return false;
|
|
2793
|
-
return normalized.endsWith("token");
|
|
2794
|
-
}
|
|
2795
|
-
function isCredentialSensitiveKey(key, sensitiveKeys = DEFAULT_CREDENTIAL_SENSITIVE_KEYS) {
|
|
2796
|
-
if (!key) return false;
|
|
2797
|
-
const normalized = normalizeSensitiveKey(key);
|
|
2798
|
-
if (!normalized) return false;
|
|
2799
|
-
if (NON_CREDENTIAL_TOKEN_CONFIG_KEYS.has(normalized)) return false;
|
|
2800
|
-
const allowPrefixCompound = keyHasExplicitSeparator(key);
|
|
2801
|
-
for (const sensitive of sensitiveKeys) {
|
|
2802
|
-
const s = normalizeSensitiveKey(sensitive);
|
|
2803
|
-
if (!s) continue;
|
|
2804
|
-
if (s === "token") {
|
|
2805
|
-
if (isTokenCredentialKey(normalized)) return true;
|
|
2806
|
-
continue;
|
|
2807
|
-
}
|
|
2808
|
-
if (normalized === s) return true;
|
|
2809
|
-
if (normalized.endsWith(`_${s}`)) return true;
|
|
2810
|
-
if (allowPrefixCompound && normalized.startsWith(`${s}_`)) return true;
|
|
2811
|
-
}
|
|
2812
|
-
return false;
|
|
2813
|
-
}
|
|
2814
|
-
|
|
2815
|
-
// packages/core/src/checks/logical-events.ts
|
|
2816
|
-
function isRecord6(value) {
|
|
2817
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2818
|
-
}
|
|
2819
|
-
function legacyEvent(event) {
|
|
2820
|
-
const value = event.attributes?.legacyEvent;
|
|
2821
|
-
return typeof value === "string" ? value : void 0;
|
|
2822
|
-
}
|
|
2823
|
-
function stepIdOf(event) {
|
|
2824
|
-
const value = event.attributes?.stepId;
|
|
2825
|
-
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2826
|
-
return void 0;
|
|
2827
|
-
}
|
|
2828
|
-
function cloneEvent(event) {
|
|
2829
|
-
return {
|
|
2830
|
-
...event,
|
|
2831
|
-
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
2832
|
-
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
2833
|
-
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
2834
|
-
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
2835
|
-
};
|
|
2836
|
-
}
|
|
2837
|
-
function mergeAttributes(start, complete) {
|
|
2838
|
-
const merged = {
|
|
2839
|
-
...isRecord6(complete.attributes) ? complete.attributes : {},
|
|
2840
|
-
...isRecord6(start.attributes) ? start.attributes : {}
|
|
2841
|
-
};
|
|
2842
|
-
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
2843
|
-
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
2844
|
-
if (complete.attributes?.errorStack !== void 0) {
|
|
2845
|
-
merged.errorStack = complete.attributes.errorStack;
|
|
2846
|
-
}
|
|
2847
|
-
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
2848
|
-
}
|
|
2849
|
-
function pairStartComplete(start, complete) {
|
|
2850
|
-
const attributes = mergeAttributes(start, complete);
|
|
2851
|
-
const paired = {
|
|
2852
|
-
...cloneEvent(start),
|
|
2853
|
-
status: complete.status,
|
|
2854
|
-
timestamp: complete.timestamp ?? start.timestamp,
|
|
2855
|
-
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
2856
|
-
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
2857
|
-
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
2858
|
-
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
2859
|
-
...attributes !== void 0 ? { attributes } : {}
|
|
2860
|
-
};
|
|
2861
|
-
return {
|
|
2862
|
-
...paired,
|
|
2863
|
-
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
2864
|
-
projection: {
|
|
2865
|
-
paired: true,
|
|
2866
|
-
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
2867
|
-
parentNormalized: false,
|
|
2868
|
-
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
2869
|
-
}
|
|
2870
|
-
};
|
|
2871
|
-
}
|
|
2872
|
-
function asLogical(event, extras) {
|
|
2873
|
-
return {
|
|
2874
|
-
...cloneEvent(event),
|
|
2875
|
-
sourceEventIds: Object.freeze([event.eventId]),
|
|
2876
|
-
projection: {
|
|
2877
|
-
paired: false,
|
|
2878
|
-
absorbedEventIds: Object.freeze([]),
|
|
2879
|
-
parentNormalized: extras?.parentNormalized === true,
|
|
2880
|
-
...{}
|
|
2881
|
-
}
|
|
2882
|
-
};
|
|
2409
|
+
function asLogical(event, extras) {
|
|
2410
|
+
return {
|
|
2411
|
+
...cloneEvent(event),
|
|
2412
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
2413
|
+
projection: {
|
|
2414
|
+
paired: false,
|
|
2415
|
+
absorbedEventIds: Object.freeze([]),
|
|
2416
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
2417
|
+
...{}
|
|
2418
|
+
}
|
|
2419
|
+
};
|
|
2883
2420
|
}
|
|
2884
2421
|
function projectLogicalEvents(events) {
|
|
2885
2422
|
const diagnostics = [];
|
|
@@ -2967,1441 +2504,1908 @@ function projectLogicalEvents(events) {
|
|
|
2967
2504
|
message: `No matching start for complete ${complete.eventId}.`,
|
|
2968
2505
|
eventIds: [complete.eventId]
|
|
2969
2506
|
});
|
|
2970
|
-
const alone = asLogical(complete);
|
|
2971
|
-
logical.push(alone);
|
|
2972
|
-
logicalByRawId.set(complete.eventId, alone);
|
|
2973
|
-
const stepId = stepIdOf(complete);
|
|
2974
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
2975
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2976
|
-
}
|
|
2507
|
+
const alone = asLogical(complete);
|
|
2508
|
+
logical.push(alone);
|
|
2509
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
2510
|
+
const stepId = stepIdOf(complete);
|
|
2511
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
2512
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2515
|
+
for (const event of others) {
|
|
2516
|
+
const alone = asLogical(event);
|
|
2517
|
+
logical.push(alone);
|
|
2518
|
+
logicalByRawId.set(event.eventId, alone);
|
|
2519
|
+
const stepId = stepIdOf(event);
|
|
2520
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
2521
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2522
|
+
}
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2525
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
2526
|
+
const normalized = [];
|
|
2527
|
+
for (const event of logical) {
|
|
2528
|
+
const originalParentId = event.parentId;
|
|
2529
|
+
if (!originalParentId) {
|
|
2530
|
+
normalized.push(event);
|
|
2531
|
+
continue;
|
|
2532
|
+
}
|
|
2533
|
+
let nextParent = originalParentId;
|
|
2534
|
+
let remapped = false;
|
|
2535
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
2536
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
2537
|
+
nextParent = viaAbsorbed.eventId;
|
|
2538
|
+
remapped = true;
|
|
2539
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
2540
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
2541
|
+
if (viaStep) {
|
|
2542
|
+
nextParent = viaStep;
|
|
2543
|
+
remapped = true;
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
if (!remapped) {
|
|
2547
|
+
if (originalParentId === event.eventId) {
|
|
2548
|
+
diagnostics.push({
|
|
2549
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
2550
|
+
message: formatProgrammaticDiagnostic(
|
|
2551
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2552
|
+
`Removed self-parent edge on ${event.eventId}.`
|
|
2553
|
+
),
|
|
2554
|
+
eventIds: [event.eventId]
|
|
2555
|
+
});
|
|
2556
|
+
const { parentId: _drop, ...rest } = event;
|
|
2557
|
+
normalized.push({
|
|
2558
|
+
...rest,
|
|
2559
|
+
projection: {
|
|
2560
|
+
...event.projection,
|
|
2561
|
+
parentNormalized: true,
|
|
2562
|
+
originalParentId
|
|
2563
|
+
}
|
|
2564
|
+
});
|
|
2565
|
+
continue;
|
|
2566
|
+
}
|
|
2567
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
2568
|
+
const mapping = event.attributes?.parentMapping;
|
|
2569
|
+
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
2570
|
+
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
2571
|
+
if (!unresolved) {
|
|
2572
|
+
diagnostics.push({
|
|
2573
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
2574
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
2575
|
+
eventIds: [event.eventId]
|
|
2576
|
+
});
|
|
2577
|
+
}
|
|
2578
|
+
}
|
|
2579
|
+
normalized.push(event);
|
|
2580
|
+
continue;
|
|
2581
|
+
}
|
|
2582
|
+
if (nextParent === event.eventId) {
|
|
2583
|
+
diagnostics.push({
|
|
2584
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
2585
|
+
message: formatProgrammaticDiagnostic(
|
|
2586
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2587
|
+
`Removed self-parent edge on ${event.eventId} (was ${originalParentId}).`
|
|
2588
|
+
),
|
|
2589
|
+
eventIds: [event.eventId]
|
|
2590
|
+
});
|
|
2591
|
+
const { parentId: _drop, ...rest } = event;
|
|
2592
|
+
normalized.push({
|
|
2593
|
+
...rest,
|
|
2594
|
+
projection: {
|
|
2595
|
+
...event.projection,
|
|
2596
|
+
parentNormalized: true,
|
|
2597
|
+
originalParentId
|
|
2598
|
+
}
|
|
2599
|
+
});
|
|
2600
|
+
continue;
|
|
2601
|
+
}
|
|
2602
|
+
diagnostics.push({
|
|
2603
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
2604
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
2605
|
+
eventIds: [event.eventId]
|
|
2606
|
+
});
|
|
2607
|
+
normalized.push({
|
|
2608
|
+
...event,
|
|
2609
|
+
parentId: nextParent,
|
|
2610
|
+
projection: {
|
|
2611
|
+
...event.projection,
|
|
2612
|
+
parentNormalized: true,
|
|
2613
|
+
originalParentId
|
|
2614
|
+
}
|
|
2615
|
+
});
|
|
2616
|
+
}
|
|
2617
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
2618
|
+
normalized.sort((a, b) => {
|
|
2619
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
2620
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
2621
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
2622
|
+
});
|
|
2623
|
+
return {
|
|
2624
|
+
logicalEvents: Object.freeze(normalized),
|
|
2625
|
+
diagnostics: Object.freeze(diagnostics)
|
|
2626
|
+
};
|
|
2627
|
+
}
|
|
2628
|
+
function resolveCanonicalToolName(event) {
|
|
2629
|
+
const attrs2 = event.attributes;
|
|
2630
|
+
const direct = pickString(attrs2, ["toolName", "tool"]);
|
|
2631
|
+
if (direct) return direct;
|
|
2632
|
+
const metadata = attrs2?.metadata;
|
|
2633
|
+
if (isRecord6(metadata)) {
|
|
2634
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
2635
|
+
if (nested) return nested;
|
|
2636
|
+
}
|
|
2637
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
2638
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
2639
|
+
}
|
|
2640
|
+
return event.name;
|
|
2641
|
+
}
|
|
2642
|
+
function pickString(record, keys) {
|
|
2643
|
+
if (!record) return void 0;
|
|
2644
|
+
for (const key of keys) {
|
|
2645
|
+
const value = record[key];
|
|
2646
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
2647
|
+
}
|
|
2648
|
+
return void 0;
|
|
2649
|
+
}
|
|
2650
|
+
|
|
2651
|
+
// packages/core/src/checks/derived-failure.ts
|
|
2652
|
+
function isRecord7(value) {
|
|
2653
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2654
|
+
}
|
|
2655
|
+
function pickString2(record, keys) {
|
|
2656
|
+
if (!record) return void 0;
|
|
2657
|
+
for (const key of keys) {
|
|
2658
|
+
const value = record[key];
|
|
2659
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2660
|
+
}
|
|
2661
|
+
return void 0;
|
|
2662
|
+
}
|
|
2663
|
+
function pickNumber(record, keys) {
|
|
2664
|
+
if (!record) return void 0;
|
|
2665
|
+
for (const key of keys) {
|
|
2666
|
+
const value = record[key];
|
|
2667
|
+
if (typeof value === "number" && Number.isFinite(value) && value > 0) {
|
|
2668
|
+
return value;
|
|
2669
|
+
}
|
|
2670
|
+
}
|
|
2671
|
+
return void 0;
|
|
2672
|
+
}
|
|
2673
|
+
function eventMetadata(event) {
|
|
2674
|
+
const attrs2 = isRecord7(event.attributes) ? event.attributes : void 0;
|
|
2675
|
+
const nested = attrs2 !== void 0 && isRecord7(attrs2.metadata) ? attrs2.metadata : void 0;
|
|
2676
|
+
return {
|
|
2677
|
+
...attrs2 ?? {},
|
|
2678
|
+
...nested ?? {}
|
|
2679
|
+
};
|
|
2680
|
+
}
|
|
2681
|
+
function canonicalName(event) {
|
|
2682
|
+
if (event.kind === "TOOL") return resolveCanonicalToolName(event);
|
|
2683
|
+
return event.name;
|
|
2684
|
+
}
|
|
2685
|
+
function linkKeys(event) {
|
|
2686
|
+
const meta = eventMetadata(event);
|
|
2687
|
+
const keys = [];
|
|
2688
|
+
for (const key of ["linkedStepId", "toolCallId", "mcpToolCallId"]) {
|
|
2689
|
+
const value = pickString2(meta, [key]);
|
|
2690
|
+
if (value !== void 0) keys.push(`${key}:${value}`);
|
|
2691
|
+
}
|
|
2692
|
+
const stepId = pickString2(meta, ["stepId"]);
|
|
2693
|
+
if (stepId !== void 0) keys.push(`stepId:${stepId}`);
|
|
2694
|
+
return keys;
|
|
2695
|
+
}
|
|
2696
|
+
function buildRunContexts(logicalEvents) {
|
|
2697
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
2698
|
+
for (const event of logicalEvents) {
|
|
2699
|
+
const existing = byRun.get(event.runId) ?? {
|
|
2700
|
+
runId: event.runId,
|
|
2701
|
+
name: event.name
|
|
2702
|
+
};
|
|
2703
|
+
const meta = eventMetadata(event);
|
|
2704
|
+
if (event.kind === "RUN" || existing.name === event.runId) {
|
|
2705
|
+
existing.name = event.name || existing.name;
|
|
2706
|
+
}
|
|
2707
|
+
if (event.kind === "RUN" && event.status !== void 0 && event.status !== "running") {
|
|
2708
|
+
existing.status = event.status;
|
|
2709
|
+
}
|
|
2710
|
+
existing.retryOf ??= pickString2(meta, ["retryOf"]);
|
|
2711
|
+
existing.attempt ??= pickNumber(meta, ["attempt", "retryAttempt", "retryCount"]);
|
|
2712
|
+
existing.sessionId ??= pickString2(meta, ["sessionId", "conversationId"]);
|
|
2713
|
+
existing.groupId ??= pickString2(meta, ["groupId"]);
|
|
2714
|
+
existing.parentGroupId ??= pickString2(meta, ["parentGroupId"]);
|
|
2715
|
+
existing.fallbackOf ??= pickString2(meta, ["fallbackOf", "fallbackFrom"]);
|
|
2716
|
+
byRun.set(event.runId, existing);
|
|
2717
|
+
}
|
|
2718
|
+
return byRun;
|
|
2719
|
+
}
|
|
2720
|
+
function sameCorrelationScope(a, b) {
|
|
2721
|
+
if (a.sessionId && b.sessionId && a.sessionId === b.sessionId) return true;
|
|
2722
|
+
if (a.groupId && b.groupId && a.groupId === b.groupId) return true;
|
|
2723
|
+
if (a.parentGroupId && b.parentGroupId && a.parentGroupId === b.parentGroupId) {
|
|
2724
|
+
return true;
|
|
2725
|
+
}
|
|
2726
|
+
return false;
|
|
2727
|
+
}
|
|
2728
|
+
function isSuccessful(event) {
|
|
2729
|
+
return event.status === "ok";
|
|
2730
|
+
}
|
|
2731
|
+
function isFailure(event) {
|
|
2732
|
+
return event.status === "error";
|
|
2733
|
+
}
|
|
2734
|
+
function compareEventOrder(a, b) {
|
|
2735
|
+
const byTime = a.timestamp.localeCompare(b.timestamp);
|
|
2736
|
+
if (byTime !== 0) return byTime;
|
|
2737
|
+
return a.eventId.localeCompare(b.eventId);
|
|
2738
|
+
}
|
|
2739
|
+
function collectCandidates(failure, logicalEvents, runs) {
|
|
2740
|
+
const failureRun = runs.get(failure.runId);
|
|
2741
|
+
const failureLinks = new Set(linkKeys(failure));
|
|
2742
|
+
const failureName = canonicalName(failure);
|
|
2743
|
+
const failureAttempt = pickNumber(eventMetadata(failure), ["attempt", "retryAttempt", "retryCount"]) ?? failureRun?.attempt;
|
|
2744
|
+
const candidates = [];
|
|
2745
|
+
for (const event of logicalEvents) {
|
|
2746
|
+
if (event.eventId === failure.eventId) continue;
|
|
2747
|
+
if (event.status === "running") continue;
|
|
2748
|
+
const eventRun = runs.get(event.runId);
|
|
2749
|
+
const eventMeta = eventMetadata(event);
|
|
2750
|
+
const sameRun = event.runId === failure.runId;
|
|
2751
|
+
if (eventRun?.retryOf === failure.runId) {
|
|
2752
|
+
candidates.push({
|
|
2753
|
+
event,
|
|
2754
|
+
basis: "retryOf",
|
|
2755
|
+
confidence: "explicit",
|
|
2756
|
+
viaRunId: event.runId
|
|
2757
|
+
});
|
|
2758
|
+
continue;
|
|
2977
2759
|
}
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2760
|
+
if (eventRun?.fallbackOf === failure.runId || pickString2(eventMeta, ["fallbackOf", "fallbackFrom"]) === failure.runId) {
|
|
2761
|
+
candidates.push({
|
|
2762
|
+
event,
|
|
2763
|
+
basis: "fallbackOf",
|
|
2764
|
+
confidence: "explicit",
|
|
2765
|
+
viaRunId: event.runId
|
|
2766
|
+
});
|
|
2767
|
+
continue;
|
|
2986
2768
|
}
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2769
|
+
if (sameRun && compareEventOrder(failure, event) >= 0) continue;
|
|
2770
|
+
const eventLinks = linkKeys(event);
|
|
2771
|
+
const sharedLink = eventLinks.find((key) => failureLinks.has(key));
|
|
2772
|
+
if (sharedLink !== void 0) {
|
|
2773
|
+
candidates.push({
|
|
2774
|
+
event,
|
|
2775
|
+
basis: sharedLink.split(":")[0] ?? "linkedId",
|
|
2776
|
+
confidence: "explicit"
|
|
2777
|
+
});
|
|
2994
2778
|
continue;
|
|
2995
2779
|
}
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
const
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
}
|
|
2780
|
+
const eventAttempt = pickNumber(eventMeta, ["attempt", "retryAttempt", "retryCount"]) ?? eventRun?.attempt;
|
|
2781
|
+
const sameParent = failure.parentId !== void 0 && event.parentId !== void 0 && failure.parentId === event.parentId;
|
|
2782
|
+
const differentParent = failure.parentId !== void 0 && event.parentId !== void 0 && failure.parentId !== event.parentId;
|
|
2783
|
+
const sessionScoped = failureRun !== void 0 && eventRun !== void 0 && sameCorrelationScope(failureRun, eventRun);
|
|
2784
|
+
if (canonicalName(event) === failureName && failureAttempt !== void 0 && eventAttempt !== void 0 && eventAttempt > failureAttempt && !differentParent && (sameParent || sessionScoped || sameRun)) {
|
|
2785
|
+
candidates.push({
|
|
2786
|
+
event,
|
|
2787
|
+
basis: "attempt-progression",
|
|
2788
|
+
confidence: "correlated",
|
|
2789
|
+
...event.runId !== failure.runId ? { viaRunId: event.runId } : {}
|
|
2790
|
+
});
|
|
3008
2791
|
}
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
`Removed self-parent edge on ${event.eventId}.`
|
|
3016
|
-
),
|
|
3017
|
-
eventIds: [event.eventId]
|
|
3018
|
-
});
|
|
3019
|
-
const { parentId: _drop, ...rest } = event;
|
|
3020
|
-
normalized.push({
|
|
3021
|
-
...rest,
|
|
3022
|
-
projection: {
|
|
3023
|
-
...event.projection,
|
|
3024
|
-
parentNormalized: true,
|
|
3025
|
-
originalParentId
|
|
3026
|
-
}
|
|
3027
|
-
});
|
|
3028
|
-
continue;
|
|
3029
|
-
}
|
|
3030
|
-
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
3031
|
-
const mapping = event.attributes?.parentMapping;
|
|
3032
|
-
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
3033
|
-
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
3034
|
-
if (!unresolved) {
|
|
3035
|
-
diagnostics.push({
|
|
3036
|
-
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
3037
|
-
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
3038
|
-
eventIds: [event.eventId]
|
|
3039
|
-
});
|
|
3040
|
-
}
|
|
3041
|
-
}
|
|
3042
|
-
normalized.push(event);
|
|
3043
|
-
continue;
|
|
2792
|
+
}
|
|
2793
|
+
const byId = /* @__PURE__ */ new Map();
|
|
2794
|
+
for (const candidate of candidates) {
|
|
2795
|
+
const prev = byId.get(candidate.event.eventId);
|
|
2796
|
+
if (!prev || prev.confidence !== "explicit" && candidate.confidence === "explicit") {
|
|
2797
|
+
byId.set(candidate.event.eventId, candidate);
|
|
3044
2798
|
}
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
2799
|
+
}
|
|
2800
|
+
return [...byId.values()].sort((a, b) => compareEventOrder(a.event, b.event));
|
|
2801
|
+
}
|
|
2802
|
+
function classifyFailure(failure, candidates, runs, logicalEvents) {
|
|
2803
|
+
const successful = candidates.filter((c) => isSuccessful(c.event));
|
|
2804
|
+
const unsuccessful = candidates.filter((c) => !isSuccessful(c.event));
|
|
2805
|
+
const retryRunIds = Object.freeze(
|
|
2806
|
+
[...new Set(candidates.map((c) => c.viaRunId).filter((id) => id !== void 0))].sort(
|
|
2807
|
+
(a, b) => a.localeCompare(b)
|
|
2808
|
+
)
|
|
2809
|
+
);
|
|
2810
|
+
if (successful.length > 1) {
|
|
2811
|
+
const distinctRuns = new Set(successful.map((c) => c.event.runId));
|
|
2812
|
+
const distinctParents = new Set(
|
|
2813
|
+
successful.map((c) => c.event.parentId ?? "").filter((id) => id !== "")
|
|
2814
|
+
);
|
|
2815
|
+
if (distinctRuns.size > 1 || distinctParents.size > 1) {
|
|
2816
|
+
return {
|
|
2817
|
+
eventId: failure.eventId,
|
|
2818
|
+
runId: failure.runId,
|
|
2819
|
+
name: failure.name,
|
|
2820
|
+
kind: failure.kind,
|
|
2821
|
+
role: "unknown",
|
|
2822
|
+
confidence: "unknown",
|
|
2823
|
+
basis: Object.freeze(["ambiguous-recovery-candidates"]),
|
|
2824
|
+
recoveryEventIds: Object.freeze(
|
|
2825
|
+
successful.map((c) => c.event.eventId).sort((a, b) => a.localeCompare(b))
|
|
3051
2826
|
),
|
|
3052
|
-
|
|
3053
|
-
}
|
|
3054
|
-
const { parentId: _drop, ...rest } = event;
|
|
3055
|
-
normalized.push({
|
|
3056
|
-
...rest,
|
|
3057
|
-
projection: {
|
|
3058
|
-
...event.projection,
|
|
3059
|
-
parentNormalized: true,
|
|
3060
|
-
originalParentId
|
|
3061
|
-
}
|
|
3062
|
-
});
|
|
3063
|
-
continue;
|
|
2827
|
+
retryRunIds
|
|
2828
|
+
};
|
|
3064
2829
|
}
|
|
3065
|
-
diagnostics.push({
|
|
3066
|
-
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
3067
|
-
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
3068
|
-
eventIds: [event.eventId]
|
|
3069
|
-
});
|
|
3070
|
-
normalized.push({
|
|
3071
|
-
...event,
|
|
3072
|
-
parentId: nextParent,
|
|
3073
|
-
projection: {
|
|
3074
|
-
...event.projection,
|
|
3075
|
-
parentNormalized: true,
|
|
3076
|
-
originalParentId
|
|
3077
|
-
}
|
|
3078
|
-
});
|
|
3079
2830
|
}
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
2831
|
+
if (successful.length >= 1) {
|
|
2832
|
+
const best = successful[0];
|
|
2833
|
+
return {
|
|
2834
|
+
eventId: failure.eventId,
|
|
2835
|
+
runId: failure.runId,
|
|
2836
|
+
name: failure.name,
|
|
2837
|
+
kind: failure.kind,
|
|
2838
|
+
role: "recovered",
|
|
2839
|
+
confidence: best.confidence,
|
|
2840
|
+
basis: Object.freeze([best.basis]),
|
|
2841
|
+
recoveryEventIds: Object.freeze(
|
|
2842
|
+
successful.map((c) => c.event.eventId).sort((a, b) => a.localeCompare(b))
|
|
2843
|
+
),
|
|
2844
|
+
retryRunIds
|
|
2845
|
+
};
|
|
2846
|
+
}
|
|
2847
|
+
if (candidates.length > 0) {
|
|
2848
|
+
const best = candidates[0];
|
|
2849
|
+
return {
|
|
2850
|
+
eventId: failure.eventId,
|
|
2851
|
+
runId: failure.runId,
|
|
2852
|
+
name: failure.name,
|
|
2853
|
+
kind: failure.kind,
|
|
2854
|
+
role: "transient",
|
|
2855
|
+
confidence: best.confidence,
|
|
2856
|
+
basis: Object.freeze([
|
|
2857
|
+
best.basis,
|
|
2858
|
+
unsuccessful.some((c) => c.event.status === void 0) ? "retry-incomplete" : "retry-without-success"
|
|
2859
|
+
]),
|
|
2860
|
+
recoveryEventIds: Object.freeze([]),
|
|
2861
|
+
retryRunIds
|
|
2862
|
+
};
|
|
2863
|
+
}
|
|
2864
|
+
const failureMeta = eventMetadata(failure);
|
|
2865
|
+
const declaredSuccessor = pickString2(failureMeta, [
|
|
2866
|
+
"retriedBy",
|
|
2867
|
+
"nextRetryRunId",
|
|
2868
|
+
"retryRunId"
|
|
2869
|
+
]);
|
|
2870
|
+
if (declaredSuccessor !== void 0 && !runs.has(declaredSuccessor)) {
|
|
2871
|
+
return {
|
|
2872
|
+
eventId: failure.eventId,
|
|
2873
|
+
runId: failure.runId,
|
|
2874
|
+
name: failure.name,
|
|
2875
|
+
kind: failure.kind,
|
|
2876
|
+
role: "transient",
|
|
2877
|
+
confidence: "explicit",
|
|
2878
|
+
basis: Object.freeze(["retry-declared", "retry-run-missing"]),
|
|
2879
|
+
recoveryEventIds: Object.freeze([]),
|
|
2880
|
+
retryRunIds: Object.freeze([declaredSuccessor])
|
|
2881
|
+
};
|
|
2882
|
+
}
|
|
2883
|
+
for (const run of runs.values()) {
|
|
2884
|
+
if (run.retryOf === failure.runId) {
|
|
2885
|
+
return {
|
|
2886
|
+
eventId: failure.eventId,
|
|
2887
|
+
runId: failure.runId,
|
|
2888
|
+
name: failure.name,
|
|
2889
|
+
kind: failure.kind,
|
|
2890
|
+
role: "transient",
|
|
2891
|
+
confidence: "explicit",
|
|
2892
|
+
basis: Object.freeze(["retryOf", "retry-run-missing-or-empty"]),
|
|
2893
|
+
recoveryEventIds: Object.freeze([]),
|
|
2894
|
+
retryRunIds: Object.freeze([run.runId])
|
|
2895
|
+
};
|
|
2896
|
+
}
|
|
2897
|
+
}
|
|
2898
|
+
const failureRun = runs.get(failure.runId);
|
|
2899
|
+
const hasSuccessorDeclared = [...runs.values()].some((run) => run.retryOf === failure.runId);
|
|
2900
|
+
const isFinalInChain = failureRun !== void 0 && !hasSuccessorDeclared && (failureRun.retryOf !== void 0 || failureRun.attempt !== void 0 && failureRun.attempt > 1 || pickNumber(eventMetadata(failure), ["attempt"]) !== void 0);
|
|
2901
|
+
if (isFinalInChain && failureRun?.status === "error" && !logicalEvents.some(
|
|
2902
|
+
(event) => event.runId === failure.runId && event.eventId !== failure.eventId && isSuccessful(event) && canonicalName(event) === canonicalName(failure)
|
|
2903
|
+
)) {
|
|
2904
|
+
return {
|
|
2905
|
+
eventId: failure.eventId,
|
|
2906
|
+
runId: failure.runId,
|
|
2907
|
+
name: failure.name,
|
|
2908
|
+
kind: failure.kind,
|
|
2909
|
+
role: "terminal",
|
|
2910
|
+
confidence: failureRun.retryOf !== void 0 ? "explicit" : "correlated",
|
|
2911
|
+
basis: Object.freeze(["final-retry-chain-member", "enclosing-run-error"]),
|
|
2912
|
+
recoveryEventIds: Object.freeze([]),
|
|
2913
|
+
retryRunIds: Object.freeze(
|
|
2914
|
+
failureRun.retryOf !== void 0 ? [failureRun.retryOf] : []
|
|
2915
|
+
)
|
|
2916
|
+
};
|
|
2917
|
+
}
|
|
3086
2918
|
return {
|
|
3087
|
-
|
|
3088
|
-
|
|
2919
|
+
eventId: failure.eventId,
|
|
2920
|
+
runId: failure.runId,
|
|
2921
|
+
name: failure.name,
|
|
2922
|
+
kind: failure.kind,
|
|
2923
|
+
role: "unknown",
|
|
2924
|
+
confidence: "unknown",
|
|
2925
|
+
basis: Object.freeze(["no-explicit-or-correlated-recovery"]),
|
|
2926
|
+
recoveryEventIds: Object.freeze([]),
|
|
2927
|
+
retryRunIds: Object.freeze([])
|
|
3089
2928
|
};
|
|
3090
2929
|
}
|
|
3091
|
-
function
|
|
3092
|
-
const
|
|
3093
|
-
const
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
function pickString(record, keys) {
|
|
3106
|
-
if (!record) return void 0;
|
|
3107
|
-
for (const key of keys) {
|
|
3108
|
-
const value = record[key];
|
|
3109
|
-
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
3110
|
-
}
|
|
3111
|
-
return void 0;
|
|
3112
|
-
}
|
|
3113
|
-
|
|
3114
|
-
// packages/core/src/checks/derived-failure.ts
|
|
3115
|
-
function isRecord7(value) {
|
|
3116
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3117
|
-
}
|
|
3118
|
-
function pickString2(record, keys) {
|
|
3119
|
-
if (!record) return void 0;
|
|
3120
|
-
for (const key of keys) {
|
|
3121
|
-
const value = record[key];
|
|
3122
|
-
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2930
|
+
function deriveFailureFacts(logicalEvents) {
|
|
2931
|
+
const runs = buildRunContexts(logicalEvents);
|
|
2932
|
+
const failures = logicalEvents.filter((event) => isFailure(event)).sort(compareEventOrder);
|
|
2933
|
+
const failureFacts = failures.map(
|
|
2934
|
+
(failure) => classifyFailure(failure, collectCandidates(failure, logicalEvents, runs), runs, logicalEvents)
|
|
2935
|
+
);
|
|
2936
|
+
const byRole = /* @__PURE__ */ new Map([
|
|
2937
|
+
["transient", []],
|
|
2938
|
+
["recovered", []],
|
|
2939
|
+
["terminal", []],
|
|
2940
|
+
["unknown", []]
|
|
2941
|
+
]);
|
|
2942
|
+
for (const fact of failureFacts) {
|
|
2943
|
+
byRole.get(fact.role).push(fact);
|
|
3123
2944
|
}
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
2945
|
+
for (const [role, list] of byRole) {
|
|
2946
|
+
byRole.set(
|
|
2947
|
+
role,
|
|
2948
|
+
Object.freeze(
|
|
2949
|
+
[...list].sort((a, b) => {
|
|
2950
|
+
const byRun = a.runId.localeCompare(b.runId);
|
|
2951
|
+
if (byRun !== 0) return byRun;
|
|
2952
|
+
return a.eventId.localeCompare(b.eventId);
|
|
2953
|
+
})
|
|
2954
|
+
)
|
|
2955
|
+
);
|
|
3133
2956
|
}
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
2957
|
+
const failureRoleCounts = {
|
|
2958
|
+
transient: byRole.get("transient").length,
|
|
2959
|
+
recovered: byRole.get("recovered").length,
|
|
2960
|
+
terminal: byRole.get("terminal").length,
|
|
2961
|
+
unknown: byRole.get("unknown").length
|
|
2962
|
+
};
|
|
3139
2963
|
return {
|
|
3140
|
-
|
|
3141
|
-
|
|
2964
|
+
failureFacts: Object.freeze(failureFacts),
|
|
2965
|
+
failuresByRole: byRole,
|
|
2966
|
+
failureRoleCounts
|
|
3142
2967
|
};
|
|
3143
2968
|
}
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
2969
|
+
|
|
2970
|
+
// packages/core/src/checks/relationship-facts.ts
|
|
2971
|
+
function attrs(event) {
|
|
2972
|
+
return event.attributes && typeof event.attributes === "object" ? event.attributes : {};
|
|
3147
2973
|
}
|
|
3148
|
-
function
|
|
3149
|
-
const
|
|
3150
|
-
|
|
3151
|
-
for (const key of ["linkedStepId", "toolCallId", "mcpToolCallId"]) {
|
|
3152
|
-
const value = pickString2(meta, [key]);
|
|
3153
|
-
if (value !== void 0) keys.push(`${key}:${value}`);
|
|
3154
|
-
}
|
|
3155
|
-
const stepId = pickString2(meta, ["stepId"]);
|
|
3156
|
-
if (stepId !== void 0) keys.push(`stepId:${stepId}`);
|
|
3157
|
-
return keys;
|
|
2974
|
+
function stringAttr(record, key) {
|
|
2975
|
+
const value = record[key];
|
|
2976
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
3158
2977
|
}
|
|
3159
|
-
function
|
|
3160
|
-
const
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
}
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
2978
|
+
function deriveRelationshipFacts(events) {
|
|
2979
|
+
const relationships = [];
|
|
2980
|
+
const diagnostics = [];
|
|
2981
|
+
const byId = new Map(events.map((event) => [event.eventId, event]));
|
|
2982
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2983
|
+
const push = (edge) => {
|
|
2984
|
+
const key = `${edge.type}:${edge.fromEventId}:${edge.toEventId ?? ""}:${edge.externalRef ?? ""}`;
|
|
2985
|
+
if (seen.has(key)) return;
|
|
2986
|
+
seen.add(key);
|
|
2987
|
+
relationships.push(edge);
|
|
2988
|
+
};
|
|
2989
|
+
for (const event of events) {
|
|
2990
|
+
if (event.parentId) {
|
|
2991
|
+
push({
|
|
2992
|
+
type: "parent-child",
|
|
2993
|
+
fromEventId: event.parentId,
|
|
2994
|
+
toEventId: event.eventId,
|
|
2995
|
+
confidence: byId.has(event.parentId) ? "explicit" : "unknown",
|
|
2996
|
+
basis: ["persisted.parentId"]
|
|
2997
|
+
});
|
|
2998
|
+
if (!byId.has(event.parentId)) {
|
|
2999
|
+
diagnostics.push({
|
|
3000
|
+
code: "AI_RELATIONSHIP_PARENT_MISSING",
|
|
3001
|
+
message: `parentId ${event.parentId} is not present in the event set.`,
|
|
3002
|
+
eventId: event.eventId
|
|
3003
|
+
});
|
|
3004
|
+
}
|
|
3169
3005
|
}
|
|
3170
|
-
|
|
3171
|
-
|
|
3006
|
+
const bag = attrs(event);
|
|
3007
|
+
const meta = extractSessionWorkflowMetadata(bag) ?? {};
|
|
3008
|
+
const nested = bag.metadata && typeof bag.metadata === "object" ? extractSessionWorkflowMetadata(bag.metadata) : void 0;
|
|
3009
|
+
const workflow = { ...meta, ...nested };
|
|
3010
|
+
if (workflow.retryOf) {
|
|
3011
|
+
const target = events.find((candidate) => candidate.runId === workflow.retryOf);
|
|
3012
|
+
push({
|
|
3013
|
+
type: "retry-of",
|
|
3014
|
+
fromEventId: event.eventId,
|
|
3015
|
+
...target ? { toEventId: target.eventId } : {},
|
|
3016
|
+
externalRef: workflow.retryOf,
|
|
3017
|
+
confidence: target ? "explicit" : "correlated",
|
|
3018
|
+
basis: ["attributes.retryOf"]
|
|
3019
|
+
});
|
|
3020
|
+
}
|
|
3021
|
+
const attemptOf = stringAttr(bag, "attemptOf") ?? stringAttr(bag, "operationId");
|
|
3022
|
+
if (attemptOf && workflow.attempt !== void 0) {
|
|
3023
|
+
push({
|
|
3024
|
+
type: "attempt-of",
|
|
3025
|
+
fromEventId: event.eventId,
|
|
3026
|
+
externalRef: attemptOf,
|
|
3027
|
+
confidence: "explicit",
|
|
3028
|
+
basis: workflow.attempt !== void 0 ? ["attributes.attempt", "attributes.operationId"] : ["attributes.operationId"]
|
|
3029
|
+
});
|
|
3030
|
+
}
|
|
3031
|
+
const fallbackOf = stringAttr(bag, "fallbackOf");
|
|
3032
|
+
if (fallbackOf) {
|
|
3033
|
+
push({
|
|
3034
|
+
type: "fallback-of",
|
|
3035
|
+
fromEventId: event.eventId,
|
|
3036
|
+
externalRef: fallbackOf,
|
|
3037
|
+
confidence: "explicit",
|
|
3038
|
+
basis: ["attributes.fallbackOf"]
|
|
3039
|
+
});
|
|
3040
|
+
}
|
|
3041
|
+
const remediationOf = stringAttr(bag, "remediationOf");
|
|
3042
|
+
if (remediationOf) {
|
|
3043
|
+
push({
|
|
3044
|
+
type: "remediation-of",
|
|
3045
|
+
fromEventId: event.eventId,
|
|
3046
|
+
externalRef: remediationOf,
|
|
3047
|
+
confidence: "explicit",
|
|
3048
|
+
basis: ["attributes.remediationOf"]
|
|
3049
|
+
});
|
|
3050
|
+
}
|
|
3051
|
+
const evidenceFor = stringAttr(bag, "evidenceFor");
|
|
3052
|
+
if (evidenceFor) {
|
|
3053
|
+
push({
|
|
3054
|
+
type: "evidence-for",
|
|
3055
|
+
fromEventId: event.eventId,
|
|
3056
|
+
...byId.has(evidenceFor) ? { toEventId: evidenceFor } : { externalRef: evidenceFor },
|
|
3057
|
+
confidence: byId.has(evidenceFor) ? "explicit" : "correlated",
|
|
3058
|
+
basis: ["attributes.evidenceFor"]
|
|
3059
|
+
});
|
|
3060
|
+
}
|
|
3061
|
+
const acceptedBy = stringAttr(bag, "acceptedBy");
|
|
3062
|
+
if (acceptedBy) {
|
|
3063
|
+
push({
|
|
3064
|
+
type: "accepted-by",
|
|
3065
|
+
fromEventId: event.eventId,
|
|
3066
|
+
...byId.has(acceptedBy) ? { toEventId: acceptedBy } : { externalRef: acceptedBy },
|
|
3067
|
+
confidence: byId.has(acceptedBy) ? "explicit" : "correlated",
|
|
3068
|
+
basis: ["attributes.acceptedBy"]
|
|
3069
|
+
});
|
|
3070
|
+
}
|
|
3071
|
+
const supersedes = stringAttr(bag, "supersedes");
|
|
3072
|
+
if (supersedes) {
|
|
3073
|
+
push({
|
|
3074
|
+
type: "supersedes",
|
|
3075
|
+
fromEventId: event.eventId,
|
|
3076
|
+
...byId.has(supersedes) ? { toEventId: supersedes } : { externalRef: supersedes },
|
|
3077
|
+
confidence: byId.has(supersedes) ? "explicit" : "correlated",
|
|
3078
|
+
basis: ["attributes.supersedes"]
|
|
3079
|
+
});
|
|
3080
|
+
}
|
|
3081
|
+
const sourceLineage = stringAttr(bag, "sourceLineage") ?? stringAttr(bag, "sourceEventId");
|
|
3082
|
+
if (sourceLineage) {
|
|
3083
|
+
push({
|
|
3084
|
+
type: "source-lineage",
|
|
3085
|
+
fromEventId: event.eventId,
|
|
3086
|
+
externalRef: sourceLineage,
|
|
3087
|
+
confidence: "explicit",
|
|
3088
|
+
basis: ["attributes.sourceLineage"]
|
|
3089
|
+
});
|
|
3090
|
+
}
|
|
3091
|
+
const unsupported = stringAttr(bag, "relationshipType");
|
|
3092
|
+
if (unsupported && unsupported !== "parent-child" && ![
|
|
3093
|
+
"source-lineage",
|
|
3094
|
+
"attempt-of",
|
|
3095
|
+
"retry-of",
|
|
3096
|
+
"fallback-of",
|
|
3097
|
+
"remediation-of",
|
|
3098
|
+
"evidence-for",
|
|
3099
|
+
"accepted-by",
|
|
3100
|
+
"supersedes"
|
|
3101
|
+
].includes(unsupported)) {
|
|
3102
|
+
diagnostics.push({
|
|
3103
|
+
code: "AI_RELATIONSHIP_UNSUPPORTED_TYPE",
|
|
3104
|
+
message: `Unsupported relationshipType ${unsupported} was reported without flattening.`,
|
|
3105
|
+
eventId: event.eventId
|
|
3106
|
+
});
|
|
3172
3107
|
}
|
|
3173
|
-
existing.retryOf ??= pickString2(meta, ["retryOf"]);
|
|
3174
|
-
existing.attempt ??= pickNumber(meta, ["attempt", "retryAttempt", "retryCount"]);
|
|
3175
|
-
existing.sessionId ??= pickString2(meta, ["sessionId", "conversationId"]);
|
|
3176
|
-
existing.groupId ??= pickString2(meta, ["groupId"]);
|
|
3177
|
-
existing.parentGroupId ??= pickString2(meta, ["parentGroupId"]);
|
|
3178
|
-
existing.fallbackOf ??= pickString2(meta, ["fallbackOf", "fallbackFrom"]);
|
|
3179
|
-
byRun.set(event.runId, existing);
|
|
3180
3108
|
}
|
|
3181
|
-
return
|
|
3109
|
+
return {
|
|
3110
|
+
relationships: Object.freeze(relationships),
|
|
3111
|
+
diagnostics: Object.freeze(diagnostics)
|
|
3112
|
+
};
|
|
3182
3113
|
}
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3114
|
+
|
|
3115
|
+
// packages/core/src/checks/trace-facts.ts
|
|
3116
|
+
function summarizeSemanticParity(events) {
|
|
3117
|
+
const projection = projectLogicalEvents(events);
|
|
3118
|
+
const logical = projection.logicalEvents;
|
|
3119
|
+
const finishedTools = logical.filter(
|
|
3120
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
3121
|
+
);
|
|
3122
|
+
const finishedToolNames = Object.freeze(
|
|
3123
|
+
finishedTools.map((event) => resolveCanonicalToolName(event)).sort((a, b) => a.localeCompare(b))
|
|
3124
|
+
);
|
|
3125
|
+
const derived = deriveFailureFacts(logical);
|
|
3126
|
+
return {
|
|
3127
|
+
rawEventCount: events.length,
|
|
3128
|
+
logicalEventCount: logical.length,
|
|
3129
|
+
runningLogicalCount: logical.filter((event) => event.status === "running").length,
|
|
3130
|
+
finishedToolNames,
|
|
3131
|
+
finishedToolCount: finishedTools.length,
|
|
3132
|
+
pairedCount: logical.filter((event) => event.projection.paired).length,
|
|
3133
|
+
parentRemapCount: projection.diagnostics.filter(
|
|
3134
|
+
(item) => item.code === "AI_LOGICAL_PARENT_REMAPPED"
|
|
3135
|
+
).length,
|
|
3136
|
+
diagnostics: projection.diagnostics,
|
|
3137
|
+
failureRoleCounts: derived.failureRoleCounts
|
|
3138
|
+
};
|
|
3139
|
+
}
|
|
3140
|
+
var TRACE_FACTS_INPUT_NOT_NORMALIZED = formatProgrammaticDiagnostic(
|
|
3141
|
+
"AI_TRACE_FACTS_INPUT_NOT_NORMALIZED"
|
|
3142
|
+
);
|
|
3143
|
+
function isTraceReadResult(input) {
|
|
3144
|
+
if (typeof input !== "object" || input === null || Array.isArray(input)) {
|
|
3145
|
+
return false;
|
|
3188
3146
|
}
|
|
3189
|
-
|
|
3147
|
+
const record = input;
|
|
3148
|
+
return Array.isArray(record.events) && Array.isArray(record.runs) && typeof record.format === "string" && Array.isArray(record.warnings);
|
|
3190
3149
|
}
|
|
3191
|
-
function
|
|
3192
|
-
|
|
3150
|
+
function looksLikeRawV01TraceEvents(input) {
|
|
3151
|
+
if (!Array.isArray(input) || input.length === 0) return false;
|
|
3152
|
+
const first = input[0];
|
|
3153
|
+
if (typeof first !== "object" || first === null) return false;
|
|
3154
|
+
const row = first;
|
|
3155
|
+
return typeof row.event === "string" && (row.schemaVersion === "0.1" || row.eventId === void 0);
|
|
3193
3156
|
}
|
|
3194
|
-
function
|
|
3195
|
-
|
|
3157
|
+
function isPersistedInspectEventArray(input) {
|
|
3158
|
+
if (!Array.isArray(input)) return false;
|
|
3159
|
+
if (input.length === 0) return true;
|
|
3160
|
+
const first = input[0];
|
|
3161
|
+
if (typeof first !== "object" || first === null) return false;
|
|
3162
|
+
const row = first;
|
|
3163
|
+
return typeof row.eventId === "string" && (row.schemaVersion === "0.2" || row.schemaVersion === "1.0" || row.schemaVersion === "0.1") && typeof row.event !== "string";
|
|
3196
3164
|
}
|
|
3197
|
-
function
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3165
|
+
function resolveTraceFactsEvents(input) {
|
|
3166
|
+
if (isTraceReadResult(input)) {
|
|
3167
|
+
return input.events;
|
|
3168
|
+
}
|
|
3169
|
+
if (looksLikeRawV01TraceEvents(input)) {
|
|
3170
|
+
throw new TypeError(TRACE_FACTS_INPUT_NOT_NORMALIZED);
|
|
3171
|
+
}
|
|
3172
|
+
if (isPersistedInspectEventArray(input)) {
|
|
3173
|
+
return input;
|
|
3174
|
+
}
|
|
3175
|
+
throw new TypeError(TRACE_FACTS_INPUT_NOT_NORMALIZED);
|
|
3201
3176
|
}
|
|
3202
|
-
function
|
|
3203
|
-
const
|
|
3204
|
-
const
|
|
3205
|
-
const
|
|
3206
|
-
const
|
|
3207
|
-
const
|
|
3208
|
-
for (const event of logicalEvents) {
|
|
3209
|
-
if (event.
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
if (eventRun?.retryOf === failure.runId) {
|
|
3215
|
-
candidates.push({
|
|
3216
|
-
event,
|
|
3217
|
-
basis: "retryOf",
|
|
3218
|
-
confidence: "explicit",
|
|
3219
|
-
viaRunId: event.runId
|
|
3220
|
-
});
|
|
3221
|
-
continue;
|
|
3222
|
-
}
|
|
3223
|
-
if (eventRun?.fallbackOf === failure.runId || pickString2(eventMeta, ["fallbackOf", "fallbackFrom"]) === failure.runId) {
|
|
3224
|
-
candidates.push({
|
|
3225
|
-
event,
|
|
3226
|
-
basis: "fallbackOf",
|
|
3227
|
-
confidence: "explicit",
|
|
3228
|
-
viaRunId: event.runId
|
|
3229
|
-
});
|
|
3230
|
-
continue;
|
|
3177
|
+
function buildTraceFacts(input) {
|
|
3178
|
+
const events = resolveTraceFactsEvents(input);
|
|
3179
|
+
const projection = projectLogicalEvents(events);
|
|
3180
|
+
const toolsByName = /* @__PURE__ */ new Map();
|
|
3181
|
+
const llmEvents = [];
|
|
3182
|
+
const outcomeEvents = [];
|
|
3183
|
+
for (const event of projection.logicalEvents) {
|
|
3184
|
+
if (event.kind === "TOOL" && event.status !== "running") {
|
|
3185
|
+
const name = resolveCanonicalToolName(event);
|
|
3186
|
+
const list = toolsByName.get(name) ?? [];
|
|
3187
|
+
list.push(event);
|
|
3188
|
+
toolsByName.set(name, list);
|
|
3231
3189
|
}
|
|
3232
|
-
if (
|
|
3233
|
-
|
|
3234
|
-
const sharedLink = eventLinks.find((key) => failureLinks.has(key));
|
|
3235
|
-
if (sharedLink !== void 0) {
|
|
3236
|
-
candidates.push({
|
|
3237
|
-
event,
|
|
3238
|
-
basis: sharedLink.split(":")[0] ?? "linkedId",
|
|
3239
|
-
confidence: "explicit"
|
|
3240
|
-
});
|
|
3241
|
-
continue;
|
|
3190
|
+
if (event.kind === "LLM" && event.status !== "running") {
|
|
3191
|
+
llmEvents.push(event);
|
|
3242
3192
|
}
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
const differentParent = failure.parentId !== void 0 && event.parentId !== void 0 && failure.parentId !== event.parentId;
|
|
3246
|
-
const sessionScoped = failureRun !== void 0 && eventRun !== void 0 && sameCorrelationScope(failureRun, eventRun);
|
|
3247
|
-
if (canonicalName(event) === failureName && failureAttempt !== void 0 && eventAttempt !== void 0 && eventAttempt > failureAttempt && !differentParent && (sameParent || sessionScoped || sameRun)) {
|
|
3248
|
-
candidates.push({
|
|
3249
|
-
event,
|
|
3250
|
-
basis: "attempt-progression",
|
|
3251
|
-
confidence: "correlated",
|
|
3252
|
-
...event.runId !== failure.runId ? { viaRunId: event.runId } : {}
|
|
3253
|
-
});
|
|
3193
|
+
if (event.kind === "OUTCOME") {
|
|
3194
|
+
outcomeEvents.push(event);
|
|
3254
3195
|
}
|
|
3255
3196
|
}
|
|
3256
|
-
const
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3197
|
+
for (const [name, list] of [...toolsByName.entries()]) {
|
|
3198
|
+
toolsByName.set(name, Object.freeze([...list]));
|
|
3199
|
+
}
|
|
3200
|
+
const derived = deriveFailureFacts(projection.logicalEvents);
|
|
3201
|
+
const relationship = deriveRelationshipFacts(events);
|
|
3202
|
+
const summary = summarizeSemanticParity(events);
|
|
3203
|
+
return {
|
|
3204
|
+
rawEvents: Object.freeze([...events]),
|
|
3205
|
+
logicalEvents: projection.logicalEvents,
|
|
3206
|
+
diagnostics: projection.diagnostics,
|
|
3207
|
+
toolsByName,
|
|
3208
|
+
llmEvents: Object.freeze(llmEvents),
|
|
3209
|
+
outcomeEvents: Object.freeze(outcomeEvents),
|
|
3210
|
+
summary: {
|
|
3211
|
+
...summary,
|
|
3212
|
+
failureRoleCounts: derived.failureRoleCounts
|
|
3213
|
+
},
|
|
3214
|
+
failureFacts: derived.failureFacts,
|
|
3215
|
+
failuresByRole: derived.failuresByRole,
|
|
3216
|
+
relationships: relationship.relationships,
|
|
3217
|
+
relationshipDiagnostics: relationship.diagnostics
|
|
3218
|
+
};
|
|
3219
|
+
}
|
|
3220
|
+
|
|
3221
|
+
// packages/core/src/checks/index.ts
|
|
3222
|
+
var SEVERITY_RANK = {
|
|
3223
|
+
error: 0,
|
|
3224
|
+
warning: 1,
|
|
3225
|
+
info: 2
|
|
3226
|
+
};
|
|
3227
|
+
var STATUS_RANK = {
|
|
3228
|
+
fail: 0,
|
|
3229
|
+
warning: 1,
|
|
3230
|
+
pass: 2
|
|
3231
|
+
};
|
|
3232
|
+
var DEFAULT_SENSITIVE_KEYS = DEFAULT_CREDENTIAL_SENSITIVE_KEYS;
|
|
3233
|
+
var DEFAULT_RAW_CONTENT_KEYS = [
|
|
3234
|
+
"body",
|
|
3235
|
+
"headers",
|
|
3236
|
+
"input",
|
|
3237
|
+
"messages",
|
|
3238
|
+
"output",
|
|
3239
|
+
"payload",
|
|
3240
|
+
"prompt",
|
|
3241
|
+
"requestbody",
|
|
3242
|
+
"request_body",
|
|
3243
|
+
"responsebody",
|
|
3244
|
+
"response_body",
|
|
3245
|
+
"rawprompt",
|
|
3246
|
+
"raw_prompt",
|
|
3247
|
+
"rawoutput",
|
|
3248
|
+
"raw_output",
|
|
3249
|
+
"toolinput",
|
|
3250
|
+
"tool_input",
|
|
3251
|
+
"tooloutput",
|
|
3252
|
+
"tool_output",
|
|
3253
|
+
// Framework / agent metadata that carries user or task text
|
|
3254
|
+
"currenttask",
|
|
3255
|
+
"current_task",
|
|
3256
|
+
"task",
|
|
3257
|
+
"userinput",
|
|
3258
|
+
"user_input",
|
|
3259
|
+
"requesttext",
|
|
3260
|
+
"request_text",
|
|
3261
|
+
"conversationtext",
|
|
3262
|
+
"conversation_text"
|
|
3263
|
+
];
|
|
3264
|
+
var DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES = [
|
|
3265
|
+
"tokenUsage",
|
|
3266
|
+
"usage",
|
|
3267
|
+
"tokens"
|
|
3268
|
+
];
|
|
3269
|
+
var SAFE_USAGE_LEAF_KEYS = /* @__PURE__ */ new Set([
|
|
3270
|
+
"input",
|
|
3271
|
+
"output",
|
|
3272
|
+
"total",
|
|
3273
|
+
"cached",
|
|
3274
|
+
"input_tokens",
|
|
3275
|
+
"inputtokens",
|
|
3276
|
+
"output_tokens",
|
|
3277
|
+
"outputtokens",
|
|
3278
|
+
"total_tokens",
|
|
3279
|
+
"totaltokens",
|
|
3280
|
+
"prompt_tokens",
|
|
3281
|
+
"prompttokens",
|
|
3282
|
+
"completion_tokens",
|
|
3283
|
+
"completiontokens"
|
|
3284
|
+
]);
|
|
3285
|
+
var DEFAULT_SECRET_PATTERNS = [
|
|
3286
|
+
{ id: "bearer-token", pattern: /Bearer\s+[A-Za-z0-9._~+/-]{12,}=*/ },
|
|
3287
|
+
{ id: "openai-key", pattern: /sk-[A-Za-z0-9_-]{16,}/ },
|
|
3288
|
+
{ id: "aws-access-key", pattern: /AKIA[0-9A-Z]{16}/ },
|
|
3289
|
+
{ id: "github-token", pattern: /gh[opsu]_[A-Za-z0-9_]{20,}/ },
|
|
3290
|
+
// Keep in sync with packages/redact/src/key-value-secret.ts (KEY_VALUE_SECRET_PATTERN_SOURCE).
|
|
3291
|
+
{
|
|
3292
|
+
id: "key-value-secret",
|
|
3293
|
+
pattern: /\b(?:api[_-]?key|internal[_-]?token|access[_-]?token|auth[_-]?token|password|secret|token)=([^\s"'\\]{8,})/i
|
|
3294
|
+
}
|
|
3295
|
+
];
|
|
3296
|
+
function compareStrings(a, b) {
|
|
3297
|
+
return (a ?? "").localeCompare(b ?? "");
|
|
3298
|
+
}
|
|
3299
|
+
function diagnostic(code, message, ruleId) {
|
|
3300
|
+
return {
|
|
3301
|
+
code,
|
|
3302
|
+
message,
|
|
3303
|
+
severity: "error",
|
|
3304
|
+
...ruleId ? { ruleId } : {}
|
|
3305
|
+
};
|
|
3306
|
+
}
|
|
3307
|
+
function emptySummary() {
|
|
3308
|
+
return {
|
|
3309
|
+
passed: 0,
|
|
3310
|
+
failed: 0,
|
|
3311
|
+
warnings: 0,
|
|
3312
|
+
errors: 0,
|
|
3313
|
+
rulesEvaluated: 0
|
|
3314
|
+
};
|
|
3315
|
+
}
|
|
3316
|
+
function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
3317
|
+
return {
|
|
3318
|
+
ok: false,
|
|
3319
|
+
status: "error",
|
|
3320
|
+
format: input.read.format,
|
|
3321
|
+
...selectedRun ? { runId: selectedRun.runId } : {},
|
|
3322
|
+
summary: {
|
|
3323
|
+
...emptySummary(),
|
|
3324
|
+
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
3325
|
+
rulesEvaluated: ruleExecutions.length
|
|
3326
|
+
},
|
|
3327
|
+
findings: [],
|
|
3328
|
+
diagnostics: [...diagnostics],
|
|
3329
|
+
ruleExecutions: [...ruleExecutions]
|
|
3330
|
+
};
|
|
3331
|
+
}
|
|
3332
|
+
function flattenNodes(nodes) {
|
|
3333
|
+
return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
|
|
3334
|
+
}
|
|
3335
|
+
function buildFacts(input, selectedRun) {
|
|
3336
|
+
const scopedRuns = selectedRun ? [selectedRun] : input.read.runs;
|
|
3337
|
+
const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
|
|
3338
|
+
const scopedEvents = selectedRun === void 0 ? input.read.events : input.read.events.filter((event) => scopedRunIds.has(event.runId));
|
|
3339
|
+
const nodes = flattenNodes(scopedRuns.flatMap((run) => run.children));
|
|
3340
|
+
const nodesByEventId = /* @__PURE__ */ new Map();
|
|
3341
|
+
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
3342
|
+
for (const node of nodes) {
|
|
3343
|
+
nodesByEventId.set(node.event.eventId, node);
|
|
3344
|
+
const parentId = node.event.parentId;
|
|
3345
|
+
if (parentId) {
|
|
3346
|
+
const children = childrenByParentId.get(parentId) ?? [];
|
|
3347
|
+
children.push(node);
|
|
3348
|
+
childrenByParentId.set(parentId, children);
|
|
3261
3349
|
}
|
|
3262
3350
|
}
|
|
3263
|
-
|
|
3351
|
+
const projection = projectLogicalEvents(scopedEvents);
|
|
3352
|
+
return {
|
|
3353
|
+
format: input.read.format,
|
|
3354
|
+
runs: Object.freeze([...input.read.runs]),
|
|
3355
|
+
events: Object.freeze([...scopedEvents]),
|
|
3356
|
+
logicalEvents: projection.logicalEvents,
|
|
3357
|
+
logicalProjectionDiagnostics: projection.diagnostics,
|
|
3358
|
+
readerWarnings: Object.freeze([...input.read.warnings]),
|
|
3359
|
+
unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
|
|
3360
|
+
sourceFiles: Object.freeze([...input.read.sourceFiles]),
|
|
3361
|
+
nodesByEventId,
|
|
3362
|
+
childrenByParentId,
|
|
3363
|
+
rootNodes: Object.freeze(scopedRuns.flatMap((run) => run.children))
|
|
3364
|
+
};
|
|
3264
3365
|
}
|
|
3265
|
-
function
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
const retryRunIds = Object.freeze(
|
|
3269
|
-
[...new Set(candidates.map((c) => c.viaRunId).filter((id) => id !== void 0))].sort(
|
|
3270
|
-
(a, b) => a.localeCompare(b)
|
|
3271
|
-
)
|
|
3272
|
-
);
|
|
3273
|
-
if (successful.length > 1) {
|
|
3274
|
-
const distinctRuns = new Set(successful.map((c) => c.event.runId));
|
|
3275
|
-
const distinctParents = new Set(
|
|
3276
|
-
successful.map((c) => c.event.parentId ?? "").filter((id) => id !== "")
|
|
3277
|
-
);
|
|
3278
|
-
if (distinctRuns.size > 1 || distinctParents.size > 1) {
|
|
3366
|
+
function resolveSelectedRun(input, runId) {
|
|
3367
|
+
if (input.selectedRun) {
|
|
3368
|
+
if (runId && input.selectedRun.runId !== runId) {
|
|
3279
3369
|
return {
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
basis: Object.freeze(["ambiguous-recovery-candidates"]),
|
|
3287
|
-
recoveryEventIds: Object.freeze(
|
|
3288
|
-
successful.map((c) => c.event.eventId).sort((a, b) => a.localeCompare(b))
|
|
3289
|
-
),
|
|
3290
|
-
retryRunIds
|
|
3370
|
+
diagnostics: [
|
|
3371
|
+
diagnostic(
|
|
3372
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
3373
|
+
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
3374
|
+
)
|
|
3375
|
+
]
|
|
3291
3376
|
};
|
|
3292
3377
|
}
|
|
3378
|
+
return { run: input.selectedRun, diagnostics: [] };
|
|
3293
3379
|
}
|
|
3294
|
-
if (
|
|
3295
|
-
const
|
|
3296
|
-
|
|
3297
|
-
eventId: failure.eventId,
|
|
3298
|
-
runId: failure.runId,
|
|
3299
|
-
name: failure.name,
|
|
3300
|
-
kind: failure.kind,
|
|
3301
|
-
role: "recovered",
|
|
3302
|
-
confidence: best.confidence,
|
|
3303
|
-
basis: Object.freeze([best.basis]),
|
|
3304
|
-
recoveryEventIds: Object.freeze(
|
|
3305
|
-
successful.map((c) => c.event.eventId).sort((a, b) => a.localeCompare(b))
|
|
3306
|
-
),
|
|
3307
|
-
retryRunIds
|
|
3308
|
-
};
|
|
3309
|
-
}
|
|
3310
|
-
if (candidates.length > 0) {
|
|
3311
|
-
const best = candidates[0];
|
|
3312
|
-
return {
|
|
3313
|
-
eventId: failure.eventId,
|
|
3314
|
-
runId: failure.runId,
|
|
3315
|
-
name: failure.name,
|
|
3316
|
-
kind: failure.kind,
|
|
3317
|
-
role: "transient",
|
|
3318
|
-
confidence: best.confidence,
|
|
3319
|
-
basis: Object.freeze([
|
|
3320
|
-
best.basis,
|
|
3321
|
-
unsuccessful.some((c) => c.event.status === void 0) ? "retry-incomplete" : "retry-without-success"
|
|
3322
|
-
]),
|
|
3323
|
-
recoveryEventIds: Object.freeze([]),
|
|
3324
|
-
retryRunIds
|
|
3325
|
-
};
|
|
3326
|
-
}
|
|
3327
|
-
const failureMeta = eventMetadata(failure);
|
|
3328
|
-
const declaredSuccessor = pickString2(failureMeta, [
|
|
3329
|
-
"retriedBy",
|
|
3330
|
-
"nextRetryRunId",
|
|
3331
|
-
"retryRunId"
|
|
3332
|
-
]);
|
|
3333
|
-
if (declaredSuccessor !== void 0 && !runs.has(declaredSuccessor)) {
|
|
3334
|
-
return {
|
|
3335
|
-
eventId: failure.eventId,
|
|
3336
|
-
runId: failure.runId,
|
|
3337
|
-
name: failure.name,
|
|
3338
|
-
kind: failure.kind,
|
|
3339
|
-
role: "transient",
|
|
3340
|
-
confidence: "explicit",
|
|
3341
|
-
basis: Object.freeze(["retry-declared", "retry-run-missing"]),
|
|
3342
|
-
recoveryEventIds: Object.freeze([]),
|
|
3343
|
-
retryRunIds: Object.freeze([declaredSuccessor])
|
|
3344
|
-
};
|
|
3345
|
-
}
|
|
3346
|
-
for (const run of runs.values()) {
|
|
3347
|
-
if (run.retryOf === failure.runId) {
|
|
3380
|
+
if (runId) {
|
|
3381
|
+
const run = input.read.runs.find((candidate) => candidate.runId === runId);
|
|
3382
|
+
if (!run) {
|
|
3348
3383
|
return {
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
kind: failure.kind,
|
|
3353
|
-
role: "transient",
|
|
3354
|
-
confidence: "explicit",
|
|
3355
|
-
basis: Object.freeze(["retryOf", "retry-run-missing-or-empty"]),
|
|
3356
|
-
recoveryEventIds: Object.freeze([]),
|
|
3357
|
-
retryRunIds: Object.freeze([run.runId])
|
|
3384
|
+
diagnostics: [
|
|
3385
|
+
diagnostic("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
3386
|
+
]
|
|
3358
3387
|
};
|
|
3359
3388
|
}
|
|
3389
|
+
return { run, diagnostics: [] };
|
|
3360
3390
|
}
|
|
3361
|
-
|
|
3362
|
-
|
|
3363
|
-
|
|
3364
|
-
if (
|
|
3365
|
-
(event) => event.runId === failure.runId && event.eventId !== failure.eventId && isSuccessful(event) && canonicalName(event) === canonicalName(failure)
|
|
3366
|
-
)) {
|
|
3391
|
+
if (input.read.runs.length === 1) {
|
|
3392
|
+
return { run: input.read.runs[0], diagnostics: [] };
|
|
3393
|
+
}
|
|
3394
|
+
if (input.read.runs.length === 0) {
|
|
3367
3395
|
return {
|
|
3368
|
-
|
|
3369
|
-
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
|
|
3373
|
-
|
|
3374
|
-
|
|
3375
|
-
|
|
3376
|
-
|
|
3377
|
-
failureRun.retryOf !== void 0 ? [failureRun.retryOf] : []
|
|
3378
|
-
)
|
|
3396
|
+
diagnostics: [
|
|
3397
|
+
diagnostic(
|
|
3398
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
3399
|
+
formatProgrammaticDiagnostic(
|
|
3400
|
+
"AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
3401
|
+
"No runs are available for checks."
|
|
3402
|
+
)
|
|
3403
|
+
)
|
|
3404
|
+
]
|
|
3379
3405
|
};
|
|
3380
|
-
}
|
|
3381
|
-
return {
|
|
3382
|
-
eventId: failure.eventId,
|
|
3383
|
-
runId: failure.runId,
|
|
3384
|
-
name: failure.name,
|
|
3385
|
-
kind: failure.kind,
|
|
3386
|
-
role: "unknown",
|
|
3387
|
-
confidence: "unknown",
|
|
3388
|
-
basis: Object.freeze(["no-explicit-or-correlated-recovery"]),
|
|
3389
|
-
recoveryEventIds: Object.freeze([]),
|
|
3390
|
-
retryRunIds: Object.freeze([])
|
|
3391
|
-
};
|
|
3392
|
-
}
|
|
3393
|
-
function deriveFailureFacts(logicalEvents) {
|
|
3394
|
-
const runs = buildRunContexts(logicalEvents);
|
|
3395
|
-
const failures = logicalEvents.filter((event) => isFailure(event)).sort(compareEventOrder);
|
|
3396
|
-
const failureFacts = failures.map(
|
|
3397
|
-
(failure) => classifyFailure(failure, collectCandidates(failure, logicalEvents, runs), runs, logicalEvents)
|
|
3398
|
-
);
|
|
3399
|
-
const byRole = /* @__PURE__ */ new Map([
|
|
3400
|
-
["transient", []],
|
|
3401
|
-
["recovered", []],
|
|
3402
|
-
["terminal", []],
|
|
3403
|
-
["unknown", []]
|
|
3404
|
-
]);
|
|
3405
|
-
for (const fact of failureFacts) {
|
|
3406
|
-
byRole.get(fact.role).push(fact);
|
|
3407
|
-
}
|
|
3408
|
-
for (const [role, list] of byRole) {
|
|
3409
|
-
byRole.set(
|
|
3410
|
-
role,
|
|
3411
|
-
Object.freeze(
|
|
3412
|
-
[...list].sort((a, b) => {
|
|
3413
|
-
const byRun = a.runId.localeCompare(b.runId);
|
|
3414
|
-
if (byRun !== 0) return byRun;
|
|
3415
|
-
return a.eventId.localeCompare(b.eventId);
|
|
3416
|
-
})
|
|
3417
|
-
)
|
|
3418
|
-
);
|
|
3419
|
-
}
|
|
3420
|
-
const failureRoleCounts = {
|
|
3421
|
-
transient: byRole.get("transient").length,
|
|
3422
|
-
recovered: byRole.get("recovered").length,
|
|
3423
|
-
terminal: byRole.get("terminal").length,
|
|
3424
|
-
unknown: byRole.get("unknown").length
|
|
3425
|
-
};
|
|
3406
|
+
}
|
|
3426
3407
|
return {
|
|
3427
|
-
|
|
3428
|
-
|
|
3429
|
-
|
|
3408
|
+
diagnostics: [
|
|
3409
|
+
diagnostic(
|
|
3410
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
3411
|
+
formatProgrammaticDiagnostic("AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED")
|
|
3412
|
+
)
|
|
3413
|
+
]
|
|
3430
3414
|
};
|
|
3431
3415
|
}
|
|
3432
|
-
|
|
3433
|
-
// packages/core/src/checks/relationship-facts.ts
|
|
3434
|
-
function attrs(event) {
|
|
3435
|
-
return event.attributes && typeof event.attributes === "object" ? event.attributes : {};
|
|
3436
|
-
}
|
|
3437
|
-
function stringAttr(record, key) {
|
|
3438
|
-
const value = record[key];
|
|
3439
|
-
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
3440
|
-
}
|
|
3441
|
-
function deriveRelationshipFacts(events) {
|
|
3442
|
-
const relationships = [];
|
|
3416
|
+
function selectRules(rules, selectedIds) {
|
|
3443
3417
|
const diagnostics = [];
|
|
3444
|
-
const byId = new Map(
|
|
3445
|
-
const
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
|
|
3449
|
-
|
|
3450
|
-
|
|
3451
|
-
};
|
|
3452
|
-
for (const event of events) {
|
|
3453
|
-
if (event.parentId) {
|
|
3454
|
-
push({
|
|
3455
|
-
type: "parent-child",
|
|
3456
|
-
fromEventId: event.parentId,
|
|
3457
|
-
toEventId: event.eventId,
|
|
3458
|
-
confidence: byId.has(event.parentId) ? "explicit" : "unknown",
|
|
3459
|
-
basis: ["persisted.parentId"]
|
|
3460
|
-
});
|
|
3461
|
-
if (!byId.has(event.parentId)) {
|
|
3462
|
-
diagnostics.push({
|
|
3463
|
-
code: "AI_RELATIONSHIP_PARENT_MISSING",
|
|
3464
|
-
message: `parentId ${event.parentId} is not present in the event set.`,
|
|
3465
|
-
eventId: event.eventId
|
|
3466
|
-
});
|
|
3467
|
-
}
|
|
3468
|
-
}
|
|
3469
|
-
const bag = attrs(event);
|
|
3470
|
-
const meta = extractSessionWorkflowMetadata(bag) ?? {};
|
|
3471
|
-
const nested = bag.metadata && typeof bag.metadata === "object" ? extractSessionWorkflowMetadata(bag.metadata) : void 0;
|
|
3472
|
-
const workflow = { ...meta, ...nested };
|
|
3473
|
-
if (workflow.retryOf) {
|
|
3474
|
-
const target = events.find((candidate) => candidate.runId === workflow.retryOf);
|
|
3475
|
-
push({
|
|
3476
|
-
type: "retry-of",
|
|
3477
|
-
fromEventId: event.eventId,
|
|
3478
|
-
...target ? { toEventId: target.eventId } : {},
|
|
3479
|
-
externalRef: workflow.retryOf,
|
|
3480
|
-
confidence: target ? "explicit" : "correlated",
|
|
3481
|
-
basis: ["attributes.retryOf"]
|
|
3482
|
-
});
|
|
3483
|
-
}
|
|
3484
|
-
const attemptOf = stringAttr(bag, "attemptOf") ?? stringAttr(bag, "operationId");
|
|
3485
|
-
if (attemptOf && workflow.attempt !== void 0) {
|
|
3486
|
-
push({
|
|
3487
|
-
type: "attempt-of",
|
|
3488
|
-
fromEventId: event.eventId,
|
|
3489
|
-
externalRef: attemptOf,
|
|
3490
|
-
confidence: "explicit",
|
|
3491
|
-
basis: workflow.attempt !== void 0 ? ["attributes.attempt", "attributes.operationId"] : ["attributes.operationId"]
|
|
3492
|
-
});
|
|
3493
|
-
}
|
|
3494
|
-
const fallbackOf = stringAttr(bag, "fallbackOf");
|
|
3495
|
-
if (fallbackOf) {
|
|
3496
|
-
push({
|
|
3497
|
-
type: "fallback-of",
|
|
3498
|
-
fromEventId: event.eventId,
|
|
3499
|
-
externalRef: fallbackOf,
|
|
3500
|
-
confidence: "explicit",
|
|
3501
|
-
basis: ["attributes.fallbackOf"]
|
|
3502
|
-
});
|
|
3503
|
-
}
|
|
3504
|
-
const remediationOf = stringAttr(bag, "remediationOf");
|
|
3505
|
-
if (remediationOf) {
|
|
3506
|
-
push({
|
|
3507
|
-
type: "remediation-of",
|
|
3508
|
-
fromEventId: event.eventId,
|
|
3509
|
-
externalRef: remediationOf,
|
|
3510
|
-
confidence: "explicit",
|
|
3511
|
-
basis: ["attributes.remediationOf"]
|
|
3512
|
-
});
|
|
3513
|
-
}
|
|
3514
|
-
const evidenceFor = stringAttr(bag, "evidenceFor");
|
|
3515
|
-
if (evidenceFor) {
|
|
3516
|
-
push({
|
|
3517
|
-
type: "evidence-for",
|
|
3518
|
-
fromEventId: event.eventId,
|
|
3519
|
-
...byId.has(evidenceFor) ? { toEventId: evidenceFor } : { externalRef: evidenceFor },
|
|
3520
|
-
confidence: byId.has(evidenceFor) ? "explicit" : "correlated",
|
|
3521
|
-
basis: ["attributes.evidenceFor"]
|
|
3522
|
-
});
|
|
3523
|
-
}
|
|
3524
|
-
const acceptedBy = stringAttr(bag, "acceptedBy");
|
|
3525
|
-
if (acceptedBy) {
|
|
3526
|
-
push({
|
|
3527
|
-
type: "accepted-by",
|
|
3528
|
-
fromEventId: event.eventId,
|
|
3529
|
-
...byId.has(acceptedBy) ? { toEventId: acceptedBy } : { externalRef: acceptedBy },
|
|
3530
|
-
confidence: byId.has(acceptedBy) ? "explicit" : "correlated",
|
|
3531
|
-
basis: ["attributes.acceptedBy"]
|
|
3532
|
-
});
|
|
3418
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3419
|
+
for (const rule of rules) {
|
|
3420
|
+
if (byId.has(rule.id)) {
|
|
3421
|
+
diagnostics.push(
|
|
3422
|
+
diagnostic("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
3423
|
+
);
|
|
3424
|
+
continue;
|
|
3533
3425
|
}
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3542
|
-
|
|
3426
|
+
byId.set(rule.id, rule);
|
|
3427
|
+
}
|
|
3428
|
+
if (selectedIds && selectedIds.length > 0) {
|
|
3429
|
+
const selected = new Set(selectedIds);
|
|
3430
|
+
for (const id of selected) {
|
|
3431
|
+
if (!byId.has(id)) {
|
|
3432
|
+
diagnostics.push(
|
|
3433
|
+
diagnostic("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
3434
|
+
);
|
|
3435
|
+
}
|
|
3543
3436
|
}
|
|
3544
|
-
|
|
3545
|
-
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
|
|
3437
|
+
return {
|
|
3438
|
+
rules: [...byId.values()].filter((rule) => selected.has(rule.id)).sort(compareRules),
|
|
3439
|
+
diagnostics
|
|
3440
|
+
};
|
|
3441
|
+
}
|
|
3442
|
+
return { rules: [...byId.values()].sort(compareRules), diagnostics };
|
|
3443
|
+
}
|
|
3444
|
+
function compareRules(a, b) {
|
|
3445
|
+
return a.id.localeCompare(b.id);
|
|
3446
|
+
}
|
|
3447
|
+
function eventTimestamp(finding, eventById) {
|
|
3448
|
+
const eventId = finding.evidence[0]?.eventId;
|
|
3449
|
+
return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
|
|
3450
|
+
}
|
|
3451
|
+
function compareFindings(eventById) {
|
|
3452
|
+
return (a, b) => {
|
|
3453
|
+
if (SEVERITY_RANK[a.severity] !== SEVERITY_RANK[b.severity]) {
|
|
3454
|
+
return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity];
|
|
3553
3455
|
}
|
|
3554
|
-
const
|
|
3555
|
-
if (
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
"retry-of",
|
|
3559
|
-
"fallback-of",
|
|
3560
|
-
"remediation-of",
|
|
3561
|
-
"evidence-for",
|
|
3562
|
-
"accepted-by",
|
|
3563
|
-
"supersedes"
|
|
3564
|
-
].includes(unsupported)) {
|
|
3565
|
-
diagnostics.push({
|
|
3566
|
-
code: "AI_RELATIONSHIP_UNSUPPORTED_TYPE",
|
|
3567
|
-
message: `Unsupported relationshipType ${unsupported} was reported without flattening.`,
|
|
3568
|
-
eventId: event.eventId
|
|
3569
|
-
});
|
|
3456
|
+
const byRule = a.ruleId.localeCompare(b.ruleId);
|
|
3457
|
+
if (byRule !== 0) return byRule;
|
|
3458
|
+
if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
|
|
3459
|
+
return STATUS_RANK[a.status] - STATUS_RANK[b.status];
|
|
3570
3460
|
}
|
|
3571
|
-
|
|
3461
|
+
const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
|
|
3462
|
+
if (byRun !== 0) return byRun;
|
|
3463
|
+
const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
|
|
3464
|
+
if (byTime !== 0) return byTime;
|
|
3465
|
+
const byEvent = compareStrings(a.evidence[0]?.eventId, b.evidence[0]?.eventId);
|
|
3466
|
+
if (byEvent !== 0) return byEvent;
|
|
3467
|
+
return compareStrings(a.evidence[0]?.path, b.evidence[0]?.path);
|
|
3468
|
+
};
|
|
3469
|
+
}
|
|
3470
|
+
function normalizeFinding(rule, finding) {
|
|
3572
3471
|
return {
|
|
3573
|
-
|
|
3574
|
-
|
|
3472
|
+
ruleId: finding.ruleId || rule.id,
|
|
3473
|
+
severity: finding.severity ?? rule.defaultSeverity,
|
|
3474
|
+
status: finding.status,
|
|
3475
|
+
message: finding.message,
|
|
3476
|
+
...finding.expected !== void 0 ? { expected: finding.expected } : {},
|
|
3477
|
+
...finding.actual !== void 0 ? { actual: finding.actual } : {},
|
|
3478
|
+
evidence: [...finding.evidence ?? []],
|
|
3479
|
+
...finding.category !== void 0 ? { category: finding.category } : {},
|
|
3480
|
+
...finding.confidence !== void 0 ? { confidence: finding.confidence } : {},
|
|
3481
|
+
...finding.detector !== void 0 ? { detector: finding.detector } : {},
|
|
3482
|
+
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
3575
3483
|
};
|
|
3576
3484
|
}
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3485
|
+
function summarize(findings, diagnostics, rulesEvaluated) {
|
|
3486
|
+
return {
|
|
3487
|
+
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
3488
|
+
failed: findings.filter(
|
|
3489
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
3490
|
+
).length,
|
|
3491
|
+
warnings: findings.filter(
|
|
3492
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
3493
|
+
).length,
|
|
3494
|
+
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
3495
|
+
rulesEvaluated
|
|
3496
|
+
};
|
|
3497
|
+
}
|
|
3498
|
+
function classifyRuleExecution(findings, threw) {
|
|
3499
|
+
if (findings.some((finding) => finding.status === "fail" && finding.severity === "error")) {
|
|
3500
|
+
return "fail";
|
|
3501
|
+
}
|
|
3502
|
+
if (findings.some(
|
|
3503
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
3504
|
+
)) {
|
|
3505
|
+
return "warning";
|
|
3506
|
+
}
|
|
3507
|
+
return "pass";
|
|
3508
|
+
}
|
|
3509
|
+
function eventEvidence(event, path17) {
|
|
3510
|
+
return {
|
|
3511
|
+
runId: event.runId,
|
|
3512
|
+
eventId: event.eventId,
|
|
3513
|
+
parentId: event.parentId,
|
|
3514
|
+
traceId: event.trace?.traceId,
|
|
3515
|
+
spanId: event.trace?.spanId,
|
|
3516
|
+
kind: event.kind,
|
|
3517
|
+
name: event.name,
|
|
3518
|
+
status: event.status,
|
|
3519
|
+
...path17 ? { path: path17 } : {}
|
|
3520
|
+
};
|
|
3521
|
+
}
|
|
3522
|
+
function runEvidence(run) {
|
|
3523
|
+
return run ? [{ runId: run.runId, name: run.name, status: run.status }] : [];
|
|
3524
|
+
}
|
|
3525
|
+
function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
3589
3526
|
return {
|
|
3590
|
-
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
|
|
3594
|
-
|
|
3595
|
-
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
|
|
3599
|
-
|
|
3600
|
-
|
|
3527
|
+
ruleId,
|
|
3528
|
+
severity: meta?.severity ?? "error",
|
|
3529
|
+
status: meta?.status ?? "fail",
|
|
3530
|
+
message,
|
|
3531
|
+
...expected !== void 0 ? { expected } : {},
|
|
3532
|
+
...actual !== void 0 ? { actual } : {},
|
|
3533
|
+
evidence: [...evidence],
|
|
3534
|
+
...meta?.category !== void 0 ? { category: meta.category } : {},
|
|
3535
|
+
...meta?.confidence !== void 0 ? { confidence: meta.confidence } : {},
|
|
3536
|
+
...meta?.detector !== void 0 ? { detector: meta.detector } : {},
|
|
3537
|
+
...meta?.action !== void 0 ? { action: meta.action } : {}
|
|
3601
3538
|
};
|
|
3602
3539
|
}
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
);
|
|
3606
|
-
function isTraceReadResult(input) {
|
|
3607
|
-
if (typeof input !== "object" || input === null || Array.isArray(input)) {
|
|
3608
|
-
return false;
|
|
3609
|
-
}
|
|
3610
|
-
const record = input;
|
|
3611
|
-
return Array.isArray(record.events) && Array.isArray(record.runs) && typeof record.format === "string" && Array.isArray(record.warnings);
|
|
3540
|
+
function semanticEvents(context) {
|
|
3541
|
+
return context.logicalEvents ?? context.events;
|
|
3612
3542
|
}
|
|
3613
|
-
function
|
|
3614
|
-
|
|
3615
|
-
const first = input[0];
|
|
3616
|
-
if (typeof first !== "object" || first === null) return false;
|
|
3617
|
-
const row = first;
|
|
3618
|
-
return typeof row.event === "string" && (row.schemaVersion === "0.1" || row.eventId === void 0);
|
|
3543
|
+
function isRecord8(value) {
|
|
3544
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3619
3545
|
}
|
|
3620
|
-
function
|
|
3621
|
-
|
|
3622
|
-
if (input.length === 0) return true;
|
|
3623
|
-
const first = input[0];
|
|
3624
|
-
if (typeof first !== "object" || first === null) return false;
|
|
3625
|
-
const row = first;
|
|
3626
|
-
return typeof row.eventId === "string" && (row.schemaVersion === "0.2" || row.schemaVersion === "1.0" || row.schemaVersion === "0.1") && typeof row.event !== "string";
|
|
3546
|
+
function normalizedKey(value) {
|
|
3547
|
+
return value.toLowerCase().replace(/[^a-z0-9_]/g, "");
|
|
3627
3548
|
}
|
|
3628
|
-
function
|
|
3629
|
-
|
|
3630
|
-
|
|
3549
|
+
function lastPathSegment(path17) {
|
|
3550
|
+
const parts = path17.split(".");
|
|
3551
|
+
return parts[parts.length - 1] ?? path17;
|
|
3552
|
+
}
|
|
3553
|
+
function valueType(value) {
|
|
3554
|
+
if (Array.isArray(value)) return "array";
|
|
3555
|
+
if (value === null) return "null";
|
|
3556
|
+
return typeof value;
|
|
3557
|
+
}
|
|
3558
|
+
function serializedByteLength(value) {
|
|
3559
|
+
try {
|
|
3560
|
+
return Buffer.byteLength(JSON.stringify(value), "utf-8");
|
|
3561
|
+
} catch {
|
|
3562
|
+
return void 0;
|
|
3631
3563
|
}
|
|
3632
|
-
|
|
3633
|
-
|
|
3564
|
+
}
|
|
3565
|
+
function pushValueEntries(entries, event, value, path17, key, depth = 0) {
|
|
3566
|
+
entries.push({ event, path: path17, key, value });
|
|
3567
|
+
if (depth >= 8) return;
|
|
3568
|
+
if (Array.isArray(value)) {
|
|
3569
|
+
for (const [index, item] of value.entries()) {
|
|
3570
|
+
pushValueEntries(entries, event, item, `${path17}.${index}`, String(index), depth + 1);
|
|
3571
|
+
}
|
|
3572
|
+
return;
|
|
3634
3573
|
}
|
|
3635
|
-
if (
|
|
3636
|
-
|
|
3574
|
+
if (!isRecord8(value)) return;
|
|
3575
|
+
for (const nestedKey of Object.keys(value).sort((a, b) => a.localeCompare(b))) {
|
|
3576
|
+
pushValueEntries(
|
|
3577
|
+
entries,
|
|
3578
|
+
event,
|
|
3579
|
+
value[nestedKey],
|
|
3580
|
+
`${path17}.${nestedKey}`,
|
|
3581
|
+
nestedKey,
|
|
3582
|
+
depth + 1
|
|
3583
|
+
);
|
|
3637
3584
|
}
|
|
3638
|
-
throw new TypeError(TRACE_FACTS_INPUT_NOT_NORMALIZED);
|
|
3639
3585
|
}
|
|
3640
|
-
function
|
|
3641
|
-
const
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
const name = resolveCanonicalToolName(event);
|
|
3649
|
-
const list = toolsByName.get(name) ?? [];
|
|
3650
|
-
list.push(event);
|
|
3651
|
-
toolsByName.set(name, list);
|
|
3652
|
-
}
|
|
3653
|
-
if (event.kind === "LLM" && event.status !== "running") {
|
|
3654
|
-
llmEvents.push(event);
|
|
3586
|
+
function eventValueEntries(event, options = {}) {
|
|
3587
|
+
const entries = [];
|
|
3588
|
+
if (event.attributes !== void 0) {
|
|
3589
|
+
pushValueEntries(entries, event, event.attributes, "attributes", "attributes");
|
|
3590
|
+
}
|
|
3591
|
+
if (options.includeSummaries) {
|
|
3592
|
+
if (event.inputSummary !== void 0) {
|
|
3593
|
+
pushValueEntries(entries, event, event.inputSummary, "inputSummary", "inputSummary");
|
|
3655
3594
|
}
|
|
3656
|
-
if (event.
|
|
3657
|
-
|
|
3595
|
+
if (event.outputSummary !== void 0) {
|
|
3596
|
+
pushValueEntries(entries, event, event.outputSummary, "outputSummary", "outputSummary");
|
|
3658
3597
|
}
|
|
3659
3598
|
}
|
|
3660
|
-
|
|
3661
|
-
|
|
3599
|
+
if (options.includeError && event.error !== void 0) {
|
|
3600
|
+
pushValueEntries(entries, event, event.error, "error", "error");
|
|
3662
3601
|
}
|
|
3663
|
-
|
|
3664
|
-
const relationship = deriveRelationshipFacts(events);
|
|
3665
|
-
const summary = summarizeSemanticParity(events);
|
|
3666
|
-
return {
|
|
3667
|
-
rawEvents: Object.freeze([...events]),
|
|
3668
|
-
logicalEvents: projection.logicalEvents,
|
|
3669
|
-
diagnostics: projection.diagnostics,
|
|
3670
|
-
toolsByName,
|
|
3671
|
-
llmEvents: Object.freeze(llmEvents),
|
|
3672
|
-
outcomeEvents: Object.freeze(outcomeEvents),
|
|
3673
|
-
summary: {
|
|
3674
|
-
...summary,
|
|
3675
|
-
failureRoleCounts: derived.failureRoleCounts
|
|
3676
|
-
},
|
|
3677
|
-
failureFacts: derived.failureFacts,
|
|
3678
|
-
failuresByRole: derived.failuresByRole,
|
|
3679
|
-
relationships: relationship.relationships,
|
|
3680
|
-
relationshipDiagnostics: relationship.diagnostics
|
|
3681
|
-
};
|
|
3602
|
+
return entries;
|
|
3682
3603
|
}
|
|
3683
|
-
|
|
3684
|
-
|
|
3685
|
-
|
|
3686
|
-
|
|
3687
|
-
// packages/core/src/checks/index.ts
|
|
3688
|
-
var SEVERITY_RANK = {
|
|
3689
|
-
error: 0,
|
|
3690
|
-
warning: 1,
|
|
3691
|
-
info: 2
|
|
3692
|
-
};
|
|
3693
|
-
var STATUS_RANK = {
|
|
3694
|
-
fail: 0,
|
|
3695
|
-
warning: 1,
|
|
3696
|
-
pass: 2
|
|
3697
|
-
};
|
|
3698
|
-
var DEFAULT_SENSITIVE_KEYS = DEFAULT_CREDENTIAL_SENSITIVE_KEYS;
|
|
3699
|
-
var DEFAULT_RAW_CONTENT_KEYS = [
|
|
3700
|
-
"body",
|
|
3701
|
-
"headers",
|
|
3702
|
-
"input",
|
|
3703
|
-
"messages",
|
|
3704
|
-
"output",
|
|
3705
|
-
"payload",
|
|
3706
|
-
"prompt",
|
|
3707
|
-
"requestbody",
|
|
3708
|
-
"request_body",
|
|
3709
|
-
"responsebody",
|
|
3710
|
-
"response_body",
|
|
3711
|
-
"rawprompt",
|
|
3712
|
-
"raw_prompt",
|
|
3713
|
-
"rawoutput",
|
|
3714
|
-
"raw_output",
|
|
3715
|
-
"toolinput",
|
|
3716
|
-
"tool_input",
|
|
3717
|
-
"tooloutput",
|
|
3718
|
-
"tool_output",
|
|
3719
|
-
// Framework / agent metadata that carries user or task text
|
|
3720
|
-
"currenttask",
|
|
3721
|
-
"current_task",
|
|
3722
|
-
"task",
|
|
3723
|
-
"userinput",
|
|
3724
|
-
"user_input",
|
|
3725
|
-
"requesttext",
|
|
3726
|
-
"request_text",
|
|
3727
|
-
"conversationtext",
|
|
3728
|
-
"conversation_text"
|
|
3729
|
-
];
|
|
3730
|
-
var DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES = [
|
|
3731
|
-
"tokenUsage",
|
|
3732
|
-
"usage",
|
|
3733
|
-
"tokens"
|
|
3734
|
-
];
|
|
3735
|
-
var SAFE_USAGE_LEAF_KEYS = /* @__PURE__ */ new Set([
|
|
3736
|
-
"input",
|
|
3737
|
-
"output",
|
|
3738
|
-
"total",
|
|
3739
|
-
"cached",
|
|
3740
|
-
"input_tokens",
|
|
3741
|
-
"inputtokens",
|
|
3742
|
-
"output_tokens",
|
|
3743
|
-
"outputtokens",
|
|
3744
|
-
"total_tokens",
|
|
3745
|
-
"totaltokens",
|
|
3746
|
-
"prompt_tokens",
|
|
3747
|
-
"prompttokens",
|
|
3748
|
-
"completion_tokens",
|
|
3749
|
-
"completiontokens"
|
|
3750
|
-
]);
|
|
3751
|
-
var DEFAULT_SECRET_PATTERNS = [
|
|
3752
|
-
{ id: "bearer-token", pattern: /Bearer\s+[A-Za-z0-9._~+/-]{12,}=*/ },
|
|
3753
|
-
{ id: "openai-key", pattern: /sk-[A-Za-z0-9_-]{16,}/ },
|
|
3754
|
-
{ id: "aws-access-key", pattern: /AKIA[0-9A-Z]{16}/ },
|
|
3755
|
-
{ id: "github-token", pattern: /gh[opsu]_[A-Za-z0-9_]{20,}/ },
|
|
3756
|
-
// Keep in sync with packages/redact/src/key-value-secret.ts (KEY_VALUE_SECRET_PATTERN_SOURCE).
|
|
3757
|
-
{
|
|
3758
|
-
id: "key-value-secret",
|
|
3759
|
-
pattern: /\b(?:api[_-]?key|internal[_-]?token|access[_-]?token|auth[_-]?token|password|secret|token)=([^\s"'\\]{8,})/i
|
|
3760
|
-
}
|
|
3761
|
-
];
|
|
3762
|
-
function compareStrings(a, b) {
|
|
3763
|
-
return (a ?? "").localeCompare(b ?? "");
|
|
3604
|
+
function limitFindings(findings, maxFindings) {
|
|
3605
|
+
if (maxFindings === void 0 || findings.length <= maxFindings) return findings;
|
|
3606
|
+
return findings.slice(0, Math.max(0, maxFindings));
|
|
3764
3607
|
}
|
|
3765
|
-
function
|
|
3608
|
+
function hasRedactionMarker(value, markers) {
|
|
3609
|
+
return markers.some((marker) => value.includes(marker)) || /^\[HASH:[A-Za-z0-9_-]+\]$/.test(value);
|
|
3610
|
+
}
|
|
3611
|
+
function isSensitiveKey(key, sensitiveKeys) {
|
|
3612
|
+
return isCredentialSensitiveKey(key, sensitiveKeys);
|
|
3613
|
+
}
|
|
3614
|
+
function isRawContentKey(key, forbiddenKeys) {
|
|
3615
|
+
if (!key) return false;
|
|
3616
|
+
const normalized = normalizedKey(key);
|
|
3617
|
+
return forbiddenKeys.some((forbidden) => normalized === normalizedKey(forbidden));
|
|
3618
|
+
}
|
|
3619
|
+
function isSafeRawContentMetricPath(path17, key, safePathPrefixes) {
|
|
3620
|
+
const leaf = normalizedKey(key ?? lastPathSegment(path17));
|
|
3621
|
+
if (!SAFE_USAGE_LEAF_KEYS.has(leaf)) return false;
|
|
3622
|
+
const parts = path17.split(".").filter(Boolean);
|
|
3623
|
+
if (parts.length < 2) return false;
|
|
3624
|
+
const parent = parts[parts.length - 2] ?? "";
|
|
3625
|
+
const parentNorm = normalizedKey(parent);
|
|
3626
|
+
return safePathPrefixes.some((prefix) => parentNorm === normalizedKey(prefix));
|
|
3627
|
+
}
|
|
3628
|
+
function isRawContentPath(path17, key, forbiddenKeys, safePathPrefixes) {
|
|
3629
|
+
if (isSafeRawContentMetricPath(path17, key, safePathPrefixes)) return false;
|
|
3630
|
+
return isRawContentKey(key ?? lastPathSegment(path17), forbiddenKeys);
|
|
3631
|
+
}
|
|
3632
|
+
function createRunStatusRule(options = {}) {
|
|
3633
|
+
const expected = options.expected ?? "ok";
|
|
3634
|
+
const allowIncomplete = options.allowIncomplete === true;
|
|
3766
3635
|
return {
|
|
3767
|
-
|
|
3768
|
-
|
|
3769
|
-
|
|
3770
|
-
|
|
3636
|
+
id: "run.status",
|
|
3637
|
+
category: "run",
|
|
3638
|
+
defaultSeverity: "error",
|
|
3639
|
+
evaluate(context) {
|
|
3640
|
+
const findings = [];
|
|
3641
|
+
const actual = context.selectedRun?.status ?? "unknown";
|
|
3642
|
+
if (actual !== expected) {
|
|
3643
|
+
findings.push(
|
|
3644
|
+
failFinding(
|
|
3645
|
+
"run.status",
|
|
3646
|
+
`Run status ${actual} did not match expected ${expected}.`,
|
|
3647
|
+
runEvidence(context.selectedRun),
|
|
3648
|
+
expected,
|
|
3649
|
+
actual
|
|
3650
|
+
)
|
|
3651
|
+
);
|
|
3652
|
+
}
|
|
3653
|
+
if (!allowIncomplete) {
|
|
3654
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
3655
|
+
if (running.length > 0) {
|
|
3656
|
+
findings.push(
|
|
3657
|
+
failFinding(
|
|
3658
|
+
"run.status",
|
|
3659
|
+
"Run contains incomplete running events.",
|
|
3660
|
+
running.map((event) => eventEvidence(event)),
|
|
3661
|
+
"no running events",
|
|
3662
|
+
running.length
|
|
3663
|
+
)
|
|
3664
|
+
);
|
|
3665
|
+
}
|
|
3666
|
+
}
|
|
3667
|
+
return findings;
|
|
3668
|
+
}
|
|
3771
3669
|
};
|
|
3772
3670
|
}
|
|
3773
|
-
function
|
|
3671
|
+
function createSafetyRedactionRule(options = {}) {
|
|
3672
|
+
const sensitiveKeys = options.sensitiveKeys ?? DEFAULT_SENSITIVE_KEYS;
|
|
3673
|
+
const markers = options.redactedMarkers ?? ["[REDACTED]", "[REDACTED:"];
|
|
3774
3674
|
return {
|
|
3775
|
-
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
|
|
3675
|
+
id: "safety.redaction",
|
|
3676
|
+
category: "safety",
|
|
3677
|
+
defaultSeverity: "error",
|
|
3678
|
+
evaluate(context) {
|
|
3679
|
+
const findings = [];
|
|
3680
|
+
for (const event of context.events) {
|
|
3681
|
+
for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
|
|
3682
|
+
if (!isSensitiveKey(entry.key ?? lastPathSegment(entry.path), sensitiveKeys)) continue;
|
|
3683
|
+
if (typeof entry.value === "string" && hasRedactionMarker(entry.value, markers)) continue;
|
|
3684
|
+
findings.push(
|
|
3685
|
+
failFinding(
|
|
3686
|
+
"safety.redaction",
|
|
3687
|
+
`Sensitive-looking field at ${entry.path} is not redacted.`,
|
|
3688
|
+
[eventEvidence(event, entry.path)],
|
|
3689
|
+
"redaction marker",
|
|
3690
|
+
{ path: entry.path, valueType: valueType(entry.value) },
|
|
3691
|
+
{
|
|
3692
|
+
category: "credential",
|
|
3693
|
+
confidence: "high",
|
|
3694
|
+
detector: "safety.redaction",
|
|
3695
|
+
action: "redact"
|
|
3696
|
+
}
|
|
3697
|
+
)
|
|
3698
|
+
);
|
|
3699
|
+
}
|
|
3700
|
+
}
|
|
3701
|
+
return limitFindings(findings, options.maxFindings);
|
|
3702
|
+
}
|
|
3780
3703
|
};
|
|
3781
3704
|
}
|
|
3782
|
-
function
|
|
3705
|
+
function createSafetyRawContentRule(options = {}) {
|
|
3706
|
+
const forbiddenKeys = options.forbiddenKeys ?? DEFAULT_RAW_CONTENT_KEYS;
|
|
3707
|
+
const safePathPrefixes = options.safePathPrefixes ?? DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES;
|
|
3783
3708
|
return {
|
|
3784
|
-
|
|
3785
|
-
|
|
3786
|
-
|
|
3787
|
-
|
|
3788
|
-
|
|
3789
|
-
|
|
3790
|
-
|
|
3791
|
-
|
|
3792
|
-
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3709
|
+
id: "safety.rawPrompt",
|
|
3710
|
+
category: "safety",
|
|
3711
|
+
defaultSeverity: "error",
|
|
3712
|
+
evaluate(context) {
|
|
3713
|
+
const findings = [];
|
|
3714
|
+
for (const event of context.events) {
|
|
3715
|
+
for (const entry of eventValueEntries(event, { includeSummaries: options.includeSummaries })) {
|
|
3716
|
+
const key = entry.key ?? lastPathSegment(entry.path);
|
|
3717
|
+
if (!isRawContentPath(entry.path, key, forbiddenKeys, safePathPrefixes)) continue;
|
|
3718
|
+
findings.push(
|
|
3719
|
+
failFinding(
|
|
3720
|
+
"safety.rawPrompt",
|
|
3721
|
+
`Raw content-like field ${entry.path} is present.`,
|
|
3722
|
+
[eventEvidence(event, entry.path)],
|
|
3723
|
+
"metadata-only trace fields",
|
|
3724
|
+
{ path: entry.path, valueType: valueType(entry.value) },
|
|
3725
|
+
{
|
|
3726
|
+
category: "raw-content",
|
|
3727
|
+
confidence: "high",
|
|
3728
|
+
detector: "safety.rawPrompt",
|
|
3729
|
+
action: "redact-or-omit"
|
|
3730
|
+
}
|
|
3731
|
+
)
|
|
3732
|
+
);
|
|
3733
|
+
}
|
|
3734
|
+
}
|
|
3735
|
+
return limitFindings(findings, options.maxFindings);
|
|
3736
|
+
}
|
|
3796
3737
|
};
|
|
3797
3738
|
}
|
|
3798
|
-
function
|
|
3799
|
-
|
|
3739
|
+
function createSafetySecretPatternRule(options = {}) {
|
|
3740
|
+
const patterns = options.patterns ?? DEFAULT_SECRET_PATTERNS;
|
|
3741
|
+
const maxStringLength = options.maxStringLength ?? 4096;
|
|
3742
|
+
return {
|
|
3743
|
+
id: "safety.secretPattern",
|
|
3744
|
+
category: "safety",
|
|
3745
|
+
defaultSeverity: "error",
|
|
3746
|
+
evaluate(context) {
|
|
3747
|
+
const findings = [];
|
|
3748
|
+
for (const event of context.events) {
|
|
3749
|
+
for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
|
|
3750
|
+
if (typeof entry.value !== "string") continue;
|
|
3751
|
+
const sample = entry.value.slice(0, maxStringLength);
|
|
3752
|
+
for (const pattern of patterns) {
|
|
3753
|
+
pattern.pattern.lastIndex = 0;
|
|
3754
|
+
if (!pattern.pattern.test(sample)) continue;
|
|
3755
|
+
pattern.pattern.lastIndex = 0;
|
|
3756
|
+
findings.push(
|
|
3757
|
+
failFinding(
|
|
3758
|
+
"safety.secretPattern",
|
|
3759
|
+
`Secret-like pattern ${pattern.id} matched at ${entry.path}.`,
|
|
3760
|
+
[eventEvidence(event, entry.path)],
|
|
3761
|
+
"no secret-like strings",
|
|
3762
|
+
{ pattern: pattern.id, path: entry.path },
|
|
3763
|
+
{
|
|
3764
|
+
category: "credential",
|
|
3765
|
+
confidence: "high",
|
|
3766
|
+
detector: pattern.id,
|
|
3767
|
+
action: "redact"
|
|
3768
|
+
}
|
|
3769
|
+
)
|
|
3770
|
+
);
|
|
3771
|
+
break;
|
|
3772
|
+
}
|
|
3773
|
+
}
|
|
3774
|
+
}
|
|
3775
|
+
return limitFindings(findings, options.maxFindings);
|
|
3776
|
+
}
|
|
3777
|
+
};
|
|
3800
3778
|
}
|
|
3801
|
-
function
|
|
3802
|
-
|
|
3803
|
-
|
|
3804
|
-
|
|
3805
|
-
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
3779
|
+
function createSafetyOversizedAttributeRule(options) {
|
|
3780
|
+
return {
|
|
3781
|
+
id: "safety.oversizedAttribute",
|
|
3782
|
+
category: "safety",
|
|
3783
|
+
defaultSeverity: "error",
|
|
3784
|
+
evaluate(context) {
|
|
3785
|
+
const findings = [];
|
|
3786
|
+
for (const event of context.events) {
|
|
3787
|
+
for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
|
|
3788
|
+
if (typeof entry.value === "string" && options.maxStringLength !== void 0 && entry.value.length > options.maxStringLength) {
|
|
3789
|
+
findings.push(
|
|
3790
|
+
failFinding(
|
|
3791
|
+
"safety.oversizedAttribute",
|
|
3792
|
+
`String at ${entry.path} exceeds ${options.maxStringLength} characters.`,
|
|
3793
|
+
[eventEvidence(event, entry.path)],
|
|
3794
|
+
{ maxStringLength: options.maxStringLength },
|
|
3795
|
+
{ path: entry.path, length: entry.value.length },
|
|
3796
|
+
{
|
|
3797
|
+
category: "size",
|
|
3798
|
+
confidence: "high",
|
|
3799
|
+
detector: "safety.oversizedAttribute",
|
|
3800
|
+
action: "truncate-or-omit"
|
|
3801
|
+
}
|
|
3802
|
+
)
|
|
3803
|
+
);
|
|
3804
|
+
}
|
|
3805
|
+
if (Array.isArray(entry.value) && options.maxArrayLength !== void 0 && entry.value.length > options.maxArrayLength) {
|
|
3806
|
+
findings.push(
|
|
3807
|
+
failFinding(
|
|
3808
|
+
"safety.oversizedAttribute",
|
|
3809
|
+
`Array at ${entry.path} exceeds ${options.maxArrayLength} items.`,
|
|
3810
|
+
[eventEvidence(event, entry.path)],
|
|
3811
|
+
{ maxArrayLength: options.maxArrayLength },
|
|
3812
|
+
{ path: entry.path, length: entry.value.length },
|
|
3813
|
+
{
|
|
3814
|
+
category: "size",
|
|
3815
|
+
confidence: "high",
|
|
3816
|
+
detector: "safety.oversizedAttribute",
|
|
3817
|
+
action: "truncate-or-omit"
|
|
3818
|
+
}
|
|
3819
|
+
)
|
|
3820
|
+
);
|
|
3821
|
+
}
|
|
3822
|
+
if (isRecord8(entry.value) && options.maxObjectKeys !== void 0 && Object.keys(entry.value).length > options.maxObjectKeys) {
|
|
3823
|
+
findings.push(
|
|
3824
|
+
failFinding(
|
|
3825
|
+
"safety.oversizedAttribute",
|
|
3826
|
+
`Object at ${entry.path} exceeds ${options.maxObjectKeys} keys.`,
|
|
3827
|
+
[eventEvidence(event, entry.path)],
|
|
3828
|
+
{ maxObjectKeys: options.maxObjectKeys },
|
|
3829
|
+
{ path: entry.path, keys: Object.keys(entry.value).length },
|
|
3830
|
+
{
|
|
3831
|
+
category: "size",
|
|
3832
|
+
confidence: "high",
|
|
3833
|
+
detector: "safety.oversizedAttribute",
|
|
3834
|
+
action: "truncate-or-omit"
|
|
3835
|
+
}
|
|
3836
|
+
)
|
|
3837
|
+
);
|
|
3838
|
+
}
|
|
3839
|
+
if (options.maxSerializedBytes !== void 0) {
|
|
3840
|
+
const bytes = serializedByteLength(entry.value);
|
|
3841
|
+
if (bytes !== void 0 && bytes > options.maxSerializedBytes) {
|
|
3842
|
+
findings.push(
|
|
3843
|
+
failFinding(
|
|
3844
|
+
"safety.oversizedAttribute",
|
|
3845
|
+
`Value at ${entry.path} exceeds ${options.maxSerializedBytes} serialized bytes.`,
|
|
3846
|
+
[eventEvidence(event, entry.path)],
|
|
3847
|
+
{ maxSerializedBytes: options.maxSerializedBytes },
|
|
3848
|
+
{ path: entry.path, bytes },
|
|
3849
|
+
{
|
|
3850
|
+
category: "size",
|
|
3851
|
+
confidence: "high",
|
|
3852
|
+
detector: "safety.oversizedAttribute",
|
|
3853
|
+
action: "truncate-or-omit"
|
|
3854
|
+
}
|
|
3855
|
+
)
|
|
3856
|
+
);
|
|
3857
|
+
}
|
|
3858
|
+
}
|
|
3859
|
+
}
|
|
3860
|
+
}
|
|
3861
|
+
return limitFindings(findings, options.maxFindings);
|
|
3815
3862
|
}
|
|
3816
|
-
}
|
|
3817
|
-
const projection = projectLogicalEvents(scopedEvents);
|
|
3818
|
-
return {
|
|
3819
|
-
format: input.read.format,
|
|
3820
|
-
runs: Object.freeze([...input.read.runs]),
|
|
3821
|
-
events: Object.freeze([...scopedEvents]),
|
|
3822
|
-
logicalEvents: projection.logicalEvents,
|
|
3823
|
-
logicalProjectionDiagnostics: projection.diagnostics,
|
|
3824
|
-
readerWarnings: Object.freeze([...input.read.warnings]),
|
|
3825
|
-
unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
|
|
3826
|
-
sourceFiles: Object.freeze([...input.read.sourceFiles]),
|
|
3827
|
-
nodesByEventId,
|
|
3828
|
-
childrenByParentId,
|
|
3829
|
-
rootNodes: Object.freeze(scopedRuns.flatMap((run) => run.children))
|
|
3830
3863
|
};
|
|
3831
3864
|
}
|
|
3832
|
-
function
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
diagnostics: [
|
|
3837
|
-
diagnostic(
|
|
3838
|
-
"AI_CHECK_INVALID_ARGUMENTS",
|
|
3839
|
-
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
3840
|
-
)
|
|
3841
|
-
]
|
|
3842
|
-
};
|
|
3843
|
-
}
|
|
3844
|
-
return { run: input.selectedRun, diagnostics: [] };
|
|
3845
|
-
}
|
|
3846
|
-
if (runId) {
|
|
3847
|
-
const run = input.read.runs.find((candidate) => candidate.runId === runId);
|
|
3848
|
-
if (!run) {
|
|
3849
|
-
return {
|
|
3850
|
-
diagnostics: [
|
|
3851
|
-
diagnostic("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
3852
|
-
]
|
|
3853
|
-
};
|
|
3854
|
-
}
|
|
3855
|
-
return { run, diagnostics: [] };
|
|
3865
|
+
function runTraceChecks(input, options = {}) {
|
|
3866
|
+
const selected = resolveSelectedRun(input, options.runId);
|
|
3867
|
+
if (selected.diagnostics.length > 0) {
|
|
3868
|
+
return errorResult(input, selected.diagnostics, selected.run);
|
|
3856
3869
|
}
|
|
3857
|
-
|
|
3858
|
-
|
|
3870
|
+
const rules = selectRules(options.rules ?? [], options.select);
|
|
3871
|
+
if (rules.diagnostics.length > 0) {
|
|
3872
|
+
return errorResult(input, rules.diagnostics, selected.run);
|
|
3859
3873
|
}
|
|
3860
|
-
if (
|
|
3861
|
-
return
|
|
3862
|
-
|
|
3874
|
+
if (rules.rules.length === 0) {
|
|
3875
|
+
return errorResult(
|
|
3876
|
+
input,
|
|
3877
|
+
[
|
|
3863
3878
|
diagnostic(
|
|
3864
|
-
"
|
|
3865
|
-
|
|
3866
|
-
"AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
3867
|
-
"No runs are available for checks."
|
|
3868
|
-
)
|
|
3879
|
+
"AI_CHECK_NO_RULES_EVALUATED",
|
|
3880
|
+
"No trace check rules were evaluated. Configure at least one rule, contract, or CLI check option."
|
|
3869
3881
|
)
|
|
3870
|
-
]
|
|
3871
|
-
|
|
3882
|
+
],
|
|
3883
|
+
selected.run
|
|
3884
|
+
);
|
|
3872
3885
|
}
|
|
3873
|
-
|
|
3874
|
-
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
)
|
|
3879
|
-
]
|
|
3886
|
+
const facts = buildFacts(input, selected.run);
|
|
3887
|
+
const context = {
|
|
3888
|
+
...facts,
|
|
3889
|
+
...selected.run ? { selectedRun: selected.run } : {},
|
|
3890
|
+
...input.sourceLabel ? { sourceLabel: input.sourceLabel } : {}
|
|
3880
3891
|
};
|
|
3881
|
-
}
|
|
3882
|
-
function selectRules(rules, selectedIds) {
|
|
3883
3892
|
const diagnostics = [];
|
|
3884
|
-
const
|
|
3885
|
-
|
|
3886
|
-
|
|
3893
|
+
const findings = [];
|
|
3894
|
+
const ruleExecutions = [];
|
|
3895
|
+
for (const rule of rules.rules) {
|
|
3896
|
+
try {
|
|
3897
|
+
const ruleFindings = rule.evaluate(context).map((finding) => normalizeFinding(rule, finding));
|
|
3898
|
+
findings.push(...ruleFindings);
|
|
3899
|
+
ruleExecutions.push({
|
|
3900
|
+
ruleId: rule.id,
|
|
3901
|
+
category: rule.category,
|
|
3902
|
+
status: classifyRuleExecution(ruleFindings, false),
|
|
3903
|
+
findingCount: ruleFindings.length,
|
|
3904
|
+
...selected.run ? { runId: selected.run.runId } : {}
|
|
3905
|
+
});
|
|
3906
|
+
} catch (error) {
|
|
3907
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3887
3908
|
diagnostics.push(
|
|
3888
|
-
diagnostic("
|
|
3909
|
+
diagnostic("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
|
|
3889
3910
|
);
|
|
3890
|
-
|
|
3911
|
+
ruleExecutions.push({
|
|
3912
|
+
ruleId: rule.id,
|
|
3913
|
+
category: rule.category,
|
|
3914
|
+
status: "error",
|
|
3915
|
+
findingCount: 0,
|
|
3916
|
+
...selected.run ? { runId: selected.run.runId } : {}
|
|
3917
|
+
});
|
|
3891
3918
|
}
|
|
3892
|
-
byId.set(rule.id, rule);
|
|
3893
3919
|
}
|
|
3894
|
-
if (
|
|
3895
|
-
|
|
3896
|
-
for (const id of selected) {
|
|
3897
|
-
if (!byId.has(id)) {
|
|
3898
|
-
diagnostics.push(
|
|
3899
|
-
diagnostic("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
3900
|
-
);
|
|
3901
|
-
}
|
|
3902
|
-
}
|
|
3903
|
-
return {
|
|
3904
|
-
rules: [...byId.values()].filter((rule) => selected.has(rule.id)).sort(compareRules),
|
|
3905
|
-
diagnostics
|
|
3906
|
-
};
|
|
3920
|
+
if (diagnostics.length > 0) {
|
|
3921
|
+
return errorResult(input, diagnostics, selected.run, ruleExecutions);
|
|
3907
3922
|
}
|
|
3908
|
-
|
|
3909
|
-
|
|
3910
|
-
|
|
3911
|
-
|
|
3912
|
-
}
|
|
3913
|
-
function eventTimestamp(finding, eventById) {
|
|
3914
|
-
const eventId = finding.evidence[0]?.eventId;
|
|
3915
|
-
return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
|
|
3916
|
-
}
|
|
3917
|
-
function compareFindings(eventById) {
|
|
3918
|
-
return (a, b) => {
|
|
3919
|
-
if (SEVERITY_RANK[a.severity] !== SEVERITY_RANK[b.severity]) {
|
|
3920
|
-
return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity];
|
|
3921
|
-
}
|
|
3922
|
-
const byRule = a.ruleId.localeCompare(b.ruleId);
|
|
3923
|
-
if (byRule !== 0) return byRule;
|
|
3924
|
-
if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
|
|
3925
|
-
return STATUS_RANK[a.status] - STATUS_RANK[b.status];
|
|
3926
|
-
}
|
|
3927
|
-
const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
|
|
3928
|
-
if (byRun !== 0) return byRun;
|
|
3929
|
-
const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
|
|
3930
|
-
if (byTime !== 0) return byTime;
|
|
3931
|
-
const byEvent = compareStrings(a.evidence[0]?.eventId, b.evidence[0]?.eventId);
|
|
3932
|
-
if (byEvent !== 0) return byEvent;
|
|
3933
|
-
return compareStrings(a.evidence[0]?.path, b.evidence[0]?.path);
|
|
3934
|
-
};
|
|
3935
|
-
}
|
|
3936
|
-
function normalizeFinding(rule, finding) {
|
|
3937
|
-
return {
|
|
3938
|
-
ruleId: finding.ruleId || rule.id,
|
|
3939
|
-
severity: finding.severity ?? rule.defaultSeverity,
|
|
3940
|
-
status: finding.status,
|
|
3941
|
-
message: finding.message,
|
|
3942
|
-
...finding.expected !== void 0 ? { expected: finding.expected } : {},
|
|
3943
|
-
...finding.actual !== void 0 ? { actual: finding.actual } : {},
|
|
3944
|
-
evidence: [...finding.evidence ?? []],
|
|
3945
|
-
...finding.category !== void 0 ? { category: finding.category } : {},
|
|
3946
|
-
...finding.confidence !== void 0 ? { confidence: finding.confidence } : {},
|
|
3947
|
-
...finding.detector !== void 0 ? { detector: finding.detector } : {},
|
|
3948
|
-
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
3949
|
-
};
|
|
3950
|
-
}
|
|
3951
|
-
function summarize(findings, diagnostics, rulesEvaluated) {
|
|
3923
|
+
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
3924
|
+
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
3925
|
+
const summary = summarize(sortedFindings, diagnostics, ruleExecutions.length);
|
|
3926
|
+
const status = summary.failed > 0 ? "fail" : "pass";
|
|
3952
3927
|
return {
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
rulesEvaluated
|
|
3928
|
+
ok: status === "pass",
|
|
3929
|
+
status,
|
|
3930
|
+
format: input.read.format,
|
|
3931
|
+
...selected.run ? { runId: selected.run.runId } : {},
|
|
3932
|
+
summary,
|
|
3933
|
+
findings: sortedFindings,
|
|
3934
|
+
diagnostics,
|
|
3935
|
+
ruleExecutions
|
|
3962
3936
|
};
|
|
3963
3937
|
}
|
|
3964
|
-
|
|
3965
|
-
|
|
3966
|
-
|
|
3967
|
-
|
|
3968
|
-
|
|
3969
|
-
|
|
3970
|
-
|
|
3971
|
-
|
|
3938
|
+
|
|
3939
|
+
// packages/core/src/checks/contract.ts
|
|
3940
|
+
new Set(OBSERVED_OUTCOME_METHODS);
|
|
3941
|
+
|
|
3942
|
+
// packages/core/src/exporters/helpers.ts
|
|
3943
|
+
var REDACT_SUBSTRINGS = [
|
|
3944
|
+
"authorization",
|
|
3945
|
+
"cookie",
|
|
3946
|
+
"token",
|
|
3947
|
+
"apikey",
|
|
3948
|
+
"password",
|
|
3949
|
+
"secret",
|
|
3950
|
+
"email"
|
|
3951
|
+
];
|
|
3952
|
+
function shouldRedactKey(key) {
|
|
3953
|
+
const k = key.toLowerCase();
|
|
3954
|
+
for (const s of REDACT_SUBSTRINGS) {
|
|
3955
|
+
if (k.includes(s)) return true;
|
|
3972
3956
|
}
|
|
3973
|
-
return
|
|
3974
|
-
}
|
|
3975
|
-
function eventEvidence(event, path17) {
|
|
3976
|
-
return {
|
|
3977
|
-
runId: event.runId,
|
|
3978
|
-
eventId: event.eventId,
|
|
3979
|
-
parentId: event.parentId,
|
|
3980
|
-
traceId: event.trace?.traceId,
|
|
3981
|
-
spanId: event.trace?.spanId,
|
|
3982
|
-
kind: event.kind,
|
|
3983
|
-
name: event.name,
|
|
3984
|
-
status: event.status,
|
|
3985
|
-
...path17 ? { path: path17 } : {}
|
|
3986
|
-
};
|
|
3987
|
-
}
|
|
3988
|
-
function runEvidence(run) {
|
|
3989
|
-
return run ? [{ runId: run.runId, name: run.name, status: run.status }] : [];
|
|
3990
|
-
}
|
|
3991
|
-
function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
3992
|
-
return {
|
|
3993
|
-
ruleId,
|
|
3994
|
-
severity: meta?.severity ?? "error",
|
|
3995
|
-
status: meta?.status ?? "fail",
|
|
3996
|
-
message,
|
|
3997
|
-
...expected !== void 0 ? { expected } : {},
|
|
3998
|
-
...actual !== void 0 ? { actual } : {},
|
|
3999
|
-
evidence: [...evidence],
|
|
4000
|
-
...meta?.category !== void 0 ? { category: meta.category } : {},
|
|
4001
|
-
...meta?.confidence !== void 0 ? { confidence: meta.confidence } : {},
|
|
4002
|
-
...meta?.detector !== void 0 ? { detector: meta.detector } : {},
|
|
4003
|
-
...meta?.action !== void 0 ? { action: meta.action } : {}
|
|
4004
|
-
};
|
|
4005
|
-
}
|
|
4006
|
-
function semanticEvents(context) {
|
|
4007
|
-
return context.logicalEvents ?? context.events;
|
|
4008
|
-
}
|
|
4009
|
-
function isRecord8(value) {
|
|
4010
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3957
|
+
return false;
|
|
4011
3958
|
}
|
|
4012
|
-
function
|
|
4013
|
-
|
|
3959
|
+
function safeString(value, maxLength) {
|
|
3960
|
+
if (value === null || value === void 0) return "";
|
|
3961
|
+
let s;
|
|
3962
|
+
if (typeof value === "string") s = value;
|
|
3963
|
+
else if (typeof value === "number" || typeof value === "boolean") s = String(value);
|
|
3964
|
+
else s = stableJson(value, false);
|
|
3965
|
+
if (maxLength !== void 0 && maxLength >= 0 && s.length > maxLength) {
|
|
3966
|
+
return `${s.slice(0, maxLength)}\u2026`;
|
|
3967
|
+
}
|
|
3968
|
+
return s;
|
|
4014
3969
|
}
|
|
4015
|
-
function
|
|
4016
|
-
|
|
4017
|
-
return parts[parts.length - 1] ?? path17;
|
|
3970
|
+
function escapeMarkdown(value) {
|
|
3971
|
+
return value.replace(/\|/g, "\\|").replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(/\n/g, " ");
|
|
4018
3972
|
}
|
|
4019
|
-
function
|
|
4020
|
-
|
|
4021
|
-
if (value === null) return "null";
|
|
4022
|
-
return typeof value;
|
|
3973
|
+
function escapeHtml(value) {
|
|
3974
|
+
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
4023
3975
|
}
|
|
4024
|
-
function
|
|
4025
|
-
|
|
4026
|
-
|
|
4027
|
-
|
|
4028
|
-
|
|
3976
|
+
function sortKeysDeep(input) {
|
|
3977
|
+
if (input === null || typeof input !== "object") return input;
|
|
3978
|
+
if (Array.isArray(input)) return input.map(sortKeysDeep);
|
|
3979
|
+
const o = input;
|
|
3980
|
+
const out = {};
|
|
3981
|
+
for (const k of Object.keys(o).sort()) {
|
|
3982
|
+
out[k] = sortKeysDeep(o[k]);
|
|
4029
3983
|
}
|
|
3984
|
+
return out;
|
|
4030
3985
|
}
|
|
4031
|
-
function
|
|
4032
|
-
|
|
4033
|
-
|
|
4034
|
-
|
|
4035
|
-
|
|
4036
|
-
|
|
3986
|
+
function stableJson(value, pretty) {
|
|
3987
|
+
const sorted = sortKeysDeep(value);
|
|
3988
|
+
return pretty === true ? JSON.stringify(sorted, null, 2) : JSON.stringify(sorted);
|
|
3989
|
+
}
|
|
3990
|
+
function compactAttributes(attrs2, options) {
|
|
3991
|
+
if (attrs2 === void 0) return {};
|
|
3992
|
+
const maxLen = options?.maxLength ?? 500;
|
|
3993
|
+
const redacted = options?.redacted ?? true;
|
|
3994
|
+
const out = {};
|
|
3995
|
+
for (const key of Object.keys(attrs2).sort()) {
|
|
3996
|
+
if (redacted && shouldRedactKey(key)) {
|
|
3997
|
+
out[key] = "[REDACTED]";
|
|
3998
|
+
continue;
|
|
4037
3999
|
}
|
|
4038
|
-
|
|
4039
|
-
|
|
4040
|
-
if (!isRecord8(value)) return;
|
|
4041
|
-
for (const nestedKey of Object.keys(value).sort((a, b) => a.localeCompare(b))) {
|
|
4042
|
-
pushValueEntries(
|
|
4043
|
-
entries,
|
|
4044
|
-
event,
|
|
4045
|
-
value[nestedKey],
|
|
4046
|
-
`${path17}.${nestedKey}`,
|
|
4047
|
-
nestedKey,
|
|
4048
|
-
depth + 1
|
|
4049
|
-
);
|
|
4000
|
+
const v = attrs2[key];
|
|
4001
|
+
out[key] = compactValue(v, maxLen, redacted);
|
|
4050
4002
|
}
|
|
4003
|
+
return out;
|
|
4051
4004
|
}
|
|
4052
|
-
function
|
|
4053
|
-
|
|
4054
|
-
|
|
4055
|
-
pushValueEntries(entries, event, event.attributes, "attributes", "attributes");
|
|
4005
|
+
function compactValue(value, maxLen, redacted) {
|
|
4006
|
+
if (value === null || typeof value !== "object") {
|
|
4007
|
+
return typeof value === "string" ? safeString(value, maxLen) : value;
|
|
4056
4008
|
}
|
|
4057
|
-
if (
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
if (event.outputSummary !== void 0) {
|
|
4062
|
-
pushValueEntries(entries, event, event.outputSummary, "outputSummary", "outputSummary");
|
|
4063
|
-
}
|
|
4009
|
+
if (Array.isArray(value)) {
|
|
4010
|
+
const arr = value.slice(0, 20).map((x) => compactValue(x, maxLen, redacted));
|
|
4011
|
+
if (value.length > 20) arr.push(`\u2026(+${value.length - 20} more)`);
|
|
4012
|
+
return arr;
|
|
4064
4013
|
}
|
|
4065
|
-
|
|
4066
|
-
|
|
4014
|
+
const o = value;
|
|
4015
|
+
const inner = {};
|
|
4016
|
+
for (const k of Object.keys(o)) {
|
|
4017
|
+
if (redacted && shouldRedactKey(k)) inner[k] = "[REDACTED]";
|
|
4018
|
+
else inner[k] = compactValue(o[k], maxLen, redacted);
|
|
4067
4019
|
}
|
|
4068
|
-
return
|
|
4069
|
-
}
|
|
4070
|
-
function limitFindings(findings, maxFindings) {
|
|
4071
|
-
if (maxFindings === void 0 || findings.length <= maxFindings) return findings;
|
|
4072
|
-
return findings.slice(0, Math.max(0, maxFindings));
|
|
4073
|
-
}
|
|
4074
|
-
function hasRedactionMarker(value, markers) {
|
|
4075
|
-
return markers.some((marker) => value.includes(marker)) || /^\[HASH:[A-Za-z0-9_-]+\]$/.test(value);
|
|
4076
|
-
}
|
|
4077
|
-
function isSensitiveKey(key, sensitiveKeys) {
|
|
4078
|
-
return isCredentialSensitiveKey(key, sensitiveKeys);
|
|
4020
|
+
return inner;
|
|
4079
4021
|
}
|
|
4080
|
-
function
|
|
4081
|
-
|
|
4082
|
-
|
|
4083
|
-
|
|
4022
|
+
function flattenTree(tree) {
|
|
4023
|
+
const out = [];
|
|
4024
|
+
function walk(nodes) {
|
|
4025
|
+
for (const n of nodes) {
|
|
4026
|
+
out.push(n);
|
|
4027
|
+
if (n.children.length > 0) walk(n.children);
|
|
4028
|
+
}
|
|
4029
|
+
}
|
|
4030
|
+
walk(tree.children);
|
|
4031
|
+
return out;
|
|
4084
4032
|
}
|
|
4085
|
-
|
|
4086
|
-
|
|
4087
|
-
|
|
4088
|
-
|
|
4089
|
-
if (
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
return safePathPrefixes.some((prefix) => parentNorm === normalizedKey(prefix));
|
|
4033
|
+
|
|
4034
|
+
// packages/core/src/diff/comparable.ts
|
|
4035
|
+
function extractOutputPreview(meta) {
|
|
4036
|
+
if (meta === void 0) return void 0;
|
|
4037
|
+
if ("outputPreview" in meta) return meta.outputPreview;
|
|
4038
|
+
if ("resultPreview" in meta) return meta.resultPreview;
|
|
4039
|
+
return void 0;
|
|
4093
4040
|
}
|
|
4094
|
-
function
|
|
4095
|
-
if (
|
|
4096
|
-
return
|
|
4041
|
+
function mapStepStatus(s) {
|
|
4042
|
+
if (s === void 0) return "running";
|
|
4043
|
+
return s;
|
|
4097
4044
|
}
|
|
4098
|
-
function
|
|
4099
|
-
const
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
|
|
4105
|
-
|
|
4106
|
-
|
|
4107
|
-
|
|
4108
|
-
|
|
4109
|
-
|
|
4110
|
-
|
|
4111
|
-
|
|
4112
|
-
|
|
4113
|
-
|
|
4114
|
-
|
|
4115
|
-
|
|
4116
|
-
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
4124
|
-
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
|
|
4132
|
-
|
|
4133
|
-
|
|
4045
|
+
function manualTraceEventsToComparableRun(events) {
|
|
4046
|
+
const started = events.find((e) => e.event === "run_started");
|
|
4047
|
+
if (!started || started.event !== "run_started") {
|
|
4048
|
+
throw new Error("Invalid trace: missing run_started");
|
|
4049
|
+
}
|
|
4050
|
+
const rs = started;
|
|
4051
|
+
const runId = rs.runId;
|
|
4052
|
+
const completedAll = events.filter((e) => e.event === "run_completed");
|
|
4053
|
+
const lastCompleted = completedAll[completedAll.length - 1];
|
|
4054
|
+
let runStatus;
|
|
4055
|
+
if (lastCompleted === void 0) runStatus = "running";
|
|
4056
|
+
else runStatus = lastCompleted.status;
|
|
4057
|
+
const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
4058
|
+
const steps = /* @__PURE__ */ new Map();
|
|
4059
|
+
let order = 0;
|
|
4060
|
+
for (const e of events) {
|
|
4061
|
+
if (e.event !== "step_started") continue;
|
|
4062
|
+
const s = e;
|
|
4063
|
+
const meta = s.metadata ? { ...s.metadata } : void 0;
|
|
4064
|
+
steps.set(s.stepId, {
|
|
4065
|
+
id: s.stepId,
|
|
4066
|
+
parentId: s.parentId,
|
|
4067
|
+
name: s.name,
|
|
4068
|
+
type: s.type,
|
|
4069
|
+
order: order++,
|
|
4070
|
+
timestamp: s.timestamp,
|
|
4071
|
+
metadata: meta
|
|
4072
|
+
});
|
|
4073
|
+
}
|
|
4074
|
+
for (const e of events) {
|
|
4075
|
+
if (e.event !== "step_completed") continue;
|
|
4076
|
+
const acc = steps.get(e.stepId);
|
|
4077
|
+
if (!acc) continue;
|
|
4078
|
+
acc.status = e.status;
|
|
4079
|
+
acc.durationMs = e.durationMs;
|
|
4080
|
+
if (e.error?.message) acc.errorMsg = e.error.message;
|
|
4081
|
+
const extra = e;
|
|
4082
|
+
if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
|
|
4083
|
+
acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
|
|
4084
|
+
}
|
|
4085
|
+
}
|
|
4086
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
4087
|
+
for (const acc of steps.values()) {
|
|
4088
|
+
let meta = acc.metadata ? { ...acc.metadata } : void 0;
|
|
4089
|
+
if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
|
|
4090
|
+
meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
|
|
4091
|
+
}
|
|
4092
|
+
const outputPreview = extractOutputPreview(meta);
|
|
4093
|
+
if (meta !== void 0 && ("outputPreview" in meta || "resultPreview" in meta)) {
|
|
4094
|
+
delete meta.outputPreview;
|
|
4095
|
+
delete meta.resultPreview;
|
|
4096
|
+
}
|
|
4097
|
+
const sc = {
|
|
4098
|
+
id: acc.id,
|
|
4099
|
+
name: acc.name,
|
|
4100
|
+
type: acc.type,
|
|
4101
|
+
status: mapStepStatus(acc.status),
|
|
4102
|
+
durationMs: acc.durationMs,
|
|
4103
|
+
error: acc.errorMsg,
|
|
4104
|
+
metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
|
|
4105
|
+
outputPreview,
|
|
4106
|
+
children: []
|
|
4107
|
+
};
|
|
4108
|
+
nodes.set(acc.id, sc);
|
|
4109
|
+
}
|
|
4110
|
+
const roots = [];
|
|
4111
|
+
const sortByOrder = (a, b) => {
|
|
4112
|
+
const oa = steps.get(a.id)?.order ?? 0;
|
|
4113
|
+
const ob = steps.get(b.id)?.order ?? 0;
|
|
4114
|
+
return oa - ob;
|
|
4115
|
+
};
|
|
4116
|
+
for (const acc of steps.values()) {
|
|
4117
|
+
const node = nodes.get(acc.id);
|
|
4118
|
+
if (acc.parentId !== void 0 && nodes.has(acc.parentId)) {
|
|
4119
|
+
nodes.get(acc.parentId).children.push(node);
|
|
4120
|
+
} else {
|
|
4121
|
+
roots.push(node);
|
|
4134
4122
|
}
|
|
4123
|
+
}
|
|
4124
|
+
roots.sort(sortByOrder);
|
|
4125
|
+
for (const n of nodes.values()) {
|
|
4126
|
+
n.children.sort(sortByOrder);
|
|
4127
|
+
}
|
|
4128
|
+
return {
|
|
4129
|
+
runId,
|
|
4130
|
+
name: rs.name,
|
|
4131
|
+
status: runStatus,
|
|
4132
|
+
durationMs,
|
|
4133
|
+
steps: roots
|
|
4135
4134
|
};
|
|
4136
4135
|
}
|
|
4137
|
-
|
|
4138
|
-
|
|
4139
|
-
|
|
4140
|
-
|
|
4141
|
-
|
|
4142
|
-
|
|
4143
|
-
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
|
|
4147
|
-
|
|
4148
|
-
|
|
4149
|
-
|
|
4150
|
-
|
|
4151
|
-
|
|
4152
|
-
|
|
4153
|
-
|
|
4154
|
-
|
|
4155
|
-
|
|
4156
|
-
{ path: entry.path, valueType: valueType(entry.value) },
|
|
4157
|
-
{
|
|
4158
|
-
category: "credential",
|
|
4159
|
-
confidence: "high",
|
|
4160
|
-
detector: "safety.redaction",
|
|
4161
|
-
action: "redact"
|
|
4162
|
-
}
|
|
4163
|
-
)
|
|
4164
|
-
);
|
|
4165
|
-
}
|
|
4136
|
+
|
|
4137
|
+
// packages/core/src/diff/engine.ts
|
|
4138
|
+
var DEFAULT_THRESHOLD_MS = 0;
|
|
4139
|
+
function pathSeg(step, index) {
|
|
4140
|
+
return { index, name: step.name, stepId: step.id };
|
|
4141
|
+
}
|
|
4142
|
+
function buildPath(segments) {
|
|
4143
|
+
return { path: [...segments] };
|
|
4144
|
+
}
|
|
4145
|
+
function pairSteps(left, right) {
|
|
4146
|
+
const usedRight = /* @__PURE__ */ new Set();
|
|
4147
|
+
const pairs = [];
|
|
4148
|
+
for (let i = 0; i < left.length; i++) {
|
|
4149
|
+
const L = left[i];
|
|
4150
|
+
let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
|
|
4151
|
+
if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
|
|
4152
|
+
const cand = right[i];
|
|
4153
|
+
if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
|
|
4154
|
+
R = cand;
|
|
4166
4155
|
}
|
|
4167
|
-
return limitFindings(findings, options.maxFindings);
|
|
4168
4156
|
}
|
|
4169
|
-
|
|
4157
|
+
if (R === void 0) {
|
|
4158
|
+
R = right.find(
|
|
4159
|
+
(r) => !usedRight.has(r.id) && r.name === L.name && (r.type ?? "") === (L.type ?? "")
|
|
4160
|
+
);
|
|
4161
|
+
}
|
|
4162
|
+
if (R !== void 0) {
|
|
4163
|
+
usedRight.add(R.id);
|
|
4164
|
+
pairs.push([L, R]);
|
|
4165
|
+
} else {
|
|
4166
|
+
pairs.push([L, void 0]);
|
|
4167
|
+
}
|
|
4168
|
+
}
|
|
4169
|
+
for (const R of right) {
|
|
4170
|
+
if (!usedRight.has(R.id)) {
|
|
4171
|
+
pairs.push([void 0, R]);
|
|
4172
|
+
}
|
|
4173
|
+
}
|
|
4174
|
+
return pairs;
|
|
4170
4175
|
}
|
|
4171
|
-
function
|
|
4172
|
-
const
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
|
|
4180
|
-
|
|
4181
|
-
|
|
4182
|
-
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
|
|
4186
|
-
|
|
4187
|
-
|
|
4188
|
-
|
|
4189
|
-
|
|
4190
|
-
|
|
4191
|
-
|
|
4192
|
-
|
|
4193
|
-
|
|
4194
|
-
|
|
4195
|
-
|
|
4196
|
-
|
|
4197
|
-
|
|
4198
|
-
|
|
4199
|
-
|
|
4200
|
-
|
|
4201
|
-
|
|
4176
|
+
function compareLeafSteps(L, R, segments, opts, out) {
|
|
4177
|
+
const path17 = buildPath(segments);
|
|
4178
|
+
if (L.name !== R.name) {
|
|
4179
|
+
out.push({
|
|
4180
|
+
kind: "structure",
|
|
4181
|
+
severity: "warning",
|
|
4182
|
+
message: "Step name differs",
|
|
4183
|
+
path: path17,
|
|
4184
|
+
left: L.name,
|
|
4185
|
+
right: R.name
|
|
4186
|
+
});
|
|
4187
|
+
}
|
|
4188
|
+
if ((L.type ?? "") !== (R.type ?? "")) {
|
|
4189
|
+
out.push({
|
|
4190
|
+
kind: "step-type",
|
|
4191
|
+
severity: "warning",
|
|
4192
|
+
message: "Step type differs",
|
|
4193
|
+
path: path17,
|
|
4194
|
+
left: L.type,
|
|
4195
|
+
right: R.type
|
|
4196
|
+
});
|
|
4197
|
+
}
|
|
4198
|
+
if ((L.status ?? "") !== (R.status ?? "")) {
|
|
4199
|
+
out.push({
|
|
4200
|
+
kind: "step-status",
|
|
4201
|
+
severity: "warning",
|
|
4202
|
+
message: "Step status differs",
|
|
4203
|
+
path: path17,
|
|
4204
|
+
left: L.status,
|
|
4205
|
+
right: R.status
|
|
4206
|
+
});
|
|
4207
|
+
}
|
|
4208
|
+
const le = L.error ?? "";
|
|
4209
|
+
const re = R.error ?? "";
|
|
4210
|
+
if (le !== re) {
|
|
4211
|
+
out.push({
|
|
4212
|
+
kind: "error",
|
|
4213
|
+
severity: "error",
|
|
4214
|
+
message: "Step error message differs",
|
|
4215
|
+
path: path17,
|
|
4216
|
+
left: le || void 0,
|
|
4217
|
+
right: re || void 0
|
|
4218
|
+
});
|
|
4219
|
+
}
|
|
4220
|
+
if (!opts.ignoreDuration) {
|
|
4221
|
+
const ld = L.durationMs;
|
|
4222
|
+
const rd = R.durationMs;
|
|
4223
|
+
const th = opts.durationThresholdMs;
|
|
4224
|
+
let differs = false;
|
|
4225
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
4226
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
4227
|
+
else differs = Math.abs(ld - rd) > th;
|
|
4228
|
+
if (differs) {
|
|
4229
|
+
out.push({
|
|
4230
|
+
kind: "duration",
|
|
4231
|
+
severity: "info",
|
|
4232
|
+
message: "Step duration differs",
|
|
4233
|
+
path: path17,
|
|
4234
|
+
left: ld,
|
|
4235
|
+
right: rd
|
|
4236
|
+
});
|
|
4202
4237
|
}
|
|
4203
|
-
}
|
|
4238
|
+
}
|
|
4239
|
+
const lm = stableJson(L.metadata ?? {});
|
|
4240
|
+
const rm = stableJson(R.metadata ?? {});
|
|
4241
|
+
if (lm !== rm) {
|
|
4242
|
+
out.push({
|
|
4243
|
+
kind: "metadata",
|
|
4244
|
+
severity: "info",
|
|
4245
|
+
message: "Step metadata differs",
|
|
4246
|
+
path: path17,
|
|
4247
|
+
left: L.metadata,
|
|
4248
|
+
right: R.metadata
|
|
4249
|
+
});
|
|
4250
|
+
}
|
|
4251
|
+
const lo = stableJson(L.outputPreview ?? null);
|
|
4252
|
+
const ro = stableJson(R.outputPreview ?? null);
|
|
4253
|
+
if (lo !== ro) {
|
|
4254
|
+
out.push({
|
|
4255
|
+
kind: "output",
|
|
4256
|
+
severity: "info",
|
|
4257
|
+
message: "Output preview differs",
|
|
4258
|
+
path: path17,
|
|
4259
|
+
left: L.outputPreview,
|
|
4260
|
+
right: R.outputPreview
|
|
4261
|
+
});
|
|
4262
|
+
}
|
|
4204
4263
|
}
|
|
4205
|
-
function
|
|
4206
|
-
|
|
4207
|
-
const
|
|
4208
|
-
|
|
4209
|
-
|
|
4210
|
-
|
|
4211
|
-
|
|
4212
|
-
|
|
4213
|
-
|
|
4214
|
-
|
|
4215
|
-
|
|
4216
|
-
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
|
|
4224
|
-
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
|
|
4230
|
-
category: "credential",
|
|
4231
|
-
confidence: "high",
|
|
4232
|
-
detector: pattern.id,
|
|
4233
|
-
action: "redact"
|
|
4234
|
-
}
|
|
4235
|
-
)
|
|
4236
|
-
);
|
|
4237
|
-
break;
|
|
4238
|
-
}
|
|
4239
|
-
}
|
|
4240
|
-
}
|
|
4241
|
-
return limitFindings(findings, options.maxFindings);
|
|
4264
|
+
function compareRecursive(L, R, segments, opts, out) {
|
|
4265
|
+
compareLeafSteps(L, R, segments, opts, out);
|
|
4266
|
+
const pairs = pairSteps(L.children, R.children);
|
|
4267
|
+
let ci = 0;
|
|
4268
|
+
for (const [lch, rch] of pairs) {
|
|
4269
|
+
if (lch !== void 0 && rch !== void 0) {
|
|
4270
|
+
compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
|
|
4271
|
+
} else if (lch !== void 0) {
|
|
4272
|
+
out.push({
|
|
4273
|
+
kind: "step-removed",
|
|
4274
|
+
severity: "warning",
|
|
4275
|
+
message: `Step only in left run: ${lch.name}`,
|
|
4276
|
+
path: buildPath([...segments, pathSeg(lch, ci)]),
|
|
4277
|
+
left: lch.id,
|
|
4278
|
+
right: void 0
|
|
4279
|
+
});
|
|
4280
|
+
} else if (rch !== void 0) {
|
|
4281
|
+
out.push({
|
|
4282
|
+
kind: "step-added",
|
|
4283
|
+
severity: "warning",
|
|
4284
|
+
message: `Step only in right run: ${rch.name}`,
|
|
4285
|
+
path: buildPath([...segments, pathSeg(rch, ci)]),
|
|
4286
|
+
left: void 0,
|
|
4287
|
+
right: rch.id
|
|
4288
|
+
});
|
|
4242
4289
|
}
|
|
4243
|
-
|
|
4290
|
+
ci += 1;
|
|
4291
|
+
}
|
|
4244
4292
|
}
|
|
4245
|
-
function
|
|
4293
|
+
function mergeDiffDefaults(options) {
|
|
4246
4294
|
return {
|
|
4247
|
-
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
const findings = [];
|
|
4252
|
-
for (const event of context.events) {
|
|
4253
|
-
for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
|
|
4254
|
-
if (typeof entry.value === "string" && options.maxStringLength !== void 0 && entry.value.length > options.maxStringLength) {
|
|
4255
|
-
findings.push(
|
|
4256
|
-
failFinding(
|
|
4257
|
-
"safety.oversizedAttribute",
|
|
4258
|
-
`String at ${entry.path} exceeds ${options.maxStringLength} characters.`,
|
|
4259
|
-
[eventEvidence(event, entry.path)],
|
|
4260
|
-
{ maxStringLength: options.maxStringLength },
|
|
4261
|
-
{ path: entry.path, length: entry.value.length },
|
|
4262
|
-
{
|
|
4263
|
-
category: "size",
|
|
4264
|
-
confidence: "high",
|
|
4265
|
-
detector: "safety.oversizedAttribute",
|
|
4266
|
-
action: "truncate-or-omit"
|
|
4267
|
-
}
|
|
4268
|
-
)
|
|
4269
|
-
);
|
|
4270
|
-
}
|
|
4271
|
-
if (Array.isArray(entry.value) && options.maxArrayLength !== void 0 && entry.value.length > options.maxArrayLength) {
|
|
4272
|
-
findings.push(
|
|
4273
|
-
failFinding(
|
|
4274
|
-
"safety.oversizedAttribute",
|
|
4275
|
-
`Array at ${entry.path} exceeds ${options.maxArrayLength} items.`,
|
|
4276
|
-
[eventEvidence(event, entry.path)],
|
|
4277
|
-
{ maxArrayLength: options.maxArrayLength },
|
|
4278
|
-
{ path: entry.path, length: entry.value.length },
|
|
4279
|
-
{
|
|
4280
|
-
category: "size",
|
|
4281
|
-
confidence: "high",
|
|
4282
|
-
detector: "safety.oversizedAttribute",
|
|
4283
|
-
action: "truncate-or-omit"
|
|
4284
|
-
}
|
|
4285
|
-
)
|
|
4286
|
-
);
|
|
4287
|
-
}
|
|
4288
|
-
if (isRecord8(entry.value) && options.maxObjectKeys !== void 0 && Object.keys(entry.value).length > options.maxObjectKeys) {
|
|
4289
|
-
findings.push(
|
|
4290
|
-
failFinding(
|
|
4291
|
-
"safety.oversizedAttribute",
|
|
4292
|
-
`Object at ${entry.path} exceeds ${options.maxObjectKeys} keys.`,
|
|
4293
|
-
[eventEvidence(event, entry.path)],
|
|
4294
|
-
{ maxObjectKeys: options.maxObjectKeys },
|
|
4295
|
-
{ path: entry.path, keys: Object.keys(entry.value).length },
|
|
4296
|
-
{
|
|
4297
|
-
category: "size",
|
|
4298
|
-
confidence: "high",
|
|
4299
|
-
detector: "safety.oversizedAttribute",
|
|
4300
|
-
action: "truncate-or-omit"
|
|
4301
|
-
}
|
|
4302
|
-
)
|
|
4303
|
-
);
|
|
4304
|
-
}
|
|
4305
|
-
if (options.maxSerializedBytes !== void 0) {
|
|
4306
|
-
const bytes = serializedByteLength(entry.value);
|
|
4307
|
-
if (bytes !== void 0 && bytes > options.maxSerializedBytes) {
|
|
4308
|
-
findings.push(
|
|
4309
|
-
failFinding(
|
|
4310
|
-
"safety.oversizedAttribute",
|
|
4311
|
-
`Value at ${entry.path} exceeds ${options.maxSerializedBytes} serialized bytes.`,
|
|
4312
|
-
[eventEvidence(event, entry.path)],
|
|
4313
|
-
{ maxSerializedBytes: options.maxSerializedBytes },
|
|
4314
|
-
{ path: entry.path, bytes },
|
|
4315
|
-
{
|
|
4316
|
-
category: "size",
|
|
4317
|
-
confidence: "high",
|
|
4318
|
-
detector: "safety.oversizedAttribute",
|
|
4319
|
-
action: "truncate-or-omit"
|
|
4320
|
-
}
|
|
4321
|
-
)
|
|
4322
|
-
);
|
|
4323
|
-
}
|
|
4324
|
-
}
|
|
4325
|
-
}
|
|
4326
|
-
}
|
|
4327
|
-
return limitFindings(findings, options.maxFindings);
|
|
4328
|
-
}
|
|
4295
|
+
ignoreDuration: false,
|
|
4296
|
+
durationThresholdMs: DEFAULT_THRESHOLD_MS,
|
|
4297
|
+
focus: "all",
|
|
4298
|
+
check: "all"
|
|
4329
4299
|
};
|
|
4330
4300
|
}
|
|
4331
|
-
function
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
|
|
4336
|
-
const
|
|
4337
|
-
|
|
4338
|
-
|
|
4301
|
+
function kindMatchesFilter(kind, merged) {
|
|
4302
|
+
return true;
|
|
4303
|
+
}
|
|
4304
|
+
function diffRuns(left, right, options) {
|
|
4305
|
+
const merged = mergeDiffDefaults();
|
|
4306
|
+
const opts = {
|
|
4307
|
+
ignoreDuration: merged.ignoreDuration,
|
|
4308
|
+
durationThresholdMs: merged.durationThresholdMs
|
|
4309
|
+
};
|
|
4310
|
+
const raw = [];
|
|
4311
|
+
if ((left.status ?? "") !== (right.status ?? "")) {
|
|
4312
|
+
raw.push({
|
|
4313
|
+
kind: "run-status",
|
|
4314
|
+
severity: "warning",
|
|
4315
|
+
message: "Run completion status differs",
|
|
4316
|
+
left: left.status,
|
|
4317
|
+
right: right.status
|
|
4318
|
+
});
|
|
4339
4319
|
}
|
|
4340
|
-
|
|
4341
|
-
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4320
|
+
{
|
|
4321
|
+
const ld = left.durationMs;
|
|
4322
|
+
const rd = right.durationMs;
|
|
4323
|
+
const th = merged.durationThresholdMs;
|
|
4324
|
+
let differs = false;
|
|
4325
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
4326
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
4327
|
+
else differs = Math.abs(ld - rd) > th;
|
|
4328
|
+
if (differs) {
|
|
4329
|
+
raw.push({
|
|
4330
|
+
kind: "duration",
|
|
4331
|
+
severity: "info",
|
|
4332
|
+
message: "Run duration differs",
|
|
4333
|
+
left: ld,
|
|
4334
|
+
right: rd
|
|
4335
|
+
});
|
|
4336
|
+
}
|
|
4351
4337
|
}
|
|
4352
|
-
const
|
|
4353
|
-
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4366
|
-
ruleId: rule.id,
|
|
4367
|
-
category: rule.category,
|
|
4368
|
-
status: classifyRuleExecution(ruleFindings, false),
|
|
4369
|
-
findingCount: ruleFindings.length,
|
|
4370
|
-
...selected.run ? { runId: selected.run.runId } : {}
|
|
4338
|
+
const pairs = pairSteps(left.steps, right.steps);
|
|
4339
|
+
let idx = 0;
|
|
4340
|
+
for (const [ls, rs] of pairs) {
|
|
4341
|
+
if (ls !== void 0 && rs !== void 0) {
|
|
4342
|
+
compareRecursive(ls, rs, [pathSeg(ls, idx)], opts, raw);
|
|
4343
|
+
idx += 1;
|
|
4344
|
+
} else if (ls !== void 0) {
|
|
4345
|
+
raw.push({
|
|
4346
|
+
kind: "step-removed",
|
|
4347
|
+
severity: "warning",
|
|
4348
|
+
message: `Step only in left run: ${ls.name}`,
|
|
4349
|
+
path: buildPath([pathSeg(ls, idx)]),
|
|
4350
|
+
left: ls.id,
|
|
4351
|
+
right: void 0
|
|
4371
4352
|
});
|
|
4372
|
-
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
|
|
4381
|
-
findingCount: 0,
|
|
4382
|
-
...selected.run ? { runId: selected.run.runId } : {}
|
|
4353
|
+
idx += 1;
|
|
4354
|
+
} else if (rs !== void 0) {
|
|
4355
|
+
raw.push({
|
|
4356
|
+
kind: "step-added",
|
|
4357
|
+
severity: "warning",
|
|
4358
|
+
message: `Step only in right run: ${rs.name}`,
|
|
4359
|
+
path: buildPath([pathSeg(rs, idx)]),
|
|
4360
|
+
left: void 0,
|
|
4361
|
+
right: rs.id
|
|
4383
4362
|
});
|
|
4363
|
+
idx += 1;
|
|
4384
4364
|
}
|
|
4385
4365
|
}
|
|
4386
|
-
|
|
4387
|
-
|
|
4366
|
+
const differences = raw.filter((d) => kindMatchesFilter(d.kind));
|
|
4367
|
+
let errors = 0;
|
|
4368
|
+
let warnings = 0;
|
|
4369
|
+
let info = 0;
|
|
4370
|
+
for (const d of differences) {
|
|
4371
|
+
if (d.severity === "error") errors += 1;
|
|
4372
|
+
else if (d.severity === "warning") warnings += 1;
|
|
4373
|
+
else info += 1;
|
|
4388
4374
|
}
|
|
4389
|
-
const
|
|
4390
|
-
const
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
|
|
4375
|
+
const firstVisible = differences[0];
|
|
4376
|
+
const firstDivergence = firstVisible !== void 0 ? {
|
|
4377
|
+
kind: "first-divergence",
|
|
4378
|
+
severity: firstVisible.severity,
|
|
4379
|
+
message: `First divergence: ${firstVisible.message}`,
|
|
4380
|
+
path: firstVisible.path,
|
|
4381
|
+
left: firstVisible.left,
|
|
4382
|
+
right: firstVisible.right
|
|
4383
|
+
} : void 0;
|
|
4384
|
+
const summary = {
|
|
4385
|
+
leftRunId: left.runId,
|
|
4386
|
+
rightRunId: right.runId,
|
|
4387
|
+
totalDifferences: differences.length,
|
|
4388
|
+
errors,
|
|
4389
|
+
warnings,
|
|
4390
|
+
info,
|
|
4391
|
+
firstDivergence
|
|
4402
4392
|
};
|
|
4393
|
+
return { summary, differences };
|
|
4403
4394
|
}
|
|
4404
4395
|
|
|
4396
|
+
// packages/core/src/evidence/zip.ts
|
|
4397
|
+
(() => {
|
|
4398
|
+
const table = new Uint32Array(256);
|
|
4399
|
+
for (let n = 0; n < 256; n += 1) {
|
|
4400
|
+
let c = n;
|
|
4401
|
+
for (let k = 0; k < 8; k += 1) {
|
|
4402
|
+
c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
|
|
4403
|
+
}
|
|
4404
|
+
table[n] = c >>> 0;
|
|
4405
|
+
}
|
|
4406
|
+
return table;
|
|
4407
|
+
})();
|
|
4408
|
+
|
|
4405
4409
|
// packages/core/src/persisted/token-usage.ts
|
|
4406
4410
|
function isRecord9(value) {
|
|
4407
4411
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|