@agent-inspect/mcp-server 6.27.0 → 6.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-2E7RWC2E.mjs → chunk-V6J5UBTW.mjs} +1942 -1938
- package/dist/chunk-V6J5UBTW.mjs.map +1 -0
- package/dist/cli.cjs +1940 -1936
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.mjs +1 -1
- package/dist/index.cjs +1940 -1936
- package/dist/index.cjs.map +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +3 -3
- package/dist/chunk-2E7RWC2E.mjs.map +0 -1
|
@@ -2140,6 +2140,9 @@ function inferEvidenceFileRole(relativePath) {
|
|
|
2140
2140
|
if (base === "check-results.json") {
|
|
2141
2141
|
return "checks";
|
|
2142
2142
|
}
|
|
2143
|
+
if (base === "contract.resolved.json") {
|
|
2144
|
+
return "contract";
|
|
2145
|
+
}
|
|
2143
2146
|
if (base === "redaction-report.json") {
|
|
2144
2147
|
return "redaction-report";
|
|
2145
2148
|
}
|
|
@@ -2206,670 +2209,204 @@ function buildEvidenceManifest(parts) {
|
|
|
2206
2209
|
},
|
|
2207
2210
|
assessment,
|
|
2208
2211
|
...parts.semantics !== void 0 ? { semantics: { ...parts.semantics } } : {},
|
|
2212
|
+
...parts.contract !== void 0 ? { contract: { ...parts.contract } } : {},
|
|
2209
2213
|
files: buildEvidenceFileEntries(parts.files)
|
|
2210
2214
|
};
|
|
2211
2215
|
}
|
|
2212
2216
|
|
|
2213
|
-
// packages/core/src/
|
|
2214
|
-
var
|
|
2217
|
+
// packages/core/src/diagnostics/programmatic.ts
|
|
2218
|
+
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
2219
|
+
AI_TRACE_INPUT_INVALID: {
|
|
2220
|
+
code: "AI_TRACE_INPUT_INVALID",
|
|
2221
|
+
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
2222
|
+
remediation: "For a file path, use openTraceFile(path).",
|
|
2223
|
+
relatedCodes: ["invalid_input"]
|
|
2224
|
+
},
|
|
2225
|
+
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
2226
|
+
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
2227
|
+
summary: "No trace reader could detect the input format.",
|
|
2228
|
+
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
2229
|
+
relatedCodes: ["unsupported_format"]
|
|
2230
|
+
},
|
|
2231
|
+
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
2232
|
+
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
2233
|
+
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
2234
|
+
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
2235
|
+
relatedCodes: ["ambiguous_format"]
|
|
2236
|
+
},
|
|
2237
|
+
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
2238
|
+
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
2239
|
+
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
2240
|
+
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
2241
|
+
},
|
|
2242
|
+
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
2243
|
+
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
2244
|
+
summary: "Multiple runs are available; select a run before executing checks.",
|
|
2245
|
+
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
2246
|
+
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
2247
|
+
},
|
|
2248
|
+
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
2249
|
+
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2250
|
+
summary: "A parentId equals its own event/step id.",
|
|
2251
|
+
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
2252
|
+
relatedCodes: [
|
|
2253
|
+
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
2254
|
+
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
2255
|
+
]
|
|
2256
|
+
},
|
|
2257
|
+
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
2258
|
+
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
2259
|
+
summary: "Trace contains a parentId cycle.",
|
|
2260
|
+
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
2261
|
+
relatedCodes: ["structure.cycle"]
|
|
2262
|
+
}
|
|
2263
|
+
});
|
|
2264
|
+
function formatProgrammaticDiagnostic(code, detail) {
|
|
2265
|
+
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
2266
|
+
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
2267
|
+
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
2268
|
+
}
|
|
2269
|
+
|
|
2270
|
+
// packages/core/src/safety/sensitive-key.ts
|
|
2271
|
+
function keyHasExplicitSeparator(value) {
|
|
2272
|
+
return /[_\-.]/.test(value);
|
|
2273
|
+
}
|
|
2274
|
+
function normalizeSensitiveKey(value) {
|
|
2275
|
+
return value.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
2276
|
+
}
|
|
2277
|
+
var NON_CREDENTIAL_TOKEN_CONFIG_KEYS = new Set(
|
|
2278
|
+
[
|
|
2279
|
+
"tokens",
|
|
2280
|
+
"max_tokens",
|
|
2281
|
+
"min_tokens",
|
|
2282
|
+
"ls_max_tokens",
|
|
2283
|
+
"token_count",
|
|
2284
|
+
"token_limit",
|
|
2285
|
+
"token_budget",
|
|
2286
|
+
"input_tokens",
|
|
2287
|
+
"output_tokens",
|
|
2288
|
+
"total_tokens",
|
|
2289
|
+
"cached_tokens",
|
|
2290
|
+
"prompt_tokens",
|
|
2291
|
+
"completion_tokens"
|
|
2292
|
+
].map(normalizeSensitiveKey)
|
|
2293
|
+
);
|
|
2294
|
+
var DEFAULT_CREDENTIAL_SENSITIVE_KEYS = [
|
|
2215
2295
|
"authorization",
|
|
2216
2296
|
"cookie",
|
|
2217
2297
|
"token",
|
|
2298
|
+
"access_token",
|
|
2299
|
+
"accesstoken",
|
|
2300
|
+
"auth_token",
|
|
2301
|
+
"authtoken",
|
|
2302
|
+
"refresh_token",
|
|
2303
|
+
"refreshtoken",
|
|
2304
|
+
"id_token",
|
|
2305
|
+
"idtoken",
|
|
2306
|
+
"bearer_token",
|
|
2307
|
+
"bearertoken",
|
|
2308
|
+
"api_token",
|
|
2309
|
+
"apitoken",
|
|
2218
2310
|
"apikey",
|
|
2311
|
+
"api_key",
|
|
2219
2312
|
"password",
|
|
2220
2313
|
"secret",
|
|
2221
2314
|
"email"
|
|
2222
2315
|
];
|
|
2223
|
-
function
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
return false;
|
|
2316
|
+
function isTokenCredentialKey(normalized) {
|
|
2317
|
+
if (NON_CREDENTIAL_TOKEN_CONFIG_KEYS.has(normalized)) return false;
|
|
2318
|
+
if (normalized === "token") return true;
|
|
2319
|
+
if (normalized.endsWith("tokens")) return false;
|
|
2320
|
+
return normalized.endsWith("token");
|
|
2229
2321
|
}
|
|
2230
|
-
function
|
|
2231
|
-
if (
|
|
2232
|
-
|
|
2233
|
-
if (
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2322
|
+
function isCredentialSensitiveKey(key, sensitiveKeys = DEFAULT_CREDENTIAL_SENSITIVE_KEYS) {
|
|
2323
|
+
if (!key) return false;
|
|
2324
|
+
const normalized = normalizeSensitiveKey(key);
|
|
2325
|
+
if (!normalized) return false;
|
|
2326
|
+
if (NON_CREDENTIAL_TOKEN_CONFIG_KEYS.has(normalized)) return false;
|
|
2327
|
+
const allowPrefixCompound = keyHasExplicitSeparator(key);
|
|
2328
|
+
for (const sensitive of sensitiveKeys) {
|
|
2329
|
+
const s = normalizeSensitiveKey(sensitive);
|
|
2330
|
+
if (!s) continue;
|
|
2331
|
+
if (s === "token") {
|
|
2332
|
+
if (isTokenCredentialKey(normalized)) return true;
|
|
2333
|
+
continue;
|
|
2334
|
+
}
|
|
2335
|
+
if (normalized === s) return true;
|
|
2336
|
+
if (normalized.endsWith(`_${s}`)) return true;
|
|
2337
|
+
if (allowPrefixCompound && normalized.startsWith(`${s}_`)) return true;
|
|
2238
2338
|
}
|
|
2239
|
-
return
|
|
2240
|
-
}
|
|
2241
|
-
function escapeMarkdown(value) {
|
|
2242
|
-
return value.replace(/\|/g, "\\|").replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(/\n/g, " ");
|
|
2339
|
+
return false;
|
|
2243
2340
|
}
|
|
2244
|
-
|
|
2245
|
-
|
|
2341
|
+
|
|
2342
|
+
// packages/core/src/checks/logical-events.ts
|
|
2343
|
+
function isRecord6(value) {
|
|
2344
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2246
2345
|
}
|
|
2247
|
-
function
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
const o = input;
|
|
2251
|
-
const out = {};
|
|
2252
|
-
for (const k of Object.keys(o).sort()) {
|
|
2253
|
-
out[k] = sortKeysDeep(o[k]);
|
|
2254
|
-
}
|
|
2255
|
-
return out;
|
|
2346
|
+
function legacyEvent(event) {
|
|
2347
|
+
const value = event.attributes?.legacyEvent;
|
|
2348
|
+
return typeof value === "string" ? value : void 0;
|
|
2256
2349
|
}
|
|
2257
|
-
function
|
|
2258
|
-
const
|
|
2259
|
-
|
|
2350
|
+
function stepIdOf(event) {
|
|
2351
|
+
const value = event.attributes?.stepId;
|
|
2352
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2353
|
+
return void 0;
|
|
2260
2354
|
}
|
|
2261
|
-
function
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
continue;
|
|
2270
|
-
}
|
|
2271
|
-
const v = attrs2[key];
|
|
2272
|
-
out[key] = compactValue(v, maxLen, redacted);
|
|
2273
|
-
}
|
|
2274
|
-
return out;
|
|
2355
|
+
function cloneEvent(event) {
|
|
2356
|
+
return {
|
|
2357
|
+
...event,
|
|
2358
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
2359
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
2360
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
2361
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
2362
|
+
};
|
|
2275
2363
|
}
|
|
2276
|
-
function
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
const o = value;
|
|
2286
|
-
const inner = {};
|
|
2287
|
-
for (const k of Object.keys(o)) {
|
|
2288
|
-
if (redacted && shouldRedactKey(k)) inner[k] = "[REDACTED]";
|
|
2289
|
-
else inner[k] = compactValue(o[k], maxLen, redacted);
|
|
2364
|
+
function mergeAttributes(start, complete) {
|
|
2365
|
+
const merged = {
|
|
2366
|
+
...isRecord6(complete.attributes) ? complete.attributes : {},
|
|
2367
|
+
...isRecord6(start.attributes) ? start.attributes : {}
|
|
2368
|
+
};
|
|
2369
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
2370
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
2371
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
2372
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
2290
2373
|
}
|
|
2291
|
-
return
|
|
2374
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
2292
2375
|
}
|
|
2293
|
-
function
|
|
2294
|
-
const
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2376
|
+
function pairStartComplete(start, complete) {
|
|
2377
|
+
const attributes = mergeAttributes(start, complete);
|
|
2378
|
+
const paired = {
|
|
2379
|
+
...cloneEvent(start),
|
|
2380
|
+
status: complete.status,
|
|
2381
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
2382
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
2383
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
2384
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
2385
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
2386
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
2387
|
+
};
|
|
2388
|
+
return {
|
|
2389
|
+
...paired,
|
|
2390
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
2391
|
+
projection: {
|
|
2392
|
+
paired: true,
|
|
2393
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
2394
|
+
parentNormalized: false,
|
|
2395
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
2299
2396
|
}
|
|
2300
|
-
}
|
|
2301
|
-
walk(tree.children);
|
|
2302
|
-
return out;
|
|
2397
|
+
};
|
|
2303
2398
|
}
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
}
|
|
2316
|
-
function manualTraceEventsToComparableRun(events) {
|
|
2317
|
-
const started = events.find((e) => e.event === "run_started");
|
|
2318
|
-
if (!started || started.event !== "run_started") {
|
|
2319
|
-
throw new Error("Invalid trace: missing run_started");
|
|
2320
|
-
}
|
|
2321
|
-
const rs = started;
|
|
2322
|
-
const runId = rs.runId;
|
|
2323
|
-
const completedAll = events.filter((e) => e.event === "run_completed");
|
|
2324
|
-
const lastCompleted = completedAll[completedAll.length - 1];
|
|
2325
|
-
let runStatus;
|
|
2326
|
-
if (lastCompleted === void 0) runStatus = "running";
|
|
2327
|
-
else runStatus = lastCompleted.status;
|
|
2328
|
-
const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
2329
|
-
const steps = /* @__PURE__ */ new Map();
|
|
2330
|
-
let order = 0;
|
|
2331
|
-
for (const e of events) {
|
|
2332
|
-
if (e.event !== "step_started") continue;
|
|
2333
|
-
const s = e;
|
|
2334
|
-
const meta = s.metadata ? { ...s.metadata } : void 0;
|
|
2335
|
-
steps.set(s.stepId, {
|
|
2336
|
-
id: s.stepId,
|
|
2337
|
-
parentId: s.parentId,
|
|
2338
|
-
name: s.name,
|
|
2339
|
-
type: s.type,
|
|
2340
|
-
order: order++,
|
|
2341
|
-
timestamp: s.timestamp,
|
|
2342
|
-
metadata: meta
|
|
2343
|
-
});
|
|
2344
|
-
}
|
|
2345
|
-
for (const e of events) {
|
|
2346
|
-
if (e.event !== "step_completed") continue;
|
|
2347
|
-
const acc = steps.get(e.stepId);
|
|
2348
|
-
if (!acc) continue;
|
|
2349
|
-
acc.status = e.status;
|
|
2350
|
-
acc.durationMs = e.durationMs;
|
|
2351
|
-
if (e.error?.message) acc.errorMsg = e.error.message;
|
|
2352
|
-
const extra = e;
|
|
2353
|
-
if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
|
|
2354
|
-
acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
|
|
2355
|
-
}
|
|
2356
|
-
}
|
|
2357
|
-
const nodes = /* @__PURE__ */ new Map();
|
|
2358
|
-
for (const acc of steps.values()) {
|
|
2359
|
-
let meta = acc.metadata ? { ...acc.metadata } : void 0;
|
|
2360
|
-
if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
|
|
2361
|
-
meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
|
|
2362
|
-
}
|
|
2363
|
-
const outputPreview = extractOutputPreview(meta);
|
|
2364
|
-
if (meta !== void 0 && ("outputPreview" in meta || "resultPreview" in meta)) {
|
|
2365
|
-
delete meta.outputPreview;
|
|
2366
|
-
delete meta.resultPreview;
|
|
2367
|
-
}
|
|
2368
|
-
const sc = {
|
|
2369
|
-
id: acc.id,
|
|
2370
|
-
name: acc.name,
|
|
2371
|
-
type: acc.type,
|
|
2372
|
-
status: mapStepStatus(acc.status),
|
|
2373
|
-
durationMs: acc.durationMs,
|
|
2374
|
-
error: acc.errorMsg,
|
|
2375
|
-
metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
|
|
2376
|
-
outputPreview,
|
|
2377
|
-
children: []
|
|
2378
|
-
};
|
|
2379
|
-
nodes.set(acc.id, sc);
|
|
2380
|
-
}
|
|
2381
|
-
const roots = [];
|
|
2382
|
-
const sortByOrder = (a, b) => {
|
|
2383
|
-
const oa = steps.get(a.id)?.order ?? 0;
|
|
2384
|
-
const ob = steps.get(b.id)?.order ?? 0;
|
|
2385
|
-
return oa - ob;
|
|
2386
|
-
};
|
|
2387
|
-
for (const acc of steps.values()) {
|
|
2388
|
-
const node = nodes.get(acc.id);
|
|
2389
|
-
if (acc.parentId !== void 0 && nodes.has(acc.parentId)) {
|
|
2390
|
-
nodes.get(acc.parentId).children.push(node);
|
|
2391
|
-
} else {
|
|
2392
|
-
roots.push(node);
|
|
2393
|
-
}
|
|
2394
|
-
}
|
|
2395
|
-
roots.sort(sortByOrder);
|
|
2396
|
-
for (const n of nodes.values()) {
|
|
2397
|
-
n.children.sort(sortByOrder);
|
|
2398
|
-
}
|
|
2399
|
-
return {
|
|
2400
|
-
runId,
|
|
2401
|
-
name: rs.name,
|
|
2402
|
-
status: runStatus,
|
|
2403
|
-
durationMs,
|
|
2404
|
-
steps: roots
|
|
2405
|
-
};
|
|
2406
|
-
}
|
|
2407
|
-
|
|
2408
|
-
// packages/core/src/diff/engine.ts
|
|
2409
|
-
var DEFAULT_THRESHOLD_MS = 0;
|
|
2410
|
-
function pathSeg(step, index) {
|
|
2411
|
-
return { index, name: step.name, stepId: step.id };
|
|
2412
|
-
}
|
|
2413
|
-
function buildPath(segments) {
|
|
2414
|
-
return { path: [...segments] };
|
|
2415
|
-
}
|
|
2416
|
-
function pairSteps(left, right) {
|
|
2417
|
-
const usedRight = /* @__PURE__ */ new Set();
|
|
2418
|
-
const pairs = [];
|
|
2419
|
-
for (let i = 0; i < left.length; i++) {
|
|
2420
|
-
const L = left[i];
|
|
2421
|
-
let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
|
|
2422
|
-
if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
|
|
2423
|
-
const cand = right[i];
|
|
2424
|
-
if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
|
|
2425
|
-
R = cand;
|
|
2426
|
-
}
|
|
2427
|
-
}
|
|
2428
|
-
if (R === void 0) {
|
|
2429
|
-
R = right.find(
|
|
2430
|
-
(r) => !usedRight.has(r.id) && r.name === L.name && (r.type ?? "") === (L.type ?? "")
|
|
2431
|
-
);
|
|
2432
|
-
}
|
|
2433
|
-
if (R !== void 0) {
|
|
2434
|
-
usedRight.add(R.id);
|
|
2435
|
-
pairs.push([L, R]);
|
|
2436
|
-
} else {
|
|
2437
|
-
pairs.push([L, void 0]);
|
|
2438
|
-
}
|
|
2439
|
-
}
|
|
2440
|
-
for (const R of right) {
|
|
2441
|
-
if (!usedRight.has(R.id)) {
|
|
2442
|
-
pairs.push([void 0, R]);
|
|
2443
|
-
}
|
|
2444
|
-
}
|
|
2445
|
-
return pairs;
|
|
2446
|
-
}
|
|
2447
|
-
function compareLeafSteps(L, R, segments, opts, out) {
|
|
2448
|
-
const path16 = buildPath(segments);
|
|
2449
|
-
if (L.name !== R.name) {
|
|
2450
|
-
out.push({
|
|
2451
|
-
kind: "structure",
|
|
2452
|
-
severity: "warning",
|
|
2453
|
-
message: "Step name differs",
|
|
2454
|
-
path: path16,
|
|
2455
|
-
left: L.name,
|
|
2456
|
-
right: R.name
|
|
2457
|
-
});
|
|
2458
|
-
}
|
|
2459
|
-
if ((L.type ?? "") !== (R.type ?? "")) {
|
|
2460
|
-
out.push({
|
|
2461
|
-
kind: "step-type",
|
|
2462
|
-
severity: "warning",
|
|
2463
|
-
message: "Step type differs",
|
|
2464
|
-
path: path16,
|
|
2465
|
-
left: L.type,
|
|
2466
|
-
right: R.type
|
|
2467
|
-
});
|
|
2468
|
-
}
|
|
2469
|
-
if ((L.status ?? "") !== (R.status ?? "")) {
|
|
2470
|
-
out.push({
|
|
2471
|
-
kind: "step-status",
|
|
2472
|
-
severity: "warning",
|
|
2473
|
-
message: "Step status differs",
|
|
2474
|
-
path: path16,
|
|
2475
|
-
left: L.status,
|
|
2476
|
-
right: R.status
|
|
2477
|
-
});
|
|
2478
|
-
}
|
|
2479
|
-
const le = L.error ?? "";
|
|
2480
|
-
const re = R.error ?? "";
|
|
2481
|
-
if (le !== re) {
|
|
2482
|
-
out.push({
|
|
2483
|
-
kind: "error",
|
|
2484
|
-
severity: "error",
|
|
2485
|
-
message: "Step error message differs",
|
|
2486
|
-
path: path16,
|
|
2487
|
-
left: le || void 0,
|
|
2488
|
-
right: re || void 0
|
|
2489
|
-
});
|
|
2490
|
-
}
|
|
2491
|
-
if (!opts.ignoreDuration) {
|
|
2492
|
-
const ld = L.durationMs;
|
|
2493
|
-
const rd = R.durationMs;
|
|
2494
|
-
const th = opts.durationThresholdMs;
|
|
2495
|
-
let differs = false;
|
|
2496
|
-
if (ld === void 0 && rd === void 0) differs = false;
|
|
2497
|
-
else if (ld === void 0 || rd === void 0) differs = true;
|
|
2498
|
-
else differs = Math.abs(ld - rd) > th;
|
|
2499
|
-
if (differs) {
|
|
2500
|
-
out.push({
|
|
2501
|
-
kind: "duration",
|
|
2502
|
-
severity: "info",
|
|
2503
|
-
message: "Step duration differs",
|
|
2504
|
-
path: path16,
|
|
2505
|
-
left: ld,
|
|
2506
|
-
right: rd
|
|
2507
|
-
});
|
|
2508
|
-
}
|
|
2509
|
-
}
|
|
2510
|
-
const lm = stableJson(L.metadata ?? {});
|
|
2511
|
-
const rm = stableJson(R.metadata ?? {});
|
|
2512
|
-
if (lm !== rm) {
|
|
2513
|
-
out.push({
|
|
2514
|
-
kind: "metadata",
|
|
2515
|
-
severity: "info",
|
|
2516
|
-
message: "Step metadata differs",
|
|
2517
|
-
path: path16,
|
|
2518
|
-
left: L.metadata,
|
|
2519
|
-
right: R.metadata
|
|
2520
|
-
});
|
|
2521
|
-
}
|
|
2522
|
-
const lo = stableJson(L.outputPreview ?? null);
|
|
2523
|
-
const ro = stableJson(R.outputPreview ?? null);
|
|
2524
|
-
if (lo !== ro) {
|
|
2525
|
-
out.push({
|
|
2526
|
-
kind: "output",
|
|
2527
|
-
severity: "info",
|
|
2528
|
-
message: "Output preview differs",
|
|
2529
|
-
path: path16,
|
|
2530
|
-
left: L.outputPreview,
|
|
2531
|
-
right: R.outputPreview
|
|
2532
|
-
});
|
|
2533
|
-
}
|
|
2534
|
-
}
|
|
2535
|
-
function compareRecursive(L, R, segments, opts, out) {
|
|
2536
|
-
compareLeafSteps(L, R, segments, opts, out);
|
|
2537
|
-
const pairs = pairSteps(L.children, R.children);
|
|
2538
|
-
let ci = 0;
|
|
2539
|
-
for (const [lch, rch] of pairs) {
|
|
2540
|
-
if (lch !== void 0 && rch !== void 0) {
|
|
2541
|
-
compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
|
|
2542
|
-
} else if (lch !== void 0) {
|
|
2543
|
-
out.push({
|
|
2544
|
-
kind: "step-removed",
|
|
2545
|
-
severity: "warning",
|
|
2546
|
-
message: `Step only in left run: ${lch.name}`,
|
|
2547
|
-
path: buildPath([...segments, pathSeg(lch, ci)]),
|
|
2548
|
-
left: lch.id,
|
|
2549
|
-
right: void 0
|
|
2550
|
-
});
|
|
2551
|
-
} else if (rch !== void 0) {
|
|
2552
|
-
out.push({
|
|
2553
|
-
kind: "step-added",
|
|
2554
|
-
severity: "warning",
|
|
2555
|
-
message: `Step only in right run: ${rch.name}`,
|
|
2556
|
-
path: buildPath([...segments, pathSeg(rch, ci)]),
|
|
2557
|
-
left: void 0,
|
|
2558
|
-
right: rch.id
|
|
2559
|
-
});
|
|
2560
|
-
}
|
|
2561
|
-
ci += 1;
|
|
2562
|
-
}
|
|
2563
|
-
}
|
|
2564
|
-
function mergeDiffDefaults(options) {
|
|
2565
|
-
return {
|
|
2566
|
-
ignoreDuration: false,
|
|
2567
|
-
durationThresholdMs: DEFAULT_THRESHOLD_MS,
|
|
2568
|
-
focus: "all",
|
|
2569
|
-
check: "all"
|
|
2570
|
-
};
|
|
2571
|
-
}
|
|
2572
|
-
function kindMatchesFilter(kind, merged) {
|
|
2573
|
-
return true;
|
|
2574
|
-
}
|
|
2575
|
-
function diffRuns(left, right, options) {
|
|
2576
|
-
const merged = mergeDiffDefaults();
|
|
2577
|
-
const opts = {
|
|
2578
|
-
ignoreDuration: merged.ignoreDuration,
|
|
2579
|
-
durationThresholdMs: merged.durationThresholdMs
|
|
2580
|
-
};
|
|
2581
|
-
const raw = [];
|
|
2582
|
-
if ((left.status ?? "") !== (right.status ?? "")) {
|
|
2583
|
-
raw.push({
|
|
2584
|
-
kind: "run-status",
|
|
2585
|
-
severity: "warning",
|
|
2586
|
-
message: "Run completion status differs",
|
|
2587
|
-
left: left.status,
|
|
2588
|
-
right: right.status
|
|
2589
|
-
});
|
|
2590
|
-
}
|
|
2591
|
-
{
|
|
2592
|
-
const ld = left.durationMs;
|
|
2593
|
-
const rd = right.durationMs;
|
|
2594
|
-
const th = merged.durationThresholdMs;
|
|
2595
|
-
let differs = false;
|
|
2596
|
-
if (ld === void 0 && rd === void 0) differs = false;
|
|
2597
|
-
else if (ld === void 0 || rd === void 0) differs = true;
|
|
2598
|
-
else differs = Math.abs(ld - rd) > th;
|
|
2599
|
-
if (differs) {
|
|
2600
|
-
raw.push({
|
|
2601
|
-
kind: "duration",
|
|
2602
|
-
severity: "info",
|
|
2603
|
-
message: "Run duration differs",
|
|
2604
|
-
left: ld,
|
|
2605
|
-
right: rd
|
|
2606
|
-
});
|
|
2607
|
-
}
|
|
2608
|
-
}
|
|
2609
|
-
const pairs = pairSteps(left.steps, right.steps);
|
|
2610
|
-
let idx = 0;
|
|
2611
|
-
for (const [ls, rs] of pairs) {
|
|
2612
|
-
if (ls !== void 0 && rs !== void 0) {
|
|
2613
|
-
compareRecursive(ls, rs, [pathSeg(ls, idx)], opts, raw);
|
|
2614
|
-
idx += 1;
|
|
2615
|
-
} else if (ls !== void 0) {
|
|
2616
|
-
raw.push({
|
|
2617
|
-
kind: "step-removed",
|
|
2618
|
-
severity: "warning",
|
|
2619
|
-
message: `Step only in left run: ${ls.name}`,
|
|
2620
|
-
path: buildPath([pathSeg(ls, idx)]),
|
|
2621
|
-
left: ls.id,
|
|
2622
|
-
right: void 0
|
|
2623
|
-
});
|
|
2624
|
-
idx += 1;
|
|
2625
|
-
} else if (rs !== void 0) {
|
|
2626
|
-
raw.push({
|
|
2627
|
-
kind: "step-added",
|
|
2628
|
-
severity: "warning",
|
|
2629
|
-
message: `Step only in right run: ${rs.name}`,
|
|
2630
|
-
path: buildPath([pathSeg(rs, idx)]),
|
|
2631
|
-
left: void 0,
|
|
2632
|
-
right: rs.id
|
|
2633
|
-
});
|
|
2634
|
-
idx += 1;
|
|
2635
|
-
}
|
|
2636
|
-
}
|
|
2637
|
-
const differences = raw.filter((d) => kindMatchesFilter(d.kind));
|
|
2638
|
-
let errors = 0;
|
|
2639
|
-
let warnings = 0;
|
|
2640
|
-
let info = 0;
|
|
2641
|
-
for (const d of differences) {
|
|
2642
|
-
if (d.severity === "error") errors += 1;
|
|
2643
|
-
else if (d.severity === "warning") warnings += 1;
|
|
2644
|
-
else info += 1;
|
|
2645
|
-
}
|
|
2646
|
-
const firstVisible = differences[0];
|
|
2647
|
-
const firstDivergence = firstVisible !== void 0 ? {
|
|
2648
|
-
kind: "first-divergence",
|
|
2649
|
-
severity: firstVisible.severity,
|
|
2650
|
-
message: `First divergence: ${firstVisible.message}`,
|
|
2651
|
-
path: firstVisible.path,
|
|
2652
|
-
left: firstVisible.left,
|
|
2653
|
-
right: firstVisible.right
|
|
2654
|
-
} : void 0;
|
|
2655
|
-
const summary = {
|
|
2656
|
-
leftRunId: left.runId,
|
|
2657
|
-
rightRunId: right.runId,
|
|
2658
|
-
totalDifferences: differences.length,
|
|
2659
|
-
errors,
|
|
2660
|
-
warnings,
|
|
2661
|
-
info,
|
|
2662
|
-
firstDivergence
|
|
2663
|
-
};
|
|
2664
|
-
return { summary, differences };
|
|
2665
|
-
}
|
|
2666
|
-
|
|
2667
|
-
// packages/core/src/evidence/zip.ts
|
|
2668
|
-
(() => {
|
|
2669
|
-
const table = new Uint32Array(256);
|
|
2670
|
-
for (let n = 0; n < 256; n += 1) {
|
|
2671
|
-
let c = n;
|
|
2672
|
-
for (let k = 0; k < 8; k += 1) {
|
|
2673
|
-
c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
|
|
2674
|
-
}
|
|
2675
|
-
table[n] = c >>> 0;
|
|
2676
|
-
}
|
|
2677
|
-
return table;
|
|
2678
|
-
})();
|
|
2679
|
-
|
|
2680
|
-
// packages/core/src/diagnostics/programmatic.ts
|
|
2681
|
-
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
2682
|
-
AI_TRACE_INPUT_INVALID: {
|
|
2683
|
-
code: "AI_TRACE_INPUT_INVALID",
|
|
2684
|
-
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
2685
|
-
remediation: "For a file path, use openTraceFile(path).",
|
|
2686
|
-
relatedCodes: ["invalid_input"]
|
|
2687
|
-
},
|
|
2688
|
-
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
2689
|
-
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
2690
|
-
summary: "No trace reader could detect the input format.",
|
|
2691
|
-
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
2692
|
-
relatedCodes: ["unsupported_format"]
|
|
2693
|
-
},
|
|
2694
|
-
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
2695
|
-
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
2696
|
-
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
2697
|
-
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
2698
|
-
relatedCodes: ["ambiguous_format"]
|
|
2699
|
-
},
|
|
2700
|
-
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
2701
|
-
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
2702
|
-
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
2703
|
-
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
2704
|
-
},
|
|
2705
|
-
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
2706
|
-
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
2707
|
-
summary: "Multiple runs are available; select a run before executing checks.",
|
|
2708
|
-
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
2709
|
-
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
2710
|
-
},
|
|
2711
|
-
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
2712
|
-
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2713
|
-
summary: "A parentId equals its own event/step id.",
|
|
2714
|
-
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
2715
|
-
relatedCodes: [
|
|
2716
|
-
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
2717
|
-
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
2718
|
-
]
|
|
2719
|
-
},
|
|
2720
|
-
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
2721
|
-
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
2722
|
-
summary: "Trace contains a parentId cycle.",
|
|
2723
|
-
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
2724
|
-
relatedCodes: ["structure.cycle"]
|
|
2725
|
-
}
|
|
2726
|
-
});
|
|
2727
|
-
function formatProgrammaticDiagnostic(code, detail) {
|
|
2728
|
-
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
2729
|
-
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
2730
|
-
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
2731
|
-
}
|
|
2732
|
-
|
|
2733
|
-
// packages/core/src/safety/sensitive-key.ts
|
|
2734
|
-
function keyHasExplicitSeparator(value) {
|
|
2735
|
-
return /[_\-.]/.test(value);
|
|
2736
|
-
}
|
|
2737
|
-
function normalizeSensitiveKey(value) {
|
|
2738
|
-
return value.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
2739
|
-
}
|
|
2740
|
-
var NON_CREDENTIAL_TOKEN_CONFIG_KEYS = new Set(
|
|
2741
|
-
[
|
|
2742
|
-
"tokens",
|
|
2743
|
-
"max_tokens",
|
|
2744
|
-
"min_tokens",
|
|
2745
|
-
"ls_max_tokens",
|
|
2746
|
-
"token_count",
|
|
2747
|
-
"token_limit",
|
|
2748
|
-
"token_budget",
|
|
2749
|
-
"input_tokens",
|
|
2750
|
-
"output_tokens",
|
|
2751
|
-
"total_tokens",
|
|
2752
|
-
"cached_tokens",
|
|
2753
|
-
"prompt_tokens",
|
|
2754
|
-
"completion_tokens"
|
|
2755
|
-
].map(normalizeSensitiveKey)
|
|
2756
|
-
);
|
|
2757
|
-
var DEFAULT_CREDENTIAL_SENSITIVE_KEYS = [
|
|
2758
|
-
"authorization",
|
|
2759
|
-
"cookie",
|
|
2760
|
-
"token",
|
|
2761
|
-
"access_token",
|
|
2762
|
-
"accesstoken",
|
|
2763
|
-
"auth_token",
|
|
2764
|
-
"authtoken",
|
|
2765
|
-
"refresh_token",
|
|
2766
|
-
"refreshtoken",
|
|
2767
|
-
"id_token",
|
|
2768
|
-
"idtoken",
|
|
2769
|
-
"bearer_token",
|
|
2770
|
-
"bearertoken",
|
|
2771
|
-
"api_token",
|
|
2772
|
-
"apitoken",
|
|
2773
|
-
"apikey",
|
|
2774
|
-
"api_key",
|
|
2775
|
-
"password",
|
|
2776
|
-
"secret",
|
|
2777
|
-
"email"
|
|
2778
|
-
];
|
|
2779
|
-
function isTokenCredentialKey(normalized) {
|
|
2780
|
-
if (NON_CREDENTIAL_TOKEN_CONFIG_KEYS.has(normalized)) return false;
|
|
2781
|
-
if (normalized === "token") return true;
|
|
2782
|
-
if (normalized.endsWith("tokens")) return false;
|
|
2783
|
-
return normalized.endsWith("token");
|
|
2784
|
-
}
|
|
2785
|
-
function isCredentialSensitiveKey(key, sensitiveKeys = DEFAULT_CREDENTIAL_SENSITIVE_KEYS) {
|
|
2786
|
-
if (!key) return false;
|
|
2787
|
-
const normalized = normalizeSensitiveKey(key);
|
|
2788
|
-
if (!normalized) return false;
|
|
2789
|
-
if (NON_CREDENTIAL_TOKEN_CONFIG_KEYS.has(normalized)) return false;
|
|
2790
|
-
const allowPrefixCompound = keyHasExplicitSeparator(key);
|
|
2791
|
-
for (const sensitive of sensitiveKeys) {
|
|
2792
|
-
const s = normalizeSensitiveKey(sensitive);
|
|
2793
|
-
if (!s) continue;
|
|
2794
|
-
if (s === "token") {
|
|
2795
|
-
if (isTokenCredentialKey(normalized)) return true;
|
|
2796
|
-
continue;
|
|
2797
|
-
}
|
|
2798
|
-
if (normalized === s) return true;
|
|
2799
|
-
if (normalized.endsWith(`_${s}`)) return true;
|
|
2800
|
-
if (allowPrefixCompound && normalized.startsWith(`${s}_`)) return true;
|
|
2801
|
-
}
|
|
2802
|
-
return false;
|
|
2803
|
-
}
|
|
2804
|
-
|
|
2805
|
-
// packages/core/src/checks/logical-events.ts
|
|
2806
|
-
function isRecord6(value) {
|
|
2807
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2808
|
-
}
|
|
2809
|
-
function legacyEvent(event) {
|
|
2810
|
-
const value = event.attributes?.legacyEvent;
|
|
2811
|
-
return typeof value === "string" ? value : void 0;
|
|
2812
|
-
}
|
|
2813
|
-
function stepIdOf(event) {
|
|
2814
|
-
const value = event.attributes?.stepId;
|
|
2815
|
-
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2816
|
-
return void 0;
|
|
2817
|
-
}
|
|
2818
|
-
function cloneEvent(event) {
|
|
2819
|
-
return {
|
|
2820
|
-
...event,
|
|
2821
|
-
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
2822
|
-
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
2823
|
-
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
2824
|
-
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
2825
|
-
};
|
|
2826
|
-
}
|
|
2827
|
-
function mergeAttributes(start, complete) {
|
|
2828
|
-
const merged = {
|
|
2829
|
-
...isRecord6(complete.attributes) ? complete.attributes : {},
|
|
2830
|
-
...isRecord6(start.attributes) ? start.attributes : {}
|
|
2831
|
-
};
|
|
2832
|
-
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
2833
|
-
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
2834
|
-
if (complete.attributes?.errorStack !== void 0) {
|
|
2835
|
-
merged.errorStack = complete.attributes.errorStack;
|
|
2836
|
-
}
|
|
2837
|
-
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
2838
|
-
}
|
|
2839
|
-
function pairStartComplete(start, complete) {
|
|
2840
|
-
const attributes = mergeAttributes(start, complete);
|
|
2841
|
-
const paired = {
|
|
2842
|
-
...cloneEvent(start),
|
|
2843
|
-
status: complete.status,
|
|
2844
|
-
timestamp: complete.timestamp ?? start.timestamp,
|
|
2845
|
-
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
2846
|
-
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
2847
|
-
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
2848
|
-
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
2849
|
-
...attributes !== void 0 ? { attributes } : {}
|
|
2850
|
-
};
|
|
2851
|
-
return {
|
|
2852
|
-
...paired,
|
|
2853
|
-
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
2854
|
-
projection: {
|
|
2855
|
-
paired: true,
|
|
2856
|
-
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
2857
|
-
parentNormalized: false,
|
|
2858
|
-
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
2859
|
-
}
|
|
2860
|
-
};
|
|
2861
|
-
}
|
|
2862
|
-
function asLogical(event, extras) {
|
|
2863
|
-
return {
|
|
2864
|
-
...cloneEvent(event),
|
|
2865
|
-
sourceEventIds: Object.freeze([event.eventId]),
|
|
2866
|
-
projection: {
|
|
2867
|
-
paired: false,
|
|
2868
|
-
absorbedEventIds: Object.freeze([]),
|
|
2869
|
-
parentNormalized: extras?.parentNormalized === true,
|
|
2870
|
-
...{}
|
|
2871
|
-
}
|
|
2872
|
-
};
|
|
2399
|
+
function asLogical(event, extras) {
|
|
2400
|
+
return {
|
|
2401
|
+
...cloneEvent(event),
|
|
2402
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
2403
|
+
projection: {
|
|
2404
|
+
paired: false,
|
|
2405
|
+
absorbedEventIds: Object.freeze([]),
|
|
2406
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
2407
|
+
...{}
|
|
2408
|
+
}
|
|
2409
|
+
};
|
|
2873
2410
|
}
|
|
2874
2411
|
function projectLogicalEvents(events) {
|
|
2875
2412
|
const diagnostics = [];
|
|
@@ -2957,1441 +2494,1908 @@ function projectLogicalEvents(events) {
|
|
|
2957
2494
|
message: `No matching start for complete ${complete.eventId}.`,
|
|
2958
2495
|
eventIds: [complete.eventId]
|
|
2959
2496
|
});
|
|
2960
|
-
const alone = asLogical(complete);
|
|
2961
|
-
logical.push(alone);
|
|
2962
|
-
logicalByRawId.set(complete.eventId, alone);
|
|
2963
|
-
const stepId = stepIdOf(complete);
|
|
2964
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
2965
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2966
|
-
}
|
|
2497
|
+
const alone = asLogical(complete);
|
|
2498
|
+
logical.push(alone);
|
|
2499
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
2500
|
+
const stepId = stepIdOf(complete);
|
|
2501
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
2502
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
for (const event of others) {
|
|
2506
|
+
const alone = asLogical(event);
|
|
2507
|
+
logical.push(alone);
|
|
2508
|
+
logicalByRawId.set(event.eventId, alone);
|
|
2509
|
+
const stepId = stepIdOf(event);
|
|
2510
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
2511
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
2512
|
+
}
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2515
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
2516
|
+
const normalized = [];
|
|
2517
|
+
for (const event of logical) {
|
|
2518
|
+
const originalParentId = event.parentId;
|
|
2519
|
+
if (!originalParentId) {
|
|
2520
|
+
normalized.push(event);
|
|
2521
|
+
continue;
|
|
2522
|
+
}
|
|
2523
|
+
let nextParent = originalParentId;
|
|
2524
|
+
let remapped = false;
|
|
2525
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
2526
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
2527
|
+
nextParent = viaAbsorbed.eventId;
|
|
2528
|
+
remapped = true;
|
|
2529
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
2530
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
2531
|
+
if (viaStep) {
|
|
2532
|
+
nextParent = viaStep;
|
|
2533
|
+
remapped = true;
|
|
2534
|
+
}
|
|
2535
|
+
}
|
|
2536
|
+
if (!remapped) {
|
|
2537
|
+
if (originalParentId === event.eventId) {
|
|
2538
|
+
diagnostics.push({
|
|
2539
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
2540
|
+
message: formatProgrammaticDiagnostic(
|
|
2541
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2542
|
+
`Removed self-parent edge on ${event.eventId}.`
|
|
2543
|
+
),
|
|
2544
|
+
eventIds: [event.eventId]
|
|
2545
|
+
});
|
|
2546
|
+
const { parentId: _drop, ...rest } = event;
|
|
2547
|
+
normalized.push({
|
|
2548
|
+
...rest,
|
|
2549
|
+
projection: {
|
|
2550
|
+
...event.projection,
|
|
2551
|
+
parentNormalized: true,
|
|
2552
|
+
originalParentId
|
|
2553
|
+
}
|
|
2554
|
+
});
|
|
2555
|
+
continue;
|
|
2556
|
+
}
|
|
2557
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
2558
|
+
const mapping = event.attributes?.parentMapping;
|
|
2559
|
+
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
2560
|
+
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
2561
|
+
if (!unresolved) {
|
|
2562
|
+
diagnostics.push({
|
|
2563
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
2564
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
2565
|
+
eventIds: [event.eventId]
|
|
2566
|
+
});
|
|
2567
|
+
}
|
|
2568
|
+
}
|
|
2569
|
+
normalized.push(event);
|
|
2570
|
+
continue;
|
|
2571
|
+
}
|
|
2572
|
+
if (nextParent === event.eventId) {
|
|
2573
|
+
diagnostics.push({
|
|
2574
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
2575
|
+
message: formatProgrammaticDiagnostic(
|
|
2576
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
2577
|
+
`Removed self-parent edge on ${event.eventId} (was ${originalParentId}).`
|
|
2578
|
+
),
|
|
2579
|
+
eventIds: [event.eventId]
|
|
2580
|
+
});
|
|
2581
|
+
const { parentId: _drop, ...rest } = event;
|
|
2582
|
+
normalized.push({
|
|
2583
|
+
...rest,
|
|
2584
|
+
projection: {
|
|
2585
|
+
...event.projection,
|
|
2586
|
+
parentNormalized: true,
|
|
2587
|
+
originalParentId
|
|
2588
|
+
}
|
|
2589
|
+
});
|
|
2590
|
+
continue;
|
|
2591
|
+
}
|
|
2592
|
+
diagnostics.push({
|
|
2593
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
2594
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
2595
|
+
eventIds: [event.eventId]
|
|
2596
|
+
});
|
|
2597
|
+
normalized.push({
|
|
2598
|
+
...event,
|
|
2599
|
+
parentId: nextParent,
|
|
2600
|
+
projection: {
|
|
2601
|
+
...event.projection,
|
|
2602
|
+
parentNormalized: true,
|
|
2603
|
+
originalParentId
|
|
2604
|
+
}
|
|
2605
|
+
});
|
|
2606
|
+
}
|
|
2607
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
2608
|
+
normalized.sort((a, b) => {
|
|
2609
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
2610
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
2611
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
2612
|
+
});
|
|
2613
|
+
return {
|
|
2614
|
+
logicalEvents: Object.freeze(normalized),
|
|
2615
|
+
diagnostics: Object.freeze(diagnostics)
|
|
2616
|
+
};
|
|
2617
|
+
}
|
|
2618
|
+
function resolveCanonicalToolName(event) {
|
|
2619
|
+
const attrs2 = event.attributes;
|
|
2620
|
+
const direct = pickString(attrs2, ["toolName", "tool"]);
|
|
2621
|
+
if (direct) return direct;
|
|
2622
|
+
const metadata = attrs2?.metadata;
|
|
2623
|
+
if (isRecord6(metadata)) {
|
|
2624
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
2625
|
+
if (nested) return nested;
|
|
2626
|
+
}
|
|
2627
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
2628
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
2629
|
+
}
|
|
2630
|
+
return event.name;
|
|
2631
|
+
}
|
|
2632
|
+
function pickString(record, keys) {
|
|
2633
|
+
if (!record) return void 0;
|
|
2634
|
+
for (const key of keys) {
|
|
2635
|
+
const value = record[key];
|
|
2636
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
2637
|
+
}
|
|
2638
|
+
return void 0;
|
|
2639
|
+
}
|
|
2640
|
+
|
|
2641
|
+
// packages/core/src/checks/derived-failure.ts
|
|
2642
|
+
function isRecord7(value) {
|
|
2643
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2644
|
+
}
|
|
2645
|
+
function pickString2(record, keys) {
|
|
2646
|
+
if (!record) return void 0;
|
|
2647
|
+
for (const key of keys) {
|
|
2648
|
+
const value = record[key];
|
|
2649
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2650
|
+
}
|
|
2651
|
+
return void 0;
|
|
2652
|
+
}
|
|
2653
|
+
function pickNumber(record, keys) {
|
|
2654
|
+
if (!record) return void 0;
|
|
2655
|
+
for (const key of keys) {
|
|
2656
|
+
const value = record[key];
|
|
2657
|
+
if (typeof value === "number" && Number.isFinite(value) && value > 0) {
|
|
2658
|
+
return value;
|
|
2659
|
+
}
|
|
2660
|
+
}
|
|
2661
|
+
return void 0;
|
|
2662
|
+
}
|
|
2663
|
+
function eventMetadata(event) {
|
|
2664
|
+
const attrs2 = isRecord7(event.attributes) ? event.attributes : void 0;
|
|
2665
|
+
const nested = attrs2 !== void 0 && isRecord7(attrs2.metadata) ? attrs2.metadata : void 0;
|
|
2666
|
+
return {
|
|
2667
|
+
...attrs2 ?? {},
|
|
2668
|
+
...nested ?? {}
|
|
2669
|
+
};
|
|
2670
|
+
}
|
|
2671
|
+
function canonicalName(event) {
|
|
2672
|
+
if (event.kind === "TOOL") return resolveCanonicalToolName(event);
|
|
2673
|
+
return event.name;
|
|
2674
|
+
}
|
|
2675
|
+
function linkKeys(event) {
|
|
2676
|
+
const meta = eventMetadata(event);
|
|
2677
|
+
const keys = [];
|
|
2678
|
+
for (const key of ["linkedStepId", "toolCallId", "mcpToolCallId"]) {
|
|
2679
|
+
const value = pickString2(meta, [key]);
|
|
2680
|
+
if (value !== void 0) keys.push(`${key}:${value}`);
|
|
2681
|
+
}
|
|
2682
|
+
const stepId = pickString2(meta, ["stepId"]);
|
|
2683
|
+
if (stepId !== void 0) keys.push(`stepId:${stepId}`);
|
|
2684
|
+
return keys;
|
|
2685
|
+
}
|
|
2686
|
+
function buildRunContexts(logicalEvents) {
|
|
2687
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
2688
|
+
for (const event of logicalEvents) {
|
|
2689
|
+
const existing = byRun.get(event.runId) ?? {
|
|
2690
|
+
runId: event.runId,
|
|
2691
|
+
name: event.name
|
|
2692
|
+
};
|
|
2693
|
+
const meta = eventMetadata(event);
|
|
2694
|
+
if (event.kind === "RUN" || existing.name === event.runId) {
|
|
2695
|
+
existing.name = event.name || existing.name;
|
|
2696
|
+
}
|
|
2697
|
+
if (event.kind === "RUN" && event.status !== void 0 && event.status !== "running") {
|
|
2698
|
+
existing.status = event.status;
|
|
2699
|
+
}
|
|
2700
|
+
existing.retryOf ??= pickString2(meta, ["retryOf"]);
|
|
2701
|
+
existing.attempt ??= pickNumber(meta, ["attempt", "retryAttempt", "retryCount"]);
|
|
2702
|
+
existing.sessionId ??= pickString2(meta, ["sessionId", "conversationId"]);
|
|
2703
|
+
existing.groupId ??= pickString2(meta, ["groupId"]);
|
|
2704
|
+
existing.parentGroupId ??= pickString2(meta, ["parentGroupId"]);
|
|
2705
|
+
existing.fallbackOf ??= pickString2(meta, ["fallbackOf", "fallbackFrom"]);
|
|
2706
|
+
byRun.set(event.runId, existing);
|
|
2707
|
+
}
|
|
2708
|
+
return byRun;
|
|
2709
|
+
}
|
|
2710
|
+
function sameCorrelationScope(a, b) {
|
|
2711
|
+
if (a.sessionId && b.sessionId && a.sessionId === b.sessionId) return true;
|
|
2712
|
+
if (a.groupId && b.groupId && a.groupId === b.groupId) return true;
|
|
2713
|
+
if (a.parentGroupId && b.parentGroupId && a.parentGroupId === b.parentGroupId) {
|
|
2714
|
+
return true;
|
|
2715
|
+
}
|
|
2716
|
+
return false;
|
|
2717
|
+
}
|
|
2718
|
+
function isSuccessful(event) {
|
|
2719
|
+
return event.status === "ok";
|
|
2720
|
+
}
|
|
2721
|
+
function isFailure(event) {
|
|
2722
|
+
return event.status === "error";
|
|
2723
|
+
}
|
|
2724
|
+
function compareEventOrder(a, b) {
|
|
2725
|
+
const byTime = a.timestamp.localeCompare(b.timestamp);
|
|
2726
|
+
if (byTime !== 0) return byTime;
|
|
2727
|
+
return a.eventId.localeCompare(b.eventId);
|
|
2728
|
+
}
|
|
2729
|
+
function collectCandidates(failure, logicalEvents, runs) {
|
|
2730
|
+
const failureRun = runs.get(failure.runId);
|
|
2731
|
+
const failureLinks = new Set(linkKeys(failure));
|
|
2732
|
+
const failureName = canonicalName(failure);
|
|
2733
|
+
const failureAttempt = pickNumber(eventMetadata(failure), ["attempt", "retryAttempt", "retryCount"]) ?? failureRun?.attempt;
|
|
2734
|
+
const candidates = [];
|
|
2735
|
+
for (const event of logicalEvents) {
|
|
2736
|
+
if (event.eventId === failure.eventId) continue;
|
|
2737
|
+
if (event.status === "running") continue;
|
|
2738
|
+
const eventRun = runs.get(event.runId);
|
|
2739
|
+
const eventMeta = eventMetadata(event);
|
|
2740
|
+
const sameRun = event.runId === failure.runId;
|
|
2741
|
+
if (eventRun?.retryOf === failure.runId) {
|
|
2742
|
+
candidates.push({
|
|
2743
|
+
event,
|
|
2744
|
+
basis: "retryOf",
|
|
2745
|
+
confidence: "explicit",
|
|
2746
|
+
viaRunId: event.runId
|
|
2747
|
+
});
|
|
2748
|
+
continue;
|
|
2967
2749
|
}
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2750
|
+
if (eventRun?.fallbackOf === failure.runId || pickString2(eventMeta, ["fallbackOf", "fallbackFrom"]) === failure.runId) {
|
|
2751
|
+
candidates.push({
|
|
2752
|
+
event,
|
|
2753
|
+
basis: "fallbackOf",
|
|
2754
|
+
confidence: "explicit",
|
|
2755
|
+
viaRunId: event.runId
|
|
2756
|
+
});
|
|
2757
|
+
continue;
|
|
2976
2758
|
}
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2759
|
+
if (sameRun && compareEventOrder(failure, event) >= 0) continue;
|
|
2760
|
+
const eventLinks = linkKeys(event);
|
|
2761
|
+
const sharedLink = eventLinks.find((key) => failureLinks.has(key));
|
|
2762
|
+
if (sharedLink !== void 0) {
|
|
2763
|
+
candidates.push({
|
|
2764
|
+
event,
|
|
2765
|
+
basis: sharedLink.split(":")[0] ?? "linkedId",
|
|
2766
|
+
confidence: "explicit"
|
|
2767
|
+
});
|
|
2984
2768
|
continue;
|
|
2985
2769
|
}
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
const
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
}
|
|
2770
|
+
const eventAttempt = pickNumber(eventMeta, ["attempt", "retryAttempt", "retryCount"]) ?? eventRun?.attempt;
|
|
2771
|
+
const sameParent = failure.parentId !== void 0 && event.parentId !== void 0 && failure.parentId === event.parentId;
|
|
2772
|
+
const differentParent = failure.parentId !== void 0 && event.parentId !== void 0 && failure.parentId !== event.parentId;
|
|
2773
|
+
const sessionScoped = failureRun !== void 0 && eventRun !== void 0 && sameCorrelationScope(failureRun, eventRun);
|
|
2774
|
+
if (canonicalName(event) === failureName && failureAttempt !== void 0 && eventAttempt !== void 0 && eventAttempt > failureAttempt && !differentParent && (sameParent || sessionScoped || sameRun)) {
|
|
2775
|
+
candidates.push({
|
|
2776
|
+
event,
|
|
2777
|
+
basis: "attempt-progression",
|
|
2778
|
+
confidence: "correlated",
|
|
2779
|
+
...event.runId !== failure.runId ? { viaRunId: event.runId } : {}
|
|
2780
|
+
});
|
|
2998
2781
|
}
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
`Removed self-parent edge on ${event.eventId}.`
|
|
3006
|
-
),
|
|
3007
|
-
eventIds: [event.eventId]
|
|
3008
|
-
});
|
|
3009
|
-
const { parentId: _drop, ...rest } = event;
|
|
3010
|
-
normalized.push({
|
|
3011
|
-
...rest,
|
|
3012
|
-
projection: {
|
|
3013
|
-
...event.projection,
|
|
3014
|
-
parentNormalized: true,
|
|
3015
|
-
originalParentId
|
|
3016
|
-
}
|
|
3017
|
-
});
|
|
3018
|
-
continue;
|
|
3019
|
-
}
|
|
3020
|
-
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
3021
|
-
const mapping = event.attributes?.parentMapping;
|
|
3022
|
-
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
3023
|
-
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
3024
|
-
if (!unresolved) {
|
|
3025
|
-
diagnostics.push({
|
|
3026
|
-
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
3027
|
-
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
3028
|
-
eventIds: [event.eventId]
|
|
3029
|
-
});
|
|
3030
|
-
}
|
|
3031
|
-
}
|
|
3032
|
-
normalized.push(event);
|
|
3033
|
-
continue;
|
|
2782
|
+
}
|
|
2783
|
+
const byId = /* @__PURE__ */ new Map();
|
|
2784
|
+
for (const candidate of candidates) {
|
|
2785
|
+
const prev = byId.get(candidate.event.eventId);
|
|
2786
|
+
if (!prev || prev.confidence !== "explicit" && candidate.confidence === "explicit") {
|
|
2787
|
+
byId.set(candidate.event.eventId, candidate);
|
|
3034
2788
|
}
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
2789
|
+
}
|
|
2790
|
+
return [...byId.values()].sort((a, b) => compareEventOrder(a.event, b.event));
|
|
2791
|
+
}
|
|
2792
|
+
function classifyFailure(failure, candidates, runs, logicalEvents) {
|
|
2793
|
+
const successful = candidates.filter((c) => isSuccessful(c.event));
|
|
2794
|
+
const unsuccessful = candidates.filter((c) => !isSuccessful(c.event));
|
|
2795
|
+
const retryRunIds = Object.freeze(
|
|
2796
|
+
[...new Set(candidates.map((c) => c.viaRunId).filter((id) => id !== void 0))].sort(
|
|
2797
|
+
(a, b) => a.localeCompare(b)
|
|
2798
|
+
)
|
|
2799
|
+
);
|
|
2800
|
+
if (successful.length > 1) {
|
|
2801
|
+
const distinctRuns = new Set(successful.map((c) => c.event.runId));
|
|
2802
|
+
const distinctParents = new Set(
|
|
2803
|
+
successful.map((c) => c.event.parentId ?? "").filter((id) => id !== "")
|
|
2804
|
+
);
|
|
2805
|
+
if (distinctRuns.size > 1 || distinctParents.size > 1) {
|
|
2806
|
+
return {
|
|
2807
|
+
eventId: failure.eventId,
|
|
2808
|
+
runId: failure.runId,
|
|
2809
|
+
name: failure.name,
|
|
2810
|
+
kind: failure.kind,
|
|
2811
|
+
role: "unknown",
|
|
2812
|
+
confidence: "unknown",
|
|
2813
|
+
basis: Object.freeze(["ambiguous-recovery-candidates"]),
|
|
2814
|
+
recoveryEventIds: Object.freeze(
|
|
2815
|
+
successful.map((c) => c.event.eventId).sort((a, b) => a.localeCompare(b))
|
|
3041
2816
|
),
|
|
3042
|
-
|
|
3043
|
-
}
|
|
3044
|
-
const { parentId: _drop, ...rest } = event;
|
|
3045
|
-
normalized.push({
|
|
3046
|
-
...rest,
|
|
3047
|
-
projection: {
|
|
3048
|
-
...event.projection,
|
|
3049
|
-
parentNormalized: true,
|
|
3050
|
-
originalParentId
|
|
3051
|
-
}
|
|
3052
|
-
});
|
|
3053
|
-
continue;
|
|
2817
|
+
retryRunIds
|
|
2818
|
+
};
|
|
3054
2819
|
}
|
|
3055
|
-
diagnostics.push({
|
|
3056
|
-
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
3057
|
-
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
3058
|
-
eventIds: [event.eventId]
|
|
3059
|
-
});
|
|
3060
|
-
normalized.push({
|
|
3061
|
-
...event,
|
|
3062
|
-
parentId: nextParent,
|
|
3063
|
-
projection: {
|
|
3064
|
-
...event.projection,
|
|
3065
|
-
parentNormalized: true,
|
|
3066
|
-
originalParentId
|
|
3067
|
-
}
|
|
3068
|
-
});
|
|
3069
2820
|
}
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
2821
|
+
if (successful.length >= 1) {
|
|
2822
|
+
const best = successful[0];
|
|
2823
|
+
return {
|
|
2824
|
+
eventId: failure.eventId,
|
|
2825
|
+
runId: failure.runId,
|
|
2826
|
+
name: failure.name,
|
|
2827
|
+
kind: failure.kind,
|
|
2828
|
+
role: "recovered",
|
|
2829
|
+
confidence: best.confidence,
|
|
2830
|
+
basis: Object.freeze([best.basis]),
|
|
2831
|
+
recoveryEventIds: Object.freeze(
|
|
2832
|
+
successful.map((c) => c.event.eventId).sort((a, b) => a.localeCompare(b))
|
|
2833
|
+
),
|
|
2834
|
+
retryRunIds
|
|
2835
|
+
};
|
|
2836
|
+
}
|
|
2837
|
+
if (candidates.length > 0) {
|
|
2838
|
+
const best = candidates[0];
|
|
2839
|
+
return {
|
|
2840
|
+
eventId: failure.eventId,
|
|
2841
|
+
runId: failure.runId,
|
|
2842
|
+
name: failure.name,
|
|
2843
|
+
kind: failure.kind,
|
|
2844
|
+
role: "transient",
|
|
2845
|
+
confidence: best.confidence,
|
|
2846
|
+
basis: Object.freeze([
|
|
2847
|
+
best.basis,
|
|
2848
|
+
unsuccessful.some((c) => c.event.status === void 0) ? "retry-incomplete" : "retry-without-success"
|
|
2849
|
+
]),
|
|
2850
|
+
recoveryEventIds: Object.freeze([]),
|
|
2851
|
+
retryRunIds
|
|
2852
|
+
};
|
|
2853
|
+
}
|
|
2854
|
+
const failureMeta = eventMetadata(failure);
|
|
2855
|
+
const declaredSuccessor = pickString2(failureMeta, [
|
|
2856
|
+
"retriedBy",
|
|
2857
|
+
"nextRetryRunId",
|
|
2858
|
+
"retryRunId"
|
|
2859
|
+
]);
|
|
2860
|
+
if (declaredSuccessor !== void 0 && !runs.has(declaredSuccessor)) {
|
|
2861
|
+
return {
|
|
2862
|
+
eventId: failure.eventId,
|
|
2863
|
+
runId: failure.runId,
|
|
2864
|
+
name: failure.name,
|
|
2865
|
+
kind: failure.kind,
|
|
2866
|
+
role: "transient",
|
|
2867
|
+
confidence: "explicit",
|
|
2868
|
+
basis: Object.freeze(["retry-declared", "retry-run-missing"]),
|
|
2869
|
+
recoveryEventIds: Object.freeze([]),
|
|
2870
|
+
retryRunIds: Object.freeze([declaredSuccessor])
|
|
2871
|
+
};
|
|
2872
|
+
}
|
|
2873
|
+
for (const run of runs.values()) {
|
|
2874
|
+
if (run.retryOf === failure.runId) {
|
|
2875
|
+
return {
|
|
2876
|
+
eventId: failure.eventId,
|
|
2877
|
+
runId: failure.runId,
|
|
2878
|
+
name: failure.name,
|
|
2879
|
+
kind: failure.kind,
|
|
2880
|
+
role: "transient",
|
|
2881
|
+
confidence: "explicit",
|
|
2882
|
+
basis: Object.freeze(["retryOf", "retry-run-missing-or-empty"]),
|
|
2883
|
+
recoveryEventIds: Object.freeze([]),
|
|
2884
|
+
retryRunIds: Object.freeze([run.runId])
|
|
2885
|
+
};
|
|
2886
|
+
}
|
|
2887
|
+
}
|
|
2888
|
+
const failureRun = runs.get(failure.runId);
|
|
2889
|
+
const hasSuccessorDeclared = [...runs.values()].some((run) => run.retryOf === failure.runId);
|
|
2890
|
+
const isFinalInChain = failureRun !== void 0 && !hasSuccessorDeclared && (failureRun.retryOf !== void 0 || failureRun.attempt !== void 0 && failureRun.attempt > 1 || pickNumber(eventMetadata(failure), ["attempt"]) !== void 0);
|
|
2891
|
+
if (isFinalInChain && failureRun?.status === "error" && !logicalEvents.some(
|
|
2892
|
+
(event) => event.runId === failure.runId && event.eventId !== failure.eventId && isSuccessful(event) && canonicalName(event) === canonicalName(failure)
|
|
2893
|
+
)) {
|
|
2894
|
+
return {
|
|
2895
|
+
eventId: failure.eventId,
|
|
2896
|
+
runId: failure.runId,
|
|
2897
|
+
name: failure.name,
|
|
2898
|
+
kind: failure.kind,
|
|
2899
|
+
role: "terminal",
|
|
2900
|
+
confidence: failureRun.retryOf !== void 0 ? "explicit" : "correlated",
|
|
2901
|
+
basis: Object.freeze(["final-retry-chain-member", "enclosing-run-error"]),
|
|
2902
|
+
recoveryEventIds: Object.freeze([]),
|
|
2903
|
+
retryRunIds: Object.freeze(
|
|
2904
|
+
failureRun.retryOf !== void 0 ? [failureRun.retryOf] : []
|
|
2905
|
+
)
|
|
2906
|
+
};
|
|
2907
|
+
}
|
|
3076
2908
|
return {
|
|
3077
|
-
|
|
3078
|
-
|
|
2909
|
+
eventId: failure.eventId,
|
|
2910
|
+
runId: failure.runId,
|
|
2911
|
+
name: failure.name,
|
|
2912
|
+
kind: failure.kind,
|
|
2913
|
+
role: "unknown",
|
|
2914
|
+
confidence: "unknown",
|
|
2915
|
+
basis: Object.freeze(["no-explicit-or-correlated-recovery"]),
|
|
2916
|
+
recoveryEventIds: Object.freeze([]),
|
|
2917
|
+
retryRunIds: Object.freeze([])
|
|
3079
2918
|
};
|
|
3080
2919
|
}
|
|
3081
|
-
function
|
|
3082
|
-
const
|
|
3083
|
-
const
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
function pickString(record, keys) {
|
|
3096
|
-
if (!record) return void 0;
|
|
3097
|
-
for (const key of keys) {
|
|
3098
|
-
const value = record[key];
|
|
3099
|
-
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
3100
|
-
}
|
|
3101
|
-
return void 0;
|
|
3102
|
-
}
|
|
3103
|
-
|
|
3104
|
-
// packages/core/src/checks/derived-failure.ts
|
|
3105
|
-
function isRecord7(value) {
|
|
3106
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3107
|
-
}
|
|
3108
|
-
function pickString2(record, keys) {
|
|
3109
|
-
if (!record) return void 0;
|
|
3110
|
-
for (const key of keys) {
|
|
3111
|
-
const value = record[key];
|
|
3112
|
-
if (typeof value === "string" && value.trim() !== "") return value;
|
|
2920
|
+
function deriveFailureFacts(logicalEvents) {
|
|
2921
|
+
const runs = buildRunContexts(logicalEvents);
|
|
2922
|
+
const failures = logicalEvents.filter((event) => isFailure(event)).sort(compareEventOrder);
|
|
2923
|
+
const failureFacts = failures.map(
|
|
2924
|
+
(failure) => classifyFailure(failure, collectCandidates(failure, logicalEvents, runs), runs, logicalEvents)
|
|
2925
|
+
);
|
|
2926
|
+
const byRole = /* @__PURE__ */ new Map([
|
|
2927
|
+
["transient", []],
|
|
2928
|
+
["recovered", []],
|
|
2929
|
+
["terminal", []],
|
|
2930
|
+
["unknown", []]
|
|
2931
|
+
]);
|
|
2932
|
+
for (const fact of failureFacts) {
|
|
2933
|
+
byRole.get(fact.role).push(fact);
|
|
3113
2934
|
}
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
2935
|
+
for (const [role, list] of byRole) {
|
|
2936
|
+
byRole.set(
|
|
2937
|
+
role,
|
|
2938
|
+
Object.freeze(
|
|
2939
|
+
[...list].sort((a, b) => {
|
|
2940
|
+
const byRun = a.runId.localeCompare(b.runId);
|
|
2941
|
+
if (byRun !== 0) return byRun;
|
|
2942
|
+
return a.eventId.localeCompare(b.eventId);
|
|
2943
|
+
})
|
|
2944
|
+
)
|
|
2945
|
+
);
|
|
3123
2946
|
}
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
2947
|
+
const failureRoleCounts = {
|
|
2948
|
+
transient: byRole.get("transient").length,
|
|
2949
|
+
recovered: byRole.get("recovered").length,
|
|
2950
|
+
terminal: byRole.get("terminal").length,
|
|
2951
|
+
unknown: byRole.get("unknown").length
|
|
2952
|
+
};
|
|
3129
2953
|
return {
|
|
3130
|
-
|
|
3131
|
-
|
|
2954
|
+
failureFacts: Object.freeze(failureFacts),
|
|
2955
|
+
failuresByRole: byRole,
|
|
2956
|
+
failureRoleCounts
|
|
3132
2957
|
};
|
|
3133
2958
|
}
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
2959
|
+
|
|
2960
|
+
// packages/core/src/checks/relationship-facts.ts
|
|
2961
|
+
function attrs(event) {
|
|
2962
|
+
return event.attributes && typeof event.attributes === "object" ? event.attributes : {};
|
|
3137
2963
|
}
|
|
3138
|
-
function
|
|
3139
|
-
const
|
|
3140
|
-
|
|
3141
|
-
for (const key of ["linkedStepId", "toolCallId", "mcpToolCallId"]) {
|
|
3142
|
-
const value = pickString2(meta, [key]);
|
|
3143
|
-
if (value !== void 0) keys.push(`${key}:${value}`);
|
|
3144
|
-
}
|
|
3145
|
-
const stepId = pickString2(meta, ["stepId"]);
|
|
3146
|
-
if (stepId !== void 0) keys.push(`stepId:${stepId}`);
|
|
3147
|
-
return keys;
|
|
2964
|
+
function stringAttr(record, key) {
|
|
2965
|
+
const value = record[key];
|
|
2966
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
3148
2967
|
}
|
|
3149
|
-
function
|
|
3150
|
-
const
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
}
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
2968
|
+
function deriveRelationshipFacts(events) {
|
|
2969
|
+
const relationships = [];
|
|
2970
|
+
const diagnostics = [];
|
|
2971
|
+
const byId = new Map(events.map((event) => [event.eventId, event]));
|
|
2972
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2973
|
+
const push = (edge) => {
|
|
2974
|
+
const key = `${edge.type}:${edge.fromEventId}:${edge.toEventId ?? ""}:${edge.externalRef ?? ""}`;
|
|
2975
|
+
if (seen.has(key)) return;
|
|
2976
|
+
seen.add(key);
|
|
2977
|
+
relationships.push(edge);
|
|
2978
|
+
};
|
|
2979
|
+
for (const event of events) {
|
|
2980
|
+
if (event.parentId) {
|
|
2981
|
+
push({
|
|
2982
|
+
type: "parent-child",
|
|
2983
|
+
fromEventId: event.parentId,
|
|
2984
|
+
toEventId: event.eventId,
|
|
2985
|
+
confidence: byId.has(event.parentId) ? "explicit" : "unknown",
|
|
2986
|
+
basis: ["persisted.parentId"]
|
|
2987
|
+
});
|
|
2988
|
+
if (!byId.has(event.parentId)) {
|
|
2989
|
+
diagnostics.push({
|
|
2990
|
+
code: "AI_RELATIONSHIP_PARENT_MISSING",
|
|
2991
|
+
message: `parentId ${event.parentId} is not present in the event set.`,
|
|
2992
|
+
eventId: event.eventId
|
|
2993
|
+
});
|
|
2994
|
+
}
|
|
3159
2995
|
}
|
|
3160
|
-
|
|
3161
|
-
|
|
2996
|
+
const bag = attrs(event);
|
|
2997
|
+
const meta = extractSessionWorkflowMetadata(bag) ?? {};
|
|
2998
|
+
const nested = bag.metadata && typeof bag.metadata === "object" ? extractSessionWorkflowMetadata(bag.metadata) : void 0;
|
|
2999
|
+
const workflow = { ...meta, ...nested };
|
|
3000
|
+
if (workflow.retryOf) {
|
|
3001
|
+
const target = events.find((candidate) => candidate.runId === workflow.retryOf);
|
|
3002
|
+
push({
|
|
3003
|
+
type: "retry-of",
|
|
3004
|
+
fromEventId: event.eventId,
|
|
3005
|
+
...target ? { toEventId: target.eventId } : {},
|
|
3006
|
+
externalRef: workflow.retryOf,
|
|
3007
|
+
confidence: target ? "explicit" : "correlated",
|
|
3008
|
+
basis: ["attributes.retryOf"]
|
|
3009
|
+
});
|
|
3010
|
+
}
|
|
3011
|
+
const attemptOf = stringAttr(bag, "attemptOf") ?? stringAttr(bag, "operationId");
|
|
3012
|
+
if (attemptOf && workflow.attempt !== void 0) {
|
|
3013
|
+
push({
|
|
3014
|
+
type: "attempt-of",
|
|
3015
|
+
fromEventId: event.eventId,
|
|
3016
|
+
externalRef: attemptOf,
|
|
3017
|
+
confidence: "explicit",
|
|
3018
|
+
basis: workflow.attempt !== void 0 ? ["attributes.attempt", "attributes.operationId"] : ["attributes.operationId"]
|
|
3019
|
+
});
|
|
3020
|
+
}
|
|
3021
|
+
const fallbackOf = stringAttr(bag, "fallbackOf");
|
|
3022
|
+
if (fallbackOf) {
|
|
3023
|
+
push({
|
|
3024
|
+
type: "fallback-of",
|
|
3025
|
+
fromEventId: event.eventId,
|
|
3026
|
+
externalRef: fallbackOf,
|
|
3027
|
+
confidence: "explicit",
|
|
3028
|
+
basis: ["attributes.fallbackOf"]
|
|
3029
|
+
});
|
|
3030
|
+
}
|
|
3031
|
+
const remediationOf = stringAttr(bag, "remediationOf");
|
|
3032
|
+
if (remediationOf) {
|
|
3033
|
+
push({
|
|
3034
|
+
type: "remediation-of",
|
|
3035
|
+
fromEventId: event.eventId,
|
|
3036
|
+
externalRef: remediationOf,
|
|
3037
|
+
confidence: "explicit",
|
|
3038
|
+
basis: ["attributes.remediationOf"]
|
|
3039
|
+
});
|
|
3040
|
+
}
|
|
3041
|
+
const evidenceFor = stringAttr(bag, "evidenceFor");
|
|
3042
|
+
if (evidenceFor) {
|
|
3043
|
+
push({
|
|
3044
|
+
type: "evidence-for",
|
|
3045
|
+
fromEventId: event.eventId,
|
|
3046
|
+
...byId.has(evidenceFor) ? { toEventId: evidenceFor } : { externalRef: evidenceFor },
|
|
3047
|
+
confidence: byId.has(evidenceFor) ? "explicit" : "correlated",
|
|
3048
|
+
basis: ["attributes.evidenceFor"]
|
|
3049
|
+
});
|
|
3050
|
+
}
|
|
3051
|
+
const acceptedBy = stringAttr(bag, "acceptedBy");
|
|
3052
|
+
if (acceptedBy) {
|
|
3053
|
+
push({
|
|
3054
|
+
type: "accepted-by",
|
|
3055
|
+
fromEventId: event.eventId,
|
|
3056
|
+
...byId.has(acceptedBy) ? { toEventId: acceptedBy } : { externalRef: acceptedBy },
|
|
3057
|
+
confidence: byId.has(acceptedBy) ? "explicit" : "correlated",
|
|
3058
|
+
basis: ["attributes.acceptedBy"]
|
|
3059
|
+
});
|
|
3060
|
+
}
|
|
3061
|
+
const supersedes = stringAttr(bag, "supersedes");
|
|
3062
|
+
if (supersedes) {
|
|
3063
|
+
push({
|
|
3064
|
+
type: "supersedes",
|
|
3065
|
+
fromEventId: event.eventId,
|
|
3066
|
+
...byId.has(supersedes) ? { toEventId: supersedes } : { externalRef: supersedes },
|
|
3067
|
+
confidence: byId.has(supersedes) ? "explicit" : "correlated",
|
|
3068
|
+
basis: ["attributes.supersedes"]
|
|
3069
|
+
});
|
|
3070
|
+
}
|
|
3071
|
+
const sourceLineage = stringAttr(bag, "sourceLineage") ?? stringAttr(bag, "sourceEventId");
|
|
3072
|
+
if (sourceLineage) {
|
|
3073
|
+
push({
|
|
3074
|
+
type: "source-lineage",
|
|
3075
|
+
fromEventId: event.eventId,
|
|
3076
|
+
externalRef: sourceLineage,
|
|
3077
|
+
confidence: "explicit",
|
|
3078
|
+
basis: ["attributes.sourceLineage"]
|
|
3079
|
+
});
|
|
3080
|
+
}
|
|
3081
|
+
const unsupported = stringAttr(bag, "relationshipType");
|
|
3082
|
+
if (unsupported && unsupported !== "parent-child" && ![
|
|
3083
|
+
"source-lineage",
|
|
3084
|
+
"attempt-of",
|
|
3085
|
+
"retry-of",
|
|
3086
|
+
"fallback-of",
|
|
3087
|
+
"remediation-of",
|
|
3088
|
+
"evidence-for",
|
|
3089
|
+
"accepted-by",
|
|
3090
|
+
"supersedes"
|
|
3091
|
+
].includes(unsupported)) {
|
|
3092
|
+
diagnostics.push({
|
|
3093
|
+
code: "AI_RELATIONSHIP_UNSUPPORTED_TYPE",
|
|
3094
|
+
message: `Unsupported relationshipType ${unsupported} was reported without flattening.`,
|
|
3095
|
+
eventId: event.eventId
|
|
3096
|
+
});
|
|
3162
3097
|
}
|
|
3163
|
-
existing.retryOf ??= pickString2(meta, ["retryOf"]);
|
|
3164
|
-
existing.attempt ??= pickNumber(meta, ["attempt", "retryAttempt", "retryCount"]);
|
|
3165
|
-
existing.sessionId ??= pickString2(meta, ["sessionId", "conversationId"]);
|
|
3166
|
-
existing.groupId ??= pickString2(meta, ["groupId"]);
|
|
3167
|
-
existing.parentGroupId ??= pickString2(meta, ["parentGroupId"]);
|
|
3168
|
-
existing.fallbackOf ??= pickString2(meta, ["fallbackOf", "fallbackFrom"]);
|
|
3169
|
-
byRun.set(event.runId, existing);
|
|
3170
3098
|
}
|
|
3171
|
-
return
|
|
3099
|
+
return {
|
|
3100
|
+
relationships: Object.freeze(relationships),
|
|
3101
|
+
diagnostics: Object.freeze(diagnostics)
|
|
3102
|
+
};
|
|
3172
3103
|
}
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3104
|
+
|
|
3105
|
+
// packages/core/src/checks/trace-facts.ts
|
|
3106
|
+
function summarizeSemanticParity(events) {
|
|
3107
|
+
const projection = projectLogicalEvents(events);
|
|
3108
|
+
const logical = projection.logicalEvents;
|
|
3109
|
+
const finishedTools = logical.filter(
|
|
3110
|
+
(event) => event.kind === "TOOL" && event.status !== "running"
|
|
3111
|
+
);
|
|
3112
|
+
const finishedToolNames = Object.freeze(
|
|
3113
|
+
finishedTools.map((event) => resolveCanonicalToolName(event)).sort((a, b) => a.localeCompare(b))
|
|
3114
|
+
);
|
|
3115
|
+
const derived = deriveFailureFacts(logical);
|
|
3116
|
+
return {
|
|
3117
|
+
rawEventCount: events.length,
|
|
3118
|
+
logicalEventCount: logical.length,
|
|
3119
|
+
runningLogicalCount: logical.filter((event) => event.status === "running").length,
|
|
3120
|
+
finishedToolNames,
|
|
3121
|
+
finishedToolCount: finishedTools.length,
|
|
3122
|
+
pairedCount: logical.filter((event) => event.projection.paired).length,
|
|
3123
|
+
parentRemapCount: projection.diagnostics.filter(
|
|
3124
|
+
(item) => item.code === "AI_LOGICAL_PARENT_REMAPPED"
|
|
3125
|
+
).length,
|
|
3126
|
+
diagnostics: projection.diagnostics,
|
|
3127
|
+
failureRoleCounts: derived.failureRoleCounts
|
|
3128
|
+
};
|
|
3129
|
+
}
|
|
3130
|
+
var TRACE_FACTS_INPUT_NOT_NORMALIZED = formatProgrammaticDiagnostic(
|
|
3131
|
+
"AI_TRACE_FACTS_INPUT_NOT_NORMALIZED"
|
|
3132
|
+
);
|
|
3133
|
+
function isTraceReadResult(input) {
|
|
3134
|
+
if (typeof input !== "object" || input === null || Array.isArray(input)) {
|
|
3135
|
+
return false;
|
|
3178
3136
|
}
|
|
3179
|
-
|
|
3137
|
+
const record = input;
|
|
3138
|
+
return Array.isArray(record.events) && Array.isArray(record.runs) && typeof record.format === "string" && Array.isArray(record.warnings);
|
|
3180
3139
|
}
|
|
3181
|
-
function
|
|
3182
|
-
|
|
3140
|
+
function looksLikeRawV01TraceEvents(input) {
|
|
3141
|
+
if (!Array.isArray(input) || input.length === 0) return false;
|
|
3142
|
+
const first = input[0];
|
|
3143
|
+
if (typeof first !== "object" || first === null) return false;
|
|
3144
|
+
const row = first;
|
|
3145
|
+
return typeof row.event === "string" && (row.schemaVersion === "0.1" || row.eventId === void 0);
|
|
3183
3146
|
}
|
|
3184
|
-
function
|
|
3185
|
-
|
|
3147
|
+
function isPersistedInspectEventArray(input) {
|
|
3148
|
+
if (!Array.isArray(input)) return false;
|
|
3149
|
+
if (input.length === 0) return true;
|
|
3150
|
+
const first = input[0];
|
|
3151
|
+
if (typeof first !== "object" || first === null) return false;
|
|
3152
|
+
const row = first;
|
|
3153
|
+
return typeof row.eventId === "string" && (row.schemaVersion === "0.2" || row.schemaVersion === "1.0" || row.schemaVersion === "0.1") && typeof row.event !== "string";
|
|
3186
3154
|
}
|
|
3187
|
-
function
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3155
|
+
function resolveTraceFactsEvents(input) {
|
|
3156
|
+
if (isTraceReadResult(input)) {
|
|
3157
|
+
return input.events;
|
|
3158
|
+
}
|
|
3159
|
+
if (looksLikeRawV01TraceEvents(input)) {
|
|
3160
|
+
throw new TypeError(TRACE_FACTS_INPUT_NOT_NORMALIZED);
|
|
3161
|
+
}
|
|
3162
|
+
if (isPersistedInspectEventArray(input)) {
|
|
3163
|
+
return input;
|
|
3164
|
+
}
|
|
3165
|
+
throw new TypeError(TRACE_FACTS_INPUT_NOT_NORMALIZED);
|
|
3191
3166
|
}
|
|
3192
|
-
function
|
|
3193
|
-
const
|
|
3194
|
-
const
|
|
3195
|
-
const
|
|
3196
|
-
const
|
|
3197
|
-
const
|
|
3198
|
-
for (const event of logicalEvents) {
|
|
3199
|
-
if (event.
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
if (eventRun?.retryOf === failure.runId) {
|
|
3205
|
-
candidates.push({
|
|
3206
|
-
event,
|
|
3207
|
-
basis: "retryOf",
|
|
3208
|
-
confidence: "explicit",
|
|
3209
|
-
viaRunId: event.runId
|
|
3210
|
-
});
|
|
3211
|
-
continue;
|
|
3212
|
-
}
|
|
3213
|
-
if (eventRun?.fallbackOf === failure.runId || pickString2(eventMeta, ["fallbackOf", "fallbackFrom"]) === failure.runId) {
|
|
3214
|
-
candidates.push({
|
|
3215
|
-
event,
|
|
3216
|
-
basis: "fallbackOf",
|
|
3217
|
-
confidence: "explicit",
|
|
3218
|
-
viaRunId: event.runId
|
|
3219
|
-
});
|
|
3220
|
-
continue;
|
|
3167
|
+
function buildTraceFacts(input) {
|
|
3168
|
+
const events = resolveTraceFactsEvents(input);
|
|
3169
|
+
const projection = projectLogicalEvents(events);
|
|
3170
|
+
const toolsByName = /* @__PURE__ */ new Map();
|
|
3171
|
+
const llmEvents = [];
|
|
3172
|
+
const outcomeEvents = [];
|
|
3173
|
+
for (const event of projection.logicalEvents) {
|
|
3174
|
+
if (event.kind === "TOOL" && event.status !== "running") {
|
|
3175
|
+
const name = resolveCanonicalToolName(event);
|
|
3176
|
+
const list = toolsByName.get(name) ?? [];
|
|
3177
|
+
list.push(event);
|
|
3178
|
+
toolsByName.set(name, list);
|
|
3221
3179
|
}
|
|
3222
|
-
if (
|
|
3223
|
-
|
|
3224
|
-
const sharedLink = eventLinks.find((key) => failureLinks.has(key));
|
|
3225
|
-
if (sharedLink !== void 0) {
|
|
3226
|
-
candidates.push({
|
|
3227
|
-
event,
|
|
3228
|
-
basis: sharedLink.split(":")[0] ?? "linkedId",
|
|
3229
|
-
confidence: "explicit"
|
|
3230
|
-
});
|
|
3231
|
-
continue;
|
|
3180
|
+
if (event.kind === "LLM" && event.status !== "running") {
|
|
3181
|
+
llmEvents.push(event);
|
|
3232
3182
|
}
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
const differentParent = failure.parentId !== void 0 && event.parentId !== void 0 && failure.parentId !== event.parentId;
|
|
3236
|
-
const sessionScoped = failureRun !== void 0 && eventRun !== void 0 && sameCorrelationScope(failureRun, eventRun);
|
|
3237
|
-
if (canonicalName(event) === failureName && failureAttempt !== void 0 && eventAttempt !== void 0 && eventAttempt > failureAttempt && !differentParent && (sameParent || sessionScoped || sameRun)) {
|
|
3238
|
-
candidates.push({
|
|
3239
|
-
event,
|
|
3240
|
-
basis: "attempt-progression",
|
|
3241
|
-
confidence: "correlated",
|
|
3242
|
-
...event.runId !== failure.runId ? { viaRunId: event.runId } : {}
|
|
3243
|
-
});
|
|
3183
|
+
if (event.kind === "OUTCOME") {
|
|
3184
|
+
outcomeEvents.push(event);
|
|
3244
3185
|
}
|
|
3245
3186
|
}
|
|
3246
|
-
const
|
|
3247
|
-
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
|
|
3187
|
+
for (const [name, list] of [...toolsByName.entries()]) {
|
|
3188
|
+
toolsByName.set(name, Object.freeze([...list]));
|
|
3189
|
+
}
|
|
3190
|
+
const derived = deriveFailureFacts(projection.logicalEvents);
|
|
3191
|
+
const relationship = deriveRelationshipFacts(events);
|
|
3192
|
+
const summary = summarizeSemanticParity(events);
|
|
3193
|
+
return {
|
|
3194
|
+
rawEvents: Object.freeze([...events]),
|
|
3195
|
+
logicalEvents: projection.logicalEvents,
|
|
3196
|
+
diagnostics: projection.diagnostics,
|
|
3197
|
+
toolsByName,
|
|
3198
|
+
llmEvents: Object.freeze(llmEvents),
|
|
3199
|
+
outcomeEvents: Object.freeze(outcomeEvents),
|
|
3200
|
+
summary: {
|
|
3201
|
+
...summary,
|
|
3202
|
+
failureRoleCounts: derived.failureRoleCounts
|
|
3203
|
+
},
|
|
3204
|
+
failureFacts: derived.failureFacts,
|
|
3205
|
+
failuresByRole: derived.failuresByRole,
|
|
3206
|
+
relationships: relationship.relationships,
|
|
3207
|
+
relationshipDiagnostics: relationship.diagnostics
|
|
3208
|
+
};
|
|
3209
|
+
}
|
|
3210
|
+
|
|
3211
|
+
// packages/core/src/checks/index.ts
|
|
3212
|
+
var SEVERITY_RANK = {
|
|
3213
|
+
error: 0,
|
|
3214
|
+
warning: 1,
|
|
3215
|
+
info: 2
|
|
3216
|
+
};
|
|
3217
|
+
var STATUS_RANK = {
|
|
3218
|
+
fail: 0,
|
|
3219
|
+
warning: 1,
|
|
3220
|
+
pass: 2
|
|
3221
|
+
};
|
|
3222
|
+
var DEFAULT_SENSITIVE_KEYS = DEFAULT_CREDENTIAL_SENSITIVE_KEYS;
|
|
3223
|
+
var DEFAULT_RAW_CONTENT_KEYS = [
|
|
3224
|
+
"body",
|
|
3225
|
+
"headers",
|
|
3226
|
+
"input",
|
|
3227
|
+
"messages",
|
|
3228
|
+
"output",
|
|
3229
|
+
"payload",
|
|
3230
|
+
"prompt",
|
|
3231
|
+
"requestbody",
|
|
3232
|
+
"request_body",
|
|
3233
|
+
"responsebody",
|
|
3234
|
+
"response_body",
|
|
3235
|
+
"rawprompt",
|
|
3236
|
+
"raw_prompt",
|
|
3237
|
+
"rawoutput",
|
|
3238
|
+
"raw_output",
|
|
3239
|
+
"toolinput",
|
|
3240
|
+
"tool_input",
|
|
3241
|
+
"tooloutput",
|
|
3242
|
+
"tool_output",
|
|
3243
|
+
// Framework / agent metadata that carries user or task text
|
|
3244
|
+
"currenttask",
|
|
3245
|
+
"current_task",
|
|
3246
|
+
"task",
|
|
3247
|
+
"userinput",
|
|
3248
|
+
"user_input",
|
|
3249
|
+
"requesttext",
|
|
3250
|
+
"request_text",
|
|
3251
|
+
"conversationtext",
|
|
3252
|
+
"conversation_text"
|
|
3253
|
+
];
|
|
3254
|
+
var DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES = [
|
|
3255
|
+
"tokenUsage",
|
|
3256
|
+
"usage",
|
|
3257
|
+
"tokens"
|
|
3258
|
+
];
|
|
3259
|
+
var SAFE_USAGE_LEAF_KEYS = /* @__PURE__ */ new Set([
|
|
3260
|
+
"input",
|
|
3261
|
+
"output",
|
|
3262
|
+
"total",
|
|
3263
|
+
"cached",
|
|
3264
|
+
"input_tokens",
|
|
3265
|
+
"inputtokens",
|
|
3266
|
+
"output_tokens",
|
|
3267
|
+
"outputtokens",
|
|
3268
|
+
"total_tokens",
|
|
3269
|
+
"totaltokens",
|
|
3270
|
+
"prompt_tokens",
|
|
3271
|
+
"prompttokens",
|
|
3272
|
+
"completion_tokens",
|
|
3273
|
+
"completiontokens"
|
|
3274
|
+
]);
|
|
3275
|
+
var DEFAULT_SECRET_PATTERNS = [
|
|
3276
|
+
{ id: "bearer-token", pattern: /Bearer\s+[A-Za-z0-9._~+/-]{12,}=*/ },
|
|
3277
|
+
{ id: "openai-key", pattern: /sk-[A-Za-z0-9_-]{16,}/ },
|
|
3278
|
+
{ id: "aws-access-key", pattern: /AKIA[0-9A-Z]{16}/ },
|
|
3279
|
+
{ id: "github-token", pattern: /gh[opsu]_[A-Za-z0-9_]{20,}/ },
|
|
3280
|
+
// Keep in sync with packages/redact/src/key-value-secret.ts (KEY_VALUE_SECRET_PATTERN_SOURCE).
|
|
3281
|
+
{
|
|
3282
|
+
id: "key-value-secret",
|
|
3283
|
+
pattern: /\b(?:api[_-]?key|internal[_-]?token|access[_-]?token|auth[_-]?token|password|secret|token)=([^\s"'\\]{8,})/i
|
|
3284
|
+
}
|
|
3285
|
+
];
|
|
3286
|
+
function compareStrings(a, b) {
|
|
3287
|
+
return (a ?? "").localeCompare(b ?? "");
|
|
3288
|
+
}
|
|
3289
|
+
function diagnostic(code, message, ruleId) {
|
|
3290
|
+
return {
|
|
3291
|
+
code,
|
|
3292
|
+
message,
|
|
3293
|
+
severity: "error",
|
|
3294
|
+
...ruleId ? { ruleId } : {}
|
|
3295
|
+
};
|
|
3296
|
+
}
|
|
3297
|
+
function emptySummary() {
|
|
3298
|
+
return {
|
|
3299
|
+
passed: 0,
|
|
3300
|
+
failed: 0,
|
|
3301
|
+
warnings: 0,
|
|
3302
|
+
errors: 0,
|
|
3303
|
+
rulesEvaluated: 0
|
|
3304
|
+
};
|
|
3305
|
+
}
|
|
3306
|
+
function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
3307
|
+
return {
|
|
3308
|
+
ok: false,
|
|
3309
|
+
status: "error",
|
|
3310
|
+
format: input.read.format,
|
|
3311
|
+
...selectedRun ? { runId: selectedRun.runId } : {},
|
|
3312
|
+
summary: {
|
|
3313
|
+
...emptySummary(),
|
|
3314
|
+
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
3315
|
+
rulesEvaluated: ruleExecutions.length
|
|
3316
|
+
},
|
|
3317
|
+
findings: [],
|
|
3318
|
+
diagnostics: [...diagnostics],
|
|
3319
|
+
ruleExecutions: [...ruleExecutions]
|
|
3320
|
+
};
|
|
3321
|
+
}
|
|
3322
|
+
function flattenNodes(nodes) {
|
|
3323
|
+
return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
|
|
3324
|
+
}
|
|
3325
|
+
function buildFacts(input, selectedRun) {
|
|
3326
|
+
const scopedRuns = selectedRun ? [selectedRun] : input.read.runs;
|
|
3327
|
+
const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
|
|
3328
|
+
const scopedEvents = selectedRun === void 0 ? input.read.events : input.read.events.filter((event) => scopedRunIds.has(event.runId));
|
|
3329
|
+
const nodes = flattenNodes(scopedRuns.flatMap((run) => run.children));
|
|
3330
|
+
const nodesByEventId = /* @__PURE__ */ new Map();
|
|
3331
|
+
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
3332
|
+
for (const node of nodes) {
|
|
3333
|
+
nodesByEventId.set(node.event.eventId, node);
|
|
3334
|
+
const parentId = node.event.parentId;
|
|
3335
|
+
if (parentId) {
|
|
3336
|
+
const children = childrenByParentId.get(parentId) ?? [];
|
|
3337
|
+
children.push(node);
|
|
3338
|
+
childrenByParentId.set(parentId, children);
|
|
3251
3339
|
}
|
|
3252
3340
|
}
|
|
3253
|
-
|
|
3341
|
+
const projection = projectLogicalEvents(scopedEvents);
|
|
3342
|
+
return {
|
|
3343
|
+
format: input.read.format,
|
|
3344
|
+
runs: Object.freeze([...input.read.runs]),
|
|
3345
|
+
events: Object.freeze([...scopedEvents]),
|
|
3346
|
+
logicalEvents: projection.logicalEvents,
|
|
3347
|
+
logicalProjectionDiagnostics: projection.diagnostics,
|
|
3348
|
+
readerWarnings: Object.freeze([...input.read.warnings]),
|
|
3349
|
+
unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
|
|
3350
|
+
sourceFiles: Object.freeze([...input.read.sourceFiles]),
|
|
3351
|
+
nodesByEventId,
|
|
3352
|
+
childrenByParentId,
|
|
3353
|
+
rootNodes: Object.freeze(scopedRuns.flatMap((run) => run.children))
|
|
3354
|
+
};
|
|
3254
3355
|
}
|
|
3255
|
-
function
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
const retryRunIds = Object.freeze(
|
|
3259
|
-
[...new Set(candidates.map((c) => c.viaRunId).filter((id) => id !== void 0))].sort(
|
|
3260
|
-
(a, b) => a.localeCompare(b)
|
|
3261
|
-
)
|
|
3262
|
-
);
|
|
3263
|
-
if (successful.length > 1) {
|
|
3264
|
-
const distinctRuns = new Set(successful.map((c) => c.event.runId));
|
|
3265
|
-
const distinctParents = new Set(
|
|
3266
|
-
successful.map((c) => c.event.parentId ?? "").filter((id) => id !== "")
|
|
3267
|
-
);
|
|
3268
|
-
if (distinctRuns.size > 1 || distinctParents.size > 1) {
|
|
3356
|
+
function resolveSelectedRun(input, runId) {
|
|
3357
|
+
if (input.selectedRun) {
|
|
3358
|
+
if (runId && input.selectedRun.runId !== runId) {
|
|
3269
3359
|
return {
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
basis: Object.freeze(["ambiguous-recovery-candidates"]),
|
|
3277
|
-
recoveryEventIds: Object.freeze(
|
|
3278
|
-
successful.map((c) => c.event.eventId).sort((a, b) => a.localeCompare(b))
|
|
3279
|
-
),
|
|
3280
|
-
retryRunIds
|
|
3360
|
+
diagnostics: [
|
|
3361
|
+
diagnostic(
|
|
3362
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
3363
|
+
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
3364
|
+
)
|
|
3365
|
+
]
|
|
3281
3366
|
};
|
|
3282
3367
|
}
|
|
3368
|
+
return { run: input.selectedRun, diagnostics: [] };
|
|
3283
3369
|
}
|
|
3284
|
-
if (
|
|
3285
|
-
const
|
|
3286
|
-
|
|
3287
|
-
eventId: failure.eventId,
|
|
3288
|
-
runId: failure.runId,
|
|
3289
|
-
name: failure.name,
|
|
3290
|
-
kind: failure.kind,
|
|
3291
|
-
role: "recovered",
|
|
3292
|
-
confidence: best.confidence,
|
|
3293
|
-
basis: Object.freeze([best.basis]),
|
|
3294
|
-
recoveryEventIds: Object.freeze(
|
|
3295
|
-
successful.map((c) => c.event.eventId).sort((a, b) => a.localeCompare(b))
|
|
3296
|
-
),
|
|
3297
|
-
retryRunIds
|
|
3298
|
-
};
|
|
3299
|
-
}
|
|
3300
|
-
if (candidates.length > 0) {
|
|
3301
|
-
const best = candidates[0];
|
|
3302
|
-
return {
|
|
3303
|
-
eventId: failure.eventId,
|
|
3304
|
-
runId: failure.runId,
|
|
3305
|
-
name: failure.name,
|
|
3306
|
-
kind: failure.kind,
|
|
3307
|
-
role: "transient",
|
|
3308
|
-
confidence: best.confidence,
|
|
3309
|
-
basis: Object.freeze([
|
|
3310
|
-
best.basis,
|
|
3311
|
-
unsuccessful.some((c) => c.event.status === void 0) ? "retry-incomplete" : "retry-without-success"
|
|
3312
|
-
]),
|
|
3313
|
-
recoveryEventIds: Object.freeze([]),
|
|
3314
|
-
retryRunIds
|
|
3315
|
-
};
|
|
3316
|
-
}
|
|
3317
|
-
const failureMeta = eventMetadata(failure);
|
|
3318
|
-
const declaredSuccessor = pickString2(failureMeta, [
|
|
3319
|
-
"retriedBy",
|
|
3320
|
-
"nextRetryRunId",
|
|
3321
|
-
"retryRunId"
|
|
3322
|
-
]);
|
|
3323
|
-
if (declaredSuccessor !== void 0 && !runs.has(declaredSuccessor)) {
|
|
3324
|
-
return {
|
|
3325
|
-
eventId: failure.eventId,
|
|
3326
|
-
runId: failure.runId,
|
|
3327
|
-
name: failure.name,
|
|
3328
|
-
kind: failure.kind,
|
|
3329
|
-
role: "transient",
|
|
3330
|
-
confidence: "explicit",
|
|
3331
|
-
basis: Object.freeze(["retry-declared", "retry-run-missing"]),
|
|
3332
|
-
recoveryEventIds: Object.freeze([]),
|
|
3333
|
-
retryRunIds: Object.freeze([declaredSuccessor])
|
|
3334
|
-
};
|
|
3335
|
-
}
|
|
3336
|
-
for (const run of runs.values()) {
|
|
3337
|
-
if (run.retryOf === failure.runId) {
|
|
3370
|
+
if (runId) {
|
|
3371
|
+
const run = input.read.runs.find((candidate) => candidate.runId === runId);
|
|
3372
|
+
if (!run) {
|
|
3338
3373
|
return {
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
kind: failure.kind,
|
|
3343
|
-
role: "transient",
|
|
3344
|
-
confidence: "explicit",
|
|
3345
|
-
basis: Object.freeze(["retryOf", "retry-run-missing-or-empty"]),
|
|
3346
|
-
recoveryEventIds: Object.freeze([]),
|
|
3347
|
-
retryRunIds: Object.freeze([run.runId])
|
|
3374
|
+
diagnostics: [
|
|
3375
|
+
diagnostic("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
3376
|
+
]
|
|
3348
3377
|
};
|
|
3349
3378
|
}
|
|
3379
|
+
return { run, diagnostics: [] };
|
|
3350
3380
|
}
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
if (
|
|
3355
|
-
(event) => event.runId === failure.runId && event.eventId !== failure.eventId && isSuccessful(event) && canonicalName(event) === canonicalName(failure)
|
|
3356
|
-
)) {
|
|
3381
|
+
if (input.read.runs.length === 1) {
|
|
3382
|
+
return { run: input.read.runs[0], diagnostics: [] };
|
|
3383
|
+
}
|
|
3384
|
+
if (input.read.runs.length === 0) {
|
|
3357
3385
|
return {
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
failureRun.retryOf !== void 0 ? [failureRun.retryOf] : []
|
|
3368
|
-
)
|
|
3386
|
+
diagnostics: [
|
|
3387
|
+
diagnostic(
|
|
3388
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
3389
|
+
formatProgrammaticDiagnostic(
|
|
3390
|
+
"AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
3391
|
+
"No runs are available for checks."
|
|
3392
|
+
)
|
|
3393
|
+
)
|
|
3394
|
+
]
|
|
3369
3395
|
};
|
|
3370
|
-
}
|
|
3371
|
-
return {
|
|
3372
|
-
eventId: failure.eventId,
|
|
3373
|
-
runId: failure.runId,
|
|
3374
|
-
name: failure.name,
|
|
3375
|
-
kind: failure.kind,
|
|
3376
|
-
role: "unknown",
|
|
3377
|
-
confidence: "unknown",
|
|
3378
|
-
basis: Object.freeze(["no-explicit-or-correlated-recovery"]),
|
|
3379
|
-
recoveryEventIds: Object.freeze([]),
|
|
3380
|
-
retryRunIds: Object.freeze([])
|
|
3381
|
-
};
|
|
3382
|
-
}
|
|
3383
|
-
function deriveFailureFacts(logicalEvents) {
|
|
3384
|
-
const runs = buildRunContexts(logicalEvents);
|
|
3385
|
-
const failures = logicalEvents.filter((event) => isFailure(event)).sort(compareEventOrder);
|
|
3386
|
-
const failureFacts = failures.map(
|
|
3387
|
-
(failure) => classifyFailure(failure, collectCandidates(failure, logicalEvents, runs), runs, logicalEvents)
|
|
3388
|
-
);
|
|
3389
|
-
const byRole = /* @__PURE__ */ new Map([
|
|
3390
|
-
["transient", []],
|
|
3391
|
-
["recovered", []],
|
|
3392
|
-
["terminal", []],
|
|
3393
|
-
["unknown", []]
|
|
3394
|
-
]);
|
|
3395
|
-
for (const fact of failureFacts) {
|
|
3396
|
-
byRole.get(fact.role).push(fact);
|
|
3397
|
-
}
|
|
3398
|
-
for (const [role, list] of byRole) {
|
|
3399
|
-
byRole.set(
|
|
3400
|
-
role,
|
|
3401
|
-
Object.freeze(
|
|
3402
|
-
[...list].sort((a, b) => {
|
|
3403
|
-
const byRun = a.runId.localeCompare(b.runId);
|
|
3404
|
-
if (byRun !== 0) return byRun;
|
|
3405
|
-
return a.eventId.localeCompare(b.eventId);
|
|
3406
|
-
})
|
|
3407
|
-
)
|
|
3408
|
-
);
|
|
3409
|
-
}
|
|
3410
|
-
const failureRoleCounts = {
|
|
3411
|
-
transient: byRole.get("transient").length,
|
|
3412
|
-
recovered: byRole.get("recovered").length,
|
|
3413
|
-
terminal: byRole.get("terminal").length,
|
|
3414
|
-
unknown: byRole.get("unknown").length
|
|
3415
|
-
};
|
|
3396
|
+
}
|
|
3416
3397
|
return {
|
|
3417
|
-
|
|
3418
|
-
|
|
3419
|
-
|
|
3398
|
+
diagnostics: [
|
|
3399
|
+
diagnostic(
|
|
3400
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
3401
|
+
formatProgrammaticDiagnostic("AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED")
|
|
3402
|
+
)
|
|
3403
|
+
]
|
|
3420
3404
|
};
|
|
3421
3405
|
}
|
|
3422
|
-
|
|
3423
|
-
// packages/core/src/checks/relationship-facts.ts
|
|
3424
|
-
function attrs(event) {
|
|
3425
|
-
return event.attributes && typeof event.attributes === "object" ? event.attributes : {};
|
|
3426
|
-
}
|
|
3427
|
-
function stringAttr(record, key) {
|
|
3428
|
-
const value = record[key];
|
|
3429
|
-
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
3430
|
-
}
|
|
3431
|
-
function deriveRelationshipFacts(events) {
|
|
3432
|
-
const relationships = [];
|
|
3406
|
+
function selectRules(rules, selectedIds) {
|
|
3433
3407
|
const diagnostics = [];
|
|
3434
|
-
const byId = new Map(
|
|
3435
|
-
const
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
};
|
|
3442
|
-
for (const event of events) {
|
|
3443
|
-
if (event.parentId) {
|
|
3444
|
-
push({
|
|
3445
|
-
type: "parent-child",
|
|
3446
|
-
fromEventId: event.parentId,
|
|
3447
|
-
toEventId: event.eventId,
|
|
3448
|
-
confidence: byId.has(event.parentId) ? "explicit" : "unknown",
|
|
3449
|
-
basis: ["persisted.parentId"]
|
|
3450
|
-
});
|
|
3451
|
-
if (!byId.has(event.parentId)) {
|
|
3452
|
-
diagnostics.push({
|
|
3453
|
-
code: "AI_RELATIONSHIP_PARENT_MISSING",
|
|
3454
|
-
message: `parentId ${event.parentId} is not present in the event set.`,
|
|
3455
|
-
eventId: event.eventId
|
|
3456
|
-
});
|
|
3457
|
-
}
|
|
3458
|
-
}
|
|
3459
|
-
const bag = attrs(event);
|
|
3460
|
-
const meta = extractSessionWorkflowMetadata(bag) ?? {};
|
|
3461
|
-
const nested = bag.metadata && typeof bag.metadata === "object" ? extractSessionWorkflowMetadata(bag.metadata) : void 0;
|
|
3462
|
-
const workflow = { ...meta, ...nested };
|
|
3463
|
-
if (workflow.retryOf) {
|
|
3464
|
-
const target = events.find((candidate) => candidate.runId === workflow.retryOf);
|
|
3465
|
-
push({
|
|
3466
|
-
type: "retry-of",
|
|
3467
|
-
fromEventId: event.eventId,
|
|
3468
|
-
...target ? { toEventId: target.eventId } : {},
|
|
3469
|
-
externalRef: workflow.retryOf,
|
|
3470
|
-
confidence: target ? "explicit" : "correlated",
|
|
3471
|
-
basis: ["attributes.retryOf"]
|
|
3472
|
-
});
|
|
3473
|
-
}
|
|
3474
|
-
const attemptOf = stringAttr(bag, "attemptOf") ?? stringAttr(bag, "operationId");
|
|
3475
|
-
if (attemptOf && workflow.attempt !== void 0) {
|
|
3476
|
-
push({
|
|
3477
|
-
type: "attempt-of",
|
|
3478
|
-
fromEventId: event.eventId,
|
|
3479
|
-
externalRef: attemptOf,
|
|
3480
|
-
confidence: "explicit",
|
|
3481
|
-
basis: workflow.attempt !== void 0 ? ["attributes.attempt", "attributes.operationId"] : ["attributes.operationId"]
|
|
3482
|
-
});
|
|
3483
|
-
}
|
|
3484
|
-
const fallbackOf = stringAttr(bag, "fallbackOf");
|
|
3485
|
-
if (fallbackOf) {
|
|
3486
|
-
push({
|
|
3487
|
-
type: "fallback-of",
|
|
3488
|
-
fromEventId: event.eventId,
|
|
3489
|
-
externalRef: fallbackOf,
|
|
3490
|
-
confidence: "explicit",
|
|
3491
|
-
basis: ["attributes.fallbackOf"]
|
|
3492
|
-
});
|
|
3493
|
-
}
|
|
3494
|
-
const remediationOf = stringAttr(bag, "remediationOf");
|
|
3495
|
-
if (remediationOf) {
|
|
3496
|
-
push({
|
|
3497
|
-
type: "remediation-of",
|
|
3498
|
-
fromEventId: event.eventId,
|
|
3499
|
-
externalRef: remediationOf,
|
|
3500
|
-
confidence: "explicit",
|
|
3501
|
-
basis: ["attributes.remediationOf"]
|
|
3502
|
-
});
|
|
3503
|
-
}
|
|
3504
|
-
const evidenceFor = stringAttr(bag, "evidenceFor");
|
|
3505
|
-
if (evidenceFor) {
|
|
3506
|
-
push({
|
|
3507
|
-
type: "evidence-for",
|
|
3508
|
-
fromEventId: event.eventId,
|
|
3509
|
-
...byId.has(evidenceFor) ? { toEventId: evidenceFor } : { externalRef: evidenceFor },
|
|
3510
|
-
confidence: byId.has(evidenceFor) ? "explicit" : "correlated",
|
|
3511
|
-
basis: ["attributes.evidenceFor"]
|
|
3512
|
-
});
|
|
3513
|
-
}
|
|
3514
|
-
const acceptedBy = stringAttr(bag, "acceptedBy");
|
|
3515
|
-
if (acceptedBy) {
|
|
3516
|
-
push({
|
|
3517
|
-
type: "accepted-by",
|
|
3518
|
-
fromEventId: event.eventId,
|
|
3519
|
-
...byId.has(acceptedBy) ? { toEventId: acceptedBy } : { externalRef: acceptedBy },
|
|
3520
|
-
confidence: byId.has(acceptedBy) ? "explicit" : "correlated",
|
|
3521
|
-
basis: ["attributes.acceptedBy"]
|
|
3522
|
-
});
|
|
3408
|
+
const byId = /* @__PURE__ */ new Map();
|
|
3409
|
+
for (const rule of rules) {
|
|
3410
|
+
if (byId.has(rule.id)) {
|
|
3411
|
+
diagnostics.push(
|
|
3412
|
+
diagnostic("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
3413
|
+
);
|
|
3414
|
+
continue;
|
|
3523
3415
|
}
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
|
|
3416
|
+
byId.set(rule.id, rule);
|
|
3417
|
+
}
|
|
3418
|
+
if (selectedIds && selectedIds.length > 0) {
|
|
3419
|
+
const selected = new Set(selectedIds);
|
|
3420
|
+
for (const id of selected) {
|
|
3421
|
+
if (!byId.has(id)) {
|
|
3422
|
+
diagnostics.push(
|
|
3423
|
+
diagnostic("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
3424
|
+
);
|
|
3425
|
+
}
|
|
3533
3426
|
}
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3542
|
-
|
|
3427
|
+
return {
|
|
3428
|
+
rules: [...byId.values()].filter((rule) => selected.has(rule.id)).sort(compareRules),
|
|
3429
|
+
diagnostics
|
|
3430
|
+
};
|
|
3431
|
+
}
|
|
3432
|
+
return { rules: [...byId.values()].sort(compareRules), diagnostics };
|
|
3433
|
+
}
|
|
3434
|
+
function compareRules(a, b) {
|
|
3435
|
+
return a.id.localeCompare(b.id);
|
|
3436
|
+
}
|
|
3437
|
+
function eventTimestamp(finding, eventById) {
|
|
3438
|
+
const eventId = finding.evidence[0]?.eventId;
|
|
3439
|
+
return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
|
|
3440
|
+
}
|
|
3441
|
+
function compareFindings(eventById) {
|
|
3442
|
+
return (a, b) => {
|
|
3443
|
+
if (SEVERITY_RANK[a.severity] !== SEVERITY_RANK[b.severity]) {
|
|
3444
|
+
return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity];
|
|
3543
3445
|
}
|
|
3544
|
-
const
|
|
3545
|
-
if (
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
"retry-of",
|
|
3549
|
-
"fallback-of",
|
|
3550
|
-
"remediation-of",
|
|
3551
|
-
"evidence-for",
|
|
3552
|
-
"accepted-by",
|
|
3553
|
-
"supersedes"
|
|
3554
|
-
].includes(unsupported)) {
|
|
3555
|
-
diagnostics.push({
|
|
3556
|
-
code: "AI_RELATIONSHIP_UNSUPPORTED_TYPE",
|
|
3557
|
-
message: `Unsupported relationshipType ${unsupported} was reported without flattening.`,
|
|
3558
|
-
eventId: event.eventId
|
|
3559
|
-
});
|
|
3446
|
+
const byRule = a.ruleId.localeCompare(b.ruleId);
|
|
3447
|
+
if (byRule !== 0) return byRule;
|
|
3448
|
+
if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
|
|
3449
|
+
return STATUS_RANK[a.status] - STATUS_RANK[b.status];
|
|
3560
3450
|
}
|
|
3561
|
-
|
|
3451
|
+
const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
|
|
3452
|
+
if (byRun !== 0) return byRun;
|
|
3453
|
+
const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
|
|
3454
|
+
if (byTime !== 0) return byTime;
|
|
3455
|
+
const byEvent = compareStrings(a.evidence[0]?.eventId, b.evidence[0]?.eventId);
|
|
3456
|
+
if (byEvent !== 0) return byEvent;
|
|
3457
|
+
return compareStrings(a.evidence[0]?.path, b.evidence[0]?.path);
|
|
3458
|
+
};
|
|
3459
|
+
}
|
|
3460
|
+
function normalizeFinding(rule, finding) {
|
|
3562
3461
|
return {
|
|
3563
|
-
|
|
3564
|
-
|
|
3462
|
+
ruleId: finding.ruleId || rule.id,
|
|
3463
|
+
severity: finding.severity ?? rule.defaultSeverity,
|
|
3464
|
+
status: finding.status,
|
|
3465
|
+
message: finding.message,
|
|
3466
|
+
...finding.expected !== void 0 ? { expected: finding.expected } : {},
|
|
3467
|
+
...finding.actual !== void 0 ? { actual: finding.actual } : {},
|
|
3468
|
+
evidence: [...finding.evidence ?? []],
|
|
3469
|
+
...finding.category !== void 0 ? { category: finding.category } : {},
|
|
3470
|
+
...finding.confidence !== void 0 ? { confidence: finding.confidence } : {},
|
|
3471
|
+
...finding.detector !== void 0 ? { detector: finding.detector } : {},
|
|
3472
|
+
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
3565
3473
|
};
|
|
3566
3474
|
}
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3475
|
+
function summarize(findings, diagnostics, rulesEvaluated) {
|
|
3476
|
+
return {
|
|
3477
|
+
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
3478
|
+
failed: findings.filter(
|
|
3479
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
3480
|
+
).length,
|
|
3481
|
+
warnings: findings.filter(
|
|
3482
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
3483
|
+
).length,
|
|
3484
|
+
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
3485
|
+
rulesEvaluated
|
|
3486
|
+
};
|
|
3487
|
+
}
|
|
3488
|
+
function classifyRuleExecution(findings, threw) {
|
|
3489
|
+
if (findings.some((finding) => finding.status === "fail" && finding.severity === "error")) {
|
|
3490
|
+
return "fail";
|
|
3491
|
+
}
|
|
3492
|
+
if (findings.some(
|
|
3493
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
3494
|
+
)) {
|
|
3495
|
+
return "warning";
|
|
3496
|
+
}
|
|
3497
|
+
return "pass";
|
|
3498
|
+
}
|
|
3499
|
+
function eventEvidence(event, path16) {
|
|
3500
|
+
return {
|
|
3501
|
+
runId: event.runId,
|
|
3502
|
+
eventId: event.eventId,
|
|
3503
|
+
parentId: event.parentId,
|
|
3504
|
+
traceId: event.trace?.traceId,
|
|
3505
|
+
spanId: event.trace?.spanId,
|
|
3506
|
+
kind: event.kind,
|
|
3507
|
+
name: event.name,
|
|
3508
|
+
status: event.status,
|
|
3509
|
+
...path16 ? { path: path16 } : {}
|
|
3510
|
+
};
|
|
3511
|
+
}
|
|
3512
|
+
function runEvidence(run) {
|
|
3513
|
+
return run ? [{ runId: run.runId, name: run.name, status: run.status }] : [];
|
|
3514
|
+
}
|
|
3515
|
+
function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
3579
3516
|
return {
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3517
|
+
ruleId,
|
|
3518
|
+
severity: meta?.severity ?? "error",
|
|
3519
|
+
status: meta?.status ?? "fail",
|
|
3520
|
+
message,
|
|
3521
|
+
...expected !== void 0 ? { expected } : {},
|
|
3522
|
+
...actual !== void 0 ? { actual } : {},
|
|
3523
|
+
evidence: [...evidence],
|
|
3524
|
+
...meta?.category !== void 0 ? { category: meta.category } : {},
|
|
3525
|
+
...meta?.confidence !== void 0 ? { confidence: meta.confidence } : {},
|
|
3526
|
+
...meta?.detector !== void 0 ? { detector: meta.detector } : {},
|
|
3527
|
+
...meta?.action !== void 0 ? { action: meta.action } : {}
|
|
3591
3528
|
};
|
|
3592
3529
|
}
|
|
3593
|
-
|
|
3594
|
-
|
|
3595
|
-
);
|
|
3596
|
-
function isTraceReadResult(input) {
|
|
3597
|
-
if (typeof input !== "object" || input === null || Array.isArray(input)) {
|
|
3598
|
-
return false;
|
|
3599
|
-
}
|
|
3600
|
-
const record = input;
|
|
3601
|
-
return Array.isArray(record.events) && Array.isArray(record.runs) && typeof record.format === "string" && Array.isArray(record.warnings);
|
|
3530
|
+
function semanticEvents(context) {
|
|
3531
|
+
return context.logicalEvents ?? context.events;
|
|
3602
3532
|
}
|
|
3603
|
-
function
|
|
3604
|
-
|
|
3605
|
-
const first = input[0];
|
|
3606
|
-
if (typeof first !== "object" || first === null) return false;
|
|
3607
|
-
const row = first;
|
|
3608
|
-
return typeof row.event === "string" && (row.schemaVersion === "0.1" || row.eventId === void 0);
|
|
3533
|
+
function isRecord8(value) {
|
|
3534
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3609
3535
|
}
|
|
3610
|
-
function
|
|
3611
|
-
|
|
3612
|
-
if (input.length === 0) return true;
|
|
3613
|
-
const first = input[0];
|
|
3614
|
-
if (typeof first !== "object" || first === null) return false;
|
|
3615
|
-
const row = first;
|
|
3616
|
-
return typeof row.eventId === "string" && (row.schemaVersion === "0.2" || row.schemaVersion === "1.0" || row.schemaVersion === "0.1") && typeof row.event !== "string";
|
|
3536
|
+
function normalizedKey(value) {
|
|
3537
|
+
return value.toLowerCase().replace(/[^a-z0-9_]/g, "");
|
|
3617
3538
|
}
|
|
3618
|
-
function
|
|
3619
|
-
|
|
3620
|
-
|
|
3539
|
+
function lastPathSegment(path16) {
|
|
3540
|
+
const parts = path16.split(".");
|
|
3541
|
+
return parts[parts.length - 1] ?? path16;
|
|
3542
|
+
}
|
|
3543
|
+
function valueType(value) {
|
|
3544
|
+
if (Array.isArray(value)) return "array";
|
|
3545
|
+
if (value === null) return "null";
|
|
3546
|
+
return typeof value;
|
|
3547
|
+
}
|
|
3548
|
+
function serializedByteLength(value) {
|
|
3549
|
+
try {
|
|
3550
|
+
return Buffer.byteLength(JSON.stringify(value), "utf-8");
|
|
3551
|
+
} catch {
|
|
3552
|
+
return void 0;
|
|
3621
3553
|
}
|
|
3622
|
-
|
|
3623
|
-
|
|
3554
|
+
}
|
|
3555
|
+
function pushValueEntries(entries, event, value, path16, key, depth = 0) {
|
|
3556
|
+
entries.push({ event, path: path16, key, value });
|
|
3557
|
+
if (depth >= 8) return;
|
|
3558
|
+
if (Array.isArray(value)) {
|
|
3559
|
+
for (const [index, item] of value.entries()) {
|
|
3560
|
+
pushValueEntries(entries, event, item, `${path16}.${index}`, String(index), depth + 1);
|
|
3561
|
+
}
|
|
3562
|
+
return;
|
|
3624
3563
|
}
|
|
3625
|
-
if (
|
|
3626
|
-
|
|
3564
|
+
if (!isRecord8(value)) return;
|
|
3565
|
+
for (const nestedKey of Object.keys(value).sort((a, b) => a.localeCompare(b))) {
|
|
3566
|
+
pushValueEntries(
|
|
3567
|
+
entries,
|
|
3568
|
+
event,
|
|
3569
|
+
value[nestedKey],
|
|
3570
|
+
`${path16}.${nestedKey}`,
|
|
3571
|
+
nestedKey,
|
|
3572
|
+
depth + 1
|
|
3573
|
+
);
|
|
3627
3574
|
}
|
|
3628
|
-
throw new TypeError(TRACE_FACTS_INPUT_NOT_NORMALIZED);
|
|
3629
3575
|
}
|
|
3630
|
-
function
|
|
3631
|
-
const
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
const name = resolveCanonicalToolName(event);
|
|
3639
|
-
const list = toolsByName.get(name) ?? [];
|
|
3640
|
-
list.push(event);
|
|
3641
|
-
toolsByName.set(name, list);
|
|
3642
|
-
}
|
|
3643
|
-
if (event.kind === "LLM" && event.status !== "running") {
|
|
3644
|
-
llmEvents.push(event);
|
|
3576
|
+
function eventValueEntries(event, options = {}) {
|
|
3577
|
+
const entries = [];
|
|
3578
|
+
if (event.attributes !== void 0) {
|
|
3579
|
+
pushValueEntries(entries, event, event.attributes, "attributes", "attributes");
|
|
3580
|
+
}
|
|
3581
|
+
if (options.includeSummaries) {
|
|
3582
|
+
if (event.inputSummary !== void 0) {
|
|
3583
|
+
pushValueEntries(entries, event, event.inputSummary, "inputSummary", "inputSummary");
|
|
3645
3584
|
}
|
|
3646
|
-
if (event.
|
|
3647
|
-
|
|
3585
|
+
if (event.outputSummary !== void 0) {
|
|
3586
|
+
pushValueEntries(entries, event, event.outputSummary, "outputSummary", "outputSummary");
|
|
3648
3587
|
}
|
|
3649
3588
|
}
|
|
3650
|
-
|
|
3651
|
-
|
|
3589
|
+
if (options.includeError && event.error !== void 0) {
|
|
3590
|
+
pushValueEntries(entries, event, event.error, "error", "error");
|
|
3652
3591
|
}
|
|
3653
|
-
|
|
3654
|
-
const relationship = deriveRelationshipFacts(events);
|
|
3655
|
-
const summary = summarizeSemanticParity(events);
|
|
3656
|
-
return {
|
|
3657
|
-
rawEvents: Object.freeze([...events]),
|
|
3658
|
-
logicalEvents: projection.logicalEvents,
|
|
3659
|
-
diagnostics: projection.diagnostics,
|
|
3660
|
-
toolsByName,
|
|
3661
|
-
llmEvents: Object.freeze(llmEvents),
|
|
3662
|
-
outcomeEvents: Object.freeze(outcomeEvents),
|
|
3663
|
-
summary: {
|
|
3664
|
-
...summary,
|
|
3665
|
-
failureRoleCounts: derived.failureRoleCounts
|
|
3666
|
-
},
|
|
3667
|
-
failureFacts: derived.failureFacts,
|
|
3668
|
-
failuresByRole: derived.failuresByRole,
|
|
3669
|
-
relationships: relationship.relationships,
|
|
3670
|
-
relationshipDiagnostics: relationship.diagnostics
|
|
3671
|
-
};
|
|
3592
|
+
return entries;
|
|
3672
3593
|
}
|
|
3673
|
-
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
// packages/core/src/checks/index.ts
|
|
3678
|
-
var SEVERITY_RANK = {
|
|
3679
|
-
error: 0,
|
|
3680
|
-
warning: 1,
|
|
3681
|
-
info: 2
|
|
3682
|
-
};
|
|
3683
|
-
var STATUS_RANK = {
|
|
3684
|
-
fail: 0,
|
|
3685
|
-
warning: 1,
|
|
3686
|
-
pass: 2
|
|
3687
|
-
};
|
|
3688
|
-
var DEFAULT_SENSITIVE_KEYS = DEFAULT_CREDENTIAL_SENSITIVE_KEYS;
|
|
3689
|
-
var DEFAULT_RAW_CONTENT_KEYS = [
|
|
3690
|
-
"body",
|
|
3691
|
-
"headers",
|
|
3692
|
-
"input",
|
|
3693
|
-
"messages",
|
|
3694
|
-
"output",
|
|
3695
|
-
"payload",
|
|
3696
|
-
"prompt",
|
|
3697
|
-
"requestbody",
|
|
3698
|
-
"request_body",
|
|
3699
|
-
"responsebody",
|
|
3700
|
-
"response_body",
|
|
3701
|
-
"rawprompt",
|
|
3702
|
-
"raw_prompt",
|
|
3703
|
-
"rawoutput",
|
|
3704
|
-
"raw_output",
|
|
3705
|
-
"toolinput",
|
|
3706
|
-
"tool_input",
|
|
3707
|
-
"tooloutput",
|
|
3708
|
-
"tool_output",
|
|
3709
|
-
// Framework / agent metadata that carries user or task text
|
|
3710
|
-
"currenttask",
|
|
3711
|
-
"current_task",
|
|
3712
|
-
"task",
|
|
3713
|
-
"userinput",
|
|
3714
|
-
"user_input",
|
|
3715
|
-
"requesttext",
|
|
3716
|
-
"request_text",
|
|
3717
|
-
"conversationtext",
|
|
3718
|
-
"conversation_text"
|
|
3719
|
-
];
|
|
3720
|
-
var DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES = [
|
|
3721
|
-
"tokenUsage",
|
|
3722
|
-
"usage",
|
|
3723
|
-
"tokens"
|
|
3724
|
-
];
|
|
3725
|
-
var SAFE_USAGE_LEAF_KEYS = /* @__PURE__ */ new Set([
|
|
3726
|
-
"input",
|
|
3727
|
-
"output",
|
|
3728
|
-
"total",
|
|
3729
|
-
"cached",
|
|
3730
|
-
"input_tokens",
|
|
3731
|
-
"inputtokens",
|
|
3732
|
-
"output_tokens",
|
|
3733
|
-
"outputtokens",
|
|
3734
|
-
"total_tokens",
|
|
3735
|
-
"totaltokens",
|
|
3736
|
-
"prompt_tokens",
|
|
3737
|
-
"prompttokens",
|
|
3738
|
-
"completion_tokens",
|
|
3739
|
-
"completiontokens"
|
|
3740
|
-
]);
|
|
3741
|
-
var DEFAULT_SECRET_PATTERNS = [
|
|
3742
|
-
{ id: "bearer-token", pattern: /Bearer\s+[A-Za-z0-9._~+/-]{12,}=*/ },
|
|
3743
|
-
{ id: "openai-key", pattern: /sk-[A-Za-z0-9_-]{16,}/ },
|
|
3744
|
-
{ id: "aws-access-key", pattern: /AKIA[0-9A-Z]{16}/ },
|
|
3745
|
-
{ id: "github-token", pattern: /gh[opsu]_[A-Za-z0-9_]{20,}/ },
|
|
3746
|
-
// Keep in sync with packages/redact/src/key-value-secret.ts (KEY_VALUE_SECRET_PATTERN_SOURCE).
|
|
3747
|
-
{
|
|
3748
|
-
id: "key-value-secret",
|
|
3749
|
-
pattern: /\b(?:api[_-]?key|internal[_-]?token|access[_-]?token|auth[_-]?token|password|secret|token)=([^\s"'\\]{8,})/i
|
|
3750
|
-
}
|
|
3751
|
-
];
|
|
3752
|
-
function compareStrings(a, b) {
|
|
3753
|
-
return (a ?? "").localeCompare(b ?? "");
|
|
3594
|
+
function limitFindings(findings, maxFindings) {
|
|
3595
|
+
if (maxFindings === void 0 || findings.length <= maxFindings) return findings;
|
|
3596
|
+
return findings.slice(0, Math.max(0, maxFindings));
|
|
3754
3597
|
}
|
|
3755
|
-
function
|
|
3598
|
+
function hasRedactionMarker(value, markers) {
|
|
3599
|
+
return markers.some((marker) => value.includes(marker)) || /^\[HASH:[A-Za-z0-9_-]+\]$/.test(value);
|
|
3600
|
+
}
|
|
3601
|
+
function isSensitiveKey(key, sensitiveKeys) {
|
|
3602
|
+
return isCredentialSensitiveKey(key, sensitiveKeys);
|
|
3603
|
+
}
|
|
3604
|
+
function isRawContentKey(key, forbiddenKeys) {
|
|
3605
|
+
if (!key) return false;
|
|
3606
|
+
const normalized = normalizedKey(key);
|
|
3607
|
+
return forbiddenKeys.some((forbidden) => normalized === normalizedKey(forbidden));
|
|
3608
|
+
}
|
|
3609
|
+
function isSafeRawContentMetricPath(path16, key, safePathPrefixes) {
|
|
3610
|
+
const leaf = normalizedKey(key ?? lastPathSegment(path16));
|
|
3611
|
+
if (!SAFE_USAGE_LEAF_KEYS.has(leaf)) return false;
|
|
3612
|
+
const parts = path16.split(".").filter(Boolean);
|
|
3613
|
+
if (parts.length < 2) return false;
|
|
3614
|
+
const parent = parts[parts.length - 2] ?? "";
|
|
3615
|
+
const parentNorm = normalizedKey(parent);
|
|
3616
|
+
return safePathPrefixes.some((prefix) => parentNorm === normalizedKey(prefix));
|
|
3617
|
+
}
|
|
3618
|
+
function isRawContentPath(path16, key, forbiddenKeys, safePathPrefixes) {
|
|
3619
|
+
if (isSafeRawContentMetricPath(path16, key, safePathPrefixes)) return false;
|
|
3620
|
+
return isRawContentKey(key ?? lastPathSegment(path16), forbiddenKeys);
|
|
3621
|
+
}
|
|
3622
|
+
function createRunStatusRule(options = {}) {
|
|
3623
|
+
const expected = options.expected ?? "ok";
|
|
3624
|
+
const allowIncomplete = options.allowIncomplete === true;
|
|
3756
3625
|
return {
|
|
3757
|
-
|
|
3758
|
-
|
|
3759
|
-
|
|
3760
|
-
|
|
3626
|
+
id: "run.status",
|
|
3627
|
+
category: "run",
|
|
3628
|
+
defaultSeverity: "error",
|
|
3629
|
+
evaluate(context) {
|
|
3630
|
+
const findings = [];
|
|
3631
|
+
const actual = context.selectedRun?.status ?? "unknown";
|
|
3632
|
+
if (actual !== expected) {
|
|
3633
|
+
findings.push(
|
|
3634
|
+
failFinding(
|
|
3635
|
+
"run.status",
|
|
3636
|
+
`Run status ${actual} did not match expected ${expected}.`,
|
|
3637
|
+
runEvidence(context.selectedRun),
|
|
3638
|
+
expected,
|
|
3639
|
+
actual
|
|
3640
|
+
)
|
|
3641
|
+
);
|
|
3642
|
+
}
|
|
3643
|
+
if (!allowIncomplete) {
|
|
3644
|
+
const running = semanticEvents(context).filter((event) => event.status === "running");
|
|
3645
|
+
if (running.length > 0) {
|
|
3646
|
+
findings.push(
|
|
3647
|
+
failFinding(
|
|
3648
|
+
"run.status",
|
|
3649
|
+
"Run contains incomplete running events.",
|
|
3650
|
+
running.map((event) => eventEvidence(event)),
|
|
3651
|
+
"no running events",
|
|
3652
|
+
running.length
|
|
3653
|
+
)
|
|
3654
|
+
);
|
|
3655
|
+
}
|
|
3656
|
+
}
|
|
3657
|
+
return findings;
|
|
3658
|
+
}
|
|
3761
3659
|
};
|
|
3762
3660
|
}
|
|
3763
|
-
function
|
|
3661
|
+
function createSafetyRedactionRule(options = {}) {
|
|
3662
|
+
const sensitiveKeys = options.sensitiveKeys ?? DEFAULT_SENSITIVE_KEYS;
|
|
3663
|
+
const markers = options.redactedMarkers ?? ["[REDACTED]", "[REDACTED:"];
|
|
3764
3664
|
return {
|
|
3765
|
-
|
|
3766
|
-
|
|
3767
|
-
|
|
3768
|
-
|
|
3769
|
-
|
|
3665
|
+
id: "safety.redaction",
|
|
3666
|
+
category: "safety",
|
|
3667
|
+
defaultSeverity: "error",
|
|
3668
|
+
evaluate(context) {
|
|
3669
|
+
const findings = [];
|
|
3670
|
+
for (const event of context.events) {
|
|
3671
|
+
for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
|
|
3672
|
+
if (!isSensitiveKey(entry.key ?? lastPathSegment(entry.path), sensitiveKeys)) continue;
|
|
3673
|
+
if (typeof entry.value === "string" && hasRedactionMarker(entry.value, markers)) continue;
|
|
3674
|
+
findings.push(
|
|
3675
|
+
failFinding(
|
|
3676
|
+
"safety.redaction",
|
|
3677
|
+
`Sensitive-looking field at ${entry.path} is not redacted.`,
|
|
3678
|
+
[eventEvidence(event, entry.path)],
|
|
3679
|
+
"redaction marker",
|
|
3680
|
+
{ path: entry.path, valueType: valueType(entry.value) },
|
|
3681
|
+
{
|
|
3682
|
+
category: "credential",
|
|
3683
|
+
confidence: "high",
|
|
3684
|
+
detector: "safety.redaction",
|
|
3685
|
+
action: "redact"
|
|
3686
|
+
}
|
|
3687
|
+
)
|
|
3688
|
+
);
|
|
3689
|
+
}
|
|
3690
|
+
}
|
|
3691
|
+
return limitFindings(findings, options.maxFindings);
|
|
3692
|
+
}
|
|
3770
3693
|
};
|
|
3771
3694
|
}
|
|
3772
|
-
function
|
|
3695
|
+
function createSafetyRawContentRule(options = {}) {
|
|
3696
|
+
const forbiddenKeys = options.forbiddenKeys ?? DEFAULT_RAW_CONTENT_KEYS;
|
|
3697
|
+
const safePathPrefixes = options.safePathPrefixes ?? DEFAULT_SAFE_RAW_CONTENT_PATH_PREFIXES;
|
|
3773
3698
|
return {
|
|
3774
|
-
|
|
3775
|
-
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
|
|
3780
|
-
|
|
3781
|
-
|
|
3782
|
-
|
|
3783
|
-
|
|
3784
|
-
|
|
3785
|
-
|
|
3699
|
+
id: "safety.rawPrompt",
|
|
3700
|
+
category: "safety",
|
|
3701
|
+
defaultSeverity: "error",
|
|
3702
|
+
evaluate(context) {
|
|
3703
|
+
const findings = [];
|
|
3704
|
+
for (const event of context.events) {
|
|
3705
|
+
for (const entry of eventValueEntries(event, { includeSummaries: options.includeSummaries })) {
|
|
3706
|
+
const key = entry.key ?? lastPathSegment(entry.path);
|
|
3707
|
+
if (!isRawContentPath(entry.path, key, forbiddenKeys, safePathPrefixes)) continue;
|
|
3708
|
+
findings.push(
|
|
3709
|
+
failFinding(
|
|
3710
|
+
"safety.rawPrompt",
|
|
3711
|
+
`Raw content-like field ${entry.path} is present.`,
|
|
3712
|
+
[eventEvidence(event, entry.path)],
|
|
3713
|
+
"metadata-only trace fields",
|
|
3714
|
+
{ path: entry.path, valueType: valueType(entry.value) },
|
|
3715
|
+
{
|
|
3716
|
+
category: "raw-content",
|
|
3717
|
+
confidence: "high",
|
|
3718
|
+
detector: "safety.rawPrompt",
|
|
3719
|
+
action: "redact-or-omit"
|
|
3720
|
+
}
|
|
3721
|
+
)
|
|
3722
|
+
);
|
|
3723
|
+
}
|
|
3724
|
+
}
|
|
3725
|
+
return limitFindings(findings, options.maxFindings);
|
|
3726
|
+
}
|
|
3786
3727
|
};
|
|
3787
3728
|
}
|
|
3788
|
-
function
|
|
3789
|
-
|
|
3729
|
+
function createSafetySecretPatternRule(options = {}) {
|
|
3730
|
+
const patterns = options.patterns ?? DEFAULT_SECRET_PATTERNS;
|
|
3731
|
+
const maxStringLength = options.maxStringLength ?? 4096;
|
|
3732
|
+
return {
|
|
3733
|
+
id: "safety.secretPattern",
|
|
3734
|
+
category: "safety",
|
|
3735
|
+
defaultSeverity: "error",
|
|
3736
|
+
evaluate(context) {
|
|
3737
|
+
const findings = [];
|
|
3738
|
+
for (const event of context.events) {
|
|
3739
|
+
for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
|
|
3740
|
+
if (typeof entry.value !== "string") continue;
|
|
3741
|
+
const sample = entry.value.slice(0, maxStringLength);
|
|
3742
|
+
for (const pattern of patterns) {
|
|
3743
|
+
pattern.pattern.lastIndex = 0;
|
|
3744
|
+
if (!pattern.pattern.test(sample)) continue;
|
|
3745
|
+
pattern.pattern.lastIndex = 0;
|
|
3746
|
+
findings.push(
|
|
3747
|
+
failFinding(
|
|
3748
|
+
"safety.secretPattern",
|
|
3749
|
+
`Secret-like pattern ${pattern.id} matched at ${entry.path}.`,
|
|
3750
|
+
[eventEvidence(event, entry.path)],
|
|
3751
|
+
"no secret-like strings",
|
|
3752
|
+
{ pattern: pattern.id, path: entry.path },
|
|
3753
|
+
{
|
|
3754
|
+
category: "credential",
|
|
3755
|
+
confidence: "high",
|
|
3756
|
+
detector: pattern.id,
|
|
3757
|
+
action: "redact"
|
|
3758
|
+
}
|
|
3759
|
+
)
|
|
3760
|
+
);
|
|
3761
|
+
break;
|
|
3762
|
+
}
|
|
3763
|
+
}
|
|
3764
|
+
}
|
|
3765
|
+
return limitFindings(findings, options.maxFindings);
|
|
3766
|
+
}
|
|
3767
|
+
};
|
|
3790
3768
|
}
|
|
3791
|
-
function
|
|
3792
|
-
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
|
|
3797
|
-
|
|
3798
|
-
|
|
3799
|
-
|
|
3800
|
-
|
|
3801
|
-
|
|
3802
|
-
|
|
3803
|
-
|
|
3804
|
-
|
|
3769
|
+
function createSafetyOversizedAttributeRule(options) {
|
|
3770
|
+
return {
|
|
3771
|
+
id: "safety.oversizedAttribute",
|
|
3772
|
+
category: "safety",
|
|
3773
|
+
defaultSeverity: "error",
|
|
3774
|
+
evaluate(context) {
|
|
3775
|
+
const findings = [];
|
|
3776
|
+
for (const event of context.events) {
|
|
3777
|
+
for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
|
|
3778
|
+
if (typeof entry.value === "string" && options.maxStringLength !== void 0 && entry.value.length > options.maxStringLength) {
|
|
3779
|
+
findings.push(
|
|
3780
|
+
failFinding(
|
|
3781
|
+
"safety.oversizedAttribute",
|
|
3782
|
+
`String at ${entry.path} exceeds ${options.maxStringLength} characters.`,
|
|
3783
|
+
[eventEvidence(event, entry.path)],
|
|
3784
|
+
{ maxStringLength: options.maxStringLength },
|
|
3785
|
+
{ path: entry.path, length: entry.value.length },
|
|
3786
|
+
{
|
|
3787
|
+
category: "size",
|
|
3788
|
+
confidence: "high",
|
|
3789
|
+
detector: "safety.oversizedAttribute",
|
|
3790
|
+
action: "truncate-or-omit"
|
|
3791
|
+
}
|
|
3792
|
+
)
|
|
3793
|
+
);
|
|
3794
|
+
}
|
|
3795
|
+
if (Array.isArray(entry.value) && options.maxArrayLength !== void 0 && entry.value.length > options.maxArrayLength) {
|
|
3796
|
+
findings.push(
|
|
3797
|
+
failFinding(
|
|
3798
|
+
"safety.oversizedAttribute",
|
|
3799
|
+
`Array at ${entry.path} exceeds ${options.maxArrayLength} items.`,
|
|
3800
|
+
[eventEvidence(event, entry.path)],
|
|
3801
|
+
{ maxArrayLength: options.maxArrayLength },
|
|
3802
|
+
{ path: entry.path, length: entry.value.length },
|
|
3803
|
+
{
|
|
3804
|
+
category: "size",
|
|
3805
|
+
confidence: "high",
|
|
3806
|
+
detector: "safety.oversizedAttribute",
|
|
3807
|
+
action: "truncate-or-omit"
|
|
3808
|
+
}
|
|
3809
|
+
)
|
|
3810
|
+
);
|
|
3811
|
+
}
|
|
3812
|
+
if (isRecord8(entry.value) && options.maxObjectKeys !== void 0 && Object.keys(entry.value).length > options.maxObjectKeys) {
|
|
3813
|
+
findings.push(
|
|
3814
|
+
failFinding(
|
|
3815
|
+
"safety.oversizedAttribute",
|
|
3816
|
+
`Object at ${entry.path} exceeds ${options.maxObjectKeys} keys.`,
|
|
3817
|
+
[eventEvidence(event, entry.path)],
|
|
3818
|
+
{ maxObjectKeys: options.maxObjectKeys },
|
|
3819
|
+
{ path: entry.path, keys: Object.keys(entry.value).length },
|
|
3820
|
+
{
|
|
3821
|
+
category: "size",
|
|
3822
|
+
confidence: "high",
|
|
3823
|
+
detector: "safety.oversizedAttribute",
|
|
3824
|
+
action: "truncate-or-omit"
|
|
3825
|
+
}
|
|
3826
|
+
)
|
|
3827
|
+
);
|
|
3828
|
+
}
|
|
3829
|
+
if (options.maxSerializedBytes !== void 0) {
|
|
3830
|
+
const bytes = serializedByteLength(entry.value);
|
|
3831
|
+
if (bytes !== void 0 && bytes > options.maxSerializedBytes) {
|
|
3832
|
+
findings.push(
|
|
3833
|
+
failFinding(
|
|
3834
|
+
"safety.oversizedAttribute",
|
|
3835
|
+
`Value at ${entry.path} exceeds ${options.maxSerializedBytes} serialized bytes.`,
|
|
3836
|
+
[eventEvidence(event, entry.path)],
|
|
3837
|
+
{ maxSerializedBytes: options.maxSerializedBytes },
|
|
3838
|
+
{ path: entry.path, bytes },
|
|
3839
|
+
{
|
|
3840
|
+
category: "size",
|
|
3841
|
+
confidence: "high",
|
|
3842
|
+
detector: "safety.oversizedAttribute",
|
|
3843
|
+
action: "truncate-or-omit"
|
|
3844
|
+
}
|
|
3845
|
+
)
|
|
3846
|
+
);
|
|
3847
|
+
}
|
|
3848
|
+
}
|
|
3849
|
+
}
|
|
3850
|
+
}
|
|
3851
|
+
return limitFindings(findings, options.maxFindings);
|
|
3805
3852
|
}
|
|
3806
|
-
}
|
|
3807
|
-
const projection = projectLogicalEvents(scopedEvents);
|
|
3808
|
-
return {
|
|
3809
|
-
format: input.read.format,
|
|
3810
|
-
runs: Object.freeze([...input.read.runs]),
|
|
3811
|
-
events: Object.freeze([...scopedEvents]),
|
|
3812
|
-
logicalEvents: projection.logicalEvents,
|
|
3813
|
-
logicalProjectionDiagnostics: projection.diagnostics,
|
|
3814
|
-
readerWarnings: Object.freeze([...input.read.warnings]),
|
|
3815
|
-
unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
|
|
3816
|
-
sourceFiles: Object.freeze([...input.read.sourceFiles]),
|
|
3817
|
-
nodesByEventId,
|
|
3818
|
-
childrenByParentId,
|
|
3819
|
-
rootNodes: Object.freeze(scopedRuns.flatMap((run) => run.children))
|
|
3820
3853
|
};
|
|
3821
3854
|
}
|
|
3822
|
-
function
|
|
3823
|
-
|
|
3824
|
-
|
|
3825
|
-
|
|
3826
|
-
diagnostics: [
|
|
3827
|
-
diagnostic(
|
|
3828
|
-
"AI_CHECK_INVALID_ARGUMENTS",
|
|
3829
|
-
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
3830
|
-
)
|
|
3831
|
-
]
|
|
3832
|
-
};
|
|
3833
|
-
}
|
|
3834
|
-
return { run: input.selectedRun, diagnostics: [] };
|
|
3835
|
-
}
|
|
3836
|
-
if (runId) {
|
|
3837
|
-
const run = input.read.runs.find((candidate) => candidate.runId === runId);
|
|
3838
|
-
if (!run) {
|
|
3839
|
-
return {
|
|
3840
|
-
diagnostics: [
|
|
3841
|
-
diagnostic("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
3842
|
-
]
|
|
3843
|
-
};
|
|
3844
|
-
}
|
|
3845
|
-
return { run, diagnostics: [] };
|
|
3855
|
+
function runTraceChecks(input, options = {}) {
|
|
3856
|
+
const selected = resolveSelectedRun(input, options.runId);
|
|
3857
|
+
if (selected.diagnostics.length > 0) {
|
|
3858
|
+
return errorResult(input, selected.diagnostics, selected.run);
|
|
3846
3859
|
}
|
|
3847
|
-
|
|
3848
|
-
|
|
3860
|
+
const rules = selectRules(options.rules ?? [], options.select);
|
|
3861
|
+
if (rules.diagnostics.length > 0) {
|
|
3862
|
+
return errorResult(input, rules.diagnostics, selected.run);
|
|
3849
3863
|
}
|
|
3850
|
-
if (
|
|
3851
|
-
return
|
|
3852
|
-
|
|
3864
|
+
if (rules.rules.length === 0) {
|
|
3865
|
+
return errorResult(
|
|
3866
|
+
input,
|
|
3867
|
+
[
|
|
3853
3868
|
diagnostic(
|
|
3854
|
-
"
|
|
3855
|
-
|
|
3856
|
-
"AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
3857
|
-
"No runs are available for checks."
|
|
3858
|
-
)
|
|
3869
|
+
"AI_CHECK_NO_RULES_EVALUATED",
|
|
3870
|
+
"No trace check rules were evaluated. Configure at least one rule, contract, or CLI check option."
|
|
3859
3871
|
)
|
|
3860
|
-
]
|
|
3861
|
-
|
|
3872
|
+
],
|
|
3873
|
+
selected.run
|
|
3874
|
+
);
|
|
3862
3875
|
}
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
)
|
|
3869
|
-
]
|
|
3876
|
+
const facts = buildFacts(input, selected.run);
|
|
3877
|
+
const context = {
|
|
3878
|
+
...facts,
|
|
3879
|
+
...selected.run ? { selectedRun: selected.run } : {},
|
|
3880
|
+
...input.sourceLabel ? { sourceLabel: input.sourceLabel } : {}
|
|
3870
3881
|
};
|
|
3871
|
-
}
|
|
3872
|
-
function selectRules(rules, selectedIds) {
|
|
3873
3882
|
const diagnostics = [];
|
|
3874
|
-
const
|
|
3875
|
-
|
|
3876
|
-
|
|
3883
|
+
const findings = [];
|
|
3884
|
+
const ruleExecutions = [];
|
|
3885
|
+
for (const rule of rules.rules) {
|
|
3886
|
+
try {
|
|
3887
|
+
const ruleFindings = rule.evaluate(context).map((finding) => normalizeFinding(rule, finding));
|
|
3888
|
+
findings.push(...ruleFindings);
|
|
3889
|
+
ruleExecutions.push({
|
|
3890
|
+
ruleId: rule.id,
|
|
3891
|
+
category: rule.category,
|
|
3892
|
+
status: classifyRuleExecution(ruleFindings, false),
|
|
3893
|
+
findingCount: ruleFindings.length,
|
|
3894
|
+
...selected.run ? { runId: selected.run.runId } : {}
|
|
3895
|
+
});
|
|
3896
|
+
} catch (error) {
|
|
3897
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3877
3898
|
diagnostics.push(
|
|
3878
|
-
diagnostic("
|
|
3899
|
+
diagnostic("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
|
|
3879
3900
|
);
|
|
3880
|
-
|
|
3901
|
+
ruleExecutions.push({
|
|
3902
|
+
ruleId: rule.id,
|
|
3903
|
+
category: rule.category,
|
|
3904
|
+
status: "error",
|
|
3905
|
+
findingCount: 0,
|
|
3906
|
+
...selected.run ? { runId: selected.run.runId } : {}
|
|
3907
|
+
});
|
|
3881
3908
|
}
|
|
3882
|
-
byId.set(rule.id, rule);
|
|
3883
3909
|
}
|
|
3884
|
-
if (
|
|
3885
|
-
|
|
3886
|
-
for (const id of selected) {
|
|
3887
|
-
if (!byId.has(id)) {
|
|
3888
|
-
diagnostics.push(
|
|
3889
|
-
diagnostic("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
3890
|
-
);
|
|
3891
|
-
}
|
|
3892
|
-
}
|
|
3893
|
-
return {
|
|
3894
|
-
rules: [...byId.values()].filter((rule) => selected.has(rule.id)).sort(compareRules),
|
|
3895
|
-
diagnostics
|
|
3896
|
-
};
|
|
3910
|
+
if (diagnostics.length > 0) {
|
|
3911
|
+
return errorResult(input, diagnostics, selected.run, ruleExecutions);
|
|
3897
3912
|
}
|
|
3898
|
-
|
|
3899
|
-
|
|
3900
|
-
|
|
3901
|
-
|
|
3902
|
-
}
|
|
3903
|
-
function eventTimestamp(finding, eventById) {
|
|
3904
|
-
const eventId = finding.evidence[0]?.eventId;
|
|
3905
|
-
return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
|
|
3906
|
-
}
|
|
3907
|
-
function compareFindings(eventById) {
|
|
3908
|
-
return (a, b) => {
|
|
3909
|
-
if (SEVERITY_RANK[a.severity] !== SEVERITY_RANK[b.severity]) {
|
|
3910
|
-
return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity];
|
|
3911
|
-
}
|
|
3912
|
-
const byRule = a.ruleId.localeCompare(b.ruleId);
|
|
3913
|
-
if (byRule !== 0) return byRule;
|
|
3914
|
-
if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
|
|
3915
|
-
return STATUS_RANK[a.status] - STATUS_RANK[b.status];
|
|
3916
|
-
}
|
|
3917
|
-
const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
|
|
3918
|
-
if (byRun !== 0) return byRun;
|
|
3919
|
-
const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
|
|
3920
|
-
if (byTime !== 0) return byTime;
|
|
3921
|
-
const byEvent = compareStrings(a.evidence[0]?.eventId, b.evidence[0]?.eventId);
|
|
3922
|
-
if (byEvent !== 0) return byEvent;
|
|
3923
|
-
return compareStrings(a.evidence[0]?.path, b.evidence[0]?.path);
|
|
3924
|
-
};
|
|
3925
|
-
}
|
|
3926
|
-
function normalizeFinding(rule, finding) {
|
|
3927
|
-
return {
|
|
3928
|
-
ruleId: finding.ruleId || rule.id,
|
|
3929
|
-
severity: finding.severity ?? rule.defaultSeverity,
|
|
3930
|
-
status: finding.status,
|
|
3931
|
-
message: finding.message,
|
|
3932
|
-
...finding.expected !== void 0 ? { expected: finding.expected } : {},
|
|
3933
|
-
...finding.actual !== void 0 ? { actual: finding.actual } : {},
|
|
3934
|
-
evidence: [...finding.evidence ?? []],
|
|
3935
|
-
...finding.category !== void 0 ? { category: finding.category } : {},
|
|
3936
|
-
...finding.confidence !== void 0 ? { confidence: finding.confidence } : {},
|
|
3937
|
-
...finding.detector !== void 0 ? { detector: finding.detector } : {},
|
|
3938
|
-
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
3939
|
-
};
|
|
3940
|
-
}
|
|
3941
|
-
function summarize(findings, diagnostics, rulesEvaluated) {
|
|
3913
|
+
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
3914
|
+
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
3915
|
+
const summary = summarize(sortedFindings, diagnostics, ruleExecutions.length);
|
|
3916
|
+
const status = summary.failed > 0 ? "fail" : "pass";
|
|
3942
3917
|
return {
|
|
3943
|
-
|
|
3944
|
-
|
|
3945
|
-
|
|
3946
|
-
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
|
|
3950
|
-
|
|
3951
|
-
rulesEvaluated
|
|
3918
|
+
ok: status === "pass",
|
|
3919
|
+
status,
|
|
3920
|
+
format: input.read.format,
|
|
3921
|
+
...selected.run ? { runId: selected.run.runId } : {},
|
|
3922
|
+
summary,
|
|
3923
|
+
findings: sortedFindings,
|
|
3924
|
+
diagnostics,
|
|
3925
|
+
ruleExecutions
|
|
3952
3926
|
};
|
|
3953
3927
|
}
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
3928
|
+
|
|
3929
|
+
// packages/core/src/checks/contract.ts
|
|
3930
|
+
new Set(OBSERVED_OUTCOME_METHODS);
|
|
3931
|
+
|
|
3932
|
+
// packages/core/src/exporters/helpers.ts
|
|
3933
|
+
var REDACT_SUBSTRINGS = [
|
|
3934
|
+
"authorization",
|
|
3935
|
+
"cookie",
|
|
3936
|
+
"token",
|
|
3937
|
+
"apikey",
|
|
3938
|
+
"password",
|
|
3939
|
+
"secret",
|
|
3940
|
+
"email"
|
|
3941
|
+
];
|
|
3942
|
+
function shouldRedactKey(key) {
|
|
3943
|
+
const k = key.toLowerCase();
|
|
3944
|
+
for (const s of REDACT_SUBSTRINGS) {
|
|
3945
|
+
if (k.includes(s)) return true;
|
|
3962
3946
|
}
|
|
3963
|
-
return
|
|
3964
|
-
}
|
|
3965
|
-
function eventEvidence(event, path16) {
|
|
3966
|
-
return {
|
|
3967
|
-
runId: event.runId,
|
|
3968
|
-
eventId: event.eventId,
|
|
3969
|
-
parentId: event.parentId,
|
|
3970
|
-
traceId: event.trace?.traceId,
|
|
3971
|
-
spanId: event.trace?.spanId,
|
|
3972
|
-
kind: event.kind,
|
|
3973
|
-
name: event.name,
|
|
3974
|
-
status: event.status,
|
|
3975
|
-
...path16 ? { path: path16 } : {}
|
|
3976
|
-
};
|
|
3977
|
-
}
|
|
3978
|
-
function runEvidence(run) {
|
|
3979
|
-
return run ? [{ runId: run.runId, name: run.name, status: run.status }] : [];
|
|
3980
|
-
}
|
|
3981
|
-
function failFinding(ruleId, message, evidence, expected, actual, meta) {
|
|
3982
|
-
return {
|
|
3983
|
-
ruleId,
|
|
3984
|
-
severity: meta?.severity ?? "error",
|
|
3985
|
-
status: meta?.status ?? "fail",
|
|
3986
|
-
message,
|
|
3987
|
-
...expected !== void 0 ? { expected } : {},
|
|
3988
|
-
...actual !== void 0 ? { actual } : {},
|
|
3989
|
-
evidence: [...evidence],
|
|
3990
|
-
...meta?.category !== void 0 ? { category: meta.category } : {},
|
|
3991
|
-
...meta?.confidence !== void 0 ? { confidence: meta.confidence } : {},
|
|
3992
|
-
...meta?.detector !== void 0 ? { detector: meta.detector } : {},
|
|
3993
|
-
...meta?.action !== void 0 ? { action: meta.action } : {}
|
|
3994
|
-
};
|
|
3995
|
-
}
|
|
3996
|
-
function semanticEvents(context) {
|
|
3997
|
-
return context.logicalEvents ?? context.events;
|
|
3998
|
-
}
|
|
3999
|
-
function isRecord8(value) {
|
|
4000
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3947
|
+
return false;
|
|
4001
3948
|
}
|
|
4002
|
-
function
|
|
4003
|
-
|
|
3949
|
+
function safeString(value, maxLength) {
|
|
3950
|
+
if (value === null || value === void 0) return "";
|
|
3951
|
+
let s;
|
|
3952
|
+
if (typeof value === "string") s = value;
|
|
3953
|
+
else if (typeof value === "number" || typeof value === "boolean") s = String(value);
|
|
3954
|
+
else s = stableJson(value, false);
|
|
3955
|
+
if (maxLength !== void 0 && maxLength >= 0 && s.length > maxLength) {
|
|
3956
|
+
return `${s.slice(0, maxLength)}\u2026`;
|
|
3957
|
+
}
|
|
3958
|
+
return s;
|
|
4004
3959
|
}
|
|
4005
|
-
function
|
|
4006
|
-
|
|
4007
|
-
return parts[parts.length - 1] ?? path16;
|
|
3960
|
+
function escapeMarkdown(value) {
|
|
3961
|
+
return value.replace(/\|/g, "\\|").replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(/\n/g, " ");
|
|
4008
3962
|
}
|
|
4009
|
-
function
|
|
4010
|
-
|
|
4011
|
-
if (value === null) return "null";
|
|
4012
|
-
return typeof value;
|
|
3963
|
+
function escapeHtml(value) {
|
|
3964
|
+
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
4013
3965
|
}
|
|
4014
|
-
function
|
|
4015
|
-
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
|
|
3966
|
+
function sortKeysDeep(input) {
|
|
3967
|
+
if (input === null || typeof input !== "object") return input;
|
|
3968
|
+
if (Array.isArray(input)) return input.map(sortKeysDeep);
|
|
3969
|
+
const o = input;
|
|
3970
|
+
const out = {};
|
|
3971
|
+
for (const k of Object.keys(o).sort()) {
|
|
3972
|
+
out[k] = sortKeysDeep(o[k]);
|
|
4019
3973
|
}
|
|
3974
|
+
return out;
|
|
4020
3975
|
}
|
|
4021
|
-
function
|
|
4022
|
-
|
|
4023
|
-
|
|
4024
|
-
|
|
4025
|
-
|
|
4026
|
-
|
|
3976
|
+
function stableJson(value, pretty) {
|
|
3977
|
+
const sorted = sortKeysDeep(value);
|
|
3978
|
+
return pretty === true ? JSON.stringify(sorted, null, 2) : JSON.stringify(sorted);
|
|
3979
|
+
}
|
|
3980
|
+
function compactAttributes(attrs2, options) {
|
|
3981
|
+
if (attrs2 === void 0) return {};
|
|
3982
|
+
const maxLen = options?.maxLength ?? 500;
|
|
3983
|
+
const redacted = options?.redacted ?? true;
|
|
3984
|
+
const out = {};
|
|
3985
|
+
for (const key of Object.keys(attrs2).sort()) {
|
|
3986
|
+
if (redacted && shouldRedactKey(key)) {
|
|
3987
|
+
out[key] = "[REDACTED]";
|
|
3988
|
+
continue;
|
|
4027
3989
|
}
|
|
4028
|
-
|
|
4029
|
-
|
|
4030
|
-
if (!isRecord8(value)) return;
|
|
4031
|
-
for (const nestedKey of Object.keys(value).sort((a, b) => a.localeCompare(b))) {
|
|
4032
|
-
pushValueEntries(
|
|
4033
|
-
entries,
|
|
4034
|
-
event,
|
|
4035
|
-
value[nestedKey],
|
|
4036
|
-
`${path16}.${nestedKey}`,
|
|
4037
|
-
nestedKey,
|
|
4038
|
-
depth + 1
|
|
4039
|
-
);
|
|
3990
|
+
const v = attrs2[key];
|
|
3991
|
+
out[key] = compactValue(v, maxLen, redacted);
|
|
4040
3992
|
}
|
|
3993
|
+
return out;
|
|
4041
3994
|
}
|
|
4042
|
-
function
|
|
4043
|
-
|
|
4044
|
-
|
|
4045
|
-
pushValueEntries(entries, event, event.attributes, "attributes", "attributes");
|
|
3995
|
+
function compactValue(value, maxLen, redacted) {
|
|
3996
|
+
if (value === null || typeof value !== "object") {
|
|
3997
|
+
return typeof value === "string" ? safeString(value, maxLen) : value;
|
|
4046
3998
|
}
|
|
4047
|
-
if (
|
|
4048
|
-
|
|
4049
|
-
|
|
4050
|
-
|
|
4051
|
-
if (event.outputSummary !== void 0) {
|
|
4052
|
-
pushValueEntries(entries, event, event.outputSummary, "outputSummary", "outputSummary");
|
|
4053
|
-
}
|
|
3999
|
+
if (Array.isArray(value)) {
|
|
4000
|
+
const arr = value.slice(0, 20).map((x) => compactValue(x, maxLen, redacted));
|
|
4001
|
+
if (value.length > 20) arr.push(`\u2026(+${value.length - 20} more)`);
|
|
4002
|
+
return arr;
|
|
4054
4003
|
}
|
|
4055
|
-
|
|
4056
|
-
|
|
4004
|
+
const o = value;
|
|
4005
|
+
const inner = {};
|
|
4006
|
+
for (const k of Object.keys(o)) {
|
|
4007
|
+
if (redacted && shouldRedactKey(k)) inner[k] = "[REDACTED]";
|
|
4008
|
+
else inner[k] = compactValue(o[k], maxLen, redacted);
|
|
4057
4009
|
}
|
|
4058
|
-
return
|
|
4059
|
-
}
|
|
4060
|
-
function limitFindings(findings, maxFindings) {
|
|
4061
|
-
if (maxFindings === void 0 || findings.length <= maxFindings) return findings;
|
|
4062
|
-
return findings.slice(0, Math.max(0, maxFindings));
|
|
4063
|
-
}
|
|
4064
|
-
function hasRedactionMarker(value, markers) {
|
|
4065
|
-
return markers.some((marker) => value.includes(marker)) || /^\[HASH:[A-Za-z0-9_-]+\]$/.test(value);
|
|
4066
|
-
}
|
|
4067
|
-
function isSensitiveKey(key, sensitiveKeys) {
|
|
4068
|
-
return isCredentialSensitiveKey(key, sensitiveKeys);
|
|
4010
|
+
return inner;
|
|
4069
4011
|
}
|
|
4070
|
-
function
|
|
4071
|
-
|
|
4072
|
-
|
|
4073
|
-
|
|
4012
|
+
function flattenTree(tree) {
|
|
4013
|
+
const out = [];
|
|
4014
|
+
function walk(nodes) {
|
|
4015
|
+
for (const n of nodes) {
|
|
4016
|
+
out.push(n);
|
|
4017
|
+
if (n.children.length > 0) walk(n.children);
|
|
4018
|
+
}
|
|
4019
|
+
}
|
|
4020
|
+
walk(tree.children);
|
|
4021
|
+
return out;
|
|
4074
4022
|
}
|
|
4075
|
-
|
|
4076
|
-
|
|
4077
|
-
|
|
4078
|
-
|
|
4079
|
-
if (
|
|
4080
|
-
|
|
4081
|
-
|
|
4082
|
-
return safePathPrefixes.some((prefix) => parentNorm === normalizedKey(prefix));
|
|
4023
|
+
|
|
4024
|
+
// packages/core/src/diff/comparable.ts
|
|
4025
|
+
function extractOutputPreview(meta) {
|
|
4026
|
+
if (meta === void 0) return void 0;
|
|
4027
|
+
if ("outputPreview" in meta) return meta.outputPreview;
|
|
4028
|
+
if ("resultPreview" in meta) return meta.resultPreview;
|
|
4029
|
+
return void 0;
|
|
4083
4030
|
}
|
|
4084
|
-
function
|
|
4085
|
-
if (
|
|
4086
|
-
return
|
|
4031
|
+
function mapStepStatus(s) {
|
|
4032
|
+
if (s === void 0) return "running";
|
|
4033
|
+
return s;
|
|
4087
4034
|
}
|
|
4088
|
-
function
|
|
4089
|
-
const
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
|
|
4105
|
-
|
|
4106
|
-
|
|
4107
|
-
|
|
4108
|
-
|
|
4109
|
-
|
|
4110
|
-
|
|
4111
|
-
|
|
4112
|
-
|
|
4113
|
-
|
|
4114
|
-
|
|
4115
|
-
|
|
4116
|
-
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
4035
|
+
function manualTraceEventsToComparableRun(events) {
|
|
4036
|
+
const started = events.find((e) => e.event === "run_started");
|
|
4037
|
+
if (!started || started.event !== "run_started") {
|
|
4038
|
+
throw new Error("Invalid trace: missing run_started");
|
|
4039
|
+
}
|
|
4040
|
+
const rs = started;
|
|
4041
|
+
const runId = rs.runId;
|
|
4042
|
+
const completedAll = events.filter((e) => e.event === "run_completed");
|
|
4043
|
+
const lastCompleted = completedAll[completedAll.length - 1];
|
|
4044
|
+
let runStatus;
|
|
4045
|
+
if (lastCompleted === void 0) runStatus = "running";
|
|
4046
|
+
else runStatus = lastCompleted.status;
|
|
4047
|
+
const durationMs = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
4048
|
+
const steps = /* @__PURE__ */ new Map();
|
|
4049
|
+
let order = 0;
|
|
4050
|
+
for (const e of events) {
|
|
4051
|
+
if (e.event !== "step_started") continue;
|
|
4052
|
+
const s = e;
|
|
4053
|
+
const meta = s.metadata ? { ...s.metadata } : void 0;
|
|
4054
|
+
steps.set(s.stepId, {
|
|
4055
|
+
id: s.stepId,
|
|
4056
|
+
parentId: s.parentId,
|
|
4057
|
+
name: s.name,
|
|
4058
|
+
type: s.type,
|
|
4059
|
+
order: order++,
|
|
4060
|
+
timestamp: s.timestamp,
|
|
4061
|
+
metadata: meta
|
|
4062
|
+
});
|
|
4063
|
+
}
|
|
4064
|
+
for (const e of events) {
|
|
4065
|
+
if (e.event !== "step_completed") continue;
|
|
4066
|
+
const acc = steps.get(e.stepId);
|
|
4067
|
+
if (!acc) continue;
|
|
4068
|
+
acc.status = e.status;
|
|
4069
|
+
acc.durationMs = e.durationMs;
|
|
4070
|
+
if (e.error?.message) acc.errorMsg = e.error.message;
|
|
4071
|
+
const extra = e;
|
|
4072
|
+
if (extra.metadata !== void 0 && typeof extra.metadata === "object") {
|
|
4073
|
+
acc.metadata = { ...acc.metadata ?? {}, ...extra.metadata };
|
|
4074
|
+
}
|
|
4075
|
+
}
|
|
4076
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
4077
|
+
for (const acc of steps.values()) {
|
|
4078
|
+
let meta = acc.metadata ? { ...acc.metadata } : void 0;
|
|
4079
|
+
if (acc.parentId !== void 0 && !steps.has(acc.parentId)) {
|
|
4080
|
+
meta = { ...meta ?? {}, agent_inspect_diff_parent_missing: true };
|
|
4081
|
+
}
|
|
4082
|
+
const outputPreview = extractOutputPreview(meta);
|
|
4083
|
+
if (meta !== void 0 && ("outputPreview" in meta || "resultPreview" in meta)) {
|
|
4084
|
+
delete meta.outputPreview;
|
|
4085
|
+
delete meta.resultPreview;
|
|
4086
|
+
}
|
|
4087
|
+
const sc = {
|
|
4088
|
+
id: acc.id,
|
|
4089
|
+
name: acc.name,
|
|
4090
|
+
type: acc.type,
|
|
4091
|
+
status: mapStepStatus(acc.status),
|
|
4092
|
+
durationMs: acc.durationMs,
|
|
4093
|
+
error: acc.errorMsg,
|
|
4094
|
+
metadata: meta && Object.keys(meta).length > 0 ? meta : void 0,
|
|
4095
|
+
outputPreview,
|
|
4096
|
+
children: []
|
|
4097
|
+
};
|
|
4098
|
+
nodes.set(acc.id, sc);
|
|
4099
|
+
}
|
|
4100
|
+
const roots = [];
|
|
4101
|
+
const sortByOrder = (a, b) => {
|
|
4102
|
+
const oa = steps.get(a.id)?.order ?? 0;
|
|
4103
|
+
const ob = steps.get(b.id)?.order ?? 0;
|
|
4104
|
+
return oa - ob;
|
|
4105
|
+
};
|
|
4106
|
+
for (const acc of steps.values()) {
|
|
4107
|
+
const node = nodes.get(acc.id);
|
|
4108
|
+
if (acc.parentId !== void 0 && nodes.has(acc.parentId)) {
|
|
4109
|
+
nodes.get(acc.parentId).children.push(node);
|
|
4110
|
+
} else {
|
|
4111
|
+
roots.push(node);
|
|
4124
4112
|
}
|
|
4113
|
+
}
|
|
4114
|
+
roots.sort(sortByOrder);
|
|
4115
|
+
for (const n of nodes.values()) {
|
|
4116
|
+
n.children.sort(sortByOrder);
|
|
4117
|
+
}
|
|
4118
|
+
return {
|
|
4119
|
+
runId,
|
|
4120
|
+
name: rs.name,
|
|
4121
|
+
status: runStatus,
|
|
4122
|
+
durationMs,
|
|
4123
|
+
steps: roots
|
|
4125
4124
|
};
|
|
4126
4125
|
}
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
|
|
4132
|
-
|
|
4133
|
-
|
|
4134
|
-
|
|
4135
|
-
|
|
4136
|
-
|
|
4137
|
-
|
|
4138
|
-
|
|
4139
|
-
|
|
4140
|
-
|
|
4141
|
-
|
|
4142
|
-
|
|
4143
|
-
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
{ path: entry.path, valueType: valueType(entry.value) },
|
|
4147
|
-
{
|
|
4148
|
-
category: "credential",
|
|
4149
|
-
confidence: "high",
|
|
4150
|
-
detector: "safety.redaction",
|
|
4151
|
-
action: "redact"
|
|
4152
|
-
}
|
|
4153
|
-
)
|
|
4154
|
-
);
|
|
4155
|
-
}
|
|
4126
|
+
|
|
4127
|
+
// packages/core/src/diff/engine.ts
|
|
4128
|
+
var DEFAULT_THRESHOLD_MS = 0;
|
|
4129
|
+
function pathSeg(step, index) {
|
|
4130
|
+
return { index, name: step.name, stepId: step.id };
|
|
4131
|
+
}
|
|
4132
|
+
function buildPath(segments) {
|
|
4133
|
+
return { path: [...segments] };
|
|
4134
|
+
}
|
|
4135
|
+
function pairSteps(left, right) {
|
|
4136
|
+
const usedRight = /* @__PURE__ */ new Set();
|
|
4137
|
+
const pairs = [];
|
|
4138
|
+
for (let i = 0; i < left.length; i++) {
|
|
4139
|
+
const L = left[i];
|
|
4140
|
+
let R = right.find((r) => !usedRight.has(r.id) && r.id === L.id);
|
|
4141
|
+
if (R === void 0 && i < right.length && !usedRight.has(right[i].id)) {
|
|
4142
|
+
const cand = right[i];
|
|
4143
|
+
if (cand.name === L.name && (cand.type ?? "") === (L.type ?? "")) {
|
|
4144
|
+
R = cand;
|
|
4156
4145
|
}
|
|
4157
|
-
return limitFindings(findings, options.maxFindings);
|
|
4158
4146
|
}
|
|
4159
|
-
|
|
4147
|
+
if (R === void 0) {
|
|
4148
|
+
R = right.find(
|
|
4149
|
+
(r) => !usedRight.has(r.id) && r.name === L.name && (r.type ?? "") === (L.type ?? "")
|
|
4150
|
+
);
|
|
4151
|
+
}
|
|
4152
|
+
if (R !== void 0) {
|
|
4153
|
+
usedRight.add(R.id);
|
|
4154
|
+
pairs.push([L, R]);
|
|
4155
|
+
} else {
|
|
4156
|
+
pairs.push([L, void 0]);
|
|
4157
|
+
}
|
|
4158
|
+
}
|
|
4159
|
+
for (const R of right) {
|
|
4160
|
+
if (!usedRight.has(R.id)) {
|
|
4161
|
+
pairs.push([void 0, R]);
|
|
4162
|
+
}
|
|
4163
|
+
}
|
|
4164
|
+
return pairs;
|
|
4160
4165
|
}
|
|
4161
|
-
function
|
|
4162
|
-
const
|
|
4163
|
-
|
|
4164
|
-
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
|
|
4168
|
-
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
|
|
4180
|
-
|
|
4181
|
-
|
|
4182
|
-
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
|
|
4186
|
-
|
|
4187
|
-
|
|
4188
|
-
|
|
4189
|
-
|
|
4190
|
-
|
|
4191
|
-
|
|
4166
|
+
function compareLeafSteps(L, R, segments, opts, out) {
|
|
4167
|
+
const path16 = buildPath(segments);
|
|
4168
|
+
if (L.name !== R.name) {
|
|
4169
|
+
out.push({
|
|
4170
|
+
kind: "structure",
|
|
4171
|
+
severity: "warning",
|
|
4172
|
+
message: "Step name differs",
|
|
4173
|
+
path: path16,
|
|
4174
|
+
left: L.name,
|
|
4175
|
+
right: R.name
|
|
4176
|
+
});
|
|
4177
|
+
}
|
|
4178
|
+
if ((L.type ?? "") !== (R.type ?? "")) {
|
|
4179
|
+
out.push({
|
|
4180
|
+
kind: "step-type",
|
|
4181
|
+
severity: "warning",
|
|
4182
|
+
message: "Step type differs",
|
|
4183
|
+
path: path16,
|
|
4184
|
+
left: L.type,
|
|
4185
|
+
right: R.type
|
|
4186
|
+
});
|
|
4187
|
+
}
|
|
4188
|
+
if ((L.status ?? "") !== (R.status ?? "")) {
|
|
4189
|
+
out.push({
|
|
4190
|
+
kind: "step-status",
|
|
4191
|
+
severity: "warning",
|
|
4192
|
+
message: "Step status differs",
|
|
4193
|
+
path: path16,
|
|
4194
|
+
left: L.status,
|
|
4195
|
+
right: R.status
|
|
4196
|
+
});
|
|
4197
|
+
}
|
|
4198
|
+
const le = L.error ?? "";
|
|
4199
|
+
const re = R.error ?? "";
|
|
4200
|
+
if (le !== re) {
|
|
4201
|
+
out.push({
|
|
4202
|
+
kind: "error",
|
|
4203
|
+
severity: "error",
|
|
4204
|
+
message: "Step error message differs",
|
|
4205
|
+
path: path16,
|
|
4206
|
+
left: le || void 0,
|
|
4207
|
+
right: re || void 0
|
|
4208
|
+
});
|
|
4209
|
+
}
|
|
4210
|
+
if (!opts.ignoreDuration) {
|
|
4211
|
+
const ld = L.durationMs;
|
|
4212
|
+
const rd = R.durationMs;
|
|
4213
|
+
const th = opts.durationThresholdMs;
|
|
4214
|
+
let differs = false;
|
|
4215
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
4216
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
4217
|
+
else differs = Math.abs(ld - rd) > th;
|
|
4218
|
+
if (differs) {
|
|
4219
|
+
out.push({
|
|
4220
|
+
kind: "duration",
|
|
4221
|
+
severity: "info",
|
|
4222
|
+
message: "Step duration differs",
|
|
4223
|
+
path: path16,
|
|
4224
|
+
left: ld,
|
|
4225
|
+
right: rd
|
|
4226
|
+
});
|
|
4192
4227
|
}
|
|
4193
|
-
}
|
|
4228
|
+
}
|
|
4229
|
+
const lm = stableJson(L.metadata ?? {});
|
|
4230
|
+
const rm = stableJson(R.metadata ?? {});
|
|
4231
|
+
if (lm !== rm) {
|
|
4232
|
+
out.push({
|
|
4233
|
+
kind: "metadata",
|
|
4234
|
+
severity: "info",
|
|
4235
|
+
message: "Step metadata differs",
|
|
4236
|
+
path: path16,
|
|
4237
|
+
left: L.metadata,
|
|
4238
|
+
right: R.metadata
|
|
4239
|
+
});
|
|
4240
|
+
}
|
|
4241
|
+
const lo = stableJson(L.outputPreview ?? null);
|
|
4242
|
+
const ro = stableJson(R.outputPreview ?? null);
|
|
4243
|
+
if (lo !== ro) {
|
|
4244
|
+
out.push({
|
|
4245
|
+
kind: "output",
|
|
4246
|
+
severity: "info",
|
|
4247
|
+
message: "Output preview differs",
|
|
4248
|
+
path: path16,
|
|
4249
|
+
left: L.outputPreview,
|
|
4250
|
+
right: R.outputPreview
|
|
4251
|
+
});
|
|
4252
|
+
}
|
|
4194
4253
|
}
|
|
4195
|
-
function
|
|
4196
|
-
|
|
4197
|
-
const
|
|
4198
|
-
|
|
4199
|
-
|
|
4200
|
-
|
|
4201
|
-
|
|
4202
|
-
|
|
4203
|
-
|
|
4204
|
-
|
|
4205
|
-
|
|
4206
|
-
|
|
4207
|
-
|
|
4208
|
-
|
|
4209
|
-
|
|
4210
|
-
|
|
4211
|
-
|
|
4212
|
-
|
|
4213
|
-
|
|
4214
|
-
|
|
4215
|
-
|
|
4216
|
-
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
category: "credential",
|
|
4221
|
-
confidence: "high",
|
|
4222
|
-
detector: pattern.id,
|
|
4223
|
-
action: "redact"
|
|
4224
|
-
}
|
|
4225
|
-
)
|
|
4226
|
-
);
|
|
4227
|
-
break;
|
|
4228
|
-
}
|
|
4229
|
-
}
|
|
4230
|
-
}
|
|
4231
|
-
return limitFindings(findings, options.maxFindings);
|
|
4254
|
+
function compareRecursive(L, R, segments, opts, out) {
|
|
4255
|
+
compareLeafSteps(L, R, segments, opts, out);
|
|
4256
|
+
const pairs = pairSteps(L.children, R.children);
|
|
4257
|
+
let ci = 0;
|
|
4258
|
+
for (const [lch, rch] of pairs) {
|
|
4259
|
+
if (lch !== void 0 && rch !== void 0) {
|
|
4260
|
+
compareRecursive(lch, rch, [...segments, pathSeg(lch, ci)], opts, out);
|
|
4261
|
+
} else if (lch !== void 0) {
|
|
4262
|
+
out.push({
|
|
4263
|
+
kind: "step-removed",
|
|
4264
|
+
severity: "warning",
|
|
4265
|
+
message: `Step only in left run: ${lch.name}`,
|
|
4266
|
+
path: buildPath([...segments, pathSeg(lch, ci)]),
|
|
4267
|
+
left: lch.id,
|
|
4268
|
+
right: void 0
|
|
4269
|
+
});
|
|
4270
|
+
} else if (rch !== void 0) {
|
|
4271
|
+
out.push({
|
|
4272
|
+
kind: "step-added",
|
|
4273
|
+
severity: "warning",
|
|
4274
|
+
message: `Step only in right run: ${rch.name}`,
|
|
4275
|
+
path: buildPath([...segments, pathSeg(rch, ci)]),
|
|
4276
|
+
left: void 0,
|
|
4277
|
+
right: rch.id
|
|
4278
|
+
});
|
|
4232
4279
|
}
|
|
4233
|
-
|
|
4280
|
+
ci += 1;
|
|
4281
|
+
}
|
|
4234
4282
|
}
|
|
4235
|
-
function
|
|
4283
|
+
function mergeDiffDefaults(options) {
|
|
4236
4284
|
return {
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
|
|
4240
|
-
|
|
4241
|
-
const findings = [];
|
|
4242
|
-
for (const event of context.events) {
|
|
4243
|
-
for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
|
|
4244
|
-
if (typeof entry.value === "string" && options.maxStringLength !== void 0 && entry.value.length > options.maxStringLength) {
|
|
4245
|
-
findings.push(
|
|
4246
|
-
failFinding(
|
|
4247
|
-
"safety.oversizedAttribute",
|
|
4248
|
-
`String at ${entry.path} exceeds ${options.maxStringLength} characters.`,
|
|
4249
|
-
[eventEvidence(event, entry.path)],
|
|
4250
|
-
{ maxStringLength: options.maxStringLength },
|
|
4251
|
-
{ path: entry.path, length: entry.value.length },
|
|
4252
|
-
{
|
|
4253
|
-
category: "size",
|
|
4254
|
-
confidence: "high",
|
|
4255
|
-
detector: "safety.oversizedAttribute",
|
|
4256
|
-
action: "truncate-or-omit"
|
|
4257
|
-
}
|
|
4258
|
-
)
|
|
4259
|
-
);
|
|
4260
|
-
}
|
|
4261
|
-
if (Array.isArray(entry.value) && options.maxArrayLength !== void 0 && entry.value.length > options.maxArrayLength) {
|
|
4262
|
-
findings.push(
|
|
4263
|
-
failFinding(
|
|
4264
|
-
"safety.oversizedAttribute",
|
|
4265
|
-
`Array at ${entry.path} exceeds ${options.maxArrayLength} items.`,
|
|
4266
|
-
[eventEvidence(event, entry.path)],
|
|
4267
|
-
{ maxArrayLength: options.maxArrayLength },
|
|
4268
|
-
{ path: entry.path, length: entry.value.length },
|
|
4269
|
-
{
|
|
4270
|
-
category: "size",
|
|
4271
|
-
confidence: "high",
|
|
4272
|
-
detector: "safety.oversizedAttribute",
|
|
4273
|
-
action: "truncate-or-omit"
|
|
4274
|
-
}
|
|
4275
|
-
)
|
|
4276
|
-
);
|
|
4277
|
-
}
|
|
4278
|
-
if (isRecord8(entry.value) && options.maxObjectKeys !== void 0 && Object.keys(entry.value).length > options.maxObjectKeys) {
|
|
4279
|
-
findings.push(
|
|
4280
|
-
failFinding(
|
|
4281
|
-
"safety.oversizedAttribute",
|
|
4282
|
-
`Object at ${entry.path} exceeds ${options.maxObjectKeys} keys.`,
|
|
4283
|
-
[eventEvidence(event, entry.path)],
|
|
4284
|
-
{ maxObjectKeys: options.maxObjectKeys },
|
|
4285
|
-
{ path: entry.path, keys: Object.keys(entry.value).length },
|
|
4286
|
-
{
|
|
4287
|
-
category: "size",
|
|
4288
|
-
confidence: "high",
|
|
4289
|
-
detector: "safety.oversizedAttribute",
|
|
4290
|
-
action: "truncate-or-omit"
|
|
4291
|
-
}
|
|
4292
|
-
)
|
|
4293
|
-
);
|
|
4294
|
-
}
|
|
4295
|
-
if (options.maxSerializedBytes !== void 0) {
|
|
4296
|
-
const bytes = serializedByteLength(entry.value);
|
|
4297
|
-
if (bytes !== void 0 && bytes > options.maxSerializedBytes) {
|
|
4298
|
-
findings.push(
|
|
4299
|
-
failFinding(
|
|
4300
|
-
"safety.oversizedAttribute",
|
|
4301
|
-
`Value at ${entry.path} exceeds ${options.maxSerializedBytes} serialized bytes.`,
|
|
4302
|
-
[eventEvidence(event, entry.path)],
|
|
4303
|
-
{ maxSerializedBytes: options.maxSerializedBytes },
|
|
4304
|
-
{ path: entry.path, bytes },
|
|
4305
|
-
{
|
|
4306
|
-
category: "size",
|
|
4307
|
-
confidence: "high",
|
|
4308
|
-
detector: "safety.oversizedAttribute",
|
|
4309
|
-
action: "truncate-or-omit"
|
|
4310
|
-
}
|
|
4311
|
-
)
|
|
4312
|
-
);
|
|
4313
|
-
}
|
|
4314
|
-
}
|
|
4315
|
-
}
|
|
4316
|
-
}
|
|
4317
|
-
return limitFindings(findings, options.maxFindings);
|
|
4318
|
-
}
|
|
4285
|
+
ignoreDuration: false,
|
|
4286
|
+
durationThresholdMs: DEFAULT_THRESHOLD_MS,
|
|
4287
|
+
focus: "all",
|
|
4288
|
+
check: "all"
|
|
4319
4289
|
};
|
|
4320
4290
|
}
|
|
4321
|
-
function
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
const
|
|
4327
|
-
|
|
4328
|
-
|
|
4291
|
+
function kindMatchesFilter(kind, merged) {
|
|
4292
|
+
return true;
|
|
4293
|
+
}
|
|
4294
|
+
function diffRuns(left, right, options) {
|
|
4295
|
+
const merged = mergeDiffDefaults();
|
|
4296
|
+
const opts = {
|
|
4297
|
+
ignoreDuration: merged.ignoreDuration,
|
|
4298
|
+
durationThresholdMs: merged.durationThresholdMs
|
|
4299
|
+
};
|
|
4300
|
+
const raw = [];
|
|
4301
|
+
if ((left.status ?? "") !== (right.status ?? "")) {
|
|
4302
|
+
raw.push({
|
|
4303
|
+
kind: "run-status",
|
|
4304
|
+
severity: "warning",
|
|
4305
|
+
message: "Run completion status differs",
|
|
4306
|
+
left: left.status,
|
|
4307
|
+
right: right.status
|
|
4308
|
+
});
|
|
4329
4309
|
}
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
|
|
4336
|
-
|
|
4337
|
-
|
|
4338
|
-
|
|
4339
|
-
|
|
4340
|
-
|
|
4310
|
+
{
|
|
4311
|
+
const ld = left.durationMs;
|
|
4312
|
+
const rd = right.durationMs;
|
|
4313
|
+
const th = merged.durationThresholdMs;
|
|
4314
|
+
let differs = false;
|
|
4315
|
+
if (ld === void 0 && rd === void 0) differs = false;
|
|
4316
|
+
else if (ld === void 0 || rd === void 0) differs = true;
|
|
4317
|
+
else differs = Math.abs(ld - rd) > th;
|
|
4318
|
+
if (differs) {
|
|
4319
|
+
raw.push({
|
|
4320
|
+
kind: "duration",
|
|
4321
|
+
severity: "info",
|
|
4322
|
+
message: "Run duration differs",
|
|
4323
|
+
left: ld,
|
|
4324
|
+
right: rd
|
|
4325
|
+
});
|
|
4326
|
+
}
|
|
4341
4327
|
}
|
|
4342
|
-
const
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
ruleId: rule.id,
|
|
4357
|
-
category: rule.category,
|
|
4358
|
-
status: classifyRuleExecution(ruleFindings, false),
|
|
4359
|
-
findingCount: ruleFindings.length,
|
|
4360
|
-
...selected.run ? { runId: selected.run.runId } : {}
|
|
4328
|
+
const pairs = pairSteps(left.steps, right.steps);
|
|
4329
|
+
let idx = 0;
|
|
4330
|
+
for (const [ls, rs] of pairs) {
|
|
4331
|
+
if (ls !== void 0 && rs !== void 0) {
|
|
4332
|
+
compareRecursive(ls, rs, [pathSeg(ls, idx)], opts, raw);
|
|
4333
|
+
idx += 1;
|
|
4334
|
+
} else if (ls !== void 0) {
|
|
4335
|
+
raw.push({
|
|
4336
|
+
kind: "step-removed",
|
|
4337
|
+
severity: "warning",
|
|
4338
|
+
message: `Step only in left run: ${ls.name}`,
|
|
4339
|
+
path: buildPath([pathSeg(ls, idx)]),
|
|
4340
|
+
left: ls.id,
|
|
4341
|
+
right: void 0
|
|
4361
4342
|
});
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4369
|
-
|
|
4370
|
-
|
|
4371
|
-
findingCount: 0,
|
|
4372
|
-
...selected.run ? { runId: selected.run.runId } : {}
|
|
4343
|
+
idx += 1;
|
|
4344
|
+
} else if (rs !== void 0) {
|
|
4345
|
+
raw.push({
|
|
4346
|
+
kind: "step-added",
|
|
4347
|
+
severity: "warning",
|
|
4348
|
+
message: `Step only in right run: ${rs.name}`,
|
|
4349
|
+
path: buildPath([pathSeg(rs, idx)]),
|
|
4350
|
+
left: void 0,
|
|
4351
|
+
right: rs.id
|
|
4373
4352
|
});
|
|
4353
|
+
idx += 1;
|
|
4374
4354
|
}
|
|
4375
4355
|
}
|
|
4376
|
-
|
|
4377
|
-
|
|
4356
|
+
const differences = raw.filter((d) => kindMatchesFilter(d.kind));
|
|
4357
|
+
let errors = 0;
|
|
4358
|
+
let warnings = 0;
|
|
4359
|
+
let info = 0;
|
|
4360
|
+
for (const d of differences) {
|
|
4361
|
+
if (d.severity === "error") errors += 1;
|
|
4362
|
+
else if (d.severity === "warning") warnings += 1;
|
|
4363
|
+
else info += 1;
|
|
4378
4364
|
}
|
|
4379
|
-
const
|
|
4380
|
-
const
|
|
4381
|
-
|
|
4382
|
-
|
|
4383
|
-
|
|
4384
|
-
|
|
4385
|
-
|
|
4386
|
-
|
|
4387
|
-
|
|
4388
|
-
|
|
4389
|
-
|
|
4390
|
-
|
|
4391
|
-
|
|
4365
|
+
const firstVisible = differences[0];
|
|
4366
|
+
const firstDivergence = firstVisible !== void 0 ? {
|
|
4367
|
+
kind: "first-divergence",
|
|
4368
|
+
severity: firstVisible.severity,
|
|
4369
|
+
message: `First divergence: ${firstVisible.message}`,
|
|
4370
|
+
path: firstVisible.path,
|
|
4371
|
+
left: firstVisible.left,
|
|
4372
|
+
right: firstVisible.right
|
|
4373
|
+
} : void 0;
|
|
4374
|
+
const summary = {
|
|
4375
|
+
leftRunId: left.runId,
|
|
4376
|
+
rightRunId: right.runId,
|
|
4377
|
+
totalDifferences: differences.length,
|
|
4378
|
+
errors,
|
|
4379
|
+
warnings,
|
|
4380
|
+
info,
|
|
4381
|
+
firstDivergence
|
|
4392
4382
|
};
|
|
4383
|
+
return { summary, differences };
|
|
4393
4384
|
}
|
|
4394
4385
|
|
|
4386
|
+
// packages/core/src/evidence/zip.ts
|
|
4387
|
+
(() => {
|
|
4388
|
+
const table = new Uint32Array(256);
|
|
4389
|
+
for (let n = 0; n < 256; n += 1) {
|
|
4390
|
+
let c = n;
|
|
4391
|
+
for (let k = 0; k < 8; k += 1) {
|
|
4392
|
+
c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
|
|
4393
|
+
}
|
|
4394
|
+
table[n] = c >>> 0;
|
|
4395
|
+
}
|
|
4396
|
+
return table;
|
|
4397
|
+
})();
|
|
4398
|
+
|
|
4395
4399
|
// packages/core/src/persisted/token-usage.ts
|
|
4396
4400
|
function isRecord9(value) {
|
|
4397
4401
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -8651,5 +8655,5 @@ async function runReadOnlyMcpServer(options = {}) {
|
|
|
8651
8655
|
}
|
|
8652
8656
|
|
|
8653
8657
|
export { MCP_MAX_REQUEST_BYTES, MCP_PROTOCOL_VERSION, MCP_SERVER_INSTRUCTIONS, READ_ONLY_TOOLS, TRACE_DATA_UNTRUSTED_WARNING, callReadOnlyTool, createMcpServerContext, handleMcpProtocolLine, runReadOnlyMcpServer };
|
|
8654
|
-
//# sourceMappingURL=chunk-
|
|
8655
|
-
//# sourceMappingURL=chunk-
|
|
8658
|
+
//# sourceMappingURL=chunk-V6J5UBTW.mjs.map
|
|
8659
|
+
//# sourceMappingURL=chunk-V6J5UBTW.mjs.map
|