@agent-inspect/viewer 6.31.10 → 6.31.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1047 -816
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.mjs +1047 -816
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -1163,870 +1163,870 @@ function buildRunTimeline(events, options = {}) {
|
|
|
1163
1163
|
};
|
|
1164
1164
|
}
|
|
1165
1165
|
|
|
1166
|
-
// packages/core/src/
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1166
|
+
// packages/core/src/diagnostics/programmatic.ts
|
|
1167
|
+
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
1168
|
+
AI_TRACE_INPUT_INVALID: {
|
|
1169
|
+
code: "AI_TRACE_INPUT_INVALID",
|
|
1170
|
+
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
1171
|
+
remediation: "For a file path, use openTraceFile(path).",
|
|
1172
|
+
relatedCodes: ["invalid_input"]
|
|
1173
|
+
},
|
|
1174
|
+
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
1175
|
+
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
1176
|
+
summary: "No trace reader could detect the input format.",
|
|
1177
|
+
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
1178
|
+
relatedCodes: ["unsupported_format"]
|
|
1179
|
+
},
|
|
1180
|
+
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
1181
|
+
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
1182
|
+
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
1183
|
+
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
1184
|
+
relatedCodes: ["ambiguous_format"]
|
|
1185
|
+
},
|
|
1186
|
+
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
1187
|
+
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
1188
|
+
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
1189
|
+
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
1190
|
+
},
|
|
1191
|
+
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
1192
|
+
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
1193
|
+
summary: "Multiple runs are available; select a run before executing checks.",
|
|
1194
|
+
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
1195
|
+
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
1196
|
+
},
|
|
1197
|
+
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
1198
|
+
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1199
|
+
summary: "A parentId equals its own event/step id.",
|
|
1200
|
+
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
1201
|
+
relatedCodes: [
|
|
1202
|
+
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
1203
|
+
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
1204
|
+
]
|
|
1205
|
+
},
|
|
1206
|
+
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
1207
|
+
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
1208
|
+
summary: "Trace contains a parentId cycle.",
|
|
1209
|
+
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
1210
|
+
relatedCodes: ["structure.cycle"]
|
|
1211
|
+
},
|
|
1212
|
+
AI_TRACE_CONTRACT_INPUT_INVALID: {
|
|
1213
|
+
code: "AI_TRACE_CONTRACT_INPUT_INVALID",
|
|
1214
|
+
summary: "evaluateTraceContract expects ({ read }, contract) or use evaluateTraceContractRead(read, contract).",
|
|
1215
|
+
remediation: "Pass TraceCheckInput as the first argument: evaluateTraceContract({ read }, contract).",
|
|
1216
|
+
relatedCodes: ["AI_CHECK_INVALID_ARGUMENTS"]
|
|
1217
|
+
},
|
|
1218
|
+
AI_INSPECT_RUN_UNSUPPORTED_OPTION: {
|
|
1219
|
+
code: "AI_INSPECT_RUN_UNSUPPORTED_OPTION",
|
|
1220
|
+
summary: 'inspectRun does not accept a "writer" option.',
|
|
1221
|
+
remediation: "Use createInspector({ writer }) then inspector.run(...). inspectRun persists via the global JSONL path (schema 0.1)."
|
|
1176
1222
|
}
|
|
1177
|
-
|
|
1223
|
+
});
|
|
1224
|
+
function formatProgrammaticDiagnostic(code, detail) {
|
|
1225
|
+
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
1226
|
+
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
1227
|
+
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
1178
1228
|
}
|
|
1179
1229
|
|
|
1180
|
-
// packages/core/src/
|
|
1181
|
-
function
|
|
1182
|
-
return typeof value === "
|
|
1183
|
-
}
|
|
1184
|
-
function finitePositiveInt(value) {
|
|
1185
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
1186
|
-
return void 0;
|
|
1187
|
-
}
|
|
1188
|
-
return Math.trunc(value);
|
|
1230
|
+
// packages/core/src/checks/logical-events.ts
|
|
1231
|
+
function isRecord5(value) {
|
|
1232
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1189
1233
|
}
|
|
1190
|
-
function
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
let found = false;
|
|
1194
|
-
const assignString = (key, value) => {
|
|
1195
|
-
if (isNonEmptyString3(value)) {
|
|
1196
|
-
out[key] = value.trim();
|
|
1197
|
-
found = true;
|
|
1198
|
-
}
|
|
1199
|
-
};
|
|
1200
|
-
assignString("sessionId", record.sessionId);
|
|
1201
|
-
assignString("conversationId", record.conversationId);
|
|
1202
|
-
assignString("groupId", record.groupId);
|
|
1203
|
-
assignString("parentGroupId", record.parentGroupId);
|
|
1204
|
-
assignString("retryOf", record.retryOf);
|
|
1205
|
-
assignString("retryReason", record.retryReason);
|
|
1206
|
-
assignString("handoffFrom", record.handoffFrom);
|
|
1207
|
-
assignString("handoffTo", record.handoffTo);
|
|
1208
|
-
assignString("subAgentId", record.subAgentId);
|
|
1209
|
-
assignString("subAgentName", record.subAgentName);
|
|
1210
|
-
assignString("jobId", record.jobId);
|
|
1211
|
-
assignString("queueName", record.queueName);
|
|
1212
|
-
assignString("workflowName", record.workflowName);
|
|
1213
|
-
assignString("workflowStep", record.workflowStep);
|
|
1214
|
-
assignString("toolCallId", record.toolCallId);
|
|
1215
|
-
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
1216
|
-
assignString("linkedStepId", record.linkedStepId);
|
|
1217
|
-
assignString("operationId", record.operationId);
|
|
1218
|
-
assignString("attemptId", record.attemptId);
|
|
1219
|
-
assignString("fallbackOf", record.fallbackOf);
|
|
1220
|
-
assignString("idempotencyKey", record.idempotencyKey);
|
|
1221
|
-
assignString("correlationId", record.correlationId);
|
|
1222
|
-
assignString("requestId", record.requestId);
|
|
1223
|
-
assignString("decisionId", record.decisionId);
|
|
1224
|
-
const attempt = finitePositiveInt(record.attempt);
|
|
1225
|
-
if (attempt !== void 0) {
|
|
1226
|
-
out.attempt = attempt;
|
|
1227
|
-
found = true;
|
|
1228
|
-
}
|
|
1229
|
-
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
1230
|
-
if (attemptNumber !== void 0) {
|
|
1231
|
-
out.attemptNumber = attemptNumber;
|
|
1232
|
-
found = true;
|
|
1233
|
-
}
|
|
1234
|
-
return found ? out : void 0;
|
|
1234
|
+
function legacyEvent(event) {
|
|
1235
|
+
const value = event.attributes?.legacyEvent;
|
|
1236
|
+
return typeof value === "string" ? value : void 0;
|
|
1235
1237
|
}
|
|
1236
|
-
function
|
|
1237
|
-
|
|
1238
|
-
if (
|
|
1239
|
-
return `group:${meta.groupId}`;
|
|
1240
|
-
}
|
|
1238
|
+
function stepIdOf(event) {
|
|
1239
|
+
const value = event.attributes?.stepId;
|
|
1240
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
1241
1241
|
return void 0;
|
|
1242
1242
|
}
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
completed: 1
|
|
1252
|
-
};
|
|
1253
|
-
var EXPLICIT_SESSION_STATUSES = /* @__PURE__ */ new Set([
|
|
1254
|
-
"running",
|
|
1255
|
-
"waiting_input",
|
|
1256
|
-
"idle",
|
|
1257
|
-
"completed",
|
|
1258
|
-
"error",
|
|
1259
|
-
"stale",
|
|
1260
|
-
"unknown"
|
|
1261
|
-
]);
|
|
1262
|
-
function isExplicitSessionStatus(value) {
|
|
1263
|
-
return typeof value === "string" && EXPLICIT_SESSION_STATUSES.has(value);
|
|
1264
|
-
}
|
|
1265
|
-
function activityMs(run) {
|
|
1266
|
-
return run.endedAt ?? run.startedAt ?? 0;
|
|
1243
|
+
function cloneEvent(event) {
|
|
1244
|
+
return {
|
|
1245
|
+
...event,
|
|
1246
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
1247
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
1248
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
1249
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
1250
|
+
};
|
|
1267
1251
|
}
|
|
1268
|
-
function
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1252
|
+
function mergeAttributes(start, complete) {
|
|
1253
|
+
const merged = {
|
|
1254
|
+
...isRecord5(complete.attributes) ? complete.attributes : {},
|
|
1255
|
+
...isRecord5(start.attributes) ? start.attributes : {}
|
|
1256
|
+
};
|
|
1257
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
1258
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
1259
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
1260
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
1273
1261
|
}
|
|
1274
|
-
return
|
|
1262
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
1275
1263
|
}
|
|
1276
|
-
function
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1264
|
+
function pairStartComplete(start, complete) {
|
|
1265
|
+
const attributes = mergeAttributes(start, complete);
|
|
1266
|
+
const paired = {
|
|
1267
|
+
...cloneEvent(start),
|
|
1268
|
+
status: complete.status,
|
|
1269
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
1270
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
1271
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
1272
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
1273
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
1274
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
1275
|
+
};
|
|
1276
|
+
return {
|
|
1277
|
+
...paired,
|
|
1278
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
1279
|
+
projection: {
|
|
1280
|
+
paired: true,
|
|
1281
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
1282
|
+
parentNormalized: false,
|
|
1283
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
1282
1284
|
}
|
|
1283
|
-
}
|
|
1284
|
-
return earliest;
|
|
1285
|
-
}
|
|
1286
|
-
function latestEndWhenAllEnded(runs) {
|
|
1287
|
-
if (runs.length === 0) return void 0;
|
|
1288
|
-
let latest;
|
|
1289
|
-
for (const run of runs) {
|
|
1290
|
-
if (run.endedAt === void 0) return void 0;
|
|
1291
|
-
if (latest === void 0 || run.endedAt > latest) latest = run.endedAt;
|
|
1292
|
-
}
|
|
1293
|
-
return latest;
|
|
1285
|
+
};
|
|
1294
1286
|
}
|
|
1295
|
-
function
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
best = raw;
|
|
1287
|
+
function asLogical(event, extras) {
|
|
1288
|
+
return {
|
|
1289
|
+
...cloneEvent(event),
|
|
1290
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
1291
|
+
projection: {
|
|
1292
|
+
paired: false,
|
|
1293
|
+
absorbedEventIds: Object.freeze([]),
|
|
1294
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
1295
|
+
...{}
|
|
1305
1296
|
}
|
|
1306
|
-
}
|
|
1307
|
-
return best;
|
|
1297
|
+
};
|
|
1308
1298
|
}
|
|
1309
|
-
function
|
|
1310
|
-
const
|
|
1311
|
-
const
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
const
|
|
1325
|
-
|
|
1326
|
-
const
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1299
|
+
function projectLogicalEvents(events) {
|
|
1300
|
+
const diagnostics = [];
|
|
1301
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
1302
|
+
for (const event of events) {
|
|
1303
|
+
const list = byRun.get(event.runId) ?? [];
|
|
1304
|
+
list.push(event);
|
|
1305
|
+
byRun.set(event.runId, list);
|
|
1306
|
+
}
|
|
1307
|
+
const absorbedIds = /* @__PURE__ */ new Set();
|
|
1308
|
+
const logicalByRawId = /* @__PURE__ */ new Map();
|
|
1309
|
+
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
1310
|
+
const logical = [];
|
|
1311
|
+
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
1312
|
+
for (const runId of orderedRuns) {
|
|
1313
|
+
const runEvents = byRun.get(runId) ?? [];
|
|
1314
|
+
const starts = [];
|
|
1315
|
+
const completes = [];
|
|
1316
|
+
const others = [];
|
|
1317
|
+
for (const event of runEvents) {
|
|
1318
|
+
const legacy = legacyEvent(event);
|
|
1319
|
+
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
1320
|
+
starts.push(event);
|
|
1321
|
+
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
1322
|
+
completes.push(event);
|
|
1323
|
+
} else {
|
|
1324
|
+
others.push(event);
|
|
1325
|
+
}
|
|
1330
1326
|
}
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1327
|
+
const usedCompletes = /* @__PURE__ */ new Set();
|
|
1328
|
+
for (const start of starts) {
|
|
1329
|
+
const stepId = stepIdOf(start);
|
|
1330
|
+
let match;
|
|
1331
|
+
if (legacyEvent(start) === "run_started") {
|
|
1332
|
+
const candidates = completes.filter(
|
|
1333
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
1334
|
+
);
|
|
1335
|
+
if (candidates.length > 1) {
|
|
1336
|
+
diagnostics.push({
|
|
1337
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1338
|
+
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
1339
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1340
|
+
});
|
|
1341
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1342
|
+
}
|
|
1343
|
+
match = candidates[0];
|
|
1344
|
+
} else if (stepId) {
|
|
1345
|
+
const candidates = completes.filter(
|
|
1346
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
1347
|
+
);
|
|
1348
|
+
if (candidates.length > 1) {
|
|
1349
|
+
diagnostics.push({
|
|
1350
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1351
|
+
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
1352
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1353
|
+
});
|
|
1354
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1355
|
+
}
|
|
1356
|
+
match = candidates[0];
|
|
1357
|
+
}
|
|
1358
|
+
if (match) {
|
|
1359
|
+
usedCompletes.add(match.eventId);
|
|
1360
|
+
absorbedIds.add(match.eventId);
|
|
1361
|
+
const paired = pairStartComplete(start, match);
|
|
1362
|
+
logical.push(paired);
|
|
1363
|
+
logicalByRawId.set(start.eventId, paired);
|
|
1364
|
+
logicalByRawId.set(match.eventId, paired);
|
|
1365
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
1366
|
+
} else {
|
|
1367
|
+
diagnostics.push({
|
|
1368
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
1369
|
+
message: `No matching complete for start ${start.eventId}.`,
|
|
1370
|
+
eventIds: [start.eventId]
|
|
1371
|
+
});
|
|
1372
|
+
const alone = asLogical(start);
|
|
1373
|
+
logical.push(alone);
|
|
1374
|
+
logicalByRawId.set(start.eventId, alone);
|
|
1375
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1376
|
+
}
|
|
1334
1377
|
}
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1378
|
+
for (const complete of completes) {
|
|
1379
|
+
if (usedCompletes.has(complete.eventId)) continue;
|
|
1380
|
+
diagnostics.push({
|
|
1381
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
1382
|
+
message: `No matching start for complete ${complete.eventId}.`,
|
|
1383
|
+
eventIds: [complete.eventId]
|
|
1384
|
+
});
|
|
1385
|
+
const alone = asLogical(complete);
|
|
1386
|
+
logical.push(alone);
|
|
1387
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
1388
|
+
const stepId = stepIdOf(complete);
|
|
1389
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1390
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
for (const event of others) {
|
|
1394
|
+
const alone = asLogical(event);
|
|
1395
|
+
logical.push(alone);
|
|
1396
|
+
logicalByRawId.set(event.eventId, alone);
|
|
1397
|
+
const stepId = stepIdOf(event);
|
|
1398
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1399
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1400
|
+
}
|
|
1338
1401
|
}
|
|
1339
1402
|
}
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1403
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
1404
|
+
const normalized = [];
|
|
1405
|
+
for (const event of logical) {
|
|
1406
|
+
const originalParentId = event.parentId;
|
|
1407
|
+
if (!originalParentId) {
|
|
1408
|
+
normalized.push(event);
|
|
1409
|
+
continue;
|
|
1410
|
+
}
|
|
1411
|
+
let nextParent = originalParentId;
|
|
1412
|
+
let remapped = false;
|
|
1413
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
1414
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
1415
|
+
nextParent = viaAbsorbed.eventId;
|
|
1416
|
+
remapped = true;
|
|
1417
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
1418
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
1419
|
+
if (viaStep) {
|
|
1420
|
+
nextParent = viaStep;
|
|
1421
|
+
remapped = true;
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
if (!remapped) {
|
|
1425
|
+
if (originalParentId === event.eventId) {
|
|
1426
|
+
diagnostics.push({
|
|
1427
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1428
|
+
message: formatProgrammaticDiagnostic(
|
|
1429
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1430
|
+
`Removed self-parent edge on ${event.eventId}.`
|
|
1431
|
+
),
|
|
1432
|
+
eventIds: [event.eventId]
|
|
1433
|
+
});
|
|
1434
|
+
const { parentId: _drop, ...rest } = event;
|
|
1435
|
+
normalized.push({
|
|
1436
|
+
...rest,
|
|
1437
|
+
projection: {
|
|
1438
|
+
...event.projection,
|
|
1439
|
+
parentNormalized: true,
|
|
1440
|
+
originalParentId
|
|
1441
|
+
}
|
|
1442
|
+
});
|
|
1443
|
+
continue;
|
|
1444
|
+
}
|
|
1445
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
1446
|
+
const mapping = event.attributes?.parentMapping;
|
|
1447
|
+
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
1448
|
+
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
1449
|
+
if (!unresolved) {
|
|
1450
|
+
diagnostics.push({
|
|
1451
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
1452
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
1453
|
+
eventIds: [event.eventId]
|
|
1454
|
+
});
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
normalized.push(event);
|
|
1458
|
+
continue;
|
|
1459
|
+
}
|
|
1460
|
+
if (nextParent === event.eventId) {
|
|
1461
|
+
diagnostics.push({
|
|
1462
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1463
|
+
message: formatProgrammaticDiagnostic(
|
|
1464
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1465
|
+
`Removed self-parent edge on ${event.eventId} (was ${originalParentId}).`
|
|
1466
|
+
),
|
|
1467
|
+
eventIds: [event.eventId]
|
|
1468
|
+
});
|
|
1469
|
+
const { parentId: _drop, ...rest } = event;
|
|
1470
|
+
normalized.push({
|
|
1471
|
+
...rest,
|
|
1472
|
+
projection: {
|
|
1473
|
+
...event.projection,
|
|
1474
|
+
parentNormalized: true,
|
|
1475
|
+
originalParentId
|
|
1476
|
+
}
|
|
1477
|
+
});
|
|
1478
|
+
continue;
|
|
1347
1479
|
}
|
|
1480
|
+
diagnostics.push({
|
|
1481
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
1482
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
1483
|
+
eventIds: [event.eventId]
|
|
1484
|
+
});
|
|
1485
|
+
normalized.push({
|
|
1486
|
+
...event,
|
|
1487
|
+
parentId: nextParent,
|
|
1488
|
+
projection: {
|
|
1489
|
+
...event.projection,
|
|
1490
|
+
parentNormalized: true,
|
|
1491
|
+
originalParentId
|
|
1492
|
+
}
|
|
1493
|
+
});
|
|
1348
1494
|
}
|
|
1349
|
-
|
|
1495
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
1496
|
+
normalized.sort((a, b) => {
|
|
1497
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
1498
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
1499
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
1500
|
+
});
|
|
1501
|
+
return {
|
|
1502
|
+
logicalEvents: Object.freeze(normalized),
|
|
1503
|
+
diagnostics: Object.freeze(diagnostics)
|
|
1504
|
+
};
|
|
1350
1505
|
}
|
|
1351
|
-
function
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
if (
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
const
|
|
1361
|
-
|
|
1362
|
-
|
|
1506
|
+
function resolveCanonicalToolName(event) {
|
|
1507
|
+
const attrs = event.attributes;
|
|
1508
|
+
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
1509
|
+
if (direct) return direct;
|
|
1510
|
+
const metadata = attrs?.metadata;
|
|
1511
|
+
if (isRecord5(metadata)) {
|
|
1512
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
1513
|
+
if (nested) return nested;
|
|
1514
|
+
}
|
|
1515
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
1516
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
1517
|
+
}
|
|
1518
|
+
return event.name;
|
|
1363
1519
|
}
|
|
1364
|
-
function
|
|
1365
|
-
|
|
1366
|
-
const
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
let correlationId;
|
|
1370
|
-
let jobId;
|
|
1371
|
-
let workflowId;
|
|
1372
|
-
for (const run of sessionRuns) {
|
|
1373
|
-
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1374
|
-
if (!correlationId && meta?.correlationId) correlationId = meta.correlationId;
|
|
1375
|
-
if (!jobId && meta?.jobId) jobId = meta.jobId;
|
|
1376
|
-
if (!workflowId && meta?.workflowName) workflowId = meta.workflowName;
|
|
1377
|
-
else if (!workflowId && meta?.workflowStep) workflowId = meta.workflowStep;
|
|
1520
|
+
function pickString(record, keys) {
|
|
1521
|
+
if (!record) return void 0;
|
|
1522
|
+
for (const key of keys) {
|
|
1523
|
+
const value = record[key];
|
|
1524
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
1378
1525
|
}
|
|
1379
|
-
|
|
1380
|
-
const lastActivity = lastMs > 0 ? new Date(lastMs).toISOString() : (/* @__PURE__ */ new Date(0)).toISOString();
|
|
1381
|
-
const retryCount2 = summary.retries.filter(
|
|
1382
|
-
(retry) => retry.retryOf !== void 0 || (retry.attempt ?? 0) > 1
|
|
1383
|
-
).length;
|
|
1384
|
-
return {
|
|
1385
|
-
...summary,
|
|
1386
|
-
status: deriveSessionStatus(sessionRuns, options),
|
|
1387
|
-
startedAt,
|
|
1388
|
-
endedAt,
|
|
1389
|
-
durationMs,
|
|
1390
|
-
correlationId,
|
|
1391
|
-
jobId,
|
|
1392
|
-
workflowId,
|
|
1393
|
-
lastError: deriveLastError(sessionRuns),
|
|
1394
|
-
lastActivity,
|
|
1395
|
-
retryCount: retryCount2,
|
|
1396
|
-
observationSummary: deriveObservationSummary(sessionRuns),
|
|
1397
|
-
checkSummary: deriveCheckSummary(sessionRuns)
|
|
1398
|
-
};
|
|
1526
|
+
return void 0;
|
|
1399
1527
|
}
|
|
1400
1528
|
|
|
1401
|
-
// packages/core/src/
|
|
1402
|
-
async function
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
}
|
|
1411
|
-
break;
|
|
1529
|
+
// packages/core/src/search.ts
|
|
1530
|
+
async function loadTraceMetadataList(_traceDir, fileNames, getPath) {
|
|
1531
|
+
const metas = [];
|
|
1532
|
+
for (const fileName of fileNames) {
|
|
1533
|
+
try {
|
|
1534
|
+
const filePath = getPath(fileName);
|
|
1535
|
+
const meta = await extractMetadata(filePath);
|
|
1536
|
+
metas.push(meta);
|
|
1537
|
+
} catch {
|
|
1412
1538
|
}
|
|
1413
|
-
} catch {
|
|
1414
1539
|
}
|
|
1415
|
-
return
|
|
1416
|
-
runId: meta.runId,
|
|
1417
|
-
name: meta.name,
|
|
1418
|
-
status: meta.status,
|
|
1419
|
-
startedAt: meta.startedAt,
|
|
1420
|
-
endedAt: meta.endedAt,
|
|
1421
|
-
durationMs: meta.durationMs,
|
|
1422
|
-
filePath: meta.filePath,
|
|
1423
|
-
metadata
|
|
1424
|
-
};
|
|
1540
|
+
return metas;
|
|
1425
1541
|
}
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1542
|
+
|
|
1543
|
+
// packages/core/src/sessions/metadata.ts
|
|
1544
|
+
function isNonEmptyString3(value) {
|
|
1545
|
+
return typeof value === "string" && value.trim() !== "";
|
|
1546
|
+
}
|
|
1547
|
+
function finitePositiveInt(value) {
|
|
1548
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
1549
|
+
return void 0;
|
|
1550
|
+
}
|
|
1551
|
+
return Math.trunc(value);
|
|
1552
|
+
}
|
|
1553
|
+
function extractSessionWorkflowMetadata(record) {
|
|
1554
|
+
if (!record) return void 0;
|
|
1555
|
+
const out = {};
|
|
1556
|
+
let found = false;
|
|
1557
|
+
const assignString = (key, value) => {
|
|
1558
|
+
if (isNonEmptyString3(value)) {
|
|
1559
|
+
out[key] = value.trim();
|
|
1560
|
+
found = true;
|
|
1561
|
+
}
|
|
1562
|
+
};
|
|
1563
|
+
assignString("sessionId", record.sessionId);
|
|
1564
|
+
assignString("conversationId", record.conversationId);
|
|
1565
|
+
assignString("groupId", record.groupId);
|
|
1566
|
+
assignString("parentGroupId", record.parentGroupId);
|
|
1567
|
+
assignString("retryOf", record.retryOf);
|
|
1568
|
+
assignString("retryReason", record.retryReason);
|
|
1569
|
+
assignString("handoffFrom", record.handoffFrom);
|
|
1570
|
+
assignString("handoffTo", record.handoffTo);
|
|
1571
|
+
assignString("subAgentId", record.subAgentId);
|
|
1572
|
+
assignString("subAgentName", record.subAgentName);
|
|
1573
|
+
assignString("jobId", record.jobId);
|
|
1574
|
+
assignString("queueName", record.queueName);
|
|
1575
|
+
assignString("workflowName", record.workflowName);
|
|
1576
|
+
assignString("workflowStep", record.workflowStep);
|
|
1577
|
+
assignString("toolCallId", record.toolCallId);
|
|
1578
|
+
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
1579
|
+
assignString("linkedStepId", record.linkedStepId);
|
|
1580
|
+
assignString("operationId", record.operationId);
|
|
1581
|
+
assignString("attemptId", record.attemptId);
|
|
1582
|
+
assignString("fallbackOf", record.fallbackOf);
|
|
1583
|
+
assignString("idempotencyKey", record.idempotencyKey);
|
|
1584
|
+
assignString("correlationId", record.correlationId);
|
|
1585
|
+
assignString("requestId", record.requestId);
|
|
1586
|
+
assignString("decisionId", record.decisionId);
|
|
1587
|
+
const attempt = finitePositiveInt(record.attempt);
|
|
1588
|
+
if (attempt !== void 0) {
|
|
1589
|
+
out.attempt = attempt;
|
|
1590
|
+
found = true;
|
|
1591
|
+
}
|
|
1592
|
+
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
1593
|
+
if (attemptNumber !== void 0) {
|
|
1594
|
+
out.attemptNumber = attemptNumber;
|
|
1595
|
+
found = true;
|
|
1430
1596
|
}
|
|
1431
|
-
return out;
|
|
1597
|
+
return found ? out : void 0;
|
|
1598
|
+
}
|
|
1599
|
+
function sessionKeyForRun(meta, options) {
|
|
1600
|
+
if (meta?.sessionId) return meta.sessionId;
|
|
1601
|
+
if (options?.correlateByGroupId && meta?.groupId) {
|
|
1602
|
+
return `group:${meta.groupId}`;
|
|
1603
|
+
}
|
|
1604
|
+
return void 0;
|
|
1432
1605
|
}
|
|
1433
1606
|
|
|
1434
|
-
// packages/core/src/sessions/
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1607
|
+
// packages/core/src/sessions/status.ts
|
|
1608
|
+
var DEFAULT_STALE_THRESHOLD_MS = 864e5;
|
|
1609
|
+
var EXPLICIT_STATUS_PRIORITY = {
|
|
1610
|
+
error: 5,
|
|
1611
|
+
waiting_input: 4,
|
|
1612
|
+
idle: 3,
|
|
1613
|
+
stale: 2,
|
|
1614
|
+
completed: 1
|
|
1615
|
+
};
|
|
1616
|
+
var EXPLICIT_SESSION_STATUSES = /* @__PURE__ */ new Set([
|
|
1617
|
+
"running",
|
|
1618
|
+
"waiting_input",
|
|
1619
|
+
"idle",
|
|
1620
|
+
"completed",
|
|
1621
|
+
"error",
|
|
1622
|
+
"stale",
|
|
1623
|
+
"unknown"
|
|
1624
|
+
]);
|
|
1625
|
+
function isExplicitSessionStatus(value) {
|
|
1626
|
+
return typeof value === "string" && EXPLICIT_SESSION_STATUSES.has(value);
|
|
1440
1627
|
}
|
|
1441
|
-
function
|
|
1442
|
-
|
|
1443
|
-
for (const runId of runIds) {
|
|
1444
|
-
const groupId = metaByRunId.get(runId)?.groupId;
|
|
1445
|
-
if (!groupId) continue;
|
|
1446
|
-
const existing = byGroup.get(groupId);
|
|
1447
|
-
if (existing) {
|
|
1448
|
-
existing.runIds.push(runId);
|
|
1449
|
-
} else {
|
|
1450
|
-
byGroup.set(groupId, {
|
|
1451
|
-
groupId,
|
|
1452
|
-
parentGroupId: metaByRunId.get(runId)?.parentGroupId,
|
|
1453
|
-
runIds: [runId]
|
|
1454
|
-
});
|
|
1455
|
-
}
|
|
1456
|
-
}
|
|
1457
|
-
return [...byGroup.values()].map((group) => ({
|
|
1458
|
-
...group,
|
|
1459
|
-
runIds: [...group.runIds].sort((a, b) => a.localeCompare(b))
|
|
1460
|
-
}));
|
|
1628
|
+
function activityMs(run) {
|
|
1629
|
+
return run.endedAt ?? run.startedAt ?? 0;
|
|
1461
1630
|
}
|
|
1462
|
-
function
|
|
1463
|
-
|
|
1464
|
-
const
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
if (seen.has(key)) return;
|
|
1468
|
-
seen.add(key);
|
|
1469
|
-
edges.push(edge);
|
|
1470
|
-
};
|
|
1471
|
-
for (const runId of runIds) {
|
|
1472
|
-
const meta = metaByRunId.get(runId);
|
|
1473
|
-
if (!meta) continue;
|
|
1474
|
-
if (meta.handoffFrom && meta.handoffTo) {
|
|
1475
|
-
pushEdge({
|
|
1476
|
-
from: meta.handoffFrom,
|
|
1477
|
-
to: meta.handoffTo,
|
|
1478
|
-
source: "manual",
|
|
1479
|
-
confidence: "explicit"
|
|
1480
|
-
});
|
|
1481
|
-
continue;
|
|
1482
|
-
}
|
|
1483
|
-
if (meta.handoffFrom) {
|
|
1484
|
-
pushEdge({
|
|
1485
|
-
from: meta.handoffFrom,
|
|
1486
|
-
to: runId,
|
|
1487
|
-
source: "manual",
|
|
1488
|
-
confidence: "explicit"
|
|
1489
|
-
});
|
|
1490
|
-
}
|
|
1491
|
-
if (meta.handoffTo) {
|
|
1492
|
-
pushEdge({
|
|
1493
|
-
from: runId,
|
|
1494
|
-
to: meta.handoffTo,
|
|
1495
|
-
source: "manual",
|
|
1496
|
-
confidence: "explicit"
|
|
1497
|
-
});
|
|
1498
|
-
}
|
|
1499
|
-
if (meta.subAgentId && meta.parentGroupId && !meta.handoffFrom && !meta.handoffTo) {
|
|
1500
|
-
pushEdge({
|
|
1501
|
-
from: meta.parentGroupId,
|
|
1502
|
-
to: meta.subAgentId,
|
|
1503
|
-
source: "inferred",
|
|
1504
|
-
confidence: "correlated"
|
|
1505
|
-
});
|
|
1506
|
-
warnings.push({
|
|
1507
|
-
code: "ambiguous-handoff-endpoints",
|
|
1508
|
-
message: "Handoff inferred from parentGroupId and subAgentId without explicit handoffFrom/handoffTo.",
|
|
1509
|
-
runId,
|
|
1510
|
-
sessionId
|
|
1511
|
-
});
|
|
1512
|
-
}
|
|
1631
|
+
function latestActivityMs(runs) {
|
|
1632
|
+
let latest = 0;
|
|
1633
|
+
for (const run of runs) {
|
|
1634
|
+
const ms = activityMs(run);
|
|
1635
|
+
if (ms > latest) latest = ms;
|
|
1513
1636
|
}
|
|
1514
|
-
return
|
|
1515
|
-
const from = a.from.localeCompare(b.from);
|
|
1516
|
-
if (from !== 0) return from;
|
|
1517
|
-
return a.to.localeCompare(b.to);
|
|
1518
|
-
});
|
|
1637
|
+
return latest;
|
|
1519
1638
|
}
|
|
1520
|
-
function
|
|
1521
|
-
|
|
1522
|
-
for (const
|
|
1523
|
-
|
|
1524
|
-
if (
|
|
1525
|
-
|
|
1526
|
-
retries.push({
|
|
1527
|
-
runId,
|
|
1528
|
-
retryOf: meta.retryOf,
|
|
1529
|
-
attempt: meta.attempt,
|
|
1530
|
-
source: "manual",
|
|
1531
|
-
confidence: "explicit"
|
|
1532
|
-
});
|
|
1533
|
-
continue;
|
|
1534
|
-
}
|
|
1535
|
-
if (meta.attempt !== void 0 && meta.attempt > 1) {
|
|
1536
|
-
retries.push({
|
|
1537
|
-
runId,
|
|
1538
|
-
attempt: meta.attempt,
|
|
1539
|
-
source: "inferred",
|
|
1540
|
-
confidence: "correlated"
|
|
1541
|
-
});
|
|
1542
|
-
warnings.push({
|
|
1543
|
-
code: "ambiguous-retry-link",
|
|
1544
|
-
message: "attempt > 1 without retryOf; retry link is correlated only.",
|
|
1545
|
-
runId,
|
|
1546
|
-
sessionId
|
|
1547
|
-
});
|
|
1639
|
+
function earliestStart(runs) {
|
|
1640
|
+
let earliest;
|
|
1641
|
+
for (const run of runs) {
|
|
1642
|
+
if (run.startedAt === void 0) continue;
|
|
1643
|
+
if (earliest === void 0 || run.startedAt < earliest) {
|
|
1644
|
+
earliest = run.startedAt;
|
|
1548
1645
|
}
|
|
1549
1646
|
}
|
|
1550
|
-
return
|
|
1647
|
+
return earliest;
|
|
1551
1648
|
}
|
|
1552
|
-
function
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
1559
|
-
);
|
|
1560
|
-
const ordered = [...runs].sort(compareRuns);
|
|
1561
|
-
const path18 = [];
|
|
1562
|
-
const visited = /* @__PURE__ */ new Set();
|
|
1563
|
-
const pushRun = (run, confidence, source) => {
|
|
1564
|
-
if (visited.has(run.runId)) return;
|
|
1565
|
-
visited.add(run.runId);
|
|
1566
|
-
path18.push({
|
|
1567
|
-
runId: run.runId,
|
|
1568
|
-
name: run.name,
|
|
1569
|
-
startedAt: run.startedAt,
|
|
1570
|
-
durationMs: run.durationMs,
|
|
1571
|
-
confidence,
|
|
1572
|
-
source
|
|
1573
|
-
});
|
|
1574
|
-
};
|
|
1575
|
-
for (const edge of handoffs) {
|
|
1576
|
-
if (edge.confidence !== "explicit") continue;
|
|
1577
|
-
const fromRun = [...runById.values()].find(
|
|
1578
|
-
(run) => run.runId === edge.from || metaRunIdMatches(run, edge.from, runById)
|
|
1579
|
-
);
|
|
1580
|
-
const toRun = [...runById.values()].find(
|
|
1581
|
-
(run) => run.runId === edge.to || metaRunIdMatches(run, edge.to, runById)
|
|
1582
|
-
);
|
|
1583
|
-
if (fromRun) pushRun(fromRun, "explicit", "manual");
|
|
1584
|
-
if (toRun) pushRun(toRun, "explicit", "manual");
|
|
1585
|
-
}
|
|
1586
|
-
for (const run of ordered) {
|
|
1587
|
-
if (visited.has(run.runId)) continue;
|
|
1588
|
-
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
1589
|
-
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
1649
|
+
function latestEndWhenAllEnded(runs) {
|
|
1650
|
+
if (runs.length === 0) return void 0;
|
|
1651
|
+
let latest;
|
|
1652
|
+
for (const run of runs) {
|
|
1653
|
+
if (run.endedAt === void 0) return void 0;
|
|
1654
|
+
if (latest === void 0 || run.endedAt > latest) latest = run.endedAt;
|
|
1590
1655
|
}
|
|
1591
|
-
return
|
|
1592
|
-
}
|
|
1593
|
-
function metaRunIdMatches(run, token, runById) {
|
|
1594
|
-
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1595
|
-
return meta?.subAgentId === token || meta?.groupId === token || runById.has(token);
|
|
1656
|
+
return latest;
|
|
1596
1657
|
}
|
|
1597
|
-
function
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
const metaByRunId = /* @__PURE__ */ new Map();
|
|
1658
|
+
function pickExplicitStatus(runs) {
|
|
1659
|
+
let best;
|
|
1660
|
+
let bestPriority = 0;
|
|
1601
1661
|
for (const run of runs) {
|
|
1602
|
-
|
|
1662
|
+
const raw = run.metadata?.sessionStatus;
|
|
1663
|
+
if (!isExplicitSessionStatus(raw)) continue;
|
|
1664
|
+
const priority = EXPLICIT_STATUS_PRIORITY[raw] ?? 0;
|
|
1665
|
+
if (priority > bestPriority) {
|
|
1666
|
+
bestPriority = priority;
|
|
1667
|
+
best = raw;
|
|
1668
|
+
}
|
|
1603
1669
|
}
|
|
1604
|
-
|
|
1605
|
-
|
|
1670
|
+
return best;
|
|
1671
|
+
}
|
|
1672
|
+
function deriveLastError(runs) {
|
|
1673
|
+
const errorRuns = runs.filter((run) => run.status === "error").sort((a, b) => activityMs(b) - activityMs(a));
|
|
1674
|
+
const latest = errorRuns[0];
|
|
1675
|
+
if (!latest) return void 0;
|
|
1676
|
+
const meta = latest.metadata ?? {};
|
|
1677
|
+
const message = typeof meta.errorMessage === "string" && meta.errorMessage.trim() !== "" ? meta.errorMessage.trim() : latest.name ?? latest.runId;
|
|
1678
|
+
const code = typeof meta.errorCode === "string" && meta.errorCode.trim() !== "" ? meta.errorCode.trim() : void 0;
|
|
1679
|
+
return { runId: latest.runId, message, code };
|
|
1680
|
+
}
|
|
1681
|
+
function deriveCheckSummary(runs) {
|
|
1682
|
+
let pass = 0;
|
|
1683
|
+
let fail = 0;
|
|
1684
|
+
let warn2 = 0;
|
|
1685
|
+
let found = false;
|
|
1606
1686
|
for (const run of runs) {
|
|
1607
|
-
const
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1687
|
+
const summary = run.metadata?.checkSummary;
|
|
1688
|
+
if (!summary || typeof summary !== "object") continue;
|
|
1689
|
+
const record = summary;
|
|
1690
|
+
if (typeof record.pass === "number") {
|
|
1691
|
+
pass += record.pass;
|
|
1692
|
+
found = true;
|
|
1693
|
+
}
|
|
1694
|
+
if (typeof record.fail === "number") {
|
|
1695
|
+
fail += record.fail;
|
|
1696
|
+
found = true;
|
|
1697
|
+
}
|
|
1698
|
+
if (typeof record.warn === "number") {
|
|
1699
|
+
warn2 += record.warn;
|
|
1700
|
+
found = true;
|
|
1614
1701
|
}
|
|
1615
|
-
const bucket = sessionsByKey.get(key) ?? [];
|
|
1616
|
-
bucket.push(run);
|
|
1617
|
-
sessionsByKey.set(key, bucket);
|
|
1618
1702
|
}
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
const
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
if (confidences.has("explicit") && confidences.has("correlated")) {
|
|
1627
|
-
warnings.push({
|
|
1628
|
-
code: "mixed-confidence-group",
|
|
1629
|
-
message: "Session aggregates explicit and correlated handoff edges.",
|
|
1630
|
-
sessionId
|
|
1631
|
-
});
|
|
1703
|
+
return found ? { pass, fail, warn: warn2 } : void 0;
|
|
1704
|
+
}
|
|
1705
|
+
function deriveObservationSummary(runs) {
|
|
1706
|
+
for (const run of [...runs].sort((a, b) => activityMs(b) - activityMs(a))) {
|
|
1707
|
+
const value = run.metadata?.observationSummary;
|
|
1708
|
+
if (typeof value === "string" && value.trim() !== "") {
|
|
1709
|
+
return value.trim();
|
|
1632
1710
|
}
|
|
1633
|
-
return enrichSessionSummary(
|
|
1634
|
-
{
|
|
1635
|
-
sessionId,
|
|
1636
|
-
runIds,
|
|
1637
|
-
groups,
|
|
1638
|
-
handoffs,
|
|
1639
|
-
retries,
|
|
1640
|
-
criticalPath
|
|
1641
|
-
},
|
|
1642
|
-
runs,
|
|
1643
|
-
{
|
|
1644
|
-
nowMs: options.nowMs,
|
|
1645
|
-
staleThresholdMs: options.staleThresholdMs
|
|
1646
|
-
}
|
|
1647
|
-
);
|
|
1648
|
-
});
|
|
1649
|
-
if (sessions.length === 0 && runs.length > 0) {
|
|
1650
|
-
warnings.push({
|
|
1651
|
-
code: "missing-session-id",
|
|
1652
|
-
message: "No sessionId (or correlated groupId) found on input runs."
|
|
1653
|
-
});
|
|
1654
1711
|
}
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1712
|
+
return void 0;
|
|
1713
|
+
}
|
|
1714
|
+
function deriveSessionStatus(runs, options = {}) {
|
|
1715
|
+
if (runs.length === 0) return "unknown";
|
|
1716
|
+
if (runs.some((run) => run.status === "running")) return "running";
|
|
1717
|
+
const explicit = pickExplicitStatus(runs);
|
|
1718
|
+
if (explicit && explicit !== "running") return explicit;
|
|
1719
|
+
if (runs.some((run) => run.status === "error")) return "error";
|
|
1720
|
+
if (runs.every((run) => run.status === "success")) return "completed";
|
|
1721
|
+
const nowMs = options.nowMs ?? Date.now();
|
|
1722
|
+
const staleThresholdMs = options.staleThresholdMs ?? DEFAULT_STALE_THRESHOLD_MS;
|
|
1723
|
+
const lastMs = latestActivityMs(runs);
|
|
1724
|
+
if (lastMs > 0 && nowMs - lastMs > staleThresholdMs) return "stale";
|
|
1725
|
+
return "unknown";
|
|
1726
|
+
}
|
|
1727
|
+
function enrichSessionSummary(summary, runs, options = {}) {
|
|
1728
|
+
const sessionRuns = runs.filter((run) => summary.runIds.includes(run.runId)).sort((a, b) => a.runId.localeCompare(b.runId));
|
|
1729
|
+
const startedAt = earliestStart(sessionRuns);
|
|
1730
|
+
const endedAt = latestEndWhenAllEnded(sessionRuns);
|
|
1731
|
+
const durationMs = startedAt !== void 0 && endedAt !== void 0 ? endedAt - startedAt : void 0;
|
|
1732
|
+
let correlationId;
|
|
1733
|
+
let jobId;
|
|
1734
|
+
let workflowId;
|
|
1735
|
+
for (const run of sessionRuns) {
|
|
1736
|
+
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1737
|
+
if (!correlationId && meta?.correlationId) correlationId = meta.correlationId;
|
|
1738
|
+
if (!jobId && meta?.jobId) jobId = meta.jobId;
|
|
1739
|
+
if (!workflowId && meta?.workflowName) workflowId = meta.workflowName;
|
|
1740
|
+
else if (!workflowId && meta?.workflowStep) workflowId = meta.workflowStep;
|
|
1741
|
+
}
|
|
1742
|
+
const lastMs = latestActivityMs(sessionRuns);
|
|
1743
|
+
const lastActivity = lastMs > 0 ? new Date(lastMs).toISOString() : (/* @__PURE__ */ new Date(0)).toISOString();
|
|
1744
|
+
const retryCount2 = summary.retries.filter(
|
|
1745
|
+
(retry) => retry.retryOf !== void 0 || (retry.attempt ?? 0) > 1
|
|
1746
|
+
).length;
|
|
1660
1747
|
return {
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1748
|
+
...summary,
|
|
1749
|
+
status: deriveSessionStatus(sessionRuns, options),
|
|
1750
|
+
startedAt,
|
|
1751
|
+
endedAt,
|
|
1752
|
+
durationMs,
|
|
1753
|
+
correlationId,
|
|
1754
|
+
jobId,
|
|
1755
|
+
workflowId,
|
|
1756
|
+
lastError: deriveLastError(sessionRuns),
|
|
1757
|
+
lastActivity,
|
|
1758
|
+
retryCount: retryCount2,
|
|
1759
|
+
observationSummary: deriveObservationSummary(sessionRuns),
|
|
1760
|
+
checkSummary: deriveCheckSummary(sessionRuns)
|
|
1665
1761
|
};
|
|
1666
1762
|
}
|
|
1667
1763
|
|
|
1668
|
-
// packages/core/src/
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
},
|
|
1682
|
-
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
1683
|
-
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
1684
|
-
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
1685
|
-
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
1686
|
-
relatedCodes: ["ambiguous_format"]
|
|
1687
|
-
},
|
|
1688
|
-
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
1689
|
-
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
1690
|
-
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
1691
|
-
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
1692
|
-
},
|
|
1693
|
-
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
1694
|
-
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
1695
|
-
summary: "Multiple runs are available; select a run before executing checks.",
|
|
1696
|
-
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
1697
|
-
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
1698
|
-
},
|
|
1699
|
-
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
1700
|
-
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1701
|
-
summary: "A parentId equals its own event/step id.",
|
|
1702
|
-
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
1703
|
-
relatedCodes: [
|
|
1704
|
-
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
1705
|
-
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
1706
|
-
]
|
|
1707
|
-
},
|
|
1708
|
-
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
1709
|
-
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
1710
|
-
summary: "Trace contains a parentId cycle.",
|
|
1711
|
-
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
1712
|
-
relatedCodes: ["structure.cycle"]
|
|
1713
|
-
},
|
|
1714
|
-
AI_TRACE_CONTRACT_INPUT_INVALID: {
|
|
1715
|
-
code: "AI_TRACE_CONTRACT_INPUT_INVALID",
|
|
1716
|
-
summary: "evaluateTraceContract expects ({ read }, contract) or use evaluateTraceContractRead(read, contract).",
|
|
1717
|
-
remediation: "Pass TraceCheckInput as the first argument: evaluateTraceContract({ read }, contract).",
|
|
1718
|
-
relatedCodes: ["AI_CHECK_INVALID_ARGUMENTS"]
|
|
1719
|
-
},
|
|
1720
|
-
AI_INSPECT_RUN_UNSUPPORTED_OPTION: {
|
|
1721
|
-
code: "AI_INSPECT_RUN_UNSUPPORTED_OPTION",
|
|
1722
|
-
summary: 'inspectRun does not accept a "writer" option.',
|
|
1723
|
-
remediation: "Use createInspector({ writer }) then inspector.run(...). inspectRun persists via the global JSONL path (schema 0.1)."
|
|
1764
|
+
// packages/core/src/sessions/load.ts
|
|
1765
|
+
async function enrichSessionRunRecord(meta) {
|
|
1766
|
+
let metadata;
|
|
1767
|
+
try {
|
|
1768
|
+
const events = await readTraceEventsFromFile(meta.filePath);
|
|
1769
|
+
for (const event of events) {
|
|
1770
|
+
if (event.event !== "run_started") continue;
|
|
1771
|
+
if (event.metadata && typeof event.metadata === "object") {
|
|
1772
|
+
metadata = event.metadata;
|
|
1773
|
+
}
|
|
1774
|
+
break;
|
|
1775
|
+
}
|
|
1776
|
+
} catch {
|
|
1724
1777
|
}
|
|
1725
|
-
});
|
|
1726
|
-
function formatProgrammaticDiagnostic(code, detail) {
|
|
1727
|
-
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
1728
|
-
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
1729
|
-
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
1730
|
-
}
|
|
1731
|
-
|
|
1732
|
-
// packages/core/src/checks/logical-events.ts
|
|
1733
|
-
function isRecord5(value) {
|
|
1734
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1735
|
-
}
|
|
1736
|
-
function legacyEvent(event) {
|
|
1737
|
-
const value = event.attributes?.legacyEvent;
|
|
1738
|
-
return typeof value === "string" ? value : void 0;
|
|
1739
|
-
}
|
|
1740
|
-
function stepIdOf(event) {
|
|
1741
|
-
const value = event.attributes?.stepId;
|
|
1742
|
-
if (typeof value === "string" && value.trim() !== "") return value;
|
|
1743
|
-
return void 0;
|
|
1744
|
-
}
|
|
1745
|
-
function cloneEvent(event) {
|
|
1746
1778
|
return {
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1779
|
+
runId: meta.runId,
|
|
1780
|
+
name: meta.name,
|
|
1781
|
+
status: meta.status,
|
|
1782
|
+
startedAt: meta.startedAt,
|
|
1783
|
+
endedAt: meta.endedAt,
|
|
1784
|
+
durationMs: meta.durationMs,
|
|
1785
|
+
filePath: meta.filePath,
|
|
1786
|
+
metadata
|
|
1752
1787
|
};
|
|
1753
1788
|
}
|
|
1754
|
-
function
|
|
1755
|
-
const
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
};
|
|
1759
|
-
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
1760
|
-
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
1761
|
-
if (complete.attributes?.errorStack !== void 0) {
|
|
1762
|
-
merged.errorStack = complete.attributes.errorStack;
|
|
1789
|
+
async function loadSessionRunRecords(metas) {
|
|
1790
|
+
const out = [];
|
|
1791
|
+
for (const meta of metas) {
|
|
1792
|
+
out.push(await enrichSessionRunRecord(meta));
|
|
1763
1793
|
}
|
|
1764
|
-
return
|
|
1794
|
+
return out;
|
|
1765
1795
|
}
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1796
|
+
|
|
1797
|
+
// packages/core/src/sessions/index.ts
|
|
1798
|
+
function compareRuns(a, b) {
|
|
1799
|
+
const aStart = a.startedAt ?? 0;
|
|
1800
|
+
const bStart = b.startedAt ?? 0;
|
|
1801
|
+
if (aStart !== bStart) return aStart - bStart;
|
|
1802
|
+
return a.runId.localeCompare(b.runId);
|
|
1803
|
+
}
|
|
1804
|
+
function buildGroups(runIds, metaByRunId) {
|
|
1805
|
+
const byGroup = /* @__PURE__ */ new Map();
|
|
1806
|
+
for (const runId of runIds) {
|
|
1807
|
+
const groupId = metaByRunId.get(runId)?.groupId;
|
|
1808
|
+
if (!groupId) continue;
|
|
1809
|
+
const existing = byGroup.get(groupId);
|
|
1810
|
+
if (existing) {
|
|
1811
|
+
existing.runIds.push(runId);
|
|
1812
|
+
} else {
|
|
1813
|
+
byGroup.set(groupId, {
|
|
1814
|
+
groupId,
|
|
1815
|
+
parentGroupId: metaByRunId.get(runId)?.parentGroupId,
|
|
1816
|
+
runIds: [runId]
|
|
1817
|
+
});
|
|
1786
1818
|
}
|
|
1787
|
-
}
|
|
1819
|
+
}
|
|
1820
|
+
return [...byGroup.values()].map((group) => ({
|
|
1821
|
+
...group,
|
|
1822
|
+
runIds: [...group.runIds].sort((a, b) => a.localeCompare(b))
|
|
1823
|
+
}));
|
|
1788
1824
|
}
|
|
1789
|
-
function
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
...{}
|
|
1798
|
-
}
|
|
1825
|
+
function buildHandoffs(runIds, metaByRunId, warnings, sessionId) {
|
|
1826
|
+
const edges = [];
|
|
1827
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1828
|
+
const pushEdge = (edge) => {
|
|
1829
|
+
const key = `${edge.from}->${edge.to}:${edge.confidence}`;
|
|
1830
|
+
if (seen.has(key)) return;
|
|
1831
|
+
seen.add(key);
|
|
1832
|
+
edges.push(edge);
|
|
1799
1833
|
};
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
1812
|
-
const logical = [];
|
|
1813
|
-
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
1814
|
-
for (const runId of orderedRuns) {
|
|
1815
|
-
const runEvents = byRun.get(runId) ?? [];
|
|
1816
|
-
const starts = [];
|
|
1817
|
-
const completes = [];
|
|
1818
|
-
const others = [];
|
|
1819
|
-
for (const event of runEvents) {
|
|
1820
|
-
const legacy = legacyEvent(event);
|
|
1821
|
-
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
1822
|
-
starts.push(event);
|
|
1823
|
-
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
1824
|
-
completes.push(event);
|
|
1825
|
-
} else {
|
|
1826
|
-
others.push(event);
|
|
1827
|
-
}
|
|
1834
|
+
for (const runId of runIds) {
|
|
1835
|
+
const meta = metaByRunId.get(runId);
|
|
1836
|
+
if (!meta) continue;
|
|
1837
|
+
if (meta.handoffFrom && meta.handoffTo) {
|
|
1838
|
+
pushEdge({
|
|
1839
|
+
from: meta.handoffFrom,
|
|
1840
|
+
to: meta.handoffTo,
|
|
1841
|
+
source: "manual",
|
|
1842
|
+
confidence: "explicit"
|
|
1843
|
+
});
|
|
1844
|
+
continue;
|
|
1828
1845
|
}
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
);
|
|
1837
|
-
if (candidates.length > 1) {
|
|
1838
|
-
diagnostics.push({
|
|
1839
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1840
|
-
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
1841
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
1842
|
-
});
|
|
1843
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1844
|
-
}
|
|
1845
|
-
match = candidates[0];
|
|
1846
|
-
} else if (stepId) {
|
|
1847
|
-
const candidates = completes.filter(
|
|
1848
|
-
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
1849
|
-
);
|
|
1850
|
-
if (candidates.length > 1) {
|
|
1851
|
-
diagnostics.push({
|
|
1852
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1853
|
-
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
1854
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
1855
|
-
});
|
|
1856
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1857
|
-
}
|
|
1858
|
-
match = candidates[0];
|
|
1859
|
-
}
|
|
1860
|
-
if (match) {
|
|
1861
|
-
usedCompletes.add(match.eventId);
|
|
1862
|
-
absorbedIds.add(match.eventId);
|
|
1863
|
-
const paired = pairStartComplete(start, match);
|
|
1864
|
-
logical.push(paired);
|
|
1865
|
-
logicalByRawId.set(start.eventId, paired);
|
|
1866
|
-
logicalByRawId.set(match.eventId, paired);
|
|
1867
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
1868
|
-
} else {
|
|
1869
|
-
diagnostics.push({
|
|
1870
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
1871
|
-
message: `No matching complete for start ${start.eventId}.`,
|
|
1872
|
-
eventIds: [start.eventId]
|
|
1873
|
-
});
|
|
1874
|
-
const alone = asLogical(start);
|
|
1875
|
-
logical.push(alone);
|
|
1876
|
-
logicalByRawId.set(start.eventId, alone);
|
|
1877
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1878
|
-
}
|
|
1846
|
+
if (meta.handoffFrom) {
|
|
1847
|
+
pushEdge({
|
|
1848
|
+
from: meta.handoffFrom,
|
|
1849
|
+
to: runId,
|
|
1850
|
+
source: "manual",
|
|
1851
|
+
confidence: "explicit"
|
|
1852
|
+
});
|
|
1879
1853
|
}
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1854
|
+
if (meta.handoffTo) {
|
|
1855
|
+
pushEdge({
|
|
1856
|
+
from: runId,
|
|
1857
|
+
to: meta.handoffTo,
|
|
1858
|
+
source: "manual",
|
|
1859
|
+
confidence: "explicit"
|
|
1886
1860
|
});
|
|
1887
|
-
const alone = asLogical(complete);
|
|
1888
|
-
logical.push(alone);
|
|
1889
|
-
logicalByRawId.set(complete.eventId, alone);
|
|
1890
|
-
const stepId = stepIdOf(complete);
|
|
1891
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1892
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1893
|
-
}
|
|
1894
1861
|
}
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1862
|
+
if (meta.subAgentId && meta.parentGroupId && !meta.handoffFrom && !meta.handoffTo) {
|
|
1863
|
+
pushEdge({
|
|
1864
|
+
from: meta.parentGroupId,
|
|
1865
|
+
to: meta.subAgentId,
|
|
1866
|
+
source: "inferred",
|
|
1867
|
+
confidence: "correlated"
|
|
1868
|
+
});
|
|
1869
|
+
warnings.push({
|
|
1870
|
+
code: "ambiguous-handoff-endpoints",
|
|
1871
|
+
message: "Handoff inferred from parentGroupId and subAgentId without explicit handoffFrom/handoffTo.",
|
|
1872
|
+
runId,
|
|
1873
|
+
sessionId
|
|
1874
|
+
});
|
|
1903
1875
|
}
|
|
1904
1876
|
}
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1877
|
+
return edges.sort((a, b) => {
|
|
1878
|
+
const from = a.from.localeCompare(b.from);
|
|
1879
|
+
if (from !== 0) return from;
|
|
1880
|
+
return a.to.localeCompare(b.to);
|
|
1881
|
+
});
|
|
1882
|
+
}
|
|
1883
|
+
function buildRetries(runIds, metaByRunId, warnings, sessionId) {
|
|
1884
|
+
const retries = [];
|
|
1885
|
+
for (const runId of runIds) {
|
|
1886
|
+
const meta = metaByRunId.get(runId);
|
|
1887
|
+
if (!meta) continue;
|
|
1888
|
+
if (meta.retryOf) {
|
|
1889
|
+
retries.push({
|
|
1890
|
+
runId,
|
|
1891
|
+
retryOf: meta.retryOf,
|
|
1892
|
+
attempt: meta.attempt,
|
|
1893
|
+
source: "manual",
|
|
1894
|
+
confidence: "explicit"
|
|
1895
|
+
});
|
|
1911
1896
|
continue;
|
|
1912
1897
|
}
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1898
|
+
if (meta.attempt !== void 0 && meta.attempt > 1) {
|
|
1899
|
+
retries.push({
|
|
1900
|
+
runId,
|
|
1901
|
+
attempt: meta.attempt,
|
|
1902
|
+
source: "inferred",
|
|
1903
|
+
confidence: "correlated"
|
|
1904
|
+
});
|
|
1905
|
+
warnings.push({
|
|
1906
|
+
code: "ambiguous-retry-link",
|
|
1907
|
+
message: "attempt > 1 without retryOf; retry link is correlated only.",
|
|
1908
|
+
runId,
|
|
1909
|
+
sessionId
|
|
1910
|
+
});
|
|
1925
1911
|
}
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1912
|
+
}
|
|
1913
|
+
return retries.sort((a, b) => a.runId.localeCompare(b.runId));
|
|
1914
|
+
}
|
|
1915
|
+
function buildCriticalPath(runs, handoffs) {
|
|
1916
|
+
const runById = new Map(runs.map((run) => [run.runId, run]));
|
|
1917
|
+
const explicitTargets = new Set(
|
|
1918
|
+
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.to)
|
|
1919
|
+
);
|
|
1920
|
+
const explicitSources = new Set(
|
|
1921
|
+
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
1922
|
+
);
|
|
1923
|
+
const ordered = [...runs].sort(compareRuns);
|
|
1924
|
+
const path18 = [];
|
|
1925
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1926
|
+
const pushRun = (run, confidence, source) => {
|
|
1927
|
+
if (visited.has(run.runId)) return;
|
|
1928
|
+
visited.add(run.runId);
|
|
1929
|
+
path18.push({
|
|
1930
|
+
runId: run.runId,
|
|
1931
|
+
name: run.name,
|
|
1932
|
+
startedAt: run.startedAt,
|
|
1933
|
+
durationMs: run.durationMs,
|
|
1934
|
+
confidence,
|
|
1935
|
+
source
|
|
1936
|
+
});
|
|
1937
|
+
};
|
|
1938
|
+
for (const edge of handoffs) {
|
|
1939
|
+
if (edge.confidence !== "explicit") continue;
|
|
1940
|
+
const fromRun = [...runById.values()].find(
|
|
1941
|
+
(run) => run.runId === edge.from || metaRunIdMatches(run, edge.from, runById)
|
|
1942
|
+
);
|
|
1943
|
+
const toRun = [...runById.values()].find(
|
|
1944
|
+
(run) => run.runId === edge.to || metaRunIdMatches(run, edge.to, runById)
|
|
1945
|
+
);
|
|
1946
|
+
if (fromRun) pushRun(fromRun, "explicit", "manual");
|
|
1947
|
+
if (toRun) pushRun(toRun, "explicit", "manual");
|
|
1948
|
+
}
|
|
1949
|
+
for (const run of ordered) {
|
|
1950
|
+
if (visited.has(run.runId)) continue;
|
|
1951
|
+
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
1952
|
+
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
1953
|
+
}
|
|
1954
|
+
return path18;
|
|
1955
|
+
}
|
|
1956
|
+
function metaRunIdMatches(run, token, runById) {
|
|
1957
|
+
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1958
|
+
return meta?.subAgentId === token || meta?.groupId === token || runById.has(token);
|
|
1959
|
+
}
|
|
1960
|
+
function buildSessionIndex(inputRuns, options = {}) {
|
|
1961
|
+
const warnings = [];
|
|
1962
|
+
const runs = [...inputRuns].sort(compareRuns);
|
|
1963
|
+
const metaByRunId = /* @__PURE__ */ new Map();
|
|
1964
|
+
for (const run of runs) {
|
|
1965
|
+
metaByRunId.set(run.runId, extractSessionWorkflowMetadata(run.metadata));
|
|
1966
|
+
}
|
|
1967
|
+
const sessionsByKey = /* @__PURE__ */ new Map();
|
|
1968
|
+
const unscopedRunIds = [];
|
|
1969
|
+
for (const run of runs) {
|
|
1970
|
+
const meta = metaByRunId.get(run.runId);
|
|
1971
|
+
const key = sessionKeyForRun(meta, {
|
|
1972
|
+
correlateByGroupId: options.correlateByGroupId === true
|
|
1973
|
+
});
|
|
1974
|
+
if (!key) {
|
|
1975
|
+
unscopedRunIds.push(run.runId);
|
|
1960
1976
|
continue;
|
|
1961
1977
|
}
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
}
|
|
1978
|
+
const bucket = sessionsByKey.get(key) ?? [];
|
|
1979
|
+
bucket.push(run);
|
|
1980
|
+
sessionsByKey.set(key, bucket);
|
|
1981
|
+
}
|
|
1982
|
+
const sessions = [...sessionsByKey.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([sessionId, sessionRuns]) => {
|
|
1983
|
+
const runIds = sessionRuns.map((run) => run.runId).sort();
|
|
1984
|
+
const handoffs = buildHandoffs(runIds, metaByRunId, warnings, sessionId);
|
|
1985
|
+
const retries = buildRetries(runIds, metaByRunId, warnings, sessionId);
|
|
1986
|
+
const groups = buildGroups(runIds, metaByRunId);
|
|
1987
|
+
const criticalPath = buildCriticalPath(sessionRuns, handoffs);
|
|
1988
|
+
const confidences = new Set(handoffs.map((edge) => edge.confidence));
|
|
1989
|
+
if (confidences.has("explicit") && confidences.has("correlated")) {
|
|
1990
|
+
warnings.push({
|
|
1991
|
+
code: "mixed-confidence-group",
|
|
1992
|
+
message: "Session aggregates explicit and correlated handoff edges.",
|
|
1993
|
+
sessionId
|
|
1979
1994
|
});
|
|
1980
|
-
continue;
|
|
1981
1995
|
}
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1996
|
+
return enrichSessionSummary(
|
|
1997
|
+
{
|
|
1998
|
+
sessionId,
|
|
1999
|
+
runIds,
|
|
2000
|
+
groups,
|
|
2001
|
+
handoffs,
|
|
2002
|
+
retries,
|
|
2003
|
+
criticalPath
|
|
2004
|
+
},
|
|
2005
|
+
runs,
|
|
2006
|
+
{
|
|
2007
|
+
nowMs: options.nowMs,
|
|
2008
|
+
staleThresholdMs: options.staleThresholdMs
|
|
1994
2009
|
}
|
|
2010
|
+
);
|
|
2011
|
+
});
|
|
2012
|
+
if (sessions.length === 0 && runs.length > 0) {
|
|
2013
|
+
warnings.push({
|
|
2014
|
+
code: "missing-session-id",
|
|
2015
|
+
message: "No sessionId (or correlated groupId) found on input runs."
|
|
1995
2016
|
});
|
|
1996
2017
|
}
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
2018
|
+
warnings.sort((a, b) => {
|
|
2019
|
+
const code = a.code.localeCompare(b.code);
|
|
2020
|
+
if (code !== 0) return code;
|
|
2021
|
+
return (a.runId ?? "").localeCompare(b.runId ?? "");
|
|
2002
2022
|
});
|
|
2003
2023
|
return {
|
|
2004
|
-
|
|
2005
|
-
|
|
2024
|
+
runs,
|
|
2025
|
+
sessions,
|
|
2026
|
+
unscopedRunIds: unscopedRunIds.sort(),
|
|
2027
|
+
warnings
|
|
2006
2028
|
};
|
|
2007
2029
|
}
|
|
2008
|
-
function resolveCanonicalToolName(event) {
|
|
2009
|
-
const attrs = event.attributes;
|
|
2010
|
-
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
2011
|
-
if (direct) return direct;
|
|
2012
|
-
const metadata = attrs?.metadata;
|
|
2013
|
-
if (isRecord5(metadata)) {
|
|
2014
|
-
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
2015
|
-
if (nested) return nested;
|
|
2016
|
-
}
|
|
2017
|
-
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
2018
|
-
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
2019
|
-
}
|
|
2020
|
-
return event.name;
|
|
2021
|
-
}
|
|
2022
|
-
function pickString(record, keys) {
|
|
2023
|
-
if (!record) return void 0;
|
|
2024
|
-
for (const key of keys) {
|
|
2025
|
-
const value = record[key];
|
|
2026
|
-
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
2027
|
-
}
|
|
2028
|
-
return void 0;
|
|
2029
|
-
}
|
|
2030
2030
|
|
|
2031
2031
|
// packages/core/src/checks/trace-facts.ts
|
|
2032
2032
|
formatProgrammaticDiagnostic(
|
|
@@ -2061,7 +2061,11 @@ function emptySummary() {
|
|
|
2061
2061
|
failed: 0,
|
|
2062
2062
|
warnings: 0,
|
|
2063
2063
|
errors: 0,
|
|
2064
|
-
rulesEvaluated: 0
|
|
2064
|
+
rulesEvaluated: 0,
|
|
2065
|
+
rulesPassed: 0,
|
|
2066
|
+
rulesWarning: 0,
|
|
2067
|
+
rulesFailed: 0,
|
|
2068
|
+
rulesError: 0
|
|
2065
2069
|
};
|
|
2066
2070
|
}
|
|
2067
2071
|
function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
@@ -2073,7 +2077,11 @@ function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
|
2073
2077
|
summary: {
|
|
2074
2078
|
...emptySummary(),
|
|
2075
2079
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2076
|
-
rulesEvaluated: ruleExecutions.length
|
|
2080
|
+
rulesEvaluated: ruleExecutions.length,
|
|
2081
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
2082
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
2083
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
2084
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
2077
2085
|
},
|
|
2078
2086
|
findings: [],
|
|
2079
2087
|
diagnostics: [...diagnostics],
|
|
@@ -2233,7 +2241,7 @@ function normalizeFinding(rule, finding) {
|
|
|
2233
2241
|
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
2234
2242
|
};
|
|
2235
2243
|
}
|
|
2236
|
-
function summarize(findings, diagnostics,
|
|
2244
|
+
function summarize(findings, diagnostics, ruleExecutions) {
|
|
2237
2245
|
return {
|
|
2238
2246
|
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
2239
2247
|
failed: findings.filter(
|
|
@@ -2243,7 +2251,11 @@ function summarize(findings, diagnostics, rulesEvaluated) {
|
|
|
2243
2251
|
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
2244
2252
|
).length,
|
|
2245
2253
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2246
|
-
rulesEvaluated
|
|
2254
|
+
rulesEvaluated: ruleExecutions.length,
|
|
2255
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
2256
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
2257
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
2258
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
2247
2259
|
};
|
|
2248
2260
|
}
|
|
2249
2261
|
function classifyRuleExecution(findings, threw) {
|
|
@@ -2720,7 +2732,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
2720
2732
|
}
|
|
2721
2733
|
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
2722
2734
|
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
2723
|
-
const summary = summarize(sortedFindings, diagnostics, ruleExecutions
|
|
2735
|
+
const summary = summarize(sortedFindings, diagnostics, ruleExecutions);
|
|
2724
2736
|
const status = summary.failed > 0 ? "fail" : "pass";
|
|
2725
2737
|
return {
|
|
2726
2738
|
ok: status === "pass",
|
|
@@ -3155,10 +3167,143 @@ function asStringArray(value, label) {
|
|
|
3155
3167
|
function asPositiveNumber(value, label) {
|
|
3156
3168
|
if (value === void 0) return void 0;
|
|
3157
3169
|
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
3158
|
-
throw new Error(`${label} must be a non-negative number.`);
|
|
3170
|
+
throw new Error(`${label} must be a non-negative finite number.`);
|
|
3171
|
+
}
|
|
3172
|
+
return value;
|
|
3173
|
+
}
|
|
3174
|
+
function asBoolean(value, label) {
|
|
3175
|
+
if (value === void 0) return void 0;
|
|
3176
|
+
if (typeof value !== "boolean") {
|
|
3177
|
+
throw new Error(`${label} must be a boolean (not a string or other type).`);
|
|
3178
|
+
}
|
|
3179
|
+
return value;
|
|
3180
|
+
}
|
|
3181
|
+
function assertObject(value, label) {
|
|
3182
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
3183
|
+
throw new Error(`${label} must be an object.`);
|
|
3159
3184
|
}
|
|
3160
3185
|
return value;
|
|
3161
3186
|
}
|
|
3187
|
+
function assertKnownKeys(raw, allowed, label) {
|
|
3188
|
+
const allowedSet = new Set(allowed);
|
|
3189
|
+
for (const key of Object.keys(raw)) {
|
|
3190
|
+
if (!allowedSet.has(key)) {
|
|
3191
|
+
throw new Error(`${label} has unknown key "${key}".`);
|
|
3192
|
+
}
|
|
3193
|
+
}
|
|
3194
|
+
}
|
|
3195
|
+
var CASE_KEYS = [
|
|
3196
|
+
"id",
|
|
3197
|
+
"trace",
|
|
3198
|
+
"runId",
|
|
3199
|
+
"input",
|
|
3200
|
+
"requireTools",
|
|
3201
|
+
"forbidTools",
|
|
3202
|
+
"maxDurationMs",
|
|
3203
|
+
"expectedObservations",
|
|
3204
|
+
"expect"
|
|
3205
|
+
];
|
|
3206
|
+
var EXPECT_KEYS = [
|
|
3207
|
+
"checkStatus",
|
|
3208
|
+
"findingRuleIds",
|
|
3209
|
+
"rejectUnexpectedFindings",
|
|
3210
|
+
"minAssertions"
|
|
3211
|
+
];
|
|
3212
|
+
var CHECKS_KEYS = ["select", "run", "tool", "llm"];
|
|
3213
|
+
var CHECKS_RUN_KEYS = ["maxDurationMs", "maxDepth"];
|
|
3214
|
+
var CHECKS_TOOL_KEYS = ["required", "forbidden"];
|
|
3215
|
+
var CHECKS_LLM_KEYS = ["allowedModels", "maxTotalTokens"];
|
|
3216
|
+
var EVAL_KEYS = [
|
|
3217
|
+
"requireSuccess",
|
|
3218
|
+
"requiredTools",
|
|
3219
|
+
"forbiddenTools",
|
|
3220
|
+
"maxDurationMs",
|
|
3221
|
+
"maxDepth",
|
|
3222
|
+
"maxRetries",
|
|
3223
|
+
"maxTotalTokens"
|
|
3224
|
+
];
|
|
3225
|
+
var ROOT_KEYS = [
|
|
3226
|
+
"name",
|
|
3227
|
+
"traces",
|
|
3228
|
+
"cases",
|
|
3229
|
+
"checks",
|
|
3230
|
+
"eval",
|
|
3231
|
+
"redactionProfile",
|
|
3232
|
+
"artifacts",
|
|
3233
|
+
"baseline",
|
|
3234
|
+
"candidate"
|
|
3235
|
+
];
|
|
3236
|
+
function asExpectConfig(value, label) {
|
|
3237
|
+
if (value === void 0) return void 0;
|
|
3238
|
+
const raw = assertObject(value, label);
|
|
3239
|
+
assertKnownKeys(raw, EXPECT_KEYS, label);
|
|
3240
|
+
const checkStatus = raw.checkStatus;
|
|
3241
|
+
if (checkStatus !== "pass" && checkStatus !== "fail" && checkStatus !== "error") {
|
|
3242
|
+
throw new Error(`${label}.checkStatus must be "pass", "fail", or "error".`);
|
|
3243
|
+
}
|
|
3244
|
+
const findingRuleIds = asStringArray(raw.findingRuleIds, `${label}.findingRuleIds`);
|
|
3245
|
+
const minAssertions = asPositiveNumber(raw.minAssertions, `${label}.minAssertions`);
|
|
3246
|
+
const rejectUnexpectedFindings = asBoolean(
|
|
3247
|
+
raw.rejectUnexpectedFindings,
|
|
3248
|
+
`${label}.rejectUnexpectedFindings`
|
|
3249
|
+
);
|
|
3250
|
+
return {
|
|
3251
|
+
checkStatus,
|
|
3252
|
+
...findingRuleIds !== void 0 ? { findingRuleIds } : {},
|
|
3253
|
+
...rejectUnexpectedFindings !== void 0 ? { rejectUnexpectedFindings } : {},
|
|
3254
|
+
...minAssertions !== void 0 ? { minAssertions } : {}
|
|
3255
|
+
};
|
|
3256
|
+
}
|
|
3257
|
+
function asChecksConfig(value, label) {
|
|
3258
|
+
const raw = assertObject(value, label);
|
|
3259
|
+
assertKnownKeys(raw, CHECKS_KEYS, label);
|
|
3260
|
+
const checks = {};
|
|
3261
|
+
const select = asStringArray(raw.select, `${label}.select`);
|
|
3262
|
+
if (select !== void 0) checks.select = select;
|
|
3263
|
+
if (raw.run !== void 0) {
|
|
3264
|
+
const run = assertObject(raw.run, `${label}.run`);
|
|
3265
|
+
assertKnownKeys(run, CHECKS_RUN_KEYS, `${label}.run`);
|
|
3266
|
+
checks.run = {
|
|
3267
|
+
...asPositiveNumber(run.maxDurationMs, `${label}.run.maxDurationMs`) !== void 0 ? { maxDurationMs: asPositiveNumber(run.maxDurationMs, `${label}.run.maxDurationMs`) } : {},
|
|
3268
|
+
...asPositiveNumber(run.maxDepth, `${label}.run.maxDepth`) !== void 0 ? { maxDepth: asPositiveNumber(run.maxDepth, `${label}.run.maxDepth`) } : {}
|
|
3269
|
+
};
|
|
3270
|
+
}
|
|
3271
|
+
if (raw.tool !== void 0) {
|
|
3272
|
+
const tool = assertObject(raw.tool, `${label}.tool`);
|
|
3273
|
+
assertKnownKeys(tool, CHECKS_TOOL_KEYS, `${label}.tool`);
|
|
3274
|
+
checks.tool = {
|
|
3275
|
+
...asStringArray(tool.required, `${label}.tool.required`) !== void 0 ? { required: asStringArray(tool.required, `${label}.tool.required`) } : {},
|
|
3276
|
+
...asStringArray(tool.forbidden, `${label}.tool.forbidden`) !== void 0 ? { forbidden: asStringArray(tool.forbidden, `${label}.tool.forbidden`) } : {}
|
|
3277
|
+
};
|
|
3278
|
+
}
|
|
3279
|
+
if (raw.llm !== void 0) {
|
|
3280
|
+
const llm = assertObject(raw.llm, `${label}.llm`);
|
|
3281
|
+
assertKnownKeys(llm, CHECKS_LLM_KEYS, `${label}.llm`);
|
|
3282
|
+
checks.llm = {
|
|
3283
|
+
...asStringArray(llm.allowedModels, `${label}.llm.allowedModels`) !== void 0 ? { allowedModels: asStringArray(llm.allowedModels, `${label}.llm.allowedModels`) } : {},
|
|
3284
|
+
...asPositiveNumber(llm.maxTotalTokens, `${label}.llm.maxTotalTokens`) !== void 0 ? {
|
|
3285
|
+
maxTotalTokens: asPositiveNumber(
|
|
3286
|
+
llm.maxTotalTokens,
|
|
3287
|
+
`${label}.llm.maxTotalTokens`
|
|
3288
|
+
)
|
|
3289
|
+
} : {}
|
|
3290
|
+
};
|
|
3291
|
+
}
|
|
3292
|
+
return checks;
|
|
3293
|
+
}
|
|
3294
|
+
function asEvalConfig(value, label) {
|
|
3295
|
+
const raw = assertObject(value, label);
|
|
3296
|
+
assertKnownKeys(raw, EVAL_KEYS, label);
|
|
3297
|
+
return {
|
|
3298
|
+
...asBoolean(raw.requireSuccess, `${label}.requireSuccess`) !== void 0 ? { requireSuccess: asBoolean(raw.requireSuccess, `${label}.requireSuccess`) } : {},
|
|
3299
|
+
...asStringArray(raw.requiredTools, `${label}.requiredTools`) !== void 0 ? { requiredTools: asStringArray(raw.requiredTools, `${label}.requiredTools`) } : {},
|
|
3300
|
+
...asStringArray(raw.forbiddenTools, `${label}.forbiddenTools`) !== void 0 ? { forbiddenTools: asStringArray(raw.forbiddenTools, `${label}.forbiddenTools`) } : {},
|
|
3301
|
+
...asPositiveNumber(raw.maxDurationMs, `${label}.maxDurationMs`) !== void 0 ? { maxDurationMs: asPositiveNumber(raw.maxDurationMs, `${label}.maxDurationMs`) } : {},
|
|
3302
|
+
...asPositiveNumber(raw.maxDepth, `${label}.maxDepth`) !== void 0 ? { maxDepth: asPositiveNumber(raw.maxDepth, `${label}.maxDepth`) } : {},
|
|
3303
|
+
...asPositiveNumber(raw.maxRetries, `${label}.maxRetries`) !== void 0 ? { maxRetries: asPositiveNumber(raw.maxRetries, `${label}.maxRetries`) } : {},
|
|
3304
|
+
...asPositiveNumber(raw.maxTotalTokens, `${label}.maxTotalTokens`) !== void 0 ? { maxTotalTokens: asPositiveNumber(raw.maxTotalTokens, `${label}.maxTotalTokens`) } : {}
|
|
3305
|
+
};
|
|
3306
|
+
}
|
|
3162
3307
|
function validateCaseConfig(value, index) {
|
|
3163
3308
|
const diagnostics = [];
|
|
3164
3309
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
@@ -3169,6 +3314,7 @@ function validateCaseConfig(value, index) {
|
|
|
3169
3314
|
}
|
|
3170
3315
|
const raw = value;
|
|
3171
3316
|
try {
|
|
3317
|
+
assertKnownKeys(raw, CASE_KEYS, `cases[${index}]`);
|
|
3172
3318
|
const id = asString(raw.id, `cases[${index}].id`);
|
|
3173
3319
|
if (id === void 0) {
|
|
3174
3320
|
diagnostics.push(
|
|
@@ -3201,7 +3347,8 @@ function validateCaseConfig(value, index) {
|
|
|
3201
3347
|
raw.expectedObservations,
|
|
3202
3348
|
`cases[${index}].expectedObservations`
|
|
3203
3349
|
)
|
|
3204
|
-
} : {}
|
|
3350
|
+
} : {},
|
|
3351
|
+
...asExpectConfig(raw.expect, `cases[${index}].expect`) !== void 0 ? { expect: asExpectConfig(raw.expect, `cases[${index}].expect`) } : {}
|
|
3205
3352
|
},
|
|
3206
3353
|
diagnostics
|
|
3207
3354
|
};
|
|
@@ -3216,6 +3363,7 @@ function normalizeSuiteConfig(value) {
|
|
|
3216
3363
|
throw new Error("Suite config must export an object.");
|
|
3217
3364
|
}
|
|
3218
3365
|
const raw = value;
|
|
3366
|
+
assertKnownKeys(raw, ROOT_KEYS, "suite config");
|
|
3219
3367
|
const name = asString(raw.name, "name");
|
|
3220
3368
|
const traces = asString(raw.traces, "traces");
|
|
3221
3369
|
if (name === void 0) throw new Error("name is required.");
|
|
@@ -3244,25 +3392,15 @@ function normalizeSuiteConfig(value) {
|
|
|
3244
3392
|
}
|
|
3245
3393
|
const config = { name, traces, cases };
|
|
3246
3394
|
if (raw.checks !== void 0) {
|
|
3247
|
-
|
|
3248
|
-
throw new Error("checks must be an object.");
|
|
3249
|
-
}
|
|
3250
|
-
config.checks = raw.checks;
|
|
3395
|
+
config.checks = asChecksConfig(raw.checks, "checks");
|
|
3251
3396
|
}
|
|
3252
3397
|
if (raw.eval !== void 0) {
|
|
3253
|
-
|
|
3254
|
-
throw new Error("eval must be an object.");
|
|
3255
|
-
}
|
|
3256
|
-
config.eval = raw.eval;
|
|
3398
|
+
config.eval = asEvalConfig(raw.eval, "eval");
|
|
3257
3399
|
}
|
|
3258
3400
|
if (raw.artifacts !== void 0) {
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
const outputDir = asString(
|
|
3263
|
-
raw.artifacts.outputDir,
|
|
3264
|
-
"artifacts.outputDir"
|
|
3265
|
-
);
|
|
3401
|
+
const artifacts = assertObject(raw.artifacts, "artifacts");
|
|
3402
|
+
assertKnownKeys(artifacts, ["outputDir"], "artifacts");
|
|
3403
|
+
const outputDir = asString(artifacts.outputDir, "artifacts.outputDir");
|
|
3266
3404
|
config.artifacts = outputDir !== void 0 ? { outputDir } : {};
|
|
3267
3405
|
}
|
|
3268
3406
|
if (raw.baseline !== void 0) {
|
|
@@ -5787,6 +5925,78 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
5787
5925
|
}
|
|
5788
5926
|
return { ok: diagnostics.length === 0, diagnostics };
|
|
5789
5927
|
}
|
|
5928
|
+
function evaluateCaseExpect(suiteCase, checkResult, observationCount) {
|
|
5929
|
+
const expectation = suiteCase.expect;
|
|
5930
|
+
if (expectation === void 0) {
|
|
5931
|
+
return { ok: true, diagnostics: [] };
|
|
5932
|
+
}
|
|
5933
|
+
const diagnostics = [];
|
|
5934
|
+
if (checkResult.status !== expectation.checkStatus) {
|
|
5935
|
+
diagnostics.push(
|
|
5936
|
+
diagnostic4(
|
|
5937
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5938
|
+
`Expected check status "${expectation.checkStatus}", got "${checkResult.status}".`,
|
|
5939
|
+
"error",
|
|
5940
|
+
suiteCase.id
|
|
5941
|
+
)
|
|
5942
|
+
);
|
|
5943
|
+
}
|
|
5944
|
+
const failedFindings = checkResult.findings.filter(
|
|
5945
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
5946
|
+
);
|
|
5947
|
+
const requiredIds = expectation.findingRuleIds ?? [];
|
|
5948
|
+
for (const ruleId of requiredIds) {
|
|
5949
|
+
if (!failedFindings.some((finding) => finding.ruleId === ruleId)) {
|
|
5950
|
+
diagnostics.push(
|
|
5951
|
+
diagnostic4(
|
|
5952
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5953
|
+
`Expected failing finding for rule "${ruleId}" was not present.`,
|
|
5954
|
+
"error",
|
|
5955
|
+
suiteCase.id
|
|
5956
|
+
)
|
|
5957
|
+
);
|
|
5958
|
+
}
|
|
5959
|
+
}
|
|
5960
|
+
const rejectUnexpected = expectation.rejectUnexpectedFindings !== false;
|
|
5961
|
+
if (rejectUnexpected && requiredIds.length > 0) {
|
|
5962
|
+
for (const finding of failedFindings) {
|
|
5963
|
+
if (!requiredIds.includes(finding.ruleId)) {
|
|
5964
|
+
diagnostics.push(
|
|
5965
|
+
diagnostic4(
|
|
5966
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5967
|
+
`Unexpected failing finding for rule "${finding.ruleId}".`,
|
|
5968
|
+
"error",
|
|
5969
|
+
suiteCase.id
|
|
5970
|
+
)
|
|
5971
|
+
);
|
|
5972
|
+
}
|
|
5973
|
+
}
|
|
5974
|
+
}
|
|
5975
|
+
if (expectation.minAssertions !== void 0) {
|
|
5976
|
+
const assertionCount = checkResult.ruleExecutions.length + observationCount;
|
|
5977
|
+
if (assertionCount < expectation.minAssertions) {
|
|
5978
|
+
diagnostics.push(
|
|
5979
|
+
diagnostic4(
|
|
5980
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5981
|
+
`Expected at least ${expectation.minAssertions} assertion(s), got ${assertionCount}.`,
|
|
5982
|
+
"error",
|
|
5983
|
+
suiteCase.id
|
|
5984
|
+
)
|
|
5985
|
+
);
|
|
5986
|
+
}
|
|
5987
|
+
}
|
|
5988
|
+
if (expectation.checkStatus === "fail" && checkResult.diagnostics.some((item) => item.severity === "error") && checkResult.status === "error") {
|
|
5989
|
+
diagnostics.push(
|
|
5990
|
+
diagnostic4(
|
|
5991
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5992
|
+
'Check ended in evaluator/config error; semantic failure expectation requires status "fail".',
|
|
5993
|
+
"error",
|
|
5994
|
+
suiteCase.id
|
|
5995
|
+
)
|
|
5996
|
+
);
|
|
5997
|
+
}
|
|
5998
|
+
return { ok: diagnostics.length === 0, diagnostics };
|
|
5999
|
+
}
|
|
5790
6000
|
async function runSuiteCase(suiteCase, config, options) {
|
|
5791
6001
|
const resolved = await resolveSuiteCaseTrace(suiteCase, options);
|
|
5792
6002
|
if (resolved.missing || resolved.tracePath === void 0) {
|
|
@@ -5839,9 +6049,21 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
5839
6049
|
status: "pass",
|
|
5840
6050
|
format: read.format,
|
|
5841
6051
|
findings: [],
|
|
5842
|
-
diagnostics: []
|
|
6052
|
+
diagnostics: [],
|
|
6053
|
+
ruleExecutions: []
|
|
6054
|
+
};
|
|
5843
6055
|
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
5844
|
-
const
|
|
6056
|
+
const checkOk = checkResult.ok;
|
|
6057
|
+
const observationsOk = observationResult.ok;
|
|
6058
|
+
const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
|
|
6059
|
+
(finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
|
|
6060
|
+
) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
|
|
6061
|
+
const expectResult = evaluateCaseExpect(
|
|
6062
|
+
suiteCase,
|
|
6063
|
+
checkResult,
|
|
6064
|
+
compiled.observationCount
|
|
6065
|
+
);
|
|
6066
|
+
const checkFailureDiagnostics = [
|
|
5845
6067
|
...checkResult.diagnostics.map(
|
|
5846
6068
|
(item) => diagnostic4(
|
|
5847
6069
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
@@ -5857,15 +6079,24 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
5857
6079
|
finding.severity,
|
|
5858
6080
|
suiteCase.id
|
|
5859
6081
|
)
|
|
5860
|
-
)
|
|
5861
|
-
...observationResult.diagnostics
|
|
6082
|
+
)
|
|
5862
6083
|
];
|
|
5863
|
-
const
|
|
5864
|
-
const
|
|
5865
|
-
|
|
5866
|
-
|
|
5867
|
-
|
|
5868
|
-
|
|
6084
|
+
const expectMatched = suiteCase.expect !== void 0 && expectResult.ok && observationsOk;
|
|
6085
|
+
const diagnostics = expectMatched ? [
|
|
6086
|
+
...observationResult.diagnostics,
|
|
6087
|
+
...expectResult.diagnostics,
|
|
6088
|
+
diagnostic4(
|
|
6089
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
6090
|
+
`Expected semantic failure matched (check.status=${checkResult.status}).`,
|
|
6091
|
+
"info",
|
|
6092
|
+
suiteCase.id
|
|
6093
|
+
)
|
|
6094
|
+
] : [
|
|
6095
|
+
...checkFailureDiagnostics,
|
|
6096
|
+
...observationResult.diagnostics,
|
|
6097
|
+
...expectResult.diagnostics
|
|
6098
|
+
];
|
|
6099
|
+
const ok = suiteCase.expect !== void 0 ? expectResult.ok && observationsOk : checkOk && observationsOk;
|
|
5869
6100
|
const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
|
|
5870
6101
|
return {
|
|
5871
6102
|
id: suiteCase.id,
|