@agent-inspect/viewer 6.31.10 → 6.31.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1047 -816
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.mjs +1047 -816
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -1170,870 +1170,870 @@ function buildRunTimeline(events, options = {}) {
|
|
|
1170
1170
|
};
|
|
1171
1171
|
}
|
|
1172
1172
|
|
|
1173
|
-
// packages/core/src/
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1173
|
+
// packages/core/src/diagnostics/programmatic.ts
|
|
1174
|
+
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
1175
|
+
AI_TRACE_INPUT_INVALID: {
|
|
1176
|
+
code: "AI_TRACE_INPUT_INVALID",
|
|
1177
|
+
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
1178
|
+
remediation: "For a file path, use openTraceFile(path).",
|
|
1179
|
+
relatedCodes: ["invalid_input"]
|
|
1180
|
+
},
|
|
1181
|
+
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
1182
|
+
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
1183
|
+
summary: "No trace reader could detect the input format.",
|
|
1184
|
+
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
1185
|
+
relatedCodes: ["unsupported_format"]
|
|
1186
|
+
},
|
|
1187
|
+
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
1188
|
+
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
1189
|
+
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
1190
|
+
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
1191
|
+
relatedCodes: ["ambiguous_format"]
|
|
1192
|
+
},
|
|
1193
|
+
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
1194
|
+
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
1195
|
+
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
1196
|
+
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
1197
|
+
},
|
|
1198
|
+
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
1199
|
+
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
1200
|
+
summary: "Multiple runs are available; select a run before executing checks.",
|
|
1201
|
+
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
1202
|
+
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
1203
|
+
},
|
|
1204
|
+
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
1205
|
+
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1206
|
+
summary: "A parentId equals its own event/step id.",
|
|
1207
|
+
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
1208
|
+
relatedCodes: [
|
|
1209
|
+
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
1210
|
+
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
1211
|
+
]
|
|
1212
|
+
},
|
|
1213
|
+
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
1214
|
+
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
1215
|
+
summary: "Trace contains a parentId cycle.",
|
|
1216
|
+
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
1217
|
+
relatedCodes: ["structure.cycle"]
|
|
1218
|
+
},
|
|
1219
|
+
AI_TRACE_CONTRACT_INPUT_INVALID: {
|
|
1220
|
+
code: "AI_TRACE_CONTRACT_INPUT_INVALID",
|
|
1221
|
+
summary: "evaluateTraceContract expects ({ read }, contract) or use evaluateTraceContractRead(read, contract).",
|
|
1222
|
+
remediation: "Pass TraceCheckInput as the first argument: evaluateTraceContract({ read }, contract).",
|
|
1223
|
+
relatedCodes: ["AI_CHECK_INVALID_ARGUMENTS"]
|
|
1224
|
+
},
|
|
1225
|
+
AI_INSPECT_RUN_UNSUPPORTED_OPTION: {
|
|
1226
|
+
code: "AI_INSPECT_RUN_UNSUPPORTED_OPTION",
|
|
1227
|
+
summary: 'inspectRun does not accept a "writer" option.',
|
|
1228
|
+
remediation: "Use createInspector({ writer }) then inspector.run(...). inspectRun persists via the global JSONL path (schema 0.1)."
|
|
1183
1229
|
}
|
|
1184
|
-
|
|
1230
|
+
});
|
|
1231
|
+
function formatProgrammaticDiagnostic(code, detail) {
|
|
1232
|
+
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
1233
|
+
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
1234
|
+
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
1185
1235
|
}
|
|
1186
1236
|
|
|
1187
|
-
// packages/core/src/
|
|
1188
|
-
function
|
|
1189
|
-
return typeof value === "
|
|
1190
|
-
}
|
|
1191
|
-
function finitePositiveInt(value) {
|
|
1192
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
1193
|
-
return void 0;
|
|
1194
|
-
}
|
|
1195
|
-
return Math.trunc(value);
|
|
1237
|
+
// packages/core/src/checks/logical-events.ts
|
|
1238
|
+
function isRecord5(value) {
|
|
1239
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1196
1240
|
}
|
|
1197
|
-
function
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
let found = false;
|
|
1201
|
-
const assignString = (key, value) => {
|
|
1202
|
-
if (isNonEmptyString3(value)) {
|
|
1203
|
-
out[key] = value.trim();
|
|
1204
|
-
found = true;
|
|
1205
|
-
}
|
|
1206
|
-
};
|
|
1207
|
-
assignString("sessionId", record.sessionId);
|
|
1208
|
-
assignString("conversationId", record.conversationId);
|
|
1209
|
-
assignString("groupId", record.groupId);
|
|
1210
|
-
assignString("parentGroupId", record.parentGroupId);
|
|
1211
|
-
assignString("retryOf", record.retryOf);
|
|
1212
|
-
assignString("retryReason", record.retryReason);
|
|
1213
|
-
assignString("handoffFrom", record.handoffFrom);
|
|
1214
|
-
assignString("handoffTo", record.handoffTo);
|
|
1215
|
-
assignString("subAgentId", record.subAgentId);
|
|
1216
|
-
assignString("subAgentName", record.subAgentName);
|
|
1217
|
-
assignString("jobId", record.jobId);
|
|
1218
|
-
assignString("queueName", record.queueName);
|
|
1219
|
-
assignString("workflowName", record.workflowName);
|
|
1220
|
-
assignString("workflowStep", record.workflowStep);
|
|
1221
|
-
assignString("toolCallId", record.toolCallId);
|
|
1222
|
-
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
1223
|
-
assignString("linkedStepId", record.linkedStepId);
|
|
1224
|
-
assignString("operationId", record.operationId);
|
|
1225
|
-
assignString("attemptId", record.attemptId);
|
|
1226
|
-
assignString("fallbackOf", record.fallbackOf);
|
|
1227
|
-
assignString("idempotencyKey", record.idempotencyKey);
|
|
1228
|
-
assignString("correlationId", record.correlationId);
|
|
1229
|
-
assignString("requestId", record.requestId);
|
|
1230
|
-
assignString("decisionId", record.decisionId);
|
|
1231
|
-
const attempt = finitePositiveInt(record.attempt);
|
|
1232
|
-
if (attempt !== void 0) {
|
|
1233
|
-
out.attempt = attempt;
|
|
1234
|
-
found = true;
|
|
1235
|
-
}
|
|
1236
|
-
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
1237
|
-
if (attemptNumber !== void 0) {
|
|
1238
|
-
out.attemptNumber = attemptNumber;
|
|
1239
|
-
found = true;
|
|
1240
|
-
}
|
|
1241
|
-
return found ? out : void 0;
|
|
1241
|
+
function legacyEvent(event) {
|
|
1242
|
+
const value = event.attributes?.legacyEvent;
|
|
1243
|
+
return typeof value === "string" ? value : void 0;
|
|
1242
1244
|
}
|
|
1243
|
-
function
|
|
1244
|
-
|
|
1245
|
-
if (
|
|
1246
|
-
return `group:${meta.groupId}`;
|
|
1247
|
-
}
|
|
1245
|
+
function stepIdOf(event) {
|
|
1246
|
+
const value = event.attributes?.stepId;
|
|
1247
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
1248
1248
|
return void 0;
|
|
1249
1249
|
}
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
completed: 1
|
|
1259
|
-
};
|
|
1260
|
-
var EXPLICIT_SESSION_STATUSES = /* @__PURE__ */ new Set([
|
|
1261
|
-
"running",
|
|
1262
|
-
"waiting_input",
|
|
1263
|
-
"idle",
|
|
1264
|
-
"completed",
|
|
1265
|
-
"error",
|
|
1266
|
-
"stale",
|
|
1267
|
-
"unknown"
|
|
1268
|
-
]);
|
|
1269
|
-
function isExplicitSessionStatus(value) {
|
|
1270
|
-
return typeof value === "string" && EXPLICIT_SESSION_STATUSES.has(value);
|
|
1271
|
-
}
|
|
1272
|
-
function activityMs(run) {
|
|
1273
|
-
return run.endedAt ?? run.startedAt ?? 0;
|
|
1250
|
+
function cloneEvent(event) {
|
|
1251
|
+
return {
|
|
1252
|
+
...event,
|
|
1253
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
1254
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
1255
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
1256
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
1257
|
+
};
|
|
1274
1258
|
}
|
|
1275
|
-
function
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1259
|
+
function mergeAttributes(start, complete) {
|
|
1260
|
+
const merged = {
|
|
1261
|
+
...isRecord5(complete.attributes) ? complete.attributes : {},
|
|
1262
|
+
...isRecord5(start.attributes) ? start.attributes : {}
|
|
1263
|
+
};
|
|
1264
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
1265
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
1266
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
1267
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
1280
1268
|
}
|
|
1281
|
-
return
|
|
1269
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
1282
1270
|
}
|
|
1283
|
-
function
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1271
|
+
function pairStartComplete(start, complete) {
|
|
1272
|
+
const attributes = mergeAttributes(start, complete);
|
|
1273
|
+
const paired = {
|
|
1274
|
+
...cloneEvent(start),
|
|
1275
|
+
status: complete.status,
|
|
1276
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
1277
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
1278
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
1279
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
1280
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
1281
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
1282
|
+
};
|
|
1283
|
+
return {
|
|
1284
|
+
...paired,
|
|
1285
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
1286
|
+
projection: {
|
|
1287
|
+
paired: true,
|
|
1288
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
1289
|
+
parentNormalized: false,
|
|
1290
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
1289
1291
|
}
|
|
1290
|
-
}
|
|
1291
|
-
return earliest;
|
|
1292
|
-
}
|
|
1293
|
-
function latestEndWhenAllEnded(runs) {
|
|
1294
|
-
if (runs.length === 0) return void 0;
|
|
1295
|
-
let latest;
|
|
1296
|
-
for (const run of runs) {
|
|
1297
|
-
if (run.endedAt === void 0) return void 0;
|
|
1298
|
-
if (latest === void 0 || run.endedAt > latest) latest = run.endedAt;
|
|
1299
|
-
}
|
|
1300
|
-
return latest;
|
|
1292
|
+
};
|
|
1301
1293
|
}
|
|
1302
|
-
function
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
best = raw;
|
|
1294
|
+
function asLogical(event, extras) {
|
|
1295
|
+
return {
|
|
1296
|
+
...cloneEvent(event),
|
|
1297
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
1298
|
+
projection: {
|
|
1299
|
+
paired: false,
|
|
1300
|
+
absorbedEventIds: Object.freeze([]),
|
|
1301
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
1302
|
+
...{}
|
|
1312
1303
|
}
|
|
1313
|
-
}
|
|
1314
|
-
return best;
|
|
1304
|
+
};
|
|
1315
1305
|
}
|
|
1316
|
-
function
|
|
1317
|
-
const
|
|
1318
|
-
const
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
const
|
|
1332
|
-
|
|
1333
|
-
const
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1306
|
+
function projectLogicalEvents(events) {
|
|
1307
|
+
const diagnostics = [];
|
|
1308
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
1309
|
+
for (const event of events) {
|
|
1310
|
+
const list = byRun.get(event.runId) ?? [];
|
|
1311
|
+
list.push(event);
|
|
1312
|
+
byRun.set(event.runId, list);
|
|
1313
|
+
}
|
|
1314
|
+
const absorbedIds = /* @__PURE__ */ new Set();
|
|
1315
|
+
const logicalByRawId = /* @__PURE__ */ new Map();
|
|
1316
|
+
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
1317
|
+
const logical = [];
|
|
1318
|
+
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
1319
|
+
for (const runId of orderedRuns) {
|
|
1320
|
+
const runEvents = byRun.get(runId) ?? [];
|
|
1321
|
+
const starts = [];
|
|
1322
|
+
const completes = [];
|
|
1323
|
+
const others = [];
|
|
1324
|
+
for (const event of runEvents) {
|
|
1325
|
+
const legacy = legacyEvent(event);
|
|
1326
|
+
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
1327
|
+
starts.push(event);
|
|
1328
|
+
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
1329
|
+
completes.push(event);
|
|
1330
|
+
} else {
|
|
1331
|
+
others.push(event);
|
|
1332
|
+
}
|
|
1337
1333
|
}
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1334
|
+
const usedCompletes = /* @__PURE__ */ new Set();
|
|
1335
|
+
for (const start of starts) {
|
|
1336
|
+
const stepId = stepIdOf(start);
|
|
1337
|
+
let match;
|
|
1338
|
+
if (legacyEvent(start) === "run_started") {
|
|
1339
|
+
const candidates = completes.filter(
|
|
1340
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
1341
|
+
);
|
|
1342
|
+
if (candidates.length > 1) {
|
|
1343
|
+
diagnostics.push({
|
|
1344
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1345
|
+
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
1346
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1347
|
+
});
|
|
1348
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1349
|
+
}
|
|
1350
|
+
match = candidates[0];
|
|
1351
|
+
} else if (stepId) {
|
|
1352
|
+
const candidates = completes.filter(
|
|
1353
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
1354
|
+
);
|
|
1355
|
+
if (candidates.length > 1) {
|
|
1356
|
+
diagnostics.push({
|
|
1357
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1358
|
+
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
1359
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1360
|
+
});
|
|
1361
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1362
|
+
}
|
|
1363
|
+
match = candidates[0];
|
|
1364
|
+
}
|
|
1365
|
+
if (match) {
|
|
1366
|
+
usedCompletes.add(match.eventId);
|
|
1367
|
+
absorbedIds.add(match.eventId);
|
|
1368
|
+
const paired = pairStartComplete(start, match);
|
|
1369
|
+
logical.push(paired);
|
|
1370
|
+
logicalByRawId.set(start.eventId, paired);
|
|
1371
|
+
logicalByRawId.set(match.eventId, paired);
|
|
1372
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
1373
|
+
} else {
|
|
1374
|
+
diagnostics.push({
|
|
1375
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
1376
|
+
message: `No matching complete for start ${start.eventId}.`,
|
|
1377
|
+
eventIds: [start.eventId]
|
|
1378
|
+
});
|
|
1379
|
+
const alone = asLogical(start);
|
|
1380
|
+
logical.push(alone);
|
|
1381
|
+
logicalByRawId.set(start.eventId, alone);
|
|
1382
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1383
|
+
}
|
|
1341
1384
|
}
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1385
|
+
for (const complete of completes) {
|
|
1386
|
+
if (usedCompletes.has(complete.eventId)) continue;
|
|
1387
|
+
diagnostics.push({
|
|
1388
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
1389
|
+
message: `No matching start for complete ${complete.eventId}.`,
|
|
1390
|
+
eventIds: [complete.eventId]
|
|
1391
|
+
});
|
|
1392
|
+
const alone = asLogical(complete);
|
|
1393
|
+
logical.push(alone);
|
|
1394
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
1395
|
+
const stepId = stepIdOf(complete);
|
|
1396
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1397
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
for (const event of others) {
|
|
1401
|
+
const alone = asLogical(event);
|
|
1402
|
+
logical.push(alone);
|
|
1403
|
+
logicalByRawId.set(event.eventId, alone);
|
|
1404
|
+
const stepId = stepIdOf(event);
|
|
1405
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1406
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1407
|
+
}
|
|
1345
1408
|
}
|
|
1346
1409
|
}
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1410
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
1411
|
+
const normalized = [];
|
|
1412
|
+
for (const event of logical) {
|
|
1413
|
+
const originalParentId = event.parentId;
|
|
1414
|
+
if (!originalParentId) {
|
|
1415
|
+
normalized.push(event);
|
|
1416
|
+
continue;
|
|
1417
|
+
}
|
|
1418
|
+
let nextParent = originalParentId;
|
|
1419
|
+
let remapped = false;
|
|
1420
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
1421
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
1422
|
+
nextParent = viaAbsorbed.eventId;
|
|
1423
|
+
remapped = true;
|
|
1424
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
1425
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
1426
|
+
if (viaStep) {
|
|
1427
|
+
nextParent = viaStep;
|
|
1428
|
+
remapped = true;
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
if (!remapped) {
|
|
1432
|
+
if (originalParentId === event.eventId) {
|
|
1433
|
+
diagnostics.push({
|
|
1434
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1435
|
+
message: formatProgrammaticDiagnostic(
|
|
1436
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1437
|
+
`Removed self-parent edge on ${event.eventId}.`
|
|
1438
|
+
),
|
|
1439
|
+
eventIds: [event.eventId]
|
|
1440
|
+
});
|
|
1441
|
+
const { parentId: _drop, ...rest } = event;
|
|
1442
|
+
normalized.push({
|
|
1443
|
+
...rest,
|
|
1444
|
+
projection: {
|
|
1445
|
+
...event.projection,
|
|
1446
|
+
parentNormalized: true,
|
|
1447
|
+
originalParentId
|
|
1448
|
+
}
|
|
1449
|
+
});
|
|
1450
|
+
continue;
|
|
1451
|
+
}
|
|
1452
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
1453
|
+
const mapping = event.attributes?.parentMapping;
|
|
1454
|
+
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
1455
|
+
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
1456
|
+
if (!unresolved) {
|
|
1457
|
+
diagnostics.push({
|
|
1458
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
1459
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
1460
|
+
eventIds: [event.eventId]
|
|
1461
|
+
});
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
normalized.push(event);
|
|
1465
|
+
continue;
|
|
1466
|
+
}
|
|
1467
|
+
if (nextParent === event.eventId) {
|
|
1468
|
+
diagnostics.push({
|
|
1469
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1470
|
+
message: formatProgrammaticDiagnostic(
|
|
1471
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1472
|
+
`Removed self-parent edge on ${event.eventId} (was ${originalParentId}).`
|
|
1473
|
+
),
|
|
1474
|
+
eventIds: [event.eventId]
|
|
1475
|
+
});
|
|
1476
|
+
const { parentId: _drop, ...rest } = event;
|
|
1477
|
+
normalized.push({
|
|
1478
|
+
...rest,
|
|
1479
|
+
projection: {
|
|
1480
|
+
...event.projection,
|
|
1481
|
+
parentNormalized: true,
|
|
1482
|
+
originalParentId
|
|
1483
|
+
}
|
|
1484
|
+
});
|
|
1485
|
+
continue;
|
|
1354
1486
|
}
|
|
1487
|
+
diagnostics.push({
|
|
1488
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
1489
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
1490
|
+
eventIds: [event.eventId]
|
|
1491
|
+
});
|
|
1492
|
+
normalized.push({
|
|
1493
|
+
...event,
|
|
1494
|
+
parentId: nextParent,
|
|
1495
|
+
projection: {
|
|
1496
|
+
...event.projection,
|
|
1497
|
+
parentNormalized: true,
|
|
1498
|
+
originalParentId
|
|
1499
|
+
}
|
|
1500
|
+
});
|
|
1355
1501
|
}
|
|
1356
|
-
|
|
1502
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
1503
|
+
normalized.sort((a, b) => {
|
|
1504
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
1505
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
1506
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
1507
|
+
});
|
|
1508
|
+
return {
|
|
1509
|
+
logicalEvents: Object.freeze(normalized),
|
|
1510
|
+
diagnostics: Object.freeze(diagnostics)
|
|
1511
|
+
};
|
|
1357
1512
|
}
|
|
1358
|
-
function
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
if (
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
const
|
|
1368
|
-
|
|
1369
|
-
|
|
1513
|
+
function resolveCanonicalToolName(event) {
|
|
1514
|
+
const attrs = event.attributes;
|
|
1515
|
+
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
1516
|
+
if (direct) return direct;
|
|
1517
|
+
const metadata = attrs?.metadata;
|
|
1518
|
+
if (isRecord5(metadata)) {
|
|
1519
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
1520
|
+
if (nested) return nested;
|
|
1521
|
+
}
|
|
1522
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
1523
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
1524
|
+
}
|
|
1525
|
+
return event.name;
|
|
1370
1526
|
}
|
|
1371
|
-
function
|
|
1372
|
-
|
|
1373
|
-
const
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
let correlationId;
|
|
1377
|
-
let jobId;
|
|
1378
|
-
let workflowId;
|
|
1379
|
-
for (const run of sessionRuns) {
|
|
1380
|
-
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1381
|
-
if (!correlationId && meta?.correlationId) correlationId = meta.correlationId;
|
|
1382
|
-
if (!jobId && meta?.jobId) jobId = meta.jobId;
|
|
1383
|
-
if (!workflowId && meta?.workflowName) workflowId = meta.workflowName;
|
|
1384
|
-
else if (!workflowId && meta?.workflowStep) workflowId = meta.workflowStep;
|
|
1527
|
+
function pickString(record, keys) {
|
|
1528
|
+
if (!record) return void 0;
|
|
1529
|
+
for (const key of keys) {
|
|
1530
|
+
const value = record[key];
|
|
1531
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
1385
1532
|
}
|
|
1386
|
-
|
|
1387
|
-
const lastActivity = lastMs > 0 ? new Date(lastMs).toISOString() : (/* @__PURE__ */ new Date(0)).toISOString();
|
|
1388
|
-
const retryCount2 = summary.retries.filter(
|
|
1389
|
-
(retry) => retry.retryOf !== void 0 || (retry.attempt ?? 0) > 1
|
|
1390
|
-
).length;
|
|
1391
|
-
return {
|
|
1392
|
-
...summary,
|
|
1393
|
-
status: deriveSessionStatus(sessionRuns, options),
|
|
1394
|
-
startedAt,
|
|
1395
|
-
endedAt,
|
|
1396
|
-
durationMs,
|
|
1397
|
-
correlationId,
|
|
1398
|
-
jobId,
|
|
1399
|
-
workflowId,
|
|
1400
|
-
lastError: deriveLastError(sessionRuns),
|
|
1401
|
-
lastActivity,
|
|
1402
|
-
retryCount: retryCount2,
|
|
1403
|
-
observationSummary: deriveObservationSummary(sessionRuns),
|
|
1404
|
-
checkSummary: deriveCheckSummary(sessionRuns)
|
|
1405
|
-
};
|
|
1533
|
+
return void 0;
|
|
1406
1534
|
}
|
|
1407
1535
|
|
|
1408
|
-
// packages/core/src/
|
|
1409
|
-
async function
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
}
|
|
1418
|
-
break;
|
|
1536
|
+
// packages/core/src/search.ts
|
|
1537
|
+
async function loadTraceMetadataList(_traceDir, fileNames, getPath) {
|
|
1538
|
+
const metas = [];
|
|
1539
|
+
for (const fileName of fileNames) {
|
|
1540
|
+
try {
|
|
1541
|
+
const filePath = getPath(fileName);
|
|
1542
|
+
const meta = await extractMetadata(filePath);
|
|
1543
|
+
metas.push(meta);
|
|
1544
|
+
} catch {
|
|
1419
1545
|
}
|
|
1420
|
-
} catch {
|
|
1421
1546
|
}
|
|
1422
|
-
return
|
|
1423
|
-
runId: meta.runId,
|
|
1424
|
-
name: meta.name,
|
|
1425
|
-
status: meta.status,
|
|
1426
|
-
startedAt: meta.startedAt,
|
|
1427
|
-
endedAt: meta.endedAt,
|
|
1428
|
-
durationMs: meta.durationMs,
|
|
1429
|
-
filePath: meta.filePath,
|
|
1430
|
-
metadata
|
|
1431
|
-
};
|
|
1547
|
+
return metas;
|
|
1432
1548
|
}
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1549
|
+
|
|
1550
|
+
// packages/core/src/sessions/metadata.ts
|
|
1551
|
+
function isNonEmptyString3(value) {
|
|
1552
|
+
return typeof value === "string" && value.trim() !== "";
|
|
1553
|
+
}
|
|
1554
|
+
function finitePositiveInt(value) {
|
|
1555
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
1556
|
+
return void 0;
|
|
1557
|
+
}
|
|
1558
|
+
return Math.trunc(value);
|
|
1559
|
+
}
|
|
1560
|
+
function extractSessionWorkflowMetadata(record) {
|
|
1561
|
+
if (!record) return void 0;
|
|
1562
|
+
const out = {};
|
|
1563
|
+
let found = false;
|
|
1564
|
+
const assignString = (key, value) => {
|
|
1565
|
+
if (isNonEmptyString3(value)) {
|
|
1566
|
+
out[key] = value.trim();
|
|
1567
|
+
found = true;
|
|
1568
|
+
}
|
|
1569
|
+
};
|
|
1570
|
+
assignString("sessionId", record.sessionId);
|
|
1571
|
+
assignString("conversationId", record.conversationId);
|
|
1572
|
+
assignString("groupId", record.groupId);
|
|
1573
|
+
assignString("parentGroupId", record.parentGroupId);
|
|
1574
|
+
assignString("retryOf", record.retryOf);
|
|
1575
|
+
assignString("retryReason", record.retryReason);
|
|
1576
|
+
assignString("handoffFrom", record.handoffFrom);
|
|
1577
|
+
assignString("handoffTo", record.handoffTo);
|
|
1578
|
+
assignString("subAgentId", record.subAgentId);
|
|
1579
|
+
assignString("subAgentName", record.subAgentName);
|
|
1580
|
+
assignString("jobId", record.jobId);
|
|
1581
|
+
assignString("queueName", record.queueName);
|
|
1582
|
+
assignString("workflowName", record.workflowName);
|
|
1583
|
+
assignString("workflowStep", record.workflowStep);
|
|
1584
|
+
assignString("toolCallId", record.toolCallId);
|
|
1585
|
+
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
1586
|
+
assignString("linkedStepId", record.linkedStepId);
|
|
1587
|
+
assignString("operationId", record.operationId);
|
|
1588
|
+
assignString("attemptId", record.attemptId);
|
|
1589
|
+
assignString("fallbackOf", record.fallbackOf);
|
|
1590
|
+
assignString("idempotencyKey", record.idempotencyKey);
|
|
1591
|
+
assignString("correlationId", record.correlationId);
|
|
1592
|
+
assignString("requestId", record.requestId);
|
|
1593
|
+
assignString("decisionId", record.decisionId);
|
|
1594
|
+
const attempt = finitePositiveInt(record.attempt);
|
|
1595
|
+
if (attempt !== void 0) {
|
|
1596
|
+
out.attempt = attempt;
|
|
1597
|
+
found = true;
|
|
1598
|
+
}
|
|
1599
|
+
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
1600
|
+
if (attemptNumber !== void 0) {
|
|
1601
|
+
out.attemptNumber = attemptNumber;
|
|
1602
|
+
found = true;
|
|
1437
1603
|
}
|
|
1438
|
-
return out;
|
|
1604
|
+
return found ? out : void 0;
|
|
1605
|
+
}
|
|
1606
|
+
function sessionKeyForRun(meta, options) {
|
|
1607
|
+
if (meta?.sessionId) return meta.sessionId;
|
|
1608
|
+
if (options?.correlateByGroupId && meta?.groupId) {
|
|
1609
|
+
return `group:${meta.groupId}`;
|
|
1610
|
+
}
|
|
1611
|
+
return void 0;
|
|
1439
1612
|
}
|
|
1440
1613
|
|
|
1441
|
-
// packages/core/src/sessions/
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1614
|
+
// packages/core/src/sessions/status.ts
|
|
1615
|
+
var DEFAULT_STALE_THRESHOLD_MS = 864e5;
|
|
1616
|
+
var EXPLICIT_STATUS_PRIORITY = {
|
|
1617
|
+
error: 5,
|
|
1618
|
+
waiting_input: 4,
|
|
1619
|
+
idle: 3,
|
|
1620
|
+
stale: 2,
|
|
1621
|
+
completed: 1
|
|
1622
|
+
};
|
|
1623
|
+
var EXPLICIT_SESSION_STATUSES = /* @__PURE__ */ new Set([
|
|
1624
|
+
"running",
|
|
1625
|
+
"waiting_input",
|
|
1626
|
+
"idle",
|
|
1627
|
+
"completed",
|
|
1628
|
+
"error",
|
|
1629
|
+
"stale",
|
|
1630
|
+
"unknown"
|
|
1631
|
+
]);
|
|
1632
|
+
function isExplicitSessionStatus(value) {
|
|
1633
|
+
return typeof value === "string" && EXPLICIT_SESSION_STATUSES.has(value);
|
|
1447
1634
|
}
|
|
1448
|
-
function
|
|
1449
|
-
|
|
1450
|
-
for (const runId of runIds) {
|
|
1451
|
-
const groupId = metaByRunId.get(runId)?.groupId;
|
|
1452
|
-
if (!groupId) continue;
|
|
1453
|
-
const existing = byGroup.get(groupId);
|
|
1454
|
-
if (existing) {
|
|
1455
|
-
existing.runIds.push(runId);
|
|
1456
|
-
} else {
|
|
1457
|
-
byGroup.set(groupId, {
|
|
1458
|
-
groupId,
|
|
1459
|
-
parentGroupId: metaByRunId.get(runId)?.parentGroupId,
|
|
1460
|
-
runIds: [runId]
|
|
1461
|
-
});
|
|
1462
|
-
}
|
|
1463
|
-
}
|
|
1464
|
-
return [...byGroup.values()].map((group) => ({
|
|
1465
|
-
...group,
|
|
1466
|
-
runIds: [...group.runIds].sort((a, b) => a.localeCompare(b))
|
|
1467
|
-
}));
|
|
1635
|
+
function activityMs(run) {
|
|
1636
|
+
return run.endedAt ?? run.startedAt ?? 0;
|
|
1468
1637
|
}
|
|
1469
|
-
function
|
|
1470
|
-
|
|
1471
|
-
const
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
if (seen.has(key)) return;
|
|
1475
|
-
seen.add(key);
|
|
1476
|
-
edges.push(edge);
|
|
1477
|
-
};
|
|
1478
|
-
for (const runId of runIds) {
|
|
1479
|
-
const meta = metaByRunId.get(runId);
|
|
1480
|
-
if (!meta) continue;
|
|
1481
|
-
if (meta.handoffFrom && meta.handoffTo) {
|
|
1482
|
-
pushEdge({
|
|
1483
|
-
from: meta.handoffFrom,
|
|
1484
|
-
to: meta.handoffTo,
|
|
1485
|
-
source: "manual",
|
|
1486
|
-
confidence: "explicit"
|
|
1487
|
-
});
|
|
1488
|
-
continue;
|
|
1489
|
-
}
|
|
1490
|
-
if (meta.handoffFrom) {
|
|
1491
|
-
pushEdge({
|
|
1492
|
-
from: meta.handoffFrom,
|
|
1493
|
-
to: runId,
|
|
1494
|
-
source: "manual",
|
|
1495
|
-
confidence: "explicit"
|
|
1496
|
-
});
|
|
1497
|
-
}
|
|
1498
|
-
if (meta.handoffTo) {
|
|
1499
|
-
pushEdge({
|
|
1500
|
-
from: runId,
|
|
1501
|
-
to: meta.handoffTo,
|
|
1502
|
-
source: "manual",
|
|
1503
|
-
confidence: "explicit"
|
|
1504
|
-
});
|
|
1505
|
-
}
|
|
1506
|
-
if (meta.subAgentId && meta.parentGroupId && !meta.handoffFrom && !meta.handoffTo) {
|
|
1507
|
-
pushEdge({
|
|
1508
|
-
from: meta.parentGroupId,
|
|
1509
|
-
to: meta.subAgentId,
|
|
1510
|
-
source: "inferred",
|
|
1511
|
-
confidence: "correlated"
|
|
1512
|
-
});
|
|
1513
|
-
warnings.push({
|
|
1514
|
-
code: "ambiguous-handoff-endpoints",
|
|
1515
|
-
message: "Handoff inferred from parentGroupId and subAgentId without explicit handoffFrom/handoffTo.",
|
|
1516
|
-
runId,
|
|
1517
|
-
sessionId
|
|
1518
|
-
});
|
|
1519
|
-
}
|
|
1638
|
+
function latestActivityMs(runs) {
|
|
1639
|
+
let latest = 0;
|
|
1640
|
+
for (const run of runs) {
|
|
1641
|
+
const ms = activityMs(run);
|
|
1642
|
+
if (ms > latest) latest = ms;
|
|
1520
1643
|
}
|
|
1521
|
-
return
|
|
1522
|
-
const from = a.from.localeCompare(b.from);
|
|
1523
|
-
if (from !== 0) return from;
|
|
1524
|
-
return a.to.localeCompare(b.to);
|
|
1525
|
-
});
|
|
1644
|
+
return latest;
|
|
1526
1645
|
}
|
|
1527
|
-
function
|
|
1528
|
-
|
|
1529
|
-
for (const
|
|
1530
|
-
|
|
1531
|
-
if (
|
|
1532
|
-
|
|
1533
|
-
retries.push({
|
|
1534
|
-
runId,
|
|
1535
|
-
retryOf: meta.retryOf,
|
|
1536
|
-
attempt: meta.attempt,
|
|
1537
|
-
source: "manual",
|
|
1538
|
-
confidence: "explicit"
|
|
1539
|
-
});
|
|
1540
|
-
continue;
|
|
1541
|
-
}
|
|
1542
|
-
if (meta.attempt !== void 0 && meta.attempt > 1) {
|
|
1543
|
-
retries.push({
|
|
1544
|
-
runId,
|
|
1545
|
-
attempt: meta.attempt,
|
|
1546
|
-
source: "inferred",
|
|
1547
|
-
confidence: "correlated"
|
|
1548
|
-
});
|
|
1549
|
-
warnings.push({
|
|
1550
|
-
code: "ambiguous-retry-link",
|
|
1551
|
-
message: "attempt > 1 without retryOf; retry link is correlated only.",
|
|
1552
|
-
runId,
|
|
1553
|
-
sessionId
|
|
1554
|
-
});
|
|
1646
|
+
function earliestStart(runs) {
|
|
1647
|
+
let earliest;
|
|
1648
|
+
for (const run of runs) {
|
|
1649
|
+
if (run.startedAt === void 0) continue;
|
|
1650
|
+
if (earliest === void 0 || run.startedAt < earliest) {
|
|
1651
|
+
earliest = run.startedAt;
|
|
1555
1652
|
}
|
|
1556
1653
|
}
|
|
1557
|
-
return
|
|
1654
|
+
return earliest;
|
|
1558
1655
|
}
|
|
1559
|
-
function
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
1566
|
-
);
|
|
1567
|
-
const ordered = [...runs].sort(compareRuns);
|
|
1568
|
-
const path18 = [];
|
|
1569
|
-
const visited = /* @__PURE__ */ new Set();
|
|
1570
|
-
const pushRun = (run, confidence, source) => {
|
|
1571
|
-
if (visited.has(run.runId)) return;
|
|
1572
|
-
visited.add(run.runId);
|
|
1573
|
-
path18.push({
|
|
1574
|
-
runId: run.runId,
|
|
1575
|
-
name: run.name,
|
|
1576
|
-
startedAt: run.startedAt,
|
|
1577
|
-
durationMs: run.durationMs,
|
|
1578
|
-
confidence,
|
|
1579
|
-
source
|
|
1580
|
-
});
|
|
1581
|
-
};
|
|
1582
|
-
for (const edge of handoffs) {
|
|
1583
|
-
if (edge.confidence !== "explicit") continue;
|
|
1584
|
-
const fromRun = [...runById.values()].find(
|
|
1585
|
-
(run) => run.runId === edge.from || metaRunIdMatches(run, edge.from, runById)
|
|
1586
|
-
);
|
|
1587
|
-
const toRun = [...runById.values()].find(
|
|
1588
|
-
(run) => run.runId === edge.to || metaRunIdMatches(run, edge.to, runById)
|
|
1589
|
-
);
|
|
1590
|
-
if (fromRun) pushRun(fromRun, "explicit", "manual");
|
|
1591
|
-
if (toRun) pushRun(toRun, "explicit", "manual");
|
|
1592
|
-
}
|
|
1593
|
-
for (const run of ordered) {
|
|
1594
|
-
if (visited.has(run.runId)) continue;
|
|
1595
|
-
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
1596
|
-
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
1656
|
+
function latestEndWhenAllEnded(runs) {
|
|
1657
|
+
if (runs.length === 0) return void 0;
|
|
1658
|
+
let latest;
|
|
1659
|
+
for (const run of runs) {
|
|
1660
|
+
if (run.endedAt === void 0) return void 0;
|
|
1661
|
+
if (latest === void 0 || run.endedAt > latest) latest = run.endedAt;
|
|
1597
1662
|
}
|
|
1598
|
-
return
|
|
1599
|
-
}
|
|
1600
|
-
function metaRunIdMatches(run, token, runById) {
|
|
1601
|
-
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1602
|
-
return meta?.subAgentId === token || meta?.groupId === token || runById.has(token);
|
|
1663
|
+
return latest;
|
|
1603
1664
|
}
|
|
1604
|
-
function
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
const metaByRunId = /* @__PURE__ */ new Map();
|
|
1665
|
+
function pickExplicitStatus(runs) {
|
|
1666
|
+
let best;
|
|
1667
|
+
let bestPriority = 0;
|
|
1608
1668
|
for (const run of runs) {
|
|
1609
|
-
|
|
1669
|
+
const raw = run.metadata?.sessionStatus;
|
|
1670
|
+
if (!isExplicitSessionStatus(raw)) continue;
|
|
1671
|
+
const priority = EXPLICIT_STATUS_PRIORITY[raw] ?? 0;
|
|
1672
|
+
if (priority > bestPriority) {
|
|
1673
|
+
bestPriority = priority;
|
|
1674
|
+
best = raw;
|
|
1675
|
+
}
|
|
1610
1676
|
}
|
|
1611
|
-
|
|
1612
|
-
|
|
1677
|
+
return best;
|
|
1678
|
+
}
|
|
1679
|
+
function deriveLastError(runs) {
|
|
1680
|
+
const errorRuns = runs.filter((run) => run.status === "error").sort((a, b) => activityMs(b) - activityMs(a));
|
|
1681
|
+
const latest = errorRuns[0];
|
|
1682
|
+
if (!latest) return void 0;
|
|
1683
|
+
const meta = latest.metadata ?? {};
|
|
1684
|
+
const message = typeof meta.errorMessage === "string" && meta.errorMessage.trim() !== "" ? meta.errorMessage.trim() : latest.name ?? latest.runId;
|
|
1685
|
+
const code = typeof meta.errorCode === "string" && meta.errorCode.trim() !== "" ? meta.errorCode.trim() : void 0;
|
|
1686
|
+
return { runId: latest.runId, message, code };
|
|
1687
|
+
}
|
|
1688
|
+
function deriveCheckSummary(runs) {
|
|
1689
|
+
let pass = 0;
|
|
1690
|
+
let fail = 0;
|
|
1691
|
+
let warn2 = 0;
|
|
1692
|
+
let found = false;
|
|
1613
1693
|
for (const run of runs) {
|
|
1614
|
-
const
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1694
|
+
const summary = run.metadata?.checkSummary;
|
|
1695
|
+
if (!summary || typeof summary !== "object") continue;
|
|
1696
|
+
const record = summary;
|
|
1697
|
+
if (typeof record.pass === "number") {
|
|
1698
|
+
pass += record.pass;
|
|
1699
|
+
found = true;
|
|
1700
|
+
}
|
|
1701
|
+
if (typeof record.fail === "number") {
|
|
1702
|
+
fail += record.fail;
|
|
1703
|
+
found = true;
|
|
1704
|
+
}
|
|
1705
|
+
if (typeof record.warn === "number") {
|
|
1706
|
+
warn2 += record.warn;
|
|
1707
|
+
found = true;
|
|
1621
1708
|
}
|
|
1622
|
-
const bucket = sessionsByKey.get(key) ?? [];
|
|
1623
|
-
bucket.push(run);
|
|
1624
|
-
sessionsByKey.set(key, bucket);
|
|
1625
1709
|
}
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
const
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
if (confidences.has("explicit") && confidences.has("correlated")) {
|
|
1634
|
-
warnings.push({
|
|
1635
|
-
code: "mixed-confidence-group",
|
|
1636
|
-
message: "Session aggregates explicit and correlated handoff edges.",
|
|
1637
|
-
sessionId
|
|
1638
|
-
});
|
|
1710
|
+
return found ? { pass, fail, warn: warn2 } : void 0;
|
|
1711
|
+
}
|
|
1712
|
+
function deriveObservationSummary(runs) {
|
|
1713
|
+
for (const run of [...runs].sort((a, b) => activityMs(b) - activityMs(a))) {
|
|
1714
|
+
const value = run.metadata?.observationSummary;
|
|
1715
|
+
if (typeof value === "string" && value.trim() !== "") {
|
|
1716
|
+
return value.trim();
|
|
1639
1717
|
}
|
|
1640
|
-
return enrichSessionSummary(
|
|
1641
|
-
{
|
|
1642
|
-
sessionId,
|
|
1643
|
-
runIds,
|
|
1644
|
-
groups,
|
|
1645
|
-
handoffs,
|
|
1646
|
-
retries,
|
|
1647
|
-
criticalPath
|
|
1648
|
-
},
|
|
1649
|
-
runs,
|
|
1650
|
-
{
|
|
1651
|
-
nowMs: options.nowMs,
|
|
1652
|
-
staleThresholdMs: options.staleThresholdMs
|
|
1653
|
-
}
|
|
1654
|
-
);
|
|
1655
|
-
});
|
|
1656
|
-
if (sessions.length === 0 && runs.length > 0) {
|
|
1657
|
-
warnings.push({
|
|
1658
|
-
code: "missing-session-id",
|
|
1659
|
-
message: "No sessionId (or correlated groupId) found on input runs."
|
|
1660
|
-
});
|
|
1661
1718
|
}
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1719
|
+
return void 0;
|
|
1720
|
+
}
|
|
1721
|
+
function deriveSessionStatus(runs, options = {}) {
|
|
1722
|
+
if (runs.length === 0) return "unknown";
|
|
1723
|
+
if (runs.some((run) => run.status === "running")) return "running";
|
|
1724
|
+
const explicit = pickExplicitStatus(runs);
|
|
1725
|
+
if (explicit && explicit !== "running") return explicit;
|
|
1726
|
+
if (runs.some((run) => run.status === "error")) return "error";
|
|
1727
|
+
if (runs.every((run) => run.status === "success")) return "completed";
|
|
1728
|
+
const nowMs = options.nowMs ?? Date.now();
|
|
1729
|
+
const staleThresholdMs = options.staleThresholdMs ?? DEFAULT_STALE_THRESHOLD_MS;
|
|
1730
|
+
const lastMs = latestActivityMs(runs);
|
|
1731
|
+
if (lastMs > 0 && nowMs - lastMs > staleThresholdMs) return "stale";
|
|
1732
|
+
return "unknown";
|
|
1733
|
+
}
|
|
1734
|
+
function enrichSessionSummary(summary, runs, options = {}) {
|
|
1735
|
+
const sessionRuns = runs.filter((run) => summary.runIds.includes(run.runId)).sort((a, b) => a.runId.localeCompare(b.runId));
|
|
1736
|
+
const startedAt = earliestStart(sessionRuns);
|
|
1737
|
+
const endedAt = latestEndWhenAllEnded(sessionRuns);
|
|
1738
|
+
const durationMs = startedAt !== void 0 && endedAt !== void 0 ? endedAt - startedAt : void 0;
|
|
1739
|
+
let correlationId;
|
|
1740
|
+
let jobId;
|
|
1741
|
+
let workflowId;
|
|
1742
|
+
for (const run of sessionRuns) {
|
|
1743
|
+
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1744
|
+
if (!correlationId && meta?.correlationId) correlationId = meta.correlationId;
|
|
1745
|
+
if (!jobId && meta?.jobId) jobId = meta.jobId;
|
|
1746
|
+
if (!workflowId && meta?.workflowName) workflowId = meta.workflowName;
|
|
1747
|
+
else if (!workflowId && meta?.workflowStep) workflowId = meta.workflowStep;
|
|
1748
|
+
}
|
|
1749
|
+
const lastMs = latestActivityMs(sessionRuns);
|
|
1750
|
+
const lastActivity = lastMs > 0 ? new Date(lastMs).toISOString() : (/* @__PURE__ */ new Date(0)).toISOString();
|
|
1751
|
+
const retryCount2 = summary.retries.filter(
|
|
1752
|
+
(retry) => retry.retryOf !== void 0 || (retry.attempt ?? 0) > 1
|
|
1753
|
+
).length;
|
|
1667
1754
|
return {
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1755
|
+
...summary,
|
|
1756
|
+
status: deriveSessionStatus(sessionRuns, options),
|
|
1757
|
+
startedAt,
|
|
1758
|
+
endedAt,
|
|
1759
|
+
durationMs,
|
|
1760
|
+
correlationId,
|
|
1761
|
+
jobId,
|
|
1762
|
+
workflowId,
|
|
1763
|
+
lastError: deriveLastError(sessionRuns),
|
|
1764
|
+
lastActivity,
|
|
1765
|
+
retryCount: retryCount2,
|
|
1766
|
+
observationSummary: deriveObservationSummary(sessionRuns),
|
|
1767
|
+
checkSummary: deriveCheckSummary(sessionRuns)
|
|
1672
1768
|
};
|
|
1673
1769
|
}
|
|
1674
1770
|
|
|
1675
|
-
// packages/core/src/
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
},
|
|
1689
|
-
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
1690
|
-
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
1691
|
-
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
1692
|
-
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
1693
|
-
relatedCodes: ["ambiguous_format"]
|
|
1694
|
-
},
|
|
1695
|
-
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
1696
|
-
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
1697
|
-
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
1698
|
-
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
1699
|
-
},
|
|
1700
|
-
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
1701
|
-
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
1702
|
-
summary: "Multiple runs are available; select a run before executing checks.",
|
|
1703
|
-
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
1704
|
-
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
1705
|
-
},
|
|
1706
|
-
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
1707
|
-
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1708
|
-
summary: "A parentId equals its own event/step id.",
|
|
1709
|
-
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
1710
|
-
relatedCodes: [
|
|
1711
|
-
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
1712
|
-
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
1713
|
-
]
|
|
1714
|
-
},
|
|
1715
|
-
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
1716
|
-
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
1717
|
-
summary: "Trace contains a parentId cycle.",
|
|
1718
|
-
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
1719
|
-
relatedCodes: ["structure.cycle"]
|
|
1720
|
-
},
|
|
1721
|
-
AI_TRACE_CONTRACT_INPUT_INVALID: {
|
|
1722
|
-
code: "AI_TRACE_CONTRACT_INPUT_INVALID",
|
|
1723
|
-
summary: "evaluateTraceContract expects ({ read }, contract) or use evaluateTraceContractRead(read, contract).",
|
|
1724
|
-
remediation: "Pass TraceCheckInput as the first argument: evaluateTraceContract({ read }, contract).",
|
|
1725
|
-
relatedCodes: ["AI_CHECK_INVALID_ARGUMENTS"]
|
|
1726
|
-
},
|
|
1727
|
-
AI_INSPECT_RUN_UNSUPPORTED_OPTION: {
|
|
1728
|
-
code: "AI_INSPECT_RUN_UNSUPPORTED_OPTION",
|
|
1729
|
-
summary: 'inspectRun does not accept a "writer" option.',
|
|
1730
|
-
remediation: "Use createInspector({ writer }) then inspector.run(...). inspectRun persists via the global JSONL path (schema 0.1)."
|
|
1771
|
+
// packages/core/src/sessions/load.ts
|
|
1772
|
+
async function enrichSessionRunRecord(meta) {
|
|
1773
|
+
let metadata;
|
|
1774
|
+
try {
|
|
1775
|
+
const events = await readTraceEventsFromFile(meta.filePath);
|
|
1776
|
+
for (const event of events) {
|
|
1777
|
+
if (event.event !== "run_started") continue;
|
|
1778
|
+
if (event.metadata && typeof event.metadata === "object") {
|
|
1779
|
+
metadata = event.metadata;
|
|
1780
|
+
}
|
|
1781
|
+
break;
|
|
1782
|
+
}
|
|
1783
|
+
} catch {
|
|
1731
1784
|
}
|
|
1732
|
-
});
|
|
1733
|
-
function formatProgrammaticDiagnostic(code, detail) {
|
|
1734
|
-
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
1735
|
-
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
1736
|
-
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
1737
|
-
}
|
|
1738
|
-
|
|
1739
|
-
// packages/core/src/checks/logical-events.ts
|
|
1740
|
-
function isRecord5(value) {
|
|
1741
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1742
|
-
}
|
|
1743
|
-
function legacyEvent(event) {
|
|
1744
|
-
const value = event.attributes?.legacyEvent;
|
|
1745
|
-
return typeof value === "string" ? value : void 0;
|
|
1746
|
-
}
|
|
1747
|
-
function stepIdOf(event) {
|
|
1748
|
-
const value = event.attributes?.stepId;
|
|
1749
|
-
if (typeof value === "string" && value.trim() !== "") return value;
|
|
1750
|
-
return void 0;
|
|
1751
|
-
}
|
|
1752
|
-
function cloneEvent(event) {
|
|
1753
1785
|
return {
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1786
|
+
runId: meta.runId,
|
|
1787
|
+
name: meta.name,
|
|
1788
|
+
status: meta.status,
|
|
1789
|
+
startedAt: meta.startedAt,
|
|
1790
|
+
endedAt: meta.endedAt,
|
|
1791
|
+
durationMs: meta.durationMs,
|
|
1792
|
+
filePath: meta.filePath,
|
|
1793
|
+
metadata
|
|
1759
1794
|
};
|
|
1760
1795
|
}
|
|
1761
|
-
function
|
|
1762
|
-
const
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
};
|
|
1766
|
-
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
1767
|
-
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
1768
|
-
if (complete.attributes?.errorStack !== void 0) {
|
|
1769
|
-
merged.errorStack = complete.attributes.errorStack;
|
|
1796
|
+
async function loadSessionRunRecords(metas) {
|
|
1797
|
+
const out = [];
|
|
1798
|
+
for (const meta of metas) {
|
|
1799
|
+
out.push(await enrichSessionRunRecord(meta));
|
|
1770
1800
|
}
|
|
1771
|
-
return
|
|
1801
|
+
return out;
|
|
1772
1802
|
}
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1803
|
+
|
|
1804
|
+
// packages/core/src/sessions/index.ts
|
|
1805
|
+
function compareRuns(a, b) {
|
|
1806
|
+
const aStart = a.startedAt ?? 0;
|
|
1807
|
+
const bStart = b.startedAt ?? 0;
|
|
1808
|
+
if (aStart !== bStart) return aStart - bStart;
|
|
1809
|
+
return a.runId.localeCompare(b.runId);
|
|
1810
|
+
}
|
|
1811
|
+
function buildGroups(runIds, metaByRunId) {
|
|
1812
|
+
const byGroup = /* @__PURE__ */ new Map();
|
|
1813
|
+
for (const runId of runIds) {
|
|
1814
|
+
const groupId = metaByRunId.get(runId)?.groupId;
|
|
1815
|
+
if (!groupId) continue;
|
|
1816
|
+
const existing = byGroup.get(groupId);
|
|
1817
|
+
if (existing) {
|
|
1818
|
+
existing.runIds.push(runId);
|
|
1819
|
+
} else {
|
|
1820
|
+
byGroup.set(groupId, {
|
|
1821
|
+
groupId,
|
|
1822
|
+
parentGroupId: metaByRunId.get(runId)?.parentGroupId,
|
|
1823
|
+
runIds: [runId]
|
|
1824
|
+
});
|
|
1793
1825
|
}
|
|
1794
|
-
}
|
|
1826
|
+
}
|
|
1827
|
+
return [...byGroup.values()].map((group) => ({
|
|
1828
|
+
...group,
|
|
1829
|
+
runIds: [...group.runIds].sort((a, b) => a.localeCompare(b))
|
|
1830
|
+
}));
|
|
1795
1831
|
}
|
|
1796
|
-
function
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
...{}
|
|
1805
|
-
}
|
|
1832
|
+
function buildHandoffs(runIds, metaByRunId, warnings, sessionId) {
|
|
1833
|
+
const edges = [];
|
|
1834
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1835
|
+
const pushEdge = (edge) => {
|
|
1836
|
+
const key = `${edge.from}->${edge.to}:${edge.confidence}`;
|
|
1837
|
+
if (seen.has(key)) return;
|
|
1838
|
+
seen.add(key);
|
|
1839
|
+
edges.push(edge);
|
|
1806
1840
|
};
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
1819
|
-
const logical = [];
|
|
1820
|
-
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
1821
|
-
for (const runId of orderedRuns) {
|
|
1822
|
-
const runEvents = byRun.get(runId) ?? [];
|
|
1823
|
-
const starts = [];
|
|
1824
|
-
const completes = [];
|
|
1825
|
-
const others = [];
|
|
1826
|
-
for (const event of runEvents) {
|
|
1827
|
-
const legacy = legacyEvent(event);
|
|
1828
|
-
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
1829
|
-
starts.push(event);
|
|
1830
|
-
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
1831
|
-
completes.push(event);
|
|
1832
|
-
} else {
|
|
1833
|
-
others.push(event);
|
|
1834
|
-
}
|
|
1841
|
+
for (const runId of runIds) {
|
|
1842
|
+
const meta = metaByRunId.get(runId);
|
|
1843
|
+
if (!meta) continue;
|
|
1844
|
+
if (meta.handoffFrom && meta.handoffTo) {
|
|
1845
|
+
pushEdge({
|
|
1846
|
+
from: meta.handoffFrom,
|
|
1847
|
+
to: meta.handoffTo,
|
|
1848
|
+
source: "manual",
|
|
1849
|
+
confidence: "explicit"
|
|
1850
|
+
});
|
|
1851
|
+
continue;
|
|
1835
1852
|
}
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
);
|
|
1844
|
-
if (candidates.length > 1) {
|
|
1845
|
-
diagnostics.push({
|
|
1846
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1847
|
-
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
1848
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
1849
|
-
});
|
|
1850
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1851
|
-
}
|
|
1852
|
-
match = candidates[0];
|
|
1853
|
-
} else if (stepId) {
|
|
1854
|
-
const candidates = completes.filter(
|
|
1855
|
-
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
1856
|
-
);
|
|
1857
|
-
if (candidates.length > 1) {
|
|
1858
|
-
diagnostics.push({
|
|
1859
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1860
|
-
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
1861
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
1862
|
-
});
|
|
1863
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1864
|
-
}
|
|
1865
|
-
match = candidates[0];
|
|
1866
|
-
}
|
|
1867
|
-
if (match) {
|
|
1868
|
-
usedCompletes.add(match.eventId);
|
|
1869
|
-
absorbedIds.add(match.eventId);
|
|
1870
|
-
const paired = pairStartComplete(start, match);
|
|
1871
|
-
logical.push(paired);
|
|
1872
|
-
logicalByRawId.set(start.eventId, paired);
|
|
1873
|
-
logicalByRawId.set(match.eventId, paired);
|
|
1874
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
1875
|
-
} else {
|
|
1876
|
-
diagnostics.push({
|
|
1877
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
1878
|
-
message: `No matching complete for start ${start.eventId}.`,
|
|
1879
|
-
eventIds: [start.eventId]
|
|
1880
|
-
});
|
|
1881
|
-
const alone = asLogical(start);
|
|
1882
|
-
logical.push(alone);
|
|
1883
|
-
logicalByRawId.set(start.eventId, alone);
|
|
1884
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1885
|
-
}
|
|
1853
|
+
if (meta.handoffFrom) {
|
|
1854
|
+
pushEdge({
|
|
1855
|
+
from: meta.handoffFrom,
|
|
1856
|
+
to: runId,
|
|
1857
|
+
source: "manual",
|
|
1858
|
+
confidence: "explicit"
|
|
1859
|
+
});
|
|
1886
1860
|
}
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1861
|
+
if (meta.handoffTo) {
|
|
1862
|
+
pushEdge({
|
|
1863
|
+
from: runId,
|
|
1864
|
+
to: meta.handoffTo,
|
|
1865
|
+
source: "manual",
|
|
1866
|
+
confidence: "explicit"
|
|
1893
1867
|
});
|
|
1894
|
-
const alone = asLogical(complete);
|
|
1895
|
-
logical.push(alone);
|
|
1896
|
-
logicalByRawId.set(complete.eventId, alone);
|
|
1897
|
-
const stepId = stepIdOf(complete);
|
|
1898
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1899
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1900
|
-
}
|
|
1901
1868
|
}
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1869
|
+
if (meta.subAgentId && meta.parentGroupId && !meta.handoffFrom && !meta.handoffTo) {
|
|
1870
|
+
pushEdge({
|
|
1871
|
+
from: meta.parentGroupId,
|
|
1872
|
+
to: meta.subAgentId,
|
|
1873
|
+
source: "inferred",
|
|
1874
|
+
confidence: "correlated"
|
|
1875
|
+
});
|
|
1876
|
+
warnings.push({
|
|
1877
|
+
code: "ambiguous-handoff-endpoints",
|
|
1878
|
+
message: "Handoff inferred from parentGroupId and subAgentId without explicit handoffFrom/handoffTo.",
|
|
1879
|
+
runId,
|
|
1880
|
+
sessionId
|
|
1881
|
+
});
|
|
1910
1882
|
}
|
|
1911
1883
|
}
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1884
|
+
return edges.sort((a, b) => {
|
|
1885
|
+
const from = a.from.localeCompare(b.from);
|
|
1886
|
+
if (from !== 0) return from;
|
|
1887
|
+
return a.to.localeCompare(b.to);
|
|
1888
|
+
});
|
|
1889
|
+
}
|
|
1890
|
+
function buildRetries(runIds, metaByRunId, warnings, sessionId) {
|
|
1891
|
+
const retries = [];
|
|
1892
|
+
for (const runId of runIds) {
|
|
1893
|
+
const meta = metaByRunId.get(runId);
|
|
1894
|
+
if (!meta) continue;
|
|
1895
|
+
if (meta.retryOf) {
|
|
1896
|
+
retries.push({
|
|
1897
|
+
runId,
|
|
1898
|
+
retryOf: meta.retryOf,
|
|
1899
|
+
attempt: meta.attempt,
|
|
1900
|
+
source: "manual",
|
|
1901
|
+
confidence: "explicit"
|
|
1902
|
+
});
|
|
1918
1903
|
continue;
|
|
1919
1904
|
}
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1905
|
+
if (meta.attempt !== void 0 && meta.attempt > 1) {
|
|
1906
|
+
retries.push({
|
|
1907
|
+
runId,
|
|
1908
|
+
attempt: meta.attempt,
|
|
1909
|
+
source: "inferred",
|
|
1910
|
+
confidence: "correlated"
|
|
1911
|
+
});
|
|
1912
|
+
warnings.push({
|
|
1913
|
+
code: "ambiguous-retry-link",
|
|
1914
|
+
message: "attempt > 1 without retryOf; retry link is correlated only.",
|
|
1915
|
+
runId,
|
|
1916
|
+
sessionId
|
|
1917
|
+
});
|
|
1932
1918
|
}
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1919
|
+
}
|
|
1920
|
+
return retries.sort((a, b) => a.runId.localeCompare(b.runId));
|
|
1921
|
+
}
|
|
1922
|
+
function buildCriticalPath(runs, handoffs) {
|
|
1923
|
+
const runById = new Map(runs.map((run) => [run.runId, run]));
|
|
1924
|
+
const explicitTargets = new Set(
|
|
1925
|
+
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.to)
|
|
1926
|
+
);
|
|
1927
|
+
const explicitSources = new Set(
|
|
1928
|
+
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
1929
|
+
);
|
|
1930
|
+
const ordered = [...runs].sort(compareRuns);
|
|
1931
|
+
const path18 = [];
|
|
1932
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1933
|
+
const pushRun = (run, confidence, source) => {
|
|
1934
|
+
if (visited.has(run.runId)) return;
|
|
1935
|
+
visited.add(run.runId);
|
|
1936
|
+
path18.push({
|
|
1937
|
+
runId: run.runId,
|
|
1938
|
+
name: run.name,
|
|
1939
|
+
startedAt: run.startedAt,
|
|
1940
|
+
durationMs: run.durationMs,
|
|
1941
|
+
confidence,
|
|
1942
|
+
source
|
|
1943
|
+
});
|
|
1944
|
+
};
|
|
1945
|
+
for (const edge of handoffs) {
|
|
1946
|
+
if (edge.confidence !== "explicit") continue;
|
|
1947
|
+
const fromRun = [...runById.values()].find(
|
|
1948
|
+
(run) => run.runId === edge.from || metaRunIdMatches(run, edge.from, runById)
|
|
1949
|
+
);
|
|
1950
|
+
const toRun = [...runById.values()].find(
|
|
1951
|
+
(run) => run.runId === edge.to || metaRunIdMatches(run, edge.to, runById)
|
|
1952
|
+
);
|
|
1953
|
+
if (fromRun) pushRun(fromRun, "explicit", "manual");
|
|
1954
|
+
if (toRun) pushRun(toRun, "explicit", "manual");
|
|
1955
|
+
}
|
|
1956
|
+
for (const run of ordered) {
|
|
1957
|
+
if (visited.has(run.runId)) continue;
|
|
1958
|
+
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
1959
|
+
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
1960
|
+
}
|
|
1961
|
+
return path18;
|
|
1962
|
+
}
|
|
1963
|
+
function metaRunIdMatches(run, token, runById) {
|
|
1964
|
+
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1965
|
+
return meta?.subAgentId === token || meta?.groupId === token || runById.has(token);
|
|
1966
|
+
}
|
|
1967
|
+
function buildSessionIndex(inputRuns, options = {}) {
|
|
1968
|
+
const warnings = [];
|
|
1969
|
+
const runs = [...inputRuns].sort(compareRuns);
|
|
1970
|
+
const metaByRunId = /* @__PURE__ */ new Map();
|
|
1971
|
+
for (const run of runs) {
|
|
1972
|
+
metaByRunId.set(run.runId, extractSessionWorkflowMetadata(run.metadata));
|
|
1973
|
+
}
|
|
1974
|
+
const sessionsByKey = /* @__PURE__ */ new Map();
|
|
1975
|
+
const unscopedRunIds = [];
|
|
1976
|
+
for (const run of runs) {
|
|
1977
|
+
const meta = metaByRunId.get(run.runId);
|
|
1978
|
+
const key = sessionKeyForRun(meta, {
|
|
1979
|
+
correlateByGroupId: options.correlateByGroupId === true
|
|
1980
|
+
});
|
|
1981
|
+
if (!key) {
|
|
1982
|
+
unscopedRunIds.push(run.runId);
|
|
1967
1983
|
continue;
|
|
1968
1984
|
}
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
}
|
|
1985
|
+
const bucket = sessionsByKey.get(key) ?? [];
|
|
1986
|
+
bucket.push(run);
|
|
1987
|
+
sessionsByKey.set(key, bucket);
|
|
1988
|
+
}
|
|
1989
|
+
const sessions = [...sessionsByKey.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([sessionId, sessionRuns]) => {
|
|
1990
|
+
const runIds = sessionRuns.map((run) => run.runId).sort();
|
|
1991
|
+
const handoffs = buildHandoffs(runIds, metaByRunId, warnings, sessionId);
|
|
1992
|
+
const retries = buildRetries(runIds, metaByRunId, warnings, sessionId);
|
|
1993
|
+
const groups = buildGroups(runIds, metaByRunId);
|
|
1994
|
+
const criticalPath = buildCriticalPath(sessionRuns, handoffs);
|
|
1995
|
+
const confidences = new Set(handoffs.map((edge) => edge.confidence));
|
|
1996
|
+
if (confidences.has("explicit") && confidences.has("correlated")) {
|
|
1997
|
+
warnings.push({
|
|
1998
|
+
code: "mixed-confidence-group",
|
|
1999
|
+
message: "Session aggregates explicit and correlated handoff edges.",
|
|
2000
|
+
sessionId
|
|
1986
2001
|
});
|
|
1987
|
-
continue;
|
|
1988
2002
|
}
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2003
|
+
return enrichSessionSummary(
|
|
2004
|
+
{
|
|
2005
|
+
sessionId,
|
|
2006
|
+
runIds,
|
|
2007
|
+
groups,
|
|
2008
|
+
handoffs,
|
|
2009
|
+
retries,
|
|
2010
|
+
criticalPath
|
|
2011
|
+
},
|
|
2012
|
+
runs,
|
|
2013
|
+
{
|
|
2014
|
+
nowMs: options.nowMs,
|
|
2015
|
+
staleThresholdMs: options.staleThresholdMs
|
|
2001
2016
|
}
|
|
2017
|
+
);
|
|
2018
|
+
});
|
|
2019
|
+
if (sessions.length === 0 && runs.length > 0) {
|
|
2020
|
+
warnings.push({
|
|
2021
|
+
code: "missing-session-id",
|
|
2022
|
+
message: "No sessionId (or correlated groupId) found on input runs."
|
|
2002
2023
|
});
|
|
2003
2024
|
}
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
2025
|
+
warnings.sort((a, b) => {
|
|
2026
|
+
const code = a.code.localeCompare(b.code);
|
|
2027
|
+
if (code !== 0) return code;
|
|
2028
|
+
return (a.runId ?? "").localeCompare(b.runId ?? "");
|
|
2009
2029
|
});
|
|
2010
2030
|
return {
|
|
2011
|
-
|
|
2012
|
-
|
|
2031
|
+
runs,
|
|
2032
|
+
sessions,
|
|
2033
|
+
unscopedRunIds: unscopedRunIds.sort(),
|
|
2034
|
+
warnings
|
|
2013
2035
|
};
|
|
2014
2036
|
}
|
|
2015
|
-
function resolveCanonicalToolName(event) {
|
|
2016
|
-
const attrs = event.attributes;
|
|
2017
|
-
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
2018
|
-
if (direct) return direct;
|
|
2019
|
-
const metadata = attrs?.metadata;
|
|
2020
|
-
if (isRecord5(metadata)) {
|
|
2021
|
-
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
2022
|
-
if (nested) return nested;
|
|
2023
|
-
}
|
|
2024
|
-
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
2025
|
-
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
2026
|
-
}
|
|
2027
|
-
return event.name;
|
|
2028
|
-
}
|
|
2029
|
-
function pickString(record, keys) {
|
|
2030
|
-
if (!record) return void 0;
|
|
2031
|
-
for (const key of keys) {
|
|
2032
|
-
const value = record[key];
|
|
2033
|
-
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
2034
|
-
}
|
|
2035
|
-
return void 0;
|
|
2036
|
-
}
|
|
2037
2037
|
|
|
2038
2038
|
// packages/core/src/checks/trace-facts.ts
|
|
2039
2039
|
formatProgrammaticDiagnostic(
|
|
@@ -2068,7 +2068,11 @@ function emptySummary() {
|
|
|
2068
2068
|
failed: 0,
|
|
2069
2069
|
warnings: 0,
|
|
2070
2070
|
errors: 0,
|
|
2071
|
-
rulesEvaluated: 0
|
|
2071
|
+
rulesEvaluated: 0,
|
|
2072
|
+
rulesPassed: 0,
|
|
2073
|
+
rulesWarning: 0,
|
|
2074
|
+
rulesFailed: 0,
|
|
2075
|
+
rulesError: 0
|
|
2072
2076
|
};
|
|
2073
2077
|
}
|
|
2074
2078
|
function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
@@ -2080,7 +2084,11 @@ function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
|
2080
2084
|
summary: {
|
|
2081
2085
|
...emptySummary(),
|
|
2082
2086
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2083
|
-
rulesEvaluated: ruleExecutions.length
|
|
2087
|
+
rulesEvaluated: ruleExecutions.length,
|
|
2088
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
2089
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
2090
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
2091
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
2084
2092
|
},
|
|
2085
2093
|
findings: [],
|
|
2086
2094
|
diagnostics: [...diagnostics],
|
|
@@ -2240,7 +2248,7 @@ function normalizeFinding(rule, finding) {
|
|
|
2240
2248
|
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
2241
2249
|
};
|
|
2242
2250
|
}
|
|
2243
|
-
function summarize(findings, diagnostics,
|
|
2251
|
+
function summarize(findings, diagnostics, ruleExecutions) {
|
|
2244
2252
|
return {
|
|
2245
2253
|
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
2246
2254
|
failed: findings.filter(
|
|
@@ -2250,7 +2258,11 @@ function summarize(findings, diagnostics, rulesEvaluated) {
|
|
|
2250
2258
|
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
2251
2259
|
).length,
|
|
2252
2260
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2253
|
-
rulesEvaluated
|
|
2261
|
+
rulesEvaluated: ruleExecutions.length,
|
|
2262
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
2263
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
2264
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
2265
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
2254
2266
|
};
|
|
2255
2267
|
}
|
|
2256
2268
|
function classifyRuleExecution(findings, threw) {
|
|
@@ -2727,7 +2739,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
2727
2739
|
}
|
|
2728
2740
|
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
2729
2741
|
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
2730
|
-
const summary = summarize(sortedFindings, diagnostics, ruleExecutions
|
|
2742
|
+
const summary = summarize(sortedFindings, diagnostics, ruleExecutions);
|
|
2731
2743
|
const status = summary.failed > 0 ? "fail" : "pass";
|
|
2732
2744
|
return {
|
|
2733
2745
|
ok: status === "pass",
|
|
@@ -3162,10 +3174,143 @@ function asStringArray(value, label) {
|
|
|
3162
3174
|
function asPositiveNumber(value, label) {
|
|
3163
3175
|
if (value === void 0) return void 0;
|
|
3164
3176
|
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
3165
|
-
throw new Error(`${label} must be a non-negative number.`);
|
|
3177
|
+
throw new Error(`${label} must be a non-negative finite number.`);
|
|
3178
|
+
}
|
|
3179
|
+
return value;
|
|
3180
|
+
}
|
|
3181
|
+
function asBoolean(value, label) {
|
|
3182
|
+
if (value === void 0) return void 0;
|
|
3183
|
+
if (typeof value !== "boolean") {
|
|
3184
|
+
throw new Error(`${label} must be a boolean (not a string or other type).`);
|
|
3185
|
+
}
|
|
3186
|
+
return value;
|
|
3187
|
+
}
|
|
3188
|
+
function assertObject(value, label) {
|
|
3189
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
3190
|
+
throw new Error(`${label} must be an object.`);
|
|
3166
3191
|
}
|
|
3167
3192
|
return value;
|
|
3168
3193
|
}
|
|
3194
|
+
function assertKnownKeys(raw, allowed, label) {
|
|
3195
|
+
const allowedSet = new Set(allowed);
|
|
3196
|
+
for (const key of Object.keys(raw)) {
|
|
3197
|
+
if (!allowedSet.has(key)) {
|
|
3198
|
+
throw new Error(`${label} has unknown key "${key}".`);
|
|
3199
|
+
}
|
|
3200
|
+
}
|
|
3201
|
+
}
|
|
3202
|
+
var CASE_KEYS = [
|
|
3203
|
+
"id",
|
|
3204
|
+
"trace",
|
|
3205
|
+
"runId",
|
|
3206
|
+
"input",
|
|
3207
|
+
"requireTools",
|
|
3208
|
+
"forbidTools",
|
|
3209
|
+
"maxDurationMs",
|
|
3210
|
+
"expectedObservations",
|
|
3211
|
+
"expect"
|
|
3212
|
+
];
|
|
3213
|
+
var EXPECT_KEYS = [
|
|
3214
|
+
"checkStatus",
|
|
3215
|
+
"findingRuleIds",
|
|
3216
|
+
"rejectUnexpectedFindings",
|
|
3217
|
+
"minAssertions"
|
|
3218
|
+
];
|
|
3219
|
+
var CHECKS_KEYS = ["select", "run", "tool", "llm"];
|
|
3220
|
+
var CHECKS_RUN_KEYS = ["maxDurationMs", "maxDepth"];
|
|
3221
|
+
var CHECKS_TOOL_KEYS = ["required", "forbidden"];
|
|
3222
|
+
var CHECKS_LLM_KEYS = ["allowedModels", "maxTotalTokens"];
|
|
3223
|
+
var EVAL_KEYS = [
|
|
3224
|
+
"requireSuccess",
|
|
3225
|
+
"requiredTools",
|
|
3226
|
+
"forbiddenTools",
|
|
3227
|
+
"maxDurationMs",
|
|
3228
|
+
"maxDepth",
|
|
3229
|
+
"maxRetries",
|
|
3230
|
+
"maxTotalTokens"
|
|
3231
|
+
];
|
|
3232
|
+
var ROOT_KEYS = [
|
|
3233
|
+
"name",
|
|
3234
|
+
"traces",
|
|
3235
|
+
"cases",
|
|
3236
|
+
"checks",
|
|
3237
|
+
"eval",
|
|
3238
|
+
"redactionProfile",
|
|
3239
|
+
"artifacts",
|
|
3240
|
+
"baseline",
|
|
3241
|
+
"candidate"
|
|
3242
|
+
];
|
|
3243
|
+
function asExpectConfig(value, label) {
|
|
3244
|
+
if (value === void 0) return void 0;
|
|
3245
|
+
const raw = assertObject(value, label);
|
|
3246
|
+
assertKnownKeys(raw, EXPECT_KEYS, label);
|
|
3247
|
+
const checkStatus = raw.checkStatus;
|
|
3248
|
+
if (checkStatus !== "pass" && checkStatus !== "fail" && checkStatus !== "error") {
|
|
3249
|
+
throw new Error(`${label}.checkStatus must be "pass", "fail", or "error".`);
|
|
3250
|
+
}
|
|
3251
|
+
const findingRuleIds = asStringArray(raw.findingRuleIds, `${label}.findingRuleIds`);
|
|
3252
|
+
const minAssertions = asPositiveNumber(raw.minAssertions, `${label}.minAssertions`);
|
|
3253
|
+
const rejectUnexpectedFindings = asBoolean(
|
|
3254
|
+
raw.rejectUnexpectedFindings,
|
|
3255
|
+
`${label}.rejectUnexpectedFindings`
|
|
3256
|
+
);
|
|
3257
|
+
return {
|
|
3258
|
+
checkStatus,
|
|
3259
|
+
...findingRuleIds !== void 0 ? { findingRuleIds } : {},
|
|
3260
|
+
...rejectUnexpectedFindings !== void 0 ? { rejectUnexpectedFindings } : {},
|
|
3261
|
+
...minAssertions !== void 0 ? { minAssertions } : {}
|
|
3262
|
+
};
|
|
3263
|
+
}
|
|
3264
|
+
function asChecksConfig(value, label) {
|
|
3265
|
+
const raw = assertObject(value, label);
|
|
3266
|
+
assertKnownKeys(raw, CHECKS_KEYS, label);
|
|
3267
|
+
const checks = {};
|
|
3268
|
+
const select = asStringArray(raw.select, `${label}.select`);
|
|
3269
|
+
if (select !== void 0) checks.select = select;
|
|
3270
|
+
if (raw.run !== void 0) {
|
|
3271
|
+
const run = assertObject(raw.run, `${label}.run`);
|
|
3272
|
+
assertKnownKeys(run, CHECKS_RUN_KEYS, `${label}.run`);
|
|
3273
|
+
checks.run = {
|
|
3274
|
+
...asPositiveNumber(run.maxDurationMs, `${label}.run.maxDurationMs`) !== void 0 ? { maxDurationMs: asPositiveNumber(run.maxDurationMs, `${label}.run.maxDurationMs`) } : {},
|
|
3275
|
+
...asPositiveNumber(run.maxDepth, `${label}.run.maxDepth`) !== void 0 ? { maxDepth: asPositiveNumber(run.maxDepth, `${label}.run.maxDepth`) } : {}
|
|
3276
|
+
};
|
|
3277
|
+
}
|
|
3278
|
+
if (raw.tool !== void 0) {
|
|
3279
|
+
const tool = assertObject(raw.tool, `${label}.tool`);
|
|
3280
|
+
assertKnownKeys(tool, CHECKS_TOOL_KEYS, `${label}.tool`);
|
|
3281
|
+
checks.tool = {
|
|
3282
|
+
...asStringArray(tool.required, `${label}.tool.required`) !== void 0 ? { required: asStringArray(tool.required, `${label}.tool.required`) } : {},
|
|
3283
|
+
...asStringArray(tool.forbidden, `${label}.tool.forbidden`) !== void 0 ? { forbidden: asStringArray(tool.forbidden, `${label}.tool.forbidden`) } : {}
|
|
3284
|
+
};
|
|
3285
|
+
}
|
|
3286
|
+
if (raw.llm !== void 0) {
|
|
3287
|
+
const llm = assertObject(raw.llm, `${label}.llm`);
|
|
3288
|
+
assertKnownKeys(llm, CHECKS_LLM_KEYS, `${label}.llm`);
|
|
3289
|
+
checks.llm = {
|
|
3290
|
+
...asStringArray(llm.allowedModels, `${label}.llm.allowedModels`) !== void 0 ? { allowedModels: asStringArray(llm.allowedModels, `${label}.llm.allowedModels`) } : {},
|
|
3291
|
+
...asPositiveNumber(llm.maxTotalTokens, `${label}.llm.maxTotalTokens`) !== void 0 ? {
|
|
3292
|
+
maxTotalTokens: asPositiveNumber(
|
|
3293
|
+
llm.maxTotalTokens,
|
|
3294
|
+
`${label}.llm.maxTotalTokens`
|
|
3295
|
+
)
|
|
3296
|
+
} : {}
|
|
3297
|
+
};
|
|
3298
|
+
}
|
|
3299
|
+
return checks;
|
|
3300
|
+
}
|
|
3301
|
+
function asEvalConfig(value, label) {
|
|
3302
|
+
const raw = assertObject(value, label);
|
|
3303
|
+
assertKnownKeys(raw, EVAL_KEYS, label);
|
|
3304
|
+
return {
|
|
3305
|
+
...asBoolean(raw.requireSuccess, `${label}.requireSuccess`) !== void 0 ? { requireSuccess: asBoolean(raw.requireSuccess, `${label}.requireSuccess`) } : {},
|
|
3306
|
+
...asStringArray(raw.requiredTools, `${label}.requiredTools`) !== void 0 ? { requiredTools: asStringArray(raw.requiredTools, `${label}.requiredTools`) } : {},
|
|
3307
|
+
...asStringArray(raw.forbiddenTools, `${label}.forbiddenTools`) !== void 0 ? { forbiddenTools: asStringArray(raw.forbiddenTools, `${label}.forbiddenTools`) } : {},
|
|
3308
|
+
...asPositiveNumber(raw.maxDurationMs, `${label}.maxDurationMs`) !== void 0 ? { maxDurationMs: asPositiveNumber(raw.maxDurationMs, `${label}.maxDurationMs`) } : {},
|
|
3309
|
+
...asPositiveNumber(raw.maxDepth, `${label}.maxDepth`) !== void 0 ? { maxDepth: asPositiveNumber(raw.maxDepth, `${label}.maxDepth`) } : {},
|
|
3310
|
+
...asPositiveNumber(raw.maxRetries, `${label}.maxRetries`) !== void 0 ? { maxRetries: asPositiveNumber(raw.maxRetries, `${label}.maxRetries`) } : {},
|
|
3311
|
+
...asPositiveNumber(raw.maxTotalTokens, `${label}.maxTotalTokens`) !== void 0 ? { maxTotalTokens: asPositiveNumber(raw.maxTotalTokens, `${label}.maxTotalTokens`) } : {}
|
|
3312
|
+
};
|
|
3313
|
+
}
|
|
3169
3314
|
function validateCaseConfig(value, index) {
|
|
3170
3315
|
const diagnostics = [];
|
|
3171
3316
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
@@ -3176,6 +3321,7 @@ function validateCaseConfig(value, index) {
|
|
|
3176
3321
|
}
|
|
3177
3322
|
const raw = value;
|
|
3178
3323
|
try {
|
|
3324
|
+
assertKnownKeys(raw, CASE_KEYS, `cases[${index}]`);
|
|
3179
3325
|
const id = asString(raw.id, `cases[${index}].id`);
|
|
3180
3326
|
if (id === void 0) {
|
|
3181
3327
|
diagnostics.push(
|
|
@@ -3208,7 +3354,8 @@ function validateCaseConfig(value, index) {
|
|
|
3208
3354
|
raw.expectedObservations,
|
|
3209
3355
|
`cases[${index}].expectedObservations`
|
|
3210
3356
|
)
|
|
3211
|
-
} : {}
|
|
3357
|
+
} : {},
|
|
3358
|
+
...asExpectConfig(raw.expect, `cases[${index}].expect`) !== void 0 ? { expect: asExpectConfig(raw.expect, `cases[${index}].expect`) } : {}
|
|
3212
3359
|
},
|
|
3213
3360
|
diagnostics
|
|
3214
3361
|
};
|
|
@@ -3223,6 +3370,7 @@ function normalizeSuiteConfig(value) {
|
|
|
3223
3370
|
throw new Error("Suite config must export an object.");
|
|
3224
3371
|
}
|
|
3225
3372
|
const raw = value;
|
|
3373
|
+
assertKnownKeys(raw, ROOT_KEYS, "suite config");
|
|
3226
3374
|
const name = asString(raw.name, "name");
|
|
3227
3375
|
const traces = asString(raw.traces, "traces");
|
|
3228
3376
|
if (name === void 0) throw new Error("name is required.");
|
|
@@ -3251,25 +3399,15 @@ function normalizeSuiteConfig(value) {
|
|
|
3251
3399
|
}
|
|
3252
3400
|
const config = { name, traces, cases };
|
|
3253
3401
|
if (raw.checks !== void 0) {
|
|
3254
|
-
|
|
3255
|
-
throw new Error("checks must be an object.");
|
|
3256
|
-
}
|
|
3257
|
-
config.checks = raw.checks;
|
|
3402
|
+
config.checks = asChecksConfig(raw.checks, "checks");
|
|
3258
3403
|
}
|
|
3259
3404
|
if (raw.eval !== void 0) {
|
|
3260
|
-
|
|
3261
|
-
throw new Error("eval must be an object.");
|
|
3262
|
-
}
|
|
3263
|
-
config.eval = raw.eval;
|
|
3405
|
+
config.eval = asEvalConfig(raw.eval, "eval");
|
|
3264
3406
|
}
|
|
3265
3407
|
if (raw.artifacts !== void 0) {
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
3269
|
-
const outputDir = asString(
|
|
3270
|
-
raw.artifacts.outputDir,
|
|
3271
|
-
"artifacts.outputDir"
|
|
3272
|
-
);
|
|
3408
|
+
const artifacts = assertObject(raw.artifacts, "artifacts");
|
|
3409
|
+
assertKnownKeys(artifacts, ["outputDir"], "artifacts");
|
|
3410
|
+
const outputDir = asString(artifacts.outputDir, "artifacts.outputDir");
|
|
3273
3411
|
config.artifacts = outputDir !== void 0 ? { outputDir } : {};
|
|
3274
3412
|
}
|
|
3275
3413
|
if (raw.baseline !== void 0) {
|
|
@@ -5794,6 +5932,78 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
5794
5932
|
}
|
|
5795
5933
|
return { ok: diagnostics.length === 0, diagnostics };
|
|
5796
5934
|
}
|
|
5935
|
+
function evaluateCaseExpect(suiteCase, checkResult, observationCount) {
|
|
5936
|
+
const expectation = suiteCase.expect;
|
|
5937
|
+
if (expectation === void 0) {
|
|
5938
|
+
return { ok: true, diagnostics: [] };
|
|
5939
|
+
}
|
|
5940
|
+
const diagnostics = [];
|
|
5941
|
+
if (checkResult.status !== expectation.checkStatus) {
|
|
5942
|
+
diagnostics.push(
|
|
5943
|
+
diagnostic4(
|
|
5944
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5945
|
+
`Expected check status "${expectation.checkStatus}", got "${checkResult.status}".`,
|
|
5946
|
+
"error",
|
|
5947
|
+
suiteCase.id
|
|
5948
|
+
)
|
|
5949
|
+
);
|
|
5950
|
+
}
|
|
5951
|
+
const failedFindings = checkResult.findings.filter(
|
|
5952
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
5953
|
+
);
|
|
5954
|
+
const requiredIds = expectation.findingRuleIds ?? [];
|
|
5955
|
+
for (const ruleId of requiredIds) {
|
|
5956
|
+
if (!failedFindings.some((finding) => finding.ruleId === ruleId)) {
|
|
5957
|
+
diagnostics.push(
|
|
5958
|
+
diagnostic4(
|
|
5959
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5960
|
+
`Expected failing finding for rule "${ruleId}" was not present.`,
|
|
5961
|
+
"error",
|
|
5962
|
+
suiteCase.id
|
|
5963
|
+
)
|
|
5964
|
+
);
|
|
5965
|
+
}
|
|
5966
|
+
}
|
|
5967
|
+
const rejectUnexpected = expectation.rejectUnexpectedFindings !== false;
|
|
5968
|
+
if (rejectUnexpected && requiredIds.length > 0) {
|
|
5969
|
+
for (const finding of failedFindings) {
|
|
5970
|
+
if (!requiredIds.includes(finding.ruleId)) {
|
|
5971
|
+
diagnostics.push(
|
|
5972
|
+
diagnostic4(
|
|
5973
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5974
|
+
`Unexpected failing finding for rule "${finding.ruleId}".`,
|
|
5975
|
+
"error",
|
|
5976
|
+
suiteCase.id
|
|
5977
|
+
)
|
|
5978
|
+
);
|
|
5979
|
+
}
|
|
5980
|
+
}
|
|
5981
|
+
}
|
|
5982
|
+
if (expectation.minAssertions !== void 0) {
|
|
5983
|
+
const assertionCount = checkResult.ruleExecutions.length + observationCount;
|
|
5984
|
+
if (assertionCount < expectation.minAssertions) {
|
|
5985
|
+
diagnostics.push(
|
|
5986
|
+
diagnostic4(
|
|
5987
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5988
|
+
`Expected at least ${expectation.minAssertions} assertion(s), got ${assertionCount}.`,
|
|
5989
|
+
"error",
|
|
5990
|
+
suiteCase.id
|
|
5991
|
+
)
|
|
5992
|
+
);
|
|
5993
|
+
}
|
|
5994
|
+
}
|
|
5995
|
+
if (expectation.checkStatus === "fail" && checkResult.diagnostics.some((item) => item.severity === "error") && checkResult.status === "error") {
|
|
5996
|
+
diagnostics.push(
|
|
5997
|
+
diagnostic4(
|
|
5998
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5999
|
+
'Check ended in evaluator/config error; semantic failure expectation requires status "fail".',
|
|
6000
|
+
"error",
|
|
6001
|
+
suiteCase.id
|
|
6002
|
+
)
|
|
6003
|
+
);
|
|
6004
|
+
}
|
|
6005
|
+
return { ok: diagnostics.length === 0, diagnostics };
|
|
6006
|
+
}
|
|
5797
6007
|
async function runSuiteCase(suiteCase, config, options) {
|
|
5798
6008
|
const resolved = await resolveSuiteCaseTrace(suiteCase, options);
|
|
5799
6009
|
if (resolved.missing || resolved.tracePath === void 0) {
|
|
@@ -5846,9 +6056,21 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
5846
6056
|
status: "pass",
|
|
5847
6057
|
format: read.format,
|
|
5848
6058
|
findings: [],
|
|
5849
|
-
diagnostics: []
|
|
6059
|
+
diagnostics: [],
|
|
6060
|
+
ruleExecutions: []
|
|
6061
|
+
};
|
|
5850
6062
|
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
5851
|
-
const
|
|
6063
|
+
const checkOk = checkResult.ok;
|
|
6064
|
+
const observationsOk = observationResult.ok;
|
|
6065
|
+
const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
|
|
6066
|
+
(finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
|
|
6067
|
+
) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
|
|
6068
|
+
const expectResult = evaluateCaseExpect(
|
|
6069
|
+
suiteCase,
|
|
6070
|
+
checkResult,
|
|
6071
|
+
compiled.observationCount
|
|
6072
|
+
);
|
|
6073
|
+
const checkFailureDiagnostics = [
|
|
5852
6074
|
...checkResult.diagnostics.map(
|
|
5853
6075
|
(item) => diagnostic4(
|
|
5854
6076
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
@@ -5864,15 +6086,24 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
5864
6086
|
finding.severity,
|
|
5865
6087
|
suiteCase.id
|
|
5866
6088
|
)
|
|
5867
|
-
)
|
|
5868
|
-
...observationResult.diagnostics
|
|
6089
|
+
)
|
|
5869
6090
|
];
|
|
5870
|
-
const
|
|
5871
|
-
const
|
|
5872
|
-
|
|
5873
|
-
|
|
5874
|
-
|
|
5875
|
-
|
|
6091
|
+
const expectMatched = suiteCase.expect !== void 0 && expectResult.ok && observationsOk;
|
|
6092
|
+
const diagnostics = expectMatched ? [
|
|
6093
|
+
...observationResult.diagnostics,
|
|
6094
|
+
...expectResult.diagnostics,
|
|
6095
|
+
diagnostic4(
|
|
6096
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
6097
|
+
`Expected semantic failure matched (check.status=${checkResult.status}).`,
|
|
6098
|
+
"info",
|
|
6099
|
+
suiteCase.id
|
|
6100
|
+
)
|
|
6101
|
+
] : [
|
|
6102
|
+
...checkFailureDiagnostics,
|
|
6103
|
+
...observationResult.diagnostics,
|
|
6104
|
+
...expectResult.diagnostics
|
|
6105
|
+
];
|
|
6106
|
+
const ok = suiteCase.expect !== void 0 ? expectResult.ok && observationsOk : checkOk && observationsOk;
|
|
5876
6107
|
const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
|
|
5877
6108
|
return {
|
|
5878
6109
|
id: suiteCase.id,
|