@agent-inspect/viewer 6.31.9 → 6.31.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +930 -791
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.mjs +930 -791
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -1170,859 +1170,870 @@ function buildRunTimeline(events, options = {}) {
|
|
|
1170
1170
|
};
|
|
1171
1171
|
}
|
|
1172
1172
|
|
|
1173
|
-
// packages/core/src/
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1173
|
+
// packages/core/src/diagnostics/programmatic.ts
|
|
1174
|
+
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
1175
|
+
AI_TRACE_INPUT_INVALID: {
|
|
1176
|
+
code: "AI_TRACE_INPUT_INVALID",
|
|
1177
|
+
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
1178
|
+
remediation: "For a file path, use openTraceFile(path).",
|
|
1179
|
+
relatedCodes: ["invalid_input"]
|
|
1180
|
+
},
|
|
1181
|
+
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
1182
|
+
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
1183
|
+
summary: "No trace reader could detect the input format.",
|
|
1184
|
+
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
1185
|
+
relatedCodes: ["unsupported_format"]
|
|
1186
|
+
},
|
|
1187
|
+
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
1188
|
+
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
1189
|
+
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
1190
|
+
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
1191
|
+
relatedCodes: ["ambiguous_format"]
|
|
1192
|
+
},
|
|
1193
|
+
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
1194
|
+
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
1195
|
+
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
1196
|
+
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
1197
|
+
},
|
|
1198
|
+
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
1199
|
+
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
1200
|
+
summary: "Multiple runs are available; select a run before executing checks.",
|
|
1201
|
+
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
1202
|
+
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
1203
|
+
},
|
|
1204
|
+
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
1205
|
+
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1206
|
+
summary: "A parentId equals its own event/step id.",
|
|
1207
|
+
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
1208
|
+
relatedCodes: [
|
|
1209
|
+
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
1210
|
+
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
1211
|
+
]
|
|
1212
|
+
},
|
|
1213
|
+
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
1214
|
+
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
1215
|
+
summary: "Trace contains a parentId cycle.",
|
|
1216
|
+
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
1217
|
+
relatedCodes: ["structure.cycle"]
|
|
1218
|
+
},
|
|
1219
|
+
AI_TRACE_CONTRACT_INPUT_INVALID: {
|
|
1220
|
+
code: "AI_TRACE_CONTRACT_INPUT_INVALID",
|
|
1221
|
+
summary: "evaluateTraceContract expects ({ read }, contract) or use evaluateTraceContractRead(read, contract).",
|
|
1222
|
+
remediation: "Pass TraceCheckInput as the first argument: evaluateTraceContract({ read }, contract).",
|
|
1223
|
+
relatedCodes: ["AI_CHECK_INVALID_ARGUMENTS"]
|
|
1224
|
+
},
|
|
1225
|
+
AI_INSPECT_RUN_UNSUPPORTED_OPTION: {
|
|
1226
|
+
code: "AI_INSPECT_RUN_UNSUPPORTED_OPTION",
|
|
1227
|
+
summary: 'inspectRun does not accept a "writer" option.',
|
|
1228
|
+
remediation: "Use createInspector({ writer }) then inspector.run(...). inspectRun persists via the global JSONL path (schema 0.1)."
|
|
1183
1229
|
}
|
|
1184
|
-
|
|
1230
|
+
});
|
|
1231
|
+
function formatProgrammaticDiagnostic(code, detail) {
|
|
1232
|
+
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
1233
|
+
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
1234
|
+
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
1185
1235
|
}
|
|
1186
1236
|
|
|
1187
|
-
// packages/core/src/
|
|
1188
|
-
function
|
|
1189
|
-
return typeof value === "
|
|
1190
|
-
}
|
|
1191
|
-
function finitePositiveInt(value) {
|
|
1192
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
1193
|
-
return void 0;
|
|
1194
|
-
}
|
|
1195
|
-
return Math.trunc(value);
|
|
1237
|
+
// packages/core/src/checks/logical-events.ts
|
|
1238
|
+
function isRecord5(value) {
|
|
1239
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1196
1240
|
}
|
|
1197
|
-
function
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
let found = false;
|
|
1201
|
-
const assignString = (key, value) => {
|
|
1202
|
-
if (isNonEmptyString3(value)) {
|
|
1203
|
-
out[key] = value.trim();
|
|
1204
|
-
found = true;
|
|
1205
|
-
}
|
|
1206
|
-
};
|
|
1207
|
-
assignString("sessionId", record.sessionId);
|
|
1208
|
-
assignString("conversationId", record.conversationId);
|
|
1209
|
-
assignString("groupId", record.groupId);
|
|
1210
|
-
assignString("parentGroupId", record.parentGroupId);
|
|
1211
|
-
assignString("retryOf", record.retryOf);
|
|
1212
|
-
assignString("retryReason", record.retryReason);
|
|
1213
|
-
assignString("handoffFrom", record.handoffFrom);
|
|
1214
|
-
assignString("handoffTo", record.handoffTo);
|
|
1215
|
-
assignString("subAgentId", record.subAgentId);
|
|
1216
|
-
assignString("subAgentName", record.subAgentName);
|
|
1217
|
-
assignString("jobId", record.jobId);
|
|
1218
|
-
assignString("queueName", record.queueName);
|
|
1219
|
-
assignString("workflowName", record.workflowName);
|
|
1220
|
-
assignString("workflowStep", record.workflowStep);
|
|
1221
|
-
assignString("toolCallId", record.toolCallId);
|
|
1222
|
-
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
1223
|
-
assignString("linkedStepId", record.linkedStepId);
|
|
1224
|
-
assignString("operationId", record.operationId);
|
|
1225
|
-
assignString("attemptId", record.attemptId);
|
|
1226
|
-
assignString("fallbackOf", record.fallbackOf);
|
|
1227
|
-
assignString("idempotencyKey", record.idempotencyKey);
|
|
1228
|
-
assignString("correlationId", record.correlationId);
|
|
1229
|
-
assignString("requestId", record.requestId);
|
|
1230
|
-
assignString("decisionId", record.decisionId);
|
|
1231
|
-
const attempt = finitePositiveInt(record.attempt);
|
|
1232
|
-
if (attempt !== void 0) {
|
|
1233
|
-
out.attempt = attempt;
|
|
1234
|
-
found = true;
|
|
1235
|
-
}
|
|
1236
|
-
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
1237
|
-
if (attemptNumber !== void 0) {
|
|
1238
|
-
out.attemptNumber = attemptNumber;
|
|
1239
|
-
found = true;
|
|
1240
|
-
}
|
|
1241
|
-
return found ? out : void 0;
|
|
1241
|
+
function legacyEvent(event) {
|
|
1242
|
+
const value = event.attributes?.legacyEvent;
|
|
1243
|
+
return typeof value === "string" ? value : void 0;
|
|
1242
1244
|
}
|
|
1243
|
-
function
|
|
1244
|
-
|
|
1245
|
-
if (
|
|
1246
|
-
return `group:${meta.groupId}`;
|
|
1247
|
-
}
|
|
1245
|
+
function stepIdOf(event) {
|
|
1246
|
+
const value = event.attributes?.stepId;
|
|
1247
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
1248
1248
|
return void 0;
|
|
1249
1249
|
}
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
completed: 1
|
|
1259
|
-
};
|
|
1260
|
-
var EXPLICIT_SESSION_STATUSES = /* @__PURE__ */ new Set([
|
|
1261
|
-
"running",
|
|
1262
|
-
"waiting_input",
|
|
1263
|
-
"idle",
|
|
1264
|
-
"completed",
|
|
1265
|
-
"error",
|
|
1266
|
-
"stale",
|
|
1267
|
-
"unknown"
|
|
1268
|
-
]);
|
|
1269
|
-
function isExplicitSessionStatus(value) {
|
|
1270
|
-
return typeof value === "string" && EXPLICIT_SESSION_STATUSES.has(value);
|
|
1271
|
-
}
|
|
1272
|
-
function activityMs(run) {
|
|
1273
|
-
return run.endedAt ?? run.startedAt ?? 0;
|
|
1250
|
+
function cloneEvent(event) {
|
|
1251
|
+
return {
|
|
1252
|
+
...event,
|
|
1253
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
1254
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
1255
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
1256
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
1257
|
+
};
|
|
1274
1258
|
}
|
|
1275
|
-
function
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1259
|
+
function mergeAttributes(start, complete) {
|
|
1260
|
+
const merged = {
|
|
1261
|
+
...isRecord5(complete.attributes) ? complete.attributes : {},
|
|
1262
|
+
...isRecord5(start.attributes) ? start.attributes : {}
|
|
1263
|
+
};
|
|
1264
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
1265
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
1266
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
1267
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
1280
1268
|
}
|
|
1281
|
-
return
|
|
1269
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
1282
1270
|
}
|
|
1283
|
-
function
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1271
|
+
function pairStartComplete(start, complete) {
|
|
1272
|
+
const attributes = mergeAttributes(start, complete);
|
|
1273
|
+
const paired = {
|
|
1274
|
+
...cloneEvent(start),
|
|
1275
|
+
status: complete.status,
|
|
1276
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
1277
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
1278
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
1279
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
1280
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
1281
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
1282
|
+
};
|
|
1283
|
+
return {
|
|
1284
|
+
...paired,
|
|
1285
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
1286
|
+
projection: {
|
|
1287
|
+
paired: true,
|
|
1288
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
1289
|
+
parentNormalized: false,
|
|
1290
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
1289
1291
|
}
|
|
1290
|
-
}
|
|
1291
|
-
return earliest;
|
|
1292
|
-
}
|
|
1293
|
-
function latestEndWhenAllEnded(runs) {
|
|
1294
|
-
if (runs.length === 0) return void 0;
|
|
1295
|
-
let latest;
|
|
1296
|
-
for (const run of runs) {
|
|
1297
|
-
if (run.endedAt === void 0) return void 0;
|
|
1298
|
-
if (latest === void 0 || run.endedAt > latest) latest = run.endedAt;
|
|
1299
|
-
}
|
|
1300
|
-
return latest;
|
|
1292
|
+
};
|
|
1301
1293
|
}
|
|
1302
|
-
function
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
best = raw;
|
|
1294
|
+
function asLogical(event, extras) {
|
|
1295
|
+
return {
|
|
1296
|
+
...cloneEvent(event),
|
|
1297
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
1298
|
+
projection: {
|
|
1299
|
+
paired: false,
|
|
1300
|
+
absorbedEventIds: Object.freeze([]),
|
|
1301
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
1302
|
+
...{}
|
|
1312
1303
|
}
|
|
1313
|
-
}
|
|
1314
|
-
return best;
|
|
1315
|
-
}
|
|
1316
|
-
function deriveLastError(runs) {
|
|
1317
|
-
const errorRuns = runs.filter((run) => run.status === "error").sort((a, b) => activityMs(b) - activityMs(a));
|
|
1318
|
-
const latest = errorRuns[0];
|
|
1319
|
-
if (!latest) return void 0;
|
|
1320
|
-
const meta = latest.metadata ?? {};
|
|
1321
|
-
const message = typeof meta.errorMessage === "string" && meta.errorMessage.trim() !== "" ? meta.errorMessage.trim() : latest.name ?? latest.runId;
|
|
1322
|
-
const code = typeof meta.errorCode === "string" && meta.errorCode.trim() !== "" ? meta.errorCode.trim() : void 0;
|
|
1323
|
-
return { runId: latest.runId, message, code };
|
|
1304
|
+
};
|
|
1324
1305
|
}
|
|
1325
|
-
function
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
if (!summary || typeof summary !== "object") continue;
|
|
1333
|
-
const record = summary;
|
|
1334
|
-
if (typeof record.pass === "number") {
|
|
1335
|
-
pass += record.pass;
|
|
1336
|
-
found = true;
|
|
1337
|
-
}
|
|
1338
|
-
if (typeof record.fail === "number") {
|
|
1339
|
-
fail += record.fail;
|
|
1340
|
-
found = true;
|
|
1341
|
-
}
|
|
1342
|
-
if (typeof record.warn === "number") {
|
|
1343
|
-
warn2 += record.warn;
|
|
1344
|
-
found = true;
|
|
1345
|
-
}
|
|
1306
|
+
function projectLogicalEvents(events) {
|
|
1307
|
+
const diagnostics = [];
|
|
1308
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
1309
|
+
for (const event of events) {
|
|
1310
|
+
const list = byRun.get(event.runId) ?? [];
|
|
1311
|
+
list.push(event);
|
|
1312
|
+
byRun.set(event.runId, list);
|
|
1346
1313
|
}
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1314
|
+
const absorbedIds = /* @__PURE__ */ new Set();
|
|
1315
|
+
const logicalByRawId = /* @__PURE__ */ new Map();
|
|
1316
|
+
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
1317
|
+
const logical = [];
|
|
1318
|
+
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
1319
|
+
for (const runId of orderedRuns) {
|
|
1320
|
+
const runEvents = byRun.get(runId) ?? [];
|
|
1321
|
+
const starts = [];
|
|
1322
|
+
const completes = [];
|
|
1323
|
+
const others = [];
|
|
1324
|
+
for (const event of runEvents) {
|
|
1325
|
+
const legacy = legacyEvent(event);
|
|
1326
|
+
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
1327
|
+
starts.push(event);
|
|
1328
|
+
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
1329
|
+
completes.push(event);
|
|
1330
|
+
} else {
|
|
1331
|
+
others.push(event);
|
|
1332
|
+
}
|
|
1354
1333
|
}
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
}
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
checkSummary: deriveCheckSummary(sessionRuns)
|
|
1405
|
-
};
|
|
1406
|
-
}
|
|
1407
|
-
|
|
1408
|
-
// packages/core/src/sessions/load.ts
|
|
1409
|
-
async function enrichSessionRunRecord(meta) {
|
|
1410
|
-
let metadata;
|
|
1411
|
-
try {
|
|
1412
|
-
const events = await readTraceEventsFromFile(meta.filePath);
|
|
1413
|
-
for (const event of events) {
|
|
1414
|
-
if (event.event !== "run_started") continue;
|
|
1415
|
-
if (event.metadata && typeof event.metadata === "object") {
|
|
1416
|
-
metadata = event.metadata;
|
|
1334
|
+
const usedCompletes = /* @__PURE__ */ new Set();
|
|
1335
|
+
for (const start of starts) {
|
|
1336
|
+
const stepId = stepIdOf(start);
|
|
1337
|
+
let match;
|
|
1338
|
+
if (legacyEvent(start) === "run_started") {
|
|
1339
|
+
const candidates = completes.filter(
|
|
1340
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
1341
|
+
);
|
|
1342
|
+
if (candidates.length > 1) {
|
|
1343
|
+
diagnostics.push({
|
|
1344
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1345
|
+
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
1346
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1347
|
+
});
|
|
1348
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1349
|
+
}
|
|
1350
|
+
match = candidates[0];
|
|
1351
|
+
} else if (stepId) {
|
|
1352
|
+
const candidates = completes.filter(
|
|
1353
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
1354
|
+
);
|
|
1355
|
+
if (candidates.length > 1) {
|
|
1356
|
+
diagnostics.push({
|
|
1357
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1358
|
+
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
1359
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1360
|
+
});
|
|
1361
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1362
|
+
}
|
|
1363
|
+
match = candidates[0];
|
|
1364
|
+
}
|
|
1365
|
+
if (match) {
|
|
1366
|
+
usedCompletes.add(match.eventId);
|
|
1367
|
+
absorbedIds.add(match.eventId);
|
|
1368
|
+
const paired = pairStartComplete(start, match);
|
|
1369
|
+
logical.push(paired);
|
|
1370
|
+
logicalByRawId.set(start.eventId, paired);
|
|
1371
|
+
logicalByRawId.set(match.eventId, paired);
|
|
1372
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
1373
|
+
} else {
|
|
1374
|
+
diagnostics.push({
|
|
1375
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
1376
|
+
message: `No matching complete for start ${start.eventId}.`,
|
|
1377
|
+
eventIds: [start.eventId]
|
|
1378
|
+
});
|
|
1379
|
+
const alone = asLogical(start);
|
|
1380
|
+
logical.push(alone);
|
|
1381
|
+
logicalByRawId.set(start.eventId, alone);
|
|
1382
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1417
1383
|
}
|
|
1418
|
-
break;
|
|
1419
1384
|
}
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
startedAt: meta.startedAt,
|
|
1427
|
-
endedAt: meta.endedAt,
|
|
1428
|
-
durationMs: meta.durationMs,
|
|
1429
|
-
filePath: meta.filePath,
|
|
1430
|
-
metadata
|
|
1431
|
-
};
|
|
1432
|
-
}
|
|
1433
|
-
async function loadSessionRunRecords(metas) {
|
|
1434
|
-
const out = [];
|
|
1435
|
-
for (const meta of metas) {
|
|
1436
|
-
out.push(await enrichSessionRunRecord(meta));
|
|
1437
|
-
}
|
|
1438
|
-
return out;
|
|
1439
|
-
}
|
|
1440
|
-
|
|
1441
|
-
// packages/core/src/sessions/index.ts
|
|
1442
|
-
function compareRuns(a, b) {
|
|
1443
|
-
const aStart = a.startedAt ?? 0;
|
|
1444
|
-
const bStart = b.startedAt ?? 0;
|
|
1445
|
-
if (aStart !== bStart) return aStart - bStart;
|
|
1446
|
-
return a.runId.localeCompare(b.runId);
|
|
1447
|
-
}
|
|
1448
|
-
function buildGroups(runIds, metaByRunId) {
|
|
1449
|
-
const byGroup = /* @__PURE__ */ new Map();
|
|
1450
|
-
for (const runId of runIds) {
|
|
1451
|
-
const groupId = metaByRunId.get(runId)?.groupId;
|
|
1452
|
-
if (!groupId) continue;
|
|
1453
|
-
const existing = byGroup.get(groupId);
|
|
1454
|
-
if (existing) {
|
|
1455
|
-
existing.runIds.push(runId);
|
|
1456
|
-
} else {
|
|
1457
|
-
byGroup.set(groupId, {
|
|
1458
|
-
groupId,
|
|
1459
|
-
parentGroupId: metaByRunId.get(runId)?.parentGroupId,
|
|
1460
|
-
runIds: [runId]
|
|
1385
|
+
for (const complete of completes) {
|
|
1386
|
+
if (usedCompletes.has(complete.eventId)) continue;
|
|
1387
|
+
diagnostics.push({
|
|
1388
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
1389
|
+
message: `No matching start for complete ${complete.eventId}.`,
|
|
1390
|
+
eventIds: [complete.eventId]
|
|
1461
1391
|
});
|
|
1392
|
+
const alone = asLogical(complete);
|
|
1393
|
+
logical.push(alone);
|
|
1394
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
1395
|
+
const stepId = stepIdOf(complete);
|
|
1396
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1397
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
for (const event of others) {
|
|
1401
|
+
const alone = asLogical(event);
|
|
1402
|
+
logical.push(alone);
|
|
1403
|
+
logicalByRawId.set(event.eventId, alone);
|
|
1404
|
+
const stepId = stepIdOf(event);
|
|
1405
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1406
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1407
|
+
}
|
|
1462
1408
|
}
|
|
1463
1409
|
}
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
const edges = [];
|
|
1471
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1472
|
-
const pushEdge = (edge) => {
|
|
1473
|
-
const key = `${edge.from}->${edge.to}:${edge.confidence}`;
|
|
1474
|
-
if (seen.has(key)) return;
|
|
1475
|
-
seen.add(key);
|
|
1476
|
-
edges.push(edge);
|
|
1477
|
-
};
|
|
1478
|
-
for (const runId of runIds) {
|
|
1479
|
-
const meta = metaByRunId.get(runId);
|
|
1480
|
-
if (!meta) continue;
|
|
1481
|
-
if (meta.handoffFrom && meta.handoffTo) {
|
|
1482
|
-
pushEdge({
|
|
1483
|
-
from: meta.handoffFrom,
|
|
1484
|
-
to: meta.handoffTo,
|
|
1485
|
-
source: "manual",
|
|
1486
|
-
confidence: "explicit"
|
|
1487
|
-
});
|
|
1410
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
1411
|
+
const normalized = [];
|
|
1412
|
+
for (const event of logical) {
|
|
1413
|
+
const originalParentId = event.parentId;
|
|
1414
|
+
if (!originalParentId) {
|
|
1415
|
+
normalized.push(event);
|
|
1488
1416
|
continue;
|
|
1489
1417
|
}
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1418
|
+
let nextParent = originalParentId;
|
|
1419
|
+
let remapped = false;
|
|
1420
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
1421
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
1422
|
+
nextParent = viaAbsorbed.eventId;
|
|
1423
|
+
remapped = true;
|
|
1424
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
1425
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
1426
|
+
if (viaStep) {
|
|
1427
|
+
nextParent = viaStep;
|
|
1428
|
+
remapped = true;
|
|
1429
|
+
}
|
|
1497
1430
|
}
|
|
1498
|
-
if (
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1431
|
+
if (!remapped) {
|
|
1432
|
+
if (originalParentId === event.eventId) {
|
|
1433
|
+
diagnostics.push({
|
|
1434
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1435
|
+
message: formatProgrammaticDiagnostic(
|
|
1436
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1437
|
+
`Removed self-parent edge on ${event.eventId}.`
|
|
1438
|
+
),
|
|
1439
|
+
eventIds: [event.eventId]
|
|
1440
|
+
});
|
|
1441
|
+
const { parentId: _drop, ...rest } = event;
|
|
1442
|
+
normalized.push({
|
|
1443
|
+
...rest,
|
|
1444
|
+
projection: {
|
|
1445
|
+
...event.projection,
|
|
1446
|
+
parentNormalized: true,
|
|
1447
|
+
originalParentId
|
|
1448
|
+
}
|
|
1449
|
+
});
|
|
1450
|
+
continue;
|
|
1451
|
+
}
|
|
1452
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
1453
|
+
const mapping = event.attributes?.parentMapping;
|
|
1454
|
+
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
1455
|
+
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
1456
|
+
if (!unresolved) {
|
|
1457
|
+
diagnostics.push({
|
|
1458
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
1459
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
1460
|
+
eventIds: [event.eventId]
|
|
1461
|
+
});
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
normalized.push(event);
|
|
1465
|
+
continue;
|
|
1505
1466
|
}
|
|
1506
|
-
if (
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1467
|
+
if (nextParent === event.eventId) {
|
|
1468
|
+
diagnostics.push({
|
|
1469
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1470
|
+
message: formatProgrammaticDiagnostic(
|
|
1471
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1472
|
+
`Removed self-parent edge on ${event.eventId} (was ${originalParentId}).`
|
|
1473
|
+
),
|
|
1474
|
+
eventIds: [event.eventId]
|
|
1512
1475
|
});
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1476
|
+
const { parentId: _drop, ...rest } = event;
|
|
1477
|
+
normalized.push({
|
|
1478
|
+
...rest,
|
|
1479
|
+
projection: {
|
|
1480
|
+
...event.projection,
|
|
1481
|
+
parentNormalized: true,
|
|
1482
|
+
originalParentId
|
|
1483
|
+
}
|
|
1518
1484
|
});
|
|
1485
|
+
continue;
|
|
1519
1486
|
}
|
|
1487
|
+
diagnostics.push({
|
|
1488
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
1489
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
1490
|
+
eventIds: [event.eventId]
|
|
1491
|
+
});
|
|
1492
|
+
normalized.push({
|
|
1493
|
+
...event,
|
|
1494
|
+
parentId: nextParent,
|
|
1495
|
+
projection: {
|
|
1496
|
+
...event.projection,
|
|
1497
|
+
parentNormalized: true,
|
|
1498
|
+
originalParentId
|
|
1499
|
+
}
|
|
1500
|
+
});
|
|
1520
1501
|
}
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1502
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
1503
|
+
normalized.sort((a, b) => {
|
|
1504
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
1505
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
1506
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
1525
1507
|
});
|
|
1508
|
+
return {
|
|
1509
|
+
logicalEvents: Object.freeze(normalized),
|
|
1510
|
+
diagnostics: Object.freeze(diagnostics)
|
|
1511
|
+
};
|
|
1526
1512
|
}
|
|
1527
|
-
function
|
|
1528
|
-
const
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1513
|
+
function resolveCanonicalToolName(event) {
|
|
1514
|
+
const attrs = event.attributes;
|
|
1515
|
+
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
1516
|
+
if (direct) return direct;
|
|
1517
|
+
const metadata = attrs?.metadata;
|
|
1518
|
+
if (isRecord5(metadata)) {
|
|
1519
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
1520
|
+
if (nested) return nested;
|
|
1521
|
+
}
|
|
1522
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
1523
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
1524
|
+
}
|
|
1525
|
+
return event.name;
|
|
1526
|
+
}
|
|
1527
|
+
function pickString(record, keys) {
|
|
1528
|
+
if (!record) return void 0;
|
|
1529
|
+
for (const key of keys) {
|
|
1530
|
+
const value = record[key];
|
|
1531
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
1532
|
+
}
|
|
1533
|
+
return void 0;
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
// packages/core/src/search.ts
|
|
1537
|
+
async function loadTraceMetadataList(_traceDir, fileNames, getPath) {
|
|
1538
|
+
const metas = [];
|
|
1539
|
+
for (const fileName of fileNames) {
|
|
1540
|
+
try {
|
|
1541
|
+
const filePath = getPath(fileName);
|
|
1542
|
+
const meta = await extractMetadata(filePath);
|
|
1543
|
+
metas.push(meta);
|
|
1544
|
+
} catch {
|
|
1555
1545
|
}
|
|
1556
1546
|
}
|
|
1557
|
-
return
|
|
1547
|
+
return metas;
|
|
1558
1548
|
}
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
source
|
|
1580
|
-
});
|
|
1549
|
+
|
|
1550
|
+
// packages/core/src/sessions/metadata.ts
|
|
1551
|
+
function isNonEmptyString3(value) {
|
|
1552
|
+
return typeof value === "string" && value.trim() !== "";
|
|
1553
|
+
}
|
|
1554
|
+
function finitePositiveInt(value) {
|
|
1555
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
1556
|
+
return void 0;
|
|
1557
|
+
}
|
|
1558
|
+
return Math.trunc(value);
|
|
1559
|
+
}
|
|
1560
|
+
function extractSessionWorkflowMetadata(record) {
|
|
1561
|
+
if (!record) return void 0;
|
|
1562
|
+
const out = {};
|
|
1563
|
+
let found = false;
|
|
1564
|
+
const assignString = (key, value) => {
|
|
1565
|
+
if (isNonEmptyString3(value)) {
|
|
1566
|
+
out[key] = value.trim();
|
|
1567
|
+
found = true;
|
|
1568
|
+
}
|
|
1581
1569
|
};
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1570
|
+
assignString("sessionId", record.sessionId);
|
|
1571
|
+
assignString("conversationId", record.conversationId);
|
|
1572
|
+
assignString("groupId", record.groupId);
|
|
1573
|
+
assignString("parentGroupId", record.parentGroupId);
|
|
1574
|
+
assignString("retryOf", record.retryOf);
|
|
1575
|
+
assignString("retryReason", record.retryReason);
|
|
1576
|
+
assignString("handoffFrom", record.handoffFrom);
|
|
1577
|
+
assignString("handoffTo", record.handoffTo);
|
|
1578
|
+
assignString("subAgentId", record.subAgentId);
|
|
1579
|
+
assignString("subAgentName", record.subAgentName);
|
|
1580
|
+
assignString("jobId", record.jobId);
|
|
1581
|
+
assignString("queueName", record.queueName);
|
|
1582
|
+
assignString("workflowName", record.workflowName);
|
|
1583
|
+
assignString("workflowStep", record.workflowStep);
|
|
1584
|
+
assignString("toolCallId", record.toolCallId);
|
|
1585
|
+
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
1586
|
+
assignString("linkedStepId", record.linkedStepId);
|
|
1587
|
+
assignString("operationId", record.operationId);
|
|
1588
|
+
assignString("attemptId", record.attemptId);
|
|
1589
|
+
assignString("fallbackOf", record.fallbackOf);
|
|
1590
|
+
assignString("idempotencyKey", record.idempotencyKey);
|
|
1591
|
+
assignString("correlationId", record.correlationId);
|
|
1592
|
+
assignString("requestId", record.requestId);
|
|
1593
|
+
assignString("decisionId", record.decisionId);
|
|
1594
|
+
const attempt = finitePositiveInt(record.attempt);
|
|
1595
|
+
if (attempt !== void 0) {
|
|
1596
|
+
out.attempt = attempt;
|
|
1597
|
+
found = true;
|
|
1592
1598
|
}
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1599
|
+
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
1600
|
+
if (attemptNumber !== void 0) {
|
|
1601
|
+
out.attemptNumber = attemptNumber;
|
|
1602
|
+
found = true;
|
|
1597
1603
|
}
|
|
1598
|
-
return
|
|
1599
|
-
}
|
|
1600
|
-
function metaRunIdMatches(run, token, runById) {
|
|
1601
|
-
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1602
|
-
return meta?.subAgentId === token || meta?.groupId === token || runById.has(token);
|
|
1604
|
+
return found ? out : void 0;
|
|
1603
1605
|
}
|
|
1604
|
-
function
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
for (const run of runs) {
|
|
1609
|
-
metaByRunId.set(run.runId, extractSessionWorkflowMetadata(run.metadata));
|
|
1606
|
+
function sessionKeyForRun(meta, options) {
|
|
1607
|
+
if (meta?.sessionId) return meta.sessionId;
|
|
1608
|
+
if (options?.correlateByGroupId && meta?.groupId) {
|
|
1609
|
+
return `group:${meta.groupId}`;
|
|
1610
1610
|
}
|
|
1611
|
-
|
|
1612
|
-
|
|
1611
|
+
return void 0;
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
// packages/core/src/sessions/status.ts
|
|
1615
|
+
var DEFAULT_STALE_THRESHOLD_MS = 864e5;
|
|
1616
|
+
var EXPLICIT_STATUS_PRIORITY = {
|
|
1617
|
+
error: 5,
|
|
1618
|
+
waiting_input: 4,
|
|
1619
|
+
idle: 3,
|
|
1620
|
+
stale: 2,
|
|
1621
|
+
completed: 1
|
|
1622
|
+
};
|
|
1623
|
+
var EXPLICIT_SESSION_STATUSES = /* @__PURE__ */ new Set([
|
|
1624
|
+
"running",
|
|
1625
|
+
"waiting_input",
|
|
1626
|
+
"idle",
|
|
1627
|
+
"completed",
|
|
1628
|
+
"error",
|
|
1629
|
+
"stale",
|
|
1630
|
+
"unknown"
|
|
1631
|
+
]);
|
|
1632
|
+
function isExplicitSessionStatus(value) {
|
|
1633
|
+
return typeof value === "string" && EXPLICIT_SESSION_STATUSES.has(value);
|
|
1634
|
+
}
|
|
1635
|
+
function activityMs(run) {
|
|
1636
|
+
return run.endedAt ?? run.startedAt ?? 0;
|
|
1637
|
+
}
|
|
1638
|
+
function latestActivityMs(runs) {
|
|
1639
|
+
let latest = 0;
|
|
1613
1640
|
for (const run of runs) {
|
|
1614
|
-
const
|
|
1615
|
-
|
|
1616
|
-
correlateByGroupId: options.correlateByGroupId === true
|
|
1617
|
-
});
|
|
1618
|
-
if (!key) {
|
|
1619
|
-
unscopedRunIds.push(run.runId);
|
|
1620
|
-
continue;
|
|
1621
|
-
}
|
|
1622
|
-
const bucket = sessionsByKey.get(key) ?? [];
|
|
1623
|
-
bucket.push(run);
|
|
1624
|
-
sessionsByKey.set(key, bucket);
|
|
1641
|
+
const ms = activityMs(run);
|
|
1642
|
+
if (ms > latest) latest = ms;
|
|
1625
1643
|
}
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
warnings.push({
|
|
1635
|
-
code: "mixed-confidence-group",
|
|
1636
|
-
message: "Session aggregates explicit and correlated handoff edges.",
|
|
1637
|
-
sessionId
|
|
1638
|
-
});
|
|
1644
|
+
return latest;
|
|
1645
|
+
}
|
|
1646
|
+
function earliestStart(runs) {
|
|
1647
|
+
let earliest;
|
|
1648
|
+
for (const run of runs) {
|
|
1649
|
+
if (run.startedAt === void 0) continue;
|
|
1650
|
+
if (earliest === void 0 || run.startedAt < earliest) {
|
|
1651
|
+
earliest = run.startedAt;
|
|
1639
1652
|
}
|
|
1640
|
-
return enrichSessionSummary(
|
|
1641
|
-
{
|
|
1642
|
-
sessionId,
|
|
1643
|
-
runIds,
|
|
1644
|
-
groups,
|
|
1645
|
-
handoffs,
|
|
1646
|
-
retries,
|
|
1647
|
-
criticalPath
|
|
1648
|
-
},
|
|
1649
|
-
runs,
|
|
1650
|
-
{
|
|
1651
|
-
nowMs: options.nowMs,
|
|
1652
|
-
staleThresholdMs: options.staleThresholdMs
|
|
1653
|
-
}
|
|
1654
|
-
);
|
|
1655
|
-
});
|
|
1656
|
-
if (sessions.length === 0 && runs.length > 0) {
|
|
1657
|
-
warnings.push({
|
|
1658
|
-
code: "missing-session-id",
|
|
1659
|
-
message: "No sessionId (or correlated groupId) found on input runs."
|
|
1660
|
-
});
|
|
1661
1653
|
}
|
|
1662
|
-
|
|
1663
|
-
const code = a.code.localeCompare(b.code);
|
|
1664
|
-
if (code !== 0) return code;
|
|
1665
|
-
return (a.runId ?? "").localeCompare(b.runId ?? "");
|
|
1666
|
-
});
|
|
1667
|
-
return {
|
|
1668
|
-
runs,
|
|
1669
|
-
sessions,
|
|
1670
|
-
unscopedRunIds: unscopedRunIds.sort(),
|
|
1671
|
-
warnings
|
|
1672
|
-
};
|
|
1654
|
+
return earliest;
|
|
1673
1655
|
}
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
remediation: "For a file path, use openTraceFile(path).",
|
|
1681
|
-
relatedCodes: ["invalid_input"]
|
|
1682
|
-
},
|
|
1683
|
-
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
1684
|
-
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
1685
|
-
summary: "No trace reader could detect the input format.",
|
|
1686
|
-
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
1687
|
-
relatedCodes: ["unsupported_format"]
|
|
1688
|
-
},
|
|
1689
|
-
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
1690
|
-
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
1691
|
-
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
1692
|
-
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
1693
|
-
relatedCodes: ["ambiguous_format"]
|
|
1694
|
-
},
|
|
1695
|
-
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
1696
|
-
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
1697
|
-
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
1698
|
-
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
1699
|
-
},
|
|
1700
|
-
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
1701
|
-
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
1702
|
-
summary: "Multiple runs are available; select a run before executing checks.",
|
|
1703
|
-
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
1704
|
-
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
1705
|
-
},
|
|
1706
|
-
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
1707
|
-
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1708
|
-
summary: "A parentId equals its own event/step id.",
|
|
1709
|
-
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
1710
|
-
relatedCodes: [
|
|
1711
|
-
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
1712
|
-
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
1713
|
-
]
|
|
1714
|
-
},
|
|
1715
|
-
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
1716
|
-
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
1717
|
-
summary: "Trace contains a parentId cycle.",
|
|
1718
|
-
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
1719
|
-
relatedCodes: ["structure.cycle"]
|
|
1656
|
+
function latestEndWhenAllEnded(runs) {
|
|
1657
|
+
if (runs.length === 0) return void 0;
|
|
1658
|
+
let latest;
|
|
1659
|
+
for (const run of runs) {
|
|
1660
|
+
if (run.endedAt === void 0) return void 0;
|
|
1661
|
+
if (latest === void 0 || run.endedAt > latest) latest = run.endedAt;
|
|
1720
1662
|
}
|
|
1721
|
-
|
|
1722
|
-
function formatProgrammaticDiagnostic(code, detail) {
|
|
1723
|
-
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
1724
|
-
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
1725
|
-
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
1663
|
+
return latest;
|
|
1726
1664
|
}
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1665
|
+
function pickExplicitStatus(runs) {
|
|
1666
|
+
let best;
|
|
1667
|
+
let bestPriority = 0;
|
|
1668
|
+
for (const run of runs) {
|
|
1669
|
+
const raw = run.metadata?.sessionStatus;
|
|
1670
|
+
if (!isExplicitSessionStatus(raw)) continue;
|
|
1671
|
+
const priority = EXPLICIT_STATUS_PRIORITY[raw] ?? 0;
|
|
1672
|
+
if (priority > bestPriority) {
|
|
1673
|
+
bestPriority = priority;
|
|
1674
|
+
best = raw;
|
|
1675
|
+
}
|
|
1676
|
+
}
|
|
1677
|
+
return best;
|
|
1731
1678
|
}
|
|
1732
|
-
function
|
|
1733
|
-
const
|
|
1734
|
-
|
|
1679
|
+
function deriveLastError(runs) {
|
|
1680
|
+
const errorRuns = runs.filter((run) => run.status === "error").sort((a, b) => activityMs(b) - activityMs(a));
|
|
1681
|
+
const latest = errorRuns[0];
|
|
1682
|
+
if (!latest) return void 0;
|
|
1683
|
+
const meta = latest.metadata ?? {};
|
|
1684
|
+
const message = typeof meta.errorMessage === "string" && meta.errorMessage.trim() !== "" ? meta.errorMessage.trim() : latest.name ?? latest.runId;
|
|
1685
|
+
const code = typeof meta.errorCode === "string" && meta.errorCode.trim() !== "" ? meta.errorCode.trim() : void 0;
|
|
1686
|
+
return { runId: latest.runId, message, code };
|
|
1735
1687
|
}
|
|
1736
|
-
function
|
|
1737
|
-
|
|
1738
|
-
|
|
1688
|
+
function deriveCheckSummary(runs) {
|
|
1689
|
+
let pass = 0;
|
|
1690
|
+
let fail = 0;
|
|
1691
|
+
let warn2 = 0;
|
|
1692
|
+
let found = false;
|
|
1693
|
+
for (const run of runs) {
|
|
1694
|
+
const summary = run.metadata?.checkSummary;
|
|
1695
|
+
if (!summary || typeof summary !== "object") continue;
|
|
1696
|
+
const record = summary;
|
|
1697
|
+
if (typeof record.pass === "number") {
|
|
1698
|
+
pass += record.pass;
|
|
1699
|
+
found = true;
|
|
1700
|
+
}
|
|
1701
|
+
if (typeof record.fail === "number") {
|
|
1702
|
+
fail += record.fail;
|
|
1703
|
+
found = true;
|
|
1704
|
+
}
|
|
1705
|
+
if (typeof record.warn === "number") {
|
|
1706
|
+
warn2 += record.warn;
|
|
1707
|
+
found = true;
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
return found ? { pass, fail, warn: warn2 } : void 0;
|
|
1711
|
+
}
|
|
1712
|
+
function deriveObservationSummary(runs) {
|
|
1713
|
+
for (const run of [...runs].sort((a, b) => activityMs(b) - activityMs(a))) {
|
|
1714
|
+
const value = run.metadata?.observationSummary;
|
|
1715
|
+
if (typeof value === "string" && value.trim() !== "") {
|
|
1716
|
+
return value.trim();
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1739
1719
|
return void 0;
|
|
1740
1720
|
}
|
|
1741
|
-
function
|
|
1742
|
-
return
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1721
|
+
function deriveSessionStatus(runs, options = {}) {
|
|
1722
|
+
if (runs.length === 0) return "unknown";
|
|
1723
|
+
if (runs.some((run) => run.status === "running")) return "running";
|
|
1724
|
+
const explicit = pickExplicitStatus(runs);
|
|
1725
|
+
if (explicit && explicit !== "running") return explicit;
|
|
1726
|
+
if (runs.some((run) => run.status === "error")) return "error";
|
|
1727
|
+
if (runs.every((run) => run.status === "success")) return "completed";
|
|
1728
|
+
const nowMs = options.nowMs ?? Date.now();
|
|
1729
|
+
const staleThresholdMs = options.staleThresholdMs ?? DEFAULT_STALE_THRESHOLD_MS;
|
|
1730
|
+
const lastMs = latestActivityMs(runs);
|
|
1731
|
+
if (lastMs > 0 && nowMs - lastMs > staleThresholdMs) return "stale";
|
|
1732
|
+
return "unknown";
|
|
1749
1733
|
}
|
|
1750
|
-
function
|
|
1751
|
-
const
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1734
|
+
function enrichSessionSummary(summary, runs, options = {}) {
|
|
1735
|
+
const sessionRuns = runs.filter((run) => summary.runIds.includes(run.runId)).sort((a, b) => a.runId.localeCompare(b.runId));
|
|
1736
|
+
const startedAt = earliestStart(sessionRuns);
|
|
1737
|
+
const endedAt = latestEndWhenAllEnded(sessionRuns);
|
|
1738
|
+
const durationMs = startedAt !== void 0 && endedAt !== void 0 ? endedAt - startedAt : void 0;
|
|
1739
|
+
let correlationId;
|
|
1740
|
+
let jobId;
|
|
1741
|
+
let workflowId;
|
|
1742
|
+
for (const run of sessionRuns) {
|
|
1743
|
+
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1744
|
+
if (!correlationId && meta?.correlationId) correlationId = meta.correlationId;
|
|
1745
|
+
if (!jobId && meta?.jobId) jobId = meta.jobId;
|
|
1746
|
+
if (!workflowId && meta?.workflowName) workflowId = meta.workflowName;
|
|
1747
|
+
else if (!workflowId && meta?.workflowStep) workflowId = meta.workflowStep;
|
|
1759
1748
|
}
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
...cloneEvent(start),
|
|
1766
|
-
status: complete.status,
|
|
1767
|
-
timestamp: complete.timestamp ?? start.timestamp,
|
|
1768
|
-
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
1769
|
-
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
1770
|
-
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
1771
|
-
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
1772
|
-
...attributes !== void 0 ? { attributes } : {}
|
|
1773
|
-
};
|
|
1749
|
+
const lastMs = latestActivityMs(sessionRuns);
|
|
1750
|
+
const lastActivity = lastMs > 0 ? new Date(lastMs).toISOString() : (/* @__PURE__ */ new Date(0)).toISOString();
|
|
1751
|
+
const retryCount2 = summary.retries.filter(
|
|
1752
|
+
(retry) => retry.retryOf !== void 0 || (retry.attempt ?? 0) > 1
|
|
1753
|
+
).length;
|
|
1774
1754
|
return {
|
|
1775
|
-
...
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1755
|
+
...summary,
|
|
1756
|
+
status: deriveSessionStatus(sessionRuns, options),
|
|
1757
|
+
startedAt,
|
|
1758
|
+
endedAt,
|
|
1759
|
+
durationMs,
|
|
1760
|
+
correlationId,
|
|
1761
|
+
jobId,
|
|
1762
|
+
workflowId,
|
|
1763
|
+
lastError: deriveLastError(sessionRuns),
|
|
1764
|
+
lastActivity,
|
|
1765
|
+
retryCount: retryCount2,
|
|
1766
|
+
observationSummary: deriveObservationSummary(sessionRuns),
|
|
1767
|
+
checkSummary: deriveCheckSummary(sessionRuns)
|
|
1783
1768
|
};
|
|
1784
1769
|
}
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1770
|
+
|
|
1771
|
+
// packages/core/src/sessions/load.ts
|
|
1772
|
+
async function enrichSessionRunRecord(meta) {
|
|
1773
|
+
let metadata;
|
|
1774
|
+
try {
|
|
1775
|
+
const events = await readTraceEventsFromFile(meta.filePath);
|
|
1776
|
+
for (const event of events) {
|
|
1777
|
+
if (event.event !== "run_started") continue;
|
|
1778
|
+
if (event.metadata && typeof event.metadata === "object") {
|
|
1779
|
+
metadata = event.metadata;
|
|
1780
|
+
}
|
|
1781
|
+
break;
|
|
1794
1782
|
}
|
|
1783
|
+
} catch {
|
|
1784
|
+
}
|
|
1785
|
+
return {
|
|
1786
|
+
runId: meta.runId,
|
|
1787
|
+
name: meta.name,
|
|
1788
|
+
status: meta.status,
|
|
1789
|
+
startedAt: meta.startedAt,
|
|
1790
|
+
endedAt: meta.endedAt,
|
|
1791
|
+
durationMs: meta.durationMs,
|
|
1792
|
+
filePath: meta.filePath,
|
|
1793
|
+
metadata
|
|
1795
1794
|
};
|
|
1796
1795
|
}
|
|
1797
|
-
function
|
|
1798
|
-
const
|
|
1799
|
-
const
|
|
1800
|
-
|
|
1801
|
-
const list = byRun.get(event.runId) ?? [];
|
|
1802
|
-
list.push(event);
|
|
1803
|
-
byRun.set(event.runId, list);
|
|
1796
|
+
async function loadSessionRunRecords(metas) {
|
|
1797
|
+
const out = [];
|
|
1798
|
+
for (const meta of metas) {
|
|
1799
|
+
out.push(await enrichSessionRunRecord(meta));
|
|
1804
1800
|
}
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
let match;
|
|
1829
|
-
if (legacyEvent(start) === "run_started") {
|
|
1830
|
-
const candidates = completes.filter(
|
|
1831
|
-
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
1832
|
-
);
|
|
1833
|
-
if (candidates.length > 1) {
|
|
1834
|
-
diagnostics.push({
|
|
1835
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1836
|
-
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
1837
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
1838
|
-
});
|
|
1839
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1840
|
-
}
|
|
1841
|
-
match = candidates[0];
|
|
1842
|
-
} else if (stepId) {
|
|
1843
|
-
const candidates = completes.filter(
|
|
1844
|
-
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
1845
|
-
);
|
|
1846
|
-
if (candidates.length > 1) {
|
|
1847
|
-
diagnostics.push({
|
|
1848
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1849
|
-
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
1850
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
1851
|
-
});
|
|
1852
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1853
|
-
}
|
|
1854
|
-
match = candidates[0];
|
|
1855
|
-
}
|
|
1856
|
-
if (match) {
|
|
1857
|
-
usedCompletes.add(match.eventId);
|
|
1858
|
-
absorbedIds.add(match.eventId);
|
|
1859
|
-
const paired = pairStartComplete(start, match);
|
|
1860
|
-
logical.push(paired);
|
|
1861
|
-
logicalByRawId.set(start.eventId, paired);
|
|
1862
|
-
logicalByRawId.set(match.eventId, paired);
|
|
1863
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
1864
|
-
} else {
|
|
1865
|
-
diagnostics.push({
|
|
1866
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
1867
|
-
message: `No matching complete for start ${start.eventId}.`,
|
|
1868
|
-
eventIds: [start.eventId]
|
|
1869
|
-
});
|
|
1870
|
-
const alone = asLogical(start);
|
|
1871
|
-
logical.push(alone);
|
|
1872
|
-
logicalByRawId.set(start.eventId, alone);
|
|
1873
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1874
|
-
}
|
|
1875
|
-
}
|
|
1876
|
-
for (const complete of completes) {
|
|
1877
|
-
if (usedCompletes.has(complete.eventId)) continue;
|
|
1878
|
-
diagnostics.push({
|
|
1879
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
1880
|
-
message: `No matching start for complete ${complete.eventId}.`,
|
|
1881
|
-
eventIds: [complete.eventId]
|
|
1801
|
+
return out;
|
|
1802
|
+
}
|
|
1803
|
+
|
|
1804
|
+
// packages/core/src/sessions/index.ts
|
|
1805
|
+
function compareRuns(a, b) {
|
|
1806
|
+
const aStart = a.startedAt ?? 0;
|
|
1807
|
+
const bStart = b.startedAt ?? 0;
|
|
1808
|
+
if (aStart !== bStart) return aStart - bStart;
|
|
1809
|
+
return a.runId.localeCompare(b.runId);
|
|
1810
|
+
}
|
|
1811
|
+
function buildGroups(runIds, metaByRunId) {
|
|
1812
|
+
const byGroup = /* @__PURE__ */ new Map();
|
|
1813
|
+
for (const runId of runIds) {
|
|
1814
|
+
const groupId = metaByRunId.get(runId)?.groupId;
|
|
1815
|
+
if (!groupId) continue;
|
|
1816
|
+
const existing = byGroup.get(groupId);
|
|
1817
|
+
if (existing) {
|
|
1818
|
+
existing.runIds.push(runId);
|
|
1819
|
+
} else {
|
|
1820
|
+
byGroup.set(groupId, {
|
|
1821
|
+
groupId,
|
|
1822
|
+
parentGroupId: metaByRunId.get(runId)?.parentGroupId,
|
|
1823
|
+
runIds: [runId]
|
|
1882
1824
|
});
|
|
1883
|
-
const alone = asLogical(complete);
|
|
1884
|
-
logical.push(alone);
|
|
1885
|
-
logicalByRawId.set(complete.eventId, alone);
|
|
1886
|
-
const stepId = stepIdOf(complete);
|
|
1887
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1888
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1889
|
-
}
|
|
1890
|
-
}
|
|
1891
|
-
for (const event of others) {
|
|
1892
|
-
const alone = asLogical(event);
|
|
1893
|
-
logical.push(alone);
|
|
1894
|
-
logicalByRawId.set(event.eventId, alone);
|
|
1895
|
-
const stepId = stepIdOf(event);
|
|
1896
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1897
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1898
|
-
}
|
|
1899
1825
|
}
|
|
1900
1826
|
}
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1827
|
+
return [...byGroup.values()].map((group) => ({
|
|
1828
|
+
...group,
|
|
1829
|
+
runIds: [...group.runIds].sort((a, b) => a.localeCompare(b))
|
|
1830
|
+
}));
|
|
1831
|
+
}
|
|
1832
|
+
function buildHandoffs(runIds, metaByRunId, warnings, sessionId) {
|
|
1833
|
+
const edges = [];
|
|
1834
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1835
|
+
const pushEdge = (edge) => {
|
|
1836
|
+
const key = `${edge.from}->${edge.to}:${edge.confidence}`;
|
|
1837
|
+
if (seen.has(key)) return;
|
|
1838
|
+
seen.add(key);
|
|
1839
|
+
edges.push(edge);
|
|
1840
|
+
};
|
|
1841
|
+
for (const runId of runIds) {
|
|
1842
|
+
const meta = metaByRunId.get(runId);
|
|
1843
|
+
if (!meta) continue;
|
|
1844
|
+
if (meta.handoffFrom && meta.handoffTo) {
|
|
1845
|
+
pushEdge({
|
|
1846
|
+
from: meta.handoffFrom,
|
|
1847
|
+
to: meta.handoffTo,
|
|
1848
|
+
source: "manual",
|
|
1849
|
+
confidence: "explicit"
|
|
1850
|
+
});
|
|
1907
1851
|
continue;
|
|
1908
1852
|
}
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
1917
|
-
if (viaStep) {
|
|
1918
|
-
nextParent = viaStep;
|
|
1919
|
-
remapped = true;
|
|
1920
|
-
}
|
|
1853
|
+
if (meta.handoffFrom) {
|
|
1854
|
+
pushEdge({
|
|
1855
|
+
from: meta.handoffFrom,
|
|
1856
|
+
to: runId,
|
|
1857
|
+
source: "manual",
|
|
1858
|
+
confidence: "explicit"
|
|
1859
|
+
});
|
|
1921
1860
|
}
|
|
1922
|
-
if (
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
}
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1861
|
+
if (meta.handoffTo) {
|
|
1862
|
+
pushEdge({
|
|
1863
|
+
from: runId,
|
|
1864
|
+
to: meta.handoffTo,
|
|
1865
|
+
source: "manual",
|
|
1866
|
+
confidence: "explicit"
|
|
1867
|
+
});
|
|
1868
|
+
}
|
|
1869
|
+
if (meta.subAgentId && meta.parentGroupId && !meta.handoffFrom && !meta.handoffTo) {
|
|
1870
|
+
pushEdge({
|
|
1871
|
+
from: meta.parentGroupId,
|
|
1872
|
+
to: meta.subAgentId,
|
|
1873
|
+
source: "inferred",
|
|
1874
|
+
confidence: "correlated"
|
|
1875
|
+
});
|
|
1876
|
+
warnings.push({
|
|
1877
|
+
code: "ambiguous-handoff-endpoints",
|
|
1878
|
+
message: "Handoff inferred from parentGroupId and subAgentId without explicit handoffFrom/handoffTo.",
|
|
1879
|
+
runId,
|
|
1880
|
+
sessionId
|
|
1881
|
+
});
|
|
1882
|
+
}
|
|
1883
|
+
}
|
|
1884
|
+
return edges.sort((a, b) => {
|
|
1885
|
+
const from = a.from.localeCompare(b.from);
|
|
1886
|
+
if (from !== 0) return from;
|
|
1887
|
+
return a.to.localeCompare(b.to);
|
|
1888
|
+
});
|
|
1889
|
+
}
|
|
1890
|
+
function buildRetries(runIds, metaByRunId, warnings, sessionId) {
|
|
1891
|
+
const retries = [];
|
|
1892
|
+
for (const runId of runIds) {
|
|
1893
|
+
const meta = metaByRunId.get(runId);
|
|
1894
|
+
if (!meta) continue;
|
|
1895
|
+
if (meta.retryOf) {
|
|
1896
|
+
retries.push({
|
|
1897
|
+
runId,
|
|
1898
|
+
retryOf: meta.retryOf,
|
|
1899
|
+
attempt: meta.attempt,
|
|
1900
|
+
source: "manual",
|
|
1901
|
+
confidence: "explicit"
|
|
1902
|
+
});
|
|
1956
1903
|
continue;
|
|
1957
1904
|
}
|
|
1958
|
-
if (
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
),
|
|
1965
|
-
eventIds: [event.eventId]
|
|
1905
|
+
if (meta.attempt !== void 0 && meta.attempt > 1) {
|
|
1906
|
+
retries.push({
|
|
1907
|
+
runId,
|
|
1908
|
+
attempt: meta.attempt,
|
|
1909
|
+
source: "inferred",
|
|
1910
|
+
confidence: "correlated"
|
|
1966
1911
|
});
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
parentNormalized: true,
|
|
1973
|
-
originalParentId
|
|
1974
|
-
}
|
|
1912
|
+
warnings.push({
|
|
1913
|
+
code: "ambiguous-retry-link",
|
|
1914
|
+
message: "attempt > 1 without retryOf; retry link is correlated only.",
|
|
1915
|
+
runId,
|
|
1916
|
+
sessionId
|
|
1975
1917
|
});
|
|
1976
|
-
continue;
|
|
1977
1918
|
}
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1919
|
+
}
|
|
1920
|
+
return retries.sort((a, b) => a.runId.localeCompare(b.runId));
|
|
1921
|
+
}
|
|
1922
|
+
function buildCriticalPath(runs, handoffs) {
|
|
1923
|
+
const runById = new Map(runs.map((run) => [run.runId, run]));
|
|
1924
|
+
const explicitTargets = new Set(
|
|
1925
|
+
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.to)
|
|
1926
|
+
);
|
|
1927
|
+
const explicitSources = new Set(
|
|
1928
|
+
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
1929
|
+
);
|
|
1930
|
+
const ordered = [...runs].sort(compareRuns);
|
|
1931
|
+
const path18 = [];
|
|
1932
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1933
|
+
const pushRun = (run, confidence, source) => {
|
|
1934
|
+
if (visited.has(run.runId)) return;
|
|
1935
|
+
visited.add(run.runId);
|
|
1936
|
+
path18.push({
|
|
1937
|
+
runId: run.runId,
|
|
1938
|
+
name: run.name,
|
|
1939
|
+
startedAt: run.startedAt,
|
|
1940
|
+
durationMs: run.durationMs,
|
|
1941
|
+
confidence,
|
|
1942
|
+
source
|
|
1982
1943
|
});
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1944
|
+
};
|
|
1945
|
+
for (const edge of handoffs) {
|
|
1946
|
+
if (edge.confidence !== "explicit") continue;
|
|
1947
|
+
const fromRun = [...runById.values()].find(
|
|
1948
|
+
(run) => run.runId === edge.from || metaRunIdMatches(run, edge.from, runById)
|
|
1949
|
+
);
|
|
1950
|
+
const toRun = [...runById.values()].find(
|
|
1951
|
+
(run) => run.runId === edge.to || metaRunIdMatches(run, edge.to, runById)
|
|
1952
|
+
);
|
|
1953
|
+
if (fromRun) pushRun(fromRun, "explicit", "manual");
|
|
1954
|
+
if (toRun) pushRun(toRun, "explicit", "manual");
|
|
1955
|
+
}
|
|
1956
|
+
for (const run of ordered) {
|
|
1957
|
+
if (visited.has(run.runId)) continue;
|
|
1958
|
+
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
1959
|
+
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
1960
|
+
}
|
|
1961
|
+
return path18;
|
|
1962
|
+
}
|
|
1963
|
+
function metaRunIdMatches(run, token, runById) {
|
|
1964
|
+
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1965
|
+
return meta?.subAgentId === token || meta?.groupId === token || runById.has(token);
|
|
1966
|
+
}
|
|
1967
|
+
function buildSessionIndex(inputRuns, options = {}) {
|
|
1968
|
+
const warnings = [];
|
|
1969
|
+
const runs = [...inputRuns].sort(compareRuns);
|
|
1970
|
+
const metaByRunId = /* @__PURE__ */ new Map();
|
|
1971
|
+
for (const run of runs) {
|
|
1972
|
+
metaByRunId.set(run.runId, extractSessionWorkflowMetadata(run.metadata));
|
|
1973
|
+
}
|
|
1974
|
+
const sessionsByKey = /* @__PURE__ */ new Map();
|
|
1975
|
+
const unscopedRunIds = [];
|
|
1976
|
+
for (const run of runs) {
|
|
1977
|
+
const meta = metaByRunId.get(run.runId);
|
|
1978
|
+
const key = sessionKeyForRun(meta, {
|
|
1979
|
+
correlateByGroupId: options.correlateByGroupId === true
|
|
1980
|
+
});
|
|
1981
|
+
if (!key) {
|
|
1982
|
+
unscopedRunIds.push(run.runId);
|
|
1983
|
+
continue;
|
|
1984
|
+
}
|
|
1985
|
+
const bucket = sessionsByKey.get(key) ?? [];
|
|
1986
|
+
bucket.push(run);
|
|
1987
|
+
sessionsByKey.set(key, bucket);
|
|
1988
|
+
}
|
|
1989
|
+
const sessions = [...sessionsByKey.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([sessionId, sessionRuns]) => {
|
|
1990
|
+
const runIds = sessionRuns.map((run) => run.runId).sort();
|
|
1991
|
+
const handoffs = buildHandoffs(runIds, metaByRunId, warnings, sessionId);
|
|
1992
|
+
const retries = buildRetries(runIds, metaByRunId, warnings, sessionId);
|
|
1993
|
+
const groups = buildGroups(runIds, metaByRunId);
|
|
1994
|
+
const criticalPath = buildCriticalPath(sessionRuns, handoffs);
|
|
1995
|
+
const confidences = new Set(handoffs.map((edge) => edge.confidence));
|
|
1996
|
+
if (confidences.has("explicit") && confidences.has("correlated")) {
|
|
1997
|
+
warnings.push({
|
|
1998
|
+
code: "mixed-confidence-group",
|
|
1999
|
+
message: "Session aggregates explicit and correlated handoff edges.",
|
|
2000
|
+
sessionId
|
|
2001
|
+
});
|
|
2002
|
+
}
|
|
2003
|
+
return enrichSessionSummary(
|
|
2004
|
+
{
|
|
2005
|
+
sessionId,
|
|
2006
|
+
runIds,
|
|
2007
|
+
groups,
|
|
2008
|
+
handoffs,
|
|
2009
|
+
retries,
|
|
2010
|
+
criticalPath
|
|
2011
|
+
},
|
|
2012
|
+
runs,
|
|
2013
|
+
{
|
|
2014
|
+
nowMs: options.nowMs,
|
|
2015
|
+
staleThresholdMs: options.staleThresholdMs
|
|
1990
2016
|
}
|
|
2017
|
+
);
|
|
2018
|
+
});
|
|
2019
|
+
if (sessions.length === 0 && runs.length > 0) {
|
|
2020
|
+
warnings.push({
|
|
2021
|
+
code: "missing-session-id",
|
|
2022
|
+
message: "No sessionId (or correlated groupId) found on input runs."
|
|
1991
2023
|
});
|
|
1992
2024
|
}
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
2025
|
+
warnings.sort((a, b) => {
|
|
2026
|
+
const code = a.code.localeCompare(b.code);
|
|
2027
|
+
if (code !== 0) return code;
|
|
2028
|
+
return (a.runId ?? "").localeCompare(b.runId ?? "");
|
|
1998
2029
|
});
|
|
1999
2030
|
return {
|
|
2000
|
-
|
|
2001
|
-
|
|
2031
|
+
runs,
|
|
2032
|
+
sessions,
|
|
2033
|
+
unscopedRunIds: unscopedRunIds.sort(),
|
|
2034
|
+
warnings
|
|
2002
2035
|
};
|
|
2003
2036
|
}
|
|
2004
|
-
function resolveCanonicalToolName(event) {
|
|
2005
|
-
const attrs = event.attributes;
|
|
2006
|
-
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
2007
|
-
if (direct) return direct;
|
|
2008
|
-
const metadata = attrs?.metadata;
|
|
2009
|
-
if (isRecord5(metadata)) {
|
|
2010
|
-
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
2011
|
-
if (nested) return nested;
|
|
2012
|
-
}
|
|
2013
|
-
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
2014
|
-
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
2015
|
-
}
|
|
2016
|
-
return event.name;
|
|
2017
|
-
}
|
|
2018
|
-
function pickString(record, keys) {
|
|
2019
|
-
if (!record) return void 0;
|
|
2020
|
-
for (const key of keys) {
|
|
2021
|
-
const value = record[key];
|
|
2022
|
-
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
2023
|
-
}
|
|
2024
|
-
return void 0;
|
|
2025
|
-
}
|
|
2026
2037
|
|
|
2027
2038
|
// packages/core/src/checks/trace-facts.ts
|
|
2028
2039
|
formatProgrammaticDiagnostic(
|
|
@@ -2057,7 +2068,11 @@ function emptySummary() {
|
|
|
2057
2068
|
failed: 0,
|
|
2058
2069
|
warnings: 0,
|
|
2059
2070
|
errors: 0,
|
|
2060
|
-
rulesEvaluated: 0
|
|
2071
|
+
rulesEvaluated: 0,
|
|
2072
|
+
rulesPassed: 0,
|
|
2073
|
+
rulesWarning: 0,
|
|
2074
|
+
rulesFailed: 0,
|
|
2075
|
+
rulesError: 0
|
|
2061
2076
|
};
|
|
2062
2077
|
}
|
|
2063
2078
|
function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
@@ -2069,7 +2084,11 @@ function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
|
2069
2084
|
summary: {
|
|
2070
2085
|
...emptySummary(),
|
|
2071
2086
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2072
|
-
rulesEvaluated: ruleExecutions.length
|
|
2087
|
+
rulesEvaluated: ruleExecutions.length,
|
|
2088
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
2089
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
2090
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
2091
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
2073
2092
|
},
|
|
2074
2093
|
findings: [],
|
|
2075
2094
|
diagnostics: [...diagnostics],
|
|
@@ -2229,7 +2248,7 @@ function normalizeFinding(rule, finding) {
|
|
|
2229
2248
|
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
2230
2249
|
};
|
|
2231
2250
|
}
|
|
2232
|
-
function summarize(findings, diagnostics,
|
|
2251
|
+
function summarize(findings, diagnostics, ruleExecutions) {
|
|
2233
2252
|
return {
|
|
2234
2253
|
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
2235
2254
|
failed: findings.filter(
|
|
@@ -2239,7 +2258,11 @@ function summarize(findings, diagnostics, rulesEvaluated) {
|
|
|
2239
2258
|
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
2240
2259
|
).length,
|
|
2241
2260
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2242
|
-
rulesEvaluated
|
|
2261
|
+
rulesEvaluated: ruleExecutions.length,
|
|
2262
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
2263
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
2264
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
2265
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
2243
2266
|
};
|
|
2244
2267
|
}
|
|
2245
2268
|
function classifyRuleExecution(findings, threw) {
|
|
@@ -2716,7 +2739,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
2716
2739
|
}
|
|
2717
2740
|
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
2718
2741
|
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
2719
|
-
const summary = summarize(sortedFindings, diagnostics, ruleExecutions
|
|
2742
|
+
const summary = summarize(sortedFindings, diagnostics, ruleExecutions);
|
|
2720
2743
|
const status = summary.failed > 0 ? "fail" : "pass";
|
|
2721
2744
|
return {
|
|
2722
2745
|
ok: status === "pass",
|
|
@@ -3155,6 +3178,28 @@ function asPositiveNumber(value, label) {
|
|
|
3155
3178
|
}
|
|
3156
3179
|
return value;
|
|
3157
3180
|
}
|
|
3181
|
+
function asExpectConfig(value, label) {
|
|
3182
|
+
if (value === void 0) return void 0;
|
|
3183
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
3184
|
+
throw new Error(`${label} must be an object.`);
|
|
3185
|
+
}
|
|
3186
|
+
const raw = value;
|
|
3187
|
+
const checkStatus = raw.checkStatus;
|
|
3188
|
+
if (checkStatus !== "pass" && checkStatus !== "fail" && checkStatus !== "error") {
|
|
3189
|
+
throw new Error(`${label}.checkStatus must be "pass", "fail", or "error".`);
|
|
3190
|
+
}
|
|
3191
|
+
const findingRuleIds = asStringArray(raw.findingRuleIds, `${label}.findingRuleIds`);
|
|
3192
|
+
const minAssertions = asPositiveNumber(raw.minAssertions, `${label}.minAssertions`);
|
|
3193
|
+
const rejectUnexpectedFindings = raw.rejectUnexpectedFindings === void 0 ? void 0 : typeof raw.rejectUnexpectedFindings === "boolean" ? raw.rejectUnexpectedFindings : (() => {
|
|
3194
|
+
throw new Error(`${label}.rejectUnexpectedFindings must be a boolean.`);
|
|
3195
|
+
})();
|
|
3196
|
+
return {
|
|
3197
|
+
checkStatus,
|
|
3198
|
+
...findingRuleIds !== void 0 ? { findingRuleIds } : {},
|
|
3199
|
+
...rejectUnexpectedFindings !== void 0 ? { rejectUnexpectedFindings } : {},
|
|
3200
|
+
...minAssertions !== void 0 ? { minAssertions } : {}
|
|
3201
|
+
};
|
|
3202
|
+
}
|
|
3158
3203
|
function validateCaseConfig(value, index) {
|
|
3159
3204
|
const diagnostics = [];
|
|
3160
3205
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
@@ -3197,7 +3242,8 @@ function validateCaseConfig(value, index) {
|
|
|
3197
3242
|
raw.expectedObservations,
|
|
3198
3243
|
`cases[${index}].expectedObservations`
|
|
3199
3244
|
)
|
|
3200
|
-
} : {}
|
|
3245
|
+
} : {},
|
|
3246
|
+
...asExpectConfig(raw.expect, `cases[${index}].expect`) !== void 0 ? { expect: asExpectConfig(raw.expect, `cases[${index}].expect`) } : {}
|
|
3201
3247
|
},
|
|
3202
3248
|
diagnostics
|
|
3203
3249
|
};
|
|
@@ -5783,6 +5829,78 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
5783
5829
|
}
|
|
5784
5830
|
return { ok: diagnostics.length === 0, diagnostics };
|
|
5785
5831
|
}
|
|
5832
|
+
function evaluateCaseExpect(suiteCase, checkResult, observationCount) {
|
|
5833
|
+
const expectation = suiteCase.expect;
|
|
5834
|
+
if (expectation === void 0) {
|
|
5835
|
+
return { ok: true, diagnostics: [] };
|
|
5836
|
+
}
|
|
5837
|
+
const diagnostics = [];
|
|
5838
|
+
if (checkResult.status !== expectation.checkStatus) {
|
|
5839
|
+
diagnostics.push(
|
|
5840
|
+
diagnostic4(
|
|
5841
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5842
|
+
`Expected check status "${expectation.checkStatus}", got "${checkResult.status}".`,
|
|
5843
|
+
"error",
|
|
5844
|
+
suiteCase.id
|
|
5845
|
+
)
|
|
5846
|
+
);
|
|
5847
|
+
}
|
|
5848
|
+
const failedFindings = checkResult.findings.filter(
|
|
5849
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
5850
|
+
);
|
|
5851
|
+
const requiredIds = expectation.findingRuleIds ?? [];
|
|
5852
|
+
for (const ruleId of requiredIds) {
|
|
5853
|
+
if (!failedFindings.some((finding) => finding.ruleId === ruleId)) {
|
|
5854
|
+
diagnostics.push(
|
|
5855
|
+
diagnostic4(
|
|
5856
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5857
|
+
`Expected failing finding for rule "${ruleId}" was not present.`,
|
|
5858
|
+
"error",
|
|
5859
|
+
suiteCase.id
|
|
5860
|
+
)
|
|
5861
|
+
);
|
|
5862
|
+
}
|
|
5863
|
+
}
|
|
5864
|
+
const rejectUnexpected = expectation.rejectUnexpectedFindings !== false;
|
|
5865
|
+
if (rejectUnexpected && requiredIds.length > 0) {
|
|
5866
|
+
for (const finding of failedFindings) {
|
|
5867
|
+
if (!requiredIds.includes(finding.ruleId)) {
|
|
5868
|
+
diagnostics.push(
|
|
5869
|
+
diagnostic4(
|
|
5870
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5871
|
+
`Unexpected failing finding for rule "${finding.ruleId}".`,
|
|
5872
|
+
"error",
|
|
5873
|
+
suiteCase.id
|
|
5874
|
+
)
|
|
5875
|
+
);
|
|
5876
|
+
}
|
|
5877
|
+
}
|
|
5878
|
+
}
|
|
5879
|
+
if (expectation.minAssertions !== void 0) {
|
|
5880
|
+
const assertionCount = checkResult.ruleExecutions.length + observationCount;
|
|
5881
|
+
if (assertionCount < expectation.minAssertions) {
|
|
5882
|
+
diagnostics.push(
|
|
5883
|
+
diagnostic4(
|
|
5884
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5885
|
+
`Expected at least ${expectation.minAssertions} assertion(s), got ${assertionCount}.`,
|
|
5886
|
+
"error",
|
|
5887
|
+
suiteCase.id
|
|
5888
|
+
)
|
|
5889
|
+
);
|
|
5890
|
+
}
|
|
5891
|
+
}
|
|
5892
|
+
if (expectation.checkStatus === "fail" && checkResult.diagnostics.some((item) => item.severity === "error") && checkResult.status === "error") {
|
|
5893
|
+
diagnostics.push(
|
|
5894
|
+
diagnostic4(
|
|
5895
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5896
|
+
'Check ended in evaluator/config error; semantic failure expectation requires status "fail".',
|
|
5897
|
+
"error",
|
|
5898
|
+
suiteCase.id
|
|
5899
|
+
)
|
|
5900
|
+
);
|
|
5901
|
+
}
|
|
5902
|
+
return { ok: diagnostics.length === 0, diagnostics };
|
|
5903
|
+
}
|
|
5786
5904
|
async function runSuiteCase(suiteCase, config, options) {
|
|
5787
5905
|
const resolved = await resolveSuiteCaseTrace(suiteCase, options);
|
|
5788
5906
|
if (resolved.missing || resolved.tracePath === void 0) {
|
|
@@ -5835,9 +5953,21 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
5835
5953
|
status: "pass",
|
|
5836
5954
|
format: read.format,
|
|
5837
5955
|
findings: [],
|
|
5838
|
-
diagnostics: []
|
|
5956
|
+
diagnostics: [],
|
|
5957
|
+
ruleExecutions: []
|
|
5958
|
+
};
|
|
5839
5959
|
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
5840
|
-
const
|
|
5960
|
+
const checkOk = checkResult.ok;
|
|
5961
|
+
const observationsOk = observationResult.ok;
|
|
5962
|
+
const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
|
|
5963
|
+
(finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
|
|
5964
|
+
) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
|
|
5965
|
+
const expectResult = evaluateCaseExpect(
|
|
5966
|
+
suiteCase,
|
|
5967
|
+
checkResult,
|
|
5968
|
+
compiled.observationCount
|
|
5969
|
+
);
|
|
5970
|
+
const checkFailureDiagnostics = [
|
|
5841
5971
|
...checkResult.diagnostics.map(
|
|
5842
5972
|
(item) => diagnostic4(
|
|
5843
5973
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
@@ -5853,15 +5983,24 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
5853
5983
|
finding.severity,
|
|
5854
5984
|
suiteCase.id
|
|
5855
5985
|
)
|
|
5856
|
-
)
|
|
5857
|
-
...observationResult.diagnostics
|
|
5986
|
+
)
|
|
5858
5987
|
];
|
|
5859
|
-
const
|
|
5860
|
-
const
|
|
5861
|
-
|
|
5862
|
-
|
|
5863
|
-
|
|
5864
|
-
|
|
5988
|
+
const expectMatched = suiteCase.expect !== void 0 && expectResult.ok && observationsOk;
|
|
5989
|
+
const diagnostics = expectMatched ? [
|
|
5990
|
+
...observationResult.diagnostics,
|
|
5991
|
+
...expectResult.diagnostics,
|
|
5992
|
+
diagnostic4(
|
|
5993
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
5994
|
+
`Expected semantic failure matched (check.status=${checkResult.status}).`,
|
|
5995
|
+
"info",
|
|
5996
|
+
suiteCase.id
|
|
5997
|
+
)
|
|
5998
|
+
] : [
|
|
5999
|
+
...checkFailureDiagnostics,
|
|
6000
|
+
...observationResult.diagnostics,
|
|
6001
|
+
...expectResult.diagnostics
|
|
6002
|
+
];
|
|
6003
|
+
const ok = suiteCase.expect !== void 0 ? expectResult.ok && observationsOk : checkOk && observationsOk;
|
|
5865
6004
|
const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
|
|
5866
6005
|
return {
|
|
5867
6006
|
id: suiteCase.id,
|