@agent-inspect/viewer 6.31.9 → 6.31.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +930 -791
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.mjs +930 -791
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -1163,859 +1163,870 @@ function buildRunTimeline(events, options = {}) {
|
|
|
1163
1163
|
};
|
|
1164
1164
|
}
|
|
1165
1165
|
|
|
1166
|
-
// packages/core/src/
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1166
|
+
// packages/core/src/diagnostics/programmatic.ts
|
|
1167
|
+
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
1168
|
+
AI_TRACE_INPUT_INVALID: {
|
|
1169
|
+
code: "AI_TRACE_INPUT_INVALID",
|
|
1170
|
+
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
1171
|
+
remediation: "For a file path, use openTraceFile(path).",
|
|
1172
|
+
relatedCodes: ["invalid_input"]
|
|
1173
|
+
},
|
|
1174
|
+
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
1175
|
+
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
1176
|
+
summary: "No trace reader could detect the input format.",
|
|
1177
|
+
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
1178
|
+
relatedCodes: ["unsupported_format"]
|
|
1179
|
+
},
|
|
1180
|
+
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
1181
|
+
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
1182
|
+
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
1183
|
+
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
1184
|
+
relatedCodes: ["ambiguous_format"]
|
|
1185
|
+
},
|
|
1186
|
+
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
1187
|
+
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
1188
|
+
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
1189
|
+
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
1190
|
+
},
|
|
1191
|
+
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
1192
|
+
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
1193
|
+
summary: "Multiple runs are available; select a run before executing checks.",
|
|
1194
|
+
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
1195
|
+
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
1196
|
+
},
|
|
1197
|
+
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
1198
|
+
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1199
|
+
summary: "A parentId equals its own event/step id.",
|
|
1200
|
+
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
1201
|
+
relatedCodes: [
|
|
1202
|
+
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
1203
|
+
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
1204
|
+
]
|
|
1205
|
+
},
|
|
1206
|
+
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
1207
|
+
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
1208
|
+
summary: "Trace contains a parentId cycle.",
|
|
1209
|
+
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
1210
|
+
relatedCodes: ["structure.cycle"]
|
|
1211
|
+
},
|
|
1212
|
+
AI_TRACE_CONTRACT_INPUT_INVALID: {
|
|
1213
|
+
code: "AI_TRACE_CONTRACT_INPUT_INVALID",
|
|
1214
|
+
summary: "evaluateTraceContract expects ({ read }, contract) or use evaluateTraceContractRead(read, contract).",
|
|
1215
|
+
remediation: "Pass TraceCheckInput as the first argument: evaluateTraceContract({ read }, contract).",
|
|
1216
|
+
relatedCodes: ["AI_CHECK_INVALID_ARGUMENTS"]
|
|
1217
|
+
},
|
|
1218
|
+
AI_INSPECT_RUN_UNSUPPORTED_OPTION: {
|
|
1219
|
+
code: "AI_INSPECT_RUN_UNSUPPORTED_OPTION",
|
|
1220
|
+
summary: 'inspectRun does not accept a "writer" option.',
|
|
1221
|
+
remediation: "Use createInspector({ writer }) then inspector.run(...). inspectRun persists via the global JSONL path (schema 0.1)."
|
|
1176
1222
|
}
|
|
1177
|
-
|
|
1223
|
+
});
|
|
1224
|
+
function formatProgrammaticDiagnostic(code, detail) {
|
|
1225
|
+
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
1226
|
+
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
1227
|
+
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
1178
1228
|
}
|
|
1179
1229
|
|
|
1180
|
-
// packages/core/src/
|
|
1181
|
-
function
|
|
1182
|
-
return typeof value === "
|
|
1183
|
-
}
|
|
1184
|
-
function finitePositiveInt(value) {
|
|
1185
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
1186
|
-
return void 0;
|
|
1187
|
-
}
|
|
1188
|
-
return Math.trunc(value);
|
|
1230
|
+
// packages/core/src/checks/logical-events.ts
|
|
1231
|
+
function isRecord5(value) {
|
|
1232
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1189
1233
|
}
|
|
1190
|
-
function
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
let found = false;
|
|
1194
|
-
const assignString = (key, value) => {
|
|
1195
|
-
if (isNonEmptyString3(value)) {
|
|
1196
|
-
out[key] = value.trim();
|
|
1197
|
-
found = true;
|
|
1198
|
-
}
|
|
1199
|
-
};
|
|
1200
|
-
assignString("sessionId", record.sessionId);
|
|
1201
|
-
assignString("conversationId", record.conversationId);
|
|
1202
|
-
assignString("groupId", record.groupId);
|
|
1203
|
-
assignString("parentGroupId", record.parentGroupId);
|
|
1204
|
-
assignString("retryOf", record.retryOf);
|
|
1205
|
-
assignString("retryReason", record.retryReason);
|
|
1206
|
-
assignString("handoffFrom", record.handoffFrom);
|
|
1207
|
-
assignString("handoffTo", record.handoffTo);
|
|
1208
|
-
assignString("subAgentId", record.subAgentId);
|
|
1209
|
-
assignString("subAgentName", record.subAgentName);
|
|
1210
|
-
assignString("jobId", record.jobId);
|
|
1211
|
-
assignString("queueName", record.queueName);
|
|
1212
|
-
assignString("workflowName", record.workflowName);
|
|
1213
|
-
assignString("workflowStep", record.workflowStep);
|
|
1214
|
-
assignString("toolCallId", record.toolCallId);
|
|
1215
|
-
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
1216
|
-
assignString("linkedStepId", record.linkedStepId);
|
|
1217
|
-
assignString("operationId", record.operationId);
|
|
1218
|
-
assignString("attemptId", record.attemptId);
|
|
1219
|
-
assignString("fallbackOf", record.fallbackOf);
|
|
1220
|
-
assignString("idempotencyKey", record.idempotencyKey);
|
|
1221
|
-
assignString("correlationId", record.correlationId);
|
|
1222
|
-
assignString("requestId", record.requestId);
|
|
1223
|
-
assignString("decisionId", record.decisionId);
|
|
1224
|
-
const attempt = finitePositiveInt(record.attempt);
|
|
1225
|
-
if (attempt !== void 0) {
|
|
1226
|
-
out.attempt = attempt;
|
|
1227
|
-
found = true;
|
|
1228
|
-
}
|
|
1229
|
-
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
1230
|
-
if (attemptNumber !== void 0) {
|
|
1231
|
-
out.attemptNumber = attemptNumber;
|
|
1232
|
-
found = true;
|
|
1233
|
-
}
|
|
1234
|
-
return found ? out : void 0;
|
|
1234
|
+
function legacyEvent(event) {
|
|
1235
|
+
const value = event.attributes?.legacyEvent;
|
|
1236
|
+
return typeof value === "string" ? value : void 0;
|
|
1235
1237
|
}
|
|
1236
|
-
function
|
|
1237
|
-
|
|
1238
|
-
if (
|
|
1239
|
-
return `group:${meta.groupId}`;
|
|
1240
|
-
}
|
|
1238
|
+
function stepIdOf(event) {
|
|
1239
|
+
const value = event.attributes?.stepId;
|
|
1240
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
1241
1241
|
return void 0;
|
|
1242
1242
|
}
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
completed: 1
|
|
1252
|
-
};
|
|
1253
|
-
var EXPLICIT_SESSION_STATUSES = /* @__PURE__ */ new Set([
|
|
1254
|
-
"running",
|
|
1255
|
-
"waiting_input",
|
|
1256
|
-
"idle",
|
|
1257
|
-
"completed",
|
|
1258
|
-
"error",
|
|
1259
|
-
"stale",
|
|
1260
|
-
"unknown"
|
|
1261
|
-
]);
|
|
1262
|
-
function isExplicitSessionStatus(value) {
|
|
1263
|
-
return typeof value === "string" && EXPLICIT_SESSION_STATUSES.has(value);
|
|
1264
|
-
}
|
|
1265
|
-
function activityMs(run) {
|
|
1266
|
-
return run.endedAt ?? run.startedAt ?? 0;
|
|
1243
|
+
function cloneEvent(event) {
|
|
1244
|
+
return {
|
|
1245
|
+
...event,
|
|
1246
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
1247
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
1248
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
1249
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
1250
|
+
};
|
|
1267
1251
|
}
|
|
1268
|
-
function
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1252
|
+
function mergeAttributes(start, complete) {
|
|
1253
|
+
const merged = {
|
|
1254
|
+
...isRecord5(complete.attributes) ? complete.attributes : {},
|
|
1255
|
+
...isRecord5(start.attributes) ? start.attributes : {}
|
|
1256
|
+
};
|
|
1257
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
1258
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
1259
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
1260
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
1273
1261
|
}
|
|
1274
|
-
return
|
|
1262
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
1275
1263
|
}
|
|
1276
|
-
function
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1264
|
+
function pairStartComplete(start, complete) {
|
|
1265
|
+
const attributes = mergeAttributes(start, complete);
|
|
1266
|
+
const paired = {
|
|
1267
|
+
...cloneEvent(start),
|
|
1268
|
+
status: complete.status,
|
|
1269
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
1270
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
1271
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
1272
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
1273
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
1274
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
1275
|
+
};
|
|
1276
|
+
return {
|
|
1277
|
+
...paired,
|
|
1278
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
1279
|
+
projection: {
|
|
1280
|
+
paired: true,
|
|
1281
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
1282
|
+
parentNormalized: false,
|
|
1283
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
1282
1284
|
}
|
|
1283
|
-
}
|
|
1284
|
-
return earliest;
|
|
1285
|
-
}
|
|
1286
|
-
function latestEndWhenAllEnded(runs) {
|
|
1287
|
-
if (runs.length === 0) return void 0;
|
|
1288
|
-
let latest;
|
|
1289
|
-
for (const run of runs) {
|
|
1290
|
-
if (run.endedAt === void 0) return void 0;
|
|
1291
|
-
if (latest === void 0 || run.endedAt > latest) latest = run.endedAt;
|
|
1292
|
-
}
|
|
1293
|
-
return latest;
|
|
1285
|
+
};
|
|
1294
1286
|
}
|
|
1295
|
-
function
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
best = raw;
|
|
1287
|
+
function asLogical(event, extras) {
|
|
1288
|
+
return {
|
|
1289
|
+
...cloneEvent(event),
|
|
1290
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
1291
|
+
projection: {
|
|
1292
|
+
paired: false,
|
|
1293
|
+
absorbedEventIds: Object.freeze([]),
|
|
1294
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
1295
|
+
...{}
|
|
1305
1296
|
}
|
|
1306
|
-
}
|
|
1307
|
-
return best;
|
|
1308
|
-
}
|
|
1309
|
-
function deriveLastError(runs) {
|
|
1310
|
-
const errorRuns = runs.filter((run) => run.status === "error").sort((a, b) => activityMs(b) - activityMs(a));
|
|
1311
|
-
const latest = errorRuns[0];
|
|
1312
|
-
if (!latest) return void 0;
|
|
1313
|
-
const meta = latest.metadata ?? {};
|
|
1314
|
-
const message = typeof meta.errorMessage === "string" && meta.errorMessage.trim() !== "" ? meta.errorMessage.trim() : latest.name ?? latest.runId;
|
|
1315
|
-
const code = typeof meta.errorCode === "string" && meta.errorCode.trim() !== "" ? meta.errorCode.trim() : void 0;
|
|
1316
|
-
return { runId: latest.runId, message, code };
|
|
1297
|
+
};
|
|
1317
1298
|
}
|
|
1318
|
-
function
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
if (!summary || typeof summary !== "object") continue;
|
|
1326
|
-
const record = summary;
|
|
1327
|
-
if (typeof record.pass === "number") {
|
|
1328
|
-
pass += record.pass;
|
|
1329
|
-
found = true;
|
|
1330
|
-
}
|
|
1331
|
-
if (typeof record.fail === "number") {
|
|
1332
|
-
fail += record.fail;
|
|
1333
|
-
found = true;
|
|
1334
|
-
}
|
|
1335
|
-
if (typeof record.warn === "number") {
|
|
1336
|
-
warn2 += record.warn;
|
|
1337
|
-
found = true;
|
|
1338
|
-
}
|
|
1299
|
+
function projectLogicalEvents(events) {
|
|
1300
|
+
const diagnostics = [];
|
|
1301
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
1302
|
+
for (const event of events) {
|
|
1303
|
+
const list = byRun.get(event.runId) ?? [];
|
|
1304
|
+
list.push(event);
|
|
1305
|
+
byRun.set(event.runId, list);
|
|
1339
1306
|
}
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1307
|
+
const absorbedIds = /* @__PURE__ */ new Set();
|
|
1308
|
+
const logicalByRawId = /* @__PURE__ */ new Map();
|
|
1309
|
+
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
1310
|
+
const logical = [];
|
|
1311
|
+
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
1312
|
+
for (const runId of orderedRuns) {
|
|
1313
|
+
const runEvents = byRun.get(runId) ?? [];
|
|
1314
|
+
const starts = [];
|
|
1315
|
+
const completes = [];
|
|
1316
|
+
const others = [];
|
|
1317
|
+
for (const event of runEvents) {
|
|
1318
|
+
const legacy = legacyEvent(event);
|
|
1319
|
+
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
1320
|
+
starts.push(event);
|
|
1321
|
+
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
1322
|
+
completes.push(event);
|
|
1323
|
+
} else {
|
|
1324
|
+
others.push(event);
|
|
1325
|
+
}
|
|
1347
1326
|
}
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
}
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
checkSummary: deriveCheckSummary(sessionRuns)
|
|
1398
|
-
};
|
|
1399
|
-
}
|
|
1400
|
-
|
|
1401
|
-
// packages/core/src/sessions/load.ts
|
|
1402
|
-
async function enrichSessionRunRecord(meta) {
|
|
1403
|
-
let metadata;
|
|
1404
|
-
try {
|
|
1405
|
-
const events = await readTraceEventsFromFile(meta.filePath);
|
|
1406
|
-
for (const event of events) {
|
|
1407
|
-
if (event.event !== "run_started") continue;
|
|
1408
|
-
if (event.metadata && typeof event.metadata === "object") {
|
|
1409
|
-
metadata = event.metadata;
|
|
1327
|
+
const usedCompletes = /* @__PURE__ */ new Set();
|
|
1328
|
+
for (const start of starts) {
|
|
1329
|
+
const stepId = stepIdOf(start);
|
|
1330
|
+
let match;
|
|
1331
|
+
if (legacyEvent(start) === "run_started") {
|
|
1332
|
+
const candidates = completes.filter(
|
|
1333
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
1334
|
+
);
|
|
1335
|
+
if (candidates.length > 1) {
|
|
1336
|
+
diagnostics.push({
|
|
1337
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1338
|
+
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
1339
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1340
|
+
});
|
|
1341
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1342
|
+
}
|
|
1343
|
+
match = candidates[0];
|
|
1344
|
+
} else if (stepId) {
|
|
1345
|
+
const candidates = completes.filter(
|
|
1346
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
1347
|
+
);
|
|
1348
|
+
if (candidates.length > 1) {
|
|
1349
|
+
diagnostics.push({
|
|
1350
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1351
|
+
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
1352
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1353
|
+
});
|
|
1354
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1355
|
+
}
|
|
1356
|
+
match = candidates[0];
|
|
1357
|
+
}
|
|
1358
|
+
if (match) {
|
|
1359
|
+
usedCompletes.add(match.eventId);
|
|
1360
|
+
absorbedIds.add(match.eventId);
|
|
1361
|
+
const paired = pairStartComplete(start, match);
|
|
1362
|
+
logical.push(paired);
|
|
1363
|
+
logicalByRawId.set(start.eventId, paired);
|
|
1364
|
+
logicalByRawId.set(match.eventId, paired);
|
|
1365
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
1366
|
+
} else {
|
|
1367
|
+
diagnostics.push({
|
|
1368
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
1369
|
+
message: `No matching complete for start ${start.eventId}.`,
|
|
1370
|
+
eventIds: [start.eventId]
|
|
1371
|
+
});
|
|
1372
|
+
const alone = asLogical(start);
|
|
1373
|
+
logical.push(alone);
|
|
1374
|
+
logicalByRawId.set(start.eventId, alone);
|
|
1375
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1410
1376
|
}
|
|
1411
|
-
break;
|
|
1412
1377
|
}
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
startedAt: meta.startedAt,
|
|
1420
|
-
endedAt: meta.endedAt,
|
|
1421
|
-
durationMs: meta.durationMs,
|
|
1422
|
-
filePath: meta.filePath,
|
|
1423
|
-
metadata
|
|
1424
|
-
};
|
|
1425
|
-
}
|
|
1426
|
-
async function loadSessionRunRecords(metas) {
|
|
1427
|
-
const out = [];
|
|
1428
|
-
for (const meta of metas) {
|
|
1429
|
-
out.push(await enrichSessionRunRecord(meta));
|
|
1430
|
-
}
|
|
1431
|
-
return out;
|
|
1432
|
-
}
|
|
1433
|
-
|
|
1434
|
-
// packages/core/src/sessions/index.ts
|
|
1435
|
-
function compareRuns(a, b) {
|
|
1436
|
-
const aStart = a.startedAt ?? 0;
|
|
1437
|
-
const bStart = b.startedAt ?? 0;
|
|
1438
|
-
if (aStart !== bStart) return aStart - bStart;
|
|
1439
|
-
return a.runId.localeCompare(b.runId);
|
|
1440
|
-
}
|
|
1441
|
-
function buildGroups(runIds, metaByRunId) {
|
|
1442
|
-
const byGroup = /* @__PURE__ */ new Map();
|
|
1443
|
-
for (const runId of runIds) {
|
|
1444
|
-
const groupId = metaByRunId.get(runId)?.groupId;
|
|
1445
|
-
if (!groupId) continue;
|
|
1446
|
-
const existing = byGroup.get(groupId);
|
|
1447
|
-
if (existing) {
|
|
1448
|
-
existing.runIds.push(runId);
|
|
1449
|
-
} else {
|
|
1450
|
-
byGroup.set(groupId, {
|
|
1451
|
-
groupId,
|
|
1452
|
-
parentGroupId: metaByRunId.get(runId)?.parentGroupId,
|
|
1453
|
-
runIds: [runId]
|
|
1378
|
+
for (const complete of completes) {
|
|
1379
|
+
if (usedCompletes.has(complete.eventId)) continue;
|
|
1380
|
+
diagnostics.push({
|
|
1381
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
1382
|
+
message: `No matching start for complete ${complete.eventId}.`,
|
|
1383
|
+
eventIds: [complete.eventId]
|
|
1454
1384
|
});
|
|
1385
|
+
const alone = asLogical(complete);
|
|
1386
|
+
logical.push(alone);
|
|
1387
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
1388
|
+
const stepId = stepIdOf(complete);
|
|
1389
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1390
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
for (const event of others) {
|
|
1394
|
+
const alone = asLogical(event);
|
|
1395
|
+
logical.push(alone);
|
|
1396
|
+
logicalByRawId.set(event.eventId, alone);
|
|
1397
|
+
const stepId = stepIdOf(event);
|
|
1398
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1399
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1400
|
+
}
|
|
1455
1401
|
}
|
|
1456
1402
|
}
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
const edges = [];
|
|
1464
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1465
|
-
const pushEdge = (edge) => {
|
|
1466
|
-
const key = `${edge.from}->${edge.to}:${edge.confidence}`;
|
|
1467
|
-
if (seen.has(key)) return;
|
|
1468
|
-
seen.add(key);
|
|
1469
|
-
edges.push(edge);
|
|
1470
|
-
};
|
|
1471
|
-
for (const runId of runIds) {
|
|
1472
|
-
const meta = metaByRunId.get(runId);
|
|
1473
|
-
if (!meta) continue;
|
|
1474
|
-
if (meta.handoffFrom && meta.handoffTo) {
|
|
1475
|
-
pushEdge({
|
|
1476
|
-
from: meta.handoffFrom,
|
|
1477
|
-
to: meta.handoffTo,
|
|
1478
|
-
source: "manual",
|
|
1479
|
-
confidence: "explicit"
|
|
1480
|
-
});
|
|
1403
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
1404
|
+
const normalized = [];
|
|
1405
|
+
for (const event of logical) {
|
|
1406
|
+
const originalParentId = event.parentId;
|
|
1407
|
+
if (!originalParentId) {
|
|
1408
|
+
normalized.push(event);
|
|
1481
1409
|
continue;
|
|
1482
1410
|
}
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1411
|
+
let nextParent = originalParentId;
|
|
1412
|
+
let remapped = false;
|
|
1413
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
1414
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
1415
|
+
nextParent = viaAbsorbed.eventId;
|
|
1416
|
+
remapped = true;
|
|
1417
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
1418
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
1419
|
+
if (viaStep) {
|
|
1420
|
+
nextParent = viaStep;
|
|
1421
|
+
remapped = true;
|
|
1422
|
+
}
|
|
1490
1423
|
}
|
|
1491
|
-
if (
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1424
|
+
if (!remapped) {
|
|
1425
|
+
if (originalParentId === event.eventId) {
|
|
1426
|
+
diagnostics.push({
|
|
1427
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1428
|
+
message: formatProgrammaticDiagnostic(
|
|
1429
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1430
|
+
`Removed self-parent edge on ${event.eventId}.`
|
|
1431
|
+
),
|
|
1432
|
+
eventIds: [event.eventId]
|
|
1433
|
+
});
|
|
1434
|
+
const { parentId: _drop, ...rest } = event;
|
|
1435
|
+
normalized.push({
|
|
1436
|
+
...rest,
|
|
1437
|
+
projection: {
|
|
1438
|
+
...event.projection,
|
|
1439
|
+
parentNormalized: true,
|
|
1440
|
+
originalParentId
|
|
1441
|
+
}
|
|
1442
|
+
});
|
|
1443
|
+
continue;
|
|
1444
|
+
}
|
|
1445
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
1446
|
+
const mapping = event.attributes?.parentMapping;
|
|
1447
|
+
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
1448
|
+
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
1449
|
+
if (!unresolved) {
|
|
1450
|
+
diagnostics.push({
|
|
1451
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
1452
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
1453
|
+
eventIds: [event.eventId]
|
|
1454
|
+
});
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
normalized.push(event);
|
|
1458
|
+
continue;
|
|
1498
1459
|
}
|
|
1499
|
-
if (
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1460
|
+
if (nextParent === event.eventId) {
|
|
1461
|
+
diagnostics.push({
|
|
1462
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1463
|
+
message: formatProgrammaticDiagnostic(
|
|
1464
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1465
|
+
`Removed self-parent edge on ${event.eventId} (was ${originalParentId}).`
|
|
1466
|
+
),
|
|
1467
|
+
eventIds: [event.eventId]
|
|
1505
1468
|
});
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1469
|
+
const { parentId: _drop, ...rest } = event;
|
|
1470
|
+
normalized.push({
|
|
1471
|
+
...rest,
|
|
1472
|
+
projection: {
|
|
1473
|
+
...event.projection,
|
|
1474
|
+
parentNormalized: true,
|
|
1475
|
+
originalParentId
|
|
1476
|
+
}
|
|
1511
1477
|
});
|
|
1478
|
+
continue;
|
|
1512
1479
|
}
|
|
1480
|
+
diagnostics.push({
|
|
1481
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
1482
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
1483
|
+
eventIds: [event.eventId]
|
|
1484
|
+
});
|
|
1485
|
+
normalized.push({
|
|
1486
|
+
...event,
|
|
1487
|
+
parentId: nextParent,
|
|
1488
|
+
projection: {
|
|
1489
|
+
...event.projection,
|
|
1490
|
+
parentNormalized: true,
|
|
1491
|
+
originalParentId
|
|
1492
|
+
}
|
|
1493
|
+
});
|
|
1513
1494
|
}
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1495
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
1496
|
+
normalized.sort((a, b) => {
|
|
1497
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
1498
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
1499
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
1518
1500
|
});
|
|
1501
|
+
return {
|
|
1502
|
+
logicalEvents: Object.freeze(normalized),
|
|
1503
|
+
diagnostics: Object.freeze(diagnostics)
|
|
1504
|
+
};
|
|
1519
1505
|
}
|
|
1520
|
-
function
|
|
1521
|
-
const
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1506
|
+
function resolveCanonicalToolName(event) {
|
|
1507
|
+
const attrs = event.attributes;
|
|
1508
|
+
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
1509
|
+
if (direct) return direct;
|
|
1510
|
+
const metadata = attrs?.metadata;
|
|
1511
|
+
if (isRecord5(metadata)) {
|
|
1512
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
1513
|
+
if (nested) return nested;
|
|
1514
|
+
}
|
|
1515
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
1516
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
1517
|
+
}
|
|
1518
|
+
return event.name;
|
|
1519
|
+
}
|
|
1520
|
+
function pickString(record, keys) {
|
|
1521
|
+
if (!record) return void 0;
|
|
1522
|
+
for (const key of keys) {
|
|
1523
|
+
const value = record[key];
|
|
1524
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
1525
|
+
}
|
|
1526
|
+
return void 0;
|
|
1527
|
+
}
|
|
1528
|
+
|
|
1529
|
+
// packages/core/src/search.ts
|
|
1530
|
+
async function loadTraceMetadataList(_traceDir, fileNames, getPath) {
|
|
1531
|
+
const metas = [];
|
|
1532
|
+
for (const fileName of fileNames) {
|
|
1533
|
+
try {
|
|
1534
|
+
const filePath = getPath(fileName);
|
|
1535
|
+
const meta = await extractMetadata(filePath);
|
|
1536
|
+
metas.push(meta);
|
|
1537
|
+
} catch {
|
|
1548
1538
|
}
|
|
1549
1539
|
}
|
|
1550
|
-
return
|
|
1540
|
+
return metas;
|
|
1551
1541
|
}
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
source
|
|
1573
|
-
});
|
|
1542
|
+
|
|
1543
|
+
// packages/core/src/sessions/metadata.ts
|
|
1544
|
+
function isNonEmptyString3(value) {
|
|
1545
|
+
return typeof value === "string" && value.trim() !== "";
|
|
1546
|
+
}
|
|
1547
|
+
function finitePositiveInt(value) {
|
|
1548
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
1549
|
+
return void 0;
|
|
1550
|
+
}
|
|
1551
|
+
return Math.trunc(value);
|
|
1552
|
+
}
|
|
1553
|
+
function extractSessionWorkflowMetadata(record) {
|
|
1554
|
+
if (!record) return void 0;
|
|
1555
|
+
const out = {};
|
|
1556
|
+
let found = false;
|
|
1557
|
+
const assignString = (key, value) => {
|
|
1558
|
+
if (isNonEmptyString3(value)) {
|
|
1559
|
+
out[key] = value.trim();
|
|
1560
|
+
found = true;
|
|
1561
|
+
}
|
|
1574
1562
|
};
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1563
|
+
assignString("sessionId", record.sessionId);
|
|
1564
|
+
assignString("conversationId", record.conversationId);
|
|
1565
|
+
assignString("groupId", record.groupId);
|
|
1566
|
+
assignString("parentGroupId", record.parentGroupId);
|
|
1567
|
+
assignString("retryOf", record.retryOf);
|
|
1568
|
+
assignString("retryReason", record.retryReason);
|
|
1569
|
+
assignString("handoffFrom", record.handoffFrom);
|
|
1570
|
+
assignString("handoffTo", record.handoffTo);
|
|
1571
|
+
assignString("subAgentId", record.subAgentId);
|
|
1572
|
+
assignString("subAgentName", record.subAgentName);
|
|
1573
|
+
assignString("jobId", record.jobId);
|
|
1574
|
+
assignString("queueName", record.queueName);
|
|
1575
|
+
assignString("workflowName", record.workflowName);
|
|
1576
|
+
assignString("workflowStep", record.workflowStep);
|
|
1577
|
+
assignString("toolCallId", record.toolCallId);
|
|
1578
|
+
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
1579
|
+
assignString("linkedStepId", record.linkedStepId);
|
|
1580
|
+
assignString("operationId", record.operationId);
|
|
1581
|
+
assignString("attemptId", record.attemptId);
|
|
1582
|
+
assignString("fallbackOf", record.fallbackOf);
|
|
1583
|
+
assignString("idempotencyKey", record.idempotencyKey);
|
|
1584
|
+
assignString("correlationId", record.correlationId);
|
|
1585
|
+
assignString("requestId", record.requestId);
|
|
1586
|
+
assignString("decisionId", record.decisionId);
|
|
1587
|
+
const attempt = finitePositiveInt(record.attempt);
|
|
1588
|
+
if (attempt !== void 0) {
|
|
1589
|
+
out.attempt = attempt;
|
|
1590
|
+
found = true;
|
|
1585
1591
|
}
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1592
|
+
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
1593
|
+
if (attemptNumber !== void 0) {
|
|
1594
|
+
out.attemptNumber = attemptNumber;
|
|
1595
|
+
found = true;
|
|
1590
1596
|
}
|
|
1591
|
-
return
|
|
1592
|
-
}
|
|
1593
|
-
function metaRunIdMatches(run, token, runById) {
|
|
1594
|
-
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1595
|
-
return meta?.subAgentId === token || meta?.groupId === token || runById.has(token);
|
|
1597
|
+
return found ? out : void 0;
|
|
1596
1598
|
}
|
|
1597
|
-
function
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
for (const run of runs) {
|
|
1602
|
-
metaByRunId.set(run.runId, extractSessionWorkflowMetadata(run.metadata));
|
|
1599
|
+
function sessionKeyForRun(meta, options) {
|
|
1600
|
+
if (meta?.sessionId) return meta.sessionId;
|
|
1601
|
+
if (options?.correlateByGroupId && meta?.groupId) {
|
|
1602
|
+
return `group:${meta.groupId}`;
|
|
1603
1603
|
}
|
|
1604
|
-
|
|
1605
|
-
|
|
1604
|
+
return void 0;
|
|
1605
|
+
}
|
|
1606
|
+
|
|
1607
|
+
// packages/core/src/sessions/status.ts
|
|
1608
|
+
var DEFAULT_STALE_THRESHOLD_MS = 864e5;
|
|
1609
|
+
var EXPLICIT_STATUS_PRIORITY = {
|
|
1610
|
+
error: 5,
|
|
1611
|
+
waiting_input: 4,
|
|
1612
|
+
idle: 3,
|
|
1613
|
+
stale: 2,
|
|
1614
|
+
completed: 1
|
|
1615
|
+
};
|
|
1616
|
+
var EXPLICIT_SESSION_STATUSES = /* @__PURE__ */ new Set([
|
|
1617
|
+
"running",
|
|
1618
|
+
"waiting_input",
|
|
1619
|
+
"idle",
|
|
1620
|
+
"completed",
|
|
1621
|
+
"error",
|
|
1622
|
+
"stale",
|
|
1623
|
+
"unknown"
|
|
1624
|
+
]);
|
|
1625
|
+
function isExplicitSessionStatus(value) {
|
|
1626
|
+
return typeof value === "string" && EXPLICIT_SESSION_STATUSES.has(value);
|
|
1627
|
+
}
|
|
1628
|
+
function activityMs(run) {
|
|
1629
|
+
return run.endedAt ?? run.startedAt ?? 0;
|
|
1630
|
+
}
|
|
1631
|
+
function latestActivityMs(runs) {
|
|
1632
|
+
let latest = 0;
|
|
1606
1633
|
for (const run of runs) {
|
|
1607
|
-
const
|
|
1608
|
-
|
|
1609
|
-
correlateByGroupId: options.correlateByGroupId === true
|
|
1610
|
-
});
|
|
1611
|
-
if (!key) {
|
|
1612
|
-
unscopedRunIds.push(run.runId);
|
|
1613
|
-
continue;
|
|
1614
|
-
}
|
|
1615
|
-
const bucket = sessionsByKey.get(key) ?? [];
|
|
1616
|
-
bucket.push(run);
|
|
1617
|
-
sessionsByKey.set(key, bucket);
|
|
1634
|
+
const ms = activityMs(run);
|
|
1635
|
+
if (ms > latest) latest = ms;
|
|
1618
1636
|
}
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
warnings.push({
|
|
1628
|
-
code: "mixed-confidence-group",
|
|
1629
|
-
message: "Session aggregates explicit and correlated handoff edges.",
|
|
1630
|
-
sessionId
|
|
1631
|
-
});
|
|
1637
|
+
return latest;
|
|
1638
|
+
}
|
|
1639
|
+
function earliestStart(runs) {
|
|
1640
|
+
let earliest;
|
|
1641
|
+
for (const run of runs) {
|
|
1642
|
+
if (run.startedAt === void 0) continue;
|
|
1643
|
+
if (earliest === void 0 || run.startedAt < earliest) {
|
|
1644
|
+
earliest = run.startedAt;
|
|
1632
1645
|
}
|
|
1633
|
-
return enrichSessionSummary(
|
|
1634
|
-
{
|
|
1635
|
-
sessionId,
|
|
1636
|
-
runIds,
|
|
1637
|
-
groups,
|
|
1638
|
-
handoffs,
|
|
1639
|
-
retries,
|
|
1640
|
-
criticalPath
|
|
1641
|
-
},
|
|
1642
|
-
runs,
|
|
1643
|
-
{
|
|
1644
|
-
nowMs: options.nowMs,
|
|
1645
|
-
staleThresholdMs: options.staleThresholdMs
|
|
1646
|
-
}
|
|
1647
|
-
);
|
|
1648
|
-
});
|
|
1649
|
-
if (sessions.length === 0 && runs.length > 0) {
|
|
1650
|
-
warnings.push({
|
|
1651
|
-
code: "missing-session-id",
|
|
1652
|
-
message: "No sessionId (or correlated groupId) found on input runs."
|
|
1653
|
-
});
|
|
1654
1646
|
}
|
|
1655
|
-
|
|
1656
|
-
const code = a.code.localeCompare(b.code);
|
|
1657
|
-
if (code !== 0) return code;
|
|
1658
|
-
return (a.runId ?? "").localeCompare(b.runId ?? "");
|
|
1659
|
-
});
|
|
1660
|
-
return {
|
|
1661
|
-
runs,
|
|
1662
|
-
sessions,
|
|
1663
|
-
unscopedRunIds: unscopedRunIds.sort(),
|
|
1664
|
-
warnings
|
|
1665
|
-
};
|
|
1647
|
+
return earliest;
|
|
1666
1648
|
}
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
remediation: "For a file path, use openTraceFile(path).",
|
|
1674
|
-
relatedCodes: ["invalid_input"]
|
|
1675
|
-
},
|
|
1676
|
-
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
1677
|
-
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
1678
|
-
summary: "No trace reader could detect the input format.",
|
|
1679
|
-
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
1680
|
-
relatedCodes: ["unsupported_format"]
|
|
1681
|
-
},
|
|
1682
|
-
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
1683
|
-
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
1684
|
-
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
1685
|
-
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
1686
|
-
relatedCodes: ["ambiguous_format"]
|
|
1687
|
-
},
|
|
1688
|
-
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
1689
|
-
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
1690
|
-
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
1691
|
-
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
1692
|
-
},
|
|
1693
|
-
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
1694
|
-
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
1695
|
-
summary: "Multiple runs are available; select a run before executing checks.",
|
|
1696
|
-
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
1697
|
-
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
1698
|
-
},
|
|
1699
|
-
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
1700
|
-
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1701
|
-
summary: "A parentId equals its own event/step id.",
|
|
1702
|
-
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
1703
|
-
relatedCodes: [
|
|
1704
|
-
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
1705
|
-
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
1706
|
-
]
|
|
1707
|
-
},
|
|
1708
|
-
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
1709
|
-
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
1710
|
-
summary: "Trace contains a parentId cycle.",
|
|
1711
|
-
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
1712
|
-
relatedCodes: ["structure.cycle"]
|
|
1649
|
+
function latestEndWhenAllEnded(runs) {
|
|
1650
|
+
if (runs.length === 0) return void 0;
|
|
1651
|
+
let latest;
|
|
1652
|
+
for (const run of runs) {
|
|
1653
|
+
if (run.endedAt === void 0) return void 0;
|
|
1654
|
+
if (latest === void 0 || run.endedAt > latest) latest = run.endedAt;
|
|
1713
1655
|
}
|
|
1714
|
-
|
|
1715
|
-
function formatProgrammaticDiagnostic(code, detail) {
|
|
1716
|
-
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
1717
|
-
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
1718
|
-
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
1656
|
+
return latest;
|
|
1719
1657
|
}
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1658
|
+
function pickExplicitStatus(runs) {
|
|
1659
|
+
let best;
|
|
1660
|
+
let bestPriority = 0;
|
|
1661
|
+
for (const run of runs) {
|
|
1662
|
+
const raw = run.metadata?.sessionStatus;
|
|
1663
|
+
if (!isExplicitSessionStatus(raw)) continue;
|
|
1664
|
+
const priority = EXPLICIT_STATUS_PRIORITY[raw] ?? 0;
|
|
1665
|
+
if (priority > bestPriority) {
|
|
1666
|
+
bestPriority = priority;
|
|
1667
|
+
best = raw;
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
return best;
|
|
1724
1671
|
}
|
|
1725
|
-
function
|
|
1726
|
-
const
|
|
1727
|
-
|
|
1672
|
+
function deriveLastError(runs) {
|
|
1673
|
+
const errorRuns = runs.filter((run) => run.status === "error").sort((a, b) => activityMs(b) - activityMs(a));
|
|
1674
|
+
const latest = errorRuns[0];
|
|
1675
|
+
if (!latest) return void 0;
|
|
1676
|
+
const meta = latest.metadata ?? {};
|
|
1677
|
+
const message = typeof meta.errorMessage === "string" && meta.errorMessage.trim() !== "" ? meta.errorMessage.trim() : latest.name ?? latest.runId;
|
|
1678
|
+
const code = typeof meta.errorCode === "string" && meta.errorCode.trim() !== "" ? meta.errorCode.trim() : void 0;
|
|
1679
|
+
return { runId: latest.runId, message, code };
|
|
1728
1680
|
}
|
|
1729
|
-
function
|
|
1730
|
-
|
|
1731
|
-
|
|
1681
|
+
function deriveCheckSummary(runs) {
|
|
1682
|
+
let pass = 0;
|
|
1683
|
+
let fail = 0;
|
|
1684
|
+
let warn2 = 0;
|
|
1685
|
+
let found = false;
|
|
1686
|
+
for (const run of runs) {
|
|
1687
|
+
const summary = run.metadata?.checkSummary;
|
|
1688
|
+
if (!summary || typeof summary !== "object") continue;
|
|
1689
|
+
const record = summary;
|
|
1690
|
+
if (typeof record.pass === "number") {
|
|
1691
|
+
pass += record.pass;
|
|
1692
|
+
found = true;
|
|
1693
|
+
}
|
|
1694
|
+
if (typeof record.fail === "number") {
|
|
1695
|
+
fail += record.fail;
|
|
1696
|
+
found = true;
|
|
1697
|
+
}
|
|
1698
|
+
if (typeof record.warn === "number") {
|
|
1699
|
+
warn2 += record.warn;
|
|
1700
|
+
found = true;
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
return found ? { pass, fail, warn: warn2 } : void 0;
|
|
1704
|
+
}
|
|
1705
|
+
function deriveObservationSummary(runs) {
|
|
1706
|
+
for (const run of [...runs].sort((a, b) => activityMs(b) - activityMs(a))) {
|
|
1707
|
+
const value = run.metadata?.observationSummary;
|
|
1708
|
+
if (typeof value === "string" && value.trim() !== "") {
|
|
1709
|
+
return value.trim();
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1732
1712
|
return void 0;
|
|
1733
1713
|
}
|
|
1734
|
-
function
|
|
1735
|
-
return
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1714
|
+
function deriveSessionStatus(runs, options = {}) {
|
|
1715
|
+
if (runs.length === 0) return "unknown";
|
|
1716
|
+
if (runs.some((run) => run.status === "running")) return "running";
|
|
1717
|
+
const explicit = pickExplicitStatus(runs);
|
|
1718
|
+
if (explicit && explicit !== "running") return explicit;
|
|
1719
|
+
if (runs.some((run) => run.status === "error")) return "error";
|
|
1720
|
+
if (runs.every((run) => run.status === "success")) return "completed";
|
|
1721
|
+
const nowMs = options.nowMs ?? Date.now();
|
|
1722
|
+
const staleThresholdMs = options.staleThresholdMs ?? DEFAULT_STALE_THRESHOLD_MS;
|
|
1723
|
+
const lastMs = latestActivityMs(runs);
|
|
1724
|
+
if (lastMs > 0 && nowMs - lastMs > staleThresholdMs) return "stale";
|
|
1725
|
+
return "unknown";
|
|
1742
1726
|
}
|
|
1743
|
-
function
|
|
1744
|
-
const
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1727
|
+
function enrichSessionSummary(summary, runs, options = {}) {
|
|
1728
|
+
const sessionRuns = runs.filter((run) => summary.runIds.includes(run.runId)).sort((a, b) => a.runId.localeCompare(b.runId));
|
|
1729
|
+
const startedAt = earliestStart(sessionRuns);
|
|
1730
|
+
const endedAt = latestEndWhenAllEnded(sessionRuns);
|
|
1731
|
+
const durationMs = startedAt !== void 0 && endedAt !== void 0 ? endedAt - startedAt : void 0;
|
|
1732
|
+
let correlationId;
|
|
1733
|
+
let jobId;
|
|
1734
|
+
let workflowId;
|
|
1735
|
+
for (const run of sessionRuns) {
|
|
1736
|
+
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1737
|
+
if (!correlationId && meta?.correlationId) correlationId = meta.correlationId;
|
|
1738
|
+
if (!jobId && meta?.jobId) jobId = meta.jobId;
|
|
1739
|
+
if (!workflowId && meta?.workflowName) workflowId = meta.workflowName;
|
|
1740
|
+
else if (!workflowId && meta?.workflowStep) workflowId = meta.workflowStep;
|
|
1752
1741
|
}
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
...cloneEvent(start),
|
|
1759
|
-
status: complete.status,
|
|
1760
|
-
timestamp: complete.timestamp ?? start.timestamp,
|
|
1761
|
-
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
1762
|
-
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
1763
|
-
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
1764
|
-
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
1765
|
-
...attributes !== void 0 ? { attributes } : {}
|
|
1766
|
-
};
|
|
1742
|
+
const lastMs = latestActivityMs(sessionRuns);
|
|
1743
|
+
const lastActivity = lastMs > 0 ? new Date(lastMs).toISOString() : (/* @__PURE__ */ new Date(0)).toISOString();
|
|
1744
|
+
const retryCount2 = summary.retries.filter(
|
|
1745
|
+
(retry) => retry.retryOf !== void 0 || (retry.attempt ?? 0) > 1
|
|
1746
|
+
).length;
|
|
1767
1747
|
return {
|
|
1768
|
-
...
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1748
|
+
...summary,
|
|
1749
|
+
status: deriveSessionStatus(sessionRuns, options),
|
|
1750
|
+
startedAt,
|
|
1751
|
+
endedAt,
|
|
1752
|
+
durationMs,
|
|
1753
|
+
correlationId,
|
|
1754
|
+
jobId,
|
|
1755
|
+
workflowId,
|
|
1756
|
+
lastError: deriveLastError(sessionRuns),
|
|
1757
|
+
lastActivity,
|
|
1758
|
+
retryCount: retryCount2,
|
|
1759
|
+
observationSummary: deriveObservationSummary(sessionRuns),
|
|
1760
|
+
checkSummary: deriveCheckSummary(sessionRuns)
|
|
1776
1761
|
};
|
|
1777
1762
|
}
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1763
|
+
|
|
1764
|
+
// packages/core/src/sessions/load.ts
|
|
1765
|
+
async function enrichSessionRunRecord(meta) {
|
|
1766
|
+
let metadata;
|
|
1767
|
+
try {
|
|
1768
|
+
const events = await readTraceEventsFromFile(meta.filePath);
|
|
1769
|
+
for (const event of events) {
|
|
1770
|
+
if (event.event !== "run_started") continue;
|
|
1771
|
+
if (event.metadata && typeof event.metadata === "object") {
|
|
1772
|
+
metadata = event.metadata;
|
|
1773
|
+
}
|
|
1774
|
+
break;
|
|
1787
1775
|
}
|
|
1776
|
+
} catch {
|
|
1777
|
+
}
|
|
1778
|
+
return {
|
|
1779
|
+
runId: meta.runId,
|
|
1780
|
+
name: meta.name,
|
|
1781
|
+
status: meta.status,
|
|
1782
|
+
startedAt: meta.startedAt,
|
|
1783
|
+
endedAt: meta.endedAt,
|
|
1784
|
+
durationMs: meta.durationMs,
|
|
1785
|
+
filePath: meta.filePath,
|
|
1786
|
+
metadata
|
|
1788
1787
|
};
|
|
1789
1788
|
}
|
|
1790
|
-
function
|
|
1791
|
-
const
|
|
1792
|
-
const
|
|
1793
|
-
|
|
1794
|
-
const list = byRun.get(event.runId) ?? [];
|
|
1795
|
-
list.push(event);
|
|
1796
|
-
byRun.set(event.runId, list);
|
|
1789
|
+
async function loadSessionRunRecords(metas) {
|
|
1790
|
+
const out = [];
|
|
1791
|
+
for (const meta of metas) {
|
|
1792
|
+
out.push(await enrichSessionRunRecord(meta));
|
|
1797
1793
|
}
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
let match;
|
|
1822
|
-
if (legacyEvent(start) === "run_started") {
|
|
1823
|
-
const candidates = completes.filter(
|
|
1824
|
-
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
1825
|
-
);
|
|
1826
|
-
if (candidates.length > 1) {
|
|
1827
|
-
diagnostics.push({
|
|
1828
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1829
|
-
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
1830
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
1831
|
-
});
|
|
1832
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1833
|
-
}
|
|
1834
|
-
match = candidates[0];
|
|
1835
|
-
} else if (stepId) {
|
|
1836
|
-
const candidates = completes.filter(
|
|
1837
|
-
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
1838
|
-
);
|
|
1839
|
-
if (candidates.length > 1) {
|
|
1840
|
-
diagnostics.push({
|
|
1841
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1842
|
-
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
1843
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
1844
|
-
});
|
|
1845
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1846
|
-
}
|
|
1847
|
-
match = candidates[0];
|
|
1848
|
-
}
|
|
1849
|
-
if (match) {
|
|
1850
|
-
usedCompletes.add(match.eventId);
|
|
1851
|
-
absorbedIds.add(match.eventId);
|
|
1852
|
-
const paired = pairStartComplete(start, match);
|
|
1853
|
-
logical.push(paired);
|
|
1854
|
-
logicalByRawId.set(start.eventId, paired);
|
|
1855
|
-
logicalByRawId.set(match.eventId, paired);
|
|
1856
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
1857
|
-
} else {
|
|
1858
|
-
diagnostics.push({
|
|
1859
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
1860
|
-
message: `No matching complete for start ${start.eventId}.`,
|
|
1861
|
-
eventIds: [start.eventId]
|
|
1862
|
-
});
|
|
1863
|
-
const alone = asLogical(start);
|
|
1864
|
-
logical.push(alone);
|
|
1865
|
-
logicalByRawId.set(start.eventId, alone);
|
|
1866
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1867
|
-
}
|
|
1868
|
-
}
|
|
1869
|
-
for (const complete of completes) {
|
|
1870
|
-
if (usedCompletes.has(complete.eventId)) continue;
|
|
1871
|
-
diagnostics.push({
|
|
1872
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
1873
|
-
message: `No matching start for complete ${complete.eventId}.`,
|
|
1874
|
-
eventIds: [complete.eventId]
|
|
1794
|
+
return out;
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
// packages/core/src/sessions/index.ts
|
|
1798
|
+
function compareRuns(a, b) {
|
|
1799
|
+
const aStart = a.startedAt ?? 0;
|
|
1800
|
+
const bStart = b.startedAt ?? 0;
|
|
1801
|
+
if (aStart !== bStart) return aStart - bStart;
|
|
1802
|
+
return a.runId.localeCompare(b.runId);
|
|
1803
|
+
}
|
|
1804
|
+
function buildGroups(runIds, metaByRunId) {
|
|
1805
|
+
const byGroup = /* @__PURE__ */ new Map();
|
|
1806
|
+
for (const runId of runIds) {
|
|
1807
|
+
const groupId = metaByRunId.get(runId)?.groupId;
|
|
1808
|
+
if (!groupId) continue;
|
|
1809
|
+
const existing = byGroup.get(groupId);
|
|
1810
|
+
if (existing) {
|
|
1811
|
+
existing.runIds.push(runId);
|
|
1812
|
+
} else {
|
|
1813
|
+
byGroup.set(groupId, {
|
|
1814
|
+
groupId,
|
|
1815
|
+
parentGroupId: metaByRunId.get(runId)?.parentGroupId,
|
|
1816
|
+
runIds: [runId]
|
|
1875
1817
|
});
|
|
1876
|
-
const alone = asLogical(complete);
|
|
1877
|
-
logical.push(alone);
|
|
1878
|
-
logicalByRawId.set(complete.eventId, alone);
|
|
1879
|
-
const stepId = stepIdOf(complete);
|
|
1880
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1881
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1882
|
-
}
|
|
1883
|
-
}
|
|
1884
|
-
for (const event of others) {
|
|
1885
|
-
const alone = asLogical(event);
|
|
1886
|
-
logical.push(alone);
|
|
1887
|
-
logicalByRawId.set(event.eventId, alone);
|
|
1888
|
-
const stepId = stepIdOf(event);
|
|
1889
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1890
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1891
|
-
}
|
|
1892
1818
|
}
|
|
1893
1819
|
}
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1820
|
+
return [...byGroup.values()].map((group) => ({
|
|
1821
|
+
...group,
|
|
1822
|
+
runIds: [...group.runIds].sort((a, b) => a.localeCompare(b))
|
|
1823
|
+
}));
|
|
1824
|
+
}
|
|
1825
|
+
function buildHandoffs(runIds, metaByRunId, warnings, sessionId) {
|
|
1826
|
+
const edges = [];
|
|
1827
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1828
|
+
const pushEdge = (edge) => {
|
|
1829
|
+
const key = `${edge.from}->${edge.to}:${edge.confidence}`;
|
|
1830
|
+
if (seen.has(key)) return;
|
|
1831
|
+
seen.add(key);
|
|
1832
|
+
edges.push(edge);
|
|
1833
|
+
};
|
|
1834
|
+
for (const runId of runIds) {
|
|
1835
|
+
const meta = metaByRunId.get(runId);
|
|
1836
|
+
if (!meta) continue;
|
|
1837
|
+
if (meta.handoffFrom && meta.handoffTo) {
|
|
1838
|
+
pushEdge({
|
|
1839
|
+
from: meta.handoffFrom,
|
|
1840
|
+
to: meta.handoffTo,
|
|
1841
|
+
source: "manual",
|
|
1842
|
+
confidence: "explicit"
|
|
1843
|
+
});
|
|
1900
1844
|
continue;
|
|
1901
1845
|
}
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
1910
|
-
if (viaStep) {
|
|
1911
|
-
nextParent = viaStep;
|
|
1912
|
-
remapped = true;
|
|
1913
|
-
}
|
|
1846
|
+
if (meta.handoffFrom) {
|
|
1847
|
+
pushEdge({
|
|
1848
|
+
from: meta.handoffFrom,
|
|
1849
|
+
to: runId,
|
|
1850
|
+
source: "manual",
|
|
1851
|
+
confidence: "explicit"
|
|
1852
|
+
});
|
|
1914
1853
|
}
|
|
1915
|
-
if (
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
}
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1854
|
+
if (meta.handoffTo) {
|
|
1855
|
+
pushEdge({
|
|
1856
|
+
from: runId,
|
|
1857
|
+
to: meta.handoffTo,
|
|
1858
|
+
source: "manual",
|
|
1859
|
+
confidence: "explicit"
|
|
1860
|
+
});
|
|
1861
|
+
}
|
|
1862
|
+
if (meta.subAgentId && meta.parentGroupId && !meta.handoffFrom && !meta.handoffTo) {
|
|
1863
|
+
pushEdge({
|
|
1864
|
+
from: meta.parentGroupId,
|
|
1865
|
+
to: meta.subAgentId,
|
|
1866
|
+
source: "inferred",
|
|
1867
|
+
confidence: "correlated"
|
|
1868
|
+
});
|
|
1869
|
+
warnings.push({
|
|
1870
|
+
code: "ambiguous-handoff-endpoints",
|
|
1871
|
+
message: "Handoff inferred from parentGroupId and subAgentId without explicit handoffFrom/handoffTo.",
|
|
1872
|
+
runId,
|
|
1873
|
+
sessionId
|
|
1874
|
+
});
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
return edges.sort((a, b) => {
|
|
1878
|
+
const from = a.from.localeCompare(b.from);
|
|
1879
|
+
if (from !== 0) return from;
|
|
1880
|
+
return a.to.localeCompare(b.to);
|
|
1881
|
+
});
|
|
1882
|
+
}
|
|
1883
|
+
function buildRetries(runIds, metaByRunId, warnings, sessionId) {
|
|
1884
|
+
const retries = [];
|
|
1885
|
+
for (const runId of runIds) {
|
|
1886
|
+
const meta = metaByRunId.get(runId);
|
|
1887
|
+
if (!meta) continue;
|
|
1888
|
+
if (meta.retryOf) {
|
|
1889
|
+
retries.push({
|
|
1890
|
+
runId,
|
|
1891
|
+
retryOf: meta.retryOf,
|
|
1892
|
+
attempt: meta.attempt,
|
|
1893
|
+
source: "manual",
|
|
1894
|
+
confidence: "explicit"
|
|
1895
|
+
});
|
|
1949
1896
|
continue;
|
|
1950
1897
|
}
|
|
1951
|
-
if (
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
),
|
|
1958
|
-
eventIds: [event.eventId]
|
|
1898
|
+
if (meta.attempt !== void 0 && meta.attempt > 1) {
|
|
1899
|
+
retries.push({
|
|
1900
|
+
runId,
|
|
1901
|
+
attempt: meta.attempt,
|
|
1902
|
+
source: "inferred",
|
|
1903
|
+
confidence: "correlated"
|
|
1959
1904
|
});
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
parentNormalized: true,
|
|
1966
|
-
originalParentId
|
|
1967
|
-
}
|
|
1905
|
+
warnings.push({
|
|
1906
|
+
code: "ambiguous-retry-link",
|
|
1907
|
+
message: "attempt > 1 without retryOf; retry link is correlated only.",
|
|
1908
|
+
runId,
|
|
1909
|
+
sessionId
|
|
1968
1910
|
});
|
|
1969
|
-
continue;
|
|
1970
1911
|
}
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1912
|
+
}
|
|
1913
|
+
return retries.sort((a, b) => a.runId.localeCompare(b.runId));
|
|
1914
|
+
}
|
|
1915
|
+
function buildCriticalPath(runs, handoffs) {
|
|
1916
|
+
const runById = new Map(runs.map((run) => [run.runId, run]));
|
|
1917
|
+
const explicitTargets = new Set(
|
|
1918
|
+
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.to)
|
|
1919
|
+
);
|
|
1920
|
+
const explicitSources = new Set(
|
|
1921
|
+
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
1922
|
+
);
|
|
1923
|
+
const ordered = [...runs].sort(compareRuns);
|
|
1924
|
+
const path18 = [];
|
|
1925
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1926
|
+
const pushRun = (run, confidence, source) => {
|
|
1927
|
+
if (visited.has(run.runId)) return;
|
|
1928
|
+
visited.add(run.runId);
|
|
1929
|
+
path18.push({
|
|
1930
|
+
runId: run.runId,
|
|
1931
|
+
name: run.name,
|
|
1932
|
+
startedAt: run.startedAt,
|
|
1933
|
+
durationMs: run.durationMs,
|
|
1934
|
+
confidence,
|
|
1935
|
+
source
|
|
1975
1936
|
});
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1937
|
+
};
|
|
1938
|
+
for (const edge of handoffs) {
|
|
1939
|
+
if (edge.confidence !== "explicit") continue;
|
|
1940
|
+
const fromRun = [...runById.values()].find(
|
|
1941
|
+
(run) => run.runId === edge.from || metaRunIdMatches(run, edge.from, runById)
|
|
1942
|
+
);
|
|
1943
|
+
const toRun = [...runById.values()].find(
|
|
1944
|
+
(run) => run.runId === edge.to || metaRunIdMatches(run, edge.to, runById)
|
|
1945
|
+
);
|
|
1946
|
+
if (fromRun) pushRun(fromRun, "explicit", "manual");
|
|
1947
|
+
if (toRun) pushRun(toRun, "explicit", "manual");
|
|
1948
|
+
}
|
|
1949
|
+
for (const run of ordered) {
|
|
1950
|
+
if (visited.has(run.runId)) continue;
|
|
1951
|
+
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
1952
|
+
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
1953
|
+
}
|
|
1954
|
+
return path18;
|
|
1955
|
+
}
|
|
1956
|
+
function metaRunIdMatches(run, token, runById) {
|
|
1957
|
+
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1958
|
+
return meta?.subAgentId === token || meta?.groupId === token || runById.has(token);
|
|
1959
|
+
}
|
|
1960
|
+
function buildSessionIndex(inputRuns, options = {}) {
|
|
1961
|
+
const warnings = [];
|
|
1962
|
+
const runs = [...inputRuns].sort(compareRuns);
|
|
1963
|
+
const metaByRunId = /* @__PURE__ */ new Map();
|
|
1964
|
+
for (const run of runs) {
|
|
1965
|
+
metaByRunId.set(run.runId, extractSessionWorkflowMetadata(run.metadata));
|
|
1966
|
+
}
|
|
1967
|
+
const sessionsByKey = /* @__PURE__ */ new Map();
|
|
1968
|
+
const unscopedRunIds = [];
|
|
1969
|
+
for (const run of runs) {
|
|
1970
|
+
const meta = metaByRunId.get(run.runId);
|
|
1971
|
+
const key = sessionKeyForRun(meta, {
|
|
1972
|
+
correlateByGroupId: options.correlateByGroupId === true
|
|
1973
|
+
});
|
|
1974
|
+
if (!key) {
|
|
1975
|
+
unscopedRunIds.push(run.runId);
|
|
1976
|
+
continue;
|
|
1977
|
+
}
|
|
1978
|
+
const bucket = sessionsByKey.get(key) ?? [];
|
|
1979
|
+
bucket.push(run);
|
|
1980
|
+
sessionsByKey.set(key, bucket);
|
|
1981
|
+
}
|
|
1982
|
+
const sessions = [...sessionsByKey.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([sessionId, sessionRuns]) => {
|
|
1983
|
+
const runIds = sessionRuns.map((run) => run.runId).sort();
|
|
1984
|
+
const handoffs = buildHandoffs(runIds, metaByRunId, warnings, sessionId);
|
|
1985
|
+
const retries = buildRetries(runIds, metaByRunId, warnings, sessionId);
|
|
1986
|
+
const groups = buildGroups(runIds, metaByRunId);
|
|
1987
|
+
const criticalPath = buildCriticalPath(sessionRuns, handoffs);
|
|
1988
|
+
const confidences = new Set(handoffs.map((edge) => edge.confidence));
|
|
1989
|
+
if (confidences.has("explicit") && confidences.has("correlated")) {
|
|
1990
|
+
warnings.push({
|
|
1991
|
+
code: "mixed-confidence-group",
|
|
1992
|
+
message: "Session aggregates explicit and correlated handoff edges.",
|
|
1993
|
+
sessionId
|
|
1994
|
+
});
|
|
1995
|
+
}
|
|
1996
|
+
return enrichSessionSummary(
|
|
1997
|
+
{
|
|
1998
|
+
sessionId,
|
|
1999
|
+
runIds,
|
|
2000
|
+
groups,
|
|
2001
|
+
handoffs,
|
|
2002
|
+
retries,
|
|
2003
|
+
criticalPath
|
|
2004
|
+
},
|
|
2005
|
+
runs,
|
|
2006
|
+
{
|
|
2007
|
+
nowMs: options.nowMs,
|
|
2008
|
+
staleThresholdMs: options.staleThresholdMs
|
|
1983
2009
|
}
|
|
2010
|
+
);
|
|
2011
|
+
});
|
|
2012
|
+
if (sessions.length === 0 && runs.length > 0) {
|
|
2013
|
+
warnings.push({
|
|
2014
|
+
code: "missing-session-id",
|
|
2015
|
+
message: "No sessionId (or correlated groupId) found on input runs."
|
|
1984
2016
|
});
|
|
1985
2017
|
}
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
2018
|
+
warnings.sort((a, b) => {
|
|
2019
|
+
const code = a.code.localeCompare(b.code);
|
|
2020
|
+
if (code !== 0) return code;
|
|
2021
|
+
return (a.runId ?? "").localeCompare(b.runId ?? "");
|
|
1991
2022
|
});
|
|
1992
2023
|
return {
|
|
1993
|
-
|
|
1994
|
-
|
|
2024
|
+
runs,
|
|
2025
|
+
sessions,
|
|
2026
|
+
unscopedRunIds: unscopedRunIds.sort(),
|
|
2027
|
+
warnings
|
|
1995
2028
|
};
|
|
1996
2029
|
}
|
|
1997
|
-
function resolveCanonicalToolName(event) {
|
|
1998
|
-
const attrs = event.attributes;
|
|
1999
|
-
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
2000
|
-
if (direct) return direct;
|
|
2001
|
-
const metadata = attrs?.metadata;
|
|
2002
|
-
if (isRecord5(metadata)) {
|
|
2003
|
-
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
2004
|
-
if (nested) return nested;
|
|
2005
|
-
}
|
|
2006
|
-
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
2007
|
-
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
2008
|
-
}
|
|
2009
|
-
return event.name;
|
|
2010
|
-
}
|
|
2011
|
-
function pickString(record, keys) {
|
|
2012
|
-
if (!record) return void 0;
|
|
2013
|
-
for (const key of keys) {
|
|
2014
|
-
const value = record[key];
|
|
2015
|
-
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
2016
|
-
}
|
|
2017
|
-
return void 0;
|
|
2018
|
-
}
|
|
2019
2030
|
|
|
2020
2031
|
// packages/core/src/checks/trace-facts.ts
|
|
2021
2032
|
formatProgrammaticDiagnostic(
|
|
@@ -2050,7 +2061,11 @@ function emptySummary() {
|
|
|
2050
2061
|
failed: 0,
|
|
2051
2062
|
warnings: 0,
|
|
2052
2063
|
errors: 0,
|
|
2053
|
-
rulesEvaluated: 0
|
|
2064
|
+
rulesEvaluated: 0,
|
|
2065
|
+
rulesPassed: 0,
|
|
2066
|
+
rulesWarning: 0,
|
|
2067
|
+
rulesFailed: 0,
|
|
2068
|
+
rulesError: 0
|
|
2054
2069
|
};
|
|
2055
2070
|
}
|
|
2056
2071
|
function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
@@ -2062,7 +2077,11 @@ function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
|
2062
2077
|
summary: {
|
|
2063
2078
|
...emptySummary(),
|
|
2064
2079
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2065
|
-
rulesEvaluated: ruleExecutions.length
|
|
2080
|
+
rulesEvaluated: ruleExecutions.length,
|
|
2081
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
2082
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
2083
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
2084
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
2066
2085
|
},
|
|
2067
2086
|
findings: [],
|
|
2068
2087
|
diagnostics: [...diagnostics],
|
|
@@ -2222,7 +2241,7 @@ function normalizeFinding(rule, finding) {
|
|
|
2222
2241
|
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
2223
2242
|
};
|
|
2224
2243
|
}
|
|
2225
|
-
function summarize(findings, diagnostics,
|
|
2244
|
+
function summarize(findings, diagnostics, ruleExecutions) {
|
|
2226
2245
|
return {
|
|
2227
2246
|
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
2228
2247
|
failed: findings.filter(
|
|
@@ -2232,7 +2251,11 @@ function summarize(findings, diagnostics, rulesEvaluated) {
|
|
|
2232
2251
|
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
2233
2252
|
).length,
|
|
2234
2253
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2235
|
-
rulesEvaluated
|
|
2254
|
+
rulesEvaluated: ruleExecutions.length,
|
|
2255
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
2256
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
2257
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
2258
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
2236
2259
|
};
|
|
2237
2260
|
}
|
|
2238
2261
|
function classifyRuleExecution(findings, threw) {
|
|
@@ -2709,7 +2732,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
2709
2732
|
}
|
|
2710
2733
|
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
2711
2734
|
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
2712
|
-
const summary = summarize(sortedFindings, diagnostics, ruleExecutions
|
|
2735
|
+
const summary = summarize(sortedFindings, diagnostics, ruleExecutions);
|
|
2713
2736
|
const status = summary.failed > 0 ? "fail" : "pass";
|
|
2714
2737
|
return {
|
|
2715
2738
|
ok: status === "pass",
|
|
@@ -3148,6 +3171,28 @@ function asPositiveNumber(value, label) {
|
|
|
3148
3171
|
}
|
|
3149
3172
|
return value;
|
|
3150
3173
|
}
|
|
3174
|
+
function asExpectConfig(value, label) {
|
|
3175
|
+
if (value === void 0) return void 0;
|
|
3176
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
3177
|
+
throw new Error(`${label} must be an object.`);
|
|
3178
|
+
}
|
|
3179
|
+
const raw = value;
|
|
3180
|
+
const checkStatus = raw.checkStatus;
|
|
3181
|
+
if (checkStatus !== "pass" && checkStatus !== "fail" && checkStatus !== "error") {
|
|
3182
|
+
throw new Error(`${label}.checkStatus must be "pass", "fail", or "error".`);
|
|
3183
|
+
}
|
|
3184
|
+
const findingRuleIds = asStringArray(raw.findingRuleIds, `${label}.findingRuleIds`);
|
|
3185
|
+
const minAssertions = asPositiveNumber(raw.minAssertions, `${label}.minAssertions`);
|
|
3186
|
+
const rejectUnexpectedFindings = raw.rejectUnexpectedFindings === void 0 ? void 0 : typeof raw.rejectUnexpectedFindings === "boolean" ? raw.rejectUnexpectedFindings : (() => {
|
|
3187
|
+
throw new Error(`${label}.rejectUnexpectedFindings must be a boolean.`);
|
|
3188
|
+
})();
|
|
3189
|
+
return {
|
|
3190
|
+
checkStatus,
|
|
3191
|
+
...findingRuleIds !== void 0 ? { findingRuleIds } : {},
|
|
3192
|
+
...rejectUnexpectedFindings !== void 0 ? { rejectUnexpectedFindings } : {},
|
|
3193
|
+
...minAssertions !== void 0 ? { minAssertions } : {}
|
|
3194
|
+
};
|
|
3195
|
+
}
|
|
3151
3196
|
function validateCaseConfig(value, index) {
|
|
3152
3197
|
const diagnostics = [];
|
|
3153
3198
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
@@ -3190,7 +3235,8 @@ function validateCaseConfig(value, index) {
|
|
|
3190
3235
|
raw.expectedObservations,
|
|
3191
3236
|
`cases[${index}].expectedObservations`
|
|
3192
3237
|
)
|
|
3193
|
-
} : {}
|
|
3238
|
+
} : {},
|
|
3239
|
+
...asExpectConfig(raw.expect, `cases[${index}].expect`) !== void 0 ? { expect: asExpectConfig(raw.expect, `cases[${index}].expect`) } : {}
|
|
3194
3240
|
},
|
|
3195
3241
|
diagnostics
|
|
3196
3242
|
};
|
|
@@ -5776,6 +5822,78 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
5776
5822
|
}
|
|
5777
5823
|
return { ok: diagnostics.length === 0, diagnostics };
|
|
5778
5824
|
}
|
|
5825
|
+
function evaluateCaseExpect(suiteCase, checkResult, observationCount) {
|
|
5826
|
+
const expectation = suiteCase.expect;
|
|
5827
|
+
if (expectation === void 0) {
|
|
5828
|
+
return { ok: true, diagnostics: [] };
|
|
5829
|
+
}
|
|
5830
|
+
const diagnostics = [];
|
|
5831
|
+
if (checkResult.status !== expectation.checkStatus) {
|
|
5832
|
+
diagnostics.push(
|
|
5833
|
+
diagnostic4(
|
|
5834
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5835
|
+
`Expected check status "${expectation.checkStatus}", got "${checkResult.status}".`,
|
|
5836
|
+
"error",
|
|
5837
|
+
suiteCase.id
|
|
5838
|
+
)
|
|
5839
|
+
);
|
|
5840
|
+
}
|
|
5841
|
+
const failedFindings = checkResult.findings.filter(
|
|
5842
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
5843
|
+
);
|
|
5844
|
+
const requiredIds = expectation.findingRuleIds ?? [];
|
|
5845
|
+
for (const ruleId of requiredIds) {
|
|
5846
|
+
if (!failedFindings.some((finding) => finding.ruleId === ruleId)) {
|
|
5847
|
+
diagnostics.push(
|
|
5848
|
+
diagnostic4(
|
|
5849
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5850
|
+
`Expected failing finding for rule "${ruleId}" was not present.`,
|
|
5851
|
+
"error",
|
|
5852
|
+
suiteCase.id
|
|
5853
|
+
)
|
|
5854
|
+
);
|
|
5855
|
+
}
|
|
5856
|
+
}
|
|
5857
|
+
const rejectUnexpected = expectation.rejectUnexpectedFindings !== false;
|
|
5858
|
+
if (rejectUnexpected && requiredIds.length > 0) {
|
|
5859
|
+
for (const finding of failedFindings) {
|
|
5860
|
+
if (!requiredIds.includes(finding.ruleId)) {
|
|
5861
|
+
diagnostics.push(
|
|
5862
|
+
diagnostic4(
|
|
5863
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5864
|
+
`Unexpected failing finding for rule "${finding.ruleId}".`,
|
|
5865
|
+
"error",
|
|
5866
|
+
suiteCase.id
|
|
5867
|
+
)
|
|
5868
|
+
);
|
|
5869
|
+
}
|
|
5870
|
+
}
|
|
5871
|
+
}
|
|
5872
|
+
if (expectation.minAssertions !== void 0) {
|
|
5873
|
+
const assertionCount = checkResult.ruleExecutions.length + observationCount;
|
|
5874
|
+
if (assertionCount < expectation.minAssertions) {
|
|
5875
|
+
diagnostics.push(
|
|
5876
|
+
diagnostic4(
|
|
5877
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5878
|
+
`Expected at least ${expectation.minAssertions} assertion(s), got ${assertionCount}.`,
|
|
5879
|
+
"error",
|
|
5880
|
+
suiteCase.id
|
|
5881
|
+
)
|
|
5882
|
+
);
|
|
5883
|
+
}
|
|
5884
|
+
}
|
|
5885
|
+
if (expectation.checkStatus === "fail" && checkResult.diagnostics.some((item) => item.severity === "error") && checkResult.status === "error") {
|
|
5886
|
+
diagnostics.push(
|
|
5887
|
+
diagnostic4(
|
|
5888
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5889
|
+
'Check ended in evaluator/config error; semantic failure expectation requires status "fail".',
|
|
5890
|
+
"error",
|
|
5891
|
+
suiteCase.id
|
|
5892
|
+
)
|
|
5893
|
+
);
|
|
5894
|
+
}
|
|
5895
|
+
return { ok: diagnostics.length === 0, diagnostics };
|
|
5896
|
+
}
|
|
5779
5897
|
async function runSuiteCase(suiteCase, config, options) {
|
|
5780
5898
|
const resolved = await resolveSuiteCaseTrace(suiteCase, options);
|
|
5781
5899
|
if (resolved.missing || resolved.tracePath === void 0) {
|
|
@@ -5828,9 +5946,21 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
5828
5946
|
status: "pass",
|
|
5829
5947
|
format: read.format,
|
|
5830
5948
|
findings: [],
|
|
5831
|
-
diagnostics: []
|
|
5949
|
+
diagnostics: [],
|
|
5950
|
+
ruleExecutions: []
|
|
5951
|
+
};
|
|
5832
5952
|
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
5833
|
-
const
|
|
5953
|
+
const checkOk = checkResult.ok;
|
|
5954
|
+
const observationsOk = observationResult.ok;
|
|
5955
|
+
const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
|
|
5956
|
+
(finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
|
|
5957
|
+
) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
|
|
5958
|
+
const expectResult = evaluateCaseExpect(
|
|
5959
|
+
suiteCase,
|
|
5960
|
+
checkResult,
|
|
5961
|
+
compiled.observationCount
|
|
5962
|
+
);
|
|
5963
|
+
const checkFailureDiagnostics = [
|
|
5834
5964
|
...checkResult.diagnostics.map(
|
|
5835
5965
|
(item) => diagnostic4(
|
|
5836
5966
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
@@ -5846,15 +5976,24 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
5846
5976
|
finding.severity,
|
|
5847
5977
|
suiteCase.id
|
|
5848
5978
|
)
|
|
5849
|
-
)
|
|
5850
|
-
...observationResult.diagnostics
|
|
5979
|
+
)
|
|
5851
5980
|
];
|
|
5852
|
-
const
|
|
5853
|
-
const
|
|
5854
|
-
|
|
5855
|
-
|
|
5856
|
-
|
|
5857
|
-
|
|
5981
|
+
const expectMatched = suiteCase.expect !== void 0 && expectResult.ok && observationsOk;
|
|
5982
|
+
const diagnostics = expectMatched ? [
|
|
5983
|
+
...observationResult.diagnostics,
|
|
5984
|
+
...expectResult.diagnostics,
|
|
5985
|
+
diagnostic4(
|
|
5986
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
5987
|
+
`Expected semantic failure matched (check.status=${checkResult.status}).`,
|
|
5988
|
+
"info",
|
|
5989
|
+
suiteCase.id
|
|
5990
|
+
)
|
|
5991
|
+
] : [
|
|
5992
|
+
...checkFailureDiagnostics,
|
|
5993
|
+
...observationResult.diagnostics,
|
|
5994
|
+
...expectResult.diagnostics
|
|
5995
|
+
];
|
|
5996
|
+
const ok = suiteCase.expect !== void 0 ? expectResult.ok && observationsOk : checkOk && observationsOk;
|
|
5858
5997
|
const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
|
|
5859
5998
|
return {
|
|
5860
5999
|
id: suiteCase.id,
|