@agent-inspect/viewer 6.31.10 → 6.31.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +934 -806
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.mjs +934 -806
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -1163,870 +1163,870 @@ function buildRunTimeline(events, options = {}) {
|
|
|
1163
1163
|
};
|
|
1164
1164
|
}
|
|
1165
1165
|
|
|
1166
|
-
// packages/core/src/
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1166
|
+
// packages/core/src/diagnostics/programmatic.ts
|
|
1167
|
+
var PROGRAMMATIC_DIAGNOSTIC_SPECS = Object.freeze({
|
|
1168
|
+
AI_TRACE_INPUT_INVALID: {
|
|
1169
|
+
code: "AI_TRACE_INPUT_INVALID",
|
|
1170
|
+
summary: 'Expected { type: "file", path }, { type: "directory", path }, { type: "string", content }, { type: "buffer", content }, or { type: "stdin" }.',
|
|
1171
|
+
remediation: "For a file path, use openTraceFile(path).",
|
|
1172
|
+
relatedCodes: ["invalid_input"]
|
|
1173
|
+
},
|
|
1174
|
+
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
1175
|
+
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
1176
|
+
summary: "No trace reader could detect the input format.",
|
|
1177
|
+
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
1178
|
+
relatedCodes: ["unsupported_format"]
|
|
1179
|
+
},
|
|
1180
|
+
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
1181
|
+
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
1182
|
+
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
1183
|
+
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
1184
|
+
relatedCodes: ["ambiguous_format"]
|
|
1185
|
+
},
|
|
1186
|
+
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
1187
|
+
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
1188
|
+
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
1189
|
+
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
1190
|
+
},
|
|
1191
|
+
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
1192
|
+
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
1193
|
+
summary: "Multiple runs are available; select a run before executing checks.",
|
|
1194
|
+
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
1195
|
+
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
1196
|
+
},
|
|
1197
|
+
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
1198
|
+
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1199
|
+
summary: "A parentId equals its own event/step id.",
|
|
1200
|
+
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
1201
|
+
relatedCodes: [
|
|
1202
|
+
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
1203
|
+
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
1204
|
+
]
|
|
1205
|
+
},
|
|
1206
|
+
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
1207
|
+
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
1208
|
+
summary: "Trace contains a parentId cycle.",
|
|
1209
|
+
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
1210
|
+
relatedCodes: ["structure.cycle"]
|
|
1211
|
+
},
|
|
1212
|
+
AI_TRACE_CONTRACT_INPUT_INVALID: {
|
|
1213
|
+
code: "AI_TRACE_CONTRACT_INPUT_INVALID",
|
|
1214
|
+
summary: "evaluateTraceContract expects ({ read }, contract) or use evaluateTraceContractRead(read, contract).",
|
|
1215
|
+
remediation: "Pass TraceCheckInput as the first argument: evaluateTraceContract({ read }, contract).",
|
|
1216
|
+
relatedCodes: ["AI_CHECK_INVALID_ARGUMENTS"]
|
|
1217
|
+
},
|
|
1218
|
+
AI_INSPECT_RUN_UNSUPPORTED_OPTION: {
|
|
1219
|
+
code: "AI_INSPECT_RUN_UNSUPPORTED_OPTION",
|
|
1220
|
+
summary: 'inspectRun does not accept a "writer" option.',
|
|
1221
|
+
remediation: "Use createInspector({ writer }) then inspector.run(...). inspectRun persists via the global JSONL path (schema 0.1)."
|
|
1176
1222
|
}
|
|
1177
|
-
|
|
1223
|
+
});
|
|
1224
|
+
function formatProgrammaticDiagnostic(code, detail) {
|
|
1225
|
+
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
1226
|
+
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
1227
|
+
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
1178
1228
|
}
|
|
1179
1229
|
|
|
1180
|
-
// packages/core/src/
|
|
1181
|
-
function
|
|
1182
|
-
return typeof value === "
|
|
1183
|
-
}
|
|
1184
|
-
function finitePositiveInt(value) {
|
|
1185
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
1186
|
-
return void 0;
|
|
1187
|
-
}
|
|
1188
|
-
return Math.trunc(value);
|
|
1230
|
+
// packages/core/src/checks/logical-events.ts
|
|
1231
|
+
function isRecord5(value) {
|
|
1232
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1189
1233
|
}
|
|
1190
|
-
function
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
let found = false;
|
|
1194
|
-
const assignString = (key, value) => {
|
|
1195
|
-
if (isNonEmptyString3(value)) {
|
|
1196
|
-
out[key] = value.trim();
|
|
1197
|
-
found = true;
|
|
1198
|
-
}
|
|
1199
|
-
};
|
|
1200
|
-
assignString("sessionId", record.sessionId);
|
|
1201
|
-
assignString("conversationId", record.conversationId);
|
|
1202
|
-
assignString("groupId", record.groupId);
|
|
1203
|
-
assignString("parentGroupId", record.parentGroupId);
|
|
1204
|
-
assignString("retryOf", record.retryOf);
|
|
1205
|
-
assignString("retryReason", record.retryReason);
|
|
1206
|
-
assignString("handoffFrom", record.handoffFrom);
|
|
1207
|
-
assignString("handoffTo", record.handoffTo);
|
|
1208
|
-
assignString("subAgentId", record.subAgentId);
|
|
1209
|
-
assignString("subAgentName", record.subAgentName);
|
|
1210
|
-
assignString("jobId", record.jobId);
|
|
1211
|
-
assignString("queueName", record.queueName);
|
|
1212
|
-
assignString("workflowName", record.workflowName);
|
|
1213
|
-
assignString("workflowStep", record.workflowStep);
|
|
1214
|
-
assignString("toolCallId", record.toolCallId);
|
|
1215
|
-
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
1216
|
-
assignString("linkedStepId", record.linkedStepId);
|
|
1217
|
-
assignString("operationId", record.operationId);
|
|
1218
|
-
assignString("attemptId", record.attemptId);
|
|
1219
|
-
assignString("fallbackOf", record.fallbackOf);
|
|
1220
|
-
assignString("idempotencyKey", record.idempotencyKey);
|
|
1221
|
-
assignString("correlationId", record.correlationId);
|
|
1222
|
-
assignString("requestId", record.requestId);
|
|
1223
|
-
assignString("decisionId", record.decisionId);
|
|
1224
|
-
const attempt = finitePositiveInt(record.attempt);
|
|
1225
|
-
if (attempt !== void 0) {
|
|
1226
|
-
out.attempt = attempt;
|
|
1227
|
-
found = true;
|
|
1228
|
-
}
|
|
1229
|
-
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
1230
|
-
if (attemptNumber !== void 0) {
|
|
1231
|
-
out.attemptNumber = attemptNumber;
|
|
1232
|
-
found = true;
|
|
1233
|
-
}
|
|
1234
|
-
return found ? out : void 0;
|
|
1234
|
+
function legacyEvent(event) {
|
|
1235
|
+
const value = event.attributes?.legacyEvent;
|
|
1236
|
+
return typeof value === "string" ? value : void 0;
|
|
1235
1237
|
}
|
|
1236
|
-
function
|
|
1237
|
-
|
|
1238
|
-
if (
|
|
1239
|
-
return `group:${meta.groupId}`;
|
|
1240
|
-
}
|
|
1238
|
+
function stepIdOf(event) {
|
|
1239
|
+
const value = event.attributes?.stepId;
|
|
1240
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
1241
1241
|
return void 0;
|
|
1242
1242
|
}
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
completed: 1
|
|
1252
|
-
};
|
|
1253
|
-
var EXPLICIT_SESSION_STATUSES = /* @__PURE__ */ new Set([
|
|
1254
|
-
"running",
|
|
1255
|
-
"waiting_input",
|
|
1256
|
-
"idle",
|
|
1257
|
-
"completed",
|
|
1258
|
-
"error",
|
|
1259
|
-
"stale",
|
|
1260
|
-
"unknown"
|
|
1261
|
-
]);
|
|
1262
|
-
function isExplicitSessionStatus(value) {
|
|
1263
|
-
return typeof value === "string" && EXPLICIT_SESSION_STATUSES.has(value);
|
|
1264
|
-
}
|
|
1265
|
-
function activityMs(run) {
|
|
1266
|
-
return run.endedAt ?? run.startedAt ?? 0;
|
|
1243
|
+
function cloneEvent(event) {
|
|
1244
|
+
return {
|
|
1245
|
+
...event,
|
|
1246
|
+
...event.attributes !== void 0 ? { attributes: { ...event.attributes } } : {},
|
|
1247
|
+
...event.error !== void 0 ? { error: { ...event.error } } : {},
|
|
1248
|
+
...event.tokenUsage !== void 0 ? { tokenUsage: { ...event.tokenUsage } } : {},
|
|
1249
|
+
...event.source !== void 0 ? { source: { ...event.source } } : {}
|
|
1250
|
+
};
|
|
1267
1251
|
}
|
|
1268
|
-
function
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1252
|
+
function mergeAttributes(start, complete) {
|
|
1253
|
+
const merged = {
|
|
1254
|
+
...isRecord5(complete.attributes) ? complete.attributes : {},
|
|
1255
|
+
...isRecord5(start.attributes) ? start.attributes : {}
|
|
1256
|
+
};
|
|
1257
|
+
merged.legacyEvent = start.attributes?.legacyEvent ?? complete.attributes?.legacyEvent;
|
|
1258
|
+
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
1259
|
+
if (complete.attributes?.errorStack !== void 0) {
|
|
1260
|
+
merged.errorStack = complete.attributes.errorStack;
|
|
1273
1261
|
}
|
|
1274
|
-
return
|
|
1262
|
+
return Object.keys(merged).length > 0 ? merged : void 0;
|
|
1275
1263
|
}
|
|
1276
|
-
function
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1264
|
+
function pairStartComplete(start, complete) {
|
|
1265
|
+
const attributes = mergeAttributes(start, complete);
|
|
1266
|
+
const paired = {
|
|
1267
|
+
...cloneEvent(start),
|
|
1268
|
+
status: complete.status,
|
|
1269
|
+
timestamp: complete.timestamp ?? start.timestamp,
|
|
1270
|
+
...complete.endedAt !== void 0 ? { endedAt: complete.endedAt } : {},
|
|
1271
|
+
...complete.durationMs !== void 0 ? { durationMs: complete.durationMs } : {},
|
|
1272
|
+
...complete.error !== void 0 ? { error: { ...complete.error } } : {},
|
|
1273
|
+
...complete.tokenUsage !== void 0 && start.tokenUsage === void 0 ? { tokenUsage: { ...complete.tokenUsage } } : {},
|
|
1274
|
+
...attributes !== void 0 ? { attributes } : {}
|
|
1275
|
+
};
|
|
1276
|
+
return {
|
|
1277
|
+
...paired,
|
|
1278
|
+
sourceEventIds: Object.freeze([start.eventId, complete.eventId]),
|
|
1279
|
+
projection: {
|
|
1280
|
+
paired: true,
|
|
1281
|
+
absorbedEventIds: Object.freeze([complete.eventId]),
|
|
1282
|
+
parentNormalized: false,
|
|
1283
|
+
...start.parentId !== void 0 ? { originalParentId: start.parentId } : {}
|
|
1282
1284
|
}
|
|
1283
|
-
}
|
|
1284
|
-
return earliest;
|
|
1285
|
-
}
|
|
1286
|
-
function latestEndWhenAllEnded(runs) {
|
|
1287
|
-
if (runs.length === 0) return void 0;
|
|
1288
|
-
let latest;
|
|
1289
|
-
for (const run of runs) {
|
|
1290
|
-
if (run.endedAt === void 0) return void 0;
|
|
1291
|
-
if (latest === void 0 || run.endedAt > latest) latest = run.endedAt;
|
|
1292
|
-
}
|
|
1293
|
-
return latest;
|
|
1285
|
+
};
|
|
1294
1286
|
}
|
|
1295
|
-
function
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
best = raw;
|
|
1287
|
+
function asLogical(event, extras) {
|
|
1288
|
+
return {
|
|
1289
|
+
...cloneEvent(event),
|
|
1290
|
+
sourceEventIds: Object.freeze([event.eventId]),
|
|
1291
|
+
projection: {
|
|
1292
|
+
paired: false,
|
|
1293
|
+
absorbedEventIds: Object.freeze([]),
|
|
1294
|
+
parentNormalized: extras?.parentNormalized === true,
|
|
1295
|
+
...{}
|
|
1305
1296
|
}
|
|
1306
|
-
}
|
|
1307
|
-
return best;
|
|
1297
|
+
};
|
|
1308
1298
|
}
|
|
1309
|
-
function
|
|
1310
|
-
const
|
|
1311
|
-
const
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
return { runId: latest.runId, message, code };
|
|
1317
|
-
}
|
|
1318
|
-
function deriveCheckSummary(runs) {
|
|
1319
|
-
let pass = 0;
|
|
1320
|
-
let fail = 0;
|
|
1321
|
-
let warn2 = 0;
|
|
1322
|
-
let found = false;
|
|
1323
|
-
for (const run of runs) {
|
|
1324
|
-
const summary = run.metadata?.checkSummary;
|
|
1325
|
-
if (!summary || typeof summary !== "object") continue;
|
|
1326
|
-
const record = summary;
|
|
1327
|
-
if (typeof record.pass === "number") {
|
|
1328
|
-
pass += record.pass;
|
|
1329
|
-
found = true;
|
|
1330
|
-
}
|
|
1331
|
-
if (typeof record.fail === "number") {
|
|
1332
|
-
fail += record.fail;
|
|
1333
|
-
found = true;
|
|
1334
|
-
}
|
|
1335
|
-
if (typeof record.warn === "number") {
|
|
1336
|
-
warn2 += record.warn;
|
|
1337
|
-
found = true;
|
|
1338
|
-
}
|
|
1299
|
+
function projectLogicalEvents(events) {
|
|
1300
|
+
const diagnostics = [];
|
|
1301
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
1302
|
+
for (const event of events) {
|
|
1303
|
+
const list = byRun.get(event.runId) ?? [];
|
|
1304
|
+
list.push(event);
|
|
1305
|
+
byRun.set(event.runId, list);
|
|
1339
1306
|
}
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1307
|
+
const absorbedIds = /* @__PURE__ */ new Set();
|
|
1308
|
+
const logicalByRawId = /* @__PURE__ */ new Map();
|
|
1309
|
+
const stepIdToLogicalId = /* @__PURE__ */ new Map();
|
|
1310
|
+
const logical = [];
|
|
1311
|
+
const orderedRuns = [...byRun.keys()].sort((a, b) => a.localeCompare(b));
|
|
1312
|
+
for (const runId of orderedRuns) {
|
|
1313
|
+
const runEvents = byRun.get(runId) ?? [];
|
|
1314
|
+
const starts = [];
|
|
1315
|
+
const completes = [];
|
|
1316
|
+
const others = [];
|
|
1317
|
+
for (const event of runEvents) {
|
|
1318
|
+
const legacy = legacyEvent(event);
|
|
1319
|
+
if (legacy === "step_started" || legacy === "run_started" && event.status === "running") {
|
|
1320
|
+
starts.push(event);
|
|
1321
|
+
} else if (legacy === "step_completed" || legacy === "run_completed") {
|
|
1322
|
+
completes.push(event);
|
|
1323
|
+
} else {
|
|
1324
|
+
others.push(event);
|
|
1325
|
+
}
|
|
1347
1326
|
}
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
}
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
checkSummary: deriveCheckSummary(sessionRuns)
|
|
1398
|
-
};
|
|
1399
|
-
}
|
|
1400
|
-
|
|
1401
|
-
// packages/core/src/sessions/load.ts
|
|
1402
|
-
async function enrichSessionRunRecord(meta) {
|
|
1403
|
-
let metadata;
|
|
1404
|
-
try {
|
|
1405
|
-
const events = await readTraceEventsFromFile(meta.filePath);
|
|
1406
|
-
for (const event of events) {
|
|
1407
|
-
if (event.event !== "run_started") continue;
|
|
1408
|
-
if (event.metadata && typeof event.metadata === "object") {
|
|
1409
|
-
metadata = event.metadata;
|
|
1327
|
+
const usedCompletes = /* @__PURE__ */ new Set();
|
|
1328
|
+
for (const start of starts) {
|
|
1329
|
+
const stepId = stepIdOf(start);
|
|
1330
|
+
let match;
|
|
1331
|
+
if (legacyEvent(start) === "run_started") {
|
|
1332
|
+
const candidates = completes.filter(
|
|
1333
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
1334
|
+
);
|
|
1335
|
+
if (candidates.length > 1) {
|
|
1336
|
+
diagnostics.push({
|
|
1337
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1338
|
+
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
1339
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1340
|
+
});
|
|
1341
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1342
|
+
}
|
|
1343
|
+
match = candidates[0];
|
|
1344
|
+
} else if (stepId) {
|
|
1345
|
+
const candidates = completes.filter(
|
|
1346
|
+
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
1347
|
+
);
|
|
1348
|
+
if (candidates.length > 1) {
|
|
1349
|
+
diagnostics.push({
|
|
1350
|
+
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1351
|
+
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
1352
|
+
eventIds: candidates.map((c) => c.eventId)
|
|
1353
|
+
});
|
|
1354
|
+
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1355
|
+
}
|
|
1356
|
+
match = candidates[0];
|
|
1357
|
+
}
|
|
1358
|
+
if (match) {
|
|
1359
|
+
usedCompletes.add(match.eventId);
|
|
1360
|
+
absorbedIds.add(match.eventId);
|
|
1361
|
+
const paired = pairStartComplete(start, match);
|
|
1362
|
+
logical.push(paired);
|
|
1363
|
+
logicalByRawId.set(start.eventId, paired);
|
|
1364
|
+
logicalByRawId.set(match.eventId, paired);
|
|
1365
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
1366
|
+
} else {
|
|
1367
|
+
diagnostics.push({
|
|
1368
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
1369
|
+
message: `No matching complete for start ${start.eventId}.`,
|
|
1370
|
+
eventIds: [start.eventId]
|
|
1371
|
+
});
|
|
1372
|
+
const alone = asLogical(start);
|
|
1373
|
+
logical.push(alone);
|
|
1374
|
+
logicalByRawId.set(start.eventId, alone);
|
|
1375
|
+
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1410
1376
|
}
|
|
1411
|
-
break;
|
|
1412
1377
|
}
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
startedAt: meta.startedAt,
|
|
1420
|
-
endedAt: meta.endedAt,
|
|
1421
|
-
durationMs: meta.durationMs,
|
|
1422
|
-
filePath: meta.filePath,
|
|
1423
|
-
metadata
|
|
1424
|
-
};
|
|
1425
|
-
}
|
|
1426
|
-
async function loadSessionRunRecords(metas) {
|
|
1427
|
-
const out = [];
|
|
1428
|
-
for (const meta of metas) {
|
|
1429
|
-
out.push(await enrichSessionRunRecord(meta));
|
|
1430
|
-
}
|
|
1431
|
-
return out;
|
|
1432
|
-
}
|
|
1433
|
-
|
|
1434
|
-
// packages/core/src/sessions/index.ts
|
|
1435
|
-
function compareRuns(a, b) {
|
|
1436
|
-
const aStart = a.startedAt ?? 0;
|
|
1437
|
-
const bStart = b.startedAt ?? 0;
|
|
1438
|
-
if (aStart !== bStart) return aStart - bStart;
|
|
1439
|
-
return a.runId.localeCompare(b.runId);
|
|
1440
|
-
}
|
|
1441
|
-
function buildGroups(runIds, metaByRunId) {
|
|
1442
|
-
const byGroup = /* @__PURE__ */ new Map();
|
|
1443
|
-
for (const runId of runIds) {
|
|
1444
|
-
const groupId = metaByRunId.get(runId)?.groupId;
|
|
1445
|
-
if (!groupId) continue;
|
|
1446
|
-
const existing = byGroup.get(groupId);
|
|
1447
|
-
if (existing) {
|
|
1448
|
-
existing.runIds.push(runId);
|
|
1449
|
-
} else {
|
|
1450
|
-
byGroup.set(groupId, {
|
|
1451
|
-
groupId,
|
|
1452
|
-
parentGroupId: metaByRunId.get(runId)?.parentGroupId,
|
|
1453
|
-
runIds: [runId]
|
|
1378
|
+
for (const complete of completes) {
|
|
1379
|
+
if (usedCompletes.has(complete.eventId)) continue;
|
|
1380
|
+
diagnostics.push({
|
|
1381
|
+
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
1382
|
+
message: `No matching start for complete ${complete.eventId}.`,
|
|
1383
|
+
eventIds: [complete.eventId]
|
|
1454
1384
|
});
|
|
1385
|
+
const alone = asLogical(complete);
|
|
1386
|
+
logical.push(alone);
|
|
1387
|
+
logicalByRawId.set(complete.eventId, alone);
|
|
1388
|
+
const stepId = stepIdOf(complete);
|
|
1389
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1390
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
for (const event of others) {
|
|
1394
|
+
const alone = asLogical(event);
|
|
1395
|
+
logical.push(alone);
|
|
1396
|
+
logicalByRawId.set(event.eventId, alone);
|
|
1397
|
+
const stepId = stepIdOf(event);
|
|
1398
|
+
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1399
|
+
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1400
|
+
}
|
|
1455
1401
|
}
|
|
1456
1402
|
}
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
const edges = [];
|
|
1464
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1465
|
-
const pushEdge = (edge) => {
|
|
1466
|
-
const key = `${edge.from}->${edge.to}:${edge.confidence}`;
|
|
1467
|
-
if (seen.has(key)) return;
|
|
1468
|
-
seen.add(key);
|
|
1469
|
-
edges.push(edge);
|
|
1470
|
-
};
|
|
1471
|
-
for (const runId of runIds) {
|
|
1472
|
-
const meta = metaByRunId.get(runId);
|
|
1473
|
-
if (!meta) continue;
|
|
1474
|
-
if (meta.handoffFrom && meta.handoffTo) {
|
|
1475
|
-
pushEdge({
|
|
1476
|
-
from: meta.handoffFrom,
|
|
1477
|
-
to: meta.handoffTo,
|
|
1478
|
-
source: "manual",
|
|
1479
|
-
confidence: "explicit"
|
|
1480
|
-
});
|
|
1403
|
+
const logicalById = new Map(logical.map((e) => [e.eventId, e]));
|
|
1404
|
+
const normalized = [];
|
|
1405
|
+
for (const event of logical) {
|
|
1406
|
+
const originalParentId = event.parentId;
|
|
1407
|
+
if (!originalParentId) {
|
|
1408
|
+
normalized.push(event);
|
|
1481
1409
|
continue;
|
|
1482
1410
|
}
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
source: "manual",
|
|
1496
|
-
confidence: "explicit"
|
|
1497
|
-
});
|
|
1498
|
-
}
|
|
1499
|
-
if (meta.subAgentId && meta.parentGroupId && !meta.handoffFrom && !meta.handoffTo) {
|
|
1500
|
-
pushEdge({
|
|
1501
|
-
from: meta.parentGroupId,
|
|
1502
|
-
to: meta.subAgentId,
|
|
1503
|
-
source: "inferred",
|
|
1504
|
-
confidence: "correlated"
|
|
1505
|
-
});
|
|
1506
|
-
warnings.push({
|
|
1507
|
-
code: "ambiguous-handoff-endpoints",
|
|
1508
|
-
message: "Handoff inferred from parentGroupId and subAgentId without explicit handoffFrom/handoffTo.",
|
|
1509
|
-
runId,
|
|
1510
|
-
sessionId
|
|
1511
|
-
});
|
|
1411
|
+
let nextParent = originalParentId;
|
|
1412
|
+
let remapped = false;
|
|
1413
|
+
const viaAbsorbed = logicalByRawId.get(originalParentId);
|
|
1414
|
+
if (viaAbsorbed && viaAbsorbed.eventId !== originalParentId) {
|
|
1415
|
+
nextParent = viaAbsorbed.eventId;
|
|
1416
|
+
remapped = true;
|
|
1417
|
+
} else if (!logicalById.has(originalParentId)) {
|
|
1418
|
+
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
1419
|
+
if (viaStep) {
|
|
1420
|
+
nextParent = viaStep;
|
|
1421
|
+
remapped = true;
|
|
1422
|
+
}
|
|
1512
1423
|
}
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
}
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1424
|
+
if (!remapped) {
|
|
1425
|
+
if (originalParentId === event.eventId) {
|
|
1426
|
+
diagnostics.push({
|
|
1427
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1428
|
+
message: formatProgrammaticDiagnostic(
|
|
1429
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1430
|
+
`Removed self-parent edge on ${event.eventId}.`
|
|
1431
|
+
),
|
|
1432
|
+
eventIds: [event.eventId]
|
|
1433
|
+
});
|
|
1434
|
+
const { parentId: _drop, ...rest } = event;
|
|
1435
|
+
normalized.push({
|
|
1436
|
+
...rest,
|
|
1437
|
+
projection: {
|
|
1438
|
+
...event.projection,
|
|
1439
|
+
parentNormalized: true,
|
|
1440
|
+
originalParentId
|
|
1441
|
+
}
|
|
1442
|
+
});
|
|
1443
|
+
continue;
|
|
1444
|
+
}
|
|
1445
|
+
if (!logicalById.has(originalParentId) && !logicalByRawId.has(originalParentId)) {
|
|
1446
|
+
const mapping = event.attributes?.parentMapping;
|
|
1447
|
+
const unresolved = mapping === "unresolved" || event.attributes?.parentUnresolved === true || event.attributes?.unresolvedParent === true || // LangGraph/framework scaffolding sentinel labels are not event ids.
|
|
1448
|
+
/^LangGraph$/i.test(originalParentId) || originalParentId.startsWith("unresolved:");
|
|
1449
|
+
if (!unresolved) {
|
|
1450
|
+
diagnostics.push({
|
|
1451
|
+
code: "AI_LOGICAL_PARENT_UNRESOLVED",
|
|
1452
|
+
message: `Parent ${originalParentId} unresolved for ${event.eventId}.`,
|
|
1453
|
+
eventIds: [event.eventId]
|
|
1454
|
+
});
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
normalized.push(event);
|
|
1533
1458
|
continue;
|
|
1534
1459
|
}
|
|
1535
|
-
if (
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1460
|
+
if (nextParent === event.eventId) {
|
|
1461
|
+
diagnostics.push({
|
|
1462
|
+
code: "AI_LOGICAL_SELF_PARENT_REMOVED",
|
|
1463
|
+
message: formatProgrammaticDiagnostic(
|
|
1464
|
+
"AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1465
|
+
`Removed self-parent edge on ${event.eventId} (was ${originalParentId}).`
|
|
1466
|
+
),
|
|
1467
|
+
eventIds: [event.eventId]
|
|
1541
1468
|
});
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1469
|
+
const { parentId: _drop, ...rest } = event;
|
|
1470
|
+
normalized.push({
|
|
1471
|
+
...rest,
|
|
1472
|
+
projection: {
|
|
1473
|
+
...event.projection,
|
|
1474
|
+
parentNormalized: true,
|
|
1475
|
+
originalParentId
|
|
1476
|
+
}
|
|
1547
1477
|
});
|
|
1478
|
+
continue;
|
|
1548
1479
|
}
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
}
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
const visited = /* @__PURE__ */ new Set();
|
|
1563
|
-
const pushRun = (run, confidence, source) => {
|
|
1564
|
-
if (visited.has(run.runId)) return;
|
|
1565
|
-
visited.add(run.runId);
|
|
1566
|
-
path18.push({
|
|
1567
|
-
runId: run.runId,
|
|
1568
|
-
name: run.name,
|
|
1569
|
-
startedAt: run.startedAt,
|
|
1570
|
-
durationMs: run.durationMs,
|
|
1571
|
-
confidence,
|
|
1572
|
-
source
|
|
1480
|
+
diagnostics.push({
|
|
1481
|
+
code: "AI_LOGICAL_PARENT_REMAPPED",
|
|
1482
|
+
message: `Remapped parent ${originalParentId} \u2192 ${nextParent} for ${event.eventId}.`,
|
|
1483
|
+
eventIds: [event.eventId]
|
|
1484
|
+
});
|
|
1485
|
+
normalized.push({
|
|
1486
|
+
...event,
|
|
1487
|
+
parentId: nextParent,
|
|
1488
|
+
projection: {
|
|
1489
|
+
...event.projection,
|
|
1490
|
+
parentNormalized: true,
|
|
1491
|
+
originalParentId
|
|
1492
|
+
}
|
|
1573
1493
|
});
|
|
1494
|
+
}
|
|
1495
|
+
const rawIndex = new Map(events.map((e, i) => [e.eventId, i]));
|
|
1496
|
+
normalized.sort((a, b) => {
|
|
1497
|
+
const ai = rawIndex.get(a.sourceEventIds[0]) ?? 0;
|
|
1498
|
+
const bi = rawIndex.get(b.sourceEventIds[0]) ?? 0;
|
|
1499
|
+
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
1500
|
+
});
|
|
1501
|
+
return {
|
|
1502
|
+
logicalEvents: Object.freeze(normalized),
|
|
1503
|
+
diagnostics: Object.freeze(diagnostics)
|
|
1574
1504
|
};
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
);
|
|
1583
|
-
if (
|
|
1584
|
-
if (toRun) pushRun(toRun, "explicit", "manual");
|
|
1505
|
+
}
|
|
1506
|
+
function resolveCanonicalToolName(event) {
|
|
1507
|
+
const attrs = event.attributes;
|
|
1508
|
+
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
1509
|
+
if (direct) return direct;
|
|
1510
|
+
const metadata = attrs?.metadata;
|
|
1511
|
+
if (isRecord5(metadata)) {
|
|
1512
|
+
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
1513
|
+
if (nested) return nested;
|
|
1585
1514
|
}
|
|
1586
|
-
for (const
|
|
1587
|
-
if (
|
|
1588
|
-
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
1589
|
-
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
1515
|
+
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
1516
|
+
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
1590
1517
|
}
|
|
1591
|
-
return
|
|
1592
|
-
}
|
|
1593
|
-
function metaRunIdMatches(run, token, runById) {
|
|
1594
|
-
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1595
|
-
return meta?.subAgentId === token || meta?.groupId === token || runById.has(token);
|
|
1518
|
+
return event.name;
|
|
1596
1519
|
}
|
|
1597
|
-
function
|
|
1598
|
-
|
|
1599
|
-
const
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
metaByRunId.set(run.runId, extractSessionWorkflowMetadata(run.metadata));
|
|
1520
|
+
function pickString(record, keys) {
|
|
1521
|
+
if (!record) return void 0;
|
|
1522
|
+
for (const key of keys) {
|
|
1523
|
+
const value = record[key];
|
|
1524
|
+
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
1603
1525
|
}
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1526
|
+
return void 0;
|
|
1527
|
+
}
|
|
1528
|
+
|
|
1529
|
+
// packages/core/src/search.ts
|
|
1530
|
+
async function loadTraceMetadataList(_traceDir, fileNames, getPath) {
|
|
1531
|
+
const metas = [];
|
|
1532
|
+
for (const fileName of fileNames) {
|
|
1533
|
+
try {
|
|
1534
|
+
const filePath = getPath(fileName);
|
|
1535
|
+
const meta = await extractMetadata(filePath);
|
|
1536
|
+
metas.push(meta);
|
|
1537
|
+
} catch {
|
|
1614
1538
|
}
|
|
1615
|
-
const bucket = sessionsByKey.get(key) ?? [];
|
|
1616
|
-
bucket.push(run);
|
|
1617
|
-
sessionsByKey.set(key, bucket);
|
|
1618
1539
|
}
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1540
|
+
return metas;
|
|
1541
|
+
}
|
|
1542
|
+
|
|
1543
|
+
// packages/core/src/sessions/metadata.ts
|
|
1544
|
+
function isNonEmptyString3(value) {
|
|
1545
|
+
return typeof value === "string" && value.trim() !== "";
|
|
1546
|
+
}
|
|
1547
|
+
function finitePositiveInt(value) {
|
|
1548
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
1549
|
+
return void 0;
|
|
1550
|
+
}
|
|
1551
|
+
return Math.trunc(value);
|
|
1552
|
+
}
|
|
1553
|
+
function extractSessionWorkflowMetadata(record) {
|
|
1554
|
+
if (!record) return void 0;
|
|
1555
|
+
const out = {};
|
|
1556
|
+
let found = false;
|
|
1557
|
+
const assignString = (key, value) => {
|
|
1558
|
+
if (isNonEmptyString3(value)) {
|
|
1559
|
+
out[key] = value.trim();
|
|
1560
|
+
found = true;
|
|
1632
1561
|
}
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1562
|
+
};
|
|
1563
|
+
assignString("sessionId", record.sessionId);
|
|
1564
|
+
assignString("conversationId", record.conversationId);
|
|
1565
|
+
assignString("groupId", record.groupId);
|
|
1566
|
+
assignString("parentGroupId", record.parentGroupId);
|
|
1567
|
+
assignString("retryOf", record.retryOf);
|
|
1568
|
+
assignString("retryReason", record.retryReason);
|
|
1569
|
+
assignString("handoffFrom", record.handoffFrom);
|
|
1570
|
+
assignString("handoffTo", record.handoffTo);
|
|
1571
|
+
assignString("subAgentId", record.subAgentId);
|
|
1572
|
+
assignString("subAgentName", record.subAgentName);
|
|
1573
|
+
assignString("jobId", record.jobId);
|
|
1574
|
+
assignString("queueName", record.queueName);
|
|
1575
|
+
assignString("workflowName", record.workflowName);
|
|
1576
|
+
assignString("workflowStep", record.workflowStep);
|
|
1577
|
+
assignString("toolCallId", record.toolCallId);
|
|
1578
|
+
assignString("mcpToolCallId", record.mcpToolCallId);
|
|
1579
|
+
assignString("linkedStepId", record.linkedStepId);
|
|
1580
|
+
assignString("operationId", record.operationId);
|
|
1581
|
+
assignString("attemptId", record.attemptId);
|
|
1582
|
+
assignString("fallbackOf", record.fallbackOf);
|
|
1583
|
+
assignString("idempotencyKey", record.idempotencyKey);
|
|
1584
|
+
assignString("correlationId", record.correlationId);
|
|
1585
|
+
assignString("requestId", record.requestId);
|
|
1586
|
+
assignString("decisionId", record.decisionId);
|
|
1587
|
+
const attempt = finitePositiveInt(record.attempt);
|
|
1588
|
+
if (attempt !== void 0) {
|
|
1589
|
+
out.attempt = attempt;
|
|
1590
|
+
found = true;
|
|
1591
|
+
}
|
|
1592
|
+
const attemptNumber = finitePositiveInt(record.attemptNumber);
|
|
1593
|
+
if (attemptNumber !== void 0) {
|
|
1594
|
+
out.attemptNumber = attemptNumber;
|
|
1595
|
+
found = true;
|
|
1596
|
+
}
|
|
1597
|
+
return found ? out : void 0;
|
|
1598
|
+
}
|
|
1599
|
+
function sessionKeyForRun(meta, options) {
|
|
1600
|
+
if (meta?.sessionId) return meta.sessionId;
|
|
1601
|
+
if (options?.correlateByGroupId && meta?.groupId) {
|
|
1602
|
+
return `group:${meta.groupId}`;
|
|
1603
|
+
}
|
|
1604
|
+
return void 0;
|
|
1605
|
+
}
|
|
1606
|
+
|
|
1607
|
+
// packages/core/src/sessions/status.ts
|
|
1608
|
+
var DEFAULT_STALE_THRESHOLD_MS = 864e5;
|
|
1609
|
+
var EXPLICIT_STATUS_PRIORITY = {
|
|
1610
|
+
error: 5,
|
|
1611
|
+
waiting_input: 4,
|
|
1612
|
+
idle: 3,
|
|
1613
|
+
stale: 2,
|
|
1614
|
+
completed: 1
|
|
1615
|
+
};
|
|
1616
|
+
var EXPLICIT_SESSION_STATUSES = /* @__PURE__ */ new Set([
|
|
1617
|
+
"running",
|
|
1618
|
+
"waiting_input",
|
|
1619
|
+
"idle",
|
|
1620
|
+
"completed",
|
|
1621
|
+
"error",
|
|
1622
|
+
"stale",
|
|
1623
|
+
"unknown"
|
|
1624
|
+
]);
|
|
1625
|
+
function isExplicitSessionStatus(value) {
|
|
1626
|
+
return typeof value === "string" && EXPLICIT_SESSION_STATUSES.has(value);
|
|
1627
|
+
}
|
|
1628
|
+
function activityMs(run) {
|
|
1629
|
+
return run.endedAt ?? run.startedAt ?? 0;
|
|
1630
|
+
}
|
|
1631
|
+
function latestActivityMs(runs) {
|
|
1632
|
+
let latest = 0;
|
|
1633
|
+
for (const run of runs) {
|
|
1634
|
+
const ms = activityMs(run);
|
|
1635
|
+
if (ms > latest) latest = ms;
|
|
1654
1636
|
}
|
|
1655
|
-
|
|
1656
|
-
const code = a.code.localeCompare(b.code);
|
|
1657
|
-
if (code !== 0) return code;
|
|
1658
|
-
return (a.runId ?? "").localeCompare(b.runId ?? "");
|
|
1659
|
-
});
|
|
1660
|
-
return {
|
|
1661
|
-
runs,
|
|
1662
|
-
sessions,
|
|
1663
|
-
unscopedRunIds: unscopedRunIds.sort(),
|
|
1664
|
-
warnings
|
|
1665
|
-
};
|
|
1637
|
+
return latest;
|
|
1666
1638
|
}
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
relatedCodes: ["invalid_input"]
|
|
1675
|
-
},
|
|
1676
|
-
AI_TRACE_FORMAT_UNSUPPORTED: {
|
|
1677
|
-
code: "AI_TRACE_FORMAT_UNSUPPORTED",
|
|
1678
|
-
summary: "No trace reader could detect the input format.",
|
|
1679
|
-
remediation: "Pass an AgentInspect JSONL file via openTraceFile, or set options.format to a registered reader.",
|
|
1680
|
-
relatedCodes: ["unsupported_format"]
|
|
1681
|
-
},
|
|
1682
|
-
AI_TRACE_FORMAT_AMBIGUOUS: {
|
|
1683
|
-
code: "AI_TRACE_FORMAT_AMBIGUOUS",
|
|
1684
|
-
summary: "Multiple trace readers matched the input with equal confidence.",
|
|
1685
|
-
remediation: "Set options.format explicitly to disambiguate the reader.",
|
|
1686
|
-
relatedCodes: ["ambiguous_format"]
|
|
1687
|
-
},
|
|
1688
|
-
AI_TRACE_FACTS_INPUT_NOT_NORMALIZED: {
|
|
1689
|
-
code: "AI_TRACE_FACTS_INPUT_NOT_NORMALIZED",
|
|
1690
|
-
summary: "TraceFacts requires TraceReadResult or PersistedInspectEvent[].",
|
|
1691
|
-
remediation: "Use openTraceFile() to normalize a JSONL trace first."
|
|
1692
|
-
},
|
|
1693
|
-
AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED: {
|
|
1694
|
-
code: "AI_TRACE_CONTRACT_RUN_SELECTION_REQUIRED",
|
|
1695
|
-
summary: "Multiple runs are available; select a run before executing checks.",
|
|
1696
|
-
remediation: "Pass options.runId or TraceCheckInput.selectedRun.",
|
|
1697
|
-
relatedCodes: ["AI_CHECK_RUN_SELECTION_REQUIRED"]
|
|
1698
|
-
},
|
|
1699
|
-
AI_TRACE_RELATIONSHIP_SELF_PARENT: {
|
|
1700
|
-
code: "AI_TRACE_RELATIONSHIP_SELF_PARENT",
|
|
1701
|
-
summary: "A parentId equals its own event/step id.",
|
|
1702
|
-
remediation: "Reject at capture (AI_LANGGRAPH_SELF_PARENT_REJECTED) or drop via logical projection; do not invent replacement parents.",
|
|
1703
|
-
relatedCodes: [
|
|
1704
|
-
"AI_LANGGRAPH_SELF_PARENT_REJECTED",
|
|
1705
|
-
"AI_LOGICAL_SELF_PARENT_REMOVED"
|
|
1706
|
-
]
|
|
1707
|
-
},
|
|
1708
|
-
AI_TRACE_RELATIONSHIP_CYCLE: {
|
|
1709
|
-
code: "AI_TRACE_RELATIONSHIP_CYCLE",
|
|
1710
|
-
summary: "Trace contains a parentId cycle.",
|
|
1711
|
-
remediation: "Use visibility-first tree linking for legacy fixtures; prefer acyclic capture for new adapter output.",
|
|
1712
|
-
relatedCodes: ["structure.cycle"]
|
|
1713
|
-
},
|
|
1714
|
-
AI_TRACE_CONTRACT_INPUT_INVALID: {
|
|
1715
|
-
code: "AI_TRACE_CONTRACT_INPUT_INVALID",
|
|
1716
|
-
summary: "evaluateTraceContract expects ({ read }, contract) or use evaluateTraceContractRead(read, contract).",
|
|
1717
|
-
remediation: "Pass TraceCheckInput as the first argument: evaluateTraceContract({ read }, contract).",
|
|
1718
|
-
relatedCodes: ["AI_CHECK_INVALID_ARGUMENTS"]
|
|
1719
|
-
},
|
|
1720
|
-
AI_INSPECT_RUN_UNSUPPORTED_OPTION: {
|
|
1721
|
-
code: "AI_INSPECT_RUN_UNSUPPORTED_OPTION",
|
|
1722
|
-
summary: 'inspectRun does not accept a "writer" option.',
|
|
1723
|
-
remediation: "Use createInspector({ writer }) then inspector.run(...). inspectRun persists via the global JSONL path (schema 0.1)."
|
|
1639
|
+
function earliestStart(runs) {
|
|
1640
|
+
let earliest;
|
|
1641
|
+
for (const run of runs) {
|
|
1642
|
+
if (run.startedAt === void 0) continue;
|
|
1643
|
+
if (earliest === void 0 || run.startedAt < earliest) {
|
|
1644
|
+
earliest = run.startedAt;
|
|
1645
|
+
}
|
|
1724
1646
|
}
|
|
1725
|
-
|
|
1726
|
-
function formatProgrammaticDiagnostic(code, detail) {
|
|
1727
|
-
const spec = PROGRAMMATIC_DIAGNOSTIC_SPECS[code];
|
|
1728
|
-
const summary = detail?.trim() ? detail.trim() : spec.summary;
|
|
1729
|
-
return `${code}: ${summary} Remediation: ${spec.remediation}`;
|
|
1647
|
+
return earliest;
|
|
1730
1648
|
}
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1649
|
+
function latestEndWhenAllEnded(runs) {
|
|
1650
|
+
if (runs.length === 0) return void 0;
|
|
1651
|
+
let latest;
|
|
1652
|
+
for (const run of runs) {
|
|
1653
|
+
if (run.endedAt === void 0) return void 0;
|
|
1654
|
+
if (latest === void 0 || run.endedAt > latest) latest = run.endedAt;
|
|
1655
|
+
}
|
|
1656
|
+
return latest;
|
|
1735
1657
|
}
|
|
1736
|
-
function
|
|
1737
|
-
|
|
1738
|
-
|
|
1658
|
+
function pickExplicitStatus(runs) {
|
|
1659
|
+
let best;
|
|
1660
|
+
let bestPriority = 0;
|
|
1661
|
+
for (const run of runs) {
|
|
1662
|
+
const raw = run.metadata?.sessionStatus;
|
|
1663
|
+
if (!isExplicitSessionStatus(raw)) continue;
|
|
1664
|
+
const priority = EXPLICIT_STATUS_PRIORITY[raw] ?? 0;
|
|
1665
|
+
if (priority > bestPriority) {
|
|
1666
|
+
bestPriority = priority;
|
|
1667
|
+
best = raw;
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
return best;
|
|
1739
1671
|
}
|
|
1740
|
-
function
|
|
1741
|
-
const
|
|
1742
|
-
|
|
1743
|
-
return void 0;
|
|
1672
|
+
function deriveLastError(runs) {
|
|
1673
|
+
const errorRuns = runs.filter((run) => run.status === "error").sort((a, b) => activityMs(b) - activityMs(a));
|
|
1674
|
+
const latest = errorRuns[0];
|
|
1675
|
+
if (!latest) return void 0;
|
|
1676
|
+
const meta = latest.metadata ?? {};
|
|
1677
|
+
const message = typeof meta.errorMessage === "string" && meta.errorMessage.trim() !== "" ? meta.errorMessage.trim() : latest.name ?? latest.runId;
|
|
1678
|
+
const code = typeof meta.errorCode === "string" && meta.errorCode.trim() !== "" ? meta.errorCode.trim() : void 0;
|
|
1679
|
+
return { runId: latest.runId, message, code };
|
|
1744
1680
|
}
|
|
1745
|
-
function
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1681
|
+
function deriveCheckSummary(runs) {
|
|
1682
|
+
let pass = 0;
|
|
1683
|
+
let fail = 0;
|
|
1684
|
+
let warn2 = 0;
|
|
1685
|
+
let found = false;
|
|
1686
|
+
for (const run of runs) {
|
|
1687
|
+
const summary = run.metadata?.checkSummary;
|
|
1688
|
+
if (!summary || typeof summary !== "object") continue;
|
|
1689
|
+
const record = summary;
|
|
1690
|
+
if (typeof record.pass === "number") {
|
|
1691
|
+
pass += record.pass;
|
|
1692
|
+
found = true;
|
|
1693
|
+
}
|
|
1694
|
+
if (typeof record.fail === "number") {
|
|
1695
|
+
fail += record.fail;
|
|
1696
|
+
found = true;
|
|
1697
|
+
}
|
|
1698
|
+
if (typeof record.warn === "number") {
|
|
1699
|
+
warn2 += record.warn;
|
|
1700
|
+
found = true;
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
return found ? { pass, fail, warn: warn2 } : void 0;
|
|
1753
1704
|
}
|
|
1754
|
-
function
|
|
1755
|
-
const
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
merged.legacyCompleteEvent = complete.attributes?.legacyEvent;
|
|
1761
|
-
if (complete.attributes?.errorStack !== void 0) {
|
|
1762
|
-
merged.errorStack = complete.attributes.errorStack;
|
|
1705
|
+
function deriveObservationSummary(runs) {
|
|
1706
|
+
for (const run of [...runs].sort((a, b) => activityMs(b) - activityMs(a))) {
|
|
1707
|
+
const value = run.metadata?.observationSummary;
|
|
1708
|
+
if (typeof value === "string" && value.trim() !== "") {
|
|
1709
|
+
return value.trim();
|
|
1710
|
+
}
|
|
1763
1711
|
}
|
|
1764
|
-
return
|
|
1712
|
+
return void 0;
|
|
1765
1713
|
}
|
|
1766
|
-
function
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1714
|
+
function deriveSessionStatus(runs, options = {}) {
|
|
1715
|
+
if (runs.length === 0) return "unknown";
|
|
1716
|
+
if (runs.some((run) => run.status === "running")) return "running";
|
|
1717
|
+
const explicit = pickExplicitStatus(runs);
|
|
1718
|
+
if (explicit && explicit !== "running") return explicit;
|
|
1719
|
+
if (runs.some((run) => run.status === "error")) return "error";
|
|
1720
|
+
if (runs.every((run) => run.status === "success")) return "completed";
|
|
1721
|
+
const nowMs = options.nowMs ?? Date.now();
|
|
1722
|
+
const staleThresholdMs = options.staleThresholdMs ?? DEFAULT_STALE_THRESHOLD_MS;
|
|
1723
|
+
const lastMs = latestActivityMs(runs);
|
|
1724
|
+
if (lastMs > 0 && nowMs - lastMs > staleThresholdMs) return "stale";
|
|
1725
|
+
return "unknown";
|
|
1726
|
+
}
|
|
1727
|
+
function enrichSessionSummary(summary, runs, options = {}) {
|
|
1728
|
+
const sessionRuns = runs.filter((run) => summary.runIds.includes(run.runId)).sort((a, b) => a.runId.localeCompare(b.runId));
|
|
1729
|
+
const startedAt = earliestStart(sessionRuns);
|
|
1730
|
+
const endedAt = latestEndWhenAllEnded(sessionRuns);
|
|
1731
|
+
const durationMs = startedAt !== void 0 && endedAt !== void 0 ? endedAt - startedAt : void 0;
|
|
1732
|
+
let correlationId;
|
|
1733
|
+
let jobId;
|
|
1734
|
+
let workflowId;
|
|
1735
|
+
for (const run of sessionRuns) {
|
|
1736
|
+
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1737
|
+
if (!correlationId && meta?.correlationId) correlationId = meta.correlationId;
|
|
1738
|
+
if (!jobId && meta?.jobId) jobId = meta.jobId;
|
|
1739
|
+
if (!workflowId && meta?.workflowName) workflowId = meta.workflowName;
|
|
1740
|
+
else if (!workflowId && meta?.workflowStep) workflowId = meta.workflowStep;
|
|
1741
|
+
}
|
|
1742
|
+
const lastMs = latestActivityMs(sessionRuns);
|
|
1743
|
+
const lastActivity = lastMs > 0 ? new Date(lastMs).toISOString() : (/* @__PURE__ */ new Date(0)).toISOString();
|
|
1744
|
+
const retryCount2 = summary.retries.filter(
|
|
1745
|
+
(retry) => retry.retryOf !== void 0 || (retry.attempt ?? 0) > 1
|
|
1746
|
+
).length;
|
|
1778
1747
|
return {
|
|
1779
|
-
...
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1748
|
+
...summary,
|
|
1749
|
+
status: deriveSessionStatus(sessionRuns, options),
|
|
1750
|
+
startedAt,
|
|
1751
|
+
endedAt,
|
|
1752
|
+
durationMs,
|
|
1753
|
+
correlationId,
|
|
1754
|
+
jobId,
|
|
1755
|
+
workflowId,
|
|
1756
|
+
lastError: deriveLastError(sessionRuns),
|
|
1757
|
+
lastActivity,
|
|
1758
|
+
retryCount: retryCount2,
|
|
1759
|
+
observationSummary: deriveObservationSummary(sessionRuns),
|
|
1760
|
+
checkSummary: deriveCheckSummary(sessionRuns)
|
|
1787
1761
|
};
|
|
1788
1762
|
}
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1763
|
+
|
|
1764
|
+
// packages/core/src/sessions/load.ts
|
|
1765
|
+
async function enrichSessionRunRecord(meta) {
|
|
1766
|
+
let metadata;
|
|
1767
|
+
try {
|
|
1768
|
+
const events = await readTraceEventsFromFile(meta.filePath);
|
|
1769
|
+
for (const event of events) {
|
|
1770
|
+
if (event.event !== "run_started") continue;
|
|
1771
|
+
if (event.metadata && typeof event.metadata === "object") {
|
|
1772
|
+
metadata = event.metadata;
|
|
1773
|
+
}
|
|
1774
|
+
break;
|
|
1798
1775
|
}
|
|
1776
|
+
} catch {
|
|
1777
|
+
}
|
|
1778
|
+
return {
|
|
1779
|
+
runId: meta.runId,
|
|
1780
|
+
name: meta.name,
|
|
1781
|
+
status: meta.status,
|
|
1782
|
+
startedAt: meta.startedAt,
|
|
1783
|
+
endedAt: meta.endedAt,
|
|
1784
|
+
durationMs: meta.durationMs,
|
|
1785
|
+
filePath: meta.filePath,
|
|
1786
|
+
metadata
|
|
1799
1787
|
};
|
|
1800
1788
|
}
|
|
1801
|
-
function
|
|
1802
|
-
const
|
|
1803
|
-
const
|
|
1804
|
-
|
|
1805
|
-
const list = byRun.get(event.runId) ?? [];
|
|
1806
|
-
list.push(event);
|
|
1807
|
-
byRun.set(event.runId, list);
|
|
1789
|
+
async function loadSessionRunRecords(metas) {
|
|
1790
|
+
const out = [];
|
|
1791
|
+
for (const meta of metas) {
|
|
1792
|
+
out.push(await enrichSessionRunRecord(meta));
|
|
1808
1793
|
}
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
let match;
|
|
1833
|
-
if (legacyEvent(start) === "run_started") {
|
|
1834
|
-
const candidates = completes.filter(
|
|
1835
|
-
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "run_completed"
|
|
1836
|
-
);
|
|
1837
|
-
if (candidates.length > 1) {
|
|
1838
|
-
diagnostics.push({
|
|
1839
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1840
|
-
message: `Multiple run_completed rows for run ${runId}; using first by eventId.`,
|
|
1841
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
1842
|
-
});
|
|
1843
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1844
|
-
}
|
|
1845
|
-
match = candidates[0];
|
|
1846
|
-
} else if (stepId) {
|
|
1847
|
-
const candidates = completes.filter(
|
|
1848
|
-
(c) => !usedCompletes.has(c.eventId) && legacyEvent(c) === "step_completed" && stepIdOf(c) === stepId
|
|
1849
|
-
);
|
|
1850
|
-
if (candidates.length > 1) {
|
|
1851
|
-
diagnostics.push({
|
|
1852
|
-
code: "AI_LOGICAL_PAIR_AMBIGUOUS",
|
|
1853
|
-
message: `Multiple step_completed rows for stepId ${stepId}; using first by eventId.`,
|
|
1854
|
-
eventIds: candidates.map((c) => c.eventId)
|
|
1855
|
-
});
|
|
1856
|
-
candidates.sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
1857
|
-
}
|
|
1858
|
-
match = candidates[0];
|
|
1859
|
-
}
|
|
1860
|
-
if (match) {
|
|
1861
|
-
usedCompletes.add(match.eventId);
|
|
1862
|
-
absorbedIds.add(match.eventId);
|
|
1863
|
-
const paired = pairStartComplete(start, match);
|
|
1864
|
-
logical.push(paired);
|
|
1865
|
-
logicalByRawId.set(start.eventId, paired);
|
|
1866
|
-
logicalByRawId.set(match.eventId, paired);
|
|
1867
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, paired.eventId);
|
|
1868
|
-
} else {
|
|
1869
|
-
diagnostics.push({
|
|
1870
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_START",
|
|
1871
|
-
message: `No matching complete for start ${start.eventId}.`,
|
|
1872
|
-
eventIds: [start.eventId]
|
|
1873
|
-
});
|
|
1874
|
-
const alone = asLogical(start);
|
|
1875
|
-
logical.push(alone);
|
|
1876
|
-
logicalByRawId.set(start.eventId, alone);
|
|
1877
|
-
if (stepId) stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1878
|
-
}
|
|
1879
|
-
}
|
|
1880
|
-
for (const complete of completes) {
|
|
1881
|
-
if (usedCompletes.has(complete.eventId)) continue;
|
|
1882
|
-
diagnostics.push({
|
|
1883
|
-
code: "AI_LOGICAL_PAIR_UNMATCHED_COMPLETE",
|
|
1884
|
-
message: `No matching start for complete ${complete.eventId}.`,
|
|
1885
|
-
eventIds: [complete.eventId]
|
|
1794
|
+
return out;
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
// packages/core/src/sessions/index.ts
|
|
1798
|
+
function compareRuns(a, b) {
|
|
1799
|
+
const aStart = a.startedAt ?? 0;
|
|
1800
|
+
const bStart = b.startedAt ?? 0;
|
|
1801
|
+
if (aStart !== bStart) return aStart - bStart;
|
|
1802
|
+
return a.runId.localeCompare(b.runId);
|
|
1803
|
+
}
|
|
1804
|
+
function buildGroups(runIds, metaByRunId) {
|
|
1805
|
+
const byGroup = /* @__PURE__ */ new Map();
|
|
1806
|
+
for (const runId of runIds) {
|
|
1807
|
+
const groupId = metaByRunId.get(runId)?.groupId;
|
|
1808
|
+
if (!groupId) continue;
|
|
1809
|
+
const existing = byGroup.get(groupId);
|
|
1810
|
+
if (existing) {
|
|
1811
|
+
existing.runIds.push(runId);
|
|
1812
|
+
} else {
|
|
1813
|
+
byGroup.set(groupId, {
|
|
1814
|
+
groupId,
|
|
1815
|
+
parentGroupId: metaByRunId.get(runId)?.parentGroupId,
|
|
1816
|
+
runIds: [runId]
|
|
1886
1817
|
});
|
|
1887
|
-
const alone = asLogical(complete);
|
|
1888
|
-
logical.push(alone);
|
|
1889
|
-
logicalByRawId.set(complete.eventId, alone);
|
|
1890
|
-
const stepId = stepIdOf(complete);
|
|
1891
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1892
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1893
|
-
}
|
|
1894
|
-
}
|
|
1895
|
-
for (const event of others) {
|
|
1896
|
-
const alone = asLogical(event);
|
|
1897
|
-
logical.push(alone);
|
|
1898
|
-
logicalByRawId.set(event.eventId, alone);
|
|
1899
|
-
const stepId = stepIdOf(event);
|
|
1900
|
-
if (stepId && !stepIdToLogicalId.has(`${runId}:${stepId}`)) {
|
|
1901
|
-
stepIdToLogicalId.set(`${runId}:${stepId}`, alone.eventId);
|
|
1902
|
-
}
|
|
1903
1818
|
}
|
|
1904
1819
|
}
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1820
|
+
return [...byGroup.values()].map((group) => ({
|
|
1821
|
+
...group,
|
|
1822
|
+
runIds: [...group.runIds].sort((a, b) => a.localeCompare(b))
|
|
1823
|
+
}));
|
|
1824
|
+
}
|
|
1825
|
+
function buildHandoffs(runIds, metaByRunId, warnings, sessionId) {
|
|
1826
|
+
const edges = [];
|
|
1827
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1828
|
+
const pushEdge = (edge) => {
|
|
1829
|
+
const key = `${edge.from}->${edge.to}:${edge.confidence}`;
|
|
1830
|
+
if (seen.has(key)) return;
|
|
1831
|
+
seen.add(key);
|
|
1832
|
+
edges.push(edge);
|
|
1833
|
+
};
|
|
1834
|
+
for (const runId of runIds) {
|
|
1835
|
+
const meta = metaByRunId.get(runId);
|
|
1836
|
+
if (!meta) continue;
|
|
1837
|
+
if (meta.handoffFrom && meta.handoffTo) {
|
|
1838
|
+
pushEdge({
|
|
1839
|
+
from: meta.handoffFrom,
|
|
1840
|
+
to: meta.handoffTo,
|
|
1841
|
+
source: "manual",
|
|
1842
|
+
confidence: "explicit"
|
|
1843
|
+
});
|
|
1911
1844
|
continue;
|
|
1912
1845
|
}
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
const viaStep = stepIdToLogicalId.get(`${event.runId}:${originalParentId}`);
|
|
1921
|
-
if (viaStep) {
|
|
1922
|
-
nextParent = viaStep;
|
|
1923
|
-
remapped = true;
|
|
1924
|
-
}
|
|
1846
|
+
if (meta.handoffFrom) {
|
|
1847
|
+
pushEdge({
|
|
1848
|
+
from: meta.handoffFrom,
|
|
1849
|
+
to: runId,
|
|
1850
|
+
source: "manual",
|
|
1851
|
+
confidence: "explicit"
|
|
1852
|
+
});
|
|
1925
1853
|
}
|
|
1926
|
-
if (
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
}
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1854
|
+
if (meta.handoffTo) {
|
|
1855
|
+
pushEdge({
|
|
1856
|
+
from: runId,
|
|
1857
|
+
to: meta.handoffTo,
|
|
1858
|
+
source: "manual",
|
|
1859
|
+
confidence: "explicit"
|
|
1860
|
+
});
|
|
1861
|
+
}
|
|
1862
|
+
if (meta.subAgentId && meta.parentGroupId && !meta.handoffFrom && !meta.handoffTo) {
|
|
1863
|
+
pushEdge({
|
|
1864
|
+
from: meta.parentGroupId,
|
|
1865
|
+
to: meta.subAgentId,
|
|
1866
|
+
source: "inferred",
|
|
1867
|
+
confidence: "correlated"
|
|
1868
|
+
});
|
|
1869
|
+
warnings.push({
|
|
1870
|
+
code: "ambiguous-handoff-endpoints",
|
|
1871
|
+
message: "Handoff inferred from parentGroupId and subAgentId without explicit handoffFrom/handoffTo.",
|
|
1872
|
+
runId,
|
|
1873
|
+
sessionId
|
|
1874
|
+
});
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
return edges.sort((a, b) => {
|
|
1878
|
+
const from = a.from.localeCompare(b.from);
|
|
1879
|
+
if (from !== 0) return from;
|
|
1880
|
+
return a.to.localeCompare(b.to);
|
|
1881
|
+
});
|
|
1882
|
+
}
|
|
1883
|
+
function buildRetries(runIds, metaByRunId, warnings, sessionId) {
|
|
1884
|
+
const retries = [];
|
|
1885
|
+
for (const runId of runIds) {
|
|
1886
|
+
const meta = metaByRunId.get(runId);
|
|
1887
|
+
if (!meta) continue;
|
|
1888
|
+
if (meta.retryOf) {
|
|
1889
|
+
retries.push({
|
|
1890
|
+
runId,
|
|
1891
|
+
retryOf: meta.retryOf,
|
|
1892
|
+
attempt: meta.attempt,
|
|
1893
|
+
source: "manual",
|
|
1894
|
+
confidence: "explicit"
|
|
1895
|
+
});
|
|
1960
1896
|
continue;
|
|
1961
1897
|
}
|
|
1962
|
-
if (
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
),
|
|
1969
|
-
eventIds: [event.eventId]
|
|
1898
|
+
if (meta.attempt !== void 0 && meta.attempt > 1) {
|
|
1899
|
+
retries.push({
|
|
1900
|
+
runId,
|
|
1901
|
+
attempt: meta.attempt,
|
|
1902
|
+
source: "inferred",
|
|
1903
|
+
confidence: "correlated"
|
|
1970
1904
|
});
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
parentNormalized: true,
|
|
1977
|
-
originalParentId
|
|
1978
|
-
}
|
|
1905
|
+
warnings.push({
|
|
1906
|
+
code: "ambiguous-retry-link",
|
|
1907
|
+
message: "attempt > 1 without retryOf; retry link is correlated only.",
|
|
1908
|
+
runId,
|
|
1909
|
+
sessionId
|
|
1979
1910
|
});
|
|
1980
|
-
continue;
|
|
1981
1911
|
}
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1912
|
+
}
|
|
1913
|
+
return retries.sort((a, b) => a.runId.localeCompare(b.runId));
|
|
1914
|
+
}
|
|
1915
|
+
function buildCriticalPath(runs, handoffs) {
|
|
1916
|
+
const runById = new Map(runs.map((run) => [run.runId, run]));
|
|
1917
|
+
const explicitTargets = new Set(
|
|
1918
|
+
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.to)
|
|
1919
|
+
);
|
|
1920
|
+
const explicitSources = new Set(
|
|
1921
|
+
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
1922
|
+
);
|
|
1923
|
+
const ordered = [...runs].sort(compareRuns);
|
|
1924
|
+
const path18 = [];
|
|
1925
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1926
|
+
const pushRun = (run, confidence, source) => {
|
|
1927
|
+
if (visited.has(run.runId)) return;
|
|
1928
|
+
visited.add(run.runId);
|
|
1929
|
+
path18.push({
|
|
1930
|
+
runId: run.runId,
|
|
1931
|
+
name: run.name,
|
|
1932
|
+
startedAt: run.startedAt,
|
|
1933
|
+
durationMs: run.durationMs,
|
|
1934
|
+
confidence,
|
|
1935
|
+
source
|
|
1986
1936
|
});
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1937
|
+
};
|
|
1938
|
+
for (const edge of handoffs) {
|
|
1939
|
+
if (edge.confidence !== "explicit") continue;
|
|
1940
|
+
const fromRun = [...runById.values()].find(
|
|
1941
|
+
(run) => run.runId === edge.from || metaRunIdMatches(run, edge.from, runById)
|
|
1942
|
+
);
|
|
1943
|
+
const toRun = [...runById.values()].find(
|
|
1944
|
+
(run) => run.runId === edge.to || metaRunIdMatches(run, edge.to, runById)
|
|
1945
|
+
);
|
|
1946
|
+
if (fromRun) pushRun(fromRun, "explicit", "manual");
|
|
1947
|
+
if (toRun) pushRun(toRun, "explicit", "manual");
|
|
1948
|
+
}
|
|
1949
|
+
for (const run of ordered) {
|
|
1950
|
+
if (visited.has(run.runId)) continue;
|
|
1951
|
+
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
1952
|
+
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
1953
|
+
}
|
|
1954
|
+
return path18;
|
|
1955
|
+
}
|
|
1956
|
+
function metaRunIdMatches(run, token, runById) {
|
|
1957
|
+
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
1958
|
+
return meta?.subAgentId === token || meta?.groupId === token || runById.has(token);
|
|
1959
|
+
}
|
|
1960
|
+
function buildSessionIndex(inputRuns, options = {}) {
|
|
1961
|
+
const warnings = [];
|
|
1962
|
+
const runs = [...inputRuns].sort(compareRuns);
|
|
1963
|
+
const metaByRunId = /* @__PURE__ */ new Map();
|
|
1964
|
+
for (const run of runs) {
|
|
1965
|
+
metaByRunId.set(run.runId, extractSessionWorkflowMetadata(run.metadata));
|
|
1966
|
+
}
|
|
1967
|
+
const sessionsByKey = /* @__PURE__ */ new Map();
|
|
1968
|
+
const unscopedRunIds = [];
|
|
1969
|
+
for (const run of runs) {
|
|
1970
|
+
const meta = metaByRunId.get(run.runId);
|
|
1971
|
+
const key = sessionKeyForRun(meta, {
|
|
1972
|
+
correlateByGroupId: options.correlateByGroupId === true
|
|
1973
|
+
});
|
|
1974
|
+
if (!key) {
|
|
1975
|
+
unscopedRunIds.push(run.runId);
|
|
1976
|
+
continue;
|
|
1977
|
+
}
|
|
1978
|
+
const bucket = sessionsByKey.get(key) ?? [];
|
|
1979
|
+
bucket.push(run);
|
|
1980
|
+
sessionsByKey.set(key, bucket);
|
|
1981
|
+
}
|
|
1982
|
+
const sessions = [...sessionsByKey.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([sessionId, sessionRuns]) => {
|
|
1983
|
+
const runIds = sessionRuns.map((run) => run.runId).sort();
|
|
1984
|
+
const handoffs = buildHandoffs(runIds, metaByRunId, warnings, sessionId);
|
|
1985
|
+
const retries = buildRetries(runIds, metaByRunId, warnings, sessionId);
|
|
1986
|
+
const groups = buildGroups(runIds, metaByRunId);
|
|
1987
|
+
const criticalPath = buildCriticalPath(sessionRuns, handoffs);
|
|
1988
|
+
const confidences = new Set(handoffs.map((edge) => edge.confidence));
|
|
1989
|
+
if (confidences.has("explicit") && confidences.has("correlated")) {
|
|
1990
|
+
warnings.push({
|
|
1991
|
+
code: "mixed-confidence-group",
|
|
1992
|
+
message: "Session aggregates explicit and correlated handoff edges.",
|
|
1993
|
+
sessionId
|
|
1994
|
+
});
|
|
1995
|
+
}
|
|
1996
|
+
return enrichSessionSummary(
|
|
1997
|
+
{
|
|
1998
|
+
sessionId,
|
|
1999
|
+
runIds,
|
|
2000
|
+
groups,
|
|
2001
|
+
handoffs,
|
|
2002
|
+
retries,
|
|
2003
|
+
criticalPath
|
|
2004
|
+
},
|
|
2005
|
+
runs,
|
|
2006
|
+
{
|
|
2007
|
+
nowMs: options.nowMs,
|
|
2008
|
+
staleThresholdMs: options.staleThresholdMs
|
|
1994
2009
|
}
|
|
2010
|
+
);
|
|
2011
|
+
});
|
|
2012
|
+
if (sessions.length === 0 && runs.length > 0) {
|
|
2013
|
+
warnings.push({
|
|
2014
|
+
code: "missing-session-id",
|
|
2015
|
+
message: "No sessionId (or correlated groupId) found on input runs."
|
|
1995
2016
|
});
|
|
1996
2017
|
}
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
return ai - bi || a.eventId.localeCompare(b.eventId);
|
|
2018
|
+
warnings.sort((a, b) => {
|
|
2019
|
+
const code = a.code.localeCompare(b.code);
|
|
2020
|
+
if (code !== 0) return code;
|
|
2021
|
+
return (a.runId ?? "").localeCompare(b.runId ?? "");
|
|
2002
2022
|
});
|
|
2003
2023
|
return {
|
|
2004
|
-
|
|
2005
|
-
|
|
2024
|
+
runs,
|
|
2025
|
+
sessions,
|
|
2026
|
+
unscopedRunIds: unscopedRunIds.sort(),
|
|
2027
|
+
warnings
|
|
2006
2028
|
};
|
|
2007
2029
|
}
|
|
2008
|
-
function resolveCanonicalToolName(event) {
|
|
2009
|
-
const attrs = event.attributes;
|
|
2010
|
-
const direct = pickString(attrs, ["toolName", "tool"]);
|
|
2011
|
-
if (direct) return direct;
|
|
2012
|
-
const metadata = attrs?.metadata;
|
|
2013
|
-
if (isRecord5(metadata)) {
|
|
2014
|
-
const nested = pickString(metadata, ["toolName", "tool"]);
|
|
2015
|
-
if (nested) return nested;
|
|
2016
|
-
}
|
|
2017
|
-
for (const prefix of ["tool:", "function:", "mcp-tools:"]) {
|
|
2018
|
-
if (event.name.startsWith(prefix)) return event.name.slice(prefix.length);
|
|
2019
|
-
}
|
|
2020
|
-
return event.name;
|
|
2021
|
-
}
|
|
2022
|
-
function pickString(record, keys) {
|
|
2023
|
-
if (!record) return void 0;
|
|
2024
|
-
for (const key of keys) {
|
|
2025
|
-
const value = record[key];
|
|
2026
|
-
if (typeof value === "string" && value.trim() !== "") return value.trim();
|
|
2027
|
-
}
|
|
2028
|
-
return void 0;
|
|
2029
|
-
}
|
|
2030
2030
|
|
|
2031
2031
|
// packages/core/src/checks/trace-facts.ts
|
|
2032
2032
|
formatProgrammaticDiagnostic(
|
|
@@ -2061,7 +2061,11 @@ function emptySummary() {
|
|
|
2061
2061
|
failed: 0,
|
|
2062
2062
|
warnings: 0,
|
|
2063
2063
|
errors: 0,
|
|
2064
|
-
rulesEvaluated: 0
|
|
2064
|
+
rulesEvaluated: 0,
|
|
2065
|
+
rulesPassed: 0,
|
|
2066
|
+
rulesWarning: 0,
|
|
2067
|
+
rulesFailed: 0,
|
|
2068
|
+
rulesError: 0
|
|
2065
2069
|
};
|
|
2066
2070
|
}
|
|
2067
2071
|
function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
@@ -2073,7 +2077,11 @@ function errorResult(input, diagnostics, selectedRun, ruleExecutions = []) {
|
|
|
2073
2077
|
summary: {
|
|
2074
2078
|
...emptySummary(),
|
|
2075
2079
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2076
|
-
rulesEvaluated: ruleExecutions.length
|
|
2080
|
+
rulesEvaluated: ruleExecutions.length,
|
|
2081
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
2082
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
2083
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
2084
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
2077
2085
|
},
|
|
2078
2086
|
findings: [],
|
|
2079
2087
|
diagnostics: [...diagnostics],
|
|
@@ -2233,7 +2241,7 @@ function normalizeFinding(rule, finding) {
|
|
|
2233
2241
|
...finding.action !== void 0 ? { action: finding.action } : {}
|
|
2234
2242
|
};
|
|
2235
2243
|
}
|
|
2236
|
-
function summarize(findings, diagnostics,
|
|
2244
|
+
function summarize(findings, diagnostics, ruleExecutions) {
|
|
2237
2245
|
return {
|
|
2238
2246
|
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
2239
2247
|
failed: findings.filter(
|
|
@@ -2243,7 +2251,11 @@ function summarize(findings, diagnostics, rulesEvaluated) {
|
|
|
2243
2251
|
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
2244
2252
|
).length,
|
|
2245
2253
|
errors: diagnostics.filter((item) => item.severity === "error").length,
|
|
2246
|
-
rulesEvaluated
|
|
2254
|
+
rulesEvaluated: ruleExecutions.length,
|
|
2255
|
+
rulesPassed: ruleExecutions.filter((item) => item.status === "pass").length,
|
|
2256
|
+
rulesWarning: ruleExecutions.filter((item) => item.status === "warning").length,
|
|
2257
|
+
rulesFailed: ruleExecutions.filter((item) => item.status === "fail").length,
|
|
2258
|
+
rulesError: ruleExecutions.filter((item) => item.status === "error").length
|
|
2247
2259
|
};
|
|
2248
2260
|
}
|
|
2249
2261
|
function classifyRuleExecution(findings, threw) {
|
|
@@ -2720,7 +2732,7 @@ function runTraceChecks(input, options = {}) {
|
|
|
2720
2732
|
}
|
|
2721
2733
|
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
2722
2734
|
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
2723
|
-
const summary = summarize(sortedFindings, diagnostics, ruleExecutions
|
|
2735
|
+
const summary = summarize(sortedFindings, diagnostics, ruleExecutions);
|
|
2724
2736
|
const status = summary.failed > 0 ? "fail" : "pass";
|
|
2725
2737
|
return {
|
|
2726
2738
|
ok: status === "pass",
|
|
@@ -3159,6 +3171,28 @@ function asPositiveNumber(value, label) {
|
|
|
3159
3171
|
}
|
|
3160
3172
|
return value;
|
|
3161
3173
|
}
|
|
3174
|
+
function asExpectConfig(value, label) {
|
|
3175
|
+
if (value === void 0) return void 0;
|
|
3176
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
3177
|
+
throw new Error(`${label} must be an object.`);
|
|
3178
|
+
}
|
|
3179
|
+
const raw = value;
|
|
3180
|
+
const checkStatus = raw.checkStatus;
|
|
3181
|
+
if (checkStatus !== "pass" && checkStatus !== "fail" && checkStatus !== "error") {
|
|
3182
|
+
throw new Error(`${label}.checkStatus must be "pass", "fail", or "error".`);
|
|
3183
|
+
}
|
|
3184
|
+
const findingRuleIds = asStringArray(raw.findingRuleIds, `${label}.findingRuleIds`);
|
|
3185
|
+
const minAssertions = asPositiveNumber(raw.minAssertions, `${label}.minAssertions`);
|
|
3186
|
+
const rejectUnexpectedFindings = raw.rejectUnexpectedFindings === void 0 ? void 0 : typeof raw.rejectUnexpectedFindings === "boolean" ? raw.rejectUnexpectedFindings : (() => {
|
|
3187
|
+
throw new Error(`${label}.rejectUnexpectedFindings must be a boolean.`);
|
|
3188
|
+
})();
|
|
3189
|
+
return {
|
|
3190
|
+
checkStatus,
|
|
3191
|
+
...findingRuleIds !== void 0 ? { findingRuleIds } : {},
|
|
3192
|
+
...rejectUnexpectedFindings !== void 0 ? { rejectUnexpectedFindings } : {},
|
|
3193
|
+
...minAssertions !== void 0 ? { minAssertions } : {}
|
|
3194
|
+
};
|
|
3195
|
+
}
|
|
3162
3196
|
function validateCaseConfig(value, index) {
|
|
3163
3197
|
const diagnostics = [];
|
|
3164
3198
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
@@ -3201,7 +3235,8 @@ function validateCaseConfig(value, index) {
|
|
|
3201
3235
|
raw.expectedObservations,
|
|
3202
3236
|
`cases[${index}].expectedObservations`
|
|
3203
3237
|
)
|
|
3204
|
-
} : {}
|
|
3238
|
+
} : {},
|
|
3239
|
+
...asExpectConfig(raw.expect, `cases[${index}].expect`) !== void 0 ? { expect: asExpectConfig(raw.expect, `cases[${index}].expect`) } : {}
|
|
3205
3240
|
},
|
|
3206
3241
|
diagnostics
|
|
3207
3242
|
};
|
|
@@ -5787,6 +5822,78 @@ function validateExpectedObservations(suiteCase, read) {
|
|
|
5787
5822
|
}
|
|
5788
5823
|
return { ok: diagnostics.length === 0, diagnostics };
|
|
5789
5824
|
}
|
|
5825
|
+
function evaluateCaseExpect(suiteCase, checkResult, observationCount) {
|
|
5826
|
+
const expectation = suiteCase.expect;
|
|
5827
|
+
if (expectation === void 0) {
|
|
5828
|
+
return { ok: true, diagnostics: [] };
|
|
5829
|
+
}
|
|
5830
|
+
const diagnostics = [];
|
|
5831
|
+
if (checkResult.status !== expectation.checkStatus) {
|
|
5832
|
+
diagnostics.push(
|
|
5833
|
+
diagnostic4(
|
|
5834
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5835
|
+
`Expected check status "${expectation.checkStatus}", got "${checkResult.status}".`,
|
|
5836
|
+
"error",
|
|
5837
|
+
suiteCase.id
|
|
5838
|
+
)
|
|
5839
|
+
);
|
|
5840
|
+
}
|
|
5841
|
+
const failedFindings = checkResult.findings.filter(
|
|
5842
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
5843
|
+
);
|
|
5844
|
+
const requiredIds = expectation.findingRuleIds ?? [];
|
|
5845
|
+
for (const ruleId of requiredIds) {
|
|
5846
|
+
if (!failedFindings.some((finding) => finding.ruleId === ruleId)) {
|
|
5847
|
+
diagnostics.push(
|
|
5848
|
+
diagnostic4(
|
|
5849
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5850
|
+
`Expected failing finding for rule "${ruleId}" was not present.`,
|
|
5851
|
+
"error",
|
|
5852
|
+
suiteCase.id
|
|
5853
|
+
)
|
|
5854
|
+
);
|
|
5855
|
+
}
|
|
5856
|
+
}
|
|
5857
|
+
const rejectUnexpected = expectation.rejectUnexpectedFindings !== false;
|
|
5858
|
+
if (rejectUnexpected && requiredIds.length > 0) {
|
|
5859
|
+
for (const finding of failedFindings) {
|
|
5860
|
+
if (!requiredIds.includes(finding.ruleId)) {
|
|
5861
|
+
diagnostics.push(
|
|
5862
|
+
diagnostic4(
|
|
5863
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5864
|
+
`Unexpected failing finding for rule "${finding.ruleId}".`,
|
|
5865
|
+
"error",
|
|
5866
|
+
suiteCase.id
|
|
5867
|
+
)
|
|
5868
|
+
);
|
|
5869
|
+
}
|
|
5870
|
+
}
|
|
5871
|
+
}
|
|
5872
|
+
if (expectation.minAssertions !== void 0) {
|
|
5873
|
+
const assertionCount = checkResult.ruleExecutions.length + observationCount;
|
|
5874
|
+
if (assertionCount < expectation.minAssertions) {
|
|
5875
|
+
diagnostics.push(
|
|
5876
|
+
diagnostic4(
|
|
5877
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5878
|
+
`Expected at least ${expectation.minAssertions} assertion(s), got ${assertionCount}.`,
|
|
5879
|
+
"error",
|
|
5880
|
+
suiteCase.id
|
|
5881
|
+
)
|
|
5882
|
+
);
|
|
5883
|
+
}
|
|
5884
|
+
}
|
|
5885
|
+
if (expectation.checkStatus === "fail" && checkResult.diagnostics.some((item) => item.severity === "error") && checkResult.status === "error") {
|
|
5886
|
+
diagnostics.push(
|
|
5887
|
+
diagnostic4(
|
|
5888
|
+
"AI_SUITE_EXPECT_MISMATCH",
|
|
5889
|
+
'Check ended in evaluator/config error; semantic failure expectation requires status "fail".',
|
|
5890
|
+
"error",
|
|
5891
|
+
suiteCase.id
|
|
5892
|
+
)
|
|
5893
|
+
);
|
|
5894
|
+
}
|
|
5895
|
+
return { ok: diagnostics.length === 0, diagnostics };
|
|
5896
|
+
}
|
|
5790
5897
|
async function runSuiteCase(suiteCase, config, options) {
|
|
5791
5898
|
const resolved = await resolveSuiteCaseTrace(suiteCase, options);
|
|
5792
5899
|
if (resolved.missing || resolved.tracePath === void 0) {
|
|
@@ -5839,9 +5946,21 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
5839
5946
|
status: "pass",
|
|
5840
5947
|
format: read.format,
|
|
5841
5948
|
findings: [],
|
|
5842
|
-
diagnostics: []
|
|
5949
|
+
diagnostics: [],
|
|
5950
|
+
ruleExecutions: []
|
|
5951
|
+
};
|
|
5843
5952
|
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
5844
|
-
const
|
|
5953
|
+
const checkOk = checkResult.ok;
|
|
5954
|
+
const observationsOk = observationResult.ok;
|
|
5955
|
+
const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
|
|
5956
|
+
(finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
|
|
5957
|
+
) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
|
|
5958
|
+
const expectResult = evaluateCaseExpect(
|
|
5959
|
+
suiteCase,
|
|
5960
|
+
checkResult,
|
|
5961
|
+
compiled.observationCount
|
|
5962
|
+
);
|
|
5963
|
+
const checkFailureDiagnostics = [
|
|
5845
5964
|
...checkResult.diagnostics.map(
|
|
5846
5965
|
(item) => diagnostic4(
|
|
5847
5966
|
"AI_SUITE_CASE_CHECK_FAILED",
|
|
@@ -5857,15 +5976,24 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
5857
5976
|
finding.severity,
|
|
5858
5977
|
suiteCase.id
|
|
5859
5978
|
)
|
|
5860
|
-
)
|
|
5861
|
-
...observationResult.diagnostics
|
|
5979
|
+
)
|
|
5862
5980
|
];
|
|
5863
|
-
const
|
|
5864
|
-
const
|
|
5865
|
-
|
|
5866
|
-
|
|
5867
|
-
|
|
5868
|
-
|
|
5981
|
+
const expectMatched = suiteCase.expect !== void 0 && expectResult.ok && observationsOk;
|
|
5982
|
+
const diagnostics = expectMatched ? [
|
|
5983
|
+
...observationResult.diagnostics,
|
|
5984
|
+
...expectResult.diagnostics,
|
|
5985
|
+
diagnostic4(
|
|
5986
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
5987
|
+
`Expected semantic failure matched (check.status=${checkResult.status}).`,
|
|
5988
|
+
"info",
|
|
5989
|
+
suiteCase.id
|
|
5990
|
+
)
|
|
5991
|
+
] : [
|
|
5992
|
+
...checkFailureDiagnostics,
|
|
5993
|
+
...observationResult.diagnostics,
|
|
5994
|
+
...expectResult.diagnostics
|
|
5995
|
+
];
|
|
5996
|
+
const ok = suiteCase.expect !== void 0 ? expectResult.ok && observationsOk : checkOk && observationsOk;
|
|
5869
5997
|
const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
|
|
5870
5998
|
return {
|
|
5871
5999
|
id: suiteCase.id,
|