agent-inspect 6.31.7 → 6.31.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +1 -1
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-QV73NM25.mjs → chunk-SWUZRRDT.mjs} +158 -21
- package/packages/cli/dist/chunk-SWUZRRDT.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +312 -52
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +156 -36
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-KFOBXFPN.mjs → src-XFIPYODW.mjs} +3 -3
- package/packages/cli/dist/{src-KFOBXFPN.mjs.map → src-XFIPYODW.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +200 -24
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +1 -1
- package/packages/core/dist/advanced.d.ts +1 -1
- package/packages/core/dist/advanced.mjs +89 -15
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.mjs +2 -2
- package/packages/core/dist/{chunk-LXYPM2ZO.mjs → chunk-2CCRUI42.mjs} +19 -318
- package/packages/core/dist/chunk-2CCRUI42.mjs.map +1 -0
- package/packages/core/dist/{chunk-QLYUWKK7.mjs → chunk-2IRRUNIN.mjs} +11 -5
- package/packages/core/dist/chunk-2IRRUNIN.mjs.map +1 -0
- package/packages/core/dist/{chunk-VQUB3WRI.mjs → chunk-3SPBELJB.mjs} +14 -3
- package/packages/core/dist/chunk-3SPBELJB.mjs.map +1 -0
- package/packages/core/dist/{chunk-3RV3JILQ.mjs → chunk-IJJSPJXV.mjs} +20 -12
- package/packages/core/dist/chunk-IJJSPJXV.mjs.map +1 -0
- package/packages/core/dist/chunk-L5WLVBW4.mjs +355 -0
- package/packages/core/dist/chunk-L5WLVBW4.mjs.map +1 -0
- package/packages/core/dist/exporters.cjs +110 -18
- package/packages/core/dist/exporters.cjs.map +1 -1
- package/packages/core/dist/exporters.mjs +110 -18
- package/packages/core/dist/exporters.mjs.map +1 -1
- package/packages/core/dist/index.cjs +12 -1
- package/packages/core/dist/index.cjs.map +1 -1
- package/packages/core/dist/index.mjs +2 -2
- package/packages/core/dist/persisted.cjs +344 -9
- package/packages/core/dist/persisted.cjs.map +1 -1
- package/packages/core/dist/persisted.d.cts +8 -1
- package/packages/core/dist/persisted.d.ts +8 -1
- package/packages/core/dist/persisted.mjs +3 -2
- package/packages/core/dist/persisted.mjs.map +1 -1
- package/packages/core/dist/readers.cjs +377 -87
- package/packages/core/dist/readers.cjs.map +1 -1
- package/packages/core/dist/readers.mjs +3 -3
- package/packages/cli/dist/chunk-QV73NM25.mjs.map +0 -1
- package/packages/core/dist/chunk-3RV3JILQ.mjs.map +0 -1
- package/packages/core/dist/chunk-FWK3Q3OJ.mjs +0 -56
- package/packages/core/dist/chunk-FWK3Q3OJ.mjs.map +0 -1
- package/packages/core/dist/chunk-LXYPM2ZO.mjs.map +0 -1
- package/packages/core/dist/chunk-QLYUWKK7.mjs.map +0 -1
- package/packages/core/dist/chunk-VQUB3WRI.mjs.map +0 -1
|
@@ -1252,11 +1252,23 @@ var init_trace_event_safety = __esm({
|
|
|
1252
1252
|
init_persisted_inspect_event();
|
|
1253
1253
|
}
|
|
1254
1254
|
});
|
|
1255
|
+
function getSharedContextStorage() {
|
|
1256
|
+
const host = globalThis;
|
|
1257
|
+
const existing = host[GLOBAL_CONTEXT_ALS_KEY];
|
|
1258
|
+
if (existing instanceof async_hooks.AsyncLocalStorage) {
|
|
1259
|
+
return existing;
|
|
1260
|
+
}
|
|
1261
|
+
const created = new async_hooks.AsyncLocalStorage();
|
|
1262
|
+
host[GLOBAL_CONTEXT_ALS_KEY] = created;
|
|
1263
|
+
return created;
|
|
1264
|
+
}
|
|
1265
|
+
var GLOBAL_CONTEXT_ALS_KEY;
|
|
1255
1266
|
var init_context = __esm({
|
|
1256
1267
|
"packages/core/src/context.ts"() {
|
|
1257
1268
|
init_correlation_metadata();
|
|
1258
1269
|
init_trace_event_safety();
|
|
1259
|
-
|
|
1270
|
+
GLOBAL_CONTEXT_ALS_KEY = "agent-inspect:execution-context-als:v1";
|
|
1271
|
+
getSharedContextStorage();
|
|
1260
1272
|
}
|
|
1261
1273
|
});
|
|
1262
1274
|
var init_inspector_runtime = __esm({
|
|
@@ -6524,6 +6536,45 @@ function createToolOrderingRule(options) {
|
|
|
6524
6536
|
requireEndpoints: false
|
|
6525
6537
|
});
|
|
6526
6538
|
}
|
|
6539
|
+
function createToolFailureRule(options) {
|
|
6540
|
+
return {
|
|
6541
|
+
id: "tool.failures",
|
|
6542
|
+
category: "tool",
|
|
6543
|
+
defaultSeverity: "error",
|
|
6544
|
+
evaluate(context) {
|
|
6545
|
+
const tools = finishedEvents(context, "TOOL");
|
|
6546
|
+
const failures = tools.filter((event) => event.status === "error");
|
|
6547
|
+
const retries = tools.map((event) => ({ event, count: retryCount(event) })).filter((item) => item.count !== void 0);
|
|
6548
|
+
const findings = [];
|
|
6549
|
+
if (options.maxFailures !== void 0 && failures.length > options.maxFailures) {
|
|
6550
|
+
findings.push(
|
|
6551
|
+
failFinding(
|
|
6552
|
+
"tool.failures",
|
|
6553
|
+
`Tool failure count ${failures.length} exceeded ${options.maxFailures}.`,
|
|
6554
|
+
failures.map((event) => eventEvidence(event)),
|
|
6555
|
+
{ maxFailures: options.maxFailures },
|
|
6556
|
+
failures.length
|
|
6557
|
+
)
|
|
6558
|
+
);
|
|
6559
|
+
}
|
|
6560
|
+
if (options.maxRetries !== void 0) {
|
|
6561
|
+
const excessiveRetries = retries.filter((item) => item.count > options.maxRetries);
|
|
6562
|
+
if (excessiveRetries.length > 0) {
|
|
6563
|
+
findings.push(
|
|
6564
|
+
failFinding(
|
|
6565
|
+
"tool.failures",
|
|
6566
|
+
`Tool retry count exceeded ${options.maxRetries}.`,
|
|
6567
|
+
excessiveRetries.map((item) => eventEvidence(item.event, "attributes.retryCount")),
|
|
6568
|
+
{ maxRetries: options.maxRetries },
|
|
6569
|
+
excessiveRetries.map((item) => ({ tool: toolName(item.event), retries: item.count }))
|
|
6570
|
+
)
|
|
6571
|
+
);
|
|
6572
|
+
}
|
|
6573
|
+
}
|
|
6574
|
+
return findings;
|
|
6575
|
+
}
|
|
6576
|
+
};
|
|
6577
|
+
}
|
|
6527
6578
|
function createLlmUsageRule(options) {
|
|
6528
6579
|
return {
|
|
6529
6580
|
id: "llm.usage",
|
|
@@ -12618,14 +12669,20 @@ var init_to_inspect_event = __esm({
|
|
|
12618
12669
|
});
|
|
12619
12670
|
|
|
12620
12671
|
// packages/core/src/persisted/tree-bridge.ts
|
|
12672
|
+
function logicalToPersisted(event) {
|
|
12673
|
+
const { sourceEventIds: _sourceEventIds, projection: _projection, ...rest } = event;
|
|
12674
|
+
return rest;
|
|
12675
|
+
}
|
|
12621
12676
|
function persistedInspectEventsToRunTrees(events, options) {
|
|
12622
|
-
const
|
|
12677
|
+
const sourceEvents = options?.coalesceLifecycle === true ? projectLogicalEvents(events).logicalEvents.map(logicalToPersisted) : events;
|
|
12678
|
+
const inspectEvents = persistedInspectEventsToInspectEvents(sourceEvents, {
|
|
12623
12679
|
skipInvalid: options?.skipInvalid
|
|
12624
12680
|
});
|
|
12625
12681
|
return new TreeBuilder().build(inspectEvents);
|
|
12626
12682
|
}
|
|
12627
12683
|
var init_tree_bridge = __esm({
|
|
12628
12684
|
"packages/core/src/persisted/tree-bridge.ts"() {
|
|
12685
|
+
init_logical_events();
|
|
12629
12686
|
init_tree_builder();
|
|
12630
12687
|
init_from_trace_event();
|
|
12631
12688
|
init_to_inspect_event();
|
|
@@ -13246,7 +13303,7 @@ function mapOpenInferenceSpan(span, index, version2) {
|
|
|
13246
13303
|
const errorMessage = isRecord11(span.status) && typeof span.status.message === "string" ? span.status.message : void 0;
|
|
13247
13304
|
const event = {
|
|
13248
13305
|
schemaVersion: "0.2",
|
|
13249
|
-
eventId: typeof rawAttributes["agent_inspect.event_id"] === "string" ? rawAttributes["agent_inspect.event_id"] : spanId
|
|
13306
|
+
eventId: typeof rawAttributes["agent_inspect.event_id"] === "string" ? rawAttributes["agent_inspect.event_id"] : `${traceId}:${spanId}`,
|
|
13250
13307
|
runId: typeof rawAttributes["agent_inspect.run_id"] === "string" ? rawAttributes["agent_inspect.run_id"] : traceId,
|
|
13251
13308
|
kind,
|
|
13252
13309
|
name,
|
|
@@ -13297,12 +13354,16 @@ function mapOpenInferenceEvents(document) {
|
|
|
13297
13354
|
const mapped = document.spans.map(
|
|
13298
13355
|
(span, index) => mapOpenInferenceSpan(span, index, document.version)
|
|
13299
13356
|
);
|
|
13300
|
-
const
|
|
13301
|
-
mapped.map((span) =>
|
|
13357
|
+
const spanKeyToEventId = new Map(
|
|
13358
|
+
mapped.map((span) => {
|
|
13359
|
+
const traceId = typeof span.event.trace?.traceId === "string" ? span.event.trace.traceId : "";
|
|
13360
|
+
return [`${traceId}:${span.spanId}`, span.event.eventId];
|
|
13361
|
+
})
|
|
13302
13362
|
);
|
|
13303
13363
|
for (const span of mapped) {
|
|
13304
13364
|
if (span.parentSpanId === void 0) continue;
|
|
13305
|
-
span.event.
|
|
13365
|
+
const traceId = typeof span.event.trace?.traceId === "string" ? span.event.trace.traceId : "";
|
|
13366
|
+
span.event.parentId = spanKeyToEventId.get(`${traceId}:${span.parentSpanId}`) ?? span.parentSpanId;
|
|
13306
13367
|
}
|
|
13307
13368
|
return {
|
|
13308
13369
|
events: mapped.map((span) => span.event),
|
|
@@ -13773,7 +13834,7 @@ function mapOtlpSpan(context) {
|
|
|
13773
13834
|
const errorMessage = isRecord11(span.status) && typeof span.status.message === "string" ? span.status.message : void 0;
|
|
13774
13835
|
const event = {
|
|
13775
13836
|
schemaVersion: "0.2",
|
|
13776
|
-
eventId: typeof parsedSpanAttributes.attributes["agent_inspect.event_id"] === "string" ? parsedSpanAttributes.attributes["agent_inspect.event_id"] : spanId
|
|
13837
|
+
eventId: typeof parsedSpanAttributes.attributes["agent_inspect.event_id"] === "string" ? parsedSpanAttributes.attributes["agent_inspect.event_id"] : `${traceId}:${spanId}`,
|
|
13777
13838
|
runId: typeof parsedSpanAttributes.attributes["agent_inspect.run_id"] === "string" ? parsedSpanAttributes.attributes["agent_inspect.run_id"] : traceId,
|
|
13778
13839
|
kind,
|
|
13779
13840
|
name: readStringField(span, ["name"]) ?? spanId,
|
|
@@ -13822,12 +13883,16 @@ function mapOtlpSpan(context) {
|
|
|
13822
13883
|
}
|
|
13823
13884
|
function mapOtlpEventsToPersisted(document) {
|
|
13824
13885
|
const mapped = document.spans.map((span) => mapOtlpSpan(span));
|
|
13825
|
-
const
|
|
13826
|
-
mapped.map((span) =>
|
|
13886
|
+
const spanKeyToEventId = new Map(
|
|
13887
|
+
mapped.map((span) => {
|
|
13888
|
+
const traceId = typeof span.event.trace?.traceId === "string" ? span.event.trace.traceId : "";
|
|
13889
|
+
return [`${traceId}:${span.spanId}`, span.event.eventId];
|
|
13890
|
+
})
|
|
13827
13891
|
);
|
|
13828
13892
|
for (const span of mapped) {
|
|
13829
13893
|
if (span.parentSpanId === void 0) continue;
|
|
13830
|
-
span.event.
|
|
13894
|
+
const traceId = typeof span.event.trace?.traceId === "string" ? span.event.trace.traceId : "";
|
|
13895
|
+
span.event.parentId = spanKeyToEventId.get(`${traceId}:${span.parentSpanId}`) ?? span.parentSpanId;
|
|
13831
13896
|
}
|
|
13832
13897
|
return {
|
|
13833
13898
|
events: mapped.map((span) => span.event),
|
|
@@ -14283,18 +14348,36 @@ var init_readers = __esm({
|
|
|
14283
14348
|
function diagnostic5(code, message, severity = "error", caseId) {
|
|
14284
14349
|
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
14285
14350
|
}
|
|
14286
|
-
function
|
|
14351
|
+
function buildCaseAssertions(suiteCase, config) {
|
|
14287
14352
|
const rules = [];
|
|
14288
14353
|
const select = new Set(config.checks?.select ?? []);
|
|
14289
|
-
|
|
14354
|
+
const configDiagnostics = [];
|
|
14355
|
+
const evalConfig = config.eval;
|
|
14356
|
+
const declaredSelect = [...select];
|
|
14357
|
+
for (const id of declaredSelect) {
|
|
14358
|
+
if (!KNOWN_SUITE_SELECT_IDS.has(id)) {
|
|
14359
|
+
configDiagnostics.push(
|
|
14360
|
+
diagnostic5(
|
|
14361
|
+
"AI_SUITE_UNKNOWN_SELECTOR",
|
|
14362
|
+
`Unknown or unsupported suite check selector "${id}".`,
|
|
14363
|
+
"error",
|
|
14364
|
+
suiteCase.id
|
|
14365
|
+
)
|
|
14366
|
+
);
|
|
14367
|
+
}
|
|
14368
|
+
}
|
|
14369
|
+
if (select.has("run.status") || evalConfig?.requireSuccess === true) {
|
|
14290
14370
|
rules.push(createRunStatusRule());
|
|
14371
|
+
select.add("run.status");
|
|
14291
14372
|
}
|
|
14292
14373
|
const requiredTools = [
|
|
14293
14374
|
...config.checks?.tool?.required ?? [],
|
|
14375
|
+
...evalConfig?.requiredTools ?? [],
|
|
14294
14376
|
...suiteCase.requireTools ?? []
|
|
14295
14377
|
];
|
|
14296
14378
|
const forbiddenTools = [
|
|
14297
14379
|
...config.checks?.tool?.forbidden ?? [],
|
|
14380
|
+
...evalConfig?.forbiddenTools ?? [],
|
|
14298
14381
|
...suiteCase.forbidTools ?? []
|
|
14299
14382
|
];
|
|
14300
14383
|
if (requiredTools.length > 0 || forbiddenTools.length > 0) {
|
|
@@ -14306,20 +14389,51 @@ function buildCaseRules(suiteCase, config) {
|
|
|
14306
14389
|
);
|
|
14307
14390
|
select.add("tool.usage");
|
|
14308
14391
|
}
|
|
14309
|
-
const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ??
|
|
14392
|
+
const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? evalConfig?.maxDurationMs;
|
|
14310
14393
|
if (maxDurationMs !== void 0) {
|
|
14311
14394
|
rules.push(createRunDurationRule({ maxDurationMs }));
|
|
14312
14395
|
select.add("run.duration");
|
|
14313
14396
|
}
|
|
14314
|
-
const
|
|
14315
|
-
if (
|
|
14316
|
-
rules.push(
|
|
14397
|
+
const maxDepth = config.checks?.run?.maxDepth ?? evalConfig?.maxDepth;
|
|
14398
|
+
if (maxDepth !== void 0) {
|
|
14399
|
+
rules.push(createRunDepthRule({ maxDepth }));
|
|
14400
|
+
select.add("run.depth");
|
|
14401
|
+
}
|
|
14402
|
+
if (evalConfig?.maxRetries !== void 0) {
|
|
14403
|
+
rules.push(createToolFailureRule({ maxRetries: evalConfig.maxRetries }));
|
|
14404
|
+
select.add("tool.failures");
|
|
14405
|
+
}
|
|
14406
|
+
const allowedModels = config.checks?.llm?.allowedModels;
|
|
14407
|
+
const maxTotalTokens = config.checks?.llm?.maxTotalTokens ?? evalConfig?.maxTotalTokens;
|
|
14408
|
+
if (allowedModels !== void 0 || maxTotalTokens !== void 0) {
|
|
14409
|
+
rules.push(
|
|
14410
|
+
createLlmUsageRule({
|
|
14411
|
+
...allowedModels !== void 0 ? { allowedModels } : {},
|
|
14412
|
+
...maxTotalTokens !== void 0 ? { maxTotalTokens } : {}
|
|
14413
|
+
})
|
|
14414
|
+
);
|
|
14317
14415
|
select.add("llm.usage");
|
|
14318
14416
|
}
|
|
14319
14417
|
if (select.has("outcome.status")) {
|
|
14320
14418
|
rules.push(createObservedOutcomeRule({ failOn: ["failed"] }));
|
|
14321
14419
|
}
|
|
14322
|
-
|
|
14420
|
+
const observationCount = suiteCase.expectedObservations?.length ?? 0;
|
|
14421
|
+
if (rules.length === 0 && observationCount === 0 && configDiagnostics.length === 0) {
|
|
14422
|
+
configDiagnostics.push(
|
|
14423
|
+
diagnostic5(
|
|
14424
|
+
"AI_SUITE_NO_ASSERTIONS",
|
|
14425
|
+
`Suite case "${suiteCase.id}" declares no effective checks, eval controls, or expected observations.`,
|
|
14426
|
+
"error",
|
|
14427
|
+
suiteCase.id
|
|
14428
|
+
)
|
|
14429
|
+
);
|
|
14430
|
+
}
|
|
14431
|
+
return {
|
|
14432
|
+
rules,
|
|
14433
|
+
select: [...select],
|
|
14434
|
+
observationCount,
|
|
14435
|
+
configDiagnostics
|
|
14436
|
+
};
|
|
14323
14437
|
}
|
|
14324
14438
|
function outcomesFromRead(read) {
|
|
14325
14439
|
const persistedOutcomes = extractOutcomesFromPersistedEvents(
|
|
@@ -14400,8 +14514,20 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
14400
14514
|
]
|
|
14401
14515
|
};
|
|
14402
14516
|
}
|
|
14403
|
-
const
|
|
14404
|
-
|
|
14517
|
+
const compiled = buildCaseAssertions(suiteCase, config);
|
|
14518
|
+
if (compiled.configDiagnostics.length > 0) {
|
|
14519
|
+
return {
|
|
14520
|
+
id: suiteCase.id,
|
|
14521
|
+
status: "error",
|
|
14522
|
+
tracePath: resolved.tracePath,
|
|
14523
|
+
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
14524
|
+
checkOk: false,
|
|
14525
|
+
...config.eval?.requireSuccess === true ? { evalOk: false } : {},
|
|
14526
|
+
message: compiled.configDiagnostics.map((item) => item.message).join("; "),
|
|
14527
|
+
diagnostics: compiled.configDiagnostics
|
|
14528
|
+
};
|
|
14529
|
+
}
|
|
14530
|
+
const checkResult = compiled.rules.length > 0 ? runTraceChecks({ read }, { rules: compiled.rules, select: compiled.select }) : {
|
|
14405
14531
|
ok: true,
|
|
14406
14532
|
status: "pass",
|
|
14407
14533
|
format: read.format,
|
|
@@ -14429,6 +14555,9 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
14429
14555
|
];
|
|
14430
14556
|
const checkOk = checkResult.ok;
|
|
14431
14557
|
const observationsOk = observationResult.ok;
|
|
14558
|
+
const evalOk = config.eval?.requireSuccess === true ? checkResult.findings.every(
|
|
14559
|
+
(finding) => finding.ruleId !== "run.status" || finding.status !== "fail"
|
|
14560
|
+
) && checkResult.diagnostics.every((item) => item.severity !== "error") : void 0;
|
|
14432
14561
|
const ok = checkOk && observationsOk;
|
|
14433
14562
|
const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
|
|
14434
14563
|
return {
|
|
@@ -14437,6 +14566,7 @@ async function runSuiteCase(suiteCase, config, options) {
|
|
|
14437
14566
|
tracePath: resolved.tracePath,
|
|
14438
14567
|
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
14439
14568
|
checkOk,
|
|
14569
|
+
...evalOk !== void 0 ? { evalOk } : {},
|
|
14440
14570
|
observationsOk,
|
|
14441
14571
|
diagnostics,
|
|
14442
14572
|
...ok ? {} : {
|
|
@@ -14481,6 +14611,7 @@ async function runSuite(options = {}) {
|
|
|
14481
14611
|
diagnostics
|
|
14482
14612
|
};
|
|
14483
14613
|
}
|
|
14614
|
+
var KNOWN_SUITE_SELECT_IDS;
|
|
14484
14615
|
var init_run = __esm({
|
|
14485
14616
|
"packages/core/src/suite/run.ts"() {
|
|
14486
14617
|
init_checks2();
|
|
@@ -14488,6 +14619,15 @@ var init_run = __esm({
|
|
|
14488
14619
|
init_readers();
|
|
14489
14620
|
init_load2();
|
|
14490
14621
|
init_resolve2();
|
|
14622
|
+
KNOWN_SUITE_SELECT_IDS = /* @__PURE__ */ new Set([
|
|
14623
|
+
"run.status",
|
|
14624
|
+
"run.duration",
|
|
14625
|
+
"run.depth",
|
|
14626
|
+
"outcome.status",
|
|
14627
|
+
"tool.usage",
|
|
14628
|
+
"tool.failures",
|
|
14629
|
+
"llm.usage"
|
|
14630
|
+
]);
|
|
14491
14631
|
}
|
|
14492
14632
|
});
|
|
14493
14633
|
|
|
@@ -16235,7 +16375,7 @@ var init_src = __esm({
|
|
|
16235
16375
|
});
|
|
16236
16376
|
|
|
16237
16377
|
// package.json
|
|
16238
|
-
var version = "6.31.
|
|
16378
|
+
var version = "6.31.9";
|
|
16239
16379
|
|
|
16240
16380
|
// packages/cli/src/list.ts
|
|
16241
16381
|
init_advanced();
|
|
@@ -18770,11 +18910,13 @@ function exportOpenInference(tree, options) {
|
|
|
18770
18910
|
endNs = startNs + unixNano(ev.durationMs);
|
|
18771
18911
|
}
|
|
18772
18912
|
const { openInferenceKind } = mapInspectKindToOI(ev.kind, warnings);
|
|
18913
|
+
const meta2 = ev.attributes;
|
|
18914
|
+
const originalSource = typeof meta2?.originalSourceType === "string" && meta2.originalSourceType.trim() !== "" ? meta2.originalSourceType.trim() : ev.source.type;
|
|
18773
18915
|
const attrs = {
|
|
18774
18916
|
"openinference.span.kind": openInferenceKind,
|
|
18775
18917
|
"agent_inspect.kind": ev.kind,
|
|
18776
18918
|
"agent_inspect.confidence": ev.confidence,
|
|
18777
|
-
"agent_inspect.source.type":
|
|
18919
|
+
"agent_inspect.source.type": originalSource,
|
|
18778
18920
|
"agent_inspect.run_id": tree.runId,
|
|
18779
18921
|
"agent_inspect.event_id": ev.eventId,
|
|
18780
18922
|
"agent_inspect.status": ev.status ?? "unset"
|
|
@@ -18782,9 +18924,9 @@ function exportOpenInference(tree, options) {
|
|
|
18782
18924
|
if (ev.durationMs !== void 0) {
|
|
18783
18925
|
attrs["agent_inspect.duration_ms"] = ev.durationMs;
|
|
18784
18926
|
}
|
|
18785
|
-
const
|
|
18786
|
-
if (
|
|
18787
|
-
attrs["llm.model_name"] =
|
|
18927
|
+
const requestModel = (typeof meta2?.model === "string" && meta2.model.trim() !== "" ? meta2.model : void 0) ?? (typeof meta2?.modelId === "string" && meta2.modelId.trim() !== "" ? meta2.modelId : void 0);
|
|
18928
|
+
if (requestModel !== void 0) {
|
|
18929
|
+
attrs["llm.model_name"] = requestModel;
|
|
18788
18930
|
}
|
|
18789
18931
|
const tokens = meta2?.tokens;
|
|
18790
18932
|
if (tokens && typeof tokens === "object" && tokens !== null) {
|
|
@@ -18794,8 +18936,9 @@ function exportOpenInference(tree, options) {
|
|
|
18794
18936
|
if (typeof outp === "number") attrs["llm.token_count.completion"] = outp;
|
|
18795
18937
|
}
|
|
18796
18938
|
if (includeAttributes && meta2 && typeof meta2 === "object") {
|
|
18939
|
+
const skipKeys = /* @__PURE__ */ new Set(["tokens", "model", "modelId", "originalSourceType"]);
|
|
18797
18940
|
for (const [k, v] of Object.entries(meta2)) {
|
|
18798
|
-
if (k
|
|
18941
|
+
if (skipKeys.has(k)) continue;
|
|
18799
18942
|
if (v !== void 0 && v !== null && typeof v !== "object") {
|
|
18800
18943
|
attrs[`agent_inspect.preview.${k}`] = typeof v === "string" ? v.slice(0, maxLen) : v;
|
|
18801
18944
|
}
|
|
@@ -18916,24 +19059,33 @@ function exportOtlpJson(tree, options) {
|
|
|
18916
19059
|
const spanId = hexFrom2(`${tree.runId}:${ev.eventId}`, 8);
|
|
18917
19060
|
const parentSpanId = ev.parentId ? hexFrom2(`${tree.runId}:${ev.parentId}`, 8) : void 0;
|
|
18918
19061
|
const { startNs, endNs } = resolveSpanTimes(ev);
|
|
19062
|
+
const meta2 = ev.attributes;
|
|
19063
|
+
const originalSource = typeof meta2?.originalSourceType === "string" && meta2.originalSourceType.trim() !== "" ? meta2.originalSourceType.trim() : ev.source.type;
|
|
18919
19064
|
const attrs = [
|
|
18920
19065
|
stringAttr2("agent_inspect.kind", ev.kind),
|
|
18921
19066
|
stringAttr2("agent_inspect.confidence", ev.confidence),
|
|
18922
|
-
stringAttr2("agent_inspect.source.type",
|
|
19067
|
+
stringAttr2("agent_inspect.source.type", originalSource),
|
|
18923
19068
|
stringAttr2("agent_inspect.run_id", tree.runId),
|
|
18924
19069
|
stringAttr2("agent_inspect.event_id", ev.eventId),
|
|
18925
19070
|
stringAttr2("agent_inspect.status", ev.status ?? "unset")
|
|
18926
19071
|
];
|
|
18927
19072
|
if (ev.durationMs !== void 0) {
|
|
18928
|
-
attrs.push(
|
|
19073
|
+
attrs.push(numberAttr("agent_inspect.duration_ms", ev.durationMs));
|
|
18929
19074
|
}
|
|
18930
19075
|
const op = genAiOperationName(ev.kind);
|
|
18931
19076
|
if (op !== void 0) {
|
|
18932
19077
|
attrs.push(stringAttr2("gen_ai.operation.name", op));
|
|
18933
19078
|
}
|
|
18934
|
-
const
|
|
18935
|
-
if (
|
|
18936
|
-
attrs.push(stringAttr2("gen_ai.request.model",
|
|
19079
|
+
const requestModel = (typeof meta2?.model === "string" && meta2.model.trim() !== "" ? meta2.model : void 0) ?? (typeof meta2?.modelId === "string" && meta2.modelId.trim() !== "" ? meta2.modelId : void 0);
|
|
19080
|
+
if (requestModel !== void 0) {
|
|
19081
|
+
attrs.push(stringAttr2("gen_ai.request.model", requestModel.slice(0, maxLen)));
|
|
19082
|
+
}
|
|
19083
|
+
const responseModel = typeof meta2?.responseModelId === "string" && meta2.responseModelId.trim() !== "" ? meta2.responseModelId : typeof meta2?.responseModel === "string" && meta2.responseModel.trim() !== "" ? meta2.responseModel : void 0;
|
|
19084
|
+
if (responseModel !== void 0) {
|
|
19085
|
+
attrs.push(stringAttr2("gen_ai.response.model", responseModel.slice(0, maxLen)));
|
|
19086
|
+
}
|
|
19087
|
+
if (typeof meta2?.provider === "string" && meta2.provider.trim() !== "") {
|
|
19088
|
+
attrs.push(stringAttr2("gen_ai.provider.name", meta2.provider.slice(0, maxLen)));
|
|
18937
19089
|
}
|
|
18938
19090
|
const tokens = meta2?.tokens;
|
|
18939
19091
|
if (tokens && typeof tokens === "object" && tokens !== null) {
|
|
@@ -18943,8 +19095,17 @@ function exportOtlpJson(tree, options) {
|
|
|
18943
19095
|
if (typeof outp === "number") attrs.push(intAttr("gen_ai.usage.output_tokens", outp));
|
|
18944
19096
|
}
|
|
18945
19097
|
if (includeAttributes && meta2 && typeof meta2 === "object") {
|
|
19098
|
+
const skipKeys = /* @__PURE__ */ new Set([
|
|
19099
|
+
"tokens",
|
|
19100
|
+
"model",
|
|
19101
|
+
"modelId",
|
|
19102
|
+
"responseModel",
|
|
19103
|
+
"responseModelId",
|
|
19104
|
+
"provider",
|
|
19105
|
+
"originalSourceType"
|
|
19106
|
+
]);
|
|
18946
19107
|
for (const [k, v] of Object.entries(meta2)) {
|
|
18947
|
-
if (k
|
|
19108
|
+
if (skipKeys.has(k)) continue;
|
|
18948
19109
|
if (typeof v === "string") {
|
|
18949
19110
|
attrs.push(stringAttr2(`agent_inspect.preview.${k}`, v.slice(0, maxLen)));
|
|
18950
19111
|
} else if (typeof v === "number" && Number.isFinite(v)) {
|
|
@@ -19014,12 +19175,42 @@ function pushPath(errors, path44, message) {
|
|
|
19014
19175
|
function isRecord16(value) {
|
|
19015
19176
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
19016
19177
|
}
|
|
19178
|
+
function isHexId(value, byteLen) {
|
|
19179
|
+
return typeof value === "string" && new RegExp(`^[0-9a-fA-F]{${byteLen * 2}}$`).test(value);
|
|
19180
|
+
}
|
|
19181
|
+
function validateAnyValue(errors, path44, value) {
|
|
19182
|
+
if (!isRecord16(value)) {
|
|
19183
|
+
pushPath(errors, path44, "must be an AnyValue object");
|
|
19184
|
+
return;
|
|
19185
|
+
}
|
|
19186
|
+
const variants = [
|
|
19187
|
+
"stringValue",
|
|
19188
|
+
"boolValue",
|
|
19189
|
+
"intValue",
|
|
19190
|
+
"doubleValue",
|
|
19191
|
+
"bytesValue",
|
|
19192
|
+
"arrayValue",
|
|
19193
|
+
"kvlistValue"
|
|
19194
|
+
].filter((key) => value[key] !== void 0);
|
|
19195
|
+
if (variants.length !== 1) {
|
|
19196
|
+
pushPath(
|
|
19197
|
+
errors,
|
|
19198
|
+
path44,
|
|
19199
|
+
`AnyValue must set exactly one variant (found ${variants.length}: ${variants.join(",")})`
|
|
19200
|
+
);
|
|
19201
|
+
}
|
|
19202
|
+
if (value.intValue !== void 0) {
|
|
19203
|
+
if (typeof value.intValue !== "string" || !/^-?\d+$/.test(value.intValue)) {
|
|
19204
|
+
pushPath(errors, `${path44}.intValue`, "must be a decimal integer string");
|
|
19205
|
+
}
|
|
19206
|
+
}
|
|
19207
|
+
}
|
|
19017
19208
|
function validateOtlpJson(content) {
|
|
19018
19209
|
const errors = [];
|
|
19019
19210
|
const warnings = [
|
|
19020
19211
|
EXPERIMENTAL,
|
|
19021
19212
|
"OTLP JSON mapping uses OTel GenAI-aligned attributes where applicable; collectors may require transformation.",
|
|
19022
|
-
"
|
|
19213
|
+
"producer-profile checks IDs, timing, and AnyValue shape; historical string enums are reader-compat warnings only."
|
|
19023
19214
|
];
|
|
19024
19215
|
let parsed;
|
|
19025
19216
|
try {
|
|
@@ -19062,11 +19253,26 @@ function validateOtlpJson(content) {
|
|
|
19062
19253
|
pushPath(errors, spanPath, "must be an object");
|
|
19063
19254
|
return;
|
|
19064
19255
|
}
|
|
19065
|
-
if (
|
|
19066
|
-
pushPath(
|
|
19256
|
+
if (!isHexId(span.traceId, 16)) {
|
|
19257
|
+
pushPath(
|
|
19258
|
+
errors,
|
|
19259
|
+
`${spanPath}.traceId`,
|
|
19260
|
+
"must be a 32-char hex trace id (16 bytes)"
|
|
19261
|
+
);
|
|
19262
|
+
}
|
|
19263
|
+
if (!isHexId(span.spanId, 8)) {
|
|
19264
|
+
pushPath(
|
|
19265
|
+
errors,
|
|
19266
|
+
`${spanPath}.spanId`,
|
|
19267
|
+
"must be a 16-char hex span id (8 bytes)"
|
|
19268
|
+
);
|
|
19067
19269
|
}
|
|
19068
|
-
if (
|
|
19069
|
-
pushPath(
|
|
19270
|
+
if (span.parentSpanId !== void 0 && !isHexId(span.parentSpanId, 8)) {
|
|
19271
|
+
pushPath(
|
|
19272
|
+
errors,
|
|
19273
|
+
`${spanPath}.parentSpanId`,
|
|
19274
|
+
"must be a 16-char hex span id when present"
|
|
19275
|
+
);
|
|
19070
19276
|
}
|
|
19071
19277
|
if (typeof span.name !== "string") {
|
|
19072
19278
|
pushPath(errors, `${spanPath}.name`, "must be a string");
|
|
@@ -19085,13 +19291,39 @@ function validateOtlpJson(content) {
|
|
|
19085
19291
|
"must be a decimal-string 64-bit integer when present"
|
|
19086
19292
|
);
|
|
19087
19293
|
}
|
|
19294
|
+
if (typeof span.startTimeUnixNano === "string" && /^\d+$/.test(span.startTimeUnixNano) && typeof span.endTimeUnixNano === "string" && /^\d+$/.test(span.endTimeUnixNano)) {
|
|
19295
|
+
try {
|
|
19296
|
+
if (BigInt(span.endTimeUnixNano) < BigInt(span.startTimeUnixNano)) {
|
|
19297
|
+
pushPath(
|
|
19298
|
+
errors,
|
|
19299
|
+
`${spanPath}.endTimeUnixNano`,
|
|
19300
|
+
"must be >= startTimeUnixNano"
|
|
19301
|
+
);
|
|
19302
|
+
}
|
|
19303
|
+
} catch {
|
|
19304
|
+
pushPath(errors, `${spanPath}.endTimeUnixNano`, "out of integer range");
|
|
19305
|
+
}
|
|
19306
|
+
}
|
|
19307
|
+
if (Array.isArray(span.attributes)) {
|
|
19308
|
+
span.attributes.forEach((attr, ai) => {
|
|
19309
|
+
const attrPath = `${spanPath}.attributes[${ai}]`;
|
|
19310
|
+
if (!isRecord16(attr)) {
|
|
19311
|
+
pushPath(errors, attrPath, "must be an object");
|
|
19312
|
+
return;
|
|
19313
|
+
}
|
|
19314
|
+
if (typeof attr.key !== "string") {
|
|
19315
|
+
pushPath(errors, `${attrPath}.key`, "must be a string");
|
|
19316
|
+
}
|
|
19317
|
+
validateAnyValue(errors, `${attrPath}.value`, attr.value);
|
|
19318
|
+
});
|
|
19319
|
+
}
|
|
19088
19320
|
if (typeof span.kind === "number") {
|
|
19089
19321
|
if (!Number.isInteger(span.kind)) {
|
|
19090
19322
|
pushPath(errors, `${spanPath}.kind`, "must be an integer SpanKind enum");
|
|
19091
19323
|
}
|
|
19092
19324
|
} else if (typeof span.kind === "string") {
|
|
19093
19325
|
warnings.push(
|
|
19094
|
-
`${spanPath}.kind: string SpanKind
|
|
19326
|
+
`${spanPath}.kind: string SpanKind is historical reader-compat only; producer wire format uses numeric enums`
|
|
19095
19327
|
);
|
|
19096
19328
|
} else if (span.kind !== void 0) {
|
|
19097
19329
|
pushPath(
|
|
@@ -19122,7 +19354,7 @@ function validateOtlpJson(content) {
|
|
|
19122
19354
|
);
|
|
19123
19355
|
} else {
|
|
19124
19356
|
warnings.push(
|
|
19125
|
-
`${spanPath}.status.code: string StatusCode is
|
|
19357
|
+
`${spanPath}.status.code: string StatusCode is historical reader-compat only; producer wire format uses numeric 0/1/2`
|
|
19126
19358
|
);
|
|
19127
19359
|
}
|
|
19128
19360
|
} else {
|
|
@@ -19603,7 +19835,9 @@ Trace directory: ${traceDir}`);
|
|
|
19603
19835
|
}
|
|
19604
19836
|
let tree;
|
|
19605
19837
|
try {
|
|
19606
|
-
const trees = persistedInspectEventsToRunTrees(events
|
|
19838
|
+
const trees = persistedInspectEventsToRunTrees(events, {
|
|
19839
|
+
coalesceLifecycle: true
|
|
19840
|
+
});
|
|
19607
19841
|
tree = trees.find((candidate) => candidate.runId === id) ?? trees[0];
|
|
19608
19842
|
if (tree === void 0) {
|
|
19609
19843
|
throw new Error("No run tree could be built from persisted events");
|
|
@@ -28899,13 +29133,15 @@ import { generateText, stepCountIs, tool } from "ai";
|
|
|
28899
29133
|
import { MockLanguageModelV3 } from "ai/test";
|
|
28900
29134
|
import { agentInspect } from "@agent-inspect/ai-sdk";
|
|
28901
29135
|
import { z } from "zod";
|
|
28902
|
-
import { readdirSync, readFileSync } from "node:fs";
|
|
29136
|
+
import { mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs";
|
|
28903
29137
|
import path from "node:path";
|
|
29138
|
+
import { randomUUID } from "node:crypto";
|
|
28904
29139
|
|
|
28905
|
-
const
|
|
29140
|
+
const SESSION = randomUUID().slice(0, 8);
|
|
29141
|
+
const TRACE_DIR = path.join(".agent-inspect", \`demo-\${SESSION}\`);
|
|
28906
29142
|
const ANSWER = "You have 2 orders";
|
|
28907
|
-
const CORRECT_RUN =
|
|
28908
|
-
const WRONG_RUN =
|
|
29143
|
+
const CORRECT_RUN = \`ai-sdk-demo-correct-\${SESSION}\`;
|
|
29144
|
+
const WRONG_RUN = \`ai-sdk-demo-wrong-\${SESSION}\`;
|
|
28909
29145
|
|
|
28910
29146
|
const usage = {
|
|
28911
29147
|
inputTokens: { total: 4, noCache: 4, cacheRead: 0, cacheWrite: 0 },
|
|
@@ -28945,7 +29181,7 @@ async function runPath(runName, toolName) {
|
|
|
28945
29181
|
type: "tool-call",
|
|
28946
29182
|
toolCallId: \`call-\${toolName}\`,
|
|
28947
29183
|
toolName,
|
|
28948
|
-
input: { userId: "u1" },
|
|
29184
|
+
input: JSON.stringify({ userId: "u1" }),
|
|
28949
29185
|
},
|
|
28950
29186
|
],
|
|
28951
29187
|
finishReason: { unified: "tool-calls", raw: "tool-calls" },
|
|
@@ -28997,15 +29233,39 @@ function selectRunIdByName(runName) {
|
|
|
28997
29233
|
return matches[0];
|
|
28998
29234
|
}
|
|
28999
29235
|
|
|
29236
|
+
function assertToolExecuted(runId, requiredTool) {
|
|
29237
|
+
const text = readFileSync(path.join(TRACE_DIR, \`\${runId}.jsonl\`), "utf8");
|
|
29238
|
+
const toolHits = text
|
|
29239
|
+
.split("\\n")
|
|
29240
|
+
.filter(Boolean)
|
|
29241
|
+
.map((line) => JSON.parse(line))
|
|
29242
|
+
.filter(
|
|
29243
|
+
(ev) =>
|
|
29244
|
+
(ev.kind === "TOOL" || ev.attributes?.stepType === "tool") &&
|
|
29245
|
+
(ev.name === requiredTool || ev.attributes?.toolName === requiredTool),
|
|
29246
|
+
);
|
|
29247
|
+
if (toolHits.length === 0) {
|
|
29248
|
+
throw new Error(\`expected TOOL events for \${requiredTool} in \${runId}\`);
|
|
29249
|
+
}
|
|
29250
|
+
}
|
|
29251
|
+
|
|
29000
29252
|
async function main() {
|
|
29001
|
-
|
|
29002
|
-
|
|
29003
|
-
|
|
29004
|
-
|
|
29005
|
-
|
|
29006
|
-
|
|
29007
|
-
|
|
29008
|
-
|
|
29253
|
+
mkdirSync(TRACE_DIR, { recursive: true });
|
|
29254
|
+
try {
|
|
29255
|
+
await runPath(CORRECT_RUN, "lookup_orders");
|
|
29256
|
+
await runPath(WRONG_RUN, "delete_orders");
|
|
29257
|
+
const correctId = selectRunIdByName(CORRECT_RUN);
|
|
29258
|
+
const wrongId = selectRunIdByName(WRONG_RUN);
|
|
29259
|
+
assertToolExecuted(correctId, "lookup_orders");
|
|
29260
|
+
assertToolExecuted(wrongId, "delete_orders");
|
|
29261
|
+
console.log(\`Trace written to \${TRACE_DIR}/\`);
|
|
29262
|
+
console.log(\`Correct path runId: \${correctId}\`);
|
|
29263
|
+
console.log(\`Wrong path runId: \${wrongId}\`);
|
|
29264
|
+
console.log(\`Next: npx agent-inspect check \${correctId} --dir \${TRACE_DIR} --required-tool lookup_orders\`);
|
|
29265
|
+
} catch (error) {
|
|
29266
|
+
rmSync(TRACE_DIR, { recursive: true, force: true });
|
|
29267
|
+
throw error;
|
|
29268
|
+
}
|
|
29009
29269
|
}
|
|
29010
29270
|
|
|
29011
29271
|
main().catch((error) => {
|