agent-inspect 4.4.0 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/docs/CLI.md +67 -0
- package/package.json +1 -1
- package/packages/cli/dist/chunk-TSIQUIPF.mjs +8505 -0
- package/packages/cli/dist/chunk-TSIQUIPF.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +10851 -9298
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +4769 -8027
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-FJOHPC4X.mjs → src-2SZWGKMM.mjs} +3 -3
- package/packages/cli/dist/{src-FJOHPC4X.mjs.map → src-2SZWGKMM.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +3741 -16
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +222 -1
- package/packages/core/dist/advanced.d.ts +222 -1
- package/packages/core/dist/advanced.mjs +1092 -9
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.mjs +2 -1648
- package/packages/core/dist/checks.mjs.map +1 -1
- package/packages/core/dist/chunk-QLYOCCEF.mjs +1650 -0
- package/packages/core/dist/chunk-QLYOCCEF.mjs.map +1 -0
- package/packages/core/dist/chunk-X2FLDF7M.mjs +1581 -0
- package/packages/core/dist/chunk-X2FLDF7M.mjs.map +1 -0
- package/packages/core/dist/readers.mjs +3 -1579
- package/packages/core/dist/readers.mjs.map +1 -1
- package/packages/cli/dist/chunk-ORHDQUT6.mjs +0 -3862
- package/packages/cli/dist/chunk-ORHDQUT6.mjs.map +0 -1
|
@@ -9,6 +9,7 @@ var process2 = require('process');
|
|
|
9
9
|
var tty = require('tty');
|
|
10
10
|
var fs = require('fs');
|
|
11
11
|
var readline = require('readline');
|
|
12
|
+
var url = require('url');
|
|
12
13
|
|
|
13
14
|
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
14
15
|
|
|
@@ -123,15 +124,15 @@ var Redactor = class {
|
|
|
123
124
|
return this.#redactNested(value);
|
|
124
125
|
}
|
|
125
126
|
if (rule.strategy === "full") return "[REDACTED]";
|
|
126
|
-
const
|
|
127
|
+
const asString2 = typeof value === "string" ? value : typeof value === "number" || typeof value === "boolean" || typeof value === "bigint" ? String(value) : void 0;
|
|
127
128
|
if (rule.strategy === "prefix") {
|
|
128
|
-
if (
|
|
129
|
+
if (asString2 === void 0) return "[REDACTED]";
|
|
129
130
|
const keep = Math.max(0, Math.floor(rule.keep));
|
|
130
|
-
return
|
|
131
|
+
return asString2.length <= keep ? `${asString2}\u2026` : `${asString2.slice(0, keep)}\u2026`;
|
|
131
132
|
}
|
|
132
133
|
if (rule.strategy === "hash") {
|
|
133
|
-
if (
|
|
134
|
-
return `[HASH:${stableHash(
|
|
134
|
+
if (asString2 === void 0) return "[HASH:unknown]";
|
|
135
|
+
return `[HASH:${stableHash(asString2)}]`;
|
|
135
136
|
}
|
|
136
137
|
return this.#redactNested(value);
|
|
137
138
|
}
|
|
@@ -2011,6 +2012,10 @@ var OBSERVED_OUTCOME_METHODS = [
|
|
|
2011
2012
|
"queue",
|
|
2012
2013
|
"custom"
|
|
2013
2014
|
];
|
|
2015
|
+
var OUTCOME_ATTRIBUTE_STATUS_KEY = "outcomeStatus";
|
|
2016
|
+
var OUTCOME_ATTRIBUTE_EXPECTATION_KEY = "expectation";
|
|
2017
|
+
var OUTCOME_ATTRIBUTE_METHOD_KEY = "method";
|
|
2018
|
+
var OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY = "observedAt";
|
|
2014
2019
|
var OUTCOME_LEGACY_EVENT = "outcome_observed";
|
|
2015
2020
|
|
|
2016
2021
|
// packages/core/src/outcomes/validate.ts
|
|
@@ -2067,6 +2072,13 @@ function normalizeObserveOutcomeInput(name, options, nowMs = Date.now()) {
|
|
|
2067
2072
|
}
|
|
2068
2073
|
|
|
2069
2074
|
// packages/core/src/outcomes/extract.ts
|
|
2075
|
+
function isOutcomeStatus(value) {
|
|
2076
|
+
return value === "passed" || value === "failed" || value === "unknown" || value === "skipped";
|
|
2077
|
+
}
|
|
2078
|
+
function parseMethod2(value) {
|
|
2079
|
+
if (typeof value !== "string" || value.trim() === "") return void 0;
|
|
2080
|
+
return value;
|
|
2081
|
+
}
|
|
2070
2082
|
function fromOutcomeObservedEvent(event) {
|
|
2071
2083
|
return {
|
|
2072
2084
|
outcomeId: event.outcomeId,
|
|
@@ -2081,6 +2093,27 @@ function fromOutcomeObservedEvent(event) {
|
|
|
2081
2093
|
observedAt: event.observedAt
|
|
2082
2094
|
};
|
|
2083
2095
|
}
|
|
2096
|
+
function fromPersistedOutcome(event) {
|
|
2097
|
+
if (event.kind !== "OUTCOME") return void 0;
|
|
2098
|
+
const attrs = event.attributes ?? {};
|
|
2099
|
+
const statusRaw = attrs[OUTCOME_ATTRIBUTE_STATUS_KEY];
|
|
2100
|
+
if (!isOutcomeStatus(statusRaw)) return void 0;
|
|
2101
|
+
const expectation = typeof attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] === "string" ? attrs[OUTCOME_ATTRIBUTE_EXPECTATION_KEY] : event.name;
|
|
2102
|
+
const observedAtRaw = attrs[OUTCOME_ATTRIBUTE_OBSERVED_AT_KEY];
|
|
2103
|
+
const observedAt = typeof observedAtRaw === "string" ? Date.parse(observedAtRaw) : typeof observedAtRaw === "number" && Number.isFinite(observedAtRaw) ? observedAtRaw : Date.parse(event.timestamp);
|
|
2104
|
+
return {
|
|
2105
|
+
outcomeId: event.eventId,
|
|
2106
|
+
runId: event.runId,
|
|
2107
|
+
...event.parentId !== void 0 ? { parentId: event.parentId } : {},
|
|
2108
|
+
name: event.name,
|
|
2109
|
+
expectation,
|
|
2110
|
+
status: statusRaw,
|
|
2111
|
+
...parseMethod2(attrs[OUTCOME_ATTRIBUTE_METHOD_KEY]) !== void 0 ? { method: parseMethod2(attrs[OUTCOME_ATTRIBUTE_METHOD_KEY]) } : {},
|
|
2112
|
+
...event.outputSummary !== void 0 ? { actual: event.outputSummary } : {},
|
|
2113
|
+
...attrs.evidence !== void 0 ? { evidence: attrs.evidence } : {},
|
|
2114
|
+
observedAt: Number.isFinite(observedAt) ? observedAt : Date.parse(event.timestamp)
|
|
2115
|
+
};
|
|
2116
|
+
}
|
|
2084
2117
|
function extractOutcomesFromTraceEvents(events) {
|
|
2085
2118
|
const out = [];
|
|
2086
2119
|
for (const event of events) {
|
|
@@ -2090,6 +2123,18 @@ function extractOutcomesFromTraceEvents(events) {
|
|
|
2090
2123
|
}
|
|
2091
2124
|
return out.sort((a, b) => a.observedAt - b.observedAt || a.name.localeCompare(b.name));
|
|
2092
2125
|
}
|
|
2126
|
+
function extractOutcomesFromPersistedEvents(events) {
|
|
2127
|
+
const out = [];
|
|
2128
|
+
for (const event of events) {
|
|
2129
|
+
const outcome = fromPersistedOutcome(event);
|
|
2130
|
+
if (outcome) out.push(outcome);
|
|
2131
|
+
}
|
|
2132
|
+
return out.sort((a, b) => a.observedAt - b.observedAt || a.name.localeCompare(b.name));
|
|
2133
|
+
}
|
|
2134
|
+
function outcomesMatchingStatus(outcomes, statuses) {
|
|
2135
|
+
const set = new Set(statuses);
|
|
2136
|
+
return outcomes.filter((outcome) => set.has(outcome.status));
|
|
2137
|
+
}
|
|
2093
2138
|
function parseObservationFilter(value) {
|
|
2094
2139
|
if (value === void 0 || value.trim() === "") return void 0;
|
|
2095
2140
|
return parseObservedOutcomeStatus(value);
|
|
@@ -2358,13 +2403,13 @@ function createInspector(options = {}) {
|
|
|
2358
2403
|
run,
|
|
2359
2404
|
step,
|
|
2360
2405
|
tool(name, fn, toolOptions) {
|
|
2361
|
-
const
|
|
2362
|
-
return step(`tool:${
|
|
2406
|
+
const toolName2 = normalizeName2(name, "unknown-tool");
|
|
2407
|
+
return step(`tool:${toolName2}`, fn, {
|
|
2363
2408
|
...toolOptions,
|
|
2364
2409
|
type: "tool",
|
|
2365
2410
|
metadata: {
|
|
2366
2411
|
...toolOptions?.metadata ?? {},
|
|
2367
|
-
toolName
|
|
2412
|
+
toolName: toolName2
|
|
2368
2413
|
}
|
|
2369
2414
|
});
|
|
2370
2415
|
},
|
|
@@ -3234,7 +3279,7 @@ function buildRunSummary(events) {
|
|
|
3234
3279
|
}
|
|
3235
3280
|
let totalSteps = 0;
|
|
3236
3281
|
let llmSteps = 0;
|
|
3237
|
-
let
|
|
3282
|
+
let toolSteps2 = 0;
|
|
3238
3283
|
let logicSteps = 0;
|
|
3239
3284
|
let errorSteps = 0;
|
|
3240
3285
|
let maxDepth = 0;
|
|
@@ -3264,7 +3309,7 @@ function buildRunSummary(events) {
|
|
|
3264
3309
|
for (const [id, s] of steps.entries()) {
|
|
3265
3310
|
totalSteps += 1;
|
|
3266
3311
|
if (s.type === "llm") llmSteps += 1;
|
|
3267
|
-
else if (s.type === "tool")
|
|
3312
|
+
else if (s.type === "tool") toolSteps2 += 1;
|
|
3268
3313
|
else logicSteps += 1;
|
|
3269
3314
|
if (s.status === "error") errorSteps += 1;
|
|
3270
3315
|
const depth = computeDepth(id);
|
|
@@ -3298,7 +3343,7 @@ function buildRunSummary(events) {
|
|
|
3298
3343
|
durationMs: durationMs2,
|
|
3299
3344
|
totalSteps,
|
|
3300
3345
|
llmSteps,
|
|
3301
|
-
toolSteps,
|
|
3346
|
+
toolSteps: toolSteps2,
|
|
3302
3347
|
logicSteps,
|
|
3303
3348
|
errorSteps,
|
|
3304
3349
|
maxDepth,
|
|
@@ -4173,8 +4218,8 @@ function matchStepLevel(m, events, opts) {
|
|
|
4173
4218
|
fields.push("step.name");
|
|
4174
4219
|
}
|
|
4175
4220
|
if (opts.toolQuery) {
|
|
4176
|
-
const
|
|
4177
|
-
if (!nameMatches(
|
|
4221
|
+
const toolName2 = typeof s.metadata?.toolName === "string" ? s.metadata.toolName : s.name;
|
|
4222
|
+
if (!nameMatches(toolName2, opts.toolQuery)) continue;
|
|
4178
4223
|
fields.push("step.tool");
|
|
4179
4224
|
}
|
|
4180
4225
|
if (opts.durationFilter) {
|
|
@@ -4926,12 +4971,12 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
4926
4971
|
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
4927
4972
|
);
|
|
4928
4973
|
const ordered = [...runs].sort(compareRuns);
|
|
4929
|
-
const
|
|
4974
|
+
const path11 = [];
|
|
4930
4975
|
const visited = /* @__PURE__ */ new Set();
|
|
4931
4976
|
const pushRun = (run, confidence, source) => {
|
|
4932
4977
|
if (visited.has(run.runId)) return;
|
|
4933
4978
|
visited.add(run.runId);
|
|
4934
|
-
|
|
4979
|
+
path11.push({
|
|
4935
4980
|
runId: run.runId,
|
|
4936
4981
|
name: run.name,
|
|
4937
4982
|
startedAt: run.startedAt,
|
|
@@ -4956,7 +5001,7 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
4956
5001
|
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
4957
5002
|
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
4958
5003
|
}
|
|
4959
|
-
return
|
|
5004
|
+
return path11;
|
|
4960
5005
|
}
|
|
4961
5006
|
function metaRunIdMatches(run, token, runById) {
|
|
4962
5007
|
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
@@ -5260,6 +5305,3668 @@ function defaultBundleOutputPath(runIds) {
|
|
|
5260
5305
|
return path__default.default.resolve(`agent-inspect-bundle-${label}-${stamp}`);
|
|
5261
5306
|
}
|
|
5262
5307
|
|
|
5308
|
+
// packages/core/src/suite/types.ts
|
|
5309
|
+
var DEFAULT_SUITE_CONFIG_NAMES = [
|
|
5310
|
+
"agent-inspect.suite.json",
|
|
5311
|
+
"agent-inspect.suite.js",
|
|
5312
|
+
"agent-inspect.suite.mjs",
|
|
5313
|
+
"agent-inspect.suite.cjs"
|
|
5314
|
+
];
|
|
5315
|
+
var DEFAULT_SUITE_ARTIFACTS_DIR = ".agent-inspect/suite-runs";
|
|
5316
|
+
function diagnostic(code, message, severity = "error", caseId) {
|
|
5317
|
+
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
5318
|
+
}
|
|
5319
|
+
function asString(value, label) {
|
|
5320
|
+
if (value === void 0) return void 0;
|
|
5321
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
5322
|
+
throw new Error(`${label} must be a non-empty string.`);
|
|
5323
|
+
}
|
|
5324
|
+
return value.trim();
|
|
5325
|
+
}
|
|
5326
|
+
function asStringArray(value, label) {
|
|
5327
|
+
if (value === void 0) return void 0;
|
|
5328
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
5329
|
+
throw new Error(`${label} must be an array of strings.`);
|
|
5330
|
+
}
|
|
5331
|
+
return value;
|
|
5332
|
+
}
|
|
5333
|
+
function asPositiveNumber(value, label) {
|
|
5334
|
+
if (value === void 0) return void 0;
|
|
5335
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
5336
|
+
throw new Error(`${label} must be a non-negative number.`);
|
|
5337
|
+
}
|
|
5338
|
+
return value;
|
|
5339
|
+
}
|
|
5340
|
+
function validateCaseConfig(value, index) {
|
|
5341
|
+
const diagnostics = [];
|
|
5342
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
5343
|
+
diagnostics.push(
|
|
5344
|
+
diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}] must be an object.`)
|
|
5345
|
+
);
|
|
5346
|
+
return { diagnostics };
|
|
5347
|
+
}
|
|
5348
|
+
const raw = value;
|
|
5349
|
+
try {
|
|
5350
|
+
const id = asString(raw.id, `cases[${index}].id`);
|
|
5351
|
+
if (id === void 0) {
|
|
5352
|
+
diagnostics.push(
|
|
5353
|
+
diagnostic("AI_SUITE_CONFIG_INVALID", `cases[${index}].id is required.`)
|
|
5354
|
+
);
|
|
5355
|
+
return { diagnostics };
|
|
5356
|
+
}
|
|
5357
|
+
const trace = asString(raw.trace, `cases[${index}].trace`);
|
|
5358
|
+
const runId = asString(raw.runId, `cases[${index}].runId`);
|
|
5359
|
+
const input = asString(raw.input, `cases[${index}].input`);
|
|
5360
|
+
return {
|
|
5361
|
+
caseConfig: {
|
|
5362
|
+
id,
|
|
5363
|
+
...trace !== void 0 ? { trace } : {},
|
|
5364
|
+
...runId !== void 0 ? { runId } : {},
|
|
5365
|
+
...input !== void 0 ? { input } : {},
|
|
5366
|
+
...asStringArray(raw.requireTools, `cases[${index}].requireTools`) !== void 0 ? { requireTools: asStringArray(raw.requireTools, `cases[${index}].requireTools`) } : {},
|
|
5367
|
+
...asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) !== void 0 ? { forbidTools: asStringArray(raw.forbidTools, `cases[${index}].forbidTools`) } : {},
|
|
5368
|
+
...asPositiveNumber(raw.maxDurationMs, `cases[${index}].maxDurationMs`) !== void 0 ? {
|
|
5369
|
+
maxDurationMs: asPositiveNumber(
|
|
5370
|
+
raw.maxDurationMs,
|
|
5371
|
+
`cases[${index}].maxDurationMs`
|
|
5372
|
+
)
|
|
5373
|
+
} : {},
|
|
5374
|
+
...asStringArray(
|
|
5375
|
+
raw.expectedObservations,
|
|
5376
|
+
`cases[${index}].expectedObservations`
|
|
5377
|
+
) !== void 0 ? {
|
|
5378
|
+
expectedObservations: asStringArray(
|
|
5379
|
+
raw.expectedObservations,
|
|
5380
|
+
`cases[${index}].expectedObservations`
|
|
5381
|
+
)
|
|
5382
|
+
} : {}
|
|
5383
|
+
},
|
|
5384
|
+
diagnostics
|
|
5385
|
+
};
|
|
5386
|
+
} catch (error) {
|
|
5387
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5388
|
+
diagnostics.push(diagnostic("AI_SUITE_CONFIG_INVALID", message));
|
|
5389
|
+
return { diagnostics };
|
|
5390
|
+
}
|
|
5391
|
+
}
|
|
5392
|
+
function normalizeSuiteConfig(value) {
|
|
5393
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
5394
|
+
throw new Error("Suite config must export an object.");
|
|
5395
|
+
}
|
|
5396
|
+
const raw = value;
|
|
5397
|
+
const name = asString(raw.name, "name");
|
|
5398
|
+
const traces = asString(raw.traces, "traces");
|
|
5399
|
+
if (name === void 0) throw new Error("name is required.");
|
|
5400
|
+
if (traces === void 0) throw new Error("traces is required.");
|
|
5401
|
+
if (!Array.isArray(raw.cases) || raw.cases.length === 0) {
|
|
5402
|
+
throw new Error("cases must be a non-empty array.");
|
|
5403
|
+
}
|
|
5404
|
+
const cases = [];
|
|
5405
|
+
for (let index = 0; index < raw.cases.length; index += 1) {
|
|
5406
|
+
const { caseConfig, diagnostics } = validateCaseConfig(raw.cases[index], index);
|
|
5407
|
+
if (diagnostics.length > 0) {
|
|
5408
|
+
throw new Error(diagnostics.map((item) => item.message).join("; "));
|
|
5409
|
+
}
|
|
5410
|
+
if (caseConfig !== void 0) cases.push(caseConfig);
|
|
5411
|
+
}
|
|
5412
|
+
const ids = /* @__PURE__ */ new Set();
|
|
5413
|
+
for (const suiteCase of cases) {
|
|
5414
|
+
if (ids.has(suiteCase.id)) {
|
|
5415
|
+
throw new Error(`Duplicate case id "${suiteCase.id}".`);
|
|
5416
|
+
}
|
|
5417
|
+
ids.add(suiteCase.id);
|
|
5418
|
+
}
|
|
5419
|
+
const redactionProfile = raw.redactionProfile === "local" || raw.redactionProfile === "share" || raw.redactionProfile === "strict" ? raw.redactionProfile : void 0;
|
|
5420
|
+
if (raw.redactionProfile !== void 0 && redactionProfile === void 0) {
|
|
5421
|
+
throw new Error('redactionProfile must be "local", "share", or "strict".');
|
|
5422
|
+
}
|
|
5423
|
+
const config = { name, traces, cases };
|
|
5424
|
+
if (raw.checks !== void 0) {
|
|
5425
|
+
if (typeof raw.checks !== "object" || Array.isArray(raw.checks)) {
|
|
5426
|
+
throw new Error("checks must be an object.");
|
|
5427
|
+
}
|
|
5428
|
+
config.checks = raw.checks;
|
|
5429
|
+
}
|
|
5430
|
+
if (raw.eval !== void 0) {
|
|
5431
|
+
if (typeof raw.eval !== "object" || Array.isArray(raw.eval)) {
|
|
5432
|
+
throw new Error("eval must be an object.");
|
|
5433
|
+
}
|
|
5434
|
+
config.eval = raw.eval;
|
|
5435
|
+
}
|
|
5436
|
+
if (raw.artifacts !== void 0) {
|
|
5437
|
+
if (typeof raw.artifacts !== "object" || Array.isArray(raw.artifacts)) {
|
|
5438
|
+
throw new Error("artifacts must be an object.");
|
|
5439
|
+
}
|
|
5440
|
+
const outputDir = asString(
|
|
5441
|
+
raw.artifacts.outputDir,
|
|
5442
|
+
"artifacts.outputDir"
|
|
5443
|
+
);
|
|
5444
|
+
config.artifacts = outputDir !== void 0 ? { outputDir } : {};
|
|
5445
|
+
}
|
|
5446
|
+
if (raw.baseline !== void 0) {
|
|
5447
|
+
const baseline = asString(raw.baseline, "baseline");
|
|
5448
|
+
if (baseline !== void 0) config.baseline = baseline;
|
|
5449
|
+
}
|
|
5450
|
+
if (raw.candidate !== void 0) {
|
|
5451
|
+
const candidate = asString(raw.candidate, "candidate");
|
|
5452
|
+
if (candidate !== void 0) config.candidate = candidate;
|
|
5453
|
+
}
|
|
5454
|
+
if (redactionProfile !== void 0) config.redactionProfile = redactionProfile;
|
|
5455
|
+
return config;
|
|
5456
|
+
}
|
|
5457
|
+
async function validateSuiteConfig(config, options) {
|
|
5458
|
+
const diagnostics = [];
|
|
5459
|
+
const tracesDir = path__default.default.resolve(options.configDir, config.traces);
|
|
5460
|
+
try {
|
|
5461
|
+
await promises.access(tracesDir);
|
|
5462
|
+
} catch {
|
|
5463
|
+
diagnostics.push(
|
|
5464
|
+
diagnostic("AI_SUITE_CONFIG_INVALID", `traces directory not found: ${tracesDir}`)
|
|
5465
|
+
);
|
|
5466
|
+
}
|
|
5467
|
+
for (const suiteCase of config.cases) {
|
|
5468
|
+
if (suiteCase.input !== void 0) {
|
|
5469
|
+
const inputPath = path__default.default.resolve(options.configDir, suiteCase.input);
|
|
5470
|
+
try {
|
|
5471
|
+
await promises.access(inputPath);
|
|
5472
|
+
} catch {
|
|
5473
|
+
diagnostics.push(
|
|
5474
|
+
diagnostic(
|
|
5475
|
+
"AI_SUITE_CONFIG_INVALID",
|
|
5476
|
+
`input fixture not found: ${suiteCase.input}`,
|
|
5477
|
+
"warning",
|
|
5478
|
+
suiteCase.id
|
|
5479
|
+
)
|
|
5480
|
+
);
|
|
5481
|
+
}
|
|
5482
|
+
}
|
|
5483
|
+
}
|
|
5484
|
+
return { ok: diagnostics.every((item) => item.severity !== "error"), diagnostics };
|
|
5485
|
+
}
|
|
5486
|
+
|
|
5487
|
+
// packages/core/src/suite/load.ts
|
|
5488
|
+
var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
5489
|
+
var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
5490
|
+
function diagnostic2(code, message) {
|
|
5491
|
+
return { code, message, severity: "error" };
|
|
5492
|
+
}
|
|
5493
|
+
async function fileExists(filePath) {
|
|
5494
|
+
try {
|
|
5495
|
+
await promises.access(filePath);
|
|
5496
|
+
return true;
|
|
5497
|
+
} catch {
|
|
5498
|
+
return false;
|
|
5499
|
+
}
|
|
5500
|
+
}
|
|
5501
|
+
async function resolveSuiteConfigPath(options = {}) {
|
|
5502
|
+
const cwd = path__default.default.resolve(options.cwd ?? process.cwd());
|
|
5503
|
+
if (options.configPath !== void 0 && options.configPath.trim() !== "") {
|
|
5504
|
+
return path__default.default.resolve(cwd, options.configPath.trim());
|
|
5505
|
+
}
|
|
5506
|
+
for (const name of DEFAULT_SUITE_CONFIG_NAMES) {
|
|
5507
|
+
const candidate = path__default.default.join(cwd, name);
|
|
5508
|
+
if (await fileExists(candidate)) return candidate;
|
|
5509
|
+
}
|
|
5510
|
+
throw new Error(
|
|
5511
|
+
`No suite config found. Create one with \`agent-inspect suite init\` or pass --config.`
|
|
5512
|
+
);
|
|
5513
|
+
}
|
|
5514
|
+
async function loadSuiteConfig(options = {}) {
|
|
5515
|
+
let configPath;
|
|
5516
|
+
try {
|
|
5517
|
+
configPath = await resolveSuiteConfigPath(options);
|
|
5518
|
+
} catch (error) {
|
|
5519
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5520
|
+
throw Object.assign(new Error(message), {
|
|
5521
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
5522
|
+
});
|
|
5523
|
+
}
|
|
5524
|
+
const extension = path__default.default.extname(configPath);
|
|
5525
|
+
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
5526
|
+
const message = "TypeScript suite configs require an explicit precompiled JavaScript config or future --config-loader support.";
|
|
5527
|
+
throw Object.assign(new Error(message), {
|
|
5528
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
5529
|
+
});
|
|
5530
|
+
}
|
|
5531
|
+
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
5532
|
+
const message = "Unsupported suite config extension. Use .json, .js, .mjs, or .cjs.";
|
|
5533
|
+
throw Object.assign(new Error(message), {
|
|
5534
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
5535
|
+
});
|
|
5536
|
+
}
|
|
5537
|
+
try {
|
|
5538
|
+
let raw;
|
|
5539
|
+
if (extension === ".json") {
|
|
5540
|
+
raw = JSON.parse(await promises.readFile(configPath, "utf-8"));
|
|
5541
|
+
} else {
|
|
5542
|
+
const mod = await import(url.pathToFileURL(configPath).href);
|
|
5543
|
+
raw = "default" in mod ? mod.default : mod;
|
|
5544
|
+
}
|
|
5545
|
+
const config = normalizeSuiteConfig(raw);
|
|
5546
|
+
return {
|
|
5547
|
+
config,
|
|
5548
|
+
configPath,
|
|
5549
|
+
configDir: path__default.default.dirname(configPath)
|
|
5550
|
+
};
|
|
5551
|
+
} catch (error) {
|
|
5552
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5553
|
+
throw Object.assign(new Error(message), {
|
|
5554
|
+
diagnostics: [diagnostic2("AI_SUITE_CONFIG_LOAD_FAILED", message)]
|
|
5555
|
+
});
|
|
5556
|
+
}
|
|
5557
|
+
}
|
|
5558
|
+
function defaultSuiteConfigTemplate() {
|
|
5559
|
+
return {
|
|
5560
|
+
name: "my-agent-suite",
|
|
5561
|
+
traces: "./.agent-inspect",
|
|
5562
|
+
cases: [
|
|
5563
|
+
{
|
|
5564
|
+
id: "example-case",
|
|
5565
|
+
runId: "example-run",
|
|
5566
|
+
requireTools: ["searchDocs"],
|
|
5567
|
+
maxDurationMs: 3e4,
|
|
5568
|
+
expectedObservations: ["answerReady"]
|
|
5569
|
+
}
|
|
5570
|
+
],
|
|
5571
|
+
checks: {
|
|
5572
|
+
select: ["run.status"]
|
|
5573
|
+
},
|
|
5574
|
+
redactionProfile: "local",
|
|
5575
|
+
artifacts: {
|
|
5576
|
+
outputDir: ".agent-inspect/suite-runs"
|
|
5577
|
+
}
|
|
5578
|
+
};
|
|
5579
|
+
}
|
|
5580
|
+
async function exists(filePath) {
|
|
5581
|
+
try {
|
|
5582
|
+
await promises.access(filePath);
|
|
5583
|
+
return true;
|
|
5584
|
+
} catch {
|
|
5585
|
+
return false;
|
|
5586
|
+
}
|
|
5587
|
+
}
|
|
5588
|
+
async function resolveSuiteCaseTrace(suiteCase, options) {
|
|
5589
|
+
if (suiteCase.trace !== void 0) {
|
|
5590
|
+
const tracePath = path__default.default.resolve(options.configDir, suiteCase.trace);
|
|
5591
|
+
if (await exists(tracePath)) {
|
|
5592
|
+
return { caseId: suiteCase.id, tracePath, missing: false };
|
|
5593
|
+
}
|
|
5594
|
+
return {
|
|
5595
|
+
caseId: suiteCase.id,
|
|
5596
|
+
tracePath,
|
|
5597
|
+
missing: true,
|
|
5598
|
+
reason: `trace file not found: ${suiteCase.trace}`
|
|
5599
|
+
};
|
|
5600
|
+
}
|
|
5601
|
+
const runKey = suiteCase.runId ?? suiteCase.id;
|
|
5602
|
+
const directPath = getTraceFilePath(runKey, options.tracesDir);
|
|
5603
|
+
if (await exists(directPath)) {
|
|
5604
|
+
return { caseId: suiteCase.id, tracePath: directPath, runId: runKey, missing: false };
|
|
5605
|
+
}
|
|
5606
|
+
const nestedPath = path__default.default.join(options.tracesDir, `${path__default.default.basename(runKey)}.jsonl`);
|
|
5607
|
+
if (await exists(nestedPath)) {
|
|
5608
|
+
return { caseId: suiteCase.id, tracePath: nestedPath, runId: runKey, missing: false };
|
|
5609
|
+
}
|
|
5610
|
+
return {
|
|
5611
|
+
caseId: suiteCase.id,
|
|
5612
|
+
runId: runKey,
|
|
5613
|
+
missing: true,
|
|
5614
|
+
reason: `no trace found for run id "${runKey}" under ${options.tracesDir}`
|
|
5615
|
+
};
|
|
5616
|
+
}
|
|
5617
|
+
|
|
5618
|
+
// packages/core/src/checks/index.ts
|
|
5619
|
+
var SEVERITY_RANK = {
|
|
5620
|
+
error: 0,
|
|
5621
|
+
warning: 1,
|
|
5622
|
+
info: 2
|
|
5623
|
+
};
|
|
5624
|
+
var STATUS_RANK = {
|
|
5625
|
+
fail: 0,
|
|
5626
|
+
warning: 1,
|
|
5627
|
+
pass: 2
|
|
5628
|
+
};
|
|
5629
|
+
function compareStrings(a, b) {
|
|
5630
|
+
return (a ?? "").localeCompare(b ?? "");
|
|
5631
|
+
}
|
|
5632
|
+
function diagnostic3(code, message, ruleId) {
|
|
5633
|
+
return {
|
|
5634
|
+
code,
|
|
5635
|
+
message,
|
|
5636
|
+
severity: "error",
|
|
5637
|
+
...ruleId ? { ruleId } : {}
|
|
5638
|
+
};
|
|
5639
|
+
}
|
|
5640
|
+
function emptySummary2() {
|
|
5641
|
+
return {
|
|
5642
|
+
passed: 0,
|
|
5643
|
+
failed: 0,
|
|
5644
|
+
warnings: 0,
|
|
5645
|
+
errors: 0
|
|
5646
|
+
};
|
|
5647
|
+
}
|
|
5648
|
+
function errorResult(input, diagnostics, selectedRun) {
|
|
5649
|
+
return {
|
|
5650
|
+
ok: false,
|
|
5651
|
+
status: "error",
|
|
5652
|
+
format: input.read.format,
|
|
5653
|
+
...selectedRun ? { runId: selectedRun.runId } : {},
|
|
5654
|
+
summary: {
|
|
5655
|
+
...emptySummary2(),
|
|
5656
|
+
errors: diagnostics.filter((item) => item.severity === "error").length
|
|
5657
|
+
},
|
|
5658
|
+
findings: [],
|
|
5659
|
+
diagnostics: [...diagnostics]
|
|
5660
|
+
};
|
|
5661
|
+
}
|
|
5662
|
+
function flattenNodes(nodes) {
|
|
5663
|
+
return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
|
|
5664
|
+
}
|
|
5665
|
+
function buildFacts2(input, selectedRun) {
|
|
5666
|
+
const scopedRuns = selectedRun ? [selectedRun] : input.read.runs;
|
|
5667
|
+
const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
|
|
5668
|
+
const scopedEvents = selectedRun === void 0 ? input.read.events : input.read.events.filter((event) => scopedRunIds.has(event.runId));
|
|
5669
|
+
const nodes = flattenNodes(scopedRuns.flatMap((run) => run.children));
|
|
5670
|
+
const nodesByEventId = /* @__PURE__ */ new Map();
|
|
5671
|
+
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
5672
|
+
for (const node of nodes) {
|
|
5673
|
+
nodesByEventId.set(node.event.eventId, node);
|
|
5674
|
+
const parentId = node.event.parentId;
|
|
5675
|
+
if (parentId) {
|
|
5676
|
+
const children = childrenByParentId.get(parentId) ?? [];
|
|
5677
|
+
children.push(node);
|
|
5678
|
+
childrenByParentId.set(parentId, children);
|
|
5679
|
+
}
|
|
5680
|
+
}
|
|
5681
|
+
return {
|
|
5682
|
+
format: input.read.format,
|
|
5683
|
+
runs: Object.freeze([...input.read.runs]),
|
|
5684
|
+
events: Object.freeze([...scopedEvents]),
|
|
5685
|
+
readerWarnings: Object.freeze([...input.read.warnings]),
|
|
5686
|
+
unsupportedFields: Object.freeze([...input.read.unsupportedFields]),
|
|
5687
|
+
sourceFiles: Object.freeze([...input.read.sourceFiles]),
|
|
5688
|
+
nodesByEventId,
|
|
5689
|
+
childrenByParentId,
|
|
5690
|
+
rootNodes: Object.freeze(scopedRuns.flatMap((run) => run.children))
|
|
5691
|
+
};
|
|
5692
|
+
}
|
|
5693
|
+
function resolveSelectedRun(input, runId) {
|
|
5694
|
+
if (input.selectedRun) {
|
|
5695
|
+
if (runId && input.selectedRun.runId !== runId) {
|
|
5696
|
+
return {
|
|
5697
|
+
diagnostics: [
|
|
5698
|
+
diagnostic3(
|
|
5699
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
5700
|
+
`Selected run ${input.selectedRun.runId} does not match requested run ${runId}.`
|
|
5701
|
+
)
|
|
5702
|
+
]
|
|
5703
|
+
};
|
|
5704
|
+
}
|
|
5705
|
+
return { run: input.selectedRun, diagnostics: [] };
|
|
5706
|
+
}
|
|
5707
|
+
if (runId) {
|
|
5708
|
+
const run = input.read.runs.find((candidate) => candidate.runId === runId);
|
|
5709
|
+
if (!run) {
|
|
5710
|
+
return {
|
|
5711
|
+
diagnostics: [
|
|
5712
|
+
diagnostic3("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
5713
|
+
]
|
|
5714
|
+
};
|
|
5715
|
+
}
|
|
5716
|
+
return { run, diagnostics: [] };
|
|
5717
|
+
}
|
|
5718
|
+
if (input.read.runs.length === 1) {
|
|
5719
|
+
return { run: input.read.runs[0], diagnostics: [] };
|
|
5720
|
+
}
|
|
5721
|
+
if (input.read.runs.length === 0) {
|
|
5722
|
+
return {
|
|
5723
|
+
diagnostics: [
|
|
5724
|
+
diagnostic3("AI_CHECK_RUN_SELECTION_REQUIRED", "No runs are available for checks.")
|
|
5725
|
+
]
|
|
5726
|
+
};
|
|
5727
|
+
}
|
|
5728
|
+
return {
|
|
5729
|
+
diagnostics: [
|
|
5730
|
+
diagnostic3(
|
|
5731
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
5732
|
+
"Multiple runs are available; select a run before executing checks."
|
|
5733
|
+
)
|
|
5734
|
+
]
|
|
5735
|
+
};
|
|
5736
|
+
}
|
|
5737
|
+
function selectRules(rules, selectedIds) {
|
|
5738
|
+
const diagnostics = [];
|
|
5739
|
+
const byId = /* @__PURE__ */ new Map();
|
|
5740
|
+
for (const rule of rules) {
|
|
5741
|
+
if (byId.has(rule.id)) {
|
|
5742
|
+
diagnostics.push(
|
|
5743
|
+
diagnostic3("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
5744
|
+
);
|
|
5745
|
+
continue;
|
|
5746
|
+
}
|
|
5747
|
+
byId.set(rule.id, rule);
|
|
5748
|
+
}
|
|
5749
|
+
if (selectedIds && selectedIds.length > 0) {
|
|
5750
|
+
const selected = new Set(selectedIds);
|
|
5751
|
+
for (const id of selected) {
|
|
5752
|
+
if (!byId.has(id)) {
|
|
5753
|
+
diagnostics.push(
|
|
5754
|
+
diagnostic3("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
5755
|
+
);
|
|
5756
|
+
}
|
|
5757
|
+
}
|
|
5758
|
+
return {
|
|
5759
|
+
rules: [...byId.values()].filter((rule) => selected.has(rule.id)).sort(compareRules),
|
|
5760
|
+
diagnostics
|
|
5761
|
+
};
|
|
5762
|
+
}
|
|
5763
|
+
return { rules: [...byId.values()].sort(compareRules), diagnostics };
|
|
5764
|
+
}
|
|
5765
|
+
function compareRules(a, b) {
|
|
5766
|
+
return a.id.localeCompare(b.id);
|
|
5767
|
+
}
|
|
5768
|
+
function eventTimestamp(finding, eventById) {
|
|
5769
|
+
const eventId = finding.evidence[0]?.eventId;
|
|
5770
|
+
return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
|
|
5771
|
+
}
|
|
5772
|
+
function compareFindings(eventById) {
|
|
5773
|
+
return (a, b) => {
|
|
5774
|
+
if (SEVERITY_RANK[a.severity] !== SEVERITY_RANK[b.severity]) {
|
|
5775
|
+
return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity];
|
|
5776
|
+
}
|
|
5777
|
+
const byRule = a.ruleId.localeCompare(b.ruleId);
|
|
5778
|
+
if (byRule !== 0) return byRule;
|
|
5779
|
+
if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
|
|
5780
|
+
return STATUS_RANK[a.status] - STATUS_RANK[b.status];
|
|
5781
|
+
}
|
|
5782
|
+
const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
|
|
5783
|
+
if (byRun !== 0) return byRun;
|
|
5784
|
+
const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
|
|
5785
|
+
if (byTime !== 0) return byTime;
|
|
5786
|
+
const byEvent = compareStrings(a.evidence[0]?.eventId, b.evidence[0]?.eventId);
|
|
5787
|
+
if (byEvent !== 0) return byEvent;
|
|
5788
|
+
return compareStrings(a.evidence[0]?.path, b.evidence[0]?.path);
|
|
5789
|
+
};
|
|
5790
|
+
}
|
|
5791
|
+
function normalizeFinding(rule, finding) {
|
|
5792
|
+
return {
|
|
5793
|
+
ruleId: finding.ruleId || rule.id,
|
|
5794
|
+
severity: finding.severity ?? rule.defaultSeverity,
|
|
5795
|
+
status: finding.status,
|
|
5796
|
+
message: finding.message,
|
|
5797
|
+
...finding.expected !== void 0 ? { expected: finding.expected } : {},
|
|
5798
|
+
...finding.actual !== void 0 ? { actual: finding.actual } : {},
|
|
5799
|
+
evidence: [...finding.evidence ?? []]
|
|
5800
|
+
};
|
|
5801
|
+
}
|
|
5802
|
+
function summarize(findings, diagnostics) {
|
|
5803
|
+
return {
|
|
5804
|
+
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
5805
|
+
failed: findings.filter(
|
|
5806
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
5807
|
+
).length,
|
|
5808
|
+
warnings: findings.filter(
|
|
5809
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
5810
|
+
).length,
|
|
5811
|
+
errors: diagnostics.filter((item) => item.severity === "error").length
|
|
5812
|
+
};
|
|
5813
|
+
}
|
|
5814
|
+
function stringAttr(event, keys) {
|
|
5815
|
+
for (const key of keys) {
|
|
5816
|
+
const value = event.attributes?.[key];
|
|
5817
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
5818
|
+
}
|
|
5819
|
+
return void 0;
|
|
5820
|
+
}
|
|
5821
|
+
function stripPrefix(name, prefixes) {
|
|
5822
|
+
for (const prefix of prefixes) {
|
|
5823
|
+
if (name.startsWith(prefix)) return name.slice(prefix.length);
|
|
5824
|
+
}
|
|
5825
|
+
return name;
|
|
5826
|
+
}
|
|
5827
|
+
function eventEvidence(event, path11) {
|
|
5828
|
+
return {
|
|
5829
|
+
runId: event.runId,
|
|
5830
|
+
eventId: event.eventId,
|
|
5831
|
+
parentId: event.parentId,
|
|
5832
|
+
traceId: event.trace?.traceId,
|
|
5833
|
+
spanId: event.trace?.spanId,
|
|
5834
|
+
kind: event.kind,
|
|
5835
|
+
name: event.name,
|
|
5836
|
+
status: event.status,
|
|
5837
|
+
...path11 ? { path: path11 } : {}
|
|
5838
|
+
};
|
|
5839
|
+
}
|
|
5840
|
+
function runEvidence(run) {
|
|
5841
|
+
return run ? [{ runId: run.runId, name: run.name, status: run.status }] : [];
|
|
5842
|
+
}
|
|
5843
|
+
function failFinding(ruleId, message, evidence, expected, actual) {
|
|
5844
|
+
return {
|
|
5845
|
+
ruleId,
|
|
5846
|
+
severity: "error",
|
|
5847
|
+
status: "fail",
|
|
5848
|
+
message,
|
|
5849
|
+
...expected !== void 0 ? { expected } : {},
|
|
5850
|
+
...actual !== void 0 ? { actual } : {},
|
|
5851
|
+
evidence: [...evidence]
|
|
5852
|
+
};
|
|
5853
|
+
}
|
|
5854
|
+
function toolName(event) {
|
|
5855
|
+
return stringAttr(event, ["toolName", "tool"]) ?? stripPrefix(event.name, ["tool:", "function:", "mcp-tools:"]);
|
|
5856
|
+
}
|
|
5857
|
+
function llmModel(event) {
|
|
5858
|
+
return stringAttr(event, ["model", "modelId", "responseModelId", "modelName", "model_name"]) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
5859
|
+
}
|
|
5860
|
+
function llmProvider(event) {
|
|
5861
|
+
return stringAttr(event, ["provider", "providerName", "provider_name"]);
|
|
5862
|
+
}
|
|
5863
|
+
function llmFinishReason(event) {
|
|
5864
|
+
return stringAttr(event, ["finishReason", "rawFinishReason", "finish_reason"]);
|
|
5865
|
+
}
|
|
5866
|
+
function finishedEvents(context, kind) {
|
|
5867
|
+
return context.events.filter(
|
|
5868
|
+
(event) => (kind === void 0 || event.kind === kind) && event.status !== "running"
|
|
5869
|
+
);
|
|
5870
|
+
}
|
|
5871
|
+
function createRunStatusRule(options = {}) {
|
|
5872
|
+
const expected = options.expected ?? "ok";
|
|
5873
|
+
const allowIncomplete = options.allowIncomplete === true;
|
|
5874
|
+
return {
|
|
5875
|
+
id: "run.status",
|
|
5876
|
+
category: "run",
|
|
5877
|
+
defaultSeverity: "error",
|
|
5878
|
+
evaluate(context) {
|
|
5879
|
+
const findings = [];
|
|
5880
|
+
const actual = context.selectedRun?.status ?? "unknown";
|
|
5881
|
+
if (actual !== expected) {
|
|
5882
|
+
findings.push(
|
|
5883
|
+
failFinding(
|
|
5884
|
+
"run.status",
|
|
5885
|
+
`Run status ${actual} did not match expected ${expected}.`,
|
|
5886
|
+
runEvidence(context.selectedRun),
|
|
5887
|
+
expected,
|
|
5888
|
+
actual
|
|
5889
|
+
)
|
|
5890
|
+
);
|
|
5891
|
+
}
|
|
5892
|
+
if (!allowIncomplete) {
|
|
5893
|
+
const running = context.events.filter((event) => event.status === "running");
|
|
5894
|
+
if (running.length > 0) {
|
|
5895
|
+
findings.push(
|
|
5896
|
+
failFinding(
|
|
5897
|
+
"run.status",
|
|
5898
|
+
"Run contains incomplete running events.",
|
|
5899
|
+
running.map((event) => eventEvidence(event)),
|
|
5900
|
+
"no running events",
|
|
5901
|
+
running.length
|
|
5902
|
+
)
|
|
5903
|
+
);
|
|
5904
|
+
}
|
|
5905
|
+
}
|
|
5906
|
+
return findings;
|
|
5907
|
+
}
|
|
5908
|
+
};
|
|
5909
|
+
}
|
|
5910
|
+
function createRunDurationRule(options) {
|
|
5911
|
+
return {
|
|
5912
|
+
id: "run.duration",
|
|
5913
|
+
category: "run",
|
|
5914
|
+
defaultSeverity: "error",
|
|
5915
|
+
evaluate(context) {
|
|
5916
|
+
const actual = context.selectedRun?.durationMs;
|
|
5917
|
+
if (actual === void 0 || actual <= options.maxDurationMs) return [];
|
|
5918
|
+
return [
|
|
5919
|
+
failFinding(
|
|
5920
|
+
"run.duration",
|
|
5921
|
+
`Run duration ${actual}ms exceeded ${options.maxDurationMs}ms.`,
|
|
5922
|
+
runEvidence(context.selectedRun),
|
|
5923
|
+
{ maxDurationMs: options.maxDurationMs },
|
|
5924
|
+
actual
|
|
5925
|
+
)
|
|
5926
|
+
];
|
|
5927
|
+
}
|
|
5928
|
+
};
|
|
5929
|
+
}
|
|
5930
|
+
function createToolUsageRule(options) {
|
|
5931
|
+
return {
|
|
5932
|
+
id: "tool.usage",
|
|
5933
|
+
category: "tool",
|
|
5934
|
+
defaultSeverity: "error",
|
|
5935
|
+
evaluate(context) {
|
|
5936
|
+
const tools = finishedEvents(context, "TOOL");
|
|
5937
|
+
const names = tools.map(toolName);
|
|
5938
|
+
const nameSet = new Set(names);
|
|
5939
|
+
const findings = [];
|
|
5940
|
+
for (const required of options.required ?? []) {
|
|
5941
|
+
if (!nameSet.has(required)) {
|
|
5942
|
+
findings.push(
|
|
5943
|
+
failFinding("tool.usage", `Required tool ${required} did not appear.`, runEvidence(context.selectedRun), required, names)
|
|
5944
|
+
);
|
|
5945
|
+
}
|
|
5946
|
+
}
|
|
5947
|
+
const forbidden = new Set(options.forbidden ?? []);
|
|
5948
|
+
const allowed = options.allowed ? new Set(options.allowed) : void 0;
|
|
5949
|
+
for (const event of tools) {
|
|
5950
|
+
const name = toolName(event);
|
|
5951
|
+
if (forbidden.has(name)) {
|
|
5952
|
+
findings.push(
|
|
5953
|
+
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence(event)], "tool absent", name)
|
|
5954
|
+
);
|
|
5955
|
+
}
|
|
5956
|
+
if (allowed && !allowed.has(name)) {
|
|
5957
|
+
findings.push(
|
|
5958
|
+
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence(event)], [...allowed].sort(), name)
|
|
5959
|
+
);
|
|
5960
|
+
}
|
|
5961
|
+
}
|
|
5962
|
+
if (options.minCount !== void 0 && tools.length < options.minCount) {
|
|
5963
|
+
findings.push(
|
|
5964
|
+
failFinding("tool.usage", `Tool count ${tools.length} was below minimum ${options.minCount}.`, runEvidence(context.selectedRun), { minCount: options.minCount }, tools.length)
|
|
5965
|
+
);
|
|
5966
|
+
}
|
|
5967
|
+
if (options.maxCount !== void 0 && tools.length > options.maxCount) {
|
|
5968
|
+
findings.push(
|
|
5969
|
+
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) => eventEvidence(event)), { maxCount: options.maxCount }, tools.length)
|
|
5970
|
+
);
|
|
5971
|
+
}
|
|
5972
|
+
return findings;
|
|
5973
|
+
}
|
|
5974
|
+
};
|
|
5975
|
+
}
|
|
5976
|
+
function createLlmUsageRule(options) {
|
|
5977
|
+
return {
|
|
5978
|
+
id: "llm.usage",
|
|
5979
|
+
category: "llm",
|
|
5980
|
+
defaultSeverity: "error",
|
|
5981
|
+
evaluate(context) {
|
|
5982
|
+
const llms = finishedEvents(context, "LLM");
|
|
5983
|
+
const findings = [];
|
|
5984
|
+
const allowedModels = options.allowedModels ? new Set(options.allowedModels) : void 0;
|
|
5985
|
+
const allowedProviders = options.allowedProviders ? new Set(options.allowedProviders) : void 0;
|
|
5986
|
+
const finishReasons = options.finishReasons ? new Set(options.finishReasons) : void 0;
|
|
5987
|
+
if (options.maxCalls !== void 0 && llms.length > options.maxCalls) {
|
|
5988
|
+
findings.push(
|
|
5989
|
+
failFinding(
|
|
5990
|
+
"llm.usage",
|
|
5991
|
+
`LLM call count ${llms.length} exceeded ${options.maxCalls}.`,
|
|
5992
|
+
llms.map((event) => eventEvidence(event)),
|
|
5993
|
+
{ maxCalls: options.maxCalls },
|
|
5994
|
+
llms.length
|
|
5995
|
+
)
|
|
5996
|
+
);
|
|
5997
|
+
}
|
|
5998
|
+
for (const event of llms) {
|
|
5999
|
+
const model = llmModel(event);
|
|
6000
|
+
const provider = llmProvider(event);
|
|
6001
|
+
const finishReason = llmFinishReason(event);
|
|
6002
|
+
if (allowedModels && (!model || !allowedModels.has(model))) {
|
|
6003
|
+
findings.push(
|
|
6004
|
+
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.model")], [...allowedModels].sort(), model ?? "unknown")
|
|
6005
|
+
);
|
|
6006
|
+
}
|
|
6007
|
+
if (allowedProviders && (!provider || !allowedProviders.has(provider))) {
|
|
6008
|
+
findings.push(
|
|
6009
|
+
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.provider")], [...allowedProviders].sort(), provider ?? "unknown")
|
|
6010
|
+
);
|
|
6011
|
+
}
|
|
6012
|
+
if (finishReasons && (!finishReason || !finishReasons.has(finishReason))) {
|
|
6013
|
+
findings.push(
|
|
6014
|
+
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.finishReason")], [...finishReasons].sort(), finishReason ?? "unknown")
|
|
6015
|
+
);
|
|
6016
|
+
}
|
|
6017
|
+
}
|
|
6018
|
+
const tokenTotals = llms.reduce(
|
|
6019
|
+
(totals, event) => ({
|
|
6020
|
+
input: totals.input + (event.tokenUsage?.input ?? 0),
|
|
6021
|
+
output: totals.output + (event.tokenUsage?.output ?? 0),
|
|
6022
|
+
total: totals.total + (event.tokenUsage?.total ?? 0),
|
|
6023
|
+
cached: totals.cached + (event.tokenUsage?.cached ?? 0)
|
|
6024
|
+
}),
|
|
6025
|
+
{ input: 0, output: 0, total: 0, cached: 0 }
|
|
6026
|
+
);
|
|
6027
|
+
const tokenLimits = [
|
|
6028
|
+
["input", options.maxInputTokens],
|
|
6029
|
+
["output", options.maxOutputTokens],
|
|
6030
|
+
["total", options.maxTotalTokens],
|
|
6031
|
+
["cached", options.maxCachedTokens]
|
|
6032
|
+
];
|
|
6033
|
+
for (const [key, limit] of tokenLimits) {
|
|
6034
|
+
if (limit !== void 0 && tokenTotals[key] > limit) {
|
|
6035
|
+
findings.push(
|
|
6036
|
+
failFinding(
|
|
6037
|
+
"llm.usage",
|
|
6038
|
+
`LLM ${key} token count ${tokenTotals[key]} exceeded ${limit}.`,
|
|
6039
|
+
llms.map((event) => eventEvidence(event, `tokenUsage.${key}`)),
|
|
6040
|
+
{ [`max${key[0].toUpperCase()}${key.slice(1)}Tokens`]: limit },
|
|
6041
|
+
tokenTotals[key]
|
|
6042
|
+
)
|
|
6043
|
+
);
|
|
6044
|
+
}
|
|
6045
|
+
}
|
|
6046
|
+
return findings;
|
|
6047
|
+
}
|
|
6048
|
+
};
|
|
6049
|
+
}
|
|
6050
|
+
function createObservedOutcomeRule(options = {}) {
|
|
6051
|
+
const failOn = options.failOn ?? ["failed"];
|
|
6052
|
+
return {
|
|
6053
|
+
id: "outcome.status",
|
|
6054
|
+
category: "run",
|
|
6055
|
+
defaultSeverity: "error",
|
|
6056
|
+
evaluate(context) {
|
|
6057
|
+
const outcomes = extractOutcomesFromPersistedEvents(context.events);
|
|
6058
|
+
const matching = outcomesMatchingStatus(outcomes, failOn);
|
|
6059
|
+
if (matching.length === 0) return [];
|
|
6060
|
+
return [
|
|
6061
|
+
failFinding(
|
|
6062
|
+
"outcome.status",
|
|
6063
|
+
`Observed outcome count ${matching.length} matched [${failOn.join(", ")}].`,
|
|
6064
|
+
matching.map((outcome) => ({
|
|
6065
|
+
runId: outcome.runId,
|
|
6066
|
+
eventId: outcome.outcomeId,
|
|
6067
|
+
...outcome.parentId !== void 0 ? { parentId: outcome.parentId } : {},
|
|
6068
|
+
kind: "OUTCOME",
|
|
6069
|
+
name: outcome.name,
|
|
6070
|
+
status: outcome.status,
|
|
6071
|
+
path: `outcome.${outcome.name}`
|
|
6072
|
+
})),
|
|
6073
|
+
{ failOn },
|
|
6074
|
+
matching.map((outcome) => ({
|
|
6075
|
+
name: outcome.name,
|
|
6076
|
+
status: outcome.status,
|
|
6077
|
+
expectation: outcome.expectation
|
|
6078
|
+
}))
|
|
6079
|
+
)
|
|
6080
|
+
];
|
|
6081
|
+
}
|
|
6082
|
+
};
|
|
6083
|
+
}
|
|
6084
|
+
function runTraceChecks(input, options = {}) {
|
|
6085
|
+
const selected = resolveSelectedRun(input, options.runId);
|
|
6086
|
+
if (selected.diagnostics.length > 0) {
|
|
6087
|
+
return errorResult(input, selected.diagnostics, selected.run);
|
|
6088
|
+
}
|
|
6089
|
+
const rules = selectRules(options.rules ?? [], options.select);
|
|
6090
|
+
if (rules.diagnostics.length > 0) {
|
|
6091
|
+
return errorResult(input, rules.diagnostics, selected.run);
|
|
6092
|
+
}
|
|
6093
|
+
const facts = buildFacts2(input, selected.run);
|
|
6094
|
+
const context = {
|
|
6095
|
+
...facts,
|
|
6096
|
+
...selected.run ? { selectedRun: selected.run } : {},
|
|
6097
|
+
...input.sourceLabel ? { sourceLabel: input.sourceLabel } : {}
|
|
6098
|
+
};
|
|
6099
|
+
const diagnostics = [];
|
|
6100
|
+
const findings = [];
|
|
6101
|
+
for (const rule of rules.rules) {
|
|
6102
|
+
try {
|
|
6103
|
+
findings.push(...rule.evaluate(context).map((finding) => normalizeFinding(rule, finding)));
|
|
6104
|
+
} catch (error) {
|
|
6105
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
6106
|
+
diagnostics.push(
|
|
6107
|
+
diagnostic3("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
|
|
6108
|
+
);
|
|
6109
|
+
}
|
|
6110
|
+
}
|
|
6111
|
+
if (diagnostics.length > 0) {
|
|
6112
|
+
return errorResult(input, diagnostics, selected.run);
|
|
6113
|
+
}
|
|
6114
|
+
const eventById = new Map(input.read.events.map((event) => [event.eventId, event]));
|
|
6115
|
+
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
6116
|
+
const summary = summarize(sortedFindings, diagnostics);
|
|
6117
|
+
const status = summary.failed > 0 ? "fail" : "pass";
|
|
6118
|
+
return {
|
|
6119
|
+
ok: status === "pass",
|
|
6120
|
+
status,
|
|
6121
|
+
format: input.read.format,
|
|
6122
|
+
...selected.run ? { runId: selected.run.runId } : {},
|
|
6123
|
+
summary,
|
|
6124
|
+
findings: sortedFindings,
|
|
6125
|
+
diagnostics
|
|
6126
|
+
};
|
|
6127
|
+
}
|
|
6128
|
+
|
|
6129
|
+
// packages/core/src/persisted/token-usage.ts
|
|
6130
|
+
function isRecord8(value) {
|
|
6131
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
6132
|
+
}
|
|
6133
|
+
function nonNegativeFinite(value) {
|
|
6134
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
|
|
6135
|
+
}
|
|
6136
|
+
function normalizeTokenUsage(value) {
|
|
6137
|
+
if (!isRecord8(value)) return void 0;
|
|
6138
|
+
const input = nonNegativeFinite(value.input);
|
|
6139
|
+
const output = nonNegativeFinite(value.output);
|
|
6140
|
+
const suppliedTotal = nonNegativeFinite(value.total);
|
|
6141
|
+
const cached = nonNegativeFinite(value.cached);
|
|
6142
|
+
const derivedTotal = input !== void 0 && output !== void 0 && Number.isFinite(input + output) ? input + output : void 0;
|
|
6143
|
+
const total = suppliedTotal ?? derivedTotal;
|
|
6144
|
+
if (input === void 0 && output === void 0 && total === void 0 && cached === void 0) {
|
|
6145
|
+
return void 0;
|
|
6146
|
+
}
|
|
6147
|
+
return {
|
|
6148
|
+
...input !== void 0 ? { input } : {},
|
|
6149
|
+
...output !== void 0 ? { output } : {},
|
|
6150
|
+
...total !== void 0 ? { total } : {},
|
|
6151
|
+
...cached !== void 0 ? { cached } : {}
|
|
6152
|
+
};
|
|
6153
|
+
}
|
|
6154
|
+
|
|
6155
|
+
// packages/core/src/persisted/from-trace-event.ts
|
|
6156
|
+
function sanitizeIdPart(value) {
|
|
6157
|
+
return value.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
6158
|
+
}
|
|
6159
|
+
function nodeIdForEvent(event) {
|
|
6160
|
+
switch (event.event) {
|
|
6161
|
+
case "run_started":
|
|
6162
|
+
case "run_completed":
|
|
6163
|
+
return event.runId;
|
|
6164
|
+
case "step_started":
|
|
6165
|
+
case "step_completed":
|
|
6166
|
+
return event.stepId;
|
|
6167
|
+
case "outcome_observed":
|
|
6168
|
+
return event.outcomeId;
|
|
6169
|
+
default:
|
|
6170
|
+
return "unknown";
|
|
6171
|
+
}
|
|
6172
|
+
}
|
|
6173
|
+
function createPersistedEventId(event, eventIndex) {
|
|
6174
|
+
const runId = sanitizeIdPart(event.runId);
|
|
6175
|
+
const ev = sanitizeIdPart(event.event);
|
|
6176
|
+
const node = sanitizeIdPart(nodeIdForEvent(event));
|
|
6177
|
+
return `manual:${runId}:${ev}:${node}:${eventIndex}`;
|
|
6178
|
+
}
|
|
6179
|
+
function toIsoTimestamp(ms) {
|
|
6180
|
+
if (typeof ms !== "number" || !Number.isFinite(ms)) {
|
|
6181
|
+
return { iso: (/* @__PURE__ */ new Date(0)).toISOString(), invalidTimestamp: true };
|
|
6182
|
+
}
|
|
6183
|
+
return { iso: new Date(ms).toISOString(), invalidTimestamp: false };
|
|
6184
|
+
}
|
|
6185
|
+
function buildSource(options) {
|
|
6186
|
+
return {
|
|
6187
|
+
type: "manual",
|
|
6188
|
+
name: options?.sourceName ?? "trace-event",
|
|
6189
|
+
version: options?.sourceVersion ?? "0.1"
|
|
6190
|
+
};
|
|
6191
|
+
}
|
|
6192
|
+
function mapStepTypeToInspectKind(type) {
|
|
6193
|
+
switch (type) {
|
|
6194
|
+
case "run":
|
|
6195
|
+
return "RUN";
|
|
6196
|
+
case "llm":
|
|
6197
|
+
return "LLM";
|
|
6198
|
+
case "tool":
|
|
6199
|
+
return "TOOL";
|
|
6200
|
+
case "decision":
|
|
6201
|
+
return "DECISION";
|
|
6202
|
+
case "logic":
|
|
6203
|
+
case "state":
|
|
6204
|
+
case "custom":
|
|
6205
|
+
return "LOGIC";
|
|
6206
|
+
default:
|
|
6207
|
+
return "LOGIC";
|
|
6208
|
+
}
|
|
6209
|
+
}
|
|
6210
|
+
function mapRunOrStepStatus(status) {
|
|
6211
|
+
return status === "success" ? "ok" : "error";
|
|
6212
|
+
}
|
|
6213
|
+
function mapErrorInfo(error) {
|
|
6214
|
+
if (!error?.message) {
|
|
6215
|
+
return {};
|
|
6216
|
+
}
|
|
6217
|
+
const out = {
|
|
6218
|
+
persisted: {
|
|
6219
|
+
message: error.message,
|
|
6220
|
+
name: "Error"
|
|
6221
|
+
}
|
|
6222
|
+
};
|
|
6223
|
+
if (typeof error.stack === "string" && error.stack.length > 0) {
|
|
6224
|
+
out.errorStack = error.stack;
|
|
6225
|
+
}
|
|
6226
|
+
return out;
|
|
6227
|
+
}
|
|
6228
|
+
function mapTokenUsageFromMetadata(metadata) {
|
|
6229
|
+
return normalizeTokenUsage(metadata?.tokens);
|
|
6230
|
+
}
|
|
6231
|
+
function compactAttributes(entries) {
|
|
6232
|
+
const out = {};
|
|
6233
|
+
for (const [key, value] of Object.entries(entries)) {
|
|
6234
|
+
if (value !== void 0) {
|
|
6235
|
+
out[key] = value;
|
|
6236
|
+
}
|
|
6237
|
+
}
|
|
6238
|
+
return Object.keys(out).length > 0 ? out : void 0;
|
|
6239
|
+
}
|
|
6240
|
+
function traceEventToPersistedInspectEvent(event, options) {
|
|
6241
|
+
const eventIndex = options?.eventIndex ?? 0;
|
|
6242
|
+
const eventId = createPersistedEventId(event, eventIndex);
|
|
6243
|
+
const source = buildSource(options);
|
|
6244
|
+
const tsMain = toIsoTimestamp(event.timestamp);
|
|
6245
|
+
switch (event.event) {
|
|
6246
|
+
case "run_started": {
|
|
6247
|
+
const tsStart = toIsoTimestamp(event.startTime);
|
|
6248
|
+
const correlation = extractCorrelationMetadata(event.metadata);
|
|
6249
|
+
const attributes = compactAttributes({
|
|
6250
|
+
legacyEvent: "run_started",
|
|
6251
|
+
metadata: event.metadata !== void 0 ? { ...event.metadata } : void 0,
|
|
6252
|
+
correlationId: correlation?.correlationId,
|
|
6253
|
+
requestId: correlation?.requestId,
|
|
6254
|
+
decisionId: correlation?.decisionId,
|
|
6255
|
+
groupId: correlation?.groupId,
|
|
6256
|
+
invalidTimestamp: tsMain.invalidTimestamp || tsStart.invalidTimestamp ? true : void 0
|
|
6257
|
+
});
|
|
6258
|
+
return {
|
|
6259
|
+
schemaVersion: "0.2",
|
|
6260
|
+
eventId,
|
|
6261
|
+
runId: event.runId,
|
|
6262
|
+
kind: "RUN",
|
|
6263
|
+
name: event.name,
|
|
6264
|
+
status: "running",
|
|
6265
|
+
timestamp: tsMain.iso,
|
|
6266
|
+
startedAt: tsStart.iso,
|
|
6267
|
+
confidence: "explicit",
|
|
6268
|
+
source,
|
|
6269
|
+
attributes
|
|
6270
|
+
};
|
|
6271
|
+
}
|
|
6272
|
+
case "run_completed": {
|
|
6273
|
+
const tsEnd = toIsoTimestamp(event.endTime);
|
|
6274
|
+
const { persisted: error, errorStack } = mapErrorInfo(event.error);
|
|
6275
|
+
const attributes = compactAttributes({
|
|
6276
|
+
legacyEvent: "run_completed",
|
|
6277
|
+
errorStack,
|
|
6278
|
+
invalidTimestamp: tsMain.invalidTimestamp || tsEnd.invalidTimestamp ? true : void 0
|
|
6279
|
+
});
|
|
6280
|
+
return {
|
|
6281
|
+
schemaVersion: "0.2",
|
|
6282
|
+
eventId,
|
|
6283
|
+
runId: event.runId,
|
|
6284
|
+
kind: "RUN",
|
|
6285
|
+
name: "run",
|
|
6286
|
+
status: mapRunOrStepStatus(event.status),
|
|
6287
|
+
timestamp: tsMain.iso,
|
|
6288
|
+
endedAt: tsEnd.iso,
|
|
6289
|
+
durationMs: event.durationMs,
|
|
6290
|
+
confidence: "explicit",
|
|
6291
|
+
source,
|
|
6292
|
+
attributes,
|
|
6293
|
+
error
|
|
6294
|
+
};
|
|
6295
|
+
}
|
|
6296
|
+
case "step_started": {
|
|
6297
|
+
const tsStart = toIsoTimestamp(event.startTime);
|
|
6298
|
+
const tokenUsage = mapTokenUsageFromMetadata(event.metadata);
|
|
6299
|
+
const attributes = compactAttributes({
|
|
6300
|
+
legacyEvent: "step_started",
|
|
6301
|
+
stepId: event.stepId,
|
|
6302
|
+
stepType: event.type,
|
|
6303
|
+
metadata: event.metadata !== void 0 ? { ...event.metadata } : void 0,
|
|
6304
|
+
invalidTimestamp: tsMain.invalidTimestamp || tsStart.invalidTimestamp ? true : void 0
|
|
6305
|
+
});
|
|
6306
|
+
const out = {
|
|
6307
|
+
schemaVersion: "0.2",
|
|
6308
|
+
eventId,
|
|
6309
|
+
runId: event.runId,
|
|
6310
|
+
kind: mapStepTypeToInspectKind(event.type),
|
|
6311
|
+
name: event.name,
|
|
6312
|
+
status: "running",
|
|
6313
|
+
timestamp: tsMain.iso,
|
|
6314
|
+
startedAt: tsStart.iso,
|
|
6315
|
+
confidence: "explicit",
|
|
6316
|
+
source,
|
|
6317
|
+
attributes
|
|
6318
|
+
};
|
|
6319
|
+
if (event.parentId !== void 0) {
|
|
6320
|
+
out.parentId = event.parentId;
|
|
6321
|
+
}
|
|
6322
|
+
if (tokenUsage !== void 0) {
|
|
6323
|
+
out.tokenUsage = tokenUsage;
|
|
6324
|
+
}
|
|
6325
|
+
return out;
|
|
6326
|
+
}
|
|
6327
|
+
case "step_completed": {
|
|
6328
|
+
const tsEnd = toIsoTimestamp(event.endTime);
|
|
6329
|
+
const { persisted: error, errorStack } = mapErrorInfo(event.error);
|
|
6330
|
+
const attributes = compactAttributes({
|
|
6331
|
+
legacyEvent: "step_completed",
|
|
6332
|
+
stepId: event.stepId,
|
|
6333
|
+
errorStack,
|
|
6334
|
+
invalidTimestamp: tsMain.invalidTimestamp || tsEnd.invalidTimestamp ? true : void 0
|
|
6335
|
+
});
|
|
6336
|
+
return {
|
|
6337
|
+
schemaVersion: "0.2",
|
|
6338
|
+
eventId,
|
|
6339
|
+
runId: event.runId,
|
|
6340
|
+
kind: "LOGIC",
|
|
6341
|
+
name: event.stepId,
|
|
6342
|
+
status: mapRunOrStepStatus(event.status),
|
|
6343
|
+
timestamp: tsMain.iso,
|
|
6344
|
+
endedAt: tsEnd.iso,
|
|
6345
|
+
durationMs: event.durationMs,
|
|
6346
|
+
confidence: "explicit",
|
|
6347
|
+
source,
|
|
6348
|
+
attributes,
|
|
6349
|
+
error
|
|
6350
|
+
};
|
|
6351
|
+
}
|
|
6352
|
+
case "outcome_observed": {
|
|
6353
|
+
const tsObserved = toIsoTimestamp(event.observedAt);
|
|
6354
|
+
const attributes = compactAttributes({
|
|
6355
|
+
legacyEvent: "outcome_observed",
|
|
6356
|
+
outcomeId: event.outcomeId,
|
|
6357
|
+
outcomeStatus: event.status,
|
|
6358
|
+
expectation: event.expectation,
|
|
6359
|
+
method: event.method,
|
|
6360
|
+
actual: event.actual,
|
|
6361
|
+
evidence: event.evidence,
|
|
6362
|
+
observedAt: tsObserved.iso,
|
|
6363
|
+
invalidTimestamp: tsMain.invalidTimestamp || tsObserved.invalidTimestamp ? true : void 0
|
|
6364
|
+
});
|
|
6365
|
+
const out = {
|
|
6366
|
+
schemaVersion: "0.2",
|
|
6367
|
+
eventId,
|
|
6368
|
+
runId: event.runId,
|
|
6369
|
+
kind: "OUTCOME",
|
|
6370
|
+
name: event.name,
|
|
6371
|
+
status: event.status === "failed" ? "error" : "ok",
|
|
6372
|
+
timestamp: tsMain.iso,
|
|
6373
|
+
confidence: "explicit",
|
|
6374
|
+
source,
|
|
6375
|
+
attributes
|
|
6376
|
+
};
|
|
6377
|
+
if (event.parentId !== void 0) {
|
|
6378
|
+
out.parentId = event.parentId;
|
|
6379
|
+
}
|
|
6380
|
+
if (event.actual !== void 0) {
|
|
6381
|
+
out.outputSummary = event.actual;
|
|
6382
|
+
}
|
|
6383
|
+
return out;
|
|
6384
|
+
}
|
|
6385
|
+
default: {
|
|
6386
|
+
const _exhaustive = event;
|
|
6387
|
+
throw new Error(`Unsupported trace event: ${_exhaustive.event}`);
|
|
6388
|
+
}
|
|
6389
|
+
}
|
|
6390
|
+
}
|
|
6391
|
+
function traceEventsToPersistedInspectEvents(events, options) {
|
|
6392
|
+
return events.map(
|
|
6393
|
+
(event, index) => traceEventToPersistedInspectEvent(event, { ...options, eventIndex: index })
|
|
6394
|
+
);
|
|
6395
|
+
}
|
|
6396
|
+
|
|
6397
|
+
// packages/core/src/logs/tree-builder.ts
|
|
6398
|
+
function inc(map, key) {
|
|
6399
|
+
map[key] = (map[key] ?? 0) + 1;
|
|
6400
|
+
}
|
|
6401
|
+
function computeRunStatus(events) {
|
|
6402
|
+
let hasRunning = false;
|
|
6403
|
+
for (const e of events) {
|
|
6404
|
+
if (e.status === "error") return "error";
|
|
6405
|
+
if (e.status === "running") hasRunning = true;
|
|
6406
|
+
}
|
|
6407
|
+
if (hasRunning) return "running";
|
|
6408
|
+
return "ok";
|
|
6409
|
+
}
|
|
6410
|
+
var TreeBuilder = class {
|
|
6411
|
+
constructor(options) {
|
|
6412
|
+
void options?.config;
|
|
6413
|
+
}
|
|
6414
|
+
build(events) {
|
|
6415
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
6416
|
+
for (const e of events) {
|
|
6417
|
+
if (!byRun.has(e.runId)) byRun.set(e.runId, []);
|
|
6418
|
+
byRun.get(e.runId).push(e);
|
|
6419
|
+
}
|
|
6420
|
+
const out = [];
|
|
6421
|
+
for (const [runId, runEvents] of byRun.entries()) {
|
|
6422
|
+
const sorted = [...runEvents].sort((a, b) => a.timestamp - b.timestamp);
|
|
6423
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
6424
|
+
for (const e of sorted) {
|
|
6425
|
+
nodes.set(e.eventId, { event: e, children: [], depth: 0 });
|
|
6426
|
+
}
|
|
6427
|
+
const roots = [];
|
|
6428
|
+
for (const node of nodes.values()) {
|
|
6429
|
+
const parentId = node.event.parentId;
|
|
6430
|
+
if (parentId && nodes.has(parentId)) {
|
|
6431
|
+
nodes.get(parentId).children.push(node);
|
|
6432
|
+
} else {
|
|
6433
|
+
roots.push(node);
|
|
6434
|
+
}
|
|
6435
|
+
}
|
|
6436
|
+
const assignDepth = (n, depth) => {
|
|
6437
|
+
n.depth = depth;
|
|
6438
|
+
for (const c of n.children) assignDepth(c, depth + 1);
|
|
6439
|
+
};
|
|
6440
|
+
for (const r of roots) assignDepth(r, 0);
|
|
6441
|
+
const confidenceBreakdown = {
|
|
6442
|
+
explicit: 0,
|
|
6443
|
+
correlated: 0,
|
|
6444
|
+
heuristic: 0,
|
|
6445
|
+
unknown: 0
|
|
6446
|
+
};
|
|
6447
|
+
const kinds = {};
|
|
6448
|
+
for (const e of sorted) {
|
|
6449
|
+
inc(confidenceBreakdown, e.confidence);
|
|
6450
|
+
kinds[e.kind] = (kinds[e.kind] ?? 0) + 1;
|
|
6451
|
+
}
|
|
6452
|
+
const startedAt = sorted.length > 0 ? sorted[0].timestamp : void 0;
|
|
6453
|
+
const endedAt = sorted.length > 0 ? sorted[sorted.length - 1].timestamp : void 0;
|
|
6454
|
+
const status = computeRunStatus(sorted);
|
|
6455
|
+
const durationMs2 = startedAt !== void 0 && endedAt !== void 0 && Number.isFinite(startedAt) && Number.isFinite(endedAt) && endedAt >= startedAt && status !== "running" ? endedAt - startedAt : void 0;
|
|
6456
|
+
const name = sorted.find((e) => e.kind === "RUN")?.name;
|
|
6457
|
+
out.push({
|
|
6458
|
+
runId,
|
|
6459
|
+
name,
|
|
6460
|
+
status,
|
|
6461
|
+
startedAt,
|
|
6462
|
+
endedAt: status === "running" ? void 0 : endedAt,
|
|
6463
|
+
durationMs: durationMs2,
|
|
6464
|
+
children: roots,
|
|
6465
|
+
metadata: {
|
|
6466
|
+
totalEvents: sorted.length,
|
|
6467
|
+
confidenceBreakdown,
|
|
6468
|
+
kinds
|
|
6469
|
+
}
|
|
6470
|
+
});
|
|
6471
|
+
}
|
|
6472
|
+
out.sort((a, b) => (b.startedAt ?? 0) - (a.startedAt ?? 0));
|
|
6473
|
+
return out;
|
|
6474
|
+
}
|
|
6475
|
+
};
|
|
6476
|
+
|
|
6477
|
+
// packages/core/src/persisted/to-inspect-event.ts
|
|
6478
|
+
function compactAttributes2(entries) {
|
|
6479
|
+
const out = {};
|
|
6480
|
+
for (const [key, value] of Object.entries(entries)) {
|
|
6481
|
+
if (value !== void 0) {
|
|
6482
|
+
out[key] = value;
|
|
6483
|
+
}
|
|
6484
|
+
}
|
|
6485
|
+
return Object.keys(out).length > 0 ? out : void 0;
|
|
6486
|
+
}
|
|
6487
|
+
function parseIsoToMs3(iso) {
|
|
6488
|
+
const parsed = Date.parse(iso);
|
|
6489
|
+
if (!Number.isFinite(parsed)) {
|
|
6490
|
+
return { ms: 0, invalidTimestamp: true };
|
|
6491
|
+
}
|
|
6492
|
+
return { ms: parsed, invalidTimestamp: false };
|
|
6493
|
+
}
|
|
6494
|
+
function mapPersistedSourceToInspect(event) {
|
|
6495
|
+
const attrs = event.attributes ?? {};
|
|
6496
|
+
const sourceName = event.source.name;
|
|
6497
|
+
if (sourceName === "pino") {
|
|
6498
|
+
return {
|
|
6499
|
+
type: "pino",
|
|
6500
|
+
file: typeof attrs.sourceFile === "string" ? attrs.sourceFile : void 0,
|
|
6501
|
+
line: typeof attrs.sourceLine === "number" ? attrs.sourceLine : void 0
|
|
6502
|
+
};
|
|
6503
|
+
}
|
|
6504
|
+
if (sourceName === "winston") {
|
|
6505
|
+
return {
|
|
6506
|
+
type: "winston",
|
|
6507
|
+
file: typeof attrs.sourceFile === "string" ? attrs.sourceFile : void 0,
|
|
6508
|
+
line: typeof attrs.sourceLine === "number" ? attrs.sourceLine : void 0
|
|
6509
|
+
};
|
|
6510
|
+
}
|
|
6511
|
+
const mapType = (t) => {
|
|
6512
|
+
switch (t) {
|
|
6513
|
+
case "manual":
|
|
6514
|
+
return "manual";
|
|
6515
|
+
case "json-log":
|
|
6516
|
+
return "json-log";
|
|
6517
|
+
case "log4js":
|
|
6518
|
+
return "log4js";
|
|
6519
|
+
case "adapter":
|
|
6520
|
+
case "ai-sdk":
|
|
6521
|
+
case "otel":
|
|
6522
|
+
return "adapter";
|
|
6523
|
+
default:
|
|
6524
|
+
return "json-log";
|
|
6525
|
+
}
|
|
6526
|
+
};
|
|
6527
|
+
return {
|
|
6528
|
+
type: mapType(event.source.type),
|
|
6529
|
+
file: typeof attrs.sourceFile === "string" ? attrs.sourceFile : void 0,
|
|
6530
|
+
line: typeof attrs.sourceLine === "number" ? attrs.sourceLine : void 0
|
|
6531
|
+
};
|
|
6532
|
+
}
|
|
6533
|
+
function buildInspectAttributes(event) {
|
|
6534
|
+
const attrs = event.attributes !== void 0 ? { ...event.attributes } : {};
|
|
6535
|
+
if (event.inputSummary !== void 0) {
|
|
6536
|
+
attrs.inputSummary = event.inputSummary;
|
|
6537
|
+
}
|
|
6538
|
+
if (event.outputSummary !== void 0) {
|
|
6539
|
+
attrs.outputSummary = event.outputSummary;
|
|
6540
|
+
}
|
|
6541
|
+
if (event.error) {
|
|
6542
|
+
if (event.error.name !== void 0) {
|
|
6543
|
+
attrs.errorName = event.error.name;
|
|
6544
|
+
}
|
|
6545
|
+
attrs.errorMessage = event.error.message;
|
|
6546
|
+
if (event.error.code !== void 0) {
|
|
6547
|
+
attrs.errorCode = event.error.code;
|
|
6548
|
+
}
|
|
6549
|
+
}
|
|
6550
|
+
if (event.tokenUsage) {
|
|
6551
|
+
attrs.tokens = { ...event.tokenUsage };
|
|
6552
|
+
}
|
|
6553
|
+
if (event.source.type === "ai-sdk" || event.source.type === "otel") {
|
|
6554
|
+
attrs.originalSourceType = event.source.type;
|
|
6555
|
+
}
|
|
6556
|
+
if (event.source.name !== void 0) {
|
|
6557
|
+
attrs.sourceName = event.source.name;
|
|
6558
|
+
}
|
|
6559
|
+
if (event.source.version !== void 0) {
|
|
6560
|
+
attrs.sourceVersion = event.source.version;
|
|
6561
|
+
}
|
|
6562
|
+
return attrs;
|
|
6563
|
+
}
|
|
6564
|
+
function persistedInspectEventToInspectEvent(event) {
|
|
6565
|
+
if (!isPersistedInspectEvent(event)) {
|
|
6566
|
+
throw new Error("Invalid PersistedInspectEvent: failed isPersistedInspectEvent");
|
|
6567
|
+
}
|
|
6568
|
+
const ts = parseIsoToMs3(event.timestamp);
|
|
6569
|
+
const attrs = buildInspectAttributes(event);
|
|
6570
|
+
if (ts.invalidTimestamp) {
|
|
6571
|
+
attrs.invalidTimestamp = true;
|
|
6572
|
+
}
|
|
6573
|
+
let status;
|
|
6574
|
+
if (event.status === "running" || event.status === "ok" || event.status === "error") {
|
|
6575
|
+
status = event.status;
|
|
6576
|
+
} else if (event.status === "unknown") {
|
|
6577
|
+
attrs.persistedStatus = "unknown";
|
|
6578
|
+
}
|
|
6579
|
+
const out = {
|
|
6580
|
+
eventId: event.eventId,
|
|
6581
|
+
runId: event.runId,
|
|
6582
|
+
name: event.name,
|
|
6583
|
+
kind: event.kind,
|
|
6584
|
+
timestamp: ts.ms,
|
|
6585
|
+
confidence: event.confidence,
|
|
6586
|
+
source: mapPersistedSourceToInspect(event),
|
|
6587
|
+
attributes: compactAttributes2(attrs)
|
|
6588
|
+
};
|
|
6589
|
+
if (event.parentId !== void 0) {
|
|
6590
|
+
out.parentId = event.parentId;
|
|
6591
|
+
}
|
|
6592
|
+
if (status !== void 0) {
|
|
6593
|
+
out.status = status;
|
|
6594
|
+
}
|
|
6595
|
+
if (event.durationMs !== void 0 && Number.isFinite(event.durationMs) && event.durationMs >= 0) {
|
|
6596
|
+
out.durationMs = event.durationMs;
|
|
6597
|
+
}
|
|
6598
|
+
return out;
|
|
6599
|
+
}
|
|
6600
|
+
function persistedInspectEventsToInspectEvents(events, options) {
|
|
6601
|
+
const skipInvalid = options?.skipInvalid === true;
|
|
6602
|
+
const out = [];
|
|
6603
|
+
for (const event of events) {
|
|
6604
|
+
if (!isPersistedInspectEvent(event)) {
|
|
6605
|
+
if (skipInvalid) {
|
|
6606
|
+
continue;
|
|
6607
|
+
}
|
|
6608
|
+
throw new Error("Invalid PersistedInspectEvent: failed isPersistedInspectEvent");
|
|
6609
|
+
}
|
|
6610
|
+
out.push(persistedInspectEventToInspectEvent(event));
|
|
6611
|
+
}
|
|
6612
|
+
return out;
|
|
6613
|
+
}
|
|
6614
|
+
|
|
6615
|
+
// packages/core/src/persisted/tree-bridge.ts
|
|
6616
|
+
function persistedInspectEventsToRunTrees(events, options) {
|
|
6617
|
+
const inspectEvents = persistedInspectEventsToInspectEvents(events, {
|
|
6618
|
+
skipInvalid: options?.skipInvalid
|
|
6619
|
+
});
|
|
6620
|
+
return new TreeBuilder().build(inspectEvents);
|
|
6621
|
+
}
|
|
6622
|
+
|
|
6623
|
+
// packages/core/src/readers/index.ts
|
|
6624
|
+
var DEFAULT_MAX_TRACE_INPUT_BYTES = 10 * 1024 * 1024;
|
|
6625
|
+
var MIN_DETECTION_CONFIDENCE = 0.5;
|
|
6626
|
+
var AMBIGUOUS_CONFIDENCE_DELTA = 0.05;
|
|
6627
|
+
var resolvedInputCache = /* @__PURE__ */ new WeakMap();
|
|
6628
|
+
var OPENINFERENCE_READER_FORMAT = "openinference-json";
|
|
6629
|
+
var OTLP_READER_FORMAT = "otlp-json";
|
|
6630
|
+
var OPENINFERENCE_SPAN_KEYS = /* @__PURE__ */ new Set([
|
|
6631
|
+
"trace_id",
|
|
6632
|
+
"traceId",
|
|
6633
|
+
"span_id",
|
|
6634
|
+
"spanId",
|
|
6635
|
+
"parent_span_id",
|
|
6636
|
+
"parentSpanId",
|
|
6637
|
+
"name",
|
|
6638
|
+
"start_time_unix_nano",
|
|
6639
|
+
"startTimeUnixNano",
|
|
6640
|
+
"end_time_unix_nano",
|
|
6641
|
+
"endTimeUnixNano",
|
|
6642
|
+
"start_time",
|
|
6643
|
+
"startTime",
|
|
6644
|
+
"end_time",
|
|
6645
|
+
"endTime",
|
|
6646
|
+
"attributes",
|
|
6647
|
+
"status",
|
|
6648
|
+
"kind",
|
|
6649
|
+
"span_kind",
|
|
6650
|
+
"spanKind"
|
|
6651
|
+
]);
|
|
6652
|
+
var OPENINFERENCE_SENSITIVE_ATTRIBUTE_KEYS = [
|
|
6653
|
+
"input.value",
|
|
6654
|
+
"output.value",
|
|
6655
|
+
"input.mime_type",
|
|
6656
|
+
"output.mime_type",
|
|
6657
|
+
"llm.input_messages",
|
|
6658
|
+
"llm.output_messages",
|
|
6659
|
+
"llm.prompts",
|
|
6660
|
+
"llm.completions",
|
|
6661
|
+
"retrieval.documents",
|
|
6662
|
+
"reranker.input_documents",
|
|
6663
|
+
"reranker.output_documents",
|
|
6664
|
+
"document.content",
|
|
6665
|
+
"gen_ai.prompt",
|
|
6666
|
+
"gen_ai.completion",
|
|
6667
|
+
"gen_ai.input.messages",
|
|
6668
|
+
"gen_ai.output.messages"
|
|
6669
|
+
];
|
|
6670
|
+
var OTLP_SPAN_KEYS = /* @__PURE__ */ new Set([
|
|
6671
|
+
"traceId",
|
|
6672
|
+
"spanId",
|
|
6673
|
+
"parentSpanId",
|
|
6674
|
+
"name",
|
|
6675
|
+
"kind",
|
|
6676
|
+
"startTimeUnixNano",
|
|
6677
|
+
"endTimeUnixNano",
|
|
6678
|
+
"attributes",
|
|
6679
|
+
"events",
|
|
6680
|
+
"status",
|
|
6681
|
+
"droppedAttributesCount",
|
|
6682
|
+
"droppedEventsCount",
|
|
6683
|
+
"droppedLinksCount",
|
|
6684
|
+
"links",
|
|
6685
|
+
"flags"
|
|
6686
|
+
]);
|
|
6687
|
+
var TraceReadError = class extends Error {
|
|
6688
|
+
code;
|
|
6689
|
+
warnings;
|
|
6690
|
+
constructor(code, message, warnings = []) {
|
|
6691
|
+
super(message);
|
|
6692
|
+
this.name = "TraceReadError";
|
|
6693
|
+
this.code = code;
|
|
6694
|
+
this.warnings = warnings;
|
|
6695
|
+
}
|
|
6696
|
+
};
|
|
6697
|
+
function normalizeCandidate(reader, candidate) {
|
|
6698
|
+
const confidence = Number.isFinite(candidate.confidence) ? Math.max(0, Math.min(1, candidate.confidence)) : 0;
|
|
6699
|
+
return {
|
|
6700
|
+
...candidate,
|
|
6701
|
+
format: candidate.format || reader.format,
|
|
6702
|
+
confidence,
|
|
6703
|
+
readerName: candidate.readerName ?? reader.name
|
|
6704
|
+
};
|
|
6705
|
+
}
|
|
6706
|
+
function sortCandidates(candidates) {
|
|
6707
|
+
return [...candidates].sort((a, b) => {
|
|
6708
|
+
if (b.confidence !== a.confidence) return b.confidence - a.confidence;
|
|
6709
|
+
return a.format.localeCompare(b.format);
|
|
6710
|
+
});
|
|
6711
|
+
}
|
|
6712
|
+
function collectWarnings(candidates) {
|
|
6713
|
+
return candidates.flatMap((candidate) => candidate.warnings ?? []);
|
|
6714
|
+
}
|
|
6715
|
+
function dedupeWarnings(warnings) {
|
|
6716
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6717
|
+
const out = [];
|
|
6718
|
+
for (const warning of warnings) {
|
|
6719
|
+
const key = [
|
|
6720
|
+
warning.code,
|
|
6721
|
+
warning.message,
|
|
6722
|
+
warning.severity ?? "",
|
|
6723
|
+
warning.sourceFile ?? "",
|
|
6724
|
+
warning.line ?? "",
|
|
6725
|
+
warning.field ?? ""
|
|
6726
|
+
].join("\0");
|
|
6727
|
+
if (seen.has(key)) continue;
|
|
6728
|
+
seen.add(key);
|
|
6729
|
+
out.push(warning);
|
|
6730
|
+
}
|
|
6731
|
+
return out;
|
|
6732
|
+
}
|
|
6733
|
+
function attachSingleSourceFile(warnings, resolved) {
|
|
6734
|
+
if (resolved.sourceFiles.length !== 1) return [...warnings];
|
|
6735
|
+
const [sourceFile] = resolved.sourceFiles;
|
|
6736
|
+
return warnings.map((warning) => ({
|
|
6737
|
+
...warning,
|
|
6738
|
+
sourceFile: warning.sourceFile ?? sourceFile
|
|
6739
|
+
}));
|
|
6740
|
+
}
|
|
6741
|
+
function findReaderByFormat(format, readers) {
|
|
6742
|
+
return readers.find((reader) => reader.format === format);
|
|
6743
|
+
}
|
|
6744
|
+
async function jsonlFilesInDirectory(dirPath) {
|
|
6745
|
+
const entries = await promises.readdir(dirPath, { withFileTypes: true });
|
|
6746
|
+
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => path__default.default.join(dirPath, entry.name)).sort((a, b) => a.localeCompare(b));
|
|
6747
|
+
}
|
|
6748
|
+
async function resolveInput(input) {
|
|
6749
|
+
const cached = resolvedInputCache.get(input);
|
|
6750
|
+
if (cached) return cached;
|
|
6751
|
+
const promise = resolveInputUncached(input);
|
|
6752
|
+
resolvedInputCache.set(input, promise);
|
|
6753
|
+
return promise;
|
|
6754
|
+
}
|
|
6755
|
+
function assertInputWithinBounds(content, sourceFile) {
|
|
6756
|
+
const bytes = Buffer.byteLength(content, "utf8");
|
|
6757
|
+
if (bytes <= DEFAULT_MAX_TRACE_INPUT_BYTES) return;
|
|
6758
|
+
throw new TraceReadError("unsupported_format", "Trace input exceeds the local reader size limit.", [
|
|
6759
|
+
{
|
|
6760
|
+
code: "input_too_large",
|
|
6761
|
+
message: `Trace input is ${bytes} bytes; max is ${DEFAULT_MAX_TRACE_INPUT_BYTES} bytes.`,
|
|
6762
|
+
severity: "error",
|
|
6763
|
+
...sourceFile !== void 0 ? { sourceFile } : {}
|
|
6764
|
+
}
|
|
6765
|
+
]);
|
|
6766
|
+
}
|
|
6767
|
+
async function resolveInputUncached(input) {
|
|
6768
|
+
if (input.type === "string") {
|
|
6769
|
+
assertInputWithinBounds(input.content);
|
|
6770
|
+
return { content: input.content, sourceFiles: [] };
|
|
6771
|
+
}
|
|
6772
|
+
if (input.type === "buffer") {
|
|
6773
|
+
const content = input.content.toString("utf-8");
|
|
6774
|
+
assertInputWithinBounds(content);
|
|
6775
|
+
return { content, sourceFiles: [] };
|
|
6776
|
+
}
|
|
6777
|
+
if (input.type === "file") {
|
|
6778
|
+
const content = await promises.readFile(input.path, "utf-8");
|
|
6779
|
+
assertInputWithinBounds(content, input.path);
|
|
6780
|
+
return { content, sourceFiles: [input.path] };
|
|
6781
|
+
}
|
|
6782
|
+
if (input.type === "directory") {
|
|
6783
|
+
const files = await jsonlFilesInDirectory(input.path);
|
|
6784
|
+
const parts = await Promise.all(
|
|
6785
|
+
files.map(async (file) => (await promises.readFile(file, "utf-8")).trimEnd())
|
|
6786
|
+
);
|
|
6787
|
+
const content = parts.filter((part) => part.trim() !== "").join("\n");
|
|
6788
|
+
assertInputWithinBounds(content, input.path);
|
|
6789
|
+
return {
|
|
6790
|
+
content,
|
|
6791
|
+
sourceFiles: files
|
|
6792
|
+
};
|
|
6793
|
+
}
|
|
6794
|
+
return void 0;
|
|
6795
|
+
}
|
|
6796
|
+
function detectJsonlFormat(content) {
|
|
6797
|
+
let saw01 = false;
|
|
6798
|
+
let saw02 = false;
|
|
6799
|
+
let saw10 = false;
|
|
6800
|
+
let validRows = 0;
|
|
6801
|
+
let invalidJsonRows = 0;
|
|
6802
|
+
let unknownSchemaRows = 0;
|
|
6803
|
+
let firstInvalidJsonLine;
|
|
6804
|
+
let firstUnknownSchemaLine;
|
|
6805
|
+
let lineNumber = 0;
|
|
6806
|
+
for (const line of content.split(/\r?\n/)) {
|
|
6807
|
+
lineNumber += 1;
|
|
6808
|
+
const trimmed = line.trim();
|
|
6809
|
+
if (trimmed === "") continue;
|
|
6810
|
+
let parsed;
|
|
6811
|
+
try {
|
|
6812
|
+
parsed = JSON.parse(trimmed);
|
|
6813
|
+
} catch {
|
|
6814
|
+
invalidJsonRows += 1;
|
|
6815
|
+
firstInvalidJsonLine ??= lineNumber;
|
|
6816
|
+
continue;
|
|
6817
|
+
}
|
|
6818
|
+
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) && "schemaVersion" in parsed) {
|
|
6819
|
+
const version = parsed.schemaVersion;
|
|
6820
|
+
if (version === "0.1") {
|
|
6821
|
+
saw01 = true;
|
|
6822
|
+
validRows += 1;
|
|
6823
|
+
continue;
|
|
6824
|
+
}
|
|
6825
|
+
if (version === "0.2") {
|
|
6826
|
+
saw02 = true;
|
|
6827
|
+
validRows += 1;
|
|
6828
|
+
continue;
|
|
6829
|
+
}
|
|
6830
|
+
if (version === "1.0") {
|
|
6831
|
+
saw10 = true;
|
|
6832
|
+
validRows += 1;
|
|
6833
|
+
continue;
|
|
6834
|
+
}
|
|
6835
|
+
}
|
|
6836
|
+
unknownSchemaRows += 1;
|
|
6837
|
+
firstUnknownSchemaLine ??= lineNumber;
|
|
6838
|
+
}
|
|
6839
|
+
const warnings = [];
|
|
6840
|
+
if (invalidJsonRows > 0) {
|
|
6841
|
+
warnings.push({
|
|
6842
|
+
code: "invalid_jsonl_rows",
|
|
6843
|
+
message: `Skipped ${invalidJsonRows} invalid JSONL row(s) during format detection.`,
|
|
6844
|
+
severity: "warning",
|
|
6845
|
+
...firstInvalidJsonLine !== void 0 ? { line: firstInvalidJsonLine } : {}
|
|
6846
|
+
});
|
|
6847
|
+
}
|
|
6848
|
+
if (unknownSchemaRows > 0) {
|
|
6849
|
+
warnings.push({
|
|
6850
|
+
code: "unknown_schema_rows",
|
|
6851
|
+
message: `Skipped ${unknownSchemaRows} row(s) with unknown schemaVersion during format detection.`,
|
|
6852
|
+
severity: "warning",
|
|
6853
|
+
...firstUnknownSchemaLine !== void 0 ? { line: firstUnknownSchemaLine } : {}
|
|
6854
|
+
});
|
|
6855
|
+
}
|
|
6856
|
+
let format = "empty";
|
|
6857
|
+
const seenFormats = [saw01, saw02, saw10].filter(Boolean).length;
|
|
6858
|
+
if (seenFormats > 1) format = "mixed";
|
|
6859
|
+
else if (saw01) format = "0.1";
|
|
6860
|
+
else if (saw02) format = "0.2";
|
|
6861
|
+
else if (saw10) format = "1.0";
|
|
6862
|
+
return { format, validRows, warnings };
|
|
6863
|
+
}
|
|
6864
|
+
function agentInspectFormatLabel(format) {
|
|
6865
|
+
switch (format) {
|
|
6866
|
+
case "0.1":
|
|
6867
|
+
return "agent-inspect-v0.1-jsonl";
|
|
6868
|
+
case "0.2":
|
|
6869
|
+
return "agent-inspect-v0.2-jsonl";
|
|
6870
|
+
case "1.0":
|
|
6871
|
+
return "agent-inspect-v1.0-jsonl";
|
|
6872
|
+
case "mixed":
|
|
6873
|
+
return "agent-inspect-mixed-jsonl";
|
|
6874
|
+
default:
|
|
6875
|
+
return "agent-inspect-jsonl";
|
|
6876
|
+
}
|
|
6877
|
+
}
|
|
6878
|
+
function persistedEventsForParsedTrace(parsed) {
|
|
6879
|
+
if ((parsed.format === "0.2" || parsed.format === "1.0") && parsed.persisted.length > 0) {
|
|
6880
|
+
return [...parsed.persisted];
|
|
6881
|
+
}
|
|
6882
|
+
if (parsed.format === "mixed" && parsed.rows.length > 0) {
|
|
6883
|
+
return parsed.rows.map((row, index) => {
|
|
6884
|
+
if (row.format === "0.2" || row.format === "1.0") return row.event;
|
|
6885
|
+
return traceEventToPersistedInspectEvent(row.event, {
|
|
6886
|
+
eventIndex: index,
|
|
6887
|
+
sourceName: "agent-inspect-jsonl-reader"
|
|
6888
|
+
});
|
|
6889
|
+
});
|
|
6890
|
+
}
|
|
6891
|
+
return traceEventsToPersistedInspectEvents(parsed.events, {
|
|
6892
|
+
sourceName: "agent-inspect-jsonl-reader"
|
|
6893
|
+
});
|
|
6894
|
+
}
|
|
6895
|
+
function isRecord9(value) {
|
|
6896
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
6897
|
+
}
|
|
6898
|
+
function isNonEmptyString4(value) {
|
|
6899
|
+
return typeof value === "string" && value.trim() !== "";
|
|
6900
|
+
}
|
|
6901
|
+
function readStringField(record, keys) {
|
|
6902
|
+
for (const key of keys) {
|
|
6903
|
+
const value = record[key];
|
|
6904
|
+
if (isNonEmptyString4(value)) return value;
|
|
6905
|
+
}
|
|
6906
|
+
return void 0;
|
|
6907
|
+
}
|
|
6908
|
+
function readRecordField(record, key) {
|
|
6909
|
+
const value = record[key];
|
|
6910
|
+
return isRecord9(value) ? value : void 0;
|
|
6911
|
+
}
|
|
6912
|
+
function parseJsonDocument(content) {
|
|
6913
|
+
return JSON.parse(content);
|
|
6914
|
+
}
|
|
6915
|
+
function looksLikeOpenInferenceSpan(value) {
|
|
6916
|
+
if (!isRecord9(value)) return false;
|
|
6917
|
+
const attributes = readRecordField(value, "attributes");
|
|
6918
|
+
return readStringField(value, ["trace_id", "traceId"]) !== void 0 && readStringField(value, ["span_id", "spanId"]) !== void 0 && (readStringField(value, ["name"]) !== void 0 || attributes?.["openinference.span.kind"] !== void 0);
|
|
6919
|
+
}
|
|
6920
|
+
function extractOpenInferenceDocument(root) {
|
|
6921
|
+
const warnings = [];
|
|
6922
|
+
const unsupportedFields = [];
|
|
6923
|
+
if (Array.isArray(root)) {
|
|
6924
|
+
const spans = root.filter(looksLikeOpenInferenceSpan);
|
|
6925
|
+
if (spans.length === 0) return void 0;
|
|
6926
|
+
if (spans.length !== root.length) {
|
|
6927
|
+
warnings.push({
|
|
6928
|
+
code: "openinference_skipped_items",
|
|
6929
|
+
message: "Skipped non-span item(s) in OpenInference span array.",
|
|
6930
|
+
severity: "warning"
|
|
6931
|
+
});
|
|
6932
|
+
}
|
|
6933
|
+
return {
|
|
6934
|
+
spans,
|
|
6935
|
+
confidence: 0.82,
|
|
6936
|
+
description: "OpenInference span array",
|
|
6937
|
+
warnings,
|
|
6938
|
+
unsupportedFields
|
|
6939
|
+
};
|
|
6940
|
+
}
|
|
6941
|
+
if (!isRecord9(root)) return void 0;
|
|
6942
|
+
const rootFormat = root.format;
|
|
6943
|
+
const rootCompatibility = root.compatibility;
|
|
6944
|
+
const version = typeof root.version === "string" && root.version.trim() !== "" ? root.version : void 0;
|
|
6945
|
+
if (Array.isArray(root.spans)) {
|
|
6946
|
+
const spans = root.spans.filter(looksLikeOpenInferenceSpan);
|
|
6947
|
+
if (spans.length === 0 && (rootFormat === "openinference" || rootCompatibility === "openinference-compatible")) {
|
|
6948
|
+
warnings.push({
|
|
6949
|
+
code: "openinference_no_valid_spans",
|
|
6950
|
+
message: "OpenInference document did not contain any valid spans.",
|
|
6951
|
+
severity: "error"
|
|
6952
|
+
});
|
|
6953
|
+
return {
|
|
6954
|
+
spans,
|
|
6955
|
+
confidence: 0.7,
|
|
6956
|
+
description: "Malformed OpenInference document",
|
|
6957
|
+
version,
|
|
6958
|
+
warnings,
|
|
6959
|
+
unsupportedFields
|
|
6960
|
+
};
|
|
6961
|
+
}
|
|
6962
|
+
if (spans.length === 0) return void 0;
|
|
6963
|
+
if (spans.length !== root.spans.length) {
|
|
6964
|
+
warnings.push({
|
|
6965
|
+
code: "openinference_skipped_spans",
|
|
6966
|
+
message: "Skipped invalid OpenInference span item(s).",
|
|
6967
|
+
severity: "warning"
|
|
6968
|
+
});
|
|
6969
|
+
}
|
|
6970
|
+
return {
|
|
6971
|
+
spans,
|
|
6972
|
+
confidence: rootFormat === "openinference" || rootCompatibility === "openinference-compatible" ? 0.9 : 0.84,
|
|
6973
|
+
description: rootFormat === "openinference" || rootCompatibility === "openinference-compatible" ? "OpenInference document" : "OpenInference spans document",
|
|
6974
|
+
version,
|
|
6975
|
+
warnings,
|
|
6976
|
+
unsupportedFields
|
|
6977
|
+
};
|
|
6978
|
+
}
|
|
6979
|
+
if (Array.isArray(root.data)) {
|
|
6980
|
+
const spans = root.data.filter(looksLikeOpenInferenceSpan);
|
|
6981
|
+
if (spans.length === 0) return void 0;
|
|
6982
|
+
if (spans.length !== root.data.length) {
|
|
6983
|
+
warnings.push({
|
|
6984
|
+
code: "openinference_skipped_data_items",
|
|
6985
|
+
message: "Skipped non-span item(s) in OpenInference data array.",
|
|
6986
|
+
severity: "warning"
|
|
6987
|
+
});
|
|
6988
|
+
}
|
|
6989
|
+
return {
|
|
6990
|
+
spans,
|
|
6991
|
+
confidence: 0.8,
|
|
6992
|
+
description: "OpenInference data document",
|
|
6993
|
+
version,
|
|
6994
|
+
warnings,
|
|
6995
|
+
unsupportedFields
|
|
6996
|
+
};
|
|
6997
|
+
}
|
|
6998
|
+
if (looksLikeOpenInferenceSpan(root)) {
|
|
6999
|
+
return {
|
|
7000
|
+
spans: [root],
|
|
7001
|
+
confidence: 0.76,
|
|
7002
|
+
description: "OpenInference single span",
|
|
7003
|
+
version,
|
|
7004
|
+
warnings,
|
|
7005
|
+
unsupportedFields
|
|
7006
|
+
};
|
|
7007
|
+
}
|
|
7008
|
+
if (rootFormat === "openinference" || rootCompatibility === "openinference-compatible") {
|
|
7009
|
+
warnings.push({
|
|
7010
|
+
code: "openinference_missing_spans",
|
|
7011
|
+
message: "OpenInference document is missing a spans array.",
|
|
7012
|
+
severity: "error"
|
|
7013
|
+
});
|
|
7014
|
+
return {
|
|
7015
|
+
spans: [],
|
|
7016
|
+
confidence: 0.7,
|
|
7017
|
+
description: "Malformed OpenInference document",
|
|
7018
|
+
version,
|
|
7019
|
+
warnings,
|
|
7020
|
+
unsupportedFields
|
|
7021
|
+
};
|
|
7022
|
+
}
|
|
7023
|
+
return void 0;
|
|
7024
|
+
}
|
|
7025
|
+
function parseUnixNanoToIso(value) {
|
|
7026
|
+
if (typeof value === "bigint" && value >= 0n) {
|
|
7027
|
+
return new Date(Number(value / 1000000n)).toISOString();
|
|
7028
|
+
}
|
|
7029
|
+
if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
|
|
7030
|
+
return new Date(Math.floor(value / 1e6)).toISOString();
|
|
7031
|
+
}
|
|
7032
|
+
if (typeof value === "string" && /^\d+$/.test(value)) {
|
|
7033
|
+
return new Date(Number(BigInt(value) / 1000000n)).toISOString();
|
|
7034
|
+
}
|
|
7035
|
+
return void 0;
|
|
7036
|
+
}
|
|
7037
|
+
function parseIsoTime(value) {
|
|
7038
|
+
if (!isNonEmptyString4(value)) return void 0;
|
|
7039
|
+
const ms = Date.parse(value);
|
|
7040
|
+
if (!Number.isFinite(ms)) return void 0;
|
|
7041
|
+
return new Date(ms).toISOString();
|
|
7042
|
+
}
|
|
7043
|
+
function readOpenInferenceTimestamp(span, nanoKeys, isoKeys) {
|
|
7044
|
+
for (const key of nanoKeys) {
|
|
7045
|
+
const iso = parseUnixNanoToIso(span[key]);
|
|
7046
|
+
if (iso !== void 0) return iso;
|
|
7047
|
+
}
|
|
7048
|
+
for (const key of isoKeys) {
|
|
7049
|
+
const iso = parseIsoTime(span[key]);
|
|
7050
|
+
if (iso !== void 0) return iso;
|
|
7051
|
+
}
|
|
7052
|
+
return void 0;
|
|
7053
|
+
}
|
|
7054
|
+
function durationBetweenIso(startedAt, endedAt) {
|
|
7055
|
+
if (startedAt === void 0 || endedAt === void 0) return void 0;
|
|
7056
|
+
const startMs = Date.parse(startedAt);
|
|
7057
|
+
const endMs = Date.parse(endedAt);
|
|
7058
|
+
if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs < startMs) {
|
|
7059
|
+
return void 0;
|
|
7060
|
+
}
|
|
7061
|
+
return endMs - startMs;
|
|
7062
|
+
}
|
|
7063
|
+
function isSensitiveOpenInferenceAttribute(key) {
|
|
7064
|
+
return OPENINFERENCE_SENSITIVE_ATTRIBUTE_KEYS.some(
|
|
7065
|
+
(sensitiveKey) => key === sensitiveKey || key.startsWith(`${sensitiveKey}.`) || key.endsWith(".message.content") || key.endsWith(".document.content")
|
|
7066
|
+
);
|
|
7067
|
+
}
|
|
7068
|
+
function summarizeAttributeValue(value) {
|
|
7069
|
+
if (typeof value === "string") {
|
|
7070
|
+
return { type: "string", length: value.length };
|
|
7071
|
+
}
|
|
7072
|
+
if (typeof value === "number") {
|
|
7073
|
+
return { type: "number", finite: Number.isFinite(value) };
|
|
7074
|
+
}
|
|
7075
|
+
if (typeof value === "boolean") {
|
|
7076
|
+
return { type: "boolean" };
|
|
7077
|
+
}
|
|
7078
|
+
if (Array.isArray(value)) {
|
|
7079
|
+
return { type: "array", length: value.length };
|
|
7080
|
+
}
|
|
7081
|
+
if (isRecord9(value)) {
|
|
7082
|
+
return { type: "object", keyCount: Object.keys(value).length };
|
|
7083
|
+
}
|
|
7084
|
+
if (value === null) {
|
|
7085
|
+
return { type: "null" };
|
|
7086
|
+
}
|
|
7087
|
+
return { type: typeof value };
|
|
7088
|
+
}
|
|
7089
|
+
function sanitizeOpenInferenceAttributes(attributes, pathPrefix) {
|
|
7090
|
+
const out = {};
|
|
7091
|
+
const warnings = [];
|
|
7092
|
+
const unsupportedFields = [];
|
|
7093
|
+
const summarizedKeys = [];
|
|
7094
|
+
for (const [key, value] of Object.entries(attributes)) {
|
|
7095
|
+
if (isSensitiveOpenInferenceAttribute(key)) {
|
|
7096
|
+
summarizedKeys.push(key);
|
|
7097
|
+
out[`${key}.summary`] = summarizeAttributeValue(value);
|
|
7098
|
+
unsupportedFields.push(`${pathPrefix}.attributes.${key}`);
|
|
7099
|
+
continue;
|
|
7100
|
+
}
|
|
7101
|
+
out[key] = value;
|
|
7102
|
+
}
|
|
7103
|
+
if (summarizedKeys.length > 0) {
|
|
7104
|
+
out["openinference.summarized_attributes"] = summarizedKeys;
|
|
7105
|
+
warnings.push({
|
|
7106
|
+
code: "openinference_sensitive_attribute_summarized",
|
|
7107
|
+
message: "OpenInference prompt/output/document attribute(s) were summarized instead of copied verbatim.",
|
|
7108
|
+
severity: "warning"
|
|
7109
|
+
});
|
|
7110
|
+
}
|
|
7111
|
+
return { attributes: out, warnings, unsupportedFields };
|
|
7112
|
+
}
|
|
7113
|
+
function mapOpenInferenceKind(span, attributes, pathPrefix) {
|
|
7114
|
+
const warnings = [];
|
|
7115
|
+
const agentInspectKind = attributes["agent_inspect.kind"];
|
|
7116
|
+
if (agentInspectKind === "RUN" || agentInspectKind === "AGENT" || agentInspectKind === "LLM" || agentInspectKind === "TOOL" || agentInspectKind === "CHAIN" || agentInspectKind === "RETRIEVER" || agentInspectKind === "DECISION" || agentInspectKind === "RESULT" || agentInspectKind === "ERROR" || agentInspectKind === "LOGIC" || agentInspectKind === "LOG" || agentInspectKind === "OUTCOME") {
|
|
7117
|
+
return { kind: agentInspectKind, warnings };
|
|
7118
|
+
}
|
|
7119
|
+
const rawKind = readStringField(span, ["kind", "span_kind", "spanKind"]) ?? (typeof attributes["openinference.span.kind"] === "string" ? attributes["openinference.span.kind"] : void 0);
|
|
7120
|
+
const normalized = rawKind?.toUpperCase();
|
|
7121
|
+
switch (normalized) {
|
|
7122
|
+
case "LLM":
|
|
7123
|
+
return { kind: "LLM", warnings };
|
|
7124
|
+
case "TOOL":
|
|
7125
|
+
return { kind: "TOOL", warnings };
|
|
7126
|
+
case "CHAIN":
|
|
7127
|
+
return { kind: "CHAIN", warnings };
|
|
7128
|
+
case "RETRIEVER":
|
|
7129
|
+
return { kind: "RETRIEVER", warnings };
|
|
7130
|
+
case "AGENT":
|
|
7131
|
+
return { kind: "AGENT", warnings };
|
|
7132
|
+
case "EMBEDDING":
|
|
7133
|
+
warnings.push({
|
|
7134
|
+
code: "openinference_kind_semantic_loss",
|
|
7135
|
+
message: "OpenInference EMBEDDING span kind mapped to AgentInspect LLM.",
|
|
7136
|
+
severity: "warning",
|
|
7137
|
+
field: `${pathPrefix}.attributes.openinference.span.kind`
|
|
7138
|
+
});
|
|
7139
|
+
return { kind: "LLM", warnings };
|
|
7140
|
+
case "RERANKER":
|
|
7141
|
+
warnings.push({
|
|
7142
|
+
code: "openinference_kind_semantic_loss",
|
|
7143
|
+
message: "OpenInference RERANKER span kind mapped to AgentInspect RETRIEVER.",
|
|
7144
|
+
severity: "warning",
|
|
7145
|
+
field: `${pathPrefix}.attributes.openinference.span.kind`
|
|
7146
|
+
});
|
|
7147
|
+
return { kind: "RETRIEVER", warnings };
|
|
7148
|
+
case "UNKNOWN":
|
|
7149
|
+
case void 0:
|
|
7150
|
+
warnings.push({
|
|
7151
|
+
code: "openinference_kind_unknown",
|
|
7152
|
+
message: "OpenInference span kind was missing or unknown; mapped to AgentInspect LOGIC.",
|
|
7153
|
+
severity: "warning",
|
|
7154
|
+
field: `${pathPrefix}.attributes.openinference.span.kind`
|
|
7155
|
+
});
|
|
7156
|
+
return { kind: "LOGIC", warnings };
|
|
7157
|
+
default:
|
|
7158
|
+
warnings.push({
|
|
7159
|
+
code: "openinference_kind_unsupported",
|
|
7160
|
+
message: `Unsupported OpenInference span kind "${rawKind}" mapped to AgentInspect LOGIC.`,
|
|
7161
|
+
severity: "warning",
|
|
7162
|
+
field: `${pathPrefix}.attributes.openinference.span.kind`
|
|
7163
|
+
});
|
|
7164
|
+
return { kind: "LOGIC", warnings };
|
|
7165
|
+
}
|
|
7166
|
+
}
|
|
7167
|
+
function mapOpenInferenceStatus(status) {
|
|
7168
|
+
if (!isRecord9(status)) return void 0;
|
|
7169
|
+
const rawCode = status.code;
|
|
7170
|
+
if (typeof rawCode !== "string") return void 0;
|
|
7171
|
+
switch (rawCode.toUpperCase()) {
|
|
7172
|
+
case "OK":
|
|
7173
|
+
return "ok";
|
|
7174
|
+
case "ERROR":
|
|
7175
|
+
return "error";
|
|
7176
|
+
case "UNSET":
|
|
7177
|
+
return "unknown";
|
|
7178
|
+
default:
|
|
7179
|
+
return "unknown";
|
|
7180
|
+
}
|
|
7181
|
+
}
|
|
7182
|
+
function readOpenInferenceTokenUsage(attributes) {
|
|
7183
|
+
const prompt = attributes["llm.token_count.prompt"];
|
|
7184
|
+
const completion = attributes["llm.token_count.completion"];
|
|
7185
|
+
const total = attributes["llm.token_count.total"];
|
|
7186
|
+
const cached = attributes["llm.token_count.prompt_details.cache_read"];
|
|
7187
|
+
const usage = {};
|
|
7188
|
+
if (typeof prompt === "number" && Number.isFinite(prompt) && prompt >= 0) {
|
|
7189
|
+
usage.input = prompt;
|
|
7190
|
+
}
|
|
7191
|
+
if (typeof completion === "number" && Number.isFinite(completion) && completion >= 0) {
|
|
7192
|
+
usage.output = completion;
|
|
7193
|
+
}
|
|
7194
|
+
if (typeof total === "number" && Number.isFinite(total) && total >= 0) {
|
|
7195
|
+
usage.total = total;
|
|
7196
|
+
}
|
|
7197
|
+
if (typeof cached === "number" && Number.isFinite(cached) && cached >= 0) {
|
|
7198
|
+
usage.cached = cached;
|
|
7199
|
+
}
|
|
7200
|
+
if (usage.total === void 0 && usage.input !== void 0 && usage.output !== void 0) {
|
|
7201
|
+
usage.total = usage.input + usage.output;
|
|
7202
|
+
}
|
|
7203
|
+
return Object.keys(usage).length > 0 ? usage : void 0;
|
|
7204
|
+
}
|
|
7205
|
+
function readOpenInferenceConfidence(attributes) {
|
|
7206
|
+
const confidence = attributes["agent_inspect.confidence"];
|
|
7207
|
+
if (confidence === "explicit" || confidence === "correlated" || confidence === "heuristic" || confidence === "unknown") {
|
|
7208
|
+
return confidence;
|
|
7209
|
+
}
|
|
7210
|
+
return "correlated";
|
|
7211
|
+
}
|
|
7212
|
+
function mapOpenInferenceSpan(span, index, version) {
|
|
7213
|
+
const pathPrefix = `spans[${index}]`;
|
|
7214
|
+
const warnings = [];
|
|
7215
|
+
const unsupportedFields = [];
|
|
7216
|
+
const rawAttributes = readRecordField(span, "attributes") ?? {};
|
|
7217
|
+
const sanitized = sanitizeOpenInferenceAttributes(rawAttributes, pathPrefix);
|
|
7218
|
+
warnings.push(...sanitized.warnings);
|
|
7219
|
+
unsupportedFields.push(...sanitized.unsupportedFields);
|
|
7220
|
+
const attributes = { ...sanitized.attributes };
|
|
7221
|
+
for (const [key, value] of Object.entries(span)) {
|
|
7222
|
+
if (OPENINFERENCE_SPAN_KEYS.has(key)) continue;
|
|
7223
|
+
unsupportedFields.push(`${pathPrefix}.${key}`);
|
|
7224
|
+
if (value === null || typeof value !== "object") {
|
|
7225
|
+
attributes[`openinference.${key}`] = value;
|
|
7226
|
+
} else {
|
|
7227
|
+
attributes[`openinference.${key}.summary`] = summarizeAttributeValue(value);
|
|
7228
|
+
warnings.push({
|
|
7229
|
+
code: "openinference_unsupported_field_summarized",
|
|
7230
|
+
message: `Unsupported OpenInference span field "${key}" was summarized.`,
|
|
7231
|
+
severity: "warning",
|
|
7232
|
+
field: `${pathPrefix}.${key}`
|
|
7233
|
+
});
|
|
7234
|
+
}
|
|
7235
|
+
}
|
|
7236
|
+
const traceId = readStringField(span, ["trace_id", "traceId"]) ?? `trace-${index}`;
|
|
7237
|
+
const spanId = readStringField(span, ["span_id", "spanId"]) ?? `span-${index}`;
|
|
7238
|
+
const parentSpanId = readStringField(span, ["parent_span_id", "parentSpanId"]);
|
|
7239
|
+
const name = readStringField(span, ["name"]) ?? spanId;
|
|
7240
|
+
const startedAt = readOpenInferenceTimestamp(
|
|
7241
|
+
span,
|
|
7242
|
+
["start_time_unix_nano", "startTimeUnixNano"],
|
|
7243
|
+
["start_time", "startTime"]
|
|
7244
|
+
);
|
|
7245
|
+
const endedAt = readOpenInferenceTimestamp(
|
|
7246
|
+
span,
|
|
7247
|
+
["end_time_unix_nano", "endTimeUnixNano"],
|
|
7248
|
+
["end_time", "endTime"]
|
|
7249
|
+
);
|
|
7250
|
+
const timestamp = startedAt ?? "1970-01-01T00:00:00.000Z";
|
|
7251
|
+
if (startedAt === void 0) {
|
|
7252
|
+
warnings.push({
|
|
7253
|
+
code: "openinference_missing_start_time",
|
|
7254
|
+
message: "OpenInference span is missing a valid start time; using Unix epoch.",
|
|
7255
|
+
severity: "warning",
|
|
7256
|
+
field: `${pathPrefix}.start_time_unix_nano`
|
|
7257
|
+
});
|
|
7258
|
+
unsupportedFields.push(`${pathPrefix}.start_time_unix_nano`);
|
|
7259
|
+
}
|
|
7260
|
+
const { kind, warnings: kindWarnings } = mapOpenInferenceKind(
|
|
7261
|
+
span,
|
|
7262
|
+
rawAttributes,
|
|
7263
|
+
pathPrefix
|
|
7264
|
+
);
|
|
7265
|
+
warnings.push(...kindWarnings);
|
|
7266
|
+
const status = mapOpenInferenceStatus(span.status);
|
|
7267
|
+
const tokenUsage = readOpenInferenceTokenUsage(rawAttributes);
|
|
7268
|
+
const errorMessage = isRecord9(span.status) && typeof span.status.message === "string" ? span.status.message : void 0;
|
|
7269
|
+
const event = {
|
|
7270
|
+
schemaVersion: "0.2",
|
|
7271
|
+
eventId: typeof rawAttributes["agent_inspect.event_id"] === "string" ? rawAttributes["agent_inspect.event_id"] : spanId,
|
|
7272
|
+
runId: typeof rawAttributes["agent_inspect.run_id"] === "string" ? rawAttributes["agent_inspect.run_id"] : traceId,
|
|
7273
|
+
kind,
|
|
7274
|
+
name,
|
|
7275
|
+
timestamp,
|
|
7276
|
+
confidence: readOpenInferenceConfidence(rawAttributes),
|
|
7277
|
+
source: {
|
|
7278
|
+
type: "otel",
|
|
7279
|
+
name: "openinference",
|
|
7280
|
+
...version !== void 0 ? { version } : {}
|
|
7281
|
+
},
|
|
7282
|
+
attributes,
|
|
7283
|
+
trace: {
|
|
7284
|
+
traceId,
|
|
7285
|
+
spanId,
|
|
7286
|
+
...parentSpanId !== void 0 ? { parentSpanId } : {}
|
|
7287
|
+
}
|
|
7288
|
+
};
|
|
7289
|
+
if (status !== void 0) {
|
|
7290
|
+
event.status = status;
|
|
7291
|
+
}
|
|
7292
|
+
if (startedAt !== void 0) {
|
|
7293
|
+
event.startedAt = startedAt;
|
|
7294
|
+
}
|
|
7295
|
+
if (endedAt !== void 0) {
|
|
7296
|
+
event.endedAt = endedAt;
|
|
7297
|
+
}
|
|
7298
|
+
const durationMs2 = durationBetweenIso(startedAt, endedAt);
|
|
7299
|
+
if (durationMs2 !== void 0) {
|
|
7300
|
+
event.durationMs = durationMs2;
|
|
7301
|
+
}
|
|
7302
|
+
if (tokenUsage !== void 0) {
|
|
7303
|
+
event.tokenUsage = tokenUsage;
|
|
7304
|
+
}
|
|
7305
|
+
if (status === "error") {
|
|
7306
|
+
event.error = {
|
|
7307
|
+
message: errorMessage !== void 0 && errorMessage.trim() !== "" ? errorMessage : "OpenInference span error"
|
|
7308
|
+
};
|
|
7309
|
+
}
|
|
7310
|
+
return {
|
|
7311
|
+
event,
|
|
7312
|
+
warnings,
|
|
7313
|
+
unsupportedFields,
|
|
7314
|
+
spanId,
|
|
7315
|
+
...parentSpanId !== void 0 ? { parentSpanId } : {}
|
|
7316
|
+
};
|
|
7317
|
+
}
|
|
7318
|
+
function mapOpenInferenceEvents(document) {
|
|
7319
|
+
const mapped = document.spans.map(
|
|
7320
|
+
(span, index) => mapOpenInferenceSpan(span, index, document.version)
|
|
7321
|
+
);
|
|
7322
|
+
const spanIdToEventId = new Map(
|
|
7323
|
+
mapped.map((span) => [span.spanId, span.event.eventId])
|
|
7324
|
+
);
|
|
7325
|
+
for (const span of mapped) {
|
|
7326
|
+
if (span.parentSpanId === void 0) continue;
|
|
7327
|
+
span.event.parentId = spanIdToEventId.get(span.parentSpanId) ?? span.parentSpanId;
|
|
7328
|
+
}
|
|
7329
|
+
return {
|
|
7330
|
+
events: mapped.map((span) => span.event),
|
|
7331
|
+
warnings: mapped.flatMap((span) => span.warnings),
|
|
7332
|
+
unsupportedFields: mapped.flatMap((span) => span.unsupportedFields)
|
|
7333
|
+
};
|
|
7334
|
+
}
|
|
7335
|
+
var openInferenceJsonReader = {
|
|
7336
|
+
format: OPENINFERENCE_READER_FORMAT,
|
|
7337
|
+
name: "OpenInference JSON",
|
|
7338
|
+
async detect(input) {
|
|
7339
|
+
const resolved = await resolveInput(input);
|
|
7340
|
+
if (!resolved) return void 0;
|
|
7341
|
+
let parsed;
|
|
7342
|
+
try {
|
|
7343
|
+
parsed = parseJsonDocument(resolved.content);
|
|
7344
|
+
} catch {
|
|
7345
|
+
return void 0;
|
|
7346
|
+
}
|
|
7347
|
+
const document = extractOpenInferenceDocument(parsed);
|
|
7348
|
+
if (!document) return void 0;
|
|
7349
|
+
return {
|
|
7350
|
+
format: OPENINFERENCE_READER_FORMAT,
|
|
7351
|
+
confidence: document.confidence,
|
|
7352
|
+
readerName: "OpenInference JSON",
|
|
7353
|
+
description: document.description,
|
|
7354
|
+
warnings: attachSingleSourceFile(document.warnings, resolved)
|
|
7355
|
+
};
|
|
7356
|
+
},
|
|
7357
|
+
async read(input) {
|
|
7358
|
+
const resolved = await resolveInput(input);
|
|
7359
|
+
if (!resolved) {
|
|
7360
|
+
throw new TraceReadError(
|
|
7361
|
+
"unsupported_format",
|
|
7362
|
+
"OpenInference JSON reader requires file, string, or buffer input."
|
|
7363
|
+
);
|
|
7364
|
+
}
|
|
7365
|
+
let parsed;
|
|
7366
|
+
try {
|
|
7367
|
+
parsed = parseJsonDocument(resolved.content);
|
|
7368
|
+
} catch {
|
|
7369
|
+
throw new TraceReadError("unsupported_format", "OpenInference JSON input is not valid JSON.", [
|
|
7370
|
+
{
|
|
7371
|
+
code: "openinference_invalid_json",
|
|
7372
|
+
message: "OpenInference JSON reader could not parse the input as JSON.",
|
|
7373
|
+
severity: "error"
|
|
7374
|
+
}
|
|
7375
|
+
]);
|
|
7376
|
+
}
|
|
7377
|
+
const document = extractOpenInferenceDocument(parsed);
|
|
7378
|
+
if (!document || document.spans.length === 0) {
|
|
7379
|
+
throw new TraceReadError(
|
|
7380
|
+
"unsupported_format",
|
|
7381
|
+
"No valid OpenInference spans found.",
|
|
7382
|
+
attachSingleSourceFile(
|
|
7383
|
+
document?.warnings ?? [
|
|
7384
|
+
{
|
|
7385
|
+
code: "openinference_no_valid_spans",
|
|
7386
|
+
message: "OpenInference JSON input did not contain valid spans.",
|
|
7387
|
+
severity: "error"
|
|
7388
|
+
}
|
|
7389
|
+
],
|
|
7390
|
+
resolved
|
|
7391
|
+
)
|
|
7392
|
+
);
|
|
7393
|
+
}
|
|
7394
|
+
const mapped = mapOpenInferenceEvents(document);
|
|
7395
|
+
const warnings = attachSingleSourceFile(
|
|
7396
|
+
[...document.warnings, ...mapped.warnings],
|
|
7397
|
+
resolved
|
|
7398
|
+
);
|
|
7399
|
+
const unsupportedFields = [
|
|
7400
|
+
...document.unsupportedFields,
|
|
7401
|
+
...mapped.unsupportedFields
|
|
7402
|
+
].sort((a, b) => a.localeCompare(b));
|
|
7403
|
+
return {
|
|
7404
|
+
format: OPENINFERENCE_READER_FORMAT,
|
|
7405
|
+
events: mapped.events,
|
|
7406
|
+
runs: persistedInspectEventsToRunTrees(mapped.events, { skipInvalid: true }),
|
|
7407
|
+
warnings,
|
|
7408
|
+
unsupportedFields,
|
|
7409
|
+
sourceFiles: resolved.sourceFiles
|
|
7410
|
+
};
|
|
7411
|
+
}
|
|
7412
|
+
};
|
|
7413
|
+
function parseOtlpAnyValue(value, field, warnings, unsupportedFields) {
|
|
7414
|
+
if (!isRecord9(value)) {
|
|
7415
|
+
unsupportedFields.push(field);
|
|
7416
|
+
warnings.push({
|
|
7417
|
+
code: "otlp_attribute_value_invalid",
|
|
7418
|
+
message: "OTLP attribute value was not an AnyValue object.",
|
|
7419
|
+
severity: "warning",
|
|
7420
|
+
field
|
|
7421
|
+
});
|
|
7422
|
+
return void 0;
|
|
7423
|
+
}
|
|
7424
|
+
if (typeof value.stringValue === "string") return value.stringValue;
|
|
7425
|
+
if (typeof value.boolValue === "boolean") return value.boolValue;
|
|
7426
|
+
if (typeof value.intValue === "number" && Number.isFinite(value.intValue)) {
|
|
7427
|
+
return value.intValue;
|
|
7428
|
+
}
|
|
7429
|
+
if (typeof value.intValue === "string" && value.intValue.trim() !== "") {
|
|
7430
|
+
const n = Number(value.intValue);
|
|
7431
|
+
if (Number.isFinite(n)) return n;
|
|
7432
|
+
}
|
|
7433
|
+
if (typeof value.doubleValue === "number" && Number.isFinite(value.doubleValue)) {
|
|
7434
|
+
return value.doubleValue;
|
|
7435
|
+
}
|
|
7436
|
+
if (isRecord9(value.arrayValue) && Array.isArray(value.arrayValue.values)) {
|
|
7437
|
+
return value.arrayValue.values.map(
|
|
7438
|
+
(item, index) => parseOtlpAnyValue(item, `${field}.arrayValue.values[${index}]`, warnings, unsupportedFields)
|
|
7439
|
+
);
|
|
7440
|
+
}
|
|
7441
|
+
if (isRecord9(value.kvlistValue) && Array.isArray(value.kvlistValue.values)) {
|
|
7442
|
+
const out = {};
|
|
7443
|
+
for (const [index, item] of value.kvlistValue.values.entries()) {
|
|
7444
|
+
if (!isRecord9(item) || typeof item.key !== "string") {
|
|
7445
|
+
unsupportedFields.push(`${field}.kvlistValue.values[${index}]`);
|
|
7446
|
+
continue;
|
|
7447
|
+
}
|
|
7448
|
+
out[item.key] = parseOtlpAnyValue(
|
|
7449
|
+
item.value,
|
|
7450
|
+
`${field}.kvlistValue.values[${index}].value`,
|
|
7451
|
+
warnings,
|
|
7452
|
+
unsupportedFields
|
|
7453
|
+
);
|
|
7454
|
+
}
|
|
7455
|
+
return out;
|
|
7456
|
+
}
|
|
7457
|
+
if (typeof value.bytesValue === "string") {
|
|
7458
|
+
unsupportedFields.push(field);
|
|
7459
|
+
warnings.push({
|
|
7460
|
+
code: "otlp_bytes_value_summarized",
|
|
7461
|
+
message: "OTLP bytesValue attribute was summarized instead of decoded.",
|
|
7462
|
+
severity: "warning",
|
|
7463
|
+
field
|
|
7464
|
+
});
|
|
7465
|
+
return { type: "bytes", length: value.bytesValue.length };
|
|
7466
|
+
}
|
|
7467
|
+
unsupportedFields.push(field);
|
|
7468
|
+
warnings.push({
|
|
7469
|
+
code: "otlp_attribute_value_unsupported",
|
|
7470
|
+
message: "OTLP attribute value used an unsupported AnyValue shape.",
|
|
7471
|
+
severity: "warning",
|
|
7472
|
+
field
|
|
7473
|
+
});
|
|
7474
|
+
return void 0;
|
|
7475
|
+
}
|
|
7476
|
+
function parseOtlpAttributes(value, pathPrefix) {
|
|
7477
|
+
const attributes = {};
|
|
7478
|
+
const warnings = [];
|
|
7479
|
+
const unsupportedFields = [];
|
|
7480
|
+
if (value === void 0) {
|
|
7481
|
+
return { attributes, warnings, unsupportedFields };
|
|
7482
|
+
}
|
|
7483
|
+
if (!Array.isArray(value)) {
|
|
7484
|
+
unsupportedFields.push(pathPrefix);
|
|
7485
|
+
warnings.push({
|
|
7486
|
+
code: "otlp_attributes_invalid",
|
|
7487
|
+
message: "OTLP attributes field was not an array.",
|
|
7488
|
+
severity: "warning",
|
|
7489
|
+
field: pathPrefix
|
|
7490
|
+
});
|
|
7491
|
+
return { attributes, warnings, unsupportedFields };
|
|
7492
|
+
}
|
|
7493
|
+
for (const [index, item] of value.entries()) {
|
|
7494
|
+
const field = `${pathPrefix}[${index}]`;
|
|
7495
|
+
if (!isRecord9(item) || typeof item.key !== "string") {
|
|
7496
|
+
unsupportedFields.push(field);
|
|
7497
|
+
warnings.push({
|
|
7498
|
+
code: "otlp_attribute_invalid",
|
|
7499
|
+
message: "Skipped OTLP attribute without a string key.",
|
|
7500
|
+
severity: "warning",
|
|
7501
|
+
field
|
|
7502
|
+
});
|
|
7503
|
+
continue;
|
|
7504
|
+
}
|
|
7505
|
+
const parsed = parseOtlpAnyValue(
|
|
7506
|
+
item.value,
|
|
7507
|
+
`${field}.value`,
|
|
7508
|
+
warnings,
|
|
7509
|
+
unsupportedFields
|
|
7510
|
+
);
|
|
7511
|
+
if (parsed !== void 0) {
|
|
7512
|
+
attributes[item.key] = parsed;
|
|
7513
|
+
}
|
|
7514
|
+
}
|
|
7515
|
+
return { attributes, warnings, unsupportedFields };
|
|
7516
|
+
}
|
|
7517
|
+
function looksLikeOtlpSpan(value) {
|
|
7518
|
+
return isRecord9(value) && readStringField(value, ["traceId"]) !== void 0 && readStringField(value, ["spanId"]) !== void 0 && readStringField(value, ["name"]) !== void 0;
|
|
7519
|
+
}
|
|
7520
|
+
function extractOtlpDocument(root) {
|
|
7521
|
+
if (!isRecord9(root) || !Array.isArray(root.resourceSpans)) return void 0;
|
|
7522
|
+
const spans = [];
|
|
7523
|
+
const warnings = [];
|
|
7524
|
+
const unsupportedFields = [];
|
|
7525
|
+
for (const [resourceIndex, resourceSpan] of root.resourceSpans.entries()) {
|
|
7526
|
+
const resourcePath = `resourceSpans[${resourceIndex}]`;
|
|
7527
|
+
if (!isRecord9(resourceSpan)) {
|
|
7528
|
+
unsupportedFields.push(resourcePath);
|
|
7529
|
+
continue;
|
|
7530
|
+
}
|
|
7531
|
+
const resource = readRecordField(resourceSpan, "resource");
|
|
7532
|
+
const resourceParsed = parseOtlpAttributes(
|
|
7533
|
+
resource?.attributes,
|
|
7534
|
+
`${resourcePath}.resource.attributes`
|
|
7535
|
+
);
|
|
7536
|
+
warnings.push(...resourceParsed.warnings);
|
|
7537
|
+
unsupportedFields.push(...resourceParsed.unsupportedFields);
|
|
7538
|
+
if (!Array.isArray(resourceSpan.scopeSpans)) {
|
|
7539
|
+
unsupportedFields.push(`${resourcePath}.scopeSpans`);
|
|
7540
|
+
warnings.push({
|
|
7541
|
+
code: "otlp_scope_spans_missing",
|
|
7542
|
+
message: "OTLP resourceSpans entry did not contain a scopeSpans array.",
|
|
7543
|
+
severity: "warning",
|
|
7544
|
+
field: `${resourcePath}.scopeSpans`
|
|
7545
|
+
});
|
|
7546
|
+
continue;
|
|
7547
|
+
}
|
|
7548
|
+
for (const [scopeIndex, scopeSpan] of resourceSpan.scopeSpans.entries()) {
|
|
7549
|
+
const scopePath = `${resourcePath}.scopeSpans[${scopeIndex}]`;
|
|
7550
|
+
if (!isRecord9(scopeSpan)) {
|
|
7551
|
+
unsupportedFields.push(scopePath);
|
|
7552
|
+
continue;
|
|
7553
|
+
}
|
|
7554
|
+
const scope = readRecordField(scopeSpan, "scope");
|
|
7555
|
+
const scopeParsed = parseOtlpAttributes(
|
|
7556
|
+
scope?.attributes,
|
|
7557
|
+
`${scopePath}.scope.attributes`
|
|
7558
|
+
);
|
|
7559
|
+
warnings.push(...scopeParsed.warnings);
|
|
7560
|
+
unsupportedFields.push(...scopeParsed.unsupportedFields);
|
|
7561
|
+
if (!Array.isArray(scopeSpan.spans)) {
|
|
7562
|
+
unsupportedFields.push(`${scopePath}.spans`);
|
|
7563
|
+
warnings.push({
|
|
7564
|
+
code: "otlp_spans_missing",
|
|
7565
|
+
message: "OTLP scopeSpans entry did not contain a spans array.",
|
|
7566
|
+
severity: "warning",
|
|
7567
|
+
field: `${scopePath}.spans`
|
|
7568
|
+
});
|
|
7569
|
+
continue;
|
|
7570
|
+
}
|
|
7571
|
+
for (const [spanIndex, span] of scopeSpan.spans.entries()) {
|
|
7572
|
+
const spanPath = `${scopePath}.spans[${spanIndex}]`;
|
|
7573
|
+
if (!looksLikeOtlpSpan(span)) {
|
|
7574
|
+
unsupportedFields.push(spanPath);
|
|
7575
|
+
warnings.push({
|
|
7576
|
+
code: "otlp_invalid_span",
|
|
7577
|
+
message: "Skipped OTLP span without required traceId, spanId, or name.",
|
|
7578
|
+
severity: "warning",
|
|
7579
|
+
field: spanPath
|
|
7580
|
+
});
|
|
7581
|
+
continue;
|
|
7582
|
+
}
|
|
7583
|
+
spans.push({
|
|
7584
|
+
span,
|
|
7585
|
+
resourceAttributes: resourceParsed.attributes,
|
|
7586
|
+
scopeAttributes: scopeParsed.attributes,
|
|
7587
|
+
scopeName: readStringField(scope ?? {}, ["name"]),
|
|
7588
|
+
scopeVersion: readStringField(scope ?? {}, ["version"]),
|
|
7589
|
+
pathPrefix: spanPath
|
|
7590
|
+
});
|
|
7591
|
+
}
|
|
7592
|
+
}
|
|
7593
|
+
}
|
|
7594
|
+
if (spans.length === 0) {
|
|
7595
|
+
warnings.push({
|
|
7596
|
+
code: "otlp_no_valid_spans",
|
|
7597
|
+
message: "OTLP JSON payload did not contain any valid spans.",
|
|
7598
|
+
severity: "error"
|
|
7599
|
+
});
|
|
7600
|
+
return {
|
|
7601
|
+
spans,
|
|
7602
|
+
confidence: 0.7,
|
|
7603
|
+
description: "Malformed OTLP JSON trace payload",
|
|
7604
|
+
warnings,
|
|
7605
|
+
unsupportedFields
|
|
7606
|
+
};
|
|
7607
|
+
}
|
|
7608
|
+
return {
|
|
7609
|
+
spans,
|
|
7610
|
+
confidence: 0.93,
|
|
7611
|
+
description: "OTLP JSON trace payload",
|
|
7612
|
+
warnings,
|
|
7613
|
+
unsupportedFields
|
|
7614
|
+
};
|
|
7615
|
+
}
|
|
7616
|
+
function mapOtlpStatus(status) {
|
|
7617
|
+
if (!isRecord9(status)) return void 0;
|
|
7618
|
+
const rawCode = status.code;
|
|
7619
|
+
if (typeof rawCode !== "string") return void 0;
|
|
7620
|
+
switch (rawCode.toUpperCase()) {
|
|
7621
|
+
case "STATUS_CODE_OK":
|
|
7622
|
+
case "OK":
|
|
7623
|
+
return "ok";
|
|
7624
|
+
case "STATUS_CODE_ERROR":
|
|
7625
|
+
case "ERROR":
|
|
7626
|
+
return "error";
|
|
7627
|
+
case "STATUS_CODE_UNSET":
|
|
7628
|
+
case "UNSET":
|
|
7629
|
+
return "unknown";
|
|
7630
|
+
default:
|
|
7631
|
+
return "unknown";
|
|
7632
|
+
}
|
|
7633
|
+
}
|
|
7634
|
+
function readOtlpKind(attributes, pathPrefix) {
|
|
7635
|
+
const warnings = [];
|
|
7636
|
+
const agentInspectKind = attributes["agent_inspect.kind"];
|
|
7637
|
+
if (agentInspectKind === "RUN" || agentInspectKind === "AGENT" || agentInspectKind === "LLM" || agentInspectKind === "TOOL" || agentInspectKind === "CHAIN" || agentInspectKind === "RETRIEVER" || agentInspectKind === "DECISION" || agentInspectKind === "RESULT" || agentInspectKind === "ERROR" || agentInspectKind === "LOGIC" || agentInspectKind === "LOG" || agentInspectKind === "OUTCOME") {
|
|
7638
|
+
return { kind: agentInspectKind, warnings };
|
|
7639
|
+
}
|
|
7640
|
+
const operation = attributes["gen_ai.operation.name"];
|
|
7641
|
+
if (typeof operation === "string") {
|
|
7642
|
+
switch (operation) {
|
|
7643
|
+
case "generate_content":
|
|
7644
|
+
case "chat":
|
|
7645
|
+
return { kind: "LLM", warnings };
|
|
7646
|
+
case "execute_tool":
|
|
7647
|
+
return { kind: "TOOL", warnings };
|
|
7648
|
+
case "invoke_agent":
|
|
7649
|
+
return { kind: "AGENT", warnings };
|
|
7650
|
+
default:
|
|
7651
|
+
warnings.push({
|
|
7652
|
+
code: "otlp_gen_ai_operation_semantic_loss",
|
|
7653
|
+
message: `OTLP GenAI operation "${operation}" mapped to AgentInspect LOGIC.`,
|
|
7654
|
+
severity: "warning",
|
|
7655
|
+
field: `${pathPrefix}.attributes.gen_ai.operation.name`
|
|
7656
|
+
});
|
|
7657
|
+
return { kind: "LOGIC", warnings };
|
|
7658
|
+
}
|
|
7659
|
+
}
|
|
7660
|
+
warnings.push({
|
|
7661
|
+
code: "otlp_kind_unknown",
|
|
7662
|
+
message: "OTLP span had no AgentInspect kind or GenAI operation; mapped to LOGIC.",
|
|
7663
|
+
severity: "warning",
|
|
7664
|
+
field: `${pathPrefix}.attributes`
|
|
7665
|
+
});
|
|
7666
|
+
return { kind: "LOGIC", warnings };
|
|
7667
|
+
}
|
|
7668
|
+
function readOtlpTokenUsage(attributes) {
|
|
7669
|
+
const input = attributes["gen_ai.usage.input_tokens"];
|
|
7670
|
+
const output = attributes["gen_ai.usage.output_tokens"];
|
|
7671
|
+
const usage = {};
|
|
7672
|
+
if (typeof input === "number" && Number.isFinite(input) && input >= 0) {
|
|
7673
|
+
usage.input = input;
|
|
7674
|
+
}
|
|
7675
|
+
if (typeof output === "number" && Number.isFinite(output) && output >= 0) {
|
|
7676
|
+
usage.output = output;
|
|
7677
|
+
}
|
|
7678
|
+
if (usage.input !== void 0 && usage.output !== void 0) {
|
|
7679
|
+
usage.total = usage.input + usage.output;
|
|
7680
|
+
}
|
|
7681
|
+
return Object.keys(usage).length > 0 ? usage : void 0;
|
|
7682
|
+
}
|
|
7683
|
+
function readOtlpConfidence(attributes) {
|
|
7684
|
+
return readOpenInferenceConfidence(attributes);
|
|
7685
|
+
}
|
|
7686
|
+
function sanitizeOtlpAttributes(attributes, pathPrefix) {
|
|
7687
|
+
const ownerPath = pathPrefix.endsWith(".attributes") ? pathPrefix.slice(0, -".attributes".length) : pathPrefix;
|
|
7688
|
+
const sanitized = sanitizeOpenInferenceAttributes(attributes, ownerPath);
|
|
7689
|
+
return {
|
|
7690
|
+
...sanitized,
|
|
7691
|
+
warnings: sanitized.warnings.map(
|
|
7692
|
+
(warning) => warning.code === "openinference_sensitive_attribute_summarized" ? {
|
|
7693
|
+
...warning,
|
|
7694
|
+
code: "otlp_sensitive_attribute_summarized",
|
|
7695
|
+
message: "OTLP prompt/output/document attribute(s) were summarized instead of copied verbatim."
|
|
7696
|
+
} : warning
|
|
7697
|
+
)
|
|
7698
|
+
};
|
|
7699
|
+
}
|
|
7700
|
+
function mapOtlpEvents(value, pathPrefix) {
|
|
7701
|
+
const warnings = [];
|
|
7702
|
+
const unsupportedFields = [];
|
|
7703
|
+
if (value === void 0) return { warnings, unsupportedFields };
|
|
7704
|
+
if (!Array.isArray(value)) {
|
|
7705
|
+
unsupportedFields.push(pathPrefix);
|
|
7706
|
+
warnings.push({
|
|
7707
|
+
code: "otlp_events_invalid",
|
|
7708
|
+
message: "OTLP events field was not an array.",
|
|
7709
|
+
severity: "warning",
|
|
7710
|
+
field: pathPrefix
|
|
7711
|
+
});
|
|
7712
|
+
return { warnings, unsupportedFields };
|
|
7713
|
+
}
|
|
7714
|
+
const events = [];
|
|
7715
|
+
for (const [index, event] of value.entries()) {
|
|
7716
|
+
const eventPath = `${pathPrefix}[${index}]`;
|
|
7717
|
+
if (!isRecord9(event)) {
|
|
7718
|
+
unsupportedFields.push(eventPath);
|
|
7719
|
+
continue;
|
|
7720
|
+
}
|
|
7721
|
+
const parsedAttributes = parseOtlpAttributes(
|
|
7722
|
+
event.attributes,
|
|
7723
|
+
`${eventPath}.attributes`
|
|
7724
|
+
);
|
|
7725
|
+
warnings.push(...parsedAttributes.warnings);
|
|
7726
|
+
unsupportedFields.push(...parsedAttributes.unsupportedFields);
|
|
7727
|
+
const sanitized = sanitizeOtlpAttributes(
|
|
7728
|
+
parsedAttributes.attributes,
|
|
7729
|
+
`${eventPath}.attributes`
|
|
7730
|
+
);
|
|
7731
|
+
warnings.push(...sanitized.warnings);
|
|
7732
|
+
unsupportedFields.push(...sanitized.unsupportedFields);
|
|
7733
|
+
const out = {};
|
|
7734
|
+
const name = readStringField(event, ["name"]);
|
|
7735
|
+
if (name !== void 0) {
|
|
7736
|
+
out.name = name;
|
|
7737
|
+
}
|
|
7738
|
+
const timestamp = parseUnixNanoToIso(event.timeUnixNano);
|
|
7739
|
+
if (timestamp !== void 0) {
|
|
7740
|
+
out.timestamp = timestamp;
|
|
7741
|
+
} else if (event.timeUnixNano !== void 0) {
|
|
7742
|
+
unsupportedFields.push(`${eventPath}.timeUnixNano`);
|
|
7743
|
+
warnings.push({
|
|
7744
|
+
code: "otlp_event_timestamp_invalid",
|
|
7745
|
+
message: "OTLP event timeUnixNano could not be parsed.",
|
|
7746
|
+
severity: "warning",
|
|
7747
|
+
field: `${eventPath}.timeUnixNano`
|
|
7748
|
+
});
|
|
7749
|
+
}
|
|
7750
|
+
if (Object.keys(sanitized.attributes).length > 0) {
|
|
7751
|
+
out.attributes = sanitized.attributes;
|
|
7752
|
+
}
|
|
7753
|
+
events.push(out);
|
|
7754
|
+
}
|
|
7755
|
+
return {
|
|
7756
|
+
events: events.length > 0 ? events : void 0,
|
|
7757
|
+
warnings,
|
|
7758
|
+
unsupportedFields
|
|
7759
|
+
};
|
|
7760
|
+
}
|
|
7761
|
+
function mapOtlpSpan(context) {
|
|
7762
|
+
const { span, pathPrefix } = context;
|
|
7763
|
+
const warnings = [];
|
|
7764
|
+
const unsupportedFields = [];
|
|
7765
|
+
const parsedSpanAttributes = parseOtlpAttributes(
|
|
7766
|
+
span.attributes,
|
|
7767
|
+
`${pathPrefix}.attributes`
|
|
7768
|
+
);
|
|
7769
|
+
warnings.push(...parsedSpanAttributes.warnings);
|
|
7770
|
+
unsupportedFields.push(...parsedSpanAttributes.unsupportedFields);
|
|
7771
|
+
const sanitizedSpanAttributes = sanitizeOtlpAttributes(
|
|
7772
|
+
parsedSpanAttributes.attributes,
|
|
7773
|
+
`${pathPrefix}.attributes`
|
|
7774
|
+
);
|
|
7775
|
+
warnings.push(...sanitizedSpanAttributes.warnings);
|
|
7776
|
+
unsupportedFields.push(...sanitizedSpanAttributes.unsupportedFields);
|
|
7777
|
+
const attributes = {
|
|
7778
|
+
...sanitizedSpanAttributes.attributes
|
|
7779
|
+
};
|
|
7780
|
+
for (const [key, value] of Object.entries(context.resourceAttributes)) {
|
|
7781
|
+
attributes[`resource.${key}`] = value;
|
|
7782
|
+
}
|
|
7783
|
+
for (const [key, value] of Object.entries(context.scopeAttributes)) {
|
|
7784
|
+
attributes[`scope.${key}`] = value;
|
|
7785
|
+
}
|
|
7786
|
+
if (context.scopeName !== void 0) {
|
|
7787
|
+
attributes["scope.name"] = context.scopeName;
|
|
7788
|
+
}
|
|
7789
|
+
if (context.scopeVersion !== void 0) {
|
|
7790
|
+
attributes["scope.version"] = context.scopeVersion;
|
|
7791
|
+
}
|
|
7792
|
+
for (const [key, value] of Object.entries(span)) {
|
|
7793
|
+
if (OTLP_SPAN_KEYS.has(key)) continue;
|
|
7794
|
+
unsupportedFields.push(`${pathPrefix}.${key}`);
|
|
7795
|
+
if (value === null || typeof value !== "object") {
|
|
7796
|
+
attributes[`otlp.${key}`] = value;
|
|
7797
|
+
} else {
|
|
7798
|
+
attributes[`otlp.${key}.summary`] = summarizeAttributeValue(value);
|
|
7799
|
+
warnings.push({
|
|
7800
|
+
code: "otlp_unsupported_field_summarized",
|
|
7801
|
+
message: `Unsupported OTLP span field "${key}" was summarized.`,
|
|
7802
|
+
severity: "warning",
|
|
7803
|
+
field: `${pathPrefix}.${key}`
|
|
7804
|
+
});
|
|
7805
|
+
}
|
|
7806
|
+
}
|
|
7807
|
+
for (const key of [
|
|
7808
|
+
"droppedAttributesCount",
|
|
7809
|
+
"droppedEventsCount",
|
|
7810
|
+
"droppedLinksCount",
|
|
7811
|
+
"links"
|
|
7812
|
+
]) {
|
|
7813
|
+
if (span[key] !== void 0) {
|
|
7814
|
+
unsupportedFields.push(`${pathPrefix}.${key}`);
|
|
7815
|
+
warnings.push({
|
|
7816
|
+
code: "otlp_span_field_not_mapped",
|
|
7817
|
+
message: `OTLP span field "${key}" is not represented in AgentInspect events.`,
|
|
7818
|
+
severity: "warning",
|
|
7819
|
+
field: `${pathPrefix}.${key}`
|
|
7820
|
+
});
|
|
7821
|
+
}
|
|
7822
|
+
}
|
|
7823
|
+
const events = mapOtlpEvents(span.events, `${pathPrefix}.events`);
|
|
7824
|
+
warnings.push(...events.warnings);
|
|
7825
|
+
unsupportedFields.push(...events.unsupportedFields);
|
|
7826
|
+
if (events.events !== void 0) {
|
|
7827
|
+
attributes["otlp.events"] = events.events;
|
|
7828
|
+
}
|
|
7829
|
+
const traceId = readStringField(span, ["traceId"]) ?? "trace-unknown";
|
|
7830
|
+
const spanId = readStringField(span, ["spanId"]) ?? "span-unknown";
|
|
7831
|
+
const parentSpanId = readStringField(span, ["parentSpanId"]);
|
|
7832
|
+
const startedAt = readOpenInferenceTimestamp(
|
|
7833
|
+
span,
|
|
7834
|
+
["startTimeUnixNano"],
|
|
7835
|
+
[]
|
|
7836
|
+
);
|
|
7837
|
+
const endedAt = readOpenInferenceTimestamp(span, ["endTimeUnixNano"], []);
|
|
7838
|
+
const timestamp = startedAt ?? "1970-01-01T00:00:00.000Z";
|
|
7839
|
+
if (startedAt === void 0) {
|
|
7840
|
+
unsupportedFields.push(`${pathPrefix}.startTimeUnixNano`);
|
|
7841
|
+
warnings.push({
|
|
7842
|
+
code: "otlp_missing_start_time",
|
|
7843
|
+
message: "OTLP span is missing a valid startTimeUnixNano; using Unix epoch.",
|
|
7844
|
+
severity: "warning",
|
|
7845
|
+
field: `${pathPrefix}.startTimeUnixNano`
|
|
7846
|
+
});
|
|
7847
|
+
}
|
|
7848
|
+
const { kind, warnings: kindWarnings } = readOtlpKind(
|
|
7849
|
+
parsedSpanAttributes.attributes,
|
|
7850
|
+
pathPrefix
|
|
7851
|
+
);
|
|
7852
|
+
warnings.push(...kindWarnings);
|
|
7853
|
+
const status = mapOtlpStatus(span.status);
|
|
7854
|
+
const tokenUsage = readOtlpTokenUsage(parsedSpanAttributes.attributes);
|
|
7855
|
+
const errorMessage = isRecord9(span.status) && typeof span.status.message === "string" ? span.status.message : void 0;
|
|
7856
|
+
const event = {
|
|
7857
|
+
schemaVersion: "0.2",
|
|
7858
|
+
eventId: typeof parsedSpanAttributes.attributes["agent_inspect.event_id"] === "string" ? parsedSpanAttributes.attributes["agent_inspect.event_id"] : spanId,
|
|
7859
|
+
runId: typeof parsedSpanAttributes.attributes["agent_inspect.run_id"] === "string" ? parsedSpanAttributes.attributes["agent_inspect.run_id"] : traceId,
|
|
7860
|
+
kind,
|
|
7861
|
+
name: readStringField(span, ["name"]) ?? spanId,
|
|
7862
|
+
timestamp,
|
|
7863
|
+
confidence: readOtlpConfidence(parsedSpanAttributes.attributes),
|
|
7864
|
+
source: {
|
|
7865
|
+
type: "otel",
|
|
7866
|
+
name: context.scopeName ?? (typeof context.resourceAttributes["service.name"] === "string" ? context.resourceAttributes["service.name"] : "otlp-json"),
|
|
7867
|
+
...context.scopeVersion !== void 0 ? { version: context.scopeVersion } : {}
|
|
7868
|
+
},
|
|
7869
|
+
attributes,
|
|
7870
|
+
trace: {
|
|
7871
|
+
traceId,
|
|
7872
|
+
spanId,
|
|
7873
|
+
...parentSpanId !== void 0 ? { parentSpanId } : {}
|
|
7874
|
+
}
|
|
7875
|
+
};
|
|
7876
|
+
if (status !== void 0) {
|
|
7877
|
+
event.status = status;
|
|
7878
|
+
}
|
|
7879
|
+
if (startedAt !== void 0) {
|
|
7880
|
+
event.startedAt = startedAt;
|
|
7881
|
+
}
|
|
7882
|
+
if (endedAt !== void 0) {
|
|
7883
|
+
event.endedAt = endedAt;
|
|
7884
|
+
}
|
|
7885
|
+
const durationMs2 = durationBetweenIso(startedAt, endedAt);
|
|
7886
|
+
if (durationMs2 !== void 0) {
|
|
7887
|
+
event.durationMs = durationMs2;
|
|
7888
|
+
}
|
|
7889
|
+
if (tokenUsage !== void 0) {
|
|
7890
|
+
event.tokenUsage = tokenUsage;
|
|
7891
|
+
}
|
|
7892
|
+
if (status === "error") {
|
|
7893
|
+
event.error = {
|
|
7894
|
+
message: errorMessage !== void 0 && errorMessage.trim() !== "" ? errorMessage : "OTLP span error"
|
|
7895
|
+
};
|
|
7896
|
+
}
|
|
7897
|
+
return {
|
|
7898
|
+
event,
|
|
7899
|
+
warnings,
|
|
7900
|
+
unsupportedFields,
|
|
7901
|
+
spanId,
|
|
7902
|
+
...parentSpanId !== void 0 ? { parentSpanId } : {}
|
|
7903
|
+
};
|
|
7904
|
+
}
|
|
7905
|
+
function mapOtlpEventsToPersisted(document) {
|
|
7906
|
+
const mapped = document.spans.map((span) => mapOtlpSpan(span));
|
|
7907
|
+
const spanIdToEventId = new Map(
|
|
7908
|
+
mapped.map((span) => [span.spanId, span.event.eventId])
|
|
7909
|
+
);
|
|
7910
|
+
for (const span of mapped) {
|
|
7911
|
+
if (span.parentSpanId === void 0) continue;
|
|
7912
|
+
span.event.parentId = spanIdToEventId.get(span.parentSpanId) ?? span.parentSpanId;
|
|
7913
|
+
}
|
|
7914
|
+
return {
|
|
7915
|
+
events: mapped.map((span) => span.event),
|
|
7916
|
+
warnings: mapped.flatMap((span) => span.warnings),
|
|
7917
|
+
unsupportedFields: mapped.flatMap((span) => span.unsupportedFields)
|
|
7918
|
+
};
|
|
7919
|
+
}
|
|
7920
|
+
var otlpJsonReader = {
|
|
7921
|
+
format: OTLP_READER_FORMAT,
|
|
7922
|
+
name: "OTLP JSON",
|
|
7923
|
+
async detect(input) {
|
|
7924
|
+
const resolved = await resolveInput(input);
|
|
7925
|
+
if (!resolved) return void 0;
|
|
7926
|
+
let parsed;
|
|
7927
|
+
try {
|
|
7928
|
+
parsed = parseJsonDocument(resolved.content);
|
|
7929
|
+
} catch {
|
|
7930
|
+
return void 0;
|
|
7931
|
+
}
|
|
7932
|
+
const document = extractOtlpDocument(parsed);
|
|
7933
|
+
if (!document) return void 0;
|
|
7934
|
+
return {
|
|
7935
|
+
format: OTLP_READER_FORMAT,
|
|
7936
|
+
confidence: document.confidence,
|
|
7937
|
+
readerName: "OTLP JSON",
|
|
7938
|
+
description: document.description,
|
|
7939
|
+
warnings: attachSingleSourceFile(document.warnings, resolved)
|
|
7940
|
+
};
|
|
7941
|
+
},
|
|
7942
|
+
async read(input) {
|
|
7943
|
+
const resolved = await resolveInput(input);
|
|
7944
|
+
if (!resolved) {
|
|
7945
|
+
throw new TraceReadError(
|
|
7946
|
+
"unsupported_format",
|
|
7947
|
+
"OTLP JSON reader requires file, string, or buffer input."
|
|
7948
|
+
);
|
|
7949
|
+
}
|
|
7950
|
+
let parsed;
|
|
7951
|
+
try {
|
|
7952
|
+
parsed = parseJsonDocument(resolved.content);
|
|
7953
|
+
} catch {
|
|
7954
|
+
throw new TraceReadError("unsupported_format", "OTLP JSON input is not valid JSON.", [
|
|
7955
|
+
{
|
|
7956
|
+
code: "otlp_invalid_json",
|
|
7957
|
+
message: "OTLP JSON reader could not parse the input as JSON.",
|
|
7958
|
+
severity: "error"
|
|
7959
|
+
}
|
|
7960
|
+
]);
|
|
7961
|
+
}
|
|
7962
|
+
const document = extractOtlpDocument(parsed);
|
|
7963
|
+
if (!document || document.spans.length === 0) {
|
|
7964
|
+
throw new TraceReadError(
|
|
7965
|
+
"unsupported_format",
|
|
7966
|
+
"No valid OTLP spans found.",
|
|
7967
|
+
attachSingleSourceFile(
|
|
7968
|
+
document?.warnings ?? [
|
|
7969
|
+
{
|
|
7970
|
+
code: "otlp_no_valid_spans",
|
|
7971
|
+
message: "OTLP JSON input did not contain valid spans.",
|
|
7972
|
+
severity: "error"
|
|
7973
|
+
}
|
|
7974
|
+
],
|
|
7975
|
+
resolved
|
|
7976
|
+
)
|
|
7977
|
+
);
|
|
7978
|
+
}
|
|
7979
|
+
const mapped = mapOtlpEventsToPersisted(document);
|
|
7980
|
+
const warnings = attachSingleSourceFile(
|
|
7981
|
+
[...document.warnings, ...mapped.warnings],
|
|
7982
|
+
resolved
|
|
7983
|
+
);
|
|
7984
|
+
const unsupportedFields = [
|
|
7985
|
+
...document.unsupportedFields,
|
|
7986
|
+
...mapped.unsupportedFields
|
|
7987
|
+
].sort((a, b) => a.localeCompare(b));
|
|
7988
|
+
return {
|
|
7989
|
+
format: OTLP_READER_FORMAT,
|
|
7990
|
+
events: mapped.events,
|
|
7991
|
+
runs: persistedInspectEventsToRunTrees(mapped.events, { skipInvalid: true }),
|
|
7992
|
+
warnings,
|
|
7993
|
+
unsupportedFields,
|
|
7994
|
+
sourceFiles: resolved.sourceFiles
|
|
7995
|
+
};
|
|
7996
|
+
}
|
|
7997
|
+
};
|
|
7998
|
+
var agentInspectJsonlReader = {
|
|
7999
|
+
format: "agent-inspect-jsonl",
|
|
8000
|
+
name: "AgentInspect JSONL",
|
|
8001
|
+
async detect(input) {
|
|
8002
|
+
const resolved = await resolveInput(input);
|
|
8003
|
+
if (!resolved) return void 0;
|
|
8004
|
+
const detected = detectJsonlFormat(resolved.content);
|
|
8005
|
+
if (detected.validRows === 0 || detected.format === "empty") {
|
|
8006
|
+
return void 0;
|
|
8007
|
+
}
|
|
8008
|
+
return {
|
|
8009
|
+
format: "agent-inspect-jsonl",
|
|
8010
|
+
confidence: 0.95,
|
|
8011
|
+
readerName: "AgentInspect JSONL",
|
|
8012
|
+
description: agentInspectFormatLabel(detected.format),
|
|
8013
|
+
warnings: attachSingleSourceFile(detected.warnings, resolved)
|
|
8014
|
+
};
|
|
8015
|
+
},
|
|
8016
|
+
async read(input) {
|
|
8017
|
+
const resolved = await resolveInput(input);
|
|
8018
|
+
if (!resolved) {
|
|
8019
|
+
throw new Error("AgentInspect JSONL reader requires file, directory, string, or buffer input.");
|
|
8020
|
+
}
|
|
8021
|
+
const parsed = parseTraceJsonl(resolved.content, { warnings: false });
|
|
8022
|
+
if (parsed.sourceEventCount === 0) {
|
|
8023
|
+
throw new Error("No valid AgentInspect JSONL events found.");
|
|
8024
|
+
}
|
|
8025
|
+
const events = persistedEventsForParsedTrace(parsed);
|
|
8026
|
+
return {
|
|
8027
|
+
format: agentInspectFormatLabel(parsed.format),
|
|
8028
|
+
events,
|
|
8029
|
+
runs: persistedInspectEventsToRunTrees(events, { skipInvalid: true }),
|
|
8030
|
+
warnings: parsed.format === "mixed" ? attachSingleSourceFile(
|
|
8031
|
+
[
|
|
8032
|
+
{
|
|
8033
|
+
code: "mixed_agent_inspect_jsonl",
|
|
8034
|
+
message: "Trace input mixes schemaVersion 0.1 and 0.2 rows; events were normalized for reading.",
|
|
8035
|
+
severity: "warning"
|
|
8036
|
+
}
|
|
8037
|
+
],
|
|
8038
|
+
resolved
|
|
8039
|
+
) : [],
|
|
8040
|
+
unsupportedFields: [],
|
|
8041
|
+
sourceFiles: resolved.sourceFiles
|
|
8042
|
+
};
|
|
8043
|
+
}
|
|
8044
|
+
};
|
|
8045
|
+
var DEFAULT_TRACE_READERS = [
|
|
8046
|
+
agentInspectJsonlReader,
|
|
8047
|
+
openInferenceJsonReader,
|
|
8048
|
+
otlpJsonReader
|
|
8049
|
+
];
|
|
8050
|
+
async function detectTraceFormat(input, options = {}) {
|
|
8051
|
+
const readers = options.readers ?? DEFAULT_TRACE_READERS;
|
|
8052
|
+
if (options.format !== void 0) {
|
|
8053
|
+
const reader = findReaderByFormat(options.format, readers);
|
|
8054
|
+
if (!reader) {
|
|
8055
|
+
return {
|
|
8056
|
+
status: "unsupported",
|
|
8057
|
+
candidates: [],
|
|
8058
|
+
warnings: [
|
|
8059
|
+
{
|
|
8060
|
+
code: "unsupported_format",
|
|
8061
|
+
message: `No trace reader is registered for format "${options.format}".`,
|
|
8062
|
+
severity: "error"
|
|
8063
|
+
}
|
|
8064
|
+
]
|
|
8065
|
+
};
|
|
8066
|
+
}
|
|
8067
|
+
return {
|
|
8068
|
+
status: "detected",
|
|
8069
|
+
format: reader.format,
|
|
8070
|
+
candidates: [
|
|
8071
|
+
{
|
|
8072
|
+
format: reader.format,
|
|
8073
|
+
confidence: 1,
|
|
8074
|
+
readerName: reader.name,
|
|
8075
|
+
description: "Explicit format override"
|
|
8076
|
+
}
|
|
8077
|
+
],
|
|
8078
|
+
warnings: []
|
|
8079
|
+
};
|
|
8080
|
+
}
|
|
8081
|
+
const candidates = [];
|
|
8082
|
+
const warnings = [];
|
|
8083
|
+
for (const reader of readers) {
|
|
8084
|
+
try {
|
|
8085
|
+
const candidate = await reader.detect(input);
|
|
8086
|
+
if (candidate !== void 0) {
|
|
8087
|
+
candidates.push(normalizeCandidate(reader, candidate));
|
|
8088
|
+
}
|
|
8089
|
+
} catch (error) {
|
|
8090
|
+
if (error instanceof TraceReadError) {
|
|
8091
|
+
warnings.push(...error.warnings);
|
|
8092
|
+
continue;
|
|
8093
|
+
}
|
|
8094
|
+
warnings.push({
|
|
8095
|
+
code: "reader_detect_failed",
|
|
8096
|
+
message: error instanceof Error && error.message.trim() !== "" ? error.message : `Trace reader "${reader.format}" failed during detection.`,
|
|
8097
|
+
severity: "warning"
|
|
8098
|
+
});
|
|
8099
|
+
}
|
|
8100
|
+
}
|
|
8101
|
+
const sorted = sortCandidates(
|
|
8102
|
+
candidates.filter((candidate) => candidate.confidence >= MIN_DETECTION_CONFIDENCE)
|
|
8103
|
+
);
|
|
8104
|
+
const candidateWarnings = collectWarnings(sorted);
|
|
8105
|
+
const lowConfidenceWarnings = candidates.length > sorted.length ? [
|
|
8106
|
+
{
|
|
8107
|
+
code: "low_confidence_candidates",
|
|
8108
|
+
message: `Ignored ${candidates.length - sorted.length} low-confidence format candidate(s).`,
|
|
8109
|
+
severity: "info"
|
|
8110
|
+
}
|
|
8111
|
+
] : [];
|
|
8112
|
+
const allWarnings = dedupeWarnings([
|
|
8113
|
+
...warnings,
|
|
8114
|
+
...candidateWarnings,
|
|
8115
|
+
...lowConfidenceWarnings
|
|
8116
|
+
]);
|
|
8117
|
+
if (sorted.length === 0) {
|
|
8118
|
+
return {
|
|
8119
|
+
status: "unsupported",
|
|
8120
|
+
candidates: [],
|
|
8121
|
+
warnings: allWarnings
|
|
8122
|
+
};
|
|
8123
|
+
}
|
|
8124
|
+
const [best, second] = sorted;
|
|
8125
|
+
if (second !== void 0 && best.confidence - second.confidence <= AMBIGUOUS_CONFIDENCE_DELTA) {
|
|
8126
|
+
return {
|
|
8127
|
+
status: "ambiguous",
|
|
8128
|
+
candidates: sorted,
|
|
8129
|
+
warnings: [
|
|
8130
|
+
...allWarnings,
|
|
8131
|
+
{
|
|
8132
|
+
code: "ambiguous_format_candidates",
|
|
8133
|
+
message: `Top trace format candidates are within ${AMBIGUOUS_CONFIDENCE_DELTA} confidence.`,
|
|
8134
|
+
severity: "warning"
|
|
8135
|
+
}
|
|
8136
|
+
]
|
|
8137
|
+
};
|
|
8138
|
+
}
|
|
8139
|
+
return {
|
|
8140
|
+
status: "detected",
|
|
8141
|
+
format: best.format,
|
|
8142
|
+
candidates: sorted,
|
|
8143
|
+
warnings: allWarnings
|
|
8144
|
+
};
|
|
8145
|
+
}
|
|
8146
|
+
async function readTrace(input, options = {}) {
|
|
8147
|
+
const readers = options.readers ?? DEFAULT_TRACE_READERS;
|
|
8148
|
+
const detection = await detectTraceFormat(input, options);
|
|
8149
|
+
if (detection.status === "unsupported" || detection.format === void 0) {
|
|
8150
|
+
throw new TraceReadError(
|
|
8151
|
+
"unsupported_format",
|
|
8152
|
+
"No trace reader could detect the input format.",
|
|
8153
|
+
detection.warnings
|
|
8154
|
+
);
|
|
8155
|
+
}
|
|
8156
|
+
if (detection.status === "ambiguous") {
|
|
8157
|
+
throw new TraceReadError(
|
|
8158
|
+
"ambiguous_format",
|
|
8159
|
+
"Multiple trace readers matched the input with equal confidence.",
|
|
8160
|
+
detection.warnings
|
|
8161
|
+
);
|
|
8162
|
+
}
|
|
8163
|
+
const reader = findReaderByFormat(detection.format, readers);
|
|
8164
|
+
if (!reader) {
|
|
8165
|
+
throw new TraceReadError(
|
|
8166
|
+
"unsupported_format",
|
|
8167
|
+
`No trace reader is registered for format "${detection.format}".`,
|
|
8168
|
+
detection.warnings
|
|
8169
|
+
);
|
|
8170
|
+
}
|
|
8171
|
+
try {
|
|
8172
|
+
const result = await reader.read(input, { format: detection.format });
|
|
8173
|
+
return {
|
|
8174
|
+
...result,
|
|
8175
|
+
format: result.format || detection.format,
|
|
8176
|
+
warnings: [...detection.warnings, ...result.warnings]
|
|
8177
|
+
};
|
|
8178
|
+
} catch (error) {
|
|
8179
|
+
if (error instanceof TraceReadError) {
|
|
8180
|
+
throw new TraceReadError(
|
|
8181
|
+
error.code,
|
|
8182
|
+
error.message,
|
|
8183
|
+
dedupeWarnings([...detection.warnings, ...error.warnings])
|
|
8184
|
+
);
|
|
8185
|
+
}
|
|
8186
|
+
throw new TraceReadError(
|
|
8187
|
+
"reader_failed",
|
|
8188
|
+
error instanceof Error && error.message.trim() !== "" ? error.message : `Trace reader "${reader.format}" failed.`,
|
|
8189
|
+
detection.warnings
|
|
8190
|
+
);
|
|
8191
|
+
}
|
|
8192
|
+
}
|
|
8193
|
+
function openTrace(input, options = {}) {
|
|
8194
|
+
return readTrace(input, options);
|
|
8195
|
+
}
|
|
8196
|
+
|
|
8197
|
+
// packages/core/src/suite/run.ts
|
|
8198
|
+
function diagnostic4(code, message, severity = "error", caseId) {
|
|
8199
|
+
return { code, message, severity, ...caseId !== void 0 ? { caseId } : {} };
|
|
8200
|
+
}
|
|
8201
|
+
function buildCaseRules(suiteCase, config) {
|
|
8202
|
+
const rules = [];
|
|
8203
|
+
const select = new Set(config.checks?.select ?? []);
|
|
8204
|
+
if (select.has("run.status")) {
|
|
8205
|
+
rules.push(createRunStatusRule());
|
|
8206
|
+
}
|
|
8207
|
+
const requiredTools = [
|
|
8208
|
+
...config.checks?.tool?.required ?? [],
|
|
8209
|
+
...suiteCase.requireTools ?? []
|
|
8210
|
+
];
|
|
8211
|
+
const forbiddenTools = [
|
|
8212
|
+
...config.checks?.tool?.forbidden ?? [],
|
|
8213
|
+
...suiteCase.forbidTools ?? []
|
|
8214
|
+
];
|
|
8215
|
+
if (requiredTools.length > 0 || forbiddenTools.length > 0) {
|
|
8216
|
+
rules.push(
|
|
8217
|
+
createToolUsageRule({
|
|
8218
|
+
required: requiredTools.length > 0 ? requiredTools : void 0,
|
|
8219
|
+
forbidden: forbiddenTools.length > 0 ? forbiddenTools : void 0
|
|
8220
|
+
})
|
|
8221
|
+
);
|
|
8222
|
+
select.add("tool.usage");
|
|
8223
|
+
}
|
|
8224
|
+
const maxDurationMs = suiteCase.maxDurationMs ?? config.checks?.run?.maxDurationMs ?? config.eval?.maxDurationMs;
|
|
8225
|
+
if (maxDurationMs !== void 0) {
|
|
8226
|
+
rules.push(createRunDurationRule({ maxDurationMs }));
|
|
8227
|
+
select.add("run.duration");
|
|
8228
|
+
}
|
|
8229
|
+
const llm = config.checks?.llm;
|
|
8230
|
+
if (llm?.allowedModels !== void 0 || llm?.maxTotalTokens !== void 0) {
|
|
8231
|
+
rules.push(createLlmUsageRule(llm));
|
|
8232
|
+
select.add("llm.usage");
|
|
8233
|
+
}
|
|
8234
|
+
if (select.has("outcome.status")) {
|
|
8235
|
+
rules.push(createObservedOutcomeRule({ failOn: ["failed"] }));
|
|
8236
|
+
}
|
|
8237
|
+
return { rules, select: [...select] };
|
|
8238
|
+
}
|
|
8239
|
+
function outcomesFromRead(read) {
|
|
8240
|
+
const persistedOutcomes = extractOutcomesFromPersistedEvents(
|
|
8241
|
+
read.events.filter((event) => event.kind === "OUTCOME")
|
|
8242
|
+
);
|
|
8243
|
+
if (persistedOutcomes.length > 0) return persistedOutcomes;
|
|
8244
|
+
const traceEvents = [];
|
|
8245
|
+
for (const event of read.events) {
|
|
8246
|
+
const legacy = event;
|
|
8247
|
+
if (typeof legacy === "object" && legacy !== null && "event" in legacy && legacy.event === "outcome_observed") {
|
|
8248
|
+
traceEvents.push(legacy);
|
|
8249
|
+
}
|
|
8250
|
+
}
|
|
8251
|
+
return traceEvents.length > 0 ? extractOutcomesFromTraceEvents(traceEvents) : [];
|
|
8252
|
+
}
|
|
8253
|
+
function validateExpectedObservations(suiteCase, read) {
|
|
8254
|
+
const expected = suiteCase.expectedObservations ?? [];
|
|
8255
|
+
if (expected.length === 0) return { ok: true, diagnostics: [] };
|
|
8256
|
+
const outcomes = outcomesFromRead(read);
|
|
8257
|
+
const diagnostics = [];
|
|
8258
|
+
for (const name of expected) {
|
|
8259
|
+
const match = outcomes.find((outcome) => outcome.name === name);
|
|
8260
|
+
if (!match) {
|
|
8261
|
+
diagnostics.push(
|
|
8262
|
+
diagnostic4(
|
|
8263
|
+
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
8264
|
+
`Expected observation "${name}" was not recorded.`,
|
|
8265
|
+
"error",
|
|
8266
|
+
suiteCase.id
|
|
8267
|
+
)
|
|
8268
|
+
);
|
|
8269
|
+
continue;
|
|
8270
|
+
}
|
|
8271
|
+
if (match.status !== "passed") {
|
|
8272
|
+
diagnostics.push(
|
|
8273
|
+
diagnostic4(
|
|
8274
|
+
"AI_SUITE_CASE_OBSERVATION_FAILED",
|
|
8275
|
+
`Observation "${name}" has status "${match.status}", expected "passed".`,
|
|
8276
|
+
"error",
|
|
8277
|
+
suiteCase.id
|
|
8278
|
+
)
|
|
8279
|
+
);
|
|
8280
|
+
}
|
|
8281
|
+
}
|
|
8282
|
+
return { ok: diagnostics.length === 0, diagnostics };
|
|
8283
|
+
}
|
|
8284
|
+
async function runSuiteCase(suiteCase, config, options) {
|
|
8285
|
+
const resolved = await resolveSuiteCaseTrace(suiteCase, options);
|
|
8286
|
+
if (resolved.missing || resolved.tracePath === void 0) {
|
|
8287
|
+
return {
|
|
8288
|
+
id: suiteCase.id,
|
|
8289
|
+
status: "skipped",
|
|
8290
|
+
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
8291
|
+
message: resolved.reason,
|
|
8292
|
+
diagnostics: [
|
|
8293
|
+
diagnostic4(
|
|
8294
|
+
"AI_SUITE_CASE_TRACE_MISSING",
|
|
8295
|
+
resolved.reason ?? "Trace not found.",
|
|
8296
|
+
"warning",
|
|
8297
|
+
suiteCase.id
|
|
8298
|
+
)
|
|
8299
|
+
]
|
|
8300
|
+
};
|
|
8301
|
+
}
|
|
8302
|
+
let read;
|
|
8303
|
+
try {
|
|
8304
|
+
read = await openTrace({ type: "file", path: resolved.tracePath });
|
|
8305
|
+
} catch (error) {
|
|
8306
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
8307
|
+
return {
|
|
8308
|
+
id: suiteCase.id,
|
|
8309
|
+
status: "error",
|
|
8310
|
+
tracePath: resolved.tracePath,
|
|
8311
|
+
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
8312
|
+
message,
|
|
8313
|
+
diagnostics: [
|
|
8314
|
+
diagnostic4("AI_SUITE_TRACE_UNREADABLE", message, "error", suiteCase.id)
|
|
8315
|
+
]
|
|
8316
|
+
};
|
|
8317
|
+
}
|
|
8318
|
+
const { rules, select } = buildCaseRules(suiteCase, config);
|
|
8319
|
+
const checkResult = rules.length > 0 ? runTraceChecks({ read }, { rules, select }) : {
|
|
8320
|
+
ok: true,
|
|
8321
|
+
status: "pass",
|
|
8322
|
+
format: read.format,
|
|
8323
|
+
findings: [],
|
|
8324
|
+
diagnostics: []
|
|
8325
|
+
};
|
|
8326
|
+
const observationResult = validateExpectedObservations(suiteCase, read);
|
|
8327
|
+
const diagnostics = [
|
|
8328
|
+
...checkResult.diagnostics.map(
|
|
8329
|
+
(item) => diagnostic4(
|
|
8330
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
8331
|
+
item.message,
|
|
8332
|
+
item.severity,
|
|
8333
|
+
suiteCase.id
|
|
8334
|
+
)
|
|
8335
|
+
),
|
|
8336
|
+
...checkResult.findings.filter((finding) => finding.status === "fail").map(
|
|
8337
|
+
(finding) => diagnostic4(
|
|
8338
|
+
"AI_SUITE_CASE_CHECK_FAILED",
|
|
8339
|
+
finding.message,
|
|
8340
|
+
finding.severity,
|
|
8341
|
+
suiteCase.id
|
|
8342
|
+
)
|
|
8343
|
+
),
|
|
8344
|
+
...observationResult.diagnostics
|
|
8345
|
+
];
|
|
8346
|
+
const checkOk = checkResult.ok;
|
|
8347
|
+
const observationsOk = observationResult.ok;
|
|
8348
|
+
const ok = checkOk && observationsOk;
|
|
8349
|
+
const status = ok ? "pass" : checkResult.status === "error" ? "error" : "fail";
|
|
8350
|
+
return {
|
|
8351
|
+
id: suiteCase.id,
|
|
8352
|
+
status,
|
|
8353
|
+
tracePath: resolved.tracePath,
|
|
8354
|
+
...resolved.runId !== void 0 ? { runId: resolved.runId } : {},
|
|
8355
|
+
checkOk,
|
|
8356
|
+
observationsOk,
|
|
8357
|
+
diagnostics,
|
|
8358
|
+
...ok ? {} : {
|
|
8359
|
+
message: diagnostics.map((item) => item.message).join("; ") || checkResult.findings.map((item) => item.message).join("; ")
|
|
8360
|
+
}
|
|
8361
|
+
};
|
|
8362
|
+
}
|
|
8363
|
+
async function runSuite(options = {}) {
|
|
8364
|
+
const startedAt = new Date(options.nowMs ?? Date.now()).toISOString();
|
|
8365
|
+
const { config, configPath, configDir } = await loadSuiteConfig(options);
|
|
8366
|
+
const tracesDir = path__default.default.resolve(configDir, config.traces);
|
|
8367
|
+
const cases = [];
|
|
8368
|
+
const diagnostics = [];
|
|
8369
|
+
for (const suiteCase of config.cases) {
|
|
8370
|
+
cases.push(
|
|
8371
|
+
await runSuiteCase(suiteCase, config, {
|
|
8372
|
+
configDir,
|
|
8373
|
+
tracesDir
|
|
8374
|
+
})
|
|
8375
|
+
);
|
|
8376
|
+
}
|
|
8377
|
+
const summary = {
|
|
8378
|
+
passed: cases.filter((item) => item.status === "pass").length,
|
|
8379
|
+
failed: cases.filter((item) => item.status === "fail").length,
|
|
8380
|
+
errors: cases.filter((item) => item.status === "error").length,
|
|
8381
|
+
skipped: cases.filter((item) => item.status === "skipped").length
|
|
8382
|
+
};
|
|
8383
|
+
const finishedAt = new Date(options.nowMs ?? Date.now()).toISOString();
|
|
8384
|
+
const ok = summary.failed === 0 && summary.errors === 0;
|
|
8385
|
+
const status = summary.errors > 0 ? "error" : summary.failed > 0 || !ok ? "fail" : "pass";
|
|
8386
|
+
return {
|
|
8387
|
+
ok,
|
|
8388
|
+
status,
|
|
8389
|
+
suiteName: config.name,
|
|
8390
|
+
configPath,
|
|
8391
|
+
tracesDir,
|
|
8392
|
+
startedAt,
|
|
8393
|
+
finishedAt,
|
|
8394
|
+
summary,
|
|
8395
|
+
cases,
|
|
8396
|
+
diagnostics
|
|
8397
|
+
};
|
|
8398
|
+
}
|
|
8399
|
+
|
|
8400
|
+
// packages/core/src/suite/report.ts
|
|
8401
|
+
function statusLabel(status) {
|
|
8402
|
+
return status.toUpperCase();
|
|
8403
|
+
}
|
|
8404
|
+
function renderSuiteReportMarkdown(result) {
|
|
8405
|
+
const lines = [];
|
|
8406
|
+
lines.push(`# Suite: ${result.suiteName}`);
|
|
8407
|
+
lines.push("");
|
|
8408
|
+
lines.push(`Status: **${statusLabel(result.status)}**`);
|
|
8409
|
+
lines.push(`Config: \`${result.configPath}\``);
|
|
8410
|
+
lines.push(`Traces: \`${result.tracesDir}\``);
|
|
8411
|
+
lines.push(`Started: ${result.startedAt}`);
|
|
8412
|
+
lines.push(`Finished: ${result.finishedAt}`);
|
|
8413
|
+
lines.push("");
|
|
8414
|
+
lines.push("## Summary");
|
|
8415
|
+
lines.push(`- Passed: ${result.summary.passed}`);
|
|
8416
|
+
lines.push(`- Failed: ${result.summary.failed}`);
|
|
8417
|
+
lines.push(`- Errors: ${result.summary.errors}`);
|
|
8418
|
+
lines.push(`- Skipped: ${result.summary.skipped}`);
|
|
8419
|
+
lines.push("");
|
|
8420
|
+
lines.push("## Cases");
|
|
8421
|
+
for (const suiteCase of result.cases) {
|
|
8422
|
+
lines.push(`### ${suiteCase.id} \u2014 ${statusLabel(suiteCase.status)}`);
|
|
8423
|
+
if (suiteCase.tracePath !== void 0) {
|
|
8424
|
+
lines.push(`- Trace: \`${suiteCase.tracePath}\``);
|
|
8425
|
+
}
|
|
8426
|
+
if (suiteCase.runId !== void 0) {
|
|
8427
|
+
lines.push(`- Run: \`${suiteCase.runId}\``);
|
|
8428
|
+
}
|
|
8429
|
+
if (suiteCase.message !== void 0 && suiteCase.message.trim() !== "") {
|
|
8430
|
+
lines.push(`- ${suiteCase.message}`);
|
|
8431
|
+
}
|
|
8432
|
+
if (suiteCase.diagnostics.length > 0) {
|
|
8433
|
+
for (const item of suiteCase.diagnostics) {
|
|
8434
|
+
lines.push(`- [${item.severity}] ${item.message}`);
|
|
8435
|
+
}
|
|
8436
|
+
}
|
|
8437
|
+
lines.push("");
|
|
8438
|
+
}
|
|
8439
|
+
return lines.join("\n").trimEnd();
|
|
8440
|
+
}
|
|
8441
|
+
function renderSuiteReport(result, options = {}) {
|
|
8442
|
+
const format = options.format ?? "markdown";
|
|
8443
|
+
if (format === "json") {
|
|
8444
|
+
return JSON.stringify(result, null, 2);
|
|
8445
|
+
}
|
|
8446
|
+
return renderSuiteReportMarkdown(result);
|
|
8447
|
+
}
|
|
8448
|
+
|
|
8449
|
+
// packages/core/src/cohort/types.ts
|
|
8450
|
+
var COHORT_METRIC_IDS = [
|
|
8451
|
+
"errorRate",
|
|
8452
|
+
"duration",
|
|
8453
|
+
"toolChoice",
|
|
8454
|
+
"toolOrdering",
|
|
8455
|
+
"llmCallCount",
|
|
8456
|
+
"tokenUsage",
|
|
8457
|
+
"retryCount",
|
|
8458
|
+
"observationFailure",
|
|
8459
|
+
"guardrailFailure",
|
|
8460
|
+
"circuitViolation",
|
|
8461
|
+
"redactionWarning"
|
|
8462
|
+
];
|
|
8463
|
+
|
|
8464
|
+
// packages/core/src/cohort/compare.ts
|
|
8465
|
+
function compareNumber(metric, label, baseline, candidate, higherIsWorse = true) {
|
|
8466
|
+
if (baseline === void 0 && candidate === void 0) return void 0;
|
|
8467
|
+
const delta = baseline !== void 0 && candidate !== void 0 ? candidate - baseline : void 0;
|
|
8468
|
+
const regression = delta !== void 0 && (higherIsWorse && delta > 0 || !higherIsWorse && delta < 0);
|
|
8469
|
+
return {
|
|
8470
|
+
metric,
|
|
8471
|
+
baseline,
|
|
8472
|
+
candidate,
|
|
8473
|
+
delta,
|
|
8474
|
+
regression,
|
|
8475
|
+
message: `${label}: baseline=${baseline ?? "n/a"} candidate=${candidate ?? "n/a"}${delta !== void 0 ? ` (delta ${delta})` : ""}`
|
|
8476
|
+
};
|
|
8477
|
+
}
|
|
8478
|
+
function pickAggregate(groups, cohortLabel, groupKey) {
|
|
8479
|
+
return groups.find(
|
|
8480
|
+
(group) => group.cohortLabel === cohortLabel && (groupKey === void 0 || group.groupKey === groupKey)
|
|
8481
|
+
);
|
|
8482
|
+
}
|
|
8483
|
+
function compareCohortAggregates(groups, options) {
|
|
8484
|
+
const baselineAgg = pickAggregate(groups, options.baseline, options.groupKey);
|
|
8485
|
+
const candidateAgg = pickAggregate(groups, options.candidate, options.groupKey);
|
|
8486
|
+
const comparisons = [];
|
|
8487
|
+
for (const metric of options.metrics) {
|
|
8488
|
+
switch (metric) {
|
|
8489
|
+
case "errorRate": {
|
|
8490
|
+
const item = compareNumber(
|
|
8491
|
+
metric,
|
|
8492
|
+
"Error rate",
|
|
8493
|
+
baselineAgg?.errorRate,
|
|
8494
|
+
candidateAgg?.errorRate
|
|
8495
|
+
);
|
|
8496
|
+
if (item) comparisons.push(item);
|
|
8497
|
+
break;
|
|
8498
|
+
}
|
|
8499
|
+
case "duration": {
|
|
8500
|
+
const item = compareNumber(
|
|
8501
|
+
metric,
|
|
8502
|
+
"Average duration (ms)",
|
|
8503
|
+
baselineAgg?.avgDurationMs,
|
|
8504
|
+
candidateAgg?.avgDurationMs
|
|
8505
|
+
);
|
|
8506
|
+
if (item) comparisons.push(item);
|
|
8507
|
+
break;
|
|
8508
|
+
}
|
|
8509
|
+
case "llmCallCount": {
|
|
8510
|
+
const item = compareNumber(
|
|
8511
|
+
metric,
|
|
8512
|
+
"Average LLM calls",
|
|
8513
|
+
baselineAgg?.avgLlmCallCount,
|
|
8514
|
+
candidateAgg?.avgLlmCallCount
|
|
8515
|
+
);
|
|
8516
|
+
if (item) comparisons.push(item);
|
|
8517
|
+
break;
|
|
8518
|
+
}
|
|
8519
|
+
case "tokenUsage": {
|
|
8520
|
+
const item = compareNumber(
|
|
8521
|
+
metric,
|
|
8522
|
+
"Average token usage",
|
|
8523
|
+
baselineAgg?.avgTokenUsage,
|
|
8524
|
+
candidateAgg?.avgTokenUsage
|
|
8525
|
+
);
|
|
8526
|
+
if (item) comparisons.push(item);
|
|
8527
|
+
break;
|
|
8528
|
+
}
|
|
8529
|
+
case "retryCount": {
|
|
8530
|
+
const item = compareNumber(
|
|
8531
|
+
metric,
|
|
8532
|
+
"Average retries",
|
|
8533
|
+
baselineAgg?.avgRetryCount,
|
|
8534
|
+
candidateAgg?.avgRetryCount
|
|
8535
|
+
);
|
|
8536
|
+
if (item) comparisons.push(item);
|
|
8537
|
+
break;
|
|
8538
|
+
}
|
|
8539
|
+
case "observationFailure": {
|
|
8540
|
+
const item = compareNumber(
|
|
8541
|
+
metric,
|
|
8542
|
+
"Observation failure rate",
|
|
8543
|
+
baselineAgg?.observationFailureRate,
|
|
8544
|
+
candidateAgg?.observationFailureRate
|
|
8545
|
+
);
|
|
8546
|
+
if (item) comparisons.push(item);
|
|
8547
|
+
break;
|
|
8548
|
+
}
|
|
8549
|
+
case "toolChoice": {
|
|
8550
|
+
const baselineValue = baselineAgg?.dominantToolChoice;
|
|
8551
|
+
const candidateValue = candidateAgg?.dominantToolChoice;
|
|
8552
|
+
comparisons.push({
|
|
8553
|
+
metric,
|
|
8554
|
+
baseline: baselineValue,
|
|
8555
|
+
candidate: candidateValue,
|
|
8556
|
+
delta: baselineValue === candidateValue ? "same" : "changed",
|
|
8557
|
+
regression: baselineValue !== candidateValue,
|
|
8558
|
+
message: `Tool choice: baseline=${baselineValue ?? "n/a"} candidate=${candidateValue ?? "n/a"}`
|
|
8559
|
+
});
|
|
8560
|
+
break;
|
|
8561
|
+
}
|
|
8562
|
+
case "toolOrdering": {
|
|
8563
|
+
const baselineValue = baselineAgg?.toolOrderingSignature;
|
|
8564
|
+
const candidateValue = candidateAgg?.toolOrderingSignature;
|
|
8565
|
+
comparisons.push({
|
|
8566
|
+
metric,
|
|
8567
|
+
baseline: baselineValue,
|
|
8568
|
+
candidate: candidateValue,
|
|
8569
|
+
delta: baselineValue === candidateValue ? "same" : "changed",
|
|
8570
|
+
regression: baselineValue !== candidateValue,
|
|
8571
|
+
message: `Tool ordering: baseline=${baselineValue ?? "n/a"} candidate=${candidateValue ?? "n/a"}`
|
|
8572
|
+
});
|
|
8573
|
+
break;
|
|
8574
|
+
}
|
|
8575
|
+
case "guardrailFailure": {
|
|
8576
|
+
const item = compareNumber(
|
|
8577
|
+
metric,
|
|
8578
|
+
"Guardrail failures",
|
|
8579
|
+
baselineAgg?.avgGuardrailFailures,
|
|
8580
|
+
candidateAgg?.avgGuardrailFailures
|
|
8581
|
+
);
|
|
8582
|
+
if (item) comparisons.push(item);
|
|
8583
|
+
break;
|
|
8584
|
+
}
|
|
8585
|
+
case "circuitViolation": {
|
|
8586
|
+
const item = compareNumber(
|
|
8587
|
+
metric,
|
|
8588
|
+
"Circuit violations",
|
|
8589
|
+
baselineAgg?.avgCircuitViolations,
|
|
8590
|
+
candidateAgg?.avgCircuitViolations
|
|
8591
|
+
);
|
|
8592
|
+
if (item) comparisons.push(item);
|
|
8593
|
+
break;
|
|
8594
|
+
}
|
|
8595
|
+
case "redactionWarning": {
|
|
8596
|
+
const item = compareNumber(
|
|
8597
|
+
metric,
|
|
8598
|
+
"Redaction warnings",
|
|
8599
|
+
baselineAgg?.avgRedactionWarnings,
|
|
8600
|
+
candidateAgg?.avgRedactionWarnings
|
|
8601
|
+
);
|
|
8602
|
+
if (item) comparisons.push(item);
|
|
8603
|
+
break;
|
|
8604
|
+
}
|
|
8605
|
+
}
|
|
8606
|
+
}
|
|
8607
|
+
return comparisons;
|
|
8608
|
+
}
|
|
8609
|
+
|
|
8610
|
+
// packages/core/src/cohort/grouping.ts
|
|
8611
|
+
function parseCohortMetricList(value) {
|
|
8612
|
+
if (value === void 0 || value.trim() === "") return [];
|
|
8613
|
+
return value.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
|
|
8614
|
+
}
|
|
8615
|
+
function parseGroupBySpec(groupBy) {
|
|
8616
|
+
const raw = (groupBy ?? "model").trim();
|
|
8617
|
+
if (raw === "model") return { kind: "model" };
|
|
8618
|
+
if (raw === "session") return { kind: "session" };
|
|
8619
|
+
if (raw === "group") return { kind: "group" };
|
|
8620
|
+
if (raw.startsWith("metadata.")) {
|
|
8621
|
+
const metadataKey = raw.slice("metadata.".length).trim();
|
|
8622
|
+
if (metadataKey === "") throw new Error("metadata group-by requires a key.");
|
|
8623
|
+
return { kind: "metadata", metadataKey };
|
|
8624
|
+
}
|
|
8625
|
+
throw new Error(`Unsupported --group-by value: ${raw}`);
|
|
8626
|
+
}
|
|
8627
|
+
function metadataString(metadata, key) {
|
|
8628
|
+
const value = metadata?.[key];
|
|
8629
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
8630
|
+
}
|
|
8631
|
+
function resolveRunGroupKey(run, groupBy) {
|
|
8632
|
+
const metadata = run.metadata ?? {};
|
|
8633
|
+
switch (groupBy.kind) {
|
|
8634
|
+
case "model":
|
|
8635
|
+
return metadataString(metadata, "model") ?? "unknown";
|
|
8636
|
+
case "session":
|
|
8637
|
+
return extractSessionWorkflowMetadata(metadata)?.sessionId ?? metadataString(metadata, "sessionId") ?? "__unscoped__";
|
|
8638
|
+
case "group":
|
|
8639
|
+
return extractSessionWorkflowMetadata(metadata)?.groupId ?? metadataString(metadata, "groupId") ?? "__unscoped__";
|
|
8640
|
+
case "metadata":
|
|
8641
|
+
return metadataString(metadata, groupBy.metadataKey) ?? "__missing__";
|
|
8642
|
+
default:
|
|
8643
|
+
return "unknown";
|
|
8644
|
+
}
|
|
8645
|
+
}
|
|
8646
|
+
function resolveCohortLabel(run, cohortKey, baseline, candidate) {
|
|
8647
|
+
const label = metadataString(run.metadata, cohortKey);
|
|
8648
|
+
if (label === void 0) return void 0;
|
|
8649
|
+
if (baseline !== void 0 && label === baseline) return baseline;
|
|
8650
|
+
if (candidate !== void 0 && label === candidate) return candidate;
|
|
8651
|
+
if (baseline === void 0 && candidate === void 0) return label;
|
|
8652
|
+
return void 0;
|
|
8653
|
+
}
|
|
8654
|
+
function filterRunsForCohort(runs, options) {
|
|
8655
|
+
const warnings = [];
|
|
8656
|
+
if (options.baseline === void 0 && options.candidate === void 0) {
|
|
8657
|
+
return { runs: [...runs], warnings };
|
|
8658
|
+
}
|
|
8659
|
+
const selected = [];
|
|
8660
|
+
for (const run of runs) {
|
|
8661
|
+
const label = resolveCohortLabel(
|
|
8662
|
+
run,
|
|
8663
|
+
options.cohortKey,
|
|
8664
|
+
options.baseline,
|
|
8665
|
+
options.candidate
|
|
8666
|
+
);
|
|
8667
|
+
if (label !== void 0) selected.push(run);
|
|
8668
|
+
}
|
|
8669
|
+
if (selected.length === 0) {
|
|
8670
|
+
warnings.push(
|
|
8671
|
+
`No runs matched baseline/candidate labels on metadata.${options.cohortKey}.`
|
|
8672
|
+
);
|
|
8673
|
+
}
|
|
8674
|
+
return { runs: selected, warnings };
|
|
8675
|
+
}
|
|
8676
|
+
|
|
8677
|
+
// packages/core/src/cohort/metrics.ts
|
|
8678
|
+
function asNumber(value) {
|
|
8679
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
8680
|
+
}
|
|
8681
|
+
function toolSteps(events) {
|
|
8682
|
+
const ordering = [];
|
|
8683
|
+
const choices = /* @__PURE__ */ new Set();
|
|
8684
|
+
const sorted = [...events].filter((event) => event.event === "step_started").sort((a, b) => a.timestamp - b.timestamp);
|
|
8685
|
+
for (const event of sorted) {
|
|
8686
|
+
const step = event;
|
|
8687
|
+
if (step.type !== "tool") continue;
|
|
8688
|
+
const name = typeof step.metadata?.toolName === "string" ? step.metadata.toolName : step.name;
|
|
8689
|
+
ordering.push(name);
|
|
8690
|
+
choices.add(name);
|
|
8691
|
+
}
|
|
8692
|
+
return { choices: [...choices].sort(), ordering };
|
|
8693
|
+
}
|
|
8694
|
+
async function computeCohortRunMetrics(input) {
|
|
8695
|
+
const events = await readTraceEventsFromFile(input.filePath);
|
|
8696
|
+
const summary = buildRunSummary(events);
|
|
8697
|
+
const tools = toolSteps(events);
|
|
8698
|
+
const outcomes = extractOutcomesFromTraceEvents(events);
|
|
8699
|
+
const observationFailures = outcomes.filter((item) => item.status === "failed").length;
|
|
8700
|
+
const metadata = input.metadata ?? {};
|
|
8701
|
+
const retryCount = asNumber(metadata.attempt) !== void 0 && asNumber(metadata.attempt) > 1 ? asNumber(metadata.attempt) - 1 : typeof metadata.retryOf === "string" ? 1 : 0;
|
|
8702
|
+
return {
|
|
8703
|
+
runId: input.runId,
|
|
8704
|
+
...input.cohortLabel !== void 0 ? { cohortLabel: input.cohortLabel } : {},
|
|
8705
|
+
groupKey: input.groupKey,
|
|
8706
|
+
status: summary.status,
|
|
8707
|
+
error: summary.status === "error",
|
|
8708
|
+
durationMs: summary.durationMs ?? input.durationMs,
|
|
8709
|
+
llmCallCount: summary.llmSteps,
|
|
8710
|
+
tokenUsageTotal: summary.totalTokens?.total,
|
|
8711
|
+
retryCount,
|
|
8712
|
+
observationFailures,
|
|
8713
|
+
guardrailFailures: asNumber(metadata.guardrailFailures) ?? 0,
|
|
8714
|
+
circuitViolations: asNumber(metadata.circuitViolations) ?? 0,
|
|
8715
|
+
redactionWarnings: asNumber(metadata.redactionWarnings) ?? 0,
|
|
8716
|
+
toolChoices: tools.choices,
|
|
8717
|
+
toolOrdering: tools.ordering
|
|
8718
|
+
};
|
|
8719
|
+
}
|
|
8720
|
+
function percentile2(values, p) {
|
|
8721
|
+
if (values.length === 0) return void 0;
|
|
8722
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
8723
|
+
const idx = Math.min(
|
|
8724
|
+
sorted.length - 1,
|
|
8725
|
+
Math.max(0, Math.ceil(p / 100 * sorted.length) - 1)
|
|
8726
|
+
);
|
|
8727
|
+
return sorted[idx];
|
|
8728
|
+
}
|
|
8729
|
+
function dominantToolChoice(runs) {
|
|
8730
|
+
const counts = /* @__PURE__ */ new Map();
|
|
8731
|
+
for (const run of runs) {
|
|
8732
|
+
const signature = run.toolChoices.join(",");
|
|
8733
|
+
if (signature === "") continue;
|
|
8734
|
+
counts.set(signature, (counts.get(signature) ?? 0) + 1);
|
|
8735
|
+
}
|
|
8736
|
+
let best;
|
|
8737
|
+
let bestCount = 0;
|
|
8738
|
+
for (const [key, count] of counts) {
|
|
8739
|
+
if (count > bestCount) {
|
|
8740
|
+
best = key;
|
|
8741
|
+
bestCount = count;
|
|
8742
|
+
}
|
|
8743
|
+
}
|
|
8744
|
+
return best;
|
|
8745
|
+
}
|
|
8746
|
+
function orderingSignature(runs) {
|
|
8747
|
+
const counts = /* @__PURE__ */ new Map();
|
|
8748
|
+
for (const run of runs) {
|
|
8749
|
+
const signature = run.toolOrdering.join(">");
|
|
8750
|
+
if (signature === "") continue;
|
|
8751
|
+
counts.set(signature, (counts.get(signature) ?? 0) + 1);
|
|
8752
|
+
}
|
|
8753
|
+
let best;
|
|
8754
|
+
let bestCount = 0;
|
|
8755
|
+
for (const [key, count] of counts) {
|
|
8756
|
+
if (count > bestCount) {
|
|
8757
|
+
best = key;
|
|
8758
|
+
bestCount = count;
|
|
8759
|
+
}
|
|
8760
|
+
}
|
|
8761
|
+
return best;
|
|
8762
|
+
}
|
|
8763
|
+
function aggregateCohortMetrics(runs, groupKey, cohortLabel) {
|
|
8764
|
+
const durations = runs.map((run) => run.durationMs).filter((value) => typeof value === "number");
|
|
8765
|
+
const tokenValues = runs.map((run) => run.tokenUsageTotal).filter((value) => typeof value === "number");
|
|
8766
|
+
const errors = runs.filter((run) => run.error).length;
|
|
8767
|
+
const observationFailures = runs.reduce((sum, run) => sum + run.observationFailures, 0);
|
|
8768
|
+
return {
|
|
8769
|
+
groupKey,
|
|
8770
|
+
...cohortLabel !== void 0 ? { cohortLabel } : {},
|
|
8771
|
+
runCount: runs.length,
|
|
8772
|
+
errorRate: runs.length > 0 ? errors / runs.length : 0,
|
|
8773
|
+
avgDurationMs: durations.length > 0 ? durations.reduce((sum, value) => sum + value, 0) / durations.length : void 0,
|
|
8774
|
+
p95DurationMs: percentile2(durations, 95),
|
|
8775
|
+
avgLlmCallCount: runs.length > 0 ? runs.reduce((sum, run) => sum + run.llmCallCount, 0) / runs.length : 0,
|
|
8776
|
+
avgTokenUsage: tokenValues.length > 0 ? tokenValues.reduce((sum, value) => sum + value, 0) / tokenValues.length : void 0,
|
|
8777
|
+
avgRetryCount: runs.length > 0 ? runs.reduce((sum, run) => sum + run.retryCount, 0) / runs.length : 0,
|
|
8778
|
+
observationFailureRate: runs.length > 0 ? observationFailures / runs.length : 0,
|
|
8779
|
+
avgGuardrailFailures: runs.length > 0 ? runs.reduce((sum, run) => sum + run.guardrailFailures, 0) / runs.length : 0,
|
|
8780
|
+
avgCircuitViolations: runs.length > 0 ? runs.reduce((sum, run) => sum + run.circuitViolations, 0) / runs.length : 0,
|
|
8781
|
+
avgRedactionWarnings: runs.length > 0 ? runs.reduce((sum, run) => sum + run.redactionWarnings, 0) / runs.length : 0,
|
|
8782
|
+
dominantToolChoice: dominantToolChoice(runs),
|
|
8783
|
+
toolOrderingSignature: orderingSignature(runs)
|
|
8784
|
+
};
|
|
8785
|
+
}
|
|
8786
|
+
|
|
8787
|
+
// packages/core/src/cohort/analyze.ts
|
|
8788
|
+
var DEFAULT_METRICS = [
|
|
8789
|
+
"errorRate",
|
|
8790
|
+
"duration",
|
|
8791
|
+
"toolChoice",
|
|
8792
|
+
"observationFailure"
|
|
8793
|
+
];
|
|
8794
|
+
function normalizeMetrics(metrics) {
|
|
8795
|
+
if (metrics === void 0 || metrics.length === 0) return [...DEFAULT_METRICS];
|
|
8796
|
+
const allowed = new Set(COHORT_METRIC_IDS);
|
|
8797
|
+
return metrics.filter((metric) => allowed.has(metric));
|
|
8798
|
+
}
|
|
8799
|
+
async function analyzeCohort(runsInput, options) {
|
|
8800
|
+
const cohortKey = options.cohortKey ?? "cohort";
|
|
8801
|
+
const groupBySpec = parseGroupBySpec(options.groupBy);
|
|
8802
|
+
const metrics = normalizeMetrics(options.metrics);
|
|
8803
|
+
const { runs: filteredRuns, warnings } = filterRunsForCohort(runsInput, {
|
|
8804
|
+
cohortKey,
|
|
8805
|
+
baseline: options.baseline,
|
|
8806
|
+
candidate: options.candidate
|
|
8807
|
+
});
|
|
8808
|
+
const runMetrics = [];
|
|
8809
|
+
for (const run of filteredRuns) {
|
|
8810
|
+
if (run.filePath === void 0) continue;
|
|
8811
|
+
const cohortLabel = resolveCohortLabel(
|
|
8812
|
+
run,
|
|
8813
|
+
cohortKey,
|
|
8814
|
+
options.baseline,
|
|
8815
|
+
options.candidate
|
|
8816
|
+
);
|
|
8817
|
+
runMetrics.push(
|
|
8818
|
+
await computeCohortRunMetrics({
|
|
8819
|
+
runId: run.runId,
|
|
8820
|
+
filePath: run.filePath,
|
|
8821
|
+
metadata: run.metadata,
|
|
8822
|
+
status: run.status,
|
|
8823
|
+
durationMs: run.durationMs,
|
|
8824
|
+
groupKey: resolveRunGroupKey(run, groupBySpec),
|
|
8825
|
+
cohortLabel
|
|
8826
|
+
})
|
|
8827
|
+
);
|
|
8828
|
+
}
|
|
8829
|
+
const groupMap = /* @__PURE__ */ new Map();
|
|
8830
|
+
for (const run of runMetrics) {
|
|
8831
|
+
const key = `${run.cohortLabel ?? "*"}::${run.groupKey}`;
|
|
8832
|
+
const bucket = groupMap.get(key) ?? [];
|
|
8833
|
+
bucket.push(run);
|
|
8834
|
+
groupMap.set(key, bucket);
|
|
8835
|
+
}
|
|
8836
|
+
const groups = [...groupMap.entries()].sort(([a], [b]) => a.localeCompare(b)).map(
|
|
8837
|
+
([, bucket]) => aggregateCohortMetrics(
|
|
8838
|
+
bucket,
|
|
8839
|
+
bucket[0].groupKey,
|
|
8840
|
+
bucket[0]?.cohortLabel
|
|
8841
|
+
)
|
|
8842
|
+
);
|
|
8843
|
+
const comparisons = options.baseline !== void 0 && options.candidate !== void 0 ? (() => {
|
|
8844
|
+
const groupKeys = [
|
|
8845
|
+
...new Set(groups.map((group) => group.groupKey))
|
|
8846
|
+
].sort((a, b) => a.localeCompare(b));
|
|
8847
|
+
const items = [];
|
|
8848
|
+
for (const groupKey of groupKeys) {
|
|
8849
|
+
items.push(
|
|
8850
|
+
...compareCohortAggregates(groups, {
|
|
8851
|
+
baseline: options.baseline,
|
|
8852
|
+
candidate: options.candidate,
|
|
8853
|
+
metrics,
|
|
8854
|
+
groupKey
|
|
8855
|
+
})
|
|
8856
|
+
);
|
|
8857
|
+
}
|
|
8858
|
+
return items;
|
|
8859
|
+
})() : [];
|
|
8860
|
+
const regression = comparisons.some((item) => item.regression);
|
|
8861
|
+
return {
|
|
8862
|
+
ok: !regression,
|
|
8863
|
+
traceDir: options.traceDir,
|
|
8864
|
+
...options.baseline !== void 0 ? { baseline: options.baseline } : {},
|
|
8865
|
+
...options.candidate !== void 0 ? { candidate: options.candidate } : {},
|
|
8866
|
+
cohortKey,
|
|
8867
|
+
groupBy: options.groupBy ?? "model",
|
|
8868
|
+
metrics,
|
|
8869
|
+
groups,
|
|
8870
|
+
comparisons,
|
|
8871
|
+
runs: runMetrics,
|
|
8872
|
+
warnings
|
|
8873
|
+
};
|
|
8874
|
+
}
|
|
8875
|
+
|
|
8876
|
+
// packages/core/src/exporters/helpers.ts
|
|
8877
|
+
function escapeHtml(value) {
|
|
8878
|
+
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
8879
|
+
}
|
|
8880
|
+
|
|
8881
|
+
// packages/core/src/cohort/render.ts
|
|
8882
|
+
function formatRate(value) {
|
|
8883
|
+
if (value === void 0) return "n/a";
|
|
8884
|
+
return `${(value * 100).toFixed(1)}%`;
|
|
8885
|
+
}
|
|
8886
|
+
function renderCohortSummaryMarkdown(result) {
|
|
8887
|
+
const lines = [];
|
|
8888
|
+
lines.push("# Cohort analysis");
|
|
8889
|
+
lines.push("");
|
|
8890
|
+
lines.push(`Trace directory: \`${result.traceDir}\``);
|
|
8891
|
+
lines.push(`Group by: \`${result.groupBy}\``);
|
|
8892
|
+
if (result.baseline !== void 0 && result.candidate !== void 0) {
|
|
8893
|
+
lines.push(
|
|
8894
|
+
`Baseline/Candidate key: \`${result.cohortKey}\` (${result.baseline} vs ${result.candidate})`
|
|
8895
|
+
);
|
|
8896
|
+
}
|
|
8897
|
+
lines.push(`Status: **${result.ok ? "PASS" : "REGRESSION"}**`);
|
|
8898
|
+
lines.push("");
|
|
8899
|
+
if (result.warnings.length > 0) {
|
|
8900
|
+
lines.push("## Warnings");
|
|
8901
|
+
for (const warning of result.warnings) lines.push(`- ${warning}`);
|
|
8902
|
+
lines.push("");
|
|
8903
|
+
}
|
|
8904
|
+
lines.push("## Groups");
|
|
8905
|
+
for (const group of result.groups) {
|
|
8906
|
+
lines.push(
|
|
8907
|
+
`### ${group.cohortLabel ?? "all"} / ${group.groupKey} (${group.runCount} runs)`
|
|
8908
|
+
);
|
|
8909
|
+
lines.push(`- Error rate: ${formatRate(group.errorRate)}`);
|
|
8910
|
+
if (group.avgDurationMs !== void 0) {
|
|
8911
|
+
lines.push(`- Avg duration: ${Math.round(group.avgDurationMs)} ms`);
|
|
8912
|
+
}
|
|
8913
|
+
if (group.dominantToolChoice !== void 0) {
|
|
8914
|
+
lines.push(`- Dominant tools: ${group.dominantToolChoice}`);
|
|
8915
|
+
}
|
|
8916
|
+
lines.push(
|
|
8917
|
+
`- Observation failure rate: ${formatRate(group.observationFailureRate)}`
|
|
8918
|
+
);
|
|
8919
|
+
lines.push("");
|
|
8920
|
+
}
|
|
8921
|
+
if (result.comparisons.length > 0) {
|
|
8922
|
+
lines.push("## Comparisons");
|
|
8923
|
+
for (const comparison of result.comparisons) {
|
|
8924
|
+
const flag = comparison.regression ? " **REGRESSION**" : "";
|
|
8925
|
+
lines.push(`- ${comparison.message}${flag}`);
|
|
8926
|
+
}
|
|
8927
|
+
lines.push("");
|
|
8928
|
+
}
|
|
8929
|
+
return lines.join("\n").trimEnd();
|
|
8930
|
+
}
|
|
8931
|
+
function renderCohortReportHtml(result) {
|
|
8932
|
+
const rows = result.groups.map(
|
|
8933
|
+
(group) => `<tr><td>${escapeHtml(group.cohortLabel ?? "all")}</td><td>${escapeHtml(group.groupKey)}</td><td>${group.runCount}</td><td>${escapeHtml(formatRate(group.errorRate))}</td><td>${group.avgDurationMs !== void 0 ? Math.round(group.avgDurationMs) : "n/a"}</td></tr>`
|
|
8934
|
+
).join("");
|
|
8935
|
+
const comparisons = result.comparisons.map(
|
|
8936
|
+
(item) => `<li>${escapeHtml(item.message)}${item.regression ? " <strong>REGRESSION</strong>" : ""}</li>`
|
|
8937
|
+
).join("");
|
|
8938
|
+
return `<!DOCTYPE html>
|
|
8939
|
+
<html lang="en">
|
|
8940
|
+
<head>
|
|
8941
|
+
<meta charset="utf-8" />
|
|
8942
|
+
<title>Cohort report</title>
|
|
8943
|
+
<style>
|
|
8944
|
+
body { font-family: system-ui, sans-serif; margin: 2rem; }
|
|
8945
|
+
table { border-collapse: collapse; width: 100%; }
|
|
8946
|
+
th, td { border: 1px solid #ddd; padding: 0.5rem; text-align: left; }
|
|
8947
|
+
th { background: #f6f6f6; }
|
|
8948
|
+
</style>
|
|
8949
|
+
</head>
|
|
8950
|
+
<body>
|
|
8951
|
+
<h1>Cohort analysis</h1>
|
|
8952
|
+
<p>Status: <strong>${result.ok ? "PASS" : "REGRESSION"}</strong></p>
|
|
8953
|
+
<p>Trace directory: <code>${escapeHtml(result.traceDir)}</code></p>
|
|
8954
|
+
<h2>Groups</h2>
|
|
8955
|
+
<table>
|
|
8956
|
+
<thead><tr><th>Cohort</th><th>Group</th><th>Runs</th><th>Error rate</th><th>Avg duration (ms)</th></tr></thead>
|
|
8957
|
+
<tbody>${rows}</tbody>
|
|
8958
|
+
</table>
|
|
8959
|
+
${result.comparisons.length > 0 ? `<h2>Comparisons</h2><ul>${comparisons}</ul>` : ""}
|
|
8960
|
+
</body>
|
|
8961
|
+
</html>`;
|
|
8962
|
+
}
|
|
8963
|
+
function renderCohortReport(result, options = {}) {
|
|
8964
|
+
const format = options.format ?? "markdown";
|
|
8965
|
+
if (format === "json") return JSON.stringify(result, null, 2);
|
|
8966
|
+
if (format === "html") return renderCohortReportHtml(result);
|
|
8967
|
+
return renderCohortSummaryMarkdown(result);
|
|
8968
|
+
}
|
|
8969
|
+
|
|
5263
8970
|
// packages/core/src/inspect-run.ts
|
|
5264
8971
|
function normalizeRunName(name) {
|
|
5265
8972
|
if (typeof name !== "string" || name.trim() === "") {
|
|
@@ -5390,9 +9097,12 @@ async function maybeInspectRun(name, fn, options) {
|
|
|
5390
9097
|
return inspectRun(name, fn, options);
|
|
5391
9098
|
}
|
|
5392
9099
|
|
|
9100
|
+
exports.COHORT_METRIC_IDS = COHORT_METRIC_IDS;
|
|
5393
9101
|
exports.DEFAULT_MAX_EVENT_BYTES = DEFAULT_MAX_EVENT_BYTES;
|
|
5394
9102
|
exports.DEFAULT_MAX_METADATA_VALUE_LENGTH = DEFAULT_MAX_METADATA_VALUE_LENGTH;
|
|
5395
9103
|
exports.DEFAULT_MAX_PREVIEW_LENGTH = DEFAULT_MAX_PREVIEW_LENGTH;
|
|
9104
|
+
exports.DEFAULT_SUITE_ARTIFACTS_DIR = DEFAULT_SUITE_ARTIFACTS_DIR;
|
|
9105
|
+
exports.DEFAULT_SUITE_CONFIG_NAMES = DEFAULT_SUITE_CONFIG_NAMES;
|
|
5396
9106
|
exports.DEFAULT_TRACE_DIR_NAME = DEFAULT_TRACE_DIR_NAME;
|
|
5397
9107
|
exports.FALLBACK_TRACE_DIR = FALLBACK_TRACE_DIR;
|
|
5398
9108
|
exports.MAX_NAME_LENGTH = MAX_NAME_LENGTH;
|
|
@@ -5404,6 +9114,7 @@ exports.TERMINAL_INDENT = TERMINAL_INDENT;
|
|
|
5404
9114
|
exports.TraceDirectory = TraceDirectory;
|
|
5405
9115
|
exports.aggregateBundleSafeStatus = aggregateBundleSafeStatus;
|
|
5406
9116
|
exports.aggregateSessionCheckResults = aggregateSessionCheckResults;
|
|
9117
|
+
exports.analyzeCohort = analyzeCohort;
|
|
5407
9118
|
exports.buildActivitySummary = buildActivitySummary;
|
|
5408
9119
|
exports.buildBundleMetadata = buildBundleMetadata;
|
|
5409
9120
|
exports.buildBundleSummaryMarkdown = buildBundleSummaryMarkdown;
|
|
@@ -5415,11 +9126,13 @@ exports.buildRunWhatSummary = buildRunWhatSummary;
|
|
|
5415
9126
|
exports.buildSessionIndex = buildSessionIndex;
|
|
5416
9127
|
exports.buildTraceStats = buildTraceStats;
|
|
5417
9128
|
exports.bundleFailsOnSafety = bundleFailsOnSafety;
|
|
9129
|
+
exports.compareCohortAggregates = compareCohortAggregates;
|
|
5418
9130
|
exports.createInspector = createInspector;
|
|
5419
9131
|
exports.createInspectorRuntime = createInspectorRuntime;
|
|
5420
9132
|
exports.createRunId = createRunId;
|
|
5421
9133
|
exports.createStepId = createStepId;
|
|
5422
9134
|
exports.defaultBundleOutputPath = defaultBundleOutputPath;
|
|
9135
|
+
exports.defaultSuiteConfigTemplate = defaultSuiteConfigTemplate;
|
|
5423
9136
|
exports.deriveSessionStatus = deriveSessionStatus;
|
|
5424
9137
|
exports.enrichSessionRunRecord = enrichSessionRunRecord;
|
|
5425
9138
|
exports.enrichSessionSummary = enrichSessionSummary;
|
|
@@ -5456,11 +9169,15 @@ exports.isStepType = isStepType;
|
|
|
5456
9169
|
exports.isTraceEvent = isTraceEvent;
|
|
5457
9170
|
exports.listTraceFiles = listTraceFiles;
|
|
5458
9171
|
exports.loadSessionRunRecords = loadSessionRunRecords;
|
|
9172
|
+
exports.loadSuiteConfig = loadSuiteConfig;
|
|
5459
9173
|
exports.loadTraceMetadataList = loadTraceMetadataList;
|
|
5460
9174
|
exports.maybeInspectRun = maybeInspectRun;
|
|
5461
9175
|
exports.normalizeBundleOutputPath = normalizeBundleOutputPath;
|
|
9176
|
+
exports.normalizeSuiteConfig = normalizeSuiteConfig;
|
|
9177
|
+
exports.parseCohortMetricList = parseCohortMetricList;
|
|
5462
9178
|
exports.parseDuration = parseDuration;
|
|
5463
9179
|
exports.parseDurationFilter = parseDurationFilter;
|
|
9180
|
+
exports.parseGroupBySpec = parseGroupBySpec;
|
|
5464
9181
|
exports.parseTraceJsonl = parseTraceJsonl;
|
|
5465
9182
|
exports.prepareMetadataForDisk = prepareMetadataForDisk;
|
|
5466
9183
|
exports.prepareTraceEventForDisk = prepareTraceEventForDisk;
|
|
@@ -5473,16 +9190,23 @@ exports.printStepStart = printStepStart;
|
|
|
5473
9190
|
exports.readTraceEvents = readTraceEvents;
|
|
5474
9191
|
exports.readTraceFile = readTraceFile;
|
|
5475
9192
|
exports.renderActivitySummaryHuman = renderActivitySummaryHuman;
|
|
9193
|
+
exports.renderCohortReport = renderCohortReport;
|
|
9194
|
+
exports.renderCohortSummaryMarkdown = renderCohortSummaryMarkdown;
|
|
5476
9195
|
exports.renderErrorLine = renderErrorLine;
|
|
5477
9196
|
exports.renderRunSummary = renderRunSummary;
|
|
5478
9197
|
exports.renderRunWhat = renderRunWhat;
|
|
5479
9198
|
exports.renderStepLine = renderStepLine;
|
|
9199
|
+
exports.renderSuiteReport = renderSuiteReport;
|
|
9200
|
+
exports.renderSuiteReportMarkdown = renderSuiteReportMarkdown;
|
|
5480
9201
|
exports.renderTimeline = renderTimeline;
|
|
5481
9202
|
exports.renderTraceStats = renderTraceStats;
|
|
5482
9203
|
exports.resolveBundleRunIds = resolveBundleRunIds;
|
|
5483
9204
|
exports.resolveRedactionProfile = resolveRedactionProfile;
|
|
9205
|
+
exports.resolveSuiteCaseTrace = resolveSuiteCaseTrace;
|
|
9206
|
+
exports.resolveSuiteConfigPath = resolveSuiteConfigPath;
|
|
5484
9207
|
exports.resolveTraceDir = resolveTraceDir;
|
|
5485
9208
|
exports.resolveTraceSafetyOptions = resolveTraceSafetyOptions;
|
|
9209
|
+
exports.runSuite = runSuite;
|
|
5486
9210
|
exports.runWithContext = runWithContext;
|
|
5487
9211
|
exports.runWithStepContext = runWithStepContext;
|
|
5488
9212
|
exports.searchTraces = searchTraces;
|
|
@@ -5493,6 +9217,7 @@ exports.traceMetasToSessionRunRecords = traceMetasToSessionRunRecords;
|
|
|
5493
9217
|
exports.truncateName = truncateName;
|
|
5494
9218
|
exports.unknownTraceFormatMessage = unknownTraceFormatMessage;
|
|
5495
9219
|
exports.validateEvent = validateEvent;
|
|
9220
|
+
exports.validateSuiteConfig = validateSuiteConfig;
|
|
5496
9221
|
exports.warn = warn;
|
|
5497
9222
|
exports.writeTraceEvent = writeTraceEvent;
|
|
5498
9223
|
//# sourceMappingURL=advanced.cjs.map
|