agentv 4.42.1-next.1 → 4.42.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{artifact-writer-7ZQBLD6J.js → artifact-writer-UA6Z6BMG.js} +4 -4
- package/dist/{chunk-PN3QFUFF.js → chunk-FB6GTT4Z.js} +6 -2
- package/dist/chunk-FB6GTT4Z.js.map +1 -0
- package/dist/{chunk-RP5425KQ.js → chunk-NEV235RP.js} +8 -8
- package/dist/chunk-NEV235RP.js.map +1 -0
- package/dist/{chunk-CVQXNBO7.js → chunk-PNI5WEAV.js} +2 -2
- package/dist/{chunk-CVQXNBO7.js.map → chunk-PNI5WEAV.js.map} +1 -1
- package/dist/{chunk-IX5KWMMM.js → chunk-SJTHFOPH.js} +837 -150
- package/dist/chunk-SJTHFOPH.js.map +1 -0
- package/dist/{chunk-2H3D2AFZ.js → chunk-SY3JIGRD.js} +74 -45
- package/dist/chunk-SY3JIGRD.js.map +1 -0
- package/dist/cli.js +5 -5
- package/dist/dashboard/assets/index-BwBJwHp4.js +118 -0
- package/dist/dashboard/assets/index-C5v6_zhR.css +1 -0
- package/dist/dashboard/assets/{index-DFcMa-UW.js → index-uXJsSG83.js} +1 -1
- package/dist/dashboard/index.html +2 -2
- package/dist/{dist-XANKSXGP.js → dist-YGHRSREL.js} +33 -3
- package/dist/index.js +5 -5
- package/dist/{interactive-MGFKEFXD.js → interactive-CEXT3GVA.js} +5 -5
- package/dist/skills/agentv-bench/references/environment-adaptation.md +28 -10
- package/dist/skills/agentv-bench/references/eval-yaml-spec.md +1 -1
- package/dist/skills/agentv-eval-writer/SKILL.md +43 -3
- package/dist/skills/agentv-eval-writer/references/custom-evaluators.md +54 -3
- package/dist/skills/agentv-eval-writer/references/python-helpers.md +31 -1
- package/dist/{ts-eval-loader-DXSXFWYI-WV7NYVUG.js → ts-eval-loader-YR5XK44N-V3P76ZCR.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-2H3D2AFZ.js.map +0 -1
- package/dist/chunk-IX5KWMMM.js.map +0 -1
- package/dist/chunk-PN3QFUFF.js.map +0 -1
- package/dist/chunk-RP5425KQ.js.map +0 -1
- package/dist/dashboard/assets/index-BEZOKdZO.css +0 -1
- package/dist/dashboard/assets/index-CxvIJei9.js +0 -118
- /package/dist/{artifact-writer-7ZQBLD6J.js.map → artifact-writer-UA6Z6BMG.js.map} +0 -0
- /package/dist/{dist-XANKSXGP.js.map → dist-YGHRSREL.js.map} +0 -0
- /package/dist/{interactive-MGFKEFXD.js.map → interactive-CEXT3GVA.js.map} +0 -0
- /package/dist/{ts-eval-loader-DXSXFWYI-WV7NYVUG.js.map → ts-eval-loader-YR5XK44N-V3P76ZCR.js.map} +0 -0
|
@@ -4056,7 +4056,7 @@ var coerce = {
|
|
|
4056
4056
|
};
|
|
4057
4057
|
var NEVER = INVALID;
|
|
4058
4058
|
|
|
4059
|
-
// ../../packages/core/dist/chunk-
|
|
4059
|
+
// ../../packages/core/dist/chunk-Z5WVZ5VI.js
|
|
4060
4060
|
import { parse } from "yaml";
|
|
4061
4061
|
import os from "node:os";
|
|
4062
4062
|
import path from "node:path";
|
|
@@ -6156,7 +6156,7 @@ async function expandFileReferences(tests, evalFileDir) {
|
|
|
6156
6156
|
return expanded;
|
|
6157
6157
|
}
|
|
6158
6158
|
|
|
6159
|
-
// ../../packages/core/dist/chunk-
|
|
6159
|
+
// ../../packages/core/dist/chunk-NFFWBMIM.js
|
|
6160
6160
|
import path50 from "node:path";
|
|
6161
6161
|
import { pathToFileURL as pathToFileURL2 } from "node:url";
|
|
6162
6162
|
import { existsSync as existsSync7 } from "node:fs";
|
|
@@ -6164,7 +6164,7 @@ import path49 from "node:path";
|
|
|
6164
6164
|
import micromatch4 from "micromatch";
|
|
6165
6165
|
import { mkdir, readFile as readFile3, writeFile } from "node:fs/promises";
|
|
6166
6166
|
import path5 from "node:path";
|
|
6167
|
-
import { createHash as
|
|
6167
|
+
import { createHash as createHash6, randomUUID as randomUUID10 } from "node:crypto";
|
|
6168
6168
|
import path47 from "node:path";
|
|
6169
6169
|
import { fileURLToPath as fileURLToPath5 } from "node:url";
|
|
6170
6170
|
import micromatch3 from "micromatch";
|
|
@@ -18820,7 +18820,7 @@ var RequestError = class _RequestError extends Error {
|
|
|
18820
18820
|
}
|
|
18821
18821
|
};
|
|
18822
18822
|
|
|
18823
|
-
// ../../packages/core/dist/chunk-
|
|
18823
|
+
// ../../packages/core/dist/chunk-NFFWBMIM.js
|
|
18824
18824
|
import { exec as execCallback } from "node:child_process";
|
|
18825
18825
|
import { readdirSync, statSync } from "node:fs";
|
|
18826
18826
|
import { readFile as readFile32, readdir as readdir2, stat as stat2 } from "node:fs/promises";
|
|
@@ -18858,6 +18858,7 @@ import { createHash } from "node:crypto";
|
|
|
18858
18858
|
import { mkdir as mkdir9, readFile as readFile6, writeFile as writeFile4 } from "node:fs/promises";
|
|
18859
18859
|
import path17 from "node:path";
|
|
18860
18860
|
import { readFile as readFile7 } from "node:fs/promises";
|
|
18861
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
18861
18862
|
import { createHash as createHash2 } from "node:crypto";
|
|
18862
18863
|
import { exec as exec2 } from "node:child_process";
|
|
18863
18864
|
import { constants as constants2, access as access2 } from "node:fs/promises";
|
|
@@ -18900,12 +18901,12 @@ import { existsSync as existsSync6 } from "node:fs";
|
|
|
18900
18901
|
import { copyFile as copyFile2, mkdir as mkdir17, readdir as readdir8, stat as stat8 } from "node:fs/promises";
|
|
18901
18902
|
import path39 from "node:path";
|
|
18902
18903
|
import { promisify as promisify6 } from "node:util";
|
|
18903
|
-
import { createHash as
|
|
18904
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
18904
18905
|
import { existsSync as existsSync3 } from "node:fs";
|
|
18905
18906
|
import { cp as cp2, mkdir as mkdir15, readFile as readFile11, readdir as readdir6, rm as rm5, unlink, writeFile as writeFile9 } from "node:fs/promises";
|
|
18906
18907
|
import path35 from "node:path";
|
|
18907
18908
|
import { execFile, spawn as spawn5 } from "node:child_process";
|
|
18908
|
-
import { createHash as
|
|
18909
|
+
import { createHash as createHash5, randomUUID as randomUUID9 } from "node:crypto";
|
|
18909
18910
|
import { existsSync as existsSync5, readFileSync as readFileSync3 } from "node:fs";
|
|
18910
18911
|
import { mkdir as mkdir16, rename, rm as rm6 } from "node:fs/promises";
|
|
18911
18912
|
import path37 from "node:path";
|
|
@@ -21870,7 +21871,7 @@ function buildTraceFromMessages(options = {}) {
|
|
|
21870
21871
|
callId: toolCall.id,
|
|
21871
21872
|
input: toolCall.input,
|
|
21872
21873
|
output: toolCall.output,
|
|
21873
|
-
status: "ok"
|
|
21874
|
+
status: toolCall.status ?? "ok"
|
|
21874
21875
|
},
|
|
21875
21876
|
metadata: {
|
|
21876
21877
|
message_index: messageIndex,
|
|
@@ -21888,7 +21889,7 @@ function buildTraceFromMessages(options = {}) {
|
|
|
21888
21889
|
name: toolCall.tool,
|
|
21889
21890
|
callId: toolCall.id,
|
|
21890
21891
|
output: toolCall.output,
|
|
21891
|
-
status: "ok"
|
|
21892
|
+
status: toolCall.status ?? "ok"
|
|
21892
21893
|
},
|
|
21893
21894
|
metadata: {
|
|
21894
21895
|
message_index: messageIndex,
|
|
@@ -22120,12 +22121,13 @@ function toolCallFromTraceArtifactEvent(event) {
|
|
|
22120
22121
|
id: tool.callId,
|
|
22121
22122
|
input: tool.input,
|
|
22122
22123
|
output: tool.output,
|
|
22124
|
+
status: tool.status,
|
|
22123
22125
|
startTime: event.timestamp,
|
|
22124
22126
|
endTime: eventEndIso(event),
|
|
22125
22127
|
durationMs: event.durationMs
|
|
22126
22128
|
};
|
|
22127
22129
|
}
|
|
22128
|
-
function attachTraceArtifactToolCall(messages, eventMessageIndex, event) {
|
|
22130
|
+
function attachTraceArtifactToolCall(messages, eventMessageIndex, eventToolIndex, event) {
|
|
22129
22131
|
const toolCall = toolCallFromTraceArtifactEvent(event);
|
|
22130
22132
|
if (!toolCall) {
|
|
22131
22133
|
return;
|
|
@@ -22141,13 +22143,17 @@ function attachTraceArtifactToolCall(messages, eventMessageIndex, event) {
|
|
|
22141
22143
|
const messageIndex = parentIndex ?? fallbackIndex;
|
|
22142
22144
|
if (messageIndex >= 0) {
|
|
22143
22145
|
const message = messages[messageIndex];
|
|
22146
|
+
const toolIndex2 = message.toolCalls?.length ?? 0;
|
|
22144
22147
|
messages[messageIndex] = {
|
|
22145
22148
|
...message,
|
|
22146
22149
|
toolCalls: [...message.toolCalls ?? [], toolCall]
|
|
22147
22150
|
};
|
|
22151
|
+
eventMessageIndex.set(event.eventId, messageIndex);
|
|
22152
|
+
eventToolIndex.set(event.eventId, { messageIndex, toolIndex: toolIndex2 });
|
|
22148
22153
|
return;
|
|
22149
22154
|
}
|
|
22150
22155
|
const syntheticIndex = messages.length;
|
|
22156
|
+
const toolIndex = 0;
|
|
22151
22157
|
messages.push({
|
|
22152
22158
|
role: "assistant",
|
|
22153
22159
|
startTime: event.timestamp,
|
|
@@ -22157,6 +22163,53 @@ function attachTraceArtifactToolCall(messages, eventMessageIndex, event) {
|
|
|
22157
22163
|
toolCalls: [toolCall]
|
|
22158
22164
|
});
|
|
22159
22165
|
eventMessageIndex.set(event.eventId, syntheticIndex);
|
|
22166
|
+
eventToolIndex.set(event.eventId, { messageIndex: syntheticIndex, toolIndex });
|
|
22167
|
+
}
|
|
22168
|
+
function findTraceArtifactToolLocation(messages, eventToolIndex, event) {
|
|
22169
|
+
if (event.parentEventId) {
|
|
22170
|
+
const parentLocation = eventToolIndex.get(event.parentEventId);
|
|
22171
|
+
if (parentLocation) {
|
|
22172
|
+
return parentLocation;
|
|
22173
|
+
}
|
|
22174
|
+
}
|
|
22175
|
+
const callId = event.tool?.callId;
|
|
22176
|
+
if (!callId) {
|
|
22177
|
+
return void 0;
|
|
22178
|
+
}
|
|
22179
|
+
for (let messageIndex = messages.length - 1; messageIndex >= 0; messageIndex--) {
|
|
22180
|
+
const toolCalls = messages[messageIndex].toolCalls ?? [];
|
|
22181
|
+
const toolIndex = toolCalls.findIndex((toolCall) => toolCall.id === callId);
|
|
22182
|
+
if (toolIndex >= 0) {
|
|
22183
|
+
return { messageIndex, toolIndex };
|
|
22184
|
+
}
|
|
22185
|
+
}
|
|
22186
|
+
return void 0;
|
|
22187
|
+
}
|
|
22188
|
+
function mergeTraceArtifactToolResult(messages, eventToolIndex, event) {
|
|
22189
|
+
const location = findTraceArtifactToolLocation(messages, eventToolIndex, event);
|
|
22190
|
+
const result = toolCallFromTraceArtifactEvent(event);
|
|
22191
|
+
if (!location || !result) {
|
|
22192
|
+
return;
|
|
22193
|
+
}
|
|
22194
|
+
const message = messages[location.messageIndex];
|
|
22195
|
+
const toolCalls = [...message.toolCalls ?? []];
|
|
22196
|
+
const existing = toolCalls[location.toolIndex];
|
|
22197
|
+
if (!existing) {
|
|
22198
|
+
return;
|
|
22199
|
+
}
|
|
22200
|
+
const resultEndTime = result.endTime ?? event.timestamp;
|
|
22201
|
+
const endTime = existing.endTime && existing.endTime !== existing.startTime ? existing.endTime : resultEndTime;
|
|
22202
|
+
toolCalls[location.toolIndex] = {
|
|
22203
|
+
...existing,
|
|
22204
|
+
output: existing.output ?? result.output,
|
|
22205
|
+
status: result.status ?? existing.status,
|
|
22206
|
+
endTime,
|
|
22207
|
+
durationMs: existing.durationMs ?? result.durationMs
|
|
22208
|
+
};
|
|
22209
|
+
messages[location.messageIndex] = {
|
|
22210
|
+
...message,
|
|
22211
|
+
toolCalls
|
|
22212
|
+
};
|
|
22160
22213
|
}
|
|
22161
22214
|
function mergeTraceArtifactFinalResponse(messages, eventMessageIndex, event) {
|
|
22162
22215
|
if (!event.parentEventId || !event.message) {
|
|
@@ -22195,6 +22248,7 @@ function traceFromTraceArtifact(artifact) {
|
|
|
22195
22248
|
);
|
|
22196
22249
|
const messages = [];
|
|
22197
22250
|
const eventMessageIndex = /* @__PURE__ */ new Map();
|
|
22251
|
+
const eventToolIndex = /* @__PURE__ */ new Map();
|
|
22198
22252
|
for (const event of events) {
|
|
22199
22253
|
if (event.type === "message" && event.message) {
|
|
22200
22254
|
eventMessageIndex.set(event.eventId, messages.length);
|
|
@@ -22215,7 +22269,11 @@ function traceFromTraceArtifact(artifact) {
|
|
|
22215
22269
|
continue;
|
|
22216
22270
|
}
|
|
22217
22271
|
if (event.type === "tool_call") {
|
|
22218
|
-
attachTraceArtifactToolCall(messages, eventMessageIndex, event);
|
|
22272
|
+
attachTraceArtifactToolCall(messages, eventMessageIndex, eventToolIndex, event);
|
|
22273
|
+
continue;
|
|
22274
|
+
}
|
|
22275
|
+
if (event.type === "tool_result") {
|
|
22276
|
+
mergeTraceArtifactToolResult(messages, eventToolIndex, event);
|
|
22219
22277
|
}
|
|
22220
22278
|
}
|
|
22221
22279
|
const computed = computeTraceSummaryFromTraceArtifact(artifact);
|
|
@@ -25122,6 +25180,7 @@ var ToolCallSchema = external_exports.object({
|
|
|
25122
25180
|
input: external_exports.unknown().optional(),
|
|
25123
25181
|
output: external_exports.unknown().optional(),
|
|
25124
25182
|
id: external_exports.string().optional(),
|
|
25183
|
+
status: external_exports.enum(["ok", "error", "timeout", "cancelled", "unknown"]).optional(),
|
|
25125
25184
|
start_time: external_exports.string().optional(),
|
|
25126
25185
|
end_time: external_exports.string().optional(),
|
|
25127
25186
|
duration_ms: external_exports.number().optional()
|
|
@@ -25178,6 +25237,7 @@ function convertMessages(messages) {
|
|
|
25178
25237
|
input: tc.input,
|
|
25179
25238
|
output: tc.output,
|
|
25180
25239
|
id: tc.id,
|
|
25240
|
+
status: tc.status,
|
|
25181
25241
|
startTime: tc.start_time,
|
|
25182
25242
|
endTime: tc.end_time,
|
|
25183
25243
|
durationMs: tc.duration_ms
|
|
@@ -29410,6 +29470,7 @@ var ToolCallWireSchema = external_exports.object({
|
|
|
29410
29470
|
input: external_exports.unknown().optional(),
|
|
29411
29471
|
output: external_exports.unknown().optional(),
|
|
29412
29472
|
id: external_exports.string().optional(),
|
|
29473
|
+
status: external_exports.enum(["ok", "error", "timeout", "cancelled", "unknown"]).optional(),
|
|
29413
29474
|
start_time: external_exports.string().optional(),
|
|
29414
29475
|
end_time: external_exports.string().optional(),
|
|
29415
29476
|
duration_ms: external_exports.number().nonnegative().optional()
|
|
@@ -29487,6 +29548,7 @@ function fromWireToolCall(wire) {
|
|
|
29487
29548
|
input: wire.input,
|
|
29488
29549
|
output: wire.output,
|
|
29489
29550
|
id: wire.id,
|
|
29551
|
+
status: wire.status,
|
|
29490
29552
|
startTime: wire.start_time,
|
|
29491
29553
|
endTime: wire.end_time,
|
|
29492
29554
|
durationMs: wire.duration_ms
|
|
@@ -29535,6 +29597,7 @@ function toWireToolCall(toolCall) {
|
|
|
29535
29597
|
input: toolCall.input,
|
|
29536
29598
|
output: toolCall.output,
|
|
29537
29599
|
id: toolCall.id,
|
|
29600
|
+
status: toolCall.status,
|
|
29538
29601
|
start_time: toolCall.startTime,
|
|
29539
29602
|
end_time: toolCall.endTime,
|
|
29540
29603
|
duration_ms: toolCall.durationMs
|
|
@@ -29759,8 +29822,265 @@ function extractTranscript(raw) {
|
|
|
29759
29822
|
const transcript = raw.transcript;
|
|
29760
29823
|
return transcript;
|
|
29761
29824
|
}
|
|
29825
|
+
var PROJECTION_IDENTITY_SCHEMA_VERSION = "agentv.projection_identity.v1";
|
|
29826
|
+
var EXPORT_DUPLICATE_POLICY_VALUES = ["skip", "update", "error"];
|
|
29827
|
+
var PROJECTION_IDENTITY_ISSUE_SEVERITY_VALUES = ["warning", "error"];
|
|
29828
|
+
var ProjectionIdentityError = class extends Error {
|
|
29829
|
+
issues;
|
|
29830
|
+
constructor(issues) {
|
|
29831
|
+
super(
|
|
29832
|
+
`Projection identity is missing required fields: ${issues.map((i) => i.field).join(", ")}`
|
|
29833
|
+
);
|
|
29834
|
+
this.name = "ProjectionIdentityError";
|
|
29835
|
+
this.issues = issues;
|
|
29836
|
+
}
|
|
29837
|
+
};
|
|
29838
|
+
var REQUIRED_STRING_FIELDS = [
|
|
29839
|
+
"runId",
|
|
29840
|
+
"testId",
|
|
29841
|
+
"target",
|
|
29842
|
+
"sourceTarget",
|
|
29843
|
+
"envelopeId",
|
|
29844
|
+
"traceId",
|
|
29845
|
+
"rootSpanId",
|
|
29846
|
+
"projectionFormat",
|
|
29847
|
+
"projectionVersion"
|
|
29848
|
+
];
|
|
29849
|
+
var DIMENSION_KEY_ORDER = [
|
|
29850
|
+
"projection_format",
|
|
29851
|
+
"projection_version",
|
|
29852
|
+
"run_id",
|
|
29853
|
+
"suite",
|
|
29854
|
+
"eval_path",
|
|
29855
|
+
"test_id",
|
|
29856
|
+
"target",
|
|
29857
|
+
"source_target",
|
|
29858
|
+
"attempt",
|
|
29859
|
+
"variant",
|
|
29860
|
+
"envelope_id",
|
|
29861
|
+
"trace_id",
|
|
29862
|
+
"root_span_id"
|
|
29863
|
+
];
|
|
29864
|
+
function normalizedString(value) {
|
|
29865
|
+
const trimmed = value?.trim();
|
|
29866
|
+
return trimmed && trimmed.length > 0 ? trimmed : void 0;
|
|
29867
|
+
}
|
|
29868
|
+
function normalizeAttempt(value) {
|
|
29869
|
+
if (value === void 0) {
|
|
29870
|
+
return 0;
|
|
29871
|
+
}
|
|
29872
|
+
if (!Number.isInteger(value) || value < 0) {
|
|
29873
|
+
throw new ProjectionIdentityError([
|
|
29874
|
+
{
|
|
29875
|
+
code: "invalid_attempt",
|
|
29876
|
+
severity: "error",
|
|
29877
|
+
field: "attempt",
|
|
29878
|
+
message: "Projection identity attempt must be a non-negative integer."
|
|
29879
|
+
}
|
|
29880
|
+
]);
|
|
29881
|
+
}
|
|
29882
|
+
return value;
|
|
29883
|
+
}
|
|
29884
|
+
function normalizeVariant(value) {
|
|
29885
|
+
if (value === null || value === void 0) {
|
|
29886
|
+
return null;
|
|
29887
|
+
}
|
|
29888
|
+
return normalizedString(value) ?? null;
|
|
29889
|
+
}
|
|
29890
|
+
function missingRequiredIssue(field) {
|
|
29891
|
+
return {
|
|
29892
|
+
code: "missing_identity_field",
|
|
29893
|
+
severity: "error",
|
|
29894
|
+
field,
|
|
29895
|
+
message: `Projection identity requires ${field}.`
|
|
29896
|
+
};
|
|
29897
|
+
}
|
|
29898
|
+
function encodeKeyValue(value) {
|
|
29899
|
+
if (value === null || value === void 0) {
|
|
29900
|
+
return "~";
|
|
29901
|
+
}
|
|
29902
|
+
return encodeURIComponent(String(value));
|
|
29903
|
+
}
|
|
29904
|
+
function dimensionEntries(dimensions) {
|
|
29905
|
+
return {
|
|
29906
|
+
projection_format: dimensions.projectionFormat,
|
|
29907
|
+
projection_version: dimensions.projectionVersion,
|
|
29908
|
+
run_id: dimensions.runId,
|
|
29909
|
+
suite: dimensions.suite ?? null,
|
|
29910
|
+
eval_path: dimensions.evalPath ?? null,
|
|
29911
|
+
test_id: dimensions.testId,
|
|
29912
|
+
target: dimensions.target,
|
|
29913
|
+
source_target: dimensions.sourceTarget,
|
|
29914
|
+
attempt: dimensions.attempt,
|
|
29915
|
+
variant: dimensions.variant,
|
|
29916
|
+
envelope_id: dimensions.envelopeId,
|
|
29917
|
+
trace_id: dimensions.traceId,
|
|
29918
|
+
root_span_id: dimensions.rootSpanId
|
|
29919
|
+
};
|
|
29920
|
+
}
|
|
29921
|
+
function buildProjectionIdentityKey(dimensions) {
|
|
29922
|
+
const entries = dimensionEntries(dimensions);
|
|
29923
|
+
const body = DIMENSION_KEY_ORDER.map((key) => `${key}=${encodeKeyValue(entries[key])}`).join("|");
|
|
29924
|
+
return `${PROJECTION_IDENTITY_SCHEMA_VERSION}:${body}`;
|
|
29925
|
+
}
|
|
29926
|
+
function buildProjectionIdentity(input, options) {
|
|
29927
|
+
const values = {
|
|
29928
|
+
runId: normalizedString(input.runId),
|
|
29929
|
+
suite: normalizedString(input.suite),
|
|
29930
|
+
evalPath: normalizedString(input.evalPath),
|
|
29931
|
+
testId: normalizedString(input.testId),
|
|
29932
|
+
target: normalizedString(input.target),
|
|
29933
|
+
sourceTarget: normalizedString(input.sourceTarget) ?? normalizedString(input.target),
|
|
29934
|
+
envelopeId: normalizedString(input.envelopeId),
|
|
29935
|
+
traceId: normalizedString(input.traceId),
|
|
29936
|
+
rootSpanId: normalizedString(input.rootSpanId),
|
|
29937
|
+
projectionFormat: normalizedString(input.projectionFormat),
|
|
29938
|
+
projectionVersion: normalizedString(input.projectionVersion)
|
|
29939
|
+
};
|
|
29940
|
+
const issues = [];
|
|
29941
|
+
for (const field of REQUIRED_STRING_FIELDS) {
|
|
29942
|
+
if (!values[field]) {
|
|
29943
|
+
issues.push(missingRequiredIssue(field));
|
|
29944
|
+
}
|
|
29945
|
+
}
|
|
29946
|
+
if (!values.suite && !values.evalPath) {
|
|
29947
|
+
issues.push({
|
|
29948
|
+
code: "missing_suite_or_eval_path",
|
|
29949
|
+
severity: "warning",
|
|
29950
|
+
field: "suite/evalPath",
|
|
29951
|
+
message: "Projection identity is more portable when at least one of suite or evalPath is present."
|
|
29952
|
+
});
|
|
29953
|
+
}
|
|
29954
|
+
const errors = issues.filter((issue2) => issue2.severity === "error");
|
|
29955
|
+
if (errors.length > 0 && (options?.missingFieldPolicy ?? "error") === "error") {
|
|
29956
|
+
throw new ProjectionIdentityError(errors);
|
|
29957
|
+
}
|
|
29958
|
+
const dimensions = {
|
|
29959
|
+
runId: values.runId ?? "unknown",
|
|
29960
|
+
suite: values.suite,
|
|
29961
|
+
evalPath: values.evalPath,
|
|
29962
|
+
testId: values.testId ?? "unknown",
|
|
29963
|
+
target: values.target ?? "unknown",
|
|
29964
|
+
sourceTarget: values.sourceTarget ?? "unknown",
|
|
29965
|
+
attempt: normalizeAttempt(input.attempt),
|
|
29966
|
+
variant: normalizeVariant(input.variant),
|
|
29967
|
+
envelopeId: values.envelopeId ?? "unknown",
|
|
29968
|
+
traceId: values.traceId ?? "unknown",
|
|
29969
|
+
rootSpanId: values.rootSpanId ?? "unknown",
|
|
29970
|
+
projectionFormat: values.projectionFormat ?? "unknown",
|
|
29971
|
+
projectionVersion: values.projectionVersion ?? "unknown"
|
|
29972
|
+
};
|
|
29973
|
+
const key = buildProjectionIdentityKey(dimensions);
|
|
29974
|
+
const digest = createHash2("sha256").update(key).digest("hex");
|
|
29975
|
+
return {
|
|
29976
|
+
schemaVersion: PROJECTION_IDENTITY_SCHEMA_VERSION,
|
|
29977
|
+
id: `agentv-prj-${digest.slice(0, 32)}`,
|
|
29978
|
+
key,
|
|
29979
|
+
dimensions,
|
|
29980
|
+
issues: issues.length > 0 ? issues : void 0
|
|
29981
|
+
};
|
|
29982
|
+
}
|
|
29983
|
+
var ProjectionIdentityIssueWireSchema = external_exports.object({
|
|
29984
|
+
code: external_exports.string(),
|
|
29985
|
+
severity: external_exports.enum(PROJECTION_IDENTITY_ISSUE_SEVERITY_VALUES),
|
|
29986
|
+
field: external_exports.string().optional(),
|
|
29987
|
+
message: external_exports.string()
|
|
29988
|
+
}).strict();
|
|
29989
|
+
var ProjectionIdentityDimensionsWireSchema = external_exports.object({
|
|
29990
|
+
run_id: external_exports.string(),
|
|
29991
|
+
suite: external_exports.string().optional(),
|
|
29992
|
+
eval_path: external_exports.string().optional(),
|
|
29993
|
+
test_id: external_exports.string(),
|
|
29994
|
+
target: external_exports.string(),
|
|
29995
|
+
source_target: external_exports.string(),
|
|
29996
|
+
attempt: external_exports.number().int().nonnegative(),
|
|
29997
|
+
variant: external_exports.string().nullable(),
|
|
29998
|
+
envelope_id: external_exports.string(),
|
|
29999
|
+
trace_id: external_exports.string(),
|
|
30000
|
+
root_span_id: external_exports.string(),
|
|
30001
|
+
projection_format: external_exports.string(),
|
|
30002
|
+
projection_version: external_exports.string()
|
|
30003
|
+
}).strict();
|
|
30004
|
+
var ProjectionIdentityWireSchema = external_exports.object({
|
|
30005
|
+
schema_version: external_exports.literal(PROJECTION_IDENTITY_SCHEMA_VERSION),
|
|
30006
|
+
id: external_exports.string(),
|
|
30007
|
+
key: external_exports.string(),
|
|
30008
|
+
dimensions: ProjectionIdentityDimensionsWireSchema,
|
|
30009
|
+
issues: external_exports.array(ProjectionIdentityIssueWireSchema).optional()
|
|
30010
|
+
}).strict();
|
|
30011
|
+
function dropUndefined2(value) {
|
|
30012
|
+
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
30013
|
+
}
|
|
30014
|
+
function toProjectionIdentityIssueWire(issue2) {
|
|
30015
|
+
return ProjectionIdentityIssueWireSchema.parse(
|
|
30016
|
+
dropUndefined2({
|
|
30017
|
+
code: issue2.code,
|
|
30018
|
+
severity: issue2.severity,
|
|
30019
|
+
field: issue2.field,
|
|
30020
|
+
message: issue2.message
|
|
30021
|
+
})
|
|
30022
|
+
);
|
|
30023
|
+
}
|
|
30024
|
+
function fromProjectionIdentityIssueWire(issue2) {
|
|
30025
|
+
return {
|
|
30026
|
+
code: issue2.code,
|
|
30027
|
+
severity: issue2.severity,
|
|
30028
|
+
field: issue2.field,
|
|
30029
|
+
message: issue2.message
|
|
30030
|
+
};
|
|
30031
|
+
}
|
|
30032
|
+
function toProjectionIdentityWire(identity) {
|
|
30033
|
+
return ProjectionIdentityWireSchema.parse(
|
|
30034
|
+
dropUndefined2({
|
|
30035
|
+
schema_version: identity.schemaVersion,
|
|
30036
|
+
id: identity.id,
|
|
30037
|
+
key: identity.key,
|
|
30038
|
+
dimensions: {
|
|
30039
|
+
run_id: identity.dimensions.runId,
|
|
30040
|
+
suite: identity.dimensions.suite,
|
|
30041
|
+
eval_path: identity.dimensions.evalPath,
|
|
30042
|
+
test_id: identity.dimensions.testId,
|
|
30043
|
+
target: identity.dimensions.target,
|
|
30044
|
+
source_target: identity.dimensions.sourceTarget,
|
|
30045
|
+
attempt: identity.dimensions.attempt,
|
|
30046
|
+
variant: identity.dimensions.variant,
|
|
30047
|
+
envelope_id: identity.dimensions.envelopeId,
|
|
30048
|
+
trace_id: identity.dimensions.traceId,
|
|
30049
|
+
root_span_id: identity.dimensions.rootSpanId,
|
|
30050
|
+
projection_format: identity.dimensions.projectionFormat,
|
|
30051
|
+
projection_version: identity.dimensions.projectionVersion
|
|
30052
|
+
},
|
|
30053
|
+
issues: identity.issues?.map(toProjectionIdentityIssueWire)
|
|
30054
|
+
})
|
|
30055
|
+
);
|
|
30056
|
+
}
|
|
30057
|
+
function fromProjectionIdentityWire(input) {
|
|
30058
|
+
const wire = ProjectionIdentityWireSchema.parse(input);
|
|
30059
|
+
return {
|
|
30060
|
+
schemaVersion: wire.schema_version,
|
|
30061
|
+
id: wire.id,
|
|
30062
|
+
key: wire.key,
|
|
30063
|
+
dimensions: {
|
|
30064
|
+
runId: wire.dimensions.run_id,
|
|
30065
|
+
suite: wire.dimensions.suite,
|
|
30066
|
+
evalPath: wire.dimensions.eval_path,
|
|
30067
|
+
testId: wire.dimensions.test_id,
|
|
30068
|
+
target: wire.dimensions.target,
|
|
30069
|
+
sourceTarget: wire.dimensions.source_target,
|
|
30070
|
+
attempt: wire.dimensions.attempt,
|
|
30071
|
+
variant: wire.dimensions.variant,
|
|
30072
|
+
envelopeId: wire.dimensions.envelope_id,
|
|
30073
|
+
traceId: wire.dimensions.trace_id,
|
|
30074
|
+
rootSpanId: wire.dimensions.root_span_id,
|
|
30075
|
+
projectionFormat: wire.dimensions.projection_format,
|
|
30076
|
+
projectionVersion: wire.dimensions.projection_version
|
|
30077
|
+
},
|
|
30078
|
+
issues: wire.issues?.map(fromProjectionIdentityIssueWire)
|
|
30079
|
+
};
|
|
30080
|
+
}
|
|
29762
30081
|
var EXECUTION_TRACE_SCHEMA_VERSION = "agentv.trace.v1";
|
|
29763
30082
|
var TRACE_ENVELOPE_FORMAT = "otlp_openinference_spans";
|
|
30083
|
+
var TRACE_ENVELOPE_PROJECTION_FORMAT = "execution_trace";
|
|
29764
30084
|
var TRANSCRIPT_MESSAGE_EVENT_NAME = "agentv.transcript.message";
|
|
29765
30085
|
var CAPTURE_CONTENT_VALUES = ["none", "metadata", "full"];
|
|
29766
30086
|
var REDACTION_LEVEL_VALUES = ["none", "partial", "full"];
|
|
@@ -29862,11 +30182,17 @@ var TraceEnvelopeScoreWireSchema = external_exports.object({
|
|
|
29862
30182
|
target_span_id: external_exports.string().optional(),
|
|
29863
30183
|
evidence: AttributeMapWireSchema.optional()
|
|
29864
30184
|
}).strict();
|
|
30185
|
+
var TraceEnvelopeExportMetadataWireSchema = external_exports.object({
|
|
30186
|
+
duplicate_policy: external_exports.enum(["skip", "update", "error"]).optional(),
|
|
30187
|
+
identity_warnings: external_exports.array(ProjectionIdentityIssueWireSchema).optional()
|
|
30188
|
+
}).strict();
|
|
29865
30189
|
var TraceEnvelopeWireSchema = external_exports.object({
|
|
29866
30190
|
schema_version: external_exports.literal(EXECUTION_TRACE_SCHEMA_VERSION),
|
|
29867
30191
|
artifact_id: external_exports.string(),
|
|
29868
30192
|
created_at: external_exports.string(),
|
|
29869
30193
|
eval: TraceEnvelopeEvalWireSchema,
|
|
30194
|
+
projection_identity: ProjectionIdentityWireSchema.optional(),
|
|
30195
|
+
export_metadata: TraceEnvelopeExportMetadataWireSchema.optional(),
|
|
29870
30196
|
replay: TraceEnvelopeReplayWireSchema.optional(),
|
|
29871
30197
|
trace: TraceEnvelopeBodyWireSchema,
|
|
29872
30198
|
source: TraceEnvelopeSourceWireSchema,
|
|
@@ -29875,7 +30201,7 @@ var TraceEnvelopeWireSchema = external_exports.object({
|
|
|
29875
30201
|
artifacts: external_exports.record(external_exports.string(), external_exports.string()).optional(),
|
|
29876
30202
|
scores: external_exports.array(TraceEnvelopeScoreWireSchema).optional()
|
|
29877
30203
|
}).strict();
|
|
29878
|
-
function
|
|
30204
|
+
function dropUndefined3(value) {
|
|
29879
30205
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
29880
30206
|
}
|
|
29881
30207
|
function isRecord(value) {
|
|
@@ -29892,7 +30218,7 @@ function definedStringRecord(value) {
|
|
|
29892
30218
|
}
|
|
29893
30219
|
function hashHex(parts, length) {
|
|
29894
30220
|
const stable = parts.map((part) => typeof part === "string" ? part : JSON.stringify(part)).join("\0");
|
|
29895
|
-
return
|
|
30221
|
+
return createHash3("sha256").update(stable).digest("hex").slice(0, length);
|
|
29896
30222
|
}
|
|
29897
30223
|
function parseTimeMs(timestamp) {
|
|
29898
30224
|
if (!timestamp) {
|
|
@@ -29961,7 +30287,7 @@ function tokenUsageAttributes(usage) {
|
|
|
29961
30287
|
if (!usage) {
|
|
29962
30288
|
return {};
|
|
29963
30289
|
}
|
|
29964
|
-
return
|
|
30290
|
+
return dropUndefined3({
|
|
29965
30291
|
"gen_ai.usage.input_tokens": usage.input,
|
|
29966
30292
|
"gen_ai.usage.output_tokens": usage.output,
|
|
29967
30293
|
"gen_ai.usage.cache_read.input_tokens": usage.cached,
|
|
@@ -29972,13 +30298,17 @@ function tokenUsageAttributes(usage) {
|
|
|
29972
30298
|
}
|
|
29973
30299
|
function sourceFromResult(result, options) {
|
|
29974
30300
|
const traceProvider = typeof result.trace.metadata?.provider === "string" ? result.trace.metadata.provider : void 0;
|
|
30301
|
+
const metadata = dropUndefined3({
|
|
30302
|
+
...result.trace.metadata,
|
|
30303
|
+
...options.source?.metadata
|
|
30304
|
+
});
|
|
29975
30305
|
return {
|
|
29976
30306
|
kind: options.source?.kind ?? "agentv_run",
|
|
29977
30307
|
path: options.source?.path,
|
|
29978
30308
|
provider: options.source?.provider ?? traceProvider ?? result.target,
|
|
29979
30309
|
format: options.source?.format ?? "agentv_result",
|
|
29980
30310
|
version: options.source?.version ?? "1",
|
|
29981
|
-
metadata:
|
|
30311
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : void 0
|
|
29982
30312
|
};
|
|
29983
30313
|
}
|
|
29984
30314
|
function capturePolicy(options) {
|
|
@@ -30013,24 +30343,37 @@ function maybeToolContentAttributes(toolCall, capture) {
|
|
|
30013
30343
|
if (capture.content !== "full") {
|
|
30014
30344
|
return {};
|
|
30015
30345
|
}
|
|
30016
|
-
return
|
|
30346
|
+
return dropUndefined3({
|
|
30017
30347
|
"gen_ai.tool.call.arguments": toolCall.input,
|
|
30018
30348
|
"gen_ai.tool.call.result": toolCall.output
|
|
30019
30349
|
});
|
|
30020
30350
|
}
|
|
30351
|
+
function toolCallStatusToSpanStatus(toolCall) {
|
|
30352
|
+
if (toolCall.status === "error" || toolCall.status === "timeout" || toolCall.status === "cancelled") {
|
|
30353
|
+
return { code: "ERROR", message: toolCall.status };
|
|
30354
|
+
}
|
|
30355
|
+
return { code: "OK" };
|
|
30356
|
+
}
|
|
30357
|
+
function spanStatusToToolCallStatus(status) {
|
|
30358
|
+
if (status.code === "ERROR") {
|
|
30359
|
+
return "error";
|
|
30360
|
+
}
|
|
30361
|
+
return "ok";
|
|
30362
|
+
}
|
|
30021
30363
|
function toTranscriptToolCallWire(toolCall, capture) {
|
|
30022
|
-
return
|
|
30364
|
+
return dropUndefined3({
|
|
30023
30365
|
tool: toolCall.tool,
|
|
30024
30366
|
input: capture.content === "full" ? toolCall.input : void 0,
|
|
30025
30367
|
output: capture.content === "full" ? toolCall.output : void 0,
|
|
30026
30368
|
id: toolCall.id,
|
|
30369
|
+
status: toolCall.status,
|
|
30027
30370
|
start_time: toolCall.startTime,
|
|
30028
30371
|
end_time: toolCall.endTime,
|
|
30029
30372
|
duration_ms: toolCall.durationMs
|
|
30030
30373
|
});
|
|
30031
30374
|
}
|
|
30032
30375
|
function toTranscriptMessageWire(message, capture) {
|
|
30033
|
-
return
|
|
30376
|
+
return dropUndefined3({
|
|
30034
30377
|
role: message.role,
|
|
30035
30378
|
name: message.name,
|
|
30036
30379
|
content: capture.content === "full" ? message.content : void 0,
|
|
@@ -30047,7 +30390,7 @@ function transcriptMessageEvent(message, index, capture) {
|
|
|
30047
30390
|
return {
|
|
30048
30391
|
name: TRANSCRIPT_MESSAGE_EVENT_NAME,
|
|
30049
30392
|
timeUnixNano: startMs !== void 0 ? msToUnixNano(startMs) : void 0,
|
|
30050
|
-
attributes:
|
|
30393
|
+
attributes: dropUndefined3({
|
|
30051
30394
|
"agentv.transcript.message.index": index,
|
|
30052
30395
|
"agentv.transcript.message": toTranscriptMessageWire(message, capture)
|
|
30053
30396
|
})
|
|
@@ -30081,10 +30424,10 @@ function scoresFromResult(result, targetSpanId) {
|
|
|
30081
30424
|
source: scoreSource(score.type),
|
|
30082
30425
|
evaluatedAt: score.endedAt ?? score.startedAt ?? result.timestamp,
|
|
30083
30426
|
targetSpanId,
|
|
30084
|
-
evidence:
|
|
30427
|
+
evidence: dropUndefined3({
|
|
30085
30428
|
span_ids: [targetSpanId],
|
|
30086
30429
|
assertions: score.assertions.map(
|
|
30087
|
-
(assertion) =>
|
|
30430
|
+
(assertion) => dropUndefined3({
|
|
30088
30431
|
text: assertion.text,
|
|
30089
30432
|
passed: assertion.passed,
|
|
30090
30433
|
evidence: assertion.evidence
|
|
@@ -30118,7 +30461,7 @@ function buildTraceEnvelopeFromEvaluationResult(result, options = {}) {
|
|
|
30118
30461
|
const rootEvents = result.trace.messages.map(
|
|
30119
30462
|
(message, index) => transcriptMessageEvent(message, index, capture)
|
|
30120
30463
|
);
|
|
30121
|
-
const rootAttributes =
|
|
30464
|
+
const rootAttributes = dropUndefined3({
|
|
30122
30465
|
"gen_ai.operation.name": "invoke_agent",
|
|
30123
30466
|
"gen_ai.provider.name": "agentv",
|
|
30124
30467
|
"gen_ai.agent.name": result.target,
|
|
@@ -30128,8 +30471,8 @@ function buildTraceEnvelopeFromEvaluationResult(result, options = {}) {
|
|
|
30128
30471
|
"agentv.suite": result.suite,
|
|
30129
30472
|
"agentv.category": result.category,
|
|
30130
30473
|
"agentv.eval_path": options.evalPath,
|
|
30131
|
-
"agentv.run_id": options.runId,
|
|
30132
|
-
"agentv.attempt": options.attempt,
|
|
30474
|
+
"agentv.run_id": options.runId ?? result.timestamp,
|
|
30475
|
+
"agentv.attempt": options.attempt ?? 0,
|
|
30133
30476
|
"agentv.variant": options.variant ?? void 0,
|
|
30134
30477
|
"agentv.execution_status": result.executionStatus,
|
|
30135
30478
|
"agentv.failure_stage": result.failureStage,
|
|
@@ -30180,7 +30523,7 @@ function buildTraceEnvelopeFromEvaluationResult(result, options = {}) {
|
|
|
30180
30523
|
startTimeUnixNano: msToUnixNano(chatTiming.startMs),
|
|
30181
30524
|
endTimeUnixNano: msToUnixNano(chatTiming.endMs),
|
|
30182
30525
|
status: { code: "OK" },
|
|
30183
|
-
attributes:
|
|
30526
|
+
attributes: dropUndefined3({
|
|
30184
30527
|
"gen_ai.operation.name": "chat",
|
|
30185
30528
|
"gen_ai.provider.name": source.provider,
|
|
30186
30529
|
"gen_ai.request.model": model,
|
|
@@ -30222,8 +30565,8 @@ function buildTraceEnvelopeFromEvaluationResult(result, options = {}) {
|
|
|
30222
30565
|
kind: "INTERNAL",
|
|
30223
30566
|
startTimeUnixNano: msToUnixNano(toolTiming.startMs),
|
|
30224
30567
|
endTimeUnixNano: msToUnixNano(toolTiming.endMs),
|
|
30225
|
-
status:
|
|
30226
|
-
attributes:
|
|
30568
|
+
status: toolCallStatusToSpanStatus(toolCall),
|
|
30569
|
+
attributes: dropUndefined3({
|
|
30227
30570
|
"gen_ai.operation.name": "execute_tool",
|
|
30228
30571
|
"gen_ai.tool.name": toolCall.tool,
|
|
30229
30572
|
"gen_ai.tool.call.id": generatedToolCallId,
|
|
@@ -30240,24 +30583,50 @@ function buildTraceEnvelopeFromEvaluationResult(result, options = {}) {
|
|
|
30240
30583
|
}
|
|
30241
30584
|
}
|
|
30242
30585
|
const artifactId = `execution-trace-${hashHex([traceId, result.timestamp, result.score], 20)}`;
|
|
30586
|
+
const sourceTarget = options.sourceTarget ?? result.target;
|
|
30587
|
+
const attempt = options.attempt ?? 0;
|
|
30588
|
+
const variant = options.variant ?? null;
|
|
30589
|
+
const runId = options.runId ?? result.timestamp;
|
|
30243
30590
|
const evalIdentity = {
|
|
30244
30591
|
evalId: options.evalId,
|
|
30245
30592
|
evalPath: options.evalPath,
|
|
30246
30593
|
suite: result.suite,
|
|
30247
30594
|
testId: result.testId,
|
|
30248
30595
|
target: result.target,
|
|
30249
|
-
sourceTarget
|
|
30250
|
-
attempt
|
|
30251
|
-
variant
|
|
30252
|
-
runId
|
|
30596
|
+
sourceTarget,
|
|
30597
|
+
attempt,
|
|
30598
|
+
variant,
|
|
30599
|
+
runId,
|
|
30253
30600
|
category: result.category,
|
|
30254
30601
|
experiment: options.experiment
|
|
30255
30602
|
};
|
|
30603
|
+
const projectionIdentity = buildProjectionIdentity({
|
|
30604
|
+
runId,
|
|
30605
|
+
suite: evalIdentity.suite,
|
|
30606
|
+
evalPath: evalIdentity.evalPath,
|
|
30607
|
+
testId: evalIdentity.testId,
|
|
30608
|
+
target: evalIdentity.target,
|
|
30609
|
+
sourceTarget,
|
|
30610
|
+
attempt,
|
|
30611
|
+
variant,
|
|
30612
|
+
envelopeId: artifactId,
|
|
30613
|
+
traceId,
|
|
30614
|
+
rootSpanId,
|
|
30615
|
+
projectionFormat: TRACE_ENVELOPE_PROJECTION_FORMAT,
|
|
30616
|
+
projectionVersion: EXECUTION_TRACE_SCHEMA_VERSION
|
|
30617
|
+
});
|
|
30256
30618
|
return {
|
|
30257
30619
|
schemaVersion: EXECUTION_TRACE_SCHEMA_VERSION,
|
|
30258
30620
|
artifactId,
|
|
30259
30621
|
createdAt: now.toISOString(),
|
|
30260
30622
|
eval: evalIdentity,
|
|
30623
|
+
projectionIdentity,
|
|
30624
|
+
exportMetadata: options.duplicatePolicy ? {
|
|
30625
|
+
duplicatePolicy: options.duplicatePolicy,
|
|
30626
|
+
identityWarnings: projectionIdentity.issues?.filter(
|
|
30627
|
+
(issue2) => issue2.severity === "warning"
|
|
30628
|
+
)
|
|
30629
|
+
} : void 0,
|
|
30261
30630
|
replay: options.replay,
|
|
30262
30631
|
trace: {
|
|
30263
30632
|
format: TRACE_ENVELOPE_FORMAT,
|
|
@@ -30276,11 +30645,13 @@ function buildTraceEnvelopeFromEvaluationResult(result, options = {}) {
|
|
|
30276
30645
|
}
|
|
30277
30646
|
function toTraceEnvelopeWire(envelope) {
|
|
30278
30647
|
return TraceEnvelopeWireSchema.parse(
|
|
30279
|
-
|
|
30648
|
+
dropUndefined3({
|
|
30280
30649
|
schema_version: envelope.schemaVersion,
|
|
30281
30650
|
artifact_id: envelope.artifactId,
|
|
30282
30651
|
created_at: envelope.createdAt,
|
|
30283
30652
|
eval: toTraceEnvelopeEvalWire(envelope.eval),
|
|
30653
|
+
projection_identity: envelope.projectionIdentity ? toProjectionIdentityWire(envelope.projectionIdentity) : void 0,
|
|
30654
|
+
export_metadata: envelope.exportMetadata ? toTraceEnvelopeExportMetadataWire(envelope.exportMetadata) : void 0,
|
|
30284
30655
|
replay: envelope.replay ? toTraceEnvelopeReplayWire(envelope.replay) : void 0,
|
|
30285
30656
|
trace: toTraceEnvelopeBodyWire(envelope.trace),
|
|
30286
30657
|
source: toTraceEnvelopeSourceWire(envelope.source),
|
|
@@ -30298,6 +30669,8 @@ function fromTraceEnvelopeWire(input) {
|
|
|
30298
30669
|
artifactId: wire.artifact_id,
|
|
30299
30670
|
createdAt: wire.created_at,
|
|
30300
30671
|
eval: fromTraceEnvelopeEvalWire(wire.eval),
|
|
30672
|
+
projectionIdentity: wire.projection_identity ? fromProjectionIdentityWire(wire.projection_identity) : void 0,
|
|
30673
|
+
exportMetadata: wire.export_metadata ? fromTraceEnvelopeExportMetadataWire(wire.export_metadata) : void 0,
|
|
30301
30674
|
replay: wire.replay ? fromTraceEnvelopeReplayWire(wire.replay) : void 0,
|
|
30302
30675
|
trace: fromTraceEnvelopeBodyWire(wire.trace),
|
|
30303
30676
|
source: fromTraceEnvelopeSourceWire(wire.source),
|
|
@@ -30309,7 +30682,7 @@ function fromTraceEnvelopeWire(input) {
|
|
|
30309
30682
|
}
|
|
30310
30683
|
function toTraceEnvelopeEvalWire(evaluation) {
|
|
30311
30684
|
return TraceEnvelopeEvalWireSchema.parse(
|
|
30312
|
-
|
|
30685
|
+
dropUndefined3({
|
|
30313
30686
|
eval_id: evaluation.evalId,
|
|
30314
30687
|
eval_path: evaluation.evalPath,
|
|
30315
30688
|
suite: evaluation.suite,
|
|
@@ -30339,9 +30712,23 @@ function fromTraceEnvelopeEvalWire(evaluation) {
|
|
|
30339
30712
|
experiment: evaluation.experiment
|
|
30340
30713
|
};
|
|
30341
30714
|
}
|
|
30715
|
+
function toTraceEnvelopeExportMetadataWire(metadata) {
|
|
30716
|
+
return TraceEnvelopeExportMetadataWireSchema.parse(
|
|
30717
|
+
dropUndefined3({
|
|
30718
|
+
duplicate_policy: metadata.duplicatePolicy,
|
|
30719
|
+
identity_warnings: metadata.identityWarnings?.map(toProjectionIdentityIssueWire)
|
|
30720
|
+
})
|
|
30721
|
+
);
|
|
30722
|
+
}
|
|
30723
|
+
function fromTraceEnvelopeExportMetadataWire(metadata) {
|
|
30724
|
+
return {
|
|
30725
|
+
duplicatePolicy: metadata.duplicate_policy,
|
|
30726
|
+
identityWarnings: metadata.identity_warnings?.map(fromProjectionIdentityIssueWire)
|
|
30727
|
+
};
|
|
30728
|
+
}
|
|
30342
30729
|
function toTraceEnvelopeReplayWire(replay) {
|
|
30343
30730
|
return TraceEnvelopeReplayWireSchema.parse(
|
|
30344
|
-
|
|
30731
|
+
dropUndefined3({
|
|
30345
30732
|
lookup_key: replay.lookupKey,
|
|
30346
30733
|
fixture_id: replay.fixtureId,
|
|
30347
30734
|
source_fixture_path: replay.sourceFixturePath
|
|
@@ -30357,7 +30744,7 @@ function fromTraceEnvelopeReplayWire(replay) {
|
|
|
30357
30744
|
}
|
|
30358
30745
|
function toTraceEnvelopeBodyWire(trace) {
|
|
30359
30746
|
return TraceEnvelopeBodyWireSchema.parse(
|
|
30360
|
-
|
|
30747
|
+
dropUndefined3({
|
|
30361
30748
|
format: trace.format,
|
|
30362
30749
|
trace_id: trace.traceId,
|
|
30363
30750
|
root_span_id: trace.rootSpanId,
|
|
@@ -30379,7 +30766,7 @@ function fromTraceEnvelopeBodyWire(trace) {
|
|
|
30379
30766
|
}
|
|
30380
30767
|
function toTraceEnvelopeSpanWire(span) {
|
|
30381
30768
|
return TraceEnvelopeSpanWireSchema.parse(
|
|
30382
|
-
|
|
30769
|
+
dropUndefined3({
|
|
30383
30770
|
trace_id: span.traceId,
|
|
30384
30771
|
span_id: span.spanId,
|
|
30385
30772
|
parent_span_id: span.parentSpanId,
|
|
@@ -30409,7 +30796,7 @@ function fromTraceEnvelopeSpanWire(span) {
|
|
|
30409
30796
|
}
|
|
30410
30797
|
function toTraceEnvelopeSpanEventWire(event) {
|
|
30411
30798
|
return TraceEnvelopeSpanEventWireSchema.parse(
|
|
30412
|
-
|
|
30799
|
+
dropUndefined3({
|
|
30413
30800
|
name: event.name,
|
|
30414
30801
|
time_unix_nano: event.timeUnixNano,
|
|
30415
30802
|
attributes: event.attributes
|
|
@@ -30425,7 +30812,7 @@ function fromTraceEnvelopeSpanEventWire(event) {
|
|
|
30425
30812
|
}
|
|
30426
30813
|
function toTraceEnvelopeSourceWire(source) {
|
|
30427
30814
|
return TraceEnvelopeSourceWireSchema.parse(
|
|
30428
|
-
|
|
30815
|
+
dropUndefined3({
|
|
30429
30816
|
kind: source.kind,
|
|
30430
30817
|
path: source.path,
|
|
30431
30818
|
provider: source.provider,
|
|
@@ -30447,7 +30834,7 @@ function fromTraceEnvelopeSourceWire(source) {
|
|
|
30447
30834
|
}
|
|
30448
30835
|
function toTraceEnvelopeCaptureWire(capture) {
|
|
30449
30836
|
return TraceEnvelopeCaptureWireSchema.parse(
|
|
30450
|
-
|
|
30837
|
+
dropUndefined3({
|
|
30451
30838
|
content: capture.content,
|
|
30452
30839
|
redaction_level: capture.redactionLevel,
|
|
30453
30840
|
redacted_fields: capture.redactedFields,
|
|
@@ -30465,7 +30852,7 @@ function fromTraceEnvelopeCaptureWire(capture) {
|
|
|
30465
30852
|
}
|
|
30466
30853
|
function toTraceEnvelopeSourceRefWire(sourceRef) {
|
|
30467
30854
|
return TraceEnvelopeSourceRefWireSchema.parse(
|
|
30468
|
-
|
|
30855
|
+
dropUndefined3({
|
|
30469
30856
|
event_id: sourceRef.eventId,
|
|
30470
30857
|
message_id: sourceRef.messageId,
|
|
30471
30858
|
span_id: sourceRef.spanId,
|
|
@@ -30491,7 +30878,7 @@ function fromTraceEnvelopeSourceRefWire(sourceRef) {
|
|
|
30491
30878
|
}
|
|
30492
30879
|
function toTraceEnvelopeConversionWarningWire(warning) {
|
|
30493
30880
|
return TraceEnvelopeConversionWarningWireSchema.parse(
|
|
30494
|
-
|
|
30881
|
+
dropUndefined3({
|
|
30495
30882
|
code: warning.code,
|
|
30496
30883
|
severity: warning.severity,
|
|
30497
30884
|
span_id: warning.spanId,
|
|
@@ -30513,7 +30900,7 @@ function fromTraceEnvelopeConversionWarningWire(warning) {
|
|
|
30513
30900
|
}
|
|
30514
30901
|
function toTraceEnvelopeScoreWire(score) {
|
|
30515
30902
|
return TraceEnvelopeScoreWireSchema.parse(
|
|
30516
|
-
|
|
30903
|
+
dropUndefined3({
|
|
30517
30904
|
name: score.name,
|
|
30518
30905
|
type: score.type,
|
|
30519
30906
|
score: score.score,
|
|
@@ -30573,6 +30960,7 @@ function toolCallFromSpan(span) {
|
|
|
30573
30960
|
id: stringAttribute(attributes, "gen_ai.tool.call.id") ?? stringAttribute(attributes, "tool.id") ?? span.spanId,
|
|
30574
30961
|
input: attributes["gen_ai.tool.call.arguments"],
|
|
30575
30962
|
output: attributes["gen_ai.tool.call.result"],
|
|
30963
|
+
status: spanStatusToToolCallStatus(span.status),
|
|
30576
30964
|
startTime: unixNanoToIso(span.startTimeUnixNano),
|
|
30577
30965
|
endTime: unixNanoToIso(span.endTimeUnixNano),
|
|
30578
30966
|
durationMs: durationMsFromSpan(span)
|
|
@@ -30610,6 +30998,7 @@ function fromTranscriptToolCallWire(wire) {
|
|
|
30610
30998
|
input: wire.input,
|
|
30611
30999
|
output: wire.output,
|
|
30612
31000
|
id: typeof wire.id === "string" ? wire.id : void 0,
|
|
31001
|
+
status: wire.status === "ok" || wire.status === "error" || wire.status === "timeout" || wire.status === "cancelled" || wire.status === "unknown" ? wire.status : void 0,
|
|
30613
31002
|
startTime: typeof wire.start_time === "string" ? wire.start_time : void 0,
|
|
30614
31003
|
endTime: typeof wire.end_time === "string" ? wire.end_time : void 0,
|
|
30615
31004
|
durationMs: numberAttribute(wire, "duration_ms")
|
|
@@ -30620,7 +31009,7 @@ function fromTranscriptMessageWire(wire) {
|
|
|
30620
31009
|
return void 0;
|
|
30621
31010
|
}
|
|
30622
31011
|
const toolCalls = Array.isArray(wire.tool_calls) ? wire.tool_calls.map(fromTranscriptToolCallWire).filter((toolCall) => toolCall !== void 0) : void 0;
|
|
30623
|
-
return
|
|
31012
|
+
return dropUndefined3({
|
|
30624
31013
|
role: wire.role,
|
|
30625
31014
|
name: typeof wire.name === "string" ? wire.name : void 0,
|
|
30626
31015
|
content: wire.content,
|
|
@@ -30658,7 +31047,7 @@ function traceEnvelopeToMessageEntries(envelope) {
|
|
|
30658
31047
|
endTime: unixNanoToIso(span.endTimeUnixNano),
|
|
30659
31048
|
durationMs: durationMsFromSpan(span),
|
|
30660
31049
|
tokenUsage: tokenUsageFromAttributes(span.attributes),
|
|
30661
|
-
metadata:
|
|
31050
|
+
metadata: dropUndefined3({
|
|
30662
31051
|
span_id: span.spanId,
|
|
30663
31052
|
trace_id: span.traceId,
|
|
30664
31053
|
parent_span_id: span.parentSpanId ?? void 0,
|
|
@@ -30725,7 +31114,7 @@ function traceEnvelopeToToolTrajectoryView(envelope) {
|
|
|
30725
31114
|
parentToolCallId: nearestAncestorToolCallId(ancestorIds, spansById),
|
|
30726
31115
|
input: toolCall.input,
|
|
30727
31116
|
output: toolCall.output,
|
|
30728
|
-
status: span.status
|
|
31117
|
+
status: spanStatusToToolCallStatus(span.status),
|
|
30729
31118
|
startTime: toolCall.startTime,
|
|
30730
31119
|
endTime: toolCall.endTime,
|
|
30731
31120
|
durationMs: toolCall.durationMs
|
|
@@ -30822,7 +31211,7 @@ function traceEnvelopeToTraceArtifact(envelope) {
|
|
|
30822
31211
|
callId: toolCall.id,
|
|
30823
31212
|
input: toolCall.input,
|
|
30824
31213
|
output: toolCall.output,
|
|
30825
|
-
status: span.status
|
|
31214
|
+
status: spanStatusToToolCallStatus(span.status)
|
|
30826
31215
|
},
|
|
30827
31216
|
sourceRef: {
|
|
30828
31217
|
spanId: span.spanId,
|
|
@@ -30865,12 +31254,12 @@ function traceEnvelopeToOtlpJson(envelope) {
|
|
|
30865
31254
|
},
|
|
30866
31255
|
scopeSpans: [
|
|
30867
31256
|
{
|
|
30868
|
-
scope:
|
|
31257
|
+
scope: dropUndefined3({
|
|
30869
31258
|
name: envelope.trace.scope?.name,
|
|
30870
31259
|
version: envelope.trace.scope?.version
|
|
30871
31260
|
}),
|
|
30872
31261
|
spans: orderedSpans(envelope.trace.spans).map(
|
|
30873
|
-
(span) =>
|
|
31262
|
+
(span) => dropUndefined3({
|
|
30874
31263
|
traceId: span.traceId,
|
|
30875
31264
|
spanId: span.spanId,
|
|
30876
31265
|
parentSpanId: span.parentSpanId ?? void 0,
|
|
@@ -30881,7 +31270,7 @@ function traceEnvelopeToOtlpJson(envelope) {
|
|
|
30881
31270
|
attributes: attributesToOtlp(span.attributes),
|
|
30882
31271
|
status: spanStatusToOtlp(span.status),
|
|
30883
31272
|
events: span.events?.map(
|
|
30884
|
-
(event) =>
|
|
31273
|
+
(event) => dropUndefined3({
|
|
30885
31274
|
name: event.name,
|
|
30886
31275
|
timeUnixNano: event.timeUnixNano,
|
|
30887
31276
|
attributes: attributesToOtlp(event.attributes)
|
|
@@ -30912,7 +31301,7 @@ function spanKindToOtlp(kind) {
|
|
|
30912
31301
|
}
|
|
30913
31302
|
function spanStatusToOtlp(status) {
|
|
30914
31303
|
const code = status.code === "OK" ? 1 : status.code === "ERROR" ? 2 : 0;
|
|
30915
|
-
return
|
|
31304
|
+
return dropUndefined3({ code, message: status.message });
|
|
30916
31305
|
}
|
|
30917
31306
|
function attributesToOtlp(attributes) {
|
|
30918
31307
|
return Object.entries(attributes ?? {}).map(([key, value]) => ({
|
|
@@ -30991,7 +31380,7 @@ function traceEnvelopeReplayRecordToProviderResponse(record2) {
|
|
|
30991
31380
|
startTime: summary.startTime,
|
|
30992
31381
|
endTime: summary.endTime,
|
|
30993
31382
|
raw: {
|
|
30994
|
-
replay_execution_trace:
|
|
31383
|
+
replay_execution_trace: dropUndefined4({
|
|
30995
31384
|
artifact_id: record2.envelope.artifactId,
|
|
30996
31385
|
source_path: record2.sourcePath,
|
|
30997
31386
|
line_number: record2.lineNumber,
|
|
@@ -31085,7 +31474,7 @@ function assertReplayableMessages(output, record2) {
|
|
|
31085
31474
|
function formatRecordLocation(record2) {
|
|
31086
31475
|
return record2.lineNumber === void 0 ? record2.sourcePath : `${record2.sourcePath}:${record2.lineNumber}`;
|
|
31087
31476
|
}
|
|
31088
|
-
function
|
|
31477
|
+
function dropUndefined4(value) {
|
|
31089
31478
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
31090
31479
|
}
|
|
31091
31480
|
var ReplayProvider = class {
|
|
@@ -33427,7 +33816,7 @@ function computeWorkspaceFingerprint(repos) {
|
|
|
33427
33816
|
const canonical = {
|
|
33428
33817
|
repos: [...repos].sort((a, b) => (a.path ?? "").localeCompare(b.path ?? "")).map(normalizeRepoForFingerprint)
|
|
33429
33818
|
};
|
|
33430
|
-
return
|
|
33819
|
+
return createHash4("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
33431
33820
|
}
|
|
33432
33821
|
async function copyDirectoryRecursive2(src, dest, skipDirs) {
|
|
33433
33822
|
await mkdir15(dest, { recursive: true });
|
|
@@ -34070,7 +34459,7 @@ ${output}` : ""}`
|
|
|
34070
34459
|
return void 0;
|
|
34071
34460
|
}
|
|
34072
34461
|
gitCachePath(repoIdentity) {
|
|
34073
|
-
const hash =
|
|
34462
|
+
const hash = createHash5("sha256").update(repoIdentity).digest("hex");
|
|
34074
34463
|
return path37.join(getAgentvDataDir(), "git-cache", hash);
|
|
34075
34464
|
}
|
|
34076
34465
|
async withMirrorCacheLock(mirrorPath, action) {
|
|
@@ -38731,7 +39120,7 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
|
|
|
38731
39120
|
return { tests: await loadTestsFromAgentSkills(evalFilePath) };
|
|
38732
39121
|
}
|
|
38733
39122
|
if (format === "typescript") {
|
|
38734
|
-
const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-
|
|
39123
|
+
const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-YR5XK44N-V3P76ZCR.js");
|
|
38735
39124
|
return loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
|
|
38736
39125
|
}
|
|
38737
39126
|
const { tests, parsed, suiteWorkspacePath } = await loadTestsFromYaml(
|
|
@@ -38760,7 +39149,7 @@ async function loadTests(evalFilePath, repoRoot, options) {
|
|
|
38760
39149
|
return loadTestsFromAgentSkills(evalFilePath);
|
|
38761
39150
|
}
|
|
38762
39151
|
if (format === "typescript") {
|
|
38763
|
-
const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-
|
|
39152
|
+
const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-YR5XK44N-V3P76ZCR.js");
|
|
38764
39153
|
const suite = await loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
|
|
38765
39154
|
return suite.tests;
|
|
38766
39155
|
}
|
|
@@ -42003,7 +42392,7 @@ function extractProviderError(response) {
|
|
|
42003
42392
|
return trimmed.length > 0 ? trimmed : void 0;
|
|
42004
42393
|
}
|
|
42005
42394
|
function createCacheKey(provider, target, evalCase, promptInputs) {
|
|
42006
|
-
const hash =
|
|
42395
|
+
const hash = createHash6("sha256");
|
|
42007
42396
|
hash.update(provider.id);
|
|
42008
42397
|
hash.update(target.name);
|
|
42009
42398
|
hash.update(evalCase.id);
|
|
@@ -42148,37 +42537,60 @@ function createFunctionProvider(taskFn) {
|
|
|
42148
42537
|
}
|
|
42149
42538
|
};
|
|
42150
42539
|
}
|
|
42151
|
-
|
|
42540
|
+
var TRANSCRIPT_ROW_SCHEMA_VERSION = "agentv.transcript.v1";
|
|
42541
|
+
function dropUndefined5(value) {
|
|
42152
42542
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
42153
42543
|
}
|
|
42544
|
+
function isRecord3(value) {
|
|
42545
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
42546
|
+
}
|
|
42547
|
+
function optionalString(value) {
|
|
42548
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
42549
|
+
}
|
|
42154
42550
|
function toTranscriptTokenUsage(usage) {
|
|
42155
42551
|
if (!usage) {
|
|
42156
42552
|
return void 0;
|
|
42157
42553
|
}
|
|
42158
|
-
return
|
|
42554
|
+
return dropUndefined5({
|
|
42159
42555
|
input: usage.input,
|
|
42160
42556
|
output: usage.output,
|
|
42161
42557
|
cached: usage.cached,
|
|
42162
42558
|
reasoning: usage.reasoning
|
|
42163
42559
|
});
|
|
42164
42560
|
}
|
|
42165
|
-
function
|
|
42166
|
-
return
|
|
42167
|
-
|
|
42168
|
-
|
|
42169
|
-
|
|
42170
|
-
|
|
42171
|
-
|
|
42172
|
-
|
|
42173
|
-
|
|
42174
|
-
|
|
42561
|
+
function toTranscriptCaptureState(capture) {
|
|
42562
|
+
return {
|
|
42563
|
+
content: capture?.content ?? "full",
|
|
42564
|
+
redaction_level: capture?.redaction_level ?? "none",
|
|
42565
|
+
...capture?.redacted_fields ? { redacted_fields: capture.redacted_fields } : {}
|
|
42566
|
+
};
|
|
42567
|
+
}
|
|
42568
|
+
function toTranscriptToolCall(toolCall, projection) {
|
|
42569
|
+
const status = projection?.status ?? (toolCall.status ? toTranscriptToolCallStatus(toolCall.status) : void 0);
|
|
42570
|
+
return {
|
|
42571
|
+
tool: projection?.tool ?? toolCall.tool,
|
|
42572
|
+
...toolCall.input !== void 0 || projection?.input !== void 0 ? { input: toolCall.input !== void 0 ? toolCall.input : projection?.input } : {},
|
|
42573
|
+
...toolCall.output !== void 0 || projection?.output !== void 0 ? { output: toolCall.output !== void 0 ? toolCall.output : projection?.output } : {},
|
|
42574
|
+
...toolCall.id || projection?.id ? { id: toolCall.id ?? projection?.id } : {},
|
|
42575
|
+
...status ? { status } : {},
|
|
42576
|
+
...toolCall.startTime || projection?.start_time ? { start_time: toolCall.startTime ?? projection?.start_time } : {},
|
|
42577
|
+
...toolCall.endTime || projection?.end_time ? { end_time: toolCall.endTime ?? projection?.end_time } : {},
|
|
42578
|
+
...toolCall.durationMs !== void 0 || projection?.duration_ms !== void 0 ? { duration_ms: toolCall.durationMs ?? projection?.duration_ms } : {},
|
|
42579
|
+
...projection?.trace ? { trace: projection.trace } : {}
|
|
42580
|
+
};
|
|
42581
|
+
}
|
|
42582
|
+
function toTranscriptToolCallStatus(status) {
|
|
42583
|
+
if (status === "timeout" || status === "cancelled") {
|
|
42584
|
+
return "error";
|
|
42585
|
+
}
|
|
42586
|
+
return status === "unknown" ? "unknown" : status;
|
|
42175
42587
|
}
|
|
42176
42588
|
function toTranscriptMessageFields(message) {
|
|
42177
|
-
return
|
|
42589
|
+
return dropUndefined5({
|
|
42178
42590
|
role: message.role,
|
|
42179
42591
|
name: message.name,
|
|
42180
42592
|
content: message.content,
|
|
42181
|
-
tool_calls: message.toolCalls?.map(toTranscriptToolCall),
|
|
42593
|
+
tool_calls: message.toolCalls?.map((toolCall) => toTranscriptToolCall(toolCall)),
|
|
42182
42594
|
start_time: message.startTime,
|
|
42183
42595
|
end_time: message.endTime,
|
|
42184
42596
|
duration_ms: message.durationMs,
|
|
@@ -42188,13 +42600,17 @@ function toTranscriptMessageFields(message) {
|
|
|
42188
42600
|
}
|
|
42189
42601
|
function toTranscriptJsonLines(entry, options) {
|
|
42190
42602
|
const source = {
|
|
42603
|
+
kind: entry.source.kind ?? "imported_transcript",
|
|
42191
42604
|
provider: entry.source.provider,
|
|
42192
42605
|
session_id: entry.source.sessionId,
|
|
42606
|
+
path: entry.source.path,
|
|
42607
|
+
format: entry.source.format,
|
|
42193
42608
|
model: entry.source.model,
|
|
42194
42609
|
timestamp: entry.source.startedAt,
|
|
42195
42610
|
git_branch: entry.source.gitBranch,
|
|
42196
42611
|
cwd: entry.source.cwd ?? entry.source.projectPath,
|
|
42197
|
-
version: entry.source.version
|
|
42612
|
+
version: entry.source.version,
|
|
42613
|
+
metadata: entry.source.metadata
|
|
42198
42614
|
};
|
|
42199
42615
|
const transcriptTokenUsage = entry.tokenUsage ? {
|
|
42200
42616
|
input: entry.tokenUsage.input,
|
|
@@ -42204,7 +42620,9 @@ function toTranscriptJsonLines(entry, options) {
|
|
|
42204
42620
|
} : void 0;
|
|
42205
42621
|
const testId = options?.testId ?? entry.source.sessionId;
|
|
42206
42622
|
const target = options?.target ?? entry.source.provider;
|
|
42623
|
+
const capture = toTranscriptCaptureState(void 0);
|
|
42207
42624
|
return entry.messages.map((message, index) => ({
|
|
42625
|
+
schema_version: TRANSCRIPT_ROW_SCHEMA_VERSION,
|
|
42208
42626
|
test_id: testId,
|
|
42209
42627
|
target,
|
|
42210
42628
|
message_index: index,
|
|
@@ -42212,9 +42630,123 @@ function toTranscriptJsonLines(entry, options) {
|
|
|
42212
42630
|
transcript_token_usage: transcriptTokenUsage,
|
|
42213
42631
|
transcript_duration_ms: entry.durationMs,
|
|
42214
42632
|
transcript_cost_usd: entry.costUsd,
|
|
42215
|
-
|
|
42633
|
+
capture,
|
|
42634
|
+
source: dropUndefined5(source)
|
|
42216
42635
|
}));
|
|
42217
42636
|
}
|
|
42637
|
+
function traceRefFromEnvelope(envelope, spanId, parentSpanId) {
|
|
42638
|
+
return dropUndefined5({
|
|
42639
|
+
schema_version: EXECUTION_TRACE_SCHEMA_VERSION,
|
|
42640
|
+
artifact_id: envelope.artifactId,
|
|
42641
|
+
trace_id: envelope.trace.traceId,
|
|
42642
|
+
span_id: spanId ?? envelope.trace.rootSpanId,
|
|
42643
|
+
parent_span_id: parentSpanId
|
|
42644
|
+
});
|
|
42645
|
+
}
|
|
42646
|
+
function sourceFromEnvelope(envelope, summary) {
|
|
42647
|
+
const metadata = envelope.source.metadata;
|
|
42648
|
+
const sessionId = optionalString(metadata?.provider_session_id) ?? optionalString(metadata?.session_id) ?? optionalString(metadata?.conversation_id) ?? envelope.eval.runId ?? envelope.eval.testId;
|
|
42649
|
+
return dropUndefined5({
|
|
42650
|
+
kind: envelope.source.kind,
|
|
42651
|
+
provider: envelope.source.provider ?? envelope.eval.target,
|
|
42652
|
+
session_id: sessionId,
|
|
42653
|
+
path: envelope.source.path,
|
|
42654
|
+
format: envelope.source.format,
|
|
42655
|
+
model: optionalString(metadata?.model),
|
|
42656
|
+
timestamp: summary.startTime ?? envelope.createdAt,
|
|
42657
|
+
cwd: optionalString(metadata?.cwd),
|
|
42658
|
+
version: envelope.source.version,
|
|
42659
|
+
metadata
|
|
42660
|
+
});
|
|
42661
|
+
}
|
|
42662
|
+
function captureFromEnvelope(envelope) {
|
|
42663
|
+
return toTranscriptCaptureState({
|
|
42664
|
+
content: envelope.capture.content,
|
|
42665
|
+
redaction_level: envelope.capture.redactionLevel,
|
|
42666
|
+
redacted_fields: envelope.capture.redactedFields
|
|
42667
|
+
});
|
|
42668
|
+
}
|
|
42669
|
+
function messageTraceRefs(envelope) {
|
|
42670
|
+
const refs = /* @__PURE__ */ new Map();
|
|
42671
|
+
for (const span of envelope.trace.spans) {
|
|
42672
|
+
if (span.attributes["gen_ai.operation.name"] !== "chat" && span.attributes["openinference.span.kind"] !== "LLM") {
|
|
42673
|
+
continue;
|
|
42674
|
+
}
|
|
42675
|
+
const index = span.attributes["agentv.message.index"];
|
|
42676
|
+
if (typeof index === "number" && Number.isInteger(index)) {
|
|
42677
|
+
refs.set(index, traceRefFromEnvelope(envelope, span.spanId, span.parentSpanId ?? void 0));
|
|
42678
|
+
}
|
|
42679
|
+
}
|
|
42680
|
+
return refs;
|
|
42681
|
+
}
|
|
42682
|
+
function toolProjections(envelope) {
|
|
42683
|
+
return traceEnvelopeToToolTrajectoryView(envelope).tools.map(
|
|
42684
|
+
(tool) => dropUndefined5({
|
|
42685
|
+
tool: tool.tool,
|
|
42686
|
+
input: tool.input,
|
|
42687
|
+
output: tool.output,
|
|
42688
|
+
id: tool.toolCallId,
|
|
42689
|
+
status: tool.status,
|
|
42690
|
+
start_time: tool.startTime,
|
|
42691
|
+
end_time: tool.endTime,
|
|
42692
|
+
duration_ms: tool.durationMs,
|
|
42693
|
+
trace: traceRefFromEnvelope(envelope, tool.spanId, tool.parentSpanId)
|
|
42694
|
+
})
|
|
42695
|
+
);
|
|
42696
|
+
}
|
|
42697
|
+
function projectedToolCalls(message, projections, usedProjectionIndexes) {
|
|
42698
|
+
if (!message.toolCalls || message.toolCalls.length === 0) {
|
|
42699
|
+
return void 0;
|
|
42700
|
+
}
|
|
42701
|
+
return message.toolCalls.map((toolCall) => {
|
|
42702
|
+
const projectionIndex = projections.findIndex((candidate, index) => {
|
|
42703
|
+
if (usedProjectionIndexes.has(index)) {
|
|
42704
|
+
return false;
|
|
42705
|
+
}
|
|
42706
|
+
if (toolCall.id && candidate.id) {
|
|
42707
|
+
return candidate.id === toolCall.id;
|
|
42708
|
+
}
|
|
42709
|
+
return candidate.tool === toolCall.tool;
|
|
42710
|
+
});
|
|
42711
|
+
const projection = projectionIndex >= 0 ? projections[projectionIndex] : void 0;
|
|
42712
|
+
if (projectionIndex >= 0) {
|
|
42713
|
+
usedProjectionIndexes.add(projectionIndex);
|
|
42714
|
+
}
|
|
42715
|
+
return toTranscriptToolCall(toolCall, projection);
|
|
42716
|
+
});
|
|
42717
|
+
}
|
|
42718
|
+
function traceEnvelopeToTranscriptJsonLines(envelope, options) {
|
|
42719
|
+
const messages = traceEnvelopeToTranscriptMessages(envelope);
|
|
42720
|
+
const summary = traceEnvelopeToTraceSummary(envelope);
|
|
42721
|
+
const source = sourceFromEnvelope(envelope, summary);
|
|
42722
|
+
const capture = captureFromEnvelope(envelope);
|
|
42723
|
+
const transcriptTokenUsage = summary.tokenUsage ? {
|
|
42724
|
+
input: summary.tokenUsage.input,
|
|
42725
|
+
output: summary.tokenUsage.output,
|
|
42726
|
+
cached: summary.tokenUsage.cached,
|
|
42727
|
+
reasoning: summary.tokenUsage.reasoning
|
|
42728
|
+
} : void 0;
|
|
42729
|
+
const refsByMessageIndex = messageTraceRefs(envelope);
|
|
42730
|
+
const toolRows = toolProjections(envelope);
|
|
42731
|
+
const usedToolIndexes = /* @__PURE__ */ new Set();
|
|
42732
|
+
return messages.map((message, index) => {
|
|
42733
|
+
const trace = refsByMessageIndex.get(index) ?? traceRefFromEnvelope(envelope);
|
|
42734
|
+
return dropUndefined5({
|
|
42735
|
+
schema_version: TRANSCRIPT_ROW_SCHEMA_VERSION,
|
|
42736
|
+
test_id: options?.testId ?? envelope.eval.testId,
|
|
42737
|
+
target: options?.target ?? envelope.eval.target,
|
|
42738
|
+
message_index: index,
|
|
42739
|
+
...toTranscriptMessageFields({ ...message, toolCalls: void 0 }),
|
|
42740
|
+
tool_calls: projectedToolCalls(message, toolRows, usedToolIndexes),
|
|
42741
|
+
transcript_token_usage: transcriptTokenUsage,
|
|
42742
|
+
transcript_duration_ms: summary.durationMs,
|
|
42743
|
+
transcript_cost_usd: summary.costUsd,
|
|
42744
|
+
capture,
|
|
42745
|
+
trace,
|
|
42746
|
+
source
|
|
42747
|
+
});
|
|
42748
|
+
});
|
|
42749
|
+
}
|
|
42218
42750
|
function traceToTranscriptJsonLines(trace, options) {
|
|
42219
42751
|
const provider = (typeof trace.metadata?.provider === "string" ? trace.metadata.provider : void 0) ?? options?.target ?? "agentv";
|
|
42220
42752
|
const sessionId = (typeof trace.metadata?.provider_session_id === "string" ? trace.metadata.provider_session_id : void 0) ?? (typeof trace.metadata?.eval_case_id === "string" ? trace.metadata.eval_case_id : void 0) ?? options?.testId ?? "trace";
|
|
@@ -42222,9 +42754,11 @@ function traceToTranscriptJsonLines(trace, options) {
|
|
|
42222
42754
|
{
|
|
42223
42755
|
messages: [...trace.messages],
|
|
42224
42756
|
source: {
|
|
42757
|
+
kind: "agentv_run",
|
|
42225
42758
|
provider,
|
|
42226
42759
|
sessionId,
|
|
42227
|
-
startedAt: trace.startTime
|
|
42760
|
+
startedAt: trace.startTime,
|
|
42761
|
+
metadata: isRecord3(trace.metadata) ? trace.metadata : void 0
|
|
42228
42762
|
},
|
|
42229
42763
|
tokenUsage: trace.tokenUsage,
|
|
42230
42764
|
durationMs: trace.durationMs,
|
|
@@ -42270,6 +42804,9 @@ function readOptionalNumber(record2, key) {
|
|
|
42270
42804
|
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
42271
42805
|
}
|
|
42272
42806
|
function fromTranscriptToolCall(wire) {
|
|
42807
|
+
if (!isRecord3(wire)) {
|
|
42808
|
+
return void 0;
|
|
42809
|
+
}
|
|
42273
42810
|
const tool = readOptionalString(wire, "tool");
|
|
42274
42811
|
if (!tool) {
|
|
42275
42812
|
return void 0;
|
|
@@ -42279,6 +42816,7 @@ function fromTranscriptToolCall(wire) {
|
|
|
42279
42816
|
input: wire.input,
|
|
42280
42817
|
output: wire.output,
|
|
42281
42818
|
id: readOptionalString(wire, "id"),
|
|
42819
|
+
status: wire.status === "ok" || wire.status === "error" || wire.status === "timeout" || wire.status === "cancelled" || wire.status === "unknown" ? wire.status : void 0,
|
|
42282
42820
|
startTime: readOptionalString(wire, "start_time"),
|
|
42283
42821
|
endTime: readOptionalString(wire, "end_time"),
|
|
42284
42822
|
durationMs: readOptionalNumber(wire, "duration_ms")
|
|
@@ -42302,13 +42840,17 @@ function groupTranscriptJsonLines(lines) {
|
|
|
42302
42840
|
for (const line of lines) {
|
|
42303
42841
|
const existing = grouped.get(line.test_id);
|
|
42304
42842
|
const source = {
|
|
42843
|
+
kind: line.source.kind,
|
|
42305
42844
|
provider: line.source.provider,
|
|
42306
42845
|
sessionId: line.source.session_id,
|
|
42846
|
+
path: line.source.path,
|
|
42847
|
+
format: line.source.format,
|
|
42307
42848
|
startedAt: line.source.timestamp,
|
|
42308
42849
|
model: line.source.model,
|
|
42309
42850
|
gitBranch: line.source.git_branch,
|
|
42310
42851
|
cwd: line.source.cwd,
|
|
42311
|
-
version: line.source.version
|
|
42852
|
+
version: line.source.version,
|
|
42853
|
+
metadata: line.source.metadata
|
|
42312
42854
|
};
|
|
42313
42855
|
const transcriptTokenUsage = line.transcript_token_usage ? {
|
|
42314
42856
|
input: line.transcript_token_usage.input,
|
|
@@ -42401,7 +42943,7 @@ var TOOL_CALL_ALIASES = {
|
|
|
42401
42943
|
endTime: "end_time",
|
|
42402
42944
|
startTime: "start_time"
|
|
42403
42945
|
};
|
|
42404
|
-
function
|
|
42946
|
+
function isRecord4(value) {
|
|
42405
42947
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
42406
42948
|
}
|
|
42407
42949
|
function normalizeKnownAliases(value, aliases) {
|
|
@@ -42417,13 +42959,13 @@ function normalizeKnownAliases(value, aliases) {
|
|
|
42417
42959
|
return normalized;
|
|
42418
42960
|
}
|
|
42419
42961
|
function normalizeToolCall2(value) {
|
|
42420
|
-
if (!
|
|
42962
|
+
if (!isRecord4(value)) {
|
|
42421
42963
|
return value;
|
|
42422
42964
|
}
|
|
42423
42965
|
return normalizeKnownAliases(value, TOOL_CALL_ALIASES);
|
|
42424
42966
|
}
|
|
42425
42967
|
function normalizeMessage(value) {
|
|
42426
|
-
if (!
|
|
42968
|
+
if (!isRecord4(value)) {
|
|
42427
42969
|
return value;
|
|
42428
42970
|
}
|
|
42429
42971
|
const normalized = normalizeKnownAliases(value, MESSAGE_ALIASES);
|
|
@@ -42433,7 +42975,7 @@ function normalizeMessage(value) {
|
|
|
42433
42975
|
return normalized;
|
|
42434
42976
|
}
|
|
42435
42977
|
function normalizeTraceSummary(value) {
|
|
42436
|
-
if (!
|
|
42978
|
+
if (!isRecord4(value)) {
|
|
42437
42979
|
return value;
|
|
42438
42980
|
}
|
|
42439
42981
|
const normalized = normalizeKnownAliases(value, TRACE_SUMMARY_ALIASES);
|
|
@@ -42466,7 +43008,7 @@ function looksLikeResultRow(value) {
|
|
|
42466
43008
|
return typeof value.test_id === "string" || Object.hasOwn(value, "score") || Object.hasOwn(value, "trace") || Object.hasOwn(value, "spans") || Object.hasOwn(value, "target") || Object.hasOwn(value, "grading_path") || Object.hasOwn(value, "timing_path");
|
|
42467
43009
|
}
|
|
42468
43010
|
function normalizeResultRow(value, context = {}) {
|
|
42469
|
-
if (!
|
|
43011
|
+
if (!isRecord4(value)) {
|
|
42470
43012
|
throw buildSchemaError(context);
|
|
42471
43013
|
}
|
|
42472
43014
|
const normalized = normalizeKnownAliases(value, RESULT_ROW_ALIASES);
|
|
@@ -42636,17 +43178,17 @@ function toIndexScore(score) {
|
|
|
42636
43178
|
function toIndexScores(scores) {
|
|
42637
43179
|
return scores?.map(toIndexScore);
|
|
42638
43180
|
}
|
|
42639
|
-
function
|
|
43181
|
+
function dropUndefined6(value) {
|
|
42640
43182
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
42641
43183
|
}
|
|
42642
|
-
function
|
|
43184
|
+
function isRecord5(value) {
|
|
42643
43185
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
42644
43186
|
}
|
|
42645
43187
|
function toIndexRerunSource(value) {
|
|
42646
|
-
if (!
|
|
43188
|
+
if (!isRecord5(value)) {
|
|
42647
43189
|
return void 0;
|
|
42648
43190
|
}
|
|
42649
|
-
return
|
|
43191
|
+
return dropUndefined6({
|
|
42650
43192
|
mode: value.mode,
|
|
42651
43193
|
source_run_dir: value.sourceRunDir,
|
|
42652
43194
|
source_index_path: value.sourceIndexPath,
|
|
@@ -42658,10 +43200,10 @@ function toIndexRerunSource(value) {
|
|
|
42658
43200
|
});
|
|
42659
43201
|
}
|
|
42660
43202
|
function toIndexPreparedAttempt(value) {
|
|
42661
|
-
if (!
|
|
43203
|
+
if (!isRecord5(value)) {
|
|
42662
43204
|
return void 0;
|
|
42663
43205
|
}
|
|
42664
|
-
return
|
|
43206
|
+
return dropUndefined6({
|
|
42665
43207
|
source: value.source,
|
|
42666
43208
|
manifest_path: value.manifestPath,
|
|
42667
43209
|
prepared_dir: value.preparedDir,
|
|
@@ -42691,6 +43233,17 @@ function toIndexMetadata(metadata) {
|
|
|
42691
43233
|
...preparedAttempt ? { prepared_attempt: preparedAttempt } : {}
|
|
42692
43234
|
};
|
|
42693
43235
|
}
|
|
43236
|
+
function buildExportMetadata(duplicatePolicy, projectionIdentity, options) {
|
|
43237
|
+
if (!duplicatePolicy) {
|
|
43238
|
+
return void 0;
|
|
43239
|
+
}
|
|
43240
|
+
const warnings = projectionIdentity?.issues?.filter((issue2) => issue2.severity === "warning").map(toProjectionIdentityIssueWire);
|
|
43241
|
+
return {
|
|
43242
|
+
duplicate_policy: duplicatePolicy,
|
|
43243
|
+
identity_warnings: warnings && warnings.length > 0 ? warnings : void 0,
|
|
43244
|
+
skipped: options?.skipped
|
|
43245
|
+
};
|
|
43246
|
+
}
|
|
42694
43247
|
function buildGradingArtifact(result) {
|
|
42695
43248
|
const assertions = buildAssertions(result);
|
|
42696
43249
|
const passed = assertions.filter((e) => e.passed).length;
|
|
@@ -42912,23 +43465,27 @@ function resolveEnvelopeEvalPath(result, testByTestId, fallbackEvalFile) {
|
|
|
42912
43465
|
function resultHasExecutionTraceTranscript(result) {
|
|
42913
43466
|
return result.output.length > 0 || result.trace.messages.length > 0;
|
|
42914
43467
|
}
|
|
42915
|
-
|
|
43468
|
+
function buildTraceEnvelopeSidecar(params) {
|
|
42916
43469
|
const hasTranscript = resultHasExecutionTraceTranscript(params.result);
|
|
42917
|
-
|
|
43470
|
+
return buildTraceEnvelopeFromEvaluationResult(params.result, {
|
|
42918
43471
|
evalPath: params.evalPath,
|
|
42919
|
-
runId: path48.basename(params.outputDir),
|
|
43472
|
+
runId: params.runId ?? path48.basename(params.outputDir),
|
|
42920
43473
|
experiment: params.experiment,
|
|
42921
43474
|
source: { path: RESULT_INDEX_FILENAME },
|
|
42922
43475
|
capture: { content: "full", redactionLevel: "none", redactedFields: [] },
|
|
42923
43476
|
artifacts: {
|
|
42924
|
-
|
|
43477
|
+
trace_path: "outputs/trace.json",
|
|
42925
43478
|
answer_path: params.result.output.length > 0 ? "outputs/answer.md" : void 0,
|
|
42926
43479
|
response_path: params.result.output.length > 0 ? "outputs/response.md" : void 0,
|
|
42927
43480
|
transcript_path: hasTranscript ? "outputs/transcript.jsonl" : void 0
|
|
42928
|
-
}
|
|
43481
|
+
},
|
|
43482
|
+
duplicatePolicy: params.duplicatePolicy
|
|
42929
43483
|
});
|
|
43484
|
+
}
|
|
43485
|
+
async function writeTraceEnvelopeSidecar(params) {
|
|
43486
|
+
const envelope = buildTraceEnvelopeSidecar(params);
|
|
42930
43487
|
await writeFile10(
|
|
42931
|
-
path48.join(params.outputsDir, "
|
|
43488
|
+
path48.join(params.outputsDir, "trace.json"),
|
|
42932
43489
|
`${JSON.stringify(toTraceEnvelopeWire(envelope), null, 2)}
|
|
42933
43490
|
`,
|
|
42934
43491
|
"utf8"
|
|
@@ -42964,10 +43521,12 @@ function buildIndexArtifactEntry(result, options) {
|
|
|
42964
43521
|
input_path: options.inputPath ? toRelativeArtifactPath(options.outputDir, options.inputPath) : void 0,
|
|
42965
43522
|
response_path: options.responsePath ? toRelativeArtifactPath(options.outputDir, options.responsePath) : void 0,
|
|
42966
43523
|
...options.extraIndexFields,
|
|
43524
|
+
projection_identity: options.projectionIdentity ? toProjectionIdentityWire(options.projectionIdentity) : void 0,
|
|
43525
|
+
export_metadata: buildExportMetadata(options.duplicatePolicy, options.projectionIdentity),
|
|
42967
43526
|
metadata: toIndexMetadata(result.metadata)
|
|
42968
43527
|
};
|
|
42969
43528
|
}
|
|
42970
|
-
function buildResultIndexArtifact(result, extraIndexFields) {
|
|
43529
|
+
function buildResultIndexArtifact(result, extraIndexFields, options) {
|
|
42971
43530
|
const artifactSubdir = buildArtifactSubdir(result);
|
|
42972
43531
|
const input = extractInput(result);
|
|
42973
43532
|
const hasAnswer = result.output.length > 0;
|
|
@@ -43000,6 +43559,8 @@ function buildResultIndexArtifact(result, extraIndexFields) {
|
|
|
43000
43559
|
transcript_path: hasTranscript ? path48.posix.join(artifactSubdir, "outputs", "transcript.jsonl") : void 0,
|
|
43001
43560
|
response_path: hasAnswer ? path48.posix.join(artifactSubdir, "outputs", "response.md") : void 0,
|
|
43002
43561
|
...extraIndexFields,
|
|
43562
|
+
projection_identity: options?.projectionIdentity ? toProjectionIdentityWire(options.projectionIdentity) : void 0,
|
|
43563
|
+
export_metadata: buildExportMetadata(options?.duplicatePolicy, options?.projectionIdentity),
|
|
43003
43564
|
metadata: toIndexMetadata(result.metadata)
|
|
43004
43565
|
};
|
|
43005
43566
|
}
|
|
@@ -43008,17 +43569,11 @@ async function writeJsonlFile(filePath, records) {
|
|
|
43008
43569
|
`;
|
|
43009
43570
|
await writeFile10(filePath, content, "utf8");
|
|
43010
43571
|
}
|
|
43011
|
-
function traceProjectionForTranscript(result, envelope) {
|
|
43012
|
-
return {
|
|
43013
|
-
...result.trace,
|
|
43014
|
-
messages: traceEnvelopeToTranscriptMessages(envelope)
|
|
43015
|
-
};
|
|
43016
|
-
}
|
|
43017
43572
|
function hasTranscriptProjection(result, envelope) {
|
|
43018
43573
|
return result.output.length > 0 || traceEnvelopeToTranscriptMessages(envelope).length > 0;
|
|
43019
43574
|
}
|
|
43020
43575
|
async function writeTranscriptJsonl(filePath, result, envelope) {
|
|
43021
|
-
const lines =
|
|
43576
|
+
const lines = traceEnvelopeToTranscriptJsonLines(envelope, {
|
|
43022
43577
|
testId: result.testId,
|
|
43023
43578
|
target: result.target
|
|
43024
43579
|
});
|
|
@@ -43027,13 +43582,70 @@ async function writeTranscriptJsonl(filePath, result, envelope) {
|
|
|
43027
43582
|
await writeFile10(filePath, content, "utf8");
|
|
43028
43583
|
}
|
|
43029
43584
|
function indexRecordKey(record2) {
|
|
43030
|
-
if (!
|
|
43585
|
+
if (!isRecord5(record2)) {
|
|
43031
43586
|
return void 0;
|
|
43032
43587
|
}
|
|
43033
43588
|
const testId = typeof record2.test_id === "string" ? record2.test_id : typeof record2.testId === "string" ? record2.testId : void 0;
|
|
43034
43589
|
const target = typeof record2.target === "string" ? record2.target : void 0;
|
|
43035
43590
|
return testId ? buildTestTargetKey(testId, target) : void 0;
|
|
43036
43591
|
}
|
|
43592
|
+
function projectionIdentityRecordKey(record2) {
|
|
43593
|
+
if (!isRecord5(record2) || !isRecord5(record2.projection_identity)) {
|
|
43594
|
+
return void 0;
|
|
43595
|
+
}
|
|
43596
|
+
const id = record2.projection_identity.id;
|
|
43597
|
+
return typeof id === "string" && id.trim().length > 0 ? id : void 0;
|
|
43598
|
+
}
|
|
43599
|
+
async function readExistingIndexRecords(outputDir) {
|
|
43600
|
+
const indexPath = path48.join(outputDir, RESULT_INDEX_FILENAME);
|
|
43601
|
+
const content = await readFile20(indexPath, "utf8").catch(() => void 0);
|
|
43602
|
+
if (content === void 0) {
|
|
43603
|
+
return [];
|
|
43604
|
+
}
|
|
43605
|
+
const records = [];
|
|
43606
|
+
for (const line of content.split("\n")) {
|
|
43607
|
+
if (line.trim().length === 0) {
|
|
43608
|
+
continue;
|
|
43609
|
+
}
|
|
43610
|
+
try {
|
|
43611
|
+
records.push(JSON.parse(line));
|
|
43612
|
+
} catch {
|
|
43613
|
+
}
|
|
43614
|
+
}
|
|
43615
|
+
return records;
|
|
43616
|
+
}
|
|
43617
|
+
function existingRecordsByProjectionIdentity(records) {
|
|
43618
|
+
const byIdentity = /* @__PURE__ */ new Map();
|
|
43619
|
+
for (const record2 of records) {
|
|
43620
|
+
const key = projectionIdentityRecordKey(record2);
|
|
43621
|
+
if (key) {
|
|
43622
|
+
byIdentity.set(key, record2);
|
|
43623
|
+
}
|
|
43624
|
+
}
|
|
43625
|
+
return byIdentity;
|
|
43626
|
+
}
|
|
43627
|
+
function skippedExistingRecord(record2, identity, duplicatePolicy) {
|
|
43628
|
+
if (!isRecord5(record2)) {
|
|
43629
|
+
return record2;
|
|
43630
|
+
}
|
|
43631
|
+
return {
|
|
43632
|
+
...record2,
|
|
43633
|
+
projection_identity: toProjectionIdentityWire(identity),
|
|
43634
|
+
export_metadata: buildExportMetadata(duplicatePolicy, identity, { skipped: true })
|
|
43635
|
+
};
|
|
43636
|
+
}
|
|
43637
|
+
function duplicateProjectionMessage(identity) {
|
|
43638
|
+
const dimensions = identity.dimensions;
|
|
43639
|
+
return [
|
|
43640
|
+
`Duplicate export projection ${identity.id}`,
|
|
43641
|
+
`test_id=${dimensions.testId}`,
|
|
43642
|
+
`target=${dimensions.target}`,
|
|
43643
|
+
`source_target=${dimensions.sourceTarget}`,
|
|
43644
|
+
`attempt=${dimensions.attempt}`,
|
|
43645
|
+
`variant=${dimensions.variant ?? "<none>"}`,
|
|
43646
|
+
`run_id=${dimensions.runId}`
|
|
43647
|
+
].join(" ");
|
|
43648
|
+
}
|
|
43037
43649
|
async function rewriteExistingIndexRecords(outputDir, replacements) {
|
|
43038
43650
|
if (replacements.length === 0) {
|
|
43039
43651
|
return;
|
|
@@ -43175,18 +43787,6 @@ async function writeArtifacts(jsonlPath, outputDir, options) {
|
|
|
43175
43787
|
const results = parseJsonlResults(content);
|
|
43176
43788
|
return writeArtifactsFromResults(results, outputDir, options);
|
|
43177
43789
|
}
|
|
43178
|
-
function buildTranscriptMessageLines(results) {
|
|
43179
|
-
const lines = [];
|
|
43180
|
-
for (const result of results) {
|
|
43181
|
-
const transcriptLines = traceToTranscriptJsonLines(result.trace, {
|
|
43182
|
-
testId: result.testId,
|
|
43183
|
-
target: result.target
|
|
43184
|
-
});
|
|
43185
|
-
lines.push(...transcriptLines.map((line) => JSON.stringify(line)));
|
|
43186
|
-
}
|
|
43187
|
-
return lines.length > 0 ? `${lines.join("\n")}
|
|
43188
|
-
` : "";
|
|
43189
|
-
}
|
|
43190
43790
|
async function collectAdditionalIndexFields(result, outputDir, testDir, testByTestId, additionalArtifacts) {
|
|
43191
43791
|
if (!additionalArtifacts) {
|
|
43192
43792
|
return void 0;
|
|
@@ -43201,6 +43801,7 @@ async function collectAdditionalIndexFields(result, outputDir, testDir, testByTe
|
|
|
43201
43801
|
}
|
|
43202
43802
|
async function writePerTestArtifacts(results, outputDir, options) {
|
|
43203
43803
|
await mkdir18(outputDir, { recursive: true });
|
|
43804
|
+
const duplicatePolicy = options?.duplicatePolicy ?? "update";
|
|
43204
43805
|
const testByTestId = new Map((options?.sourceTests ?? []).map((test) => [test.id, test]));
|
|
43205
43806
|
const indexRecords = [];
|
|
43206
43807
|
for (const result of results) {
|
|
@@ -43236,7 +43837,9 @@ async function writePerTestArtifacts(results, outputDir, options) {
|
|
|
43236
43837
|
outputDir,
|
|
43237
43838
|
outputsDir,
|
|
43238
43839
|
evalPath: resolveEnvelopeEvalPath(result, testByTestId, options?.evalFile),
|
|
43239
|
-
experiment: options?.experiment
|
|
43840
|
+
experiment: options?.experiment,
|
|
43841
|
+
runId: options?.runId,
|
|
43842
|
+
duplicatePolicy
|
|
43240
43843
|
});
|
|
43241
43844
|
if (hasTranscriptProjection(result, envelope)) {
|
|
43242
43845
|
await writeTranscriptJsonl(path48.join(outputsDir, "transcript.jsonl"), result, envelope);
|
|
@@ -43249,7 +43852,10 @@ async function writePerTestArtifacts(results, outputDir, options) {
|
|
|
43249
43852
|
options?.additionalArtifacts
|
|
43250
43853
|
);
|
|
43251
43854
|
indexRecords.push({
|
|
43252
|
-
...buildResultIndexArtifact(result, extraIndexFields
|
|
43855
|
+
...buildResultIndexArtifact(result, extraIndexFields, {
|
|
43856
|
+
projectionIdentity: envelope.projectionIdentity,
|
|
43857
|
+
duplicatePolicy
|
|
43858
|
+
}),
|
|
43253
43859
|
experiment: options?.experiment
|
|
43254
43860
|
});
|
|
43255
43861
|
}
|
|
@@ -43261,66 +43867,137 @@ async function writeArtifactsFromResults(results, outputDir, options) {
|
|
|
43261
43867
|
const benchmarkPath = path48.join(outputDir, "benchmark.json");
|
|
43262
43868
|
const indexPath = path48.join(outputDir, RESULT_INDEX_FILENAME);
|
|
43263
43869
|
await mkdir18(outputDir, { recursive: true });
|
|
43870
|
+
const duplicatePolicy = options?.duplicatePolicy ?? "update";
|
|
43871
|
+
const existingRecords = await readExistingIndexRecords(outputDir);
|
|
43872
|
+
const existingByIdentity = existingRecordsByProjectionIdentity(existingRecords);
|
|
43264
43873
|
const indexRecords = [];
|
|
43265
43874
|
const testByTestId = new Map((options?.sourceTests ?? []).map((test) => [test.id, test]));
|
|
43266
|
-
|
|
43875
|
+
const emittedIdentityIds = /* @__PURE__ */ new Set();
|
|
43876
|
+
const plans = results.map((result) => {
|
|
43267
43877
|
const grading = buildGradingArtifact(result);
|
|
43268
43878
|
const timing2 = buildTimingArtifact([result]);
|
|
43269
43879
|
const artifactSubdir = buildArtifactSubdir(result);
|
|
43270
43880
|
const testDir = path48.join(outputDir, artifactSubdir);
|
|
43271
43881
|
const gradingPath = path48.join(testDir, "grading.json");
|
|
43272
43882
|
const perTestTimingPath = path48.join(testDir, "timing.json");
|
|
43273
|
-
await mkdir18(testDir, { recursive: true });
|
|
43274
|
-
await writeFile10(gradingPath, `${JSON.stringify(grading, null, 2)}
|
|
43275
|
-
`, "utf8");
|
|
43276
|
-
await writeFile10(perTestTimingPath, `${JSON.stringify(timing2, null, 2)}
|
|
43277
|
-
`, "utf8");
|
|
43278
43883
|
const input = extractInput(result);
|
|
43279
43884
|
const inputPath = input ? path48.join(testDir, "input.md") : void 0;
|
|
43280
|
-
if (inputPath && input) {
|
|
43281
|
-
await writeFile10(inputPath, input, "utf8");
|
|
43282
|
-
}
|
|
43283
43885
|
const outputsDir = path48.join(testDir, "outputs");
|
|
43284
|
-
await mkdir18(outputsDir, { recursive: true });
|
|
43285
43886
|
const answerPath = result.output.length > 0 ? path48.join(outputsDir, "answer.md") : void 0;
|
|
43286
43887
|
const responsePath = result.output.length > 0 ? path48.join(outputsDir, "response.md") : void 0;
|
|
43287
|
-
|
|
43288
|
-
await writeFile10(answerPath, result.output, "utf8");
|
|
43289
|
-
await writeFile10(responsePath, result.output, "utf8");
|
|
43290
|
-
}
|
|
43291
|
-
const envelope = await writeTraceEnvelopeSidecar({
|
|
43888
|
+
const envelope = buildTraceEnvelopeSidecar({
|
|
43292
43889
|
result,
|
|
43293
43890
|
outputDir,
|
|
43294
43891
|
outputsDir,
|
|
43295
43892
|
evalPath: resolveEnvelopeEvalPath(result, testByTestId, options?.evalFile),
|
|
43296
|
-
experiment: options?.experiment
|
|
43893
|
+
experiment: options?.experiment,
|
|
43894
|
+
runId: options?.runId,
|
|
43895
|
+
duplicatePolicy
|
|
43297
43896
|
});
|
|
43298
43897
|
const transcriptPath = hasTranscriptProjection(result, envelope) ? path48.join(outputsDir, "transcript.jsonl") : void 0;
|
|
43299
|
-
|
|
43300
|
-
|
|
43898
|
+
const projectionIdentity = envelope.projectionIdentity;
|
|
43899
|
+
if (!projectionIdentity) {
|
|
43900
|
+
throw new Error(`Result ${result.testId ?? "unknown"} is missing projection identity`);
|
|
43901
|
+
}
|
|
43902
|
+
const identityId = projectionIdentity.id;
|
|
43903
|
+
return {
|
|
43904
|
+
result,
|
|
43905
|
+
grading,
|
|
43906
|
+
timing: timing2,
|
|
43907
|
+
testDir,
|
|
43908
|
+
gradingPath,
|
|
43909
|
+
perTestTimingPath,
|
|
43910
|
+
input,
|
|
43911
|
+
inputPath,
|
|
43912
|
+
outputsDir,
|
|
43913
|
+
answerPath,
|
|
43914
|
+
responsePath,
|
|
43915
|
+
envelope,
|
|
43916
|
+
projectionIdentity,
|
|
43917
|
+
transcriptPath,
|
|
43918
|
+
identityId
|
|
43919
|
+
};
|
|
43920
|
+
});
|
|
43921
|
+
if (duplicatePolicy === "error") {
|
|
43922
|
+
const seen = /* @__PURE__ */ new Set();
|
|
43923
|
+
for (const plan of plans) {
|
|
43924
|
+
const duplicate = seen.has(plan.identityId) || existingByIdentity.has(plan.identityId) ? plan.projectionIdentity : void 0;
|
|
43925
|
+
if (duplicate) {
|
|
43926
|
+
throw new Error(
|
|
43927
|
+
`${duplicateProjectionMessage(duplicate)}; use --duplicate-policy update or skip`
|
|
43928
|
+
);
|
|
43929
|
+
}
|
|
43930
|
+
seen.add(plan.identityId);
|
|
43931
|
+
}
|
|
43932
|
+
}
|
|
43933
|
+
for (const plan of plans) {
|
|
43934
|
+
const { result, envelope, identityId } = plan;
|
|
43935
|
+
const existing = existingByIdentity.get(identityId);
|
|
43936
|
+
if (duplicatePolicy === "skip" && existing) {
|
|
43937
|
+
indexRecords.push(skippedExistingRecord(existing, plan.projectionIdentity, duplicatePolicy));
|
|
43938
|
+
emittedIdentityIds.add(identityId);
|
|
43939
|
+
continue;
|
|
43940
|
+
}
|
|
43941
|
+
if (duplicatePolicy === "skip" && emittedIdentityIds.has(identityId)) {
|
|
43942
|
+
continue;
|
|
43943
|
+
}
|
|
43944
|
+
await mkdir18(plan.testDir, { recursive: true });
|
|
43945
|
+
await writeFile10(plan.gradingPath, `${JSON.stringify(plan.grading, null, 2)}
|
|
43946
|
+
`, "utf8");
|
|
43947
|
+
await writeFile10(plan.perTestTimingPath, `${JSON.stringify(plan.timing, null, 2)}
|
|
43948
|
+
`, "utf8");
|
|
43949
|
+
if (plan.inputPath && plan.input) {
|
|
43950
|
+
await writeFile10(plan.inputPath, plan.input, "utf8");
|
|
43951
|
+
}
|
|
43952
|
+
await mkdir18(plan.outputsDir, { recursive: true });
|
|
43953
|
+
if (plan.answerPath && plan.responsePath) {
|
|
43954
|
+
await writeFile10(plan.answerPath, result.output, "utf8");
|
|
43955
|
+
await writeFile10(plan.responsePath, result.output, "utf8");
|
|
43956
|
+
}
|
|
43957
|
+
await writeFile10(
|
|
43958
|
+
path48.join(plan.outputsDir, "execution-trace.json"),
|
|
43959
|
+
`${JSON.stringify(toTraceEnvelopeWire(envelope), null, 2)}
|
|
43960
|
+
`,
|
|
43961
|
+
"utf8"
|
|
43962
|
+
);
|
|
43963
|
+
if (plan.transcriptPath) {
|
|
43964
|
+
await writeTranscriptJsonl(plan.transcriptPath, result, envelope);
|
|
43301
43965
|
}
|
|
43302
43966
|
const extraIndexFields = await collectAdditionalIndexFields(
|
|
43303
43967
|
result,
|
|
43304
43968
|
outputDir,
|
|
43305
|
-
testDir,
|
|
43969
|
+
plan.testDir,
|
|
43306
43970
|
testByTestId,
|
|
43307
43971
|
options?.additionalArtifacts
|
|
43308
43972
|
);
|
|
43309
|
-
|
|
43973
|
+
const nextRecord = {
|
|
43310
43974
|
...buildIndexArtifactEntry(result, {
|
|
43311
43975
|
outputDir,
|
|
43312
|
-
artifactDir: testDir,
|
|
43313
|
-
gradingPath,
|
|
43314
|
-
timingPath: perTestTimingPath,
|
|
43315
|
-
outputPath: answerPath,
|
|
43316
|
-
answerPath,
|
|
43317
|
-
transcriptPath,
|
|
43318
|
-
inputPath,
|
|
43319
|
-
responsePath,
|
|
43320
|
-
extraIndexFields
|
|
43976
|
+
artifactDir: plan.testDir,
|
|
43977
|
+
gradingPath: plan.gradingPath,
|
|
43978
|
+
timingPath: plan.perTestTimingPath,
|
|
43979
|
+
outputPath: plan.answerPath,
|
|
43980
|
+
answerPath: plan.answerPath,
|
|
43981
|
+
transcriptPath: plan.transcriptPath,
|
|
43982
|
+
inputPath: plan.inputPath,
|
|
43983
|
+
responsePath: plan.responsePath,
|
|
43984
|
+
extraIndexFields,
|
|
43985
|
+
projectionIdentity: plan.projectionIdentity,
|
|
43986
|
+
duplicatePolicy
|
|
43321
43987
|
}),
|
|
43322
43988
|
experiment: options?.experiment
|
|
43323
|
-
}
|
|
43989
|
+
};
|
|
43990
|
+
if (duplicatePolicy === "update" && emittedIdentityIds.has(identityId)) {
|
|
43991
|
+
const existingIndex = indexRecords.findIndex(
|
|
43992
|
+
(record2) => projectionIdentityRecordKey(record2) === identityId
|
|
43993
|
+
);
|
|
43994
|
+
if (existingIndex >= 0) {
|
|
43995
|
+
indexRecords[existingIndex] = nextRecord;
|
|
43996
|
+
}
|
|
43997
|
+
} else {
|
|
43998
|
+
indexRecords.push(nextRecord);
|
|
43999
|
+
}
|
|
44000
|
+
emittedIdentityIds.add(identityId);
|
|
43324
44001
|
}
|
|
43325
44002
|
const timing = buildTimingArtifact(results);
|
|
43326
44003
|
await writeFile10(timingPath, `${JSON.stringify(timing, null, 2)}
|
|
@@ -43335,11 +44012,6 @@ async function writeArtifactsFromResults(results, outputDir, options) {
|
|
|
43335
44012
|
await writeFile10(benchmarkPath, `${JSON.stringify(benchmark, null, 2)}
|
|
43336
44013
|
`, "utf8");
|
|
43337
44014
|
await writeJsonlFile(indexPath, indexRecords);
|
|
43338
|
-
await writeFile10(
|
|
43339
|
-
path48.join(outputDir, "transcript.jsonl"),
|
|
43340
|
-
buildTranscriptMessageLines(results),
|
|
43341
|
-
"utf8"
|
|
43342
|
-
);
|
|
43343
44015
|
return { testArtifactDir, timingPath, benchmarkPath, indexPath };
|
|
43344
44016
|
}
|
|
43345
44017
|
async function evaluate(config2) {
|
|
@@ -43869,6 +44541,18 @@ export {
|
|
|
43869
44541
|
formatReplayLookupKey,
|
|
43870
44542
|
replayFixtureRecordToProviderResponse,
|
|
43871
44543
|
buildReplayFixtureRecord,
|
|
44544
|
+
PROJECTION_IDENTITY_SCHEMA_VERSION,
|
|
44545
|
+
EXPORT_DUPLICATE_POLICY_VALUES,
|
|
44546
|
+
ProjectionIdentityError,
|
|
44547
|
+
buildProjectionIdentityKey,
|
|
44548
|
+
buildProjectionIdentity,
|
|
44549
|
+
ProjectionIdentityIssueWireSchema,
|
|
44550
|
+
ProjectionIdentityDimensionsWireSchema,
|
|
44551
|
+
ProjectionIdentityWireSchema,
|
|
44552
|
+
toProjectionIdentityIssueWire,
|
|
44553
|
+
fromProjectionIdentityIssueWire,
|
|
44554
|
+
toProjectionIdentityWire,
|
|
44555
|
+
fromProjectionIdentityWire,
|
|
43872
44556
|
EXECUTION_TRACE_SCHEMA_VERSION,
|
|
43873
44557
|
TraceEnvelopeEvalWireSchema,
|
|
43874
44558
|
TraceEnvelopeReplayWireSchema,
|
|
@@ -43881,6 +44565,7 @@ export {
|
|
|
43881
44565
|
TraceEnvelopeSourceRefWireSchema,
|
|
43882
44566
|
TraceEnvelopeConversionWarningWireSchema,
|
|
43883
44567
|
TraceEnvelopeScoreWireSchema,
|
|
44568
|
+
TraceEnvelopeExportMetadataWireSchema,
|
|
43884
44569
|
TraceEnvelopeWireSchema,
|
|
43885
44570
|
buildTraceEnvelopeFromEvaluationResult,
|
|
43886
44571
|
toTraceEnvelopeWire,
|
|
@@ -43963,7 +44648,9 @@ export {
|
|
|
43963
44648
|
gradePreparedEvalCase,
|
|
43964
44649
|
runEvaluation,
|
|
43965
44650
|
runEvalCase,
|
|
44651
|
+
TRANSCRIPT_ROW_SCHEMA_VERSION,
|
|
43966
44652
|
toTranscriptJsonLines,
|
|
44653
|
+
traceEnvelopeToTranscriptJsonLines,
|
|
43967
44654
|
traceToTranscriptJsonLines,
|
|
43968
44655
|
traceFromTranscriptJsonLines,
|
|
43969
44656
|
groupTranscriptJsonLines,
|
|
@@ -43990,4 +44677,4 @@ export {
|
|
|
43990
44677
|
loadTsEvalFile,
|
|
43991
44678
|
loadTsEvalSuite
|
|
43992
44679
|
};
|
|
43993
|
-
//# sourceMappingURL=chunk-
|
|
44680
|
+
//# sourceMappingURL=chunk-SJTHFOPH.js.map
|