agentv 2.8.0-next.1 → 2.9.0-next.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -2
- package/dist/{chunk-RCFB5QFS.js → chunk-3INJ7ISP.js} +46 -85
- package/dist/chunk-3INJ7ISP.js.map +1 -0
- package/dist/{chunk-2SXGPQVR.js → chunk-PC3FAOHT.js} +4 -4
- package/dist/chunk-PC3FAOHT.js.map +1 -0
- package/dist/{chunk-DJCMBVB3.js → chunk-RJWTL3VS.js} +166 -75
- package/dist/chunk-RJWTL3VS.js.map +1 -0
- package/dist/cli.js +3 -3
- package/dist/{dist-T7REAXNS.js → dist-BGRU67HI.js} +2 -2
- package/dist/index.js +3 -3
- package/dist/{interactive-TE5SJPJW.js → interactive-7KFUCBIP.js} +3 -3
- package/dist/templates/.agents/skills/agentv-eval-builder/SKILL.md +1 -1
- package/dist/templates/.agents/skills/agentv-eval-builder/references/custom-evaluators.md +8 -8
- package/dist/templates/.agents/skills/agentv-eval-builder/references/eval-schema.json +12683 -237
- package/dist/templates/.agentv/config.yaml +1 -1
- package/dist/templates/.agentv/targets.yaml +10 -13
- package/package.json +2 -2
- package/dist/chunk-2SXGPQVR.js.map +0 -1
- package/dist/chunk-DJCMBVB3.js.map +0 -1
- package/dist/chunk-RCFB5QFS.js.map +0 -1
- package/dist/templates/.claude/skills/agentv-eval-builder/SKILL.md +0 -202
- package/dist/templates/.claude/skills/agentv-eval-builder/references/batch-cli-evaluator.md +0 -316
- package/dist/templates/.claude/skills/agentv-eval-builder/references/compare-command.md +0 -137
- package/dist/templates/.claude/skills/agentv-eval-builder/references/composite-evaluator.md +0 -215
- package/dist/templates/.claude/skills/agentv-eval-builder/references/config-schema.json +0 -27
- package/dist/templates/.claude/skills/agentv-eval-builder/references/custom-evaluators.md +0 -118
- package/dist/templates/.claude/skills/agentv-eval-builder/references/eval-schema.json +0 -278
- package/dist/templates/.claude/skills/agentv-eval-builder/references/example-evals.md +0 -333
- package/dist/templates/.claude/skills/agentv-eval-builder/references/rubric-evaluator.md +0 -77
- package/dist/templates/.claude/skills/agentv-eval-builder/references/structured-data-evaluators.md +0 -121
- package/dist/templates/.claude/skills/agentv-eval-builder/references/tool-trajectory-evaluator.md +0 -298
- package/dist/templates/.claude/skills/agentv-prompt-optimizer/SKILL.md +0 -78
- package/dist/templates/.github/prompts/agentv-eval-build.prompt.md +0 -5
- package/dist/templates/.github/prompts/agentv-optimize.prompt.md +0 -4
- /package/dist/{dist-T7REAXNS.js.map → dist-BGRU67HI.js.map} +0 -0
- /package/dist/{interactive-TE5SJPJW.js.map → interactive-7KFUCBIP.js.map} +0 -0
|
@@ -148,7 +148,7 @@ var require_dist = __commonJS({
|
|
|
148
148
|
}
|
|
149
149
|
});
|
|
150
150
|
|
|
151
|
-
// ../../packages/core/dist/chunk-
|
|
151
|
+
// ../../packages/core/dist/chunk-7Q4PH265.js
|
|
152
152
|
import { constants } from "node:fs";
|
|
153
153
|
import { access, readFile } from "node:fs/promises";
|
|
154
154
|
import path from "node:path";
|
|
@@ -4195,7 +4195,7 @@ var coerce = {
|
|
|
4195
4195
|
};
|
|
4196
4196
|
var NEVER = INVALID;
|
|
4197
4197
|
|
|
4198
|
-
// ../../packages/core/dist/chunk-
|
|
4198
|
+
// ../../packages/core/dist/chunk-7Q4PH265.js
|
|
4199
4199
|
var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
|
|
4200
4200
|
var TEST_MESSAGE_ROLES = TEST_MESSAGE_ROLE_VALUES;
|
|
4201
4201
|
var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
|
|
@@ -34030,14 +34030,16 @@ function computeTraceSummary(messages) {
|
|
|
34030
34030
|
}
|
|
34031
34031
|
const toolNames = Object.keys(toolCallCounts).sort();
|
|
34032
34032
|
return {
|
|
34033
|
-
|
|
34034
|
-
|
|
34035
|
-
|
|
34036
|
-
|
|
34033
|
+
trace: {
|
|
34034
|
+
eventCount: totalToolCalls,
|
|
34035
|
+
toolNames,
|
|
34036
|
+
toolCallsByName: toolCallCounts,
|
|
34037
|
+
errorCount: 0,
|
|
34038
|
+
llmCallCount,
|
|
34039
|
+
...hasAnyDuration ? { toolDurations } : {}
|
|
34040
|
+
},
|
|
34037
34041
|
startTime: earliestStart?.toISOString(),
|
|
34038
|
-
endTime: latestEnd?.toISOString()
|
|
34039
|
-
llmCallCount,
|
|
34040
|
-
...hasAnyDuration ? { toolDurations } : {}
|
|
34042
|
+
endTime: latestEnd?.toISOString()
|
|
34041
34043
|
};
|
|
34042
34044
|
}
|
|
34043
34045
|
var DEFAULT_EXPLORATION_TOOLS = [
|
|
@@ -34060,9 +34062,9 @@ function explorationRatio(summary, explorationTools = DEFAULT_EXPLORATION_TOOLS)
|
|
|
34060
34062
|
);
|
|
34061
34063
|
return explorationCalls / summary.eventCount;
|
|
34062
34064
|
}
|
|
34063
|
-
function tokensPerTool(summary) {
|
|
34064
|
-
if (!
|
|
34065
|
-
const totalTokens =
|
|
34065
|
+
function tokensPerTool(summary, tokenUsage) {
|
|
34066
|
+
if (!tokenUsage || summary.eventCount === 0) return void 0;
|
|
34067
|
+
const totalTokens = tokenUsage.input + tokenUsage.output;
|
|
34066
34068
|
return totalTokens / summary.eventCount;
|
|
34067
34069
|
}
|
|
34068
34070
|
function avgToolDurationMs(summary) {
|
|
@@ -34078,16 +34080,15 @@ function avgToolDurationMs(summary) {
|
|
|
34078
34080
|
if (totalCalls === 0) return void 0;
|
|
34079
34081
|
return totalDuration / totalCalls;
|
|
34080
34082
|
}
|
|
34081
|
-
function mergeExecutionMetrics(
|
|
34082
|
-
if (!metrics) return
|
|
34083
|
+
function mergeExecutionMetrics(computed, metrics) {
|
|
34084
|
+
if (!metrics) return computed;
|
|
34083
34085
|
return {
|
|
34084
|
-
|
|
34086
|
+
trace: computed.trace,
|
|
34085
34087
|
tokenUsage: metrics.tokenUsage,
|
|
34086
34088
|
costUsd: metrics.costUsd,
|
|
34087
34089
|
durationMs: metrics.durationMs,
|
|
34088
|
-
|
|
34089
|
-
|
|
34090
|
-
endTime: metrics.endTime ?? summary.endTime
|
|
34090
|
+
startTime: metrics.startTime ?? computed.startTime,
|
|
34091
|
+
endTime: metrics.endTime ?? computed.endTime
|
|
34091
34092
|
};
|
|
34092
34093
|
}
|
|
34093
34094
|
var ANSI_YELLOW = "\x1B[33m";
|
|
@@ -36973,10 +36974,13 @@ async function invokeModel(options) {
|
|
|
36973
36974
|
}
|
|
36974
36975
|
function mapResponse(result) {
|
|
36975
36976
|
const content = result.text ?? "";
|
|
36977
|
+
const rawUsage = result.totalUsage ?? result.usage;
|
|
36978
|
+
const tokenUsage = rawUsage?.inputTokens != null && rawUsage?.outputTokens != null ? { input: rawUsage.inputTokens, output: rawUsage.outputTokens } : void 0;
|
|
36976
36979
|
return {
|
|
36977
36980
|
raw: result,
|
|
36978
|
-
usage: toJsonObject(
|
|
36979
|
-
output: [{ role: "assistant", content }]
|
|
36981
|
+
usage: toJsonObject(rawUsage),
|
|
36982
|
+
output: [{ role: "assistant", content }],
|
|
36983
|
+
tokenUsage
|
|
36980
36984
|
};
|
|
36981
36985
|
}
|
|
36982
36986
|
function toJsonObject(value) {
|
|
@@ -42100,6 +42104,8 @@ async function createTargetProxy(options) {
|
|
|
42100
42104
|
const token = randomBytes(32).toString("hex");
|
|
42101
42105
|
let callCount = 0;
|
|
42102
42106
|
let isShutdown = false;
|
|
42107
|
+
let totalInputTokens = 0;
|
|
42108
|
+
let totalOutputTokens = 0;
|
|
42103
42109
|
const targetsList = availableTargets ?? [defaultProvider.targetName];
|
|
42104
42110
|
function resolveProvider(targetName) {
|
|
42105
42111
|
if (targetName === void 0 || targetName === defaultProvider.targetName) {
|
|
@@ -42178,11 +42184,16 @@ async function createTargetProxy(options) {
|
|
|
42178
42184
|
evalCaseId: request.evalCaseId ?? "proxy",
|
|
42179
42185
|
attempt: request.attempt ?? 1
|
|
42180
42186
|
});
|
|
42187
|
+
if (response.tokenUsage) {
|
|
42188
|
+
totalInputTokens += response.tokenUsage.input;
|
|
42189
|
+
totalOutputTokens += response.tokenUsage.output;
|
|
42190
|
+
}
|
|
42181
42191
|
const output = response.output ?? [];
|
|
42182
42192
|
const rawText = extractLastAssistantContent2(output);
|
|
42183
42193
|
const result = {
|
|
42184
42194
|
output,
|
|
42185
|
-
rawText
|
|
42195
|
+
rawText,
|
|
42196
|
+
tokenUsage: response.tokenUsage
|
|
42186
42197
|
};
|
|
42187
42198
|
sendJson(res, 200, result);
|
|
42188
42199
|
} catch (error40) {
|
|
@@ -42229,10 +42240,15 @@ async function createTargetProxy(options) {
|
|
|
42229
42240
|
evalCaseId: request.evalCaseId ?? "proxy",
|
|
42230
42241
|
attempt: request.attempt ?? 1
|
|
42231
42242
|
});
|
|
42243
|
+
if (response.tokenUsage) {
|
|
42244
|
+
totalInputTokens += response.tokenUsage.input;
|
|
42245
|
+
totalOutputTokens += response.tokenUsage.output;
|
|
42246
|
+
}
|
|
42232
42247
|
const output = response.output ?? [];
|
|
42233
42248
|
responses.push({
|
|
42234
42249
|
output,
|
|
42235
|
-
rawText: extractLastAssistantContent2(output)
|
|
42250
|
+
rawText: extractLastAssistantContent2(output),
|
|
42251
|
+
tokenUsage: response.tokenUsage
|
|
42236
42252
|
});
|
|
42237
42253
|
} catch (error40) {
|
|
42238
42254
|
const message = error40 instanceof Error ? error40.message : String(error40);
|
|
@@ -42271,7 +42287,8 @@ async function createTargetProxy(options) {
|
|
|
42271
42287
|
},
|
|
42272
42288
|
getUsageMetadata: () => ({
|
|
42273
42289
|
callCount,
|
|
42274
|
-
maxCalls
|
|
42290
|
+
maxCalls,
|
|
42291
|
+
tokenUsage: totalInputTokens > 0 || totalOutputTokens > 0 ? { input: totalInputTokens, output: totalOutputTokens } : void 0
|
|
42275
42292
|
})
|
|
42276
42293
|
};
|
|
42277
42294
|
}
|
|
@@ -42392,6 +42409,11 @@ var CodeEvaluator = class {
|
|
|
42392
42409
|
),
|
|
42393
42410
|
input: context.evalCase.input,
|
|
42394
42411
|
trace: context.trace ?? null,
|
|
42412
|
+
tokenUsage: context.tokenUsage ?? null,
|
|
42413
|
+
costUsd: context.costUsd ?? null,
|
|
42414
|
+
durationMs: context.durationMs ?? null,
|
|
42415
|
+
startTime: context.startTime ?? null,
|
|
42416
|
+
endTime: context.endTime ?? null,
|
|
42395
42417
|
fileChanges: context.fileChanges ?? null,
|
|
42396
42418
|
workspacePath: context.workspacePath ?? null,
|
|
42397
42419
|
config: this.config ?? null
|
|
@@ -42450,7 +42472,8 @@ var CodeEvaluator = class {
|
|
|
42450
42472
|
expectedAspectCount: hits.length + misses.length || 1,
|
|
42451
42473
|
reasoning,
|
|
42452
42474
|
evaluatorRawRequest,
|
|
42453
|
-
...details ? { details } : {}
|
|
42475
|
+
...details ? { details } : {},
|
|
42476
|
+
tokenUsage: proxyUsage?.tokenUsage
|
|
42454
42477
|
};
|
|
42455
42478
|
} catch (error40) {
|
|
42456
42479
|
const message = error40 instanceof Error ? error40.message : String(error40);
|
|
@@ -42472,7 +42495,8 @@ var CodeEvaluator = class {
|
|
|
42472
42495
|
}
|
|
42473
42496
|
} : {},
|
|
42474
42497
|
error: message
|
|
42475
|
-
}
|
|
42498
|
+
},
|
|
42499
|
+
tokenUsage: proxyUsage?.tokenUsage
|
|
42476
42500
|
};
|
|
42477
42501
|
} finally {
|
|
42478
42502
|
if (proxyShutdown) {
|
|
@@ -42600,7 +42624,7 @@ ${context.fileChanges}`;
|
|
|
42600
42624
|
target: judgeProvider.targetName
|
|
42601
42625
|
};
|
|
42602
42626
|
try {
|
|
42603
|
-
const { data } = await this.runWithRetry({
|
|
42627
|
+
const { data, tokenUsage } = await this.runWithRetry({
|
|
42604
42628
|
context,
|
|
42605
42629
|
judgeProvider,
|
|
42606
42630
|
systemPrompt,
|
|
@@ -42619,7 +42643,8 @@ ${context.fileChanges}`;
|
|
|
42619
42643
|
misses,
|
|
42620
42644
|
expectedAspectCount,
|
|
42621
42645
|
reasoning,
|
|
42622
|
-
evaluatorRawRequest
|
|
42646
|
+
evaluatorRawRequest,
|
|
42647
|
+
tokenUsage
|
|
42623
42648
|
};
|
|
42624
42649
|
} catch {
|
|
42625
42650
|
return {
|
|
@@ -42649,7 +42674,7 @@ ${context.fileChanges}`;
|
|
|
42649
42674
|
systemPrompt,
|
|
42650
42675
|
target: judgeProvider.targetName
|
|
42651
42676
|
};
|
|
42652
|
-
const { data } = await this.runWithRetry({
|
|
42677
|
+
const { data, tokenUsage } = await this.runWithRetry({
|
|
42653
42678
|
context,
|
|
42654
42679
|
judgeProvider,
|
|
42655
42680
|
systemPrompt,
|
|
@@ -42664,7 +42689,8 @@ ${context.fileChanges}`;
|
|
|
42664
42689
|
misses,
|
|
42665
42690
|
expectedAspectCount: rubrics.length,
|
|
42666
42691
|
reasoning: data.overall_reasoning,
|
|
42667
|
-
evaluatorRawRequest
|
|
42692
|
+
evaluatorRawRequest,
|
|
42693
|
+
tokenUsage
|
|
42668
42694
|
};
|
|
42669
42695
|
}
|
|
42670
42696
|
/**
|
|
@@ -42679,7 +42705,7 @@ ${context.fileChanges}`;
|
|
|
42679
42705
|
systemPrompt,
|
|
42680
42706
|
target: judgeProvider.targetName
|
|
42681
42707
|
};
|
|
42682
|
-
const { data } = await this.runWithRetry({
|
|
42708
|
+
const { data, tokenUsage } = await this.runWithRetry({
|
|
42683
42709
|
context,
|
|
42684
42710
|
judgeProvider,
|
|
42685
42711
|
systemPrompt,
|
|
@@ -42695,7 +42721,8 @@ ${context.fileChanges}`;
|
|
|
42695
42721
|
expectedAspectCount: rubrics.length,
|
|
42696
42722
|
reasoning: data.overall_reasoning,
|
|
42697
42723
|
evaluatorRawRequest,
|
|
42698
|
-
details
|
|
42724
|
+
details,
|
|
42725
|
+
tokenUsage
|
|
42699
42726
|
};
|
|
42700
42727
|
}
|
|
42701
42728
|
/**
|
|
@@ -42779,15 +42806,17 @@ ${context.fileChanges}`;
|
|
|
42779
42806
|
try {
|
|
42780
42807
|
const model = judgeProvider.asLanguageModel?.();
|
|
42781
42808
|
if (model) {
|
|
42782
|
-
const
|
|
42809
|
+
const result = await generateText({
|
|
42783
42810
|
model,
|
|
42784
42811
|
system: systemPrompt,
|
|
42785
42812
|
prompt: userPrompt,
|
|
42786
42813
|
...this.maxOutputTokens ? { maxTokens: this.maxOutputTokens } : {},
|
|
42787
42814
|
...typeof this.temperature === "number" ? { temperature: this.temperature } : {}
|
|
42788
42815
|
});
|
|
42789
|
-
const data2 = schema.parse(parseJsonFromText(
|
|
42790
|
-
|
|
42816
|
+
const data2 = schema.parse(parseJsonFromText(result.text));
|
|
42817
|
+
const rawUsage = result.usage;
|
|
42818
|
+
const tokenUsage = rawUsage?.inputTokens != null && rawUsage?.outputTokens != null ? { input: rawUsage.inputTokens, output: rawUsage.outputTokens } : void 0;
|
|
42819
|
+
return { data: data2, tokenUsage };
|
|
42791
42820
|
}
|
|
42792
42821
|
const response = await judgeProvider.invoke({
|
|
42793
42822
|
question: userPrompt,
|
|
@@ -42798,7 +42827,7 @@ ${context.fileChanges}`;
|
|
|
42798
42827
|
temperature: this.temperature
|
|
42799
42828
|
});
|
|
42800
42829
|
const data = schema.parse(parseJsonFromText(extractLastAssistantContent(response.output)));
|
|
42801
|
-
return { data, providerResponse: response };
|
|
42830
|
+
return { data, providerResponse: response, tokenUsage: response.tokenUsage };
|
|
42802
42831
|
} catch (e) {
|
|
42803
42832
|
lastError = e instanceof Error ? e : new Error(String(e));
|
|
42804
42833
|
}
|
|
@@ -43002,7 +43031,8 @@ var CompositeEvaluator = class {
|
|
|
43002
43031
|
reasoning: member.result.reasoning,
|
|
43003
43032
|
evaluatorRawRequest: member.result.evaluatorRawRequest,
|
|
43004
43033
|
scores: member.result.scores,
|
|
43005
|
-
details: member.result.details
|
|
43034
|
+
details: member.result.details,
|
|
43035
|
+
tokenUsage: member.result.tokenUsage
|
|
43006
43036
|
});
|
|
43007
43037
|
}
|
|
43008
43038
|
const finalScore = totalWeight > 0 ? weightedSum / totalWeight : 0;
|
|
@@ -43050,7 +43080,8 @@ var CompositeEvaluator = class {
|
|
|
43050
43080
|
reasoning: member.result.reasoning,
|
|
43051
43081
|
evaluatorRawRequest: member.result.evaluatorRawRequest,
|
|
43052
43082
|
scores: member.result.scores,
|
|
43053
|
-
details: member.result.details
|
|
43083
|
+
details: member.result.details,
|
|
43084
|
+
tokenUsage: member.result.tokenUsage
|
|
43054
43085
|
});
|
|
43055
43086
|
}
|
|
43056
43087
|
const totalCount = results.length;
|
|
@@ -43227,7 +43258,7 @@ var CostEvaluator = class {
|
|
|
43227
43258
|
}
|
|
43228
43259
|
evaluate(context) {
|
|
43229
43260
|
const { budget } = this.config;
|
|
43230
|
-
const costUsd = context.
|
|
43261
|
+
const costUsd = context.costUsd;
|
|
43231
43262
|
if (costUsd === void 0) {
|
|
43232
43263
|
return {
|
|
43233
43264
|
score: 0,
|
|
@@ -43268,7 +43299,7 @@ var ExecutionMetricsEvaluator = class {
|
|
|
43268
43299
|
this.config = options.config;
|
|
43269
43300
|
}
|
|
43270
43301
|
evaluate(context) {
|
|
43271
|
-
const { trace: trace2 } = context;
|
|
43302
|
+
const { trace: trace2, tokenUsage, costUsd, durationMs } = context;
|
|
43272
43303
|
const {
|
|
43273
43304
|
max_tool_calls,
|
|
43274
43305
|
max_llm_calls,
|
|
@@ -43278,7 +43309,8 @@ var ExecutionMetricsEvaluator = class {
|
|
|
43278
43309
|
target_exploration_ratio,
|
|
43279
43310
|
exploration_tolerance = 0.2
|
|
43280
43311
|
} = this.config;
|
|
43281
|
-
|
|
43312
|
+
const needsTrace = max_tool_calls !== void 0 || max_llm_calls !== void 0 || target_exploration_ratio !== void 0;
|
|
43313
|
+
if (needsTrace && !trace2) {
|
|
43282
43314
|
return {
|
|
43283
43315
|
score: 0,
|
|
43284
43316
|
verdict: "fail",
|
|
@@ -43293,11 +43325,12 @@ var ExecutionMetricsEvaluator = class {
|
|
|
43293
43325
|
}
|
|
43294
43326
|
};
|
|
43295
43327
|
}
|
|
43328
|
+
const narrowedTrace = trace2;
|
|
43296
43329
|
const hits = [];
|
|
43297
43330
|
const misses = [];
|
|
43298
43331
|
const actualMetrics = {};
|
|
43299
|
-
if (max_tool_calls !== void 0) {
|
|
43300
|
-
const toolCalls =
|
|
43332
|
+
if (max_tool_calls !== void 0 && narrowedTrace) {
|
|
43333
|
+
const toolCalls = narrowedTrace.eventCount;
|
|
43301
43334
|
actualMetrics.tool_calls = toolCalls;
|
|
43302
43335
|
if (toolCalls <= max_tool_calls) {
|
|
43303
43336
|
hits.push(`Tool calls ${toolCalls} <= ${max_tool_calls} max`);
|
|
@@ -43305,8 +43338,8 @@ var ExecutionMetricsEvaluator = class {
|
|
|
43305
43338
|
misses.push(`Tool calls ${toolCalls} > ${max_tool_calls} max`);
|
|
43306
43339
|
}
|
|
43307
43340
|
}
|
|
43308
|
-
if (max_llm_calls !== void 0) {
|
|
43309
|
-
const llmCalls =
|
|
43341
|
+
if (max_llm_calls !== void 0 && narrowedTrace) {
|
|
43342
|
+
const llmCalls = narrowedTrace.llmCallCount;
|
|
43310
43343
|
if (llmCalls === void 0) {
|
|
43311
43344
|
misses.push("LLM call count data not available");
|
|
43312
43345
|
} else {
|
|
@@ -43319,7 +43352,6 @@ var ExecutionMetricsEvaluator = class {
|
|
|
43319
43352
|
}
|
|
43320
43353
|
}
|
|
43321
43354
|
if (max_tokens !== void 0) {
|
|
43322
|
-
const tokenUsage = trace2.tokenUsage;
|
|
43323
43355
|
if (!tokenUsage) {
|
|
43324
43356
|
misses.push("Token usage data not available");
|
|
43325
43357
|
} else {
|
|
@@ -43333,7 +43365,6 @@ var ExecutionMetricsEvaluator = class {
|
|
|
43333
43365
|
}
|
|
43334
43366
|
}
|
|
43335
43367
|
if (max_cost_usd !== void 0) {
|
|
43336
|
-
const costUsd = trace2.costUsd;
|
|
43337
43368
|
if (costUsd === void 0) {
|
|
43338
43369
|
misses.push("Cost data not available");
|
|
43339
43370
|
} else {
|
|
@@ -43347,7 +43378,6 @@ var ExecutionMetricsEvaluator = class {
|
|
|
43347
43378
|
}
|
|
43348
43379
|
}
|
|
43349
43380
|
if (max_duration_ms !== void 0) {
|
|
43350
|
-
const durationMs = trace2.durationMs;
|
|
43351
43381
|
if (durationMs === void 0) {
|
|
43352
43382
|
misses.push("Duration data not available");
|
|
43353
43383
|
} else {
|
|
@@ -43359,8 +43389,8 @@ var ExecutionMetricsEvaluator = class {
|
|
|
43359
43389
|
}
|
|
43360
43390
|
}
|
|
43361
43391
|
}
|
|
43362
|
-
if (target_exploration_ratio !== void 0) {
|
|
43363
|
-
const ratio = explorationRatio(
|
|
43392
|
+
if (target_exploration_ratio !== void 0 && narrowedTrace) {
|
|
43393
|
+
const ratio = explorationRatio(narrowedTrace);
|
|
43364
43394
|
if (ratio === void 0) {
|
|
43365
43395
|
misses.push("Exploration ratio not available (no tool calls)");
|
|
43366
43396
|
} else {
|
|
@@ -43870,7 +43900,7 @@ var LatencyEvaluator = class {
|
|
|
43870
43900
|
}
|
|
43871
43901
|
evaluate(context) {
|
|
43872
43902
|
const { threshold } = this.config;
|
|
43873
|
-
const durationMs = context.
|
|
43903
|
+
const durationMs = context.durationMs;
|
|
43874
43904
|
if (durationMs === void 0) {
|
|
43875
43905
|
return {
|
|
43876
43906
|
score: 0,
|
|
@@ -44505,7 +44535,7 @@ var TokenUsageEvaluator = class {
|
|
|
44505
44535
|
this.config = options.config;
|
|
44506
44536
|
}
|
|
44507
44537
|
evaluate(context) {
|
|
44508
|
-
const usage = context.
|
|
44538
|
+
const usage = context.tokenUsage;
|
|
44509
44539
|
const maxTotal = this.config.max_total;
|
|
44510
44540
|
const maxInput = this.config.max_input;
|
|
44511
44541
|
const maxOutput = this.config.max_output;
|
|
@@ -46149,7 +46179,7 @@ async function runEvaluation(options) {
|
|
|
46149
46179
|
caseCost = trialCostSum;
|
|
46150
46180
|
}
|
|
46151
46181
|
} else {
|
|
46152
|
-
caseCost = result.
|
|
46182
|
+
caseCost = result.costUsd;
|
|
46153
46183
|
}
|
|
46154
46184
|
if (caseCost !== void 0) {
|
|
46155
46185
|
cumulativeBudgetCost += caseCost;
|
|
@@ -46310,17 +46340,18 @@ async function runBatchEvaluation(options) {
|
|
|
46310
46340
|
const providerResponse = batchResponse[i];
|
|
46311
46341
|
const output = providerResponse.output;
|
|
46312
46342
|
const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
|
|
46313
|
-
const
|
|
46314
|
-
|
|
46315
|
-
toolNames: [],
|
|
46316
|
-
toolCallsByName: {},
|
|
46317
|
-
errorCount: 0
|
|
46318
|
-
} : void 0;
|
|
46319
|
-
const trace2 = baseSummary ? mergeExecutionMetrics(baseSummary, {
|
|
46343
|
+
const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolNames: [], toolCallsByName: {}, errorCount: 0 } } : void 0;
|
|
46344
|
+
const merged = computed ? mergeExecutionMetrics(computed, {
|
|
46320
46345
|
tokenUsage: providerResponse.tokenUsage,
|
|
46321
46346
|
costUsd: providerResponse.costUsd,
|
|
46322
46347
|
durationMs: providerResponse.durationMs
|
|
46323
46348
|
}) : void 0;
|
|
46349
|
+
const trace2 = merged?.trace;
|
|
46350
|
+
const costUsd = merged?.costUsd;
|
|
46351
|
+
const durationMs = merged?.durationMs;
|
|
46352
|
+
const tokenUsage = merged?.tokenUsage;
|
|
46353
|
+
const startTime = merged?.startTime;
|
|
46354
|
+
const endTime = merged?.endTime;
|
|
46324
46355
|
const candidate = extractLastAssistantContent(output);
|
|
46325
46356
|
const providerError = extractProviderError(providerResponse);
|
|
46326
46357
|
let result;
|
|
@@ -46339,6 +46370,11 @@ async function runBatchEvaluation(options) {
|
|
|
46339
46370
|
agentTimeoutMs,
|
|
46340
46371
|
output,
|
|
46341
46372
|
trace: trace2,
|
|
46373
|
+
costUsd,
|
|
46374
|
+
durationMs,
|
|
46375
|
+
tokenUsage,
|
|
46376
|
+
startTime,
|
|
46377
|
+
endTime,
|
|
46342
46378
|
targetResolver,
|
|
46343
46379
|
availableTargets
|
|
46344
46380
|
});
|
|
@@ -46575,17 +46611,18 @@ async function runEvalCase(options) {
|
|
|
46575
46611
|
}
|
|
46576
46612
|
const output = providerResponse.output;
|
|
46577
46613
|
const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
|
|
46578
|
-
const
|
|
46579
|
-
|
|
46580
|
-
toolNames: [],
|
|
46581
|
-
toolCallsByName: {},
|
|
46582
|
-
errorCount: 0
|
|
46583
|
-
} : void 0;
|
|
46584
|
-
const trace2 = baseSummary ? mergeExecutionMetrics(baseSummary, {
|
|
46614
|
+
const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolNames: [], toolCallsByName: {}, errorCount: 0 } } : void 0;
|
|
46615
|
+
const merged = computed ? mergeExecutionMetrics(computed, {
|
|
46585
46616
|
tokenUsage: providerResponse.tokenUsage,
|
|
46586
46617
|
costUsd: providerResponse.costUsd,
|
|
46587
46618
|
durationMs: providerResponse.durationMs
|
|
46588
46619
|
}) : void 0;
|
|
46620
|
+
const trace2 = merged?.trace;
|
|
46621
|
+
const costUsd = merged?.costUsd;
|
|
46622
|
+
const durationMs = merged?.durationMs;
|
|
46623
|
+
const tokenUsage = merged?.tokenUsage;
|
|
46624
|
+
const startTime = merged?.startTime;
|
|
46625
|
+
const endTime = merged?.endTime;
|
|
46589
46626
|
const candidate = extractLastAssistantContent(output);
|
|
46590
46627
|
let fileChanges;
|
|
46591
46628
|
if (baselineCommit && workspacePath) {
|
|
@@ -46630,6 +46667,11 @@ async function runEvalCase(options) {
|
|
|
46630
46667
|
agentTimeoutMs,
|
|
46631
46668
|
output,
|
|
46632
46669
|
trace: trace2,
|
|
46670
|
+
costUsd,
|
|
46671
|
+
durationMs,
|
|
46672
|
+
tokenUsage,
|
|
46673
|
+
startTime,
|
|
46674
|
+
endTime,
|
|
46633
46675
|
targetResolver,
|
|
46634
46676
|
availableTargets,
|
|
46635
46677
|
fileChanges,
|
|
@@ -46686,7 +46728,7 @@ async function runEvalCaseWithTrials(options, trialsConfig) {
|
|
|
46686
46728
|
};
|
|
46687
46729
|
const result = await runEvalCase(trialOptions);
|
|
46688
46730
|
allResults.push(result);
|
|
46689
|
-
const trialCost = result.
|
|
46731
|
+
const trialCost = result.costUsd;
|
|
46690
46732
|
const trialVerdict = scoreToVerdict(result.score);
|
|
46691
46733
|
const trial = {
|
|
46692
46734
|
attempt,
|
|
@@ -46742,6 +46784,11 @@ async function evaluateCandidate(options) {
|
|
|
46742
46784
|
agentTimeoutMs,
|
|
46743
46785
|
output,
|
|
46744
46786
|
trace: trace2,
|
|
46787
|
+
costUsd,
|
|
46788
|
+
durationMs,
|
|
46789
|
+
tokenUsage,
|
|
46790
|
+
startTime,
|
|
46791
|
+
endTime,
|
|
46745
46792
|
targetResolver,
|
|
46746
46793
|
availableTargets,
|
|
46747
46794
|
fileChanges,
|
|
@@ -46762,6 +46809,11 @@ async function evaluateCandidate(options) {
|
|
|
46762
46809
|
agentTimeoutMs,
|
|
46763
46810
|
output,
|
|
46764
46811
|
trace: trace2,
|
|
46812
|
+
costUsd,
|
|
46813
|
+
durationMs,
|
|
46814
|
+
tokenUsage,
|
|
46815
|
+
startTime,
|
|
46816
|
+
endTime,
|
|
46765
46817
|
targetResolver,
|
|
46766
46818
|
availableTargets,
|
|
46767
46819
|
fileChanges,
|
|
@@ -46805,6 +46857,11 @@ async function evaluateCandidate(options) {
|
|
|
46805
46857
|
answer: candidate,
|
|
46806
46858
|
target: target.name,
|
|
46807
46859
|
reasoning: score.reasoning,
|
|
46860
|
+
tokenUsage,
|
|
46861
|
+
costUsd,
|
|
46862
|
+
durationMs,
|
|
46863
|
+
startTime,
|
|
46864
|
+
endTime,
|
|
46808
46865
|
requests,
|
|
46809
46866
|
input,
|
|
46810
46867
|
scores,
|
|
@@ -46828,6 +46885,11 @@ async function runEvaluatorsForCase(options) {
|
|
|
46828
46885
|
agentTimeoutMs,
|
|
46829
46886
|
output,
|
|
46830
46887
|
trace: trace2,
|
|
46888
|
+
costUsd,
|
|
46889
|
+
durationMs,
|
|
46890
|
+
tokenUsage,
|
|
46891
|
+
startTime,
|
|
46892
|
+
endTime,
|
|
46831
46893
|
targetResolver,
|
|
46832
46894
|
availableTargets,
|
|
46833
46895
|
fileChanges,
|
|
@@ -46849,6 +46911,11 @@ async function runEvaluatorsForCase(options) {
|
|
|
46849
46911
|
agentTimeoutMs,
|
|
46850
46912
|
output,
|
|
46851
46913
|
trace: trace2,
|
|
46914
|
+
costUsd,
|
|
46915
|
+
durationMs,
|
|
46916
|
+
tokenUsage,
|
|
46917
|
+
startTime,
|
|
46918
|
+
endTime,
|
|
46852
46919
|
targetResolver,
|
|
46853
46920
|
availableTargets,
|
|
46854
46921
|
fileChanges,
|
|
@@ -46871,6 +46938,11 @@ async function runEvaluatorsForCase(options) {
|
|
|
46871
46938
|
judgeProvider,
|
|
46872
46939
|
output,
|
|
46873
46940
|
trace: trace2,
|
|
46941
|
+
tokenUsage,
|
|
46942
|
+
costUsd,
|
|
46943
|
+
durationMs,
|
|
46944
|
+
startTime,
|
|
46945
|
+
endTime,
|
|
46874
46946
|
targetResolver,
|
|
46875
46947
|
availableTargets,
|
|
46876
46948
|
fileChanges,
|
|
@@ -46894,6 +46966,11 @@ async function runEvaluatorList(options) {
|
|
|
46894
46966
|
agentTimeoutMs,
|
|
46895
46967
|
output,
|
|
46896
46968
|
trace: trace2,
|
|
46969
|
+
costUsd,
|
|
46970
|
+
durationMs,
|
|
46971
|
+
tokenUsage,
|
|
46972
|
+
startTime,
|
|
46973
|
+
endTime,
|
|
46897
46974
|
targetResolver,
|
|
46898
46975
|
availableTargets,
|
|
46899
46976
|
fileChanges,
|
|
@@ -46912,6 +46989,11 @@ async function runEvaluatorList(options) {
|
|
|
46912
46989
|
judgeProvider,
|
|
46913
46990
|
output,
|
|
46914
46991
|
trace: trace2,
|
|
46992
|
+
tokenUsage,
|
|
46993
|
+
costUsd,
|
|
46994
|
+
durationMs,
|
|
46995
|
+
startTime,
|
|
46996
|
+
endTime,
|
|
46915
46997
|
targetResolver,
|
|
46916
46998
|
availableTargets,
|
|
46917
46999
|
fileChanges,
|
|
@@ -46951,7 +47033,8 @@ async function runEvaluatorList(options) {
|
|
|
46951
47033
|
reasoning: score2.reasoning,
|
|
46952
47034
|
evaluatorProviderRequest: score2.evaluatorRawRequest,
|
|
46953
47035
|
details: score2.details,
|
|
46954
|
-
scores: mapChildResults(score2.scores)
|
|
47036
|
+
scores: mapChildResults(score2.scores),
|
|
47037
|
+
tokenUsage: score2.tokenUsage
|
|
46955
47038
|
});
|
|
46956
47039
|
} catch (error40) {
|
|
46957
47040
|
const message = error40 instanceof Error ? error40.message : String(error40);
|
|
@@ -47199,7 +47282,8 @@ function mapChildResults(children) {
|
|
|
47199
47282
|
reasoning: child.reasoning,
|
|
47200
47283
|
evaluatorProviderRequest: child.evaluatorRawRequest,
|
|
47201
47284
|
scores: mapChildResults(child.scores),
|
|
47202
|
-
details: child.details
|
|
47285
|
+
details: child.details,
|
|
47286
|
+
tokenUsage: child.tokenUsage
|
|
47203
47287
|
}));
|
|
47204
47288
|
}
|
|
47205
47289
|
function computeWeightedMean(entries) {
|
|
@@ -47562,7 +47646,13 @@ var STRIPPED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
|
|
|
47562
47646
|
"beforeEachOutput",
|
|
47563
47647
|
"afterAllOutput",
|
|
47564
47648
|
"afterEachOutput",
|
|
47565
|
-
"fileChanges"
|
|
47649
|
+
"fileChanges",
|
|
47650
|
+
// Promoted execution metrics (debug, not needed for regression comparison)
|
|
47651
|
+
"tokenUsage",
|
|
47652
|
+
"costUsd",
|
|
47653
|
+
"durationMs",
|
|
47654
|
+
"startTime",
|
|
47655
|
+
"endTime"
|
|
47566
47656
|
]);
|
|
47567
47657
|
var STRIPPED_EVALUATOR_FIELDS = /* @__PURE__ */ new Set(["rawRequest", "evaluatorProviderRequest"]);
|
|
47568
47658
|
function trimEvaluatorResult(result) {
|
|
@@ -47683,8 +47773,8 @@ var OtelTraceExporter = class {
|
|
|
47683
47773
|
const api = this.api;
|
|
47684
47774
|
const tracer = this.tracer;
|
|
47685
47775
|
const captureContent = this.options.captureContent ?? false;
|
|
47686
|
-
const startHr = toHrTime(result.
|
|
47687
|
-
const endHr = toHrTime(result.
|
|
47776
|
+
const startHr = toHrTime(result.startTime ?? result.timestamp);
|
|
47777
|
+
const endHr = toHrTime(result.endTime ?? result.timestamp);
|
|
47688
47778
|
let parentCtx = api.ROOT_CONTEXT;
|
|
47689
47779
|
const traceparent = process.env.TRACEPARENT;
|
|
47690
47780
|
if (traceparent && this.W3CPropagator) {
|
|
@@ -47713,12 +47803,13 @@ var OtelTraceExporter = class {
|
|
|
47713
47803
|
if (result.dataset) rootSpan.setAttribute("agentv.dataset", result.dataset);
|
|
47714
47804
|
rootSpan.setAttribute("agentv.score", result.score);
|
|
47715
47805
|
if (captureContent) rootSpan.setAttribute("agentv.answer", result.answer);
|
|
47806
|
+
if (result.durationMs != null)
|
|
47807
|
+
rootSpan.setAttribute("agentv.trace.duration_ms", result.durationMs);
|
|
47808
|
+
if (result.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", result.costUsd);
|
|
47716
47809
|
if (result.trace) {
|
|
47717
47810
|
const t = result.trace;
|
|
47718
47811
|
rootSpan.setAttribute("agentv.trace.event_count", t.eventCount);
|
|
47719
47812
|
rootSpan.setAttribute("agentv.trace.tool_names", t.toolNames.join(","));
|
|
47720
|
-
if (t.durationMs != null) rootSpan.setAttribute("agentv.trace.duration_ms", t.durationMs);
|
|
47721
|
-
if (t.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", t.costUsd);
|
|
47722
47813
|
if (t.llmCallCount != null)
|
|
47723
47814
|
rootSpan.setAttribute("agentv.trace.llm_call_count", t.llmCallCount);
|
|
47724
47815
|
}
|
|
@@ -48105,4 +48196,4 @@ export {
|
|
|
48105
48196
|
OtelStreamingObserver,
|
|
48106
48197
|
createAgentKernel
|
|
48107
48198
|
};
|
|
48108
|
-
//# sourceMappingURL=chunk-
|
|
48199
|
+
//# sourceMappingURL=chunk-RJWTL3VS.js.map
|