agentv 5.2.0-next.1 → 5.3.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{artifact-writer-SN6UNL5G.js → artifact-writer-JFNIPMKW.js} +4 -4
- package/dist/{chunk-3S6A2RKR.js → chunk-6ZZCDZPD.js} +1357 -304
- package/dist/chunk-6ZZCDZPD.js.map +1 -0
- package/dist/{chunk-46K2OET3.js → chunk-BHKQHG26.js} +2 -2
- package/dist/{chunk-T4AD3H3Y.js → chunk-FVR4RQFK.js} +74 -16
- package/dist/chunk-FVR4RQFK.js.map +1 -0
- package/dist/{chunk-3X357HS4.js → chunk-T32NL3E6.js} +62 -6
- package/dist/chunk-T32NL3E6.js.map +1 -0
- package/dist/{chunk-ALVZQUZP.js → chunk-V52ATPTT.js} +2 -2
- package/dist/cli.js +5 -5
- package/dist/{dist-DEPJOMCA.js → dist-6Z7U473R.js} +11 -3
- package/dist/index.js +5 -5
- package/dist/{interactive-36ZNMQB2.js → interactive-RUY3OCBI.js} +5 -5
- package/dist/skills/agentv-bench/SKILL.md +1 -1
- package/dist/skills/agentv-bench/agents/analyzer.md +1 -1
- package/dist/skills/agentv-bench/agents/comparator.md +3 -3
- package/dist/skills/agentv-bench/agents/grader.md +4 -2
- package/dist/skills/agentv-bench/references/description-optimization.md +5 -6
- package/dist/skills/agentv-bench/references/environment-adaptation.md +2 -2
- package/dist/skills/agentv-bench/references/eval-yaml-spec.md +20 -9
- package/dist/skills/agentv-eval-migrations/references/breaking-changes.md +100 -2
- package/dist/skills/agentv-eval-writer/SKILL.md +27 -16
- package/dist/{ts-eval-loader-OMG2JBHP-OP5RR7A3.js → ts-eval-loader-DQDYRULE-V3377FOL.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-3S6A2RKR.js.map +0 -1
- package/dist/chunk-3X357HS4.js.map +0 -1
- package/dist/chunk-T4AD3H3Y.js.map +0 -1
- /package/dist/{artifact-writer-SN6UNL5G.js.map → artifact-writer-JFNIPMKW.js.map} +0 -0
- /package/dist/{chunk-46K2OET3.js.map → chunk-BHKQHG26.js.map} +0 -0
- /package/dist/{chunk-ALVZQUZP.js.map → chunk-V52ATPTT.js.map} +0 -0
- /package/dist/{dist-DEPJOMCA.js.map → dist-6Z7U473R.js.map} +0 -0
- /package/dist/{interactive-36ZNMQB2.js.map → interactive-RUY3OCBI.js.map} +0 -0
- /package/dist/{ts-eval-loader-OMG2JBHP-OP5RR7A3.js.map → ts-eval-loader-DQDYRULE-V3377FOL.js.map} +0 -0
|
@@ -8114,7 +8114,7 @@ function toFiniteNumber(value) {
|
|
|
8114
8114
|
return void 0;
|
|
8115
8115
|
}
|
|
8116
8116
|
|
|
8117
|
-
// ../../packages/core/dist/chunk-
|
|
8117
|
+
// ../../packages/core/dist/chunk-QMPBDTFI.js
|
|
8118
8118
|
var SDK_CHILD_PROTOCOL_VERSION = 1;
|
|
8119
8119
|
function providerRequestToWire(request) {
|
|
8120
8120
|
if (request.tools && request.tools.length > 0) {
|
|
@@ -8146,6 +8146,7 @@ function providerResponseFromWire(response) {
|
|
|
8146
8146
|
return {
|
|
8147
8147
|
raw: response.raw,
|
|
8148
8148
|
usage: response.usage,
|
|
8149
|
+
metadata: response.metadata,
|
|
8149
8150
|
output: response.output,
|
|
8150
8151
|
tokenUsage: response.token_usage,
|
|
8151
8152
|
costUsd: response.cost_usd,
|
|
@@ -12198,7 +12199,7 @@ var coerce = {
|
|
|
12198
12199
|
};
|
|
12199
12200
|
var NEVER = INVALID;
|
|
12200
12201
|
|
|
12201
|
-
// ../../packages/core/dist/chunk-
|
|
12202
|
+
// ../../packages/core/dist/chunk-RXN2AWVO.js
|
|
12202
12203
|
import { parse } from "yaml";
|
|
12203
12204
|
import path3 from "node:path";
|
|
12204
12205
|
import path22 from "node:path";
|
|
@@ -12292,6 +12293,13 @@ var GRADER_KIND_VALUES = [
|
|
|
12292
12293
|
"script",
|
|
12293
12294
|
"llm-grader",
|
|
12294
12295
|
"tool-trajectory",
|
|
12296
|
+
"skill-used",
|
|
12297
|
+
"not-skill-used",
|
|
12298
|
+
"trajectory:tool-used",
|
|
12299
|
+
"trajectory:tool-args-match",
|
|
12300
|
+
"trajectory:tool-sequence",
|
|
12301
|
+
"trajectory:step-count",
|
|
12302
|
+
"trajectory:goal-success",
|
|
12295
12303
|
"field-accuracy",
|
|
12296
12304
|
"latency",
|
|
12297
12305
|
"cost",
|
|
@@ -15846,6 +15854,141 @@ function subscribeToCodexLogEntries(listener) {
|
|
|
15846
15854
|
};
|
|
15847
15855
|
}
|
|
15848
15856
|
|
|
15857
|
+
// ../../packages/core/dist/chunk-7FVITOZM.js
|
|
15858
|
+
var SKILL_PATH_PATTERN = /(?:^|[/"'\s])(?:\.agents\/)?skills\/([^/"'\s]+)\/SKILL\.md(?:$|[)"'\s,;:])/g;
|
|
15859
|
+
function deriveSkillCallsFromMessages(messages) {
|
|
15860
|
+
if (!messages) {
|
|
15861
|
+
return [];
|
|
15862
|
+
}
|
|
15863
|
+
return deriveSkillCallsFromToolCalls(messages.flatMap((message) => message.toolCalls ?? []));
|
|
15864
|
+
}
|
|
15865
|
+
function deriveSkillCallsFromToolCalls(toolCalls) {
|
|
15866
|
+
if (!toolCalls) {
|
|
15867
|
+
return [];
|
|
15868
|
+
}
|
|
15869
|
+
const skillCalls = [];
|
|
15870
|
+
const heuristicKeys = /* @__PURE__ */ new Set();
|
|
15871
|
+
const pushHeuristic = (entry) => {
|
|
15872
|
+
const key = `${entry.name}\0${entry.path ?? ""}\0${entry.source ?? ""}`;
|
|
15873
|
+
if (heuristicKeys.has(key)) {
|
|
15874
|
+
return;
|
|
15875
|
+
}
|
|
15876
|
+
heuristicKeys.add(key);
|
|
15877
|
+
skillCalls.push(entry);
|
|
15878
|
+
};
|
|
15879
|
+
for (const toolCall of toolCalls) {
|
|
15880
|
+
const toolName = toolCall.tool.toLowerCase();
|
|
15881
|
+
const input = asRecord(toolCall.input);
|
|
15882
|
+
const isError = isErroredToolCall(toolCall);
|
|
15883
|
+
if (toolName === "skill") {
|
|
15884
|
+
const skillName = normalizeSkillName(input?.skill ?? input?.name);
|
|
15885
|
+
if (skillName) {
|
|
15886
|
+
skillCalls.push(
|
|
15887
|
+
dropUndefined({ name: skillName, input: toolCall.input, source: "tool", isError })
|
|
15888
|
+
);
|
|
15889
|
+
}
|
|
15890
|
+
continue;
|
|
15891
|
+
}
|
|
15892
|
+
if (toolName === "read") {
|
|
15893
|
+
const path49 = normalizeText(input?.file_path ?? input?.path ?? input?.filePath);
|
|
15894
|
+
const skillName = path49 ? skillNameFromPath(path49) : void 0;
|
|
15895
|
+
if (skillName) {
|
|
15896
|
+
pushHeuristic(
|
|
15897
|
+
dropUndefined({
|
|
15898
|
+
name: skillName,
|
|
15899
|
+
input: toolCall.input,
|
|
15900
|
+
path: path49,
|
|
15901
|
+
source: "heuristic",
|
|
15902
|
+
isError
|
|
15903
|
+
})
|
|
15904
|
+
);
|
|
15905
|
+
}
|
|
15906
|
+
continue;
|
|
15907
|
+
}
|
|
15908
|
+
if (toolName === "bash") {
|
|
15909
|
+
const command = normalizeText(input?.command);
|
|
15910
|
+
if (command) {
|
|
15911
|
+
for (const candidate of skillPathCandidates(command)) {
|
|
15912
|
+
pushHeuristic(
|
|
15913
|
+
dropUndefined({
|
|
15914
|
+
name: candidate.name,
|
|
15915
|
+
input: toolCall.input,
|
|
15916
|
+
path: candidate.path,
|
|
15917
|
+
source: "heuristic",
|
|
15918
|
+
isError
|
|
15919
|
+
})
|
|
15920
|
+
);
|
|
15921
|
+
}
|
|
15922
|
+
}
|
|
15923
|
+
}
|
|
15924
|
+
if (toolCall.output !== void 0) {
|
|
15925
|
+
const outputText = typeof toolCall.output === "string" ? toolCall.output : JSON.stringify(toolCall.output);
|
|
15926
|
+
for (const candidate of skillPathCandidates(outputText)) {
|
|
15927
|
+
pushHeuristic(
|
|
15928
|
+
dropUndefined({
|
|
15929
|
+
name: candidate.name,
|
|
15930
|
+
path: candidate.path,
|
|
15931
|
+
source: "heuristic",
|
|
15932
|
+
isError
|
|
15933
|
+
})
|
|
15934
|
+
);
|
|
15935
|
+
}
|
|
15936
|
+
}
|
|
15937
|
+
}
|
|
15938
|
+
return skillCalls;
|
|
15939
|
+
}
|
|
15940
|
+
function skillCallMetadata(skillCalls) {
|
|
15941
|
+
if (skillCalls.length === 0) {
|
|
15942
|
+
return void 0;
|
|
15943
|
+
}
|
|
15944
|
+
return { skillCalls };
|
|
15945
|
+
}
|
|
15946
|
+
function skillPathCandidates(text) {
|
|
15947
|
+
const candidates = /* @__PURE__ */ new Map();
|
|
15948
|
+
for (const rawToken of text.split(/\s+/)) {
|
|
15949
|
+
const token = rawToken.replace(/^[`"'([{<]+|[`"',;:)\]}>]+$/g, "").replace(/\\/g, "/");
|
|
15950
|
+
const match = token.match(/(?:^|\/)(?:\.agents\/)?skills\/([^/\s]+)\/SKILL\.md$/);
|
|
15951
|
+
if (match && isValidSkillName(match[1])) {
|
|
15952
|
+
candidates.set(token, { name: match[1], path: token });
|
|
15953
|
+
}
|
|
15954
|
+
}
|
|
15955
|
+
for (const match of text.replace(/\\/g, "/").matchAll(SKILL_PATH_PATTERN)) {
|
|
15956
|
+
const pathMatch = match[0].match(/(?:\.agents\/)?skills\/([^/"'\s]+)\/SKILL\.md/);
|
|
15957
|
+
const name = pathMatch?.[1];
|
|
15958
|
+
if (name && isValidSkillName(name)) {
|
|
15959
|
+
candidates.set(pathMatch[0], { name, path: pathMatch[0] });
|
|
15960
|
+
}
|
|
15961
|
+
}
|
|
15962
|
+
return Array.from(candidates.values());
|
|
15963
|
+
}
|
|
15964
|
+
function skillNameFromPath(path49) {
|
|
15965
|
+
const normalized = path49.replace(/\\/g, "/");
|
|
15966
|
+
const match = normalized.match(/(?:^|\/)(?:\.agents\/)?skills\/([^/\s]+)\/SKILL\.md$/);
|
|
15967
|
+
const name = match?.[1];
|
|
15968
|
+
return name && isValidSkillName(name) ? name : void 0;
|
|
15969
|
+
}
|
|
15970
|
+
function isErroredToolCall(toolCall) {
|
|
15971
|
+
if (toolCall.status === void 0 || toolCall.status === "ok" || toolCall.status === "unknown") {
|
|
15972
|
+
return void 0;
|
|
15973
|
+
}
|
|
15974
|
+
return true;
|
|
15975
|
+
}
|
|
15976
|
+
function normalizeSkillName(value) {
|
|
15977
|
+
return normalizeText(value);
|
|
15978
|
+
}
|
|
15979
|
+
function normalizeText(value) {
|
|
15980
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
15981
|
+
}
|
|
15982
|
+
function asRecord(value) {
|
|
15983
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
|
|
15984
|
+
}
|
|
15985
|
+
function isValidSkillName(name) {
|
|
15986
|
+
return /^[A-Za-z0-9._:-]+$/.test(name);
|
|
15987
|
+
}
|
|
15988
|
+
function dropUndefined(value) {
|
|
15989
|
+
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
15990
|
+
}
|
|
15991
|
+
|
|
15849
15992
|
// ../../packages/core/dist/chunk-YRELA7BA.js
|
|
15850
15993
|
var TOOL_NAME_MAP = /* @__PURE__ */ new Map([
|
|
15851
15994
|
// --- Claude (already canonical) ---
|
|
@@ -16093,7 +16236,7 @@ function pathToFileUri(filePath) {
|
|
|
16093
16236
|
return `file://${normalizedPath}`;
|
|
16094
16237
|
}
|
|
16095
16238
|
|
|
16096
|
-
// ../../packages/core/dist/chunk-
|
|
16239
|
+
// ../../packages/core/dist/chunk-WBZHD7AZ.js
|
|
16097
16240
|
import path48 from "node:path";
|
|
16098
16241
|
import { pathToFileURL as pathToFileURL4 } from "node:url";
|
|
16099
16242
|
import { existsSync as existsSync6 } from "node:fs";
|
|
@@ -16120,6 +16263,7 @@ import path42 from "node:path";
|
|
|
16120
16263
|
import { readFile as readFile23 } from "node:fs/promises";
|
|
16121
16264
|
import path33 from "node:path";
|
|
16122
16265
|
import { fileURLToPath as fileURLToPath4 } from "node:url";
|
|
16266
|
+
import { isDeepStrictEqual } from "node:util";
|
|
16123
16267
|
import path52 from "node:path";
|
|
16124
16268
|
import { pathToFileURL as pathToFileURL22 } from "node:url";
|
|
16125
16269
|
import { spawn } from "node:child_process";
|
|
@@ -28763,7 +28907,7 @@ var RequestError = class _RequestError extends Error {
|
|
|
28763
28907
|
}
|
|
28764
28908
|
};
|
|
28765
28909
|
|
|
28766
|
-
// ../../packages/core/dist/chunk-
|
|
28910
|
+
// ../../packages/core/dist/chunk-WBZHD7AZ.js
|
|
28767
28911
|
import { randomUUID as randomUUID5 } from "node:crypto";
|
|
28768
28912
|
import { createWriteStream as createWriteStream3 } from "node:fs";
|
|
28769
28913
|
import { mkdir as mkdir6, mkdtemp as mkdtemp3, rm as rm3, writeFile as writeFile4 } from "node:fs/promises";
|
|
@@ -28912,8 +29056,8 @@ function shouldSkipCacheForTemperature(targetConfig) {
|
|
|
28912
29056
|
return false;
|
|
28913
29057
|
}
|
|
28914
29058
|
var DEFAULT_THRESHOLD2 = 0.8;
|
|
28915
|
-
function scoreToVerdict(
|
|
28916
|
-
return
|
|
29059
|
+
function scoreToVerdict(score2, threshold = DEFAULT_THRESHOLD2) {
|
|
29060
|
+
return score2 >= threshold ? "pass" : "fail";
|
|
28917
29061
|
}
|
|
28918
29062
|
function clampScore(value) {
|
|
28919
29063
|
if (Number.isNaN(value) || !Number.isFinite(value)) {
|
|
@@ -28984,12 +29128,12 @@ var NEGATED_VERDICT = {
|
|
|
28984
29128
|
fail: "pass",
|
|
28985
29129
|
skip: "skip"
|
|
28986
29130
|
};
|
|
28987
|
-
function negateScore(
|
|
29131
|
+
function negateScore(score2) {
|
|
28988
29132
|
return {
|
|
28989
|
-
...
|
|
28990
|
-
score: clampScore(1 -
|
|
28991
|
-
verdict: NEGATED_VERDICT[
|
|
28992
|
-
assertions:
|
|
29133
|
+
...score2,
|
|
29134
|
+
score: clampScore(1 - score2.score),
|
|
29135
|
+
verdict: NEGATED_VERDICT[score2.verdict],
|
|
29136
|
+
assertions: score2.assertions.map((a) => ({
|
|
28993
29137
|
...a,
|
|
28994
29138
|
passed: !a.passed,
|
|
28995
29139
|
evidence: a.evidence ? `[Negated] ${a.evidence}` : void 0
|
|
@@ -29905,14 +30049,14 @@ function checksToAssertions(checks) {
|
|
|
29905
30049
|
}
|
|
29906
30050
|
function normalizeScriptProtocol(parsed) {
|
|
29907
30051
|
const checks = parseScriptChecks(parsed.checks);
|
|
29908
|
-
const
|
|
29909
|
-
const pass = typeof parsed.pass === "boolean" ? parsed.pass : scoreToVerdict(
|
|
30052
|
+
const score2 = optionalScore(parsed, "score") ?? (checks.length > 0 ? checks.reduce((sum, check2) => sum + (check2.score ?? (check2.pass ? 1 : 0)), 0) / checks.length : typeof parsed.pass === "boolean" ? parsed.pass ? 1 : 0 : 0);
|
|
30053
|
+
const pass = typeof parsed.pass === "boolean" ? parsed.pass : scoreToVerdict(score2) === "pass";
|
|
29910
30054
|
const reason = optionalString(parsed, "reason");
|
|
29911
30055
|
const details = parsed.details && typeof parsed.details === "object" && !Array.isArray(parsed.details) ? parsed.details : void 0;
|
|
29912
30056
|
if (typeof parsed.pass !== "boolean" && typeof parsed.score !== "number" && checks.length === 0) {
|
|
29913
30057
|
throw new Error("Script evaluator JSON must include pass, score, or checks[]");
|
|
29914
30058
|
}
|
|
29915
|
-
return { pass, score, reason, checks, details };
|
|
30059
|
+
return { pass, score: score2, reason, checks, details };
|
|
29916
30060
|
}
|
|
29917
30061
|
var ScriptGrader = class {
|
|
29918
30062
|
kind = "script";
|
|
@@ -30051,8 +30195,8 @@ var ScriptGrader = class {
|
|
|
30051
30195
|
const protocol = parsed != null ? normalizeScriptProtocol(parsed) : void 0;
|
|
30052
30196
|
const checks = protocol?.checks ?? [];
|
|
30053
30197
|
const assertions = protocol != null ? checksToAssertions(checks) : [{ text: stdout.trim() || (passed ? "exit 0" : `exit ${exitCode}`), passed }];
|
|
30054
|
-
const
|
|
30055
|
-
const verdict = protocol ? protocol.pass ? "pass" : "fail" : scoreToVerdict(
|
|
30198
|
+
const score2 = protocol?.score ?? (passed ? 1 : 0);
|
|
30199
|
+
const verdict = protocol ? protocol.pass ? "pass" : "fail" : scoreToVerdict(score2);
|
|
30056
30200
|
const reason = protocol?.reason;
|
|
30057
30201
|
const details = protocol?.details;
|
|
30058
30202
|
const proxyUsage = getProxyUsage?.();
|
|
@@ -30067,7 +30211,7 @@ var ScriptGrader = class {
|
|
|
30067
30211
|
} : {}
|
|
30068
30212
|
};
|
|
30069
30213
|
return {
|
|
30070
|
-
score,
|
|
30214
|
+
score: score2,
|
|
30071
30215
|
verdict,
|
|
30072
30216
|
reason,
|
|
30073
30217
|
checks,
|
|
@@ -30174,10 +30318,10 @@ var CostGrader = class {
|
|
|
30174
30318
|
};
|
|
30175
30319
|
}
|
|
30176
30320
|
const passed = costUsd <= budget;
|
|
30177
|
-
const
|
|
30321
|
+
const score2 = passed ? 1 : 0;
|
|
30178
30322
|
const formatCost = (n) => `$${n.toFixed(4)}`;
|
|
30179
30323
|
return {
|
|
30180
|
-
score,
|
|
30324
|
+
score: score2,
|
|
30181
30325
|
verdict: passed ? "pass" : "fail",
|
|
30182
30326
|
assertions: [
|
|
30183
30327
|
passed ? { text: `Cost ${formatCost(costUsd)} <= ${formatCost(budget)} budget`, passed: true } : { text: `Cost ${formatCost(costUsd)} > ${formatCost(budget)} budget`, passed: false }
|
|
@@ -31279,10 +31423,10 @@ var ExecutionMetricsGrader = class {
|
|
|
31279
31423
|
}
|
|
31280
31424
|
const totalChecks = assertions.length;
|
|
31281
31425
|
const passedCount = assertions.filter((a) => a.passed).length;
|
|
31282
|
-
const
|
|
31426
|
+
const score2 = totalChecks > 0 ? passedCount / totalChecks : 0;
|
|
31283
31427
|
return {
|
|
31284
|
-
score,
|
|
31285
|
-
verdict: scoreToVerdict(
|
|
31428
|
+
score: score2,
|
|
31429
|
+
verdict: scoreToVerdict(score2),
|
|
31286
31430
|
assertions,
|
|
31287
31431
|
expectedAspectCount: totalChecks || 1,
|
|
31288
31432
|
graderRawRequest: {
|
|
@@ -31618,22 +31762,22 @@ var FieldAccuracyGrader = class {
|
|
|
31618
31762
|
for (const result of results) {
|
|
31619
31763
|
assertions.push({ text: result.message, passed: result.hit });
|
|
31620
31764
|
}
|
|
31621
|
-
let
|
|
31765
|
+
let score2;
|
|
31622
31766
|
if (aggregation === "all_or_nothing") {
|
|
31623
31767
|
const hasFailed = assertions.some((a) => !a.passed);
|
|
31624
|
-
|
|
31768
|
+
score2 = hasFailed ? 0 : 1;
|
|
31625
31769
|
} else {
|
|
31626
31770
|
const totalWeight = results.reduce((sum, r) => sum + r.weight, 0);
|
|
31627
31771
|
if (totalWeight === 0) {
|
|
31628
|
-
|
|
31772
|
+
score2 = results.length === 0 ? 1 : 0;
|
|
31629
31773
|
} else {
|
|
31630
31774
|
const weightedSum = results.reduce((sum, r) => sum + r.score * r.weight, 0);
|
|
31631
|
-
|
|
31775
|
+
score2 = weightedSum / totalWeight;
|
|
31632
31776
|
}
|
|
31633
31777
|
}
|
|
31634
31778
|
return {
|
|
31635
|
-
score: clampScore(
|
|
31636
|
-
verdict: scoreToVerdict(
|
|
31779
|
+
score: clampScore(score2),
|
|
31780
|
+
verdict: scoreToVerdict(score2),
|
|
31637
31781
|
assertions,
|
|
31638
31782
|
expectedAspectCount: results.length
|
|
31639
31783
|
};
|
|
@@ -31752,9 +31896,9 @@ var LatencyGrader = class {
|
|
|
31752
31896
|
};
|
|
31753
31897
|
}
|
|
31754
31898
|
const passed = durationMs <= threshold;
|
|
31755
|
-
const
|
|
31899
|
+
const score2 = passed ? 1 : 0;
|
|
31756
31900
|
return {
|
|
31757
|
-
score,
|
|
31901
|
+
score: score2,
|
|
31758
31902
|
verdict: passed ? "pass" : "fail",
|
|
31759
31903
|
assertions: [
|
|
31760
31904
|
passed ? { text: `Duration ${durationMs}ms <= ${threshold}ms threshold`, passed: true } : { text: `Duration ${durationMs}ms > ${threshold}ms threshold`, passed: false }
|
|
@@ -32139,12 +32283,12 @@ function promptfooFallbackText(config2, fallbackText) {
|
|
|
32139
32283
|
}
|
|
32140
32284
|
return fallbackText;
|
|
32141
32285
|
}
|
|
32142
|
-
function promptfooFallbackReason(pass,
|
|
32286
|
+
function promptfooFallbackReason(pass, score2, threshold) {
|
|
32143
32287
|
if (pass) {
|
|
32144
32288
|
return "Grading passed";
|
|
32145
32289
|
}
|
|
32146
|
-
if (threshold !== void 0 &&
|
|
32147
|
-
return `Score ${
|
|
32290
|
+
if (threshold !== void 0 && score2 < threshold) {
|
|
32291
|
+
return `Score ${score2} below threshold ${threshold}`;
|
|
32148
32292
|
}
|
|
32149
32293
|
return "Grading failed";
|
|
32150
32294
|
}
|
|
@@ -32187,14 +32331,14 @@ function normalizePromptfooGradingResult(options) {
|
|
|
32187
32331
|
});
|
|
32188
32332
|
const threshold = promptfooThreshold(options.config);
|
|
32189
32333
|
const rawPass = data.pass !== void 0 ? normalizePromptfooPass(data.pass) : normalizedChecks && normalizedChecks.length > 0 ? normalizedChecks.every((check2) => check2.pass) : true;
|
|
32190
|
-
const
|
|
32334
|
+
const score2 = data.score !== void 0 ? normalizePromptfooScore(data.score, rawPass) : normalizedChecks && normalizedChecks.length > 0 ? normalizedChecks.reduce((total, check2) => {
|
|
32191
32335
|
if (typeof check2.score === "number") {
|
|
32192
32336
|
return total + check2.score;
|
|
32193
32337
|
}
|
|
32194
32338
|
return total + Number(check2.pass);
|
|
32195
32339
|
}, 0) / normalizedChecks.length : normalizePromptfooScore(void 0, rawPass);
|
|
32196
|
-
const pass = threshold !== void 0 ? rawPass &&
|
|
32197
|
-
const reason = data.reason?.trim() || promptfooFallbackReason(pass,
|
|
32340
|
+
const pass = threshold !== void 0 ? rawPass && score2 >= threshold : rawPass;
|
|
32341
|
+
const reason = data.reason?.trim() || promptfooFallbackReason(pass, score2, threshold);
|
|
32198
32342
|
const assertions = normalizedChecks && normalizedChecks.length > 0 ? normalizedChecks.map((check2) => ({
|
|
32199
32343
|
text: check2.text,
|
|
32200
32344
|
passed: check2.pass,
|
|
@@ -32207,12 +32351,12 @@ function normalizePromptfooGradingResult(options) {
|
|
|
32207
32351
|
}
|
|
32208
32352
|
];
|
|
32209
32353
|
return {
|
|
32210
|
-
score,
|
|
32354
|
+
score: score2,
|
|
32211
32355
|
verdict: pass ? "pass" : "fail",
|
|
32212
32356
|
assertions,
|
|
32213
32357
|
details: {
|
|
32214
32358
|
pass,
|
|
32215
|
-
score,
|
|
32359
|
+
score: score2,
|
|
32216
32360
|
reason,
|
|
32217
32361
|
...threshold !== void 0 ? { threshold } : {},
|
|
32218
32362
|
...normalizedChecks && normalizedChecks.length > 0 ? { checks: normalizedChecks } : {}
|
|
@@ -32318,11 +32462,11 @@ ${context.toolCalls}`;
|
|
|
32318
32462
|
});
|
|
32319
32463
|
}
|
|
32320
32464
|
const data2 = freeformEvaluationSchema.parse(raw);
|
|
32321
|
-
const
|
|
32465
|
+
const score2 = clampScore(data2.score);
|
|
32322
32466
|
const assertions = Array.isArray(data2.assertions) ? data2.assertions.slice(0, 8) : [];
|
|
32323
32467
|
return {
|
|
32324
|
-
score,
|
|
32325
|
-
verdict: scoreToVerdict(
|
|
32468
|
+
score: score2,
|
|
32469
|
+
verdict: scoreToVerdict(score2),
|
|
32326
32470
|
assertions,
|
|
32327
32471
|
details: data2.details
|
|
32328
32472
|
};
|
|
@@ -32436,9 +32580,9 @@ ${context.toolCalls}`;
|
|
|
32436
32580
|
schema: scoreRangeEvaluationSchema,
|
|
32437
32581
|
images
|
|
32438
32582
|
});
|
|
32439
|
-
const { score, verdict, assertions, details } = calculateScoreRangeResult(data, rubrics);
|
|
32583
|
+
const { score: score2, verdict, assertions, details } = calculateScoreRangeResult(data, rubrics);
|
|
32440
32584
|
return {
|
|
32441
|
-
score,
|
|
32585
|
+
score: score2,
|
|
32442
32586
|
verdict,
|
|
32443
32587
|
assertions,
|
|
32444
32588
|
expectedAspectCount: rubrics.length,
|
|
@@ -32774,9 +32918,9 @@ ${outputSchema}`;
|
|
|
32774
32918
|
};
|
|
32775
32919
|
}
|
|
32776
32920
|
const data2 = rubricEvaluationSchema.parse(parsed);
|
|
32777
|
-
const { score:
|
|
32921
|
+
const { score: score3, verdict, assertions: assertions2 } = calculateRubricScore(data2, rubrics);
|
|
32778
32922
|
return {
|
|
32779
|
-
score:
|
|
32923
|
+
score: score3,
|
|
32780
32924
|
verdict,
|
|
32781
32925
|
assertions: assertions2,
|
|
32782
32926
|
expectedAspectCount: rubrics.length,
|
|
@@ -32802,11 +32946,11 @@ ${outputSchema}`;
|
|
|
32802
32946
|
};
|
|
32803
32947
|
}
|
|
32804
32948
|
const data = freeformEvaluationSchema.parse(parsed);
|
|
32805
|
-
const
|
|
32949
|
+
const score2 = clampScore(data.score);
|
|
32806
32950
|
const assertions = Array.isArray(data.assertions) ? data.assertions.slice(0, 8) : [];
|
|
32807
32951
|
return {
|
|
32808
|
-
score,
|
|
32809
|
-
verdict: scoreToVerdict(
|
|
32952
|
+
score: score2,
|
|
32953
|
+
verdict: scoreToVerdict(score2),
|
|
32810
32954
|
assertions,
|
|
32811
32955
|
expectedAspectCount: Math.max(assertions.length, 1),
|
|
32812
32956
|
graderRawRequest,
|
|
@@ -33131,9 +33275,9 @@ function calculateRubricScore(result, rubrics) {
|
|
|
33131
33275
|
failedRequired = true;
|
|
33132
33276
|
}
|
|
33133
33277
|
}
|
|
33134
|
-
const
|
|
33135
|
-
const verdict = failedRequired ? "fail" : scoreToVerdict(
|
|
33136
|
-
return { score, verdict, assertions };
|
|
33278
|
+
const score2 = totalWeight > 0 ? Math.min(1, Math.max(0, earnedWeight / totalWeight)) : 0;
|
|
33279
|
+
const verdict = failedRequired ? "fail" : scoreToVerdict(score2);
|
|
33280
|
+
return { score: score2, verdict, assertions };
|
|
33137
33281
|
}
|
|
33138
33282
|
function buildScoreRangeFormatInstructions() {
|
|
33139
33283
|
return `You must return a valid JSON object matching this schema:
|
|
@@ -33190,10 +33334,10 @@ function calculateScoreRangeResult(result, rubrics) {
|
|
|
33190
33334
|
evidence: check2.reasoning
|
|
33191
33335
|
});
|
|
33192
33336
|
}
|
|
33193
|
-
const
|
|
33194
|
-
const verdict = failedRequired ? "fail" : scoreToVerdict(
|
|
33337
|
+
const score2 = totalWeight > 0 ? Math.min(1, Math.max(0, weightedScoreSum / totalWeight)) : 0;
|
|
33338
|
+
const verdict = failedRequired ? "fail" : scoreToVerdict(score2);
|
|
33195
33339
|
return {
|
|
33196
|
-
score,
|
|
33340
|
+
score: score2,
|
|
33197
33341
|
verdict,
|
|
33198
33342
|
assertions,
|
|
33199
33343
|
details: {
|
|
@@ -33473,6 +33617,174 @@ var SkillTriggerGrader = class {
|
|
|
33473
33617
|
};
|
|
33474
33618
|
}
|
|
33475
33619
|
};
|
|
33620
|
+
var SkillUsedGrader = class {
|
|
33621
|
+
constructor(config2) {
|
|
33622
|
+
this.config = config2;
|
|
33623
|
+
this.kind = config2.type;
|
|
33624
|
+
}
|
|
33625
|
+
kind;
|
|
33626
|
+
evaluate(context) {
|
|
33627
|
+
const inverse = this.config.type === "not-skill-used" || this.config.negate === true;
|
|
33628
|
+
const skillCalls = getSkillCalls(context);
|
|
33629
|
+
const actualSkills = skillCalls.map(formatSkillCall);
|
|
33630
|
+
const expected = resolveSkillMatchers(this.config.value);
|
|
33631
|
+
if (expected.kind === "list") {
|
|
33632
|
+
return handleListSkillAssertion({
|
|
33633
|
+
config: this.config,
|
|
33634
|
+
inverse,
|
|
33635
|
+
skillCalls,
|
|
33636
|
+
actualSkills,
|
|
33637
|
+
expected
|
|
33638
|
+
});
|
|
33639
|
+
}
|
|
33640
|
+
return handleCountSkillAssertion({
|
|
33641
|
+
config: this.config,
|
|
33642
|
+
inverse,
|
|
33643
|
+
skillCalls,
|
|
33644
|
+
actualSkills,
|
|
33645
|
+
matcher: expected.matcher
|
|
33646
|
+
});
|
|
33647
|
+
}
|
|
33648
|
+
};
|
|
33649
|
+
function getSkillCalls(context) {
|
|
33650
|
+
const responseMetadata = context.responseMetadata ?? {};
|
|
33651
|
+
const metadata = responseMetadata.metadata;
|
|
33652
|
+
const rawSkillCalls = responseMetadata.skill_calls ?? responseMetadata.skillCalls ?? (isRecord2(metadata) ? metadata.skillCalls ?? metadata.skill_calls : void 0);
|
|
33653
|
+
if (!Array.isArray(rawSkillCalls)) {
|
|
33654
|
+
return [];
|
|
33655
|
+
}
|
|
33656
|
+
return rawSkillCalls.filter(
|
|
33657
|
+
(entry) => isRecord2(entry) && typeof entry.name === "string" && entry.name.trim().length > 0 && entry.isError !== true && entry.is_error !== true
|
|
33658
|
+
);
|
|
33659
|
+
}
|
|
33660
|
+
function matchesSkill(skillCall, matcher) {
|
|
33661
|
+
if (matcher.name && skillCall.name !== matcher.name) {
|
|
33662
|
+
return false;
|
|
33663
|
+
}
|
|
33664
|
+
if (matcher.pattern && !matchesPattern(skillCall.name, matcher.pattern)) {
|
|
33665
|
+
return false;
|
|
33666
|
+
}
|
|
33667
|
+
return true;
|
|
33668
|
+
}
|
|
33669
|
+
function formatSkillCall(skillCall) {
|
|
33670
|
+
const details = [skillCall.source, skillCall.path].filter(Boolean).join(", ");
|
|
33671
|
+
return details ? `${skillCall.name} (${details})` : skillCall.name;
|
|
33672
|
+
}
|
|
33673
|
+
function resolveSkillMatchers(value) {
|
|
33674
|
+
const normalizeText2 = (text) => typeof text === "string" ? text.trim() : void 0;
|
|
33675
|
+
const validateCount = (field, count) => {
|
|
33676
|
+
if (!Number.isFinite(count) || !Number.isInteger(count) || count < 0) {
|
|
33677
|
+
throw new Error(`skill-used assertion object ${field} must be a finite non-negative integer`);
|
|
33678
|
+
}
|
|
33679
|
+
};
|
|
33680
|
+
if (typeof value === "string" && value.trim()) {
|
|
33681
|
+
const name = value.trim();
|
|
33682
|
+
return {
|
|
33683
|
+
kind: "list",
|
|
33684
|
+
matchers: [{ name }]
|
|
33685
|
+
};
|
|
33686
|
+
}
|
|
33687
|
+
if (Array.isArray(value) && value.length > 0 && value.every((item) => typeof item === "string" && item.trim())) {
|
|
33688
|
+
return {
|
|
33689
|
+
kind: "list",
|
|
33690
|
+
matchers: value.map((item) => ({ name: item.trim() }))
|
|
33691
|
+
};
|
|
33692
|
+
}
|
|
33693
|
+
if (isRecord2(value)) {
|
|
33694
|
+
const name = normalizeText2(value.name);
|
|
33695
|
+
const pattern = normalizeText2(value.pattern);
|
|
33696
|
+
if (!name && !pattern) {
|
|
33697
|
+
throw new Error("skill-used assertion object must include a name or pattern property");
|
|
33698
|
+
}
|
|
33699
|
+
if ("min" in value) {
|
|
33700
|
+
validateCount("min", value.min);
|
|
33701
|
+
}
|
|
33702
|
+
if ("max" in value) {
|
|
33703
|
+
validateCount("max", value.max);
|
|
33704
|
+
}
|
|
33705
|
+
if (typeof value.min === "number" && typeof value.max === "number" && value.max < value.min) {
|
|
33706
|
+
throw new Error("skill-used assertion object max must be greater than or equal to min");
|
|
33707
|
+
}
|
|
33708
|
+
return {
|
|
33709
|
+
kind: "count",
|
|
33710
|
+
matcher: {
|
|
33711
|
+
max: typeof value.max === "number" ? value.max : void 0,
|
|
33712
|
+
min: typeof value.min === "number" ? value.min : void 0,
|
|
33713
|
+
name,
|
|
33714
|
+
pattern
|
|
33715
|
+
}
|
|
33716
|
+
};
|
|
33717
|
+
}
|
|
33718
|
+
throw new Error("skill-used assertion must have a string, string array, or object value");
|
|
33719
|
+
}
|
|
33720
|
+
function handleListSkillAssertion(params) {
|
|
33721
|
+
const { config: config2, inverse, skillCalls, actualSkills, expected } = params;
|
|
33722
|
+
const missing = expected.matchers.filter(
|
|
33723
|
+
(matcher) => !skillCalls.some((skillCall) => matchesSkill(skillCall, matcher))
|
|
33724
|
+
);
|
|
33725
|
+
const matched = expected.matchers.filter(
|
|
33726
|
+
(matcher) => skillCalls.some((skillCall) => matchesSkill(skillCall, matcher))
|
|
33727
|
+
);
|
|
33728
|
+
const pass = inverse ? matched.length === 0 : missing.length === 0;
|
|
33729
|
+
const expectedSkills = expected.matchers.map((matcher) => matcher.name);
|
|
33730
|
+
const actualSummary = actualSkills.length > 0 ? actualSkills.join(", ") : "(none)";
|
|
33731
|
+
let reason;
|
|
33732
|
+
if (inverse) {
|
|
33733
|
+
reason = pass ? `Forbidden skill(s) were not used: ${expectedSkills.join(", ")}` : `Forbidden skill(s) were used: ${matched.map((matcher) => matcher.name).join(", ")}. Actual skills: ${actualSummary}`;
|
|
33734
|
+
} else if (pass) {
|
|
33735
|
+
reason = `Observed required skill(s): ${expectedSkills.join(", ")}. Actual skills: ${actualSummary}`;
|
|
33736
|
+
} else {
|
|
33737
|
+
reason = `Missing required skill(s): ${missing.map((matcher) => matcher.name).join(", ")}. Actual skills: ${actualSummary}`;
|
|
33738
|
+
}
|
|
33739
|
+
return skillScore(config2, pass, reason);
|
|
33740
|
+
}
|
|
33741
|
+
function handleCountSkillAssertion(params) {
|
|
33742
|
+
const { config: config2, inverse, skillCalls, actualSkills, matcher } = params;
|
|
33743
|
+
const hasExplicitMin = matcher.min !== void 0;
|
|
33744
|
+
const hasExplicitMax = matcher.max !== void 0;
|
|
33745
|
+
const min = matcher.min ?? (hasExplicitMax ? 0 : 1);
|
|
33746
|
+
const max = matcher.max;
|
|
33747
|
+
const matchingSkillCalls = skillCalls.filter((skillCall) => matchesSkill(skillCall, matcher));
|
|
33748
|
+
const count = matchingSkillCalls.length;
|
|
33749
|
+
const matcherLabel = matcher.pattern || matcher.name || "*";
|
|
33750
|
+
if (inverse) {
|
|
33751
|
+
if (hasExplicitMin || hasExplicitMax && max !== 0) {
|
|
33752
|
+
throw new Error(
|
|
33753
|
+
"not-skill-used object assertions only support name/pattern with no count bounds, or max: 0"
|
|
33754
|
+
);
|
|
33755
|
+
}
|
|
33756
|
+
const pass2 = count === 0;
|
|
33757
|
+
const actualSummary = actualSkills.length > 0 ? actualSkills.join(", ") : "(none)";
|
|
33758
|
+
const reason2 = pass2 ? `Forbidden skill "${matcherLabel}" was not used. Actual skills: ${actualSummary}` : `Forbidden skill "${matcherLabel}" was used ${count} time(s). Matches: ${matchingSkillCalls.map(formatSkillCall).join(", ")}`;
|
|
33759
|
+
return skillScore(config2, pass2, reason2);
|
|
33760
|
+
}
|
|
33761
|
+
const pass = count >= min && (max === void 0 || count <= max);
|
|
33762
|
+
let reason = `Matched skill "${matcherLabel}" ${count} time(s)`;
|
|
33763
|
+
reason += max === void 0 ? ` (expected at least ${min})` : ` (expected ${min}-${max})`;
|
|
33764
|
+
if (matchingSkillCalls.length > 0) {
|
|
33765
|
+
reason += `. Matches: ${matchingSkillCalls.map(formatSkillCall).join(", ")}`;
|
|
33766
|
+
}
|
|
33767
|
+
return skillScore(config2, pass, reason);
|
|
33768
|
+
}
|
|
33769
|
+
function skillScore(config2, pass, reason) {
|
|
33770
|
+
return {
|
|
33771
|
+
score: pass ? 1 : 0,
|
|
33772
|
+
verdict: pass ? "pass" : "fail",
|
|
33773
|
+
reason,
|
|
33774
|
+
assertions: [{ text: reason, passed: pass }],
|
|
33775
|
+
expectedAspectCount: 1,
|
|
33776
|
+
details: {
|
|
33777
|
+
assertion_type: config2.type
|
|
33778
|
+
}
|
|
33779
|
+
};
|
|
33780
|
+
}
|
|
33781
|
+
function matchesPattern(skillName, pattern) {
|
|
33782
|
+
const regexPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".");
|
|
33783
|
+
return new RegExp(`^${regexPattern}$`, "i").test(skillName);
|
|
33784
|
+
}
|
|
33785
|
+
function isRecord2(value) {
|
|
33786
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
33787
|
+
}
|
|
33476
33788
|
function stringifyPretty2(value) {
|
|
33477
33789
|
return value === void 0 ? "" : JSON.stringify(value, null, 2);
|
|
33478
33790
|
}
|
|
@@ -33952,10 +34264,10 @@ var ToolTrajectoryGrader = class {
|
|
|
33952
34264
|
}
|
|
33953
34265
|
}
|
|
33954
34266
|
const passedCount = assertions.filter((a) => a.passed).length;
|
|
33955
|
-
const
|
|
34267
|
+
const score2 = passedCount / toolNames.length;
|
|
33956
34268
|
return {
|
|
33957
|
-
score,
|
|
33958
|
-
verdict: scoreToVerdict(
|
|
34269
|
+
score: score2,
|
|
34270
|
+
verdict: scoreToVerdict(score2),
|
|
33959
34271
|
assertions,
|
|
33960
34272
|
expectedAspectCount: toolNames.length
|
|
33961
34273
|
};
|
|
@@ -34038,10 +34350,10 @@ var ToolTrajectoryGrader = class {
|
|
|
34038
34350
|
}
|
|
34039
34351
|
const effectiveLatencyAssertions = latencyAssertionCount - latencySkips;
|
|
34040
34352
|
const totalAssertions = expected.length + effectiveLatencyAssertions;
|
|
34041
|
-
const
|
|
34353
|
+
const score2 = totalAssertions > 0 ? (sequenceHits + latencyHits) / totalAssertions : 1;
|
|
34042
34354
|
return {
|
|
34043
|
-
score,
|
|
34044
|
-
verdict: scoreToVerdict(
|
|
34355
|
+
score: score2,
|
|
34356
|
+
verdict: scoreToVerdict(score2),
|
|
34045
34357
|
assertions,
|
|
34046
34358
|
expectedAspectCount: totalAssertions
|
|
34047
34359
|
};
|
|
@@ -34120,10 +34432,10 @@ var ToolTrajectoryGrader = class {
|
|
|
34120
34432
|
}
|
|
34121
34433
|
const effectiveLatencyAssertions = latencyAssertionCount - latencySkips;
|
|
34122
34434
|
const totalAssertions = expected.length + effectiveLatencyAssertions;
|
|
34123
|
-
const
|
|
34435
|
+
const score2 = totalAssertions > 0 ? (sequenceHits + latencyHits) / totalAssertions : 1;
|
|
34124
34436
|
return {
|
|
34125
|
-
score,
|
|
34126
|
-
verdict: scoreToVerdict(
|
|
34437
|
+
score: score2,
|
|
34438
|
+
verdict: scoreToVerdict(score2),
|
|
34127
34439
|
assertions,
|
|
34128
34440
|
expectedAspectCount: totalAssertions
|
|
34129
34441
|
};
|
|
@@ -34168,10 +34480,10 @@ var ToolTrajectoryGrader = class {
|
|
|
34168
34480
|
}
|
|
34169
34481
|
}
|
|
34170
34482
|
const passedCount = assertions.filter((a) => a.passed).length;
|
|
34171
|
-
const
|
|
34483
|
+
const score2 = expected.length > 0 ? passedCount / expected.length : 1;
|
|
34172
34484
|
return {
|
|
34173
|
-
score,
|
|
34174
|
-
verdict: scoreToVerdict(
|
|
34485
|
+
score: score2,
|
|
34486
|
+
verdict: scoreToVerdict(score2),
|
|
34175
34487
|
assertions,
|
|
34176
34488
|
expectedAspectCount: expected.length
|
|
34177
34489
|
};
|
|
@@ -34236,15 +34548,608 @@ var ToolTrajectoryGrader = class {
|
|
|
34236
34548
|
}
|
|
34237
34549
|
}
|
|
34238
34550
|
const passedCount = assertions.filter((a) => a.passed).length;
|
|
34239
|
-
const
|
|
34551
|
+
const score2 = toolCalls.length > 0 ? passedCount / toolCalls.length : 1;
|
|
34240
34552
|
return {
|
|
34241
|
-
score,
|
|
34242
|
-
verdict: scoreToVerdict(
|
|
34553
|
+
score: score2,
|
|
34554
|
+
verdict: scoreToVerdict(score2),
|
|
34243
34555
|
assertions,
|
|
34244
34556
|
expectedAspectCount: toolCalls.length
|
|
34245
34557
|
};
|
|
34246
34558
|
}
|
|
34247
34559
|
};
|
|
34560
|
+
var COMMAND_TOOL_NAMES = /* @__PURE__ */ new Set(["exec_command", "local_shell", "shell"]);
|
|
34561
|
+
var SEARCH_NAME_PATTERN = /(^|[\s._:/-])(search|find|lookup|retriev(?:e|al))($|[\s._:/-])/i;
|
|
34562
|
+
function isRecord22(value) {
|
|
34563
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
34564
|
+
}
|
|
34565
|
+
function matchesPattern2(value, pattern) {
|
|
34566
|
+
const regexPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".");
|
|
34567
|
+
return new RegExp(`^${regexPattern}$`, "i").test(value);
|
|
34568
|
+
}
|
|
34569
|
+
function normalizeStructuredValue(value) {
|
|
34570
|
+
if (value === void 0 || value === null) {
|
|
34571
|
+
return void 0;
|
|
34572
|
+
}
|
|
34573
|
+
if (typeof value !== "string") {
|
|
34574
|
+
return value;
|
|
34575
|
+
}
|
|
34576
|
+
const trimmed = value.trim();
|
|
34577
|
+
if (!trimmed) {
|
|
34578
|
+
return void 0;
|
|
34579
|
+
}
|
|
34580
|
+
try {
|
|
34581
|
+
return JSON.parse(trimmed);
|
|
34582
|
+
} catch {
|
|
34583
|
+
return trimmed;
|
|
34584
|
+
}
|
|
34585
|
+
}
|
|
34586
|
+
function getFirstString(values) {
|
|
34587
|
+
for (const value of values) {
|
|
34588
|
+
if (typeof value === "string" && value.trim()) {
|
|
34589
|
+
return value.trim();
|
|
34590
|
+
}
|
|
34591
|
+
}
|
|
34592
|
+
return void 0;
|
|
34593
|
+
}
|
|
34594
|
+
function commandFromInput(input) {
|
|
34595
|
+
if (!isRecord22(input)) {
|
|
34596
|
+
return void 0;
|
|
34597
|
+
}
|
|
34598
|
+
const command = input.cmd ?? input.command ?? input.commands;
|
|
34599
|
+
if (typeof command === "string" && command.trim()) {
|
|
34600
|
+
return command.trim();
|
|
34601
|
+
}
|
|
34602
|
+
if (Array.isArray(command)) {
|
|
34603
|
+
const joined = command.map((part) => String(part).trim()).filter(Boolean).join(command === input.commands ? "; " : " ");
|
|
34604
|
+
return joined || void 0;
|
|
34605
|
+
}
|
|
34606
|
+
return void 0;
|
|
34607
|
+
}
|
|
34608
|
+
function executableFromCommand(command) {
|
|
34609
|
+
return command.trim().split(/\s+/)[0] || void 0;
|
|
34610
|
+
}
|
|
34611
|
+
function classifyMessageStep(event) {
|
|
34612
|
+
const itemType = event.metadata?.["codex.item.type"] ?? event.message?.metadata?.["codex.item.type"];
|
|
34613
|
+
if (itemType === "reasoning") {
|
|
34614
|
+
return "reasoning";
|
|
34615
|
+
}
|
|
34616
|
+
return "message";
|
|
34617
|
+
}
|
|
34618
|
+
function eventToTrajectoryStep(event) {
|
|
34619
|
+
const attributes = {
|
|
34620
|
+
...event.metadata ?? {},
|
|
34621
|
+
...event.message?.metadata ?? {},
|
|
34622
|
+
...event.tool?.metadata ?? {}
|
|
34623
|
+
};
|
|
34624
|
+
if (event.type === "tool_call" && event.tool) {
|
|
34625
|
+
const toolName = event.tool.name;
|
|
34626
|
+
const args = normalizeStructuredValue(event.tool.input);
|
|
34627
|
+
const command = COMMAND_TOOL_NAMES.has(toolName.toLowerCase()) ? commandFromInput(args) : void 0;
|
|
34628
|
+
if (command) {
|
|
34629
|
+
const aliases = /* @__PURE__ */ new Set([command, toolName]);
|
|
34630
|
+
const executable = executableFromCommand(command);
|
|
34631
|
+
if (executable) {
|
|
34632
|
+
aliases.add(executable);
|
|
34633
|
+
}
|
|
34634
|
+
return {
|
|
34635
|
+
aliases: [...aliases],
|
|
34636
|
+
...args === void 0 ? {} : { args },
|
|
34637
|
+
attributes,
|
|
34638
|
+
name: command,
|
|
34639
|
+
spanName: toolName,
|
|
34640
|
+
startTime: event.timestamp,
|
|
34641
|
+
type: "command"
|
|
34642
|
+
};
|
|
34643
|
+
}
|
|
34644
|
+
return {
|
|
34645
|
+
aliases: [toolName],
|
|
34646
|
+
...args === void 0 ? {} : { args },
|
|
34647
|
+
attributes,
|
|
34648
|
+
name: toolName,
|
|
34649
|
+
spanName: toolName,
|
|
34650
|
+
startTime: event.timestamp,
|
|
34651
|
+
type: "tool"
|
|
34652
|
+
};
|
|
34653
|
+
}
|
|
34654
|
+
const messageName = event.message?.name ?? event.type;
|
|
34655
|
+
const query = getFirstString([event.metadata?.query, event.metadata?.search_query]);
|
|
34656
|
+
if (query && SEARCH_NAME_PATTERN.test(messageName)) {
|
|
34657
|
+
return {
|
|
34658
|
+
aliases: [query, messageName],
|
|
34659
|
+
attributes,
|
|
34660
|
+
name: query,
|
|
34661
|
+
spanName: messageName,
|
|
34662
|
+
startTime: event.timestamp,
|
|
34663
|
+
type: "search"
|
|
34664
|
+
};
|
|
34665
|
+
}
|
|
34666
|
+
const messageType = event.type === "message" || event.type === "final_response";
|
|
34667
|
+
const type = messageType ? classifyMessageStep(event) : "span";
|
|
34668
|
+
return {
|
|
34669
|
+
aliases: [messageName, type],
|
|
34670
|
+
attributes,
|
|
34671
|
+
name: messageName,
|
|
34672
|
+
spanName: messageName,
|
|
34673
|
+
startTime: event.timestamp,
|
|
34674
|
+
type
|
|
34675
|
+
};
|
|
34676
|
+
}
|
|
34677
|
+
function extractTrajectorySteps(trace) {
|
|
34678
|
+
return [...trace.events].sort((left, right) => {
|
|
34679
|
+
const leftTime = left.timestamp ? Date.parse(left.timestamp) : Number.NaN;
|
|
34680
|
+
const rightTime = right.timestamp ? Date.parse(right.timestamp) : Number.NaN;
|
|
34681
|
+
if (Number.isFinite(leftTime) && Number.isFinite(rightTime) && leftTime !== rightTime) {
|
|
34682
|
+
return leftTime - rightTime;
|
|
34683
|
+
}
|
|
34684
|
+
return left.ordinal - right.ordinal;
|
|
34685
|
+
}).map(eventToTrajectoryStep);
|
|
34686
|
+
}
|
|
34687
|
+
function normalizeMatcher(matcher, defaultType) {
|
|
34688
|
+
if (typeof matcher === "string") {
|
|
34689
|
+
return { pattern: matcher, ...defaultType ? { type: defaultType } : {} };
|
|
34690
|
+
}
|
|
34691
|
+
return { ...matcher, ...matcher.type ? {} : defaultType ? { type: defaultType } : {} };
|
|
34692
|
+
}
|
|
34693
|
+
function matchesTrajectoryStep(step, matcher, defaultType) {
|
|
34694
|
+
const normalized = normalizeMatcher(matcher, defaultType);
|
|
34695
|
+
if (normalized.type) {
|
|
34696
|
+
const allowedTypes = Array.isArray(normalized.type) ? normalized.type : [normalized.type];
|
|
34697
|
+
if (!allowedTypes.includes(step.type)) {
|
|
34698
|
+
return false;
|
|
34699
|
+
}
|
|
34700
|
+
}
|
|
34701
|
+
const matchPattern = normalized.pattern ?? normalized.name;
|
|
34702
|
+
return matchPattern ? step.aliases.some((alias) => matchesPattern2(alias, matchPattern)) : true;
|
|
34703
|
+
}
|
|
34704
|
+
function formatStep(step) {
|
|
34705
|
+
return `${step.type}:${step.name}`;
|
|
34706
|
+
}
|
|
34707
|
+
function formatArgs(args) {
|
|
34708
|
+
if (args === void 0) {
|
|
34709
|
+
return "(none)";
|
|
34710
|
+
}
|
|
34711
|
+
try {
|
|
34712
|
+
return JSON.stringify(args) ?? String(args);
|
|
34713
|
+
} catch {
|
|
34714
|
+
return String(args);
|
|
34715
|
+
}
|
|
34716
|
+
}
|
|
34717
|
+
function formatStepList(stepLabels) {
|
|
34718
|
+
return stepLabels.length > 0 ? stepLabels.join(", ") : "(none)";
|
|
34719
|
+
}
|
|
34720
|
+
function score(pass, text) {
|
|
34721
|
+
return {
|
|
34722
|
+
score: pass ? 1 : 0,
|
|
34723
|
+
verdict: scoreToVerdict(pass ? 1 : 0),
|
|
34724
|
+
assertions: [{ text, passed: pass }],
|
|
34725
|
+
expectedAspectCount: 1
|
|
34726
|
+
};
|
|
34727
|
+
}
|
|
34728
|
+
function missingTrace(type) {
|
|
34729
|
+
return score(false, `No trace data available for ${type} assertion`);
|
|
34730
|
+
}
|
|
34731
|
+
function requireNamedMatcher(matcher, assertionType, index) {
|
|
34732
|
+
if (matcher.pattern || matcher.name) {
|
|
34733
|
+
return;
|
|
34734
|
+
}
|
|
34735
|
+
const label = index === void 0 ? "object" : `step ${index + 1}`;
|
|
34736
|
+
throw new Error(`${assertionType} assertion ${label} must include a name or pattern property`);
|
|
34737
|
+
}
|
|
34738
|
+
function resolveToolMatchers(value) {
|
|
34739
|
+
if (typeof value === "string") {
|
|
34740
|
+
return { kind: "list", matchers: [normalizeMatcher(value, "tool")] };
|
|
34741
|
+
}
|
|
34742
|
+
if (Array.isArray(value) && value.every((item) => typeof item === "string")) {
|
|
34743
|
+
return { kind: "list", matchers: value.map((item) => normalizeMatcher(item, "tool")) };
|
|
34744
|
+
}
|
|
34745
|
+
if (isRecord22(value)) {
|
|
34746
|
+
const matcher = normalizeMatcher(value, "tool");
|
|
34747
|
+
return {
|
|
34748
|
+
kind: "count",
|
|
34749
|
+
matcher: {
|
|
34750
|
+
...matcher,
|
|
34751
|
+
...typeof value.min === "number" ? { min: value.min } : {},
|
|
34752
|
+
...typeof value.max === "number" ? { max: value.max } : {}
|
|
34753
|
+
}
|
|
34754
|
+
};
|
|
34755
|
+
}
|
|
34756
|
+
throw new Error(
|
|
34757
|
+
"trajectory:tool-used assertion must have a string, string array, or object value"
|
|
34758
|
+
);
|
|
34759
|
+
}
|
|
34760
|
+
function evaluateToolUsed(config2, trace) {
|
|
34761
|
+
const steps = extractTrajectorySteps(trace).filter((step) => step.type === "tool");
|
|
34762
|
+
const expected = resolveToolMatchers(config2.value);
|
|
34763
|
+
const inverse = config2.inverse === true;
|
|
34764
|
+
if (expected.kind === "list") {
|
|
34765
|
+
if (expected.matchers.length === 0) {
|
|
34766
|
+
throw new Error("trajectory:tool-used assertion requires at least one expected tool");
|
|
34767
|
+
}
|
|
34768
|
+
const missing = expected.matchers.filter(
|
|
34769
|
+
(matcher2) => !steps.some((step) => matchesTrajectoryStep(step, matcher2))
|
|
34770
|
+
);
|
|
34771
|
+
const matched = expected.matchers.filter(
|
|
34772
|
+
(matcher2) => steps.some((step) => matchesTrajectoryStep(step, matcher2))
|
|
34773
|
+
);
|
|
34774
|
+
const pass2 = inverse ? matched.length === 0 : missing.length === 0;
|
|
34775
|
+
const actualTools = steps.map(formatStep);
|
|
34776
|
+
const expectedTools = expected.matchers.map(
|
|
34777
|
+
(matcher2) => matcher2.pattern ?? matcher2.name ?? "*"
|
|
34778
|
+
);
|
|
34779
|
+
if (inverse) {
|
|
34780
|
+
return score(
|
|
34781
|
+
pass2,
|
|
34782
|
+
pass2 ? `Forbidden tool(s) were not used: ${expectedTools.join(", ")}` : `Forbidden tool(s) were used: ${matched.map((matcher2) => matcher2.pattern ?? matcher2.name ?? "*").join(", ")}. Actual tools: ${formatStepList(actualTools)}`
|
|
34783
|
+
);
|
|
34784
|
+
}
|
|
34785
|
+
return score(
|
|
34786
|
+
pass2,
|
|
34787
|
+
pass2 ? `Observed required tool(s): ${expectedTools.join(", ")}. Actual tools: ${formatStepList(actualTools)}` : `Missing required tool(s): ${missing.map((matcher2) => matcher2.pattern ?? matcher2.name ?? "*").join(", ")}. Actual tools: ${formatStepList(actualTools)}`
|
|
34788
|
+
);
|
|
34789
|
+
}
|
|
34790
|
+
const matcher = expected.matcher;
|
|
34791
|
+
const min = matcher.min ?? 1;
|
|
34792
|
+
const max = matcher.max;
|
|
34793
|
+
requireNamedMatcher(matcher, "trajectory:tool-used");
|
|
34794
|
+
const matches = steps.filter((step) => matchesTrajectoryStep(step, matcher));
|
|
34795
|
+
const count = matches.length;
|
|
34796
|
+
const basePass = count >= min && (max === void 0 || count <= max);
|
|
34797
|
+
const pass = inverse ? !basePass : basePass;
|
|
34798
|
+
const label = matcher.pattern ?? matcher.name ?? "*";
|
|
34799
|
+
if (inverse) {
|
|
34800
|
+
return score(
|
|
34801
|
+
pass,
|
|
34802
|
+
basePass ? `Tool "${label}" matched ${count} time(s), which violates the inverse assertion` : `Tool "${label}" did not satisfy the forbidden match condition`
|
|
34803
|
+
);
|
|
34804
|
+
}
|
|
34805
|
+
let reason = `Matched tool "${label}" ${count} time(s)`;
|
|
34806
|
+
reason += max === void 0 ? ` (expected at least ${min})` : ` (expected ${min}-${max})`;
|
|
34807
|
+
if (matches.length > 0) {
|
|
34808
|
+
reason += `. Matches: ${matches.map(formatStep).join(", ")}`;
|
|
34809
|
+
}
|
|
34810
|
+
return score(pass, reason);
|
|
34811
|
+
}
|
|
34812
|
+
function resolveSequenceValue(value) {
|
|
34813
|
+
if (Array.isArray(value)) {
|
|
34814
|
+
return { mode: "in_order", steps: value };
|
|
34815
|
+
}
|
|
34816
|
+
if (isRecord22(value)) {
|
|
34817
|
+
return {
|
|
34818
|
+
mode: value.mode === "exact" ? "exact" : "in_order",
|
|
34819
|
+
steps: Array.isArray(value.steps) ? value.steps : []
|
|
34820
|
+
};
|
|
34821
|
+
}
|
|
34822
|
+
throw new Error("trajectory:tool-sequence assertion must have an array or object value");
|
|
34823
|
+
}
|
|
34824
|
+
function evaluateToolSequence(config2, trace) {
|
|
34825
|
+
const toolSteps = extractTrajectorySteps(trace).filter((step) => step.type === "tool");
|
|
34826
|
+
const value = resolveSequenceValue(config2.value);
|
|
34827
|
+
const expected = value.steps.map((step, index) => {
|
|
34828
|
+
const matcher = normalizeMatcher(step, "tool");
|
|
34829
|
+
requireNamedMatcher(matcher, "trajectory:tool-sequence", index);
|
|
34830
|
+
return matcher;
|
|
34831
|
+
});
|
|
34832
|
+
if (expected.length === 0) {
|
|
34833
|
+
throw new Error("trajectory:tool-sequence assertion requires at least one expected step");
|
|
34834
|
+
}
|
|
34835
|
+
const actualTools = toolSteps.map(formatStep);
|
|
34836
|
+
let basePass = false;
|
|
34837
|
+
let reason = "";
|
|
34838
|
+
if (value.mode === "exact") {
|
|
34839
|
+
basePass = toolSteps.length === expected.length && expected.every((matcher, index) => matchesTrajectoryStep(toolSteps[index], matcher));
|
|
34840
|
+
reason = basePass ? `Observed exact tool sequence: ${formatStepList(actualTools)}` : `Expected exact tool sequence of ${expected.map((matcher) => matcher.pattern ?? matcher.name ?? "*").join(", ")}, but actual tools were ${formatStepList(actualTools)}`;
|
|
34841
|
+
} else {
|
|
34842
|
+
let expectedIndex = 0;
|
|
34843
|
+
const matchedSteps = [];
|
|
34844
|
+
for (const step of toolSteps) {
|
|
34845
|
+
if (expectedIndex >= expected.length) {
|
|
34846
|
+
break;
|
|
34847
|
+
}
|
|
34848
|
+
if (matchesTrajectoryStep(step, expected[expectedIndex])) {
|
|
34849
|
+
matchedSteps.push(formatStep(step));
|
|
34850
|
+
expectedIndex += 1;
|
|
34851
|
+
}
|
|
34852
|
+
}
|
|
34853
|
+
basePass = expectedIndex === expected.length;
|
|
34854
|
+
reason = basePass ? `Observed tool sequence in order: ${matchedSteps.join(", ")}. Actual tools: ${formatStepList(actualTools)}` : `Expected tool "${expected[expectedIndex]?.pattern ?? expected[expectedIndex]?.name ?? "*"}" was not observed in order. Actual tools: ${formatStepList(actualTools)}`;
|
|
34855
|
+
}
|
|
34856
|
+
const inverse = config2.inverse === true;
|
|
34857
|
+
return score(
|
|
34858
|
+
inverse ? !basePass : basePass,
|
|
34859
|
+
inverse ? basePass ? `Forbidden tool sequence was observed. Actual tools: ${formatStepList(actualTools)}` : "Forbidden tool sequence was not observed" : reason
|
|
34860
|
+
);
|
|
34861
|
+
}
|
|
34862
|
+
function matchesExpectedArgsPartial(actual, expected) {
|
|
34863
|
+
if (Array.isArray(expected)) {
|
|
34864
|
+
return Array.isArray(actual) && actual.length === expected.length && expected.every((item, index) => matchesExpectedArgsPartial(actual[index], item));
|
|
34865
|
+
}
|
|
34866
|
+
if (isRecord22(expected)) {
|
|
34867
|
+
if (!isRecord22(actual)) {
|
|
34868
|
+
return false;
|
|
34869
|
+
}
|
|
34870
|
+
return Object.entries(expected).every(
|
|
34871
|
+
([key, expectedValue]) => Object.hasOwn(actual, key) && matchesExpectedArgsPartial(actual[key], expectedValue)
|
|
34872
|
+
);
|
|
34873
|
+
}
|
|
34874
|
+
return isDeepStrictEqual(actual, expected);
|
|
34875
|
+
}
|
|
34876
|
+
function stripKeys(value, shouldStrip) {
|
|
34877
|
+
if (!isRecord22(value)) {
|
|
34878
|
+
return { cleaned: value, stripped: [] };
|
|
34879
|
+
}
|
|
34880
|
+
const cleaned = {};
|
|
34881
|
+
const stripped = [];
|
|
34882
|
+
for (const [key, entryValue] of Object.entries(value)) {
|
|
34883
|
+
if (shouldStrip(key, entryValue)) {
|
|
34884
|
+
stripped.push(key);
|
|
34885
|
+
continue;
|
|
34886
|
+
}
|
|
34887
|
+
Object.defineProperty(cleaned, key, {
|
|
34888
|
+
value: entryValue,
|
|
34889
|
+
enumerable: true,
|
|
34890
|
+
configurable: true,
|
|
34891
|
+
writable: true
|
|
34892
|
+
});
|
|
34893
|
+
}
|
|
34894
|
+
return { cleaned, stripped };
|
|
34895
|
+
}
|
|
34896
|
+
function stripDefaults(actual, defaults) {
|
|
34897
|
+
if (!defaults) {
|
|
34898
|
+
return { cleaned: actual, stripped: [] };
|
|
34899
|
+
}
|
|
34900
|
+
return stripKeys(
|
|
34901
|
+
actual,
|
|
34902
|
+
(key, value) => Object.hasOwn(defaults, key) && isDeepStrictEqual(value, defaults[key])
|
|
34903
|
+
);
|
|
34904
|
+
}
|
|
34905
|
+
function stripIgnored(value, ignore) {
|
|
34906
|
+
return stripKeys(
|
|
34907
|
+
value,
|
|
34908
|
+
(key) => ignore.some((entry) => /[*?]/.test(entry) ? matchesPattern2(key, entry) : entry === key)
|
|
34909
|
+
);
|
|
34910
|
+
}
|
|
34911
|
+
function resolveArgsMatchValue(value) {
|
|
34912
|
+
if (!isRecord22(value)) {
|
|
34913
|
+
throw new Error("trajectory:tool-args-match assertion must have an object value");
|
|
34914
|
+
}
|
|
34915
|
+
const matcher = normalizeMatcher(value, "tool");
|
|
34916
|
+
requireNamedMatcher(matcher, "trajectory:tool-args-match");
|
|
34917
|
+
const expectedArgs = Object.hasOwn(value, "args") ? value.args : value.arguments;
|
|
34918
|
+
if (expectedArgs === void 0) {
|
|
34919
|
+
throw new Error(
|
|
34920
|
+
"trajectory:tool-args-match assertion must include an args or arguments property"
|
|
34921
|
+
);
|
|
34922
|
+
}
|
|
34923
|
+
const mode = value.mode ?? "partial";
|
|
34924
|
+
if (mode !== "partial" && mode !== "exact") {
|
|
34925
|
+
throw new Error('trajectory:tool-args-match assertion mode must be "partial" or "exact"');
|
|
34926
|
+
}
|
|
34927
|
+
const defaults = value.defaults;
|
|
34928
|
+
if (defaults !== void 0 && !isRecord22(defaults)) {
|
|
34929
|
+
throw new Error(
|
|
34930
|
+
"trajectory:tool-args-match assertion defaults must be an object mapping argument names to default values"
|
|
34931
|
+
);
|
|
34932
|
+
}
|
|
34933
|
+
const rawIgnore = value.ignore;
|
|
34934
|
+
const ignore = rawIgnore === void 0 ? [] : Array.isArray(rawIgnore) ? rawIgnore : [rawIgnore];
|
|
34935
|
+
if (ignore.some((entry) => typeof entry !== "string" || entry.trim().length === 0)) {
|
|
34936
|
+
throw new Error(
|
|
34937
|
+
"trajectory:tool-args-match assertion ignore must be a non-empty string or an array of non-empty strings"
|
|
34938
|
+
);
|
|
34939
|
+
}
|
|
34940
|
+
return {
|
|
34941
|
+
matcher,
|
|
34942
|
+
expectedArgs,
|
|
34943
|
+
mode,
|
|
34944
|
+
defaults,
|
|
34945
|
+
ignore
|
|
34946
|
+
};
|
|
34947
|
+
}
|
|
34948
|
+
function argsMatch2(actual, expected, mode, defaults, ignore) {
|
|
34949
|
+
const cleanedActual = stripDefaults(stripIgnored(actual, ignore).cleaned, defaults).cleaned;
|
|
34950
|
+
const cleanedExpected = stripIgnored(expected, ignore).cleaned;
|
|
34951
|
+
return mode === "exact" ? isDeepStrictEqual(cleanedActual, cleanedExpected) : matchesExpectedArgsPartial(cleanedActual, cleanedExpected);
|
|
34952
|
+
}
|
|
34953
|
+
function evaluateToolArgsMatch(config2, trace) {
|
|
34954
|
+
const toolSteps = extractTrajectorySteps(trace).filter((step) => step.type === "tool");
|
|
34955
|
+
const { matcher, expectedArgs, mode, defaults, ignore } = resolveArgsMatchValue(config2.value);
|
|
34956
|
+
const label = matcher.pattern ?? matcher.name ?? "*";
|
|
34957
|
+
const actualTools = toolSteps.map(formatStep);
|
|
34958
|
+
const matchingSteps = toolSteps.filter((step) => matchesTrajectoryStep(step, matcher));
|
|
34959
|
+
const stepsWithArgs = matchingSteps.filter((step) => step.args !== void 0);
|
|
34960
|
+
const matchedStep = stepsWithArgs.find(
|
|
34961
|
+
(step) => argsMatch2(step.args, expectedArgs, mode, defaults, ignore)
|
|
34962
|
+
);
|
|
34963
|
+
const basePass = matchedStep !== void 0;
|
|
34964
|
+
const inverse = config2.inverse === true;
|
|
34965
|
+
const pass = inverse ? !basePass : basePass;
|
|
34966
|
+
const observedArgs = stepsWithArgs.length > 0 ? stepsWithArgs.map((step) => formatArgs(step.args)).join(", ") : "(none)";
|
|
34967
|
+
if (inverse) {
|
|
34968
|
+
if (basePass) {
|
|
34969
|
+
return score(
|
|
34970
|
+
pass,
|
|
34971
|
+
`Forbidden argument match for tool "${label}" was observed on ${formatStep(
|
|
34972
|
+
matchedStep
|
|
34973
|
+
)}. Args: ${formatArgs(matchedStep?.args)}`
|
|
34974
|
+
);
|
|
34975
|
+
}
|
|
34976
|
+
return score(
|
|
34977
|
+
pass,
|
|
34978
|
+
matchingSteps.length === 0 ? `Forbidden argument match for tool "${label}" was not observed because no tool call matched it` : `Forbidden argument match for tool "${label}" was not observed. Observed args: ${observedArgs}`
|
|
34979
|
+
);
|
|
34980
|
+
}
|
|
34981
|
+
if (basePass) {
|
|
34982
|
+
const ignoredArgs = stripIgnored(matchedStep.args, ignore).stripped;
|
|
34983
|
+
const ignoredDefaults = stripDefaults(
|
|
34984
|
+
stripIgnored(matchedStep.args, ignore).cleaned,
|
|
34985
|
+
defaults
|
|
34986
|
+
).stripped;
|
|
34987
|
+
const ignoredArgsSuffix = ignoredArgs.length > 0 ? `. Ignored argument(s): ${ignoredArgs.join(", ")}` : "";
|
|
34988
|
+
const ignoredDefaultsSuffix = ignoredDefaults.length > 0 ? `. Ignored default argument(s): ${ignoredDefaults.join(", ")}` : "";
|
|
34989
|
+
return score(
|
|
34990
|
+
true,
|
|
34991
|
+
`Tool "${label}" matched expected arguments (${mode}) on ${formatStep(
|
|
34992
|
+
matchedStep
|
|
34993
|
+
)}. Args: ${formatArgs(matchedStep.args)}${ignoredArgsSuffix}${ignoredDefaultsSuffix}`
|
|
34994
|
+
);
|
|
34995
|
+
}
|
|
34996
|
+
if (matchingSteps.length === 0) {
|
|
34997
|
+
return score(
|
|
34998
|
+
false,
|
|
34999
|
+
`No tool call matched "${label}". Actual tools: ${formatStepList(actualTools)}`
|
|
35000
|
+
);
|
|
35001
|
+
}
|
|
35002
|
+
if (stepsWithArgs.length === 0) {
|
|
35003
|
+
return score(
|
|
35004
|
+
false,
|
|
35005
|
+
`Tool "${label}" was observed but no arguments were captured. Actual tools: ${formatStepList(
|
|
35006
|
+
actualTools
|
|
35007
|
+
)}`
|
|
35008
|
+
);
|
|
35009
|
+
}
|
|
35010
|
+
return score(
|
|
35011
|
+
false,
|
|
35012
|
+
`No call to tool "${label}" matched expected arguments (${mode}): ${formatArgs(
|
|
35013
|
+
expectedArgs
|
|
35014
|
+
)}. Observed args: ${observedArgs}`
|
|
35015
|
+
);
|
|
35016
|
+
}
|
|
35017
|
+
function resolveStepCountValue(value) {
|
|
35018
|
+
if (!isRecord22(value)) {
|
|
35019
|
+
throw new Error("trajectory:step-count assertion must have an object value");
|
|
35020
|
+
}
|
|
35021
|
+
const matcher = normalizeMatcher(value);
|
|
35022
|
+
return {
|
|
35023
|
+
...matcher,
|
|
35024
|
+
...typeof value.min === "number" ? { min: value.min } : {},
|
|
35025
|
+
...typeof value.max === "number" ? { max: value.max } : {}
|
|
35026
|
+
};
|
|
35027
|
+
}
|
|
35028
|
+
function evaluateStepCount(config2, trace) {
|
|
35029
|
+
const matcher = resolveStepCountValue(config2.value);
|
|
35030
|
+
if (matcher.min === void 0 && matcher.max === void 0) {
|
|
35031
|
+
throw new Error("trajectory:step-count assertion must include a min or max property");
|
|
35032
|
+
}
|
|
35033
|
+
const matchingSteps = extractTrajectorySteps(trace).filter(
|
|
35034
|
+
(step) => matchesTrajectoryStep(step, matcher)
|
|
35035
|
+
);
|
|
35036
|
+
const count = matchingSteps.length;
|
|
35037
|
+
const basePass = (matcher.min === void 0 || count >= matcher.min) && (matcher.max === void 0 || count <= matcher.max);
|
|
35038
|
+
const inverse = config2.inverse === true;
|
|
35039
|
+
const filterParts = [];
|
|
35040
|
+
if (matcher.type) {
|
|
35041
|
+
const types = Array.isArray(matcher.type) ? matcher.type : [matcher.type];
|
|
35042
|
+
filterParts.push(`type=${types.join("|")}`);
|
|
35043
|
+
}
|
|
35044
|
+
const pattern = matcher.pattern ?? matcher.name;
|
|
35045
|
+
if (pattern) {
|
|
35046
|
+
filterParts.push(`pattern=${pattern}`);
|
|
35047
|
+
}
|
|
35048
|
+
let reason = `Matched ${count} trajectory step(s)`;
|
|
35049
|
+
if (filterParts.length > 0) {
|
|
35050
|
+
reason += ` for ${filterParts.join(", ")}`;
|
|
35051
|
+
}
|
|
35052
|
+
if (matcher.min !== void 0 && matcher.max !== void 0) {
|
|
35053
|
+
reason += ` (expected ${matcher.min}-${matcher.max})`;
|
|
35054
|
+
} else if (matcher.min !== void 0) {
|
|
35055
|
+
reason += ` (expected at least ${matcher.min})`;
|
|
35056
|
+
} else {
|
|
35057
|
+
reason += ` (expected at most ${matcher.max})`;
|
|
35058
|
+
}
|
|
35059
|
+
if (matchingSteps.length > 0) {
|
|
35060
|
+
reason += `. Matches: ${matchingSteps.map(formatStep).join(", ")}`;
|
|
35061
|
+
}
|
|
35062
|
+
return score(
|
|
35063
|
+
inverse ? !basePass : basePass,
|
|
35064
|
+
inverse ? basePass ? "Trajectory step count satisfied the forbidden range" : "Trajectory step count did not satisfy the forbidden range" : reason
|
|
35065
|
+
);
|
|
35066
|
+
}
|
|
35067
|
+
function resolveGoal(value) {
|
|
35068
|
+
if (typeof value === "string" && value.trim()) {
|
|
35069
|
+
return value.trim();
|
|
35070
|
+
}
|
|
35071
|
+
if (isRecord22(value) && typeof value.goal === "string" && value.goal.trim()) {
|
|
35072
|
+
return value.goal.trim();
|
|
35073
|
+
}
|
|
35074
|
+
throw new Error(
|
|
35075
|
+
"trajectory:goal-success assertion must have a string value or an object with a goal property"
|
|
35076
|
+
);
|
|
35077
|
+
}
|
|
35078
|
+
function summarizeTrajectoryForJudge(trace) {
|
|
35079
|
+
const steps = extractTrajectorySteps(trace).map((step, index) => ({
|
|
35080
|
+
index: index + 1,
|
|
35081
|
+
type: step.type,
|
|
35082
|
+
name: step.name,
|
|
35083
|
+
...step.spanName === step.name ? {} : { spanName: step.spanName }
|
|
35084
|
+
}));
|
|
35085
|
+
return JSON.stringify({ stepCount: steps.length, steps }, null, 2);
|
|
35086
|
+
}
|
|
35087
|
+
function buildGoalCriteria(goal, trajectory, output) {
|
|
35088
|
+
return [
|
|
35089
|
+
"Determine whether the agent successfully achieved the goal using the observed trajectory and final output.",
|
|
35090
|
+
"",
|
|
35091
|
+
`Goal: ${goal}`,
|
|
35092
|
+
"",
|
|
35093
|
+
"Trajectory:",
|
|
35094
|
+
trajectory,
|
|
35095
|
+
"",
|
|
35096
|
+
"Final output:",
|
|
35097
|
+
output
|
|
35098
|
+
].join("\n");
|
|
35099
|
+
}
|
|
35100
|
+
var TrajectoryGrader = class {
|
|
35101
|
+
kind;
|
|
35102
|
+
config;
|
|
35103
|
+
llmGrader;
|
|
35104
|
+
constructor(options) {
|
|
35105
|
+
this.config = options.config;
|
|
35106
|
+
this.kind = options.config.type;
|
|
35107
|
+
this.llmGrader = options.llmGrader;
|
|
35108
|
+
}
|
|
35109
|
+
async evaluate(context) {
|
|
35110
|
+
const trace = context.trace;
|
|
35111
|
+
if (!trace) {
|
|
35112
|
+
return missingTrace(this.config.type);
|
|
35113
|
+
}
|
|
35114
|
+
switch (this.config.type) {
|
|
35115
|
+
case "trajectory:tool-used":
|
|
35116
|
+
return evaluateToolUsed(this.config, trace);
|
|
35117
|
+
case "trajectory:tool-args-match":
|
|
35118
|
+
return evaluateToolArgsMatch(this.config, trace);
|
|
35119
|
+
case "trajectory:tool-sequence":
|
|
35120
|
+
return evaluateToolSequence(this.config, trace);
|
|
35121
|
+
case "trajectory:step-count":
|
|
35122
|
+
return evaluateStepCount(this.config, trace);
|
|
35123
|
+
case "trajectory:goal-success": {
|
|
35124
|
+
if (!this.llmGrader) {
|
|
35125
|
+
throw new Error("trajectory:goal-success assertion requires an LLM grader");
|
|
35126
|
+
}
|
|
35127
|
+
const goal = resolveGoal(this.config.value);
|
|
35128
|
+
const result = await this.llmGrader.evaluate({
|
|
35129
|
+
...context,
|
|
35130
|
+
evalCase: {
|
|
35131
|
+
...context.evalCase,
|
|
35132
|
+
criteria: buildGoalCriteria(
|
|
35133
|
+
goal,
|
|
35134
|
+
summarizeTrajectoryForJudge(trace),
|
|
35135
|
+
context.candidate
|
|
35136
|
+
)
|
|
35137
|
+
},
|
|
35138
|
+
evaluator: this.config
|
|
35139
|
+
});
|
|
35140
|
+
if (this.config.inverse !== true) {
|
|
35141
|
+
return result;
|
|
35142
|
+
}
|
|
35143
|
+
const negated = negateScore(result);
|
|
35144
|
+
const assertion = {
|
|
35145
|
+
text: result.score >= 0.5 ? `Agent unexpectedly achieved the goal: ${goal}` : `Agent did not achieve the forbidden goal: ${goal}`,
|
|
35146
|
+
passed: negated.score >= 0.5
|
|
35147
|
+
};
|
|
35148
|
+
return { ...negated, assertions: [assertion] };
|
|
35149
|
+
}
|
|
35150
|
+
}
|
|
35151
|
+
}
|
|
35152
|
+
};
|
|
34248
35153
|
function runContainsAssertion(output, value) {
|
|
34249
35154
|
const passed = output.includes(value);
|
|
34250
35155
|
return {
|
|
@@ -34672,6 +35577,7 @@ var ClaudeCliProvider = class {
|
|
|
34672
35577
|
args,
|
|
34673
35578
|
exitCode: result.exitCode
|
|
34674
35579
|
},
|
|
35580
|
+
metadata: skillCallMetadata(deriveSkillCallsFromToolCalls(completedToolCalls)),
|
|
34675
35581
|
output,
|
|
34676
35582
|
tokenUsage,
|
|
34677
35583
|
costUsd,
|
|
@@ -36640,6 +37546,7 @@ ${basePrompt}` : basePrompt;
|
|
|
36640
37546
|
inputFiles,
|
|
36641
37547
|
logFile: logger?.filePath
|
|
36642
37548
|
},
|
|
37549
|
+
metadata: skillCallMetadata(deriveSkillCallsFromMessages(messages)),
|
|
36643
37550
|
output: messages,
|
|
36644
37551
|
durationMs,
|
|
36645
37552
|
targetExecution: {
|
|
@@ -40308,7 +41215,7 @@ function toWireRecord(record2) {
|
|
|
40308
41215
|
start_time: record2.startTime,
|
|
40309
41216
|
end_time: record2.endTime
|
|
40310
41217
|
};
|
|
40311
|
-
const parsed = ReplayFixtureWireSchema.parse(
|
|
41218
|
+
const parsed = ReplayFixtureWireSchema.parse(dropUndefined2(wire));
|
|
40312
41219
|
return parsed;
|
|
40313
41220
|
}
|
|
40314
41221
|
function toWireMessage(message) {
|
|
@@ -40336,7 +41243,7 @@ function toWireToolCall(toolCall) {
|
|
|
40336
41243
|
duration_ms: toolCall.durationMs
|
|
40337
41244
|
};
|
|
40338
41245
|
}
|
|
40339
|
-
function
|
|
41246
|
+
function dropUndefined2(value) {
|
|
40340
41247
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
40341
41248
|
}
|
|
40342
41249
|
function formatZodError(error40) {
|
|
@@ -40452,13 +41359,14 @@ function formatReplayLookupKey(lookup) {
|
|
|
40452
41359
|
function replayFixtureRecordToProviderResponse(record2) {
|
|
40453
41360
|
return {
|
|
40454
41361
|
output: record2.output,
|
|
41362
|
+
metadata: skillCallMetadata(deriveSkillCallsFromMessages(record2.output)),
|
|
40455
41363
|
tokenUsage: record2.tokenUsage,
|
|
40456
41364
|
costUsd: record2.costUsd,
|
|
40457
41365
|
durationMs: record2.durationMs,
|
|
40458
41366
|
startTime: record2.startTime,
|
|
40459
41367
|
endTime: record2.endTime,
|
|
40460
41368
|
raw: {
|
|
40461
|
-
replay_fixture:
|
|
41369
|
+
replay_fixture: dropUndefined2({
|
|
40462
41370
|
fixture_id: record2.fixtureId,
|
|
40463
41371
|
suite: record2.suite,
|
|
40464
41372
|
eval_path: record2.evalPath,
|
|
@@ -40531,7 +41439,7 @@ function buildFixtureId(input) {
|
|
|
40531
41439
|
return `${input.sourceTarget}-${input.testId}-${digest}`;
|
|
40532
41440
|
}
|
|
40533
41441
|
function buildSourceMetadata(target, sourceTarget) {
|
|
40534
|
-
return
|
|
41442
|
+
return dropUndefined2({
|
|
40535
41443
|
provider: target.kind,
|
|
40536
41444
|
target_name: sourceTarget,
|
|
40537
41445
|
resolved_target: target.name,
|
|
@@ -40596,7 +41504,7 @@ var EXTERNAL_TRACE_KEYS = [
|
|
|
40596
41504
|
"target"
|
|
40597
41505
|
];
|
|
40598
41506
|
var TRACEPARENT_RE = /^[\da-f]{2}-[\da-f]{32}-[\da-f]{16}-[\da-f]{2}$/i;
|
|
40599
|
-
function
|
|
41507
|
+
function isRecord3(value) {
|
|
40600
41508
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
40601
41509
|
}
|
|
40602
41510
|
function stringValue(value) {
|
|
@@ -40741,7 +41649,7 @@ function omitExternalTraceMetadataKeys(value) {
|
|
|
40741
41649
|
return entries.length > 0 ? Object.fromEntries(entries) : void 0;
|
|
40742
41650
|
}
|
|
40743
41651
|
function sanitizeExternalTraceMetadata(value) {
|
|
40744
|
-
if (!
|
|
41652
|
+
if (!isRecord3(value)) {
|
|
40745
41653
|
return void 0;
|
|
40746
41654
|
}
|
|
40747
41655
|
return sanitizeExternalTraceObject(externalTraceInputFromRecord(value));
|
|
@@ -41095,12 +42003,12 @@ var ProjectionIdentityWireSchema = external_exports.object({
|
|
|
41095
42003
|
dimensions: ProjectionIdentityDimensionsWireSchema,
|
|
41096
42004
|
issues: external_exports.array(ProjectionIdentityIssueWireSchema).optional()
|
|
41097
42005
|
}).strict();
|
|
41098
|
-
function
|
|
42006
|
+
function dropUndefined22(value) {
|
|
41099
42007
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
41100
42008
|
}
|
|
41101
42009
|
function toProjectionIdentityIssueWire(issue2) {
|
|
41102
42010
|
return ProjectionIdentityIssueWireSchema.parse(
|
|
41103
|
-
|
|
42011
|
+
dropUndefined22({
|
|
41104
42012
|
code: issue2.code,
|
|
41105
42013
|
severity: issue2.severity,
|
|
41106
42014
|
field: issue2.field,
|
|
@@ -41118,7 +42026,7 @@ function fromProjectionIdentityIssueWire(issue2) {
|
|
|
41118
42026
|
}
|
|
41119
42027
|
function toProjectionIdentityWire(identity) {
|
|
41120
42028
|
return ProjectionIdentityWireSchema.parse(
|
|
41121
|
-
|
|
42029
|
+
dropUndefined22({
|
|
41122
42030
|
schema_version: identity.schemaVersion,
|
|
41123
42031
|
id: identity.id,
|
|
41124
42032
|
key: identity.key,
|
|
@@ -41292,7 +42200,7 @@ var TraceEnvelopeWireSchema = external_exports.object({
|
|
|
41292
42200
|
function dropUndefined3(value) {
|
|
41293
42201
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
41294
42202
|
}
|
|
41295
|
-
function
|
|
42203
|
+
function isRecord4(value) {
|
|
41296
42204
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
41297
42205
|
}
|
|
41298
42206
|
function definedStringRecord(value) {
|
|
@@ -41503,25 +42411,25 @@ function scoresFromResult(result, targetSpanId) {
|
|
|
41503
42411
|
if (!result.scores || result.scores.length === 0) {
|
|
41504
42412
|
return void 0;
|
|
41505
42413
|
}
|
|
41506
|
-
return result.scores.map((
|
|
41507
|
-
name:
|
|
41508
|
-
type:
|
|
41509
|
-
score:
|
|
41510
|
-
weight:
|
|
41511
|
-
verdict:
|
|
41512
|
-
source: scoreSource(
|
|
41513
|
-
evaluatedAt:
|
|
42414
|
+
return result.scores.map((score2) => ({
|
|
42415
|
+
name: score2.name,
|
|
42416
|
+
type: score2.type,
|
|
42417
|
+
score: score2.score,
|
|
42418
|
+
weight: score2.weight,
|
|
42419
|
+
verdict: score2.verdict,
|
|
42420
|
+
source: scoreSource(score2.type),
|
|
42421
|
+
evaluatedAt: score2.endedAt ?? score2.startedAt ?? result.timestamp,
|
|
41514
42422
|
targetSpanId,
|
|
41515
42423
|
evidence: dropUndefined3({
|
|
41516
42424
|
span_ids: [targetSpanId],
|
|
41517
|
-
assertions: (
|
|
42425
|
+
assertions: (score2.assertions ?? []).map(
|
|
41518
42426
|
(assertion) => dropUndefined3({
|
|
41519
42427
|
text: assertion.text,
|
|
41520
42428
|
passed: assertion.passed,
|
|
41521
42429
|
evidence: assertion.evidence
|
|
41522
42430
|
})
|
|
41523
42431
|
),
|
|
41524
|
-
details:
|
|
42432
|
+
details: score2.details
|
|
41525
42433
|
})
|
|
41526
42434
|
}));
|
|
41527
42435
|
}
|
|
@@ -41990,32 +42898,32 @@ function fromTraceEnvelopeConversionWarningWire(warning) {
|
|
|
41990
42898
|
details: warning.details
|
|
41991
42899
|
};
|
|
41992
42900
|
}
|
|
41993
|
-
function toTraceEnvelopeScoreWire(
|
|
42901
|
+
function toTraceEnvelopeScoreWire(score2) {
|
|
41994
42902
|
return TraceEnvelopeScoreWireSchema.parse(
|
|
41995
42903
|
dropUndefined3({
|
|
41996
|
-
name:
|
|
41997
|
-
type:
|
|
41998
|
-
score:
|
|
41999
|
-
weight:
|
|
42000
|
-
verdict:
|
|
42001
|
-
source:
|
|
42002
|
-
evaluated_at:
|
|
42003
|
-
target_span_id:
|
|
42004
|
-
evidence:
|
|
42904
|
+
name: score2.name,
|
|
42905
|
+
type: score2.type,
|
|
42906
|
+
score: score2.score,
|
|
42907
|
+
weight: score2.weight,
|
|
42908
|
+
verdict: score2.verdict,
|
|
42909
|
+
source: score2.source,
|
|
42910
|
+
evaluated_at: score2.evaluatedAt,
|
|
42911
|
+
target_span_id: score2.targetSpanId,
|
|
42912
|
+
evidence: score2.evidence
|
|
42005
42913
|
})
|
|
42006
42914
|
);
|
|
42007
42915
|
}
|
|
42008
|
-
function fromTraceEnvelopeScoreWire(
|
|
42916
|
+
function fromTraceEnvelopeScoreWire(score2) {
|
|
42009
42917
|
return {
|
|
42010
|
-
name:
|
|
42011
|
-
type:
|
|
42012
|
-
score:
|
|
42013
|
-
weight:
|
|
42014
|
-
verdict:
|
|
42015
|
-
source:
|
|
42016
|
-
evaluatedAt:
|
|
42017
|
-
targetSpanId:
|
|
42018
|
-
evidence:
|
|
42918
|
+
name: score2.name,
|
|
42919
|
+
type: score2.type,
|
|
42920
|
+
score: score2.score,
|
|
42921
|
+
weight: score2.weight,
|
|
42922
|
+
verdict: score2.verdict,
|
|
42923
|
+
source: score2.source,
|
|
42924
|
+
evaluatedAt: score2.evaluated_at,
|
|
42925
|
+
targetSpanId: score2.target_span_id,
|
|
42926
|
+
evidence: score2.evidence
|
|
42019
42927
|
};
|
|
42020
42928
|
}
|
|
42021
42929
|
function isToolSpan(span) {
|
|
@@ -42082,7 +42990,7 @@ function nearestAncestorToolCallId(ancestorIds, spansById) {
|
|
|
42082
42990
|
return void 0;
|
|
42083
42991
|
}
|
|
42084
42992
|
function fromTranscriptToolCallWire(wire) {
|
|
42085
|
-
if (!
|
|
42993
|
+
if (!isRecord4(wire) || typeof wire.tool !== "string") {
|
|
42086
42994
|
return void 0;
|
|
42087
42995
|
}
|
|
42088
42996
|
return {
|
|
@@ -42097,7 +43005,7 @@ function fromTranscriptToolCallWire(wire) {
|
|
|
42097
43005
|
};
|
|
42098
43006
|
}
|
|
42099
43007
|
function fromTranscriptMessageWire(wire) {
|
|
42100
|
-
if (!
|
|
43008
|
+
if (!isRecord4(wire) || typeof wire.role !== "string") {
|
|
42101
43009
|
return void 0;
|
|
42102
43010
|
}
|
|
42103
43011
|
const toolCalls = Array.isArray(wire.tool_calls) ? wire.tool_calls.map(fromTranscriptToolCallWire).filter((toolCall) => toolCall !== void 0) : void 0;
|
|
@@ -42109,8 +43017,8 @@ function fromTranscriptMessageWire(wire) {
|
|
|
42109
43017
|
startTime: typeof wire.start_time === "string" ? wire.start_time : void 0,
|
|
42110
43018
|
endTime: typeof wire.end_time === "string" ? wire.end_time : void 0,
|
|
42111
43019
|
durationMs: numberAttribute(wire, "duration_ms"),
|
|
42112
|
-
metadata:
|
|
42113
|
-
tokenUsage:
|
|
43020
|
+
metadata: isRecord4(wire.metadata) ? wire.metadata : void 0,
|
|
43021
|
+
tokenUsage: isRecord4(wire.token_usage) ? tokenUsageFromAttributes({
|
|
42114
43022
|
"gen_ai.usage.input_tokens": wire.token_usage.input,
|
|
42115
43023
|
"gen_ai.usage.output_tokens": wire.token_usage.output,
|
|
42116
43024
|
"gen_ai.usage.cache_read.input_tokens": wire.token_usage.cached,
|
|
@@ -42466,6 +43374,7 @@ function traceEnvelopeReplayRecordToProviderResponse(record2) {
|
|
|
42466
43374
|
const wire = toTraceEnvelopeWire(record2.envelope);
|
|
42467
43375
|
return {
|
|
42468
43376
|
output,
|
|
43377
|
+
metadata: skillCallMetadata(deriveSkillCallsFromMessages(output)),
|
|
42469
43378
|
tokenUsage: summary.tokenUsage,
|
|
42470
43379
|
costUsd: summary.costUsd,
|
|
42471
43380
|
durationMs: summary.durationMs,
|
|
@@ -42581,9 +43490,10 @@ var CANONICAL_TRANSCRIPT_ARTIFACT_PATH = "transcript.json";
|
|
|
42581
43490
|
var CANONICAL_METRICS_ARTIFACT_PATH = "metrics.json";
|
|
42582
43491
|
var CANONICAL_FILE_CHANGES_ARTIFACT_PATH = "outputs/file_changes.diff";
|
|
42583
43492
|
var TRANSCRIPT_SCHEMA_VERSION = "agentv.normalized_transcript.v1";
|
|
42584
|
-
var
|
|
43493
|
+
var LEGACY_METRICS_SCHEMA_VERSION = "agentv.metrics.v1";
|
|
43494
|
+
var METRICS_SCHEMA_VERSION = "agentv.metrics.v2";
|
|
42585
43495
|
var TRANSCRIPT_JSON_MEDIA_TYPE = "application/vnd.agentv.normalized-transcript.v1+json";
|
|
42586
|
-
var METRICS_JSON_MEDIA_TYPE = "application/vnd.agentv.metrics.
|
|
43496
|
+
var METRICS_JSON_MEDIA_TYPE = "application/vnd.agentv.metrics.v2+json";
|
|
42587
43497
|
function toResultArtifactPointerWire(pointer) {
|
|
42588
43498
|
return {
|
|
42589
43499
|
ref: pointer.ref,
|
|
@@ -42668,7 +43578,7 @@ var COMMAND_KEYS = /* @__PURE__ */ new Set([
|
|
|
42668
43578
|
"_extractedcommand"
|
|
42669
43579
|
]);
|
|
42670
43580
|
var URL_KEYS = /* @__PURE__ */ new Set(["url", "uri", "href", "extractedurl", "_extractedurl"]);
|
|
42671
|
-
function
|
|
43581
|
+
function isRecord5(value) {
|
|
42672
43582
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
42673
43583
|
}
|
|
42674
43584
|
function normalizedKey(key) {
|
|
@@ -42747,7 +43657,7 @@ function stringValuesByKey(value, keys, maxDepth = 6) {
|
|
|
42747
43657
|
}
|
|
42748
43658
|
return;
|
|
42749
43659
|
}
|
|
42750
|
-
if (!
|
|
43660
|
+
if (!isRecord5(entry)) {
|
|
42751
43661
|
return;
|
|
42752
43662
|
}
|
|
42753
43663
|
for (const [key, nested] of Object.entries(entry)) {
|
|
@@ -42799,7 +43709,7 @@ function collectThinkingBlocks(messages) {
|
|
|
42799
43709
|
}
|
|
42800
43710
|
for (const block of content) {
|
|
42801
43711
|
const entry = block;
|
|
42802
|
-
if (!
|
|
43712
|
+
if (!isRecord5(entry)) {
|
|
42803
43713
|
continue;
|
|
42804
43714
|
}
|
|
42805
43715
|
if (entry.type === "thinking" || entry.type === "reasoning") {
|
|
@@ -42814,7 +43724,7 @@ function errorFromToolCall(toolCall, toolName) {
|
|
|
42814
43724
|
return void 0;
|
|
42815
43725
|
}
|
|
42816
43726
|
const output = toolCall.output;
|
|
42817
|
-
const message = typeof output === "string" ? output :
|
|
43727
|
+
const message = typeof output === "string" ? output : isRecord5(output) && typeof output.message === "string" ? output.message : `Tool ${toolCall.tool} ${toolCall.status}`;
|
|
42818
43728
|
return {
|
|
42819
43729
|
message,
|
|
42820
43730
|
...toolCall.id ? { tool_call_id: toolCall.id } : {},
|
|
@@ -42877,7 +43787,7 @@ var TRANSCRIPT_ROW_SCHEMA_VERSION = "agentv.transcript.v1";
|
|
|
42877
43787
|
function dropUndefined5(value) {
|
|
42878
43788
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
42879
43789
|
}
|
|
42880
|
-
function
|
|
43790
|
+
function isRecord6(value) {
|
|
42881
43791
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
42882
43792
|
}
|
|
42883
43793
|
function optionalString2(value) {
|
|
@@ -43110,7 +44020,7 @@ function normalizedContentBlocks(message, messageIndex, providerId) {
|
|
|
43110
44020
|
} else if (Array.isArray(content)) {
|
|
43111
44021
|
for (const contentBlock of content) {
|
|
43112
44022
|
const block = contentBlock;
|
|
43113
|
-
if (!
|
|
44023
|
+
if (!isRecord6(block) || typeof block.type !== "string") {
|
|
43114
44024
|
continue;
|
|
43115
44025
|
}
|
|
43116
44026
|
if (block.type === "text" && typeof block.text === "string") {
|
|
@@ -43303,7 +44213,7 @@ function traceToTranscriptJsonLines(trace, options) {
|
|
|
43303
44213
|
provider,
|
|
43304
44214
|
sessionId,
|
|
43305
44215
|
startedAt: trace.startTime,
|
|
43306
|
-
metadata:
|
|
44216
|
+
metadata: isRecord6(trace.metadata) ? trace.metadata : void 0
|
|
43307
44217
|
},
|
|
43308
44218
|
tokenUsage: trace.tokenUsage,
|
|
43309
44219
|
durationMs: trace.durationMs,
|
|
@@ -43349,7 +44259,7 @@ function readOptionalNumber(record2, key) {
|
|
|
43349
44259
|
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
43350
44260
|
}
|
|
43351
44261
|
function fromTranscriptToolCall(wire) {
|
|
43352
|
-
if (!
|
|
44262
|
+
if (!isRecord6(wire)) {
|
|
43353
44263
|
return void 0;
|
|
43354
44264
|
}
|
|
43355
44265
|
const tool = readOptionalString3(wire, "tool");
|
|
@@ -43457,6 +44367,7 @@ function transcriptReplayRecordToProviderResponse(record2) {
|
|
|
43457
44367
|
const entry = record2.entry;
|
|
43458
44368
|
return {
|
|
43459
44369
|
output: entry.messages,
|
|
44370
|
+
metadata: skillCallMetadata(deriveSkillCallsFromMessages(entry.messages)),
|
|
43460
44371
|
tokenUsage: entry.tokenUsage,
|
|
43461
44372
|
durationMs: entry.durationMs,
|
|
43462
44373
|
costUsd: entry.costUsd ?? void 0,
|
|
@@ -45189,7 +46100,7 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
|
|
|
45189
46100
|
};
|
|
45190
46101
|
}
|
|
45191
46102
|
}
|
|
45192
|
-
function
|
|
46103
|
+
function isRecord7(value) {
|
|
45193
46104
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
45194
46105
|
}
|
|
45195
46106
|
function extractTargetsArray(parsed, absolutePath) {
|
|
@@ -45200,7 +46111,7 @@ function extractTargetsArray(parsed, absolutePath) {
|
|
|
45200
46111
|
return targets;
|
|
45201
46112
|
}
|
|
45202
46113
|
function assertTargetDefinition(value, index, filePath) {
|
|
45203
|
-
if (!
|
|
46114
|
+
if (!isRecord7(value)) {
|
|
45204
46115
|
throw new Error(`targets.yaml entry at index ${index} in ${filePath} must be an object`);
|
|
45205
46116
|
}
|
|
45206
46117
|
const id = value.id;
|
|
@@ -45241,7 +46152,7 @@ async function readTargetDefinitions(filePath) {
|
|
|
45241
46152
|
}
|
|
45242
46153
|
const raw = await readFile8(absolutePath, "utf8");
|
|
45243
46154
|
const parsed = parseYamlValue(raw);
|
|
45244
|
-
if (!
|
|
46155
|
+
if (!isRecord7(parsed)) {
|
|
45245
46156
|
throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`);
|
|
45246
46157
|
}
|
|
45247
46158
|
const targets = extractTargetsArray(parsed, absolutePath);
|
|
@@ -45522,11 +46433,11 @@ var InlineAssertGrader = class {
|
|
|
45522
46433
|
criteria: context.evalCase.criteria,
|
|
45523
46434
|
metadata: context.evalCase.metadata
|
|
45524
46435
|
});
|
|
45525
|
-
const
|
|
46436
|
+
const score2 = clampScore(result.score);
|
|
45526
46437
|
return {
|
|
45527
|
-
score,
|
|
45528
|
-
verdict: scoreToVerdict(
|
|
45529
|
-
assertions: [{ text: result.name, passed:
|
|
46438
|
+
score: score2,
|
|
46439
|
+
verdict: scoreToVerdict(score2),
|
|
46440
|
+
assertions: [{ text: result.name, passed: score2 >= 0.5 }],
|
|
45530
46441
|
expectedAspectCount: 1,
|
|
45531
46442
|
details: result.metadata ? result.metadata : void 0
|
|
45532
46443
|
};
|
|
@@ -45627,17 +46538,17 @@ function normalizeScriptResult(result, fallbackText, threshold) {
|
|
|
45627
46538
|
};
|
|
45628
46539
|
}
|
|
45629
46540
|
if (typeof result === "number") {
|
|
45630
|
-
const
|
|
45631
|
-
const passed2 =
|
|
46541
|
+
const score3 = clampScore(result);
|
|
46542
|
+
const passed2 = score3 >= passThreshold;
|
|
45632
46543
|
return {
|
|
45633
|
-
score:
|
|
46544
|
+
score: score3,
|
|
45634
46545
|
verdict: passed2 ? "pass" : "fail",
|
|
45635
46546
|
assertions: [{ text: passed2 ? "Assertion passed" : fallbackText, passed: passed2 }],
|
|
45636
46547
|
expectedAspectCount: 1
|
|
45637
46548
|
};
|
|
45638
46549
|
}
|
|
45639
|
-
const
|
|
45640
|
-
const passed = result.pass ??
|
|
46550
|
+
const score2 = typeof result.score === "number" ? clampScore(result.score) : result.pass === true ? 1 : result.pass === false ? 0 : 0;
|
|
46551
|
+
const passed = result.pass ?? score2 >= passThreshold;
|
|
45641
46552
|
const assertions = result.assertions && result.assertions.length > 0 ? result.assertions : [
|
|
45642
46553
|
{
|
|
45643
46554
|
text: result.reason ?? (passed ? "Assertion passed" : fallbackText),
|
|
@@ -45645,7 +46556,7 @@ function normalizeScriptResult(result, fallbackText, threshold) {
|
|
|
45645
46556
|
}
|
|
45646
46557
|
];
|
|
45647
46558
|
return {
|
|
45648
|
-
score,
|
|
46559
|
+
score: score2,
|
|
45649
46560
|
verdict: passed ? "pass" : "fail",
|
|
45650
46561
|
assertions,
|
|
45651
46562
|
expectedAspectCount: assertions.length || 1,
|
|
@@ -45809,12 +46720,12 @@ var AssertSetGrader = class {
|
|
|
45809
46720
|
tokenUsage: result.tokenUsage
|
|
45810
46721
|
});
|
|
45811
46722
|
}
|
|
45812
|
-
const totalWeight = scores.reduce((sum,
|
|
45813
|
-
const
|
|
46723
|
+
const totalWeight = scores.reduce((sum, score3) => sum + (score3.weight ?? 1), 0) || 1;
|
|
46724
|
+
const score2 = scores.reduce((sum, item) => sum + item.score * (item.weight ?? 1), 0) / totalWeight;
|
|
45814
46725
|
const threshold = this.config.threshold;
|
|
45815
|
-
const passed = threshold !== void 0 ?
|
|
46726
|
+
const passed = threshold !== void 0 ? score2 >= threshold : scores.every((item) => (item.weight ?? 1) === 0 || item.verdict === "pass");
|
|
45816
46727
|
return {
|
|
45817
|
-
score,
|
|
46728
|
+
score: score2,
|
|
45818
46729
|
verdict: passed ? "pass" : "fail",
|
|
45819
46730
|
assertions: scores.flatMap((item) => item.assertions),
|
|
45820
46731
|
expectedAspectCount: scores.reduce((sum, item) => sum + item.assertions.length, 0) || 1,
|
|
@@ -46016,6 +46927,12 @@ var toolTrajectoryFactory = (config2) => {
|
|
|
46016
46927
|
config: config2
|
|
46017
46928
|
});
|
|
46018
46929
|
};
|
|
46930
|
+
var trajectoryFactory = (config2, context) => {
|
|
46931
|
+
return new TrajectoryGrader({
|
|
46932
|
+
config: config2,
|
|
46933
|
+
llmGrader: context.llmGrader
|
|
46934
|
+
});
|
|
46935
|
+
};
|
|
46019
46936
|
var fieldAccuracyFactory = (config2) => {
|
|
46020
46937
|
return new FieldAccuracyGrader({
|
|
46021
46938
|
config: config2
|
|
@@ -46038,6 +46955,9 @@ var executionMetricsFactory = (config2) => {
|
|
|
46038
46955
|
var skillTriggerFactory = (config2) => {
|
|
46039
46956
|
return new SkillTriggerGrader(config2);
|
|
46040
46957
|
};
|
|
46958
|
+
var skillUsedFactory = (config2) => {
|
|
46959
|
+
return new SkillUsedGrader(config2);
|
|
46960
|
+
};
|
|
46041
46961
|
var containsFactory = (config2) => {
|
|
46042
46962
|
const c = config2;
|
|
46043
46963
|
return new DeterministicAssertionGrader("contains", (ctx) => {
|
|
@@ -46171,7 +47091,7 @@ var endsWithFactory = (config2) => {
|
|
|
46171
47091
|
};
|
|
46172
47092
|
function createBuiltinRegistry() {
|
|
46173
47093
|
const registry2 = new GraderRegistry();
|
|
46174
|
-
registry2.register("llm-grader", llmGraderFactory).register("llm-rubric", llmRubricFactory).register("script", scriptFactory).register("tool-trajectory", toolTrajectoryFactory).register("field-accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token-usage", tokenUsageFactory).register("execution-metrics", executionMetricsFactory).register("skill-trigger", skillTriggerFactory).register("assert-set", assertSetFactory).register("contains", containsFactory).register("contains-any", containsAnyFactory).register("contains-all", containsAllFactory).register("icontains", icontainsFactory).register("icontains-any", icontainsAnyFactory).register("icontains-all", icontainsAllFactory).register("starts-with", startsWithFactory).register("ends-with", endsWithFactory).register("regex", regexFactory).register("is-json", isJsonFactory).register("equals", equalsFactory).register("javascript", javascriptFactory).register("python", pythonFactory).register("webhook", webhookFactory).register("similar", similarFactory).register("inline-assert", (config2) => {
|
|
47094
|
+
registry2.register("llm-grader", llmGraderFactory).register("llm-rubric", llmRubricFactory).register("script", scriptFactory).register("tool-trajectory", toolTrajectoryFactory).register("trajectory:tool-used", trajectoryFactory).register("trajectory:tool-args-match", trajectoryFactory).register("trajectory:tool-sequence", trajectoryFactory).register("trajectory:step-count", trajectoryFactory).register("trajectory:goal-success", trajectoryFactory).register("field-accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token-usage", tokenUsageFactory).register("execution-metrics", executionMetricsFactory).register("skill-used", skillUsedFactory).register("not-skill-used", skillUsedFactory).register("skill-trigger", skillTriggerFactory).register("assert-set", assertSetFactory).register("contains", containsFactory).register("contains-any", containsAnyFactory).register("contains-all", containsAllFactory).register("icontains", icontainsFactory).register("icontains-any", icontainsAnyFactory).register("icontains-all", icontainsAllFactory).register("starts-with", startsWithFactory).register("ends-with", endsWithFactory).register("regex", regexFactory).register("is-json", isJsonFactory).register("equals", equalsFactory).register("javascript", javascriptFactory).register("python", pythonFactory).register("webhook", webhookFactory).register("similar", similarFactory).register("inline-assert", (config2) => {
|
|
46175
47095
|
const fn = config2[INLINE_ASSERT_FN];
|
|
46176
47096
|
if (!fn) {
|
|
46177
47097
|
throw new Error(
|
|
@@ -49161,7 +50081,7 @@ var REPEAT_FIELDS = /* @__PURE__ */ new Set([
|
|
|
49161
50081
|
"cost_limit_usd"
|
|
49162
50082
|
]);
|
|
49163
50083
|
function normalizeExperimentConfig(rawConfig) {
|
|
49164
|
-
if (!
|
|
50084
|
+
if (!isRecord8(rawConfig)) {
|
|
49165
50085
|
throw new Error("Experiment config must be an object.");
|
|
49166
50086
|
}
|
|
49167
50087
|
const name = readOptionalString22(rawConfig.name, "name");
|
|
@@ -49202,7 +50122,7 @@ function normalizeExperimentConfig(rawConfig) {
|
|
|
49202
50122
|
};
|
|
49203
50123
|
}
|
|
49204
50124
|
function normalizeExperimentRunOverride(rawConfig) {
|
|
49205
|
-
if (!
|
|
50125
|
+
if (!isRecord8(rawConfig)) {
|
|
49206
50126
|
throw new Error("Run override must be an object.");
|
|
49207
50127
|
}
|
|
49208
50128
|
for (const key of Object.keys(rawConfig)) {
|
|
@@ -49272,7 +50192,7 @@ function readRepeat(raw) {
|
|
|
49272
50192
|
strategy: "pass_any"
|
|
49273
50193
|
};
|
|
49274
50194
|
}
|
|
49275
|
-
if (!
|
|
50195
|
+
if (!isRecord8(raw)) {
|
|
49276
50196
|
throw new Error("Experiment repeat must be a positive integer or object.");
|
|
49277
50197
|
}
|
|
49278
50198
|
for (const key of Object.keys(raw)) {
|
|
@@ -49304,7 +50224,7 @@ function readTargets(raw) {
|
|
|
49304
50224
|
if (typeof entry === "string" && entry.trim().length > 0) {
|
|
49305
50225
|
return entry.trim();
|
|
49306
50226
|
}
|
|
49307
|
-
if (!
|
|
50227
|
+
if (!isRecord8(entry)) {
|
|
49308
50228
|
throw new Error(`Experiment targets[${index}] must be a string or object.`);
|
|
49309
50229
|
}
|
|
49310
50230
|
const name = readRequiredString3(entry.name, `targets[${index}].name`);
|
|
@@ -49415,7 +50335,7 @@ function readOptionalRecord(raw) {
|
|
|
49415
50335
|
if (raw === void 0) {
|
|
49416
50336
|
return void 0;
|
|
49417
50337
|
}
|
|
49418
|
-
if (!
|
|
50338
|
+
if (!isRecord8(raw)) {
|
|
49419
50339
|
throw new Error("Experiment object field must be an object.");
|
|
49420
50340
|
}
|
|
49421
50341
|
return raw;
|
|
@@ -49436,7 +50356,7 @@ function rejectExperimentWorkers(raw) {
|
|
|
49436
50356
|
"Experiment workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency."
|
|
49437
50357
|
);
|
|
49438
50358
|
}
|
|
49439
|
-
function
|
|
50359
|
+
function isRecord8(value) {
|
|
49440
50360
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
49441
50361
|
}
|
|
49442
50362
|
function rejectExperimentLifecycleCommands(rawConfig) {
|
|
@@ -50321,18 +51241,83 @@ var UNSUPPORTED_PROMPTFOO_ASSERTION_TYPES = /* @__PURE__ */ new Set([
|
|
|
50321
51241
|
"human",
|
|
50322
51242
|
"max-score",
|
|
50323
51243
|
"tool-call-f1",
|
|
50324
|
-
"skill-used",
|
|
50325
|
-
"trajectory:goal-success",
|
|
50326
|
-
"trajectory:tool-args-match",
|
|
50327
|
-
"trajectory:step-count",
|
|
50328
|
-
"trajectory:tool-sequence",
|
|
50329
|
-
"trajectory:tool-used",
|
|
50330
51244
|
"trace-error-spans",
|
|
50331
51245
|
"trace-span-count",
|
|
50332
51246
|
"trace-span-duration",
|
|
50333
51247
|
"search-rubric",
|
|
50334
51248
|
"word-count"
|
|
50335
51249
|
]);
|
|
51250
|
+
var TRAJECTORY_GRADER_TYPES = /* @__PURE__ */ new Set([
|
|
51251
|
+
"trajectory:tool-used",
|
|
51252
|
+
"trajectory:tool-args-match",
|
|
51253
|
+
"trajectory:tool-sequence",
|
|
51254
|
+
"trajectory:step-count",
|
|
51255
|
+
"trajectory:goal-success"
|
|
51256
|
+
]);
|
|
51257
|
+
function isTrajectoryGraderKind(value) {
|
|
51258
|
+
return TRAJECTORY_GRADER_TYPES.has(value);
|
|
51259
|
+
}
|
|
51260
|
+
function formatJsonValue(value) {
|
|
51261
|
+
if (typeof value === "string") {
|
|
51262
|
+
return value;
|
|
51263
|
+
}
|
|
51264
|
+
try {
|
|
51265
|
+
return JSON.stringify(value);
|
|
51266
|
+
} catch {
|
|
51267
|
+
return String(value);
|
|
51268
|
+
}
|
|
51269
|
+
}
|
|
51270
|
+
function hasToolTrajectoryLatencyCheck(rawEvaluator) {
|
|
51271
|
+
const expected = rawEvaluator.expected;
|
|
51272
|
+
return Array.isArray(expected) && expected.some(
|
|
51273
|
+
(item) => isJsonObject2(item) && (item.max_duration_ms !== void 0 || item.maxDurationMs !== void 0)
|
|
51274
|
+
);
|
|
51275
|
+
}
|
|
51276
|
+
function staleSkillTriggerMessage(rawEvaluator) {
|
|
51277
|
+
const skillName = asString3(rawEvaluator.skill);
|
|
51278
|
+
const shouldTrigger = rawEvaluator.should_trigger !== false;
|
|
51279
|
+
if (!skillName) {
|
|
51280
|
+
return "Authored assertion type 'skill-trigger' has been removed. Use 'skill-used' with value: <skill> for expected skill use, or 'not-skill-used' with value: <skill> when the skill must not be used.";
|
|
51281
|
+
}
|
|
51282
|
+
const replacementType = shouldTrigger ? "skill-used" : "not-skill-used";
|
|
51283
|
+
return `Authored assertion type 'skill-trigger' has been removed. Replace skill: ${skillName} with type: ${replacementType}, value: ${skillName}.`;
|
|
51284
|
+
}
|
|
51285
|
+
function staleToolTrajectoryMessage(rawEvaluator) {
|
|
51286
|
+
const mode = asString3(rawEvaluator.mode);
|
|
51287
|
+
if (hasToolTrajectoryLatencyCheck(rawEvaluator)) {
|
|
51288
|
+
return "Authored assertion type 'tool-trajectory' has been removed. Per-tool latency checks such as max_duration_ms have no Promptfoo trajectory:* equivalent in AgentV yet; use a custom script assertion or track this as unsupported future scope.";
|
|
51289
|
+
}
|
|
51290
|
+
if (mode === "any_order" && isJsonObject2(rawEvaluator.minimums)) {
|
|
51291
|
+
const entries = Object.entries(rawEvaluator.minimums);
|
|
51292
|
+
const [toolName, min] = entries[0] ?? [];
|
|
51293
|
+
const example = typeof toolName === "string" ? ` For example: type: trajectory:tool-used, value: { name: ${toolName}, min: ${formatJsonValue(min)} }.` : "";
|
|
51294
|
+
return `Authored assertion type 'tool-trajectory' has been removed. Replace mode: any_order minimums with one Promptfoo trajectory:tool-used assertion per tool.${example}`;
|
|
51295
|
+
}
|
|
51296
|
+
if ((mode === "in_order" || mode === "exact") && Array.isArray(rawEvaluator.expected)) {
|
|
51297
|
+
const steps = rawEvaluator.expected.filter(isJsonObject2).map((item) => item.tool).filter((tool) => typeof tool === "string");
|
|
51298
|
+
const hasArgs = rawEvaluator.expected.some(
|
|
51299
|
+
(item) => isJsonObject2(item) && item.args !== void 0 && item.args !== "any"
|
|
51300
|
+
);
|
|
51301
|
+
const sequenceHint = steps.length > 0 ? ` Use type: trajectory:tool-sequence, value: { mode: ${mode}, steps: ${formatJsonValue(steps)} }.` : " Use type: trajectory:tool-sequence with value: { mode, steps }.";
|
|
51302
|
+
const argsHint = hasArgs ? " Move expected args checks to trajectory:tool-args-match with Promptfoo mode: partial or exact." : "";
|
|
51303
|
+
return `Authored assertion type 'tool-trajectory' has been removed.${sequenceHint}${argsHint}`;
|
|
51304
|
+
}
|
|
51305
|
+
return "Authored assertion type 'tool-trajectory' has been removed. Use Promptfoo trajectory:* assertions: trajectory:tool-used for tool presence/counts, trajectory:tool-sequence for in_order/exact steps, and trajectory:tool-args-match for argument checks. AgentV-specific latency checks are unsupported future scope.";
|
|
51306
|
+
}
|
|
51307
|
+
function staleAuthoredGraderMessage(rawEvaluator) {
|
|
51308
|
+
const rawType = rawEvaluator.type;
|
|
51309
|
+
if (typeof rawType !== "string") {
|
|
51310
|
+
return void 0;
|
|
51311
|
+
}
|
|
51312
|
+
const type = rawType.replace(/_/g, "-");
|
|
51313
|
+
if (type === "skill-trigger") {
|
|
51314
|
+
return staleSkillTriggerMessage(rawEvaluator);
|
|
51315
|
+
}
|
|
51316
|
+
if (type === "tool-trajectory") {
|
|
51317
|
+
return staleToolTrajectoryMessage(rawEvaluator);
|
|
51318
|
+
}
|
|
51319
|
+
return void 0;
|
|
51320
|
+
}
|
|
50336
51321
|
function assertSupportedPromptfooType(type, evalId, name) {
|
|
50337
51322
|
const baseType = type.startsWith("not-") ? type.slice(4) : type;
|
|
50338
51323
|
if (!UNSUPPORTED_PROMPTFOO_ASSERTION_TYPES.has(baseType)) {
|
|
@@ -50607,15 +51592,24 @@ async function parseGraderList(candidateEvaluators, searchRoots, evalId, default
|
|
|
50607
51592
|
const rawEvaluator = withInheritedAssertionConfig(rawEvaluatorEntry, inheritedAssertionConfig);
|
|
50608
51593
|
const rawName = asString3(rawEvaluator.metric);
|
|
50609
51594
|
const rawType = rawEvaluator.type;
|
|
50610
|
-
const
|
|
50611
|
-
|
|
50612
|
-
|
|
51595
|
+
const normalizedType = typeof rawType === "string" ? normalizeGraderType(rawType) : rawType;
|
|
51596
|
+
const negatedType = typeof normalizedType === "string" && normalizedType.startsWith("not-") ? normalizedType.slice(4) : void 0;
|
|
51597
|
+
const typeValue = negatedType && isTrajectoryGraderKind(negatedType) ? negatedType : normalizedType;
|
|
51598
|
+
const inverse = negatedType && isTrajectoryGraderKind(negatedType) ? true : void 0;
|
|
51599
|
+
if (typeof normalizedType === "string") {
|
|
51600
|
+
const staleMessage = staleAuthoredGraderMessage(rawEvaluator);
|
|
51601
|
+
if (staleMessage) {
|
|
51602
|
+
throw new Error(
|
|
51603
|
+
`Unsupported grader '${rawType}' in '${evalId}'${rawName ? ` for evaluator '${rawName}'` : ""}. ${staleMessage}`
|
|
51604
|
+
);
|
|
51605
|
+
}
|
|
51606
|
+
const replacement = removedGraderReplacement(normalizedType);
|
|
50613
51607
|
if (replacement) {
|
|
50614
51608
|
throw new Error(
|
|
50615
51609
|
`Unsupported grader '${rawType}' in '${evalId}'. Use '${replacement}' instead.`
|
|
50616
51610
|
);
|
|
50617
51611
|
}
|
|
50618
|
-
assertSupportedPromptfooType(
|
|
51612
|
+
assertSupportedPromptfooType(normalizedType, evalId, rawName);
|
|
50619
51613
|
}
|
|
50620
51614
|
const isCustomType = typeof typeValue === "string" && !isGraderKind(typeValue);
|
|
50621
51615
|
if (typeof typeValue !== "string") {
|
|
@@ -50648,6 +51642,25 @@ async function parseGraderList(candidateEvaluators, searchRoots, evalId, default
|
|
|
50648
51642
|
evaluators.push(transform2 !== void 0 ? { ...config22, transform: transform2 } : config22);
|
|
50649
51643
|
};
|
|
50650
51644
|
const inheritedInternalPreprocessors = defaultPreprocessors;
|
|
51645
|
+
if (isTrajectoryGraderKind(typeValue)) {
|
|
51646
|
+
const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
|
|
51647
|
+
const { required: required22, min_score: min_score2 } = parseRequiredAndMinScore(
|
|
51648
|
+
rawEvaluator.required,
|
|
51649
|
+
rawEvaluator.min_score,
|
|
51650
|
+
name,
|
|
51651
|
+
evalId
|
|
51652
|
+
);
|
|
51653
|
+
pushEvaluator({
|
|
51654
|
+
name,
|
|
51655
|
+
type: typeValue,
|
|
51656
|
+
...rawEvaluator.value !== void 0 ? { value: rawEvaluator.value } : {},
|
|
51657
|
+
...weight2 !== void 0 ? { weight: weight2 } : {},
|
|
51658
|
+
...required22 !== void 0 ? { required: required22 } : {},
|
|
51659
|
+
...min_score2 !== void 0 ? { min_score: min_score2 } : {},
|
|
51660
|
+
...inverse || negate ? { inverse: true } : {}
|
|
51661
|
+
});
|
|
51662
|
+
continue;
|
|
51663
|
+
}
|
|
50651
51664
|
if (isCustomType) {
|
|
50652
51665
|
const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
|
|
50653
51666
|
const { required: required22, min_score: min_score2 } = parseRequiredAndMinScore(
|
|
@@ -50862,18 +51875,18 @@ async function parseGraderList(candidateEvaluators, searchRoots, evalId, default
|
|
|
50862
51875
|
}
|
|
50863
51876
|
}
|
|
50864
51877
|
const rawArgsMatch = rawEvaluator.args_match ?? rawEvaluator.argsMatch;
|
|
50865
|
-
let
|
|
51878
|
+
let argsMatch3;
|
|
50866
51879
|
if (rawArgsMatch !== void 0) {
|
|
50867
51880
|
if (Array.isArray(rawArgsMatch)) {
|
|
50868
51881
|
const fieldList = rawArgsMatch.filter(
|
|
50869
51882
|
(f) => typeof f === "string" && f.length > 0
|
|
50870
51883
|
);
|
|
50871
51884
|
if (fieldList.length > 0) {
|
|
50872
|
-
|
|
51885
|
+
argsMatch3 = fieldList;
|
|
50873
51886
|
}
|
|
50874
51887
|
} else if (typeof rawArgsMatch === "string") {
|
|
50875
51888
|
if (rawArgsMatch === "exact" || rawArgsMatch === "superset" || rawArgsMatch === "subset" || rawArgsMatch === "ignore") {
|
|
50876
|
-
|
|
51889
|
+
argsMatch3 = rawArgsMatch;
|
|
50877
51890
|
} else {
|
|
50878
51891
|
logWarning2(
|
|
50879
51892
|
`Invalid args_match '${rawArgsMatch}' for tool-trajectory evaluator '${name}' in '${evalId}': must be exact, superset, subset, ignore, or a string array`
|
|
@@ -50959,11 +51972,30 @@ async function parseGraderList(candidateEvaluators, searchRoots, evalId, default
|
|
|
50959
51972
|
...required22 !== void 0 ? { required: required22 } : {},
|
|
50960
51973
|
...min_score2 !== void 0 ? { min_score: min_score2 } : {},
|
|
50961
51974
|
...negate !== void 0 ? { negate } : {},
|
|
50962
|
-
...
|
|
51975
|
+
...argsMatch3 !== void 0 ? { argsMatch: argsMatch3 } : {}
|
|
50963
51976
|
};
|
|
50964
51977
|
pushEvaluator(config22);
|
|
50965
51978
|
continue;
|
|
50966
51979
|
}
|
|
51980
|
+
if (typeValue === "skill-used" || typeValue === "not-skill-used") {
|
|
51981
|
+
const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
|
|
51982
|
+
const { required: required22, min_score: min_score2 } = parseRequiredAndMinScore(
|
|
51983
|
+
rawEvaluator.required,
|
|
51984
|
+
rawEvaluator.min_score,
|
|
51985
|
+
name,
|
|
51986
|
+
evalId
|
|
51987
|
+
);
|
|
51988
|
+
pushEvaluator({
|
|
51989
|
+
name,
|
|
51990
|
+
type: typeValue,
|
|
51991
|
+
value: rawEvaluator.value,
|
|
51992
|
+
...weight2 !== void 0 ? { weight: weight2 } : {},
|
|
51993
|
+
...required22 !== void 0 ? { required: required22 } : {},
|
|
51994
|
+
...min_score2 !== void 0 ? { min_score: min_score2 } : {},
|
|
51995
|
+
...negate !== void 0 ? { negate } : {}
|
|
51996
|
+
});
|
|
51997
|
+
continue;
|
|
51998
|
+
}
|
|
50967
51999
|
if (typeValue === "field-accuracy") {
|
|
50968
52000
|
const rawFields = rawEvaluator.fields;
|
|
50969
52001
|
if (!Array.isArray(rawFields)) {
|
|
@@ -51667,6 +52699,17 @@ function generateAssertionName(typeValue, rawEvaluator) {
|
|
|
51667
52699
|
const value = asString3(rawEvaluator.value);
|
|
51668
52700
|
const arrayValue = Array.isArray(rawEvaluator.value) ? rawEvaluator.value : void 0;
|
|
51669
52701
|
switch (typeValue) {
|
|
52702
|
+
case "skill-used":
|
|
52703
|
+
case "not-skill-used": {
|
|
52704
|
+
const rawValue = rawEvaluator.value;
|
|
52705
|
+
if (typeof rawValue === "string" && rawValue.trim()) {
|
|
52706
|
+
return `${typeValue}-${rawValue.trim()}`;
|
|
52707
|
+
}
|
|
52708
|
+
if (Array.isArray(rawValue)) {
|
|
52709
|
+
return `${typeValue}-${rawValue.length}`;
|
|
52710
|
+
}
|
|
52711
|
+
return typeValue;
|
|
52712
|
+
}
|
|
51670
52713
|
case "skill-trigger": {
|
|
51671
52714
|
const skillValue = asString3(rawEvaluator.skill);
|
|
51672
52715
|
return skillValue ? `skill-trigger-${skillValue}` : "skill-trigger";
|
|
@@ -53538,7 +54581,7 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
|
|
|
53538
54581
|
return { tests: await loadTestsFromJsonl(evalFilePath, repoRoot, options) };
|
|
53539
54582
|
}
|
|
53540
54583
|
if (format === "typescript") {
|
|
53541
|
-
const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-
|
|
54584
|
+
const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-DQDYRULE-V3377FOL.js");
|
|
53542
54585
|
return loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
|
|
53543
54586
|
}
|
|
53544
54587
|
const { tests, parsed } = await loadTestsFromYaml(evalFilePath, repoRoot, options);
|
|
@@ -53560,7 +54603,7 @@ async function loadTests(evalFilePath, repoRoot, options) {
|
|
|
53560
54603
|
return loadTestsFromJsonl(evalFilePath, repoRoot, options);
|
|
53561
54604
|
}
|
|
53562
54605
|
if (format === "typescript") {
|
|
53563
|
-
const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-
|
|
54606
|
+
const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-DQDYRULE-V3377FOL.js");
|
|
53564
54607
|
const suite = await loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
|
|
53565
54608
|
return suite.tests;
|
|
53566
54609
|
}
|
|
@@ -55111,17 +56154,17 @@ ${detailBlock}${ANSI_RESET5}`);
|
|
|
55111
56154
|
function pathFromRoot(root) {
|
|
55112
56155
|
return root instanceof URL ? fileURLToPath3(root) : String(root);
|
|
55113
56156
|
}
|
|
55114
|
-
function classifyQualityStatus(
|
|
55115
|
-
return
|
|
56157
|
+
function classifyQualityStatus(score2, threshold = DEFAULT_THRESHOLD2) {
|
|
56158
|
+
return score2 >= threshold ? "ok" : "quality_failure";
|
|
55116
56159
|
}
|
|
55117
56160
|
function buildSkippedEvaluatorError(scores) {
|
|
55118
|
-
const skippedScores = scores?.filter((
|
|
56161
|
+
const skippedScores = scores?.filter((score2) => score2.verdict === "skip") ?? [];
|
|
55119
56162
|
if (skippedScores.length === 0) {
|
|
55120
56163
|
return void 0;
|
|
55121
56164
|
}
|
|
55122
|
-
const messages = skippedScores.map((
|
|
55123
|
-
const label =
|
|
55124
|
-
const assertionMessage =
|
|
56165
|
+
const messages = skippedScores.map((score2) => {
|
|
56166
|
+
const label = score2.name || score2.type;
|
|
56167
|
+
const assertionMessage = score2.assertions.find((assertion) => !assertion.passed)?.text ?? "Grader skipped";
|
|
55125
56168
|
return `${label}: ${assertionMessage}`;
|
|
55126
56169
|
});
|
|
55127
56170
|
return messages.length === 1 ? messages[0] : `Graders skipped: ${messages.join(" | ")}`;
|
|
@@ -55424,7 +56467,7 @@ async function gradePreparedEvalCase(options) {
|
|
|
55424
56467
|
const gradingOutput = evalCase.outputTransform ? [
|
|
55425
56468
|
{ role: "assistant", content: preparedCandidate.candidate }
|
|
55426
56469
|
] : preparedTrace ? outputMessages : void 0;
|
|
55427
|
-
const { score, scores } = await runEvaluatorsForCase({
|
|
56470
|
+
const { score: score2, scores } = await runEvaluatorsForCase({
|
|
55428
56471
|
evalCase,
|
|
55429
56472
|
candidate: preparedCandidate.candidate,
|
|
55430
56473
|
candidateValue: preparedCandidate.candidateValue,
|
|
@@ -55458,7 +56501,7 @@ async function gradePreparedEvalCase(options) {
|
|
|
55458
56501
|
...graderTokens ? { tokenUsage: graderTokens } : {}
|
|
55459
56502
|
};
|
|
55460
56503
|
const skippedEvaluatorError = buildSkippedEvaluatorError(scores);
|
|
55461
|
-
const executionStatus = skippedEvaluatorError ? "execution_error" : classifyQualityStatus(
|
|
56504
|
+
const executionStatus = skippedEvaluatorError ? "execution_error" : classifyQualityStatus(score2.score, effectiveThreshold);
|
|
55462
56505
|
const baseResult = {
|
|
55463
56506
|
timestamp: timestamp.toISOString(),
|
|
55464
56507
|
testId: authoredResultTestId(evalCase),
|
|
@@ -55467,10 +56510,10 @@ async function gradePreparedEvalCase(options) {
|
|
|
55467
56510
|
suite: evalCase.suite,
|
|
55468
56511
|
category: evalCase.category,
|
|
55469
56512
|
conversationId: evalCase.conversation_id,
|
|
55470
|
-
score: skippedEvaluatorError ? 0 :
|
|
55471
|
-
reason:
|
|
55472
|
-
checks:
|
|
55473
|
-
assertions:
|
|
56513
|
+
score: skippedEvaluatorError ? 0 : score2.score,
|
|
56514
|
+
reason: score2.reason,
|
|
56515
|
+
checks: score2.checks,
|
|
56516
|
+
assertions: score2.assertions,
|
|
55474
56517
|
target: target.name,
|
|
55475
56518
|
input,
|
|
55476
56519
|
output: preparedCandidate.candidate,
|
|
@@ -56946,7 +57989,7 @@ async function runEvalCaseWithTrials(options, trialsConfig) {
|
|
|
56946
57989
|
break;
|
|
56947
57990
|
}
|
|
56948
57991
|
}
|
|
56949
|
-
const { score, aggregation } = aggregateTrials(trialResults, trialsConfig);
|
|
57992
|
+
const { score: score2, aggregation } = aggregateTrials(trialResults, trialsConfig);
|
|
56950
57993
|
const bestTrialIndex = trialResults.reduce(
|
|
56951
57994
|
(bestIdx, t, idx) => t.score > trialResults[bestIdx].score ? idx : bestIdx,
|
|
56952
57995
|
0
|
|
@@ -56960,7 +58003,7 @@ async function runEvalCaseWithTrials(options, trialsConfig) {
|
|
|
56960
58003
|
const aggregateExecutionError = aggregateExecutionStatus === "execution_error" ? baseResult.executionError : void 0;
|
|
56961
58004
|
return {
|
|
56962
58005
|
...baseResult,
|
|
56963
|
-
score,
|
|
58006
|
+
score: score2,
|
|
56964
58007
|
sampleIndex: void 0,
|
|
56965
58008
|
retryIndex: void 0,
|
|
56966
58009
|
trials: trialResults,
|
|
@@ -57020,6 +58063,9 @@ function toJsonValue(value) {
|
|
|
57020
58063
|
function buildProviderResponseTransformMetadata(response) {
|
|
57021
58064
|
const metadata = {};
|
|
57022
58065
|
const fields = [
|
|
58066
|
+
["metadata", response.metadata],
|
|
58067
|
+
["skill_calls", response.metadata?.skillCalls],
|
|
58068
|
+
["attempted_skill_calls", response.metadata?.attemptedSkillCalls],
|
|
57023
58069
|
["raw", response.raw],
|
|
57024
58070
|
["usage", response.usage],
|
|
57025
58071
|
["token_usage", response.tokenUsage],
|
|
@@ -57118,7 +58164,7 @@ async function evaluateCandidate(options) {
|
|
|
57118
58164
|
const gradingOutput = evalCase.outputTransform ? [
|
|
57119
58165
|
{ role: "assistant", content: preparedCandidate.candidate }
|
|
57120
58166
|
] : output;
|
|
57121
|
-
const { score, scores } = await runEvaluatorsForCase({
|
|
58167
|
+
const { score: score2, scores } = await runEvaluatorsForCase({
|
|
57122
58168
|
evalCase,
|
|
57123
58169
|
candidate: preparedCandidate.candidate,
|
|
57124
58170
|
candidateValue: preparedCandidate.candidateValue,
|
|
@@ -57166,7 +58212,7 @@ async function evaluateCandidate(options) {
|
|
|
57166
58212
|
};
|
|
57167
58213
|
}
|
|
57168
58214
|
}
|
|
57169
|
-
const evaluatorRequest = scores ? void 0 :
|
|
58215
|
+
const evaluatorRequest = scores ? void 0 : score2.graderRawRequest;
|
|
57170
58216
|
const effectiveAgentRequest = agentRequest && Object.keys(agentRequest).length > 0 ? agentRequest : void 0;
|
|
57171
58217
|
const requests = effectiveAgentRequest || lmRequest || evaluatorRequest ? {
|
|
57172
58218
|
...effectiveAgentRequest ? { agent: effectiveAgentRequest } : {},
|
|
@@ -57183,10 +58229,10 @@ async function evaluateCandidate(options) {
|
|
|
57183
58229
|
suite: evalCase.suite,
|
|
57184
58230
|
category: evalCase.category,
|
|
57185
58231
|
conversationId: evalCase.conversation_id,
|
|
57186
|
-
score:
|
|
57187
|
-
reason:
|
|
57188
|
-
checks:
|
|
57189
|
-
assertions:
|
|
58232
|
+
score: score2.score,
|
|
58233
|
+
reason: score2.reason,
|
|
58234
|
+
checks: score2.checks,
|
|
58235
|
+
assertions: score2.assertions,
|
|
57190
58236
|
target: target.name,
|
|
57191
58237
|
tokenUsage,
|
|
57192
58238
|
costUsd,
|
|
@@ -57200,7 +58246,7 @@ async function evaluateCandidate(options) {
|
|
|
57200
58246
|
trace: evaluationTrace,
|
|
57201
58247
|
rawProviderLogPath,
|
|
57202
58248
|
fileChanges,
|
|
57203
|
-
executionStatus: classifyQualityStatus(
|
|
58249
|
+
executionStatus: classifyQualityStatus(score2.score, evalThreshold)
|
|
57204
58250
|
};
|
|
57205
58251
|
}
|
|
57206
58252
|
async function runEvaluatorsForCase(options) {
|
|
@@ -57288,7 +58334,7 @@ async function runEvaluatorsForCase(options) {
|
|
|
57288
58334
|
throw new Error(`No evaluator registered for kind '${evaluatorKind}'`);
|
|
57289
58335
|
}
|
|
57290
58336
|
const implicitEvaluator = evaluatorKind === "llm-grader" && !evalCase.assertions ? buildImplicitLlmGraderConfig(evalCase) : void 0;
|
|
57291
|
-
const
|
|
58337
|
+
const score2 = await activeEvaluator.evaluate({
|
|
57292
58338
|
evalCase,
|
|
57293
58339
|
candidate,
|
|
57294
58340
|
candidateValue,
|
|
@@ -57315,7 +58361,7 @@ async function runEvaluatorsForCase(options) {
|
|
|
57315
58361
|
dependencyResults,
|
|
57316
58362
|
...implicitEvaluator ? { evaluator: implicitEvaluator } : {}
|
|
57317
58363
|
});
|
|
57318
|
-
return { score };
|
|
58364
|
+
return { score: score2 };
|
|
57319
58365
|
}
|
|
57320
58366
|
function buildImplicitLlmGraderConfig(evalCase) {
|
|
57321
58367
|
if (!evalCase.preprocessors || evalCase.preprocessors.length === 0) {
|
|
@@ -57435,11 +58481,11 @@ async function runEvaluatorList(options) {
|
|
|
57435
58481
|
evalContext,
|
|
57436
58482
|
evaluatorConfig
|
|
57437
58483
|
);
|
|
57438
|
-
const
|
|
58484
|
+
const score3 = await evaluatorInstance.evaluate(transformedContext.context);
|
|
57439
58485
|
const endedAt = /* @__PURE__ */ new Date();
|
|
57440
58486
|
const weight = evaluatorConfig.weight ?? 1;
|
|
57441
58487
|
scored.push({
|
|
57442
|
-
score:
|
|
58488
|
+
score: score3,
|
|
57443
58489
|
name: evaluatorConfig.name,
|
|
57444
58490
|
type: evaluatorConfig.type,
|
|
57445
58491
|
weight,
|
|
@@ -57449,17 +58495,17 @@ async function runEvaluatorList(options) {
|
|
|
57449
58495
|
scores.push({
|
|
57450
58496
|
name: evaluatorConfig.name,
|
|
57451
58497
|
type: evaluatorConfig.type,
|
|
57452
|
-
score:
|
|
58498
|
+
score: score3.score,
|
|
57453
58499
|
weight,
|
|
57454
|
-
verdict:
|
|
57455
|
-
reason:
|
|
57456
|
-
checks:
|
|
57457
|
-
assertions:
|
|
57458
|
-
input: transformedContext.input ??
|
|
57459
|
-
target:
|
|
57460
|
-
details: transformedContext.details &&
|
|
57461
|
-
scores: mapChildResults(
|
|
57462
|
-
tokenUsage:
|
|
58500
|
+
verdict: score3.verdict,
|
|
58501
|
+
reason: score3.reason,
|
|
58502
|
+
checks: score3.checks,
|
|
58503
|
+
assertions: score3.assertions,
|
|
58504
|
+
input: transformedContext.input ?? score3.graderRawRequest,
|
|
58505
|
+
target: score3.graderTarget,
|
|
58506
|
+
details: transformedContext.details && score3.details ? { ...transformedContext.details, ...score3.details } : transformedContext.details ?? score3.details,
|
|
58507
|
+
scores: mapChildResults(score3.scores),
|
|
58508
|
+
tokenUsage: score3.tokenUsage,
|
|
57463
58509
|
durationMs: endedAt.getTime() - startedAt.getTime(),
|
|
57464
58510
|
startedAt: startedAt.toISOString(),
|
|
57465
58511
|
endedAt: endedAt.toISOString()
|
|
@@ -57528,13 +58574,13 @@ async function runEvaluatorList(options) {
|
|
|
57528
58574
|
) : 0;
|
|
57529
58575
|
const assertions = scored.flatMap((entry) => entry.score.assertions);
|
|
57530
58576
|
const expectedAspectCount = assertions.length || 1;
|
|
57531
|
-
const
|
|
58577
|
+
const score2 = {
|
|
57532
58578
|
score: aggregateScore,
|
|
57533
58579
|
verdict: scoreToVerdict(aggregateScore, effectiveThreshold),
|
|
57534
58580
|
assertions,
|
|
57535
58581
|
expectedAspectCount
|
|
57536
58582
|
};
|
|
57537
|
-
return { score, scores };
|
|
58583
|
+
return { score: score2, scores };
|
|
57538
58584
|
}
|
|
57539
58585
|
function formatFilter(filter) {
|
|
57540
58586
|
return typeof filter === "string" ? filter : filter.join(", ");
|
|
@@ -58255,7 +59301,7 @@ var MetricsWireSchema = external_exports.object({
|
|
|
58255
59301
|
reasoning_blocks: external_exports.array(ReasoningBlockWireSchema),
|
|
58256
59302
|
thinking_blocks: external_exports.number().int().nonnegative()
|
|
58257
59303
|
}).strict();
|
|
58258
|
-
var
|
|
59304
|
+
var MetricsArtifactBaseWireSchema = external_exports.object({
|
|
58259
59305
|
schema_version: external_exports.literal(METRICS_SCHEMA_VERSION),
|
|
58260
59306
|
artifact_id: external_exports.string(),
|
|
58261
59307
|
generated_at: external_exports.string(),
|
|
@@ -58263,12 +59309,6 @@ var MetricsArtifactWireSchema = external_exports.object({
|
|
|
58263
59309
|
target: external_exports.string(),
|
|
58264
59310
|
suite: external_exports.string().optional(),
|
|
58265
59311
|
category: external_exports.string().optional(),
|
|
58266
|
-
trace: external_exports.object({
|
|
58267
|
-
schema_version: external_exports.literal(EXECUTION_TRACE_SCHEMA_VERSION),
|
|
58268
|
-
artifact_id: external_exports.string(),
|
|
58269
|
-
trace_id: external_exports.string(),
|
|
58270
|
-
root_span_id: external_exports.string()
|
|
58271
|
-
}).strict(),
|
|
58272
59312
|
source_artifacts: external_exports.object({
|
|
58273
59313
|
transcript_path: external_exports.string().optional(),
|
|
58274
59314
|
grading_path: external_exports.string().optional(),
|
|
@@ -58278,17 +59318,32 @@ var MetricsArtifactWireSchema = external_exports.object({
|
|
|
58278
59318
|
tokens: MetricsTokensWireSchema.optional(),
|
|
58279
59319
|
cost: MetricsCostWireSchema.optional(),
|
|
58280
59320
|
execution: external_exports.record(external_exports.string(), external_exports.unknown()).optional(),
|
|
58281
|
-
trajectory: external_exports.record(external_exports.string(), external_exports.unknown()).optional()
|
|
58282
|
-
metrics: MetricsWireSchema
|
|
59321
|
+
trajectory: external_exports.record(external_exports.string(), external_exports.unknown()).optional()
|
|
58283
59322
|
}).strict();
|
|
59323
|
+
var MetricsArtifactWireSchema = MetricsArtifactBaseWireSchema.merge(MetricsWireSchema);
|
|
58284
59324
|
function dropUndefined7(value) {
|
|
58285
59325
|
return Object.fromEntries(
|
|
58286
59326
|
Object.entries(value).filter(([, entryValue]) => entryValue !== void 0)
|
|
58287
59327
|
);
|
|
58288
59328
|
}
|
|
58289
|
-
function
|
|
59329
|
+
function isRecord9(value) {
|
|
58290
59330
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
58291
59331
|
}
|
|
59332
|
+
function normalizeMetricsArtifactWire(value) {
|
|
59333
|
+
if (!isRecord9(value)) {
|
|
59334
|
+
return void 0;
|
|
59335
|
+
}
|
|
59336
|
+
const nestedMetrics = isRecord9(value.metrics) ? value.metrics : {};
|
|
59337
|
+
const normalized = Object.fromEntries(
|
|
59338
|
+
Object.entries({
|
|
59339
|
+
...value,
|
|
59340
|
+
...nestedMetrics,
|
|
59341
|
+
schema_version: METRICS_SCHEMA_VERSION
|
|
59342
|
+
}).filter(([key]) => key !== "trace" && key !== "metrics")
|
|
59343
|
+
);
|
|
59344
|
+
const parsed = MetricsArtifactWireSchema.safeParse(normalized);
|
|
59345
|
+
return parsed.success ? parsed.data : void 0;
|
|
59346
|
+
}
|
|
58292
59347
|
function normalizedKey2(key) {
|
|
58293
59348
|
return key.replace(/[^A-Za-z0-9]+/g, "").toLowerCase();
|
|
58294
59349
|
}
|
|
@@ -58314,7 +59369,7 @@ function collectStringValuesByKey(value, keys, options = {}) {
|
|
|
58314
59369
|
}
|
|
58315
59370
|
return;
|
|
58316
59371
|
}
|
|
58317
|
-
if (!
|
|
59372
|
+
if (!isRecord9(entry)) {
|
|
58318
59373
|
return;
|
|
58319
59374
|
}
|
|
58320
59375
|
for (const [key, nested] of Object.entries(entry)) {
|
|
@@ -58350,7 +59405,7 @@ function findNumberByKey(value, keys, maxDepth = 4) {
|
|
|
58350
59405
|
}
|
|
58351
59406
|
return void 0;
|
|
58352
59407
|
}
|
|
58353
|
-
if (!
|
|
59408
|
+
if (!isRecord9(entry)) {
|
|
58354
59409
|
return void 0;
|
|
58355
59410
|
}
|
|
58356
59411
|
for (const [key, nested] of Object.entries(entry)) {
|
|
@@ -58451,7 +59506,7 @@ function toolSucceeded(toolCall) {
|
|
|
58451
59506
|
if (toolCall.status === "error" || toolCall.status === "timeout" || toolCall.status === "cancelled") {
|
|
58452
59507
|
return false;
|
|
58453
59508
|
}
|
|
58454
|
-
const explicitSuccess = booleanValue(
|
|
59509
|
+
const explicitSuccess = booleanValue(isRecord9(toolCall.output) ? toolCall.output.success : void 0) ?? booleanValue(isRecord9(toolCall.output) ? toolCall.output.ok : void 0);
|
|
58455
59510
|
if (explicitSuccess !== void 0) {
|
|
58456
59511
|
return explicitSuccess;
|
|
58457
59512
|
}
|
|
@@ -58635,7 +59690,7 @@ function buildWebFetches(calls) {
|
|
|
58635
59690
|
return [
|
|
58636
59691
|
dropUndefined7({
|
|
58637
59692
|
url: url2,
|
|
58638
|
-
method: stringValue2(
|
|
59693
|
+
method: stringValue2(isRecord9(call.toolCall.input) ? call.toolCall.input.method : void 0),
|
|
58639
59694
|
status: findNumberByKey(call.toolCall.output, STATUS_CODE_KEY_SET),
|
|
58640
59695
|
success: toolSucceeded(call.toolCall),
|
|
58641
59696
|
tool_call_id: call.toolCall.id
|
|
@@ -58647,7 +59702,7 @@ function errorFromUnknown(value, fallback) {
|
|
|
58647
59702
|
if (typeof value === "string" && value.trim().length > 0) {
|
|
58648
59703
|
return dropUndefined7({ message: value.trim(), ...fallback });
|
|
58649
59704
|
}
|
|
58650
|
-
if (!
|
|
59705
|
+
if (!isRecord9(value)) {
|
|
58651
59706
|
return void 0;
|
|
58652
59707
|
}
|
|
58653
59708
|
const message = stringValue2(value.message) ?? stringValue2(value.error) ?? stringValue2(value.reason);
|
|
@@ -58710,7 +59765,7 @@ function contentBlockText(value) {
|
|
|
58710
59765
|
if (typeof value === "string") {
|
|
58711
59766
|
return value.trim().length > 0 ? value : void 0;
|
|
58712
59767
|
}
|
|
58713
|
-
if (!
|
|
59768
|
+
if (!isRecord9(value)) {
|
|
58714
59769
|
return void 0;
|
|
58715
59770
|
}
|
|
58716
59771
|
return stringValue2(value.text) ?? stringValue2(value.content) ?? stringValue2(value.value) ?? stringValue2(value.summary);
|
|
@@ -58764,7 +59819,7 @@ function buildReasoningBlocks(messages, events) {
|
|
|
58764
59819
|
if (Array.isArray(content)) {
|
|
58765
59820
|
for (const item of content) {
|
|
58766
59821
|
const block = item;
|
|
58767
|
-
if (!
|
|
59822
|
+
if (!isRecord9(block)) {
|
|
58768
59823
|
continue;
|
|
58769
59824
|
}
|
|
58770
59825
|
const blockType = block.type;
|
|
@@ -58843,18 +59898,12 @@ function buildMetricsArtifact(result, envelope, options = {}) {
|
|
|
58843
59898
|
target: result.target ?? "unknown",
|
|
58844
59899
|
suite: result.suite,
|
|
58845
59900
|
category: result.category,
|
|
58846
|
-
trace: {
|
|
58847
|
-
schema_version: EXECUTION_TRACE_SCHEMA_VERSION,
|
|
58848
|
-
artifact_id: envelope.artifactId,
|
|
58849
|
-
trace_id: envelope.trace.traceId,
|
|
58850
|
-
root_span_id: envelope.trace.rootSpanId
|
|
58851
|
-
},
|
|
58852
59901
|
source_artifacts: dropUndefined7({
|
|
58853
59902
|
transcript_path: options.transcriptPath,
|
|
58854
59903
|
grading_path: options.gradingPath,
|
|
58855
59904
|
file_changes_path: options.fileChangesPath
|
|
58856
59905
|
}),
|
|
58857
|
-
|
|
59906
|
+
...buildMetrics(result)
|
|
58858
59907
|
})
|
|
58859
59908
|
);
|
|
58860
59909
|
}
|
|
@@ -58929,7 +59978,7 @@ var TOOL_CALL_ALIASES = {
|
|
|
58929
59978
|
endTime: "end_time",
|
|
58930
59979
|
startTime: "start_time"
|
|
58931
59980
|
};
|
|
58932
|
-
function
|
|
59981
|
+
function isRecord10(value) {
|
|
58933
59982
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
58934
59983
|
}
|
|
58935
59984
|
function normalizeKnownAliases(value, aliases) {
|
|
@@ -58945,13 +59994,13 @@ function normalizeKnownAliases(value, aliases) {
|
|
|
58945
59994
|
return normalized;
|
|
58946
59995
|
}
|
|
58947
59996
|
function normalizeToolCall2(value) {
|
|
58948
|
-
if (!
|
|
59997
|
+
if (!isRecord10(value)) {
|
|
58949
59998
|
return value;
|
|
58950
59999
|
}
|
|
58951
60000
|
return normalizeKnownAliases(value, TOOL_CALL_ALIASES);
|
|
58952
60001
|
}
|
|
58953
60002
|
function normalizeMessage(value) {
|
|
58954
|
-
if (!
|
|
60003
|
+
if (!isRecord10(value)) {
|
|
58955
60004
|
return value;
|
|
58956
60005
|
}
|
|
58957
60006
|
const normalized = normalizeKnownAliases(value, MESSAGE_ALIASES);
|
|
@@ -58961,7 +60010,7 @@ function normalizeMessage(value) {
|
|
|
58961
60010
|
return normalized;
|
|
58962
60011
|
}
|
|
58963
60012
|
function normalizeTraceSummary(value) {
|
|
58964
|
-
if (!
|
|
60013
|
+
if (!isRecord10(value)) {
|
|
58965
60014
|
return value;
|
|
58966
60015
|
}
|
|
58967
60016
|
const normalized = normalizeKnownAliases(value, TRACE_SUMMARY_ALIASES);
|
|
@@ -59003,7 +60052,7 @@ function looksLikeResultRow(value) {
|
|
|
59003
60052
|
return typeof value.test_id === "string" || Object.hasOwn(value, "score") || Object.hasOwn(value, "trace") || Object.hasOwn(value, "spans") || Object.hasOwn(value, "target") || Object.hasOwn(value, "summary_path") || Object.hasOwn(value, "grading_path") || Object.hasOwn(value, "timing_path");
|
|
59004
60053
|
}
|
|
59005
60054
|
function normalizeResultRow(value, context = {}) {
|
|
59006
|
-
if (!
|
|
60055
|
+
if (!isRecord10(value)) {
|
|
59007
60056
|
throw buildSchemaError(context);
|
|
59008
60057
|
}
|
|
59009
60058
|
for (const field of Object.keys(
|
|
@@ -59066,11 +60115,11 @@ function stringField2(record2, key) {
|
|
|
59066
60115
|
}
|
|
59067
60116
|
function resultProjectionDimensions(result) {
|
|
59068
60117
|
const projectionIdentity = result.projectionIdentity;
|
|
59069
|
-
if (!
|
|
60118
|
+
if (!isRecord11(projectionIdentity)) {
|
|
59070
60119
|
return void 0;
|
|
59071
60120
|
}
|
|
59072
60121
|
const dimensions = projectionIdentity.dimensions;
|
|
59073
|
-
return
|
|
60122
|
+
return isRecord11(dimensions) ? dimensions : void 0;
|
|
59074
60123
|
}
|
|
59075
60124
|
function buildEvaluationResultTargetKey(result) {
|
|
59076
60125
|
const dimensions = resultProjectionDimensions(result);
|
|
@@ -59206,7 +60255,7 @@ function normalizeEnvironmentSummaries(value) {
|
|
|
59206
60255
|
return void 0;
|
|
59207
60256
|
}
|
|
59208
60257
|
const summaries = value.filter((entry) => {
|
|
59209
|
-
if (!
|
|
60258
|
+
if (!isRecord11(entry)) {
|
|
59210
60259
|
return false;
|
|
59211
60260
|
}
|
|
59212
60261
|
return entry.schema_version === "agentv.environment_summary.v1" && (entry.type === "host" || entry.type === "docker") && typeof entry.workdir === "string" && typeof entry.recipe_sha256 === "string" && (entry.authored_kind === "inline" || entry.authored_kind === "file");
|
|
@@ -59214,7 +60263,7 @@ function normalizeEnvironmentSummaries(value) {
|
|
|
59214
60263
|
return summaries.length > 0 ? summaries : void 0;
|
|
59215
60264
|
}
|
|
59216
60265
|
function normalizeStringRecord(value) {
|
|
59217
|
-
if (!
|
|
60266
|
+
if (!isRecord11(value)) {
|
|
59218
60267
|
return void 0;
|
|
59219
60268
|
}
|
|
59220
60269
|
const out = {};
|
|
@@ -59290,7 +60339,7 @@ function parseWorkspaceChanges(fileChanges) {
|
|
|
59290
60339
|
}
|
|
59291
60340
|
function resultVerdict(result) {
|
|
59292
60341
|
const scores = result.scores ?? [];
|
|
59293
|
-
if (scores.length > 0 && scores.every((
|
|
60342
|
+
if (scores.length > 0 && scores.every((score2) => score2.verdict === "skip")) {
|
|
59294
60343
|
return "skip";
|
|
59295
60344
|
}
|
|
59296
60345
|
if (result.executionStatus === "ok") {
|
|
@@ -59298,8 +60347,8 @@ function resultVerdict(result) {
|
|
|
59298
60347
|
}
|
|
59299
60348
|
return "fail";
|
|
59300
60349
|
}
|
|
59301
|
-
function passFromVerdict(verdict,
|
|
59302
|
-
return verdict ? verdict === "pass" :
|
|
60350
|
+
function passFromVerdict(verdict, score2) {
|
|
60351
|
+
return verdict ? verdict === "pass" : score2 >= DEFAULT_THRESHOLD2;
|
|
59303
60352
|
}
|
|
59304
60353
|
function assertionMetadata(metadata) {
|
|
59305
60354
|
const compact = dropUndefined8(metadata);
|
|
@@ -59334,10 +60383,10 @@ function assertionToComponent(assertion, parent) {
|
|
|
59334
60383
|
};
|
|
59335
60384
|
}
|
|
59336
60385
|
function checkToComponent(check2, parent) {
|
|
59337
|
-
const
|
|
60386
|
+
const score2 = clampScore(check2.score ?? (check2.pass ? 1 : 0));
|
|
59338
60387
|
return {
|
|
59339
60388
|
pass: check2.pass,
|
|
59340
|
-
score,
|
|
60389
|
+
score: score2,
|
|
59341
60390
|
reason: check2.reason || check2.evidence || check2.text,
|
|
59342
60391
|
assertion: assertionMetadata({
|
|
59343
60392
|
...parent,
|
|
@@ -59359,59 +60408,59 @@ function sanitizePublicGradingMetadata(value) {
|
|
|
59359
60408
|
if (Array.isArray(value)) {
|
|
59360
60409
|
return value.map(sanitizePublicGradingMetadata);
|
|
59361
60410
|
}
|
|
59362
|
-
if (!
|
|
60411
|
+
if (!isRecord11(value)) {
|
|
59363
60412
|
return value;
|
|
59364
60413
|
}
|
|
59365
60414
|
const entries = Object.entries(value).filter(([key]) => !PUBLIC_GRADING_FORBIDDEN_METADATA_KEYS.has(key)).map(([key, entry]) => [key, sanitizePublicGradingMetadata(entry)]).filter(([, entry]) => entry !== void 0);
|
|
59366
60415
|
return entries.length > 0 ? Object.fromEntries(entries) : void 0;
|
|
59367
60416
|
}
|
|
59368
|
-
function scoreMetadata(
|
|
60417
|
+
function scoreMetadata(score2) {
|
|
59369
60418
|
const metadata = dropUndefined8({
|
|
59370
|
-
details: sanitizePublicGradingMetadata(
|
|
59371
|
-
token_usage:
|
|
59372
|
-
duration_ms:
|
|
59373
|
-
started_at:
|
|
59374
|
-
ended_at:
|
|
60419
|
+
details: sanitizePublicGradingMetadata(score2.details),
|
|
60420
|
+
token_usage: score2.tokenUsage,
|
|
60421
|
+
duration_ms: score2.durationMs,
|
|
60422
|
+
started_at: score2.startedAt,
|
|
60423
|
+
ended_at: score2.endedAt
|
|
59375
60424
|
});
|
|
59376
60425
|
return Object.keys(metadata).length > 0 ? metadata : void 0;
|
|
59377
60426
|
}
|
|
59378
|
-
function scoreReason(
|
|
59379
|
-
const reason =
|
|
60427
|
+
function scoreReason(score2, pass) {
|
|
60428
|
+
const reason = score2.reason?.trim();
|
|
59380
60429
|
if (reason) {
|
|
59381
60430
|
return reason;
|
|
59382
60431
|
}
|
|
59383
|
-
const failed = (
|
|
60432
|
+
const failed = (score2.checks ?? []).find((check2) => !check2.pass);
|
|
59384
60433
|
if (failed?.reason) {
|
|
59385
60434
|
return failed.reason;
|
|
59386
60435
|
}
|
|
59387
|
-
const failedAssertion = (
|
|
60436
|
+
const failedAssertion = (score2.assertions ?? []).find((assertion) => !assertion.passed);
|
|
59388
60437
|
if (failedAssertion?.evidence || failedAssertion?.text) {
|
|
59389
60438
|
return failedAssertion.evidence ?? failedAssertion.text;
|
|
59390
60439
|
}
|
|
59391
60440
|
return pass ? "Grader passed." : "Grader failed.";
|
|
59392
60441
|
}
|
|
59393
|
-
function scoreToComponent(
|
|
59394
|
-
const pass = passFromVerdict(
|
|
60442
|
+
function scoreToComponent(score2) {
|
|
60443
|
+
const pass = passFromVerdict(score2.verdict, score2.score);
|
|
59395
60444
|
const assertion = assertionMetadata({
|
|
59396
|
-
name:
|
|
59397
|
-
type:
|
|
59398
|
-
weight:
|
|
59399
|
-
target:
|
|
60445
|
+
name: score2.name,
|
|
60446
|
+
type: score2.type,
|
|
60447
|
+
weight: score2.weight,
|
|
60448
|
+
target: score2.target
|
|
59400
60449
|
});
|
|
59401
60450
|
const componentResults = [
|
|
59402
|
-
...(
|
|
59403
|
-
...(
|
|
59404
|
-
...
|
|
60451
|
+
...(score2.checks ?? []).map((check2) => checkToComponent(check2, assertion)),
|
|
60452
|
+
...(score2.scores ?? []).map(scoreToComponent),
|
|
60453
|
+
...score2.checks && score2.checks.length > 0 ? [] : (score2.assertions ?? []).map((entry) => assertionToComponent(entry, assertion))
|
|
59405
60454
|
];
|
|
59406
|
-
const namedScores = collectNamedScores(
|
|
60455
|
+
const namedScores = collectNamedScores(score2.scores);
|
|
59407
60456
|
return dropUndefined8({
|
|
59408
60457
|
pass,
|
|
59409
|
-
score: clampScore(
|
|
59410
|
-
reason: scoreReason(
|
|
60458
|
+
score: clampScore(score2.score),
|
|
60459
|
+
reason: scoreReason(score2, pass),
|
|
59411
60460
|
component_results: componentResults.length > 0 ? componentResults : void 0,
|
|
59412
60461
|
assertion,
|
|
59413
60462
|
named_scores: namedScores,
|
|
59414
|
-
metadata: scoreMetadata(
|
|
60463
|
+
metadata: scoreMetadata(score2)
|
|
59415
60464
|
});
|
|
59416
60465
|
}
|
|
59417
60466
|
function buildComponentResults(result) {
|
|
@@ -59426,39 +60475,39 @@ function buildComponentResults(result) {
|
|
|
59426
60475
|
}
|
|
59427
60476
|
return void 0;
|
|
59428
60477
|
}
|
|
59429
|
-
function toIndexScore(
|
|
59430
|
-
const pass = passFromVerdict(
|
|
60478
|
+
function toIndexScore(score2) {
|
|
60479
|
+
const pass = passFromVerdict(score2.verdict, score2.score);
|
|
59431
60480
|
return dropUndefined8({
|
|
59432
|
-
name:
|
|
59433
|
-
type:
|
|
60481
|
+
name: score2.name,
|
|
60482
|
+
type: score2.type,
|
|
59434
60483
|
pass,
|
|
59435
|
-
score:
|
|
59436
|
-
reason: scoreReason(
|
|
59437
|
-
weight:
|
|
59438
|
-
target:
|
|
59439
|
-
scores:
|
|
59440
|
-
named_scores: collectNamedScores(
|
|
59441
|
-
token_usage:
|
|
59442
|
-
duration_ms:
|
|
59443
|
-
started_at:
|
|
59444
|
-
ended_at:
|
|
60484
|
+
score: score2.score,
|
|
60485
|
+
reason: scoreReason(score2, pass),
|
|
60486
|
+
weight: score2.weight,
|
|
60487
|
+
target: score2.target,
|
|
60488
|
+
scores: score2.scores?.map(toIndexScore),
|
|
60489
|
+
named_scores: collectNamedScores(score2.scores),
|
|
60490
|
+
token_usage: score2.tokenUsage,
|
|
60491
|
+
duration_ms: score2.durationMs,
|
|
60492
|
+
started_at: score2.startedAt,
|
|
60493
|
+
ended_at: score2.endedAt
|
|
59445
60494
|
});
|
|
59446
60495
|
}
|
|
59447
60496
|
function toIndexScores(scores) {
|
|
59448
60497
|
return scores?.map(toIndexScore);
|
|
59449
60498
|
}
|
|
59450
60499
|
function collectNamedScores(scores, out = {}) {
|
|
59451
|
-
for (const
|
|
59452
|
-
if (
|
|
59453
|
-
out[
|
|
60500
|
+
for (const score2 of scores ?? []) {
|
|
60501
|
+
if (score2.name) {
|
|
60502
|
+
out[score2.name] = score2.score;
|
|
59454
60503
|
}
|
|
59455
|
-
collectNamedScores(
|
|
60504
|
+
collectNamedScores(score2.scores, out);
|
|
59456
60505
|
}
|
|
59457
60506
|
return Object.keys(out).length > 0 ? out : void 0;
|
|
59458
60507
|
}
|
|
59459
60508
|
function resultDerivedMetrics(result) {
|
|
59460
60509
|
const value = result.metadata?.derived_metrics ?? result.metadata?.derivedMetrics;
|
|
59461
|
-
return
|
|
60510
|
+
return isRecord11(value) ? value : void 0;
|
|
59462
60511
|
}
|
|
59463
60512
|
function resultProvenance(result) {
|
|
59464
60513
|
const value = result.metadata?.provenance;
|
|
@@ -59533,11 +60582,11 @@ function toTrialAggregationArtifact(aggregation) {
|
|
|
59533
60582
|
function dropUndefined8(value) {
|
|
59534
60583
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
|
|
59535
60584
|
}
|
|
59536
|
-
function
|
|
60585
|
+
function isRecord11(value) {
|
|
59537
60586
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
59538
60587
|
}
|
|
59539
60588
|
function toIndexRerunSource(value) {
|
|
59540
|
-
if (!
|
|
60589
|
+
if (!isRecord11(value)) {
|
|
59541
60590
|
return void 0;
|
|
59542
60591
|
}
|
|
59543
60592
|
return dropUndefined8({
|
|
@@ -59638,7 +60687,7 @@ function buildRepeatCaseSummaryArtifact(result, timing, fingerprint) {
|
|
|
59638
60687
|
};
|
|
59639
60688
|
}
|
|
59640
60689
|
function toFilePathList(entries) {
|
|
59641
|
-
return entries.map((entry) =>
|
|
60690
|
+
return entries.map((entry) => isRecord11(entry) && typeof entry.path === "string" ? entry.path : void 0).filter((entry) => entry !== void 0);
|
|
59642
60691
|
}
|
|
59643
60692
|
function resultTranscriptProviderId(result) {
|
|
59644
60693
|
const provider = result.trace.metadata?.provider;
|
|
@@ -59715,7 +60764,7 @@ async function writeEnvironmentArtifact(params) {
|
|
|
59715
60764
|
};
|
|
59716
60765
|
}
|
|
59717
60766
|
function buildAgentVRunResultArtifact(params) {
|
|
59718
|
-
const metrics = params.metricsArtifact
|
|
60767
|
+
const metrics = params.metricsArtifact;
|
|
59719
60768
|
const fileChangesPath = params.hasFileChanges ? `./${CANONICAL_FILE_CHANGES_ARTIFACT_PATH}` : void 0;
|
|
59720
60769
|
return dropUndefined8({
|
|
59721
60770
|
execution_status: params.trial.executionStatus ?? params.result.executionStatus,
|
|
@@ -59909,7 +60958,7 @@ async function writeTrialRunArtifacts(params) {
|
|
|
59909
60958
|
);
|
|
59910
60959
|
}
|
|
59911
60960
|
function toIndexPreparedAttempt(value) {
|
|
59912
|
-
if (!
|
|
60961
|
+
if (!isRecord11(value)) {
|
|
59913
60962
|
return void 0;
|
|
59914
60963
|
}
|
|
59915
60964
|
return dropUndefined8({
|
|
@@ -59994,7 +61043,7 @@ function timingMetadataSource(metadata, sourceKey) {
|
|
|
59994
61043
|
const usageSources = metadata?.usage_sources;
|
|
59995
61044
|
const usageSummary = metadata?.usage_summary;
|
|
59996
61045
|
const legacyKey = sourceKey === "duration" ? "duration_source" : sourceKey === "cost" ? "cost_source" : "token_usage_source";
|
|
59997
|
-
const value =
|
|
61046
|
+
const value = isRecord11(usageSources) ? usageSources[sourceKey] : isRecord11(usageSummary) ? usageSummary[legacyKey] : metadata?.[legacyKey];
|
|
59998
61047
|
return typeof value === "string" && TIMING_SOURCE_VALUES2.includes(value) ? value : void 0;
|
|
59999
61048
|
}
|
|
60000
61049
|
function sumMessageTokenUsage(messages) {
|
|
@@ -60152,12 +61201,12 @@ function buildRunSummaryArtifact(results, evalFile = "", experiment, runId, plan
|
|
|
60152
61201
|
if (isExecutionError(result)) {
|
|
60153
61202
|
continue;
|
|
60154
61203
|
}
|
|
60155
|
-
for (const
|
|
60156
|
-
const key = `${
|
|
61204
|
+
for (const score2 of result.scores ?? []) {
|
|
61205
|
+
const key = `${score2.name}:${score2.type}`;
|
|
60157
61206
|
if (!evaluatorScores.has(key)) {
|
|
60158
61207
|
evaluatorScores.set(key, []);
|
|
60159
61208
|
}
|
|
60160
|
-
evaluatorScores.get(key)?.push(
|
|
61209
|
+
evaluatorScores.get(key)?.push(score2.score);
|
|
60161
61210
|
}
|
|
60162
61211
|
}
|
|
60163
61212
|
let perEvaluatorSummary;
|
|
@@ -60323,7 +61372,7 @@ async function writeInitialRunSummaryArtifact(runDir, options) {
|
|
|
60323
61372
|
async function readRunConfigArtifact(runDir) {
|
|
60324
61373
|
try {
|
|
60325
61374
|
const parsed = JSON.parse(await readFile16(runInternalPath(runDir, RUN_CONFIG_FILENAME), "utf8"));
|
|
60326
|
-
if (!
|
|
61375
|
+
if (!isRecord11(parsed) || parsed.schema_version !== "agentv.run_config.v1") {
|
|
60327
61376
|
return void 0;
|
|
60328
61377
|
}
|
|
60329
61378
|
return parsed;
|
|
@@ -60333,7 +61382,7 @@ async function readRunConfigArtifact(runDir) {
|
|
|
60333
61382
|
}
|
|
60334
61383
|
function buildAggregateGradingArtifact(results) {
|
|
60335
61384
|
const qualityResults = results.filter((r) => !isExecutionError(r));
|
|
60336
|
-
const
|
|
61385
|
+
const score2 = qualityResults.length > 0 ? Math.round(
|
|
60337
61386
|
qualityResults.reduce((sum, result) => sum + clampScore(result.score), 0) / qualityResults.length * 1e3
|
|
60338
61387
|
) / 1e3 : 0;
|
|
60339
61388
|
const pass = qualityResults.length > 0 && qualityResults.every((result) => resultVerdict(result) === "pass");
|
|
@@ -60347,7 +61396,7 @@ function buildAggregateGradingArtifact(results) {
|
|
|
60347
61396
|
}));
|
|
60348
61397
|
return dropUndefined8({
|
|
60349
61398
|
pass,
|
|
60350
|
-
score,
|
|
61399
|
+
score: score2,
|
|
60351
61400
|
reason: results.length === 0 ? "No results to summarize." : pass ? "All quality results passed." : "One or more quality results failed.",
|
|
60352
61401
|
component_results: componentResults.length > 0 ? componentResults : void 0,
|
|
60353
61402
|
named_scores: collectNamedScores(qualityResults.flatMap((result) => result.scores ?? [])),
|
|
@@ -60734,7 +61783,7 @@ async function writeMetricsArtifact(params) {
|
|
|
60734
61783
|
return artifactWithTiming;
|
|
60735
61784
|
}
|
|
60736
61785
|
function indexRecordKey(record2) {
|
|
60737
|
-
if (!
|
|
61786
|
+
if (!isRecord11(record2)) {
|
|
60738
61787
|
return void 0;
|
|
60739
61788
|
}
|
|
60740
61789
|
const testId = typeof record2.test_id === "string" ? record2.test_id : typeof record2.testId === "string" ? record2.testId : void 0;
|
|
@@ -60752,7 +61801,7 @@ function indexRecordReplacementKeys(record2) {
|
|
|
60752
61801
|
return Array.from(new Set(keys));
|
|
60753
61802
|
}
|
|
60754
61803
|
function projectionIdentityRecordKey(record2) {
|
|
60755
|
-
if (!
|
|
61804
|
+
if (!isRecord11(record2) || !isRecord11(record2.projection_identity)) {
|
|
60756
61805
|
return void 0;
|
|
60757
61806
|
}
|
|
60758
61807
|
const id = record2.projection_identity.id;
|
|
@@ -60790,7 +61839,7 @@ function existingRecordsByProjectionIdentity(records) {
|
|
|
60790
61839
|
return byIdentity;
|
|
60791
61840
|
}
|
|
60792
61841
|
function skippedExistingRecord(record2, identity, duplicatePolicy) {
|
|
60793
|
-
if (!
|
|
61842
|
+
if (!isRecord11(record2)) {
|
|
60794
61843
|
return record2;
|
|
60795
61844
|
}
|
|
60796
61845
|
return {
|
|
@@ -60869,23 +61918,23 @@ async function readJsonFile2(filePath) {
|
|
|
60869
61918
|
}
|
|
60870
61919
|
}
|
|
60871
61920
|
function summaryRunId(summary, fallback) {
|
|
60872
|
-
if (!
|
|
61921
|
+
if (!isRecord11(summary)) {
|
|
60873
61922
|
return fallback;
|
|
60874
61923
|
}
|
|
60875
|
-
const runId =
|
|
61924
|
+
const runId = isRecord11(summary.metadata) ? summary.metadata.run_id : void 0;
|
|
60876
61925
|
return typeof runId === "string" && runId.trim().length > 0 ? runId : fallback;
|
|
60877
61926
|
}
|
|
60878
61927
|
function summaryTimestamp(summary) {
|
|
60879
|
-
if (!
|
|
61928
|
+
if (!isRecord11(summary) || !isRecord11(summary.metadata)) {
|
|
60880
61929
|
return void 0;
|
|
60881
61930
|
}
|
|
60882
61931
|
return typeof summary.metadata.timestamp === "string" ? summary.metadata.timestamp : void 0;
|
|
60883
61932
|
}
|
|
60884
61933
|
function summaryTargets(summary) {
|
|
60885
|
-
return
|
|
61934
|
+
return isRecord11(summary) && isRecord11(summary.metadata) ? summary.metadata.targets : void 0;
|
|
60886
61935
|
}
|
|
60887
61936
|
function summaryTags(summary) {
|
|
60888
|
-
return
|
|
61937
|
+
return isRecord11(summary) && isRecord11(summary.metadata) ? summary.metadata.tags : void 0;
|
|
60889
61938
|
}
|
|
60890
61939
|
async function readJsonlFile(filePath) {
|
|
60891
61940
|
const content = await readTextIfExists(filePath);
|
|
@@ -60905,7 +61954,7 @@ async function readJsonlFile(filePath) {
|
|
|
60905
61954
|
return records;
|
|
60906
61955
|
}
|
|
60907
61956
|
function buildCrossRunRunRecord(params) {
|
|
60908
|
-
const summary =
|
|
61957
|
+
const summary = isRecord11(params.summary) ? params.summary : {};
|
|
60909
61958
|
return dropUndefined8({
|
|
60910
61959
|
run_id: params.runId,
|
|
60911
61960
|
run_dir: params.runDirName,
|
|
@@ -60920,7 +61969,7 @@ function buildCrossRunRunRecord(params) {
|
|
|
60920
61969
|
});
|
|
60921
61970
|
}
|
|
60922
61971
|
function buildCrossRunCaseRecord(params) {
|
|
60923
|
-
if (!
|
|
61972
|
+
if (!isRecord11(params.caseRecord)) {
|
|
60924
61973
|
return void 0;
|
|
60925
61974
|
}
|
|
60926
61975
|
return dropUndefined8({
|
|
@@ -61895,9 +62944,11 @@ export {
|
|
|
61895
62944
|
extractImageBlocks,
|
|
61896
62945
|
formatToolCalls,
|
|
61897
62946
|
SkillTriggerGrader,
|
|
62947
|
+
SkillUsedGrader,
|
|
61898
62948
|
assembleLlmGraderPrompt,
|
|
61899
62949
|
TokenUsageGrader,
|
|
61900
62950
|
ToolTrajectoryGrader,
|
|
62951
|
+
TrajectoryGrader,
|
|
61901
62952
|
runContainsAssertion,
|
|
61902
62953
|
runContainsAnyAssertion,
|
|
61903
62954
|
runContainsAllAssertion,
|
|
@@ -61981,6 +63032,7 @@ export {
|
|
|
61981
63032
|
CANONICAL_METRICS_ARTIFACT_PATH,
|
|
61982
63033
|
CANONICAL_FILE_CHANGES_ARTIFACT_PATH,
|
|
61983
63034
|
TRANSCRIPT_SCHEMA_VERSION,
|
|
63035
|
+
LEGACY_METRICS_SCHEMA_VERSION,
|
|
61984
63036
|
METRICS_SCHEMA_VERSION,
|
|
61985
63037
|
TRANSCRIPT_JSON_MEDIA_TYPE,
|
|
61986
63038
|
METRICS_JSON_MEDIA_TYPE,
|
|
@@ -62072,6 +63124,7 @@ export {
|
|
|
62072
63124
|
runEvalCase,
|
|
62073
63125
|
MetricsWireSchema,
|
|
62074
63126
|
MetricsArtifactWireSchema,
|
|
63127
|
+
normalizeMetricsArtifactWire,
|
|
62075
63128
|
buildMetricsArtifact,
|
|
62076
63129
|
ResultRowSchemaError,
|
|
62077
63130
|
normalizeResultRow,
|
|
@@ -62104,4 +63157,4 @@ export {
|
|
|
62104
63157
|
chokidar/index.js:
|
|
62105
63158
|
(*! chokidar - MIT License (c) 2012 Paul Miller (paulmillr.com) *)
|
|
62106
63159
|
*/
|
|
62107
|
-
//# sourceMappingURL=chunk-
|
|
63160
|
+
//# sourceMappingURL=chunk-6ZZCDZPD.js.map
|