agentv 5.2.0-next.1 → 5.3.0-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/dist/{artifact-writer-SN6UNL5G.js → artifact-writer-JFNIPMKW.js} +4 -4
  2. package/dist/{chunk-3S6A2RKR.js → chunk-6ZZCDZPD.js} +1357 -304
  3. package/dist/chunk-6ZZCDZPD.js.map +1 -0
  4. package/dist/{chunk-46K2OET3.js → chunk-BHKQHG26.js} +2 -2
  5. package/dist/{chunk-T4AD3H3Y.js → chunk-FVR4RQFK.js} +74 -16
  6. package/dist/chunk-FVR4RQFK.js.map +1 -0
  7. package/dist/{chunk-3X357HS4.js → chunk-T32NL3E6.js} +62 -6
  8. package/dist/chunk-T32NL3E6.js.map +1 -0
  9. package/dist/{chunk-ALVZQUZP.js → chunk-V52ATPTT.js} +2 -2
  10. package/dist/cli.js +5 -5
  11. package/dist/{dist-DEPJOMCA.js → dist-6Z7U473R.js} +11 -3
  12. package/dist/index.js +5 -5
  13. package/dist/{interactive-36ZNMQB2.js → interactive-RUY3OCBI.js} +5 -5
  14. package/dist/skills/agentv-bench/SKILL.md +1 -1
  15. package/dist/skills/agentv-bench/agents/analyzer.md +1 -1
  16. package/dist/skills/agentv-bench/agents/comparator.md +3 -3
  17. package/dist/skills/agentv-bench/agents/grader.md +4 -2
  18. package/dist/skills/agentv-bench/references/description-optimization.md +5 -6
  19. package/dist/skills/agentv-bench/references/environment-adaptation.md +2 -2
  20. package/dist/skills/agentv-bench/references/eval-yaml-spec.md +20 -9
  21. package/dist/skills/agentv-eval-migrations/references/breaking-changes.md +100 -2
  22. package/dist/skills/agentv-eval-writer/SKILL.md +27 -16
  23. package/dist/{ts-eval-loader-OMG2JBHP-OP5RR7A3.js → ts-eval-loader-DQDYRULE-V3377FOL.js} +2 -2
  24. package/package.json +1 -1
  25. package/dist/chunk-3S6A2RKR.js.map +0 -1
  26. package/dist/chunk-3X357HS4.js.map +0 -1
  27. package/dist/chunk-T4AD3H3Y.js.map +0 -1
  28. /package/dist/{artifact-writer-SN6UNL5G.js.map → artifact-writer-JFNIPMKW.js.map} +0 -0
  29. /package/dist/{chunk-46K2OET3.js.map → chunk-BHKQHG26.js.map} +0 -0
  30. /package/dist/{chunk-ALVZQUZP.js.map → chunk-V52ATPTT.js.map} +0 -0
  31. /package/dist/{dist-DEPJOMCA.js.map → dist-6Z7U473R.js.map} +0 -0
  32. /package/dist/{interactive-36ZNMQB2.js.map → interactive-RUY3OCBI.js.map} +0 -0
  33. /package/dist/{ts-eval-loader-OMG2JBHP-OP5RR7A3.js.map → ts-eval-loader-DQDYRULE-V3377FOL.js.map} +0 -0
@@ -8114,7 +8114,7 @@ function toFiniteNumber(value) {
8114
8114
  return void 0;
8115
8115
  }
8116
8116
 
8117
- // ../../packages/core/dist/chunk-3A6FPNWI.js
8117
+ // ../../packages/core/dist/chunk-QMPBDTFI.js
8118
8118
  var SDK_CHILD_PROTOCOL_VERSION = 1;
8119
8119
  function providerRequestToWire(request) {
8120
8120
  if (request.tools && request.tools.length > 0) {
@@ -8146,6 +8146,7 @@ function providerResponseFromWire(response) {
8146
8146
  return {
8147
8147
  raw: response.raw,
8148
8148
  usage: response.usage,
8149
+ metadata: response.metadata,
8149
8150
  output: response.output,
8150
8151
  tokenUsage: response.token_usage,
8151
8152
  costUsd: response.cost_usd,
@@ -12198,7 +12199,7 @@ var coerce = {
12198
12199
  };
12199
12200
  var NEVER = INVALID;
12200
12201
 
12201
- // ../../packages/core/dist/chunk-DYGAVJOU.js
12202
+ // ../../packages/core/dist/chunk-RXN2AWVO.js
12202
12203
  import { parse } from "yaml";
12203
12204
  import path3 from "node:path";
12204
12205
  import path22 from "node:path";
@@ -12292,6 +12293,13 @@ var GRADER_KIND_VALUES = [
12292
12293
  "script",
12293
12294
  "llm-grader",
12294
12295
  "tool-trajectory",
12296
+ "skill-used",
12297
+ "not-skill-used",
12298
+ "trajectory:tool-used",
12299
+ "trajectory:tool-args-match",
12300
+ "trajectory:tool-sequence",
12301
+ "trajectory:step-count",
12302
+ "trajectory:goal-success",
12295
12303
  "field-accuracy",
12296
12304
  "latency",
12297
12305
  "cost",
@@ -15846,6 +15854,141 @@ function subscribeToCodexLogEntries(listener) {
15846
15854
  };
15847
15855
  }
15848
15856
 
15857
+ // ../../packages/core/dist/chunk-7FVITOZM.js
15858
+ var SKILL_PATH_PATTERN = /(?:^|[/"'\s])(?:\.agents\/)?skills\/([^/"'\s]+)\/SKILL\.md(?:$|[)"'\s,;:])/g;
15859
+ function deriveSkillCallsFromMessages(messages) {
15860
+ if (!messages) {
15861
+ return [];
15862
+ }
15863
+ return deriveSkillCallsFromToolCalls(messages.flatMap((message) => message.toolCalls ?? []));
15864
+ }
15865
+ function deriveSkillCallsFromToolCalls(toolCalls) {
15866
+ if (!toolCalls) {
15867
+ return [];
15868
+ }
15869
+ const skillCalls = [];
15870
+ const heuristicKeys = /* @__PURE__ */ new Set();
15871
+ const pushHeuristic = (entry) => {
15872
+ const key = `${entry.name}\0${entry.path ?? ""}\0${entry.source ?? ""}`;
15873
+ if (heuristicKeys.has(key)) {
15874
+ return;
15875
+ }
15876
+ heuristicKeys.add(key);
15877
+ skillCalls.push(entry);
15878
+ };
15879
+ for (const toolCall of toolCalls) {
15880
+ const toolName = toolCall.tool.toLowerCase();
15881
+ const input = asRecord(toolCall.input);
15882
+ const isError = isErroredToolCall(toolCall);
15883
+ if (toolName === "skill") {
15884
+ const skillName = normalizeSkillName(input?.skill ?? input?.name);
15885
+ if (skillName) {
15886
+ skillCalls.push(
15887
+ dropUndefined({ name: skillName, input: toolCall.input, source: "tool", isError })
15888
+ );
15889
+ }
15890
+ continue;
15891
+ }
15892
+ if (toolName === "read") {
15893
+ const path49 = normalizeText(input?.file_path ?? input?.path ?? input?.filePath);
15894
+ const skillName = path49 ? skillNameFromPath(path49) : void 0;
15895
+ if (skillName) {
15896
+ pushHeuristic(
15897
+ dropUndefined({
15898
+ name: skillName,
15899
+ input: toolCall.input,
15900
+ path: path49,
15901
+ source: "heuristic",
15902
+ isError
15903
+ })
15904
+ );
15905
+ }
15906
+ continue;
15907
+ }
15908
+ if (toolName === "bash") {
15909
+ const command = normalizeText(input?.command);
15910
+ if (command) {
15911
+ for (const candidate of skillPathCandidates(command)) {
15912
+ pushHeuristic(
15913
+ dropUndefined({
15914
+ name: candidate.name,
15915
+ input: toolCall.input,
15916
+ path: candidate.path,
15917
+ source: "heuristic",
15918
+ isError
15919
+ })
15920
+ );
15921
+ }
15922
+ }
15923
+ }
15924
+ if (toolCall.output !== void 0) {
15925
+ const outputText = typeof toolCall.output === "string" ? toolCall.output : JSON.stringify(toolCall.output);
15926
+ for (const candidate of skillPathCandidates(outputText)) {
15927
+ pushHeuristic(
15928
+ dropUndefined({
15929
+ name: candidate.name,
15930
+ path: candidate.path,
15931
+ source: "heuristic",
15932
+ isError
15933
+ })
15934
+ );
15935
+ }
15936
+ }
15937
+ }
15938
+ return skillCalls;
15939
+ }
15940
+ function skillCallMetadata(skillCalls) {
15941
+ if (skillCalls.length === 0) {
15942
+ return void 0;
15943
+ }
15944
+ return { skillCalls };
15945
+ }
15946
+ function skillPathCandidates(text) {
15947
+ const candidates = /* @__PURE__ */ new Map();
15948
+ for (const rawToken of text.split(/\s+/)) {
15949
+ const token = rawToken.replace(/^[`"'([{<]+|[`"',;:)\]}>]+$/g, "").replace(/\\/g, "/");
15950
+ const match = token.match(/(?:^|\/)(?:\.agents\/)?skills\/([^/\s]+)\/SKILL\.md$/);
15951
+ if (match && isValidSkillName(match[1])) {
15952
+ candidates.set(token, { name: match[1], path: token });
15953
+ }
15954
+ }
15955
+ for (const match of text.replace(/\\/g, "/").matchAll(SKILL_PATH_PATTERN)) {
15956
+ const pathMatch = match[0].match(/(?:\.agents\/)?skills\/([^/"'\s]+)\/SKILL\.md/);
15957
+ const name = pathMatch?.[1];
15958
+ if (name && isValidSkillName(name)) {
15959
+ candidates.set(pathMatch[0], { name, path: pathMatch[0] });
15960
+ }
15961
+ }
15962
+ return Array.from(candidates.values());
15963
+ }
15964
+ function skillNameFromPath(path49) {
15965
+ const normalized = path49.replace(/\\/g, "/");
15966
+ const match = normalized.match(/(?:^|\/)(?:\.agents\/)?skills\/([^/\s]+)\/SKILL\.md$/);
15967
+ const name = match?.[1];
15968
+ return name && isValidSkillName(name) ? name : void 0;
15969
+ }
15970
+ function isErroredToolCall(toolCall) {
15971
+ if (toolCall.status === void 0 || toolCall.status === "ok" || toolCall.status === "unknown") {
15972
+ return void 0;
15973
+ }
15974
+ return true;
15975
+ }
15976
+ function normalizeSkillName(value) {
15977
+ return normalizeText(value);
15978
+ }
15979
+ function normalizeText(value) {
15980
+ return typeof value === "string" && value.trim() ? value.trim() : void 0;
15981
+ }
15982
+ function asRecord(value) {
15983
+ return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
15984
+ }
15985
+ function isValidSkillName(name) {
15986
+ return /^[A-Za-z0-9._:-]+$/.test(name);
15987
+ }
15988
+ function dropUndefined(value) {
15989
+ return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
15990
+ }
15991
+
15849
15992
  // ../../packages/core/dist/chunk-YRELA7BA.js
15850
15993
  var TOOL_NAME_MAP = /* @__PURE__ */ new Map([
15851
15994
  // --- Claude (already canonical) ---
@@ -16093,7 +16236,7 @@ function pathToFileUri(filePath) {
16093
16236
  return `file://${normalizedPath}`;
16094
16237
  }
16095
16238
 
16096
- // ../../packages/core/dist/chunk-3B7EZ2GI.js
16239
+ // ../../packages/core/dist/chunk-WBZHD7AZ.js
16097
16240
  import path48 from "node:path";
16098
16241
  import { pathToFileURL as pathToFileURL4 } from "node:url";
16099
16242
  import { existsSync as existsSync6 } from "node:fs";
@@ -16120,6 +16263,7 @@ import path42 from "node:path";
16120
16263
  import { readFile as readFile23 } from "node:fs/promises";
16121
16264
  import path33 from "node:path";
16122
16265
  import { fileURLToPath as fileURLToPath4 } from "node:url";
16266
+ import { isDeepStrictEqual } from "node:util";
16123
16267
  import path52 from "node:path";
16124
16268
  import { pathToFileURL as pathToFileURL22 } from "node:url";
16125
16269
  import { spawn } from "node:child_process";
@@ -28763,7 +28907,7 @@ var RequestError = class _RequestError extends Error {
28763
28907
  }
28764
28908
  };
28765
28909
 
28766
- // ../../packages/core/dist/chunk-3B7EZ2GI.js
28910
+ // ../../packages/core/dist/chunk-WBZHD7AZ.js
28767
28911
  import { randomUUID as randomUUID5 } from "node:crypto";
28768
28912
  import { createWriteStream as createWriteStream3 } from "node:fs";
28769
28913
  import { mkdir as mkdir6, mkdtemp as mkdtemp3, rm as rm3, writeFile as writeFile4 } from "node:fs/promises";
@@ -28912,8 +29056,8 @@ function shouldSkipCacheForTemperature(targetConfig) {
28912
29056
  return false;
28913
29057
  }
28914
29058
  var DEFAULT_THRESHOLD2 = 0.8;
28915
- function scoreToVerdict(score, threshold = DEFAULT_THRESHOLD2) {
28916
- return score >= threshold ? "pass" : "fail";
29059
+ function scoreToVerdict(score2, threshold = DEFAULT_THRESHOLD2) {
29060
+ return score2 >= threshold ? "pass" : "fail";
28917
29061
  }
28918
29062
  function clampScore(value) {
28919
29063
  if (Number.isNaN(value) || !Number.isFinite(value)) {
@@ -28984,12 +29128,12 @@ var NEGATED_VERDICT = {
28984
29128
  fail: "pass",
28985
29129
  skip: "skip"
28986
29130
  };
28987
- function negateScore(score) {
29131
+ function negateScore(score2) {
28988
29132
  return {
28989
- ...score,
28990
- score: clampScore(1 - score.score),
28991
- verdict: NEGATED_VERDICT[score.verdict],
28992
- assertions: score.assertions.map((a) => ({
29133
+ ...score2,
29134
+ score: clampScore(1 - score2.score),
29135
+ verdict: NEGATED_VERDICT[score2.verdict],
29136
+ assertions: score2.assertions.map((a) => ({
28993
29137
  ...a,
28994
29138
  passed: !a.passed,
28995
29139
  evidence: a.evidence ? `[Negated] ${a.evidence}` : void 0
@@ -29905,14 +30049,14 @@ function checksToAssertions(checks) {
29905
30049
  }
29906
30050
  function normalizeScriptProtocol(parsed) {
29907
30051
  const checks = parseScriptChecks(parsed.checks);
29908
- const score = optionalScore(parsed, "score") ?? (checks.length > 0 ? checks.reduce((sum, check2) => sum + (check2.score ?? (check2.pass ? 1 : 0)), 0) / checks.length : typeof parsed.pass === "boolean" ? parsed.pass ? 1 : 0 : 0);
29909
- const pass = typeof parsed.pass === "boolean" ? parsed.pass : scoreToVerdict(score) === "pass";
30052
+ const score2 = optionalScore(parsed, "score") ?? (checks.length > 0 ? checks.reduce((sum, check2) => sum + (check2.score ?? (check2.pass ? 1 : 0)), 0) / checks.length : typeof parsed.pass === "boolean" ? parsed.pass ? 1 : 0 : 0);
30053
+ const pass = typeof parsed.pass === "boolean" ? parsed.pass : scoreToVerdict(score2) === "pass";
29910
30054
  const reason = optionalString(parsed, "reason");
29911
30055
  const details = parsed.details && typeof parsed.details === "object" && !Array.isArray(parsed.details) ? parsed.details : void 0;
29912
30056
  if (typeof parsed.pass !== "boolean" && typeof parsed.score !== "number" && checks.length === 0) {
29913
30057
  throw new Error("Script evaluator JSON must include pass, score, or checks[]");
29914
30058
  }
29915
- return { pass, score, reason, checks, details };
30059
+ return { pass, score: score2, reason, checks, details };
29916
30060
  }
29917
30061
  var ScriptGrader = class {
29918
30062
  kind = "script";
@@ -30051,8 +30195,8 @@ var ScriptGrader = class {
30051
30195
  const protocol = parsed != null ? normalizeScriptProtocol(parsed) : void 0;
30052
30196
  const checks = protocol?.checks ?? [];
30053
30197
  const assertions = protocol != null ? checksToAssertions(checks) : [{ text: stdout.trim() || (passed ? "exit 0" : `exit ${exitCode}`), passed }];
30054
- const score = protocol?.score ?? (passed ? 1 : 0);
30055
- const verdict = protocol ? protocol.pass ? "pass" : "fail" : scoreToVerdict(score);
30198
+ const score2 = protocol?.score ?? (passed ? 1 : 0);
30199
+ const verdict = protocol ? protocol.pass ? "pass" : "fail" : scoreToVerdict(score2);
30056
30200
  const reason = protocol?.reason;
30057
30201
  const details = protocol?.details;
30058
30202
  const proxyUsage = getProxyUsage?.();
@@ -30067,7 +30211,7 @@ var ScriptGrader = class {
30067
30211
  } : {}
30068
30212
  };
30069
30213
  return {
30070
- score,
30214
+ score: score2,
30071
30215
  verdict,
30072
30216
  reason,
30073
30217
  checks,
@@ -30174,10 +30318,10 @@ var CostGrader = class {
30174
30318
  };
30175
30319
  }
30176
30320
  const passed = costUsd <= budget;
30177
- const score = passed ? 1 : 0;
30321
+ const score2 = passed ? 1 : 0;
30178
30322
  const formatCost = (n) => `$${n.toFixed(4)}`;
30179
30323
  return {
30180
- score,
30324
+ score: score2,
30181
30325
  verdict: passed ? "pass" : "fail",
30182
30326
  assertions: [
30183
30327
  passed ? { text: `Cost ${formatCost(costUsd)} <= ${formatCost(budget)} budget`, passed: true } : { text: `Cost ${formatCost(costUsd)} > ${formatCost(budget)} budget`, passed: false }
@@ -31279,10 +31423,10 @@ var ExecutionMetricsGrader = class {
31279
31423
  }
31280
31424
  const totalChecks = assertions.length;
31281
31425
  const passedCount = assertions.filter((a) => a.passed).length;
31282
- const score = totalChecks > 0 ? passedCount / totalChecks : 0;
31426
+ const score2 = totalChecks > 0 ? passedCount / totalChecks : 0;
31283
31427
  return {
31284
- score,
31285
- verdict: scoreToVerdict(score),
31428
+ score: score2,
31429
+ verdict: scoreToVerdict(score2),
31286
31430
  assertions,
31287
31431
  expectedAspectCount: totalChecks || 1,
31288
31432
  graderRawRequest: {
@@ -31618,22 +31762,22 @@ var FieldAccuracyGrader = class {
31618
31762
  for (const result of results) {
31619
31763
  assertions.push({ text: result.message, passed: result.hit });
31620
31764
  }
31621
- let score;
31765
+ let score2;
31622
31766
  if (aggregation === "all_or_nothing") {
31623
31767
  const hasFailed = assertions.some((a) => !a.passed);
31624
- score = hasFailed ? 0 : 1;
31768
+ score2 = hasFailed ? 0 : 1;
31625
31769
  } else {
31626
31770
  const totalWeight = results.reduce((sum, r) => sum + r.weight, 0);
31627
31771
  if (totalWeight === 0) {
31628
- score = results.length === 0 ? 1 : 0;
31772
+ score2 = results.length === 0 ? 1 : 0;
31629
31773
  } else {
31630
31774
  const weightedSum = results.reduce((sum, r) => sum + r.score * r.weight, 0);
31631
- score = weightedSum / totalWeight;
31775
+ score2 = weightedSum / totalWeight;
31632
31776
  }
31633
31777
  }
31634
31778
  return {
31635
- score: clampScore(score),
31636
- verdict: scoreToVerdict(score),
31779
+ score: clampScore(score2),
31780
+ verdict: scoreToVerdict(score2),
31637
31781
  assertions,
31638
31782
  expectedAspectCount: results.length
31639
31783
  };
@@ -31752,9 +31896,9 @@ var LatencyGrader = class {
31752
31896
  };
31753
31897
  }
31754
31898
  const passed = durationMs <= threshold;
31755
- const score = passed ? 1 : 0;
31899
+ const score2 = passed ? 1 : 0;
31756
31900
  return {
31757
- score,
31901
+ score: score2,
31758
31902
  verdict: passed ? "pass" : "fail",
31759
31903
  assertions: [
31760
31904
  passed ? { text: `Duration ${durationMs}ms <= ${threshold}ms threshold`, passed: true } : { text: `Duration ${durationMs}ms > ${threshold}ms threshold`, passed: false }
@@ -32139,12 +32283,12 @@ function promptfooFallbackText(config2, fallbackText) {
32139
32283
  }
32140
32284
  return fallbackText;
32141
32285
  }
32142
- function promptfooFallbackReason(pass, score, threshold) {
32286
+ function promptfooFallbackReason(pass, score2, threshold) {
32143
32287
  if (pass) {
32144
32288
  return "Grading passed";
32145
32289
  }
32146
- if (threshold !== void 0 && score < threshold) {
32147
- return `Score ${score} below threshold ${threshold}`;
32290
+ if (threshold !== void 0 && score2 < threshold) {
32291
+ return `Score ${score2} below threshold ${threshold}`;
32148
32292
  }
32149
32293
  return "Grading failed";
32150
32294
  }
@@ -32187,14 +32331,14 @@ function normalizePromptfooGradingResult(options) {
32187
32331
  });
32188
32332
  const threshold = promptfooThreshold(options.config);
32189
32333
  const rawPass = data.pass !== void 0 ? normalizePromptfooPass(data.pass) : normalizedChecks && normalizedChecks.length > 0 ? normalizedChecks.every((check2) => check2.pass) : true;
32190
- const score = data.score !== void 0 ? normalizePromptfooScore(data.score, rawPass) : normalizedChecks && normalizedChecks.length > 0 ? normalizedChecks.reduce((total, check2) => {
32334
+ const score2 = data.score !== void 0 ? normalizePromptfooScore(data.score, rawPass) : normalizedChecks && normalizedChecks.length > 0 ? normalizedChecks.reduce((total, check2) => {
32191
32335
  if (typeof check2.score === "number") {
32192
32336
  return total + check2.score;
32193
32337
  }
32194
32338
  return total + Number(check2.pass);
32195
32339
  }, 0) / normalizedChecks.length : normalizePromptfooScore(void 0, rawPass);
32196
- const pass = threshold !== void 0 ? rawPass && score >= threshold : rawPass;
32197
- const reason = data.reason?.trim() || promptfooFallbackReason(pass, score, threshold);
32340
+ const pass = threshold !== void 0 ? rawPass && score2 >= threshold : rawPass;
32341
+ const reason = data.reason?.trim() || promptfooFallbackReason(pass, score2, threshold);
32198
32342
  const assertions = normalizedChecks && normalizedChecks.length > 0 ? normalizedChecks.map((check2) => ({
32199
32343
  text: check2.text,
32200
32344
  passed: check2.pass,
@@ -32207,12 +32351,12 @@ function normalizePromptfooGradingResult(options) {
32207
32351
  }
32208
32352
  ];
32209
32353
  return {
32210
- score,
32354
+ score: score2,
32211
32355
  verdict: pass ? "pass" : "fail",
32212
32356
  assertions,
32213
32357
  details: {
32214
32358
  pass,
32215
- score,
32359
+ score: score2,
32216
32360
  reason,
32217
32361
  ...threshold !== void 0 ? { threshold } : {},
32218
32362
  ...normalizedChecks && normalizedChecks.length > 0 ? { checks: normalizedChecks } : {}
@@ -32318,11 +32462,11 @@ ${context.toolCalls}`;
32318
32462
  });
32319
32463
  }
32320
32464
  const data2 = freeformEvaluationSchema.parse(raw);
32321
- const score = clampScore(data2.score);
32465
+ const score2 = clampScore(data2.score);
32322
32466
  const assertions = Array.isArray(data2.assertions) ? data2.assertions.slice(0, 8) : [];
32323
32467
  return {
32324
- score,
32325
- verdict: scoreToVerdict(score),
32468
+ score: score2,
32469
+ verdict: scoreToVerdict(score2),
32326
32470
  assertions,
32327
32471
  details: data2.details
32328
32472
  };
@@ -32436,9 +32580,9 @@ ${context.toolCalls}`;
32436
32580
  schema: scoreRangeEvaluationSchema,
32437
32581
  images
32438
32582
  });
32439
- const { score, verdict, assertions, details } = calculateScoreRangeResult(data, rubrics);
32583
+ const { score: score2, verdict, assertions, details } = calculateScoreRangeResult(data, rubrics);
32440
32584
  return {
32441
- score,
32585
+ score: score2,
32442
32586
  verdict,
32443
32587
  assertions,
32444
32588
  expectedAspectCount: rubrics.length,
@@ -32774,9 +32918,9 @@ ${outputSchema}`;
32774
32918
  };
32775
32919
  }
32776
32920
  const data2 = rubricEvaluationSchema.parse(parsed);
32777
- const { score: score2, verdict, assertions: assertions2 } = calculateRubricScore(data2, rubrics);
32921
+ const { score: score3, verdict, assertions: assertions2 } = calculateRubricScore(data2, rubrics);
32778
32922
  return {
32779
- score: score2,
32923
+ score: score3,
32780
32924
  verdict,
32781
32925
  assertions: assertions2,
32782
32926
  expectedAspectCount: rubrics.length,
@@ -32802,11 +32946,11 @@ ${outputSchema}`;
32802
32946
  };
32803
32947
  }
32804
32948
  const data = freeformEvaluationSchema.parse(parsed);
32805
- const score = clampScore(data.score);
32949
+ const score2 = clampScore(data.score);
32806
32950
  const assertions = Array.isArray(data.assertions) ? data.assertions.slice(0, 8) : [];
32807
32951
  return {
32808
- score,
32809
- verdict: scoreToVerdict(score),
32952
+ score: score2,
32953
+ verdict: scoreToVerdict(score2),
32810
32954
  assertions,
32811
32955
  expectedAspectCount: Math.max(assertions.length, 1),
32812
32956
  graderRawRequest,
@@ -33131,9 +33275,9 @@ function calculateRubricScore(result, rubrics) {
33131
33275
  failedRequired = true;
33132
33276
  }
33133
33277
  }
33134
- const score = totalWeight > 0 ? Math.min(1, Math.max(0, earnedWeight / totalWeight)) : 0;
33135
- const verdict = failedRequired ? "fail" : scoreToVerdict(score);
33136
- return { score, verdict, assertions };
33278
+ const score2 = totalWeight > 0 ? Math.min(1, Math.max(0, earnedWeight / totalWeight)) : 0;
33279
+ const verdict = failedRequired ? "fail" : scoreToVerdict(score2);
33280
+ return { score: score2, verdict, assertions };
33137
33281
  }
33138
33282
  function buildScoreRangeFormatInstructions() {
33139
33283
  return `You must return a valid JSON object matching this schema:
@@ -33190,10 +33334,10 @@ function calculateScoreRangeResult(result, rubrics) {
33190
33334
  evidence: check2.reasoning
33191
33335
  });
33192
33336
  }
33193
- const score = totalWeight > 0 ? Math.min(1, Math.max(0, weightedScoreSum / totalWeight)) : 0;
33194
- const verdict = failedRequired ? "fail" : scoreToVerdict(score);
33337
+ const score2 = totalWeight > 0 ? Math.min(1, Math.max(0, weightedScoreSum / totalWeight)) : 0;
33338
+ const verdict = failedRequired ? "fail" : scoreToVerdict(score2);
33195
33339
  return {
33196
- score,
33340
+ score: score2,
33197
33341
  verdict,
33198
33342
  assertions,
33199
33343
  details: {
@@ -33473,6 +33617,174 @@ var SkillTriggerGrader = class {
33473
33617
  };
33474
33618
  }
33475
33619
  };
33620
+ var SkillUsedGrader = class {
33621
+ constructor(config2) {
33622
+ this.config = config2;
33623
+ this.kind = config2.type;
33624
+ }
33625
+ kind;
33626
+ evaluate(context) {
33627
+ const inverse = this.config.type === "not-skill-used" || this.config.negate === true;
33628
+ const skillCalls = getSkillCalls(context);
33629
+ const actualSkills = skillCalls.map(formatSkillCall);
33630
+ const expected = resolveSkillMatchers(this.config.value);
33631
+ if (expected.kind === "list") {
33632
+ return handleListSkillAssertion({
33633
+ config: this.config,
33634
+ inverse,
33635
+ skillCalls,
33636
+ actualSkills,
33637
+ expected
33638
+ });
33639
+ }
33640
+ return handleCountSkillAssertion({
33641
+ config: this.config,
33642
+ inverse,
33643
+ skillCalls,
33644
+ actualSkills,
33645
+ matcher: expected.matcher
33646
+ });
33647
+ }
33648
+ };
33649
+ function getSkillCalls(context) {
33650
+ const responseMetadata = context.responseMetadata ?? {};
33651
+ const metadata = responseMetadata.metadata;
33652
+ const rawSkillCalls = responseMetadata.skill_calls ?? responseMetadata.skillCalls ?? (isRecord2(metadata) ? metadata.skillCalls ?? metadata.skill_calls : void 0);
33653
+ if (!Array.isArray(rawSkillCalls)) {
33654
+ return [];
33655
+ }
33656
+ return rawSkillCalls.filter(
33657
+ (entry) => isRecord2(entry) && typeof entry.name === "string" && entry.name.trim().length > 0 && entry.isError !== true && entry.is_error !== true
33658
+ );
33659
+ }
33660
+ function matchesSkill(skillCall, matcher) {
33661
+ if (matcher.name && skillCall.name !== matcher.name) {
33662
+ return false;
33663
+ }
33664
+ if (matcher.pattern && !matchesPattern(skillCall.name, matcher.pattern)) {
33665
+ return false;
33666
+ }
33667
+ return true;
33668
+ }
33669
+ function formatSkillCall(skillCall) {
33670
+ const details = [skillCall.source, skillCall.path].filter(Boolean).join(", ");
33671
+ return details ? `${skillCall.name} (${details})` : skillCall.name;
33672
+ }
33673
+ function resolveSkillMatchers(value) {
33674
+ const normalizeText2 = (text) => typeof text === "string" ? text.trim() : void 0;
33675
+ const validateCount = (field, count) => {
33676
+ if (!Number.isFinite(count) || !Number.isInteger(count) || count < 0) {
33677
+ throw new Error(`skill-used assertion object ${field} must be a finite non-negative integer`);
33678
+ }
33679
+ };
33680
+ if (typeof value === "string" && value.trim()) {
33681
+ const name = value.trim();
33682
+ return {
33683
+ kind: "list",
33684
+ matchers: [{ name }]
33685
+ };
33686
+ }
33687
+ if (Array.isArray(value) && value.length > 0 && value.every((item) => typeof item === "string" && item.trim())) {
33688
+ return {
33689
+ kind: "list",
33690
+ matchers: value.map((item) => ({ name: item.trim() }))
33691
+ };
33692
+ }
33693
+ if (isRecord2(value)) {
33694
+ const name = normalizeText2(value.name);
33695
+ const pattern = normalizeText2(value.pattern);
33696
+ if (!name && !pattern) {
33697
+ throw new Error("skill-used assertion object must include a name or pattern property");
33698
+ }
33699
+ if ("min" in value) {
33700
+ validateCount("min", value.min);
33701
+ }
33702
+ if ("max" in value) {
33703
+ validateCount("max", value.max);
33704
+ }
33705
+ if (typeof value.min === "number" && typeof value.max === "number" && value.max < value.min) {
33706
+ throw new Error("skill-used assertion object max must be greater than or equal to min");
33707
+ }
33708
+ return {
33709
+ kind: "count",
33710
+ matcher: {
33711
+ max: typeof value.max === "number" ? value.max : void 0,
33712
+ min: typeof value.min === "number" ? value.min : void 0,
33713
+ name,
33714
+ pattern
33715
+ }
33716
+ };
33717
+ }
33718
+ throw new Error("skill-used assertion must have a string, string array, or object value");
33719
+ }
33720
+ function handleListSkillAssertion(params) {
33721
+ const { config: config2, inverse, skillCalls, actualSkills, expected } = params;
33722
+ const missing = expected.matchers.filter(
33723
+ (matcher) => !skillCalls.some((skillCall) => matchesSkill(skillCall, matcher))
33724
+ );
33725
+ const matched = expected.matchers.filter(
33726
+ (matcher) => skillCalls.some((skillCall) => matchesSkill(skillCall, matcher))
33727
+ );
33728
+ const pass = inverse ? matched.length === 0 : missing.length === 0;
33729
+ const expectedSkills = expected.matchers.map((matcher) => matcher.name);
33730
+ const actualSummary = actualSkills.length > 0 ? actualSkills.join(", ") : "(none)";
33731
+ let reason;
33732
+ if (inverse) {
33733
+ reason = pass ? `Forbidden skill(s) were not used: ${expectedSkills.join(", ")}` : `Forbidden skill(s) were used: ${matched.map((matcher) => matcher.name).join(", ")}. Actual skills: ${actualSummary}`;
33734
+ } else if (pass) {
33735
+ reason = `Observed required skill(s): ${expectedSkills.join(", ")}. Actual skills: ${actualSummary}`;
33736
+ } else {
33737
+ reason = `Missing required skill(s): ${missing.map((matcher) => matcher.name).join(", ")}. Actual skills: ${actualSummary}`;
33738
+ }
33739
+ return skillScore(config2, pass, reason);
33740
+ }
33741
+ function handleCountSkillAssertion(params) {
33742
+ const { config: config2, inverse, skillCalls, actualSkills, matcher } = params;
33743
+ const hasExplicitMin = matcher.min !== void 0;
33744
+ const hasExplicitMax = matcher.max !== void 0;
33745
+ const min = matcher.min ?? (hasExplicitMax ? 0 : 1);
33746
+ const max = matcher.max;
33747
+ const matchingSkillCalls = skillCalls.filter((skillCall) => matchesSkill(skillCall, matcher));
33748
+ const count = matchingSkillCalls.length;
33749
+ const matcherLabel = matcher.pattern || matcher.name || "*";
33750
+ if (inverse) {
33751
+ if (hasExplicitMin || hasExplicitMax && max !== 0) {
33752
+ throw new Error(
33753
+ "not-skill-used object assertions only support name/pattern with no count bounds, or max: 0"
33754
+ );
33755
+ }
33756
+ const pass2 = count === 0;
33757
+ const actualSummary = actualSkills.length > 0 ? actualSkills.join(", ") : "(none)";
33758
+ const reason2 = pass2 ? `Forbidden skill "${matcherLabel}" was not used. Actual skills: ${actualSummary}` : `Forbidden skill "${matcherLabel}" was used ${count} time(s). Matches: ${matchingSkillCalls.map(formatSkillCall).join(", ")}`;
33759
+ return skillScore(config2, pass2, reason2);
33760
+ }
33761
+ const pass = count >= min && (max === void 0 || count <= max);
33762
+ let reason = `Matched skill "${matcherLabel}" ${count} time(s)`;
33763
+ reason += max === void 0 ? ` (expected at least ${min})` : ` (expected ${min}-${max})`;
33764
+ if (matchingSkillCalls.length > 0) {
33765
+ reason += `. Matches: ${matchingSkillCalls.map(formatSkillCall).join(", ")}`;
33766
+ }
33767
+ return skillScore(config2, pass, reason);
33768
+ }
33769
+ function skillScore(config2, pass, reason) {
33770
+ return {
33771
+ score: pass ? 1 : 0,
33772
+ verdict: pass ? "pass" : "fail",
33773
+ reason,
33774
+ assertions: [{ text: reason, passed: pass }],
33775
+ expectedAspectCount: 1,
33776
+ details: {
33777
+ assertion_type: config2.type
33778
+ }
33779
+ };
33780
+ }
33781
+ function matchesPattern(skillName, pattern) {
33782
+ const regexPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".");
33783
+ return new RegExp(`^${regexPattern}$`, "i").test(skillName);
33784
+ }
33785
+ function isRecord2(value) {
33786
+ return value !== null && typeof value === "object" && !Array.isArray(value);
33787
+ }
33476
33788
  function stringifyPretty2(value) {
33477
33789
  return value === void 0 ? "" : JSON.stringify(value, null, 2);
33478
33790
  }
@@ -33952,10 +34264,10 @@ var ToolTrajectoryGrader = class {
33952
34264
  }
33953
34265
  }
33954
34266
  const passedCount = assertions.filter((a) => a.passed).length;
33955
- const score = passedCount / toolNames.length;
34267
+ const score2 = passedCount / toolNames.length;
33956
34268
  return {
33957
- score,
33958
- verdict: scoreToVerdict(score),
34269
+ score: score2,
34270
+ verdict: scoreToVerdict(score2),
33959
34271
  assertions,
33960
34272
  expectedAspectCount: toolNames.length
33961
34273
  };
@@ -34038,10 +34350,10 @@ var ToolTrajectoryGrader = class {
34038
34350
  }
34039
34351
  const effectiveLatencyAssertions = latencyAssertionCount - latencySkips;
34040
34352
  const totalAssertions = expected.length + effectiveLatencyAssertions;
34041
- const score = totalAssertions > 0 ? (sequenceHits + latencyHits) / totalAssertions : 1;
34353
+ const score2 = totalAssertions > 0 ? (sequenceHits + latencyHits) / totalAssertions : 1;
34042
34354
  return {
34043
- score,
34044
- verdict: scoreToVerdict(score),
34355
+ score: score2,
34356
+ verdict: scoreToVerdict(score2),
34045
34357
  assertions,
34046
34358
  expectedAspectCount: totalAssertions
34047
34359
  };
@@ -34120,10 +34432,10 @@ var ToolTrajectoryGrader = class {
34120
34432
  }
34121
34433
  const effectiveLatencyAssertions = latencyAssertionCount - latencySkips;
34122
34434
  const totalAssertions = expected.length + effectiveLatencyAssertions;
34123
- const score = totalAssertions > 0 ? (sequenceHits + latencyHits) / totalAssertions : 1;
34435
+ const score2 = totalAssertions > 0 ? (sequenceHits + latencyHits) / totalAssertions : 1;
34124
34436
  return {
34125
- score,
34126
- verdict: scoreToVerdict(score),
34437
+ score: score2,
34438
+ verdict: scoreToVerdict(score2),
34127
34439
  assertions,
34128
34440
  expectedAspectCount: totalAssertions
34129
34441
  };
@@ -34168,10 +34480,10 @@ var ToolTrajectoryGrader = class {
34168
34480
  }
34169
34481
  }
34170
34482
  const passedCount = assertions.filter((a) => a.passed).length;
34171
- const score = expected.length > 0 ? passedCount / expected.length : 1;
34483
+ const score2 = expected.length > 0 ? passedCount / expected.length : 1;
34172
34484
  return {
34173
- score,
34174
- verdict: scoreToVerdict(score),
34485
+ score: score2,
34486
+ verdict: scoreToVerdict(score2),
34175
34487
  assertions,
34176
34488
  expectedAspectCount: expected.length
34177
34489
  };
@@ -34236,15 +34548,608 @@ var ToolTrajectoryGrader = class {
34236
34548
  }
34237
34549
  }
34238
34550
  const passedCount = assertions.filter((a) => a.passed).length;
34239
- const score = toolCalls.length > 0 ? passedCount / toolCalls.length : 1;
34551
+ const score2 = toolCalls.length > 0 ? passedCount / toolCalls.length : 1;
34240
34552
  return {
34241
- score,
34242
- verdict: scoreToVerdict(score),
34553
+ score: score2,
34554
+ verdict: scoreToVerdict(score2),
34243
34555
  assertions,
34244
34556
  expectedAspectCount: toolCalls.length
34245
34557
  };
34246
34558
  }
34247
34559
  };
34560
+ var COMMAND_TOOL_NAMES = /* @__PURE__ */ new Set(["exec_command", "local_shell", "shell"]);
34561
+ var SEARCH_NAME_PATTERN = /(^|[\s._:/-])(search|find|lookup|retriev(?:e|al))($|[\s._:/-])/i;
34562
+ function isRecord22(value) {
34563
+ return typeof value === "object" && value !== null && !Array.isArray(value);
34564
+ }
34565
+ function matchesPattern2(value, pattern) {
34566
+ const regexPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".");
34567
+ return new RegExp(`^${regexPattern}$`, "i").test(value);
34568
+ }
34569
+ function normalizeStructuredValue(value) {
34570
+ if (value === void 0 || value === null) {
34571
+ return void 0;
34572
+ }
34573
+ if (typeof value !== "string") {
34574
+ return value;
34575
+ }
34576
+ const trimmed = value.trim();
34577
+ if (!trimmed) {
34578
+ return void 0;
34579
+ }
34580
+ try {
34581
+ return JSON.parse(trimmed);
34582
+ } catch {
34583
+ return trimmed;
34584
+ }
34585
+ }
34586
+ function getFirstString(values) {
34587
+ for (const value of values) {
34588
+ if (typeof value === "string" && value.trim()) {
34589
+ return value.trim();
34590
+ }
34591
+ }
34592
+ return void 0;
34593
+ }
34594
+ function commandFromInput(input) {
34595
+ if (!isRecord22(input)) {
34596
+ return void 0;
34597
+ }
34598
+ const command = input.cmd ?? input.command ?? input.commands;
34599
+ if (typeof command === "string" && command.trim()) {
34600
+ return command.trim();
34601
+ }
34602
+ if (Array.isArray(command)) {
34603
+ const joined = command.map((part) => String(part).trim()).filter(Boolean).join(command === input.commands ? "; " : " ");
34604
+ return joined || void 0;
34605
+ }
34606
+ return void 0;
34607
+ }
34608
+ function executableFromCommand(command) {
34609
+ return command.trim().split(/\s+/)[0] || void 0;
34610
+ }
34611
+ function classifyMessageStep(event) {
34612
+ const itemType = event.metadata?.["codex.item.type"] ?? event.message?.metadata?.["codex.item.type"];
34613
+ if (itemType === "reasoning") {
34614
+ return "reasoning";
34615
+ }
34616
+ return "message";
34617
+ }
34618
+ function eventToTrajectoryStep(event) {
34619
+ const attributes = {
34620
+ ...event.metadata ?? {},
34621
+ ...event.message?.metadata ?? {},
34622
+ ...event.tool?.metadata ?? {}
34623
+ };
34624
+ if (event.type === "tool_call" && event.tool) {
34625
+ const toolName = event.tool.name;
34626
+ const args = normalizeStructuredValue(event.tool.input);
34627
+ const command = COMMAND_TOOL_NAMES.has(toolName.toLowerCase()) ? commandFromInput(args) : void 0;
34628
+ if (command) {
34629
+ const aliases = /* @__PURE__ */ new Set([command, toolName]);
34630
+ const executable = executableFromCommand(command);
34631
+ if (executable) {
34632
+ aliases.add(executable);
34633
+ }
34634
+ return {
34635
+ aliases: [...aliases],
34636
+ ...args === void 0 ? {} : { args },
34637
+ attributes,
34638
+ name: command,
34639
+ spanName: toolName,
34640
+ startTime: event.timestamp,
34641
+ type: "command"
34642
+ };
34643
+ }
34644
+ return {
34645
+ aliases: [toolName],
34646
+ ...args === void 0 ? {} : { args },
34647
+ attributes,
34648
+ name: toolName,
34649
+ spanName: toolName,
34650
+ startTime: event.timestamp,
34651
+ type: "tool"
34652
+ };
34653
+ }
34654
+ const messageName = event.message?.name ?? event.type;
34655
+ const query = getFirstString([event.metadata?.query, event.metadata?.search_query]);
34656
+ if (query && SEARCH_NAME_PATTERN.test(messageName)) {
34657
+ return {
34658
+ aliases: [query, messageName],
34659
+ attributes,
34660
+ name: query,
34661
+ spanName: messageName,
34662
+ startTime: event.timestamp,
34663
+ type: "search"
34664
+ };
34665
+ }
34666
+ const messageType = event.type === "message" || event.type === "final_response";
34667
+ const type = messageType ? classifyMessageStep(event) : "span";
34668
+ return {
34669
+ aliases: [messageName, type],
34670
+ attributes,
34671
+ name: messageName,
34672
+ spanName: messageName,
34673
+ startTime: event.timestamp,
34674
+ type
34675
+ };
34676
+ }
34677
+ function extractTrajectorySteps(trace) {
34678
+ return [...trace.events].sort((left, right) => {
34679
+ const leftTime = left.timestamp ? Date.parse(left.timestamp) : Number.NaN;
34680
+ const rightTime = right.timestamp ? Date.parse(right.timestamp) : Number.NaN;
34681
+ if (Number.isFinite(leftTime) && Number.isFinite(rightTime) && leftTime !== rightTime) {
34682
+ return leftTime - rightTime;
34683
+ }
34684
+ return left.ordinal - right.ordinal;
34685
+ }).map(eventToTrajectoryStep);
34686
+ }
34687
+ function normalizeMatcher(matcher, defaultType) {
34688
+ if (typeof matcher === "string") {
34689
+ return { pattern: matcher, ...defaultType ? { type: defaultType } : {} };
34690
+ }
34691
+ return { ...matcher, ...matcher.type ? {} : defaultType ? { type: defaultType } : {} };
34692
+ }
34693
+ function matchesTrajectoryStep(step, matcher, defaultType) {
34694
+ const normalized = normalizeMatcher(matcher, defaultType);
34695
+ if (normalized.type) {
34696
+ const allowedTypes = Array.isArray(normalized.type) ? normalized.type : [normalized.type];
34697
+ if (!allowedTypes.includes(step.type)) {
34698
+ return false;
34699
+ }
34700
+ }
34701
+ const matchPattern = normalized.pattern ?? normalized.name;
34702
+ return matchPattern ? step.aliases.some((alias) => matchesPattern2(alias, matchPattern)) : true;
34703
+ }
34704
+ function formatStep(step) {
34705
+ return `${step.type}:${step.name}`;
34706
+ }
34707
+ function formatArgs(args) {
34708
+ if (args === void 0) {
34709
+ return "(none)";
34710
+ }
34711
+ try {
34712
+ return JSON.stringify(args) ?? String(args);
34713
+ } catch {
34714
+ return String(args);
34715
+ }
34716
+ }
34717
+ function formatStepList(stepLabels) {
34718
+ return stepLabels.length > 0 ? stepLabels.join(", ") : "(none)";
34719
+ }
34720
+ function score(pass, text) {
34721
+ return {
34722
+ score: pass ? 1 : 0,
34723
+ verdict: scoreToVerdict(pass ? 1 : 0),
34724
+ assertions: [{ text, passed: pass }],
34725
+ expectedAspectCount: 1
34726
+ };
34727
+ }
34728
+ function missingTrace(type) {
34729
+ return score(false, `No trace data available for ${type} assertion`);
34730
+ }
34731
+ function requireNamedMatcher(matcher, assertionType, index) {
34732
+ if (matcher.pattern || matcher.name) {
34733
+ return;
34734
+ }
34735
+ const label = index === void 0 ? "object" : `step ${index + 1}`;
34736
+ throw new Error(`${assertionType} assertion ${label} must include a name or pattern property`);
34737
+ }
34738
+ function resolveToolMatchers(value) {
34739
+ if (typeof value === "string") {
34740
+ return { kind: "list", matchers: [normalizeMatcher(value, "tool")] };
34741
+ }
34742
+ if (Array.isArray(value) && value.every((item) => typeof item === "string")) {
34743
+ return { kind: "list", matchers: value.map((item) => normalizeMatcher(item, "tool")) };
34744
+ }
34745
+ if (isRecord22(value)) {
34746
+ const matcher = normalizeMatcher(value, "tool");
34747
+ return {
34748
+ kind: "count",
34749
+ matcher: {
34750
+ ...matcher,
34751
+ ...typeof value.min === "number" ? { min: value.min } : {},
34752
+ ...typeof value.max === "number" ? { max: value.max } : {}
34753
+ }
34754
+ };
34755
+ }
34756
+ throw new Error(
34757
+ "trajectory:tool-used assertion must have a string, string array, or object value"
34758
+ );
34759
+ }
34760
+ function evaluateToolUsed(config2, trace) {
34761
+ const steps = extractTrajectorySteps(trace).filter((step) => step.type === "tool");
34762
+ const expected = resolveToolMatchers(config2.value);
34763
+ const inverse = config2.inverse === true;
34764
+ if (expected.kind === "list") {
34765
+ if (expected.matchers.length === 0) {
34766
+ throw new Error("trajectory:tool-used assertion requires at least one expected tool");
34767
+ }
34768
+ const missing = expected.matchers.filter(
34769
+ (matcher2) => !steps.some((step) => matchesTrajectoryStep(step, matcher2))
34770
+ );
34771
+ const matched = expected.matchers.filter(
34772
+ (matcher2) => steps.some((step) => matchesTrajectoryStep(step, matcher2))
34773
+ );
34774
+ const pass2 = inverse ? matched.length === 0 : missing.length === 0;
34775
+ const actualTools = steps.map(formatStep);
34776
+ const expectedTools = expected.matchers.map(
34777
+ (matcher2) => matcher2.pattern ?? matcher2.name ?? "*"
34778
+ );
34779
+ if (inverse) {
34780
+ return score(
34781
+ pass2,
34782
+ pass2 ? `Forbidden tool(s) were not used: ${expectedTools.join(", ")}` : `Forbidden tool(s) were used: ${matched.map((matcher2) => matcher2.pattern ?? matcher2.name ?? "*").join(", ")}. Actual tools: ${formatStepList(actualTools)}`
34783
+ );
34784
+ }
34785
+ return score(
34786
+ pass2,
34787
+ pass2 ? `Observed required tool(s): ${expectedTools.join(", ")}. Actual tools: ${formatStepList(actualTools)}` : `Missing required tool(s): ${missing.map((matcher2) => matcher2.pattern ?? matcher2.name ?? "*").join(", ")}. Actual tools: ${formatStepList(actualTools)}`
34788
+ );
34789
+ }
34790
+ const matcher = expected.matcher;
34791
+ const min = matcher.min ?? 1;
34792
+ const max = matcher.max;
34793
+ requireNamedMatcher(matcher, "trajectory:tool-used");
34794
+ const matches = steps.filter((step) => matchesTrajectoryStep(step, matcher));
34795
+ const count = matches.length;
34796
+ const basePass = count >= min && (max === void 0 || count <= max);
34797
+ const pass = inverse ? !basePass : basePass;
34798
+ const label = matcher.pattern ?? matcher.name ?? "*";
34799
+ if (inverse) {
34800
+ return score(
34801
+ pass,
34802
+ basePass ? `Tool "${label}" matched ${count} time(s), which violates the inverse assertion` : `Tool "${label}" did not satisfy the forbidden match condition`
34803
+ );
34804
+ }
34805
+ let reason = `Matched tool "${label}" ${count} time(s)`;
34806
+ reason += max === void 0 ? ` (expected at least ${min})` : ` (expected ${min}-${max})`;
34807
+ if (matches.length > 0) {
34808
+ reason += `. Matches: ${matches.map(formatStep).join(", ")}`;
34809
+ }
34810
+ return score(pass, reason);
34811
+ }
34812
+ function resolveSequenceValue(value) {
34813
+ if (Array.isArray(value)) {
34814
+ return { mode: "in_order", steps: value };
34815
+ }
34816
+ if (isRecord22(value)) {
34817
+ return {
34818
+ mode: value.mode === "exact" ? "exact" : "in_order",
34819
+ steps: Array.isArray(value.steps) ? value.steps : []
34820
+ };
34821
+ }
34822
+ throw new Error("trajectory:tool-sequence assertion must have an array or object value");
34823
+ }
34824
+ function evaluateToolSequence(config2, trace) {
34825
+ const toolSteps = extractTrajectorySteps(trace).filter((step) => step.type === "tool");
34826
+ const value = resolveSequenceValue(config2.value);
34827
+ const expected = value.steps.map((step, index) => {
34828
+ const matcher = normalizeMatcher(step, "tool");
34829
+ requireNamedMatcher(matcher, "trajectory:tool-sequence", index);
34830
+ return matcher;
34831
+ });
34832
+ if (expected.length === 0) {
34833
+ throw new Error("trajectory:tool-sequence assertion requires at least one expected step");
34834
+ }
34835
+ const actualTools = toolSteps.map(formatStep);
34836
+ let basePass = false;
34837
+ let reason = "";
34838
+ if (value.mode === "exact") {
34839
+ basePass = toolSteps.length === expected.length && expected.every((matcher, index) => matchesTrajectoryStep(toolSteps[index], matcher));
34840
+ reason = basePass ? `Observed exact tool sequence: ${formatStepList(actualTools)}` : `Expected exact tool sequence of ${expected.map((matcher) => matcher.pattern ?? matcher.name ?? "*").join(", ")}, but actual tools were ${formatStepList(actualTools)}`;
34841
+ } else {
34842
+ let expectedIndex = 0;
34843
+ const matchedSteps = [];
34844
+ for (const step of toolSteps) {
34845
+ if (expectedIndex >= expected.length) {
34846
+ break;
34847
+ }
34848
+ if (matchesTrajectoryStep(step, expected[expectedIndex])) {
34849
+ matchedSteps.push(formatStep(step));
34850
+ expectedIndex += 1;
34851
+ }
34852
+ }
34853
+ basePass = expectedIndex === expected.length;
34854
+ reason = basePass ? `Observed tool sequence in order: ${matchedSteps.join(", ")}. Actual tools: ${formatStepList(actualTools)}` : `Expected tool "${expected[expectedIndex]?.pattern ?? expected[expectedIndex]?.name ?? "*"}" was not observed in order. Actual tools: ${formatStepList(actualTools)}`;
34855
+ }
34856
+ const inverse = config2.inverse === true;
34857
+ return score(
34858
+ inverse ? !basePass : basePass,
34859
+ inverse ? basePass ? `Forbidden tool sequence was observed. Actual tools: ${formatStepList(actualTools)}` : "Forbidden tool sequence was not observed" : reason
34860
+ );
34861
+ }
34862
+ function matchesExpectedArgsPartial(actual, expected) {
34863
+ if (Array.isArray(expected)) {
34864
+ return Array.isArray(actual) && actual.length === expected.length && expected.every((item, index) => matchesExpectedArgsPartial(actual[index], item));
34865
+ }
34866
+ if (isRecord22(expected)) {
34867
+ if (!isRecord22(actual)) {
34868
+ return false;
34869
+ }
34870
+ return Object.entries(expected).every(
34871
+ ([key, expectedValue]) => Object.hasOwn(actual, key) && matchesExpectedArgsPartial(actual[key], expectedValue)
34872
+ );
34873
+ }
34874
+ return isDeepStrictEqual(actual, expected);
34875
+ }
34876
+ function stripKeys(value, shouldStrip) {
34877
+ if (!isRecord22(value)) {
34878
+ return { cleaned: value, stripped: [] };
34879
+ }
34880
+ const cleaned = {};
34881
+ const stripped = [];
34882
+ for (const [key, entryValue] of Object.entries(value)) {
34883
+ if (shouldStrip(key, entryValue)) {
34884
+ stripped.push(key);
34885
+ continue;
34886
+ }
34887
+ Object.defineProperty(cleaned, key, {
34888
+ value: entryValue,
34889
+ enumerable: true,
34890
+ configurable: true,
34891
+ writable: true
34892
+ });
34893
+ }
34894
+ return { cleaned, stripped };
34895
+ }
34896
+ function stripDefaults(actual, defaults) {
34897
+ if (!defaults) {
34898
+ return { cleaned: actual, stripped: [] };
34899
+ }
34900
+ return stripKeys(
34901
+ actual,
34902
+ (key, value) => Object.hasOwn(defaults, key) && isDeepStrictEqual(value, defaults[key])
34903
+ );
34904
+ }
34905
+ function stripIgnored(value, ignore) {
34906
+ return stripKeys(
34907
+ value,
34908
+ (key) => ignore.some((entry) => /[*?]/.test(entry) ? matchesPattern2(key, entry) : entry === key)
34909
+ );
34910
+ }
34911
+ function resolveArgsMatchValue(value) {
34912
+ if (!isRecord22(value)) {
34913
+ throw new Error("trajectory:tool-args-match assertion must have an object value");
34914
+ }
34915
+ const matcher = normalizeMatcher(value, "tool");
34916
+ requireNamedMatcher(matcher, "trajectory:tool-args-match");
34917
+ const expectedArgs = Object.hasOwn(value, "args") ? value.args : value.arguments;
34918
+ if (expectedArgs === void 0) {
34919
+ throw new Error(
34920
+ "trajectory:tool-args-match assertion must include an args or arguments property"
34921
+ );
34922
+ }
34923
+ const mode = value.mode ?? "partial";
34924
+ if (mode !== "partial" && mode !== "exact") {
34925
+ throw new Error('trajectory:tool-args-match assertion mode must be "partial" or "exact"');
34926
+ }
34927
+ const defaults = value.defaults;
34928
+ if (defaults !== void 0 && !isRecord22(defaults)) {
34929
+ throw new Error(
34930
+ "trajectory:tool-args-match assertion defaults must be an object mapping argument names to default values"
34931
+ );
34932
+ }
34933
+ const rawIgnore = value.ignore;
34934
+ const ignore = rawIgnore === void 0 ? [] : Array.isArray(rawIgnore) ? rawIgnore : [rawIgnore];
34935
+ if (ignore.some((entry) => typeof entry !== "string" || entry.trim().length === 0)) {
34936
+ throw new Error(
34937
+ "trajectory:tool-args-match assertion ignore must be a non-empty string or an array of non-empty strings"
34938
+ );
34939
+ }
34940
+ return {
34941
+ matcher,
34942
+ expectedArgs,
34943
+ mode,
34944
+ defaults,
34945
+ ignore
34946
+ };
34947
+ }
34948
+ function argsMatch2(actual, expected, mode, defaults, ignore) {
34949
+ const cleanedActual = stripDefaults(stripIgnored(actual, ignore).cleaned, defaults).cleaned;
34950
+ const cleanedExpected = stripIgnored(expected, ignore).cleaned;
34951
+ return mode === "exact" ? isDeepStrictEqual(cleanedActual, cleanedExpected) : matchesExpectedArgsPartial(cleanedActual, cleanedExpected);
34952
+ }
34953
+ function evaluateToolArgsMatch(config2, trace) {
34954
+ const toolSteps = extractTrajectorySteps(trace).filter((step) => step.type === "tool");
34955
+ const { matcher, expectedArgs, mode, defaults, ignore } = resolveArgsMatchValue(config2.value);
34956
+ const label = matcher.pattern ?? matcher.name ?? "*";
34957
+ const actualTools = toolSteps.map(formatStep);
34958
+ const matchingSteps = toolSteps.filter((step) => matchesTrajectoryStep(step, matcher));
34959
+ const stepsWithArgs = matchingSteps.filter((step) => step.args !== void 0);
34960
+ const matchedStep = stepsWithArgs.find(
34961
+ (step) => argsMatch2(step.args, expectedArgs, mode, defaults, ignore)
34962
+ );
34963
+ const basePass = matchedStep !== void 0;
34964
+ const inverse = config2.inverse === true;
34965
+ const pass = inverse ? !basePass : basePass;
34966
+ const observedArgs = stepsWithArgs.length > 0 ? stepsWithArgs.map((step) => formatArgs(step.args)).join(", ") : "(none)";
34967
+ if (inverse) {
34968
+ if (basePass) {
34969
+ return score(
34970
+ pass,
34971
+ `Forbidden argument match for tool "${label}" was observed on ${formatStep(
34972
+ matchedStep
34973
+ )}. Args: ${formatArgs(matchedStep?.args)}`
34974
+ );
34975
+ }
34976
+ return score(
34977
+ pass,
34978
+ matchingSteps.length === 0 ? `Forbidden argument match for tool "${label}" was not observed because no tool call matched it` : `Forbidden argument match for tool "${label}" was not observed. Observed args: ${observedArgs}`
34979
+ );
34980
+ }
34981
+ if (basePass) {
34982
+ const ignoredArgs = stripIgnored(matchedStep.args, ignore).stripped;
34983
+ const ignoredDefaults = stripDefaults(
34984
+ stripIgnored(matchedStep.args, ignore).cleaned,
34985
+ defaults
34986
+ ).stripped;
34987
+ const ignoredArgsSuffix = ignoredArgs.length > 0 ? `. Ignored argument(s): ${ignoredArgs.join(", ")}` : "";
34988
+ const ignoredDefaultsSuffix = ignoredDefaults.length > 0 ? `. Ignored default argument(s): ${ignoredDefaults.join(", ")}` : "";
34989
+ return score(
34990
+ true,
34991
+ `Tool "${label}" matched expected arguments (${mode}) on ${formatStep(
34992
+ matchedStep
34993
+ )}. Args: ${formatArgs(matchedStep.args)}${ignoredArgsSuffix}${ignoredDefaultsSuffix}`
34994
+ );
34995
+ }
34996
+ if (matchingSteps.length === 0) {
34997
+ return score(
34998
+ false,
34999
+ `No tool call matched "${label}". Actual tools: ${formatStepList(actualTools)}`
35000
+ );
35001
+ }
35002
+ if (stepsWithArgs.length === 0) {
35003
+ return score(
35004
+ false,
35005
+ `Tool "${label}" was observed but no arguments were captured. Actual tools: ${formatStepList(
35006
+ actualTools
35007
+ )}`
35008
+ );
35009
+ }
35010
+ return score(
35011
+ false,
35012
+ `No call to tool "${label}" matched expected arguments (${mode}): ${formatArgs(
35013
+ expectedArgs
35014
+ )}. Observed args: ${observedArgs}`
35015
+ );
35016
+ }
35017
+ function resolveStepCountValue(value) {
35018
+ if (!isRecord22(value)) {
35019
+ throw new Error("trajectory:step-count assertion must have an object value");
35020
+ }
35021
+ const matcher = normalizeMatcher(value);
35022
+ return {
35023
+ ...matcher,
35024
+ ...typeof value.min === "number" ? { min: value.min } : {},
35025
+ ...typeof value.max === "number" ? { max: value.max } : {}
35026
+ };
35027
+ }
35028
+ function evaluateStepCount(config2, trace) {
35029
+ const matcher = resolveStepCountValue(config2.value);
35030
+ if (matcher.min === void 0 && matcher.max === void 0) {
35031
+ throw new Error("trajectory:step-count assertion must include a min or max property");
35032
+ }
35033
+ const matchingSteps = extractTrajectorySteps(trace).filter(
35034
+ (step) => matchesTrajectoryStep(step, matcher)
35035
+ );
35036
+ const count = matchingSteps.length;
35037
+ const basePass = (matcher.min === void 0 || count >= matcher.min) && (matcher.max === void 0 || count <= matcher.max);
35038
+ const inverse = config2.inverse === true;
35039
+ const filterParts = [];
35040
+ if (matcher.type) {
35041
+ const types = Array.isArray(matcher.type) ? matcher.type : [matcher.type];
35042
+ filterParts.push(`type=${types.join("|")}`);
35043
+ }
35044
+ const pattern = matcher.pattern ?? matcher.name;
35045
+ if (pattern) {
35046
+ filterParts.push(`pattern=${pattern}`);
35047
+ }
35048
+ let reason = `Matched ${count} trajectory step(s)`;
35049
+ if (filterParts.length > 0) {
35050
+ reason += ` for ${filterParts.join(", ")}`;
35051
+ }
35052
+ if (matcher.min !== void 0 && matcher.max !== void 0) {
35053
+ reason += ` (expected ${matcher.min}-${matcher.max})`;
35054
+ } else if (matcher.min !== void 0) {
35055
+ reason += ` (expected at least ${matcher.min})`;
35056
+ } else {
35057
+ reason += ` (expected at most ${matcher.max})`;
35058
+ }
35059
+ if (matchingSteps.length > 0) {
35060
+ reason += `. Matches: ${matchingSteps.map(formatStep).join(", ")}`;
35061
+ }
35062
+ return score(
35063
+ inverse ? !basePass : basePass,
35064
+ inverse ? basePass ? "Trajectory step count satisfied the forbidden range" : "Trajectory step count did not satisfy the forbidden range" : reason
35065
+ );
35066
+ }
35067
+ function resolveGoal(value) {
35068
+ if (typeof value === "string" && value.trim()) {
35069
+ return value.trim();
35070
+ }
35071
+ if (isRecord22(value) && typeof value.goal === "string" && value.goal.trim()) {
35072
+ return value.goal.trim();
35073
+ }
35074
+ throw new Error(
35075
+ "trajectory:goal-success assertion must have a string value or an object with a goal property"
35076
+ );
35077
+ }
35078
+ function summarizeTrajectoryForJudge(trace) {
35079
+ const steps = extractTrajectorySteps(trace).map((step, index) => ({
35080
+ index: index + 1,
35081
+ type: step.type,
35082
+ name: step.name,
35083
+ ...step.spanName === step.name ? {} : { spanName: step.spanName }
35084
+ }));
35085
+ return JSON.stringify({ stepCount: steps.length, steps }, null, 2);
35086
+ }
35087
+ function buildGoalCriteria(goal, trajectory, output) {
35088
+ return [
35089
+ "Determine whether the agent successfully achieved the goal using the observed trajectory and final output.",
35090
+ "",
35091
+ `Goal: ${goal}`,
35092
+ "",
35093
+ "Trajectory:",
35094
+ trajectory,
35095
+ "",
35096
+ "Final output:",
35097
+ output
35098
+ ].join("\n");
35099
+ }
35100
+ var TrajectoryGrader = class {
35101
+ kind;
35102
+ config;
35103
+ llmGrader;
35104
+ constructor(options) {
35105
+ this.config = options.config;
35106
+ this.kind = options.config.type;
35107
+ this.llmGrader = options.llmGrader;
35108
+ }
35109
+ async evaluate(context) {
35110
+ const trace = context.trace;
35111
+ if (!trace) {
35112
+ return missingTrace(this.config.type);
35113
+ }
35114
+ switch (this.config.type) {
35115
+ case "trajectory:tool-used":
35116
+ return evaluateToolUsed(this.config, trace);
35117
+ case "trajectory:tool-args-match":
35118
+ return evaluateToolArgsMatch(this.config, trace);
35119
+ case "trajectory:tool-sequence":
35120
+ return evaluateToolSequence(this.config, trace);
35121
+ case "trajectory:step-count":
35122
+ return evaluateStepCount(this.config, trace);
35123
+ case "trajectory:goal-success": {
35124
+ if (!this.llmGrader) {
35125
+ throw new Error("trajectory:goal-success assertion requires an LLM grader");
35126
+ }
35127
+ const goal = resolveGoal(this.config.value);
35128
+ const result = await this.llmGrader.evaluate({
35129
+ ...context,
35130
+ evalCase: {
35131
+ ...context.evalCase,
35132
+ criteria: buildGoalCriteria(
35133
+ goal,
35134
+ summarizeTrajectoryForJudge(trace),
35135
+ context.candidate
35136
+ )
35137
+ },
35138
+ evaluator: this.config
35139
+ });
35140
+ if (this.config.inverse !== true) {
35141
+ return result;
35142
+ }
35143
+ const negated = negateScore(result);
35144
+ const assertion = {
35145
+ text: result.score >= 0.5 ? `Agent unexpectedly achieved the goal: ${goal}` : `Agent did not achieve the forbidden goal: ${goal}`,
35146
+ passed: negated.score >= 0.5
35147
+ };
35148
+ return { ...negated, assertions: [assertion] };
35149
+ }
35150
+ }
35151
+ }
35152
+ };
34248
35153
  function runContainsAssertion(output, value) {
34249
35154
  const passed = output.includes(value);
34250
35155
  return {
@@ -34672,6 +35577,7 @@ var ClaudeCliProvider = class {
34672
35577
  args,
34673
35578
  exitCode: result.exitCode
34674
35579
  },
35580
+ metadata: skillCallMetadata(deriveSkillCallsFromToolCalls(completedToolCalls)),
34675
35581
  output,
34676
35582
  tokenUsage,
34677
35583
  costUsd,
@@ -36640,6 +37546,7 @@ ${basePrompt}` : basePrompt;
36640
37546
  inputFiles,
36641
37547
  logFile: logger?.filePath
36642
37548
  },
37549
+ metadata: skillCallMetadata(deriveSkillCallsFromMessages(messages)),
36643
37550
  output: messages,
36644
37551
  durationMs,
36645
37552
  targetExecution: {
@@ -40308,7 +41215,7 @@ function toWireRecord(record2) {
40308
41215
  start_time: record2.startTime,
40309
41216
  end_time: record2.endTime
40310
41217
  };
40311
- const parsed = ReplayFixtureWireSchema.parse(dropUndefined(wire));
41218
+ const parsed = ReplayFixtureWireSchema.parse(dropUndefined2(wire));
40312
41219
  return parsed;
40313
41220
  }
40314
41221
  function toWireMessage(message) {
@@ -40336,7 +41243,7 @@ function toWireToolCall(toolCall) {
40336
41243
  duration_ms: toolCall.durationMs
40337
41244
  };
40338
41245
  }
40339
- function dropUndefined(value) {
41246
+ function dropUndefined2(value) {
40340
41247
  return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
40341
41248
  }
40342
41249
  function formatZodError(error40) {
@@ -40452,13 +41359,14 @@ function formatReplayLookupKey(lookup) {
40452
41359
  function replayFixtureRecordToProviderResponse(record2) {
40453
41360
  return {
40454
41361
  output: record2.output,
41362
+ metadata: skillCallMetadata(deriveSkillCallsFromMessages(record2.output)),
40455
41363
  tokenUsage: record2.tokenUsage,
40456
41364
  costUsd: record2.costUsd,
40457
41365
  durationMs: record2.durationMs,
40458
41366
  startTime: record2.startTime,
40459
41367
  endTime: record2.endTime,
40460
41368
  raw: {
40461
- replay_fixture: dropUndefined({
41369
+ replay_fixture: dropUndefined2({
40462
41370
  fixture_id: record2.fixtureId,
40463
41371
  suite: record2.suite,
40464
41372
  eval_path: record2.evalPath,
@@ -40531,7 +41439,7 @@ function buildFixtureId(input) {
40531
41439
  return `${input.sourceTarget}-${input.testId}-${digest}`;
40532
41440
  }
40533
41441
  function buildSourceMetadata(target, sourceTarget) {
40534
- return dropUndefined({
41442
+ return dropUndefined2({
40535
41443
  provider: target.kind,
40536
41444
  target_name: sourceTarget,
40537
41445
  resolved_target: target.name,
@@ -40596,7 +41504,7 @@ var EXTERNAL_TRACE_KEYS = [
40596
41504
  "target"
40597
41505
  ];
40598
41506
  var TRACEPARENT_RE = /^[\da-f]{2}-[\da-f]{32}-[\da-f]{16}-[\da-f]{2}$/i;
40599
- function isRecord2(value) {
41507
+ function isRecord3(value) {
40600
41508
  return typeof value === "object" && value !== null && !Array.isArray(value);
40601
41509
  }
40602
41510
  function stringValue(value) {
@@ -40741,7 +41649,7 @@ function omitExternalTraceMetadataKeys(value) {
40741
41649
  return entries.length > 0 ? Object.fromEntries(entries) : void 0;
40742
41650
  }
40743
41651
  function sanitizeExternalTraceMetadata(value) {
40744
- if (!isRecord2(value)) {
41652
+ if (!isRecord3(value)) {
40745
41653
  return void 0;
40746
41654
  }
40747
41655
  return sanitizeExternalTraceObject(externalTraceInputFromRecord(value));
@@ -41095,12 +42003,12 @@ var ProjectionIdentityWireSchema = external_exports.object({
41095
42003
  dimensions: ProjectionIdentityDimensionsWireSchema,
41096
42004
  issues: external_exports.array(ProjectionIdentityIssueWireSchema).optional()
41097
42005
  }).strict();
41098
- function dropUndefined2(value) {
42006
+ function dropUndefined22(value) {
41099
42007
  return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
41100
42008
  }
41101
42009
  function toProjectionIdentityIssueWire(issue2) {
41102
42010
  return ProjectionIdentityIssueWireSchema.parse(
41103
- dropUndefined2({
42011
+ dropUndefined22({
41104
42012
  code: issue2.code,
41105
42013
  severity: issue2.severity,
41106
42014
  field: issue2.field,
@@ -41118,7 +42026,7 @@ function fromProjectionIdentityIssueWire(issue2) {
41118
42026
  }
41119
42027
  function toProjectionIdentityWire(identity) {
41120
42028
  return ProjectionIdentityWireSchema.parse(
41121
- dropUndefined2({
42029
+ dropUndefined22({
41122
42030
  schema_version: identity.schemaVersion,
41123
42031
  id: identity.id,
41124
42032
  key: identity.key,
@@ -41292,7 +42200,7 @@ var TraceEnvelopeWireSchema = external_exports.object({
41292
42200
  function dropUndefined3(value) {
41293
42201
  return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
41294
42202
  }
41295
- function isRecord22(value) {
42203
+ function isRecord4(value) {
41296
42204
  return typeof value === "object" && value !== null && !Array.isArray(value);
41297
42205
  }
41298
42206
  function definedStringRecord(value) {
@@ -41503,25 +42411,25 @@ function scoresFromResult(result, targetSpanId) {
41503
42411
  if (!result.scores || result.scores.length === 0) {
41504
42412
  return void 0;
41505
42413
  }
41506
- return result.scores.map((score) => ({
41507
- name: score.name,
41508
- type: score.type,
41509
- score: score.score,
41510
- weight: score.weight,
41511
- verdict: score.verdict,
41512
- source: scoreSource(score.type),
41513
- evaluatedAt: score.endedAt ?? score.startedAt ?? result.timestamp,
42414
+ return result.scores.map((score2) => ({
42415
+ name: score2.name,
42416
+ type: score2.type,
42417
+ score: score2.score,
42418
+ weight: score2.weight,
42419
+ verdict: score2.verdict,
42420
+ source: scoreSource(score2.type),
42421
+ evaluatedAt: score2.endedAt ?? score2.startedAt ?? result.timestamp,
41514
42422
  targetSpanId,
41515
42423
  evidence: dropUndefined3({
41516
42424
  span_ids: [targetSpanId],
41517
- assertions: (score.assertions ?? []).map(
42425
+ assertions: (score2.assertions ?? []).map(
41518
42426
  (assertion) => dropUndefined3({
41519
42427
  text: assertion.text,
41520
42428
  passed: assertion.passed,
41521
42429
  evidence: assertion.evidence
41522
42430
  })
41523
42431
  ),
41524
- details: score.details
42432
+ details: score2.details
41525
42433
  })
41526
42434
  }));
41527
42435
  }
@@ -41990,32 +42898,32 @@ function fromTraceEnvelopeConversionWarningWire(warning) {
41990
42898
  details: warning.details
41991
42899
  };
41992
42900
  }
41993
- function toTraceEnvelopeScoreWire(score) {
42901
+ function toTraceEnvelopeScoreWire(score2) {
41994
42902
  return TraceEnvelopeScoreWireSchema.parse(
41995
42903
  dropUndefined3({
41996
- name: score.name,
41997
- type: score.type,
41998
- score: score.score,
41999
- weight: score.weight,
42000
- verdict: score.verdict,
42001
- source: score.source,
42002
- evaluated_at: score.evaluatedAt,
42003
- target_span_id: score.targetSpanId,
42004
- evidence: score.evidence
42904
+ name: score2.name,
42905
+ type: score2.type,
42906
+ score: score2.score,
42907
+ weight: score2.weight,
42908
+ verdict: score2.verdict,
42909
+ source: score2.source,
42910
+ evaluated_at: score2.evaluatedAt,
42911
+ target_span_id: score2.targetSpanId,
42912
+ evidence: score2.evidence
42005
42913
  })
42006
42914
  );
42007
42915
  }
42008
- function fromTraceEnvelopeScoreWire(score) {
42916
+ function fromTraceEnvelopeScoreWire(score2) {
42009
42917
  return {
42010
- name: score.name,
42011
- type: score.type,
42012
- score: score.score,
42013
- weight: score.weight,
42014
- verdict: score.verdict,
42015
- source: score.source,
42016
- evaluatedAt: score.evaluated_at,
42017
- targetSpanId: score.target_span_id,
42018
- evidence: score.evidence
42918
+ name: score2.name,
42919
+ type: score2.type,
42920
+ score: score2.score,
42921
+ weight: score2.weight,
42922
+ verdict: score2.verdict,
42923
+ source: score2.source,
42924
+ evaluatedAt: score2.evaluated_at,
42925
+ targetSpanId: score2.target_span_id,
42926
+ evidence: score2.evidence
42019
42927
  };
42020
42928
  }
42021
42929
  function isToolSpan(span) {
@@ -42082,7 +42990,7 @@ function nearestAncestorToolCallId(ancestorIds, spansById) {
42082
42990
  return void 0;
42083
42991
  }
42084
42992
  function fromTranscriptToolCallWire(wire) {
42085
- if (!isRecord22(wire) || typeof wire.tool !== "string") {
42993
+ if (!isRecord4(wire) || typeof wire.tool !== "string") {
42086
42994
  return void 0;
42087
42995
  }
42088
42996
  return {
@@ -42097,7 +43005,7 @@ function fromTranscriptToolCallWire(wire) {
42097
43005
  };
42098
43006
  }
42099
43007
  function fromTranscriptMessageWire(wire) {
42100
- if (!isRecord22(wire) || typeof wire.role !== "string") {
43008
+ if (!isRecord4(wire) || typeof wire.role !== "string") {
42101
43009
  return void 0;
42102
43010
  }
42103
43011
  const toolCalls = Array.isArray(wire.tool_calls) ? wire.tool_calls.map(fromTranscriptToolCallWire).filter((toolCall) => toolCall !== void 0) : void 0;
@@ -42109,8 +43017,8 @@ function fromTranscriptMessageWire(wire) {
42109
43017
  startTime: typeof wire.start_time === "string" ? wire.start_time : void 0,
42110
43018
  endTime: typeof wire.end_time === "string" ? wire.end_time : void 0,
42111
43019
  durationMs: numberAttribute(wire, "duration_ms"),
42112
- metadata: isRecord22(wire.metadata) ? wire.metadata : void 0,
42113
- tokenUsage: isRecord22(wire.token_usage) ? tokenUsageFromAttributes({
43020
+ metadata: isRecord4(wire.metadata) ? wire.metadata : void 0,
43021
+ tokenUsage: isRecord4(wire.token_usage) ? tokenUsageFromAttributes({
42114
43022
  "gen_ai.usage.input_tokens": wire.token_usage.input,
42115
43023
  "gen_ai.usage.output_tokens": wire.token_usage.output,
42116
43024
  "gen_ai.usage.cache_read.input_tokens": wire.token_usage.cached,
@@ -42466,6 +43374,7 @@ function traceEnvelopeReplayRecordToProviderResponse(record2) {
42466
43374
  const wire = toTraceEnvelopeWire(record2.envelope);
42467
43375
  return {
42468
43376
  output,
43377
+ metadata: skillCallMetadata(deriveSkillCallsFromMessages(output)),
42469
43378
  tokenUsage: summary.tokenUsage,
42470
43379
  costUsd: summary.costUsd,
42471
43380
  durationMs: summary.durationMs,
@@ -42581,9 +43490,10 @@ var CANONICAL_TRANSCRIPT_ARTIFACT_PATH = "transcript.json";
42581
43490
  var CANONICAL_METRICS_ARTIFACT_PATH = "metrics.json";
42582
43491
  var CANONICAL_FILE_CHANGES_ARTIFACT_PATH = "outputs/file_changes.diff";
42583
43492
  var TRANSCRIPT_SCHEMA_VERSION = "agentv.normalized_transcript.v1";
42584
- var METRICS_SCHEMA_VERSION = "agentv.metrics.v1";
43493
+ var LEGACY_METRICS_SCHEMA_VERSION = "agentv.metrics.v1";
43494
+ var METRICS_SCHEMA_VERSION = "agentv.metrics.v2";
42585
43495
  var TRANSCRIPT_JSON_MEDIA_TYPE = "application/vnd.agentv.normalized-transcript.v1+json";
42586
- var METRICS_JSON_MEDIA_TYPE = "application/vnd.agentv.metrics.v1+json";
43496
+ var METRICS_JSON_MEDIA_TYPE = "application/vnd.agentv.metrics.v2+json";
42587
43497
  function toResultArtifactPointerWire(pointer) {
42588
43498
  return {
42589
43499
  ref: pointer.ref,
@@ -42668,7 +43578,7 @@ var COMMAND_KEYS = /* @__PURE__ */ new Set([
42668
43578
  "_extractedcommand"
42669
43579
  ]);
42670
43580
  var URL_KEYS = /* @__PURE__ */ new Set(["url", "uri", "href", "extractedurl", "_extractedurl"]);
42671
- function isRecord3(value) {
43581
+ function isRecord5(value) {
42672
43582
  return typeof value === "object" && value !== null && !Array.isArray(value);
42673
43583
  }
42674
43584
  function normalizedKey(key) {
@@ -42747,7 +43657,7 @@ function stringValuesByKey(value, keys, maxDepth = 6) {
42747
43657
  }
42748
43658
  return;
42749
43659
  }
42750
- if (!isRecord3(entry)) {
43660
+ if (!isRecord5(entry)) {
42751
43661
  return;
42752
43662
  }
42753
43663
  for (const [key, nested] of Object.entries(entry)) {
@@ -42799,7 +43709,7 @@ function collectThinkingBlocks(messages) {
42799
43709
  }
42800
43710
  for (const block of content) {
42801
43711
  const entry = block;
42802
- if (!isRecord3(entry)) {
43712
+ if (!isRecord5(entry)) {
42803
43713
  continue;
42804
43714
  }
42805
43715
  if (entry.type === "thinking" || entry.type === "reasoning") {
@@ -42814,7 +43724,7 @@ function errorFromToolCall(toolCall, toolName) {
42814
43724
  return void 0;
42815
43725
  }
42816
43726
  const output = toolCall.output;
42817
- const message = typeof output === "string" ? output : isRecord3(output) && typeof output.message === "string" ? output.message : `Tool ${toolCall.tool} ${toolCall.status}`;
43727
+ const message = typeof output === "string" ? output : isRecord5(output) && typeof output.message === "string" ? output.message : `Tool ${toolCall.tool} ${toolCall.status}`;
42818
43728
  return {
42819
43729
  message,
42820
43730
  ...toolCall.id ? { tool_call_id: toolCall.id } : {},
@@ -42877,7 +43787,7 @@ var TRANSCRIPT_ROW_SCHEMA_VERSION = "agentv.transcript.v1";
42877
43787
  function dropUndefined5(value) {
42878
43788
  return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
42879
43789
  }
42880
- function isRecord4(value) {
43790
+ function isRecord6(value) {
42881
43791
  return typeof value === "object" && value !== null && !Array.isArray(value);
42882
43792
  }
42883
43793
  function optionalString2(value) {
@@ -43110,7 +44020,7 @@ function normalizedContentBlocks(message, messageIndex, providerId) {
43110
44020
  } else if (Array.isArray(content)) {
43111
44021
  for (const contentBlock of content) {
43112
44022
  const block = contentBlock;
43113
- if (!isRecord4(block) || typeof block.type !== "string") {
44023
+ if (!isRecord6(block) || typeof block.type !== "string") {
43114
44024
  continue;
43115
44025
  }
43116
44026
  if (block.type === "text" && typeof block.text === "string") {
@@ -43303,7 +44213,7 @@ function traceToTranscriptJsonLines(trace, options) {
43303
44213
  provider,
43304
44214
  sessionId,
43305
44215
  startedAt: trace.startTime,
43306
- metadata: isRecord4(trace.metadata) ? trace.metadata : void 0
44216
+ metadata: isRecord6(trace.metadata) ? trace.metadata : void 0
43307
44217
  },
43308
44218
  tokenUsage: trace.tokenUsage,
43309
44219
  durationMs: trace.durationMs,
@@ -43349,7 +44259,7 @@ function readOptionalNumber(record2, key) {
43349
44259
  return typeof value === "number" && Number.isFinite(value) ? value : void 0;
43350
44260
  }
43351
44261
  function fromTranscriptToolCall(wire) {
43352
- if (!isRecord4(wire)) {
44262
+ if (!isRecord6(wire)) {
43353
44263
  return void 0;
43354
44264
  }
43355
44265
  const tool = readOptionalString3(wire, "tool");
@@ -43457,6 +44367,7 @@ function transcriptReplayRecordToProviderResponse(record2) {
43457
44367
  const entry = record2.entry;
43458
44368
  return {
43459
44369
  output: entry.messages,
44370
+ metadata: skillCallMetadata(deriveSkillCallsFromMessages(entry.messages)),
43460
44371
  tokenUsage: entry.tokenUsage,
43461
44372
  durationMs: entry.durationMs,
43462
44373
  costUsd: entry.costUsd ?? void 0,
@@ -45189,7 +46100,7 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
45189
46100
  };
45190
46101
  }
45191
46102
  }
45192
- function isRecord5(value) {
46103
+ function isRecord7(value) {
45193
46104
  return typeof value === "object" && value !== null && !Array.isArray(value);
45194
46105
  }
45195
46106
  function extractTargetsArray(parsed, absolutePath) {
@@ -45200,7 +46111,7 @@ function extractTargetsArray(parsed, absolutePath) {
45200
46111
  return targets;
45201
46112
  }
45202
46113
  function assertTargetDefinition(value, index, filePath) {
45203
- if (!isRecord5(value)) {
46114
+ if (!isRecord7(value)) {
45204
46115
  throw new Error(`targets.yaml entry at index ${index} in ${filePath} must be an object`);
45205
46116
  }
45206
46117
  const id = value.id;
@@ -45241,7 +46152,7 @@ async function readTargetDefinitions(filePath) {
45241
46152
  }
45242
46153
  const raw = await readFile8(absolutePath, "utf8");
45243
46154
  const parsed = parseYamlValue(raw);
45244
- if (!isRecord5(parsed)) {
46155
+ if (!isRecord7(parsed)) {
45245
46156
  throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`);
45246
46157
  }
45247
46158
  const targets = extractTargetsArray(parsed, absolutePath);
@@ -45522,11 +46433,11 @@ var InlineAssertGrader = class {
45522
46433
  criteria: context.evalCase.criteria,
45523
46434
  metadata: context.evalCase.metadata
45524
46435
  });
45525
- const score = clampScore(result.score);
46436
+ const score2 = clampScore(result.score);
45526
46437
  return {
45527
- score,
45528
- verdict: scoreToVerdict(score),
45529
- assertions: [{ text: result.name, passed: score >= 0.5 }],
46438
+ score: score2,
46439
+ verdict: scoreToVerdict(score2),
46440
+ assertions: [{ text: result.name, passed: score2 >= 0.5 }],
45530
46441
  expectedAspectCount: 1,
45531
46442
  details: result.metadata ? result.metadata : void 0
45532
46443
  };
@@ -45627,17 +46538,17 @@ function normalizeScriptResult(result, fallbackText, threshold) {
45627
46538
  };
45628
46539
  }
45629
46540
  if (typeof result === "number") {
45630
- const score2 = clampScore(result);
45631
- const passed2 = score2 >= passThreshold;
46541
+ const score3 = clampScore(result);
46542
+ const passed2 = score3 >= passThreshold;
45632
46543
  return {
45633
- score: score2,
46544
+ score: score3,
45634
46545
  verdict: passed2 ? "pass" : "fail",
45635
46546
  assertions: [{ text: passed2 ? "Assertion passed" : fallbackText, passed: passed2 }],
45636
46547
  expectedAspectCount: 1
45637
46548
  };
45638
46549
  }
45639
- const score = typeof result.score === "number" ? clampScore(result.score) : result.pass === true ? 1 : result.pass === false ? 0 : 0;
45640
- const passed = result.pass ?? score >= passThreshold;
46550
+ const score2 = typeof result.score === "number" ? clampScore(result.score) : result.pass === true ? 1 : result.pass === false ? 0 : 0;
46551
+ const passed = result.pass ?? score2 >= passThreshold;
45641
46552
  const assertions = result.assertions && result.assertions.length > 0 ? result.assertions : [
45642
46553
  {
45643
46554
  text: result.reason ?? (passed ? "Assertion passed" : fallbackText),
@@ -45645,7 +46556,7 @@ function normalizeScriptResult(result, fallbackText, threshold) {
45645
46556
  }
45646
46557
  ];
45647
46558
  return {
45648
- score,
46559
+ score: score2,
45649
46560
  verdict: passed ? "pass" : "fail",
45650
46561
  assertions,
45651
46562
  expectedAspectCount: assertions.length || 1,
@@ -45809,12 +46720,12 @@ var AssertSetGrader = class {
45809
46720
  tokenUsage: result.tokenUsage
45810
46721
  });
45811
46722
  }
45812
- const totalWeight = scores.reduce((sum, score2) => sum + (score2.weight ?? 1), 0) || 1;
45813
- const score = scores.reduce((sum, item) => sum + item.score * (item.weight ?? 1), 0) / totalWeight;
46723
+ const totalWeight = scores.reduce((sum, score3) => sum + (score3.weight ?? 1), 0) || 1;
46724
+ const score2 = scores.reduce((sum, item) => sum + item.score * (item.weight ?? 1), 0) / totalWeight;
45814
46725
  const threshold = this.config.threshold;
45815
- const passed = threshold !== void 0 ? score >= threshold : scores.every((item) => (item.weight ?? 1) === 0 || item.verdict === "pass");
46726
+ const passed = threshold !== void 0 ? score2 >= threshold : scores.every((item) => (item.weight ?? 1) === 0 || item.verdict === "pass");
45816
46727
  return {
45817
- score,
46728
+ score: score2,
45818
46729
  verdict: passed ? "pass" : "fail",
45819
46730
  assertions: scores.flatMap((item) => item.assertions),
45820
46731
  expectedAspectCount: scores.reduce((sum, item) => sum + item.assertions.length, 0) || 1,
@@ -46016,6 +46927,12 @@ var toolTrajectoryFactory = (config2) => {
46016
46927
  config: config2
46017
46928
  });
46018
46929
  };
46930
+ var trajectoryFactory = (config2, context) => {
46931
+ return new TrajectoryGrader({
46932
+ config: config2,
46933
+ llmGrader: context.llmGrader
46934
+ });
46935
+ };
46019
46936
  var fieldAccuracyFactory = (config2) => {
46020
46937
  return new FieldAccuracyGrader({
46021
46938
  config: config2
@@ -46038,6 +46955,9 @@ var executionMetricsFactory = (config2) => {
46038
46955
  var skillTriggerFactory = (config2) => {
46039
46956
  return new SkillTriggerGrader(config2);
46040
46957
  };
46958
+ var skillUsedFactory = (config2) => {
46959
+ return new SkillUsedGrader(config2);
46960
+ };
46041
46961
  var containsFactory = (config2) => {
46042
46962
  const c = config2;
46043
46963
  return new DeterministicAssertionGrader("contains", (ctx) => {
@@ -46171,7 +47091,7 @@ var endsWithFactory = (config2) => {
46171
47091
  };
46172
47092
  function createBuiltinRegistry() {
46173
47093
  const registry2 = new GraderRegistry();
46174
- registry2.register("llm-grader", llmGraderFactory).register("llm-rubric", llmRubricFactory).register("script", scriptFactory).register("tool-trajectory", toolTrajectoryFactory).register("field-accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token-usage", tokenUsageFactory).register("execution-metrics", executionMetricsFactory).register("skill-trigger", skillTriggerFactory).register("assert-set", assertSetFactory).register("contains", containsFactory).register("contains-any", containsAnyFactory).register("contains-all", containsAllFactory).register("icontains", icontainsFactory).register("icontains-any", icontainsAnyFactory).register("icontains-all", icontainsAllFactory).register("starts-with", startsWithFactory).register("ends-with", endsWithFactory).register("regex", regexFactory).register("is-json", isJsonFactory).register("equals", equalsFactory).register("javascript", javascriptFactory).register("python", pythonFactory).register("webhook", webhookFactory).register("similar", similarFactory).register("inline-assert", (config2) => {
47094
+ registry2.register("llm-grader", llmGraderFactory).register("llm-rubric", llmRubricFactory).register("script", scriptFactory).register("tool-trajectory", toolTrajectoryFactory).register("trajectory:tool-used", trajectoryFactory).register("trajectory:tool-args-match", trajectoryFactory).register("trajectory:tool-sequence", trajectoryFactory).register("trajectory:step-count", trajectoryFactory).register("trajectory:goal-success", trajectoryFactory).register("field-accuracy", fieldAccuracyFactory).register("latency", latencyFactory).register("cost", costFactory).register("token-usage", tokenUsageFactory).register("execution-metrics", executionMetricsFactory).register("skill-used", skillUsedFactory).register("not-skill-used", skillUsedFactory).register("skill-trigger", skillTriggerFactory).register("assert-set", assertSetFactory).register("contains", containsFactory).register("contains-any", containsAnyFactory).register("contains-all", containsAllFactory).register("icontains", icontainsFactory).register("icontains-any", icontainsAnyFactory).register("icontains-all", icontainsAllFactory).register("starts-with", startsWithFactory).register("ends-with", endsWithFactory).register("regex", regexFactory).register("is-json", isJsonFactory).register("equals", equalsFactory).register("javascript", javascriptFactory).register("python", pythonFactory).register("webhook", webhookFactory).register("similar", similarFactory).register("inline-assert", (config2) => {
46175
47095
  const fn = config2[INLINE_ASSERT_FN];
46176
47096
  if (!fn) {
46177
47097
  throw new Error(
@@ -49161,7 +50081,7 @@ var REPEAT_FIELDS = /* @__PURE__ */ new Set([
49161
50081
  "cost_limit_usd"
49162
50082
  ]);
49163
50083
  function normalizeExperimentConfig(rawConfig) {
49164
- if (!isRecord6(rawConfig)) {
50084
+ if (!isRecord8(rawConfig)) {
49165
50085
  throw new Error("Experiment config must be an object.");
49166
50086
  }
49167
50087
  const name = readOptionalString22(rawConfig.name, "name");
@@ -49202,7 +50122,7 @@ function normalizeExperimentConfig(rawConfig) {
49202
50122
  };
49203
50123
  }
49204
50124
  function normalizeExperimentRunOverride(rawConfig) {
49205
- if (!isRecord6(rawConfig)) {
50125
+ if (!isRecord8(rawConfig)) {
49206
50126
  throw new Error("Run override must be an object.");
49207
50127
  }
49208
50128
  for (const key of Object.keys(rawConfig)) {
@@ -49272,7 +50192,7 @@ function readRepeat(raw) {
49272
50192
  strategy: "pass_any"
49273
50193
  };
49274
50194
  }
49275
- if (!isRecord6(raw)) {
50195
+ if (!isRecord8(raw)) {
49276
50196
  throw new Error("Experiment repeat must be a positive integer or object.");
49277
50197
  }
49278
50198
  for (const key of Object.keys(raw)) {
@@ -49304,7 +50224,7 @@ function readTargets(raw) {
49304
50224
  if (typeof entry === "string" && entry.trim().length > 0) {
49305
50225
  return entry.trim();
49306
50226
  }
49307
- if (!isRecord6(entry)) {
50227
+ if (!isRecord8(entry)) {
49308
50228
  throw new Error(`Experiment targets[${index}] must be a string or object.`);
49309
50229
  }
49310
50230
  const name = readRequiredString3(entry.name, `targets[${index}].name`);
@@ -49415,7 +50335,7 @@ function readOptionalRecord(raw) {
49415
50335
  if (raw === void 0) {
49416
50336
  return void 0;
49417
50337
  }
49418
- if (!isRecord6(raw)) {
50338
+ if (!isRecord8(raw)) {
49419
50339
  throw new Error("Experiment object field must be an object.");
49420
50340
  }
49421
50341
  return raw;
@@ -49436,7 +50356,7 @@ function rejectExperimentWorkers(raw) {
49436
50356
  "Experiment workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency."
49437
50357
  );
49438
50358
  }
49439
- function isRecord6(value) {
50359
+ function isRecord8(value) {
49440
50360
  return typeof value === "object" && value !== null && !Array.isArray(value);
49441
50361
  }
49442
50362
  function rejectExperimentLifecycleCommands(rawConfig) {
@@ -50321,18 +51241,83 @@ var UNSUPPORTED_PROMPTFOO_ASSERTION_TYPES = /* @__PURE__ */ new Set([
50321
51241
  "human",
50322
51242
  "max-score",
50323
51243
  "tool-call-f1",
50324
- "skill-used",
50325
- "trajectory:goal-success",
50326
- "trajectory:tool-args-match",
50327
- "trajectory:step-count",
50328
- "trajectory:tool-sequence",
50329
- "trajectory:tool-used",
50330
51244
  "trace-error-spans",
50331
51245
  "trace-span-count",
50332
51246
  "trace-span-duration",
50333
51247
  "search-rubric",
50334
51248
  "word-count"
50335
51249
  ]);
51250
+ var TRAJECTORY_GRADER_TYPES = /* @__PURE__ */ new Set([
51251
+ "trajectory:tool-used",
51252
+ "trajectory:tool-args-match",
51253
+ "trajectory:tool-sequence",
51254
+ "trajectory:step-count",
51255
+ "trajectory:goal-success"
51256
+ ]);
51257
+ function isTrajectoryGraderKind(value) {
51258
+ return TRAJECTORY_GRADER_TYPES.has(value);
51259
+ }
51260
+ function formatJsonValue(value) {
51261
+ if (typeof value === "string") {
51262
+ return value;
51263
+ }
51264
+ try {
51265
+ return JSON.stringify(value);
51266
+ } catch {
51267
+ return String(value);
51268
+ }
51269
+ }
51270
+ function hasToolTrajectoryLatencyCheck(rawEvaluator) {
51271
+ const expected = rawEvaluator.expected;
51272
+ return Array.isArray(expected) && expected.some(
51273
+ (item) => isJsonObject2(item) && (item.max_duration_ms !== void 0 || item.maxDurationMs !== void 0)
51274
+ );
51275
+ }
51276
+ function staleSkillTriggerMessage(rawEvaluator) {
51277
+ const skillName = asString3(rawEvaluator.skill);
51278
+ const shouldTrigger = rawEvaluator.should_trigger !== false;
51279
+ if (!skillName) {
51280
+ return "Authored assertion type 'skill-trigger' has been removed. Use 'skill-used' with value: <skill> for expected skill use, or 'not-skill-used' with value: <skill> when the skill must not be used.";
51281
+ }
51282
+ const replacementType = shouldTrigger ? "skill-used" : "not-skill-used";
51283
+ return `Authored assertion type 'skill-trigger' has been removed. Replace skill: ${skillName} with type: ${replacementType}, value: ${skillName}.`;
51284
+ }
51285
+ function staleToolTrajectoryMessage(rawEvaluator) {
51286
+ const mode = asString3(rawEvaluator.mode);
51287
+ if (hasToolTrajectoryLatencyCheck(rawEvaluator)) {
51288
+ return "Authored assertion type 'tool-trajectory' has been removed. Per-tool latency checks such as max_duration_ms have no Promptfoo trajectory:* equivalent in AgentV yet; use a custom script assertion or track this as unsupported future scope.";
51289
+ }
51290
+ if (mode === "any_order" && isJsonObject2(rawEvaluator.minimums)) {
51291
+ const entries = Object.entries(rawEvaluator.minimums);
51292
+ const [toolName, min] = entries[0] ?? [];
51293
+ const example = typeof toolName === "string" ? ` For example: type: trajectory:tool-used, value: { name: ${toolName}, min: ${formatJsonValue(min)} }.` : "";
51294
+ return `Authored assertion type 'tool-trajectory' has been removed. Replace mode: any_order minimums with one Promptfoo trajectory:tool-used assertion per tool.${example}`;
51295
+ }
51296
+ if ((mode === "in_order" || mode === "exact") && Array.isArray(rawEvaluator.expected)) {
51297
+ const steps = rawEvaluator.expected.filter(isJsonObject2).map((item) => item.tool).filter((tool) => typeof tool === "string");
51298
+ const hasArgs = rawEvaluator.expected.some(
51299
+ (item) => isJsonObject2(item) && item.args !== void 0 && item.args !== "any"
51300
+ );
51301
+ const sequenceHint = steps.length > 0 ? ` Use type: trajectory:tool-sequence, value: { mode: ${mode}, steps: ${formatJsonValue(steps)} }.` : " Use type: trajectory:tool-sequence with value: { mode, steps }.";
51302
+ const argsHint = hasArgs ? " Move expected args checks to trajectory:tool-args-match with Promptfoo mode: partial or exact." : "";
51303
+ return `Authored assertion type 'tool-trajectory' has been removed.${sequenceHint}${argsHint}`;
51304
+ }
51305
+ return "Authored assertion type 'tool-trajectory' has been removed. Use Promptfoo trajectory:* assertions: trajectory:tool-used for tool presence/counts, trajectory:tool-sequence for in_order/exact steps, and trajectory:tool-args-match for argument checks. AgentV-specific latency checks are unsupported future scope.";
51306
+ }
51307
+ function staleAuthoredGraderMessage(rawEvaluator) {
51308
+ const rawType = rawEvaluator.type;
51309
+ if (typeof rawType !== "string") {
51310
+ return void 0;
51311
+ }
51312
+ const type = rawType.replace(/_/g, "-");
51313
+ if (type === "skill-trigger") {
51314
+ return staleSkillTriggerMessage(rawEvaluator);
51315
+ }
51316
+ if (type === "tool-trajectory") {
51317
+ return staleToolTrajectoryMessage(rawEvaluator);
51318
+ }
51319
+ return void 0;
51320
+ }
50336
51321
  function assertSupportedPromptfooType(type, evalId, name) {
50337
51322
  const baseType = type.startsWith("not-") ? type.slice(4) : type;
50338
51323
  if (!UNSUPPORTED_PROMPTFOO_ASSERTION_TYPES.has(baseType)) {
@@ -50607,15 +51592,24 @@ async function parseGraderList(candidateEvaluators, searchRoots, evalId, default
50607
51592
  const rawEvaluator = withInheritedAssertionConfig(rawEvaluatorEntry, inheritedAssertionConfig);
50608
51593
  const rawName = asString3(rawEvaluator.metric);
50609
51594
  const rawType = rawEvaluator.type;
50610
- const typeValue = typeof rawType === "string" ? normalizeGraderType(rawType) : rawType;
50611
- if (typeof typeValue === "string") {
50612
- const replacement = removedGraderReplacement(typeValue);
51595
+ const normalizedType = typeof rawType === "string" ? normalizeGraderType(rawType) : rawType;
51596
+ const negatedType = typeof normalizedType === "string" && normalizedType.startsWith("not-") ? normalizedType.slice(4) : void 0;
51597
+ const typeValue = negatedType && isTrajectoryGraderKind(negatedType) ? negatedType : normalizedType;
51598
+ const inverse = negatedType && isTrajectoryGraderKind(negatedType) ? true : void 0;
51599
+ if (typeof normalizedType === "string") {
51600
+ const staleMessage = staleAuthoredGraderMessage(rawEvaluator);
51601
+ if (staleMessage) {
51602
+ throw new Error(
51603
+ `Unsupported grader '${rawType}' in '${evalId}'${rawName ? ` for evaluator '${rawName}'` : ""}. ${staleMessage}`
51604
+ );
51605
+ }
51606
+ const replacement = removedGraderReplacement(normalizedType);
50613
51607
  if (replacement) {
50614
51608
  throw new Error(
50615
51609
  `Unsupported grader '${rawType}' in '${evalId}'. Use '${replacement}' instead.`
50616
51610
  );
50617
51611
  }
50618
- assertSupportedPromptfooType(typeValue, evalId, rawName);
51612
+ assertSupportedPromptfooType(normalizedType, evalId, rawName);
50619
51613
  }
50620
51614
  const isCustomType = typeof typeValue === "string" && !isGraderKind(typeValue);
50621
51615
  if (typeof typeValue !== "string") {
@@ -50648,6 +51642,25 @@ async function parseGraderList(candidateEvaluators, searchRoots, evalId, default
50648
51642
  evaluators.push(transform2 !== void 0 ? { ...config22, transform: transform2 } : config22);
50649
51643
  };
50650
51644
  const inheritedInternalPreprocessors = defaultPreprocessors;
51645
+ if (isTrajectoryGraderKind(typeValue)) {
51646
+ const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
51647
+ const { required: required22, min_score: min_score2 } = parseRequiredAndMinScore(
51648
+ rawEvaluator.required,
51649
+ rawEvaluator.min_score,
51650
+ name,
51651
+ evalId
51652
+ );
51653
+ pushEvaluator({
51654
+ name,
51655
+ type: typeValue,
51656
+ ...rawEvaluator.value !== void 0 ? { value: rawEvaluator.value } : {},
51657
+ ...weight2 !== void 0 ? { weight: weight2 } : {},
51658
+ ...required22 !== void 0 ? { required: required22 } : {},
51659
+ ...min_score2 !== void 0 ? { min_score: min_score2 } : {},
51660
+ ...inverse || negate ? { inverse: true } : {}
51661
+ });
51662
+ continue;
51663
+ }
50651
51664
  if (isCustomType) {
50652
51665
  const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
50653
51666
  const { required: required22, min_score: min_score2 } = parseRequiredAndMinScore(
@@ -50862,18 +51875,18 @@ async function parseGraderList(candidateEvaluators, searchRoots, evalId, default
50862
51875
  }
50863
51876
  }
50864
51877
  const rawArgsMatch = rawEvaluator.args_match ?? rawEvaluator.argsMatch;
50865
- let argsMatch2;
51878
+ let argsMatch3;
50866
51879
  if (rawArgsMatch !== void 0) {
50867
51880
  if (Array.isArray(rawArgsMatch)) {
50868
51881
  const fieldList = rawArgsMatch.filter(
50869
51882
  (f) => typeof f === "string" && f.length > 0
50870
51883
  );
50871
51884
  if (fieldList.length > 0) {
50872
- argsMatch2 = fieldList;
51885
+ argsMatch3 = fieldList;
50873
51886
  }
50874
51887
  } else if (typeof rawArgsMatch === "string") {
50875
51888
  if (rawArgsMatch === "exact" || rawArgsMatch === "superset" || rawArgsMatch === "subset" || rawArgsMatch === "ignore") {
50876
- argsMatch2 = rawArgsMatch;
51889
+ argsMatch3 = rawArgsMatch;
50877
51890
  } else {
50878
51891
  logWarning2(
50879
51892
  `Invalid args_match '${rawArgsMatch}' for tool-trajectory evaluator '${name}' in '${evalId}': must be exact, superset, subset, ignore, or a string array`
@@ -50959,11 +51972,30 @@ async function parseGraderList(candidateEvaluators, searchRoots, evalId, default
50959
51972
  ...required22 !== void 0 ? { required: required22 } : {},
50960
51973
  ...min_score2 !== void 0 ? { min_score: min_score2 } : {},
50961
51974
  ...negate !== void 0 ? { negate } : {},
50962
- ...argsMatch2 !== void 0 ? { argsMatch: argsMatch2 } : {}
51975
+ ...argsMatch3 !== void 0 ? { argsMatch: argsMatch3 } : {}
50963
51976
  };
50964
51977
  pushEvaluator(config22);
50965
51978
  continue;
50966
51979
  }
51980
+ if (typeValue === "skill-used" || typeValue === "not-skill-used") {
51981
+ const weight2 = validateWeight(rawEvaluator.weight, name, evalId);
51982
+ const { required: required22, min_score: min_score2 } = parseRequiredAndMinScore(
51983
+ rawEvaluator.required,
51984
+ rawEvaluator.min_score,
51985
+ name,
51986
+ evalId
51987
+ );
51988
+ pushEvaluator({
51989
+ name,
51990
+ type: typeValue,
51991
+ value: rawEvaluator.value,
51992
+ ...weight2 !== void 0 ? { weight: weight2 } : {},
51993
+ ...required22 !== void 0 ? { required: required22 } : {},
51994
+ ...min_score2 !== void 0 ? { min_score: min_score2 } : {},
51995
+ ...negate !== void 0 ? { negate } : {}
51996
+ });
51997
+ continue;
51998
+ }
50967
51999
  if (typeValue === "field-accuracy") {
50968
52000
  const rawFields = rawEvaluator.fields;
50969
52001
  if (!Array.isArray(rawFields)) {
@@ -51667,6 +52699,17 @@ function generateAssertionName(typeValue, rawEvaluator) {
51667
52699
  const value = asString3(rawEvaluator.value);
51668
52700
  const arrayValue = Array.isArray(rawEvaluator.value) ? rawEvaluator.value : void 0;
51669
52701
  switch (typeValue) {
52702
+ case "skill-used":
52703
+ case "not-skill-used": {
52704
+ const rawValue = rawEvaluator.value;
52705
+ if (typeof rawValue === "string" && rawValue.trim()) {
52706
+ return `${typeValue}-${rawValue.trim()}`;
52707
+ }
52708
+ if (Array.isArray(rawValue)) {
52709
+ return `${typeValue}-${rawValue.length}`;
52710
+ }
52711
+ return typeValue;
52712
+ }
51670
52713
  case "skill-trigger": {
51671
52714
  const skillValue = asString3(rawEvaluator.skill);
51672
52715
  return skillValue ? `skill-trigger-${skillValue}` : "skill-trigger";
@@ -53538,7 +54581,7 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
53538
54581
  return { tests: await loadTestsFromJsonl(evalFilePath, repoRoot, options) };
53539
54582
  }
53540
54583
  if (format === "typescript") {
53541
- const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-OMG2JBHP-OP5RR7A3.js");
54584
+ const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-DQDYRULE-V3377FOL.js");
53542
54585
  return loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
53543
54586
  }
53544
54587
  const { tests, parsed } = await loadTestsFromYaml(evalFilePath, repoRoot, options);
@@ -53560,7 +54603,7 @@ async function loadTests(evalFilePath, repoRoot, options) {
53560
54603
  return loadTestsFromJsonl(evalFilePath, repoRoot, options);
53561
54604
  }
53562
54605
  if (format === "typescript") {
53563
- const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-OMG2JBHP-OP5RR7A3.js");
54606
+ const { loadTsEvalSuite: loadTsEvalSuite2 } = await import("./ts-eval-loader-DQDYRULE-V3377FOL.js");
53564
54607
  const suite = await loadTsEvalSuite2(evalFilePath, resolveToAbsolutePath(repoRoot), options);
53565
54608
  return suite.tests;
53566
54609
  }
@@ -55111,17 +56154,17 @@ ${detailBlock}${ANSI_RESET5}`);
55111
56154
  function pathFromRoot(root) {
55112
56155
  return root instanceof URL ? fileURLToPath3(root) : String(root);
55113
56156
  }
55114
- function classifyQualityStatus(score, threshold = DEFAULT_THRESHOLD2) {
55115
- return score >= threshold ? "ok" : "quality_failure";
56157
+ function classifyQualityStatus(score2, threshold = DEFAULT_THRESHOLD2) {
56158
+ return score2 >= threshold ? "ok" : "quality_failure";
55116
56159
  }
55117
56160
  function buildSkippedEvaluatorError(scores) {
55118
- const skippedScores = scores?.filter((score) => score.verdict === "skip") ?? [];
56161
+ const skippedScores = scores?.filter((score2) => score2.verdict === "skip") ?? [];
55119
56162
  if (skippedScores.length === 0) {
55120
56163
  return void 0;
55121
56164
  }
55122
- const messages = skippedScores.map((score) => {
55123
- const label = score.name || score.type;
55124
- const assertionMessage = score.assertions.find((assertion) => !assertion.passed)?.text ?? "Grader skipped";
56165
+ const messages = skippedScores.map((score2) => {
56166
+ const label = score2.name || score2.type;
56167
+ const assertionMessage = score2.assertions.find((assertion) => !assertion.passed)?.text ?? "Grader skipped";
55125
56168
  return `${label}: ${assertionMessage}`;
55126
56169
  });
55127
56170
  return messages.length === 1 ? messages[0] : `Graders skipped: ${messages.join(" | ")}`;
@@ -55424,7 +56467,7 @@ async function gradePreparedEvalCase(options) {
55424
56467
  const gradingOutput = evalCase.outputTransform ? [
55425
56468
  { role: "assistant", content: preparedCandidate.candidate }
55426
56469
  ] : preparedTrace ? outputMessages : void 0;
55427
- const { score, scores } = await runEvaluatorsForCase({
56470
+ const { score: score2, scores } = await runEvaluatorsForCase({
55428
56471
  evalCase,
55429
56472
  candidate: preparedCandidate.candidate,
55430
56473
  candidateValue: preparedCandidate.candidateValue,
@@ -55458,7 +56501,7 @@ async function gradePreparedEvalCase(options) {
55458
56501
  ...graderTokens ? { tokenUsage: graderTokens } : {}
55459
56502
  };
55460
56503
  const skippedEvaluatorError = buildSkippedEvaluatorError(scores);
55461
- const executionStatus = skippedEvaluatorError ? "execution_error" : classifyQualityStatus(score.score, effectiveThreshold);
56504
+ const executionStatus = skippedEvaluatorError ? "execution_error" : classifyQualityStatus(score2.score, effectiveThreshold);
55462
56505
  const baseResult = {
55463
56506
  timestamp: timestamp.toISOString(),
55464
56507
  testId: authoredResultTestId(evalCase),
@@ -55467,10 +56510,10 @@ async function gradePreparedEvalCase(options) {
55467
56510
  suite: evalCase.suite,
55468
56511
  category: evalCase.category,
55469
56512
  conversationId: evalCase.conversation_id,
55470
- score: skippedEvaluatorError ? 0 : score.score,
55471
- reason: score.reason,
55472
- checks: score.checks,
55473
- assertions: score.assertions,
56513
+ score: skippedEvaluatorError ? 0 : score2.score,
56514
+ reason: score2.reason,
56515
+ checks: score2.checks,
56516
+ assertions: score2.assertions,
55474
56517
  target: target.name,
55475
56518
  input,
55476
56519
  output: preparedCandidate.candidate,
@@ -56946,7 +57989,7 @@ async function runEvalCaseWithTrials(options, trialsConfig) {
56946
57989
  break;
56947
57990
  }
56948
57991
  }
56949
- const { score, aggregation } = aggregateTrials(trialResults, trialsConfig);
57992
+ const { score: score2, aggregation } = aggregateTrials(trialResults, trialsConfig);
56950
57993
  const bestTrialIndex = trialResults.reduce(
56951
57994
  (bestIdx, t, idx) => t.score > trialResults[bestIdx].score ? idx : bestIdx,
56952
57995
  0
@@ -56960,7 +58003,7 @@ async function runEvalCaseWithTrials(options, trialsConfig) {
56960
58003
  const aggregateExecutionError = aggregateExecutionStatus === "execution_error" ? baseResult.executionError : void 0;
56961
58004
  return {
56962
58005
  ...baseResult,
56963
- score,
58006
+ score: score2,
56964
58007
  sampleIndex: void 0,
56965
58008
  retryIndex: void 0,
56966
58009
  trials: trialResults,
@@ -57020,6 +58063,9 @@ function toJsonValue(value) {
57020
58063
  function buildProviderResponseTransformMetadata(response) {
57021
58064
  const metadata = {};
57022
58065
  const fields = [
58066
+ ["metadata", response.metadata],
58067
+ ["skill_calls", response.metadata?.skillCalls],
58068
+ ["attempted_skill_calls", response.metadata?.attemptedSkillCalls],
57023
58069
  ["raw", response.raw],
57024
58070
  ["usage", response.usage],
57025
58071
  ["token_usage", response.tokenUsage],
@@ -57118,7 +58164,7 @@ async function evaluateCandidate(options) {
57118
58164
  const gradingOutput = evalCase.outputTransform ? [
57119
58165
  { role: "assistant", content: preparedCandidate.candidate }
57120
58166
  ] : output;
57121
- const { score, scores } = await runEvaluatorsForCase({
58167
+ const { score: score2, scores } = await runEvaluatorsForCase({
57122
58168
  evalCase,
57123
58169
  candidate: preparedCandidate.candidate,
57124
58170
  candidateValue: preparedCandidate.candidateValue,
@@ -57166,7 +58212,7 @@ async function evaluateCandidate(options) {
57166
58212
  };
57167
58213
  }
57168
58214
  }
57169
- const evaluatorRequest = scores ? void 0 : score.graderRawRequest;
58215
+ const evaluatorRequest = scores ? void 0 : score2.graderRawRequest;
57170
58216
  const effectiveAgentRequest = agentRequest && Object.keys(agentRequest).length > 0 ? agentRequest : void 0;
57171
58217
  const requests = effectiveAgentRequest || lmRequest || evaluatorRequest ? {
57172
58218
  ...effectiveAgentRequest ? { agent: effectiveAgentRequest } : {},
@@ -57183,10 +58229,10 @@ async function evaluateCandidate(options) {
57183
58229
  suite: evalCase.suite,
57184
58230
  category: evalCase.category,
57185
58231
  conversationId: evalCase.conversation_id,
57186
- score: score.score,
57187
- reason: score.reason,
57188
- checks: score.checks,
57189
- assertions: score.assertions,
58232
+ score: score2.score,
58233
+ reason: score2.reason,
58234
+ checks: score2.checks,
58235
+ assertions: score2.assertions,
57190
58236
  target: target.name,
57191
58237
  tokenUsage,
57192
58238
  costUsd,
@@ -57200,7 +58246,7 @@ async function evaluateCandidate(options) {
57200
58246
  trace: evaluationTrace,
57201
58247
  rawProviderLogPath,
57202
58248
  fileChanges,
57203
- executionStatus: classifyQualityStatus(score.score, evalThreshold)
58249
+ executionStatus: classifyQualityStatus(score2.score, evalThreshold)
57204
58250
  };
57205
58251
  }
57206
58252
  async function runEvaluatorsForCase(options) {
@@ -57288,7 +58334,7 @@ async function runEvaluatorsForCase(options) {
57288
58334
  throw new Error(`No evaluator registered for kind '${evaluatorKind}'`);
57289
58335
  }
57290
58336
  const implicitEvaluator = evaluatorKind === "llm-grader" && !evalCase.assertions ? buildImplicitLlmGraderConfig(evalCase) : void 0;
57291
- const score = await activeEvaluator.evaluate({
58337
+ const score2 = await activeEvaluator.evaluate({
57292
58338
  evalCase,
57293
58339
  candidate,
57294
58340
  candidateValue,
@@ -57315,7 +58361,7 @@ async function runEvaluatorsForCase(options) {
57315
58361
  dependencyResults,
57316
58362
  ...implicitEvaluator ? { evaluator: implicitEvaluator } : {}
57317
58363
  });
57318
- return { score };
58364
+ return { score: score2 };
57319
58365
  }
57320
58366
  function buildImplicitLlmGraderConfig(evalCase) {
57321
58367
  if (!evalCase.preprocessors || evalCase.preprocessors.length === 0) {
@@ -57435,11 +58481,11 @@ async function runEvaluatorList(options) {
57435
58481
  evalContext,
57436
58482
  evaluatorConfig
57437
58483
  );
57438
- const score2 = await evaluatorInstance.evaluate(transformedContext.context);
58484
+ const score3 = await evaluatorInstance.evaluate(transformedContext.context);
57439
58485
  const endedAt = /* @__PURE__ */ new Date();
57440
58486
  const weight = evaluatorConfig.weight ?? 1;
57441
58487
  scored.push({
57442
- score: score2,
58488
+ score: score3,
57443
58489
  name: evaluatorConfig.name,
57444
58490
  type: evaluatorConfig.type,
57445
58491
  weight,
@@ -57449,17 +58495,17 @@ async function runEvaluatorList(options) {
57449
58495
  scores.push({
57450
58496
  name: evaluatorConfig.name,
57451
58497
  type: evaluatorConfig.type,
57452
- score: score2.score,
58498
+ score: score3.score,
57453
58499
  weight,
57454
- verdict: score2.verdict,
57455
- reason: score2.reason,
57456
- checks: score2.checks,
57457
- assertions: score2.assertions,
57458
- input: transformedContext.input ?? score2.graderRawRequest,
57459
- target: score2.graderTarget,
57460
- details: transformedContext.details && score2.details ? { ...transformedContext.details, ...score2.details } : transformedContext.details ?? score2.details,
57461
- scores: mapChildResults(score2.scores),
57462
- tokenUsage: score2.tokenUsage,
58500
+ verdict: score3.verdict,
58501
+ reason: score3.reason,
58502
+ checks: score3.checks,
58503
+ assertions: score3.assertions,
58504
+ input: transformedContext.input ?? score3.graderRawRequest,
58505
+ target: score3.graderTarget,
58506
+ details: transformedContext.details && score3.details ? { ...transformedContext.details, ...score3.details } : transformedContext.details ?? score3.details,
58507
+ scores: mapChildResults(score3.scores),
58508
+ tokenUsage: score3.tokenUsage,
57463
58509
  durationMs: endedAt.getTime() - startedAt.getTime(),
57464
58510
  startedAt: startedAt.toISOString(),
57465
58511
  endedAt: endedAt.toISOString()
@@ -57528,13 +58574,13 @@ async function runEvaluatorList(options) {
57528
58574
  ) : 0;
57529
58575
  const assertions = scored.flatMap((entry) => entry.score.assertions);
57530
58576
  const expectedAspectCount = assertions.length || 1;
57531
- const score = {
58577
+ const score2 = {
57532
58578
  score: aggregateScore,
57533
58579
  verdict: scoreToVerdict(aggregateScore, effectiveThreshold),
57534
58580
  assertions,
57535
58581
  expectedAspectCount
57536
58582
  };
57537
- return { score, scores };
58583
+ return { score: score2, scores };
57538
58584
  }
57539
58585
  function formatFilter(filter) {
57540
58586
  return typeof filter === "string" ? filter : filter.join(", ");
@@ -58255,7 +59301,7 @@ var MetricsWireSchema = external_exports.object({
58255
59301
  reasoning_blocks: external_exports.array(ReasoningBlockWireSchema),
58256
59302
  thinking_blocks: external_exports.number().int().nonnegative()
58257
59303
  }).strict();
58258
- var MetricsArtifactWireSchema = external_exports.object({
59304
+ var MetricsArtifactBaseWireSchema = external_exports.object({
58259
59305
  schema_version: external_exports.literal(METRICS_SCHEMA_VERSION),
58260
59306
  artifact_id: external_exports.string(),
58261
59307
  generated_at: external_exports.string(),
@@ -58263,12 +59309,6 @@ var MetricsArtifactWireSchema = external_exports.object({
58263
59309
  target: external_exports.string(),
58264
59310
  suite: external_exports.string().optional(),
58265
59311
  category: external_exports.string().optional(),
58266
- trace: external_exports.object({
58267
- schema_version: external_exports.literal(EXECUTION_TRACE_SCHEMA_VERSION),
58268
- artifact_id: external_exports.string(),
58269
- trace_id: external_exports.string(),
58270
- root_span_id: external_exports.string()
58271
- }).strict(),
58272
59312
  source_artifacts: external_exports.object({
58273
59313
  transcript_path: external_exports.string().optional(),
58274
59314
  grading_path: external_exports.string().optional(),
@@ -58278,17 +59318,32 @@ var MetricsArtifactWireSchema = external_exports.object({
58278
59318
  tokens: MetricsTokensWireSchema.optional(),
58279
59319
  cost: MetricsCostWireSchema.optional(),
58280
59320
  execution: external_exports.record(external_exports.string(), external_exports.unknown()).optional(),
58281
- trajectory: external_exports.record(external_exports.string(), external_exports.unknown()).optional(),
58282
- metrics: MetricsWireSchema
59321
+ trajectory: external_exports.record(external_exports.string(), external_exports.unknown()).optional()
58283
59322
  }).strict();
59323
+ var MetricsArtifactWireSchema = MetricsArtifactBaseWireSchema.merge(MetricsWireSchema);
58284
59324
  function dropUndefined7(value) {
58285
59325
  return Object.fromEntries(
58286
59326
  Object.entries(value).filter(([, entryValue]) => entryValue !== void 0)
58287
59327
  );
58288
59328
  }
58289
- function isRecord7(value) {
59329
+ function isRecord9(value) {
58290
59330
  return typeof value === "object" && value !== null && !Array.isArray(value);
58291
59331
  }
59332
+ function normalizeMetricsArtifactWire(value) {
59333
+ if (!isRecord9(value)) {
59334
+ return void 0;
59335
+ }
59336
+ const nestedMetrics = isRecord9(value.metrics) ? value.metrics : {};
59337
+ const normalized = Object.fromEntries(
59338
+ Object.entries({
59339
+ ...value,
59340
+ ...nestedMetrics,
59341
+ schema_version: METRICS_SCHEMA_VERSION
59342
+ }).filter(([key]) => key !== "trace" && key !== "metrics")
59343
+ );
59344
+ const parsed = MetricsArtifactWireSchema.safeParse(normalized);
59345
+ return parsed.success ? parsed.data : void 0;
59346
+ }
58292
59347
  function normalizedKey2(key) {
58293
59348
  return key.replace(/[^A-Za-z0-9]+/g, "").toLowerCase();
58294
59349
  }
@@ -58314,7 +59369,7 @@ function collectStringValuesByKey(value, keys, options = {}) {
58314
59369
  }
58315
59370
  return;
58316
59371
  }
58317
- if (!isRecord7(entry)) {
59372
+ if (!isRecord9(entry)) {
58318
59373
  return;
58319
59374
  }
58320
59375
  for (const [key, nested] of Object.entries(entry)) {
@@ -58350,7 +59405,7 @@ function findNumberByKey(value, keys, maxDepth = 4) {
58350
59405
  }
58351
59406
  return void 0;
58352
59407
  }
58353
- if (!isRecord7(entry)) {
59408
+ if (!isRecord9(entry)) {
58354
59409
  return void 0;
58355
59410
  }
58356
59411
  for (const [key, nested] of Object.entries(entry)) {
@@ -58451,7 +59506,7 @@ function toolSucceeded(toolCall) {
58451
59506
  if (toolCall.status === "error" || toolCall.status === "timeout" || toolCall.status === "cancelled") {
58452
59507
  return false;
58453
59508
  }
58454
- const explicitSuccess = booleanValue(isRecord7(toolCall.output) ? toolCall.output.success : void 0) ?? booleanValue(isRecord7(toolCall.output) ? toolCall.output.ok : void 0);
59509
+ const explicitSuccess = booleanValue(isRecord9(toolCall.output) ? toolCall.output.success : void 0) ?? booleanValue(isRecord9(toolCall.output) ? toolCall.output.ok : void 0);
58455
59510
  if (explicitSuccess !== void 0) {
58456
59511
  return explicitSuccess;
58457
59512
  }
@@ -58635,7 +59690,7 @@ function buildWebFetches(calls) {
58635
59690
  return [
58636
59691
  dropUndefined7({
58637
59692
  url: url2,
58638
- method: stringValue2(isRecord7(call.toolCall.input) ? call.toolCall.input.method : void 0),
59693
+ method: stringValue2(isRecord9(call.toolCall.input) ? call.toolCall.input.method : void 0),
58639
59694
  status: findNumberByKey(call.toolCall.output, STATUS_CODE_KEY_SET),
58640
59695
  success: toolSucceeded(call.toolCall),
58641
59696
  tool_call_id: call.toolCall.id
@@ -58647,7 +59702,7 @@ function errorFromUnknown(value, fallback) {
58647
59702
  if (typeof value === "string" && value.trim().length > 0) {
58648
59703
  return dropUndefined7({ message: value.trim(), ...fallback });
58649
59704
  }
58650
- if (!isRecord7(value)) {
59705
+ if (!isRecord9(value)) {
58651
59706
  return void 0;
58652
59707
  }
58653
59708
  const message = stringValue2(value.message) ?? stringValue2(value.error) ?? stringValue2(value.reason);
@@ -58710,7 +59765,7 @@ function contentBlockText(value) {
58710
59765
  if (typeof value === "string") {
58711
59766
  return value.trim().length > 0 ? value : void 0;
58712
59767
  }
58713
- if (!isRecord7(value)) {
59768
+ if (!isRecord9(value)) {
58714
59769
  return void 0;
58715
59770
  }
58716
59771
  return stringValue2(value.text) ?? stringValue2(value.content) ?? stringValue2(value.value) ?? stringValue2(value.summary);
@@ -58764,7 +59819,7 @@ function buildReasoningBlocks(messages, events) {
58764
59819
  if (Array.isArray(content)) {
58765
59820
  for (const item of content) {
58766
59821
  const block = item;
58767
- if (!isRecord7(block)) {
59822
+ if (!isRecord9(block)) {
58768
59823
  continue;
58769
59824
  }
58770
59825
  const blockType = block.type;
@@ -58843,18 +59898,12 @@ function buildMetricsArtifact(result, envelope, options = {}) {
58843
59898
  target: result.target ?? "unknown",
58844
59899
  suite: result.suite,
58845
59900
  category: result.category,
58846
- trace: {
58847
- schema_version: EXECUTION_TRACE_SCHEMA_VERSION,
58848
- artifact_id: envelope.artifactId,
58849
- trace_id: envelope.trace.traceId,
58850
- root_span_id: envelope.trace.rootSpanId
58851
- },
58852
59901
  source_artifacts: dropUndefined7({
58853
59902
  transcript_path: options.transcriptPath,
58854
59903
  grading_path: options.gradingPath,
58855
59904
  file_changes_path: options.fileChangesPath
58856
59905
  }),
58857
- metrics: buildMetrics(result)
59906
+ ...buildMetrics(result)
58858
59907
  })
58859
59908
  );
58860
59909
  }
@@ -58929,7 +59978,7 @@ var TOOL_CALL_ALIASES = {
58929
59978
  endTime: "end_time",
58930
59979
  startTime: "start_time"
58931
59980
  };
58932
- function isRecord8(value) {
59981
+ function isRecord10(value) {
58933
59982
  return typeof value === "object" && value !== null && !Array.isArray(value);
58934
59983
  }
58935
59984
  function normalizeKnownAliases(value, aliases) {
@@ -58945,13 +59994,13 @@ function normalizeKnownAliases(value, aliases) {
58945
59994
  return normalized;
58946
59995
  }
58947
59996
  function normalizeToolCall2(value) {
58948
- if (!isRecord8(value)) {
59997
+ if (!isRecord10(value)) {
58949
59998
  return value;
58950
59999
  }
58951
60000
  return normalizeKnownAliases(value, TOOL_CALL_ALIASES);
58952
60001
  }
58953
60002
  function normalizeMessage(value) {
58954
- if (!isRecord8(value)) {
60003
+ if (!isRecord10(value)) {
58955
60004
  return value;
58956
60005
  }
58957
60006
  const normalized = normalizeKnownAliases(value, MESSAGE_ALIASES);
@@ -58961,7 +60010,7 @@ function normalizeMessage(value) {
58961
60010
  return normalized;
58962
60011
  }
58963
60012
  function normalizeTraceSummary(value) {
58964
- if (!isRecord8(value)) {
60013
+ if (!isRecord10(value)) {
58965
60014
  return value;
58966
60015
  }
58967
60016
  const normalized = normalizeKnownAliases(value, TRACE_SUMMARY_ALIASES);
@@ -59003,7 +60052,7 @@ function looksLikeResultRow(value) {
59003
60052
  return typeof value.test_id === "string" || Object.hasOwn(value, "score") || Object.hasOwn(value, "trace") || Object.hasOwn(value, "spans") || Object.hasOwn(value, "target") || Object.hasOwn(value, "summary_path") || Object.hasOwn(value, "grading_path") || Object.hasOwn(value, "timing_path");
59004
60053
  }
59005
60054
  function normalizeResultRow(value, context = {}) {
59006
- if (!isRecord8(value)) {
60055
+ if (!isRecord10(value)) {
59007
60056
  throw buildSchemaError(context);
59008
60057
  }
59009
60058
  for (const field of Object.keys(
@@ -59066,11 +60115,11 @@ function stringField2(record2, key) {
59066
60115
  }
59067
60116
  function resultProjectionDimensions(result) {
59068
60117
  const projectionIdentity = result.projectionIdentity;
59069
- if (!isRecord9(projectionIdentity)) {
60118
+ if (!isRecord11(projectionIdentity)) {
59070
60119
  return void 0;
59071
60120
  }
59072
60121
  const dimensions = projectionIdentity.dimensions;
59073
- return isRecord9(dimensions) ? dimensions : void 0;
60122
+ return isRecord11(dimensions) ? dimensions : void 0;
59074
60123
  }
59075
60124
  function buildEvaluationResultTargetKey(result) {
59076
60125
  const dimensions = resultProjectionDimensions(result);
@@ -59206,7 +60255,7 @@ function normalizeEnvironmentSummaries(value) {
59206
60255
  return void 0;
59207
60256
  }
59208
60257
  const summaries = value.filter((entry) => {
59209
- if (!isRecord9(entry)) {
60258
+ if (!isRecord11(entry)) {
59210
60259
  return false;
59211
60260
  }
59212
60261
  return entry.schema_version === "agentv.environment_summary.v1" && (entry.type === "host" || entry.type === "docker") && typeof entry.workdir === "string" && typeof entry.recipe_sha256 === "string" && (entry.authored_kind === "inline" || entry.authored_kind === "file");
@@ -59214,7 +60263,7 @@ function normalizeEnvironmentSummaries(value) {
59214
60263
  return summaries.length > 0 ? summaries : void 0;
59215
60264
  }
59216
60265
  function normalizeStringRecord(value) {
59217
- if (!isRecord9(value)) {
60266
+ if (!isRecord11(value)) {
59218
60267
  return void 0;
59219
60268
  }
59220
60269
  const out = {};
@@ -59290,7 +60339,7 @@ function parseWorkspaceChanges(fileChanges) {
59290
60339
  }
59291
60340
  function resultVerdict(result) {
59292
60341
  const scores = result.scores ?? [];
59293
- if (scores.length > 0 && scores.every((score) => score.verdict === "skip")) {
60342
+ if (scores.length > 0 && scores.every((score2) => score2.verdict === "skip")) {
59294
60343
  return "skip";
59295
60344
  }
59296
60345
  if (result.executionStatus === "ok") {
@@ -59298,8 +60347,8 @@ function resultVerdict(result) {
59298
60347
  }
59299
60348
  return "fail";
59300
60349
  }
59301
- function passFromVerdict(verdict, score) {
59302
- return verdict ? verdict === "pass" : score >= DEFAULT_THRESHOLD2;
60350
+ function passFromVerdict(verdict, score2) {
60351
+ return verdict ? verdict === "pass" : score2 >= DEFAULT_THRESHOLD2;
59303
60352
  }
59304
60353
  function assertionMetadata(metadata) {
59305
60354
  const compact = dropUndefined8(metadata);
@@ -59334,10 +60383,10 @@ function assertionToComponent(assertion, parent) {
59334
60383
  };
59335
60384
  }
59336
60385
  function checkToComponent(check2, parent) {
59337
- const score = clampScore(check2.score ?? (check2.pass ? 1 : 0));
60386
+ const score2 = clampScore(check2.score ?? (check2.pass ? 1 : 0));
59338
60387
  return {
59339
60388
  pass: check2.pass,
59340
- score,
60389
+ score: score2,
59341
60390
  reason: check2.reason || check2.evidence || check2.text,
59342
60391
  assertion: assertionMetadata({
59343
60392
  ...parent,
@@ -59359,59 +60408,59 @@ function sanitizePublicGradingMetadata(value) {
59359
60408
  if (Array.isArray(value)) {
59360
60409
  return value.map(sanitizePublicGradingMetadata);
59361
60410
  }
59362
- if (!isRecord9(value)) {
60411
+ if (!isRecord11(value)) {
59363
60412
  return value;
59364
60413
  }
59365
60414
  const entries = Object.entries(value).filter(([key]) => !PUBLIC_GRADING_FORBIDDEN_METADATA_KEYS.has(key)).map(([key, entry]) => [key, sanitizePublicGradingMetadata(entry)]).filter(([, entry]) => entry !== void 0);
59366
60415
  return entries.length > 0 ? Object.fromEntries(entries) : void 0;
59367
60416
  }
59368
- function scoreMetadata(score) {
60417
+ function scoreMetadata(score2) {
59369
60418
  const metadata = dropUndefined8({
59370
- details: sanitizePublicGradingMetadata(score.details),
59371
- token_usage: score.tokenUsage,
59372
- duration_ms: score.durationMs,
59373
- started_at: score.startedAt,
59374
- ended_at: score.endedAt
60419
+ details: sanitizePublicGradingMetadata(score2.details),
60420
+ token_usage: score2.tokenUsage,
60421
+ duration_ms: score2.durationMs,
60422
+ started_at: score2.startedAt,
60423
+ ended_at: score2.endedAt
59375
60424
  });
59376
60425
  return Object.keys(metadata).length > 0 ? metadata : void 0;
59377
60426
  }
59378
- function scoreReason(score, pass) {
59379
- const reason = score.reason?.trim();
60427
+ function scoreReason(score2, pass) {
60428
+ const reason = score2.reason?.trim();
59380
60429
  if (reason) {
59381
60430
  return reason;
59382
60431
  }
59383
- const failed = (score.checks ?? []).find((check2) => !check2.pass);
60432
+ const failed = (score2.checks ?? []).find((check2) => !check2.pass);
59384
60433
  if (failed?.reason) {
59385
60434
  return failed.reason;
59386
60435
  }
59387
- const failedAssertion = (score.assertions ?? []).find((assertion) => !assertion.passed);
60436
+ const failedAssertion = (score2.assertions ?? []).find((assertion) => !assertion.passed);
59388
60437
  if (failedAssertion?.evidence || failedAssertion?.text) {
59389
60438
  return failedAssertion.evidence ?? failedAssertion.text;
59390
60439
  }
59391
60440
  return pass ? "Grader passed." : "Grader failed.";
59392
60441
  }
59393
- function scoreToComponent(score) {
59394
- const pass = passFromVerdict(score.verdict, score.score);
60442
+ function scoreToComponent(score2) {
60443
+ const pass = passFromVerdict(score2.verdict, score2.score);
59395
60444
  const assertion = assertionMetadata({
59396
- name: score.name,
59397
- type: score.type,
59398
- weight: score.weight,
59399
- target: score.target
60445
+ name: score2.name,
60446
+ type: score2.type,
60447
+ weight: score2.weight,
60448
+ target: score2.target
59400
60449
  });
59401
60450
  const componentResults = [
59402
- ...(score.checks ?? []).map((check2) => checkToComponent(check2, assertion)),
59403
- ...(score.scores ?? []).map(scoreToComponent),
59404
- ...score.checks && score.checks.length > 0 ? [] : (score.assertions ?? []).map((entry) => assertionToComponent(entry, assertion))
60451
+ ...(score2.checks ?? []).map((check2) => checkToComponent(check2, assertion)),
60452
+ ...(score2.scores ?? []).map(scoreToComponent),
60453
+ ...score2.checks && score2.checks.length > 0 ? [] : (score2.assertions ?? []).map((entry) => assertionToComponent(entry, assertion))
59405
60454
  ];
59406
- const namedScores = collectNamedScores(score.scores);
60455
+ const namedScores = collectNamedScores(score2.scores);
59407
60456
  return dropUndefined8({
59408
60457
  pass,
59409
- score: clampScore(score.score),
59410
- reason: scoreReason(score, pass),
60458
+ score: clampScore(score2.score),
60459
+ reason: scoreReason(score2, pass),
59411
60460
  component_results: componentResults.length > 0 ? componentResults : void 0,
59412
60461
  assertion,
59413
60462
  named_scores: namedScores,
59414
- metadata: scoreMetadata(score)
60463
+ metadata: scoreMetadata(score2)
59415
60464
  });
59416
60465
  }
59417
60466
  function buildComponentResults(result) {
@@ -59426,39 +60475,39 @@ function buildComponentResults(result) {
59426
60475
  }
59427
60476
  return void 0;
59428
60477
  }
59429
- function toIndexScore(score) {
59430
- const pass = passFromVerdict(score.verdict, score.score);
60478
+ function toIndexScore(score2) {
60479
+ const pass = passFromVerdict(score2.verdict, score2.score);
59431
60480
  return dropUndefined8({
59432
- name: score.name,
59433
- type: score.type,
60481
+ name: score2.name,
60482
+ type: score2.type,
59434
60483
  pass,
59435
- score: score.score,
59436
- reason: scoreReason(score, pass),
59437
- weight: score.weight,
59438
- target: score.target,
59439
- scores: score.scores?.map(toIndexScore),
59440
- named_scores: collectNamedScores(score.scores),
59441
- token_usage: score.tokenUsage,
59442
- duration_ms: score.durationMs,
59443
- started_at: score.startedAt,
59444
- ended_at: score.endedAt
60484
+ score: score2.score,
60485
+ reason: scoreReason(score2, pass),
60486
+ weight: score2.weight,
60487
+ target: score2.target,
60488
+ scores: score2.scores?.map(toIndexScore),
60489
+ named_scores: collectNamedScores(score2.scores),
60490
+ token_usage: score2.tokenUsage,
60491
+ duration_ms: score2.durationMs,
60492
+ started_at: score2.startedAt,
60493
+ ended_at: score2.endedAt
59445
60494
  });
59446
60495
  }
59447
60496
  function toIndexScores(scores) {
59448
60497
  return scores?.map(toIndexScore);
59449
60498
  }
59450
60499
  function collectNamedScores(scores, out = {}) {
59451
- for (const score of scores ?? []) {
59452
- if (score.name) {
59453
- out[score.name] = score.score;
60500
+ for (const score2 of scores ?? []) {
60501
+ if (score2.name) {
60502
+ out[score2.name] = score2.score;
59454
60503
  }
59455
- collectNamedScores(score.scores, out);
60504
+ collectNamedScores(score2.scores, out);
59456
60505
  }
59457
60506
  return Object.keys(out).length > 0 ? out : void 0;
59458
60507
  }
59459
60508
  function resultDerivedMetrics(result) {
59460
60509
  const value = result.metadata?.derived_metrics ?? result.metadata?.derivedMetrics;
59461
- return isRecord9(value) ? value : void 0;
60510
+ return isRecord11(value) ? value : void 0;
59462
60511
  }
59463
60512
  function resultProvenance(result) {
59464
60513
  const value = result.metadata?.provenance;
@@ -59533,11 +60582,11 @@ function toTrialAggregationArtifact(aggregation) {
59533
60582
  function dropUndefined8(value) {
59534
60583
  return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
59535
60584
  }
59536
- function isRecord9(value) {
60585
+ function isRecord11(value) {
59537
60586
  return typeof value === "object" && value !== null && !Array.isArray(value);
59538
60587
  }
59539
60588
  function toIndexRerunSource(value) {
59540
- if (!isRecord9(value)) {
60589
+ if (!isRecord11(value)) {
59541
60590
  return void 0;
59542
60591
  }
59543
60592
  return dropUndefined8({
@@ -59638,7 +60687,7 @@ function buildRepeatCaseSummaryArtifact(result, timing, fingerprint) {
59638
60687
  };
59639
60688
  }
59640
60689
  function toFilePathList(entries) {
59641
- return entries.map((entry) => isRecord9(entry) && typeof entry.path === "string" ? entry.path : void 0).filter((entry) => entry !== void 0);
60690
+ return entries.map((entry) => isRecord11(entry) && typeof entry.path === "string" ? entry.path : void 0).filter((entry) => entry !== void 0);
59642
60691
  }
59643
60692
  function resultTranscriptProviderId(result) {
59644
60693
  const provider = result.trace.metadata?.provider;
@@ -59715,7 +60764,7 @@ async function writeEnvironmentArtifact(params) {
59715
60764
  };
59716
60765
  }
59717
60766
  function buildAgentVRunResultArtifact(params) {
59718
- const metrics = params.metricsArtifact.metrics;
60767
+ const metrics = params.metricsArtifact;
59719
60768
  const fileChangesPath = params.hasFileChanges ? `./${CANONICAL_FILE_CHANGES_ARTIFACT_PATH}` : void 0;
59720
60769
  return dropUndefined8({
59721
60770
  execution_status: params.trial.executionStatus ?? params.result.executionStatus,
@@ -59909,7 +60958,7 @@ async function writeTrialRunArtifacts(params) {
59909
60958
  );
59910
60959
  }
59911
60960
  function toIndexPreparedAttempt(value) {
59912
- if (!isRecord9(value)) {
60961
+ if (!isRecord11(value)) {
59913
60962
  return void 0;
59914
60963
  }
59915
60964
  return dropUndefined8({
@@ -59994,7 +61043,7 @@ function timingMetadataSource(metadata, sourceKey) {
59994
61043
  const usageSources = metadata?.usage_sources;
59995
61044
  const usageSummary = metadata?.usage_summary;
59996
61045
  const legacyKey = sourceKey === "duration" ? "duration_source" : sourceKey === "cost" ? "cost_source" : "token_usage_source";
59997
- const value = isRecord9(usageSources) ? usageSources[sourceKey] : isRecord9(usageSummary) ? usageSummary[legacyKey] : metadata?.[legacyKey];
61046
+ const value = isRecord11(usageSources) ? usageSources[sourceKey] : isRecord11(usageSummary) ? usageSummary[legacyKey] : metadata?.[legacyKey];
59998
61047
  return typeof value === "string" && TIMING_SOURCE_VALUES2.includes(value) ? value : void 0;
59999
61048
  }
60000
61049
  function sumMessageTokenUsage(messages) {
@@ -60152,12 +61201,12 @@ function buildRunSummaryArtifact(results, evalFile = "", experiment, runId, plan
60152
61201
  if (isExecutionError(result)) {
60153
61202
  continue;
60154
61203
  }
60155
- for (const score of result.scores ?? []) {
60156
- const key = `${score.name}:${score.type}`;
61204
+ for (const score2 of result.scores ?? []) {
61205
+ const key = `${score2.name}:${score2.type}`;
60157
61206
  if (!evaluatorScores.has(key)) {
60158
61207
  evaluatorScores.set(key, []);
60159
61208
  }
60160
- evaluatorScores.get(key)?.push(score.score);
61209
+ evaluatorScores.get(key)?.push(score2.score);
60161
61210
  }
60162
61211
  }
60163
61212
  let perEvaluatorSummary;
@@ -60323,7 +61372,7 @@ async function writeInitialRunSummaryArtifact(runDir, options) {
60323
61372
  async function readRunConfigArtifact(runDir) {
60324
61373
  try {
60325
61374
  const parsed = JSON.parse(await readFile16(runInternalPath(runDir, RUN_CONFIG_FILENAME), "utf8"));
60326
- if (!isRecord9(parsed) || parsed.schema_version !== "agentv.run_config.v1") {
61375
+ if (!isRecord11(parsed) || parsed.schema_version !== "agentv.run_config.v1") {
60327
61376
  return void 0;
60328
61377
  }
60329
61378
  return parsed;
@@ -60333,7 +61382,7 @@ async function readRunConfigArtifact(runDir) {
60333
61382
  }
60334
61383
  function buildAggregateGradingArtifact(results) {
60335
61384
  const qualityResults = results.filter((r) => !isExecutionError(r));
60336
- const score = qualityResults.length > 0 ? Math.round(
61385
+ const score2 = qualityResults.length > 0 ? Math.round(
60337
61386
  qualityResults.reduce((sum, result) => sum + clampScore(result.score), 0) / qualityResults.length * 1e3
60338
61387
  ) / 1e3 : 0;
60339
61388
  const pass = qualityResults.length > 0 && qualityResults.every((result) => resultVerdict(result) === "pass");
@@ -60347,7 +61396,7 @@ function buildAggregateGradingArtifact(results) {
60347
61396
  }));
60348
61397
  return dropUndefined8({
60349
61398
  pass,
60350
- score,
61399
+ score: score2,
60351
61400
  reason: results.length === 0 ? "No results to summarize." : pass ? "All quality results passed." : "One or more quality results failed.",
60352
61401
  component_results: componentResults.length > 0 ? componentResults : void 0,
60353
61402
  named_scores: collectNamedScores(qualityResults.flatMap((result) => result.scores ?? [])),
@@ -60734,7 +61783,7 @@ async function writeMetricsArtifact(params) {
60734
61783
  return artifactWithTiming;
60735
61784
  }
60736
61785
  function indexRecordKey(record2) {
60737
- if (!isRecord9(record2)) {
61786
+ if (!isRecord11(record2)) {
60738
61787
  return void 0;
60739
61788
  }
60740
61789
  const testId = typeof record2.test_id === "string" ? record2.test_id : typeof record2.testId === "string" ? record2.testId : void 0;
@@ -60752,7 +61801,7 @@ function indexRecordReplacementKeys(record2) {
60752
61801
  return Array.from(new Set(keys));
60753
61802
  }
60754
61803
  function projectionIdentityRecordKey(record2) {
60755
- if (!isRecord9(record2) || !isRecord9(record2.projection_identity)) {
61804
+ if (!isRecord11(record2) || !isRecord11(record2.projection_identity)) {
60756
61805
  return void 0;
60757
61806
  }
60758
61807
  const id = record2.projection_identity.id;
@@ -60790,7 +61839,7 @@ function existingRecordsByProjectionIdentity(records) {
60790
61839
  return byIdentity;
60791
61840
  }
60792
61841
  function skippedExistingRecord(record2, identity, duplicatePolicy) {
60793
- if (!isRecord9(record2)) {
61842
+ if (!isRecord11(record2)) {
60794
61843
  return record2;
60795
61844
  }
60796
61845
  return {
@@ -60869,23 +61918,23 @@ async function readJsonFile2(filePath) {
60869
61918
  }
60870
61919
  }
60871
61920
  function summaryRunId(summary, fallback) {
60872
- if (!isRecord9(summary)) {
61921
+ if (!isRecord11(summary)) {
60873
61922
  return fallback;
60874
61923
  }
60875
- const runId = isRecord9(summary.metadata) ? summary.metadata.run_id : void 0;
61924
+ const runId = isRecord11(summary.metadata) ? summary.metadata.run_id : void 0;
60876
61925
  return typeof runId === "string" && runId.trim().length > 0 ? runId : fallback;
60877
61926
  }
60878
61927
  function summaryTimestamp(summary) {
60879
- if (!isRecord9(summary) || !isRecord9(summary.metadata)) {
61928
+ if (!isRecord11(summary) || !isRecord11(summary.metadata)) {
60880
61929
  return void 0;
60881
61930
  }
60882
61931
  return typeof summary.metadata.timestamp === "string" ? summary.metadata.timestamp : void 0;
60883
61932
  }
60884
61933
  function summaryTargets(summary) {
60885
- return isRecord9(summary) && isRecord9(summary.metadata) ? summary.metadata.targets : void 0;
61934
+ return isRecord11(summary) && isRecord11(summary.metadata) ? summary.metadata.targets : void 0;
60886
61935
  }
60887
61936
  function summaryTags(summary) {
60888
- return isRecord9(summary) && isRecord9(summary.metadata) ? summary.metadata.tags : void 0;
61937
+ return isRecord11(summary) && isRecord11(summary.metadata) ? summary.metadata.tags : void 0;
60889
61938
  }
60890
61939
  async function readJsonlFile(filePath) {
60891
61940
  const content = await readTextIfExists(filePath);
@@ -60905,7 +61954,7 @@ async function readJsonlFile(filePath) {
60905
61954
  return records;
60906
61955
  }
60907
61956
  function buildCrossRunRunRecord(params) {
60908
- const summary = isRecord9(params.summary) ? params.summary : {};
61957
+ const summary = isRecord11(params.summary) ? params.summary : {};
60909
61958
  return dropUndefined8({
60910
61959
  run_id: params.runId,
60911
61960
  run_dir: params.runDirName,
@@ -60920,7 +61969,7 @@ function buildCrossRunRunRecord(params) {
60920
61969
  });
60921
61970
  }
60922
61971
  function buildCrossRunCaseRecord(params) {
60923
- if (!isRecord9(params.caseRecord)) {
61972
+ if (!isRecord11(params.caseRecord)) {
60924
61973
  return void 0;
60925
61974
  }
60926
61975
  return dropUndefined8({
@@ -61895,9 +62944,11 @@ export {
61895
62944
  extractImageBlocks,
61896
62945
  formatToolCalls,
61897
62946
  SkillTriggerGrader,
62947
+ SkillUsedGrader,
61898
62948
  assembleLlmGraderPrompt,
61899
62949
  TokenUsageGrader,
61900
62950
  ToolTrajectoryGrader,
62951
+ TrajectoryGrader,
61901
62952
  runContainsAssertion,
61902
62953
  runContainsAnyAssertion,
61903
62954
  runContainsAllAssertion,
@@ -61981,6 +63032,7 @@ export {
61981
63032
  CANONICAL_METRICS_ARTIFACT_PATH,
61982
63033
  CANONICAL_FILE_CHANGES_ARTIFACT_PATH,
61983
63034
  TRANSCRIPT_SCHEMA_VERSION,
63035
+ LEGACY_METRICS_SCHEMA_VERSION,
61984
63036
  METRICS_SCHEMA_VERSION,
61985
63037
  TRANSCRIPT_JSON_MEDIA_TYPE,
61986
63038
  METRICS_JSON_MEDIA_TYPE,
@@ -62072,6 +63124,7 @@ export {
62072
63124
  runEvalCase,
62073
63125
  MetricsWireSchema,
62074
63126
  MetricsArtifactWireSchema,
63127
+ normalizeMetricsArtifactWire,
62075
63128
  buildMetricsArtifact,
62076
63129
  ResultRowSchemaError,
62077
63130
  normalizeResultRow,
@@ -62104,4 +63157,4 @@ export {
62104
63157
  chokidar/index.js:
62105
63158
  (*! chokidar - MIT License (c) 2012 Paul Miller (paulmillr.com) *)
62106
63159
  */
62107
- //# sourceMappingURL=chunk-3S6A2RKR.js.map
63160
+ //# sourceMappingURL=chunk-6ZZCDZPD.js.map