@agentv/core 3.10.3 → 3.11.0

package/dist/evaluation/validation/index.js

@@ -8,7 +8,7 @@ import {
   isEvaluatorKind,
   loadCasesFromFile,
   resolveFileReference
-} from "../../chunk-VCFYWLFV.js";
+} from "../../chunk-AVTN5AB7.js";
 
 // src/evaluation/validation/file-type.ts
 import { readFile } from "node:fs/promises";

package/dist/index.cjs

@@ -1854,6 +1854,64 @@ var import_node_path8 = __toESM(require("path"), 1);
 var import_micromatch2 = __toESM(require("micromatch"), 1);
 var import_yaml4 = require("yaml");
 
+// src/evaluation/input-message-utils.ts
+function flattenInputMessages(messages) {
+  return messages.flatMap((message) => extractContentSegments(message.content));
+}
+function collectResolvedInputFilePaths(messages) {
+  const filePaths = [];
+  for (const message of messages) {
+    if (!Array.isArray(message.content)) {
+      continue;
+    }
+    for (const segment of message.content) {
+      if (isJsonObject(segment) && segment.type === "file" && typeof segment.resolvedPath === "string") {
+        filePaths.push(segment.resolvedPath);
+      }
+    }
+  }
+  return filePaths;
+}
+function extractContentSegments(content) {
+  if (typeof content === "string") {
+    return content.trim().length > 0 ? [{ type: "text", value: content }] : [];
+  }
+  if (isJsonObject(content)) {
+    const rendered = JSON.stringify(content, null, 2);
+    return rendered.trim().length > 0 ? [{ type: "text", value: rendered }] : [];
+  }
+  if (!Array.isArray(content)) {
+    return [];
+  }
+  const segments = [];
+  for (const segment of content) {
+    if (!isJsonObject(segment)) {
+      continue;
+    }
+    segments.push(cloneJsonObject(segment));
+  }
+  return segments;
+}
+function cloneJsonObject(source) {
+  const entries = Object.entries(source).map(([key, value]) => [key, cloneJsonValue(value)]);
+  return Object.fromEntries(entries);
+}
+function cloneJsonValue(value) {
+  if (value === null) {
+    return null;
+  }
+  if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => cloneJsonValue(item));
+  }
+  if (typeof value === "object") {
+    return cloneJsonObject(value);
+  }
+  return value;
+}
+
 // src/evaluation/interpolation.ts
 var ENV_VAR_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
 function interpolateEnv(value, env) {
@@ -1941,7 +1999,6 @@ function parseAgentSkillsEvals(parsed, source = "evals.json", baseDir) {
       id: String(id),
       question: prompt,
       input: [{ role: "user", content: prompt }],
-      input_segments: [{ type: "text", value: prompt }],
       expected_output: evalCase.expected_output ? [{ role: "assistant", content: evalCase.expected_output }] : [],
       reference_answer: evalCase.expected_output,
       file_paths: filePaths,
@@ -2194,7 +2251,7 @@ async function loadConfig(evalFilePath, repoRoot) {
     }
     try {
       const rawConfig = await (0, import_promises4.readFile)(configPath, "utf8");
-      const parsed = (0, import_yaml2.parse)(rawConfig);
+      const parsed = interpolateEnv((0, import_yaml2.parse)(rawConfig), process.env);
       if (!isJsonObject(parsed)) {
         logWarning(`Invalid .agentv/config.yaml format at ${configPath}`);
         continue;
@@ -2412,6 +2469,27 @@ function parseExecutionDefaults(raw, configPath) {
   } else if (otelFile !== void 0) {
     logWarning(`Invalid execution.otel_file in ${configPath}, expected non-empty string`);
   }
+  if (typeof obj.export_otel === "boolean") {
+    result.export_otel = obj.export_otel;
+  } else if (obj.export_otel !== void 0) {
+    logWarning(`Invalid execution.export_otel in ${configPath}, expected boolean`);
+  }
+  const otelBackend = obj.otel_backend;
+  if (typeof otelBackend === "string" && otelBackend.trim().length > 0) {
+    result.otel_backend = otelBackend.trim();
+  } else if (otelBackend !== void 0) {
+    logWarning(`Invalid execution.otel_backend in ${configPath}, expected non-empty string`);
+  }
+  if (typeof obj.otel_capture_content === "boolean") {
+    result.otel_capture_content = obj.otel_capture_content;
+  } else if (obj.otel_capture_content !== void 0) {
+    logWarning(`Invalid execution.otel_capture_content in ${configPath}, expected boolean`);
+  }
+  if (typeof obj.otel_group_turns === "boolean") {
+    result.otel_group_turns = obj.otel_group_turns;
+  } else if (obj.otel_group_turns !== void 0) {
+    logWarning(`Invalid execution.otel_group_turns in ${configPath}, expected boolean`);
+  }
   if (typeof obj.pool_workspaces === "boolean") {
     result.pool_workspaces = obj.pool_workspaces;
   } else if (obj.pool_workspaces !== void 0) {
@@ -3882,27 +3960,28 @@ var ANSI_YELLOW5 = "\x1B[33m";
 var ANSI_RESET6 = "\x1B[0m";
 async function processMessages(options) {
   const { messages, searchRoots, repoRootPath, textParts, messageType, verbose } = options;
-  const segments = [];
+  const processedMessages = [];
   for (const message of messages) {
     const content = message.content;
     if (typeof content === "string") {
-      segments.push({ type: "text", value: content });
       if (textParts) {
         textParts.push(content);
       }
+      processedMessages.push({ ...message, content });
       continue;
     }
     if (isJsonObject(content)) {
       const rendered = JSON.stringify(content, null, 2);
-      segments.push({ type: "text", value: rendered });
       if (textParts) {
         textParts.push(rendered);
       }
+      processedMessages.push({ ...message, content: cloneJsonObject(content) });
       continue;
     }
     if (!Array.isArray(content)) {
       continue;
     }
+    const processedContent = [];
     for (const rawSegment of content) {
       if (!isJsonObject(rawSegment)) {
         continue;
@@ -3925,8 +4004,8 @@ async function processMessages(options) {
         }
         try {
           const fileContent = (await (0, import_promises6.readFile)(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
-          segments.push({
-            type: "file",
+          processedContent.push({
+            ...cloneJsonObject(rawSegment),
             path: displayPath,
             text: fileContent,
             resolvedPath: import_node_path6.default.resolve(resolvedPath)
@@ -3943,37 +4022,19 @@ async function processMessages(options) {
         continue;
       }
       const clonedSegment = cloneJsonObject(rawSegment);
-      segments.push(clonedSegment);
+      processedContent.push(clonedSegment);
       const inlineValue = clonedSegment.value;
       if (typeof inlineValue === "string" && textParts) {
         textParts.push(inlineValue);
       }
     }
+    processedMessages.push({ ...message, content: processedContent });
   }
-  return segments;
+  return processedMessages;
 }
 function asString3(value) {
   return typeof value === "string" ? value : void 0;
 }
-function cloneJsonObject(source) {
-  const entries = Object.entries(source).map(([key, value]) => [key, cloneJsonValue(value)]);
-  return Object.fromEntries(entries);
-}
-function cloneJsonValue(value) {
-  if (value === null) {
-    return null;
-  }
-  if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
-    return value;
-  }
-  if (Array.isArray(value)) {
-    return value.map((item) => cloneJsonValue(item));
-  }
-  if (typeof value === "object") {
-    return cloneJsonObject(value);
-  }
-  return value;
-}
 function logWarning3(message, details) {
   if (details && details.length > 0) {
     const detailBlock = details.join("\n");
@@ -4222,10 +4283,10 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
         );
       }
     }
-    const inputMessages = resolveInputMessages(evalcase);
+    const rawInputMessages = resolveInputMessages(evalcase);
     const expectedMessages = resolveExpectedMessages(evalcase) ?? [];
     const hasEvaluationSpec = !!outcome || expectedMessages.length > 0 || evalcase.assert !== void 0;
-    if (!id || !hasEvaluationSpec || !inputMessages || inputMessages.length === 0) {
+    if (!id || !hasEvaluationSpec || !rawInputMessages || rawInputMessages.length === 0) {
       logError2(
         `Skipping incomplete test at line ${lineNumber}: ${id ?? "unknown"}. Missing required fields: id, input, and at least one of criteria/expected_output/assert`
       );
@@ -4233,8 +4294,8 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
     }
     const hasExpectedMessages = expectedMessages.length > 0;
     const inputTextParts = [];
-    const inputSegments = await processMessages({
-      messages: inputMessages,
+    const inputMessages = await processMessages({
+      messages: rawInputMessages,
       searchRoots,
       repoRootPath,
       textParts: inputTextParts,
@@ -4280,19 +4341,13 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
       }
     }
     warnUnconsumedCriteria(outcome, evaluators, id ?? "unknown");
-    const userFilePaths = [];
-    for (const segment of inputSegments) {
-      if (segment.type === "file" && typeof segment.resolvedPath === "string") {
-        userFilePaths.push(segment.resolvedPath);
-      }
-    }
+    const userFilePaths = collectResolvedInputFilePaths(inputMessages);
     const testCase = {
       id,
       eval_set: evalSetName,
       conversation_id: conversationId,
       question,
       input: inputMessages,
-      input_segments: inputSegments,
       expected_output: outputSegments,
       reference_answer: referenceAnswer,
       file_paths: userFilePaths,
@@ -4358,50 +4413,9 @@ function parseMetadata(suite) {
 
 // src/evaluation/formatting/prompt-builder.ts
 async function buildPromptInputs(testCase, mode = "lm") {
-  const segmentsByMessage = [];
-  const fileContentsByPath = /* @__PURE__ */ new Map();
-  for (const segment of testCase.input_segments) {
-    if (segment.type === "file" && typeof segment.path === "string" && typeof segment.text === "string") {
-      fileContentsByPath.set(segment.path, segment.text);
-    }
-  }
-  for (const message of testCase.input) {
-    const messageSegments = [];
-    if (typeof message.content === "string") {
-      if (message.content.trim().length > 0) {
-        messageSegments.push({ type: "text", value: message.content });
-      }
-    } else if (Array.isArray(message.content)) {
-      for (const segment of message.content) {
-        if (typeof segment === "string") {
-          if (segment.trim().length > 0) {
-            messageSegments.push({ type: "text", value: segment });
-          }
-        } else if (isJsonObject(segment)) {
-          const type = asString5(segment.type);
-          if (type === "file") {
-            const value = asString5(segment.value);
-            if (!value) continue;
-            const fileText = fileContentsByPath.get(value);
-            if (fileText !== void 0) {
-              messageSegments.push({ type: "file", text: fileText, path: value });
-            }
-          } else if (type === "text") {
-            const textValue = asString5(segment.value);
-            if (textValue && textValue.trim().length > 0) {
-              messageSegments.push({ type: "text", value: textValue });
-            }
-          }
-        }
-      }
-    } else if (isJsonObject(message.content)) {
-      const rendered = JSON.stringify(message.content, null, 2);
-      if (rendered.trim().length > 0) {
-        messageSegments.push({ type: "text", value: rendered });
-      }
-    }
-    segmentsByMessage.push(messageSegments);
-  }
+  const segmentsByMessage = testCase.input.map(
+    (message) => extractContentSegments(message.content)
+  );
   const useRoleMarkers = needsRoleMarkers(testCase.input, segmentsByMessage);
   let question;
   if (useRoleMarkers) {
@@ -4429,7 +4443,7 @@ ${messageContent}`);
     question = messageParts.join("\n\n");
   } else {
     const questionParts = [];
-    for (const segment of testCase.input_segments) {
+    for (const segment of flattenInputMessages(testCase.input)) {
       const formattedContent = formatSegment(segment, mode);
       if (formattedContent) {
         questionParts.push(formattedContent);
@@ -4516,9 +4530,6 @@ function buildChatPromptFromSegments(options) {
   }
   return chatPrompt.length > 0 ? chatPrompt : void 0;
 }
-function asString5(value) {
-  return typeof value === "string" ? value : void 0;
-}
 
 // src/evaluation/yaml-parser.ts
 var ANSI_YELLOW7 = "\x1B[33m";
@@ -4601,7 +4612,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
     throw new Error(`Invalid test file format: ${evalFilePath}`);
   }
   const suite = interpolated;
-  const evalSetNameFromSuite = asString6(suite.name)?.trim();
+  const evalSetNameFromSuite = asString5(suite.name)?.trim();
   const fallbackEvalSet = import_node_path8.default.basename(absoluteTestPath).replace(/\.ya?ml$/i, "") || "eval";
   const evalSetName = evalSetNameFromSuite && evalSetNameFromSuite.length > 0 ? evalSetNameFromSuite : fallbackEvalSet;
   const rawTestcases = resolveTests(suite);
@@ -4620,7 +4631,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
   const suiteInputMessages = expandInputShorthand(suite.input);
   const suiteInputFiles = suite.input_files;
   const rawGlobalExecution = isJsonObject(suite.execution) ? suite.execution : void 0;
-  const _globalTarget = asString6(rawGlobalExecution?.target) ?? asString6(suite.target);
+  const _globalTarget = asString5(rawGlobalExecution?.target) ?? asString5(suite.target);
   const suiteAssertions = suite.assertions ?? suite.assert;
   if (suite.assert !== void 0 && suite.assertions === void 0) {
     logWarning5("'assert' is deprecated at the suite level. Use 'assertions' instead.");
@@ -4633,17 +4644,17 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
       continue;
     }
     const evalcase = rawEvalcase;
-    const id = asString6(evalcase.id);
+    const id = asString5(evalcase.id);
     if (filterPattern && (!id || !import_micromatch2.default.isMatch(id, filterPattern))) {
       continue;
     }
-    const conversationId = asString6(evalcase.conversation_id);
-    let outcome = asString6(evalcase.criteria);
+    const conversationId = asString5(evalcase.conversation_id);
+    let outcome = asString5(evalcase.criteria);
     if (!outcome && evalcase.expected_outcome !== void 0) {
-      outcome = asString6(evalcase.expected_outcome);
+      outcome = asString5(evalcase.expected_outcome);
       if (outcome) {
         logWarning5(
-          `Test '${asString6(evalcase.id) ?? "unknown"}': 'expected_outcome' is deprecated. Use 'criteria' instead.`
+          `Test '${asString5(evalcase.id) ?? "unknown"}': 'expected_outcome' is deprecated. Use 'criteria' instead.`
         );
       }
     }
@@ -4660,10 +4671,9 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
       continue;
     }
     const effectiveSuiteInputMessages = suiteInputMessages && !skipDefaults ? suiteInputMessages : void 0;
-    const inputMessages = effectiveSuiteInputMessages ? [...effectiveSuiteInputMessages, ...testInputMessages] : testInputMessages;
     const hasExpectedMessages = expectedMessages.length > 0;
     const inputTextParts = [];
-    const suiteInputSegments = effectiveSuiteInputMessages ? await processMessages({
+    const suiteResolvedInputMessages = effectiveSuiteInputMessages ? await processMessages({
       messages: effectiveSuiteInputMessages,
       searchRoots,
       repoRootPath,
@@ -4671,7 +4681,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
       messageType: "input",
       verbose
     }) : [];
-    const testInputSegments = await processMessages({
+    const testResolvedInputMessages = await processMessages({
       messages: testInputMessages,
       searchRoots,
       repoRootPath,
@@ -4679,7 +4689,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
       messageType: "input",
       verbose
     });
-    const inputSegments = [...suiteInputSegments, ...testInputSegments];
+    const inputMessages = [...suiteResolvedInputMessages, ...testResolvedInputMessages];
     const outputSegments = hasExpectedMessages ? await processExpectedMessages({
       messages: expectedMessages,
       searchRoots,
@@ -4717,12 +4727,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
       }
     }
     warnUnconsumedCriteria(outcome, evaluators, id ?? "unknown");
-    const userFilePaths = [];
-    for (const segment of inputSegments) {
-      if (segment.type === "file" && typeof segment.resolvedPath === "string") {
-        userFilePaths.push(segment.resolvedPath);
-      }
-    }
+    const userFilePaths = collectResolvedInputFilePaths(inputMessages);
     const caseWorkspace = await resolveWorkspaceConfig(evalcase.workspace, evalFileDir);
     const mergedWorkspace = mergeWorkspaceConfigs(suiteWorkspace, caseWorkspace);
     const metadata = isJsonObject(evalcase.metadata) ? evalcase.metadata : void 0;
@@ -4733,7 +4738,6 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
       conversation_id: conversationId,
       question,
       input: inputMessages,
-      input_segments: inputSegments,
       expected_output: outputSegments,
       reference_answer: referenceAnswer,
       file_paths: userFilePaths,
@@ -4942,7 +4946,7 @@ function mergeWorkspaceConfigs(suiteLevel, caseLevel) {
     path: caseLevel.path ?? suiteLevel.path
   };
 }
-function asString6(value) {
+function asString5(value) {
   return typeof value === "string" ? value : void 0;
 }
 function logWarning5(message, details) {
@@ -8813,7 +8817,7 @@ var PiAgentSdkProvider = class {
     const { Agent, getModel, getEnvApiKey } = await loadPiModules();
     const startTimeIso = (/* @__PURE__ */ new Date()).toISOString();
     const startMs = Date.now();
-    const providerName = this.config.provider ?? "anthropic";
+    const providerName = this.config.subprovider ?? "anthropic";
     const modelId = this.config.model ?? "claude-sonnet-4-20250514";
     const model = getModel(providerName, modelId);
     const systemPrompt = this.config.systemPrompt ?? "Answer directly and concisely.";
@@ -8925,7 +8929,7 @@ var PiAgentSdkProvider = class {
           messages: agentMessages,
           systemPrompt,
           model: this.config.model,
-          provider: this.config.provider
+          subprovider: this.config.subprovider
         },
         output,
         tokenUsage,
@@ -9161,8 +9165,8 @@ var PiCodingAgentProvider = class {
   }
   buildPiArgs(prompt, inputFiles, _captureFileChanges) {
     const args = [];
-    if (this.config.provider) {
-      args.push("--provider", this.config.provider);
+    if (this.config.subprovider) {
+      args.push("--provider", this.config.subprovider);
     }
     if (this.config.model) {
       args.push("--model", this.config.model);
@@ -9220,7 +9224,7 @@ ${prompt}` : prompt;
   buildEnv() {
     const env = { ...process.env };
     if (this.config.apiKey) {
-      const provider = this.config.provider?.toLowerCase() ?? "google";
+      const provider = this.config.subprovider?.toLowerCase() ?? "google";
       switch (provider) {
         case "google":
         case "gemini":
@@ -10531,7 +10535,7 @@ function normalizeCopilotLogFormat(value) {
 }
 function resolvePiCodingAgentConfig(target, env, evalFilePath) {
   const executableSource = target.executable ?? target.command ?? target.binary;
-  const providerSource = target.pi_provider ?? target.piProvider ?? target.llm_provider;
+  const subproviderSource = target.subprovider;
   const modelSource = target.model ?? target.pi_model ?? target.piModel;
   const apiKeySource = target.api_key ?? target.apiKey;
   const toolsSource = target.tools ?? target.pi_tools ?? target.piTools;
@@ -10547,10 +10551,15 @@ function resolvePiCodingAgentConfig(target, env, evalFilePath) {
     allowLiteral: true,
     optionalEnv: true
   }) ?? "pi";
-  const provider = resolveOptionalString(providerSource, env, `${target.name} pi provider`, {
-    allowLiteral: true,
-    optionalEnv: true
-  });
+  const subprovider = resolveOptionalString(
+    subproviderSource,
+    env,
+    `${target.name} pi subprovider`,
+    {
+      allowLiteral: true,
+      optionalEnv: true
+    }
+  );
   const model = resolveOptionalString(modelSource, env, `${target.name} pi model`, {
     allowLiteral: true,
     optionalEnv: true
@@ -10598,7 +10607,7 @@ function resolvePiCodingAgentConfig(target, env, evalFilePath) {
   const systemPrompt = typeof systemPromptSource === "string" && systemPromptSource.trim().length > 0 ? systemPromptSource.trim() : void 0;
   return {
     executable,
-    provider,
+    subprovider,
     model,
     apiKey,
     tools,
@@ -10613,15 +10622,15 @@ function resolvePiCodingAgentConfig(target, env, evalFilePath) {
   };
 }
 function resolvePiAgentSdkConfig(target, env) {
-  const providerSource = target.pi_provider ?? target.piProvider ?? target.llm_provider;
+  const subproviderSource = target.subprovider;
   const modelSource = target.model ?? target.pi_model ?? target.piModel;
   const apiKeySource = target.api_key ?? target.apiKey;
   const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds;
   const systemPromptSource = target.system_prompt ?? target.systemPrompt;
-  const provider = resolveOptionalString(
-    providerSource,
+  const subprovider = resolveOptionalString(
+    subproviderSource,
     env,
-    `${target.name} pi-agent-sdk provider`,
+    `${target.name} pi-agent-sdk subprovider`,
     {
       allowLiteral: true,
       optionalEnv: true
@@ -10638,7 +10647,7 @@ function resolvePiAgentSdkConfig(target, env) {
   const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} pi-agent-sdk timeout`);
   const systemPrompt = typeof systemPromptSource === "string" && systemPromptSource.trim().length > 0 ? systemPromptSource.trim() : void 0;
   return {
-    provider,
+    subprovider,
     model,
     apiKey,
     timeoutMs,
@@ -13300,7 +13309,8 @@ var freeformEvaluationSchema = import_zod4.z.object({
       passed: import_zod4.z.boolean().describe("Whether this aspect was satisfied"),
       evidence: import_zod4.z.string().describe("Concise evidence (1-2 sentences)").optional()
     })
-  ).describe("Per-aspect evaluation results \u2014 one entry per aspect checked").optional()
+  ).describe("Per-aspect evaluation results \u2014 one entry per aspect checked").optional(),
+  details: import_zod4.z.record(import_zod4.z.unknown()).describe("Optional structured metadata for domain-specific metrics").optional()
 });
 var rubricCheckResultSchema = import_zod4.z.object({
   id: import_zod4.z.string().describe("The ID of the rubric item being checked"),
@@ -13362,7 +13372,7 @@ var LlmGraderEvaluator = class {
   async evaluateFreeform(context2, graderProvider) {
     const formattedQuestion = context2.promptInputs.question && context2.promptInputs.question.trim().length > 0 ? context2.promptInputs.question : context2.evalCase.question;
     const variables = {
-      [TEMPLATE_VARIABLES.INPUT]: JSON.stringify(context2.evalCase.input_segments, null, 2),
+      [TEMPLATE_VARIABLES.INPUT]: JSON.stringify(context2.evalCase.input, null, 2),
       [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: JSON.stringify(
         context2.evalCase.expected_output,
         null,
@@ -13405,6 +13415,7 @@ ${context2.fileChanges}`;
         expectedAspectCount: Math.max(assertions.length, 1),
         evaluatorRawRequest,
         graderTarget: graderProvider.targetName,
+        details: data.details,
         tokenUsage
       };
     } catch (e) {
@@ -13824,7 +13835,7 @@ ${outputSchema}`;
         expectedAspectCount: Math.max(assertions.length, 1),
         evaluatorRawRequest,
         graderTarget,
-        details
+        details: data.details && Object.keys(data.details).length > 0 ? { ...details, ...data.details } : details
       };
     } catch {
       return {
@@ -13971,7 +13982,8 @@ function buildOutputSchema() {
     '      "passed": <boolean>,',
     '      "evidence": "<concise evidence, 1-2 sentences, optional>"',
     "    }",
-    "  ]",
+    "  ],",
+    '  "details": {<optional object with domain-specific structured metrics>}',
     "}"
   ].join("\n");
 }
@@ -15335,7 +15347,7 @@ function assembleLlmGraderPrompt(input) {
 function assembleFreeform(evalCase, candidate, promptInputs, fileChanges, evaluatorTemplateOverride) {
   const formattedQuestion = promptInputs.question && promptInputs.question.trim().length > 0 ? promptInputs.question : evalCase.question;
   const variables = {
-    [TEMPLATE_VARIABLES.INPUT]: JSON.stringify(evalCase.input_segments, null, 2),
+    [TEMPLATE_VARIABLES.INPUT]: JSON.stringify(evalCase.input, null, 2),
     [TEMPLATE_VARIABLES.EXPECTED_OUTPUT]: JSON.stringify(evalCase.expected_output, null, 2),
     [TEMPLATE_VARIABLES.OUTPUT]: JSON.stringify([], null, 2),
     [TEMPLATE_VARIABLES.CRITERIA]: evalCase.criteria.trim(),
@@ -17616,6 +17628,18 @@ var QUALITY_PASS_THRESHOLD = 0.8;
 function classifyQualityStatus(score) {
   return score >= QUALITY_PASS_THRESHOLD ? "ok" : "quality_failure";
 }
+function buildSkippedEvaluatorError(scores) {
+  const skippedScores = scores?.filter((score) => score.verdict === "skip") ?? [];
+  if (skippedScores.length === 0) {
+    return void 0;
+  }
+  const messages = skippedScores.map((score) => {
+    const label = score.name || score.type;
+    const assertionMessage = score.assertions.find((assertion) => !assertion.passed)?.text ?? "Evaluator skipped";
+    return `${label}: ${assertionMessage}`;
+  });
+  return messages.length === 1 ? messages[0] : `Evaluators skipped: ${messages.join(" | ")}`;
+}
 function usesFileReferencePrompt(provider) {
   return isAgentProvider(provider) || provider.kind === "cli";
 }
@@ -18880,7 +18904,8 @@ async function runEvalCase(options) {
       durationMs: totalDurationMs,
       ...evalRunTokenUsage ? { tokenUsage: evalRunTokenUsage } : {}
     };
-    const executionStatus = providerError ? "execution_error" : classifyQualityStatus(result.score);
+    const skippedEvaluatorError = buildSkippedEvaluatorError(result.scores);
+    const executionStatus = providerError || skippedEvaluatorError ? "execution_error" : classifyQualityStatus(result.score);
     const finalResult = providerError ? {
       ...result,
       evalRun,
@@ -18892,7 +18917,26 @@ async function runEvalCase(options) {
       beforeAllOutput,
       beforeEachOutput,
       afterEachOutput
-    } : { ...result, evalRun, executionStatus, beforeAllOutput, beforeEachOutput, afterEachOutput };
+    } : skippedEvaluatorError ? {
+      ...result,
+      score: 0,
+      evalRun,
+      error: skippedEvaluatorError,
+      executionStatus,
+      failureStage: "evaluator",
+      failureReasonCode: "evaluator_error",
+      executionError: { message: skippedEvaluatorError, stage: "evaluator" },
+      beforeAllOutput,
+      beforeEachOutput,
+      afterEachOutput
+    } : {
+      ...result,
+      evalRun,
+      executionStatus,
+      beforeAllOutput,
+      beforeEachOutput,
+      afterEachOutput
+    };
     const isFailure = !!finalResult.error || finalResult.score < 0.5;
     if (workspacePath && !isSharedWorkspace) {
       if (forceCleanup) {
@@ -19637,11 +19681,6 @@ async function evaluate(config) {
     evalCases = (config.tests ?? []).map((test) => {
       const input = typeof test.input === "string" ? [{ role: "user", content: test.input }] : test.input;
       const question = typeof test.input === "string" ? test.input : test.input.find((m) => m.role === "user")?.content ?? "";
-      const inputSegments = input.map((m) => ({
-        type: "text",
-        value: typeof m.content === "string" ? m.content : JSON.stringify(m.content),
-        messageIndex: 0
-      }));
       const expectedOutputValue = test.expectedOutput ?? test.expected_output;
       const expectedOutput = expectedOutputValue ? [
         { role: "assistant", content: expectedOutputValue }
@@ -19670,7 +19709,6 @@ async function evaluate(config) {
         criteria: test.criteria ?? "",
         question: String(question),
         input,
-        input_segments: inputSegments,
         expected_output: expectedOutput,
         reference_answer: expectedOutputValue,
         file_paths: [],