@agentv/core 0.6.1 → 0.7.0

package/dist/index.js

@@ -6,7 +6,7 @@ import {
   findGitRoot,
   readTextFile,
   resolveFileReference
-} from "./chunk-OW3SHBIJ.js";
+} from "./chunk-L7I5UTJU.js";
 
 // src/evaluation/types.ts
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
@@ -134,6 +134,87 @@ function extractCodeBlocks(segments) {
   }
   return codeBlocks;
 }
+async function processMessages(options) {
+  const {
+    messages,
+    searchRoots,
+    repoRootPath,
+    guidelinePatterns,
+    guidelinePaths,
+    textParts,
+    messageType,
+    verbose
+  } = options;
+  const segments = [];
+  for (const message of messages) {
+    const content = message.content;
+    if (typeof content === "string") {
+      segments.push({ type: "text", value: content });
+      if (textParts) {
+        textParts.push(content);
+      }
+      continue;
+    }
+    for (const rawSegment of content) {
+      if (!isJsonObject(rawSegment)) {
+        continue;
+      }
+      const segmentType = asString(rawSegment.type);
+      if (segmentType === "file") {
+        const rawValue = asString(rawSegment.value);
+        if (!rawValue) {
+          continue;
+        }
+        const { displayPath, resolvedPath, attempted } = await resolveFileReference(
+          rawValue,
+          searchRoots
+        );
+        if (!resolvedPath) {
+          const attempts = attempted.length ? ["  Tried:", ...attempted.map((candidate) => `    ${candidate}`)] : void 0;
+          const context = messageType === "input" ? "" : " in expected_messages";
+          logWarning(`File not found${context}: ${displayPath}`, attempts);
+          continue;
+        }
+        try {
+          const fileContent = (await readFile(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
+          if (messageType === "input" && guidelinePatterns && guidelinePaths) {
+            const relativeToRepo = path.relative(repoRootPath, resolvedPath);
+            if (isGuidelineFile(relativeToRepo, guidelinePatterns)) {
+              guidelinePaths.push(path.resolve(resolvedPath));
+              if (verbose) {
+                console.log(`  [Guideline] Found: ${displayPath}`);
+                console.log(`    Resolved to: ${resolvedPath}`);
+              }
+              continue;
+            }
+          }
+          segments.push({
+            type: "file",
+            path: displayPath,
+            text: fileContent,
+            resolvedPath: path.resolve(resolvedPath)
+          });
+          if (verbose) {
+            const label = messageType === "input" ? "[File]" : "[Expected Output File]";
+            console.log(`  ${label} Found: ${displayPath}`);
+            console.log(`    Resolved to: ${resolvedPath}`);
+          }
+        } catch (error) {
+          const context = messageType === "input" ? "" : " expected output";
+          logWarning(`Could not read${context} file ${resolvedPath}: ${error.message}`);
+        }
+        continue;
+      }
+      const clonedSegment = cloneJsonObject(rawSegment);
+      segments.push(clonedSegment);
+      const inlineValue = clonedSegment.value;
+      if (typeof inlineValue === "string" && textParts) {
+        textParts.push(inlineValue);
+      }
+    }
+  }
+  return segments;
+}
 async function loadEvalCases(evalFilePath, repoRoot, options) {
   const verbose = options?.verbose ?? false;
   const absoluteTestPath = path.resolve(evalFilePath);
@@ -219,77 +300,34 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
         }
       }
     }
-    const userSegments = [];
     const guidelinePaths = [];
-    const userTextParts = [];
-    for (const userMessage of userMessages) {
-      const content = userMessage.content;
-      if (typeof content === "string") {
-        userSegments.push({ type: "text", value: content });
-        userTextParts.push(content);
-        continue;
-      }
-      for (const rawSegment of content) {
-        if (!isJsonObject(rawSegment)) {
-          continue;
-        }
-        const segmentType = asString(rawSegment.type);
-        if (segmentType === "file") {
-          const rawValue = asString(rawSegment.value);
-          if (!rawValue) {
-            continue;
-          }
-          const { displayPath, resolvedPath, attempted } = await resolveFileReference(
-            rawValue,
-            searchRoots
-          );
-          if (!resolvedPath) {
-            const attempts = attempted.length ? ["  Tried:", ...attempted.map((candidate) => `    ${candidate}`)] : void 0;
-            logWarning(`File not found: ${displayPath}`, attempts);
-            continue;
-          }
-          try {
-            const fileContent = (await readFile(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
-            const relativeToRepo = path.relative(repoRootPath, resolvedPath);
-            if (isGuidelineFile(relativeToRepo, guidelinePatterns)) {
-              guidelinePaths.push(path.resolve(resolvedPath));
-              if (verbose) {
-                console.log(`  [Guideline] Found: ${displayPath}`);
-                console.log(`    Resolved to: ${resolvedPath}`);
-              }
-            } else {
-              userSegments.push({
-                type: "file",
-                path: displayPath,
-                text: fileContent,
-                resolvedPath: path.resolve(resolvedPath)
-              });
-              if (verbose) {
-                console.log(`  [File] Found: ${displayPath}`);
-                console.log(`    Resolved to: ${resolvedPath}`);
-              }
-            }
-          } catch (error) {
-            logWarning(`Could not read file ${resolvedPath}: ${error.message}`);
-          }
-          continue;
-        }
-        const clonedSegment = cloneJsonObject(rawSegment);
-        userSegments.push(clonedSegment);
-        const inlineValue = clonedSegment.value;
-        if (typeof inlineValue === "string") {
-          userTextParts.push(inlineValue);
-        }
-      }
-    }
-    const codeSnippets = extractCodeBlocks(userSegments);
+    const inputTextParts = [];
+    const inputSegments = await processMessages({
+      messages: userMessages,
+      searchRoots,
+      repoRootPath,
+      guidelinePatterns,
+      guidelinePaths,
+      textParts: inputTextParts,
+      messageType: "input",
+      verbose
+    });
+    const outputSegments = await processMessages({
+      messages: assistantMessages,
+      searchRoots,
+      repoRootPath,
+      guidelinePatterns,
+      messageType: "output",
+      verbose
+    });
+    const codeSnippets = extractCodeBlocks(inputSegments);
     const assistantContent = assistantMessages[0]?.content;
-    const expectedAssistantRaw = await resolveAssistantContent(assistantContent, searchRoots, verbose);
-    const userTextPrompt = userTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
+    const referenceAnswer = await resolveAssistantContent(assistantContent, searchRoots, verbose);
+    const question = inputTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
     const testCaseEvaluatorKind = coerceEvaluator(evalcase.evaluator, id) ?? globalEvaluator;
     const evaluators = await parseEvaluators(evalcase, searchRoots, id ?? "unknown");
     const userFilePaths = [];
-    for (const segment of userSegments) {
+    for (const segment of inputSegments) {
       if (segment.type === "file" && typeof segment.resolvedPath === "string") {
         userFilePaths.push(segment.resolvedPath);
       }
@@ -302,15 +340,16 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
       id,
       dataset: datasetName,
       conversation_id: conversationId,
-      task: userTextPrompt,
-      user_segments: userSegments,
+      question,
+      input_segments: inputSegments,
+      output_segments: outputSegments,
       system_message: systemMessageContent,
-      expected_assistant_raw: expectedAssistantRaw,
+      reference_answer: referenceAnswer,
       guideline_paths: guidelinePaths.map((guidelinePath) => path.resolve(guidelinePath)),
       guideline_patterns: guidelinePatterns,
       file_paths: allFilePaths,
       code_snippets: codeSnippets,
-      outcome,
+      expected_outcome: outcome,
       evaluator: testCaseEvaluatorKind,
       evaluators
     };
@@ -346,36 +385,36 @@ ${content}`);
       logWarning(`Could not read guideline file ${absolutePath}: ${error.message}`);
     }
   }
-  const requestParts = [];
-  for (const segment of testCase.user_segments) {
+  const questionParts = [];
+  for (const segment of testCase.input_segments) {
     const typeValue = segment.type;
     if (typeof typeValue === "string" && typeValue === "file") {
       const pathValue = segment.path;
       const textValue = segment.text;
       const label = typeof pathValue === "string" ? pathValue : "file";
       const body = typeof textValue === "string" ? textValue : "";
-      requestParts.push(`=== ${label} ===
+      questionParts.push(`=== ${label} ===
 ${body}`);
       continue;
     }
     if (typeof typeValue === "string" && typeValue === "text") {
       const value = segment.value;
       if (typeof value === "string") {
-        requestParts.push(value);
+        questionParts.push(value);
       }
       continue;
     }
     const genericValue = segment.value;
     if (typeof genericValue === "string") {
-      requestParts.push(genericValue);
+      questionParts.push(genericValue);
     }
   }
   if (testCase.code_snippets.length > 0) {
-    requestParts.push(testCase.code_snippets.join("\n"));
+    questionParts.push(testCase.code_snippets.join("\n"));
   }
-  const request = requestParts.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
+  const question = questionParts.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
   const guidelines = guidelineContents.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
-  return { request, guidelines, systemMessage: testCase.system_message };
+  return { question, guidelines, systemMessage: testCase.system_message };
 }
 async function fileExists2(absolutePath) {
   try {
@@ -587,7 +626,7 @@ function buildChatPrompt(request) {
 ${request.guidelines.trim()}`);
   }
   const systemContent = systemSegments.join("\n\n");
-  const userContent = request.prompt.trim();
+  const userContent = request.question.trim();
   const prompt = [
     {
       role: "system",
@@ -885,7 +924,7 @@ var CliProvider = class {
       healthcheck.commandTemplate,
       buildTemplateValues(
         {
-          prompt: "",
+          question: "",
           guidelines: "",
           inputFiles: [],
           evalCaseId: "",
@@ -912,7 +951,7 @@ var CliProvider = class {
 function buildTemplateValues(request, config) {
   const inputFiles = normalizeInputFiles(request.inputFiles);
   return {
-    PROMPT: shellEscape(request.prompt ?? ""),
+    PROMPT: shellEscape(request.question ?? ""),
     GUIDELINES: shellEscape(request.guidelines ?? ""),
     EVAL_ID: shellEscape(request.evalCaseId ?? ""),
     ATTEMPT: shellEscape(String(request.attempt ?? 0)),
@@ -976,6 +1015,59 @@ import { tmpdir } from "node:os";
 import path4 from "node:path";
 import { promisify as promisify2 } from "node:util";
 
+// src/evaluation/providers/codex-log-tracker.ts
+var GLOBAL_LOGS_KEY = Symbol.for("agentv.codexLogs");
+var GLOBAL_SUBSCRIBERS_KEY = Symbol.for("agentv.codexLogSubscribers");
+function getCodexLogStore() {
+  const globalObject = globalThis;
+  const existing = globalObject[GLOBAL_LOGS_KEY];
+  if (existing) {
+    return existing;
+  }
+  const created = [];
+  globalObject[GLOBAL_LOGS_KEY] = created;
+  return created;
+}
+function getSubscriberStore() {
+  const globalObject = globalThis;
+  const existing = globalObject[GLOBAL_SUBSCRIBERS_KEY];
+  if (existing) {
+    return existing;
+  }
+  const created = /* @__PURE__ */ new Set();
+  globalObject[GLOBAL_SUBSCRIBERS_KEY] = created;
+  return created;
+}
+function notifySubscribers(entry) {
+  const subscribers = Array.from(getSubscriberStore());
+  for (const listener of subscribers) {
+    try {
+      listener(entry);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      console.warn(`Codex log subscriber failed: ${message}`);
+    }
+  }
+}
+function recordCodexLogEntry(entry) {
+  getCodexLogStore().push(entry);
+  notifySubscribers(entry);
+}
+function consumeCodexLogEntries() {
+  const store = getCodexLogStore();
+  if (store.length === 0) {
+    return [];
+  }
+  return store.splice(0, store.length);
+}
+function subscribeToCodexLogEntries(listener) {
+  const store = getSubscriberStore();
+  store.add(listener);
+  return () => {
+    store.delete(listener);
+  };
+}
+
 // src/evaluation/providers/preread.ts
 import path3 from "node:path";
 function buildPromptDocument(request, inputFiles, options) {
@@ -993,7 +1085,7 @@ function buildPromptDocument(request, inputFiles, options) {
   if (prereadBlock.length > 0) {
     parts.push("\n", prereadBlock);
   }
-  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
+  parts.push("\n[[ ## user_query ## ]]\n", request.question.trim());
   return parts.join("\n").trim();
 }
 function normalizeInputFiles2(inputFiles) {
@@ -1077,59 +1169,6 @@ function pathToFileUri(filePath) {
   return `file://${normalizedPath}`;
 }
 
-// src/evaluation/providers/codex-log-tracker.ts
-var GLOBAL_LOGS_KEY = Symbol.for("agentv.codexLogs");
-var GLOBAL_SUBSCRIBERS_KEY = Symbol.for("agentv.codexLogSubscribers");
-function getCodexLogStore() {
-  const globalObject = globalThis;
-  const existing = globalObject[GLOBAL_LOGS_KEY];
-  if (existing) {
-    return existing;
-  }
-  const created = [];
-  globalObject[GLOBAL_LOGS_KEY] = created;
-  return created;
-}
-function getSubscriberStore() {
-  const globalObject = globalThis;
-  const existing = globalObject[GLOBAL_SUBSCRIBERS_KEY];
-  if (existing) {
-    return existing;
-  }
-  const created = /* @__PURE__ */ new Set();
-  globalObject[GLOBAL_SUBSCRIBERS_KEY] = created;
-  return created;
-}
-function notifySubscribers(entry) {
-  const subscribers = Array.from(getSubscriberStore());
-  for (const listener of subscribers) {
-    try {
-      listener(entry);
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      console.warn(`Codex log subscriber failed: ${message}`);
-    }
-  }
-}
-function recordCodexLogEntry(entry) {
-  getCodexLogStore().push(entry);
-  notifySubscribers(entry);
-}
-function consumeCodexLogEntries() {
-  const store = getCodexLogStore();
-  if (store.length === 0) {
-    return [];
-  }
-  return store.splice(0, store.length);
-}
-function subscribeToCodexLogEntries(listener) {
-  const store = getSubscriberStore();
-  store.add(listener);
-  return () => {
-    store.delete(listener);
-  };
-}
-
 // src/evaluation/providers/codex.ts
 var execAsync2 = promisify2(execCallback);
 var WORKSPACE_PREFIX = "agentv-codex-";
@@ -1863,7 +1902,7 @@ var MockProvider = class {
     return {
       text: this.cannedResponse,
       raw: {
-        prompt: request.prompt,
+        question: request.question,
         guidelines: request.guidelines
       }
     };
@@ -2479,7 +2518,7 @@ function buildPromptDocument2(request, attachments, guidelinePatterns) {
   if (prereadBlock.length > 0) {
     parts.push("\n", prereadBlock);
   }
-  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
+  parts.push("\n[[ ## user_query ## ]]\n", request.question.trim());
   return parts.join("\n").trim();
 }
 function buildMandatoryPrereadBlock2(guidelineFiles, attachmentFiles) {
@@ -2716,30 +2755,7 @@ function resolveAndCreateProvider(definition, env = process.env) {
 }
 
 // src/evaluation/evaluators.ts
-import { ax, f } from "@ax-llm/ax";
 import { randomUUID as randomUUID2 } from "node:crypto";
-var LLM_JUDGE_SIGNATURE = f().input(
-  "evaluationContext",
-  f.object(
-    {
-      expectedOutcome: f.string("The expected outcome for the original task"),
-      request: f.string("The original task request"),
-      referenceAnswer: f.string("The gold standard reference answer"),
-      generatedAnswer: f.string("The answer to evaluate"),
-      guidelines: f.string("Additional evaluation guidelines or instructions").optional()
-    },
-    "Complete evaluation context for the judge"
-  )
-).output(
-  "evaluation",
-  f.object({
-    score: f.number("Score between 0.0 and 1.0").min(0).max(1),
-    hits: f.string("Brief specific achievement").array(),
-    misses: f.string("Brief specific failure or omission").array(),
-    reasoning: f.string("Concise explanation for the score").max(500)
-  })
-).build();
-var LLM_JUDGE = ax(LLM_JUDGE_SIGNATURE);
 var LlmJudgeEvaluator = class {
   kind = "llm_judge";
   resolveJudgeProvider;
@@ -2757,52 +2773,29 @@ var LlmJudgeEvaluator = class {
     if (!judgeProvider) {
       throw new Error("No judge provider available for LLM grading");
     }
-    if (providerSupportsAx(judgeProvider)) {
-      return this.evaluateWithAx(context, judgeProvider);
-    }
     return this.evaluateWithPrompt(context, judgeProvider);
   }
-  async evaluateWithAx(context, judgeProvider) {
-    const ai = judgeProvider.getAxAI();
-    const guidelines = context.promptInputs.guidelines?.trim();
-    const evaluationContext = {
-      expectedOutcome: context.evalCase.outcome.trim(),
-      request: context.evalCase.task.trim(),
-      referenceAnswer: context.evalCase.expected_assistant_raw.trim(),
-      generatedAnswer: context.candidate.trim(),
-      ...guidelines ? { guidelines } : {}
-    };
-    const options = this.buildJudgeForwardOptions(context);
-    const result = await LLM_JUDGE.forward(ai, { evaluationContext }, options);
-    const evaluation = result.evaluation;
-    const expectedAspectCount = Math.max(
-      evaluation.hits.length + evaluation.misses.length,
-      1
-    );
-    return {
-      score: evaluation.score,
-      hits: evaluation.hits,
-      misses: evaluation.misses,
-      expectedAspectCount,
-      reasoning: evaluation.reasoning,
-      evaluatorRawRequest: {
-        id: randomUUID2(),
-        provider: judgeProvider.id,
-        target: context.target.name,
-        method: "ax-structured-output",
-        signature: LLM_JUDGE_SIGNATURE.toString()
-      }
-    };
-  }
   async evaluateWithPrompt(context, judgeProvider) {
-    const prompt = buildQualityPrompt(context.evalCase, context.candidate);
-    const systemPrompt = context.systemPrompt ?? this.customPrompt ?? QUALITY_SYSTEM_PROMPT;
+    let prompt = buildQualityPrompt(context.evalCase, context.candidate);
+    let systemPrompt = context.systemPrompt ?? this.customPrompt ?? QUALITY_SYSTEM_PROMPT;
+    if (systemPrompt && hasTemplateVariables(systemPrompt)) {
+      const variables = {
+        input_messages: JSON.stringify(context.evalCase.input_segments, null, 2),
+        output_messages: JSON.stringify(context.evalCase.output_segments, null, 2),
+        candidate_answer: context.candidate,
+        reference_answer: context.evalCase.reference_answer,
+        expected_outcome: context.evalCase.expected_outcome,
+        question: context.evalCase.question
+      };
+      prompt = substituteVariables(systemPrompt, variables);
+      systemPrompt = QUALITY_SYSTEM_PROMPT;
+    }
     const metadata = {
       ...systemPrompt !== void 0 ? { systemPrompt } : {},
       ...context.judgeModel !== void 0 ? { model: context.judgeModel } : {}
     };
     const response = await judgeProvider.invoke({
-      prompt,
+      question: prompt,
       metadata,
       evalCaseId: context.evalCase.id,
       attempt: context.attempt,
@@ -2832,33 +2825,11 @@ var LlmJudgeEvaluator = class {
       evaluatorRawRequest
     };
   }
-  buildJudgeForwardOptions(context) {
-    const modelConfig = this.buildJudgeModelConfig();
-    if (modelConfig === void 0 && context.judgeModel === void 0) {
-      return void 0;
-    }
-    return {
-      ...context.judgeModel ? { model: context.judgeModel } : {},
-      ...modelConfig ? { modelConfig } : {}
-    };
-  }
-  buildJudgeModelConfig() {
-    if (this.maxOutputTokens === void 0 && this.temperature === void 0) {
-      return void 0;
-    }
-    return {
-      ...this.maxOutputTokens !== void 0 ? { maxTokens: this.maxOutputTokens } : {},
-      ...this.temperature !== void 0 ? { temperature: this.temperature } : {}
-    };
-  }
 };
-function providerSupportsAx(provider) {
-  return typeof provider.getAxAI === "function";
-}
 var QUALITY_SYSTEM_PROMPT = [
-  "You are an expert evaluator. Your goal is to grade the generated_answer based on how well it achieves the expected_outcome for the original task.",
+  "You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.",
   "",
-  "Use the reference_answer as a gold standard for a high-quality response. The generated_answer does not need to match it verbatim, but it should capture the key points and follow the same spirit.",
+  "Use the reference_answer as a gold standard for a high-quality response. The candidate_answer does not need to match it verbatim, but it should capture the key points and follow the same spirit.",
   "",
   "Be concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.",
   "",
@@ -2871,18 +2842,18 @@ var QUALITY_SYSTEM_PROMPT = [
   '  "reasoning": "<string, concise explanation for the score, 1-2 sentences max>"',
   "}"
 ].join("\n");
-function buildQualityPrompt(testCase, candidate) {
+function buildQualityPrompt(evalCase, candidate) {
   const parts = [
     "[[ ## expected_outcome ## ]]",
-    testCase.outcome.trim(),
+    evalCase.expected_outcome.trim(),
     "",
-    "[[ ## request ## ]]",
-    testCase.task.trim(),
+    "[[ ## question ## ]]",
+    evalCase.question.trim(),
     "",
     "[[ ## reference_answer ## ]]",
-    testCase.expected_assistant_raw.trim(),
+    evalCase.reference_answer.trim(),
     "",
-    "[[ ## generated_answer ## ]]",
+    "[[ ## candidate_answer ## ]]",
     candidate.trim(),
     "",
     "Respond with a single JSON object matching the schema described in the system prompt."
@@ -2982,14 +2953,14 @@ var CodeEvaluator = class {
   async evaluate(context) {
     const inputPayload = JSON.stringify(
       {
-        task: context.evalCase.task,
-        outcome: context.evalCase.outcome,
-        expected: context.evalCase.expected_assistant_raw,
-        output: context.candidate,
+        question: context.evalCase.question,
+        expected_outcome: context.evalCase.expected_outcome,
+        reference_answer: context.evalCase.reference_answer,
+        candidate_answer: context.candidate,
         system_message: context.promptInputs.systemMessage ?? "",
         guideline_paths: context.evalCase.guideline_paths,
-        attachments: context.evalCase.file_paths,
-        user_segments: context.evalCase.user_segments
+        input_files: context.evalCase.file_paths,
+        input_segments: context.evalCase.input_segments
       },
       null,
       2
@@ -3075,6 +3046,14 @@ function parseJsonSafe(payload) {
     return void 0;
   }
 }
+function hasTemplateVariables(text) {
+  return /\$\{[a-zA-Z0-9_]+\}/.test(text);
+}
+function substituteVariables(template, variables) {
+  return template.replace(/\$\{([a-zA-Z0-9_]+)\}/g, (match, varName) => {
+    return variables[varName] ?? match;
+  });
+}
 
 // src/evaluation/orchestrator.ts
 import { createHash, randomUUID as randomUUID3 } from "node:crypto";
@@ -3431,7 +3410,7 @@ async function runBatchEvaluation(options) {
   const batchRequests = evalCases.map((evalCase, index) => {
     const promptInputs = promptInputsList[index];
     return {
-      prompt: promptInputs.request,
+      question: promptInputs.question,
       guidelines: promptInputs.guidelines,
       guideline_patterns: evalCase.guideline_patterns,
       inputFiles: evalCase.file_paths,
@@ -3618,7 +3597,7 @@ async function evaluateCandidate(options) {
   });
   const completedAt = nowFn();
   const rawRequest = {
-    request: promptInputs.request,
+    question: promptInputs.question,
     guidelines: promptInputs.guidelines,
     guideline_paths: evalCase.guideline_paths,
     system_message: promptInputs.systemMessage ?? ""
@@ -3630,7 +3609,7 @@ async function evaluateCandidate(options) {
     score: score.score,
     hits: score.hits,
     misses: score.misses,
-    model_answer: candidate,
+    candidate_answer: candidate,
     expected_aspect_count: score.expectedAspectCount,
     target: target.name,
     timestamp: completedAt.toISOString(),
@@ -3840,7 +3819,7 @@ async function dumpPrompt(directory, evalCase, promptInputs) {
   await mkdir2(path7.dirname(filePath), { recursive: true });
   const payload = {
     eval_id: evalCase.id,
-    request: promptInputs.request,
+    question: promptInputs.question,
     guidelines: promptInputs.guidelines,
     guideline_paths: evalCase.guideline_paths
   };
@@ -3862,7 +3841,7 @@ async function invokeProvider(provider, options) {
   }
   try {
     return await provider.invoke({
-      prompt: promptInputs.request,
+      question: promptInputs.question,
       guidelines: promptInputs.guidelines,
       guideline_patterns: evalCase.guideline_patterns,
       inputFiles: evalCase.file_paths,
@@ -3882,7 +3861,7 @@ async function invokeProvider(provider, options) {
 function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs) {
   const message = error instanceof Error ? error.message : String(error);
   const rawRequest = {
-    request: promptInputs.request,
+    question: promptInputs.question,
     guidelines: promptInputs.guidelines,
     guideline_paths: evalCase.guideline_paths,
     system_message: promptInputs.systemMessage ?? "",
@@ -3895,7 +3874,7 @@ function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs)
     score: 0,
     hits: [],
     misses: [`Error: ${message}`],
-    model_answer: `Error occurred: ${message}`,
+    candidate_answer: `Error occurred: ${message}`,
     expected_aspect_count: 0,
     target: targetName,
     timestamp: timestamp.toISOString(),
@@ -3908,7 +3887,7 @@ function createCacheKey(provider, target, evalCase, promptInputs) {
   hash.update(provider.id);
   hash.update(target.name);
   hash.update(evalCase.id);
-  hash.update(promptInputs.request);
+  hash.update(promptInputs.question);
   hash.update(promptInputs.guidelines);
   hash.update(promptInputs.systemMessage ?? "");
   return hash.digest("hex");