@agentv/core 0.6.1 → 0.7.2

package/dist/index.js

@@ -4,9 +4,10 @@ import {
   buildSearchRoots,
   fileExists,
   findGitRoot,
+  isAgentProvider,
   readTextFile,
   resolveFileReference
-} from "./chunk-OW3SHBIJ.js";
+} from "./chunk-UQLHF3T7.js";
 
 // src/evaluation/types.ts
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
@@ -134,6 +135,87 @@ function extractCodeBlocks(segments) {
   }
   return codeBlocks;
 }
+async function processMessages(options) {
+  const {
+    messages,
+    searchRoots,
+    repoRootPath,
+    guidelinePatterns,
+    guidelinePaths,
+    textParts,
+    messageType,
+    verbose
+  } = options;
+  const segments = [];
+  for (const message of messages) {
+    const content = message.content;
+    if (typeof content === "string") {
+      segments.push({ type: "text", value: content });
+      if (textParts) {
+        textParts.push(content);
+      }
+      continue;
+    }
+    for (const rawSegment of content) {
+      if (!isJsonObject(rawSegment)) {
+        continue;
+      }
+      const segmentType = asString(rawSegment.type);
+      if (segmentType === "file") {
+        const rawValue = asString(rawSegment.value);
+        if (!rawValue) {
+          continue;
+        }
+        const { displayPath, resolvedPath, attempted } = await resolveFileReference(
+          rawValue,
+          searchRoots
+        );
+        if (!resolvedPath) {
+          const attempts = attempted.length ? ["  Tried:", ...attempted.map((candidate) => `    ${candidate}`)] : void 0;
+          const context = messageType === "input" ? "" : " in expected_messages";
+          logWarning(`File not found${context}: ${displayPath}`, attempts);
+          continue;
+        }
+        try {
+          const fileContent = (await readFile(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
+          if (messageType === "input" && guidelinePatterns && guidelinePaths) {
+            const relativeToRepo = path.relative(repoRootPath, resolvedPath);
+            if (isGuidelineFile(relativeToRepo, guidelinePatterns)) {
+              guidelinePaths.push(path.resolve(resolvedPath));
+              if (verbose) {
+                console.log(`  [Guideline] Found: ${displayPath}`);
+                console.log(`    Resolved to: ${resolvedPath}`);
+              }
+              continue;
+            }
+          }
+          segments.push({
+            type: "file",
+            path: displayPath,
+            text: fileContent,
+            resolvedPath: path.resolve(resolvedPath)
+          });
+          if (verbose) {
+            const label = messageType === "input" ? "[File]" : "[Expected Output File]";
+            console.log(`  ${label} Found: ${displayPath}`);
+            console.log(`    Resolved to: ${resolvedPath}`);
+          }
+        } catch (error) {
+          const context = messageType === "input" ? "" : " expected output";
+          logWarning(`Could not read${context} file ${resolvedPath}: ${error.message}`);
+        }
+        continue;
+      }
+      const clonedSegment = cloneJsonObject(rawSegment);
+      segments.push(clonedSegment);
+      const inlineValue = clonedSegment.value;
+      if (typeof inlineValue === "string" && textParts) {
+        textParts.push(inlineValue);
+      }
+    }
+  }
+  return segments;
+}
 async function loadEvalCases(evalFilePath, repoRoot, options) {
   const verbose = options?.verbose ?? false;
   const absoluteTestPath = path.resolve(evalFilePath);
@@ -219,77 +301,34 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
         }
       }
     }
-    const userSegments = [];
     const guidelinePaths = [];
-    const userTextParts = [];
-    for (const userMessage of userMessages) {
-      const content = userMessage.content;
-      if (typeof content === "string") {
-        userSegments.push({ type: "text", value: content });
-        userTextParts.push(content);
-        continue;
-      }
-      for (const rawSegment of content) {
-        if (!isJsonObject(rawSegment)) {
-          continue;
-        }
-        const segmentType = asString(rawSegment.type);
-        if (segmentType === "file") {
-          const rawValue = asString(rawSegment.value);
-          if (!rawValue) {
-            continue;
-          }
-          const { displayPath, resolvedPath, attempted } = await resolveFileReference(
-            rawValue,
-            searchRoots
-          );
-          if (!resolvedPath) {
-            const attempts = attempted.length ? ["  Tried:", ...attempted.map((candidate) => `    ${candidate}`)] : void 0;
-            logWarning(`File not found: ${displayPath}`, attempts);
-            continue;
-          }
-          try {
-            const fileContent = (await readFile(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
-            const relativeToRepo = path.relative(repoRootPath, resolvedPath);
-            if (isGuidelineFile(relativeToRepo, guidelinePatterns)) {
-              guidelinePaths.push(path.resolve(resolvedPath));
-              if (verbose) {
-                console.log(`  [Guideline] Found: ${displayPath}`);
-                console.log(`    Resolved to: ${resolvedPath}`);
-              }
-            } else {
-              userSegments.push({
-                type: "file",
-                path: displayPath,
-                text: fileContent,
-                resolvedPath: path.resolve(resolvedPath)
-              });
-              if (verbose) {
-                console.log(`  [File] Found: ${displayPath}`);
-                console.log(`    Resolved to: ${resolvedPath}`);
-              }
-            }
-          } catch (error) {
-            logWarning(`Could not read file ${resolvedPath}: ${error.message}`);
-          }
-          continue;
-        }
-        const clonedSegment = cloneJsonObject(rawSegment);
-        userSegments.push(clonedSegment);
-        const inlineValue = clonedSegment.value;
-        if (typeof inlineValue === "string") {
-          userTextParts.push(inlineValue);
-        }
-      }
-    }
-    const codeSnippets = extractCodeBlocks(userSegments);
+    const inputTextParts = [];
+    const inputSegments = await processMessages({
+      messages: userMessages,
+      searchRoots,
+      repoRootPath,
+      guidelinePatterns,
+      guidelinePaths,
+      textParts: inputTextParts,
+      messageType: "input",
+      verbose
+    });
+    const outputSegments = await processMessages({
+      messages: assistantMessages,
+      searchRoots,
+      repoRootPath,
+      guidelinePatterns,
+      messageType: "output",
+      verbose
+    });
+    const codeSnippets = extractCodeBlocks(inputSegments);
     const assistantContent = assistantMessages[0]?.content;
-    const expectedAssistantRaw = await resolveAssistantContent(assistantContent, searchRoots, verbose);
-    const userTextPrompt = userTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
+    const referenceAnswer = await resolveAssistantContent(assistantContent, searchRoots, verbose);
+    const question = inputTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
     const testCaseEvaluatorKind = coerceEvaluator(evalcase.evaluator, id) ?? globalEvaluator;
     const evaluators = await parseEvaluators(evalcase, searchRoots, id ?? "unknown");
     const userFilePaths = [];
-    for (const segment of userSegments) {
+    for (const segment of inputSegments) {
       if (segment.type === "file" && typeof segment.resolvedPath === "string") {
         userFilePaths.push(segment.resolvedPath);
       }
@@ -302,15 +341,16 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
       id,
       dataset: datasetName,
       conversation_id: conversationId,
-      task: userTextPrompt,
-      user_segments: userSegments,
+      question,
+      input_segments: inputSegments,
+      output_segments: outputSegments,
       system_message: systemMessageContent,
-      expected_assistant_raw: expectedAssistantRaw,
+      reference_answer: referenceAnswer,
       guideline_paths: guidelinePaths.map((guidelinePath) => path.resolve(guidelinePath)),
       guideline_patterns: guidelinePatterns,
       file_paths: allFilePaths,
       code_snippets: codeSnippets,
-      outcome,
+      expected_outcome: outcome,
       evaluator: testCaseEvaluatorKind,
       evaluators
     };
@@ -346,36 +386,36 @@ ${content}`);
       logWarning(`Could not read guideline file ${absolutePath}: ${error.message}`);
     }
   }
-  const requestParts = [];
-  for (const segment of testCase.user_segments) {
+  const questionParts = [];
+  for (const segment of testCase.input_segments) {
     const typeValue = segment.type;
     if (typeof typeValue === "string" && typeValue === "file") {
       const pathValue = segment.path;
       const textValue = segment.text;
       const label = typeof pathValue === "string" ? pathValue : "file";
       const body = typeof textValue === "string" ? textValue : "";
-      requestParts.push(`=== ${label} ===
+      questionParts.push(`=== ${label} ===
 ${body}`);
       continue;
     }
     if (typeof typeValue === "string" && typeValue === "text") {
       const value = segment.value;
       if (typeof value === "string") {
-        requestParts.push(value);
+        questionParts.push(value);
       }
       continue;
     }
     const genericValue = segment.value;
     if (typeof genericValue === "string") {
-      requestParts.push(genericValue);
+      questionParts.push(genericValue);
     }
   }
   if (testCase.code_snippets.length > 0) {
-    requestParts.push(testCase.code_snippets.join("\n"));
+    questionParts.push(testCase.code_snippets.join("\n"));
   }
-  const request = requestParts.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
+  const question = questionParts.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
   const guidelines = guidelineContents.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
-  return { request, guidelines, systemMessage: testCase.system_message };
+  return { question, guidelines, systemMessage: testCase.system_message };
 }
 async function fileExists2(absolutePath) {
   try {
@@ -587,7 +627,7 @@ function buildChatPrompt(request) {
 ${request.guidelines.trim()}`);
   }
   const systemContent = systemSegments.join("\n\n");
-  const userContent = request.prompt.trim();
+  const userContent = request.question.trim();
   const prompt = [
     {
       role: "system",
@@ -885,7 +925,7 @@ var CliProvider = class {
       healthcheck.commandTemplate,
       buildTemplateValues(
         {
-          prompt: "",
+          question: "",
           guidelines: "",
           inputFiles: [],
           evalCaseId: "",
@@ -912,7 +952,7 @@ var CliProvider = class {
 function buildTemplateValues(request, config) {
   const inputFiles = normalizeInputFiles(request.inputFiles);
   return {
-    PROMPT: shellEscape(request.prompt ?? ""),
+    PROMPT: shellEscape(request.question ?? ""),
     GUIDELINES: shellEscape(request.guidelines ?? ""),
     EVAL_ID: shellEscape(request.evalCaseId ?? ""),
     ATTEMPT: shellEscape(String(request.attempt ?? 0)),
@@ -971,11 +1011,64 @@ function formatTimeoutSuffix(timeoutMs) {
 import { exec as execCallback, spawn } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { constants as constants2, createWriteStream } from "node:fs";
-import { access as access2, copyFile, mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+import { access as access2, mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import path4 from "node:path";
 import { promisify as promisify2 } from "node:util";
 
+// src/evaluation/providers/codex-log-tracker.ts
+var GLOBAL_LOGS_KEY = Symbol.for("agentv.codexLogs");
+var GLOBAL_SUBSCRIBERS_KEY = Symbol.for("agentv.codexLogSubscribers");
+function getCodexLogStore() {
+  const globalObject = globalThis;
+  const existing = globalObject[GLOBAL_LOGS_KEY];
+  if (existing) {
+    return existing;
+  }
+  const created = [];
+  globalObject[GLOBAL_LOGS_KEY] = created;
+  return created;
+}
+function getSubscriberStore() {
+  const globalObject = globalThis;
+  const existing = globalObject[GLOBAL_SUBSCRIBERS_KEY];
+  if (existing) {
+    return existing;
+  }
+  const created = /* @__PURE__ */ new Set();
+  globalObject[GLOBAL_SUBSCRIBERS_KEY] = created;
+  return created;
+}
+function notifySubscribers(entry) {
+  const subscribers = Array.from(getSubscriberStore());
+  for (const listener of subscribers) {
+    try {
+      listener(entry);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      console.warn(`Codex log subscriber failed: ${message}`);
+    }
+  }
+}
+function recordCodexLogEntry(entry) {
+  getCodexLogStore().push(entry);
+  notifySubscribers(entry);
+}
+function consumeCodexLogEntries() {
+  const store = getCodexLogStore();
+  if (store.length === 0) {
+    return [];
+  }
+  return store.splice(0, store.length);
+}
+function subscribeToCodexLogEntries(listener) {
+  const store = getSubscriberStore();
+  store.add(listener);
+  return () => {
+    store.delete(listener);
+  };
+}
+
 // src/evaluation/providers/preread.ts
 import path3 from "node:path";
 function buildPromptDocument(request, inputFiles, options) {
@@ -993,7 +1086,7 @@ function buildPromptDocument(request, inputFiles, options) {
   if (prereadBlock.length > 0) {
     parts.push("\n", prereadBlock);
   }
-  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
+  parts.push("\n[[ ## user_query ## ]]\n", request.question.trim());
   return parts.join("\n").trim();
 }
 function normalizeInputFiles2(inputFiles) {
@@ -1077,64 +1170,10 @@ function pathToFileUri(filePath) {
   return `file://${normalizedPath}`;
 }
 
-// src/evaluation/providers/codex-log-tracker.ts
-var GLOBAL_LOGS_KEY = Symbol.for("agentv.codexLogs");
-var GLOBAL_SUBSCRIBERS_KEY = Symbol.for("agentv.codexLogSubscribers");
-function getCodexLogStore() {
-  const globalObject = globalThis;
-  const existing = globalObject[GLOBAL_LOGS_KEY];
-  if (existing) {
-    return existing;
-  }
-  const created = [];
-  globalObject[GLOBAL_LOGS_KEY] = created;
-  return created;
-}
-function getSubscriberStore() {
-  const globalObject = globalThis;
-  const existing = globalObject[GLOBAL_SUBSCRIBERS_KEY];
-  if (existing) {
-    return existing;
-  }
-  const created = /* @__PURE__ */ new Set();
-  globalObject[GLOBAL_SUBSCRIBERS_KEY] = created;
-  return created;
-}
-function notifySubscribers(entry) {
-  const subscribers = Array.from(getSubscriberStore());
-  for (const listener of subscribers) {
-    try {
-      listener(entry);
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      console.warn(`Codex log subscriber failed: ${message}`);
-    }
-  }
-}
-function recordCodexLogEntry(entry) {
-  getCodexLogStore().push(entry);
-  notifySubscribers(entry);
-}
-function consumeCodexLogEntries() {
-  const store = getCodexLogStore();
-  if (store.length === 0) {
-    return [];
-  }
-  return store.splice(0, store.length);
-}
-function subscribeToCodexLogEntries(listener) {
-  const store = getSubscriberStore();
-  store.add(listener);
-  return () => {
-    store.delete(listener);
-  };
-}
-
 // src/evaluation/providers/codex.ts
 var execAsync2 = promisify2(execCallback);
 var WORKSPACE_PREFIX = "agentv-codex-";
 var PROMPT_FILENAME = "prompt.md";
-var FILES_DIR = "files";
 var JSONL_TYPE_ITEM_COMPLETED = "item.completed";
 var CodexProvider = class {
   id;
@@ -1157,21 +1196,10 @@ var CodexProvider = class {
     }
     await this.ensureEnvironmentReady();
     const inputFiles = normalizeInputFiles2(request.inputFiles);
-    const originalGuidelines = new Set(
-      collectGuidelineFiles(inputFiles, request.guideline_patterns).map((file) => path4.resolve(file))
-    );
     const workspaceRoot = await this.createWorkspace();
     const logger = await this.createStreamLogger(request).catch(() => void 0);
     try {
-      const { mirroredInputFiles, guidelineMirrors } = await this.mirrorInputFiles(
-        inputFiles,
-        workspaceRoot,
-        originalGuidelines
-      );
-      const promptContent = buildPromptDocument(request, mirroredInputFiles, {
-        guidelinePatterns: request.guideline_patterns,
-        guidelineOverrides: guidelineMirrors
-      });
+      const promptContent = buildPromptDocument(request, inputFiles);
       const promptFile = path4.join(workspaceRoot, PROMPT_FILENAME);
       await writeFile(promptFile, promptContent, "utf8");
       const args = this.buildCodexArgs();
@@ -1200,7 +1228,7 @@ var CodexProvider = class {
           executable: this.resolvedExecutable ?? this.config.executable,
           promptFile,
           workspace: workspaceRoot,
-          inputFiles: mirroredInputFiles,
+          inputFiles,
           logFile: logger?.filePath
         }
       };
@@ -1255,37 +1283,6 @@ var CodexProvider = class {
       throw error;
     }
   }
-  async mirrorInputFiles(inputFiles, workspaceRoot, guidelineOriginals) {
-    if (!inputFiles || inputFiles.length === 0) {
-      return {
-        mirroredInputFiles: void 0,
-        guidelineMirrors: /* @__PURE__ */ new Set()
-      };
-    }
-    const filesRoot = path4.join(workspaceRoot, FILES_DIR);
-    await mkdir(filesRoot, { recursive: true });
-    const mirrored = [];
-    const guidelineMirrors = /* @__PURE__ */ new Set();
-    const nameCounts = /* @__PURE__ */ new Map();
-    for (const inputFile of inputFiles) {
-      const absoluteSource = path4.resolve(inputFile);
-      const baseName = path4.basename(absoluteSource);
-      const count = nameCounts.get(baseName) ?? 0;
-      nameCounts.set(baseName, count + 1);
-      const finalName = count === 0 ? baseName : `${baseName}.${count}`;
-      const destination = path4.join(filesRoot, finalName);
-      await copyFile(absoluteSource, destination);
-      const resolvedDestination = path4.resolve(destination);
-      mirrored.push(resolvedDestination);
-      if (guidelineOriginals.has(absoluteSource)) {
-        guidelineMirrors.add(resolvedDestination);
-      }
-    }
-    return {
-      mirroredInputFiles: mirrored,
-      guidelineMirrors
-    };
-  }
   async createWorkspace() {
     return await mkdtemp(path4.join(tmpdir(), WORKSPACE_PREFIX));
   }
@@ -1863,7 +1860,7 @@ var MockProvider = class {
     return {
       text: this.cannedResponse,
       raw: {
-        prompt: request.prompt,
+        question: request.question,
         guidelines: request.guidelines
       }
     };
@@ -2256,23 +2253,25 @@ function resolveOptionalString(source, env, description, options) {
   if (trimmed.length === 0) {
     return void 0;
   }
-  const envValue = env[trimmed];
-  if (envValue !== void 0) {
-    if (envValue.trim().length === 0) {
-      throw new Error(`Environment variable '${trimmed}' for ${description} is empty`);
+  const envVarMatch = trimmed.match(/^\$\{\{\s*([A-Z0-9_]+)\s*\}\}$/i);
+  if (envVarMatch) {
+    const varName = envVarMatch[1];
+    const envValue = env[varName];
+    if (envValue !== void 0) {
+      if (envValue.trim().length === 0) {
+        throw new Error(`Environment variable '${varName}' for ${description} is empty`);
+      }
+      return envValue;
     }
-    return envValue;
-  }
-  const allowLiteral = options?.allowLiteral ?? false;
-  const optionalEnv = options?.optionalEnv ?? false;
-  const looksLikeEnv = isLikelyEnvReference(trimmed);
-  if (looksLikeEnv) {
+    const optionalEnv = options?.optionalEnv ?? false;
     if (optionalEnv) {
       return void 0;
     }
-    if (!allowLiteral) {
-      throw new Error(`Environment variable '${trimmed}' required for ${description} is not set`);
-    }
+    throw new Error(`Environment variable '${varName}' required for ${description} is not set`);
+  }
+  const allowLiteral = options?.allowLiteral ?? false;
+  if (!allowLiteral) {
+    throw new Error(`${description} must use \${{ VARIABLE_NAME }} syntax for environment variables or be marked as allowing literals`);
   }
   return trimmed;
 }
@@ -2319,9 +2318,6 @@ function resolveOptionalBoolean(source) {
   }
   throw new Error("expected boolean value");
 }
-function isLikelyEnvReference(value) {
-  return /^[A-Z0-9_]+$/.test(value);
-}
 function resolveOptionalStringArray(source, env, description) {
   if (source === void 0 || source === null) {
     return void 0;
@@ -2342,21 +2338,25 @@ function resolveOptionalStringArray(source, env, description) {
     if (trimmed.length === 0) {
       throw new Error(`${description}[${i}] cannot be empty`);
     }
-    const envValue = env[trimmed];
-    if (envValue !== void 0) {
-      if (envValue.trim().length === 0) {
-        throw new Error(`Environment variable '${trimmed}' for ${description}[${i}] is empty`);
+    const envVarMatch = trimmed.match(/^\$\{\{\s*([A-Z0-9_]+)\s*\}\}$/i);
+    if (envVarMatch) {
+      const varName = envVarMatch[1];
+      const envValue = env[varName];
+      if (envValue !== void 0) {
+        if (envValue.trim().length === 0) {
+          throw new Error(`Environment variable '${varName}' for ${description}[${i}] is empty`);
+        }
+        resolved.push(envValue);
+        continue;
       }
-      resolved.push(envValue);
-    } else {
-      resolved.push(trimmed);
+      throw new Error(`Environment variable '${varName}' for ${description}[${i}] is not set`);
     }
+    resolved.push(trimmed);
   }
   return resolved.length > 0 ? resolved : void 0;
 }
 
 // src/evaluation/providers/vscode.ts
-import { readFile as readFile2 } from "node:fs/promises";
 import path5 from "node:path";
 import { dispatchAgentSession, dispatchBatchAgent, getSubagentRoot, provisionSubagents } from "subagent";
 var VSCodeProvider = class {
@@ -2400,7 +2400,7 @@ var VSCodeProvider = class {
         }
       };
     }
-    const responseText = await readFile2(session.responseFile, "utf8");
+    const responseText = await readTextFile(session.responseFile);
     return {
       text: responseText,
       raw: {
@@ -2454,7 +2454,7 @@ var VSCodeProvider = class {
     }
     const responses = [];
     for (const [index, responseFile] of session.responseFiles.entries()) {
-      const responseText = await readFile2(responseFile, "utf8");
+      const responseText = await readTextFile(responseFile);
       responses.push({
         text: responseText,
         raw: {
@@ -2479,7 +2479,7 @@ function buildPromptDocument2(request, attachments, guidelinePatterns) {
   if (prereadBlock.length > 0) {
     parts.push("\n", prereadBlock);
   }
-  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
+  parts.push("\n[[ ## user_query ## ]]\n", request.question.trim());
   return parts.join("\n").trim();
 }
 function buildMandatoryPrereadBlock2(guidelineFiles, attachmentFiles) {
@@ -2604,7 +2604,7 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
 
 // src/evaluation/providers/targets-file.ts
 import { constants as constants3 } from "node:fs";
-import { access as access3, readFile as readFile3 } from "node:fs/promises";
+import { access as access3, readFile as readFile2 } from "node:fs/promises";
 import path6 from "node:path";
 import { parse as parse2 } from "yaml";
 function isRecord(value) {
@@ -2672,7 +2672,7 @@ async function readTargetDefinitions(filePath) {
   if (!await fileExists3(absolutePath)) {
     throw new Error(`targets.yaml not found at ${absolutePath}`);
   }
-  const raw = await readFile3(absolutePath, "utf8");
+  const raw = await readFile2(absolutePath, "utf8");
   const parsed = parse2(raw);
   if (!isRecord(parsed)) {
     throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with '$schema' and 'targets' fields`);
@@ -2716,30 +2716,7 @@ function resolveAndCreateProvider(definition, env = process.env) {
 }
 
 // src/evaluation/evaluators.ts
-import { ax, f } from "@ax-llm/ax";
 import { randomUUID as randomUUID2 } from "node:crypto";
-var LLM_JUDGE_SIGNATURE = f().input(
-  "evaluationContext",
-  f.object(
-    {
-      expectedOutcome: f.string("The expected outcome for the original task"),
-      request: f.string("The original task request"),
-      referenceAnswer: f.string("The gold standard reference answer"),
-      generatedAnswer: f.string("The answer to evaluate"),
-      guidelines: f.string("Additional evaluation guidelines or instructions").optional()
-    },
-    "Complete evaluation context for the judge"
-  )
-).output(
-  "evaluation",
-  f.object({
-    score: f.number("Score between 0.0 and 1.0").min(0).max(1),
-    hits: f.string("Brief specific achievement").array(),
-    misses: f.string("Brief specific failure or omission").array(),
-    reasoning: f.string("Concise explanation for the score").max(500)
-  })
-).build();
-var LLM_JUDGE = ax(LLM_JUDGE_SIGNATURE);
 var LlmJudgeEvaluator = class {
   kind = "llm_judge";
   resolveJudgeProvider;
@@ -2757,52 +2734,29 @@ var LlmJudgeEvaluator = class {
     if (!judgeProvider) {
       throw new Error("No judge provider available for LLM grading");
     }
-    if (providerSupportsAx(judgeProvider)) {
-      return this.evaluateWithAx(context, judgeProvider);
-    }
     return this.evaluateWithPrompt(context, judgeProvider);
   }
-  async evaluateWithAx(context, judgeProvider) {
-    const ai = judgeProvider.getAxAI();
-    const guidelines = context.promptInputs.guidelines?.trim();
-    const evaluationContext = {
-      expectedOutcome: context.evalCase.outcome.trim(),
-      request: context.evalCase.task.trim(),
-      referenceAnswer: context.evalCase.expected_assistant_raw.trim(),
-      generatedAnswer: context.candidate.trim(),
-      ...guidelines ? { guidelines } : {}
-    };
-    const options = this.buildJudgeForwardOptions(context);
-    const result = await LLM_JUDGE.forward(ai, { evaluationContext }, options);
-    const evaluation = result.evaluation;
-    const expectedAspectCount = Math.max(
-      evaluation.hits.length + evaluation.misses.length,
-      1
-    );
-    return {
-      score: evaluation.score,
-      hits: evaluation.hits,
-      misses: evaluation.misses,
-      expectedAspectCount,
-      reasoning: evaluation.reasoning,
-      evaluatorRawRequest: {
-        id: randomUUID2(),
-        provider: judgeProvider.id,
-        target: context.target.name,
-        method: "ax-structured-output",
-        signature: LLM_JUDGE_SIGNATURE.toString()
-      }
-    };
-  }
   async evaluateWithPrompt(context, judgeProvider) {
-    const prompt = buildQualityPrompt(context.evalCase, context.candidate);
-    const systemPrompt = context.systemPrompt ?? this.customPrompt ?? QUALITY_SYSTEM_PROMPT;
+    let prompt = buildQualityPrompt(context.evalCase, context.candidate);
+    let systemPrompt = context.systemPrompt ?? this.customPrompt ?? QUALITY_SYSTEM_PROMPT;
+    if (systemPrompt && hasTemplateVariables(systemPrompt)) {
+      const variables = {
+        input_messages: JSON.stringify(context.evalCase.input_segments, null, 2),
+        output_messages: JSON.stringify(context.evalCase.output_segments, null, 2),
+        candidate_answer: context.candidate,
+        reference_answer: context.evalCase.reference_answer,
+        expected_outcome: context.evalCase.expected_outcome,
+        question: context.evalCase.question
+      };
+      prompt = substituteVariables(systemPrompt, variables);
+      systemPrompt = QUALITY_SYSTEM_PROMPT;
+    }
     const metadata = {
       ...systemPrompt !== void 0 ? { systemPrompt } : {},
       ...context.judgeModel !== void 0 ? { model: context.judgeModel } : {}
     };
     const response = await judgeProvider.invoke({
-      prompt,
+      question: prompt,
       metadata,
       evalCaseId: context.evalCase.id,
       attempt: context.attempt,
@@ -2832,33 +2786,11 @@ var LlmJudgeEvaluator = class {
       evaluatorRawRequest
     };
   }
-  buildJudgeForwardOptions(context) {
-    const modelConfig = this.buildJudgeModelConfig();
-    if (modelConfig === void 0 && context.judgeModel === void 0) {
-      return void 0;
-    }
-    return {
-      ...context.judgeModel ? { model: context.judgeModel } : {},
-      ...modelConfig ? { modelConfig } : {}
-    };
-  }
-  buildJudgeModelConfig() {
-    if (this.maxOutputTokens === void 0 && this.temperature === void 0) {
-      return void 0;
-    }
-    return {
-      ...this.maxOutputTokens !== void 0 ? { maxTokens: this.maxOutputTokens } : {},
-      ...this.temperature !== void 0 ? { temperature: this.temperature } : {}
-    };
-  }
 };
-function providerSupportsAx(provider) {
-  return typeof provider.getAxAI === "function";
-}
 var QUALITY_SYSTEM_PROMPT = [
-  "You are an expert evaluator. Your goal is to grade the generated_answer based on how well it achieves the expected_outcome for the original task.",
+  "You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.",
   "",
-  "Use the reference_answer as a gold standard for a high-quality response. The generated_answer does not need to match it verbatim, but it should capture the key points and follow the same spirit.",
+  "Use the reference_answer as a gold standard for a high-quality response. The candidate_answer does not need to match it verbatim, but it should capture the key points and follow the same spirit.",
   "",
   "Be concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.",
   "",
@@ -2871,18 +2803,18 @@ var QUALITY_SYSTEM_PROMPT = [
   '  "reasoning": "<string, concise explanation for the score, 1-2 sentences max>"',
   "}"
 ].join("\n");
-function buildQualityPrompt(testCase, candidate) {
+function buildQualityPrompt(evalCase, candidate) {
   const parts = [
     "[[ ## expected_outcome ## ]]",
-    testCase.outcome.trim(),
+    evalCase.expected_outcome.trim(),
     "",
-    "[[ ## request ## ]]",
-    testCase.task.trim(),
+    "[[ ## question ## ]]",
+    evalCase.question.trim(),
     "",
     "[[ ## reference_answer ## ]]",
-    testCase.expected_assistant_raw.trim(),
+    evalCase.reference_answer.trim(),
     "",
-    "[[ ## generated_answer ## ]]",
+    "[[ ## candidate_answer ## ]]",
     candidate.trim(),
     "",
     "Respond with a single JSON object matching the schema described in the system prompt."
@@ -2982,14 +2914,14 @@ var CodeEvaluator = class {
   async evaluate(context) {
     const inputPayload = JSON.stringify(
       {
-        task: context.evalCase.task,
-        outcome: context.evalCase.outcome,
-        expected: context.evalCase.expected_assistant_raw,
-        output: context.candidate,
+        question: context.evalCase.question,
+        expected_outcome: context.evalCase.expected_outcome,
+        reference_answer: context.evalCase.reference_answer,
+        candidate_answer: context.candidate,
         system_message: context.promptInputs.systemMessage ?? "",
         guideline_paths: context.evalCase.guideline_paths,
-        attachments: context.evalCase.file_paths,
-        user_segments: context.evalCase.user_segments
+        input_files: context.evalCase.file_paths,
+        input_segments: context.evalCase.input_segments
       },
       null,
       2
@@ -3075,6 +3007,14 @@ function parseJsonSafe(payload) {
     return void 0;
   }
 }
+function hasTemplateVariables(text) {
+  return /\$\{[a-zA-Z0-9_]+\}/.test(text);
+}
+function substituteVariables(template, variables) {
+  return template.replace(/\$\{([a-zA-Z0-9_]+)\}/g, (match, varName) => {
+    return variables[varName] ?? match;
+  });
+}
 
 // src/evaluation/orchestrator.ts
 import { createHash, randomUUID as randomUUID3 } from "node:crypto";
@@ -3397,7 +3337,8 @@ async function runEvaluation(options) {
         target.name,
         (now ?? (() => /* @__PURE__ */ new Date()))(),
         outcome.reason,
-        promptInputs
+        promptInputs,
+        primaryProvider
       );
       results.push(errorResult);
       if (onResult) {
@@ -3431,7 +3372,7 @@ async function runBatchEvaluation(options) {
   const batchRequests = evalCases.map((evalCase, index) => {
     const promptInputs = promptInputsList[index];
     return {
-      prompt: promptInputs.request,
+      question: promptInputs.question,
       guidelines: promptInputs.guidelines,
       guideline_patterns: evalCase.guideline_patterns,
       inputFiles: evalCase.file_paths,
@@ -3481,7 +3422,7 @@ async function runBatchEvaluation(options) {
         agentTimeoutMs
       });
     } catch (error) {
-      const errorResult = buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs);
+      const errorResult = buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs, provider);
       results.push(errorResult);
       if (onResult) {
         await onResult(errorResult);
@@ -3558,7 +3499,7 @@ async function runEvalCase(options) {
         attempt += 1;
         continue;
       }
-      return buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs);
+      return buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs, provider);
     }
   }
   if (!providerResponse) {
@@ -3567,7 +3508,8 @@ async function runEvalCase(options) {
       target.name,
       nowFn(),
       lastError ?? new Error("Provider did not return a response"),
-      promptInputs
+      promptInputs,
+      provider
     );
   }
   if (cacheKey && cache && !cachedResponse) {
@@ -3587,7 +3529,7 @@ async function runEvalCase(options) {
       agentTimeoutMs
     });
   } catch (error) {
-    return buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs);
+    return buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs, provider);
   }
 }
 async function evaluateCandidate(options) {
@@ -3618,8 +3560,8 @@ async function evaluateCandidate(options) {
   });
   const completedAt = nowFn();
   const rawRequest = {
-    request: promptInputs.request,
-    guidelines: promptInputs.guidelines,
+    question: promptInputs.question,
+    ...isAgentProvider(provider) ? {} : { guidelines: promptInputs.guidelines },
     guideline_paths: evalCase.guideline_paths,
     system_message: promptInputs.systemMessage ?? ""
   };
@@ -3630,7 +3572,7 @@ async function evaluateCandidate(options) {
     score: score.score,
     hits: score.hits,
     misses: score.misses,
-    model_answer: candidate,
+    candidate_answer: candidate,
     expected_aspect_count: score.expectedAspectCount,
     target: target.name,
     timestamp: completedAt.toISOString(),
@@ -3840,7 +3782,7 @@ async function dumpPrompt(directory, evalCase, promptInputs) {
   await mkdir2(path7.dirname(filePath), { recursive: true });
   const payload = {
     eval_id: evalCase.id,
-    request: promptInputs.request,
+    question: promptInputs.question,
     guidelines: promptInputs.guidelines,
     guideline_paths: evalCase.guideline_paths
   };
@@ -3862,7 +3804,7 @@ async function invokeProvider(provider, options) {
   }
   try {
     return await provider.invoke({
-      prompt: promptInputs.request,
+      question: promptInputs.question,
       guidelines: promptInputs.guidelines,
       guideline_patterns: evalCase.guideline_patterns,
       inputFiles: evalCase.file_paths,
@@ -3879,11 +3821,11 @@ async function invokeProvider(provider, options) {
     }
   }
 }
-function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs) {
+function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs, provider) {
   const message = error instanceof Error ? error.message : String(error);
   const rawRequest = {
-    request: promptInputs.request,
-    guidelines: promptInputs.guidelines,
+    question: promptInputs.question,
+    ...isAgentProvider(provider) ? {} : { guidelines: promptInputs.guidelines },
     guideline_paths: evalCase.guideline_paths,
     system_message: promptInputs.systemMessage ?? "",
     error: message
@@ -3895,7 +3837,7 @@ function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs)
     score: 0,
     hits: [],
     misses: [`Error: ${message}`],
-    model_answer: `Error occurred: ${message}`,
+    candidate_answer: `Error occurred: ${message}`,
     expected_aspect_count: 0,
     target: targetName,
     timestamp: timestamp.toISOString(),
@@ -3908,7 +3850,7 @@ function createCacheKey(provider, target, evalCase, promptInputs) {
   hash.update(provider.id);
   hash.update(target.name);
   hash.update(evalCase.id);
-  hash.update(promptInputs.request);
+  hash.update(promptInputs.question);
   hash.update(promptInputs.guidelines);
   hash.update(promptInputs.systemMessage ?? "");
   return hash.digest("hex");