@agentv/core 1.2.0 → 1.4.0

package/dist/index.cjs

@@ -54,8 +54,6 @@ __export(index_exports, {
   isJsonValue: () => isJsonValue,
   isTestMessage: () => isTestMessage,
   isTestMessageRole: () => isTestMessageRole,
-  isTraceEvent: () => isTraceEvent,
-  isTraceEventType: () => isTraceEventType,
   listTargetNames: () => listTargetNames,
   loadEvalCases: () => loadEvalCases,
   normalizeLineEndings: () => normalizeLineEndings,
@@ -135,33 +133,22 @@ function getHitCount(result) {
 }
 
 // src/evaluation/trace.ts
-function isTraceEventType(value) {
-  return typeof value === "string" && ["model_step", "tool_call", "tool_result", "message", "error"].includes(value);
-}
-function isTraceEvent(value) {
-  if (typeof value !== "object" || value === null) {
-    return false;
-  }
-  const candidate = value;
-  return isTraceEventType(candidate.type) && typeof candidate.timestamp === "string";
-}
-function computeTraceSummary(trace) {
+function computeTraceSummary(messages) {
   const toolCallCounts = {};
-  let errorCount = 0;
-  for (const event of trace) {
-    if (event.type === "tool_call" && event.name) {
-      toolCallCounts[event.name] = (toolCallCounts[event.name] ?? 0) + 1;
-    }
-    if (event.type === "error") {
-      errorCount++;
+  let totalToolCalls = 0;
+  for (const message of messages) {
+    if (!message.toolCalls) continue;
+    for (const toolCall of message.toolCalls) {
+      toolCallCounts[toolCall.tool] = (toolCallCounts[toolCall.tool] ?? 0) + 1;
+      totalToolCalls++;
     }
   }
   const toolNames = Object.keys(toolCallCounts).sort();
   return {
-    eventCount: trace.length,
+    eventCount: totalToolCalls,
     toolNames,
     toolCallsByName: toolCallCounts,
-    errorCount
+    errorCount: 0
   };
 }
 
@@ -437,7 +424,8 @@ var TEMPLATE_VARIABLES = {
   QUESTION: "question",
   EXPECTED_OUTCOME: "expected_outcome",
   REFERENCE_ANSWER: "reference_answer",
-  INPUT_MESSAGES: "input_messages"
+  INPUT_MESSAGES: "input_messages",
+  OUTPUT_MESSAGES: "output_messages"
 };
 var VALID_TEMPLATE_VARIABLES = new Set(Object.values(TEMPLATE_VARIABLES));
 var REQUIRED_TEMPLATE_VARIABLES = /* @__PURE__ */ new Set([
@@ -835,6 +823,17 @@ async function processMessages(options) {
       }
       continue;
     }
+    if (isJsonObject(content)) {
+      const rendered = JSON.stringify(content, null, 2);
+      segments.push({ type: "text", value: rendered });
+      if (textParts) {
+        textParts.push(rendered);
+      }
+      continue;
+    }
+    if (!Array.isArray(content)) {
+      continue;
+    }
     for (const rawSegment of content) {
       if (!isJsonObject(rawSegment)) {
         continue;
@@ -1061,6 +1060,11 @@ async function buildPromptInputs(testCase, mode = "lm") {
           }
         }
       }
+    } else if (isJsonObject(message.content)) {
+      const rendered = JSON.stringify(message.content, null, 2);
+      if (rendered.trim().length > 0) {
+        messageSegments.push({ type: "text", value: rendered });
+      }
     }
     segmentsByMessage.push(messageSegments);
   }
@@ -1304,16 +1308,16 @@ async function loadEvalCases(evalFilePath, repoRoot, options) {
     }) : [];
     const codeSnippets = extractCodeBlocks(inputSegments);
     let referenceAnswer = "";
-    if (outputSegments.length > 1) {
-      referenceAnswer = JSON.stringify(outputSegments, null, 2);
-    } else if (outputSegments.length === 1) {
-      const singleMessage = outputSegments[0];
-      if (typeof singleMessage.content === "string") {
-        referenceAnswer = singleMessage.content;
-      } else if (singleMessage.content) {
-        referenceAnswer = JSON.stringify(singleMessage, null, 2);
-      } else if (singleMessage.tool_calls) {
-        referenceAnswer = JSON.stringify(singleMessage, null, 2);
+    if (outputSegments.length > 0) {
+      const lastMessage = outputSegments[outputSegments.length - 1];
+      const content = lastMessage.content;
+      const toolCalls = lastMessage.tool_calls;
+      if (typeof content === "string") {
+        referenceAnswer = content;
+      } else if (content !== void 0 && content !== null) {
+        referenceAnswer = JSON.stringify(content, null, 2);
+      } else if (toolCalls !== void 0 && toolCalls !== null) {
+        referenceAnswer = JSON.stringify(toolCalls, null, 2);
       }
     }
     const question = inputTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
@@ -1756,11 +1760,11 @@ async function invokeModel(options) {
   return mapResponse(result);
 }
 function mapResponse(result) {
+  const content = result.text ?? "";
   return {
-    text: result.text ?? "",
-    reasoning: result.reasoningText ?? void 0,
     raw: result,
-    usage: toJsonObject(result.totalUsage ?? result.usage)
+    usage: toJsonObject(result.totalUsage ?? result.usage),
+    outputMessages: [{ role: "assistant", content }]
   };
 }
 function toJsonObject(value) {
@@ -1909,10 +1913,11 @@ var CliProvider = class {
   id;
   kind = "cli";
   targetName;
-  supportsBatch = false;
+  supportsBatch = true;
   config;
   runCommand;
   verbose;
+  keepTempFiles;
   healthcheckPromise;
   constructor(targetName, config, runner = defaultCommandRunner) {
     this.targetName = targetName;
@@ -1920,6 +1925,7 @@ var CliProvider = class {
     this.config = config;
     this.runCommand = runner;
     this.verbose = config.verbose ?? false;
+    this.keepTempFiles = config.keepTempFiles ?? false;
   }
   async invoke(request) {
     if (request.signal?.aborted) {
@@ -1929,6 +1935,11 @@ var CliProvider = class {
     const outputFilePath = generateOutputFilePath(request.evalCaseId);
     const templateValues = buildTemplateValues(request, this.config, outputFilePath);
     const renderedCommand = renderTemplate(this.config.commandTemplate, templateValues);
+    if (this.verbose) {
+      console.log(
+        `[cli-provider:${this.targetName}] cwd=${this.config.cwd ?? ""} command=${renderedCommand}`
+      );
+    }
     const result = await this.runCommand(renderedCommand, {
       cwd: this.config.cwd,
       env: process.env,
@@ -1952,8 +1963,7 @@ var CliProvider = class {
     const responseContent = await this.readAndCleanupOutputFile(outputFilePath);
     const parsed = this.parseOutputContent(responseContent);
     return {
-      text: parsed.text,
-      trace: parsed.trace,
+      outputMessages: parsed.outputMessages,
       raw: {
         command: renderedCommand,
         stderr: result.stderr,
@@ -1963,30 +1973,225 @@ var CliProvider = class {
       }
     };
   }
+  async invokeBatch(requests) {
+    if (requests.length === 0) {
+      return [];
+    }
+    for (const request of requests) {
+      if (request.signal?.aborted) {
+        throw new Error("CLI provider batch request was aborted before execution");
+      }
+    }
+    const controller = new AbortController();
+    for (const request of requests) {
+      request.signal?.addEventListener("abort", () => controller.abort(), { once: true });
+    }
+    await this.ensureHealthy(controller.signal);
+    const outputFilePath = generateOutputFilePath("batch", ".jsonl");
+    const batchInputFiles = [];
+    for (const request of requests) {
+      if (request.inputFiles && request.inputFiles.length > 0) {
+        batchInputFiles.push(...request.inputFiles);
+      }
+    }
+    const templateValues = buildTemplateValues(
+      {
+        question: "",
+        guidelines: "",
+        inputFiles: batchInputFiles,
+        evalCaseId: "batch",
+        attempt: 0
+      },
+      this.config,
+      outputFilePath
+    );
+    const renderedCommand = renderTemplate(this.config.commandTemplate, templateValues);
+    if (this.verbose) {
+      console.log(
+        `[cli-provider:${this.targetName}] (batch size=${requests.length}) cwd=${this.config.cwd ?? ""} command=${renderedCommand}`
+      );
+    }
+    const result = await this.runCommand(renderedCommand, {
+      cwd: this.config.cwd,
+      env: process.env,
+      timeoutMs: this.config.timeoutMs,
+      signal: controller.signal
+    });
+    if (result.failed || (result.exitCode ?? 0) !== 0) {
+      if (controller.signal.aborted) {
+        throw new Error("CLI provider request was aborted");
+      }
+      if (result.timedOut) {
+        throw new Error(
+          `CLI provider timed out${formatTimeoutSuffix(this.config.timeoutMs ?? void 0)}`
+        );
+      }
+      const codeText = result.exitCode !== null ? result.exitCode : "unknown";
+      const detail = result.stderr.trim() || result.stdout.trim();
+      const message = detail ? `${detail} (exit code ${codeText})` : `CLI exited with code ${codeText}`;
+      throw new Error(message);
+    }
+    const responseContent = await this.readAndCleanupOutputFile(outputFilePath);
+    const recordsById = this.parseJsonlBatchOutput(responseContent);
+    const requestedIds = requests.map((request) => request.evalCaseId).filter((id) => typeof id === "string" && id.trim().length > 0);
+    const missingIds = requestedIds.filter((id) => !recordsById.has(id));
+    if (missingIds.length > 0) {
+      throw new Error(`CLI batch output missing ids: ${missingIds.join(", ")}`);
+    }
+    const responses = requests.map((request) => {
+      const evalCaseId = request.evalCaseId;
+      if (!evalCaseId) {
+        return {
+          outputMessages: [],
+          raw: {
+            command: renderedCommand,
+            stderr: result.stderr,
+            exitCode: result.exitCode ?? 0,
+            cwd: this.config.cwd,
+            outputFile: outputFilePath
+          }
+        };
+      }
+      const parsed = recordsById.get(evalCaseId);
+      if (!parsed) {
+        return {
+          outputMessages: [],
+          raw: {
+            command: renderedCommand,
+            stderr: result.stderr,
+            exitCode: result.exitCode ?? 0,
+            cwd: this.config.cwd,
+            outputFile: outputFilePath
+          }
+        };
+      }
+      return {
+        outputMessages: parsed.outputMessages,
+        raw: {
+          command: renderedCommand,
+          stderr: result.stderr,
+          exitCode: result.exitCode ?? 0,
+          cwd: this.config.cwd,
+          outputFile: outputFilePath,
+          recordId: evalCaseId
+        }
+      };
+    });
+    return responses;
+  }
   /**
    * Parse output content from CLI.
-   * If the content is valid JSON with a 'text' field, extract text and optional trace.
-   * Otherwise, treat the entire content as plain text.
+   * If the content is valid JSON with 'output_messages' or 'text' field, extract them.
+   * If only 'text' is provided, wrap it in outputMessages.
+   * Otherwise, treat the entire content as plain text wrapped in outputMessages.
    */
   parseOutputContent(content) {
     try {
       const parsed = JSON.parse(content);
-      if (typeof parsed === "object" && parsed !== null && "text" in parsed) {
+      if (typeof parsed === "object" && parsed !== null) {
         const obj = parsed;
-        const text = typeof obj.text === "string" ? obj.text : String(obj.text);
-        const trace = this.parseTrace(obj.trace);
-        return { text, trace };
+        const outputMessages = this.parseOutputMessages(obj.output_messages);
+        if (outputMessages && outputMessages.length > 0) {
+          return { outputMessages };
+        }
+        if ("text" in obj) {
+          const text = typeof obj.text === "string" ? obj.text : String(obj.text);
+          return { outputMessages: [{ role: "assistant", content: text }] };
+        }
       }
     } catch {
     }
-    return { text: content };
+    return { outputMessages: [{ role: "assistant", content }] };
+  }
+  /**
+   * Parse output_messages from JSONL (snake_case) and convert to OutputMessage[] (camelCase).
+   */
+  parseOutputMessages(outputMessages) {
+    if (!Array.isArray(outputMessages)) {
+      return void 0;
+    }
+    const messages = [];
+    for (const msg of outputMessages) {
+      if (typeof msg !== "object" || msg === null) {
+        continue;
+      }
+      const rawMsg = msg;
+      if (typeof rawMsg.role !== "string") {
+        continue;
+      }
+      const message = {
+        role: rawMsg.role,
+        name: typeof rawMsg.name === "string" ? rawMsg.name : void 0,
+        content: rawMsg.content,
+        toolCalls: this.parseToolCalls(rawMsg.tool_calls),
+        timestamp: typeof rawMsg.timestamp === "string" ? rawMsg.timestamp : void 0,
+        metadata: typeof rawMsg.metadata === "object" && rawMsg.metadata !== null ? rawMsg.metadata : void 0
+      };
+      messages.push(message);
+    }
+    return messages.length > 0 ? messages : void 0;
   }
-  parseTrace(trace) {
-    if (!Array.isArray(trace)) {
+  /**
+   * Parse tool_calls from JSONL (snake_case) and convert to ToolCall[] format.
+   */
+  parseToolCalls(toolCalls) {
+    if (!Array.isArray(toolCalls)) {
       return void 0;
     }
-    const validEvents = trace.filter(isTraceEvent);
-    return validEvents.length > 0 ? validEvents : void 0;
+    const calls = [];
+    for (const call of toolCalls) {
+      if (typeof call !== "object" || call === null) {
+        continue;
+      }
+      const rawCall = call;
+      if (typeof rawCall.tool !== "string") {
+        continue;
+      }
+      calls.push({
+        tool: rawCall.tool,
+        input: rawCall.input,
+        output: rawCall.output,
+        id: typeof rawCall.id === "string" ? rawCall.id : void 0,
+        timestamp: typeof rawCall.timestamp === "string" ? rawCall.timestamp : void 0
+      });
+    }
+    return calls.length > 0 ? calls : void 0;
+  }
+  parseJsonlBatchOutput(content) {
+    const records = /* @__PURE__ */ new Map();
+    const lines = content.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
+    for (const line of lines) {
+      let parsed;
+      try {
+        parsed = JSON.parse(line);
+      } catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        throw new Error(`CLI batch output contains invalid JSONL line: ${reason}`);
+      }
+      if (typeof parsed !== "object" || parsed === null) {
+        throw new Error("CLI batch output JSONL line must be an object");
+      }
+      const obj = parsed;
+      const id = typeof obj.id === "string" ? obj.id : void 0;
+      if (!id || id.trim().length === 0) {
+        throw new Error("CLI batch output JSONL line missing required string field: id");
+      }
+      if (records.has(id)) {
+        throw new Error(`CLI batch output contains duplicate id: ${id}`);
+      }
+      const parsedOutputMessages = this.parseOutputMessages(obj.output_messages);
+      let outputMessages;
+      if (parsedOutputMessages && parsedOutputMessages.length > 0) {
+        outputMessages = parsedOutputMessages;
+      } else {
+        const text = typeof obj.text === "string" ? obj.text : obj.text === void 0 ? "" : JSON.stringify(obj.text);
+        outputMessages = text ? [{ role: "assistant", content: text }] : [];
+      }
+      records.set(id, {
+        outputMessages
+      });
+    }
+    return records;
   }
   async readAndCleanupOutputFile(filePath) {
     try {
@@ -1996,8 +2201,10 @@ var CliProvider = class {
       const errorMsg = error instanceof Error ? error.message : String(error);
       throw new Error(`Failed to read output file '${filePath}': ${errorMsg}`);
     } finally {
-      await import_promises8.default.unlink(filePath).catch(() => {
-      });
+      if (!this.keepTempFiles) {
+        await import_promises8.default.unlink(filePath).catch(() => {
+        });
+      }
     }
   }
   async ensureHealthy(signal) {
@@ -2049,7 +2256,7 @@ var CliProvider = class {
     );
     if (this.verbose) {
       console.log(
-        `[cli-provider:${this.targetName}] (healthcheck) CLI_EVALS_DIR=${process.env.CLI_EVALS_DIR ?? ""} cwd=${healthcheck.cwd ?? this.config.cwd ?? ""} command=${renderedCommand}`
+        `[cli-provider:${this.targetName}] (healthcheck) cwd=${healthcheck.cwd ?? this.config.cwd ?? ""} command=${renderedCommand}`
       );
     }
     const result = await this.runCommand(renderedCommand, {
@@ -2117,11 +2324,11 @@ function shellEscape(value) {
   }
   return `'${value.replace(/'/g, `'"'"'`)}'`;
 }
-function generateOutputFilePath(evalCaseId) {
+function generateOutputFilePath(evalCaseId, extension = ".json") {
   const safeEvalId = evalCaseId || "unknown";
   const timestamp = Date.now();
   const random = Math.random().toString(36).substring(2, 9);
-  return import_node_path8.default.join(import_node_os.default.tmpdir(), `agentv-${safeEvalId}-${timestamp}-${random}.json`);
+  return import_node_path8.default.join(import_node_os.default.tmpdir(), `agentv-${safeEvalId}-${timestamp}-${random}${extension}`);
 }
 function formatTimeoutSuffix(timeoutMs) {
   if (!timeoutMs || timeoutMs <= 0) {
@@ -2340,7 +2547,6 @@ var CodexProvider = class {
       const parsed = parseCodexJson(result.stdout);
       const assistantText = extractAssistantText(parsed);
       return {
-        text: assistantText,
         raw: {
           response: parsed,
           stdout: result.stdout,
@@ -2352,7 +2558,8 @@ var CodexProvider = class {
           workspace: workspaceRoot,
           inputFiles,
           logFile: logger?.filePath
-        }
+        },
+        outputMessages: [{ role: "assistant", content: assistantText }]
       };
     } finally {
       await logger?.close();
@@ -2974,7 +3181,6 @@ var MockProvider = class {
   delayMs;
   delayMinMs;
   delayMaxMs;
-  trace;
   constructor(targetName, config) {
     this.id = `mock:${targetName}`;
     this.targetName = targetName;
@@ -2982,7 +3188,6 @@ var MockProvider = class {
     this.delayMs = config.delayMs ?? 0;
     this.delayMinMs = config.delayMinMs ?? 0;
     this.delayMaxMs = config.delayMaxMs ?? 0;
-    this.trace = config.trace;
   }
   async invoke(request) {
     const delay = this.calculateDelay();
@@ -2990,12 +3195,11 @@ var MockProvider = class {
       await new Promise((resolve) => setTimeout(resolve, delay));
     }
     return {
-      text: this.cannedResponse,
+      outputMessages: [{ role: "assistant", content: this.cannedResponse }],
       raw: {
         question: request.question,
         guidelines: request.guidelines
-      },
-      trace: this.trace
+      }
     };
   }
   calculateDelay() {
@@ -3263,8 +3467,7 @@ function normalizeCodexLogFormat(value) {
 }
 function resolveMockConfig(target) {
   const response = typeof target.response === "string" ? target.response : void 0;
-  const trace = Array.isArray(target.trace) ? target.trace : void 0;
-  return { response, trace };
+  return { response };
 }
 function resolveVSCodeConfig(target, env, insiders) {
   const workspaceTemplateEnvVar = resolveOptionalLiteralString(
@@ -3301,10 +3504,17 @@ function resolveCliConfig(target, env, evalFilePath) {
   const filesFormat = resolveOptionalLiteralString(
     target.files_format ?? target.filesFormat ?? target.attachments_format ?? target.attachmentsFormat
   );
+  const verbose = resolveOptionalBoolean(target.verbose ?? target.cli_verbose ?? target.cliVerbose);
+  const keepTempFiles = resolveOptionalBoolean(
+    target.keep_temp_files ?? target.keepTempFiles ?? target.keep_output_files ?? target.keepOutputFiles
+  );
   let cwd = resolveOptionalString(target.cwd, env, `${target.name} working directory`, {
     allowLiteral: true,
     optionalEnv: true
   });
+  if (cwd && evalFilePath && !import_node_path11.default.isAbsolute(cwd)) {
+    cwd = import_node_path11.default.resolve(import_node_path11.default.dirname(import_node_path11.default.resolve(evalFilePath)), cwd);
+  }
   if (!cwd && evalFilePath) {
     cwd = import_node_path11.default.dirname(import_node_path11.default.resolve(evalFilePath));
   }
@@ -3312,7 +3522,7 @@ function resolveCliConfig(target, env, evalFilePath) {
     target.timeout_seconds ?? target.timeoutSeconds,
     `${target.name} timeout`
   );
-  const healthcheck = resolveCliHealthcheck(target.healthcheck, env, target.name);
+  const healthcheck = resolveCliHealthcheck(target.healthcheck, env, target.name, evalFilePath);
   const commandTemplate = resolveString(
     commandTemplateSource,
     env,
@@ -3325,7 +3535,9 @@ function resolveCliConfig(target, env, evalFilePath) {
     filesFormat,
     cwd,
     timeoutMs,
-    healthcheck
+    healthcheck,
+    verbose,
+    keepTempFiles
   };
 }
 function resolveTimeoutMs(source, description) {
@@ -3338,7 +3550,7 @@ function resolveTimeoutMs(source, description) {
   }
   return Math.floor(seconds * 1e3);
 }
-function resolveCliHealthcheck(source, env, targetName) {
+function resolveCliHealthcheck(source, env, targetName, evalFilePath) {
   if (source === void 0 || source === null) {
     return void 0;
   }
@@ -3371,11 +3583,12 @@ function resolveCliHealthcheck(source, env, targetName) {
       allowLiteral: true,
       optionalEnv: true
     });
+    const resolvedCwd = cwd && evalFilePath && !import_node_path11.default.isAbsolute(cwd) ? import_node_path11.default.resolve(import_node_path11.default.dirname(import_node_path11.default.resolve(evalFilePath)), cwd) : cwd;
     return {
       type: "command",
       commandTemplate,
       timeoutMs,
-      cwd
+      cwd: resolvedCwd
     };
   }
   throw new Error(`${targetName} healthcheck type must be 'http' or 'command'`);
@@ -3619,7 +3832,7 @@ var VSCodeProvider = class {
     }
     if (this.config.dryRun) {
       return {
-        text: "",
+        outputMessages: [],
         raw: {
           session,
           inputFiles
@@ -3628,7 +3841,7 @@ var VSCodeProvider = class {
     }
     const responseText = await readTextFile(session.responseFile);
     return {
-      text: responseText,
+      outputMessages: [{ role: "assistant", content: responseText }],
       raw: {
         session,
         inputFiles
@@ -3666,7 +3879,7 @@ var VSCodeProvider = class {
     }
     if (this.config.dryRun) {
       return normalizedRequests.map(({ inputFiles }) => ({
-        text: "",
+        outputMessages: [],
         raw: {
           session,
           inputFiles,
@@ -3683,7 +3896,7 @@ var VSCodeProvider = class {
     for (const [index, responseFile] of session.responseFiles.entries()) {
       const responseText = await readTextFile(responseFile);
       responses.push({
-        text: responseText,
+        outputMessages: [{ role: "assistant", content: responseText }],
         raw: {
           session,
           inputFiles: normalizedRequests[index]?.inputFiles,
@@ -3923,6 +4136,33 @@ function resolveAndCreateProvider(definition, env = process.env) {
 // src/evaluation/evaluators.ts
 var import_ai2 = require("ai");
 var import_zod2 = require("zod");
+
+// src/evaluation/providers/types.ts
+var AGENT_PROVIDER_KINDS = [
+  "codex",
+  "vscode",
+  "vscode-insiders"
+];
+function extractLastAssistantContent(messages) {
+  if (!messages || messages.length === 0) {
+    return "";
+  }
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msg = messages[i];
+    if (msg.role === "assistant" && msg.content !== void 0) {
+      if (typeof msg.content === "string") {
+        return msg.content;
+      }
+      return JSON.stringify(msg.content);
+    }
+  }
+  return "";
+}
+function isAgentProvider(provider) {
+  return provider ? AGENT_PROVIDER_KINDS.includes(provider.kind) : false;
+}
+
+// src/evaluation/evaluators.ts
 var DEFAULT_EVALUATOR_TEMPLATE = `You are an expert evaluator. Your goal is to grade the candidate_answer based on how well it achieves the expected_outcome for the original task.
 
 Use the reference_answer as a gold standard for a high-quality response (if provided). The reference_answer may be a simple text response, or it may contain a sequence of expected agent messages including tool calls. When it contains multiple messages, the last message represents the final expected answer. The candidate_answer does not need to match it verbatim, but should capture the key points and follow the same spirit.
@@ -3987,6 +4227,7 @@ var LlmJudgeEvaluator = class {
         null,
         2
       ),
+      [TEMPLATE_VARIABLES.OUTPUT_MESSAGES]: JSON.stringify(context.outputMessages ?? [], null, 2),
       [TEMPLATE_VARIABLES.CANDIDATE_ANSWER]: context.candidate.trim(),
       [TEMPLATE_VARIABLES.REFERENCE_ANSWER]: (context.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.EXPECTED_OUTCOME]: context.evalCase.expected_outcome.trim(),
@@ -4011,7 +4252,7 @@ var LlmJudgeEvaluator = class {
       const score = clampScore(data.score);
       const hits = Array.isArray(data.hits) ? data.hits.filter(isNonEmptyString).slice(0, 4) : [];
       const misses = Array.isArray(data.misses) ? data.misses.filter(isNonEmptyString).slice(0, 4) : [];
-      const reasoning = data.reasoning ?? providerResponse?.reasoning;
+      const reasoning = data.reasoning;
       const expectedAspectCount = Math.max(hits.length + misses.length, 1);
       return {
         score,
@@ -4113,7 +4354,9 @@ var LlmJudgeEvaluator = class {
           maxOutputTokens: this.maxOutputTokens,
           temperature: this.temperature
         });
-        const data = schema.parse(parseJsonFromText(response.text ?? ""));
+        const data = schema.parse(
+          parseJsonFromText(extractLastAssistantContent(response.outputMessages))
+        );
         return { data, providerResponse: response };
       } catch (e) {
         lastError = e instanceof Error ? e : new Error(String(e));
@@ -4196,15 +4439,16 @@ var CodeEvaluator = class {
       {
         question: context.evalCase.question,
         expected_outcome: context.evalCase.expected_outcome,
+        expected_messages: context.evalCase.expected_messages,
         reference_answer: context.evalCase.reference_answer,
         candidate_answer: context.candidate,
+        output_messages: context.outputMessages ?? null,
         guideline_files: context.evalCase.guideline_paths,
         input_files: context.evalCase.file_paths.filter(
           (path15) => !context.evalCase.guideline_paths.includes(path15)
         ),
         input_messages: context.evalCase.input_messages,
-        candidate_trace_file: context.candidateTraceRef ?? null,
-        candidate_trace_summary: context.candidateTraceSummary ?? null
+        candidate_trace_summary: context.traceSummary ?? null
       },
       null,
       2
@@ -4331,8 +4575,19 @@ var ToolTrajectoryEvaluator = class {
     this.config = options.config;
   }
   evaluate(context) {
-    const { candidateTrace, candidateTraceSummary } = context;
-    if (!candidateTrace || !candidateTraceSummary) {
+    const { outputMessages, traceSummary } = context;
+    const toolCalls = this.extractToolCallsFromMessages(outputMessages);
+    if (toolCalls.length === 0 && !traceSummary) {
+      return {
+        score: 0,
+        verdict: "fail",
+        hits: [],
+        misses: ["No trace available for evaluation"],
+        expectedAspectCount: 1
+      };
+    }
+    const summary = toolCalls.length > 0 ? this.buildSummary(toolCalls) : traceSummary;
+    if (!summary) {
       return {
         score: 0,
         verdict: "fail",
@@ -4343,11 +4598,11 @@ var ToolTrajectoryEvaluator = class {
     }
     switch (this.config.mode) {
       case "any_order":
-        return this.evaluateAnyOrder(candidateTraceSummary);
+        return this.evaluateAnyOrder(summary);
       case "in_order":
-        return this.evaluateInOrder(candidateTrace);
+        return this.evaluateInOrder(toolCalls);
       case "exact":
-        return this.evaluateExact(candidateTrace);
+        return this.evaluateExact(toolCalls);
       default:
         return {
           score: 0,
@@ -4358,6 +4613,39 @@ var ToolTrajectoryEvaluator = class {
         };
     }
   }
+  /**
+   * Extract tool calls from output messages.
+   */
+  extractToolCallsFromMessages(messages) {
+    if (!messages) {
+      return [];
+    }
+    const toolCalls = [];
+    for (const message of messages) {
+      if (message.toolCalls) {
+        for (const call of message.toolCalls) {
+          toolCalls.push({ name: call.tool });
+        }
+      }
+    }
+    return toolCalls;
+  }
+  /**
+   * Build a summary from extracted tool calls.
+   */
+  buildSummary(toolCalls) {
+    const toolCallsByName = {};
+    for (const call of toolCalls) {
+      toolCallsByName[call.name] = (toolCallsByName[call.name] ?? 0) + 1;
+    }
+    const toolNames = Object.keys(toolCallsByName).sort();
+    return {
+      eventCount: toolCalls.length,
+      toolNames,
+      toolCallsByName,
+      errorCount: 0
+    };
+  }
   evaluateAnyOrder(summary) {
     const minimums = this.config.minimums ?? {};
     const toolNames = Object.keys(minimums);
@@ -4390,7 +4678,7 @@ var ToolTrajectoryEvaluator = class {
       expectedAspectCount: toolNames.length
     };
   }
-  evaluateInOrder(trace) {
+  evaluateInOrder(toolCalls) {
     const expected = this.config.expected ?? [];
     if (expected.length === 0) {
       return {
@@ -4401,15 +4689,14 @@ var ToolTrajectoryEvaluator = class {
         expectedAspectCount: 0
       };
     }
-    const actualToolCalls = trace.filter((e) => e.type === "tool_call" && e.name);
     const hits = [];
     const misses = [];
     let actualIndex = 0;
     for (let i = 0; i < expected.length; i++) {
       const expectedTool = expected[i].tool;
       let found = false;
-      while (actualIndex < actualToolCalls.length) {
-        if (actualToolCalls[actualIndex].name === expectedTool) {
+      while (actualIndex < toolCalls.length) {
+        if (toolCalls[actualIndex].name === expectedTool) {
           hits.push(`Found ${expectedTool} at position ${actualIndex}`);
           actualIndex++;
           found = true;
@@ -4430,7 +4717,7 @@ var ToolTrajectoryEvaluator = class {
       expectedAspectCount: expected.length
     };
   }
-  evaluateExact(trace) {
+  evaluateExact(toolCalls) {
     const expected = this.config.expected ?? [];
     if (expected.length === 0) {
       return {
@@ -4441,16 +4728,15 @@ var ToolTrajectoryEvaluator = class {
         expectedAspectCount: 0
       };
     }
-    const actualToolCalls = trace.filter((e) => e.type === "tool_call" && e.name);
     const hits = [];
     const misses = [];
-    if (actualToolCalls.length !== expected.length) {
-      misses.push(`Expected ${expected.length} tool calls, got ${actualToolCalls.length}`);
+    if (toolCalls.length !== expected.length) {
+      misses.push(`Expected ${expected.length} tool calls, got ${toolCalls.length}`);
     }
-    const checkLength = Math.min(expected.length, actualToolCalls.length);
+    const checkLength = Math.min(expected.length, toolCalls.length);
     for (let i = 0; i < checkLength; i++) {
       const expectedTool = expected[i].tool;
-      const actualTool = actualToolCalls[i].name;
+      const actualTool = toolCalls[i].name;
       if (actualTool === expectedTool) {
         hits.push(`Position ${i}: ${expectedTool} \u2713`);
       } else {
@@ -4664,11 +4950,13 @@ var CompositeEvaluator = class {
         evalCaseId: context.evalCase.id,
         attempt: context.attempt
       });
-      const data = freeformEvaluationSchema.parse(parseJsonFromText(response.text ?? ""));
+      const data = freeformEvaluationSchema.parse(
+        parseJsonFromText(extractLastAssistantContent(response.outputMessages))
+      );
       const score = clampScore(data.score);
       const hits = Array.isArray(data.hits) ? data.hits.filter(isNonEmptyString).slice(0, 4) : [];
       const misses = Array.isArray(data.misses) ? data.misses.filter(isNonEmptyString).slice(0, 4) : [];
-      const reasoning = data.reasoning ?? response.reasoning;
+      const reasoning = data.reasoning;
       return {
         score,
         verdict: scoreToVerdict(score),
@@ -4837,16 +5125,6 @@ function validateConcurrency(concurrency) {
   }
 }
 
-// src/evaluation/providers/types.ts
-var AGENT_PROVIDER_KINDS = [
-  "codex",
-  "vscode",
-  "vscode-insiders"
-];
-function isAgentProvider(provider) {
-  return provider ? AGENT_PROVIDER_KINDS.includes(provider.kind) : false;
-}
-
 // src/evaluation/orchestrator.ts
 async function runEvaluation(options) {
   const {
@@ -5101,11 +5379,14 @@ async function runBatchEvaluation(options) {
     const evalCase = evalCases[i];
     const promptInputs = promptInputsList[i];
     const providerResponse = batchResponse[i];
+    const outputMessages = providerResponse.outputMessages;
+    const traceSummary = outputMessages ? computeTraceSummary(outputMessages) : void 0;
+    const candidate = extractLastAssistantContent(outputMessages);
     let result;
     try {
       result = await evaluateCandidate({
         evalCase,
-        candidate: providerResponse.text ?? "",
+        candidate,
         target,
         provider,
         evaluators: evaluatorRegistry,
@@ -5113,7 +5394,9 @@ async function runBatchEvaluation(options) {
         nowFn,
         attempt: 0,
         judgeProvider: await resolveJudgeProvider(target),
-        agentTimeoutMs
+        agentTimeoutMs,
+        outputMessages,
+        traceSummary
       });
     } catch (error) {
       const errorResult = buildErrorResult(
@@ -5217,21 +5500,13 @@ async function runEvalCase(options) {
   if (cacheKey && cache && !cachedResponse) {
     await cache.set(cacheKey, providerResponse);
   }
-  let candidateTrace = providerResponse.trace;
-  if (!candidateTrace && providerResponse.traceRef) {
-    try {
-      const rawTrace = await readJsonFile(providerResponse.traceRef);
-      if (Array.isArray(rawTrace) && rawTrace.every(isTraceEvent)) {
-        candidateTrace = rawTrace;
-      }
-    } catch {
-    }
-  }
-  const candidateTraceSummary = candidateTrace ? computeTraceSummary(candidateTrace) : void 0;
+  const outputMessages = providerResponse.outputMessages;
+  const traceSummary = outputMessages ? computeTraceSummary(outputMessages) : void 0;
+  const candidate = extractLastAssistantContent(outputMessages);
   try {
     return await evaluateCandidate({
       evalCase,
-      candidate: providerResponse.text ?? "",
+      candidate,
       target,
       provider,
       evaluators,
@@ -5240,9 +5515,8 @@ async function runEvalCase(options) {
       attempt,
       judgeProvider,
       agentTimeoutMs,
-      candidateTrace,
-      candidateTraceRef: providerResponse.traceRef,
-      candidateTraceSummary
+      outputMessages,
+      traceSummary
     });
   } catch (error) {
     return buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs, provider);
@@ -5260,9 +5534,8 @@ async function evaluateCandidate(options) {
     attempt,
     judgeProvider,
     agentTimeoutMs,
-    candidateTrace,
-    candidateTraceRef,
-    candidateTraceSummary
+    outputMessages,
+    traceSummary
   } = options;
   const gradeTimestamp = nowFn();
   const { score, evaluatorResults } = await runEvaluatorsForCase({
@@ -5276,9 +5549,8 @@ async function evaluateCandidate(options) {
     now: gradeTimestamp,
     judgeProvider,
     agentTimeoutMs,
-    candidateTrace,
-    candidateTraceRef,
-    candidateTraceSummary
+    outputMessages,
+    traceSummary
   });
   const completedAt = nowFn();
   let agentProviderRequest;
@@ -5316,7 +5588,7 @@ async function evaluateCandidate(options) {
     lm_provider_request: lmProviderRequest,
     evaluator_provider_request: evaluatorResults ? void 0 : score.evaluatorRawRequest,
     evaluator_results: evaluatorResults,
-    trace_summary: candidateTraceSummary
+    trace_summary: traceSummary
   };
 }
 async function runEvaluatorsForCase(options) {
@@ -5331,9 +5603,8 @@ async function runEvaluatorsForCase(options) {
     now,
     judgeProvider,
     agentTimeoutMs,
-    candidateTrace,
-    candidateTraceRef,
-    candidateTraceSummary
+    outputMessages,
+    traceSummary
   } = options;
   if (evalCase.evaluators && evalCase.evaluators.length > 0) {
     return runEvaluatorList({
@@ -5348,9 +5619,8 @@ async function runEvaluatorsForCase(options) {
       now,
       judgeProvider,
       agentTimeoutMs,
-      candidateTrace,
-      candidateTraceRef,
-      candidateTraceSummary
+      outputMessages,
+      traceSummary
     });
   }
   const evaluatorKind = evalCase.evaluator ?? "llm_judge";
@@ -5367,9 +5637,8 @@ async function runEvaluatorsForCase(options) {
     promptInputs,
     now,
     judgeProvider,
-    candidateTrace,
-    candidateTraceRef,
-    candidateTraceSummary
+    outputMessages,
+    traceSummary
   });
   return { score };
 }
@@ -5386,9 +5655,8 @@ async function runEvaluatorList(options) {
     now,
     judgeProvider,
     agentTimeoutMs,
-    candidateTrace,
-    candidateTraceRef,
-    candidateTraceSummary
+    outputMessages,
+    traceSummary
   } = options;
   const scored = [];
   const evaluatorResults = [];
@@ -5435,8 +5703,8 @@ async function runEvaluatorList(options) {
           attempt,
           promptInputs,
           now,
-          candidateTraceRef,
-          candidateTraceSummary
+          outputMessages,
+          traceSummary
         });
         const weight = evaluator.weight ?? 1;
         scored.push({ score: score2, name: evaluator.name, type: "code_judge", weight });
@@ -5522,9 +5790,8 @@ async function runEvaluatorList(options) {
           attempt,
           promptInputs,
           now,
-          candidateTrace,
-          candidateTraceRef,
-          candidateTraceSummary
+          outputMessages,
+          traceSummary
         });
         const weight = evaluator.weight ?? 1;
         scored.push({ score: score2, name: evaluator.name, type: evaluator.type, weight });
@@ -5918,8 +6185,6 @@ function createAgentKernel() {
   isJsonValue,
   isTestMessage,
   isTestMessageRole,
-  isTraceEvent,
-  isTraceEventType,
   listTargetNames,
   loadEvalCases,
   normalizeLineEndings,