agentv 1.2.0 → 1.3.1

package/dist/{chunk-IVIT4U6S.js → chunk-6R2YRXCQ.js}

@@ -164,7 +164,7 @@ import { access as access6, mkdir as mkdir7 } from "node:fs/promises";
 import path19 from "node:path";
 import { pathToFileURL } from "node:url";
 
-// ../../packages/core/dist/chunk-V3JCB3HI.js
+// ../../packages/core/dist/chunk-4A6L2F6L.js
 import { constants } from "node:fs";
 import { access, readFile } from "node:fs/promises";
 import path from "node:path";
@@ -4211,7 +4211,7 @@ var coerce = {
 };
 var NEVER = INVALID;
 
-// ../../packages/core/dist/chunk-V3JCB3HI.js
+// ../../packages/core/dist/chunk-4A6L2F6L.js
 async function fileExists(filePath) {
   try {
     await access(filePath, constants.F_OK);
@@ -4612,10 +4612,14 @@ function resolveCliConfig(target, env, evalFilePath) {
   const filesFormat = resolveOptionalLiteralString(
     target.files_format ?? target.filesFormat ?? target.attachments_format ?? target.attachmentsFormat
   );
+  const verbose = resolveOptionalBoolean(target.verbose ?? target.cli_verbose ?? target.cliVerbose);
   let cwd = resolveOptionalString(target.cwd, env, `${target.name} working directory`, {
     allowLiteral: true,
     optionalEnv: true
   });
+  if (cwd && evalFilePath && !path2.isAbsolute(cwd)) {
+    cwd = path2.resolve(path2.dirname(path2.resolve(evalFilePath)), cwd);
+  }
   if (!cwd && evalFilePath) {
     cwd = path2.dirname(path2.resolve(evalFilePath));
   }
@@ -4623,7 +4627,7 @@ function resolveCliConfig(target, env, evalFilePath) {
     target.timeout_seconds ?? target.timeoutSeconds,
     `${target.name} timeout`
   );
-  const healthcheck = resolveCliHealthcheck(target.healthcheck, env, target.name);
+  const healthcheck = resolveCliHealthcheck(target.healthcheck, env, target.name, evalFilePath);
   const commandTemplate = resolveString(
     commandTemplateSource,
     env,
@@ -4636,7 +4640,8 @@ function resolveCliConfig(target, env, evalFilePath) {
     filesFormat,
     cwd,
     timeoutMs,
-    healthcheck
+    healthcheck,
+    verbose
   };
 }
 function resolveTimeoutMs(source2, description) {
@@ -4649,7 +4654,7 @@ function resolveTimeoutMs(source2, description) {
   }
   return Math.floor(seconds * 1e3);
 }
-function resolveCliHealthcheck(source2, env, targetName) {
+function resolveCliHealthcheck(source2, env, targetName, evalFilePath) {
   if (source2 === void 0 || source2 === null) {
     return void 0;
   }
@@ -4682,11 +4687,12 @@ function resolveCliHealthcheck(source2, env, targetName) {
       allowLiteral: true,
       optionalEnv: true
     });
+    const resolvedCwd = cwd && evalFilePath && !path2.isAbsolute(cwd) ? path2.resolve(path2.dirname(path2.resolve(evalFilePath)), cwd) : cwd;
     return {
       type: "command",
       commandTemplate,
       timeoutMs,
-      cwd
+      cwd: resolvedCwd
     };
   }
   throw new Error(`${targetName} healthcheck type must be 'http' or 'command'`);
@@ -35253,6 +35259,17 @@ async function processMessages(options) {
       }
       continue;
     }
+    if (isJsonObject(content)) {
+      const rendered = JSON.stringify(content, null, 2);
+      segments.push({ type: "text", value: rendered });
+      if (textParts) {
+        textParts.push(rendered);
+      }
+      continue;
+    }
+    if (!Array.isArray(content)) {
+      continue;
+    }
     for (const rawSegment of content) {
       if (!isJsonObject(rawSegment)) {
         continue;
@@ -35475,6 +35492,11 @@ async function buildPromptInputs(testCase, mode = "lm") {
           }
         }
       }
+    } else if (isJsonObject(message.content)) {
+      const rendered = JSON.stringify(message.content, null, 2);
+      if (rendered.trim().length > 0) {
+        messageSegments.push({ type: "text", value: rendered });
+      }
     }
     segmentsByMessage.push(messageSegments);
   }
@@ -36193,7 +36215,7 @@ var CliProvider = class {
   id;
   kind = "cli";
   targetName;
-  supportsBatch = false;
+  supportsBatch = true;
   config;
   runCommand;
   verbose;
@@ -36213,6 +36235,11 @@ var CliProvider = class {
     const outputFilePath = generateOutputFilePath(request.evalCaseId);
     const templateValues = buildTemplateValues(request, this.config, outputFilePath);
     const renderedCommand = renderTemplate2(this.config.commandTemplate, templateValues);
+    if (this.verbose) {
+      console.log(
+        `[cli-provider:${this.targetName}] cwd=${this.config.cwd ?? ""} command=${renderedCommand}`
+      );
+    }
     const result = await this.runCommand(renderedCommand, {
       cwd: this.config.cwd,
       env: process.env,
@@ -36247,6 +36274,114 @@ var CliProvider = class {
       }
     };
   }
+  async invokeBatch(requests) {
+    if (requests.length === 0) {
+      return [];
+    }
+    for (const request of requests) {
+      if (request.signal?.aborted) {
+        throw new Error("CLI provider batch request was aborted before execution");
+      }
+    }
+    const controller = new AbortController();
+    for (const request of requests) {
+      request.signal?.addEventListener("abort", () => controller.abort(), { once: true });
+    }
+    await this.ensureHealthy(controller.signal);
+    const outputFilePath = generateOutputFilePath("batch", ".jsonl");
+    const batchInputFiles = [];
+    for (const request of requests) {
+      if (request.inputFiles && request.inputFiles.length > 0) {
+        batchInputFiles.push(...request.inputFiles);
+      }
+    }
+    const templateValues = buildTemplateValues(
+      {
+        question: "",
+        guidelines: "",
+        inputFiles: batchInputFiles,
+        evalCaseId: "batch",
+        attempt: 0
+      },
+      this.config,
+      outputFilePath
+    );
+    const renderedCommand = renderTemplate2(this.config.commandTemplate, templateValues);
+    if (this.verbose) {
+      console.log(
+        `[cli-provider:${this.targetName}] (batch size=${requests.length}) cwd=${this.config.cwd ?? ""} command=${renderedCommand}`
+      );
+    }
+    const result = await this.runCommand(renderedCommand, {
+      cwd: this.config.cwd,
+      env: process.env,
+      timeoutMs: this.config.timeoutMs,
+      signal: controller.signal
+    });
+    if (result.failed || (result.exitCode ?? 0) !== 0) {
+      if (controller.signal.aborted) {
+        throw new Error("CLI provider request was aborted");
+      }
+      if (result.timedOut) {
+        throw new Error(
+          `CLI provider timed out${formatTimeoutSuffix(this.config.timeoutMs ?? void 0)}`
+        );
+      }
+      const codeText = result.exitCode !== null ? result.exitCode : "unknown";
+      const detail = result.stderr.trim() || result.stdout.trim();
+      const message = detail ? `${detail} (exit code ${codeText})` : `CLI exited with code ${codeText}`;
+      throw new Error(message);
+    }
+    const responseContent = await this.readAndCleanupOutputFile(outputFilePath);
+    const recordsById = this.parseJsonlBatchOutput(responseContent);
+    const requestedIds = requests.map((request) => request.evalCaseId).filter((id) => typeof id === "string" && id.trim().length > 0);
+    const missingIds = requestedIds.filter((id) => !recordsById.has(id));
+    if (missingIds.length > 0) {
+      throw new Error(`CLI batch output missing ids: ${missingIds.join(", ")}`);
+    }
+    const responses = requests.map((request) => {
+      const evalCaseId = request.evalCaseId;
+      if (!evalCaseId) {
+        return {
+          text: "",
+          raw: {
+            command: renderedCommand,
+            stderr: result.stderr,
+            exitCode: result.exitCode ?? 0,
+            cwd: this.config.cwd,
+            outputFile: outputFilePath
+          }
+        };
+      }
+      const parsed = recordsById.get(evalCaseId);
+      if (!parsed) {
+        return {
+          text: "",
+          raw: {
+            command: renderedCommand,
+            stderr: result.stderr,
+            exitCode: result.exitCode ?? 0,
+            cwd: this.config.cwd,
+            outputFile: outputFilePath
+          }
+        };
+      }
+      return {
+        text: parsed.text,
+        trace: parsed.trace,
+        traceRef: parsed.traceRef,
+        raw: {
+          command: renderedCommand,
+          stderr: result.stderr,
+          exitCode: result.exitCode ?? 0,
+          cwd: this.config.cwd,
+          outputFile: outputFilePath,
+          recordId: evalCaseId
+        }
+      };
+    });
+    return responses;
+  }
   /**
    * Parse output content from CLI.
    * If the content is valid JSON with a 'text' field, extract text and optional trace.
@@ -36272,6 +36407,38 @@ var CliProvider = class {
     const validEvents = trace2.filter(isTraceEvent);
     return validEvents.length > 0 ? validEvents : void 0;
   }
+  parseJsonlBatchOutput(content) {
+    const records = /* @__PURE__ */ new Map();
+    const lines = content.split(/\r?\n/).map((line2) => line2.trim()).filter((line2) => line2.length > 0);
+    for (const line2 of lines) {
+      let parsed;
+      try {
+        parsed = JSON.parse(line2);
+      } catch (error40) {
+        const reason = error40 instanceof Error ? error40.message : String(error40);
+        throw new Error(`CLI batch output contains invalid JSONL line: ${reason}`);
+      }
+      if (typeof parsed !== "object" || parsed === null) {
+        throw new Error("CLI batch output JSONL line must be an object");
+      }
+      const obj = parsed;
+      const id = typeof obj.id === "string" ? obj.id : void 0;
+      if (!id || id.trim().length === 0) {
+        throw new Error("CLI batch output JSONL line missing required string field: id");
+      }
+      if (records.has(id)) {
+        throw new Error(`CLI batch output contains duplicate id: ${id}`);
+      }
+      const text2 = typeof obj.text === "string" ? obj.text : obj.text === void 0 ? "" : JSON.stringify(obj.text);
+      const traceRef = typeof obj.traceRef === "string" ? obj.traceRef : typeof obj.trace_ref === "string" ? obj.trace_ref : void 0;
+      records.set(id, {
+        text: text2,
+        trace: this.parseTrace(obj.trace),
+        traceRef
+      });
+    }
+    return records;
+  }
   async readAndCleanupOutputFile(filePath) {
     try {
       const content = await readTextFile(filePath);
@@ -36333,7 +36500,7 @@ var CliProvider = class {
     );
     if (this.verbose) {
       console.log(
-        `[cli-provider:${this.targetName}] (healthcheck) CLI_EVALS_DIR=${process.env.CLI_EVALS_DIR ?? ""} cwd=${healthcheck.cwd ?? this.config.cwd ?? ""} command=${renderedCommand}`
+        `[cli-provider:${this.targetName}] (healthcheck) cwd=${healthcheck.cwd ?? this.config.cwd ?? ""} command=${renderedCommand}`
       );
     }
     const result = await this.runCommand(renderedCommand, {
@@ -36401,11 +36568,11 @@ function shellEscape(value) {
   }
   return `'${value.replace(/'/g, `'"'"'`)}'`;
 }
-function generateOutputFilePath(evalCaseId) {
+function generateOutputFilePath(evalCaseId, extension = ".json") {
   const safeEvalId = evalCaseId || "unknown";
   const timestamp = Date.now();
   const random = Math.random().toString(36).substring(2, 9);
-  return path72.join(os2.tmpdir(), `agentv-${safeEvalId}-${timestamp}-${random}.json`);
+  return path72.join(os2.tmpdir(), `agentv-${safeEvalId}-${timestamp}-${random}${extension}`);
 }
 function formatTimeoutSuffix(timeoutMs) {
   if (!timeoutMs || timeoutMs <= 0) {
@@ -37895,6 +38062,7 @@ var CodeEvaluator = class {
       {
         question: context.evalCase.question,
         expected_outcome: context.evalCase.expected_outcome,
+        expected_messages: context.evalCase.expected_messages,
         reference_answer: context.evalCase.reference_answer,
         candidate_answer: context.candidate,
         guideline_files: context.evalCase.guideline_paths,
@@ -39963,61 +40131,56 @@ function getDefaultExtension(format) {
 }
 
 // src/commands/eval/progress-display.ts
-import { stripVTControlCharacters } from "node:util";
-var ESC = "\x1B[";
-var CLEAR_LINE = `${ESC}K`;
-var MOVE_CURSOR_UP = `${ESC}1A`;
 var ProgressDisplay = class {
   workers = /* @__PURE__ */ new Map();
-  maxWorkers;
   totalTests = 0;
   completedTests = 0;
-  renderTimer;
-  renderScheduled = false;
-  isInteractive;
   logPaths = [];
   logPathSet = /* @__PURE__ */ new Set();
   hasPrintedLogHeader = false;
-  windowHeight = 0;
   started = false;
   finished = false;
-  constructor(maxWorkers) {
-    this.maxWorkers = maxWorkers;
-    this.isInteractive = process.stdout.isTTY && !process.env.CI;
+  verbose;
+  constructor(_maxWorkers, options) {
+    this.verbose = options?.verbose ?? false;
   }
   isInteractiveMode() {
-    return this.isInteractive;
+    return false;
   }
   start() {
     this.started = true;
     this.finished = false;
-    if (this.isInteractive) {
-      this.write("\n");
-      this.renderTimer = setInterval(() => {
-        this.scheduleRender();
-      }, 1e3);
-      this.renderTimer.unref?.();
-    }
   }
   setTotalTests(count) {
     this.totalTests = count;
   }
   updateWorker(progress) {
+    const previous = this.workers.get(progress.workerId);
     this.workers.set(progress.workerId, progress);
     if (progress.status === "completed" || progress.status === "failed") {
       this.completedTests++;
     }
-    if (this.isInteractive) {
-      this.scheduleRender();
-    } else {
-      const targetSuffix = progress.targetLabel ? ` | ${progress.targetLabel}` : "";
-      if (progress.status === "completed") {
-        console.log(`\u2713 Eval ${progress.evalId}${targetSuffix} completed`);
-      } else if (progress.status === "failed") {
+    const targetSuffix = progress.targetLabel ? ` | ${progress.targetLabel}` : "";
+    const countPrefix = `${this.completedTests}/${this.totalTests}`;
+    switch (progress.status) {
+      case "pending":
+        if (this.verbose && !previous) {
+          console.log(`${countPrefix}   \u23F3 ${progress.evalId}${targetSuffix}`);
+        }
+        break;
+      case "running":
+        if (!previous || previous.status === "pending") {
+          console.log(`${countPrefix}   \u{1F504} ${progress.evalId}${targetSuffix}`);
+        }
+        break;
+      case "completed":
+        console.log(`${countPrefix}   \u2705 ${progress.evalId}${targetSuffix}`);
+        break;
+      case "failed":
         console.log(
-          `\u2717 Eval ${progress.evalId}${targetSuffix} failed${progress.error ? `: ${progress.error}` : ""}`
+          `${countPrefix}   \u274C ${progress.evalId}${targetSuffix}${progress.error ? `: ${progress.error}` : ""}`
         );
-      }
+        break;
     }
   }
   addLogPaths(paths) {
@@ -40033,10 +40196,6 @@ var ProgressDisplay = class {
       return;
     }
     this.logPaths.push(...newPaths);
-    if (this.isInteractive) {
-      this.scheduleRender();
-      return;
-    }
     if (!this.hasPrintedLogHeader) {
       console.log("");
       console.log("Codex CLI logs:");
@@ -40047,112 +40206,11 @@ var ProgressDisplay = class {
       console.log(`${startIndex + offset + 1}. ${path27}`);
     });
   }
-  scheduleRender() {
-    if (this.renderScheduled || this.finished) {
-      return;
-    }
-    this.renderScheduled = true;
-    setTimeout(() => {
-      this.renderScheduled = false;
-      this.render();
-    }, 100);
-  }
-  write(content) {
-    process.stdout.write(content);
-  }
-  clearWindow() {
-    if (this.windowHeight === 0) {
-      return;
-    }
-    this.write(`\r${CLEAR_LINE}`);
-    for (let i = 1; i < this.windowHeight; i++) {
-      this.write(`${MOVE_CURSOR_UP}\r${CLEAR_LINE}`);
-    }
-    this.windowHeight = 0;
-  }
-  getRenderedRowCount(rows) {
-    const columns = process.stdout.columns || 80;
-    let count = 0;
-    for (const row of rows) {
-      const text2 = stripVTControlCharacters(row);
-      count += Math.max(1, Math.ceil(text2.length / columns));
-    }
-    return count;
-  }
-  render() {
-    if (!this.isInteractive || !this.started || this.finished) {
-      return;
-    }
-    const lines = [];
-    const sortedWorkers = Array.from(this.workers.values()).sort((a, b) => a.workerId - b.workerId);
-    for (const worker of sortedWorkers) {
-      const line2 = this.formatWorkerLine(worker);
-      lines.push(line2);
-    }
-    if (this.logPaths.length > 0) {
-      lines.push("");
-      lines.push("Codex CLI logs:");
-      this.logPaths.forEach((path27, index) => {
-        lines.push(`${index + 1}. ${path27}`);
-      });
-    }
-    const rowCount = this.getRenderedRowCount(lines);
-    this.clearWindow();
-    if (lines.length > 0) {
-      this.write(lines.join("\n"));
-    }
-    this.windowHeight = rowCount;
-  }
-  formatWorkerLine(worker) {
-    const workerLabel = `${worker.workerId}.`.padEnd(4);
-    const statusIcon = this.getStatusIcon(worker.status);
-    const targetLabel = worker.targetLabel ? `  | ${worker.targetLabel}` : "";
-    const columns = process.stdout.columns || 80;
-    const maxLineLength = Math.max(40, columns - 4);
-    const reservedLength = workerLabel.length + statusIcon.length + targetLabel.length + 4;
-    const availableLabelLength = Math.max(15, maxLineLength - reservedLength);
-    let testLabel = worker.evalId;
-    if (testLabel.length > availableLabelLength) {
-      testLabel = `${testLabel.substring(0, Math.max(0, availableLabelLength - 3))}...`;
-    }
-    return `${workerLabel} ${statusIcon} ${testLabel}${targetLabel}`;
-  }
-  getStatusIcon(status) {
-    switch (status) {
-      case "pending":
-        return "\u23F3";
-      case "running":
-        return "\u{1F504}";
-      case "completed":
-        return "\u2705";
-      case "failed":
-        return "\u274C";
-      default:
-        return "  ";
-    }
-  }
   finish() {
-    if (this.renderTimer) {
-      clearInterval(this.renderTimer);
-      this.renderTimer = void 0;
-    }
     this.finished = true;
-    if (this.isInteractive && this.started) {
-      this.clearWindow();
-      const sortedWorkers = Array.from(this.workers.values()).sort(
-        (a, b) => a.workerId - b.workerId
-      );
-      for (const worker of sortedWorkers) {
-        this.write(`${this.formatWorkerLine(worker)}
-`);
-      }
-      this.write("\n");
-    }
+    console.log("");
   }
   clear() {
-    if (this.isInteractive) {
-      this.clearWindow();
-    }
   }
 };
 
@@ -40656,6 +40714,9 @@ var CLI_SETTINGS = /* @__PURE__ */ new Set([
   ...COMMON_SETTINGS,
   "command_template",
   "commandTemplate",
+  "verbose",
+  "cli_verbose",
+  "cliVerbose",
   "files_format",
   "filesFormat",
   "attachments_format",
@@ -40789,6 +40850,15 @@ async function validateTargetsFile(filePath) {
     if (healthcheck !== void 0) {
       validateCliHealthcheck(healthcheck, absolutePath2, `${location}.healthcheck`, errors2);
     }
+    const verbose = target.verbose ?? target.cli_verbose ?? target.cliVerbose;
+    if (verbose !== void 0 && typeof verbose !== "boolean") {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location: `${location}.verbose`,
+        message: "'verbose' must be a boolean when provided"
+      });
+    }
   }
   function validateCliHealthcheck(healthcheck, absolutePath2, location, errors2) {
     if (!isObject22(healthcheck)) {
@@ -41432,8 +41502,8 @@ function createEvaluationCache() {
     }
   };
 }
-function createProgressReporter(maxWorkers) {
-  const display = new ProgressDisplay(maxWorkers);
+function createProgressReporter(maxWorkers, options) {
+  const display = new ProgressDisplay(maxWorkers, options);
   return {
     isInteractive: display.isInteractiveMode(),
     start: () => display.start(),
@@ -41461,6 +41531,22 @@ function createDisplayIdTracker() {
     }
   };
 }
+function applyVerboseOverride(selection, cliVerbose) {
+  const { resolvedTarget } = selection;
+  if (resolvedTarget.kind !== "cli") {
+    return selection;
+  }
+  return {
+    ...selection,
+    resolvedTarget: {
+      ...resolvedTarget,
+      config: {
+        ...resolvedTarget.config,
+        verbose: cliVerbose
+      }
+    }
+  };
+}
 async function prepareFileMetadata(params) {
   const { testFilePath, repoRoot, cwd, options } = params;
   await ensureFileExists(testFilePath, "Test file");
@@ -41520,7 +41606,7 @@ async function runSingleEvalFile(params) {
     evalCases
   } = params;
   await ensureFileExists(testFilePath, "Test file");
-  const resolvedTargetSelection = selection;
+  const resolvedTargetSelection = applyVerboseOverride(selection, options.verbose);
   const providerLabel = options.dryRun ? `${resolvedTargetSelection.resolvedTarget.kind} (dry-run)` : resolvedTargetSelection.resolvedTarget.kind;
   const targetMessage = options.verbose ? `Using target (${resolvedTargetSelection.targetSource}): ${resolvedTargetSelection.targetName} [provider=${providerLabel}] via ${resolvedTargetSelection.targetsFilePath}` : `Using target: ${inlineTargetLabel}`;
   if (!progressReporter.isInteractive || options.verbose) {
@@ -41633,7 +41719,7 @@ async function runEvalCommand(input) {
   if (totalEvalCount === 0) {
     throw new Error("No eval cases matched the provided filters.");
   }
-  const progressReporter = createProgressReporter(totalWorkers);
+  const progressReporter = createProgressReporter(totalWorkers, { verbose: options.verbose });
   progressReporter.start();
   progressReporter.setTotal(totalEvalCount);
   const seenCodexLogPaths = /* @__PURE__ */ new Set();
@@ -42504,4 +42590,4 @@ export {
   app,
   runCli
 };
-//# sourceMappingURL=chunk-IVIT4U6S.js.map
+//# sourceMappingURL=chunk-6R2YRXCQ.js.map