@agentv/core 0.5.0 → 0.5.3

package/dist/index.js

@@ -953,1323 +953,1581 @@ function formatTimeoutSuffix(timeoutMs) {
   return ` after ${seconds}s`;
 }
 
-// src/evaluation/providers/mock.ts
-var DEFAULT_MOCK_RESPONSE = '{"answer":"Mock provider response. Configure targets.yaml to supply a custom value."}';
-var MockProvider = class {
+// src/evaluation/providers/codex.ts
+import { exec as execCallback, spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { constants as constants2, createWriteStream } from "node:fs";
+import { access as access2, copyFile, mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path4 from "node:path";
+import { promisify as promisify2 } from "node:util";
+
+// src/evaluation/providers/preread.ts
+import path3 from "node:path";
+function buildPromptDocument(request, inputFiles, options) {
+  const parts = [];
+  const guidelineFiles = collectGuidelineFiles(
+    inputFiles,
+    options?.guidelinePatterns ?? request.guideline_patterns,
+    options?.guidelineOverrides
+  );
+  const inputFilesList = collectInputFiles(inputFiles);
+  const nonGuidelineInputFiles = inputFilesList.filter(
+    (file) => !guidelineFiles.includes(file)
+  );
+  const prereadBlock = buildMandatoryPrereadBlock(guidelineFiles, nonGuidelineInputFiles);
+  if (prereadBlock.length > 0) {
+    parts.push("\n", prereadBlock);
+  }
+  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
+  return parts.join("\n").trim();
+}
+function normalizeInputFiles2(inputFiles) {
+  if (!inputFiles || inputFiles.length === 0) {
+    return void 0;
+  }
+  const deduped = /* @__PURE__ */ new Map();
+  for (const inputFile of inputFiles) {
+    const absolutePath = path3.resolve(inputFile);
+    if (!deduped.has(absolutePath)) {
+      deduped.set(absolutePath, absolutePath);
+    }
+  }
+  return Array.from(deduped.values());
+}
+function collectGuidelineFiles(inputFiles, guidelinePatterns, overrides) {
+  if (!inputFiles || inputFiles.length === 0) {
+    return [];
+  }
+  const unique = /* @__PURE__ */ new Map();
+  for (const inputFile of inputFiles) {
+    const absolutePath = path3.resolve(inputFile);
+    if (overrides?.has(absolutePath)) {
+      if (!unique.has(absolutePath)) {
+        unique.set(absolutePath, absolutePath);
+      }
+      continue;
+    }
+    const normalized = absolutePath.split(path3.sep).join("/");
+    if (isGuidelineFile(normalized, guidelinePatterns)) {
+      if (!unique.has(absolutePath)) {
+        unique.set(absolutePath, absolutePath);
+      }
+    }
+  }
+  return Array.from(unique.values());
+}
+function collectInputFiles(inputFiles) {
+  if (!inputFiles || inputFiles.length === 0) {
+    return [];
+  }
+  const unique = /* @__PURE__ */ new Map();
+  for (const inputFile of inputFiles) {
+    const absolutePath = path3.resolve(inputFile);
+    if (!unique.has(absolutePath)) {
+      unique.set(absolutePath, absolutePath);
+    }
+  }
+  return Array.from(unique.values());
+}
+function buildMandatoryPrereadBlock(guidelineFiles, inputFiles) {
+  if (guidelineFiles.length === 0 && inputFiles.length === 0) {
+    return "";
+  }
+  const buildList = (files) => files.map((absolutePath) => {
+    const fileName = path3.basename(absolutePath);
+    const fileUri = pathToFileUri(absolutePath);
+    return `* [${fileName}](${fileUri})`;
+  });
+  const sections = [];
+  if (guidelineFiles.length > 0) {
+    sections.push(`Read all guideline files:
+${buildList(guidelineFiles).join("\n")}.`);
+  }
+  if (inputFiles.length > 0) {
+    sections.push(`Read all input files:
+${buildList(inputFiles).join("\n")}.`);
+  }
+  sections.push(
+    "If any file is missing, fail with ERROR: missing-file <filename> and stop.",
+    "Then apply system_instructions on the user query below."
+  );
+  return sections.join("\n");
+}
+function pathToFileUri(filePath) {
+  const absolutePath = path3.isAbsolute(filePath) ? filePath : path3.resolve(filePath);
+  const normalizedPath = absolutePath.replace(/\\/g, "/");
+  if (/^[a-zA-Z]:\//.test(normalizedPath)) {
+    return `file:///${normalizedPath}`;
+  }
+  return `file://${normalizedPath}`;
+}
+
+// src/evaluation/providers/codex.ts
+var execAsync2 = promisify2(execCallback);
+var WORKSPACE_PREFIX = "agentv-codex-";
+var PROMPT_FILENAME = "prompt.md";
+var FILES_DIR = "files";
+var JSONL_TYPE_ITEM_COMPLETED = "item.completed";
+var CodexProvider = class {
   id;
-  kind = "mock";
+  kind = "codex";
   targetName;
-  cannedResponse;
-  delayMs;
-  delayMinMs;
-  delayMaxMs;
-  constructor(targetName, config) {
-    this.id = `mock:${targetName}`;
+  supportsBatch = false;
+  config;
+  runCodex;
+  environmentCheck;
+  resolvedExecutable;
+  constructor(targetName, config, runner = defaultCodexRunner) {
+    this.id = `codex:${targetName}`;
     this.targetName = targetName;
-    this.cannedResponse = config.response ?? DEFAULT_MOCK_RESPONSE;
-    this.delayMs = config.delayMs ?? 0;
-    this.delayMinMs = config.delayMinMs ?? 0;
-    this.delayMaxMs = config.delayMaxMs ?? 0;
+    this.config = config;
+    this.runCodex = runner;
   }
   async invoke(request) {
-    const delay = this.calculateDelay();
-    if (delay > 0) {
-      await new Promise((resolve) => setTimeout(resolve, delay));
+    if (request.signal?.aborted) {
+      throw new Error("Codex provider request was aborted before execution");
     }
-    return {
-      text: this.cannedResponse,
-      raw: {
-        prompt: request.prompt,
-        guidelines: request.guidelines
+    await this.ensureEnvironmentReady();
+    const inputFiles = normalizeInputFiles2(request.inputFiles);
+    const originalGuidelines = new Set(
+      collectGuidelineFiles(inputFiles, request.guideline_patterns).map((file) => path4.resolve(file))
+    );
+    const workspaceRoot = await this.createWorkspace();
+    const logger = await this.createStreamLogger(request).catch(() => void 0);
+    try {
+      const { mirroredInputFiles, guidelineMirrors } = await this.mirrorInputFiles(
+        inputFiles,
+        workspaceRoot,
+        originalGuidelines
+      );
+      const promptContent = buildPromptDocument(request, mirroredInputFiles, {
+        guidelinePatterns: request.guideline_patterns,
+        guidelineOverrides: guidelineMirrors
+      });
+      const promptFile = path4.join(workspaceRoot, PROMPT_FILENAME);
+      await writeFile(promptFile, promptContent, "utf8");
+      const args = this.buildCodexArgs();
+      const cwd = this.resolveCwd(workspaceRoot);
+      const result = await this.executeCodex(args, cwd, promptContent, request.signal, logger);
+      if (result.timedOut) {
+        throw new Error(
+          `Codex CLI timed out${formatTimeoutSuffix2(this.config.timeoutMs ?? void 0)}`
+        );
       }
-    };
+      if (result.exitCode !== 0) {
+        const detail = pickDetail(result.stderr, result.stdout);
+        const prefix = `Codex CLI exited with code ${result.exitCode}`;
+        throw new Error(detail ? `${prefix}: ${detail}` : prefix);
+      }
+      const parsed = parseCodexJson(result.stdout);
+      const assistantText = extractAssistantText(parsed);
+      return {
+        text: assistantText,
+        raw: {
+          response: parsed,
+          stdout: result.stdout,
+          stderr: result.stderr,
+          exitCode: result.exitCode,
+          args,
+          executable: this.resolvedExecutable ?? this.config.executable,
+          promptFile,
+          workspace: workspaceRoot,
+          inputFiles: mirroredInputFiles,
+          logFile: logger?.filePath
+        }
+      };
+    } finally {
+      await logger?.close();
+      await this.cleanupWorkspace(workspaceRoot);
+    }
   }
-  calculateDelay() {
-    if (this.delayMinMs > 0 || this.delayMaxMs > 0) {
-      const min = Math.max(0, this.delayMinMs);
-      const max = Math.max(min, this.delayMaxMs);
-      return Math.floor(Math.random() * (max - min + 1)) + min;
+  async ensureEnvironmentReady() {
+    if (!this.environmentCheck) {
+      this.environmentCheck = this.validateEnvironment();
     }
-    return this.delayMs;
+    await this.environmentCheck;
   }
-};
-
-// src/evaluation/providers/targets.ts
-import { z } from "zod";
-var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID", "ATTEMPT", "FILES"]);
-var BASE_TARGET_SCHEMA = z.object({
-  name: z.string().min(1, "target name is required"),
-  provider: z.string().min(1, "provider is required"),
-  settings: z.record(z.unknown()).optional(),
-  judge_target: z.string().optional(),
-  workers: z.number().int().min(1).optional()
-});
-var DEFAULT_AZURE_API_VERSION = "2024-10-01-preview";
-function normalizeAzureApiVersion(value) {
-  if (!value) {
-    return DEFAULT_AZURE_API_VERSION;
+  async validateEnvironment() {
+    this.resolvedExecutable = await locateExecutable(this.config.executable);
   }
-  const trimmed = value.trim();
-  if (trimmed.length === 0) {
-    return DEFAULT_AZURE_API_VERSION;
+  resolveCwd(workspaceRoot) {
+    if (!this.config.cwd) {
+      return workspaceRoot;
+    }
+    return path4.resolve(this.config.cwd);
   }
-  const withoutPrefix = trimmed.replace(/^api[-_]?version\s*=\s*/i, "").trim();
-  return withoutPrefix.length > 0 ? withoutPrefix : DEFAULT_AZURE_API_VERSION;
-}
-function resolveTargetDefinition(definition, env = process.env) {
-  const parsed = BASE_TARGET_SCHEMA.parse(definition);
-  const provider = parsed.provider.toLowerCase();
-  const providerBatching = resolveOptionalBoolean(
-    parsed.settings?.provider_batching ?? parsed.settings?.providerBatching
-  );
-  switch (provider) {
-    case "azure":
-    case "azure-openai":
-      return {
-        kind: "azure",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveAzureConfig(parsed, env)
-      };
-    case "anthropic":
-      return {
-        kind: "anthropic",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveAnthropicConfig(parsed, env)
-      };
-    case "gemini":
-    case "google":
-    case "google-gemini":
-      return {
-        kind: "gemini",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveGeminiConfig(parsed, env)
-      };
-    case "codex":
-    case "codex-cli":
-      return {
-        kind: "codex",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveCodexConfig(parsed, env)
-      };
-    case "mock":
-      return {
-        kind: "mock",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveMockConfig(parsed)
-      };
-    case "vscode":
-    case "vscode-insiders":
-      return {
-        kind: provider,
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders")
-      };
-    case "cli":
+  buildCodexArgs() {
+    const args = ["--ask-for-approval", "never", "exec", "--json", "--color", "never", "--skip-git-repo-check"];
+    if (this.config.args && this.config.args.length > 0) {
+      args.push(...this.config.args);
+    }
+    args.push("-");
+    return args;
+  }
+  async executeCodex(args, cwd, promptContent, signal, logger) {
+    try {
+      return await this.runCodex({
+        executable: this.resolvedExecutable ?? this.config.executable,
+        args,
+        cwd,
+        prompt: promptContent,
+        timeoutMs: this.config.timeoutMs,
+        env: process.env,
+        signal,
+        onStdoutChunk: logger ? (chunk) => logger.handleStdoutChunk(chunk) : void 0,
+        onStderrChunk: logger ? (chunk) => logger.handleStderrChunk(chunk) : void 0
+      });
+    } catch (error) {
+      const err = error;
+      if (err.code === "ENOENT") {
+        throw new Error(
+          `Codex executable '${this.config.executable}' was not found. Update the target settings.executable or add it to PATH.`
+        );
+      }
+      throw error;
+    }
+  }
+  async mirrorInputFiles(inputFiles, workspaceRoot, guidelineOriginals) {
+    if (!inputFiles || inputFiles.length === 0) {
       return {
-        kind: "cli",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveCliConfig(parsed, env)
+        mirroredInputFiles: void 0,
+        guidelineMirrors: /* @__PURE__ */ new Set()
       };
-    default:
-      throw new Error(`Unsupported provider '${parsed.provider}' in target '${parsed.name}'`);
+    }
+    const filesRoot = path4.join(workspaceRoot, FILES_DIR);
+    await mkdir(filesRoot, { recursive: true });
+    const mirrored = [];
+    const guidelineMirrors = /* @__PURE__ */ new Set();
+    const nameCounts = /* @__PURE__ */ new Map();
+    for (const inputFile of inputFiles) {
+      const absoluteSource = path4.resolve(inputFile);
+      const baseName = path4.basename(absoluteSource);
+      const count = nameCounts.get(baseName) ?? 0;
+      nameCounts.set(baseName, count + 1);
+      const finalName = count === 0 ? baseName : `${baseName}.${count}`;
+      const destination = path4.join(filesRoot, finalName);
+      await copyFile(absoluteSource, destination);
+      const resolvedDestination = path4.resolve(destination);
+      mirrored.push(resolvedDestination);
+      if (guidelineOriginals.has(absoluteSource)) {
+        guidelineMirrors.add(resolvedDestination);
+      }
+    }
+    return {
+      mirroredInputFiles: mirrored,
+      guidelineMirrors
+    };
   }
+  async createWorkspace() {
+    return await mkdtemp(path4.join(tmpdir(), WORKSPACE_PREFIX));
+  }
+  async cleanupWorkspace(workspaceRoot) {
+    try {
+      await rm(workspaceRoot, { recursive: true, force: true });
+    } catch {
+    }
+  }
+  resolveLogDirectory() {
+    const disabled = isCodexLogStreamingDisabled();
+    if (disabled) {
+      return void 0;
+    }
+    if (this.config.logDir) {
+      return path4.resolve(this.config.logDir);
+    }
+    return path4.join(process.cwd(), ".agentv", "logs", "codex");
+  }
+  async createStreamLogger(request) {
+    const logDir = this.resolveLogDirectory();
+    if (!logDir) {
+      return void 0;
+    }
+    try {
+      await mkdir(logDir, { recursive: true });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      console.warn(`Skipping Codex stream logging (could not create ${logDir}): ${message}`);
+      return void 0;
+    }
+    const filePath = path4.join(logDir, buildLogFilename(request, this.targetName));
+    try {
+      const logger = await CodexStreamLogger.create({
+        filePath,
+        targetName: this.targetName,
+        evalCaseId: request.evalCaseId,
+        attempt: request.attempt,
+        format: this.config.logFormat ?? "summary"
+      });
+      console.log(`Streaming Codex CLI output to ${filePath}`);
+      return logger;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      console.warn(`Skipping Codex stream logging for ${filePath}: ${message}`);
+      return void 0;
+    }
+  }
+};
+var CodexStreamLogger = class _CodexStreamLogger {
+  filePath;
+  stream;
+  startedAt = Date.now();
+  stdoutBuffer = "";
+  stderrBuffer = "";
+  format;
+  constructor(filePath, format) {
+    this.filePath = filePath;
+    this.format = format;
+    this.stream = createWriteStream(filePath, { flags: "a" });
+  }
+  static async create(options) {
+    const logger = new _CodexStreamLogger(options.filePath, options.format);
+    const header = [
+      "# Codex CLI stream log",
+      `# target: ${options.targetName}`,
+      options.evalCaseId ? `# eval: ${options.evalCaseId}` : void 0,
+      options.attempt !== void 0 ? `# attempt: ${options.attempt + 1}` : void 0,
+      `# started: ${(/* @__PURE__ */ new Date()).toISOString()}`,
+      ""
+    ].filter((line) => Boolean(line));
+    logger.writeLines(header);
+    return logger;
+  }
+  handleStdoutChunk(chunk) {
+    this.stdoutBuffer += chunk;
+    this.flushBuffer("stdout");
+  }
+  handleStderrChunk(chunk) {
+    this.stderrBuffer += chunk;
+    this.flushBuffer("stderr");
+  }
+  async close() {
+    this.flushBuffer("stdout");
+    this.flushBuffer("stderr");
+    this.flushRemainder();
+    await new Promise((resolve, reject) => {
+      this.stream.once("error", reject);
+      this.stream.end(() => resolve());
+    });
+  }
+  writeLines(lines) {
+    for (const line of lines) {
+      this.stream.write(`${line}
+`);
+    }
+  }
+  flushBuffer(source) {
+    const buffer = source === "stdout" ? this.stdoutBuffer : this.stderrBuffer;
+    const lines = buffer.split(/\r?\n/);
+    const remainder = lines.pop() ?? "";
+    if (source === "stdout") {
+      this.stdoutBuffer = remainder;
+    } else {
+      this.stderrBuffer = remainder;
+    }
+    for (const line of lines) {
+      const formatted = this.formatLine(line, source);
+      if (formatted) {
+        this.stream.write(formatted);
+        this.stream.write("\n");
+      }
+    }
+  }
+  formatLine(rawLine, source) {
+    const trimmed = rawLine.trim();
+    if (trimmed.length === 0) {
+      return void 0;
+    }
+    const message = this.format === "json" ? formatCodexJsonLog(trimmed) : formatCodexLogMessage(trimmed, source);
+    return `[+${formatElapsed(this.startedAt)}] [${source}] ${message}`;
+  }
+  flushRemainder() {
+    const stdoutRemainder = this.stdoutBuffer.trim();
+    if (stdoutRemainder.length > 0) {
+      const formatted = this.formatLine(stdoutRemainder, "stdout");
+      if (formatted) {
+        this.stream.write(formatted);
+        this.stream.write("\n");
+      }
+    }
+    const stderrRemainder = this.stderrBuffer.trim();
+    if (stderrRemainder.length > 0) {
+      const formatted = this.formatLine(stderrRemainder, "stderr");
+      if (formatted) {
+        this.stream.write(formatted);
+        this.stream.write("\n");
+      }
+    }
+    this.stdoutBuffer = "";
+    this.stderrBuffer = "";
+  }
+};
+function isCodexLogStreamingDisabled() {
+  const envValue = process.env.AGENTV_CODEX_STREAM_LOGS;
+  if (!envValue) {
+    return false;
+  }
+  const normalized = envValue.trim().toLowerCase();
+  return normalized === "false" || normalized === "0" || normalized === "off";
 }
-function resolveAzureConfig(target, env) {
-  const settings = target.settings ?? {};
-  const endpointSource = settings.endpoint ?? settings.resource ?? settings.resourceName;
-  const apiKeySource = settings.api_key ?? settings.apiKey;
-  const deploymentSource = settings.deployment ?? settings.deploymentName ?? settings.model;
-  const versionSource = settings.version ?? settings.api_version;
-  const temperatureSource = settings.temperature;
-  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
-  const resourceName = resolveString(endpointSource, env, `${target.name} endpoint`);
-  const apiKey = resolveString(apiKeySource, env, `${target.name} api key`);
-  const deploymentName = resolveString(deploymentSource, env, `${target.name} deployment`);
-  const version = normalizeAzureApiVersion(
-    resolveOptionalString(versionSource, env, `${target.name} api version`)
-  );
-  const temperature = resolveOptionalNumber(temperatureSource, `${target.name} temperature`);
-  const maxOutputTokens = resolveOptionalNumber(
-    maxTokensSource,
-    `${target.name} max output tokens`
-  );
-  return {
-    resourceName,
-    deploymentName,
-    apiKey,
-    version,
-    temperature,
-    maxOutputTokens
-  };
-}
-function resolveAnthropicConfig(target, env) {
-  const settings = target.settings ?? {};
-  const apiKeySource = settings.api_key ?? settings.apiKey;
-  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
-  const temperatureSource = settings.temperature;
-  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
-  const thinkingBudgetSource = settings.thinking_budget ?? settings.thinkingBudget;
-  const apiKey = resolveString(apiKeySource, env, `${target.name} Anthropic api key`);
-  const model = resolveString(modelSource, env, `${target.name} Anthropic model`);
-  return {
-    apiKey,
-    model,
-    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
-    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`),
-    thinkingBudget: resolveOptionalNumber(thinkingBudgetSource, `${target.name} thinking budget`)
-  };
-}
-function resolveGeminiConfig(target, env) {
-  const settings = target.settings ?? {};
-  const apiKeySource = settings.api_key ?? settings.apiKey;
-  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
-  const temperatureSource = settings.temperature;
-  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
-  const apiKey = resolveString(apiKeySource, env, `${target.name} Google API key`);
-  const model = resolveOptionalString(modelSource, env, `${target.name} Gemini model`, {
-    allowLiteral: true,
-    optionalEnv: true
-  }) ?? "gemini-2.5-flash";
-  return {
-    apiKey,
-    model,
-    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
-    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`)
-  };
-}
-function resolveCodexConfig(target, env) {
-  const settings = target.settings ?? {};
-  const executableSource = settings.executable ?? settings.command ?? settings.binary;
-  const argsSource = settings.args ?? settings.arguments;
-  const cwdSource = settings.cwd;
-  const timeoutSource = settings.timeout_seconds ?? settings.timeoutSeconds;
-  const executable = resolveOptionalString(executableSource, env, `${target.name} codex executable`, {
-    allowLiteral: true,
-    optionalEnv: true
-  }) ?? "codex";
-  const args = resolveOptionalStringArray(argsSource, env, `${target.name} codex args`);
-  const cwd = resolveOptionalString(cwdSource, env, `${target.name} codex cwd`, {
-    allowLiteral: true,
-    optionalEnv: true
-  });
-  const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} codex timeout`);
-  return {
-    executable,
-    args,
-    cwd,
-    timeoutMs
-  };
-}
-function resolveMockConfig(target) {
-  const settings = target.settings ?? {};
-  const response = typeof settings.response === "string" ? settings.response : void 0;
-  return { response };
-}
-function resolveVSCodeConfig(target, env, insiders) {
-  const settings = target.settings ?? {};
-  const workspaceTemplateEnvVar = resolveOptionalLiteralString(settings.workspace_template ?? settings.workspaceTemplate);
-  const workspaceTemplate = workspaceTemplateEnvVar ? resolveOptionalString(workspaceTemplateEnvVar, env, `${target.name} workspace template path`, {
-    allowLiteral: false,
-    optionalEnv: true
-  }) : void 0;
-  const commandSource = settings.vscode_cmd ?? settings.command;
-  const waitSource = settings.wait;
-  const dryRunSource = settings.dry_run ?? settings.dryRun;
-  const subagentRootSource = settings.subagent_root ?? settings.subagentRoot;
-  const defaultCommand = insiders ? "code-insiders" : "code";
-  const command = resolveOptionalLiteralString(commandSource) ?? defaultCommand;
-  return {
-    command,
-    waitForResponse: resolveOptionalBoolean(waitSource) ?? true,
-    dryRun: resolveOptionalBoolean(dryRunSource) ?? false,
-    subagentRoot: resolveOptionalString(subagentRootSource, env, `${target.name} subagent root`, {
-      allowLiteral: true,
-      optionalEnv: true
-    }),
-    workspaceTemplate
-  };
-}
-function resolveCliConfig(target, env) {
-  const settings = target.settings ?? {};
-  const commandTemplateSource = settings.command_template ?? settings.commandTemplate;
-  const filesFormat = resolveOptionalLiteralString(
-    settings.files_format ?? settings.filesFormat ?? settings.attachments_format ?? settings.attachmentsFormat
-  );
-  const cwd = resolveOptionalString(settings.cwd, env, `${target.name} working directory`, {
-    allowLiteral: true,
-    optionalEnv: true
-  });
-  const envOverrides = resolveEnvOverrides(settings.env, env, target.name);
-  const timeoutMs = resolveTimeoutMs(settings.timeout_seconds ?? settings.timeoutSeconds, `${target.name} timeout`);
-  const healthcheck = resolveCliHealthcheck(settings.healthcheck, env, target.name);
-  const commandTemplate = resolveString(
-    commandTemplateSource,
-    env,
-    `${target.name} CLI command template`,
-    true
-  );
-  assertSupportedCliPlaceholders(commandTemplate, `${target.name} CLI command template`);
-  return {
-    commandTemplate,
-    filesFormat,
-    cwd,
-    env: envOverrides,
-    timeoutMs,
-    healthcheck
-  };
+function buildLogFilename(request, targetName) {
+  const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+  const evalId = sanitizeForFilename(request.evalCaseId ?? "codex");
+  const attemptSuffix = request.attempt !== void 0 ? `_attempt-${request.attempt + 1}` : "";
+  const target = sanitizeForFilename(targetName);
+  return `${timestamp}_${target}_${evalId}${attemptSuffix}_${randomUUID().slice(0, 8)}.log`;
 }
-function resolveEnvOverrides(source, env, targetName) {
-  if (source === void 0 || source === null) {
+function sanitizeForFilename(value) {
+  const sanitized = value.replace(/[^A-Za-z0-9._-]+/g, "_");
+  return sanitized.length > 0 ? sanitized : "codex";
+}
+function formatElapsed(startedAt) {
+  const elapsedSeconds = Math.floor((Date.now() - startedAt) / 1e3);
+  const hours = Math.floor(elapsedSeconds / 3600);
+  const minutes = Math.floor(elapsedSeconds % 3600 / 60);
+  const seconds = elapsedSeconds % 60;
+  if (hours > 0) {
+    return `${hours.toString().padStart(2, "0")}:${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`;
+  }
+  return `${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`;
+}
+function formatCodexLogMessage(rawLine, source) {
+  const parsed = tryParseJsonValue(rawLine);
+  if (parsed) {
+    const summary = summarizeCodexEvent(parsed);
+    if (summary) {
+      return summary;
+    }
+  }
+  if (source === "stderr") {
+    return `stderr: ${rawLine}`;
+  }
+  return rawLine;
+}
+function formatCodexJsonLog(rawLine) {
+  const parsed = tryParseJsonValue(rawLine);
+  if (!parsed) {
+    return rawLine;
+  }
+  try {
+    return JSON.stringify(parsed, null, 2);
+  } catch {
+    return rawLine;
+  }
+}
+function summarizeCodexEvent(event) {
+  if (!event || typeof event !== "object") {
     return void 0;
   }
-  if (typeof source !== "object" || Array.isArray(source)) {
-    throw new Error(`${targetName} env overrides must be an object map of strings`);
+  const record = event;
+  const type = typeof record.type === "string" ? record.type : void 0;
+  let message = extractFromEvent(event) ?? extractFromItem(record.item) ?? flattenContent(record.output ?? record.content);
+  if (!message && type === JSONL_TYPE_ITEM_COMPLETED) {
+    const item = record.item;
+    if (item && typeof item === "object") {
+      const candidate = flattenContent(
+        item.text ?? item.content ?? item.output
+      );
+      if (candidate) {
+        message = candidate;
+      }
+    }
   }
-  const entries = Object.entries(source);
-  const resolved = {};
-  for (const [key, value] of entries) {
-    if (typeof value !== "string") {
-      throw new Error(`${targetName} env override '${key}' must be a string`);
+  if (!message) {
+    const itemType = typeof record.item?.type === "string" ? record.item.type : void 0;
+    if (type && itemType) {
+      return `${type}:${itemType}`;
+    }
+    if (type) {
+      return type;
     }
-    const resolvedValue = resolveString(value, env, `${targetName} env override '${key}'`);
-    resolved[key] = resolvedValue;
   }
-  return Object.keys(resolved).length > 0 ? resolved : void 0;
+  if (type && message) {
+    return `${type}: ${message}`;
+  }
+  if (message) {
+    return message;
+  }
+  return type;
 }
-function resolveTimeoutMs(source, description) {
-  const seconds = resolveOptionalNumber(source, `${description} (seconds)`);
-  if (seconds === void 0) {
+function tryParseJsonValue(rawLine) {
+  try {
+    return JSON.parse(rawLine);
+  } catch {
     return void 0;
   }
-  if (seconds <= 0) {
-    throw new Error(`${description} must be greater than zero seconds`);
+}
+async function locateExecutable(candidate) {
+  const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
+  if (includesPathSeparator) {
+    const resolved = path4.isAbsolute(candidate) ? candidate : path4.resolve(candidate);
+    const executablePath = await ensureWindowsExecutableVariant(resolved);
+    await access2(executablePath, constants2.F_OK);
+    return executablePath;
   }
-  return Math.floor(seconds * 1e3);
+  const locator = process.platform === "win32" ? "where" : "which";
+  try {
+    const { stdout } = await execAsync2(`${locator} ${candidate}`);
+    const lines = stdout.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
+    const preferred = selectExecutableCandidate(lines);
+    if (preferred) {
+      const executablePath = await ensureWindowsExecutableVariant(preferred);
+      await access2(executablePath, constants2.F_OK);
+      return executablePath;
+    }
+  } catch {
+  }
+  throw new Error(`Codex executable '${candidate}' was not found on PATH`);
 }
-function resolveCliHealthcheck(source, env, targetName) {
-  if (source === void 0 || source === null) {
+function selectExecutableCandidate(candidates) {
+  if (candidates.length === 0) {
     return void 0;
   }
-  if (typeof source !== "object" || Array.isArray(source)) {
-    throw new Error(`${targetName} healthcheck must be an object`);
+  if (process.platform !== "win32") {
+    return candidates[0];
   }
-  const candidate = source;
-  const type = candidate.type;
-  const timeoutMs = resolveTimeoutMs(
-    candidate.timeout_seconds ?? candidate.timeoutSeconds,
-    `${targetName} healthcheck timeout`
-  );
-  if (type === "http") {
-    const url = resolveString(candidate.url, env, `${targetName} healthcheck URL`);
-    return {
-      type: "http",
-      url,
-      timeoutMs
-    };
+  const extensions = getWindowsExecutableExtensions();
+  for (const ext of extensions) {
+    const match = candidates.find((candidate) => candidate.toLowerCase().endsWith(ext));
+    if (match) {
+      return match;
+    }
   }
-  if (type === "command") {
-    const commandTemplate = resolveString(
-      candidate.command_template ?? candidate.commandTemplate,
-      env,
-      `${targetName} healthcheck command template`,
-      true
-    );
-    assertSupportedCliPlaceholders(commandTemplate, `${targetName} healthcheck command template`);
-    const cwd = resolveOptionalString(candidate.cwd, env, `${targetName} healthcheck cwd`, {
-      allowLiteral: true,
-      optionalEnv: true
-    });
-    return {
-      type: "command",
-      commandTemplate,
-      timeoutMs,
-      cwd
-    };
+  return candidates[0];
+}
+async function ensureWindowsExecutableVariant(candidate) {
+  if (process.platform !== "win32") {
+    return candidate;
   }
-  throw new Error(`${targetName} healthcheck type must be 'http' or 'command'`);
+  if (hasExecutableExtension(candidate)) {
+    return candidate;
+  }
+  const extensions = getWindowsExecutableExtensions();
+  for (const ext of extensions) {
+    const withExtension = `${candidate}${ext}`;
+    try {
+      await access2(withExtension, constants2.F_OK);
+      return withExtension;
+    } catch {
+    }
+  }
+  return candidate;
 }
-function assertSupportedCliPlaceholders(template, description) {
-  const placeholders = extractCliPlaceholders(template);
-  for (const placeholder of placeholders) {
-    if (!CLI_PLACEHOLDERS.has(placeholder)) {
-      throw new Error(
-        `${description} includes unsupported placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS).join(", ")}`
-      );
+function hasExecutableExtension(candidate) {
+  const lower = candidate.toLowerCase();
+  return getWindowsExecutableExtensions().some((ext) => lower.endsWith(ext));
+}
+var DEFAULT_WINDOWS_EXTENSIONS = [".com", ".exe", ".bat", ".cmd", ".ps1"];
+function getWindowsExecutableExtensions() {
+  if (process.platform !== "win32") {
+    return [];
+  }
+  const fromEnv = process.env.PATHEXT?.split(";").map((ext) => ext.trim().toLowerCase()).filter((ext) => ext.length > 0);
+  return fromEnv && fromEnv.length > 0 ? fromEnv : DEFAULT_WINDOWS_EXTENSIONS;
+}
+function parseCodexJson(output) {
+  const trimmed = output.trim();
+  if (trimmed.length === 0) {
+    throw new Error("Codex CLI produced no output in --json mode");
+  }
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    const lineObjects = parseJsonLines(trimmed);
+    if (lineObjects) {
+      return lineObjects;
+    }
+    const lastBrace = trimmed.lastIndexOf("{");
+    if (lastBrace >= 0) {
+      const candidate = trimmed.slice(lastBrace);
+      try {
+        return JSON.parse(candidate);
+      } catch {
+      }
     }
+    const preview = trimmed.slice(0, 200);
+    throw new Error(`Codex CLI emitted invalid JSON: ${preview}${trimmed.length > 200 ? "\u2026" : ""}`);
   }
 }
-function extractCliPlaceholders(template) {
-  const matches = template.matchAll(/\{([A-Z_]+)\}/g);
-  const results = [];
-  for (const match of matches) {
-    if (match[1]) {
-      results.push(match[1]);
+function extractAssistantText(parsed) {
+  if (Array.isArray(parsed)) {
+    const text = extractFromEventStream(parsed);
+    if (text) {
+      return text;
+    }
+  }
+  if (!parsed || typeof parsed !== "object") {
+    throw new Error("Codex CLI JSON response did not include an assistant message");
+  }
+  const record = parsed;
+  const eventText = extractFromEvent(record);
+  if (eventText) {
+    return eventText;
+  }
+  const messages = Array.isArray(record.messages) ? record.messages : void 0;
+  if (messages) {
+    for (let index = messages.length - 1; index >= 0; index -= 1) {
+      const entry = messages[index];
+      if (!entry || typeof entry !== "object") {
+        continue;
+      }
+      const role = entry.role;
+      if (role !== "assistant") {
+        continue;
+      }
+      const content = entry.content;
+      const flattened = flattenContent(content);
+      if (flattened) {
+        return flattened;
+      }
+    }
+  }
+  const response = record.response;
+  if (response && typeof response === "object") {
+    const content = response.content;
+    const flattened = flattenContent(content);
+    if (flattened) {
+      return flattened;
     }
   }
-  return results;
+  const output = record.output;
+  const flattenedOutput = flattenContent(output);
+  if (flattenedOutput) {
+    return flattenedOutput;
+  }
+  throw new Error("Codex CLI JSON response did not include an assistant message");
 }
-function resolveString(source, env, description, allowLiteral = false) {
-  const value = resolveOptionalString(source, env, description, {
-    allowLiteral,
-    optionalEnv: false
-  });
-  if (value === void 0) {
-    throw new Error(`${description} is required`);
+function extractFromEventStream(events) {
+  for (let index = events.length - 1; index >= 0; index -= 1) {
+    const candidate = events[index];
+    const text = extractFromEvent(candidate);
+    if (text) {
+      return text;
+    }
   }
-  return value;
+  return void 0;
 }
-function resolveOptionalString(source, env, description, options) {
-  if (source === void 0 || source === null) {
-    return void 0;
-  }
-  if (typeof source !== "string") {
-    throw new Error(`${description} must be a string`);
-  }
-  const trimmed = source.trim();
-  if (trimmed.length === 0) {
+function extractFromEvent(event) {
+  if (!event || typeof event !== "object") {
     return void 0;
   }
-  const envValue = env[trimmed];
-  if (envValue !== void 0) {
-    if (envValue.trim().length === 0) {
-      throw new Error(`Environment variable '${trimmed}' for ${description} is empty`);
+  const record = event;
+  const type = typeof record.type === "string" ? record.type : void 0;
+  if (type === JSONL_TYPE_ITEM_COMPLETED) {
+    const item = record.item;
+    const text = extractFromItem(item);
+    if (text) {
+      return text;
     }
-    return envValue;
   }
-  const allowLiteral = options?.allowLiteral ?? false;
-  const optionalEnv = options?.optionalEnv ?? false;
-  const looksLikeEnv = isLikelyEnvReference(trimmed);
-  if (looksLikeEnv) {
-    if (optionalEnv) {
-      return void 0;
-    }
-    if (!allowLiteral) {
-      throw new Error(`Environment variable '${trimmed}' required for ${description} is not set`);
-    }
+  const output = record.output ?? record.content;
+  const flattened = flattenContent(output);
+  if (flattened) {
+    return flattened;
   }
-  return trimmed;
+  return void 0;
 }
-function resolveOptionalLiteralString(source) {
-  if (source === void 0 || source === null) {
+function extractFromItem(item) {
+  if (!item || typeof item !== "object") {
     return void 0;
   }
-  if (typeof source !== "string") {
-    throw new Error("expected string value");
+  const record = item;
+  const itemType = typeof record.type === "string" ? record.type : void 0;
+  if (itemType === "agent_message" || itemType === "response" || itemType === "output") {
+    const text = flattenContent(record.text ?? record.content ?? record.output);
+    if (text) {
+      return text;
+    }
   }
-  const trimmed = source.trim();
-  return trimmed.length > 0 ? trimmed : void 0;
+  return void 0;
 }
-function resolveOptionalNumber(source, description) {
-  if (source === void 0 || source === null || source === "") {
-    return void 0;
+function flattenContent(value) {
+  if (typeof value === "string") {
+    return value;
   }
-  if (typeof source === "number") {
-    return Number.isFinite(source) ? source : void 0;
+  if (Array.isArray(value)) {
+    const parts = value.map((segment) => {
+      if (typeof segment === "string") {
+        return segment;
+      }
+      if (segment && typeof segment === "object" && "text" in segment) {
+        const text = segment.text;
+        return typeof text === "string" ? text : void 0;
+      }
+      return void 0;
+    }).filter((part) => typeof part === "string" && part.length > 0);
+    return parts.length > 0 ? parts.join(" \n") : void 0;
   }
-  if (typeof source === "string") {
-    const numeric = Number(source);
-    if (Number.isFinite(numeric)) {
-      return numeric;
-    }
+  if (value && typeof value === "object" && "text" in value) {
+    const text = value.text;
+    return typeof text === "string" ? text : void 0;
   }
-  throw new Error(`${description} must be a number`);
+  return void 0;
 }
-function resolveOptionalBoolean(source) {
-  if (source === void 0 || source === null || source === "") {
+function parseJsonLines(output) {
+  const lines = output.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
+  if (lines.length <= 1) {
     return void 0;
   }
-  if (typeof source === "boolean") {
-    return source;
-  }
-  if (typeof source === "string") {
-    const lowered = source.trim().toLowerCase();
-    if (lowered === "true" || lowered === "1") {
-      return true;
-    }
-    if (lowered === "false" || lowered === "0") {
-      return false;
+  const parsed = [];
+  for (const line of lines) {
+    try {
+      parsed.push(JSON.parse(line));
+    } catch {
+      return void 0;
     }
   }
-  throw new Error("expected boolean value");
-}
-function isLikelyEnvReference(value) {
-  return /^[A-Z0-9_]+$/.test(value);
+  return parsed;
 }
-function resolveOptionalStringArray(source, env, description) {
-  if (source === void 0 || source === null) {
-    return void 0;
-  }
-  if (!Array.isArray(source)) {
-    throw new Error(`${description} must be an array of strings`);
+function pickDetail(stderr, stdout) {
+  const errorText = stderr.trim();
+  if (errorText.length > 0) {
+    return errorText;
   }
-  if (source.length === 0) {
-    return void 0;
+  const stdoutText = stdout.trim();
+  return stdoutText.length > 0 ? stdoutText : void 0;
+}
+function formatTimeoutSuffix2(timeoutMs) {
+  if (!timeoutMs || timeoutMs <= 0) {
+    return "";
   }
-  const resolved = [];
-  for (let i = 0; i < source.length; i++) {
-    const item = source[i];
-    if (typeof item !== "string") {
-      throw new Error(`${description}[${i}] must be a string`);
+  const seconds = Math.ceil(timeoutMs / 1e3);
+  return ` after ${seconds}s`;
+}
+async function defaultCodexRunner(options) {
+  return await new Promise((resolve, reject) => {
+    const child = spawn(options.executable, options.args, {
+      cwd: options.cwd,
+      env: options.env,
+      stdio: ["pipe", "pipe", "pipe"],
+      shell: shouldShellExecute(options.executable)
+    });
+    let stdout = "";
+    let stderr = "";
+    let timedOut = false;
+    const onAbort = () => {
+      child.kill("SIGTERM");
+    };
+    if (options.signal) {
+      if (options.signal.aborted) {
+        onAbort();
+      } else {
+        options.signal.addEventListener("abort", onAbort, { once: true });
+      }
     }
-    const trimmed = item.trim();
-    if (trimmed.length === 0) {
-      throw new Error(`${description}[${i}] cannot be empty`);
+    let timeoutHandle;
+    if (options.timeoutMs && options.timeoutMs > 0) {
+      timeoutHandle = setTimeout(() => {
+        timedOut = true;
+        child.kill("SIGTERM");
+      }, options.timeoutMs);
+      timeoutHandle.unref?.();
     }
-    const envValue = env[trimmed];
-    if (envValue !== void 0) {
-      if (envValue.trim().length === 0) {
-        throw new Error(`Environment variable '${trimmed}' for ${description}[${i}] is empty`);
+    child.stdout.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+      options.onStdoutChunk?.(chunk);
+    });
+    child.stderr.setEncoding("utf8");
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+      options.onStderrChunk?.(chunk);
+    });
+    child.stdin.end(options.prompt);
+    const cleanup = () => {
+      if (timeoutHandle) {
+        clearTimeout(timeoutHandle);
       }
-      resolved.push(envValue);
-    } else {
-      resolved.push(trimmed);
-    }
+      if (options.signal) {
+        options.signal.removeEventListener("abort", onAbort);
+      }
+    };
+    child.on("error", (error) => {
+      cleanup();
+      reject(error);
+    });
+    child.on("close", (code) => {
+      cleanup();
+      resolve({
+        stdout,
+        stderr,
+        exitCode: typeof code === "number" ? code : -1,
+        timedOut
+      });
+    });
+  });
+}
+function shouldShellExecute(executable) {
+  if (process.platform !== "win32") {
+    return false;
   }
-  return resolved.length > 0 ? resolved : void 0;
+  const lower = executable.toLowerCase();
+  return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
 }
 
-// src/evaluation/providers/vscode.ts
-import { readFile as readFile2 } from "node:fs/promises";
-import path3 from "node:path";
-import { dispatchAgentSession, dispatchBatchAgent, getSubagentRoot, provisionSubagents } from "subagent";
-var VSCodeProvider = class {
+// src/evaluation/providers/mock.ts
+var DEFAULT_MOCK_RESPONSE = '{"answer":"Mock provider response. Configure targets.yaml to supply a custom value."}';
+var MockProvider = class {
   id;
-  kind;
+  kind = "mock";
   targetName;
-  supportsBatch = true;
-  config;
-  constructor(targetName, config, kind) {
-    this.id = `${kind}:${targetName}`;
-    this.kind = kind;
+  cannedResponse;
+  delayMs;
+  delayMinMs;
+  delayMaxMs;
+  constructor(targetName, config) {
+    this.id = `mock:${targetName}`;
     this.targetName = targetName;
-    this.config = config;
+    this.cannedResponse = config.response ?? DEFAULT_MOCK_RESPONSE;
+    this.delayMs = config.delayMs ?? 0;
+    this.delayMinMs = config.delayMinMs ?? 0;
+    this.delayMaxMs = config.delayMaxMs ?? 0;
   }
   async invoke(request) {
-    if (request.signal?.aborted) {
-      throw new Error("VS Code provider request was aborted before dispatch");
-    }
-    const inputFiles = normalizeAttachments(request.inputFiles);
-    const promptContent = buildPromptDocument(request, inputFiles, request.guideline_patterns);
-    const session = await dispatchAgentSession({
-      userQuery: promptContent,
-      extraAttachments: inputFiles,
-      wait: this.config.waitForResponse,
-      dryRun: this.config.dryRun,
-      vscodeCmd: this.config.command,
-      subagentRoot: this.config.subagentRoot,
-      workspaceTemplate: this.config.workspaceTemplate,
-      silent: true
-    });
-    if (session.exitCode !== 0 || !session.responseFile) {
-      const failure = session.error ?? "VS Code subagent did not produce a response";
-      throw new Error(failure);
-    }
-    if (this.config.dryRun) {
-      return {
-        text: "",
-        raw: {
-          session,
-          inputFiles
-        }
-      };
+    const delay = this.calculateDelay();
+    if (delay > 0) {
+      await new Promise((resolve) => setTimeout(resolve, delay));
     }
-    const responseText = await readFile2(session.responseFile, "utf8");
     return {
-      text: responseText,
+      text: this.cannedResponse,
       raw: {
-        session,
-        inputFiles
+        prompt: request.prompt,
+        guidelines: request.guidelines
       }
     };
   }
-  async invokeBatch(requests) {
-    if (requests.length === 0) {
-      return [];
-    }
-    const normalizedRequests = requests.map((req) => ({
-      request: req,
-      inputFiles: normalizeAttachments(req.inputFiles)
-    }));
-    const combinedInputFiles = mergeAttachments(
-      normalizedRequests.map(({ inputFiles }) => inputFiles)
-    );
-    const userQueries = normalizedRequests.map(
-      ({ request, inputFiles }) => buildPromptDocument(request, inputFiles, request.guideline_patterns)
-    );
-    const session = await dispatchBatchAgent({
-      userQueries,
-      extraAttachments: combinedInputFiles,
-      wait: this.config.waitForResponse,
-      dryRun: this.config.dryRun,
-      vscodeCmd: this.config.command,
-      subagentRoot: this.config.subagentRoot,
-      workspaceTemplate: this.config.workspaceTemplate,
-      silent: true
-    });
-    if (session.exitCode !== 0 || !session.responseFiles) {
-      const failure = session.error ?? "VS Code subagent did not produce batch responses";
-      throw new Error(failure);
-    }
-    if (this.config.dryRun) {
-      return normalizedRequests.map(({ inputFiles }) => ({
-        text: "",
-        raw: {
-          session,
-          inputFiles,
-          allInputFiles: combinedInputFiles
-        }
-      }));
-    }
-    if (session.responseFiles.length !== requests.length) {
-      throw new Error(
-        `VS Code batch returned ${session.responseFiles.length} responses for ${requests.length} requests`
-      );
-    }
-    const responses = [];
-    for (const [index, responseFile] of session.responseFiles.entries()) {
-      const responseText = await readFile2(responseFile, "utf8");
-      responses.push({
-        text: responseText,
-        raw: {
-          session,
-          inputFiles: normalizedRequests[index]?.inputFiles,
-          allInputFiles: combinedInputFiles,
-          responseFile
-        }
-      });
+  calculateDelay() {
+    if (this.delayMinMs > 0 || this.delayMaxMs > 0) {
+      const min = Math.max(0, this.delayMinMs);
+      const max = Math.max(min, this.delayMaxMs);
+      return Math.floor(Math.random() * (max - min + 1)) + min;
     }
-    return responses;
+    return this.delayMs;
   }
 };
-function buildPromptDocument(request, attachments, guidelinePatterns) {
-  const parts = [];
-  const guidelineFiles = collectGuidelineFiles(attachments, guidelinePatterns);
-  const attachmentFiles = collectAttachmentFiles(attachments);
-  const nonGuidelineAttachments = attachmentFiles.filter(
-    (file) => !guidelineFiles.includes(file)
-  );
-  const prereadBlock = buildMandatoryPrereadBlock(guidelineFiles, nonGuidelineAttachments);
-  if (prereadBlock.length > 0) {
-    parts.push("\n", prereadBlock);
-  }
-  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
-  return parts.join("\n").trim();
-}
-function buildMandatoryPrereadBlock(guidelineFiles, attachmentFiles) {
-  if (guidelineFiles.length === 0 && attachmentFiles.length === 0) {
-    return "";
-  }
-  const buildList = (files) => files.map((absolutePath) => {
-    const fileName = path3.basename(absolutePath);
-    const fileUri = pathToFileUri(absolutePath);
-    return `* [${fileName}](${fileUri})`;
-  });
-  const sections = [];
-  if (guidelineFiles.length > 0) {
-    sections.push(`Read all guideline files:
-${buildList(guidelineFiles).join("\n")}.`);
+
+// src/evaluation/providers/targets.ts
+import { z } from "zod";
+var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID", "ATTEMPT", "FILES"]);
+var BASE_TARGET_SCHEMA = z.object({
+  name: z.string().min(1, "target name is required"),
+  provider: z.string().min(1, "provider is required"),
+  settings: z.record(z.unknown()).optional(),
+  judge_target: z.string().optional(),
+  workers: z.number().int().min(1).optional()
+});
+var DEFAULT_AZURE_API_VERSION = "2024-10-01-preview";
+function normalizeAzureApiVersion(value) {
+  if (!value) {
+    return DEFAULT_AZURE_API_VERSION;
   }
-  if (attachmentFiles.length > 0) {
-    sections.push(`Read all attachment files:
-${buildList(attachmentFiles).join("\n")}.`);
+  const trimmed = value.trim();
+  if (trimmed.length === 0) {
+    return DEFAULT_AZURE_API_VERSION;
   }
-  sections.push(
-    "If any file is missing, fail with ERROR: missing-file <filename> and stop.",
-    "Then apply system_instructions on the user query below."
-  );
-  return sections.join("\n");
+  const withoutPrefix = trimmed.replace(/^api[-_]?version\s*=\s*/i, "").trim();
+  return withoutPrefix.length > 0 ? withoutPrefix : DEFAULT_AZURE_API_VERSION;
 }
-function collectGuidelineFiles(attachments, guidelinePatterns) {
-  if (!attachments || attachments.length === 0) {
-    return [];
-  }
-  const unique = /* @__PURE__ */ new Map();
-  for (const attachment of attachments) {
-    const absolutePath = path3.resolve(attachment);
-    const normalized = absolutePath.split(path3.sep).join("/");
-    if (isGuidelineFile(normalized, guidelinePatterns)) {
-      if (!unique.has(absolutePath)) {
-        unique.set(absolutePath, absolutePath);
-      }
-    }
+function resolveTargetDefinition(definition, env = process.env) {
+  const parsed = BASE_TARGET_SCHEMA.parse(definition);
+  const provider = parsed.provider.toLowerCase();
+  const providerBatching = resolveOptionalBoolean(
+    parsed.settings?.provider_batching ?? parsed.settings?.providerBatching
+  );
+  switch (provider) {
+    case "azure":
+    case "azure-openai":
+      return {
+        kind: "azure",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveAzureConfig(parsed, env)
+      };
+    case "anthropic":
+      return {
+        kind: "anthropic",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveAnthropicConfig(parsed, env)
+      };
+    case "gemini":
+    case "google":
+    case "google-gemini":
+      return {
+        kind: "gemini",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveGeminiConfig(parsed, env)
+      };
+    case "codex":
+    case "codex-cli":
+      return {
+        kind: "codex",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveCodexConfig(parsed, env)
+      };
+    case "mock":
+      return {
+        kind: "mock",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveMockConfig(parsed)
+      };
+    case "vscode":
+    case "vscode-insiders":
+      return {
+        kind: provider,
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders")
+      };
+    case "cli":
+      return {
+        kind: "cli",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveCliConfig(parsed, env)
+      };
+    default:
+      throw new Error(`Unsupported provider '${parsed.provider}' in target '${parsed.name}'`);
   }
-  return Array.from(unique.values());
 }
-function collectAttachmentFiles(attachments) {
-  if (!attachments || attachments.length === 0) {
-    return [];
-  }
-  const unique = /* @__PURE__ */ new Map();
-  for (const attachment of attachments) {
-    const absolutePath = path3.resolve(attachment);
-    if (!unique.has(absolutePath)) {
-      unique.set(absolutePath, absolutePath);
-    }
-  }
-  return Array.from(unique.values());
+function resolveAzureConfig(target, env) {
+  const settings = target.settings ?? {};
+  const endpointSource = settings.endpoint ?? settings.resource ?? settings.resourceName;
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const deploymentSource = settings.deployment ?? settings.deploymentName ?? settings.model;
+  const versionSource = settings.version ?? settings.api_version;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const resourceName = resolveString(endpointSource, env, `${target.name} endpoint`);
+  const apiKey = resolveString(apiKeySource, env, `${target.name} api key`);
+  const deploymentName = resolveString(deploymentSource, env, `${target.name} deployment`);
+  const version = normalizeAzureApiVersion(
+    resolveOptionalString(versionSource, env, `${target.name} api version`)
+  );
+  const temperature = resolveOptionalNumber(temperatureSource, `${target.name} temperature`);
+  const maxOutputTokens = resolveOptionalNumber(
+    maxTokensSource,
+    `${target.name} max output tokens`
+  );
+  return {
+    resourceName,
+    deploymentName,
+    apiKey,
+    version,
+    temperature,
+    maxOutputTokens
+  };
+}
+function resolveAnthropicConfig(target, env) {
+  const settings = target.settings ?? {};
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const thinkingBudgetSource = settings.thinking_budget ?? settings.thinkingBudget;
+  const apiKey = resolveString(apiKeySource, env, `${target.name} Anthropic api key`);
+  const model = resolveString(modelSource, env, `${target.name} Anthropic model`);
+  return {
+    apiKey,
+    model,
+    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
+    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`),
+    thinkingBudget: resolveOptionalNumber(thinkingBudgetSource, `${target.name} thinking budget`)
+  };
 }
-function pathToFileUri(filePath) {
-  const absolutePath = path3.isAbsolute(filePath) ? filePath : path3.resolve(filePath);
-  const normalizedPath = absolutePath.replace(/\\/g, "/");
-  if (/^[a-zA-Z]:\//.test(normalizedPath)) {
-    return `file:///${normalizedPath}`;
-  }
-  return `file://${normalizedPath}`;
+function resolveGeminiConfig(target, env) {
+  const settings = target.settings ?? {};
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const apiKey = resolveString(apiKeySource, env, `${target.name} Google API key`);
+  const model = resolveOptionalString(modelSource, env, `${target.name} Gemini model`, {
+    allowLiteral: true,
+    optionalEnv: true
+  }) ?? "gemini-2.5-flash";
+  return {
+    apiKey,
+    model,
+    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
+    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`)
+  };
 }
-function normalizeAttachments(attachments) {
-  if (!attachments || attachments.length === 0) {
+function resolveCodexConfig(target, env) {
+  const settings = target.settings ?? {};
+  const executableSource = settings.executable ?? settings.command ?? settings.binary;
+  const argsSource = settings.args ?? settings.arguments;
+  const cwdSource = settings.cwd;
+  const timeoutSource = settings.timeout_seconds ?? settings.timeoutSeconds;
+  const logDirSource = settings.log_dir ?? settings.logDir ?? settings.log_directory ?? settings.logDirectory;
+  const logFormatSource = settings.log_format ?? settings.logFormat ?? settings.log_output_format ?? settings.logOutputFormat ?? env.AGENTV_CODEX_LOG_FORMAT;
+  const executable = resolveOptionalString(executableSource, env, `${target.name} codex executable`, {
+    allowLiteral: true,
+    optionalEnv: true
+  }) ?? "codex";
+  const args = resolveOptionalStringArray(argsSource, env, `${target.name} codex args`);
+  const cwd = resolveOptionalString(cwdSource, env, `${target.name} codex cwd`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} codex timeout`);
+  const logDir = resolveOptionalString(logDirSource, env, `${target.name} codex log directory`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  const logFormat = normalizeCodexLogFormat(logFormatSource);
+  return {
+    executable,
+    args,
+    cwd,
+    timeoutMs,
+    logDir,
+    logFormat
+  };
+}
+function normalizeCodexLogFormat(value) {
+  if (value === void 0 || value === null) {
     return void 0;
   }
-  const deduped = /* @__PURE__ */ new Set();
-  for (const attachment of attachments) {
-    deduped.add(path3.resolve(attachment));
+  if (typeof value !== "string") {
+    throw new Error("codex log format must be 'summary' or 'json'");
   }
-  return Array.from(deduped);
-}
-function mergeAttachments(all) {
-  const deduped = /* @__PURE__ */ new Set();
-  for (const list of all) {
-    if (!list) continue;
-    for (const inputFile of list) {
-      deduped.add(path3.resolve(inputFile));
-    }
+  const normalized = value.trim().toLowerCase();
+  if (normalized === "json" || normalized === "summary") {
+    return normalized;
   }
-  return deduped.size > 0 ? Array.from(deduped) : void 0;
+  throw new Error("codex log format must be 'summary' or 'json'");
 }
-async function ensureVSCodeSubagents(options) {
-  const { kind, count, verbose = false } = options;
-  const vscodeCmd = kind === "vscode-insiders" ? "code-insiders" : "code";
-  const subagentRoot = getSubagentRoot(vscodeCmd);
-  try {
-    if (verbose) {
-      console.log(`Provisioning ${count} subagent(s) via: subagent ${vscodeCmd} provision`);
-    }
-    const result = await provisionSubagents({
-      targetRoot: subagentRoot,
-      subagents: count,
-      dryRun: false
-    });
-    if (verbose) {
-      if (result.created.length > 0) {
-        console.log(`Created ${result.created.length} new subagent(s)`);
-      }
-      if (result.skippedExisting.length > 0) {
-        console.log(`Reusing ${result.skippedExisting.length} existing unlocked subagent(s)`);
-      }
-      console.log(`
-total unlocked subagents available: ${result.created.length + result.skippedExisting.length}`);
-    }
-    return {
-      provisioned: true,
-      message: `Provisioned ${count} subagent(s): ${result.created.length} created, ${result.skippedExisting.length} reused`
-    };
-  } catch (error) {
-    const errorMessage = error instanceof Error ? error.message : String(error);
-    if (verbose) {
-      console.warn(`Provisioning failed (continuing anyway): ${errorMessage}`);
-    }
-    return {
-      provisioned: false,
-      message: `Provisioning failed: ${errorMessage}`
-    };
-  }
+function resolveMockConfig(target) {
+  const settings = target.settings ?? {};
+  const response = typeof settings.response === "string" ? settings.response : void 0;
+  return { response };
 }
-
-// src/evaluation/providers/codex.ts
-import { exec as execCallback, spawn } from "node:child_process";
-import { constants as constants2 } from "node:fs";
-import { access as access2, copyFile, mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import path5 from "node:path";
-import { promisify as promisify2 } from "node:util";
-
-// src/evaluation/providers/preread.ts
-import path4 from "node:path";
-function buildPromptDocument2(request, inputFiles, options) {
-  const parts = [];
-  const guidelineFiles = collectGuidelineFiles2(
-    inputFiles,
-    options?.guidelinePatterns ?? request.guideline_patterns,
-    options?.guidelineOverrides
+function resolveVSCodeConfig(target, env, insiders) {
+  const settings = target.settings ?? {};
+  const workspaceTemplateEnvVar = resolveOptionalLiteralString(settings.workspace_template ?? settings.workspaceTemplate);
+  const workspaceTemplate = workspaceTemplateEnvVar ? resolveOptionalString(workspaceTemplateEnvVar, env, `${target.name} workspace template path`, {
+    allowLiteral: false,
+    optionalEnv: true
+  }) : void 0;
+  const commandSource = settings.vscode_cmd ?? settings.command;
+  const waitSource = settings.wait;
+  const dryRunSource = settings.dry_run ?? settings.dryRun;
+  const subagentRootSource = settings.subagent_root ?? settings.subagentRoot;
+  const defaultCommand = insiders ? "code-insiders" : "code";
+  const command = resolveOptionalLiteralString(commandSource) ?? defaultCommand;
+  return {
+    command,
+    waitForResponse: resolveOptionalBoolean(waitSource) ?? true,
+    dryRun: resolveOptionalBoolean(dryRunSource) ?? false,
+    subagentRoot: resolveOptionalString(subagentRootSource, env, `${target.name} subagent root`, {
+      allowLiteral: true,
+      optionalEnv: true
+    }),
+    workspaceTemplate
+  };
+}
+function resolveCliConfig(target, env) {
+  const settings = target.settings ?? {};
+  const commandTemplateSource = settings.command_template ?? settings.commandTemplate;
+  const filesFormat = resolveOptionalLiteralString(
+    settings.files_format ?? settings.filesFormat ?? settings.attachments_format ?? settings.attachmentsFormat
   );
-  const inputFilesList = collectInputFiles(inputFiles);
-  const nonGuidelineInputFiles = inputFilesList.filter(
-    (file) => !guidelineFiles.includes(file)
+  const cwd = resolveOptionalString(settings.cwd, env, `${target.name} working directory`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  const envOverrides = resolveEnvOverrides(settings.env, env, target.name);
+  const timeoutMs = resolveTimeoutMs(settings.timeout_seconds ?? settings.timeoutSeconds, `${target.name} timeout`);
+  const healthcheck = resolveCliHealthcheck(settings.healthcheck, env, target.name);
+  const commandTemplate = resolveString(
+    commandTemplateSource,
+    env,
+    `${target.name} CLI command template`,
+    true
   );
-  const prereadBlock = buildMandatoryPrereadBlock2(guidelineFiles, nonGuidelineInputFiles);
-  if (prereadBlock.length > 0) {
-    parts.push("\n", prereadBlock);
-  }
-  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
-  return parts.join("\n").trim();
+  assertSupportedCliPlaceholders(commandTemplate, `${target.name} CLI command template`);
+  return {
+    commandTemplate,
+    filesFormat,
+    cwd,
+    env: envOverrides,
+    timeoutMs,
+    healthcheck
+  };
 }
-function normalizeInputFiles2(inputFiles) {
-  if (!inputFiles || inputFiles.length === 0) {
+function resolveEnvOverrides(source, env, targetName) {
+  if (source === void 0 || source === null) {
     return void 0;
   }
-  const deduped = /* @__PURE__ */ new Map();
-  for (const inputFile of inputFiles) {
-    const absolutePath = path4.resolve(inputFile);
-    if (!deduped.has(absolutePath)) {
-      deduped.set(absolutePath, absolutePath);
-    }
-  }
-  return Array.from(deduped.values());
-}
-function collectGuidelineFiles2(inputFiles, guidelinePatterns, overrides) {
-  if (!inputFiles || inputFiles.length === 0) {
-    return [];
-  }
-  const unique = /* @__PURE__ */ new Map();
-  for (const inputFile of inputFiles) {
-    const absolutePath = path4.resolve(inputFile);
-    if (overrides?.has(absolutePath)) {
-      if (!unique.has(absolutePath)) {
-        unique.set(absolutePath, absolutePath);
-      }
-      continue;
-    }
-    const normalized = absolutePath.split(path4.sep).join("/");
-    if (isGuidelineFile(normalized, guidelinePatterns)) {
-      if (!unique.has(absolutePath)) {
-        unique.set(absolutePath, absolutePath);
-      }
-    }
-  }
-  return Array.from(unique.values());
-}
-function collectInputFiles(inputFiles) {
-  if (!inputFiles || inputFiles.length === 0) {
-    return [];
+  if (typeof source !== "object" || Array.isArray(source)) {
+    throw new Error(`${targetName} env overrides must be an object map of strings`);
   }
-  const unique = /* @__PURE__ */ new Map();
-  for (const inputFile of inputFiles) {
-    const absolutePath = path4.resolve(inputFile);
-    if (!unique.has(absolutePath)) {
-      unique.set(absolutePath, absolutePath);
+  const entries = Object.entries(source);
+  const resolved = {};
+  for (const [key, value] of entries) {
+    if (typeof value !== "string") {
+      throw new Error(`${targetName} env override '${key}' must be a string`);
     }
+    const resolvedValue = resolveString(value, env, `${targetName} env override '${key}'`);
+    resolved[key] = resolvedValue;
   }
-  return Array.from(unique.values());
+  return Object.keys(resolved).length > 0 ? resolved : void 0;
 }
-function buildMandatoryPrereadBlock2(guidelineFiles, inputFiles) {
-  if (guidelineFiles.length === 0 && inputFiles.length === 0) {
-    return "";
-  }
-  const buildList = (files) => files.map((absolutePath) => {
-    const fileName = path4.basename(absolutePath);
-    const fileUri = pathToFileUri2(absolutePath);
-    return `* [${fileName}](${fileUri})`;
-  });
-  const sections = [];
-  if (guidelineFiles.length > 0) {
-    sections.push(`Read all guideline files:
-${buildList(guidelineFiles).join("\n")}.`);
+function resolveTimeoutMs(source, description) {
+  const seconds = resolveOptionalNumber(source, `${description} (seconds)`);
+  if (seconds === void 0) {
+    return void 0;
   }
-  if (inputFiles.length > 0) {
-    sections.push(`Read all input files:
-${buildList(inputFiles).join("\n")}.`);
+  if (seconds <= 0) {
+    throw new Error(`${description} must be greater than zero seconds`);
   }
-  sections.push(
-    "If any file is missing, fail with ERROR: missing-file <filename> and stop.",
-    "Then apply system_instructions on the user query below."
-  );
-  return sections.join("\n");
+  return Math.floor(seconds * 1e3);
 }
-function pathToFileUri2(filePath) {
-  const absolutePath = path4.isAbsolute(filePath) ? filePath : path4.resolve(filePath);
-  const normalizedPath = absolutePath.replace(/\\/g, "/");
-  if (/^[a-zA-Z]:\//.test(normalizedPath)) {
-    return `file:///${normalizedPath}`;
+function resolveCliHealthcheck(source, env, targetName) {
+  if (source === void 0 || source === null) {
+    return void 0;
   }
-  return `file://${normalizedPath}`;
-}
-
-// src/evaluation/providers/codex.ts
-var execAsync2 = promisify2(execCallback);
-var WORKSPACE_PREFIX = "agentv-codex-";
-var PROMPT_FILENAME = "prompt.md";
-var FILES_DIR = "files";
-var JSONL_TYPE_ITEM_COMPLETED = "item.completed";
-var CodexProvider = class {
-  id;
-  kind = "codex";
-  targetName;
-  supportsBatch = false;
-  config;
-  runCodex;
-  environmentCheck;
-  resolvedExecutable;
-  constructor(targetName, config, runner = defaultCodexRunner) {
-    this.id = `codex:${targetName}`;
-    this.targetName = targetName;
-    this.config = config;
-    this.runCodex = runner;
+  if (typeof source !== "object" || Array.isArray(source)) {
+    throw new Error(`${targetName} healthcheck must be an object`);
   }
-  async invoke(request) {
-    if (request.signal?.aborted) {
-      throw new Error("Codex provider request was aborted before execution");
-    }
-    await this.ensureEnvironmentReady();
-    const inputFiles = normalizeInputFiles2(request.inputFiles);
-    const originalGuidelines = new Set(
-      collectGuidelineFiles2(inputFiles, request.guideline_patterns).map((file) => path5.resolve(file))
+  const candidate = source;
+  const type = candidate.type;
+  const timeoutMs = resolveTimeoutMs(
+    candidate.timeout_seconds ?? candidate.timeoutSeconds,
+    `${targetName} healthcheck timeout`
+  );
+  if (type === "http") {
+    const url = resolveString(candidate.url, env, `${targetName} healthcheck URL`);
+    return {
+      type: "http",
+      url,
+      timeoutMs
+    };
+  }
+  if (type === "command") {
+    const commandTemplate = resolveString(
+      candidate.command_template ?? candidate.commandTemplate,
+      env,
+      `${targetName} healthcheck command template`,
+      true
     );
-    const workspaceRoot = await this.createWorkspace();
-    try {
-      const { mirroredInputFiles, guidelineMirrors } = await this.mirrorInputFiles(
-        inputFiles,
-        workspaceRoot,
-        originalGuidelines
+    assertSupportedCliPlaceholders(commandTemplate, `${targetName} healthcheck command template`);
+    const cwd = resolveOptionalString(candidate.cwd, env, `${targetName} healthcheck cwd`, {
+      allowLiteral: true,
+      optionalEnv: true
+    });
+    return {
+      type: "command",
+      commandTemplate,
+      timeoutMs,
+      cwd
+    };
+  }
+  throw new Error(`${targetName} healthcheck type must be 'http' or 'command'`);
+}
+function assertSupportedCliPlaceholders(template, description) {
+  const placeholders = extractCliPlaceholders(template);
+  for (const placeholder of placeholders) {
+    if (!CLI_PLACEHOLDERS.has(placeholder)) {
+      throw new Error(
+        `${description} includes unsupported placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS).join(", ")}`
       );
-      const promptContent = buildPromptDocument2(request, mirroredInputFiles, {
-        guidelinePatterns: request.guideline_patterns,
-        guidelineOverrides: guidelineMirrors
-      });
-      const promptFile = path5.join(workspaceRoot, PROMPT_FILENAME);
-      await writeFile(promptFile, promptContent, "utf8");
-      const args = this.buildCodexArgs();
-      const cwd = this.resolveCwd(workspaceRoot);
-      const result = await this.executeCodex(args, cwd, promptContent, request.signal);
-      if (result.timedOut) {
-        throw new Error(
-          `Codex CLI timed out${formatTimeoutSuffix2(this.config.timeoutMs ?? void 0)}`
-        );
-      }
-      if (result.exitCode !== 0) {
-        const detail = pickDetail(result.stderr, result.stdout);
-        const prefix = `Codex CLI exited with code ${result.exitCode}`;
-        throw new Error(detail ? `${prefix}: ${detail}` : prefix);
-      }
-      const parsed = parseCodexJson(result.stdout);
-      const assistantText = extractAssistantText(parsed);
-      return {
-        text: assistantText,
-        raw: {
-          response: parsed,
-          stdout: result.stdout,
-          stderr: result.stderr,
-          exitCode: result.exitCode,
-          args,
-          executable: this.resolvedExecutable ?? this.config.executable,
-          promptFile,
-          workspace: workspaceRoot,
-          inputFiles: mirroredInputFiles
-        }
-      };
-    } finally {
-      await this.cleanupWorkspace(workspaceRoot);
     }
   }
-  async ensureEnvironmentReady() {
-    if (!this.environmentCheck) {
-      this.environmentCheck = this.validateEnvironment();
+}
+function extractCliPlaceholders(template) {
+  const matches = template.matchAll(/\{([A-Z_]+)\}/g);
+  const results = [];
+  for (const match of matches) {
+    if (match[1]) {
+      results.push(match[1]);
     }
-    await this.environmentCheck;
   }
-  async validateEnvironment() {
-    this.resolvedExecutable = await locateExecutable(this.config.executable);
+  return results;
+}
+function resolveString(source, env, description, allowLiteral = false) {
+  const value = resolveOptionalString(source, env, description, {
+    allowLiteral,
+    optionalEnv: false
+  });
+  if (value === void 0) {
+    throw new Error(`${description} is required`);
   }
-  resolveCwd(workspaceRoot) {
-    if (!this.config.cwd) {
-      return workspaceRoot;
-    }
-    return path5.resolve(this.config.cwd);
+  return value;
+}
+function resolveOptionalString(source, env, description, options) {
+  if (source === void 0 || source === null) {
+    return void 0;
   }
-  buildCodexArgs() {
-    const args = ["--ask-for-approval", "never", "exec", "--json", "--color", "never", "--skip-git-repo-check"];
-    if (this.config.args && this.config.args.length > 0) {
-      args.push(...this.config.args);
-    }
-    args.push("-");
-    return args;
+  if (typeof source !== "string") {
+    throw new Error(`${description} must be a string`);
   }
-  async executeCodex(args, cwd, promptContent, signal) {
-    try {
-      return await this.runCodex({
-        executable: this.resolvedExecutable ?? this.config.executable,
-        args,
-        cwd,
-        prompt: promptContent,
-        timeoutMs: this.config.timeoutMs,
-        env: process.env,
-        signal
-      });
-    } catch (error) {
-      const err = error;
-      if (err.code === "ENOENT") {
-        throw new Error(
-          `Codex executable '${this.config.executable}' was not found. Update the target settings.executable or add it to PATH.`
-        );
-      }
-      throw error;
-    }
+  const trimmed = source.trim();
+  if (trimmed.length === 0) {
+    return void 0;
   }
-  async mirrorInputFiles(inputFiles, workspaceRoot, guidelineOriginals) {
-    if (!inputFiles || inputFiles.length === 0) {
-      return {
-        mirroredInputFiles: void 0,
-        guidelineMirrors: /* @__PURE__ */ new Set()
-      };
-    }
-    const filesRoot = path5.join(workspaceRoot, FILES_DIR);
-    await mkdir(filesRoot, { recursive: true });
-    const mirrored = [];
-    const guidelineMirrors = /* @__PURE__ */ new Set();
-    const nameCounts = /* @__PURE__ */ new Map();
-    for (const inputFile of inputFiles) {
-      const absoluteSource = path5.resolve(inputFile);
-      const baseName = path5.basename(absoluteSource);
-      const count = nameCounts.get(baseName) ?? 0;
-      nameCounts.set(baseName, count + 1);
-      const finalName = count === 0 ? baseName : `${baseName}.${count}`;
-      const destination = path5.join(filesRoot, finalName);
-      await copyFile(absoluteSource, destination);
-      const resolvedDestination = path5.resolve(destination);
-      mirrored.push(resolvedDestination);
-      if (guidelineOriginals.has(absoluteSource)) {
-        guidelineMirrors.add(resolvedDestination);
-      }
+  const envValue = env[trimmed];
+  if (envValue !== void 0) {
+    if (envValue.trim().length === 0) {
+      throw new Error(`Environment variable '${trimmed}' for ${description} is empty`);
     }
-    return {
-      mirroredInputFiles: mirrored,
-      guidelineMirrors
-    };
-  }
-  async createWorkspace() {
-    return await mkdtemp(path5.join(tmpdir(), WORKSPACE_PREFIX));
+    return envValue;
   }
-  async cleanupWorkspace(workspaceRoot) {
-    try {
-      await rm(workspaceRoot, { recursive: true, force: true });
-    } catch {
+  const allowLiteral = options?.allowLiteral ?? false;
+  const optionalEnv = options?.optionalEnv ?? false;
+  const looksLikeEnv = isLikelyEnvReference(trimmed);
+  if (looksLikeEnv) {
+    if (optionalEnv) {
+      return void 0;
+    }
+    if (!allowLiteral) {
+      throw new Error(`Environment variable '${trimmed}' required for ${description} is not set`);
     }
   }
-};
-async function locateExecutable(candidate) {
-  const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
-  if (includesPathSeparator) {
-    const resolved = path5.isAbsolute(candidate) ? candidate : path5.resolve(candidate);
-    const executablePath = await ensureWindowsExecutableVariant(resolved);
-    await access2(executablePath, constants2.F_OK);
-    return executablePath;
+  return trimmed;
+}
+function resolveOptionalLiteralString(source) {
+  if (source === void 0 || source === null) {
+    return void 0;
   }
-  const locator = process.platform === "win32" ? "where" : "which";
-  try {
-    const { stdout } = await execAsync2(`${locator} ${candidate}`);
-    const lines = stdout.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
-    const preferred = selectExecutableCandidate(lines);
-    if (preferred) {
-      const executablePath = await ensureWindowsExecutableVariant(preferred);
-      await access2(executablePath, constants2.F_OK);
-      return executablePath;
-    }
-  } catch {
+  if (typeof source !== "string") {
+    throw new Error("expected string value");
   }
-  throw new Error(`Codex executable '${candidate}' was not found on PATH`);
+  const trimmed = source.trim();
+  return trimmed.length > 0 ? trimmed : void 0;
 }
-function selectExecutableCandidate(candidates) {
-  if (candidates.length === 0) {
+function resolveOptionalNumber(source, description) {
+  if (source === void 0 || source === null || source === "") {
     return void 0;
   }
-  if (process.platform !== "win32") {
-    return candidates[0];
+  if (typeof source === "number") {
+    return Number.isFinite(source) ? source : void 0;
   }
-  const extensions = getWindowsExecutableExtensions();
-  for (const ext of extensions) {
-    const match = candidates.find((candidate) => candidate.toLowerCase().endsWith(ext));
-    if (match) {
-      return match;
+  if (typeof source === "string") {
+    const numeric = Number(source);
+    if (Number.isFinite(numeric)) {
+      return numeric;
     }
   }
-  return candidates[0];
+  throw new Error(`${description} must be a number`);
 }
-async function ensureWindowsExecutableVariant(candidate) {
-  if (process.platform !== "win32") {
-    return candidate;
+function resolveOptionalBoolean(source) {
+  if (source === void 0 || source === null || source === "") {
+    return void 0;
   }
-  if (hasExecutableExtension(candidate)) {
-    return candidate;
+  if (typeof source === "boolean") {
+    return source;
   }
-  const extensions = getWindowsExecutableExtensions();
-  for (const ext of extensions) {
-    const withExtension = `${candidate}${ext}`;
-    try {
-      await access2(withExtension, constants2.F_OK);
-      return withExtension;
-    } catch {
+  if (typeof source === "string") {
+    const lowered = source.trim().toLowerCase();
+    if (lowered === "true" || lowered === "1") {
+      return true;
+    }
+    if (lowered === "false" || lowered === "0") {
+      return false;
     }
   }
-  return candidate;
+  throw new Error("expected boolean value");
 }
-function hasExecutableExtension(candidate) {
-  const lower = candidate.toLowerCase();
-  return getWindowsExecutableExtensions().some((ext) => lower.endsWith(ext));
+function isLikelyEnvReference(value) {
+  return /^[A-Z0-9_]+$/.test(value);
 }
-var DEFAULT_WINDOWS_EXTENSIONS = [".com", ".exe", ".bat", ".cmd", ".ps1"];
-function getWindowsExecutableExtensions() {
-  if (process.platform !== "win32") {
-    return [];
+function resolveOptionalStringArray(source, env, description) {
+  if (source === void 0 || source === null) {
+    return void 0;
   }
-  const fromEnv = process.env.PATHEXT?.split(";").map((ext) => ext.trim().toLowerCase()).filter((ext) => ext.length > 0);
-  return fromEnv && fromEnv.length > 0 ? fromEnv : DEFAULT_WINDOWS_EXTENSIONS;
-}
-function parseCodexJson(output) {
-  const trimmed = output.trim();
-  if (trimmed.length === 0) {
-    throw new Error("Codex CLI produced no output in --json mode");
+  if (!Array.isArray(source)) {
+    throw new Error(`${description} must be an array of strings`);
   }
-  try {
-    return JSON.parse(trimmed);
-  } catch {
-    const lineObjects = parseJsonLines(trimmed);
-    if (lineObjects) {
-      return lineObjects;
+  if (source.length === 0) {
+    return void 0;
+  }
+  const resolved = [];
+  for (let i = 0; i < source.length; i++) {
+    const item = source[i];
+    if (typeof item !== "string") {
+      throw new Error(`${description}[${i}] must be a string`);
     }
-    const lastBrace = trimmed.lastIndexOf("{");
-    if (lastBrace >= 0) {
-      const candidate = trimmed.slice(lastBrace);
-      try {
-        return JSON.parse(candidate);
-      } catch {
+    const trimmed = item.trim();
+    if (trimmed.length === 0) {
+      throw new Error(`${description}[${i}] cannot be empty`);
+    }
+    const envValue = env[trimmed];
+    if (envValue !== void 0) {
+      if (envValue.trim().length === 0) {
+        throw new Error(`Environment variable '${trimmed}' for ${description}[${i}] is empty`);
       }
+      resolved.push(envValue);
+    } else {
+      resolved.push(trimmed);
     }
-    const preview = trimmed.slice(0, 200);
-    throw new Error(`Codex CLI emitted invalid JSON: ${preview}${trimmed.length > 200 ? "\u2026" : ""}`);
   }
+  return resolved.length > 0 ? resolved : void 0;
 }
-function extractAssistantText(parsed) {
-  if (Array.isArray(parsed)) {
-    const text = extractFromEventStream(parsed);
-    if (text) {
-      return text;
-    }
-  }
-  if (!parsed || typeof parsed !== "object") {
-    throw new Error("Codex CLI JSON response did not include an assistant message");
-  }
-  const record = parsed;
-  const eventText = extractFromEvent(record);
-  if (eventText) {
-    return eventText;
+
+// src/evaluation/providers/vscode.ts
+import { readFile as readFile2 } from "node:fs/promises";
+import path5 from "node:path";
+import { dispatchAgentSession, dispatchBatchAgent, getSubagentRoot, provisionSubagents } from "subagent";
+var VSCodeProvider = class {
+  id;
+  kind;
+  targetName;
+  supportsBatch = true;
+  config;
+  constructor(targetName, config, kind) {
+    this.id = `${kind}:${targetName}`;
+    this.kind = kind;
+    this.targetName = targetName;
+    this.config = config;
   }
-  const messages = Array.isArray(record.messages) ? record.messages : void 0;
-  if (messages) {
-    for (let index = messages.length - 1; index >= 0; index -= 1) {
-      const entry = messages[index];
-      if (!entry || typeof entry !== "object") {
-        continue;
-      }
-      const role = entry.role;
-      if (role !== "assistant") {
-        continue;
-      }
-      const content = entry.content;
-      const flattened = flattenContent(content);
-      if (flattened) {
-        return flattened;
-      }
+  async invoke(request) {
+    if (request.signal?.aborted) {
+      throw new Error("VS Code provider request was aborted before dispatch");
+    }
+    const inputFiles = normalizeAttachments(request.inputFiles);
+    const promptContent = buildPromptDocument2(request, inputFiles, request.guideline_patterns);
+    const session = await dispatchAgentSession({
+      userQuery: promptContent,
+      extraAttachments: inputFiles,
+      wait: this.config.waitForResponse,
+      dryRun: this.config.dryRun,
+      vscodeCmd: this.config.command,
+      subagentRoot: this.config.subagentRoot,
+      workspaceTemplate: this.config.workspaceTemplate,
+      silent: true
+    });
+    if (session.exitCode !== 0 || !session.responseFile) {
+      const failure = session.error ?? "VS Code subagent did not produce a response";
+      throw new Error(failure);
+    }
+    if (this.config.dryRun) {
+      return {
+        text: "",
+        raw: {
+          session,
+          inputFiles
+        }
+      };
     }
+    const responseText = await readFile2(session.responseFile, "utf8");
+    return {
+      text: responseText,
+      raw: {
+        session,
+        inputFiles
+      }
+    };
   }
-  const response = record.response;
-  if (response && typeof response === "object") {
-    const content = response.content;
-    const flattened = flattenContent(content);
-    if (flattened) {
-      return flattened;
+  async invokeBatch(requests) {
+    if (requests.length === 0) {
+      return [];
+    }
+    const normalizedRequests = requests.map((req) => ({
+      request: req,
+      inputFiles: normalizeAttachments(req.inputFiles)
+    }));
+    const combinedInputFiles = mergeAttachments(
+      normalizedRequests.map(({ inputFiles }) => inputFiles)
+    );
+    const userQueries = normalizedRequests.map(
+      ({ request, inputFiles }) => buildPromptDocument2(request, inputFiles, request.guideline_patterns)
+    );
+    const session = await dispatchBatchAgent({
+      userQueries,
+      extraAttachments: combinedInputFiles,
+      wait: this.config.waitForResponse,
+      dryRun: this.config.dryRun,
+      vscodeCmd: this.config.command,
+      subagentRoot: this.config.subagentRoot,
+      workspaceTemplate: this.config.workspaceTemplate,
+      silent: true
+    });
+    if (session.exitCode !== 0 || !session.responseFiles) {
+      const failure = session.error ?? "VS Code subagent did not produce batch responses";
+      throw new Error(failure);
+    }
+    if (this.config.dryRun) {
+      return normalizedRequests.map(({ inputFiles }) => ({
+        text: "",
+        raw: {
+          session,
+          inputFiles,
+          allInputFiles: combinedInputFiles
+        }
+      }));
+    }
+    if (session.responseFiles.length !== requests.length) {
+      throw new Error(
+        `VS Code batch returned ${session.responseFiles.length} responses for ${requests.length} requests`
+      );
     }
-  }
-  const output = record.output;
-  const flattenedOutput = flattenContent(output);
-  if (flattenedOutput) {
-    return flattenedOutput;
-  }
-  throw new Error("Codex CLI JSON response did not include an assistant message");
-}
-function extractFromEventStream(events) {
-  for (let index = events.length - 1; index >= 0; index -= 1) {
-    const candidate = events[index];
-    const text = extractFromEvent(candidate);
-    if (text) {
-      return text;
+    const responses = [];
+    for (const [index, responseFile] of session.responseFiles.entries()) {
+      const responseText = await readFile2(responseFile, "utf8");
+      responses.push({
+        text: responseText,
+        raw: {
+          session,
+          inputFiles: normalizedRequests[index]?.inputFiles,
+          allInputFiles: combinedInputFiles,
+          responseFile
+        }
+      });
     }
+    return responses;
   }
-  return void 0;
+};
+function buildPromptDocument2(request, attachments, guidelinePatterns) {
+  const parts = [];
+  const guidelineFiles = collectGuidelineFiles2(attachments, guidelinePatterns);
+  const attachmentFiles = collectAttachmentFiles(attachments);
+  const nonGuidelineAttachments = attachmentFiles.filter(
+    (file) => !guidelineFiles.includes(file)
+  );
+  const prereadBlock = buildMandatoryPrereadBlock2(guidelineFiles, nonGuidelineAttachments);
+  if (prereadBlock.length > 0) {
+    parts.push("\n", prereadBlock);
+  }
+  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
+  return parts.join("\n").trim();
 }
-function extractFromEvent(event) {
-  if (!event || typeof event !== "object") {
-    return void 0;
+function buildMandatoryPrereadBlock2(guidelineFiles, attachmentFiles) {
+  if (guidelineFiles.length === 0 && attachmentFiles.length === 0) {
+    return "";
   }
-  const record = event;
-  const type = typeof record.type === "string" ? record.type : void 0;
-  if (type === JSONL_TYPE_ITEM_COMPLETED) {
-    const item = record.item;
-    const text = extractFromItem(item);
-    if (text) {
-      return text;
-    }
+  const buildList = (files) => files.map((absolutePath) => {
+    const fileName = path5.basename(absolutePath);
+    const fileUri = pathToFileUri2(absolutePath);
+    return `* [${fileName}](${fileUri})`;
+  });
+  const sections = [];
+  if (guidelineFiles.length > 0) {
+    sections.push(`Read all guideline files:
+${buildList(guidelineFiles).join("\n")}.`);
   }
-  const output = record.output ?? record.content;
-  const flattened = flattenContent(output);
-  if (flattened) {
-    return flattened;
+  if (attachmentFiles.length > 0) {
+    sections.push(`Read all attachment files:
+${buildList(attachmentFiles).join("\n")}.`);
   }
-  return void 0;
+  sections.push(
+    "If any file is missing, fail with ERROR: missing-file <filename> and stop.",
+    "Then apply system_instructions on the user query below."
+  );
+  return sections.join("\n");
 }
-function extractFromItem(item) {
-  if (!item || typeof item !== "object") {
-    return void 0;
+function collectGuidelineFiles2(attachments, guidelinePatterns) {
+  if (!attachments || attachments.length === 0) {
+    return [];
   }
-  const record = item;
-  const itemType = typeof record.type === "string" ? record.type : void 0;
-  if (itemType === "agent_message" || itemType === "response" || itemType === "output") {
-    const text = flattenContent(record.text ?? record.content ?? record.output);
-    if (text) {
-      return text;
+  const unique = /* @__PURE__ */ new Map();
+  for (const attachment of attachments) {
+    const absolutePath = path5.resolve(attachment);
+    const normalized = absolutePath.split(path5.sep).join("/");
+    if (isGuidelineFile(normalized, guidelinePatterns)) {
+      if (!unique.has(absolutePath)) {
+        unique.set(absolutePath, absolutePath);
+      }
     }
   }
-  return void 0;
+  return Array.from(unique.values());
 }
-function flattenContent(value) {
-  if (typeof value === "string") {
-    return value;
+function collectAttachmentFiles(attachments) {
+  if (!attachments || attachments.length === 0) {
+    return [];
   }
-  if (Array.isArray(value)) {
-    const parts = value.map((segment) => {
-      if (typeof segment === "string") {
-        return segment;
-      }
-      if (segment && typeof segment === "object" && "text" in segment) {
-        const text = segment.text;
-        return typeof text === "string" ? text : void 0;
-      }
-      return void 0;
-    }).filter((part) => typeof part === "string" && part.length > 0);
-    return parts.length > 0 ? parts.join(" \n") : void 0;
+  const unique = /* @__PURE__ */ new Map();
+  for (const attachment of attachments) {
+    const absolutePath = path5.resolve(attachment);
+    if (!unique.has(absolutePath)) {
+      unique.set(absolutePath, absolutePath);
+    }
   }
-  if (value && typeof value === "object" && "text" in value) {
-    const text = value.text;
-    return typeof text === "string" ? text : void 0;
+  return Array.from(unique.values());
+}
+function pathToFileUri2(filePath) {
+  const absolutePath = path5.isAbsolute(filePath) ? filePath : path5.resolve(filePath);
+  const normalizedPath = absolutePath.replace(/\\/g, "/");
+  if (/^[a-zA-Z]:\//.test(normalizedPath)) {
+    return `file:///${normalizedPath}`;
   }
-  return void 0;
+  return `file://${normalizedPath}`;
 }
-function parseJsonLines(output) {
-  const lines = output.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
-  if (lines.length <= 1) {
+function normalizeAttachments(attachments) {
+  if (!attachments || attachments.length === 0) {
     return void 0;
   }
-  const parsed = [];
-  for (const line of lines) {
-    try {
-      parsed.push(JSON.parse(line));
-    } catch {
-      return void 0;
-    }
-  }
-  return parsed;
-}
-function pickDetail(stderr, stdout) {
-  const errorText = stderr.trim();
-  if (errorText.length > 0) {
-    return errorText;
+  const deduped = /* @__PURE__ */ new Set();
+  for (const attachment of attachments) {
+    deduped.add(path5.resolve(attachment));
   }
-  const stdoutText = stdout.trim();
-  return stdoutText.length > 0 ? stdoutText : void 0;
+  return Array.from(deduped);
 }
-function formatTimeoutSuffix2(timeoutMs) {
-  if (!timeoutMs || timeoutMs <= 0) {
-    return "";
+function mergeAttachments(all) {
+  const deduped = /* @__PURE__ */ new Set();
+  for (const list of all) {
+    if (!list) continue;
+    for (const inputFile of list) {
+      deduped.add(path5.resolve(inputFile));
+    }
   }
-  const seconds = Math.ceil(timeoutMs / 1e3);
-  return ` after ${seconds}s`;
+  return deduped.size > 0 ? Array.from(deduped) : void 0;
 }
-async function defaultCodexRunner(options) {
-  return await new Promise((resolve, reject) => {
-    const child = spawn(options.executable, options.args, {
-      cwd: options.cwd,
-      env: options.env,
-      stdio: ["pipe", "pipe", "pipe"],
-      shell: shouldShellExecute(options.executable)
-    });
-    let stdout = "";
-    let stderr = "";
-    let timedOut = false;
-    const onAbort = () => {
-      child.kill("SIGTERM");
-    };
-    if (options.signal) {
-      if (options.signal.aborted) {
-        onAbort();
-      } else {
-        options.signal.addEventListener("abort", onAbort, { once: true });
-      }
-    }
-    let timeoutHandle;
-    if (options.timeoutMs && options.timeoutMs > 0) {
-      timeoutHandle = setTimeout(() => {
-        timedOut = true;
-        child.kill("SIGTERM");
-      }, options.timeoutMs);
-      timeoutHandle.unref?.();
+async function ensureVSCodeSubagents(options) {
+  const { kind, count, verbose = false } = options;
+  const vscodeCmd = kind === "vscode-insiders" ? "code-insiders" : "code";
+  const subagentRoot = getSubagentRoot(vscodeCmd);
+  try {
+    if (verbose) {
+      console.log(`Provisioning ${count} subagent(s) via: subagent ${vscodeCmd} provision`);
     }
-    child.stdout.setEncoding("utf8");
-    child.stdout.on("data", (chunk) => {
-      stdout += chunk;
-    });
-    child.stderr.setEncoding("utf8");
-    child.stderr.on("data", (chunk) => {
-      stderr += chunk;
+    const result = await provisionSubagents({
+      targetRoot: subagentRoot,
+      subagents: count,
+      dryRun: false
     });
-    child.stdin.end(options.prompt);
-    const cleanup = () => {
-      if (timeoutHandle) {
-        clearTimeout(timeoutHandle);
+    if (verbose) {
+      if (result.created.length > 0) {
+        console.log(`Created ${result.created.length} new subagent(s)`);
       }
-      if (options.signal) {
-        options.signal.removeEventListener("abort", onAbort);
+      if (result.skippedExisting.length > 0) {
+        console.log(`Reusing ${result.skippedExisting.length} existing unlocked subagent(s)`);
       }
+      console.log(`
+total unlocked subagents available: ${result.created.length + result.skippedExisting.length}`);
+    }
+    return {
+      provisioned: true,
+      message: `Provisioned ${count} subagent(s): ${result.created.length} created, ${result.skippedExisting.length} reused`
+    };
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    if (verbose) {
+      console.warn(`Provisioning failed (continuing anyway): ${errorMessage}`);
+    }
+    return {
+      provisioned: false,
+      message: `Provisioning failed: ${errorMessage}`
     };
-    child.on("error", (error) => {
-      cleanup();
-      reject(error);
-    });
-    child.on("close", (code) => {
-      cleanup();
-      resolve({
-        stdout,
-        stderr,
-        exitCode: typeof code === "number" ? code : -1,
-        timedOut
-      });
-    });
-  });
-}
-function shouldShellExecute(executable) {
-  if (process.platform !== "win32") {
-    return false;
   }
-  const lower = executable.toLowerCase();
-  return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
 }
 
 // src/evaluation/providers/targets-file.ts
@@ -2386,7 +2644,7 @@ function resolveAndCreateProvider(definition, env = process.env) {
 }
 
 // src/evaluation/evaluators.ts
-import { randomUUID } from "node:crypto";
+import { randomUUID as randomUUID2 } from "node:crypto";
 var LlmJudgeEvaluator = class {
   kind = "llm_judge";
   resolveJudgeProvider;
@@ -2424,7 +2682,7 @@ var LlmJudgeEvaluator = class {
     const misses = Array.isArray(parsed.misses) ? parsed.misses.filter(isNonEmptyString).slice(0, 4) : [];
     const reasoning = parsed.reasoning ?? response.reasoning;
     const evaluatorRawRequest = {
-      id: randomUUID(),
+      id: randomUUID2(),
       provider: judgeProvider.id,
       prompt,
       target: context.target.name,
@@ -2663,7 +2921,7 @@ function parseJsonSafe(payload) {
 }
 
 // src/evaluation/orchestrator.ts
-import { createHash, randomUUID as randomUUID2 } from "node:crypto";
+import { createHash, randomUUID as randomUUID3 } from "node:crypto";
 import { mkdir as mkdir2, readFile as readFile4, writeFile as writeFile2 } from "node:fs/promises";
 import path7 from "node:path";
 
@@ -3436,7 +3694,7 @@ function sanitizeFilename(value) {
     return "prompt";
   }
   const sanitized = value.replace(/[^A-Za-z0-9._-]+/g, "_");
-  return sanitized.length > 0 ? sanitized : randomUUID2();
+  return sanitized.length > 0 ? sanitized : randomUUID3();
 }
 async function invokeProvider(provider, options) {
   const { evalCase, promptInputs, attempt, agentTimeoutMs, signal } = options;