@agentv/core 0.5.0 → 0.5.1

package/dist/index.cjs

@@ -1112,1323 +1112,1323 @@ function formatTimeoutSuffix(timeoutMs) {
   return ` after ${seconds}s`;
 }
 
-// src/evaluation/providers/mock.ts
-var DEFAULT_MOCK_RESPONSE = '{"answer":"Mock provider response. Configure targets.yaml to supply a custom value."}';
-var MockProvider = class {
-  id;
-  kind = "mock";
-  targetName;
-  cannedResponse;
-  delayMs;
-  delayMinMs;
-  delayMaxMs;
-  constructor(targetName, config) {
-    this.id = `mock:${targetName}`;
-    this.targetName = targetName;
-    this.cannedResponse = config.response ?? DEFAULT_MOCK_RESPONSE;
-    this.delayMs = config.delayMs ?? 0;
-    this.delayMinMs = config.delayMinMs ?? 0;
-    this.delayMaxMs = config.delayMaxMs ?? 0;
+// src/evaluation/providers/codex.ts
+var import_node_child_process2 = require("child_process");
+var import_node_fs3 = require("fs");
+var import_promises3 = require("fs/promises");
+var import_node_os = require("os");
+var import_node_path5 = __toESM(require("path"), 1);
+var import_node_util2 = require("util");
+
+// src/evaluation/providers/preread.ts
+var import_node_path4 = __toESM(require("path"), 1);
+function buildPromptDocument(request, inputFiles, options) {
+  const parts = [];
+  const guidelineFiles = collectGuidelineFiles(
+    inputFiles,
+    options?.guidelinePatterns ?? request.guideline_patterns,
+    options?.guidelineOverrides
+  );
+  const inputFilesList = collectInputFiles(inputFiles);
+  const nonGuidelineInputFiles = inputFilesList.filter(
+    (file) => !guidelineFiles.includes(file)
+  );
+  const prereadBlock = buildMandatoryPrereadBlock(guidelineFiles, nonGuidelineInputFiles);
+  if (prereadBlock.length > 0) {
+    parts.push("\n", prereadBlock);
   }
-  async invoke(request) {
-    const delay = this.calculateDelay();
-    if (delay > 0) {
-      await new Promise((resolve) => setTimeout(resolve, delay));
-    }
-    return {
-      text: this.cannedResponse,
-      raw: {
-        prompt: request.prompt,
-        guidelines: request.guidelines
-      }
-    };
+  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
+  return parts.join("\n").trim();
+}
+function normalizeInputFiles2(inputFiles) {
+  if (!inputFiles || inputFiles.length === 0) {
+    return void 0;
   }
-  calculateDelay() {
-    if (this.delayMinMs > 0 || this.delayMaxMs > 0) {
-      const min = Math.max(0, this.delayMinMs);
-      const max = Math.max(min, this.delayMaxMs);
-      return Math.floor(Math.random() * (max - min + 1)) + min;
+  const deduped = /* @__PURE__ */ new Map();
+  for (const inputFile of inputFiles) {
+    const absolutePath = import_node_path4.default.resolve(inputFile);
+    if (!deduped.has(absolutePath)) {
+      deduped.set(absolutePath, absolutePath);
     }
-    return this.delayMs;
   }
-};
-
-// src/evaluation/providers/targets.ts
-var import_zod = require("zod");
-var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID", "ATTEMPT", "FILES"]);
-var BASE_TARGET_SCHEMA = import_zod.z.object({
-  name: import_zod.z.string().min(1, "target name is required"),
-  provider: import_zod.z.string().min(1, "provider is required"),
-  settings: import_zod.z.record(import_zod.z.unknown()).optional(),
-  judge_target: import_zod.z.string().optional(),
-  workers: import_zod.z.number().int().min(1).optional()
-});
-var DEFAULT_AZURE_API_VERSION = "2024-10-01-preview";
-function normalizeAzureApiVersion(value) {
-  if (!value) {
-    return DEFAULT_AZURE_API_VERSION;
+  return Array.from(deduped.values());
+}
+function collectGuidelineFiles(inputFiles, guidelinePatterns, overrides) {
+  if (!inputFiles || inputFiles.length === 0) {
+    return [];
   }
-  const trimmed = value.trim();
-  if (trimmed.length === 0) {
-    return DEFAULT_AZURE_API_VERSION;
+  const unique = /* @__PURE__ */ new Map();
+  for (const inputFile of inputFiles) {
+    const absolutePath = import_node_path4.default.resolve(inputFile);
+    if (overrides?.has(absolutePath)) {
+      if (!unique.has(absolutePath)) {
+        unique.set(absolutePath, absolutePath);
+      }
+      continue;
+    }
+    const normalized = absolutePath.split(import_node_path4.default.sep).join("/");
+    if (isGuidelineFile(normalized, guidelinePatterns)) {
+      if (!unique.has(absolutePath)) {
+        unique.set(absolutePath, absolutePath);
+      }
+    }
   }
-  const withoutPrefix = trimmed.replace(/^api[-_]?version\s*=\s*/i, "").trim();
-  return withoutPrefix.length > 0 ? withoutPrefix : DEFAULT_AZURE_API_VERSION;
+  return Array.from(unique.values());
 }
-function resolveTargetDefinition(definition, env = process.env) {
-  const parsed = BASE_TARGET_SCHEMA.parse(definition);
-  const provider = parsed.provider.toLowerCase();
-  const providerBatching = resolveOptionalBoolean(
-    parsed.settings?.provider_batching ?? parsed.settings?.providerBatching
-  );
-  switch (provider) {
-    case "azure":
-    case "azure-openai":
-      return {
-        kind: "azure",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveAzureConfig(parsed, env)
-      };
-    case "anthropic":
-      return {
-        kind: "anthropic",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveAnthropicConfig(parsed, env)
-      };
-    case "gemini":
-    case "google":
-    case "google-gemini":
-      return {
-        kind: "gemini",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveGeminiConfig(parsed, env)
-      };
-    case "codex":
-    case "codex-cli":
-      return {
-        kind: "codex",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveCodexConfig(parsed, env)
-      };
-    case "mock":
-      return {
-        kind: "mock",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveMockConfig(parsed)
-      };
-    case "vscode":
-    case "vscode-insiders":
-      return {
-        kind: provider,
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders")
-      };
-    case "cli":
-      return {
-        kind: "cli",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveCliConfig(parsed, env)
-      };
-    default:
-      throw new Error(`Unsupported provider '${parsed.provider}' in target '${parsed.name}'`);
+function collectInputFiles(inputFiles) {
+  if (!inputFiles || inputFiles.length === 0) {
+    return [];
   }
+  const unique = /* @__PURE__ */ new Map();
+  for (const inputFile of inputFiles) {
+    const absolutePath = import_node_path4.default.resolve(inputFile);
+    if (!unique.has(absolutePath)) {
+      unique.set(absolutePath, absolutePath);
+    }
+  }
+  return Array.from(unique.values());
 }
-function resolveAzureConfig(target, env) {
-  const settings = target.settings ?? {};
-  const endpointSource = settings.endpoint ?? settings.resource ?? settings.resourceName;
-  const apiKeySource = settings.api_key ?? settings.apiKey;
-  const deploymentSource = settings.deployment ?? settings.deploymentName ?? settings.model;
-  const versionSource = settings.version ?? settings.api_version;
-  const temperatureSource = settings.temperature;
-  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
-  const resourceName = resolveString(endpointSource, env, `${target.name} endpoint`);
-  const apiKey = resolveString(apiKeySource, env, `${target.name} api key`);
-  const deploymentName = resolveString(deploymentSource, env, `${target.name} deployment`);
-  const version = normalizeAzureApiVersion(
-    resolveOptionalString(versionSource, env, `${target.name} api version`)
-  );
-  const temperature = resolveOptionalNumber(temperatureSource, `${target.name} temperature`);
-  const maxOutputTokens = resolveOptionalNumber(
-    maxTokensSource,
-    `${target.name} max output tokens`
-  );
-  return {
-    resourceName,
-    deploymentName,
-    apiKey,
-    version,
-    temperature,
-    maxOutputTokens
-  };
-}
-function resolveAnthropicConfig(target, env) {
-  const settings = target.settings ?? {};
-  const apiKeySource = settings.api_key ?? settings.apiKey;
-  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
-  const temperatureSource = settings.temperature;
-  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
-  const thinkingBudgetSource = settings.thinking_budget ?? settings.thinkingBudget;
-  const apiKey = resolveString(apiKeySource, env, `${target.name} Anthropic api key`);
-  const model = resolveString(modelSource, env, `${target.name} Anthropic model`);
-  return {
-    apiKey,
-    model,
-    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
-    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`),
-    thinkingBudget: resolveOptionalNumber(thinkingBudgetSource, `${target.name} thinking budget`)
-  };
-}
-function resolveGeminiConfig(target, env) {
-  const settings = target.settings ?? {};
-  const apiKeySource = settings.api_key ?? settings.apiKey;
-  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
-  const temperatureSource = settings.temperature;
-  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
-  const apiKey = resolveString(apiKeySource, env, `${target.name} Google API key`);
-  const model = resolveOptionalString(modelSource, env, `${target.name} Gemini model`, {
-    allowLiteral: true,
-    optionalEnv: true
-  }) ?? "gemini-2.5-flash";
-  return {
-    apiKey,
-    model,
-    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
-    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`)
-  };
-}
-function resolveCodexConfig(target, env) {
-  const settings = target.settings ?? {};
-  const executableSource = settings.executable ?? settings.command ?? settings.binary;
-  const argsSource = settings.args ?? settings.arguments;
-  const cwdSource = settings.cwd;
-  const timeoutSource = settings.timeout_seconds ?? settings.timeoutSeconds;
-  const executable = resolveOptionalString(executableSource, env, `${target.name} codex executable`, {
-    allowLiteral: true,
-    optionalEnv: true
-  }) ?? "codex";
-  const args = resolveOptionalStringArray(argsSource, env, `${target.name} codex args`);
-  const cwd = resolveOptionalString(cwdSource, env, `${target.name} codex cwd`, {
-    allowLiteral: true,
-    optionalEnv: true
-  });
-  const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} codex timeout`);
-  return {
-    executable,
-    args,
-    cwd,
-    timeoutMs
-  };
-}
-function resolveMockConfig(target) {
-  const settings = target.settings ?? {};
-  const response = typeof settings.response === "string" ? settings.response : void 0;
-  return { response };
-}
-function resolveVSCodeConfig(target, env, insiders) {
-  const settings = target.settings ?? {};
-  const workspaceTemplateEnvVar = resolveOptionalLiteralString(settings.workspace_template ?? settings.workspaceTemplate);
-  const workspaceTemplate = workspaceTemplateEnvVar ? resolveOptionalString(workspaceTemplateEnvVar, env, `${target.name} workspace template path`, {
-    allowLiteral: false,
-    optionalEnv: true
-  }) : void 0;
-  const commandSource = settings.vscode_cmd ?? settings.command;
-  const waitSource = settings.wait;
-  const dryRunSource = settings.dry_run ?? settings.dryRun;
-  const subagentRootSource = settings.subagent_root ?? settings.subagentRoot;
-  const defaultCommand = insiders ? "code-insiders" : "code";
-  const command = resolveOptionalLiteralString(commandSource) ?? defaultCommand;
-  return {
-    command,
-    waitForResponse: resolveOptionalBoolean(waitSource) ?? true,
-    dryRun: resolveOptionalBoolean(dryRunSource) ?? false,
-    subagentRoot: resolveOptionalString(subagentRootSource, env, `${target.name} subagent root`, {
-      allowLiteral: true,
-      optionalEnv: true
-    }),
-    workspaceTemplate
-  };
-}
-function resolveCliConfig(target, env) {
-  const settings = target.settings ?? {};
-  const commandTemplateSource = settings.command_template ?? settings.commandTemplate;
-  const filesFormat = resolveOptionalLiteralString(
-    settings.files_format ?? settings.filesFormat ?? settings.attachments_format ?? settings.attachmentsFormat
-  );
-  const cwd = resolveOptionalString(settings.cwd, env, `${target.name} working directory`, {
-    allowLiteral: true,
-    optionalEnv: true
+function buildMandatoryPrereadBlock(guidelineFiles, inputFiles) {
+  if (guidelineFiles.length === 0 && inputFiles.length === 0) {
+    return "";
+  }
+  const buildList = (files) => files.map((absolutePath) => {
+    const fileName = import_node_path4.default.basename(absolutePath);
+    const fileUri = pathToFileUri(absolutePath);
+    return `* [${fileName}](${fileUri})`;
   });
-  const envOverrides = resolveEnvOverrides(settings.env, env, target.name);
-  const timeoutMs = resolveTimeoutMs(settings.timeout_seconds ?? settings.timeoutSeconds, `${target.name} timeout`);
-  const healthcheck = resolveCliHealthcheck(settings.healthcheck, env, target.name);
-  const commandTemplate = resolveString(
-    commandTemplateSource,
-    env,
-    `${target.name} CLI command template`,
-    true
+  const sections = [];
+  if (guidelineFiles.length > 0) {
+    sections.push(`Read all guideline files:
+${buildList(guidelineFiles).join("\n")}.`);
+  }
+  if (inputFiles.length > 0) {
+    sections.push(`Read all input files:
+${buildList(inputFiles).join("\n")}.`);
+  }
+  sections.push(
+    "If any file is missing, fail with ERROR: missing-file <filename> and stop.",
+    "Then apply system_instructions on the user query below."
   );
-  assertSupportedCliPlaceholders(commandTemplate, `${target.name} CLI command template`);
-  return {
-    commandTemplate,
-    filesFormat,
-    cwd,
-    env: envOverrides,
-    timeoutMs,
-    healthcheck
-  };
+  return sections.join("\n");
 }
-function resolveEnvOverrides(source, env, targetName) {
-  if (source === void 0 || source === null) {
-    return void 0;
+function pathToFileUri(filePath) {
+  const absolutePath = import_node_path4.default.isAbsolute(filePath) ? filePath : import_node_path4.default.resolve(filePath);
+  const normalizedPath = absolutePath.replace(/\\/g, "/");
+  if (/^[a-zA-Z]:\//.test(normalizedPath)) {
+    return `file:///${normalizedPath}`;
+  }
+  return `file://${normalizedPath}`;
+}
+
+// src/evaluation/providers/codex.ts
+var execAsync2 = (0, import_node_util2.promisify)(import_node_child_process2.exec);
+var WORKSPACE_PREFIX = "agentv-codex-";
+var PROMPT_FILENAME = "prompt.md";
+var FILES_DIR = "files";
+var JSONL_TYPE_ITEM_COMPLETED = "item.completed";
+var CodexProvider = class {
+  id;
+  kind = "codex";
+  targetName;
+  supportsBatch = false;
+  config;
+  runCodex;
+  environmentCheck;
+  resolvedExecutable;
+  constructor(targetName, config, runner = defaultCodexRunner) {
+    this.id = `codex:${targetName}`;
+    this.targetName = targetName;
+    this.config = config;
+    this.runCodex = runner;
+  }
+  async invoke(request) {
+    if (request.signal?.aborted) {
+      throw new Error("Codex provider request was aborted before execution");
+    }
+    await this.ensureEnvironmentReady();
+    const inputFiles = normalizeInputFiles2(request.inputFiles);
+    const originalGuidelines = new Set(
+      collectGuidelineFiles(inputFiles, request.guideline_patterns).map((file) => import_node_path5.default.resolve(file))
+    );
+    const workspaceRoot = await this.createWorkspace();
+    try {
+      const { mirroredInputFiles, guidelineMirrors } = await this.mirrorInputFiles(
+        inputFiles,
+        workspaceRoot,
+        originalGuidelines
+      );
+      const promptContent = buildPromptDocument(request, mirroredInputFiles, {
+        guidelinePatterns: request.guideline_patterns,
+        guidelineOverrides: guidelineMirrors
+      });
+      const promptFile = import_node_path5.default.join(workspaceRoot, PROMPT_FILENAME);
+      await (0, import_promises3.writeFile)(promptFile, promptContent, "utf8");
+      const args = this.buildCodexArgs();
+      const cwd = this.resolveCwd(workspaceRoot);
+      const result = await this.executeCodex(args, cwd, promptContent, request.signal);
+      if (result.timedOut) {
+        throw new Error(
+          `Codex CLI timed out${formatTimeoutSuffix2(this.config.timeoutMs ?? void 0)}`
+        );
+      }
+      if (result.exitCode !== 0) {
+        const detail = pickDetail(result.stderr, result.stdout);
+        const prefix = `Codex CLI exited with code ${result.exitCode}`;
+        throw new Error(detail ? `${prefix}: ${detail}` : prefix);
+      }
+      const parsed = parseCodexJson(result.stdout);
+      const assistantText = extractAssistantText(parsed);
+      return {
+        text: assistantText,
+        raw: {
+          response: parsed,
+          stdout: result.stdout,
+          stderr: result.stderr,
+          exitCode: result.exitCode,
+          args,
+          executable: this.resolvedExecutable ?? this.config.executable,
+          promptFile,
+          workspace: workspaceRoot,
+          inputFiles: mirroredInputFiles
+        }
+      };
+    } finally {
+      await this.cleanupWorkspace(workspaceRoot);
+    }
   }
-  if (typeof source !== "object" || Array.isArray(source)) {
-    throw new Error(`${targetName} env overrides must be an object map of strings`);
+  async ensureEnvironmentReady() {
+    if (!this.environmentCheck) {
+      this.environmentCheck = this.validateEnvironment();
+    }
+    await this.environmentCheck;
   }
-  const entries = Object.entries(source);
-  const resolved = {};
-  for (const [key, value] of entries) {
-    if (typeof value !== "string") {
-      throw new Error(`${targetName} env override '${key}' must be a string`);
+  async validateEnvironment() {
+    this.resolvedExecutable = await locateExecutable(this.config.executable);
+  }
+  resolveCwd(workspaceRoot) {
+    if (!this.config.cwd) {
+      return workspaceRoot;
     }
-    const resolvedValue = resolveString(value, env, `${targetName} env override '${key}'`);
-    resolved[key] = resolvedValue;
+    return import_node_path5.default.resolve(this.config.cwd);
   }
-  return Object.keys(resolved).length > 0 ? resolved : void 0;
+  buildCodexArgs() {
+    const args = ["--ask-for-approval", "never", "exec", "--json", "--color", "never", "--skip-git-repo-check"];
+    if (this.config.args && this.config.args.length > 0) {
+      args.push(...this.config.args);
+    }
+    args.push("-");
+    return args;
+  }
+  async executeCodex(args, cwd, promptContent, signal) {
+    try {
+      return await this.runCodex({
+        executable: this.resolvedExecutable ?? this.config.executable,
+        args,
+        cwd,
+        prompt: promptContent,
+        timeoutMs: this.config.timeoutMs,
+        env: process.env,
+        signal
+      });
+    } catch (error) {
+      const err = error;
+      if (err.code === "ENOENT") {
+        throw new Error(
+          `Codex executable '${this.config.executable}' was not found. Update the target settings.executable or add it to PATH.`
+        );
+      }
+      throw error;
+    }
+  }
+  async mirrorInputFiles(inputFiles, workspaceRoot, guidelineOriginals) {
+    if (!inputFiles || inputFiles.length === 0) {
+      return {
+        mirroredInputFiles: void 0,
+        guidelineMirrors: /* @__PURE__ */ new Set()
+      };
+    }
+    const filesRoot = import_node_path5.default.join(workspaceRoot, FILES_DIR);
+    await (0, import_promises3.mkdir)(filesRoot, { recursive: true });
+    const mirrored = [];
+    const guidelineMirrors = /* @__PURE__ */ new Set();
+    const nameCounts = /* @__PURE__ */ new Map();
+    for (const inputFile of inputFiles) {
+      const absoluteSource = import_node_path5.default.resolve(inputFile);
+      const baseName = import_node_path5.default.basename(absoluteSource);
+      const count = nameCounts.get(baseName) ?? 0;
+      nameCounts.set(baseName, count + 1);
+      const finalName = count === 0 ? baseName : `${baseName}.${count}`;
+      const destination = import_node_path5.default.join(filesRoot, finalName);
+      await (0, import_promises3.copyFile)(absoluteSource, destination);
+      const resolvedDestination = import_node_path5.default.resolve(destination);
+      mirrored.push(resolvedDestination);
+      if (guidelineOriginals.has(absoluteSource)) {
+        guidelineMirrors.add(resolvedDestination);
+      }
+    }
+    return {
+      mirroredInputFiles: mirrored,
+      guidelineMirrors
+    };
+  }
+  async createWorkspace() {
+    return await (0, import_promises3.mkdtemp)(import_node_path5.default.join((0, import_node_os.tmpdir)(), WORKSPACE_PREFIX));
+  }
+  async cleanupWorkspace(workspaceRoot) {
+    try {
+      await (0, import_promises3.rm)(workspaceRoot, { recursive: true, force: true });
+    } catch {
+    }
+  }
+};
+async function locateExecutable(candidate) {
+  const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
+  if (includesPathSeparator) {
+    const resolved = import_node_path5.default.isAbsolute(candidate) ? candidate : import_node_path5.default.resolve(candidate);
+    const executablePath = await ensureWindowsExecutableVariant(resolved);
+    await (0, import_promises3.access)(executablePath, import_node_fs3.constants.F_OK);
+    return executablePath;
+  }
+  const locator = process.platform === "win32" ? "where" : "which";
+  try {
+    const { stdout } = await execAsync2(`${locator} ${candidate}`);
+    const lines = stdout.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
+    const preferred = selectExecutableCandidate(lines);
+    if (preferred) {
+      const executablePath = await ensureWindowsExecutableVariant(preferred);
+      await (0, import_promises3.access)(executablePath, import_node_fs3.constants.F_OK);
+      return executablePath;
+    }
+  } catch {
+  }
+  throw new Error(`Codex executable '${candidate}' was not found on PATH`);
 }
-function resolveTimeoutMs(source, description) {
-  const seconds = resolveOptionalNumber(source, `${description} (seconds)`);
-  if (seconds === void 0) {
+function selectExecutableCandidate(candidates) {
+  if (candidates.length === 0) {
     return void 0;
   }
-  if (seconds <= 0) {
-    throw new Error(`${description} must be greater than zero seconds`);
+  if (process.platform !== "win32") {
+    return candidates[0];
   }
-  return Math.floor(seconds * 1e3);
+  const extensions = getWindowsExecutableExtensions();
+  for (const ext of extensions) {
+    const match = candidates.find((candidate) => candidate.toLowerCase().endsWith(ext));
+    if (match) {
+      return match;
+    }
+  }
+  return candidates[0];
 }
-function resolveCliHealthcheck(source, env, targetName) {
-  if (source === void 0 || source === null) {
-    return void 0;
+async function ensureWindowsExecutableVariant(candidate) {
+  if (process.platform !== "win32") {
+    return candidate;
   }
-  if (typeof source !== "object" || Array.isArray(source)) {
-    throw new Error(`${targetName} healthcheck must be an object`);
+  if (hasExecutableExtension(candidate)) {
+    return candidate;
   }
-  const candidate = source;
-  const type = candidate.type;
-  const timeoutMs = resolveTimeoutMs(
-    candidate.timeout_seconds ?? candidate.timeoutSeconds,
-    `${targetName} healthcheck timeout`
-  );
-  if (type === "http") {
-    const url = resolveString(candidate.url, env, `${targetName} healthcheck URL`);
-    return {
-      type: "http",
-      url,
-      timeoutMs
-    };
+  const extensions = getWindowsExecutableExtensions();
+  for (const ext of extensions) {
+    const withExtension = `${candidate}${ext}`;
+    try {
+      await (0, import_promises3.access)(withExtension, import_node_fs3.constants.F_OK);
+      return withExtension;
+    } catch {
+    }
   }
-  if (type === "command") {
-    const commandTemplate = resolveString(
-      candidate.command_template ?? candidate.commandTemplate,
-      env,
-      `${targetName} healthcheck command template`,
-      true
-    );
-    assertSupportedCliPlaceholders(commandTemplate, `${targetName} healthcheck command template`);
-    const cwd = resolveOptionalString(candidate.cwd, env, `${targetName} healthcheck cwd`, {
-      allowLiteral: true,
-      optionalEnv: true
-    });
-    return {
-      type: "command",
-      commandTemplate,
-      timeoutMs,
-      cwd
-    };
+  return candidate;
+}
+function hasExecutableExtension(candidate) {
+  const lower = candidate.toLowerCase();
+  return getWindowsExecutableExtensions().some((ext) => lower.endsWith(ext));
+}
+var DEFAULT_WINDOWS_EXTENSIONS = [".com", ".exe", ".bat", ".cmd", ".ps1"];
+function getWindowsExecutableExtensions() {
+  if (process.platform !== "win32") {
+    return [];
   }
-  throw new Error(`${targetName} healthcheck type must be 'http' or 'command'`);
+  const fromEnv = process.env.PATHEXT?.split(";").map((ext) => ext.trim().toLowerCase()).filter((ext) => ext.length > 0);
+  return fromEnv && fromEnv.length > 0 ? fromEnv : DEFAULT_WINDOWS_EXTENSIONS;
 }
-function assertSupportedCliPlaceholders(template, description) {
-  const placeholders = extractCliPlaceholders(template);
-  for (const placeholder of placeholders) {
-    if (!CLI_PLACEHOLDERS.has(placeholder)) {
-      throw new Error(
-        `${description} includes unsupported placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS).join(", ")}`
-      );
+function parseCodexJson(output) {
+  const trimmed = output.trim();
+  if (trimmed.length === 0) {
+    throw new Error("Codex CLI produced no output in --json mode");
+  }
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    const lineObjects = parseJsonLines(trimmed);
+    if (lineObjects) {
+      return lineObjects;
+    }
+    const lastBrace = trimmed.lastIndexOf("{");
+    if (lastBrace >= 0) {
+      const candidate = trimmed.slice(lastBrace);
+      try {
+        return JSON.parse(candidate);
+      } catch {
+      }
+    }
+    const preview = trimmed.slice(0, 200);
+    throw new Error(`Codex CLI emitted invalid JSON: ${preview}${trimmed.length > 200 ? "\u2026" : ""}`);
+  }
+}
+function extractAssistantText(parsed) {
+  if (Array.isArray(parsed)) {
+    const text = extractFromEventStream(parsed);
+    if (text) {
+      return text;
+    }
+  }
+  if (!parsed || typeof parsed !== "object") {
+    throw new Error("Codex CLI JSON response did not include an assistant message");
+  }
+  const record = parsed;
+  const eventText = extractFromEvent(record);
+  if (eventText) {
+    return eventText;
+  }
+  const messages = Array.isArray(record.messages) ? record.messages : void 0;
+  if (messages) {
+    for (let index = messages.length - 1; index >= 0; index -= 1) {
+      const entry = messages[index];
+      if (!entry || typeof entry !== "object") {
+        continue;
+      }
+      const role = entry.role;
+      if (role !== "assistant") {
+        continue;
+      }
+      const content = entry.content;
+      const flattened = flattenContent(content);
+      if (flattened) {
+        return flattened;
+      }
+    }
+  }
+  const response = record.response;
+  if (response && typeof response === "object") {
+    const content = response.content;
+    const flattened = flattenContent(content);
+    if (flattened) {
+      return flattened;
     }
   }
-}
-function extractCliPlaceholders(template) {
-  const matches = template.matchAll(/\{([A-Z_]+)\}/g);
-  const results = [];
-  for (const match of matches) {
-    if (match[1]) {
-      results.push(match[1]);
-    }
+  const output = record.output;
+  const flattenedOutput = flattenContent(output);
+  if (flattenedOutput) {
+    return flattenedOutput;
   }
-  return results;
+  throw new Error("Codex CLI JSON response did not include an assistant message");
 }
-function resolveString(source, env, description, allowLiteral = false) {
-  const value = resolveOptionalString(source, env, description, {
-    allowLiteral,
-    optionalEnv: false
-  });
-  if (value === void 0) {
-    throw new Error(`${description} is required`);
+function extractFromEventStream(events) {
+  for (let index = events.length - 1; index >= 0; index -= 1) {
+    const candidate = events[index];
+    const text = extractFromEvent(candidate);
+    if (text) {
+      return text;
+    }
   }
-  return value;
+  return void 0;
 }
-function resolveOptionalString(source, env, description, options) {
-  if (source === void 0 || source === null) {
-    return void 0;
-  }
-  if (typeof source !== "string") {
-    throw new Error(`${description} must be a string`);
-  }
-  const trimmed = source.trim();
-  if (trimmed.length === 0) {
+function extractFromEvent(event) {
+  if (!event || typeof event !== "object") {
     return void 0;
   }
-  const envValue = env[trimmed];
-  if (envValue !== void 0) {
-    if (envValue.trim().length === 0) {
-      throw new Error(`Environment variable '${trimmed}' for ${description} is empty`);
+  const record = event;
+  const type = typeof record.type === "string" ? record.type : void 0;
+  if (type === JSONL_TYPE_ITEM_COMPLETED) {
+    const item = record.item;
+    const text = extractFromItem(item);
+    if (text) {
+      return text;
     }
-    return envValue;
   }
-  const allowLiteral = options?.allowLiteral ?? false;
-  const optionalEnv = options?.optionalEnv ?? false;
-  const looksLikeEnv = isLikelyEnvReference(trimmed);
-  if (looksLikeEnv) {
-    if (optionalEnv) {
-      return void 0;
-    }
-    if (!allowLiteral) {
-      throw new Error(`Environment variable '${trimmed}' required for ${description} is not set`);
-    }
+  const output = record.output ?? record.content;
+  const flattened = flattenContent(output);
+  if (flattened) {
+    return flattened;
   }
-  return trimmed;
+  return void 0;
 }
-function resolveOptionalLiteralString(source) {
-  if (source === void 0 || source === null) {
+function extractFromItem(item) {
+  if (!item || typeof item !== "object") {
     return void 0;
   }
-  if (typeof source !== "string") {
-    throw new Error("expected string value");
+  const record = item;
+  const itemType = typeof record.type === "string" ? record.type : void 0;
+  if (itemType === "agent_message" || itemType === "response" || itemType === "output") {
+    const text = flattenContent(record.text ?? record.content ?? record.output);
+    if (text) {
+      return text;
+    }
   }
-  const trimmed = source.trim();
-  return trimmed.length > 0 ? trimmed : void 0;
+  return void 0;
 }
-function resolveOptionalNumber(source, description) {
-  if (source === void 0 || source === null || source === "") {
-    return void 0;
+function flattenContent(value) {
+  if (typeof value === "string") {
+    return value;
   }
-  if (typeof source === "number") {
-    return Number.isFinite(source) ? source : void 0;
+  if (Array.isArray(value)) {
+    const parts = value.map((segment) => {
+      if (typeof segment === "string") {
+        return segment;
+      }
+      if (segment && typeof segment === "object" && "text" in segment) {
+        const text = segment.text;
+        return typeof text === "string" ? text : void 0;
+      }
+      return void 0;
+    }).filter((part) => typeof part === "string" && part.length > 0);
+    return parts.length > 0 ? parts.join(" \n") : void 0;
   }
-  if (typeof source === "string") {
-    const numeric = Number(source);
-    if (Number.isFinite(numeric)) {
-      return numeric;
-    }
+  if (value && typeof value === "object" && "text" in value) {
+    const text = value.text;
+    return typeof text === "string" ? text : void 0;
   }
-  throw new Error(`${description} must be a number`);
+  return void 0;
 }
-function resolveOptionalBoolean(source) {
-  if (source === void 0 || source === null || source === "") {
+function parseJsonLines(output) {
+  const lines = output.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
+  if (lines.length <= 1) {
     return void 0;
   }
-  if (typeof source === "boolean") {
-    return source;
-  }
-  if (typeof source === "string") {
-    const lowered = source.trim().toLowerCase();
-    if (lowered === "true" || lowered === "1") {
-      return true;
-    }
-    if (lowered === "false" || lowered === "0") {
-      return false;
+  const parsed = [];
+  for (const line of lines) {
+    try {
+      parsed.push(JSON.parse(line));
+    } catch {
+      return void 0;
     }
   }
-  throw new Error("expected boolean value");
-}
-function isLikelyEnvReference(value) {
-  return /^[A-Z0-9_]+$/.test(value);
+  return parsed;
 }
-function resolveOptionalStringArray(source, env, description) {
-  if (source === void 0 || source === null) {
-    return void 0;
-  }
-  if (!Array.isArray(source)) {
-    throw new Error(`${description} must be an array of strings`);
+function pickDetail(stderr, stdout) {
+  const errorText = stderr.trim();
+  if (errorText.length > 0) {
+    return errorText;
   }
-  if (source.length === 0) {
-    return void 0;
+  const stdoutText = stdout.trim();
+  return stdoutText.length > 0 ? stdoutText : void 0;
+}
+function formatTimeoutSuffix2(timeoutMs) {
+  if (!timeoutMs || timeoutMs <= 0) {
+    return "";
   }
-  const resolved = [];
-  for (let i = 0; i < source.length; i++) {
-    const item = source[i];
-    if (typeof item !== "string") {
-      throw new Error(`${description}[${i}] must be a string`);
+  const seconds = Math.ceil(timeoutMs / 1e3);
+  return ` after ${seconds}s`;
+}
+async function defaultCodexRunner(options) {
+  return await new Promise((resolve, reject) => {
+    const child = (0, import_node_child_process2.spawn)(options.executable, options.args, {
+      cwd: options.cwd,
+      env: options.env,
+      stdio: ["pipe", "pipe", "pipe"],
+      shell: shouldShellExecute(options.executable)
+    });
+    let stdout = "";
+    let stderr = "";
+    let timedOut = false;
+    const onAbort = () => {
+      child.kill("SIGTERM");
+    };
+    if (options.signal) {
+      if (options.signal.aborted) {
+        onAbort();
+      } else {
+        options.signal.addEventListener("abort", onAbort, { once: true });
+      }
     }
-    const trimmed = item.trim();
-    if (trimmed.length === 0) {
-      throw new Error(`${description}[${i}] cannot be empty`);
+    let timeoutHandle;
+    if (options.timeoutMs && options.timeoutMs > 0) {
+      timeoutHandle = setTimeout(() => {
+        timedOut = true;
+        child.kill("SIGTERM");
+      }, options.timeoutMs);
+      timeoutHandle.unref?.();
     }
-    const envValue = env[trimmed];
-    if (envValue !== void 0) {
-      if (envValue.trim().length === 0) {
-        throw new Error(`Environment variable '${trimmed}' for ${description}[${i}] is empty`);
+    child.stdout.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.setEncoding("utf8");
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.stdin.end(options.prompt);
+    const cleanup = () => {
+      if (timeoutHandle) {
+        clearTimeout(timeoutHandle);
       }
-      resolved.push(envValue);
-    } else {
-      resolved.push(trimmed);
-    }
+      if (options.signal) {
+        options.signal.removeEventListener("abort", onAbort);
+      }
+    };
+    child.on("error", (error) => {
+      cleanup();
+      reject(error);
+    });
+    child.on("close", (code) => {
+      cleanup();
+      resolve({
+        stdout,
+        stderr,
+        exitCode: typeof code === "number" ? code : -1,
+        timedOut
+      });
+    });
+  });
+}
+function shouldShellExecute(executable) {
+  if (process.platform !== "win32") {
+    return false;
   }
-  return resolved.length > 0 ? resolved : void 0;
+  const lower = executable.toLowerCase();
+  return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
 }
 
-// src/evaluation/providers/vscode.ts
-var import_promises3 = require("fs/promises");
-var import_node_path4 = __toESM(require("path"), 1);
-var import_subagent = require("subagent");
-var VSCodeProvider = class {
+// src/evaluation/providers/mock.ts
+var DEFAULT_MOCK_RESPONSE = '{"answer":"Mock provider response. Configure targets.yaml to supply a custom value."}';
+var MockProvider = class {
   id;
-  kind;
+  kind = "mock";
   targetName;
-  supportsBatch = true;
-  config;
-  constructor(targetName, config, kind) {
-    this.id = `${kind}:${targetName}`;
-    this.kind = kind;
+  cannedResponse;
+  delayMs;
+  delayMinMs;
+  delayMaxMs;
+  constructor(targetName, config) {
+    this.id = `mock:${targetName}`;
     this.targetName = targetName;
-    this.config = config;
+    this.cannedResponse = config.response ?? DEFAULT_MOCK_RESPONSE;
+    this.delayMs = config.delayMs ?? 0;
+    this.delayMinMs = config.delayMinMs ?? 0;
+    this.delayMaxMs = config.delayMaxMs ?? 0;
   }
   async invoke(request) {
-    if (request.signal?.aborted) {
-      throw new Error("VS Code provider request was aborted before dispatch");
-    }
-    const inputFiles = normalizeAttachments(request.inputFiles);
-    const promptContent = buildPromptDocument(request, inputFiles, request.guideline_patterns);
-    const session = await (0, import_subagent.dispatchAgentSession)({
-      userQuery: promptContent,
-      extraAttachments: inputFiles,
-      wait: this.config.waitForResponse,
-      dryRun: this.config.dryRun,
-      vscodeCmd: this.config.command,
-      subagentRoot: this.config.subagentRoot,
-      workspaceTemplate: this.config.workspaceTemplate,
-      silent: true
-    });
-    if (session.exitCode !== 0 || !session.responseFile) {
-      const failure = session.error ?? "VS Code subagent did not produce a response";
-      throw new Error(failure);
-    }
-    if (this.config.dryRun) {
-      return {
-        text: "",
-        raw: {
-          session,
-          inputFiles
-        }
-      };
+    const delay = this.calculateDelay();
+    if (delay > 0) {
+      await new Promise((resolve) => setTimeout(resolve, delay));
     }
-    const responseText = await (0, import_promises3.readFile)(session.responseFile, "utf8");
     return {
-      text: responseText,
+      text: this.cannedResponse,
       raw: {
-        session,
-        inputFiles
+        prompt: request.prompt,
+        guidelines: request.guidelines
       }
     };
   }
-  async invokeBatch(requests) {
-    if (requests.length === 0) {
-      return [];
-    }
-    const normalizedRequests = requests.map((req) => ({
-      request: req,
-      inputFiles: normalizeAttachments(req.inputFiles)
-    }));
-    const combinedInputFiles = mergeAttachments(
-      normalizedRequests.map(({ inputFiles }) => inputFiles)
-    );
-    const userQueries = normalizedRequests.map(
-      ({ request, inputFiles }) => buildPromptDocument(request, inputFiles, request.guideline_patterns)
-    );
-    const session = await (0, import_subagent.dispatchBatchAgent)({
-      userQueries,
-      extraAttachments: combinedInputFiles,
-      wait: this.config.waitForResponse,
-      dryRun: this.config.dryRun,
-      vscodeCmd: this.config.command,
-      subagentRoot: this.config.subagentRoot,
-      workspaceTemplate: this.config.workspaceTemplate,
-      silent: true
-    });
-    if (session.exitCode !== 0 || !session.responseFiles) {
-      const failure = session.error ?? "VS Code subagent did not produce batch responses";
-      throw new Error(failure);
-    }
-    if (this.config.dryRun) {
-      return normalizedRequests.map(({ inputFiles }) => ({
-        text: "",
-        raw: {
-          session,
-          inputFiles,
-          allInputFiles: combinedInputFiles
-        }
-      }));
-    }
-    if (session.responseFiles.length !== requests.length) {
-      throw new Error(
-        `VS Code batch returned ${session.responseFiles.length} responses for ${requests.length} requests`
-      );
-    }
-    const responses = [];
-    for (const [index, responseFile] of session.responseFiles.entries()) {
-      const responseText = await (0, import_promises3.readFile)(responseFile, "utf8");
-      responses.push({
-        text: responseText,
-        raw: {
-          session,
-          inputFiles: normalizedRequests[index]?.inputFiles,
-          allInputFiles: combinedInputFiles,
-          responseFile
-        }
-      });
+  calculateDelay() {
+    if (this.delayMinMs > 0 || this.delayMaxMs > 0) {
+      const min = Math.max(0, this.delayMinMs);
+      const max = Math.max(min, this.delayMaxMs);
+      return Math.floor(Math.random() * (max - min + 1)) + min;
     }
-    return responses;
+    return this.delayMs;
   }
 };
-function buildPromptDocument(request, attachments, guidelinePatterns) {
-  const parts = [];
-  const guidelineFiles = collectGuidelineFiles(attachments, guidelinePatterns);
-  const attachmentFiles = collectAttachmentFiles(attachments);
-  const nonGuidelineAttachments = attachmentFiles.filter(
-    (file) => !guidelineFiles.includes(file)
-  );
-  const prereadBlock = buildMandatoryPrereadBlock(guidelineFiles, nonGuidelineAttachments);
-  if (prereadBlock.length > 0) {
-    parts.push("\n", prereadBlock);
-  }
-  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
-  return parts.join("\n").trim();
-}
-function buildMandatoryPrereadBlock(guidelineFiles, attachmentFiles) {
-  if (guidelineFiles.length === 0 && attachmentFiles.length === 0) {
-    return "";
+
+// src/evaluation/providers/targets.ts
+var import_zod = require("zod");
+var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID", "ATTEMPT", "FILES"]);
+var BASE_TARGET_SCHEMA = import_zod.z.object({
+  name: import_zod.z.string().min(1, "target name is required"),
+  provider: import_zod.z.string().min(1, "provider is required"),
+  settings: import_zod.z.record(import_zod.z.unknown()).optional(),
+  judge_target: import_zod.z.string().optional(),
+  workers: import_zod.z.number().int().min(1).optional()
+});
+var DEFAULT_AZURE_API_VERSION = "2024-10-01-preview";
+function normalizeAzureApiVersion(value) {
+  if (!value) {
+    return DEFAULT_AZURE_API_VERSION;
   }
-  const buildList = (files) => files.map((absolutePath) => {
-    const fileName = import_node_path4.default.basename(absolutePath);
-    const fileUri = pathToFileUri(absolutePath);
-    return `* [${fileName}](${fileUri})`;
-  });
-  const sections = [];
-  if (guidelineFiles.length > 0) {
-    sections.push(`Read all guideline files:
-${buildList(guidelineFiles).join("\n")}.`);
+  const trimmed = value.trim();
+  if (trimmed.length === 0) {
+    return DEFAULT_AZURE_API_VERSION;
   }
-  if (attachmentFiles.length > 0) {
-    sections.push(`Read all attachment files:
-${buildList(attachmentFiles).join("\n")}.`);
+  const withoutPrefix = trimmed.replace(/^api[-_]?version\s*=\s*/i, "").trim();
+  return withoutPrefix.length > 0 ? withoutPrefix : DEFAULT_AZURE_API_VERSION;
+}
+function resolveTargetDefinition(definition, env = process.env) {
+  const parsed = BASE_TARGET_SCHEMA.parse(definition);
+  const provider = parsed.provider.toLowerCase();
+  const providerBatching = resolveOptionalBoolean(
+    parsed.settings?.provider_batching ?? parsed.settings?.providerBatching
+  );
+  switch (provider) {
+    case "azure":
+    case "azure-openai":
+      return {
+        kind: "azure",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveAzureConfig(parsed, env)
+      };
+    case "anthropic":
+      return {
+        kind: "anthropic",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveAnthropicConfig(parsed, env)
+      };
+    case "gemini":
+    case "google":
+    case "google-gemini":
+      return {
+        kind: "gemini",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveGeminiConfig(parsed, env)
+      };
+    case "codex":
+    case "codex-cli":
+      return {
+        kind: "codex",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveCodexConfig(parsed, env)
+      };
+    case "mock":
+      return {
+        kind: "mock",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveMockConfig(parsed)
+      };
+    case "vscode":
+    case "vscode-insiders":
+      return {
+        kind: provider,
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders")
+      };
+    case "cli":
+      return {
+        kind: "cli",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveCliConfig(parsed, env)
+      };
+    default:
+      throw new Error(`Unsupported provider '${parsed.provider}' in target '${parsed.name}'`);
   }
-  sections.push(
-    "If any file is missing, fail with ERROR: missing-file <filename> and stop.",
-    "Then apply system_instructions on the user query below."
+}
+function resolveAzureConfig(target, env) {
+  const settings = target.settings ?? {};
+  const endpointSource = settings.endpoint ?? settings.resource ?? settings.resourceName;
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const deploymentSource = settings.deployment ?? settings.deploymentName ?? settings.model;
+  const versionSource = settings.version ?? settings.api_version;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const resourceName = resolveString(endpointSource, env, `${target.name} endpoint`);
+  const apiKey = resolveString(apiKeySource, env, `${target.name} api key`);
+  const deploymentName = resolveString(deploymentSource, env, `${target.name} deployment`);
+  const version = normalizeAzureApiVersion(
+    resolveOptionalString(versionSource, env, `${target.name} api version`)
+  );
+  const temperature = resolveOptionalNumber(temperatureSource, `${target.name} temperature`);
+  const maxOutputTokens = resolveOptionalNumber(
+    maxTokensSource,
+    `${target.name} max output tokens`
   );
-  return sections.join("\n");
-}
-function collectGuidelineFiles(attachments, guidelinePatterns) {
-  if (!attachments || attachments.length === 0) {
-    return [];
-  }
-  const unique = /* @__PURE__ */ new Map();
-  for (const attachment of attachments) {
-    const absolutePath = import_node_path4.default.resolve(attachment);
-    const normalized = absolutePath.split(import_node_path4.default.sep).join("/");
-    if (isGuidelineFile(normalized, guidelinePatterns)) {
-      if (!unique.has(absolutePath)) {
-        unique.set(absolutePath, absolutePath);
-      }
-    }
-  }
-  return Array.from(unique.values());
+  return {
+    resourceName,
+    deploymentName,
+    apiKey,
+    version,
+    temperature,
+    maxOutputTokens
+  };
 }
-function collectAttachmentFiles(attachments) {
-  if (!attachments || attachments.length === 0) {
-    return [];
-  }
-  const unique = /* @__PURE__ */ new Map();
-  for (const attachment of attachments) {
-    const absolutePath = import_node_path4.default.resolve(attachment);
-    if (!unique.has(absolutePath)) {
-      unique.set(absolutePath, absolutePath);
-    }
-  }
-  return Array.from(unique.values());
+function resolveAnthropicConfig(target, env) {
+  const settings = target.settings ?? {};
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const thinkingBudgetSource = settings.thinking_budget ?? settings.thinkingBudget;
+  const apiKey = resolveString(apiKeySource, env, `${target.name} Anthropic api key`);
+  const model = resolveString(modelSource, env, `${target.name} Anthropic model`);
+  return {
+    apiKey,
+    model,
+    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
+    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`),
+    thinkingBudget: resolveOptionalNumber(thinkingBudgetSource, `${target.name} thinking budget`)
+  };
 }
-function pathToFileUri(filePath) {
-  const absolutePath = import_node_path4.default.isAbsolute(filePath) ? filePath : import_node_path4.default.resolve(filePath);
-  const normalizedPath = absolutePath.replace(/\\/g, "/");
-  if (/^[a-zA-Z]:\//.test(normalizedPath)) {
-    return `file:///${normalizedPath}`;
-  }
-  return `file://${normalizedPath}`;
+function resolveGeminiConfig(target, env) {
+  const settings = target.settings ?? {};
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const apiKey = resolveString(apiKeySource, env, `${target.name} Google API key`);
+  const model = resolveOptionalString(modelSource, env, `${target.name} Gemini model`, {
+    allowLiteral: true,
+    optionalEnv: true
+  }) ?? "gemini-2.5-flash";
+  return {
+    apiKey,
+    model,
+    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
+    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`)
+  };
 }
-function normalizeAttachments(attachments) {
-  if (!attachments || attachments.length === 0) {
-    return void 0;
-  }
-  const deduped = /* @__PURE__ */ new Set();
-  for (const attachment of attachments) {
-    deduped.add(import_node_path4.default.resolve(attachment));
-  }
-  return Array.from(deduped);
+function resolveCodexConfig(target, env) {
+  const settings = target.settings ?? {};
+  const executableSource = settings.executable ?? settings.command ?? settings.binary;
+  const argsSource = settings.args ?? settings.arguments;
+  const cwdSource = settings.cwd;
+  const timeoutSource = settings.timeout_seconds ?? settings.timeoutSeconds;
+  const executable = resolveOptionalString(executableSource, env, `${target.name} codex executable`, {
+    allowLiteral: true,
+    optionalEnv: true
+  }) ?? "codex";
+  const args = resolveOptionalStringArray(argsSource, env, `${target.name} codex args`);
+  const cwd = resolveOptionalString(cwdSource, env, `${target.name} codex cwd`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} codex timeout`);
+  return {
+    executable,
+    args,
+    cwd,
+    timeoutMs
+  };
 }
-function mergeAttachments(all) {
-  const deduped = /* @__PURE__ */ new Set();
-  for (const list of all) {
-    if (!list) continue;
-    for (const inputFile of list) {
-      deduped.add(import_node_path4.default.resolve(inputFile));
-    }
-  }
-  return deduped.size > 0 ? Array.from(deduped) : void 0;
+function resolveMockConfig(target) {
+  const settings = target.settings ?? {};
+  const response = typeof settings.response === "string" ? settings.response : void 0;
+  return { response };
 }
-async function ensureVSCodeSubagents(options) {
-  const { kind, count, verbose = false } = options;
-  const vscodeCmd = kind === "vscode-insiders" ? "code-insiders" : "code";
-  const subagentRoot = (0, import_subagent.getSubagentRoot)(vscodeCmd);
-  try {
-    if (verbose) {
-      console.log(`Provisioning ${count} subagent(s) via: subagent ${vscodeCmd} provision`);
-    }
-    const result = await (0, import_subagent.provisionSubagents)({
-      targetRoot: subagentRoot,
-      subagents: count,
-      dryRun: false
-    });
-    if (verbose) {
-      if (result.created.length > 0) {
-        console.log(`Created ${result.created.length} new subagent(s)`);
-      }
-      if (result.skippedExisting.length > 0) {
-        console.log(`Reusing ${result.skippedExisting.length} existing unlocked subagent(s)`);
-      }
-      console.log(`
-total unlocked subagents available: ${result.created.length + result.skippedExisting.length}`);
-    }
-    return {
-      provisioned: true,
-      message: `Provisioned ${count} subagent(s): ${result.created.length} created, ${result.skippedExisting.length} reused`
-    };
-  } catch (error) {
-    const errorMessage = error instanceof Error ? error.message : String(error);
-    if (verbose) {
-      console.warn(`Provisioning failed (continuing anyway): ${errorMessage}`);
-    }
-    return {
-      provisioned: false,
-      message: `Provisioning failed: ${errorMessage}`
-    };
-  }
+function resolveVSCodeConfig(target, env, insiders) {
+  const settings = target.settings ?? {};
+  const workspaceTemplateEnvVar = resolveOptionalLiteralString(settings.workspace_template ?? settings.workspaceTemplate);
+  const workspaceTemplate = workspaceTemplateEnvVar ? resolveOptionalString(workspaceTemplateEnvVar, env, `${target.name} workspace template path`, {
+    allowLiteral: false,
+    optionalEnv: true
+  }) : void 0;
+  const commandSource = settings.vscode_cmd ?? settings.command;
+  const waitSource = settings.wait;
+  const dryRunSource = settings.dry_run ?? settings.dryRun;
+  const subagentRootSource = settings.subagent_root ?? settings.subagentRoot;
+  const defaultCommand = insiders ? "code-insiders" : "code";
+  const command = resolveOptionalLiteralString(commandSource) ?? defaultCommand;
+  return {
+    command,
+    waitForResponse: resolveOptionalBoolean(waitSource) ?? true,
+    dryRun: resolveOptionalBoolean(dryRunSource) ?? false,
+    subagentRoot: resolveOptionalString(subagentRootSource, env, `${target.name} subagent root`, {
+      allowLiteral: true,
+      optionalEnv: true
+    }),
+    workspaceTemplate
+  };
 }
-
-// src/evaluation/providers/codex.ts
-var import_node_child_process2 = require("child_process");
-var import_node_fs3 = require("fs");
-var import_promises4 = require("fs/promises");
-var import_node_os = require("os");
-var import_node_path6 = __toESM(require("path"), 1);
-var import_node_util2 = require("util");
-
-// src/evaluation/providers/preread.ts
-var import_node_path5 = __toESM(require("path"), 1);
-function buildPromptDocument2(request, inputFiles, options) {
-  const parts = [];
-  const guidelineFiles = collectGuidelineFiles2(
-    inputFiles,
-    options?.guidelinePatterns ?? request.guideline_patterns,
-    options?.guidelineOverrides
+function resolveCliConfig(target, env) {
+  const settings = target.settings ?? {};
+  const commandTemplateSource = settings.command_template ?? settings.commandTemplate;
+  const filesFormat = resolveOptionalLiteralString(
+    settings.files_format ?? settings.filesFormat ?? settings.attachments_format ?? settings.attachmentsFormat
   );
-  const inputFilesList = collectInputFiles(inputFiles);
-  const nonGuidelineInputFiles = inputFilesList.filter(
-    (file) => !guidelineFiles.includes(file)
+  const cwd = resolveOptionalString(settings.cwd, env, `${target.name} working directory`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  const envOverrides = resolveEnvOverrides(settings.env, env, target.name);
+  const timeoutMs = resolveTimeoutMs(settings.timeout_seconds ?? settings.timeoutSeconds, `${target.name} timeout`);
+  const healthcheck = resolveCliHealthcheck(settings.healthcheck, env, target.name);
+  const commandTemplate = resolveString(
+    commandTemplateSource,
+    env,
+    `${target.name} CLI command template`,
+    true
   );
-  const prereadBlock = buildMandatoryPrereadBlock2(guidelineFiles, nonGuidelineInputFiles);
-  if (prereadBlock.length > 0) {
-    parts.push("\n", prereadBlock);
-  }
-  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
-  return parts.join("\n").trim();
+  assertSupportedCliPlaceholders(commandTemplate, `${target.name} CLI command template`);
+  return {
+    commandTemplate,
+    filesFormat,
+    cwd,
+    env: envOverrides,
+    timeoutMs,
+    healthcheck
+  };
 }
-function normalizeInputFiles2(inputFiles) {
-  if (!inputFiles || inputFiles.length === 0) {
+function resolveEnvOverrides(source, env, targetName) {
+  if (source === void 0 || source === null) {
     return void 0;
   }
-  const deduped = /* @__PURE__ */ new Map();
-  for (const inputFile of inputFiles) {
-    const absolutePath = import_node_path5.default.resolve(inputFile);
-    if (!deduped.has(absolutePath)) {
-      deduped.set(absolutePath, absolutePath);
-    }
-  }
-  return Array.from(deduped.values());
-}
-function collectGuidelineFiles2(inputFiles, guidelinePatterns, overrides) {
-  if (!inputFiles || inputFiles.length === 0) {
-    return [];
+  if (typeof source !== "object" || Array.isArray(source)) {
+    throw new Error(`${targetName} env overrides must be an object map of strings`);
   }
-  const unique = /* @__PURE__ */ new Map();
-  for (const inputFile of inputFiles) {
-    const absolutePath = import_node_path5.default.resolve(inputFile);
-    if (overrides?.has(absolutePath)) {
-      if (!unique.has(absolutePath)) {
-        unique.set(absolutePath, absolutePath);
-      }
-      continue;
-    }
-    const normalized = absolutePath.split(import_node_path5.default.sep).join("/");
-    if (isGuidelineFile(normalized, guidelinePatterns)) {
-      if (!unique.has(absolutePath)) {
-        unique.set(absolutePath, absolutePath);
-      }
+  const entries = Object.entries(source);
+  const resolved = {};
+  for (const [key, value] of entries) {
+    if (typeof value !== "string") {
+      throw new Error(`${targetName} env override '${key}' must be a string`);
     }
+    const resolvedValue = resolveString(value, env, `${targetName} env override '${key}'`);
+    resolved[key] = resolvedValue;
   }
-  return Array.from(unique.values());
+  return Object.keys(resolved).length > 0 ? resolved : void 0;
 }
-function collectInputFiles(inputFiles) {
-  if (!inputFiles || inputFiles.length === 0) {
-    return [];
+function resolveTimeoutMs(source, description) {
+  const seconds = resolveOptionalNumber(source, `${description} (seconds)`);
+  if (seconds === void 0) {
+    return void 0;
   }
-  const unique = /* @__PURE__ */ new Map();
-  for (const inputFile of inputFiles) {
-    const absolutePath = import_node_path5.default.resolve(inputFile);
-    if (!unique.has(absolutePath)) {
-      unique.set(absolutePath, absolutePath);
-    }
+  if (seconds <= 0) {
+    throw new Error(`${description} must be greater than zero seconds`);
   }
-  return Array.from(unique.values());
+  return Math.floor(seconds * 1e3);
 }
-function buildMandatoryPrereadBlock2(guidelineFiles, inputFiles) {
-  if (guidelineFiles.length === 0 && inputFiles.length === 0) {
-    return "";
-  }
-  const buildList = (files) => files.map((absolutePath) => {
-    const fileName = import_node_path5.default.basename(absolutePath);
-    const fileUri = pathToFileUri2(absolutePath);
-    return `* [${fileName}](${fileUri})`;
-  });
-  const sections = [];
-  if (guidelineFiles.length > 0) {
-    sections.push(`Read all guideline files:
-${buildList(guidelineFiles).join("\n")}.`);
+function resolveCliHealthcheck(source, env, targetName) {
+  if (source === void 0 || source === null) {
+    return void 0;
   }
-  if (inputFiles.length > 0) {
-    sections.push(`Read all input files:
-${buildList(inputFiles).join("\n")}.`);
+  if (typeof source !== "object" || Array.isArray(source)) {
+    throw new Error(`${targetName} healthcheck must be an object`);
   }
-  sections.push(
-    "If any file is missing, fail with ERROR: missing-file <filename> and stop.",
-    "Then apply system_instructions on the user query below."
+  const candidate = source;
+  const type = candidate.type;
+  const timeoutMs = resolveTimeoutMs(
+    candidate.timeout_seconds ?? candidate.timeoutSeconds,
+    `${targetName} healthcheck timeout`
   );
-  return sections.join("\n");
-}
-function pathToFileUri2(filePath) {
-  const absolutePath = import_node_path5.default.isAbsolute(filePath) ? filePath : import_node_path5.default.resolve(filePath);
-  const normalizedPath = absolutePath.replace(/\\/g, "/");
-  if (/^[a-zA-Z]:\//.test(normalizedPath)) {
-    return `file:///${normalizedPath}`;
-  }
-  return `file://${normalizedPath}`;
-}
-
-// src/evaluation/providers/codex.ts
-var execAsync2 = (0, import_node_util2.promisify)(import_node_child_process2.exec);
-var WORKSPACE_PREFIX = "agentv-codex-";
-var PROMPT_FILENAME = "prompt.md";
-var FILES_DIR = "files";
-var JSONL_TYPE_ITEM_COMPLETED = "item.completed";
-var CodexProvider = class {
-  id;
-  kind = "codex";
-  targetName;
-  supportsBatch = false;
-  config;
-  runCodex;
-  environmentCheck;
-  resolvedExecutable;
-  constructor(targetName, config, runner = defaultCodexRunner) {
-    this.id = `codex:${targetName}`;
-    this.targetName = targetName;
-    this.config = config;
-    this.runCodex = runner;
+  if (type === "http") {
+    const url = resolveString(candidate.url, env, `${targetName} healthcheck URL`);
+    return {
+      type: "http",
+      url,
+      timeoutMs
+    };
   }
-  async invoke(request) {
-    if (request.signal?.aborted) {
-      throw new Error("Codex provider request was aborted before execution");
-    }
-    await this.ensureEnvironmentReady();
-    const inputFiles = normalizeInputFiles2(request.inputFiles);
-    const originalGuidelines = new Set(
-      collectGuidelineFiles2(inputFiles, request.guideline_patterns).map((file) => import_node_path6.default.resolve(file))
+  if (type === "command") {
+    const commandTemplate = resolveString(
+      candidate.command_template ?? candidate.commandTemplate,
+      env,
+      `${targetName} healthcheck command template`,
+      true
     );
-    const workspaceRoot = await this.createWorkspace();
-    try {
-      const { mirroredInputFiles, guidelineMirrors } = await this.mirrorInputFiles(
-        inputFiles,
-        workspaceRoot,
-        originalGuidelines
+    assertSupportedCliPlaceholders(commandTemplate, `${targetName} healthcheck command template`);
+    const cwd = resolveOptionalString(candidate.cwd, env, `${targetName} healthcheck cwd`, {
+      allowLiteral: true,
+      optionalEnv: true
+    });
+    return {
+      type: "command",
+      commandTemplate,
+      timeoutMs,
+      cwd
+    };
+  }
+  throw new Error(`${targetName} healthcheck type must be 'http' or 'command'`);
+}
+function assertSupportedCliPlaceholders(template, description) {
+  const placeholders = extractCliPlaceholders(template);
+  for (const placeholder of placeholders) {
+    if (!CLI_PLACEHOLDERS.has(placeholder)) {
+      throw new Error(
+        `${description} includes unsupported placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS).join(", ")}`
       );
-      const promptContent = buildPromptDocument2(request, mirroredInputFiles, {
-        guidelinePatterns: request.guideline_patterns,
-        guidelineOverrides: guidelineMirrors
-      });
-      const promptFile = import_node_path6.default.join(workspaceRoot, PROMPT_FILENAME);
-      await (0, import_promises4.writeFile)(promptFile, promptContent, "utf8");
-      const args = this.buildCodexArgs();
-      const cwd = this.resolveCwd(workspaceRoot);
-      const result = await this.executeCodex(args, cwd, promptContent, request.signal);
-      if (result.timedOut) {
-        throw new Error(
-          `Codex CLI timed out${formatTimeoutSuffix2(this.config.timeoutMs ?? void 0)}`
-        );
-      }
-      if (result.exitCode !== 0) {
-        const detail = pickDetail(result.stderr, result.stdout);
-        const prefix = `Codex CLI exited with code ${result.exitCode}`;
-        throw new Error(detail ? `${prefix}: ${detail}` : prefix);
-      }
-      const parsed = parseCodexJson(result.stdout);
-      const assistantText = extractAssistantText(parsed);
-      return {
-        text: assistantText,
-        raw: {
-          response: parsed,
-          stdout: result.stdout,
-          stderr: result.stderr,
-          exitCode: result.exitCode,
-          args,
-          executable: this.resolvedExecutable ?? this.config.executable,
-          promptFile,
-          workspace: workspaceRoot,
-          inputFiles: mirroredInputFiles
-        }
-      };
-    } finally {
-      await this.cleanupWorkspace(workspaceRoot);
     }
   }
-  async ensureEnvironmentReady() {
-    if (!this.environmentCheck) {
-      this.environmentCheck = this.validateEnvironment();
+}
+function extractCliPlaceholders(template) {
+  const matches = template.matchAll(/\{([A-Z_]+)\}/g);
+  const results = [];
+  for (const match of matches) {
+    if (match[1]) {
+      results.push(match[1]);
     }
-    await this.environmentCheck;
   }
-  async validateEnvironment() {
-    this.resolvedExecutable = await locateExecutable(this.config.executable);
+  return results;
+}
+function resolveString(source, env, description, allowLiteral = false) {
+  const value = resolveOptionalString(source, env, description, {
+    allowLiteral,
+    optionalEnv: false
+  });
+  if (value === void 0) {
+    throw new Error(`${description} is required`);
   }
-  resolveCwd(workspaceRoot) {
-    if (!this.config.cwd) {
-      return workspaceRoot;
-    }
-    return import_node_path6.default.resolve(this.config.cwd);
+  return value;
+}
+function resolveOptionalString(source, env, description, options) {
+  if (source === void 0 || source === null) {
+    return void 0;
   }
-  buildCodexArgs() {
-    const args = ["--ask-for-approval", "never", "exec", "--json", "--color", "never", "--skip-git-repo-check"];
-    if (this.config.args && this.config.args.length > 0) {
-      args.push(...this.config.args);
-    }
-    args.push("-");
-    return args;
+  if (typeof source !== "string") {
+    throw new Error(`${description} must be a string`);
   }
-  async executeCodex(args, cwd, promptContent, signal) {
-    try {
-      return await this.runCodex({
-        executable: this.resolvedExecutable ?? this.config.executable,
-        args,
-        cwd,
-        prompt: promptContent,
-        timeoutMs: this.config.timeoutMs,
-        env: process.env,
-        signal
-      });
-    } catch (error) {
-      const err = error;
-      if (err.code === "ENOENT") {
-        throw new Error(
-          `Codex executable '${this.config.executable}' was not found. Update the target settings.executable or add it to PATH.`
-        );
-      }
-      throw error;
-    }
+  const trimmed = source.trim();
+  if (trimmed.length === 0) {
+    return void 0;
   }
-  async mirrorInputFiles(inputFiles, workspaceRoot, guidelineOriginals) {
-    if (!inputFiles || inputFiles.length === 0) {
-      return {
-        mirroredInputFiles: void 0,
-        guidelineMirrors: /* @__PURE__ */ new Set()
-      };
-    }
-    const filesRoot = import_node_path6.default.join(workspaceRoot, FILES_DIR);
-    await (0, import_promises4.mkdir)(filesRoot, { recursive: true });
-    const mirrored = [];
-    const guidelineMirrors = /* @__PURE__ */ new Set();
-    const nameCounts = /* @__PURE__ */ new Map();
-    for (const inputFile of inputFiles) {
-      const absoluteSource = import_node_path6.default.resolve(inputFile);
-      const baseName = import_node_path6.default.basename(absoluteSource);
-      const count = nameCounts.get(baseName) ?? 0;
-      nameCounts.set(baseName, count + 1);
-      const finalName = count === 0 ? baseName : `${baseName}.${count}`;
-      const destination = import_node_path6.default.join(filesRoot, finalName);
-      await (0, import_promises4.copyFile)(absoluteSource, destination);
-      const resolvedDestination = import_node_path6.default.resolve(destination);
-      mirrored.push(resolvedDestination);
-      if (guidelineOriginals.has(absoluteSource)) {
-        guidelineMirrors.add(resolvedDestination);
-      }
+  const envValue = env[trimmed];
+  if (envValue !== void 0) {
+    if (envValue.trim().length === 0) {
+      throw new Error(`Environment variable '${trimmed}' for ${description} is empty`);
     }
-    return {
-      mirroredInputFiles: mirrored,
-      guidelineMirrors
-    };
-  }
-  async createWorkspace() {
-    return await (0, import_promises4.mkdtemp)(import_node_path6.default.join((0, import_node_os.tmpdir)(), WORKSPACE_PREFIX));
+    return envValue;
   }
-  async cleanupWorkspace(workspaceRoot) {
-    try {
-      await (0, import_promises4.rm)(workspaceRoot, { recursive: true, force: true });
-    } catch {
+  const allowLiteral = options?.allowLiteral ?? false;
+  const optionalEnv = options?.optionalEnv ?? false;
+  const looksLikeEnv = isLikelyEnvReference(trimmed);
+  if (looksLikeEnv) {
+    if (optionalEnv) {
+      return void 0;
+    }
+    if (!allowLiteral) {
+      throw new Error(`Environment variable '${trimmed}' required for ${description} is not set`);
     }
   }
-};
-async function locateExecutable(candidate) {
-  const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
-  if (includesPathSeparator) {
-    const resolved = import_node_path6.default.isAbsolute(candidate) ? candidate : import_node_path6.default.resolve(candidate);
-    const executablePath = await ensureWindowsExecutableVariant(resolved);
-    await (0, import_promises4.access)(executablePath, import_node_fs3.constants.F_OK);
-    return executablePath;
+  return trimmed;
+}
+function resolveOptionalLiteralString(source) {
+  if (source === void 0 || source === null) {
+    return void 0;
   }
-  const locator = process.platform === "win32" ? "where" : "which";
-  try {
-    const { stdout } = await execAsync2(`${locator} ${candidate}`);
-    const lines = stdout.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
-    const preferred = selectExecutableCandidate(lines);
-    if (preferred) {
-      const executablePath = await ensureWindowsExecutableVariant(preferred);
-      await (0, import_promises4.access)(executablePath, import_node_fs3.constants.F_OK);
-      return executablePath;
-    }
-  } catch {
+  if (typeof source !== "string") {
+    throw new Error("expected string value");
   }
-  throw new Error(`Codex executable '${candidate}' was not found on PATH`);
+  const trimmed = source.trim();
+  return trimmed.length > 0 ? trimmed : void 0;
 }
-function selectExecutableCandidate(candidates) {
-  if (candidates.length === 0) {
+function resolveOptionalNumber(source, description) {
+  if (source === void 0 || source === null || source === "") {
     return void 0;
   }
-  if (process.platform !== "win32") {
-    return candidates[0];
+  if (typeof source === "number") {
+    return Number.isFinite(source) ? source : void 0;
   }
-  const extensions = getWindowsExecutableExtensions();
-  for (const ext of extensions) {
-    const match = candidates.find((candidate) => candidate.toLowerCase().endsWith(ext));
-    if (match) {
-      return match;
+  if (typeof source === "string") {
+    const numeric = Number(source);
+    if (Number.isFinite(numeric)) {
+      return numeric;
     }
   }
-  return candidates[0];
+  throw new Error(`${description} must be a number`);
 }
-async function ensureWindowsExecutableVariant(candidate) {
-  if (process.platform !== "win32") {
-    return candidate;
+function resolveOptionalBoolean(source) {
+  if (source === void 0 || source === null || source === "") {
+    return void 0;
   }
-  if (hasExecutableExtension(candidate)) {
-    return candidate;
+  if (typeof source === "boolean") {
+    return source;
   }
-  const extensions = getWindowsExecutableExtensions();
-  for (const ext of extensions) {
-    const withExtension = `${candidate}${ext}`;
-    try {
-      await (0, import_promises4.access)(withExtension, import_node_fs3.constants.F_OK);
-      return withExtension;
-    } catch {
+  if (typeof source === "string") {
+    const lowered = source.trim().toLowerCase();
+    if (lowered === "true" || lowered === "1") {
+      return true;
+    }
+    if (lowered === "false" || lowered === "0") {
+      return false;
     }
   }
-  return candidate;
+  throw new Error("expected boolean value");
 }
-function hasExecutableExtension(candidate) {
-  const lower = candidate.toLowerCase();
-  return getWindowsExecutableExtensions().some((ext) => lower.endsWith(ext));
+function isLikelyEnvReference(value) {
+  return /^[A-Z0-9_]+$/.test(value);
 }
-var DEFAULT_WINDOWS_EXTENSIONS = [".com", ".exe", ".bat", ".cmd", ".ps1"];
-function getWindowsExecutableExtensions() {
-  if (process.platform !== "win32") {
-    return [];
+function resolveOptionalStringArray(source, env, description) {
+  if (source === void 0 || source === null) {
+    return void 0;
   }
-  const fromEnv = process.env.PATHEXT?.split(";").map((ext) => ext.trim().toLowerCase()).filter((ext) => ext.length > 0);
-  return fromEnv && fromEnv.length > 0 ? fromEnv : DEFAULT_WINDOWS_EXTENSIONS;
-}
-function parseCodexJson(output) {
-  const trimmed = output.trim();
-  if (trimmed.length === 0) {
-    throw new Error("Codex CLI produced no output in --json mode");
+  if (!Array.isArray(source)) {
+    throw new Error(`${description} must be an array of strings`);
   }
-  try {
-    return JSON.parse(trimmed);
-  } catch {
-    const lineObjects = parseJsonLines(trimmed);
-    if (lineObjects) {
-      return lineObjects;
+  if (source.length === 0) {
+    return void 0;
+  }
+  const resolved = [];
+  for (let i = 0; i < source.length; i++) {
+    const item = source[i];
+    if (typeof item !== "string") {
+      throw new Error(`${description}[${i}] must be a string`);
     }
-    const lastBrace = trimmed.lastIndexOf("{");
-    if (lastBrace >= 0) {
-      const candidate = trimmed.slice(lastBrace);
-      try {
-        return JSON.parse(candidate);
-      } catch {
+    const trimmed = item.trim();
+    if (trimmed.length === 0) {
+      throw new Error(`${description}[${i}] cannot be empty`);
+    }
+    const envValue = env[trimmed];
+    if (envValue !== void 0) {
+      if (envValue.trim().length === 0) {
+        throw new Error(`Environment variable '${trimmed}' for ${description}[${i}] is empty`);
       }
+      resolved.push(envValue);
+    } else {
+      resolved.push(trimmed);
     }
-    const preview = trimmed.slice(0, 200);
-    throw new Error(`Codex CLI emitted invalid JSON: ${preview}${trimmed.length > 200 ? "\u2026" : ""}`);
   }
+  return resolved.length > 0 ? resolved : void 0;
 }
-function extractAssistantText(parsed) {
-  if (Array.isArray(parsed)) {
-    const text = extractFromEventStream(parsed);
-    if (text) {
-      return text;
-    }
-  }
-  if (!parsed || typeof parsed !== "object") {
-    throw new Error("Codex CLI JSON response did not include an assistant message");
-  }
-  const record = parsed;
-  const eventText = extractFromEvent(record);
-  if (eventText) {
-    return eventText;
+
+// src/evaluation/providers/vscode.ts
+var import_promises4 = require("fs/promises");
+var import_node_path6 = __toESM(require("path"), 1);
+var import_subagent = require("subagent");
+var VSCodeProvider = class {
+  id;
+  kind;
+  targetName;
+  supportsBatch = true;
+  config;
+  constructor(targetName, config, kind) {
+    this.id = `${kind}:${targetName}`;
+    this.kind = kind;
+    this.targetName = targetName;
+    this.config = config;
   }
-  const messages = Array.isArray(record.messages) ? record.messages : void 0;
-  if (messages) {
-    for (let index = messages.length - 1; index >= 0; index -= 1) {
-      const entry = messages[index];
-      if (!entry || typeof entry !== "object") {
-        continue;
-      }
-      const role = entry.role;
-      if (role !== "assistant") {
-        continue;
-      }
-      const content = entry.content;
-      const flattened = flattenContent(content);
-      if (flattened) {
-        return flattened;
-      }
+  async invoke(request) {
+    if (request.signal?.aborted) {
+      throw new Error("VS Code provider request was aborted before dispatch");
+    }
+    const inputFiles = normalizeAttachments(request.inputFiles);
+    const promptContent = buildPromptDocument2(request, inputFiles, request.guideline_patterns);
+    const session = await (0, import_subagent.dispatchAgentSession)({
+      userQuery: promptContent,
+      extraAttachments: inputFiles,
+      wait: this.config.waitForResponse,
+      dryRun: this.config.dryRun,
+      vscodeCmd: this.config.command,
+      subagentRoot: this.config.subagentRoot,
+      workspaceTemplate: this.config.workspaceTemplate,
+      silent: true
+    });
+    if (session.exitCode !== 0 || !session.responseFile) {
+      const failure = session.error ?? "VS Code subagent did not produce a response";
+      throw new Error(failure);
+    }
+    if (this.config.dryRun) {
+      return {
+        text: "",
+        raw: {
+          session,
+          inputFiles
+        }
+      };
     }
+    const responseText = await (0, import_promises4.readFile)(session.responseFile, "utf8");
+    return {
+      text: responseText,
+      raw: {
+        session,
+        inputFiles
+      }
+    };
   }
-  const response = record.response;
-  if (response && typeof response === "object") {
-    const content = response.content;
-    const flattened = flattenContent(content);
-    if (flattened) {
-      return flattened;
+  async invokeBatch(requests) {
+    if (requests.length === 0) {
+      return [];
+    }
+    const normalizedRequests = requests.map((req) => ({
+      request: req,
+      inputFiles: normalizeAttachments(req.inputFiles)
+    }));
+    const combinedInputFiles = mergeAttachments(
+      normalizedRequests.map(({ inputFiles }) => inputFiles)
+    );
+    const userQueries = normalizedRequests.map(
+      ({ request, inputFiles }) => buildPromptDocument2(request, inputFiles, request.guideline_patterns)
+    );
+    const session = await (0, import_subagent.dispatchBatchAgent)({
+      userQueries,
+      extraAttachments: combinedInputFiles,
+      wait: this.config.waitForResponse,
+      dryRun: this.config.dryRun,
+      vscodeCmd: this.config.command,
+      subagentRoot: this.config.subagentRoot,
+      workspaceTemplate: this.config.workspaceTemplate,
+      silent: true
+    });
+    if (session.exitCode !== 0 || !session.responseFiles) {
+      const failure = session.error ?? "VS Code subagent did not produce batch responses";
+      throw new Error(failure);
+    }
+    if (this.config.dryRun) {
+      return normalizedRequests.map(({ inputFiles }) => ({
+        text: "",
+        raw: {
+          session,
+          inputFiles,
+          allInputFiles: combinedInputFiles
+        }
+      }));
+    }
+    if (session.responseFiles.length !== requests.length) {
+      throw new Error(
+        `VS Code batch returned ${session.responseFiles.length} responses for ${requests.length} requests`
+      );
     }
-  }
-  const output = record.output;
-  const flattenedOutput = flattenContent(output);
-  if (flattenedOutput) {
-    return flattenedOutput;
-  }
-  throw new Error("Codex CLI JSON response did not include an assistant message");
-}
-function extractFromEventStream(events) {
-  for (let index = events.length - 1; index >= 0; index -= 1) {
-    const candidate = events[index];
-    const text = extractFromEvent(candidate);
-    if (text) {
-      return text;
+    const responses = [];
+    for (const [index, responseFile] of session.responseFiles.entries()) {
+      const responseText = await (0, import_promises4.readFile)(responseFile, "utf8");
+      responses.push({
+        text: responseText,
+        raw: {
+          session,
+          inputFiles: normalizedRequests[index]?.inputFiles,
+          allInputFiles: combinedInputFiles,
+          responseFile
+        }
+      });
     }
+    return responses;
   }
-  return void 0;
+};
+function buildPromptDocument2(request, attachments, guidelinePatterns) {
+  const parts = [];
+  const guidelineFiles = collectGuidelineFiles2(attachments, guidelinePatterns);
+  const attachmentFiles = collectAttachmentFiles(attachments);
+  const nonGuidelineAttachments = attachmentFiles.filter(
+    (file) => !guidelineFiles.includes(file)
+  );
+  const prereadBlock = buildMandatoryPrereadBlock2(guidelineFiles, nonGuidelineAttachments);
+  if (prereadBlock.length > 0) {
+    parts.push("\n", prereadBlock);
+  }
+  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
+  return parts.join("\n").trim();
 }
-function extractFromEvent(event) {
-  if (!event || typeof event !== "object") {
-    return void 0;
+function buildMandatoryPrereadBlock2(guidelineFiles, attachmentFiles) {
+  if (guidelineFiles.length === 0 && attachmentFiles.length === 0) {
+    return "";
   }
-  const record = event;
-  const type = typeof record.type === "string" ? record.type : void 0;
-  if (type === JSONL_TYPE_ITEM_COMPLETED) {
-    const item = record.item;
-    const text = extractFromItem(item);
-    if (text) {
-      return text;
-    }
+  const buildList = (files) => files.map((absolutePath) => {
+    const fileName = import_node_path6.default.basename(absolutePath);
+    const fileUri = pathToFileUri2(absolutePath);
+    return `* [${fileName}](${fileUri})`;
+  });
+  const sections = [];
+  if (guidelineFiles.length > 0) {
+    sections.push(`Read all guideline files:
+${buildList(guidelineFiles).join("\n")}.`);
   }
-  const output = record.output ?? record.content;
-  const flattened = flattenContent(output);
-  if (flattened) {
-    return flattened;
+  if (attachmentFiles.length > 0) {
+    sections.push(`Read all attachment files:
+${buildList(attachmentFiles).join("\n")}.`);
   }
-  return void 0;
+  sections.push(
+    "If any file is missing, fail with ERROR: missing-file <filename> and stop.",
+    "Then apply system_instructions on the user query below."
+  );
+  return sections.join("\n");
 }
-function extractFromItem(item) {
-  if (!item || typeof item !== "object") {
-    return void 0;
+function collectGuidelineFiles2(attachments, guidelinePatterns) {
+  if (!attachments || attachments.length === 0) {
+    return [];
   }
-  const record = item;
-  const itemType = typeof record.type === "string" ? record.type : void 0;
-  if (itemType === "agent_message" || itemType === "response" || itemType === "output") {
-    const text = flattenContent(record.text ?? record.content ?? record.output);
-    if (text) {
-      return text;
+  const unique = /* @__PURE__ */ new Map();
+  for (const attachment of attachments) {
+    const absolutePath = import_node_path6.default.resolve(attachment);
+    const normalized = absolutePath.split(import_node_path6.default.sep).join("/");
+    if (isGuidelineFile(normalized, guidelinePatterns)) {
+      if (!unique.has(absolutePath)) {
+        unique.set(absolutePath, absolutePath);
+      }
     }
   }
-  return void 0;
+  return Array.from(unique.values());
 }
-function flattenContent(value) {
-  if (typeof value === "string") {
-    return value;
+function collectAttachmentFiles(attachments) {
+  if (!attachments || attachments.length === 0) {
+    return [];
   }
-  if (Array.isArray(value)) {
-    const parts = value.map((segment) => {
-      if (typeof segment === "string") {
-        return segment;
-      }
-      if (segment && typeof segment === "object" && "text" in segment) {
-        const text = segment.text;
-        return typeof text === "string" ? text : void 0;
-      }
-      return void 0;
-    }).filter((part) => typeof part === "string" && part.length > 0);
-    return parts.length > 0 ? parts.join(" \n") : void 0;
+  const unique = /* @__PURE__ */ new Map();
+  for (const attachment of attachments) {
+    const absolutePath = import_node_path6.default.resolve(attachment);
+    if (!unique.has(absolutePath)) {
+      unique.set(absolutePath, absolutePath);
+    }
   }
-  if (value && typeof value === "object" && "text" in value) {
-    const text = value.text;
-    return typeof text === "string" ? text : void 0;
+  return Array.from(unique.values());
+}
+function pathToFileUri2(filePath) {
+  const absolutePath = import_node_path6.default.isAbsolute(filePath) ? filePath : import_node_path6.default.resolve(filePath);
+  const normalizedPath = absolutePath.replace(/\\/g, "/");
+  if (/^[a-zA-Z]:\//.test(normalizedPath)) {
+    return `file:///${normalizedPath}`;
   }
-  return void 0;
+  return `file://${normalizedPath}`;
 }
-function parseJsonLines(output) {
-  const lines = output.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
-  if (lines.length <= 1) {
+function normalizeAttachments(attachments) {
+  if (!attachments || attachments.length === 0) {
     return void 0;
   }
-  const parsed = [];
-  for (const line of lines) {
-    try {
-      parsed.push(JSON.parse(line));
-    } catch {
-      return void 0;
-    }
-  }
-  return parsed;
-}
-function pickDetail(stderr, stdout) {
-  const errorText = stderr.trim();
-  if (errorText.length > 0) {
-    return errorText;
+  const deduped = /* @__PURE__ */ new Set();
+  for (const attachment of attachments) {
+    deduped.add(import_node_path6.default.resolve(attachment));
   }
-  const stdoutText = stdout.trim();
-  return stdoutText.length > 0 ? stdoutText : void 0;
+  return Array.from(deduped);
 }
-function formatTimeoutSuffix2(timeoutMs) {
-  if (!timeoutMs || timeoutMs <= 0) {
-    return "";
+function mergeAttachments(all) {
+  const deduped = /* @__PURE__ */ new Set();
+  for (const list of all) {
+    if (!list) continue;
+    for (const inputFile of list) {
+      deduped.add(import_node_path6.default.resolve(inputFile));
+    }
   }
-  const seconds = Math.ceil(timeoutMs / 1e3);
-  return ` after ${seconds}s`;
+  return deduped.size > 0 ? Array.from(deduped) : void 0;
 }
-async function defaultCodexRunner(options) {
-  return await new Promise((resolve, reject) => {
-    const child = (0, import_node_child_process2.spawn)(options.executable, options.args, {
-      cwd: options.cwd,
-      env: options.env,
-      stdio: ["pipe", "pipe", "pipe"],
-      shell: shouldShellExecute(options.executable)
-    });
-    let stdout = "";
-    let stderr = "";
-    let timedOut = false;
-    const onAbort = () => {
-      child.kill("SIGTERM");
-    };
-    if (options.signal) {
-      if (options.signal.aborted) {
-        onAbort();
-      } else {
-        options.signal.addEventListener("abort", onAbort, { once: true });
-      }
-    }
-    let timeoutHandle;
-    if (options.timeoutMs && options.timeoutMs > 0) {
-      timeoutHandle = setTimeout(() => {
-        timedOut = true;
-        child.kill("SIGTERM");
-      }, options.timeoutMs);
-      timeoutHandle.unref?.();
+async function ensureVSCodeSubagents(options) {
+  const { kind, count, verbose = false } = options;
+  const vscodeCmd = kind === "vscode-insiders" ? "code-insiders" : "code";
+  const subagentRoot = (0, import_subagent.getSubagentRoot)(vscodeCmd);
+  try {
+    if (verbose) {
+      console.log(`Provisioning ${count} subagent(s) via: subagent ${vscodeCmd} provision`);
     }
-    child.stdout.setEncoding("utf8");
-    child.stdout.on("data", (chunk) => {
-      stdout += chunk;
-    });
-    child.stderr.setEncoding("utf8");
-    child.stderr.on("data", (chunk) => {
-      stderr += chunk;
+    const result = await (0, import_subagent.provisionSubagents)({
+      targetRoot: subagentRoot,
+      subagents: count,
+      dryRun: false
     });
-    child.stdin.end(options.prompt);
-    const cleanup = () => {
-      if (timeoutHandle) {
-        clearTimeout(timeoutHandle);
+    if (verbose) {
+      if (result.created.length > 0) {
+        console.log(`Created ${result.created.length} new subagent(s)`);
       }
-      if (options.signal) {
-        options.signal.removeEventListener("abort", onAbort);
+      if (result.skippedExisting.length > 0) {
+        console.log(`Reusing ${result.skippedExisting.length} existing unlocked subagent(s)`);
       }
+      console.log(`
+total unlocked subagents available: ${result.created.length + result.skippedExisting.length}`);
+    }
+    return {
+      provisioned: true,
+      message: `Provisioned ${count} subagent(s): ${result.created.length} created, ${result.skippedExisting.length} reused`
+    };
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    if (verbose) {
+      console.warn(`Provisioning failed (continuing anyway): ${errorMessage}`);
+    }
+    return {
+      provisioned: false,
+      message: `Provisioning failed: ${errorMessage}`
     };
-    child.on("error", (error) => {
-      cleanup();
-      reject(error);
-    });
-    child.on("close", (code) => {
-      cleanup();
-      resolve({
-        stdout,
-        stderr,
-        exitCode: typeof code === "number" ? code : -1,
-        timedOut
-      });
-    });
-  });
-}
-function shouldShellExecute(executable) {
-  if (process.platform !== "win32") {
-    return false;
   }
-  const lower = executable.toLowerCase();
-  return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
 }
 
 // src/evaluation/providers/targets-file.ts