agentv 0.2.11 → 0.5.1

package/dist/{chunk-7MGIZBZG.js → chunk-HPH4YWGU.js}

@@ -585,7 +585,7 @@ var require_utc = __commonJS({
 import { Command } from "commander";
 import { readFileSync as readFileSync2 } from "node:fs";
 
-// ../../packages/core/dist/chunk-P4GOYWYH.js
+// ../../packages/core/dist/chunk-NL7K4CAK.js
 import { constants } from "node:fs";
 import { access } from "node:fs/promises";
 import path from "node:path";
@@ -692,6 +692,8 @@ var KNOWN_PROVIDERS = [
   "azure",
   "anthropic",
   "gemini",
+  "codex",
+  "cli",
   "mock",
   "vscode",
   "vscode-insiders"
@@ -703,6 +705,8 @@ var PROVIDER_ALIASES = [
   // alias for "gemini"
   "google-gemini",
   // alias for "gemini"
+  "codex-cli",
+  // alias for "codex"
   "openai",
   // legacy/future support
   "bedrock",
@@ -5031,6 +5035,18 @@ var _c = pr();
 var ya = new Error("Agent description must be at least 20 characters (explain in detail what the agent does)");
 var ba = new Error("Agent definition is the prompt you give to the LLM for the agent. It must be detailed and at least 100 characters");
 
+// ../../packages/core/dist/index.js
+import { exec as execWithCallback } from "node:child_process";
+import path22 from "node:path";
+import { promisify as promisify2 } from "node:util";
+import { exec as execCallback, spawn as spawn2 } from "node:child_process";
+import { constants as constants22 } from "node:fs";
+import { access as access22, copyFile as copyFile2, mkdtemp, mkdir as mkdir3, rm as rm2, writeFile as writeFile3 } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path42 from "node:path";
+import { promisify as promisify22 } from "node:util";
+import path32 from "node:path";
+
 // ../../node_modules/.pnpm/zod@3.25.76/node_modules/zod/v3/external.js
 var external_exports = {};
 __export(external_exports, {
@@ -9074,7 +9090,7 @@ var NEVER = INVALID;
 
 // ../../packages/core/dist/index.js
 import { readFile as readFile22 } from "node:fs/promises";
-import path22 from "node:path";
+import path52 from "node:path";
 
 // ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/vscode/agentDispatch.js
 import { exec, spawn } from "child_process";
@@ -11012,14 +11028,14 @@ async function provisionSubagents(options) {
 }
 
 // ../../packages/core/dist/index.js
-import { constants as constants22 } from "node:fs";
-import { access as access22, readFile as readFile3 } from "node:fs/promises";
-import path32 from "node:path";
+import { constants as constants32 } from "node:fs";
+import { access as access32, readFile as readFile3 } from "node:fs/promises";
+import path62 from "node:path";
 import { parse as parse22 } from "yaml";
 import { randomUUID } from "node:crypto";
 import { createHash, randomUUID as randomUUID2 } from "node:crypto";
-import { mkdir as mkdir3, writeFile as writeFile22 } from "node:fs/promises";
-import path42 from "node:path";
+import { mkdir as mkdir22, readFile as readFile4, writeFile as writeFile22 } from "node:fs/promises";
+import path72 from "node:path";
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
 var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
 function isTestMessageRole(value) {
@@ -11059,10 +11075,10 @@ function isTestMessage(value) {
   }
   return candidate.content.every(isJsonObject);
 }
-var GRADER_KIND_VALUES = ["heuristic", "llm_judge"];
-var GRADER_KIND_SET = new Set(GRADER_KIND_VALUES);
-function isGraderKind(value) {
-  return typeof value === "string" && GRADER_KIND_SET.has(value);
+var EVALUATOR_KIND_VALUES = ["code", "llm_judge"];
+var EVALUATOR_KIND_SET = new Set(EVALUATOR_KIND_VALUES);
+function isEvaluatorKind(value) {
+  return typeof value === "string" && EVALUATOR_KIND_SET.has(value);
 }
 var CODE_BLOCK_PATTERN = /```[\s\S]*?```/g;
 var ANSI_YELLOW = "\x1B[33m";
@@ -11159,7 +11175,7 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
   if (!Array.isArray(rawTestcases)) {
     throw new Error(`Invalid test file format: ${evalFilePath} - missing 'evalcases' field`);
   }
-  const globalGrader = coerceGrader(suite.grader) ?? "llm_judge";
+  const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm_judge";
   const results = [];
   for (const rawEvalcase of rawTestcases) {
     if (!isJsonObject(rawEvalcase)) {
@@ -11282,7 +11298,8 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
     const assistantContent = assistantMessages[0]?.content;
     const expectedAssistantRaw = await resolveAssistantContent(assistantContent, searchRoots, verbose);
     const userTextPrompt = userTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
-    const testCaseGrader = coerceGrader(evalcase.grader) ?? globalGrader;
+    const testCaseEvaluatorKind = coerceEvaluator(evalcase.evaluator, id) ?? globalEvaluator;
+    const evaluators = await parseEvaluators(evalcase, searchRoots, id ?? "unknown");
     const userFilePaths = [];
     for (const segment of userSegments) {
       if (segment.type === "file" && typeof segment.resolvedPath === "string") {
@@ -11305,7 +11322,8 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
       file_paths: allFilePaths,
       code_snippets: codeSnippets,
       outcome,
-      grader: testCaseGrader
+      evaluator: testCaseEvaluatorKind,
+      evaluators
     };
     if (verbose) {
       console.log(`
@@ -11466,14 +11484,88 @@ async function resolveAssistantContent(content, searchRoots, verbose) {
   }
   return parts.join(" ");
 }
-function coerceGrader(candidate) {
+async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
+  const execution = rawEvalCase.execution;
+  const candidateEvaluators = isJsonObject(execution) ? execution.evaluators ?? rawEvalCase.evaluators : rawEvalCase.evaluators;
+  if (candidateEvaluators === void 0) {
+    return void 0;
+  }
+  if (!Array.isArray(candidateEvaluators)) {
+    logWarning(`Skipping evaluators for '${evalId}': expected array`);
+    return void 0;
+  }
+  const evaluators = [];
+  for (const rawEvaluator of candidateEvaluators) {
+    if (!isJsonObject(rawEvaluator)) {
+      logWarning(`Skipping invalid evaluator entry for '${evalId}' (expected object)`);
+      continue;
+    }
+    const name = asString(rawEvaluator.name);
+    const typeValue = rawEvaluator.type;
+    if (!name || !isEvaluatorKind(typeValue)) {
+      logWarning(`Skipping evaluator with invalid name/type in '${evalId}'`);
+      continue;
+    }
+    if (typeValue === "code") {
+      const script = asString(rawEvaluator.script);
+      if (!script) {
+        logWarning(`Skipping code evaluator '${name}' in '${evalId}': missing script`);
+        continue;
+      }
+      const cwd = asString(rawEvaluator.cwd);
+      let resolvedCwd;
+      if (cwd) {
+        const resolved = await resolveFileReference(cwd, searchRoots);
+        if (resolved.resolvedPath) {
+          resolvedCwd = path8.resolve(resolved.resolvedPath);
+        } else {
+          logWarning(
+            `Code evaluator '${name}' in '${evalId}': cwd not found (${resolved.displayPath})`,
+            resolved.attempted.length > 0 ? resolved.attempted.map((attempt) => `  Tried: ${attempt}`) : void 0
+          );
+        }
+      }
+      evaluators.push({
+        name,
+        type: "code",
+        script,
+        cwd,
+        resolvedCwd
+      });
+      continue;
+    }
+    const prompt = asString(rawEvaluator.prompt);
+    let promptPath;
+    if (prompt) {
+      const resolved = await resolveFileReference(prompt, searchRoots);
+      if (resolved.resolvedPath) {
+        promptPath = path8.resolve(resolved.resolvedPath);
+      } else {
+        logWarning(
+          `Inline prompt used for evaluator '${name}' in '${evalId}' (file not found: ${resolved.displayPath})`,
+          resolved.attempted.length > 0 ? resolved.attempted.map((attempt) => `  Tried: ${attempt}`) : void 0
+        );
+      }
+    }
+    const model = asString(rawEvaluator.model);
+    evaluators.push({
+      name,
+      type: "llm_judge",
+      prompt,
+      promptPath,
+      model
+    });
+  }
+  return evaluators.length > 0 ? evaluators : void 0;
+}
+function coerceEvaluator(candidate, contextId) {
   if (typeof candidate !== "string") {
     return void 0;
   }
-  if (isGraderKind(candidate)) {
+  if (isEvaluatorKind(candidate)) {
     return candidate;
   }
-  logWarning(`Unknown grader '${candidate}', falling back to default`);
+  logWarning(`Unknown evaluator '${candidate}' in ${contextId}, falling back to default`);
   return void 0;
 }
 function logWarning(message, details) {
@@ -11665,194 +11757,1003 @@ var GeminiProvider = class {
     return mapResponse(ensureChatResponse(response));
   }
 };
-var DEFAULT_MOCK_RESPONSE = '{"answer":"Mock provider response. Configure targets.yaml to supply a custom value."}';
-var MockProvider = class {
+var execAsync2 = promisify2(execWithCallback);
+var DEFAULT_MAX_BUFFER = 10 * 1024 * 1024;
+async function defaultCommandRunner(command, options) {
+  const execOptions = {
+    cwd: options.cwd,
+    env: options.env,
+    timeout: options.timeoutMs,
+    signal: options.signal,
+    maxBuffer: DEFAULT_MAX_BUFFER,
+    shell: process.platform === "win32" ? "powershell.exe" : void 0
+  };
+  try {
+    const { stdout, stderr } = await execAsync2(command, execOptions);
+    return {
+      stdout,
+      stderr,
+      exitCode: 0,
+      failed: false,
+      timedOut: false,
+      signal: null
+    };
+  } catch (error) {
+    const execError = error;
+    return {
+      stdout: execError.stdout ?? "",
+      stderr: execError.stderr ?? "",
+      exitCode: typeof execError.code === "number" ? execError.code : null,
+      failed: true,
+      timedOut: execError.timedOut === true || execError.killed === true,
+      signal: execError.signal ?? null
+    };
+  }
+}
+var CliProvider = class {
   id;
-  kind = "mock";
+  kind = "cli";
   targetName;
-  cannedResponse;
-  delayMs;
-  delayMinMs;
-  delayMaxMs;
-  constructor(targetName, config) {
-    this.id = `mock:${targetName}`;
+  supportsBatch = false;
+  config;
+  runCommand;
+  healthcheckPromise;
+  constructor(targetName, config, runner = defaultCommandRunner) {
     this.targetName = targetName;
-    this.cannedResponse = config.response ?? DEFAULT_MOCK_RESPONSE;
-    this.delayMs = config.delayMs ?? 0;
-    this.delayMinMs = config.delayMinMs ?? 0;
-    this.delayMaxMs = config.delayMaxMs ?? 0;
+    this.id = `cli:${targetName}`;
+    this.config = config;
+    this.runCommand = runner;
   }
   async invoke(request) {
-    const delay = this.calculateDelay();
-    if (delay > 0) {
-      await new Promise((resolve) => setTimeout(resolve, delay));
+    if (request.signal?.aborted) {
+      throw new Error("CLI provider request was aborted before execution");
+    }
+    await this.ensureHealthy(request.signal);
+    const templateValues = buildTemplateValues(request, this.config);
+    const renderedCommand = renderTemplate(this.config.commandTemplate, templateValues);
+    const env = this.config.env ? { ...process.env, ...this.config.env } : process.env;
+    const result = await this.runCommand(renderedCommand, {
+      cwd: this.config.cwd,
+      env,
+      timeoutMs: this.config.timeoutMs,
+      signal: request.signal
+    });
+    if (result.failed || (result.exitCode ?? 0) !== 0) {
+      if (request.signal?.aborted) {
+        throw new Error("CLI provider request was aborted");
+      }
+      if (result.timedOut) {
+        throw new Error(
+          `CLI provider timed out${formatTimeoutSuffix(this.config.timeoutMs ?? void 0)}`
+        );
+      }
+      const codeText = result.exitCode !== null ? result.exitCode : "unknown";
+      const detail = result.stderr.trim() || result.stdout.trim();
+      const message = detail ? `${detail} (exit code ${codeText})` : `CLI exited with code ${codeText}`;
+      throw new Error(message);
     }
     return {
-      text: this.cannedResponse,
+      text: result.stdout,
       raw: {
-        prompt: request.prompt,
-        guidelines: request.guidelines
+        command: renderedCommand,
+        stderr: result.stderr,
+        exitCode: result.exitCode ?? 0,
+        cwd: this.config.cwd
       }
     };
   }
-  calculateDelay() {
-    if (this.delayMinMs > 0 || this.delayMaxMs > 0) {
-      const min = Math.max(0, this.delayMinMs);
-      const max = Math.max(min, this.delayMaxMs);
-      return Math.floor(Math.random() * (max - min + 1)) + min;
+  async ensureHealthy(signal) {
+    if (!this.config.healthcheck) {
+      return;
+    }
+    if (!this.healthcheckPromise) {
+      this.healthcheckPromise = this.runHealthcheck(this.config.healthcheck, signal);
+    }
+    return this.healthcheckPromise;
+  }
+  async runHealthcheck(healthcheck, signal) {
+    if (!healthcheck) {
+      return;
+    }
+    const timeoutMs = healthcheck.timeoutMs ?? this.config.timeoutMs;
+    if (healthcheck.type === "http") {
+      const controller = new AbortController();
+      const timer = timeoutMs ? setTimeout(() => controller.abort(), timeoutMs) : void 0;
+      signal?.addEventListener("abort", () => controller.abort(), { once: true });
+      try {
+        const response = await fetch(healthcheck.url, { method: "GET", signal: controller.signal });
+        if (!response.ok) {
+          throw new Error(`HTTP ${response.status} ${response.statusText}`);
+        }
+      } catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        throw new Error(`CLI healthcheck failed for '${this.targetName}': ${reason}`);
+      } finally {
+        if (timer !== void 0) {
+          clearTimeout(timer);
+        }
+      }
+      return;
+    }
+    const renderedCommand = renderTemplate(
+      healthcheck.commandTemplate,
+      buildTemplateValues(
+        {
+          prompt: "",
+          guidelines: "",
+          inputFiles: [],
+          evalCaseId: "",
+          attempt: 0
+        },
+        this.config
+      )
+    );
+    const env = this.config.env ? { ...process.env, ...this.config.env } : process.env;
+    const result = await this.runCommand(renderedCommand, {
+      cwd: healthcheck.cwd ?? this.config.cwd,
+      env,
+      timeoutMs,
+      signal
+    });
+    if (result.failed || (result.exitCode ?? 0) !== 0) {
+      const codeText = result.exitCode !== null ? result.exitCode : "unknown";
+      const detail = result.stderr.trim() || result.stdout.trim();
+      const message = detail ? `${detail} (exit code ${codeText})` : `CLI healthcheck command exited with code ${codeText}`;
+      throw new Error(`CLI healthcheck failed for '${this.targetName}': ${message}`);
     }
-    return this.delayMs;
   }
 };
-var BASE_TARGET_SCHEMA = external_exports.object({
-  name: external_exports.string().min(1, "target name is required"),
-  provider: external_exports.string().min(1, "provider is required"),
-  settings: external_exports.record(external_exports.unknown()).optional(),
-  judge_target: external_exports.string().optional(),
-  workers: external_exports.number().int().min(1).optional()
-});
-var DEFAULT_AZURE_API_VERSION = "2024-10-01-preview";
-function normalizeAzureApiVersion(value) {
-  if (!value) {
-    return DEFAULT_AZURE_API_VERSION;
+function buildTemplateValues(request, config) {
+  const inputFiles = normalizeInputFiles(request.inputFiles);
+  return {
+    PROMPT: shellEscape(request.prompt ?? ""),
+    GUIDELINES: shellEscape(request.guidelines ?? ""),
+    EVAL_ID: shellEscape(request.evalCaseId ?? ""),
+    ATTEMPT: shellEscape(String(request.attempt ?? 0)),
+    FILES: formatFileList(inputFiles, config.filesFormat)
+  };
+}
+function normalizeInputFiles(inputFiles) {
+  if (!inputFiles || inputFiles.length === 0) {
+    return void 0;
   }
-  const trimmed = value.trim();
-  if (trimmed.length === 0) {
-    return DEFAULT_AZURE_API_VERSION;
+  const unique = /* @__PURE__ */ new Map();
+  for (const inputFile of inputFiles) {
+    const absolutePath = path22.resolve(inputFile);
+    if (!unique.has(absolutePath)) {
+      unique.set(absolutePath, absolutePath);
+    }
   }
-  const withoutPrefix = trimmed.replace(/^api[-_]?version\s*=\s*/i, "").trim();
-  return withoutPrefix.length > 0 ? withoutPrefix : DEFAULT_AZURE_API_VERSION;
+  return Array.from(unique.values());
 }
-function resolveTargetDefinition(definition, env = process.env) {
-  const parsed = BASE_TARGET_SCHEMA.parse(definition);
-  const provider = parsed.provider.toLowerCase();
-  const providerBatching = resolveOptionalBoolean(
-    parsed.settings?.provider_batching ?? parsed.settings?.providerBatching
-  );
-  switch (provider) {
-    case "azure":
-    case "azure-openai":
-      return {
-        kind: "azure",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveAzureConfig(parsed, env)
-      };
-    case "anthropic":
-      return {
-        kind: "anthropic",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveAnthropicConfig(parsed, env)
-      };
-    case "gemini":
-    case "google":
-    case "google-gemini":
-      return {
-        kind: "gemini",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveGeminiConfig(parsed, env)
-      };
-    case "mock":
-      return {
-        kind: "mock",
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveMockConfig(parsed)
-      };
-    case "vscode":
-    case "vscode-insiders":
-      return {
-        kind: provider,
-        name: parsed.name,
-        judgeTarget: parsed.judge_target,
-        workers: parsed.workers,
-        providerBatching,
-        config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders")
-      };
-    default:
-      throw new Error(`Unsupported provider '${parsed.provider}' in target '${parsed.name}'`);
+function formatFileList(files, template) {
+  if (!files || files.length === 0) {
+    return "";
   }
+  const formatter = template ?? "{path}";
+  return files.map((filePath) => {
+    const escapedPath = shellEscape(filePath);
+    const escapedName = shellEscape(path22.basename(filePath));
+    return formatter.replaceAll("{path}", escapedPath).replaceAll("{basename}", escapedName);
+  }).join(" ");
 }
-function resolveAzureConfig(target, env) {
-  const settings = target.settings ?? {};
-  const endpointSource = settings.endpoint ?? settings.resource ?? settings.resourceName;
-  const apiKeySource = settings.api_key ?? settings.apiKey;
-  const deploymentSource = settings.deployment ?? settings.deploymentName ?? settings.model;
-  const versionSource = settings.version ?? settings.api_version;
-  const temperatureSource = settings.temperature;
-  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
-  const resourceName = resolveString(endpointSource, env, `${target.name} endpoint`);
-  const apiKey = resolveString(apiKeySource, env, `${target.name} api key`);
-  const deploymentName = resolveString(deploymentSource, env, `${target.name} deployment`);
-  const version = normalizeAzureApiVersion(
-    resolveOptionalString(versionSource, env, `${target.name} api version`)
-  );
-  const temperature = resolveOptionalNumber(temperatureSource, `${target.name} temperature`);
-  const maxOutputTokens = resolveOptionalNumber(
-    maxTokensSource,
-    `${target.name} max output tokens`
-  );
-  return {
-    resourceName,
-    deploymentName,
-    apiKey,
-    version,
-    temperature,
-    maxOutputTokens
-  };
+function renderTemplate(template, values) {
+  return template.replace(/\{([A-Z_]+)\}/g, (match, key2) => {
+    const replacement = values[key2];
+    return replacement !== void 0 ? replacement : match;
+  });
 }
-function resolveAnthropicConfig(target, env) {
-  const settings = target.settings ?? {};
-  const apiKeySource = settings.api_key ?? settings.apiKey;
-  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
-  const temperatureSource = settings.temperature;
-  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
-  const thinkingBudgetSource = settings.thinking_budget ?? settings.thinkingBudget;
-  const apiKey = resolveString(apiKeySource, env, `${target.name} Anthropic api key`);
-  const model = resolveString(modelSource, env, `${target.name} Anthropic model`);
-  return {
-    apiKey,
-    model,
-    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
-    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`),
-    thinkingBudget: resolveOptionalNumber(thinkingBudgetSource, `${target.name} thinking budget`)
-  };
+function shellEscape(value) {
+  if (value.length === 0) {
+    return "''";
+  }
+  if (process.platform === "win32") {
+    const escaped = value.replace(/"/g, '\\"');
+    return `"${escaped}"`;
+  }
+  return `'${value.replace(/'/g, `'"'"'`)}'`;
 }
-function resolveGeminiConfig(target, env) {
-  const settings = target.settings ?? {};
-  const apiKeySource = settings.api_key ?? settings.apiKey;
-  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
-  const temperatureSource = settings.temperature;
-  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
-  const apiKey = resolveString(apiKeySource, env, `${target.name} Google API key`);
-  const model = resolveOptionalString(modelSource, env, `${target.name} Gemini model`, {
-    allowLiteral: true,
-    optionalEnv: true
-  }) ?? "gemini-2.5-flash";
-  return {
-    apiKey,
-    model,
-    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
-    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`)
-  };
+function formatTimeoutSuffix(timeoutMs) {
+  if (!timeoutMs || timeoutMs <= 0) {
+    return "";
+  }
+  const seconds = Math.ceil(timeoutMs / 1e3);
+  return ` after ${seconds}s`;
 }
-function resolveMockConfig(target) {
-  const settings = target.settings ?? {};
-  const response = typeof settings.response === "string" ? settings.response : void 0;
-  return { response };
+function buildPromptDocument(request, inputFiles, options) {
+  const parts = [];
+  const guidelineFiles = collectGuidelineFiles(
+    inputFiles,
+    options?.guidelinePatterns ?? request.guideline_patterns,
+    options?.guidelineOverrides
+  );
+  const inputFilesList = collectInputFiles(inputFiles);
+  const nonGuidelineInputFiles = inputFilesList.filter(
+    (file) => !guidelineFiles.includes(file)
+  );
+  const prereadBlock = buildMandatoryPrereadBlock(guidelineFiles, nonGuidelineInputFiles);
+  if (prereadBlock.length > 0) {
+    parts.push("\n", prereadBlock);
+  }
+  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
+  return parts.join("\n").trim();
 }
-function resolveVSCodeConfig(target, env, insiders) {
-  const settings = target.settings ?? {};
+function normalizeInputFiles2(inputFiles) {
+  if (!inputFiles || inputFiles.length === 0) {
+    return void 0;
+  }
+  const deduped = /* @__PURE__ */ new Map();
+  for (const inputFile of inputFiles) {
+    const absolutePath = path32.resolve(inputFile);
+    if (!deduped.has(absolutePath)) {
+      deduped.set(absolutePath, absolutePath);
+    }
+  }
+  return Array.from(deduped.values());
+}
+function collectGuidelineFiles(inputFiles, guidelinePatterns, overrides) {
+  if (!inputFiles || inputFiles.length === 0) {
+    return [];
+  }
+  const unique = /* @__PURE__ */ new Map();
+  for (const inputFile of inputFiles) {
+    const absolutePath = path32.resolve(inputFile);
+    if (overrides?.has(absolutePath)) {
+      if (!unique.has(absolutePath)) {
+        unique.set(absolutePath, absolutePath);
+      }
+      continue;
+    }
+    const normalized = absolutePath.split(path32.sep).join("/");
+    if (isGuidelineFile(normalized, guidelinePatterns)) {
+      if (!unique.has(absolutePath)) {
+        unique.set(absolutePath, absolutePath);
+      }
+    }
+  }
+  return Array.from(unique.values());
+}
+function collectInputFiles(inputFiles) {
+  if (!inputFiles || inputFiles.length === 0) {
+    return [];
+  }
+  const unique = /* @__PURE__ */ new Map();
+  for (const inputFile of inputFiles) {
+    const absolutePath = path32.resolve(inputFile);
+    if (!unique.has(absolutePath)) {
+      unique.set(absolutePath, absolutePath);
+    }
+  }
+  return Array.from(unique.values());
+}
+function buildMandatoryPrereadBlock(guidelineFiles, inputFiles) {
+  if (guidelineFiles.length === 0 && inputFiles.length === 0) {
+    return "";
+  }
+  const buildList = (files) => files.map((absolutePath) => {
+    const fileName = path32.basename(absolutePath);
+    const fileUri = pathToFileUri2(absolutePath);
+    return `* [${fileName}](${fileUri})`;
+  });
+  const sections = [];
+  if (guidelineFiles.length > 0) {
+    sections.push(`Read all guideline files:
+${buildList(guidelineFiles).join("\n")}.`);
+  }
+  if (inputFiles.length > 0) {
+    sections.push(`Read all input files:
+${buildList(inputFiles).join("\n")}.`);
+  }
+  sections.push(
+    "If any file is missing, fail with ERROR: missing-file <filename> and stop.",
+    "Then apply system_instructions on the user query below."
+  );
+  return sections.join("\n");
+}
+function pathToFileUri2(filePath) {
+  const absolutePath = path32.isAbsolute(filePath) ? filePath : path32.resolve(filePath);
+  const normalizedPath = absolutePath.replace(/\\/g, "/");
+  if (/^[a-zA-Z]:\//.test(normalizedPath)) {
+    return `file:///${normalizedPath}`;
+  }
+  return `file://${normalizedPath}`;
+}
+var execAsync22 = promisify22(execCallback);
+var WORKSPACE_PREFIX = "agentv-codex-";
+var PROMPT_FILENAME = "prompt.md";
+var FILES_DIR = "files";
+var JSONL_TYPE_ITEM_COMPLETED = "item.completed";
+var CodexProvider = class {
+  id;
+  kind = "codex";
+  targetName;
+  supportsBatch = false;
+  config;
+  runCodex;
+  environmentCheck;
+  resolvedExecutable;
+  constructor(targetName, config, runner = defaultCodexRunner) {
+    this.id = `codex:${targetName}`;
+    this.targetName = targetName;
+    this.config = config;
+    this.runCodex = runner;
+  }
+  async invoke(request) {
+    if (request.signal?.aborted) {
+      throw new Error("Codex provider request was aborted before execution");
+    }
+    await this.ensureEnvironmentReady();
+    const inputFiles = normalizeInputFiles2(request.inputFiles);
+    const originalGuidelines = new Set(
+      collectGuidelineFiles(inputFiles, request.guideline_patterns).map((file) => path42.resolve(file))
+    );
+    const workspaceRoot = await this.createWorkspace();
+    try {
+      const { mirroredInputFiles, guidelineMirrors } = await this.mirrorInputFiles(
+        inputFiles,
+        workspaceRoot,
+        originalGuidelines
+      );
+      const promptContent = buildPromptDocument(request, mirroredInputFiles, {
+        guidelinePatterns: request.guideline_patterns,
+        guidelineOverrides: guidelineMirrors
+      });
+      const promptFile = path42.join(workspaceRoot, PROMPT_FILENAME);
+      await writeFile3(promptFile, promptContent, "utf8");
+      const args = this.buildCodexArgs();
+      const cwd = this.resolveCwd(workspaceRoot);
+      const result = await this.executeCodex(args, cwd, promptContent, request.signal);
+      if (result.timedOut) {
+        throw new Error(
+          `Codex CLI timed out${formatTimeoutSuffix2(this.config.timeoutMs ?? void 0)}`
+        );
+      }
+      if (result.exitCode !== 0) {
+        const detail = pickDetail(result.stderr, result.stdout);
+        const prefix = `Codex CLI exited with code ${result.exitCode}`;
+        throw new Error(detail ? `${prefix}: ${detail}` : prefix);
+      }
+      const parsed = parseCodexJson(result.stdout);
+      const assistantText = extractAssistantText(parsed);
+      return {
+        text: assistantText,
+        raw: {
+          response: parsed,
+          stdout: result.stdout,
+          stderr: result.stderr,
+          exitCode: result.exitCode,
+          args,
+          executable: this.resolvedExecutable ?? this.config.executable,
+          promptFile,
+          workspace: workspaceRoot,
+          inputFiles: mirroredInputFiles
+        }
+      };
+    } finally {
+      await this.cleanupWorkspace(workspaceRoot);
+    }
+  }
+  async ensureEnvironmentReady() {
+    if (!this.environmentCheck) {
+      this.environmentCheck = this.validateEnvironment();
+    }
+    await this.environmentCheck;
+  }
+  async validateEnvironment() {
+    this.resolvedExecutable = await locateExecutable(this.config.executable);
+  }
+  resolveCwd(workspaceRoot) {
+    if (!this.config.cwd) {
+      return workspaceRoot;
+    }
+    return path42.resolve(this.config.cwd);
+  }
+  buildCodexArgs() {
+    const args = ["--ask-for-approval", "never", "exec", "--json", "--color", "never", "--skip-git-repo-check"];
+    if (this.config.args && this.config.args.length > 0) {
+      args.push(...this.config.args);
+    }
+    args.push("-");
+    return args;
+  }
+  async executeCodex(args, cwd, promptContent, signal) {
+    try {
+      return await this.runCodex({
+        executable: this.resolvedExecutable ?? this.config.executable,
+        args,
+        cwd,
+        prompt: promptContent,
+        timeoutMs: this.config.timeoutMs,
+        env: process.env,
+        signal
+      });
+    } catch (error) {
+      const err = error;
+      if (err.code === "ENOENT") {
+        throw new Error(
+          `Codex executable '${this.config.executable}' was not found. Update the target settings.executable or add it to PATH.`
+        );
+      }
+      throw error;
+    }
+  }
+  async mirrorInputFiles(inputFiles, workspaceRoot, guidelineOriginals) {
+    if (!inputFiles || inputFiles.length === 0) {
+      return {
+        mirroredInputFiles: void 0,
+        guidelineMirrors: /* @__PURE__ */ new Set()
+      };
+    }
+    const filesRoot = path42.join(workspaceRoot, FILES_DIR);
+    await mkdir3(filesRoot, { recursive: true });
+    const mirrored = [];
+    const guidelineMirrors = /* @__PURE__ */ new Set();
+    const nameCounts = /* @__PURE__ */ new Map();
+    for (const inputFile of inputFiles) {
+      const absoluteSource = path42.resolve(inputFile);
+      const baseName = path42.basename(absoluteSource);
+      const count = nameCounts.get(baseName) ?? 0;
+      nameCounts.set(baseName, count + 1);
+      const finalName = count === 0 ? baseName : `${baseName}.${count}`;
+      const destination = path42.join(filesRoot, finalName);
+      await copyFile2(absoluteSource, destination);
+      const resolvedDestination = path42.resolve(destination);
+      mirrored.push(resolvedDestination);
+      if (guidelineOriginals.has(absoluteSource)) {
+        guidelineMirrors.add(resolvedDestination);
+      }
+    }
+    return {
+      mirroredInputFiles: mirrored,
+      guidelineMirrors
+    };
+  }
+  async createWorkspace() {
+    return await mkdtemp(path42.join(tmpdir(), WORKSPACE_PREFIX));
+  }
+  async cleanupWorkspace(workspaceRoot) {
+    try {
+      await rm2(workspaceRoot, { recursive: true, force: true });
+    } catch {
+    }
+  }
+};
+async function locateExecutable(candidate) {
+  const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
+  if (includesPathSeparator) {
+    const resolved = path42.isAbsolute(candidate) ? candidate : path42.resolve(candidate);
+    const executablePath = await ensureWindowsExecutableVariant(resolved);
+    await access22(executablePath, constants22.F_OK);
+    return executablePath;
+  }
+  const locator = process.platform === "win32" ? "where" : "which";
+  try {
+    const { stdout } = await execAsync22(`${locator} ${candidate}`);
+    const lines = stdout.split(/\r?\n/).map((line2) => line2.trim()).filter((line2) => line2.length > 0);
+    const preferred = selectExecutableCandidate(lines);
+    if (preferred) {
+      const executablePath = await ensureWindowsExecutableVariant(preferred);
+      await access22(executablePath, constants22.F_OK);
+      return executablePath;
+    }
+  } catch {
+  }
+  throw new Error(`Codex executable '${candidate}' was not found on PATH`);
+}
+function selectExecutableCandidate(candidates) {
+  if (candidates.length === 0) {
+    return void 0;
+  }
+  if (process.platform !== "win32") {
+    return candidates[0];
+  }
+  const extensions = getWindowsExecutableExtensions();
+  for (const ext of extensions) {
+    const match = candidates.find((candidate) => candidate.toLowerCase().endsWith(ext));
+    if (match) {
+      return match;
+    }
+  }
+  return candidates[0];
+}
+async function ensureWindowsExecutableVariant(candidate) {
+  if (process.platform !== "win32") {
+    return candidate;
+  }
+  if (hasExecutableExtension(candidate)) {
+    return candidate;
+  }
+  const extensions = getWindowsExecutableExtensions();
+  for (const ext of extensions) {
+    const withExtension = `${candidate}${ext}`;
+    try {
+      await access22(withExtension, constants22.F_OK);
+      return withExtension;
+    } catch {
+    }
+  }
+  return candidate;
+}
+function hasExecutableExtension(candidate) {
+  const lower = candidate.toLowerCase();
+  return getWindowsExecutableExtensions().some((ext) => lower.endsWith(ext));
+}
+var DEFAULT_WINDOWS_EXTENSIONS = [".com", ".exe", ".bat", ".cmd", ".ps1"];
+function getWindowsExecutableExtensions() {
+  if (process.platform !== "win32") {
+    return [];
+  }
+  const fromEnv = process.env.PATHEXT?.split(";").map((ext) => ext.trim().toLowerCase()).filter((ext) => ext.length > 0);
+  return fromEnv && fromEnv.length > 0 ? fromEnv : DEFAULT_WINDOWS_EXTENSIONS;
+}
+function parseCodexJson(output) {
+  const trimmed = output.trim();
+  if (trimmed.length === 0) {
+    throw new Error("Codex CLI produced no output in --json mode");
+  }
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    const lineObjects = parseJsonLines(trimmed);
+    if (lineObjects) {
+      return lineObjects;
+    }
+    const lastBrace = trimmed.lastIndexOf("{");
+    if (lastBrace >= 0) {
+      const candidate = trimmed.slice(lastBrace);
+      try {
+        return JSON.parse(candidate);
+      } catch {
+      }
+    }
+    const preview = trimmed.slice(0, 200);
+    throw new Error(`Codex CLI emitted invalid JSON: ${preview}${trimmed.length > 200 ? "\u2026" : ""}`);
+  }
+}
+function extractAssistantText(parsed) {
+  if (Array.isArray(parsed)) {
+    const text = extractFromEventStream(parsed);
+    if (text) {
+      return text;
+    }
+  }
+  if (!parsed || typeof parsed !== "object") {
+    throw new Error("Codex CLI JSON response did not include an assistant message");
+  }
+  const record = parsed;
+  const eventText = extractFromEvent(record);
+  if (eventText) {
+    return eventText;
+  }
+  const messages = Array.isArray(record.messages) ? record.messages : void 0;
+  if (messages) {
+    for (let index = messages.length - 1; index >= 0; index -= 1) {
+      const entry = messages[index];
+      if (!entry || typeof entry !== "object") {
+        continue;
+      }
+      const role = entry.role;
+      if (role !== "assistant") {
+        continue;
+      }
+      const content = entry.content;
+      const flattened = flattenContent(content);
+      if (flattened) {
+        return flattened;
+      }
+    }
+  }
+  const response = record.response;
+  if (response && typeof response === "object") {
+    const content = response.content;
+    const flattened = flattenContent(content);
+    if (flattened) {
+      return flattened;
+    }
+  }
+  const output = record.output;
+  const flattenedOutput = flattenContent(output);
+  if (flattenedOutput) {
+    return flattenedOutput;
+  }
+  throw new Error("Codex CLI JSON response did not include an assistant message");
+}
+function extractFromEventStream(events) {
+  for (let index = events.length - 1; index >= 0; index -= 1) {
+    const candidate = events[index];
+    const text = extractFromEvent(candidate);
+    if (text) {
+      return text;
+    }
+  }
+  return void 0;
+}
+function extractFromEvent(event) {
+  if (!event || typeof event !== "object") {
+    return void 0;
+  }
+  const record = event;
+  const type = typeof record.type === "string" ? record.type : void 0;
+  if (type === JSONL_TYPE_ITEM_COMPLETED) {
+    const item = record.item;
+    const text = extractFromItem(item);
+    if (text) {
+      return text;
+    }
+  }
+  const output = record.output ?? record.content;
+  const flattened = flattenContent(output);
+  if (flattened) {
+    return flattened;
+  }
+  return void 0;
+}
+function extractFromItem(item) {
+  if (!item || typeof item !== "object") {
+    return void 0;
+  }
+  const record = item;
+  const itemType = typeof record.type === "string" ? record.type : void 0;
+  if (itemType === "agent_message" || itemType === "response" || itemType === "output") {
+    const text = flattenContent(record.text ?? record.content ?? record.output);
+    if (text) {
+      return text;
+    }
+  }
+  return void 0;
+}
+function flattenContent(value) {
+  if (typeof value === "string") {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    const parts = value.map((segment) => {
+      if (typeof segment === "string") {
+        return segment;
+      }
+      if (segment && typeof segment === "object" && "text" in segment) {
+        const text = segment.text;
+        return typeof text === "string" ? text : void 0;
+      }
+      return void 0;
+    }).filter((part) => typeof part === "string" && part.length > 0);
+    return parts.length > 0 ? parts.join(" \n") : void 0;
+  }
+  if (value && typeof value === "object" && "text" in value) {
+    const text = value.text;
+    return typeof text === "string" ? text : void 0;
+  }
+  return void 0;
+}
+function parseJsonLines(output) {
+  const lines = output.split(/\r?\n/).map((line2) => line2.trim()).filter((line2) => line2.length > 0);
+  if (lines.length <= 1) {
+    return void 0;
+  }
+  const parsed = [];
+  for (const line2 of lines) {
+    try {
+      parsed.push(JSON.parse(line2));
+    } catch {
+      return void 0;
+    }
+  }
+  return parsed;
+}
+function pickDetail(stderr, stdout) {
+  const errorText = stderr.trim();
+  if (errorText.length > 0) {
+    return errorText;
+  }
+  const stdoutText = stdout.trim();
+  return stdoutText.length > 0 ? stdoutText : void 0;
+}
+function formatTimeoutSuffix2(timeoutMs) {
+  if (!timeoutMs || timeoutMs <= 0) {
+    return "";
+  }
+  const seconds = Math.ceil(timeoutMs / 1e3);
+  return ` after ${seconds}s`;
+}
+async function defaultCodexRunner(options) {
+  return await new Promise((resolve, reject) => {
+    const child = spawn2(options.executable, options.args, {
+      cwd: options.cwd,
+      env: options.env,
+      stdio: ["pipe", "pipe", "pipe"],
+      shell: shouldShellExecute(options.executable)
+    });
+    let stdout = "";
+    let stderr = "";
+    let timedOut = false;
+    const onAbort = () => {
+      child.kill("SIGTERM");
+    };
+    if (options.signal) {
+      if (options.signal.aborted) {
+        onAbort();
+      } else {
+        options.signal.addEventListener("abort", onAbort, { once: true });
+      }
+    }
+    let timeoutHandle;
+    if (options.timeoutMs && options.timeoutMs > 0) {
+      timeoutHandle = setTimeout(() => {
+        timedOut = true;
+        child.kill("SIGTERM");
+      }, options.timeoutMs);
+      timeoutHandle.unref?.();
+    }
+    child.stdout.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.setEncoding("utf8");
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.stdin.end(options.prompt);
+    const cleanup = () => {
+      if (timeoutHandle) {
+        clearTimeout(timeoutHandle);
+      }
+      if (options.signal) {
+        options.signal.removeEventListener("abort", onAbort);
+      }
+    };
+    child.on("error", (error) => {
+      cleanup();
+      reject(error);
+    });
+    child.on("close", (code) => {
+      cleanup();
+      resolve({
+        stdout,
+        stderr,
+        exitCode: typeof code === "number" ? code : -1,
+        timedOut
+      });
+    });
+  });
+}
+function shouldShellExecute(executable) {
+  if (process.platform !== "win32") {
+    return false;
+  }
+  const lower = executable.toLowerCase();
+  return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
+}
+var DEFAULT_MOCK_RESPONSE = '{"answer":"Mock provider response. Configure targets.yaml to supply a custom value."}';
+var MockProvider = class {
+  id;
+  kind = "mock";
+  targetName;
+  cannedResponse;
+  delayMs;
+  delayMinMs;
+  delayMaxMs;
+  constructor(targetName, config) {
+    this.id = `mock:${targetName}`;
+    this.targetName = targetName;
+    this.cannedResponse = config.response ?? DEFAULT_MOCK_RESPONSE;
+    this.delayMs = config.delayMs ?? 0;
+    this.delayMinMs = config.delayMinMs ?? 0;
+    this.delayMaxMs = config.delayMaxMs ?? 0;
+  }
+  async invoke(request) {
+    const delay = this.calculateDelay();
+    if (delay > 0) {
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+    return {
+      text: this.cannedResponse,
+      raw: {
+        prompt: request.prompt,
+        guidelines: request.guidelines
+      }
+    };
+  }
+  calculateDelay() {
+    if (this.delayMinMs > 0 || this.delayMaxMs > 0) {
+      const min = Math.max(0, this.delayMinMs);
+      const max = Math.max(min, this.delayMaxMs);
+      return Math.floor(Math.random() * (max - min + 1)) + min;
+    }
+    return this.delayMs;
+  }
+};
+var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID", "ATTEMPT", "FILES"]);
+var BASE_TARGET_SCHEMA = external_exports.object({
+  name: external_exports.string().min(1, "target name is required"),
+  provider: external_exports.string().min(1, "provider is required"),
+  settings: external_exports.record(external_exports.unknown()).optional(),
+  judge_target: external_exports.string().optional(),
+  workers: external_exports.number().int().min(1).optional()
+});
+var DEFAULT_AZURE_API_VERSION = "2024-10-01-preview";
+function normalizeAzureApiVersion(value) {
+  if (!value) {
+    return DEFAULT_AZURE_API_VERSION;
+  }
+  const trimmed = value.trim();
+  if (trimmed.length === 0) {
+    return DEFAULT_AZURE_API_VERSION;
+  }
+  const withoutPrefix = trimmed.replace(/^api[-_]?version\s*=\s*/i, "").trim();
+  return withoutPrefix.length > 0 ? withoutPrefix : DEFAULT_AZURE_API_VERSION;
+}
+function resolveTargetDefinition(definition, env = process.env) {
+  const parsed = BASE_TARGET_SCHEMA.parse(definition);
+  const provider = parsed.provider.toLowerCase();
+  const providerBatching = resolveOptionalBoolean(
+    parsed.settings?.provider_batching ?? parsed.settings?.providerBatching
+  );
+  switch (provider) {
+    case "azure":
+    case "azure-openai":
+      return {
+        kind: "azure",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveAzureConfig(parsed, env)
+      };
+    case "anthropic":
+      return {
+        kind: "anthropic",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveAnthropicConfig(parsed, env)
+      };
+    case "gemini":
+    case "google":
+    case "google-gemini":
+      return {
+        kind: "gemini",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveGeminiConfig(parsed, env)
+      };
+    case "codex":
+    case "codex-cli":
+      return {
+        kind: "codex",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveCodexConfig(parsed, env)
+      };
+    case "mock":
+      return {
+        kind: "mock",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveMockConfig(parsed)
+      };
+    case "vscode":
+    case "vscode-insiders":
+      return {
+        kind: provider,
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders")
+      };
+    case "cli":
+      return {
+        kind: "cli",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        providerBatching,
+        config: resolveCliConfig(parsed, env)
+      };
+    default:
+      throw new Error(`Unsupported provider '${parsed.provider}' in target '${parsed.name}'`);
+  }
+}
+function resolveAzureConfig(target, env) {
+  const settings = target.settings ?? {};
+  const endpointSource = settings.endpoint ?? settings.resource ?? settings.resourceName;
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const deploymentSource = settings.deployment ?? settings.deploymentName ?? settings.model;
+  const versionSource = settings.version ?? settings.api_version;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const resourceName = resolveString(endpointSource, env, `${target.name} endpoint`);
+  const apiKey = resolveString(apiKeySource, env, `${target.name} api key`);
+  const deploymentName = resolveString(deploymentSource, env, `${target.name} deployment`);
+  const version = normalizeAzureApiVersion(
+    resolveOptionalString(versionSource, env, `${target.name} api version`)
+  );
+  const temperature = resolveOptionalNumber(temperatureSource, `${target.name} temperature`);
+  const maxOutputTokens = resolveOptionalNumber(
+    maxTokensSource,
+    `${target.name} max output tokens`
+  );
+  return {
+    resourceName,
+    deploymentName,
+    apiKey,
+    version,
+    temperature,
+    maxOutputTokens
+  };
+}
+function resolveAnthropicConfig(target, env) {
+  const settings = target.settings ?? {};
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const thinkingBudgetSource = settings.thinking_budget ?? settings.thinkingBudget;
+  const apiKey = resolveString(apiKeySource, env, `${target.name} Anthropic api key`);
+  const model = resolveString(modelSource, env, `${target.name} Anthropic model`);
+  return {
+    apiKey,
+    model,
+    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
+    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`),
+    thinkingBudget: resolveOptionalNumber(thinkingBudgetSource, `${target.name} thinking budget`)
+  };
+}
+function resolveGeminiConfig(target, env) {
+  const settings = target.settings ?? {};
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const apiKey = resolveString(apiKeySource, env, `${target.name} Google API key`);
+  const model = resolveOptionalString(modelSource, env, `${target.name} Gemini model`, {
+    allowLiteral: true,
+    optionalEnv: true
+  }) ?? "gemini-2.5-flash";
+  return {
+    apiKey,
+    model,
+    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
+    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`)
+  };
+}
+function resolveCodexConfig(target, env) {
+  const settings = target.settings ?? {};
+  const executableSource = settings.executable ?? settings.command ?? settings.binary;
+  const argsSource = settings.args ?? settings.arguments;
+  const cwdSource = settings.cwd;
+  const timeoutSource = settings.timeout_seconds ?? settings.timeoutSeconds;
+  const executable = resolveOptionalString(executableSource, env, `${target.name} codex executable`, {
+    allowLiteral: true,
+    optionalEnv: true
+  }) ?? "codex";
+  const args = resolveOptionalStringArray(argsSource, env, `${target.name} codex args`);
+  const cwd = resolveOptionalString(cwdSource, env, `${target.name} codex cwd`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} codex timeout`);
+  return {
+    executable,
+    args,
+    cwd,
+    timeoutMs
+  };
+}
+function resolveMockConfig(target) {
+  const settings = target.settings ?? {};
+  const response = typeof settings.response === "string" ? settings.response : void 0;
+  return { response };
+}
+function resolveVSCodeConfig(target, env, insiders) {
+  const settings = target.settings ?? {};
   const workspaceTemplateEnvVar = resolveOptionalLiteralString(settings.workspace_template ?? settings.workspaceTemplate);
   const workspaceTemplate = workspaceTemplateEnvVar ? resolveOptionalString(workspaceTemplateEnvVar, env, `${target.name} workspace template path`, {
     allowLiteral: false,
@@ -11875,6 +12776,125 @@ function resolveVSCodeConfig(target, env, insiders) {
     workspaceTemplate
   };
 }
+function resolveCliConfig(target, env) {
+  const settings = target.settings ?? {};
+  const commandTemplateSource = settings.command_template ?? settings.commandTemplate;
+  const filesFormat = resolveOptionalLiteralString(
+    settings.files_format ?? settings.filesFormat ?? settings.attachments_format ?? settings.attachmentsFormat
+  );
+  const cwd = resolveOptionalString(settings.cwd, env, `${target.name} working directory`, {
+    allowLiteral: true,
+    optionalEnv: true
+  });
+  const envOverrides = resolveEnvOverrides(settings.env, env, target.name);
+  const timeoutMs = resolveTimeoutMs(settings.timeout_seconds ?? settings.timeoutSeconds, `${target.name} timeout`);
+  const healthcheck = resolveCliHealthcheck(settings.healthcheck, env, target.name);
+  const commandTemplate = resolveString(
+    commandTemplateSource,
+    env,
+    `${target.name} CLI command template`,
+    true
+  );
+  assertSupportedCliPlaceholders(commandTemplate, `${target.name} CLI command template`);
+  return {
+    commandTemplate,
+    filesFormat,
+    cwd,
+    env: envOverrides,
+    timeoutMs,
+    healthcheck
+  };
+}
+function resolveEnvOverrides(source2, env, targetName) {
+  if (source2 === void 0 || source2 === null) {
+    return void 0;
+  }
+  if (typeof source2 !== "object" || Array.isArray(source2)) {
+    throw new Error(`${targetName} env overrides must be an object map of strings`);
+  }
+  const entries = Object.entries(source2);
+  const resolved = {};
+  for (const [key2, value] of entries) {
+    if (typeof value !== "string") {
+      throw new Error(`${targetName} env override '${key2}' must be a string`);
+    }
+    const resolvedValue = resolveString(value, env, `${targetName} env override '${key2}'`);
+    resolved[key2] = resolvedValue;
+  }
+  return Object.keys(resolved).length > 0 ? resolved : void 0;
+}
+function resolveTimeoutMs(source2, description) {
+  const seconds = resolveOptionalNumber(source2, `${description} (seconds)`);
+  if (seconds === void 0) {
+    return void 0;
+  }
+  if (seconds <= 0) {
+    throw new Error(`${description} must be greater than zero seconds`);
+  }
+  return Math.floor(seconds * 1e3);
+}
+function resolveCliHealthcheck(source2, env, targetName) {
+  if (source2 === void 0 || source2 === null) {
+    return void 0;
+  }
+  if (typeof source2 !== "object" || Array.isArray(source2)) {
+    throw new Error(`${targetName} healthcheck must be an object`);
+  }
+  const candidate = source2;
+  const type = candidate.type;
+  const timeoutMs = resolveTimeoutMs(
+    candidate.timeout_seconds ?? candidate.timeoutSeconds,
+    `${targetName} healthcheck timeout`
+  );
+  if (type === "http") {
+    const url = resolveString(candidate.url, env, `${targetName} healthcheck URL`);
+    return {
+      type: "http",
+      url,
+      timeoutMs
+    };
+  }
+  if (type === "command") {
+    const commandTemplate = resolveString(
+      candidate.command_template ?? candidate.commandTemplate,
+      env,
+      `${targetName} healthcheck command template`,
+      true
+    );
+    assertSupportedCliPlaceholders(commandTemplate, `${targetName} healthcheck command template`);
+    const cwd = resolveOptionalString(candidate.cwd, env, `${targetName} healthcheck cwd`, {
+      allowLiteral: true,
+      optionalEnv: true
+    });
+    return {
+      type: "command",
+      commandTemplate,
+      timeoutMs,
+      cwd
+    };
+  }
+  throw new Error(`${targetName} healthcheck type must be 'http' or 'command'`);
+}
+function assertSupportedCliPlaceholders(template, description) {
+  const placeholders = extractCliPlaceholders(template);
+  for (const placeholder of placeholders) {
+    if (!CLI_PLACEHOLDERS.has(placeholder)) {
+      throw new Error(
+        `${description} includes unsupported placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS).join(", ")}`
+      );
+    }
+  }
+}
+function extractCliPlaceholders(template) {
+  const matches = template.matchAll(/\{([A-Z_]+)\}/g);
+  const results = [];
+  for (const match of matches) {
+    if (match[1]) {
+      results.push(match[1]);
+    }
+  }
+  return results;
+}
 function resolveString(source2, env, description, allowLiteral = false) {
   const value = resolveOptionalString(source2, env, description, {
     allowLiteral,
@@ -11905,11 +12925,14 @@ function resolveOptionalString(source2, env, description, options) {
   }
   const allowLiteral = options?.allowLiteral ?? false;
   const optionalEnv = options?.optionalEnv ?? false;
-  if (!allowLiteral && isLikelyEnvReference(trimmed)) {
+  const looksLikeEnv = isLikelyEnvReference(trimmed);
+  if (looksLikeEnv) {
     if (optionalEnv) {
       return void 0;
     }
-    throw new Error(`Environment variable '${trimmed}' required for ${description} is not set`);
+    if (!allowLiteral) {
+      throw new Error(`Environment variable '${trimmed}' required for ${description} is not set`);
+    }
   }
   return trimmed;
 }
@@ -11959,6 +12982,38 @@ function resolveOptionalBoolean(source2) {
 function isLikelyEnvReference(value) {
   return /^[A-Z0-9_]+$/.test(value);
 }
+function resolveOptionalStringArray(source2, env, description) {
+  if (source2 === void 0 || source2 === null) {
+    return void 0;
+  }
+  if (!Array.isArray(source2)) {
+    throw new Error(`${description} must be an array of strings`);
+  }
+  if (source2.length === 0) {
+    return void 0;
+  }
+  const resolved = [];
+  for (let i6 = 0; i6 < source2.length; i6++) {
+    const item = source2[i6];
+    if (typeof item !== "string") {
+      throw new Error(`${description}[${i6}] must be a string`);
+    }
+    const trimmed = item.trim();
+    if (trimmed.length === 0) {
+      throw new Error(`${description}[${i6}] cannot be empty`);
+    }
+    const envValue = env[trimmed];
+    if (envValue !== void 0) {
+      if (envValue.trim().length === 0) {
+        throw new Error(`Environment variable '${trimmed}' for ${description}[${i6}] is empty`);
+      }
+      resolved.push(envValue);
+    } else {
+      resolved.push(trimmed);
+    }
+  }
+  return resolved.length > 0 ? resolved : void 0;
+}
 var VSCodeProvider = class {
   id;
   kind;
@@ -11975,12 +13030,11 @@ var VSCodeProvider = class {
     if (request.signal?.aborted) {
       throw new Error("VS Code provider request was aborted before dispatch");
     }
-    const attachments = normalizeAttachments(request.attachments);
-    const promptContent = buildPromptDocument(request, attachments, request.guideline_patterns);
+    const inputFiles = normalizeAttachments(request.inputFiles);
+    const promptContent = buildPromptDocument2(request, inputFiles, request.guideline_patterns);
     const session = await dispatchAgentSession({
       userQuery: promptContent,
-      // Use full prompt content instead of just request.prompt
-      extraAttachments: attachments,
+      extraAttachments: inputFiles,
       wait: this.config.waitForResponse,
       dryRun: this.config.dryRun,
       vscodeCmd: this.config.command,
@@ -11997,7 +13051,7 @@ var VSCodeProvider = class {
         text: "",
         raw: {
           session,
-          attachments
+          inputFiles
         }
       };
     }
@@ -12006,7 +13060,7 @@ var VSCodeProvider = class {
       text: responseText,
       raw: {
         session,
-        attachments
+        inputFiles
       }
     };
   }
@@ -12016,17 +13070,17 @@ var VSCodeProvider = class {
     }
     const normalizedRequests = requests.map((req) => ({
       request: req,
-      attachments: normalizeAttachments(req.attachments)
+      inputFiles: normalizeAttachments(req.inputFiles)
     }));
-    const combinedAttachments = mergeAttachments(
-      normalizedRequests.map(({ attachments }) => attachments)
+    const combinedInputFiles = mergeAttachments(
+      normalizedRequests.map(({ inputFiles }) => inputFiles)
     );
     const userQueries = normalizedRequests.map(
-      ({ request, attachments }) => buildPromptDocument(request, attachments, request.guideline_patterns)
+      ({ request, inputFiles }) => buildPromptDocument2(request, inputFiles, request.guideline_patterns)
     );
     const session = await dispatchBatchAgent({
       userQueries,
-      extraAttachments: combinedAttachments,
+      extraAttachments: combinedInputFiles,
       wait: this.config.waitForResponse,
       dryRun: this.config.dryRun,
       vscodeCmd: this.config.command,
@@ -12039,12 +13093,12 @@ var VSCodeProvider = class {
       throw new Error(failure);
     }
     if (this.config.dryRun) {
-      return normalizedRequests.map(({ attachments }) => ({
+      return normalizedRequests.map(({ inputFiles }) => ({
         text: "",
         raw: {
           session,
-          attachments,
-          allAttachments: combinedAttachments
+          inputFiles,
+          allInputFiles: combinedInputFiles
         }
       }));
     }
@@ -12060,8 +13114,8 @@ var VSCodeProvider = class {
         text: responseText,
         raw: {
           session,
-          attachments: normalizedRequests[index]?.attachments,
-          allAttachments: combinedAttachments,
+          inputFiles: normalizedRequests[index]?.inputFiles,
+          allInputFiles: combinedInputFiles,
           responseFile
         }
       });
@@ -12069,27 +13123,27 @@ var VSCodeProvider = class {
     return responses;
   }
 };
-function buildPromptDocument(request, attachments, guidelinePatterns) {
+function buildPromptDocument2(request, attachments, guidelinePatterns) {
   const parts = [];
-  const guidelineFiles = collectGuidelineFiles(attachments, guidelinePatterns);
+  const guidelineFiles = collectGuidelineFiles2(attachments, guidelinePatterns);
   const attachmentFiles = collectAttachmentFiles(attachments);
   const nonGuidelineAttachments = attachmentFiles.filter(
     (file) => !guidelineFiles.includes(file)
   );
-  const prereadBlock = buildMandatoryPrereadBlock(guidelineFiles, nonGuidelineAttachments);
+  const prereadBlock = buildMandatoryPrereadBlock2(guidelineFiles, nonGuidelineAttachments);
   if (prereadBlock.length > 0) {
     parts.push("\n", prereadBlock);
   }
   parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
   return parts.join("\n").trim();
 }
-function buildMandatoryPrereadBlock(guidelineFiles, attachmentFiles) {
+function buildMandatoryPrereadBlock2(guidelineFiles, attachmentFiles) {
   if (guidelineFiles.length === 0 && attachmentFiles.length === 0) {
     return "";
   }
   const buildList = (files) => files.map((absolutePath) => {
-    const fileName = path22.basename(absolutePath);
-    const fileUri = pathToFileUri2(absolutePath);
+    const fileName = path52.basename(absolutePath);
+    const fileUri = pathToFileUri22(absolutePath);
     return `* [${fileName}](${fileUri})`;
   });
   const sections = [];
@@ -12107,14 +13161,14 @@ ${buildList(attachmentFiles).join("\n")}.`);
   );
   return sections.join("\n");
 }
-function collectGuidelineFiles(attachments, guidelinePatterns) {
+function collectGuidelineFiles2(attachments, guidelinePatterns) {
   if (!attachments || attachments.length === 0) {
     return [];
   }
   const unique = /* @__PURE__ */ new Map();
   for (const attachment of attachments) {
-    const absolutePath = path22.resolve(attachment);
-    const normalized = absolutePath.split(path22.sep).join("/");
+    const absolutePath = path52.resolve(attachment);
+    const normalized = absolutePath.split(path52.sep).join("/");
     if (isGuidelineFile(normalized, guidelinePatterns)) {
       if (!unique.has(absolutePath)) {
         unique.set(absolutePath, absolutePath);
@@ -12129,15 +13183,15 @@ function collectAttachmentFiles(attachments) {
   }
   const unique = /* @__PURE__ */ new Map();
   for (const attachment of attachments) {
-    const absolutePath = path22.resolve(attachment);
+    const absolutePath = path52.resolve(attachment);
     if (!unique.has(absolutePath)) {
       unique.set(absolutePath, absolutePath);
     }
   }
   return Array.from(unique.values());
 }
-function pathToFileUri2(filePath) {
-  const absolutePath = path22.isAbsolute(filePath) ? filePath : path22.resolve(filePath);
+function pathToFileUri22(filePath) {
+  const absolutePath = path52.isAbsolute(filePath) ? filePath : path52.resolve(filePath);
   const normalizedPath = absolutePath.replace(/\\/g, "/");
   if (/^[a-zA-Z]:\//.test(normalizedPath)) {
     return `file:///${normalizedPath}`;
@@ -12150,7 +13204,7 @@ function normalizeAttachments(attachments) {
   }
   const deduped = /* @__PURE__ */ new Set();
   for (const attachment of attachments) {
-    deduped.add(path22.resolve(attachment));
+    deduped.add(path52.resolve(attachment));
   }
   return Array.from(deduped);
 }
@@ -12158,8 +13212,8 @@ function mergeAttachments(all) {
   const deduped = /* @__PURE__ */ new Set();
   for (const list of all) {
     if (!list) continue;
-    for (const attachment of list) {
-      deduped.add(path22.resolve(attachment));
+    for (const inputFile of list) {
+      deduped.add(path52.resolve(inputFile));
     }
   }
   return deduped.size > 0 ? Array.from(deduped) : void 0;
@@ -12256,14 +13310,14 @@ function assertTargetDefinition(value, index, filePath) {
 }
 async function fileExists3(filePath) {
   try {
-    await access22(filePath, constants22.F_OK);
+    await access32(filePath, constants32.F_OK);
     return true;
   } catch {
     return false;
   }
 }
 async function readTargetDefinitions(filePath) {
-  const absolutePath = path32.resolve(filePath);
+  const absolutePath = path62.resolve(filePath);
   if (!await fileExists3(absolutePath)) {
     throw new Error(`targets.yaml not found at ${absolutePath}`);
   }
@@ -12288,233 +13342,43 @@ function createProvider(target) {
       return new AnthropicProvider(target.name, target.config);
     case "gemini":
       return new GeminiProvider(target.name, target.config);
+    case "cli":
+      return new CliProvider(target.name, target.config);
+    case "codex":
+      return new CodexProvider(target.name, target.config);
     case "mock":
       return new MockProvider(target.name, target.config);
     case "vscode":
-    case "vscode-insiders":
-      return new VSCodeProvider(target.name, target.config, target.kind);
-    default: {
-      const neverTarget = target;
-      throw new Error(`Unsupported provider kind ${neverTarget.kind}`);
-    }
-  }
-}
-var KEY_TERM_MATCH_THRESHOLD = 0.5;
-var ACTION_WORDS = /* @__PURE__ */ new Set([
-  "use",
-  "avoid",
-  "prefer",
-  "replace",
-  "consider",
-  "ensure",
-  "remove",
-  "add"
-]);
-var STOP_WORDS = /* @__PURE__ */ new Set([
-  "the",
-  "a",
-  "an",
-  "and",
-  "or",
-  "but",
-  "in",
-  "on",
-  "at",
-  "to",
-  "for",
-  "of",
-  "with",
-  "by",
-  "is",
-  "are",
-  "was",
-  "were",
-  "be",
-  "been",
-  "being",
-  "have",
-  "has",
-  "had",
-  "do",
-  "does",
-  "did",
-  "will",
-  "would",
-  "could",
-  "should"
-]);
-var ERROR_PREFIXES = [
-  "error:",
-  "err:",
-  "vs code command failed",
-  "exception",
-  "traceback",
-  "no response file was generated",
-  "timed out",
-  "cli not found"
-];
-function extractAspects(expectedResponse) {
-  const lines = expectedResponse.split(/\r?\n/).map((line2) => line2.trim());
-  const aspects = [];
-  for (const line2 of lines) {
-    if (line2.length === 0) {
-      continue;
-    }
-    const bulletMatch = /^([-*•]|[0-9]+\.)\s*(.+)$/.exec(line2);
-    if (bulletMatch) {
-      const normalized = normalizeAspect(bulletMatch[2]);
-      if (normalized.length > 0) {
-        aspects.push(normalized);
-      }
-      continue;
-    }
-    const lowered = line2.toLowerCase();
-    if (Array.from(ACTION_WORDS).some((word) => lowered.startsWith(word))) {
-      const normalized = normalizeAspect(line2);
-      if (normalized.length > 0) {
-        aspects.push(normalized);
-      }
-    }
-  }
-  return aspects;
-}
-function calculateHits(candidateResponse, expectedAspects) {
-  const { normalizedText, words } = normalizeCandidate(candidateResponse);
-  const hits = [];
-  for (const aspect of expectedAspects) {
-    if (matchesAspect(aspect, normalizedText, words)) {
-      hits.push(aspect);
-    }
-  }
-  return hits;
-}
-function scoreCandidateResponse(candidateResponse, expectedAspects) {
-  if (expectedAspects.length === 0) {
-    if (isErrorLike(candidateResponse)) {
-      return {
-        score: 0,
-        hits: [],
-        misses: ["Model produced an error instead of an answer."],
-        hitCount: 0,
-        totalAspects: 0,
-        rawAspects: []
-      };
-    }
-    return {
-      score: 1,
-      hits: [],
-      misses: [],
-      hitCount: 0,
-      totalAspects: 0,
-      rawAspects: []
-    };
-  }
-  const hits = calculateHits(candidateResponse, expectedAspects);
-  const misses = expectedAspects.filter((aspect) => !hits.includes(aspect));
-  const score = expectedAspects.length > 0 ? hits.length / expectedAspects.length : 0;
-  return {
-    score,
-    hits,
-    misses,
-    hitCount: hits.length,
-    totalAspects: expectedAspects.length,
-    rawAspects: expectedAspects
-  };
-}
-function isErrorLike(text) {
-  if (!text) {
-    return false;
-  }
-  const lowered = text.trim().toLowerCase();
-  return ERROR_PREFIXES.some((prefix) => lowered.startsWith(prefix));
-}
-function normalizeAspect(aspect) {
-  const sanitized = aspect.toLowerCase().replace(/[^\w\s]/g, " ").replace(/\s+/g, " ").trim();
-  return sanitized;
-}
-function normalizeCandidate(candidate) {
-  const lowered = candidate.toLowerCase();
-  const normalizedText = lowered.replace(/[^\w\s]/g, " ");
-  const words = new Set(normalizedText.split(/\s+/).filter((word) => word.length > 0));
-  return { normalizedText, words };
-}
-function matchesAspect(aspect, candidateNormalized, candidateWords) {
-  const keyTerms = extractKeyTerms(aspect);
-  if (keyTerms.length === 0) {
-    return false;
-  }
-  const matches = keyTerms.filter((term) => candidateWords.has(term)).length;
-  const ratio = matches / keyTerms.length;
-  if (ratio >= KEY_TERM_MATCH_THRESHOLD) {
-    return true;
-  }
-  const aspectWords = aspect.split(" ");
-  if (aspectWords.length >= 2) {
-    for (let index = 0; index < aspectWords.length - 1; index += 1) {
-      const phrase = `${aspectWords[index]} ${aspectWords[index + 1]}`;
-      if (candidateNormalized.includes(phrase)) {
-        return true;
-      }
-    }
-  }
-  return false;
-}
-function extractKeyTerms(aspect, maxTerms = 5) {
-  const terms = [];
-  const words = aspect.split(" ");
-  for (const word of words) {
-    if (word.length <= 2) {
-      continue;
-    }
-    if (STOP_WORDS.has(word)) {
-      continue;
-    }
-    terms.push(word);
-    if (terms.length >= maxTerms) {
-      break;
+    case "vscode-insiders":
+      return new VSCodeProvider(target.name, target.config, target.kind);
+    default: {
+      const neverTarget = target;
+      throw new Error(`Unsupported provider kind ${neverTarget.kind}`);
     }
   }
-  return terms;
 }
-var HeuristicGrader = class {
-  kind = "heuristic";
-  grade(context2) {
-    const expectedAspects = extractAspects(context2.evalCase.expected_assistant_raw);
-    const result = scoreCandidateResponse(context2.candidate, expectedAspects);
-    const misses = [...result.misses];
-    if (expectedAspects.length === 0 && isErrorLike(context2.candidate)) {
-      const firstLine = context2.candidate.split(/\r?\n/)[0]?.trim();
-      if (firstLine && !misses.includes(firstLine)) {
-        misses.unshift(firstLine);
-      }
-    }
-    return {
-      score: result.score,
-      hits: result.hits,
-      misses,
-      expectedAspectCount: result.totalAspects,
-      rawAspects: result.rawAspects
-    };
-  }
-};
-var QualityGrader = class {
+var LlmJudgeEvaluator = class {
   kind = "llm_judge";
   resolveJudgeProvider;
   maxOutputTokens;
   temperature;
+  customPrompt;
   constructor(options) {
     this.resolveJudgeProvider = options.resolveJudgeProvider;
     this.maxOutputTokens = options.maxOutputTokens;
     this.temperature = options.temperature;
+    this.customPrompt = options.customPrompt;
   }
-  async grade(context2) {
+  async evaluate(context2) {
     const judgeProvider = await this.resolveJudgeProvider(context2);
     if (!judgeProvider) {
       throw new Error("No judge provider available for LLM grading");
     }
     const prompt = buildQualityPrompt(context2.evalCase, context2.candidate);
+    const systemPrompt = context2.systemPrompt ?? this.customPrompt ?? QUALITY_SYSTEM_PROMPT;
     const metadata = {
-      systemPrompt: QUALITY_SYSTEM_PROMPT
+      ...systemPrompt !== void 0 ? { systemPrompt } : {},
+      ...context2.judgeModel !== void 0 ? { model: context2.judgeModel } : {}
     };
     const response = await judgeProvider.invoke({
       prompt,
@@ -12529,12 +13393,13 @@ var QualityGrader = class {
     const hits = Array.isArray(parsed.hits) ? parsed.hits.filter(isNonEmptyString).slice(0, 4) : [];
     const misses = Array.isArray(parsed.misses) ? parsed.misses.filter(isNonEmptyString).slice(0, 4) : [];
     const reasoning = parsed.reasoning ?? response.reasoning;
-    const graderRawRequest = {
+    const evaluatorRawRequest = {
       id: randomUUID(),
       provider: judgeProvider.id,
       prompt,
-      systemPrompt: QUALITY_SYSTEM_PROMPT,
-      target: context2.target.name
+      target: context2.target.name,
+      ...systemPrompt !== void 0 ? { systemPrompt } : {},
+      ...context2.judgeModel !== void 0 ? { model: context2.judgeModel } : {}
     };
     return {
       score,
@@ -12542,7 +13407,7 @@ var QualityGrader = class {
       misses,
       expectedAspectCount: hits.length + misses.length || 1,
       reasoning,
-      graderRawRequest
+      evaluatorRawRequest
     };
   }
 };
@@ -12660,6 +13525,112 @@ function extractJsonBlob(text) {
 function isNonEmptyString(value) {
   return typeof value === "string" && value.trim().length > 0;
 }
+var CodeEvaluator = class {
+  kind = "code";
+  script;
+  cwd;
+  agentTimeoutMs;
+  constructor(options) {
+    this.script = options.script;
+    this.cwd = options.cwd;
+    this.agentTimeoutMs = options.agentTimeoutMs;
+  }
+  async evaluate(context2) {
+    const inputPayload = JSON.stringify(
+      {
+        task: context2.evalCase.task,
+        outcome: context2.evalCase.outcome,
+        expected: context2.evalCase.expected_assistant_raw,
+        output: context2.candidate,
+        system_message: context2.promptInputs.systemMessage ?? "",
+        guideline_paths: context2.evalCase.guideline_paths,
+        attachments: context2.evalCase.file_paths,
+        user_segments: context2.evalCase.user_segments
+      },
+      null,
+      2
+    );
+    try {
+      const stdout = await executeScript(this.script, inputPayload, this.agentTimeoutMs, this.cwd);
+      const parsed = parseJsonSafe(stdout);
+      const score = clampScore(typeof parsed?.score === "number" ? parsed.score : 0);
+      const hits = Array.isArray(parsed?.hits) ? parsed.hits.filter(isNonEmptyString) : [];
+      const misses = Array.isArray(parsed?.misses) ? parsed.misses.filter(isNonEmptyString) : [];
+      const reasoning = typeof parsed?.reasoning === "string" ? parsed.reasoning : void 0;
+      return {
+        score,
+        hits,
+        misses,
+        expectedAspectCount: hits.length + misses.length || 1,
+        reasoning,
+        evaluatorRawRequest: {
+          script: this.script,
+          ...this.cwd ? { cwd: this.cwd } : {}
+        }
+      };
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      return {
+        score: 0,
+        hits: [],
+        misses: [`Code evaluator failed: ${message}`],
+        expectedAspectCount: 1,
+        reasoning: message,
+        evaluatorRawRequest: {
+          script: this.script,
+          ...this.cwd ? { cwd: this.cwd } : {},
+          error: message
+        }
+      };
+    }
+  }
+};
+async function executeScript(scriptPath, input, agentTimeoutMs, cwd) {
+  const { spawn: spawn22 } = await import("node:child_process");
+  return await new Promise((resolve, reject) => {
+    const child = spawn22(scriptPath, {
+      shell: true,
+      cwd
+    });
+    let stdout = "";
+    let stderr = "";
+    const timeout = agentTimeoutMs ? setTimeout(() => {
+      child.kill();
+      reject(new Error(`Code evaluator timed out after ${agentTimeoutMs}ms`));
+    }, agentTimeoutMs) : void 0;
+    child.stdout?.on("data", (data) => {
+      stdout += data.toString();
+    });
+    child.stderr?.on("data", (data) => {
+      stderr += data.toString();
+    });
+    child.on("error", (error) => {
+      if (timeout !== void 0) {
+        clearTimeout(timeout);
+      }
+      reject(error);
+    });
+    child.on("exit", (code) => {
+      if (timeout !== void 0) {
+        clearTimeout(timeout);
+      }
+      if (code && code !== 0 && stderr.length > 0) {
+        reject(new Error(`Code evaluator exited with code ${code}: ${stderr.trim()}`));
+        return;
+      }
+      resolve(stdout.trim());
+    });
+    child.stdin?.write(input);
+    child.stdin?.end();
+  });
+}
+function parseJsonSafe(payload) {
+  try {
+    return JSON.parse(payload);
+  } catch {
+    return void 0;
+  }
+}
 var Node = class {
   value;
   next;
@@ -12800,7 +13771,7 @@ async function runEvaluation(options) {
     targets,
     env,
     providerFactory,
-    graders,
+    evaluators,
     maxRetries,
     agentTimeoutMs,
     promptDumpDir,
@@ -12859,7 +13830,7 @@ async function runEvaluation(options) {
     }
     return getOrCreateProvider(resolvedJudge);
   };
-  const graderRegistry = buildGraderRegistry(graders, resolveJudgeProvider);
+  const evaluatorRegistry = buildEvaluatorRegistry(evaluators, resolveJudgeProvider);
   const primaryProvider = getOrCreateProvider(target);
   const providerSupportsBatch = target.providerBatching === true && primaryProvider.supportsBatch === true && typeof primaryProvider.invokeBatch === "function";
   if (target.providerBatching && !providerSupportsBatch && verbose) {
@@ -12882,13 +13853,14 @@ async function runEvaluation(options) {
         evalCases: filteredEvalCases,
         provider: primaryProvider,
         target,
-        graderRegistry,
+        evaluatorRegistry,
         promptDumpDir,
         nowFn: now ?? (() => /* @__PURE__ */ new Date()),
         onProgress,
         onResult,
         verbose,
-        resolveJudgeProvider
+        resolveJudgeProvider,
+        agentTimeoutMs
       });
     } catch (error) {
       if (verbose) {
@@ -12919,7 +13891,7 @@ async function runEvaluation(options) {
           evalCase,
           provider: primaryProvider,
           target,
-          graders: graderRegistry,
+          evaluators: evaluatorRegistry,
           maxRetries,
           agentTimeoutMs,
           promptDumpDir,
@@ -12985,12 +13957,13 @@ async function runBatchEvaluation(options) {
     evalCases,
     provider,
     target,
-    graderRegistry,
+    evaluatorRegistry,
     promptDumpDir,
     nowFn,
     onProgress,
     onResult,
-    resolveJudgeProvider
+    resolveJudgeProvider,
+    agentTimeoutMs
   } = options;
   const promptInputsList = [];
   for (const evalCase of evalCases) {
@@ -13006,7 +13979,7 @@ async function runBatchEvaluation(options) {
       prompt: promptInputs.request,
       guidelines: promptInputs.guidelines,
       guideline_patterns: evalCase.guideline_patterns,
-      attachments: evalCase.file_paths,
+      inputFiles: evalCase.file_paths,
       evalCaseId: evalCase.id,
       metadata: {
         systemPrompt: promptInputs.systemMessage ?? ""
@@ -13038,23 +14011,19 @@ async function runBatchEvaluation(options) {
     const evalCase = evalCases[i6];
     const promptInputs = promptInputsList[i6];
     const providerResponse = batchResponse[i6];
-    const now = nowFn();
-    const graderKind = evalCase.grader ?? "heuristic";
-    const activeGrader = graderRegistry[graderKind] ?? graderRegistry.heuristic;
-    if (!activeGrader) {
-      throw new Error(`No grader registered for kind '${graderKind}'`);
-    }
-    let grade;
+    let result;
     try {
-      grade = await activeGrader.grade({
+      result = await evaluateCandidate({
         evalCase,
         candidate: providerResponse.text ?? "",
         target,
         provider,
-        attempt: 0,
+        evaluators: evaluatorRegistry,
         promptInputs,
-        now,
-        judgeProvider: await resolveJudgeProvider(target)
+        nowFn,
+        attempt: 0,
+        judgeProvider: await resolveJudgeProvider(target),
+        agentTimeoutMs
       });
     } catch (error) {
       const errorResult = buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs);
@@ -13073,28 +14042,6 @@ async function runBatchEvaluation(options) {
       }
       continue;
     }
-    const completedAt = nowFn();
-    const rawRequest = {
-      request: promptInputs.request,
-      guidelines: promptInputs.guidelines,
-      guideline_paths: evalCase.guideline_paths,
-      system_message: promptInputs.systemMessage ?? ""
-    };
-    const result = {
-      eval_id: evalCase.id,
-      conversation_id: evalCase.conversation_id,
-      score: grade.score,
-      hits: grade.hits,
-      misses: grade.misses,
-      model_answer: providerResponse.text ?? "",
-      expected_aspect_count: grade.expectedAspectCount,
-      target: target.name,
-      timestamp: completedAt.toISOString(),
-      reasoning: grade.reasoning,
-      raw_aspects: grade.rawAspects,
-      raw_request: rawRequest,
-      grader_raw_request: grade.graderRawRequest
-    };
     results.push(result);
     if (onResult) {
       await onResult(result);
@@ -13116,7 +14063,7 @@ async function runEvalCase(options) {
     evalCase,
     provider,
     target,
-    graders,
+    evaluators,
     now,
     maxRetries,
     agentTimeoutMs,
@@ -13171,27 +14118,49 @@ async function runEvalCase(options) {
   if (cacheKey && cache && !cachedResponse) {
     await cache.set(cacheKey, providerResponse);
   }
-  const graderKind = evalCase.grader ?? "heuristic";
-  const activeGrader = graders[graderKind] ?? graders.heuristic;
-  if (!activeGrader) {
-    throw new Error(`No grader registered for kind '${graderKind}'`);
-  }
-  let grade;
   try {
-    const gradeTimestamp = nowFn();
-    grade = await activeGrader.grade({
+    return await evaluateCandidate({
       evalCase,
       candidate: providerResponse.text ?? "",
       target,
       provider,
-      attempt,
+      evaluators,
       promptInputs,
-      now: gradeTimestamp,
-      judgeProvider
+      nowFn,
+      attempt,
+      judgeProvider,
+      agentTimeoutMs
     });
   } catch (error) {
     return buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs);
   }
+}
+async function evaluateCandidate(options) {
+  const {
+    evalCase,
+    candidate,
+    target,
+    provider,
+    evaluators,
+    promptInputs,
+    nowFn,
+    attempt,
+    judgeProvider,
+    agentTimeoutMs
+  } = options;
+  const gradeTimestamp = nowFn();
+  const { score, evaluatorResults } = await runEvaluatorsForCase({
+    evalCase,
+    candidate,
+    target,
+    provider,
+    evaluators,
+    attempt,
+    promptInputs,
+    now: gradeTimestamp,
+    judgeProvider,
+    agentTimeoutMs
+  });
   const completedAt = nowFn();
   const rawRequest = {
     request: promptInputs.request,
@@ -13202,18 +14171,191 @@ async function runEvalCase(options) {
   return {
     eval_id: evalCase.id,
     conversation_id: evalCase.conversation_id,
-    score: grade.score,
-    hits: grade.hits,
-    misses: grade.misses,
-    model_answer: providerResponse.text ?? "",
-    expected_aspect_count: grade.expectedAspectCount,
+    score: score.score,
+    hits: score.hits,
+    misses: score.misses,
+    model_answer: candidate,
+    expected_aspect_count: score.expectedAspectCount,
     target: target.name,
     timestamp: completedAt.toISOString(),
-    reasoning: grade.reasoning,
-    raw_aspects: grade.rawAspects,
+    reasoning: score.reasoning,
+    raw_aspects: score.rawAspects,
     raw_request: rawRequest,
-    grader_raw_request: grade.graderRawRequest
+    evaluator_raw_request: evaluatorResults ? void 0 : score.evaluatorRawRequest,
+    evaluator_results: evaluatorResults
+  };
+}
+async function runEvaluatorsForCase(options) {
+  const { evalCase, candidate, target, provider, evaluators, attempt, promptInputs, now, judgeProvider, agentTimeoutMs } = options;
+  if (evalCase.evaluators && evalCase.evaluators.length > 0) {
+    return runEvaluatorList({
+      evalCase,
+      evaluators: evalCase.evaluators,
+      candidate,
+      target,
+      provider,
+      evaluatorRegistry: evaluators,
+      attempt,
+      promptInputs,
+      now,
+      judgeProvider,
+      agentTimeoutMs
+    });
+  }
+  const evaluatorKind = evalCase.evaluator ?? "llm_judge";
+  const activeEvaluator = evaluators[evaluatorKind] ?? evaluators.llm_judge;
+  if (!activeEvaluator) {
+    throw new Error(`No evaluator registered for kind '${evaluatorKind}'`);
+  }
+  const score = await activeEvaluator.evaluate({
+    evalCase,
+    candidate,
+    target,
+    provider,
+    attempt,
+    promptInputs,
+    now,
+    judgeProvider
+  });
+  return { score };
+}
+async function runEvaluatorList(options) {
+  const {
+    evalCase,
+    evaluators,
+    candidate,
+    target,
+    provider,
+    evaluatorRegistry,
+    attempt,
+    promptInputs,
+    now,
+    judgeProvider,
+    agentTimeoutMs
+  } = options;
+  const scored = [];
+  const evaluatorResults = [];
+  for (const evaluator of evaluators ?? []) {
+    try {
+      if (evaluator.type === "llm_judge") {
+        const score2 = await runLlmJudgeEvaluator({
+          config: evaluator,
+          evalCase,
+          candidate,
+          target,
+          provider,
+          evaluatorRegistry,
+          attempt,
+          promptInputs,
+          now,
+          judgeProvider
+        });
+        scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
+        evaluatorResults.push({
+          name: evaluator.name,
+          type: evaluator.type,
+          score: score2.score,
+          hits: score2.hits,
+          misses: score2.misses,
+          reasoning: score2.reasoning,
+          evaluator_raw_request: score2.evaluatorRawRequest
+        });
+        continue;
+      }
+      if (evaluator.type === "code") {
+        const codeEvaluator = new CodeEvaluator({
+          script: evaluator.script,
+          cwd: evaluator.resolvedCwd ?? evaluator.cwd,
+          agentTimeoutMs
+        });
+        const score2 = await codeEvaluator.evaluate({
+          evalCase,
+          candidate,
+          target,
+          provider,
+          attempt,
+          promptInputs,
+          now
+        });
+        scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
+        evaluatorResults.push({
+          name: evaluator.name,
+          type: evaluator.type,
+          score: score2.score,
+          hits: score2.hits,
+          misses: score2.misses,
+          reasoning: score2.reasoning,
+          evaluator_raw_request: score2.evaluatorRawRequest
+        });
+        continue;
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      const fallbackScore = {
+        score: 0,
+        hits: [],
+        misses: [`Evaluator '${evaluator.name}' failed: ${message}`],
+        expectedAspectCount: 1,
+        reasoning: message
+      };
+      scored.push({ score: fallbackScore, name: evaluator.name ?? "unknown", type: evaluator.type ?? "unknown" });
+      evaluatorResults.push({
+        name: evaluator.name ?? "unknown",
+        type: evaluator.type ?? "unknown",
+        score: 0,
+        hits: [],
+        misses: [`Evaluator '${evaluator.name ?? "unknown"}' failed: ${message}`],
+        reasoning: message
+      });
+    }
+  }
+  const aggregateScore = scored.length > 0 ? scored.reduce((total, entry) => total + entry.score.score, 0) / scored.length : 0;
+  const hits = scored.flatMap((entry) => entry.score.hits);
+  const misses = scored.flatMap((entry) => entry.score.misses);
+  const expectedAspectCount = scored.reduce((total, entry) => total + (entry.score.expectedAspectCount ?? 0), 0);
+  const rawAspects = scored.flatMap((entry) => entry.score.rawAspects ?? []);
+  const reasoningParts = scored.map((entry) => entry.score.reasoning ? `${entry.name}: ${entry.score.reasoning}` : void 0).filter(isNonEmptyString2);
+  const reasoning = reasoningParts.length > 0 ? reasoningParts.join(" | ") : void 0;
+  const score = {
+    score: aggregateScore,
+    hits,
+    misses,
+    expectedAspectCount,
+    reasoning,
+    rawAspects: rawAspects.length > 0 ? rawAspects : void 0
   };
+  return { score, evaluatorResults };
+}
+async function runLlmJudgeEvaluator(options) {
+  const { config, evalCase, candidate, target, provider, evaluatorRegistry, attempt, promptInputs, now, judgeProvider } = options;
+  const customPrompt = await resolveCustomPrompt(config);
+  return evaluatorRegistry.llm_judge.evaluate({
+    evalCase,
+    candidate,
+    target,
+    provider,
+    attempt,
+    promptInputs,
+    now,
+    judgeProvider,
+    systemPrompt: customPrompt,
+    evaluator: config,
+    judgeModel: config.model
+  });
+}
+async function resolveCustomPrompt(config) {
+  if (config.promptPath) {
+    try {
+      return await readFile4(config.promptPath, "utf8");
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      console.warn(`Could not read custom prompt at ${config.promptPath}: ${message}`);
+    }
+  }
+  return config.prompt;
+}
+function isNonEmptyString2(value) {
+  return typeof value === "string" && value.trim().length > 0;
 }
 function filterEvalCases(evalCases, evalId) {
   if (!evalId) {
@@ -13221,9 +14363,8 @@ function filterEvalCases(evalCases, evalId) {
   }
   return evalCases.filter((evalCase) => evalCase.id === evalId);
 }
-function buildGraderRegistry(overrides, resolveJudgeProvider) {
-  const heuristic = overrides?.heuristic ?? new HeuristicGrader();
-  const llmJudge = overrides?.llm_judge ?? new QualityGrader({
+function buildEvaluatorRegistry(overrides, resolveJudgeProvider) {
+  const llmJudge = overrides?.llm_judge ?? new LlmJudgeEvaluator({
     resolveJudgeProvider: async (context2) => {
       if (context2.judgeProvider) {
         return context2.judgeProvider;
@@ -13233,15 +14374,14 @@ function buildGraderRegistry(overrides, resolveJudgeProvider) {
   });
   return {
     ...overrides,
-    heuristic,
     llm_judge: llmJudge
   };
 }
 async function dumpPrompt(directory, evalCase, promptInputs) {
   const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
   const filename = `${timestamp}_${sanitizeFilename(evalCase.id)}.json`;
-  const filePath = path42.resolve(directory, filename);
-  await mkdir3(path42.dirname(filePath), { recursive: true });
+  const filePath = path72.resolve(directory, filename);
+  await mkdir22(path72.dirname(filePath), { recursive: true });
   const payload = {
     eval_id: evalCase.id,
     request: promptInputs.request,
@@ -13258,7 +14398,7 @@ function sanitizeFilename(value) {
   return sanitized.length > 0 ? sanitized : randomUUID2();
 }
 async function invokeProvider(provider, options) {
-  const { evalCase, target, promptInputs, attempt, agentTimeoutMs, signal } = options;
+  const { evalCase, promptInputs, attempt, agentTimeoutMs, signal } = options;
   const controller = new AbortController();
   const timeout = agentTimeoutMs ? setTimeout(() => controller.abort(), agentTimeoutMs) : void 0;
   if (signal) {
@@ -13269,7 +14409,7 @@ async function invokeProvider(provider, options) {
       prompt: promptInputs.request,
       guidelines: promptInputs.guidelines,
       guideline_patterns: evalCase.guideline_patterns,
-      attachments: evalCase.file_paths,
+      inputFiles: evalCase.file_paths,
       evalCaseId: evalCase.id,
       attempt,
       metadata: {
@@ -13980,7 +15120,7 @@ function formatEvaluationSummary(summary) {
 
 // src/commands/eval/targets.ts
 import { constants as constants5 } from "node:fs";
-import { access as access5, readFile as readFile4 } from "node:fs/promises";
+import { access as access5, readFile as readFile5 } from "node:fs/promises";
 import path12 from "node:path";
 import { parse as parse4 } from "yaml";
 var TARGET_FILE_CANDIDATES = [
@@ -13999,7 +15139,7 @@ async function fileExists5(filePath) {
 }
 async function readTestSuiteTarget(testFilePath) {
   try {
-    const raw = await readFile4(path12.resolve(testFilePath), "utf8");
+    const raw = await readFile5(path12.resolve(testFilePath), "utf8");
     const parsed = parse4(raw);
     if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
       const targetValue = parsed.target;
@@ -14386,58 +15526,105 @@ function registerEvalCommand(program) {
 // src/commands/init/index.ts
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import path15 from "node:path";
+import * as readline from "node:readline/promises";
 
 // src/templates/index.ts
-import { readFileSync } from "node:fs";
+import { readFileSync, readdirSync, statSync } from "node:fs";
 import path14 from "node:path";
 import { fileURLToPath as fileURLToPath2 } from "node:url";
 var TemplateManager = class {
-  static getTemplates() {
+  static getGithubTemplates() {
+    return this.getTemplatesFromDir("github");
+  }
+  static getAgentvTemplates() {
+    return this.getTemplatesFromDir("agentv");
+  }
+  static getTemplatesFromDir(subdir) {
     const currentDir = path14.dirname(fileURLToPath2(import.meta.url));
     let templatesDir;
     if (currentDir.includes(path14.sep + "dist")) {
-      templatesDir = path14.join(currentDir, "templates");
+      templatesDir = path14.join(currentDir, "templates", subdir);
     } else {
-      templatesDir = currentDir;
+      templatesDir = path14.join(currentDir, subdir);
     }
-    const evalBuildPrompt = readFileSync(
-      path14.join(templatesDir, "eval-build.prompt.md"),
-      "utf-8"
-    );
-    const evalSchema = readFileSync(
-      path14.join(templatesDir, "eval-schema.json"),
-      "utf-8"
-    );
-    const configSchema = readFileSync(
-      path14.join(templatesDir, "config-schema.json"),
-      "utf-8"
-    );
-    return [
-      {
-        path: "prompts/eval-build.prompt.md",
-        content: evalBuildPrompt
-      },
-      {
-        path: "contexts/eval-schema.json",
-        content: evalSchema
-      },
-      {
-        path: "contexts/config-schema.json",
-        content: configSchema
+    return this.readTemplatesRecursively(templatesDir, "");
+  }
+  static readTemplatesRecursively(dir, relativePath) {
+    const templates = [];
+    const entries = readdirSync(dir);
+    for (const entry of entries) {
+      const fullPath = path14.join(dir, entry);
+      const stat4 = statSync(fullPath);
+      const entryRelativePath = relativePath ? path14.join(relativePath, entry) : entry;
+      if (stat4.isDirectory()) {
+        templates.push(...this.readTemplatesRecursively(fullPath, entryRelativePath));
+      } else {
+        const content = readFileSync(fullPath, "utf-8");
+        templates.push({
+          path: entryRelativePath.split(path14.sep).join("/"),
+          // Normalize to forward slashes
+          content
+        });
       }
-    ];
+    }
+    return templates;
   }
 };
 
 // src/commands/init/index.ts
+async function promptYesNo(message) {
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout
+  });
+  try {
+    const answer = await rl.question(`${message} (y/N): `);
+    return answer.toLowerCase() === "y" || answer.toLowerCase() === "yes";
+  } finally {
+    rl.close();
+  }
+}
 async function initCommand(options = {}) {
   const targetPath = path15.resolve(options.targetPath ?? ".");
   const githubDir = path15.join(targetPath, ".github");
+  const agentvDir = path15.join(targetPath, ".agentv");
+  const githubTemplates = TemplateManager.getGithubTemplates();
+  const agentvTemplates = TemplateManager.getAgentvTemplates();
+  const existingFiles = [];
+  if (existsSync(githubDir)) {
+    for (const template of githubTemplates) {
+      const targetFilePath = path15.join(githubDir, template.path);
+      if (existsSync(targetFilePath)) {
+        existingFiles.push(path15.relative(targetPath, targetFilePath));
+      }
+    }
+  }
+  if (existsSync(agentvDir)) {
+    for (const template of agentvTemplates) {
+      const targetFilePath = path15.join(agentvDir, template.path);
+      if (existsSync(targetFilePath)) {
+        existingFiles.push(path15.relative(targetPath, targetFilePath));
+      }
+    }
+  }
+  if (existingFiles.length > 0) {
+    console.log("We detected an existing setup:");
+    existingFiles.forEach((file) => console.log(`  - ${file}`));
+    console.log();
+    const shouldReplace = await promptYesNo("Do you want to replace these files?");
+    if (!shouldReplace) {
+      console.log("\nInit cancelled. No files were changed.");
+      return;
+    }
+    console.log();
+  }
   if (!existsSync(githubDir)) {
     mkdirSync(githubDir, { recursive: true });
   }
-  const templates = TemplateManager.getTemplates();
-  for (const template of templates) {
+  if (!existsSync(agentvDir)) {
+    mkdirSync(agentvDir, { recursive: true });
+  }
+  for (const template of githubTemplates) {
     const targetFilePath = path15.join(githubDir, template.path);
     const targetDirPath = path15.dirname(targetFilePath);
     if (!existsSync(targetDirPath)) {
@@ -14446,11 +15633,35 @@ async function initCommand(options = {}) {
     writeFileSync(targetFilePath, template.content, "utf-8");
     console.log(`Created ${path15.relative(targetPath, targetFilePath)}`);
   }
+  for (const template of agentvTemplates) {
+    const targetFilePath = path15.join(agentvDir, template.path);
+    const targetDirPath = path15.dirname(targetFilePath);
+    if (!existsSync(targetDirPath)) {
+      mkdirSync(targetDirPath, { recursive: true });
+    }
+    writeFileSync(targetFilePath, template.content, "utf-8");
+    console.log(`Created ${path15.relative(targetPath, targetFilePath)}`);
+  }
   console.log("\nAgentV initialized successfully!");
   console.log(`
 Files installed to ${path15.relative(targetPath, githubDir)}:`);
-  templates.forEach((t) => console.log(`  - ${t.path}`));
-  console.log("\nYou can now create eval files using the schema and prompt templates.");
+  githubTemplates.forEach((t) => console.log(`  - ${t.path}`));
+  console.log(`
+Files installed to ${path15.relative(targetPath, agentvDir)}:`);
+  agentvTemplates.forEach((t) => console.log(`  - ${t.path}`));
+  console.log("\nYou can now:");
+  console.log("  1. Edit .agentv/.env with your API credentials");
+  console.log("  2. Configure targets in .agentv/targets.yaml");
+  console.log("  3. Create eval files using the schema and prompt templates");
+}
+
+// src/commands/status.ts
+function registerStatusCommand(program) {
+  program.command("status").description("Show the latest AgentV kernel status").action(() => {
+    const kernel = createAgentKernel();
+    console.log(`Kernel status: ${kernel.status}`);
+  });
+  return program;
 }
 
 // src/commands/validate/format-output.ts
@@ -14525,7 +15736,7 @@ function isTTY() {
 }
 
 // ../../packages/core/dist/evaluation/validation/index.js
-import { readFile as readFile5 } from "node:fs/promises";
+import { readFile as readFile6 } from "node:fs/promises";
 import { parse as parse5 } from "yaml";
 import { readFile as readFile23 } from "node:fs/promises";
 import path16 from "node:path";
@@ -14543,7 +15754,7 @@ var SCHEMA_TARGETS_V2 = "agentv-targets-v2";
 var SCHEMA_CONFIG_V22 = "agentv-config-v2";
 async function detectFileType(filePath) {
   try {
-    const content = await readFile5(filePath, "utf8");
+    const content = await readFile6(filePath, "utf8");
     const parsed = parse5(content);
     if (typeof parsed !== "object" || parsed === null) {
       return "unknown";
@@ -14762,6 +15973,7 @@ function validateMessages(messages, location, filePath, errors) {
 function isObject2(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
+var CLI_PLACEHOLDERS2 = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID", "ATTEMPT", "FILES"]);
 async function validateTargetsFile(filePath) {
   const errors = [];
   const absolutePath = path23.resolve(filePath);
@@ -14782,6 +15994,182 @@ async function validateTargetsFile(filePath) {
       errors
     };
   }
+  function validateCliSettings(settings, absolutePath2, location, errors2) {
+    if (!isObject2(settings)) {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location,
+        message: "CLI provider requires a 'settings' object"
+      });
+      return;
+    }
+    const commandTemplate = settings["command_template"] ?? settings["commandTemplate"];
+    if (typeof commandTemplate !== "string" || commandTemplate.trim().length === 0) {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location: `${location}.commandTemplate`,
+        message: "CLI provider requires 'commandTemplate' as a non-empty string"
+      });
+    } else {
+      recordUnknownPlaceholders(commandTemplate, absolutePath2, `${location}.commandTemplate`, errors2);
+    }
+    const attachmentsFormat = settings["attachments_format"] ?? settings["attachmentsFormat"];
+    if (attachmentsFormat !== void 0 && typeof attachmentsFormat !== "string") {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location: `${location}.attachmentsFormat`,
+        message: "'attachmentsFormat' must be a string when provided"
+      });
+    }
+    const filesFormat = settings["files_format"] ?? settings["filesFormat"];
+    if (filesFormat !== void 0 && typeof filesFormat !== "string") {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location: `${location}.filesFormat`,
+        message: "'filesFormat' must be a string when provided"
+      });
+    }
+    const cwd = settings["cwd"];
+    if (cwd !== void 0 && typeof cwd !== "string") {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location: `${location}.cwd`,
+        message: "'cwd' must be a string when provided"
+      });
+    }
+    const timeoutSeconds = settings["timeout_seconds"] ?? settings["timeoutSeconds"];
+    if (timeoutSeconds !== void 0) {
+      const numericTimeout = Number(timeoutSeconds);
+      if (!Number.isFinite(numericTimeout) || numericTimeout <= 0) {
+        errors2.push({
+          severity: "error",
+          filePath: absolutePath2,
+          location: `${location}.timeoutSeconds`,
+          message: "'timeoutSeconds' must be a positive number when provided"
+        });
+      }
+    }
+    const envOverrides = settings["env"];
+    if (envOverrides !== void 0) {
+      if (!isObject2(envOverrides)) {
+        errors2.push({
+          severity: "error",
+          filePath: absolutePath2,
+          location: `${location}.env`,
+          message: "'env' must be an object with string values"
+        });
+      } else {
+        for (const [key2, value] of Object.entries(envOverrides)) {
+          if (typeof value !== "string" || value.trim().length === 0) {
+            errors2.push({
+              severity: "error",
+              filePath: absolutePath2,
+              location: `${location}.env.${key2}`,
+              message: `Environment override '${key2}' must be a non-empty string`
+            });
+          }
+        }
+      }
+    }
+    const healthcheck = settings["healthcheck"];
+    if (healthcheck !== void 0) {
+      validateCliHealthcheck(healthcheck, absolutePath2, `${location}.healthcheck`, errors2);
+    }
+  }
+  function validateCliHealthcheck(healthcheck, absolutePath2, location, errors2) {
+    if (!isObject2(healthcheck)) {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location,
+        message: "'healthcheck' must be an object when provided"
+      });
+      return;
+    }
+    const type = healthcheck["type"];
+    if (type !== "http" && type !== "command") {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location: `${location}.type`,
+        message: "healthcheck.type must be either 'http' or 'command'"
+      });
+      return;
+    }
+    const timeoutSeconds = healthcheck["timeout_seconds"] ?? healthcheck["timeoutSeconds"];
+    if (timeoutSeconds !== void 0) {
+      const numericTimeout = Number(timeoutSeconds);
+      if (!Number.isFinite(numericTimeout) || numericTimeout <= 0) {
+        errors2.push({
+          severity: "error",
+          filePath: absolutePath2,
+          location: `${location}.timeoutSeconds`,
+          message: "healthcheck.timeoutSeconds must be a positive number when provided"
+        });
+      }
+    }
+    if (type === "http") {
+      const url = healthcheck["url"];
+      if (typeof url !== "string" || url.trim().length === 0) {
+        errors2.push({
+          severity: "error",
+          filePath: absolutePath2,
+          location: `${location}.url`,
+          message: "healthcheck.url must be a non-empty string for http checks"
+        });
+      }
+      return;
+    }
+    const commandTemplate = healthcheck["command_template"] ?? healthcheck["commandTemplate"];
+    if (typeof commandTemplate !== "string" || commandTemplate.trim().length === 0) {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location: `${location}.commandTemplate`,
+        message: "healthcheck.commandTemplate must be a non-empty string for command checks"
+      });
+    } else {
+      recordUnknownPlaceholders(commandTemplate, absolutePath2, `${location}.commandTemplate`, errors2);
+    }
+    const cwd = healthcheck["cwd"];
+    if (cwd !== void 0 && typeof cwd !== "string") {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location: `${location}.cwd`,
+        message: "healthcheck.cwd must be a string when provided"
+      });
+    }
+  }
+  function recordUnknownPlaceholders(template, absolutePath2, location, errors2) {
+    const placeholders = extractPlaceholders(template);
+    for (const placeholder of placeholders) {
+      if (!CLI_PLACEHOLDERS2.has(placeholder)) {
+        errors2.push({
+          severity: "error",
+          filePath: absolutePath2,
+          location,
+          message: `Unknown CLI placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS2).join(", ")}`
+        });
+      }
+    }
+  }
+  function extractPlaceholders(template) {
+    const matches = template.matchAll(/\{([A-Z_]+)\}/g);
+    const result = [];
+    for (const match of matches) {
+      const placeholder = match[1];
+      if (placeholder) {
+        result.push(placeholder);
+      }
+    }
+    return result;
+  }
   if (!isObject2(parsed)) {
     errors.push({
       severity: "error",
@@ -14843,6 +16231,7 @@ async function validateTargetsFile(filePath) {
       });
     }
     const provider = target["provider"];
+    const providerValue = typeof provider === "string" ? provider.trim().toLowerCase() : void 0;
     if (typeof provider !== "string" || provider.trim().length === 0) {
       errors.push({
         severity: "error",
@@ -14859,7 +16248,7 @@ async function validateTargetsFile(filePath) {
       });
     }
     const settings = target["settings"];
-    if (settings !== void 0 && !isObject2(settings)) {
+    if (providerValue !== "cli" && settings !== void 0 && !isObject2(settings)) {
       errors.push({
         severity: "error",
         filePath: absolutePath,
@@ -14867,6 +16256,9 @@ async function validateTargetsFile(filePath) {
         message: "Invalid 'settings' field (must be an object)"
       });
     }
+    if (providerValue === "cli") {
+      validateCliSettings(settings, absolutePath, `${location}.settings`, errors);
+    }
     const judgeTarget = target["judge_target"];
     if (judgeTarget !== void 0 && typeof judgeTarget !== "string") {
       errors.push({
@@ -15198,15 +16590,6 @@ function registerValidateCommand(program) {
   return program;
 }
 
-// src/commands/status.ts
-function registerStatusCommand(program) {
-  program.command("status").description("Show the latest AgentV kernel status").action(() => {
-    const kernel = createAgentKernel();
-    console.log(`Kernel status: ${kernel.status}`);
-  });
-  return program;
-}
-
 // src/index.ts
 var packageJson = JSON.parse(readFileSync2(new URL("../package.json", import.meta.url), "utf8"));
 function createProgram() {
@@ -15235,4 +16618,4 @@ export {
   createProgram,
   runCli
 };
-//# sourceMappingURL=chunk-7MGIZBZG.js.map
+//# sourceMappingURL=chunk-HPH4YWGU.js.map