agentv 3.1.0 → 3.2.0

package/dist/chunk-PCQA43SA.js

@@ -0,0 +1,4248 @@
+import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
+import {
+  CLI_PLACEHOLDERS,
+  KNOWN_PROVIDERS,
+  PROVIDER_ALIASES,
+  ResponseCache,
+  buildDirectoryChain,
+  buildSearchRoots,
+  ensureVSCodeSubagents,
+  findGitRoot,
+  isEvaluatorKind,
+  listTargetNames,
+  loadConfig,
+  loadTestSuite,
+  loadTsConfig,
+  normalizeLineEndings,
+  readTargetDefinitions,
+  readTestSuiteMetadata,
+  resolveFileReference,
+  resolveTargetDefinition,
+  runEvaluation,
+  shouldEnableCache,
+  shouldSkipCacheForTemperature,
+  subscribeToCodexLogEntries,
+  subscribeToCopilotCliLogEntries,
+  subscribeToCopilotSdkLogEntries,
+  subscribeToPiLogEntries
+} from "./chunk-OR4WXZAF.js";
+
+// package.json
+var package_default = {
+  name: "agentv",
+  version: "3.2.0",
+  description: "CLI entry point for AgentV",
+  type: "module",
+  repository: {
+    type: "git",
+    url: "https://github.com/EntityProcess/agentv.git"
+  },
+  homepage: "https://github.com/EntityProcess/agentv#readme",
+  bugs: {
+    url: "https://github.com/EntityProcess/agentv/issues"
+  },
+  bin: {
+    agentv: "./dist/cli.js"
+  },
+  files: ["dist", "README.md"],
+  scripts: {
+    dev: "bun src/cli.ts",
+    build: "tsup && bun run copy-readme",
+    "copy-readme": `bun -e "import { cpSync } from 'fs'; cpSync('../../README.md', 'README.md')"`,
+    prepublishOnly: "bun run copy-readme",
+    typecheck: "tsc --noEmit",
+    lint: "biome check .",
+    format: "biome format --write .",
+    fix: "biome check --write .",
+    test: "bun test",
+    "test:watch": "bun test --watch"
+  },
+  dependencies: {
+    "@anthropic-ai/claude-agent-sdk": "^0.2.49",
+    "@github/copilot-sdk": "^0.1.25",
+    "@inquirer/prompts": "^8.2.1",
+    "@mariozechner/pi-agent-core": "^0.54.2",
+    "@mariozechner/pi-ai": "^0.54.2",
+    "@openai/codex-sdk": "^0.104.0",
+    "cmd-ts": "^0.14.3",
+    dotenv: "^16.4.5",
+    "fast-glob": "^3.3.3",
+    json5: "^2.2.3",
+    micromatch: "^4.0.8",
+    semver: "^7.7.4",
+    yaml: "^2.6.1"
+  },
+  devDependencies: {
+    "@agentv/core": "workspace:*",
+    "@types/semver": "^7.7.1",
+    execa: "^9.3.0"
+  }
+};
+
+// src/commands/eval/shared.ts
+import { constants } from "node:fs";
+import { access, stat } from "node:fs/promises";
+import path from "node:path";
+import fg from "fast-glob";
+async function resolveEvalPaths(evalPaths, cwd) {
+  const normalizedInputs = evalPaths.map((value) => value?.trim()).filter((value) => value);
+  if (normalizedInputs.length === 0) {
+    throw new Error("No eval paths provided.");
+  }
+  const unmatched = [];
+  const results = /* @__PURE__ */ new Set();
+  for (const pattern of normalizedInputs) {
+    const candidatePath = path.isAbsolute(pattern) ? path.normalize(pattern) : path.resolve(cwd, pattern);
+    try {
+      const stats = await stat(candidatePath);
+      if (stats.isFile() && /\.(ya?ml|jsonl|json)$/i.test(candidatePath)) {
+        results.add(candidatePath);
+        continue;
+      }
+    } catch {
+    }
+    const globPattern = pattern.includes("\\") ? pattern.replace(/\\/g, "/") : pattern;
+    const matches = await fg(globPattern, {
+      cwd,
+      absolute: true,
+      onlyFiles: true,
+      unique: true,
+      dot: true,
+      followSymbolicLinks: true
+    });
+    const yamlMatches = matches.filter((filePath) => /\.(ya?ml|jsonl|json)$/i.test(filePath));
+    if (yamlMatches.length === 0) {
+      unmatched.push(pattern);
+      continue;
+    }
+    for (const filePath of yamlMatches) {
+      results.add(path.normalize(filePath));
+    }
+  }
+  if (unmatched.length > 0) {
+    throw new Error(
+      `No eval files matched: ${unmatched.join(
+        ", "
+      )}. Provide YAML, JSONL, or JSON paths or globs (e.g., "evals/**/*.yaml", "evals/**/*.jsonl", "evals.json").`
+    );
+  }
+  const sorted = Array.from(results);
+  sorted.sort();
+  return sorted;
+}
+async function findRepoRoot(start) {
+  const fallback = path.resolve(start);
+  let current = fallback;
+  while (current !== void 0) {
+    const candidate = path.join(current, ".git");
+    try {
+      await access(candidate, constants.F_OK);
+      return current;
+    } catch {
+      const parent = path.dirname(current);
+      if (parent === current) {
+        break;
+      }
+      current = parent;
+    }
+  }
+  return fallback;
+}
+
+// src/utils/targets.ts
+import { constants as constants2 } from "node:fs";
+import { access as access2 } from "node:fs/promises";
+import path2 from "node:path";
+var TARGET_FILE_CANDIDATES = [
+  "targets.yaml",
+  "targets.yml",
+  path2.join(".agentv", "targets.yaml"),
+  path2.join(".agentv", "targets.yml")
+];
+async function fileExists(filePath) {
+  try {
+    await access2(filePath, constants2.F_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+async function discoverTargetsFile(options) {
+  const { explicitPath, testFilePath, repoRoot, cwd } = options;
+  if (explicitPath) {
+    const resolvedExplicit = path2.resolve(explicitPath);
+    if (await fileExists(resolvedExplicit)) {
+      return resolvedExplicit;
+    }
+    for (const candidate of TARGET_FILE_CANDIDATES) {
+      const nested = path2.join(resolvedExplicit, candidate);
+      if (await fileExists(nested)) {
+        return nested;
+      }
+    }
+    throw new Error(`targets.yaml not found at provided path: ${resolvedExplicit}`);
+  }
+  const directories = [...buildDirectoryChain(testFilePath, repoRoot)];
+  const resolvedCwd = path2.resolve(cwd);
+  if (!directories.includes(resolvedCwd)) {
+    directories.push(resolvedCwd);
+  }
+  for (const directory of directories) {
+    for (const candidate of TARGET_FILE_CANDIDATES) {
+      const fullPath = path2.join(directory, candidate);
+      if (await fileExists(fullPath)) {
+        return fullPath;
+      }
+    }
+  }
+  throw new Error("Unable to locate targets.yaml. Use --targets to specify the file explicitly.");
+}
+
+// src/commands/eval/run-eval.ts
+import { constants as constants4 } from "node:fs";
+import { access as access4 } from "node:fs/promises";
+import path12 from "node:path";
+import { pathToFileURL } from "node:url";
+
+// src/version-check.ts
+import { coerce, satisfies, validRange } from "semver";
+var ANSI_YELLOW = "\x1B[33m";
+var ANSI_RED = "\x1B[31m";
+var ANSI_RESET = "\x1B[0m";
+function checkVersion(requiredVersion) {
+  const currentVersion = package_default.version;
+  if (!requiredVersion.trim() || !validRange(requiredVersion)) {
+    throw new Error(
+      `Invalid required_version "${requiredVersion}" in .agentv/config.yaml. Must be a valid semver range (e.g., ">=2.11.0", "^2.11.0").`
+    );
+  }
+  return {
+    satisfied: satisfies(coerce(currentVersion) ?? currentVersion, requiredVersion),
+    currentVersion,
+    requiredRange: requiredVersion
+  };
+}
+async function enforceRequiredVersion(requiredVersion, options) {
+  let result;
+  try {
+    result = checkVersion(requiredVersion);
+  } catch (err) {
+    console.error(`${ANSI_RED}Error: ${err.message}${ANSI_RESET}`);
+    process.exit(1);
+  }
+  if (result.satisfied) {
+    return;
+  }
+  const warning = `${ANSI_YELLOW}Warning: This project requires agentv ${result.requiredRange} but you have ${result.currentVersion}.${ANSI_RESET}
+  Run \`agentv self update\` to upgrade.`;
+  if (options?.strict) {
+    console.error(warning);
+    console.error(
+      `${ANSI_RED}Aborting: --strict mode requires the installed version to satisfy the required range.${ANSI_RESET}`
+    );
+    process.exit(1);
+  }
+  if (process.stdin.isTTY && process.stdout.isTTY) {
+    console.warn(warning);
+    const shouldContinue = await promptContinue();
+    if (!shouldContinue) {
+      process.exit(1);
+    }
+  } else {
+    process.stderr.write(`${warning}
+`);
+  }
+}
+async function promptContinue() {
+  const { confirm } = await import("@inquirer/prompts");
+  return confirm({ message: "Continue anyway?", default: false });
+}
+
+// src/commands/eval/artifact-writer.ts
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import path3 from "node:path";
+var PASS_THRESHOLD = 0.8;
+function computeStats(values) {
+  if (values.length === 0) {
+    return { mean: 0, stddev: 0 };
+  }
+  const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
+  const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / values.length;
+  return {
+    mean: Math.round(mean * 1e3) / 1e3,
+    stddev: Math.round(Math.sqrt(variance) * 1e3) / 1e3
+  };
+}
+function computePassRate(result) {
+  const scores = result.scores;
+  if (scores && scores.length > 0) {
+    const passed = scores.filter((s) => s.score >= PASS_THRESHOLD).length;
+    return passed / scores.length;
+  }
+  return result.score >= PASS_THRESHOLD ? 1 : 0;
+}
+function countToolCalls(result) {
+  const toolCalls = {};
+  let total = 0;
+  const trace = result.trace;
+  if (trace?.steps) {
+    for (const step of trace.steps) {
+      if (step.toolName || step.type === "tool") {
+        const name = step.toolName ?? "unknown";
+        toolCalls[name] = (toolCalls[name] ?? 0) + 1;
+        total += 1;
+      }
+    }
+  }
+  return { toolCalls, total };
+}
+function parseWorkspaceChanges(fileChanges) {
+  if (!fileChanges) {
+    return void 0;
+  }
+  let filesModified = 0;
+  let filesCreated = 0;
+  const lines = fileChanges.split("\n");
+  for (const line of lines) {
+    if (line.startsWith("--- /dev/null")) {
+      filesCreated += 1;
+    } else if (line.startsWith("--- a/")) {
+      filesModified += 1;
+    }
+  }
+  const summaryLines = lines.slice(0, 20);
+  const diffSummary = lines.length > 20 ? `${summaryLines.join("\n")}
+... (${lines.length - 20} more lines)` : fileChanges;
+  return {
+    files_modified: filesModified,
+    files_created: filesCreated,
+    diff_summary: diffSummary
+  };
+}
+function buildExpectations(result) {
+  const expectations = [];
+  if (result.scores && result.scores.length > 0) {
+    for (const evaluator of result.scores) {
+      for (const hit of evaluator.hits) {
+        expectations.push({
+          text: hit,
+          passed: true,
+          evidence: evaluator.reasoning ?? ""
+        });
+      }
+      for (const miss of evaluator.misses) {
+        expectations.push({
+          text: miss,
+          passed: false,
+          evidence: evaluator.reasoning ?? ""
+        });
+      }
+    }
+  } else {
+    for (const hit of result.hits) {
+      expectations.push({ text: hit, passed: true, evidence: result.reasoning ?? "" });
+    }
+    for (const miss of result.misses) {
+      expectations.push({ text: miss, passed: false, evidence: result.reasoning ?? "" });
+    }
+  }
+  return expectations;
+}
+function buildEvaluators(scores) {
+  if (!scores || scores.length === 0) {
+    return void 0;
+  }
+  return scores.map((s) => ({
+    name: s.name,
+    type: s.type,
+    score: s.score,
+    reasoning: s.reasoning ?? "",
+    weight: s.weight,
+    verdict: s.verdict,
+    hits: s.hits,
+    misses: s.misses,
+    details: s.details
+  }));
+}
+function buildGradingArtifact(result) {
+  const expectations = buildExpectations(result);
+  const passed = expectations.filter((e) => e.passed).length;
+  const failed = expectations.filter((e) => !e.passed).length;
+  const total = expectations.length;
+  const { toolCalls, total: totalToolCalls } = countToolCalls(result);
+  const errorsEncountered = result.error ? 1 : 0;
+  return {
+    expectations,
+    summary: {
+      passed,
+      failed,
+      total,
+      pass_rate: total > 0 ? Math.round(passed / total * 1e3) / 1e3 : 0
+    },
+    execution_metrics: {
+      tool_calls: toolCalls,
+      total_tool_calls: totalToolCalls,
+      errors_encountered: errorsEncountered
+    },
+    evaluators: buildEvaluators(result.scores),
+    workspace_changes: parseWorkspaceChanges(result.fileChanges),
+    conversation: result.conversationId ? {
+      turns: result.trace ? result.trace.steps?.length ?? 0 : 0,
+      conversation_id: result.conversationId
+    } : void 0
+  };
+}
+function buildTimingArtifact(results) {
+  let totalInput = 0;
+  let totalOutput = 0;
+  let totalDurationMs = 0;
+  for (const result of results) {
+    const usage = result.tokenUsage;
+    if (usage) {
+      totalInput += usage.input ?? 0;
+      totalOutput += usage.output ?? 0;
+    }
+    if (result.durationMs != null) {
+      totalDurationMs += result.durationMs;
+    }
+  }
+  return {
+    total_tokens: totalInput + totalOutput,
+    duration_ms: totalDurationMs,
+    total_duration_seconds: Math.round(totalDurationMs / 1e3 * 1e3) / 1e3,
+    token_usage: {
+      input: totalInput,
+      output: totalOutput
+    }
+  };
+}
+function buildBenchmarkArtifact(results, evalFile = "") {
+  const targetSet = /* @__PURE__ */ new Set();
+  const testIdSet = /* @__PURE__ */ new Set();
+  for (const result of results) {
+    targetSet.add(result.target);
+    testIdSet.add(result.testId);
+  }
+  const targets = [...targetSet].sort();
+  const testIds = [...testIdSet].sort();
+  const runSummary = {};
+  const notes = [];
+  for (const target of targets) {
+    const targetResults = results.filter((r) => r.target === target);
+    const passRates = targetResults.map(computePassRate);
+    const timings = targetResults.filter((r) => r.durationMs != null).map((r) => r.durationMs / 1e3);
+    const tokens = targetResults.filter((r) => r.tokenUsage != null).map((r) => {
+      const usage = r.tokenUsage;
+      return (usage.input ?? 0) + (usage.output ?? 0);
+    });
+    const entry = {
+      pass_rate: computeStats(passRates),
+      time_seconds: computeStats(timings),
+      tokens: computeStats(tokens)
+    };
+    const toolCallCounts = targetResults.map((r) => countToolCalls(r).total);
+    if (toolCallCounts.some((c) => c > 0)) {
+      entry.tool_calls = computeStats(toolCallCounts);
+    }
+    const costs = targetResults.filter((r) => r.costUsd != null).map((r) => r.costUsd);
+    if (costs.length > 0) {
+      entry.cost_usd = computeStats(costs);
+    }
+    runSummary[target] = entry;
+  }
+  const evaluatorScores = /* @__PURE__ */ new Map();
+  for (const result of results) {
+    if (result.scores) {
+      for (const score of result.scores) {
+        const key = `${score.name}:${score.type}`;
+        if (!evaluatorScores.has(key)) {
+          evaluatorScores.set(key, []);
+        }
+        evaluatorScores.get(key)?.push(score.score);
+      }
+    }
+  }
+  let perEvaluatorSummary;
+  if (evaluatorScores.size > 0) {
+    perEvaluatorSummary = {};
+    for (const [key, scores] of evaluatorScores) {
+      perEvaluatorSummary[key] = computeStats(scores);
+    }
+  }
+  const errorCount = results.filter((r) => r.executionStatus === "execution_error").length;
+  if (errorCount > 0) {
+    notes.push(
+      `${errorCount} test(s) had execution errors and are included in pass_rate as failures`
+    );
+  }
+  if (results.length === 0) {
+    notes.push("No results to summarize");
+  }
+  const firstResult = results[0];
+  const timestamp = firstResult?.timestamp ?? (/* @__PURE__ */ new Date()).toISOString();
+  return {
+    metadata: {
+      eval_file: evalFile,
+      timestamp,
+      targets,
+      tests_run: testIds
+    },
+    run_summary: runSummary,
+    per_evaluator_summary: perEvaluatorSummary,
+    notes
+  };
+}
+async function writeArtifactsFromResults(results, outputDir, options) {
+  const gradingDir = path3.join(outputDir, "grading");
+  const timingPath = path3.join(outputDir, "timing.json");
+  const benchmarkPath = path3.join(outputDir, "benchmark.json");
+  await mkdir(gradingDir, { recursive: true });
+  for (const result of results) {
+    const grading = buildGradingArtifact(result);
+    const safeTestId = result.testId.replace(/[/\\:*?"<>|]/g, "_");
+    const gradingPath = path3.join(gradingDir, `${safeTestId}.json`);
+    await writeFile(gradingPath, `${JSON.stringify(grading, null, 2)}
+`, "utf8");
+  }
+  const timing = buildTimingArtifact(results);
+  await writeFile(timingPath, `${JSON.stringify(timing, null, 2)}
+`, "utf8");
+  const benchmark = buildBenchmarkArtifact(results, options?.evalFile);
+  await writeFile(benchmarkPath, `${JSON.stringify(benchmark, null, 2)}
+`, "utf8");
+  return { gradingDir, timingPath, benchmarkPath };
+}
+
+// src/commands/eval/benchmark-writer.ts
+import { writeFile as writeFile2 } from "node:fs/promises";
+var PASS_THRESHOLD2 = 0.8;
+function computeStats2(values) {
+  if (values.length === 0) {
+    return { mean: 0, stddev: 0 };
+  }
+  const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
+  const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / values.length;
+  return {
+    mean: Math.round(mean * 1e3) / 1e3,
+    stddev: Math.round(Math.sqrt(variance) * 1e3) / 1e3
+  };
+}
+function computePassRate2(result) {
+  const scores = result.scores;
+  if (scores && scores.length > 0) {
+    const passed = scores.filter((s) => s.score >= PASS_THRESHOLD2).length;
+    return passed / scores.length;
+  }
+  return result.score >= PASS_THRESHOLD2 ? 1 : 0;
+}
+function buildBenchmarkJson(results) {
+  const passRates = results.map(computePassRate2);
+  const timings = results.filter((r) => r.durationMs != null).map((r) => r.durationMs / 1e3);
+  const tokens = results.filter((r) => r.tokenUsage != null).map((r) => {
+    const usage = r.tokenUsage;
+    return (usage.input ?? 0) + (usage.output ?? 0);
+  });
+  return {
+    run_summary: {
+      with_skill: {
+        pass_rate: computeStats2(passRates),
+        time_seconds: computeStats2(timings),
+        tokens: computeStats2(tokens)
+      }
+    }
+  };
+}
+async function writeBenchmarkJson(outputPath, results) {
+  const benchmark = buildBenchmarkJson(results);
+  await writeFile2(outputPath, `${JSON.stringify(benchmark, null, 2)}
+`, "utf8");
+}
+
+// src/commands/eval/env.ts
+import { constants as constants3 } from "node:fs";
+import { access as access3 } from "node:fs/promises";
+import path4 from "node:path";
+import { config as loadDotenv } from "dotenv";
+function uniqueDirs(directories) {
+  const seen = /* @__PURE__ */ new Set();
+  const result = [];
+  for (const dir of directories) {
+    const absolute = path4.resolve(dir);
+    if (seen.has(absolute)) {
+      continue;
+    }
+    seen.add(absolute);
+    result.push(absolute);
+  }
+  return result;
+}
+async function fileExists2(filePath) {
+  try {
+    await access3(filePath, constants3.F_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function collectAncestorDirectories(start, boundary) {
+  const directories = [];
+  const boundaryDir = path4.resolve(boundary);
+  let current = path4.resolve(start);
+  while (current !== void 0) {
+    directories.push(current);
+    if (current === boundaryDir) {
+      break;
+    }
+    const parent = path4.dirname(current);
+    if (parent === current) {
+      break;
+    }
+    current = parent;
+  }
+  return directories;
+}
+async function loadEnvFromHierarchy(options) {
+  const { testFilePath, repoRoot, verbose } = options;
+  const testDir = path4.dirname(path4.resolve(testFilePath));
+  const cwd = process.cwd();
+  const searchDirs = uniqueDirs([...collectAncestorDirectories(testDir, repoRoot), repoRoot, cwd]);
+  const envFiles = [];
+  for (const dir of searchDirs) {
+    const candidate = path4.join(dir, ".env");
+    if (await fileExists2(candidate)) {
+      envFiles.push(candidate);
+    }
+  }
+  if (envFiles.length === 0) {
+    if (verbose) {
+      console.log("No .env file found in hierarchy");
+    }
+    return void 0;
+  }
+  for (let i = 0; i < envFiles.length; i++) {
+    const envFile = envFiles[i];
+    loadDotenv({ path: envFile, override: false });
+    if (verbose) {
+      console.log(`Loaded environment from: ${envFile}`);
+    }
+  }
+  return envFiles[0];
+}
+
+// src/commands/eval/output-writer.ts
+import path10 from "node:path";
+
+// src/commands/eval/html-writer.ts
+import { mkdir as mkdir2, writeFile as writeFile3 } from "node:fs/promises";
+import path5 from "node:path";
+
+// ../../node_modules/.bun/async-mutex@0.5.0/node_modules/async-mutex/index.mjs
+var E_TIMEOUT = new Error("timeout while waiting for mutex to become available");
+var E_ALREADY_LOCKED = new Error("mutex already locked");
+var E_CANCELED = new Error("request for lock canceled");
+var __awaiter$2 = function(thisArg, _arguments, P, generator) {
+  function adopt(value) {
+    return value instanceof P ? value : new P(function(resolve) {
+      resolve(value);
+    });
+  }
+  return new (P || (P = Promise))(function(resolve, reject) {
+    function fulfilled(value) {
+      try {
+        step(generator.next(value));
+      } catch (e) {
+        reject(e);
+      }
+    }
+    function rejected(value) {
+      try {
+        step(generator["throw"](value));
+      } catch (e) {
+        reject(e);
+      }
+    }
+    function step(result) {
+      result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
+    }
+    step((generator = generator.apply(thisArg, _arguments || [])).next());
+  });
+};
+var Semaphore = class {
+  constructor(_value, _cancelError = E_CANCELED) {
+    this._value = _value;
+    this._cancelError = _cancelError;
+    this._queue = [];
+    this._weightedWaiters = [];
+  }
+  acquire(weight = 1, priority = 0) {
+    if (weight <= 0)
+      throw new Error(`invalid weight ${weight}: must be positive`);
+    return new Promise((resolve, reject) => {
+      const task = { resolve, reject, weight, priority };
+      const i = findIndexFromEnd(this._queue, (other) => priority <= other.priority);
+      if (i === -1 && weight <= this._value) {
+        this._dispatchItem(task);
+      } else {
+        this._queue.splice(i + 1, 0, task);
+      }
+    });
+  }
+  runExclusive(callback_1) {
+    return __awaiter$2(this, arguments, void 0, function* (callback, weight = 1, priority = 0) {
+      const [value, release] = yield this.acquire(weight, priority);
+      try {
+        return yield callback(value);
+      } finally {
+        release();
+      }
+    });
+  }
+  waitForUnlock(weight = 1, priority = 0) {
+    if (weight <= 0)
+      throw new Error(`invalid weight ${weight}: must be positive`);
+    if (this._couldLockImmediately(weight, priority)) {
+      return Promise.resolve();
+    } else {
+      return new Promise((resolve) => {
+        if (!this._weightedWaiters[weight - 1])
+          this._weightedWaiters[weight - 1] = [];
+        insertSorted(this._weightedWaiters[weight - 1], { resolve, priority });
+      });
+    }
+  }
+  isLocked() {
+    return this._value <= 0;
+  }
+  getValue() {
+    return this._value;
+  }
+  setValue(value) {
+    this._value = value;
+    this._dispatchQueue();
+  }
+  release(weight = 1) {
+    if (weight <= 0)
+      throw new Error(`invalid weight ${weight}: must be positive`);
+    this._value += weight;
+    this._dispatchQueue();
+  }
+  cancel() {
+    this._queue.forEach((entry) => entry.reject(this._cancelError));
+    this._queue = [];
+  }
+  _dispatchQueue() {
+    this._drainUnlockWaiters();
+    while (this._queue.length > 0 && this._queue[0].weight <= this._value) {
+      this._dispatchItem(this._queue.shift());
+      this._drainUnlockWaiters();
+    }
+  }
+  _dispatchItem(item) {
+    const previousValue = this._value;
+    this._value -= item.weight;
+    item.resolve([previousValue, this._newReleaser(item.weight)]);
+  }
+  _newReleaser(weight) {
+    let called = false;
+    return () => {
+      if (called)
+        return;
+      called = true;
+      this.release(weight);
+    };
+  }
+  _drainUnlockWaiters() {
+    if (this._queue.length === 0) {
+      for (let weight = this._value; weight > 0; weight--) {
+        const waiters = this._weightedWaiters[weight - 1];
+        if (!waiters)
+          continue;
+        waiters.forEach((waiter) => waiter.resolve());
+        this._weightedWaiters[weight - 1] = [];
+      }
+    } else {
+      const queuedPriority = this._queue[0].priority;
+      for (let weight = this._value; weight > 0; weight--) {
+        const waiters = this._weightedWaiters[weight - 1];
+        if (!waiters)
+          continue;
+        const i = waiters.findIndex((waiter) => waiter.priority <= queuedPriority);
+        (i === -1 ? waiters : waiters.splice(0, i)).forEach((waiter) => waiter.resolve());
+      }
+    }
+  }
+  _couldLockImmediately(weight, priority) {
+    return (this._queue.length === 0 || this._queue[0].priority < priority) && weight <= this._value;
+  }
+};
+function insertSorted(a, v) {
+  const i = findIndexFromEnd(a, (other) => v.priority <= other.priority);
+  a.splice(i + 1, 0, v);
+}
+function findIndexFromEnd(a, predicate) {
+  for (let i = a.length - 1; i >= 0; i--) {
+    if (predicate(a[i])) {
+      return i;
+    }
+  }
+  return -1;
+}
+var __awaiter$1 = function(thisArg, _arguments, P, generator) {
+  function adopt(value) {
+    return value instanceof P ? value : new P(function(resolve) {
+      resolve(value);
+    });
+  }
+  return new (P || (P = Promise))(function(resolve, reject) {
+    function fulfilled(value) {
+      try {
+        step(generator.next(value));
+      } catch (e) {
+        reject(e);
+      }
+    }
+    function rejected(value) {
+      try {
+        step(generator["throw"](value));
+      } catch (e) {
+        reject(e);
+      }
+    }
+    function step(result) {
+      result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
+    }
+    step((generator = generator.apply(thisArg, _arguments || [])).next());
+  });
+};
+var Mutex = class {
+  constructor(cancelError) {
+    this._semaphore = new Semaphore(1, cancelError);
+  }
+  acquire() {
+    return __awaiter$1(this, arguments, void 0, function* (priority = 0) {
+      const [, releaser] = yield this._semaphore.acquire(1, priority);
+      return releaser;
+    });
+  }
+  runExclusive(callback, priority = 0) {
+    return this._semaphore.runExclusive(() => callback(), 1, priority);
+  }
+  isLocked() {
+    return this._semaphore.isLocked();
+  }
+  waitForUnlock(priority = 0) {
+    return this._semaphore.waitForUnlock(1, priority);
+  }
+  release() {
+    if (this._semaphore.isLocked())
+      this._semaphore.release();
+  }
+  cancel() {
+    return this._semaphore.cancel();
+  }
+};
+
+// src/commands/eval/html-writer.ts
+var HtmlWriter = class _HtmlWriter {
+  filePath;
+  results = [];
+  mutex = new Mutex();
+  closed = false;
+  isLive = true;
+  constructor(filePath) {
+    this.filePath = filePath;
+  }
+  static async open(filePath) {
+    await mkdir2(path5.dirname(filePath), { recursive: true });
+    const writer = new _HtmlWriter(filePath);
+    await writer.writeHtml();
+    return writer;
+  }
+  async append(result) {
+    await this.mutex.runExclusive(async () => {
+      if (this.closed) {
+        throw new Error("Cannot write to closed HTML writer");
+      }
+      this.results.push(result);
+      await this.writeHtml();
+    });
+  }
+  async close() {
+    await this.mutex.runExclusive(async () => {
+      if (this.closed) {
+        return;
+      }
+      this.closed = true;
+      this.isLive = false;
+      await this.writeHtml();
+    });
+  }
+  async writeHtml() {
+    const html = generateHtml(this.results, this.isLive);
+    await writeFile3(this.filePath, html, "utf8");
+  }
+};
+function generateHtml(results, isLive) {
+  const lightResults = results.map((r) => {
+    const { requests, trace, ...rest } = r;
+    return rest;
+  });
+  const dataJson = JSON.stringify(lightResults).replace(/<\//g, "<\\/");
+  const metaRefresh = isLive ? '    <meta http-equiv="refresh" content="2">\n' : "";
+  const liveIndicator = isLive ? '<span class="live-badge">\u25CF LIVE</span>' : `<span class="timestamp">${escapeHtml((/* @__PURE__ */ new Date()).toISOString())}</span>`;
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+${metaRefresh}    <title>AgentV Evaluation Report</title>
+    <style>
+${STYLES}
+    </style>
+</head>
+<body>
+    <header class="header">
+        <div class="header-left">
+            <h1 class="header-title">AgentV</h1>
+            <span class="header-subtitle">Evaluation Report</span>
+        </div>
+        <div class="header-right">${liveIndicator}</div>
+    </header>
+    <nav class="tabs" id="tabs">
+        <button class="tab active" data-tab="overview">Overview</button>
+        <button class="tab" data-tab="tests">Test Cases</button>
+    </nav>
+    <main id="app"></main>
+    <script>
+    var DATA = ${dataJson};
+    var IS_LIVE = ${String(isLive)};
+${SCRIPT}
+    </script>
+</body>
+</html>`;
+}
+function escapeHtml(s) {
+  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+var STYLES = `
+*{margin:0;padding:0;box-sizing:border-box}
+:root{
+  --bg:#f6f8fa;--surface:#fff;--border:#d0d7de;--border-light:#e8ebee;
+  --text:#1f2328;--text-muted:#656d76;
+  --primary:#0969da;--primary-bg:#ddf4ff;
+  --success:#1a7f37;--success-bg:#dafbe1;
+  --danger:#cf222e;--danger-bg:#ffebe9;
+  --warning:#9a6700;--warning-bg:#fff8c5;
+  --radius:6px;
+  --shadow:0 1px 3px rgba(31,35,40,.04),0 1px 2px rgba(31,35,40,.06);
+  --font:-apple-system,BlinkMacSystemFont,"Segoe UI","Noto Sans",Helvetica,Arial,sans-serif;
+  --mono:ui-monospace,SFMono-Regular,"SF Mono",Menlo,Consolas,monospace;
+}
+body{font-family:var(--font);background:var(--bg);color:var(--text);line-height:1.5;font-size:14px}
+
+/* Header */
+.header{background:var(--surface);border-bottom:1px solid var(--border);padding:12px 24px;display:flex;align-items:center;justify-content:space-between}
+.header-left{display:flex;align-items:baseline;gap:12px}
+.header-title{font-size:18px;font-weight:600}
+.header-subtitle{font-size:14px;color:var(--text-muted)}
+.live-badge{color:var(--success);font-size:12px;font-weight:600;animation:pulse 2s infinite}
+@keyframes pulse{0%,100%{opacity:1}50%{opacity:.4}}
+.timestamp{font-size:12px;color:var(--text-muted);font-family:var(--mono)}
+
+/* Tabs */
+.tabs{background:var(--surface);border-bottom:1px solid var(--border);padding:0 24px;display:flex}
+.tab{background:none;border:none;padding:10px 16px;font-size:14px;color:var(--text-muted);cursor:pointer;border-bottom:2px solid transparent;font-family:var(--font);transition:color .15s,border-color .15s}
+.tab:hover{color:var(--text)}
+.tab.active{color:var(--text);font-weight:600;border-bottom-color:var(--primary)}
+
+#app{max-width:1280px;margin:0 auto;padding:24px}
+
+/* Stat cards */
+.stats-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(140px,1fr));gap:12px;margin-bottom:24px}
+.stat-card{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);padding:16px;text-align:center;box-shadow:var(--shadow)}
+.stat-card.pass .stat-value{color:var(--success)}
+.stat-card.fail .stat-value{color:var(--danger)}
+.stat-card.error .stat-value{color:var(--danger)}
+.stat-card.warn .stat-value{color:var(--warning)}
+.stat-card.total .stat-value{color:var(--primary)}
+.stat-value{font-size:28px;font-weight:700;line-height:1.2}
+.stat-label{font-size:12px;color:var(--text-muted);text-transform:uppercase;letter-spacing:.5px;margin-top:4px}
+
+/* Sections */
+.section{margin-bottom:24px}
+.section-title{font-size:16px;font-weight:600;margin-bottom:12px}
+
+/* Tables */
+.table-wrap{overflow-x:auto;background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);box-shadow:var(--shadow)}
+.data-table{width:100%;border-collapse:collapse;font-size:13px}
+.data-table th{background:var(--bg);border-bottom:1px solid var(--border);padding:8px 12px;text-align:left;font-weight:600;font-size:12px;color:var(--text-muted);text-transform:uppercase;letter-spacing:.3px;white-space:nowrap}
+.data-table th.sortable{cursor:pointer;user-select:none}
+.data-table th.sortable:hover{color:var(--text)}
+.data-table td{padding:8px 12px;border-bottom:1px solid var(--border-light);vertical-align:middle}
+.data-table tbody tr:last-child td{border-bottom:none}
+
+/* Status icons */
+.status-icon{display:inline-flex;align-items:center;justify-content:center;width:22px;height:22px;border-radius:50%;font-size:12px;font-weight:700}
+.status-icon.pass{background:var(--success-bg);color:var(--success)}
+.status-icon.fail{background:var(--danger-bg);color:var(--danger)}
+.status-icon.error{background:var(--warning-bg);color:var(--warning)}
+
+/* Score colors */
+.score-high{color:var(--success);font-weight:600}
+.score-mid{color:var(--warning);font-weight:600}
+.score-low{color:var(--danger);font-weight:600}
+
+/* Pass-rate bar */
+.bar-bg{width:100px;height:8px;background:var(--border-light);border-radius:4px;overflow:hidden}
+.bar-fill{height:100%;border-radius:4px;transition:width .3s}
+.bar-fill.score-high{background:var(--success)}
+.bar-fill.score-mid{background:var(--warning)}
+.bar-fill.score-low{background:var(--danger)}
+
+/* Histogram */
+.histogram{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);padding:16px;box-shadow:var(--shadow)}
+.hist-row{display:flex;align-items:center;gap:12px;margin-bottom:8px}
+.hist-row:last-child{margin-bottom:0}
+.hist-label{width:60px;font-size:12px;color:var(--text-muted);text-align:right;flex-shrink:0}
+.hist-bar-bg{flex:1;height:20px;background:var(--border-light);border-radius:3px;overflow:hidden}
+.hist-bar{height:100%;border-radius:3px;transition:width .3s}
+.hist-count{width:30px;font-size:12px;color:var(--text-muted);text-align:right;flex-shrink:0}
+
+/* Filters */
+.filter-bar{display:flex;gap:8px;margin-bottom:16px;align-items:center;flex-wrap:wrap}
+.filter-select,.filter-search{padding:6px 10px;border:1px solid var(--border);border-radius:var(--radius);font-size:13px;background:var(--surface);color:var(--text);font-family:var(--font)}
+.filter-search{flex:1;min-width:200px}
+.filter-count{font-size:12px;color:var(--text-muted);margin-left:auto}
+
+/* Test rows */
+.test-row{cursor:pointer;transition:background .1s}
+.test-row:hover{background:var(--bg)!important}
+.test-row.expanded{background:var(--primary-bg)!important}
+.expand-col{width:32px;text-align:center}
+.expand-icon{color:var(--text-muted);font-size:12px}
+.fw-medium{font-weight:500}
+.text-pass{color:var(--success)}.text-fail{color:var(--danger)}.text-error{color:var(--warning)}
+
+/* Detail panel */
+.detail-row td{padding:0!important;background:var(--bg)!important}
+.detail-panel{padding:16px 24px}
+.detail-grid{display:grid;grid-template-columns:1fr 1fr;gap:16px;margin-bottom:16px}
+.detail-block h4{font-size:12px;color:var(--text-muted);text-transform:uppercase;letter-spacing:.3px;margin-bottom:6px}
+.detail-pre{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);padding:12px;font-family:var(--mono);font-size:12px;white-space:pre-wrap;word-break:break-word;max-height:300px;overflow-y:auto;line-height:1.6}
+.detail-panel h4{font-size:13px;font-weight:600;margin:16px 0 8px}
+.eval-table{width:100%;border-collapse:collapse;font-size:13px;background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);margin-bottom:12px}
+.eval-table th{background:var(--bg);padding:6px 10px;text-align:left;font-size:11px;font-weight:600;color:var(--text-muted);text-transform:uppercase;border-bottom:1px solid var(--border)}
+.eval-table td{padding:8px 10px;border-bottom:1px solid var(--border-light)}
+.reasoning-cell{max-width:500px;font-size:12px;color:var(--text-muted)}
+.expect-list{list-style:none;padding:0;margin-bottom:12px}
+.expect-list li{padding:4px 8px 4px 24px;position:relative;font-size:13px}
+.expect-list.pass li::before{content:"\\2713";position:absolute;left:4px;color:var(--success);font-weight:700}
+.expect-list.fail li::before{content:"\\2717";position:absolute;left:4px;color:var(--danger);font-weight:700}
+.error-box{background:var(--danger-bg);border:1px solid var(--danger);border-radius:var(--radius);padding:12px;margin-bottom:12px}
+.error-box h4{color:var(--danger);margin:0 0 6px}
+.error-box pre{font-family:var(--mono);font-size:12px;white-space:pre-wrap;word-break:break-word}
+.detail-meta{font-size:12px;color:var(--text-muted);margin-top:12px;padding-top:12px;border-top:1px solid var(--border-light)}
+.empty-state{text-align:center;padding:48px 24px;color:var(--text-muted)}
+.empty-state h3{font-size:16px;margin-bottom:8px;color:var(--text)}
+`;
+var SCRIPT = `
+(function(){
+  /* ---- helpers ---- */
+  function esc(s){
+    if(s==null)return"";
+    return String(s).replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;");
+  }
+  function getStatus(r){
+    if(r.executionStatus==="execution_error")return"error";
+    if(r.executionStatus==="quality_failure")return"fail";
+    if(r.executionStatus==="ok")return"pass";
+    if(r.error)return"error";
+    return r.score>=0.5?"pass":"fail";
+  }
+  function sIcon(s){
+    if(s==="pass")return'<span class="status-icon pass">\\u2713</span>';
+    if(s==="fail")return'<span class="status-icon fail">\\u2717</span>';
+    return'<span class="status-icon error">!</span>';
+  }
+  function fmtDur(ms){
+    if(ms==null)return"\\u2014";
+    if(ms<1000)return ms+"ms";
+    if(ms<60000)return(ms/1000).toFixed(1)+"s";
+    return Math.floor(ms/60000)+"m "+Math.round((ms%60000)/1000)+"s";
+  }
+  function fmtTok(n){
+    if(n==null)return"\\u2014";
+    if(n>=1e6)return(n/1e6).toFixed(1)+"M";
+    if(n>=1e3)return(n/1e3).toFixed(1)+"K";
+    return String(n);
+  }
+  function fmtCost(u){if(u==null)return"\\u2014";if(u<0.01)return"<$0.01";return"$"+u.toFixed(2);}
+  function fmtPct(v){if(v==null)return"\\u2014";return(v*100).toFixed(1)+"%";}
+  function sCls(v){if(v==null)return"";if(v>=0.9)return"score-high";if(v>=0.5)return"score-mid";return"score-low";}
+
+  /* ---- compute stats ---- */
+  function computeStats(d){
+    var t=d.length,p=0,f=0,e=0,dur=0,ti=0,to=0,cost=0,sc=[];
+    for(var i=0;i<d.length;i++){
+      var r=d[i],s=getStatus(r);
+      if(s==="pass")p++;else if(s==="fail")f++;else e++;
+      if(r.durationMs)dur+=r.durationMs;
+      if(r.tokenUsage){ti+=(r.tokenUsage.input||0);to+=(r.tokenUsage.output||0);}
+      if(r.costUsd)cost+=r.costUsd;
+      if(s!=="error")sc.push(r.score);
+    }
+    var g=t-e;
+    return{total:t,passed:p,failed:f,errors:e,passRate:g>0?p/g:0,dur:dur,tokens:ti+to,inTok:ti,outTok:to,cost:cost,scores:sc};
+  }
+  function computeTargets(d){
+    var m={};
+    for(var i=0;i<d.length;i++){
+      var r=d[i],tgt=r.target||"unknown";
+      if(!m[tgt])m[tgt]={target:tgt,results:[],p:0,f:0,e:0,ts:0,sc:0,dur:0,tok:0,cost:0};
+      var o=m[tgt];o.results.push(r);
+      var s=getStatus(r);
+      if(s==="pass")o.p++;else if(s==="fail")o.f++;else o.e++;
+      if(s!=="error"){o.ts+=r.score;o.sc++;}
+      if(r.durationMs)o.dur+=r.durationMs;
+      if(r.tokenUsage)o.tok+=(r.tokenUsage.input||0)+(r.tokenUsage.output||0);
+      if(r.costUsd)o.cost+=r.costUsd;
+    }
+    var a=[];for(var k in m)a.push(m[k]);return a;
+  }
+  function getEvalNames(){
+    var n={};
+    for(var i=0;i<DATA.length;i++){
+      var sc=DATA[i].scores;
+      if(sc)for(var j=0;j<sc.length;j++)n[sc[j].name]=true;
+    }
+    return Object.keys(n);
+  }
+  function getEvalScore(r,name){
+    if(!r.scores)return null;
+    for(var i=0;i<r.scores.length;i++)if(r.scores[i].name===name)return r.scores[i].score;
+    return null;
+  }
+
+  var stats=computeStats(DATA);
+  var tgtStats=computeTargets(DATA);
+  var tgtNames=tgtStats.map(function(t){return t.target;});
+
+  /* ---- state ---- */
+  var state={tab:"overview",filter:{status:"all",target:"all",search:""},sort:{col:"testId",dir:"asc"},expanded:{}};
+
+  /* ---- DOM refs ---- */
+  var app=document.getElementById("app");
+  var tabBtns=document.querySelectorAll(".tab");
+
+  /* ---- tabs ---- */
+  function setTab(t){
+    state.tab=t;
+    for(var i=0;i<tabBtns.length;i++)tabBtns[i].classList.toggle("active",tabBtns[i].getAttribute("data-tab")===t);
+    render();
+  }
+  for(var i=0;i<tabBtns.length;i++){
+    tabBtns[i].addEventListener("click",(function(b){return function(){setTab(b.getAttribute("data-tab"));};})(tabBtns[i]));
+  }
+
+  /* ---- render ---- */
+  function render(){
+    if(DATA.length===0){app.innerHTML='<div class="empty-state"><h3>No results yet</h3><p>'+(IS_LIVE?"Waiting for evaluation results\\u2026 Page will auto-refresh.":"Run an evaluation to generate results.")+"</p></div>";return;}
+    if(state.tab==="overview")renderOverview();else renderTests();
+  }
+
+  /* ---- stat card helper ---- */
+  function card(label,value,type){
+    return'<div class="stat-card '+type+'"><div class="stat-value">'+value+'</div><div class="stat-label">'+label+"</div></div>";
+  }
+
+  /* ---- overview ---- */
+  function renderOverview(){
+    var h='<div class="stats-grid">';
+    h+=card("Total Tests",stats.total,"total");
+    h+=card("Passed",stats.passed,"pass");
+    h+=card("Failed",stats.failed,"fail");
+    h+=card("Errors",stats.errors,"error");
+    var prCls=stats.passRate>=0.9?"pass":stats.passRate>=0.5?"warn":"fail";
+    h+=card("Pass Rate",fmtPct(stats.passRate),prCls);
+    h+=card("Duration",fmtDur(stats.dur),"neutral");
+    h+=card("Tokens",fmtTok(stats.tokens),"neutral");
+    h+=card("Est. Cost",fmtCost(stats.cost),"neutral");
+    h+="</div>";
+
+    /* targets table */
+    if(tgtStats.length>1){
+      h+='<div class="section"><h2 class="section-title">Targets</h2><div class="table-wrap"><table class="data-table">';
+      h+="<thead><tr><th>Target</th><th>Pass Rate</th><th></th><th>Passed</th><th>Failed</th><th>Errors</th><th>Avg Score</th><th>Duration</th><th>Tokens</th><th>Cost</th></tr></thead><tbody>";
+      for(var i=0;i<tgtStats.length;i++){
+        var t=tgtStats[i],g=t.p+t.f,pr=g>0?t.p/g:0,avg=t.sc>0?t.ts/t.sc:0;
+        h+="<tr><td class=\\"fw-medium\\">"+esc(t.target)+"</td><td>"+fmtPct(pr)+'</td><td><div class="bar-bg"><div class="bar-fill '+sCls(pr)+'" style="width:'+(pr*100)+'%"></div></div></td>';
+        h+='<td class="text-pass">'+t.p+'</td><td class="text-fail">'+t.f+'</td><td class="text-error">'+t.e+"</td>";
+        h+='<td class="'+sCls(avg)+'">'+fmtPct(avg)+"</td><td>"+fmtDur(t.dur)+"</td><td>"+fmtTok(t.tok)+"</td><td>"+fmtCost(t.cost)+"</td></tr>";
+      }
+      h+="</tbody></table></div></div>";
+    }
+
+    /* histogram */
+    if(stats.scores.length>0){
+      var bk=[0,0,0,0,0];
+      for(var i=0;i<stats.scores.length;i++){var idx=Math.min(Math.floor(stats.scores[i]*5),4);bk[idx]++;}
+      var mx=Math.max.apply(null,bk);
+      var lb=["0\\u201320%","20\\u201340%","40\\u201360%","60\\u201380%","80\\u2013100%"];
+      h+='<div class="section"><h2 class="section-title">Score Distribution</h2><div class="histogram">';
+      for(var i=0;i<bk.length;i++){
+        var pct=mx>0?(bk[i]/mx*100):0;
+        h+='<div class="hist-row"><span class="hist-label">'+lb[i]+'</span><div class="hist-bar-bg"><div class="hist-bar '+(i>=4?"score-high":i>=2?"score-mid":"score-low")+'" style="width:'+pct+'%"></div></div><span class="hist-count">'+bk[i]+"</span></div>";
+      }
+      h+="</div></div>";
+    }
+    app.innerHTML=h;
+  }
+
+  /* ---- test cases ---- */
+  function renderTests(){
+    var evalNames=getEvalNames();
+    var h='<div class="filter-bar">';
+    h+='<select id="flt-status" class="filter-select"><option value="all">All Status</option><option value="pass">Passed</option><option value="fail">Failed</option><option value="error">Errors</option></select>';
+    if(tgtNames.length>1){
+      h+='<select id="flt-target" class="filter-select"><option value="all">All Targets</option>';
+      for(var i=0;i<tgtNames.length;i++)h+='<option value="'+esc(tgtNames[i])+'">'+esc(tgtNames[i])+"</option>";
+      h+="</select>";
+    }
+    h+='<input type="text" id="flt-search" class="filter-search" placeholder="Search tests..." value="'+esc(state.filter.search)+'">';
+    h+='<span class="filter-count" id="flt-count"></span></div>';
+
+    h+='<div class="table-wrap"><table class="data-table" id="test-tbl"><thead><tr>';
+    h+='<th class="expand-col"></th>';
+    h+=sHdr("Status","status");
+    h+=sHdr("Test ID","testId");
+    if(tgtNames.length>1)h+=sHdr("Target","target");
+    h+=sHdr("Score","score");
+    for(var i=0;i<evalNames.length;i++)h+="<th>"+esc(evalNames[i])+"</th>";
+    h+=sHdr("Duration","durationMs");
+    h+=sHdr("Cost","costUsd");
+    h+="</tr></thead><tbody id=\\"test-body\\"></tbody></table></div>";
+    app.innerHTML=h;
+
+    /* wire events */
+    var selS=document.getElementById("flt-status");
+    selS.value=state.filter.status;
+    selS.addEventListener("change",function(e){state.filter.status=e.target.value;renderRows();});
+    var selT=document.getElementById("flt-target");
+    if(selT){selT.value=state.filter.target;selT.addEventListener("change",function(e){state.filter.target=e.target.value;renderRows();});}
+    document.getElementById("flt-search").addEventListener("input",function(e){state.filter.search=e.target.value;renderRows();});
+    var ths=document.querySelectorAll("th[data-sort]");
+    for(var i=0;i<ths.length;i++){
+      ths[i].addEventListener("click",(function(th){return function(){
+        var c=th.getAttribute("data-sort");
+        if(state.sort.col===c)state.sort.dir=state.sort.dir==="asc"?"desc":"asc";
+        else{state.sort.col=c;state.sort.dir="asc";}
+        renderTests();
+      };})(ths[i]));
+    }
+    renderRows();
+  }
+
+  function sHdr(label,col){
+    var arrow="";
+    if(state.sort.col===col)arrow=state.sort.dir==="asc"?" \\u2191":" \\u2193";
+    return'<th class="sortable" data-sort="'+col+'">'+label+arrow+"</th>";
+  }
+
+  function filtered(){
+    var out=[];
+    for(var i=0;i<DATA.length;i++){
+      var r=DATA[i],s=getStatus(r);
+      if(state.filter.status!=="all"&&s!==state.filter.status)continue;
+      if(state.filter.target!=="all"&&r.target!==state.filter.target)continue;
+      if(state.filter.search&&r.testId.toLowerCase().indexOf(state.filter.search.toLowerCase())===-1)continue;
+      out.push(r);
+    }
+    var col=state.sort.col,dir=state.sort.dir==="asc"?1:-1;
+    out.sort(function(a,b){
+      var va=col==="status"?getStatus(a):a[col],vb=col==="status"?getStatus(b):b[col];
+      if(va==null&&vb==null)return 0;if(va==null)return 1;if(vb==null)return-1;
+      if(typeof va==="string")return va.localeCompare(vb)*dir;
+      return(va-vb)*dir;
+    });
+    return out;
+  }
+
+  function renderRows(){
+    var rows=filtered(),evalNames=getEvalNames();
+    var tbody=document.getElementById("test-body");
+    var colSpan=5+evalNames.length+(tgtNames.length>1?1:0);
+    document.getElementById("flt-count").textContent=rows.length+" of "+DATA.length+" tests";
+    var h="";
+    for(var i=0;i<rows.length;i++){
+      var r=rows[i],s=getStatus(r),key=r.testId+":"+r.target,exp=!!state.expanded[key];
+      h+='<tr class="test-row '+s+(exp?" expanded":"")+'" data-key="'+esc(key)+'">';
+      h+='<td class="expand-col"><span class="expand-icon">'+(exp?"\\u25BE":"\\u25B8")+"</span></td>";
+      h+="<td>"+sIcon(s)+"</td>";
+      h+='<td class="fw-medium">'+esc(r.testId)+"</td>";
+      if(tgtNames.length>1)h+="<td>"+esc(r.target)+"</td>";
+      h+='<td class="'+sCls(r.score)+'">'+fmtPct(r.score)+"</td>";
+      for(var j=0;j<evalNames.length;j++){
+        var es=getEvalScore(r,evalNames[j]);
+        h+='<td class="'+sCls(es)+'">'+(es!=null?fmtPct(es):"\\u2014")+"</td>";
+      }
+      h+="<td>"+fmtDur(r.durationMs)+"</td><td>"+fmtCost(r.costUsd)+"</td></tr>";
+      if(exp)h+='<tr class="detail-row"><td colspan="'+colSpan+'">'+renderDetail(r)+"</td></tr>";
+    }
+    if(rows.length===0)h+='<tr><td colspan="'+colSpan+'" class="empty-state">No matching tests</td></tr>';
+    tbody.innerHTML=h;
+
+    /* row click */
+    var trs=tbody.querySelectorAll(".test-row");
+    for(var k=0;k<trs.length;k++){
+      trs[k].addEventListener("click",(function(tr){return function(){
+        var key=tr.getAttribute("data-key");
+        state.expanded[key]=!state.expanded[key];
+        renderRows();
+      };})(trs[k]));
+    }
+  }
+
+  /* ---- detail panel ---- */
+  function renderDetail(r){
+    var h='<div class="detail-panel">';
+
+    /* input / output */
+    h+='<div class="detail-grid">';
+    if(r.input!=null){
+      h+='<div class="detail-block"><h4>Input</h4><pre class="detail-pre">'+esc(typeof r.input==="string"?r.input:JSON.stringify(r.input,null,2))+"</pre></div>";
+    }
+    h+='<div class="detail-block"><h4>Output</h4><pre class="detail-pre">'+esc(r.answer||"")+"</pre></div>";
+    h+="</div>";
+
+    /* evaluator results */
+    if(r.scores&&r.scores.length>0){
+      h+="<h4>Evaluator Results</h4>";
+      h+='<table class="eval-table"><thead><tr><th>Evaluator</th><th>Score</th><th>Status</th><th>Reasoning</th></tr></thead><tbody>';
+      for(var i=0;i<r.scores.length;i++){
+        var ev=r.scores[i],evS=ev.score>=0.5?"pass":"fail";
+        h+="<tr><td class=\\"fw-medium\\">"+esc(ev.name)+'</td><td class="'+sCls(ev.score)+'">'+fmtPct(ev.score)+"</td><td>"+sIcon(evS)+'</td><td class="reasoning-cell">'+esc(ev.reasoning||"")+"</td></tr>";
+      }
+      h+="</tbody></table>";
+    }
+
+    /* hits / misses */
+    if(r.hits&&r.hits.length>0){
+      h+='<h4>Passed Expectations</h4><ul class="expect-list pass">';
+      for(var i=0;i<r.hits.length;i++)h+="<li>"+esc(r.hits[i])+"</li>";
+      h+="</ul>";
+    }
+    if(r.misses&&r.misses.length>0){
+      h+='<h4>Failed Expectations</h4><ul class="expect-list fail">';
+      for(var i=0;i<r.misses.length;i++)h+="<li>"+esc(r.misses[i])+"</li>";
+      h+="</ul>";
+    }
+
+    /* error */
+    if(r.error)h+='<div class="error-box"><h4>Error</h4><pre>'+esc(r.error)+"</pre></div>";
+
+    /* metadata */
+    h+='<div class="detail-meta">';
+    var m=[];
+    if(r.tokenUsage)m.push(fmtTok(r.tokenUsage.input)+" in / "+fmtTok(r.tokenUsage.output)+" out tokens");
+    if(r.durationMs)m.push(fmtDur(r.durationMs));
+    if(r.target)m.push(r.target);
+    if(r.costUsd)m.push(fmtCost(r.costUsd));
+    if(r.timestamp)m.push(r.timestamp);
+    h+=esc(m.join(" \\u00B7 "));
+    h+="</div></div>";
+    return h;
+  }
+
+  /* ---- init ---- */
+  render();
+})();
+`;
+
+// src/commands/eval/json-writer.ts
+import { mkdir as mkdir3, writeFile as writeFile4 } from "node:fs/promises";
+import path6 from "node:path";
+
+// src/utils/case-conversion.ts
+function toSnakeCase(str) {
+  if (/^[A-Z]/.test(str)) {
+    return str;
+  }
+  return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
+}
+function toSnakeCaseDeep(obj) {
+  if (obj === null || obj === void 0) {
+    return obj;
+  }
+  if (Array.isArray(obj)) {
+    return obj.map((item) => toSnakeCaseDeep(item));
+  }
+  if (typeof obj === "object") {
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      const snakeKey = toSnakeCase(key);
+      result[snakeKey] = toSnakeCaseDeep(value);
+    }
+    return result;
+  }
+  return obj;
+}
+
+// src/commands/eval/json-writer.ts
+var JsonWriter = class _JsonWriter {
+  filePath;
+  results = [];
+  closed = false;
+  constructor(filePath) {
+    this.filePath = filePath;
+  }
+  static async open(filePath) {
+    await mkdir3(path6.dirname(filePath), { recursive: true });
+    return new _JsonWriter(filePath);
+  }
+  async append(result) {
+    if (this.closed) {
+      throw new Error("Cannot write to closed JSON writer");
+    }
+    this.results.push(result);
+  }
+  async close() {
+    if (this.closed) {
+      return;
+    }
+    this.closed = true;
+    const passed = this.results.filter((r) => r.score >= 0.5).length;
+    const failed = this.results.length - passed;
+    const total = this.results.length;
+    const output = {
+      stats: {
+        total,
+        passed,
+        failed,
+        passRate: total > 0 ? passed / total : 0
+      },
+      results: this.results
+    };
+    const snakeCaseOutput = toSnakeCaseDeep(output);
+    await writeFile4(this.filePath, `${JSON.stringify(snakeCaseOutput, null, 2)}
+`, "utf8");
+  }
+};
+
+// src/commands/eval/jsonl-writer.ts
+import { createWriteStream } from "node:fs";
+import { mkdir as mkdir4 } from "node:fs/promises";
+import path7 from "node:path";
+import { finished } from "node:stream/promises";
+var JsonlWriter = class _JsonlWriter {
+  stream;
+  mutex = new Mutex();
+  closed = false;
+  constructor(stream) {
+    this.stream = stream;
+  }
+  static async open(filePath) {
+    await mkdir4(path7.dirname(filePath), { recursive: true });
+    const stream = createWriteStream(filePath, { flags: "w", encoding: "utf8" });
+    return new _JsonlWriter(stream);
+  }
+  async append(record) {
+    await this.mutex.runExclusive(async () => {
+      if (this.closed) {
+        throw new Error("Cannot write to closed JSONL writer");
+      }
+      const snakeCaseRecord = toSnakeCaseDeep(record);
+      const line = `${JSON.stringify(snakeCaseRecord)}
+`;
+      if (!this.stream.write(line)) {
+        await new Promise((resolve, reject) => {
+          this.stream.once("drain", resolve);
+          this.stream.once("error", reject);
+        });
+      }
+    });
+  }
+  async close() {
+    if (this.closed) {
+      return;
+    }
+    this.closed = true;
+    this.stream.end();
+    await finished(this.stream);
+  }
+};
+
+// src/commands/eval/junit-writer.ts
+import { mkdir as mkdir5, writeFile as writeFile5 } from "node:fs/promises";
+import path8 from "node:path";
+function escapeXml(str) {
+  return str.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+var JunitWriter = class _JunitWriter {
+  filePath;
+  results = [];
+  closed = false;
+  constructor(filePath) {
+    this.filePath = filePath;
+  }
+  static async open(filePath) {
+    await mkdir5(path8.dirname(filePath), { recursive: true });
+    return new _JunitWriter(filePath);
+  }
+  async append(result) {
+    if (this.closed) {
+      throw new Error("Cannot write to closed JUnit writer");
+    }
+    this.results.push(result);
+  }
+  async close() {
+    if (this.closed) {
+      return;
+    }
+    this.closed = true;
+    const grouped = /* @__PURE__ */ new Map();
+    for (const result of this.results) {
+      const suite = result.dataset ?? "default";
+      const existing = grouped.get(suite);
+      if (existing) {
+        existing.push(result);
+      } else {
+        grouped.set(suite, [result]);
+      }
+    }
+    const suiteXmls = [];
+    for (const [suiteName, results] of grouped) {
+      const failures = results.filter((r) => r.score < 0.5).length;
+      const errors = results.filter((r) => r.error !== void 0).length;
+      const testCases = results.map((r) => {
+        const time = r.durationMs ? (r.durationMs / 1e3).toFixed(3) : "0.000";
+        let inner = "";
+        if (r.error) {
+          inner = `
+      <error message="${escapeXml(r.error)}">${escapeXml(r.error)}</error>
+    `;
+        } else if (r.score < 0.5) {
+          const message = `score=${r.score.toFixed(3)}`;
+          const detail = [
+            `Score: ${r.score.toFixed(3)}`,
+            r.reasoning ? `Reasoning: ${r.reasoning}` : "",
+            r.misses.length > 0 ? `Misses: ${r.misses.join(", ")}` : ""
+          ].filter(Boolean).join("\n");
+          inner = `
+      <failure message="${escapeXml(message)}">${escapeXml(detail)}</failure>
+    `;
+        }
+        return `    <testcase name="${escapeXml(r.testId)}" classname="${escapeXml(suiteName)}" time="${time}">${inner}</testcase>`;
+      });
+      suiteXmls.push(
+        `  <testsuite name="${escapeXml(suiteName)}" tests="${results.length}" failures="${failures}" errors="${errors}">
+${testCases.join("\n")}
+  </testsuite>`
+      );
+    }
+    const totalTests = this.results.length;
+    const totalFailures = this.results.filter((r) => r.score < 0.5).length;
+    const totalErrors = this.results.filter((r) => r.error !== void 0).length;
+    const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="${totalTests}" failures="${totalFailures}" errors="${totalErrors}">
+${suiteXmls.join("\n")}
+</testsuites>
+`;
+    await writeFile5(this.filePath, xml, "utf8");
+  }
+};
+
+// src/commands/eval/yaml-writer.ts
+import { createWriteStream as createWriteStream2 } from "node:fs";
+import { mkdir as mkdir6 } from "node:fs/promises";
+import path9 from "node:path";
+import { finished as finished2 } from "node:stream/promises";
+import { stringify as stringifyYaml } from "yaml";
+var YamlWriter = class _YamlWriter {
+  stream;
+  mutex = new Mutex();
+  closed = false;
+  isFirst = true;
+  constructor(stream) {
+    this.stream = stream;
+  }
+  static async open(filePath) {
+    await mkdir6(path9.dirname(filePath), { recursive: true });
+    const stream = createWriteStream2(filePath, { flags: "w", encoding: "utf8" });
+    return new _YamlWriter(stream);
+  }
+  async append(record) {
+    await this.mutex.runExclusive(async () => {
+      if (this.closed) {
+        throw new Error("Cannot write to closed YAML writer");
+      }
+      const snakeCaseRecord = toSnakeCaseDeep(record);
+      const yamlDoc = stringifyYaml(snakeCaseRecord, {
+        indent: 2,
+        lineWidth: 0
+        // Disable line wrapping
+        // Let YAML library choose appropriate string style based on content
+        // (will use block literal for multiline strings with actual newlines)
+      });
+      const normalizedYaml = normalizeLineEndings(yamlDoc);
+      const separator = this.isFirst ? "---\n" : "\n---\n";
+      this.isFirst = false;
+      const content = `${separator}${normalizedYaml}`;
+      if (!this.stream.write(content)) {
+        await new Promise((resolve, reject) => {
+          this.stream.once("drain", resolve);
+          this.stream.once("error", reject);
+        });
+      }
+    });
+  }
+  async close() {
+    if (this.closed) {
+      return;
+    }
+    this.closed = true;
+    this.stream.end();
+    await finished2(this.stream);
+  }
+};
+
+// src/commands/eval/output-writer.ts
+async function createOutputWriter(filePath, format) {
+  switch (format) {
+    case "jsonl":
+      return JsonlWriter.open(filePath);
+    case "yaml":
+      return YamlWriter.open(filePath);
+    case "html":
+      return HtmlWriter.open(filePath);
+    default: {
+      const exhaustiveCheck = format;
+      throw new Error(`Unsupported output format: ${exhaustiveCheck}`);
+    }
+  }
+}
+function getDefaultExtension(format) {
+  switch (format) {
+    case "jsonl":
+      return ".jsonl";
+    case "yaml":
+      return ".yaml";
+    case "html":
+      return ".html";
+    default: {
+      const exhaustiveCheck = format;
+      throw new Error(`Unsupported output format: ${exhaustiveCheck}`);
+    }
+  }
+}
+var SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([".jsonl", ".json", ".xml", ".yaml", ".yml", ".html", ".htm"]);
+function createWriterFromPath(filePath) {
+  const ext = path10.extname(filePath).toLowerCase();
+  switch (ext) {
+    case ".jsonl":
+      return JsonlWriter.open(filePath);
+    case ".json":
+      return JsonWriter.open(filePath);
+    case ".xml":
+      return JunitWriter.open(filePath);
+    case ".yaml":
+    case ".yml":
+      return YamlWriter.open(filePath);
+    case ".html":
+    case ".htm":
+      return HtmlWriter.open(filePath);
+    default:
+      throw new Error(
+        `Unsupported output file extension "${ext}". Supported: ${[...SUPPORTED_EXTENSIONS].join(", ")}`
+      );
+  }
+}
+async function createMultiWriter(filePaths) {
+  const writers = await Promise.all(filePaths.map((fp) => createWriterFromPath(fp)));
+  return {
+    async append(result) {
+      await Promise.all(writers.map((w) => w.append(result)));
+    },
+    async close() {
+      await Promise.all(writers.map((w) => w.close()));
+    }
+  };
+}
+
+// src/commands/eval/progress-display.ts
+var ProgressDisplay = class {
+  workers = /* @__PURE__ */ new Map();
+  totalTests = 0;
+  completedTests = 0;
+  logPaths = [];
+  logPathSet = /* @__PURE__ */ new Set();
+  hasPrintedLogHeader = false;
+  started = false;
+  finished = false;
+  verbose;
+  constructor(_maxWorkers, options) {
+    this.verbose = options?.verbose ?? false;
+  }
+  isInteractiveMode() {
+    return false;
+  }
+  start() {
+    this.started = true;
+    this.finished = false;
+  }
+  setTotalTests(count) {
+    this.totalTests = count;
+  }
+  updateWorker(progress) {
+    const previous = this.workers.get(progress.workerId);
+    this.workers.set(progress.workerId, progress);
+    if (progress.status === "completed" || progress.status === "failed") {
+      this.completedTests++;
+    }
+    const targetSuffix = progress.targetLabel ? ` | ${progress.targetLabel}` : "";
+    const countPrefix = `${this.completedTests}/${this.totalTests}`;
+    switch (progress.status) {
+      case "pending":
+        if (this.verbose && !previous) {
+          console.log(`${countPrefix}   \u23F3 ${progress.testId}${targetSuffix}`);
+        }
+        break;
+      case "running":
+        if (!previous || previous.status === "pending") {
+          console.log(`${countPrefix}   \u{1F504} ${progress.testId}${targetSuffix}`);
+        }
+        break;
+      case "completed":
+        console.log(`${countPrefix}   \u2705 ${progress.testId}${targetSuffix}`);
+        break;
+      case "failed":
+        console.log(
+          `${countPrefix}   \u274C ${progress.testId}${targetSuffix}${progress.error ? `: ${progress.error}` : ""}`
+        );
+        break;
+    }
+  }
+  addLogPaths(paths, provider) {
+    const newPaths = [];
+    for (const path13 of paths) {
+      if (this.logPathSet.has(path13)) {
+        continue;
+      }
+      this.logPathSet.add(path13);
+      newPaths.push(path13);
+    }
+    if (newPaths.length === 0) {
+      return;
+    }
+    this.logPaths.push(...newPaths);
+    if (!this.hasPrintedLogHeader) {
+      console.log("");
+      const label = provider === "pi" ? "Pi Coding Agent" : provider === "copilot" ? "Copilot CLI" : "Codex CLI";
+      console.log(`${label} logs:`);
+      this.hasPrintedLogHeader = true;
+    }
+    const startIndex = this.logPaths.length - newPaths.length;
+    newPaths.forEach((path13, offset) => {
+      console.log(`${startIndex + offset + 1}. ${path13}`);
+    });
+  }
+  finish() {
+    this.finished = true;
+    console.log("");
+  }
+  clear() {
+  }
+};
+
+// src/commands/eval/retry-errors.ts
+import { createReadStream } from "node:fs";
+import { createInterface } from "node:readline";
+async function loadErrorTestIds(jsonlPath) {
+  const ids = [];
+  const rl = createInterface({
+    input: createReadStream(jsonlPath),
+    crlfDelay: Number.POSITIVE_INFINITY
+  });
+  for await (const line of rl) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const parsed = JSON.parse(trimmed);
+      if (parsed.executionStatus === "execution_error" && parsed.testId) {
+        ids.push(parsed.testId);
+      }
+    } catch {
+    }
+  }
+  return [...new Set(ids)];
+}
+async function loadNonErrorResults(jsonlPath) {
+  const results = [];
+  const rl = createInterface({
+    input: createReadStream(jsonlPath),
+    crlfDelay: Number.POSITIVE_INFINITY
+  });
+  for await (const line of rl) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const parsed = JSON.parse(trimmed);
+      if (!parsed.testId || parsed.score === void 0) continue;
+      if (parsed.executionStatus !== "execution_error") {
+        results.push(parsed);
+      }
+    } catch {
+    }
+  }
+  return results;
+}
+
+// src/commands/eval/statistics.ts
+var HISTOGRAM_BREAKPOINTS = [0, 0.2, 0.4, 0.6, 0.8, 1];
+function computeMean(values) {
+  if (values.length === 0) {
+    return 0;
+  }
+  const sum = values.reduce((acc, value) => acc + value, 0);
+  return sum / values.length;
+}
+function computeMedian(values) {
+  if (values.length === 0) {
+    return 0;
+  }
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  if (sorted.length % 2 === 0) {
+    return (sorted[mid - 1] + sorted[mid]) / 2;
+  }
+  return sorted[mid];
+}
+function computeStandardDeviation(values) {
+  if (values.length < 2) {
+    return void 0;
+  }
+  const mean = computeMean(values);
+  const variance = values.reduce((acc, value) => acc + (value - mean) ** 2, 0) / (values.length - 1);
+  return Math.sqrt(variance);
+}
+function buildHistogram(values) {
+  const bins = [];
+  for (let index = 0; index < HISTOGRAM_BREAKPOINTS.length - 1; index += 1) {
+    bins.push({
+      range: [HISTOGRAM_BREAKPOINTS[index], HISTOGRAM_BREAKPOINTS[index + 1]],
+      count: 0
+    });
+  }
+  for (const value of values) {
+    for (const bin of bins) {
+      const [start, end] = bin.range;
+      const isLastBin = end === HISTOGRAM_BREAKPOINTS[HISTOGRAM_BREAKPOINTS.length - 1];
+      const withinRange = isLastBin ? value >= start && value <= end : value >= start && value < end + 1e-9;
+      if (withinRange) {
+        bin.count += 1;
+        break;
+      }
+    }
+  }
+  return bins;
+}
+function calculateEvaluationSummary(results) {
+  const total = results.length;
+  const errors = results.filter((result) => result.error !== void 0).map((result) => ({ testId: result.testId, error: result.error }));
+  const errorCount = errors.length;
+  if (total === 0) {
+    return {
+      total: 0,
+      mean: 0,
+      median: 0,
+      min: 0,
+      max: 0,
+      standardDeviation: void 0,
+      histogram: buildHistogram([]),
+      topResults: [],
+      bottomResults: [],
+      errorCount: 0,
+      errors: [],
+      executionErrorCount: 0,
+      qualityFailureCount: 0,
+      passedCount: 0,
+      byFailureStage: {},
+      byFailureReason: {}
+    };
+  }
+  const executionErrors = results.filter((r) => r.executionStatus === "execution_error");
+  const qualityResults = results.filter((r) => r.executionStatus !== "execution_error");
+  const qualityScores = qualityResults.map((r) => r.score);
+  const mean = computeMean(qualityScores);
+  const median = computeMedian(qualityScores);
+  const min = qualityScores.length > 0 ? Math.min(...qualityScores) : 0;
+  const max = qualityScores.length > 0 ? Math.max(...qualityScores) : 0;
+  const standardDeviation = computeStandardDeviation(qualityScores);
+  const histogram = buildHistogram(qualityScores);
+  const sortedResults = [...qualityResults].sort((a, b) => b.score - a.score);
+  const topResults = sortedResults.slice(0, Math.min(3, sortedResults.length));
+  const bottomResults = sortedResults.slice(-Math.min(3, sortedResults.length));
+  const executionErrorCount = executionErrors.length;
+  const qualityFailureCount = results.filter((r) => r.executionStatus === "quality_failure").length;
+  const passedCount = results.filter((r) => r.executionStatus === "ok").length;
+  const byFailureStage = {};
+  const byFailureReason = {};
+  for (const result of executionErrors) {
+    if (result.failureStage) {
+      byFailureStage[result.failureStage] = (byFailureStage[result.failureStage] ?? 0) + 1;
+    }
+    if (result.failureReasonCode) {
+      byFailureReason[result.failureReasonCode] = (byFailureReason[result.failureReasonCode] ?? 0) + 1;
+    }
+  }
+  return {
+    total,
+    mean,
+    median,
+    min,
+    max,
+    standardDeviation,
+    histogram,
+    topResults,
+    bottomResults,
+    errorCount,
+    errors,
+    executionErrorCount,
+    qualityFailureCount,
+    passedCount,
+    byFailureStage,
+    byFailureReason
+  };
+}
+function formatScore(value) {
+  return value.toFixed(3);
+}
+function formatEvaluationSummary(summary) {
+  if (summary.total === 0) {
+    return "\nNo results to summarize";
+  }
+  const lines = [];
+  if (summary.errorCount > 0) {
+    lines.push("\n==================================================");
+    lines.push("EXECUTION ERRORS");
+    lines.push("==================================================");
+    for (const error of summary.errors) {
+      lines.push(`
+\u274C ${error.testId}`);
+      lines.push(`   ${error.error}`);
+    }
+    lines.push("");
+  }
+  lines.push("\n==================================================");
+  lines.push("EVALUATION SUMMARY");
+  lines.push("==================================================");
+  lines.push(`Total tests: ${summary.total}`);
+  lines.push(`Passed: ${summary.passedCount}`);
+  if (summary.qualityFailureCount > 0) {
+    lines.push(`Quality failures: ${summary.qualityFailureCount}`);
+  }
+  if (summary.executionErrorCount > 0) {
+    lines.push(`Execution errors: ${summary.executionErrorCount}`);
+  }
+  if (summary.executionErrorCount > 0) {
+    const qualityCount = summary.total - summary.executionErrorCount;
+    lines.push(
+      `Mean score: ${formatScore(summary.mean)} (${qualityCount} quality tests, ${summary.executionErrorCount} execution errors excluded)`
+    );
+  } else {
+    lines.push(`Mean score: ${formatScore(summary.mean)}`);
+  }
+  lines.push(`Median score: ${formatScore(summary.median)}`);
+  lines.push(`Min score: ${formatScore(summary.min)}`);
+  lines.push(`Max score: ${formatScore(summary.max)}`);
+  if (typeof summary.standardDeviation === "number") {
+    lines.push(`Std deviation: ${formatScore(summary.standardDeviation)}`);
+  }
+  lines.push("\nScore distribution:");
+  for (const bin of summary.histogram) {
+    const [start, end] = bin.range;
+    lines.push(`  ${start.toFixed(1)}-${end.toFixed(1)}: ${bin.count}`);
+  }
+  lines.push("\nTop performing tests:");
+  summary.topResults.forEach((result, index) => {
+    lines.push(`  ${index + 1}. ${result.testId}: ${formatScore(result.score)}`);
+  });
+  lines.push("\nLowest performing tests:");
+  summary.bottomResults.forEach((result, index) => {
+    lines.push(`  ${index + 1}. ${result.testId}: ${formatScore(result.score)}`);
+  });
+  const failureStageEntries = Object.entries(summary.byFailureStage);
+  if (failureStageEntries.length > 0) {
+    lines.push("\nExecution errors by stage:");
+    for (const [stage, count] of failureStageEntries) {
+      lines.push(`  ${stage}: ${count}`);
+    }
+  }
+  const failureReasonEntries = Object.entries(summary.byFailureReason);
+  if (failureReasonEntries.length > 0) {
+    lines.push("\nExecution errors by reason:");
+    for (const [reason, count] of failureReasonEntries) {
+      lines.push(`  ${reason}: ${count}`);
+    }
+  }
+  return lines.join("\n");
+}
+function formatMatrixSummary(results) {
+  const targetSet = /* @__PURE__ */ new Set();
+  const testIdSet = /* @__PURE__ */ new Set();
+  for (const result of results) {
+    targetSet.add(result.target);
+    testIdSet.add(result.testId);
+  }
+  const targets = [...targetSet].sort();
+  const testIds = [...testIdSet].sort();
+  if (targets.length < 2) {
+    return "";
+  }
+  const scoreMap = /* @__PURE__ */ new Map();
+  for (const result of results) {
+    if (!scoreMap.has(result.testId)) {
+      scoreMap.set(result.testId, /* @__PURE__ */ new Map());
+    }
+    scoreMap.get(result.testId)?.set(result.target, result.score);
+  }
+  const lines = [];
+  lines.push("\n==================================================");
+  lines.push("MATRIX RESULTS (tests \xD7 targets)");
+  lines.push("==================================================");
+  const testIdColWidth = Math.max(7, ...testIds.map((id) => id.length));
+  const targetColWidth = Math.max(7, ...targets.map((t) => t.length));
+  const header = `${"Test".padEnd(testIdColWidth)}  ${targets.map((t) => t.padEnd(targetColWidth)).join("  ")}`;
+  lines.push(header);
+  lines.push("-".repeat(header.length));
+  for (const testId of testIds) {
+    const cells = targets.map((target) => {
+      const score = scoreMap.get(testId)?.get(target);
+      return score !== void 0 ? formatScore(score).padEnd(targetColWidth) : "-".padEnd(targetColWidth);
+    });
+    lines.push(`${testId.padEnd(testIdColWidth)}  ${cells.join("  ")}`);
+  }
+  lines.push("-".repeat(header.length));
+  const avgCells = targets.map((target) => {
+    const scores = results.filter((r) => r.target === target).map((r) => r.score);
+    const avg = scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0;
+    return formatScore(avg).padEnd(targetColWidth);
+  });
+  lines.push(`${"Average".padEnd(testIdColWidth)}  ${avgCells.join("  ")}`);
+  return lines.join("\n");
+}
+
+// ../../packages/core/dist/evaluation/validation/index.js
+import { readFile as readFile2 } from "node:fs/promises";
+import path11 from "node:path";
+import { parse } from "yaml";
+import { readFile as readFile22 } from "node:fs/promises";
+import path22 from "node:path";
+import { parse as parse2 } from "yaml";
+import { readFile as readFile3 } from "node:fs/promises";
+import path32 from "node:path";
+import { parse as parse3 } from "yaml";
+import { readFile as readFile4 } from "node:fs/promises";
+import { parse as parse4 } from "yaml";
+import { readFile as readFile5 } from "node:fs/promises";
+import path42 from "node:path";
+import { parse as parse5 } from "yaml";
+var SCHEMA_EVAL_V2 = "agentv-eval-v2";
+var SCHEMA_TARGETS_V2 = "agentv-targets-v2.2";
+var SCHEMA_CONFIG_V2 = "agentv-config-v2";
+async function detectFileType(filePath) {
+  try {
+    const content = await readFile2(filePath, "utf8");
+    const parsed = parse(content);
+    if (typeof parsed !== "object" || parsed === null) {
+      return inferFileTypeFromPath(filePath);
+    }
+    const record = parsed;
+    const schema = record.$schema;
+    if (typeof schema !== "string") {
+      return inferFileTypeFromPath(filePath);
+    }
+    switch (schema) {
+      case SCHEMA_EVAL_V2:
+        return "eval";
+      case SCHEMA_TARGETS_V2:
+        return "targets";
+      case SCHEMA_CONFIG_V2:
+        return "config";
+      default:
+        return inferFileTypeFromPath(filePath);
+    }
+  } catch {
+    return inferFileTypeFromPath(filePath);
+  }
+}
+function inferFileTypeFromPath(filePath) {
+  const normalized = path11.normalize(filePath).replace(/\\/g, "/");
+  const basename = path11.basename(filePath);
+  if (normalized.includes("/.agentv/")) {
+    if (basename === "config.yaml" || basename === "config.yml") {
+      return "config";
+    }
+    if (basename === "targets.yaml" || basename === "targets.yml") {
+      return "targets";
+    }
+  }
+  return "eval";
+}
+var ASSERTION_TYPES_WITH_STRING_VALUE = /* @__PURE__ */ new Set([
+  "contains",
+  "icontains",
+  "starts-with",
+  "ends-with",
+  "equals",
+  "regex"
+]);
+var ASSERTION_TYPES_WITH_ARRAY_VALUE = /* @__PURE__ */ new Set([
+  "contains-any",
+  "contains-all",
+  "icontains-any",
+  "icontains-all"
+]);
+var VALID_TEST_FILE_EXTENSIONS = /* @__PURE__ */ new Set([".yaml", ".yml", ".jsonl"]);
+var NAME_PATTERN = /^[a-z0-9-]+$/;
+function isObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+async function validateEvalFile(filePath) {
+  const errors = [];
+  const absolutePath = path22.resolve(filePath);
+  let parsed;
+  try {
+    const content = await readFile22(absolutePath, "utf8");
+    parsed = parse2(content);
+  } catch (error) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      message: `Failed to parse YAML: ${error.message}`
+    });
+    return {
+      valid: false,
+      filePath: absolutePath,
+      fileType: "eval",
+      errors
+    };
+  }
+  if (!isObject(parsed)) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      message: "File must contain a YAML object"
+    });
+    return {
+      valid: false,
+      filePath: absolutePath,
+      fileType: "eval",
+      errors
+    };
+  }
+  validateMetadata(parsed, absolutePath, errors);
+  const suiteInput = parsed.input;
+  if (suiteInput !== void 0) {
+    if (typeof suiteInput === "string") {
+    } else if (Array.isArray(suiteInput)) {
+      validateMessages(suiteInput, "input", absolutePath, errors);
+    } else {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: "input",
+        message: "Invalid suite-level 'input' field (must be a string or array of messages)"
+      });
+    }
+  }
+  let cases = parsed.tests;
+  if (cases === void 0 && "eval_cases" in parsed) {
+    cases = parsed.eval_cases;
+    errors.push({
+      severity: "warning",
+      filePath: absolutePath,
+      location: "eval_cases",
+      message: "'eval_cases' is deprecated. Use 'tests' instead."
+    });
+  }
+  if (cases === void 0 && "evalcases" in parsed) {
+    cases = parsed.evalcases;
+    errors.push({
+      severity: "warning",
+      filePath: absolutePath,
+      location: "evalcases",
+      message: "'evalcases' is deprecated. Use 'tests' instead."
+    });
+  }
+  if (typeof cases === "string") {
+    validateTestsStringPath(cases, absolutePath, errors);
+    return {
+      valid: errors.filter((e) => e.severity === "error").length === 0,
+      filePath: absolutePath,
+      fileType: "eval",
+      errors
+    };
+  }
+  if (!Array.isArray(cases)) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      location: "tests",
+      message: "Missing or invalid 'tests' field (must be an array or a file path string)"
+    });
+    return {
+      valid: errors.length === 0,
+      filePath: absolutePath,
+      fileType: "eval",
+      errors
+    };
+  }
+  for (let i = 0; i < cases.length; i++) {
+    const evalCase = cases[i];
+    const location = `tests[${i}]`;
+    if (!isObject(evalCase)) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location,
+        message: "Eval case must be an object"
+      });
+      continue;
+    }
+    const id = evalCase.id;
+    if (typeof id !== "string" || id.trim().length === 0) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.id`,
+        message: "Missing or invalid 'id' field (must be a non-empty string)"
+      });
+    }
+    let criteria = evalCase.criteria;
+    if (criteria === void 0 && "expected_outcome" in evalCase) {
+      criteria = evalCase.expected_outcome;
+      errors.push({
+        severity: "warning",
+        filePath: absolutePath,
+        location: `${location}.expected_outcome`,
+        message: "'expected_outcome' is deprecated. Use 'criteria' instead."
+      });
+    }
+    if (criteria !== void 0 && (typeof criteria !== "string" || criteria.trim().length === 0)) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.criteria`,
+        message: "Invalid 'criteria' field (must be a non-empty string if provided)"
+      });
+    }
+    const inputField = evalCase.input;
+    if (inputField !== void 0) {
+      if (typeof inputField === "string") {
+      } else if (Array.isArray(inputField)) {
+        validateMessages(inputField, `${location}.input`, absolutePath, errors);
+      } else {
+        errors.push({
+          severity: "error",
+          filePath: absolutePath,
+          location: `${location}.input`,
+          message: "Invalid 'input' field (must be a string or array of messages)"
+        });
+      }
+    } else {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.input`,
+        message: "Missing 'input' field (must be a string or array of messages)"
+      });
+    }
+    const expectedOutputField = evalCase.expected_output;
+    if (expectedOutputField !== void 0) {
+      if (typeof expectedOutputField === "string") {
+      } else if (Array.isArray(expectedOutputField)) {
+        if (expectedOutputField.length > 0 && isObject(expectedOutputField[0]) && "role" in expectedOutputField[0]) {
+          validateMessages(
+            expectedOutputField,
+            `${location}.expected_output`,
+            absolutePath,
+            errors
+          );
+        }
+      } else if (isObject(expectedOutputField)) {
+      } else {
+        errors.push({
+          severity: "error",
+          filePath: absolutePath,
+          location: `${location}.expected_output`,
+          message: "Invalid 'expected_output' field (must be a string, object, or array)"
+        });
+      }
+    }
+    const assertField = evalCase.assertions ?? evalCase.assert;
+    if (assertField !== void 0) {
+      validateAssertArray(assertField, location, absolutePath, errors);
+    }
+  }
+  if (isObject(parsed.workspace)) {
+    validateWorkspaceRepoConfig(parsed.workspace, absolutePath, errors);
+  }
+  return {
+    valid: errors.filter((e) => e.severity === "error").length === 0,
+    filePath: absolutePath,
+    fileType: "eval",
+    errors
+  };
+}
+function validateWorkspaceRepoConfig(workspace, filePath, errors) {
+  const repos = workspace.repos;
+  const hooks = workspace.hooks;
+  const afterEachHook = isObject(hooks) ? hooks.after_each : void 0;
+  const isolation = workspace.isolation;
+  if (Array.isArray(repos)) {
+    for (const repo of repos) {
+      if (!isObject(repo)) continue;
+      const checkout = repo.checkout;
+      const clone = repo.clone;
+      if (isObject(checkout) && isObject(clone)) {
+        const ancestor = checkout.ancestor;
+        const depth = clone.depth;
+        if (typeof ancestor === "number" && typeof depth === "number" && depth < ancestor + 1) {
+          errors.push({
+            severity: "warning",
+            filePath,
+            location: `workspace.repos[path=${repo.path}]`,
+            message: `clone.depth (${depth}) may be insufficient for checkout.ancestor (${ancestor}). Recommend depth >= ${ancestor + 1}.`
+          });
+        }
+      }
+    }
+  }
+  if (isObject(afterEachHook) && afterEachHook.reset && afterEachHook.reset !== "none") {
+    if (!Array.isArray(repos) || repos.length === 0) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location: "workspace.hooks.after_each",
+        message: `hooks.after_each.reset '${afterEachHook.reset}' has no effect without repos.`
+      });
+    }
+  }
+  if (isObject(afterEachHook) && afterEachHook.reset && isolation === "per_test") {
+    errors.push({
+      severity: "warning",
+      filePath,
+      location: "workspace.hooks.after_each",
+      message: "hooks.after_each.reset is redundant with isolation: per_test (each test gets a fresh workspace)."
+    });
+  }
+}
+function validateMessages(messages, location, filePath, errors) {
+  for (let i = 0; i < messages.length; i++) {
+    const message = messages[i];
+    const msgLocation = `${location}[${i}]`;
+    if (!isObject(message)) {
+      errors.push({
+        severity: "error",
+        filePath,
+        location: msgLocation,
+        message: "Message must be an object"
+      });
+      continue;
+    }
+    const role = message.role;
+    const validRoles = ["system", "user", "assistant"];
+    if (!validRoles.includes(role)) {
+      errors.push({
+        severity: "error",
+        filePath,
+        location: `${msgLocation}.role`,
+        message: `Invalid role '${role}'. Must be one of: ${validRoles.join(", ")}`
+      });
+    }
+    const content = message.content;
+    if (typeof content === "string") {
+      validateContentForRoleMarkers(content, `${msgLocation}.content`, filePath, errors);
+    } else if (Array.isArray(content)) {
+      for (let j = 0; j < content.length; j++) {
+        const contentItem = content[j];
+        const contentLocation = `${msgLocation}.content[${j}]`;
+        if (typeof contentItem === "string") {
+          validateContentForRoleMarkers(contentItem, contentLocation, filePath, errors);
+        } else if (isObject(contentItem)) {
+          const type = contentItem.type;
+          if (typeof type !== "string") {
+            errors.push({
+              severity: "error",
+              filePath,
+              location: `${contentLocation}.type`,
+              message: "Content object must have a 'type' field"
+            });
+          }
+          if (type === "text") {
+            const value = contentItem.value;
+            if (typeof value !== "string") {
+              errors.push({
+                severity: "error",
+                filePath,
+                location: `${contentLocation}.value`,
+                message: "Content with type 'text' must have a 'value' field"
+              });
+            } else {
+              validateContentForRoleMarkers(value, `${contentLocation}.value`, filePath, errors);
+            }
+          }
+        } else {
+          errors.push({
+            severity: "error",
+            filePath,
+            location: contentLocation,
+            message: "Content array items must be strings or objects"
+          });
+        }
+      }
+    } else {
+      errors.push({
+        severity: "error",
+        filePath,
+        location: `${msgLocation}.content`,
+        message: "Missing or invalid 'content' field (must be a string or array)"
+      });
+    }
+  }
+}
+function validateMetadata(parsed, filePath, errors) {
+  const name = parsed.name;
+  if (name !== void 0) {
+    if (typeof name === "string") {
+      if (!NAME_PATTERN.test(name)) {
+        errors.push({
+          severity: "warning",
+          filePath,
+          location: "name",
+          message: `Invalid 'name' format '${name}'. Must match pattern /^[a-z0-9-]+$/ (lowercase alphanumeric with hyphens).`
+        });
+      }
+    }
+    if (!("description" in parsed) || parsed.description === void 0) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location: "name",
+        message: "When 'name' is present, 'description' should also be provided."
+      });
+    }
+  }
+}
+function validateTestsStringPath(testsPath, filePath, errors) {
+  const ext = path22.extname(testsPath);
+  if (!VALID_TEST_FILE_EXTENSIONS.has(ext)) {
+    errors.push({
+      severity: "warning",
+      filePath,
+      location: "tests",
+      message: `Unsupported file extension '${ext}' for tests path '${testsPath}'. Supported extensions: ${[...VALID_TEST_FILE_EXTENSIONS].join(", ")}`
+    });
+  }
+}
+function validateAssertArray(assertField, parentLocation, filePath, errors) {
+  if (!Array.isArray(assertField)) {
+    errors.push({
+      severity: "warning",
+      filePath,
+      location: `${parentLocation}.assertions`,
+      message: "'assertions' must be an array of assertion objects."
+    });
+    return;
+  }
+  const objectItems = [];
+  for (let i = 0; i < assertField.length; i++) {
+    const item = assertField[i];
+    if (typeof item === "string") {
+      if (item.trim().length === 0) {
+        errors.push({
+          severity: "warning",
+          filePath,
+          location: `${parentLocation}.assertions[${i}]`,
+          message: "Empty string assertion item will be ignored."
+        });
+      }
+      continue;
+    }
+    if (!isObject(item)) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location: `${parentLocation}.assertions[${i}]`,
+        message: "Assertion item must be a string or an object with a type field."
+      });
+      continue;
+    }
+    objectItems.push({ item, index: i });
+  }
+  for (const { item, index } of objectItems) {
+    const location = `${parentLocation}.assertions[${index}]`;
+    const rawTypeValue = item.type;
+    if (rawTypeValue === void 0 || typeof rawTypeValue !== "string") {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location: `${location}.type`,
+        message: "Assertion item is missing a 'type' field."
+      });
+      continue;
+    }
+    const typeValue = rawTypeValue.replace(/_/g, "-");
+    if (!isEvaluatorKind(typeValue)) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location: `${location}.type`,
+        message: `Unknown assertion type '${rawTypeValue}'.`
+      });
+      continue;
+    }
+    if (ASSERTION_TYPES_WITH_STRING_VALUE.has(typeValue)) {
+      const value = item.value;
+      if (value === void 0 || typeof value !== "string") {
+        errors.push({
+          severity: "warning",
+          filePath,
+          location: `${location}.value`,
+          message: `Assertion type '${typeValue}' requires a 'value' field (string).`
+        });
+        continue;
+      }
+      if (typeValue === "regex") {
+        try {
+          new RegExp(value);
+        } catch {
+          errors.push({
+            severity: "warning",
+            filePath,
+            location: `${location}.value`,
+            message: `Invalid regex pattern '${value}': not a valid regular expression.`
+          });
+        }
+      }
+    }
+    if (ASSERTION_TYPES_WITH_ARRAY_VALUE.has(typeValue)) {
+      const value = item.value;
+      if (!Array.isArray(value) || value.length === 0) {
+        errors.push({
+          severity: "warning",
+          filePath,
+          location: `${location}.value`,
+          message: `Assertion type '${typeValue}' requires a 'value' field (non-empty string array).`
+        });
+        continue;
+      }
+    }
+    const required = item.required;
+    if (required !== void 0) {
+      validateRequiredField(required, location, filePath, errors);
+    }
+  }
+}
+function validateRequiredField(required, parentLocation, filePath, errors) {
+  if (typeof required === "boolean") {
+    return;
+  }
+  if (typeof required === "number") {
+    if (required <= 0 || required > 1) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location: `${parentLocation}.required`,
+        message: `Invalid 'required' value ${required}. When a number, it must be between 0 (exclusive) and 1 (inclusive).`
+      });
+    }
+    return;
+  }
+  errors.push({
+    severity: "warning",
+    filePath,
+    location: `${parentLocation}.required`,
+    message: `Invalid 'required' value. Must be a boolean or a number between 0 (exclusive) and 1 (inclusive).`
+  });
+}
+function validateContentForRoleMarkers(content, location, filePath, errors) {
+  const markers = ["@[System]:", "@[User]:", "@[Assistant]:", "@[Tool]:"];
+  for (const marker of markers) {
+    if (content.toLowerCase().includes(marker.toLowerCase())) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        location,
+        message: `Content contains potential role marker '${marker}'. This may confuse agentic providers or cause prompt injection.`
+      });
+    }
+  }
+}
+function isObject2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+var COMMON_SETTINGS = /* @__PURE__ */ new Set(["provider_batching", "providerBatching"]);
+var RETRY_SETTINGS = /* @__PURE__ */ new Set([
+  "max_retries",
+  "maxRetries",
+  "retry_initial_delay_ms",
+  "retryInitialDelayMs",
+  "retry_max_delay_ms",
+  "retryMaxDelayMs",
+  "retry_backoff_factor",
+  "retryBackoffFactor",
+  "retry_status_codes",
+  "retryStatusCodes"
+]);
+var AZURE_SETTINGS = /* @__PURE__ */ new Set([
+  ...COMMON_SETTINGS,
+  ...RETRY_SETTINGS,
+  "endpoint",
+  "resource",
+  "resourceName",
+  "api_key",
+  "apiKey",
+  "deployment",
+  "deploymentName",
+  "model",
+  "version",
+  "api_version",
+  "temperature",
+  "max_output_tokens",
+  "maxTokens"
+]);
+var ANTHROPIC_SETTINGS = /* @__PURE__ */ new Set([
+  ...COMMON_SETTINGS,
+  ...RETRY_SETTINGS,
+  "api_key",
+  "apiKey",
+  "model",
+  "deployment",
+  "variant",
+  "temperature",
+  "max_output_tokens",
+  "maxTokens",
+  "thinking_budget",
+  "thinkingBudget"
+]);
+var GEMINI_SETTINGS = /* @__PURE__ */ new Set([
+  ...COMMON_SETTINGS,
+  ...RETRY_SETTINGS,
+  "api_key",
+  "apiKey",
+  "model",
+  "deployment",
+  "variant",
+  "temperature",
+  "max_output_tokens",
+  "maxTokens"
+]);
+var CODEX_SETTINGS = /* @__PURE__ */ new Set([
+  ...COMMON_SETTINGS,
+  "model",
+  "executable",
+  "command",
+  "binary",
+  "args",
+  "arguments",
+  "cwd",
+  "timeout_seconds",
+  "timeoutSeconds",
+  "log_dir",
+  "logDir",
+  "log_directory",
+  "logDirectory",
+  "log_format",
+  "logFormat",
+  "log_output_format",
+  "logOutputFormat",
+  "system_prompt",
+  "systemPrompt",
+  "workspace_template",
+  "workspaceTemplate"
+]);
+var COPILOT_SDK_SETTINGS = /* @__PURE__ */ new Set([
+  ...COMMON_SETTINGS,
+  "cli_url",
+  "cliUrl",
+  "cli_path",
+  "cliPath",
+  "github_token",
+  "githubToken",
+  "model",
+  "cwd",
+  "timeout_seconds",
+  "timeoutSeconds",
+  "log_dir",
+  "logDir",
+  "log_format",
+  "logFormat",
+  "system_prompt",
+  "systemPrompt",
+  "workspace_template",
+  "workspaceTemplate"
+]);
+var COPILOT_CLI_SETTINGS = /* @__PURE__ */ new Set([
+  ...COMMON_SETTINGS,
+  "executable",
+  "command",
+  "binary",
+  "args",
+  "arguments",
+  "model",
+  "cwd",
+  "timeout_seconds",
+  "timeoutSeconds",
+  "log_dir",
+  "logDir",
+  "log_format",
+  "logFormat",
+  "system_prompt",
+  "systemPrompt",
+  "workspace_template",
+  "workspaceTemplate"
+]);
+var VSCODE_SETTINGS = /* @__PURE__ */ new Set([
+  ...COMMON_SETTINGS,
+  "executable",
+  "workspace_template",
+  "workspaceTemplate",
+  "wait",
+  "dry_run",
+  "dryRun",
+  "subagent_root",
+  "subagentRoot",
+  "timeout_seconds",
+  "timeoutSeconds"
+]);
+var MOCK_SETTINGS = /* @__PURE__ */ new Set([
+  ...COMMON_SETTINGS,
+  "response",
+  "delayMs",
+  "delayMinMs",
+  "delayMaxMs",
+  "trace"
+  // For testing tool-trajectory evaluator
+]);
+var CLAUDE_SETTINGS = /* @__PURE__ */ new Set([
+  ...COMMON_SETTINGS,
+  "model",
+  "cwd",
+  "timeout_seconds",
+  "timeoutSeconds",
+  "log_dir",
+  "logDir",
+  "log_directory",
+  "logDirectory",
+  "log_format",
+  "logFormat",
+  "log_output_format",
+  "logOutputFormat",
+  "system_prompt",
+  "systemPrompt",
+  "workspace_template",
+  "workspaceTemplate",
+  "max_turns",
+  "maxTurns",
+  "max_budget_usd",
+  "maxBudgetUsd"
+]);
+function getKnownSettings(provider) {
+  const normalizedProvider = provider.toLowerCase();
+  switch (normalizedProvider) {
+    case "azure":
+    case "azure-openai":
+      return AZURE_SETTINGS;
+    case "anthropic":
+      return ANTHROPIC_SETTINGS;
+    case "gemini":
+    case "google":
+    case "google-gemini":
+      return GEMINI_SETTINGS;
+    case "codex":
+    case "codex-cli":
+      return CODEX_SETTINGS;
+    case "copilot-sdk":
+    case "copilot_sdk":
+      return COPILOT_SDK_SETTINGS;
+    case "copilot":
+    case "copilot-cli":
+      return COPILOT_CLI_SETTINGS;
+    case "claude":
+    case "claude-code":
+    case "claude-cli":
+    case "claude-sdk":
+      return CLAUDE_SETTINGS;
+    case "vscode":
+    case "vscode-insiders":
+      return VSCODE_SETTINGS;
+    case "mock":
+      return MOCK_SETTINGS;
+    case "cli":
+      return null;
+    default:
+      return null;
+  }
+}
+function validateUnknownSettings(target, provider, absolutePath, location, errors) {
+  const removedTargetFields = /* @__PURE__ */ new Set(["workspace_template", "workspaceTemplate"]);
+  const knownSettings = getKnownSettings(provider);
+  if (!knownSettings) {
+    return;
+  }
+  const baseFields = /* @__PURE__ */ new Set([
+    "name",
+    "provider",
+    "grader_target",
+    "judge_target",
+    "workers",
+    "$schema",
+    "targets"
+  ]);
+  for (const key of Object.keys(target)) {
+    if (removedTargetFields.has(key)) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.${key}`,
+        message: "target-level workspace_template has been removed. Use eval-level workspace.template."
+      });
+      continue;
+    }
+    if (!baseFields.has(key) && !knownSettings.has(key)) {
+      errors.push({
+        severity: "warning",
+        filePath: absolutePath,
+        location: `${location}.${key}`,
+        message: `Unknown setting '${key}' for ${provider} provider. This property will be ignored.`
+      });
+    }
+  }
+}
+async function validateTargetsFile(filePath) {
+  const errors = [];
+  const absolutePath = path32.resolve(filePath);
+  let parsed;
+  try {
+    const content = await readFile3(absolutePath, "utf8");
+    parsed = parse3(content);
+  } catch (error) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      message: `Failed to parse YAML: ${error.message}`
+    });
+    return {
+      valid: false,
+      filePath: absolutePath,
+      fileType: "targets",
+      errors
+    };
+  }
+  function validateCliSettings(target, absolutePath2, location, errors2) {
+    const command = target.command;
+    if (typeof command !== "string" || command.trim().length === 0) {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location: `${location}.command`,
+        message: "CLI provider requires 'command' as a non-empty string"
+      });
+    } else {
+      recordUnknownPlaceholders(command, absolutePath2, `${location}.command`, errors2);
+    }
+    const healthcheck = target.healthcheck;
+    if (healthcheck !== void 0) {
+      validateCliHealthcheck(healthcheck, absolutePath2, `${location}.healthcheck`, errors2);
+    }
+  }
+  function validateCliHealthcheck(healthcheck, absolutePath2, location, errors2) {
+    if (!isObject2(healthcheck)) {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location,
+        message: "'healthcheck' must be an object when provided"
+      });
+      return;
+    }
+    const timeoutSeconds = healthcheck.timeout_seconds ?? healthcheck.timeoutSeconds;
+    if (timeoutSeconds !== void 0) {
+      const numericTimeout = Number(timeoutSeconds);
+      if (!Number.isFinite(numericTimeout) || numericTimeout <= 0) {
+        errors2.push({
+          severity: "error",
+          filePath: absolutePath2,
+          location: `${location}.timeoutSeconds`,
+          message: "healthcheck.timeoutSeconds must be a positive number when provided"
+        });
+      }
+    }
+    const hasUrl = typeof healthcheck.url === "string" && healthcheck.url.trim().length > 0;
+    const hasCommand = typeof healthcheck.command === "string" && healthcheck.command.trim().length > 0;
+    if (!hasUrl && !hasCommand) {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location,
+        message: "healthcheck must have either 'url' (HTTP) or 'command' (command)"
+      });
+      return;
+    }
+    if (hasUrl) {
+      return;
+    }
+    recordUnknownPlaceholders(
+      healthcheck.command,
+      absolutePath2,
+      `${location}.command`,
+      errors2
+    );
+    const cwd = healthcheck.cwd;
+    if (cwd !== void 0 && typeof cwd !== "string") {
+      errors2.push({
+        severity: "error",
+        filePath: absolutePath2,
+        location: `${location}.cwd`,
+        message: "healthcheck.cwd must be a string when provided"
+      });
+    }
+  }
+  function recordUnknownPlaceholders(template, absolutePath2, location, errors2) {
+    const placeholders = extractPlaceholders(template);
+    for (const placeholder of placeholders) {
+      if (!CLI_PLACEHOLDERS.has(placeholder)) {
+        errors2.push({
+          severity: "error",
+          filePath: absolutePath2,
+          location,
+          message: `Unknown CLI placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS).join(", ")}`
+        });
+      }
+    }
+  }
+  function extractPlaceholders(template) {
+    const matches = template.matchAll(/\{([A-Z_]+)\}/g);
+    const result = [];
+    for (const match of matches) {
+      const placeholder = match[1];
+      if (placeholder) {
+        result.push(placeholder);
+      }
+    }
+    return result;
+  }
+  if (!isObject2(parsed)) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      message: "File must contain a YAML object"
+    });
+    return {
+      valid: false,
+      filePath: absolutePath,
+      fileType: "targets",
+      errors
+    };
+  }
+  const targets = parsed.targets;
+  if (!Array.isArray(targets)) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      location: "targets",
+      message: "Missing or invalid 'targets' field (must be an array)"
+    });
+    return {
+      valid: errors.length === 0,
+      filePath: absolutePath,
+      fileType: "targets",
+      errors
+    };
+  }
+  const knownProviders = [...KNOWN_PROVIDERS, ...PROVIDER_ALIASES];
+  for (let i = 0; i < targets.length; i++) {
+    const target = targets[i];
+    const location = `targets[${i}]`;
+    if (!isObject2(target)) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location,
+        message: "Target must be an object"
+      });
+      continue;
+    }
+    const name = target.name;
+    if (typeof name !== "string" || name.trim().length === 0) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.name`,
+        message: "Missing or invalid 'name' field (must be a non-empty string)"
+      });
+    }
+    const provider = target.provider;
+    const providerValue = typeof provider === "string" ? provider.trim().toLowerCase() : void 0;
+    if (typeof provider !== "string" || provider.trim().length === 0) {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.provider`,
+        message: "Missing or invalid 'provider' field (must be a non-empty string)"
+      });
+    } else if (!knownProviders.includes(provider)) {
+      errors.push({
+        severity: "warning",
+        filePath: absolutePath,
+        location: `${location}.provider`,
+        message: `Unknown provider '${provider}'. Known providers: ${knownProviders.join(", ")}`
+      });
+    }
+    if (providerValue === "cli") {
+      validateCliSettings(target, absolutePath, location, errors);
+    }
+    if (typeof provider === "string") {
+      validateUnknownSettings(target, provider, absolutePath, location, errors);
+    }
+    const graderTarget = target.grader_target ?? target.judge_target;
+    if (graderTarget !== void 0 && typeof graderTarget !== "string") {
+      errors.push({
+        severity: "error",
+        filePath: absolutePath,
+        location: `${location}.grader_target`,
+        message: "Invalid 'grader_target' field (must be a string)"
+      });
+    }
+  }
+  return {
+    valid: errors.filter((e) => e.severity === "error").length === 0,
+    filePath: absolutePath,
+    fileType: "targets",
+    errors
+  };
+}
+async function validateConfigFile(filePath) {
+  const errors = [];
+  try {
+    const content = await readFile4(filePath, "utf8");
+    const parsed = parse4(content);
+    if (typeof parsed !== "object" || parsed === null) {
+      errors.push({
+        severity: "error",
+        filePath,
+        message: "Config file must contain a valid YAML object"
+      });
+      return { valid: false, filePath, fileType: "config", errors };
+    }
+    const config = parsed;
+    const guidelinePatterns = config.guideline_patterns;
+    if (guidelinePatterns !== void 0) {
+      if (!Array.isArray(guidelinePatterns)) {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: "guideline_patterns",
+          message: "Field 'guideline_patterns' must be an array"
+        });
+      } else if (!guidelinePatterns.every((p) => typeof p === "string")) {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: "guideline_patterns",
+          message: "All entries in 'guideline_patterns' must be strings"
+        });
+      } else if (guidelinePatterns.length === 0) {
+        errors.push({
+          severity: "warning",
+          filePath,
+          location: "guideline_patterns",
+          message: "Field 'guideline_patterns' is empty. Consider removing it or adding patterns."
+        });
+      }
+    }
+    const evalPatterns = config.eval_patterns;
+    if (evalPatterns !== void 0) {
+      if (!Array.isArray(evalPatterns)) {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: "eval_patterns",
+          message: "Field 'eval_patterns' must be an array"
+        });
+      } else if (!evalPatterns.every((p) => typeof p === "string")) {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: "eval_patterns",
+          message: "All entries in 'eval_patterns' must be strings"
+        });
+      } else if (evalPatterns.length === 0) {
+        errors.push({
+          severity: "warning",
+          filePath,
+          location: "eval_patterns",
+          message: "Field 'eval_patterns' is empty. Consider removing it or adding patterns."
+        });
+      }
+    }
+    const requiredVersion = config.required_version;
+    if (requiredVersion !== void 0) {
+      if (typeof requiredVersion !== "string" || requiredVersion.trim().length === 0) {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: "required_version",
+          message: `Field 'required_version' must be a non-empty string (e.g. ">=3.1.0")`
+        });
+      }
+    }
+    const allowedFields = /* @__PURE__ */ new Set([
+      "$schema",
+      "guideline_patterns",
+      "eval_patterns",
+      "required_version",
+      "execution"
+    ]);
+    const unexpectedFields = Object.keys(config).filter((key) => !allowedFields.has(key));
+    if (unexpectedFields.length > 0) {
+      errors.push({
+        severity: "warning",
+        filePath,
+        message: `Unexpected fields: ${unexpectedFields.join(", ")}`
+      });
+    }
+    return {
+      valid: errors.filter((e) => e.severity === "error").length === 0,
+      filePath,
+      fileType: "config",
+      errors
+    };
+  } catch (error) {
+    errors.push({
+      severity: "error",
+      filePath,
+      message: `Failed to parse config file: ${error.message}`
+    });
+    return { valid: false, filePath, fileType: "config", errors };
+  }
+}
+function isObject3(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+async function validateFileReferences(evalFilePath) {
+  const errors = [];
+  const absolutePath = path42.resolve(evalFilePath);
+  const gitRoot = await findGitRoot(absolutePath);
+  if (!gitRoot) {
+    errors.push({
+      severity: "error",
+      filePath: absolutePath,
+      message: "Cannot validate file references: git repository root not found"
+    });
+    return errors;
+  }
+  const searchRoots = buildSearchRoots(absolutePath, gitRoot);
+  let parsed;
+  try {
+    const content = await readFile5(absolutePath, "utf8");
+    parsed = parse5(content);
+  } catch {
+    return errors;
+  }
+  if (!isObject3(parsed)) {
+    return errors;
+  }
+  let cases = parsed.tests;
+  if (cases === void 0 && "eval_cases" in parsed) {
+    cases = parsed.eval_cases;
+  }
+  if (cases === void 0 && "evalcases" in parsed) {
+    cases = parsed.evalcases;
+  }
+  if (!Array.isArray(cases)) {
+    return errors;
+  }
+  for (let i = 0; i < cases.length; i++) {
+    const evalCase = cases[i];
+    if (!isObject3(evalCase)) {
+      continue;
+    }
+    const inputField = evalCase.input;
+    if (Array.isArray(inputField)) {
+      await validateMessagesFileRefs(
+        inputField,
+        `tests[${i}].input`,
+        searchRoots,
+        absolutePath,
+        errors
+      );
+    }
+    const expectedOutputField = evalCase.expected_output;
+    if (Array.isArray(expectedOutputField)) {
+      await validateMessagesFileRefs(
+        expectedOutputField,
+        `tests[${i}].expected_output`,
+        searchRoots,
+        absolutePath,
+        errors
+      );
+    }
+  }
+  return errors;
+}
+async function validateMessagesFileRefs(messages, location, searchRoots, filePath, errors) {
+  for (let i = 0; i < messages.length; i++) {
+    const message = messages[i];
+    if (!isObject3(message)) {
+      continue;
+    }
+    const content = message.content;
+    if (typeof content === "string") {
+      continue;
+    }
+    if (!Array.isArray(content)) {
+      continue;
+    }
+    for (let j = 0; j < content.length; j++) {
+      const contentItem = content[j];
+      if (!isObject3(contentItem)) {
+        continue;
+      }
+      const type = contentItem.type;
+      if (type !== "file") {
+        continue;
+      }
+      const value = contentItem.value;
+      if (typeof value !== "string") {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: `${location}[${i}].content[${j}].value`,
+          message: "File reference must have a 'value' field with the file path"
+        });
+        continue;
+      }
+      const { resolvedPath } = await resolveFileReference(value, searchRoots);
+      if (!resolvedPath) {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: `${location}[${i}].content[${j}]`,
+          message: `Referenced file not found: ${value}`
+        });
+      } else {
+        try {
+          const fileContent = await readFile5(resolvedPath, "utf8");
+          if (fileContent.trim().length === 0) {
+            errors.push({
+              severity: "warning",
+              filePath,
+              location: `${location}[${i}].content[${j}]`,
+              message: `Referenced file is empty: ${value}`
+            });
+          }
+        } catch (error) {
+          errors.push({
+            severity: "error",
+            filePath,
+            location: `${location}[${i}].content[${j}]`,
+            message: `Cannot read referenced file: ${value} (${error.message})`
+          });
+        }
+      }
+    }
+  }
+}
+
+// src/commands/eval/targets.ts
+var ANSI_YELLOW2 = "\x1B[33m";
+var ANSI_RED2 = "\x1B[31m";
+var ANSI_RESET2 = "\x1B[0m";
+function isTTY() {
+  return process.stdout.isTTY ?? false;
+}
+async function readTestSuiteTarget(testFilePath) {
+  const metadata = await readTestSuiteMetadata(testFilePath);
+  return metadata.target;
+}
+function pickTargetName(options) {
+  const cliName = options.cliTargetName?.trim();
+  if (cliName && cliName !== "default") {
+    return { name: cliName, source: "cli" };
+  }
+  const fileName = options.fileTargetName?.trim();
+  if (fileName && fileName.length > 0) {
+    return { name: fileName, source: "test-file" };
+  }
+  return { name: "default", source: "default" };
+}
+async function selectTarget(options) {
+  const {
+    testFilePath,
+    repoRoot,
+    cwd,
+    explicitTargetsPath,
+    cliTargetName,
+    dryRun,
+    dryRunDelay,
+    dryRunDelayMin,
+    dryRunDelayMax,
+    env
+  } = options;
+  const targetsFilePath = await discoverTargetsFile({
+    explicitPath: explicitTargetsPath,
+    testFilePath,
+    repoRoot,
+    cwd
+  });
+  const validationResult = await validateTargetsFile(targetsFilePath);
+  const warnings = validationResult.errors.filter((e) => e.severity === "warning");
+  const useColors = isTTY();
+  if (warnings.length > 0) {
+    console.warn(`
+Warnings in ${targetsFilePath}:`);
+    for (const warning of warnings) {
+      const location = warning.location ? ` [${warning.location}]` : "";
+      const prefix = useColors ? `${ANSI_YELLOW2}  \u26A0${ANSI_RESET2}` : "  \u26A0";
+      const message = useColors ? `${ANSI_YELLOW2}${warning.message}${ANSI_RESET2}` : warning.message;
+      console.warn(`${prefix}${location} ${message}`);
+    }
+    console.warn("");
+  }
+  const errors = validationResult.errors.filter((e) => e.severity === "error");
+  if (errors.length > 0) {
+    console.error(`
+Errors in ${targetsFilePath}:`);
+    for (const error of errors) {
+      const location = error.location ? ` [${error.location}]` : "";
+      const prefix = useColors ? `${ANSI_RED2}  \u2717${ANSI_RESET2}` : "  \u2717";
+      const message = useColors ? `${ANSI_RED2}${error.message}${ANSI_RESET2}` : error.message;
+      console.error(`${prefix}${location} ${message}`);
+    }
+    throw new Error(`Targets file validation failed with ${errors.length} error(s)`);
+  }
+  const definitions = await readTargetDefinitions(targetsFilePath);
+  const fileTargetName = await readTestSuiteTarget(testFilePath);
+  const targetChoice = pickTargetName({ cliTargetName, fileTargetName });
+  const targetDefinition = definitions.find(
+    (definition) => definition.name === targetChoice.name
+  );
+  if (!targetDefinition) {
+    const available = listTargetNames(definitions).join(", ");
+    throw new Error(
+      `Target '${targetChoice.name}' not found in ${targetsFilePath}. Available targets: ${available}`
+    );
+  }
+  if (dryRun) {
+    const mockTarget = {
+      kind: "mock",
+      name: `${targetDefinition.name}-dry-run`,
+      graderTarget: void 0,
+      config: {
+        response: '{"answer":"Mock dry-run response"}',
+        delayMs: dryRunDelay,
+        delayMinMs: dryRunDelayMin,
+        delayMaxMs: dryRunDelayMax
+      }
+    };
+    return {
+      definitions,
+      resolvedTarget: mockTarget,
+      targetName: targetChoice.name,
+      targetSource: targetChoice.source,
+      targetsFilePath
+    };
+  }
+  try {
+    const resolvedTarget = resolveTargetDefinition(targetDefinition, env, testFilePath);
+    return {
+      definitions,
+      resolvedTarget,
+      targetName: targetChoice.name,
+      targetSource: targetChoice.source,
+      targetsFilePath
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new Error(`Failed to resolve target '${targetChoice.name}': ${message}`);
+  }
+}
+async function selectMultipleTargets(options) {
+  const {
+    testFilePath,
+    repoRoot,
+    cwd,
+    explicitTargetsPath,
+    dryRun,
+    dryRunDelay,
+    dryRunDelayMin,
+    dryRunDelayMax,
+    env,
+    targetNames
+  } = options;
+  const targetsFilePath = await discoverTargetsFile({
+    explicitPath: explicitTargetsPath,
+    testFilePath,
+    repoRoot,
+    cwd
+  });
+  const validationResult = await validateTargetsFile(targetsFilePath);
+  const warnings = validationResult.errors.filter((e) => e.severity === "warning");
+  const useColors = isTTY();
+  if (warnings.length > 0) {
+    console.warn(`
+Warnings in ${targetsFilePath}:`);
+    for (const warning of warnings) {
+      const location = warning.location ? ` [${warning.location}]` : "";
+      const prefix = useColors ? `${ANSI_YELLOW2}  \u26A0${ANSI_RESET2}` : "  \u26A0";
+      const message = useColors ? `${ANSI_YELLOW2}${warning.message}${ANSI_RESET2}` : warning.message;
+      console.warn(`${prefix}${location} ${message}`);
+    }
+    console.warn("");
+  }
+  const errors = validationResult.errors.filter((e) => e.severity === "error");
+  if (errors.length > 0) {
+    console.error(`
+Errors in ${targetsFilePath}:`);
+    for (const error of errors) {
+      const location = error.location ? ` [${error.location}]` : "";
+      const prefix = useColors ? `${ANSI_RED2}  \u2717${ANSI_RESET2}` : "  \u2717";
+      const message = useColors ? `${ANSI_RED2}${error.message}${ANSI_RESET2}` : error.message;
+      console.error(`${prefix}${location} ${message}`);
+    }
+    throw new Error(`Targets file validation failed with ${errors.length} error(s)`);
+  }
+  const definitions = await readTargetDefinitions(targetsFilePath);
+  const results = [];
+  for (const name of targetNames) {
+    const targetDefinition = definitions.find(
+      (definition) => definition.name === name
+    );
+    if (!targetDefinition) {
+      const available = listTargetNames(definitions).join(", ");
+      throw new Error(
+        `Target '${name}' not found in ${targetsFilePath}. Available targets: ${available}`
+      );
+    }
+    if (dryRun) {
+      const mockTarget = {
+        kind: "mock",
+        name: `${targetDefinition.name}-dry-run`,
+        graderTarget: void 0,
+        config: {
+          response: '{"answer":"Mock dry-run response"}',
+          delayMs: dryRunDelay,
+          delayMinMs: dryRunDelayMin,
+          delayMaxMs: dryRunDelayMax
+        }
+      };
+      results.push({
+        definitions,
+        resolvedTarget: mockTarget,
+        targetName: name,
+        targetSource: "cli",
+        targetsFilePath
+      });
+    } else {
+      try {
+        const resolvedTarget = resolveTargetDefinition(targetDefinition, env, testFilePath);
+        results.push({
+          definitions,
+          resolvedTarget,
+          targetName: name,
+          targetSource: "cli",
+          targetsFilePath
+        });
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to resolve target '${name}': ${message}`);
+      }
+    }
+  }
+  return results;
+}
+
+// src/commands/eval/run-eval.ts
+var DEFAULT_WORKERS = 3;
+function normalizeBoolean(value) {
+  return value === true;
+}
+function normalizeString(value) {
+  if (typeof value !== "string") {
+    return void 0;
+  }
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? trimmed : void 0;
+}
+function resolveTimestampPlaceholder(value) {
+  if (!value.includes("{timestamp}")) {
+    return value;
+  }
+  const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+  return value.replaceAll("{timestamp}", timestamp);
+}
+function normalizeNumber(value, fallback) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  if (typeof value === "string") {
+    const parsed = Number.parseInt(value, 10);
+    if (!Number.isNaN(parsed)) {
+      return parsed;
+    }
+  }
+  return fallback;
+}
+function normalizeOptionalNumber(value) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  if (typeof value === "string") {
+    const parsed = Number.parseInt(value, 10);
+    if (!Number.isNaN(parsed)) {
+      return parsed;
+    }
+  }
+  return void 0;
+}
+function normalizeWorkspaceMode(value) {
+  return value === "pooled" || value === "temp" || value === "static" ? value : void 0;
+}
+function normalizeOptions(rawOptions, config, yamlExecution) {
+  const cliFormat = normalizeString(rawOptions.outputFormat);
+  const configFormat = config?.output?.format;
+  const formatStr = cliFormat ?? configFormat ?? "jsonl";
+  const format = formatStr === "yaml" ? "yaml" : "jsonl";
+  const cliWorkers = normalizeOptionalNumber(rawOptions.workers);
+  const configWorkers = config?.execution?.workers;
+  const workers = cliWorkers ?? configWorkers ?? 0;
+  const rawOutputPaths = rawOptions.output;
+  const outputPaths = Array.isArray(rawOutputPaths) ? rawOutputPaths.filter((v) => typeof v === "string" && v.trim().length > 0) : [];
+  const rawTarget = rawOptions.target;
+  let cliTargets = [];
+  let singleTarget;
+  if (Array.isArray(rawTarget)) {
+    cliTargets = rawTarget.filter((v) => typeof v === "string" && v.trim().length > 0);
+    singleTarget = cliTargets.length === 1 ? cliTargets[0] : void 0;
+  } else if (typeof rawTarget === "string") {
+    const trimmed = rawTarget.trim();
+    if (trimmed.length > 0 && trimmed !== "default") {
+      cliTargets = [trimmed];
+      singleTarget = trimmed;
+    }
+  }
+  const cliAgentTimeout = normalizeOptionalNumber(rawOptions.agentTimeout);
+  const configAgentTimeoutSeconds = config?.execution?.agentTimeoutMs != null ? config.execution.agentTimeoutMs / 1e3 : void 0;
+  const cliMaxRetries = normalizeOptionalNumber(rawOptions.maxRetries);
+  const configMaxRetries = config?.execution?.maxRetries;
+  const cliCache = normalizeBoolean(rawOptions.cache);
+  const cliNoCache = normalizeBoolean(rawOptions.noCache);
+  const configCacheEnabled = config?.cache?.enabled;
+  const resolvedCache = cliCache || !cliNoCache && configCacheEnabled === true;
+  const resolvedNoCache = cliNoCache;
+  const cliOut = normalizeString(rawOptions.out);
+  const configOut = config?.output?.dir;
+  const cliWorkspacePath = normalizeString(rawOptions.workspacePath);
+  const cliWorkspaceModeRaw = normalizeString(rawOptions.workspaceMode);
+  const cliWorkspaceMode = normalizeWorkspaceMode(rawOptions.workspaceMode);
+  if (cliWorkspacePath && cliWorkspaceModeRaw && cliWorkspaceMode !== "static") {
+    throw new Error("--workspace-path requires --workspace-mode=static (or omit --workspace-mode)");
+  }
+  const yamlExecutionRecord = yamlExecution;
+  const yamlWorkspaceMode = normalizeWorkspaceMode(yamlExecutionRecord?.workspace_mode);
+  const yamlWorkspacePath = normalizeString(yamlExecutionRecord?.workspace_path);
+  const workspacePath = cliWorkspacePath ?? yamlWorkspacePath;
+  const workspaceMode = cliWorkspacePath ? "static" : cliWorkspaceMode ?? yamlWorkspaceMode;
+  return {
+    target: singleTarget,
+    cliTargets,
+    targetsPath: normalizeString(rawOptions.targets),
+    filter: normalizeString(rawOptions.filter),
+    workers: workers > 0 ? workers : void 0,
+    outPath: cliOut ?? configOut,
+    outputPaths,
+    format,
+    dryRun: normalizeBoolean(rawOptions.dryRun),
+    dryRunDelay: normalizeNumber(rawOptions.dryRunDelay, 0),
+    dryRunDelayMin: normalizeNumber(rawOptions.dryRunDelayMin, 0),
+    dryRunDelayMax: normalizeNumber(rawOptions.dryRunDelayMax, 0),
+    agentTimeoutSeconds: cliAgentTimeout ?? configAgentTimeoutSeconds,
+    maxRetries: cliMaxRetries ?? configMaxRetries ?? 2,
+    cache: resolvedCache,
+    noCache: resolvedNoCache,
+    // Boolean OR: config `true` cannot be overridden to `false` from CLI.
+    // Intentional — there are no --no-verbose / --no-keep-workspaces flags.
+    // Precedence: CLI > YAML config > TS config
+    verbose: normalizeBoolean(rawOptions.verbose) || yamlExecution?.verbose === true || config?.execution?.verbose === true,
+    // Precedence: CLI > YAML config > TS config
+    otelFile: normalizeString(rawOptions.otelFile) ?? (yamlExecution?.otel_file ? resolveTimestampPlaceholder(yamlExecution.otel_file) : void 0) ?? (config?.execution?.otelFile ? resolveTimestampPlaceholder(config.execution.otelFile) : void 0),
+    traceFile: normalizeString(rawOptions.traceFile) ?? (yamlExecution?.trace_file ? resolveTimestampPlaceholder(yamlExecution.trace_file) : void 0) ?? (config?.execution?.traceFile ? resolveTimestampPlaceholder(config.execution.traceFile) : void 0),
+    exportOtel: normalizeBoolean(rawOptions.exportOtel),
+    otelBackend: normalizeString(rawOptions.otelBackend),
+    otelCaptureContent: normalizeBoolean(rawOptions.otelCaptureContent),
+    otelGroupTurns: normalizeBoolean(rawOptions.otelGroupTurns),
+    retryErrors: normalizeString(rawOptions.retryErrors),
+    workspaceMode,
+    workspacePath,
+    benchmarkJson: normalizeString(rawOptions.benchmarkJson),
+    artifacts: normalizeString(rawOptions.artifacts),
+    graderTarget: normalizeString(rawOptions.graderTarget),
+    model: normalizeString(rawOptions.model)
+  };
+}
+async function ensureFileExists(filePath, description) {
+  try {
+    await access4(filePath, constants4.F_OK);
+  } catch {
+    throw new Error(`${description} not found: ${filePath}`);
+  }
+}
+function buildDefaultOutputPath(cwd, format) {
+  const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+  const baseName = "eval";
+  const extension = getDefaultExtension(format);
+  return path12.join(cwd, ".agentv", "results", `${baseName}_${timestamp}${extension}`);
+}
+function createProgressReporter(maxWorkers, options) {
+  const display = new ProgressDisplay(maxWorkers, options);
+  return {
+    isInteractive: display.isInteractiveMode(),
+    start: () => display.start(),
+    setTotal: (total) => display.setTotalTests(total),
+    update: (workerId, progress) => display.updateWorker({ ...progress, workerId }),
+    finish: () => display.finish(),
+    addLogPaths: (paths, provider) => display.addLogPaths(paths, provider)
+  };
+}
+function makeEvalKey(testFilePath, evalId) {
+  return `${path12.resolve(testFilePath)}::${evalId}`;
+}
+function createDisplayIdTracker() {
+  const map = /* @__PURE__ */ new Map();
+  let nextId = 1;
+  return {
+    getOrAssign(evalKey) {
+      const existing = map.get(evalKey);
+      if (existing !== void 0) {
+        return existing;
+      }
+      const assigned = nextId++;
+      map.set(evalKey, assigned);
+      return assigned;
+    }
+  };
+}
+function applyVerboseOverride(selection, cliVerbose) {
+  const { resolvedTarget } = selection;
+  if (resolvedTarget.kind !== "cli") {
+    return selection;
+  }
+  return {
+    ...selection,
+    resolvedTarget: {
+      ...resolvedTarget,
+      config: {
+        ...resolvedTarget.config,
+        verbose: cliVerbose
+      }
+    }
+  };
+}
+async function prepareFileMetadata(params) {
+  const { testFilePath, repoRoot, cwd, options } = params;
+  await ensureFileExists(testFilePath, "Test file");
+  await loadEnvFromHierarchy({
+    testFilePath,
+    repoRoot,
+    verbose: options.verbose
+  });
+  const suite = await loadTestSuite(testFilePath, repoRoot, {
+    verbose: options.verbose,
+    filter: options.filter
+  });
+  const filteredIds = suite.tests.map((value) => value.id);
+  const cliTargets = options.cliTargets;
+  const suiteTargets = suite.targets;
+  let targetNames;
+  if (cliTargets.length > 0) {
+    targetNames = cliTargets;
+  } else if (suiteTargets && suiteTargets.length > 0) {
+    targetNames = suiteTargets;
+  } else {
+    targetNames = [];
+  }
+  let selections;
+  if (targetNames.length > 1) {
+    const multiSelections = await selectMultipleTargets({
+      testFilePath,
+      repoRoot,
+      cwd,
+      explicitTargetsPath: options.targetsPath,
+      dryRun: options.dryRun,
+      dryRunDelay: options.dryRunDelay,
+      dryRunDelayMin: options.dryRunDelayMin,
+      dryRunDelayMax: options.dryRunDelayMax,
+      env: process.env,
+      targetNames
+    });
+    selections = multiSelections.map((sel) => {
+      const providerLabel = options.dryRun ? `${sel.resolvedTarget.kind} (dry-run)` : sel.resolvedTarget.kind;
+      return {
+        selection: sel,
+        inlineTargetLabel: `${sel.targetName} [provider=${providerLabel}]`
+      };
+    });
+  } else {
+    const selection = await selectTarget({
+      testFilePath,
+      repoRoot,
+      cwd,
+      explicitTargetsPath: options.targetsPath,
+      cliTargetName: targetNames.length === 1 ? targetNames[0] : options.target,
+      dryRun: options.dryRun,
+      dryRunDelay: options.dryRunDelay,
+      dryRunDelayMin: options.dryRunDelayMin,
+      dryRunDelayMax: options.dryRunDelayMax,
+      env: process.env
+    });
+    const providerLabel = options.dryRun ? `${selection.resolvedTarget.kind} (dry-run)` : selection.resolvedTarget.kind;
+    selections = [
+      {
+        selection,
+        inlineTargetLabel: `${selection.targetName} [provider=${providerLabel}]`
+      }
+    ];
+  }
+  return {
+    evalIds: filteredIds,
+    evalCases: suite.tests,
+    selections,
+    trialsConfig: suite.trials,
+    suiteTargets,
+    yamlCache: suite.cacheConfig?.enabled,
+    yamlCachePath: suite.cacheConfig?.cachePath,
+    totalBudgetUsd: suite.totalBudgetUsd,
+    failOnError: suite.failOnError
+  };
+}
+async function runWithLimit(items, limit, task) {
+  const safeLimit = Math.max(1, limit);
+  let index = 0;
+  const workers = Array.from({ length: safeLimit }, async () => {
+    while (index < items.length) {
+      const current = items[index];
+      index += 1;
+      await task(current);
+    }
+  });
+  await Promise.all(workers);
+}
+async function runSingleEvalFile(params) {
+  const {
+    testFilePath,
+    cwd,
+    repoRoot,
+    options,
+    outputWriter,
+    otelExporter,
+    cache,
+    evaluationRunner,
+    workersOverride,
+    progressReporter,
+    seenEvalCases,
+    displayIdTracker,
+    selection,
+    inlineTargetLabel,
+    evalCases,
+    trialsConfig,
+    matrixMode,
+    totalBudgetUsd,
+    failOnError
+  } = params;
+  const targetName = selection.targetName;
+  await ensureFileExists(testFilePath, "Test file");
+  const resolvedTargetSelection = applyVerboseOverride(selection, options.verbose);
+  const providerLabel = options.dryRun ? `${resolvedTargetSelection.resolvedTarget.kind} (dry-run)` : resolvedTargetSelection.resolvedTarget.kind;
+  const targetMessage = options.verbose ? `Using target (${resolvedTargetSelection.targetSource}): ${resolvedTargetSelection.targetName} [provider=${providerLabel}] via ${resolvedTargetSelection.targetsFilePath}` : `Using target: ${inlineTargetLabel}`;
+  if (!progressReporter.isInteractive || options.verbose) {
+    console.log(targetMessage);
+  }
+  const agentTimeoutMs = options.agentTimeoutSeconds != null ? Math.max(0, options.agentTimeoutSeconds) * 1e3 : void 0;
+  const workerPreference = workersOverride ?? options.workers;
+  let resolvedWorkers = workerPreference ?? resolvedTargetSelection.resolvedTarget.workers ?? DEFAULT_WORKERS;
+  if (resolvedWorkers < 1 || resolvedWorkers > 50) {
+    throw new Error(`Workers must be between 1 and 50, got: ${resolvedWorkers}`);
+  }
+  const isVSCodeProvider = ["vscode", "vscode-insiders"].includes(
+    resolvedTargetSelection.resolvedTarget.kind
+  );
+  if (isVSCodeProvider && resolvedWorkers > 1) {
+    console.warn(
+      `Warning: VSCode providers require window focus. Limiting workers from ${resolvedWorkers} to 1 to prevent race conditions.`
+    );
+    resolvedWorkers = 1;
+  }
+  if (isVSCodeProvider && !options.dryRun) {
+    const vsConfig = resolvedTargetSelection.resolvedTarget.config;
+    await ensureVSCodeSubagents({
+      kind: resolvedTargetSelection.resolvedTarget.kind,
+      count: resolvedWorkers,
+      verbose: options.verbose,
+      vscodeCmd: vsConfig.executable
+    });
+  }
+  const streamingObserver = otelExporter?.createStreamingObserver() ?? null;
+  const results = await evaluationRunner({
+    testFilePath,
+    repoRoot,
+    target: resolvedTargetSelection.resolvedTarget,
+    targets: resolvedTargetSelection.definitions,
+    env: process.env,
+    maxRetries: Math.max(0, options.maxRetries),
+    agentTimeoutMs,
+    cache,
+    useCache: (() => {
+      if (!cache) return false;
+      const targetConfig = resolvedTargetSelection.resolvedTarget.config;
+      if (shouldSkipCacheForTemperature(targetConfig)) {
+        if (options.verbose) {
+          console.log("Cache skipped: target temperature > 0");
+        }
+        return false;
+      }
+      return true;
+    })(),
+    evalCases,
+    verbose: options.verbose,
+    maxConcurrency: resolvedWorkers,
+    workspaceMode: options.workspaceMode,
+    workspacePath: options.workspacePath,
+    trials: trialsConfig,
+    totalBudgetUsd,
+    failOnError,
+    graderTarget: options.graderTarget,
+    model: options.model,
+    streamCallbacks: streamingObserver?.getStreamCallbacks(),
+    onResult: async (result) => {
+      streamingObserver?.finalizeEvalCase(result.score, result.error);
+      const { output: _, ...resultWithoutTrace } = result;
+      await outputWriter.append(resultWithoutTrace);
+      if (otelExporter && !streamingObserver) {
+        try {
+          await otelExporter.exportResult(result);
+        } catch (err) {
+          if (options.verbose) {
+            console.warn(
+              `OTel export warning: ${err instanceof Error ? err.message : String(err)}`
+            );
+          }
+        }
+      }
+    },
+    onProgress: async (event) => {
+      const evalKeyId = matrixMode ? `${event.testId}@${targetName}` : event.testId;
+      const evalKey = makeEvalKey(testFilePath, evalKeyId);
+      if (event.status === "pending" && !seenEvalCases.has(evalKey)) {
+        seenEvalCases.add(evalKey);
+        progressReporter.setTotal(seenEvalCases.size);
+      }
+      const displayId = displayIdTracker.getOrAssign(evalKey);
+      if (event.status === "running" && streamingObserver) {
+        streamingObserver.startEvalCase(event.testId, targetName, testFilePath);
+      }
+      progressReporter.update(displayId, {
+        workerId: displayId,
+        testId: matrixMode ? `${event.testId}@${targetName}` : event.testId,
+        status: event.status,
+        startedAt: event.startedAt,
+        completedAt: event.completedAt,
+        error: event.error,
+        targetLabel: inlineTargetLabel
+      });
+    }
+  });
+  return { results: [...results] };
+}
+async function runEvalCommand(input) {
+  const cwd = process.cwd();
+  let config = null;
+  try {
+    config = await loadTsConfig(cwd);
+  } catch (err) {
+    console.warn(
+      `Warning: Failed to load agentv config: ${err instanceof Error ? err.message : String(err)}`
+    );
+  }
+  const repoRoot = await findRepoRoot(cwd);
+  const yamlConfig = await loadConfig(path12.join(cwd, "_"), repoRoot);
+  if (yamlConfig?.required_version) {
+    await enforceRequiredVersion(yamlConfig.required_version, {
+      strict: normalizeBoolean(input.rawOptions.strict)
+    });
+  }
+  let options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution);
+  if (options.graderTarget === "agentv" && !options.model) {
+    throw new Error("--grader-target agentv requires --model (e.g., --model openai:gpt-5-mini)");
+  }
+  let retryNonErrorResults;
+  if (options.retryErrors) {
+    const retryPath = path12.resolve(options.retryErrors);
+    await ensureFileExists(retryPath, "Retry-errors JSONL file");
+    const errorIds = await loadErrorTestIds(retryPath);
+    if (errorIds.length === 0) {
+      console.log("No execution errors found in the previous output. Nothing to retry.");
+      return;
+    }
+    console.log(`Retrying ${errorIds.length} execution-error test(s): ${errorIds.join(", ")}`);
+    const filterPattern = errorIds.length === 1 ? errorIds[0] : `{${errorIds.join(",")}}`;
+    options = { ...options, filter: filterPattern };
+    retryNonErrorResults = await loadNonErrorResults(retryPath);
+  }
+  if (options.workspacePath) {
+    const resolvedWorkspace = path12.resolve(options.workspacePath);
+    try {
+      const { stat: stat2 } = await import("node:fs/promises");
+      const stats = await stat2(resolvedWorkspace);
+      if (!stats.isDirectory()) {
+        throw new Error(`--workspace-path is not a directory: ${resolvedWorkspace}`);
+      }
+    } catch (err) {
+      if (err.code === "ENOENT") {
+        throw new Error(`--workspace-path does not exist: ${resolvedWorkspace}`);
+      }
+      throw err;
+    }
+    options = { ...options, workspacePath: resolvedWorkspace };
+  }
+  if (options.verbose) {
+    console.log(`Repository root: ${repoRoot}`);
+  }
+  let otelExporter = null;
+  const useFileExport = !!(options.otelFile || options.traceFile);
+  if (options.exportOtel || useFileExport) {
+    try {
+      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-3BMOAU4X.js");
+      let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
+      let headers = {};
+      if (options.otelBackend) {
+        const preset = OTEL_BACKEND_PRESETS[options.otelBackend];
+        if (preset) {
+          endpoint = preset.endpoint;
+          headers = preset.headers(process.env);
+        } else {
+          console.warn(`Unknown OTel backend preset: ${options.otelBackend}`);
+        }
+      }
+      if (process.env.OTEL_EXPORTER_OTLP_HEADERS) {
+        for (const pair of process.env.OTEL_EXPORTER_OTLP_HEADERS.split(",")) {
+          const [key, ...rest] = pair.split("=");
+          if (key) headers[key.trim()] = rest.join("=").trim();
+        }
+      }
+      const captureContent = options.otelCaptureContent || process.env.AGENTV_OTEL_CAPTURE_CONTENT === "true";
+      otelExporter = new OtelTraceExporter({
+        endpoint,
+        headers,
+        captureContent,
+        groupTurns: options.otelGroupTurns,
+        otlpFilePath: options.otelFile ? path12.resolve(options.otelFile) : void 0,
+        traceFilePath: options.traceFile ? path12.resolve(options.traceFile) : void 0
+      });
+      const initialized = await otelExporter.init();
+      if (!initialized) {
+        console.warn(
+          "OTel export requested but @opentelemetry packages not available. Install them to enable export."
+        );
+        otelExporter = null;
+      }
+    } catch (err) {
+      console.warn(
+        `OTel export initialization failed: ${err instanceof Error ? err.message : String(err)}`
+      );
+      otelExporter = null;
+    }
+  }
+  const outputPath = options.outPath ? path12.resolve(options.outPath) : buildDefaultOutputPath(cwd, options.format);
+  const extraOutputPaths = options.outputPaths.map((p) => path12.resolve(p));
+  const allOutputPaths = extraOutputPaths.length > 0 ? [outputPath, ...extraOutputPaths] : [outputPath];
+  const uniqueOutputPaths = [...new Set(allOutputPaths)];
+  let outputWriter;
+  if (uniqueOutputPaths.length === 1) {
+    outputWriter = await createOutputWriter(outputPath, options.format);
+    console.log(`Output path: ${outputPath}`);
+  } else {
+    outputWriter = await createMultiWriter(uniqueOutputPaths);
+    console.log("Output paths:");
+    for (const p of uniqueOutputPaths) {
+      console.log(`  ${p}`);
+    }
+  }
+  const resolvedTestFiles = input.testFiles.map((file) => path12.resolve(file));
+  if (options.otelFile) {
+    console.log(`OTLP JSON file: ${path12.resolve(options.otelFile)}`);
+  }
+  if (options.traceFile) {
+    console.log(`Trace file: ${path12.resolve(options.traceFile)}`);
+  }
+  const evaluationRunner = await resolveEvaluationRunner();
+  const allResults = [];
+  const seenEvalCases = /* @__PURE__ */ new Set();
+  const displayIdTracker = createDisplayIdTracker();
+  const totalWorkers = options.workers ?? DEFAULT_WORKERS;
+  const fileConcurrency = Math.min(
+    Math.max(1, totalWorkers),
+    Math.max(1, resolvedTestFiles.length)
+  );
+  const perFileWorkers = options.workers ? Math.max(1, Math.floor(totalWorkers / fileConcurrency)) : void 0;
+  const fileMetadata = /* @__PURE__ */ new Map();
+  const tsFiles = [];
+  const yamlFiles = [];
+  for (const testFilePath of resolvedTestFiles) {
+    if (/\.(ts|js|mts|mjs)$/.test(testFilePath)) {
+      tsFiles.push(testFilePath);
+    } else {
+      yamlFiles.push(testFilePath);
+    }
+  }
+  for (const tsFile of tsFiles) {
+    await ensureFileExists(tsFile, "TypeScript eval file");
+    await import(pathToFileURL(tsFile).href);
+  }
+  if (yamlFiles.length === 0 && tsFiles.length > 0) {
+    return;
+  }
+  for (const testFilePath of yamlFiles) {
+    const meta = await prepareFileMetadata({
+      testFilePath,
+      repoRoot,
+      cwd,
+      options
+    });
+    fileMetadata.set(testFilePath, meta);
+  }
+  const firstMeta = fileMetadata.values().next().value;
+  const yamlCacheEnabled = firstMeta?.yamlCache;
+  const yamlCachePath = firstMeta?.yamlCachePath;
+  const cacheEnabled = shouldEnableCache({
+    cliCache: options.cache,
+    cliNoCache: options.noCache,
+    yamlCache: yamlCacheEnabled
+  });
+  const cache = cacheEnabled ? new ResponseCache(yamlCachePath ? path12.resolve(yamlCachePath) : void 0) : void 0;
+  const useCache = cacheEnabled;
+  if (cacheEnabled) {
+    console.log(`Response cache: enabled${yamlCachePath ? ` (${yamlCachePath})` : ""}`);
+  }
+  const isMatrixMode = Array.from(fileMetadata.values()).some((meta) => meta.selections.length > 1);
+  let totalEvalCount = 0;
+  for (const meta of fileMetadata.values()) {
+    const suiteTargetNames = meta.selections.map((s) => s.selection.targetName);
+    for (const test of meta.evalCases) {
+      const testTargetNames = test.targets && test.targets.length > 0 ? test.targets.filter((t) => suiteTargetNames.includes(t)) : suiteTargetNames;
+      totalEvalCount += testTargetNames.length > 0 ? testTargetNames.length : 1;
+    }
+  }
+  if (totalEvalCount === 0) {
+    throw new Error("No tests matched the provided filters.");
+  }
+  const progressReporter = createProgressReporter(totalWorkers, { verbose: options.verbose });
+  progressReporter.start();
+  progressReporter.setTotal(totalEvalCount);
+  const seenCodexLogPaths = /* @__PURE__ */ new Set();
+  const unsubscribeCodexLogs = subscribeToCodexLogEntries((entry) => {
+    if (!entry.filePath || seenCodexLogPaths.has(entry.filePath)) {
+      return;
+    }
+    seenCodexLogPaths.add(entry.filePath);
+    progressReporter.addLogPaths([entry.filePath], "codex");
+  });
+  const seenPiLogPaths = /* @__PURE__ */ new Set();
+  const unsubscribePiLogs = subscribeToPiLogEntries((entry) => {
+    if (!entry.filePath || seenPiLogPaths.has(entry.filePath)) {
+      return;
+    }
+    seenPiLogPaths.add(entry.filePath);
+    progressReporter.addLogPaths([entry.filePath], "pi");
+  });
+  const seenCopilotLogPaths = /* @__PURE__ */ new Set();
+  const unsubscribeCopilotSdkLogs = subscribeToCopilotSdkLogEntries((entry) => {
+    if (!entry.filePath || seenCopilotLogPaths.has(entry.filePath)) {
+      return;
+    }
+    seenCopilotLogPaths.add(entry.filePath);
+    progressReporter.addLogPaths([entry.filePath], "copilot");
+  });
+  const unsubscribeCopilotCliLogs = subscribeToCopilotCliLogEntries((entry) => {
+    if (!entry.filePath || seenCopilotLogPaths.has(entry.filePath)) {
+      return;
+    }
+    seenCopilotLogPaths.add(entry.filePath);
+    progressReporter.addLogPaths([entry.filePath], "copilot");
+  });
+  for (const [testFilePath, meta] of fileMetadata.entries()) {
+    for (const { selection, inlineTargetLabel } of meta.selections) {
+      for (const testId of meta.evalIds) {
+        const evalKey = makeEvalKey(
+          testFilePath,
+          meta.selections.length > 1 ? `${testId}@${selection.targetName}` : testId
+        );
+        seenEvalCases.add(evalKey);
+        const displayId = displayIdTracker.getOrAssign(evalKey);
+        progressReporter.update(displayId, {
+          workerId: displayId,
+          testId: meta.selections.length > 1 ? `${testId}@${selection.targetName}` : testId,
+          status: "pending",
+          targetLabel: inlineTargetLabel
+        });
+      }
+    }
+  }
+  try {
+    await runWithLimit(resolvedTestFiles, fileConcurrency, async (testFilePath) => {
+      const targetPrep = fileMetadata.get(testFilePath);
+      if (!targetPrep) {
+        throw new Error(`Missing metadata for ${testFilePath}`);
+      }
+      const targetResults = await Promise.all(
+        targetPrep.selections.map(async ({ selection, inlineTargetLabel }) => {
+          const targetName = selection.targetName;
+          const applicableEvalCases = targetPrep.selections.length > 1 ? targetPrep.evalCases.filter((test) => {
+            if (test.targets && test.targets.length > 0) {
+              return test.targets.includes(targetName);
+            }
+            return true;
+          }) : targetPrep.evalCases;
+          if (applicableEvalCases.length === 0) {
+            return [];
+          }
+          const result = await runSingleEvalFile({
+            testFilePath,
+            cwd,
+            repoRoot,
+            options,
+            outputWriter,
+            otelExporter,
+            cache,
+            evaluationRunner,
+            workersOverride: perFileWorkers,
+            progressReporter,
+            seenEvalCases,
+            displayIdTracker,
+            selection,
+            inlineTargetLabel,
+            evalCases: applicableEvalCases,
+            trialsConfig: targetPrep.trialsConfig,
+            matrixMode: targetPrep.selections.length > 1,
+            totalBudgetUsd: targetPrep.totalBudgetUsd,
+            failOnError: targetPrep.failOnError
+          });
+          return result.results;
+        })
+      );
+      for (const results of targetResults) {
+        allResults.push(...results);
+      }
+    });
+    progressReporter.finish();
+    if (retryNonErrorResults && retryNonErrorResults.length > 0) {
+      for (const preserved of retryNonErrorResults) {
+        await outputWriter.append(preserved);
+      }
+      allResults.push(...retryNonErrorResults);
+      console.log(
+        `Merged ${retryNonErrorResults.length} non-error result(s) from previous output.`
+      );
+    }
+    const summary = calculateEvaluationSummary(allResults);
+    console.log(formatEvaluationSummary(summary));
+    if (isMatrixMode && allResults.length > 0) {
+      console.log(formatMatrixSummary(allResults));
+    }
+    if (options.benchmarkJson && allResults.length > 0) {
+      const benchmarkPath = path12.resolve(options.benchmarkJson);
+      await writeBenchmarkJson(benchmarkPath, allResults);
+      console.log(`Benchmark written to: ${benchmarkPath}`);
+    }
+    if (options.artifacts && allResults.length > 0) {
+      const artifactsDir = path12.resolve(options.artifacts);
+      const evalFile = resolvedTestFiles.length === 1 ? resolvedTestFiles[0] : "";
+      const {
+        gradingDir,
+        timingPath,
+        benchmarkPath: abp
+      } = await writeArtifactsFromResults(allResults, artifactsDir, { evalFile });
+      console.log(`Artifacts written to: ${artifactsDir}`);
+      console.log(`  Grading: ${gradingDir} (${allResults.length} files)`);
+      console.log(`  Timing:  ${timingPath}`);
+      console.log(`  Benchmark: ${abp}`);
+    }
+    const failedWithWorkspaces = allResults.filter(
+      (r) => r.workspacePath && (r.error || r.score < 0.5)
+    );
+    if (failedWithWorkspaces.length > 0) {
+      console.log("\nWorkspaces preserved for debugging:");
+      for (const result of failedWithWorkspaces) {
+        console.log(`  ${result.testId}: ${result.workspacePath}`);
+      }
+    }
+    if (allResults.length > 0) {
+      if (uniqueOutputPaths.length === 1) {
+        console.log(`
+Results written to: ${outputPath}`);
+      } else {
+        console.log("\nResults written to:");
+        for (const p of uniqueOutputPaths) {
+          console.log(`  ${p}`);
+        }
+      }
+    }
+  } finally {
+    unsubscribeCodexLogs();
+    unsubscribePiLogs();
+    unsubscribeCopilotSdkLogs();
+    unsubscribeCopilotCliLogs();
+    await outputWriter.close().catch(() => void 0);
+    if (otelExporter) {
+      try {
+        await otelExporter.shutdown();
+      } catch {
+      }
+    }
+  }
+}
+async function resolveEvaluationRunner() {
+  const overridePath = process.env.AGENTEVO_CLI_EVAL_RUNNER;
+  if (!overridePath) {
+    return runEvaluation;
+  }
+  const resolved = path12.isAbsolute(overridePath) ? overridePath : path12.resolve(process.cwd(), overridePath);
+  const moduleUrl = pathToFileURL(resolved).href;
+  const mod = await import(moduleUrl);
+  const candidate = mod.runEvaluation;
+  if (typeof candidate !== "function") {
+    throw new Error(
+      `Module '${resolved}' must export a 'runEvaluation' function to override the default implementation`
+    );
+  }
+  return candidate;
+}
+
+export {
+  package_default,
+  toSnakeCaseDeep,
+  HtmlWriter,
+  resolveEvalPaths,
+  findRepoRoot,
+  detectFileType,
+  validateEvalFile,
+  validateTargetsFile,
+  validateConfigFile,
+  validateFileReferences,
+  TARGET_FILE_CANDIDATES,
+  fileExists,
+  selectTarget,
+  runEvalCommand
+};
+//# sourceMappingURL=chunk-PCQA43SA.js.map