agentv 4.17.1 → 4.19.0-next.1

package/dist/{chunk-ZUNYOUFO.js → chunk-YGNBTBXW.js}

@@ -1,4 +1,36 @@
 import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
+import {
+  RESULT_INDEX_FILENAME,
+  RESULT_RUNS_DIRNAME,
+  aggregateRunDir,
+  buildDefaultRunDir,
+  buildTestTargetKey,
+  deduplicateByTestIdTarget,
+  isDirectoryPath,
+  normalizeExperimentName,
+  parseJsonlResults,
+  resolveExistingRunPrimaryPath,
+  resolveRunIndexPath,
+  resolveRunManifestPath,
+  resolveWorkspaceOrFilePath,
+  toSnakeCaseDeep,
+  writeArtifactsFromResults
+} from "./chunk-NL6P5MUH.js";
+import {
+  ResponseCache,
+  commitAndPushResultsBranch,
+  createDraftResultsPr,
+  deriveCategory,
+  directorySizeBytes,
+  getResultsRepoStatus,
+  loadTsConfig,
+  prepareResultsRepoBranch,
+  resolveResultsRepoRunsDir,
+  shouldEnableCache,
+  shouldSkipCacheForTemperature,
+  stageResultsArtifacts,
+  syncResultsRepo
+} from "./chunk-R2QDYORI.js";
 import {
   CLI_PLACEHOLDERS,
   COMMON_TARGET_SETTINGS,
@@ -6,48 +38,34 @@ import {
   DEFAULT_THRESHOLD,
   KNOWN_PROVIDERS,
   PROVIDER_ALIASES,
-  ResponseCache,
   buildDirectoryChain,
   buildSearchRoots,
-  commitAndPushResultsBranch,
-  createDraftResultsPr,
-  deriveCategory,
-  directorySizeBytes,
   ensureVSCodeSubagents,
   findDeprecatedCamelCaseTargetWarnings,
   findGitRoot,
-  getResultsRepoStatus,
   interpolateEnv,
-  isEvaluatorKind,
+  isGraderKind,
   listTargetNames,
   loadCasesFromFile,
   loadConfig,
   loadTestSuite,
-  loadTsConfig,
   normalizeLineEndings,
-  prepareResultsRepoBranch,
   readTargetDefinitions,
   readTestSuiteMetadata,
   resolveFileReference,
-  resolveResultsRepoRunsDir,
   resolveTargetDefinition,
   runEvaluation,
-  shouldEnableCache,
-  shouldSkipCacheForTemperature,
-  stageResultsArtifacts,
   subscribeToCodexLogEntries,
   subscribeToCopilotCliLogEntries,
   subscribeToCopilotSdkLogEntries,
   subscribeToPiLogEntries,
-  syncResultsRepo,
-  toCamelCaseDeep,
-  toTranscriptJsonLines
-} from "./chunk-IRU2UOWN.js";
+  toCamelCaseDeep
+} from "./chunk-PTYQS37Y.js";
 
 // package.json
 var package_default = {
   name: "agentv",
-  version: "4.17.1",
+  version: "4.19.0-next.1",
   description: "CLI entry point for AgentV",
   type: "module",
   repository: {
@@ -132,12 +150,15 @@ async function resolveEvalPaths(evalPaths, cwd) {
     const candidatePath = path.isAbsolute(pattern) ? path.normalize(pattern) : path.resolve(cwd, pattern);
     try {
       const stats = await stat(candidatePath);
-      if (stats.isFile() && /\.(ya?ml|jsonl|json)$/i.test(candidatePath)) {
+      if (stats.isFile() && /\.(ya?ml|jsonl|json|[cm]?ts)$/i.test(candidatePath)) {
         results.add(candidatePath);
         continue;
       }
       if (stats.isDirectory()) {
-        const dirGlob = path.posix.join(candidatePath.replace(/\\/g, "/"), "**/*.eval.{yaml,yml}");
+        const dirGlob = path.posix.join(
+          candidatePath.replace(/\\/g, "/"),
+          "**/{*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,*.eval.ts,*.eval.mts}"
+        );
         const dirMatches = await fg(dirGlob, {
           absolute: true,
           onlyFiles: true,
@@ -163,7 +184,9 @@ async function resolveEvalPaths(evalPaths, cwd) {
       followSymbolicLinks: true,
       ignore: ignorePatterns
     });
-    const yamlMatches = matches.filter((filePath) => /\.(ya?ml|jsonl|json)$/i.test(filePath));
+    const yamlMatches = matches.filter(
+      (filePath) => /\.(ya?ml|jsonl|json|[cm]?ts)$/i.test(filePath)
+    );
     for (const filePath of yamlMatches) {
       results.add(path.normalize(filePath));
     }
@@ -185,7 +208,7 @@ async function resolveEvalPaths(evalPaths, cwd) {
     throw new Error(
       `No eval files matched any provided paths or globs: ${includePatterns.join(
         ", "
-      )}. Provide YAML, JSONL, or JSON paths or globs (e.g., "evals/**/*.yaml", "evals/**/*.jsonl", "evals.json").`
+      )}. Provide YAML, JSONL, JSON, or TypeScript paths or globs (e.g., "evals/**/eval.yaml", "evals/**/*.eval.ts").`
     );
   }
   const sorted = Array.from(results);
@@ -261,9 +284,9 @@ async function discoverTargetsFile(options) {
 }
 
 // src/commands/eval/run-eval.ts
-import { constants as constants4, mkdirSync } from "node:fs";
-import { access as access5 } from "node:fs/promises";
-import path17 from "node:path";
+import { constants as constants4, existsSync as existsSync2, mkdirSync } from "node:fs";
+import { access as access5, readFile as readFile8 } from "node:fs/promises";
+import path15 from "node:path";
 import { pathToFileURL } from "node:url";
 
 // src/version-check.ts
@@ -321,88 +344,15 @@ async function promptContinue() {
 }
 
 // src/commands/results/remote.ts
-import path6 from "node:path";
-
-// src/commands/inspect/utils.ts
-import { readFileSync as readFileSync2, readdirSync, statSync as statSync2 } from "node:fs";
 import path5 from "node:path";
 
-// src/commands/eval/result-layout.ts
-import { existsSync, statSync } from "node:fs";
-import path3 from "node:path";
-var RESULT_INDEX_FILENAME = "index.jsonl";
-var RESULT_RUNS_DIRNAME = "runs";
-var DEFAULT_EXPERIMENT_NAME = "default";
-function normalizeExperimentName(experiment) {
-  const trimmed = experiment?.trim();
-  if (!trimmed) {
-    return DEFAULT_EXPERIMENT_NAME;
-  }
-  if (!/^[A-Za-z0-9._-]+$/.test(trimmed)) {
-    throw new Error(
-      `Invalid experiment name "${trimmed}". Use only letters, numbers, ".", "_" and "-".`
-    );
-  }
-  return trimmed;
-}
-function createRunDirName(timestamp = /* @__PURE__ */ new Date()) {
-  return timestamp.toISOString().replace(/[:.]/g, "-");
-}
-function buildDefaultRunDir(cwd, experiment, timestamp = /* @__PURE__ */ new Date()) {
-  return path3.join(
-    cwd,
-    ".agentv",
-    "results",
-    RESULT_RUNS_DIRNAME,
-    normalizeExperimentName(experiment),
-    createRunDirName(timestamp)
-  );
-}
-function resolveRunIndexPath(runDir) {
-  return path3.join(runDir, RESULT_INDEX_FILENAME);
-}
-function isRunManifestPath(filePath) {
-  return path3.basename(filePath) === RESULT_INDEX_FILENAME;
-}
-function resolveExistingRunPrimaryPath(runDir) {
-  const indexPath = resolveRunIndexPath(runDir);
-  if (existsSync(indexPath)) {
-    return indexPath;
-  }
-  return void 0;
-}
-function isDirectoryPath(filePath) {
-  try {
-    return statSync(filePath).isDirectory();
-  } catch {
-    return false;
-  }
-}
-function resolveWorkspaceOrFilePath(filePath) {
-  if (!isDirectoryPath(filePath)) {
-    return filePath;
-  }
-  const existing = resolveExistingRunPrimaryPath(filePath);
-  if (!existing) {
-    throw new Error(`Result workspace is missing ${RESULT_INDEX_FILENAME}: ${filePath}`);
-  }
-  return existing;
-}
-function resolveRunManifestPath(filePath) {
-  if (isDirectoryPath(filePath)) {
-    return resolveWorkspaceOrFilePath(filePath);
-  }
-  if (!isRunManifestPath(filePath)) {
-    throw new Error(
-      `Expected a run workspace directory or ${RESULT_INDEX_FILENAME} manifest: ${filePath}`
-    );
-  }
-  return filePath;
-}
+// src/commands/inspect/utils.ts
+import { readFileSync as readFileSync2, readdirSync, statSync } from "node:fs";
+import path4 from "node:path";
 
 // src/commands/results/manifest.ts
-import { existsSync as existsSync2, readFileSync } from "node:fs";
-import path4 from "node:path";
+import { existsSync, readFileSync } from "node:fs";
+import path3 from "node:path";
 function parseJsonlLines(content) {
   return content.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
 }
@@ -421,8 +371,8 @@ function readOptionalText(baseDir, relativePath) {
   if (!relativePath) {
     return void 0;
   }
-  const absolutePath = path4.join(baseDir, relativePath);
-  if (!existsSync2(absolutePath)) {
+  const absolutePath = path3.join(baseDir, relativePath);
+  if (!existsSync(absolutePath)) {
     return void 0;
   }
   return readFileSync(absolutePath, "utf8");
@@ -510,8 +460,8 @@ function parseResultManifest(content) {
   return parseJsonlLines(content);
 }
 function resolveResultSourcePath(source, cwd) {
-  const resolved = path4.isAbsolute(source) ? source : path4.resolve(cwd ?? process.cwd(), source);
-  if (isDirectoryPath(resolved) || path4.basename(resolved) === RESULT_INDEX_FILENAME) {
+  const resolved = path3.isAbsolute(source) ? source : path3.resolve(cwd ?? process.cwd(), source);
+  if (isDirectoryPath(resolved) || path3.basename(resolved) === RESULT_INDEX_FILENAME) {
     return resolveRunManifestPath(resolved);
   }
   return resolved;
@@ -520,7 +470,7 @@ function loadManifestResults(sourceFile) {
   const resolvedSourceFile = resolveRunManifestPath(sourceFile);
   const content = readFileSync(resolvedSourceFile, "utf8");
   const records = parseResultManifest(content);
-  const baseDir = path4.dirname(resolvedSourceFile);
+  const baseDir = path3.dirname(resolvedSourceFile);
   return records.map((record) => hydrateManifestRecord(baseDir, record));
 }
 function loadLightweightResults(sourceFile) {
@@ -566,10 +516,10 @@ function padLeft(str, len) {
 }
 function loadResultFile(filePath) {
   const resolvedFilePath = resolveTraceResultPath(filePath);
-  if (path5.extname(resolvedFilePath) === ".json") {
+  if (path4.extname(resolvedFilePath) === ".json") {
     return loadOtlpTraceFile(resolvedFilePath);
   }
-  if (path5.basename(resolvedFilePath) === RESULT_INDEX_FILENAME) {
+  if (path4.basename(resolvedFilePath) === RESULT_INDEX_FILENAME) {
     return loadManifestAsRawResults(resolvedFilePath);
   }
   return loadJsonlRecords(resolvedFilePath);
@@ -848,7 +798,7 @@ function toTraceSummary(result) {
   return toCamelCaseDeep(rawTrace);
 }
 function buildRunId(relativeRunPath) {
-  const normalized = relativeRunPath.split(path5.sep).join("/");
+  const normalized = relativeRunPath.split(path4.sep).join("/");
   const segments = normalized.split("/").filter(Boolean);
   if (segments.length >= 2) {
     const experiment = segments.slice(0, -1).join("/");
@@ -863,10 +813,10 @@ function buildRunId(relativeRunPath) {
 function collectRunManifestPaths(runsDir, currentDir, files) {
   const primaryPath = resolveExistingRunPrimaryPath(currentDir);
   if (primaryPath) {
-    const relativeRunPath = path5.relative(runsDir, currentDir);
+    const relativeRunPath = path4.relative(runsDir, currentDir);
     files.push({
       filePath: primaryPath,
-      displayName: path5.basename(currentDir),
+      displayName: path4.basename(currentDir),
       runId: buildRunId(relativeRunPath)
     });
     return;
@@ -874,7 +824,7 @@ function collectRunManifestPaths(runsDir, currentDir, files) {
   const entries = readdirSync(currentDir, { withFileTypes: true });
   for (const entry of entries) {
     if (entry.isDirectory()) {
-      collectRunManifestPaths(runsDir, path5.join(currentDir, entry.name), files);
+      collectRunManifestPaths(runsDir, path4.join(currentDir, entry.name), files);
     }
   }
 }
@@ -884,7 +834,7 @@ function listResultFilesFromRunsDir(runsDir, limit) {
     const entries = readdirSync(runsDir, { withFileTypes: true });
     for (const entry of entries) {
       if (entry.isDirectory()) {
-        collectRunManifestPaths(runsDir, path5.join(runsDir, entry.name), files);
+        collectRunManifestPaths(runsDir, path4.join(runsDir, entry.name), files);
       }
     }
   } catch {
@@ -894,7 +844,7 @@ function listResultFilesFromRunsDir(runsDir, limit) {
   const metas = [];
   for (const { filePath, displayName, runId } of limited) {
     try {
-      const fileStat = statSync2(filePath);
+      const fileStat = statSync(filePath);
       const results = loadResultFile(filePath);
       const testCount = results.length;
       const passCount = results.filter((r) => r.score >= DEFAULT_THRESHOLD).length;
@@ -919,7 +869,7 @@ function listResultFilesFromRunsDir(runsDir, limit) {
 }
 function listResultFiles(cwd, limit) {
   return listResultFilesFromRunsDir(
-    path5.join(cwd, ".agentv", "results", RESULT_RUNS_DIRNAME),
+    path4.join(cwd, ".agentv", "results", RESULT_RUNS_DIRNAME),
     limit
   );
 }
@@ -971,17 +921,17 @@ function slugify(value) {
   return value.trim().replace(/[^A-Za-z0-9._/-]+/g, "-").replace(/\/+/g, "/").replace(/^-+|-+$/g, "").slice(0, 120);
 }
 function getRelativeRunPath(cwd, runDir) {
-  const relative = path6.relative(path6.join(cwd, ".agentv", "results", "runs"), runDir);
-  if (!relative.startsWith("..") && !path6.isAbsolute(relative)) {
+  const relative = path5.relative(path5.join(cwd, ".agentv", "results", "runs"), runDir);
+  if (!relative.startsWith("..") && !path5.isAbsolute(relative)) {
     return relative;
   }
-  const experiment = path6.basename(path6.dirname(runDir));
-  const runName = path6.basename(runDir);
-  return experiment && experiment !== runName ? path6.join(experiment, runName) : runName;
+  const experiment = path5.basename(path5.dirname(runDir));
+  const runName = path5.basename(runDir);
+  return experiment && experiment !== runName ? path5.join(experiment, runName) : runName;
 }
 function buildBranchName(config, payload) {
-  const timestamp = path6.basename(payload.run_dir);
-  const evalStem = payload.test_files.length === 1 ? path6.basename(payload.test_files[0]).replace(/\.eval\.ya?ml$/i, "").replace(/\.[^.]+$/i, "") : `${payload.test_files.length}-evals`;
+  const timestamp = path5.basename(payload.run_dir);
+  const evalStem = payload.test_files.length === 1 ? path5.basename(payload.test_files[0]).replace(/\.eval\.ya?ml$/i, "").replace(/\.[^.]+$/i, "") : `${payload.test_files.length}-evals`;
   const experiment = slugify(payload.experiment ?? "default");
   const branchLeaf = slugify(`${experiment}-${evalStem}-${timestamp}`) || timestamp;
   return `${config.branch_prefix}/${branchLeaf}`;
@@ -1010,7 +960,7 @@ function buildPrBody(payload) {
     "",
     sections,
     "",
-    `Run: ${path6.basename(payload.run_dir)}`,
+    `Run: ${path5.basename(payload.run_dir)}`,
     `Experiment: ${payload.experiment ?? "default"}`,
     `Eval Files: ${payload.test_files.join(", ")}`
   ].join("\n");
@@ -1025,7 +975,7 @@ async function maybeWarnLargeArtifact(runDir) {
 }
 async function loadNormalizedResultsConfig(cwd) {
   const repoRoot = await findRepoRoot(cwd) ?? cwd;
-  const config = await loadConfig(path6.join(cwd, "_"), repoRoot);
+  const config = await loadConfig(path5.join(cwd, "_"), repoRoot);
   if (!config?.results?.export) {
     return void 0;
   }
@@ -1109,7 +1059,7 @@ async function maybeAutoExportRunArtifacts(payload) {
     const prepared = await prepareResultsRepoBranch(config, branchName);
     try {
       const relativeRunPath = getRelativeRunPath(payload.cwd, payload.run_dir);
-      const destinationDir = path6.join(prepared.repoDir, config.path, relativeRunPath);
+      const destinationDir = path5.join(prepared.repoDir, config.path, relativeRunPath);
       await stageResultsArtifacts({
         repoDir: prepared.repoDir,
         sourceDir: payload.run_dir,
@@ -1143,36 +1093,8 @@ async function maybeAutoExportRunArtifacts(payload) {
   }
 }
 
-// src/commands/eval/artifact-writer.ts
-import { mkdir, readFile, writeFile } from "node:fs/promises";
-import path7 from "node:path";
-
-// src/utils/case-conversion.ts
-function toSnakeCase(str) {
-  if (/^[A-Z]/.test(str)) {
-    return str;
-  }
-  return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
-}
-function toSnakeCaseDeep(obj) {
-  if (obj === null || obj === void 0) {
-    return obj;
-  }
-  if (Array.isArray(obj)) {
-    return obj.map((item) => toSnakeCaseDeep(item));
-  }
-  if (typeof obj === "object") {
-    const result = {};
-    for (const [key, value] of Object.entries(obj)) {
-      const snakeKey = toSnakeCase(key);
-      result[snakeKey] = toSnakeCaseDeep(value);
-    }
-    return result;
-  }
-  return obj;
-}
-
-// src/commands/eval/artifact-writer.ts
+// src/commands/eval/benchmark-writer.ts
+import { writeFile } from "node:fs/promises";
 function computeStats(values) {
   if (values.length === 0) {
     return { mean: 0, stddev: 0 };
@@ -1185,361 +1107,6 @@ function computeStats(values) {
   };
 }
 function computePassRate(result) {
-  const scores = result.scores;
-  if (scores && scores.length > 0) {
-    const passed = scores.filter((s) => s.score >= DEFAULT_THRESHOLD).length;
-    return passed / scores.length;
-  }
-  return (result.score ?? 0) >= DEFAULT_THRESHOLD ? 1 : 0;
-}
-function countToolCalls(result) {
-  const toolCalls = {};
-  let total = 0;
-  const trace = result.trace;
-  if (trace?.steps) {
-    for (const step of trace.steps) {
-      if (step.toolName || step.type === "tool") {
-        const name = step.toolName ?? "unknown";
-        toolCalls[name] = (toolCalls[name] ?? 0) + 1;
-        total += 1;
-      }
-    }
-  }
-  return { toolCalls, total };
-}
-function parseWorkspaceChanges(fileChanges) {
-  if (!fileChanges) {
-    return void 0;
-  }
-  let filesModified = 0;
-  let filesCreated = 0;
-  const lines = fileChanges.split("\n");
-  for (const line of lines) {
-    if (line.startsWith("--- /dev/null")) {
-      filesCreated += 1;
-    } else if (line.startsWith("--- a/")) {
-      filesModified += 1;
-    }
-  }
-  const summaryLines = lines.slice(0, 20);
-  const diffSummary = lines.length > 20 ? `${summaryLines.join("\n")}
-... (${lines.length - 20} more lines)` : fileChanges;
-  return {
-    files_modified: filesModified,
-    files_created: filesCreated,
-    diff_summary: diffSummary
-  };
-}
-function buildAssertions(result) {
-  if (!result.assertions) return [];
-  return result.assertions.map((a) => ({
-    text: a.text,
-    passed: a.passed,
-    evidence: a.evidence ?? ""
-  }));
-}
-function buildEvaluators(scores) {
-  if (!scores || scores.length === 0) {
-    return void 0;
-  }
-  return scores.map((s) => ({
-    name: s.name,
-    type: s.type,
-    score: s.score,
-    reasoning: "",
-    weight: s.weight,
-    verdict: s.verdict,
-    assertions: s.assertions,
-    details: s.details
-  }));
-}
-function buildGradingArtifact(result) {
-  const assertions = buildAssertions(result);
-  const passed = assertions.filter((e) => e.passed).length;
-  const failed = assertions.filter((e) => !e.passed).length;
-  const total = assertions.length;
-  const { toolCalls, total: totalToolCalls } = countToolCalls(result);
-  const errorsEncountered = result.error ? 1 : 0;
-  return {
-    assertions,
-    summary: {
-      passed,
-      failed,
-      total,
-      pass_rate: total > 0 ? Math.round(passed / total * 1e3) / 1e3 : 0
-    },
-    execution_metrics: {
-      tool_calls: toolCalls,
-      total_tool_calls: totalToolCalls,
-      errors_encountered: errorsEncountered
-    },
-    graders: buildEvaluators(result.scores),
-    workspace_changes: parseWorkspaceChanges(result.fileChanges),
-    conversation: result.conversationId ? {
-      turns: result.trace ? result.trace.steps?.length ?? 0 : 0,
-      conversation_id: result.conversationId
-    } : void 0
-  };
-}
-function buildTimingArtifact(results) {
-  let totalInput = 0;
-  let totalOutput = 0;
-  let totalReasoning = 0;
-  let totalDurationMs = 0;
-  for (const result of results) {
-    const usage = result.tokenUsage;
-    if (usage) {
-      totalInput += usage.input ?? 0;
-      totalOutput += usage.output ?? 0;
-      totalReasoning += usage.reasoning ?? 0;
-    }
-    if (result.durationMs != null) {
-      totalDurationMs += result.durationMs;
-    }
-  }
-  return {
-    total_tokens: totalInput + totalOutput,
-    duration_ms: totalDurationMs,
-    total_duration_seconds: Math.round(totalDurationMs / 1e3 * 1e3) / 1e3,
-    token_usage: {
-      input: totalInput,
-      output: totalOutput,
-      reasoning: totalReasoning
-    }
-  };
-}
-function buildBenchmarkArtifact(results, evalFile = "", experiment) {
-  const targetSet = /* @__PURE__ */ new Set();
-  const testIdSet = /* @__PURE__ */ new Set();
-  for (const result of results) {
-    targetSet.add(result.target ?? "unknown");
-    testIdSet.add(result.testId ?? "unknown");
-  }
-  const targets = [...targetSet].sort();
-  const testIds = [...testIdSet].sort();
-  const runSummary = {};
-  const notes = [];
-  for (const target of targets) {
-    const targetResults = results.filter((r) => r.target === target);
-    const passRates = targetResults.map(computePassRate);
-    const timings = targetResults.filter((r) => r.durationMs != null).map((r) => r.durationMs / 1e3);
-    const tokens = targetResults.filter((r) => r.tokenUsage != null).map((r) => {
-      const usage = r.tokenUsage;
-      return (usage.input ?? 0) + (usage.output ?? 0);
-    });
-    const entry = {
-      pass_rate: computeStats(passRates),
-      time_seconds: computeStats(timings),
-      tokens: computeStats(tokens)
-    };
-    const toolCallCounts = targetResults.map((r) => countToolCalls(r).total);
-    if (toolCallCounts.some((c2) => c2 > 0)) {
-      entry.tool_calls = computeStats(toolCallCounts);
-    }
-    const costs = targetResults.filter((r) => r.costUsd != null).map((r) => r.costUsd);
-    if (costs.length > 0) {
-      entry.cost_usd = computeStats(costs);
-    }
-    runSummary[target] = entry;
-  }
-  const evaluatorScores = /* @__PURE__ */ new Map();
-  for (const result of results) {
-    if (result.scores) {
-      for (const score of result.scores) {
-        const key = `${score.name}:${score.type}`;
-        if (!evaluatorScores.has(key)) {
-          evaluatorScores.set(key, []);
-        }
-        evaluatorScores.get(key)?.push(score.score);
-      }
-    }
-  }
-  let perEvaluatorSummary;
-  if (evaluatorScores.size > 0) {
-    perEvaluatorSummary = {};
-    for (const [key, scores] of evaluatorScores) {
-      perEvaluatorSummary[key] = computeStats(scores);
-    }
-  }
-  const errorCount = results.filter(
-    (r) => r.executionStatus != null && r.executionStatus === "execution_error"
-  ).length;
-  if (errorCount > 0) {
-    notes.push(
-      `${errorCount} test(s) had execution errors and are included in pass_rate as failures`
-    );
-  }
-  if (results.length === 0) {
-    notes.push("No results to summarize");
-  }
-  const firstResult = results[0];
-  const timestamp = firstResult?.timestamp ?? (/* @__PURE__ */ new Date()).toISOString();
-  return {
-    metadata: {
-      eval_file: evalFile,
-      timestamp,
-      targets,
-      tests_run: testIds,
-      experiment
-    },
-    run_summary: runSummary,
-    per_grader_summary: perEvaluatorSummary,
-    notes
-  };
-}
-function safeArtifactPathSegment(value, fallback) {
-  const trimmed = value?.trim();
-  if (!trimmed) {
-    return fallback;
-  }
-  return trimmed.replace(/[/\\:*?"<>|]/g, "_");
-}
-function safeTestId(testId) {
-  return safeArtifactPathSegment(testId, "unknown");
-}
-function getSuite(result) {
-  return result.suite;
-}
-function buildArtifactSubdir(result) {
-  const segments = [];
-  const evalSet = getSuite(result);
-  if (evalSet) {
-    segments.push(safeArtifactPathSegment(evalSet, "default"));
-  }
-  segments.push(safeTestId(result.testId));
-  return path7.posix.join(...segments);
-}
-function formatOutputMarkdown(output) {
-  return output.map((msg) => `@[${msg.role}]:
-${String(msg.content ?? "")}`).join("\n\n");
-}
-function extractInput(result) {
-  const input = result.input;
-  if (!input) return null;
-  if (typeof input === "string") return input;
-  if (Array.isArray(input) && input.length > 0) {
-    return formatOutputMarkdown(input);
-  }
-  return null;
-}
-function buildResultIndexArtifact(result) {
-  const artifactSubdir = buildArtifactSubdir(result);
-  const input = extractInput(result);
-  const hasResponse = Array.isArray(result.output) && result.output.length > 0;
-  return {
-    timestamp: result.timestamp,
-    test_id: result.testId ?? "unknown",
-    suite: getSuite(result),
-    category: result.category,
-    conversation_id: result.conversationId,
-    score: result.score,
-    target: result.target ?? "unknown",
-    scores: result.scores ? toSnakeCaseDeep(result.scores) : void 0,
-    execution_status: result.executionStatus,
-    error: result.error,
-    failure_stage: result.failureStage,
-    failure_reason_code: result.failureReasonCode,
-    workspace_path: result.workspacePath,
-    grading_path: path7.posix.join(artifactSubdir, "grading.json"),
-    timing_path: path7.posix.join(artifactSubdir, "timing.json"),
-    input_path: input ? path7.posix.join(artifactSubdir, "input.md") : void 0,
-    output_path: hasResponse ? path7.posix.join(artifactSubdir, "outputs", "response.md") : void 0,
-    response_path: hasResponse ? path7.posix.join(artifactSubdir, "outputs", "response.md") : void 0
-  };
-}
-async function writeJsonlFile(filePath, records) {
-  const content = records.length === 0 ? "" : `${records.map((record) => JSON.stringify(toSnakeCaseDeep(record))).join("\n")}
-`;
-  await writeFile(filePath, content, "utf8");
-}
-function buildTranscriptMessageLines(results) {
-  const lines = [];
-  for (const result of results) {
-    const transcriptLines = toTranscriptJsonLines(
-      {
-        messages: [...result.input ?? [], ...result.output],
-        source: {
-          provider: result.target,
-          sessionId: result.conversationId ?? result.testId,
-          startedAt: result.timestamp
-        },
-        tokenUsage: result.tokenUsage,
-        durationMs: result.durationMs,
-        costUsd: result.costUsd
-      },
-      {
-        testId: result.testId,
-        target: result.target
-      }
-    );
-    lines.push(...transcriptLines.map((line) => JSON.stringify(line)));
-  }
-  return lines.length > 0 ? `${lines.join("\n")}
-` : "";
-}
-async function writeArtifactsFromResults(results, outputDir, options) {
-  const testArtifactDir = outputDir;
-  const timingPath = path7.join(outputDir, "timing.json");
-  const benchmarkPath = path7.join(outputDir, "benchmark.json");
-  const indexPath = path7.join(outputDir, RESULT_INDEX_FILENAME);
-  await mkdir(outputDir, { recursive: true });
-  const indexRecords = [];
-  for (const result of results) {
-    const grading = buildGradingArtifact(result);
-    const timing2 = buildTimingArtifact([result]);
-    const artifactSubdir = buildArtifactSubdir(result);
-    const testDir = path7.join(outputDir, artifactSubdir);
-    const gradingPath = path7.join(testDir, "grading.json");
-    const perTestTimingPath = path7.join(testDir, "timing.json");
-    await mkdir(testDir, { recursive: true });
-    await writeFile(gradingPath, `${JSON.stringify(grading, null, 2)}
-`, "utf8");
-    await writeFile(perTestTimingPath, `${JSON.stringify(timing2, null, 2)}
-`, "utf8");
-    const input = extractInput(result);
-    if (input) {
-      await writeFile(path7.join(testDir, "input.md"), input, "utf8");
-    }
-    if (result.output && result.output.length > 0) {
-      const outputsDir = path7.join(testDir, "outputs");
-      await mkdir(outputsDir, { recursive: true });
-      await writeFile(
-        path7.join(outputsDir, "response.md"),
-        formatOutputMarkdown(result.output),
-        "utf8"
-      );
-    }
-    indexRecords.push({
-      ...buildResultIndexArtifact(result),
-      experiment: options?.experiment
-    });
-  }
-  const timing = buildTimingArtifact(results);
-  await writeFile(timingPath, `${JSON.stringify(timing, null, 2)}
-`, "utf8");
-  const benchmark = buildBenchmarkArtifact(results, options?.evalFile, options?.experiment);
-  await writeFile(benchmarkPath, `${JSON.stringify(benchmark, null, 2)}
-`, "utf8");
-  await writeJsonlFile(indexPath, indexRecords);
-  const transcriptPath = path7.join(outputDir, "transcript.jsonl");
-  await writeFile(transcriptPath, buildTranscriptMessageLines(results), "utf8");
-  return { testArtifactDir, timingPath, benchmarkPath, indexPath };
-}
-
-// src/commands/eval/benchmark-writer.ts
-import { writeFile as writeFile2 } from "node:fs/promises";
-function computeStats2(values) {
-  if (values.length === 0) {
-    return { mean: 0, stddev: 0 };
-  }
-  const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
-  const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / values.length;
-  return {
-    mean: Math.round(mean * 1e3) / 1e3,
-    stddev: Math.round(Math.sqrt(variance) * 1e3) / 1e3
-  };
-}
-function computePassRate2(result) {
   const scores = result.scores;
   if (scores && scores.length > 0) {
     const passed = scores.filter((s) => s.score >= DEFAULT_THRESHOLD).length;
@@ -1548,7 +1115,7 @@ function computePassRate2(result) {
   return result.score >= DEFAULT_THRESHOLD ? 1 : 0;
 }
 function buildBenchmarkJson(results) {
-  const passRates = results.map(computePassRate2);
+  const passRates = results.map(computePassRate);
   const timings = results.filter((r) => r.durationMs != null).map((r) => r.durationMs / 1e3);
   const tokens = results.filter((r) => r.tokenUsage != null).map((r) => {
     const usage = r.tokenUsage;
@@ -1557,29 +1124,29 @@ function buildBenchmarkJson(results) {
   return {
     run_summary: {
       with_skill: {
-        pass_rate: computeStats2(passRates),
-        time_seconds: computeStats2(timings),
-        tokens: computeStats2(tokens)
+        pass_rate: computeStats(passRates),
+        time_seconds: computeStats(timings),
+        tokens: computeStats(tokens)
       }
     }
   };
 }
 async function writeBenchmarkJson(outputPath, results) {
   const benchmark = buildBenchmarkJson(results);
-  await writeFile2(outputPath, `${JSON.stringify(benchmark, null, 2)}
+  await writeFile(outputPath, `${JSON.stringify(benchmark, null, 2)}
 `, "utf8");
 }
 
 // src/commands/eval/env.ts
 import { constants as constants3 } from "node:fs";
 import { access as access3 } from "node:fs/promises";
-import path8 from "node:path";
+import path6 from "node:path";
 import { config as loadDotenv } from "dotenv";
 function uniqueDirs(directories) {
   const seen = /* @__PURE__ */ new Set();
   const result = [];
   for (const dir of directories) {
-    const absolute = path8.resolve(dir);
+    const absolute = path6.resolve(dir);
     if (seen.has(absolute)) {
       continue;
     }
@@ -1598,14 +1165,14 @@ async function fileExists2(filePath) {
 }
 function collectAncestorDirectories(start, boundary) {
   const directories = [];
-  const boundaryDir = path8.resolve(boundary);
-  let current = path8.resolve(start);
+  const boundaryDir = path6.resolve(boundary);
+  let current = path6.resolve(start);
   while (current !== void 0) {
     directories.push(current);
     if (current === boundaryDir) {
       break;
     }
-    const parent = path8.dirname(current);
+    const parent = path6.dirname(current);
     if (parent === current) {
       break;
     }
@@ -1615,12 +1182,12 @@ function collectAncestorDirectories(start, boundary) {
 }
 async function loadEnvFromHierarchy(options) {
   const { testFilePath, repoRoot, verbose } = options;
-  const testDir = path8.dirname(path8.resolve(testFilePath));
+  const testDir = path6.dirname(path6.resolve(testFilePath));
   const cwd = process.cwd();
   const searchDirs = uniqueDirs([...collectAncestorDirectories(testDir, repoRoot), repoRoot, cwd]);
   const envFiles = [];
   for (const dir of searchDirs) {
-    const candidate = path8.join(dir, ".env");
+    const candidate = path6.join(dir, ".env");
     if (await fileExists2(candidate)) {
       envFiles.push(candidate);
     }
@@ -1642,11 +1209,11 @@ async function loadEnvFromHierarchy(options) {
 }
 
 // src/commands/eval/output-writer.ts
-import path14 from "node:path";
+import path12 from "node:path";
 
 // src/commands/eval/html-writer.ts
-import { mkdir as mkdir2, writeFile as writeFile3 } from "node:fs/promises";
-import path9 from "node:path";
+import { mkdir, writeFile as writeFile2 } from "node:fs/promises";
+import path7 from "node:path";
 
 // ../../node_modules/.bun/async-mutex@0.5.0/node_modules/async-mutex/index.mjs
 var E_TIMEOUT = new Error("timeout while waiting for mutex to become available");
@@ -1865,7 +1432,7 @@ var HtmlWriter = class _HtmlWriter {
     this.filePath = filePath;
   }
   static async open(filePath) {
-    await mkdir2(path9.dirname(filePath), { recursive: true });
+    await mkdir(path7.dirname(filePath), { recursive: true });
     const writer = new _HtmlWriter(filePath);
     await writer.writeHtml();
     return writer;
@@ -1891,7 +1458,7 @@ var HtmlWriter = class _HtmlWriter {
   }
   async writeHtml() {
     const html = generateHtml(this.results, this.isLive);
-    await writeFile3(this.filePath, html, "utf8");
+    await writeFile2(this.filePath, html, "utf8");
   }
 };
 function generateHtml(results, isLive) {
@@ -2326,10 +1893,10 @@ var SCRIPT = `
     h+='<div class="detail-block"><h4>Output</h4><pre class="detail-pre">'+esc(r.output?JSON.stringify(r.output,null,2):"")+"</pre></div>";
     h+="</div>";
 
-    /* evaluator results */
+    /* grader results */
     if(r.scores&&r.scores.length>0){
-      h+="<h4>Evaluator Results</h4>";
-      h+='<table class="eval-table"><thead><tr><th>Evaluator</th><th>Score</th><th>Status</th><th>Assertions</th></tr></thead><tbody>';
+      h+="<h4>Grader Results</h4>";
+      h+='<table class="eval-table"><thead><tr><th>Grader</th><th>Score</th><th>Status</th><th>Assertions</th></tr></thead><tbody>';
       for(var i=0;i<r.scores.length;i++){
         var ev=r.scores[i],evS=ev.score>=0.5?"pass":"fail";
         var evAssertions=ev.assertions||[];
@@ -2375,8 +1942,8 @@ var SCRIPT = `
 `;
 
 // src/commands/eval/json-writer.ts
-import { mkdir as mkdir3, writeFile as writeFile4 } from "node:fs/promises";
-import path10 from "node:path";
+import { mkdir as mkdir2, writeFile as writeFile3 } from "node:fs/promises";
+import path8 from "node:path";
 var JsonWriter = class _JsonWriter {
   filePath;
   results = [];
@@ -2385,7 +1952,7 @@ var JsonWriter = class _JsonWriter {
     this.filePath = filePath;
   }
   static async open(filePath) {
-    await mkdir3(path10.dirname(filePath), { recursive: true });
+    await mkdir2(path8.dirname(filePath), { recursive: true });
     return new _JsonWriter(filePath);
   }
   async append(result) {
@@ -2412,15 +1979,15 @@ var JsonWriter = class _JsonWriter {
       results: this.results
     };
     const snakeCaseOutput = toSnakeCaseDeep(output);
-    await writeFile4(this.filePath, `${JSON.stringify(snakeCaseOutput, null, 2)}
+    await writeFile3(this.filePath, `${JSON.stringify(snakeCaseOutput, null, 2)}
 `, "utf8");
   }
 };
 
 // src/commands/eval/jsonl-writer.ts
 import { createWriteStream } from "node:fs";
-import { mkdir as mkdir4 } from "node:fs/promises";
-import path11 from "node:path";
+import { mkdir as mkdir3 } from "node:fs/promises";
+import path9 from "node:path";
 import { finished } from "node:stream/promises";
 var JsonlWriter = class _JsonlWriter {
   stream;
@@ -2429,9 +1996,10 @@ var JsonlWriter = class _JsonlWriter {
   constructor(stream) {
     this.stream = stream;
   }
-  static async open(filePath) {
-    await mkdir4(path11.dirname(filePath), { recursive: true });
-    const stream = createWriteStream(filePath, { flags: "w", encoding: "utf8" });
+  static async open(filePath, options) {
+    await mkdir3(path9.dirname(filePath), { recursive: true });
+    const flags = options?.append ? "a" : "w";
+    const stream = createWriteStream(filePath, { flags, encoding: "utf8" });
     return new _JsonlWriter(stream);
   }
   async append(record) {
@@ -2461,8 +2029,8 @@ var JsonlWriter = class _JsonlWriter {
 };
 
 // src/commands/eval/junit-writer.ts
-import { mkdir as mkdir5, writeFile as writeFile5 } from "node:fs/promises";
-import path12 from "node:path";
+import { mkdir as mkdir4, writeFile as writeFile4 } from "node:fs/promises";
+import path10 from "node:path";
 function escapeXml(str) {
   return str.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
 }
@@ -2476,7 +2044,7 @@ var JunitWriter = class _JunitWriter {
     this.threshold = options?.threshold ?? 0.5;
   }
   static async open(filePath, options) {
-    await mkdir5(path12.dirname(filePath), { recursive: true });
+    await mkdir4(path10.dirname(filePath), { recursive: true });
     return new _JunitWriter(filePath, options);
   }
   async append(result) {
@@ -2545,14 +2113,14 @@ ${testCases.join("\n")}
 ${suiteXmls.join("\n")}
 </testsuites>
 `;
-    await writeFile5(this.filePath, xml, "utf8");
+    await writeFile4(this.filePath, xml, "utf8");
   }
 };
 
 // src/commands/eval/yaml-writer.ts
 import { createWriteStream as createWriteStream2 } from "node:fs";
-import { mkdir as mkdir6 } from "node:fs/promises";
-import path13 from "node:path";
+import { mkdir as mkdir5 } from "node:fs/promises";
+import path11 from "node:path";
 import { finished as finished2 } from "node:stream/promises";
 import { stringify as stringifyYaml } from "yaml";
 var YamlWriter = class _YamlWriter {
@@ -2564,7 +2132,7 @@ var YamlWriter = class _YamlWriter {
     this.stream = stream;
   }
   static async open(filePath) {
-    await mkdir6(path13.dirname(filePath), { recursive: true });
+    await mkdir5(path11.dirname(filePath), { recursive: true });
     const stream = createWriteStream2(filePath, { flags: "w", encoding: "utf8" });
     return new _YamlWriter(stream);
   }
@@ -2604,10 +2172,10 @@ var YamlWriter = class _YamlWriter {
 };
 
 // src/commands/eval/output-writer.ts
-async function createOutputWriter(filePath, format) {
+async function createOutputWriter(filePath, format, options) {
   switch (format) {
     case "jsonl":
-      return JsonlWriter.open(filePath);
+      return JsonlWriter.open(filePath, { append: options?.append });
     case "yaml":
       return YamlWriter.open(filePath);
     case "html":
@@ -2620,7 +2188,7 @@ async function createOutputWriter(filePath, format) {
 }
 var SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([".jsonl", ".json", ".xml", ".yaml", ".yml", ".html", ".htm"]);
 function createWriterFromPath(filePath, options) {
-  const ext = path14.extname(filePath).toLowerCase();
+  const ext = path12.extname(filePath).toLowerCase();
   switch (ext) {
     case ".jsonl":
       return JsonlWriter.open(filePath);
@@ -2719,12 +2287,12 @@ var ProgressDisplay = class {
   }
   addLogPaths(paths) {
     const newPaths = [];
-    for (const path19 of paths) {
-      if (this.logPathSet.has(path19)) {
+    for (const path17 of paths) {
+      if (this.logPathSet.has(path17)) {
         continue;
       }
-      this.logPathSet.add(path19);
-      newPaths.push(path19);
+      this.logPathSet.add(path17);
+      newPaths.push(path17);
     }
     if (newPaths.length === 0) {
       return;
@@ -2777,8 +2345,8 @@ async function loadNonErrorResults(jsonlPath) {
 }
 
 // src/commands/eval/run-cache.ts
-import { mkdir as mkdir7, readFile as readFile2, writeFile as writeFile6 } from "node:fs/promises";
-import path15 from "node:path";
+import { mkdir as mkdir6, readFile, writeFile as writeFile5 } from "node:fs/promises";
+import path13 from "node:path";
 var CACHE_FILENAME = "cache.json";
 function resolveRunCacheFile(cache) {
   if (cache.lastRunDir) {
@@ -2787,27 +2355,27 @@ function resolveRunCacheFile(cache) {
   return "";
 }
 function cachePath(cwd) {
-  return path15.join(cwd, ".agentv", CACHE_FILENAME);
+  return path13.join(cwd, ".agentv", CACHE_FILENAME);
 }
 async function loadRunCache(cwd) {
   try {
-    const content = await readFile2(cachePath(cwd), "utf-8");
+    const content = await readFile(cachePath(cwd), "utf-8");
     return JSON.parse(content);
   } catch {
     return void 0;
   }
 }
 async function saveRunCache(cwd, resultPath) {
-  if (path15.basename(resultPath) !== RESULT_INDEX_FILENAME) {
+  if (path13.basename(resultPath) !== RESULT_INDEX_FILENAME) {
     return;
   }
-  const dir = path15.join(cwd, ".agentv");
-  await mkdir7(dir, { recursive: true });
+  const dir = path13.join(cwd, ".agentv");
+  await mkdir6(dir, { recursive: true });
   const cache = {
-    lastRunDir: path15.dirname(resultPath),
+    lastRunDir: path13.dirname(resultPath),
     timestamp: (/* @__PURE__ */ new Date()).toISOString()
   };
-  await writeFile6(cachePath(cwd), `${JSON.stringify(cache, null, 2)}
+  await writeFile5(cachePath(cwd), `${JSON.stringify(cache, null, 2)}
 `, "utf-8");
 }
 
@@ -3072,13 +2640,13 @@ function formatMatrixSummary(results) {
 }
 
 // ../../packages/core/dist/evaluation/validation/index.js
-import { readFile as readFile3 } from "node:fs/promises";
-import path16 from "node:path";
+import { readFile as readFile2 } from "node:fs/promises";
+import path14 from "node:path";
 import { parse } from "yaml";
 import { readFile as readFile22, readdir } from "node:fs/promises";
 import path22 from "node:path";
 import { parse as parse2 } from "yaml";
-import { readFile as readFile32 } from "node:fs/promises";
+import { readFile as readFile3 } from "node:fs/promises";
 import path32 from "node:path";
 import { parse as parse3 } from "yaml";
 import { readFile as readFile4 } from "node:fs/promises";
@@ -3097,7 +2665,7 @@ var SCHEMA_TARGETS_V2 = "agentv-targets-v2.2";
 var SCHEMA_CONFIG_V2 = "agentv-config-v2";
 async function detectFileType(filePath) {
   try {
-    const content = await readFile3(filePath, "utf8");
+    const content = await readFile2(filePath, "utf8");
     const parsed = parse(content);
     if (Array.isArray(parsed)) {
       return "cases";
@@ -3125,8 +2693,8 @@ async function detectFileType(filePath) {
   }
 }
 function inferFileTypeFromPath(filePath) {
-  const normalized = path16.normalize(filePath).replace(/\\/g, "/");
-  const basename = path16.basename(filePath);
+  const normalized = path14.normalize(filePath).replace(/\\/g, "/");
+  const basename = path14.basename(filePath);
   if (normalized.includes("/.agentv/")) {
     if (basename === "config.yaml" || basename === "config.yml") {
       return "config";
@@ -3723,7 +3291,7 @@ function validateAssertArray(assertField, parentLocation, filePath, errors, cust
       continue;
     }
     const typeValue = rawTypeValue.replace(/_/g, "-");
-    if (!isEvaluatorKind(typeValue) && !customAssertionTypes.has(typeValue)) {
+    if (!isGraderKind(typeValue) && !customAssertionTypes.has(typeValue)) {
       errors.push({
         severity: "warning",
         filePath,
@@ -3897,7 +3465,7 @@ async function validateCasesFile(filePath) {
   const absolutePath = path32.resolve(filePath);
   let parsed;
   try {
-    const content = await readFile32(absolutePath, "utf8");
+    const content = await readFile3(absolutePath, "utf8");
     parsed = parse3(content);
   } catch (error) {
     errors.push({
@@ -5005,6 +4573,12 @@ Errors in ${targetsFilePath}:`);
 
 // src/commands/eval/run-eval.ts
 var DEFAULT_WORKERS = 3;
+function shouldSkipExistingResultForResume(result, rerunFailed) {
+  if (rerunFailed) {
+    return result.executionStatus === "ok";
+  }
+  return result.executionStatus !== "execution_error";
+}
 function normalizeBoolean(value) {
   return value === true;
 }
@@ -5175,6 +4749,8 @@ function normalizeOptions(rawOptions, config, yamlExecution) {
     otelCaptureContent: normalizeBoolean(rawOptions.otelCaptureContent) || yamlExecution?.otel_capture_content === true,
     otelGroupTurns: normalizeBoolean(rawOptions.otelGroupTurns) || yamlExecution?.otel_group_turns === true,
     retryErrors: normalizeString(rawOptions.retryErrors),
+    resume: normalizeBoolean(rawOptions.resume) || normalizeBoolean(rawOptions.rerunFailed),
+    rerunFailed: normalizeBoolean(rawOptions.rerunFailed),
     workspaceMode,
     workspacePath,
     // Precedence: CLI > YAML config > TS config
@@ -5201,7 +4777,7 @@ async function ensureFileExists(filePath, description) {
 function buildDefaultOutputPathForExperiment(cwd, experiment) {
   const runDir = buildDefaultRunDir(cwd, experiment);
   mkdirSync(runDir, { recursive: true });
-  return path17.join(runDir, "index.jsonl");
+  return path15.join(runDir, "index.jsonl");
 }
 function createProgressReporter(maxWorkers, options) {
   const display = new ProgressDisplay(maxWorkers, options);
@@ -5215,7 +4791,7 @@ function createProgressReporter(maxWorkers, options) {
   };
 }
 function makeTestCaseKey(testFilePath, testId) {
-  return `${path17.resolve(testFilePath)}::${testId}`;
+  return `${path15.resolve(testFilePath)}::${testId}`;
 }
 function resolveTargetLabel(requestedName, resolvedName) {
   if (resolvedName !== requestedName) {
@@ -5277,7 +4853,7 @@ async function prepareFileMetadata(params) {
     repoRoot,
     verbose: options.verbose
   });
-  const relativePath = path17.relative(cwd, testFilePath);
+  const relativePath = path15.relative(cwd, testFilePath);
   const category = deriveCategory(relativePath);
   const suite = await loadTestSuite(testFilePath, repoRoot, {
     verbose: options.verbose,
@@ -5302,7 +4878,53 @@ async function prepareFileMetadata(params) {
     selections = [
       {
         selection: transcriptSelection,
-        inlineTargetLabel: `transcript (${path17.basename(options.transcript)})`
+        inlineTargetLabel: `transcript (${path15.basename(options.transcript)})`
+      }
+    ];
+  } else if (suite.inlineTarget && options.cliTargets.length === 0) {
+    const targetDefinition = suite.inlineTarget;
+    const resolvedTarget = options.dryRun ? {
+      kind: "mock",
+      name: `${targetDefinition.name}-dry-run`,
+      graderTarget: void 0,
+      config: {
+        response: '{"answer":"Mock dry-run response"}',
+        delayMs: options.dryRunDelay,
+        delayMinMs: options.dryRunDelayMin,
+        delayMaxMs: options.dryRunDelayMax
+      }
+    } : resolveTargetDefinition(targetDefinition, process.env, testFilePath, {
+      emitDeprecationWarnings: false
+    });
+    selections = [
+      {
+        selection: {
+          definitions: [targetDefinition],
+          resolvedTarget,
+          targetName: targetDefinition.name,
+          targetSource: "test-file",
+          targetsFilePath: testFilePath
+        },
+        inlineTargetLabel: resolveTargetLabel(targetDefinition.name, resolvedTarget.name)
+      }
+    ];
+  } else if (suite.providerFactory && options.cliTargets.length === 0) {
+    const taskTarget = {
+      kind: "mock",
+      name: "custom-task",
+      graderTarget: void 0,
+      config: {}
+    };
+    selections = [
+      {
+        selection: {
+          definitions: [],
+          resolvedTarget: taskTarget,
+          targetName: "custom-task",
+          targetSource: "test-file",
+          targetsFilePath: testFilePath
+        },
+        inlineTargetLabel: "custom-task"
       }
     ];
   } else {
@@ -5372,10 +4994,11 @@ async function prepareFileMetadata(params) {
     yamlWorkers: suite.workers,
     yamlCache: suite.cacheConfig?.enabled,
     yamlCachePath: suite.cacheConfig?.cachePath,
-    totalBudgetUsd: suite.totalBudgetUsd,
+    budgetUsd: suite.budgetUsd,
     failOnError: suite.failOnError,
     threshold: suite.threshold,
-    tags: suite.metadata?.tags
+    tags: suite.metadata?.tags,
+    providerFactory: suite.providerFactory
   };
 }
 async function runSingleEvalFile(params) {
@@ -5398,7 +5021,7 @@ async function runSingleEvalFile(params) {
     testCases,
     trialsConfig,
     matrixMode,
-    totalBudgetUsd,
+    budgetUsd,
     failOnError,
     providerFactory
   } = params;
@@ -5464,7 +5087,7 @@ async function runSingleEvalFile(params) {
     workspacePath: options.workspacePath,
     keepWorkspaces: options.keepWorkspaces,
     trials: trialsConfig,
-    totalBudgetUsd,
+    budgetUsd,
     failOnError,
     graderTarget: options.graderTarget,
     model: options.model,
@@ -5537,7 +5160,7 @@ async function runEvalCommand(input) {
     );
   }
   const repoRoot = await findRepoRoot(cwd);
-  const yamlConfig = await loadConfig(path17.join(cwd, "_"), repoRoot);
+  const yamlConfig = await loadConfig(path15.join(cwd, "_"), repoRoot);
   if (yamlConfig?.required_version) {
     await enforceRequiredVersion(yamlConfig.required_version, {
       strict: normalizeBoolean(input.rawOptions.strict)
@@ -5552,7 +5175,7 @@ async function runEvalCommand(input) {
   }
   let retryNonErrorResults;
   if (options.retryErrors) {
-    const retryPath = path17.resolve(options.retryErrors);
+    const retryPath = path15.resolve(options.retryErrors);
     await ensureFileExists(retryPath, "Retry-errors JSONL file");
     const completedIds = await loadFullyCompletedTestIds(retryPath);
     const errorIds = await loadErrorTestIds(retryPath);
@@ -5565,8 +5188,37 @@ async function runEvalCommand(input) {
       console.log(`Skipping ${completedIds.length} already-completed test(s).`);
     }
   }
+  let resumeSkipKeys;
+  let isResumeAppend = false;
+  if (options.resume && !options.retryErrors) {
+    const explicitResumeDir = options.outputDir ?? options.artifacts;
+    if (explicitResumeDir) {
+      const resumeIndexPath = path15.join(path15.resolve(explicitResumeDir), "index.jsonl");
+      if (existsSync2(resumeIndexPath)) {
+        const content = await readFile8(resumeIndexPath, "utf8");
+        const existingResults = parseJsonlResults(content);
+        resumeSkipKeys = /* @__PURE__ */ new Set();
+        for (const r of existingResults) {
+          if (shouldSkipExistingResultForResume(r, options.rerunFailed)) {
+            resumeSkipKeys.add(buildTestTargetKey(r.testId, r.target));
+          }
+        }
+        isResumeAppend = true;
+        const modeLabel = options.rerunFailed ? "Rerun-failed" : "Resume";
+        console.log(
+          `${modeLabel}: found ${existingResults.length} existing result(s), skipping ${resumeSkipKeys.size} completed.`
+        );
+      } else {
+        console.log("Resume: no existing index.jsonl found, starting fresh run.");
+      }
+    } else {
+      console.warn(
+        "Warning: --resume requires --output <dir> to identify the run directory. Ignoring --resume."
+      );
+    }
+  }
   if (options.workspacePath) {
-    const resolvedWorkspace = path17.resolve(options.workspacePath);
+    const resolvedWorkspace = path15.resolve(options.workspacePath);
     try {
       const { stat: stat2 } = await import("node:fs/promises");
       const stats = await stat2(resolvedWorkspace);
@@ -5607,25 +5259,25 @@ async function runEvalCommand(input) {
   let outputPath;
   let usesDefaultArtifactWorkspace;
   if (explicitDir) {
-    runDir = path17.resolve(explicitDir);
+    runDir = path15.resolve(explicitDir);
     mkdirSync(runDir, { recursive: true });
-    outputPath = path17.join(runDir, "index.jsonl");
+    outputPath = path15.join(runDir, "index.jsonl");
     usesDefaultArtifactWorkspace = true;
   } else if (options.outPath) {
-    outputPath = path17.resolve(options.outPath);
-    runDir = path17.dirname(outputPath);
+    outputPath = path15.resolve(options.outPath);
+    runDir = path15.dirname(outputPath);
     mkdirSync(runDir, { recursive: true });
     usesDefaultArtifactWorkspace = false;
   } else {
     outputPath = buildDefaultOutputPathForExperiment(cwd, options.experiment);
-    runDir = path17.dirname(outputPath);
+    runDir = path15.dirname(outputPath);
     usesDefaultArtifactWorkspace = true;
   }
   let otelExporter = null;
   const useFileExport = !!options.otelFile;
   if (options.exportOtel || useFileExport) {
     try {
-      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-U5EXNMON.js");
+      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-RTIUSC6L.js");
       let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
       let headers = {};
       if (options.otelBackend) {
@@ -5649,7 +5301,7 @@ async function runEvalCommand(input) {
         headers,
         captureContent,
         groupTurns: options.otelGroupTurns,
-        otlpFilePath: options.otelFile ? path17.resolve(options.otelFile) : void 0
+        otlpFilePath: options.otelFile ? path15.resolve(options.otelFile) : void 0
       });
       const initialized = await otelExporter.init();
       if (!initialized) {
@@ -5666,7 +5318,7 @@ async function runEvalCommand(input) {
     }
   }
   const primaryWritePath = outputPath;
-  const resolvedExportPaths = options.exportPaths.map((p) => path17.resolve(p));
+  const resolvedExportPaths = options.exportPaths.map((p) => path15.resolve(p));
   console.log(`Artifact directory: ${runDir}`);
   if (resolvedExportPaths.length > 0) {
     console.log("Export files:");
@@ -5674,9 +5326,9 @@ async function runEvalCommand(input) {
       console.log(`  ${p}`);
     }
   }
-  const resolvedTestFiles = input.testFiles.map((file) => path17.resolve(file));
+  const resolvedTestFiles = input.testFiles.map((file) => path15.resolve(file));
   if (options.otelFile) {
-    console.log(`OTLP JSON file: ${path17.resolve(options.otelFile)}`);
+    console.log(`OTLP JSON file: ${path15.resolve(options.otelFile)}`);
   }
   const evaluationRunner = await resolveEvaluationRunner();
   const allResults = [];
@@ -5685,23 +5337,7 @@ async function runEvalCommand(input) {
   const displayIdTracker = createDisplayIdTracker();
   const perFileWorkers = options.workers;
   const fileMetadata = /* @__PURE__ */ new Map();
-  const tsFiles = [];
-  const yamlFiles = [];
   for (const testFilePath of resolvedTestFiles) {
-    if (/\.(ts|js|mts|mjs)$/.test(testFilePath)) {
-      tsFiles.push(testFilePath);
-    } else {
-      yamlFiles.push(testFilePath);
-    }
-  }
-  for (const tsFile of tsFiles) {
-    await ensureFileExists(tsFile, "TypeScript eval file");
-    await import(pathToFileURL(tsFile).href);
-  }
-  if (yamlFiles.length === 0 && tsFiles.length > 0) {
-    return;
-  }
-  for (const testFilePath of yamlFiles) {
     const meta = await prepareFileMetadata({
       testFilePath,
       repoRoot,
@@ -5716,7 +5352,7 @@ async function runEvalCommand(input) {
     for (const [testFilePath, meta] of fileMetadata.entries()) {
       if (!matchesTagFilters(meta.tags, options.tags, options.excludeTags)) {
         fileMetadata.delete(testFilePath);
-        skippedFiles.push(path17.relative(cwd, testFilePath));
+        skippedFiles.push(path15.relative(cwd, testFilePath));
       }
     }
     if (skippedFiles.length > 0 && options.verbose) {
@@ -5737,7 +5373,7 @@ async function runEvalCommand(input) {
     cliNoCache: options.noCache,
     yamlCache: yamlCacheEnabled
   });
-  const cache = cacheEnabled ? new ResponseCache(yamlCachePath ? path17.resolve(yamlCachePath) : void 0) : void 0;
+  const cache = cacheEnabled ? new ResponseCache(yamlCachePath ? path15.resolve(yamlCachePath) : void 0) : void 0;
   if (cacheEnabled) {
     console.log(`Response cache: enabled${yamlCachePath ? ` (${yamlCachePath})` : ""}`);
   }
@@ -5747,14 +5383,25 @@ async function runEvalCommand(input) {
     throw new Error("--threshold must be between 0 and 1");
   }
   const writerOptions = resolvedThreshold !== void 0 ? { threshold: resolvedThreshold } : void 0;
-  const outputWriter = await createOutputWriter(primaryWritePath, "jsonl");
+  const outputWriter = await createOutputWriter(primaryWritePath, "jsonl", {
+    append: isResumeAppend
+  });
   const isMatrixMode = Array.from(fileMetadata.values()).some((meta) => meta.selections.length > 1);
   let totalEvalCount = 0;
+  let resumeSkippedCount = 0;
   for (const meta of fileMetadata.values()) {
     const suiteTargetNames = meta.selections.map((s) => s.selection.targetName);
     for (const test of meta.testCases) {
       const testTargetNames = test.targets && test.targets.length > 0 ? test.targets.filter((t) => suiteTargetNames.includes(t)) : suiteTargetNames;
-      totalEvalCount += testTargetNames.length > 0 ? testTargetNames.length : 1;
+      const effectiveTargets = testTargetNames.length > 0 ? testTargetNames : ["unknown"];
+      for (const tn of effectiveTargets) {
+        const key = `${test.id}::${tn}`;
+        if (resumeSkipKeys?.has(key)) {
+          resumeSkippedCount++;
+        } else {
+          totalEvalCount++;
+        }
+      }
     }
   }
   if (totalEvalCount === 0) {
@@ -5762,6 +5409,10 @@ async function runEvalCommand(input) {
       console.log("No execution errors or missing cases in the previous run. Nothing to retry.");
       return;
     }
+    if (resumeSkipKeys && resumeSkippedCount > 0) {
+      console.log(`Nothing to resume \u2014 all ${resumeSkippedCount} test(s) already completed.`);
+      return;
+    }
     throw new Error("No tests matched the provided filters.");
   }
   const progressReporter = createProgressReporter(options.workers ?? DEFAULT_WORKERS, {
@@ -5821,7 +5472,7 @@ async function runEvalCommand(input) {
   const activeTestFiles = resolvedTestFiles.filter((f) => fileMetadata.has(f));
   let transcriptProviderFactory;
   if (options.transcript) {
-    const { TranscriptProvider } = await import("./dist-U5EXNMON.js");
+    const { TranscriptProvider } = await import("./dist-RTIUSC6L.js");
     const transcriptProvider = await TranscriptProvider.fromFile(options.transcript);
     const totalTests = [...fileMetadata.values()].reduce(
       (sum, meta) => sum + meta.testCases.length,
@@ -5852,7 +5503,10 @@ async function runEvalCommand(input) {
             }
             return true;
           }) : targetPrep.testCases;
-          if (applicableTestCases.length === 0) {
+          const filteredTestCases = resumeSkipKeys ? applicableTestCases.filter(
+            (test) => !resumeSkipKeys.has(buildTestTargetKey(test.id, targetName))
+          ) : applicableTestCases;
+          if (filteredTestCases.length === 0) {
             return [];
           }
           try {
@@ -5872,15 +5526,15 @@ async function runEvalCommand(input) {
               displayIdTracker,
               selection,
               inlineTargetLabel,
-              testCases: applicableTestCases,
+              testCases: filteredTestCases,
               trialsConfig: options.transcript ? void 0 : targetPrep.trialsConfig,
               matrixMode: targetPrep.selections.length > 1,
-              totalBudgetUsd: targetPrep.totalBudgetUsd,
+              budgetUsd: targetPrep.budgetUsd,
               failOnError: targetPrep.failOnError,
               threshold: resolvedThreshold,
-              providerFactory: transcriptProviderFactory
+              providerFactory: transcriptProviderFactory ?? targetPrep.providerFactory
             });
-            const evalFile = path17.relative(cwd, testFilePath);
+            const evalFile = path15.relative(cwd, testFilePath);
             const existingSummary = remoteEvalSummaries.find(
               (summary2) => summary2.evalFile === evalFile
             );
@@ -5897,10 +5551,10 @@ async function runEvalCommand(input) {
             const message = fileError instanceof Error ? fileError.message : String(fileError);
             console.error(
               `
-[ERROR] \u26A0 Eval file failed: ${path17.basename(testFilePath)} \u2014 ${message}
+[ERROR] \u26A0 Eval file failed: ${path15.basename(testFilePath)} \u2014 ${message}
 `
             );
-            const errorResults = applicableTestCases.map((testCase) => ({
+            const errorResults = filteredTestCases.map((testCase) => ({
               timestamp: (/* @__PURE__ */ new Date()).toISOString(),
               testId: testCase.id,
               score: 0,
@@ -5936,37 +5590,60 @@ async function runEvalCommand(input) {
         `Merged ${retryNonErrorResults.length} non-error result(s) from previous output.`
       );
     }
+    await outputWriter.close().catch(() => void 0);
+    let summaryResults = allResults;
+    if (isResumeAppend && usesDefaultArtifactWorkspace) {
+      const content = await readFile8(outputPath, "utf8");
+      summaryResults = deduplicateByTestIdTarget(parseJsonlResults(content));
+    }
     const thresholdOpts = resolvedThreshold !== void 0 ? { threshold: resolvedThreshold } : void 0;
-    const summary = calculateEvaluationSummary(allResults, thresholdOpts);
+    const summary = calculateEvaluationSummary(summaryResults, thresholdOpts);
     console.log(formatEvaluationSummary(summary, thresholdOpts));
     const allExecutionErrors = summary.total > 0 && summary.executionErrorCount === summary.total;
     const thresholdFailed = resolvedThreshold !== void 0 && summary.qualityFailureCount > 0;
-    if (isMatrixMode && allResults.length > 0) {
-      console.log(formatMatrixSummary(allResults));
+    if (isMatrixMode && summaryResults.length > 0) {
+      console.log(formatMatrixSummary(summaryResults));
     }
     if (options.benchmarkJson && allResults.length > 0) {
-      const benchmarkPath = path17.resolve(options.benchmarkJson);
+      const benchmarkPath = path15.resolve(options.benchmarkJson);
       await writeBenchmarkJson(benchmarkPath, allResults);
       console.log(`Benchmark written to: ${benchmarkPath}`);
     }
     if (usesDefaultArtifactWorkspace && allResults.length > 0) {
       const evalFile = activeTestFiles.length === 1 ? activeTestFiles[0] : "";
-      const {
-        testArtifactDir,
-        timingPath,
-        benchmarkPath: workspaceBenchmarkPath,
-        indexPath
-      } = await writeArtifactsFromResults(allResults, runDir, {
-        evalFile,
-        experiment: normalizeExperimentName(options.experiment)
-      });
-      console.log(`Artifact workspace written to: ${runDir}`);
-      console.log(`  Index: ${indexPath}`);
-      console.log(
-        `  Per-test artifacts: ${testArtifactDir} (${allResults.length} test directories)`
-      );
-      console.log(`  Timing: ${timingPath}`);
-      console.log(`  Benchmark: ${workspaceBenchmarkPath}`);
+      if (isResumeAppend) {
+        const { writePerTestArtifacts } = await import("./artifact-writer-YATMDPWI.js");
+        await writePerTestArtifacts(allResults, runDir, {
+          experiment: normalizeExperimentName(options.experiment)
+        });
+        const { benchmarkPath: workspaceBenchmarkPath, timingPath } = await aggregateRunDir(
+          runDir,
+          { evalFile, experiment: normalizeExperimentName(options.experiment) }
+        );
+        const indexPath = path15.join(runDir, "index.jsonl");
+        console.log(`Artifact workspace updated: ${runDir}`);
+        console.log(`  Index: ${indexPath}`);
+        console.log(`  Per-test artifacts: ${runDir} (${allResults.length} new test directories)`);
+        console.log(`  Timing: ${timingPath}`);
+        console.log(`  Benchmark: ${workspaceBenchmarkPath}`);
+      } else {
+        const {
+          testArtifactDir,
+          timingPath,
+          benchmarkPath: workspaceBenchmarkPath,
+          indexPath
+        } = await writeArtifactsFromResults(allResults, runDir, {
+          evalFile,
+          experiment: normalizeExperimentName(options.experiment)
+        });
+        console.log(`Artifact workspace written to: ${runDir}`);
+        console.log(`  Index: ${indexPath}`);
+        console.log(
+          `  Per-test artifacts: ${testArtifactDir} (${allResults.length} test directories)`
+        );
+        console.log(`  Timing: ${timingPath}`);
+        console.log(`  Benchmark: ${workspaceBenchmarkPath}`);
+      }
     }
     if (resolvedExportPaths.length > 0 && allResults.length > 0) {
       for (const exportPath of resolvedExportPaths) {
@@ -5977,7 +5654,7 @@ async function runEvalCommand(input) {
         await writer.close();
       }
       console.log(
-        `Export file(s) written: ${resolvedExportPaths.map((p) => path17.relative(cwd, p)).join(", ")}`
+        `Export file(s) written: ${resolvedExportPaths.map((p) => path15.relative(cwd, p)).join(", ")}`
       );
     }
     const resultsWithWorkspaces = allResults.filter((r) => r.workspacePath);
@@ -6015,14 +5692,14 @@ Results written to: ${outputPath}`);
         experiment: normalizeExperimentName(options.experiment)
       });
     }
-    if (summary.executionErrorCount > 0 && !options.retryErrors) {
-      const evalFileArgs = activeTestFiles.map((f) => path17.relative(cwd, f)).join(" ");
+    if (summary.executionErrorCount > 0 && !options.retryErrors && !options.resume) {
+      const evalFileArgs = activeTestFiles.map((f) => path15.relative(cwd, f)).join(" ");
       const targetFlag = options.target ? ` --target ${options.target}` : "";
-      const relativeOutputPath = path17.relative(cwd, outputPath);
+      const relativeRunDir = path15.relative(cwd, runDir);
       console.log(
         `
 Tip: ${summary.executionErrorCount} execution error(s) detected. Re-run failed tests with:
-  agentv eval run ${evalFileArgs}${targetFlag} --retry-errors ${relativeOutputPath}`
+  agentv eval run ${evalFileArgs}${targetFlag} --output ${relativeRunDir} --rerun-failed`
       );
     }
     return {
@@ -6052,7 +5729,7 @@ async function resolveEvaluationRunner() {
   if (!overridePath) {
     return runEvaluation;
   }
-  const resolved = path17.isAbsolute(overridePath) ? overridePath : path17.resolve(process.cwd(), overridePath);
+  const resolved = path15.isAbsolute(overridePath) ? overridePath : path15.resolve(process.cwd(), overridePath);
   const moduleUrl = pathToFileURL(resolved).href;
   const mod = await import(moduleUrl);
   const candidate = mod.runEvaluation;
@@ -6065,11 +5742,11 @@ async function resolveEvaluationRunner() {
 }
 
 // src/commands/eval/discover.ts
-import path18 from "node:path";
+import path16 from "node:path";
 import fg2 from "fast-glob";
 async function discoverEvalFiles(cwd) {
   const repoRoot = await findRepoRoot(cwd);
-  const config = await loadConfig(path18.join(cwd, "_"), repoRoot);
+  const config = await loadConfig(path16.join(cwd, "_"), repoRoot);
   const patterns = config?.eval_patterns && config.eval_patterns.length > 0 ? config.eval_patterns : DEFAULT_EVAL_PATTERNS;
   const ignore = ["**/node_modules/**", "**/dist/**"];
   const matches = await fg2(patterns, {
@@ -6081,7 +5758,7 @@ async function discoverEvalFiles(cwd) {
     caseSensitiveMatch: false
   });
   const evalFiles = matches.map((absPath) => {
-    const relativePath = path18.relative(cwd, absPath);
+    const relativePath = path16.relative(cwd, absPath);
     const category = deriveCategory(relativePath);
     return { path: absPath, relativePath, category };
   });
@@ -6103,10 +5780,6 @@ function filterByCategory(files, category) {
 
 export {
   package_default,
-  toSnakeCaseDeep,
-  RESULT_INDEX_FILENAME,
-  buildDefaultRunDir,
-  resolveRunManifestPath,
   parseResultManifest,
   resolveResultSourcePath,
   loadManifestResults,
@@ -6132,7 +5805,6 @@ export {
   listMergedResultFiles,
   findRunById,
   maybeAutoExportRunArtifacts,
-  writeArtifactsFromResults,
   resolveRunCacheFile,
   loadRunCache,
   detectFileType,
@@ -6150,4 +5822,4 @@ export {
   getCategories,
   filterByCategory
 };
-//# sourceMappingURL=chunk-ZUNYOUFO.js.map
+//# sourceMappingURL=chunk-YGNBTBXW.js.map