agentv 4.44.0-next.1 → 5.0.0-next.1

package/dist/{chunk-CXHYTYHU.js → chunk-IBI7A5RP.js}

@@ -2,16 +2,21 @@ import { createRequire } from 'node:module'; const require = createRequire(impor
 import {
   toSnakeCaseDeep as toSnakeCaseDeep2,
   writeArtifactsFromResults
-} from "./chunk-KAGZRUO6.js";
+} from "./chunk-6JVM4GJI.js";
 import {
   RunBudgetTracker,
+  buildExperimentArtifactMetadata,
   buildWipBranchName,
+  confirmResultsMergeAndPull,
   deleteWipBranch,
   deriveCategory,
+  deriveExperimentNameFromPath,
   directPushResultsWithDetails,
   directorySizeBytes,
   getResultsRepoSyncStatus,
+  isExperimentFileReference,
   listGitRuns,
+  loadExperimentConfig,
   loadTsConfig,
   materializeGitRun,
   normalizeResultsConfig,
@@ -19,8 +24,10 @@ import {
   resolveResultsRepoRunsDir,
   setupWipWorktree,
   syncResultsRepoForProject
-} from "./chunk-NCNDA3F6.js";
+} from "./chunk-UG4JHZYM.js";
 import {
+  AGENTV_LOCAL_CONFIG_FILE_NAME,
+  AGENTV_LOCAL_CONFIG_YML_FILE_NAME,
   CLI_PLACEHOLDERS,
   COMMON_TARGET_SETTINGS,
   DEFAULT_EVAL_PATTERNS,
@@ -32,27 +39,33 @@ import {
   aggregateRunDir,
   buildDirectoryChain,
   buildSearchRoots,
+  buildSearchRoots2,
   buildTestTargetKey,
   buildTraceFromMessages,
   deduplicateByTestIdTarget,
   ensureVSCodeSubagents,
+  external_exports,
   findDeprecatedCamelCaseTargetWarnings,
   findGitRoot,
   getAgentvConfigDir,
+  getLocalConfigPath,
   getProject,
   getProjectForPath,
   interpolateEnv,
+  isAgentVConfigFileName,
   isGraderKind,
   listTargetNames,
   loadCasesFromDirectory,
   loadCasesFromFile,
   loadConfig,
+  loadPromptMdFallback,
   loadTestSuite,
   normalizeResultRow,
   parseJsonlResults,
   parseYamlValue,
   readTargetDefinitions,
   readTestSuiteMetadata,
+  resolveDefaultExperimentReference,
   resolveFileReference,
   resolveResultsConfigForProject,
   resolveTargetDefinition,
@@ -66,8 +79,8 @@ import {
   toCamelCaseDeep,
   toSnakeCaseDeep,
   traceFromTranscriptJsonLines,
-  writeInitialBenchmarkArtifact
-} from "./chunk-5DYGN547.js";
+  writeInitialRunSummaryArtifact
+} from "./chunk-GZJMODLW.js";
 
 // src/commands/eval/shared.ts
 import { constants } from "node:fs";
@@ -183,7 +196,7 @@ async function findRepoRoot(start) {
 // package.json
 var package_default = {
   name: "agentv",
-  version: "4.44.0-next.1",
+  version: "5.0.0-next.1",
   description: "CLI entry point for AgentV",
   type: "module",
   repository: {
@@ -291,10 +304,49 @@ async function discoverTargetsFile(options) {
   throw new Error("Unable to locate targets.yaml. Use --targets to specify the file explicitly.");
 }
 
+// src/commands/eval/discover.ts
+import path3 from "node:path";
+import fg2 from "fast-glob";
+async function discoverEvalFiles(cwd) {
+  const repoRoot = await findRepoRoot(cwd);
+  const config = await loadConfig(path3.join(cwd, "_"), repoRoot);
+  const patterns = config?.eval_patterns && config.eval_patterns.length > 0 ? config.eval_patterns : DEFAULT_EVAL_PATTERNS;
+  const ignore = ["**/node_modules/**", "**/dist/**"];
+  const matches = await fg2(patterns, {
+    cwd,
+    absolute: true,
+    onlyFiles: true,
+    ignore,
+    followSymbolicLinks: true,
+    caseSensitiveMatch: false
+  });
+  const evalFiles = matches.map((absPath) => {
+    const relativePath = path3.relative(cwd, absPath);
+    const category = deriveCategory(relativePath);
+    return { path: absPath, relativePath, category };
+  });
+  evalFiles.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
+  return evalFiles;
+}
+function getCategories(files) {
+  const categories = /* @__PURE__ */ new Set();
+  for (const file of files) {
+    categories.add(file.category);
+  }
+  const sorted = Array.from(categories);
+  sorted.sort();
+  return sorted;
+}
+function filterByCategory(files, category) {
+  return files.filter((f) => f.category === category);
+}
+
 // src/commands/eval/run-eval.ts
+import { spawn } from "node:child_process";
 import { constants as constants4, existsSync as existsSync7, mkdirSync as mkdirSync2 } from "node:fs";
 import { access as access6, readFile as readFile8 } from "node:fs/promises";
-import path14 from "node:path";
+import { createRequire as createNodeRequire } from "node:module";
+import path15 from "node:path";
 import { pathToFileURL as pathToFileURL2 } from "node:url";
 
 // src/version-check.ts
@@ -350,11 +402,11 @@ function enforceRequiredVersion(requiredVersion, options) {
 
 // src/commands/results/remote.ts
 import { existsSync as existsSync5 } from "node:fs";
-import path8 from "node:path";
+import path9 from "node:path";
 
 // src/commands/eval/result-layout.ts
 import { existsSync, statSync } from "node:fs";
-import path3 from "node:path";
+import path4 from "node:path";
 var RESULT_INDEX_FILENAME = "index.jsonl";
 var RESULTS_DIRNAME = "results";
 var DEFAULT_EXPERIMENT_NAME = "default";
@@ -385,19 +437,19 @@ function defaultRunPathSegments(experiment, runDirName) {
   return [normalizedExperiment, runDirName];
 }
 function buildResultsRootDir(cwd) {
-  return path3.join(cwd, ".agentv", RESULTS_DIRNAME);
+  return path4.join(cwd, ".agentv", RESULTS_DIRNAME);
 }
 function buildDefaultRunDirFromName(cwd, experiment, runDirName) {
-  return path3.join(buildResultsRootDir(cwd), ...defaultRunPathSegments(experiment, runDirName));
+  return path4.join(buildResultsRootDir(cwd), ...defaultRunPathSegments(experiment, runDirName));
 }
 function buildDefaultRunDir(cwd, experiment, timestamp = /* @__PURE__ */ new Date()) {
   return buildDefaultRunDirFromName(cwd, experiment, createRunDirName(timestamp));
 }
 function resolveRunIndexPath(runDir) {
-  return path3.join(runDir, RESULT_INDEX_FILENAME);
+  return path4.join(runDir, RESULT_INDEX_FILENAME);
 }
 function isRunManifestPath(filePath) {
-  return path3.basename(filePath) === RESULT_INDEX_FILENAME;
+  return path4.basename(filePath) === RESULT_INDEX_FILENAME;
 }
 function resolveExistingRunPrimaryPath(runDir) {
   const indexPath = resolveRunIndexPath(runDir);
@@ -435,24 +487,24 @@ function resolveRunManifestPath(filePath) {
   return filePath;
 }
 function relativeRunPathFromCwd(cwd, runDir) {
-  const relative = path3.relative(path3.resolve(buildResultsRootDir(cwd)), path3.resolve(runDir));
-  if (!relative || relative.startsWith("..") || path3.isAbsolute(relative)) {
+  const relative = path4.relative(path4.resolve(buildResultsRootDir(cwd)), path4.resolve(runDir));
+  if (!relative || relative.startsWith("..") || path4.isAbsolute(relative)) {
     return void 0;
   }
-  const parts = relative.split(path3.sep).filter(Boolean);
+  const parts = relative.split(path4.sep).filter(Boolean);
   if (parts.length < 2 || isReservedResultsNamespace(parts[0])) {
     return void 0;
   }
-  return parts.join(path3.posix.sep);
+  return parts.join(path4.posix.sep);
 }
 
 // src/commands/inspect/utils.ts
 import { readFileSync as readFileSync2, readdirSync, statSync as statSync2 } from "node:fs";
-import path5 from "node:path";
+import path6 from "node:path";
 
 // src/commands/results/manifest.ts
 import { existsSync as existsSync2, readFileSync } from "node:fs";
-import path4 from "node:path";
+import path5 from "node:path";
 function parseJsonlLines(content) {
   return content.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
 }
@@ -479,7 +531,7 @@ function readOptionalText(baseDir, relativePath) {
   if (!relativePath) {
     return void 0;
   }
-  const absolutePath = path4.join(baseDir, relativePath);
+  const absolutePath = path5.join(baseDir, relativePath);
   if (!existsSync2(absolutePath)) {
     return void 0;
   }
@@ -601,8 +653,8 @@ function parseResultManifest(content) {
   return parseResultRows(content);
 }
 function resolveResultSourcePath(source, cwd) {
-  const resolved = path4.isAbsolute(source) ? source : path4.resolve(cwd ?? process.cwd(), source);
-  if (isDirectoryPath(resolved) || path4.basename(resolved) === RESULT_INDEX_FILENAME) {
+  const resolved = path5.isAbsolute(source) ? source : path5.resolve(cwd ?? process.cwd(), source);
+  if (isDirectoryPath(resolved) || path5.basename(resolved) === RESULT_INDEX_FILENAME) {
     return resolveRunManifestPath(resolved);
   }
   return resolved;
@@ -611,7 +663,7 @@ function loadManifestResults(sourceFile, options = {}) {
   const resolvedSourceFile = resolveRunManifestPath(sourceFile);
   const content = readFileSync(resolvedSourceFile, "utf8");
   const records = parseResultRows(content, resolvedSourceFile);
-  const baseDir = path4.dirname(resolvedSourceFile);
+  const baseDir = path5.dirname(resolvedSourceFile);
   return records.map((record) => hydrateManifestRecord(baseDir, record, options));
 }
 function loadLightweightResults(sourceFile) {
@@ -659,10 +711,10 @@ function padLeft(str, len) {
 }
 function loadResultFile(filePath) {
   const resolvedFilePath = resolveTraceResultPath(filePath);
-  if (path5.extname(resolvedFilePath) === ".json") {
+  if (path6.extname(resolvedFilePath) === ".json") {
     return loadOtlpTraceFile(resolvedFilePath);
   }
-  if (path5.basename(resolvedFilePath) === RESULT_INDEX_FILENAME) {
+  if (path6.basename(resolvedFilePath) === RESULT_INDEX_FILENAME) {
     return loadManifestAsRawResults(resolvedFilePath);
   }
   return loadJsonlRecords(resolvedFilePath);
@@ -960,7 +1012,7 @@ function toTraceSummary(result) {
   return toCamelCaseDeep(rawTrace);
 }
 function buildRunId(relativeRunPath) {
-  const normalized = relativeRunPath.split(path5.sep).join("/");
+  const normalized = relativeRunPath.split(path6.sep).join("/");
   const segments = normalized.split("/").filter(Boolean);
   if (segments.length >= 2) {
     const experiment = segments.slice(0, -1).join("/");
@@ -974,8 +1026,8 @@ function buildRunId(relativeRunPath) {
 }
 function readRunDisplayName(runDir) {
   try {
-    const benchmark = JSON.parse(readFileSync2(path5.join(runDir, "benchmark.json"), "utf8"));
-    const displayName = benchmark.metadata?.display_name;
+    const summary = JSON.parse(readFileSync2(path6.join(runDir, "summary.json"), "utf8"));
+    const displayName = summary.metadata?.display_name;
     return typeof displayName === "string" && displayName.trim() ? displayName.trim() : void 0;
   } catch {
     return void 0;
@@ -984,8 +1036,8 @@ function readRunDisplayName(runDir) {
 function collectRunManifestPaths(runsDir, currentDir, files) {
   const primaryPath = resolveExistingRunPrimaryPath(currentDir);
   if (primaryPath) {
-    const relativeRunPath = path5.relative(runsDir, currentDir);
-    const sortName = path5.basename(currentDir);
+    const relativeRunPath = path6.relative(runsDir, currentDir);
+    const sortName = path6.basename(currentDir);
     files.push({
       filePath: primaryPath,
       displayName: readRunDisplayName(currentDir) ?? sortName,
@@ -997,7 +1049,7 @@ function collectRunManifestPaths(runsDir, currentDir, files) {
   const entries = readdirSync(currentDir, { withFileTypes: true });
   for (const entry of entries) {
     if (entry.isDirectory()) {
-      collectRunManifestPaths(runsDir, path5.join(currentDir, entry.name), files);
+      collectRunManifestPaths(runsDir, path6.join(currentDir, entry.name), files);
     }
   }
 }
@@ -1010,7 +1062,7 @@ function listResultFilesFromRoot(runsDir, options) {
         if (options?.skipTopLevelDirs?.has(entry.name) || isReservedResultsNamespace(entry.name)) {
           continue;
         }
-        collectRunManifestPaths(runsDir, path5.join(runsDir, entry.name), files);
+        collectRunManifestPaths(runsDir, path6.join(runsDir, entry.name), files);
       }
     }
   } catch {
@@ -1091,7 +1143,7 @@ function formatScore(score) {
 // src/commands/results/remote-metadata.ts
 import { execFileSync } from "node:child_process";
 import { existsSync as existsSync4, mkdirSync, readFileSync as readFileSync4, rmSync, writeFileSync as writeFileSync2 } from "node:fs";
-import path7 from "node:path";
+import path8 from "node:path";
 
 // src/commands/results/run-state.ts
 import { createHash } from "node:crypto";
@@ -1132,12 +1184,12 @@ function materializeRunState(input) {
 
 // src/commands/results/run-tags.ts
 import { existsSync as existsSync3, readFileSync as readFileSync3, unlinkSync, writeFileSync } from "node:fs";
-import path6 from "node:path";
+import path7 from "node:path";
 var RUN_TAGS_FILENAME = "tags.json";
 var MAX_TAGS_PER_RUN = 20;
 var MAX_TAG_LENGTH = 60;
 function runTagsPath(manifestPath) {
-  return path6.join(path6.dirname(manifestPath), RUN_TAGS_FILENAME);
+  return path7.join(path7.dirname(manifestPath), RUN_TAGS_FILENAME);
 }
 function readRunTags(manifestPath) {
   const fp = runTagsPath(manifestPath);
@@ -1202,7 +1254,7 @@ function normalizeTags(tags) {
 
 // src/commands/results/remote-metadata.ts
 var RESULTS_RUNS_DIR = "runs";
-var REMOTE_METADATA_RUNS_DIR = path7.join("metadata", "runs");
+var REMOTE_METADATA_RUNS_DIR = path8.join("metadata", "runs");
 function cleanGitEnv() {
   const env = {};
   for (const [key, value] of Object.entries(process.env)) {
@@ -1228,7 +1280,7 @@ function tryRunGit(repoDir, args) {
   }
 }
 function toGitPath(filePath) {
-  return filePath.split(path7.sep).join("/");
+  return filePath.split(path8.sep).join("/");
 }
 function readTagsFile(filePath) {
   if (!existsSync4(filePath)) return void 0;
@@ -1282,27 +1334,27 @@ function resolveComparisonRef(repoDir) {
   return tryRunGit(repoDir, ["rev-parse", "--verify", "HEAD"]) ? "HEAD" : void 0;
 }
 function resolveRemoteRunMetadataPaths(repoDir, manifestPath) {
-  const runsRoot = path7.resolve(repoDir, RESULTS_RUNS_DIR);
-  const manifestDir = path7.resolve(path7.dirname(manifestPath));
-  const runRelativePath = path7.relative(runsRoot, manifestDir);
-  if (runRelativePath.length === 0 || runRelativePath.startsWith("..") || path7.isAbsolute(runRelativePath)) {
+  const runsRoot = path8.resolve(repoDir, RESULTS_RUNS_DIR);
+  const manifestDir = path8.resolve(path8.dirname(manifestPath));
+  const runRelativePath = path8.relative(runsRoot, manifestDir);
+  if (runRelativePath.length === 0 || runRelativePath.startsWith("..") || path8.isAbsolute(runRelativePath)) {
     throw new Error(
       `Remote run manifest is outside the results repo runs directory: ${manifestPath}`
     );
   }
-  const overlayTagsPath = path7.join(
+  const overlayTagsPath = path8.join(
     repoDir,
     REMOTE_METADATA_RUNS_DIR,
     runRelativePath,
     RUN_TAGS_FILENAME
   );
-  const artifactTagsPath = path7.join(runsRoot, runRelativePath, RUN_TAGS_FILENAME);
+  const artifactTagsPath = path8.join(runsRoot, runRelativePath, RUN_TAGS_FILENAME);
   return {
     runRelativePath,
     artifactTagsPath,
-    artifactTagsGitPath: toGitPath(path7.relative(repoDir, artifactTagsPath)),
+    artifactTagsGitPath: toGitPath(path8.relative(repoDir, artifactTagsPath)),
     overlayTagsPath,
-    overlayTagsGitPath: toGitPath(path7.relative(repoDir, overlayTagsPath))
+    overlayTagsGitPath: toGitPath(path8.relative(repoDir, overlayTagsPath))
   };
 }
 function readRemoteRunTagsContext(repoDir, manifestPath, comparisonRef = resolveComparisonRef(repoDir)) {
@@ -1363,7 +1415,7 @@ function writeRemoteRunTags(repoDir, manifestPath, tags, comparisonRef, expected
     updated_at: updatedAt,
     tag_revision: createTagRevision(cleaned, updatedAt)
   };
-  mkdirSync(path7.dirname(context.paths.overlayTagsPath), { recursive: true });
+  mkdirSync(path8.dirname(context.paths.overlayTagsPath), { recursive: true });
   writeFileSync2(context.paths.overlayTagsPath, `${JSON.stringify(entry, null, 2)}
 `, "utf8");
   return readRemoteRunTags(repoDir, manifestPath, comparisonRef);
@@ -1407,7 +1459,7 @@ function invalidateGitRunsCache(repoDir) {
   }
 }
 function relativeLocalRunPath(cwd, manifestPath) {
-  const manifestDir = path8.resolve(path8.dirname(manifestPath));
+  const manifestDir = path9.resolve(path9.dirname(manifestPath));
   return relativeRunPathFromCwd(cwd, manifestDir);
 }
 function remoteMetadataManifestPath(cwd, config, meta) {
@@ -1421,7 +1473,7 @@ function remoteMetadataManifestPath(cwd, config, meta) {
   if (!relativeRunPath) {
     return void 0;
   }
-  return path8.join(config.path, "runs", ...relativeRunPath.split("/"), "index.jsonl");
+  return path9.join(config.path, "runs", ...relativeRunPath.split("/"), "index.jsonl");
 }
 var REMOTE_RUN_PREFIX = "remote::";
 var SIZE_WARNING_BYTES = 10 * 1024 * 1024;
@@ -1434,7 +1486,7 @@ function getRelativeRunPath(cwd, runDir) {
     return relative;
   }
   throw new Error(
-    `Run workspace must use .agentv/results/<experiment>/<timestamp>: ${path8.resolve(runDir)}`
+    `Run workspace must use .agentv/results/<experiment>/<timestamp>: ${path9.resolve(runDir)}`
   );
 }
 function buildCommitTitle(payload) {
@@ -1453,7 +1505,7 @@ async function maybeWarnLargeArtifact(runDir) {
 }
 async function loadNormalizedResultsConfig(cwd, projectId, overrides) {
   const repoRoot = await findRepoRoot(cwd) ?? cwd;
-  const config = await loadConfig(path8.join(cwd, "_"), repoRoot);
+  const config = await loadConfig(path9.join(cwd, "_"), repoRoot);
   const project = projectId !== void 0 ? getProject(projectId) : getProjectForPath(repoRoot) ?? getProjectForPath(cwd);
   const projectResults = project?.results ? {
     mode: "github",
@@ -1571,6 +1623,32 @@ async function syncRemoteResults(cwd, projectId) {
     };
   }
 }
+async function confirmRemoteResultsMerge(cwd, projectId) {
+  const config = await loadNormalizedResultsConfig(cwd, projectId);
+  if (!config) {
+    return {
+      ...await getResultsRepoSyncStatus(),
+      run_count: 0
+    };
+  }
+  try {
+    const status = await confirmResultsMergeAndPull(config);
+    invalidateGitRunsCache(config.path);
+    return {
+      ...status,
+      run_count: await getRemoteRunCount(config, status)
+    };
+  } catch (error) {
+    const status = await getResultsRepoSyncStatus(config);
+    return {
+      ...status,
+      run_count: await getRemoteRunCount(config, status),
+      last_error: getStatusMessage(error),
+      blocked: true,
+      block_reason: getStatusMessage(error)
+    };
+  }
+}
 function dedupeSyncedRunCopies(runs) {
   const byRunId = /* @__PURE__ */ new Map();
   for (const run of runs) {
@@ -1613,7 +1691,7 @@ async function listMergedResultFiles(cwd, limit, projectId) {
         raw_filename: r.run_id,
         source: "remote",
         on_remote: true,
-        path: path8.join(config.path, r.manifest_path),
+        path: path9.join(config.path, r.manifest_path),
         displayName: r.display_name,
         timestamp: r.timestamp,
         testCount: r.test_count,
@@ -1668,11 +1746,11 @@ async function ensureRemoteRunAvailable(cwd, meta, projectId) {
   if (!config) {
     throw new Error("Remote results are not configured");
   }
-  const relativeManifestPath = path8.relative(config.path, meta.path).split(path8.sep).join("/");
+  const relativeManifestPath = path9.relative(config.path, meta.path).split(path9.sep).join("/");
   if (relativeManifestPath.length === 0 || relativeManifestPath === meta.path || relativeManifestPath.startsWith("../")) {
     throw new Error(`Remote manifest path is outside the results repo clone: ${meta.path}`);
   }
-  const relativeRunPath = path8.posix.relative("runs", path8.posix.dirname(relativeManifestPath));
+  const relativeRunPath = path9.posix.relative("runs", path9.posix.dirname(relativeManifestPath));
   await materializeGitRun(config.path, relativeRunPath, getResultsStorageRef(config));
 }
 async function readRemoteRunTagState(cwd, meta, projectId) {
@@ -1781,13 +1859,13 @@ async function maybeAutoExportRunArtifacts(payload) {
 // src/commands/eval/env.ts
 import { constants as constants3 } from "node:fs";
 import { access as access3 } from "node:fs/promises";
-import path9 from "node:path";
+import path10 from "node:path";
 import { config as loadDotenv } from "dotenv";
 function uniqueDirs(directories) {
   const seen = /* @__PURE__ */ new Set();
   const result = [];
   for (const dir of directories) {
-    const absolute = path9.resolve(dir);
+    const absolute = path10.resolve(dir);
     if (seen.has(absolute)) {
       continue;
     }
@@ -1806,14 +1884,14 @@ async function fileExists2(filePath) {
 }
 function collectAncestorDirectories(start, boundary) {
   const directories = [];
-  const boundaryDir = path9.resolve(boundary);
-  let current = path9.resolve(start);
+  const boundaryDir = path10.resolve(boundary);
+  let current = path10.resolve(start);
   while (current !== void 0) {
     directories.push(current);
     if (current === boundaryDir) {
       break;
     }
-    const parent = path9.dirname(current);
+    const parent = path10.dirname(current);
     if (parent === current) {
       break;
     }
@@ -1823,12 +1901,12 @@ function collectAncestorDirectories(start, boundary) {
 }
 async function loadEnvFromHierarchy(options) {
   const { testFilePath, repoRoot, verbose } = options;
-  const testDir = path9.dirname(path9.resolve(testFilePath));
+  const testDir = path10.dirname(path10.resolve(testFilePath));
   const cwd = process.cwd();
   const searchDirs = uniqueDirs([...collectAncestorDirectories(testDir, repoRoot), repoRoot, cwd]);
   const envFiles = [];
   for (const dir of searchDirs) {
-    const candidate = path9.join(dir, ".env");
+    const candidate = path10.join(dir, ".env");
     if (await fileExists2(candidate)) {
       envFiles.push(candidate);
     }
@@ -1851,7 +1929,7 @@ async function loadEnvFromHierarchy(options) {
 
 // src/commands/eval/otel-backends.ts
 import { access as access4 } from "node:fs/promises";
-import path10 from "node:path";
+import path11 from "node:path";
 import { pathToFileURL } from "node:url";
 var RESOLVER_EXTENSIONS = [".mjs", ".js"];
 var builtinOtelBackendResolvers = [
@@ -1915,7 +1993,7 @@ async function findLocalOtelBackendResolver(name, cwd) {
   }
   for (const dir of getResolverSearchDirs(cwd)) {
     for (const ext of RESOLVER_EXTENSIONS) {
-      const candidate = path10.join(dir, `${name}${ext}`);
+      const candidate = path11.join(dir, `${name}${ext}`);
       try {
         await access4(candidate);
         return candidate;
@@ -1927,11 +2005,11 @@ async function findLocalOtelBackendResolver(name, cwd) {
 }
 function getResolverSearchDirs(cwd) {
   const dirs = [];
-  let current = path10.resolve(cwd);
-  const root = path10.parse(current).root;
+  let current = path11.resolve(cwd);
+  const root = path11.parse(current).root;
   while (current !== root) {
-    dirs.push(path10.join(current, ".agentv", "otel-backends"));
-    current = path10.dirname(current);
+    dirs.push(path11.join(current, ".agentv", "otel-backends"));
+    current = path11.dirname(current);
   }
   return dirs;
 }
@@ -1960,7 +2038,7 @@ function trimTrailingSlash(value) {
 // src/commands/eval/jsonl-writer.ts
 import { createWriteStream } from "node:fs";
 import { mkdir } from "node:fs/promises";
-import path11 from "node:path";
+import path12 from "node:path";
 import { finished } from "node:stream/promises";
 
 // ../../node_modules/.bun/async-mutex@0.5.0/node_modules/async-mutex/index.mjs
@@ -2178,7 +2256,7 @@ var JsonlWriter = class _JsonlWriter {
     this.stream = stream;
   }
   static async open(filePath, options) {
-    await mkdir(path11.dirname(filePath), { recursive: true });
+    await mkdir(path12.dirname(filePath), { recursive: true });
     const flags = options?.append ? "a" : "w";
     const stream = createWriteStream(filePath, { flags, encoding: "utf8" });
     return new _JsonlWriter(stream);
@@ -2363,7 +2441,7 @@ async function loadNonErrorResults(jsonlPath) {
 // src/commands/eval/run-cache.ts
 import { existsSync as existsSync6 } from "node:fs";
 import { mkdir as mkdir2, readFile, writeFile } from "node:fs/promises";
-import path12 from "node:path";
+import path13 from "node:path";
 var CACHE_FILENAME = "cache.json";
 function resolveRunCacheFile(cache) {
   if (cache.lastRunDir) {
@@ -2372,7 +2450,7 @@ function resolveRunCacheFile(cache) {
   return "";
 }
 function cachePath(cwd) {
-  return path12.join(cwd, ".agentv", CACHE_FILENAME);
+  return path13.join(cwd, ".agentv", CACHE_FILENAME);
 }
 async function loadRunCache(cwd) {
   try {
@@ -2389,13 +2467,13 @@ async function resolveCachedRunDir(cwd) {
   return cache.lastRunDir;
 }
 async function saveRunCache(cwd, resultPath) {
-  if (path12.basename(resultPath) !== RESULT_INDEX_FILENAME) {
+  if (path13.basename(resultPath) !== RESULT_INDEX_FILENAME) {
     return;
   }
-  const dir = path12.join(cwd, ".agentv");
+  const dir = path13.join(cwd, ".agentv");
   await mkdir2(dir, { recursive: true });
   const cache = {
-    lastRunDir: path12.dirname(resultPath),
+    lastRunDir: path13.dirname(resultPath),
     timestamp: (/* @__PURE__ */ new Date()).toISOString()
   };
   await writeFile(cachePath(cwd), `${JSON.stringify(cache, null, 2)}
@@ -2664,7 +2742,7 @@ function formatMatrixSummary(results) {
 
 // ../../packages/core/dist/evaluation/validation/index.js
 import { readFile as readFile2 } from "node:fs/promises";
-import path13 from "node:path";
+import path14 from "node:path";
 import { readFile as readFile22, readdir, stat as stat2 } from "node:fs/promises";
 import path22 from "node:path";
 import { readFile as readFile3 } from "node:fs/promises";
@@ -2710,10 +2788,13 @@ async function detectFileType(filePath) {
   }
 }
 function inferFileTypeFromPath(filePath) {
-  const normalized = path13.normalize(filePath).replace(/\\/g, "/");
-  const basename = path13.basename(filePath);
+  const normalized = path14.normalize(filePath).replace(/\\/g, "/");
+  const basename = path14.basename(filePath);
+  if (basename === AGENTV_LOCAL_CONFIG_FILE_NAME || basename === AGENTV_LOCAL_CONFIG_YML_FILE_NAME) {
+    return "config";
+  }
   if (normalized.includes("/.agentv/")) {
-    if (basename === "config.yaml" || basename === "config.yml") {
+    if (isAgentVConfigFileName(basename)) {
       return "config";
     }
     if (basename === "targets.yaml" || basename === "targets.yml") {
@@ -2726,6 +2807,62 @@ function inferFileTypeFromPath(filePath) {
   }
   return "unknown";
 }
+var JsonObjectSchema = external_exports.object({}).catchall(external_exports.unknown());
+var StringOrStringArraySchema = external_exports.union([external_exports.string().min(1), external_exports.array(external_exports.string().min(1)).min(1)]);
+var ExperimentScriptSchema = external_exports.union([
+  external_exports.string().min(1),
+  external_exports.object({
+    command: StringOrStringArraySchema.optional(),
+    script: StringOrStringArraySchema.optional(),
+    timeout_seconds: external_exports.number().gt(0).optional(),
+    cwd: external_exports.string().min(1).optional(),
+    env: external_exports.record(external_exports.string()).optional()
+  }).strict().refine((value) => value.command !== void 0 || value.script !== void 0, {
+    message: "Experiment step must define command or script."
+  })
+]);
+var ExperimentRepeatSchema = external_exports.object({
+  count: external_exports.number().int().min(1),
+  strategy: external_exports.enum(["pass_at_k", "mean", "confidence_interval"]).optional(),
+  cost_limit_usd: external_exports.number().min(0).optional(),
+  costLimitUsd: external_exports.number().min(0).optional()
+}).strict();
+var ExperimentTargetRefSchema = external_exports.union([
+  external_exports.string().min(1),
+  external_exports.object({
+    name: external_exports.string().min(1),
+    use_target: external_exports.string().min(1).optional(),
+    hooks: JsonObjectSchema.optional()
+  }).strict()
+]);
+var ExperimentSuiteSelectSchema = external_exports.object({
+  test_ids: external_exports.array(external_exports.string().min(1)).min(1)
+}).strict();
+var ExperimentSuiteRefSchema = external_exports.object({
+  ref: external_exports.string().min(1),
+  select: ExperimentSuiteSelectSchema.optional()
+}).strict();
+var ExperimentFileSchema = external_exports.object({
+  name: external_exports.string().min(1).optional(),
+  agent: external_exports.string().min(1).optional(),
+  target: external_exports.string().min(1).optional(),
+  targets: external_exports.array(ExperimentTargetRefSchema).min(1).optional(),
+  model: external_exports.string().min(1).optional(),
+  agent_options: JsonObjectSchema.optional(),
+  suites: external_exports.array(ExperimentSuiteRefSchema).min(1).optional(),
+  scripts: external_exports.array(ExperimentScriptSchema).optional(),
+  repeat: ExperimentRepeatSchema.optional(),
+  runs: external_exports.number().int().min(1).optional(),
+  early_exit: external_exports.boolean().optional(),
+  timeout_seconds: external_exports.number().gt(0).optional(),
+  workers: external_exports.number().int().min(1).optional(),
+  budget_usd: external_exports.number().gt(0).optional(),
+  sandbox: external_exports.enum(["auto", "docker", "vercel"]).optional(),
+  workspace: JsonObjectSchema.optional(),
+  setup: external_exports.array(ExperimentScriptSchema).optional()
+}).strict().refine((value) => value.repeat === void 0 || value.runs === void 0, {
+  message: "Use repeat or runs, not both."
+});
 var ASSERTION_TYPES_WITH_STRING_VALUE = /* @__PURE__ */ new Set([
   "contains",
   "icontains",
@@ -3023,8 +3160,16 @@ async function validateEvalFile(filePath) {
         message: "Invalid 'criteria' field (must be a non-empty string if provided)"
       });
     }
+    const caseExecution = isObject(evalCase.execution) ? evalCase.execution : void 0;
+    const skipDefaults = caseExecution?.skip_defaults === true;
+    const hasPromptMdFallback = evalCase.input === void 0 ? await loadPromptMdFallback({
+      evalFilePath: absolutePath,
+      searchRoots: buildSearchRoots2(absolutePath, process.cwd()),
+      testInputFiles: evalCase.input_files,
+      suiteInputFiles: skipDefaults ? void 0 : parsed.input_files
+    }) !== void 0 : false;
     validateInputField(evalCase.input, `${location}.input`, absolutePath, errors, {
-      required: true
+      required: !hasPromptMdFallback
     });
     const expectedOutputField = evalCase.expected_output;
     if (expectedOutputField !== void 0) {
@@ -3247,7 +3392,7 @@ function validateInputField(inputField, location, filePath, errors, options = {}
         severity: "error",
         filePath,
         location,
-        message: "Missing 'input' field (must be a string, object, or array of messages)"
+        message: "Missing 'input' field (provide a string, object, message array, or PROMPT.md next to EVAL.yaml / referenced in input_files)"
       });
     }
     return;
@@ -4241,7 +4386,9 @@ async function validateConfigFile(filePath, options = {}) {
 }
 function inferConfigScope(filePath) {
   const globalConfigPath = path52.resolve(getAgentvConfigDir(), "config.yaml");
-  if (path52.resolve(filePath) === globalConfigPath) {
+  const globalLocalConfigPath = path52.resolve(getLocalConfigPath(globalConfigPath));
+  const resolvedPath = path52.resolve(filePath);
+  if (resolvedPath === globalConfigPath || resolvedPath === globalLocalConfigPath) {
     return "global";
   }
   return filePath.split(/[\\/]/).includes(".agentv") ? "project" : "global";
@@ -4474,12 +4621,19 @@ function validateResultsSyncAndBranchPrefix(errors, filePath, resultsRecord, loc
           `Field '${location}.sync.require_push' must be a boolean`
         );
       }
-      if (syncRecord.push_conflict_policy !== void 0 && syncRecord.push_conflict_policy !== "block" && syncRecord.push_conflict_policy !== "backup_and_force_push") {
+      if (syncRecord.push_conflict_policy === "backup_and_force_push") {
         addError(
           errors,
           filePath,
           `${location}.sync.push_conflict_policy`,
-          `Field '${location}.sync.push_conflict_policy' must be 'block' or 'backup_and_force_push'`
+          `Field '${location}.sync.push_conflict_policy' uses removed value 'backup_and_force_push'; remove it or set it to 'block'. AgentV never force-pushes result branches.`
+        );
+      } else if (syncRecord.push_conflict_policy !== void 0 && syncRecord.push_conflict_policy !== "block") {
+        addError(
+          errors,
+          filePath,
+          `${location}.sync.push_conflict_policy`,
+          `Field '${location}.sync.push_conflict_policy' must be 'block'`
         );
       }
     }
@@ -5226,6 +5380,8 @@ var WipCheckpointLoop = class {
 
 // src/commands/eval/run-eval.ts
 var DEFAULT_WORKERS = 3;
+var loadCjsModule = createNodeRequire(import.meta.url);
+var micromatch = loadCjsModule("micromatch");
 function shouldSkipExistingResultForResume(result, rerunFailed) {
   if (rerunFailed) {
     return result.executionStatus === "ok";
@@ -5306,13 +5462,13 @@ function normalizeSourceMetadataByEvalFile(value) {
     const entries = [...value.entries()].filter(
       (entry) => typeof entry[0] === "string" && typeof entry[1] === "object" && entry[1] !== null && !Array.isArray(entry[1])
     );
-    return entries.length > 0 ? new Map(entries.map(([key, metadata]) => [path14.resolve(key), metadata])) : void 0;
+    return entries.length > 0 ? new Map(entries.map(([key, metadata]) => [path15.resolve(key), metadata])) : void 0;
   }
   if (typeof value === "object" && value !== null && !Array.isArray(value)) {
     const entries = Object.entries(value).filter(
       (entry) => typeof entry[1] === "object" && entry[1] !== null && !Array.isArray(entry[1])
     );
-    return entries.length > 0 ? new Map(entries.map(([key, metadata]) => [path14.resolve(key), metadata])) : void 0;
+    return entries.length > 0 ? new Map(entries.map(([key, metadata]) => [path15.resolve(key), metadata])) : void 0;
   }
   return void 0;
 }
@@ -5326,10 +5482,10 @@ var LEGACY_OUTPUT_FILE_EXTENSIONS = /* @__PURE__ */ new Set([
   ".htm"
 ]);
 function looksLikeLegacyOutputFilePath(value) {
-  return LEGACY_OUTPUT_FILE_EXTENSIONS.has(path14.extname(value).toLowerCase());
+  return LEGACY_OUTPUT_FILE_EXTENSIONS.has(path15.extname(value).toLowerCase());
 }
 function outputFileMigrationMessage(value) {
-  const ext = path14.extname(value).toLowerCase();
+  const ext = path15.extname(value).toLowerCase();
   const removalHint = ext === ".xml" ? "JUnit XML export from agentv eval has been removed." : "Flat result file export from agentv eval has been removed.";
   return `--output expects a run directory, not a file path: ${value}
 ${removalHint} Set --output <dir> for the canonical run artifacts; AgentV always writes <dir>/index.jsonl.`;
@@ -5337,7 +5493,7 @@ ${removalHint} Set --output <dir> for the canonical run artifacts; AgentV always
 function artifactsMigrationMessage(artifactsDir, outputDir) {
   const lines = [`--artifacts was removed from agentv eval. Use --output ${artifactsDir} instead.`];
   if (outputDir && looksLikeLegacyOutputFilePath(outputDir)) {
-    const ext = path14.extname(outputDir).toLowerCase();
+    const ext = path15.extname(outputDir).toLowerCase();
     lines.push(
       ext === ".xml" ? "JUnit XML export from agentv eval has been removed." : "Flat result file export from agentv eval has been removed."
     );
@@ -5493,7 +5649,7 @@ function normalizeOptions(rawOptions, config, yamlExecution) {
   };
 }
 function withSourceMetadata(result, testFilePath, options) {
-  const sourceMetadata = options.sourceMetadataByEvalFile?.get(path14.resolve(testFilePath));
+  const sourceMetadata = options.sourceMetadataByEvalFile?.get(path15.resolve(testFilePath));
   if (!sourceMetadata) {
     return result;
   }
@@ -5515,7 +5671,258 @@ async function ensureFileExists(filePath, description) {
 function buildDefaultOutputPathForExperiment(cwd, experiment, runDirName) {
   const runDir = buildDefaultRunDirFromName(cwd, experiment, runDirName);
   mkdirSync2(runDir, { recursive: true });
-  return path14.join(runDir, "index.jsonl");
+  return path15.join(runDir, "index.jsonl");
+}
+function normalizeTsDefaultExperiment(config) {
+  return normalizeString(config?.experiments?.default) ?? normalizeString(config?.defaultExperiment);
+}
+async function resolveExperimentForRun(params) {
+  const experimentRef = params.explicitExperiment ?? params.yamlDefaultExperiment ?? params.tsDefaultExperiment;
+  if (!experimentRef) {
+    return {};
+  }
+  const experimentPath = resolveExperimentFilePath(params.cwd, experimentRef);
+  if (!experimentPath) {
+    if (isExperimentFileReference(experimentRef)) {
+      throw new Error(`Experiment file not found: ${experimentRef}`);
+    }
+    return { name: experimentRef };
+  }
+  const config = await loadExperimentConfig(experimentPath);
+  return {
+    name: config.name ?? deriveExperimentNameFromPath(experimentPath),
+    config
+  };
+}
+function resolveExperimentFilePath(cwd, experimentRef) {
+  if (isExperimentFileReference(experimentRef)) {
+    const experimentPath = path15.isAbsolute(experimentRef) ? experimentRef : path15.resolve(cwd, experimentRef);
+    return existsSync7(experimentPath) ? experimentPath : void 0;
+  }
+  for (const ext of ["yaml", "yml", "ts", "js", "mts", "mjs"]) {
+    const candidate = path15.resolve(cwd, "experiments", `${experimentRef}.${ext}`);
+    if (existsSync7(candidate)) {
+      return candidate;
+    }
+  }
+  return void 0;
+}
+function applyExperimentOptions(options, experiment) {
+  if (!experiment) {
+    return options;
+  }
+  const experimentTargetRefs = buildExperimentTargetRefs(experiment);
+  const experimentTargetNames = experimentTargetRefs?.map((target) => target.name) ?? [];
+  const experimentTarget = experiment.target && experiment.target.trim().length > 0 ? experiment.target : void 0;
+  const nextCliTargets = options.cliTargets.length > 0 ? options.cliTargets : experimentTargetNames.length > 0 ? experimentTargetNames : experimentTarget ? [experimentTarget] : options.cliTargets;
+  const workspaceMode = options.workspaceMode ?? readExperimentWorkspaceMode(experiment.workspace?.mode);
+  const workspacePath = options.workspacePath ?? readExperimentWorkspacePath(experiment.workspace);
+  return {
+    ...options,
+    target: options.target ?? (nextCliTargets.length === 1 ? nextCliTargets[0] : void 0),
+    cliTargets: nextCliTargets,
+    agentTimeoutSeconds: options.agentTimeoutSeconds ?? experiment.timeoutSeconds,
+    workers: options.workers ?? experiment.workers,
+    workspaceMode: workspacePath ? "static" : workspaceMode,
+    workspacePath,
+    budgetUsd: options.budgetUsd ?? experiment.budgetUsd,
+    experimentConfig: experiment,
+    experimentMetadata: buildExperimentArtifactMetadata(experiment),
+    experimentTargetRefs: options.cliTargets.length === 0 ? experimentTargetRefs : void 0,
+    experimentTrialsConfig: buildExperimentTrialsConfig(experiment)
+  };
+}
+function buildExperimentTargetRefs(experiment) {
+  if (!experiment.targets || experiment.targets.length === 0) {
+    return void 0;
+  }
+  return experiment.targets.map((target) => {
+    if (typeof target === "string") {
+      return { name: target };
+    }
+    return {
+      name: target.name,
+      ...target.useTarget !== void 0 && { use_target: target.useTarget },
+      ...target.hooks !== void 0 && {
+        hooks: target.hooks
+      }
+    };
+  });
+}
+function buildExperimentTrialsConfig(experiment) {
+  if (experiment.repeat) {
+    if (experiment.repeat.count <= 1) {
+      return void 0;
+    }
+    return {
+      count: experiment.repeat.count,
+      strategy: experiment.repeat.strategy,
+      ...experiment.repeat.costLimitUsd !== void 0 && {
+        costLimitUsd: experiment.repeat.costLimitUsd
+      },
+      ...experiment.earlyExit !== void 0 && { earlyExit: experiment.earlyExit }
+    };
+  }
+  if (!experiment.runs || experiment.runs <= 1) {
+    return void 0;
+  }
+  return {
+    count: experiment.runs,
+    strategy: "pass_at_k",
+    ...experiment.earlyExit !== void 0 && { earlyExit: experiment.earlyExit }
+  };
+}
+function readExperimentWorkspaceMode(value) {
+  return value === "pooled" || value === "temp" || value === "static" ? value : void 0;
+}
+function readExperimentWorkspacePath(workspace) {
+  const value = workspace?.path;
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : void 0;
+}
+function matchesTestFilter(id, filter) {
+  return typeof filter === "string" ? micromatch.isMatch(id, filter) : filter.some((pattern) => micromatch.isMatch(id, pattern));
+}
+async function resolveExperimentSuiteSelection(suites, cwd) {
+  if (!suites || suites.length === 0) {
+    return void 0;
+  }
+  const testFiles = /* @__PURE__ */ new Set();
+  const selectedTestIdsByEvalFile = /* @__PURE__ */ new Map();
+  for (const suite of suites) {
+    const resolvedSuiteFiles = await resolveEvalPaths([suite.ref], cwd);
+    for (const testFilePath of resolvedSuiteFiles) {
+      const resolvedPath = path15.resolve(testFilePath);
+      testFiles.add(resolvedPath);
+      if (suite.select?.testIds && suite.select.testIds.length > 0) {
+        const existing = selectedTestIdsByEvalFile.get(resolvedPath) ?? [];
+        selectedTestIdsByEvalFile.set(resolvedPath, [...existing, ...suite.select.testIds]);
+      }
+    }
+  }
+  const filtersByEvalFile = /* @__PURE__ */ new Map();
+  for (const [testFilePath, testIds] of selectedTestIdsByEvalFile.entries()) {
+    const uniqueTestIds = [...new Set(testIds)];
+    filtersByEvalFile.set(
+      testFilePath,
+      uniqueTestIds.length === 1 ? uniqueTestIds[0] : uniqueTestIds
+    );
+  }
+  return {
+    testFiles: [...testFiles],
+    filtersByEvalFile
+  };
+}
+async function runExperimentSteps(params) {
+  const steps = params.steps ?? [];
+  if (steps.length === 0) {
+    return;
+  }
+  for (let index = 0; index < steps.length; index++) {
+    const step = steps[index];
+    const command = buildExperimentStepCommand(step);
+    const cwd = resolveExperimentStepCwd(params.cwd, params.experimentConfig, step.cwd);
+    console.log(`Experiment ${params.label} ${index + 1}/${steps.length}: ${command.display}`);
+    await runExperimentCommand(command.argv, {
+      cwd,
+      env: step.env,
+      timeoutMs: step.timeoutSeconds ? step.timeoutSeconds * 1e3 : void 0,
+      label: `experiment ${params.label}`
+    });
+  }
+}
+async function runExperimentSetup(params) {
+  const setup = params.config?.setup;
+  if (typeof setup === "function") {
+    console.log("Experiment setup: running TypeScript setup()");
+    await setup({
+      cwd: params.cwd,
+      runDir: params.runDir,
+      experiment: params.config,
+      env: process.env
+    });
+    return;
+  }
+  await runExperimentSteps({
+    label: "setup",
+    steps: setup,
+    cwd: params.cwd,
+    experimentConfig: params.config
+  });
+}
+function buildExperimentStepCommand(step) {
+  if (step.command && step.command.length > 0) {
+    return { argv: step.command, display: step.command.join(" ") };
+  }
+  if (typeof step.script === "string" && step.script.trim().length > 0) {
+    return {
+      argv: shellCommand(step.script),
+      display: step.script
+    };
+  }
+  if (Array.isArray(step.script) && step.script.length > 0) {
+    return { argv: step.script, display: step.script.join(" ") };
+  }
+  throw new Error("Experiment step must define command or script.");
+}
+function shellCommand(script) {
+  return process.platform === "win32" ? ["cmd", "/c", script] : ["sh", "-c", script];
+}
+function resolveExperimentStepCwd(cwd, experimentConfig, stepCwd) {
+  const base = experimentConfig?.sourcePath ? path15.dirname(experimentConfig.sourcePath) : cwd;
+  if (!stepCwd) {
+    return base;
+  }
+  return path15.isAbsolute(stepCwd) ? stepCwd : path15.resolve(base, stepCwd);
+}
+async function runExperimentCommand(argv, options) {
+  if (argv.length === 0) {
+    throw new Error(`${options.label} command must not be empty.`);
+  }
+  await new Promise((resolve, reject) => {
+    const cmd = argv[0];
+    if (!cmd) {
+      reject(new Error(`${options.label} command must not be empty.`));
+      return;
+    }
+    const args = argv.slice(1);
+    const child = spawn(cmd, args, {
+      cwd: options.cwd,
+      env: options.env ? { ...process.env, ...options.env } : process.env,
+      stdio: "inherit"
+    });
+    let completed = false;
+    const timeout = options.timeoutMs !== void 0 ? setTimeout(() => {
+      if (!completed) {
+        completed = true;
+        child.kill("SIGKILL");
+        reject(new Error(`${options.label} timed out after ${options.timeoutMs}ms`));
+      }
+    }, options.timeoutMs) : void 0;
+    child.on("error", (error) => {
+      if (completed) {
+        return;
+      }
+      completed = true;
+      if (timeout !== void 0) {
+        clearTimeout(timeout);
+      }
+      reject(error);
+    });
+    child.on("exit", (code) => {
+      if (completed) {
+        return;
+      }
+      completed = true;
+      if (timeout !== void 0) {
+        clearTimeout(timeout);
+      }
+      if (code === 0) {
+        resolve();
+      } else {
+        reject(new Error(`${options.label} exited with code ${code ?? "unknown"}`));
+      }
+    });
+  });
 }
 function createProgressReporter(maxWorkers, options) {
   const display = new ProgressDisplay(maxWorkers, options);
@@ -5529,7 +5936,7 @@ function createProgressReporter(maxWorkers, options) {
   };
 }
 function makeTestCaseKey(testFilePath, testId) {
-  return `${path14.resolve(testFilePath)}::${testId}`;
+  return `${path15.resolve(testFilePath)}::${testId}`;
 }
 function resolveTargetLabel(requestedName, resolvedName) {
   if (resolvedName !== requestedName) {
@@ -5584,22 +5991,40 @@ function applyVerboseOverride(selection, cliVerbose) {
   };
 }
 async function prepareFileMetadata(params) {
-  const { testFilePath, repoRoot, cwd, options } = params;
+  const { testFilePath, repoRoot, cwd, options, suiteFilter } = params;
   await ensureFileExists(testFilePath, "Test file");
   await loadEnvFromHierarchy({
     testFilePath,
     repoRoot,
     verbose: options.verbose
   });
-  const relativePath = path14.relative(cwd, testFilePath);
+  const relativePath = path15.relative(cwd, testFilePath);
   const category = deriveCategory(relativePath);
   const suite = await loadTestSuite(testFilePath, repoRoot, {
     verbose: options.verbose,
-    filter: options.filter,
+    filter: suiteFilter ?? options.filter,
     category
   });
-  const testIds = suite.tests.map((value) => value.id);
+  const testCases = suiteFilter && options.filter ? suite.tests.filter((testCase) => matchesTestFilter(testCase.id, options.filter ?? "")) : suite.tests;
+  const testIds = testCases.map((value) => value.id);
   const suiteTargets = suite.targets;
+  if (testCases.length === 0) {
+    return {
+      testIds,
+      testCases,
+      selections: [],
+      trialsConfig: options.experimentTrialsConfig,
+      suiteTargets,
+      yamlWorkers: suite.workers,
+      yamlCache: suite.cacheConfig?.enabled,
+      yamlCachePath: suite.cacheConfig?.cachePath,
+      budgetUsd: suite.budgetUsd,
+      failOnError: suite.failOnError,
+      threshold: suite.threshold,
+      tags: suite.metadata?.tags,
+      providerFactory: suite.providerFactory
+    };
+  }
   let selections;
   if (options.transcript) {
     const transcriptSelection = {
@@ -5616,7 +6041,7 @@ async function prepareFileMetadata(params) {
     selections = [
       {
         selection: transcriptSelection,
-        inlineTargetLabel: `transcript (${path14.basename(options.transcript)})`
+        inlineTargetLabel: `transcript (${path15.basename(options.transcript)})`
       }
     ];
   } else if (suite.inlineTarget && options.cliTargets.length === 0) {
@@ -5671,15 +6096,20 @@ async function prepareFileMetadata(params) {
     const cliTargets = options.cliTargets;
     const suiteTargets2 = suite.targets;
     const suiteTargetRefs = suite.targetRefs;
+    const experimentTargetRefs = options.experimentTargetRefs;
     let targetNames;
+    let targetRefs;
     if (cliTargets.length > 0) {
       targetNames = cliTargets;
+      targetRefs = experimentTargetRefs;
     } else if (suiteTargets2 && suiteTargets2.length > 0) {
       targetNames = suiteTargets2;
+      targetRefs = suiteTargetRefs;
     } else {
       targetNames = [];
+      targetRefs = void 0;
     }
-    if (targetNames.length > 1) {
+    if (targetNames.length > 1 || targetNames.length === 1 && targetRefs) {
       const multiSelections = await selectMultipleTargets({
         testFilePath,
         repoRoot,
@@ -5691,7 +6121,7 @@ async function prepareFileMetadata(params) {
         dryRunDelayMax: options.dryRunDelayMax,
         env: process.env,
         targetNames,
-        targetRefs: suiteTargetRefs
+        targetRefs
       });
       selections = multiSelections.map((sel) => ({
         selection: sel,
@@ -5710,9 +6140,7 @@ async function prepareFileMetadata(params) {
         dryRunDelayMax: options.dryRunDelayMax,
         env: process.env
       });
-      const singleTargetHooks = suiteTargetRefs?.find(
-        (ref) => ref.name === selection.targetName
-      )?.hooks;
+      const singleTargetHooks = targetRefs?.find((ref) => ref.name === selection.targetName)?.hooks;
       const augmentedSelection = singleTargetHooks ? { ...selection, targetHooks: singleTargetHooks } : selection;
       selections = [
         {
@@ -5727,9 +6155,9 @@ async function prepareFileMetadata(params) {
   }
   return {
     testIds,
-    testCases: suite.tests,
+    testCases,
     selections,
-    trialsConfig: suite.trials,
+    trialsConfig: options.experimentTrialsConfig,
     suiteTargets,
     yamlWorkers: suite.workers,
     yamlCache: suite.cacheConfig?.enabled,
@@ -5782,7 +6210,7 @@ async function runSingleEvalFile(params) {
   } = params;
   const targetName = selection.targetName;
   const replayRecording = options.recordReplay ? {
-    fixturesPath: path14.resolve(options.recordReplay),
+    fixturesPath: path15.resolve(options.recordReplay),
     sourceTarget: targetName,
     variant: options.recordReplayVariant
   } : void 0;
@@ -5926,7 +6354,7 @@ async function runEvalCommand(input) {
     );
   }
   const repoRoot = await findRepoRoot(cwd);
-  const yamlConfig = await loadConfig(path14.join(cwd, "_"), repoRoot);
+  const yamlConfig = await loadConfig(path15.join(cwd, "_"), repoRoot);
   let requiredVersionCheck;
   if (yamlConfig?.required_version) {
     requiredVersionCheck = await enforceRequiredVersion(yamlConfig.required_version, {
@@ -5934,6 +6362,33 @@ async function runEvalCommand(input) {
     });
   }
   let options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution);
+  const resolvedExperiment = await resolveExperimentForRun({
+    cwd,
+    explicitExperiment: options.experiment,
+    yamlDefaultExperiment: resolveDefaultExperimentReference(yamlConfig),
+    tsDefaultExperiment: normalizeTsDefaultExperiment(config)
+  });
+  options = {
+    ...applyExperimentOptions(options, resolvedExperiment.config),
+    experiment: resolvedExperiment.name
+  };
+  const suiteSelection = await resolveExperimentSuiteSelection(
+    options.experimentConfig?.suites,
+    cwd
+  );
+  const evalPathInputs = input.testFiles.length > 0 ? [...input.testFiles] : suiteSelection ? [...suiteSelection.testFiles] : [];
+  if (evalPathInputs.length === 0 && process.stdin.isTTY) {
+    const { launchInteractiveWizard } = await import("./interactive-2WMVAASS.js");
+    await launchInteractiveWizard();
+    return void 0;
+  }
+  const resolvedTestFiles = await resolveEvalPaths(evalPathInputs, cwd);
+  options = {
+    ...options,
+    ...suiteSelection !== void 0 && {
+      suiteFiltersByEvalFile: suiteSelection.filtersByEvalFile
+    }
+  };
   if (!process.env.AGENTV_EXPERIMENT) {
     process.env.AGENTV_EXPERIMENT = normalizeExperimentName(options.experiment);
   }
@@ -5962,7 +6417,7 @@ async function runEvalCommand(input) {
   }
   let retryNonErrorResults;
   if (options.retryErrors) {
-    const retryPath = path14.resolve(options.retryErrors);
+    const retryPath = path15.resolve(options.retryErrors);
     await ensureFileExists(retryPath, "Retry-errors JSONL file");
     const completedIds = await loadFullyCompletedTestIds(retryPath);
     const errorIds = await loadErrorTestIds(retryPath);
@@ -5980,7 +6435,7 @@ async function runEvalCommand(input) {
     if (cachedDir) {
       options = { ...options, outputDir: cachedDir };
       const flagLabel = options.rerunFailed ? "rerun-failed" : "resume";
-      const displayDir = path14.relative(cwd, cachedDir) || cachedDir;
+      const displayDir = path15.relative(cwd, cachedDir) || cachedDir;
       console.log(`Auto-detected last run dir for --${flagLabel}: ${displayDir}`);
     }
   }
@@ -5989,7 +6444,7 @@ async function runEvalCommand(input) {
   if (options.resume && !options.retryErrors) {
     const explicitResumeDir = options.outputDir;
     if (explicitResumeDir) {
-      const resumeIndexPath = path14.join(path14.resolve(explicitResumeDir), "index.jsonl");
+      const resumeIndexPath = path15.join(path15.resolve(explicitResumeDir), "index.jsonl");
       if (existsSync7(resumeIndexPath)) {
         const content = await readFile8(resumeIndexPath, "utf8");
         const existingResults = parseJsonlResults(content);
@@ -6014,7 +6469,7 @@ async function runEvalCommand(input) {
     }
   }
   if (options.workspacePath) {
-    const resolvedWorkspace = path14.resolve(options.workspacePath);
+    const resolvedWorkspace = path15.resolve(options.workspacePath);
     try {
       const { stat: stat3 } = await import("node:fs/promises");
       const stats = await stat3(resolvedWorkspace);
@@ -6037,22 +6492,22 @@ async function runEvalCommand(input) {
   let outputPath;
   const runDirName = process.env.AGENTV_RUN_TIMESTAMP?.trim() || createRunDirName();
   if (explicitDir) {
-    runDir = path14.resolve(explicitDir);
+    runDir = path15.resolve(explicitDir);
     mkdirSync2(runDir, { recursive: true });
-    outputPath = path14.join(runDir, "index.jsonl");
+    outputPath = path15.join(runDir, "index.jsonl");
   } else {
     outputPath = buildDefaultOutputPathForExperiment(cwd, options.experiment, runDirName);
-    runDir = path14.dirname(outputPath);
+    runDir = path15.dirname(outputPath);
   }
   if (!process.env.AGENTV_RUN_TIMESTAMP) {
-    process.env.AGENTV_RUN_TIMESTAMP = path14.basename(runDir);
+    process.env.AGENTV_RUN_TIMESTAMP = path15.basename(runDir);
   }
   process.env.AGENTV_RUN_DIR = runDir;
   let otelExporter = null;
   const useFileExport = !!options.otelFile;
   if (options.exportOtel || useFileExport) {
     try {
-      const { OtelTraceExporter } = await import("./dist-L4YCQOQ6.js");
+      const { OtelTraceExporter } = await import("./dist-KJ6PELQY.js");
       let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
       let headers = {};
       let resourceAttributes = {};
@@ -6085,7 +6540,7 @@ async function runEvalCommand(input) {
         resourceAttributes,
         captureContent,
         groupTurns: options.otelGroupTurns,
-        otlpFilePath: options.otelFile ? path14.resolve(options.otelFile) : void 0
+        otlpFilePath: options.otelFile ? path15.resolve(options.otelFile) : void 0
       });
       const initialized = await otelExporter.init();
       if (!initialized) {
@@ -6103,9 +6558,13 @@ async function runEvalCommand(input) {
   }
   const primaryWritePath = outputPath;
   console.log(`Artifact directory: ${runDir}`);
-  const resolvedTestFiles = input.testFiles.map((file) => path14.resolve(file));
+  await runExperimentSetup({
+    config: options.experimentConfig,
+    cwd,
+    runDir
+  });
   if (options.otelFile) {
-    console.log(`OTLP JSON file: ${path14.resolve(options.otelFile)}`);
+    console.log(`OTLP JSON file: ${path15.resolve(options.otelFile)}`);
   }
   const evaluationRunner = await resolveEvaluationRunner();
   const allResults = [];
@@ -6123,7 +6582,8 @@ async function runEvalCommand(input) {
       testFilePath,
       repoRoot,
       cwd,
-      options
+      options,
+      suiteFilter: options.suiteFiltersByEvalFile?.get(path15.resolve(testFilePath))
     });
     fileMetadata.set(testFilePath, meta);
   }
@@ -6133,7 +6593,7 @@ async function runEvalCommand(input) {
     for (const [testFilePath, meta] of fileMetadata.entries()) {
       if (!matchesTagFilters(meta.tags, options.tags, options.excludeTags)) {
         fileMetadata.delete(testFilePath);
-        skippedFiles.push(path14.relative(cwd, testFilePath));
+        skippedFiles.push(path15.relative(cwd, testFilePath));
       }
     }
     if (skippedFiles.length > 0 && options.verbose) {
@@ -6156,12 +6616,12 @@ async function runEvalCommand(input) {
     tsConfigCache: options.tsConfigCache
   });
   const activeCachePath = options.cachePath ?? yamlCachePath ?? options.tsConfigCachePath;
-  const cache = cacheEnabled ? new ResponseCache(activeCachePath ? path14.resolve(activeCachePath) : void 0) : void 0;
+  const cache = cacheEnabled ? new ResponseCache(activeCachePath ? path15.resolve(activeCachePath) : void 0) : void 0;
   if (cache) {
     console.log(`Response cache: enabled (${cache.cachePath})`);
   }
   if (options.recordReplay) {
-    console.log(`Replay recording: ${path14.resolve(options.recordReplay)}`);
+    console.log(`Replay recording: ${path15.resolve(options.recordReplay)}`);
   }
   const yamlThreshold = firstMeta?.threshold;
   const resolvedThreshold = options.threshold ?? yamlThreshold;
@@ -6257,7 +6717,7 @@ async function runEvalCommand(input) {
   const activeTestFiles = resolvedTestFiles.filter((f) => fileMetadata.has(f));
   let transcriptProviderFactory;
   if (options.transcript) {
-    const { TranscriptProvider } = await import("./dist-L4YCQOQ6.js");
+    const { TranscriptProvider } = await import("./dist-KJ6PELQY.js");
     const transcriptProvider = await TranscriptProvider.fromFile(options.transcript);
     const totalTests = [...fileMetadata.values()].reduce(
       (sum, meta) => sum + meta.testCases.length,
@@ -6275,10 +6735,11 @@ async function runEvalCommand(input) {
   }
   if (!isResumeAppend && totalEvalCount > 0) {
     const evalFile = activeTestFiles.length === 1 ? activeTestFiles[0] : "";
-    await writeInitialBenchmarkArtifact(runDir, {
+    await writeInitialRunSummaryArtifact(runDir, {
       evalFile,
       plannedTestCount: totalEvalCount,
-      experiment: normalizeExperimentName(options.experiment)
+      experiment: normalizeExperimentName(options.experiment),
+      experimentMetadata: options.experimentMetadata
     });
   }
   let wipLoop;
@@ -6306,7 +6767,7 @@ async function runEvalCommand(input) {
         if (!targetPrep2) continue;
         const budgetMsg = `Run budget exceeded ($${runBudgetTracker.currentCostUsd.toFixed(4)} / $${runBudgetTracker.budgetCapUsd.toFixed(4)})`;
         console.log(`
-\u26A0 ${budgetMsg} \u2014 skipping ${path14.basename(testFilePath)}`);
+\u26A0 ${budgetMsg} \u2014 skipping ${path15.basename(testFilePath)}`);
         for (const { selection } of targetPrep2.selections) {
           const skippedResults = targetPrep2.testCases.map((testCase) => ({
             timestamp: (/* @__PURE__ */ new Date()).toISOString(),
@@ -6385,7 +6846,7 @@ async function runEvalCommand(input) {
               threshold: resolvedThreshold,
               providerFactory: transcriptProviderFactory ?? targetPrep.providerFactory
             });
-            const evalFile = path14.relative(cwd, testFilePath);
+            const evalFile = path15.relative(cwd, testFilePath);
             const existingSummary = remoteEvalSummaries.find(
               (summary2) => summary2.evalFile === evalFile
             );
@@ -6402,7 +6863,7 @@ async function runEvalCommand(input) {
             const message = fileError instanceof Error ? fileError.message : String(fileError);
             console.error(
               `
-[ERROR] \u26A0 Eval file failed: ${path14.basename(testFilePath)} \u2014 ${message}
+[ERROR] \u26A0 Eval file failed: ${path15.basename(testFilePath)} \u2014 ${message}
 `
             );
             const errorResults = filteredTestCases.map(
@@ -6481,7 +6942,7 @@ ${formatRequiredVersionFailureNote(requiredVersionCheck)}`);
       );
       const taskBundleTargets = buildTaskBundleTargetSelections(activeTestFiles, fileMetadata);
       if (isResumeAppend) {
-        const { writePerTestArtifacts } = await import("./artifact-writer-72MQQELS.js");
+        const { writePerTestArtifacts } = await import("./artifact-writer-QD52FC56.js");
         await writePerTestArtifacts(allResults, runDir, {
           experiment: normalizeExperimentName(options.experiment),
           cwd,
@@ -6489,37 +6950,36 @@ ${formatRequiredVersionFailureNote(requiredVersionCheck)}`);
           sourceTests,
           taskBundleTargets
         });
-        const { benchmarkPath: workspaceBenchmarkPath, timingPath } = await aggregateRunDir(
-          runDir,
-          { evalFile, experiment: normalizeExperimentName(options.experiment) }
-        );
-        const indexPath = path14.join(runDir, "index.jsonl");
+        const { summaryPath } = await aggregateRunDir(runDir, {
+          evalFile,
+          experiment: normalizeExperimentName(options.experiment),
+          experimentMetadata: options.experimentMetadata
+        });
+        const indexPath = path15.join(runDir, "index.jsonl");
         console.log(`Artifact workspace updated: ${runDir}`);
         console.log(`  Index: ${indexPath}`);
         console.log(`  Per-test artifacts: ${runDir} (${allResults.length} new test directories)`);
-        console.log(`  Timing: ${timingPath}`);
-        console.log(`  Benchmark: ${workspaceBenchmarkPath}`);
+        console.log(`  Summary: ${summaryPath}`);
       } else {
-        const {
-          testArtifactDir,
-          timingPath,
-          benchmarkPath: workspaceBenchmarkPath,
-          indexPath
-        } = await writeArtifactsFromResults(allResults, runDir, {
-          evalFile,
-          experiment: normalizeExperimentName(options.experiment),
-          cwd,
-          repoRoot,
-          sourceTests,
-          taskBundleTargets
-        });
+        const { testArtifactDir, summaryPath, indexPath } = await writeArtifactsFromResults(
+          allResults,
+          runDir,
+          {
+            evalFile,
+            experiment: normalizeExperimentName(options.experiment),
+            experimentMetadata: options.experimentMetadata,
+            cwd,
+            repoRoot,
+            sourceTests,
+            taskBundleTargets
+          }
+        );
         console.log(`Artifact workspace written to: ${runDir}`);
         console.log(`  Index: ${indexPath}`);
         console.log(
           `  Per-test artifacts: ${testArtifactDir} (${allResults.length} test directories)`
         );
-        console.log(`  Timing: ${timingPath}`);
-        console.log(`  Benchmark: ${workspaceBenchmarkPath}`);
+        console.log(`  Summary: ${summaryPath}`);
       }
     }
     const resultsWithWorkspaces = allResults.filter((r) => r.workspacePath);
@@ -6559,9 +7019,9 @@ Results written to: ${outputPath}`);
       });
     }
     if (summary.executionErrorCount > 0 && !options.retryErrors && !options.resume) {
-      const evalFileArgs = activeTestFiles.map((f) => path14.relative(cwd, f)).join(" ");
+      const evalFileArgs = activeTestFiles.map((f) => path15.relative(cwd, f)).join(" ");
       const targetFlag = options.target ? ` --target ${options.target}` : "";
-      const relativeRunDir = path14.relative(cwd, runDir);
+      const relativeRunDir = path15.relative(cwd, runDir);
       console.log(
         `
 Tip: ${summary.executionErrorCount} execution error(s) detected. Re-run failed tests with:
@@ -6579,6 +7039,12 @@ Tip: ${summary.executionErrorCount} execution error(s) detected. Re-run failed t
       wipCleanedUp = true;
       await wipLoop.stopAndDeleteWipBranch();
     }
+    await runExperimentSteps({
+      label: "script",
+      steps: options.experimentConfig?.scripts,
+      cwd,
+      experimentConfig: options.experimentConfig
+    });
     return {
       executionErrorCount: summary.executionErrorCount,
       outputPath,
@@ -6610,7 +7076,7 @@ async function resolveEvaluationRunner() {
   if (!overridePath) {
     return runEvaluation;
   }
-  const resolved = path14.isAbsolute(overridePath) ? overridePath : path14.resolve(process.cwd(), overridePath);
+  const resolved = path15.isAbsolute(overridePath) ? overridePath : path15.resolve(process.cwd(), overridePath);
   const moduleUrl = pathToFileURL2(resolved).href;
   const mod = await import(moduleUrl);
   const candidate = mod.runEvaluation;
@@ -6622,43 +7088,6 @@ async function resolveEvaluationRunner() {
   return candidate;
 }
 
-// src/commands/eval/discover.ts
-import path15 from "node:path";
-import fg2 from "fast-glob";
-async function discoverEvalFiles(cwd) {
-  const repoRoot = await findRepoRoot(cwd);
-  const config = await loadConfig(path15.join(cwd, "_"), repoRoot);
-  const patterns = config?.eval_patterns && config.eval_patterns.length > 0 ? config.eval_patterns : DEFAULT_EVAL_PATTERNS;
-  const ignore = ["**/node_modules/**", "**/dist/**"];
-  const matches = await fg2(patterns, {
-    cwd,
-    absolute: true,
-    onlyFiles: true,
-    ignore,
-    followSymbolicLinks: true,
-    caseSensitiveMatch: false
-  });
-  const evalFiles = matches.map((absPath) => {
-    const relativePath = path15.relative(cwd, absPath);
-    const category = deriveCategory(relativePath);
-    return { path: absPath, relativePath, category };
-  });
-  evalFiles.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
-  return evalFiles;
-}
-function getCategories(files) {
-  const categories = /* @__PURE__ */ new Set();
-  for (const file of files) {
-    categories.add(file.category);
-  }
-  const sorted = Array.from(categories);
-  sorted.sort();
-  return sorted;
-}
-function filterByCategory(files, category) {
-  return files.filter((f) => f.category === category);
-}
-
 export {
   resolveEvalPaths,
   findRepoRoot,
@@ -6714,6 +7143,7 @@ export {
   isRemoteRunId,
   getRemoteResultsStatus,
   syncRemoteResults,
+  confirmRemoteResultsMerge,
   listMergedResultFiles,
   findRunById,
   ensureRemoteRunAvailable,
@@ -6723,9 +7153,9 @@ export {
   maybeAutoExportRunArtifacts,
   resolveRunCacheFile,
   loadRunCache,
-  runEvalCommand,
   discoverEvalFiles,
   getCategories,
-  filterByCategory
+  filterByCategory,
+  runEvalCommand
 };
-//# sourceMappingURL=chunk-CXHYTYHU.js.map
+//# sourceMappingURL=chunk-IBI7A5RP.js.map