agentv 5.0.0-next.1 → 5.2.0-next.1

package/dist/{interactive-2WMVAASS.js → interactive-36ZNMQB2.js}

@@ -6,20 +6,20 @@ import {
   filterByCategory,
   findRepoRoot,
   getCategories,
+  resolveExistingRunPrimaryPath,
   runEvalCommand
-} from "./chunk-IBI7A5RP.js";
-import "./chunk-6JVM4GJI.js";
-import "./chunk-UG4JHZYM.js";
-import "./chunk-76FOHROU.js";
-import "./chunk-RCIEHFQ2.js";
+} from "./chunk-T4AD3H3Y.js";
+import "./chunk-46K2OET3.js";
+import "./chunk-ALVZQUZP.js";
 import {
   getAgentvConfigDir,
   listTargetNames,
   readTargetDefinitions
-} from "./chunk-GZJMODLW.js";
-import "./chunk-IJEZFZJR.js";
+} from "./chunk-3S6A2RKR.js";
+import "./chunk-7BGERE6L.js";
+import "./chunk-PEUTJS7B.js";
 import "./chunk-M7BUKBAF.js";
-import "./chunk-5H446C7X.js";
+import "./chunk-QMRVH5ZP.js";
 
 // src/commands/eval/interactive.ts
 import { existsSync } from "node:fs";
@@ -70,7 +70,6 @@ ${ANSI_DIM}Resuming run at ${relativeDir}...${ANSI_RESET}
         evalPaths: lastConfig.evalPaths,
         target: lastConfig.target,
         workers: lastConfig.workers,
-        dryRun: lastConfig.dryRun,
         cache: lastConfig.cache
       },
       { resumeOutputDir: lastConfig.outputDir }
@@ -85,7 +84,6 @@ ${ANSI_DIM}Rerunning last configuration...${ANSI_RESET}
       evalPaths: lastConfig.evalPaths,
       target: lastConfig.target,
       workers: lastConfig.workers,
-      dryRun: lastConfig.dryRun,
       cache: lastConfig.cache
     });
     return;
@@ -103,8 +101,8 @@ ${ANSI_DIM}Rerunning last configuration...${ANSI_RESET}
 async function promptMainMenu(lastConfig) {
   const choices = [];
   if (lastConfig?.outputDir) {
-    const indexPath = path2.join(lastConfig.outputDir, "index.jsonl");
-    if (existsSync(indexPath)) {
+    const indexPath = resolveExistingRunPrimaryPath(lastConfig.outputDir);
+    if (indexPath && existsSync(indexPath)) {
       const dirLabel = path2.basename(lastConfig.outputDir);
       choices.push({
         name: "\u23EF  Resume last run",
@@ -245,7 +243,7 @@ async function promptAdvancedOptions() {
     default: false
   });
   if (!customize) {
-    return { workers: 3, dryRun: false, cache: false };
+    return { workers: 3, cache: false };
   }
   const workers = await number({
     message: "Number of parallel workers (1-50)",
@@ -253,15 +251,11 @@ async function promptAdvancedOptions() {
     min: 1,
     max: 50
   }) ?? 3;
-  const dryRun = await confirm({
-    message: "Enable dry-run mode (mock responses)?",
-    default: false
-  });
   const cache = await confirm({
     message: "Enable response cache?",
     default: false
   });
-  return { workers, dryRun, cache };
+  return { workers, cache };
 }
 async function promptReviewAndConfirm(config, cwd) {
   const evalDisplay = config.evalPaths.map((p) => {
@@ -275,7 +269,6 @@ ${ANSI_BOLD}Review Configuration${ANSI_RESET}`);
 ${evalDisplay}`);
   console.log(`${ANSI_GREEN}Target:${ANSI_RESET}    ${config.target}`);
   console.log(`${ANSI_GREEN}Workers:${ANSI_RESET}   ${config.workers}`);
-  console.log(`${ANSI_GREEN}Dry run:${ANSI_RESET}   ${config.dryRun ? "yes" : "no"}`);
   console.log(`${ANSI_GREEN}Cache:${ANSI_RESET}     ${config.cache ? "yes" : "no"}`);
   console.log(`${ANSI_DIM}${"\u2500".repeat(40)}${ANSI_RESET}`);
   return confirm({
@@ -288,12 +281,8 @@ async function executeConfig(config, opts) {
   const rawOptions = {
     target: config.target,
     workers: config.workers,
-    dryRun: config.dryRun,
     cache: config.cache,
     ...opts?.resumeOutputDir ? { output: opts.resumeOutputDir, resume: true } : {},
-    dryRunDelay: 0,
-    dryRunDelayMin: 0,
-    dryRunDelayMax: 0,
     agentTimeout: 120,
     maxRetries: 2,
     verbose: false,
@@ -312,7 +301,6 @@ async function executeConfig(config, opts) {
       evalPaths: config.evalPaths,
       target: config.target,
       workers: config.workers,
-      dryRun: config.dryRun,
       cache: config.cache,
       outputDir: path2.dirname(result.outputPath)
     });
@@ -335,13 +323,9 @@ ${ANSI_DIM}Retrying execution errors...${ANSI_RESET}
   const rawOptions = {
     target: config.target,
     workers: config.workers,
-    dryRun: config.dryRun,
     cache: config.cache,
     retryErrors: outputPath,
     out: outputPath,
-    dryRunDelay: 0,
-    dryRunDelayMin: 0,
-    dryRunDelayMax: 0,
     agentTimeout: 120,
     maxRetries: 2,
     verbose: false,
@@ -360,4 +344,4 @@ ${ANSI_DIM}Retrying execution errors...${ANSI_RESET}
 export {
   launchInteractiveWizard
 };
-//# sourceMappingURL=interactive-2WMVAASS.js.map
+//# sourceMappingURL=interactive-36ZNMQB2.js.map

package/dist/interactive-36ZNMQB2.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/commands/eval/interactive.ts","../src/commands/eval/last-config.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport path from 'node:path';\nimport { listTargetNames, readTargetDefinitions } from '@agentv/core';\nimport { checkbox, confirm, number, search, select } from '@inquirer/prompts';\n\nimport { TARGET_FILE_CANDIDATES, fileExists } from '../../utils/targets.js';\nimport {\n  type DiscoveredEvalFile,\n  discoverEvalFiles,\n  filterByCategory,\n  getCategories,\n} from './discover.js';\nimport { type LastConfig, loadLastConfig, saveLastConfig } from './last-config.js';\nimport { resolveExistingRunPrimaryPath } from './result-layout.js';\nimport { runEvalCommand } from './run-eval.js';\nimport { findRepoRoot } from './shared.js';\n\nconst ANSI_BOLD = '\\x1b[1m';\nconst ANSI_DIM = '\\x1b[2m';\nconst ANSI_CYAN = '\\x1b[36m';\nconst ANSI_GREEN = '\\x1b[32m';\nconst ANSI_RESET = '\\x1b[0m';\n\nexport interface InteractiveConfig {\n  readonly evalPaths: readonly string[];\n  readonly target: string;\n  readonly workers: number;\n  readonly cache: boolean;\n}\n\n/**\n * Launch the interactive wizard when `agentv eval` is called with no arguments.\n */\nexport async function launchInteractiveWizard(): Promise<void> {\n  const cwd = process.cwd();\n\n  console.log(`\\n${ANSI_BOLD}${ANSI_CYAN}AgentV Interactive Mode${ANSI_RESET}\\n`);\n\n  const lastConfig = await loadLastConfig();\n  const action = await promptMainMenu(lastConfig);\n\n  if (action === 'exit') {\n    return;\n  }\n\n  if (action === 'resume' && lastConfig?.outputDir) {\n    const relativeDir = path.relative(cwd, lastConfig.outputDir) || lastConfig.outputDir;\n    console.log(`\\n${ANSI_DIM}Resuming run at ${relativeDir}...${ANSI_RESET}\\n`);\n    await executeConfig(\n      {\n        evalPaths: lastConfig.evalPaths,\n        target: lastConfig.target,\n        workers: lastConfig.workers,\n        cache: lastConfig.cache,\n      },\n      { resumeOutputDir: lastConfig.outputDir },\n    );\n    return;\n  }\n\n  if (action === 'rerun' && lastConfig) {\n    console.log(`\\n${ANSI_DIM}Rerunning last configuration...${ANSI_RESET}\\n`);\n    await executeConfig({\n      evalPaths: lastConfig.evalPaths,\n      target: lastConfig.target,\n      workers: lastConfig.workers,\n      cache: lastConfig.cache,\n    });\n    return;\n  }\n\n  // Run new evaluation flow\n  const config = await promptNewEvaluation(cwd);\n  if (!config) {\n    return;\n  }\n\n  // Review & confirm\n  const confirmed = await promptReviewAndConfirm(config, cwd);\n  if (!confirmed) {\n    return;\n  }\n\n  await executeConfig(config);\n}\n\nasync function promptMainMenu(\n  lastConfig: LastConfig | undefined,\n): Promise<'new' | 'rerun' | 'resume' | 'exit'> {\n  type MenuChoice = 'new' | 'rerun' | 'resume' | 'exit';\n  const choices: Array<{ name: string; value: MenuChoice; description?: string }> = [];\n\n  // Resume entry: only when the prior run has a known artifact dir with a manifest.\n  if (lastConfig?.outputDir) {\n    const indexPath = resolveExistingRunPrimaryPath(lastConfig.outputDir);\n    if (indexPath && existsSync(indexPath)) {\n      const dirLabel = path.basename(lastConfig.outputDir);\n      choices.push({\n        name: '⏯  Resume last run',\n        value: 'resume',\n        description: `${dirLabel} (target: ${lastConfig.target})`,\n      });\n    }\n  }\n\n  if (lastConfig) {\n    const evalCount = lastConfig.evalPaths.length;\n    choices.push({\n      name: '🔄 Rerun last config',\n      value: 'rerun',\n      description: `${evalCount} eval file(s), target: ${lastConfig.target}`,\n    });\n  }\n\n  choices.push({ name: '🚀 Run new evaluation', value: 'new' }, { name: '✕ Exit', value: 'exit' });\n\n  return select<MenuChoice>({\n    message: 'What would you like to do?',\n    choices,\n  });\n}\n\nasync function promptNewEvaluation(cwd: string): Promise<InteractiveConfig | undefined> {\n  // Step 1: Discover eval files\n  console.log(`\\n${ANSI_DIM}Scanning for eval files...${ANSI_RESET}`);\n  const allFiles = await discoverEvalFiles(cwd);\n\n  if (allFiles.length === 0) {\n    console.log(\n      '\\n⚠  No eval files found in the current directory.\\n' +\n        '   Place .yaml or .jsonl eval files in your project, or use:\\n' +\n        '   agentv eval <path-to-eval.yaml>\\n',\n    );\n    return undefined;\n  }\n\n  console.log(`${ANSI_DIM}Found ${allFiles.length} eval file(s)${ANSI_RESET}\\n`);\n\n  // Step 2: Select eval files (optionally filter by category first)\n  const selectedFiles = await promptEvalSelection(allFiles);\n  if (selectedFiles.length === 0) {\n    console.log('\\nNo eval files selected.');\n    return undefined;\n  }\n\n  // Step 3: Select target\n  const target = await promptTargetSelection(cwd, selectedFiles[0].path);\n\n  // Step 4: Advanced options\n  const advanced = await promptAdvancedOptions();\n\n  return {\n    evalPaths: selectedFiles.map((f) => f.path),\n    target,\n    ...advanced,\n  };\n}\n\nasync function promptEvalSelection(\n  allFiles: readonly DiscoveredEvalFile[],\n): Promise<DiscoveredEvalFile[]> {\n  const categories = getCategories(allFiles);\n\n  // If only one category or few files, skip category selection\n  let filesToSelect: readonly DiscoveredEvalFile[];\n\n  if (categories.length > 1) {\n    const selectedCategory = await search<string>({\n      message: 'Select a category (type to search)',\n      source: async (term) => {\n        const filtered = term\n          ? categories.filter((c) => c.toLowerCase().includes(term.toLowerCase()))\n          : categories;\n        return [\n          { name: '(all categories)', value: '__all__' },\n          ...filtered.map((c) => {\n            const count = filterByCategory(allFiles, c).length;\n            return { name: `${c} (${count} file${count > 1 ? 's' : ''})`, value: c };\n          }),\n        ];\n      },\n    });\n\n    filesToSelect =\n      selectedCategory === '__all__' ? allFiles : filterByCategory(allFiles, selectedCategory);\n  } else {\n    filesToSelect = allFiles;\n  }\n\n  return checkbox<DiscoveredEvalFile>({\n    message: 'Select eval files to run (space to toggle, enter to confirm)',\n    choices: filesToSelect.map((f) => ({\n      name: f.relativePath,\n      value: f,\n      checked: filesToSelect.length <= 5, // auto-select if few files\n    })),\n    required: true,\n  });\n}\n\nasync function promptTargetSelection(cwd: string, firstEvalPath: string): Promise<string> {\n  const repoRoot = await findRepoRoot(cwd);\n\n  // Try to find targets.yaml — search near the eval file first, then cwd/repoRoot\n  const targetsPath = await findTargetsFile(cwd, repoRoot, firstEvalPath);\n\n  if (!targetsPath) {\n    console.log(`${ANSI_DIM}No targets.yaml found. Using default target.${ANSI_RESET}`);\n    return 'default';\n  }\n\n  const definitions = await readTargetDefinitions(targetsPath);\n  const targetNames = listTargetNames(definitions);\n\n  if (targetNames.length === 0) {\n    return 'default';\n  }\n\n  if (targetNames.length === 1) {\n    console.log(`${ANSI_DIM}Using target: ${targetNames[0]}${ANSI_RESET}`);\n    return targetNames[0];\n  }\n\n  return search<string>({\n    message: 'Select a target (type to search)',\n    source: async (term) => {\n      const filtered = term\n        ? targetNames.filter((t) => t.toLowerCase().includes(term.toLowerCase()))\n        : targetNames;\n      return filtered.map((t) => {\n        const def = definitions.find((d) => d.name === t);\n        return {\n          name: t,\n          value: t,\n          description: def ? `provider: ${def.provider}` : undefined,\n        };\n      });\n    },\n  });\n}\n\nasync function findTargetsFile(\n  cwd: string,\n  repoRoot: string,\n  evalFilePath?: string,\n): Promise<string | undefined> {\n  // Build directory chain: eval file dir → cwd → repoRoot (mirrors discoverTargetsFile)\n  const dirsToSearch: string[] = [];\n\n  if (evalFilePath) {\n    const evalDir = path.dirname(evalFilePath);\n    if (!dirsToSearch.includes(evalDir)) {\n      dirsToSearch.push(evalDir);\n    }\n  }\n\n  if (!dirsToSearch.includes(cwd)) {\n    dirsToSearch.push(cwd);\n  }\n\n  if (repoRoot !== cwd && !dirsToSearch.includes(repoRoot)) {\n    dirsToSearch.push(repoRoot);\n  }\n\n  for (const dir of dirsToSearch) {\n    for (const candidate of TARGET_FILE_CANDIDATES) {\n      const fullPath = `${dir}/${candidate}`;\n      if (await fileExists(fullPath)) {\n        return fullPath;\n      }\n    }\n  }\n\n  return undefined;\n}\n\nasync function promptAdvancedOptions(): Promise<{\n  workers: number;\n  cache: boolean;\n}> {\n  const customize = await confirm({\n    message: 'Configure advanced options?',\n    default: false,\n  });\n\n  if (!customize) {\n    return { workers: 3, cache: false };\n  }\n\n  const workers =\n    (await number({\n      message: 'Number of parallel workers (1-50)',\n      default: 3,\n      min: 1,\n      max: 50,\n    })) ?? 3;\n\n  const cache = await confirm({\n    message: 'Enable response cache?',\n    default: false,\n  });\n\n  return { workers, cache };\n}\n\nasync function promptReviewAndConfirm(config: InteractiveConfig, cwd: string): Promise<boolean> {\n  const evalDisplay = config.evalPaths\n    .map((p) => {\n      const rel = p.startsWith(cwd) ? p.slice(cwd.length + 1) : p;\n      return `  ${rel}`;\n    })\n    .join('\\n');\n\n  console.log(`\\n${ANSI_BOLD}Review Configuration${ANSI_RESET}`);\n  console.log(`${ANSI_DIM}${'─'.repeat(40)}${ANSI_RESET}`);\n  console.log(`${ANSI_GREEN}Eval files:${ANSI_RESET}\\n${evalDisplay}`);\n  console.log(`${ANSI_GREEN}Target:${ANSI_RESET}    ${config.target}`);\n  console.log(`${ANSI_GREEN}Workers:${ANSI_RESET}   ${config.workers}`);\n  console.log(`${ANSI_GREEN}Cache:${ANSI_RESET}     ${config.cache ? 'yes' : 'no'}`);\n  console.log(`${ANSI_DIM}${'─'.repeat(40)}${ANSI_RESET}`);\n\n  return confirm({\n    message: 'Run evaluation with this configuration?',\n    default: true,\n  });\n}\n\nasync function executeConfig(\n  config: InteractiveConfig,\n  opts?: { resumeOutputDir?: string },\n): Promise<void> {\n  const cwd = process.cwd();\n  const rawOptions: Record<string, unknown> = {\n    target: config.target,\n    workers: config.workers,\n    cache: config.cache,\n    ...(opts?.resumeOutputDir ? { output: opts.resumeOutputDir, resume: true } : {}),\n    agentTimeout: 120,\n    maxRetries: 2,\n    verbose: false,\n    keepWorkspaces: false,\n    cleanupWorkspaces: false,\n    trace: false,\n  };\n\n  const result = await runEvalCommand({\n    testFiles: [...config.evalPaths],\n    rawOptions,\n  });\n\n  // Persist config with the resolved artifact dir so the wizard can offer\n  // \"Resume last run\" on the next invocation. Done after a successful run so\n  // the saved outputDir always points at a real run manifest.\n  if (result) {\n    await saveLastConfig({\n      timestamp: new Date().toISOString(),\n      cwd,\n      evalPaths: config.evalPaths,\n      target: config.target,\n      workers: config.workers,\n      cache: config.cache,\n      outputDir: path.dirname(result.outputPath),\n    });\n  }\n\n  // Prompt to retry errors when execution errors were detected in a TTY\n  if (result && result.executionErrorCount > 0 && process.stdin.isTTY) {\n    await promptRetryErrors(config, result.outputPath);\n  }\n}\n\nasync function promptRetryErrors(config: InteractiveConfig, outputPath: string): Promise<void> {\n  const shouldRetry = await confirm({\n    message: `Retry ${ANSI_BOLD}execution errors${ANSI_RESET} from this run?`,\n    default: true,\n  });\n\n  if (!shouldRetry) {\n    return;\n  }\n\n  console.log(`\\n${ANSI_DIM}Retrying execution errors...${ANSI_RESET}\\n`);\n\n  const rawOptions: Record<string, unknown> = {\n    target: config.target,\n    workers: config.workers,\n    cache: config.cache,\n    retryErrors: outputPath,\n    out: outputPath,\n    agentTimeout: 120,\n    maxRetries: 2,\n    verbose: false,\n    keepWorkspaces: false,\n    cleanupWorkspaces: false,\n    trace: false,\n  };\n\n  const retryResult = await runEvalCommand({\n    testFiles: [...config.evalPaths],\n    rawOptions,\n  });\n\n  // Allow chained retries if there are still errors\n  if (retryResult && retryResult.executionErrorCount > 0 && process.stdin.isTTY) {\n    await promptRetryErrors(config, retryResult.outputPath);\n  }\n}\n","import { mkdir, readFile, writeFile } from 'node:fs/promises';\nimport path from 'node:path';\nimport { getAgentvConfigDir } from '@agentv/core';\n\nconst CONFIG_DIR = getAgentvConfigDir();\nconst LAST_CONFIG_PATH = path.join(CONFIG_DIR, 'last-config.json');\n\nexport interface LastConfig {\n  readonly timestamp: string;\n  readonly cwd: string;\n  readonly evalPaths: readonly string[];\n  readonly target: string;\n  readonly workers: number;\n  readonly cache: boolean;\n  /**\n   * Resolved artifact directory of the last completed wizard run. Used to\n   * power the wizard's \"Resume last run\" entry. Optional for backward\n   * compatibility with configs saved before this field existed.\n   */\n  readonly outputDir?: string;\n}\n\nexport async function loadLastConfig(): Promise<LastConfig | undefined> {\n  try {\n    const content = await readFile(LAST_CONFIG_PATH, 'utf-8');\n    return JSON.parse(content) as LastConfig;\n  } catch {\n    return undefined;\n  }\n}\n\nexport async function saveLastConfig(config: LastConfig): Promise<void> {\n  await mkdir(CONFIG_DIR, { recursive: true });\n  await writeFile(LAST_CONFIG_PATH, JSON.stringify(config, null, 2), 'utf-8');\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,kBAAkB;AAC3B,OAAOA,WAAU;AAEjB,SAAS,UAAU,SAAS,QAAQ,QAAQ,cAAc;;;ACH1D,SAAS,OAAO,UAAU,iBAAiB;AAC3C,OAAO,UAAU;AAGjB,IAAM,aAAa,mBAAmB;AACtC,IAAM,mBAAmB,KAAK,KAAK,YAAY,kBAAkB;AAiBjE,eAAsB,iBAAkD;AACtE,MAAI;AACF,UAAM,UAAU,MAAM,SAAS,kBAAkB,OAAO;AACxD,WAAO,KAAK,MAAM,OAAO;AAAA,EAC3B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,eAAsB,eAAe,QAAmC;AACtE,QAAM,MAAM,YAAY,EAAE,WAAW,KAAK,CAAC;AAC3C,QAAM,UAAU,kBAAkB,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AAC5E;;;ADjBA,IAAM,YAAY;AAClB,IAAM,WAAW;AACjB,IAAM,YAAY;AAClB,IAAM,aAAa;AACnB,IAAM,aAAa;AAYnB,eAAsB,0BAAyC;AAC7D,QAAM,MAAM,QAAQ,IAAI;AAExB,UAAQ,IAAI;AAAA,EAAK,SAAS,GAAG,SAAS,0BAA0B,UAAU;AAAA,CAAI;AAE9E,QAAM,aAAa,MAAM,eAAe;AACxC,QAAM,SAAS,MAAM,eAAe,UAAU;AAE9C,MAAI,WAAW,QAAQ;AACrB;AAAA,EACF;AAEA,MAAI,WAAW,YAAY,YAAY,WAAW;AAChD,UAAM,cAAcC,MAAK,SAAS,KAAK,WAAW,SAAS,KAAK,WAAW;AAC3E,YAAQ,IAAI;AAAA,EAAK,QAAQ,mBAAmB,WAAW,MAAM,UAAU;AAAA,CAAI;AAC3E,UAAM;AAAA,MACJ;AAAA,QACE,WAAW,WAAW;AAAA,QACtB,QAAQ,WAAW;AAAA,QACnB,SAAS,WAAW;AAAA,QACpB,OAAO,WAAW;AAAA,MACpB;AAAA,MACA,EAAE,iBAAiB,WAAW,UAAU;AAAA,IAC1C;AACA;AAAA,EACF;AAEA,MAAI,WAAW,WAAW,YAAY;AACpC,YAAQ,IAAI;AAAA,EAAK,QAAQ,kCAAkC,UAAU;AAAA,CAAI;AACzE,UAAM,cAAc;AAAA,MAClB,WAAW,WAAW;AAAA,MACtB,QAAQ,WAAW;AAAA,MACnB,SAAS,WAAW;AAAA,MACpB,OAAO,WAAW;AAAA,IACpB,CAAC;AACD;AAAA,EACF;AAGA,QAAM,SAAS,MAAM,oBAAoB,GAAG;AAC5C,MAAI,CAAC,QAAQ;AACX;AAAA,EACF;AAGA,QAAM,YAAY,MAAM,uBAAuB,QAAQ,GAAG;AAC1D,MAAI,CAAC,WAAW;AACd;AAAA,EACF;AAEA,QAAM,cAAc,MAAM;AAC5B;AAEA,eAAe,eACb,YAC8C;AAE9C,QAAM,UAA4E,CAAC;AAGnF,MAAI,YAAY,WAAW;AACzB,UAAM,YAAY,8BAA8B,WAAW,SAAS;AACpE,QAAI,aAAa,WAAW,SAAS,GAAG;AACtC,YAAM,WAAWA,MAAK,SAAS,WAAW,SAAS;AACnD,cAAQ,KAAK;AAAA,QACX,MAAM;AAAA,QACN,OAAO;AAAA,QACP,aAAa,GAAG,QAAQ,aAAa,WAAW,MAAM;AAAA,MACxD,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,YAAY;AACd,UAAM,YAAY,WAAW,UAAU;AACvC,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,OAAO;AAAA,MACP,aAAa,GAAG,SAAS,0BAA0B,WAAW,MAAM;AAAA,IACtE,CAAC;AAAA,EACH;AAEA,UAAQ,KAAK,EAAE,MAAM,gCAAyB,OAAO,MAAM,GAAG,EAAE,MAAM,eAAU,OAAO,OAAO,CAAC;AAE/F,SAAO,OAAmB;AAAA,IACxB,SAAS;AAAA,IACT;AAAA,EACF,CAAC;AACH;AAEA,eAAe,oBAAoB,KAAqD;AAEtF,UAAQ,IAAI;AAAA,EAAK,QAAQ,6BAA6B,UAAU,EAAE;AAClE,QAAM,WAAW,MAAM,kBAAkB,GAAG;AAE5C,MAAI,SAAS,WAAW,GAAG;AACzB,YAAQ;AAAA,MACN;AAAA,IAGF;AACA,WAAO;AAAA,EACT;AAEA,UAAQ,IAAI,GAAG,QAAQ,SAAS,SAAS,MAAM,gBAAgB,UAAU;AAAA,CAAI;AAG7E,QAAM,gBAAgB,MAAM,oBAAoB,QAAQ;AACxD,MAAI,cAAc,WAAW,GAAG;AAC9B,YAAQ,IAAI,2BAA2B;AACvC,WAAO;AAAA,EACT;AAGA,QAAM,SAAS,MAAM,sBAAsB,KAAK,cAAc,CAAC,EAAE,IAAI;AAGrE,QAAM,WAAW,MAAM,sBAAsB;AAE7C,SAAO;AAAA,IACL,WAAW,cAAc,IAAI,CAAC,MAAM,EAAE,IAAI;AAAA,IAC1C;AAAA,IACA,GAAG;AAAA,EACL;AACF;AAEA,eAAe,oBACb,UAC+B;AAC/B,QAAM,aAAa,cAAc,QAAQ;AAGzC,MAAI;AAEJ,MAAI,WAAW,SAAS,GAAG;AACzB,UAAM,mBAAmB,MAAM,OAAe;AAAA,MAC5C,SAAS;AAAA,MACT,QAAQ,OAAO,SAAS;AACtB,cAAM,WAAW,OACb,WAAW,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,SAAS,KAAK,YAAY,CAAC,CAAC,IACrE;AACJ,eAAO;AAAA,UACL,EAAE,MAAM,oBAAoB,OAAO,UAAU;AAAA,UAC7C,GAAG,SAAS,IAAI,CAAC,MAAM;AACrB,kBAAM,QAAQ,iBAAiB,UAAU,CAAC,EAAE;AAC5C,mBAAO,EAAE,MAAM,GAAG,CAAC,KAAK,KAAK,QAAQ,QAAQ,IAAI,MAAM,EAAE,KAAK,OAAO,EAAE;AAAA,UACzE,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC;AAED,oBACE,qBAAqB,YAAY,WAAW,iBAAiB,UAAU,gBAAgB;AAAA,EAC3F,OAAO;AACL,oBAAgB;AAAA,EAClB;AAEA,SAAO,SAA6B;AAAA,IAClC,SAAS;AAAA,IACT,SAAS,cAAc,IAAI,CAAC,OAAO;AAAA,MACjC,MAAM,EAAE;AAAA,MACR,OAAO;AAAA,MACP,SAAS,cAAc,UAAU;AAAA;AAAA,IACnC,EAAE;AAAA,IACF,UAAU;AAAA,EACZ,CAAC;AACH;AAEA,eAAe,sBAAsB,KAAa,eAAwC;AACxF,QAAM,WAAW,MAAM,aAAa,GAAG;AAGvC,QAAM,cAAc,MAAM,gBAAgB,KAAK,UAAU,aAAa;AAEtE,MAAI,CAAC,aAAa;AAChB,YAAQ,IAAI,GAAG,QAAQ,+CAA+C,UAAU,EAAE;AAClF,WAAO;AAAA,EACT;AAEA,QAAM,cAAc,MAAM,sBAAsB,WAAW;AAC3D,QAAM,cAAc,gBAAgB,WAAW;AAE/C,MAAI,YAAY,WAAW,GAAG;AAC5B,WAAO;AAAA,EACT;AAEA,MAAI,YAAY,WAAW,GAAG;AAC5B,YAAQ,IAAI,GAAG,QAAQ,iBAAiB,YAAY,CAAC,CAAC,GAAG,UAAU,EAAE;AACrE,WAAO,YAAY,CAAC;AAAA,EACtB;AAEA,SAAO,OAAe;AAAA,IACpB,SAAS;AAAA,IACT,QAAQ,OAAO,SAAS;AACtB,YAAM,WAAW,OACb,YAAY,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,SAAS,KAAK,YAAY,CAAC,CAAC,IACtE;AACJ,aAAO,SAAS,IAAI,CAAC,MAAM;AACzB,cAAM,MAAM,YAAY,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC;AAChD,eAAO;AAAA,UACL,MAAM;AAAA,UACN,OAAO;AAAA,UACP,aAAa,MAAM,aAAa,IAAI,QAAQ,KAAK;AAAA,QACnD;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACH;AAEA,eAAe,gBACb,KACA,UACA,cAC6B;AAE7B,QAAM,eAAyB,CAAC;AAEhC,MAAI,cAAc;AAChB,UAAM,UAAUA,MAAK,QAAQ,YAAY;AACzC,QAAI,CAAC,aAAa,SAAS,OAAO,GAAG;AACnC,mBAAa,KAAK,OAAO;AAAA,IAC3B;AAAA,EACF;AAEA,MAAI,CAAC,aAAa,SAAS,GAAG,GAAG;AAC/B,iBAAa,KAAK,GAAG;AAAA,EACvB;AAEA,MAAI,aAAa,OAAO,CAAC,aAAa,SAAS,QAAQ,GAAG;AACxD,iBAAa,KAAK,QAAQ;AAAA,EAC5B;AAEA,aAAW,OAAO,cAAc;AAC9B,eAAW,aAAa,wBAAwB;AAC9C,YAAM,WAAW,GAAG,GAAG,IAAI,SAAS;AACpC,UAAI,MAAM,WAAW,QAAQ,GAAG;AAC9B,eAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAEA,eAAe,wBAGZ;AACD,QAAM,YAAY,MAAM,QAAQ;AAAA,IAC9B,SAAS;AAAA,IACT,SAAS;AAAA,EACX,CAAC;AAED,MAAI,CAAC,WAAW;AACd,WAAO,EAAE,SAAS,GAAG,OAAO,MAAM;AAAA,EACpC;AAEA,QAAM,UACH,MAAM,OAAO;AAAA,IACZ,SAAS;AAAA,IACT,SAAS;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AAAA,EACP,CAAC,KAAM;AAET,QAAM,QAAQ,MAAM,QAAQ;AAAA,IAC1B,SAAS;AAAA,IACT,SAAS;AAAA,EACX,CAAC;AAED,SAAO,EAAE,SAAS,MAAM;AAC1B;AAEA,eAAe,uBAAuB,QAA2B,KAA+B;AAC9F,QAAM,cAAc,OAAO,UACxB,IAAI,CAAC,MAAM;AACV,UAAM,MAAM,EAAE,WAAW,GAAG,IAAI,EAAE,MAAM,IAAI,SAAS,CAAC,IAAI;AAC1D,WAAO,KAAK,GAAG;AAAA,EACjB,CAAC,EACA,KAAK,IAAI;AAEZ,UAAQ,IAAI;AAAA,EAAK,SAAS,uBAAuB,UAAU,EAAE;AAC7D,UAAQ,IAAI,GAAG,QAAQ,GAAG,SAAI,OAAO,EAAE,CAAC,GAAG,UAAU,EAAE;AACvD,UAAQ,IAAI,GAAG,UAAU,cAAc,UAAU;AAAA,EAAK,WAAW,EAAE;AACnE,UAAQ,IAAI,GAAG,UAAU,UAAU,UAAU,OAAO,OAAO,MAAM,EAAE;AACnE,UAAQ,IAAI,GAAG,UAAU,WAAW,UAAU,MAAM,OAAO,OAAO,EAAE;AACpE,UAAQ,IAAI,GAAG,UAAU,SAAS,UAAU,QAAQ,OAAO,QAAQ,QAAQ,IAAI,EAAE;AACjF,UAAQ,IAAI,GAAG,QAAQ,GAAG,SAAI,OAAO,EAAE,CAAC,GAAG,UAAU,EAAE;AAEvD,SAAO,QAAQ;AAAA,IACb,SAAS;AAAA,IACT,SAAS;AAAA,EACX,CAAC;AACH;AAEA,eAAe,cACb,QACA,MACe;AACf,QAAM,MAAM,QAAQ,IAAI;AACxB,QAAM,aAAsC;AAAA,IAC1C,QAAQ,OAAO;AAAA,IACf,SAAS,OAAO;AAAA,IAChB,OAAO,OAAO;AAAA,IACd,GAAI,MAAM,kBAAkB,EAAE,QAAQ,KAAK,iBAAiB,QAAQ,KAAK,IAAI,CAAC;AAAA,IAC9E,cAAc;AAAA,IACd,YAAY;AAAA,IACZ,SAAS;AAAA,IACT,gBAAgB;AAAA,IAChB,mBAAmB;AAAA,IACnB,OAAO;AAAA,EACT;AAEA,QAAM,SAAS,MAAM,eAAe;AAAA,IAClC,WAAW,CAAC,GAAG,OAAO,SAAS;AAAA,IAC/B;AAAA,EACF,CAAC;AAKD,MAAI,QAAQ;AACV,UAAM,eAAe;AAAA,MACnB,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC;AAAA,MACA,WAAW,OAAO;AAAA,MAClB,QAAQ,OAAO;AAAA,MACf,SAAS,OAAO;AAAA,MAChB,OAAO,OAAO;AAAA,MACd,WAAWA,MAAK,QAAQ,OAAO,UAAU;AAAA,IAC3C,CAAC;AAAA,EACH;AAGA,MAAI,UAAU,OAAO,sBAAsB,KAAK,QAAQ,MAAM,OAAO;AACnE,UAAM,kBAAkB,QAAQ,OAAO,UAAU;AAAA,EACnD;AACF;AAEA,eAAe,kBAAkB,QAA2B,YAAmC;AAC7F,QAAM,cAAc,MAAM,QAAQ;AAAA,IAChC,SAAS,SAAS,SAAS,mBAAmB,UAAU;AAAA,IACxD,SAAS;AAAA,EACX,CAAC;AAED,MAAI,CAAC,aAAa;AAChB;AAAA,EACF;AAEA,UAAQ,IAAI;AAAA,EAAK,QAAQ,+BAA+B,UAAU;AAAA,CAAI;AAEtE,QAAM,aAAsC;AAAA,IAC1C,QAAQ,OAAO;AAAA,IACf,SAAS,OAAO;AAAA,IAChB,OAAO,OAAO;AAAA,IACd,aAAa;AAAA,IACb,KAAK;AAAA,IACL,cAAc;AAAA,IACd,YAAY;AAAA,IACZ,SAAS;AAAA,IACT,gBAAgB;AAAA,IAChB,mBAAmB;AAAA,IACnB,OAAO;AAAA,EACT;AAEA,QAAM,cAAc,MAAM,eAAe;AAAA,IACvC,WAAW,CAAC,GAAG,OAAO,SAAS;AAAA,IAC/B;AAAA,EACF,CAAC;AAGD,MAAI,eAAe,YAAY,sBAAsB,KAAK,QAAQ,MAAM,OAAO;AAC7E,UAAM,kBAAkB,QAAQ,YAAY,UAAU;AAAA,EACxD;AACF;","names":["path","path"]}

package/dist/skills/agentv-bench/SKILL.md

@@ -65,24 +65,28 @@ Before running or optimizing, understand what you're working with.
 
 AgentV supports two evaluation formats:
 
-**EVAL.yaml** (native, full features) — supports workspaces, code graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, multi-provider targets. Use this for agent evaluation.
+**EVAL.yaml** (native, full features) — supports environment recipes, script graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, and multi-provider targets. Use this for agent evaluation.
 
 ```yaml
 # example.eval.yaml
+prompts:
+  - "{{ task }}"
+
+environment:
+  type: host
+  workdir: ./workspace-template
+  setup:
+    command: ["bash", "-lc", "bun install && bun run build"]
+    cwd: "."
+
 tests:
   - id: basic-code-review
-    input: "Review this TypeScript file for bugs and suggest improvements"
-    criteria: "Identifies the null pointer bug on line 12 and suggests a fix"
-    assertions:
+    vars:
+      task: "Review this TypeScript file for bugs and suggest improvements"
+    assert:
       - type: contains
         value: "null"
       - Review identifies the null pointer bug and suggests a concrete fix
-
-workspace:
-  template: ./workspace-template
-  hooks:
-    before_each:
-      reset: fast
 ```
 
 Multi-skill evaluation is handled naturally via input messages — describe the task in the test input, and the agent uses whatever skills it needs.
@@ -113,9 +117,9 @@ Start with 2-3 realistic test cases — the kind of thing a real user would actu
 
 Good assertions are objectively verifiable and have descriptive names. Subjective quality ("the output is good") is better evaluated qualitatively — don't force assertions onto things that need human judgment.
 
-**Grader types** (cheapest to most expensive): `exact`, `contains`, `regex`, `is-json`, `field-accuracy`, `composite`, `code-grader`, `tool-trajectory`, `llm-grader`. See `references/eval-yaml-spec.md` for full config and grading recipes for each type.
+**Grader types** (cheapest to most expensive): `exact`, `contains`, `regex`, `is-json`, `field-accuracy`, `assert-set`, `script`, `tool-trajectory`, `llm-rubric`. See `references/eval-yaml-spec.md` for full config and grading recipes for each type.
 
-Prefer deterministic graders over LLM graders whenever possible. If an assertion can be checked with `contains` or `regex`, don't use `llm-grader`.
+Prefer deterministic graders over LLM graders whenever possible. If an assertion can be checked with `contains` or `regex`, don't use `llm-rubric`.
 
 ---
 
@@ -129,24 +133,24 @@ Each run produces a new `.agentv/results/default/<timestamp>/` directory automat
 
 **User instruction takes priority.** If the user says "run in subagent mode", "use subagent mode", or "use CLI mode", use that mode directly.
 
-If the user has not specified a mode, default to `subagent`.
+If the user has not specified a mode, default to AgentV CLI mode.
 
-| `AGENT_EVAL_MODE` | Mode | How |
-|----------------------|------|-----|
-| `subagent` (default) | **Subagent mode** | Subagent-driven eval — parses eval.yaml, spawns executor + grader subagents. Zero CLI dependency. |
-| `cli` | **AgentV CLI** | `agentv eval <path>` — end-to-end, multi-provider |
+| Mode | When to use | How |
+|------|-------------|-----|
+| **AgentV CLI** (default) | Normal eval runs, multi-provider benchmarking, CI, artifact generation, and Dashboard-compatible results. | `agentv eval <path>` |
+| **Subagent mode** | Explicit user request, no usable CLI/runtime, or rare provider/request-cost constraints. | Read `references/subagent-pipeline.md` first. |
 
-Set `AGENT_EVAL_MODE` in `.env` at the project root as the default when no mode is specified. If absent, default to `subagent`. **User instruction always overrides this.**
+Do not read `AGENT_EVAL_MODE` or another environment variable to decide the default mode. **User instruction always overrides the default.**
 
-**`subagent`** — Parses eval.yaml directly, spawns executor subagents to run each test case in the current workspace, then spawns grader subagents to evaluate all assertion types natively. No CLI or external API calls required. Read `references/subagent-pipeline.md` for the detailed procedure.
+**Subagent mode** — Parses eval.yaml directly, spawns executor subagents to run each test case in the current workspace, then spawns grader subagents to evaluate assertion types natively. This is an opt-in fallback. Read `references/subagent-pipeline.md` for the detailed procedure before using it.
 
-**`cli`** — AgentV CLI handles execution, grading, and artifact generation end-to-end. Works with all providers. Use when you need multi-provider benchmarking or CLI-specific features.
+**AgentV CLI mode** — AgentV CLI handles execution, grading, and artifact generation end-to-end. Works with all providers. Use this unless the user explicitly asks for subagent mode or the CLI path is unavailable.
 
 ### Running evaluations
 
-**AgentV CLI mode** (end-to-end, EVAL.yaml):
+**AgentV CLI mode** (default, end-to-end, EVAL.yaml):
 ```bash
-agentv eval <eval-path> --output .agentv/artifacts/
+agentv eval <eval-path>
 ```
 
 **Subagent mode** — read `references/subagent-pipeline.md` for the detailed procedure. In brief: use `pipeline input` to extract inputs, dispatch one `executor` subagent per test case (all in parallel), then proceed to grading below.
@@ -196,19 +200,19 @@ grading until all responses are present.
 
 **In subagent mode**, grading has three phases. **All three are required — do not stop after phase 1.**
 
-**Phase 1: Code graders** (deterministic, zero-cost)
+**Phase 1: script graders** (deterministic, zero-cost)
 
 ```bash
 agentv pipeline grade <run-dir>
 ```
 
 This evaluates all deterministic assertions against `response.md` files. Two types are handled:
-- **`code-grader` scripts** — external scripts executed against the response (arbitrary logic, any language)
+- **`script-grader` scripts** — external scripts executed against the response (arbitrary logic, any language)
 - **Built-in assertion types** — evaluated in-process: `contains`, `contains-any`, `contains-all`, `icontains`, `regex`, `equals`, `starts-with`, `ends-with`, `is-json`, and variants
 
-Both types are configured by `pipeline input` into `code_graders/<name>.json` and graded by `pipeline grade`. Results are written to `<test-id>/code_grader_results/<name>.json`. Alternatively, pass `--grader-type code` to `pipeline run` to run these inline.
+Both types are configured by `pipeline input` into `script_graders/<name>.json` and graded by `pipeline grade`. Results are written to `<test-id>/script_grader_results/<name>.json`. Alternatively, pass `--grader-type code` to `pipeline run` to run these inline.
 
-**Do not dispatch LLM grader subagents for tests that only have `contains`, `regex`, or other built-in assertions** — `pipeline grade` handles them entirely, at zero cost. To detect which tests need Phase 2, check whether `<test-id>/llm_graders/` contains any `.json` config files — `pipeline input` only writes there for `llm-grader` assertions. Tests with an empty (or missing) `llm_graders/` directory are done after Phase 1.
+**Do not dispatch LLM grader subagents for tests that only have `contains`, `regex`, or other built-in assertions** — `pipeline grade` handles them entirely, at zero cost. To detect which tests need Phase 2, check whether `<test-id>/llm_graders/` contains any `.json` config files — `pipeline input` only writes there for `llm-rubric` assertions. Tests with an empty (or missing) `llm_graders/` directory are done after Phase 1.
 
 **Phase 2: LLM grading** (semantic — do NOT skip this phase)
 
@@ -242,7 +246,7 @@ agentv pipeline bench <run-dir>
 agentv results validate <run-dir>
 ```
 
-`pipeline bench` reads LLM grader results from `llm_grader_results/<name>.json` per test automatically, merges with code-grader scores, computes weighted pass_rate, and writes `grading.json` + `index.jsonl` + `summary.json`.
+`pipeline bench` reads LLM grader results from `llm_grader_results/<name>.json` per test automatically, merges with script-grader scores, computes weighted pass_rate, and writes `grading.json` + `index.jsonl` + `summary.json`.
 
 > **Diagnosing `pass_rate=0`:** If `pipeline bench` reports `pass_rate=0` across the board, do **not** assume the tests genuinely failed. First verify the grading pipeline ran correctly: check that `<test-id>/llm_grader_results/<name>.json` exists and is non-empty for each test. If these files are absent or empty, the grader subagents failed to produce output (most common cause: `agents/grader.md` was not embedded in the subagent prompts — see Phase 2). Treat `pass_rate=0` as a real signal only after confirming grader results exist.
 
@@ -255,11 +259,11 @@ All artifacts use established schemas — see `references/schemas.md` for the fu
 
 Write artifacts to `.agentv/artifacts/` or the iteration directory.
 
-### Workspace features (EVAL.yaml only)
+### Environment features (EVAL.yaml only)
 
-- **Workspace isolation** — clone repos, run setup/teardown hooks (before_all, before_each, after_each, after_all)
-- **Materialization modes** — `pooled` (reuse slots), `temp` (fresh per run), `static` (existing dir)
-- **Multi-repo** — clone multiple repos with sparse checkout and shallow clone support
+- **Environment recipes** — prepare host or Docker testbeds with `type`, `workdir`, and argv-only `setup.command`
+- **Static local override** — use `--workspace-path` only when an existing local directory should be reused as-is
+- **Multi-repo** — materialize repos from setup scripts when a task needs several checkouts
 - **File change tracking** — grade by diffing workspace files before/after agent execution
 
 ---
@@ -276,7 +280,7 @@ Read the JSONL results and look for:
 - **Always-fail tests** — task impossible, eval broken, or assertion misconfigured. Don't optimize against broken evals.
 - **Flaky tests** — non-deterministic results across runs. Investigate before treating failures as real.
 - **Systematic failures** — same failure pattern across multiple tests. This usually points to a missing instruction or wrong approach.
-- **Deterministic upgrade candidates** — `llm-grader` assertions that could be replaced with `contains`, `regex`, or `is-json` (cheaper, faster, more reliable).
+- **Deterministic upgrade candidates** — `llm-rubric` assertions that could be replaced with `contains`, `regex`, or `is-json` (cheaper, faster, more reliable).
 
 ### Dispatch subagents
 
@@ -289,7 +293,7 @@ Read the JSONL results and look for:
 Use CLI tools for deeper investigation:
 ```bash
 agentv inspect <results-file>          # Detailed execution trace inspection
-agentv compare <file-a> <file-b>     # Structured diff between runs
+agentv results compare <file-a> <file-b>     # Structured diff between runs
 ```
 
 Look for: tool call patterns, error recovery behavior, conversation flow, wasted steps.
@@ -360,7 +364,7 @@ After improving:
 
 ### Automated keep/discard
 
-For autonomous iteration, use `agentv compare --json` to automatically decide whether to keep or discard each change based on wins/losses/ties. Read `references/autoresearch.md` for the full decision rules, logging format, and integration with the iteration loop.
+For autonomous iteration, use `agentv results compare --json` to automatically decide whether to keep or discard each change based on wins/losses/ties. Read `references/autoresearch.md` for the full decision rules, logging format, and integration with the iteration loop.
 
 ---
 

package/dist/skills/agentv-bench/agents/analyzer.md

@@ -28,7 +28,7 @@ If `eval-path` is provided, also read the EVAL.yaml to understand grader configu
 
 ### Step 2: Deterministic-Upgrade Analysis
 
-For each grader entry in `scores` where `type` is `"llm-grader"` or `"rubrics"`, inspect the `reasoning` and `assertions` fields for patterns that indicate a deterministic assertion would suffice:
+For each grader entry in `scores` where `type` is `"llm-rubric"` or `"rubrics"`, inspect the `reasoning` and `assertions` fields for patterns that indicate a deterministic assertion would suffice:
 
 | Signal | Detection | Suggested Upgrade |
 |--------|-----------|-------------------|
@@ -48,7 +48,7 @@ For each grader entry in `scores` where `type` is `"llm-grader"` or `"rubrics"`,
 
 ### Step 3: Weak Assertion Detection
 
-Scan the EVAL.yaml `assertions` entries (if `eval-path` provided) and the `reasoning` fields in results for weak assertions:
+Scan the EVAL.yaml `assert` entries (if `eval-path` provided) and the `reasoning` fields in results for weak assertions:
 
 | Weakness | Detection | Improvement |
 |----------|-----------|-------------|
@@ -56,7 +56,7 @@ Scan the EVAL.yaml `assertions` entries (if `eval-path` provided) and the `reaso
 | Tautological | Contains "is correct", "is good", "works properly", "is valid" without specifying what correct/good means | Define explicit pass/fail conditions |
 | Compound criteria | Single assertion checks multiple independent things (uses "and", "also", "additionally" joining distinct checks) | Split into separate assertions, one per concern |
 | Missing expected value | `type: equals` or `type: contains` without a `value` field | Add the expected value |
-| Overly broad LLM-grader | LLM-grader with no rubric items, just a single vague `prompt` string | Convert to `type: rubrics` with enumerated criteria, or use deterministic checks |
+| Overly broad LLM-grader | LLM-grader with no rubric items, just a single vague `prompt` string | Convert to `type: llm-rubric` with enumerated criteria, or use deterministic checks |
 
 ### Step 4: Cost/Quality Signals
 
@@ -94,7 +94,7 @@ Produce a structured report in this exact format:
 
 | # | Test ID | Grader | Current Type | Evidence | Suggested Type | Suggested Config |
 |---|---------|-----------|-------------|----------|----------------|-----------------|
-| 1 | <test_id> | <grader name> | llm-grader | <brief evidence> | contains | `value: "exact string"` |
+| 1 | <test_id> | <grader name> | llm-rubric | <brief evidence> | contains | `value: "exact string"` |
 
 ### Weak Assertions
 
@@ -122,8 +122,8 @@ If a section has no findings, include the header with "None found." underneath.
 
 - **Be specific:** Every suggestion must include the test case ID, grader name, evidence from the results, and a concrete replacement config.
 - **Be conservative:** Only suggest deterministic upgrades when the pattern is clear and consistent. Partial or ambiguous evidence should be noted but not acted on.
-- **Prioritize by impact:** Order suggestions by estimated cost savings (`llm-grader` → deterministic saves the most).
-- **Handle all grader types:** Process `code-grader`, `tool-trajectory`, `llm-grader`, `rubrics`, `composite`, and all deterministic types. Only LLM-based types are candidates for deterministic upgrades.
+- **Prioritize by impact:** Order suggestions by estimated cost savings (`llm-rubric` → deterministic saves the most).
+- **Handle all grader types:** Process `script`, `tool-trajectory`, `llm-rubric`, `rubrics`, `assert-set`, and all deterministic types. Only LLM-based types are candidates for deterministic upgrades.
 - **Multi-provider awareness:** When results span multiple targets, note if a suggestion applies to all targets or is target-specific.
 - **No false positives:** It is better to miss a suggestion than to recommend an incorrect upgrade. If unsure, add the finding to a "Needs Review" subsection with your reasoning.
 

package/dist/skills/agentv-bench/agents/comparator.md

@@ -24,8 +24,7 @@ You are the Blind Comparator for AgentV's evaluation workflow. Your job is to co
 You will receive:
 - `outputs`: Array of evaluation outputs to compare. Each contains:
   - `target_id`: The provider/configuration identifier (DO NOT read this during scoring)
-  - `answer`: The candidate response text
-  - `evaluator_results`: Array of grader scores and details (code-grader, tool-trajectory, llm-grader, deterministic)
+  - `evaluator_results`: Array of grader scores and details (script, tool-trajectory, llm-rubric, deterministic)
   - `workspace_changes`: File changes made during workspace evaluation (if applicable)
   - `tool_calls`: Tool invocations and results from multi-turn conversations (if applicable)
   - `conversation`: Full multi-turn conversation history (if applicable)
@@ -91,9 +90,9 @@ For each content criterion, define:
 
 **Grader-Specific Scoring** — when grader results are present:
 
-- **code-grader**: Factor in pass/fail results, test coverage, assertion hit rates
+- **script-grader**: Factor in pass/fail results, test coverage, assertion hit rates
 - **tool-trajectory**: Factor in tool call accuracy, sequence correctness, unnecessary tool calls
-- **llm-grader**: Factor in existing LLM grader scores as a reference signal (not as ground truth)
+- **llm-rubric**: Factor in existing LLM grader scores as a reference signal (not as ground truth)
 - **deterministic**: Factor in exact match / keyword hit rates
 
 ### Phase 3: Scoring
@@ -236,7 +235,7 @@ Also produce a human-readable markdown summary:
 - **Be evidence-based**: Every score must cite specific evidence from the output.
 - **Evaluate substance over style**: Correct, complete answers with rough formatting score higher than polished but incorrect answers.
 - **Handle missing data gracefully**: If an output lacks workspace changes or tool calls but others have them, score what is present — do not penalize for data the target wasn't expected to produce.
-- **Respect grader signals**: When code-grader or tool-trajectory results exist, they represent objective ground truth. Weight these heavily.
+- **Respect grader signals**: When script-grader or tool-trajectory results exist, they represent objective ground truth. Weight these heavily.
 
 ## Edge Cases
 

package/dist/skills/agentv-bench/agents/grader.md

@@ -3,7 +3,7 @@ name: grader
 description: >-
   Grade a candidate response for an AgentV evaluation test case. Evaluates all
   assertion types natively — deterministic checks via string operations, LLM grading
-  via Claude's own reasoning, code-grader via Bash script execution. Zero CLI dependency.
+  via Claude's own reasoning, script-grader via Bash script execution. Zero CLI dependency.
   Dispatch this agent after a candidate completes a test case.
 model: inherit
 color: yellow
@@ -12,7 +12,7 @@ tools: ["Read", "Bash", "Glob", "Grep", "Write"]
 
 You are the grader for an AgentV evaluation test case. You have two jobs: **grade the outputs** and **critique the evals themselves**. A passing grade on a weak assertion is worse than useless — it creates false confidence. When you notice an assertion that's trivially satisfied, or an important outcome that no assertion checks, say so.
 
-**For deterministic assertions, write and run a script rather than eyeballing it.** Scripts are faster, more reliable, and can be reused. Use LLM reasoning only for assertions that genuinely require semantic understanding (`llm-grader`, `rubric`).
+**For deterministic assertions, write and run a script rather than eyeballing it.** Scripts are faster, more reliable, and can be reused. Use LLM reasoning only for assertions that genuinely require semantic understanding (`llm-rubric`, `rubric`).
 
 **You will receive these parameters:**
 - `eval-path`: Path to the eval YAML file
@@ -63,7 +63,7 @@ For each assertion in the test's `assertions[]`, evaluate it natively based on i
 
 | Type | How to evaluate |
 |------|----------------|
-| `llm-grader` | Read the `prompt` field. Evaluate the response against those criteria. Score 0.0-1.0 with evidence. |
+| `llm-rubric` | Read `value` for free-form criteria, `rubrics` for itemized criteria, or `prompt_content`/`prompt` for a custom grading prompt. Evaluate the response against those criteria. Score 0.0-1.0 with evidence. |
 | `rubric` / `rubrics` | Read rubric items/criteria. Score each item 0.0-1.0. Aggregate as weighted average. |
 
 For LLM-graded types: be rigorous and fair. Score based on substance, not exact wording. If a `criteria` field exists on the test case, use it as additional context for your evaluation. If `expected_output` exists, use it as a reference answer (not as the only correct answer).
@@ -72,7 +72,7 @@ For LLM-graded types: be rigorous and fair. Score based on substance, not exact
 
 | Type | How to evaluate |
 |------|----------------|
-| `code-grader` | Run: `bun <script-path>` or `python <script-path>`. Pass response via file. Parse stdout JSON: `{"score": N, "reason": "..."}` |
+| `script-grader` | Run: `bun <script-path>` or `python <script-path>`. Pass response via file. Parse stdout JSON: `{"score": N, "reason": "..."}` |
 
 **Composite assertions** — evaluate sub-assertions, then aggregate per the configured mode (weighted_average, min, max, all_pass).
 

package/dist/skills/agentv-bench/references/autoresearch.md

@@ -11,7 +11,7 @@ After each iteration, you can automatically decide whether to keep or discard th
 After re-running test cases, compare the new results against the previous iteration's baseline:
 
 ```bash
-agentv compare <baseline>.jsonl <candidate>.jsonl --json
+agentv results compare <baseline>.jsonl <candidate>.jsonl --json
 ```
 
 Where `<baseline>.jsonl` is the `index.jsonl` from the previous best iteration and `<candidate>.jsonl` is the `index.jsonl` from the run you just completed.
@@ -75,7 +75,7 @@ The automated keep/discard replaces the manual compare-and-present cycle (steps
 
 1. Apply change to prompts/skills/config
 2. Re-run all test cases
-3. Run `agentv compare baseline.jsonl candidate.jsonl --json`
+3. Run `agentv results compare baseline.jsonl candidate.jsonl --json`
 4. Apply keep/discard rules → promote or revert
 5. Log the decision
 6. If this is iteration 3, 6, or 9 → present progress to the user (human checkpoint)
@@ -244,7 +244,7 @@ The trajectory chart fetches `iterations.jsonl` directly via HTTP on each auto-r
 
 Apply the automated keep/discard rules from the section above:
 
-1. Run `agentv compare <baseline>.jsonl <candidate>.jsonl --json` where `<baseline>` is the best iteration's `index.jsonl` (or the first run's `index.jsonl` for cycle 1) and `<candidate>` is this cycle's `index.jsonl`.
+1. Run `agentv results compare <baseline>.jsonl <candidate>.jsonl --json` where `<baseline>` is the best iteration's `index.jsonl` (or the first run's `index.jsonl` for cycle 1) and `<candidate>` is this cycle's `index.jsonl`.
 2. If `wins > losses` → **KEEP**.
 3. If `wins <= losses` → **DISCARD**.
 4. If `mean_delta == 0` and the artifact is simpler → **KEEP** (simpler is better at equal performance). Simplicity: for files, compare line count; for directories, compare total size via `du -sb`.

package/dist/skills/agentv-bench/references/description-optimization.md

@@ -35,12 +35,12 @@ it's a natural utterance):
 tests:
   - id: should-trigger-casual-optimize
     input: "ok so I have this agent that keeps failing on the code review tasks, can you help me figure out why and fix it"
-    assertions:
+    assert:
       - type: skill-trigger
         skill: agentv-bench
   - id: should-not-trigger-build-error
     input: "my TypeScript build is failing with type errors in src/auth.ts"
-    assertions:
+    assert:
       - type: skill-trigger
         skill: agentv-bench
         should_trigger: false

package/dist/skills/agentv-bench/references/environment-adaptation.md

@@ -29,28 +29,28 @@ to any platform with skill-discovery mechanisms. All listed providers support sk
 ## Unsupported Providers: Use a Code-Grader
 
 The built-in `skill-trigger` grader covers Claude, Copilot, Pi, Codex and VS Code out
-of the box. For providers with different tool-call formats, write a code-grader that inspects
+of the box. For providers with different tool-call formats, write a script-grader that inspects
 the agent's transcript messages or tool call trace.
 
-A code-grader receives the full evaluation context including the final `output` string,
+A script-grader receives the full evaluation context including the final `output` string,
 transcript `messages`, and structured `trace`. Inspect `messages` or `trace.events` for
 tool calls; reserve `output` for final-answer text checks.
 
 ```yaml
-# Example: code-grader for Codex skill-trigger detection
+# Example: script-grader for Codex skill-trigger detection
 tests:
   - id: should-trigger-codex
     input: "Analyze this CSV file"
-    assertions:
-      - type: code-grader
+    assert:
+      - type: script
         command: [bun, run, ./judges/codex-skill-trigger.ts]
 ```
 
 ```typescript
 // judges/codex-skill-trigger.ts
-import { defineCodeGrader } from '@agentv/sdk';
+import { defineScriptGrader } from '@agentv/sdk';
 
-export default defineCodeGrader(({ messages }) => {
+export default defineScriptGrader(({ messages }) => {
   const skillName = 'csv-analyzer';
   const toolCalls = messages.flatMap((msg) => msg.toolCalls ?? []);
   const firstTool = toolCalls[0];
@@ -65,7 +65,7 @@ export default defineCodeGrader(({ messages }) => {
     if (cmd.includes(skillName)) {
       return {
         score: 1,
-        assertions: [
+        assert: [
           { text: `Skill "${skillName}" triggered via command`, passed: true, evidence: cmd },
         ],
       };
@@ -78,14 +78,14 @@ export default defineCodeGrader(({ messages }) => {
     if (path.includes(skillName)) {
       return {
         score: 1,
-        assertions: [{ text: 'Skill file accessed', passed: true, evidence: path }],
+        assert: [{ text: 'Skill file accessed', passed: true, evidence: path }],
       };
     }
   }
 
   return {
     score: 0,
-    assertions: [
+    assert: [
       {
         text: `First tool was not a skill invocation for "${skillName}"`,
         passed: false,