agentgrader 1.0.7 → 1.2.0

package/dist/index.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import 'dotenv/config';
 import { cac } from 'cac';
+import { initDb, saveTestCase, saveAgentConfig, getRun, getTraces, getRunsByMatrixId } from '@agentgrader/store';
 import { randomUUID } from 'crypto';
-import { resolve, dirname, isAbsolute } from 'path';
+import { resolve, dirname, isAbsolute, basename } from 'path';
 import { render, Box, Text } from 'ink';
-import { initDb, saveTestCase, saveAgentConfig, getRun, getTraces, getRunsByMatrixId } from '@agentgrader/store';
 import { runSingle, runBenchmark, validateTestCase, TestCaseSchema, AgentConfigSchema } from '@agentgrader/core';
 import { DockerSandboxProvider } from '@agentgrader/sandbox-docker';
 import { AiSdkAgentAdapter } from '@agentgrader/agent-openrouter';
@@ -13,9 +13,158 @@ import { expandMatrix, MatrixSchema, aggregateResults, paretoFront } from '@agen
 import { jsx, jsxs } from 'react/jsx-runtime';
 import { mkdirSync, writeFileSync, readFileSync, readdirSync, statSync, existsSync } from 'fs';
 import { parse, stringify } from 'yaml';
-import { ZodError } from 'zod';
+import { z, ZodError } from 'zod';
 import { execFileSync } from 'child_process';
 
+var CONTENT_PREVIEW_MAX = 200;
+var ANSI = {
+  reset: "\x1B[0m",
+  gray: "\x1B[90m",
+  yellow: "\x1B[33m",
+  cyan: "\x1B[36m",
+  blue: "\x1B[34m"};
+function paint(text, code) {
+  if (!process.stdout.isTTY) return text;
+  return `${code}${text}${ANSI.reset}`;
+}
+function truncateContent(content, full) {
+  if (full || content.length <= CONTENT_PREVIEW_MAX) return content;
+  return `${content.slice(0, CONTENT_PREVIEW_MAX)}...`;
+}
+function normalizeContent(content) {
+  return (content ?? "").trim();
+}
+function formatStepSummary(step, full) {
+  if (!step) return "(no step)";
+  const label = step.tool ? `${step.kind}:${step.tool}` : step.kind;
+  if (!step.content) return label;
+  const preview = truncateContent(step.content.replace(/\n/g, " "), full);
+  return `${label} ${preview}`;
+}
+function stepsByIndex(traces) {
+  const map = /* @__PURE__ */ new Map();
+  for (const step of traces) {
+    map.set(step.stepIndex, step);
+  }
+  return map;
+}
+function stepsDiverge(a, b) {
+  if (!a || !b) return true;
+  if (a.kind !== b.kind) return true;
+  if ((a.tool ?? "") !== (b.tool ?? "")) return true;
+  if (normalizeContent(a.content) !== normalizeContent(b.content)) return true;
+  return false;
+}
+function formatRunStatus(run) {
+  const passed = run.passed === true ? " (passed)" : run.passed === false ? " (failed)" : "";
+  return `${run.status}${passed}`;
+}
+function printRunHeader(label, run) {
+  const tag = label === "A" ? paint(`Run A (${run.id})`, ANSI.cyan) : paint(`Run B (${run.id})`, ANSI.blue);
+  console.log(tag);
+  console.log(`  test case:    ${run.testCaseId}`);
+  console.log(`  agent config: ${run.agentConfigId}`);
+  console.log(`  status:       ${formatRunStatus(run)}`);
+  console.log(`  steps:        ${run.stepsCount}`);
+  console.log(`  cost:         $${run.costUsd.toFixed(4)}`);
+  console.log(`  duration:     ${run.durationMs}ms`);
+  if (run.error) console.log(`  error:        ${run.error}`);
+}
+async function compareCommand(runIdA, runIdB, opts) {
+  const db = initDb();
+  const [runA, runB, tracesA, tracesB] = await Promise.all([
+    getRun(db, runIdA),
+    getRun(db, runIdB),
+    getTraces(db, runIdA),
+    getTraces(db, runIdB)
+  ]);
+  if (!runA) {
+    console.error(`Run not found: ${runIdA}`);
+    process.exit(1);
+  }
+  if (!runB) {
+    console.error(`Run not found: ${runIdB}`);
+    process.exit(1);
+  }
+  console.log("");
+  printRunHeader("A", runA);
+  console.log("");
+  printRunHeader("B", runB);
+  console.log("");
+  if (runA.testCaseId !== runB.testCaseId) {
+    console.log(
+      paint(
+        "\u26A0\uFE0F  Comparing runs of different test cases - step alignment may not be meaningful.",
+        ANSI.yellow
+      )
+    );
+    console.log("");
+  }
+  const mapA = stepsByIndex(tracesA);
+  const mapB = stepsByIndex(tracesB);
+  const maxIndex = Math.max(
+    tracesA.length > 0 ? Math.max(...tracesA.map((s) => s.stepIndex)) : -1,
+    tracesB.length > 0 ? Math.max(...tracesB.map((s) => s.stepIndex)) : -1,
+    -1
+  );
+  if (maxIndex < 0) {
+    console.log("No steps recorded for either run.");
+    return;
+  }
+  const divergentIndices = /* @__PURE__ */ new Set();
+  for (let i = 0; i <= maxIndex; i++) {
+    if (stepsDiverge(mapA.get(i), mapB.get(i))) {
+      divergentIndices.add(i);
+    }
+  }
+  const visibleIndices = /* @__PURE__ */ new Set();
+  if (opts.onlyDiff) {
+    for (const idx of divergentIndices) {
+      visibleIndices.add(idx);
+      if (idx > 0) visibleIndices.add(idx - 1);
+      if (idx < maxIndex) visibleIndices.add(idx + 1);
+    }
+  } else {
+    for (let i = 0; i <= maxIndex; i++) visibleIndices.add(i);
+  }
+  const sortedVisible = [...visibleIndices].sort((a, b) => a - b);
+  if (sortedVisible.length === 0) {
+    console.log("No divergent steps (nothing to show with --only-diff).");
+  } else {
+    console.log("Step comparison:");
+    for (const i of sortedVisible) {
+      const stepA = mapA.get(i);
+      const stepB = mapB.get(i);
+      const divergent = stepsDiverge(stepA, stepB);
+      if (divergent) {
+        console.log(paint(`[step ${i}] DIVERGENT`, ANSI.yellow));
+        console.log(`  A: ${formatStepSummary(stepA, opts.full ?? false)}`);
+        console.log(`  B: ${formatStepSummary(stepB, opts.full ?? false)}`);
+      } else {
+        const line = formatStepSummary(stepA ?? stepB, opts.full ?? false);
+        console.log(paint(`[step ${i}] (same)`, ANSI.gray));
+        console.log(`  ${line}`);
+      }
+    }
+  }
+  const totalSteps = maxIndex + 1;
+  const diffCount = divergentIndices.size;
+  let firstDivergence = null;
+  for (let i = 0; i <= maxIndex; i++) {
+    if (divergentIndices.has(i)) {
+      firstDivergence = i;
+      break;
+    }
+  }
+  console.log("");
+  console.log(`${diffCount} of ${totalSteps} step(s) differ.`);
+  if (firstDivergence !== null) {
+    console.log(`First divergence at step ${firstDivergence}.`);
+  } else {
+    console.log("No divergence detected.");
+  }
+  console.log("");
+}
 var CONFIG_COL_WIDTH = 24;
 var CONFIG_LABEL_MAX = 20;
 function truncateLabel(name, max = CONFIG_LABEL_MAX) {
@@ -179,6 +328,125 @@ function loadAgentConfig(yamlPath) {
   }
   return config;
 }
+
+// src/lib/resolve-agent-config-paths.ts
+function globToRegex(glob) {
+  const escaped = glob.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+  return new RegExp(`^${escaped}$`);
+}
+function collectYamlFilesRecursive(dir) {
+  const files = [];
+  for (const entry of readdirSync(dir)) {
+    if (entry.startsWith(".")) continue;
+    const fullPath = resolve(dir, entry);
+    const stat = statSync(fullPath);
+    if (stat.isDirectory()) {
+      files.push(...collectYamlFilesRecursive(fullPath));
+    } else if (entry.endsWith(".yaml") || entry.endsWith(".yml")) {
+      files.push(fullPath);
+    }
+  }
+  return files;
+}
+function findAgentConfigYamlFilesInDir(dir) {
+  const resolvedDir = resolve(dir);
+  const files = [];
+  for (const entry of readdirSync(resolvedDir)) {
+    if (entry.startsWith(".")) continue;
+    const fullPath = resolve(resolvedDir, entry);
+    if (!statSync(fullPath).isFile()) continue;
+    if (entry.endsWith(".yaml") || entry.endsWith(".yml")) {
+      files.push(fullPath);
+    }
+  }
+  return files.sort();
+}
+function expandAgentConfigGlob(globPattern, baseDir) {
+  const base = resolve(baseDir);
+  const normalized = globPattern.replace(/^\.\//, "");
+  if (normalized.includes("**")) {
+    const [prefix, suffixPart] = normalized.split("**");
+    const searchRoot = prefix.replace(/\/$/, "") ? resolve(base, prefix.replace(/\/$/, "")) : base;
+    const suffix = (suffixPart ?? "").replace(/^\//, "") || "*.yaml";
+    const regex2 = globToRegex(suffix);
+    return collectYamlFilesRecursive(searchRoot).filter((filePath) => regex2.test(basename(filePath))).sort();
+  }
+  const slashIdx = normalized.lastIndexOf("/");
+  const cwd = slashIdx === -1 ? base : resolve(base, normalized.slice(0, slashIdx));
+  const fileGlob = slashIdx === -1 ? normalized : normalized.slice(slashIdx + 1);
+  const regex = globToRegex(fileGlob);
+  return readdirSync(cwd).filter((entry) => {
+    if (entry.startsWith(".")) return false;
+    const fullPath = resolve(cwd, entry);
+    return statSync(fullPath).isFile() && regex.test(entry);
+  }).map((entry) => resolve(cwd, entry)).sort();
+}
+function resolveAgentConfigPathList(input) {
+  const paths = /* @__PURE__ */ new Set();
+  if (input.commaSeparated) {
+    for (const part of input.commaSeparated.split(",")) {
+      const trimmed = part.trim();
+      if (trimmed) paths.add(resolve(trimmed));
+    }
+  }
+  if (input.dir) {
+    for (const file of findAgentConfigYamlFilesInDir(input.dir)) {
+      paths.add(file);
+    }
+  }
+  const baseDir = input.relativeTo ? resolve(input.relativeTo) : process.cwd();
+  if (input.explicitPaths) {
+    for (const p of input.explicitPaths) {
+      paths.add(resolve(baseDir, p));
+    }
+  }
+  if (input.globs) {
+    for (const pattern of input.globs) {
+      for (const file of expandAgentConfigGlob(pattern, baseDir)) {
+        paths.add(file);
+      }
+    }
+  }
+  const sorted = [...paths].sort();
+  if (sorted.length === 0) {
+    throw new Error("No agent config YAML files found.");
+  }
+  return sorted;
+}
+function loadAgentConfigsFromPaths(paths) {
+  return paths.map((p) => loadAgentConfig(p));
+}
+
+// src/lib/load-bench-manifest.ts
+var AgentsSchema = z.object({
+  paths: z.array(z.string()).optional(),
+  glob: z.union([z.string(), z.array(z.string())]).optional()
+}).refine((data) => (data.paths?.length ?? 0) > 0 || data.glob !== void 0, {
+  message: "agents must specify at least one of paths or glob"
+});
+var BenchManifestSchema = z.object({
+  name: z.string().optional(),
+  suite: z.string(),
+  agents: AgentsSchema,
+  concurrency: z.number().optional()
+});
+function loadBenchManifest(yamlPath) {
+  const path = resolve(yamlPath);
+  const raw = parse(readFileSync(path, "utf-8"));
+  return BenchManifestSchema.parse(raw);
+}
+function resolveManifestAgentConfigPaths(manifest, manifestPath) {
+  const manifestDir = dirname(resolve(manifestPath));
+  const globs = manifest.agents.glob ? Array.isArray(manifest.agents.glob) ? manifest.agents.glob : [manifest.agents.glob] : void 0;
+  return resolveAgentConfigPathList({
+    explicitPaths: manifest.agents.paths,
+    globs,
+    relativeTo: manifestDir
+  });
+}
+function resolveManifestSuiteDir(manifest, manifestPath) {
+  return resolve(dirname(resolve(manifestPath)), manifest.suite);
+}
 function loadMatrix(yamlPath) {
   const path = resolve(yamlPath);
   const fileContent = readFileSync(path, "utf-8");
@@ -272,26 +540,57 @@ function findTestCaseYamlFiles(dir) {
   return files;
 }
 async function runBenchCommand(opts) {
-  const suiteDir = resolve(opts.suite);
-  const concurrency = opts.concurrency || 2;
+  let suiteDir;
+  let concurrency = opts.concurrency ?? 2;
   let agentConfigs;
   let matrixId;
-  if (opts.matrix) {
-    const matrix = loadMatrix(opts.matrix);
-    agentConfigs = expandMatrix(matrix);
-    matrixId = randomUUID();
+  if (opts.manifest) {
+    const manifestPath = resolve(opts.manifest);
+    const manifest = loadBenchManifest(manifestPath);
+    suiteDir = resolveManifestSuiteDir(manifest, manifestPath);
+    if (manifest.concurrency !== void 0 && opts.concurrency === void 0) {
+      concurrency = manifest.concurrency;
+    }
+    if (opts.matrix) {
+      throw new Error("Use either --manifest or --matrix, not both.");
+    }
+    const configPaths = resolveManifestAgentConfigPaths(manifest, manifestPath);
+    agentConfigs = loadAgentConfigsFromPaths(configPaths);
     console.log(
-      `Matrix "${matrix.name}" expanded to ${agentConfigs.length} agent config(s) (matrixId: ${matrixId})`
+      `Bench manifest "${manifest.name ?? manifestPath}" loaded ${agentConfigs.length} agent config(s) from ${configPaths.length} file(s).`
     );
-  } else if (opts.configs) {
-    const configPaths = opts.configs.split(",").map((c) => resolve(c.trim()));
-    agentConfigs = configPaths.map((p) => loadAgentConfig(p));
   } else {
-    throw new Error("Either --configs or --matrix must be provided.");
+    if (!opts.suite) {
+      throw new Error("--suite is required unless --manifest is provided.");
+    }
+    suiteDir = resolve(opts.suite);
+    if (opts.matrix) {
+      if (opts.configs || opts.configsDir) {
+        throw new Error("Use either --matrix or --configs/--configs-dir, not both.");
+      }
+      const matrix = loadMatrix(opts.matrix);
+      agentConfigs = expandMatrix(matrix);
+      matrixId = randomUUID();
+      console.log(
+        `Matrix "${matrix.name}" expanded to ${agentConfigs.length} agent config(s) (matrixId: ${matrixId})`
+      );
+    } else {
+      const configPaths = resolveAgentConfigPathList({
+        commaSeparated: opts.configs,
+        dir: opts.configsDir
+      });
+      agentConfigs = loadAgentConfigsFromPaths(configPaths);
+      if (opts.configsDir) {
+        console.log(`Loaded ${agentConfigs.length} agent config(s) from ${opts.configsDir}.`);
+      }
+    }
+  }
+  if (agentConfigs.length === 0) {
+    throw new Error("No agent configs to benchmark.");
   }
   const yamlFiles = findTestCaseYamlFiles(suiteDir);
   if (yamlFiles.length === 0) {
-    console.error(`No test cases found in suite directory: ${opts.suite}`);
+    console.error(`No test cases found in suite directory: ${suiteDir}`);
     process.exit(1);
   }
   const testCases = [];
@@ -879,25 +1178,51 @@ cli.command("run <testCase>", "Run a single agent test case").option("--config <
     process.exit(1);
   }
 });
-cli.command("bench", "Run a benchmark matrix of multiple test cases and configs").option("--configs <configs>", "Comma-separated paths to AgentConfig YAML files").option("--config <config>", "Alias for --configs (single config path)").option("--suite <suite>", "Path to test suite directory containing test cases").option("--concurrency <concurrency>", "Number of parallel sandbox executions", { default: 2 }).option(
+cli.command("bench", "Run a benchmark matrix of multiple test cases and configs").option("--configs <configs>", "Comma-separated paths to AgentConfig YAML files").option("--config <config>", "Alias for --configs (single config path)").option(
+  "--configs-dir <dir>",
+  "Directory of AgentConfig YAML files (all .yaml/.yml files in the folder)"
+).option(
+  "--manifest <manifest>",
+  "Path to a bench manifest YAML (suite + agent paths/glob in one file)"
+).option("--suite <suite>", "Path to test suite directory containing test cases").option("--concurrency <concurrency>", "Number of parallel sandbox executions", { default: 2 }).option(
   "--matrix <matrix>",
   "Path to an optimizer matrix YAML file - expands into agent configs and prints a Pareto summary afterwards (alternative to --configs)"
-).example("agr bench --suite tasks --configs agent.yaml,agent-openrouter.yaml").example("agr bench --suite tasks --matrix optimizer-matrix.yaml").action(async (options) => {
+).example("agr bench --manifest bench.yaml").example("agr bench --suite tasks --configs-dir ./agents").example("agr bench --suite tasks --configs agent.yaml,agent-openrouter.yaml").example("agr bench --suite tasks --matrix optimizer-matrix.yaml").action(async (options) => {
   if (!options.configs && options.config) {
     options.configs = options.config;
   }
-  if (!options.suite || !options.configs && !options.matrix) {
+  const agentSourceCount = [
+    options.configs,
+    options.configsDir,
+    options.matrix,
+    options.manifest
+  ].filter(Boolean).length;
+  if (options.manifest) {
+    if (agentSourceCount > 1) {
+      console.error(
+        "Error: --manifest cannot be combined with --configs, --configs-dir, or --matrix."
+      );
+      process.exit(1);
+    }
+  } else if (!options.suite || agentSourceCount === 0) {
     console.error(
-      "Error: --suite and either --configs, --config, or --matrix are required for benchmarking."
+      "Error: provide --manifest, or --suite with one of --configs, --config, --configs-dir, or --matrix."
+    );
+    process.exit(1);
+  } else if (agentSourceCount > 1) {
+    console.error(
+      "Error: use only one agent source: --configs, --configs-dir, or --matrix."
     );
     process.exit(1);
   }
   try {
     await runBenchCommand({
       configs: options.configs,
+      configsDir: options.configsDir,
       suite: options.suite,
       concurrency: Number(options.concurrency),
-      matrix: options.matrix
+      matrix: options.matrix,
+      manifest: options.manifest
     });
   } catch (err) {
     console.error(`Error executing benchmark: ${err.message}`);
@@ -943,6 +1268,17 @@ cli.command("trace <runId>", "Show the step trace and metrics for a single run")
     process.exit(1);
   }
 });
+cli.command("compare <runIdA> <runIdB>", "Compare the step traces of two runs side by side").option("--full", "Print full step content without truncation").option(
+  "--only-diff",
+  "Show only divergent steps plus one step of context before and after each"
+).example("agr compare <runIdA> <runIdB> --only-diff").action(async (runIdA, runIdB, options) => {
+  try {
+    await compareCommand(runIdA, runIdB, options);
+  } catch (err) {
+    console.error(`Error executing compare: ${err.message}`);
+    process.exit(1);
+  }
+});
 cli.help();
 try {
   cli.parse();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentgrader",
-  "version": "1.0.7",
+  "version": "1.2.0",
   "description": "CLI for the Agentgrader benchmarking framework — run and bench coding agents",
   "license": "MIT",
   "type": "module",