agentgrader 1.0.6 → 1.1.0

package/dist/index.js

@@ -2,7 +2,7 @@
 import 'dotenv/config';
 import { cac } from 'cac';
 import { randomUUID } from 'crypto';
-import { resolve, dirname, isAbsolute } from 'path';
+import { resolve, dirname, isAbsolute, basename } from 'path';
 import { render, Box, Text } from 'ink';
 import { initDb, saveTestCase, saveAgentConfig, getRun, getTraces, getRunsByMatrixId } from '@agentgrader/store';
 import { runSingle, runBenchmark, validateTestCase, TestCaseSchema, AgentConfigSchema } from '@agentgrader/core';
@@ -13,9 +13,15 @@ import { expandMatrix, MatrixSchema, aggregateResults, paretoFront } from '@agen
 import { jsx, jsxs } from 'react/jsx-runtime';
 import { mkdirSync, writeFileSync, readFileSync, readdirSync, statSync, existsSync } from 'fs';
 import { parse, stringify } from 'yaml';
-import { ZodError } from 'zod';
+import { z, ZodError } from 'zod';
 import { execFileSync } from 'child_process';
 
+var CONFIG_COL_WIDTH = 24;
+var CONFIG_LABEL_MAX = 20;
+function truncateLabel(name, max = CONFIG_LABEL_MAX) {
+  if (name.length <= max) return name;
+  return `${name.slice(0, max - 1)}\u2026`;
+}
 var Dashboard = ({ runs, testCases, configs, isFinished }) => {
   let totalCost = 0;
   let totalSteps = 0;
@@ -67,7 +73,7 @@ var Dashboard = ({ runs, testCases, configs, isFinished }) => {
           r.testCaseId
         ] }),
         /* @__PURE__ */ jsx(Text, { color: "gray", children: " with " }),
-        /* @__PURE__ */ jsx(Text, { color: "blue", children: r.agentConfigId }),
+        /* @__PURE__ */ jsx(Text, { color: "blue", wrap: "truncate-end", children: truncateLabel(r.agentConfigId) }),
         /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
           " (Steps: ",
           r.stepsCount,
@@ -81,22 +87,22 @@ var Dashboard = ({ runs, testCases, configs, isFinished }) => {
     /* @__PURE__ */ jsxs(Box, { flexDirection: "column", borderStyle: "single", borderColor: "gray", padding: 1, children: [
       /* @__PURE__ */ jsxs(Box, { flexDirection: "row", marginBottom: 1, children: [
         /* @__PURE__ */ jsx(Box, { width: 25, children: /* @__PURE__ */ jsx(Text, { bold: true, color: "cyan", children: "Test Case" }) }),
-        configs.map((cfg) => /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsx(Text, { bold: true, color: "blue", children: cfg }) }, cfg))
+        configs.map((cfg) => /* @__PURE__ */ jsx(Box, { width: CONFIG_COL_WIDTH, children: /* @__PURE__ */ jsx(Text, { bold: true, color: "blue", wrap: "truncate-end", children: truncateLabel(cfg) }) }, cfg))
       ] }),
       testCases.map((tc) => /* @__PURE__ */ jsxs(Box, { flexDirection: "row", children: [
-        /* @__PURE__ */ jsx(Box, { width: 25, children: /* @__PURE__ */ jsx(Text, { children: tc }) }),
+        /* @__PURE__ */ jsx(Box, { width: 25, children: /* @__PURE__ */ jsx(Text, { wrap: "truncate-end", children: tc }) }),
         configs.map((cfg) => {
           const key = `${tc}_${cfg}`;
           const run = runs[key];
           if (!run) {
-            return /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsx(Text, { color: "gray", children: "queued" }) }, cfg);
+            return /* @__PURE__ */ jsx(Box, { width: CONFIG_COL_WIDTH, children: /* @__PURE__ */ jsx(Text, { color: "gray", children: "queued" }) }, cfg);
           }
           if (run.status === "running") {
-            return /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsx(Text, { color: "yellow", children: "running..." }) }, cfg);
+            return /* @__PURE__ */ jsx(Box, { width: CONFIG_COL_WIDTH, children: /* @__PURE__ */ jsx(Text, { color: "yellow", children: "running..." }) }, cfg);
           }
           if (run.status === "failed" || !run.passed) {
             const seconds2 = (run.durationMs / 1e3).toFixed(1);
-            return /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsxs(Text, { color: "red", children: [
+            return /* @__PURE__ */ jsx(Box, { width: CONFIG_COL_WIDTH, children: /* @__PURE__ */ jsxs(Text, { color: "red", wrap: "truncate-end", children: [
               "\u2717 ",
               seconds2,
               "s ($",
@@ -105,7 +111,7 @@ var Dashboard = ({ runs, testCases, configs, isFinished }) => {
             ] }) }, cfg);
           }
           const seconds = (run.durationMs / 1e3).toFixed(1);
-          return /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsxs(Text, { color: "green", children: [
+          return /* @__PURE__ */ jsx(Box, { width: CONFIG_COL_WIDTH, children: /* @__PURE__ */ jsxs(Text, { color: "green", wrap: "truncate-end", children: [
             "\u2713 ",
             seconds,
             "s ($",
@@ -173,6 +179,125 @@ function loadAgentConfig(yamlPath) {
   }
   return config;
 }
+
+// src/lib/resolve-agent-config-paths.ts
+function globToRegex(glob) {
+  const escaped = glob.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+  return new RegExp(`^${escaped}$`);
+}
+function collectYamlFilesRecursive(dir) {
+  const files = [];
+  for (const entry of readdirSync(dir)) {
+    if (entry.startsWith(".")) continue;
+    const fullPath = resolve(dir, entry);
+    const stat = statSync(fullPath);
+    if (stat.isDirectory()) {
+      files.push(...collectYamlFilesRecursive(fullPath));
+    } else if (entry.endsWith(".yaml") || entry.endsWith(".yml")) {
+      files.push(fullPath);
+    }
+  }
+  return files;
+}
+function findAgentConfigYamlFilesInDir(dir) {
+  const resolvedDir = resolve(dir);
+  const files = [];
+  for (const entry of readdirSync(resolvedDir)) {
+    if (entry.startsWith(".")) continue;
+    const fullPath = resolve(resolvedDir, entry);
+    if (!statSync(fullPath).isFile()) continue;
+    if (entry.endsWith(".yaml") || entry.endsWith(".yml")) {
+      files.push(fullPath);
+    }
+  }
+  return files.sort();
+}
+function expandAgentConfigGlob(globPattern, baseDir) {
+  const base = resolve(baseDir);
+  const normalized = globPattern.replace(/^\.\//, "");
+  if (normalized.includes("**")) {
+    const [prefix, suffixPart] = normalized.split("**");
+    const searchRoot = prefix.replace(/\/$/, "") ? resolve(base, prefix.replace(/\/$/, "")) : base;
+    const suffix = (suffixPart ?? "").replace(/^\//, "") || "*.yaml";
+    const regex2 = globToRegex(suffix);
+    return collectYamlFilesRecursive(searchRoot).filter((filePath) => regex2.test(basename(filePath))).sort();
+  }
+  const slashIdx = normalized.lastIndexOf("/");
+  const cwd = slashIdx === -1 ? base : resolve(base, normalized.slice(0, slashIdx));
+  const fileGlob = slashIdx === -1 ? normalized : normalized.slice(slashIdx + 1);
+  const regex = globToRegex(fileGlob);
+  return readdirSync(cwd).filter((entry) => {
+    if (entry.startsWith(".")) return false;
+    const fullPath = resolve(cwd, entry);
+    return statSync(fullPath).isFile() && regex.test(entry);
+  }).map((entry) => resolve(cwd, entry)).sort();
+}
+function resolveAgentConfigPathList(input) {
+  const paths = /* @__PURE__ */ new Set();
+  if (input.commaSeparated) {
+    for (const part of input.commaSeparated.split(",")) {
+      const trimmed = part.trim();
+      if (trimmed) paths.add(resolve(trimmed));
+    }
+  }
+  if (input.dir) {
+    for (const file of findAgentConfigYamlFilesInDir(input.dir)) {
+      paths.add(file);
+    }
+  }
+  const baseDir = input.relativeTo ? resolve(input.relativeTo) : process.cwd();
+  if (input.explicitPaths) {
+    for (const p of input.explicitPaths) {
+      paths.add(resolve(baseDir, p));
+    }
+  }
+  if (input.globs) {
+    for (const pattern of input.globs) {
+      for (const file of expandAgentConfigGlob(pattern, baseDir)) {
+        paths.add(file);
+      }
+    }
+  }
+  const sorted = [...paths].sort();
+  if (sorted.length === 0) {
+    throw new Error("No agent config YAML files found.");
+  }
+  return sorted;
+}
+function loadAgentConfigsFromPaths(paths) {
+  return paths.map((p) => loadAgentConfig(p));
+}
+
+// src/lib/load-bench-manifest.ts
+var AgentsSchema = z.object({
+  paths: z.array(z.string()).optional(),
+  glob: z.union([z.string(), z.array(z.string())]).optional()
+}).refine((data) => (data.paths?.length ?? 0) > 0 || data.glob !== void 0, {
+  message: "agents must specify at least one of paths or glob"
+});
+var BenchManifestSchema = z.object({
+  name: z.string().optional(),
+  suite: z.string(),
+  agents: AgentsSchema,
+  concurrency: z.number().optional()
+});
+function loadBenchManifest(yamlPath) {
+  const path = resolve(yamlPath);
+  const raw = parse(readFileSync(path, "utf-8"));
+  return BenchManifestSchema.parse(raw);
+}
+function resolveManifestAgentConfigPaths(manifest, manifestPath) {
+  const manifestDir = dirname(resolve(manifestPath));
+  const globs = manifest.agents.glob ? Array.isArray(manifest.agents.glob) ? manifest.agents.glob : [manifest.agents.glob] : void 0;
+  return resolveAgentConfigPathList({
+    explicitPaths: manifest.agents.paths,
+    globs,
+    relativeTo: manifestDir
+  });
+}
+function resolveManifestSuiteDir(manifest, manifestPath) {
+  return resolve(dirname(resolve(manifestPath)), manifest.suite);
+}
 function loadMatrix(yamlPath) {
   const path = resolve(yamlPath);
   const fileContent = readFileSync(path, "utf-8");
@@ -266,26 +391,57 @@ function findTestCaseYamlFiles(dir) {
   return files;
 }
 async function runBenchCommand(opts) {
-  const suiteDir = resolve(opts.suite);
-  const concurrency = opts.concurrency || 2;
+  let suiteDir;
+  let concurrency = opts.concurrency ?? 2;
   let agentConfigs;
   let matrixId;
-  if (opts.matrix) {
-    const matrix = loadMatrix(opts.matrix);
-    agentConfigs = expandMatrix(matrix);
-    matrixId = randomUUID();
+  if (opts.manifest) {
+    const manifestPath = resolve(opts.manifest);
+    const manifest = loadBenchManifest(manifestPath);
+    suiteDir = resolveManifestSuiteDir(manifest, manifestPath);
+    if (manifest.concurrency !== void 0 && opts.concurrency === void 0) {
+      concurrency = manifest.concurrency;
+    }
+    if (opts.matrix) {
+      throw new Error("Use either --manifest or --matrix, not both.");
+    }
+    const configPaths = resolveManifestAgentConfigPaths(manifest, manifestPath);
+    agentConfigs = loadAgentConfigsFromPaths(configPaths);
     console.log(
-      `Matrix "${matrix.name}" expanded to ${agentConfigs.length} agent config(s) (matrixId: ${matrixId})`
+      `Bench manifest "${manifest.name ?? manifestPath}" loaded ${agentConfigs.length} agent config(s) from ${configPaths.length} file(s).`
     );
-  } else if (opts.configs) {
-    const configPaths = opts.configs.split(",").map((c) => resolve(c.trim()));
-    agentConfigs = configPaths.map((p) => loadAgentConfig(p));
   } else {
-    throw new Error("Either --configs or --matrix must be provided.");
+    if (!opts.suite) {
+      throw new Error("--suite is required unless --manifest is provided.");
+    }
+    suiteDir = resolve(opts.suite);
+    if (opts.matrix) {
+      if (opts.configs || opts.configsDir) {
+        throw new Error("Use either --matrix or --configs/--configs-dir, not both.");
+      }
+      const matrix = loadMatrix(opts.matrix);
+      agentConfigs = expandMatrix(matrix);
+      matrixId = randomUUID();
+      console.log(
+        `Matrix "${matrix.name}" expanded to ${agentConfigs.length} agent config(s) (matrixId: ${matrixId})`
+      );
+    } else {
+      const configPaths = resolveAgentConfigPathList({
+        commaSeparated: opts.configs,
+        dir: opts.configsDir
+      });
+      agentConfigs = loadAgentConfigsFromPaths(configPaths);
+      if (opts.configsDir) {
+        console.log(`Loaded ${agentConfigs.length} agent config(s) from ${opts.configsDir}.`);
+      }
+    }
+  }
+  if (agentConfigs.length === 0) {
+    throw new Error("No agent configs to benchmark.");
   }
   const yamlFiles = findTestCaseYamlFiles(suiteDir);
   if (yamlFiles.length === 0) {
-    console.error(`No test cases found in suite directory: ${opts.suite}`);
+    console.error(`No test cases found in suite directory: ${suiteDir}`);
     process.exit(1);
   }
   const testCases = [];
@@ -461,6 +617,11 @@ async function validateCommand(testCasePath, opts) {
     console.log(
       "Note: this was a static-only validation (no test_command configured) - Docker/patch execution checks were skipped."
     );
+    if (report.ok && !opts?.strict) {
+      console.log(
+        "Tip: run with --strict to enforce test_command, fail_to_pass, and pass_to_pass as a CI gate."
+      );
+    }
   }
   console.log("");
   console.log(report.ok ? "\u2705 Validation passed." : "\u274C Validation failed.");
@@ -506,8 +667,8 @@ async function importPrCommand(repo, prNumber, opts) {
   if (testDiff.trim()) {
     writeFileSync(resolve(outDir, "test_patch.patch"), testDiff);
   }
+  const fixtureDir = resolve(outDir, "fixture");
   if (opts.cloneFixture) {
-    const fixtureDir = resolve(outDir, "fixture");
     console.log(`
 Cloning ${owner}/${repoName} into ${fixtureDir}...`);
     execFileSync("git", ["clone", `https://github.com/${owner}/${repoName}.git`, fixtureDir], {
@@ -516,8 +677,12 @@ Cloning ${owner}/${repoName} into ${fixtureDir}...`);
     console.log(`Checking out base commit ${pr.base.sha}...`);
     execFileSync("git", ["checkout", pr.base.sha], { cwd: fixtureDir, stdio: "inherit" });
   }
-  const projectKind = opts.cloneFixture ? detectProjectKind(resolve(outDir, "fixture")) : "unknown";
-  const { success, test_command } = projectTestDefaults(projectKind, opts.cloneFixture ?? false);
+  const projectKind = opts.cloneFixture ? detectProjectKind(fixtureDir) : "unknown";
+  const { success, test_command, testCommandHint } = projectTestDefaults(
+    projectKind,
+    opts.cloneFixture ?? false,
+    fixtureDir
+  );
   const yamlDoc = {
     name: slug,
     description: pr.title,
@@ -535,7 +700,7 @@ Cloning ${owner}/${repoName} into ${fixtureDir}...`);
   if (testDiff.trim()) yamlDoc.test_patch = "./test_patch.patch";
   if (expectedFiles.length > 0) yamlDoc.expected_files = expectedFiles;
   if (forbidModified.length > 0) yamlDoc.forbid_modified = forbidModified;
-  writeFileSync(resolve(outDir, "agr.yaml"), buildAgrYaml(yamlDoc, projectKind));
+  writeFileSync(resolve(outDir, "agr.yaml"), buildAgrYaml(yamlDoc, testCommandHint));
   console.log(`
 Imported PR #${pr.number}: "${pr.title}"`);
   console.log(`Wrote scaffold to: ${outDir}`);
@@ -551,6 +716,9 @@ Imported PR #${pr.number}: "${pr.title}"`);
     console.log(
       `  3. Run "agr validate ${resolve(outDir, "agr.yaml")}" to verify the test case`
     );
+    console.log(
+      "\nNote: test_command/success defaults were NOT auto-detected because --clone-fixture was not set. Re-run with --clone-fixture to get language-specific defaults, or fill these fields manually."
+    );
   } else {
     console.log("  1. Fill in fail_to_pass and pass_to_pass in agr.yaml");
     console.log(
@@ -570,50 +738,81 @@ function detectProjectKind(fixtureDir) {
   if (existsSync(resolve(fixtureDir, "go.mod"))) return "go";
   return "unknown";
 }
-function projectTestDefaults(kind, cloned) {
+function projectTestDefaults(kind, cloned, fixtureDir) {
   if (!cloned) {
     return {
       success: [
         { run: "<TODO: install dependencies and run the test suite>", expect: { exit_code: 0 } }
       ],
-      test_command: "<TODO: shell command that runs tests with TAP output>"
+      test_command: "<TODO: shell command that runs tests with TAP output>",
+      testCommandHint: "none"
     };
   }
   switch (kind) {
     case "python":
       return {
         success: [{ run: "pip install -e . && pytest", expect: { exit_code: 0 } }],
-        test_command: "pytest --tap-stream"
+        test_command: "pytest --tap-stream",
+        testCommandHint: "python"
       };
     case "node":
-      return {
-        success: [{ run: "npm install && npm test", expect: { exit_code: 0 } }],
-        test_command: "tsx --test --test-reporter=tap src/**/*.test.ts"
-      };
+      return detectNodeTestRunner(fixtureDir);
     case "go":
       return {
         success: [{ run: "go test ./...", expect: { exit_code: 0 } }],
-        test_command: "<TODO: configure a TAP-producing test command for go>"
+        test_command: "<TODO: configure a TAP-producing test command for go>",
+        testCommandHint: "go"
       };
     default:
       return {
         success: [
           { run: "<TODO: install dependencies and run the test suite>", expect: { exit_code: 0 } }
         ],
-        test_command: "<TODO: shell command that runs tests with TAP output>"
+        test_command: "<TODO: shell command that runs tests with TAP output>",
+        testCommandHint: "none"
       };
   }
 }
-function buildAgrYaml(doc, projectKind) {
+function detectNodeTestRunner(fixtureDir) {
+  const success = [{ run: "npm install && npm test", expect: { exit_code: 0 } }];
+  const fallback = {
+    success,
+    test_command: "tsx --test --test-reporter=tap src/**/*.test.ts",
+    testCommandHint: "node-unknown"
+  };
+  try {
+    const pkgPath = resolve(fixtureDir, "package.json");
+    if (!existsSync(pkgPath)) return fallback;
+    const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
+    const deps = { ...pkg.dependencies, ...pkg.devDependencies };
+    if (deps.ava) {
+      return { success, test_command: "npx ava --tap", testCommandHint: "ava" };
+    }
+    if (deps.vitest) {
+      return { success, test_command: "npx vitest run --reporter=tap", testCommandHint: "vitest" };
+    }
+    if (deps.jest) {
+      return { success, test_command: "npx jest --ci", testCommandHint: "jest" };
+    }
+    return fallback;
+  } catch {
+    return fallback;
+  }
+}
+function buildAgrYaml(doc, testCommandHint) {
   let yaml = stringify(doc);
   const testListComment = "# TODO: run the test suite (see test_command above) and add real test names here.\n# agr validate checks pre/post-patch status once these fields are filled in.";
   yaml = yaml.replace(/^fail_to_pass:/m, `${testListComment}
 fail_to_pass:`);
-  if (projectKind === "python") {
-    yaml = yaml.replace(
-      /^test_command: (.+)$/m,
-      "# Requires pytest-tap for TAP output (pip install pytest-tap).\n$&"
-    );
+  const testCommandComments = {
+    python: "# Requires pytest-tap for TAP output (pip install pytest-tap).",
+    jest: "# jest does not output TAP by default; consider jest-tap-reporter",
+    "node-unknown": "# test_command could not be auto-detected reliably - verify this matches the project's actual test setup"
+  };
+  const comment = testCommandComments[testCommandHint];
+  if (comment) {
+    yaml = yaml.replace(/^test_command: (.+)$/m, `${comment}
+$&`);
   }
   return yaml;
 }
@@ -830,25 +1029,51 @@ cli.command("run <testCase>", "Run a single agent test case").option("--config <
     process.exit(1);
   }
 });
-cli.command("bench", "Run a benchmark matrix of multiple test cases and configs").option("--configs <configs>", "Comma-separated paths to AgentConfig YAML files").option("--config <config>", "Alias for --configs (single config path)").option("--suite <suite>", "Path to test suite directory containing test cases").option("--concurrency <concurrency>", "Number of parallel sandbox executions", { default: 2 }).option(
+cli.command("bench", "Run a benchmark matrix of multiple test cases and configs").option("--configs <configs>", "Comma-separated paths to AgentConfig YAML files").option("--config <config>", "Alias for --configs (single config path)").option(
+  "--configs-dir <dir>",
+  "Directory of AgentConfig YAML files (all .yaml/.yml files in the folder)"
+).option(
+  "--manifest <manifest>",
+  "Path to a bench manifest YAML (suite + agent paths/glob in one file)"
+).option("--suite <suite>", "Path to test suite directory containing test cases").option("--concurrency <concurrency>", "Number of parallel sandbox executions", { default: 2 }).option(
   "--matrix <matrix>",
   "Path to an optimizer matrix YAML file - expands into agent configs and prints a Pareto summary afterwards (alternative to --configs)"
-).example("agr bench --suite tasks --configs agent.yaml,agent-openrouter.yaml").example("agr bench --suite tasks --matrix optimizer-matrix.yaml").action(async (options) => {
+).example("agr bench --manifest bench.yaml").example("agr bench --suite tasks --configs-dir ./agents").example("agr bench --suite tasks --configs agent.yaml,agent-openrouter.yaml").example("agr bench --suite tasks --matrix optimizer-matrix.yaml").action(async (options) => {
   if (!options.configs && options.config) {
     options.configs = options.config;
   }
-  if (!options.suite || !options.configs && !options.matrix) {
+  const agentSourceCount = [
+    options.configs,
+    options.configsDir,
+    options.matrix,
+    options.manifest
+  ].filter(Boolean).length;
+  if (options.manifest) {
+    if (agentSourceCount > 1) {
+      console.error(
+        "Error: --manifest cannot be combined with --configs, --configs-dir, or --matrix."
+      );
+      process.exit(1);
+    }
+  } else if (!options.suite || agentSourceCount === 0) {
+    console.error(
+      "Error: provide --manifest, or --suite with one of --configs, --config, --configs-dir, or --matrix."
+    );
+    process.exit(1);
+  } else if (agentSourceCount > 1) {
     console.error(
-      "Error: --suite and either --configs, --config, or --matrix are required for benchmarking."
+      "Error: use only one agent source: --configs, --configs-dir, or --matrix."
     );
     process.exit(1);
   }
   try {
     await runBenchCommand({
       configs: options.configs,
+      configsDir: options.configsDir,
       suite: options.suite,
       concurrency: Number(options.concurrency),
-      matrix: options.matrix
+      matrix: options.matrix,
+      manifest: options.manifest
     });
   } catch (err) {
     console.error(`Error executing benchmark: ${err.message}`);
@@ -872,7 +1097,10 @@ cli.command(
 cli.command(
   "import-pr <repo> <prNumber>",
   "Scaffold a test case from a GitHub pull request (e.g. owner/repo 1234)"
-).option("--out <dir>", "Output directory for the scaffolded test case").option("--clone-fixture", "Clone the repo and check out the PR's base commit into ./fixture").option("--validate", "Run `agr validate` against the scaffolded test case afterwards").example("agr import-pr astropy/astropy 12907 --clone-fixture --validate").action(async (repo, prNumber, options) => {
+).option("--out <dir>", "Output directory for the scaffolded test case").option(
+  "--clone-fixture",
+  "Clone the repo and check out the PR's base commit into ./fixture (required for language/test-command auto-detection)"
+).option("--validate", "Run `agr validate` against the scaffolded test case afterwards").example("agr import-pr astropy/astropy 12907 --clone-fixture --validate").action(async (repo, prNumber, options) => {
   try {
     await importPrCommand(repo, prNumber, options);
   } catch (err) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentgrader",
-  "version": "1.0.6",
+  "version": "1.1.0",
   "description": "CLI for the Agentgrader benchmarking framework — run and bench coding agents",
   "license": "MIT",
   "type": "module",
@@ -19,9 +19,9 @@
     "dev": "bun run src/index.ts"
   },
   "dependencies": {
-    "@agentgrader/agent-openrouter": "^2.0.1",
+    "@agentgrader/agent-openrouter": "^2.0.3",
     "@agentgrader/core": "^1.1.3",
-    "@agentgrader/optimizer": "^0.1.0",
+    "@agentgrader/optimizer": "^0.1.1",
     "@agentgrader/sandbox-docker": "^2.0.2",
     "@agentgrader/scorer-static": "^0.1.0",
     "@agentgrader/store": "^1.0.3",