agentgrader 1.2.0 → 1.3.0

package/dist/index.js

@@ -476,6 +476,9 @@ function loadTestCase(yamlPath) {
       (toolkit) => isAbsolute(toolkit) ? toolkit : resolve(dir, toolkit)
     );
   }
+  if (testCase.agent_config) {
+    testCase.agent_config = isAbsolute(testCase.agent_config) ? testCase.agent_config : resolve(dir, testCase.agent_config);
+  }
   if (testCase.solution && looksLikeFilePath(testCase.solution)) {
     testCase.solution = readPatchFile(dir, testCase.solution);
   }
@@ -484,6 +487,26 @@ function loadTestCase(yamlPath) {
   }
   return testCase;
 }
+function resolveSharedAgentConfigFromTestCases(testCases) {
+  if (testCases.length === 0) {
+    throw new Error("No test cases loaded.");
+  }
+  const paths = /* @__PURE__ */ new Set();
+  for (const tc of testCases) {
+    if (!tc.agent_config) {
+      throw new Error(
+        "Either --configs, --configs-dir, --matrix, or --manifest must be provided, or every test case in the suite must define the same agent_config in agr.yaml."
+      );
+    }
+    paths.add(tc.agent_config);
+  }
+  if (paths.size > 1) {
+    throw new Error(
+      `Multiple agent_config values found across test cases (${[...paths].join(", ")}). Use --configs, --configs-dir, or --matrix to specify agent configs explicitly.`
+    );
+  }
+  return [...paths][0];
+}
 function looksLikeFilePath(value) {
   const trimmed = value.trimStart();
   if (trimmed.startsWith("diff ") || trimmed.startsWith("---") || trimmed.startsWith("***")) {
@@ -542,7 +565,7 @@ function findTestCaseYamlFiles(dir) {
 async function runBenchCommand(opts) {
   let suiteDir;
   let concurrency = opts.concurrency ?? 2;
-  let agentConfigs;
+  let agentConfigs = [];
   let matrixId;
   if (opts.manifest) {
     const manifestPath = resolve(opts.manifest);
@@ -574,7 +597,7 @@ async function runBenchCommand(opts) {
       console.log(
         `Matrix "${matrix.name}" expanded to ${agentConfigs.length} agent config(s) (matrixId: ${matrixId})`
       );
-    } else {
+    } else if (opts.configs || opts.configsDir) {
       const configPaths = resolveAgentConfigPathList({
         commaSeparated: opts.configs,
         dir: opts.configsDir
@@ -585,9 +608,6 @@ async function runBenchCommand(opts) {
       }
     }
   }
-  if (agentConfigs.length === 0) {
-    throw new Error("No agent configs to benchmark.");
-  }
   const yamlFiles = findTestCaseYamlFiles(suiteDir);
   if (yamlFiles.length === 0) {
     console.error(`No test cases found in suite directory: ${suiteDir}`);
@@ -597,6 +617,16 @@ async function runBenchCommand(opts) {
   for (const f of yamlFiles) {
     testCases.push(loadTestCase(f));
   }
+  if (agentConfigs.length === 0) {
+    const sharedAgentConfig = resolveSharedAgentConfigFromTestCases(testCases);
+    const configPaths = resolveAgentConfigPathList({
+      explicitPaths: [sharedAgentConfig]
+    });
+    agentConfigs = loadAgentConfigsFromPaths(configPaths);
+    console.log(
+      `Using shared agent_config from agr.yaml: ${sharedAgentConfig} (${agentConfigs.length} config).`
+    );
+  }
   const db = initDb();
   for (const tc of testCases) {
     await saveTestCase(db, testCaseToDbRow(tc));
@@ -1034,6 +1064,11 @@ async function runSingleCommand(testCasePath, opts) {
   };
   if (opts.config) {
     agentConfig = loadAgentConfig(opts.config);
+  } else if (testCase.agent_config) {
+    agentConfig = loadAgentConfig(testCase.agent_config);
+    console.log(
+      `Using agent config from agr.yaml: ${testCase.agent_config} (model: ${agentConfig.model})`
+    );
   }
   console.log(`Starting run for "${testCase.name}" using model "${agentConfig.model}"...`);
   const sandboxProvider = new DockerSandboxProvider();
@@ -1204,9 +1239,9 @@ cli.command("bench", "Run a benchmark matrix of multiple test cases and configs"
       );
       process.exit(1);
     }
-  } else if (!options.suite || agentSourceCount === 0) {
+  } else if (!options.suite) {
     console.error(
-      "Error: provide --manifest, or --suite with one of --configs, --config, --configs-dir, or --matrix."
+      "Error: provide --manifest, or --suite with one of --configs, --config, --configs-dir, --matrix, or a shared agent_config in every agr.yaml."
     );
     process.exit(1);
   } else if (agentSourceCount > 1) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentgrader",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "CLI for the Agentgrader benchmarking framework — run and bench coding agents",
   "license": "MIT",
   "type": "module",
@@ -19,11 +19,11 @@
     "dev": "bun run src/index.ts"
   },
   "dependencies": {
-    "@agentgrader/agent-openrouter": "^2.0.3",
-    "@agentgrader/core": "^1.1.3",
-    "@agentgrader/optimizer": "^0.1.1",
-    "@agentgrader/sandbox-docker": "^2.0.2",
-    "@agentgrader/scorer-static": "^0.1.0",
+    "@agentgrader/agent-openrouter": "^3.0.0",
+    "@agentgrader/core": "^1.2.0",
+    "@agentgrader/optimizer": "^1.0.0",
+    "@agentgrader/sandbox-docker": "^3.0.0",
+    "@agentgrader/scorer-static": "^1.0.0",
     "@agentgrader/store": "^1.0.3",
     "cac": "^6.7.14",
     "dotenv": "^17.4.2",