agentgrader 1.0.0

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+
+export {  }

package/dist/index.js

@@ -0,0 +1,564 @@
+#!/usr/bin/env node
+import { cac } from 'cac';
+import { resolve, dirname, isAbsolute } from 'path';
+import { render, Box, Text } from 'ink';
+import { initDb, saveTestCase, saveAgentConfig } from '@agentgrader/store';
+import { runSingle, runBenchmark, validateTestCase, TestCaseSchema, AgentConfigSchema } from '@agentgrader/core';
+import { DockerSandboxProvider } from '@agentgrader/sandbox-docker';
+import { OpenRouterAgentAdapter } from '@agentgrader/agent-openrouter';
+import { jsx, jsxs } from 'react/jsx-runtime';
+import { mkdirSync, writeFileSync, readFileSync, readdirSync, statSync } from 'fs';
+import { stringify, parse } from 'yaml';
+import { randomUUID } from 'crypto';
+
+var Dashboard = ({ runs, testCases, configs, isFinished }) => {
+  let totalCost = 0;
+  let totalSteps = 0;
+  let passedCount = 0;
+  let completedCount = 0;
+  for (const run of Object.values(runs)) {
+    totalCost += run.costUsd || 0;
+    totalSteps += run.stepsCount || 0;
+    if (run.status === "completed" || run.status === "failed") {
+      completedCount++;
+      if (run.passed) passedCount++;
+    }
+  }
+  const totalRuns = testCases.length * configs.length;
+  return /* @__PURE__ */ jsxs(Box, { flexDirection: "column", padding: 1, children: [
+    /* @__PURE__ */ jsxs(Box, { borderStyle: "round", borderColor: "cyan", paddingX: 2, marginBottom: 1, flexDirection: "column", children: [
+      /* @__PURE__ */ jsx(Text, { color: "cyan", bold: true, children: "\u{1F525} AGENTGRADER BENCHMARK \u{1F525}" }),
+      /* @__PURE__ */ jsx(Text, { color: "gray", children: "Docker Sandboxes \u2022 Parallel Execution \u2022 Mastra Orchestration" })
+    ] }),
+    /* @__PURE__ */ jsxs(Box, { marginBottom: 1, children: [
+      /* @__PURE__ */ jsx(Text, { bold: true, children: "Progress: " }),
+      /* @__PURE__ */ jsxs(Text, { color: "yellow", children: [
+        completedCount,
+        " / ",
+        totalRuns,
+        " completed"
+      ] }),
+      /* @__PURE__ */ jsx(Text, { children: " | " }),
+      /* @__PURE__ */ jsx(Text, { bold: true, children: "Passed: " }),
+      /* @__PURE__ */ jsxs(Text, { color: "green", children: [
+        passedCount,
+        " / ",
+        completedCount || 0,
+        " solves"
+      ] }),
+      /* @__PURE__ */ jsx(Text, { children: " | " }),
+      /* @__PURE__ */ jsx(Text, { bold: true, children: "Total Cost: " }),
+      /* @__PURE__ */ jsxs(Text, { color: "magenta", children: [
+        "$",
+        totalCost.toFixed(4)
+      ] })
+    ] }),
+    !isFinished && /* @__PURE__ */ jsxs(Box, { flexDirection: "column", marginBottom: 1, children: [
+      /* @__PURE__ */ jsx(Text, { bold: true, underline: true, color: "yellow", children: "Active Runs:" }),
+      Object.values(runs).filter((r) => r.status === "running").map((r) => /* @__PURE__ */ jsxs(Box, { marginLeft: 2, children: [
+        /* @__PURE__ */ jsx(Text, { color: "yellow", children: "\u25CF" }),
+        /* @__PURE__ */ jsxs(Text, { bold: true, children: [
+          " ",
+          r.testCaseId
+        ] }),
+        /* @__PURE__ */ jsx(Text, { color: "gray", children: " with " }),
+        /* @__PURE__ */ jsx(Text, { color: "blue", children: r.agentConfigId }),
+        /* @__PURE__ */ jsxs(Text, { color: "gray", children: [
+          " (Steps: ",
+          r.stepsCount,
+          ", Cost: $",
+          r.costUsd.toFixed(4),
+          ")"
+        ] })
+      ] }, r.runId)),
+      Object.values(runs).filter((r) => r.status === "running").length === 0 && /* @__PURE__ */ jsx(Box, { marginLeft: 2, children: /* @__PURE__ */ jsx(Text, { color: "gray", children: "No active runs (waiting or queuing)..." }) })
+    ] }),
+    /* @__PURE__ */ jsxs(Box, { flexDirection: "column", borderStyle: "single", borderColor: "gray", padding: 1, children: [
+      /* @__PURE__ */ jsxs(Box, { flexDirection: "row", marginBottom: 1, children: [
+        /* @__PURE__ */ jsx(Box, { width: 25, children: /* @__PURE__ */ jsx(Text, { bold: true, color: "cyan", children: "Test Case" }) }),
+        configs.map((cfg) => /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsx(Text, { bold: true, color: "blue", children: cfg }) }, cfg))
+      ] }),
+      testCases.map((tc) => /* @__PURE__ */ jsxs(Box, { flexDirection: "row", children: [
+        /* @__PURE__ */ jsx(Box, { width: 25, children: /* @__PURE__ */ jsx(Text, { children: tc }) }),
+        configs.map((cfg) => {
+          const key = `${tc}_${cfg}`;
+          const run = runs[key];
+          if (!run) {
+            return /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsx(Text, { color: "gray", children: "queued" }) }, cfg);
+          }
+          if (run.status === "running") {
+            return /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsx(Text, { color: "yellow", children: "running..." }) }, cfg);
+          }
+          if (run.status === "failed" || !run.passed) {
+            const seconds2 = (run.durationMs / 1e3).toFixed(1);
+            return /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsxs(Text, { color: "red", children: [
+              "\u2717 ",
+              seconds2,
+              "s ($",
+              run.costUsd.toFixed(3),
+              ")"
+            ] }) }, cfg);
+          }
+          const seconds = (run.durationMs / 1e3).toFixed(1);
+          return /* @__PURE__ */ jsx(Box, { width: 20, children: /* @__PURE__ */ jsxs(Text, { color: "green", children: [
+            "\u2713 ",
+            seconds,
+            "s ($",
+            run.costUsd.toFixed(3),
+            ")"
+          ] }) }, cfg);
+        })
+      ] }, tc))
+    ] }),
+    isFinished && /* @__PURE__ */ jsxs(Box, { marginTop: 1, flexDirection: "column", borderStyle: "double", borderColor: "green", padding: 1, children: [
+      /* @__PURE__ */ jsx(Text, { color: "green", bold: true, children: "Benchmark finished successfully!" }),
+      /* @__PURE__ */ jsxs(Text, { children: [
+        "Total runs executed: ",
+        totalRuns
+      ] }),
+      /* @__PURE__ */ jsxs(Text, { children: [
+        "Successful solves: ",
+        passedCount,
+        " (",
+        (passedCount / totalRuns * 100).toFixed(0),
+        "%)"
+      ] }),
+      /* @__PURE__ */ jsxs(Text, { children: [
+        "Total model API cost: $",
+        totalCost.toFixed(4)
+      ] }),
+      /* @__PURE__ */ jsxs(Text, { children: [
+        "Average steps per run: ",
+        (totalSteps / totalRuns).toFixed(1)
+      ] })
+    ] })
+  ] });
+};
+function loadAgentConfig(yamlPath) {
+  const path = resolve(yamlPath);
+  const fileContent = readFileSync(path, "utf-8");
+  const raw = parse(fileContent);
+  const dir = dirname(path);
+  const config = AgentConfigSchema.parse(raw);
+  config.id = config.id || config.name;
+  if (config.toolkits) {
+    config.toolkits = config.toolkits.map(
+      (toolkit) => isAbsolute(toolkit) ? toolkit : resolve(dir, toolkit)
+    );
+  }
+  return config;
+}
+function loadTestCase(yamlPath) {
+  const path = resolve(yamlPath);
+  const fileContent = readFileSync(path, "utf-8");
+  const raw = parse(fileContent);
+  const dir = dirname(path);
+  if (raw.fixture && !String(raw.fixture).startsWith("/") && !String(raw.fixture).startsWith("http")) {
+    raw.fixture = resolve(dir, raw.fixture);
+  }
+  const testCase = TestCaseSchema.parse(raw);
+  testCase.id = testCase.id || testCase.name;
+  if (testCase.toolkits) {
+    testCase.toolkits = testCase.toolkits.map(
+      (toolkit) => isAbsolute(toolkit) ? toolkit : resolve(dir, toolkit)
+    );
+  }
+  if (testCase.solution && looksLikeFilePath(testCase.solution)) {
+    testCase.solution = readPatchFile(dir, testCase.solution);
+  }
+  if (testCase.test_patch && looksLikeFilePath(testCase.test_patch)) {
+    testCase.test_patch = readPatchFile(dir, testCase.test_patch);
+  }
+  return testCase;
+}
+function looksLikeFilePath(value) {
+  const trimmed = value.trimStart();
+  if (trimmed.startsWith("diff ") || trimmed.startsWith("---") || trimmed.startsWith("***")) {
+    return false;
+  }
+  return !value.includes("\n");
+}
+function readPatchFile(dir, relPath) {
+  const full = isAbsolute(relPath) ? relPath : resolve(dir, relPath);
+  try {
+    return readFileSync(full, "utf-8");
+  } catch (err) {
+    throw new Error(`Failed to read patch file "${relPath}" (resolved to ${full}): ${err.message}`);
+  }
+}
+function testCaseToDbRow(testCase) {
+  return {
+    id: testCase.id || testCase.name,
+    name: testCase.name,
+    description: testCase.description,
+    fixture: testCase.fixture,
+    prompt: testCase.prompt,
+    success: JSON.stringify(testCase.success),
+    timeoutSeconds: testCase.timeout_seconds,
+    createdAt: Math.floor(Date.now() / 1e3),
+    tags: testCase.tags ? JSON.stringify(testCase.tags) : null,
+    testCommand: testCase.test_command ?? null,
+    failToPass: testCase.fail_to_pass ? JSON.stringify(testCase.fail_to_pass) : null,
+    passToPass: testCase.pass_to_pass ? JSON.stringify(testCase.pass_to_pass) : null,
+    forbidModified: testCase.forbid_modified ? JSON.stringify(testCase.forbid_modified) : null,
+    expectedFiles: testCase.expected_files ? JSON.stringify(testCase.expected_files) : null,
+    solution: testCase.solution ?? null,
+    testPatch: testCase.test_patch ?? null,
+    sourceCreatedAt: testCase.created_at ?? null
+  };
+}
+function findTestCaseYamlFiles(dir) {
+  const files = [];
+  try {
+    const entries = readdirSync(dir);
+    for (const entry of entries) {
+      const fullPath = resolve(dir, entry);
+      const stat = statSync(fullPath);
+      if (stat.isDirectory()) {
+        if (entry !== "fixture" && entry !== "node_modules" && !entry.startsWith(".")) {
+          files.push(...findTestCaseYamlFiles(fullPath));
+        }
+      } else if (entry === "agr.yaml" || entry.endsWith(".yaml") && !entry.includes("config")) {
+        files.push(fullPath);
+      }
+    }
+  } catch {
+  }
+  return files;
+}
+async function runBenchCommand(opts) {
+  const suiteDir = resolve(opts.suite);
+  const configPaths = opts.configs.split(",").map((c) => resolve(c.trim()));
+  const concurrency = opts.concurrency || 2;
+  const agentConfigs = configPaths.map((p) => loadAgentConfig(p));
+  const yamlFiles = findTestCaseYamlFiles(suiteDir);
+  if (yamlFiles.length === 0) {
+    console.error(`No test cases found in suite directory: ${opts.suite}`);
+    process.exit(1);
+  }
+  const testCases = [];
+  for (const f of yamlFiles) {
+    testCases.push(loadTestCase(f));
+  }
+  const db = initDb();
+  for (const tc of testCases) {
+    await saveTestCase(db, testCaseToDbRow(tc));
+  }
+  for (const ac of agentConfigs) {
+    await saveAgentConfig(db, {
+      id: ac.id || ac.name,
+      name: ac.name,
+      model: ac.model,
+      maxSteps: ac.max_steps,
+      temperature: ac.temperature,
+      systemPrompt: ac.system_prompt,
+      tools: JSON.stringify(ac.tools),
+      createdAt: Math.floor(Date.now() / 1e3)
+    });
+  }
+  const sandboxProvider = new DockerSandboxProvider();
+  const adapter = new OpenRouterAgentAdapter();
+  const runStates = {};
+  const testCaseIds = testCases.map((tc) => tc.id || tc.name);
+  const configIds = agentConfigs.map((ac) => ac.id || ac.name);
+  const { rerender, waitUntilExit } = render(
+    /* @__PURE__ */ jsx(
+      Dashboard,
+      {
+        runs: runStates,
+        testCases: testCaseIds,
+        configs: configIds,
+        isFinished: false
+      }
+    )
+  );
+  const onRunUpdate = (run) => {
+    const key = `${run.testCaseId}_${run.agentConfigId}`;
+    runStates[key] = run;
+    rerender(
+      /* @__PURE__ */ jsx(
+        Dashboard,
+        {
+          runs: runStates,
+          testCases: testCaseIds,
+          configs: configIds,
+          isFinished: false
+        }
+      )
+    );
+  };
+  try {
+    await runBenchmark({
+      testCases,
+      agentConfigs,
+      adapter,
+      sandboxProvider,
+      db,
+      concurrency,
+      onRunUpdate
+    });
+  } catch (err) {
+    console.error("Benchmark runner encountered an error:", err);
+  }
+  rerender(
+    /* @__PURE__ */ jsx(
+      Dashboard,
+      {
+        runs: runStates,
+        testCases: testCaseIds,
+        configs: configIds,
+        isFinished: true
+      }
+    )
+  );
+  printTagBreakdown(testCases, agentConfigs, runStates);
+  process.exit(0);
+}
+function printTagBreakdown(testCases, agentConfigs, runStates) {
+  const tagStats = {};
+  for (const tc of testCases) {
+    const tags2 = tc.tags ?? [];
+    if (tags2.length === 0) continue;
+    for (const ac of agentConfigs) {
+      const key = `${tc.id || tc.name}_${ac.id || ac.name}`;
+      const run = runStates[key];
+      if (!run || run.status === "running") continue;
+      for (const tag of tags2) {
+        const stats = tagStats[tag] ??= { passed: 0, total: 0 };
+        stats.total++;
+        if (run.passed) stats.passed++;
+      }
+    }
+  }
+  const tags = Object.keys(tagStats).sort();
+  if (tags.length === 0) return;
+  console.log("\n================ TAG BREAKDOWN ================");
+  for (const tag of tags) {
+    const { passed, total } = tagStats[tag];
+    const pct = total > 0 ? (passed / total * 100).toFixed(0) : "0";
+    console.log(`${tag.padEnd(24)} ${passed}/${total} (${pct}%)`);
+  }
+  console.log("=================================================\n");
+}
+var TEST_FILE_PATTERN = /(^|\/)(tests?|specs?|__tests__)(\/|$)|\.(test|spec)\.[jt]sx?$/i;
+async function importPrCommand(repo, prNumber, opts) {
+  const [owner, repoName] = repo.split("/");
+  if (!owner || !repoName) {
+    throw new Error(`Invalid repo "${repo}" - expected format "owner/repo".`);
+  }
+  const headers = {
+    Accept: "application/vnd.github+json",
+    "User-Agent": "agentgrader-import-pr"
+  };
+  if (process.env.GITHUB_TOKEN) {
+    headers.Authorization = `Bearer ${process.env.GITHUB_TOKEN}`;
+  }
+  const apiUrl = `https://api.github.com/repos/${owner}/${repoName}/pulls/${prNumber}`;
+  console.log(`Fetching PR #${prNumber} from ${owner}/${repoName}...`);
+  const prRes = await fetch(apiUrl, { headers });
+  if (!prRes.ok) {
+    throw new Error(`Failed to fetch PR metadata: ${prRes.status} ${prRes.statusText}`);
+  }
+  const pr = await prRes.json();
+  console.log("Fetching PR diff...");
+  const diffRes = await fetch(apiUrl, {
+    headers: { ...headers, Accept: "application/vnd.github.v3.diff" }
+  });
+  if (!diffRes.ok) {
+    throw new Error(`Failed to fetch PR diff: ${diffRes.status} ${diffRes.statusText}`);
+  }
+  const fullDiff = await diffRes.text();
+  const { solutionDiff, testDiff, expectedFiles, forbidModified } = splitDiff(fullDiff);
+  const slug = `${repoName}-pr-${pr.number}`;
+  const outDir = resolve(opts.out || `./imported/${slug}`);
+  mkdirSync(outDir, { recursive: true });
+  if (solutionDiff.trim()) {
+    writeFileSync(resolve(outDir, "solution.patch"), solutionDiff);
+  }
+  if (testDiff.trim()) {
+    writeFileSync(resolve(outDir, "test_patch.patch"), testDiff);
+  }
+  const yamlDoc = {
+    name: slug,
+    description: pr.title,
+    fixture: "./fixture",
+    prompt: buildPrompt(pr),
+    success: [{ run: "npm install && npm test", expect: { exit_code: 0 } }],
+    timeout_seconds: 600,
+    tags: ["imported", repoName],
+    created_at: pr.created_at,
+    // TODO: fill these in after setting up ./fixture (checked out at
+    // base.sha below) and running the test suite to discover real test names.
+    test_command: "<TODO: e.g. tsx --test --test-reporter=tap src/**/*.test.ts>",
+    fail_to_pass: ["<TODO: fill in via `agr validate`>"],
+    pass_to_pass: ["<TODO: fill in via `agr validate`>"]
+  };
+  if (solutionDiff.trim()) yamlDoc.solution = "./solution.patch";
+  if (testDiff.trim()) yamlDoc.test_patch = "./test_patch.patch";
+  if (expectedFiles.length > 0) yamlDoc.expected_files = expectedFiles;
+  if (forbidModified.length > 0) yamlDoc.forbid_modified = forbidModified;
+  writeFileSync(resolve(outDir, "agr.yaml"), stringify(yamlDoc));
+  console.log(`
+Imported PR #${pr.number}: "${pr.title}"`);
+  console.log(`Wrote scaffold to: ${outDir}`);
+  console.log("  - agr.yaml");
+  if (solutionDiff.trim())
+    console.log(`  - solution.patch (${expectedFiles.length} file(s) changed)`);
+  if (testDiff.trim())
+    console.log(`  - test_patch.patch (${forbidModified.length} test file(s) changed)`);
+  console.log("\nNext steps:");
+  console.log(`  1. Check out ${owner}/${repoName}@${pr.base.sha} into ${outDir}/fixture`);
+  console.log("  2. Fill in test_command, fail_to_pass, and pass_to_pass in agr.yaml");
+  console.log(
+    `  3. Run "agr validate ${resolve(outDir, "agr.yaml")}" to verify the test case`
+  );
+}
+function buildPrompt(pr) {
+  const body = (pr.body || "").trim();
+  return body ? `${pr.title}
+
+${body}` : pr.title;
+}
+function splitDiff(diff) {
+  const fileDiffs = diff.split(/(?=^diff --git )/m).filter((d) => d.trim().length > 0);
+  const solutionParts = [];
+  const testParts = [];
+  const expectedFiles = [];
+  const forbidModified = [];
+  for (const fileDiff of fileDiffs) {
+    const match = fileDiff.match(/^diff --git a\/(.+?) b\/(.+)$/m);
+    const path = match?.[2];
+    if (path && TEST_FILE_PATTERN.test(path)) {
+      testParts.push(fileDiff);
+      forbidModified.push(path);
+    } else {
+      solutionParts.push(fileDiff);
+      if (path) expectedFiles.push(path);
+    }
+  }
+  return {
+    solutionDiff: solutionParts.join(""),
+    testDiff: testParts.join(""),
+    expectedFiles,
+    forbidModified
+  };
+}
+async function runSingleCommand(testCasePath, opts) {
+  const testCase = loadTestCase(testCasePath);
+  let agentConfig = {
+    id: "baseline",
+    name: "Baseline Agent",
+    model: "gpt-4o-mini",
+    max_steps: 20
+  };
+  if (opts.config) {
+    agentConfig = loadAgentConfig(opts.config);
+  }
+  console.log(`Starting run for "${testCase.name}" using model "${agentConfig.model}"...`);
+  const sandboxProvider = new DockerSandboxProvider();
+  const adapter = new OpenRouterAgentAdapter();
+  const db = initDb();
+  await saveTestCase(db, testCaseToDbRow(testCase));
+  const runId = randomUUID();
+  try {
+    const result = await runSingle({
+      testCase,
+      agentConfig,
+      adapter,
+      sandboxProvider,
+      db,
+      runId
+    });
+    console.log("\n================ RUN SUMMARY ================");
+    console.log(`Status:    ${result.passed ? "\u2705 PASSED" : "\u274C FAILED"}`);
+    console.log(`Steps:     ${result.stepsCount}`);
+    console.log(`Cost:      $${result.costUsd.toFixed(4)}`);
+    console.log(`Duration:  ${(result.durationMs / 1e3).toFixed(1)}s`);
+    if (result.error) {
+      console.log(`Error:     ${result.error}`);
+    }
+    if (result.metrics?.regression) {
+      console.log(`Regression: ${result.metrics.regression.detail}`);
+    }
+    if (result.metrics?.diff) {
+      console.log(`Diff scope: ${result.metrics.diff.detail.split("\n")[0]}`);
+    }
+    if (result.metrics?.localization) {
+      console.log(`Localization: ${result.metrics.localization.detail.split("\n")[0]}`);
+    }
+    console.log("=============================================\n");
+  } catch (err) {
+    console.error(`Run failed with error: ${err.message}`);
+    process.exit(1);
+  }
+  process.exit(0);
+}
+async function validateCommand(testCasePath) {
+  const testCase = loadTestCase(testCasePath);
+  console.log(`Validating "${testCase.name}" (${testCasePath})...
+`);
+  const sandboxProvider = new DockerSandboxProvider();
+  const report = await validateTestCase({ testCase, sandboxProvider });
+  for (const check of report.checks) {
+    const icon = check.passed ? "\u2705" : "\u274C";
+    console.log(`${icon} ${check.name}`);
+    if (check.detail && check.detail !== "ok") {
+      const indented = check.detail.split("\n").map((line) => `   ${line}`).join("\n");
+      console.log(indented);
+    }
+  }
+  console.log("");
+  console.log(report.ok ? "\u2705 Validation passed." : "\u274C Validation failed.");
+  process.exit(report.ok ? 0 : 1);
+}
+
+// src/index.ts
+var cli = cac("agr");
+cli.command("run <testCase>", "Run a single agent test case").option("--config <config>", "Path to an AgentConfig YAML file").action(async (testCase, options) => {
+  try {
+    await runSingleCommand(testCase, options);
+  } catch (err) {
+    console.error(`Error executing run: ${err.message}`);
+    process.exit(1);
+  }
+});
+cli.command("bench", "Run a benchmark matrix of multiple test cases and configs").option("--configs <configs>", "Comma-separated paths to AgentConfig YAML files").option("--suite <suite>", "Path to test suite directory containing test cases").option("--concurrency <concurrency>", "Number of parallel sandbox executions", { default: 2 }).action(async (options) => {
+  if (!options.configs || !options.suite) {
+    console.error("Error: --configs and --suite are required for benchmarking.");
+    process.exit(1);
+  }
+  try {
+    await runBenchCommand({
+      configs: options.configs,
+      suite: options.suite,
+      concurrency: Number(options.concurrency)
+    });
+  } catch (err) {
+    console.error(`Error executing benchmark: ${err.message}`);
+    process.exit(1);
+  }
+});
+cli.command(
+  "validate <testCase>",
+  "Validate a test case definition (fixture, fail_to_pass/pass_to_pass, gold patch)"
+).action(async (testCase) => {
+  try {
+    await validateCommand(testCase);
+  } catch (err) {
+    console.error(`Error executing validate: ${err.message}`);
+    process.exit(1);
+  }
+});
+cli.command(
+  "import-pr <repo> <prNumber>",
+  "Scaffold a test case from a GitHub pull request (e.g. owner/repo 1234)"
+).option("--out <dir>", "Output directory for the scaffolded test case").action(async (repo, prNumber, options) => {
+  try {
+    await importPrCommand(repo, prNumber, options);
+  } catch (err) {
+    console.error(`Error executing import-pr: ${err.message}`);
+    process.exit(1);
+  }
+});
+cli.help();
+cli.parse();

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "agentgrader",
+  "version": "1.0.0",
+  "description": "CLI for the Agentgrader benchmarking framework — run and bench coding agents",
+  "license": "MIT",
+  "type": "module",
+  "bin": {
+    "agr": "./dist/index.js",
+    "agentgrader": "./dist/index.js"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "build:watch": "tsup --watch",
+    "dev": "bun run src/index.ts"
+  },
+  "dependencies": {
+    "@agentgrader/agent-openrouter": "workspace:*",
+    "@agentgrader/core": "workspace:*",
+    "@agentgrader/sandbox-docker": "workspace:*",
+    "@agentgrader/store": "workspace:*",
+    "cac": "^6.7.14",
+    "ink": "^4.4.1",
+    "react": "^18.2.0",
+    "yaml": "^2.5.1"
+  },
+  "devDependencies": {
+    "@types/react": "^18.2.0",
+    "tsup": "^8.5.1"
+  }
+}