@agentgrader/core 1.1.2 → 1.2.0

package/dist/index.d.ts

@@ -72,6 +72,7 @@ declare const TestCaseSchema: z.ZodObject<{
     created_at: z.ZodOptional<z.ZodString>;
     image: z.ZodOptional<z.ZodString>;
     toolkits: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    agent_config: z.ZodOptional<z.ZodString>;
 }, "strip", z.ZodTypeAny, {
     name: string;
     fixture: string;
@@ -98,6 +99,7 @@ declare const TestCaseSchema: z.ZodObject<{
     created_at?: string | undefined;
     image?: string | undefined;
     toolkits?: string[] | undefined;
+    agent_config?: string | undefined;
 }, {
     name: string;
     fixture: string;
@@ -124,6 +126,7 @@ declare const TestCaseSchema: z.ZodObject<{
     created_at?: string | undefined;
     image?: string | undefined;
     toolkits?: string[] | undefined;
+    agent_config?: string | undefined;
 }>;
 type TestCase = z.infer<typeof TestCaseSchema>;
 
@@ -653,6 +656,7 @@ interface RunSingleInput {
     extraScorers?: Scorer[];
     /** links this run to an optimizer matrix run, if any */
     matrixId?: string;
+    onStep?: (step: StepEvent) => void;
 }
 interface RunSingleResult {
     runId: string;

package/dist/index.js

@@ -50,7 +50,8 @@ var TestCaseSchema = z.object({
   // paths to toolkit directories (custom CLI tools + .claude/skills/) to
   // inject into the sandbox and surface to the agent via the system prompt,
   // in addition to any toolkits configured on the agent
-  toolkits: z.array(z.string()).optional()
+  toolkits: z.array(z.string()).optional(),
+  agent_config: z.string().optional()
 });
 var SkillFrontmatterSchema = z.object({
   /** lowercase letters, numbers, hyphens; max 64 chars */
@@ -83,7 +84,9 @@ var AgentConfigSchema = z.object({
   max_steps: z.number().default(30),
   temperature: z.number().optional(),
   system_prompt: z.string().optional(),
-  tools: z.array(z.string()).optional(),
+  tools: z.array(z.string()).optional().describe(
+    "Optional allowlist of tool names (local: executeCommand, readFile, writeFile, submit; MCP: <mcpServerName>_<toolName>). submit is always included implicitly."
+  ),
   // paths to toolkit directories (custom CLI tools + .claude/skills/) to
   // inject into the sandbox and surface to the agent via the system prompt
   toolkits: z.array(z.string()).optional(),
@@ -641,6 +644,7 @@ async function runSingle(input) {
         tokensIn += stepEvent.tokensIn || 0;
         tokensOut += stepEvent.tokensOut || 0;
         costUsd += stepEvent.costUsd || 0;
+        input.onStep?.(stepEvent);
         if (db) {
           addTrace(db, {
             runId,
@@ -1008,7 +1012,7 @@ async function validateTestCase(input) {
   checks.push(...checkStaticFields(testCase));
   if (!testCase.test_command) {
     checks.push({
-      name: "execution-checks",
+      name: "execution-checks (skipped - no test_command)",
       passed: true,
       detail: "No test_command configured; skipping pre/post-patch execution checks."
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentgrader/core",
-  "version": "1.1.2",
+  "version": "1.2.0",
   "description": "Core schemas, contracts, and runner for the Agentgrader benchmarking framework",
   "license": "MIT",
   "type": "module",
@@ -22,7 +22,7 @@
     "dev": "bun run src/index.ts"
   },
   "dependencies": {
-    "@agentgrader/store": "^1.0.2",
+    "@agentgrader/store": "^1.0.3",
     "@mastra/core": "^1.41.0",
     "yaml": "^2.5.1",
     "zod": "^3.23.8"