@agentgrader/core 1.0.1 → 1.1.2

package/dist/index.d.ts

@@ -131,6 +131,7 @@ declare const AgentConfigSchema: z.ZodObject<{
     id: z.ZodOptional<z.ZodString>;
     name: z.ZodString;
     model: z.ZodString;
+    provider: z.ZodOptional<z.ZodString>;
     max_steps: z.ZodDefault<z.ZodNumber>;
     temperature: z.ZodOptional<z.ZodNumber>;
     system_prompt: z.ZodOptional<z.ZodString>;
@@ -167,6 +168,7 @@ declare const AgentConfigSchema: z.ZodObject<{
     max_steps: number;
     id?: string | undefined;
     toolkits?: string[] | undefined;
+    provider?: string | undefined;
     temperature?: number | undefined;
     system_prompt?: string | undefined;
     tools?: string[] | undefined;
@@ -184,6 +186,7 @@ declare const AgentConfigSchema: z.ZodObject<{
     model: string;
     id?: string | undefined;
     toolkits?: string[] | undefined;
+    provider?: string | undefined;
     max_steps?: number | undefined;
     temperature?: number | undefined;
     system_prompt?: string | undefined;
@@ -300,6 +303,7 @@ declare const RunSchema: z.ZodObject<{
     error: z.ZodOptional<z.ZodString>;
     finalDiff: z.ZodOptional<z.ZodString>;
     metrics: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    matrixId: z.ZodOptional<z.ZodString>;
     createdAt: z.ZodNumber;
     completedAt: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
@@ -319,6 +323,7 @@ declare const RunSchema: z.ZodObject<{
     error?: string | undefined;
     finalDiff?: string | undefined;
     metrics?: Record<string, any> | undefined;
+    matrixId?: string | undefined;
     completedAt?: number | undefined;
 }, {
     status: "running" | "completed" | "failed";
@@ -337,6 +342,7 @@ declare const RunSchema: z.ZodObject<{
     error?: string | undefined;
     finalDiff?: string | undefined;
     metrics?: Record<string, any> | undefined;
+    matrixId?: string | undefined;
     completedAt?: number | undefined;
 }>;
 type Run = z.infer<typeof RunSchema>;
@@ -493,6 +499,26 @@ interface ScorerResult {
     passed: boolean;
     detail: string;
     score?: number;
+    /**
+     * non-functional code-quality signals. populated by additive scorers
+     * (e.g. staticqualityscorer, llmjudgescorer) that annotate a run without
+     * affecting `passed`/`score`. all fields optional - a scorer only fills
+     * in what it actually measured.
+     */
+    quality?: {
+        /** total +/- lines in the agent's diff */
+        diffLines?: number;
+        /** number of files touched by the agent's diff */
+        filesModified?: number;
+        /** TODO/FIXME/HACK/XXX markers introduced by the diff */
+        todosIntroduced?: number;
+        /** linter (e.g. Biome) error+warning count on the changed files */
+        linterViolations?: number;
+        /** 0-1 holistic quality score from an LLM judge */
+        llmJudgeScore?: number;
+        /** prose rationale from an LLM judge */
+        llmJudgeDetail?: string;
+    };
 }
 interface Scorer {
     readonly name: string;
@@ -618,6 +644,15 @@ interface RunSingleInput {
     sandboxProvider: SandboxProvider;
     db?: AgrDb;
     runId: string;
+    /**
+     * additive, non-blocking scorers (e.g. staticqualityscorer,
+     * llmjudgescorer) run after the core pass/fail scoring. their results
+     * never affect `passed`/`score` - each scorer's `scorerresult` is merged
+     * into `metrics` under its own `name`.
+     */
+    extraScorers?: Scorer[];
+    /** links this run to an optimizer matrix run, if any */
+    matrixId?: string;
 }
 interface RunSingleResult {
     runId: string;
@@ -637,13 +672,19 @@ declare function runSingle(input: RunSingleInput): Promise<RunSingleResult>;
 interface BenchmarkInput {
     testCases: TestCase[];
     agentConfigs: AgentConfig[];
-    adapter: AgentAdapter;
+    adapter?: AgentAdapter;
+    adapters?: AgentAdapter[];
     sandboxProvider: SandboxProvider;
     db?: AgrDb;
     concurrency?: number;
+    /** additive, non-blocking quality scorers run for every test case x config combination */
+    extraScorers?: Scorer[];
+    /** links every run in this benchmark to an optimizer matrix run, if any */
+    matrixId?: string;
     onRunUpdate?: (run: RunSingleResult & {
         testCaseId: string;
         agentConfigId: string;
+        adapterName?: string;
         status: "running" | "completed" | "failed";
     }) => void;
 }

package/dist/index.js

@@ -79,6 +79,7 @@ var AgentConfigSchema = z.object({
   id: z.string().optional(),
   name: z.string(),
   model: z.string(),
+  provider: z.string().optional(),
   max_steps: z.number().default(30),
   temperature: z.number().optional(),
   system_prompt: z.string().optional(),
@@ -105,8 +106,12 @@ var RunSchema = z.object({
   error: z.string().optional(),
   finalDiff: z.string().optional(),
   // extended scoring metrics: regression (FAIL_TO_PASS/PASS_TO_PASS), diff stats,
-  // localization precision/recall, etc. Stored as JSON.
+  // localization precision/recall, additive quality scorers (keyed by scorer
+  // name, ScorerResult-shaped), etc. Stored as JSON.
   metrics: z.record(z.any()).optional(),
+  // links this run back to the optimizer matrix run that generated its
+  // agentConfig, if any (see @agentgrader/optimizer).
+  matrixId: z.string().optional(),
   createdAt: z.number(),
   completedAt: z.number().optional()
 });
@@ -575,7 +580,7 @@ function buildSkillsPromptAddendum(skills) {
 
 // src/runner/run-single.ts
 async function runSingle(input) {
-  const { testCase, agentConfig, adapter, sandboxProvider, db, runId } = input;
+  const { testCase, agentConfig, adapter, sandboxProvider, db, runId, extraScorers, matrixId } = input;
   const startTime = Date.now();
   let sandbox = null;
   let passed = false;
@@ -598,6 +603,7 @@ async function runSingle(input) {
       agentConfigId: agentConfig.id || agentConfig.name,
       sandboxProvider: sandboxProvider.name,
       status: "running",
+      matrixId,
       createdAt: Math.floor(startTime / 1e3)
     });
   }
@@ -701,6 +707,21 @@ ${addendum}` : addendum
     }),
     execute: async () => {
       if (!sandbox) throw new Error("Sandbox not initialized");
+      if (extraScorers && extraScorers.length > 0 && agentResult) {
+        const trace2 = { runId, steps: emittedSteps };
+        for (const scorer of extraScorers) {
+          try {
+            metrics[scorer.name] = await scorer.score({
+              testCase,
+              result: agentResult,
+              trace: trace2,
+              sandbox
+            });
+          } catch (e) {
+            metrics[scorer.name] = { passed: true, detail: `Scorer error: ${e.message}` };
+          }
+        }
+      }
       const cmdScorer = new CommandScorer();
       const cmdResult = await cmdScorer.score({
         testCase,
@@ -790,10 +811,17 @@ ${addendum}` : addendum
       inputData: {},
       initialState: runState
     });
-    const scoreResults = res.results?.score;
-    passed = scoreResults?.passed ?? false;
-    score = scoreResults?.score ?? 0;
-    errorMsg = scoreResults?.passed ? void 0 : scoreResults?.detail;
+    const scoreStep2 = res.steps?.score;
+    if (res.status === "success" && scoreStep2?.status === "success") {
+      const scoreOutput = scoreStep2.output;
+      passed = scoreOutput?.passed ?? false;
+      score = scoreOutput?.score ?? 0;
+      errorMsg = scoreOutput?.passed ? void 0 : scoreOutput?.detail;
+    } else {
+      passed = false;
+      score = 0;
+      errorMsg = res.error?.message ?? "Workflow did not complete successfully";
+    }
   } catch (err) {
     errorMsg = err.message || "Unknown execution error";
     passed = false;
@@ -836,7 +864,11 @@ ${addendum}` : addendum
   };
 }
 async function runBenchmark(input) {
-  const { testCases, agentConfigs, adapter, sandboxProvider, db, concurrency = 2, onRunUpdate } = input;
+  const { testCases, agentConfigs, adapter, adapters, sandboxProvider, db, concurrency = 2, extraScorers, matrixId, onRunUpdate } = input;
+  const actualAdapters = adapters || (adapter ? [adapter] : []);
+  if (actualAdapters.length === 0) {
+    throw new Error("You must provide either 'adapter' or 'adapters' to runBenchmark.");
+  }
   const generateCombinationsStep = createStep({
     id: "generateCombinations",
     inputSchema: z.any(),
@@ -846,10 +878,13 @@ async function runBenchmark(input) {
       const combinations = [];
       for (const tc of initData.testCases) {
         for (const config of initData.agentConfigs) {
-          combinations.push({
-            testCase: tc,
-            agentConfig: config
-          });
+          for (const adapterName of initData.adapterNames) {
+            combinations.push({
+              testCase: tc,
+              agentConfig: config,
+              adapterName
+            });
+          }
         }
       }
       return combinations;
@@ -860,7 +895,7 @@ async function runBenchmark(input) {
     inputSchema: z.any(),
     outputSchema: z.any(),
     execute: async ({ inputData, requestContext }) => {
-      const { testCase, agentConfig } = inputData;
+      const { testCase, agentConfig, adapterName } = inputData;
       const ctx = requestContext?.context || requestContext;
       const getVal = (key) => {
         if (ctx instanceof Map) return ctx.get(key);
@@ -868,16 +903,23 @@ async function runBenchmark(input) {
         if (typeof ctx?.get === "function") return ctx.get(key);
         return void 0;
       };
-      const adapter2 = getVal("adapter");
+      const adaptersFromCtx = getVal("adapters");
+      const singleAdapter = getVal("adapter");
+      const adapterList = adaptersFromCtx || (singleAdapter ? [singleAdapter] : []);
+      const adapter2 = adapterList.find((a) => a.name === adapterName);
+      if (!adapter2) throw new Error(`Adapter ${adapterName} not found in execution context`);
       const sandboxProvider2 = getVal("sandboxProvider");
       const db2 = getVal("db");
       const onRunUpdate2 = getVal("onRunUpdate");
+      const extraScorers2 = getVal("extraScorers");
+      const matrixId2 = getVal("matrixId");
       const runId = randomUUID();
       if (onRunUpdate2) {
         onRunUpdate2({
           runId,
           testCaseId: testCase.id || testCase.name,
           agentConfigId: agentConfig.id || agentConfig.name,
+          adapterName: adapter2.name,
           status: "running",
           passed: false,
           stepsCount: 0,
@@ -894,13 +936,16 @@ async function runBenchmark(input) {
           adapter: adapter2,
           sandboxProvider: sandboxProvider2,
           db: db2,
-          runId
+          runId,
+          extraScorers: extraScorers2,
+          matrixId: matrixId2
         });
         if (onRunUpdate2) {
           onRunUpdate2({
             ...res2,
             testCaseId: testCase.id || testCase.name,
             agentConfigId: agentConfig.id || agentConfig.name,
+            adapterName: adapter2.name,
             status: res2.error ? "failed" : "completed"
           });
         }
@@ -921,6 +966,7 @@ async function runBenchmark(input) {
             ...failedResult,
             testCaseId: testCase.id || testCase.name,
             agentConfigId: agentConfig.id || agentConfig.name,
+            adapterName: adapter2.name,
             status: "failed"
           });
         }
@@ -936,17 +982,20 @@ async function runBenchmark(input) {
   const runState = {};
   const executionContext = /* @__PURE__ */ new Map([
     ["adapter", adapter],
+    ["adapters", actualAdapters],
     ["sandboxProvider", sandboxProvider],
     ["db", db],
-    ["onRunUpdate", onRunUpdate]
+    ["onRunUpdate", onRunUpdate],
+    ["extraScorers", extraScorers],
+    ["matrixId", matrixId]
   ]);
   const run = await workflow.createRun();
   const res = await run.start({
-    inputData: { testCases, agentConfigs },
+    inputData: { testCases, agentConfigs, adapterNames: actualAdapters.map((a) => a.name) },
     initialState: runState,
     requestContext: executionContext
   });
-  const rawRuns = res.results?.executeSingleRunStepResult || [];
+  const rawRuns = res.steps?.executeSingleRun?.output || [];
   return {
     runs: Array.isArray(rawRuns) ? rawRuns : []
   };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentgrader/core",
-  "version": "1.0.1",
+  "version": "1.1.2",
   "description": "Core schemas, contracts, and runner for the Agentgrader benchmarking framework",
   "license": "MIT",
   "type": "module",
@@ -22,7 +22,7 @@
     "dev": "bun run src/index.ts"
   },
   "dependencies": {
-    "@agentgrader/store": "^1.0.1",
+    "@agentgrader/store": "^1.0.2",
     "@mastra/core": "^1.41.0",
     "yaml": "^2.5.1",
     "zod": "^3.23.8"