@agentgrader/scorer-llm-judge 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,30 @@
+import { Scorer, TestCase, AgentResult, ScorerResult } from '@agentgrader/core';
+
+interface LlmJudgeScorerOptions {
+    /** which AI SDK provider to judge with. Defaults to "anthropic". */
+    provider?: "anthropic" | "openai";
+    /** model slug for the chosen provider. Defaults to a small, cheap model suitable for judging. */
+    model?: string;
+}
+/**
+ * Additive, non-blocking LLM-as-judge scorer: asks a model to rate the
+ * agent's diff (0-1) for code quality and fit to the task, given the
+ * original prompt. Never blocks a run (`passed` is always `true`) - it
+ * only annotates `metrics["llm-judge"].quality.{llmJudgeScore,llmJudgeDetail}`.
+ *
+ * Degrades gracefully (no `quality` field) if the diff is empty or the
+ * judge call fails (e.g. missing API key, network unavailable).
+ */
+declare class LlmJudgeScorer implements Scorer {
+    readonly name = "llm-judge";
+    private readonly provider;
+    private readonly modelName;
+    constructor(options?: LlmJudgeScorerOptions);
+    score(input: {
+        testCase: TestCase;
+        result: AgentResult;
+    }): Promise<ScorerResult>;
+    private resolveModel;
+}
+
+export { LlmJudgeScorer, type LlmJudgeScorerOptions };

package/dist/index.js

@@ -0,0 +1,61 @@
+import { createAnthropic } from '@ai-sdk/anthropic';
+import { createOpenAI } from '@ai-sdk/openai';
+import { generateObject } from 'ai';
+import { z } from 'zod';
+
+// src/index.ts
+var JudgeResponseSchema = z.object({
+  score: z.number().min(0).max(1),
+  rationale: z.string()
+});
+var DEFAULT_MODELS = {
+  anthropic: "claude-3-5-haiku-20241022",
+  openai: "gpt-4o-mini"
+};
+var LlmJudgeScorer = class {
+  name = "llm-judge";
+  provider;
+  modelName;
+  constructor(options = {}) {
+    this.provider = options.provider ?? "anthropic";
+    this.modelName = options.model ?? DEFAULT_MODELS[this.provider];
+  }
+  async score(input) {
+    const diff = input.result.finalDiff ?? "";
+    if (!diff.trim()) {
+      return { passed: true, detail: "llm-judge: no diff to judge" };
+    }
+    try {
+      const { object } = await generateObject({
+        model: this.resolveModel(),
+        schema: JudgeResponseSchema,
+        system: "You are a senior software engineer reviewing a coding agent's patch. Rate the patch from 0 (poor: incorrect, sloppy, or unrelated changes) to 1 (excellent: correct, minimal, idiomatic, and focused on the task). Respond with a score and a one-sentence rationale.",
+        prompt: `Task:
+${input.testCase.prompt}
+
+Patch:
+${diff}`
+      });
+      return {
+        passed: true,
+        detail: `llm-judge: ${object.score.toFixed(2)} - ${object.rationale}`,
+        quality: {
+          llmJudgeScore: object.score,
+          llmJudgeDetail: object.rationale
+        }
+      };
+    } catch (e) {
+      return { passed: true, detail: `llm-judge unavailable: ${e.message}` };
+    }
+  }
+  resolveModel() {
+    if (this.provider === "openai") {
+      const openai = createOpenAI({ apiKey: process.env.OPENAI_API_KEY || "mock-key" });
+      return openai(this.modelName);
+    }
+    const anthropic = createAnthropic({ apiKey: process.env.ANTHROPIC_API_KEY || "mock-key" });
+    return anthropic(this.modelName);
+  }
+};
+
+export { LlmJudgeScorer };

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@agentgrader/scorer-llm-judge",
+  "version": "0.1.0",
+  "description": "Additive, non-blocking LLM-as-judge code-quality scorer for the Agentgrader framework",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsup src/index.ts --format esm --dts --clean --treeshake",
+    "build:watch": "tsup src/index.ts --format esm --dts --watch"
+  },
+  "dependencies": {
+    "@agentgrader/core": "^1.1.0",
+    "@ai-sdk/anthropic": "^3.0.82",
+    "@ai-sdk/openai": "^1.1.0",
+    "ai": "^4.1.0",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "tsup": "^8.5.1"
+  },
+  "peerDependencies": {
+    "@agentgrader/core": "^1.1.0"
+  }
+}