@scalvert/eval-core 0.5.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Steve Calvert
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,109 @@
+# @scalvert/eval-core
+
+[![CI Build](https://github.com/scalvert/eval-core/actions/workflows/ci-build.yml/badge.svg)](https://github.com/scalvert/eval-core/actions/workflows/ci-build.yml)
+[![npm version](https://badge.fury.io/js/%40scalvert%2Feval-core.svg)](https://www.npmjs.com/package/@scalvert/eval-core)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+General-purpose LLM evaluation primitives. Run test cases against a model, score responses with an LLM judge, and compare results against a saved baseline.
+
+## Installation
+
+```sh
+npm install @scalvert/eval-core
+```
+
+## Usage
+
+### Running evaluations
+
+The core workflow: provide a `respond` function (calls your model), a `judge` function (scores the response), and a set of test cases.
+
+```typescript
+import { runEval, createAnthropicJudge, type ResponseFn } from '@scalvert/eval-core';
+
+const respond: ResponseFn = async (input) => {
+  // Call your model here — any provider, any API
+  return { response: 'model output', inputTokens: 100, outputTokens: 50 };
+};
+
+const judge = createAnthropicJudge({
+  model: 'claude-sonnet-4-6',
+  threshold: 0.7,
+});
+
+const result = await runEval({
+  testCases: [{ name: 'greeting', input: 'Say hello', rubric: 'Response is a friendly greeting' }],
+  respond,
+  judge,
+  concurrency: 3,
+});
+
+console.log(result.passRate); // 0.0–1.0
+```
+
+### Comparing runs against a baseline
+
+```typescript
+import { compareRuns, loadBaseline, saveBaseline } from '@scalvert/eval-core';
+
+// Save a run as the baseline
+await saveBaseline(result, 'baseline.json');
+
+// Later, compare a new run against it
+const baseline = await loadBaseline('baseline.json');
+if (baseline) {
+  const { passRateDelta, regressions, improvements } = compareRuns(newResult, baseline);
+}
+```
+
+### Validating test case JSON
+
+```typescript
+import { TestCaseSchema } from '@scalvert/eval-core';
+import { z } from 'zod';
+
+const testCases = z.array(TestCaseSchema).parse(JSON.parse(rawJson));
+```
+
+### Custom judges
+
+`createAnthropicJudge` is a convenience — you can pass any function matching `JudgeFn`:
+
+```typescript
+import type { JudgeFn } from '@scalvert/eval-core';
+
+const myJudge: JudgeFn = async ({ input, response, rubric }) => {
+  // Your own scoring logic
+  return { passed: true, score: 0.95, reasoning: 'Looks good' };
+};
+```
+
+## API
+
+### `runEval(options)`
+
+Runs test cases with concurrency control, returning a `RunResult` with pass rate, per-case scores, and token usage.
+
+### `createAnthropicJudge(config)`
+
+Factory that returns a `JudgeFn` using the Anthropic messages API. Scores responses against a rubric and applies a threshold.
+
+### `compareRuns(current, baseline)`
+
+Returns `passRateDelta`, `regressions` (names that went from pass to fail), and `improvements` (fail to pass).
+
+### `saveBaseline(result, filePath)` / `loadBaseline(filePath)`
+
+Persist and load `RunResult` objects as JSON. `loadBaseline` returns `null` for missing files and throws on malformed data.
+
+### `buildPassMap(result)`
+
+Returns a `Map<string, boolean>` of test name to pass/fail status.
+
+### `calculateCost(pricing, inputTokens, outputTokens)`
+
+Calculates cost from token counts given a `Pricing` object (`{ inputPerMillion, outputPerMillion }`).
+
+## License
+
+MIT

package/dist/index.d.ts

@@ -0,0 +1,224 @@
+import { z } from 'zod';
+
+interface TestCase {
+    name: string;
+    input: string;
+    rubric: string;
+}
+interface TestResult {
+    name: string;
+    passed: boolean;
+    score: number;
+    reasoning: string;
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number;
+    durationMs: number;
+}
+interface RunResult {
+    runId: string;
+    timestamp: string;
+    passRate: number;
+    results: TestResult[];
+    totalInputTokens: number;
+    totalOutputTokens: number;
+    totalCostUsd: number;
+}
+type JudgeFn = (options: {
+    input: string;
+    response: string;
+    rubric: string;
+}) => Promise<{
+    passed: boolean;
+    score: number;
+    reasoning: string;
+    inputTokens?: number;
+    outputTokens?: number;
+}>;
+type ResponseFn = (input: string) => Promise<{
+    response: string;
+    inputTokens?: number;
+    outputTokens?: number;
+}>;
+declare const TestCaseSchema: z.ZodObject<{
+    name: z.ZodString;
+    input: z.ZodString;
+    rubric: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    name: string;
+    input: string;
+    rubric: string;
+}, {
+    name: string;
+    input: string;
+    rubric: string;
+}>;
+declare const TestResultSchema: z.ZodObject<{
+    name: z.ZodString;
+    passed: z.ZodBoolean;
+    score: z.ZodNumber;
+    reasoning: z.ZodString;
+    inputTokens: z.ZodNumber;
+    outputTokens: z.ZodNumber;
+    costUsd: z.ZodNumber;
+    durationMs: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    name: string;
+    passed: boolean;
+    score: number;
+    reasoning: string;
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number;
+    durationMs: number;
+}, {
+    name: string;
+    passed: boolean;
+    score: number;
+    reasoning: string;
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number;
+    durationMs: number;
+}>;
+declare const RunResultSchema: z.ZodObject<{
+    runId: z.ZodString;
+    timestamp: z.ZodString;
+    passRate: z.ZodNumber;
+    results: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        passed: z.ZodBoolean;
+        score: z.ZodNumber;
+        reasoning: z.ZodString;
+        inputTokens: z.ZodNumber;
+        outputTokens: z.ZodNumber;
+        costUsd: z.ZodNumber;
+        durationMs: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        name: string;
+        passed: boolean;
+        score: number;
+        reasoning: string;
+        inputTokens: number;
+        outputTokens: number;
+        costUsd: number;
+        durationMs: number;
+    }, {
+        name: string;
+        passed: boolean;
+        score: number;
+        reasoning: string;
+        inputTokens: number;
+        outputTokens: number;
+        costUsd: number;
+        durationMs: number;
+    }>, "many">;
+    totalInputTokens: z.ZodNumber;
+    totalOutputTokens: z.ZodNumber;
+    totalCostUsd: z.ZodNumber;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    runId: z.ZodString;
+    timestamp: z.ZodString;
+    passRate: z.ZodNumber;
+    results: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        passed: z.ZodBoolean;
+        score: z.ZodNumber;
+        reasoning: z.ZodString;
+        inputTokens: z.ZodNumber;
+        outputTokens: z.ZodNumber;
+        costUsd: z.ZodNumber;
+        durationMs: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        name: string;
+        passed: boolean;
+        score: number;
+        reasoning: string;
+        inputTokens: number;
+        outputTokens: number;
+        costUsd: number;
+        durationMs: number;
+    }, {
+        name: string;
+        passed: boolean;
+        score: number;
+        reasoning: string;
+        inputTokens: number;
+        outputTokens: number;
+        costUsd: number;
+        durationMs: number;
+    }>, "many">;
+    totalInputTokens: z.ZodNumber;
+    totalOutputTokens: z.ZodNumber;
+    totalCostUsd: z.ZodNumber;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    runId: z.ZodString;
+    timestamp: z.ZodString;
+    passRate: z.ZodNumber;
+    results: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        passed: z.ZodBoolean;
+        score: z.ZodNumber;
+        reasoning: z.ZodString;
+        inputTokens: z.ZodNumber;
+        outputTokens: z.ZodNumber;
+        costUsd: z.ZodNumber;
+        durationMs: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        name: string;
+        passed: boolean;
+        score: number;
+        reasoning: string;
+        inputTokens: number;
+        outputTokens: number;
+        costUsd: number;
+        durationMs: number;
+    }, {
+        name: string;
+        passed: boolean;
+        score: number;
+        reasoning: string;
+        inputTokens: number;
+        outputTokens: number;
+        costUsd: number;
+        durationMs: number;
+    }>, "many">;
+    totalInputTokens: z.ZodNumber;
+    totalOutputTokens: z.ZodNumber;
+    totalCostUsd: z.ZodNumber;
+}, z.ZodTypeAny, "passthrough">>;
+
+interface Pricing {
+    inputPerMillion: number;
+    outputPerMillion: number;
+}
+declare function calculateCost(pricing: Pricing, inputTokens: number, outputTokens: number): number;
+
+interface RunEvalOptions {
+    testCases: TestCase[];
+    respond: ResponseFn;
+    judge: JudgeFn;
+    concurrency?: number;
+    pricing?: Pricing;
+}
+declare function runEval(options: RunEvalOptions): Promise<RunResult>;
+
+interface AnthropicJudgeConfig {
+    apiKey?: string;
+    model: string;
+    threshold: number;
+}
+declare function createAnthropicJudge(config: AnthropicJudgeConfig): JudgeFn;
+
+interface CompareResult {
+    passRateDelta: number;
+    regressions: string[];
+    improvements: string[];
+}
+declare function compareRuns(current: RunResult, baseline: RunResult): CompareResult;
+
+declare function saveBaseline(result: RunResult, filePath: string): Promise<void>;
+declare function loadBaseline(filePath: string): Promise<RunResult | null>;
+declare function buildPassMap(result: RunResult): Map<string, boolean>;
+
+export { type AnthropicJudgeConfig, type CompareResult, type JudgeFn, type Pricing, type ResponseFn, type RunEvalOptions, type RunResult, RunResultSchema, type TestCase, TestCaseSchema, type TestResult, TestResultSchema, buildPassMap, calculateCost, compareRuns, createAnthropicJudge, loadBaseline, runEval, saveBaseline };

package/dist/index.js

@@ -0,0 +1,232 @@
+// src/types.ts
+import { z } from "zod";
+var TestCaseSchema = z.object({
+  name: z.string().min(1),
+  input: z.string(),
+  rubric: z.string()
+});
+var TestResultSchema = z.object({
+  name: z.string(),
+  passed: z.boolean(),
+  score: z.number().min(0).max(1),
+  reasoning: z.string(),
+  inputTokens: z.number(),
+  outputTokens: z.number(),
+  costUsd: z.number(),
+  durationMs: z.number()
+});
+var RunResultSchema = z.object({
+  runId: z.string(),
+  timestamp: z.string(),
+  passRate: z.number().min(0).max(1),
+  results: z.array(TestResultSchema),
+  totalInputTokens: z.number(),
+  totalOutputTokens: z.number(),
+  totalCostUsd: z.number()
+}).passthrough();
+
+// src/cost.ts
+function calculateCost(pricing, inputTokens, outputTokens) {
+  return inputTokens / 1e6 * pricing.inputPerMillion + outputTokens / 1e6 * pricing.outputPerMillion;
+}
+
+// src/runner.ts
+async function runEval(options) {
+  const { testCases, respond, judge, concurrency = 3, pricing } = options;
+  const runId = `run-${Date.now()}`;
+  const timestamp = (/* @__PURE__ */ new Date()).toISOString();
+  const results = [];
+  const queue = [...testCases];
+  const running = /* @__PURE__ */ new Set();
+  async function processCase(testCase) {
+    const start = performance.now();
+    const respondResult = await respond(testCase.input);
+    const judgeResult = await judge({
+      input: testCase.input,
+      response: respondResult.response,
+      rubric: testCase.rubric
+    });
+    const durationMs = Math.round(performance.now() - start);
+    const inputTokens = (respondResult.inputTokens ?? 0) + (judgeResult.inputTokens ?? 0);
+    const outputTokens = (respondResult.outputTokens ?? 0) + (judgeResult.outputTokens ?? 0);
+    const costUsd = pricing ? calculateCost(pricing, inputTokens, outputTokens) : 0;
+    results.push({
+      name: testCase.name,
+      passed: judgeResult.passed,
+      score: judgeResult.score,
+      reasoning: judgeResult.reasoning,
+      inputTokens,
+      outputTokens,
+      costUsd,
+      durationMs
+    });
+  }
+  for (const testCase of queue) {
+    if (running.size >= concurrency) {
+      await Promise.race(running);
+    }
+    const promise = processCase(testCase).then(() => {
+      running.delete(promise);
+    });
+    running.add(promise);
+  }
+  await Promise.all(running);
+  const passedCount = results.filter((r) => r.passed).length;
+  const totalInputTokens = results.reduce((sum, r) => sum + r.inputTokens, 0);
+  const totalOutputTokens = results.reduce((sum, r) => sum + r.outputTokens, 0);
+  const totalCostUsd = results.reduce((sum, r) => sum + r.costUsd, 0);
+  return {
+    runId,
+    timestamp,
+    passRate: testCases.length > 0 ? passedCount / testCases.length : 0,
+    results,
+    totalInputTokens,
+    totalOutputTokens,
+    totalCostUsd
+  };
+}
+
+// src/judge.ts
+import Anthropic from "@anthropic-ai/sdk";
+import { z as z2 } from "zod";
+var JudgeResponseSchema = z2.object({
+  pass: z2.boolean(),
+  score: z2.number().min(0).max(1),
+  reasoning: z2.string()
+});
+function createAnthropicJudge(config) {
+  const client = new Anthropic({ apiKey: config.apiKey });
+  return async ({ input, response, rubric }) => {
+    const message = await client.messages.create({
+      model: config.model,
+      max_tokens: 1024,
+      system: 'You are an objective evaluator. Score whether a model response satisfies a rubric. Respond ONLY with JSON: { "pass": boolean, "score": number (0.0\u20131.0), "reasoning": string }',
+      messages: [
+        {
+          role: "user",
+          content: `Rubric: ${rubric}
+
+User message: ${input}
+
+Response to evaluate:
+${response}
+
+Does this response satisfy the rubric?`
+        }
+      ]
+    });
+    const text = message.content[0].type === "text" ? message.content[0].text : "";
+    const parsed = parseJudgeResponse(text);
+    return {
+      passed: parsed.score >= config.threshold,
+      score: parsed.score,
+      reasoning: parsed.reasoning,
+      inputTokens: message.usage.input_tokens,
+      outputTokens: message.usage.output_tokens
+    };
+  };
+}
+function parseJudgeResponse(text) {
+  let jsonText = text.trim();
+  if (jsonText.startsWith("```json")) {
+    jsonText = jsonText.slice(7);
+  }
+  if (jsonText.startsWith("```")) {
+    jsonText = jsonText.slice(3);
+  }
+  if (jsonText.endsWith("```")) {
+    jsonText = jsonText.slice(0, -3);
+  }
+  jsonText = jsonText.trim();
+  let parsed;
+  try {
+    parsed = JSON.parse(jsonText);
+  } catch {
+    const jsonMatch = jsonText.match(/\{[\s\S]*"pass"[\s\S]*\}/);
+    if (jsonMatch) {
+      parsed = JSON.parse(jsonMatch[0]);
+    } else {
+      throw new Error(`Failed to parse judge response as JSON: ${text}`);
+    }
+  }
+  const result = JudgeResponseSchema.safeParse(parsed);
+  if (!result.success) {
+    throw new Error(
+      `Judge returned invalid response. Expected {pass, score, reasoning} but got: ${jsonText.slice(0, 500)}
+Validation errors: ${JSON.stringify(result.error.issues)}`
+    );
+  }
+  return result.data;
+}
+
+// src/compare.ts
+function compareRuns(current, baseline) {
+  const baselineMap = /* @__PURE__ */ new Map();
+  for (const r of baseline.results) {
+    baselineMap.set(r.name, r.passed);
+  }
+  const currentMap = /* @__PURE__ */ new Map();
+  for (const r of current.results) {
+    currentMap.set(r.name, r.passed);
+  }
+  const regressions = [];
+  const improvements = [];
+  for (const [name, currentPassed] of currentMap) {
+    const baselinePassed = baselineMap.get(name);
+    if (baselinePassed === void 0) continue;
+    if (baselinePassed && !currentPassed) regressions.push(name);
+    if (!baselinePassed && currentPassed) improvements.push(name);
+  }
+  return {
+    passRateDelta: current.passRate - baseline.passRate,
+    regressions,
+    improvements
+  };
+}
+
+// src/baseline.ts
+import { readFile, writeFile, mkdir } from "fs/promises";
+import { dirname } from "path";
+async function saveBaseline(result, filePath) {
+  await mkdir(dirname(filePath), { recursive: true });
+  await writeFile(filePath, JSON.stringify(result, null, 2), "utf8");
+}
+async function loadBaseline(filePath) {
+  let raw;
+  try {
+    raw = await readFile(filePath, "utf8");
+  } catch (error) {
+    if (error instanceof Error && "code" in error && error.code === "ENOENT") {
+      return null;
+    }
+    throw error;
+  }
+  const parsed = JSON.parse(raw);
+  const result = RunResultSchema.safeParse(parsed);
+  if (!result.success) {
+    throw new Error(
+      `Malformed baseline file: ${filePath}
+Validation errors: ${JSON.stringify(result.error.issues)}`
+    );
+  }
+  return result.data;
+}
+function buildPassMap(result) {
+  const map = /* @__PURE__ */ new Map();
+  for (const r of result.results) {
+    map.set(r.name, r.passed);
+  }
+  return map;
+}
+export {
+  RunResultSchema,
+  TestCaseSchema,
+  TestResultSchema,
+  buildPassMap,
+  calculateCost,
+  compareRuns,
+  createAnthropicJudge,
+  loadBaseline,
+  runEval,
+  saveBaseline
+};

package/package.json

@@ -0,0 +1,89 @@
+{
+  "name": "@scalvert/eval-core",
+  "version": "0.5.0",
+  "description": "General-purpose LLM evaluation primitives",
+  "keywords": [
+    "llm",
+    "evaluation",
+    "testing",
+    "judge",
+    "baseline"
+  ],
+  "homepage": "https://github.com/scalvert/eval-core#readme",
+  "bugs": {
+    "url": "https://github.com/scalvert/eval-core/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/scalvert/eval-core.git"
+  },
+  "license": "MIT",
+  "author": "Steve Calvert <steve.calvert@gmail.com>",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsup src/index.ts --format esm --dts --clean",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "lint": "eslint . && npm run format:check",
+    "prepublishOnly": "npm run build",
+    "test": "npm run lint && npm run test:vitest",
+    "test:vitest": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.39.0",
+    "zod": "^3.22.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.24.0",
+    "@release-it-plugins/lerna-changelog": "^5.0.0",
+    "@types/node": "^22.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.29.1",
+    "@typescript-eslint/parser": "^8.29.1",
+    "eslint": "^9.24.0",
+    "eslint-config-prettier": "^10.1.2",
+    "eslint-plugin-jsdoc": "^61.5.0",
+    "eslint-plugin-prettier": "^5.2.6",
+    "eslint-plugin-unicorn": "^62.0.0",
+    "prettier": "^3.5.3",
+    "release-it": "^15.5.0",
+    "tsup": "^8.0.0",
+    "typescript": "^5.4.0",
+    "vite": "^7.3.0",
+    "vitest": "^4.0.16"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "release-it": {
+    "plugins": {
+      "@release-it-plugins/lerna-changelog": {
+        "infile": "CHANGELOG.md",
+        "launchEditor": true
+      }
+    },
+    "git": {
+      "tagName": "v${version}"
+    },
+    "github": {
+      "release": true,
+      "tokenRef": "GITHUB_AUTH"
+    }
+  }
+}