@hazeljs/eval 2.0.13 → 2.0.14

package/README.md

@@ -23,9 +23,10 @@ npm install @hazeljs/core
 - **Retrieval metrics** — `precisionAtK`, `recallAtK`, `meanReciprocalRank`, `ndcgAtK`
 - **RAG helpers** — `evaluateRetrieval`, `answerContextOverlap` (lightweight faithfulness proxy)
 - **Agent trajectories** — `trajectoryScore`, `toolCallAccuracy`
-- **LLM judge** — `parseJudgeScore`, `buildRelevanceJudgePrompt`, `buildFaithfulnessJudgePrompt` (wire to your model)
+- **LLM judge** — `parseJudgeScore`, `buildRelevanceJudgePrompt`, `buildFaithfulnessJudgePrompt`; pass `judge` on `EvalRunOptions` to blend into `runGoldenDataset`
 - **Runner** — `runGoldenDataset` with configurable concurrency and thresholds
 - **CI** — `reportEvalForCi` with optional non-zero exit via `process.exitCode`
+- **AI call goldens** — put `metadata.expectedCalls` on cases and use `@hazeljs/ai` `runAiGolden` / `FakeModel` for provider-request regression
 
 ## Quick start
 
@@ -46,7 +47,13 @@ const result = await runGoldenDataset(
       retrievedIds: ['doc-1', 'doc-2'],
     };
   },
-  { concurrency: 2, minAverageScore: 0.75 }
+  {
+    concurrency: 2,
+    minAverageScore: 0.75,
+    // optional: blend LLM-as-judge (skip per-case with metadata.skipJudge)
+    // judge: async (prompt) => ai.chat(prompt).text(),
+    // judgePrompt: 'relevance',
+  }
 );
 
 reportEvalForCi(result, { exitOnFail: true });

package/dist/golden-runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"golden-runner.d.ts","sourceRoot":"","sources":["../src/golden-runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAc,aAAa,EAAE,MAAM,SAAS,CAAC;AAIxF,MAAM,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,mDAAmD;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,KAAK,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAAC;AAEjF;;GAEG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,aAAa,EACtB,MAAM,EAAE,UAAU,EAClB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,CAyFxB"}
+{"version":3,"file":"golden-runner.d.ts","sourceRoot":"","sources":["../src/golden-runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAc,aAAa,EAAE,MAAM,SAAS,CAAC;AASxF,MAAM,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,mDAAmD;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,KAAK,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAAC;AAEjF;;GAEG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,aAAa,EACtB,MAAM,EAAE,UAAU,EAClB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,aAAa,CAAC,CA6GxB"}

package/dist/golden-runner.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.runGoldenDataset = runGoldenDataset;
 const agent_trajectory_1 = require("./agent-trajectory");
 const retrieval_metrics_1 = require("./retrieval-metrics");
+const llm_judge_1 = require("./llm-judge");
 /**
  * Run a golden dataset against an async runner (your app, agent, or RAG pipeline).
  */
@@ -22,30 +23,48 @@ async function runGoldenDataset(dataset, runner, options) {
                     metadata: c.metadata,
                 });
                 let score = 0;
+                let scoredParts = 0;
                 const details = {};
                 if (c.expectedOutput !== undefined) {
                     const out = result.output ?? '';
                     const exp = c.expectedOutput;
                     const passed = out.includes(exp) || out.trim().toLowerCase() === exp.trim().toLowerCase();
-                    score = passed ? 1 : 0;
+                    score += passed ? 1 : 0;
+                    scoredParts += 1;
                     details.outputMatch = passed;
                 }
                 if (c.expectedToolCalls && c.expectedToolCalls.length > 0) {
                     const traj = (0, agent_trajectory_1.trajectoryScore)(c.expectedToolCalls, result.toolCalls ?? []);
-                    score = score === 0 ? traj : (score + traj) / 2;
+                    score += traj;
+                    scoredParts += 1;
                     details.trajectoryScore = traj;
                 }
                 if (c.expectedRetrievedIds && c.expectedRetrievedIds.length > 0) {
                     const rel = new Set(c.expectedRetrievedIds);
                     const p = (0, retrieval_metrics_1.precisionAtK)(result.retrievedIds ?? [], rel, 5);
-                    score = score === 0 ? p : (score + p) / 2;
+                    score += p;
+                    scoredParts += 1;
                     details.precisionAt5 = p;
                 }
-                if (c.expectedOutput === undefined &&
-                    (!c.expectedToolCalls || c.expectedToolCalls.length === 0) &&
-                    (!c.expectedRetrievedIds || c.expectedRetrievedIds.length === 0)) {
+                const skipJudge = Boolean(c.metadata?.skipJudge);
+                if (options?.judge && !skipJudge) {
+                    const promptKind = options.judgePrompt ?? 'relevance';
+                    const judgePrompt = promptKind === 'faithfulness'
+                        ? (0, llm_judge_1.buildFaithfulnessJudgePrompt)(String(c.metadata?.judgeContext ?? ''), result.output ?? '')
+                        : (0, llm_judge_1.buildRelevanceJudgePrompt)(c.input, result.output ?? '');
+                    const raw = await options.judge(judgePrompt);
+                    const judged = await (0, llm_judge_1.parseJudgeScore)(raw);
+                    score += judged.score;
+                    scoredParts += 1;
+                    details.judgeScore = judged.score;
+                    details.judgeReasoning = judged.reasoning;
+                }
+                if (scoredParts === 0) {
                     score = 1;
                 }
+                else {
+                    score = score / scoredParts;
+                }
                 const threshold = options?.minAverageScore ?? 0.7;
                 return {
                     caseId: c.id,

package/dist/index.d.ts

@@ -1,7 +1,7 @@
 /**
  * @hazeljs/eval — Evaluation toolkit for AI-native HazelJS apps
  */
-export type { GoldenCase, GoldenDataset, EvalRunOptions, CaseResult, EvalRunResult } from './types';
+export type { GoldenCase, GoldenDataset, EvalRunOptions, CaseResult, EvalRunResult, ExpectedCallSlice, } from './types';
 export { precisionAtK, recallAtK, meanReciprocalRank, ndcgAtK } from './retrieval-metrics';
 export { evaluateRetrieval, answerContextOverlap, type RagRetrievalEvalInput, type RagRetrievalEvalResult, } from './rag-metrics';
 export { trajectoryScore, toolCallAccuracy, type AgentTrajectory } from './agent-trajectory';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEpG,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAE3F,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,KAAK,qBAAqB,EAC1B,KAAK,sBAAsB,GAC5B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAE7F,OAAO,EACL,eAAe,EACf,yBAAyB,EACzB,4BAA4B,EAC5B,KAAK,UAAU,EACf,KAAK,UAAU,GAChB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,gBAAgB,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAEpE,OAAO,EAAE,eAAe,EAAE,KAAK,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAExE,OAAO,EAAE,yBAAyB,EAAE,MAAM,kBAAkB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,YAAY,EACV,UAAU,EACV,aAAa,EACb,cAAc,EACd,UAAU,EACV,aAAa,EACb,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAEjB,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAE3F,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,KAAK,qBAAqB,EAC1B,KAAK,sBAAsB,GAC5B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAE7F,OAAO,EACL,eAAe,EACf,yBAAyB,EACzB,4BAA4B,EAC5B,KAAK,UAAU,EACf,KAAK,UAAU,GAChB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,gBAAgB,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAEpE,OAAO,EAAE,eAAe,EAAE,KAAK,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAExE,OAAO,EAAE,yBAAyB,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/types.d.ts

@@ -1,6 +1,21 @@
 /**
  * Core types for @hazeljs/eval
  */
+import type { LLMJudgeFn } from './llm-judge';
+/**
+ * Optional provider-call golden shape (used by `@hazeljs/ai` FakeModel matchers).
+ * Store under `GoldenCase.metadata.expectedCalls`.
+ */
+export interface ExpectedCallSlice {
+    /** Exact model id if asserted */
+    model?: string;
+    /** Subset of message roles in order (e.g. ['system','user']) */
+    messageRoles?: string[];
+    /** Tool / function names expected to be offered on the request */
+    toolNames?: string[];
+    /** Substring that must appear in some message content */
+    messageIncludes?: string;
+}
 export interface GoldenCase {
     id: string;
     input: string;
@@ -10,6 +25,12 @@ export interface GoldenCase {
     expectedToolCalls?: string[];
     /** Relevant document IDs or content snippets retrieval should surface */
     expectedRetrievedIds?: string[];
+    /**
+     * Free-form case metadata. Recognized keys:
+     * - `expectedCalls`: ExpectedCallSlice[] for FakeModel request goldens
+     * - `judgeContext`: string used when `EvalRunOptions.judgePrompt === 'faithfulness'`
+     * - `skipJudge`: boolean to skip LLM judge for this case
+     */
     metadata?: Record<string, unknown>;
 }
 export interface GoldenDataset {
@@ -22,6 +43,13 @@ export interface EvalRunOptions {
     concurrency?: number;
     /** Fail CI when average score below threshold */
     minAverageScore?: number;
+    /**
+     * Optional LLM-as-judge. When set, blends judge score with existing matchers
+     * (unless `metadata.skipJudge` is true).
+     */
+    judge?: LLMJudgeFn;
+    /** Which rubric prompt to build (default relevance). */
+    judgePrompt?: 'relevance' | 'faithfulness';
 }
 export interface CaseResult {
     caseId: string;

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,2DAA2D;IAC3D,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,yEAAyE;IACzE,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iDAAiD;IACjD,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;CACjB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAChC,iCAAiC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,kEAAkE;IAClE,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,yDAAyD;IACzD,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,2DAA2D;IAC3D,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,yEAAyE;IACzE,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iDAAiD;IACjD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,wDAAwD;IACxD,WAAW,CAAC,EAAE,WAAW,GAAG,cAAc,CAAC;CAC5C;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;CACjB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hazeljs/eval",
-  "version": "2.0.13",
+  "version": "2.0.14",
   "description": "Evaluation toolkit for HazelJS AI apps — golden datasets, RAG metrics, agent trajectories, LLM-as-judge, CI reports",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -19,10 +19,10 @@
     "zod": "^3.25.76"
   },
   "peerDependencies": {
-    "@hazeljs/core": "^2.0.13"
+    "@hazeljs/core": "^2.0.14"
   },
   "devDependencies": {
-    "@hazeljs/core": "^2.0.13",
+    "@hazeljs/core": "^2.0.14",
     "@types/jest": "^29.5.14",
     "@types/node": "^20.19.39",
     "@typescript-eslint/eslint-plugin": "^8.58.0",