jev-recipes 0.8.3 → 0.8.5

package/README.md

@@ -31,7 +31,21 @@ For example, give `route` a support message and descriptions of your teams. It r
 
 Evaluate your own cases with the [installed evaluator](https://github.com/agencyenterprise/jev-recipes/blob/main/docs/evaluation.md), retain model responses, and replay confidence policies offline. The [agent workflow](https://github.com/agencyenterprise/jev-recipes/blob/main/examples/agent-loop/README.md) and [customer queue](https://github.com/agencyenterprise/jev-recipes/blob/main/examples/customer-queue/README.md) show how decisions fit into application code. [Direct and Gateway integrations](https://github.com/agencyenterprise/jev-recipes/blob/main/docs/integrations.md) use the same recipe interface.
 
-Build the searchable static catalog with `npm run site:build`, then preview it with `npm run site:preview`. It includes fixture exploration, related-recipe comparisons, and saved evaluation evidence. Current measurements and missing or older evidence are labeled explicitly.
+## Build a complete support flow
+
+The [support conversation starter](https://github.com/agencyenterprise/jev-recipes/blob/main/examples/support-routing/README.md) checks for missing details, asks a configured question, and resumes with the customer's answer. It proposes a queue or requests human review when the conversation remains unresolved.
+
+```sh
+cd examples/support-routing
+npm ci --ignore-scripts
+npm run dev
+```
+
+Open `http://localhost:3000` and choose **Ask and continue**. This folder is portable and needs no repository build. Edit `config.mjs` to supply your queues, required information, questions, and review policy. Run `npm run demo` for the command-line version. Saved scenarios work without a key; the starter guide explains live calls and application integration.
+
+The existing [rules/Jev/fallback comparison](https://github.com/agencyenterprise/jev-recipes/blob/main/evals/support-routing/README.md) covers single-turn routing, not this clarification loop. Its evidence remains experimental.
+
+Build the searchable static catalog with `npm run site:build`, then preview it with `npm run site:preview`. It includes fixture exploration, related-recipe comparisons, and saved evaluation evidence. Current measurements and missing, unknown-origin, or older evidence are labeled explicitly. Use `npm run eval:audit` after building to check retained evidence offline; see [evaluation and archive format](https://github.com/agencyenterprise/jev-recipes/blob/main/docs/evaluation.md#archive-format-and-evidence-origin).
 
 ## Hear a real application: Jevthoven
 

package/dist/catalog/evidence.d.ts

@@ -1,10 +1,13 @@
 export interface RecipeEvidence {
-    kind: 'fixture' | 'earlier' | 'synthetic' | 'public-dataset' | 'human-reviewed' | 'mixed' | 'unspecified';
+    kind: 'fixture' | 'unknown' | 'earlier' | 'synthetic' | 'public-dataset' | 'human-reviewed' | 'mixed' | 'unspecified';
     label: string;
     experimental: boolean;
     measurement: {
         model: string;
         date: string | null;
+        operation?: string | null;
+        replayedAt?: string | null;
+        scoringRevision?: number;
         split: string | null;
         cases: number;
         ready: number | null;

package/dist/catalog/generated/metadata.js

@@ -45,12 +45,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'typesafe-ai/jev',
                 date: '2026-09-27T21:19:58.394Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 20,
@@ -86,21 +89,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 44,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {
@@ -306,21 +298,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 50,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {
@@ -583,12 +564,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:26.559Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 25,
                 ready: 24,
@@ -648,12 +632,15 @@ export const recipeMetadata = [
             { id: 'followup-timing', reason: 'Use followup-timing for when another contact is wanted.' },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:29.897Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 24,
@@ -906,21 +893,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 31,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {
@@ -963,21 +939,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 32,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {
@@ -1081,21 +1046,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 53,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {
@@ -1116,21 +1070,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 50,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {
@@ -1298,12 +1241,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'typesafe-ai/jev',
                 date: '2026-09-27T21:20:01.789Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 30,
@@ -1449,12 +1395,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:27.401Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 33,
@@ -1545,12 +1494,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
             experimental: true,
             measurement: {
                 model: 'typesafe-ai/jev',
                 date: '2026-09-27T21:21:18.930Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 20,
                 ready: 11,
@@ -1778,21 +1730,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 43,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {
@@ -1972,12 +1913,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:33.689Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 30,
@@ -2268,12 +2212,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:32.456Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 40,
@@ -2412,12 +2359,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:31.169Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 30,
@@ -2501,12 +2451,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:24.179Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 30,
@@ -2551,12 +2504,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:28.731Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 37,
@@ -2750,21 +2706,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 44,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {
@@ -2828,12 +2773,15 @@ export const recipeMetadata = [
             { id: 'route', reason: 'Use route to choose a handler when escalation is not the decision.' },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:21.635Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 28,
@@ -2958,21 +2906,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 44,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {
@@ -3686,12 +3623,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:10.135Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 26,
@@ -4118,12 +4058,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'public-dataset',
-            label: 'Current public-dataset measurement',
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
             experimental: true,
             measurement: {
                 model: 'typesafe-ai/jev',
                 date: '2026-09-27T21:21:33.447Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 167,
                 ready: 82,
@@ -4446,21 +4389,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 43,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {
@@ -4693,12 +4625,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:14.022Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 40,
@@ -4733,12 +4668,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:25.450Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 40,
@@ -5117,21 +5055,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 57,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {
@@ -5173,12 +5100,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:22.956Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 40,
@@ -5231,21 +5161,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 44,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {
@@ -5347,12 +5266,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:15.276Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 39,
@@ -5535,12 +5457,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:19.034Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 24,
@@ -5580,12 +5505,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-version measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:20.238Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 25,
@@ -6204,12 +6132,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'public-dataset',
-            label: 'Current public-dataset measurement',
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
             experimental: true,
             measurement: {
                 model: 'typesafe-ai/jev',
                 date: '2026-09-27T21:21:20.879Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 189,
                 ready: 136,
@@ -6404,12 +6335,15 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'synthetic',
-            label: 'Current synthetic measurement',
-            experimental: false,
+            kind: 'earlier',
+            label: 'Earlier-evaluator measurement',
+            experimental: true,
             measurement: {
                 model: 'jev-1.13.0',
                 date: '2026-09-27T07:56:16.449Z',
+                operation: 'live',
+                replayedAt: null,
+                scoringRevision: 1,
                 split: 'held-out',
                 cases: 40,
                 ready: 31,
@@ -6618,21 +6552,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 50,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {
@@ -6657,21 +6580,10 @@ export const recipeMetadata = [
             },
         ],
         evidence: {
-            kind: 'earlier',
-            label: 'Earlier-version measurement',
+            kind: 'unknown',
+            label: 'Unknown response origin',
             experimental: true,
-            measurement: {
-                model: 'jev-1.13.0',
-                date: null,
-                split: null,
-                cases: 44,
-                ready: null,
-                reviewRate: null,
-                readyAccuracy: null,
-                failed: null,
-                provenance: [],
-                acceptanceMet: null,
-            },
+            measurement: null,
         },
     },
     {

package/dist/evaluation/archive.d.ts

@@ -4,7 +4,13 @@ import type { EvaluationRow } from './engine.js';
 import { buildReport } from './report.js';
 import type { EvaluationOptions, EvaluationPolicy } from './schema.js';
 declare const archiveSchema: z.ZodObject<{
-    format: z.ZodLiteral<1>;
+    format: z.ZodLiteral<2>;
+    scoringRevision: z.ZodLiteral<2>;
+    sourceMode: z.ZodEnum<{
+        fixture: "fixture";
+        live: "live";
+    }>;
+    evaluatedAt: z.ZodISODateTime;
     runId: z.ZodString;
     createdAt: z.ZodISODateTime;
     mode: z.ZodEnum<{
@@ -125,6 +131,7 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             } | undefined;
         };
         mode: "fixture" | "live" | "replay";
+        scoringRevision: 2;
         report: {
             contestedAccuracy: number | null;
             adversarialAccuracy: number | null;
@@ -310,6 +317,7 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             } | undefined;
         };
         mode: "fixture" | "live" | "replay";
+        scoringRevision: 2;
         report: {
             contestedAccuracy: number | null;
             adversarialAccuracy: number | null;

package/dist/evaluation/archive.js

@@ -7,8 +7,13 @@ import { datasetFingerprints, fingerprint, loadEvaluationRecipe, recipeFingerpri
 import { confidenceThresholds, evaluateCases } from './engine.js';
 import { buildReport } from './report.js';
 import { evaluationCaseSchema, evaluationOptionsSchema, evaluationPolicySchema, priceSchema, } from './schema.js';
-const archiveSchema = z.object({
-    format: z.literal(1),
+import { scoringRevision } from './comparison.js';
+const archiveSchema = z
+    .object({
+    format: z.literal(2),
+    scoringRevision: z.literal(scoringRevision),
+    sourceMode: z.enum(['live', 'fixture']),
+    evaluatedAt: z.iso.datetime(),
     runId: z.string(),
     createdAt: z.iso.datetime(),
     mode: z.enum(['live', 'fixture', 'replay']),
@@ -53,6 +58,21 @@ const archiveSchema = z.object({
     thresholds: z.array(z.number()),
     sourceRun: z.string().optional(),
     report: z.record(z.string(), z.unknown()),
+})
+    .superRefine((run, context) => {
+    if (run.mode !== 'replay' && (run.sourceMode !== run.mode || run.evaluatedAt !== run.createdAt))
+        context.addIssue({
+            code: 'custom',
+            message: 'Direct runs must retain their own response mode and date.',
+        });
+    for (const [rowIndex, row] of run.rows.entries())
+        for (const [exchangeIndex, exchange] of row.exchanges.entries())
+            if (Object.hasOwn(exchange, 'response') === Object.hasOwn(exchange, 'error'))
+                context.addIssue({
+                    code: 'custom',
+                    path: ['rows', rowIndex, 'exchanges', exchangeIndex],
+                    message: 'A completed exchange must contain exactly one response or error.',
+                });
 });
 export async function evaluate(name, values, options = {}) {
     const settings = evaluationOptionsSchema.parse(options);
@@ -73,10 +93,14 @@ export async function evaluate(name, values, options = {}) {
             throw new Error('The frozen policy belongs to a different recipe version or requested model.');
     }
     const manifest = JSON.parse(await readFile(new URL('../../package.json', import.meta.url), 'utf8'));
+    const createdAt = new Date().toISOString();
     const metadata = {
-        format: 1,
+        format: 2,
+        scoringRevision,
+        sourceMode: options.mode ?? 'live',
+        evaluatedAt: createdAt,
         runId: randomUUID(),
-        createdAt: new Date().toISOString(),
+        createdAt,
         mode: options.mode ?? 'live',
         packageVersion: manifest.version,
         recipe: recipe.id,
@@ -130,6 +154,8 @@ export async function replay(run, policy = run.policy, out) {
     const cases = validateCases(recipe, run.cases);
     const metadata = {
         ...run,
+        format: 2,
+        scoringRevision,
         runId: randomUUID(),
         createdAt: new Date().toISOString(),
         mode: 'replay',
@@ -158,8 +184,10 @@ export async function readRun(directory) {
             throw error;
         return gunzipSync(await readFile(join(directory, 'run.json.gz'))).toString('utf8');
     });
-    const parsed = archiveSchema.parse(JSON.parse(content));
-    const run = parsed;
+    const raw = JSON.parse(content);
+    if (raw.format !== 2)
+        throw new Error('Unsupported archive format. This evaluator reads format 2; historical files remain available for direct inspection.');
+    const run = archiveSchema.parse(raw);
     const hashes = datasetFingerprints(run.cases);
     if (Object.entries(hashes).some(([key, hash]) => run[key] !== hash))
         throw new Error('Archive dataset fingerprints do not match its cases.');
@@ -196,7 +224,10 @@ function reportEvidence(run) {
     }
     return {
         runId: run.runId,
-        evaluatedAt: run.createdAt,
+        sourceMode: run.sourceMode,
+        evaluatedAt: run.evaluatedAt,
+        createdAt: run.createdAt,
+        scoringRevision: run.scoringRevision,
         packageVersion: run.packageVersion,
         recipeFingerprint: run.recipeFingerprint,
         datasetFingerprint: run.datasetFingerprint,
@@ -207,6 +238,8 @@ function reportEvidence(run) {
     };
 }
 export function compare(baseline, candidate) {
+    if (baseline.scoringRevision !== candidate.scoringRevision)
+        throw new Error('Comparison requires the same scoring revision. Replay compatible archives before comparing.');
     if (baseline.recipe !== candidate.recipe ||
         baseline.inputFingerprint !== candidate.inputFingerprint ||
         baseline.answerKeyFingerprint !== candidate.answerKeyFingerprint)
@@ -220,6 +253,7 @@ export function compare(baseline, candidate) {
         models: run.report.models,
         policy: run.policy,
         mode: run.mode,
+        scoringRevision: run.scoringRevision,
         report: run.report,
     });
     return {

package/dist/evaluation/comparison.d.ts

@@ -0,0 +1,8 @@
+export declare const scoringRevision: 2;
+type Schema = Record<string, unknown>;
+export declare function resultSchemaFor(recipe: string): Schema;
+export declare function comparableDecision(value: unknown, recipe: string): unknown;
+export declare function matchesExpected(actual: unknown, expected: unknown, recipe: string): boolean;
+export declare function isMetadataPath(schema: Schema, segments: string[]): boolean;
+export declare function schemaAtPath(schema: Schema, segments: string[]): Schema | undefined;
+export {};

package/dist/evaluation/comparison.js

@@ -0,0 +1,83 @@
+import { isDeepStrictEqual } from 'node:util';
+import { describeRecipe } from '../catalog/index.js';
+import { recipeNameSchema } from '../catalog/schema.js';
+import { isRecord } from './decisions.js';
+export const scoringRevision = 2;
+const metadataFields = new Set([
+    'model',
+    'usage',
+    'confidence',
+    'probability',
+    'probabilities',
+    'score',
+    'status',
+]);
+const resultSchemas = new Map();
+export function resultSchemaFor(recipe) {
+    const cached = resultSchemas.get(recipe);
+    if (cached)
+        return cached;
+    const schema = describeRecipe(recipeNameSchema.parse(recipe)).resultSchema;
+    resultSchemas.set(recipe, schema);
+    return schema;
+}
+// Expected keys are result paths. Below an unconstrained payload, every key is data.
+export function comparableDecision(value, recipe) {
+    if (!isRecord(value))
+        return value;
+    const schema = resultSchemaFor(recipe);
+    return Object.fromEntries(Object.entries(value)
+        .filter(([path]) => !isMetadataPath(schema, path.split('.')))
+        .map(([path, field]) => [path, normalize(field, schemaAtPath(schema, path.split('.')))]));
+}
+export function matchesExpected(actual, expected, recipe) {
+    return isDeepStrictEqual(comparableDecision(actual, recipe), comparableDecision(expected, recipe));
+}
+function normalize(value, schema) {
+    if (!schema)
+        return value;
+    if (Array.isArray(value))
+        return value.map((item, index) => normalize(item, schemaAtPath(schema, [String(index)])));
+    if (!isRecord(value))
+        return value;
+    return Object.fromEntries(Object.entries(value)
+        .filter(([name]) => !isMetadataPath(schema, [name]))
+        .map(([name, field]) => [name, normalize(field, schemaAtPath(schema, [name]))]));
+}
+export function isMetadataPath(schema, segments) {
+    if (!segments.length)
+        return false;
+    if (Array.isArray(schema.anyOf))
+        return schema.anyOf.some((branch) => isRecord(branch) && isMetadataPath(branch, segments));
+    const [first, ...rest] = segments;
+    if (isRecord(schema.properties) &&
+        Object.hasOwn(schema.properties, first) &&
+        metadataFields.has(first))
+        return true;
+    const child = schemaAtPath(schema, [first]);
+    return child !== undefined && isMetadataPath(child, rest);
+}
+export function schemaAtPath(schema, segments) {
+    if (!segments.length)
+        return schema;
+    if (Array.isArray(schema.anyOf)) {
+        const candidates = schema.anyOf.flatMap((branch) => {
+            const field = isRecord(branch) ? schemaAtPath(branch, segments) : undefined;
+            return field ? [field] : [];
+        });
+        return candidates.length ? { anyOf: candidates } : undefined;
+    }
+    const [first, ...rest] = segments;
+    if (isRecord(schema.properties) &&
+        Object.hasOwn(schema.properties, first) &&
+        isRecord(schema.properties[first]))
+        return schemaAtPath(schema.properties[first], rest);
+    if (schema.type === 'array' && /^(0|[1-9]\d*)$/.test(first) && isRecord(schema.items))
+        return schemaAtPath(schema.items, rest);
+    if (isRecord(schema.additionalProperties))
+        return schemaAtPath(schema.additionalProperties, rest);
+    if (schema.additionalProperties === true ||
+        (schema.type === undefined && schema.enum === undefined && schema.const === undefined))
+        return {};
+    return undefined;
+}

package/dist/evaluation/dataset.js

@@ -1,9 +1,9 @@
 import { createHash } from 'node:crypto';
 import { readFile } from 'node:fs/promises';
 import { z } from 'zod';
-import { describeRecipe } from '../catalog/index.js';
 import { recipeNameSchema } from '../catalog/schema.js';
-import { isRecord, comparableDecision } from './decisions.js';
+import { isRecord } from './decisions.js';
+import { comparableDecision, isMetadataPath, schemaAtPath, resultSchemaFor } from './comparison.js';
 import { evaluationCaseSchema } from './schema.js';
 export async function loadEvaluationRecipe(name) {
     const id = recipeNameSchema.parse(name);
@@ -30,7 +30,7 @@ export async function readEvaluationCases(path) {
 export function validateCases(recipe, values) {
     if (!values.length)
         throw new Error('The case file is empty.');
-    const resultSchema = describeRecipe(recipeNameSchema.parse(recipe.id)).resultSchema;
+    const resultSchema = resultSchemaFor(recipe.id);
     const cases = values.map((value) => evaluationCaseSchema.parse(value));
     const ids = new Set();
     const familySplits = new Map();
@@ -45,17 +45,7 @@ export function validateCases(recipe, values) {
         }
         recipe.inputSchema.parse(entry.input);
         for (const [path, expected] of Object.entries(entry.expected)) {
-            if (path
-                .split('.')
-                .some((segment) => [
-                'status',
-                'confidence',
-                'probabilities',
-                'probability',
-                'model',
-                'usage',
-                'score',
-            ].includes(segment)))
+            if (isMetadataPath(resultSchema, path.split('.')))
                 throw new Error(`${entry.id}: expected must name an ungated decision, not ${path}.`);
             const field = schemaAtPath(resultSchema, path.split('.'));
             if (!field)
@@ -63,30 +53,11 @@ export function validateCases(recipe, values) {
             if (!acceptsValue(field, expected))
                 throw new Error(`${entry.id}: expected ${path} does not match the result schema.`);
         }
-        if (isEmptyDecision(comparableDecision(entry.expected)))
+        if (isEmptyDecision(comparableDecision(entry.expected, recipe.id)))
             throw new Error(`${entry.id}: expected must contain a decision to compare.`);
     }
     return cases;
 }
-function schemaAtPath(schema, segments) {
-    if (!segments.length)
-        return schema;
-    if (Array.isArray(schema.anyOf)) {
-        const candidates = schema.anyOf.flatMap((branch) => {
-            const field = isRecord(branch) ? schemaAtPath(branch, segments) : undefined;
-            return field ? [field] : [];
-        });
-        return candidates.length ? { anyOf: candidates } : undefined;
-    }
-    const [first, ...rest] = segments;
-    if (isRecord(schema.properties) && isRecord(schema.properties[first]))
-        return schemaAtPath(schema.properties[first], rest);
-    if (schema.type === 'array' && /^(0|[1-9]\d*)$/.test(first) && isRecord(schema.items))
-        return schemaAtPath(schema.items, rest);
-    if (isRecord(schema.additionalProperties))
-        return schemaAtPath(schema.additionalProperties, rest);
-    return undefined;
-}
 function acceptsValue(schema, value) {
     if (Array.isArray(schema.anyOf))
         return schema.anyOf.some((candidate) => isRecord(candidate) && acceptsValue(candidate, value));

package/dist/evaluation/decisions.d.ts

@@ -1,5 +1,3 @@
-export declare function comparableDecision(value: unknown): unknown;
-export declare function matchesExpected(actual: unknown, expected: unknown): boolean;
 export declare function isRecord(value: unknown): value is Record<string, unknown>;
 export declare function valueAtPath(record: unknown, path: string): unknown;
 export declare function pickPaths(record: unknown, paths: string[]): Record<string, unknown>;

package/dist/evaluation/decisions.js

@@ -1,25 +1,3 @@
-import { isDeepStrictEqual } from 'node:util';
-const metadataFields = new Set([
-    'model',
-    'usage',
-    'confidence',
-    'probability',
-    'probabilities',
-    'score',
-    'status',
-]);
-export function comparableDecision(value) {
-    if (Array.isArray(value))
-        return value.map(comparableDecision);
-    if (isRecord(value))
-        return Object.fromEntries(Object.entries(value)
-            .filter(([name]) => !metadataFields.has(name))
-            .map(([name, field]) => [name, comparableDecision(field)]));
-    return value;
-}
-export function matchesExpected(actual, expected) {
-    return isDeepStrictEqual(comparableDecision(actual), comparableDecision(expected));
-}
 export function isRecord(value) {
     return value !== null && typeof value === 'object' && !Array.isArray(value);
 }

package/dist/evaluation/engine.js

@@ -1,6 +1,7 @@
 import { isDeepStrictEqual } from 'node:util';
 import { createClient } from '../src/client.js';
-import { caseConfidence, isReady, mapWithConcurrencyLimit, matchesExpected, pickPaths, } from './decisions.js';
+import { caseConfidence, isReady, mapWithConcurrencyLimit, pickPaths } from './decisions.js';
+import { matchesExpected } from './comparison.js';
 export const confidenceThresholds = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95];
 export async function evaluateCases(recipe, cases, options = {}) {
     let requests = 0;
@@ -50,7 +51,7 @@ async function evaluateCase(recipe, goldenCase, client, options) {
     try {
         row.result = await recipe.run(goldenCase.input, recipeOptions);
         row.actual = pickPaths(row.result, Object.keys(goldenCase.expected));
-        row.correct = matchesExpected(row.actual, row.expected);
+        row.correct = matchesExpected(row.actual, row.expected, recipe.id);
         row.ready = isReady(row.result);
         row.confidence = caseConfidence(row.result, Object.keys(goldenCase.expected));
         const model = pickPaths(row.result, ['model']).model;
@@ -72,7 +73,7 @@ async function evaluateCase(recipe, goldenCase, client, options) {
         row.decisions.push({
             minConfidence,
             ready: isReady(result),
-            correct: matchesExpected(pickPaths(result, Object.keys(row.expected)), row.expected),
+            correct: matchesExpected(pickPaths(result, Object.keys(row.expected)), row.expected, recipe.id),
         });
     }
     return row;

package/dist/evaluation/report.d.ts

@@ -4,6 +4,9 @@ import type { EvaluationCase, EvaluationPolicy } from './schema.js';
 export interface ReportEvidence {
     runId: string;
     evaluatedAt: string;
+    createdAt: string;
+    sourceMode: 'live' | 'fixture';
+    scoringRevision: number;
     packageVersion: string;
     recipeFingerprint: string;
     datasetFingerprint: string;

package/dist/evaluation/report.js

@@ -1,4 +1,5 @@
-import { isRecord, matchesExpected, roundedTo } from './decisions.js';
+import { isRecord, roundedTo } from './decisions.js';
+import { matchesExpected } from './comparison.js';
 export function buildReport(id, rows, confidenceThresholds, price, evidence) {
     const completed = rows.filter((row) => row.error === undefined);
     const ready = completed.filter((row) => row.ready);
@@ -41,7 +42,7 @@ export function buildReport(id, rows, confidenceThresholds, price, evidence) {
         reviewRate: completed.length
             ? roundedTo(3, (completed.length - ready.length) / completed.length)
             : null,
-        ...itemAccuracyOf(rows),
+        ...itemAccuracyOf(id, rows),
         contestedAccuracy: accuracyOf(rows.filter((row) => row.contested)),
         adversarialAccuracy: accuracyOf(rows.filter((row) => row.adversarial)),
         calibration: [0, 0.5, 0.6, 0.7, 0.8, 0.9].map((lower, index, edges) => {
@@ -90,14 +91,15 @@ export function buildReport(id, rows, confidenceThresholds, price, evidence) {
 function accuracyOf(rows) {
     return rows.length ? roundedTo(3, rows.filter((row) => row.correct).length / rows.length) : null;
 }
-function itemAccuracyOf(rows) {
+function itemAccuracyOf(recipe, rows) {
     const paths = rows.flatMap((row) => Object.entries(row.expected).map(([path, expected]) => ({
+        path,
         expected,
         actual: row.error === undefined ? row.actual?.[path] : undefined,
     })));
     if (paths.length <= rows.length)
         return {};
-    const correct = paths.filter((entry) => matchesExpected(entry.actual, entry.expected)).length;
+    const correct = paths.filter(({ path, actual, expected }) => matchesExpected({ [path]: actual }, { [path]: expected }, recipe)).length;
     return {
         items: paths.length,
         correctItems: correct,

package/dist/src/batch.js

@@ -11,7 +11,20 @@ export async function evaluateInBatches(items, batchSize, evaluate) {
     const chunks = [];
     for (let offset = 0; offset < items.length; offset += batchSize)
         chunks.push({ chunk: items.slice(offset, offset + batchSize), offset });
-    const evaluations = await Promise.all(chunks.map(({ chunk, offset }) => evaluate(chunk, offset)));
+    let firstFailure;
+    const settled = await Promise.allSettled(chunks.map(async ({ chunk, offset }) => {
+        try {
+            return await evaluate(chunk, offset);
+        }
+        catch (error) {
+            firstFailure ??= { error };
+            throw error;
+        }
+    }));
+    // Let request recorders finish before a caller archives this failed batch.
+    if (firstFailure)
+        throw firstFailure.error;
+    const evaluations = settled.flatMap((result) => result.status === 'fulfilled' ? [result.value] : []);
     return {
         items: evaluations.flatMap((evaluation) => evaluation.items),
         model: evaluations[0]?.model ?? 'none',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jev-recipes",
-  "version": "0.8.3",
+  "version": "0.8.5",
   "description": "Small, composable Jev decisions for retrieval, conversations, and agent workflows.",
   "type": "module",
   "license": "MIT",
@@ -1054,6 +1054,7 @@
     "eval": "node --env-file-if-exists=.env evals/run.mjs",
     "eval:smoke": "node --env-file-if-exists=.env evals/smoke.mjs",
     "eval:validate": "node evals/validate.mjs",
+    "eval:audit": "node scripts/audit-evidence.mjs",
     "jev-recipes": "node --env-file-if-exists=.env dist/cli/index.js",
     "ci": "node scripts/ci.mjs",
     "pack:check": "node scripts/pack-check.mjs",