jev-recipes 0.8.4 → 0.9.0

package/dist/evaluation/archive.d.ts

@@ -4,7 +4,13 @@ import type { EvaluationRow } from './engine.js';
 import { buildReport } from './report.js';
 import type { EvaluationOptions, EvaluationPolicy } from './schema.js';
 declare const archiveSchema: z.ZodObject<{
-    format: z.ZodLiteral<1>;
+    format: z.ZodLiteral<2>;
+    scoringRevision: z.ZodLiteral<2>;
+    sourceMode: z.ZodEnum<{
+        fixture: "fixture";
+        live: "live";
+    }>;
+    evaluatedAt: z.ZodISODateTime;
     runId: z.ZodString;
     createdAt: z.ZodISODateTime;
     mode: z.ZodEnum<{
@@ -125,6 +131,7 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             } | undefined;
         };
         mode: "fixture" | "live" | "replay";
+        scoringRevision: 2;
         report: {
             contestedAccuracy: number | null;
             adversarialAccuracy: number | null;
@@ -310,6 +317,7 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             } | undefined;
         };
         mode: "fixture" | "live" | "replay";
+        scoringRevision: 2;
         report: {
             contestedAccuracy: number | null;
             adversarialAccuracy: number | null;

package/dist/evaluation/archive.js

@@ -7,8 +7,13 @@ import { datasetFingerprints, fingerprint, loadEvaluationRecipe, recipeFingerpri
 import { confidenceThresholds, evaluateCases } from './engine.js';
 import { buildReport } from './report.js';
 import { evaluationCaseSchema, evaluationOptionsSchema, evaluationPolicySchema, priceSchema, } from './schema.js';
-const archiveSchema = z.object({
-    format: z.literal(1),
+import { scoringRevision } from './comparison.js';
+const archiveSchema = z
+    .object({
+    format: z.literal(2),
+    scoringRevision: z.literal(scoringRevision),
+    sourceMode: z.enum(['live', 'fixture']),
+    evaluatedAt: z.iso.datetime(),
     runId: z.string(),
     createdAt: z.iso.datetime(),
     mode: z.enum(['live', 'fixture', 'replay']),
@@ -53,6 +58,21 @@ const archiveSchema = z.object({
     thresholds: z.array(z.number()),
     sourceRun: z.string().optional(),
     report: z.record(z.string(), z.unknown()),
+})
+    .superRefine((run, context) => {
+    if (run.mode !== 'replay' && (run.sourceMode !== run.mode || run.evaluatedAt !== run.createdAt))
+        context.addIssue({
+            code: 'custom',
+            message: 'Direct runs must retain their own response mode and date.',
+        });
+    for (const [rowIndex, row] of run.rows.entries())
+        for (const [exchangeIndex, exchange] of row.exchanges.entries())
+            if (Object.hasOwn(exchange, 'response') === Object.hasOwn(exchange, 'error'))
+                context.addIssue({
+                    code: 'custom',
+                    path: ['rows', rowIndex, 'exchanges', exchangeIndex],
+                    message: 'A completed exchange must contain exactly one response or error.',
+                });
 });
 export async function evaluate(name, values, options = {}) {
     const settings = evaluationOptionsSchema.parse(options);
@@ -73,10 +93,14 @@ export async function evaluate(name, values, options = {}) {
             throw new Error('The frozen policy belongs to a different recipe version or requested model.');
     }
     const manifest = JSON.parse(await readFile(new URL('../../package.json', import.meta.url), 'utf8'));
+    const createdAt = new Date().toISOString();
     const metadata = {
-        format: 1,
+        format: 2,
+        scoringRevision,
+        sourceMode: options.mode ?? 'live',
+        evaluatedAt: createdAt,
         runId: randomUUID(),
-        createdAt: new Date().toISOString(),
+        createdAt,
         mode: options.mode ?? 'live',
         packageVersion: manifest.version,
         recipe: recipe.id,
@@ -130,6 +154,8 @@ export async function replay(run, policy = run.policy, out) {
     const cases = validateCases(recipe, run.cases);
     const metadata = {
         ...run,
+        format: 2,
+        scoringRevision,
         runId: randomUUID(),
         createdAt: new Date().toISOString(),
         mode: 'replay',
@@ -158,8 +184,10 @@ export async function readRun(directory) {
             throw error;
         return gunzipSync(await readFile(join(directory, 'run.json.gz'))).toString('utf8');
     });
-    const parsed = archiveSchema.parse(JSON.parse(content));
-    const run = parsed;
+    const raw = JSON.parse(content);
+    if (raw.format !== 2)
+        throw new Error('Unsupported archive format. This evaluator reads format 2; historical files remain available for direct inspection.');
+    const run = archiveSchema.parse(raw);
     const hashes = datasetFingerprints(run.cases);
     if (Object.entries(hashes).some(([key, hash]) => run[key] !== hash))
         throw new Error('Archive dataset fingerprints do not match its cases.');
@@ -196,7 +224,10 @@ function reportEvidence(run) {
     }
     return {
         runId: run.runId,
-        evaluatedAt: run.createdAt,
+        sourceMode: run.sourceMode,
+        evaluatedAt: run.evaluatedAt,
+        createdAt: run.createdAt,
+        scoringRevision: run.scoringRevision,
         packageVersion: run.packageVersion,
         recipeFingerprint: run.recipeFingerprint,
         datasetFingerprint: run.datasetFingerprint,
@@ -207,6 +238,8 @@ function reportEvidence(run) {
     };
 }
 export function compare(baseline, candidate) {
+    if (baseline.scoringRevision !== candidate.scoringRevision)
+        throw new Error('Comparison requires the same scoring revision. Replay compatible archives before comparing.');
     if (baseline.recipe !== candidate.recipe ||
         baseline.inputFingerprint !== candidate.inputFingerprint ||
         baseline.answerKeyFingerprint !== candidate.answerKeyFingerprint)
@@ -220,6 +253,7 @@ export function compare(baseline, candidate) {
         models: run.report.models,
         policy: run.policy,
         mode: run.mode,
+        scoringRevision: run.scoringRevision,
         report: run.report,
     });
     return {

package/dist/evaluation/comparison.d.ts

@@ -0,0 +1,8 @@
+export declare const scoringRevision: 2;
+type Schema = Record<string, unknown>;
+export declare function resultSchemaFor(recipe: string): Schema;
+export declare function comparableDecision(value: unknown, recipe: string): unknown;
+export declare function matchesExpected(actual: unknown, expected: unknown, recipe: string): boolean;
+export declare function isMetadataPath(schema: Schema, segments: string[]): boolean;
+export declare function schemaAtPath(schema: Schema, segments: string[]): Schema | undefined;
+export {};

package/dist/evaluation/comparison.js

@@ -0,0 +1,83 @@
+import { isDeepStrictEqual } from 'node:util';
+import { describeRecipe } from '../catalog/index.js';
+import { recipeNameSchema } from '../catalog/schema.js';
+import { isRecord } from './decisions.js';
+export const scoringRevision = 2;
+const metadataFields = new Set([
+    'model',
+    'usage',
+    'confidence',
+    'probability',
+    'probabilities',
+    'score',
+    'status',
+]);
+const resultSchemas = new Map();
+export function resultSchemaFor(recipe) {
+    const cached = resultSchemas.get(recipe);
+    if (cached)
+        return cached;
+    const schema = describeRecipe(recipeNameSchema.parse(recipe)).resultSchema;
+    resultSchemas.set(recipe, schema);
+    return schema;
+}
+// Expected keys are result paths. Below an unconstrained payload, every key is data.
+export function comparableDecision(value, recipe) {
+    if (!isRecord(value))
+        return value;
+    const schema = resultSchemaFor(recipe);
+    return Object.fromEntries(Object.entries(value)
+        .filter(([path]) => !isMetadataPath(schema, path.split('.')))
+        .map(([path, field]) => [path, normalize(field, schemaAtPath(schema, path.split('.')))]));
+}
+export function matchesExpected(actual, expected, recipe) {
+    return isDeepStrictEqual(comparableDecision(actual, recipe), comparableDecision(expected, recipe));
+}
+function normalize(value, schema) {
+    if (!schema)
+        return value;
+    if (Array.isArray(value))
+        return value.map((item, index) => normalize(item, schemaAtPath(schema, [String(index)])));
+    if (!isRecord(value))
+        return value;
+    return Object.fromEntries(Object.entries(value)
+        .filter(([name]) => !isMetadataPath(schema, [name]))
+        .map(([name, field]) => [name, normalize(field, schemaAtPath(schema, [name]))]));
+}
+export function isMetadataPath(schema, segments) {
+    if (!segments.length)
+        return false;
+    if (Array.isArray(schema.anyOf))
+        return schema.anyOf.some((branch) => isRecord(branch) && isMetadataPath(branch, segments));
+    const [first, ...rest] = segments;
+    if (isRecord(schema.properties) &&
+        Object.hasOwn(schema.properties, first) &&
+        metadataFields.has(first))
+        return true;
+    const child = schemaAtPath(schema, [first]);
+    return child !== undefined && isMetadataPath(child, rest);
+}
+export function schemaAtPath(schema, segments) {
+    if (!segments.length)
+        return schema;
+    if (Array.isArray(schema.anyOf)) {
+        const candidates = schema.anyOf.flatMap((branch) => {
+            const field = isRecord(branch) ? schemaAtPath(branch, segments) : undefined;
+            return field ? [field] : [];
+        });
+        return candidates.length ? { anyOf: candidates } : undefined;
+    }
+    const [first, ...rest] = segments;
+    if (isRecord(schema.properties) &&
+        Object.hasOwn(schema.properties, first) &&
+        isRecord(schema.properties[first]))
+        return schemaAtPath(schema.properties[first], rest);
+    if (schema.type === 'array' && /^(0|[1-9]\d*)$/.test(first) && isRecord(schema.items))
+        return schemaAtPath(schema.items, rest);
+    if (isRecord(schema.additionalProperties))
+        return schemaAtPath(schema.additionalProperties, rest);
+    if (schema.additionalProperties === true ||
+        (schema.type === undefined && schema.enum === undefined && schema.const === undefined))
+        return {};
+    return undefined;
+}

package/dist/evaluation/dataset.js

@@ -1,9 +1,9 @@
 import { createHash } from 'node:crypto';
 import { readFile } from 'node:fs/promises';
 import { z } from 'zod';
-import { describeRecipe } from '../catalog/index.js';
 import { recipeNameSchema } from '../catalog/schema.js';
-import { isRecord, comparableDecision } from './decisions.js';
+import { isRecord } from './decisions.js';
+import { comparableDecision, isMetadataPath, schemaAtPath, resultSchemaFor } from './comparison.js';
 import { evaluationCaseSchema } from './schema.js';
 export async function loadEvaluationRecipe(name) {
     const id = recipeNameSchema.parse(name);
@@ -30,7 +30,7 @@ export async function readEvaluationCases(path) {
 export function validateCases(recipe, values) {
     if (!values.length)
         throw new Error('The case file is empty.');
-    const resultSchema = describeRecipe(recipeNameSchema.parse(recipe.id)).resultSchema;
+    const resultSchema = resultSchemaFor(recipe.id);
     const cases = values.map((value) => evaluationCaseSchema.parse(value));
     const ids = new Set();
     const familySplits = new Map();
@@ -45,17 +45,7 @@ export function validateCases(recipe, values) {
         }
         recipe.inputSchema.parse(entry.input);
         for (const [path, expected] of Object.entries(entry.expected)) {
-            if (path
-                .split('.')
-                .some((segment) => [
-                'status',
-                'confidence',
-                'probabilities',
-                'probability',
-                'model',
-                'usage',
-                'score',
-            ].includes(segment)))
+            if (isMetadataPath(resultSchema, path.split('.')))
                 throw new Error(`${entry.id}: expected must name an ungated decision, not ${path}.`);
             const field = schemaAtPath(resultSchema, path.split('.'));
             if (!field)
@@ -63,30 +53,11 @@ export function validateCases(recipe, values) {
             if (!acceptsValue(field, expected))
                 throw new Error(`${entry.id}: expected ${path} does not match the result schema.`);
         }
-        if (isEmptyDecision(comparableDecision(entry.expected)))
+        if (isEmptyDecision(comparableDecision(entry.expected, recipe.id)))
             throw new Error(`${entry.id}: expected must contain a decision to compare.`);
     }
     return cases;
 }
-function schemaAtPath(schema, segments) {
-    if (!segments.length)
-        return schema;
-    if (Array.isArray(schema.anyOf)) {
-        const candidates = schema.anyOf.flatMap((branch) => {
-            const field = isRecord(branch) ? schemaAtPath(branch, segments) : undefined;
-            return field ? [field] : [];
-        });
-        return candidates.length ? { anyOf: candidates } : undefined;
-    }
-    const [first, ...rest] = segments;
-    if (isRecord(schema.properties) && isRecord(schema.properties[first]))
-        return schemaAtPath(schema.properties[first], rest);
-    if (schema.type === 'array' && /^(0|[1-9]\d*)$/.test(first) && isRecord(schema.items))
-        return schemaAtPath(schema.items, rest);
-    if (isRecord(schema.additionalProperties))
-        return schemaAtPath(schema.additionalProperties, rest);
-    return undefined;
-}
 function acceptsValue(schema, value) {
     if (Array.isArray(schema.anyOf))
         return schema.anyOf.some((candidate) => isRecord(candidate) && acceptsValue(candidate, value));

package/dist/evaluation/decisions.d.ts

@@ -1,5 +1,3 @@
-export declare function comparableDecision(value: unknown): unknown;
-export declare function matchesExpected(actual: unknown, expected: unknown): boolean;
 export declare function isRecord(value: unknown): value is Record<string, unknown>;
 export declare function valueAtPath(record: unknown, path: string): unknown;
 export declare function pickPaths(record: unknown, paths: string[]): Record<string, unknown>;

package/dist/evaluation/decisions.js

@@ -1,25 +1,3 @@
-import { isDeepStrictEqual } from 'node:util';
-const metadataFields = new Set([
-    'model',
-    'usage',
-    'confidence',
-    'probability',
-    'probabilities',
-    'score',
-    'status',
-]);
-export function comparableDecision(value) {
-    if (Array.isArray(value))
-        return value.map(comparableDecision);
-    if (isRecord(value))
-        return Object.fromEntries(Object.entries(value)
-            .filter(([name]) => !metadataFields.has(name))
-            .map(([name, field]) => [name, comparableDecision(field)]));
-    return value;
-}
-export function matchesExpected(actual, expected) {
-    return isDeepStrictEqual(comparableDecision(actual), comparableDecision(expected));
-}
 export function isRecord(value) {
     return value !== null && typeof value === 'object' && !Array.isArray(value);
 }

package/dist/evaluation/engine.js

@@ -1,6 +1,7 @@
 import { isDeepStrictEqual } from 'node:util';
 import { createClient } from '../src/client.js';
-import { caseConfidence, isReady, mapWithConcurrencyLimit, matchesExpected, pickPaths, } from './decisions.js';
+import { caseConfidence, isReady, mapWithConcurrencyLimit, pickPaths } from './decisions.js';
+import { matchesExpected } from './comparison.js';
 export const confidenceThresholds = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95];
 export async function evaluateCases(recipe, cases, options = {}) {
     let requests = 0;
@@ -50,7 +51,7 @@ async function evaluateCase(recipe, goldenCase, client, options) {
     try {
         row.result = await recipe.run(goldenCase.input, recipeOptions);
         row.actual = pickPaths(row.result, Object.keys(goldenCase.expected));
-        row.correct = matchesExpected(row.actual, row.expected);
+        row.correct = matchesExpected(row.actual, row.expected, recipe.id);
         row.ready = isReady(row.result);
         row.confidence = caseConfidence(row.result, Object.keys(goldenCase.expected));
         const model = pickPaths(row.result, ['model']).model;
@@ -72,7 +73,7 @@ async function evaluateCase(recipe, goldenCase, client, options) {
         row.decisions.push({
             minConfidence,
             ready: isReady(result),
-            correct: matchesExpected(pickPaths(result, Object.keys(row.expected)), row.expected),
+            correct: matchesExpected(pickPaths(result, Object.keys(row.expected)), row.expected, recipe.id),
         });
     }
     return row;

package/dist/evaluation/report.d.ts

@@ -4,6 +4,9 @@ import type { EvaluationCase, EvaluationPolicy } from './schema.js';
 export interface ReportEvidence {
     runId: string;
     evaluatedAt: string;
+    createdAt: string;
+    sourceMode: 'live' | 'fixture';
+    scoringRevision: number;
     packageVersion: string;
     recipeFingerprint: string;
     datasetFingerprint: string;

package/dist/evaluation/report.js

@@ -1,4 +1,5 @@
-import { isRecord, matchesExpected, roundedTo } from './decisions.js';
+import { isRecord, roundedTo } from './decisions.js';
+import { matchesExpected } from './comparison.js';
 export function buildReport(id, rows, confidenceThresholds, price, evidence) {
     const completed = rows.filter((row) => row.error === undefined);
     const ready = completed.filter((row) => row.ready);
@@ -41,7 +42,7 @@ export function buildReport(id, rows, confidenceThresholds, price, evidence) {
         reviewRate: completed.length
             ? roundedTo(3, (completed.length - ready.length) / completed.length)
             : null,
-        ...itemAccuracyOf(rows),
+        ...itemAccuracyOf(id, rows),
         contestedAccuracy: accuracyOf(rows.filter((row) => row.contested)),
         adversarialAccuracy: accuracyOf(rows.filter((row) => row.adversarial)),
         calibration: [0, 0.5, 0.6, 0.7, 0.8, 0.9].map((lower, index, edges) => {
@@ -90,14 +91,15 @@ export function buildReport(id, rows, confidenceThresholds, price, evidence) {
 function accuracyOf(rows) {
     return rows.length ? roundedTo(3, rows.filter((row) => row.correct).length / rows.length) : null;
 }
-function itemAccuracyOf(rows) {
+function itemAccuracyOf(recipe, rows) {
     const paths = rows.flatMap((row) => Object.entries(row.expected).map(([path, expected]) => ({
+        path,
         expected,
         actual: row.error === undefined ? row.actual?.[path] : undefined,
     })));
     if (paths.length <= rows.length)
         return {};
-    const correct = paths.filter((entry) => matchesExpected(entry.actual, entry.expected)).length;
+    const correct = paths.filter(({ path, actual, expected }) => matchesExpected({ [path]: actual }, { [path]: expected }, recipe)).length;
     return {
         items: paths.length,
         correctItems: correct,

package/dist/src/batch.js

@@ -11,7 +11,20 @@ export async function evaluateInBatches(items, batchSize, evaluate) {
     const chunks = [];
     for (let offset = 0; offset < items.length; offset += batchSize)
         chunks.push({ chunk: items.slice(offset, offset + batchSize), offset });
-    const evaluations = await Promise.all(chunks.map(({ chunk, offset }) => evaluate(chunk, offset)));
+    let firstFailure;
+    const settled = await Promise.allSettled(chunks.map(async ({ chunk, offset }) => {
+        try {
+            return await evaluate(chunk, offset);
+        }
+        catch (error) {
+            firstFailure ??= { error };
+            throw error;
+        }
+    }));
+    // Let request recorders finish before a caller archives this failed batch.
+    if (firstFailure)
+        throw firstFailure.error;
+    const evaluations = settled.flatMap((result) => result.status === 'fulfilled' ? [result.value] : []);
     return {
         items: evaluations.flatMap((evaluation) => evaluation.items),
         model: evaluations[0]?.model ?? 'none',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jev-recipes",
-  "version": "0.8.4",
+  "version": "0.9.0",
   "description": "Small, composable Jev decisions for retrieval, conversations, and agent workflows.",
   "type": "module",
   "license": "MIT",
@@ -27,6 +27,14 @@
       "types": "./dist/evaluation/index.d.ts",
       "import": "./dist/evaluation/index.js"
     },
+    "./ai-sdk": {
+      "types": "./dist/adapters/ai-sdk/index.d.ts",
+      "import": "./dist/adapters/ai-sdk/index.js"
+    },
+    "./langchain": {
+      "types": "./dist/adapters/langchain/index.d.ts",
+      "import": "./dist/adapters/langchain/index.js"
+    },
     "./package.json": "./package.json",
     "./action-compare": {
       "types": "./dist/recipes/action-compare/index.d.ts",
@@ -1035,6 +1043,8 @@
     "dist/cli/*.js",
     "dist/evaluation/**/*.js",
     "dist/evaluation/**/*.d.ts",
+    "dist/adapters/**/*.js",
+    "dist/adapters/**/*.d.ts",
     "dist/cli/*.d.ts",
     "dist/recipes/*/demo.json",
     "README.md",
@@ -1054,6 +1064,7 @@
     "eval": "node --env-file-if-exists=.env evals/run.mjs",
     "eval:smoke": "node --env-file-if-exists=.env evals/smoke.mjs",
     "eval:validate": "node evals/validate.mjs",
+    "eval:audit": "node scripts/audit-evidence.mjs",
     "jev-recipes": "node --env-file-if-exists=.env dist/cli/index.js",
     "ci": "node scripts/ci.mjs",
     "pack:check": "node scripts/pack-check.mjs",
@@ -1066,15 +1077,29 @@
     "test:coverage": "vitest run --coverage",
     "eval:featured": "node --env-file-if-exists=.env evals/featured.mjs",
     "site:build": "npm run generate:check && npm run build && node site/build.mjs",
-    "site:preview": "node site/preview.mjs"
+    "site:preview": "node --env-file-if-exists=.env site/preview.mjs"
   },
   "dependencies": {
     "@typesafe-ai/sdk": "^0.6.0",
     "zod": "^4.6.5"
   },
+  "peerDependencies": {
+    "ai": ">=7.0.0",
+    "@langchain/core": ">=1.0.0"
+  },
+  "peerDependenciesMeta": {
+    "ai": {
+      "optional": true
+    },
+    "@langchain/core": {
+      "optional": true
+    }
+  },
   "devDependencies": {
+    "@langchain/core": "^1.2.13",
     "@types/node": "^22.0.0",
     "@vitest/coverage-v8": "4.1.11",
+    "ai": "^7.0.122",
     "prettier": "3.9.8",
     "rimraf": "6.1.3",
     "typescript": "~5.9.3",