jev-recipes 0.9.6 → 0.9.9

package/dist/adapters/ai-sdk/index.js

@@ -90,8 +90,9 @@ export function routeModelStep(options) {
     if (!candidates.length)
         throw new Error('Supply at least one model candidate.');
     const decisions = new WeakMap();
-    return async ({ initialMessages }) => {
-        if (!decisions.has(initialMessages)) {
+    return async ({ initialMessages, steps }) => {
+        // The SDK owns one steps array per generation, even when callers reuse messages.
+        if (!decisions.has(steps)) {
             const decision = await modelRoute({
                 request: request ?? requireText(firstUserText(initialMessages), 'request'),
                 models: candidates.map(({ id, text }) => ({ id, text })),
@@ -102,9 +103,9 @@ export function routeModelStep(options) {
                 ? candidates.find((candidate) => candidate.id === decision.selection)?.model
                 : fallback;
             onDecision?.({ decision, model: selected });
-            decisions.set(initialMessages, selected);
+            decisions.set(steps, selected);
         }
-        const model = decisions.get(initialMessages);
+        const model = decisions.get(steps);
         return model === undefined ? {} : { model };
     };
 }

package/dist/evaluation/archive.d.ts

@@ -142,9 +142,9 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             }[];
             thresholdEvaluation: string;
             thresholds: {
+                readyAccuracy: number | null;
                 minConfidence: number;
                 deferRate: number | null;
-                readyAccuracy: number | null;
                 readyCases: number;
                 reviewCases: number;
                 failedCases: number;
@@ -225,9 +225,9 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             }[];
             thresholdEvaluation: string;
             thresholds: {
+                readyAccuracy: number | null;
                 minConfidence: number;
                 deferRate: number | null;
-                readyAccuracy: number | null;
                 readyCases: number;
                 reviewCases: number;
                 failedCases: number;
@@ -328,9 +328,9 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             }[];
             thresholdEvaluation: string;
             thresholds: {
+                readyAccuracy: number | null;
                 minConfidence: number;
                 deferRate: number | null;
-                readyAccuracy: number | null;
                 readyCases: number;
                 reviewCases: number;
                 failedCases: number;
@@ -411,9 +411,9 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             }[];
             thresholdEvaluation: string;
             thresholds: {
+                readyAccuracy: number | null;
                 minConfidence: number;
                 deferRate: number | null;
-                readyAccuracy: number | null;
                 readyCases: number;
                 reviewCases: number;
                 failedCases: number;

package/dist/evaluation/comparison.d.ts

@@ -3,6 +3,7 @@ type Schema = Record<string, unknown>;
 export declare function resultSchemaFor(recipe: string): Schema;
 export declare function comparableDecision(value: unknown, recipe: string): unknown;
 export declare function matchesExpected(actual: unknown, expected: unknown, recipe: string): boolean;
+export declare function isReady(result: unknown, recipe: string): boolean;
 export declare function isMetadataPath(schema: Schema, segments: string[]): boolean;
 export declare function schemaAtPath(schema: Schema, segments: string[]): Schema | undefined;
 export {};

package/dist/evaluation/comparison.js

@@ -33,6 +33,25 @@ export function comparableDecision(value, recipe) {
 export function matchesExpected(actual, expected, recipe) {
     return isDeepStrictEqual(comparableDecision(actual, recipe), comparableDecision(expected, recipe));
 }
+export function isReady(result, recipe) {
+    if (!isRecord(result))
+        return false;
+    const schema = resultSchemaFor(recipe);
+    if (isMetadataPath(schema, ['status']) && result.status !== undefined)
+        return result.status === 'ready';
+    return !hasReview(result, schema);
+}
+function hasReview(value, schema) {
+    if (!schema)
+        return false;
+    if (Array.isArray(value))
+        return value.some((item, index) => hasReview(item, schemaAtPath(schema, [String(index)])));
+    if (!isRecord(value))
+        return false;
+    return Object.entries(value).some(([name, field]) => name === 'status' && isMetadataPath(schema, [name])
+        ? field === 'review'
+        : hasReview(field, schemaAtPath(schema, [name])));
+}
 function normalize(value, schema) {
     if (!schema)
         return value;

package/dist/evaluation/decisions.d.ts

@@ -1,8 +1,6 @@
 export declare function isRecord(value: unknown): value is Record<string, unknown>;
 export declare function valueAtPath(record: unknown, path: string): unknown;
 export declare function pickPaths(record: unknown, paths: string[]): Record<string, unknown>;
-export declare function hasReviewAnywhere(value: unknown): boolean;
-export declare function isReady(result: unknown): boolean;
 export declare function caseConfidence(result: unknown, expectedPaths?: string[]): number | undefined;
 export declare function mapWithConcurrencyLimit<T, R>(items: T[], limit: number, worker: (item: T, index: number) => Promise<R>): Promise<R[]>;
 export declare function roundedTo(places: number, value: number): number;

package/dist/evaluation/decisions.js

@@ -11,20 +11,6 @@ export function valueAtPath(record, path) {
 export function pickPaths(record, paths) {
     return Object.fromEntries(paths.map((path) => [path, valueAtPath(record, path)]));
 }
-export function hasReviewAnywhere(value) {
-    if (Array.isArray(value))
-        return value.some(hasReviewAnywhere);
-    if (isRecord(value))
-        return Object.entries(value).some(([name, field]) => name === 'status' ? field === 'review' : hasReviewAnywhere(field));
-    return false;
-}
-export function isReady(result) {
-    if (!isRecord(result))
-        return false;
-    if (result.status !== undefined)
-        return result.status === 'ready';
-    return !hasReviewAnywhere(result);
-}
 export function caseConfidence(result, expectedPaths = []) {
     if (isRecord(result) && typeof result.confidence === 'number')
         return result.confidence;

package/dist/evaluation/engine.js

@@ -1,7 +1,7 @@
 import { isDeepStrictEqual } from 'node:util';
 import { createClient } from '../src/client.js';
-import { caseConfidence, isReady, mapWithConcurrencyLimit, pickPaths } from './decisions.js';
-import { matchesExpected } from './comparison.js';
+import { caseConfidence, mapWithConcurrencyLimit, pickPaths } from './decisions.js';
+import { isReady, matchesExpected } from './comparison.js';
 export const confidenceThresholds = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95];
 export async function evaluateCases(recipe, cases, options = {}) {
     let requests = 0;
@@ -52,7 +52,7 @@ async function evaluateCase(recipe, goldenCase, client, options) {
         row.result = await recipe.run(goldenCase.input, recipeOptions);
         row.actual = pickPaths(row.result, Object.keys(goldenCase.expected));
         row.correct = matchesExpected(row.actual, row.expected, recipe.id);
-        row.ready = isReady(row.result);
+        row.ready = isReady(row.result, recipe.id);
         row.confidence = caseConfidence(row.result, Object.keys(goldenCase.expected));
         const model = pickPaths(row.result, ['model']).model;
         if (typeof model === 'string')
@@ -72,7 +72,7 @@ async function evaluateCase(recipe, goldenCase, client, options) {
         playback.assertComplete();
         row.decisions.push({
             minConfidence,
-            ready: isReady(result),
+            ready: isReady(result, recipe.id),
             correct: matchesExpected(pickPaths(result, Object.keys(row.expected)), row.expected, recipe.id),
         });
     }

package/dist/evaluation/report.d.ts

@@ -29,9 +29,9 @@ export declare function buildReport(id: string, rows: EvaluationRow[], confidenc
     }[];
     thresholdEvaluation: string;
     thresholds: {
+        readyAccuracy: number | null;
         minConfidence: number;
         deferRate: number | null;
-        readyAccuracy: number | null;
         readyCases: number;
         reviewCases: number;
         failedCases: number;
@@ -112,9 +112,9 @@ export declare function buildReport(id: string, rows: EvaluationRow[], confidenc
     }[];
     thresholdEvaluation: string;
     thresholds: {
+        readyAccuracy: number | null;
         minConfidence: number;
         deferRate: number | null;
-        readyAccuracy: number | null;
         readyCases: number;
         reviewCases: number;
         failedCases: number;

package/dist/evaluation/report.js

@@ -1,26 +1,34 @@
 import { isRecord, roundedTo } from './decisions.js';
 import { matchesExpected } from './comparison.js';
+import { resultMetadataSchema } from '../src/schema.js';
 export function buildReport(id, rows, confidenceThresholds, price, evidence) {
     const completed = rows.filter((row) => row.error === undefined);
     const ready = completed.filter((row) => row.ready);
-    const thresholds = confidenceThresholds.map((minConfidence) => {
+    const thresholdEvaluations = confidenceThresholds.map((minConfidence) => {
         const decisions = completed
             .flatMap((row) => row.decisions)
             .filter((decision) => decision.minConfidence === minConfidence);
         const accepted = decisions.filter((decision) => decision.ready);
+        const correctReadyCases = accepted.filter((decision) => decision.correct).length;
         return {
             minConfidence,
             deferRate: completed.length
                 ? roundedTo(3, (completed.length - accepted.length) / completed.length)
                 : null,
-            readyAccuracy: accuracyOf(accepted),
+            readyAccuracy: accepted.length ? correctReadyCases / accepted.length : null,
             readyCases: accepted.length,
             reviewCases: completed.length - accepted.length,
             failedCases: rows.length - completed.length,
-            readyAccuracyInterval95: wilsonInterval(accepted.filter((entry) => entry.correct).length, accepted.length),
+            readyAccuracyInterval95: wilsonInterval(correctReadyCases, accepted.length),
         };
     });
-    const usage = usageOf(rows);
+    const suggestedMinConfidence = thresholdEvaluations.find((entry) => entry.readyAccuracy !== null && entry.readyAccuracy >= 0.95)?.minConfidence ?? null;
+    const thresholds = thresholdEvaluations.map((threshold) => ({
+        ...threshold,
+        readyAccuracy: threshold.readyAccuracy === null ? null : roundedTo(3, threshold.readyAccuracy),
+    }));
+    const usage = summarizeUsage(rows);
+    const cost = estimateKnownCost(usage, price);
     const models = [...new Set(rows.flatMap((row) => (row.model ? [row.model] : [])))].sort();
     return {
         ...(evidence ? { evidence } : {}),
@@ -60,21 +68,10 @@ export function buildReport(id, rows, confidenceThresholds, price, evidence) {
                 ? 'recipe-replay'
                 : 'not-applicable',
         thresholds,
-        suggestedMinConfidence: thresholds.find((entry) => entry.readyAccuracy !== null && entry.readyAccuracy >= 0.95)
-            ?.minConfidence ?? null,
+        suggestedMinConfidence,
         latencyMs: latencyOf(rows.map((row) => row.durationMs)),
         usage,
-        cost: price &&
-            usage.models.every((model) => model === price.model) &&
-            usage.unknownResponses === 0 &&
-            usage.failedRequests === 0
-            ? {
-                ...price,
-                estimated: roundedTo(8, (usage.input_tokens * price.inputPerMillion +
-                    usage.output_tokens * price.outputPerMillion) /
-                    1_000_000),
-            }
-            : null,
+        cost,
         confusion: confusionOf(rows),
         failures: rows
             .filter((row) => !row.correct)
@@ -126,7 +123,7 @@ function latencyOf(values) {
         totalCaseTime: roundedTo(2, values.reduce((sum, value) => sum + value, 0)),
     };
 }
-function usageOf(rows) {
+function summarizeUsage(rows) {
     let input_tokens = 0, output_tokens = 0, unknownResponses = 0, failedRequests = 0;
     const models = new Set();
     const exchanges = rows.flatMap((row) => row.exchanges);
@@ -135,18 +132,17 @@ function usageOf(rows) {
             failedRequests++;
             continue;
         }
-        const response = exchange.response;
-        if (isRecord(response) && typeof response.model === 'string')
-            models.add(response.model);
-        if (!isRecord(response) ||
-            !isRecord(response.usage) ||
-            typeof response.usage.input_tokens !== 'number' ||
-            typeof response.usage.output_tokens !== 'number') {
+        const response = isRecord(exchange.response) ? exchange.response : {};
+        const usage = resultMetadataSchema.shape.usage.safeParse(response.usage);
+        const model = resultMetadataSchema.shape.model.safeParse(response.model);
+        if (!usage.success || !model.success)
             unknownResponses++;
+        if (model.success)
+            models.add(model.data);
+        if (!usage.success)
             continue;
-        }
-        input_tokens += response.usage.input_tokens;
-        output_tokens += response.usage.output_tokens;
+        input_tokens += usage.data.input_tokens;
+        output_tokens += usage.data.output_tokens;
     }
     return {
         requests: exchanges.length,
@@ -157,6 +153,18 @@ function usageOf(rows) {
         models: [...models].sort(),
     };
 }
+function estimateKnownCost(usage, price) {
+    if (!price ||
+        usage.unknownResponses > 0 ||
+        usage.failedRequests > 0 ||
+        usage.models.some((model) => model !== price.model))
+        return null;
+    return {
+        ...price,
+        estimated: roundedTo(8, (usage.input_tokens * price.inputPerMillion + usage.output_tokens * price.outputPerMillion) /
+            1_000_000),
+    };
+}
 function confusionOf(rows) {
     const byExpected = new Map();
     for (const row of rows) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jev-recipes",
-  "version": "0.9.6",
+  "version": "0.9.9",
   "description": "248 small Jev decisions for JavaScript and TypeScript: route, rerank, gate, grade, compare, and label text for agents, RAG, support, code review, and music. Each returns a typed result with explicit uncertainty. Plug into Vercel AI SDK or LangChain agents, import one recipe, or run the CLI with JSON from any language.",
   "type": "module",
   "license": "MIT",