jev-recipes 0.9.6 → 0.9.7

package/dist/evaluation/archive.d.ts

@@ -142,9 +142,9 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             }[];
             thresholdEvaluation: string;
             thresholds: {
+                readyAccuracy: number | null;
                 minConfidence: number;
                 deferRate: number | null;
-                readyAccuracy: number | null;
                 readyCases: number;
                 reviewCases: number;
                 failedCases: number;
@@ -225,9 +225,9 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             }[];
             thresholdEvaluation: string;
             thresholds: {
+                readyAccuracy: number | null;
                 minConfidence: number;
                 deferRate: number | null;
-                readyAccuracy: number | null;
                 readyCases: number;
                 reviewCases: number;
                 failedCases: number;
@@ -328,9 +328,9 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             }[];
             thresholdEvaluation: string;
             thresholds: {
+                readyAccuracy: number | null;
                 minConfidence: number;
                 deferRate: number | null;
-                readyAccuracy: number | null;
                 readyCases: number;
                 reviewCases: number;
                 failedCases: number;
@@ -411,9 +411,9 @@ export declare function compare(baseline: EvaluationRun, candidate: EvaluationRu
             }[];
             thresholdEvaluation: string;
             thresholds: {
+                readyAccuracy: number | null;
                 minConfidence: number;
                 deferRate: number | null;
-                readyAccuracy: number | null;
                 readyCases: number;
                 reviewCases: number;
                 failedCases: number;

package/dist/evaluation/report.d.ts

@@ -29,9 +29,9 @@ export declare function buildReport(id: string, rows: EvaluationRow[], confidenc
     }[];
     thresholdEvaluation: string;
     thresholds: {
+        readyAccuracy: number | null;
         minConfidence: number;
         deferRate: number | null;
-        readyAccuracy: number | null;
         readyCases: number;
         reviewCases: number;
         failedCases: number;
@@ -112,9 +112,9 @@ export declare function buildReport(id: string, rows: EvaluationRow[], confidenc
     }[];
     thresholdEvaluation: string;
     thresholds: {
+        readyAccuracy: number | null;
         minConfidence: number;
         deferRate: number | null;
-        readyAccuracy: number | null;
         readyCases: number;
         reviewCases: number;
         failedCases: number;

package/dist/evaluation/report.js

@@ -1,26 +1,34 @@
 import { isRecord, roundedTo } from './decisions.js';
 import { matchesExpected } from './comparison.js';
+import { resultMetadataSchema } from '../src/schema.js';
 export function buildReport(id, rows, confidenceThresholds, price, evidence) {
     const completed = rows.filter((row) => row.error === undefined);
     const ready = completed.filter((row) => row.ready);
-    const thresholds = confidenceThresholds.map((minConfidence) => {
+    const thresholdEvaluations = confidenceThresholds.map((minConfidence) => {
         const decisions = completed
             .flatMap((row) => row.decisions)
             .filter((decision) => decision.minConfidence === minConfidence);
         const accepted = decisions.filter((decision) => decision.ready);
+        const correctReadyCases = accepted.filter((decision) => decision.correct).length;
         return {
             minConfidence,
             deferRate: completed.length
                 ? roundedTo(3, (completed.length - accepted.length) / completed.length)
                 : null,
-            readyAccuracy: accuracyOf(accepted),
+            readyAccuracy: accepted.length ? correctReadyCases / accepted.length : null,
             readyCases: accepted.length,
             reviewCases: completed.length - accepted.length,
             failedCases: rows.length - completed.length,
-            readyAccuracyInterval95: wilsonInterval(accepted.filter((entry) => entry.correct).length, accepted.length),
+            readyAccuracyInterval95: wilsonInterval(correctReadyCases, accepted.length),
         };
     });
-    const usage = usageOf(rows);
+    const suggestedMinConfidence = thresholdEvaluations.find((entry) => entry.readyAccuracy !== null && entry.readyAccuracy >= 0.95)?.minConfidence ?? null;
+    const thresholds = thresholdEvaluations.map((threshold) => ({
+        ...threshold,
+        readyAccuracy: threshold.readyAccuracy === null ? null : roundedTo(3, threshold.readyAccuracy),
+    }));
+    const usage = summarizeUsage(rows);
+    const cost = estimateKnownCost(usage, price);
     const models = [...new Set(rows.flatMap((row) => (row.model ? [row.model] : [])))].sort();
     return {
         ...(evidence ? { evidence } : {}),
@@ -60,21 +68,10 @@ export function buildReport(id, rows, confidenceThresholds, price, evidence) {
                 ? 'recipe-replay'
                 : 'not-applicable',
         thresholds,
-        suggestedMinConfidence: thresholds.find((entry) => entry.readyAccuracy !== null && entry.readyAccuracy >= 0.95)
-            ?.minConfidence ?? null,
+        suggestedMinConfidence,
         latencyMs: latencyOf(rows.map((row) => row.durationMs)),
         usage,
-        cost: price &&
-            usage.models.every((model) => model === price.model) &&
-            usage.unknownResponses === 0 &&
-            usage.failedRequests === 0
-            ? {
-                ...price,
-                estimated: roundedTo(8, (usage.input_tokens * price.inputPerMillion +
-                    usage.output_tokens * price.outputPerMillion) /
-                    1_000_000),
-            }
-            : null,
+        cost,
         confusion: confusionOf(rows),
         failures: rows
             .filter((row) => !row.correct)
@@ -126,7 +123,7 @@ function latencyOf(values) {
         totalCaseTime: roundedTo(2, values.reduce((sum, value) => sum + value, 0)),
     };
 }
-function usageOf(rows) {
+function summarizeUsage(rows) {
     let input_tokens = 0, output_tokens = 0, unknownResponses = 0, failedRequests = 0;
     const models = new Set();
     const exchanges = rows.flatMap((row) => row.exchanges);
@@ -135,18 +132,17 @@ function usageOf(rows) {
             failedRequests++;
             continue;
         }
-        const response = exchange.response;
-        if (isRecord(response) && typeof response.model === 'string')
-            models.add(response.model);
-        if (!isRecord(response) ||
-            !isRecord(response.usage) ||
-            typeof response.usage.input_tokens !== 'number' ||
-            typeof response.usage.output_tokens !== 'number') {
+        const response = isRecord(exchange.response) ? exchange.response : {};
+        const usage = resultMetadataSchema.shape.usage.safeParse(response.usage);
+        const model = resultMetadataSchema.shape.model.safeParse(response.model);
+        if (!usage.success || !model.success)
             unknownResponses++;
+        if (model.success)
+            models.add(model.data);
+        if (!usage.success)
             continue;
-        }
-        input_tokens += response.usage.input_tokens;
-        output_tokens += response.usage.output_tokens;
+        input_tokens += usage.data.input_tokens;
+        output_tokens += usage.data.output_tokens;
     }
     return {
         requests: exchanges.length,
@@ -157,6 +153,18 @@ function usageOf(rows) {
         models: [...models].sort(),
     };
 }
+function estimateKnownCost(usage, price) {
+    if (!price ||
+        usage.unknownResponses > 0 ||
+        usage.failedRequests > 0 ||
+        usage.models.some((model) => model !== price.model))
+        return null;
+    return {
+        ...price,
+        estimated: roundedTo(8, (usage.input_tokens * price.inputPerMillion + usage.output_tokens * price.outputPerMillion) /
+            1_000_000),
+    };
+}
 function confusionOf(rows) {
     const byExpected = new Map();
     for (const row of rows) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jev-recipes",
-  "version": "0.9.6",
+  "version": "0.9.7",
   "description": "248 small Jev decisions for JavaScript and TypeScript: route, rerank, gate, grade, compare, and label text for agents, RAG, support, code review, and music. Each returns a typed result with explicit uncertainty. Plug into Vercel AI SDK or LangChain agents, import one recipe, or run the CLI with JSON from any language.",
   "type": "module",
   "license": "MIT",