jev-recipes 0.7.0 → 0.8.0

package/dist/cli/schema.d.ts

@@ -28,6 +28,7 @@ export declare const commandArgumentsSchema: z.ZodUnion<readonly [z.ZodTuple<[z.
     "bug-report-completeness": "bug-report-completeness";
     "buying-intent": "buying-intent";
     "cache-match": "cache-match";
+    "callback-responsibility": "callback-responsibility";
     "cancellation-check": "cancellation-check";
     "care-urgency-wording": "care-urgency-wording";
     "category-fit": "category-fit";
@@ -55,6 +56,7 @@ export declare const commandArgumentsSchema: z.ZodUnion<readonly [z.ZodTuple<[z.
     "consent-request": "consent-request";
     "consent-scope-fit": "consent-scope-fit";
     "constraint-strength": "constraint-strength";
+    "contact-opt-out": "contact-opt-out";
     "content-facets": "content-facets";
     "content-freshness-signal": "content-freshness-signal";
     "context-prune": "context-prune";
@@ -95,6 +97,7 @@ export declare const commandArgumentsSchema: z.ZodUnion<readonly [z.ZodTuple<[z.
     "financial-advice-signal": "financial-advice-signal";
     "finger-actions": "finger-actions";
     "followup-link": "followup-link";
+    "followup-timing": "followup-timing";
     "forecast-confidence-wording": "forecast-confidence-wording";
     "format-fit": "format-fit";
     "freshness-needed": "freshness-needed";
@@ -275,6 +278,7 @@ export declare const commandArgumentsSchema: z.ZodUnion<readonly [z.ZodTuple<[z.
     "bug-report-completeness": "bug-report-completeness";
     "buying-intent": "buying-intent";
     "cache-match": "cache-match";
+    "callback-responsibility": "callback-responsibility";
     "cancellation-check": "cancellation-check";
     "care-urgency-wording": "care-urgency-wording";
     "category-fit": "category-fit";
@@ -302,6 +306,7 @@ export declare const commandArgumentsSchema: z.ZodUnion<readonly [z.ZodTuple<[z.
     "consent-request": "consent-request";
     "consent-scope-fit": "consent-scope-fit";
     "constraint-strength": "constraint-strength";
+    "contact-opt-out": "contact-opt-out";
     "content-facets": "content-facets";
     "content-freshness-signal": "content-freshness-signal";
     "context-prune": "context-prune";
@@ -342,6 +347,7 @@ export declare const commandArgumentsSchema: z.ZodUnion<readonly [z.ZodTuple<[z.
     "financial-advice-signal": "financial-advice-signal";
     "finger-actions": "finger-actions";
     "followup-link": "followup-link";
+    "followup-timing": "followup-timing";
     "forecast-confidence-wording": "forecast-confidence-wording";
     "format-fit": "format-fit";
     "freshness-needed": "freshness-needed";
@@ -519,6 +525,7 @@ export declare const commandArgumentsSchema: z.ZodUnion<readonly [z.ZodTuple<[z.
     "bug-report-completeness": "bug-report-completeness";
     "buying-intent": "buying-intent";
     "cache-match": "cache-match";
+    "callback-responsibility": "callback-responsibility";
     "cancellation-check": "cancellation-check";
     "care-urgency-wording": "care-urgency-wording";
     "category-fit": "category-fit";
@@ -546,6 +553,7 @@ export declare const commandArgumentsSchema: z.ZodUnion<readonly [z.ZodTuple<[z.
     "consent-request": "consent-request";
     "consent-scope-fit": "consent-scope-fit";
     "constraint-strength": "constraint-strength";
+    "contact-opt-out": "contact-opt-out";
     "content-facets": "content-facets";
     "content-freshness-signal": "content-freshness-signal";
     "context-prune": "context-prune";
@@ -586,6 +594,7 @@ export declare const commandArgumentsSchema: z.ZodUnion<readonly [z.ZodTuple<[z.
     "financial-advice-signal": "financial-advice-signal";
     "finger-actions": "finger-actions";
     "followup-link": "followup-link";
+    "followup-timing": "followup-timing";
     "forecast-confidence-wording": "forecast-confidence-wording";
     "format-fit": "format-fit";
     "freshness-needed": "freshness-needed";

package/dist/evaluation/archive.d.ts

@@ -0,0 +1,501 @@
+import { z } from 'zod';
+import type { DecisionClient } from '../src/schema.js';
+import type { EvaluationRow } from './engine.js';
+import { buildReport } from './report.js';
+import type { EvaluationOptions, EvaluationPolicy } from './schema.js';
+declare const archiveSchema: z.ZodObject<{
+    format: z.ZodLiteral<1>;
+    runId: z.ZodString;
+    createdAt: z.ZodISODateTime;
+    mode: z.ZodEnum<{
+        live: "live";
+        fixture: "fixture";
+        replay: "replay";
+    }>;
+    packageVersion: z.ZodString;
+    recipe: z.ZodString;
+    recipeFingerprint: z.ZodString;
+    datasetFingerprint: z.ZodString;
+    inputFingerprint: z.ZodString;
+    answerKeyFingerprint: z.ZodString;
+    split: z.ZodEnum<{
+        development: "development";
+        "held-out": "held-out";
+    }>;
+    requestedModel: z.ZodString;
+    policy: z.ZodObject<{
+        minConfidence: z.ZodOptional<z.ZodNumber>;
+        selectedOn: z.ZodOptional<z.ZodObject<{
+            runId: z.ZodString;
+            datasetFingerprint: z.ZodString;
+            recipeFingerprint: z.ZodString;
+            model: z.ZodString;
+            split: z.ZodLiteral<"development">;
+        }, z.core.$strip>>;
+    }, z.core.$strip>;
+    price: z.ZodOptional<z.ZodObject<{
+        model: z.ZodString;
+        inputPerMillion: z.ZodNumber;
+        outputPerMillion: z.ZodNumber;
+        currency: z.ZodLiteral<"USD">;
+        date: z.ZodISODate;
+        source: z.ZodString;
+    }, z.core.$strip>>;
+    cases: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        input: z.ZodNonOptional<z.ZodUnknown & z.ZodType<Record<string, import("../src/data.js").JsonValue>, unknown, z.core.$ZodTypeInternals<Record<string, import("../src/data.js").JsonValue>, unknown>>>;
+        expected: z.ZodNonOptional<z.ZodUnknown & z.ZodType<Record<string, import("../src/data.js").JsonValue>, unknown, z.core.$ZodTypeInternals<Record<string, import("../src/data.js").JsonValue>, unknown>>>;
+        rationale: z.ZodString;
+        family: z.ZodOptional<z.ZodString>;
+        split: z.ZodDefault<z.ZodEnum<{
+            development: "development";
+            "held-out": "held-out";
+        }>>;
+        provenance: z.ZodDefault<z.ZodObject<{
+            method: z.ZodEnum<{
+                "author-synthetic": "author-synthetic";
+                "human-reviewed": "human-reviewed";
+                "public-dataset": "public-dataset";
+                unspecified: "unspecified";
+            }>;
+            source: z.ZodString;
+        }, z.core.$strip>>;
+        contested: z.ZodDefault<z.ZodBoolean>;
+        adversarial: z.ZodDefault<z.ZodBoolean>;
+    }, z.core.$strip>>;
+    rows: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        expected: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        contested: z.ZodBoolean;
+        adversarial: z.ZodBoolean;
+        actual: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        result: z.ZodOptional<z.ZodUnknown>;
+        error: z.ZodOptional<z.ZodString>;
+        confidence: z.ZodOptional<z.ZodNumber>;
+        model: z.ZodOptional<z.ZodString>;
+        correct: z.ZodBoolean;
+        ready: z.ZodBoolean;
+        durationMs: z.ZodNumber;
+        decisions: z.ZodArray<z.ZodObject<{
+            minConfidence: z.ZodNumber;
+            ready: z.ZodBoolean;
+            correct: z.ZodBoolean;
+        }, z.core.$strip>>;
+        exchanges: z.ZodArray<z.ZodObject<{
+            request: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+            response: z.ZodOptional<z.ZodUnknown>;
+            error: z.ZodOptional<z.ZodString>;
+            durationMs: z.ZodNumber;
+        }, z.core.$strip>>;
+    }, z.core.$loose>>;
+    thresholds: z.ZodArray<z.ZodNumber>;
+    sourceRun: z.ZodOptional<z.ZodString>;
+    report: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+}, z.core.$strip>;
+export interface EvaluationRun extends Omit<z.infer<typeof archiveSchema>, 'rows' | 'report'> {
+    rows: EvaluationRow[];
+    report: ReturnType<typeof buildReport>;
+}
+export declare function evaluate(name: string, values: unknown[], options?: EvaluationOptions & {
+    out?: string;
+    client?: DecisionClient;
+    mode?: 'live' | 'fixture';
+}): Promise<EvaluationRun>;
+export declare function replay(run: EvaluationRun, policy?: EvaluationPolicy, out?: string): Promise<EvaluationRun>;
+export declare function readRun(directory: string): Promise<EvaluationRun>;
+export declare function freezePolicy(run: EvaluationRun): EvaluationPolicy;
+export declare function compare(baseline: EvaluationRun, candidate: EvaluationRun): {
+    recipe: string;
+    cases: number;
+    split: "development" | "held-out";
+    baseline: {
+        runId: string;
+        packageVersion: string;
+        recipeFingerprint: string;
+        requestedModel: string;
+        models: string[];
+        policy: {
+            minConfidence?: number | undefined;
+            selectedOn?: {
+                runId: string;
+                datasetFingerprint: string;
+                recipeFingerprint: string;
+                model: string;
+                split: "development";
+            } | undefined;
+        };
+        mode: "live" | "fixture" | "replay";
+        report: {
+            contestedAccuracy: number | null;
+            adversarialAccuracy: number | null;
+            calibration: {
+                confidence: string;
+                cases: number;
+                accuracy: number | null;
+            }[];
+            thresholdEvaluation: string;
+            thresholds: {
+                minConfidence: number;
+                deferRate: number | null;
+                readyAccuracy: number | null;
+                readyCases: number;
+                reviewCases: number;
+                failedCases: number;
+                readyAccuracyInterval95: [number, number] | null;
+            }[];
+            suggestedMinConfidence: number | null;
+            latencyMs: {
+                p50: number | null;
+                p95: number | null;
+                totalCaseTime: number;
+            };
+            usage: {
+                requests: number;
+                input_tokens: number;
+                output_tokens: number;
+                unknownResponses: number;
+                failedRequests: number;
+                models: string[];
+            };
+            cost: {
+                estimated: number;
+                model: string;
+                inputPerMillion: number;
+                outputPerMillion: number;
+                currency: "USD";
+                date: string;
+                source: string;
+            } | null;
+            confusion: {
+                [k: string]: {
+                    total: number;
+                    correct: number;
+                    misses: Record<string, number>;
+                };
+            };
+            failures: ({
+                adversarial?: true;
+                contested?: true;
+                confidence?: number;
+                actual: Record<string, unknown> | undefined;
+                id: string;
+                expected: Record<string, unknown>;
+            } | {
+                adversarial?: true;
+                contested?: true;
+                confidence?: number;
+                error: string;
+                id: string;
+                expected: Record<string, unknown>;
+            })[];
+            items?: never;
+            correctItems?: never;
+            itemAccuracy?: never;
+            recipe: string;
+            model: string;
+            models: string[];
+            cases: number;
+            completed: number;
+            failed: number;
+            correct: number;
+            wrong: number;
+            ready: number;
+            review: number;
+            accuracy: number | null;
+            completedAccuracy: number | null;
+            accuracyInterval95: [number, number] | null;
+            readyAccuracy: number | null;
+            readyAccuracyInterval95: [number, number] | null;
+            reviewRate: number | null;
+            evidence?: import("./report.js").ReportEvidence;
+        } | {
+            contestedAccuracy: number | null;
+            adversarialAccuracy: number | null;
+            calibration: {
+                confidence: string;
+                cases: number;
+                accuracy: number | null;
+            }[];
+            thresholdEvaluation: string;
+            thresholds: {
+                minConfidence: number;
+                deferRate: number | null;
+                readyAccuracy: number | null;
+                readyCases: number;
+                reviewCases: number;
+                failedCases: number;
+                readyAccuracyInterval95: [number, number] | null;
+            }[];
+            suggestedMinConfidence: number | null;
+            latencyMs: {
+                p50: number | null;
+                p95: number | null;
+                totalCaseTime: number;
+            };
+            usage: {
+                requests: number;
+                input_tokens: number;
+                output_tokens: number;
+                unknownResponses: number;
+                failedRequests: number;
+                models: string[];
+            };
+            cost: {
+                estimated: number;
+                model: string;
+                inputPerMillion: number;
+                outputPerMillion: number;
+                currency: "USD";
+                date: string;
+                source: string;
+            } | null;
+            confusion: {
+                [k: string]: {
+                    total: number;
+                    correct: number;
+                    misses: Record<string, number>;
+                };
+            };
+            failures: ({
+                adversarial?: true;
+                contested?: true;
+                confidence?: number;
+                actual: Record<string, unknown> | undefined;
+                id: string;
+                expected: Record<string, unknown>;
+            } | {
+                adversarial?: true;
+                contested?: true;
+                confidence?: number;
+                error: string;
+                id: string;
+                expected: Record<string, unknown>;
+            })[];
+            items: number;
+            correctItems: number;
+            itemAccuracy: number;
+            recipe: string;
+            model: string;
+            models: string[];
+            cases: number;
+            completed: number;
+            failed: number;
+            correct: number;
+            wrong: number;
+            ready: number;
+            review: number;
+            accuracy: number | null;
+            completedAccuracy: number | null;
+            accuracyInterval95: [number, number] | null;
+            readyAccuracy: number | null;
+            readyAccuracyInterval95: [number, number] | null;
+            reviewRate: number | null;
+            evidence?: import("./report.js").ReportEvidence;
+        };
+    };
+    candidate: {
+        runId: string;
+        packageVersion: string;
+        recipeFingerprint: string;
+        requestedModel: string;
+        models: string[];
+        policy: {
+            minConfidence?: number | undefined;
+            selectedOn?: {
+                runId: string;
+                datasetFingerprint: string;
+                recipeFingerprint: string;
+                model: string;
+                split: "development";
+            } | undefined;
+        };
+        mode: "live" | "fixture" | "replay";
+        report: {
+            contestedAccuracy: number | null;
+            adversarialAccuracy: number | null;
+            calibration: {
+                confidence: string;
+                cases: number;
+                accuracy: number | null;
+            }[];
+            thresholdEvaluation: string;
+            thresholds: {
+                minConfidence: number;
+                deferRate: number | null;
+                readyAccuracy: number | null;
+                readyCases: number;
+                reviewCases: number;
+                failedCases: number;
+                readyAccuracyInterval95: [number, number] | null;
+            }[];
+            suggestedMinConfidence: number | null;
+            latencyMs: {
+                p50: number | null;
+                p95: number | null;
+                totalCaseTime: number;
+            };
+            usage: {
+                requests: number;
+                input_tokens: number;
+                output_tokens: number;
+                unknownResponses: number;
+                failedRequests: number;
+                models: string[];
+            };
+            cost: {
+                estimated: number;
+                model: string;
+                inputPerMillion: number;
+                outputPerMillion: number;
+                currency: "USD";
+                date: string;
+                source: string;
+            } | null;
+            confusion: {
+                [k: string]: {
+                    total: number;
+                    correct: number;
+                    misses: Record<string, number>;
+                };
+            };
+            failures: ({
+                adversarial?: true;
+                contested?: true;
+                confidence?: number;
+                actual: Record<string, unknown> | undefined;
+                id: string;
+                expected: Record<string, unknown>;
+            } | {
+                adversarial?: true;
+                contested?: true;
+                confidence?: number;
+                error: string;
+                id: string;
+                expected: Record<string, unknown>;
+            })[];
+            items?: never;
+            correctItems?: never;
+            itemAccuracy?: never;
+            recipe: string;
+            model: string;
+            models: string[];
+            cases: number;
+            completed: number;
+            failed: number;
+            correct: number;
+            wrong: number;
+            ready: number;
+            review: number;
+            accuracy: number | null;
+            completedAccuracy: number | null;
+            accuracyInterval95: [number, number] | null;
+            readyAccuracy: number | null;
+            readyAccuracyInterval95: [number, number] | null;
+            reviewRate: number | null;
+            evidence?: import("./report.js").ReportEvidence;
+        } | {
+            contestedAccuracy: number | null;
+            adversarialAccuracy: number | null;
+            calibration: {
+                confidence: string;
+                cases: number;
+                accuracy: number | null;
+            }[];
+            thresholdEvaluation: string;
+            thresholds: {
+                minConfidence: number;
+                deferRate: number | null;
+                readyAccuracy: number | null;
+                readyCases: number;
+                reviewCases: number;
+                failedCases: number;
+                readyAccuracyInterval95: [number, number] | null;
+            }[];
+            suggestedMinConfidence: number | null;
+            latencyMs: {
+                p50: number | null;
+                p95: number | null;
+                totalCaseTime: number;
+            };
+            usage: {
+                requests: number;
+                input_tokens: number;
+                output_tokens: number;
+                unknownResponses: number;
+                failedRequests: number;
+                models: string[];
+            };
+            cost: {
+                estimated: number;
+                model: string;
+                inputPerMillion: number;
+                outputPerMillion: number;
+                currency: "USD";
+                date: string;
+                source: string;
+            } | null;
+            confusion: {
+                [k: string]: {
+                    total: number;
+                    correct: number;
+                    misses: Record<string, number>;
+                };
+            };
+            failures: ({
+                adversarial?: true;
+                contested?: true;
+                confidence?: number;
+                actual: Record<string, unknown> | undefined;
+                id: string;
+                expected: Record<string, unknown>;
+            } | {
+                adversarial?: true;
+                contested?: true;
+                confidence?: number;
+                error: string;
+                id: string;
+                expected: Record<string, unknown>;
+            })[];
+            items: number;
+            correctItems: number;
+            itemAccuracy: number;
+            recipe: string;
+            model: string;
+            models: string[];
+            cases: number;
+            completed: number;
+            failed: number;
+            correct: number;
+            wrong: number;
+            ready: number;
+            review: number;
+            accuracy: number | null;
+            completedAccuracy: number | null;
+            accuracyInterval95: [number, number] | null;
+            readyAccuracy: number | null;
+            readyAccuracyInterval95: [number, number] | null;
+            reviewRate: number | null;
+            evidence?: import("./report.js").ReportEvidence;
+        };
+    };
+    delta: {
+        accuracy: number | null;
+        readyAccuracy: number | null;
+        review: number;
+        failed: number;
+    };
+    changed: {
+        id: string;
+        before: {
+            correct: boolean;
+            ready: boolean;
+            error: string | undefined;
+        };
+        after: {
+            correct: boolean;
+            ready: boolean;
+            error: string | undefined;
+        };
+    }[];
+};
+export {};