jev-recipes 0.7.0 → 0.8.1

package/dist/evaluation/engine.js

@@ -0,0 +1,130 @@
+import { isDeepStrictEqual } from 'node:util';
+import { createClient } from '../src/client.js';
+import { caseConfidence, isReady, mapWithConcurrencyLimit, matchesExpected, pickPaths, } from './decisions.js';
+export const confidenceThresholds = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95];
+export async function evaluateCases(recipe, cases, options = {}) {
+    let requests = 0;
+    let client = options.client;
+    const limitedClient = {
+        async systemOne(request, requestOptions) {
+            if (requests >= (options.maxRequests ?? 1000))
+                throw new Error('Evaluation request limit reached.');
+            requests++;
+            client ??= createClient();
+            return client.systemOne(request, requestOptions);
+        },
+    };
+    return mapWithConcurrencyLimit(cases, options.concurrency ?? 4, async (goldenCase, index) => {
+        const recorded = options.recorded?.find((row) => row.id === goldenCase.id);
+        if (options.recorded && !recorded)
+            throw new Error(`Missing archived case: ${goldenCase.id}`);
+        const playback = recorded ? replayResponses(recorded.exchanges) : undefined;
+        const row = await evaluateCase(recipe, goldenCase, playback?.client ?? limitedClient, options);
+        playback?.assertComplete();
+        if (recorded) {
+            row.durationMs = recorded.durationMs;
+            row.exchanges = recorded.exchanges;
+        }
+        await options.onCase?.(row, index);
+        return row;
+    });
+}
+async function evaluateCase(recipe, goldenCase, client, options) {
+    const row = {
+        id: goldenCase.id,
+        expected: goldenCase.expected,
+        contested: goldenCase.contested,
+        adversarial: goldenCase.adversarial,
+        correct: false,
+        ready: false,
+        durationMs: 0,
+        exchanges: [],
+        decisions: [],
+    };
+    const started = performance.now();
+    const recipeOptions = { client: recordResponses(client, row.exchanges) };
+    if (options.model !== undefined)
+        recipeOptions.model = options.model;
+    if (options.signal !== undefined)
+        recipeOptions.signal = options.signal;
+    try {
+        row.result = await recipe.run(goldenCase.input, recipeOptions);
+        row.actual = pickPaths(row.result, Object.keys(goldenCase.expected));
+        row.correct = matchesExpected(row.actual, row.expected);
+        row.ready = isReady(row.result);
+        row.confidence = caseConfidence(row.result, Object.keys(goldenCase.expected));
+        const model = pickPaths(row.result, ['model']).model;
+        if (typeof model === 'string')
+            row.model = model;
+    }
+    catch (error) {
+        if (error instanceof ReplayMismatch)
+            throw error;
+        row.error = safeError(error);
+    }
+    row.durationMs = Math.round((performance.now() - started) * 100) / 100;
+    if (row.error !== undefined)
+        return row;
+    for (const minConfidence of options.thresholds ?? []) {
+        const playback = replayResponses(row.exchanges);
+        const result = await recipe.run({ ...goldenCase.input, minConfidence }, { ...recipeOptions, client: playback.client });
+        playback.assertComplete();
+        row.decisions.push({
+            minConfidence,
+            ready: isReady(result),
+            correct: matchesExpected(pickPaths(result, Object.keys(row.expected)), row.expected),
+        });
+    }
+    return row;
+}
+function recordResponses(client, exchanges) {
+    return {
+        async systemOne(request, options) {
+            const exchange = { request: structuredClone(request), durationMs: 0 };
+            exchanges.push(exchange);
+            const started = performance.now();
+            try {
+                const response = await client.systemOne(request, options);
+                exchange.response = structuredClone(response);
+                return response;
+            }
+            catch (error) {
+                exchange.error = safeError(error);
+                throw error;
+            }
+            finally {
+                exchange.durationMs = Math.round((performance.now() - started) * 100) / 100;
+            }
+        },
+    };
+}
+class ReplayMismatch extends Error {
+}
+export function replayResponses(exchanges) {
+    const remaining = [...exchanges];
+    return {
+        client: {
+            async systemOne(request) {
+                const index = remaining.findIndex((exchange) => isDeepStrictEqual(exchange.request, request));
+                if (index === -1)
+                    throw new ReplayMismatch('Replay requested an unrecorded model response. The prompt or model has changed.');
+                const [exchange] = remaining.splice(index, 1);
+                if (exchange.error !== undefined)
+                    throw new Error(exchange.error);
+                return structuredClone(exchange.response);
+            },
+        },
+        assertComplete() {
+            if (remaining.length)
+                throw new ReplayMismatch('Replay did not consume every recorded request. The recipe has changed.');
+        },
+    };
+}
+function safeError(error) {
+    let message = error instanceof Error ? error.message : String(error);
+    for (const [name, value] of Object.entries(process.env)) {
+        if (/KEY|TOKEN|SECRET|PASSWORD/i.test(name) && value && value.length >= 6)
+            message = message.replaceAll(value, '[redacted]');
+    }
+    return message.replace(/Bearer\s+\S+/gi, 'Bearer [redacted]');
+}

package/dist/evaluation/index.d.ts

@@ -0,0 +1,5 @@
+export { evaluate, replay, compare, readRun, freezePolicy } from './archive.js';
+export type { EvaluationRun } from './archive.js';
+export { readEvaluationCases } from './dataset.js';
+export { evaluationCaseSchema, evaluationOptionsSchema, evaluationPolicySchema } from './schema.js';
+export type { EvaluationCase, EvaluationPolicy, EvaluationOptions, Price } from './schema.js';

package/dist/evaluation/index.js

@@ -0,0 +1,3 @@
+export { evaluate, replay, compare, readRun, freezePolicy } from './archive.js';
+export { readEvaluationCases } from './dataset.js';
+export { evaluationCaseSchema, evaluationOptionsSchema, evaluationPolicySchema } from './schema.js';

package/dist/evaluation/report.d.ts

@@ -0,0 +1,186 @@
+import type { EvaluationRow } from './engine.js';
+import type { Price } from './schema.js';
+import type { EvaluationCase, EvaluationPolicy } from './schema.js';
+export interface ReportEvidence {
+    runId: string;
+    evaluatedAt: string;
+    packageVersion: string;
+    recipeFingerprint: string;
+    datasetFingerprint: string;
+    split: 'development' | 'held-out';
+    mode: 'live' | 'fixture' | 'replay';
+    policy: EvaluationPolicy;
+    provenance: {
+        method: EvaluationCase['provenance']['method'];
+        source: string;
+        cases: number;
+    }[];
+}
+export declare function buildReport(id: string, rows: EvaluationRow[], confidenceThresholds: number[], price?: Price, evidence?: ReportEvidence): {
+    contestedAccuracy: number | null;
+    adversarialAccuracy: number | null;
+    calibration: {
+        confidence: string;
+        cases: number;
+        accuracy: number | null;
+    }[];
+    thresholdEvaluation: string;
+    thresholds: {
+        minConfidence: number;
+        deferRate: number | null;
+        readyAccuracy: number | null;
+        readyCases: number;
+        reviewCases: number;
+        failedCases: number;
+        readyAccuracyInterval95: [number, number] | null;
+    }[];
+    suggestedMinConfidence: number | null;
+    latencyMs: {
+        p50: number | null;
+        p95: number | null;
+        totalCaseTime: number;
+    };
+    usage: {
+        requests: number;
+        input_tokens: number;
+        output_tokens: number;
+        unknownResponses: number;
+        failedRequests: number;
+        models: string[];
+    };
+    cost: {
+        estimated: number;
+        model: string;
+        inputPerMillion: number;
+        outputPerMillion: number;
+        currency: "USD";
+        date: string;
+        source: string;
+    } | null;
+    confusion: {
+        [k: string]: {
+            total: number;
+            correct: number;
+            misses: Record<string, number>;
+        };
+    };
+    failures: ({
+        adversarial?: true;
+        contested?: true;
+        confidence?: number;
+        actual: Record<string, unknown> | undefined;
+        id: string;
+        expected: Record<string, unknown>;
+    } | {
+        adversarial?: true;
+        contested?: true;
+        confidence?: number;
+        error: string;
+        id: string;
+        expected: Record<string, unknown>;
+    })[];
+    items?: never;
+    correctItems?: never;
+    itemAccuracy?: never;
+    recipe: string;
+    model: string;
+    models: string[];
+    cases: number;
+    completed: number;
+    failed: number;
+    correct: number;
+    wrong: number;
+    ready: number;
+    review: number;
+    accuracy: number | null;
+    completedAccuracy: number | null;
+    accuracyInterval95: [number, number] | null;
+    readyAccuracy: number | null;
+    readyAccuracyInterval95: [number, number] | null;
+    reviewRate: number | null;
+    evidence?: ReportEvidence;
+} | {
+    contestedAccuracy: number | null;
+    adversarialAccuracy: number | null;
+    calibration: {
+        confidence: string;
+        cases: number;
+        accuracy: number | null;
+    }[];
+    thresholdEvaluation: string;
+    thresholds: {
+        minConfidence: number;
+        deferRate: number | null;
+        readyAccuracy: number | null;
+        readyCases: number;
+        reviewCases: number;
+        failedCases: number;
+        readyAccuracyInterval95: [number, number] | null;
+    }[];
+    suggestedMinConfidence: number | null;
+    latencyMs: {
+        p50: number | null;
+        p95: number | null;
+        totalCaseTime: number;
+    };
+    usage: {
+        requests: number;
+        input_tokens: number;
+        output_tokens: number;
+        unknownResponses: number;
+        failedRequests: number;
+        models: string[];
+    };
+    cost: {
+        estimated: number;
+        model: string;
+        inputPerMillion: number;
+        outputPerMillion: number;
+        currency: "USD";
+        date: string;
+        source: string;
+    } | null;
+    confusion: {
+        [k: string]: {
+            total: number;
+            correct: number;
+            misses: Record<string, number>;
+        };
+    };
+    failures: ({
+        adversarial?: true;
+        contested?: true;
+        confidence?: number;
+        actual: Record<string, unknown> | undefined;
+        id: string;
+        expected: Record<string, unknown>;
+    } | {
+        adversarial?: true;
+        contested?: true;
+        confidence?: number;
+        error: string;
+        id: string;
+        expected: Record<string, unknown>;
+    })[];
+    items: number;
+    correctItems: number;
+    itemAccuracy: number;
+    recipe: string;
+    model: string;
+    models: string[];
+    cases: number;
+    completed: number;
+    failed: number;
+    correct: number;
+    wrong: number;
+    ready: number;
+    review: number;
+    accuracy: number | null;
+    completedAccuracy: number | null;
+    accuracyInterval95: [number, number] | null;
+    readyAccuracy: number | null;
+    readyAccuracyInterval95: [number, number] | null;
+    reviewRate: number | null;
+    evidence?: ReportEvidence;
+};
+export declare function wilsonInterval(correct: number, total: number): [number, number] | null;

package/dist/evaluation/report.js

@@ -0,0 +1,177 @@
+import { isRecord, matchesExpected, roundedTo } from './decisions.js';
+export function buildReport(id, rows, confidenceThresholds, price, evidence) {
+    const completed = rows.filter((row) => row.error === undefined);
+    const ready = completed.filter((row) => row.ready);
+    const thresholds = confidenceThresholds.map((minConfidence) => {
+        const decisions = completed
+            .flatMap((row) => row.decisions)
+            .filter((decision) => decision.minConfidence === minConfidence);
+        const accepted = decisions.filter((decision) => decision.ready);
+        return {
+            minConfidence,
+            deferRate: completed.length
+                ? roundedTo(3, (completed.length - accepted.length) / completed.length)
+                : null,
+            readyAccuracy: accuracyOf(accepted),
+            readyCases: accepted.length,
+            reviewCases: completed.length - accepted.length,
+            failedCases: rows.length - completed.length,
+            readyAccuracyInterval95: wilsonInterval(accepted.filter((entry) => entry.correct).length, accepted.length),
+        };
+    });
+    const usage = usageOf(rows);
+    const models = [...new Set(rows.flatMap((row) => (row.model ? [row.model] : [])))].sort();
+    return {
+        ...(evidence ? { evidence } : {}),
+        recipe: id,
+        model: models.length === 1 ? models[0] : models.length ? 'mixed' : 'unknown',
+        models,
+        cases: rows.length,
+        completed: completed.length,
+        failed: rows.length - completed.length,
+        correct: rows.filter((row) => row.correct).length,
+        wrong: completed.filter((row) => !row.correct).length,
+        ready: ready.length,
+        review: completed.length - ready.length,
+        accuracy: accuracyOf(rows),
+        completedAccuracy: accuracyOf(completed),
+        accuracyInterval95: wilsonInterval(rows.filter((row) => row.correct).length, rows.length),
+        readyAccuracy: accuracyOf(ready),
+        readyAccuracyInterval95: wilsonInterval(ready.filter((row) => row.correct).length, ready.length),
+        reviewRate: completed.length
+            ? roundedTo(3, (completed.length - ready.length) / completed.length)
+            : null,
+        ...itemAccuracyOf(rows),
+        contestedAccuracy: accuracyOf(rows.filter((row) => row.contested)),
+        adversarialAccuracy: accuracyOf(rows.filter((row) => row.adversarial)),
+        calibration: [0, 0.5, 0.6, 0.7, 0.8, 0.9].map((lower, index, edges) => {
+            const upper = edges[index + 1] ?? 1.000001;
+            const bucket = rows.filter((row) => row.confidence !== undefined && row.confidence >= lower && row.confidence < upper);
+            return {
+                confidence: `${lower}-${Math.min(upper, 1)}`,
+                cases: bucket.length,
+                accuracy: accuracyOf(bucket),
+            };
+        }),
+        thresholdEvaluation: evidence?.split === 'held-out'
+            ? 'frozen-policy'
+            : confidenceThresholds.length
+                ? 'recipe-replay'
+                : 'not-applicable',
+        thresholds,
+        suggestedMinConfidence: thresholds.find((entry) => entry.readyAccuracy !== null && entry.readyAccuracy >= 0.95)
+            ?.minConfidence ?? null,
+        latencyMs: latencyOf(rows.map((row) => row.durationMs)),
+        usage,
+        cost: price &&
+            usage.models.every((model) => model === price.model) &&
+            usage.unknownResponses === 0 &&
+            usage.failedRequests === 0
+            ? {
+                ...price,
+                estimated: roundedTo(8, (usage.input_tokens * price.inputPerMillion +
+                    usage.output_tokens * price.outputPerMillion) /
+                    1_000_000),
+            }
+            : null,
+        confusion: confusionOf(rows),
+        failures: rows
+            .filter((row) => !row.correct)
+            .map(({ id, expected, actual, error, confidence, contested, adversarial }) => ({
+            id,
+            expected,
+            ...(error === undefined ? { actual } : { error }),
+            ...(confidence === undefined ? {} : { confidence: roundedTo(3, confidence) }),
+            ...(contested ? { contested } : {}),
+            ...(adversarial ? { adversarial } : {}),
+        })),
+    };
+}
+function accuracyOf(rows) {
+    return rows.length ? roundedTo(3, rows.filter((row) => row.correct).length / rows.length) : null;
+}
+function itemAccuracyOf(rows) {
+    const paths = rows.flatMap((row) => Object.entries(row.expected).map(([path, expected]) => ({
+        expected,
+        actual: row.error === undefined ? row.actual?.[path] : undefined,
+    })));
+    if (paths.length <= rows.length)
+        return {};
+    const correct = paths.filter((entry) => matchesExpected(entry.actual, entry.expected)).length;
+    return {
+        items: paths.length,
+        correctItems: correct,
+        itemAccuracy: roundedTo(3, correct / paths.length),
+    };
+}
+export function wilsonInterval(correct, total) {
+    if (!total)
+        return null;
+    const z = 1.959963984540054;
+    const proportion = correct / total;
+    const denominator = 1 + (z * z) / total;
+    const center = (proportion + (z * z) / (2 * total)) / denominator;
+    const margin = (z * Math.sqrt((proportion * (1 - proportion)) / total + (z * z) / (4 * total * total))) /
+        denominator;
+    return [roundedTo(4, Math.max(0, center - margin)), roundedTo(4, Math.min(1, center + margin))];
+}
+function latencyOf(values) {
+    const ordered = [...values].sort((a, b) => a - b);
+    const percentile = (fraction) => ordered[Math.max(0, Math.ceil(ordered.length * fraction) - 1)] ?? null;
+    return {
+        p50: percentile(0.5),
+        p95: percentile(0.95),
+        totalCaseTime: roundedTo(2, values.reduce((sum, value) => sum + value, 0)),
+    };
+}
+function usageOf(rows) {
+    let input_tokens = 0, output_tokens = 0, unknownResponses = 0, failedRequests = 0;
+    const models = new Set();
+    const exchanges = rows.flatMap((row) => row.exchanges);
+    for (const exchange of exchanges) {
+        if (exchange.error !== undefined) {
+            failedRequests++;
+            continue;
+        }
+        const response = exchange.response;
+        if (isRecord(response) && typeof response.model === 'string')
+            models.add(response.model);
+        if (!isRecord(response) ||
+            !isRecord(response.usage) ||
+            typeof response.usage.input_tokens !== 'number' ||
+            typeof response.usage.output_tokens !== 'number') {
+            unknownResponses++;
+            continue;
+        }
+        input_tokens += response.usage.input_tokens;
+        output_tokens += response.usage.output_tokens;
+    }
+    return {
+        requests: exchanges.length,
+        input_tokens,
+        output_tokens,
+        unknownResponses,
+        failedRequests,
+        models: [...models].sort(),
+    };
+}
+function confusionOf(rows) {
+    const byExpected = new Map();
+    for (const row of rows) {
+        const label = JSON.stringify(row.expected);
+        const entry = byExpected.get(label) ?? {
+            total: 0,
+            correct: 0,
+            misses: Object.create(null),
+        };
+        entry.total++;
+        if (row.correct)
+            entry.correct++;
+        else {
+            const actual = row.error === undefined ? JSON.stringify(row.actual) : `error: ${row.error}`;
+            entry.misses[actual] = (entry.misses[actual] ?? 0) + 1;
+        }
+        byExpected.set(label, entry);
+    }
+    return Object.fromEntries([...byExpected.entries()].sort(([a], [b]) => a.localeCompare(b)));
+}

package/dist/evaluation/schema.d.ts

@@ -0,0 +1,82 @@
+import { z } from 'zod';
+export declare const provenanceSchema: z.ZodObject<{
+    method: z.ZodEnum<{
+        "author-synthetic": "author-synthetic";
+        "human-reviewed": "human-reviewed";
+        "public-dataset": "public-dataset";
+        unspecified: "unspecified";
+    }>;
+    source: z.ZodString;
+}, z.core.$strip>;
+export declare const evaluationCaseSchema: z.ZodObject<{
+    id: z.ZodString;
+    input: z.ZodNonOptional<z.ZodUnknown & z.ZodType<Record<string, import("../src/data.js").JsonValue>, unknown, z.core.$ZodTypeInternals<Record<string, import("../src/data.js").JsonValue>, unknown>>>;
+    expected: z.ZodNonOptional<z.ZodUnknown & z.ZodType<Record<string, import("../src/data.js").JsonValue>, unknown, z.core.$ZodTypeInternals<Record<string, import("../src/data.js").JsonValue>, unknown>>>;
+    rationale: z.ZodString;
+    family: z.ZodOptional<z.ZodString>;
+    split: z.ZodDefault<z.ZodEnum<{
+        development: "development";
+        "held-out": "held-out";
+    }>>;
+    provenance: z.ZodDefault<z.ZodObject<{
+        method: z.ZodEnum<{
+            "author-synthetic": "author-synthetic";
+            "human-reviewed": "human-reviewed";
+            "public-dataset": "public-dataset";
+            unspecified: "unspecified";
+        }>;
+        source: z.ZodString;
+    }, z.core.$strip>>;
+    contested: z.ZodDefault<z.ZodBoolean>;
+    adversarial: z.ZodDefault<z.ZodBoolean>;
+}, z.core.$strip>;
+export declare const evaluationPolicySchema: z.ZodObject<{
+    minConfidence: z.ZodOptional<z.ZodNumber>;
+    selectedOn: z.ZodOptional<z.ZodObject<{
+        runId: z.ZodString;
+        datasetFingerprint: z.ZodString;
+        recipeFingerprint: z.ZodString;
+        model: z.ZodString;
+        split: z.ZodLiteral<"development">;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const priceSchema: z.ZodObject<{
+    model: z.ZodString;
+    inputPerMillion: z.ZodNumber;
+    outputPerMillion: z.ZodNumber;
+    currency: z.ZodLiteral<"USD">;
+    date: z.ZodISODate;
+    source: z.ZodString;
+}, z.core.$strip>;
+export declare const evaluationOptionsSchema: z.ZodObject<{
+    concurrency: z.ZodDefault<z.ZodNumber>;
+    maxCases: z.ZodDefault<z.ZodNumber>;
+    maxRequests: z.ZodDefault<z.ZodNumber>;
+    model: z.ZodOptional<z.ZodString>;
+    split: z.ZodDefault<z.ZodEnum<{
+        development: "development";
+        "held-out": "held-out";
+    }>>;
+    policy: z.ZodDefault<z.ZodObject<{
+        minConfidence: z.ZodOptional<z.ZodNumber>;
+        selectedOn: z.ZodOptional<z.ZodObject<{
+            runId: z.ZodString;
+            datasetFingerprint: z.ZodString;
+            recipeFingerprint: z.ZodString;
+            model: z.ZodString;
+            split: z.ZodLiteral<"development">;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    price: z.ZodOptional<z.ZodObject<{
+        model: z.ZodString;
+        inputPerMillion: z.ZodNumber;
+        outputPerMillion: z.ZodNumber;
+        currency: z.ZodLiteral<"USD">;
+        date: z.ZodISODate;
+        source: z.ZodString;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type EvaluationCase = z.infer<typeof evaluationCaseSchema>;
+export type EvaluationPolicy = z.infer<typeof evaluationPolicySchema>;
+export type EvaluationOptions = z.input<typeof evaluationOptionsSchema>;
+export type Price = z.infer<typeof priceSchema>;

package/dist/evaluation/schema.js

@@ -0,0 +1,50 @@
+import { z } from 'zod';
+import { nonEmptyText, probability } from '../src/schema.js';
+import { recordWithOwnKeys, jsonValueSchema } from '../src/data.js';
+export const provenanceSchema = z.object({
+    method: z.enum(['author-synthetic', 'human-reviewed', 'public-dataset', 'unspecified']),
+    source: nonEmptyText,
+});
+export const evaluationCaseSchema = z.object({
+    id: nonEmptyText,
+    input: recordWithOwnKeys(z.string(), jsonValueSchema),
+    expected: recordWithOwnKeys(nonEmptyText, jsonValueSchema).refine((expected) => Object.keys(expected).length > 0, 'Provide at least one expected result field.'),
+    rationale: nonEmptyText,
+    family: nonEmptyText.optional(),
+    split: z.enum(['development', 'held-out']).default('development'),
+    provenance: provenanceSchema.default({
+        method: 'unspecified',
+        source: 'Not recorded by the dataset author.',
+    }),
+    contested: z.boolean().default(false),
+    adversarial: z.boolean().default(false),
+});
+export const evaluationPolicySchema = z.object({
+    minConfidence: probability.optional(),
+    selectedOn: z
+        .object({
+        runId: nonEmptyText,
+        datasetFingerprint: nonEmptyText,
+        recipeFingerprint: nonEmptyText,
+        model: nonEmptyText,
+        split: z.literal('development'),
+    })
+        .optional(),
+});
+export const priceSchema = z.object({
+    model: nonEmptyText,
+    inputPerMillion: z.number().nonnegative(),
+    outputPerMillion: z.number().nonnegative(),
+    currency: z.literal('USD'),
+    date: z.iso.date(),
+    source: nonEmptyText,
+});
+export const evaluationOptionsSchema = z.object({
+    concurrency: z.number().int().min(1).max(32).default(4),
+    maxCases: z.number().int().positive().default(1000),
+    maxRequests: z.number().int().positive().default(1000),
+    model: nonEmptyText.optional(),
+    split: z.enum(['development', 'held-out']).default('development'),
+    policy: evaluationPolicySchema.default({}),
+    price: priceSchema.optional(),
+});

package/dist/recipes/callback-responsibility/demo.json

@@ -0,0 +1,25 @@
+{
+  "input": {
+    "conversation": "Avery: I will call you when the replacement arrives. Sam: Thanks, I will wait for your call.",
+    "roles": "Avery is the business representative. Sam is the customer.",
+    "minConfidence": 0.8
+  },
+  "response": {
+    "model": "demo-fixture",
+    "answers": {
+      "decision": {
+        "type": "choice",
+        "choice": "business",
+        "confidence": 0.94,
+        "probabilities": {
+          "business": 0.94,
+          "customer": 0.01,
+          "either": 0.01,
+          "none": 0.01,
+          "unclear": 0.03
+        }
+      }
+    },
+    "usage": { "input_tokens": 0, "output_tokens": 0 }
+  }
+}

package/dist/recipes/callback-responsibility/index.d.ts

@@ -0,0 +1,5 @@
+import type { RecipeOptions } from '../../src/schema.js';
+import type { CallbackResponsibilityInput, CallbackResponsibilityResult } from './schema.js';
+export declare function callbackResponsibility(input: CallbackResponsibilityInput, options?: RecipeOptions): Promise<CallbackResponsibilityResult>;
+export { callbackResponsibilityInputSchema, callbackResponsibilityResultSchema, callbackResponsibilityVerdictSchema, } from './schema.js';
+export type { CallbackResponsibilityInput, CallbackResponsibilityResult, CallbackResponsibilityVerdict, } from './schema.js';