jev-recipes 0.7.0 → 0.8.1

package/dist/evaluation/archive.js

@@ -0,0 +1,276 @@
+import { randomUUID } from 'node:crypto';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { z } from 'zod';
+import { gunzipSync } from 'node:zlib';
+import { datasetFingerprints, fingerprint, loadEvaluationRecipe, recipeFingerprint, validateCases, } from './dataset.js';
+import { confidenceThresholds, evaluateCases } from './engine.js';
+import { buildReport } from './report.js';
+import { evaluationCaseSchema, evaluationOptionsSchema, evaluationPolicySchema, priceSchema, } from './schema.js';
+const archiveSchema = z.object({
+    format: z.literal(1),
+    runId: z.string(),
+    createdAt: z.iso.datetime(),
+    mode: z.enum(['live', 'fixture', 'replay']),
+    packageVersion: z.string(),
+    recipe: z.string(),
+    recipeFingerprint: z.string(),
+    datasetFingerprint: z.string(),
+    inputFingerprint: z.string(),
+    answerKeyFingerprint: z.string(),
+    split: z.enum(['development', 'held-out']),
+    requestedModel: z.string(),
+    policy: evaluationPolicySchema,
+    price: priceSchema.optional(),
+    cases: z.array(evaluationCaseSchema).min(1),
+    rows: z.array(z
+        .object({
+        id: z.string(),
+        expected: z.record(z.string(), z.unknown()),
+        contested: z.boolean(),
+        adversarial: z.boolean(),
+        actual: z.record(z.string(), z.unknown()).optional(),
+        result: z.unknown().optional(),
+        error: z.string().optional(),
+        confidence: z.number().min(0).max(1).optional(),
+        model: z.string().optional(),
+        correct: z.boolean(),
+        ready: z.boolean(),
+        durationMs: z.number().nonnegative(),
+        decisions: z.array(z.object({
+            minConfidence: z.number().min(0).max(1),
+            ready: z.boolean(),
+            correct: z.boolean(),
+        })),
+        exchanges: z.array(z.object({
+            request: z.record(z.string(), z.unknown()),
+            response: z.unknown().optional(),
+            error: z.string().optional(),
+            durationMs: z.number().nonnegative(),
+        })),
+    })
+        .passthrough()),
+    thresholds: z.array(z.number()),
+    sourceRun: z.string().optional(),
+    report: z.record(z.string(), z.unknown()),
+});
+export async function evaluate(name, values, options = {}) {
+    const settings = evaluationOptionsSchema.parse(options);
+    const recipe = await loadEvaluationRecipe(name);
+    const allCases = validateCases(recipe, values);
+    const cases = allCases.filter((entry) => entry.split === settings.split);
+    if (!cases.length)
+        throw new Error(`No ${settings.split} cases in the dataset.`);
+    if (cases.length > settings.maxCases)
+        throw new Error(`${cases.length} cases exceed maxCases=${settings.maxCases}. Set an explicit larger limit.`);
+    const sourceFingerprint = await recipeFingerprint(recipe.id);
+    if (settings.split === 'held-out') {
+        const chosen = settings.policy.selectedOn;
+        if (!chosen)
+            throw new Error('Held-out evaluation requires a policy frozen on a development run.');
+        if (chosen.recipeFingerprint !== sourceFingerprint ||
+            chosen.model !== (settings.model ?? 'provider-default'))
+            throw new Error('The frozen policy belongs to a different recipe version or requested model.');
+    }
+    const manifest = JSON.parse(await readFile(new URL('../../package.json', import.meta.url), 'utf8'));
+    const metadata = {
+        format: 1,
+        runId: randomUUID(),
+        createdAt: new Date().toISOString(),
+        mode: options.mode ?? 'live',
+        packageVersion: manifest.version,
+        recipe: recipe.id,
+        recipeFingerprint: sourceFingerprint,
+        ...datasetFingerprints(cases),
+        split: settings.split,
+        requestedModel: settings.model ?? 'provider-default',
+        policy: settings.policy,
+        ...(settings.price ? { price: settings.price } : {}),
+        cases,
+        thresholds: settings.split === 'development' && Object.hasOwn(recipe.inputSchema.shape, 'minConfidence')
+            ? confidenceThresholds
+            : [],
+    };
+    if (settings.policy.minConfidence !== undefined &&
+        !Object.hasOwn(recipe.inputSchema.shape, 'minConfidence'))
+        throw new Error('This recipe has no minConfidence policy.');
+    if (options.out)
+        await startArchive(options.out, metadata);
+    const rows = await evaluateCases(recipe, applyPolicy(cases, settings.policy), {
+        concurrency: settings.concurrency,
+        maxRequests: settings.maxRequests,
+        thresholds: metadata.thresholds,
+        ...(options.client ? { client: options.client } : {}),
+        ...(settings.model ? { model: settings.model } : {}),
+        ...(options.out
+            ? {
+                onCase: (row, index) => writeJson(join(options.out, 'cases', `${String(index).padStart(6, '0')}.json`), row),
+            }
+            : {}),
+    });
+    const run = {
+        ...metadata,
+        rows,
+        report: buildReport(recipe.id, rows, metadata.thresholds, settings.price, reportEvidence(metadata)),
+    };
+    if (options.out)
+        await finishArchive(options.out, run);
+    return run;
+}
+export async function replay(run, policy = run.policy, out) {
+    const settings = evaluationPolicySchema.parse(policy);
+    const recipe = await loadEvaluationRecipe(run.recipe);
+    if ((await recipeFingerprint(run.recipe)) !== run.recipeFingerprint)
+        throw new Error('The installed recipe differs from this archive. Replay requires the recorded recipe version.');
+    if (run.split === 'held-out' && fingerprint(settings) !== fingerprint(run.policy))
+        throw new Error('Do not tune a policy on held-out data. Replay a development run to choose a policy.');
+    if (settings.minConfidence !== undefined &&
+        !Object.hasOwn(recipe.inputSchema.shape, 'minConfidence'))
+        throw new Error('This recipe has no minConfidence policy.');
+    const cases = validateCases(recipe, run.cases);
+    const metadata = {
+        ...run,
+        runId: randomUUID(),
+        createdAt: new Date().toISOString(),
+        mode: 'replay',
+        sourceRun: run.runId,
+        policy: settings,
+    };
+    if (out)
+        await startArchive(out, { ...metadata, rows: undefined, report: undefined });
+    const rows = await evaluateCases(recipe, applyPolicy(cases, settings), {
+        recorded: run.rows,
+        thresholds: run.thresholds,
+        ...(run.requestedModel === 'provider-default' ? {} : { model: run.requestedModel }),
+    });
+    const result = {
+        ...metadata,
+        rows,
+        report: buildReport(run.recipe, rows, run.thresholds, run.price, reportEvidence(metadata)),
+    };
+    if (out)
+        await finishArchive(out, result);
+    return result;
+}
+export async function readRun(directory) {
+    const content = await readFile(join(directory, 'run.json'), 'utf8').catch(async (error) => {
+        if (error.code !== 'ENOENT')
+            throw error;
+        return gunzipSync(await readFile(join(directory, 'run.json.gz'))).toString('utf8');
+    });
+    const parsed = archiveSchema.parse(JSON.parse(content));
+    const run = parsed;
+    const hashes = datasetFingerprints(run.cases);
+    if (Object.entries(hashes).some(([key, hash]) => run[key] !== hash))
+        throw new Error('Archive dataset fingerprints do not match its cases.');
+    if (run.rows.length !== run.cases.length ||
+        new Set(run.rows.map((row) => row.id)).size !== run.cases.length ||
+        run.cases.some((entry) => !run.rows.some((row) => row.id === entry.id && fingerprint(row.expected) === fingerprint(entry.expected))))
+        throw new Error('Archive case records are incomplete.');
+    return {
+        ...run,
+        report: buildReport(run.recipe, run.rows, run.thresholds, run.price, reportEvidence(run)),
+    };
+}
+export function freezePolicy(run) {
+    if (run.split !== 'development')
+        throw new Error('Choose the policy on a development run.');
+    return {
+        ...run.policy,
+        selectedOn: {
+            runId: run.runId,
+            datasetFingerprint: run.datasetFingerprint,
+            recipeFingerprint: run.recipeFingerprint,
+            model: run.requestedModel,
+            split: 'development',
+        },
+    };
+}
+function reportEvidence(run) {
+    const provenance = new Map();
+    for (const entry of run.cases) {
+        const key = JSON.stringify(entry.provenance);
+        const counted = provenance.get(key) ?? { ...entry.provenance, cases: 0 };
+        counted.cases++;
+        provenance.set(key, counted);
+    }
+    return {
+        runId: run.runId,
+        evaluatedAt: run.createdAt,
+        packageVersion: run.packageVersion,
+        recipeFingerprint: run.recipeFingerprint,
+        datasetFingerprint: run.datasetFingerprint,
+        split: run.split,
+        mode: run.mode,
+        policy: run.policy,
+        provenance: [...provenance.values()],
+    };
+}
+export function compare(baseline, candidate) {
+    if (baseline.recipe !== candidate.recipe ||
+        baseline.inputFingerprint !== candidate.inputFingerprint ||
+        baseline.answerKeyFingerprint !== candidate.answerKeyFingerprint)
+        throw new Error('Comparison requires the same recipe, case IDs, inputs, split, and answer key.');
+    const delta = (first, second) => first === null || second === null ? null : Number((second - first).toFixed(6));
+    const describe = (run) => ({
+        runId: run.runId,
+        packageVersion: run.packageVersion,
+        recipeFingerprint: run.recipeFingerprint,
+        requestedModel: run.requestedModel,
+        models: run.report.models,
+        policy: run.policy,
+        mode: run.mode,
+        report: run.report,
+    });
+    return {
+        recipe: baseline.recipe,
+        cases: baseline.cases.length,
+        split: baseline.split,
+        baseline: describe(baseline),
+        candidate: describe(candidate),
+        delta: {
+            accuracy: delta(baseline.report.accuracy, candidate.report.accuracy),
+            readyAccuracy: delta(baseline.report.readyAccuracy, candidate.report.readyAccuracy),
+            review: candidate.report.review - baseline.report.review,
+            failed: candidate.report.failed - baseline.report.failed,
+        },
+        changed: candidate.rows.flatMap((row) => {
+            const before = baseline.rows.find((entry) => entry.id === row.id);
+            return before.correct !== row.correct ||
+                before.ready !== row.ready ||
+                before.error !== row.error
+                ? [
+                    {
+                        id: row.id,
+                        before: { correct: before.correct, ready: before.ready, error: before.error },
+                        after: { correct: row.correct, ready: row.ready, error: row.error },
+                    },
+                ]
+                : [];
+        }),
+    };
+}
+function applyPolicy(cases, policy) {
+    return cases.map((entry) => ({
+        ...entry,
+        input: {
+            ...entry.input,
+            ...(policy.minConfidence === undefined ? {} : { minConfidence: policy.minConfidence }),
+        },
+    }));
+}
+async function startArchive(directory, metadata) {
+    await mkdir(dirname(directory), { recursive: true });
+    await mkdir(directory, { mode: 0o700 });
+    await mkdir(join(directory, 'cases'), { mode: 0o700 });
+    await writeJson(join(directory, 'manifest.json'), { status: 'running', metadata });
+}
+async function finishArchive(directory, run) {
+    await writeJson(join(directory, 'run.json'), run);
+    await writeJson(join(directory, 'report.json'), run.report);
+    await writeJson(join(directory, 'policy.json'), run.split === 'development' ? freezePolicy(run) : run.policy);
+    await writeFile(join(directory, 'manifest.json'), JSON.stringify({ status: 'complete', runId: run.runId }, null, 2) + '\n', { mode: 0o600 });
+}
+async function writeJson(path, value) {
+    await writeFile(path, JSON.stringify(value, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
+}

package/dist/evaluation/dataset.d.ts

@@ -0,0 +1,12 @@
+import type { EvaluationCase } from './schema.js';
+import type { EvaluationRecipe } from './engine.js';
+export declare function loadEvaluationRecipe(name: string): Promise<EvaluationRecipe>;
+export declare function readEvaluationCases(path: string): Promise<unknown[]>;
+export declare function validateCases(recipe: EvaluationRecipe, values: unknown[]): EvaluationCase[];
+export declare function fingerprint(value: unknown): string;
+export declare function datasetFingerprints(cases: EvaluationCase[]): {
+    datasetFingerprint: string;
+    inputFingerprint: string;
+    answerKeyFingerprint: string;
+};
+export declare function recipeFingerprint(id: string): Promise<string>;

package/dist/evaluation/dataset.js

@@ -0,0 +1,162 @@
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { z } from 'zod';
+import { describeRecipe } from '../catalog/index.js';
+import { recipeNameSchema } from '../catalog/schema.js';
+import { isRecord, comparableDecision } from './decisions.js';
+import { evaluationCaseSchema } from './schema.js';
+export async function loadEvaluationRecipe(name) {
+    const id = recipeNameSchema.parse(name);
+    const module = (await import(new URL(`../recipes/${id}/index.js`, import.meta.url).href));
+    const run = Object.values(module).find((value) => typeof value === 'function');
+    const inputSchema = Object.entries(module).find(([key]) => key.endsWith('InputSchema'))?.[1];
+    if (typeof run !== 'function' || !(inputSchema instanceof z.ZodObject))
+        throw new Error(`Cannot load recipe ${id}.`);
+    return { id, run: run, inputSchema };
+}
+export async function readEvaluationCases(path) {
+    const text = await readFile(path, 'utf8');
+    return text.split(/\r?\n/).flatMap((line, index) => {
+        if (!line.trim())
+            return [];
+        try {
+            return [JSON.parse(line)];
+        }
+        catch {
+            throw new Error(`Invalid JSON on line ${index + 1} of ${path}.`);
+        }
+    });
+}
+export function validateCases(recipe, values) {
+    if (!values.length)
+        throw new Error('The case file is empty.');
+    const resultSchema = describeRecipe(recipeNameSchema.parse(recipe.id)).resultSchema;
+    const cases = values.map((value) => evaluationCaseSchema.parse(value));
+    const ids = new Set();
+    const familySplits = new Map();
+    for (const entry of cases) {
+        if (ids.has(entry.id))
+            throw new Error(`Duplicate case ID: ${entry.id}`);
+        ids.add(entry.id);
+        if (entry.family) {
+            if (familySplits.has(entry.family) && familySplits.get(entry.family) !== entry.split)
+                throw new Error(`Case family ${entry.family} crosses development and held-out splits.`);
+            familySplits.set(entry.family, entry.split);
+        }
+        recipe.inputSchema.parse(entry.input);
+        for (const [path, expected] of Object.entries(entry.expected)) {
+            if (path
+                .split('.')
+                .some((segment) => [
+                'status',
+                'confidence',
+                'probabilities',
+                'probability',
+                'model',
+                'usage',
+                'score',
+            ].includes(segment)))
+                throw new Error(`${entry.id}: expected must name an ungated decision, not ${path}.`);
+            const field = schemaAtPath(resultSchema, path.split('.'));
+            if (!field)
+                throw new Error(`${entry.id}: unknown result field ${path}.`);
+            if (!acceptsValue(field, expected))
+                throw new Error(`${entry.id}: expected ${path} does not match the result schema.`);
+        }
+        if (isEmptyDecision(comparableDecision(entry.expected)))
+            throw new Error(`${entry.id}: expected must contain a decision to compare.`);
+    }
+    return cases;
+}
+function schemaAtPath(schema, segments) {
+    if (!segments.length)
+        return schema;
+    if (Array.isArray(schema.anyOf)) {
+        const candidates = schema.anyOf.flatMap((branch) => {
+            const field = isRecord(branch) ? schemaAtPath(branch, segments) : undefined;
+            return field ? [field] : [];
+        });
+        return candidates.length ? { anyOf: candidates } : undefined;
+    }
+    const [first, ...rest] = segments;
+    if (isRecord(schema.properties) && isRecord(schema.properties[first]))
+        return schemaAtPath(schema.properties[first], rest);
+    if (schema.type === 'array' && /^(0|[1-9]\d*)$/.test(first) && isRecord(schema.items))
+        return schemaAtPath(schema.items, rest);
+    if (isRecord(schema.additionalProperties))
+        return schemaAtPath(schema.additionalProperties, rest);
+    return undefined;
+}
+function acceptsValue(schema, value) {
+    if (Array.isArray(schema.anyOf))
+        return schema.anyOf.some((candidate) => isRecord(candidate) && acceptsValue(candidate, value));
+    if (Array.isArray(schema.enum) && !schema.enum.includes(value))
+        return false;
+    if (schema.const !== undefined && schema.const !== value)
+        return false;
+    if (Array.isArray(schema.type))
+        return schema.type.some((type) => acceptsValue({ ...schema, type }, value));
+    if (schema.type === 'null')
+        return value === null;
+    if (schema.type === 'array')
+        return (Array.isArray(value) &&
+            (!isRecord(schema.items) ||
+                value.every((item) => acceptsValue(schema.items, item))));
+    if (schema.type === 'object') {
+        if (!isRecord(value))
+            return false;
+        return Object.entries(value).every(([name, item]) => {
+            const field = schemaAtPath(schema, [name]);
+            return field ? acceptsValue(field, item) : schema.additionalProperties !== false;
+        });
+    }
+    if (schema.type === 'integer')
+        return typeof value === 'number' && Number.isInteger(value);
+    return schema.type === undefined || typeof value === schema.type;
+}
+function isEmptyDecision(value) {
+    if (Array.isArray(value))
+        return value.length > 0 && value.every(isEmptyDecision);
+    if (isRecord(value))
+        return Object.values(value).every(isEmptyDecision);
+    return value === undefined;
+}
+export function fingerprint(value) {
+    return createHash('sha256')
+        .update(JSON.stringify(canonicalValue(value)))
+        .digest('hex');
+}
+function canonicalValue(value) {
+    if (Array.isArray(value))
+        return value.map(canonicalValue);
+    if (isRecord(value))
+        return Object.fromEntries(Object.keys(value)
+            .sort()
+            .map((key) => [key, canonicalValue(value[key])]));
+    return value;
+}
+export function datasetFingerprints(cases) {
+    const ordered = [...cases].sort((a, b) => a.id.localeCompare(b.id));
+    return {
+        datasetFingerprint: fingerprint(ordered),
+        inputFingerprint: fingerprint(ordered.map(({ id, input, split }) => ({ id, input, split }))),
+        answerKeyFingerprint: fingerprint(ordered.map(({ id, expected }) => ({ id, expected }))),
+    };
+}
+export async function recipeFingerprint(id) {
+    recipeNameSchema.parse(id);
+    const sources = new Map();
+    const root = new URL('../', import.meta.url);
+    async function visit(url) {
+        if (sources.has(url.href))
+            return;
+        const source = await readFile(url, 'utf8');
+        sources.set(url.href, source);
+        for (const match of source.matchAll(/(?:from\s*|import\s*)['"]([^'"]+)['"]/g)) {
+            if (match[1].startsWith('.'))
+                await visit(new URL(match[1], url));
+        }
+    }
+    await visit(new URL(`recipes/${id}/index.js`, root));
+    return fingerprint(Object.fromEntries([...sources].map(([url, source]) => [url.slice(root.href.length), source])));
+}

package/dist/evaluation/decisions.d.ts

@@ -0,0 +1,10 @@
+export declare function comparableDecision(value: unknown): unknown;
+export declare function matchesExpected(actual: unknown, expected: unknown): boolean;
+export declare function isRecord(value: unknown): value is Record<string, unknown>;
+export declare function valueAtPath(record: unknown, path: string): unknown;
+export declare function pickPaths(record: unknown, paths: string[]): Record<string, unknown>;
+export declare function hasReviewAnywhere(value: unknown): boolean;
+export declare function isReady(result: unknown): boolean;
+export declare function caseConfidence(result: unknown, expectedPaths?: string[]): number | undefined;
+export declare function mapWithConcurrencyLimit<T, R>(items: T[], limit: number, worker: (item: T, index: number) => Promise<R>): Promise<R[]>;
+export declare function roundedTo(places: number, value: number): number;

package/dist/evaluation/decisions.js

@@ -0,0 +1,87 @@
+import { isDeepStrictEqual } from 'node:util';
+const metadataFields = new Set([
+    'model',
+    'usage',
+    'confidence',
+    'probability',
+    'probabilities',
+    'score',
+    'status',
+]);
+export function comparableDecision(value) {
+    if (Array.isArray(value))
+        return value.map(comparableDecision);
+    if (isRecord(value))
+        return Object.fromEntries(Object.entries(value)
+            .filter(([name, field]) => !metadataFields.has(name) && typeof field !== 'number')
+            .map(([name, field]) => [name, comparableDecision(field)]));
+    return value;
+}
+export function matchesExpected(actual, expected) {
+    return isDeepStrictEqual(comparableDecision(actual), comparableDecision(expected));
+}
+export function isRecord(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+export function valueAtPath(record, path) {
+    return path
+        .split('.')
+        .reduce((value, segment) => value !== null && typeof value === 'object' && Object.hasOwn(value, segment)
+        ? value[segment]
+        : undefined, record);
+}
+export function pickPaths(record, paths) {
+    return Object.fromEntries(paths.map((path) => [path, valueAtPath(record, path)]));
+}
+export function hasReviewAnywhere(value) {
+    if (Array.isArray(value))
+        return value.some(hasReviewAnywhere);
+    if (isRecord(value))
+        return Object.entries(value).some(([name, field]) => name === 'status' ? field === 'review' : hasReviewAnywhere(field));
+    return false;
+}
+export function isReady(result) {
+    if (!isRecord(result))
+        return false;
+    if (result.status !== undefined)
+        return result.status === 'ready';
+    return !hasReviewAnywhere(result);
+}
+export function caseConfidence(result, expectedPaths = []) {
+    if (isRecord(result) && typeof result.confidence === 'number')
+        return result.confidence;
+    const scoped = expectedPaths.flatMap((path) => {
+        const segments = path.split('.');
+        for (let end = segments.length; end > 0; end--) {
+            const node = valueAtPath(result, segments.slice(0, end).join('.'));
+            if (isRecord(node) && typeof node.confidence === 'number')
+                return [node.confidence];
+        }
+        return [];
+    });
+    const values = scoped.length ? scoped : collectConfidences(result);
+    return values.length ? Math.min(...values) : undefined;
+}
+function collectConfidences(value) {
+    if (Array.isArray(value))
+        return value.flatMap(collectConfidences);
+    if (isRecord(value))
+        return Object.entries(value).flatMap(([name, field]) => name === 'confidence' && typeof field === 'number' ? [field] : collectConfidences(field));
+    return [];
+}
+export async function mapWithConcurrencyLimit(items, limit, worker) {
+    if (!Number.isInteger(limit) || limit < 1 || limit > 32)
+        throw new Error('Concurrency must be an integer from 1 to 32.');
+    const results = new Array(items.length);
+    let nextIndex = 0;
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {
+        while (nextIndex < items.length) {
+            const index = nextIndex++;
+            results[index] = await worker(items[index], index);
+        }
+    }));
+    return results;
+}
+export function roundedTo(places, value) {
+    return Number(value.toFixed(places));
+}

package/dist/evaluation/engine.d.ts

@@ -0,0 +1,50 @@
+import type { SystemOneRequest } from '@typesafe-ai/sdk';
+import type { z } from 'zod';
+import type { RecipeOptions } from '../src/schema.js';
+import type { EvaluationCase } from './schema.js';
+export interface EvaluationRecipe {
+    id: string;
+    run: (input: unknown, options?: RecipeOptions) => Promise<unknown>;
+    inputSchema: z.ZodObject;
+}
+export interface Exchange {
+    request: SystemOneRequest;
+    response?: unknown;
+    error?: string;
+    durationMs: number;
+}
+export interface EvaluationRow {
+    id: string;
+    expected: Record<string, unknown>;
+    contested: boolean;
+    adversarial: boolean;
+    actual?: Record<string, unknown>;
+    result?: unknown;
+    error?: string;
+    confidence?: number | undefined;
+    model?: string | undefined;
+    correct: boolean;
+    ready: boolean;
+    durationMs: number;
+    exchanges: Exchange[];
+    decisions: {
+        minConfidence: number;
+        ready: boolean;
+        correct: boolean;
+    }[];
+}
+export interface EngineOptions extends RecipeOptions {
+    concurrency?: number;
+    maxRequests?: number;
+    thresholds?: number[];
+    recorded?: EvaluationRow[];
+    onCase?: (row: EvaluationRow, index: number) => Promise<void>;
+}
+export declare const confidenceThresholds: number[];
+export declare function evaluateCases(recipe: EvaluationRecipe, cases: EvaluationCase[], options?: EngineOptions): Promise<EvaluationRow[]>;
+export declare function replayResponses(exchanges: Exchange[]): {
+    client: {
+        systemOne(request: SystemOneRequest): Promise<unknown>;
+    };
+    assertComplete(): void;
+};