@outputai/evals 0.1.4 → 0.1.5

package/dist/render.spec.js

@@ -0,0 +1,171 @@
+import { describe, it, expect } from 'vitest';
+import { renderEvalOutput, computeExitCode } from './render.js';
+const makeOutput = (cases, overrides) => {
+    const passed = cases.filter(c => c.verdict === 'pass').length;
+    const partial = cases.filter(c => c.verdict === 'partial').length;
+    const failed = cases.filter(c => c.verdict === 'fail').length;
+    const total = cases.length;
+    return {
+        cases,
+        summary: {
+            total,
+            passed,
+            partial,
+            failed,
+            acceptableRate: total > 0 ? (passed + partial) / total : 0,
+            ...overrides
+        }
+    };
+};
+describe('computeExitCode', () => {
+    it('returns 0 when all cases pass', () => {
+        const output = makeOutput([
+            { datasetName: 'a', verdict: 'pass', evaluators: [] }
+        ]);
+        expect(computeExitCode(output)).toBe(0);
+    });
+    it('returns 0 when cases are partial but none fail', () => {
+        const output = makeOutput([
+            { datasetName: 'a', verdict: 'partial', evaluators: [] }
+        ]);
+        expect(computeExitCode(output)).toBe(0);
+    });
+    it('returns 1 when any case fails', () => {
+        const output = makeOutput([
+            { datasetName: 'a', verdict: 'pass', evaluators: [] },
+            { datasetName: 'b', verdict: 'fail', evaluators: [] }
+        ]);
+        expect(computeExitCode(output)).toBe(1);
+    });
+    it('returns 0 for empty cases', () => {
+        const output = makeOutput([]);
+        expect(computeExitCode(output)).toBe(0);
+    });
+});
+describe('renderEvalOutput', () => {
+    it('includes eval name when provided', () => {
+        const output = makeOutput([
+            { datasetName: 'test', verdict: 'pass', evaluators: [] }
+        ]);
+        const rendered = renderEvalOutput(output, 'my_eval');
+        expect(rendered).toContain('my_eval');
+    });
+    it('includes dataset names', () => {
+        const output = makeOutput([
+            { datasetName: 'basic_input', verdict: 'pass', evaluators: [] },
+            { datasetName: 'edge_case', verdict: 'fail', evaluators: [] }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('basic_input');
+        expect(rendered).toContain('edge_case');
+    });
+    it('includes verdict labels', () => {
+        const output = makeOutput([
+            { datasetName: 'a', verdict: 'pass', evaluators: [] },
+            { datasetName: 'b', verdict: 'fail', evaluators: [] }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('PASS');
+        expect(rendered).toContain('FAIL');
+    });
+    it('shows reasoning for failed evaluators', () => {
+        const output = makeOutput([
+            {
+                datasetName: 'test',
+                verdict: 'fail',
+                evaluators: [{
+                        name: 'check',
+                        verdict: 'fail',
+                        criticality: 'required',
+                        reasoning: 'Expected 15, got null'
+                    }]
+            }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('Expected 15, got null');
+    });
+    it('hides reasoning for passing evaluators', () => {
+        const output = makeOutput([
+            {
+                datasetName: 'test',
+                verdict: 'pass',
+                evaluators: [{
+                        name: 'check',
+                        verdict: 'pass',
+                        criticality: 'required',
+                        reasoning: 'All good'
+                    }]
+            }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).not.toContain('All good');
+    });
+    it('shows summary with acceptable rate', () => {
+        const output = makeOutput([
+            { datasetName: 'a', verdict: 'pass', evaluators: [] },
+            { datasetName: 'b', verdict: 'fail', evaluators: [] }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('1 passed');
+        expect(rendered).toContain('1 failed');
+        expect(rendered).toContain('50% acceptable');
+    });
+    it('includes PARTIAL verdict label', () => {
+        const output = makeOutput([
+            { datasetName: 'a', verdict: 'partial', evaluators: [] }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('PARTIAL');
+    });
+    it('shows reasoning for partial evaluators', () => {
+        const output = makeOutput([
+            {
+                datasetName: 'test',
+                verdict: 'partial',
+                evaluators: [{
+                        name: 'check',
+                        verdict: 'partial',
+                        criticality: 'required',
+                        reasoning: 'Close but not quite'
+                    }]
+            }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('Close but not quite');
+    });
+    it('renders feedback issues', () => {
+        const output = makeOutput([
+            {
+                datasetName: 'test',
+                verdict: 'fail',
+                evaluators: [{
+                        name: 'check',
+                        verdict: 'fail',
+                        criticality: 'required',
+                        feedback: [
+                            { issue: 'Missing required field' },
+                            { issue: 'Invalid format' }
+                        ]
+                    }]
+            }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('Missing required field');
+        expect(rendered).toContain('Invalid format');
+    });
+    it('marks informational evaluators with (info) prefix', () => {
+        const output = makeOutput([
+            {
+                datasetName: 'test',
+                verdict: 'pass',
+                evaluators: [{
+                        name: 'info_check',
+                        verdict: 'fail',
+                        criticality: 'informational'
+                    }]
+            }
+        ]);
+        const rendered = renderEvalOutput(output);
+        expect(rendered).toContain('(info)');
+    });
+});

package/dist/schemas.d.ts

@@ -0,0 +1,95 @@
+import { z } from '@outputai/core';
+export declare const VERDICT: {
+    readonly PASS: "pass";
+    readonly PARTIAL: "partial";
+    readonly FAIL: "fail";
+};
+export declare const CRITICALITY: {
+    readonly REQUIRED: "required";
+    readonly INFORMATIONAL: "informational";
+};
+export declare const VerdictSchema: z.ZodEnum<{
+    pass: "pass";
+    partial: "partial";
+    fail: "fail";
+}>;
+export type Verdict = z.infer<typeof VerdictSchema>;
+export declare const CriticalitySchema: z.ZodEnum<{
+    required: "required";
+    informational: "informational";
+}>;
+export type Criticality = z.infer<typeof CriticalitySchema>;
+export type InterpretConfig = {
+    type: 'verdict';
+} | {
+    type: 'boolean';
+} | {
+    type: 'number';
+    pass: number;
+    partial?: number;
+} | {
+    type: 'string';
+    pass: string[];
+    partial?: string[];
+};
+export declare const InterpretConfigSchema: z.ZodType<InterpretConfig>;
+export interface EvaluatorResult {
+    name: string;
+    verdict: Verdict;
+    criticality: Criticality;
+    confidence?: number;
+    reasoning?: string;
+    feedback?: unknown[];
+}
+export declare const EvaluatorResultSchema: z.ZodType<EvaluatorResult>;
+export interface EvalCase {
+    datasetName: string;
+    verdict: Verdict;
+    evaluators: EvaluatorResult[];
+}
+export declare const EvalCaseSchema: z.ZodType<EvalCase>;
+export interface EvalSummary {
+    total: number;
+    passed: number;
+    partial: number;
+    failed: number;
+    acceptableRate: number;
+}
+export interface EvalOutput {
+    cases: EvalCase[];
+    summary: EvalSummary;
+}
+export declare const EvalOutputSchema: z.ZodType<EvalOutput>;
+export interface GroundTruth {
+    evals?: Record<string, Record<string, unknown>>;
+    [key: string]: unknown;
+}
+export interface LastOutput {
+    output: unknown;
+    executionTimeMs?: number;
+    date: string;
+}
+export interface LastEval {
+    output: EvalCase;
+    executionTimeMs?: number;
+    date: string;
+}
+export declare const LastOutputSchema: z.ZodType<LastOutput>;
+export declare const LastEvalSchema: z.ZodType<LastEval>;
+export interface Dataset {
+    name: string;
+    input: Record<string, unknown>;
+    ground_truth?: GroundTruth;
+    last_output?: LastOutput;
+    last_eval?: LastEval;
+    [key: string]: unknown;
+}
+export declare const DatasetSchema: z.ZodObject<{
+    name: z.ZodString;
+    input: z.ZodRecord<z.ZodString, z.ZodAny>;
+    ground_truth: z.ZodOptional<z.ZodObject<{
+        evals: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodAny>>>;
+    }, z.core.$loose>>;
+    last_output: z.ZodOptional<z.ZodType<LastOutput, unknown, z.core.$ZodTypeInternals<LastOutput, unknown>>>;
+    last_eval: z.ZodOptional<z.ZodType<LastEval, unknown, z.core.$ZodTypeInternals<LastEval, unknown>>>;
+}, z.core.$loose>;

package/dist/schemas.js

@@ -0,0 +1,60 @@
+import { z } from '@outputai/core';
+export const VERDICT = {
+    PASS: 'pass',
+    PARTIAL: 'partial',
+    FAIL: 'fail'
+};
+export const CRITICALITY = {
+    REQUIRED: 'required',
+    INFORMATIONAL: 'informational'
+};
+export const VerdictSchema = z.enum([VERDICT.PASS, VERDICT.PARTIAL, VERDICT.FAIL]);
+export const CriticalitySchema = z.enum([CRITICALITY.REQUIRED, CRITICALITY.INFORMATIONAL]);
+export const InterpretConfigSchema = z.discriminatedUnion('type', [
+    z.object({ type: z.literal('verdict') }),
+    z.object({ type: z.literal('boolean') }),
+    z.object({ type: z.literal('number'), pass: z.number(), partial: z.number().optional() }),
+    z.object({ type: z.literal('string'), pass: z.array(z.string()), partial: z.array(z.string()).optional() })
+]);
+export const EvaluatorResultSchema = z.object({
+    name: z.string(),
+    verdict: VerdictSchema,
+    criticality: CriticalitySchema,
+    confidence: z.number().optional(),
+    reasoning: z.string().optional(),
+    feedback: z.array(z.any()).optional()
+});
+export const EvalCaseSchema = z.object({
+    datasetName: z.string(),
+    verdict: VerdictSchema,
+    evaluators: z.array(EvaluatorResultSchema)
+});
+export const EvalOutputSchema = z.object({
+    cases: z.array(EvalCaseSchema),
+    summary: z.object({
+        total: z.number(),
+        passed: z.number(),
+        partial: z.number(),
+        failed: z.number(),
+        acceptableRate: z.number()
+    })
+});
+export const LastOutputSchema = z.object({
+    output: z.any(),
+    executionTimeMs: z.number().optional(),
+    date: z.string()
+});
+export const LastEvalSchema = z.object({
+    output: EvalCaseSchema,
+    executionTimeMs: z.number().optional(),
+    date: z.string()
+});
+export const DatasetSchema = z.object({
+    name: z.string(),
+    input: z.record(z.string(), z.any()),
+    ground_truth: z.object({
+        evals: z.record(z.string(), z.record(z.string(), z.any())).optional()
+    }).passthrough().optional(),
+    last_output: LastOutputSchema.optional(),
+    last_eval: LastEvalSchema.optional()
+}).passthrough();

package/dist/verdict.d.ts

@@ -0,0 +1,27 @@
+import { EvaluationVerdictResult, EvaluationBooleanResult, EvaluationNumberResult, EvaluationStringResult, EvaluationFeedback } from '@outputai/core';
+type FeedbackArg = ConstructorParameters<typeof EvaluationFeedback>[0];
+export declare const Verdict: {
+    pass(reasoning?: string): EvaluationVerdictResult;
+    partial(confidence: number, reasoning?: string, feedback?: FeedbackArg[]): EvaluationVerdictResult;
+    fail(reasoning: string, feedback?: FeedbackArg[]): EvaluationVerdictResult;
+    equals(actual: unknown, expected: unknown): EvaluationBooleanResult;
+    closeTo(actual: number, expected: number, tolerance: number): EvaluationBooleanResult;
+    gt(actual: number, threshold: number): EvaluationBooleanResult;
+    gte(actual: number, threshold: number): EvaluationBooleanResult;
+    lt(actual: number, threshold: number): EvaluationBooleanResult;
+    lte(actual: number, threshold: number): EvaluationBooleanResult;
+    inRange(actual: number, min: number, max: number): EvaluationBooleanResult;
+    contains(haystack: string, needle: string): EvaluationBooleanResult;
+    matches(value: string, pattern: RegExp): EvaluationBooleanResult;
+    includesAll(actual: unknown[], expected: unknown[]): EvaluationBooleanResult;
+    includesAny(actual: unknown[], expected: unknown[]): EvaluationBooleanResult;
+    isTrue(value: boolean): EvaluationBooleanResult;
+    isFalse(value: boolean): EvaluationBooleanResult;
+    fromJudge({ verdict, reasoning }: {
+        verdict: "pass" | "partial" | "fail";
+        reasoning: string;
+    }): EvaluationVerdictResult;
+    score(value: number, reasoning?: string): EvaluationNumberResult;
+    label(value: string, reasoning?: string): EvaluationStringResult;
+};
+export {};

package/dist/verdict.js

@@ -0,0 +1,109 @@
+import { EvaluationVerdictResult, EvaluationBooleanResult, EvaluationNumberResult, EvaluationStringResult, EvaluationFeedback } from '@outputai/core';
+const boolResult = (value, reasoning) => new EvaluationBooleanResult({ value, confidence: 1.0, reasoning });
+export const Verdict = {
+    // --- Original helpers ---
+    pass(reasoning) {
+        return new EvaluationVerdictResult({ value: 'pass', confidence: 1.0, reasoning });
+    },
+    partial(confidence, reasoning, feedback) {
+        return new EvaluationVerdictResult({
+            value: 'partial',
+            confidence,
+            reasoning,
+            feedback: feedback ? feedback.map(f => new EvaluationFeedback(f)) : undefined
+        });
+    },
+    fail(reasoning, feedback) {
+        return new EvaluationVerdictResult({
+            value: 'fail',
+            confidence: 0.0,
+            reasoning,
+            feedback: feedback ? feedback.map(f => new EvaluationFeedback(f)) : undefined
+        });
+    },
+    // --- Deterministic assertions (confidence 1.0 for both pass and fail) ---
+    equals(actual, expected) {
+        const passed = actual === expected;
+        return boolResult(passed, passed ?
+            `Value equals expected: ${JSON.stringify(expected)}` :
+            `Expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
+    },
+    closeTo(actual, expected, tolerance) {
+        const passed = Math.abs(actual - expected) <= tolerance;
+        return boolResult(passed, passed ?
+            `${actual} is within ${tolerance} of ${expected}` :
+            `${actual} is not within ${tolerance} of ${expected} (diff: ${Math.abs(actual - expected)})`);
+    },
+    gt(actual, threshold) {
+        const passed = actual > threshold;
+        return boolResult(passed, passed ?
+            `${actual} > ${threshold}` :
+            `${actual} is not greater than ${threshold}`);
+    },
+    gte(actual, threshold) {
+        const passed = actual >= threshold;
+        return boolResult(passed, passed ?
+            `${actual} >= ${threshold}` :
+            `${actual} is not greater than or equal to ${threshold}`);
+    },
+    lt(actual, threshold) {
+        const passed = actual < threshold;
+        return boolResult(passed, passed ?
+            `${actual} < ${threshold}` :
+            `${actual} is not less than ${threshold}`);
+    },
+    lte(actual, threshold) {
+        const passed = actual <= threshold;
+        return boolResult(passed, passed ?
+            `${actual} <= ${threshold}` :
+            `${actual} is not less than or equal to ${threshold}`);
+    },
+    inRange(actual, min, max) {
+        const passed = actual >= min && actual <= max;
+        return boolResult(passed, passed ?
+            `${actual} is in range [${min}, ${max}]` :
+            `${actual} is not in range [${min}, ${max}]`);
+    },
+    contains(haystack, needle) {
+        const passed = haystack.includes(needle);
+        return boolResult(passed, passed ?
+            `String contains "${needle}"` :
+            `String does not contain "${needle}"`);
+    },
+    matches(value, pattern) {
+        const passed = pattern.test(value);
+        return boolResult(passed, passed ?
+            `Value matches ${pattern}` :
+            `Value does not match ${pattern}`);
+    },
+    includesAll(actual, expected) {
+        const missing = expected.filter(e => !actual.includes(e));
+        const passed = missing.length === 0;
+        return boolResult(passed, passed ?
+            'Array includes all expected values' :
+            `Array is missing: ${JSON.stringify(missing)}`);
+    },
+    includesAny(actual, expected) {
+        const found = expected.filter(e => actual.includes(e));
+        const passed = found.length > 0;
+        return boolResult(passed, passed ?
+            `Array includes: ${JSON.stringify(found)}` :
+            `Array includes none of: ${JSON.stringify(expected)}`);
+    },
+    isTrue(value) {
+        return boolResult(value === true, value === true ? 'Value is true' : `Expected true, got ${value}`);
+    },
+    isFalse(value) {
+        return boolResult(value === false, value === false ? 'Value is false' : `Expected false, got ${value}`);
+    },
+    // --- LLM judge helpers ---
+    fromJudge({ verdict, reasoning }) {
+        return new EvaluationVerdictResult({ value: verdict, confidence: 0.9, reasoning });
+    },
+    score(value, reasoning) {
+        return new EvaluationNumberResult({ value, confidence: 0.9, reasoning });
+    },
+    label(value, reasoning) {
+        return new EvaluationStringResult({ value, confidence: 0.9, reasoning });
+    }
+};

package/dist/verdict.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/verdict.spec.js

@@ -0,0 +1,181 @@
+import { describe, it, expect } from 'vitest';
+import { Verdict } from './verdict.js';
+import { EvaluationBooleanResult, EvaluationVerdictResult, EvaluationNumberResult, EvaluationStringResult } from '@outputai/core';
+describe('Verdict', () => {
+    // --- Original helpers regression ---
+    describe('pass', () => {
+        it('returns EvaluationVerdictResult with value "pass"', () => {
+            const r = Verdict.pass('looks good');
+            expect(r).toBeInstanceOf(EvaluationVerdictResult);
+            expect(r.value).toBe('pass');
+            expect(r.confidence).toBe(1.0);
+            expect(r.reasoning).toBe('looks good');
+        });
+    });
+    describe('partial', () => {
+        it('returns EvaluationVerdictResult with value "partial"', () => {
+            const r = Verdict.partial(0.7, 'mostly ok');
+            expect(r).toBeInstanceOf(EvaluationVerdictResult);
+            expect(r.value).toBe('partial');
+            expect(r.confidence).toBe(0.7);
+        });
+    });
+    describe('fail', () => {
+        it('returns EvaluationVerdictResult with value "fail"', () => {
+            const r = Verdict.fail('wrong');
+            expect(r).toBeInstanceOf(EvaluationVerdictResult);
+            expect(r.value).toBe('fail');
+            expect(r.confidence).toBe(0.0);
+        });
+    });
+    // --- Deterministic assertions ---
+    describe('equals', () => {
+        it('passes when values are strictly equal', () => {
+            const r = Verdict.equals(42, 42);
+            expect(r).toBeInstanceOf(EvaluationBooleanResult);
+            expect(r.value).toBe(true);
+            expect(r.confidence).toBe(1.0);
+        });
+        it('fails when values differ', () => {
+            const r = Verdict.equals(42, 43);
+            expect(r.value).toBe(false);
+            expect(r.confidence).toBe(1.0);
+        });
+    });
+    describe('closeTo', () => {
+        it('passes within tolerance', () => {
+            const r = Verdict.closeTo(3.14, 3.15, 0.02);
+            expect(r.value).toBe(true);
+        });
+        it('fails outside tolerance', () => {
+            const r = Verdict.closeTo(3.14, 3.20, 0.02);
+            expect(r.value).toBe(false);
+        });
+    });
+    describe('gt', () => {
+        it('passes when actual > threshold', () => {
+            expect(Verdict.gt(5, 3).value).toBe(true);
+        });
+        it('fails when actual <= threshold', () => {
+            expect(Verdict.gt(3, 3).value).toBe(false);
+        });
+    });
+    describe('gte', () => {
+        it('passes when actual >= threshold', () => {
+            expect(Verdict.gte(3, 3).value).toBe(true);
+        });
+        it('fails when actual < threshold', () => {
+            expect(Verdict.gte(2, 3).value).toBe(false);
+        });
+    });
+    describe('lt', () => {
+        it('passes when actual < threshold', () => {
+            expect(Verdict.lt(2, 3).value).toBe(true);
+        });
+        it('fails when actual >= threshold', () => {
+            expect(Verdict.lt(3, 3).value).toBe(false);
+        });
+    });
+    describe('lte', () => {
+        it('passes when actual <= threshold', () => {
+            expect(Verdict.lte(3, 3).value).toBe(true);
+        });
+        it('fails when actual > threshold', () => {
+            expect(Verdict.lte(4, 3).value).toBe(false);
+        });
+    });
+    describe('inRange', () => {
+        it('passes when value is in range', () => {
+            expect(Verdict.inRange(5, 1, 10).value).toBe(true);
+        });
+        it('passes at boundaries', () => {
+            expect(Verdict.inRange(1, 1, 10).value).toBe(true);
+            expect(Verdict.inRange(10, 1, 10).value).toBe(true);
+        });
+        it('fails when value is out of range', () => {
+            expect(Verdict.inRange(11, 1, 10).value).toBe(false);
+        });
+    });
+    describe('contains', () => {
+        it('passes when haystack includes needle', () => {
+            expect(Verdict.contains('hello world', 'world').value).toBe(true);
+        });
+        it('fails when haystack does not include needle', () => {
+            expect(Verdict.contains('hello world', 'xyz').value).toBe(false);
+        });
+    });
+    describe('matches', () => {
+        it('passes when value matches pattern', () => {
+            expect(Verdict.matches('abc123', /\d+/).value).toBe(true);
+        });
+        it('fails when value does not match', () => {
+            expect(Verdict.matches('abc', /\d+/).value).toBe(false);
+        });
+    });
+    describe('includesAll', () => {
+        it('passes when all expected values are present', () => {
+            expect(Verdict.includesAll([1, 2, 3], [1, 3]).value).toBe(true);
+        });
+        it('fails when some expected values are missing', () => {
+            const r = Verdict.includesAll([1, 2], [2, 3]);
+            expect(r.value).toBe(false);
+            expect(r.reasoning).toContain('3');
+        });
+    });
+    describe('includesAny', () => {
+        it('passes when at least one expected value is present', () => {
+            expect(Verdict.includesAny([1, 2, 3], [5, 2]).value).toBe(true);
+        });
+        it('fails when no expected values are present', () => {
+            expect(Verdict.includesAny([1, 2], [5, 6]).value).toBe(false);
+        });
+    });
+    describe('isTrue', () => {
+        it('passes for true', () => {
+            expect(Verdict.isTrue(true).value).toBe(true);
+        });
+        it('fails for false', () => {
+            expect(Verdict.isTrue(false).value).toBe(false);
+        });
+    });
+    describe('isFalse', () => {
+        it('passes for false', () => {
+            expect(Verdict.isFalse(false).value).toBe(true);
+        });
+        it('fails for true', () => {
+            expect(Verdict.isFalse(true).value).toBe(false);
+        });
+    });
+    // --- LLM judge helpers ---
+    describe('fromJudge', () => {
+        it('returns EvaluationVerdictResult with 0.9 confidence', () => {
+            const r = Verdict.fromJudge({ verdict: 'pass', reasoning: 'on topic' });
+            expect(r).toBeInstanceOf(EvaluationVerdictResult);
+            expect(r.value).toBe('pass');
+            expect(r.confidence).toBe(0.9);
+            expect(r.reasoning).toBe('on topic');
+        });
+        it('handles all verdict values', () => {
+            expect(Verdict.fromJudge({ verdict: 'partial', reasoning: 'ok' }).value).toBe('partial');
+            expect(Verdict.fromJudge({ verdict: 'fail', reasoning: 'bad' }).value).toBe('fail');
+        });
+    });
+    describe('score', () => {
+        it('returns EvaluationNumberResult with 0.9 confidence', () => {
+            const r = Verdict.score(0.85, 'good quality');
+            expect(r).toBeInstanceOf(EvaluationNumberResult);
+            expect(r.value).toBe(0.85);
+            expect(r.confidence).toBe(0.9);
+            expect(r.reasoning).toBe('good quality');
+        });
+    });
+    describe('label', () => {
+        it('returns EvaluationStringResult with 0.9 confidence', () => {
+            const r = Verdict.label('formal', 'professional writing');
+            expect(r).toBeInstanceOf(EvaluationStringResult);
+            expect(r.value).toBe('formal');
+            expect(r.confidence).toBe(0.9);
+            expect(r.reasoning).toBe('professional writing');
+        });
+    });
+});

package/dist/verify.d.ts

@@ -0,0 +1,41 @@
+import { z } from '@outputai/core';
+import type { EvaluationResult } from '@outputai/core';
+export type CheckContext<TInput = unknown, TOutput = unknown> = {
+    input: TInput;
+    output: TOutput;
+    context: {
+        ground_truth: Record<string, unknown>;
+    };
+};
+type CheckFn<TInput, TOutput> = (ctx: CheckContext<TInput, TOutput>) => EvaluationResult | Promise<EvaluationResult>;
+type VerifyOptions<I extends z.ZodType = z.ZodType<unknown>, O extends z.ZodType = z.ZodType<unknown>> = {
+    name: string;
+    input?: I;
+    output?: O;
+};
+export declare const verify: <I extends z.ZodType, O extends z.ZodType>(options: VerifyOptions<I, O>, fn: CheckFn<z.infer<I>, z.infer<O>>) => (input: ({
+    input: z.ZodAny | I;
+    output: z.ZodAny | O;
+    ground_truth: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+} extends infer T_1 extends z.core.$ZodLooseShape ? { -readonly [k in keyof T_1 as {
+    input: z.ZodAny | I;
+    output: z.ZodAny | O;
+    ground_truth: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}[k] extends {
+    _zod: {
+        optout: "optional";
+    };
+} ? never : k]: T_1[k]["_zod"]["output"]; } : never) & ({
+    input: z.ZodAny | I;
+    output: z.ZodAny | O;
+    ground_truth: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+} extends infer T_2 extends z.core.$ZodLooseShape ? { -readonly [k_1 in keyof T_2 as {
+    input: z.ZodAny | I;
+    output: z.ZodAny | O;
+    ground_truth: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}[k_1] extends {
+    _zod: {
+        optout: "optional";
+    };
+} ? k_1 : never]?: T_2[k_1]["_zod"]["output"] | undefined; } : never) extends infer T ? { [K in keyof T]: T[K]; } : never) => Promise<EvaluationResult>;
+export {};