@outputai/evals 0.1.0

package/dist/verdict.spec.js

@@ -0,0 +1,181 @@
+import { describe, it, expect } from 'vitest';
+import { Verdict } from './verdict.js';
+import { EvaluationBooleanResult, EvaluationVerdictResult, EvaluationNumberResult, EvaluationStringResult } from '@outputai/core';
+describe('Verdict', () => {
+    // --- Original helpers regression ---
+    describe('pass', () => {
+        it('returns EvaluationVerdictResult with value "pass"', () => {
+            const r = Verdict.pass('looks good');
+            expect(r).toBeInstanceOf(EvaluationVerdictResult);
+            expect(r.value).toBe('pass');
+            expect(r.confidence).toBe(1.0);
+            expect(r.reasoning).toBe('looks good');
+        });
+    });
+    describe('partial', () => {
+        it('returns EvaluationVerdictResult with value "partial"', () => {
+            const r = Verdict.partial(0.7, 'mostly ok');
+            expect(r).toBeInstanceOf(EvaluationVerdictResult);
+            expect(r.value).toBe('partial');
+            expect(r.confidence).toBe(0.7);
+        });
+    });
+    describe('fail', () => {
+        it('returns EvaluationVerdictResult with value "fail"', () => {
+            const r = Verdict.fail('wrong');
+            expect(r).toBeInstanceOf(EvaluationVerdictResult);
+            expect(r.value).toBe('fail');
+            expect(r.confidence).toBe(0.0);
+        });
+    });
+    // --- Deterministic assertions ---
+    describe('equals', () => {
+        it('passes when values are strictly equal', () => {
+            const r = Verdict.equals(42, 42);
+            expect(r).toBeInstanceOf(EvaluationBooleanResult);
+            expect(r.value).toBe(true);
+            expect(r.confidence).toBe(1.0);
+        });
+        it('fails when values differ', () => {
+            const r = Verdict.equals(42, 43);
+            expect(r.value).toBe(false);
+            expect(r.confidence).toBe(1.0);
+        });
+    });
+    describe('closeTo', () => {
+        it('passes within tolerance', () => {
+            const r = Verdict.closeTo(3.14, 3.15, 0.02);
+            expect(r.value).toBe(true);
+        });
+        it('fails outside tolerance', () => {
+            const r = Verdict.closeTo(3.14, 3.20, 0.02);
+            expect(r.value).toBe(false);
+        });
+    });
+    describe('gt', () => {
+        it('passes when actual > threshold', () => {
+            expect(Verdict.gt(5, 3).value).toBe(true);
+        });
+        it('fails when actual <= threshold', () => {
+            expect(Verdict.gt(3, 3).value).toBe(false);
+        });
+    });
+    describe('gte', () => {
+        it('passes when actual >= threshold', () => {
+            expect(Verdict.gte(3, 3).value).toBe(true);
+        });
+        it('fails when actual < threshold', () => {
+            expect(Verdict.gte(2, 3).value).toBe(false);
+        });
+    });
+    describe('lt', () => {
+        it('passes when actual < threshold', () => {
+            expect(Verdict.lt(2, 3).value).toBe(true);
+        });
+        it('fails when actual >= threshold', () => {
+            expect(Verdict.lt(3, 3).value).toBe(false);
+        });
+    });
+    describe('lte', () => {
+        it('passes when actual <= threshold', () => {
+            expect(Verdict.lte(3, 3).value).toBe(true);
+        });
+        it('fails when actual > threshold', () => {
+            expect(Verdict.lte(4, 3).value).toBe(false);
+        });
+    });
+    describe('inRange', () => {
+        it('passes when value is in range', () => {
+            expect(Verdict.inRange(5, 1, 10).value).toBe(true);
+        });
+        it('passes at boundaries', () => {
+            expect(Verdict.inRange(1, 1, 10).value).toBe(true);
+            expect(Verdict.inRange(10, 1, 10).value).toBe(true);
+        });
+        it('fails when value is out of range', () => {
+            expect(Verdict.inRange(11, 1, 10).value).toBe(false);
+        });
+    });
+    describe('contains', () => {
+        it('passes when haystack includes needle', () => {
+            expect(Verdict.contains('hello world', 'world').value).toBe(true);
+        });
+        it('fails when haystack does not include needle', () => {
+            expect(Verdict.contains('hello world', 'xyz').value).toBe(false);
+        });
+    });
+    describe('matches', () => {
+        it('passes when value matches pattern', () => {
+            expect(Verdict.matches('abc123', /\d+/).value).toBe(true);
+        });
+        it('fails when value does not match', () => {
+            expect(Verdict.matches('abc', /\d+/).value).toBe(false);
+        });
+    });
+    describe('includesAll', () => {
+        it('passes when all expected values are present', () => {
+            expect(Verdict.includesAll([1, 2, 3], [1, 3]).value).toBe(true);
+        });
+        it('fails when some expected values are missing', () => {
+            const r = Verdict.includesAll([1, 2], [2, 3]);
+            expect(r.value).toBe(false);
+            expect(r.reasoning).toContain('3');
+        });
+    });
+    describe('includesAny', () => {
+        it('passes when at least one expected value is present', () => {
+            expect(Verdict.includesAny([1, 2, 3], [5, 2]).value).toBe(true);
+        });
+        it('fails when no expected values are present', () => {
+            expect(Verdict.includesAny([1, 2], [5, 6]).value).toBe(false);
+        });
+    });
+    describe('isTrue', () => {
+        it('passes for true', () => {
+            expect(Verdict.isTrue(true).value).toBe(true);
+        });
+        it('fails for false', () => {
+            expect(Verdict.isTrue(false).value).toBe(false);
+        });
+    });
+    describe('isFalse', () => {
+        it('passes for false', () => {
+            expect(Verdict.isFalse(false).value).toBe(true);
+        });
+        it('fails for true', () => {
+            expect(Verdict.isFalse(true).value).toBe(false);
+        });
+    });
+    // --- LLM judge helpers ---
+    describe('fromJudge', () => {
+        it('returns EvaluationVerdictResult with 0.9 confidence', () => {
+            const r = Verdict.fromJudge({ verdict: 'pass', reasoning: 'on topic' });
+            expect(r).toBeInstanceOf(EvaluationVerdictResult);
+            expect(r.value).toBe('pass');
+            expect(r.confidence).toBe(0.9);
+            expect(r.reasoning).toBe('on topic');
+        });
+        it('handles all verdict values', () => {
+            expect(Verdict.fromJudge({ verdict: 'partial', reasoning: 'ok' }).value).toBe('partial');
+            expect(Verdict.fromJudge({ verdict: 'fail', reasoning: 'bad' }).value).toBe('fail');
+        });
+    });
+    describe('score', () => {
+        it('returns EvaluationNumberResult with 0.9 confidence', () => {
+            const r = Verdict.score(0.85, 'good quality');
+            expect(r).toBeInstanceOf(EvaluationNumberResult);
+            expect(r.value).toBe(0.85);
+            expect(r.confidence).toBe(0.9);
+            expect(r.reasoning).toBe('good quality');
+        });
+    });
+    describe('label', () => {
+        it('returns EvaluationStringResult with 0.9 confidence', () => {
+            const r = Verdict.label('formal', 'professional writing');
+            expect(r).toBeInstanceOf(EvaluationStringResult);
+            expect(r.value).toBe('formal');
+            expect(r.confidence).toBe(0.9);
+            expect(r.reasoning).toBe('professional writing');
+        });
+    });
+});

package/dist/verify.d.ts

@@ -0,0 +1,41 @@
+import { z } from '@outputai/core';
+import type { EvaluationResult } from '@outputai/core';
+export type CheckContext<TInput = unknown, TOutput = unknown> = {
+    input: TInput;
+    output: TOutput;
+    context: {
+        ground_truth: Record<string, unknown>;
+    };
+};
+type CheckFn<TInput, TOutput> = (ctx: CheckContext<TInput, TOutput>) => EvaluationResult | Promise<EvaluationResult>;
+type VerifyOptions<I extends z.ZodType = z.ZodType<unknown>, O extends z.ZodType = z.ZodType<unknown>> = {
+    name: string;
+    input?: I;
+    output?: O;
+};
+export declare const verify: <I extends z.ZodType, O extends z.ZodType>(options: VerifyOptions<I, O>, fn: CheckFn<z.infer<I>, z.infer<O>>) => (input: ({
+    input: z.ZodAny | I;
+    output: z.ZodAny | O;
+    ground_truth: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+} extends infer T_1 extends z.core.$ZodLooseShape ? { -readonly [k in keyof T_1 as {
+    input: z.ZodAny | I;
+    output: z.ZodAny | O;
+    ground_truth: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}[k] extends {
+    _zod: {
+        optout: "optional";
+    };
+} ? never : k]: T_1[k]["_zod"]["output"]; } : never) & ({
+    input: z.ZodAny | I;
+    output: z.ZodAny | O;
+    ground_truth: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+} extends infer T_2 extends z.core.$ZodLooseShape ? { -readonly [k_1 in keyof T_2 as {
+    input: z.ZodAny | I;
+    output: z.ZodAny | O;
+    ground_truth: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}[k_1] extends {
+    _zod: {
+        optout: "optional";
+    };
+} ? k_1 : never]?: T_2[k_1]["_zod"]["output"] | undefined; } : never) extends infer T ? { [K in keyof T]: T[K]; } : never) => Promise<EvaluationResult>;
+export {};

package/dist/verify.js

@@ -0,0 +1,18 @@
+import { evaluator, z } from '@outputai/core';
+export const verify = (options, fn) => {
+    const inputSchema = z.object({
+        input: options.input ?? z.any(),
+        output: options.output ?? z.any(),
+        ground_truth: z.record(z.string(), z.unknown()).optional()
+    });
+    const wrappedFn = async (data) => {
+        const groundTruth = data.ground_truth ?? {};
+        return fn({ input: data.input, output: data.output, context: { ground_truth: groundTruth } });
+    };
+    return evaluator({
+        name: options.name,
+        description: options.name,
+        inputSchema,
+        fn: wrappedFn
+    });
+};

package/dist/verify.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/verify.spec.js

@@ -0,0 +1,70 @@
+import { describe, it, expect } from 'vitest';
+import { verify } from './verify.js';
+import { getMetadata } from '@outputai/core/sdk_utils';
+import { EvaluationBooleanResult, z } from '@outputai/core';
+import { Verdict } from './verdict.js';
+describe('verify', () => {
+    it('returns a function with metadata matching the given name', () => {
+        const ev = verify({ name: 'my_eval' }, () => Verdict.isTrue(true));
+        const meta = getMetadata(ev);
+        expect(meta).not.toBeNull();
+        expect(meta.name).toBe('my_eval');
+        expect(meta.description).toBe('my_eval');
+    });
+    it('passes input and output to the user function', async () => {
+        const captured = [];
+        const ev = verify({ name: 'capture_test' }, ctx => {
+            captured.push(ctx);
+            return Verdict.isTrue(true);
+        });
+        await ev({
+            input: { values: [1, 2] },
+            output: { result: 3 },
+            ground_truth: { key: 'val' }
+        });
+        expect(captured).toHaveLength(1);
+        expect(captured[0].input).toEqual({ values: [1, 2] });
+        expect(captured[0].output).toEqual({ result: 3 });
+        expect(captured[0].context.ground_truth).toEqual({ key: 'val' });
+    });
+    it('defaults ground_truth to empty object when undefined', async () => {
+        const captured = [];
+        const ev = verify({ name: 'default_ground_truth_test' }, ({ context }) => {
+            captured.push(context.ground_truth);
+            return Verdict.isTrue(true);
+        });
+        await ev({ input: {}, output: {} });
+        expect(captured[0]).toEqual({});
+    });
+    it('returns EvaluationResult from user fn as-is', async () => {
+        const ev = verify({
+            name: 'return_test',
+            input: z.object({ x: z.number() }),
+            output: z.object({ result: z.number() })
+        }, ({ input, output }) => Verdict.equals(output.result, input.x));
+        const result = await ev({ input: { x: 5 }, output: { result: 5 } });
+        expect(result).toBeInstanceOf(EvaluationBooleanResult);
+        expect(result.value).toBe(true);
+        expect(result.confidence).toBe(1.0);
+    });
+    it('returns failing result when assertion fails', async () => {
+        const ev = verify({
+            name: 'fail_test',
+            input: z.object({ x: z.number() }),
+            output: z.object({ result: z.number() })
+        }, ({ input, output }) => Verdict.equals(output.result, input.x));
+        const result = await ev({ input: { x: 5 }, output: { result: 10 } });
+        expect(result).toBeInstanceOf(EvaluationBooleanResult);
+        expect(result.value).toBe(false);
+        expect(result.confidence).toBe(1.0);
+    });
+    it('provides type inference via Zod schemas without as-casts', async () => {
+        const ev = verify({
+            name: 'typed_test',
+            input: z.object({ values: z.array(z.number()) }),
+            output: z.object({ sum: z.number() })
+        }, ({ input, output }) => Verdict.equals(output.sum, input.values.reduce((a, b) => a + b, 0)));
+        const result = await ev({ input: { values: [1, 2, 3] }, output: { sum: 6 } });
+        expect(result.value).toBe(true);
+    });
+});

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@outputai/evals",
+  "version": "0.1.0",
+  "description": "Offline evaluation framework for Output.ai workflows",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "output-workflow-bundle": "./dist/index_sandbox.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "./dist"
+  ],
+  "dependencies": {
+    "@outputai/core": "0.1.0",
+    "@outputai/llm": "0.1.0"
+  },
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "rm -rf ./dist && tsc"
+  }
+}