@outputai/evals 0.8.2-next.2da7213.0 → 0.8.2-next.ad732b1.0

package/dist/eval_workflow.js

@@ -1,17 +1,20 @@
 import { workflow, executeInParallel, z } from '@outputai/core';
-import { getMetadata } from '@outputai/core/sdk_utils';
+import { ComponentMetadata } from '@outputai/core/sdk/helpers';
 import { VERDICT, CRITICALITY, DatasetSchema, EvalOutputSchema } from './schemas.js';
 import { interpretResult } from './interpret.js';
 import { aggregateCaseVerdict } from './aggregate.js';
 export function evalWorkflow({ name, evals, fn, config = {} }) {
     const concurrency = config.concurrency ?? 10;
     const evalDefs = evals.map(def => {
-        const meta = getMetadata(def.evaluator);
-        if (!meta) {
-            throw new Error('Evaluator passed to evalWorkflow is missing metadata. Ensure it was created with evaluator().');
+        if (!ComponentMetadata.has(def.evaluator)) {
+            throw new Error('Evaluator passed to evalWorkflow was not created with evaluator().');
+        }
+        const name = ComponentMetadata.getName(def.evaluator);
+        if (!name) {
+            throw new Error('Evaluator component doesn\'t have a name.');
         }
         return {
-            name: meta.name,
+            name,
             criticality: def.criticality ?? CRITICALITY.REQUIRED,
             interpret: def.interpret
         };

package/dist/eval_workflow.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/eval_workflow.spec.js

@@ -0,0 +1,92 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { evalWorkflow } from './eval_workflow.js';
+const workflowMock = vi.hoisted(() => vi.fn(({ fn }) => fn));
+const executeInParallelMock = vi.hoisted(() => vi.fn(async ({ jobs }) => {
+    const results = await Promise.all(jobs.map(async (job) => ({ ok: true, result: await job() })));
+    return results;
+}));
+const hasMock = vi.hoisted(() => vi.fn());
+const getNameMock = vi.hoisted(() => vi.fn());
+vi.mock('@outputai/core', async (importOriginal) => {
+    const actual = await importOriginal();
+    return {
+        ...actual,
+        workflow: workflowMock,
+        executeInParallel: executeInParallelMock
+    };
+});
+vi.mock('@outputai/core/sdk/helpers', () => ({
+    ComponentMetadata: {
+        has: hasMock,
+        getName: getNameMock
+    }
+}));
+describe('evalWorkflow', () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+        hasMock.mockReturnValue(true);
+        getNameMock.mockReturnValue('quality_eval');
+    });
+    it('resolves evaluator names with ComponentMetadata.getName', async () => {
+        const evaluator = async () => ({ value: true, confidence: 1, feedback: [], dimensions: [] });
+        const workflowFn = evalWorkflow({
+            name: 'quality',
+            evals: [{
+                    evaluator,
+                    interpret: { type: 'boolean' }
+                }]
+        });
+        const invokeEvaluator = vi.fn().mockResolvedValue({ value: true, confidence: 1, feedback: [], dimensions: [] });
+        const output = await workflowFn.call({ invokeEvaluator }, {
+            datasets: [{
+                    name: 'case_1',
+                    input: { prompt: 'hello' },
+                    last_output: { output: { answer: 'hello' } },
+                    ground_truth: {
+                        shared: 'value',
+                        evals: {
+                            quality_eval: { expected: 'hello' }
+                        }
+                    }
+                }]
+        });
+        expect(hasMock).toHaveBeenCalledWith(evaluator);
+        expect(getNameMock).toHaveBeenCalledWith(evaluator);
+        expect(invokeEvaluator).toHaveBeenCalledWith('quality_eval', {
+            input: { prompt: 'hello' },
+            output: { answer: 'hello' },
+            ground_truth: { shared: 'value', expected: 'hello' }
+        });
+        expect(output.cases[0].evaluators[0]).toMatchObject({
+            name: 'quality_eval',
+            verdict: 'pass',
+            criticality: 'required'
+        });
+    });
+    it('rejects non-component evaluators', () => {
+        const evaluator = async () => ({ value: true, confidence: 1 });
+        hasMock.mockReturnValue(false);
+        expect(() => evalWorkflow({
+            name: 'quality',
+            evals: [{
+                    evaluator,
+                    interpret: { type: 'boolean' }
+                }]
+        })).toThrow('Evaluator passed to evalWorkflow was not created with evaluator().');
+        expect(hasMock).toHaveBeenCalledWith(evaluator);
+        expect(getNameMock).not.toHaveBeenCalled();
+    });
+    it('rejects component evaluators without a component name', () => {
+        const evaluator = async () => ({ value: true, confidence: 1 });
+        getNameMock.mockReturnValue(undefined);
+        expect(() => evalWorkflow({
+            name: 'quality',
+            evals: [{
+                    evaluator,
+                    interpret: { type: 'boolean' }
+                }]
+        })).toThrow('Evaluator component doesn\'t have a name.');
+        expect(hasMock).toHaveBeenCalledWith(evaluator);
+        expect(getNameMock).toHaveBeenCalledWith(evaluator);
+    });
+});

package/dist/index_sandbox.js

@@ -29,10 +29,10 @@
  * is fixed to generate output from the modified AST even when rewriteFnBodies() returns
  * false — i.e. when collectTargetImports() stripped imports but no fn bodies needed
  * rewriting. This also requires injecting metadata stubs for the stripped evaluator
- * references, since evalWorkflow() calls getMetadata(def.evaluator) at module init time
- * and the identifiers would otherwise be undefined. Once the rewriter properly strips the
- * evaluator → judge → @outputai/llm chain, this file and the "output-workflow-bundle" export
- * condition in package.json can be deleted.
+ * references, since evalWorkflow() calls ComponentMetadata.getName(def.evaluator) at
+ * module init time and the identifiers would otherwise be undefined. Once the rewriter
+ * properly strips the evaluator → judge → @outputai/llm chain, this file and the
+ * "output-workflow-bundle" export condition in package.json can be deleted.
  */
 export { evalWorkflow } from './eval_workflow.js';
 export { Verdict } from './verdict.js';

package/dist/judge.js

@@ -1,30 +1,30 @@
 import { EvaluationVerdictResult, EvaluationNumberResult, EvaluationBooleanResult, EvaluationStringResult, z } from '@outputai/core';
-import { resolveInvocationDir } from '@outputai/core/sdk_utils';
+import { Path } from '@outputai/core/sdk/helpers';
 import { generateText, Output } from '@outputai/llm';
 const verdictSchema = z.object({ verdict: z.enum(['pass', 'partial', 'fail']), reasoning: z.string() });
 const scoreSchema = z.object({ score: z.number(), reasoning: z.string() });
 const booleanSchema = z.object({ result: z.boolean(), reasoning: z.string() });
 const labelSchema = z.object({ label: z.string(), reasoning: z.string() });
 export async function judgeVerdict({ prompt, variables, schema = verdictSchema }) {
-    const promptDir = resolveInvocationDir();
+    const promptDir = Path.resolveInvocationDir();
     const response = await generateText({ prompt, variables, promptDir, output: Output.object({ schema }) });
     const result = response.output;
     return new EvaluationVerdictResult({ value: result.verdict, confidence: 0.9, reasoning: result.reasoning });
 }
 export async function judgeScore({ prompt, variables, schema = scoreSchema }) {
-    const promptDir = resolveInvocationDir();
+    const promptDir = Path.resolveInvocationDir();
     const response = await generateText({ prompt, variables, promptDir, output: Output.object({ schema }) });
     const result = response.output;
     return new EvaluationNumberResult({ value: result.score, confidence: 0.9, reasoning: result.reasoning });
 }
 export async function judgeBoolean({ prompt, variables, schema = booleanSchema }) {
-    const promptDir = resolveInvocationDir();
+    const promptDir = Path.resolveInvocationDir();
     const response = await generateText({ prompt, variables, promptDir, output: Output.object({ schema }) });
     const result = response.output;
     return new EvaluationBooleanResult({ value: result.result, confidence: 0.9, reasoning: result.reasoning });
 }
 export async function judgeLabel({ prompt, variables, schema = labelSchema }) {
-    const promptDir = resolveInvocationDir();
+    const promptDir = Path.resolveInvocationDir();
     const response = await generateText({ prompt, variables, promptDir, output: Output.object({ schema }) });
     const result = response.output;
     return new EvaluationStringResult({ value: result.label, confidence: 0.9, reasoning: result.reasoning });

package/dist/verify.spec.js

@@ -1,16 +1,8 @@
 import { describe, it, expect } from 'vitest';
 import { verify } from './verify.js';
-import { getMetadata } from '@outputai/core/sdk_utils';
 import { EvaluationBooleanResult, z } from '@outputai/core';
 import { Verdict } from './verdict.js';
 describe('verify', () => {
-    it('returns a function with metadata matching the given name', () => {
-        const ev = verify({ name: 'my_eval' }, () => Verdict.isTrue(true));
-        const meta = getMetadata(ev);
-        expect(meta).not.toBeNull();
-        expect(meta.name).toBe('my_eval');
-        expect(meta.description).toBe('my_eval');
-    });
     it('passes input and output to the user function', async () => {
         const captured = [];
         const ev = verify({ name: 'capture_test' }, ctx => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/evals",
-  "version": "0.8.2-next.2da7213.0",
+  "version": "0.8.2-next.ad732b1.0",
   "description": "Offline evaluation framework for Output.ai workflows",
   "type": "module",
   "main": "dist/index.js",
@@ -16,8 +16,8 @@
     "./dist"
   ],
   "dependencies": {
-    "@outputai/core": "0.8.2-next.2da7213.0",
-    "@outputai/llm": "0.8.2-next.2da7213.0"
+    "@outputai/core": "0.8.2-next.ad732b1.0",
+    "@outputai/llm": "0.8.2-next.ad732b1.0"
   },
   "license": "Apache-2.0",
   "publishConfig": {