@outputai/evals 0.10.0 → 0.10.1-next.09ed166.0

package/dist/eval_workflow.js

@@ -20,14 +20,14 @@ export function evalWorkflow({ name, evals, fn, config = {} }) {
         };
     });
     const defaultFn = async function (input) {
-        const { invokeEvaluator } = this;
+        const invoke = globalThis[Symbol.for('@outputai/core:__invoke_activity')];
         const jobs = input.datasets.map((dataset) => async () => {
             const evaluatorResults = [];
             for (const evalDef of evalDefs) {
                 const { evals: perEvalTruth, ...globalTruth } = dataset.ground_truth ?? {};
                 const evalTruth = perEvalTruth?.[evalDef.name] ?? {};
                 const mergedTruth = { ...globalTruth, ...evalTruth };
-                const result = await invokeEvaluator(evalDef.name, {
+                const result = await invoke(evalDef.name, {
                     input: dataset.input,
                     output: dataset.last_output?.output,
                     ground_truth: mergedTruth

package/dist/eval_workflow.spec.js

@@ -1,5 +1,6 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest';
 import { evalWorkflow } from './eval_workflow.js';
+const INVOKE_ACTIVITY_SYMBOL = Symbol.for('@outputai/core:__invoke_activity');
 const workflowMock = vi.hoisted(() => vi.fn(({ fn }) => fn));
 const executeInParallelMock = vi.hoisted(() => vi.fn(async ({ jobs }) => {
     const results = await Promise.all(jobs.map(async (job) => ({ ok: true, result: await job() })));
@@ -24,6 +25,7 @@ vi.mock('@outputai/core/sdk/helpers', () => ({
 describe('evalWorkflow', () => {
     beforeEach(() => {
         vi.clearAllMocks();
+        delete globalThis[INVOKE_ACTIVITY_SYMBOL];
         hasMock.mockReturnValue(true);
         getNameMock.mockReturnValue('quality_eval');
     });
@@ -36,8 +38,9 @@ describe('evalWorkflow', () => {
                     interpret: { type: 'boolean' }
                 }]
         });
-        const invokeEvaluator = vi.fn().mockResolvedValue({ value: true, confidence: 1, feedback: [], dimensions: [] });
-        const output = await workflowFn.call({ invokeEvaluator }, {
+        const invokeActivity = vi.fn().mockResolvedValue({ value: true, confidence: 1, feedback: [], dimensions: [] });
+        globalThis[INVOKE_ACTIVITY_SYMBOL] = invokeActivity;
+        const output = await workflowFn({
             datasets: [{
                     name: 'case_1',
                     input: { prompt: 'hello' },
@@ -52,7 +55,7 @@ describe('evalWorkflow', () => {
         });
         expect(hasMock).toHaveBeenCalledWith(evaluator);
         expect(getNameMock).toHaveBeenCalledWith(evaluator);
-        expect(invokeEvaluator).toHaveBeenCalledWith('quality_eval', {
+        expect(invokeActivity).toHaveBeenCalledWith('quality_eval', {
             input: { prompt: 'hello' },
             output: { answer: 'hello' },
             ground_truth: { shared: 'value', expected: 'hello' }

package/dist/index_sandbox.js

@@ -9,8 +9,8 @@
  * The webpack rewriter (workflow_rewriter/index.mjs) is supposed to strip evaluator
  * imports from the AST via collectTargetImports(), which calls path.remove() on each
  * matched import declaration. However, after collecting imports, the rewriter checks
- * if rewriteFnBodies() performed any rewrites. For eval workflows — which have no
- * fn body, only a config object — rewriteFnBodies() returns false. When that happens,
+ * if rewriteActivityCalls() performed any rewrites. For eval workflows — which have no
+ * function-body activity calls, only a config object — rewriteActivityCalls() returns false. When that happens,
  * the rewriter returns the original source string unchanged (lines 46-48), discarding
  * the import stripping that collectTargetImports() already applied to the AST.
  *
@@ -26,8 +26,8 @@
  *
  * Remove when:
  * The workflow rewriter in sdk/core/src/worker/webpack_loaders/workflow_rewriter/index.mjs
- * is fixed to generate output from the modified AST even when rewriteFnBodies() returns
- * false — i.e. when collectTargetImports() stripped imports but no fn bodies needed
+ * is fixed to generate output from the modified AST even when rewriteActivityCalls() returns
+ * false — i.e. when collectTargetImports() stripped imports but no activity calls needed
  * rewriting. This also requires injecting metadata stubs for the stripped evaluator
  * references, since evalWorkflow() calls ComponentMetadata.getName(def.evaluator) at
  * module init time and the identifiers would otherwise be undefined. Once the rewriter

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/evals",
-  "version": "0.10.0",
+  "version": "0.10.1-next.09ed166.0",
   "description": "Offline evaluation framework for Output.ai workflows",
   "type": "module",
   "main": "dist/index.js",
@@ -19,8 +19,8 @@
     "./dist"
   ],
   "dependencies": {
-    "@outputai/core": "0.10.0",
-    "@outputai/llm": "0.10.0"
+    "@outputai/core": "0.10.1-next.09ed166.0",
+    "@outputai/llm": "0.10.1-next.09ed166.0"
   },
   "license": "Apache-2.0",
   "publishConfig": {