@outputai/cli 0.8.2-next.e1cd79b.0 → 0.8.2-next.ec4c07d.0

package/dist/commands/workflow/run.js

@@ -1,7 +1,5 @@
 import { Args, Command, Flags } from '@oclif/core';
 import { postWorkflowRun } from '#api/generated/api.js';
-import { OUTPUT_FORMAT } from '#utils/constants.js';
-import { formatOutput } from '#utils/output_formatter.js';
 import { formatWorkflowResult, ERROR_STATUSES } from '#utils/format_workflow_result.js';
 import { handleApiError } from '#utils/error_handler.js';
 import { resolveInput } from '#utils/resolve_input.js';
@@ -30,9 +28,10 @@ async function executeWorkflow(args) {
 }
 export default class WorkflowRun extends Command {
     static description = 'Execute a workflow synchronously and wait for completion';
+    static enableJsonFlag = true;
     static examples = [
         '<%= config.bin %> <%= command.id %> simple basic_input',
-        '<%= config.bin %> <%= command.id %> simple my_scenario --format json',
+        '<%= config.bin %> <%= command.id %> simple my_scenario --json',
         '<%= config.bin %> <%= command.id %> simple --input \'{"values":[1,2,3]}\'',
         '<%= config.bin %> <%= command.id %> simple --input input.json',
         '<%= config.bin %> <%= command.id %> simple --input \'{"key":"value"}\' --catalog my-catalog'
@@ -60,17 +59,11 @@ export default class WorkflowRun extends Command {
             deprecateAliases: true,
             description: 'Catalog name for workflow execution (defaults to OUTPUT_CATALOG_ID)',
             env: 'OUTPUT_CATALOG_ID'
-        }),
-        format: Flags.string({
-            char: 'f',
-            description: 'Output format',
-            options: [OUTPUT_FORMAT.JSON, OUTPUT_FORMAT.TEXT],
-            default: OUTPUT_FORMAT.TEXT
         })
     };
     async run() {
         const { args, flags } = await this.parse(WorkflowRun);
-        const input = await resolveInput(args.workflowName, args.scenario, flags.input, 'run');
+        const input = await resolveInput(args.workflowName, args.scenario, flags.input, 'run', flags.catalog);
         this.log(`Executing workflow: ${args.workflowName}...`);
         const response = await executeWorkflow({
             body: {
@@ -85,11 +78,11 @@ export default class WorkflowRun extends Command {
             this.error('API returned invalid response', { exit: 1 });
         }
         const data = response.data;
-        const output = formatOutput(data, flags.format, formatWorkflowResult);
-        this.log(`\n${output}`);
+        this.log(`\n${formatWorkflowResult(data)}`);
         if (ERROR_STATUSES.has(data.status)) {
             process.exitCode = 1;
         }
+        return data;
     }
     async catch(error) {
         return handleApiError(error, (...args) => this.error(...args), {

package/dist/commands/workflow/run.spec.js

@@ -26,13 +26,11 @@ describe('workflow run command', () => {
             expect(WorkflowRun.description).toContain('Execute a workflow');
             expect(WorkflowRun.args).toHaveProperty('workflowName');
             expect(WorkflowRun.flags).toHaveProperty('input');
-            expect(WorkflowRun.flags).toHaveProperty('format');
             expect(WorkflowRun.flags).toHaveProperty('catalog');
         });
         it('should have correct flag configuration', async () => {
             const WorkflowRun = (await import('./run.js')).default;
-            expect(WorkflowRun.flags.format.options).toEqual(['json', 'text']);
-            expect(WorkflowRun.flags.format.default).toBe('text');
+            expect(WorkflowRun.enableJsonFlag).toBe(true);
             expect(WorkflowRun.flags.input.required).toBe(false);
         });
         it('should have optional scenario argument', async () => {
@@ -66,11 +64,28 @@ describe('workflow run command', () => {
                 headers: new Headers()
             });
             await cmd.run();
+            expect(resolveInput).toHaveBeenCalledWith('my_workflow', undefined, undefined, 'run', undefined);
             expect(postWorkflowRun).toHaveBeenCalledTimes(1);
             expect(postWorkflowRun).toHaveBeenCalledWith({ workflowName: 'my_workflow', input: { key: 'value' }, catalog: undefined }, expect.objectContaining({ config: { timeout: 600000 } }));
             expect(cmd.log).toHaveBeenCalledWith('Executing workflow: my_workflow...');
             expect(cmd.log).toHaveBeenCalledWith(expect.stringMatching(/\n/));
         });
+        it('threads the resolved catalog to resolveInput and postWorkflowRun', async () => {
+            const { cmd, postWorkflowRun, resolveInput } = await createCommand();
+            cmd.parse = vi.fn().mockResolvedValue({
+                args: { workflowName: 'my_workflow', scenario: 'basic' },
+                flags: { input: undefined, catalog: 'my-catalog', format: 'text' }
+            });
+            resolveInput.mockResolvedValue({ key: 'value' });
+            postWorkflowRun.mockResolvedValue({
+                data: { status: 'completed', result: {} },
+                status: 200,
+                headers: new Headers()
+            });
+            await cmd.run();
+            expect(resolveInput).toHaveBeenCalledWith('my_workflow', 'basic', undefined, 'run', 'my-catalog');
+            expect(postWorkflowRun).toHaveBeenCalledWith(expect.objectContaining({ catalog: 'my-catalog' }), expect.anything());
+        });
         it('retries when response has Retry-After and succeeds on second attempt', async () => {
             const { cmd, postWorkflowRun, resolveInput } = await createCommand();
             resolveInput.mockResolvedValue({});

package/dist/commands/workflow/runs/list.d.ts

@@ -1,6 +1,8 @@
 import { Command } from '@oclif/core';
+import { type WorkflowRun } from '#services/workflow_runs.js';
 export default class WorkflowRunsList extends Command {
     static description: string;
+    static enableJsonFlag: boolean;
     static examples: string[];
     static args: {
         workflowName: import("@oclif/core/interfaces").Arg<string | undefined, Record<string, unknown>>;
@@ -10,6 +12,6 @@ export default class WorkflowRunsList extends Command {
         limit: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
         format: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
     };
-    run(): Promise<void>;
+    run(): Promise<WorkflowRun[]>;
     catch(error: Error): Promise<void>;
 }

package/dist/commands/workflow/runs/list.js

@@ -5,7 +5,6 @@ import { formatDate, formatDurationFromTimestamps } from '#utils/date_formatter.
 import { handleApiError } from '#utils/error_handler.js';
 const OUTPUT_FORMAT = {
     TABLE: 'table',
-    JSON: 'json',
     TEXT: 'text'
 };
 function createRunsTable(runs) {
@@ -37,13 +36,7 @@ function formatRunsAsText(runs) {
         return `${run.workflowId} (${run.workflowType}) - ${run.status} [${duration}]`;
     }).join('\n');
 }
-function formatRunsAsJson(runs) {
-    return JSON.stringify(runs, null, 2);
-}
 function formatRuns(runs, format) {
-    if (format === OUTPUT_FORMAT.JSON) {
-        return formatRunsAsJson(runs);
-    }
     if (format === OUTPUT_FORMAT.TABLE) {
         return createRunsTable(runs);
     }
@@ -51,12 +44,13 @@ function formatRuns(runs, format) {
 }
 export default class WorkflowRunsList extends Command {
     static description = 'List workflow runs with optional filtering by workflow type';
+    static enableJsonFlag = true;
     static examples = [
         '<%= config.bin %> <%= command.id %>',
         '<%= config.bin %> <%= command.id %> simple',
         '<%= config.bin %> <%= command.id %> simple --limit 10',
         '<%= config.bin %> <%= command.id %> --catalog my-catalog',
-        '<%= config.bin %> <%= command.id %> --format json',
+        '<%= config.bin %> <%= command.id %> --json',
         '<%= config.bin %> <%= command.id %> --format table'
     ];
     static args = {
@@ -78,8 +72,8 @@ export default class WorkflowRunsList extends Command {
         }),
         format: Flags.string({
             char: 'f',
-            description: 'Output format',
-            options: [OUTPUT_FORMAT.TABLE, OUTPUT_FORMAT.JSON, OUTPUT_FORMAT.TEXT],
+            description: 'Output format (use --json for JSON output)',
+            options: [OUTPUT_FORMAT.TABLE, OUTPUT_FORMAT.TEXT],
             default: OUTPUT_FORMAT.TABLE
         })
     };
@@ -90,17 +84,19 @@ export default class WorkflowRunsList extends Command {
             catalog: flags.catalog,
             limit: flags.limit
         });
+        if (this.jsonEnabled()) {
+            return runs;
+        }
         if (runs.length === 0) {
             const filterMsg = args.workflowName ? ` for workflow type "${args.workflowName}"` : '';
             this.log(`No workflow runs found${filterMsg}.`);
-            return;
+            return runs;
         }
         const output = formatRuns(runs, flags.format);
         this.log(output);
-        if (flags.format !== OUTPUT_FORMAT.JSON) {
-            const filterMsg = args.workflowName ? ` of type "${args.workflowName}"` : '';
-            this.log(`\nFound ${count} run(s)${filterMsg}`);
-        }
+        const filterMsg = args.workflowName ? ` of type "${args.workflowName}"` : '';
+        this.log(`\nFound ${count} run(s)${filterMsg}`);
+        return runs;
     }
     async catch(error) {
         return handleApiError(error, (...args) => this.error(...args), {

package/dist/commands/workflow/start.js

@@ -37,7 +37,7 @@ export default class WorkflowStart extends Command {
     };
     async run() {
         const { args, flags } = await this.parse(WorkflowStart);
-        const input = await resolveInput(args.workflowName, args.scenario, flags.input, 'start');
+        const input = await resolveInput(args.workflowName, args.scenario, flags.input, 'start', flags.catalog);
         this.log(`Starting workflow: ${args.workflowName}...`);
         const response = await postWorkflowStart({
             workflowName: args.workflowName,

package/dist/commands/workflow/start.spec.js

@@ -1,11 +1,17 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
 import { describe, it, expect, vi, beforeEach } from 'vitest';
-vi.mock('../../api/generated/api.js', () => ({
+vi.mock('#api/generated/api.js', () => ({
     postWorkflowStart: vi.fn()
 }));
+vi.mock('#utils/resolve_input.js', () => ({
+    resolveInput: vi.fn()
+}));
 describe('workflow start command', () => {
-    beforeEach(() => {
+    beforeEach(async () => {
         vi.clearAllMocks();
         delete process.env.OUTPUT_CATALOG_ID;
+        const { resolveInput } = await import('#utils/resolve_input.js');
+        vi.mocked(resolveInput).mockResolvedValue({});
     });
     describe('command definition', () => {
         it('should export a valid OCLIF command', async () => {
@@ -25,5 +31,50 @@ describe('workflow start command', () => {
             expect(WorkflowStart.args).toHaveProperty('scenario');
             expect(WorkflowStart.args.scenario.required).toBe(false);
         });
+        it('binds the catalog flag to OUTPUT_CATALOG_ID', async () => {
+            const WorkflowStart = (await import('./start.js')).default;
+            expect(WorkflowStart.flags.catalog.env).toBe('OUTPUT_CATALOG_ID');
+            expect(WorkflowStart.flags.catalog.char).toBe('c');
+        });
+    });
+    describe('run()', () => {
+        const createCommand = async (flagOverrides = {}) => {
+            const WorkflowStart = (await import('./start.js')).default;
+            const { postWorkflowStart } = await import('#api/generated/api.js');
+            const { resolveInput } = await import('#utils/resolve_input.js');
+            const cmd = new WorkflowStart(['my_workflow'], {});
+            cmd.log = vi.fn();
+            cmd.error = vi.fn(() => {
+                throw new Error('error called');
+            });
+            cmd.parse = vi.fn().mockResolvedValue({
+                args: { workflowName: 'my_workflow', scenario: undefined },
+                flags: { input: undefined, catalog: undefined, ...flagOverrides }
+            });
+            return { cmd, postWorkflowStart: vi.mocked(postWorkflowStart), resolveInput: vi.mocked(resolveInput) };
+        };
+        it('threads the resolved catalog to resolveInput and postWorkflowStart', async () => {
+            const { cmd, postWorkflowStart, resolveInput } = await createCommand({ catalog: 'my-catalog' });
+            resolveInput.mockResolvedValue({ key: 'value' });
+            postWorkflowStart.mockResolvedValue({
+                data: { workflowId: 'wf-123' },
+                status: 200,
+                headers: new Headers()
+            });
+            await cmd.run();
+            expect(resolveInput).toHaveBeenCalledWith('my_workflow', undefined, undefined, 'start', 'my-catalog');
+            expect(postWorkflowStart).toHaveBeenCalledWith(expect.objectContaining({ workflowName: 'my_workflow', catalog: 'my-catalog' }));
+        });
+        it('passes undefined catalog through when none is set', async () => {
+            const { cmd, postWorkflowStart, resolveInput } = await createCommand();
+            resolveInput.mockResolvedValue({});
+            postWorkflowStart.mockResolvedValue({
+                data: { workflowId: 'wf-123' },
+                status: 200,
+                headers: new Headers()
+            });
+            await cmd.run();
+            expect(resolveInput).toHaveBeenCalledWith('my_workflow', undefined, undefined, 'start', undefined);
+        });
     });
 });

package/dist/commands/workflow/status.d.ts

@@ -1,13 +1,12 @@
 import { Command } from '@oclif/core';
+import { WorkflowStatusResponse } from '#api/generated/api.js';
 export default class WorkflowStatus extends Command {
     static description: string;
+    static enableJsonFlag: boolean;
     static examples: string[];
     static args: {
         workflowId: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
     };
-    static flags: {
-        format: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
-    };
-    run(): Promise<void>;
+    run(): Promise<WorkflowStatusResponse>;
     catch(error: Error): Promise<void>;
 }

package/dist/commands/workflow/status.js

@@ -1,14 +1,27 @@
-import { Args, Command, Flags } from '@oclif/core';
+import { Args, Command } from '@oclif/core';
 import { getWorkflowIdStatus } from '#api/generated/api.js';
-import { OUTPUT_FORMAT } from '#utils/constants.js';
-import { formatOutput } from '#utils/output_formatter.js';
 import { handleApiError } from '#utils/error_handler.js';
 import { normalizeWorkflowStatus } from '#utils/normalize_workflow_status.js';
+function formatStatusText(result) {
+    const lines = [
+        `Workflow ID: ${result.workflowId || 'unknown'}`,
+        `Status: ${result.status || 'unknown'}`,
+        ''
+    ];
+    if (result.startedAt) {
+        lines.push(`Started At: ${new Date(result.startedAt).toISOString()}`);
+    }
+    if (result.completedAt) {
+        lines.push(`Completed At: ${new Date(result.completedAt).toISOString()}`);
+    }
+    return lines.join('\n');
+}
 export default class WorkflowStatus extends Command {
     static description = 'Get workflow execution status';
+    static enableJsonFlag = true;
     static examples = [
         '<%= config.bin %> <%= command.id %> wf-12345',
-        '<%= config.bin %> <%= command.id %> wf-12345 --format json'
+        '<%= config.bin %> <%= command.id %> wf-12345 --json'
     ];
     static args = {
         workflowId: Args.string({
@@ -16,16 +29,8 @@ export default class WorkflowStatus extends Command {
             required: true
         })
     };
-    static flags = {
-        format: Flags.string({
-            char: 'f',
-            description: 'Output format',
-            options: [OUTPUT_FORMAT.JSON, OUTPUT_FORMAT.TEXT],
-            default: OUTPUT_FORMAT.TEXT
-        })
-    };
     async run() {
-        const { args, flags } = await this.parse(WorkflowStatus);
+        const { args } = await this.parse(WorkflowStatus);
         this.log(`Fetching status for workflow: ${args.workflowId}...`);
         const response = await getWorkflowIdStatus(args.workflowId);
         if (!response || !response.data) {
@@ -36,23 +41,8 @@ export default class WorkflowStatus extends Command {
             ...rawData,
             status: normalizeWorkflowStatus(rawData.status)
         };
-        const output = formatOutput(data, flags.format, (result) => {
-            const lines = [
-                `Workflow ID: ${result.workflowId || 'unknown'}`,
-                `Status: ${result.status || 'unknown'}`,
-                ''
-            ];
-            if (result.startedAt) {
-                const startDate = new Date(result.startedAt);
-                lines.push(`Started At: ${startDate.toISOString()}`);
-            }
-            if (result.completedAt) {
-                const completedDate = new Date(result.completedAt);
-                lines.push(`Completed At: ${completedDate.toISOString()}`);
-            }
-            return lines.join('\n');
-        });
-        this.log(`\n${output}`);
+        this.log(`\n${formatStatusText(data)}`);
+        return data;
     }
     async catch(error) {
         return handleApiError(error, (...args) => this.error(...args), {

package/dist/commands/workflow/status.spec.js

@@ -22,12 +22,10 @@ describe('workflow status command', () => {
             expect(WorkflowStatus).toBeDefined();
             expect(WorkflowStatus.description).toContain('Get workflow execution status');
             expect(WorkflowStatus.args).toHaveProperty('workflowId');
-            expect(WorkflowStatus.flags).toHaveProperty('format');
         });
-        it('should have correct flag configuration', async () => {
+        it('enables the built-in --json flag', async () => {
             const WorkflowStatus = (await import('./status.js')).default;
-            expect(WorkflowStatus.flags.format.options).toEqual(['json', 'text']);
-            expect(WorkflowStatus.flags.format.default).toBe('text');
+            expect(WorkflowStatus.enableJsonFlag).toBe(true);
         });
     });
 });

package/dist/commands/workflow/test_eval.d.ts

@@ -1,18 +1,20 @@
 import { Command } from '@oclif/core';
+import type { EvalOutput } from '@outputai/evals';
 export default class WorkflowTest extends Command {
     static aliases: string[];
     static description: string;
+    static enableJsonFlag: boolean;
     static examples: string[];
     static args: {
         workflowName: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
     };
     static flags: {
+        catalog: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         cached: import("@oclif/core/interfaces").BooleanFlag<boolean>;
         save: import("@oclif/core/interfaces").BooleanFlag<boolean>;
         dataset: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
-        format: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
     };
-    run(): Promise<void>;
+    run(): Promise<EvalOutput>;
     private ensureEvalWorkflowRegistered;
     private validateDatasets;
     private runWorkflowForDatasets;

package/dist/commands/workflow/test_eval.js

@@ -9,12 +9,13 @@ import { getEvalWorkflowName, renderEvalOutput, computeExitCode, EvalOutputSchem
 export default class WorkflowTest extends Command {
     static aliases = ['workflow:test'];
     static description = 'Run evaluations against a workflow using its datasets';
+    static enableJsonFlag = true;
     static examples = [
         '<%= config.bin %> <%= command.id %> simple',
         '<%= config.bin %> <%= command.id %> simple --cached',
         '<%= config.bin %> <%= command.id %> simple --save',
         '<%= config.bin %> <%= command.id %> simple --dataset basic_input,edge_case',
-        '<%= config.bin %> <%= command.id %> simple --format json'
+        '<%= config.bin %> <%= command.id %> simple --json'
     ];
     static args = {
         workflowName: Args.string({
@@ -23,6 +24,14 @@ export default class WorkflowTest extends Command {
         })
     };
     static flags = {
+        catalog: Flags.string({
+            char: 'c',
+            aliases: ['task-queue'],
+            charAliases: ['q'],
+            deprecateAliases: true,
+            description: 'Catalog name for workflow execution (defaults to OUTPUT_CATALOG_ID)',
+            env: 'OUTPUT_CATALOG_ID'
+        }),
         cached: Flags.boolean({
             description: 'Use cached output from dataset files (skip workflow execution)',
             default: false,
@@ -36,19 +45,13 @@ export default class WorkflowTest extends Command {
         dataset: Flags.string({
             description: 'Comma-separated list of dataset names to run',
             char: 'd'
-        }),
-        format: Flags.string({
-            char: 'f',
-            description: 'Output format',
-            options: ['json', 'text'],
-            default: 'text'
         })
     };
     async run() {
         const { args, flags } = await this.parse(WorkflowTest);
         const filterNames = flags.dataset?.split(',').map(s => s.trim());
         const evalName = getEvalWorkflowName(args.workflowName);
-        await this.ensureEvalWorkflowRegistered(args.workflowName, evalName);
+        await this.ensureEvalWorkflowRegistered(args.workflowName, evalName, flags.catalog);
         const { datasets, dir } = await readAllDatasets(args.workflowName, filterNames);
         if (datasets.length === 0) {
             this.error(`No datasets found for workflow "${args.workflowName}".\n` +
@@ -56,11 +59,12 @@ export default class WorkflowTest extends Command {
         }
         const preparedDatasets = flags.cached ?
             this.validateDatasets(datasets) :
-            await this.runWorkflowForDatasets(args.workflowName, datasets, flags.save, dir);
+            await this.runWorkflowForDatasets(args.workflowName, datasets, flags.save, dir, flags.catalog);
         this.log(`Running eval workflow "${evalName}"...\n`);
         const response = await postWorkflowRun({
             workflowName: evalName,
-            input: { datasets: preparedDatasets }
+            input: { datasets: preparedDatasets },
+            catalog: flags.catalog
         }, {
             config: { timeout: 600000 }
         });
@@ -72,20 +76,15 @@ export default class WorkflowTest extends Command {
         if (flags.save) {
             await this.saveEvalResults(evalOutput, preparedDatasets, dir);
         }
-        if (flags.format === 'json') {
-            this.log(JSON.stringify(evalOutput, null, 2));
-        }
-        else {
+        if (!this.jsonEnabled()) {
             this.log(renderEvalOutput(evalOutput, evalName));
         }
-        const exitCode = computeExitCode(evalOutput);
-        if (exitCode !== 0) {
-            this.exit(exitCode);
-        }
+        process.exitCode = computeExitCode(evalOutput);
+        return evalOutput;
     }
-    async ensureEvalWorkflowRegistered(workflowName, evalName) {
-        const catalog = await fetchWorkflowCatalog().catch(() => null);
-        if (catalog && !catalog.some(w => w.name === evalName)) {
+    async ensureEvalWorkflowRegistered(workflowName, evalName, catalog) {
+        const workflows = await fetchWorkflowCatalog(catalog).catch(() => null);
+        if (workflows && !workflows.some(w => w.name === evalName)) {
             this.error(await diagnoseMissingEvalWorkflow(workflowName), { exit: 1 });
         }
     }
@@ -98,7 +97,7 @@ export default class WorkflowTest extends Command {
         }
         return datasets;
     }
-    async runWorkflowForDatasets(workflowName, datasets, save, dir) {
+    async runWorkflowForDatasets(workflowName, datasets, save, dir, catalog) {
         this.log(`Running workflow "${workflowName}" for ${datasets.length} dataset(s)...\n`);
         const results = [];
         for (const dataset of datasets) {
@@ -106,7 +105,8 @@ export default class WorkflowTest extends Command {
             const startMs = Date.now();
             const response = await postWorkflowRun({
                 workflowName,
-                input: dataset.input
+                input: dataset.input,
+                catalog
             }, {
                 config: { timeout: 600000 }
             });

package/dist/commands/workflow/test_eval.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/commands/workflow/test_eval.spec.js

@@ -0,0 +1,121 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { getEvalWorkflowName, renderEvalOutput } from '@outputai/evals';
+vi.mock('#api/generated/api.js', () => ({
+    postWorkflowRun: vi.fn()
+}));
+vi.mock('#api/workflow_catalog.js', () => ({
+    fetchWorkflowCatalog: vi.fn()
+}));
+vi.mock('#services/datasets.js', () => ({
+    readAllDatasets: vi.fn(),
+    writeDataset: vi.fn()
+}));
+vi.mock('#utils/eval_diagnostics.js', () => ({
+    diagnoseMissingEvalWorkflow: vi.fn().mockResolvedValue('missing eval workflow')
+}));
+const passingOutput = {
+    cases: [{ datasetName: 'd1', verdict: 'pass', evaluators: [] }],
+    summary: { total: 1, passed: 1, partial: 0, failed: 0, acceptableRate: 1 }
+};
+const failingOutput = {
+    cases: [{ datasetName: 'd1', verdict: 'fail', evaluators: [] }],
+    summary: { total: 1, passed: 0, partial: 0, failed: 1, acceptableRate: 0 }
+};
+describe('workflow test command', () => {
+    const exitState = { original: undefined };
+    beforeEach(async () => {
+        vi.clearAllMocks();
+        exitState.original = process.exitCode;
+        process.exitCode = undefined;
+        const { readAllDatasets } = await import('#services/datasets.js');
+        const { fetchWorkflowCatalog } = await import('#api/workflow_catalog.js');
+        vi.mocked(readAllDatasets).mockResolvedValue({
+            datasets: [{ name: 'd1', input: {}, last_output: { output: {}, date: '2026-01-01' } }],
+            dir: '/tmp/datasets'
+        });
+        // Catalog includes both eval names so ensureEvalWorkflowRegistered passes deterministically.
+        vi.mocked(fetchWorkflowCatalog).mockResolvedValue([
+            { name: getEvalWorkflowName('simple') },
+            { name: getEvalWorkflowName('my_workflow') }
+        ]);
+    });
+    afterEach(() => {
+        process.exitCode = exitState.original;
+    });
+    describe('command definition', () => {
+        it('enables the built-in --json flag', async () => {
+            const WorkflowTest = (await import('./test_eval.js')).default;
+            expect(WorkflowTest.enableJsonFlag).toBe(true);
+        });
+        it('binds the catalog flag to OUTPUT_CATALOG_ID', async () => {
+            const WorkflowTest = (await import('./test_eval.js')).default;
+            expect(WorkflowTest.flags).toHaveProperty('catalog');
+            expect(WorkflowTest.flags.catalog.env).toBe('OUTPUT_CATALOG_ID');
+            expect(WorkflowTest.flags.catalog.char).toBe('c');
+        });
+    });
+    describe('run()', () => {
+        const createCommand = async (jsonEnabled) => {
+            const WorkflowTest = (await import('./test_eval.js')).default;
+            const { postWorkflowRun } = await import('#api/generated/api.js');
+            const cmd = new WorkflowTest(['simple'], {});
+            cmd.log = vi.fn();
+            cmd.jsonEnabled = vi.fn().mockReturnValue(jsonEnabled);
+            cmd.parse = vi.fn().mockResolvedValue({
+                args: { workflowName: 'simple' },
+                flags: { cached: true, save: false, dataset: undefined }
+            });
+            return { cmd, postWorkflowRun: vi.mocked(postWorkflowRun) };
+        };
+        it('sets a non-zero exit code and returns the eval output when a case fails', async () => {
+            const { cmd, postWorkflowRun } = await createCommand(false);
+            postWorkflowRun.mockResolvedValue({ data: { output: failingOutput } });
+            const result = await cmd.run();
+            expect(result).toEqual(failingOutput);
+            expect(process.exitCode).toBe(1);
+        });
+        it('leaves the exit code at zero and returns the eval output when all cases pass', async () => {
+            const { cmd, postWorkflowRun } = await createCommand(false);
+            postWorkflowRun.mockResolvedValue({ data: { output: passingOutput } });
+            const result = await cmd.run();
+            expect(result).toEqual(passingOutput);
+            expect(process.exitCode).toBe(0);
+        });
+        it('renders the human-readable summary in text mode', async () => {
+            const { cmd, postWorkflowRun } = await createCommand(false);
+            postWorkflowRun.mockResolvedValue({ data: { output: passingOutput } });
+            await cmd.run();
+            const rendered = renderEvalOutput(passingOutput, getEvalWorkflowName('simple'));
+            expect(cmd.log).toHaveBeenCalledWith(rendered);
+        });
+        it('suppresses the rendered summary in JSON mode but still returns and sets exit code', async () => {
+            const { cmd, postWorkflowRun } = await createCommand(true);
+            postWorkflowRun.mockResolvedValue({ data: { output: failingOutput } });
+            const result = await cmd.run();
+            const rendered = renderEvalOutput(failingOutput, getEvalWorkflowName('simple'));
+            expect(cmd.log).not.toHaveBeenCalledWith(rendered);
+            expect(result).toEqual(failingOutput);
+            expect(process.exitCode).toBe(1);
+        });
+        it('routes registration, dataset runs, and the eval run to the resolved catalog', async () => {
+            const WorkflowTest = (await import('./test_eval.js')).default;
+            const { postWorkflowRun } = await import('#api/generated/api.js');
+            const { fetchWorkflowCatalog } = await import('#api/workflow_catalog.js');
+            const cmd = new WorkflowTest(['my_workflow'], {});
+            cmd.log = vi.fn();
+            cmd.jsonEnabled = vi.fn().mockReturnValue(false);
+            cmd.parse = vi.fn().mockResolvedValue({
+                args: { workflowName: 'my_workflow' },
+                flags: { catalog: 'my-catalog', cached: false, save: false, dataset: undefined }
+            });
+            vi.mocked(postWorkflowRun)
+                .mockResolvedValueOnce({ data: { output: {} }, status: 200, headers: new Headers() })
+                .mockResolvedValueOnce({ data: { output: passingOutput }, status: 200, headers: new Headers() });
+            await cmd.run();
+            expect(vi.mocked(fetchWorkflowCatalog)).toHaveBeenCalledWith('my-catalog');
+            expect(postWorkflowRun).toHaveBeenNthCalledWith(1, expect.objectContaining({ workflowName: 'my_workflow', catalog: 'my-catalog' }), expect.anything());
+            expect(postWorkflowRun).toHaveBeenNthCalledWith(2, expect.objectContaining({ workflowName: getEvalWorkflowName('my_workflow'), catalog: 'my-catalog' }), expect.anything());
+        });
+    });
+});

package/dist/generated/framework_version.json

@@ -1,3 +1,3 @@
 {
-    "framework": "0.8.2-next.e1cd79b.0"
+    "framework": "0.8.2-next.ec4c07d.0"
 }

package/dist/hooks/init.d.ts

@@ -2,6 +2,7 @@ import { Hook } from '@oclif/core';
 export declare const INTERACTIVE_FLAGS: string[];
 export declare const GLOBAL_FLAGS: Set<string>;
 export declare const hasInteractiveFlag: (argv: string[]) => boolean;
+export declare const hasJsonFlag: (argv: string[]) => boolean;
 export declare const stripGlobalFlags: (argv: string[]) => void;
 declare const hook: Hook<'init'>;
 export default hook;