@outputai/cli 0.8.1 → 0.8.2-next.c12766f.0

package/dist/commands/workflow/status.d.ts

@@ -1,13 +1,12 @@
 import { Command } from '@oclif/core';
+import { WorkflowStatusResponse } from '#api/generated/api.js';
 export default class WorkflowStatus extends Command {
     static description: string;
+    static enableJsonFlag: boolean;
     static examples: string[];
     static args: {
         workflowId: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
     };
-    static flags: {
-        format: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
-    };
-    run(): Promise<void>;
+    run(): Promise<WorkflowStatusResponse>;
     catch(error: Error): Promise<void>;
 }

package/dist/commands/workflow/status.js

@@ -1,14 +1,27 @@
-import { Args, Command, Flags } from '@oclif/core';
+import { Args, Command } from '@oclif/core';
 import { getWorkflowIdStatus } from '#api/generated/api.js';
-import { OUTPUT_FORMAT } from '#utils/constants.js';
-import { formatOutput } from '#utils/output_formatter.js';
 import { handleApiError } from '#utils/error_handler.js';
 import { normalizeWorkflowStatus } from '#utils/normalize_workflow_status.js';
+function formatStatusText(result) {
+    const lines = [
+        `Workflow ID: ${result.workflowId || 'unknown'}`,
+        `Status: ${result.status || 'unknown'}`,
+        ''
+    ];
+    if (result.startedAt) {
+        lines.push(`Started At: ${new Date(result.startedAt).toISOString()}`);
+    }
+    if (result.completedAt) {
+        lines.push(`Completed At: ${new Date(result.completedAt).toISOString()}`);
+    }
+    return lines.join('\n');
+}
 export default class WorkflowStatus extends Command {
     static description = 'Get workflow execution status';
+    static enableJsonFlag = true;
     static examples = [
         '<%= config.bin %> <%= command.id %> wf-12345',
-        '<%= config.bin %> <%= command.id %> wf-12345 --format json'
+        '<%= config.bin %> <%= command.id %> wf-12345 --json'
     ];
     static args = {
         workflowId: Args.string({
@@ -16,16 +29,8 @@ export default class WorkflowStatus extends Command {
             required: true
         })
     };
-    static flags = {
-        format: Flags.string({
-            char: 'f',
-            description: 'Output format',
-            options: [OUTPUT_FORMAT.JSON, OUTPUT_FORMAT.TEXT],
-            default: OUTPUT_FORMAT.TEXT
-        })
-    };
     async run() {
-        const { args, flags } = await this.parse(WorkflowStatus);
+        const { args } = await this.parse(WorkflowStatus);
         this.log(`Fetching status for workflow: ${args.workflowId}...`);
         const response = await getWorkflowIdStatus(args.workflowId);
         if (!response || !response.data) {
@@ -36,23 +41,8 @@ export default class WorkflowStatus extends Command {
             ...rawData,
             status: normalizeWorkflowStatus(rawData.status)
         };
-        const output = formatOutput(data, flags.format, (result) => {
-            const lines = [
-                `Workflow ID: ${result.workflowId || 'unknown'}`,
-                `Status: ${result.status || 'unknown'}`,
-                ''
-            ];
-            if (result.startedAt) {
-                const startDate = new Date(result.startedAt);
-                lines.push(`Started At: ${startDate.toISOString()}`);
-            }
-            if (result.completedAt) {
-                const completedDate = new Date(result.completedAt);
-                lines.push(`Completed At: ${completedDate.toISOString()}`);
-            }
-            return lines.join('\n');
-        });
-        this.log(`\n${output}`);
+        this.log(`\n${formatStatusText(data)}`);
+        return data;
     }
     async catch(error) {
         return handleApiError(error, (...args) => this.error(...args), {

package/dist/commands/workflow/status.spec.js

@@ -22,12 +22,10 @@ describe('workflow status command', () => {
             expect(WorkflowStatus).toBeDefined();
             expect(WorkflowStatus.description).toContain('Get workflow execution status');
             expect(WorkflowStatus.args).toHaveProperty('workflowId');
-            expect(WorkflowStatus.flags).toHaveProperty('format');
         });
-        it('should have correct flag configuration', async () => {
+        it('enables the built-in --json flag', async () => {
             const WorkflowStatus = (await import('./status.js')).default;
-            expect(WorkflowStatus.flags.format.options).toEqual(['json', 'text']);
-            expect(WorkflowStatus.flags.format.default).toBe('text');
+            expect(WorkflowStatus.enableJsonFlag).toBe(true);
         });
     });
 });

package/dist/commands/workflow/test_eval.d.ts

@@ -1,7 +1,9 @@
 import { Command } from '@oclif/core';
+import type { EvalOutput } from '@outputai/evals';
 export default class WorkflowTest extends Command {
     static aliases: string[];
     static description: string;
+    static enableJsonFlag: boolean;
     static examples: string[];
     static args: {
         workflowName: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
@@ -10,9 +12,8 @@ export default class WorkflowTest extends Command {
         cached: import("@oclif/core/interfaces").BooleanFlag<boolean>;
         save: import("@oclif/core/interfaces").BooleanFlag<boolean>;
         dataset: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
-        format: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
     };
-    run(): Promise<void>;
+    run(): Promise<EvalOutput>;
     private ensureEvalWorkflowRegistered;
     private validateDatasets;
     private runWorkflowForDatasets;

package/dist/commands/workflow/test_eval.js

@@ -9,12 +9,13 @@ import { getEvalWorkflowName, renderEvalOutput, computeExitCode, EvalOutputSchem
 export default class WorkflowTest extends Command {
     static aliases = ['workflow:test'];
     static description = 'Run evaluations against a workflow using its datasets';
+    static enableJsonFlag = true;
     static examples = [
         '<%= config.bin %> <%= command.id %> simple',
         '<%= config.bin %> <%= command.id %> simple --cached',
         '<%= config.bin %> <%= command.id %> simple --save',
         '<%= config.bin %> <%= command.id %> simple --dataset basic_input,edge_case',
-        '<%= config.bin %> <%= command.id %> simple --format json'
+        '<%= config.bin %> <%= command.id %> simple --json'
     ];
     static args = {
         workflowName: Args.string({
@@ -36,12 +37,6 @@ export default class WorkflowTest extends Command {
         dataset: Flags.string({
             description: 'Comma-separated list of dataset names to run',
             char: 'd'
-        }),
-        format: Flags.string({
-            char: 'f',
-            description: 'Output format',
-            options: ['json', 'text'],
-            default: 'text'
         })
     };
     async run() {
@@ -72,16 +67,11 @@ export default class WorkflowTest extends Command {
         if (flags.save) {
             await this.saveEvalResults(evalOutput, preparedDatasets, dir);
         }
-        if (flags.format === 'json') {
-            this.log(JSON.stringify(evalOutput, null, 2));
-        }
-        else {
+        if (!this.jsonEnabled()) {
             this.log(renderEvalOutput(evalOutput, evalName));
         }
-        const exitCode = computeExitCode(evalOutput);
-        if (exitCode !== 0) {
-            this.exit(exitCode);
-        }
+        process.exitCode = computeExitCode(evalOutput);
+        return evalOutput;
     }
     async ensureEvalWorkflowRegistered(workflowName, evalName) {
         const catalog = await fetchWorkflowCatalog().catch(() => null);

package/dist/commands/workflow/test_eval.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/commands/workflow/test_eval.spec.js

@@ -0,0 +1,84 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { getEvalWorkflowName, renderEvalOutput } from '@outputai/evals';
+vi.mock('#api/generated/api.js', () => ({
+    postWorkflowRun: vi.fn()
+}));
+vi.mock('#services/datasets.js', () => ({
+    readAllDatasets: vi.fn(),
+    writeDataset: vi.fn()
+}));
+const passingOutput = {
+    cases: [{ datasetName: 'd1', verdict: 'pass', evaluators: [] }],
+    summary: { total: 1, passed: 1, partial: 0, failed: 0, acceptableRate: 1 }
+};
+const failingOutput = {
+    cases: [{ datasetName: 'd1', verdict: 'fail', evaluators: [] }],
+    summary: { total: 1, passed: 0, partial: 0, failed: 1, acceptableRate: 0 }
+};
+describe('workflow test command', () => {
+    const exitState = { original: undefined };
+    beforeEach(async () => {
+        vi.clearAllMocks();
+        exitState.original = process.exitCode;
+        process.exitCode = undefined;
+        const { readAllDatasets } = await import('#services/datasets.js');
+        vi.mocked(readAllDatasets).mockResolvedValue({
+            datasets: [{ name: 'd1', input: {}, last_output: { output: {}, date: '2026-01-01' } }],
+            dir: '/tmp/datasets'
+        });
+    });
+    afterEach(() => {
+        process.exitCode = exitState.original;
+    });
+    describe('command definition', () => {
+        it('enables the built-in --json flag', async () => {
+            const WorkflowTest = (await import('./test_eval.js')).default;
+            expect(WorkflowTest.enableJsonFlag).toBe(true);
+        });
+    });
+    describe('run()', () => {
+        const createCommand = async (jsonEnabled) => {
+            const WorkflowTest = (await import('./test_eval.js')).default;
+            const { postWorkflowRun } = await import('#api/generated/api.js');
+            const cmd = new WorkflowTest(['simple'], {});
+            cmd.log = vi.fn();
+            cmd.jsonEnabled = vi.fn().mockReturnValue(jsonEnabled);
+            cmd.parse = vi.fn().mockResolvedValue({
+                args: { workflowName: 'simple' },
+                flags: { cached: true, save: false, dataset: undefined }
+            });
+            return { cmd, postWorkflowRun: vi.mocked(postWorkflowRun) };
+        };
+        it('sets a non-zero exit code and returns the eval output when a case fails', async () => {
+            const { cmd, postWorkflowRun } = await createCommand(false);
+            postWorkflowRun.mockResolvedValue({ data: { output: failingOutput } });
+            const result = await cmd.run();
+            expect(result).toEqual(failingOutput);
+            expect(process.exitCode).toBe(1);
+        });
+        it('leaves the exit code at zero and returns the eval output when all cases pass', async () => {
+            const { cmd, postWorkflowRun } = await createCommand(false);
+            postWorkflowRun.mockResolvedValue({ data: { output: passingOutput } });
+            const result = await cmd.run();
+            expect(result).toEqual(passingOutput);
+            expect(process.exitCode).toBe(0);
+        });
+        it('renders the human-readable summary in text mode', async () => {
+            const { cmd, postWorkflowRun } = await createCommand(false);
+            postWorkflowRun.mockResolvedValue({ data: { output: passingOutput } });
+            await cmd.run();
+            const rendered = renderEvalOutput(passingOutput, getEvalWorkflowName('simple'));
+            expect(cmd.log).toHaveBeenCalledWith(rendered);
+        });
+        it('suppresses the rendered summary in JSON mode but still returns and sets exit code', async () => {
+            const { cmd, postWorkflowRun } = await createCommand(true);
+            postWorkflowRun.mockResolvedValue({ data: { output: failingOutput } });
+            const result = await cmd.run();
+            const rendered = renderEvalOutput(failingOutput, getEvalWorkflowName('simple'));
+            expect(cmd.log).not.toHaveBeenCalledWith(rendered);
+            expect(result).toEqual(failingOutput);
+            expect(process.exitCode).toBe(1);
+        });
+    });
+});

package/dist/generated/framework_version.json

@@ -1,3 +1,3 @@
 {
-    "framework": "0.8.1"
+    "framework": "0.8.2-next.c12766f.0"
 }

package/dist/hooks/init.d.ts

@@ -2,6 +2,7 @@ import { Hook } from '@oclif/core';
 export declare const INTERACTIVE_FLAGS: string[];
 export declare const GLOBAL_FLAGS: Set<string>;
 export declare const hasInteractiveFlag: (argv: string[]) => boolean;
+export declare const hasJsonFlag: (argv: string[]) => boolean;
 export declare const stripGlobalFlags: (argv: string[]) => void;
 declare const hook: Hook<'init'>;
 export default hook;

package/dist/hooks/init.js

@@ -6,6 +6,10 @@ const debug = debugFactory('output-cli:init');
 export const INTERACTIVE_FLAGS = ['--yes', '--non-interactive'];
 export const GLOBAL_FLAGS = new Set(INTERACTIVE_FLAGS);
 export const hasInteractiveFlag = (argv) => argv.some(arg => INTERACTIVE_FLAGS.includes(arg));
+// The version banner must never reach stdout in JSON mode, where it would
+// corrupt the machine-readable output. oclif only suppresses `this.log` inside
+// the command, not hook output, so we detect `--json` ourselves.
+export const hasJsonFlag = (argv) => argv.includes('--json');
 export const stripGlobalFlags = (argv) => {
     const kept = argv.filter(arg => !GLOBAL_FLAGS.has(arg));
     if (kept.length !== argv.length) {
@@ -19,37 +23,43 @@ const hook = async function (opts) {
     if (interactive) {
         setNonInteractive(true);
     }
-    try {
-        // Only the local cache is read here; the registry roundtrip happens in a
-        // detached child so it never delays the invoked command.
-        const result = await readCachedResult(this.config.version, this.config.cacheDir);
-        if (!result) {
-            spawnBackgroundRefresh(this.config.version, this.config.cacheDir);
-            return;
-        }
-        if (!result.updateAvailable) {
-            return;
-        }
-        const border = ux.colorize('dim', '─'.repeat(80));
-        const warning = ux.colorize('yellow', 'Uhoh! Your Output.ai CLI is behind!');
-        const latestVer = ux.colorize('green', `v${result.latestVersion}`);
-        const currentVer = ux.colorize('yellow', `v${result.currentVersion}`);
-        const updateCmd = ux.colorize('cyan', 'npx output update');
-        ux.stdout('');
-        ux.stdout(border);
-        ux.stdout('');
-        ux.stdout(`  ⚠️  ${warning}`);
-        ux.stdout('');
-        ux.stdout(`     Latest is ${latestVer}, and you're using ${currentVer}`);
-        ux.stdout('');
-        ux.stdout(`     Run \`${updateCmd}\` to update`);
-        ux.stdout('');
-        ux.stdout(border);
-        ux.stdout('');
+    // Guard only the version-check IO: a broken or unreadable cache must never
+    // block CLI execution, so a failure is treated as "no cached result". Banner
+    // rendering below is intentionally left unguarded — a fault there is a real
+    // bug that should surface, not fail dark.
+    const result = await readCachedResult(this.config.version, this.config.cacheDir)
+        .catch((error) => {
+        debug('Version check failed: %O', error);
+        return null;
+    });
+    if (!result) {
+        spawnBackgroundRefresh(this.config.version, this.config.cacheDir);
+        return;
     }
-    catch (error) {
-        // Never block CLI execution
-        debug('Version banner failed: %O', error);
+    if (!result.updateAvailable) {
+        return;
     }
+    // Skip the banner entirely in JSON mode: even on stderr it is pure noise to
+    // a script consuming the command's structured output.
+    if (hasJsonFlag(opts.argv) || hasJsonFlag(process.argv)) {
+        return;
+    }
+    const border = ux.colorize('dim', '─'.repeat(80));
+    const warning = ux.colorize('yellow', 'Uhoh! Your Output.ai CLI is behind!');
+    const latestVer = ux.colorize('green', `v${result.latestVersion}`);
+    const currentVer = ux.colorize('yellow', `v${result.currentVersion}`);
+    const updateCmd = ux.colorize('cyan', 'npx output update');
+    // Advisory notice goes to stderr so stdout stays clean for piping in every mode.
+    ux.stderr('');
+    ux.stderr(border);
+    ux.stderr('');
+    ux.stderr(`  ⚠️  ${warning}`);
+    ux.stderr('');
+    ux.stderr(`     Latest is ${latestVer}, and you're using ${currentVer}`);
+    ux.stderr('');
+    ux.stderr(`     Run \`${updateCmd}\` to update`);
+    ux.stderr('');
+    ux.stderr(border);
+    ux.stderr('');
 };
 export default hook;

package/dist/hooks/init.spec.js

@@ -12,11 +12,12 @@ vi.mock('#utils/interactive.js', () => ({
 vi.mock('@oclif/core', () => ({
     ux: {
         stdout: vi.fn(),
+        stderr: vi.fn(),
         colorize: vi.fn((_color, text) => text)
     }
 }));
 import { ux } from '@oclif/core';
-import hook, { hasInteractiveFlag, stripGlobalFlags } from './init.js';
+import hook, { hasInteractiveFlag, hasJsonFlag, stripGlobalFlags } from './init.js';
 describe('init hook', () => {
     beforeEach(() => {
         vi.clearAllMocks();
@@ -24,7 +25,7 @@ describe('init hook', () => {
     const createHookContext = (version = '0.8.4') => ({
         config: { version, cacheDir: '/tmp/test-cache' }
     });
-    it('should display warning when cached result says an update is available', async () => {
+    it('should display warning on stderr when cached result says an update is available', async () => {
         vi.mocked(readCachedResult).mockResolvedValue({
             updateAvailable: true,
             currentVersion: '0.8.4',
@@ -34,13 +35,25 @@ describe('init hook', () => {
         await hook.call(ctx, { argv: [], id: undefined });
         expect(readCachedResult).toHaveBeenCalledWith('0.8.4', '/tmp/test-cache');
         expect(spawnBackgroundRefresh).not.toHaveBeenCalled();
-        expect(ux.stdout).toHaveBeenCalled();
-        const output = vi.mocked(ux.stdout).mock.calls.map(c => c[0]).join('\n');
+        expect(ux.stderr).toHaveBeenCalled();
+        expect(ux.stdout).not.toHaveBeenCalled();
+        const output = vi.mocked(ux.stderr).mock.calls.map(c => c[0]).join('\n');
         expect(output).toContain('Uhoh');
         expect(output).toContain('v1.0.0');
         expect(output).toContain('v0.8.4');
         expect(output).toContain('npx output update');
     });
+    it('should suppress the warning entirely in JSON mode', async () => {
+        vi.mocked(readCachedResult).mockResolvedValue({
+            updateAvailable: true,
+            currentVersion: '0.8.4',
+            latestVersion: '1.0.0'
+        });
+        const ctx = createHookContext();
+        await hook.call(ctx, { argv: ['--json'], id: undefined });
+        expect(ux.stderr).not.toHaveBeenCalled();
+        expect(ux.stdout).not.toHaveBeenCalled();
+    });
     it('should not display anything when up to date', async () => {
         vi.mocked(readCachedResult).mockResolvedValue({
             updateAvailable: false,
@@ -121,6 +134,17 @@ describe('init hook', () => {
             expect(hasInteractiveFlag([])).toBe(false);
         });
     });
+    describe('hasJsonFlag', () => {
+        it('returns true when --json is present', () => {
+            expect(hasJsonFlag(['workflow', 'runs', 'list', '--json'])).toBe(true);
+        });
+        it('returns false when --json is absent', () => {
+            expect(hasJsonFlag(['workflow', 'runs', 'list', '--format', 'table'])).toBe(false);
+        });
+        it('returns false for an empty argv', () => {
+            expect(hasJsonFlag([])).toBe(false);
+        });
+    });
     describe('stripGlobalFlags', () => {
         it('mutates argv in place to remove global flags', () => {
             const argv = ['init', '--yes', 'foo', '--non-interactive'];

package/dist/utils/format_workflow_result.spec.js

@@ -1,6 +1,5 @@
 import { describe, it, expect } from 'vitest';
 import { formatWorkflowResult } from './format_workflow_result.js';
-import { formatOutput } from './output_formatter.js';
 describe('formatWorkflowResult', () => {
     it('should display output for completed workflows', () => {
         const result = formatWorkflowResult({
@@ -76,16 +75,4 @@ describe('formatWorkflowResult', () => {
         expect(result).toContain('Status: failed');
         expect(result).not.toContain('Error:');
     });
-    it('should work with formatOutput for json format', () => {
-        const data = {
-            workflowId: 'wf-456',
-            status: 'failed',
-            output: null,
-            error: 'Activity task failed'
-        };
-        const output = formatOutput(data, 'json', formatWorkflowResult);
-        const parsed = JSON.parse(output);
-        expect(parsed.status).toBe('failed');
-        expect(parsed.error).toBe('Activity task failed');
-    });
 });

package/dist/utils/trace_formatter.js

@@ -1,6 +1,5 @@
 import Table from 'cli-table3';
 import { ux } from '@oclif/core';
-import { formatOutput } from '#utils/output_formatter.js';
 import { formatDuration } from '#utils/date_formatter.js';
 import { getErrorMessage } from '#utils/error_utils.js';
 import { isTraceEvent, isValidTimestamp } from '#types/trace.js';
@@ -369,7 +368,7 @@ const formatAsText = (trace) => {
 export function format(traceData, outputFormat = 'text') {
     const trace = typeof traceData === 'string' ? JSON.parse(traceData) : traceData;
     if (outputFormat === 'json') {
-        return formatOutput(trace, 'json');
+        return JSON.stringify(trace, null, 2);
     }
     return formatAsText(trace);
 }