@outputai/cli 0.1.3 → 0.1.4-dev.0

package/dist/commands/workflow/terminate.d.ts

@@ -0,0 +1,13 @@
+import { Command } from '@oclif/core';
+export default class WorkflowTerminate extends Command {
+    static description: string;
+    static examples: string[];
+    static args: {
+        workflowId: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
+    };
+    static flags: {
+        reason: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+    };
+    run(): Promise<void>;
+    catch(error: Error): Promise<void>;
+}

package/dist/commands/workflow/terminate.js

@@ -0,0 +1,39 @@
+import { Args, Command, Flags } from '@oclif/core';
+import { postWorkflowIdTerminate } from '#api/generated/api.js';
+import { handleApiError } from '#utils/error_handler.js';
+export default class WorkflowTerminate extends Command {
+    static description = 'Terminate a workflow execution (force stop)';
+    static examples = [
+        '<%= config.bin %> <%= command.id %> wf-12345',
+        '<%= config.bin %> <%= command.id %> wf-12345 --reason "Cleaning up old workflows"'
+    ];
+    static args = {
+        workflowId: Args.string({
+            description: 'The workflow ID to terminate',
+            required: true
+        })
+    };
+    static flags = {
+        reason: Flags.string({
+            char: 'r',
+            description: 'Reason for termination'
+        })
+    };
+    async run() {
+        const { args, flags } = await this.parse(WorkflowTerminate);
+        this.log(`Terminating workflow: ${args.workflowId}...`);
+        await postWorkflowIdTerminate(args.workflowId, { reason: flags.reason });
+        const output = [
+            'Workflow terminated successfully',
+            '',
+            `Workflow ID: ${args.workflowId}`,
+            flags.reason ? `Reason: ${flags.reason}` : ''
+        ].filter(Boolean).join('\n');
+        this.log(`\n${output}`);
+    }
+    async catch(error) {
+        return handleApiError(error, (...args) => this.error(...args), {
+            404: 'Workflow not found. Check the workflow ID.'
+        });
+    }
+}

package/dist/commands/workflow/test_eval.d.ts

@@ -0,0 +1,20 @@
+import { Command } from '@oclif/core';
+export default class WorkflowTest extends Command {
+    static aliases: string[];
+    static description: string;
+    static examples: string[];
+    static args: {
+        workflowName: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
+    };
+    static flags: {
+        cached: import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        save: import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        dataset: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        format: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+    };
+    run(): Promise<void>;
+    private validateDatasets;
+    private runWorkflowForDatasets;
+    private saveEvalResults;
+    catch(error: Error): Promise<void>;
+}

package/dist/commands/workflow/test_eval.js

@@ -0,0 +1,151 @@
+import { join } from 'node:path';
+import { Args, Command, Flags } from '@oclif/core';
+import { postWorkflowRun } from '#api/generated/api.js';
+import { readAllDatasets, writeDataset } from '#services/datasets.js';
+import { handleApiError } from '#utils/error_handler.js';
+import { getEvalWorkflowName, renderEvalOutput, computeExitCode, EvalOutputSchema } from '@outputai/evals';
+export default class WorkflowTest extends Command {
+    static aliases = ['workflow:test'];
+    static description = 'Run evaluations against a workflow using its datasets';
+    static examples = [
+        '<%= config.bin %> <%= command.id %> simple',
+        '<%= config.bin %> <%= command.id %> simple --cached',
+        '<%= config.bin %> <%= command.id %> simple --save',
+        '<%= config.bin %> <%= command.id %> simple --dataset basic_input,edge_case',
+        '<%= config.bin %> <%= command.id %> simple --format json'
+    ];
+    static args = {
+        workflowName: Args.string({
+            description: 'Name of the workflow to test',
+            required: true
+        })
+    };
+    static flags = {
+        cached: Flags.boolean({
+            description: 'Use cached output from dataset files (skip workflow execution)',
+            default: false,
+            exclusive: ['save']
+        }),
+        save: Flags.boolean({
+            description: 'Run workflow and save output back to dataset files',
+            default: false,
+            exclusive: ['cached']
+        }),
+        dataset: Flags.string({
+            description: 'Comma-separated list of dataset names to run',
+            char: 'd'
+        }),
+        format: Flags.string({
+            char: 'f',
+            description: 'Output format',
+            options: ['json', 'text'],
+            default: 'text'
+        })
+    };
+    async run() {
+        const { args, flags } = await this.parse(WorkflowTest);
+        const filterNames = flags.dataset?.split(',').map(s => s.trim());
+        const { datasets, dir } = await readAllDatasets(args.workflowName, filterNames);
+        if (datasets.length === 0) {
+            this.error(`No datasets found for workflow "${args.workflowName}".\n` +
+                'Generate datasets first: output workflow dataset generate', { exit: 1 });
+        }
+        const preparedDatasets = flags.cached ?
+            this.validateDatasets(datasets) :
+            await this.runWorkflowForDatasets(args.workflowName, datasets, flags.save, dir);
+        const evalName = getEvalWorkflowName(args.workflowName);
+        this.log(`Running eval workflow "${evalName}"...\n`);
+        const response = await postWorkflowRun({
+            workflowName: evalName,
+            input: { datasets: preparedDatasets }
+        }, {
+            config: { timeout: 600000 }
+        });
+        const evalData = response?.data;
+        if (!evalData?.output) {
+            this.error('Eval workflow returned no output', { exit: 1 });
+        }
+        const evalOutput = EvalOutputSchema.parse(evalData.output);
+        if (flags.save) {
+            await this.saveEvalResults(evalOutput, preparedDatasets, dir);
+        }
+        if (flags.format === 'json') {
+            this.log(JSON.stringify(evalOutput, null, 2));
+        }
+        else {
+            this.log(renderEvalOutput(evalOutput, evalName));
+        }
+        const exitCode = computeExitCode(evalOutput);
+        if (exitCode !== 0) {
+            this.exit(exitCode);
+        }
+    }
+    validateDatasets(datasets) {
+        const missing = datasets.filter(d => d.last_output?.output === undefined);
+        if (missing.length > 0) {
+            const names = missing.map(d => d.name).join(', ');
+            this.error(`Datasets missing cached output: ${names}\n` +
+                'Run without --cached to execute the workflow, or use --save to cache output.', { exit: 1 });
+        }
+        return datasets;
+    }
+    async runWorkflowForDatasets(workflowName, datasets, save, dir) {
+        this.log(`Running workflow "${workflowName}" for ${datasets.length} dataset(s)...\n`);
+        const results = [];
+        for (const dataset of datasets) {
+            this.log(`  Running "${dataset.name}"...`);
+            const startMs = Date.now();
+            const response = await postWorkflowRun({
+                workflowName,
+                input: dataset.input
+            }, {
+                config: { timeout: 600000 }
+            });
+            const executionTimeMs = Date.now() - startMs;
+            if (!response?.data) {
+                this.error(`Workflow execution failed for dataset "${dataset.name}"`, { exit: 1 });
+            }
+            const updated = {
+                ...dataset,
+                last_output: {
+                    output: response.data.output,
+                    executionTimeMs,
+                    date: new Date().toISOString()
+                }
+            };
+            if (save) {
+                const filePath = join(dir, `${dataset.name}.yml`);
+                await writeDataset(updated, filePath);
+                this.log(`    Saved output to ${filePath}`);
+            }
+            results.push(updated);
+        }
+        this.log('');
+        return results;
+    }
+    async saveEvalResults(evalOutput, datasets, dir) {
+        const now = new Date().toISOString();
+        for (const evalCase of evalOutput.cases) {
+            const dataset = datasets.find(d => d.name === evalCase.datasetName);
+            if (!dataset) {
+                continue;
+            }
+            const updated = {
+                ...dataset,
+                last_eval: {
+                    output: evalCase,
+                    date: now
+                }
+            };
+            const filePath = join(dir, `${dataset.name}.yml`);
+            await writeDataset(updated, filePath);
+            this.log(`  Saved eval result to ${filePath}`);
+        }
+    }
+    async catch(error) {
+        return handleApiError(error, (...args) => this.error(...args), {
+            404: 'Workflow or eval workflow not found. Check the workflow name.',
+            500: 'Workflow execution failed.'
+        });
+    }
+}

package/dist/config.d.ts

@@ -0,0 +1,47 @@
+/**
+ * CLI configuration
+ */
+export declare const config: {
+    /**
+     * Base URL for the Output.ai API server
+     * Can be overridden with OUTPUT_API_URL environment variable
+     */
+    apiUrl: string;
+    /**
+     * API authentication token
+     * Set via OUTPUT_API_AUTH_TOKEN environment variable
+     */
+    apiToken: string | undefined;
+    /**
+     * Default timeout for API requests (in milliseconds)
+     */
+    requestTimeout: number;
+    /**
+     * Docker Compose project name
+     * Can be overridden with DOCKER_SERVICE_NAME environment variable
+     */
+    dockerServiceName: string;
+    /**
+     * Set the debug mode
+     */
+    debugMode: boolean;
+    /**
+     * Where the env vars are stored, defaults to `.env`
+     */
+    envFile: string;
+    /**
+     * Agent configuration directory name
+     */
+    agentConfigDir: string;
+    /**
+     * S3 configuration for remote trace storage
+     * Set via OUTPUT_TRACE_REMOTE_S3_BUCKET, OUTPUT_AWS_REGION,
+     * OUTPUT_AWS_ACCESS_KEY_ID, and OUTPUT_AWS_SECRET_ACCESS_KEY environment variables
+     */
+    s3: {
+        bucket: string | undefined;
+        region: string | undefined;
+        accessKeyId: string | undefined;
+        secretAccessKey: string | undefined;
+    };
+};

package/dist/config.js

@@ -0,0 +1,47 @@
+/**
+ * CLI configuration
+ */
+export const config = {
+    /**
+     * Base URL for the Output.ai API server
+     * Can be overridden with OUTPUT_API_URL environment variable
+     */
+    apiUrl: process.env.OUTPUT_API_URL || 'http://localhost:3001',
+    /**
+     * API authentication token
+     * Set via OUTPUT_API_AUTH_TOKEN environment variable
+     */
+    apiToken: process.env.OUTPUT_API_AUTH_TOKEN,
+    /**
+     * Default timeout for API requests (in milliseconds)
+     */
+    requestTimeout: 30000,
+    /**
+     * Docker Compose project name
+     * Can be overridden with DOCKER_SERVICE_NAME environment variable
+     */
+    dockerServiceName: process.env.DOCKER_SERVICE_NAME || 'output-sdk',
+    /**
+     * Set the debug mode
+     */
+    debugMode: process.env.OUTPUT_DEBUG === 'true',
+    /**
+     * Where the env vars are stored, defaults to `.env`
+     */
+    envFile: process.env.OUTPUT_CLI_ENV || '.env',
+    /**
+     * Agent configuration directory name
+     */
+    agentConfigDir: '.outputai',
+    /**
+     * S3 configuration for remote trace storage
+     * Set via OUTPUT_TRACE_REMOTE_S3_BUCKET, OUTPUT_AWS_REGION,
+     * OUTPUT_AWS_ACCESS_KEY_ID, and OUTPUT_AWS_SECRET_ACCESS_KEY environment variables
+     */
+    s3: {
+        bucket: process.env.OUTPUT_TRACE_REMOTE_S3_BUCKET,
+        region: process.env.OUTPUT_AWS_REGION,
+        accessKeyId: process.env.OUTPUT_AWS_ACCESS_KEY_ID,
+        secretAccessKey: process.env.OUTPUT_AWS_SECRET_ACCESS_KEY
+    }
+};

package/dist/generated/framework_version.json

@@ -0,0 +1,3 @@
+{
+    "framework": "0.1.3"
+}

package/dist/hooks/init.d.ts

@@ -0,0 +1,3 @@
+import { Hook } from '@oclif/core';
+declare const hook: Hook<'init'>;
+export default hook;

package/dist/hooks/init.js

@@ -0,0 +1,30 @@
+import { ux } from '@oclif/core';
+import { checkForUpdate } from '#services/version_check.js';
+const hook = async function () {
+    try {
+        const result = await checkForUpdate(this.config.version, this.config.cacheDir);
+        if (!result.updateAvailable) {
+            return;
+        }
+        const border = ux.colorize('dim', '─'.repeat(80));
+        const warning = ux.colorize('yellow', 'Uhoh! Your Output.ai CLI is behind!');
+        const latestVer = ux.colorize('green', `v${result.latestVersion}`);
+        const currentVer = ux.colorize('yellow', `v${result.currentVersion}`);
+        const updateCmd = ux.colorize('cyan', 'npx output update');
+        ux.stdout('');
+        ux.stdout(border);
+        ux.stdout('');
+        ux.stdout(`  ⚠️  ${warning}`);
+        ux.stdout('');
+        ux.stdout(`     Latest is ${latestVer}, and you're using ${currentVer}`);
+        ux.stdout('');
+        ux.stdout(`     Run \`${updateCmd}\` to update`);
+        ux.stdout('');
+        ux.stdout(border);
+        ux.stdout('');
+    }
+    catch {
+        // Never block CLI execution
+    }
+};
+export default hook;

package/dist/hooks/init.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/hooks/init.spec.js

@@ -0,0 +1,54 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { checkForUpdate } from '#services/version_check.js';
+vi.mock('#services/version_check.js', () => ({
+    checkForUpdate: vi.fn()
+}));
+vi.mock('@oclif/core', () => ({
+    ux: {
+        stdout: vi.fn(),
+        colorize: vi.fn((_color, text) => text)
+    }
+}));
+import { ux } from '@oclif/core';
+import hook from './init.js';
+describe('init hook', () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+    });
+    const createHookContext = (version = '0.8.4') => ({
+        config: { version, cacheDir: '/tmp/test-cache' }
+    });
+    it('should display warning when update is available', async () => {
+        vi.mocked(checkForUpdate).mockResolvedValue({
+            updateAvailable: true,
+            currentVersion: '0.8.4',
+            latestVersion: '1.0.0'
+        });
+        const ctx = createHookContext();
+        await hook.call(ctx, {});
+        expect(checkForUpdate).toHaveBeenCalledWith('0.8.4', '/tmp/test-cache');
+        expect(ux.stdout).toHaveBeenCalled();
+        const output = vi.mocked(ux.stdout).mock.calls.map(c => c[0]).join('\n');
+        expect(output).toContain('Uhoh');
+        expect(output).toContain('v1.0.0');
+        expect(output).toContain('v0.8.4');
+        expect(output).toContain('npx output update');
+    });
+    it('should not display anything when up to date', async () => {
+        vi.mocked(checkForUpdate).mockResolvedValue({
+            updateAvailable: false,
+            currentVersion: '0.8.4',
+            latestVersion: '0.8.4'
+        });
+        const ctx = createHookContext();
+        await hook.call(ctx, {});
+        expect(ux.stdout).not.toHaveBeenCalled();
+    });
+    it('should silently handle errors', async () => {
+        vi.mocked(checkForUpdate).mockRejectedValue(new Error('network failure'));
+        const ctx = createHookContext();
+        await hook.call(ctx, {});
+        expect(ux.stdout).not.toHaveBeenCalled();
+    });
+});

package/dist/index.d.ts

@@ -0,0 +1 @@
+export { run } from '@oclif/core';

package/dist/index.js

@@ -0,0 +1 @@
+export { run } from '@oclif/core';

package/dist/index.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/index.spec.js

@@ -0,0 +1,6 @@
+import { describe, it, expect } from 'vitest';
+describe('CLI', () => {
+    it('should be initialized', () => {
+        expect(true).toBe(true);
+    });
+});

package/dist/services/claude_client.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Claude Agent SDK client for workflow planning
+ */
+import { Options } from '@anthropic-ai/claude-agent-sdk';
+export declare const PLAN_COMMAND_OPTIONS: Options;
+export declare const BUILD_COMMAND_OPTIONS: Options;
+export declare class ClaudeInvocationError extends Error {
+    cause?: Error | undefined;
+    constructor(message: string, cause?: Error | undefined);
+}
+export declare function replyToClaude(message: string, options?: Options): Promise<string>;
+/**
+ * Invoke claude-code with /outputai:plan_workflow slash command
+ * The SDK loads custom commands from .claude/commands/ when settingSources includes 'project'.
+ * ensureOutputAISystem() scaffolds the command files to that location.
+ * @param description - Workflow description
+ * @returns Plan output from claude-code
+ */
+export declare function invokePlanWorkflow(description: string): Promise<string>;
+/**
+ * Invoke claude-code with /outputai:build_workflow slash command
+ * The SDK loads custom commands from .claude/commands/ when settingSources includes 'project'.
+ * ensureOutputAISystem() scaffolds the command files to that location.
+ * @param planFilePath - Absolute path to the plan file
+ * @param workflowDir - Absolute path to the workflow directory
+ * @param workflowName - Name of the workflow
+ * @param additionalInstructions - Optional additional instructions
+ * @returns Implementation output from claude-code
+ */
+export declare function invokeBuildWorkflow(planFilePath: string, workflowDir: string, workflowName: string, additionalInstructions?: string): Promise<string>;

package/dist/services/claude_client.integration.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/services/claude_client.integration.test.js

@@ -0,0 +1,43 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { invokePlanWorkflow } from './claude_client.js';
+import { query } from '@anthropic-ai/claude-agent-sdk';
+describe('invokePlanWorkflow - Integration Tests', () => {
+    beforeEach(() => {
+        // Ensure API key is set
+        if (!process.env.ANTHROPIC_API_KEY) {
+            throw new Error('ANTHROPIC_API_KEY must be set for integration tests');
+        }
+    });
+    it('should debug actual message format from Claude Agent SDK', async () => {
+        const description = 'Simple test workflow that takes a number and returns it doubled';
+        console.log('\n===== DEBUGGING ACTUAL MESSAGE FORMAT =====');
+        console.log('Description:', description);
+        const messages = [];
+        try {
+            for await (const message of query({
+                prompt: `/outputai:plan_workflow ${description}`,
+                options: { maxTurns: 1 }
+            })) {
+                console.log('\nReceived message:', JSON.stringify(message, null, 2));
+                messages.push(message);
+            }
+        }
+        catch (error) {
+            console.error('Error during query:', error);
+            throw error;
+        }
+        console.log(`\nTotal messages received: ${messages.length}`);
+        console.log('===== END DEBUG =====');
+        // This test is just for debugging - we expect messages
+        expect(messages.length).toBeGreaterThan(0);
+    }, 60000); // 60 second timeout
+    it('should successfully invoke /outputai:plan_workflow slash command and return content', async () => {
+        const description = 'Simple workflow that takes a number and doubles it';
+        const result = await invokePlanWorkflow(description);
+        console.log('\n===== PLAN RESULT =====');
+        console.log(result);
+        console.log('===== END RESULT =====');
+        expect(result).toBeTruthy();
+        expect(result.length).toBeGreaterThan(0);
+    }, 60000); // 60 second timeout
+});