@outputai/cli 0.8.1-next.e92f632.0 → 0.8.2-dev.e78f6b4.0

package/dist/commands/workflow/test_eval.spec.js

@@ -0,0 +1,121 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { getEvalWorkflowName, renderEvalOutput } from '@outputai/evals';
+vi.mock('#api/generated/api.js', () => ({
+    postWorkflowRun: vi.fn()
+}));
+vi.mock('#api/workflow_catalog.js', () => ({
+    fetchWorkflowCatalog: vi.fn()
+}));
+vi.mock('#services/datasets.js', () => ({
+    readAllDatasets: vi.fn(),
+    writeDataset: vi.fn()
+}));
+vi.mock('#utils/eval_diagnostics.js', () => ({
+    diagnoseMissingEvalWorkflow: vi.fn().mockResolvedValue('missing eval workflow')
+}));
+const passingOutput = {
+    cases: [{ datasetName: 'd1', verdict: 'pass', evaluators: [] }],
+    summary: { total: 1, passed: 1, partial: 0, failed: 0, acceptableRate: 1 }
+};
+const failingOutput = {
+    cases: [{ datasetName: 'd1', verdict: 'fail', evaluators: [] }],
+    summary: { total: 1, passed: 0, partial: 0, failed: 1, acceptableRate: 0 }
+};
+describe('workflow test command', () => {
+    const exitState = { original: undefined };
+    beforeEach(async () => {
+        vi.clearAllMocks();
+        exitState.original = process.exitCode;
+        process.exitCode = undefined;
+        const { readAllDatasets } = await import('#services/datasets.js');
+        const { fetchWorkflowCatalog } = await import('#api/workflow_catalog.js');
+        vi.mocked(readAllDatasets).mockResolvedValue({
+            datasets: [{ name: 'd1', input: {}, last_output: { output: {}, date: '2026-01-01' } }],
+            dir: '/tmp/datasets'
+        });
+        // Catalog includes both eval names so ensureEvalWorkflowRegistered passes deterministically.
+        vi.mocked(fetchWorkflowCatalog).mockResolvedValue([
+            { name: getEvalWorkflowName('simple') },
+            { name: getEvalWorkflowName('my_workflow') }
+        ]);
+    });
+    afterEach(() => {
+        process.exitCode = exitState.original;
+    });
+    describe('command definition', () => {
+        it('enables the built-in --json flag', async () => {
+            const WorkflowTest = (await import('./test_eval.js')).default;
+            expect(WorkflowTest.enableJsonFlag).toBe(true);
+        });
+        it('binds the catalog flag to OUTPUT_CATALOG_ID', async () => {
+            const WorkflowTest = (await import('./test_eval.js')).default;
+            expect(WorkflowTest.flags).toHaveProperty('catalog');
+            expect(WorkflowTest.flags.catalog.env).toBe('OUTPUT_CATALOG_ID');
+            expect(WorkflowTest.flags.catalog.char).toBe('c');
+        });
+    });
+    describe('run()', () => {
+        const createCommand = async (jsonEnabled) => {
+            const WorkflowTest = (await import('./test_eval.js')).default;
+            const { postWorkflowRun } = await import('#api/generated/api.js');
+            const cmd = new WorkflowTest(['simple'], {});
+            cmd.log = vi.fn();
+            cmd.jsonEnabled = vi.fn().mockReturnValue(jsonEnabled);
+            cmd.parse = vi.fn().mockResolvedValue({
+                args: { workflowName: 'simple' },
+                flags: { cached: true, save: false, dataset: undefined }
+            });
+            return { cmd, postWorkflowRun: vi.mocked(postWorkflowRun) };
+        };
+        it('sets a non-zero exit code and returns the eval output when a case fails', async () => {
+            const { cmd, postWorkflowRun } = await createCommand(false);
+            postWorkflowRun.mockResolvedValue({ data: { output: failingOutput } });
+            const result = await cmd.run();
+            expect(result).toEqual(failingOutput);
+            expect(process.exitCode).toBe(1);
+        });
+        it('leaves the exit code at zero and returns the eval output when all cases pass', async () => {
+            const { cmd, postWorkflowRun } = await createCommand(false);
+            postWorkflowRun.mockResolvedValue({ data: { output: passingOutput } });
+            const result = await cmd.run();
+            expect(result).toEqual(passingOutput);
+            expect(process.exitCode).toBe(0);
+        });
+        it('renders the human-readable summary in text mode', async () => {
+            const { cmd, postWorkflowRun } = await createCommand(false);
+            postWorkflowRun.mockResolvedValue({ data: { output: passingOutput } });
+            await cmd.run();
+            const rendered = renderEvalOutput(passingOutput, getEvalWorkflowName('simple'));
+            expect(cmd.log).toHaveBeenCalledWith(rendered);
+        });
+        it('suppresses the rendered summary in JSON mode but still returns and sets exit code', async () => {
+            const { cmd, postWorkflowRun } = await createCommand(true);
+            postWorkflowRun.mockResolvedValue({ data: { output: failingOutput } });
+            const result = await cmd.run();
+            const rendered = renderEvalOutput(failingOutput, getEvalWorkflowName('simple'));
+            expect(cmd.log).not.toHaveBeenCalledWith(rendered);
+            expect(result).toEqual(failingOutput);
+            expect(process.exitCode).toBe(1);
+        });
+        it('routes registration, dataset runs, and the eval run to the resolved catalog', async () => {
+            const WorkflowTest = (await import('./test_eval.js')).default;
+            const { postWorkflowRun } = await import('#api/generated/api.js');
+            const { fetchWorkflowCatalog } = await import('#api/workflow_catalog.js');
+            const cmd = new WorkflowTest(['my_workflow'], {});
+            cmd.log = vi.fn();
+            cmd.jsonEnabled = vi.fn().mockReturnValue(false);
+            cmd.parse = vi.fn().mockResolvedValue({
+                args: { workflowName: 'my_workflow' },
+                flags: { catalog: 'my-catalog', cached: false, save: false, dataset: undefined }
+            });
+            vi.mocked(postWorkflowRun)
+                .mockResolvedValueOnce({ data: { output: {} }, status: 200, headers: new Headers() })
+                .mockResolvedValueOnce({ data: { output: passingOutput }, status: 200, headers: new Headers() });
+            await cmd.run();
+            expect(vi.mocked(fetchWorkflowCatalog)).toHaveBeenCalledWith('my-catalog');
+            expect(postWorkflowRun).toHaveBeenNthCalledWith(1, expect.objectContaining({ workflowName: 'my_workflow', catalog: 'my-catalog' }), expect.anything());
+            expect(postWorkflowRun).toHaveBeenNthCalledWith(2, expect.objectContaining({ workflowName: getEvalWorkflowName('my_workflow'), catalog: 'my-catalog' }), expect.anything());
+        });
+    });
+});

package/dist/generated/framework_version.json

@@ -1,3 +1,3 @@
 {
-    "framework": "0.8.1-next.e92f632.0"
+    "framework": "0.8.2-dev.e78f6b4.0"
 }

package/dist/hooks/init.d.ts

@@ -2,6 +2,7 @@ import { Hook } from '@oclif/core';
 export declare const INTERACTIVE_FLAGS: string[];
 export declare const GLOBAL_FLAGS: Set<string>;
 export declare const hasInteractiveFlag: (argv: string[]) => boolean;
+export declare const hasJsonFlag: (argv: string[]) => boolean;
 export declare const stripGlobalFlags: (argv: string[]) => void;
 declare const hook: Hook<'init'>;
 export default hook;

package/dist/hooks/init.js

@@ -6,6 +6,10 @@ const debug = debugFactory('output-cli:init');
 export const INTERACTIVE_FLAGS = ['--yes', '--non-interactive'];
 export const GLOBAL_FLAGS = new Set(INTERACTIVE_FLAGS);
 export const hasInteractiveFlag = (argv) => argv.some(arg => INTERACTIVE_FLAGS.includes(arg));
+// The version banner must never reach stdout in JSON mode, where it would
+// corrupt the machine-readable output. oclif only suppresses `this.log` inside
+// the command, not hook output, so we detect `--json` ourselves.
+export const hasJsonFlag = (argv) => argv.includes('--json');
 export const stripGlobalFlags = (argv) => {
     const kept = argv.filter(arg => !GLOBAL_FLAGS.has(arg));
     if (kept.length !== argv.length) {
@@ -19,37 +23,43 @@ const hook = async function (opts) {
     if (interactive) {
         setNonInteractive(true);
     }
-    try {
-        // Only the local cache is read here; the registry roundtrip happens in a
-        // detached child so it never delays the invoked command.
-        const result = await readCachedResult(this.config.version, this.config.cacheDir);
-        if (!result) {
-            spawnBackgroundRefresh(this.config.version, this.config.cacheDir);
-            return;
-        }
-        if (!result.updateAvailable) {
-            return;
-        }
-        const border = ux.colorize('dim', '─'.repeat(80));
-        const warning = ux.colorize('yellow', 'Uhoh! Your Output.ai CLI is behind!');
-        const latestVer = ux.colorize('green', `v${result.latestVersion}`);
-        const currentVer = ux.colorize('yellow', `v${result.currentVersion}`);
-        const updateCmd = ux.colorize('cyan', 'npx output update');
-        ux.stdout('');
-        ux.stdout(border);
-        ux.stdout('');
-        ux.stdout(`  ⚠️  ${warning}`);
-        ux.stdout('');
-        ux.stdout(`     Latest is ${latestVer}, and you're using ${currentVer}`);
-        ux.stdout('');
-        ux.stdout(`     Run \`${updateCmd}\` to update`);
-        ux.stdout('');
-        ux.stdout(border);
-        ux.stdout('');
+    // Guard only the version-check IO: a broken or unreadable cache must never
+    // block CLI execution, so a failure is treated as "no cached result". Banner
+    // rendering below is intentionally left unguarded — a fault there is a real
+    // bug that should surface, not fail dark.
+    const result = await readCachedResult(this.config.version, this.config.cacheDir)
+        .catch((error) => {
+        debug('Version check failed: %O', error);
+        return null;
+    });
+    if (!result) {
+        spawnBackgroundRefresh(this.config.version, this.config.cacheDir);
+        return;
     }
-    catch (error) {
-        // Never block CLI execution
-        debug('Version banner failed: %O', error);
+    if (!result.updateAvailable) {
+        return;
     }
+    // Skip the banner entirely in JSON mode: even on stderr it is pure noise to
+    // a script consuming the command's structured output.
+    if (hasJsonFlag(opts.argv) || hasJsonFlag(process.argv)) {
+        return;
+    }
+    const border = ux.colorize('dim', '─'.repeat(80));
+    const warning = ux.colorize('yellow', 'Uhoh! Your Output.ai CLI is behind!');
+    const latestVer = ux.colorize('green', `v${result.latestVersion}`);
+    const currentVer = ux.colorize('yellow', `v${result.currentVersion}`);
+    const updateCmd = ux.colorize('cyan', 'npx output update');
+    // Advisory notice goes to stderr so stdout stays clean for piping in every mode.
+    ux.stderr('');
+    ux.stderr(border);
+    ux.stderr('');
+    ux.stderr(`  ⚠️  ${warning}`);
+    ux.stderr('');
+    ux.stderr(`     Latest is ${latestVer}, and you're using ${currentVer}`);
+    ux.stderr('');
+    ux.stderr(`     Run \`${updateCmd}\` to update`);
+    ux.stderr('');
+    ux.stderr(border);
+    ux.stderr('');
 };
 export default hook;

package/dist/hooks/init.spec.js

@@ -12,11 +12,12 @@ vi.mock('#utils/interactive.js', () => ({
 vi.mock('@oclif/core', () => ({
     ux: {
         stdout: vi.fn(),
+        stderr: vi.fn(),
         colorize: vi.fn((_color, text) => text)
     }
 }));
 import { ux } from '@oclif/core';
-import hook, { hasInteractiveFlag, stripGlobalFlags } from './init.js';
+import hook, { hasInteractiveFlag, hasJsonFlag, stripGlobalFlags } from './init.js';
 describe('init hook', () => {
     beforeEach(() => {
         vi.clearAllMocks();
@@ -24,7 +25,7 @@ describe('init hook', () => {
     const createHookContext = (version = '0.8.4') => ({
         config: { version, cacheDir: '/tmp/test-cache' }
     });
-    it('should display warning when cached result says an update is available', async () => {
+    it('should display warning on stderr when cached result says an update is available', async () => {
         vi.mocked(readCachedResult).mockResolvedValue({
             updateAvailable: true,
             currentVersion: '0.8.4',
@@ -34,13 +35,25 @@ describe('init hook', () => {
         await hook.call(ctx, { argv: [], id: undefined });
         expect(readCachedResult).toHaveBeenCalledWith('0.8.4', '/tmp/test-cache');
         expect(spawnBackgroundRefresh).not.toHaveBeenCalled();
-        expect(ux.stdout).toHaveBeenCalled();
-        const output = vi.mocked(ux.stdout).mock.calls.map(c => c[0]).join('\n');
+        expect(ux.stderr).toHaveBeenCalled();
+        expect(ux.stdout).not.toHaveBeenCalled();
+        const output = vi.mocked(ux.stderr).mock.calls.map(c => c[0]).join('\n');
         expect(output).toContain('Uhoh');
         expect(output).toContain('v1.0.0');
         expect(output).toContain('v0.8.4');
         expect(output).toContain('npx output update');
     });
+    it('should suppress the warning entirely in JSON mode', async () => {
+        vi.mocked(readCachedResult).mockResolvedValue({
+            updateAvailable: true,
+            currentVersion: '0.8.4',
+            latestVersion: '1.0.0'
+        });
+        const ctx = createHookContext();
+        await hook.call(ctx, { argv: ['--json'], id: undefined });
+        expect(ux.stderr).not.toHaveBeenCalled();
+        expect(ux.stdout).not.toHaveBeenCalled();
+    });
     it('should not display anything when up to date', async () => {
         vi.mocked(readCachedResult).mockResolvedValue({
             updateAvailable: false,
@@ -121,6 +134,17 @@ describe('init hook', () => {
             expect(hasInteractiveFlag([])).toBe(false);
         });
     });
+    describe('hasJsonFlag', () => {
+        it('returns true when --json is present', () => {
+            expect(hasJsonFlag(['workflow', 'runs', 'list', '--json'])).toBe(true);
+        });
+        it('returns false when --json is absent', () => {
+            expect(hasJsonFlag(['workflow', 'runs', 'list', '--format', 'table'])).toBe(false);
+        });
+        it('returns false for an empty argv', () => {
+            expect(hasJsonFlag([])).toBe(false);
+        });
+    });
     describe('stripGlobalFlags', () => {
         it('mutates argv in place to remove global flags', () => {
             const argv = ['init', '--yes', 'foo', '--non-interactive'];

package/dist/services/coding_agents.spec.js

@@ -3,13 +3,13 @@ import { checkAgentStructure, prepareTemplateVariables, initializeAgentConfig, e
 import { access } from 'node:fs/promises';
 import fs from 'node:fs/promises';
 vi.mock('node:fs/promises');
-vi.mock('../utils/paths.js', () => ({
+vi.mock('#utils/paths.js', () => ({
     getTemplateDir: vi.fn().mockReturnValue('/templates')
 }));
-vi.mock('../utils/template.js', () => ({
+vi.mock('#utils/template.js', () => ({
     processTemplate: vi.fn().mockImplementation((content) => content)
 }));
-vi.mock('../utils/claude.js', () => ({
+vi.mock('#utils/claude.js', () => ({
     executeClaudeCommand: vi.fn().mockResolvedValue(undefined)
 }));
 vi.mock('@oclif/core', () => ({
@@ -152,14 +152,14 @@ describe('coding_agents service', () => {
             vi.mocked(fs.writeFile).mockResolvedValue(undefined);
         });
         it('should call registerPluginMarketplace and installOutputAIPlugin', async () => {
-            const { executeClaudeCommand } = await import('../utils/claude.js');
+            const { executeClaudeCommand } = await import('#utils/claude.js');
             await ensureClaudePlugin('/test/project');
             expect(executeClaudeCommand).toHaveBeenCalledWith(['plugin', 'marketplace', 'add', 'growthxai/output'], '/test/project', { ignoreFailure: true });
             expect(executeClaudeCommand).toHaveBeenCalledWith(['plugin', 'marketplace', 'update', 'outputai'], '/test/project');
             expect(executeClaudeCommand).toHaveBeenCalledWith(['plugin', 'install', 'outputai@outputai', '--scope', 'project'], '/test/project');
         });
         it('should show error and prompt user when plugin commands fail', async () => {
-            const { executeClaudeCommand } = await import('../utils/claude.js');
+            const { executeClaudeCommand } = await import('#utils/claude.js');
             const { confirm } = await import('#utils/prompt.js');
             vi.mocked(executeClaudeCommand)
                 .mockResolvedValueOnce(undefined) // marketplace add
@@ -171,7 +171,7 @@ describe('coding_agents service', () => {
             }));
         });
         it('should allow user to proceed without plugin setup if they confirm', async () => {
-            const { executeClaudeCommand } = await import('../utils/claude.js');
+            const { executeClaudeCommand } = await import('#utils/claude.js');
             const { confirm } = await import('#utils/prompt.js');
             vi.mocked(executeClaudeCommand)
                 .mockRejectedValue(new Error('All plugin commands fail'));
@@ -221,7 +221,7 @@ describe('coding_agents service', () => {
             vi.mocked(fs.writeFile).mockResolvedValue(undefined);
         });
         it('should show error and prompt user when registerPluginMarketplace fails', async () => {
-            const { executeClaudeCommand } = await import('../utils/claude.js');
+            const { executeClaudeCommand } = await import('#utils/claude.js');
             const { confirm } = await import('#utils/prompt.js');
             vi.mocked(executeClaudeCommand)
                 .mockResolvedValueOnce(undefined) // marketplace add
@@ -233,7 +233,7 @@ describe('coding_agents service', () => {
             }));
         });
         it('should show error and prompt user when installOutputAIPlugin fails', async () => {
-            const { executeClaudeCommand } = await import('../utils/claude.js');
+            const { executeClaudeCommand } = await import('#utils/claude.js');
             const { confirm } = await import('#utils/prompt.js');
             vi.mocked(executeClaudeCommand)
                 .mockResolvedValueOnce(undefined) // marketplace add
@@ -246,7 +246,7 @@ describe('coding_agents service', () => {
             }));
         });
         it('should allow user to proceed without plugin setup if they confirm', async () => {
-            const { executeClaudeCommand } = await import('../utils/claude.js');
+            const { executeClaudeCommand } = await import('#utils/claude.js');
             const { confirm } = await import('#utils/prompt.js');
             vi.mocked(executeClaudeCommand)
                 .mockRejectedValue(new Error('All plugin commands fail'));
@@ -256,7 +256,7 @@ describe('coding_agents service', () => {
             expect(fs.mkdir).toHaveBeenCalled();
         });
         it('should rethrow plugin error in non-interactive mode without prompting', async () => {
-            const { executeClaudeCommand } = await import('../utils/claude.js');
+            const { executeClaudeCommand } = await import('#utils/claude.js');
             const { confirm } = await import('#utils/prompt.js');
             const { isInteractive } = await import('#utils/interactive.js');
             vi.mocked(isInteractive).mockReturnValueOnce(false);

package/dist/services/datasets.d.ts

@@ -6,8 +6,8 @@ export interface DatasetInfo {
     lastOutputDate?: string;
     lastEvalDate?: string;
 }
-export declare function resolveDatasetsDir(workflowName: string, basePath?: string): string | null;
-export declare function resolveDefaultDatasetsDir(workflowName: string, basePath?: string): string;
+export declare function resolveDatasetsDir(workflowName: string, basePath?: string, workflowPath?: string): Promise<string | null>;
+export declare function resolveDefaultDatasetsDir(workflowName: string, basePath?: string, workflowPath?: string): Promise<string>;
 export declare function readDatasetFile(filePath: string): Promise<Dataset[]>;
 export declare function readAllDatasets(workflowName: string, filterNames?: string[], basePath?: string): Promise<{
     datasets: Dataset[];

package/dist/services/datasets.js

@@ -5,24 +5,26 @@ import yaml from 'js-yaml';
 import { DatasetSchema } from '@outputai/evals';
 import { getTrace } from '#services/trace_reader.js';
 import { sanitizeSecrets } from '#utils/secret_sanitizer.js';
+import { resolveWorkflowDir, WORKFLOWS_PATHS } from '#utils/workflow_dir.js';
+import { getWorkflowsBasePath } from '#utils/paths.js';
 const DATASETS_DIR = 'tests/datasets';
-const WORKFLOWS_PATHS = ['src/workflows', 'workflows'];
-export function resolveDatasetsDir(workflowName, basePath = process.cwd()) {
-    for (const workflowsDir of WORKFLOWS_PATHS) {
-        const candidate = resolve(basePath, workflowsDir, workflowName, DATASETS_DIR);
-        if (existsSync(candidate)) {
-            return candidate;
-        }
+export async function resolveDatasetsDir(workflowName, basePath, workflowPath) {
+    const workflowDir = await resolveWorkflowDir(workflowName, basePath, workflowPath);
+    if (!workflowDir) {
+        return null;
     }
-    return null;
+    const datasetsDir = resolve(workflowDir, DATASETS_DIR);
+    return existsSync(datasetsDir) ? datasetsDir : null;
 }
-export function resolveDefaultDatasetsDir(workflowName, basePath = process.cwd()) {
-    const existing = resolveDatasetsDir(workflowName, basePath);
-    if (existing) {
-        return existing;
+export async function resolveDefaultDatasetsDir(workflowName, basePath, workflowPath) {
+    // Write into the resolved workflow's tests/datasets — even when it doesn't
+    // exist yet (first generation) — so nested workflows get the right location.
+    const workflowDir = await resolveWorkflowDir(workflowName, basePath, workflowPath);
+    if (workflowDir) {
+        return resolve(workflowDir, DATASETS_DIR);
     }
-    // Default to first workflows path
-    return resolve(basePath, WORKFLOWS_PATHS[0], workflowName, DATASETS_DIR);
+    // Workflow not found (unknown name / API unavailable): flat convention.
+    return resolve(basePath ?? getWorkflowsBasePath(), WORKFLOWS_PATHS[0], workflowName, DATASETS_DIR);
 }
 export async function readDatasetFile(filePath) {
     const raw = yaml.load(await readFile(filePath, 'utf-8'));
@@ -39,9 +41,9 @@ export async function readDatasetFile(filePath) {
     });
 }
 export async function readAllDatasets(workflowName, filterNames, basePath) {
-    const dir = resolveDatasetsDir(workflowName, basePath);
+    const dir = await resolveDatasetsDir(workflowName, basePath);
     if (!dir) {
-        return { datasets: [], dir: resolveDefaultDatasetsDir(workflowName, basePath) };
+        return { datasets: [], dir: await resolveDefaultDatasetsDir(workflowName, basePath) };
     }
     const files = await readdir(dir);
     const ymlFiles = files.filter(f => f.endsWith('.yml') || f.endsWith('.yaml'));
@@ -76,7 +78,7 @@ export async function writeDataset(dataset, filePath) {
     await writeFile(filePath, yaml.dump(fileObj, { lineWidth: 120, noRefs: true, sortKeys: false }), 'utf-8');
 }
 export async function listDatasets(workflowName, basePath) {
-    const dir = resolveDatasetsDir(workflowName, basePath);
+    const dir = await resolveDatasetsDir(workflowName, basePath);
     if (!dir) {
         return [];
     }

package/dist/services/datasets.test.js

@@ -1,12 +1,20 @@
-import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
 import { mkdtemp, rm, writeFile, readFile, mkdir } from 'node:fs/promises';
 import { join } from 'node:path';
 import { tmpdir } from 'node:os';
 import yaml from 'js-yaml';
-import { readDatasetFile, readAllDatasets, writeDataset, listDatasets } from './datasets.js';
+import { readDatasetFile, readAllDatasets, writeDataset, listDatasets, resolveDefaultDatasetsDir } from './datasets.js';
+import * as catalog from '#api/workflow_catalog.js';
+vi.mock('#api/workflow_catalog.js', () => ({
+    fetchWorkflowCatalog: vi.fn()
+}));
 const ctx = { tmpDir: '' };
 beforeEach(async () => {
     ctx.tmpDir = await mkdtemp(join(tmpdir(), 'output-datasets-test-'));
+    // Default: no API. Flat-layout cases resolve offline before this is hit;
+    // nested cases override with a resolved catalog.
+    vi.mocked(catalog.fetchWorkflowCatalog).mockReset();
+    vi.mocked(catalog.fetchWorkflowCatalog).mockRejectedValue(new Error('API unavailable'));
 });
 afterEach(async () => {
     await rm(ctx.tmpDir, { recursive: true, force: true });
@@ -102,6 +110,17 @@ describe('readAllDatasets', () => {
         expect(datasets).toHaveLength(2);
         expect(datasets.map(d => d.name).sort()).toEqual(['case_2', 'case_3']);
     });
+    it('resolves a nested workflow folder via the catalog', async () => {
+        const datasetsDir = join(ctx.tmpDir, 'src', 'workflows', 'a', 'b', 'c', 'tests', 'datasets');
+        await mkdir(datasetsDir, { recursive: true });
+        await writeYaml(join(datasetsDir, 'cases.yml'), {
+            nested_case: { input: { x: 1 } }
+        });
+        vi.mocked(catalog.fetchWorkflowCatalog).mockResolvedValue([{ name: 'a_b_c', path: '/app/dist/workflows/a/b/c/workflow.js' }]);
+        const { datasets, dir } = await readAllDatasets('a_b_c', undefined, ctx.tmpDir);
+        expect(datasets.map(d => d.name)).toEqual(['nested_case']);
+        expect(dir).toBe(datasetsDir);
+    });
     it('returns empty datasets and a default dir when workflow has no datasets dir', async () => {
         const { datasets, dir } = await readAllDatasets('nonexistent_workflow', undefined, ctx.tmpDir);
         expect(datasets).toHaveLength(0);
@@ -200,3 +219,19 @@ describe('listDatasets', () => {
         expect(infos).toHaveLength(0);
     });
 });
+// ---------------------------------------------------------------------------
+// resolveDefaultDatasetsDir
+// ---------------------------------------------------------------------------
+describe('resolveDefaultDatasetsDir', () => {
+    it('returns the nested workflow tests/datasets for first-time generation via catalog', async () => {
+        // Workflow dir exists; its tests/datasets does not yet (first generation).
+        await mkdir(join(ctx.tmpDir, 'src', 'workflows', 'a', 'b', 'c'), { recursive: true });
+        vi.mocked(catalog.fetchWorkflowCatalog).mockResolvedValue([{ name: 'a_b_c', path: '/app/dist/workflows/a/b/c/workflow.js' }]);
+        const dir = await resolveDefaultDatasetsDir('a_b_c', ctx.tmpDir);
+        expect(dir).toBe(join(ctx.tmpDir, 'src', 'workflows', 'a', 'b', 'c', 'tests', 'datasets'));
+    });
+    it('falls back to the flat convention when the workflow cannot be resolved', async () => {
+        const dir = await resolveDefaultDatasetsDir('unknown_flow', ctx.tmpDir);
+        expect(dir).toBe(join(ctx.tmpDir, 'src', 'workflows', 'unknown_flow', 'tests', 'datasets'));
+    });
+});

package/dist/utils/eval_diagnostics.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Explain why a `<wf>_eval` workflow isn't registered. When the eval source
+ * exists on disk but isn't in the catalog, it almost always means tests/evals
+ * never compiled to dist (a tsconfig exclude dropped it) — so point at that
+ * instead of a bare WorkflowNotFoundError.
+ */
+export declare function diagnoseMissingEvalWorkflow(workflowName: string, basePath?: string): Promise<string>;

package/dist/utils/eval_diagnostics.js

@@ -0,0 +1,35 @@
+import { existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { getEvalWorkflowName } from '@outputai/evals';
+import { resolveWorkflowDir } from '#utils/workflow_dir.js';
+const EVAL_WORKFLOW_FILES = ['tests/evals/workflow.ts', 'tests/evals/workflow.js'];
+/**
+ * Explain why a `<wf>_eval` workflow isn't registered. When the eval source
+ * exists on disk but isn't in the catalog, it almost always means tests/evals
+ * never compiled to dist (a tsconfig exclude dropped it) — so point at that
+ * instead of a bare WorkflowNotFoundError.
+ */
+export async function diagnoseMissingEvalWorkflow(workflowName, basePath) {
+    const evalName = getEvalWorkflowName(workflowName);
+    const workflowDir = await resolveWorkflowDir(workflowName, basePath);
+    const evalSource = workflowDir ?
+        EVAL_WORKFLOW_FILES.map(file => resolve(workflowDir, file)).find(existsSync) :
+        undefined;
+    if (evalSource) {
+        return [
+            `Eval workflow "${evalName}" is not registered, but its source exists at:`,
+            `  ${evalSource}`,
+            '',
+            'This usually means tests/evals did not compile to dist. Check your tsconfig:',
+            '  - Ensure tests/evals is not excluded (avoid excluding "src/**/tests").',
+            '  - Prefer excluding "**/*.spec.ts" and "**/*.test.ts" instead.',
+            '',
+            'Rebuild the worker so dist/.../tests/evals/workflow.js exists, then retry.'
+        ].join('\n');
+    }
+    return [
+        `No eval workflow defined for "${workflowName}".`,
+        '',
+        `Create an eval workflow at tests/evals/workflow.ts that registers "${evalName}".`
+    ].join('\n');
+}

package/dist/utils/format_workflow_result.spec.js

@@ -1,6 +1,5 @@
 import { describe, it, expect } from 'vitest';
 import { formatWorkflowResult } from './format_workflow_result.js';
-import { formatOutput } from './output_formatter.js';
 describe('formatWorkflowResult', () => {
     it('should display output for completed workflows', () => {
         const result = formatWorkflowResult({
@@ -76,16 +75,4 @@ describe('formatWorkflowResult', () => {
         expect(result).toContain('Status: failed');
         expect(result).not.toContain('Error:');
     });
-    it('should work with formatOutput for json format', () => {
-        const data = {
-            workflowId: 'wf-456',
-            status: 'failed',
-            output: null,
-            error: 'Activity task failed'
-        };
-        const output = formatOutput(data, 'json', formatWorkflowResult);
-        const parsed = JSON.parse(output);
-        expect(parsed.status).toBe('failed');
-        expect(parsed.error).toBe('Activity task failed');
-    });
 });

package/dist/utils/resolve_input.d.ts

@@ -1 +1 @@
-export declare function resolveInput(workflowName: string, scenario: string | undefined, inputFlag: string | undefined, commandName: string): Promise<unknown>;
+export declare function resolveInput(workflowName: string, scenario: string | undefined, inputFlag: string | undefined, commandName: string, catalog?: string): Promise<unknown>;

package/dist/utils/resolve_input.js

@@ -1,7 +1,7 @@
 import { ux } from '@oclif/core';
 import { parseInputFlag } from '#utils/input_parser.js';
 import { resolveScenarioPath, getScenarioNotFoundMessage } from '#utils/scenario_resolver.js';
-export async function resolveInput(workflowName, scenario, inputFlag, commandName) {
+export async function resolveInput(workflowName, scenario, inputFlag, commandName, catalog) {
     if (inputFlag && scenario) {
         return ux.error('Cannot use both scenario argument and --input flag. Choose one.', { exit: 1 });
     }
@@ -9,7 +9,7 @@ export async function resolveInput(workflowName, scenario, inputFlag, commandNam
         return parseInputFlag(inputFlag);
     }
     if (scenario) {
-        const resolution = await resolveScenarioPath(workflowName, scenario);
+        const resolution = await resolveScenarioPath(workflowName, scenario, undefined, undefined, catalog);
         if (!resolution.found) {
             return ux.error(getScenarioNotFoundMessage(workflowName, scenario, resolution.searchedPaths), { exit: 1 });
         }

package/dist/utils/scenario_resolver.d.ts

@@ -1,10 +1,9 @@
+export { extractWorkflowRelativePath, findWorkflowDirectoryFromPath } from '#utils/workflow_dir.js';
 export interface ScenarioResolutionResult {
     found: boolean;
     path?: string;
     searchedPaths: string[];
 }
-export declare function extractWorkflowRelativePath(path: string): string | null;
-export declare function findWorkflowDirectoryFromPath(workflowPath: string | undefined, basePath?: string): string | null;
-export declare function resolveScenarioPath(workflowName: string, scenarioName: string, basePath?: string, workflowPath?: string): Promise<ScenarioResolutionResult>;
+export declare function resolveScenarioPath(workflowName: string, scenarioName: string, basePath?: string, workflowPath?: string, catalog?: string): Promise<ScenarioResolutionResult>;
 export declare function listScenariosForWorkflow(workflowName: string, workflowPath?: string, basePath?: string): string[];
 export declare function getScenarioNotFoundMessage(workflowName: string, scenarioName: string, searchedPaths: string[]): string;