@outputai/cli 0.1.13-dev.af42b51.0 → 0.1.13-dev.e3e3291.0

package/dist/assets/docker/docker-compose-dev.yml

@@ -81,7 +81,7 @@ services:
         condition: service_healthy
       worker:
         condition: service_healthy
-    image: outputai/api:${OUTPUT_API_VERSION:-0.1.13-dev.af42b51.0}
+    image: outputai/api:${OUTPUT_API_VERSION:-0.1.13-dev.e3e3291.0}
     init: true
     networks:
       - main

package/dist/commands/workflow/test_eval.js

@@ -114,7 +114,7 @@ export default class WorkflowTest extends Command {
                 }
             };
             if (save) {
-                const filePath = join(dir, `${dataset.name}.yml`);
+                const filePath = dataset._source ?? join(dir, `${dataset.name}.yml`);
                 await writeDataset(updated, filePath);
                 this.log(`    Saved output to ${filePath}`);
             }
@@ -137,7 +137,7 @@ export default class WorkflowTest extends Command {
                     date: now
                 }
             };
-            const filePath = join(dir, `${dataset.name}.yml`);
+            const filePath = dataset._source ?? join(dir, `${dataset.name}.yml`);
             await writeDataset(updated, filePath);
             this.log(`  Saved eval result to ${filePath}`);
         }

package/dist/generated/framework_version.json

@@ -1,3 +1,3 @@
 {
-    "framework": "0.1.13-dev.af42b51.0"
+    "framework": "0.1.13-dev.e3e3291.0"
 }

package/dist/services/datasets.d.ts

@@ -8,7 +8,7 @@ export interface DatasetInfo {
 }
 export declare function resolveDatasetsDir(workflowName: string, basePath?: string): string | null;
 export declare function resolveDefaultDatasetsDir(workflowName: string, basePath?: string): string;
-export declare function readDataset(filePath: string): Promise<Dataset>;
+export declare function readDatasetFile(filePath: string): Promise<Dataset[]>;
 export declare function readAllDatasets(workflowName: string, filterNames?: string[], basePath?: string): Promise<{
     datasets: Dataset[];
     dir: string;

package/dist/services/datasets.js

@@ -24,10 +24,19 @@ export function resolveDefaultDatasetsDir(workflowName, basePath = process.cwd()
     // Default to first workflows path
     return resolve(basePath, WORKFLOWS_PATHS[0], workflowName, DATASETS_DIR);
 }
-export async function readDataset(filePath) {
-    const content = await readFile(filePath, 'utf-8');
-    const raw = yaml.load(content);
-    return DatasetSchema.parse(raw);
+export async function readDatasetFile(filePath) {
+    const raw = yaml.load(await readFile(filePath, 'utf-8'));
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
+        throw new Error(`Invalid dataset file: ${filePath}`);
+    }
+    return Object.entries(raw).map(([name, body]) => {
+        if (!body || typeof body !== 'object' || !('input' in body)) {
+            throw new Error(`Dataset case "${name}" in ${filePath} is missing required "input" field`);
+        }
+        const dataset = DatasetSchema.parse({ name, ...body });
+        dataset._source = filePath;
+        return dataset;
+    });
 }
 export async function readAllDatasets(workflowName, filterNames, basePath) {
     const dir = resolveDatasetsDir(workflowName, basePath);
@@ -36,42 +45,35 @@ export async function readAllDatasets(workflowName, filterNames, basePath) {
     }
     const files = await readdir(dir);
     const ymlFiles = files.filter(f => f.endsWith('.yml') || f.endsWith('.yaml'));
+    const seen = new Set();
     const datasets = [];
     for (const file of ymlFiles) {
-        const dataset = await readDataset(join(dir, file));
-        if (filterNames && !filterNames.includes(dataset.name)) {
-            continue;
+        const cases = await readDatasetFile(join(dir, file));
+        for (const dataset of cases) {
+            if (seen.has(dataset.name)) {
+                throw new Error(`Duplicate dataset case name "${dataset.name}" found in ${file}`);
+            }
+            seen.add(dataset.name);
+            if (filterNames && !filterNames.includes(dataset.name)) {
+                continue;
+            }
+            datasets.push(dataset);
         }
-        datasets.push(dataset);
     }
     return { datasets, dir };
 }
-async function mergeWithExisting(dataset, filePath) {
-    if (!existsSync(filePath)) {
-        return dataset;
-    }
-    try {
-        const existing = await readDataset(filePath);
-        return {
-            ...existing,
-            ...dataset,
-            ground_truth: dataset.ground_truth ?? existing.ground_truth,
-            last_output: dataset.last_output ?? existing.last_output,
-            last_eval: dataset.last_eval ?? existing.last_eval
-        };
-    }
-    catch {
-        return dataset;
-    }
-}
 export async function writeDataset(dataset, filePath) {
-    const merged = await mergeWithExisting(dataset, filePath);
     const dir = resolve(filePath, '..');
     if (!existsSync(dir)) {
         await mkdir(dir, { recursive: true });
     }
-    const content = yaml.dump(merged, { lineWidth: 120, noRefs: true, sortKeys: false });
-    await writeFile(filePath, content, 'utf-8');
+    const loaded = existsSync(filePath) ? yaml.load(await readFile(filePath, 'utf-8')) : null;
+    const fileObj = (loaded && typeof loaded === 'object' && !Array.isArray(loaded)) ?
+        loaded :
+        {};
+    const { name, _source, ...caseBody } = dataset;
+    fileObj[name] = { ...fileObj[name], ...caseBody };
+    await writeFile(filePath, yaml.dump(fileObj, { lineWidth: 120, noRefs: true, sortKeys: false }), 'utf-8');
 }
 export async function listDatasets(workflowName, basePath) {
     const dir = resolveDatasetsDir(workflowName, basePath);
@@ -84,14 +86,16 @@ export async function listDatasets(workflowName, basePath) {
     for (const file of ymlFiles) {
         const filePath = join(dir, file);
         try {
-            const dataset = await readDataset(filePath);
-            results.push({
-                name: dataset.name,
-                path: filePath,
-                hasLastOutput: dataset.last_output?.output !== undefined,
-                lastOutputDate: dataset.last_output?.date,
-                lastEvalDate: dataset.last_eval?.date
-            });
+            const cases = await readDatasetFile(filePath);
+            for (const dataset of cases) {
+                results.push({
+                    name: dataset.name,
+                    path: filePath,
+                    hasLastOutput: dataset.last_output?.output !== undefined,
+                    lastOutputDate: dataset.last_output?.date,
+                    lastEvalDate: dataset.last_eval?.date
+                });
+            }
         }
         catch {
             results.push({

package/dist/services/datasets.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/services/datasets.test.js

@@ -0,0 +1,202 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdtemp, rm, writeFile, readFile, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import yaml from 'js-yaml';
+import { readDatasetFile, readAllDatasets, writeDataset, listDatasets } from './datasets.js';
+const ctx = { tmpDir: '' };
+beforeEach(async () => {
+    ctx.tmpDir = await mkdtemp(join(tmpdir(), 'output-datasets-test-'));
+});
+afterEach(async () => {
+    await rm(ctx.tmpDir, { recursive: true, force: true });
+});
+function writeYaml(filePath, obj) {
+    return writeFile(filePath, yaml.dump(obj, { lineWidth: 120, noRefs: true, sortKeys: false }), 'utf-8');
+}
+// ---------------------------------------------------------------------------
+// readDatasetFile
+// ---------------------------------------------------------------------------
+describe('readDatasetFile', () => {
+    it('parses a multi-case file and returns all cases', async () => {
+        const filePath = join(ctx.tmpDir, 'cases.yml');
+        await writeYaml(filePath, {
+            case_a: { input: { query: 'foo' }, ground_truth: { expected: 1 } },
+            case_b: { input: { query: 'bar' } },
+            case_c: { input: { query: 'baz' }, ground_truth: { expected: 3 } }
+        });
+        const datasets = await readDatasetFile(filePath);
+        expect(datasets).toHaveLength(3);
+        expect(datasets.map(d => d.name)).toEqual(['case_a', 'case_b', 'case_c']);
+        expect(datasets[0].input).toEqual({ query: 'foo' });
+        expect(datasets[0].ground_truth).toEqual({ expected: 1 });
+    });
+    it('attaches _source with the absolute file path to each dataset', async () => {
+        const filePath = join(ctx.tmpDir, 'cases.yml');
+        await writeYaml(filePath, {
+            my_case: { input: { x: 1 } }
+        });
+        const [dataset] = await readDatasetFile(filePath);
+        expect(dataset._source).toBe(filePath);
+    });
+    it('throws when a case is missing input', async () => {
+        const filePath = join(ctx.tmpDir, 'bad.yml');
+        await writeYaml(filePath, {
+            good_case: { input: { x: 1 } },
+            bad_case: { ground_truth: { expected: 42 } }
+        });
+        await expect(readDatasetFile(filePath)).rejects.toThrow('Dataset case "bad_case" in');
+    });
+    it('throws when file content is not an object', async () => {
+        const filePath = join(ctx.tmpDir, 'bad.yml');
+        await writeFile(filePath, 'just a string', 'utf-8');
+        await expect(readDatasetFile(filePath)).rejects.toThrow('Invalid dataset file');
+    });
+    it('throws with clear message when file content is a YAML array', async () => {
+        const filePath = join(ctx.tmpDir, 'bad.yml');
+        await writeFile(filePath, '- foo\n- bar\n', 'utf-8');
+        await expect(readDatasetFile(filePath)).rejects.toThrow('Invalid dataset file');
+    });
+    it('preserves last_output and last_eval fields', async () => {
+        const filePath = join(ctx.tmpDir, 'cases.yml');
+        await writeYaml(filePath, {
+            cached_case: {
+                input: { q: 'hello' },
+                last_output: { output: { result: 42 }, executionTimeMs: 100, date: '2026-01-01T00:00:00.000Z' }
+            }
+        });
+        const [dataset] = await readDatasetFile(filePath);
+        expect(dataset.last_output?.output).toEqual({ result: 42 });
+        expect(dataset.last_output?.executionTimeMs).toBe(100);
+    });
+});
+// ---------------------------------------------------------------------------
+// readAllDatasets
+// ---------------------------------------------------------------------------
+describe('readAllDatasets', () => {
+    it('flattens cases from multiple files', async () => {
+        const datasetsDir = join(ctx.tmpDir, 'src', 'workflows', 'my_workflow', 'tests', 'datasets');
+        await mkdir(datasetsDir, { recursive: true });
+        await writeYaml(join(datasetsDir, 'group_a.yml'), {
+            case_1: { input: { x: 1 } },
+            case_2: { input: { x: 2 } }
+        });
+        await writeYaml(join(datasetsDir, 'group_b.yml'), {
+            case_3: { input: { x: 3 } }
+        });
+        const { datasets } = await readAllDatasets('my_workflow', undefined, ctx.tmpDir);
+        expect(datasets).toHaveLength(3);
+        expect(datasets.map(d => d.name).sort()).toEqual(['case_1', 'case_2', 'case_3']);
+    });
+    it('filters by case name across files', async () => {
+        const datasetsDir = join(ctx.tmpDir, 'src', 'workflows', 'my_workflow', 'tests', 'datasets');
+        await mkdir(datasetsDir, { recursive: true });
+        await writeYaml(join(datasetsDir, 'group_a.yml'), {
+            case_1: { input: { x: 1 } },
+            case_2: { input: { x: 2 } }
+        });
+        await writeYaml(join(datasetsDir, 'group_b.yml'), {
+            case_3: { input: { x: 3 } }
+        });
+        const { datasets } = await readAllDatasets('my_workflow', ['case_2', 'case_3'], ctx.tmpDir);
+        expect(datasets).toHaveLength(2);
+        expect(datasets.map(d => d.name).sort()).toEqual(['case_2', 'case_3']);
+    });
+    it('returns empty datasets and a default dir when workflow has no datasets dir', async () => {
+        const { datasets, dir } = await readAllDatasets('nonexistent_workflow', undefined, ctx.tmpDir);
+        expect(datasets).toHaveLength(0);
+        expect(dir).toContain('nonexistent_workflow');
+    });
+    it('throws when the same case name appears in two different files', async () => {
+        const datasetsDir = join(ctx.tmpDir, 'src', 'workflows', 'my_workflow', 'tests', 'datasets');
+        await mkdir(datasetsDir, { recursive: true });
+        await writeYaml(join(datasetsDir, 'group_a.yml'), { case_1: { input: { x: 1 } } });
+        await writeYaml(join(datasetsDir, 'group_b.yml'), { case_1: { input: { x: 2 } } });
+        await expect(readAllDatasets('my_workflow', undefined, ctx.tmpDir)).rejects.toThrow('Duplicate dataset case name "case_1"');
+    });
+});
+// ---------------------------------------------------------------------------
+// writeDataset
+// ---------------------------------------------------------------------------
+describe('writeDataset', () => {
+    it('creates a new file with one case keyed by name', async () => {
+        const filePath = join(ctx.tmpDir, 'cases.yml');
+        const dataset = { name: 'new_case', input: { q: 'hello' } };
+        await writeDataset(dataset, filePath);
+        const raw = yaml.load(await readFile(filePath, 'utf-8'));
+        expect(raw).toHaveProperty('new_case');
+        expect(raw.new_case.input).toEqual({ q: 'hello' });
+        expect(raw.new_case).not.toHaveProperty('name');
+    });
+    it('does not write _source into the file', async () => {
+        const filePath = join(ctx.tmpDir, 'cases.yml');
+        const dataset = { name: 'my_case', input: { q: 'x' }, _source: '/some/path.yml' };
+        await writeDataset(dataset, filePath);
+        const raw = yaml.load(await readFile(filePath, 'utf-8'));
+        expect(raw.my_case).not.toHaveProperty('_source');
+    });
+    it('updates only the target case, leaving other cases untouched', async () => {
+        const filePath = join(ctx.tmpDir, 'cases.yml');
+        await writeYaml(filePath, {
+            case_a: { input: { x: 1 }, ground_truth: { expected: 1 } },
+            case_b: { input: { x: 2 }, ground_truth: { expected: 2 } }
+        });
+        await writeDataset({ name: 'case_a', input: { x: 1 }, last_output: { output: { result: 1 }, date: '2026-01-01T00:00:00.000Z' } }, filePath);
+        const raw = yaml.load(await readFile(filePath, 'utf-8'));
+        expect(raw).toHaveProperty('case_b');
+        expect(raw.case_b.ground_truth).toEqual({ expected: 2 });
+    });
+    it('preserves existing fields when writing last_output then last_eval', async () => {
+        const filePath = join(ctx.tmpDir, 'cases.yml');
+        await writeYaml(filePath, {
+            my_case: { input: { q: 'hello' }, ground_truth: { expected: 42 } }
+        });
+        await writeDataset({ name: 'my_case', input: { q: 'hello' }, last_output: { output: { result: 42 }, executionTimeMs: 50, date: '2026-01-01T00:00:00.000Z' } }, filePath);
+        await writeDataset({
+            name: 'my_case', input: { q: 'hello' },
+            last_eval: { output: { datasetName: 'my_case', verdict: 'pass', evaluators: [] }, date: '2026-01-01T00:01:00.000Z' }
+        }, filePath);
+        const raw = yaml.load(await readFile(filePath, 'utf-8'));
+        const caseObj = raw.my_case;
+        expect(caseObj).toHaveProperty('last_output');
+        expect(caseObj).toHaveProperty('last_eval');
+        expect(caseObj).toHaveProperty('ground_truth');
+    });
+    it('creates parent directories if they do not exist', async () => {
+        const filePath = join(ctx.tmpDir, 'deep', 'nested', 'cases.yml');
+        await writeDataset({ name: 'my_case', input: { q: 'x' } }, filePath);
+        const raw = yaml.load(await readFile(filePath, 'utf-8'));
+        expect(raw).toHaveProperty('my_case');
+    });
+    it('recovers gracefully when existing file contains non-object YAML', async () => {
+        const filePath = join(ctx.tmpDir, 'cases.yml');
+        await writeFile(filePath, 'just a string', 'utf-8');
+        await writeDataset({ name: 'my_case', input: { q: 'x' } }, filePath);
+        const raw = yaml.load(await readFile(filePath, 'utf-8'));
+        expect(raw).toHaveProperty('my_case');
+    });
+});
+// ---------------------------------------------------------------------------
+// listDatasets
+// ---------------------------------------------------------------------------
+describe('listDatasets', () => {
+    it('returns one DatasetInfo per case across all files', async () => {
+        const datasetsDir = join(ctx.tmpDir, 'src', 'workflows', 'my_workflow', 'tests', 'datasets');
+        await mkdir(datasetsDir, { recursive: true });
+        await writeYaml(join(datasetsDir, 'core.yml'), {
+            case_1: { input: { x: 1 }, last_output: { output: { r: 1 }, date: '2026-01-01T00:00:00.000Z' } },
+            case_2: { input: { x: 2 } }
+        });
+        const infos = await listDatasets('my_workflow', ctx.tmpDir);
+        expect(infos).toHaveLength(2);
+        const case1 = infos.find(i => i.name === 'case_1');
+        expect(case1.hasLastOutput).toBe(true);
+        expect(case1.path).toContain('core.yml');
+        const case2 = infos.find(i => i.name === 'case_2');
+        expect(case2.hasLastOutput).toBe(false);
+    });
+    it('returns empty array when no datasets directory exists', async () => {
+        const infos = await listDatasets('nonexistent_workflow', ctx.tmpDir);
+        expect(infos).toHaveLength(0);
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/cli",
-  "version": "0.1.13-dev.af42b51.0",
+  "version": "0.1.13-dev.e3e3291.0",
   "description": "CLI for Output.ai workflow generation",
   "type": "module",
   "main": "dist/index.js",
@@ -35,9 +35,9 @@
     "react": "19.2.4",
     "semver": "7.7.4",
     "yaml": "^2.8.3",
-    "@outputai/credentials": "0.1.13-dev.af42b51.0",
-    "@outputai/evals": "0.1.13-dev.af42b51.0",
-    "@outputai/llm": "0.1.13-dev.af42b51.0"
+    "@outputai/evals": "0.1.13-dev.e3e3291.0",
+    "@outputai/credentials": "0.1.13-dev.e3e3291.0",
+    "@outputai/llm": "0.1.13-dev.e3e3291.0"
   },
   "devDependencies": {
     "@types/cli-progress": "3.11.6",