@outputai/cli 0.1.13-next.c93cdb4.0 → 0.1.13-next.cc1ead7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/generated/api.d.ts +322 -84
- package/dist/api/generated/api.js +67 -9
- package/dist/assets/docker/docker-compose-dev.yml +2 -2
- package/dist/commands/migrate.d.ts +11 -0
- package/dist/commands/migrate.js +40 -0
- package/dist/commands/workflow/generate.js +3 -1
- package/dist/commands/workflow/generate.spec.js +12 -0
- package/dist/commands/workflow/list.d.ts +1 -0
- package/dist/commands/workflow/list.js +12 -6
- package/dist/commands/workflow/list.spec.js +21 -0
- package/dist/commands/workflow/plan.js +1 -1
- package/dist/commands/workflow/run.js +2 -1
- package/dist/commands/workflow/run.spec.js +1 -0
- package/dist/commands/workflow/start.js +2 -1
- package/dist/commands/workflow/start.spec.js +1 -0
- package/dist/commands/workflow/test_eval.js +2 -2
- package/dist/components/command_footer.d.ts +8 -0
- package/dist/components/command_footer.js +4 -0
- package/dist/components/status_icon.d.ts +11 -0
- package/dist/components/status_icon.js +25 -0
- package/dist/components/workflow_summary.d.ts +10 -0
- package/dist/components/workflow_summary.js +4 -0
- package/dist/generated/framework_version.json +1 -1
- package/dist/services/claude_client.d.ts +14 -2
- package/dist/services/claude_client.integration.test.js +2 -2
- package/dist/services/claude_client.js +42 -6
- package/dist/services/claude_client.spec.js +3 -3
- package/dist/services/datasets.d.ts +1 -1
- package/dist/services/datasets.js +41 -37
- package/dist/services/datasets.test.d.ts +1 -0
- package/dist/services/datasets.test.js +202 -0
- package/dist/services/docker.d.ts +1 -3
- package/dist/services/docker.js +38 -13
- package/dist/services/messages.d.ts +1 -1
- package/dist/services/messages.js +2 -2
- package/dist/services/workflow_builder.d.ts +1 -1
- package/dist/services/workflow_builder.js +1 -1
- package/dist/templates/agent_instructions/CLAUDE.md.template +36 -2
- package/dist/templates/agent_instructions/dotclaude/settings.json.template +2 -2
- package/dist/utils/date_formatter.d.ts +11 -1
- package/dist/utils/date_formatter.js +26 -1
- package/dist/utils/format_workflow_result.d.ts +3 -3
- package/dist/utils/open_url.d.ts +1 -0
- package/dist/utils/open_url.js +12 -0
- package/dist/utils/workflow_dir_parser.d.ts +5 -0
- package/dist/utils/workflow_dir_parser.js +39 -0
- package/dist/utils/workflow_dir_parser.spec.d.ts +1 -0
- package/dist/utils/workflow_dir_parser.spec.js +74 -0
- package/dist/views/dev.js +62 -26
- package/dist/views/workflow/list.d.ts +6 -0
- package/dist/views/workflow/list.js +127 -0
- package/package.json +14 -14
|
@@ -38,10 +38,25 @@ date: <plan-date>
|
|
|
38
38
|
3. Implement all workflow files following Output.ai patterns and best practices.
|
|
39
39
|
|
|
40
40
|
4. After you mark all todos as complete, provide a summary of what was implemented.
|
|
41
|
+
`,
|
|
42
|
+
MIGRATE: `
|
|
43
|
+
! IMPORTANT !
|
|
44
|
+
1. Use TodoWrite to track your progress through the migration.
|
|
45
|
+
|
|
46
|
+
2. Fetch the migration guide from https://docs.output.ai/migrations — do not invent migration steps from memory.
|
|
47
|
+
|
|
48
|
+
3. If the specific guide URL 404s, fall back to the /migrations index and chain the guides that cover the version range.
|
|
49
|
+
|
|
50
|
+
4. Confirm the planned changes with the user before editing files.
|
|
51
|
+
|
|
52
|
+
5. After you mark all todos as complete, provide a summary of which files changed and whether the type check passed.
|
|
41
53
|
`
|
|
42
54
|
};
|
|
43
|
-
|
|
44
|
-
|
|
55
|
+
// Slash-command naming convention used by the outputai plugin:
|
|
56
|
+
// `output-<kebab-name>` — skills under coding_assistants/.../skills/, which surface as top-level slash commands without the plugin prefix.
|
|
57
|
+
const PLAN_COMMAND = 'output-plan-workflow';
|
|
58
|
+
const BUILD_COMMAND = 'output-build-workflow';
|
|
59
|
+
const MIGRATE_COMMAND = 'output-migrate';
|
|
45
60
|
const GLOBAL_CLAUDE_OPTIONS = {
|
|
46
61
|
settingSources: ['user', 'project', 'local']
|
|
47
62
|
};
|
|
@@ -51,6 +66,9 @@ export const PLAN_COMMAND_OPTIONS = {
|
|
|
51
66
|
export const BUILD_COMMAND_OPTIONS = {
|
|
52
67
|
permissionMode: 'bypassPermissions'
|
|
53
68
|
};
|
|
69
|
+
export const MIGRATE_COMMAND_OPTIONS = {
|
|
70
|
+
permissionMode: 'bypassPermissions'
|
|
71
|
+
};
|
|
54
72
|
export class ClaudeInvocationError extends Error {
|
|
55
73
|
cause;
|
|
56
74
|
constructor(message, cause) {
|
|
@@ -77,7 +95,7 @@ function validateEnvironment() {
|
|
|
77
95
|
}
|
|
78
96
|
}
|
|
79
97
|
function validateSystem(systemMessage) {
|
|
80
|
-
const requiredCommands = [PLAN_COMMAND, BUILD_COMMAND];
|
|
98
|
+
const requiredCommands = [PLAN_COMMAND, BUILD_COMMAND, MIGRATE_COMMAND];
|
|
81
99
|
const availableCommands = systemMessage.slash_commands;
|
|
82
100
|
const missingCommands = requiredCommands.filter(command => !availableCommands.includes(command));
|
|
83
101
|
return {
|
|
@@ -105,7 +123,10 @@ function getTodoWriteMessage(message) {
|
|
|
105
123
|
if (message.type !== 'assistant') {
|
|
106
124
|
return null;
|
|
107
125
|
}
|
|
108
|
-
const todoWriteMessage = message.message.content.find((c) =>
|
|
126
|
+
const todoWriteMessage = message.message.content.find((c) => {
|
|
127
|
+
const block = c;
|
|
128
|
+
return block.type === 'tool_use' && block.name === 'TodoWrite';
|
|
129
|
+
});
|
|
109
130
|
return todoWriteMessage ?? null;
|
|
110
131
|
}
|
|
111
132
|
function applyInstructions(message, instructions) {
|
|
@@ -190,7 +211,7 @@ export async function replyToClaude(message, { anthropicOpts, applyAdditionalIns
|
|
|
190
211
|
return singleQuery(applyInstructions(message, applyAdditionalInstructions), { continue: true, ...anthropicOpts });
|
|
191
212
|
}
|
|
192
213
|
/**
|
|
193
|
-
* Invoke claude-code with /
|
|
214
|
+
* Invoke claude-code with /output-plan-workflow slash command
|
|
194
215
|
* The SDK loads custom commands from .claude/commands/ when settingSources includes 'project'.
|
|
195
216
|
* ensureOutputAISystem() scaffolds the command files to that location.
|
|
196
217
|
* @param description - Workflow description
|
|
@@ -200,7 +221,7 @@ export async function invokePlanWorkflow(description) {
|
|
|
200
221
|
return singleQuery(applyInstructions(`/${PLAN_COMMAND} ${description}`, ADDITIONAL_INSTRUCTIONS.PLAN), PLAN_COMMAND_OPTIONS);
|
|
201
222
|
}
|
|
202
223
|
/**
|
|
203
|
-
* Invoke claude-code with /
|
|
224
|
+
* Invoke claude-code with /output-build-workflow slash command
|
|
204
225
|
* The SDK loads custom commands from .claude/commands/ when settingSources includes 'project'.
|
|
205
226
|
* ensureOutputAISystem() scaffolds the command files to that location.
|
|
206
227
|
* @param planFilePath - Absolute path to the plan file
|
|
@@ -216,3 +237,18 @@ export async function invokeBuildWorkflow(planFilePath, workflowDir, workflowNam
|
|
|
216
237
|
`/${BUILD_COMMAND} ${commandArgs}`;
|
|
217
238
|
return singleQuery(applyInstructions(fullCommand, ADDITIONAL_INSTRUCTIONS.BUILD), BUILD_COMMAND_OPTIONS);
|
|
218
239
|
}
|
|
240
|
+
/**
|
|
241
|
+
* Invoke claude-code with /output-migrate slash command (registered via the output-migrate skill).
|
|
242
|
+
* The slash command fetches migration instructions from docs.output.ai —
|
|
243
|
+
* this CLI wrapper just passes through the version arguments.
|
|
244
|
+
* @param fromVersion - Current framework version (empty string = auto-detect)
|
|
245
|
+
* @param toVersion - Target version (empty string = use npm "latest")
|
|
246
|
+
* @param additionalInstructions - Optional user-supplied guidance
|
|
247
|
+
* @returns Migration summary from claude-code
|
|
248
|
+
*/
|
|
249
|
+
export async function invokeMigrate(fromVersion, toVersion, additionalInstructions) {
|
|
250
|
+
const from = fromVersion || 'auto';
|
|
251
|
+
const to = toVersion || 'latest';
|
|
252
|
+
const commandArgs = [from, to, additionalInstructions].filter(Boolean).join(' ');
|
|
253
|
+
return singleQuery(applyInstructions(`/${MIGRATE_COMMAND} ${commandArgs}`, ADDITIONAL_INSTRUCTIONS.MIGRATE), MIGRATE_COMMAND_OPTIONS);
|
|
254
|
+
}
|
|
@@ -13,7 +13,7 @@ describe('invokePlanWorkflow', () => {
|
|
|
13
13
|
// Clean up environment variables
|
|
14
14
|
delete process.env.ANTHROPIC_API_KEY;
|
|
15
15
|
});
|
|
16
|
-
it('should invoke /
|
|
16
|
+
it('should invoke /output-plan-workflow slash command with settingSources', async () => {
|
|
17
17
|
const { query } = await import('@anthropic-ai/claude-agent-sdk');
|
|
18
18
|
process.env.ANTHROPIC_API_KEY = 'test-key';
|
|
19
19
|
async function* mockIterator() {
|
|
@@ -22,7 +22,7 @@ describe('invokePlanWorkflow', () => {
|
|
|
22
22
|
vi.mocked(query).mockReturnValue(mockIterator());
|
|
23
23
|
await invokePlanWorkflow('Test workflow');
|
|
24
24
|
const calls = vi.mocked(query).mock.calls;
|
|
25
|
-
expect(calls[0]?.[0]?.prompt).toContain('/
|
|
25
|
+
expect(calls[0]?.[0]?.prompt).toContain('/output-plan-workflow Test workflow');
|
|
26
26
|
expect(calls[0]?.[0]?.options?.settingSources).toEqual(['user', 'project', 'local']);
|
|
27
27
|
expect(calls[0]?.[0]?.options?.allowedTools).toEqual(['Read', 'Grep', 'WebSearch', 'WebFetch', 'TodoWrite']);
|
|
28
28
|
});
|
|
@@ -36,7 +36,7 @@ describe('invokePlanWorkflow', () => {
|
|
|
36
36
|
const description = 'Build a user authentication system';
|
|
37
37
|
await invokePlanWorkflow(description);
|
|
38
38
|
const calls = vi.mocked(query).mock.calls;
|
|
39
|
-
expect(calls[0]?.[0]?.prompt).toContain(`/
|
|
39
|
+
expect(calls[0]?.[0]?.prompt).toContain(`/output-plan-workflow ${description}`);
|
|
40
40
|
expect(calls[0]?.[0]?.options?.settingSources).toEqual(['user', 'project', 'local']);
|
|
41
41
|
});
|
|
42
42
|
it('should return plan output from claude-code', async () => {
|
|
@@ -8,7 +8,7 @@ export interface DatasetInfo {
|
|
|
8
8
|
}
|
|
9
9
|
export declare function resolveDatasetsDir(workflowName: string, basePath?: string): string | null;
|
|
10
10
|
export declare function resolveDefaultDatasetsDir(workflowName: string, basePath?: string): string;
|
|
11
|
-
export declare function
|
|
11
|
+
export declare function readDatasetFile(filePath: string): Promise<Dataset[]>;
|
|
12
12
|
export declare function readAllDatasets(workflowName: string, filterNames?: string[], basePath?: string): Promise<{
|
|
13
13
|
datasets: Dataset[];
|
|
14
14
|
dir: string;
|
|
@@ -24,10 +24,19 @@ export function resolveDefaultDatasetsDir(workflowName, basePath = process.cwd()
|
|
|
24
24
|
// Default to first workflows path
|
|
25
25
|
return resolve(basePath, WORKFLOWS_PATHS[0], workflowName, DATASETS_DIR);
|
|
26
26
|
}
|
|
27
|
-
export async function
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
27
|
+
export async function readDatasetFile(filePath) {
|
|
28
|
+
const raw = yaml.load(await readFile(filePath, 'utf-8'));
|
|
29
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
|
30
|
+
throw new Error(`Invalid dataset file: ${filePath}`);
|
|
31
|
+
}
|
|
32
|
+
return Object.entries(raw).map(([name, body]) => {
|
|
33
|
+
if (!body || typeof body !== 'object' || !('input' in body)) {
|
|
34
|
+
throw new Error(`Dataset case "${name}" in ${filePath} is missing required "input" field`);
|
|
35
|
+
}
|
|
36
|
+
const dataset = DatasetSchema.parse({ name, ...body });
|
|
37
|
+
dataset._source = filePath;
|
|
38
|
+
return dataset;
|
|
39
|
+
});
|
|
31
40
|
}
|
|
32
41
|
export async function readAllDatasets(workflowName, filterNames, basePath) {
|
|
33
42
|
const dir = resolveDatasetsDir(workflowName, basePath);
|
|
@@ -36,42 +45,35 @@ export async function readAllDatasets(workflowName, filterNames, basePath) {
|
|
|
36
45
|
}
|
|
37
46
|
const files = await readdir(dir);
|
|
38
47
|
const ymlFiles = files.filter(f => f.endsWith('.yml') || f.endsWith('.yaml'));
|
|
48
|
+
const seen = new Set();
|
|
39
49
|
const datasets = [];
|
|
40
50
|
for (const file of ymlFiles) {
|
|
41
|
-
const
|
|
42
|
-
|
|
43
|
-
|
|
51
|
+
const cases = await readDatasetFile(join(dir, file));
|
|
52
|
+
for (const dataset of cases) {
|
|
53
|
+
if (seen.has(dataset.name)) {
|
|
54
|
+
throw new Error(`Duplicate dataset case name "${dataset.name}" found in ${file}`);
|
|
55
|
+
}
|
|
56
|
+
seen.add(dataset.name);
|
|
57
|
+
if (filterNames && !filterNames.includes(dataset.name)) {
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
datasets.push(dataset);
|
|
44
61
|
}
|
|
45
|
-
datasets.push(dataset);
|
|
46
62
|
}
|
|
47
63
|
return { datasets, dir };
|
|
48
64
|
}
|
|
49
|
-
async function mergeWithExisting(dataset, filePath) {
|
|
50
|
-
if (!existsSync(filePath)) {
|
|
51
|
-
return dataset;
|
|
52
|
-
}
|
|
53
|
-
try {
|
|
54
|
-
const existing = await readDataset(filePath);
|
|
55
|
-
return {
|
|
56
|
-
...existing,
|
|
57
|
-
...dataset,
|
|
58
|
-
ground_truth: dataset.ground_truth ?? existing.ground_truth,
|
|
59
|
-
last_output: dataset.last_output ?? existing.last_output,
|
|
60
|
-
last_eval: dataset.last_eval ?? existing.last_eval
|
|
61
|
-
};
|
|
62
|
-
}
|
|
63
|
-
catch {
|
|
64
|
-
return dataset;
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
65
|
export async function writeDataset(dataset, filePath) {
|
|
68
|
-
const merged = await mergeWithExisting(dataset, filePath);
|
|
69
66
|
const dir = resolve(filePath, '..');
|
|
70
67
|
if (!existsSync(dir)) {
|
|
71
68
|
await mkdir(dir, { recursive: true });
|
|
72
69
|
}
|
|
73
|
-
const
|
|
74
|
-
|
|
70
|
+
const loaded = existsSync(filePath) ? yaml.load(await readFile(filePath, 'utf-8')) : null;
|
|
71
|
+
const fileObj = (loaded && typeof loaded === 'object' && !Array.isArray(loaded)) ?
|
|
72
|
+
loaded :
|
|
73
|
+
{};
|
|
74
|
+
const { name, _source, ...caseBody } = dataset;
|
|
75
|
+
fileObj[name] = { ...fileObj[name], ...caseBody };
|
|
76
|
+
await writeFile(filePath, yaml.dump(fileObj, { lineWidth: 120, noRefs: true, sortKeys: false }), 'utf-8');
|
|
75
77
|
}
|
|
76
78
|
export async function listDatasets(workflowName, basePath) {
|
|
77
79
|
const dir = resolveDatasetsDir(workflowName, basePath);
|
|
@@ -84,14 +86,16 @@ export async function listDatasets(workflowName, basePath) {
|
|
|
84
86
|
for (const file of ymlFiles) {
|
|
85
87
|
const filePath = join(dir, file);
|
|
86
88
|
try {
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
89
|
+
const cases = await readDatasetFile(filePath);
|
|
90
|
+
for (const dataset of cases) {
|
|
91
|
+
results.push({
|
|
92
|
+
name: dataset.name,
|
|
93
|
+
path: filePath,
|
|
94
|
+
hasLastOutput: dataset.last_output?.output !== undefined,
|
|
95
|
+
lastOutputDate: dataset.last_output?.date,
|
|
96
|
+
lastEvalDate: dataset.last_eval?.date
|
|
97
|
+
});
|
|
98
|
+
}
|
|
95
99
|
}
|
|
96
100
|
catch {
|
|
97
101
|
results.push({
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { mkdtemp, rm, writeFile, readFile, mkdir } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import yaml from 'js-yaml';
|
|
6
|
+
import { readDatasetFile, readAllDatasets, writeDataset, listDatasets } from './datasets.js';
|
|
7
|
+
const ctx = { tmpDir: '' };
|
|
8
|
+
beforeEach(async () => {
|
|
9
|
+
ctx.tmpDir = await mkdtemp(join(tmpdir(), 'output-datasets-test-'));
|
|
10
|
+
});
|
|
11
|
+
afterEach(async () => {
|
|
12
|
+
await rm(ctx.tmpDir, { recursive: true, force: true });
|
|
13
|
+
});
|
|
14
|
+
function writeYaml(filePath, obj) {
|
|
15
|
+
return writeFile(filePath, yaml.dump(obj, { lineWidth: 120, noRefs: true, sortKeys: false }), 'utf-8');
|
|
16
|
+
}
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
// readDatasetFile
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
describe('readDatasetFile', () => {
|
|
21
|
+
it('parses a multi-case file and returns all cases', async () => {
|
|
22
|
+
const filePath = join(ctx.tmpDir, 'cases.yml');
|
|
23
|
+
await writeYaml(filePath, {
|
|
24
|
+
case_a: { input: { query: 'foo' }, ground_truth: { expected: 1 } },
|
|
25
|
+
case_b: { input: { query: 'bar' } },
|
|
26
|
+
case_c: { input: { query: 'baz' }, ground_truth: { expected: 3 } }
|
|
27
|
+
});
|
|
28
|
+
const datasets = await readDatasetFile(filePath);
|
|
29
|
+
expect(datasets).toHaveLength(3);
|
|
30
|
+
expect(datasets.map(d => d.name)).toEqual(['case_a', 'case_b', 'case_c']);
|
|
31
|
+
expect(datasets[0].input).toEqual({ query: 'foo' });
|
|
32
|
+
expect(datasets[0].ground_truth).toEqual({ expected: 1 });
|
|
33
|
+
});
|
|
34
|
+
it('attaches _source with the absolute file path to each dataset', async () => {
|
|
35
|
+
const filePath = join(ctx.tmpDir, 'cases.yml');
|
|
36
|
+
await writeYaml(filePath, {
|
|
37
|
+
my_case: { input: { x: 1 } }
|
|
38
|
+
});
|
|
39
|
+
const [dataset] = await readDatasetFile(filePath);
|
|
40
|
+
expect(dataset._source).toBe(filePath);
|
|
41
|
+
});
|
|
42
|
+
it('throws when a case is missing input', async () => {
|
|
43
|
+
const filePath = join(ctx.tmpDir, 'bad.yml');
|
|
44
|
+
await writeYaml(filePath, {
|
|
45
|
+
good_case: { input: { x: 1 } },
|
|
46
|
+
bad_case: { ground_truth: { expected: 42 } }
|
|
47
|
+
});
|
|
48
|
+
await expect(readDatasetFile(filePath)).rejects.toThrow('Dataset case "bad_case" in');
|
|
49
|
+
});
|
|
50
|
+
it('throws when file content is not an object', async () => {
|
|
51
|
+
const filePath = join(ctx.tmpDir, 'bad.yml');
|
|
52
|
+
await writeFile(filePath, 'just a string', 'utf-8');
|
|
53
|
+
await expect(readDatasetFile(filePath)).rejects.toThrow('Invalid dataset file');
|
|
54
|
+
});
|
|
55
|
+
it('throws with clear message when file content is a YAML array', async () => {
|
|
56
|
+
const filePath = join(ctx.tmpDir, 'bad.yml');
|
|
57
|
+
await writeFile(filePath, '- foo\n- bar\n', 'utf-8');
|
|
58
|
+
await expect(readDatasetFile(filePath)).rejects.toThrow('Invalid dataset file');
|
|
59
|
+
});
|
|
60
|
+
it('preserves last_output and last_eval fields', async () => {
|
|
61
|
+
const filePath = join(ctx.tmpDir, 'cases.yml');
|
|
62
|
+
await writeYaml(filePath, {
|
|
63
|
+
cached_case: {
|
|
64
|
+
input: { q: 'hello' },
|
|
65
|
+
last_output: { output: { result: 42 }, executionTimeMs: 100, date: '2026-01-01T00:00:00.000Z' }
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
const [dataset] = await readDatasetFile(filePath);
|
|
69
|
+
expect(dataset.last_output?.output).toEqual({ result: 42 });
|
|
70
|
+
expect(dataset.last_output?.executionTimeMs).toBe(100);
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
// ---------------------------------------------------------------------------
|
|
74
|
+
// readAllDatasets
|
|
75
|
+
// ---------------------------------------------------------------------------
|
|
76
|
+
describe('readAllDatasets', () => {
|
|
77
|
+
it('flattens cases from multiple files', async () => {
|
|
78
|
+
const datasetsDir = join(ctx.tmpDir, 'src', 'workflows', 'my_workflow', 'tests', 'datasets');
|
|
79
|
+
await mkdir(datasetsDir, { recursive: true });
|
|
80
|
+
await writeYaml(join(datasetsDir, 'group_a.yml'), {
|
|
81
|
+
case_1: { input: { x: 1 } },
|
|
82
|
+
case_2: { input: { x: 2 } }
|
|
83
|
+
});
|
|
84
|
+
await writeYaml(join(datasetsDir, 'group_b.yml'), {
|
|
85
|
+
case_3: { input: { x: 3 } }
|
|
86
|
+
});
|
|
87
|
+
const { datasets } = await readAllDatasets('my_workflow', undefined, ctx.tmpDir);
|
|
88
|
+
expect(datasets).toHaveLength(3);
|
|
89
|
+
expect(datasets.map(d => d.name).sort()).toEqual(['case_1', 'case_2', 'case_3']);
|
|
90
|
+
});
|
|
91
|
+
it('filters by case name across files', async () => {
|
|
92
|
+
const datasetsDir = join(ctx.tmpDir, 'src', 'workflows', 'my_workflow', 'tests', 'datasets');
|
|
93
|
+
await mkdir(datasetsDir, { recursive: true });
|
|
94
|
+
await writeYaml(join(datasetsDir, 'group_a.yml'), {
|
|
95
|
+
case_1: { input: { x: 1 } },
|
|
96
|
+
case_2: { input: { x: 2 } }
|
|
97
|
+
});
|
|
98
|
+
await writeYaml(join(datasetsDir, 'group_b.yml'), {
|
|
99
|
+
case_3: { input: { x: 3 } }
|
|
100
|
+
});
|
|
101
|
+
const { datasets } = await readAllDatasets('my_workflow', ['case_2', 'case_3'], ctx.tmpDir);
|
|
102
|
+
expect(datasets).toHaveLength(2);
|
|
103
|
+
expect(datasets.map(d => d.name).sort()).toEqual(['case_2', 'case_3']);
|
|
104
|
+
});
|
|
105
|
+
it('returns empty datasets and a default dir when workflow has no datasets dir', async () => {
|
|
106
|
+
const { datasets, dir } = await readAllDatasets('nonexistent_workflow', undefined, ctx.tmpDir);
|
|
107
|
+
expect(datasets).toHaveLength(0);
|
|
108
|
+
expect(dir).toContain('nonexistent_workflow');
|
|
109
|
+
});
|
|
110
|
+
it('throws when the same case name appears in two different files', async () => {
|
|
111
|
+
const datasetsDir = join(ctx.tmpDir, 'src', 'workflows', 'my_workflow', 'tests', 'datasets');
|
|
112
|
+
await mkdir(datasetsDir, { recursive: true });
|
|
113
|
+
await writeYaml(join(datasetsDir, 'group_a.yml'), { case_1: { input: { x: 1 } } });
|
|
114
|
+
await writeYaml(join(datasetsDir, 'group_b.yml'), { case_1: { input: { x: 2 } } });
|
|
115
|
+
await expect(readAllDatasets('my_workflow', undefined, ctx.tmpDir)).rejects.toThrow('Duplicate dataset case name "case_1"');
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
// ---------------------------------------------------------------------------
|
|
119
|
+
// writeDataset
|
|
120
|
+
// ---------------------------------------------------------------------------
|
|
121
|
+
describe('writeDataset', () => {
|
|
122
|
+
it('creates a new file with one case keyed by name', async () => {
|
|
123
|
+
const filePath = join(ctx.tmpDir, 'cases.yml');
|
|
124
|
+
const dataset = { name: 'new_case', input: { q: 'hello' } };
|
|
125
|
+
await writeDataset(dataset, filePath);
|
|
126
|
+
const raw = yaml.load(await readFile(filePath, 'utf-8'));
|
|
127
|
+
expect(raw).toHaveProperty('new_case');
|
|
128
|
+
expect(raw.new_case.input).toEqual({ q: 'hello' });
|
|
129
|
+
expect(raw.new_case).not.toHaveProperty('name');
|
|
130
|
+
});
|
|
131
|
+
it('does not write _source into the file', async () => {
|
|
132
|
+
const filePath = join(ctx.tmpDir, 'cases.yml');
|
|
133
|
+
const dataset = { name: 'my_case', input: { q: 'x' }, _source: '/some/path.yml' };
|
|
134
|
+
await writeDataset(dataset, filePath);
|
|
135
|
+
const raw = yaml.load(await readFile(filePath, 'utf-8'));
|
|
136
|
+
expect(raw.my_case).not.toHaveProperty('_source');
|
|
137
|
+
});
|
|
138
|
+
it('updates only the target case, leaving other cases untouched', async () => {
|
|
139
|
+
const filePath = join(ctx.tmpDir, 'cases.yml');
|
|
140
|
+
await writeYaml(filePath, {
|
|
141
|
+
case_a: { input: { x: 1 }, ground_truth: { expected: 1 } },
|
|
142
|
+
case_b: { input: { x: 2 }, ground_truth: { expected: 2 } }
|
|
143
|
+
});
|
|
144
|
+
await writeDataset({ name: 'case_a', input: { x: 1 }, last_output: { output: { result: 1 }, date: '2026-01-01T00:00:00.000Z' } }, filePath);
|
|
145
|
+
const raw = yaml.load(await readFile(filePath, 'utf-8'));
|
|
146
|
+
expect(raw).toHaveProperty('case_b');
|
|
147
|
+
expect(raw.case_b.ground_truth).toEqual({ expected: 2 });
|
|
148
|
+
});
|
|
149
|
+
it('preserves existing fields when writing last_output then last_eval', async () => {
|
|
150
|
+
const filePath = join(ctx.tmpDir, 'cases.yml');
|
|
151
|
+
await writeYaml(filePath, {
|
|
152
|
+
my_case: { input: { q: 'hello' }, ground_truth: { expected: 42 } }
|
|
153
|
+
});
|
|
154
|
+
await writeDataset({ name: 'my_case', input: { q: 'hello' }, last_output: { output: { result: 42 }, executionTimeMs: 50, date: '2026-01-01T00:00:00.000Z' } }, filePath);
|
|
155
|
+
await writeDataset({
|
|
156
|
+
name: 'my_case', input: { q: 'hello' },
|
|
157
|
+
last_eval: { output: { datasetName: 'my_case', verdict: 'pass', evaluators: [] }, date: '2026-01-01T00:01:00.000Z' }
|
|
158
|
+
}, filePath);
|
|
159
|
+
const raw = yaml.load(await readFile(filePath, 'utf-8'));
|
|
160
|
+
const caseObj = raw.my_case;
|
|
161
|
+
expect(caseObj).toHaveProperty('last_output');
|
|
162
|
+
expect(caseObj).toHaveProperty('last_eval');
|
|
163
|
+
expect(caseObj).toHaveProperty('ground_truth');
|
|
164
|
+
});
|
|
165
|
+
it('creates parent directories if they do not exist', async () => {
|
|
166
|
+
const filePath = join(ctx.tmpDir, 'deep', 'nested', 'cases.yml');
|
|
167
|
+
await writeDataset({ name: 'my_case', input: { q: 'x' } }, filePath);
|
|
168
|
+
const raw = yaml.load(await readFile(filePath, 'utf-8'));
|
|
169
|
+
expect(raw).toHaveProperty('my_case');
|
|
170
|
+
});
|
|
171
|
+
it('recovers gracefully when existing file contains non-object YAML', async () => {
|
|
172
|
+
const filePath = join(ctx.tmpDir, 'cases.yml');
|
|
173
|
+
await writeFile(filePath, 'just a string', 'utf-8');
|
|
174
|
+
await writeDataset({ name: 'my_case', input: { q: 'x' } }, filePath);
|
|
175
|
+
const raw = yaml.load(await readFile(filePath, 'utf-8'));
|
|
176
|
+
expect(raw).toHaveProperty('my_case');
|
|
177
|
+
});
|
|
178
|
+
});
|
|
179
|
+
// ---------------------------------------------------------------------------
|
|
180
|
+
// listDatasets
|
|
181
|
+
// ---------------------------------------------------------------------------
|
|
182
|
+
describe('listDatasets', () => {
|
|
183
|
+
it('returns one DatasetInfo per case across all files', async () => {
|
|
184
|
+
const datasetsDir = join(ctx.tmpDir, 'src', 'workflows', 'my_workflow', 'tests', 'datasets');
|
|
185
|
+
await mkdir(datasetsDir, { recursive: true });
|
|
186
|
+
await writeYaml(join(datasetsDir, 'core.yml'), {
|
|
187
|
+
case_1: { input: { x: 1 }, last_output: { output: { r: 1 }, date: '2026-01-01T00:00:00.000Z' } },
|
|
188
|
+
case_2: { input: { x: 2 } }
|
|
189
|
+
});
|
|
190
|
+
const infos = await listDatasets('my_workflow', ctx.tmpDir);
|
|
191
|
+
expect(infos).toHaveLength(2);
|
|
192
|
+
const case1 = infos.find(i => i.name === 'case_1');
|
|
193
|
+
expect(case1.hasLastOutput).toBe(true);
|
|
194
|
+
expect(case1.path).toContain('core.yml');
|
|
195
|
+
const case2 = infos.find(i => i.name === 'case_2');
|
|
196
|
+
expect(case2.hasLastOutput).toBe(false);
|
|
197
|
+
});
|
|
198
|
+
it('returns empty array when no datasets directory exists', async () => {
|
|
199
|
+
const infos = await listDatasets('nonexistent_workflow', ctx.tmpDir);
|
|
200
|
+
expect(infos).toHaveLength(0);
|
|
201
|
+
});
|
|
202
|
+
});
|
|
@@ -21,8 +21,6 @@ export declare class DockerComposeConfigNotFoundError extends Error {
|
|
|
21
21
|
constructor(dockerComposePath: string);
|
|
22
22
|
}
|
|
23
23
|
declare const isDockerInstalled: () => boolean;
|
|
24
|
-
declare const isDockerComposeAvailable: () => boolean;
|
|
25
|
-
declare const isDockerDaemonRunning: () => boolean;
|
|
26
24
|
export declare function validateDockerEnvironment(): void;
|
|
27
25
|
export declare function getDefaultDockerComposePath(): string;
|
|
28
26
|
export declare function parseServiceStatus(jsonOutput: string): ServiceStatus[];
|
|
@@ -37,4 +35,4 @@ export type PullPolicy = 'always' | 'missing' | 'never';
|
|
|
37
35
|
export declare function startDockerCompose(dockerComposePath: string, pullPolicy?: PullPolicy): Promise<DockerComposeProcess>;
|
|
38
36
|
export declare function startDockerComposeDetached(dockerComposePath: string, pullPolicy?: PullPolicy): void;
|
|
39
37
|
export declare function stopDockerCompose(dockerComposePath: string): Promise<void>;
|
|
40
|
-
export { isDockerInstalled,
|
|
38
|
+
export { isDockerInstalled, DockerValidationError };
|
package/dist/services/docker.js
CHANGED
|
@@ -2,6 +2,7 @@ import { execFileSync, execSync, spawn } from 'node:child_process';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
4
|
import { ux } from '@oclif/core';
|
|
5
|
+
import semver from 'semver';
|
|
5
6
|
const DEFAULT_COMPOSE_PATH = '../assets/docker/docker-compose-dev.yml';
|
|
6
7
|
export const SERVICE_HEALTH = {
|
|
7
8
|
HEALTHY: 'healthy',
|
|
@@ -30,27 +31,51 @@ const checkDockerCommand = (command) => {
|
|
|
30
31
|
return false;
|
|
31
32
|
}
|
|
32
33
|
};
|
|
34
|
+
const getCommandVersion = (command, pattern = /(\d+\.\d+\.\d+)/) => {
|
|
35
|
+
try {
|
|
36
|
+
const output = execSync(command, { stdio: 'pipe', encoding: 'utf-8' }).trim();
|
|
37
|
+
const match = output.match(pattern);
|
|
38
|
+
return match ? match[1] : null;
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
};
|
|
33
44
|
const isDockerInstalled = () => checkDockerCommand('docker --version');
|
|
34
|
-
const
|
|
35
|
-
const isDockerDaemonRunning = () => checkDockerCommand('docker ps');
|
|
36
|
-
const DOCKER_VALIDATIONS = [
|
|
45
|
+
const PREREQUISITES = [
|
|
37
46
|
{
|
|
38
|
-
|
|
39
|
-
|
|
47
|
+
name: 'Docker',
|
|
48
|
+
semverRange: '>=20.0.0',
|
|
49
|
+
getVersion: () => getCommandVersion('docker --version'),
|
|
50
|
+
errorMessage: (current, required) => current === null ?
|
|
51
|
+
'Docker is not installed. Please install Docker to use the dev command.\nVisit: https://docs.docker.com/get-docker/' :
|
|
52
|
+
`Docker version ${required} is required (found v${current}).\nVisit: https://docs.docker.com/get-docker/`
|
|
40
53
|
},
|
|
41
54
|
{
|
|
42
|
-
|
|
43
|
-
|
|
55
|
+
name: 'Docker Compose',
|
|
56
|
+
semverRange: '>=2.24.0',
|
|
57
|
+
getVersion: () => getCommandVersion('docker compose version --short'),
|
|
58
|
+
errorMessage: (current, required) => current === null ?
|
|
59
|
+
'Docker Compose is not installed. Please install Docker Compose to use the dev command.\nVisit: https://docs.docker.com/compose/install/' :
|
|
60
|
+
`Docker Compose ${required} is required (found v${current}).\nPlease update Docker Compose: https://docs.docker.com/compose/install/`
|
|
44
61
|
},
|
|
45
62
|
{
|
|
46
|
-
|
|
47
|
-
|
|
63
|
+
name: 'Docker Daemon',
|
|
64
|
+
semverRange: '*',
|
|
65
|
+
getVersion: () => checkDockerCommand('docker ps') ? '0.0.0' : null,
|
|
66
|
+
errorMessage: () => 'Docker daemon is not running. Please start Docker and try again.'
|
|
48
67
|
}
|
|
49
68
|
];
|
|
50
69
|
export function validateDockerEnvironment() {
|
|
51
|
-
const
|
|
52
|
-
|
|
53
|
-
|
|
70
|
+
for (const prereq of PREREQUISITES) {
|
|
71
|
+
const raw = prereq.getVersion();
|
|
72
|
+
const version = raw ? semver.valid(semver.coerce(raw)) : null;
|
|
73
|
+
if (!version) {
|
|
74
|
+
throw new DockerValidationError(prereq.errorMessage(null, prereq.semverRange));
|
|
75
|
+
}
|
|
76
|
+
if (!semver.satisfies(version, prereq.semverRange)) {
|
|
77
|
+
throw new DockerValidationError(prereq.errorMessage(version, prereq.semverRange));
|
|
78
|
+
}
|
|
54
79
|
}
|
|
55
80
|
}
|
|
56
81
|
export function getDefaultDockerComposePath() {
|
|
@@ -129,4 +154,4 @@ export async function stopDockerCompose(dockerComposePath) {
|
|
|
129
154
|
ux.stdout('⏹️ Stopping services...\n');
|
|
130
155
|
execFileSync('docker', ['compose', '-f', dockerComposePath, 'down'], { stdio: 'inherit', cwd: process.cwd() });
|
|
131
156
|
}
|
|
132
|
-
export { isDockerInstalled,
|
|
157
|
+
export { isDockerInstalled, DockerValidationError };
|
|
@@ -3,4 +3,4 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export declare const getEjectSuccessMessage: (destPath: string, outputFile: string, binName: string) => string;
|
|
5
5
|
export declare const getProjectSuccessMessage: (folderName: string, installSuccess: boolean, credentialsConfigured?: boolean) => string;
|
|
6
|
-
export declare const getWorkflowGenerateSuccessMessage: (workflowName: string, targetDir: string, filesCreated: string[]) => string;
|
|
6
|
+
export declare const getWorkflowGenerateSuccessMessage: (workflowName: string, workflowId: string, scenarioName: string | undefined, targetDir: string, filesCreated: string[]) => string;
|
|
@@ -234,7 +234,7 @@ ${ux.colorize('dim', ' to manage your project secrets.')}
|
|
|
234
234
|
${ux.colorize('green', ux.colorize('bold', 'Happy building with Output! 🚀'))}
|
|
235
235
|
`;
|
|
236
236
|
};
|
|
237
|
-
export const getWorkflowGenerateSuccessMessage = (workflowName, targetDir, filesCreated) => {
|
|
237
|
+
export const getWorkflowGenerateSuccessMessage = (workflowName, workflowId, scenarioName, targetDir, filesCreated) => {
|
|
238
238
|
const divider = ux.colorize('dim', '─'.repeat(80));
|
|
239
239
|
const bulletPoint = ux.colorize('green', '▸');
|
|
240
240
|
const formattedFiles = filesCreated.map(file => {
|
|
@@ -256,7 +256,7 @@ export const getWorkflowGenerateSuccessMessage = (workflowName, targetDir, files
|
|
|
256
256
|
},
|
|
257
257
|
{
|
|
258
258
|
step: 'Test your workflow',
|
|
259
|
-
command: `npx output workflow run ${
|
|
259
|
+
command: `npx output workflow run ${workflowId}${scenarioName ? ` ${scenarioName}` : ''}`,
|
|
260
260
|
note: 'Run after starting services with "npx output dev"'
|
|
261
261
|
}
|
|
262
262
|
];
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Build a workflow from a plan file using the /
|
|
2
|
+
* Build a workflow from a plan file using the /output-build-workflow slash command
|
|
3
3
|
* @param planFilePath - Absolute path to the plan file
|
|
4
4
|
* @param workflowDir - Absolute path to the workflow directory
|
|
5
5
|
* @param workflowName - Name of the workflow
|
|
@@ -31,7 +31,7 @@ function isEmpty(modification) {
|
|
|
31
31
|
return modification.trim() === '';
|
|
32
32
|
}
|
|
33
33
|
/**
|
|
34
|
-
* Build a workflow from a plan file using the /
|
|
34
|
+
* Build a workflow from a plan file using the /output-build-workflow slash command
|
|
35
35
|
* @param planFilePath - Absolute path to the plan file
|
|
36
36
|
* @param workflowDir - Absolute path to the workflow directory
|
|
37
37
|
* @param workflowName - Name of the workflow
|
|
@@ -4,16 +4,50 @@ This is an **Output.ai** project - a framework for building reliable, production
|
|
|
4
4
|
|
|
5
5
|
## Getting Started
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Install Claude Code plugins for full framework documentation and AI-assisted development:
|
|
8
8
|
|
|
9
9
|
```bash
|
|
10
10
|
claude plugin marketplace add growthxai/output
|
|
11
11
|
claude plugin install outputai@outputai --scope project
|
|
12
12
|
```
|
|
13
13
|
|
|
14
|
+
## Commands
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
npm run output:dev # Start dev environment (worker + Temporal)
|
|
18
|
+
npm run output:worker:build # Build TypeScript to dist/
|
|
19
|
+
npm run output:worker:watch # Build + restart on file changes
|
|
20
|
+
npm run output:worker # Install, build, and start worker
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Project Structure
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
src/
|
|
27
|
+
workflows/ # Each subfolder is one workflow
|
|
28
|
+
<name>/
|
|
29
|
+
workflow.ts # Workflow orchestration (must be deterministic - no I/O)
|
|
30
|
+
steps.ts # Step functions (all I/O: HTTP, LLM, DB)
|
|
31
|
+
evaluators.ts # Evaluator functions (LLM-based quality assessment)
|
|
32
|
+
types.ts # Zod schemas and TypeScript types
|
|
33
|
+
prompts/ # .prompt files (Liquid.js templates with YAML frontmatter)
|
|
34
|
+
scenarios/ # Test scenario JSON files
|
|
35
|
+
clients/ # Shared HTTP clients (use @outputai/http, not fetch/axios)
|
|
36
|
+
shared/ # Shared utilities across workflows
|
|
37
|
+
config/
|
|
38
|
+
costs.yml # Token/API pricing overrides
|
|
39
|
+
credentials.yml.enc # Encrypted secrets (edit via: output credentials edit)
|
|
40
|
+
credentials.yml.template # Credential structure reference
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Key Conventions
|
|
44
|
+
|
|
45
|
+
- **Workflows are deterministic**: No I/O, no `Date.now()`, no `Math.random()` in `workflow.ts`. All side effects go in steps or evaluators.
|
|
46
|
+
- **HTTP clients**: Always use `httpClient` from `@outputai/http` -- never raw `fetch` or `axios`. This enables automatic tracing and cost tracking.
|
|
47
|
+
- **LLM calls**: Use `generateText` from `@outputai/llm` with `.prompt` files. Never call LLM APIs directly.
|
|
48
|
+
|
|
14
49
|
---
|
|
15
50
|
|
|
16
51
|
## Project-Specific Instructions
|
|
17
52
|
|
|
18
53
|
<!-- Add your project-specific instructions below -->
|
|
19
|
-
|