@outputai/cli 0.8.2-next.42a0ddf.0 → 0.8.2-next.7929835.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/docker/docker-compose-dev.yml +1 -1
- package/dist/commands/workflow/dataset/generate.d.ts +1 -0
- package/dist/commands/workflow/dataset/generate.js +15 -6
- package/dist/commands/workflow/dataset/generate.spec.d.ts +1 -0
- package/dist/commands/workflow/dataset/generate.spec.js +69 -0
- package/dist/commands/workflow/generate.spec.js +2 -2
- package/dist/commands/workflow/run.js +1 -1
- package/dist/commands/workflow/run.spec.js +17 -0
- package/dist/commands/workflow/start.js +1 -1
- package/dist/commands/workflow/start.spec.js +53 -2
- package/dist/commands/workflow/test_eval.d.ts +1 -0
- package/dist/commands/workflow/test_eval.js +18 -8
- package/dist/commands/workflow/test_eval.spec.js +37 -0
- package/dist/generated/framework_version.json +1 -1
- package/dist/services/coding_agents.spec.js +10 -10
- package/dist/utils/resolve_input.d.ts +1 -1
- package/dist/utils/resolve_input.js +2 -2
- package/dist/utils/scenario_resolver.d.ts +1 -1
- package/dist/utils/scenario_resolver.js +2 -2
- package/dist/utils/scenario_resolver.spec.js +14 -0
- package/dist/utils/workflow_dir.d.ts +1 -1
- package/dist/utils/workflow_dir.js +2 -2
- package/oclif.manifest.json +33 -1
- package/package.json +5 -5
|
@@ -7,6 +7,7 @@ export default class DatasetGenerate extends Command {
|
|
|
7
7
|
scenario: import("@oclif/core/interfaces").Arg<string | undefined, Record<string, unknown>>;
|
|
8
8
|
};
|
|
9
9
|
static flags: {
|
|
10
|
+
catalog: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
10
11
|
trace: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
11
12
|
name: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
12
13
|
download: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
@@ -26,6 +26,14 @@ export default class DatasetGenerate extends Command {
|
|
|
26
26
|
})
|
|
27
27
|
};
|
|
28
28
|
static flags = {
|
|
29
|
+
catalog: Flags.string({
|
|
30
|
+
char: 'c',
|
|
31
|
+
aliases: ['task-queue'],
|
|
32
|
+
charAliases: ['q'],
|
|
33
|
+
deprecateAliases: true,
|
|
34
|
+
description: 'Catalog name for workflow execution (defaults to OUTPUT_CATALOG_ID)',
|
|
35
|
+
env: 'OUTPUT_CATALOG_ID'
|
|
36
|
+
}),
|
|
29
37
|
trace: Flags.string({
|
|
30
38
|
char: 't',
|
|
31
39
|
description: 'Path to a local trace file to extract dataset from',
|
|
@@ -61,15 +69,16 @@ export default class DatasetGenerate extends Command {
|
|
|
61
69
|
await this.generateFromTrace(args.workflowName, flags.trace, flags.name);
|
|
62
70
|
return;
|
|
63
71
|
}
|
|
64
|
-
await this.generateFromScenario(args.workflowName, args.scenario, flags.input, flags.name);
|
|
72
|
+
await this.generateFromScenario(args.workflowName, args.scenario, flags.input, flags.name, flags.catalog);
|
|
65
73
|
}
|
|
66
|
-
async generateFromScenario(workflowName, scenario, inputFlag, nameOverride) {
|
|
67
|
-
const resolvedInput = await this.resolveScenarioInput(workflowName, scenario, inputFlag);
|
|
74
|
+
async generateFromScenario(workflowName, scenario, inputFlag, nameOverride, catalog) {
|
|
75
|
+
const resolvedInput = await this.resolveScenarioInput(workflowName, scenario, inputFlag, catalog);
|
|
68
76
|
const datasetName = nameOverride ?? scenario ?? 'dataset';
|
|
69
77
|
this.log(`Running workflow "${workflowName}"...`);
|
|
70
78
|
const response = await postWorkflowRun({
|
|
71
79
|
workflowName,
|
|
72
|
-
input: resolvedInput
|
|
80
|
+
input: resolvedInput,
|
|
81
|
+
catalog
|
|
73
82
|
}, {
|
|
74
83
|
config: { timeout: 600000 }
|
|
75
84
|
});
|
|
@@ -116,7 +125,7 @@ export default class DatasetGenerate extends Command {
|
|
|
116
125
|
}
|
|
117
126
|
this.log(`\nGenerated ${traces.length} dataset(s)`);
|
|
118
127
|
}
|
|
119
|
-
async resolveScenarioInput(workflowName, scenario, inputFlag) {
|
|
128
|
+
async resolveScenarioInput(workflowName, scenario, inputFlag, catalog) {
|
|
120
129
|
if (inputFlag && scenario) {
|
|
121
130
|
return ux.error('Cannot use both scenario argument and --input flag. Choose one.', { exit: 1 });
|
|
122
131
|
}
|
|
@@ -124,7 +133,7 @@ export default class DatasetGenerate extends Command {
|
|
|
124
133
|
return parseInputFlag(inputFlag);
|
|
125
134
|
}
|
|
126
135
|
if (scenario) {
|
|
127
|
-
const resolution = await resolveScenarioPath(workflowName, scenario);
|
|
136
|
+
const resolution = await resolveScenarioPath(workflowName, scenario, undefined, undefined, catalog);
|
|
128
137
|
if (!resolution.found) {
|
|
129
138
|
return ux.error(getScenarioNotFoundMessage(workflowName, scenario, resolution.searchedPaths), { exit: 1 });
|
|
130
139
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
2
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
3
|
+
vi.mock('#api/generated/api.js', () => ({
|
|
4
|
+
postWorkflowRun: vi.fn()
|
|
5
|
+
}));
|
|
6
|
+
vi.mock('#utils/scenario_resolver.js', () => ({
|
|
7
|
+
resolveScenarioPath: vi.fn(),
|
|
8
|
+
getScenarioNotFoundMessage: vi.fn().mockReturnValue('not found')
|
|
9
|
+
}));
|
|
10
|
+
vi.mock('#utils/input_parser.js', () => ({
|
|
11
|
+
parseInputFlag: vi.fn()
|
|
12
|
+
}));
|
|
13
|
+
vi.mock('#services/datasets.js', () => ({
|
|
14
|
+
writeDataset: vi.fn(),
|
|
15
|
+
resolveDefaultDatasetsDir: vi.fn().mockResolvedValue('/datasets'),
|
|
16
|
+
buildDataset: vi.fn().mockReturnValue({ name: 'basic' }),
|
|
17
|
+
getExecutionTime: vi.fn().mockResolvedValue(100),
|
|
18
|
+
extractDatasetName: vi.fn()
|
|
19
|
+
}));
|
|
20
|
+
describe('workflow dataset generate command', () => {
|
|
21
|
+
beforeEach(() => {
|
|
22
|
+
vi.clearAllMocks();
|
|
23
|
+
delete process.env.OUTPUT_CATALOG_ID;
|
|
24
|
+
});
|
|
25
|
+
describe('command definition', () => {
|
|
26
|
+
it('binds the catalog flag to OUTPUT_CATALOG_ID', async () => {
|
|
27
|
+
const DatasetGenerate = (await import('./generate.js')).default;
|
|
28
|
+
expect(DatasetGenerate.flags).toHaveProperty('catalog');
|
|
29
|
+
expect(DatasetGenerate.flags.catalog.env).toBe('OUTPUT_CATALOG_ID');
|
|
30
|
+
expect(DatasetGenerate.flags.catalog.char).toBe('c');
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
describe('run()', () => {
|
|
34
|
+
const createCommand = async (flagOverrides = {}) => {
|
|
35
|
+
const DatasetGenerate = (await import('./generate.js')).default;
|
|
36
|
+
const { postWorkflowRun } = await import('#api/generated/api.js');
|
|
37
|
+
const { resolveScenarioPath } = await import('#utils/scenario_resolver.js');
|
|
38
|
+
const { parseInputFlag } = await import('#utils/input_parser.js');
|
|
39
|
+
const cmd = new DatasetGenerate(['my_workflow'], {});
|
|
40
|
+
cmd.log = vi.fn();
|
|
41
|
+
cmd.error = vi.fn(() => {
|
|
42
|
+
throw new Error('error called');
|
|
43
|
+
});
|
|
44
|
+
cmd.parse = vi.fn().mockResolvedValue({
|
|
45
|
+
args: { workflowName: 'my_workflow', scenario: 'basic' },
|
|
46
|
+
flags: { catalog: undefined, trace: undefined, name: undefined, download: false, limit: 5, input: undefined, ...flagOverrides }
|
|
47
|
+
});
|
|
48
|
+
return {
|
|
49
|
+
cmd,
|
|
50
|
+
postWorkflowRun: vi.mocked(postWorkflowRun),
|
|
51
|
+
resolveScenarioPath: vi.mocked(resolveScenarioPath),
|
|
52
|
+
parseInputFlag: vi.mocked(parseInputFlag)
|
|
53
|
+
};
|
|
54
|
+
};
|
|
55
|
+
it('resolves the scenario and runs the workflow against the resolved catalog', async () => {
|
|
56
|
+
const { cmd, postWorkflowRun, resolveScenarioPath, parseInputFlag } = await createCommand({ catalog: 'my-catalog' });
|
|
57
|
+
resolveScenarioPath.mockResolvedValue({ found: true, path: '/scenarios/basic.json', searchedPaths: [] });
|
|
58
|
+
parseInputFlag.mockResolvedValue({ foo: 'bar' });
|
|
59
|
+
postWorkflowRun.mockResolvedValue({
|
|
60
|
+
data: { workflowId: 'wf-1', output: { ok: true } },
|
|
61
|
+
status: 200,
|
|
62
|
+
headers: new Headers()
|
|
63
|
+
});
|
|
64
|
+
await cmd.run();
|
|
65
|
+
expect(resolveScenarioPath).toHaveBeenCalledWith('my_workflow', 'basic', undefined, undefined, 'my-catalog');
|
|
66
|
+
expect(postWorkflowRun).toHaveBeenCalledWith(expect.objectContaining({ workflowName: 'my_workflow', catalog: 'my-catalog' }), expect.anything());
|
|
67
|
+
});
|
|
68
|
+
});
|
|
69
|
+
});
|
|
@@ -4,8 +4,8 @@ import Generate from './generate.js';
|
|
|
4
4
|
import { generateWorkflow } from '#services/workflow_generator.js';
|
|
5
5
|
import { parseWorkflowDir } from '#utils/workflow_dir_parser.js';
|
|
6
6
|
import { InvalidNameError, WorkflowExistsError } from '#types/errors.js';
|
|
7
|
-
vi.mock('
|
|
8
|
-
vi.mock('
|
|
7
|
+
vi.mock('#services/workflow_generator.js');
|
|
8
|
+
vi.mock('#utils/workflow_dir_parser.js');
|
|
9
9
|
describe('Generate Command', () => {
|
|
10
10
|
let mockGenerateWorkflow;
|
|
11
11
|
let mockParseWorkflowDir;
|
|
@@ -63,7 +63,7 @@ export default class WorkflowRun extends Command {
|
|
|
63
63
|
};
|
|
64
64
|
async run() {
|
|
65
65
|
const { args, flags } = await this.parse(WorkflowRun);
|
|
66
|
-
const input = await resolveInput(args.workflowName, args.scenario, flags.input, 'run');
|
|
66
|
+
const input = await resolveInput(args.workflowName, args.scenario, flags.input, 'run', flags.catalog);
|
|
67
67
|
this.log(`Executing workflow: ${args.workflowName}...`);
|
|
68
68
|
const response = await executeWorkflow({
|
|
69
69
|
body: {
|
|
@@ -64,11 +64,28 @@ describe('workflow run command', () => {
|
|
|
64
64
|
headers: new Headers()
|
|
65
65
|
});
|
|
66
66
|
await cmd.run();
|
|
67
|
+
expect(resolveInput).toHaveBeenCalledWith('my_workflow', undefined, undefined, 'run', undefined);
|
|
67
68
|
expect(postWorkflowRun).toHaveBeenCalledTimes(1);
|
|
68
69
|
expect(postWorkflowRun).toHaveBeenCalledWith({ workflowName: 'my_workflow', input: { key: 'value' }, catalog: undefined }, expect.objectContaining({ config: { timeout: 600000 } }));
|
|
69
70
|
expect(cmd.log).toHaveBeenCalledWith('Executing workflow: my_workflow...');
|
|
70
71
|
expect(cmd.log).toHaveBeenCalledWith(expect.stringMatching(/\n/));
|
|
71
72
|
});
|
|
73
|
+
it('threads the resolved catalog to resolveInput and postWorkflowRun', async () => {
|
|
74
|
+
const { cmd, postWorkflowRun, resolveInput } = await createCommand();
|
|
75
|
+
cmd.parse = vi.fn().mockResolvedValue({
|
|
76
|
+
args: { workflowName: 'my_workflow', scenario: 'basic' },
|
|
77
|
+
flags: { input: undefined, catalog: 'my-catalog', format: 'text' }
|
|
78
|
+
});
|
|
79
|
+
resolveInput.mockResolvedValue({ key: 'value' });
|
|
80
|
+
postWorkflowRun.mockResolvedValue({
|
|
81
|
+
data: { status: 'completed', result: {} },
|
|
82
|
+
status: 200,
|
|
83
|
+
headers: new Headers()
|
|
84
|
+
});
|
|
85
|
+
await cmd.run();
|
|
86
|
+
expect(resolveInput).toHaveBeenCalledWith('my_workflow', 'basic', undefined, 'run', 'my-catalog');
|
|
87
|
+
expect(postWorkflowRun).toHaveBeenCalledWith(expect.objectContaining({ catalog: 'my-catalog' }), expect.anything());
|
|
88
|
+
});
|
|
72
89
|
it('retries when response has Retry-After and succeeds on second attempt', async () => {
|
|
73
90
|
const { cmd, postWorkflowRun, resolveInput } = await createCommand();
|
|
74
91
|
resolveInput.mockResolvedValue({});
|
|
@@ -37,7 +37,7 @@ export default class WorkflowStart extends Command {
|
|
|
37
37
|
};
|
|
38
38
|
async run() {
|
|
39
39
|
const { args, flags } = await this.parse(WorkflowStart);
|
|
40
|
-
const input = await resolveInput(args.workflowName, args.scenario, flags.input, 'start');
|
|
40
|
+
const input = await resolveInput(args.workflowName, args.scenario, flags.input, 'start', flags.catalog);
|
|
41
41
|
this.log(`Starting workflow: ${args.workflowName}...`);
|
|
42
42
|
const response = await postWorkflowStart({
|
|
43
43
|
workflowName: args.workflowName,
|
|
@@ -1,11 +1,17 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
1
2
|
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
-
vi.mock('
|
|
3
|
+
vi.mock('#api/generated/api.js', () => ({
|
|
3
4
|
postWorkflowStart: vi.fn()
|
|
4
5
|
}));
|
|
6
|
+
vi.mock('#utils/resolve_input.js', () => ({
|
|
7
|
+
resolveInput: vi.fn()
|
|
8
|
+
}));
|
|
5
9
|
describe('workflow start command', () => {
|
|
6
|
-
beforeEach(() => {
|
|
10
|
+
beforeEach(async () => {
|
|
7
11
|
vi.clearAllMocks();
|
|
8
12
|
delete process.env.OUTPUT_CATALOG_ID;
|
|
13
|
+
const { resolveInput } = await import('#utils/resolve_input.js');
|
|
14
|
+
vi.mocked(resolveInput).mockResolvedValue({});
|
|
9
15
|
});
|
|
10
16
|
describe('command definition', () => {
|
|
11
17
|
it('should export a valid OCLIF command', async () => {
|
|
@@ -25,5 +31,50 @@ describe('workflow start command', () => {
|
|
|
25
31
|
expect(WorkflowStart.args).toHaveProperty('scenario');
|
|
26
32
|
expect(WorkflowStart.args.scenario.required).toBe(false);
|
|
27
33
|
});
|
|
34
|
+
it('binds the catalog flag to OUTPUT_CATALOG_ID', async () => {
|
|
35
|
+
const WorkflowStart = (await import('./start.js')).default;
|
|
36
|
+
expect(WorkflowStart.flags.catalog.env).toBe('OUTPUT_CATALOG_ID');
|
|
37
|
+
expect(WorkflowStart.flags.catalog.char).toBe('c');
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
describe('run()', () => {
|
|
41
|
+
const createCommand = async (flagOverrides = {}) => {
|
|
42
|
+
const WorkflowStart = (await import('./start.js')).default;
|
|
43
|
+
const { postWorkflowStart } = await import('#api/generated/api.js');
|
|
44
|
+
const { resolveInput } = await import('#utils/resolve_input.js');
|
|
45
|
+
const cmd = new WorkflowStart(['my_workflow'], {});
|
|
46
|
+
cmd.log = vi.fn();
|
|
47
|
+
cmd.error = vi.fn(() => {
|
|
48
|
+
throw new Error('error called');
|
|
49
|
+
});
|
|
50
|
+
cmd.parse = vi.fn().mockResolvedValue({
|
|
51
|
+
args: { workflowName: 'my_workflow', scenario: undefined },
|
|
52
|
+
flags: { input: undefined, catalog: undefined, ...flagOverrides }
|
|
53
|
+
});
|
|
54
|
+
return { cmd, postWorkflowStart: vi.mocked(postWorkflowStart), resolveInput: vi.mocked(resolveInput) };
|
|
55
|
+
};
|
|
56
|
+
it('threads the resolved catalog to resolveInput and postWorkflowStart', async () => {
|
|
57
|
+
const { cmd, postWorkflowStart, resolveInput } = await createCommand({ catalog: 'my-catalog' });
|
|
58
|
+
resolveInput.mockResolvedValue({ key: 'value' });
|
|
59
|
+
postWorkflowStart.mockResolvedValue({
|
|
60
|
+
data: { workflowId: 'wf-123' },
|
|
61
|
+
status: 200,
|
|
62
|
+
headers: new Headers()
|
|
63
|
+
});
|
|
64
|
+
await cmd.run();
|
|
65
|
+
expect(resolveInput).toHaveBeenCalledWith('my_workflow', undefined, undefined, 'start', 'my-catalog');
|
|
66
|
+
expect(postWorkflowStart).toHaveBeenCalledWith(expect.objectContaining({ workflowName: 'my_workflow', catalog: 'my-catalog' }));
|
|
67
|
+
});
|
|
68
|
+
it('passes undefined catalog through when none is set', async () => {
|
|
69
|
+
const { cmd, postWorkflowStart, resolveInput } = await createCommand();
|
|
70
|
+
resolveInput.mockResolvedValue({});
|
|
71
|
+
postWorkflowStart.mockResolvedValue({
|
|
72
|
+
data: { workflowId: 'wf-123' },
|
|
73
|
+
status: 200,
|
|
74
|
+
headers: new Headers()
|
|
75
|
+
});
|
|
76
|
+
await cmd.run();
|
|
77
|
+
expect(resolveInput).toHaveBeenCalledWith('my_workflow', undefined, undefined, 'start', undefined);
|
|
78
|
+
});
|
|
28
79
|
});
|
|
29
80
|
});
|
|
@@ -9,6 +9,7 @@ export default class WorkflowTest extends Command {
|
|
|
9
9
|
workflowName: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
|
|
10
10
|
};
|
|
11
11
|
static flags: {
|
|
12
|
+
catalog: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
12
13
|
cached: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
13
14
|
save: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
14
15
|
dataset: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
@@ -24,6 +24,14 @@ export default class WorkflowTest extends Command {
|
|
|
24
24
|
})
|
|
25
25
|
};
|
|
26
26
|
static flags = {
|
|
27
|
+
catalog: Flags.string({
|
|
28
|
+
char: 'c',
|
|
29
|
+
aliases: ['task-queue'],
|
|
30
|
+
charAliases: ['q'],
|
|
31
|
+
deprecateAliases: true,
|
|
32
|
+
description: 'Catalog name for workflow execution (defaults to OUTPUT_CATALOG_ID)',
|
|
33
|
+
env: 'OUTPUT_CATALOG_ID'
|
|
34
|
+
}),
|
|
27
35
|
cached: Flags.boolean({
|
|
28
36
|
description: 'Use cached output from dataset files (skip workflow execution)',
|
|
29
37
|
default: false,
|
|
@@ -43,7 +51,7 @@ export default class WorkflowTest extends Command {
|
|
|
43
51
|
const { args, flags } = await this.parse(WorkflowTest);
|
|
44
52
|
const filterNames = flags.dataset?.split(',').map(s => s.trim());
|
|
45
53
|
const evalName = getEvalWorkflowName(args.workflowName);
|
|
46
|
-
await this.ensureEvalWorkflowRegistered(args.workflowName, evalName);
|
|
54
|
+
await this.ensureEvalWorkflowRegistered(args.workflowName, evalName, flags.catalog);
|
|
47
55
|
const { datasets, dir } = await readAllDatasets(args.workflowName, filterNames);
|
|
48
56
|
if (datasets.length === 0) {
|
|
49
57
|
this.error(`No datasets found for workflow "${args.workflowName}".\n` +
|
|
@@ -51,11 +59,12 @@ export default class WorkflowTest extends Command {
|
|
|
51
59
|
}
|
|
52
60
|
const preparedDatasets = flags.cached ?
|
|
53
61
|
this.validateDatasets(datasets) :
|
|
54
|
-
await this.runWorkflowForDatasets(args.workflowName, datasets, flags.save, dir);
|
|
62
|
+
await this.runWorkflowForDatasets(args.workflowName, datasets, flags.save, dir, flags.catalog);
|
|
55
63
|
this.log(`Running eval workflow "${evalName}"...\n`);
|
|
56
64
|
const response = await postWorkflowRun({
|
|
57
65
|
workflowName: evalName,
|
|
58
|
-
input: { datasets: preparedDatasets }
|
|
66
|
+
input: { datasets: preparedDatasets },
|
|
67
|
+
catalog: flags.catalog
|
|
59
68
|
}, {
|
|
60
69
|
config: { timeout: 600000 }
|
|
61
70
|
});
|
|
@@ -73,9 +82,9 @@ export default class WorkflowTest extends Command {
|
|
|
73
82
|
process.exitCode = computeExitCode(evalOutput);
|
|
74
83
|
return evalOutput;
|
|
75
84
|
}
|
|
76
|
-
async ensureEvalWorkflowRegistered(workflowName, evalName) {
|
|
77
|
-
const
|
|
78
|
-
if (
|
|
85
|
+
async ensureEvalWorkflowRegistered(workflowName, evalName, catalog) {
|
|
86
|
+
const workflows = await fetchWorkflowCatalog(catalog).catch(() => null);
|
|
87
|
+
if (workflows && !workflows.some(w => w.name === evalName)) {
|
|
79
88
|
this.error(await diagnoseMissingEvalWorkflow(workflowName), { exit: 1 });
|
|
80
89
|
}
|
|
81
90
|
}
|
|
@@ -88,7 +97,7 @@ export default class WorkflowTest extends Command {
|
|
|
88
97
|
}
|
|
89
98
|
return datasets;
|
|
90
99
|
}
|
|
91
|
-
async runWorkflowForDatasets(workflowName, datasets, save, dir) {
|
|
100
|
+
async runWorkflowForDatasets(workflowName, datasets, save, dir, catalog) {
|
|
92
101
|
this.log(`Running workflow "${workflowName}" for ${datasets.length} dataset(s)...\n`);
|
|
93
102
|
const results = [];
|
|
94
103
|
for (const dataset of datasets) {
|
|
@@ -96,7 +105,8 @@ export default class WorkflowTest extends Command {
|
|
|
96
105
|
const startMs = Date.now();
|
|
97
106
|
const response = await postWorkflowRun({
|
|
98
107
|
workflowName,
|
|
99
|
-
input: dataset.input
|
|
108
|
+
input: dataset.input,
|
|
109
|
+
catalog
|
|
100
110
|
}, {
|
|
101
111
|
config: { timeout: 600000 }
|
|
102
112
|
});
|
|
@@ -4,10 +4,16 @@ import { getEvalWorkflowName, renderEvalOutput } from '@outputai/evals';
|
|
|
4
4
|
vi.mock('#api/generated/api.js', () => ({
|
|
5
5
|
postWorkflowRun: vi.fn()
|
|
6
6
|
}));
|
|
7
|
+
vi.mock('#api/workflow_catalog.js', () => ({
|
|
8
|
+
fetchWorkflowCatalog: vi.fn()
|
|
9
|
+
}));
|
|
7
10
|
vi.mock('#services/datasets.js', () => ({
|
|
8
11
|
readAllDatasets: vi.fn(),
|
|
9
12
|
writeDataset: vi.fn()
|
|
10
13
|
}));
|
|
14
|
+
vi.mock('#utils/eval_diagnostics.js', () => ({
|
|
15
|
+
diagnoseMissingEvalWorkflow: vi.fn().mockResolvedValue('missing eval workflow')
|
|
16
|
+
}));
|
|
11
17
|
const passingOutput = {
|
|
12
18
|
cases: [{ datasetName: 'd1', verdict: 'pass', evaluators: [] }],
|
|
13
19
|
summary: { total: 1, passed: 1, partial: 0, failed: 0, acceptableRate: 1 }
|
|
@@ -23,10 +29,16 @@ describe('workflow test command', () => {
|
|
|
23
29
|
exitState.original = process.exitCode;
|
|
24
30
|
process.exitCode = undefined;
|
|
25
31
|
const { readAllDatasets } = await import('#services/datasets.js');
|
|
32
|
+
const { fetchWorkflowCatalog } = await import('#api/workflow_catalog.js');
|
|
26
33
|
vi.mocked(readAllDatasets).mockResolvedValue({
|
|
27
34
|
datasets: [{ name: 'd1', input: {}, last_output: { output: {}, date: '2026-01-01' } }],
|
|
28
35
|
dir: '/tmp/datasets'
|
|
29
36
|
});
|
|
37
|
+
// Catalog includes both eval names so ensureEvalWorkflowRegistered passes deterministically.
|
|
38
|
+
vi.mocked(fetchWorkflowCatalog).mockResolvedValue([
|
|
39
|
+
{ name: getEvalWorkflowName('simple') },
|
|
40
|
+
{ name: getEvalWorkflowName('my_workflow') }
|
|
41
|
+
]);
|
|
30
42
|
});
|
|
31
43
|
afterEach(() => {
|
|
32
44
|
process.exitCode = exitState.original;
|
|
@@ -36,6 +48,12 @@ describe('workflow test command', () => {
|
|
|
36
48
|
const WorkflowTest = (await import('./test_eval.js')).default;
|
|
37
49
|
expect(WorkflowTest.enableJsonFlag).toBe(true);
|
|
38
50
|
});
|
|
51
|
+
it('binds the catalog flag to OUTPUT_CATALOG_ID', async () => {
|
|
52
|
+
const WorkflowTest = (await import('./test_eval.js')).default;
|
|
53
|
+
expect(WorkflowTest.flags).toHaveProperty('catalog');
|
|
54
|
+
expect(WorkflowTest.flags.catalog.env).toBe('OUTPUT_CATALOG_ID');
|
|
55
|
+
expect(WorkflowTest.flags.catalog.char).toBe('c');
|
|
56
|
+
});
|
|
39
57
|
});
|
|
40
58
|
describe('run()', () => {
|
|
41
59
|
const createCommand = async (jsonEnabled) => {
|
|
@@ -80,5 +98,24 @@ describe('workflow test command', () => {
|
|
|
80
98
|
expect(result).toEqual(failingOutput);
|
|
81
99
|
expect(process.exitCode).toBe(1);
|
|
82
100
|
});
|
|
101
|
+
it('routes registration, dataset runs, and the eval run to the resolved catalog', async () => {
|
|
102
|
+
const WorkflowTest = (await import('./test_eval.js')).default;
|
|
103
|
+
const { postWorkflowRun } = await import('#api/generated/api.js');
|
|
104
|
+
const { fetchWorkflowCatalog } = await import('#api/workflow_catalog.js');
|
|
105
|
+
const cmd = new WorkflowTest(['my_workflow'], {});
|
|
106
|
+
cmd.log = vi.fn();
|
|
107
|
+
cmd.jsonEnabled = vi.fn().mockReturnValue(false);
|
|
108
|
+
cmd.parse = vi.fn().mockResolvedValue({
|
|
109
|
+
args: { workflowName: 'my_workflow' },
|
|
110
|
+
flags: { catalog: 'my-catalog', cached: false, save: false, dataset: undefined }
|
|
111
|
+
});
|
|
112
|
+
vi.mocked(postWorkflowRun)
|
|
113
|
+
.mockResolvedValueOnce({ data: { output: {} }, status: 200, headers: new Headers() })
|
|
114
|
+
.mockResolvedValueOnce({ data: { output: passingOutput }, status: 200, headers: new Headers() });
|
|
115
|
+
await cmd.run();
|
|
116
|
+
expect(vi.mocked(fetchWorkflowCatalog)).toHaveBeenCalledWith('my-catalog');
|
|
117
|
+
expect(postWorkflowRun).toHaveBeenNthCalledWith(1, expect.objectContaining({ workflowName: 'my_workflow', catalog: 'my-catalog' }), expect.anything());
|
|
118
|
+
expect(postWorkflowRun).toHaveBeenNthCalledWith(2, expect.objectContaining({ workflowName: getEvalWorkflowName('my_workflow'), catalog: 'my-catalog' }), expect.anything());
|
|
119
|
+
});
|
|
83
120
|
});
|
|
84
121
|
});
|
|
@@ -3,13 +3,13 @@ import { checkAgentStructure, prepareTemplateVariables, initializeAgentConfig, e
|
|
|
3
3
|
import { access } from 'node:fs/promises';
|
|
4
4
|
import fs from 'node:fs/promises';
|
|
5
5
|
vi.mock('node:fs/promises');
|
|
6
|
-
vi.mock('
|
|
6
|
+
vi.mock('#utils/paths.js', () => ({
|
|
7
7
|
getTemplateDir: vi.fn().mockReturnValue('/templates')
|
|
8
8
|
}));
|
|
9
|
-
vi.mock('
|
|
9
|
+
vi.mock('#utils/template.js', () => ({
|
|
10
10
|
processTemplate: vi.fn().mockImplementation((content) => content)
|
|
11
11
|
}));
|
|
12
|
-
vi.mock('
|
|
12
|
+
vi.mock('#utils/claude.js', () => ({
|
|
13
13
|
executeClaudeCommand: vi.fn().mockResolvedValue(undefined)
|
|
14
14
|
}));
|
|
15
15
|
vi.mock('@oclif/core', () => ({
|
|
@@ -152,14 +152,14 @@ describe('coding_agents service', () => {
|
|
|
152
152
|
vi.mocked(fs.writeFile).mockResolvedValue(undefined);
|
|
153
153
|
});
|
|
154
154
|
it('should call registerPluginMarketplace and installOutputAIPlugin', async () => {
|
|
155
|
-
const { executeClaudeCommand } = await import('
|
|
155
|
+
const { executeClaudeCommand } = await import('#utils/claude.js');
|
|
156
156
|
await ensureClaudePlugin('/test/project');
|
|
157
157
|
expect(executeClaudeCommand).toHaveBeenCalledWith(['plugin', 'marketplace', 'add', 'growthxai/output'], '/test/project', { ignoreFailure: true });
|
|
158
158
|
expect(executeClaudeCommand).toHaveBeenCalledWith(['plugin', 'marketplace', 'update', 'outputai'], '/test/project');
|
|
159
159
|
expect(executeClaudeCommand).toHaveBeenCalledWith(['plugin', 'install', 'outputai@outputai', '--scope', 'project'], '/test/project');
|
|
160
160
|
});
|
|
161
161
|
it('should show error and prompt user when plugin commands fail', async () => {
|
|
162
|
-
const { executeClaudeCommand } = await import('
|
|
162
|
+
const { executeClaudeCommand } = await import('#utils/claude.js');
|
|
163
163
|
const { confirm } = await import('#utils/prompt.js');
|
|
164
164
|
vi.mocked(executeClaudeCommand)
|
|
165
165
|
.mockResolvedValueOnce(undefined) // marketplace add
|
|
@@ -171,7 +171,7 @@ describe('coding_agents service', () => {
|
|
|
171
171
|
}));
|
|
172
172
|
});
|
|
173
173
|
it('should allow user to proceed without plugin setup if they confirm', async () => {
|
|
174
|
-
const { executeClaudeCommand } = await import('
|
|
174
|
+
const { executeClaudeCommand } = await import('#utils/claude.js');
|
|
175
175
|
const { confirm } = await import('#utils/prompt.js');
|
|
176
176
|
vi.mocked(executeClaudeCommand)
|
|
177
177
|
.mockRejectedValue(new Error('All plugin commands fail'));
|
|
@@ -221,7 +221,7 @@ describe('coding_agents service', () => {
|
|
|
221
221
|
vi.mocked(fs.writeFile).mockResolvedValue(undefined);
|
|
222
222
|
});
|
|
223
223
|
it('should show error and prompt user when registerPluginMarketplace fails', async () => {
|
|
224
|
-
const { executeClaudeCommand } = await import('
|
|
224
|
+
const { executeClaudeCommand } = await import('#utils/claude.js');
|
|
225
225
|
const { confirm } = await import('#utils/prompt.js');
|
|
226
226
|
vi.mocked(executeClaudeCommand)
|
|
227
227
|
.mockResolvedValueOnce(undefined) // marketplace add
|
|
@@ -233,7 +233,7 @@ describe('coding_agents service', () => {
|
|
|
233
233
|
}));
|
|
234
234
|
});
|
|
235
235
|
it('should show error and prompt user when installOutputAIPlugin fails', async () => {
|
|
236
|
-
const { executeClaudeCommand } = await import('
|
|
236
|
+
const { executeClaudeCommand } = await import('#utils/claude.js');
|
|
237
237
|
const { confirm } = await import('#utils/prompt.js');
|
|
238
238
|
vi.mocked(executeClaudeCommand)
|
|
239
239
|
.mockResolvedValueOnce(undefined) // marketplace add
|
|
@@ -246,7 +246,7 @@ describe('coding_agents service', () => {
|
|
|
246
246
|
}));
|
|
247
247
|
});
|
|
248
248
|
it('should allow user to proceed without plugin setup if they confirm', async () => {
|
|
249
|
-
const { executeClaudeCommand } = await import('
|
|
249
|
+
const { executeClaudeCommand } = await import('#utils/claude.js');
|
|
250
250
|
const { confirm } = await import('#utils/prompt.js');
|
|
251
251
|
vi.mocked(executeClaudeCommand)
|
|
252
252
|
.mockRejectedValue(new Error('All plugin commands fail'));
|
|
@@ -256,7 +256,7 @@ describe('coding_agents service', () => {
|
|
|
256
256
|
expect(fs.mkdir).toHaveBeenCalled();
|
|
257
257
|
});
|
|
258
258
|
it('should rethrow plugin error in non-interactive mode without prompting', async () => {
|
|
259
|
-
const { executeClaudeCommand } = await import('
|
|
259
|
+
const { executeClaudeCommand } = await import('#utils/claude.js');
|
|
260
260
|
const { confirm } = await import('#utils/prompt.js');
|
|
261
261
|
const { isInteractive } = await import('#utils/interactive.js');
|
|
262
262
|
vi.mocked(isInteractive).mockReturnValueOnce(false);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare function resolveInput(workflowName: string, scenario: string | undefined, inputFlag: string | undefined, commandName: string): Promise<unknown>;
|
|
1
|
+
export declare function resolveInput(workflowName: string, scenario: string | undefined, inputFlag: string | undefined, commandName: string, catalog?: string): Promise<unknown>;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { ux } from '@oclif/core';
|
|
2
2
|
import { parseInputFlag } from '#utils/input_parser.js';
|
|
3
3
|
import { resolveScenarioPath, getScenarioNotFoundMessage } from '#utils/scenario_resolver.js';
|
|
4
|
-
export async function resolveInput(workflowName, scenario, inputFlag, commandName) {
|
|
4
|
+
export async function resolveInput(workflowName, scenario, inputFlag, commandName, catalog) {
|
|
5
5
|
if (inputFlag && scenario) {
|
|
6
6
|
return ux.error('Cannot use both scenario argument and --input flag. Choose one.', { exit: 1 });
|
|
7
7
|
}
|
|
@@ -9,7 +9,7 @@ export async function resolveInput(workflowName, scenario, inputFlag, commandNam
|
|
|
9
9
|
return parseInputFlag(inputFlag);
|
|
10
10
|
}
|
|
11
11
|
if (scenario) {
|
|
12
|
-
const resolution = await resolveScenarioPath(workflowName, scenario);
|
|
12
|
+
const resolution = await resolveScenarioPath(workflowName, scenario, undefined, undefined, catalog);
|
|
13
13
|
if (!resolution.found) {
|
|
14
14
|
return ux.error(getScenarioNotFoundMessage(workflowName, scenario, resolution.searchedPaths), { exit: 1 });
|
|
15
15
|
}
|
|
@@ -4,6 +4,6 @@ export interface ScenarioResolutionResult {
|
|
|
4
4
|
path?: string;
|
|
5
5
|
searchedPaths: string[];
|
|
6
6
|
}
|
|
7
|
-
export declare function resolveScenarioPath(workflowName: string, scenarioName: string, basePath?: string, workflowPath?: string): Promise<ScenarioResolutionResult>;
|
|
7
|
+
export declare function resolveScenarioPath(workflowName: string, scenarioName: string, basePath?: string, workflowPath?: string, catalog?: string): Promise<ScenarioResolutionResult>;
|
|
8
8
|
export declare function listScenariosForWorkflow(workflowName: string, workflowPath?: string, basePath?: string): string[];
|
|
9
9
|
export declare function getScenarioNotFoundMessage(workflowName: string, scenarioName: string, searchedPaths: string[]): string;
|
|
@@ -26,7 +26,7 @@ function resolveScenarioFromScenarioDirs(scenariosDirs, scenarioFileName) {
|
|
|
26
26
|
{ found: true, path, searchedPaths } :
|
|
27
27
|
{ found: false, searchedPaths };
|
|
28
28
|
}
|
|
29
|
-
export async function resolveScenarioPath(workflowName, scenarioName, basePath = getWorkflowsBasePath(), workflowPath) {
|
|
29
|
+
export async function resolveScenarioPath(workflowName, scenarioName, basePath = getWorkflowsBasePath(), workflowPath, catalog) {
|
|
30
30
|
const scenarioFileName = scenarioName.endsWith('.json') ?
|
|
31
31
|
scenarioName :
|
|
32
32
|
`${scenarioName}.json`;
|
|
@@ -36,7 +36,7 @@ export async function resolveScenarioPath(workflowName, scenarioName, basePath =
|
|
|
36
36
|
return pathResult;
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
-
const catalogPath = workflowPath ? null : await fetchWorkflowPath(workflowName);
|
|
39
|
+
const catalogPath = workflowPath ? null : await fetchWorkflowPath(workflowName, catalog);
|
|
40
40
|
if (catalogPath) {
|
|
41
41
|
const result = resolveScenarioFromScenarioDirs(candidateScenarioDirsFromPath(catalogPath, basePath), scenarioFileName);
|
|
42
42
|
if (result.found) {
|
|
@@ -156,6 +156,20 @@ describe('resolveScenarioPath', () => {
|
|
|
156
156
|
expect(result.path).toContain('complex/deep_test.json');
|
|
157
157
|
});
|
|
158
158
|
});
|
|
159
|
+
describe('catalog routing', () => {
|
|
160
|
+
it('forwards the provided catalog to the catalog lookup', async () => {
|
|
161
|
+
mockCatalog([{ name: 'my_workflow', path: '/app/dist/workflows/my_workflow/workflow.js' }]);
|
|
162
|
+
vi.mocked(fs.existsSync).mockReturnValue(false);
|
|
163
|
+
await resolveScenarioPath('my_workflow', 'test', '/project', undefined, 'os-workflows');
|
|
164
|
+
expect(catalog.fetchWorkflowCatalog).toHaveBeenCalledWith('os-workflows');
|
|
165
|
+
});
|
|
166
|
+
it('looks up the default catalog when no catalog is provided', async () => {
|
|
167
|
+
mockCatalog([{ name: 'my_workflow', path: '/app/dist/workflows/my_workflow/workflow.js' }]);
|
|
168
|
+
vi.mocked(fs.existsSync).mockReturnValue(false);
|
|
169
|
+
await resolveScenarioPath('my_workflow', 'test', '/project');
|
|
170
|
+
expect(catalog.fetchWorkflowCatalog).toHaveBeenCalledWith(undefined);
|
|
171
|
+
});
|
|
172
|
+
});
|
|
159
173
|
});
|
|
160
174
|
describe('listScenariosForWorkflow', () => {
|
|
161
175
|
beforeEach(() => {
|
|
@@ -2,7 +2,7 @@ export declare const WORKFLOWS_PATHS: string[];
|
|
|
2
2
|
export declare function extractWorkflowRelativePath(path: string): string | null;
|
|
3
3
|
export declare function candidateWorkflowDirsFromPath(workflowPath: string, basePath: string): string[];
|
|
4
4
|
export declare function findWorkflowDirectoryFromPath(workflowPath: string | undefined, basePath?: string): string | null;
|
|
5
|
-
export declare function fetchWorkflowPath(workflowName: string): Promise<string | null>;
|
|
5
|
+
export declare function fetchWorkflowPath(workflowName: string, catalog?: string): Promise<string | null>;
|
|
6
6
|
/**
|
|
7
7
|
* Resolve the on-disk directory of a registered workflow by name.
|
|
8
8
|
*
|
|
@@ -23,9 +23,9 @@ export function findWorkflowDirectoryFromPath(workflowPath, basePath = getWorkfl
|
|
|
23
23
|
}
|
|
24
24
|
return candidateWorkflowDirsFromPath(workflowPath, basePath).find(existsSync) ?? null;
|
|
25
25
|
}
|
|
26
|
-
export async function fetchWorkflowPath(workflowName) {
|
|
26
|
+
export async function fetchWorkflowPath(workflowName, catalog) {
|
|
27
27
|
try {
|
|
28
|
-
const workflows = await fetchWorkflowCatalog();
|
|
28
|
+
const workflows = await fetchWorkflowCatalog(catalog);
|
|
29
29
|
const workflow = workflows.find(w => w.name === workflowName);
|
|
30
30
|
return workflow?.path ?? null;
|
|
31
31
|
}
|
package/oclif.manifest.json
CHANGED
|
@@ -1164,6 +1164,22 @@
|
|
|
1164
1164
|
"allowNo": false,
|
|
1165
1165
|
"type": "boolean"
|
|
1166
1166
|
},
|
|
1167
|
+
"catalog": {
|
|
1168
|
+
"aliases": [
|
|
1169
|
+
"task-queue"
|
|
1170
|
+
],
|
|
1171
|
+
"char": "c",
|
|
1172
|
+
"charAliases": [
|
|
1173
|
+
"q"
|
|
1174
|
+
],
|
|
1175
|
+
"deprecateAliases": true,
|
|
1176
|
+
"description": "Catalog name for workflow execution (defaults to OUTPUT_CATALOG_ID)",
|
|
1177
|
+
"env": "OUTPUT_CATALOG_ID",
|
|
1178
|
+
"name": "catalog",
|
|
1179
|
+
"hasDynamicHelp": false,
|
|
1180
|
+
"multiple": false,
|
|
1181
|
+
"type": "option"
|
|
1182
|
+
},
|
|
1167
1183
|
"cached": {
|
|
1168
1184
|
"description": "Use cached output from dataset files (skip workflow execution)",
|
|
1169
1185
|
"exclusive": [
|
|
@@ -1228,6 +1244,22 @@
|
|
|
1228
1244
|
"<%= config.bin %> <%= command.id %> simple --download --limit 5"
|
|
1229
1245
|
],
|
|
1230
1246
|
"flags": {
|
|
1247
|
+
"catalog": {
|
|
1248
|
+
"aliases": [
|
|
1249
|
+
"task-queue"
|
|
1250
|
+
],
|
|
1251
|
+
"char": "c",
|
|
1252
|
+
"charAliases": [
|
|
1253
|
+
"q"
|
|
1254
|
+
],
|
|
1255
|
+
"deprecateAliases": true,
|
|
1256
|
+
"description": "Catalog name for workflow execution (defaults to OUTPUT_CATALOG_ID)",
|
|
1257
|
+
"env": "OUTPUT_CATALOG_ID",
|
|
1258
|
+
"name": "catalog",
|
|
1259
|
+
"hasDynamicHelp": false,
|
|
1260
|
+
"multiple": false,
|
|
1261
|
+
"type": "option"
|
|
1262
|
+
},
|
|
1231
1263
|
"trace": {
|
|
1232
1264
|
"char": "t",
|
|
1233
1265
|
"description": "Path to a local trace file to extract dataset from",
|
|
@@ -1422,5 +1454,5 @@
|
|
|
1422
1454
|
]
|
|
1423
1455
|
}
|
|
1424
1456
|
},
|
|
1425
|
-
"version": "0.8.2-next.
|
|
1457
|
+
"version": "0.8.2-next.7929835.0"
|
|
1426
1458
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@outputai/cli",
|
|
3
|
-
"version": "0.8.2-next.
|
|
3
|
+
"version": "0.8.2-next.7929835.0",
|
|
4
4
|
"description": "CLI for Output.ai workflow generation",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -34,11 +34,11 @@
|
|
|
34
34
|
"ky": "1.14.3",
|
|
35
35
|
"react": "19.2.5",
|
|
36
36
|
"semver": "7.7.4",
|
|
37
|
-
"undici": "8.
|
|
37
|
+
"undici": "8.5.0",
|
|
38
38
|
"yaml": "^2.8.3",
|
|
39
|
-
"@outputai/
|
|
40
|
-
"@outputai/
|
|
41
|
-
"@outputai/
|
|
39
|
+
"@outputai/evals": "0.8.2-next.7929835.0",
|
|
40
|
+
"@outputai/llm": "0.8.2-next.7929835.0",
|
|
41
|
+
"@outputai/credentials": "0.8.2-next.7929835.0"
|
|
42
42
|
},
|
|
43
43
|
"devDependencies": {
|
|
44
44
|
"@types/cli-progress": "3.11.6",
|