@outputai/cli 0.8.1-next.e92f632.0 → 0.8.2-dev.e78f6b4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/dist/assets/docker/docker-compose-dev.yml +1 -1
  2. package/dist/commands/workflow/cost.d.ts +3 -2
  3. package/dist/commands/workflow/cost.js +4 -11
  4. package/dist/commands/workflow/cost.spec.js +1 -3
  5. package/dist/commands/workflow/dataset/generate.d.ts +1 -0
  6. package/dist/commands/workflow/dataset/generate.js +18 -9
  7. package/dist/commands/workflow/dataset/generate.spec.d.ts +1 -0
  8. package/dist/commands/workflow/dataset/generate.spec.js +69 -0
  9. package/dist/commands/workflow/dataset/list.d.ts +3 -1
  10. package/dist/commands/workflow/dataset/list.js +10 -14
  11. package/dist/commands/workflow/debug.d.ts +2 -6
  12. package/dist/commands/workflow/debug.js +10 -29
  13. package/dist/commands/workflow/debug.spec.js +2 -5
  14. package/dist/commands/workflow/generate.spec.js +2 -2
  15. package/dist/commands/workflow/list.d.ts +4 -1
  16. package/dist/commands/workflow/list.js +15 -19
  17. package/dist/commands/workflow/list.spec.js +2 -1
  18. package/dist/commands/workflow/result.d.ts +3 -4
  19. package/dist/commands/workflow/result.js +6 -15
  20. package/dist/commands/workflow/result.spec.js +2 -4
  21. package/dist/commands/workflow/run.d.ts +3 -2
  22. package/dist/commands/workflow/run.js +5 -12
  23. package/dist/commands/workflow/run.spec.js +18 -3
  24. package/dist/commands/workflow/runs/list.d.ts +3 -1
  25. package/dist/commands/workflow/runs/list.js +11 -15
  26. package/dist/commands/workflow/start.js +1 -1
  27. package/dist/commands/workflow/start.spec.js +53 -2
  28. package/dist/commands/workflow/status.d.ts +3 -4
  29. package/dist/commands/workflow/status.js +20 -30
  30. package/dist/commands/workflow/status.spec.js +2 -4
  31. package/dist/commands/workflow/test_eval.d.ts +5 -2
  32. package/dist/commands/workflow/test_eval.js +28 -19
  33. package/dist/commands/workflow/test_eval.spec.d.ts +1 -0
  34. package/dist/commands/workflow/test_eval.spec.js +121 -0
  35. package/dist/generated/framework_version.json +1 -1
  36. package/dist/hooks/init.d.ts +1 -0
  37. package/dist/hooks/init.js +40 -30
  38. package/dist/hooks/init.spec.js +28 -4
  39. package/dist/services/coding_agents.spec.js +10 -10
  40. package/dist/services/datasets.d.ts +2 -2
  41. package/dist/services/datasets.js +19 -17
  42. package/dist/services/datasets.test.js +37 -2
  43. package/dist/utils/eval_diagnostics.d.ts +7 -0
  44. package/dist/utils/eval_diagnostics.js +35 -0
  45. package/dist/utils/format_workflow_result.spec.js +0 -13
  46. package/dist/utils/resolve_input.d.ts +1 -1
  47. package/dist/utils/resolve_input.js +2 -2
  48. package/dist/utils/scenario_resolver.d.ts +2 -3
  49. package/dist/utils/scenario_resolver.js +5 -35
  50. package/dist/utils/scenario_resolver.spec.js +14 -0
  51. package/dist/utils/trace_formatter.js +1 -2
  52. package/dist/utils/workflow_dir.d.ts +13 -0
  53. package/dist/utils/workflow_dir.js +58 -0
  54. package/dist/utils/workflow_dir.spec.d.ts +1 -0
  55. package/dist/utils/workflow_dir.spec.js +60 -0
  56. package/oclif.manifest.json +214 -201
  57. package/package.json +5 -5
  58. package/dist/utils/constants.d.ts +0 -5
  59. package/dist/utils/constants.js +0 -4
  60. package/dist/utils/output_formatter.d.ts +0 -2
  61. package/dist/utils/output_formatter.js +0 -11
@@ -1,43 +1,13 @@
1
1
  import { existsSync, readdirSync } from 'node:fs';
2
- import { dirname, resolve } from 'node:path';
3
- import { fetchWorkflowCatalog } from '#api/workflow_catalog.js';
2
+ import { resolve } from 'node:path';
4
3
  import { getWorkflowsBasePath } from '#utils/paths.js';
4
+ import { WORKFLOWS_PATHS, candidateWorkflowDirsFromPath, fetchWorkflowPath } from '#utils/workflow_dir.js';
5
+ export { extractWorkflowRelativePath, findWorkflowDirectoryFromPath } from '#utils/workflow_dir.js';
5
6
  const SCENARIOS_DIR = 'scenarios';
6
- const WORKFLOWS_PATHS = ['src/workflows', 'workflows'];
7
- export function extractWorkflowRelativePath(path) {
8
- const match = path.match(/(?:^|\/)workflows\/(.+)\/workflow\.[jt]s$/);
9
- return match ? match[1] : null;
10
- }
11
- function unique(values) {
12
- return [...new Set(values)];
13
- }
14
- function workflowPathSuffixes(workflowPath) {
15
- const parts = dirname(workflowPath).split(/[/\\]+/).filter(Boolean);
16
- return parts.map((_, index) => parts.slice(index));
17
- }
18
- function candidateWorkflowDirsFromPath(workflowPath, basePath) {
19
- return unique(workflowPathSuffixes(workflowPath).flatMap(suffix => WORKFLOWS_PATHS.map(workflowsDir => resolve(basePath, workflowsDir, ...suffix))));
20
- }
21
7
  function candidateScenarioDirsFromPath(workflowPath, basePath) {
22
8
  return candidateWorkflowDirsFromPath(workflowPath, basePath)
23
9
  .map(workflowDir => resolve(workflowDir, SCENARIOS_DIR));
24
10
  }
25
- export function findWorkflowDirectoryFromPath(workflowPath, basePath = getWorkflowsBasePath()) {
26
- if (!workflowPath) {
27
- return null;
28
- }
29
- return candidateWorkflowDirsFromPath(workflowPath, basePath).find(existsSync) ?? null;
30
- }
31
- async function fetchWorkflowPath(workflowName) {
32
- try {
33
- const workflows = await fetchWorkflowCatalog();
34
- const workflow = workflows.find(w => w.name === workflowName);
35
- return workflow?.path ?? null;
36
- }
37
- catch {
38
- return null;
39
- }
40
- }
41
11
  function resolveScenarioFromDirectory(relativeDir, scenarioFileName, basePath) {
42
12
  const searchedPaths = [];
43
13
  for (const workflowsDir of WORKFLOWS_PATHS) {
@@ -56,7 +26,7 @@ function resolveScenarioFromScenarioDirs(scenariosDirs, scenarioFileName) {
56
26
  { found: true, path, searchedPaths } :
57
27
  { found: false, searchedPaths };
58
28
  }
59
- export async function resolveScenarioPath(workflowName, scenarioName, basePath = getWorkflowsBasePath(), workflowPath) {
29
+ export async function resolveScenarioPath(workflowName, scenarioName, basePath = getWorkflowsBasePath(), workflowPath, catalog) {
60
30
  const scenarioFileName = scenarioName.endsWith('.json') ?
61
31
  scenarioName :
62
32
  `${scenarioName}.json`;
@@ -66,7 +36,7 @@ export async function resolveScenarioPath(workflowName, scenarioName, basePath =
66
36
  return pathResult;
67
37
  }
68
38
  }
69
- const catalogPath = workflowPath ? null : await fetchWorkflowPath(workflowName);
39
+ const catalogPath = workflowPath ? null : await fetchWorkflowPath(workflowName, catalog);
70
40
  if (catalogPath) {
71
41
  const result = resolveScenarioFromScenarioDirs(candidateScenarioDirsFromPath(catalogPath, basePath), scenarioFileName);
72
42
  if (result.found) {
@@ -156,6 +156,20 @@ describe('resolveScenarioPath', () => {
156
156
  expect(result.path).toContain('complex/deep_test.json');
157
157
  });
158
158
  });
159
+ describe('catalog routing', () => {
160
+ it('forwards the provided catalog to the catalog lookup', async () => {
161
+ mockCatalog([{ name: 'my_workflow', path: '/app/dist/workflows/my_workflow/workflow.js' }]);
162
+ vi.mocked(fs.existsSync).mockReturnValue(false);
163
+ await resolveScenarioPath('my_workflow', 'test', '/project', undefined, 'os-workflows');
164
+ expect(catalog.fetchWorkflowCatalog).toHaveBeenCalledWith('os-workflows');
165
+ });
166
+ it('looks up the default catalog when no catalog is provided', async () => {
167
+ mockCatalog([{ name: 'my_workflow', path: '/app/dist/workflows/my_workflow/workflow.js' }]);
168
+ vi.mocked(fs.existsSync).mockReturnValue(false);
169
+ await resolveScenarioPath('my_workflow', 'test', '/project');
170
+ expect(catalog.fetchWorkflowCatalog).toHaveBeenCalledWith(undefined);
171
+ });
172
+ });
159
173
  });
160
174
  describe('listScenariosForWorkflow', () => {
161
175
  beforeEach(() => {
@@ -1,6 +1,5 @@
1
1
  import Table from 'cli-table3';
2
2
  import { ux } from '@oclif/core';
3
- import { formatOutput } from '#utils/output_formatter.js';
4
3
  import { formatDuration } from '#utils/date_formatter.js';
5
4
  import { getErrorMessage } from '#utils/error_utils.js';
6
5
  import { isTraceEvent, isValidTimestamp } from '#types/trace.js';
@@ -369,7 +368,7 @@ const formatAsText = (trace) => {
369
368
  export function format(traceData, outputFormat = 'text') {
370
369
  const trace = typeof traceData === 'string' ? JSON.parse(traceData) : traceData;
371
370
  if (outputFormat === 'json') {
372
- return formatOutput(trace, 'json');
371
+ return JSON.stringify(trace, null, 2);
373
372
  }
374
373
  return formatAsText(trace);
375
374
  }
@@ -0,0 +1,13 @@
1
+ export declare const WORKFLOWS_PATHS: string[];
2
+ export declare function extractWorkflowRelativePath(path: string): string | null;
3
+ export declare function candidateWorkflowDirsFromPath(workflowPath: string, basePath: string): string[];
4
+ export declare function findWorkflowDirectoryFromPath(workflowPath: string | undefined, basePath?: string): string | null;
5
+ export declare function fetchWorkflowPath(workflowName: string, catalog?: string): Promise<string | null>;
6
+ /**
7
+ * Resolve the on-disk directory of a registered workflow by name.
8
+ *
9
+ * Flat-first so flat-layout projects resolve offline (no catalog round-trip)
10
+ * exactly as before; nested folders fall through to the worker catalog, whose
11
+ * `path` is re-rooted under `src/workflows` / `workflows`.
12
+ */
13
+ export declare function resolveWorkflowDir(workflowName: string, basePath?: string, workflowPath?: string): Promise<string | null>;
@@ -0,0 +1,58 @@
1
+ import { existsSync } from 'node:fs';
2
+ import { dirname, resolve } from 'node:path';
3
+ import { fetchWorkflowCatalog } from '#api/workflow_catalog.js';
4
+ import { getWorkflowsBasePath } from '#utils/paths.js';
5
+ export const WORKFLOWS_PATHS = ['src/workflows', 'workflows'];
6
+ export function extractWorkflowRelativePath(path) {
7
+ const match = path.match(/(?:^|\/)workflows\/(.+)\/workflow\.[jt]s$/);
8
+ return match ? match[1] : null;
9
+ }
10
+ function unique(values) {
11
+ return [...new Set(values)];
12
+ }
13
+ function workflowPathSuffixes(workflowPath) {
14
+ const parts = dirname(workflowPath).split(/[/\\]+/).filter(Boolean);
15
+ return parts.map((_, index) => parts.slice(index));
16
+ }
17
+ export function candidateWorkflowDirsFromPath(workflowPath, basePath) {
18
+ return unique(workflowPathSuffixes(workflowPath).flatMap(suffix => WORKFLOWS_PATHS.map(workflowsDir => resolve(basePath, workflowsDir, ...suffix))));
19
+ }
20
+ export function findWorkflowDirectoryFromPath(workflowPath, basePath = getWorkflowsBasePath()) {
21
+ if (!workflowPath) {
22
+ return null;
23
+ }
24
+ return candidateWorkflowDirsFromPath(workflowPath, basePath).find(existsSync) ?? null;
25
+ }
26
+ export async function fetchWorkflowPath(workflowName, catalog) {
27
+ try {
28
+ const workflows = await fetchWorkflowCatalog(catalog);
29
+ const workflow = workflows.find(w => w.name === workflowName);
30
+ return workflow?.path ?? null;
31
+ }
32
+ catch {
33
+ return null;
34
+ }
35
+ }
36
+ /**
37
+ * Resolve the on-disk directory of a registered workflow by name.
38
+ *
39
+ * Flat-first so flat-layout projects resolve offline (no catalog round-trip)
40
+ * exactly as before; nested folders fall through to the worker catalog, whose
41
+ * `path` is re-rooted under `src/workflows` / `workflows`.
42
+ */
43
+ export async function resolveWorkflowDir(workflowName, basePath = getWorkflowsBasePath(), workflowPath) {
44
+ for (const workflowsDir of WORKFLOWS_PATHS) {
45
+ const candidate = resolve(basePath, workflowsDir, workflowName);
46
+ if (existsSync(candidate)) {
47
+ return candidate;
48
+ }
49
+ }
50
+ if (workflowPath) {
51
+ const found = findWorkflowDirectoryFromPath(workflowPath, basePath);
52
+ if (found) {
53
+ return found;
54
+ }
55
+ }
56
+ const catalogPath = await fetchWorkflowPath(workflowName);
57
+ return findWorkflowDirectoryFromPath(catalogPath ?? undefined, basePath);
58
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,60 @@
1
+ import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest';
2
+ import { resolveWorkflowDir } from './workflow_dir.js';
3
+ import * as fs from 'node:fs';
4
+ import * as catalog from '#api/workflow_catalog.js';
5
+ vi.mock('node:fs', () => ({
6
+ existsSync: vi.fn()
7
+ }));
8
+ vi.mock('#api/workflow_catalog.js', () => ({
9
+ fetchWorkflowCatalog: vi.fn()
10
+ }));
11
+ function mockCatalog(workflows) {
12
+ vi.mocked(catalog.fetchWorkflowCatalog).mockResolvedValue(workflows);
13
+ }
14
+ function mockCatalogFailure() {
15
+ vi.mocked(catalog.fetchWorkflowCatalog).mockRejectedValue(new Error('API unavailable'));
16
+ }
17
+ describe('resolveWorkflowDir', () => {
18
+ beforeEach(() => {
19
+ vi.resetAllMocks();
20
+ });
21
+ afterEach(() => {
22
+ vi.restoreAllMocks();
23
+ });
24
+ it('resolves a flat layout without querying the catalog', async () => {
25
+ vi.mocked(fs.existsSync).mockImplementation(path => String(path) === '/project/src/workflows/simple');
26
+ const result = await resolveWorkflowDir('simple', '/project');
27
+ expect(result).toBe('/project/src/workflows/simple');
28
+ expect(catalog.fetchWorkflowCatalog).not.toHaveBeenCalled();
29
+ });
30
+ it('resolves the workflows/ fallback without querying the catalog', async () => {
31
+ vi.mocked(fs.existsSync).mockImplementation(path => String(path) === '/project/workflows/simple');
32
+ const result = await resolveWorkflowDir('simple', '/project');
33
+ expect(result).toBe('/project/workflows/simple');
34
+ expect(catalog.fetchWorkflowCatalog).not.toHaveBeenCalled();
35
+ });
36
+ it('resolves a nested folder via the catalog path', async () => {
37
+ mockCatalog([{ name: 'a_b_c', path: '/app/dist/workflows/a/b/c/workflow.js' }]);
38
+ vi.mocked(fs.existsSync).mockImplementation(path => String(path) === '/project/src/workflows/a/b/c');
39
+ const result = await resolveWorkflowDir('a_b_c', '/project');
40
+ expect(result).toBe('/project/src/workflows/a/b/c');
41
+ });
42
+ it('returns null when the catalog is unavailable and no flat dir exists', async () => {
43
+ mockCatalogFailure();
44
+ vi.mocked(fs.existsSync).mockReturnValue(false);
45
+ const result = await resolveWorkflowDir('a_b_c', '/project');
46
+ expect(result).toBeNull();
47
+ });
48
+ it('returns null when the workflow is not in the catalog', async () => {
49
+ mockCatalog([{ name: 'other', path: '/app/dist/workflows/other/workflow.js' }]);
50
+ vi.mocked(fs.existsSync).mockReturnValue(false);
51
+ const result = await resolveWorkflowDir('a_b_c', '/project');
52
+ expect(result).toBeNull();
53
+ });
54
+ it('uses a provided workflowPath without querying the catalog', async () => {
55
+ vi.mocked(fs.existsSync).mockImplementation(path => String(path) === '/project/src/workflows/writing/editor');
56
+ const result = await resolveWorkflowDir('writing_editor', '/project', '/app/build-output/writing/editor/workflow.js');
57
+ expect(result).toBe('/project/src/workflows/writing/editor');
58
+ expect(catalog.fetchWorkflowCatalog).not.toHaveBeenCalled();
59
+ });
60
+ });