@outputai/core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +11 -0
- package/bin/healthcheck.mjs +36 -0
- package/bin/healthcheck.spec.js +90 -0
- package/bin/worker.sh +26 -0
- package/package.json +67 -0
- package/src/activity_integration/context.d.ts +27 -0
- package/src/activity_integration/context.js +17 -0
- package/src/activity_integration/context.spec.js +42 -0
- package/src/activity_integration/events.d.ts +7 -0
- package/src/activity_integration/events.js +10 -0
- package/src/activity_integration/index.d.ts +9 -0
- package/src/activity_integration/index.js +3 -0
- package/src/activity_integration/tracing.d.ts +32 -0
- package/src/activity_integration/tracing.js +37 -0
- package/src/async_storage.js +19 -0
- package/src/bus.js +3 -0
- package/src/consts.js +32 -0
- package/src/errors.d.ts +15 -0
- package/src/errors.js +14 -0
- package/src/hooks/index.d.ts +28 -0
- package/src/hooks/index.js +32 -0
- package/src/index.d.ts +49 -0
- package/src/index.js +4 -0
- package/src/interface/evaluation_result.d.ts +173 -0
- package/src/interface/evaluation_result.js +215 -0
- package/src/interface/evaluator.d.ts +70 -0
- package/src/interface/evaluator.js +34 -0
- package/src/interface/evaluator.spec.js +565 -0
- package/src/interface/index.d.ts +9 -0
- package/src/interface/index.js +26 -0
- package/src/interface/step.d.ts +138 -0
- package/src/interface/step.js +22 -0
- package/src/interface/types.d.ts +27 -0
- package/src/interface/validations/runtime.js +20 -0
- package/src/interface/validations/runtime.spec.js +29 -0
- package/src/interface/validations/schema_utils.js +8 -0
- package/src/interface/validations/schema_utils.spec.js +67 -0
- package/src/interface/validations/static.js +136 -0
- package/src/interface/validations/static.spec.js +366 -0
- package/src/interface/webhook.d.ts +84 -0
- package/src/interface/webhook.js +64 -0
- package/src/interface/webhook.spec.js +122 -0
- package/src/interface/workflow.d.ts +273 -0
- package/src/interface/workflow.js +128 -0
- package/src/interface/workflow.spec.js +467 -0
- package/src/interface/workflow_context.js +31 -0
- package/src/interface/workflow_utils.d.ts +76 -0
- package/src/interface/workflow_utils.js +50 -0
- package/src/interface/workflow_utils.spec.js +190 -0
- package/src/interface/zod_integration.spec.js +646 -0
- package/src/internal_activities/index.js +66 -0
- package/src/internal_activities/index.spec.js +102 -0
- package/src/logger.js +73 -0
- package/src/tracing/internal_interface.js +71 -0
- package/src/tracing/processors/local/index.js +111 -0
- package/src/tracing/processors/local/index.spec.js +149 -0
- package/src/tracing/processors/s3/configs.js +31 -0
- package/src/tracing/processors/s3/configs.spec.js +64 -0
- package/src/tracing/processors/s3/index.js +114 -0
- package/src/tracing/processors/s3/index.spec.js +153 -0
- package/src/tracing/processors/s3/redis_client.js +62 -0
- package/src/tracing/processors/s3/redis_client.spec.js +185 -0
- package/src/tracing/processors/s3/s3_client.js +27 -0
- package/src/tracing/processors/s3/s3_client.spec.js +62 -0
- package/src/tracing/tools/build_trace_tree.js +83 -0
- package/src/tracing/tools/build_trace_tree.spec.js +135 -0
- package/src/tracing/tools/utils.js +21 -0
- package/src/tracing/tools/utils.spec.js +14 -0
- package/src/tracing/trace_engine.js +97 -0
- package/src/tracing/trace_engine.spec.js +199 -0
- package/src/utils/index.d.ts +134 -0
- package/src/utils/index.js +2 -0
- package/src/utils/resolve_invocation_dir.js +34 -0
- package/src/utils/resolve_invocation_dir.spec.js +102 -0
- package/src/utils/utils.js +211 -0
- package/src/utils/utils.spec.js +448 -0
- package/src/worker/bundler_options.js +43 -0
- package/src/worker/catalog_workflow/catalog.js +114 -0
- package/src/worker/catalog_workflow/index.js +54 -0
- package/src/worker/catalog_workflow/index.spec.js +196 -0
- package/src/worker/catalog_workflow/workflow.js +24 -0
- package/src/worker/configs.js +49 -0
- package/src/worker/configs.spec.js +130 -0
- package/src/worker/index.js +89 -0
- package/src/worker/index.spec.js +177 -0
- package/src/worker/interceptors/activity.js +62 -0
- package/src/worker/interceptors/activity.spec.js +212 -0
- package/src/worker/interceptors/workflow.js +70 -0
- package/src/worker/interceptors/workflow.spec.js +167 -0
- package/src/worker/interceptors.js +10 -0
- package/src/worker/loader.js +151 -0
- package/src/worker/loader.spec.js +236 -0
- package/src/worker/loader_tools.js +132 -0
- package/src/worker/loader_tools.spec.js +156 -0
- package/src/worker/log_hooks.js +95 -0
- package/src/worker/log_hooks.spec.js +217 -0
- package/src/worker/sandboxed_utils.js +18 -0
- package/src/worker/shutdown.js +26 -0
- package/src/worker/shutdown.spec.js +82 -0
- package/src/worker/sinks.js +74 -0
- package/src/worker/start_catalog.js +36 -0
- package/src/worker/start_catalog.spec.js +118 -0
- package/src/worker/webpack_loaders/consts.js +9 -0
- package/src/worker/webpack_loaders/tools.js +548 -0
- package/src/worker/webpack_loaders/tools.spec.js +330 -0
- package/src/worker/webpack_loaders/workflow_rewriter/collect_target_imports.js +221 -0
- package/src/worker/webpack_loaders/workflow_rewriter/collect_target_imports.spec.js +336 -0
- package/src/worker/webpack_loaders/workflow_rewriter/index.mjs +61 -0
- package/src/worker/webpack_loaders/workflow_rewriter/index.spec.js +216 -0
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_fn_bodies.js +196 -0
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_fn_bodies.spec.js +123 -0
- package/src/worker/webpack_loaders/workflow_validator/index.mjs +205 -0
- package/src/worker/webpack_loaders/workflow_validator/index.spec.js +613 -0
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
|
|
5
|
+
vi.mock( '#consts', () => ( {
|
|
6
|
+
ACTIVITY_SEND_HTTP_REQUEST: '__internal#sendHttpRequest',
|
|
7
|
+
ACTIVITY_GET_TRACE_DESTINATIONS: '__internal#getTraceDestinations',
|
|
8
|
+
WORKFLOWS_INDEX_FILENAME: '__workflows_entrypoint.js',
|
|
9
|
+
WORKFLOW_CATALOG: 'catalog',
|
|
10
|
+
ACTIVITY_OPTIONS_FILENAME: '__activity_options.js',
|
|
11
|
+
SHARED_STEP_PREFIX: '$shared'
|
|
12
|
+
} ) );
|
|
13
|
+
|
|
14
|
+
const sendHttpRequestMock = vi.fn();
|
|
15
|
+
const getTraceDestinationsMock = vi.fn();
|
|
16
|
+
vi.mock( '#internal_activities', () => ( {
|
|
17
|
+
sendHttpRequest: sendHttpRequestMock,
|
|
18
|
+
getTraceDestinations: getTraceDestinationsMock
|
|
19
|
+
} ) );
|
|
20
|
+
|
|
21
|
+
const importComponentsMock = vi.fn();
|
|
22
|
+
vi.mock( './loader_tools.js', async importOriginal => {
|
|
23
|
+
const actual = await importOriginal();
|
|
24
|
+
return { ...actual, importComponents: importComponentsMock };
|
|
25
|
+
} );
|
|
26
|
+
|
|
27
|
+
const fsMocks = vi.hoisted( () => ( {
|
|
28
|
+
mkdirSync: vi.fn(),
|
|
29
|
+
writeFileSync: vi.fn(),
|
|
30
|
+
existsSync: vi.fn().mockReturnValue( false )
|
|
31
|
+
} ) );
|
|
32
|
+
vi.mock( 'node:fs', () => ( {
|
|
33
|
+
mkdirSync: fsMocks.mkdirSync,
|
|
34
|
+
writeFileSync: fsMocks.writeFileSync,
|
|
35
|
+
existsSync: fsMocks.existsSync
|
|
36
|
+
} ) );
|
|
37
|
+
|
|
38
|
+
describe( 'worker/loader', () => {
|
|
39
|
+
beforeEach( () => {
|
|
40
|
+
vi.clearAllMocks();
|
|
41
|
+
} );
|
|
42
|
+
|
|
43
|
+
it( 'loadActivities returns map including system activity and writes options file', async () => {
|
|
44
|
+
const { loadActivities } = await import( './loader.js' );
|
|
45
|
+
|
|
46
|
+
// First call: workflow directory scan (options.activityOptions propagated to activity options file)
|
|
47
|
+
importComponentsMock.mockImplementationOnce( async function *() {
|
|
48
|
+
yield {
|
|
49
|
+
fn: () => {},
|
|
50
|
+
metadata: { name: 'Act1', options: { activityOptions: { retry: { maximumAttempts: 3 } } } },
|
|
51
|
+
path: '/a/steps.js'
|
|
52
|
+
};
|
|
53
|
+
} );
|
|
54
|
+
// Second call: shared activities scan (no results)
|
|
55
|
+
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
56
|
+
|
|
57
|
+
const workflows = [ { name: 'A', path: '/a/workflow.js' } ];
|
|
58
|
+
const activities = await loadActivities( '/root', workflows );
|
|
59
|
+
expect( activities['A#Act1'] ).toBeTypeOf( 'function' );
|
|
60
|
+
expect( activities['__internal#sendHttpRequest'] ).toBe( sendHttpRequestMock );
|
|
61
|
+
|
|
62
|
+
// options file written with the collected activityOptions map
|
|
63
|
+
expect( fsMocks.writeFileSync ).toHaveBeenCalledTimes( 1 );
|
|
64
|
+
const [ writtenPath, contents ] = fsMocks.writeFileSync.mock.calls[0];
|
|
65
|
+
expect( writtenPath ).toMatch( /temp\/__activity_options\.js$/ );
|
|
66
|
+
expect( contents ).toContain( 'export default' );
|
|
67
|
+
expect( JSON.parse( contents.replace( /^export default\s*/, '' ).replace( /;\s*$/, '' ) ) ).toEqual( {
|
|
68
|
+
'A#Act1': { retry: { maximumAttempts: 3 } }
|
|
69
|
+
} );
|
|
70
|
+
expect( fsMocks.mkdirSync ).toHaveBeenCalled();
|
|
71
|
+
} );
|
|
72
|
+
|
|
73
|
+
it( 'loadActivities omits activity options when component has no options or no activityOptions', async () => {
|
|
74
|
+
const { loadActivities } = await import( './loader.js' );
|
|
75
|
+
importComponentsMock.mockImplementationOnce( async function *() {
|
|
76
|
+
yield { fn: () => {}, metadata: { name: 'NoOptions' }, path: '/a/steps.js' };
|
|
77
|
+
yield { fn: () => {}, metadata: { name: 'EmptyOptions', options: {} }, path: '/a/steps2.js' };
|
|
78
|
+
} );
|
|
79
|
+
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
80
|
+
|
|
81
|
+
await loadActivities( '/root', [ { name: 'A', path: '/a/workflow.js' } ] );
|
|
82
|
+
const written = JSON.parse(
|
|
83
|
+
fsMocks.writeFileSync.mock.calls[0][1].replace( /^export default\s*/, '' ).replace( /;\s*$/, '' )
|
|
84
|
+
);
|
|
85
|
+
expect( written['A#NoOptions'] ).toBeUndefined();
|
|
86
|
+
expect( written['A#EmptyOptions'] ).toBeUndefined();
|
|
87
|
+
} );
|
|
88
|
+
|
|
89
|
+
it( 'loadWorkflows returns array of workflows with metadata', async () => {
|
|
90
|
+
const { loadWorkflows } = await import( './loader.js' );
|
|
91
|
+
|
|
92
|
+
importComponentsMock.mockImplementationOnce( async function *() {
|
|
93
|
+
yield { metadata: { name: 'Flow1', description: 'd' }, path: '/b/workflow.js' };
|
|
94
|
+
} );
|
|
95
|
+
|
|
96
|
+
const workflows = await loadWorkflows( '/root' );
|
|
97
|
+
expect( workflows ).toEqual( [ { name: 'Flow1', description: 'd', path: '/b/workflow.js' } ] );
|
|
98
|
+
} );
|
|
99
|
+
|
|
100
|
+
it( 'createWorkflowsEntryPoint writes index and returns its path', async () => {
|
|
101
|
+
const { createWorkflowsEntryPoint } = await import( './loader.js' );
|
|
102
|
+
|
|
103
|
+
const workflows = [ { name: 'W', path: '/abs/wf.js' } ];
|
|
104
|
+
const entry = createWorkflowsEntryPoint( workflows );
|
|
105
|
+
|
|
106
|
+
expect( fsMocks.writeFileSync ).toHaveBeenCalledTimes( 1 );
|
|
107
|
+
const [ writtenPath, contents ] = fsMocks.writeFileSync.mock.calls[0];
|
|
108
|
+
expect( entry ).toBe( writtenPath );
|
|
109
|
+
expect( contents ).toContain( 'export { default as W } from \'/abs/wf.js\';' );
|
|
110
|
+
expect( contents ).toContain( 'export { default as catalog }' );
|
|
111
|
+
expect( fsMocks.mkdirSync ).toHaveBeenCalledTimes( 1 );
|
|
112
|
+
} );
|
|
113
|
+
|
|
114
|
+
it( 'loadActivities uses folder-based matchers for steps/evaluators and shared', async () => {
|
|
115
|
+
const { loadActivities } = await import( './loader.js' );
|
|
116
|
+
// First call (workflow dir): no results
|
|
117
|
+
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
118
|
+
// Second call (shared): no results
|
|
119
|
+
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
120
|
+
|
|
121
|
+
const workflows = [ { name: 'A', path: '/a/workflow.js' } ];
|
|
122
|
+
await loadActivities( '/root', workflows );
|
|
123
|
+
|
|
124
|
+
// First invocation should target the workflow directory with folder/file matchers
|
|
125
|
+
expect( importComponentsMock ).toHaveBeenCalledTimes( 2 );
|
|
126
|
+
const [ firstDir, firstMatchers ] = importComponentsMock.mock.calls[0];
|
|
127
|
+
expect( firstDir ).toBe( '/a' );
|
|
128
|
+
expect( Array.isArray( firstMatchers ) ).toBe( true );
|
|
129
|
+
// Should match folder-based steps and evaluators files
|
|
130
|
+
expect( firstMatchers.some( fn => fn( '/a/steps/foo.js' ) ) ).toBe( true );
|
|
131
|
+
expect( firstMatchers.some( fn => fn( '/a/evaluators/bar.js' ) ) ).toBe( true );
|
|
132
|
+
// And also direct file names
|
|
133
|
+
expect( firstMatchers.some( fn => fn( '/a/steps.js' ) ) ).toBe( true );
|
|
134
|
+
expect( firstMatchers.some( fn => fn( '/a/evaluators.js' ) ) ).toBe( true );
|
|
135
|
+
|
|
136
|
+
// Second invocation should target root with shared matchers
|
|
137
|
+
const [ secondDir, secondMatchers ] = importComponentsMock.mock.calls[1];
|
|
138
|
+
expect( secondDir ).toBe( '/root' );
|
|
139
|
+
expect( secondMatchers.some( fn => fn( '/root/shared/steps/baz.js' ) ) ).toBe( true );
|
|
140
|
+
expect( secondMatchers.some( fn => fn( '/root/shared/evaluators/qux.js' ) ) ).toBe( true );
|
|
141
|
+
} );
|
|
142
|
+
|
|
143
|
+
it( 'loadActivities includes nested workflow steps and shared evaluators', async () => {
|
|
144
|
+
const { loadActivities } = await import( './loader.js' );
|
|
145
|
+
// Workflow dir scan returns a nested step
|
|
146
|
+
importComponentsMock.mockImplementationOnce( async function *() {
|
|
147
|
+
yield { fn: () => {}, metadata: { name: 'ActNested' }, path: '/a/steps/foo.js' };
|
|
148
|
+
} );
|
|
149
|
+
// Shared scan returns a shared evaluator
|
|
150
|
+
importComponentsMock.mockImplementationOnce( async function *() {
|
|
151
|
+
yield { fn: () => {}, metadata: { name: 'SharedEval' }, path: '/root/shared/evaluators/bar.js' };
|
|
152
|
+
} );
|
|
153
|
+
|
|
154
|
+
const workflows = [ { name: 'A', path: '/a/workflow.js' } ];
|
|
155
|
+
const activities = await loadActivities( '/root', workflows );
|
|
156
|
+
expect( activities['A#ActNested'] ).toBeTypeOf( 'function' );
|
|
157
|
+
expect( activities['$shared#SharedEval'] ).toBeTypeOf( 'function' );
|
|
158
|
+
} );
|
|
159
|
+
|
|
160
|
+
it( 'loadWorkflows throws when workflow is under shared directory', async () => {
|
|
161
|
+
const { loadWorkflows } = await import( './loader.js' );
|
|
162
|
+
importComponentsMock.mockImplementationOnce( async function *() {
|
|
163
|
+
yield { metadata: { name: 'Invalid' }, path: '/root/shared/workflow.js' };
|
|
164
|
+
} );
|
|
165
|
+
await expect( loadWorkflows( '/root' ) ).rejects.toThrow( 'Workflow directory can\'t be named \"shared\"' );
|
|
166
|
+
} );
|
|
167
|
+
|
|
168
|
+
it( 'collects workflow nested steps and evaluators across multiple subfolders', async () => {
|
|
169
|
+
const { loadActivities } = await import( './loader.js' );
|
|
170
|
+
// Workflow dir scan returns nested steps and evaluators
|
|
171
|
+
importComponentsMock.mockImplementationOnce( async function *() {
|
|
172
|
+
yield { fn: () => {}, metadata: { name: 'StepPrimary' }, path: '/a/steps/primary/foo.js' };
|
|
173
|
+
yield { fn: () => {}, metadata: { name: 'StepSecondary' }, path: '/a/steps/secondary/bar.js' };
|
|
174
|
+
yield { fn: () => {}, metadata: { name: 'EvalPrimary' }, path: '/a/evaluators/primary/baz.js' };
|
|
175
|
+
yield { fn: () => {}, metadata: { name: 'EvalSecondary' }, path: '/a/evaluators/secondary/qux.js' };
|
|
176
|
+
} );
|
|
177
|
+
// Shared scan returns nothing for this test
|
|
178
|
+
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
179
|
+
|
|
180
|
+
const workflows = [ { name: 'A', path: '/a/workflow.js' } ];
|
|
181
|
+
const activities = await loadActivities( '/root', workflows );
|
|
182
|
+
expect( activities['A#StepPrimary'] ).toBeTypeOf( 'function' );
|
|
183
|
+
expect( activities['A#StepSecondary'] ).toBeTypeOf( 'function' );
|
|
184
|
+
expect( activities['A#EvalPrimary'] ).toBeTypeOf( 'function' );
|
|
185
|
+
expect( activities['A#EvalSecondary'] ).toBeTypeOf( 'function' );
|
|
186
|
+
} );
|
|
187
|
+
|
|
188
|
+
it( 'collects shared nested steps and evaluators across multiple subfolders', async () => {
|
|
189
|
+
const { loadActivities } = await import( './loader.js' );
|
|
190
|
+
// Workflow dir scan returns nothing for this test
|
|
191
|
+
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
192
|
+
// Shared scan returns nested steps and evaluators
|
|
193
|
+
importComponentsMock.mockImplementationOnce( async function *() {
|
|
194
|
+
yield { fn: () => {}, metadata: { name: 'SharedStepPrimary' }, path: '/root/shared/steps/primary/a.js' };
|
|
195
|
+
yield { fn: () => {}, metadata: { name: 'SharedStepSecondary' }, path: '/root/shared/steps/secondary/b.js' };
|
|
196
|
+
yield { fn: () => {}, metadata: { name: 'SharedEvalPrimary' }, path: '/root/shared/evaluators/primary/c.js' };
|
|
197
|
+
yield { fn: () => {}, metadata: { name: 'SharedEvalSecondary' }, path: '/root/shared/evaluators/secondary/d.js' };
|
|
198
|
+
} );
|
|
199
|
+
|
|
200
|
+
const workflows = [ { name: 'A', path: '/a/workflow.js' } ];
|
|
201
|
+
const activities = await loadActivities( '/root', workflows );
|
|
202
|
+
expect( activities['$shared#SharedStepPrimary'] ).toBeTypeOf( 'function' );
|
|
203
|
+
expect( activities['$shared#SharedStepSecondary'] ).toBeTypeOf( 'function' );
|
|
204
|
+
expect( activities['$shared#SharedEvalPrimary'] ).toBeTypeOf( 'function' );
|
|
205
|
+
expect( activities['$shared#SharedEvalSecondary'] ).toBeTypeOf( 'function' );
|
|
206
|
+
} );
|
|
207
|
+
|
|
208
|
+
describe( 'loadHooks', () => {
|
|
209
|
+
it( 'resolves without importing when package.json does not exist', async () => {
|
|
210
|
+
fsMocks.existsSync.mockReturnValue( false );
|
|
211
|
+
const { loadHooks } = await import( './loader.js' );
|
|
212
|
+
await expect( loadHooks( '/root' ) ).resolves.toBeUndefined();
|
|
213
|
+
expect( fsMocks.existsSync ).toHaveBeenCalledWith( join( '/root', 'package.json' ) );
|
|
214
|
+
} );
|
|
215
|
+
|
|
216
|
+
it( 'imports hook files listed in package.json output.hookFiles', async () => {
|
|
217
|
+
vi.doUnmock( 'node:fs' );
|
|
218
|
+
vi.resetModules();
|
|
219
|
+
const fs = await import( 'node:fs' );
|
|
220
|
+
const tmpDir = fs.mkdtempSync( join( tmpdir(), 'loader-spec-' ) );
|
|
221
|
+
try {
|
|
222
|
+
fs.writeFileSync( join( tmpDir, 'package.json' ), JSON.stringify( {
|
|
223
|
+
output: { hookFiles: [ 'hook.js' ] }
|
|
224
|
+
} ) );
|
|
225
|
+
fs.writeFileSync( join( tmpDir, 'hook.js' ), 'globalThis.__loadHooksTestLoaded = true;' );
|
|
226
|
+
|
|
227
|
+
const { loadHooks } = await import( './loader.js' );
|
|
228
|
+
await loadHooks( tmpDir );
|
|
229
|
+
expect( globalThis.__loadHooksTestLoaded ).toBe( true );
|
|
230
|
+
} finally {
|
|
231
|
+
delete globalThis.__loadHooksTestLoaded;
|
|
232
|
+
fs.rmSync( tmpDir, { recursive: true, force: true } );
|
|
233
|
+
}
|
|
234
|
+
} );
|
|
235
|
+
} );
|
|
236
|
+
} );
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { resolve, sep } from 'path';
|
|
2
|
+
import { pathToFileURL } from 'url';
|
|
3
|
+
import { METADATA_ACCESS_SYMBOL } from '#consts';
|
|
4
|
+
import { readdirSync } from 'fs';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* @typedef {object} CollectedFile
|
|
8
|
+
* @property {string} path - The file path
|
|
9
|
+
* @property {string} url - The resolved url of the file, ready to be imported
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* @typedef {object} Component
|
|
13
|
+
* @property {Function} fn - The loaded component function
|
|
14
|
+
* @property {object} metadata - Associated metadata with the component
|
|
15
|
+
* @property {string} path - Associated metadata with the component
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Recursive traverse directories collection files with paths that match one of the given matches.
|
|
20
|
+
*
|
|
21
|
+
* @param {string} path - The path to scan
|
|
22
|
+
* @param {function[]} matchers - Boolean functions to match files to add to collection
|
|
23
|
+
* @returns {CollectedFile[]} An array containing the collected files
|
|
24
|
+
*/
|
|
25
|
+
const findByNameRecursively = ( parentPath, matchers, ignoreDirNames = [ 'vendor', 'node_modules' ] ) => {
|
|
26
|
+
const collection = [];
|
|
27
|
+
for ( const entry of readdirSync( parentPath, { withFileTypes: true } ) ) {
|
|
28
|
+
if ( ignoreDirNames.includes( entry.name ) ) {
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const path = resolve( parentPath, entry.name );
|
|
33
|
+
if ( entry.isDirectory() ) {
|
|
34
|
+
collection.push( ...findByNameRecursively( path, matchers ) );
|
|
35
|
+
} else if ( matchers.some( m => m( path ) ) ) {
|
|
36
|
+
collection.push( { path, url: pathToFileURL( path ).href } );
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return collection;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Scan a path for files testing each path against a matching function.
|
|
45
|
+
*
|
|
46
|
+
* For each file found, dynamic import it and for each exports on that file, yields it.
|
|
47
|
+
*
|
|
48
|
+
* @remarks
|
|
49
|
+
* - Only yields exports that have the METADATA_ACCESS_SYMBOL, as they are output components (steps, evaluators, etc).
|
|
50
|
+
*
|
|
51
|
+
* @generator
|
|
52
|
+
* @async
|
|
53
|
+
* @function importComponents
|
|
54
|
+
* @param {string} target - Place to look for files
|
|
55
|
+
* @param {function[]} matchers - Boolean functions to match files
|
|
56
|
+
* @yields {Component}
|
|
57
|
+
*/
|
|
58
|
+
export async function *importComponents( target, matchers ) {
|
|
59
|
+
for ( const { url, path } of findByNameRecursively( target, matchers ) ) {
|
|
60
|
+
const imported = await import( url );
|
|
61
|
+
for ( const fn of Object.values( imported ) ) {
|
|
62
|
+
const metadata = fn[METADATA_ACCESS_SYMBOL];
|
|
63
|
+
if ( !metadata ) {
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
yield { fn, metadata, path };
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Returns matchers that need to be built using a relative path
|
|
73
|
+
*
|
|
74
|
+
* @param {string} path
|
|
75
|
+
* @returns {object} The object containing the matchers
|
|
76
|
+
*/
|
|
77
|
+
export const activityMatchersBuilder = path => ( {
|
|
78
|
+
/**
|
|
79
|
+
* Matches a file called steps.js, located at the path
|
|
80
|
+
* @param {string} path - Path to test
|
|
81
|
+
* @returns {boolean}
|
|
82
|
+
*/
|
|
83
|
+
stepsFile: v => v === `${path}${sep}steps.js`,
|
|
84
|
+
/**
|
|
85
|
+
* Matches a file called evaluators.js, located at the path
|
|
86
|
+
* @param {string} path - Path to test
|
|
87
|
+
* @returns {boolean}
|
|
88
|
+
*/
|
|
89
|
+
evaluatorsFile: v => v === `${path}${sep}evaluators.js`,
|
|
90
|
+
/**
|
|
91
|
+
* Matches all files on any levels inside a folder called steps/, located at the path
|
|
92
|
+
* @param {string} path - Path to test
|
|
93
|
+
* @returns {boolean}
|
|
94
|
+
*/
|
|
95
|
+
stepsDir: v => v.startsWith( `${path}${sep}steps${sep}` ),
|
|
96
|
+
/**
|
|
97
|
+
* Matches all files on any levels inside a folder called evaluators/, located at the path
|
|
98
|
+
* @param {string} path - Path to test
|
|
99
|
+
* @returns {boolean}
|
|
100
|
+
*/
|
|
101
|
+
evaluatorsDir: v => v.startsWith( `${path}${sep}evaluators${sep}` )
|
|
102
|
+
} );
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Matchers that can be used to access conditions without initializing them
|
|
106
|
+
*/
|
|
107
|
+
export const staticMatchers = {
|
|
108
|
+
/**
|
|
109
|
+
* Matches a workflow.js file
|
|
110
|
+
* @param {string} path - Path to test
|
|
111
|
+
* @returns {boolean}
|
|
112
|
+
*/
|
|
113
|
+
workflowFile: v => v.endsWith( `${sep}workflow.js` ),
|
|
114
|
+
/**
|
|
115
|
+
* Matches a workflow.js that is inside a shared folder: eg foo/shared/workflow.js
|
|
116
|
+
* @param {string} path - Path to test
|
|
117
|
+
* @returns {boolean}
|
|
118
|
+
*/
|
|
119
|
+
workflowPathHasShared: v => v.endsWith( `${sep}shared${sep}workflow.js` ),
|
|
120
|
+
/**
|
|
121
|
+
* Matches the shared folder for steps src/shared/steps/../step_file.js
|
|
122
|
+
* @param {string} path - Path to test
|
|
123
|
+
* @returns {boolean}
|
|
124
|
+
*/
|
|
125
|
+
sharedStepsDir: v => v.includes( `${sep}shared${sep}steps${sep}` ) && v.endsWith( '.js' ),
|
|
126
|
+
/**
|
|
127
|
+
* Matches the shared folder for evaluators src/shared/evaluators/../evaluator_file.js
|
|
128
|
+
* @param {string} path - Path to test
|
|
129
|
+
* @returns {boolean}
|
|
130
|
+
*/
|
|
131
|
+
sharedEvaluatorsDir: v => v.includes( `${sep}shared${sep}evaluators${sep}` ) && v.endsWith( '.js' )
|
|
132
|
+
};
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { describe, it, expect, afterEach } from 'vitest';
|
|
2
|
+
import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
|
3
|
+
import { join, sep } from 'node:path';
|
|
4
|
+
import { importComponents, staticMatchers } from './loader_tools.js';
|
|
5
|
+
|
|
6
|
+
describe( '.importComponents', () => {
|
|
7
|
+
const TEMP_BASE = join( process.cwd(), 'sdk/core/temp_test_modules' );
|
|
8
|
+
afterEach( () => {
|
|
9
|
+
rmSync( TEMP_BASE, { recursive: true, force: true } );
|
|
10
|
+
} );
|
|
11
|
+
it( 'imports modules and yields metadata from exports tagged with METADATA_ACCESS_SYMBOL', async () => {
|
|
12
|
+
const root = join( process.cwd(), 'sdk/core/temp_test_modules', `meta-${Date.now()}` );
|
|
13
|
+
mkdirSync( root, { recursive: true } );
|
|
14
|
+
const file = join( root, 'meta.module.js' );
|
|
15
|
+
writeFileSync( file, [
|
|
16
|
+
'import { METADATA_ACCESS_SYMBOL } from "#consts";',
|
|
17
|
+
'export const StepA = () => {};',
|
|
18
|
+
'StepA[METADATA_ACCESS_SYMBOL] = { kind: "step", name: "a" };',
|
|
19
|
+
'export const FlowB = () => {};',
|
|
20
|
+
'FlowB[METADATA_ACCESS_SYMBOL] = { kind: "workflow", name: "b" };'
|
|
21
|
+
].join( '\n' ) );
|
|
22
|
+
|
|
23
|
+
const collected = [];
|
|
24
|
+
for await ( const m of importComponents( root, [ v => v.endsWith( 'meta.module.js' ) ] ) ) {
|
|
25
|
+
collected.push( m );
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
expect( collected.length ).toBe( 2 );
|
|
29
|
+
expect( collected.map( m => m.metadata.name ).sort() ).toEqual( [ 'a', 'b' ] );
|
|
30
|
+
expect( collected.map( m => m.metadata.kind ).sort() ).toEqual( [ 'step', 'workflow' ] );
|
|
31
|
+
for ( const m of collected ) {
|
|
32
|
+
expect( m.path ).toBe( file );
|
|
33
|
+
expect( typeof m.fn ).toBe( 'function' );
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
rmSync( root, { recursive: true, force: true } );
|
|
37
|
+
} );
|
|
38
|
+
|
|
39
|
+
it( 'ignores exports without metadata symbol', async () => {
|
|
40
|
+
const root = join( process.cwd(), 'sdk/core/temp_test_modules', `meta-${Date.now()}-nometa` );
|
|
41
|
+
mkdirSync( root, { recursive: true } );
|
|
42
|
+
const file = join( root, 'meta.module.js' );
|
|
43
|
+
writeFileSync( file, [
|
|
44
|
+
'export const Plain = () => {};',
|
|
45
|
+
'export const AlsoPlain = {}'
|
|
46
|
+
].join( '\n' ) );
|
|
47
|
+
|
|
48
|
+
const collected = [];
|
|
49
|
+
for await ( const m of importComponents( root, [ v => v.endsWith( 'meta.module.js' ) ] ) ) {
|
|
50
|
+
collected.push( m );
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
expect( collected.length ).toBe( 0 );
|
|
54
|
+
rmSync( root, { recursive: true, force: true } );
|
|
55
|
+
} );
|
|
56
|
+
|
|
57
|
+
it( 'skips files inside ignored directories (node_modules, vendor)', async () => {
|
|
58
|
+
const root = join( process.cwd(), 'sdk/core/temp_test_modules', `meta-${Date.now()}-ignoredirs` );
|
|
59
|
+
const okDir = join( root, 'ok' );
|
|
60
|
+
const nmDir = join( root, 'node_modules' );
|
|
61
|
+
const vendorDir = join( root, 'vendor' );
|
|
62
|
+
mkdirSync( okDir, { recursive: true } );
|
|
63
|
+
mkdirSync( nmDir, { recursive: true } );
|
|
64
|
+
mkdirSync( vendorDir, { recursive: true } );
|
|
65
|
+
|
|
66
|
+
const okFile = join( okDir, 'meta.module.js' );
|
|
67
|
+
const nmFile = join( nmDir, 'meta.module.js' );
|
|
68
|
+
const vendorFile = join( vendorDir, 'meta.module.js' );
|
|
69
|
+
|
|
70
|
+
const fileContents = [
|
|
71
|
+
'import { METADATA_ACCESS_SYMBOL } from "#consts";',
|
|
72
|
+
'export const C = () => {};',
|
|
73
|
+
'C[METADATA_ACCESS_SYMBOL] = { kind: "step", name: "c" };'
|
|
74
|
+
].join( '\n' );
|
|
75
|
+
writeFileSync( okFile, fileContents );
|
|
76
|
+
writeFileSync( nmFile, fileContents );
|
|
77
|
+
writeFileSync( vendorFile, fileContents );
|
|
78
|
+
|
|
79
|
+
const collected = [];
|
|
80
|
+
for await ( const m of importComponents( root, [ v => v.endsWith( 'meta.module.js' ) ] ) ) {
|
|
81
|
+
collected.push( m );
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
expect( collected.length ).toBe( 1 );
|
|
85
|
+
expect( collected[0].path ).toBe( okFile );
|
|
86
|
+
|
|
87
|
+
rmSync( root, { recursive: true, force: true } );
|
|
88
|
+
} );
|
|
89
|
+
|
|
90
|
+
it( 'supports partial matching by folder name', async () => {
|
|
91
|
+
const root = join( process.cwd(), 'sdk/core/temp_test_modules', `meta-${Date.now()}-foldermatch` );
|
|
92
|
+
const okDir = join( root, 'features', 'ok' );
|
|
93
|
+
const otherDir = join( root, 'features', 'other' );
|
|
94
|
+
mkdirSync( okDir, { recursive: true } );
|
|
95
|
+
mkdirSync( otherDir, { recursive: true } );
|
|
96
|
+
|
|
97
|
+
const okFile = join( okDir, 'alpha.js' );
|
|
98
|
+
const otherFile = join( otherDir, 'beta.js' );
|
|
99
|
+
const src = [
|
|
100
|
+
'import { METADATA_ACCESS_SYMBOL } from "#consts";',
|
|
101
|
+
'export const X = () => {};',
|
|
102
|
+
'X[METADATA_ACCESS_SYMBOL] = { kind: "step", name: "x" };'
|
|
103
|
+
].join( '\n' );
|
|
104
|
+
writeFileSync( okFile, src );
|
|
105
|
+
writeFileSync( otherFile, src );
|
|
106
|
+
|
|
107
|
+
// Match any JS under a folder named "ok"
|
|
108
|
+
const matcher = v => v.includes( `${join( 'features', 'ok' )}${sep}` );
|
|
109
|
+
const collected = [];
|
|
110
|
+
for await ( const m of importComponents( root, [ matcher ] ) ) {
|
|
111
|
+
collected.push( m );
|
|
112
|
+
}
|
|
113
|
+
expect( collected.length ).toBe( 1 );
|
|
114
|
+
expect( collected[0].path ).toBe( okFile );
|
|
115
|
+
|
|
116
|
+
rmSync( root, { recursive: true, force: true } );
|
|
117
|
+
} );
|
|
118
|
+
} );
|
|
119
|
+
|
|
120
|
+
describe( '.staticMatchers', () => {
|
|
121
|
+
describe( '.sharedStepsDir', () => {
|
|
122
|
+
it( 'matches .js files inside shared/steps/', () => {
|
|
123
|
+
expect( staticMatchers.sharedStepsDir( `${sep}app${sep}dist${sep}shared${sep}steps${sep}tools.js` ) ).toBe( true );
|
|
124
|
+
} );
|
|
125
|
+
|
|
126
|
+
it( 'matches .js files in nested subdirectories of shared/steps/', () => {
|
|
127
|
+
expect( staticMatchers.sharedStepsDir( `${sep}app${sep}dist${sep}shared${sep}steps${sep}utils${sep}helper.js` ) ).toBe( true );
|
|
128
|
+
} );
|
|
129
|
+
|
|
130
|
+
it( 'rejects .ts files inside shared/steps/', () => {
|
|
131
|
+
expect( staticMatchers.sharedStepsDir( `${sep}app${sep}src${sep}shared${sep}steps${sep}tools.ts` ) ).toBe( false );
|
|
132
|
+
} );
|
|
133
|
+
|
|
134
|
+
it( 'rejects non-.js files inside shared/steps/', () => {
|
|
135
|
+
expect( staticMatchers.sharedStepsDir( `${sep}app${sep}dist${sep}shared${sep}steps${sep}readme.md` ) ).toBe( false );
|
|
136
|
+
} );
|
|
137
|
+
} );
|
|
138
|
+
|
|
139
|
+
describe( '.sharedEvaluatorsDir', () => {
|
|
140
|
+
it( 'matches .js files inside shared/evaluators/', () => {
|
|
141
|
+
expect( staticMatchers.sharedEvaluatorsDir( `${sep}app${sep}dist${sep}shared${sep}evaluators${sep}quality.js` ) ).toBe( true );
|
|
142
|
+
} );
|
|
143
|
+
|
|
144
|
+
it( 'matches .js files in nested subdirectories of shared/evaluators/', () => {
|
|
145
|
+
expect( staticMatchers.sharedEvaluatorsDir( `${sep}app${sep}dist${sep}shared${sep}evaluators${sep}utils${sep}helper.js` ) ).toBe( true );
|
|
146
|
+
} );
|
|
147
|
+
|
|
148
|
+
it( 'rejects .ts files inside shared/evaluators/', () => {
|
|
149
|
+
expect( staticMatchers.sharedEvaluatorsDir( `${sep}app${sep}src${sep}shared${sep}evaluators${sep}quality.ts` ) ).toBe( false );
|
|
150
|
+
} );
|
|
151
|
+
|
|
152
|
+
it( 'rejects non-.js files inside shared/evaluators/', () => {
|
|
153
|
+
expect( staticMatchers.sharedEvaluatorsDir( `${sep}app${sep}dist${sep}shared${sep}evaluators${sep}readme.md` ) ).toBe( false );
|
|
154
|
+
} );
|
|
155
|
+
} );
|
|
156
|
+
} );
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { messageBus } from '#bus';
|
|
2
|
+
import { createChildLogger } from '#logger';
|
|
3
|
+
import { BusEventType, ComponentType, LifecycleEvent, WORKFLOW_CATALOG } from '#consts';
|
|
4
|
+
|
|
5
|
+
const activityLog = createChildLogger( 'Activity' );
|
|
6
|
+
const workflowLog = createChildLogger( 'Workflow' );
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Intercepts internal bus events for activity and workflow lifecycle and log them
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/*
|
|
13
|
+
╔═════════════════╗
|
|
14
|
+
║ Activity events ║
|
|
15
|
+
╚═════════════════╝
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Returns true if activity event should be logged
|
|
20
|
+
*/
|
|
21
|
+
const shouldLogActivityEvent = ( { kind } ) => kind !== ComponentType.INTERNAL_STEP;
|
|
22
|
+
|
|
23
|
+
messageBus.on( BusEventType.ACTIVITY_START, ( { id, name, kind, workflowId, workflowName } ) =>
|
|
24
|
+
shouldLogActivityEvent( { kind } ) && activityLog.info( `Started ${name} ${kind}`, {
|
|
25
|
+
event: LifecycleEvent.START,
|
|
26
|
+
activityId: id,
|
|
27
|
+
activityName: name,
|
|
28
|
+
activityKind: kind,
|
|
29
|
+
workflowId,
|
|
30
|
+
workflowName
|
|
31
|
+
} )
|
|
32
|
+
);
|
|
33
|
+
|
|
34
|
+
messageBus.on( BusEventType.ACTIVITY_END, ( { id, name, kind, workflowId, workflowName, duration } ) =>
|
|
35
|
+
shouldLogActivityEvent( { kind } ) && activityLog.info( `Ended ${name} ${kind}`, {
|
|
36
|
+
event: LifecycleEvent.END,
|
|
37
|
+
activityId: id,
|
|
38
|
+
activityName: name,
|
|
39
|
+
activityKind: kind,
|
|
40
|
+
workflowId,
|
|
41
|
+
workflowName,
|
|
42
|
+
durationMs: duration
|
|
43
|
+
} )
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
messageBus.on( BusEventType.ACTIVITY_ERROR, ( { id, name, kind, workflowId, workflowName, duration, error } ) =>
|
|
47
|
+
shouldLogActivityEvent( { kind } ) && activityLog.error( `Error ${name} ${kind}: ${error.constructor.name}`, {
|
|
48
|
+
event: LifecycleEvent.ERROR,
|
|
49
|
+
activityId: id,
|
|
50
|
+
activityName: name,
|
|
51
|
+
activityKind: kind,
|
|
52
|
+
workflowId,
|
|
53
|
+
workflowName,
|
|
54
|
+
durationMs: duration,
|
|
55
|
+
error: error.message
|
|
56
|
+
} )
|
|
57
|
+
);
|
|
58
|
+
|
|
59
|
+
/*
|
|
60
|
+
╔═════════════════╗
|
|
61
|
+
║ Workflow events ║
|
|
62
|
+
╚═════════════════╝
|
|
63
|
+
*/
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Returns true if activity event should be logged
|
|
67
|
+
*/
|
|
68
|
+
const shouldLogWorkflowEvent = ( { name } ) => name !== WORKFLOW_CATALOG;
|
|
69
|
+
|
|
70
|
+
messageBus.on( BusEventType.WORKFLOW_START, ( { id, name } ) =>
|
|
71
|
+
shouldLogWorkflowEvent( { name } ) && workflowLog.info( `Started ${name} workflow`, {
|
|
72
|
+
event: LifecycleEvent.START,
|
|
73
|
+
workflowId: id,
|
|
74
|
+
workflowName: name
|
|
75
|
+
} )
|
|
76
|
+
);
|
|
77
|
+
|
|
78
|
+
messageBus.on( BusEventType.WORKFLOW_END, ( { id, name, duration } ) =>
|
|
79
|
+
shouldLogWorkflowEvent( { name } ) && workflowLog.info( `Ended ${name} workflow`, {
|
|
80
|
+
event: LifecycleEvent.END,
|
|
81
|
+
workflowId: id,
|
|
82
|
+
workflowName: name,
|
|
83
|
+
durationMs: duration
|
|
84
|
+
} )
|
|
85
|
+
);
|
|
86
|
+
|
|
87
|
+
messageBus.on( BusEventType.WORKFLOW_ERROR, ( { id, name, duration, error } ) =>
|
|
88
|
+
shouldLogWorkflowEvent( { name } ) && workflowLog.error( `Error ${name} workflow: ${error.constructor.name}`, {
|
|
89
|
+
event: LifecycleEvent.ERROR,
|
|
90
|
+
workflowId: id,
|
|
91
|
+
workflowName: name,
|
|
92
|
+
durationMs: duration,
|
|
93
|
+
error: error.message
|
|
94
|
+
} )
|
|
95
|
+
);
|