@outputai/core 0.9.3-next.c318502.0 → 0.10.1-dev.b7b2fbe.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/consts.js +2 -1
- package/src/helpers/object.js +45 -36
- package/src/helpers/object.spec.js +118 -75
- package/src/interface/workflow.js +92 -85
- package/src/interface/workflow.spec.js +135 -142
- package/src/sdk/helpers/objects.d.ts +24 -19
- package/src/worker/configs.js +4 -0
- package/src/worker/configs.spec.js +21 -0
- package/src/worker/configs_tuner_schema.js +81 -0
- package/src/worker/configs_tuner_schema.spec.js +127 -0
- package/src/worker/index.js +12 -3
- package/src/worker/index.spec.js +28 -0
- package/src/worker/interceptors/workflow.js +5 -5
- package/src/worker/interceptors/workflow.spec.js +7 -7
- package/src/worker/loader/activities.js +56 -40
- package/src/worker/loader/activities.spec.js +21 -22
- package/src/worker/loader/workflows.spec.js +1 -2
- package/src/worker/webpack_loaders/consts.js +6 -0
- package/src/worker/webpack_loaders/tools.js +1 -146
- package/src/worker/webpack_loaders/tools.spec.js +0 -23
- package/src/worker/webpack_loaders/workflow_rewriter/collect_target_imports.js +24 -32
- package/src/worker/webpack_loaders/workflow_rewriter/collect_target_imports.spec.js +82 -279
- package/src/worker/webpack_loaders/workflow_rewriter/index.mjs +6 -7
- package/src/worker/webpack_loaders/workflow_rewriter/index.spec.js +57 -119
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_activity_calls.js +88 -0
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_activity_calls.spec.js +165 -0
- package/src/worker/webpack_loaders/workflow_validator/index.mjs +96 -29
- package/src/worker/webpack_loaders/workflow_validator/index.spec.js +110 -583
- package/src/worker/webpack_loaders/{npm_workflow_export_resolve.js → workflow_validator/npm_workflow_export_resolve.js} +1 -1
- package/src/worker/webpack_loaders/{npm_workflow_export_resolve.spec.js → workflow_validator/npm_workflow_export_resolve.spec.js} +1 -1
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_fn_bodies.js +0 -193
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_fn_bodies.spec.js +0 -123
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { workerTunerEnvSchema } from './configs_tuner_schema.js';
|
|
3
|
+
|
|
4
|
+
const parseWorkerTuner = value => workerTunerEnvSchema.parse( value );
|
|
5
|
+
|
|
6
|
+
describe( 'worker/configs_tuner_schema', () => {
|
|
7
|
+
it( 'treats empty string as unset', () => {
|
|
8
|
+
expect( parseWorkerTuner( '' ) ).toBeUndefined();
|
|
9
|
+
} );
|
|
10
|
+
|
|
11
|
+
it( 'parses resource-based Temporal worker tuner JSON', () => {
|
|
12
|
+
const workerTuner = {
|
|
13
|
+
tunerOptions: {
|
|
14
|
+
targetMemoryUsage: 0.8,
|
|
15
|
+
targetCpuUsage: 0.9
|
|
16
|
+
},
|
|
17
|
+
activityTaskSlotOptions: {
|
|
18
|
+
minimumSlots: 1,
|
|
19
|
+
maximumSlots: 100,
|
|
20
|
+
rampThrottle: '50ms'
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
expect( parseWorkerTuner( JSON.stringify( workerTuner ) ) ).toEqual( workerTuner );
|
|
25
|
+
} );
|
|
26
|
+
|
|
27
|
+
it( 'parses per-task Temporal worker tuner JSON', () => {
|
|
28
|
+
const resourceBasedSupplier = {
|
|
29
|
+
type: 'resource-based',
|
|
30
|
+
tunerOptions: {
|
|
31
|
+
targetMemoryUsage: 0.8,
|
|
32
|
+
targetCpuUsage: 0.9
|
|
33
|
+
},
|
|
34
|
+
minimumSlots: 1,
|
|
35
|
+
maximumSlots: 100,
|
|
36
|
+
rampThrottle: '50ms'
|
|
37
|
+
};
|
|
38
|
+
const workerTuner = {
|
|
39
|
+
workflowTaskSlotSupplier: {
|
|
40
|
+
type: 'fixed-size',
|
|
41
|
+
numSlots: 10
|
|
42
|
+
},
|
|
43
|
+
activityTaskSlotSupplier: resourceBasedSupplier,
|
|
44
|
+
localActivityTaskSlotSupplier: {
|
|
45
|
+
type: 'fixed-size',
|
|
46
|
+
numSlots: 10
|
|
47
|
+
},
|
|
48
|
+
nexusTaskSlotSupplier: {
|
|
49
|
+
type: 'fixed-size',
|
|
50
|
+
numSlots: 10
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
expect( parseWorkerTuner( JSON.stringify( workerTuner ) ) ).toEqual( workerTuner );
|
|
55
|
+
} );
|
|
56
|
+
|
|
57
|
+
it( 'throws when tuner is not valid JSON', () => {
|
|
58
|
+
expect( () => parseWorkerTuner( '{invalid' ) ).toThrow();
|
|
59
|
+
} );
|
|
60
|
+
|
|
61
|
+
it( 'throws when tuner is not a JSON object', () => {
|
|
62
|
+
expect( () => parseWorkerTuner( '[]' ) ).toThrow();
|
|
63
|
+
} );
|
|
64
|
+
|
|
65
|
+
it( 'throws when tuner is missing required per-task suppliers', () => {
|
|
66
|
+
const workerTuner = {
|
|
67
|
+
workflowTaskSlotSupplier: {
|
|
68
|
+
type: 'fixed-size',
|
|
69
|
+
numSlots: 10
|
|
70
|
+
},
|
|
71
|
+
activityTaskSlotSupplier: {
|
|
72
|
+
type: 'fixed-size',
|
|
73
|
+
numSlots: 10
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
expect( () => parseWorkerTuner( JSON.stringify( workerTuner ) ) ).toThrow();
|
|
78
|
+
} );
|
|
79
|
+
|
|
80
|
+
it( 'throws when tuner uses custom suppliers', () => {
|
|
81
|
+
const workerTuner = {
|
|
82
|
+
workflowTaskSlotSupplier: {
|
|
83
|
+
type: 'custom'
|
|
84
|
+
},
|
|
85
|
+
activityTaskSlotSupplier: {
|
|
86
|
+
type: 'fixed-size',
|
|
87
|
+
numSlots: 10
|
|
88
|
+
},
|
|
89
|
+
localActivityTaskSlotSupplier: {
|
|
90
|
+
type: 'fixed-size',
|
|
91
|
+
numSlots: 10
|
|
92
|
+
},
|
|
93
|
+
nexusTaskSlotSupplier: {
|
|
94
|
+
type: 'fixed-size',
|
|
95
|
+
numSlots: 10
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
expect( () => parseWorkerTuner( JSON.stringify( workerTuner ) ) ).toThrow();
|
|
100
|
+
} );
|
|
101
|
+
|
|
102
|
+
it( 'throws when target usage is outside range', () => {
|
|
103
|
+
const workerTuner = {
|
|
104
|
+
tunerOptions: {
|
|
105
|
+
targetMemoryUsage: 1.5,
|
|
106
|
+
targetCpuUsage: 0.9
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
expect( () => parseWorkerTuner( JSON.stringify( workerTuner ) ) ).toThrow();
|
|
111
|
+
} );
|
|
112
|
+
|
|
113
|
+
it( 'throws when minimum slots exceeds maximum slots', () => {
|
|
114
|
+
const workerTuner = {
|
|
115
|
+
tunerOptions: {
|
|
116
|
+
targetMemoryUsage: 0.8,
|
|
117
|
+
targetCpuUsage: 0.9
|
|
118
|
+
},
|
|
119
|
+
activityTaskSlotOptions: {
|
|
120
|
+
minimumSlots: 10,
|
|
121
|
+
maximumSlots: 5
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
expect( () => parseWorkerTuner( JSON.stringify( workerTuner ) ) ).toThrow();
|
|
126
|
+
} );
|
|
127
|
+
} );
|
package/src/worker/index.js
CHANGED
|
@@ -36,7 +36,8 @@ const {
|
|
|
36
36
|
maxConcurrentActivityTaskPolls,
|
|
37
37
|
maxConcurrentWorkflowTaskPolls,
|
|
38
38
|
shutdownForceTime,
|
|
39
|
-
shutdownGraceTime
|
|
39
|
+
shutdownGraceTime,
|
|
40
|
+
workerTuner
|
|
40
41
|
} = configs;
|
|
41
42
|
|
|
42
43
|
const state = {
|
|
@@ -88,6 +89,9 @@ const execute = async () => {
|
|
|
88
89
|
state.catalogJob = new CatalogJob( { connection: state.connection, namespace, catalog, catalogHash } );
|
|
89
90
|
|
|
90
91
|
log.info( 'Creating worker...' );
|
|
92
|
+
if ( workerTuner ) {
|
|
93
|
+
log.info( 'Using worker tuner options', { ...workerTuner } );
|
|
94
|
+
}
|
|
91
95
|
const worker = await Worker.create( {
|
|
92
96
|
connection: state.connection,
|
|
93
97
|
namespace,
|
|
@@ -96,8 +100,13 @@ const execute = async () => {
|
|
|
96
100
|
activities,
|
|
97
101
|
sinks,
|
|
98
102
|
interceptors: initInterceptors( { activities, workflows } ),
|
|
99
|
-
|
|
100
|
-
|
|
103
|
+
// tuner isn't compatible with concurrent task executions configs
|
|
104
|
+
...( workerTuner ? {
|
|
105
|
+
tuner: workerTuner
|
|
106
|
+
} : {
|
|
107
|
+
maxConcurrentWorkflowTaskExecutions,
|
|
108
|
+
maxConcurrentActivityTaskExecutions
|
|
109
|
+
} ),
|
|
101
110
|
maxCachedWorkflows,
|
|
102
111
|
maxConcurrentActivityTaskPolls,
|
|
103
112
|
maxConcurrentWorkflowTaskPolls,
|
package/src/worker/index.spec.js
CHANGED
|
@@ -45,6 +45,7 @@ const {
|
|
|
45
45
|
maxCachedWorkflows: 1000,
|
|
46
46
|
maxConcurrentActivityTaskPolls: 5,
|
|
47
47
|
maxConcurrentWorkflowTaskPolls: 5,
|
|
48
|
+
workerTuner: undefined,
|
|
48
49
|
processFailureShutdownDelay: 0,
|
|
49
50
|
shutdownForceTime: undefined,
|
|
50
51
|
shutdownGraceTime: undefined
|
|
@@ -185,6 +186,7 @@ describe( 'worker/index', () => {
|
|
|
185
186
|
resetPromises();
|
|
186
187
|
configValues.apiKey = undefined;
|
|
187
188
|
configValues.grpcProxy = undefined;
|
|
189
|
+
configValues.workerTuner = undefined;
|
|
188
190
|
configValues.shutdownForceTime = undefined;
|
|
189
191
|
configValues.shutdownGraceTime = undefined;
|
|
190
192
|
catalogJobInstance.error = null;
|
|
@@ -263,6 +265,32 @@ describe( 'worker/index', () => {
|
|
|
263
265
|
expect( mockLog.info ).toHaveBeenCalledWith( 'Bye' );
|
|
264
266
|
} );
|
|
265
267
|
|
|
268
|
+
it( 'passes worker tuner instead of incompatible execution concurrency options', async () => {
|
|
269
|
+
configValues.workerTuner = {
|
|
270
|
+
tunerOptions: {
|
|
271
|
+
targetMemoryUsage: 0.8,
|
|
272
|
+
targetCpuUsage: 0.9
|
|
273
|
+
}
|
|
274
|
+
};
|
|
275
|
+
const { Worker } = await import( '@temporalio/worker' );
|
|
276
|
+
|
|
277
|
+
await importWorker();
|
|
278
|
+
|
|
279
|
+
await vi.waitFor( () => expect( Worker.create ).toHaveBeenCalled() );
|
|
280
|
+
|
|
281
|
+
const workerOptions = Worker.create.mock.calls[0][0];
|
|
282
|
+
expect( workerOptions ).toEqual( expect.objectContaining( {
|
|
283
|
+
tuner: configValues.workerTuner,
|
|
284
|
+
maxCachedWorkflows: configValues.maxCachedWorkflows,
|
|
285
|
+
maxConcurrentActivityTaskPolls: configValues.maxConcurrentActivityTaskPolls,
|
|
286
|
+
maxConcurrentWorkflowTaskPolls: configValues.maxConcurrentWorkflowTaskPolls
|
|
287
|
+
} ) );
|
|
288
|
+
expect( workerOptions ).not.toHaveProperty( 'maxConcurrentWorkflowTaskExecutions' );
|
|
289
|
+
expect( workerOptions ).not.toHaveProperty( 'maxConcurrentActivityTaskExecutions' );
|
|
290
|
+
|
|
291
|
+
await settleWorker();
|
|
292
|
+
} );
|
|
293
|
+
|
|
266
294
|
it( 'enables TLS when apiKey is set', async () => {
|
|
267
295
|
configValues.apiKey = 'secret';
|
|
268
296
|
const { NativeConnection } = await import( '@temporalio/worker' );
|
|
@@ -7,7 +7,7 @@ import { METADATA_ACCESS_SYMBOL, WorkflowSpecialOutput } from '#consts';
|
|
|
7
7
|
import { createWorkflowDetails } from '#helpers/temporal_context';
|
|
8
8
|
|
|
9
9
|
// this is a dynamic generated file with activity configs overwrites
|
|
10
|
-
import
|
|
10
|
+
import activityOptionsMap from '../temp/__activity_options.js';
|
|
11
11
|
|
|
12
12
|
/*
|
|
13
13
|
This interceptor adds Memo and serialized workflowInfo() to the Activity invocation headers.
|
|
@@ -22,10 +22,10 @@ class HeadersInjectionInterceptor {
|
|
|
22
22
|
...memo,
|
|
23
23
|
workflowDetails: createWorkflowDetails( info )
|
|
24
24
|
} ) );
|
|
25
|
-
//
|
|
26
|
-
const
|
|
27
|
-
if (
|
|
28
|
-
input.options = deepMerge(
|
|
25
|
+
// Apply component-level activity options on top of workflow options.
|
|
26
|
+
const activityOptionsOverrides = activityOptionsMap[input.activityType];
|
|
27
|
+
if ( activityOptionsOverrides ) {
|
|
28
|
+
input.options = deepMerge( input.options ?? {}, activityOptionsOverrides );
|
|
29
29
|
}
|
|
30
30
|
return next( input );
|
|
31
31
|
}
|
|
@@ -56,12 +56,13 @@ vi.mock( './headers.js', () => ( { memoToHeaders: ( ...args ) => memoToHeadersMo
|
|
|
56
56
|
const deepMergeMock = vi.fn( ( a, b ) => ( { ...( a || {} ), ...( b || {} ) } ) );
|
|
57
57
|
vi.mock( '#helpers/object', () => ( { deepMerge: ( ...args ) => deepMergeMock( ...args ) } ) );
|
|
58
58
|
|
|
59
|
-
const
|
|
60
|
-
vi.mock( '../temp/__activity_options.js', () => ( { default:
|
|
59
|
+
const activityOptionsDefault = {};
|
|
60
|
+
vi.mock( '../temp/__activity_options.js', () => ( { default: activityOptionsDefault } ) );
|
|
61
61
|
|
|
62
62
|
describe( 'workflow interceptors', () => {
|
|
63
63
|
beforeEach( () => {
|
|
64
64
|
vi.clearAllMocks();
|
|
65
|
+
Object.keys( activityOptionsDefault ).forEach( key => delete activityOptionsDefault[key] );
|
|
65
66
|
isCancellationMock.mockReturnValue( false );
|
|
66
67
|
workflowInfoMock.mockReturnValue( workflowInfo );
|
|
67
68
|
} );
|
|
@@ -91,11 +92,11 @@ describe( 'workflow interceptors', () => {
|
|
|
91
92
|
expect( out ).toBe( 'result' );
|
|
92
93
|
} );
|
|
93
94
|
|
|
94
|
-
it( 'merges
|
|
95
|
-
|
|
95
|
+
it( 'merges component activity options over the scheduled activity input options', async () => {
|
|
96
|
+
activityOptionsDefault['MyWorkflow#step1'] = { scheduleToCloseTimeout: 60 };
|
|
96
97
|
workflowInfoMock.mockReturnValue( {
|
|
97
98
|
...workflowInfo,
|
|
98
|
-
memo: { traceInfo: workflowInfo.memo.traceInfo
|
|
99
|
+
memo: { traceInfo: workflowInfo.memo.traceInfo }
|
|
99
100
|
} );
|
|
100
101
|
memoToHeadersMock.mockReturnValue( {} );
|
|
101
102
|
deepMergeMock.mockReturnValue( { heartbeatTimeout: 10, scheduleToCloseTimeout: 60 } );
|
|
@@ -103,14 +104,13 @@ describe( 'workflow interceptors', () => {
|
|
|
103
104
|
const { interceptors } = await import( './workflow.js' );
|
|
104
105
|
const { outbound } = interceptors();
|
|
105
106
|
const interceptor = outbound[0];
|
|
106
|
-
const input = { headers: {}, activityType: 'MyWorkflow#step1' };
|
|
107
|
+
const input = { headers: {}, activityType: 'MyWorkflow#step1', options: { heartbeatTimeout: 10 } };
|
|
107
108
|
const next = vi.fn().mockResolvedValue( undefined );
|
|
108
109
|
|
|
109
110
|
await interceptor.scheduleActivity( input, next );
|
|
110
111
|
|
|
111
112
|
expect( deepMergeMock ).toHaveBeenCalledWith( { heartbeatTimeout: 10 }, { scheduleToCloseTimeout: 60 } );
|
|
112
113
|
expect( input.options ).toEqual( { heartbeatTimeout: 10, scheduleToCloseTimeout: 60 } );
|
|
113
|
-
delete stepOptionsDefault['MyWorkflow#step1'];
|
|
114
114
|
} );
|
|
115
115
|
} );
|
|
116
116
|
|
|
@@ -2,74 +2,90 @@ import { dirname } from 'node:path';
|
|
|
2
2
|
import { getTraceDestinations, sendHttpRequest } from '#internal_activities';
|
|
3
3
|
import { findSharedActivitiesFromWorkflows, importComponents, matchFiles, writeFileInTempDir } from './tools.js';
|
|
4
4
|
import { buildActivityMatcher, staticMatchers } from './matchers.js';
|
|
5
|
-
import { ACTIVITY_SEND_HTTP_REQUEST, ACTIVITY_OPTIONS_FILENAME,
|
|
5
|
+
import { ACTIVITY_SEND_HTTP_REQUEST, ACTIVITY_OPTIONS_FILENAME, ACTIVITY_GET_TRACE_DESTINATIONS } from '#consts';
|
|
6
6
|
import { createChildLogger } from '#logger';
|
|
7
7
|
import { ValidationError } from '#errors';
|
|
8
8
|
|
|
9
9
|
const log = createChildLogger( 'Activities Loader' );
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
|
-
* Load activities
|
|
13
|
-
* -
|
|
14
|
-
* -
|
|
15
|
-
* -
|
|
12
|
+
* Load activities
|
|
13
|
+
* - Scan local project and external workflow npm packages and look for shared activities.
|
|
14
|
+
* - For each workflow, register shared activities under the workflow namespace.
|
|
15
|
+
* - For each workflow, load activities declared relative to that workflow directory.
|
|
16
16
|
*
|
|
17
|
-
* Builds a map of activities,
|
|
18
|
-
* - Shared activity keys have a common prefix followed by the activity name;
|
|
19
|
-
* - Internal activities are registered with a fixed key;
|
|
20
|
-
* - Workflow activities keys are composed using the workflow name and the activity name;
|
|
17
|
+
* Builds a map of activities, key is workflowType#activityType, value is the component wrapper function.
|
|
21
18
|
*
|
|
22
19
|
* @param {string} rootDir
|
|
23
20
|
* @param {import('./workflows.js').Workflow[]} workflows
|
|
24
21
|
* @returns {object}
|
|
25
22
|
*/
|
|
26
23
|
export async function loadActivities( rootDir, workflows ) {
|
|
27
|
-
const activities =
|
|
28
|
-
const
|
|
24
|
+
const activities = new Map();
|
|
25
|
+
const activityOptions = new Map();
|
|
29
26
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
const dir = dirname( workflowPath );
|
|
33
|
-
for await ( const { fn, metadata, path } of importComponents( matchFiles( dir, [ buildActivityMatcher( dir ) ] ) ) ) {
|
|
34
|
-
// Activities loaded from a workflow path will use the workflow name as a namespace, which is unique across the platform, avoiding collision
|
|
35
|
-
const activityKey = `${workflowName}#${metadata.name}`;
|
|
36
|
-
|
|
37
|
-
log.info( metadata.name, { workflow: workflowName, type: metadata.type, ...( external && { external } ), path } );
|
|
38
|
-
|
|
39
|
-
if ( activities[activityKey] ) {
|
|
40
|
-
throw new ValidationError( `Activity "${metadata.name}" in workflow "${workflowName}" conflicts with another \
|
|
41
|
-
activity in the same workflow. Activity names must be unique within a workflow.` );
|
|
42
|
-
}
|
|
43
|
-
activities[activityKey] = fn;
|
|
44
|
-
// propagate the custom options set on the step()/evaluator() constructor
|
|
45
|
-
activityOptionsMap[activityKey] = metadata.options?.activityOptions ?? undefined;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
27
|
+
const sharedActivities = new Map();
|
|
28
|
+
const sharedActivitiesOptions = new Map();
|
|
48
29
|
|
|
49
30
|
// Load shared activities/evaluators from local and external npm modules
|
|
50
31
|
const localSharedActivities = matchFiles( rootDir, [ staticMatchers.sharedStepsDir, staticMatchers.sharedEvaluatorsDir ] );
|
|
51
32
|
const externalSharedActivities = findSharedActivitiesFromWorkflows( workflows.filter( w => w.external ) );
|
|
52
33
|
for await ( const { fn, metadata, path } of importComponents( [ ...localSharedActivities, ...externalSharedActivities ] ) ) {
|
|
53
34
|
const external = externalSharedActivities.some( a => a.path === path );
|
|
54
|
-
// Uses a global namespace for shared activities
|
|
55
|
-
const activityKey = `${SHARED_STEP_PREFIX}#${metadata.name}`;
|
|
56
35
|
|
|
36
|
+
if ( sharedActivities.has( metadata.name ) ) {
|
|
37
|
+
throw new ValidationError( `Shared activity "${metadata.name}" conflicts with another shared activity.` +
|
|
38
|
+
' Shared activity names must be unique.' );
|
|
39
|
+
}
|
|
57
40
|
log.info( metadata.name, { shared: true, type: metadata.type, ...( external && { external } ), path } );
|
|
58
41
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
42
|
+
sharedActivities.set( metadata.name, fn );
|
|
43
|
+
if ( metadata.options?.activityOptions ) {
|
|
44
|
+
sharedActivitiesOptions.set( metadata.name, metadata.options.activityOptions );
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Discover and load workflow activities
|
|
49
|
+
for ( const { path: workflowPath, name: workflowName, external } of workflows ) {
|
|
50
|
+
const dir = dirname( workflowPath );
|
|
51
|
+
|
|
52
|
+
// Add shared activities to this workflow namespace
|
|
53
|
+
for ( const [ name, fn ] of sharedActivities ) {
|
|
54
|
+
const id = `${workflowName}#${name}`;
|
|
55
|
+
activities.set( id, fn );
|
|
56
|
+
if ( sharedActivitiesOptions.has( name ) ) {
|
|
57
|
+
activityOptions.set( id, sharedActivitiesOptions.get( name ) );
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
for await ( const { fn, metadata, path } of importComponents( matchFiles( dir, [ buildActivityMatcher( dir ) ] ) ) ) {
|
|
62
|
+
// Activities loaded from a workflow path will use the workflow name as a namespace, which is unique across the platform, avoiding collision
|
|
63
|
+
const id = `${workflowName}#${metadata.name}`;
|
|
64
|
+
|
|
65
|
+
if ( sharedActivities.has( metadata.name ) ) {
|
|
66
|
+
throw new ValidationError( `Activity "${metadata.name}" in workflow "${workflowName}" conflicts with a shared activity.` +
|
|
67
|
+
' Workflow activity names must not overlap with shared activity names.' );
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if ( activities.has( id ) ) {
|
|
71
|
+
throw new ValidationError( `Activity "${metadata.name}" in workflow "${workflowName}" conflicts with another activity in the same workflow.` +
|
|
72
|
+
' Activity names must be unique within a workflow.' );
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
log.info( metadata.name, { workflow: workflowName, type: metadata.type, ...( external && { external } ), path } );
|
|
76
|
+
activities.set( id, fn );
|
|
77
|
+
if ( metadata.options?.activityOptions ) {
|
|
78
|
+
activityOptions.set( id, metadata.options.activityOptions );
|
|
79
|
+
}
|
|
62
80
|
}
|
|
63
|
-
activities[activityKey] = fn;
|
|
64
|
-
activityOptionsMap[activityKey] = metadata.options?.activityOptions ?? undefined;
|
|
65
81
|
}
|
|
66
82
|
|
|
67
83
|
// writes down the activity option overrides
|
|
68
|
-
const optionsContent = `export default ${JSON.stringify(
|
|
84
|
+
const optionsContent = `export default ${JSON.stringify( Object.fromEntries( activityOptions ), undefined, 2 )};`;
|
|
69
85
|
const optionsFile = writeFileInTempDir( optionsContent, ACTIVITY_OPTIONS_FILENAME );
|
|
70
86
|
|
|
71
87
|
// system activities
|
|
72
|
-
activities
|
|
73
|
-
activities
|
|
74
|
-
return { activities, optionsFile };
|
|
88
|
+
activities.set( ACTIVITY_SEND_HTTP_REQUEST, sendHttpRequest );
|
|
89
|
+
activities.set( ACTIVITY_GET_TRACE_DESTINATIONS, getTraceDestinations );
|
|
90
|
+
return { activities: Object.fromEntries( activities ), optionsFile };
|
|
75
91
|
};
|
|
@@ -5,8 +5,7 @@ vi.mock( '#consts', () => ( {
|
|
|
5
5
|
ACTIVITY_GET_TRACE_DESTINATIONS: '__internal#getTraceDestinations',
|
|
6
6
|
WORKFLOWS_INDEX_FILENAME: '__workflows_entrypoint.js',
|
|
7
7
|
WORKFLOW_CATALOG: 'catalog',
|
|
8
|
-
ACTIVITY_OPTIONS_FILENAME: '__activity_options.js'
|
|
9
|
-
SHARED_STEP_PREFIX: '$shared'
|
|
8
|
+
ACTIVITY_OPTIONS_FILENAME: '__activity_options.js'
|
|
10
9
|
} ) );
|
|
11
10
|
|
|
12
11
|
const sendHttpRequestMock = vi.fn();
|
|
@@ -63,6 +62,7 @@ describe( 'loadActivities', () => {
|
|
|
63
62
|
it( 'loadActivities returns map including system activity and writes options file', async () => {
|
|
64
63
|
const { loadActivities } = await import( './activities.js' );
|
|
65
64
|
|
|
65
|
+
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
66
66
|
importComponentsMock.mockImplementationOnce( async function *() {
|
|
67
67
|
yield {
|
|
68
68
|
fn: () => {},
|
|
@@ -70,7 +70,6 @@ describe( 'loadActivities', () => {
|
|
|
70
70
|
path: '/a/steps.js'
|
|
71
71
|
};
|
|
72
72
|
} );
|
|
73
|
-
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
74
73
|
|
|
75
74
|
const workflows = [ { name: 'A', path: '/a/workflow.js' } ];
|
|
76
75
|
const { activities, optionsFile } = await loadActivities( '/root', workflows );
|
|
@@ -89,11 +88,11 @@ describe( 'loadActivities', () => {
|
|
|
89
88
|
|
|
90
89
|
it( 'loadActivities omits activity options when component has no options or no activityOptions', async () => {
|
|
91
90
|
const { loadActivities } = await import( './activities.js' );
|
|
91
|
+
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
92
92
|
importComponentsMock.mockImplementationOnce( async function *() {
|
|
93
93
|
yield { fn: () => {}, metadata: { name: 'NoOptions' }, path: '/a/steps.js' };
|
|
94
94
|
yield { fn: () => {}, metadata: { name: 'EmptyOptions', options: {} }, path: '/a/steps2.js' };
|
|
95
95
|
} );
|
|
96
|
-
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
97
96
|
|
|
98
97
|
await loadActivities( '/root', [ { name: 'A', path: '/a/workflow.js' } ] );
|
|
99
98
|
const written = JSON.parse(
|
|
@@ -105,6 +104,7 @@ describe( 'loadActivities', () => {
|
|
|
105
104
|
|
|
106
105
|
it( 'loadActivities throws when two activities in the same workflow share a name', async () => {
|
|
107
106
|
const { loadActivities } = await import( './activities.js' );
|
|
107
|
+
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
108
108
|
importComponentsMock.mockImplementationOnce( async function *() {
|
|
109
109
|
yield { fn: () => {}, metadata: { name: 'DuplicateActivity' }, path: '/a/steps.js' };
|
|
110
110
|
yield { fn: () => {}, metadata: { name: 'DuplicateActivity' }, path: '/a/evaluators.js' };
|
|
@@ -118,7 +118,6 @@ Activity names must be unique within a workflow.'
|
|
|
118
118
|
|
|
119
119
|
it( 'loadActivities throws when two shared activities share a name', async () => {
|
|
120
120
|
const { loadActivities } = await import( './activities.js' );
|
|
121
|
-
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
122
121
|
importComponentsMock.mockImplementationOnce( async function *() {
|
|
123
122
|
yield { fn: () => {}, metadata: { name: 'DuplicateShared' }, path: '/root/shared/steps/a.js' };
|
|
124
123
|
yield { fn: () => {}, metadata: { name: 'DuplicateShared' }, path: '/root/shared/evaluators/a.js' };
|
|
@@ -133,8 +132,8 @@ Activity names must be unique within a workflow.'
|
|
|
133
132
|
const { loadActivities } = await import( './activities.js' );
|
|
134
133
|
const workflowFiles = [ { path: '/a/steps/foo.js' } ];
|
|
135
134
|
const sharedFiles = [ { path: '/root/shared/steps/baz.js' } ];
|
|
136
|
-
matchFilesMock.mockReturnValueOnce( workflowFiles );
|
|
137
135
|
matchFilesMock.mockReturnValueOnce( sharedFiles );
|
|
136
|
+
matchFilesMock.mockReturnValueOnce( workflowFiles );
|
|
138
137
|
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
139
138
|
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
140
139
|
|
|
@@ -143,12 +142,12 @@ Activity names must be unique within a workflow.'
|
|
|
143
142
|
|
|
144
143
|
expect( matchFilesMock ).toHaveBeenCalledTimes( 2 );
|
|
145
144
|
expect( buildActivityMatcherMock ).toHaveBeenCalledWith( '/a' );
|
|
146
|
-
expect( matchFilesMock ).toHaveBeenNthCalledWith( 1, '/
|
|
147
|
-
expect( matchFilesMock ).toHaveBeenNthCalledWith( 2, '/
|
|
145
|
+
expect( matchFilesMock ).toHaveBeenNthCalledWith( 1, '/root', [ sharedStepsDirMock, sharedEvaluatorsDirMock ] );
|
|
146
|
+
expect( matchFilesMock ).toHaveBeenNthCalledWith( 2, '/a', [ activityMatcherMock ] );
|
|
148
147
|
|
|
149
148
|
expect( importComponentsMock ).toHaveBeenCalledTimes( 2 );
|
|
150
|
-
expect( importComponentsMock ).toHaveBeenNthCalledWith( 1,
|
|
151
|
-
expect( importComponentsMock ).toHaveBeenNthCalledWith( 2,
|
|
149
|
+
expect( importComponentsMock ).toHaveBeenNthCalledWith( 1, sharedFiles );
|
|
150
|
+
expect( importComponentsMock ).toHaveBeenNthCalledWith( 2, workflowFiles );
|
|
152
151
|
} );
|
|
153
152
|
|
|
154
153
|
it( 'loads shared activities from external workflow packages', async () => {
|
|
@@ -157,8 +156,6 @@ Activity names must be unique within a workflow.'
|
|
|
157
156
|
const localWorkflow = { name: 'Local', path: '/root/workflows/local/workflow.js' };
|
|
158
157
|
const externalWorkflow = { name: 'External', path: '/root/node_modules/pkg/workflows/a/workflow.js', external: true };
|
|
159
158
|
findSharedActivitiesFromWorkflowsMock.mockReturnValue( externalSharedFiles );
|
|
160
|
-
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
161
|
-
importComponentsMock.mockImplementationOnce( async function *() {} );
|
|
162
159
|
importComponentsMock.mockImplementationOnce( async function *() {
|
|
163
160
|
yield {
|
|
164
161
|
fn: () => {},
|
|
@@ -170,27 +167,29 @@ Activity names must be unique within a workflow.'
|
|
|
170
167
|
const { activities } = await loadActivities( '/root', [ localWorkflow, externalWorkflow ] );
|
|
171
168
|
|
|
172
169
|
expect( findSharedActivitiesFromWorkflowsMock ).toHaveBeenCalledWith( [ externalWorkflow ] );
|
|
173
|
-
expect( importComponentsMock ).toHaveBeenNthCalledWith(
|
|
174
|
-
expect( activities['
|
|
170
|
+
expect( importComponentsMock ).toHaveBeenNthCalledWith( 1, externalSharedFiles );
|
|
171
|
+
expect( activities['Local#ExternalShared'] ).toBeTypeOf( 'function' );
|
|
172
|
+
expect( activities['External#ExternalShared'] ).toBeTypeOf( 'function' );
|
|
175
173
|
const written = JSON.parse(
|
|
176
174
|
writeFileInTempDirMock.mock.calls[0][0].replace( /^export default\s*/, '' ).replace( /;\s*$/, '' )
|
|
177
175
|
);
|
|
178
|
-
expect( written['
|
|
176
|
+
expect( written['Local#ExternalShared'] ).toEqual( { retry: { maximumAttempts: 2 } } );
|
|
177
|
+
expect( written['External#ExternalShared'] ).toEqual( { retry: { maximumAttempts: 2 } } );
|
|
179
178
|
} );
|
|
180
179
|
|
|
181
180
|
it( 'loadActivities includes nested workflow steps and shared evaluators', async () => {
|
|
182
181
|
const { loadActivities } = await import( './activities.js' );
|
|
183
182
|
importComponentsMock.mockImplementationOnce( async function *() {
|
|
184
|
-
yield { fn: () => {}, metadata: { name: '
|
|
183
|
+
yield { fn: () => {}, metadata: { name: 'SharedEval' }, path: '/root/shared/evaluators/bar.js' };
|
|
185
184
|
} );
|
|
186
185
|
importComponentsMock.mockImplementationOnce( async function *() {
|
|
187
|
-
yield { fn: () => {}, metadata: { name: '
|
|
186
|
+
yield { fn: () => {}, metadata: { name: 'ActNested' }, path: '/a/steps/foo.js' };
|
|
188
187
|
} );
|
|
189
188
|
|
|
190
189
|
const workflows = [ { name: 'A', path: '/a/workflow.js' } ];
|
|
191
190
|
const { activities } = await loadActivities( '/root', workflows );
|
|
192
191
|
expect( activities['A#ActNested'] ).toBeTypeOf( 'function' );
|
|
193
|
-
expect( activities['
|
|
192
|
+
expect( activities['A#SharedEval'] ).toBeTypeOf( 'function' );
|
|
194
193
|
} );
|
|
195
194
|
|
|
196
195
|
it( 'collects shared nested steps and evaluators across multiple subfolders', async () => {
|
|
@@ -205,9 +204,9 @@ Activity names must be unique within a workflow.'
|
|
|
205
204
|
|
|
206
205
|
const workflows = [ { name: 'A', path: '/a/workflow.js' } ];
|
|
207
206
|
const { activities } = await loadActivities( '/root', workflows );
|
|
208
|
-
expect( activities['
|
|
209
|
-
expect( activities['
|
|
210
|
-
expect( activities['
|
|
211
|
-
expect( activities['
|
|
207
|
+
expect( activities['A#SharedStepPrimary'] ).toBeTypeOf( 'function' );
|
|
208
|
+
expect( activities['A#SharedStepSecondary'] ).toBeTypeOf( 'function' );
|
|
209
|
+
expect( activities['A#SharedEvalPrimary'] ).toBeTypeOf( 'function' );
|
|
210
|
+
expect( activities['A#SharedEvalSecondary'] ).toBeTypeOf( 'function' );
|
|
212
211
|
} );
|
|
213
212
|
} );
|
|
@@ -5,8 +5,7 @@ vi.mock( '#consts', () => ( {
|
|
|
5
5
|
ACTIVITY_GET_TRACE_DESTINATIONS: '__internal#getTraceDestinations',
|
|
6
6
|
WORKFLOWS_INDEX_FILENAME: '__workflows_entrypoint.js',
|
|
7
7
|
WORKFLOW_CATALOG: 'catalog',
|
|
8
|
-
ACTIVITY_OPTIONS_FILENAME: '__activity_options.js'
|
|
9
|
-
SHARED_STEP_PREFIX: '$shared'
|
|
8
|
+
ACTIVITY_OPTIONS_FILENAME: '__activity_options.js'
|
|
10
9
|
} ) );
|
|
11
10
|
|
|
12
11
|
const { importComponentsMock, findWorkflowsInNodeModulesMock, matchFilesMock, writeFileInTempDirMock } = vi.hoisted( () => ( {
|