@outputai/core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +11 -0
- package/bin/healthcheck.mjs +36 -0
- package/bin/healthcheck.spec.js +90 -0
- package/bin/worker.sh +26 -0
- package/package.json +67 -0
- package/src/activity_integration/context.d.ts +27 -0
- package/src/activity_integration/context.js +17 -0
- package/src/activity_integration/context.spec.js +42 -0
- package/src/activity_integration/events.d.ts +7 -0
- package/src/activity_integration/events.js +10 -0
- package/src/activity_integration/index.d.ts +9 -0
- package/src/activity_integration/index.js +3 -0
- package/src/activity_integration/tracing.d.ts +32 -0
- package/src/activity_integration/tracing.js +37 -0
- package/src/async_storage.js +19 -0
- package/src/bus.js +3 -0
- package/src/consts.js +32 -0
- package/src/errors.d.ts +15 -0
- package/src/errors.js +14 -0
- package/src/hooks/index.d.ts +28 -0
- package/src/hooks/index.js +32 -0
- package/src/index.d.ts +49 -0
- package/src/index.js +4 -0
- package/src/interface/evaluation_result.d.ts +173 -0
- package/src/interface/evaluation_result.js +215 -0
- package/src/interface/evaluator.d.ts +70 -0
- package/src/interface/evaluator.js +34 -0
- package/src/interface/evaluator.spec.js +565 -0
- package/src/interface/index.d.ts +9 -0
- package/src/interface/index.js +26 -0
- package/src/interface/step.d.ts +138 -0
- package/src/interface/step.js +22 -0
- package/src/interface/types.d.ts +27 -0
- package/src/interface/validations/runtime.js +20 -0
- package/src/interface/validations/runtime.spec.js +29 -0
- package/src/interface/validations/schema_utils.js +8 -0
- package/src/interface/validations/schema_utils.spec.js +67 -0
- package/src/interface/validations/static.js +136 -0
- package/src/interface/validations/static.spec.js +366 -0
- package/src/interface/webhook.d.ts +84 -0
- package/src/interface/webhook.js +64 -0
- package/src/interface/webhook.spec.js +122 -0
- package/src/interface/workflow.d.ts +273 -0
- package/src/interface/workflow.js +128 -0
- package/src/interface/workflow.spec.js +467 -0
- package/src/interface/workflow_context.js +31 -0
- package/src/interface/workflow_utils.d.ts +76 -0
- package/src/interface/workflow_utils.js +50 -0
- package/src/interface/workflow_utils.spec.js +190 -0
- package/src/interface/zod_integration.spec.js +646 -0
- package/src/internal_activities/index.js +66 -0
- package/src/internal_activities/index.spec.js +102 -0
- package/src/logger.js +73 -0
- package/src/tracing/internal_interface.js +71 -0
- package/src/tracing/processors/local/index.js +111 -0
- package/src/tracing/processors/local/index.spec.js +149 -0
- package/src/tracing/processors/s3/configs.js +31 -0
- package/src/tracing/processors/s3/configs.spec.js +64 -0
- package/src/tracing/processors/s3/index.js +114 -0
- package/src/tracing/processors/s3/index.spec.js +153 -0
- package/src/tracing/processors/s3/redis_client.js +62 -0
- package/src/tracing/processors/s3/redis_client.spec.js +185 -0
- package/src/tracing/processors/s3/s3_client.js +27 -0
- package/src/tracing/processors/s3/s3_client.spec.js +62 -0
- package/src/tracing/tools/build_trace_tree.js +83 -0
- package/src/tracing/tools/build_trace_tree.spec.js +135 -0
- package/src/tracing/tools/utils.js +21 -0
- package/src/tracing/tools/utils.spec.js +14 -0
- package/src/tracing/trace_engine.js +97 -0
- package/src/tracing/trace_engine.spec.js +199 -0
- package/src/utils/index.d.ts +134 -0
- package/src/utils/index.js +2 -0
- package/src/utils/resolve_invocation_dir.js +34 -0
- package/src/utils/resolve_invocation_dir.spec.js +102 -0
- package/src/utils/utils.js +211 -0
- package/src/utils/utils.spec.js +448 -0
- package/src/worker/bundler_options.js +43 -0
- package/src/worker/catalog_workflow/catalog.js +114 -0
- package/src/worker/catalog_workflow/index.js +54 -0
- package/src/worker/catalog_workflow/index.spec.js +196 -0
- package/src/worker/catalog_workflow/workflow.js +24 -0
- package/src/worker/configs.js +49 -0
- package/src/worker/configs.spec.js +130 -0
- package/src/worker/index.js +89 -0
- package/src/worker/index.spec.js +177 -0
- package/src/worker/interceptors/activity.js +62 -0
- package/src/worker/interceptors/activity.spec.js +212 -0
- package/src/worker/interceptors/workflow.js +70 -0
- package/src/worker/interceptors/workflow.spec.js +167 -0
- package/src/worker/interceptors.js +10 -0
- package/src/worker/loader.js +151 -0
- package/src/worker/loader.spec.js +236 -0
- package/src/worker/loader_tools.js +132 -0
- package/src/worker/loader_tools.spec.js +156 -0
- package/src/worker/log_hooks.js +95 -0
- package/src/worker/log_hooks.spec.js +217 -0
- package/src/worker/sandboxed_utils.js +18 -0
- package/src/worker/shutdown.js +26 -0
- package/src/worker/shutdown.spec.js +82 -0
- package/src/worker/sinks.js +74 -0
- package/src/worker/start_catalog.js +36 -0
- package/src/worker/start_catalog.spec.js +118 -0
- package/src/worker/webpack_loaders/consts.js +9 -0
- package/src/worker/webpack_loaders/tools.js +548 -0
- package/src/worker/webpack_loaders/tools.spec.js +330 -0
- package/src/worker/webpack_loaders/workflow_rewriter/collect_target_imports.js +221 -0
- package/src/worker/webpack_loaders/workflow_rewriter/collect_target_imports.spec.js +336 -0
- package/src/worker/webpack_loaders/workflow_rewriter/index.mjs +61 -0
- package/src/worker/webpack_loaders/workflow_rewriter/index.spec.js +216 -0
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_fn_bodies.js +196 -0
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_fn_bodies.spec.js +123 -0
- package/src/worker/webpack_loaders/workflow_validator/index.mjs +205 -0
- package/src/worker/webpack_loaders/workflow_validator/index.spec.js +613 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { Context } from '@temporalio/activity';
|
|
2
|
+
import { Storage } from '#async_storage';
|
|
3
|
+
import * as Tracing from '#tracing';
|
|
4
|
+
import { headersToObject } from '../sandboxed_utils.js';
|
|
5
|
+
import { BusEventType, METADATA_ACCESS_SYMBOL } from '#consts';
|
|
6
|
+
import { activityHeartbeatEnabled, activityHeartbeatIntervalMs } from '../configs.js';
|
|
7
|
+
import { messageBus } from '#bus';
|
|
8
|
+
|
|
9
|
+
/*
|
|
10
|
+
This interceptor wraps every activity execution with cross-cutting concerns:
|
|
11
|
+
|
|
12
|
+
1. Tracing: records start/end/error events and sets up AsyncLocalStorage context
|
|
13
|
+
so nested operations (e.g. HTTP calls inside steps) can be traced back to the parent activity.
|
|
14
|
+
|
|
15
|
+
2. Heartbeating: sends periodic heartbeat signals to Temporal so it can detect dead workers
|
|
16
|
+
without waiting for the full startToCloseTimeout (which can be up to 1h+).
|
|
17
|
+
This is critical during deploys — when a worker restarts, Temporal will notice
|
|
18
|
+
the missing heartbeat within the heartbeatTimeout window and retry the activity
|
|
19
|
+
on a new worker, instead of waiting the entire startToCloseTimeout.
|
|
20
|
+
|
|
21
|
+
Context information comes from two sources:
|
|
22
|
+
- Temporal's Activity Context (workflowId, activityId, activityType)
|
|
23
|
+
- Headers injected by the workflow interceptor (executionContext)
|
|
24
|
+
*/
|
|
25
|
+
export class ActivityExecutionInterceptor {
|
|
26
|
+
constructor( { activities, workflows } ) {
|
|
27
|
+
this.activities = activities;
|
|
28
|
+
this.workflowsMap = workflows.reduce( ( map, w ) => map.set( w.name, w ), new Map() );
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
async execute( input, next ) {
|
|
32
|
+
const startDate = Date.now();
|
|
33
|
+
const { workflowExecution: { workflowId }, activityId: id, activityType: name, workflowType: workflowName } = Context.current().info;
|
|
34
|
+
const { executionContext } = headersToObject( input.headers );
|
|
35
|
+
const { type: kind } = this.activities?.[name]?.[METADATA_ACCESS_SYMBOL];
|
|
36
|
+
const workflowFilename = this.workflowsMap.get( workflowName ).path;
|
|
37
|
+
|
|
38
|
+
messageBus.emit( BusEventType.ACTIVITY_START, { id, name, kind, workflowId, workflowName } );
|
|
39
|
+
Tracing.addEventStart( { id, name, kind, parentId: workflowId, details: input.args[0], executionContext } );
|
|
40
|
+
|
|
41
|
+
const intervals = { heartbeat: null };
|
|
42
|
+
try {
|
|
43
|
+
// Sends heartbeat to communicate that activity is still alive
|
|
44
|
+
intervals.heartbeat = activityHeartbeatEnabled && setInterval( () => Context.current().heartbeat(), activityHeartbeatIntervalMs );
|
|
45
|
+
|
|
46
|
+
// Wraps the execution with accessible metadata for the activity
|
|
47
|
+
const output = await Storage.runWithContext( async _ => next( input ), { parentId: id, executionContext, workflowFilename } );
|
|
48
|
+
|
|
49
|
+
messageBus.emit( BusEventType.ACTIVITY_END, { id, name, kind, workflowId, workflowName, duration: Date.now() - startDate } );
|
|
50
|
+
Tracing.addEventEnd( { id, details: output, executionContext } );
|
|
51
|
+
return output;
|
|
52
|
+
|
|
53
|
+
} catch ( error ) {
|
|
54
|
+
messageBus.emit( BusEventType.ACTIVITY_ERROR, { id, name, kind, workflowId, workflowName, duration: Date.now() - startDate, error } );
|
|
55
|
+
Tracing.addEventError( { id, details: error, executionContext } );
|
|
56
|
+
|
|
57
|
+
throw error;
|
|
58
|
+
} finally {
|
|
59
|
+
clearInterval( intervals.heartbeat );
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
};
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { BusEventType } from '#consts';
|
|
3
|
+
|
|
4
|
+
const METADATA_ACCESS_SYMBOL = vi.hoisted( () => Symbol( '__metadata' ) );
|
|
5
|
+
|
|
6
|
+
const heartbeatMock = vi.fn();
|
|
7
|
+
const runWithContextMock = vi.hoisted( () => vi.fn().mockImplementation( async fn => fn() ) );
|
|
8
|
+
const contextInfoMock = {
|
|
9
|
+
workflowExecution: { workflowId: 'wf-1' },
|
|
10
|
+
activityId: 'act-1',
|
|
11
|
+
activityType: 'myWorkflow#myStep',
|
|
12
|
+
workflowType: 'myWorkflow'
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
vi.mock( '@temporalio/activity', () => ( {
|
|
16
|
+
Context: {
|
|
17
|
+
current: () => ( {
|
|
18
|
+
info: contextInfoMock,
|
|
19
|
+
heartbeat: heartbeatMock
|
|
20
|
+
} )
|
|
21
|
+
}
|
|
22
|
+
} ) );
|
|
23
|
+
|
|
24
|
+
vi.mock( '#async_storage', () => ( {
|
|
25
|
+
Storage: {
|
|
26
|
+
runWithContext: runWithContextMock
|
|
27
|
+
}
|
|
28
|
+
} ) );
|
|
29
|
+
|
|
30
|
+
const addEventStartMock = vi.fn();
|
|
31
|
+
const addEventEndMock = vi.fn();
|
|
32
|
+
const addEventErrorMock = vi.fn();
|
|
33
|
+
vi.mock( '#tracing', () => ( {
|
|
34
|
+
addEventStart: addEventStartMock,
|
|
35
|
+
addEventEnd: addEventEndMock,
|
|
36
|
+
addEventError: addEventErrorMock
|
|
37
|
+
} ) );
|
|
38
|
+
|
|
39
|
+
vi.mock( '../sandboxed_utils.js', () => ( {
|
|
40
|
+
headersToObject: () => ( { executionContext: { workflowId: 'wf-1' } } )
|
|
41
|
+
} ) );
|
|
42
|
+
|
|
43
|
+
const messageBusEmitMock = vi.fn();
|
|
44
|
+
vi.mock( '#bus', () => ( { messageBus: { emit: messageBusEmitMock } } ) );
|
|
45
|
+
|
|
46
|
+
vi.mock( '#consts', async importOriginal => {
|
|
47
|
+
const actual = await importOriginal();
|
|
48
|
+
return {
|
|
49
|
+
...actual, get METADATA_ACCESS_SYMBOL() {
|
|
50
|
+
return METADATA_ACCESS_SYMBOL;
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
} );
|
|
54
|
+
|
|
55
|
+
vi.mock( '../configs.js', () => ( {
|
|
56
|
+
get activityHeartbeatEnabled() {
|
|
57
|
+
return process.env.OUTPUT_ACTIVITY_HEARTBEAT_ENABLED !== 'false';
|
|
58
|
+
},
|
|
59
|
+
get activityHeartbeatIntervalMs() {
|
|
60
|
+
return parseInt( process.env.OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS || '120000', 10 );
|
|
61
|
+
}
|
|
62
|
+
} ) );
|
|
63
|
+
|
|
64
|
+
const makeActivities = () => ( {
|
|
65
|
+
'myWorkflow#myStep': { [METADATA_ACCESS_SYMBOL]: { type: 'step' } }
|
|
66
|
+
} );
|
|
67
|
+
|
|
68
|
+
const makeWorkflows = () => [ { name: 'myWorkflow', path: '/workflows/myWorkflow.js' } ];
|
|
69
|
+
|
|
70
|
+
const makeInput = () => ( {
|
|
71
|
+
args: [ { someInput: 'data' } ],
|
|
72
|
+
headers: {}
|
|
73
|
+
} );
|
|
74
|
+
|
|
75
|
+
describe( 'ActivityExecutionInterceptor', () => {
|
|
76
|
+
beforeEach( () => {
|
|
77
|
+
vi.clearAllMocks();
|
|
78
|
+
vi.useFakeTimers();
|
|
79
|
+
vi.resetModules();
|
|
80
|
+
// Default: heartbeat enabled with 50ms interval for fast tests
|
|
81
|
+
vi.stubEnv( 'OUTPUT_ACTIVITY_HEARTBEAT_ENABLED', 'true' );
|
|
82
|
+
vi.stubEnv( 'OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS', '50' );
|
|
83
|
+
} );
|
|
84
|
+
|
|
85
|
+
afterEach( () => {
|
|
86
|
+
vi.useRealTimers();
|
|
87
|
+
vi.unstubAllEnvs();
|
|
88
|
+
} );
|
|
89
|
+
|
|
90
|
+
it( 'records trace start and end events on successful execution', async () => {
|
|
91
|
+
const { ActivityExecutionInterceptor } = await import( './activity.js' );
|
|
92
|
+
const interceptor = new ActivityExecutionInterceptor( { activities: makeActivities(), workflows: makeWorkflows() } );
|
|
93
|
+
const next = vi.fn().mockResolvedValue( { result: 'ok' } );
|
|
94
|
+
|
|
95
|
+
const promise = interceptor.execute( makeInput(), next );
|
|
96
|
+
vi.advanceTimersByTime( 0 );
|
|
97
|
+
const output = await promise;
|
|
98
|
+
|
|
99
|
+
expect( output ).toEqual( { result: 'ok' } );
|
|
100
|
+
expect( messageBusEmitMock ).toHaveBeenCalledWith( BusEventType.ACTIVITY_START, expect.objectContaining( {
|
|
101
|
+
id: 'act-1', name: 'myWorkflow#myStep', kind: 'step', workflowId: 'wf-1', workflowName: 'myWorkflow'
|
|
102
|
+
} ) );
|
|
103
|
+
expect( messageBusEmitMock ).toHaveBeenCalledWith( BusEventType.ACTIVITY_END, expect.objectContaining( {
|
|
104
|
+
id: 'act-1', name: 'myWorkflow#myStep', kind: 'step', workflowId: 'wf-1', workflowName: 'myWorkflow', duration: expect.any( Number )
|
|
105
|
+
} ) );
|
|
106
|
+
expect( addEventStartMock ).toHaveBeenCalledOnce();
|
|
107
|
+
expect( addEventEndMock ).toHaveBeenCalledOnce();
|
|
108
|
+
expect( addEventErrorMock ).not.toHaveBeenCalled();
|
|
109
|
+
expect( runWithContextMock ).toHaveBeenCalledWith(
|
|
110
|
+
expect.any( Function ),
|
|
111
|
+
{
|
|
112
|
+
parentId: 'act-1',
|
|
113
|
+
executionContext: { workflowId: 'wf-1' },
|
|
114
|
+
workflowFilename: '/workflows/myWorkflow.js'
|
|
115
|
+
}
|
|
116
|
+
);
|
|
117
|
+
} );
|
|
118
|
+
|
|
119
|
+
it( 'records trace error event on failed execution', async () => {
|
|
120
|
+
const { ActivityExecutionInterceptor } = await import( './activity.js' );
|
|
121
|
+
const interceptor = new ActivityExecutionInterceptor( { activities: makeActivities(), workflows: makeWorkflows() } );
|
|
122
|
+
const error = new Error( 'step failed' );
|
|
123
|
+
const next = vi.fn().mockRejectedValue( error );
|
|
124
|
+
|
|
125
|
+
const promise = interceptor.execute( makeInput(), next );
|
|
126
|
+
vi.advanceTimersByTime( 0 );
|
|
127
|
+
|
|
128
|
+
await expect( promise ).rejects.toThrow( 'step failed' );
|
|
129
|
+
expect( messageBusEmitMock ).toHaveBeenCalledWith( BusEventType.ACTIVITY_START, expect.any( Object ) );
|
|
130
|
+
expect( messageBusEmitMock ).toHaveBeenCalledWith( BusEventType.ACTIVITY_ERROR, expect.objectContaining( {
|
|
131
|
+
id: 'act-1', name: 'myWorkflow#myStep', kind: 'step', workflowId: 'wf-1', workflowName: 'myWorkflow',
|
|
132
|
+
duration: expect.any( Number ), error: expect.any( Error )
|
|
133
|
+
} ) );
|
|
134
|
+
expect( addEventStartMock ).toHaveBeenCalledOnce();
|
|
135
|
+
expect( addEventErrorMock ).toHaveBeenCalledOnce();
|
|
136
|
+
expect( addEventEndMock ).not.toHaveBeenCalled();
|
|
137
|
+
} );
|
|
138
|
+
|
|
139
|
+
it( 'sends periodic heartbeats during activity execution', async () => {
|
|
140
|
+
const { ActivityExecutionInterceptor } = await import( './activity.js' );
|
|
141
|
+
const interceptor = new ActivityExecutionInterceptor( { activities: makeActivities(), workflows: makeWorkflows() } );
|
|
142
|
+
|
|
143
|
+
// next() resolves only after we manually resolve it, simulating a long-running activity
|
|
144
|
+
const deferred = { resolve: null };
|
|
145
|
+
const next = vi.fn().mockImplementation( () => new Promise( r => {
|
|
146
|
+
deferred.resolve = r;
|
|
147
|
+
} ) );
|
|
148
|
+
|
|
149
|
+
const promise = interceptor.execute( makeInput(), next );
|
|
150
|
+
|
|
151
|
+
expect( heartbeatMock ).not.toHaveBeenCalled();
|
|
152
|
+
|
|
153
|
+
vi.advanceTimersByTime( 50 );
|
|
154
|
+
expect( heartbeatMock ).toHaveBeenCalledTimes( 1 );
|
|
155
|
+
|
|
156
|
+
vi.advanceTimersByTime( 50 );
|
|
157
|
+
expect( heartbeatMock ).toHaveBeenCalledTimes( 2 );
|
|
158
|
+
|
|
159
|
+
vi.advanceTimersByTime( 50 );
|
|
160
|
+
expect( heartbeatMock ).toHaveBeenCalledTimes( 3 );
|
|
161
|
+
|
|
162
|
+
deferred.resolve( { result: 'done' } );
|
|
163
|
+
await promise;
|
|
164
|
+
} );
|
|
165
|
+
|
|
166
|
+
it( 'clears heartbeat interval after activity completes', async () => {
|
|
167
|
+
const { ActivityExecutionInterceptor } = await import( './activity.js' );
|
|
168
|
+
const interceptor = new ActivityExecutionInterceptor( { activities: makeActivities(), workflows: makeWorkflows() } );
|
|
169
|
+
const next = vi.fn().mockResolvedValue( { result: 'ok' } );
|
|
170
|
+
|
|
171
|
+
const promise = interceptor.execute( makeInput(), next );
|
|
172
|
+
vi.advanceTimersByTime( 0 );
|
|
173
|
+
await promise;
|
|
174
|
+
|
|
175
|
+
heartbeatMock.mockClear();
|
|
176
|
+
vi.advanceTimersByTime( 500 );
|
|
177
|
+
expect( heartbeatMock ).not.toHaveBeenCalled();
|
|
178
|
+
} );
|
|
179
|
+
|
|
180
|
+
it( 'clears heartbeat interval after activity fails', async () => {
|
|
181
|
+
const { ActivityExecutionInterceptor } = await import( './activity.js' );
|
|
182
|
+
const interceptor = new ActivityExecutionInterceptor( { activities: makeActivities(), workflows: makeWorkflows() } );
|
|
183
|
+
const next = vi.fn().mockRejectedValue( new Error( 'boom' ) );
|
|
184
|
+
|
|
185
|
+
const promise = interceptor.execute( makeInput(), next );
|
|
186
|
+
vi.advanceTimersByTime( 0 );
|
|
187
|
+
await promise.catch( () => {} );
|
|
188
|
+
|
|
189
|
+
heartbeatMock.mockClear();
|
|
190
|
+
vi.advanceTimersByTime( 500 );
|
|
191
|
+
expect( heartbeatMock ).not.toHaveBeenCalled();
|
|
192
|
+
} );
|
|
193
|
+
|
|
194
|
+
it( 'does not heartbeat when OUTPUT_ACTIVITY_HEARTBEAT_ENABLED is false', async () => {
|
|
195
|
+
vi.stubEnv( 'OUTPUT_ACTIVITY_HEARTBEAT_ENABLED', 'false' );
|
|
196
|
+
const { ActivityExecutionInterceptor } = await import( './activity.js' );
|
|
197
|
+
const interceptor = new ActivityExecutionInterceptor( { activities: makeActivities(), workflows: makeWorkflows() } );
|
|
198
|
+
|
|
199
|
+
const deferred = { resolve: null };
|
|
200
|
+
const next = vi.fn().mockImplementation( () => new Promise( r => {
|
|
201
|
+
deferred.resolve = r;
|
|
202
|
+
} ) );
|
|
203
|
+
|
|
204
|
+
const promise = interceptor.execute( makeInput(), next );
|
|
205
|
+
|
|
206
|
+
vi.advanceTimersByTime( 200 );
|
|
207
|
+
expect( heartbeatMock ).not.toHaveBeenCalled();
|
|
208
|
+
|
|
209
|
+
deferred.resolve( { result: 'done' } );
|
|
210
|
+
await promise;
|
|
211
|
+
} );
|
|
212
|
+
} );
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// THIS RUNS IN THE TEMPORAL'S SANDBOX ENVIRONMENT
|
|
2
|
+
import { workflowInfo, proxySinks, ApplicationFailure, ContinueAsNew } from '@temporalio/workflow';
|
|
3
|
+
import { memoToHeaders } from '../sandboxed_utils.js';
|
|
4
|
+
import { deepMerge } from '#utils';
|
|
5
|
+
import { METADATA_ACCESS_SYMBOL } from '#consts';
|
|
6
|
+
// this is a dynamic generated file with activity configs overwrites
|
|
7
|
+
import stepOptions from '../temp/__activity_options.js';
|
|
8
|
+
|
|
9
|
+
/*
|
|
10
|
+
This is not an AI comment!
|
|
11
|
+
|
|
12
|
+
This interceptor adds information value from workflowInfo().memo as Activity invocation headers.
|
|
13
|
+
|
|
14
|
+
This is a strategy to share values between the workflow context and activity context.
|
|
15
|
+
|
|
16
|
+
We also want to preserve existing headers that might have been inject somewhere else and
|
|
17
|
+
*/
|
|
18
|
+
class HeadersInjectionInterceptor {
|
|
19
|
+
async scheduleActivity( input, next ) {
|
|
20
|
+
const memo = workflowInfo().memo ?? {};
|
|
21
|
+
Object.assign( input.headers, memoToHeaders( memo ) );
|
|
22
|
+
// apply per-invocation options passed as second argument by rewritten calls
|
|
23
|
+
const options = stepOptions[input.activityType];
|
|
24
|
+
if ( options ) {
|
|
25
|
+
input.options = deepMerge( memo.activityOptions, options );
|
|
26
|
+
}
|
|
27
|
+
return next( input );
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
const sinks = proxySinks();
|
|
32
|
+
|
|
33
|
+
class WorkflowExecutionInterceptor {
|
|
34
|
+
async execute( input, next ) {
|
|
35
|
+
sinks.workflow.start( input.args[0] );
|
|
36
|
+
try {
|
|
37
|
+
const output = await next( input );
|
|
38
|
+
sinks.workflow.end( output );
|
|
39
|
+
return output;
|
|
40
|
+
} catch ( error ) {
|
|
41
|
+
/*
|
|
42
|
+
* When the error is a ContinueAsNew instance, it represents the point where the actual workflow code was
|
|
43
|
+
* delegated to another run. In this case the result in the traces will be the string below and
|
|
44
|
+
* a new trace file will be generated
|
|
45
|
+
*/
|
|
46
|
+
if ( error instanceof ContinueAsNew ) {
|
|
47
|
+
sinks.workflow.end( '<continued_as_new>' );
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
sinks.workflow.error( error );
|
|
52
|
+
const failure = new ApplicationFailure( error.message, error.constructor.name, undefined, undefined, error );
|
|
53
|
+
|
|
54
|
+
/*
|
|
55
|
+
* If intercepted error has metadata, set it to .details property of Temporal's ApplicationFailure instance.
|
|
56
|
+
* This make it possible for this information be retrieved by Temporal's client instance.
|
|
57
|
+
* Ref: https://typescript.temporal.io/api/classes/common.ApplicationFailure#details
|
|
58
|
+
*/
|
|
59
|
+
if ( error[METADATA_ACCESS_SYMBOL] ) {
|
|
60
|
+
failure.details = [ error[METADATA_ACCESS_SYMBOL] ];
|
|
61
|
+
}
|
|
62
|
+
throw failure;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
export const interceptors = () => ( {
|
|
68
|
+
inbound: [ new WorkflowExecutionInterceptor() ],
|
|
69
|
+
outbound: [ new HeadersInjectionInterceptor( workflowInfo().workflowType ) ]
|
|
70
|
+
} );
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
+
|
|
3
|
+
const METADATA_ACCESS_SYMBOL = vi.hoisted( () => Symbol( '__metadata' ) );
|
|
4
|
+
|
|
5
|
+
const workflowInfoMock = vi.fn();
|
|
6
|
+
const workflowStartMock = vi.fn();
|
|
7
|
+
const workflowEndMock = vi.fn();
|
|
8
|
+
const workflowErrorMock = vi.fn();
|
|
9
|
+
vi.mock( '@temporalio/workflow', () => ( {
|
|
10
|
+
workflowInfo: ( ...args ) => workflowInfoMock( ...args ),
|
|
11
|
+
proxySinks: () => ( {
|
|
12
|
+
workflow: { start: workflowStartMock, end: workflowEndMock, error: workflowErrorMock }
|
|
13
|
+
} ),
|
|
14
|
+
ApplicationFailure: class ApplicationFailure {
|
|
15
|
+
constructor( message, type, nonRetryable, cause, originalError ) {
|
|
16
|
+
this.message = message;
|
|
17
|
+
this.type = type;
|
|
18
|
+
this.nonRetryable = nonRetryable;
|
|
19
|
+
this.cause = cause;
|
|
20
|
+
this.originalError = originalError;
|
|
21
|
+
this.details = undefined;
|
|
22
|
+
}
|
|
23
|
+
},
|
|
24
|
+
ContinueAsNew: class ContinueAsNew extends Error {
|
|
25
|
+
constructor() {
|
|
26
|
+
super( 'ContinueAsNew' );
|
|
27
|
+
this.name = 'ContinueAsNew';
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
} ) );
|
|
31
|
+
|
|
32
|
+
const memoToHeadersMock = vi.fn( memo => ( memo ? { ...memo, __asHeaders: true } : {} ) );
|
|
33
|
+
vi.mock( '../sandboxed_utils.js', () => ( { memoToHeaders: ( ...args ) => memoToHeadersMock( ...args ) } ) );
|
|
34
|
+
|
|
35
|
+
const deepMergeMock = vi.fn( ( a, b ) => ( { ...( a || {} ), ...( b || {} ) } ) );
|
|
36
|
+
vi.mock( '#utils', () => ( { deepMerge: ( ...args ) => deepMergeMock( ...args ) } ) );
|
|
37
|
+
|
|
38
|
+
vi.mock( '#consts', async importOriginal => {
|
|
39
|
+
const actual = await importOriginal();
|
|
40
|
+
return {
|
|
41
|
+
...actual, get METADATA_ACCESS_SYMBOL() {
|
|
42
|
+
return METADATA_ACCESS_SYMBOL;
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
} );
|
|
46
|
+
|
|
47
|
+
const stepOptionsDefault = {};
|
|
48
|
+
vi.mock( '../temp/__activity_options.js', () => ( { default: stepOptionsDefault } ) );
|
|
49
|
+
|
|
50
|
+
describe( 'workflow interceptors', () => {
|
|
51
|
+
beforeEach( () => {
|
|
52
|
+
vi.clearAllMocks();
|
|
53
|
+
workflowInfoMock.mockReturnValue( { workflowType: 'MyWorkflow', memo: { executionContext: { id: 'ctx-1' } } } );
|
|
54
|
+
} );
|
|
55
|
+
|
|
56
|
+
describe( 'HeadersInjectionInterceptor', () => {
|
|
57
|
+
it( 'assigns memo as headers via memoToHeaders and calls next', async () => {
|
|
58
|
+
const { interceptors } = await import( './workflow.js' );
|
|
59
|
+
const { outbound } = interceptors();
|
|
60
|
+
const interceptor = outbound[0];
|
|
61
|
+
const input = { headers: { existing: 'header' }, activityType: 'MyWorkflow#step1' };
|
|
62
|
+
const next = vi.fn().mockResolvedValue( 'result' );
|
|
63
|
+
|
|
64
|
+
memoToHeadersMock.mockReturnValue( { executionContext: { id: 'ctx-1' } } );
|
|
65
|
+
|
|
66
|
+
const out = await interceptor.scheduleActivity( input, next );
|
|
67
|
+
|
|
68
|
+
expect( memoToHeadersMock ).toHaveBeenCalledWith( { executionContext: { id: 'ctx-1' } } );
|
|
69
|
+
expect( input.headers ).toEqual( { existing: 'header', executionContext: { id: 'ctx-1' } } );
|
|
70
|
+
expect( next ).toHaveBeenCalledWith( input );
|
|
71
|
+
expect( out ).toBe( 'result' );
|
|
72
|
+
} );
|
|
73
|
+
|
|
74
|
+
it( 'merges stepOptions with memo.activityOptions when stepOptions exist for activityType', async () => {
|
|
75
|
+
stepOptionsDefault['MyWorkflow#step1'] = { scheduleToCloseTimeout: 60 };
|
|
76
|
+
workflowInfoMock.mockReturnValue( {
|
|
77
|
+
workflowType: 'MyWorkflow',
|
|
78
|
+
memo: { executionContext: {}, activityOptions: { heartbeatTimeout: 10 } }
|
|
79
|
+
} );
|
|
80
|
+
memoToHeadersMock.mockReturnValue( {} );
|
|
81
|
+
deepMergeMock.mockReturnValue( { heartbeatTimeout: 10, scheduleToCloseTimeout: 60 } );
|
|
82
|
+
|
|
83
|
+
const { interceptors } = await import( './workflow.js' );
|
|
84
|
+
const { outbound } = interceptors();
|
|
85
|
+
const interceptor = outbound[0];
|
|
86
|
+
const input = { headers: {}, activityType: 'MyWorkflow#step1' };
|
|
87
|
+
const next = vi.fn().mockResolvedValue( undefined );
|
|
88
|
+
|
|
89
|
+
await interceptor.scheduleActivity( input, next );
|
|
90
|
+
|
|
91
|
+
expect( deepMergeMock ).toHaveBeenCalledWith( { heartbeatTimeout: 10 }, { scheduleToCloseTimeout: 60 } );
|
|
92
|
+
expect( input.options ).toEqual( { heartbeatTimeout: 10, scheduleToCloseTimeout: 60 } );
|
|
93
|
+
delete stepOptionsDefault['MyWorkflow#step1'];
|
|
94
|
+
} );
|
|
95
|
+
} );
|
|
96
|
+
|
|
97
|
+
describe( 'WorkflowExecutionInterceptor', () => {
|
|
98
|
+
it( 'calls sinks.workflow.start, next, then sinks.workflow.end on success', async () => {
|
|
99
|
+
const { interceptors } = await import( './workflow.js' );
|
|
100
|
+
const { inbound } = interceptors();
|
|
101
|
+
const interceptor = inbound[0];
|
|
102
|
+
const input = { args: [ { input: 'data' } ] };
|
|
103
|
+
const next = vi.fn().mockResolvedValue( { output: 'ok' } );
|
|
104
|
+
|
|
105
|
+
const result = await interceptor.execute( input, next );
|
|
106
|
+
|
|
107
|
+
expect( workflowStartMock ).toHaveBeenCalledWith( { input: 'data' } );
|
|
108
|
+
expect( next ).toHaveBeenCalledWith( input );
|
|
109
|
+
expect( workflowEndMock ).toHaveBeenCalledWith( { output: 'ok' } );
|
|
110
|
+
expect( result ).toEqual( { output: 'ok' } );
|
|
111
|
+
expect( workflowErrorMock ).not.toHaveBeenCalled();
|
|
112
|
+
} );
|
|
113
|
+
|
|
114
|
+
it( 'calls sinks.workflow.error and throws ApplicationFailure on error', async () => {
|
|
115
|
+
const { interceptors } = await import( './workflow.js' );
|
|
116
|
+
const { inbound } = interceptors();
|
|
117
|
+
const interceptor = inbound[0];
|
|
118
|
+
const input = { args: [ {} ] };
|
|
119
|
+
const err = new Error( 'workflow failed' );
|
|
120
|
+
const next = vi.fn().mockRejectedValue( err );
|
|
121
|
+
|
|
122
|
+
await expect( interceptor.execute( input, next ) ).rejects.toMatchObject( {
|
|
123
|
+
message: 'workflow failed',
|
|
124
|
+
type: 'Error',
|
|
125
|
+
originalError: err
|
|
126
|
+
} );
|
|
127
|
+
expect( workflowStartMock ).toHaveBeenCalled();
|
|
128
|
+
expect( workflowErrorMock ).toHaveBeenCalledWith( err );
|
|
129
|
+
expect( workflowEndMock ).not.toHaveBeenCalled();
|
|
130
|
+
} );
|
|
131
|
+
|
|
132
|
+
it( 'sets failure.details from error metadata when present', async () => {
|
|
133
|
+
const { interceptors } = await import( './workflow.js' );
|
|
134
|
+
const { ApplicationFailure } = await import( '@temporalio/workflow' );
|
|
135
|
+
const { inbound } = interceptors();
|
|
136
|
+
const interceptor = inbound[0];
|
|
137
|
+
const meta = { code: 'CUSTOM' };
|
|
138
|
+
const err = new Error( 'custom' );
|
|
139
|
+
err[METADATA_ACCESS_SYMBOL] = meta;
|
|
140
|
+
const next = vi.fn().mockRejectedValue( err );
|
|
141
|
+
|
|
142
|
+
const error = await ( async () => {
|
|
143
|
+
try {
|
|
144
|
+
await interceptor.execute( { args: [ {} ] }, next );
|
|
145
|
+
return null;
|
|
146
|
+
} catch ( error ) {
|
|
147
|
+
return error;
|
|
148
|
+
}
|
|
149
|
+
} )();
|
|
150
|
+
expect( error ).toBeInstanceOf( ApplicationFailure );
|
|
151
|
+
expect( error.details ).toEqual( [ meta ] );
|
|
152
|
+
} );
|
|
153
|
+
|
|
154
|
+
it( 'on ContinueAsNew calls sinks.trace.addWorkflowEventEnd and rethrows', async () => {
|
|
155
|
+
const { ContinueAsNew } = await import( '@temporalio/workflow' );
|
|
156
|
+
const { interceptors } = await import( './workflow.js' );
|
|
157
|
+
const { inbound } = interceptors();
|
|
158
|
+
const interceptor = inbound[0];
|
|
159
|
+
const continueErr = new ContinueAsNew();
|
|
160
|
+
const next = vi.fn().mockRejectedValue( continueErr );
|
|
161
|
+
|
|
162
|
+
await expect( interceptor.execute( { args: [ {} ] }, next ) ).rejects.toThrow( ContinueAsNew );
|
|
163
|
+
expect( workflowEndMock ).toHaveBeenCalledWith( '<continued_as_new>' );
|
|
164
|
+
expect( workflowErrorMock ).not.toHaveBeenCalled();
|
|
165
|
+
} );
|
|
166
|
+
} );
|
|
167
|
+
} );
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { dirname, join } from 'path';
|
|
2
|
+
import { fileURLToPath } from 'node:url';
|
|
3
|
+
import { ActivityExecutionInterceptor } from './interceptors/activity.js';
|
|
4
|
+
|
|
5
|
+
const __dirname = dirname( fileURLToPath( import.meta.url ) );
|
|
6
|
+
|
|
7
|
+
export const initInterceptors = ( { activities, workflows } ) => ( {
|
|
8
|
+
workflowModules: [ join( __dirname, './interceptors/workflow.js' ) ],
|
|
9
|
+
activityInbound: [ () => new ActivityExecutionInterceptor( { activities, workflows } ) ]
|
|
10
|
+
} );
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { dirname, join } from 'node:path';
|
|
2
|
+
import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { EOL } from 'node:os';
|
|
4
|
+
import { fileURLToPath } from 'url';
|
|
5
|
+
import { getTraceDestinations, sendHttpRequest } from '#internal_activities';
|
|
6
|
+
import { importComponents, staticMatchers, activityMatchersBuilder } from './loader_tools.js';
|
|
7
|
+
import {
|
|
8
|
+
ACTIVITY_SEND_HTTP_REQUEST,
|
|
9
|
+
ACTIVITY_OPTIONS_FILENAME,
|
|
10
|
+
SHARED_STEP_PREFIX,
|
|
11
|
+
WORKFLOWS_INDEX_FILENAME,
|
|
12
|
+
WORKFLOW_CATALOG,
|
|
13
|
+
ACTIVITY_GET_TRACE_DESTINATIONS
|
|
14
|
+
} from '#consts';
|
|
15
|
+
import { createChildLogger } from '#logger';
|
|
16
|
+
|
|
17
|
+
const log = createChildLogger( 'Scanner' );
|
|
18
|
+
|
|
19
|
+
const __dirname = dirname( fileURLToPath( import.meta.url ) );
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Writes to file the activity options
|
|
23
|
+
*
|
|
24
|
+
* @param {object} optionsMap
|
|
25
|
+
*/
|
|
26
|
+
const writeActivityOptionsFile = map => {
|
|
27
|
+
const path = join( __dirname, 'temp', ACTIVITY_OPTIONS_FILENAME );
|
|
28
|
+
mkdirSync( dirname( path ), { recursive: true } );
|
|
29
|
+
writeFileSync( path, `export default ${JSON.stringify( map, undefined, 2 )};`, 'utf-8' );
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Creates the activity key that will identify it on Temporal.
|
|
34
|
+
*
|
|
35
|
+
* It composes it using a namespace and the name of the activity.
|
|
36
|
+
*
|
|
37
|
+
* No two activities with the same name can exist on the same namespace.
|
|
38
|
+
*
|
|
39
|
+
* @param {object} options
|
|
40
|
+
* @param {string} namespace
|
|
41
|
+
* @param {string} activityName
|
|
42
|
+
* @returns {string} key
|
|
43
|
+
*/
|
|
44
|
+
const generateActivityKey = ( { namespace, activityName } ) => `${namespace}#${activityName}`;
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Load activities:
|
|
48
|
+
*
|
|
49
|
+
* - Scans activities based on workflows, using each workflow folder as a point to lookup for steps, evaluators files;
|
|
50
|
+
* - Scans shared activities in the rootDir;
|
|
51
|
+
* - Loads internal activities as well;
|
|
52
|
+
*
|
|
53
|
+
* Builds a map of activities, where they is generated according to the type of activity and the value is the function itself and return it.
|
|
54
|
+
* - Shared activity keys have a common prefix followed by the activity name;
|
|
55
|
+
* - Internal activities are registered with a fixed key;
|
|
56
|
+
* - Workflow activities keys are composed using the workflow name and the activity name;
|
|
57
|
+
*
|
|
58
|
+
* @param {string} rootDir
|
|
59
|
+
* @param {object[]} workflows
|
|
60
|
+
* @returns {object}
|
|
61
|
+
*/
|
|
62
|
+
export async function loadActivities( rootDir, workflows ) {
|
|
63
|
+
const activities = {};
|
|
64
|
+
const activityOptionsMap = {};
|
|
65
|
+
|
|
66
|
+
// Load workflow based activities
|
|
67
|
+
for ( const { path: workflowPath, name: workflowName } of workflows ) {
|
|
68
|
+
const dir = dirname( workflowPath );
|
|
69
|
+
for await ( const { fn, metadata, path } of importComponents( dir, Object.values( activityMatchersBuilder( dir ) ) ) ) {
|
|
70
|
+
log.info( 'Component loaded', { type: metadata.type, name: metadata.name, path, workflow: workflowName } );
|
|
71
|
+
// Activities loaded from a workflow path will use the workflow name as a namespace, which is unique across the platform, avoiding collision
|
|
72
|
+
const activityKey = generateActivityKey( { namespace: workflowName, activityName: metadata.name } );
|
|
73
|
+
activities[activityKey] = fn;
|
|
74
|
+
// propagate the custom options set on the step()/evaluator() constructor
|
|
75
|
+
activityOptionsMap[activityKey] = metadata.options?.activityOptions ?? undefined;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// Load shared activities/evaluators
|
|
80
|
+
for await ( const { fn, metadata, path } of importComponents( rootDir, [ staticMatchers.sharedStepsDir, staticMatchers.sharedEvaluatorsDir ] ) ) {
|
|
81
|
+
log.info( 'Shared component loaded', { type: metadata.type, name: metadata.name, path } );
|
|
82
|
+
// The namespace for shared activities is fixed
|
|
83
|
+
const activityKey = generateActivityKey( { namespace: SHARED_STEP_PREFIX, activityName: metadata.name } );
|
|
84
|
+
activities[activityKey] = fn;
|
|
85
|
+
activityOptionsMap[activityKey] = metadata.options?.activityOptions ?? undefined;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// writes down the activity option overrides
|
|
89
|
+
writeActivityOptionsFile( activityOptionsMap );
|
|
90
|
+
|
|
91
|
+
// system activities
|
|
92
|
+
activities[ACTIVITY_SEND_HTTP_REQUEST] = sendHttpRequest;
|
|
93
|
+
activities[ACTIVITY_GET_TRACE_DESTINATIONS] = getTraceDestinations;
|
|
94
|
+
return activities;
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Scan and find workflow.js files and import them.
|
|
99
|
+
*
|
|
100
|
+
* Creates an array containing their metadata and path and return it.
|
|
101
|
+
*
|
|
102
|
+
* @param {string} rootDir
|
|
103
|
+
* @returns {object[]}
|
|
104
|
+
*/
|
|
105
|
+
export async function loadWorkflows( rootDir ) {
|
|
106
|
+
const workflows = [];
|
|
107
|
+
for await ( const { metadata, path } of importComponents( rootDir, [ staticMatchers.workflowFile ] ) ) {
|
|
108
|
+
if ( staticMatchers.workflowPathHasShared( path ) ) {
|
|
109
|
+
throw new Error( 'Workflow directory can\'t be named "shared"' );
|
|
110
|
+
}
|
|
111
|
+
log.info( 'Workflow loaded', { name: metadata.name, path } );
|
|
112
|
+
workflows.push( { ...metadata, path } );
|
|
113
|
+
}
|
|
114
|
+
return workflows;
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Loads the hook files from package.json's output config section.
|
|
119
|
+
*
|
|
120
|
+
* @param {string} rootDir
|
|
121
|
+
* @returns {void}
|
|
122
|
+
*/
|
|
123
|
+
export async function loadHooks( rootDir ) {
|
|
124
|
+
const packageFile = join( rootDir, 'package.json' );
|
|
125
|
+
if ( existsSync( packageFile ) ) {
|
|
126
|
+
const pkg = await import( packageFile, { with: { type: 'json' } } );
|
|
127
|
+
for ( const path of pkg.default.output?.hookFiles ?? [] ) {
|
|
128
|
+
const hookFile = join( rootDir, path );
|
|
129
|
+
await import( hookFile );
|
|
130
|
+
log.info( 'Hook file loaded', { path } );
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Creates a temporary index file importing all workflows for Temporal.
|
|
137
|
+
*
|
|
138
|
+
* @param {object[]} workflows
|
|
139
|
+
* @returns
|
|
140
|
+
*/
|
|
141
|
+
export function createWorkflowsEntryPoint( workflows ) {
|
|
142
|
+
const path = join( __dirname, 'temp', WORKFLOWS_INDEX_FILENAME );
|
|
143
|
+
|
|
144
|
+
// default system catalog workflow
|
|
145
|
+
const catalog = { name: WORKFLOW_CATALOG, path: join( __dirname, './catalog_workflow/workflow.js' ) };
|
|
146
|
+
const content = [ ... workflows, catalog ].map( ( { name, path } ) => `export { default as ${name} } from '${path}';` ).join( EOL );
|
|
147
|
+
|
|
148
|
+
mkdirSync( dirname( path ), { recursive: true } );
|
|
149
|
+
writeFileSync( path, content, 'utf-8' );
|
|
150
|
+
return path;
|
|
151
|
+
};
|