@outputai/core 0.10.1-next.b7b2fbe.0 → 0.10.1-next.bf95a6f.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +13 -7
- package/src/bus.js +58 -20
- package/src/bus.spec.js +135 -77
- package/src/consts.js +7 -3
- package/src/errors.d.ts +12 -2
- package/src/errors.js +17 -3
- package/src/errors.spec.js +19 -0
- package/src/helpers/activity_options.js +21 -0
- package/src/helpers/activity_options.spec.js +55 -0
- package/src/helpers/error_serializer.js +236 -0
- package/src/helpers/error_serializer.spec.js +413 -0
- package/src/helpers/errors.js +23 -14
- package/src/helpers/errors.spec.js +31 -75
- package/src/helpers/fetch.js +1 -38
- package/src/helpers/fetch.spec.js +1 -39
- package/src/helpers/function.js +13 -0
- package/src/helpers/function.spec.js +23 -1
- package/src/helpers/object.js +48 -35
- package/src/helpers/object.spec.js +123 -75
- package/src/helpers/redact.js +51 -0
- package/src/helpers/redact.spec.js +66 -0
- package/src/helpers/string.js +10 -0
- package/src/helpers/string.spec.js +36 -1
- package/src/hooks/index.d.ts +76 -78
- package/src/hooks/index.js +57 -49
- package/src/hooks/index.spec.js +104 -105
- package/src/hooks/pending_hooks.js +24 -0
- package/src/hooks/pending_hooks.spec.js +62 -0
- package/src/interface/evaluator.d.ts +10 -9
- package/src/interface/evaluator.js +1 -6
- package/src/interface/evaluator.spec.js +10 -10
- package/src/interface/index.js +2 -1
- package/src/interface/step.d.ts +16 -15
- package/src/interface/step.js +1 -6
- package/src/interface/step.spec.js +10 -10
- package/src/interface/validations/index.js +33 -53
- package/src/interface/validations/index.spec.js +25 -17
- package/src/interface/webhook.js +1 -2
- package/src/interface/webhook.spec.js +25 -19
- package/src/interface/workflow.d.ts +25 -18
- package/src/interface/workflow.js +64 -85
- package/src/interface/workflow.spec.js +220 -226
- package/src/interface/workflow_activity_options.js +2 -5
- package/src/interface/workflow_utils.d.ts +15 -0
- package/src/interface/workflow_utils.js +19 -0
- package/src/interface/workflow_utils.spec.js +82 -1
- package/src/sdk/helpers/objects.d.ts +24 -19
- package/src/sdk/runtime/events.d.ts +4 -6
- package/src/sdk/runtime/events.js +2 -15
- package/src/sdk/runtime/events.spec.js +14 -92
- package/src/temporal/client.js +47 -0
- package/src/temporal/client.spec.js +124 -0
- package/src/temporal/index.d.ts +32 -0
- package/src/temporal/index.js +1 -0
- package/src/tracing/trace_engine.js +9 -11
- package/src/tracing/trace_engine.spec.js +67 -28
- package/src/worker/catalog_workflow/catalog_job.js +3 -3
- package/src/worker/catalog_workflow/catalog_job.spec.js +1 -1
- package/src/worker/configs.js +0 -3
- package/src/worker/global_functions.js +2 -2
- package/src/worker/global_functions.spec.js +3 -3
- package/src/worker/index.js +24 -11
- package/src/worker/index.spec.js +36 -16
- package/src/worker/interceptors/activity.js +34 -21
- package/src/worker/interceptors/activity.spec.js +77 -172
- package/src/worker/interceptors/workflow.js +58 -16
- package/src/worker/interceptors/workflow.spec.js +183 -56
- package/src/worker/loader/activities.js +56 -40
- package/src/worker/loader/activities.spec.js +21 -22
- package/src/worker/loader/workflows.js +16 -3
- package/src/worker/loader/workflows.spec.js +36 -5
- package/src/worker/log_hooks.js +22 -19
- package/src/worker/log_hooks.spec.js +14 -6
- package/src/worker/sinks.js +8 -8
- package/src/worker/sinks.spec.js +9 -9
- package/src/worker/telemetry.js +2 -1
- package/src/worker/temporal_logger.js +33 -0
- package/src/worker/temporal_logger.spec.js +90 -0
- package/src/worker/webpack_loaders/consts.js +6 -0
- package/src/worker/webpack_loaders/tools.js +1 -146
- package/src/worker/webpack_loaders/tools.spec.js +0 -23
- package/src/worker/webpack_loaders/workflow_rewriter/collect_target_imports.js +24 -32
- package/src/worker/webpack_loaders/workflow_rewriter/collect_target_imports.spec.js +82 -279
- package/src/worker/webpack_loaders/workflow_rewriter/index.mjs +6 -7
- package/src/worker/webpack_loaders/workflow_rewriter/index.spec.js +57 -119
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_activity_calls.js +88 -0
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_activity_calls.spec.js +165 -0
- package/src/worker/webpack_loaders/workflow_validator/index.mjs +96 -29
- package/src/worker/webpack_loaders/workflow_validator/index.spec.js +110 -583
- package/src/worker/webpack_loaders/{npm_workflow_export_resolve.js → workflow_validator/npm_workflow_export_resolve.js} +1 -1
- package/src/worker/webpack_loaders/{npm_workflow_export_resolve.spec.js → workflow_validator/npm_workflow_export_resolve.spec.js} +1 -1
- package/src/helpers/aggregations.js +0 -44
- package/src/helpers/aggregations.spec.js +0 -92
- package/src/tracing/tools/utils.js +0 -42
- package/src/tracing/tools/utils.spec.js +0 -83
- package/src/worker/proxy.js +0 -20
- package/src/worker/proxy.spec.js +0 -51
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_fn_bodies.js +0 -193
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_fn_bodies.spec.js +0 -123
|
@@ -5,10 +5,15 @@ vi.mock( '#async_storage', () => ( {
|
|
|
5
5
|
Storage: { load: storageLoadMock }
|
|
6
6
|
} ) );
|
|
7
7
|
|
|
8
|
-
const
|
|
8
|
+
const serializeErrorMock = vi.fn( error => ( {
|
|
9
|
+
name: error.name,
|
|
10
|
+
message: error.message
|
|
11
|
+
} ) );
|
|
12
|
+
vi.mock( '#helpers/error_serializer', () => ( { serializeError: serializeErrorMock } ) );
|
|
13
|
+
|
|
9
14
|
const logErrorMock = vi.fn();
|
|
10
15
|
vi.mock( '#logger', () => ( {
|
|
11
|
-
createChildLogger: () => ( {
|
|
16
|
+
createChildLogger: () => ( { error: logErrorMock } )
|
|
12
17
|
} ) );
|
|
13
18
|
|
|
14
19
|
const localInitMock = vi.fn( async () => {} );
|
|
@@ -71,6 +76,24 @@ describe( 'tracing/trace_engine', () => {
|
|
|
71
76
|
expect( payload.traceInfo ).toBe( traceInfo );
|
|
72
77
|
} );
|
|
73
78
|
|
|
79
|
+
it( 'init() logs processor failures with serialized stacks', async () => {
|
|
80
|
+
process.env.OUTPUT_TRACE_LOCAL_ON = '1';
|
|
81
|
+
const processorError = new Error( 'processor failed' );
|
|
82
|
+
localExecMock.mockRejectedValueOnce( processorError );
|
|
83
|
+
const { init, addEventAction } = await loadTraceEngine();
|
|
84
|
+
await init();
|
|
85
|
+
|
|
86
|
+
addEventAction( 'start', {
|
|
87
|
+
kind: 'step', name: 'N', id: '1', parentId: 'p', details: null, traceInfo
|
|
88
|
+
} );
|
|
89
|
+
|
|
90
|
+
await vi.waitFor( () => expect( logErrorMock ).toHaveBeenCalledWith( 'Processor execution error', {
|
|
91
|
+
processor: 'LOCAL',
|
|
92
|
+
error: { name: 'Error', message: 'processor failed' }
|
|
93
|
+
} ) );
|
|
94
|
+
expect( serializeErrorMock ).toHaveBeenCalledWith( processorError );
|
|
95
|
+
} );
|
|
96
|
+
|
|
74
97
|
it( 'addEventAction() emits an entry consumed by processors', async () => {
|
|
75
98
|
process.env.OUTPUT_TRACE_LOCAL_ON = 'on';
|
|
76
99
|
const { init, addEventAction } = await loadTraceEngine();
|
|
@@ -87,34 +110,34 @@ describe( 'tracing/trace_engine', () => {
|
|
|
87
110
|
expect( payload.entry.details ).toBe( 'done' );
|
|
88
111
|
} );
|
|
89
112
|
|
|
90
|
-
it( 'addEventAction()
|
|
91
|
-
process.env.OUTPUT_TRACE_LOCAL_ON = '
|
|
113
|
+
it( 'addEventAction() serializes error details before emitting', async () => {
|
|
114
|
+
process.env.OUTPUT_TRACE_LOCAL_ON = 'on';
|
|
92
115
|
const { init, addEventAction } = await loadTraceEngine();
|
|
93
116
|
await init();
|
|
117
|
+
const error = new TypeError( 'step failed' );
|
|
94
118
|
|
|
95
|
-
addEventAction( '
|
|
96
|
-
kind: 'step', name: '
|
|
97
|
-
traceInfo
|
|
119
|
+
addEventAction( 'error', {
|
|
120
|
+
kind: 'step', name: 'S', id: '3', parentId: 'p3', details: error,
|
|
121
|
+
traceInfo
|
|
122
|
+
} );
|
|
123
|
+
|
|
124
|
+
expect( serializeErrorMock ).toHaveBeenCalledWith( error );
|
|
125
|
+
expect( localExecMock.mock.calls[0][0].entry.details ).toEqual( {
|
|
126
|
+
name: 'TypeError',
|
|
127
|
+
message: 'step failed'
|
|
98
128
|
} );
|
|
99
|
-
expect( localExecMock ).not.toHaveBeenCalled();
|
|
100
129
|
} );
|
|
101
130
|
|
|
102
|
-
it( 'addEventAction()
|
|
131
|
+
it( 'addEventAction() does not emit when traceInfo is absent', async () => {
|
|
103
132
|
process.env.OUTPUT_TRACE_LOCAL_ON = '1';
|
|
104
133
|
const { init, addEventAction } = await loadTraceEngine();
|
|
105
134
|
await init();
|
|
106
135
|
|
|
107
136
|
addEventAction( 'start', {
|
|
108
|
-
kind: '
|
|
109
|
-
traceInfo
|
|
110
|
-
} );
|
|
111
|
-
expect( localExecMock ).toHaveBeenCalledTimes( 1 );
|
|
112
|
-
expect( localExecMock.mock.calls[0][0].entry ).toMatchObject( {
|
|
113
|
-
kind: 'internal_step',
|
|
114
|
-
name: 'Internal',
|
|
115
|
-
id: '1',
|
|
116
|
-
parentId: 'p'
|
|
137
|
+
kind: 'step', name: 'X', id: '1', parentId: 'p', details: {},
|
|
138
|
+
traceInfo: undefined
|
|
117
139
|
} );
|
|
140
|
+
expect( localExecMock ).not.toHaveBeenCalled();
|
|
118
141
|
} );
|
|
119
142
|
|
|
120
143
|
it( 'addEventActionWithContext() uses storage when available', async () => {
|
|
@@ -135,13 +158,34 @@ describe( 'tracing/trace_engine', () => {
|
|
|
135
158
|
expect( payload.entry.action ).toBe( 'tick' );
|
|
136
159
|
} );
|
|
137
160
|
|
|
138
|
-
it( 'addEventActionWithContext()
|
|
161
|
+
it( 'addEventActionWithContext() unwraps transparent errors without mutating options', async () => {
|
|
139
162
|
process.env.OUTPUT_TRACE_LOCAL_ON = 'true';
|
|
140
|
-
const addAttributeMock = vi.fn();
|
|
141
163
|
storageLoadMock.mockReturnValue( {
|
|
142
164
|
parentId: 'ctx-p',
|
|
143
|
-
traceInfo
|
|
144
|
-
|
|
165
|
+
traceInfo
|
|
166
|
+
} );
|
|
167
|
+
const { init, addEventActionWithContext } = await loadTraceEngine();
|
|
168
|
+
const { TransparentFatalError } = await import( '#errors' );
|
|
169
|
+
await init();
|
|
170
|
+
|
|
171
|
+
const cause = new TypeError( 'provider rejected request' );
|
|
172
|
+
const error = new TransparentFatalError( cause );
|
|
173
|
+
const options = { kind: 'llm', name: 'generate', id: 'llm-1', details: error };
|
|
174
|
+
addEventActionWithContext( 'error', options );
|
|
175
|
+
|
|
176
|
+
expect( options.details ).toBe( error );
|
|
177
|
+
expect( serializeErrorMock ).toHaveBeenCalledWith( cause );
|
|
178
|
+
expect( localExecMock.mock.calls[0][0].entry.details ).toEqual( {
|
|
179
|
+
name: 'TypeError',
|
|
180
|
+
message: 'provider rejected request'
|
|
181
|
+
} );
|
|
182
|
+
} );
|
|
183
|
+
|
|
184
|
+
it( 'addEventActionWithContext() emits validated ADD_ATTR trace entries', async () => {
|
|
185
|
+
process.env.OUTPUT_TRACE_LOCAL_ON = 'true';
|
|
186
|
+
storageLoadMock.mockReturnValue( {
|
|
187
|
+
parentId: 'ctx-p',
|
|
188
|
+
traceInfo
|
|
145
189
|
} );
|
|
146
190
|
const { init, addEventActionWithContext } = await loadTraceEngine();
|
|
147
191
|
const { EventAction } = await import( './trace_consts.js' );
|
|
@@ -151,8 +195,6 @@ describe( 'tracing/trace_engine', () => {
|
|
|
151
195
|
const attribute = new Attribute.HTTPRequestCount( 'https://example.test', 'req-1' );
|
|
152
196
|
addEventActionWithContext( EventAction.ADD_ATTR, { kind: 'http', name: 'request', id: 'req-1', details: attribute } );
|
|
153
197
|
|
|
154
|
-
expect( addAttributeMock ).toHaveBeenCalledTimes( 1 );
|
|
155
|
-
expect( addAttributeMock ).toHaveBeenCalledWith( attribute );
|
|
156
198
|
expect( localExecMock ).toHaveBeenCalledTimes( 1 );
|
|
157
199
|
expect( localExecMock.mock.calls[0][0] ).toEqual( {
|
|
158
200
|
traceInfo,
|
|
@@ -170,11 +212,9 @@ describe( 'tracing/trace_engine', () => {
|
|
|
170
212
|
|
|
171
213
|
it( 'addEventActionWithContext() throws on invalid ADD_ATTR signal payloads', async () => {
|
|
172
214
|
process.env.OUTPUT_TRACE_LOCAL_ON = 'true';
|
|
173
|
-
const addAttributeMock = vi.fn();
|
|
174
215
|
storageLoadMock.mockReturnValue( {
|
|
175
216
|
parentId: 'ctx-p',
|
|
176
|
-
traceInfo
|
|
177
|
-
addAttribute: addAttributeMock
|
|
217
|
+
traceInfo
|
|
178
218
|
} );
|
|
179
219
|
const { init, addEventActionWithContext } = await loadTraceEngine();
|
|
180
220
|
const { EventAction } = await import( './trace_consts.js' );
|
|
@@ -186,7 +226,6 @@ describe( 'tracing/trace_engine', () => {
|
|
|
186
226
|
{ kind: 'http', name: 'request', id: 'req-1', details: invalidAttribute }
|
|
187
227
|
) ).toThrow( /not a BaseAttribute instance/ );
|
|
188
228
|
|
|
189
|
-
expect( addAttributeMock ).not.toHaveBeenCalled();
|
|
190
229
|
expect( localExecMock ).not.toHaveBeenCalled();
|
|
191
230
|
} );
|
|
192
231
|
|
|
@@ -38,7 +38,7 @@ export class CatalogJob {
|
|
|
38
38
|
throw e;
|
|
39
39
|
}
|
|
40
40
|
if ( !( e instanceof WorkflowNotFoundError ) ) {
|
|
41
|
-
log.warn( 'Error describing catalog workflow', { error: e } );
|
|
41
|
+
log.warn( 'Error describing catalog workflow', { error: e.message } );
|
|
42
42
|
}
|
|
43
43
|
return false;
|
|
44
44
|
} );
|
|
@@ -52,7 +52,7 @@ export class CatalogJob {
|
|
|
52
52
|
throw e;
|
|
53
53
|
}
|
|
54
54
|
if ( !( e instanceof WorkflowNotFoundError ) ) {
|
|
55
|
-
log.warn( 'Error describing catalog workflow', { error: e } );
|
|
55
|
+
log.warn( 'Error describing catalog workflow', { error: e.message } );
|
|
56
56
|
}
|
|
57
57
|
return false;
|
|
58
58
|
} );
|
|
@@ -66,7 +66,7 @@ export class CatalogJob {
|
|
|
66
66
|
throw e;
|
|
67
67
|
}
|
|
68
68
|
if ( !( e instanceof WorkflowNotFoundError ) ) {
|
|
69
|
-
log.warn( 'Error completing previous catalog workflow', { error: e } );
|
|
69
|
+
log.warn( 'Error completing previous catalog workflow', { error: e.message } );
|
|
70
70
|
}
|
|
71
71
|
} );
|
|
72
72
|
};
|
|
@@ -121,7 +121,7 @@ describe( 'CatalogJob', () => {
|
|
|
121
121
|
|
|
122
122
|
await job.run();
|
|
123
123
|
|
|
124
|
-
expect( mockLog.warn ).toHaveBeenCalledWith( 'Error completing previous catalog workflow', { error: completeError } );
|
|
124
|
+
expect( mockLog.warn ).toHaveBeenCalledWith( 'Error completing previous catalog workflow', { error: completeError.message } );
|
|
125
125
|
expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', expect.any( Object ) );
|
|
126
126
|
expect( job.error ).toBeNull();
|
|
127
127
|
} );
|
package/src/worker/configs.js
CHANGED
|
@@ -36,8 +36,6 @@ const envVarSchema = z.object( {
|
|
|
36
36
|
OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 2 * 60 * 1000 ) ), // 2min
|
|
37
37
|
// Whether to send activity heartbeats (enabled by default)
|
|
38
38
|
OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: z.transform( v => v === undefined ? true : isStringboolTrue( v ) ),
|
|
39
|
-
// Time to allow for hooks to flush before shutdown
|
|
40
|
-
OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 3000 ) ),
|
|
41
39
|
// Set temporal worker shutdown force time
|
|
42
40
|
TEMPORAL_SHUTDOWN_FORCE_TIME: durationSchema,
|
|
43
41
|
// Set temporal worker shutdown grace time
|
|
@@ -70,7 +68,6 @@ export const catalogId = envVars.OUTPUT_CATALOG_ID;
|
|
|
70
68
|
export const workerTelemetryIntervalMs = envVars.OUTPUT_WORKER_TELEMETRY_INTERVAL_MS;
|
|
71
69
|
export const activityHeartbeatIntervalMs = envVars.OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS;
|
|
72
70
|
export const activityHeartbeatEnabled = envVars.OUTPUT_ACTIVITY_HEARTBEAT_ENABLED;
|
|
73
|
-
export const processFailureShutdownDelay = envVars.OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY;
|
|
74
71
|
export const shutdownForceTime = envVars.TEMPORAL_SHUTDOWN_FORCE_TIME;
|
|
75
72
|
export const shutdownGraceTime = envVars.TEMPORAL_SHUTDOWN_GRACE_TIME;
|
|
76
73
|
export const grpcProxy = envVars.TEMPORAL_GRPC_PROXY;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { mainEventBus } from '#bus';
|
|
2
2
|
import { ACTIVITY_LOGGER_SYMBOL, BusEventType } from '#consts';
|
|
3
3
|
import { activityInfo as activityInfoFn } from '@temporalio/activity';
|
|
4
4
|
import { assignImmutableProperty } from '#helpers/object';
|
|
@@ -9,6 +9,6 @@ import { assignImmutableProperty } from '#helpers/object';
|
|
|
9
9
|
export const bindGlobalFunctions = () => {
|
|
10
10
|
/** Defines the activity logger function, accessible in activity context via logger interface */
|
|
11
11
|
assignImmutableProperty( globalThis, ACTIVITY_LOGGER_SYMBOL, ( { level, message, metadata } ) =>
|
|
12
|
-
|
|
12
|
+
mainEventBus.emit( BusEventType.ACTIVITY_LOG, { level, message, metadata, activityInfo: activityInfoFn() } )
|
|
13
13
|
);
|
|
14
14
|
};
|
|
@@ -10,7 +10,7 @@ const {
|
|
|
10
10
|
}
|
|
11
11
|
} ) );
|
|
12
12
|
|
|
13
|
-
const
|
|
13
|
+
const mainEventBusMock = vi.hoisted( () => ( {
|
|
14
14
|
emit: vi.fn()
|
|
15
15
|
} ) );
|
|
16
16
|
const activityInfoMock = vi.hoisted( () => vi.fn( () => ( {
|
|
@@ -19,7 +19,7 @@ const activityInfoMock = vi.hoisted( () => vi.fn( () => ( {
|
|
|
19
19
|
} ) ) );
|
|
20
20
|
|
|
21
21
|
vi.mock( '#consts', () => ( { ACTIVITY_LOGGER_SYMBOL, BusEventType } ) );
|
|
22
|
-
vi.mock( '#bus', () => ( {
|
|
22
|
+
vi.mock( '#bus', () => ( { mainEventBus: mainEventBusMock } ) );
|
|
23
23
|
vi.mock( '@temporalio/activity', () => ( {
|
|
24
24
|
activityInfo: activityInfoMock
|
|
25
25
|
} ) );
|
|
@@ -42,7 +42,7 @@ describe( 'worker/global_functions', () => {
|
|
|
42
42
|
} );
|
|
43
43
|
|
|
44
44
|
expect( activityInfoMock ).toHaveBeenCalledTimes( 1 );
|
|
45
|
-
expect(
|
|
45
|
+
expect( mainEventBusMock.emit ).toHaveBeenCalledWith( BusEventType.ACTIVITY_LOG, {
|
|
46
46
|
level: 'debug',
|
|
47
47
|
message: 'activity detail',
|
|
48
48
|
metadata,
|
package/src/worker/index.js
CHANGED
|
@@ -12,13 +12,16 @@ import { initInterceptors } from './interceptors/index.js';
|
|
|
12
12
|
import { createChildLogger } from '#logger';
|
|
13
13
|
import { setupInterruptionHandler } from './interruption.js';
|
|
14
14
|
import { CatalogJob } from './catalog_workflow/catalog_job.js';
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
15
|
+
import { mainEventBus } from '#bus';
|
|
16
|
+
import { flushPendingHooks } from '#hooks/pending_hooks';
|
|
17
17
|
import { BusEventType } from '#consts';
|
|
18
18
|
import { setupTelemetry } from './telemetry.js';
|
|
19
19
|
import { TemporalConnectionMonitor } from './connection_monitor.js';
|
|
20
20
|
import { bindGlobalFunctions } from './global_functions.js';
|
|
21
|
+
import { setupClientConfig } from '#temporal/client';
|
|
21
22
|
import { runOnce } from '#helpers/function';
|
|
23
|
+
import { serializeError } from '#helpers/error_serializer';
|
|
24
|
+
import { setupTemporalLogger } from './temporal_logger.js';
|
|
22
25
|
|
|
23
26
|
import './log_hooks.js';
|
|
24
27
|
|
|
@@ -51,6 +54,9 @@ const state = {
|
|
|
51
54
|
const callerDir = process.argv[2];
|
|
52
55
|
|
|
53
56
|
const execute = async () => {
|
|
57
|
+
log.info( 'Setup Temporal Logger' );
|
|
58
|
+
setupTemporalLogger();
|
|
59
|
+
|
|
54
60
|
log.info( 'Loading config...', { callerDir } );
|
|
55
61
|
await loadHooks( callerDir );
|
|
56
62
|
|
|
@@ -60,8 +66,7 @@ const execute = async () => {
|
|
|
60
66
|
log.info( 'Loading activities...', { callerDir } );
|
|
61
67
|
const { activities } = await loadActivities( callerDir, workflows );
|
|
62
68
|
|
|
63
|
-
|
|
64
|
-
bootstrapFetchProxy();
|
|
69
|
+
mainEventBus.emit( BusEventType.WORKER_BEFORE_START );
|
|
65
70
|
|
|
66
71
|
log.info( 'Initializing tracing...' );
|
|
67
72
|
await initTracing();
|
|
@@ -82,6 +87,9 @@ const execute = async () => {
|
|
|
82
87
|
|
|
83
88
|
state.connection = await NativeConnection.connect( { address, tls: Boolean( apiKey ), apiKey, proxy } );
|
|
84
89
|
|
|
90
|
+
// add configs for the /temporal endpoint
|
|
91
|
+
setupClientConfig( { connection: state.connection, namespace } );
|
|
92
|
+
|
|
85
93
|
log.info( 'Creating connection monitor...' );
|
|
86
94
|
state.connectionMonitor = new TemporalConnectionMonitor( state.connection );
|
|
87
95
|
|
|
@@ -214,13 +222,18 @@ execute()
|
|
|
214
222
|
.catch( e => log.warn( 'Connection close error', { error: e.message } ) );
|
|
215
223
|
}
|
|
216
224
|
} )
|
|
217
|
-
.then( () =>
|
|
218
|
-
|
|
219
|
-
|
|
225
|
+
.then( async () => {
|
|
226
|
+
log.info( 'Flushing hook callbacks...' );
|
|
227
|
+
await flushPendingHooks();
|
|
228
|
+
log.info( 'Bye' );
|
|
229
|
+
} )
|
|
230
|
+
.catch( async error => {
|
|
231
|
+
log.error( 'Fatal error', { error: serializeError( error ) } );
|
|
220
232
|
|
|
221
|
-
|
|
233
|
+
mainEventBus.emit( BusEventType.RUNTIME_ERROR, { error } );
|
|
222
234
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
235
|
+
log.info( 'Flushing hook callbacks...' );
|
|
236
|
+
await flushPendingHooks();
|
|
237
|
+
log.info( 'Exiting...' );
|
|
238
|
+
setTimeout( () => process.exit( 1 ) );
|
|
226
239
|
} );
|
package/src/worker/index.spec.js
CHANGED
|
@@ -6,14 +6,18 @@ const {
|
|
|
6
6
|
configValues,
|
|
7
7
|
connectionMonitorInstance,
|
|
8
8
|
createCatalogMock,
|
|
9
|
+
flushPendingHooksMock,
|
|
9
10
|
initInterceptorsMock,
|
|
10
|
-
|
|
11
|
+
mainEventBusMock,
|
|
11
12
|
mockConnection,
|
|
12
13
|
mockLog,
|
|
13
14
|
mockWorker,
|
|
14
15
|
promises,
|
|
15
16
|
resetPromises,
|
|
17
|
+
serializeErrorMock,
|
|
18
|
+
setupClientConfigMock,
|
|
16
19
|
setupInterruptionHandlerMock,
|
|
20
|
+
setupTemporalLoggerMock,
|
|
17
21
|
setupTelemetryMock
|
|
18
22
|
} = vi.hoisted( () => {
|
|
19
23
|
const createDeferred = () => {
|
|
@@ -46,7 +50,6 @@ const {
|
|
|
46
50
|
maxConcurrentActivityTaskPolls: 5,
|
|
47
51
|
maxConcurrentWorkflowTaskPolls: 5,
|
|
48
52
|
workerTuner: undefined,
|
|
49
|
-
processFailureShutdownDelay: 0,
|
|
50
53
|
shutdownForceTime: undefined,
|
|
51
54
|
shutdownGraceTime: undefined
|
|
52
55
|
};
|
|
@@ -105,30 +108,36 @@ const {
|
|
|
105
108
|
connectionMonitorInstance,
|
|
106
109
|
createCatalogMock: vi.fn().mockReturnValue( { workflows: [], activities: {} } ),
|
|
107
110
|
createWorkflowsEntryPointMock: vi.fn().mockReturnValue( '/fake/workflows/path.js' ),
|
|
111
|
+
flushPendingHooksMock: vi.fn().mockResolvedValue( undefined ),
|
|
108
112
|
hashSourceCodeMock: vi.fn().mockResolvedValue( 'catalog-hash' ),
|
|
109
113
|
initInterceptorsMock: vi.fn().mockReturnValue( [] ),
|
|
110
114
|
loadActivitiesMock: vi.fn().mockResolvedValue( {} ),
|
|
111
115
|
loadHooksMock: vi.fn().mockResolvedValue( undefined ),
|
|
112
116
|
loadWorkflowsMock: vi.fn().mockResolvedValue( [] ),
|
|
113
|
-
|
|
117
|
+
mainEventBusMock: { emit: vi.fn(), on: vi.fn() },
|
|
114
118
|
mockConnection: { close: vi.fn().mockResolvedValue( undefined ) },
|
|
115
119
|
mockLog: { error: vi.fn(), info: vi.fn(), warn: vi.fn() },
|
|
116
120
|
mockWorker,
|
|
117
121
|
promises,
|
|
118
122
|
resetPromises,
|
|
123
|
+
serializeErrorMock: vi.fn( error => ( {
|
|
124
|
+
name: error.name,
|
|
125
|
+
message: error.message,
|
|
126
|
+
stack: error.stack
|
|
127
|
+
} ) ),
|
|
128
|
+
setupClientConfigMock: vi.fn(),
|
|
119
129
|
setupInterruptionHandlerMock: vi.fn(),
|
|
130
|
+
setupTemporalLoggerMock: vi.fn(),
|
|
120
131
|
setupTelemetryMock: vi.fn()
|
|
121
132
|
};
|
|
122
133
|
} );
|
|
123
134
|
|
|
124
135
|
vi.mock( '#logger', () => ( { createChildLogger: () => mockLog } ) );
|
|
125
|
-
vi.mock( '#
|
|
126
|
-
const actual = await importOriginal();
|
|
127
|
-
return { ...actual };
|
|
128
|
-
} );
|
|
136
|
+
vi.mock( '#helpers/error_serializer', () => ( { serializeError: serializeErrorMock } ) );
|
|
129
137
|
const initTracing = vi.fn().mockResolvedValue( undefined );
|
|
130
138
|
vi.mock( '#tracing', () => ( { init: initTracing } ) );
|
|
131
|
-
vi.mock( '#bus', () => ( {
|
|
139
|
+
vi.mock( '#bus', () => ( { mainEventBus: mainEventBusMock } ) );
|
|
140
|
+
vi.mock( '#hooks/pending_hooks', () => ( { flushPendingHooks: flushPendingHooksMock } ) );
|
|
132
141
|
|
|
133
142
|
const loadWorkflowsMock = vi.fn().mockResolvedValue( { workflows: [], entrypoint: '/fake/workflows/path.js' } );
|
|
134
143
|
const loadActivitiesMock = vi.fn().mockResolvedValue( { activities: {} } );
|
|
@@ -145,7 +154,6 @@ vi.mock( './sinks.js', () => ( { sinks: {} } ) );
|
|
|
145
154
|
vi.mock( './catalog_workflow/index.js', () => ( { createCatalog: createCatalogMock } ) );
|
|
146
155
|
vi.mock( './bundler_options.js', () => ( { webpackConfigHook: vi.fn() } ) );
|
|
147
156
|
vi.mock( './interceptors/index.js', () => ( { initInterceptors: initInterceptorsMock } ) );
|
|
148
|
-
vi.mock( './proxy.js', () => ( { bootstrapFetchProxy: vi.fn() } ) );
|
|
149
157
|
vi.mock( './telemetry.js', () => ( { setupTelemetry: setupTelemetryMock } ) );
|
|
150
158
|
vi.mock( './interruption.js', () => ( { setupInterruptionHandler: setupInterruptionHandlerMock } ) );
|
|
151
159
|
vi.mock( './global_functions.js', () => ( { bindGlobalFunctions: bindGlobalFunctionsMock } ) );
|
|
@@ -160,6 +168,8 @@ vi.mock( './catalog_workflow/catalog_job.js', () => ( {
|
|
|
160
168
|
} )
|
|
161
169
|
} ) );
|
|
162
170
|
vi.mock( './log_hooks.js', () => ( {} ) );
|
|
171
|
+
vi.mock( './temporal_logger.js', () => ( { setupTemporalLogger: setupTemporalLoggerMock } ) );
|
|
172
|
+
vi.mock( '#temporal/client', () => ( { setupClientConfig: setupClientConfigMock } ) );
|
|
163
173
|
vi.mock( '@temporalio/worker', () => ( {
|
|
164
174
|
NativeConnection: { connect: vi.fn().mockResolvedValue( mockConnection ) },
|
|
165
175
|
Worker: { create: vi.fn().mockResolvedValue( mockWorker ) }
|
|
@@ -216,6 +226,7 @@ describe( 'worker/index', () => {
|
|
|
216
226
|
await vi.waitFor( () => expect( Worker.create ).toHaveBeenCalled() );
|
|
217
227
|
|
|
218
228
|
expect( loadHooksMock ).toHaveBeenCalledWith( '/test/caller/dir' );
|
|
229
|
+
expect( setupTemporalLoggerMock ).toHaveBeenCalledOnce();
|
|
219
230
|
expect( loadWorkflowsMock ).toHaveBeenCalledWith( '/test/caller/dir' );
|
|
220
231
|
expect( loadActivitiesMock ).toHaveBeenCalledWith( '/test/caller/dir', [] );
|
|
221
232
|
expect( initTracing ).toHaveBeenCalled();
|
|
@@ -227,6 +238,10 @@ describe( 'worker/index', () => {
|
|
|
227
238
|
apiKey: undefined,
|
|
228
239
|
proxy: undefined
|
|
229
240
|
} );
|
|
241
|
+
expect( setupClientConfigMock ).toHaveBeenCalledWith( {
|
|
242
|
+
connection: mockConnection,
|
|
243
|
+
namespace: configValues.namespace
|
|
244
|
+
} );
|
|
230
245
|
expect( bindGlobalFunctionsMock ).toHaveBeenCalledTimes( 1 );
|
|
231
246
|
expect( bindGlobalFunctionsMock.mock.invocationCallOrder[0] ).toBeLessThan(
|
|
232
247
|
NativeConnection.connect.mock.invocationCallOrder[0]
|
|
@@ -262,6 +277,7 @@ describe( 'worker/index', () => {
|
|
|
262
277
|
expect( catalogJobInstance.run ).toHaveBeenCalled();
|
|
263
278
|
|
|
264
279
|
await settleWorker();
|
|
280
|
+
await vi.waitFor( () => expect( flushPendingHooksMock ).toHaveBeenCalledOnce() );
|
|
265
281
|
expect( mockLog.info ).toHaveBeenCalledWith( 'Bye' );
|
|
266
282
|
} );
|
|
267
283
|
|
|
@@ -354,11 +370,15 @@ describe( 'worker/index', () => {
|
|
|
354
370
|
|
|
355
371
|
await vi.waitFor( () => {
|
|
356
372
|
expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.objectContaining( {
|
|
357
|
-
error: 'Big Failure'
|
|
373
|
+
error: expect.objectContaining( { message: 'Big Failure' } )
|
|
358
374
|
} ) );
|
|
359
375
|
} );
|
|
360
|
-
expect(
|
|
376
|
+
expect( serializeErrorMock ).toHaveBeenCalledWith( error );
|
|
377
|
+
expect( mainEventBusMock.emit ).toHaveBeenCalledWith( expect.any( String ), { error } );
|
|
361
378
|
await vi.waitFor( () => expect( exitMock ).toHaveBeenCalledWith( 1 ) );
|
|
379
|
+
expect( flushPendingHooksMock ).toHaveBeenCalledOnce();
|
|
380
|
+
expect( mainEventBusMock.emit.mock.invocationCallOrder[0] ).toBeLessThan( flushPendingHooksMock.mock.invocationCallOrder[0] );
|
|
381
|
+
expect( flushPendingHooksMock.mock.invocationCallOrder[0] ).toBeLessThan( exitMock.mock.invocationCallOrder[0] );
|
|
362
382
|
} );
|
|
363
383
|
|
|
364
384
|
it( 'throws connection monitor errors after graceful shutdown', async () => {
|
|
@@ -373,12 +393,12 @@ describe( 'worker/index', () => {
|
|
|
373
393
|
|
|
374
394
|
await vi.waitFor( () => {
|
|
375
395
|
expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.objectContaining( {
|
|
376
|
-
error: 'connection lost'
|
|
396
|
+
error: expect.objectContaining( { message: 'connection lost' } )
|
|
377
397
|
} ) );
|
|
378
398
|
} );
|
|
379
399
|
expect( mockWorker.shutdown ).toHaveBeenCalledOnce();
|
|
380
400
|
expect( catalogJobInstance.interrupt ).toHaveBeenCalledOnce();
|
|
381
|
-
expect(
|
|
401
|
+
expect( mainEventBusMock.emit ).toHaveBeenCalledWith( expect.any( String ), { error } );
|
|
382
402
|
} );
|
|
383
403
|
|
|
384
404
|
it( 'throws catalog job errors after graceful shutdown', async () => {
|
|
@@ -393,12 +413,12 @@ describe( 'worker/index', () => {
|
|
|
393
413
|
|
|
394
414
|
await vi.waitFor( () => {
|
|
395
415
|
expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.objectContaining( {
|
|
396
|
-
error: 'catalog failed'
|
|
416
|
+
error: expect.objectContaining( { message: 'catalog failed' } )
|
|
397
417
|
} ) );
|
|
398
418
|
} );
|
|
399
419
|
expect( mockWorker.shutdown ).toHaveBeenCalledOnce();
|
|
400
420
|
expect( connectionMonitorInstance.stop ).toHaveBeenCalledOnce();
|
|
401
|
-
expect(
|
|
421
|
+
expect( mainEventBusMock.emit ).toHaveBeenCalledWith( expect.any( String ), { error } );
|
|
402
422
|
} );
|
|
403
423
|
|
|
404
424
|
it( 'cleans up partial startup failures after connecting', async () => {
|
|
@@ -413,7 +433,7 @@ describe( 'worker/index', () => {
|
|
|
413
433
|
expect( catalogJobInstance.interrupt ).not.toHaveBeenCalled();
|
|
414
434
|
await vi.waitFor( () => {
|
|
415
435
|
expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.objectContaining( {
|
|
416
|
-
error: 'worker create failed'
|
|
436
|
+
error: expect.objectContaining( { message: 'worker create failed' } )
|
|
417
437
|
} ) );
|
|
418
438
|
} );
|
|
419
439
|
} );
|
|
@@ -1,12 +1,14 @@
|
|
|
1
|
-
import { Context, activityInfo as activityInfoFn } from '@temporalio/activity';
|
|
1
|
+
import { Context, activityInfo as activityInfoFn, ApplicationFailure } from '@temporalio/activity';
|
|
2
|
+
import { CompleteAsyncError, TemporalFailure } from '@temporalio/common';
|
|
2
3
|
import { Storage } from '#async_storage';
|
|
3
4
|
import * as Tracing from '#tracing';
|
|
4
5
|
import { headersToObject } from './headers.js';
|
|
5
|
-
import {
|
|
6
|
+
import { ActivitySpecialOutput, BusEventType, METADATA_ACCESS_SYMBOL } from '#consts';
|
|
6
7
|
import { activityHeartbeatEnabled, activityHeartbeatIntervalMs } from '../configs.js';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
8
|
+
import { mainEventBus } from '#bus';
|
|
9
|
+
import { inheritsFromAnyNamedType } from '#helpers/errors';
|
|
10
|
+
import { serializeError } from '#helpers/error_serializer';
|
|
11
|
+
import { FatalError, TransparentFatalError } from '#errors';
|
|
10
12
|
|
|
11
13
|
/*
|
|
12
14
|
This interceptor wraps every activity execution with cross-cutting concerns:
|
|
@@ -51,12 +53,9 @@ export class ActivityExecutionInterceptor {
|
|
|
51
53
|
}
|
|
52
54
|
|
|
53
55
|
const state = {
|
|
54
|
-
heartbeat: null
|
|
55
|
-
attributes: []
|
|
56
|
+
heartbeat: null
|
|
56
57
|
};
|
|
57
58
|
|
|
58
|
-
const addAttribute = attribute => state.attributes.push( attribute );
|
|
59
|
-
|
|
60
59
|
// Adds context accessible information
|
|
61
60
|
const storageContext = {
|
|
62
61
|
parentId: activityId,
|
|
@@ -64,11 +63,10 @@ export class ActivityExecutionInterceptor {
|
|
|
64
63
|
activityInfo,
|
|
65
64
|
workflowDetails,
|
|
66
65
|
workflowFilename,
|
|
67
|
-
traceInfo
|
|
68
|
-
addAttribute
|
|
66
|
+
traceInfo
|
|
69
67
|
};
|
|
70
68
|
|
|
71
|
-
|
|
69
|
+
mainEventBus.emit( BusEventType.ACTIVITY_START, { activityInfo, workflowDetails, outputActivityKind } );
|
|
72
70
|
Tracing.addEventStart( { id: activityId, name: activityType, kind: outputActivityKind, parentId: runId, details: input.args[0], traceInfo } );
|
|
73
71
|
|
|
74
72
|
try {
|
|
@@ -77,19 +75,34 @@ export class ActivityExecutionInterceptor {
|
|
|
77
75
|
|
|
78
76
|
const output = await Storage.runWithContext( async _ => next( input ), storageContext );
|
|
79
77
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
messageBus.emit( BusEventType.ACTIVITY_END, { activityInfo, aggregations, workflowDetails, outputActivityKind } );
|
|
78
|
+
mainEventBus.emit( BusEventType.ACTIVITY_END, { activityInfo, workflowDetails, outputActivityKind } );
|
|
83
79
|
Tracing.addEventEnd( { id: activityId, details: output, traceInfo } );
|
|
84
80
|
|
|
85
|
-
return
|
|
81
|
+
return output;
|
|
86
82
|
|
|
87
83
|
} catch ( error ) {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
84
|
+
// Record async completion handoff as a trace end without emitting a bus event.
|
|
85
|
+
if ( error instanceof CompleteAsyncError ) {
|
|
86
|
+
Tracing.addEventEnd( { id: activityId, details: ActivitySpecialOutput.ASYNC_HANDOFF, traceInfo } );
|
|
87
|
+
throw error;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const unwrappedError = error instanceof TransparentFatalError ? error.cause : error;
|
|
91
|
+
|
|
92
|
+
Tracing.addEventError( { id: activityId, details: unwrappedError, traceInfo } );
|
|
93
|
+
mainEventBus.emit( BusEventType.ACTIVITY_ERROR, { activityInfo, workflowDetails, outputActivityKind, error: unwrappedError } );
|
|
94
|
+
|
|
95
|
+
// Native Temporal errors are just re-thrown
|
|
96
|
+
if ( error instanceof TemporalFailure ) {
|
|
97
|
+
throw error;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const nonRetryable = error instanceof FatalError || inheritsFromAnyNamedType( error, activityInfo.retryPolicy?.nonRetryableErrorTypes ?? [] );
|
|
101
|
+
throw ApplicationFailure.fromError( unwrappedError, {
|
|
102
|
+
nonRetryable,
|
|
103
|
+
cause: unwrappedError,
|
|
104
|
+
details: [ { error: serializeError( unwrappedError, { dropKeys: [ 'stack' ] } ) } ]
|
|
105
|
+
} );
|
|
93
106
|
} finally {
|
|
94
107
|
clearInterval( state.heartbeat );
|
|
95
108
|
}
|