@outputai/core 0.10.1-dev.b7b2fbe.0 → 0.10.1-next.1070949.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -7
- package/src/bus.js +58 -20
- package/src/bus.spec.js +135 -77
- package/src/consts.js +5 -2
- package/src/errors.d.ts +12 -2
- package/src/errors.js +17 -3
- package/src/errors.spec.js +19 -0
- package/src/helpers/activity_options.js +21 -0
- package/src/helpers/activity_options.spec.js +55 -0
- package/src/helpers/error_serializer.js +236 -0
- package/src/helpers/error_serializer.spec.js +413 -0
- package/src/helpers/errors.js +23 -14
- package/src/helpers/errors.spec.js +31 -75
- package/src/helpers/fetch.js +1 -38
- package/src/helpers/fetch.spec.js +1 -39
- package/src/helpers/function.js +13 -0
- package/src/helpers/function.spec.js +23 -1
- package/src/helpers/object.js +5 -1
- package/src/helpers/object.spec.js +5 -0
- package/src/helpers/redact.js +51 -0
- package/src/helpers/redact.spec.js +66 -0
- package/src/helpers/string.js +10 -0
- package/src/helpers/string.spec.js +36 -1
- package/src/hooks/index.d.ts +76 -78
- package/src/hooks/index.js +57 -49
- package/src/hooks/index.spec.js +104 -105
- package/src/hooks/pending_hooks.js +24 -0
- package/src/hooks/pending_hooks.spec.js +62 -0
- package/src/interface/evaluator.d.ts +10 -9
- package/src/interface/evaluator.js +1 -6
- package/src/interface/evaluator.spec.js +10 -10
- package/src/interface/index.js +2 -1
- package/src/interface/step.d.ts +16 -15
- package/src/interface/step.js +1 -6
- package/src/interface/step.spec.js +10 -10
- package/src/interface/validations/index.js +33 -53
- package/src/interface/validations/index.spec.js +25 -17
- package/src/interface/webhook.js +1 -2
- package/src/interface/webhook.spec.js +25 -19
- package/src/interface/workflow.d.ts +25 -18
- package/src/interface/workflow.js +24 -52
- package/src/interface/workflow.spec.js +109 -108
- package/src/interface/workflow_activity_options.js +2 -5
- package/src/interface/workflow_utils.d.ts +15 -0
- package/src/interface/workflow_utils.js +19 -0
- package/src/interface/workflow_utils.spec.js +82 -1
- package/src/sdk/runtime/events.d.ts +4 -6
- package/src/sdk/runtime/events.js +2 -15
- package/src/sdk/runtime/events.spec.js +14 -92
- package/src/tracing/trace_engine.js +9 -11
- package/src/tracing/trace_engine.spec.js +67 -28
- package/src/worker/catalog_workflow/catalog_job.js +3 -3
- package/src/worker/catalog_workflow/catalog_job.spec.js +1 -1
- package/src/worker/configs.js +0 -3
- package/src/worker/global_functions.js +2 -2
- package/src/worker/global_functions.spec.js +3 -3
- package/src/worker/index.js +20 -11
- package/src/worker/index.spec.js +29 -16
- package/src/worker/interceptors/activity.js +34 -21
- package/src/worker/interceptors/activity.spec.js +77 -172
- package/src/worker/interceptors/workflow.js +53 -11
- package/src/worker/interceptors/workflow.spec.js +176 -49
- package/src/worker/loader/workflows.js +16 -3
- package/src/worker/loader/workflows.spec.js +37 -5
- package/src/worker/log_hooks.js +22 -19
- package/src/worker/log_hooks.spec.js +14 -6
- package/src/worker/sinks.js +8 -8
- package/src/worker/sinks.spec.js +9 -9
- package/src/worker/telemetry.js +2 -1
- package/src/worker/temporal_logger.js +33 -0
- package/src/worker/temporal_logger.spec.js +90 -0
- package/src/helpers/aggregations.js +0 -44
- package/src/helpers/aggregations.spec.js +0 -92
- package/src/tracing/tools/utils.js +0 -42
- package/src/tracing/tools/utils.spec.js +0 -83
- package/src/worker/proxy.js +0 -20
- package/src/worker/proxy.spec.js +0 -51
|
@@ -1,118 +1,40 @@
|
|
|
1
1
|
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
2
|
|
|
3
|
-
const loadMock = vi.hoisted( () => vi.fn() );
|
|
4
3
|
const emitMock = vi.hoisted( () => vi.fn() );
|
|
5
4
|
|
|
6
|
-
vi.mock( '#async_storage', () => ( {
|
|
7
|
-
Storage: { load: loadMock }
|
|
8
|
-
} ) );
|
|
9
|
-
|
|
10
5
|
vi.mock( '#bus', () => ( {
|
|
11
|
-
|
|
6
|
+
stepEventBus: { emit: emitMock }
|
|
12
7
|
} ) );
|
|
13
8
|
|
|
14
9
|
import { Event } from './events.js';
|
|
15
10
|
|
|
16
|
-
// `eventId` stamping is the bus layer's responsibility (see bus.spec.js).
|
|
17
|
-
// Assertions here use `objectContaining` so they don't have to know about that enrichment.
|
|
18
11
|
describe( 'Event.emit', () => {
|
|
19
12
|
beforeEach( () => {
|
|
20
13
|
vi.clearAllMocks();
|
|
21
14
|
} );
|
|
22
15
|
|
|
23
|
-
it( 'forwards
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
workflowType: 'workflow'
|
|
29
|
-
};
|
|
30
|
-
const workflowDetails = {
|
|
31
|
-
workflowId: 'wf-1',
|
|
32
|
-
runId: 'run-1',
|
|
33
|
-
workflowType: 'workflow'
|
|
34
|
-
};
|
|
35
|
-
loadMock.mockReturnValue( {
|
|
36
|
-
activityInfo,
|
|
37
|
-
workflowDetails,
|
|
38
|
-
outputActivityKind: 'step'
|
|
39
|
-
} );
|
|
16
|
+
it( 'forwards class instance payloads unchanged', () => {
|
|
17
|
+
class Payload {
|
|
18
|
+
value = 'test';
|
|
19
|
+
}
|
|
20
|
+
const payload = new Payload();
|
|
40
21
|
|
|
41
|
-
Event.emit( '
|
|
22
|
+
Event.emit( 'test:event', payload );
|
|
42
23
|
|
|
43
|
-
expect( emitMock ).toHaveBeenCalledWith( '
|
|
44
|
-
activityInfo,
|
|
45
|
-
workflowDetails,
|
|
46
|
-
outputActivityKind: 'step',
|
|
47
|
-
modelId: 'gpt-4o'
|
|
48
|
-
} ) );
|
|
24
|
+
expect( emitMock ).toHaveBeenCalledWith( 'sdk:test:event', payload );
|
|
49
25
|
} );
|
|
50
26
|
|
|
51
|
-
it( 'emits
|
|
52
|
-
|
|
27
|
+
it( 'emits plain object payloads on the SDK event channel', () => {
|
|
28
|
+
const payload = { modelId: 'gpt-4o' };
|
|
53
29
|
|
|
54
|
-
Event.emit( '
|
|
30
|
+
Event.emit( 'cost:llm:request', payload );
|
|
55
31
|
|
|
56
|
-
expect( emitMock ).toHaveBeenCalledWith( '
|
|
32
|
+
expect( emitMock ).toHaveBeenCalledWith( 'sdk:cost:llm:request', payload );
|
|
57
33
|
} );
|
|
58
34
|
|
|
59
|
-
it( '
|
|
60
|
-
const activityInfo = {
|
|
61
|
-
activityId: 'act-2',
|
|
62
|
-
activityType: 'step',
|
|
63
|
-
workflowExecution: { workflowId: 'wf-2', runId: 'run-2' },
|
|
64
|
-
workflowType: 'workflow'
|
|
65
|
-
};
|
|
66
|
-
const workflowDetails = {
|
|
67
|
-
workflowId: 'wf-2',
|
|
68
|
-
runId: 'run-2',
|
|
69
|
-
workflowType: 'workflow'
|
|
70
|
-
};
|
|
71
|
-
loadMock.mockReturnValue( {
|
|
72
|
-
activityInfo,
|
|
73
|
-
workflowDetails,
|
|
74
|
-
outputActivityKind: 'step'
|
|
75
|
-
} );
|
|
76
|
-
|
|
35
|
+
it( 'forwards a missing payload as undefined', () => {
|
|
77
36
|
Event.emit( 'lifecycle:start' );
|
|
78
37
|
|
|
79
|
-
expect( emitMock ).toHaveBeenCalledWith( '
|
|
80
|
-
activityInfo,
|
|
81
|
-
workflowDetails,
|
|
82
|
-
outputActivityKind: 'step'
|
|
83
|
-
} ) );
|
|
84
|
-
} );
|
|
85
|
-
|
|
86
|
-
it( 'does not let payload override activityInfo, workflowDetails, or outputActivityKind', () => {
|
|
87
|
-
const activityInfo = {
|
|
88
|
-
activityId: 'act-3',
|
|
89
|
-
activityType: 'step',
|
|
90
|
-
workflowExecution: { workflowId: 'wf-3', runId: 'run-3' },
|
|
91
|
-
workflowType: 'workflow'
|
|
92
|
-
};
|
|
93
|
-
const workflowDetails = {
|
|
94
|
-
workflowId: 'wf-3',
|
|
95
|
-
runId: 'run-3',
|
|
96
|
-
workflowType: 'workflow'
|
|
97
|
-
};
|
|
98
|
-
loadMock.mockReturnValue( {
|
|
99
|
-
activityInfo,
|
|
100
|
-
workflowDetails,
|
|
101
|
-
outputActivityKind: 'step'
|
|
102
|
-
} );
|
|
103
|
-
|
|
104
|
-
Event.emit( 'cost:http:request', {
|
|
105
|
-
activityInfo: { activityId: 'should-be-overridden' },
|
|
106
|
-
workflowDetails: { workflowId: 'should-be-overridden' },
|
|
107
|
-
outputActivityKind: 'should-be-overridden',
|
|
108
|
-
url: 'https://example.com'
|
|
109
|
-
} );
|
|
110
|
-
|
|
111
|
-
expect( emitMock ).toHaveBeenCalledWith( 'external:cost:http:request', expect.objectContaining( {
|
|
112
|
-
activityInfo,
|
|
113
|
-
workflowDetails,
|
|
114
|
-
outputActivityKind: 'step',
|
|
115
|
-
url: 'https://example.com'
|
|
116
|
-
} ) );
|
|
38
|
+
expect( emitMock ).toHaveBeenCalledWith( 'sdk:lifecycle:start', undefined );
|
|
117
39
|
} );
|
|
118
40
|
} );
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { Storage } from '#async_storage';
|
|
2
2
|
import { EventEmitter } from 'node:events';
|
|
3
|
-
import { serializeError } from '
|
|
3
|
+
import { serializeError } from '#helpers/error_serializer';
|
|
4
4
|
import { isStringboolTrue } from '#helpers/string';
|
|
5
5
|
import * as localProcessor from './processors/local/index.js';
|
|
6
6
|
import * as s3Processor from './processors/s3/index.js';
|
|
7
7
|
import { createChildLogger } from '#logger';
|
|
8
8
|
import { EventAction } from './trace_consts.js';
|
|
9
9
|
import { BaseAttribute } from './trace_attribute.js';
|
|
10
|
+
import { TransparentFatalError } from '#errors';
|
|
10
11
|
|
|
11
12
|
const log = createChildLogger( 'Tracing' );
|
|
12
13
|
|
|
@@ -51,7 +52,7 @@ export const init = async () => {
|
|
|
51
52
|
try {
|
|
52
53
|
await p.exec( ...args );
|
|
53
54
|
} catch ( error ) {
|
|
54
|
-
log.error( 'Processor execution error', { processor: p.name, error: error
|
|
55
|
+
log.error( 'Processor execution error', { processor: p.name, error: serializeError( error ) } );
|
|
55
56
|
}
|
|
56
57
|
} );
|
|
57
58
|
}
|
|
@@ -87,15 +88,12 @@ export const addEventAction = ( action, { kind, name, id, parentId, details, tra
|
|
|
87
88
|
export function addEventActionWithContext( action, options ) {
|
|
88
89
|
const storeContent = Storage.load();
|
|
89
90
|
if ( storeContent ) { // If there is no storageContext this was not called from a Temporal environment
|
|
90
|
-
const { parentId, traceInfo
|
|
91
|
-
if ( action === EventAction.ADD_ATTR ) {
|
|
92
|
-
|
|
93
|
-
if ( !( attribute instanceof BaseAttribute ) ) {
|
|
94
|
-
throw new Error( `Event ${EventAction.ADD_ATTR} argument is not a BaseAttribute instance` );
|
|
95
|
-
} else {
|
|
96
|
-
addAttribute( options.details );
|
|
97
|
-
}
|
|
91
|
+
const { parentId, traceInfo } = storeContent;
|
|
92
|
+
if ( action === EventAction.ADD_ATTR && !( options.details instanceof BaseAttribute ) ) {
|
|
93
|
+
throw new Error( `Event ${EventAction.ADD_ATTR} argument is not a BaseAttribute instance` );
|
|
98
94
|
}
|
|
99
|
-
|
|
95
|
+
/** Unwraps Transparent error set in details so it behaves like in the interceptors */
|
|
96
|
+
const details = options.details instanceof TransparentFatalError ? options.details.cause : options.details;
|
|
97
|
+
addEventAction( action, { ...options, details, parentId, traceInfo } );
|
|
100
98
|
}
|
|
101
99
|
};
|
|
@@ -5,10 +5,15 @@ vi.mock( '#async_storage', () => ( {
|
|
|
5
5
|
Storage: { load: storageLoadMock }
|
|
6
6
|
} ) );
|
|
7
7
|
|
|
8
|
-
const
|
|
8
|
+
const serializeErrorMock = vi.fn( error => ( {
|
|
9
|
+
name: error.name,
|
|
10
|
+
message: error.message
|
|
11
|
+
} ) );
|
|
12
|
+
vi.mock( '#helpers/error_serializer', () => ( { serializeError: serializeErrorMock } ) );
|
|
13
|
+
|
|
9
14
|
const logErrorMock = vi.fn();
|
|
10
15
|
vi.mock( '#logger', () => ( {
|
|
11
|
-
createChildLogger: () => ( {
|
|
16
|
+
createChildLogger: () => ( { error: logErrorMock } )
|
|
12
17
|
} ) );
|
|
13
18
|
|
|
14
19
|
const localInitMock = vi.fn( async () => {} );
|
|
@@ -71,6 +76,24 @@ describe( 'tracing/trace_engine', () => {
|
|
|
71
76
|
expect( payload.traceInfo ).toBe( traceInfo );
|
|
72
77
|
} );
|
|
73
78
|
|
|
79
|
+
it( 'init() logs processor failures with serialized stacks', async () => {
|
|
80
|
+
process.env.OUTPUT_TRACE_LOCAL_ON = '1';
|
|
81
|
+
const processorError = new Error( 'processor failed' );
|
|
82
|
+
localExecMock.mockRejectedValueOnce( processorError );
|
|
83
|
+
const { init, addEventAction } = await loadTraceEngine();
|
|
84
|
+
await init();
|
|
85
|
+
|
|
86
|
+
addEventAction( 'start', {
|
|
87
|
+
kind: 'step', name: 'N', id: '1', parentId: 'p', details: null, traceInfo
|
|
88
|
+
} );
|
|
89
|
+
|
|
90
|
+
await vi.waitFor( () => expect( logErrorMock ).toHaveBeenCalledWith( 'Processor execution error', {
|
|
91
|
+
processor: 'LOCAL',
|
|
92
|
+
error: { name: 'Error', message: 'processor failed' }
|
|
93
|
+
} ) );
|
|
94
|
+
expect( serializeErrorMock ).toHaveBeenCalledWith( processorError );
|
|
95
|
+
} );
|
|
96
|
+
|
|
74
97
|
it( 'addEventAction() emits an entry consumed by processors', async () => {
|
|
75
98
|
process.env.OUTPUT_TRACE_LOCAL_ON = 'on';
|
|
76
99
|
const { init, addEventAction } = await loadTraceEngine();
|
|
@@ -87,34 +110,34 @@ describe( 'tracing/trace_engine', () => {
|
|
|
87
110
|
expect( payload.entry.details ).toBe( 'done' );
|
|
88
111
|
} );
|
|
89
112
|
|
|
90
|
-
it( 'addEventAction()
|
|
91
|
-
process.env.OUTPUT_TRACE_LOCAL_ON = '
|
|
113
|
+
it( 'addEventAction() serializes error details before emitting', async () => {
|
|
114
|
+
process.env.OUTPUT_TRACE_LOCAL_ON = 'on';
|
|
92
115
|
const { init, addEventAction } = await loadTraceEngine();
|
|
93
116
|
await init();
|
|
117
|
+
const error = new TypeError( 'step failed' );
|
|
94
118
|
|
|
95
|
-
addEventAction( '
|
|
96
|
-
kind: 'step', name: '
|
|
97
|
-
traceInfo
|
|
119
|
+
addEventAction( 'error', {
|
|
120
|
+
kind: 'step', name: 'S', id: '3', parentId: 'p3', details: error,
|
|
121
|
+
traceInfo
|
|
122
|
+
} );
|
|
123
|
+
|
|
124
|
+
expect( serializeErrorMock ).toHaveBeenCalledWith( error );
|
|
125
|
+
expect( localExecMock.mock.calls[0][0].entry.details ).toEqual( {
|
|
126
|
+
name: 'TypeError',
|
|
127
|
+
message: 'step failed'
|
|
98
128
|
} );
|
|
99
|
-
expect( localExecMock ).not.toHaveBeenCalled();
|
|
100
129
|
} );
|
|
101
130
|
|
|
102
|
-
it( 'addEventAction()
|
|
131
|
+
it( 'addEventAction() does not emit when traceInfo is absent', async () => {
|
|
103
132
|
process.env.OUTPUT_TRACE_LOCAL_ON = '1';
|
|
104
133
|
const { init, addEventAction } = await loadTraceEngine();
|
|
105
134
|
await init();
|
|
106
135
|
|
|
107
136
|
addEventAction( 'start', {
|
|
108
|
-
kind: '
|
|
109
|
-
traceInfo
|
|
110
|
-
} );
|
|
111
|
-
expect( localExecMock ).toHaveBeenCalledTimes( 1 );
|
|
112
|
-
expect( localExecMock.mock.calls[0][0].entry ).toMatchObject( {
|
|
113
|
-
kind: 'internal_step',
|
|
114
|
-
name: 'Internal',
|
|
115
|
-
id: '1',
|
|
116
|
-
parentId: 'p'
|
|
137
|
+
kind: 'step', name: 'X', id: '1', parentId: 'p', details: {},
|
|
138
|
+
traceInfo: undefined
|
|
117
139
|
} );
|
|
140
|
+
expect( localExecMock ).not.toHaveBeenCalled();
|
|
118
141
|
} );
|
|
119
142
|
|
|
120
143
|
it( 'addEventActionWithContext() uses storage when available', async () => {
|
|
@@ -135,13 +158,34 @@ describe( 'tracing/trace_engine', () => {
|
|
|
135
158
|
expect( payload.entry.action ).toBe( 'tick' );
|
|
136
159
|
} );
|
|
137
160
|
|
|
138
|
-
it( 'addEventActionWithContext()
|
|
161
|
+
it( 'addEventActionWithContext() unwraps transparent errors without mutating options', async () => {
|
|
139
162
|
process.env.OUTPUT_TRACE_LOCAL_ON = 'true';
|
|
140
|
-
const addAttributeMock = vi.fn();
|
|
141
163
|
storageLoadMock.mockReturnValue( {
|
|
142
164
|
parentId: 'ctx-p',
|
|
143
|
-
traceInfo
|
|
144
|
-
|
|
165
|
+
traceInfo
|
|
166
|
+
} );
|
|
167
|
+
const { init, addEventActionWithContext } = await loadTraceEngine();
|
|
168
|
+
const { TransparentFatalError } = await import( '#errors' );
|
|
169
|
+
await init();
|
|
170
|
+
|
|
171
|
+
const cause = new TypeError( 'provider rejected request' );
|
|
172
|
+
const error = new TransparentFatalError( cause );
|
|
173
|
+
const options = { kind: 'llm', name: 'generate', id: 'llm-1', details: error };
|
|
174
|
+
addEventActionWithContext( 'error', options );
|
|
175
|
+
|
|
176
|
+
expect( options.details ).toBe( error );
|
|
177
|
+
expect( serializeErrorMock ).toHaveBeenCalledWith( cause );
|
|
178
|
+
expect( localExecMock.mock.calls[0][0].entry.details ).toEqual( {
|
|
179
|
+
name: 'TypeError',
|
|
180
|
+
message: 'provider rejected request'
|
|
181
|
+
} );
|
|
182
|
+
} );
|
|
183
|
+
|
|
184
|
+
it( 'addEventActionWithContext() emits validated ADD_ATTR trace entries', async () => {
|
|
185
|
+
process.env.OUTPUT_TRACE_LOCAL_ON = 'true';
|
|
186
|
+
storageLoadMock.mockReturnValue( {
|
|
187
|
+
parentId: 'ctx-p',
|
|
188
|
+
traceInfo
|
|
145
189
|
} );
|
|
146
190
|
const { init, addEventActionWithContext } = await loadTraceEngine();
|
|
147
191
|
const { EventAction } = await import( './trace_consts.js' );
|
|
@@ -151,8 +195,6 @@ describe( 'tracing/trace_engine', () => {
|
|
|
151
195
|
const attribute = new Attribute.HTTPRequestCount( 'https://example.test', 'req-1' );
|
|
152
196
|
addEventActionWithContext( EventAction.ADD_ATTR, { kind: 'http', name: 'request', id: 'req-1', details: attribute } );
|
|
153
197
|
|
|
154
|
-
expect( addAttributeMock ).toHaveBeenCalledTimes( 1 );
|
|
155
|
-
expect( addAttributeMock ).toHaveBeenCalledWith( attribute );
|
|
156
198
|
expect( localExecMock ).toHaveBeenCalledTimes( 1 );
|
|
157
199
|
expect( localExecMock.mock.calls[0][0] ).toEqual( {
|
|
158
200
|
traceInfo,
|
|
@@ -170,11 +212,9 @@ describe( 'tracing/trace_engine', () => {
|
|
|
170
212
|
|
|
171
213
|
it( 'addEventActionWithContext() throws on invalid ADD_ATTR signal payloads', async () => {
|
|
172
214
|
process.env.OUTPUT_TRACE_LOCAL_ON = 'true';
|
|
173
|
-
const addAttributeMock = vi.fn();
|
|
174
215
|
storageLoadMock.mockReturnValue( {
|
|
175
216
|
parentId: 'ctx-p',
|
|
176
|
-
traceInfo
|
|
177
|
-
addAttribute: addAttributeMock
|
|
217
|
+
traceInfo
|
|
178
218
|
} );
|
|
179
219
|
const { init, addEventActionWithContext } = await loadTraceEngine();
|
|
180
220
|
const { EventAction } = await import( './trace_consts.js' );
|
|
@@ -186,7 +226,6 @@ describe( 'tracing/trace_engine', () => {
|
|
|
186
226
|
{ kind: 'http', name: 'request', id: 'req-1', details: invalidAttribute }
|
|
187
227
|
) ).toThrow( /not a BaseAttribute instance/ );
|
|
188
228
|
|
|
189
|
-
expect( addAttributeMock ).not.toHaveBeenCalled();
|
|
190
229
|
expect( localExecMock ).not.toHaveBeenCalled();
|
|
191
230
|
} );
|
|
192
231
|
|
|
@@ -38,7 +38,7 @@ export class CatalogJob {
|
|
|
38
38
|
throw e;
|
|
39
39
|
}
|
|
40
40
|
if ( !( e instanceof WorkflowNotFoundError ) ) {
|
|
41
|
-
log.warn( 'Error describing catalog workflow', { error: e } );
|
|
41
|
+
log.warn( 'Error describing catalog workflow', { error: e.message } );
|
|
42
42
|
}
|
|
43
43
|
return false;
|
|
44
44
|
} );
|
|
@@ -52,7 +52,7 @@ export class CatalogJob {
|
|
|
52
52
|
throw e;
|
|
53
53
|
}
|
|
54
54
|
if ( !( e instanceof WorkflowNotFoundError ) ) {
|
|
55
|
-
log.warn( 'Error describing catalog workflow', { error: e } );
|
|
55
|
+
log.warn( 'Error describing catalog workflow', { error: e.message } );
|
|
56
56
|
}
|
|
57
57
|
return false;
|
|
58
58
|
} );
|
|
@@ -66,7 +66,7 @@ export class CatalogJob {
|
|
|
66
66
|
throw e;
|
|
67
67
|
}
|
|
68
68
|
if ( !( e instanceof WorkflowNotFoundError ) ) {
|
|
69
|
-
log.warn( 'Error completing previous catalog workflow', { error: e } );
|
|
69
|
+
log.warn( 'Error completing previous catalog workflow', { error: e.message } );
|
|
70
70
|
}
|
|
71
71
|
} );
|
|
72
72
|
};
|
|
@@ -121,7 +121,7 @@ describe( 'CatalogJob', () => {
|
|
|
121
121
|
|
|
122
122
|
await job.run();
|
|
123
123
|
|
|
124
|
-
expect( mockLog.warn ).toHaveBeenCalledWith( 'Error completing previous catalog workflow', { error: completeError } );
|
|
124
|
+
expect( mockLog.warn ).toHaveBeenCalledWith( 'Error completing previous catalog workflow', { error: completeError.message } );
|
|
125
125
|
expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', expect.any( Object ) );
|
|
126
126
|
expect( job.error ).toBeNull();
|
|
127
127
|
} );
|
package/src/worker/configs.js
CHANGED
|
@@ -36,8 +36,6 @@ const envVarSchema = z.object( {
|
|
|
36
36
|
OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 2 * 60 * 1000 ) ), // 2min
|
|
37
37
|
// Whether to send activity heartbeats (enabled by default)
|
|
38
38
|
OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: z.transform( v => v === undefined ? true : isStringboolTrue( v ) ),
|
|
39
|
-
// Time to allow for hooks to flush before shutdown
|
|
40
|
-
OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 3000 ) ),
|
|
41
39
|
// Set temporal worker shutdown force time
|
|
42
40
|
TEMPORAL_SHUTDOWN_FORCE_TIME: durationSchema,
|
|
43
41
|
// Set temporal worker shutdown grace time
|
|
@@ -70,7 +68,6 @@ export const catalogId = envVars.OUTPUT_CATALOG_ID;
|
|
|
70
68
|
export const workerTelemetryIntervalMs = envVars.OUTPUT_WORKER_TELEMETRY_INTERVAL_MS;
|
|
71
69
|
export const activityHeartbeatIntervalMs = envVars.OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS;
|
|
72
70
|
export const activityHeartbeatEnabled = envVars.OUTPUT_ACTIVITY_HEARTBEAT_ENABLED;
|
|
73
|
-
export const processFailureShutdownDelay = envVars.OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY;
|
|
74
71
|
export const shutdownForceTime = envVars.TEMPORAL_SHUTDOWN_FORCE_TIME;
|
|
75
72
|
export const shutdownGraceTime = envVars.TEMPORAL_SHUTDOWN_GRACE_TIME;
|
|
76
73
|
export const grpcProxy = envVars.TEMPORAL_GRPC_PROXY;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { mainEventBus } from '#bus';
|
|
2
2
|
import { ACTIVITY_LOGGER_SYMBOL, BusEventType } from '#consts';
|
|
3
3
|
import { activityInfo as activityInfoFn } from '@temporalio/activity';
|
|
4
4
|
import { assignImmutableProperty } from '#helpers/object';
|
|
@@ -9,6 +9,6 @@ import { assignImmutableProperty } from '#helpers/object';
|
|
|
9
9
|
export const bindGlobalFunctions = () => {
|
|
10
10
|
/** Defines the activity logger function, accessible in activity context via logger interface */
|
|
11
11
|
assignImmutableProperty( globalThis, ACTIVITY_LOGGER_SYMBOL, ( { level, message, metadata } ) =>
|
|
12
|
-
|
|
12
|
+
mainEventBus.emit( BusEventType.ACTIVITY_LOG, { level, message, metadata, activityInfo: activityInfoFn() } )
|
|
13
13
|
);
|
|
14
14
|
};
|
|
@@ -10,7 +10,7 @@ const {
|
|
|
10
10
|
}
|
|
11
11
|
} ) );
|
|
12
12
|
|
|
13
|
-
const
|
|
13
|
+
const mainEventBusMock = vi.hoisted( () => ( {
|
|
14
14
|
emit: vi.fn()
|
|
15
15
|
} ) );
|
|
16
16
|
const activityInfoMock = vi.hoisted( () => vi.fn( () => ( {
|
|
@@ -19,7 +19,7 @@ const activityInfoMock = vi.hoisted( () => vi.fn( () => ( {
|
|
|
19
19
|
} ) ) );
|
|
20
20
|
|
|
21
21
|
vi.mock( '#consts', () => ( { ACTIVITY_LOGGER_SYMBOL, BusEventType } ) );
|
|
22
|
-
vi.mock( '#bus', () => ( {
|
|
22
|
+
vi.mock( '#bus', () => ( { mainEventBus: mainEventBusMock } ) );
|
|
23
23
|
vi.mock( '@temporalio/activity', () => ( {
|
|
24
24
|
activityInfo: activityInfoMock
|
|
25
25
|
} ) );
|
|
@@ -42,7 +42,7 @@ describe( 'worker/global_functions', () => {
|
|
|
42
42
|
} );
|
|
43
43
|
|
|
44
44
|
expect( activityInfoMock ).toHaveBeenCalledTimes( 1 );
|
|
45
|
-
expect(
|
|
45
|
+
expect( mainEventBusMock.emit ).toHaveBeenCalledWith( BusEventType.ACTIVITY_LOG, {
|
|
46
46
|
level: 'debug',
|
|
47
47
|
message: 'activity detail',
|
|
48
48
|
metadata,
|
package/src/worker/index.js
CHANGED
|
@@ -12,13 +12,15 @@ import { initInterceptors } from './interceptors/index.js';
|
|
|
12
12
|
import { createChildLogger } from '#logger';
|
|
13
13
|
import { setupInterruptionHandler } from './interruption.js';
|
|
14
14
|
import { CatalogJob } from './catalog_workflow/catalog_job.js';
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
15
|
+
import { mainEventBus } from '#bus';
|
|
16
|
+
import { flushPendingHooks } from '#hooks/pending_hooks';
|
|
17
17
|
import { BusEventType } from '#consts';
|
|
18
18
|
import { setupTelemetry } from './telemetry.js';
|
|
19
19
|
import { TemporalConnectionMonitor } from './connection_monitor.js';
|
|
20
20
|
import { bindGlobalFunctions } from './global_functions.js';
|
|
21
21
|
import { runOnce } from '#helpers/function';
|
|
22
|
+
import { serializeError } from '#helpers/error_serializer';
|
|
23
|
+
import { setupTemporalLogger } from './temporal_logger.js';
|
|
22
24
|
|
|
23
25
|
import './log_hooks.js';
|
|
24
26
|
|
|
@@ -51,6 +53,9 @@ const state = {
|
|
|
51
53
|
const callerDir = process.argv[2];
|
|
52
54
|
|
|
53
55
|
const execute = async () => {
|
|
56
|
+
log.info( 'Setup Temporal Logger' );
|
|
57
|
+
setupTemporalLogger();
|
|
58
|
+
|
|
54
59
|
log.info( 'Loading config...', { callerDir } );
|
|
55
60
|
await loadHooks( callerDir );
|
|
56
61
|
|
|
@@ -60,8 +65,7 @@ const execute = async () => {
|
|
|
60
65
|
log.info( 'Loading activities...', { callerDir } );
|
|
61
66
|
const { activities } = await loadActivities( callerDir, workflows );
|
|
62
67
|
|
|
63
|
-
|
|
64
|
-
bootstrapFetchProxy();
|
|
68
|
+
mainEventBus.emit( BusEventType.WORKER_BEFORE_START );
|
|
65
69
|
|
|
66
70
|
log.info( 'Initializing tracing...' );
|
|
67
71
|
await initTracing();
|
|
@@ -214,13 +218,18 @@ execute()
|
|
|
214
218
|
.catch( e => log.warn( 'Connection close error', { error: e.message } ) );
|
|
215
219
|
}
|
|
216
220
|
} )
|
|
217
|
-
.then( () =>
|
|
218
|
-
|
|
219
|
-
|
|
221
|
+
.then( async () => {
|
|
222
|
+
log.info( 'Flushing hook callbacks...' );
|
|
223
|
+
await flushPendingHooks();
|
|
224
|
+
log.info( 'Bye' );
|
|
225
|
+
} )
|
|
226
|
+
.catch( async error => {
|
|
227
|
+
log.error( 'Fatal error', { error: serializeError( error ) } );
|
|
220
228
|
|
|
221
|
-
|
|
229
|
+
mainEventBus.emit( BusEventType.RUNTIME_ERROR, { error } );
|
|
222
230
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
231
|
+
log.info( 'Flushing hook callbacks...' );
|
|
232
|
+
await flushPendingHooks();
|
|
233
|
+
log.info( 'Exiting...' );
|
|
234
|
+
setTimeout( () => process.exit( 1 ) );
|
|
226
235
|
} );
|
package/src/worker/index.spec.js
CHANGED
|
@@ -6,14 +6,17 @@ const {
|
|
|
6
6
|
configValues,
|
|
7
7
|
connectionMonitorInstance,
|
|
8
8
|
createCatalogMock,
|
|
9
|
+
flushPendingHooksMock,
|
|
9
10
|
initInterceptorsMock,
|
|
10
|
-
|
|
11
|
+
mainEventBusMock,
|
|
11
12
|
mockConnection,
|
|
12
13
|
mockLog,
|
|
13
14
|
mockWorker,
|
|
14
15
|
promises,
|
|
15
16
|
resetPromises,
|
|
17
|
+
serializeErrorMock,
|
|
16
18
|
setupInterruptionHandlerMock,
|
|
19
|
+
setupTemporalLoggerMock,
|
|
17
20
|
setupTelemetryMock
|
|
18
21
|
} = vi.hoisted( () => {
|
|
19
22
|
const createDeferred = () => {
|
|
@@ -46,7 +49,6 @@ const {
|
|
|
46
49
|
maxConcurrentActivityTaskPolls: 5,
|
|
47
50
|
maxConcurrentWorkflowTaskPolls: 5,
|
|
48
51
|
workerTuner: undefined,
|
|
49
|
-
processFailureShutdownDelay: 0,
|
|
50
52
|
shutdownForceTime: undefined,
|
|
51
53
|
shutdownGraceTime: undefined
|
|
52
54
|
};
|
|
@@ -105,30 +107,35 @@ const {
|
|
|
105
107
|
connectionMonitorInstance,
|
|
106
108
|
createCatalogMock: vi.fn().mockReturnValue( { workflows: [], activities: {} } ),
|
|
107
109
|
createWorkflowsEntryPointMock: vi.fn().mockReturnValue( '/fake/workflows/path.js' ),
|
|
110
|
+
flushPendingHooksMock: vi.fn().mockResolvedValue( undefined ),
|
|
108
111
|
hashSourceCodeMock: vi.fn().mockResolvedValue( 'catalog-hash' ),
|
|
109
112
|
initInterceptorsMock: vi.fn().mockReturnValue( [] ),
|
|
110
113
|
loadActivitiesMock: vi.fn().mockResolvedValue( {} ),
|
|
111
114
|
loadHooksMock: vi.fn().mockResolvedValue( undefined ),
|
|
112
115
|
loadWorkflowsMock: vi.fn().mockResolvedValue( [] ),
|
|
113
|
-
|
|
116
|
+
mainEventBusMock: { emit: vi.fn(), on: vi.fn() },
|
|
114
117
|
mockConnection: { close: vi.fn().mockResolvedValue( undefined ) },
|
|
115
118
|
mockLog: { error: vi.fn(), info: vi.fn(), warn: vi.fn() },
|
|
116
119
|
mockWorker,
|
|
117
120
|
promises,
|
|
118
121
|
resetPromises,
|
|
122
|
+
serializeErrorMock: vi.fn( error => ( {
|
|
123
|
+
name: error.name,
|
|
124
|
+
message: error.message,
|
|
125
|
+
stack: error.stack
|
|
126
|
+
} ) ),
|
|
119
127
|
setupInterruptionHandlerMock: vi.fn(),
|
|
128
|
+
setupTemporalLoggerMock: vi.fn(),
|
|
120
129
|
setupTelemetryMock: vi.fn()
|
|
121
130
|
};
|
|
122
131
|
} );
|
|
123
132
|
|
|
124
133
|
vi.mock( '#logger', () => ( { createChildLogger: () => mockLog } ) );
|
|
125
|
-
vi.mock( '#
|
|
126
|
-
const actual = await importOriginal();
|
|
127
|
-
return { ...actual };
|
|
128
|
-
} );
|
|
134
|
+
vi.mock( '#helpers/error_serializer', () => ( { serializeError: serializeErrorMock } ) );
|
|
129
135
|
const initTracing = vi.fn().mockResolvedValue( undefined );
|
|
130
136
|
vi.mock( '#tracing', () => ( { init: initTracing } ) );
|
|
131
|
-
vi.mock( '#bus', () => ( {
|
|
137
|
+
vi.mock( '#bus', () => ( { mainEventBus: mainEventBusMock } ) );
|
|
138
|
+
vi.mock( '#hooks/pending_hooks', () => ( { flushPendingHooks: flushPendingHooksMock } ) );
|
|
132
139
|
|
|
133
140
|
const loadWorkflowsMock = vi.fn().mockResolvedValue( { workflows: [], entrypoint: '/fake/workflows/path.js' } );
|
|
134
141
|
const loadActivitiesMock = vi.fn().mockResolvedValue( { activities: {} } );
|
|
@@ -145,7 +152,6 @@ vi.mock( './sinks.js', () => ( { sinks: {} } ) );
|
|
|
145
152
|
vi.mock( './catalog_workflow/index.js', () => ( { createCatalog: createCatalogMock } ) );
|
|
146
153
|
vi.mock( './bundler_options.js', () => ( { webpackConfigHook: vi.fn() } ) );
|
|
147
154
|
vi.mock( './interceptors/index.js', () => ( { initInterceptors: initInterceptorsMock } ) );
|
|
148
|
-
vi.mock( './proxy.js', () => ( { bootstrapFetchProxy: vi.fn() } ) );
|
|
149
155
|
vi.mock( './telemetry.js', () => ( { setupTelemetry: setupTelemetryMock } ) );
|
|
150
156
|
vi.mock( './interruption.js', () => ( { setupInterruptionHandler: setupInterruptionHandlerMock } ) );
|
|
151
157
|
vi.mock( './global_functions.js', () => ( { bindGlobalFunctions: bindGlobalFunctionsMock } ) );
|
|
@@ -160,6 +166,7 @@ vi.mock( './catalog_workflow/catalog_job.js', () => ( {
|
|
|
160
166
|
} )
|
|
161
167
|
} ) );
|
|
162
168
|
vi.mock( './log_hooks.js', () => ( {} ) );
|
|
169
|
+
vi.mock( './temporal_logger.js', () => ( { setupTemporalLogger: setupTemporalLoggerMock } ) );
|
|
163
170
|
vi.mock( '@temporalio/worker', () => ( {
|
|
164
171
|
NativeConnection: { connect: vi.fn().mockResolvedValue( mockConnection ) },
|
|
165
172
|
Worker: { create: vi.fn().mockResolvedValue( mockWorker ) }
|
|
@@ -216,6 +223,7 @@ describe( 'worker/index', () => {
|
|
|
216
223
|
await vi.waitFor( () => expect( Worker.create ).toHaveBeenCalled() );
|
|
217
224
|
|
|
218
225
|
expect( loadHooksMock ).toHaveBeenCalledWith( '/test/caller/dir' );
|
|
226
|
+
expect( setupTemporalLoggerMock ).toHaveBeenCalledOnce();
|
|
219
227
|
expect( loadWorkflowsMock ).toHaveBeenCalledWith( '/test/caller/dir' );
|
|
220
228
|
expect( loadActivitiesMock ).toHaveBeenCalledWith( '/test/caller/dir', [] );
|
|
221
229
|
expect( initTracing ).toHaveBeenCalled();
|
|
@@ -262,6 +270,7 @@ describe( 'worker/index', () => {
|
|
|
262
270
|
expect( catalogJobInstance.run ).toHaveBeenCalled();
|
|
263
271
|
|
|
264
272
|
await settleWorker();
|
|
273
|
+
await vi.waitFor( () => expect( flushPendingHooksMock ).toHaveBeenCalledOnce() );
|
|
265
274
|
expect( mockLog.info ).toHaveBeenCalledWith( 'Bye' );
|
|
266
275
|
} );
|
|
267
276
|
|
|
@@ -354,11 +363,15 @@ describe( 'worker/index', () => {
|
|
|
354
363
|
|
|
355
364
|
await vi.waitFor( () => {
|
|
356
365
|
expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.objectContaining( {
|
|
357
|
-
error: 'Big Failure'
|
|
366
|
+
error: expect.objectContaining( { message: 'Big Failure' } )
|
|
358
367
|
} ) );
|
|
359
368
|
} );
|
|
360
|
-
expect(
|
|
369
|
+
expect( serializeErrorMock ).toHaveBeenCalledWith( error );
|
|
370
|
+
expect( mainEventBusMock.emit ).toHaveBeenCalledWith( expect.any( String ), { error } );
|
|
361
371
|
await vi.waitFor( () => expect( exitMock ).toHaveBeenCalledWith( 1 ) );
|
|
372
|
+
expect( flushPendingHooksMock ).toHaveBeenCalledOnce();
|
|
373
|
+
expect( mainEventBusMock.emit.mock.invocationCallOrder[0] ).toBeLessThan( flushPendingHooksMock.mock.invocationCallOrder[0] );
|
|
374
|
+
expect( flushPendingHooksMock.mock.invocationCallOrder[0] ).toBeLessThan( exitMock.mock.invocationCallOrder[0] );
|
|
362
375
|
} );
|
|
363
376
|
|
|
364
377
|
it( 'throws connection monitor errors after graceful shutdown', async () => {
|
|
@@ -373,12 +386,12 @@ describe( 'worker/index', () => {
|
|
|
373
386
|
|
|
374
387
|
await vi.waitFor( () => {
|
|
375
388
|
expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.objectContaining( {
|
|
376
|
-
error: 'connection lost'
|
|
389
|
+
error: expect.objectContaining( { message: 'connection lost' } )
|
|
377
390
|
} ) );
|
|
378
391
|
} );
|
|
379
392
|
expect( mockWorker.shutdown ).toHaveBeenCalledOnce();
|
|
380
393
|
expect( catalogJobInstance.interrupt ).toHaveBeenCalledOnce();
|
|
381
|
-
expect(
|
|
394
|
+
expect( mainEventBusMock.emit ).toHaveBeenCalledWith( expect.any( String ), { error } );
|
|
382
395
|
} );
|
|
383
396
|
|
|
384
397
|
it( 'throws catalog job errors after graceful shutdown', async () => {
|
|
@@ -393,12 +406,12 @@ describe( 'worker/index', () => {
|
|
|
393
406
|
|
|
394
407
|
await vi.waitFor( () => {
|
|
395
408
|
expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.objectContaining( {
|
|
396
|
-
error: 'catalog failed'
|
|
409
|
+
error: expect.objectContaining( { message: 'catalog failed' } )
|
|
397
410
|
} ) );
|
|
398
411
|
} );
|
|
399
412
|
expect( mockWorker.shutdown ).toHaveBeenCalledOnce();
|
|
400
413
|
expect( connectionMonitorInstance.stop ).toHaveBeenCalledOnce();
|
|
401
|
-
expect(
|
|
414
|
+
expect( mainEventBusMock.emit ).toHaveBeenCalledWith( expect.any( String ), { error } );
|
|
402
415
|
} );
|
|
403
416
|
|
|
404
417
|
it( 'cleans up partial startup failures after connecting', async () => {
|
|
@@ -413,7 +426,7 @@ describe( 'worker/index', () => {
|
|
|
413
426
|
expect( catalogJobInstance.interrupt ).not.toHaveBeenCalled();
|
|
414
427
|
await vi.waitFor( () => {
|
|
415
428
|
expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.objectContaining( {
|
|
416
|
-
error: 'worker create failed'
|
|
429
|
+
error: expect.objectContaining( { message: 'worker create failed' } )
|
|
417
430
|
} ) );
|
|
418
431
|
} );
|
|
419
432
|
} );
|