@outputai/core 0.8.1-next.86368bb.0 → 0.8.1-next.aa8ed5e.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
package/src/worker/configs.js
CHANGED
|
@@ -5,6 +5,13 @@ class InvalidEnvVarsErrors extends Error { }
|
|
|
5
5
|
|
|
6
6
|
const coalesceEmptyString = v => v === '' ? undefined : v;
|
|
7
7
|
|
|
8
|
+
const durationSchema = z.preprocess(
|
|
9
|
+
coalesceEmptyString,
|
|
10
|
+
z.string()
|
|
11
|
+
.regex( /^\d+$|^\d+(\.\d+)?\s?(ms|s|m|h|d)$/i )
|
|
12
|
+
.optional()
|
|
13
|
+
);
|
|
14
|
+
|
|
8
15
|
const envVarSchema = z.object( {
|
|
9
16
|
OUTPUT_CATALOG_ID: z.string().regex( /^[a-z0-9_.@-]+$/i ),
|
|
10
17
|
OUTPUT_WORKER_TELEMETRY_INTERVAL_MS: z.preprocess( coalesceEmptyString, z.coerce.number().int().nonnegative().default( 0 ) ),
|
|
@@ -28,6 +35,10 @@ const envVarSchema = z.object( {
|
|
|
28
35
|
OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: z.transform( v => v === undefined ? true : isStringboolTrue( v ) ),
|
|
29
36
|
// Time to allow for hooks to flush before shutdown
|
|
30
37
|
OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 3000 ) ),
|
|
38
|
+
// Set temporal worker shutdown force time
|
|
39
|
+
TEMPORAL_SHUTDOWN_FORCE_TIME: durationSchema,
|
|
40
|
+
// Set temporal worker shutdown grace time
|
|
41
|
+
TEMPORAL_SHUTDOWN_GRACE_TIME: durationSchema,
|
|
31
42
|
// HTTP CONNECT proxy for Temporal gRPC connections (e.g. "proxy-host:8080").
|
|
32
43
|
// Must be a bare host:port — no scheme (Temporal's native HTTP CONNECT
|
|
33
44
|
// option is not a URL).
|
|
@@ -56,4 +67,6 @@ export const workerTelemetryIntervalMs = envVars.OUTPUT_WORKER_TELEMETRY_INTERVA
|
|
|
56
67
|
export const activityHeartbeatIntervalMs = envVars.OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS;
|
|
57
68
|
export const activityHeartbeatEnabled = envVars.OUTPUT_ACTIVITY_HEARTBEAT_ENABLED;
|
|
58
69
|
export const processFailureShutdownDelay = envVars.OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY;
|
|
70
|
+
export const shutdownForceTime = envVars.TEMPORAL_SHUTDOWN_FORCE_TIME;
|
|
71
|
+
export const shutdownGraceTime = envVars.TEMPORAL_SHUTDOWN_GRACE_TIME;
|
|
59
72
|
export const grpcProxy = envVars.TEMPORAL_GRPC_PROXY;
|
|
@@ -12,7 +12,9 @@ const CONFIG_KEYS = [
|
|
|
12
12
|
'TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASK_POLLS',
|
|
13
13
|
'OUTPUT_WORKER_TELEMETRY_INTERVAL_MS',
|
|
14
14
|
'OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS',
|
|
15
|
-
'OUTPUT_ACTIVITY_HEARTBEAT_ENABLED'
|
|
15
|
+
'OUTPUT_ACTIVITY_HEARTBEAT_ENABLED',
|
|
16
|
+
'TEMPORAL_SHUTDOWN_FORCE_TIME',
|
|
17
|
+
'TEMPORAL_SHUTDOWN_GRACE_TIME'
|
|
16
18
|
];
|
|
17
19
|
|
|
18
20
|
const setEnv = ( overrides = {} ) => {
|
|
@@ -65,6 +67,8 @@ describe( 'worker/configs', () => {
|
|
|
65
67
|
expect( configs.workerTelemetryIntervalMs ).toBe( 0 );
|
|
66
68
|
expect( configs.activityHeartbeatIntervalMs ).toBe( 2 * 60 * 1000 );
|
|
67
69
|
expect( configs.activityHeartbeatEnabled ).toBe( true );
|
|
70
|
+
expect( configs.shutdownForceTime ).toBeUndefined();
|
|
71
|
+
expect( configs.shutdownGraceTime ).toBeUndefined();
|
|
68
72
|
expect( configs.taskQueue ).toBe( 'test-catalog' );
|
|
69
73
|
expect( configs.catalogId ).toBe( 'test-catalog' );
|
|
70
74
|
} );
|
|
@@ -100,6 +104,35 @@ describe( 'worker/configs', () => {
|
|
|
100
104
|
expect( configs.activityHeartbeatIntervalMs ).toBe( 60000 );
|
|
101
105
|
} );
|
|
102
106
|
|
|
107
|
+
it( 'parses Temporal shutdown durations', async () => {
|
|
108
|
+
setEnv( {
|
|
109
|
+
TEMPORAL_SHUTDOWN_FORCE_TIME: '15000',
|
|
110
|
+
TEMPORAL_SHUTDOWN_GRACE_TIME: '15s'
|
|
111
|
+
} );
|
|
112
|
+
const configs = await loadConfigs();
|
|
113
|
+
|
|
114
|
+
expect( configs.shutdownForceTime ).toBe( '15000' );
|
|
115
|
+
expect( configs.shutdownGraceTime ).toBe( '15s' );
|
|
116
|
+
} );
|
|
117
|
+
|
|
118
|
+
it( 'treats empty Temporal shutdown durations as unset', async () => {
|
|
119
|
+
setEnv( {
|
|
120
|
+
TEMPORAL_SHUTDOWN_FORCE_TIME: '',
|
|
121
|
+
TEMPORAL_SHUTDOWN_GRACE_TIME: ''
|
|
122
|
+
} );
|
|
123
|
+
const configs = await loadConfigs();
|
|
124
|
+
|
|
125
|
+
expect( configs.shutdownForceTime ).toBeUndefined();
|
|
126
|
+
expect( configs.shutdownGraceTime ).toBeUndefined();
|
|
127
|
+
} );
|
|
128
|
+
|
|
129
|
+
it( 'throws when Temporal shutdown durations are invalid', async () => {
|
|
130
|
+
setEnv( { TEMPORAL_SHUTDOWN_FORCE_TIME: 'soon' } );
|
|
131
|
+
vi.resetModules();
|
|
132
|
+
|
|
133
|
+
await expect( import( './configs.js' ) ).rejects.toThrow();
|
|
134
|
+
} );
|
|
135
|
+
|
|
103
136
|
it( 'allows zero for worker telemetry interval', async () => {
|
|
104
137
|
setEnv( { OUTPUT_WORKER_TELEMETRY_INTERVAL_MS: '0' } );
|
|
105
138
|
const configs = await loadConfigs();
|
|
@@ -2,13 +2,6 @@ import { createChildLogger } from '#logger';
|
|
|
2
2
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
3
3
|
import { CancellablePromise } from '#utils';
|
|
4
4
|
|
|
5
|
-
const ServingStatus = {
|
|
6
|
-
UNKNOWN: 0,
|
|
7
|
-
SERVING: 1,
|
|
8
|
-
NOT_SERVING: 2,
|
|
9
|
-
SERVICE_UNKNOWN: 3
|
|
10
|
-
};
|
|
11
|
-
|
|
12
5
|
const log = createChildLogger( 'Connection' );
|
|
13
6
|
|
|
14
7
|
export class TemporalConnectionMonitor {
|
|
@@ -28,24 +21,20 @@ export class TemporalConnectionMonitor {
|
|
|
28
21
|
throw new Error( 'Connection health check timed out' );
|
|
29
22
|
} );
|
|
30
23
|
|
|
31
|
-
#healthcheck = async () => this.#connection.
|
|
24
|
+
#healthcheck = async () => this.#connection.workflowService.getSystemInfo( {} );
|
|
32
25
|
|
|
33
26
|
#sleep = async () => delay( this.#CHECK_INTERVAL_MS, 0, { ref: false } );
|
|
34
27
|
|
|
35
28
|
#watch = async () => {
|
|
36
29
|
while ( !this.#cancellation.completed ) {
|
|
37
30
|
try {
|
|
38
|
-
|
|
31
|
+
await Promise.race( [ this.#healthcheck(), this.#getTimeout(), this.#cancellation.promise ] );
|
|
39
32
|
|
|
40
33
|
// cancellation won the race
|
|
41
34
|
if ( this.#cancellation.completed ) {
|
|
42
35
|
break;
|
|
43
36
|
}
|
|
44
37
|
|
|
45
|
-
if ( health?.status !== ServingStatus.SERVING ) {
|
|
46
|
-
throw new Error( `Connection not serving (status ${health?.status})` );
|
|
47
|
-
}
|
|
48
|
-
|
|
49
38
|
log.info( this.#failures === 0 ? 'Healthy' : 'Recovered' );
|
|
50
39
|
this.#failures = 0;
|
|
51
40
|
} catch ( error ) {
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
|
2
2
|
import { TemporalConnectionMonitor } from './connection_monitor.js';
|
|
3
3
|
|
|
4
|
-
const SERVING = 1;
|
|
5
|
-
const NOT_SERVING = 2;
|
|
6
4
|
const CHECK_TIMEOUT_MS = 50;
|
|
7
5
|
const CHECK_INTERVAL_MS = 100;
|
|
8
6
|
|
|
@@ -26,7 +24,7 @@ vi.mock( 'node:timers/promises', () => ( { setTimeout: delayMock } ) );
|
|
|
26
24
|
vi.mock( '#logger', () => ( { createChildLogger: vi.fn( () => mockLogger ) } ) );
|
|
27
25
|
|
|
28
26
|
const createConnection = check => ( {
|
|
29
|
-
|
|
27
|
+
workflowService: { getSystemInfo: check }
|
|
30
28
|
} );
|
|
31
29
|
|
|
32
30
|
const createMonitor = ( check, overrides = {} ) => new TemporalConnectionMonitor( createConnection( check ), {
|
|
@@ -52,8 +50,8 @@ describe( 'TemporalConnectionMonitor', () => {
|
|
|
52
50
|
scheduledDelays.length = 0;
|
|
53
51
|
} );
|
|
54
52
|
|
|
55
|
-
it( 'logs healthy when the
|
|
56
|
-
const check = vi.fn().mockResolvedValue( {
|
|
53
|
+
it( 'logs healthy when the workflow service is reachable', async () => {
|
|
54
|
+
const check = vi.fn().mockResolvedValue( {} );
|
|
57
55
|
const monitor = createMonitor( check );
|
|
58
56
|
|
|
59
57
|
const run = monitor.start();
|
|
@@ -90,7 +88,7 @@ describe( 'TemporalConnectionMonitor', () => {
|
|
|
90
88
|
it( 'logs recovered after a transient failure succeeds', async () => {
|
|
91
89
|
const check = vi.fn()
|
|
92
90
|
.mockRejectedValueOnce( new Error( 'temporary outage' ) )
|
|
93
|
-
.mockResolvedValueOnce( {
|
|
91
|
+
.mockResolvedValueOnce( {} );
|
|
94
92
|
const monitor = createMonitor( check );
|
|
95
93
|
|
|
96
94
|
monitor.start();
|
|
@@ -134,21 +132,6 @@ describe( 'TemporalConnectionMonitor', () => {
|
|
|
134
132
|
expect( monitor.running ).toBe( false );
|
|
135
133
|
} );
|
|
136
134
|
|
|
137
|
-
it( 'treats non-serving health status as a failure', async () => {
|
|
138
|
-
const check = vi.fn().mockResolvedValue( { status: NOT_SERVING } );
|
|
139
|
-
const monitor = createMonitor( check );
|
|
140
|
-
|
|
141
|
-
monitor.start();
|
|
142
|
-
await flushPromises();
|
|
143
|
-
|
|
144
|
-
expect( mockLogger.warn ).toHaveBeenCalledWith( 'Connection unhealthy', {
|
|
145
|
-
error: `Connection not serving (status ${NOT_SERVING})`,
|
|
146
|
-
failures: 1
|
|
147
|
-
} );
|
|
148
|
-
|
|
149
|
-
await monitor.stop();
|
|
150
|
-
} );
|
|
151
|
-
|
|
152
135
|
it( 'returns the same lifecycle promise when started more than once', async () => {
|
|
153
136
|
const check = vi.fn().mockReturnValue( new Promise( () => {} ) );
|
|
154
137
|
const monitor = createMonitor( check );
|
package/src/worker/index.js
CHANGED
|
@@ -33,7 +33,9 @@ const {
|
|
|
33
33
|
maxConcurrentActivityTaskExecutions,
|
|
34
34
|
maxCachedWorkflows,
|
|
35
35
|
maxConcurrentActivityTaskPolls,
|
|
36
|
-
maxConcurrentWorkflowTaskPolls
|
|
36
|
+
maxConcurrentWorkflowTaskPolls,
|
|
37
|
+
shutdownForceTime,
|
|
38
|
+
shutdownGraceTime
|
|
37
39
|
} = configs;
|
|
38
40
|
|
|
39
41
|
const state = {
|
|
@@ -73,6 +75,7 @@ const execute = async () => {
|
|
|
73
75
|
if ( proxy ) {
|
|
74
76
|
log.info( 'Using gRPC proxy', { targetHost: grpcProxy } );
|
|
75
77
|
}
|
|
78
|
+
|
|
76
79
|
state.connection = await NativeConnection.connect( { address, tls: Boolean( apiKey ), apiKey, proxy } );
|
|
77
80
|
|
|
78
81
|
log.info( 'Creating connection monitor...' );
|
|
@@ -95,7 +98,9 @@ const execute = async () => {
|
|
|
95
98
|
maxCachedWorkflows,
|
|
96
99
|
maxConcurrentActivityTaskPolls,
|
|
97
100
|
maxConcurrentWorkflowTaskPolls,
|
|
98
|
-
bundlerOptions: { webpackConfigHook }
|
|
101
|
+
bundlerOptions: { webpackConfigHook },
|
|
102
|
+
...( shutdownForceTime !== undefined && { shutdownForceTime } ),
|
|
103
|
+
...( shutdownGraceTime !== undefined && { shutdownGraceTime } )
|
|
99
104
|
} );
|
|
100
105
|
|
|
101
106
|
log.info( 'Setting up telemetry...' );
|
package/src/worker/index.spec.js
CHANGED
|
@@ -44,7 +44,9 @@ const {
|
|
|
44
44
|
maxCachedWorkflows: 1000,
|
|
45
45
|
maxConcurrentActivityTaskPolls: 5,
|
|
46
46
|
maxConcurrentWorkflowTaskPolls: 5,
|
|
47
|
-
processFailureShutdownDelay: 0
|
|
47
|
+
processFailureShutdownDelay: 0,
|
|
48
|
+
shutdownForceTime: undefined,
|
|
49
|
+
shutdownGraceTime: undefined
|
|
48
50
|
};
|
|
49
51
|
|
|
50
52
|
const connectionMonitorInstance = {
|
|
@@ -180,6 +182,8 @@ describe( 'worker/index', () => {
|
|
|
180
182
|
resetPromises();
|
|
181
183
|
configValues.apiKey = undefined;
|
|
182
184
|
configValues.grpcProxy = undefined;
|
|
185
|
+
configValues.shutdownForceTime = undefined;
|
|
186
|
+
configValues.shutdownGraceTime = undefined;
|
|
183
187
|
catalogJobInstance.error = null;
|
|
184
188
|
catalogJobInstance.errorCb = null;
|
|
185
189
|
catalogJobInstance.running = false;
|
|
@@ -237,6 +241,8 @@ describe( 'worker/index', () => {
|
|
|
237
241
|
maxConcurrentActivityTaskPolls: configValues.maxConcurrentActivityTaskPolls,
|
|
238
242
|
maxConcurrentWorkflowTaskPolls: configValues.maxConcurrentWorkflowTaskPolls
|
|
239
243
|
} ) );
|
|
244
|
+
expect( Worker.create.mock.calls[0][0] ).not.toHaveProperty( 'shutdownForceTime' );
|
|
245
|
+
expect( Worker.create.mock.calls[0][0] ).not.toHaveProperty( 'shutdownGraceTime' );
|
|
240
246
|
expect( initInterceptorsMock ).toHaveBeenCalledWith( { activities: {}, workflows: [] } );
|
|
241
247
|
expect( setupTelemetryMock ).toHaveBeenCalledWith( { worker: mockWorker } );
|
|
242
248
|
expect( setupInterruptionHandlerMock ).toHaveBeenCalledWith( expect.any( Function ) );
|
|
@@ -264,6 +270,21 @@ describe( 'worker/index', () => {
|
|
|
264
270
|
await settleWorker();
|
|
265
271
|
} );
|
|
266
272
|
|
|
273
|
+
it( 'passes configured shutdown durations to the worker', async () => {
|
|
274
|
+
configValues.shutdownForceTime = '30s';
|
|
275
|
+
configValues.shutdownGraceTime = '10s';
|
|
276
|
+
const { Worker } = await import( '@temporalio/worker' );
|
|
277
|
+
|
|
278
|
+
await importWorker();
|
|
279
|
+
|
|
280
|
+
await vi.waitFor( () => expect( Worker.create ).toHaveBeenCalledWith( expect.objectContaining( {
|
|
281
|
+
shutdownForceTime: '30s',
|
|
282
|
+
shutdownGraceTime: '10s'
|
|
283
|
+
} ) ) );
|
|
284
|
+
|
|
285
|
+
await settleWorker();
|
|
286
|
+
} );
|
|
287
|
+
|
|
267
288
|
it( 'runs graceful shutdown when interrupted', async () => {
|
|
268
289
|
await importWorker();
|
|
269
290
|
|