@outputai/core 0.12.1-next.ce54578.0 → 0.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/core",
-  "version": "0.12.1-next.ce54578.0",
+  "version": "0.13.0",
   "description": "The core module of the output framework",
   "type": "module",
   "engines": {

package/src/hooks/pending_hooks.js

@@ -3,17 +3,17 @@ import { createChildLogger } from '#logger';
 export const pendingHooks = new Set();
 
 const log = createChildLogger( 'Hooks' );
-const flushTimeoutMs = 30_000;
 
 /**
  * Await all pending hooks to flush for a certain time
+ * @param {number} timeoutMs - How long to await the pending hooks before giving up
  */
-export const flushPendingHooks = async () => {
+export const flushPendingHooks = async timeoutMs => {
   const state = { timeout: null };
   try {
     const flushed = await Promise.race( [
       Promise.allSettled( [ ...pendingHooks ] ).then( _ => true ),
-      new Promise( r => state.timeout = setTimeout( () => r( false ), flushTimeoutMs ) )
+      new Promise( r => state.timeout = setTimeout( () => r( false ), timeoutMs ) )
     ] );
     if ( !flushed ) {
       log.warn( 'Some hook callbacks exceeded the timeout and will not be awaited', { count: pendingHooks.size } );

package/src/hooks/pending_hooks.spec.js

@@ -24,7 +24,7 @@ describe( 'pending hooks', () => {
     } );
     pendingHooks.add( hookPromise );
 
-    const flushPromise = flushPendingHooks();
+    const flushPromise = flushPendingHooks( 5000 );
     const state = { flushed: false };
     flushPromise.then( () => {
       state.flushed = true;
@@ -42,8 +42,8 @@ describe( 'pending hooks', () => {
     vi.useFakeTimers();
     pendingHooks.add( new Promise( () => {} ) );
 
-    const flushPromise = flushPendingHooks();
-    await vi.advanceTimersByTimeAsync( 30_000 );
+    const flushPromise = flushPendingHooks( 5000 );
+    await vi.advanceTimersByTimeAsync( 5000 );
 
     await expect( flushPromise ).resolves.toBeUndefined();
   } );
@@ -53,8 +53,8 @@ describe( 'pending hooks', () => {
     pendingHooks.add( new Promise( () => {} ) );
     pendingHooks.add( new Promise( () => {} ) );
 
-    const flushPromise = flushPendingHooks();
-    await vi.advanceTimersByTimeAsync( 30_000 );
+    const flushPromise = flushPendingHooks( 5000 );
+    await vi.advanceTimersByTimeAsync( 5000 );
     await flushPromise;
 
     expect( logMock.warn ).toHaveBeenCalledWith( expect.any( String ), { count: 2 } );

package/src/worker/configs.js

@@ -36,10 +36,13 @@ const envVarSchema = z.object( {
   OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 2 * 60 * 1000 ) ), // 2min
   // Whether to send activity heartbeats (enabled by default)
   OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: z.transform( v => v === undefined ? true : isStringboolTrue( v ) ),
-  // Set temporal worker shutdown force time, kept under the 30s most platforms allow before SIGKILL
-  TEMPORAL_SHUTDOWN_FORCE_TIME: durationSchema.default( '25s' ),
+  // Set temporal worker shutdown force time. Defaults budget a shutdown against the 30s most
+  // platforms allow before SIGKILL: force time + OUTPUT_HOOK_FLUSH_TIMEOUT_MS + 5s of margin.
+  TEMPORAL_SHUTDOWN_FORCE_TIME: durationSchema.default( '20s' ),
   // Set temporal worker shutdown grace time, when in-flight activities are asked to cancel
-  TEMPORAL_SHUTDOWN_GRACE_TIME: durationSchema.default( '20s' ),
+  TEMPORAL_SHUTDOWN_GRACE_TIME: durationSchema.default( '15s' ),
+  // How long the worker awaits pending hook callbacks once the drain is over, before exiting anyway
+  OUTPUT_HOOK_FLUSH_TIMEOUT_MS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 5 * 1000 ) ), // 5s
   // HTTP CONNECT proxy for Temporal gRPC connections (e.g. "proxy-host:8080").
   // Must be a bare host:port — no scheme (Temporal's native HTTP CONNECT
   // option is not a URL).
@@ -70,4 +73,5 @@ export const activityHeartbeatIntervalMs = envVars.OUTPUT_ACTIVITY_HEARTBEAT_INT
 export const activityHeartbeatEnabled = envVars.OUTPUT_ACTIVITY_HEARTBEAT_ENABLED;
 export const shutdownForceTime = envVars.TEMPORAL_SHUTDOWN_FORCE_TIME;
 export const shutdownGraceTime = envVars.TEMPORAL_SHUTDOWN_GRACE_TIME;
+export const hookFlushTimeoutMs = envVars.OUTPUT_HOOK_FLUSH_TIMEOUT_MS;
 export const grpcProxy = envVars.TEMPORAL_GRPC_PROXY;

package/src/worker/configs.spec.js

@@ -15,7 +15,8 @@ const CONFIG_KEYS = [
   'OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS',
   'OUTPUT_ACTIVITY_HEARTBEAT_ENABLED',
   'TEMPORAL_SHUTDOWN_FORCE_TIME',
-  'TEMPORAL_SHUTDOWN_GRACE_TIME'
+  'TEMPORAL_SHUTDOWN_GRACE_TIME',
+  'OUTPUT_HOOK_FLUSH_TIMEOUT_MS'
 ];
 
 const setEnv = ( overrides = {} ) => {
@@ -69,8 +70,9 @@ describe( 'worker/configs', () => {
     expect( configs.workerTelemetryIntervalMs ).toBe( 0 );
     expect( configs.activityHeartbeatIntervalMs ).toBe( 2 * 60 * 1000 );
     expect( configs.activityHeartbeatEnabled ).toBe( true );
-    expect( configs.shutdownForceTime ).toBe( '25s' );
-    expect( configs.shutdownGraceTime ).toBe( '20s' );
+    expect( configs.shutdownForceTime ).toBe( '20s' );
+    expect( configs.shutdownGraceTime ).toBe( '15s' );
+    expect( configs.hookFlushTimeoutMs ).toBe( 5000 );
     expect( configs.taskQueue ).toBe( 'test-catalog' );
     expect( configs.catalogId ).toBe( 'test-catalog' );
   } );
@@ -139,12 +141,14 @@ describe( 'worker/configs', () => {
   it( 'falls back to the defaults when Temporal shutdown durations are empty', async () => {
     setEnv( {
       TEMPORAL_SHUTDOWN_FORCE_TIME: '',
-      TEMPORAL_SHUTDOWN_GRACE_TIME: ''
+      TEMPORAL_SHUTDOWN_GRACE_TIME: '',
+      OUTPUT_HOOK_FLUSH_TIMEOUT_MS: ''
     } );
     const configs = await loadConfigs();
 
-    expect( configs.shutdownForceTime ).toBe( '25s' );
-    expect( configs.shutdownGraceTime ).toBe( '20s' );
+    expect( configs.shutdownForceTime ).toBe( '20s' );
+    expect( configs.shutdownGraceTime ).toBe( '15s' );
+    expect( configs.hookFlushTimeoutMs ).toBe( 5000 );
   } );
 
   it( 'throws when Temporal shutdown durations are invalid', async () => {

package/src/worker/index.js

@@ -1,4 +1,4 @@
-import { Worker, NativeConnection } from '@temporalio/worker';
+import { Worker, NativeConnection, GracefulShutdownPeriodExpiredError } from '@temporalio/worker';
 import * as configs from './configs.js';
 import { loadActivities } from './loader/activities.js';
 import { loadWorkflows } from './loader/workflows.js';
@@ -41,6 +41,7 @@ const {
   maxConcurrentWorkflowTaskPolls,
   shutdownForceTime,
   shutdownGraceTime,
+  hookFlushTimeoutMs,
   workerTuner
 } = configs;
 
@@ -171,9 +172,11 @@ execute()
     }
 
     log.info( 'Flushing hook callbacks...' );
-    await flushPendingHooks();
+    await flushPendingHooks( hookFlushTimeoutMs );
 
-    const exitCode = shutdownFailures.some( v => v.service === 'worker' ) || hasError ? 1 : 0;
+    // An expired drain is the bound working as intended, not a worker failure, so it keeps exit code 0.
+    const workerFailedToStop = shutdownFailures.some( v => v.service === 'worker' && !( v.error instanceof GracefulShutdownPeriodExpiredError ) );
+    const exitCode = workerFailedToStop || hasError ? 1 : 0;
     setTimeout( () => {
       log.info( 'Bye' );
       process.exit( exitCode );

package/src/worker/index.spec.js

@@ -8,6 +8,7 @@ const {
   connectionMonitorInstance,
   createCatalogMock,
   flushPendingHooksMock,
+  GracefulShutdownPeriodExpiredError,
   hashSourceCodeMock,
   initInterceptorsMock,
   interruption,
@@ -58,7 +59,8 @@ const {
     maxConcurrentWorkflowTaskPolls: 5,
     workerTuner: undefined,
     shutdownForceTime: undefined,
-    shutdownGraceTime: undefined
+    shutdownGraceTime: undefined,
+    hookFlushTimeoutMs: 5000
   };
 
   /**
@@ -116,6 +118,11 @@ const {
     name = 'UncaughtError';
   };
 
+  /** Stands in for the Temporal error thrown when the drain outlives the force time */
+  class GracefulShutdownPeriodExpiredError extends Error {
+    name = 'GracefulShutdownPeriodExpiredError';
+  };
+
   const interruption = { controller: null, KillSignError, UncaughtError };
 
   return {
@@ -125,6 +132,7 @@ const {
     connectionMonitorInstance,
     createCatalogMock: vi.fn().mockReturnValue( { workflowNames: [ 'demo' ] } ),
     flushPendingHooksMock: vi.fn().mockResolvedValue( undefined ),
+    GracefulShutdownPeriodExpiredError,
     hashSourceCodeMock: vi.fn().mockResolvedValue( 'catalog-hash' ),
     initInterceptorsMock: vi.fn().mockReturnValue( [] ),
     interruption,
@@ -196,7 +204,8 @@ vi.mock( './catalog_workflow/catalog_publisher.js', () => ( {
 vi.mock( './shutdown.js', () => ( { shutdownServices: shutdownServicesMock } ) );
 vi.mock( '@temporalio/worker', () => ( {
   NativeConnection: { connect: vi.fn().mockResolvedValue( mockConnection ) },
-  Worker: { create: vi.fn().mockResolvedValue( mockWorker ) }
+  Worker: { create: vi.fn().mockResolvedValue( mockWorker ) },
+  GracefulShutdownPeriodExpiredError
 } ) );
 
 const importWorker = async () => {
@@ -403,6 +412,18 @@ describe( 'worker/index', () => {
       expect( process.exit ).toHaveBeenCalledWith( 0 );
     } );
 
+    it( 'still exits cleanly when the drain outlived the force time', async () => {
+      await bootWorker();
+      const error = new GracefulShutdownPeriodExpiredError( 'Timed out while waiting for worker to shutdown gracefully' );
+      shutdownServicesMock.mockResolvedValueOnce( [ { service: 'worker', error } ] );
+
+      interruption.controller.abort( new interruption.KillSignError( 'SIGTERM' ) );
+      await waitForExit();
+
+      expect( mockLog.error ).not.toHaveBeenCalled();
+      expect( process.exit ).toHaveBeenCalledWith( 0 );
+    } );
+
     it( 'does not report the worker as terminated when a signal ended the run', async () => {
       await bootWorker();
 
@@ -428,7 +449,7 @@ describe( 'worker/index', () => {
       interruption.controller.abort( new interruption.KillSignError( 'SIGTERM' ) );
       await waitForExit();
 
-      expect( flushPendingHooksMock ).toHaveBeenCalledOnce();
+      expect( flushPendingHooksMock ).toHaveBeenCalledWith( 5000 );
       expect( shutdownServicesMock.mock.invocationCallOrder[0] )
         .toBeLessThan( flushPendingHooksMock.mock.invocationCallOrder[0] );
       expect( flushPendingHooksMock.mock.invocationCallOrder[0] )