@outputai/core 0.10.1-next.af37678.0 → 0.10.1-next.be4ec7f.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/core",
-  "version": "0.10.1-next.af37678.0",
+  "version": "0.10.1-next.be4ec7f.0",
   "description": "The core module of the output framework",
   "type": "module",
   "engines": {
@@ -69,6 +69,7 @@
     "#consts": "./src/consts.js",
     "#errors": "./src/errors.js",
     "#logger": "./src/logger/index.js",
+    "#hooks/*": "./src/hooks/*.js",
     "#helpers/*": "./src/helpers/*.js",
     "#tracing": "./src/tracing/internal_interface.js",
     "#trace_attribute": "./src/tracing/trace_attribute.js",

package/src/helpers/fetch.js

@@ -1,6 +1,6 @@
 import { FatalError } from '#errors';
 
-/** Matches red int "hot-red-pie", but not int "redact" */
+/** Matches red in "hot-red-pie", but not in "redact" */
 const wordMatcher = term => new RegExp( `(?<![a-z\\d])${term}(?![a-z\\d])`, 'i' );
 
 /** Matches red in "acquired", but not in "redact" */

package/src/hooks/index.js

@@ -1,22 +1,24 @@
 import { mainEventBus, stepEventBus } from '#bus';
 import { BusEventType } from '#consts';
 import { createChildLogger } from '#logger';
+import { pendingHooks } from './pending_hooks.js';
 
 const log = createChildLogger( 'Hooks' );
 
 /**
- * Invokes a function within a try catch and log the error
+ * Invokes a function within a try catch and log the error.
  *
  * @param {Function} fn
  * @param {any} args - Args to invoke the function with
  * @param {string} hookName - hookName to identify this hook function in the logs
  */
-const callHookCb = async ( fn, args, hookName ) => {
-  try {
-    await fn( args );
-  } catch ( error ) {
-    log.error( `${hookName} hook error`, { message: error.message, stack: error.stack } );
-  }
+const callHookCb = ( fn, args, hookName ) => {
+  const promise = Promise.resolve()
+    .then( () => fn( args ) )
+    .catch( e => log.error( `${hookName} hook error`, { message: e.message, stack: e.stack } ) )
+    .finally( () => pendingHooks.delete( promise ) );
+  pendingHooks.add( promise );
+  return promise;
 };
 
 // General Life-cycle

package/src/hooks/index.spec.js

@@ -38,6 +38,7 @@ import {
   onWorkflowError,
   onWorkflowStart
 } from './index.js';
+import { pendingHooks } from './pending_hooks.js';
 
 const workflowDetails = {
   workflowId: 'wf-1',
@@ -67,6 +68,7 @@ const aggregations = {
 describe( 'hooks/index', () => {
   beforeEach( () => {
     vi.clearAllMocks();
+    pendingHooks.clear();
     Object.keys( mainOnHandlers ).forEach( k => {
       delete mainOnHandlers[k];
     } );
@@ -158,6 +160,23 @@ describe( 'hooks/index', () => {
     } );
   } );
 
+  it( 'tracks a hook callback until it settles', async () => {
+    const deferred = { resolve: null };
+    const handlerPromise = new Promise( resolve => {
+      deferred.resolve = resolve;
+    } );
+    const handler = vi.fn( () => handlerPromise );
+    onActivityEnd( handler );
+
+    const callbackPromise = mainOnHandlers[BusEventType.ACTIVITY_END]( { activityInfo } );
+
+    expect( pendingHooks.size ).toBe( 1 );
+
+    deferred.resolve();
+    await callbackPromise;
+    expect( pendingHooks.size ).toBe( 0 );
+  } );
+
   describe( 'workflow lifecycle hooks', () => {
     const cases = [
       [ 'onWorkflowStart', onWorkflowStart, BusEventType.WORKFLOW_START, {} ],

package/src/hooks/pending_hooks.js

@@ -0,0 +1,24 @@
+import { createChildLogger } from '#logger';
+
+export const pendingHooks = new Set();
+
+const log = createChildLogger( 'Hooks' );
+const flushTimeoutMs = 30_000;
+
+/**
+ * Await all pending hooks to flush for a certain time
+ */
+export const flushPendingHooks = async () => {
+  const state = { timeout: null };
+  try {
+    const flushed = await Promise.race( [
+      Promise.allSettled( [ ...pendingHooks ] ).then( _ => true ),
+      new Promise( r => state.timeout = setTimeout( () => r( false ), flushTimeoutMs ) )
+    ] );
+    if ( !flushed ) {
+      log.warn( 'Some hook callbacks exceeded the timeout and will not be awaited', { count: pendingHooks.size } );
+    }
+  } finally {
+    clearTimeout( state.timeout );
+  }
+};

package/src/hooks/pending_hooks.spec.js

@@ -0,0 +1,62 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const logMock = vi.hoisted( () => ( { warn: vi.fn() } ) );
+
+vi.mock( '#logger', () => ( { createChildLogger: () => logMock } ) );
+
+import { flushPendingHooks, pendingHooks } from './pending_hooks.js';
+
+describe( 'pending hooks', () => {
+  beforeEach( () => {
+    pendingHooks.clear();
+    vi.clearAllMocks();
+  } );
+
+  afterEach( () => {
+    pendingHooks.clear();
+    vi.useRealTimers();
+  } );
+
+  it( 'waits for pending hooks to settle', async () => {
+    const deferred = { resolve: null };
+    const hookPromise = new Promise( resolve => {
+      deferred.resolve = resolve;
+    } );
+    pendingHooks.add( hookPromise );
+
+    const flushPromise = flushPendingHooks();
+    const state = { flushed: false };
+    flushPromise.then( () => {
+      state.flushed = true;
+    } );
+
+    await Promise.resolve();
+    expect( state.flushed ).toBe( false );
+
+    deferred.resolve();
+    await flushPromise;
+    expect( state.flushed ).toBe( true );
+  } );
+
+  it( 'stops waiting after the timeout', async () => {
+    vi.useFakeTimers();
+    pendingHooks.add( new Promise( () => {} ) );
+
+    const flushPromise = flushPendingHooks();
+    await vi.advanceTimersByTimeAsync( 30_000 );
+
+    await expect( flushPromise ).resolves.toBeUndefined();
+  } );
+
+  it( 'logs the number of hooks that exceeded the timeout', async () => {
+    vi.useFakeTimers();
+    pendingHooks.add( new Promise( () => {} ) );
+    pendingHooks.add( new Promise( () => {} ) );
+
+    const flushPromise = flushPendingHooks();
+    await vi.advanceTimersByTimeAsync( 30_000 );
+    await flushPromise;
+
+    expect( logMock.warn ).toHaveBeenCalledWith( expect.any( String ), { count: 2 } );
+  } );
+} );

package/src/worker/configs.js

@@ -36,8 +36,6 @@ const envVarSchema = z.object( {
   OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 2 * 60 * 1000 ) ), // 2min
   // Whether to send activity heartbeats (enabled by default)
   OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: z.transform( v => v === undefined ? true : isStringboolTrue( v ) ),
-  // Time to allow for hooks to flush before shutdown
-  OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 3000 ) ),
   // Set temporal worker shutdown force time
   TEMPORAL_SHUTDOWN_FORCE_TIME: durationSchema,
   // Set temporal worker shutdown grace time
@@ -70,7 +68,6 @@ export const catalogId = envVars.OUTPUT_CATALOG_ID;
 export const workerTelemetryIntervalMs = envVars.OUTPUT_WORKER_TELEMETRY_INTERVAL_MS;
 export const activityHeartbeatIntervalMs = envVars.OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS;
 export const activityHeartbeatEnabled = envVars.OUTPUT_ACTIVITY_HEARTBEAT_ENABLED;
-export const processFailureShutdownDelay = envVars.OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY;
 export const shutdownForceTime = envVars.TEMPORAL_SHUTDOWN_FORCE_TIME;
 export const shutdownGraceTime = envVars.TEMPORAL_SHUTDOWN_GRACE_TIME;
 export const grpcProxy = envVars.TEMPORAL_GRPC_PROXY;

package/src/worker/index.js

@@ -13,6 +13,7 @@ import { createChildLogger } from '#logger';
 import { setupInterruptionHandler } from './interruption.js';
 import { CatalogJob } from './catalog_workflow/catalog_job.js';
 import { mainEventBus } from '#bus';
+import { flushPendingHooks } from '#hooks/pending_hooks';
 import { BusEventType } from '#consts';
 import { setupTelemetry } from './telemetry.js';
 import { TemporalConnectionMonitor } from './connection_monitor.js';
@@ -212,13 +213,18 @@ execute()
         .catch( e => log.warn( 'Connection close error', { error: e.message } ) );
     }
   } )
-  .then( () => log.info( 'Bye' ) )
-  .catch( error => {
+  .then( async () => {
+    log.info( 'Flushing hook callbacks...' );
+    await flushPendingHooks();
+    log.info( 'Bye' );
+  } )
+  .catch( async error => {
     log.error( 'Fatal error', { error: error.message, stack: error.stack } );
 
     mainEventBus.emit( BusEventType.RUNTIME_ERROR, { error } );
 
-    const timeToFlushEvent = configs.processFailureShutdownDelay;
-    log.info( `Exiting in ${timeToFlushEvent}ms` );
-    setTimeout( () => process.exit( 1 ), timeToFlushEvent );
+    log.info( 'Flushing hook callbacks...' );
+    await flushPendingHooks();
+    log.info( 'Exiting...' );
+    setTimeout( () => process.exit( 1 ) );
   } );

package/src/worker/index.spec.js

@@ -6,6 +6,7 @@ const {
   configValues,
   connectionMonitorInstance,
   createCatalogMock,
+  flushPendingHooksMock,
   initInterceptorsMock,
   mainEventBusMock,
   mockConnection,
@@ -46,7 +47,6 @@ const {
     maxConcurrentActivityTaskPolls: 5,
     maxConcurrentWorkflowTaskPolls: 5,
     workerTuner: undefined,
-    processFailureShutdownDelay: 0,
     shutdownForceTime: undefined,
     shutdownGraceTime: undefined
   };
@@ -105,6 +105,7 @@ const {
     connectionMonitorInstance,
     createCatalogMock: vi.fn().mockReturnValue( { workflows: [], activities: {} } ),
     createWorkflowsEntryPointMock: vi.fn().mockReturnValue( '/fake/workflows/path.js' ),
+    flushPendingHooksMock: vi.fn().mockResolvedValue( undefined ),
     hashSourceCodeMock: vi.fn().mockResolvedValue( 'catalog-hash' ),
     initInterceptorsMock: vi.fn().mockReturnValue( [] ),
     loadActivitiesMock: vi.fn().mockResolvedValue( {} ),
@@ -129,6 +130,7 @@ vi.mock( '#consts', async importOriginal => {
 const initTracing = vi.fn().mockResolvedValue( undefined );
 vi.mock( '#tracing', () => ( { init: initTracing } ) );
 vi.mock( '#bus', () => ( { mainEventBus: mainEventBusMock } ) );
+vi.mock( '#hooks/pending_hooks', () => ( { flushPendingHooks: flushPendingHooksMock } ) );
 
 const loadWorkflowsMock = vi.fn().mockResolvedValue( { workflows: [], entrypoint: '/fake/workflows/path.js' } );
 const loadActivitiesMock = vi.fn().mockResolvedValue( { activities: {} } );
@@ -261,6 +263,7 @@ describe( 'worker/index', () => {
     expect( catalogJobInstance.run ).toHaveBeenCalled();
 
     await settleWorker();
+    await vi.waitFor( () => expect( flushPendingHooksMock ).toHaveBeenCalledOnce() );
     expect( mockLog.info ).toHaveBeenCalledWith( 'Bye' );
   } );
 
@@ -358,6 +361,9 @@ describe( 'worker/index', () => {
     } );
     expect( mainEventBusMock.emit ).toHaveBeenCalledWith( expect.any( String ), { error } );
     await vi.waitFor( () => expect( exitMock ).toHaveBeenCalledWith( 1 ) );
+    expect( flushPendingHooksMock ).toHaveBeenCalledOnce();
+    expect( mainEventBusMock.emit.mock.invocationCallOrder[0] ).toBeLessThan( flushPendingHooksMock.mock.invocationCallOrder[0] );
+    expect( flushPendingHooksMock.mock.invocationCallOrder[0] ).toBeLessThan( exitMock.mock.invocationCallOrder[0] );
   } );
 
   it( 'throws connection monitor errors after graceful shutdown', async () => {