@outputai/core 0.10.1-next.09ed166.0 → 0.10.1-next.2e221c7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/core",
-  "version": "0.10.1-next.09ed166.0",
+  "version": "0.10.1-next.2e221c7.0",
   "description": "The core module of the output framework",
   "type": "module",
   "engines": {
@@ -69,6 +69,7 @@
     "#consts": "./src/consts.js",
     "#errors": "./src/errors.js",
     "#logger": "./src/logger/index.js",
+    "#hooks/*": "./src/hooks/*.js",
     "#helpers/*": "./src/helpers/*.js",
     "#tracing": "./src/tracing/internal_interface.js",
     "#trace_attribute": "./src/tracing/trace_attribute.js",

package/src/helpers/fetch.js

@@ -1,6 +1,6 @@
 import { FatalError } from '#errors';
 
-/** Matches red int "hot-red-pie", but not int "redact" */
+/** Matches red in "hot-red-pie", but not in "redact" */
 const wordMatcher = term => new RegExp( `(?<![a-z\\d])${term}(?![a-z\\d])`, 'i' );
 
 /** Matches red in "acquired", but not in "redact" */

package/src/hooks/index.js

@@ -1,22 +1,24 @@
 import { mainEventBus, stepEventBus } from '#bus';
 import { BusEventType } from '#consts';
 import { createChildLogger } from '#logger';
+import { pendingHooks } from './pending_hooks.js';
 
 const log = createChildLogger( 'Hooks' );
 
 /**
- * Invokes a function within a try catch and log the error
+ * Invokes a function within a try catch and log the error.
  *
  * @param {Function} fn
  * @param {any} args - Args to invoke the function with
  * @param {string} hookName - hookName to identify this hook function in the logs
  */
-const callHookCb = async ( fn, args, hookName ) => {
-  try {
-    await fn( args );
-  } catch ( error ) {
-    log.error( `${hookName} hook error`, { message: error.message, stack: error.stack } );
-  }
+const callHookCb = ( fn, args, hookName ) => {
+  const promise = Promise.resolve()
+    .then( () => fn( args ) )
+    .catch( e => log.error( `${hookName} hook error`, { message: e.message, stack: e.stack } ) )
+    .finally( () => pendingHooks.delete( promise ) );
+  pendingHooks.add( promise );
+  return promise;
 };
 
 // General Life-cycle

package/src/hooks/index.spec.js

@@ -38,6 +38,7 @@ import {
   onWorkflowError,
   onWorkflowStart
 } from './index.js';
+import { pendingHooks } from './pending_hooks.js';
 
 const workflowDetails = {
   workflowId: 'wf-1',
@@ -67,6 +68,7 @@ const aggregations = {
 describe( 'hooks/index', () => {
   beforeEach( () => {
     vi.clearAllMocks();
+    pendingHooks.clear();
     Object.keys( mainOnHandlers ).forEach( k => {
       delete mainOnHandlers[k];
     } );
@@ -158,6 +160,23 @@ describe( 'hooks/index', () => {
     } );
   } );
 
+  it( 'tracks a hook callback until it settles', async () => {
+    const deferred = { resolve: null };
+    const handlerPromise = new Promise( resolve => {
+      deferred.resolve = resolve;
+    } );
+    const handler = vi.fn( () => handlerPromise );
+    onActivityEnd( handler );
+
+    const callbackPromise = mainOnHandlers[BusEventType.ACTIVITY_END]( { activityInfo } );
+
+    expect( pendingHooks.size ).toBe( 1 );
+
+    deferred.resolve();
+    await callbackPromise;
+    expect( pendingHooks.size ).toBe( 0 );
+  } );
+
   describe( 'workflow lifecycle hooks', () => {
     const cases = [
       [ 'onWorkflowStart', onWorkflowStart, BusEventType.WORKFLOW_START, {} ],

package/src/hooks/pending_hooks.js

@@ -0,0 +1,24 @@
+import { createChildLogger } from '#logger';
+
+export const pendingHooks = new Set();
+
+const log = createChildLogger( 'Hooks' );
+const flushTimeoutMs = 30_000;
+
+/**
+ * Await all pending hooks to flush for a certain time
+ */
+export const flushPendingHooks = async () => {
+  const state = { timeout: null };
+  try {
+    const flushed = await Promise.race( [
+      Promise.allSettled( [ ...pendingHooks ] ).then( _ => true ),
+      new Promise( r => state.timeout = setTimeout( () => r( false ), flushTimeoutMs ) )
+    ] );
+    if ( !flushed ) {
+      log.warn( 'Some hook callbacks exceeded the timeout and will not be awaited', { count: pendingHooks.size } );
+    }
+  } finally {
+    clearTimeout( state.timeout );
+  }
+};

package/src/hooks/pending_hooks.spec.js

@@ -0,0 +1,62 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const logMock = vi.hoisted( () => ( { warn: vi.fn() } ) );
+
+vi.mock( '#logger', () => ( { createChildLogger: () => logMock } ) );
+
+import { flushPendingHooks, pendingHooks } from './pending_hooks.js';
+
+describe( 'pending hooks', () => {
+  beforeEach( () => {
+    pendingHooks.clear();
+    vi.clearAllMocks();
+  } );
+
+  afterEach( () => {
+    pendingHooks.clear();
+    vi.useRealTimers();
+  } );
+
+  it( 'waits for pending hooks to settle', async () => {
+    const deferred = { resolve: null };
+    const hookPromise = new Promise( resolve => {
+      deferred.resolve = resolve;
+    } );
+    pendingHooks.add( hookPromise );
+
+    const flushPromise = flushPendingHooks();
+    const state = { flushed: false };
+    flushPromise.then( () => {
+      state.flushed = true;
+    } );
+
+    await Promise.resolve();
+    expect( state.flushed ).toBe( false );
+
+    deferred.resolve();
+    await flushPromise;
+    expect( state.flushed ).toBe( true );
+  } );
+
+  it( 'stops waiting after the timeout', async () => {
+    vi.useFakeTimers();
+    pendingHooks.add( new Promise( () => {} ) );
+
+    const flushPromise = flushPendingHooks();
+    await vi.advanceTimersByTimeAsync( 30_000 );
+
+    await expect( flushPromise ).resolves.toBeUndefined();
+  } );
+
+  it( 'logs the number of hooks that exceeded the timeout', async () => {
+    vi.useFakeTimers();
+    pendingHooks.add( new Promise( () => {} ) );
+    pendingHooks.add( new Promise( () => {} ) );
+
+    const flushPromise = flushPendingHooks();
+    await vi.advanceTimersByTimeAsync( 30_000 );
+    await flushPromise;
+
+    expect( logMock.warn ).toHaveBeenCalledWith( expect.any( String ), { count: 2 } );
+  } );
+} );

package/src/interface/index.js

@@ -8,7 +8,7 @@ import {
 import { evaluator } from './evaluator.js';
 import { step } from './step.js';
 import { workflow } from './workflow.js';
-import { executeInParallel } from './workflow_utils.js';
+import { executeInParallel, hasErrorType } from './workflow_utils.js';
 import { sendHttpRequest, sendPostRequestAndAwaitWebhook } from './webhook.js';
 import { Logger } from './logger.js';
 
@@ -23,6 +23,7 @@ export {
   EvaluationVerdictResult,
   EvaluationFeedback,
   executeInParallel,
+  hasErrorType,
   sendHttpRequest,
   sendPostRequestAndAwaitWebhook
 };

package/src/interface/workflow_utils.d.ts

@@ -74,3 +74,18 @@ export declare function executeInParallel<T extends readonly ( () => Promise<unk
     onJobCompleted?: ( result: ParallelResults<T>[number] ) => void;
   }
 ): Promise<ParallelResults<T>>;
+
+/**
+ * Checks whether an error or any error in its cause chain matches an Error class.
+ *
+ * Temporal Application Failures are matched through their serialized `type`,
+ * in addition to standard `instanceof` and `name` checks.
+ *
+ * @param error - Error or failure chain to inspect
+ * @param ErrorType - Error class to match
+ * @returns Whether the Error class appears in the chain
+ */
+export declare function hasErrorType<T extends Error>(
+  error: unknown,
+  ErrorType: abstract new ( ...args: never[] ) => T
+): boolean;

package/src/interface/workflow_utils.js

@@ -48,3 +48,22 @@ export const executeInParallel = async ( { jobs, concurrency = Infinity, onJobCo
 
   return results.sort( ( a, b ) => a.index - b.index );
 };
+
+/**
+ * Check if an error, or any parts of its cause chain are instances of a given Error class.
+ * Also check .name and .type
+ *
+ * @param {Error} error
+ * @param {Error} ErrorType
+ * @returns {boolean}
+ */
+export const hasErrorType = ( error, ErrorType, depth = 0 ) => {
+  if ( depth > 20 || typeof ErrorType !== 'function' ) {
+    return false;
+  }
+  const expectedType = ErrorType.name;
+  if ( error instanceof ErrorType || error?.type === expectedType || error?.name === expectedType || error === expectedType ) {
+    return true;
+  }
+  return error?.cause ? hasErrorType( error.cause, ErrorType, depth + 1 ) : false;
+};

package/src/interface/workflow_utils.spec.js

@@ -1,5 +1,5 @@
 import { describe, it, expect, vi } from 'vitest';
-import { executeInParallel } from './workflow_utils.js';
+import { hasErrorType, executeInParallel } from './workflow_utils.js';
 
 describe( 'executeInParallel', () => {
   it( 'returns empty array for empty jobs', async () => {
@@ -188,3 +188,84 @@ describe( 'executeInParallel', () => {
     ] );
   } );
 } );
+
+describe( 'hasErrorType', () => {
+  class FooError extends Error {}
+
+  it( 'matches a direct error instance', () => {
+    expect( hasErrorType( new FooError( 'foo' ), FooError ) ).toBe( true );
+  } );
+
+  it( 'matches an error instance in the cause chain', () => {
+    const error = new Error( 'outer', {
+      cause: new Error( 'middle', {
+        cause: new FooError( 'foo' )
+      } )
+    } );
+
+    expect( hasErrorType( error, FooError ) ).toBe( true );
+  } );
+
+  it( 'matches Temporal ApplicationFailure type in the cause chain', () => {
+    const applicationFailure = Object.assign( new Error( 'foo' ), {
+      name: 'ApplicationFailure',
+      type: 'FooError'
+    } );
+    const activityFailure = Object.assign( new Error( 'Activity execution failed', {
+      cause: applicationFailure
+    } ), {
+      name: 'ActivityFailure'
+    } );
+
+    expect( hasErrorType( activityFailure, FooError ) ).toBe( true );
+  } );
+
+  it( 'matches a serialized error name', () => {
+    const error = Object.assign( new Error( 'foo' ), { name: 'FooError' } );
+
+    expect( hasErrorType( error, FooError ) ).toBe( true );
+  } );
+
+  it( 'matches an error serialized as its type name', () => {
+    expect( hasErrorType( 'FooError', FooError ) ).toBe( true );
+  } );
+
+  it( 'matches an error type in a plain object cause chain', () => {
+    const error = {
+      name: 'ActivityFailure',
+      cause: {
+        name: 'ApplicationFailure',
+        type: 'FooError'
+      }
+    };
+
+    expect( hasErrorType( error, FooError ) ).toBe( true );
+  } );
+
+  it.each( [ null, undefined, '', 42, false ] )( 'returns false for unsupported error value %j', error => {
+    expect( hasErrorType( error, FooError ) ).toBe( false );
+  } );
+
+  it( 'returns false when ErrorType is not a class', () => {
+    expect( hasErrorType( new FooError( 'foo' ), null ) ).toBe( false );
+  } );
+
+  it( 'returns false when the requested error type is absent', () => {
+    const error = new Error( 'outer', { cause: new TypeError( 'inner' ) } );
+
+    expect( hasErrorType( error, FooError ) ).toBe( false );
+  } );
+
+  it( 'keeps plain Error matching backwards compatible', () => {
+    expect( hasErrorType( new Error( 'foo' ), Error ) ).toBe( true );
+  } );
+
+  it( 'stops traversing cause chains after the depth limit', () => {
+    const createCauseChain = depth => depth === 20 ?
+      new FooError( 'foo' ) :
+      new Error( `depth-${depth}`, { cause: createCauseChain( depth + 1 ) } );
+    const root = new Error( 'root', { cause: createCauseChain( 0 ) } );
+
+    expect( hasErrorType( root, FooError ) ).toBe( false );
+  } );
+} );

package/src/worker/configs.js

@@ -36,8 +36,6 @@ const envVarSchema = z.object( {
   OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 2 * 60 * 1000 ) ), // 2min
   // Whether to send activity heartbeats (enabled by default)
   OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: z.transform( v => v === undefined ? true : isStringboolTrue( v ) ),
-  // Time to allow for hooks to flush before shutdown
-  OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 3000 ) ),
   // Set temporal worker shutdown force time
   TEMPORAL_SHUTDOWN_FORCE_TIME: durationSchema,
   // Set temporal worker shutdown grace time
@@ -70,7 +68,6 @@ export const catalogId = envVars.OUTPUT_CATALOG_ID;
 export const workerTelemetryIntervalMs = envVars.OUTPUT_WORKER_TELEMETRY_INTERVAL_MS;
 export const activityHeartbeatIntervalMs = envVars.OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS;
 export const activityHeartbeatEnabled = envVars.OUTPUT_ACTIVITY_HEARTBEAT_ENABLED;
-export const processFailureShutdownDelay = envVars.OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY;
 export const shutdownForceTime = envVars.TEMPORAL_SHUTDOWN_FORCE_TIME;
 export const shutdownGraceTime = envVars.TEMPORAL_SHUTDOWN_GRACE_TIME;
 export const grpcProxy = envVars.TEMPORAL_GRPC_PROXY;

package/src/worker/index.js

@@ -12,8 +12,8 @@ import { initInterceptors } from './interceptors/index.js';
 import { createChildLogger } from '#logger';
 import { setupInterruptionHandler } from './interruption.js';
 import { CatalogJob } from './catalog_workflow/catalog_job.js';
-import { bootstrapFetchProxy } from './proxy.js';
 import { mainEventBus } from '#bus';
+import { flushPendingHooks } from '#hooks/pending_hooks';
 import { BusEventType } from '#consts';
 import { setupTelemetry } from './telemetry.js';
 import { TemporalConnectionMonitor } from './connection_monitor.js';
@@ -61,7 +61,6 @@ const execute = async () => {
   const { activities } = await loadActivities( callerDir, workflows );
 
   mainEventBus.emit( BusEventType.WORKER_BEFORE_START );
-  bootstrapFetchProxy();
 
   log.info( 'Initializing tracing...' );
   await initTracing();
@@ -214,13 +213,18 @@ execute()
         .catch( e => log.warn( 'Connection close error', { error: e.message } ) );
     }
   } )
-  .then( () => log.info( 'Bye' ) )
-  .catch( error => {
+  .then( async () => {
+    log.info( 'Flushing hook callbacks...' );
+    await flushPendingHooks();
+    log.info( 'Bye' );
+  } )
+  .catch( async error => {
     log.error( 'Fatal error', { error: error.message, stack: error.stack } );
 
     mainEventBus.emit( BusEventType.RUNTIME_ERROR, { error } );
 
-    const timeToFlushEvent = configs.processFailureShutdownDelay;
-    log.info( `Exiting in ${timeToFlushEvent}ms` );
-    setTimeout( () => process.exit( 1 ), timeToFlushEvent );
+    log.info( 'Flushing hook callbacks...' );
+    await flushPendingHooks();
+    log.info( 'Exiting...' );
+    setTimeout( () => process.exit( 1 ) );
   } );

package/src/worker/index.spec.js

@@ -6,6 +6,7 @@ const {
   configValues,
   connectionMonitorInstance,
   createCatalogMock,
+  flushPendingHooksMock,
   initInterceptorsMock,
   mainEventBusMock,
   mockConnection,
@@ -46,7 +47,6 @@ const {
     maxConcurrentActivityTaskPolls: 5,
     maxConcurrentWorkflowTaskPolls: 5,
     workerTuner: undefined,
-    processFailureShutdownDelay: 0,
     shutdownForceTime: undefined,
     shutdownGraceTime: undefined
   };
@@ -105,6 +105,7 @@ const {
     connectionMonitorInstance,
     createCatalogMock: vi.fn().mockReturnValue( { workflows: [], activities: {} } ),
     createWorkflowsEntryPointMock: vi.fn().mockReturnValue( '/fake/workflows/path.js' ),
+    flushPendingHooksMock: vi.fn().mockResolvedValue( undefined ),
     hashSourceCodeMock: vi.fn().mockResolvedValue( 'catalog-hash' ),
     initInterceptorsMock: vi.fn().mockReturnValue( [] ),
     loadActivitiesMock: vi.fn().mockResolvedValue( {} ),
@@ -129,6 +130,7 @@ vi.mock( '#consts', async importOriginal => {
 const initTracing = vi.fn().mockResolvedValue( undefined );
 vi.mock( '#tracing', () => ( { init: initTracing } ) );
 vi.mock( '#bus', () => ( { mainEventBus: mainEventBusMock } ) );
+vi.mock( '#hooks/pending_hooks', () => ( { flushPendingHooks: flushPendingHooksMock } ) );
 
 const loadWorkflowsMock = vi.fn().mockResolvedValue( { workflows: [], entrypoint: '/fake/workflows/path.js' } );
 const loadActivitiesMock = vi.fn().mockResolvedValue( { activities: {} } );
@@ -145,7 +147,6 @@ vi.mock( './sinks.js', () => ( { sinks: {} } ) );
 vi.mock( './catalog_workflow/index.js', () => ( { createCatalog: createCatalogMock } ) );
 vi.mock( './bundler_options.js', () => ( { webpackConfigHook: vi.fn() } ) );
 vi.mock( './interceptors/index.js', () => ( { initInterceptors: initInterceptorsMock } ) );
-vi.mock( './proxy.js', () => ( { bootstrapFetchProxy: vi.fn() } ) );
 vi.mock( './telemetry.js', () => ( { setupTelemetry: setupTelemetryMock } ) );
 vi.mock( './interruption.js', () => ( { setupInterruptionHandler: setupInterruptionHandlerMock } ) );
 vi.mock( './global_functions.js', () => ( { bindGlobalFunctions: bindGlobalFunctionsMock } ) );
@@ -262,6 +263,7 @@ describe( 'worker/index', () => {
     expect( catalogJobInstance.run ).toHaveBeenCalled();
 
     await settleWorker();
+    await vi.waitFor( () => expect( flushPendingHooksMock ).toHaveBeenCalledOnce() );
     expect( mockLog.info ).toHaveBeenCalledWith( 'Bye' );
   } );
 
@@ -359,6 +361,9 @@ describe( 'worker/index', () => {
     } );
     expect( mainEventBusMock.emit ).toHaveBeenCalledWith( expect.any( String ), { error } );
     await vi.waitFor( () => expect( exitMock ).toHaveBeenCalledWith( 1 ) );
+    expect( flushPendingHooksMock ).toHaveBeenCalledOnce();
+    expect( mainEventBusMock.emit.mock.invocationCallOrder[0] ).toBeLessThan( flushPendingHooksMock.mock.invocationCallOrder[0] );
+    expect( flushPendingHooksMock.mock.invocationCallOrder[0] ).toBeLessThan( exitMock.mock.invocationCallOrder[0] );
   } );
 
   it( 'throws connection monitor errors after graceful shutdown', async () => {

package/src/worker/proxy.js

@@ -1,20 +0,0 @@
-import { EnvHttpProxyAgent, setGlobalDispatcher } from 'undici';
-import { createChildLogger } from '#logger';
-
-const log = createChildLogger( 'Proxy' );
-
-/**
- * Routes all `fetch()` calls (including those inside Temporal activities)
- * through an HTTP/HTTPS proxy when standard proxy env vars are set.
- * No-op when none are set.
- */
-export const bootstrapFetchProxy = () => {
-  const httpProxyUrl = process.env.http_proxy ?? process.env.HTTP_PROXY;
-  const httpsProxyUrl = process.env.https_proxy ?? process.env.HTTPS_PROXY;
-
-  if ( httpProxyUrl?.length > 0 || httpsProxyUrl?.length > 0 ) {
-    log.info( 'Proxy env vars detected, setting up global fetch dispatcher EnvHttpProxyAgent', { httpProxyUrl, httpsProxyUrl } );
-    /* Ignore HTTP/2. Check: https://github.com/growthxai/output/issues/299 */
-    setGlobalDispatcher( new EnvHttpProxyAgent( { allowH2: false } ) );
-  }
-};

package/src/worker/proxy.spec.js

@@ -1,51 +0,0 @@
-import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
-
-const mockSetGlobalDispatcher = vi.fn();
-const MockEnvHttpProxyAgent = vi.fn();
-const mockLogger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
-
-vi.mock( 'undici', () => ( {
-  EnvHttpProxyAgent: MockEnvHttpProxyAgent,
-  setGlobalDispatcher: mockSetGlobalDispatcher
-} ) );
-
-vi.mock( '#logger', () => ( {
-  createChildLogger: () => mockLogger
-} ) );
-
-describe( 'worker/proxy', () => {
-  const proxyEnv = [ 'http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY' ];
-
-  beforeEach( () => {
-    vi.clearAllMocks();
-    for ( const key of proxyEnv ) {
-      delete process.env[key];
-    }
-  } );
-
-  afterEach( () => {
-    vi.resetModules();
-  } );
-
-  it( 'does nothing when no proxy env vars are set', async () => {
-    const { bootstrapFetchProxy } = await import( './proxy.js' );
-    bootstrapFetchProxy();
-
-    expect( MockEnvHttpProxyAgent ).not.toHaveBeenCalled();
-    expect( mockSetGlobalDispatcher ).not.toHaveBeenCalled();
-    expect( mockLogger.info ).not.toHaveBeenCalled();
-  } );
-
-  it( 'sets global dispatcher when proxy URL is available', async () => {
-    process.env.HTTP_PROXY = 'http://proxy:8080/';
-    const { bootstrapFetchProxy } = await import( './proxy.js' );
-    bootstrapFetchProxy();
-
-    expect( MockEnvHttpProxyAgent ).toHaveBeenCalledWith( { allowH2: false } );
-    expect( mockSetGlobalDispatcher ).toHaveBeenCalledTimes( 1 );
-    expect( mockLogger.info ).toHaveBeenCalledWith(
-      'Proxy env vars detected, setting up global fetch dispatcher EnvHttpProxyAgent',
-      { httpProxyUrl: 'http://proxy:8080/', httpsProxyUrl: undefined }
-    );
-  } );
-} );