@outputai/core 0.7.1-next.de30052.0 → 0.8.0

package/src/worker/connection_monitor.spec.js

@@ -0,0 +1,199 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { TemporalConnectionMonitor } from './connection_monitor.js';
+
+const SERVING = 1;
+const NOT_SERVING = 2;
+const CHECK_TIMEOUT_MS = 50;
+const CHECK_INTERVAL_MS = 100;
+
+const { scheduledDelays, delayMock, mockLogger } = vi.hoisted( () => {
+  const scheduledDelays = [];
+  const delayMock = vi.fn( ( ms, value, options ) => new Promise( resolve => {
+    scheduledDelays.push( { ms, value, options, resolve } );
+  } ) );
+
+  return {
+    scheduledDelays,
+    delayMock,
+    mockLogger: {
+      info: vi.fn(),
+      warn: vi.fn()
+    }
+  };
+} );
+
+vi.mock( 'node:timers/promises', () => ( { setTimeout: delayMock } ) );
+vi.mock( '#logger', () => ( { createChildLogger: vi.fn( () => mockLogger ) } ) );
+
+const createConnection = check => ( {
+  healthService: { check }
+} );
+
+const createMonitor = ( check, overrides = {} ) => new TemporalConnectionMonitor( createConnection( check ), {
+  checkIntervalMs: CHECK_INTERVAL_MS,
+  checkTimeoutMs: CHECK_TIMEOUT_MS,
+  ...overrides
+} );
+
+const flushPromises = async () => Array
+  .from( { length: 10 } )
+  .reduce( promise => promise.then( () => Promise.resolve() ), Promise.resolve() );
+
+const resolveNextDelay = ms => {
+  const index = scheduledDelays.findIndex( delay => delay.ms === ms );
+  expect( index ).not.toBe( -1 );
+  const [ scheduled ] = scheduledDelays.splice( index, 1 );
+  scheduled.resolve( scheduled.value );
+};
+
+describe( 'TemporalConnectionMonitor', () => {
+  beforeEach( () => {
+    vi.clearAllMocks();
+    scheduledDelays.length = 0;
+  } );
+
+  it( 'logs healthy when the connection is serving', async () => {
+    const check = vi.fn().mockResolvedValue( { status: SERVING } );
+    const monitor = createMonitor( check );
+
+    const run = monitor.start();
+    await flushPromises();
+
+    expect( check ).toHaveBeenCalledWith( {} );
+    expect( mockLogger.info ).toHaveBeenCalledWith( 'Healthy' );
+    expect( delayMock ).toHaveBeenCalledWith( CHECK_TIMEOUT_MS, 0, { ref: false } );
+    expect( delayMock ).toHaveBeenCalledWith( CHECK_INTERVAL_MS, 0, { ref: false } );
+    expect( monitor.running ).toBe( true );
+
+    await monitor.stop();
+    await run;
+
+    expect( monitor.running ).toBe( false );
+  } );
+
+  it( 'logs transient timeout failures before retrying', async () => {
+    const check = vi.fn().mockReturnValue( new Promise( () => {} ) );
+    const monitor = createMonitor( check );
+
+    monitor.start();
+    resolveNextDelay( CHECK_TIMEOUT_MS );
+    await flushPromises();
+
+    expect( mockLogger.warn ).toHaveBeenCalledWith( 'Connection unhealthy', {
+      error: 'Connection health check timed out',
+      failures: 1
+    } );
+
+    await monitor.stop();
+  } );
+
+  it( 'logs recovered after a transient failure succeeds', async () => {
+    const check = vi.fn()
+      .mockRejectedValueOnce( new Error( 'temporary outage' ) )
+      .mockResolvedValueOnce( { status: SERVING } );
+    const monitor = createMonitor( check );
+
+    monitor.start();
+    await flushPromises();
+
+    expect( mockLogger.warn ).toHaveBeenCalledWith( 'Connection unhealthy', {
+      error: 'temporary outage',
+      failures: 1
+    } );
+
+    resolveNextDelay( CHECK_INTERVAL_MS );
+    await flushPromises();
+
+    expect( mockLogger.info ).toHaveBeenCalledWith( 'Recovered' );
+
+    await monitor.stop();
+  } );
+
+  it( 'stores connection loss error and calls callback after max consecutive failures', async () => {
+    const error = new Error( 'connection refused' );
+    const check = vi.fn().mockRejectedValue( error );
+    const connectionLost = vi.fn();
+    const monitor = createMonitor( check );
+
+    monitor.onConnectionLost( connectionLost );
+    monitor.start();
+
+    await flushPromises();
+    resolveNextDelay( CHECK_INTERVAL_MS );
+    await flushPromises();
+    resolveNextDelay( CHECK_INTERVAL_MS );
+    await flushPromises();
+
+    expect( mockLogger.warn ).toHaveBeenCalledTimes( 3 );
+    expect( mockLogger.warn ).toHaveBeenCalledWith( 'Connection lost', {
+      error: 'connection refused',
+      failures: 3
+    } );
+    expect( connectionLost ).toHaveBeenCalledWith( error );
+    expect( monitor.connectionLossError ).toBe( error );
+    expect( monitor.running ).toBe( false );
+  } );
+
+  it( 'treats non-serving health status as a failure', async () => {
+    const check = vi.fn().mockResolvedValue( { status: NOT_SERVING } );
+    const monitor = createMonitor( check );
+
+    monitor.start();
+    await flushPromises();
+
+    expect( mockLogger.warn ).toHaveBeenCalledWith( 'Connection unhealthy', {
+      error: `Connection not serving (status ${NOT_SERVING})`,
+      failures: 1
+    } );
+
+    await monitor.stop();
+  } );
+
+  it( 'returns the same lifecycle promise when started more than once', async () => {
+    const check = vi.fn().mockReturnValue( new Promise( () => {} ) );
+    const monitor = createMonitor( check );
+
+    const firstRun = monitor.start();
+    const secondRun = monitor.start();
+
+    expect( secondRun ).toBe( firstRun );
+    expect( check ).toHaveBeenCalledOnce();
+
+    await monitor.stop();
+  } );
+
+  it( 'stops without calling connection lost callback for in-flight health checks', async () => {
+    const check = vi.fn().mockReturnValue( new Promise( () => {} ) );
+    const connectionLost = vi.fn();
+    const monitor = createMonitor( check, { maxFailures: 1 } );
+
+    monitor.onConnectionLost( connectionLost );
+    monitor.start();
+
+    expect( monitor.running ).toBe( true );
+
+    await monitor.stop();
+
+    expect( connectionLost ).not.toHaveBeenCalled();
+    expect( monitor.connectionLossError ).toBeNull();
+    expect( monitor.running ).toBe( false );
+  } );
+
+  it( 'applies timing and failure threshold overrides', async () => {
+    const error = new Error( 'fast failure' );
+    const check = vi.fn().mockRejectedValue( error );
+    const connectionLost = vi.fn();
+    const monitor = createMonitor( check, {
+      maxFailures: 1,
+      checkIntervalMs: 7,
+      checkTimeoutMs: 3
+    } );
+
+    monitor.onConnectionLost( connectionLost );
+    await monitor.start();
+
+    expect( delayMock ).toHaveBeenCalledWith( 3, 0, { ref: false } );
+    expect( delayMock ).not.toHaveBeenCalledWith( 7, 0, { ref: false } );
+    expect( connectionLost ).toHaveBeenCalledWith( error );
+  } );
+} );

package/src/worker/index.js

@@ -1,55 +1,64 @@
 import { Worker, NativeConnection } from '@temporalio/worker';
 import * as configs from './configs.js';
-import { loadActivities, loadHooks, loadWorkflows, createWorkflowsEntryPoint } from './loader.js';
+import { loadActivities } from './loader/activities.js';
+import { loadWorkflows } from './loader/workflows.js';
+import { loadHooks } from './loader/hooks.js';
+import { hashSourceCode } from './loader/tools.js';
 import { sinks } from './sinks.js';
 import { createCatalog } from './catalog_workflow/index.js';
 import { init as initTracing } from '#tracing';
 import { webpackConfigHook } from './bundler_options.js';
 import { initInterceptors } from './interceptors/index.js';
 import { createChildLogger } from '#logger';
-import { registerShutdown } from './shutdown.js';
-import { startCatalog } from './start_catalog.js';
+import { setupInterruptionHandler } from './interruption.js';
+import { CatalogJob } from './catalog_workflow/catalog_job.js';
 import { bootstrapFetchProxy } from './proxy.js';
 import { messageBus } from '#bus';
 import { BusEventType } from '#consts';
-import { hashSourceCode } from './loader_tools.js';
-import { setupTelemetry } from './setup_telemetry.js';
+import { setupTelemetry } from './telemetry.js';
+import { TemporalConnectionMonitor } from './connection_monitor.js';
+import { runOnce } from '#utils';
+
 import './log_hooks.js';
 
 const log = createChildLogger( 'Worker' );
 
+const {
+  address,
+  apiKey,
+  namespace,
+  taskQueue,
+  grpcProxy,
+  maxConcurrentWorkflowTaskExecutions,
+  maxConcurrentActivityTaskExecutions,
+  maxCachedWorkflows,
+  maxConcurrentActivityTaskPolls,
+  maxConcurrentWorkflowTaskPolls
+} = configs;
+
+const state = {
+  connection: null,
+  connectionMonitor: null,
+  catalogJob: null,
+  workerError: null
+};
+
 // Get caller directory from command line arguments
 const callerDir = process.argv[2];
 
-( async () => {
-  const {
-    address,
-    apiKey,
-    namespace,
-    taskQueue,
-    grpcProxy,
-    maxConcurrentWorkflowTaskExecutions,
-    maxConcurrentActivityTaskExecutions,
-    maxCachedWorkflows,
-    maxConcurrentActivityTaskPolls,
-    maxConcurrentWorkflowTaskPolls
-  } = configs;
-
+const execute = async () => {
   log.info( 'Loading config...', { callerDir } );
   await loadHooks( callerDir );
 
   log.info( 'Loading workflows...', { callerDir } );
-  const workflows = await loadWorkflows( callerDir );
+  const { workflows, entrypoint: workflowsPath } = await loadWorkflows( callerDir );
 
   log.info( 'Loading activities...', { callerDir } );
-  const activities = await loadActivities( callerDir, workflows );
+  const { activities } = await loadActivities( callerDir, workflows );
 
   messageBus.emit( BusEventType.WORKER_BEFORE_START );
   bootstrapFetchProxy();
 
-  log.info( 'Creating worker entry point...' );
-  const workflowsPath = createWorkflowsEntryPoint( workflows );
-
   log.info( 'Initializing tracing...' );
   await initTracing();
 
@@ -64,17 +73,23 @@ const callerDir = process.argv[2];
   if ( proxy ) {
     log.info( 'Using gRPC proxy', { targetHost: grpcProxy } );
   }
-  const connection = await NativeConnection.connect( { address, tls: Boolean( apiKey ), apiKey, proxy } );
+  state.connection = await NativeConnection.connect( { address, tls: Boolean( apiKey ), apiKey, proxy } );
+
+  log.info( 'Creating connection monitor...' );
+  state.connectionMonitor = new TemporalConnectionMonitor( state.connection );
+
+  log.info( 'Creating catalog job manager...' );
+  state.catalogJob = new CatalogJob( { connection: state.connection, namespace, catalog, catalogHash } );
 
   log.info( 'Creating worker...' );
   const worker = await Worker.create( {
-    connection,
+    connection: state.connection,
     namespace,
     taskQueue,
     workflowsPath,
     activities,
     sinks,
-    interceptors: initInterceptors( { activities, workflows, connection } ),
+    interceptors: initInterceptors( { activities, workflows } ),
     maxConcurrentWorkflowTaskExecutions,
     maxConcurrentActivityTaskExecutions,
     maxCachedWorkflows,
@@ -83,22 +98,112 @@ const callerDir = process.argv[2];
     bundlerOptions: { webpackConfigHook }
   } );
 
-  registerShutdown( { worker, log } );
-
+  log.info( 'Setting up telemetry...' );
   setupTelemetry( { worker } );
 
-  log.info( 'Running worker...' );
-  await Promise.all( [ worker.run(), startCatalog( { connection, namespace, catalog, catalogHash } ) ] );
+  /**
+   * NOTE
+   * Temporal worker shutdown is a bit odd.
+   * worker.run() is an async job that only resolves when calling worker.shutdown().
+   * But worker.shutdown() is not async and returns nothing, so there is no way to await it.
+   * All code that needs to run after shutdown needs to be after `await worker.run()`.
+   *
+   * The following code needs to cover these scenarios:
+   * 1. Connection monitor detects connection loss
+   * 2. Catalog.run() has a failure
+   * 3. Interruption is received
+   * 4. Worker throws an error
+   *
+   * For each scenario all promises in the Promise.all() need to be completed via functions:
+   * connectionMonitor.stop(), catalogJob.interrupt(), worker.shutdown()
+   */
+
+  /**
+   * Graceful shutdown
+   * Triggers the actions that will resolve all promises in the Promise.all(), so the code can resume
+   */
+  const shutdown = runOnce( () => {
+    log.info( 'Shutdown started...' );
+    if ( worker.getStatus().runState === 'RUNNING' ) {
+      worker.shutdown();
+    }
+    state.connectionMonitor.stop();
+    state.catalogJob.interrupt();
+  } );
 
-  log.info( 'Closing connection...' );
-  await connection.close();
+  /** When receiving an interruption, call shutdown */
+  setupInterruptionHandler( shutdown );
 
-  log.info( 'Bye' );
+  /** When the connection is lost, call shutdown */
+  state.connectionMonitor.onConnectionLost( shutdown );
 
-  process.exit( 0 );
-} )().catch( error => {
-  log.error( 'Fatal error', { message: error.message, stack: error.stack } );
-  messageBus.emit( BusEventType.RUNTIME_ERROR, { error } );
-  log.info( `Exiting in ${configs.processFailureShutdownDelay}ms` );
-  setTimeout( () => process.exit( 1 ), configs.processFailureShutdownDelay );
-} );
+  /** If the catalog job manager fails, call shutdown */
+  state.catalogJob.onError( shutdown );
+
+  /**
+   * Runs the worker, connection monitor and catalogJob (ephemeral)
+   * None of these will reject in normal conditions. Errors need to be inspected later
+   * They will resolve only when calling the actions in shutdown(),
+   * except catalogJob, which can resolve by itself given a bit of time
+   */
+  log.info( 'Running worker...' );
+  await Promise.all( [
+    // When the worker fails, store the error and call shutdown
+    worker.run().catch( error => {
+      state.workerError = error;
+      shutdown();
+    } ),
+    state.connectionMonitor.start(),
+    state.catalogJob.run()
+  ] );
+
+  log.info( 'Worker terminated' );
+
+  /** After the Promise.all() is resolved, check which services had an error, since none rejects the promise */
+  const error =
+    state.connectionMonitor.connectionLossError ??
+    state.catalogJob.error ??
+    state.workerError;
+
+  /** If any error is found, throws it, so this process can exit with code=1 (failure) */
+  if ( error ) {
+    throw error;
+  }
+};
+
+execute()
+  .finally( async () => {
+    /**
+     * This will make sure that if we had any uncaught failures, everything is tore down.
+     * Ignore any errors here in order to not mask actually errors from before and because at this point
+     * the code is already shutting down, so no need to throw anyway.
+     *
+     * worker.shutdown() is not tried here, because it cannot be awaited. By this point is safe to say
+     * that worker never started, already crashed or was stopped anyway.
+     */
+    if ( state.connectionMonitor?.running ) {
+      log.info( 'Stopping connection monitor...' );
+      await state.connectionMonitor.stop()
+        .catch( e => log.warn( 'Connection monitor stop error', { error: e.message } ) );
+    }
+    if ( state.catalogJob?.running ) {
+      log.info( 'Interrupting catalog job...' );
+      await state.catalogJob.interrupt()
+        .catch( e => log.warn( 'Catalog job interruption error', { error: e.message } ) );
+    }
+    if ( state.connection ) {
+      log.info( 'Closing connection...' );
+      await state.connection.close()
+        .catch( e => log.warn( 'Connection close error', { error: e.message } ) );
+    }
+  } )
+  .then( () => log.info( 'Bye' ) )
+  .catch( error => {
+    log.error( 'Fatal error', { error: error.message, stack: error.stack } );
+
+    messageBus.emit( BusEventType.RUNTIME_ERROR, { error } );
+
+    const timeToFlushEvent = configs.processFailureShutdownDelay;
+    log.info( `Exiting in ${timeToFlushEvent}ms` );
+    setTimeout( () => process.exit( 1 ), timeToFlushEvent );
+  } );