@outputai/core 0.12.0 → 0.12.1-next.1243f78.0

package/src/worker/index.js

@@ -10,8 +10,8 @@ import { init as initTracing } from '#tracing';
 import { webpackConfigHook } from './bundler_options.js';
 import { initInterceptors } from './interceptors/index.js';
 import { createChildLogger } from '#logger';
-import { setupInterruptionHandler } from './interruption.js';
-import { CatalogJob } from './catalog_workflow/catalog_job.js';
+import { setupInterruptionHandler, KillSignError } from './interruption.js';
+import { CatalogPublisher } from './catalog_workflow/catalog_publisher.js';
 import { mainEventBus } from '#bus';
 import { flushPendingHooks } from '#hooks/pending_hooks';
 import { BusEventType } from '#consts';
@@ -19,9 +19,10 @@ import { setupTelemetry } from './telemetry.js';
 import { TemporalConnectionMonitor } from './connection_monitor.js';
 import { bindGlobalFunctions } from './global_functions.js';
 import { setupClientConfig } from '#temporal/client';
-import { runOnce } from '#helpers/function';
 import { serializeError } from '#helpers/error_serializer';
 import { setupTemporalLogger } from './temporal_logger.js';
+import { WorkerRunner } from './worker_runner.js';
+import { shutdownServices } from './shutdown.js';
 
 import './log_hooks.js';
 
@@ -43,197 +44,138 @@ const {
   workerTuner
 } = configs;
 
-const state = {
+const services = {
   connection: null,
   connectionMonitor: null,
-  catalogJob: null,
-  workerError: null
+  catalogPublisher: null,
+  workerRunner: null
 };
 
 // Get caller directory from command line arguments
 const callerDir = process.argv[2];
 
+const abortController = new AbortController();
+const { signal } = abortController;
+
+setupInterruptionHandler( abortController );
+
+/** Run a given function or abort code if global signal was aborted */
+const run = cb => signal.throwIfAborted() ?? cb();
+
 const execute = async () => {
-  log.info( 'Setup Temporal Logger' );
-  setupTemporalLogger();
+  log.info( 'Setting up Temporal Logger...' );
+  run( setupTemporalLogger );
 
   log.info( 'Loading config...', { callerDir } );
-  await loadHooks( callerDir );
+  await run( () => loadHooks( callerDir ) );
 
   log.info( 'Loading workflows...', { callerDir } );
-  const { workflows, entrypoint: workflowsPath } = await loadWorkflows( callerDir );
+  const { workflows, entrypoint: workflowsPath } = await run( () => loadWorkflows( callerDir ) );
 
   log.info( 'Loading activities...', { callerDir } );
-  const { activities } = await loadActivities( callerDir, workflows );
+  const { activities } = await run( () => loadActivities( callerDir, workflows ) );
 
   mainEventBus.emit( BusEventType.WORKER_BEFORE_START );
 
   log.info( 'Initializing tracing...' );
-  await initTracing();
+  await run( initTracing );
 
   log.info( 'Creating workflows catalog...' );
-  const catalog = createCatalog( { workflows, activities } );
+  const catalog = run( () => createCatalog( { workflows, activities } ) );
 
   log.info( 'Computing catalog source code hash...' );
-  const catalogHash = await hashSourceCode( callerDir );
+  const catalogHash = await run( () => hashSourceCode( callerDir ) );
+
+  log.info( 'Binding globals...' );
+  run( bindGlobalFunctions );
 
   log.info( 'Connecting Temporal...' );
   const proxy = grpcProxy ? { type: 'http-connect', targetHost: grpcProxy } : undefined;
   if ( proxy ) {
     log.info( 'Using gRPC proxy', { targetHost: grpcProxy } );
   }
+  const connection = await run( () => NativeConnection.connect( { address, tls: Boolean( apiKey ), apiKey, proxy } ) );
+  services.connection = connection;
 
-  bindGlobalFunctions();
+  log.info( 'Setting up temporal endpoint...' );
+  run( () => setupClientConfig( { connection, namespace } ) );
 
-  state.connection = await NativeConnection.connect( { address, tls: Boolean( apiKey ), apiKey, proxy } );
-
-  // add configs for the /temporal endpoint
-  setupClientConfig( { connection: state.connection, namespace } );
+  log.info( 'Creating catalog publisher...' );
+  services.catalogPublisher = run( () => new CatalogPublisher( { connection, namespace, catalog, catalogHash, signal } ) );
 
   log.info( 'Creating connection monitor...' );
-  state.connectionMonitor = new TemporalConnectionMonitor( state.connection );
-
-  log.info( 'Creating catalog job manager...' );
-  state.catalogJob = new CatalogJob( { connection: state.connection, namespace, catalog, catalogHash } );
-
-  log.info( 'Creating worker...' );
-  if ( workerTuner ) {
-    log.info( 'Using worker tuner options', { ...workerTuner } );
-  }
-  const worker = await Worker.create( {
-    connection: state.connection,
-    namespace,
-    taskQueue,
-    workflowsPath,
-    activities,
-    sinks,
-    interceptors: initInterceptors( { activities, workflows } ),
-    // tuner isn't compatible with concurrent task executions configs
-    ...( workerTuner ? {
-      tuner: workerTuner
-    } : {
-      maxConcurrentWorkflowTaskExecutions,
-      maxConcurrentActivityTaskExecutions
-    } ),
-    maxCachedWorkflows,
-    maxConcurrentActivityTaskPolls,
-    maxConcurrentWorkflowTaskPolls,
-    bundlerOptions: { webpackConfigHook },
-    ...( shutdownForceTime !== undefined && { shutdownForceTime } ),
-    ...( shutdownGraceTime !== undefined && { shutdownGraceTime } )
-  } );
+  services.connectionMonitor = run( () => new TemporalConnectionMonitor( { connection, signal } ) );
 
-  log.info( 'Setting up telemetry...' );
-  setupTelemetry( { worker } );
+  log.info( 'Publishing catalog workflow...' );
+  await run( () => services.catalogPublisher.run() );
 
-  /**
-   * NOTE
-   * Temporal worker shutdown is a bit odd.
-   * worker.run() is an async job that only resolves when calling worker.shutdown().
-   * But worker.shutdown() is not async and returns nothing, so there is no way to await it.
-   * All code that needs to run after shutdown needs to be after `await worker.run()`.
-   *
-   * The following code needs to cover these scenarios:
-   * 1. Connection monitor detects connection loss
-   * 2. Catalog.run() has a failure
-   * 3. Interruption is received
-   * 4. Worker throws an error
-   *
-   * For each scenario all promises in the Promise.all() need to be completed via functions:
-   * connectionMonitor.stop(), catalogJob.interrupt(), worker.shutdown()
-   */
-
-  /**
-   * Graceful shutdown
-   * Triggers the actions that will resolve all promises in the Promise.all(), so the code can resume
-   */
-  const shutdown = runOnce( () => {
-    log.info( 'Shutdown started...' );
-    if ( worker.getStatus().runState === 'RUNNING' ) {
-      worker.shutdown();
+  log.info( 'Creating Temporal worker...' );
+  const worker = await run( () => {
+    if ( workerTuner ) {
+      log.info( 'Using worker tuner options', { ...workerTuner } );
     }
-    state.connectionMonitor.stop();
-    state.catalogJob.interrupt();
-  } );
 
-  /** When receiving an interruption, call shutdown */
-  setupInterruptionHandler( shutdown );
+    return Worker.create( {
+      connection,
+      namespace,
+      taskQueue,
+      workflowsPath,
+      activities,
+      sinks,
+      interceptors: initInterceptors( { activities, workflows } ),
+      // tuner isn't compatible with concurrent task executions configs
+      ...( workerTuner ? {
+        tuner: workerTuner
+      } : {
+        maxConcurrentWorkflowTaskExecutions,
+        maxConcurrentActivityTaskExecutions
+      } ),
+      maxCachedWorkflows,
+      maxConcurrentActivityTaskPolls,
+      maxConcurrentWorkflowTaskPolls,
+      bundlerOptions: { webpackConfigHook },
+      ...( shutdownForceTime !== undefined && { shutdownForceTime } ),
+      ...( shutdownGraceTime !== undefined && { shutdownGraceTime } )
+    } );
+  } );
 
-  /** When the connection is lost, call shutdown */
-  state.connectionMonitor.onConnectionLost( shutdown );
+  log.info( 'Setting up telemetry...' );
+  run( () => setupTelemetry( { worker } ) );
 
-  /** If the catalog job manager fails, call shutdown */
-  state.catalogJob.onError( shutdown );
+  log.info( 'Creating worker runner...' );
+  services.workerRunner = new WorkerRunner( { worker, signal } );
 
   /**
-   * Runs the worker, connection monitor and catalogJob (ephemeral)
-   * None of these will reject in normal conditions. Errors need to be inspected later
-   * They will resolve only when calling the actions in shutdown(),
-   * except catalogJob, which can resolve by itself given a bit of time
+   * Runs the worker and connection monitor together
+   * They will stop only upon receiving a wired kill signal (eg SIGINT), which resolves the promise
    */
   log.info( 'Running worker...' );
-  await Promise.all( [
-    // When the worker fails, store the error and call shutdown
-    worker.run().catch( error => {
-      state.workerError = error;
-      shutdown();
-    } ),
-    state.connectionMonitor.start(),
-    state.catalogJob.run()
-  ] );
+  await run( () => Promise.race( [
+    services.workerRunner.start(),
+    services.connectionMonitor.start()
+  ] ) );
 
+  signal.throwIfAborted();
   log.info( 'Worker terminated' );
-
-  /** After the Promise.all() is resolved, check which services had an error, since none rejects the promise */
-  const error =
-    state.connectionMonitor.connectionLossError ??
-    state.catalogJob.error ??
-    state.workerError;
-
-  /** If any error is found, throws it, so this process can exit with code=1 (failure) */
-  if ( error ) {
-    throw error;
-  }
 };
 
 execute()
+  .catch( async error => abortController.abort( error ) )
   .finally( async () => {
-    /**
-     * This will make sure that if we had any uncaught failures, everything is tore down.
-     * Ignore any errors here in order to not mask actually errors from before and because at this point
-     * the code is already shutting down, so no need to throw anyway.
-     *
-     * worker.shutdown() is not tried here, because it cannot be awaited. By this point is safe to say
-     * that worker never started, already crashed or was stopped anyway.
-     */
-    if ( state.connectionMonitor?.running ) {
-      log.info( 'Stopping connection monitor...' );
-      await state.connectionMonitor.stop()
-        .catch( e => log.warn( 'Connection monitor stop error', { error: e.message } ) );
-    }
-    if ( state.catalogJob?.running ) {
-      log.info( 'Interrupting catalog job...' );
-      await state.catalogJob.interrupt()
-        .catch( e => log.warn( 'Catalog job interruption error', { error: e.message } ) );
-    }
-    if ( state.connection ) {
-      log.info( 'Closing connection...' );
-      await state.connection.close()
-        .catch( e => log.warn( 'Connection close error', { error: e.message } ) );
-    }
-  } )
-  .then( async () => {
-    log.info( 'Flushing hook callbacks...' );
-    await flushPendingHooks();
-    log.info( 'Bye' );
-  } )
-  .catch( async error => {
-    log.error( 'Fatal error', { error: serializeError( error ) } );
+    const shutdownFailures = await shutdownServices( services );
 
-    mainEventBus.emit( BusEventType.RUNTIME_ERROR, { error } );
+    const hasError = signal.aborted && !( signal.reason instanceof KillSignError );
+    if ( hasError ) {
+      log.error( 'Worker error', { error: serializeError( signal.reason ) } );
+      mainEventBus.emit( BusEventType.RUNTIME_ERROR, { error: signal.reason } );
+    }
 
     log.info( 'Flushing hook callbacks...' );
     await flushPendingHooks();
-    log.info( 'Exiting...' );
-    setTimeout( () => process.exit( 1 ) );
+
+    const exitCode = shutdownFailures.some( v => v.service === 'worker' ) || hasError ? 1 : 0;
+    setTimeout( () => {
+      log.info( 'Bye' );
+      process.exit( exitCode );
+    } );
   } );