@outputai/core 0.7.0 → 0.7.1-next.0e958f3.0

package/src/worker/index.js

@@ -7,34 +7,44 @@ import { init as initTracing } from '#tracing';
 import { webpackConfigHook } from './bundler_options.js';
 import { initInterceptors } from './interceptors/index.js';
 import { createChildLogger } from '#logger';
-import { registerShutdown } from './shutdown.js';
-import { startCatalog } from './start_catalog.js';
+import { setupInterruptionHandler } from './interruption.js';
+import { CatalogJob } from './catalog_workflow/catalog_job.js';
 import { bootstrapFetchProxy } from './proxy.js';
 import { messageBus } from '#bus';
 import { BusEventType } from '#consts';
 import { hashSourceCode } from './loader_tools.js';
-import { setupTelemetry } from './setup_telemetry.js';
+import { setupTelemetry } from './telemetry.js';
+import { TemporalConnectionMonitor } from './connection_monitor.js';
+import { runOnce } from '#utils';
+
 import './log_hooks.js';
 
 const log = createChildLogger( 'Worker' );
 
+const {
+  address,
+  apiKey,
+  namespace,
+  taskQueue,
+  grpcProxy,
+  maxConcurrentWorkflowTaskExecutions,
+  maxConcurrentActivityTaskExecutions,
+  maxCachedWorkflows,
+  maxConcurrentActivityTaskPolls,
+  maxConcurrentWorkflowTaskPolls
+} = configs;
+
+const state = {
+  connection: null,
+  connectionMonitor: null,
+  catalogJob: null,
+  workerError: null
+};
+
 // Get caller directory from command line arguments
 const callerDir = process.argv[2];
 
-( async () => {
-  const {
-    address,
-    apiKey,
-    namespace,
-    taskQueue,
-    grpcProxy,
-    maxConcurrentWorkflowTaskExecutions,
-    maxConcurrentActivityTaskExecutions,
-    maxCachedWorkflows,
-    maxConcurrentActivityTaskPolls,
-    maxConcurrentWorkflowTaskPolls
-  } = configs;
-
+const execute = async () => {
   log.info( 'Loading config...', { callerDir } );
   await loadHooks( callerDir );
 
@@ -64,17 +74,23 @@ const callerDir = process.argv[2];
   if ( proxy ) {
     log.info( 'Using gRPC proxy', { targetHost: grpcProxy } );
   }
-  const connection = await NativeConnection.connect( { address, tls: Boolean( apiKey ), apiKey, proxy } );
+  state.connection = await NativeConnection.connect( { address, tls: Boolean( apiKey ), apiKey, proxy } );
+
+  log.info( 'Creating connection monitor...' );
+  state.connectionMonitor = new TemporalConnectionMonitor( state.connection );
+
+  log.info( 'Creating catalog job manager...' );
+  state.catalogJob = new CatalogJob( { connection: state.connection, namespace, catalog, catalogHash } );
 
   log.info( 'Creating worker...' );
   const worker = await Worker.create( {
-    connection,
+    connection: state.connection,
     namespace,
     taskQueue,
     workflowsPath,
     activities,
     sinks,
-    interceptors: initInterceptors( { activities, workflows, connection } ),
+    interceptors: initInterceptors( { activities, workflows } ),
     maxConcurrentWorkflowTaskExecutions,
     maxConcurrentActivityTaskExecutions,
     maxCachedWorkflows,
@@ -83,22 +99,112 @@ const callerDir = process.argv[2];
     bundlerOptions: { webpackConfigHook }
   } );
 
-  registerShutdown( { worker, log } );
-
+  log.info( 'Setting up telemetry...' );
   setupTelemetry( { worker } );
 
-  log.info( 'Running worker...' );
-  await Promise.all( [ worker.run(), startCatalog( { connection, namespace, catalog, catalogHash } ) ] );
+  /**
+   * NOTE
+   * Temporal worker shutdown is a bit odd.
+   * worker.run() is an async job that only resolves when calling worker.shutdown().
+   * But worker.shutdown() is not async and returns nothing, so there is no way to await it.
+   * All code that needs to run after shutdown needs to be after `await worker.run()`.
+   *
+   * The following code needs to cover these scenarios:
+   * 1. Connection monitor detects connection loss
+   * 2. Catalog.run() has a failure
+   * 3. Interruption is received
+   * 4. Worker throws an error
+   *
+   * For each scenario all promises in the Promise.all() need to be completed via functions:
+   * connectionMonitor.stop(), catalogJob.interrupt(), worker.shutdown()
+   */
+
+  /**
+   * Graceful shutdown
+   * Triggers the actions that will resolve all promises in the Promise.all(), so the code can resume
+   */
+  const shutdown = runOnce( () => {
+    log.info( 'Shutdown started...' );
+    if ( worker.getStatus().runState === 'RUNNING' ) {
+      worker.shutdown();
+    }
+    state.connectionMonitor.stop();
+    state.catalogJob.interrupt();
+  } );
 
-  log.info( 'Closing connection...' );
-  await connection.close();
+  /** When receiving an interruption, call shutdown */
+  setupInterruptionHandler( shutdown );
 
-  log.info( 'Bye' );
+  /** When the connection is lost, call shutdown */
+  state.connectionMonitor.onConnectionLost( shutdown );
 
-  process.exit( 0 );
-} )().catch( error => {
-  log.error( 'Fatal error', { message: error.message, stack: error.stack } );
-  messageBus.emit( BusEventType.RUNTIME_ERROR, { error } );
-  log.info( `Exiting in ${configs.processFailureShutdownDelay}ms` );
-  setTimeout( () => process.exit( 1 ), configs.processFailureShutdownDelay );
-} );
+  /** If the catalog job manager fails, call shutdown */
+  state.catalogJob.onError( shutdown );
+
+  /**
+   * Runs the worker, connection monitor and catalogJob (ephemeral)
+   * None of these will reject in normal conditions. Errors need to be inspected later
+   * They will resolve only when calling the actions in shutdown(),
+   * except catalogJob, which can resolve by itself given a bit of time
+   */
+  log.info( 'Running worker...' );
+  await Promise.all( [
+    // When the worker fails, store the error and call shutdown
+    worker.run().catch( error => {
+      state.workerError = error;
+      shutdown();
+    } ),
+    state.connectionMonitor.start(),
+    state.catalogJob.run()
+  ] );
+
+  log.info( 'Worker terminated' );
+
+  /** After the Promise.all() is resolved, check which services had an error, since none rejects the promise */
+  const error =
+    state.connectionMonitor.connectionLossError ??
+    state.catalogJob.error ??
+    state.workerError;
+
+  /** If any error is found, throws it, so this process can exit with code=1 (failure) */
+  if ( error ) {
+    throw error;
+  }
+};
+
+execute()
+  .finally( async () => {
+    /**
+     * This will make sure that if we had any uncaught failures, everything is tore down.
+     * Ignore any errors here in order to not mask actually errors from before and because at this point
+     * the code is already shutting down, so no need to throw anyway.
+     *
+     * worker.shutdown() is not tried here, because it cannot be awaited. By this point is safe to say
+     * that worker never started, already crashed or was stopped anyway.
+     */
+    if ( state.connectionMonitor?.running ) {
+      log.info( 'Stopping connection monitor...' );
+      await state.connectionMonitor.stop()
+        .catch( e => log.warn( 'Connection monitor stop error', { error: e.message } ) );
+    }
+    if ( state.catalogJob?.running ) {
+      log.info( 'Interrupting catalog job...' );
+      await state.catalogJob.interrupt()
+        .catch( e => log.warn( 'Catalog job interruption error', { error: e.message } ) );
+    }
+    if ( state.connection ) {
+      log.info( 'Closing connection...' );
+      await state.connection.close()
+        .catch( e => log.warn( 'Connection close error', { error: e.message } ) );
+    }
+  } )
+  .then( () => log.info( 'Bye' ) )
+  .catch( error => {
+    log.error( 'Fatal error', { error: error.message, stack: error.stack } );
+
+    messageBus.emit( BusEventType.RUNTIME_ERROR, { error } );
+
+    const timeToFlushEvent = configs.processFailureShutdownDelay;
+    log.info( `Exiting in ${timeToFlushEvent}ms` );
+    setTimeout( () => process.exit( 1 ), timeToFlushEvent );
+  } );

package/src/worker/index.spec.js

@@ -1,125 +1,232 @@
-import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const {
+  catalogJobInstance,
+  configValues,
+  connectionMonitorInstance,
+  createCatalogMock,
+  createWorkflowsEntryPointMock,
+  hashSourceCodeMock,
+  initInterceptorsMock,
+  loadActivitiesMock,
+  loadHooksMock,
+  loadWorkflowsMock,
+  messageBusMock,
+  mockConnection,
+  mockLog,
+  mockWorker,
+  promises,
+  resetPromises,
+  setupInterruptionHandlerMock,
+  setupTelemetryMock
+} = vi.hoisted( () => {
+  const createDeferred = () => {
+    const state = {};
+    state.promise = new Promise( ( resolve, reject ) => {
+      state.resolve = resolve;
+      state.reject = reject;
+    } );
+    return state;
+  };
+
+  const promises = {};
+  const resetPromises = () => {
+    promises.workerRun = createDeferred();
+    promises.connectionMonitor = createDeferred();
+    promises.catalogJob = createDeferred();
+  };
+  resetPromises();
+
+  const configValues = {
+    address: 'localhost:7233',
+    apiKey: undefined,
+    namespace: 'default',
+    taskQueue: 'test-queue',
+    catalogId: 'test-catalog',
+    grpcProxy: undefined,
+    maxConcurrentWorkflowTaskExecutions: 200,
+    maxConcurrentActivityTaskExecutions: 40,
+    maxCachedWorkflows: 1000,
+    maxConcurrentActivityTaskPolls: 5,
+    maxConcurrentWorkflowTaskPolls: 5,
+    processFailureShutdownDelay: 0
+  };
+
+  const connectionMonitorInstance = {
+    running: false,
+    connectionLossError: null,
+    onConnectionLost: vi.fn( cb => {
+      connectionMonitorInstance.connectionLostCb = cb;
+    } ),
+    start: vi.fn( () => {
+      connectionMonitorInstance.running = true;
+      return promises.connectionMonitor.promise.finally( () => {
+        connectionMonitorInstance.running = false;
+      } );
+    } ),
+    stop: vi.fn( () => {
+      connectionMonitorInstance.running = false;
+      promises.connectionMonitor.resolve();
+      return promises.connectionMonitor.promise;
+    } ),
+    connectionLostCb: null
+  };
+
+  const catalogJobInstance = {
+    running: false,
+    error: null,
+    onError: vi.fn( cb => {
+      catalogJobInstance.errorCb = cb;
+    } ),
+    run: vi.fn( () => {
+      catalogJobInstance.running = true;
+      return promises.catalogJob.promise.finally( () => {
+        catalogJobInstance.running = false;
+      } );
+    } ),
+    interrupt: vi.fn( () => {
+      catalogJobInstance.running = false;
+      promises.catalogJob.resolve();
+      return promises.catalogJob.promise;
+    } ),
+    errorCb: null
+  };
+
+  const mockWorker = {
+    getStatus: vi.fn( () => ( { runState: mockWorker.runState } ) ),
+    run: vi.fn( () => promises.workerRun.promise ),
+    runState: 'RUNNING',
+    shutdown: vi.fn()
+  };
+
+  return {
+    catalogJobInstance,
+    configValues,
+    connectionMonitorInstance,
+    createCatalogMock: vi.fn().mockReturnValue( { workflows: [], activities: {} } ),
+    createWorkflowsEntryPointMock: vi.fn().mockReturnValue( '/fake/workflows/path.js' ),
+    hashSourceCodeMock: vi.fn().mockResolvedValue( 'catalog-hash' ),
+    initInterceptorsMock: vi.fn().mockReturnValue( [] ),
+    loadActivitiesMock: vi.fn().mockResolvedValue( {} ),
+    loadHooksMock: vi.fn().mockResolvedValue( undefined ),
+    loadWorkflowsMock: vi.fn().mockResolvedValue( [] ),
+    messageBusMock: { emit: vi.fn(), on: vi.fn() },
+    mockConnection: { close: vi.fn().mockResolvedValue( undefined ) },
+    mockLog: { error: vi.fn(), info: vi.fn(), warn: vi.fn() },
+    mockWorker,
+    promises,
+    resetPromises,
+    setupInterruptionHandlerMock: vi.fn(),
+    setupTelemetryMock: vi.fn()
+  };
+} );
 
-const mockLog = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
 vi.mock( '#logger', () => ( { createChildLogger: () => mockLog } ) );
-
 vi.mock( '#consts', async importOriginal => {
   const actual = await importOriginal();
   return { ...actual };
 } );
-
 vi.mock( '#tracing', () => ( { init: vi.fn().mockResolvedValue( undefined ) } ) );
-
-const configValues = {
-  address: 'localhost:7233',
-  apiKey: undefined,
-  namespace: 'default',
-  taskQueue: 'test-queue',
-  catalogId: 'test-catalog',
-  grpcProxy: undefined,
-  maxConcurrentWorkflowTaskExecutions: 200,
-  maxConcurrentActivityTaskExecutions: 40,
-  maxCachedWorkflows: 1000,
-  maxConcurrentActivityTaskPolls: 5,
-  maxConcurrentWorkflowTaskPolls: 5,
-  processFailureShutdownDelay: 0
-};
-vi.mock( './configs.js', () => configValues );
-
-const messageBusMock = { on: vi.fn(), emit: vi.fn() };
 vi.mock( '#bus', () => ( { messageBus: messageBusMock } ) );
-
-const loadWorkflowsMock = vi.fn().mockResolvedValue( [] );
-const loadActivitiesMock = vi.fn().mockResolvedValue( {} );
-const loadHooksMock = vi.fn().mockResolvedValue( undefined );
-const createWorkflowsEntryPointMock = vi.fn().mockReturnValue( '/fake/workflows/path.js' );
+vi.mock( './configs.js', () => configValues );
 vi.mock( './loader.js', () => ( {
-  loadWorkflows: loadWorkflowsMock,
+  createWorkflowsEntryPoint: createWorkflowsEntryPointMock,
   loadActivities: loadActivitiesMock,
   loadHooks: loadHooksMock,
-  createWorkflowsEntryPoint: createWorkflowsEntryPointMock
+  loadWorkflows: loadWorkflowsMock
 } ) );
-
-const hashSourceCodeMock = vi.fn().mockResolvedValue( 'catalog-hash' );
 vi.mock( './loader_tools.js', () => ( { hashSourceCode: hashSourceCodeMock } ) );
-
 vi.mock( './sinks.js', () => ( { sinks: {} } ) );
-
-const createCatalogMock = vi.fn().mockReturnValue( { workflows: [], activities: {} } );
 vi.mock( './catalog_workflow/index.js', () => ( { createCatalog: createCatalogMock } ) );
-
 vi.mock( './bundler_options.js', () => ( { webpackConfigHook: vi.fn() } ) );
-
-const initInterceptorsMock = vi.fn().mockReturnValue( [] );
 vi.mock( './interceptors/index.js', () => ( { initInterceptors: initInterceptorsMock } ) );
-
-const startCatalogMock = vi.fn().mockResolvedValue( undefined );
-vi.mock( './start_catalog.js', () => ( { startCatalog: startCatalogMock } ) );
-
-const bootstrapFetchProxyMock = vi.fn();
-vi.mock( './proxy.js', () => ( { bootstrapFetchProxy: bootstrapFetchProxyMock } ) );
-
-const registerShutdownMock = vi.fn();
-vi.mock( './shutdown.js', () => ( { registerShutdown: registerShutdownMock } ) );
-
-const setupTelemetryMock = vi.fn();
-vi.mock( './setup_telemetry.js', () => ( { setupTelemetry: setupTelemetryMock } ) );
-
+vi.mock( './proxy.js', () => ( { bootstrapFetchProxy: vi.fn() } ) );
+vi.mock( './telemetry.js', () => ( { setupTelemetry: setupTelemetryMock } ) );
+vi.mock( './interruption.js', () => ( { setupInterruptionHandler: setupInterruptionHandlerMock } ) );
+vi.mock( './connection_monitor.js', () => ( {
+  TemporalConnectionMonitor: vi.fn( function () {
+    return connectionMonitorInstance;
+  } )
+} ) );
+vi.mock( './catalog_workflow/catalog_job.js', () => ( {
+  CatalogJob: vi.fn( function () {
+    return catalogJobInstance;
+  } )
+} ) );
 vi.mock( './log_hooks.js', () => ( {} ) );
-
-const runState = { resolve: null };
-const runPromise = new Promise( r => {
-  runState.resolve = r;
-} );
-const shutdownMock = vi.fn();
-const mockConnection = { close: vi.fn().mockResolvedValue( undefined ) };
-const mockWorker = { run: () => runPromise, shutdown: shutdownMock };
-
 vi.mock( '@temporalio/worker', () => ( {
-  Worker: { create: vi.fn().mockResolvedValue( mockWorker ) },
-  NativeConnection: { connect: vi.fn().mockResolvedValue( mockConnection ) }
+  NativeConnection: { connect: vi.fn().mockResolvedValue( mockConnection ) },
+  Worker: { create: vi.fn().mockResolvedValue( mockWorker ) }
 } ) );
 
+const importWorker = async () => {
+  vi.resetModules();
+  await import( './index.js' );
+};
+
+const settleWorker = async () => {
+  promises.catalogJob.resolve();
+  promises.connectionMonitor.resolve();
+  promises.workerRun.resolve();
+  await vi.waitFor( () => expect( mockConnection.close ).toHaveBeenCalled() );
+};
+
 describe( 'worker/index', () => {
   const exitMock = vi.fn();
   const originalArgv = process.argv;
-  const originalExit = process.exit;
 
   beforeEach( () => {
     vi.clearAllMocks();
+    resetPromises();
+    configValues.apiKey = undefined;
+    configValues.grpcProxy = undefined;
+    catalogJobInstance.error = null;
+    catalogJobInstance.errorCb = null;
+    catalogJobInstance.running = false;
+    connectionMonitorInstance.connectionLossError = null;
+    connectionMonitorInstance.connectionLostCb = null;
+    connectionMonitorInstance.running = false;
+    mockConnection.close.mockResolvedValue( undefined );
+    mockWorker.runState = 'RUNNING';
     process.argv = [ ...originalArgv.slice( 0, 2 ), '/test/caller/dir' ];
-    process.exit = exitMock;
+    vi.spyOn( process, 'exit' ).mockImplementation( exitMock );
   } );
 
   afterEach( () => {
     process.argv = originalArgv;
-    process.exit = originalExit;
-    configValues.apiKey = undefined;
+    vi.restoreAllMocks();
   } );
 
-  it( 'loads configs, workflows, activities and creates worker with correct options', async () => {
-    const { Worker, NativeConnection } = await import( '@temporalio/worker' );
-    const { init: initTracing } = await import( '#tracing' );
+  it( 'creates the worker lifecycle jobs with expected dependencies', async () => {
+    const { NativeConnection, Worker } = await import( '@temporalio/worker' );
+    const { TemporalConnectionMonitor } = await import( './connection_monitor.js' );
+    const { CatalogJob } = await import( './catalog_workflow/catalog_job.js' );
 
-    import( './index.js' );
+    await importWorker();
 
-    await vi.waitFor( () => {
-      expect( loadHooksMock ).toHaveBeenCalledWith( '/test/caller/dir' );
-    } );
+    await vi.waitFor( () => expect( Worker.create ).toHaveBeenCalled() );
+
+    expect( loadHooksMock ).toHaveBeenCalledWith( '/test/caller/dir' );
     expect( loadWorkflowsMock ).toHaveBeenCalledWith( '/test/caller/dir' );
     expect( loadActivitiesMock ).toHaveBeenCalledWith( '/test/caller/dir', [] );
     expect( createWorkflowsEntryPointMock ).toHaveBeenCalledWith( [] );
-    expect( initTracing ).toHaveBeenCalled();
     expect( createCatalogMock ).toHaveBeenCalledWith( { workflows: [], activities: {} } );
     expect( hashSourceCodeMock ).toHaveBeenCalledWith( '/test/caller/dir' );
-    expect( bootstrapFetchProxyMock ).toHaveBeenCalled();
     expect( NativeConnection.connect ).toHaveBeenCalledWith( {
       address: configValues.address,
       tls: false,
       apiKey: undefined,
       proxy: undefined
     } );
+    expect( TemporalConnectionMonitor ).toHaveBeenCalledWith( mockConnection );
+    expect( CatalogJob ).toHaveBeenCalledWith( {
+      connection: mockConnection,
+      namespace: configValues.namespace,
+      catalog: { workflows: [], activities: {} },
+      catalogHash: 'catalog-hash'
+    } );
     expect( Worker.create ).toHaveBeenCalledWith( expect.objectContaining( {
+      connection: mockConnection,
       namespace: configValues.namespace,
       taskQueue: configValues.taskQueue,
       workflowsPath: '/fake/workflows/path.js',
@@ -130,63 +237,128 @@ describe( 'worker/index', () => {
       maxConcurrentActivityTaskPolls: configValues.maxConcurrentActivityTaskPolls,
       maxConcurrentWorkflowTaskPolls: configValues.maxConcurrentWorkflowTaskPolls
     } ) );
-    expect( initInterceptorsMock ).toHaveBeenCalledWith( { activities: {}, workflows: [], connection: mockConnection } );
-    expect( registerShutdownMock ).toHaveBeenCalledWith( { worker: mockWorker, log: mockLog } );
+    expect( initInterceptorsMock ).toHaveBeenCalledWith( { activities: {}, workflows: [] } );
     expect( setupTelemetryMock ).toHaveBeenCalledWith( { worker: mockWorker } );
-    expect( startCatalogMock ).toHaveBeenCalledWith( {
-      connection: mockConnection,
-      namespace: configValues.namespace,
-      catalog: { workflows: [], activities: {} },
-      catalogHash: 'catalog-hash'
-    } );
-
-    runState.resolve();
-    await vi.waitFor( () => {
-      expect( mockConnection.close ).toHaveBeenCalled();
-    } );
-    expect( exitMock ).toHaveBeenCalledWith( 0 );
+    expect( setupInterruptionHandlerMock ).toHaveBeenCalledWith( expect.any( Function ) );
+    expect( connectionMonitorInstance.onConnectionLost ).toHaveBeenCalledWith( expect.any( Function ) );
+    expect( catalogJobInstance.onError ).toHaveBeenCalledWith( expect.any( Function ) );
+    expect( mockWorker.run ).toHaveBeenCalled();
+    expect( connectionMonitorInstance.start ).toHaveBeenCalled();
+    expect( catalogJobInstance.run ).toHaveBeenCalled();
+
+    await settleWorker();
+    expect( mockLog.info ).toHaveBeenCalledWith( 'Bye' );
   } );
 
   it( 'enables TLS when apiKey is set', async () => {
     configValues.apiKey = 'secret';
-    vi.resetModules();
-
     const { NativeConnection } = await import( '@temporalio/worker' );
-    import( './index.js' );
+
+    await importWorker();
+
+    await vi.waitFor( () => expect( NativeConnection.connect ).toHaveBeenCalledWith( expect.objectContaining( {
+      apiKey: 'secret',
+      tls: true
+    } ) ) );
+
+    await settleWorker();
+  } );
+
+  it( 'runs graceful shutdown when interrupted', async () => {
+    await importWorker();
+
+    await vi.waitFor( () => expect( setupInterruptionHandlerMock ).toHaveBeenCalled() );
+    const [ shutdown ] = setupInterruptionHandlerMock.mock.calls[0];
+
+    shutdown();
+
+    expect( mockWorker.shutdown ).toHaveBeenCalledOnce();
+    expect( connectionMonitorInstance.stop ).toHaveBeenCalledOnce();
+    expect( catalogJobInstance.interrupt ).toHaveBeenCalledOnce();
+
+    promises.workerRun.resolve();
+    await vi.waitFor( () => expect( mockConnection.close ).toHaveBeenCalled() );
+    expect( mockLog.info ).toHaveBeenCalledWith( 'Bye' );
+  } );
+
+  it( 'does not call worker.shutdown when worker has already failed', async () => {
+    const error = new Error( 'Big Failure' );
+
+    await importWorker();
+    await vi.waitFor( () => expect( mockWorker.run ).toHaveBeenCalled() );
+
+    mockWorker.runState = 'FAILED';
+    promises.workerRun.reject( error );
+
+    await vi.waitFor( () => expect( connectionMonitorInstance.stop ).toHaveBeenCalled() );
+    expect( mockWorker.shutdown ).not.toHaveBeenCalled();
+
+    promises.connectionMonitor.resolve();
+    promises.catalogJob.resolve();
 
     await vi.waitFor( () => {
-      expect( NativeConnection.connect ).toHaveBeenCalledWith( expect.objectContaining( {
-        tls: true,
-        apiKey: 'secret'
+      expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.objectContaining( {
+        error: 'Big Failure'
       } ) );
     } );
-    runState.resolve();
-    await vi.waitFor( () => expect( exitMock ).toHaveBeenCalled() );
+    expect( messageBusMock.emit ).toHaveBeenCalledWith( expect.any( String ), { error } );
+    await vi.waitFor( () => expect( exitMock ).toHaveBeenCalledWith( 1 ) );
   } );
 
-  it( 'calls registerShutdown with worker and log', async () => {
-    vi.resetModules();
+  it( 'throws connection monitor errors after graceful shutdown', async () => {
+    const error = new Error( 'connection lost' );
 
-    import( './index.js' );
+    await importWorker();
+    await vi.waitFor( () => expect( connectionMonitorInstance.onConnectionLost ).toHaveBeenCalled() );
+
+    connectionMonitorInstance.connectionLossError = error;
+    connectionMonitorInstance.connectionLostCb( error );
+    promises.workerRun.resolve();
 
     await vi.waitFor( () => {
-      expect( registerShutdownMock ).toHaveBeenCalledWith( { worker: mockWorker, log: mockLog } );
+      expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.objectContaining( {
+        error: 'connection lost'
+      } ) );
     } );
-    runState.resolve();
-    await vi.waitFor( () => expect( exitMock ).toHaveBeenCalled() );
+    expect( mockWorker.shutdown ).toHaveBeenCalledOnce();
+    expect( catalogJobInstance.interrupt ).toHaveBeenCalledOnce();
+    expect( messageBusMock.emit ).toHaveBeenCalledWith( expect.any( String ), { error } );
   } );
 
-  it( 'calls process.exit(1) on fatal error', async () => {
-    loadWorkflowsMock.mockRejectedValueOnce( new Error( 'load failed' ) );
-    vi.resetModules();
+  it( 'throws catalog job errors after graceful shutdown', async () => {
+    const error = new Error( 'catalog failed' );
+
+    await importWorker();
+    await vi.waitFor( () => expect( catalogJobInstance.onError ).toHaveBeenCalled() );
 
-    import( './index.js' );
+    catalogJobInstance.error = error;
+    catalogJobInstance.errorCb( error );
+    promises.workerRun.resolve();
 
     await vi.waitFor( () => {
-      expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.any( Object ) );
+      expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.objectContaining( {
+        error: 'catalog failed'
+      } ) );
     } );
+    expect( mockWorker.shutdown ).toHaveBeenCalledOnce();
+    expect( connectionMonitorInstance.stop ).toHaveBeenCalledOnce();
+    expect( messageBusMock.emit ).toHaveBeenCalledWith( expect.any( String ), { error } );
+  } );
+
+  it( 'cleans up partial startup failures after connecting', async () => {
+    const { Worker } = await import( '@temporalio/worker' );
+    const error = new Error( 'worker create failed' );
+    Worker.create.mockRejectedValueOnce( error );
+
+    await importWorker();
+
+    await vi.waitFor( () => expect( mockConnection.close ).toHaveBeenCalled() );
+    expect( connectionMonitorInstance.stop ).not.toHaveBeenCalled();
+    expect( catalogJobInstance.interrupt ).not.toHaveBeenCalled();
     await vi.waitFor( () => {
-      expect( exitMock ).toHaveBeenCalledWith( 1 );
+      expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.objectContaining( {
+        error: 'worker create failed'
+      } ) );
     } );
   } );
 } );