@outputai/core 0.12.1-next.a4a41b6.0 → 0.12.1-next.d1d27f7.0

package/src/worker/catalog_workflow/catalog_publisher.spec.js

@@ -0,0 +1,317 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { WorkflowNotFoundError } from '@temporalio/client';
+import { WorkflowExecutionAlreadyStartedError, WorkflowIdConflictPolicy } from '@temporalio/common';
+import { CatalogPublisher } from './catalog_publisher.js';
+
+const {
+  catalogId,
+  describeMock,
+  getHandleMock,
+  mockLog,
+  serializeErrorMock,
+  sleepMock,
+  startMock,
+  taskQueue,
+  terminateMock
+} = vi.hoisted( () => ( {
+  catalogId: 'test-catalog',
+  describeMock: vi.fn(),
+  getHandleMock: vi.fn(),
+  mockLog: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+  serializeErrorMock: vi.fn( error => ( { message: error.message } ) ),
+  sleepMock: vi.fn(),
+  startMock: vi.fn(),
+  taskQueue: 'test-queue',
+  terminateMock: vi.fn()
+} ) );
+
+vi.mock( '#logger', () => ( { createChildLogger: () => mockLog } ) );
+vi.mock( '#helpers/error_serializer', () => ( { serializeError: serializeErrorMock } ) );
+vi.mock( '#consts', () => ( { WORKFLOW_CATALOG: 'catalog' } ) );
+vi.mock( '#helpers/promise', () => ( { sleepCancellable: sleepMock } ) );
+vi.mock( '../configs.js', () => ( { catalogId, taskQueue } ) );
+vi.mock( '@temporalio/client', async importOriginal => {
+  const actual = await importOriginal();
+  return {
+    ...actual,
+    Client: vi.fn().mockImplementation( function () {
+      return { workflow: { start: startMock, getHandle: getHandleMock } };
+    } )
+  };
+} );
+
+const RETRY_DELAY = 10_000;
+const namespace = 'default';
+const catalogHash = 'catalog-hash';
+const catalog = { workflows: [], workflowNames: { workflow: 'workflow' }, activities: {} };
+
+const startArguments = {
+  taskQueue,
+  workflowId: catalogId,
+  workflowIdConflictPolicy: WorkflowIdConflictPolicy.FAIL,
+  workflowTaskTimeout: 30_000,
+  args: [ catalog ],
+  memo: {
+    workflowNames: catalog.workflowNames,
+    hash: catalogHash
+  }
+};
+
+const compactError = message => ( { error: { message } } );
+
+const running = hash => ( { closeTime: undefined, runId: 'run-1', memo: { hash } } );
+const notFound = () => new WorkflowNotFoundError( 'not found' );
+const alreadyStarted = () => new WorkflowExecutionAlreadyStartedError( 'already started', catalogId, 'catalog' );
+
+const createPublisher = ( signal = new AbortController().signal ) =>
+  new CatalogPublisher( { connection: {}, namespace, catalog, catalogHash, signal } );
+
+const flushPromises = async () => Array
+  .from( { length: 10 } )
+  .reduce( promise => promise.then( () => Promise.resolve() ), Promise.resolve() );
+
+describe( 'CatalogPublisher', () => {
+  beforeEach( () => {
+    vi.clearAllMocks();
+    getHandleMock.mockReturnValue( { describe: describeMock, terminate: terminateMock } );
+    describeMock.mockRejectedValue( notFound() );
+    terminateMock.mockResolvedValue( undefined );
+    startMock.mockResolvedValue( undefined );
+    sleepMock.mockResolvedValue( undefined );
+  } );
+
+  describe( 'publish paths', () => {
+    it( 'starts the catalog when nothing is running', async () => {
+      const publisher = createPublisher();
+
+      await publisher.run();
+
+      expect( terminateMock ).not.toHaveBeenCalled();
+      expect( startMock ).toHaveBeenCalledWith( 'catalog', startArguments );
+      expect( mockLog.info ).toHaveBeenCalledWith( 'Catalog workflow started' );
+      expect( mockLog.warn ).not.toHaveBeenCalled();
+      expect( publisher.running ).toBe( false );
+    } );
+
+    it( 'starts the catalog when the previous one is closed', async () => {
+      describeMock.mockResolvedValue( { closeTime: '2024-01-01T00:00:00Z' } );
+
+      await createPublisher().run();
+
+      expect( terminateMock ).not.toHaveBeenCalled();
+      expect( startMock ).toHaveBeenCalledWith( 'catalog', startArguments );
+    } );
+
+    it( 'skips the restart when the running hash matches', async () => {
+      describeMock.mockResolvedValue( running( catalogHash ) );
+
+      await createPublisher().run();
+
+      expect( terminateMock ).not.toHaveBeenCalled();
+      expect( startMock ).not.toHaveBeenCalled();
+      expect( mockLog.info ).toHaveBeenCalledWith( 'Current catalog workflow hash matches worker, restart skipped' );
+    } );
+
+    it( 'terminates and starts when the running catalog carries no memo', async () => {
+      describeMock.mockResolvedValue( { closeTime: undefined, runId: 'run-1', memo: undefined } );
+
+      await createPublisher().run();
+
+      expect( mockLog.warn ).not.toHaveBeenCalled();
+      expect( terminateMock ).toHaveBeenCalledOnce();
+      expect( startMock ).toHaveBeenCalledWith( 'catalog', startArguments );
+    } );
+
+    it( 'terminates the pinned run then starts when the running hash differs', async () => {
+      describeMock.mockResolvedValue( running( 'old-hash' ) );
+
+      await createPublisher().run();
+
+      expect( getHandleMock ).toHaveBeenCalledWith( catalogId, 'run-1' );
+      expect( terminateMock ).toHaveBeenCalledWith( `Systematic terminating. Superseded by catalog ${catalogHash}` );
+      expect( startMock ).toHaveBeenCalledWith( 'catalog', startArguments );
+    } );
+  } );
+
+  describe( 'race paths', () => {
+    it( 'ignores an already started error when the published hash matches', async () => {
+      describeMock
+        .mockRejectedValueOnce( notFound() )
+        .mockResolvedValueOnce( running( catalogHash ) );
+      startMock.mockRejectedValue( alreadyStarted() );
+
+      await createPublisher().run();
+
+      expect( startMock ).toHaveBeenCalledOnce();
+      expect( terminateMock ).not.toHaveBeenCalled();
+      expect( mockLog.info ).toHaveBeenCalledWith( 'Found a catalog workflow with matching hashes, start skipped' );
+      expect( mockLog.info ).not.toHaveBeenCalledWith( 'Catalog workflow started' );
+    } );
+
+    it( 'retires the winner on retry when an already started error exposes a different hash', async () => {
+      describeMock
+        .mockRejectedValueOnce( notFound() )
+        .mockResolvedValue( running( 'other-hash' ) );
+      startMock
+        .mockRejectedValueOnce( alreadyStarted() )
+        .mockResolvedValue( undefined );
+
+      await createPublisher().run();
+
+      expect( sleepMock ).toHaveBeenCalledOnce();
+      expect( terminateMock ).toHaveBeenCalledOnce();
+      expect( startMock ).toHaveBeenCalledTimes( 2 );
+      expect( mockLog.warn ).toHaveBeenCalledWith(
+        'Error while starting catalog workflow, retrying...',
+        compactError( 'already started' )
+      );
+    } );
+
+    it( 'warns and attempts a start when describing fails', async () => {
+      const error = new Error( 'describe failed' );
+      describeMock.mockRejectedValue( error );
+
+      await createPublisher().run();
+
+      expect( serializeErrorMock ).toHaveBeenCalledWith( error, { dropKeys: [ 'stack' ] } );
+      expect( mockLog.warn ).toHaveBeenCalledWith(
+        'Ignoring error while describing catalog workflow',
+        compactError( 'describe failed' )
+      );
+      expect( startMock ).toHaveBeenCalledWith( 'catalog', startArguments );
+    } );
+
+    it( 'warns and still starts when terminating the previous catalog fails', async () => {
+      describeMock.mockResolvedValue( running( 'old-hash' ) );
+      terminateMock.mockRejectedValue( new Error( 'terminate failed' ) );
+
+      await createPublisher().run();
+
+      expect( mockLog.warn ).toHaveBeenCalledWith(
+        'Ignoring error while terminating previous catalog workflow',
+        compactError( 'terminate failed' )
+      );
+      expect( startMock ).toHaveBeenCalledWith( 'catalog', startArguments );
+    } );
+  } );
+
+  describe( 'failure', () => {
+    it( 'rejects with the original error after exhausting the retries', async () => {
+      const error = new Error( 'start failed' );
+      startMock.mockRejectedValue( error );
+      const publisher = createPublisher();
+
+      await expect( publisher.run() ).rejects.toBe( error );
+
+      expect( startMock ).toHaveBeenCalledTimes( 3 );
+      expect( sleepMock ).toHaveBeenCalledTimes( 2 );
+      expect( publisher.running ).toBe( false );
+    } );
+
+    it( 'waits the retry delay with the abort signal between attempts', async () => {
+      startMock.mockRejectedValue( new Error( 'start failed' ) );
+
+      await expect( createPublisher().run() ).rejects.toThrow( 'start failed' );
+
+      expect( sleepMock ).toHaveBeenCalledWith( RETRY_DELAY, expect.any( AbortSignal ) );
+    } );
+  } );
+
+  describe( 'abort paths', () => {
+    it( 'publishes nothing when already aborted', async () => {
+      const controller = new AbortController();
+      controller.abort();
+
+      await expect( createPublisher( controller.signal ).run() ).resolves.toBeUndefined();
+
+      expect( describeMock ).not.toHaveBeenCalled();
+      expect( startMock ).not.toHaveBeenCalled();
+    } );
+
+    it( 'stops retrying when the signal aborts during the backoff', async () => {
+      const controller = new AbortController();
+      startMock.mockRejectedValue( new Error( 'start failed' ) );
+      sleepMock.mockImplementation( async () => controller.abort() );
+
+      await expect( createPublisher( controller.signal ).run() ).resolves.toBeUndefined();
+
+      expect( startMock ).toHaveBeenCalledOnce();
+      expect( sleepMock ).toHaveBeenCalledOnce();
+    } );
+
+    it( 'stops retrying when interrupted during the backoff', async () => {
+      startMock.mockRejectedValue( new Error( 'start failed' ) );
+      const publisher = createPublisher();
+      sleepMock.mockImplementation( async () => {
+        publisher.interrupt();
+      } );
+
+      await expect( publisher.run() ).resolves.toBeUndefined();
+
+      expect( startMock ).toHaveBeenCalledOnce();
+    } );
+
+    it( 'still starts when the abort lands between terminating and starting', async () => {
+      const controller = new AbortController();
+      describeMock.mockResolvedValue( running( 'old-hash' ) );
+      terminateMock.mockImplementation( async () => controller.abort() );
+
+      await createPublisher( controller.signal ).run();
+
+      expect( terminateMock ).toHaveBeenCalledOnce();
+      expect( startMock ).toHaveBeenCalledWith( 'catalog', startArguments );
+    } );
+  } );
+
+  describe( 'lifecycle', () => {
+    it( 'reports running while the sequence is pending', async () => {
+      const deferred = {};
+      describeMock.mockReturnValue( new Promise( resolve => {
+        deferred.resolve = resolve;
+      } ) );
+      const publisher = createPublisher();
+
+      const run = publisher.run();
+      await flushPromises();
+
+      expect( publisher.running ).toBe( true );
+
+      deferred.resolve( running( catalogHash ) );
+      await run;
+
+      expect( publisher.running ).toBe( false );
+    } );
+
+    it( 'returns the same execution and publishes once when run more than once', async () => {
+      const publisher = createPublisher();
+
+      const first = publisher.run();
+      const second = publisher.run();
+
+      expect( second ).toBe( first );
+
+      await first;
+
+      expect( describeMock ).toHaveBeenCalledOnce();
+      expect( startMock ).toHaveBeenCalledOnce();
+    } );
+
+    it( 'resolves when interrupted before running', async () => {
+      const publisher = createPublisher();
+
+      await expect( publisher.interrupt() ).resolves.toBeUndefined();
+
+      expect( publisher.running ).toBe( false );
+      expect( describeMock ).not.toHaveBeenCalled();
+    } );
+
+    it( 'reports the failure when interrupting after a rejected run', async () => {
+      startMock.mockRejectedValue( new Error( 'start failed' ) );
+      const publisher = createPublisher();
+
+      await expect( publisher.run() ).rejects.toThrow( 'start failed' );
+
+      await expect( publisher.interrupt() ).rejects.toThrow( 'start failed' );
+    } );
+  } );
+} );

package/src/worker/configs.js

@@ -36,10 +36,13 @@ const envVarSchema = z.object( {
   OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 2 * 60 * 1000 ) ), // 2min
   // Whether to send activity heartbeats (enabled by default)
   OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: z.transform( v => v === undefined ? true : isStringboolTrue( v ) ),
-  // Set temporal worker shutdown force time
-  TEMPORAL_SHUTDOWN_FORCE_TIME: durationSchema,
-  // Set temporal worker shutdown grace time
-  TEMPORAL_SHUTDOWN_GRACE_TIME: durationSchema,
+  // Set temporal worker shutdown force time. Defaults budget a shutdown against the 30s most
+  // platforms allow before SIGKILL: force time + OUTPUT_HOOK_FLUSH_TIMEOUT_MS + 5s of margin.
+  TEMPORAL_SHUTDOWN_FORCE_TIME: durationSchema.default( '20s' ),
+  // Set temporal worker shutdown grace time, when in-flight activities are asked to cancel
+  TEMPORAL_SHUTDOWN_GRACE_TIME: durationSchema.default( '15s' ),
+  // How long the worker awaits pending hook callbacks once the drain is over, before exiting anyway
+  OUTPUT_HOOK_FLUSH_TIMEOUT_MS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 5 * 1000 ) ), // 5s
   // HTTP CONNECT proxy for Temporal gRPC connections (e.g. "proxy-host:8080").
   // Must be a bare host:port — no scheme (Temporal's native HTTP CONNECT
   // option is not a URL).
@@ -70,4 +73,5 @@ export const activityHeartbeatIntervalMs = envVars.OUTPUT_ACTIVITY_HEARTBEAT_INT
 export const activityHeartbeatEnabled = envVars.OUTPUT_ACTIVITY_HEARTBEAT_ENABLED;
 export const shutdownForceTime = envVars.TEMPORAL_SHUTDOWN_FORCE_TIME;
 export const shutdownGraceTime = envVars.TEMPORAL_SHUTDOWN_GRACE_TIME;
+export const hookFlushTimeoutMs = envVars.OUTPUT_HOOK_FLUSH_TIMEOUT_MS;
 export const grpcProxy = envVars.TEMPORAL_GRPC_PROXY;

package/src/worker/configs.spec.js

@@ -15,7 +15,8 @@ const CONFIG_KEYS = [
   'OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS',
   'OUTPUT_ACTIVITY_HEARTBEAT_ENABLED',
   'TEMPORAL_SHUTDOWN_FORCE_TIME',
-  'TEMPORAL_SHUTDOWN_GRACE_TIME'
+  'TEMPORAL_SHUTDOWN_GRACE_TIME',
+  'OUTPUT_HOOK_FLUSH_TIMEOUT_MS'
 ];
 
 const setEnv = ( overrides = {} ) => {
@@ -69,8 +70,9 @@ describe( 'worker/configs', () => {
     expect( configs.workerTelemetryIntervalMs ).toBe( 0 );
     expect( configs.activityHeartbeatIntervalMs ).toBe( 2 * 60 * 1000 );
     expect( configs.activityHeartbeatEnabled ).toBe( true );
-    expect( configs.shutdownForceTime ).toBeUndefined();
-    expect( configs.shutdownGraceTime ).toBeUndefined();
+    expect( configs.shutdownForceTime ).toBe( '20s' );
+    expect( configs.shutdownGraceTime ).toBe( '15s' );
+    expect( configs.hookFlushTimeoutMs ).toBe( 5000 );
     expect( configs.taskQueue ).toBe( 'test-catalog' );
     expect( configs.catalogId ).toBe( 'test-catalog' );
   } );
@@ -136,15 +138,17 @@ describe( 'worker/configs', () => {
     expect( configs.shutdownGraceTime ).toBe( '15s' );
   } );
 
-  it( 'treats empty Temporal shutdown durations as unset', async () => {
+  it( 'falls back to the defaults when Temporal shutdown durations are empty', async () => {
     setEnv( {
       TEMPORAL_SHUTDOWN_FORCE_TIME: '',
-      TEMPORAL_SHUTDOWN_GRACE_TIME: ''
+      TEMPORAL_SHUTDOWN_GRACE_TIME: '',
+      OUTPUT_HOOK_FLUSH_TIMEOUT_MS: ''
     } );
     const configs = await loadConfigs();
 
-    expect( configs.shutdownForceTime ).toBeUndefined();
-    expect( configs.shutdownGraceTime ).toBeUndefined();
+    expect( configs.shutdownForceTime ).toBe( '20s' );
+    expect( configs.shutdownGraceTime ).toBe( '15s' );
+    expect( configs.hookFlushTimeoutMs ).toBe( 5000 );
   } );
 
   it( 'throws when Temporal shutdown durations are invalid', async () => {

package/src/worker/connection_monitor.js

@@ -1,101 +1,129 @@
+import { sleepCancellable } from '#helpers/promise';
 import { createChildLogger } from '#logger';
 import { setTimeout as delay } from 'node:timers/promises';
-import { CancellablePromise } from '#helpers/promise';
 
 const log = createChildLogger( 'Connection' );
 
+/** Raised when the connection health check fails repeatedly. */
+export class ConnectionLostError extends Error {
+  name = 'ConnectionLostError';
+};
+
+const MAX_FAILURES = 3;
+const CHECK_INTERVAL_MS = 60_000;
+const CHECK_TIMEOUT_MS = 5_000;
+const HEALTHY_LOG_EVERY_LOOPS = 60;
+
+/**
+ * Watches the Temporal connection health.
+ * Rejects with a ConnectionLostError once the connection is considered lost.
+ */
 export class TemporalConnectionMonitor {
-  #MAX_FAILURES = 3;
-  #CHECK_INTERVAL_MS = 60_000;
-  #CHECK_TIMEOUT_MS = 5_000;
+  #maxFailures = MAX_FAILURES;
+  #checkIntervalMs = CHECK_INTERVAL_MS;
+  #checkTimeoutMs = CHECK_TIMEOUT_MS;
 
-  #cancellation = new CancellablePromise();
+  #abortCtrl = new AbortController();
   #failures = 0;
-  #error = null;
+  #loops = 0;
   #running = false;
-  #watchPromise = null;
+  #execution = null;
   #connection = null;
-  #connectionLostCb = null;
-
-  #getTimeout = async () => delay( this.#CHECK_TIMEOUT_MS, 0, { ref: false } ).then( () => {
-    throw new Error( 'Connection health check timed out' );
-  } );
-
-  #healthcheck = async () => this.#connection.workflowService.getSystemInfo( {} );
+  #signal = null;
 
-  #sleep = async () => delay( this.#CHECK_INTERVAL_MS, 0, { ref: false } );
-
-  #watch = async () => {
-    while ( !this.#cancellation.completed ) {
+  #watch = async signal => {
+    while ( !signal.aborted ) {
       try {
-        await Promise.race( [ this.#healthcheck(), this.#getTimeout(), this.#cancellation.promise ] );
-
-        // cancellation won the race
-        if ( this.#cancellation.completed ) {
-          break;
+        await Promise.race( [
+          this.#connection.workflowService.getSystemInfo( {} ),
+          delay( this.#checkTimeoutMs, 0, { ref: false, signal } ).then( () => {
+            throw new Error( 'Connection health check timed out' );
+          } )
+        ] );
+
+        if ( this.#failures > 0 ) {
+          log.info( 'Recovered' );
+        }
+        if ( this.#loops === 0 || this.#loops % HEALTHY_LOG_EVERY_LOOPS === 0 ) {
+          log.info( 'Healthy' );
         }
-
-        log.info( this.#failures === 0 ? 'Healthy' : 'Recovered' );
         this.#failures = 0;
+
       } catch ( error ) {
-        // cancellation will ignore warnings and not throw errors;
-        if ( this.#cancellation.completed ) {
-          break;
+        // aborting will ignore warnings and not report errors
+        if ( signal.aborted ) {
+          return;
         }
 
-        if ( ++this.#failures >= this.#MAX_FAILURES ) {
-          log.warn( 'Connection lost', { error: error.message, failures: this.#failures } );
-          this.#error = error;
-          this.#connectionLostCb?.( error );
-          this.#cancellation.complete();
-          break;
-        } else {
-          log.warn( 'Connection unhealthy', { error: error.message, failures: this.#failures } );
+        const failureMessage = error?.message ?? String( error );
+        if ( ++this.#failures >= this.#maxFailures ) {
+          log.warn( 'Connection lost', { error: failureMessage, failures: this.#failures } );
+          this.#abortCtrl.abort();
+          throw new ConnectionLostError( 'Connection lost', { cause: error } );
         }
+
+        log.warn( 'Connection unhealthy', { error: failureMessage, failures: this.#failures } );
       }
 
-      await Promise.race( [ this.#sleep(), this.#cancellation.promise ] );
+      this.#loops++;
+      await sleepCancellable( this.#checkIntervalMs, signal );
     }
   };
 
-  constructor( connection, overrides = {} ) {
+  /**
+   * @param {object} options
+   * @param {import('@temporalio/worker').NativeConnection} options.connection - Temporal connection
+   * @param {AbortSignal} options.signal - stops watching when aborted
+   * @param {object} [options.overrides] - health check tuning, mostly for tests
+   */
+  constructor( { connection, signal, overrides = {} } ) {
     this.#connection = connection;
+    this.#signal = signal;
     if ( Number.isFinite( overrides?.maxFailures ) ) {
-      this.#MAX_FAILURES = overrides.maxFailures;
+      this.#maxFailures = overrides.maxFailures;
     }
     if ( Number.isFinite( overrides?.checkIntervalMs ) ) {
-      this.#CHECK_INTERVAL_MS = overrides.checkIntervalMs;
+      this.#checkIntervalMs = overrides.checkIntervalMs;
     }
     if ( Number.isFinite( overrides?.checkTimeoutMs ) ) {
-      this.#CHECK_TIMEOUT_MS = overrides.checkTimeoutMs;
+      this.#checkTimeoutMs = overrides.checkTimeoutMs;
     }
   }
 
-  onConnectionLost( cb ) {
-    this.#connectionLostCb = cb;
-  }
-
+  /** Returns whether the monitor is watching */
   get running() {
     return this.#running;
   }
 
+  /**
+   * Starts watching the connection. Aborting the signal stops the watch.
+   * @returns {Promise<void>} resolves when the watch has stopped, rejects when the connection is lost
+   */
   start() {
-    if ( this.#watchPromise ) {
-      return this.#watchPromise;
+    if ( this.#execution ) {
+      return this.#execution;
+    }
+
+    const signal = AbortSignal.any( [ this.#signal, this.#abortCtrl.signal ] );
+
+    if ( signal.aborted ) {
+      return Promise.resolve();
     }
+
     this.#running = true;
-    this.#watchPromise = this.#watch().finally( () => {
+    this.#execution = this.#watch( signal ).finally( () => {
       this.#running = false;
     } );
-    return this.#watchPromise;
-  }
 
-  stop() {
-    this.#cancellation.complete();
-    return this.#watchPromise ?? Promise.resolve();
+    return this.#execution;
   }
 
-  get connectionLossError() {
-    return this.#error;
+  /**
+   * Stops watching, without reporting a connection loss.
+   * @returns {Promise<void>} resolves when the watch has fully stopped, rejects when it failed
+   */
+  stop() {
+    this.#abortCtrl.abort();
+    return this.#execution ?? Promise.resolve();
   }
 };