@outputai/core 0.7.1-next.de30052.0 → 0.8.0

package/src/worker/catalog_workflow/catalog_job.js

@@ -0,0 +1,148 @@
+import { Client, WorkflowNotFoundError } from '@temporalio/client';
+import { WorkflowExecutionAlreadyStartedError, WorkflowIdConflictPolicy } from '@temporalio/common';
+import { WORKFLOW_CATALOG } from '#consts';
+import { catalogId, taskQueue } from '../configs.js';
+import { createChildLogger } from '#logger';
+import { CancellablePromise } from '#utils';
+
+const log = createChildLogger( 'Catalog' );
+
+class CancellationError extends Error {};
+
+// Note, functions don't log on "WorkflowNotFound" errors,
+// because they happen when the catalog is not running at all.
+
+/** Make sure the latest version of the catalog workflow is running. Stateful. */
+export class CatalogJob {
+  #cancellation = new CancellablePromise();
+  #connection = null;
+  #namespace = null;
+  #catalog = null;
+  #catalogHash = null;
+
+  #running = false;
+  #executePromise = null;
+  #onErrorCb = null;
+  #error = null;
+
+  #runCancellable = promise => Promise
+    .race( [ promise, this.#cancellation.promise.then( () => {
+      throw new CancellationError();
+    } ) ] );
+
+  /** Check if the currently running catalog has the same hash as instance. */
+  async #checkCatalogIsTheSame( handle ) {
+    log.info( 'Checking running catalog hash against worker hash...' );
+    return this.#runCancellable( handle.query( 'get_hash' ) ).then( h => h === this.#catalogHash ).catch( e => {
+      if ( e instanceof CancellationError ) {
+        throw e;
+      }
+      if ( !( e instanceof WorkflowNotFoundError ) ) {
+        log.warn( 'Error retrieving catalog hash', { error: e } );
+      }
+      return false;
+    } );
+  };
+
+  /**  Check if the catalog workflow is running. */
+  async #checkCatalogRunning( handle ) {
+    log.info( 'Checking if the catalog workflow is running...' );
+    return this.#runCancellable( handle.describe() ).then( d => !d.closeTime ).catch( e => {
+      if ( e instanceof CancellationError ) {
+        throw e;
+      }
+      if ( !( e instanceof WorkflowNotFoundError ) ) {
+        log.warn( 'Error describing catalog workflow', { error: e } );
+      }
+      return false;
+    } );
+  };
+
+  /** Complete previous running catalog workflow. */
+  async #completePreviousCatalog( handle ) {
+    log.info( 'Completing previous catalog workflow...' );
+    return this.#runCancellable( handle.executeUpdate( 'complete', { args: [] } ) ).catch( e => {
+      if ( e instanceof CancellationError ) {
+        throw e;
+      }
+      if ( !( e instanceof WorkflowNotFoundError ) ) {
+        log.warn( 'Error completing previous catalog workflow', { error: e } );
+      }
+    } );
+  };
+
+  /** Run the sequence to start the catalog */
+  async #execute() {
+    const client = new Client( { connection: this.#connection, namespace: this.#namespace } );
+    const handle = client.workflow.getHandle( catalogId );
+
+    if ( await this.#checkCatalogRunning( handle ) ) {
+      if ( await this.#checkCatalogIsTheSame( handle ) ) {
+        log.info( 'Current catalog workflow hash matches worker, restart skipped' );
+        return;
+      }
+      await this.#completePreviousCatalog( handle );
+    }
+
+    const startArguments = {
+      taskQueue,
+      workflowId: catalogId,
+      workflowIdConflictPolicy: WorkflowIdConflictPolicy.FAIL,
+      args: [ this.#catalog, this.#catalogHash ]
+    };
+
+    log.info( 'Starting catalog workflow...' );
+    try {
+      await this.#runCancellable( client.workflow.start( WORKFLOW_CATALOG, startArguments ) );
+    } catch ( error ) {
+      // if the error was caused by the catalog existing and its hash is the same as the one from the worker, just ignore the error
+      if ( error instanceof WorkflowExecutionAlreadyStartedError && await this.#checkCatalogIsTheSame( handle ) ) {
+        log.info( 'Ignoring start error: it failed because execution already started but catalog hash matches worker' );
+      } else {
+        throw error;
+      }
+    }
+    log.info( 'Startup completed' );
+  }
+
+  constructor( { connection, namespace, catalog, catalogHash } ) {
+    this.#connection = connection;
+    this.#namespace = namespace;
+    this.#catalog = catalog;
+    this.#catalogHash = catalogHash;
+  }
+
+  onError( cb ) {
+    this.#onErrorCb = cb;
+  }
+
+  get running() {
+    return this.#running;
+  }
+
+  interrupt() {
+    if ( this.#running ) {
+      this.#cancellation.complete();
+    }
+    return this.#executePromise ?? Promise.resolve();
+  }
+
+  get error() {
+    return this.#error;
+  }
+
+  async run() {
+    this.#running = true;
+    this.#executePromise = this.#execute()
+      .catch( error => {
+        if ( !( error instanceof CancellationError ) ) {
+          this.#error = error;
+          this.#onErrorCb?.( error );
+        }
+      } )
+      .finally( () => {
+        this.#running = false;
+      } );
+    return this.#executePromise;
+  }
+};

package/src/worker/catalog_workflow/catalog_job.spec.js

@@ -0,0 +1,232 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { WorkflowNotFoundError } from '@temporalio/client';
+import { WorkflowExecutionAlreadyStartedError, WorkflowIdConflictPolicy } from '@temporalio/common';
+import { CatalogJob } from './catalog_job.js';
+
+const {
+  catalogId,
+  describeMock,
+  executeUpdateMock,
+  mockLog,
+  queryMock,
+  taskQueue,
+  workflowStartMock
+} = vi.hoisted( () => ( {
+  catalogId: 'test-catalog',
+  describeMock: vi.fn(),
+  executeUpdateMock: vi.fn(),
+  mockLog: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+  queryMock: vi.fn(),
+  taskQueue: 'test-queue',
+  workflowStartMock: vi.fn()
+} ) );
+
+vi.mock( '#logger', () => ( { createChildLogger: () => mockLog } ) );
+vi.mock( '#consts', () => ( { WORKFLOW_CATALOG: 'catalog' } ) );
+vi.mock( '../configs.js', () => ( { catalogId, taskQueue } ) );
+vi.mock( '@temporalio/client', async importOriginal => {
+  const actual = await importOriginal();
+  return {
+    ...actual,
+    Client: vi.fn().mockImplementation( function () {
+      return {
+        workflow: {
+          start: workflowStartMock,
+          getHandle: () => ( { describe: describeMock, query: queryMock, executeUpdate: executeUpdateMock } )
+        }
+      };
+    } )
+  };
+} );
+
+const mockConnection = {};
+const namespace = 'default';
+const catalog = { workflows: [], activities: {} };
+const catalogHash = 'catalog-hash';
+
+const createJob = () => new CatalogJob( { connection: mockConnection, namespace, catalog, catalogHash } );
+
+const flushPromises = async () => Array
+  .from( { length: 10 } )
+  .reduce( promise => promise.then( () => Promise.resolve() ), Promise.resolve() );
+
+describe( 'CatalogJob', () => {
+  beforeEach( () => {
+    vi.clearAllMocks();
+    describeMock.mockResolvedValue( { closeTime: '2024-01-01T00:00:00Z' } );
+    executeUpdateMock.mockResolvedValue( undefined );
+    queryMock.mockResolvedValue( catalogHash );
+    workflowStartMock.mockResolvedValue( undefined );
+  } );
+
+  it( 'completes a previous running stale catalog before starting the new workflow', async () => {
+    describeMock.mockResolvedValue( { closeTime: undefined } );
+    queryMock.mockResolvedValue( 'old-catalog-hash' );
+    const job = createJob();
+
+    await job.run();
+
+    expect( describeMock ).toHaveBeenCalled();
+    expect( queryMock ).toHaveBeenCalledWith( 'get_hash' );
+    expect( executeUpdateMock ).toHaveBeenCalledWith( 'complete', { args: [] } );
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', {
+      taskQueue,
+      workflowId: catalogId,
+      workflowIdConflictPolicy: WorkflowIdConflictPolicy.FAIL,
+      args: [ catalog, catalogHash ]
+    } );
+    expect( job.error ).toBeNull();
+    expect( job.running ).toBe( false );
+  } );
+
+  it( 'keeps the existing catalog workflow when the running hash matches', async () => {
+    describeMock.mockResolvedValue( { closeTime: undefined } );
+    queryMock.mockResolvedValue( catalogHash );
+    const job = createJob();
+
+    await job.run();
+
+    expect( queryMock ).toHaveBeenCalledWith( 'get_hash' );
+    expect( executeUpdateMock ).not.toHaveBeenCalled();
+    expect( workflowStartMock ).not.toHaveBeenCalled();
+    expect( mockLog.info ).toHaveBeenCalledWith( 'Current catalog workflow hash matches worker, restart skipped' );
+    expect( job.error ).toBeNull();
+  } );
+
+  it( 'starts the catalog workflow when no previous catalog exists', async () => {
+    describeMock.mockRejectedValue( new WorkflowNotFoundError( 'not found' ) );
+    const job = createJob();
+
+    await job.run();
+
+    expect( describeMock ).toHaveBeenCalled();
+    expect( queryMock ).not.toHaveBeenCalled();
+    expect( executeUpdateMock ).not.toHaveBeenCalled();
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', {
+      taskQueue,
+      workflowId: catalogId,
+      workflowIdConflictPolicy: WorkflowIdConflictPolicy.FAIL,
+      args: [ catalog, catalogHash ]
+    } );
+    expect( mockLog.warn ).not.toHaveBeenCalled();
+  } );
+
+  it( 'starts the catalog workflow when the previous catalog is closed', async () => {
+    const job = createJob();
+
+    await job.run();
+
+    expect( describeMock ).toHaveBeenCalled();
+    expect( queryMock ).not.toHaveBeenCalled();
+    expect( executeUpdateMock ).not.toHaveBeenCalled();
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', {
+      taskQueue,
+      workflowId: catalogId,
+      workflowIdConflictPolicy: WorkflowIdConflictPolicy.FAIL,
+      args: [ catalog, catalogHash ]
+    } );
+  } );
+
+  it( 'warns and continues when describing or completing the previous catalog fails', async () => {
+    const completeError = new Error( 'complete failed' );
+    describeMock.mockResolvedValue( { closeTime: undefined } );
+    queryMock.mockResolvedValue( 'old-catalog-hash' );
+    executeUpdateMock.mockRejectedValue( completeError );
+    const job = createJob();
+
+    await job.run();
+
+    expect( mockLog.warn ).toHaveBeenCalledWith( 'Error completing previous catalog workflow', { error: completeError } );
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', expect.any( Object ) );
+    expect( job.error ).toBeNull();
+  } );
+
+  it( 'ignores an already-started error when the running catalog hash matches', async () => {
+    const alreadyStartedError = new WorkflowExecutionAlreadyStartedError( 'already started', catalogId, 'catalog' );
+    describeMock.mockRejectedValue( new WorkflowNotFoundError( 'not found' ) );
+    workflowStartMock.mockRejectedValue( alreadyStartedError );
+    queryMock.mockResolvedValue( catalogHash );
+    const job = createJob();
+
+    await job.run();
+
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', expect.any( Object ) );
+    expect( queryMock ).toHaveBeenCalledWith( 'get_hash' );
+    expect( mockLog.info ).toHaveBeenCalledWith(
+      'Ignoring start error: it failed because execution already started but catalog hash matches worker'
+    );
+    expect( job.error ).toBeNull();
+  } );
+
+  it( 'stores start errors and calls the error callback', async () => {
+    const error = new Error( 'start failed' );
+    const onError = vi.fn();
+    workflowStartMock.mockRejectedValue( error );
+    const job = createJob();
+
+    job.onError( onError );
+    await job.run();
+
+    expect( job.error ).toBe( error );
+    expect( onError ).toHaveBeenCalledWith( error );
+    expect( job.running ).toBe( false );
+  } );
+
+  it( 'stores stale already-started errors and calls the error callback', async () => {
+    const alreadyStartedError = new WorkflowExecutionAlreadyStartedError( 'already started', catalogId, 'catalog' );
+    const onError = vi.fn();
+    describeMock.mockRejectedValue( new WorkflowNotFoundError( 'not found' ) );
+    workflowStartMock.mockRejectedValue( alreadyStartedError );
+    queryMock.mockResolvedValue( 'old-catalog-hash' );
+    const job = createJob();
+
+    job.onError( onError );
+    await job.run();
+
+    expect( job.error ).toBe( alreadyStartedError );
+    expect( onError ).toHaveBeenCalledWith( alreadyStartedError );
+    expect( job.running ).toBe( false );
+  } );
+
+  it( 'reports running while a catalog call is pending', async () => {
+    describeMock.mockReturnValue( new Promise( () => {} ) );
+    const job = createJob();
+
+    const run = job.run();
+    await flushPromises();
+
+    expect( job.running ).toBe( true );
+
+    await job.interrupt();
+    await run;
+
+    expect( job.running ).toBe( false );
+    expect( job.error ).toBeNull();
+  } );
+
+  it( 'interrupts pending catalog calls without storing an error or calling the callback', async () => {
+    const onError = vi.fn();
+    describeMock.mockReturnValue( new Promise( () => {} ) );
+    const job = createJob();
+
+    job.onError( onError );
+    const run = job.run();
+    await flushPromises();
+
+    const interrupt = job.interrupt();
+
+    await expect( interrupt ).resolves.toBeUndefined();
+    await expect( run ).resolves.toBeUndefined();
+    expect( job.error ).toBeNull();
+    expect( onError ).not.toHaveBeenCalled();
+    expect( job.running ).toBe( false );
+  } );
+
+  it( 'returns a resolved promise when interrupted before running', async () => {
+    const job = createJob();
+
+    await expect( job.interrupt() ).resolves.toBeUndefined();
+    expect( job.running ).toBe( false );
+    expect( job.error ).toBeNull();
+  } );
+} );

package/src/worker/check.js

@@ -0,0 +1,24 @@
+import { bundleWorkflows } from './bundle.js';
+
+// `output-worker --check` entry: bundle the workflows exactly as the worker would, then
+// exit 0 (ok) / 1 (fail). The exit code is the signal. Workflow-discovery logs write to
+// stdout via the worker logger, so mute stdout to keep output clean for CI/tooling; only
+// real errors surface, on stderr. Catches bad workflow imports (e.g. `node:` built-ins)
+// at build/CI time instead of crash-looping the worker at startup.
+const callerDir = process.argv[2] ?? process.cwd();
+
+process.stdout.write = ( ...args ) => {
+  const callback = args.find( arg => typeof arg === 'function' );
+  if ( callback ) {
+    callback();
+  }
+  return true;
+};
+
+( async () => {
+  await bundleWorkflows( callerDir );
+  process.exit( 0 );
+} )().catch( error => {
+  console.error( error );
+  process.exit( 1 );
+} );

package/src/worker/connection_monitor.js

@@ -0,0 +1,112 @@
+import { createChildLogger } from '#logger';
+import { setTimeout as delay } from 'node:timers/promises';
+import { CancellablePromise } from '#utils';
+
+const ServingStatus = {
+  UNKNOWN: 0,
+  SERVING: 1,
+  NOT_SERVING: 2,
+  SERVICE_UNKNOWN: 3
+};
+
+const log = createChildLogger( 'Connection' );
+
+export class TemporalConnectionMonitor {
+  #MAX_FAILURES = 3;
+  #CHECK_INTERVAL_MS = 60_000;
+  #CHECK_TIMEOUT_MS = 5_000;
+
+  #cancellation = new CancellablePromise();
+  #failures = 0;
+  #error = null;
+  #running = false;
+  #watchPromise = null;
+  #connection = null;
+  #connectionLostCb = null;
+
+  #getTimeout = async () => delay( this.#CHECK_TIMEOUT_MS, 0, { ref: false } ).then( () => {
+    throw new Error( 'Connection health check timed out' );
+  } );
+
+  #healthcheck = async () => this.#connection.healthService.check( {} );
+
+  #sleep = async () => delay( this.#CHECK_INTERVAL_MS, 0, { ref: false } );
+
+  #watch = async () => {
+    while ( !this.#cancellation.completed ) {
+      try {
+        const health = await Promise.race( [ this.#healthcheck(), this.#getTimeout(), this.#cancellation.promise ] );
+
+        // cancellation won the race
+        if ( this.#cancellation.completed ) {
+          break;
+        }
+
+        if ( health?.status !== ServingStatus.SERVING ) {
+          throw new Error( `Connection not serving (status ${health?.status})` );
+        }
+
+        log.info( this.#failures === 0 ? 'Healthy' : 'Recovered' );
+        this.#failures = 0;
+      } catch ( error ) {
+        // cancellation will ignore warnings and not throw errors;
+        if ( this.#cancellation.completed ) {
+          break;
+        }
+
+        if ( ++this.#failures >= this.#MAX_FAILURES ) {
+          log.warn( 'Connection lost', { error: error.message, failures: this.#failures } );
+          this.#error = error;
+          this.#connectionLostCb?.( error );
+          this.#cancellation.complete();
+          break;
+        } else {
+          log.warn( 'Connection unhealthy', { error: error.message, failures: this.#failures } );
+        }
+      }
+
+      await Promise.race( [ this.#sleep(), this.#cancellation.promise ] );
+    }
+  };
+
+  constructor( connection, overrides = {} ) {
+    this.#connection = connection;
+    if ( Number.isFinite( overrides?.maxFailures ) ) {
+      this.#MAX_FAILURES = overrides.maxFailures;
+    }
+    if ( Number.isFinite( overrides?.checkIntervalMs ) ) {
+      this.#CHECK_INTERVAL_MS = overrides.checkIntervalMs;
+    }
+    if ( Number.isFinite( overrides?.checkTimeoutMs ) ) {
+      this.#CHECK_TIMEOUT_MS = overrides.checkTimeoutMs;
+    }
+  }
+
+  onConnectionLost( cb ) {
+    this.#connectionLostCb = cb;
+  }
+
+  get running() {
+    return this.#running;
+  }
+
+  start() {
+    if ( this.#watchPromise ) {
+      return this.#watchPromise;
+    }
+    this.#running = true;
+    this.#watchPromise = this.#watch().finally( () => {
+      this.#running = false;
+    } );
+    return this.#watchPromise;
+  }
+
+  stop() {
+    this.#cancellation.complete();
+    return this.#watchPromise ?? Promise.resolve();
+  }
+
+  get connectionLossError() {
+    return this.#error;
+  }
+};