@outputai/core 0.7.0 → 0.7.1-next.0e958f3.0

package/src/worker/catalog_workflow/catalog_job.spec.js

@@ -0,0 +1,232 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { WorkflowNotFoundError } from '@temporalio/client';
+import { WorkflowExecutionAlreadyStartedError, WorkflowIdConflictPolicy } from '@temporalio/common';
+import { CatalogJob } from './catalog_job.js';
+
+const {
+  catalogId,
+  describeMock,
+  executeUpdateMock,
+  mockLog,
+  queryMock,
+  taskQueue,
+  workflowStartMock
+} = vi.hoisted( () => ( {
+  catalogId: 'test-catalog',
+  describeMock: vi.fn(),
+  executeUpdateMock: vi.fn(),
+  mockLog: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+  queryMock: vi.fn(),
+  taskQueue: 'test-queue',
+  workflowStartMock: vi.fn()
+} ) );
+
+vi.mock( '#logger', () => ( { createChildLogger: () => mockLog } ) );
+vi.mock( '#consts', () => ( { WORKFLOW_CATALOG: 'catalog' } ) );
+vi.mock( '../configs.js', () => ( { catalogId, taskQueue } ) );
+vi.mock( '@temporalio/client', async importOriginal => {
+  const actual = await importOriginal();
+  return {
+    ...actual,
+    Client: vi.fn().mockImplementation( function () {
+      return {
+        workflow: {
+          start: workflowStartMock,
+          getHandle: () => ( { describe: describeMock, query: queryMock, executeUpdate: executeUpdateMock } )
+        }
+      };
+    } )
+  };
+} );
+
+const mockConnection = {};
+const namespace = 'default';
+const catalog = { workflows: [], activities: {} };
+const catalogHash = 'catalog-hash';
+
+const createJob = () => new CatalogJob( { connection: mockConnection, namespace, catalog, catalogHash } );
+
+const flushPromises = async () => Array
+  .from( { length: 10 } )
+  .reduce( promise => promise.then( () => Promise.resolve() ), Promise.resolve() );
+
+describe( 'CatalogJob', () => {
+  beforeEach( () => {
+    vi.clearAllMocks();
+    describeMock.mockResolvedValue( { closeTime: '2024-01-01T00:00:00Z' } );
+    executeUpdateMock.mockResolvedValue( undefined );
+    queryMock.mockResolvedValue( catalogHash );
+    workflowStartMock.mockResolvedValue( undefined );
+  } );
+
+  it( 'completes a previous running stale catalog before starting the new workflow', async () => {
+    describeMock.mockResolvedValue( { closeTime: undefined } );
+    queryMock.mockResolvedValue( 'old-catalog-hash' );
+    const job = createJob();
+
+    await job.run();
+
+    expect( describeMock ).toHaveBeenCalled();
+    expect( queryMock ).toHaveBeenCalledWith( 'get_hash' );
+    expect( executeUpdateMock ).toHaveBeenCalledWith( 'complete', { args: [] } );
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', {
+      taskQueue,
+      workflowId: catalogId,
+      workflowIdConflictPolicy: WorkflowIdConflictPolicy.FAIL,
+      args: [ catalog, catalogHash ]
+    } );
+    expect( job.error ).toBeNull();
+    expect( job.running ).toBe( false );
+  } );
+
+  it( 'keeps the existing catalog workflow when the running hash matches', async () => {
+    describeMock.mockResolvedValue( { closeTime: undefined } );
+    queryMock.mockResolvedValue( catalogHash );
+    const job = createJob();
+
+    await job.run();
+
+    expect( queryMock ).toHaveBeenCalledWith( 'get_hash' );
+    expect( executeUpdateMock ).not.toHaveBeenCalled();
+    expect( workflowStartMock ).not.toHaveBeenCalled();
+    expect( mockLog.info ).toHaveBeenCalledWith( 'Current catalog workflow hash matches worker, restart skipped' );
+    expect( job.error ).toBeNull();
+  } );
+
+  it( 'starts the catalog workflow when no previous catalog exists', async () => {
+    describeMock.mockRejectedValue( new WorkflowNotFoundError( 'not found' ) );
+    const job = createJob();
+
+    await job.run();
+
+    expect( describeMock ).toHaveBeenCalled();
+    expect( queryMock ).not.toHaveBeenCalled();
+    expect( executeUpdateMock ).not.toHaveBeenCalled();
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', {
+      taskQueue,
+      workflowId: catalogId,
+      workflowIdConflictPolicy: WorkflowIdConflictPolicy.FAIL,
+      args: [ catalog, catalogHash ]
+    } );
+    expect( mockLog.warn ).not.toHaveBeenCalled();
+  } );
+
+  it( 'starts the catalog workflow when the previous catalog is closed', async () => {
+    const job = createJob();
+
+    await job.run();
+
+    expect( describeMock ).toHaveBeenCalled();
+    expect( queryMock ).not.toHaveBeenCalled();
+    expect( executeUpdateMock ).not.toHaveBeenCalled();
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', {
+      taskQueue,
+      workflowId: catalogId,
+      workflowIdConflictPolicy: WorkflowIdConflictPolicy.FAIL,
+      args: [ catalog, catalogHash ]
+    } );
+  } );
+
+  it( 'warns and continues when describing or completing the previous catalog fails', async () => {
+    const completeError = new Error( 'complete failed' );
+    describeMock.mockResolvedValue( { closeTime: undefined } );
+    queryMock.mockResolvedValue( 'old-catalog-hash' );
+    executeUpdateMock.mockRejectedValue( completeError );
+    const job = createJob();
+
+    await job.run();
+
+    expect( mockLog.warn ).toHaveBeenCalledWith( 'Error completing previous catalog workflow', { error: completeError } );
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', expect.any( Object ) );
+    expect( job.error ).toBeNull();
+  } );
+
+  it( 'ignores an already-started error when the running catalog hash matches', async () => {
+    const alreadyStartedError = new WorkflowExecutionAlreadyStartedError( 'already started', catalogId, 'catalog' );
+    describeMock.mockRejectedValue( new WorkflowNotFoundError( 'not found' ) );
+    workflowStartMock.mockRejectedValue( alreadyStartedError );
+    queryMock.mockResolvedValue( catalogHash );
+    const job = createJob();
+
+    await job.run();
+
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', expect.any( Object ) );
+    expect( queryMock ).toHaveBeenCalledWith( 'get_hash' );
+    expect( mockLog.info ).toHaveBeenCalledWith(
+      'Ignoring start error: it failed because execution already started but catalog hash matches worker'
+    );
+    expect( job.error ).toBeNull();
+  } );
+
+  it( 'stores start errors and calls the error callback', async () => {
+    const error = new Error( 'start failed' );
+    const onError = vi.fn();
+    workflowStartMock.mockRejectedValue( error );
+    const job = createJob();
+
+    job.onError( onError );
+    await job.run();
+
+    expect( job.error ).toBe( error );
+    expect( onError ).toHaveBeenCalledWith( error );
+    expect( job.running ).toBe( false );
+  } );
+
+  it( 'stores stale already-started errors and calls the error callback', async () => {
+    const alreadyStartedError = new WorkflowExecutionAlreadyStartedError( 'already started', catalogId, 'catalog' );
+    const onError = vi.fn();
+    describeMock.mockRejectedValue( new WorkflowNotFoundError( 'not found' ) );
+    workflowStartMock.mockRejectedValue( alreadyStartedError );
+    queryMock.mockResolvedValue( 'old-catalog-hash' );
+    const job = createJob();
+
+    job.onError( onError );
+    await job.run();
+
+    expect( job.error ).toBe( alreadyStartedError );
+    expect( onError ).toHaveBeenCalledWith( alreadyStartedError );
+    expect( job.running ).toBe( false );
+  } );
+
+  it( 'reports running while a catalog call is pending', async () => {
+    describeMock.mockReturnValue( new Promise( () => {} ) );
+    const job = createJob();
+
+    const run = job.run();
+    await flushPromises();
+
+    expect( job.running ).toBe( true );
+
+    await job.interrupt();
+    await run;
+
+    expect( job.running ).toBe( false );
+    expect( job.error ).toBeNull();
+  } );
+
+  it( 'interrupts pending catalog calls without storing an error or calling the callback', async () => {
+    const onError = vi.fn();
+    describeMock.mockReturnValue( new Promise( () => {} ) );
+    const job = createJob();
+
+    job.onError( onError );
+    const run = job.run();
+    await flushPromises();
+
+    const interrupt = job.interrupt();
+
+    await expect( interrupt ).resolves.toBeUndefined();
+    await expect( run ).resolves.toBeUndefined();
+    expect( job.error ).toBeNull();
+    expect( onError ).not.toHaveBeenCalled();
+    expect( job.running ).toBe( false );
+  } );
+
+  it( 'returns a resolved promise when interrupted before running', async () => {
+    const job = createJob();
+
+    await expect( job.interrupt() ).resolves.toBeUndefined();
+    expect( job.running ).toBe( false );
+    expect( job.error ).toBeNull();
+  } );
+} );

package/src/worker/check.js

@@ -0,0 +1,24 @@
+import { bundleWorkflows } from './bundle.js';
+
+// `output-worker --check` entry: bundle the workflows exactly as the worker would, then
+// exit 0 (ok) / 1 (fail). The exit code is the signal. Workflow-discovery logs write to
+// stdout via the worker logger, so mute stdout to keep output clean for CI/tooling; only
+// real errors surface, on stderr. Catches bad workflow imports (e.g. `node:` built-ins)
+// at build/CI time instead of crash-looping the worker at startup.
+const callerDir = process.argv[2] ?? process.cwd();
+
+process.stdout.write = ( ...args ) => {
+  const callback = args.find( arg => typeof arg === 'function' );
+  if ( callback ) {
+    callback();
+  }
+  return true;
+};
+
+( async () => {
+  await bundleWorkflows( callerDir );
+  process.exit( 0 );
+} )().catch( error => {
+  console.error( error );
+  process.exit( 1 );
+} );

package/src/worker/connection_monitor.js

@@ -0,0 +1,112 @@
+import { createChildLogger } from '#logger';
+import { setTimeout as delay } from 'node:timers/promises';
+import { CancellablePromise } from '#utils';
+
+const ServingStatus = {
+  UNKNOWN: 0,
+  SERVING: 1,
+  NOT_SERVING: 2,
+  SERVICE_UNKNOWN: 3
+};
+
+const log = createChildLogger( 'Connection' );
+
+export class TemporalConnectionMonitor {
+  #MAX_FAILURES = 3;
+  #CHECK_INTERVAL_MS = 60_000;
+  #CHECK_TIMEOUT_MS = 5_000;
+
+  #cancellation = new CancellablePromise();
+  #failures = 0;
+  #error = null;
+  #running = false;
+  #watchPromise = null;
+  #connection = null;
+  #connectionLostCb = null;
+
+  #getTimeout = async () => delay( this.#CHECK_TIMEOUT_MS, 0, { ref: false } ).then( () => {
+    throw new Error( 'Connection health check timed out' );
+  } );
+
+  #healthcheck = async () => this.#connection.healthService.check( {} );
+
+  #sleep = async () => delay( this.#CHECK_INTERVAL_MS, 0, { ref: false } );
+
+  #watch = async () => {
+    while ( !this.#cancellation.completed ) {
+      try {
+        const health = await Promise.race( [ this.#healthcheck(), this.#getTimeout(), this.#cancellation.promise ] );
+
+        // cancellation won the race
+        if ( this.#cancellation.completed ) {
+          break;
+        }
+
+        if ( health?.status !== ServingStatus.SERVING ) {
+          throw new Error( `Connection not serving (status ${health?.status})` );
+        }
+
+        log.info( this.#failures === 0 ? 'Healthy' : 'Recovered' );
+        this.#failures = 0;
+      } catch ( error ) {
+        // cancellation will ignore warnings and not throw errors;
+        if ( this.#cancellation.completed ) {
+          break;
+        }
+
+        if ( ++this.#failures >= this.#MAX_FAILURES ) {
+          log.warn( 'Connection lost', { error: error.message, failures: this.#failures } );
+          this.#error = error;
+          this.#connectionLostCb?.( error );
+          this.#cancellation.complete();
+          break;
+        } else {
+          log.warn( 'Connection unhealthy', { error: error.message, failures: this.#failures } );
+        }
+      }
+
+      await Promise.race( [ this.#sleep(), this.#cancellation.promise ] );
+    }
+  };
+
+  constructor( connection, overrides = {} ) {
+    this.#connection = connection;
+    if ( Number.isFinite( overrides?.maxFailures ) ) {
+      this.#MAX_FAILURES = overrides.maxFailures;
+    }
+    if ( Number.isFinite( overrides?.checkIntervalMs ) ) {
+      this.#CHECK_INTERVAL_MS = overrides.checkIntervalMs;
+    }
+    if ( Number.isFinite( overrides?.checkTimeoutMs ) ) {
+      this.#CHECK_TIMEOUT_MS = overrides.checkTimeoutMs;
+    }
+  }
+
+  onConnectionLost( cb ) {
+    this.#connectionLostCb = cb;
+  }
+
+  get running() {
+    return this.#running;
+  }
+
+  start() {
+    if ( this.#watchPromise ) {
+      return this.#watchPromise;
+    }
+    this.#running = true;
+    this.#watchPromise = this.#watch().finally( () => {
+      this.#running = false;
+    } );
+    return this.#watchPromise;
+  }
+
+  stop() {
+    this.#cancellation.complete();
+    return this.#watchPromise ?? Promise.resolve();
+  }
+
+  get connectionLossError() {
+    return this.#error;
+  }
+};

package/src/worker/connection_monitor.spec.js

@@ -0,0 +1,199 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { TemporalConnectionMonitor } from './connection_monitor.js';
+
+const SERVING = 1;
+const NOT_SERVING = 2;
+const CHECK_TIMEOUT_MS = 50;
+const CHECK_INTERVAL_MS = 100;
+
+const { scheduledDelays, delayMock, mockLogger } = vi.hoisted( () => {
+  const scheduledDelays = [];
+  const delayMock = vi.fn( ( ms, value, options ) => new Promise( resolve => {
+    scheduledDelays.push( { ms, value, options, resolve } );
+  } ) );
+
+  return {
+    scheduledDelays,
+    delayMock,
+    mockLogger: {
+      info: vi.fn(),
+      warn: vi.fn()
+    }
+  };
+} );
+
+vi.mock( 'node:timers/promises', () => ( { setTimeout: delayMock } ) );
+vi.mock( '#logger', () => ( { createChildLogger: vi.fn( () => mockLogger ) } ) );
+
+const createConnection = check => ( {
+  healthService: { check }
+} );
+
+const createMonitor = ( check, overrides = {} ) => new TemporalConnectionMonitor( createConnection( check ), {
+  checkIntervalMs: CHECK_INTERVAL_MS,
+  checkTimeoutMs: CHECK_TIMEOUT_MS,
+  ...overrides
+} );
+
+const flushPromises = async () => Array
+  .from( { length: 10 } )
+  .reduce( promise => promise.then( () => Promise.resolve() ), Promise.resolve() );
+
+const resolveNextDelay = ms => {
+  const index = scheduledDelays.findIndex( delay => delay.ms === ms );
+  expect( index ).not.toBe( -1 );
+  const [ scheduled ] = scheduledDelays.splice( index, 1 );
+  scheduled.resolve( scheduled.value );
+};
+
+describe( 'TemporalConnectionMonitor', () => {
+  beforeEach( () => {
+    vi.clearAllMocks();
+    scheduledDelays.length = 0;
+  } );
+
+  it( 'logs healthy when the connection is serving', async () => {
+    const check = vi.fn().mockResolvedValue( { status: SERVING } );
+    const monitor = createMonitor( check );
+
+    const run = monitor.start();
+    await flushPromises();
+
+    expect( check ).toHaveBeenCalledWith( {} );
+    expect( mockLogger.info ).toHaveBeenCalledWith( 'Healthy' );
+    expect( delayMock ).toHaveBeenCalledWith( CHECK_TIMEOUT_MS, 0, { ref: false } );
+    expect( delayMock ).toHaveBeenCalledWith( CHECK_INTERVAL_MS, 0, { ref: false } );
+    expect( monitor.running ).toBe( true );
+
+    await monitor.stop();
+    await run;
+
+    expect( monitor.running ).toBe( false );
+  } );
+
+  it( 'logs transient timeout failures before retrying', async () => {
+    const check = vi.fn().mockReturnValue( new Promise( () => {} ) );
+    const monitor = createMonitor( check );
+
+    monitor.start();
+    resolveNextDelay( CHECK_TIMEOUT_MS );
+    await flushPromises();
+
+    expect( mockLogger.warn ).toHaveBeenCalledWith( 'Connection unhealthy', {
+      error: 'Connection health check timed out',
+      failures: 1
+    } );
+
+    await monitor.stop();
+  } );
+
+  it( 'logs recovered after a transient failure succeeds', async () => {
+    const check = vi.fn()
+      .mockRejectedValueOnce( new Error( 'temporary outage' ) )
+      .mockResolvedValueOnce( { status: SERVING } );
+    const monitor = createMonitor( check );
+
+    monitor.start();
+    await flushPromises();
+
+    expect( mockLogger.warn ).toHaveBeenCalledWith( 'Connection unhealthy', {
+      error: 'temporary outage',
+      failures: 1
+    } );
+
+    resolveNextDelay( CHECK_INTERVAL_MS );
+    await flushPromises();
+
+    expect( mockLogger.info ).toHaveBeenCalledWith( 'Recovered' );
+
+    await monitor.stop();
+  } );
+
+  it( 'stores connection loss error and calls callback after max consecutive failures', async () => {
+    const error = new Error( 'connection refused' );
+    const check = vi.fn().mockRejectedValue( error );
+    const connectionLost = vi.fn();
+    const monitor = createMonitor( check );
+
+    monitor.onConnectionLost( connectionLost );
+    monitor.start();
+
+    await flushPromises();
+    resolveNextDelay( CHECK_INTERVAL_MS );
+    await flushPromises();
+    resolveNextDelay( CHECK_INTERVAL_MS );
+    await flushPromises();
+
+    expect( mockLogger.warn ).toHaveBeenCalledTimes( 3 );
+    expect( mockLogger.warn ).toHaveBeenCalledWith( 'Connection lost', {
+      error: 'connection refused',
+      failures: 3
+    } );
+    expect( connectionLost ).toHaveBeenCalledWith( error );
+    expect( monitor.connectionLossError ).toBe( error );
+    expect( monitor.running ).toBe( false );
+  } );
+
+  it( 'treats non-serving health status as a failure', async () => {
+    const check = vi.fn().mockResolvedValue( { status: NOT_SERVING } );
+    const monitor = createMonitor( check );
+
+    monitor.start();
+    await flushPromises();
+
+    expect( mockLogger.warn ).toHaveBeenCalledWith( 'Connection unhealthy', {
+      error: `Connection not serving (status ${NOT_SERVING})`,
+      failures: 1
+    } );
+
+    await monitor.stop();
+  } );
+
+  it( 'returns the same lifecycle promise when started more than once', async () => {
+    const check = vi.fn().mockReturnValue( new Promise( () => {} ) );
+    const monitor = createMonitor( check );
+
+    const firstRun = monitor.start();
+    const secondRun = monitor.start();
+
+    expect( secondRun ).toBe( firstRun );
+    expect( check ).toHaveBeenCalledOnce();
+
+    await monitor.stop();
+  } );
+
+  it( 'stops without calling connection lost callback for in-flight health checks', async () => {
+    const check = vi.fn().mockReturnValue( new Promise( () => {} ) );
+    const connectionLost = vi.fn();
+    const monitor = createMonitor( check, { maxFailures: 1 } );
+
+    monitor.onConnectionLost( connectionLost );
+    monitor.start();
+
+    expect( monitor.running ).toBe( true );
+
+    await monitor.stop();
+
+    expect( connectionLost ).not.toHaveBeenCalled();
+    expect( monitor.connectionLossError ).toBeNull();
+    expect( monitor.running ).toBe( false );
+  } );
+
+  it( 'applies timing and failure threshold overrides', async () => {
+    const error = new Error( 'fast failure' );
+    const check = vi.fn().mockRejectedValue( error );
+    const connectionLost = vi.fn();
+    const monitor = createMonitor( check, {
+      maxFailures: 1,
+      checkIntervalMs: 7,
+      checkTimeoutMs: 3
+    } );
+
+    monitor.onConnectionLost( connectionLost );
+    await monitor.start();
+
+    expect( delayMock ).toHaveBeenCalledWith( 3, 0, { ref: false } );
+    expect( delayMock ).not.toHaveBeenCalledWith( 7, 0, { ref: false } );
+    expect( connectionLost ).toHaveBeenCalledWith( error );
+  } );
+} );