@outputai/core 0.1.0

package/src/worker/log_hooks.spec.js

@@ -0,0 +1,217 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import {
+  BusEventType,
+  ComponentType,
+  LifecycleEvent,
+  WORKFLOW_CATALOG
+} from '#consts';
+
+const activityLogMock = vi.hoisted( () => ( { info: vi.fn(), error: vi.fn() } ) );
+const workflowLogMock = vi.hoisted( () => ( { info: vi.fn(), error: vi.fn() } ) );
+const createChildLoggerMock = vi.hoisted( () =>
+  vi.fn( name => ( name === 'Activity' ? activityLogMock : workflowLogMock ) )
+);
+
+const onHandlers = vi.hoisted( () => ( {} ) );
+const messageBusMock = vi.hoisted( () => ( {
+  on: vi.fn( ( eventType, handler ) => {
+    onHandlers[eventType] = handler;
+  } )
+} ) );
+
+vi.mock( '#logger', () => ( { createChildLogger: createChildLoggerMock } ) );
+vi.mock( '#bus', () => ( { messageBus: messageBusMock } ) );
+
+import './log_hooks.js';
+
+describe( 'log_hooks', () => {
+  beforeEach( () => {
+    vi.clearAllMocks();
+  } );
+
+  describe( 'activity events', () => {
+    const basePayload = {
+      id: 'act-1',
+      name: 'myWorkflow#myStep',
+      kind: 'step',
+      workflowId: 'wf-1',
+      workflowName: 'myWorkflow'
+    };
+
+    it( 'ACTIVITY_START logs full message and second arg', () => {
+      onHandlers[BusEventType.ACTIVITY_START]( basePayload );
+
+      expect( activityLogMock.info ).toHaveBeenCalledTimes( 1 );
+      expect( activityLogMock.info ).toHaveBeenCalledWith(
+        'Started myWorkflow#myStep step',
+        {
+          event: LifecycleEvent.START,
+          activityId: 'act-1',
+          activityName: 'myWorkflow#myStep',
+          activityKind: 'step',
+          workflowId: 'wf-1',
+          workflowName: 'myWorkflow'
+        }
+      );
+    } );
+
+    it( 'ACTIVITY_START does not log when kind is INTERNAL_STEP', () => {
+      onHandlers[BusEventType.ACTIVITY_START]( {
+        ...basePayload,
+        kind: ComponentType.INTERNAL_STEP
+      } );
+
+      expect( activityLogMock.info ).not.toHaveBeenCalled();
+    } );
+
+    it( 'ACTIVITY_END logs full message and second arg', () => {
+      onHandlers[BusEventType.ACTIVITY_END]( { ...basePayload, duration: 42 } );
+
+      expect( activityLogMock.info ).toHaveBeenCalledTimes( 1 );
+      expect( activityLogMock.info ).toHaveBeenCalledWith(
+        'Ended myWorkflow#myStep step',
+        {
+          event: LifecycleEvent.END,
+          activityId: 'act-1',
+          activityName: 'myWorkflow#myStep',
+          activityKind: 'step',
+          workflowId: 'wf-1',
+          workflowName: 'myWorkflow',
+          durationMs: 42
+        }
+      );
+    } );
+
+    it( 'ACTIVITY_END does not log when kind is INTERNAL_STEP', () => {
+      onHandlers[BusEventType.ACTIVITY_END]( {
+        ...basePayload,
+        kind: ComponentType.INTERNAL_STEP,
+        duration: 10
+      } );
+
+      expect( activityLogMock.info ).not.toHaveBeenCalled();
+    } );
+
+    it( 'ACTIVITY_ERROR logs full message and second arg', () => {
+      const err = new Error( 'step failed' );
+      onHandlers[BusEventType.ACTIVITY_ERROR]( {
+        ...basePayload,
+        duration: 100,
+        error: err
+      } );
+
+      expect( activityLogMock.error ).toHaveBeenCalledTimes( 1 );
+      expect( activityLogMock.error ).toHaveBeenCalledWith(
+        'Error myWorkflow#myStep step: Error',
+        {
+          event: LifecycleEvent.ERROR,
+          activityId: 'act-1',
+          activityName: 'myWorkflow#myStep',
+          activityKind: 'step',
+          workflowId: 'wf-1',
+          workflowName: 'myWorkflow',
+          durationMs: 100,
+          error: 'step failed'
+        }
+      );
+    } );
+
+    it( 'ACTIVITY_ERROR does not log when kind is INTERNAL_STEP', () => {
+      onHandlers[BusEventType.ACTIVITY_ERROR]( {
+        ...basePayload,
+        kind: ComponentType.INTERNAL_STEP,
+        duration: 5,
+        error: new Error( 'x' )
+      } );
+
+      expect( activityLogMock.error ).not.toHaveBeenCalled();
+    } );
+  } );
+
+  describe( 'workflow events', () => {
+    const basePayload = { id: 'wf-1', name: 'myWorkflow' };
+
+    it( 'WORKFLOW_START logs full message and second arg', () => {
+      onHandlers[BusEventType.WORKFLOW_START]( basePayload );
+
+      expect( workflowLogMock.info ).toHaveBeenCalledTimes( 1 );
+      expect( workflowLogMock.info ).toHaveBeenCalledWith(
+        'Started myWorkflow workflow',
+        {
+          event: LifecycleEvent.START,
+          workflowId: 'wf-1',
+          workflowName: 'myWorkflow'
+        }
+      );
+    } );
+
+    it( 'WORKFLOW_START does not log when name is WORKFLOW_CATALOG', () => {
+      onHandlers[BusEventType.WORKFLOW_START]( {
+        id: 'cat-1',
+        name: WORKFLOW_CATALOG
+      } );
+
+      expect( workflowLogMock.info ).not.toHaveBeenCalled();
+    } );
+
+    it( 'WORKFLOW_END logs full message and second arg', () => {
+      onHandlers[BusEventType.WORKFLOW_END]( {
+        ...basePayload,
+        duration: 200
+      } );
+
+      expect( workflowLogMock.info ).toHaveBeenCalledTimes( 1 );
+      expect( workflowLogMock.info ).toHaveBeenCalledWith(
+        'Ended myWorkflow workflow',
+        {
+          event: LifecycleEvent.END,
+          workflowId: 'wf-1',
+          workflowName: 'myWorkflow',
+          durationMs: 200
+        }
+      );
+    } );
+
+    it( 'WORKFLOW_END does not log when name is WORKFLOW_CATALOG', () => {
+      onHandlers[BusEventType.WORKFLOW_END]( {
+        id: 'cat-1',
+        name: WORKFLOW_CATALOG,
+        duration: 50
+      } );
+
+      expect( workflowLogMock.info ).not.toHaveBeenCalled();
+    } );
+
+    it( 'WORKFLOW_ERROR logs full message and second arg', () => {
+      const err = new TypeError( 'workflow boom' );
+      onHandlers[BusEventType.WORKFLOW_ERROR]( {
+        ...basePayload,
+        duration: 150,
+        error: err
+      } );
+
+      expect( workflowLogMock.error ).toHaveBeenCalledTimes( 1 );
+      expect( workflowLogMock.error ).toHaveBeenCalledWith(
+        'Error myWorkflow workflow: TypeError',
+        {
+          event: LifecycleEvent.ERROR,
+          workflowId: 'wf-1',
+          workflowName: 'myWorkflow',
+          durationMs: 150,
+          error: 'workflow boom'
+        }
+      );
+    } );
+
+    it( 'WORKFLOW_ERROR does not log when name is WORKFLOW_CATALOG', () => {
+      onHandlers[BusEventType.WORKFLOW_ERROR]( {
+        id: 'cat-1',
+        name: WORKFLOW_CATALOG,
+        duration: 1,
+        error: new Error( 'x' )
+      } );
+
+      expect( workflowLogMock.error ).not.toHaveBeenCalled();
+    } );
+  } );
+} );

package/src/worker/sandboxed_utils.js

@@ -0,0 +1,18 @@
+// THIS IS SAFE TO RUN AT TEMPORAL'S SANDBOX ENVIRONMENT
+import { defaultPayloadConverter } from '@temporalio/common';
+
+/*
+  @important: They plain JS values need to be converted to "payload":
+  - https://typescript.temporal.io/api/namespaces/common/#headers
+  - https://community.temporal.io/t/specify-temporal-headers-when-starting-workflow/6712
+*/
+export const memoToHeaders = memo =>
+  Object.fromEntries(
+    Object.entries( memo ?? {} ).map( ( [ k, v ] ) => [ k, defaultPayloadConverter.toPayload( v ) ] )
+  );
+
+// And the opposite of the function above
+export const headersToObject = headers =>
+  Object.fromEntries(
+    Object.entries( headers ?? {} ).map( ( [ k, v ] ) => [ k, defaultPayloadConverter.fromPayload( v ) ] )
+  );

package/src/worker/shutdown.js

@@ -0,0 +1,26 @@
+const FORCE_QUIT_GRACE_MS = 1000;
+
+export const registerShutdown = ( { worker, log } ) => {
+  const state = { isShuttingDown: false, shutdownStartedAt: null };
+
+  const shutdown = signal => {
+    if ( state.isShuttingDown ) {
+      const elapsed = Date.now() - state.shutdownStartedAt;
+
+      // If running with npx, 2 kill signals are received in rapid succession,
+      // this ignores the second interruption when it is right after the first.
+      if ( elapsed < FORCE_QUIT_GRACE_MS ) {
+        return;
+      }
+      log.warn( 'Force quitting...' );
+      process.exit( 1 );
+    }
+    state.isShuttingDown = true;
+    state.shutdownStartedAt = Date.now();
+    log.info( 'Shutting down...', { signal } );
+    worker.shutdown();
+  };
+
+  process.on( 'SIGTERM', () => shutdown( 'SIGTERM' ) );
+  process.on( 'SIGINT', () => shutdown( 'SIGINT' ) );
+};

package/src/worker/shutdown.spec.js

@@ -0,0 +1,82 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { registerShutdown } from './shutdown.js';
+
+describe( 'worker/shutdown', () => {
+  const mockLog = { info: vi.fn(), warn: vi.fn() };
+  const shutdownMock = vi.fn();
+  const mockWorker = { shutdown: shutdownMock };
+  const onHandlers = {};
+  const exitMock = vi.fn();
+  const originalOn = process.on;
+  const originalExit = process.exit;
+
+  beforeEach( () => {
+    vi.clearAllMocks();
+    Object.keys( onHandlers ).forEach( k => delete onHandlers[k] );
+    process.on = vi.fn( ( event, handler ) => {
+      onHandlers[event] = handler;
+    } );
+    process.exit = exitMock;
+  } );
+
+  afterEach( () => {
+    process.on = originalOn;
+    process.exit = originalExit;
+  } );
+
+  it( 'registers SIGTERM and SIGINT handlers', () => {
+    registerShutdown( { worker: mockWorker, log: mockLog } );
+
+    expect( process.on ).toHaveBeenCalledWith( 'SIGTERM', expect.any( Function ) );
+    expect( process.on ).toHaveBeenCalledWith( 'SIGINT', expect.any( Function ) );
+  } );
+
+  it( 'on first signal: logs, calls worker.shutdown(), does not exit', () => {
+    registerShutdown( { worker: mockWorker, log: mockLog } );
+
+    onHandlers.SIGTERM();
+
+    expect( mockLog.info ).toHaveBeenCalledWith( 'Shutting down...', { signal: 'SIGTERM' } );
+    expect( shutdownMock ).toHaveBeenCalledTimes( 1 );
+    expect( mockLog.warn ).not.toHaveBeenCalled();
+    expect( exitMock ).not.toHaveBeenCalled();
+  } );
+
+  it( 'on first SIGINT: logs with SIGINT', () => {
+    registerShutdown( { worker: mockWorker, log: mockLog } );
+
+    onHandlers.SIGINT();
+
+    expect( mockLog.info ).toHaveBeenCalledWith( 'Shutting down...', { signal: 'SIGINT' } );
+    expect( shutdownMock ).toHaveBeenCalledTimes( 1 );
+  } );
+
+  it( 'on second signal within grace period: ignores (no force quit)', () => {
+    vi.useFakeTimers();
+    registerShutdown( { worker: mockWorker, log: mockLog } );
+
+    onHandlers.SIGTERM();
+    onHandlers.SIGINT();
+
+    expect( mockLog.info ).toHaveBeenCalledTimes( 1 );
+    expect( shutdownMock ).toHaveBeenCalledTimes( 1 );
+    expect( mockLog.warn ).not.toHaveBeenCalled();
+    expect( exitMock ).not.toHaveBeenCalled();
+
+    vi.useRealTimers();
+  } );
+
+  it( 'on second signal after grace period: logs force quit and exits with 1', () => {
+    vi.useFakeTimers();
+    registerShutdown( { worker: mockWorker, log: mockLog } );
+
+    onHandlers.SIGTERM();
+    vi.advanceTimersByTime( 1001 );
+    onHandlers.SIGINT();
+
+    expect( mockLog.warn ).toHaveBeenCalledWith( 'Force quitting...' );
+    expect( exitMock ).toHaveBeenCalledWith( 1 );
+
+    vi.useRealTimers();
+  } );
+} );

package/src/worker/sinks.js

@@ -0,0 +1,74 @@
+import { BusEventType, ComponentType } from '#consts';
+import * as Tracing from '#tracing';
+import { messageBus } from '#bus';
+
+// This sink allow for sandbox Temporal environment to send trace logs back to the main thread.
+export const sinks = {
+
+  /**
+   * Workflow lifecycle sinks
+   */
+  workflow: {
+    start: {
+      fn: ( workflowInfo, input ) => {
+        const { workflowId: id, workflowType: name, memo: { parentId, executionContext } } = workflowInfo;
+        messageBus.emit( BusEventType.WORKFLOW_START, { id, name } );
+        if ( executionContext ) { // filters out internal workflows
+          Tracing.addEventStart( { id, kind: ComponentType.WORKFLOW, name, details: input, parentId, executionContext } );
+        }
+      },
+      callDuringReplay: false
+    },
+
+    end: {
+      fn: ( workflowInfo, output ) => {
+        const { workflowId: id, workflowType: name, startTime, memo: { executionContext } } = workflowInfo;
+        messageBus.emit( BusEventType.WORKFLOW_END, { id, name, duration: Date.now() - startTime.getTime() } );
+        if ( executionContext ) { // filters out internal workflows
+          Tracing.addEventEnd( { id, details: output, executionContext } );
+        }
+      },
+      callDuringReplay: false
+    },
+
+    error: {
+      fn: ( workflowInfo, error ) => {
+        const { workflowId: id, workflowType: name, startTime, memo: { executionContext } } = workflowInfo;
+        messageBus.emit( BusEventType.WORKFLOW_ERROR, { id, name, error, duration: Date.now() - startTime.getTime() } );
+        if ( executionContext ) { // filters out internal workflows
+          Tracing.addEventError( { id, details: error, executionContext } );
+        }
+      },
+      callDuringReplay: false
+    }
+  },
+
+  /**
+   * Generic trace sinks
+   */
+  trace: {
+    start: {
+      fn: ( workflowInfo, { id, name, kind, details } ) => {
+        const { memo: { executionContext, parentId } } = workflowInfo;
+        Tracing.addEventStart( { id, kind, name, details, parentId, executionContext } );
+      },
+      callDuringReplay: false
+    },
+
+    end: {
+      fn: ( workflowInfo, { id, details } ) => {
+        const { memo: { executionContext } } = workflowInfo;
+        Tracing.addEventEnd( { id, details, executionContext } );
+      },
+      callDuringReplay: false
+    },
+
+    error: {
+      fn: ( workflowInfo, { id, details } ) => {
+        const { memo: { executionContext } } = workflowInfo;
+        Tracing.addEventError( { id, details, executionContext } );
+      },
+      callDuringReplay: false
+    }
+  }
+};

package/src/worker/start_catalog.js

@@ -0,0 +1,36 @@
+import { Client, WorkflowNotFoundError } from '@temporalio/client';
+import { WorkflowIdConflictPolicy } from '@temporalio/common';
+import { WORKFLOW_CATALOG } from '#consts';
+import { catalogId, taskQueue } from './configs.js';
+import { createChildLogger } from '#logger';
+
+const log = createChildLogger( 'Catalog' );
+
+export const startCatalog = async ( { connection, namespace, catalog } ) => {
+  const client = new Client( { connection, namespace } );
+  const catalogWorkflowHandle = client.workflow.getHandle( catalogId );
+
+  try {
+    const catalogWorkflowDescription = await catalogWorkflowHandle.describe();
+    if ( !catalogWorkflowDescription.closeTime ) {
+      log.info( 'Completing previous catalog workflow...' );
+      await catalogWorkflowHandle.executeUpdate( 'complete', { args: [] } );
+    }
+  } catch ( error ) {
+    // When "not found", it's either a cold start or the catalog was already stopped/terminated, ignore it.
+    // Otherwise, create a log and try the next operation:
+    // A. If the workflow is still running, the start() will fail and throw;
+    // B. If the workflow is no running, the start() will succeed, and the error was transient;
+    if ( !( error instanceof WorkflowNotFoundError ) ) {
+      log.warn( 'Error interacting with previous catalog workflow', { error } );
+    }
+  }
+
+  log.info( 'Starting catalog workflow...' );
+  await client.workflow.start( WORKFLOW_CATALOG, {
+    taskQueue,
+    workflowId: catalogId, // use the name of the task queue as the catalog name, ensuring uniqueness
+    workflowIdConflictPolicy: WorkflowIdConflictPolicy.FAIL,
+    args: [ catalog ]
+  } );
+};

package/src/worker/start_catalog.spec.js

@@ -0,0 +1,118 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { WorkflowNotFoundError } from '@temporalio/client';
+
+const mockLog = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+vi.mock( '#logger', () => ( { createChildLogger: () => mockLog } ) );
+
+vi.mock( '#consts', () => ( { WORKFLOW_CATALOG: 'catalog' } ) );
+
+const catalogId = 'test-catalog';
+const taskQueue = 'test-queue';
+vi.mock( './configs.js', () => ( { catalogId, taskQueue } ) );
+
+const describeMock = vi.fn();
+const executeUpdateMock = vi.fn();
+const workflowStartMock = vi.fn().mockResolvedValue( undefined );
+vi.mock( '@temporalio/client', async importOriginal => {
+  const actual = await importOriginal();
+  return {
+    ...actual,
+    Client: vi.fn().mockImplementation( function () {
+      return {
+        workflow: {
+          start: workflowStartMock,
+          getHandle: () => ( { describe: describeMock, executeUpdate: executeUpdateMock } )
+        }
+      };
+    } )
+  };
+} );
+
+vi.mock( '@temporalio/common', () => ( { WorkflowIdConflictPolicy: { FAIL: 'FAIL' } } ) );
+
+describe( 'worker/start_catalog', () => {
+  const mockConnection = {};
+  const namespace = 'default';
+  const catalog = { workflows: [], activities: {} };
+
+  beforeEach( () => {
+    vi.clearAllMocks();
+    workflowStartMock.mockResolvedValue( undefined );
+  } );
+
+  it( 'when previous catalog still running: completes it then starts catalog workflow', async () => {
+    describeMock.mockResolvedValue( { closeTime: undefined } );
+    executeUpdateMock.mockResolvedValue( undefined );
+
+    const { startCatalog } = await import( './start_catalog.js' );
+    await startCatalog( { connection: mockConnection, namespace, catalog } );
+
+    expect( describeMock ).toHaveBeenCalled();
+    expect( mockLog.info ).toHaveBeenCalledWith( 'Completing previous catalog workflow...' );
+    expect( executeUpdateMock ).toHaveBeenCalledWith( 'complete', { args: [] } );
+    expect( mockLog.info ).toHaveBeenCalledWith( 'Starting catalog workflow...' );
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', {
+      taskQueue,
+      workflowId: catalogId,
+      workflowIdConflictPolicy: 'FAIL',
+      args: [ catalog ]
+    } );
+  } );
+
+  it( 'when no previous catalog: ignores and starts catalog workflow', async () => {
+    describeMock.mockRejectedValue( new WorkflowNotFoundError( 'not found' ) );
+
+    const { startCatalog } = await import( './start_catalog.js' );
+    await startCatalog( { connection: mockConnection, namespace, catalog } );
+
+    expect( describeMock ).toHaveBeenCalled();
+    expect( mockLog.warn ).not.toHaveBeenCalled();
+    expect( mockLog.info ).toHaveBeenCalledWith( 'Starting catalog workflow...' );
+    expect( executeUpdateMock ).not.toHaveBeenCalled();
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', {
+      taskQueue,
+      workflowId: catalogId,
+      workflowIdConflictPolicy: 'FAIL',
+      args: [ catalog ]
+    } );
+  } );
+
+  it( 'when previous catalog already closed: skips complete and starts catalog workflow', async () => {
+    describeMock.mockResolvedValue( { closeTime: '2024-01-01T00:00:00Z' } );
+
+    const { startCatalog } = await import( './start_catalog.js' );
+    await startCatalog( { connection: mockConnection, namespace, catalog } );
+
+    expect( describeMock ).toHaveBeenCalled();
+    expect( mockLog.info ).not.toHaveBeenCalledWith( 'Completing previous catalog workflow...' );
+    expect( executeUpdateMock ).not.toHaveBeenCalled();
+    expect( mockLog.info ).toHaveBeenCalledWith( 'Starting catalog workflow...' );
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', {
+      taskQueue,
+      workflowId: catalogId,
+      workflowIdConflictPolicy: 'FAIL',
+      args: [ catalog ]
+    } );
+  } );
+
+  it( 'when describe or complete fails with other error: logs warn and still starts catalog workflow', async () => {
+    describeMock.mockResolvedValue( { closeTime: undefined } );
+    executeUpdateMock.mockRejectedValue( new Error( 'Connection refused' ) );
+
+    const { startCatalog } = await import( './start_catalog.js' );
+    await startCatalog( { connection: mockConnection, namespace, catalog } );
+
+    expect( describeMock ).toHaveBeenCalled();
+    expect( executeUpdateMock ).toHaveBeenCalledWith( 'complete', { args: [] } );
+    expect( mockLog.warn ).toHaveBeenCalledWith( 'Error interacting with previous catalog workflow', {
+      error: expect.any( Error )
+    } );
+    expect( mockLog.info ).toHaveBeenCalledWith( 'Starting catalog workflow...' );
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', {
+      taskQueue,
+      workflowId: catalogId,
+      workflowIdConflictPolicy: 'FAIL',
+      args: [ catalog ]
+    } );
+  } );
+} );

package/src/worker/webpack_loaders/consts.js

@@ -0,0 +1,9 @@
+export const NodeType = {
+  CONST: 'const'
+};
+
+export const ComponentFile = {
+  EVALUATORS: 'evaluators',
+  STEPS: 'steps',
+  WORKFLOW: 'workflow'
+};