@outputai/core 0.5.3-next.69060d7.0 → 0.5.3-next.bdf47aa.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@outputai/core",
-  "version": "0.5.3-next.69060d7.0",
+  "version": "0.5.3-next.bdf47aa.0",
   "description": "The core module of the output framework",
   "type": "module",
   "exports": {
@@ -44,6 +44,7 @@
     "@temporalio/worker": "1.17.0",
     "@temporalio/workflow": "1.17.0",
     "decimal.js": "10.6.0",
+    "folder-hash": "4.1.3",
     "json-stream-stringify": "3.1.6",
     "redis": "5.12.1",
     "stacktrace-parser": "0.1.11",

package/src/worker/catalog_workflow/workflow.js

@@ -7,12 +7,15 @@ import { defineQuery, setHandler, condition, defineUpdate } from '@temporalio/wo
  *
  * @param {object} catalog - The catalog information
  */
-export default async function catalogWorkflow( catalog ) {
+export default async function catalogWorkflow( catalog, catalogHash ) {
   const state = { canEnd: false };
 
   // Returns the catalog
   setHandler( defineQuery( 'get' ), () => catalog );
 
+  // Returns the catalog hash
+  setHandler( defineQuery( 'get_hash' ), () => catalogHash );
+
   // Politely respond to a ping
   setHandler( defineQuery( 'ping' ), () => 'pong' );
 

package/src/worker/index.js

@@ -13,6 +13,7 @@ import { bootstrapFetchProxy } from './proxy.js';
 import { messageBus } from '#bus';
 import './log_hooks.js';
 import { BusEventType } from '#consts';
+import { hashSourceCode } from './loader_tools.js';
 
 const log = createChildLogger( 'Worker' );
 
@@ -54,6 +55,9 @@ const callerDir = process.argv[2];
   log.info( 'Creating workflows catalog...' );
   const catalog = createCatalog( { workflows, activities } );
 
+  log.info( 'Computing catalog source code hash...' );
+  const catalogHash = await hashSourceCode( callerDir );
+
   log.info( 'Connecting Temporal...' );
   const proxy = grpcProxy ? { type: 'http-connect', targetHost: grpcProxy } : undefined;
   if ( proxy ) {
@@ -81,7 +85,7 @@ const callerDir = process.argv[2];
   registerShutdown( { worker, log } );
 
   log.info( 'Running worker...' );
-  await Promise.all( [ worker.run(), startCatalog( { connection, namespace, catalog } ) ] );
+  await Promise.all( [ worker.run(), startCatalog( { connection, namespace, catalog, catalogHash } ) ] );
 
   log.info( 'Closing connection...' );
   await connection.close();

package/src/worker/index.spec.js

@@ -40,6 +40,9 @@ vi.mock( './loader.js', () => ( {
   createWorkflowsEntryPoint: createWorkflowsEntryPointMock
 } ) );
 
+const hashSourceCodeMock = vi.fn().mockResolvedValue( 'catalog-hash' );
+vi.mock( './loader_tools.js', () => ( { hashSourceCode: hashSourceCodeMock } ) );
+
 vi.mock( './sinks.js', () => ( { sinks: {} } ) );
 
 const createCatalogMock = vi.fn().mockReturnValue( { workflows: [], activities: {} } );
@@ -105,6 +108,7 @@ describe( 'worker/index', () => {
     expect( createWorkflowsEntryPointMock ).toHaveBeenCalledWith( [] );
     expect( initTracing ).toHaveBeenCalled();
     expect( createCatalogMock ).toHaveBeenCalledWith( { workflows: [], activities: {} } );
+    expect( hashSourceCodeMock ).toHaveBeenCalledWith( '/test/caller/dir' );
     expect( bootstrapFetchProxyMock ).toHaveBeenCalled();
     expect( NativeConnection.connect ).toHaveBeenCalledWith( {
       address: configValues.address,
@@ -128,7 +132,8 @@ describe( 'worker/index', () => {
     expect( startCatalogMock ).toHaveBeenCalledWith( {
       connection: mockConnection,
       namespace: configValues.namespace,
-      catalog: { workflows: [], activities: {} }
+      catalog: { workflows: [], activities: {} },
+      catalogHash: 'catalog-hash'
     } );
 
     runState.resolve();

package/src/worker/interceptors.js

@@ -6,5 +6,9 @@ const __dirname = dirname( fileURLToPath( import.meta.url ) );
 
 export const initInterceptors = ( { activities, workflows, connection } ) => ( {
   workflowModules: [ join( __dirname, './interceptors/workflow.js' ) ],
-  activityInbound: [ () => new ActivityExecutionInterceptor( { activities, workflows, connection } ) ]
+  activity: [
+    () => ( {
+      inbound: new ActivityExecutionInterceptor( { activities, workflows, connection } )
+    } )
+  ]
 } );

package/src/worker/loader_tools.js

@@ -2,6 +2,7 @@ import { dirname, resolve, sep } from 'path';
 import { pathToFileURL } from 'url';
 import { METADATA_ACCESS_SYMBOL } from '#consts';
 import { existsSync, lstatSync, readdirSync, readFileSync, realpathSync } from 'fs';
+import { hashElement } from 'folder-hash';
 
 /**
  * Returns the real path for symlink
@@ -302,3 +303,27 @@ export async function *importComponents( files ) {
     }
   }
 };
+
+/**
+ * Creates a hash of all source code in a given dir
+ *
+ * @param {string} rootDir
+ * @returns {string} Hash value
+ */
+export const hashSourceCode = async rootDir => {
+  try {
+    const { hash } = await hashElement( rootDir, {
+      folders: {
+        exclude: [ '.*', 'node_modules', 'test_coverage', 'vendor', 'test' ],
+        ignoreRootName: true
+      },
+      files: {
+        include: [ '*.js', '*.cjs', '*.mjs', '*.ts', '*.yaml', '*.yml', '*.json', '*.prompt' ],
+        ignoreRootName: true
+      }
+    } );
+    return hash;
+  } catch ( error ) {
+    throw new Error( `Error calculating hash from "${error}": ${error.message}`, { cause: error } );
+  }
+};

package/src/worker/start_catalog.js

@@ -1,36 +1,96 @@
 import { Client, WorkflowNotFoundError } from '@temporalio/client';
-import { WorkflowIdConflictPolicy } from '@temporalio/common';
+import { WorkflowExecutionAlreadyStartedError, WorkflowIdConflictPolicy } from '@temporalio/common';
 import { WORKFLOW_CATALOG } from '#consts';
 import { catalogId, taskQueue } from './configs.js';
 import { createChildLogger } from '#logger';
 
 const log = createChildLogger( 'Catalog' );
 
-export const startCatalog = async ( { connection, namespace, catalog } ) => {
-  const client = new Client( { connection, namespace } );
-  const catalogWorkflowHandle = client.workflow.getHandle( catalogId );
+// Note, functions don't log on "WorkflowNotFound" errors,
+// because they happen when the catalog is not running at all.
 
+/**
+ * Check if the currently running catalog has the same hash as the passed argument.
+ * @param {import('@temporalio/client').WorkflowHandle} handle
+ * @param {string} hash
+ * @returns {boolean}
+ */
+const checkCatalogIsTheSame = async ( handle, hash ) => {
   try {
-    const catalogWorkflowDescription = await catalogWorkflowHandle.describe();
-    if ( !catalogWorkflowDescription.closeTime ) {
-      log.info( 'Completing previous catalog workflow...' );
-      await catalogWorkflowHandle.executeUpdate( 'complete', { args: [] } );
+    log.info( 'Checking running catalog hash against worker hash....' );
+    const runningHash = await handle.query( 'get_hash' );
+    return runningHash === hash;
+  } catch ( error ) {
+    if ( !( error instanceof WorkflowNotFoundError ) ) {
+      log.warn( 'Error retrieving catalog hash', { error } );
     }
+    return false;
+  }
+};
+
+/**
+ * Check if the catalog workflow is running.
+ * @param {import('@temporalio/client').WorkflowHandle} handle
+ * @returns
+ */
+const checkCatalogRunning = async handle => {
+  try {
+    log.info( 'Checking if the catalog workflow is running....' );
+    const description = await handle.describe();
+    return !description.closeTime;
   } catch ( error ) {
-    // When "not found", it's either a cold start or the catalog was already stopped/terminated, ignore it.
-    // Otherwise, create a log and try the next operation:
-    // A. If the workflow is still running, the start() will fail and throw;
-    // B. If the workflow is no running, the start() will succeed, and the error was transient;
     if ( !( error instanceof WorkflowNotFoundError ) ) {
-      log.warn( 'Error interacting with previous catalog workflow', { error } );
+      log.warn( 'Error describing catalog workflow', { error } );
     }
+    return false;
   }
+};
 
-  log.info( 'Starting catalog workflow...' );
-  await client.workflow.start( WORKFLOW_CATALOG, {
+/**
+ * Complete previous running catalog workflow.
+ * @param {import('@temporalio/client').WorkflowHandle} handle
+ */
+const completePreviousCatalog = async handle => {
+  try {
+    log.info( 'Completing previous catalog workflow...' );
+    await handle.executeUpdate( 'complete', { args: [] } );
+  } catch ( error ) {
+    if ( !( error instanceof WorkflowNotFoundError ) ) {
+      log.warn( 'Error completing previous catalog workflow', { error } );
+    }
+  }
+};
+
+export const startCatalog = async ( { connection, namespace, catalog, catalogHash } ) => {
+  const client = new Client( { connection, namespace } );
+  const handle = client.workflow.getHandle( catalogId );
+
+  if ( await checkCatalogRunning( handle ) ) {
+    if ( await checkCatalogIsTheSame( handle, catalogHash ) ) {
+      log.info( 'Current catalog workflow hash matches worker, restart skipped' );
+      return;
+    }
+    await completePreviousCatalog( handle );
+  }
+
+  const startArguments = {
     taskQueue,
-    workflowId: catalogId, // use the name of the task queue as the catalog name, ensuring uniqueness
+    workflowId: catalogId,
     workflowIdConflictPolicy: WorkflowIdConflictPolicy.FAIL,
-    args: [ catalog ]
-  } );
+    args: [ catalog, catalogHash ]
+  };
+
+  log.info( 'Starting catalog workflow...' );
+  try {
+    await client.workflow.start( WORKFLOW_CATALOG, startArguments );
+  } catch ( error ) {
+    // if the error was caused by the catalog existing and its hash is the same as the one from the worker, just ignore the error
+    if ( error instanceof WorkflowExecutionAlreadyStartedError && await checkCatalogIsTheSame( handle, catalogHash ) ) {
+      log.info( 'Ignoring start error: it failed because execution already started but catalog hash matches worker' );
+    } else {
+      throw error;
+    }
+  }
+
+  log.info( 'Startup completed' );
 };

package/src/worker/start_catalog.spec.js

@@ -1,5 +1,6 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest';
 import { WorkflowNotFoundError } from '@temporalio/client';
+import { WorkflowExecutionAlreadyStartedError } from '@temporalio/common';
 
 const mockLog = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
 vi.mock( '#logger', () => ( { createChildLogger: () => mockLog } ) );
@@ -11,6 +12,7 @@ const taskQueue = 'test-queue';
 vi.mock( './configs.js', () => ( { catalogId, taskQueue } ) );
 
 const describeMock = vi.fn();
+const queryMock = vi.fn();
 const executeUpdateMock = vi.fn();
 const workflowStartMock = vi.fn().mockResolvedValue( undefined );
 vi.mock( '@temporalio/client', async importOriginal => {
@@ -21,33 +23,40 @@ vi.mock( '@temporalio/client', async importOriginal => {
       return {
         workflow: {
           start: workflowStartMock,
-          getHandle: () => ( { describe: describeMock, executeUpdate: executeUpdateMock } )
+          getHandle: () => ( { describe: describeMock, query: queryMock, executeUpdate: executeUpdateMock } )
         }
       };
     } )
   };
 } );
 
-vi.mock( '@temporalio/common', () => ( { WorkflowIdConflictPolicy: { FAIL: 'FAIL' } } ) );
-
 describe( 'worker/start_catalog', () => {
   const mockConnection = {};
   const namespace = 'default';
   const catalog = { workflows: [], activities: {} };
+  const catalogHash = 'catalog-hash';
 
   beforeEach( () => {
-    vi.clearAllMocks();
+    mockLog.info.mockClear();
+    mockLog.warn.mockClear();
+    mockLog.error.mockClear();
+    describeMock.mockReset();
+    queryMock.mockReset();
+    executeUpdateMock.mockReset();
+    workflowStartMock.mockReset();
     workflowStartMock.mockResolvedValue( undefined );
   } );
 
-  it( 'when previous catalog still running: completes it then starts catalog workflow', async () => {
+  it( 'when previous catalog still running with different hash: completes it then starts catalog workflow', async () => {
     describeMock.mockResolvedValue( { closeTime: undefined } );
+    queryMock.mockResolvedValue( 'old-catalog-hash' );
     executeUpdateMock.mockResolvedValue( undefined );
 
     const { startCatalog } = await import( './start_catalog.js' );
-    await startCatalog( { connection: mockConnection, namespace, catalog } );
+    await startCatalog( { connection: mockConnection, namespace, catalog, catalogHash } );
 
     expect( describeMock ).toHaveBeenCalled();
+    expect( queryMock ).toHaveBeenCalledWith( 'get_hash' );
     expect( mockLog.info ).toHaveBeenCalledWith( 'Completing previous catalog workflow...' );
     expect( executeUpdateMock ).toHaveBeenCalledWith( 'complete', { args: [] } );
     expect( mockLog.info ).toHaveBeenCalledWith( 'Starting catalog workflow...' );
@@ -55,17 +64,32 @@ describe( 'worker/start_catalog', () => {
       taskQueue,
       workflowId: catalogId,
       workflowIdConflictPolicy: 'FAIL',
-      args: [ catalog ]
+      args: [ catalog, catalogHash ]
     } );
   } );
 
+  it( 'when previous catalog still running with same hash: keeps existing catalog workflow', async () => {
+    describeMock.mockResolvedValue( { closeTime: undefined } );
+    queryMock.mockResolvedValue( catalogHash );
+
+    const { startCatalog } = await import( './start_catalog.js' );
+    await startCatalog( { connection: mockConnection, namespace, catalog, catalogHash } );
+
+    expect( describeMock ).toHaveBeenCalled();
+    expect( queryMock ).toHaveBeenCalledWith( 'get_hash' );
+    expect( mockLog.info ).toHaveBeenCalledWith( 'Current catalog workflow hash matches worker, restart skipped' );
+    expect( executeUpdateMock ).not.toHaveBeenCalled();
+    expect( workflowStartMock ).not.toHaveBeenCalled();
+  } );
+
   it( 'when no previous catalog: ignores and starts catalog workflow', async () => {
     describeMock.mockRejectedValue( new WorkflowNotFoundError( 'not found' ) );
 
     const { startCatalog } = await import( './start_catalog.js' );
-    await startCatalog( { connection: mockConnection, namespace, catalog } );
+    await startCatalog( { connection: mockConnection, namespace, catalog, catalogHash } );
 
     expect( describeMock ).toHaveBeenCalled();
+    expect( queryMock ).not.toHaveBeenCalled();
     expect( mockLog.warn ).not.toHaveBeenCalled();
     expect( mockLog.info ).toHaveBeenCalledWith( 'Starting catalog workflow...' );
     expect( executeUpdateMock ).not.toHaveBeenCalled();
@@ -73,7 +97,7 @@ describe( 'worker/start_catalog', () => {
       taskQueue,
       workflowId: catalogId,
       workflowIdConflictPolicy: 'FAIL',
-      args: [ catalog ]
+      args: [ catalog, catalogHash ]
     } );
   } );
 
@@ -81,9 +105,10 @@ describe( 'worker/start_catalog', () => {
     describeMock.mockResolvedValue( { closeTime: '2024-01-01T00:00:00Z' } );
 
     const { startCatalog } = await import( './start_catalog.js' );
-    await startCatalog( { connection: mockConnection, namespace, catalog } );
+    await startCatalog( { connection: mockConnection, namespace, catalog, catalogHash } );
 
     expect( describeMock ).toHaveBeenCalled();
+    expect( queryMock ).not.toHaveBeenCalled();
     expect( mockLog.info ).not.toHaveBeenCalledWith( 'Completing previous catalog workflow...' );
     expect( executeUpdateMock ).not.toHaveBeenCalled();
     expect( mockLog.info ).toHaveBeenCalledWith( 'Starting catalog workflow...' );
@@ -91,20 +116,22 @@ describe( 'worker/start_catalog', () => {
       taskQueue,
       workflowId: catalogId,
       workflowIdConflictPolicy: 'FAIL',
-      args: [ catalog ]
+      args: [ catalog, catalogHash ]
     } );
   } );
 
   it( 'when describe or complete fails with other error: logs warn and still starts catalog workflow', async () => {
     describeMock.mockResolvedValue( { closeTime: undefined } );
+    queryMock.mockResolvedValue( 'old-catalog-hash' );
     executeUpdateMock.mockRejectedValue( new Error( 'Connection refused' ) );
 
     const { startCatalog } = await import( './start_catalog.js' );
-    await startCatalog( { connection: mockConnection, namespace, catalog } );
+    await startCatalog( { connection: mockConnection, namespace, catalog, catalogHash } );
 
     expect( describeMock ).toHaveBeenCalled();
+    expect( queryMock ).toHaveBeenCalledWith( 'get_hash' );
     expect( executeUpdateMock ).toHaveBeenCalledWith( 'complete', { args: [] } );
-    expect( mockLog.warn ).toHaveBeenCalledWith( 'Error interacting with previous catalog workflow', {
+    expect( mockLog.warn ).toHaveBeenCalledWith( 'Error completing previous catalog workflow', {
       error: expect.any( Error )
     } );
     expect( mockLog.info ).toHaveBeenCalledWith( 'Starting catalog workflow...' );
@@ -112,7 +139,41 @@ describe( 'worker/start_catalog', () => {
       taskQueue,
       workflowId: catalogId,
       workflowIdConflictPolicy: 'FAIL',
-      args: [ catalog ]
+      args: [ catalog, catalogHash ]
+    } );
+  } );
+
+  it( 'when another worker starts matching catalog concurrently: ignores already-started error', async () => {
+    const alreadyStartedError = new WorkflowExecutionAlreadyStartedError( 'already started', catalogId, 'catalog' );
+    describeMock.mockRejectedValue( new WorkflowNotFoundError( 'not found' ) );
+    workflowStartMock.mockRejectedValue( alreadyStartedError );
+    queryMock.mockResolvedValue( catalogHash );
+
+    const { startCatalog } = await import( './start_catalog.js' );
+    await startCatalog( { connection: mockConnection, namespace, catalog, catalogHash } );
+
+    expect( workflowStartMock ).toHaveBeenCalledWith( 'catalog', {
+      taskQueue,
+      workflowId: catalogId,
+      workflowIdConflictPolicy: 'FAIL',
+      args: [ catalog, catalogHash ]
     } );
+    expect( queryMock ).toHaveBeenCalledWith( 'get_hash' );
+    expect( mockLog.info ).toHaveBeenCalledWith(
+      'Ignoring start error: it failed because execution already started but catalog hash matches worker'
+    );
+  } );
+
+  it( 'when another worker starts stale catalog concurrently: rethrows already-started error', async () => {
+    const alreadyStartedError = new WorkflowExecutionAlreadyStartedError( 'already started', catalogId, 'catalog' );
+    describeMock.mockRejectedValue( new WorkflowNotFoundError( 'not found' ) );
+    workflowStartMock.mockRejectedValue( alreadyStartedError );
+    queryMock.mockResolvedValue( 'old-catalog-hash' );
+
+    const { startCatalog } = await import( './start_catalog.js' );
+    await expect( startCatalog( { connection: mockConnection, namespace, catalog, catalogHash } ) )
+      .rejects.toBe( alreadyStartedError );
+
+    expect( queryMock ).toHaveBeenCalledWith( 'get_hash' );
   } );
 } );