@zintrust/workers 0.1.28 → 0.1.30

package/src/WorkerShutdown.ts

@@ -5,7 +5,7 @@
  * Coordinates orderly shutdown of all worker modules and the WorkerFactory.
  */
 
-import { Logger } from '@zintrust/core';
+import { Cloudflare, Logger } from '@zintrust/core';
 import { WorkerFactory } from './WorkerFactory';
 
 // ============================================================================
@@ -36,6 +36,12 @@ interface IShutdownState {
   reason: string | null;
 }
 
+type DurableShutdownState = {
+  shuttingDown: boolean;
+  startedAt?: string;
+  reason?: string;
+};
+
 // ============================================================================
 // Implementation
 // ============================================================================
@@ -47,8 +53,67 @@ const state: IShutdownState = {
   reason: null,
 };
 
+const getDurableShutdownStub = (): {
+  fetch: (input: string | URL, init?: RequestInit) => Promise<Response>;
+} | null => {
+  const env = Cloudflare.getWorkersEnv();
+  if (env === null) return null;
+
+  const namespace = env['WORKER_SHUTDOWN'] as
+    | {
+        idFromName?: (name: string) => unknown;
+        get?: (id: unknown) => {
+          fetch: (input: string | URL, init?: RequestInit) => Promise<Response>;
+        };
+      }
+    | undefined;
+
+  if (
+    !namespace ||
+    typeof namespace.idFromName !== 'function' ||
+    typeof namespace.get !== 'function'
+  ) {
+    return null;
+  }
+
+  const id = namespace.idFromName('zintrust-shutdown');
+  return namespace.get(id) ?? null;
+};
+
+const requestDurableShutdown = async (reason = 'manual'): Promise<boolean> => {
+  const stub = getDurableShutdownStub();
+  if (!stub) {
+    Logger.warn('Worker shutdown Durable Object binding not configured');
+    return false;
+  }
+
+  const res = await stub.fetch('https://worker-shutdown/shutdown', {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify({ reason }),
+  });
+
+  return res.ok;
+};
+
+const getDurableShutdownState = async (): Promise<DurableShutdownState | null> => {
+  const stub = getDurableShutdownStub();
+  if (!stub) return null;
+
+  const res = await stub.fetch('https://worker-shutdown/status');
+  if (!res.ok) return null;
+  return (await res.json()) as DurableShutdownState;
+};
+
 let shutdownHandlersRegistered = false;
 
+const signalHandlers: {
+  sigterm?: () => Promise<void>;
+  sighup?: () => Promise<void>;
+  uncaughtException?: (error: Error) => Promise<void>;
+  unhandledRejection?: (reason: unknown) => void;
+} = {};
+
 /**
  * Perform graceful shutdown of all worker modules
  */
@@ -115,14 +180,15 @@ function registerShutdownHandlers(): void {
   Logger.debug('Registering worker management system shutdown handlers');
 
   // SIGTERM - graceful shutdown (Docker, systemd, etc.)
-  process.on('SIGTERM', async () => {
+  signalHandlers.sigterm = async (): Promise<void> => {
     Logger.info('📨 Received SIGTERM signal');
     try {
       await shutdown({ signal: 'SIGTERM', timeout: 30000, forceExit: true });
     } catch (error) {
       Logger.error('Error during SIGTERM shutdown', error);
     }
-  });
+  };
+  process.on('SIGTERM', signalHandlers.sigterm);
 
   // SIGINT - user interrupt (Ctrl+C) - REMOVED: handled by bootstrap.ts to prevent race condition
   // process.on('SIGINT', async () => {
@@ -135,17 +201,18 @@ function registerShutdownHandlers(): void {
   // });
 
   // SIGHUP - terminal closed
-  process.on('SIGHUP', async () => {
+  signalHandlers.sighup = async (): Promise<void> => {
     Logger.info('📨 Received SIGHUP signal');
     try {
       await shutdown({ signal: 'SIGHUP', timeout: 30000, forceExit: true });
     } catch (error) {
       Logger.error('Error during SIGHUP shutdown', error);
     }
-  });
+  };
+  process.on('SIGHUP', signalHandlers.sighup);
 
   // Handle uncaught errors during shutdown
-  process.on('uncaughtException', async (error: Error) => {
+  signalHandlers.uncaughtException = async (error: Error): Promise<void> => {
     Logger.error('💥 Uncaught exception during worker operations', error);
     try {
       await shutdown({ signal: 'uncaughtException', timeout: 10000, forceExit: true });
@@ -153,19 +220,43 @@ function registerShutdownHandlers(): void {
       // Ignore errors during emergency shutdown
     }
     process.exit(1);
-  });
+  };
+  process.on('uncaughtException', signalHandlers.uncaughtException);
 
-  process.on('unhandledRejection', (reason: unknown) => {
+  signalHandlers.unhandledRejection = (reason: unknown): void => {
     // Only log the error - don't shut down the entire application
     Logger.error('💥 Unhandled promise rejection detected', reason);
     Logger.warn('⚠️  This error has been logged but will not shut down the server');
     Logger.warn('⚠️  Check the error context and fix the underlying issue');
-  });
+  };
+  process.on('unhandledRejection', signalHandlers.unhandledRejection);
 
   shutdownHandlersRegistered = true;
   Logger.debug('Worker management system shutdown handlers registered');
 }
 
+/**
+ * Unregister process signal handlers (for hot reload/testing)
+ */
+function unregisterShutdownHandlers(): void {
+  if (!shutdownHandlersRegistered) return;
+
+  if (signalHandlers.sigterm) process.off('SIGTERM', signalHandlers.sigterm);
+  if (signalHandlers.sighup) process.off('SIGHUP', signalHandlers.sighup);
+  if (signalHandlers.uncaughtException)
+    process.off('uncaughtException', signalHandlers.uncaughtException);
+  if (signalHandlers.unhandledRejection)
+    process.off('unhandledRejection', signalHandlers.unhandledRejection);
+
+  signalHandlers.sigterm = undefined;
+  signalHandlers.sighup = undefined;
+  signalHandlers.uncaughtException = undefined;
+  signalHandlers.unhandledRejection = undefined;
+
+  shutdownHandlersRegistered = false;
+  Logger.debug('Worker management system shutdown handlers unregistered');
+}
+
 /**
  * Check if system is currently shutting down
  */
@@ -195,6 +286,11 @@ export const WorkerShutdown = Object.freeze({
    */
   registerShutdownHandlers,
 
+  /**
+   * Unregister process signal handlers (for hot reload/testing)
+   */
+  unregisterShutdownHandlers,
+
   /**
    * Check if system is currently shutting down
    */
@@ -204,4 +300,14 @@ export const WorkerShutdown = Object.freeze({
    * Get current shutdown state
    */
   getShutdownState,
+
+  /**
+   * Request shutdown via Durable Object (Workers)
+   */
+  requestDurableShutdown,
+
+  /**
+   * Read shutdown state from Durable Object (Workers)
+   */
+  getDurableShutdownState,
 });

package/src/WorkerShutdownDurableObject.ts

@@ -0,0 +1,64 @@
+import { Logger } from '@zintrust/core';
+
+type DurableObjectState = {
+  storage: {
+    get: (key: string) => Promise<unknown>;
+    put: (key: string, value: unknown) => Promise<void>;
+  };
+};
+
+type ShutdownState = {
+  shuttingDown: boolean;
+  startedAt?: string;
+  reason?: string;
+};
+
+const loadState = async (state: DurableObjectState): Promise<ShutdownState> => {
+  const stored = (await state.storage.get('shutdown')) as ShutdownState | undefined;
+  return stored ?? { shuttingDown: false };
+};
+
+const saveState = async (state: DurableObjectState, value: ShutdownState): Promise<void> => {
+  await state.storage.put('shutdown', value);
+};
+
+// eslint-disable-next-line no-restricted-syntax
+export class ZinTrustWorkerShutdownDurableObject {
+  private readonly state: DurableObjectState;
+
+  constructor(state: DurableObjectState) {
+    this.state = state;
+  }
+
+  async fetch(request: Request): Promise<Response> {
+    const url = new URL(request.url);
+    const path = url.pathname;
+
+    if (request.method === 'GET' && path === '/status') {
+      const current = await loadState(this.state);
+      return new Response(JSON.stringify(current), {
+        status: 200,
+        headers: { 'content-type': 'application/json' },
+      });
+    }
+
+    if (request.method === 'POST' && path === '/shutdown') {
+      const payload = (await request.json().catch(() => ({}))) as { reason?: string };
+      const next: ShutdownState = {
+        shuttingDown: true,
+        startedAt: new Date().toISOString(),
+        reason: payload.reason ?? 'manual',
+      };
+
+      await saveState(this.state, next);
+      Logger.info('Worker shutdown requested via Durable Object', next);
+
+      return new Response(JSON.stringify({ ok: true }), {
+        status: 202,
+        headers: { 'content-type': 'application/json' },
+      });
+    }
+
+    return new Response('Not Found', { status: 404 });
+  }
+}

package/src/createQueueWorker.ts

@@ -1,4 +1,4 @@
-import type { QueueMessage } from '@zintrust/core';
+import type { BullMQPayload, QueueMessage } from '@zintrust/core';
 import { Logger, Queue } from '@zintrust/core';
 
 type QueueWorker = {
@@ -8,11 +8,13 @@ type QueueWorker = {
     queueName?: string;
     driverName?: string;
     maxItems?: number;
+    maxDurationMs?: number;
   }) => Promise<number>;
   startWorker: (opts?: {
     queueName?: string;
     driverName?: string;
     signal?: AbortSignal;
+    maxDurationMs?: number;
   }) => Promise<number>;
 };
 
@@ -54,7 +56,7 @@ const createProcessOne = <TPayload>(
         dueAt: new Date(timestamp).toISOString(),
       });
       // Re-queue original payload
-      await Queue.enqueue(queueName, message.payload, driverName);
+      await Queue.enqueue(queueName, message.payload as BullMQPayload, driverName);
       await Queue.ack(queueName, message.id, driverName);
       return false;
     }
@@ -75,7 +77,7 @@ const createProcessOne = <TPayload>(
       });
 
       if (attempts < options.maxAttempts) {
-        await Queue.enqueue(queueName, message.payload, driverName);
+        await Queue.enqueue(queueName, message.payload as BullMQPayload, driverName);
         Logger.info(`${options.kindLabel} re-queued for retry`, {
           ...baseLogFields,
           attempts: attempts + 1,
@@ -83,7 +85,8 @@ const createProcessOne = <TPayload>(
       }
 
       await Queue.ack(queueName, message.id, driverName);
-      return false;
+      // We processed the message (even if it failed), so return true to continue processing
+      return true;
     }
   };
 };
@@ -109,29 +112,49 @@ const createProcessAll = (
 const createRunOnce = (
   defaultQueueName: string,
   processOne: (queueName?: string, driverName?: string) => Promise<boolean>
-): ((opts?: { queueName?: string; driverName?: string; maxItems?: number }) => Promise<number>) => {
+): ((opts?: {
+  queueName?: string;
+  driverName?: string;
+  maxItems?: number;
+  maxDurationMs?: number;
+  concurrency?: number;
+}) => Promise<number>) => {
   return async (opts = {}): Promise<number> => {
-    const { queueName = defaultQueueName, driverName, maxItems } = opts;
-    let processed = 0;
-
-    if (maxItems === undefined) {
+    const {
+      queueName = defaultQueueName,
+      driverName,
+      maxItems,
+      maxDurationMs = 30000,
+      concurrency = 1,
+    } = opts;
+    const startTime = Date.now();
+    let totalProcessed = 0;
+
+    // Helper for single worker loop
+    const runWorker = async (): Promise<number> => {
+      let workerProcessed = 0;
       while (true) {
+        if (maxDurationMs > 0 && Date.now() - startTime > maxDurationMs) {
+          break;
+        }
+        if (maxItems !== undefined && totalProcessed >= maxItems) {
+          break;
+        }
+
         // eslint-disable-next-line no-await-in-loop
         const didProcess = await processOne(queueName, driverName);
         if (!didProcess) break;
-        processed++;
+
+        workerProcessed++;
+        totalProcessed++; // Shared counter (approximation in parallel)
       }
-      return processed;
-    }
+      return workerProcessed;
+    };
 
-    for (let i = 0; i < maxItems; i++) {
-      // eslint-disable-next-line no-await-in-loop
-      const didProcess = await processOne(queueName, driverName);
-      if (!didProcess) break;
-      processed++;
-    }
+    // Run workers in parallel
+    await Promise.all(Array.from({ length: Math.max(1, concurrency) }).map(() => runWorker()));
 
-    return processed;
+    return totalProcessed;
   };
 };
 
@@ -143,22 +166,42 @@ const createStartWorker = (
   queueName?: string;
   driverName?: string;
   signal?: AbortSignal;
+  maxDurationMs?: number;
+  concurrency?: number;
 }) => Promise<number>) => {
   return async (opts = {}): Promise<number> => {
-    const { queueName = defaultQueueName, driverName, signal } = opts;
+    const {
+      queueName = defaultQueueName,
+      driverName,
+      signal,
+      maxDurationMs = 300000,
+      concurrency = 1,
+    } = opts;
+
+    Logger.info(`Starting ${kindLabel} worker (drain-until-empty)`, { queueName, concurrency });
+
+    const startTime = Date.now();
+    let totalProcessed = 0;
+
+    const runWorker = async (): Promise<void> => {
+      while (signal?.aborted !== true) {
+        if (maxDurationMs > 0 && Date.now() - startTime > maxDurationMs) {
+          Logger.warn(`${kindLabel} worker timeout reached`, { queueName, totalProcessed });
+          break;
+        }
 
-    Logger.info(`Starting ${kindLabel} worker (drain-until-empty)`, { queueName });
+        // eslint-disable-next-line no-await-in-loop
+        const didProcess = await processOne(queueName, driverName);
+        if (!didProcess) break;
+        totalProcessed++;
+      }
+    };
 
-    let processedCount = 0;
-    while (signal?.aborted !== true) {
-      // eslint-disable-next-line no-await-in-loop
-      const didProcess = await processOne(queueName, driverName);
-      if (!didProcess) break;
-      processedCount++;
-    }
+    // Run workers in parallel
+    await Promise.all(Array.from({ length: Math.max(1, concurrency) }).map(() => runWorker()));
 
-    Logger.info(`${kindLabel} worker finished (queue drained)`, { queueName, processedCount });
-    return processedCount;
+    Logger.info(`${kindLabel} worker finished (queue drained)`, { queueName, totalProcessed });
+    return totalProcessed;
   };
 };
 

package/src/dashboard/types.ts

@@ -1,5 +1,7 @@
 // Worker Configuration Types
 export interface WorkerConfiguration {
+  processorSpec?: string | null;
+  activeStatus?: boolean | null;
   [key: string]: string | number | boolean | null | undefined | object;
 }
 
@@ -49,6 +51,7 @@ export interface WorkerData {
   avgTime: number;
   memory: number;
   autoStart: boolean;
+  activeStatus?: boolean;
   details?: {
     configuration: WorkerConfiguration;
     health: WorkerHealth;
@@ -95,6 +98,7 @@ export type GetWorkersQuery = {
   driver?: WorkerDriver;
   search?: string;
   includeDetails?: boolean;
+  includeInactive?: boolean;
 };
 
 // UI Options Types
@@ -116,6 +120,7 @@ export type RawWorkerData = {
   processed?: number;
   version?: string;
   autoStart?: boolean;
+  activeStatus?: boolean;
   queueName?: string;
   details?: {
     configuration: WorkerConfiguration;