@zintrust/workers 0.1.27

package/src/WorkerRegistry.ts

@@ -0,0 +1,443 @@
+/**
+ * Worker Registry
+ * Central registry for all background workers with lifecycle management
+ * Sealed namespace for immutability
+ */
+
+import { ErrorFactory, Logger, type WorkerConfig, type WorkerStatus } from '@zintrust/core';
+
+export type WorkerMetadata = {
+  name: string;
+  status: WorkerStatus;
+  version: string;
+  region: string;
+  queueName: string;
+  concurrency: number;
+  startedAt: Date | null;
+  stoppedAt: Date | null;
+  lastProcessedAt: Date | null;
+  restartCount: number;
+  processedCount: number;
+  errorCount: number;
+  lockKey: string | null;
+  priority: number;
+  memoryUsage: number;
+  cpuUsage: number;
+  circuitState: 'closed' | 'open' | 'half-open';
+  queues: ReadonlyArray<string>;
+  plugins: ReadonlyArray<string>;
+  datacenter: string;
+  canaryPercentage: number;
+  config: Partial<WorkerConfig>;
+};
+
+export type WorkerInstance = {
+  metadata: WorkerMetadata;
+  instance: unknown; // The actual worker instance (BullMQ Worker, etc.)
+  start: () => void;
+  stop: () => Promise<void>;
+  drain: () => Promise<void>;
+  sleep: () => Promise<void>;
+  wakeup: () => void;
+  getStatus: () => WorkerStatus;
+  getHealth: () => 'green' | 'yellow' | 'red';
+};
+
+export type RegisterWorkerOptions = {
+  name: string;
+  config: Partial<WorkerConfig>;
+  version?: string;
+  region?: string;
+  queues?: ReadonlyArray<string>;
+  factory: () => Promise<WorkerInstance>;
+};
+
+export type WorkerRegistrySnapshot = {
+  timestamp: Date;
+  totalWorkers: number;
+  runningWorkers: number;
+  stoppedWorkers: number;
+  sleepingWorkers: number;
+  unhealthyWorkers: number;
+  workers: ReadonlyArray<{
+    name: string;
+    status: WorkerStatus;
+    health: 'green' | 'yellow' | 'red';
+    uptime: number | null;
+    processedCount: number;
+    errorCount: number;
+  }>;
+};
+
+type Rego = { workers: string[]; count: number };
+
+// Internal storage
+const workers = new Map<string, WorkerInstance>();
+const registrations = new Map<string, RegisterWorkerOptions>();
+
+/**
+ * Helper: Calculate uptime in seconds
+ */
+const calculateUptime = (startedAt: Date | null): number | null => {
+  if (!startedAt) return null;
+  return Math.floor((Date.now() - startedAt.getTime()) / 1000);
+};
+
+/**
+ * Helper: Validate worker name
+ */
+const validateWorkerName = (name: string): void => {
+  if (!name || typeof name !== 'string') {
+    throw ErrorFactory.createWorkerError('Worker name must be a non-empty string');
+  }
+  if (!/^[a-z0-9-]+$/.test(name)) {
+    throw ErrorFactory.createWorkerError(
+      'Worker name must contain only lowercase letters, numbers, and hyphens'
+    );
+  }
+};
+
+/**
+ * Worker Registry - Sealed namespace
+ */
+export const WorkerRegistry = Object.freeze({
+  /**
+   * Register a worker with the registry
+   */
+  register(options: RegisterWorkerOptions): void {
+    validateWorkerName(options.name);
+
+    if (registrations.has(options.name)) {
+      Logger.warn(`Worker "${options.name}" is already registered. Skipping.`);
+      return;
+    }
+
+    registrations.set(options.name, options);
+    Logger.info(`Worker "${options.name}" registered successfully`);
+  },
+
+  /**
+   * Start a worker
+   */
+  async start(name: string, version?: string): Promise<void> {
+    validateWorkerName(name);
+
+    const registration = registrations.get(name);
+    if (!registration) {
+      throw ErrorFactory.createWorkerError(`Worker "${name}" is not registered`);
+    }
+
+    if (workers.has(name)) {
+      const existing = workers.get(name);
+      if (existing?.metadata.status === 'running') {
+        Logger.warn(`Worker "${name}" is already running`);
+        return;
+      }
+    }
+
+    const versionSuffix = version === undefined ? '' : ` version ${version}`;
+    Logger.info(`Starting worker "${name}"${versionSuffix}...`);
+
+    try {
+      const instance = await registration.factory();
+      instance.metadata.status = 'starting';
+      instance.metadata.version = version ?? '1.0.0';
+
+      workers.set(name, instance);
+
+      instance.start();
+
+      instance.metadata.status = 'running';
+      instance.metadata.startedAt = new Date();
+      instance.metadata.stoppedAt = null;
+
+      Logger.info(`Worker "${name}" started successfully`);
+    } catch (error) {
+      Logger.error(`Failed to start worker "${name}"`, error);
+      throw error;
+    }
+  },
+
+  /**
+   * Stop a worker
+   */
+  async stop(name: string): Promise<void> {
+    validateWorkerName(name);
+
+    const instance = workers.get(name);
+    if (!instance) {
+      Logger.warn(`Worker "${name}" is not running`);
+      return;
+    }
+
+    if (instance.metadata.status === 'stopped') {
+      Logger.warn(`Worker "${name}" is already stopped`);
+      return;
+    }
+
+    Logger.info(`Stopping worker "${name}"...`);
+
+    try {
+      instance.metadata.status = 'stopping';
+      await instance.stop();
+      instance.metadata.status = 'stopped';
+      instance.metadata.stoppedAt = new Date();
+
+      Logger.info(`Worker "${name}" stopped successfully`);
+    } catch (error) {
+      Logger.error(`Failed to stop worker "${name}"`, error);
+      throw error;
+    }
+  },
+
+  /**
+   * Restart a worker (stop + start)
+   */
+  async restart(name: string): Promise<void> {
+    validateWorkerName(name);
+
+    const instance = workers.get(name);
+    if (instance) {
+      await WorkerRegistry.stop(name);
+      instance.metadata.restartCount += 1;
+    }
+
+    await WorkerRegistry.start(name);
+    Logger.info(`Worker "${name}" restarted successfully`);
+  },
+
+  /**
+   * Sleep a worker (pause processing but keep lock)
+   */
+  async sleep(name: string): Promise<void> {
+    validateWorkerName(name);
+
+    const instance = workers.get(name);
+    if (!instance) {
+      throw ErrorFactory.createWorkerError(`Worker "${name}" is not running`);
+    }
+
+    if (instance.metadata.status === 'sleeping') {
+      Logger.warn(`Worker "${name}" is already sleeping`);
+      return;
+    }
+
+    Logger.info(`Putting worker "${name}" to sleep...`);
+
+    try {
+      await instance.sleep();
+      instance.metadata.status = 'sleeping';
+      Logger.info(`Worker "${name}" is now sleeping`);
+    } catch (error) {
+      Logger.error(`Failed to sleep worker "${name}"`, error);
+      throw error;
+    }
+  },
+
+  /**
+   * Wakeup a worker (resume from sleep)
+   */
+  async wakeup(name: string): Promise<void> {
+    validateWorkerName(name);
+
+    const instance = workers.get(name);
+    if (!instance) {
+      throw ErrorFactory.createWorkerError(`Worker "${name}" is not found`);
+    }
+
+    if (instance.metadata.status !== 'sleeping') {
+      Logger.warn(`Worker "${name}" is not sleeping (status: ${instance.metadata.status})`);
+      return;
+    }
+
+    Logger.info(`Waking up worker "${name}"...`);
+
+    try {
+      instance.wakeup();
+      instance.metadata.status = 'running';
+      Logger.info(`Worker "${name}" is now awake and running`);
+    } catch (error) {
+      Logger.error(`Failed to wake up worker "${name}"`, error);
+      throw error;
+    }
+  },
+
+  /**
+   * Get worker status
+   */
+  status(name: string): WorkerMetadata | null {
+    validateWorkerName(name);
+
+    const instance = workers.get(name);
+    if (!instance) {
+      return null;
+    }
+
+    return { ...instance.metadata };
+  },
+
+  /**
+   * List all registered workers
+   */
+  list(): ReadonlyArray<string> {
+    return Array.from(registrations.keys());
+  },
+
+  /**
+   * List all running workers
+   */
+  listRunning(): ReadonlyArray<string> {
+    const running: string[] = [];
+    for (const [name, instance] of workers.entries()) {
+      if (instance.metadata.status === 'running') {
+        running.push(name);
+      }
+    }
+    return running;
+  },
+
+  /**
+   * Stop all running workers
+   */
+  async stopAll(): Promise<void> {
+    Logger.info('Stopping all running workers...');
+
+    const running = WorkerRegistry.listRunning();
+    const tasks = running.map(async (name) => WorkerRegistry.stop(name));
+
+    try {
+      await Promise.all(tasks);
+      Logger.info(`Stopped ${running.length} workers successfully`);
+    } catch (error) {
+      Logger.error('Failed to stop some workers', error);
+      throw error;
+    }
+  },
+
+  /**
+   * Get worker metrics
+   */
+  getMetrics(
+    name: string
+  ): Pick<WorkerMetadata, 'processedCount' | 'errorCount' | 'memoryUsage' | 'cpuUsage'> | null {
+    validateWorkerName(name);
+
+    const instance = workers.get(name);
+    if (!instance) {
+      return null;
+    }
+
+    return {
+      processedCount: instance.metadata.processedCount,
+      errorCount: instance.metadata.errorCount,
+      memoryUsage: instance.metadata.memoryUsage,
+      cpuUsage: instance.metadata.cpuUsage,
+    };
+  },
+
+  /**
+   * Get worker health status
+   */
+  getHealth(name: string): 'green' | 'yellow' | 'red' | null {
+    validateWorkerName(name);
+
+    const instance = workers.get(name);
+    if (!instance) {
+      return null;
+    }
+
+    return instance.getHealth();
+  },
+
+  /**
+   * Get registry snapshot
+   */
+  getSnapshot(): WorkerRegistrySnapshot {
+    const allWorkers = Array.from(workers.entries()).map(([name, instance]) => ({
+      name,
+      status: instance.metadata.status,
+      health: instance.getHealth(),
+      uptime: calculateUptime(instance.metadata.startedAt),
+      processedCount: instance.metadata.processedCount,
+      errorCount: instance.metadata.errorCount,
+    }));
+
+    const statusCounts = allWorkers.reduce(
+      (acc, w) => {
+        if (w.status === 'running') acc.running++;
+        else if (w.status === 'stopped') acc.stopped++;
+        else if (w.status === 'sleeping') acc.sleeping++;
+        if (w.health === 'red') acc.unhealthy++;
+        return acc;
+      },
+      { running: 0, stopped: 0, sleeping: 0, unhealthy: 0 }
+    );
+
+    return {
+      timestamp: new Date(),
+      totalWorkers: registrations.size,
+      runningWorkers: statusCounts.running,
+      stoppedWorkers: statusCounts.stopped,
+      sleepingWorkers: statusCounts.sleeping,
+      unhealthyWorkers: statusCounts.unhealthy,
+      workers: allWorkers,
+    };
+  },
+
+  /**
+   * Get worker topology (cluster view)
+   */
+  getTopology(): Record<string, { workers: string[]; count: number }> {
+    const topology: Record<string, Rego> = {};
+
+    for (const [name, instance] of workers.entries()) {
+      const region = instance.metadata.region;
+      if (topology[region].count <= 0) {
+        topology[region] = { workers: [], count: 0 };
+      }
+      topology[region].workers.push(name);
+      topology[region].count++;
+    }
+
+    return topology;
+  },
+
+  /**
+   * Unregister a worker and clear its instance
+   */
+  unregister(name: string): void {
+    validateWorkerName(name);
+
+    const instance = workers.get(name);
+    if (instance?.metadata.status === 'running') {
+      Logger.warn(`Worker "${name}" is still running during unregister`);
+    }
+
+    workers.delete(name);
+    registrations.delete(name);
+
+    Logger.info(`Worker "${name}" unregistered`);
+  },
+
+  /**
+   * Check if worker is registered
+   */
+  isRegistered(name: string): boolean {
+    return registrations.has(name);
+  },
+
+  /**
+   * Check if worker is running
+   */
+  isRunning(name: string): boolean {
+    const instance = workers.get(name);
+    return instance?.metadata.status === 'running';
+  },
+
+  /**
+   * Get worker instance (internal use)
+   */
+  getInstance(name: string): WorkerInstance | null {
+    return workers.get(name) ?? null;
+  },
+});

package/src/WorkerShutdown.ts

@@ -0,0 +1,210 @@
+/**
+ * Worker Shutdown Coordinator
+ *
+ * Centralized graceful shutdown handling for the worker management system.
+ * Coordinates orderly shutdown of all worker modules and the WorkerFactory.
+ */
+
+import { Logger } from '@zintrust/core';
+import { WorkerFactory } from './WorkerFactory';
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface IShutdownOptions {
+  /**
+   * Timeout for graceful shutdown in milliseconds
+   */
+  timeout?: number;
+
+  /**
+   * Whether to force exit after timeout
+   */
+  forceExit?: boolean;
+
+  /**
+   * Signal that triggered shutdown (SIGTERM, SIGINT, etc.)
+   */
+  signal?: string;
+}
+
+interface IShutdownState {
+  isShuttingDown: boolean;
+  completedAt: Date | null;
+  startedAt: Date | null;
+  reason: string | null;
+}
+
+// ============================================================================
+// Implementation
+// ============================================================================
+
+const state: IShutdownState = {
+  isShuttingDown: false,
+  completedAt: null,
+  startedAt: null,
+  reason: null,
+};
+
+let shutdownHandlersRegistered = false;
+
+/**
+ * Perform graceful shutdown of all worker modules
+ */
+async function shutdown(options: IShutdownOptions = {}): Promise<void> {
+  const { timeout = 30000, forceExit = true, signal = 'unknown' } = options;
+
+  // Prevent concurrent shutdowns
+  if (state.isShuttingDown) {
+    Logger.warn('Shutdown already in progress, ignoring duplicate request');
+    return;
+  }
+
+  state.isShuttingDown = true;
+  state.startedAt = new Date();
+  state.reason = `Signal: ${signal}`;
+
+  Logger.info('🛑 Initiating graceful shutdown of worker management system', {
+    signal,
+    timeout,
+    forceExit,
+  });
+
+  // Setup timeout for forced shutdown
+  let timeoutHandle: NodeJS.Timeout | null = null;
+  if (forceExit && timeout > 0) {
+    // eslint-disable-next-line no-restricted-syntax
+    timeoutHandle = setTimeout(() => {
+      Logger.error('❌ Graceful shutdown timeout exceeded, forcing exit', { timeout });
+      process.exit(1);
+    }, timeout);
+  }
+
+  try {
+    // Shutdown WorkerFactory - this will coordinate shutdown of all modules
+    await WorkerFactory.shutdown();
+
+    state.completedAt = new Date();
+    const duration = state.completedAt.getTime() - (state.startedAt?.getTime() ?? 0);
+
+    Logger.info('✅ Worker management system shutdown complete', {
+      duration: `${duration}ms`,
+      signal,
+    });
+
+    // Clear timeout if successful
+    if (timeoutHandle) {
+      clearTimeout(timeoutHandle);
+    }
+  } catch (error) {
+    Logger.error('❌ Error during worker management system shutdown', error);
+    throw error;
+  }
+}
+
+/**
+ * Register process signal handlers for graceful shutdown
+ */
+function registerShutdownHandlers(): void {
+  if (shutdownHandlersRegistered) {
+    Logger.debug('Shutdown handlers already registered, skipping');
+    return;
+  }
+
+  Logger.debug('Registering worker management system shutdown handlers');
+
+  // SIGTERM - graceful shutdown (Docker, systemd, etc.)
+  process.on('SIGTERM', async () => {
+    Logger.info('📨 Received SIGTERM signal');
+    try {
+      await shutdown({ signal: 'SIGTERM', timeout: 30000, forceExit: true });
+    } catch (error) {
+      Logger.error('Error during SIGTERM shutdown', error);
+    }
+  });
+
+  // SIGINT - user interrupt (Ctrl+C) - REMOVED: handled by bootstrap.ts to prevent race condition
+  // process.on('SIGINT', async () => {
+  //   Logger.info('📨 Received SIGINT signal');
+  //   try {
+  //     await shutdown({ signal: 'SIGINT', timeout: 30000, forceExit: true });
+  //   } catch (error) {
+  //     Logger.error('Error during SIGINT shutdown', error);
+  //   }
+  // });
+
+  // SIGHUP - terminal closed
+  process.on('SIGHUP', async () => {
+    Logger.info('📨 Received SIGHUP signal');
+    try {
+      await shutdown({ signal: 'SIGHUP', timeout: 30000, forceExit: true });
+    } catch (error) {
+      Logger.error('Error during SIGHUP shutdown', error);
+    }
+  });
+
+  // Handle uncaught errors during shutdown
+  process.on('uncaughtException', async (error: Error) => {
+    Logger.error('💥 Uncaught exception during worker operations', error);
+    try {
+      await shutdown({ signal: 'uncaughtException', timeout: 10000, forceExit: true });
+    } catch {
+      // Ignore errors during emergency shutdown
+    }
+    process.exit(1);
+  });
+
+  process.on('unhandledRejection', async (reason: unknown) => {
+    Logger.error('💥 Unhandled promise rejection during worker operations', reason);
+    try {
+      await shutdown({ signal: 'unhandledRejection', timeout: 10000, forceExit: true });
+    } catch {
+      // Ignore errors during emergency shutdown
+    }
+    process.exit(1);
+  });
+
+  shutdownHandlersRegistered = true;
+  Logger.debug('Worker management system shutdown handlers registered');
+}
+
+/**
+ * Check if system is currently shutting down
+ */
+function isShuttingDown(): boolean {
+  return state.isShuttingDown;
+}
+
+/**
+ * Get current shutdown state
+ */
+function getShutdownState(): Readonly<IShutdownState> {
+  return { ...state };
+}
+
+// ============================================================================
+// Public API (Sealed Namespace)
+// ============================================================================
+
+export const WorkerShutdown = Object.freeze({
+  /**
+   * Perform graceful shutdown of all worker modules
+   */
+  shutdown,
+
+  /**
+   * Register process signal handlers for graceful shutdown
+   */
+  registerShutdownHandlers,
+
+  /**
+   * Check if system is currently shutting down
+   */
+  isShuttingDown,
+
+  /**
+   * Get current shutdown state
+   */
+  getShutdownState,
+});