@zintrust/workers 0.1.27

package/dist/HealthMonitor.js

@@ -0,0 +1,301 @@
+import { ErrorFactory, Logger } from '@zintrust/core';
+import { WorkerCreationStatus, WorkerFactory } from './WorkerFactory';
+const DEFAULT_CONFIG = {
+    enabled: true,
+    tickIntervalMs: 1000,
+    concurrencyLimit: 50,
+    checkTimeoutMs: 5000,
+    intervalHealthyMs: 30000,
+    intervalSuspectMs: 5000,
+    failureThreshold: 2,
+    historyLimit: 50,
+};
+// Module-level state (Singleton by nature of ESM)
+const registry = new Map();
+let config = { ...DEFAULT_CONFIG };
+let timer = null;
+let runningChecks = 0;
+// Internal Helpers
+const persistStatusChange = async (name, status, lastError) => {
+    try {
+        await WorkerFactory.updateStatus(name, status, lastError);
+    }
+    catch (err) {
+        Logger.error(`Failed to persist status change for ${name}`, err);
+    }
+};
+const verifyWorkerHealth = async (worker, _name, _queueName) => {
+    // Check if isClosing exists (isClosing check safe for mocks)
+    const workerAny = worker;
+    const isClosingFn = workerAny['isClosing'];
+    if (worker.isPaused() ||
+        (typeof isClosingFn === 'function' && isClosingFn())) {
+        return false;
+    }
+    const isRunning = await worker.isRunning();
+    if (!isRunning)
+        return false;
+    const client = await worker.client;
+    const pingResult = await client.ping();
+    if (pingResult !== 'PONG') {
+        throw ErrorFactory.createWorkerError(`Redis ping failed: ${pingResult}`);
+    }
+    Logger.debug(`Worker health verification passed for ${_name} ${_queueName}`);
+    return true;
+};
+const updateState = (state, isHealthy, errorMsg, latency) => {
+    const now = new Date();
+    state.lastCheck = now;
+    // Determine status (healthy > degraded > critical)
+    let status;
+    if (isHealthy) {
+        status = 'healthy';
+    }
+    else if (state.consecutiveFailures < config.failureThreshold) {
+        status = 'degraded';
+    }
+    else {
+        status = 'critical';
+    }
+    // Create Check Result
+    const result = {
+        timestamp: now,
+        status,
+        latency,
+        message: errorMsg,
+    };
+    // Add to history
+    state.history.push(result);
+    if (state.history.length > config.historyLimit) {
+        state.history.shift();
+    }
+    // Callbacks
+    if (!isHealthy && state.config?.degradedCallback) {
+        state.config.degradedCallback(state.name, result);
+    }
+    if (result.status === 'critical' && state.config?.criticalCallback) {
+        state.config.criticalCallback(state.name, result);
+    }
+    if (isHealthy) {
+        state.consecutiveFailures = 0;
+        if (state.status !== WorkerCreationStatus.RUNNING) {
+            persistStatusChange(state.name, WorkerCreationStatus.RUNNING);
+            state.status = WorkerCreationStatus.RUNNING;
+            Logger.info(`Worker ${state.name} recovered to RUNNING`);
+        }
+        const jitter = Math.floor(Math.random() * 500); //NOSONAR
+        state.nextCheck = new Date(now.getTime() + config.intervalHealthyMs + jitter);
+    }
+    else {
+        state.consecutiveFailures++;
+        if (state.consecutiveFailures >= config.failureThreshold &&
+            state.status !== WorkerCreationStatus.FAILED) {
+            persistStatusChange(state.name, WorkerCreationStatus.FAILED, errorMsg);
+            state.status = WorkerCreationStatus.FAILED;
+            Logger.warn(`Worker ${state.name} marked FAILED after ${state.consecutiveFailures} checks`, {
+                error: errorMsg,
+            });
+        }
+        const jitter = Math.floor(Math.random() * 500); //NOSONAR
+        state.nextCheck = new Date(now.getTime() + config.intervalSuspectMs + jitter);
+    }
+};
+const performCheck = async (state) => {
+    const startTime = Date.now();
+    let isHealthy = false;
+    let errorMsg;
+    try {
+        if (!state.worker) {
+            throw ErrorFactory.createWorkerError('Worker instance not available');
+        }
+        isHealthy = await Promise.race([
+            verifyWorkerHealth(state.worker, state.name, state.queueName || 'unknown'),
+            new Promise((_, reject) => {
+                // eslint-disable-next-line
+                const id = setTimeout(() => {
+                    reject(ErrorFactory.createWorkerError('Health check timeout'));
+                }, config.checkTimeoutMs);
+                // Unref to prevent holding event loop if everything else finishes
+                id.unref();
+            }),
+        ]);
+    }
+    catch (err) {
+        isHealthy = false;
+        errorMsg = err.message;
+    }
+    const duration = Date.now() - startTime;
+    updateState(state, isHealthy, errorMsg, duration);
+};
+const scheduleCheck = async (state) => {
+    state.inProgress = true;
+    runningChecks++;
+    performCheck(state).finally(() => {
+        state.inProgress = false;
+        runningChecks--;
+    });
+};
+const tick = async () => {
+    const now = new Date();
+    const candidates = [];
+    for (const state of registry.values()) {
+        if (runningChecks >= config.concurrencyLimit)
+            break;
+        // Skip if checks are paused or if worker instance is missing (wait for register)
+        if (!state.worker && !state.queueName)
+            continue;
+        if (!state.inProgress && state.nextCheck <= now) {
+            candidates.push(state);
+        }
+    }
+    for (const candidate of candidates) {
+        if (runningChecks >= config.concurrencyLimit)
+            break;
+        scheduleCheck(candidate);
+    }
+};
+const start = () => {
+    if (timer)
+        return;
+    timer = setInterval(() => tick(), config.tickIntervalMs);
+    Logger.debug('HealthMonitor started');
+};
+const stop = () => {
+    if (timer) {
+        clearInterval(timer);
+        timer = null;
+    }
+    Logger.debug('HealthMonitor stopped');
+};
+// Exported Public Methods
+const configure = (newConfig) => {
+    config = { ...config, ...newConfig };
+    if (timer) {
+        stop();
+        start();
+    }
+};
+const register = (name, worker, queueName) => {
+    let state = registry.get(name);
+    if (state) {
+        // update existing entry (maybe created by startMonitoring)
+        state.worker = worker;
+        state.queueName = queueName;
+    }
+    else {
+        // Add jitter
+        const initialDelay = Math.floor(Math.random() * 5000); //NOSONAR
+        state = {
+            name,
+            worker,
+            queueName,
+            status: WorkerCreationStatus.STARTING,
+            lastCheck: new Date(),
+            nextCheck: new Date(Date.now() + initialDelay),
+            consecutiveFailures: 0,
+            inProgress: false,
+            history: [],
+        };
+        registry.set(name, state);
+    }
+    if (!timer) {
+        start();
+    }
+};
+const startMonitoring = (name, monitorConfig) => {
+    const state = registry.get(name);
+    if (state) {
+        if (monitorConfig)
+            state.config = { ...state.config, ...monitorConfig };
+    }
+    else {
+        // Worker instance not yet registered, create placeholder
+        const initialDelay = Math.floor(Math.random() * 5000); //NOSONAR
+        registry.set(name, {
+            name,
+            status: WorkerCreationStatus.STARTING,
+            lastCheck: new Date(),
+            nextCheck: new Date(Date.now() + initialDelay),
+            consecutiveFailures: 0,
+            inProgress: false,
+            history: [],
+            config: monitorConfig,
+        });
+    }
+    if (!timer)
+        start();
+};
+const unregister = (name) => {
+    registry.delete(name);
+    if (registry.size === 0) {
+        stop();
+    }
+};
+const stopMonitoring = (name) => {
+    unregister(name);
+};
+const updateConfig = (name, monitorConfig) => {
+    startMonitoring(name, monitorConfig);
+};
+const getCurrentHealth = (name) => {
+    const state = registry.get(name);
+    if (!state || state.history.length === 0)
+        return null;
+    return state.history[state.history.length - 1];
+};
+const getHealthHistory = (name, limit) => {
+    const state = registry.get(name);
+    if (!state)
+        return [];
+    const history = state.history;
+    return limit ? history.slice(-limit) : history;
+};
+const getHealthTrend = (name) => {
+    const history = getHealthHistory(name, 10);
+    const uptime = history.filter((h) => h.status === 'healthy').length / (history.length || 1);
+    return { uptime, samples: history.length };
+};
+const getSummary = async () => {
+    const summary = {
+        total: registry.size,
+        healthy: 0,
+        degraded: 0,
+        critical: 0,
+        details: [],
+    };
+    for (const [name, state] of registry) {
+        const lastResult = state.history[state.history.length - 1];
+        const status = lastResult?.status || 'unknown';
+        if (status === 'healthy')
+            summary.healthy++;
+        else if (status === 'degraded')
+            summary.degraded++;
+        else if (status === 'critical')
+            summary.critical++;
+        summary.details.push({
+            name,
+            status,
+            lastCheck: state.lastCheck,
+        });
+    }
+    return summary;
+};
+const shutdown = () => {
+    stop();
+    registry.clear();
+};
+export const HealthMonitor = Object.freeze({
+    configure,
+    register,
+    unregister,
+    start,
+    stop,
+    startMonitoring,
+    stopMonitoring,
+    updateConfig,
+    getCurrentHealth,
+    getHealthHistory,
+    getHealthTrend,
+    getSummary,
+    shutdown,
+});

package/dist/MultiQueueWorker.d.ts

@@ -0,0 +1,89 @@
+/**
+ * Multi-Queue Worker Support
+ * Enable workers to process multiple queues with different priorities
+ * Sealed namespace for immutability
+ */
+import { type Job } from 'bullmq';
+export type QueueConfig = {
+    name: string;
+    concurrency: number;
+    priority: number;
+    enabled: boolean;
+    rateLimit?: {
+        max: number;
+        duration: number;
+    };
+};
+export type MultiQueueWorkerConfig = {
+    workerName: string;
+    queues: QueueConfig[];
+    processor: (job: Job) => Promise<unknown>;
+    sharedConcurrency?: number;
+    defaultConcurrency?: number;
+};
+export type QueueStats = {
+    queueName: string;
+    processed: number;
+    failed: number;
+    active: number;
+    waiting: number;
+    enabled: boolean;
+    lastProcessedAt?: Date;
+};
+/**
+ * Multi-Queue Worker Manager - Sealed namespace
+ */
+export declare const MultiQueueWorker: Readonly<{
+    /**
+     * Create multi-queue worker
+     */
+    create(config: MultiQueueWorkerConfig): void;
+    /**
+     * Start processing for a specific queue
+     */
+    startQueue(workerName: string, queueName: string): Promise<void>;
+    /**
+     * Stop processing for a specific queue
+     */
+    stopQueue(workerName: string, queueName: string): Promise<void>;
+    /**
+     * Start all queues
+     */
+    startAll(workerName: string): Promise<void>;
+    /**
+     * Stop all queues
+     */
+    stopAll(workerName: string): Promise<void>;
+    /**
+     * Update queue priority
+     */
+    updateQueuePriority(workerName: string, queueName: string, priority: number): void;
+    /**
+     * Update queue concurrency
+     */
+    updateQueueConcurrency(workerName: string, queueName: string, concurrency: number): Promise<void>;
+    /**
+     * Get stats for a queue
+     */
+    getQueueStats(workerName: string, queueName: string): Promise<QueueStats>;
+    /**
+     * Get stats for all queues
+     */
+    getAllStats(workerName: string): Promise<ReadonlyArray<QueueStats>>;
+    /**
+     * Get configuration
+     */
+    getConfig(workerName: string): MultiQueueWorkerConfig | null;
+    /**
+     * List all multi-queue workers
+     */
+    list(): string[];
+    /**
+     * Remove multi-queue worker
+     */
+    remove(workerName: string): Promise<void>;
+    /**
+     * Shutdown all multi-queue workers
+     */
+    shutdown(): Promise<void>;
+}>;

package/dist/MultiQueueWorker.js

@@ -0,0 +1,277 @@
+/**
+ * Multi-Queue Worker Support
+ * Enable workers to process multiple queues with different priorities
+ * Sealed namespace for immutability
+ */
+import { ErrorFactory, Logger } from '@zintrust/core';
+import { Worker } from 'bullmq';
+import { PriorityQueue } from './PriorityQueue';
+// Internal state
+const multiQueueWorkers = new Map();
+/**
+ * Helper: Create worker for a queue
+ */
+const createQueueWorker = (workerName, queueConfig, processor) => {
+    const queue = PriorityQueue.getQueueInstance(queueConfig.name);
+    const connection = queue.opts.connection;
+    const workerOptions = {
+        connection,
+        concurrency: queueConfig.concurrency,
+        limiter: queueConfig.rateLimit,
+        autorun: queueConfig.enabled,
+        prefix: queue.opts.prefix,
+    };
+    const worker = new Worker(queueConfig.name, async (job) => {
+        Logger.debug(`Processing job from queue: ${queueConfig.name}`, {
+            jobId: job.id,
+            workerName,
+        });
+        return processor(job);
+    }, workerOptions);
+    // Set up event listeners
+    worker.on('completed', (job) => {
+        const stats = multiQueueWorkers.get(workerName)?.stats.get(queueConfig.name);
+        if (stats) {
+            stats.processed++;
+            stats.lastProcessedAt = new Date();
+        }
+        Logger.debug(`Job completed from queue: ${queueConfig.name}`, {
+            jobId: job.id,
+            workerName,
+        });
+    });
+    worker.on('failed', (job, error) => {
+        const stats = multiQueueWorkers.get(workerName)?.stats.get(queueConfig.name);
+        if (stats) {
+            stats.failed++;
+        }
+        Logger.error(`Job failed from queue: ${queueConfig.name}`, error, 'workers');
+        Logger.debug('Queue job failure details', {
+            jobId: job?.id,
+            workerName,
+            queueName: queueConfig.name,
+        });
+    });
+    worker.on('active', (_job) => {
+        const stats = multiQueueWorkers.get(workerName)?.stats.get(queueConfig.name);
+        if (stats) {
+            stats.active++;
+        }
+    });
+    return worker;
+};
+/**
+ * Helper: Initialize stats for queue
+ */
+const initializeQueueStats = (queueName, enabled) => {
+    return {
+        queueName,
+        processed: 0,
+        failed: 0,
+        active: 0,
+        waiting: 0,
+        enabled,
+    };
+};
+/**
+ * Multi-Queue Worker Manager - Sealed namespace
+ */
+export const MultiQueueWorker = Object.freeze({
+    /**
+     * Create multi-queue worker
+     */
+    create(config) {
+        if (multiQueueWorkers.has(config.workerName)) {
+            throw ErrorFactory.createWorkerError(`Multi-queue worker "${config.workerName}" already exists`);
+        }
+        const workers = new Map();
+        const stats = new Map();
+        // Sort queues by priority (higher first)
+        const sortedQueues = [...config.queues].sort((a, b) => b.priority - a.priority);
+        // Create workers for each queue
+        for (const queueConfig of sortedQueues) {
+            const worker = createQueueWorker(config.workerName, queueConfig, config.processor);
+            workers.set(queueConfig.name, worker);
+            stats.set(queueConfig.name, initializeQueueStats(queueConfig.name, queueConfig.enabled));
+        }
+        multiQueueWorkers.set(config.workerName, {
+            config,
+            workers,
+            stats,
+        });
+        Logger.info(`Multi-queue worker created: ${config.workerName}`, {
+            queues: sortedQueues.map((q) => q.name),
+            totalConcurrency: sortedQueues.reduce((sum, q) => sum + q.concurrency, 0),
+        });
+    },
+    /**
+     * Start processing for a specific queue
+     */
+    async startQueue(workerName, queueName) {
+        const mqw = multiQueueWorkers.get(workerName);
+        if (!mqw) {
+            throw ErrorFactory.createNotFoundError(`Multi-queue worker "${workerName}" not found`);
+        }
+        const worker = mqw.workers.get(queueName);
+        if (!worker) {
+            throw ErrorFactory.createNotFoundError(`Queue "${queueName}" not found in worker "${workerName}"`);
+        }
+        await worker.run();
+        const stats = mqw.stats.get(queueName);
+        if (stats) {
+            stats.enabled = true;
+        }
+        Logger.info(`Queue started: ${queueName}`, { workerName });
+    },
+    /**
+     * Stop processing for a specific queue
+     */
+    async stopQueue(workerName, queueName) {
+        const mqw = multiQueueWorkers.get(workerName);
+        if (!mqw) {
+            throw ErrorFactory.createNotFoundError(`Multi-queue worker "${workerName}" not found`);
+        }
+        const worker = mqw.workers.get(queueName);
+        if (!worker) {
+            throw ErrorFactory.createNotFoundError(`Queue "${queueName}" not found in worker "${workerName}"`);
+        }
+        await worker.pause();
+        const stats = mqw.stats.get(queueName);
+        if (stats) {
+            stats.enabled = false;
+        }
+        Logger.info(`Queue stopped: ${queueName}`, { workerName });
+    },
+    /**
+     * Start all queues
+     */
+    async startAll(workerName) {
+        const mqw = multiQueueWorkers.get(workerName);
+        if (!mqw) {
+            throw ErrorFactory.createNotFoundError(`Multi-queue worker "${workerName}" not found`);
+        }
+        const promises = Array.from(mqw.workers.keys()).map(async (queueName) => MultiQueueWorker.startQueue(workerName, queueName));
+        await Promise.all(promises);
+        Logger.info(`All queues started for worker: ${workerName}`);
+    },
+    /**
+     * Stop all queues
+     */
+    async stopAll(workerName) {
+        const mqw = multiQueueWorkers.get(workerName);
+        if (!mqw) {
+            throw ErrorFactory.createNotFoundError(`Multi-queue worker "${workerName}" not found`);
+        }
+        const promises = Array.from(mqw.workers.keys()).map(async (queueName) => MultiQueueWorker.stopQueue(workerName, queueName));
+        await Promise.all(promises);
+        Logger.info(`All queues stopped for worker: ${workerName}`);
+    },
+    /**
+     * Update queue priority
+     */
+    updateQueuePriority(workerName, queueName, priority) {
+        const mqw = multiQueueWorkers.get(workerName);
+        if (!mqw) {
+            throw ErrorFactory.createNotFoundError(`Multi-queue worker "${workerName}" not found`);
+        }
+        const queueConfig = mqw.config.queues.find((q) => q.name === queueName);
+        if (!queueConfig) {
+            throw ErrorFactory.createNotFoundError(`Queue "${queueName}" not found in worker "${workerName}"`);
+        }
+        queueConfig.priority = priority;
+        Logger.info(`Queue priority updated: ${queueName}`, { workerName, priority });
+    },
+    /**
+     * Update queue concurrency
+     */
+    async updateQueueConcurrency(workerName, queueName, concurrency) {
+        const mqw = multiQueueWorkers.get(workerName);
+        if (!mqw) {
+            throw ErrorFactory.createNotFoundError(`Multi-queue worker "${workerName}" not found`);
+        }
+        const worker = mqw.workers.get(queueName);
+        if (!worker) {
+            throw ErrorFactory.createNotFoundError(`Queue "${queueName}" not found in worker "${workerName}"`);
+        }
+        const queueConfig = mqw.config.queues.find((q) => q.name === queueName);
+        if (!queueConfig) {
+            throw ErrorFactory.createNotFoundError(`Queue "${queueName}" not found in worker "${workerName}"`);
+        }
+        queueConfig.concurrency = concurrency;
+        // Update worker concurrency (requires restart in BullMQ)
+        await worker.close();
+        const newWorker = createQueueWorker(workerName, queueConfig, mqw.config.processor);
+        mqw.workers.set(queueName, newWorker);
+        Logger.info(`Queue concurrency updated: ${queueName}`, { workerName, concurrency });
+    },
+    /**
+     * Get stats for a queue
+     */
+    async getQueueStats(workerName, queueName) {
+        const mqw = multiQueueWorkers.get(workerName);
+        if (!mqw) {
+            throw ErrorFactory.createNotFoundError(`Multi-queue worker "${workerName}" not found`);
+        }
+        const stats = mqw.stats.get(queueName);
+        if (!stats) {
+            throw ErrorFactory.createNotFoundError(`Stats for queue "${queueName}" not found in worker "${workerName}"`);
+        }
+        // Update waiting count from queue
+        PriorityQueue.getQueueInstance(queueName);
+        const queueInfo = await PriorityQueue.getQueueInfo(queueName);
+        stats.waiting = queueInfo.jobCounts.waiting;
+        return { ...stats };
+    },
+    /**
+     * Get stats for all queues
+     */
+    async getAllStats(workerName) {
+        const mqw = multiQueueWorkers.get(workerName);
+        if (!mqw) {
+            throw ErrorFactory.createNotFoundError(`Multi-queue worker "${workerName}" not found`);
+        }
+        const allStats = await Promise.all(Array.from(mqw.workers.keys()).map(async (queueName) => MultiQueueWorker.getQueueStats(workerName, queueName)));
+        return allStats;
+    },
+    /**
+     * Get configuration
+     */
+    getConfig(workerName) {
+        const mqw = multiQueueWorkers.get(workerName);
+        return mqw ? { ...mqw.config } : null;
+    },
+    /**
+     * List all multi-queue workers
+     */
+    list() {
+        return Array.from(multiQueueWorkers.keys());
+    },
+    /**
+     * Remove multi-queue worker
+     */
+    async remove(workerName) {
+        const mqw = multiQueueWorkers.get(workerName);
+        if (!mqw) {
+            throw ErrorFactory.createNotFoundError(`Multi-queue worker "${workerName}" not found`);
+        }
+        // Close all workers
+        const closePromises = Array.from(mqw.workers.values()).map(async (worker) => worker.close());
+        await Promise.all(closePromises);
+        multiQueueWorkers.delete(workerName);
+        Logger.info(`Multi-queue worker removed: ${workerName}`);
+    },
+    /**
+     * Shutdown all multi-queue workers
+     */
+    async shutdown() {
+        Logger.info('MultiQueueWorker shutting down...');
+        const shutdownPromises = Array.from(multiQueueWorkers.values()).map(async (mqw) => {
+            const closePromises = Array.from(mqw.workers.values()).map(async (worker) => worker.close());
+            await Promise.all(closePromises);
+        });
+        await Promise.all(shutdownPromises);
+        multiQueueWorkers.clear();
+        Logger.info('MultiQueueWorker shutdown complete');
+    },
+});
+// Graceful shutdown handled by WorkerShutdown

package/dist/NotificationWorker.d.ts

@@ -0,0 +1,21 @@
+/**
+ * NotificationWorker - Processes queued notifications
+ *
+ * This worker dequeues notification messages and sends them using the Notification service.
+ * Use with Queue.dequeue() in a background process or cron job.
+ */
+export declare const NotificationWorker: Readonly<{
+    processOne: (queueName?: string, driverName?: string) => Promise<boolean>;
+    processAll: (queueName?: string, driverName?: string) => Promise<number>;
+    runOnce: (opts?: {
+        queueName?: string;
+        driverName?: string;
+        maxItems?: number;
+    }) => Promise<number>;
+    startWorker: (opts?: {
+        queueName?: string;
+        driverName?: string;
+        signal?: AbortSignal;
+    }) => Promise<number>;
+}>;
+export default NotificationWorker;