@zintrust/workers 0.1.27

package/dist/WorkerMetrics.js

@@ -0,0 +1,509 @@
+/**
+ * Worker Metrics Manager
+ * Time-series metrics persistence with Redis Sorted Sets
+ * Sealed namespace for immutability
+ */
+import { ErrorFactory, Logger, appConfig, createRedisConnection, } from '@zintrust/core';
+const PREFIX = appConfig.prefix;
+// Redis key prefixes
+const METRICS_PREFIX = `${PREFIX}:worker:metrics:`;
+const HEALTH_PREFIX = `${PREFIX}:worker:health:`;
+// Retention periods (in seconds)
+const RETENTION = {
+    hourly: 7 * 24 * 60 * 60, // 7 days
+    daily: 30 * 24 * 60 * 60, // 30 days
+    monthly: 365 * 24 * 60 * 60, // 1 year
+};
+// Internal state
+let redisClient = null;
+/**
+ * Helper: Get Redis key for metrics
+ */
+const getMetricsKey = (workerName, metricType, granularity) => {
+    return `${METRICS_PREFIX}${workerName}:${metricType}:${granularity}`;
+};
+/**
+ * Helper: Get Redis key for health scores
+ */
+const getHealthKey = (workerName) => {
+    return `${HEALTH_PREFIX}${workerName}`;
+};
+/**
+ * Helper: Round timestamp to granularity
+ */
+const roundTimestamp = (date, granularity) => {
+    const timestamp = date.getTime();
+    switch (granularity) {
+        case 'hourly':
+            // Round to nearest hour
+            return new Date(Math.floor(timestamp / (60 * 60 * 1000)) * 60 * 60 * 1000);
+        case 'daily': {
+            // Round to start of day (UTC)
+            const d = new Date(timestamp);
+            d.setUTCHours(0, 0, 0, 0);
+            return d;
+        }
+        case 'monthly': {
+            // Round to start of month (UTC)
+            const m = new Date(timestamp);
+            m.setUTCDate(1);
+            m.setUTCHours(0, 0, 0, 0);
+            return m;
+        }
+    }
+};
+/**
+ * Helper: Clean up old metrics based on retention policy
+ */
+const cleanupOldMetrics = async (client, key, granularity) => {
+    try {
+        const retentionSeconds = RETENTION[granularity];
+        const cutoffTimestamp = Date.now() - retentionSeconds * 1000;
+        // Remove entries older than retention period
+        await client.zremrangebyscore(key, '-inf', cutoffTimestamp);
+        // Set expiry on the key (2x retention period for safety)
+        await client.expire(key, retentionSeconds * 2);
+    }
+    catch (error) {
+        Logger.error(`Failed to cleanup old metrics for key "${key}"`, error);
+    }
+};
+/**
+ * Helper: Calculate health score based on metrics
+ */
+const calculateHealthScore = (metrics) => {
+    // Error rate factor (0-100, lower is better)
+    // 0% errors = 100, 10%+ errors = 0
+    const errorRateFactor = Math.max(0, 100 - metrics.errorRate * 1000);
+    // Throughput factor (0-100, higher is better)
+    // Normalized: >100 jobs/min = 100, 0 jobs/min = 0
+    const throughputFactor = Math.min(100, metrics.throughput);
+    // Latency factor (0-100, lower is better)
+    // <1s = 100, >10s = 0
+    const latencyFactor = Math.max(0, 100 - (metrics.avgDuration / 10000) * 100);
+    // Resource usage factor (0-100, lower is better)
+    // <50% = 100, >90% = 0
+    const avgResourceUsage = (metrics.memoryUsage + metrics.cpuUsage) / 2;
+    const resourceFactor = Math.max(0, 100 - Math.max(0, avgResourceUsage - 50) * 2.5);
+    // Weighted average: errors are most important
+    const score = errorRateFactor * 0.4 + throughputFactor * 0.2 + latencyFactor * 0.2 + resourceFactor * 0.2;
+    let status;
+    if (score >= 80) {
+        status = 'healthy';
+    }
+    else if (score >= 50) {
+        status = 'degraded';
+    }
+    else {
+        status = 'unhealthy';
+    }
+    return {
+        score: Math.round(score),
+        status,
+        factors: {
+            errorRate: Math.round(errorRateFactor),
+            throughput: Math.round(throughputFactor),
+            latency: Math.round(latencyFactor),
+            resourceUsage: Math.round(resourceFactor),
+        },
+    };
+};
+/**
+ * Worker Metrics Manager - Sealed namespace
+ */
+export const WorkerMetrics = Object.freeze({
+    /**
+     * Initialize the metrics manager with Redis connection
+     */
+    initialize(config) {
+        if (redisClient) {
+            Logger.warn('WorkerMetrics already initialized');
+            return;
+        }
+        redisClient = createRedisConnection(config);
+        Logger.info('WorkerMetrics initialized');
+    },
+    /**
+     * Record a metric point
+     */
+    async record(workerName, metricType, value, metadata) {
+        if (!redisClient) {
+            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized. Call initialize() first.');
+        }
+        const now = new Date();
+        // Record at all granularities
+        const granularities = ['hourly', 'daily', 'monthly'];
+        await Promise.all(granularities.map(async (granularity) => {
+            const roundedTimestamp = roundTimestamp(now, granularity);
+            const key = getMetricsKey(workerName, metricType, granularity);
+            const point = {
+                timestamp: roundedTimestamp,
+                value,
+                metadata,
+            };
+            // Store in sorted set with timestamp as score
+            const score = roundedTimestamp.getTime();
+            const data = JSON.stringify(point);
+            await redisClient?.zadd(key, score, data);
+            // Cleanup old metrics (lightweight: ~1% based on time slice)
+            const client = redisClient;
+            if (client && Date.now() % 100 === 0) {
+                cleanupOldMetrics(client, key, granularity).catch((err) => {
+                    Logger.error('Failed to cleanup old metrics', err);
+                });
+            }
+        }));
+        Logger.debug(`Recorded metric: ${workerName}/${metricType} = ${value}`);
+    },
+    /**
+     * Record multiple metrics at once (batch operation)
+     */
+    async recordBatch(workerName, metrics) {
+        await Promise.all(metrics.map(async (m) => WorkerMetrics.record(workerName, m.metricType, m.value, m.metadata)));
+    },
+    /**
+     * Query metrics for a time range
+     */
+    async query(options) {
+        if (!redisClient) {
+            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
+        }
+        const { workerName, metricType, granularity, startDate, endDate, limit = 1000 } = options;
+        const key = getMetricsKey(workerName, metricType, granularity);
+        const minScore = startDate ? startDate.getTime() : '-inf';
+        const maxScore = endDate ? endDate.getTime() : '+inf';
+        try {
+            // Get data from sorted set
+            const results = await redisClient.zrangebyscore(key, minScore, maxScore, 'LIMIT', 0, limit);
+            const points = results.map((data) => JSON.parse(data));
+            return {
+                workerName,
+                metricType,
+                granularity,
+                points,
+            };
+        }
+        catch (error) {
+            Logger.error(`Error querying metrics for ${workerName}/${metricType}`, error);
+            throw error;
+        }
+    },
+    /**
+     * Get aggregated metrics for a time range
+     */
+    async aggregate(options) {
+        const entry = await WorkerMetrics.query(options);
+        if (entry.points.length === 0) {
+            return {
+                workerName: entry.workerName,
+                metricType: entry.metricType,
+                period: {
+                    start: options.startDate ?? new Date(0),
+                    end: options.endDate ?? new Date(),
+                },
+                total: 0,
+                average: 0,
+                min: 0,
+                max: 0,
+                count: 0,
+            };
+        }
+        const values = entry.points.map((p) => p.value);
+        const total = values.reduce((sum, val) => sum + val, 0);
+        const average = total / values.length;
+        const min = Math.min(...values);
+        const max = Math.max(...values);
+        return {
+            workerName: entry.workerName,
+            metricType: entry.metricType,
+            period: {
+                start: entry.points[0].timestamp,
+                end: entry.points.at(-1)?.timestamp ?? new Date(),
+            },
+            total,
+            average,
+            min,
+            max,
+            count: values.length,
+        };
+    },
+    async aggregateBatch(optionsList) {
+        if (!redisClient) {
+            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
+        }
+        if (optionsList.length === 0)
+            return [];
+        const pipeline = redisClient.pipeline();
+        for (const options of optionsList) {
+            const { workerName, metricType, granularity, startDate, endDate, limit = 1000 } = options;
+            const key = getMetricsKey(workerName, metricType, granularity);
+            const minScore = startDate ? startDate.getTime() : '-inf';
+            const maxScore = endDate ? endDate.getTime() : '+inf';
+            pipeline.zrangebyscore(key, minScore, maxScore, 'LIMIT', 0, limit);
+        }
+        const results = await pipeline.exec();
+        if (!results) {
+            throw ErrorFactory.createWorkerError('Failed to execute metrics pipeline');
+        }
+        return optionsList.map((options, index) => {
+            const [err, data] = results[index];
+            if (err) {
+                Logger.error(`Error querying metrics for ${options.workerName}/${options.metricType}`, err);
+                return {
+                    workerName: options.workerName,
+                    metricType: options.metricType,
+                    period: { start: options.startDate ?? new Date(), end: options.endDate ?? new Date() },
+                    total: 0,
+                    average: 0,
+                    min: 0,
+                    max: 0,
+                    count: 0,
+                };
+            }
+            const points = data.map((d) => JSON.parse(d));
+            if (points.length === 0) {
+                return {
+                    workerName: options.workerName,
+                    metricType: options.metricType,
+                    period: { start: options.startDate ?? new Date(0), end: options.endDate ?? new Date() },
+                    total: 0,
+                    average: 0,
+                    min: 0,
+                    max: 0,
+                    count: 0,
+                };
+            }
+            const values = points.map((p) => p.value);
+            const total = values.reduce((sum, val) => sum + val, 0);
+            const average = total / values.length;
+            const min = Math.min(...values);
+            const max = Math.max(...values);
+            return {
+                workerName: options.workerName,
+                metricType: options.metricType,
+                period: {
+                    start: points[0].timestamp,
+                    end: points.at(-1)?.timestamp ?? new Date(),
+                },
+                total,
+                average,
+                min,
+                max,
+                count: values.length,
+            };
+        });
+    },
+    /**
+     * Calculate and store health score
+     */
+    async calculateHealth(workerName) {
+        if (!redisClient) {
+            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
+        }
+        const now = new Date();
+        const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);
+        try {
+            // Get recent metrics (last hour)
+            const [processed, errors, duration, memory, cpu] = await Promise.all([
+                WorkerMetrics.aggregate({
+                    workerName,
+                    metricType: 'processed',
+                    granularity: 'hourly',
+                    startDate: oneHourAgo,
+                    endDate: now,
+                }),
+                WorkerMetrics.aggregate({
+                    workerName,
+                    metricType: 'errors',
+                    granularity: 'hourly',
+                    startDate: oneHourAgo,
+                    endDate: now,
+                }),
+                WorkerMetrics.aggregate({
+                    workerName,
+                    metricType: 'duration',
+                    granularity: 'hourly',
+                    startDate: oneHourAgo,
+                    endDate: now,
+                }),
+                WorkerMetrics.aggregate({
+                    workerName,
+                    metricType: 'memory',
+                    granularity: 'hourly',
+                    startDate: oneHourAgo,
+                    endDate: now,
+                }),
+                WorkerMetrics.aggregate({
+                    workerName,
+                    metricType: 'cpu',
+                    granularity: 'hourly',
+                    startDate: oneHourAgo,
+                    endDate: now,
+                }),
+            ]);
+            const totalJobs = processed.total + errors.total;
+            const errorRate = totalJobs > 0 ? errors.total / totalJobs : 0;
+            const throughput = processed.total; // Jobs in last hour
+            const avgDuration = duration.average || 0;
+            const memoryUsage = memory.average || 0;
+            const cpuUsage = cpu.average || 0;
+            const healthData = calculateHealthScore({
+                errorRate,
+                throughput,
+                avgDuration,
+                memoryUsage,
+                cpuUsage,
+            });
+            const healthScore = {
+                workerName,
+                timestamp: now,
+                score: healthData.score,
+                factors: healthData.factors,
+                status: healthData.status,
+            };
+            // Store health score in sorted set (keep last 24 hours)
+            const key = getHealthKey(workerName);
+            const score = now.getTime();
+            const data = JSON.stringify(healthScore);
+            await redisClient.zadd(key, score, data);
+            // Keep only last 24 hours
+            const cutoff = now.getTime() - 24 * 60 * 60 * 1000;
+            await redisClient.zremrangebyscore(key, '-inf', cutoff);
+            // Set expiry (48 hours)
+            await redisClient.expire(key, 48 * 60 * 60);
+            Logger.debug(`Health score for ${workerName}: ${healthScore.score} (${healthScore.status})`);
+            return healthScore;
+        }
+        catch (error) {
+            Logger.error(`Error calculating health score for ${workerName}`, error);
+            throw error;
+        }
+    },
+    /**
+     * Get recent health scores
+     */
+    async getHealthHistory(workerName, hours = 24) {
+        if (!redisClient) {
+            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
+        }
+        try {
+            const key = getHealthKey(workerName);
+            const now = Date.now();
+            const startTime = now - hours * 60 * 60 * 1000;
+            const results = await redisClient.zrangebyscore(key, startTime, now);
+            return results.map((data) => JSON.parse(data));
+        }
+        catch (error) {
+            Logger.error(`Error retrieving health history for ${workerName}`, error);
+            return [];
+        }
+    },
+    /**
+     * Get latest health score
+     */
+    async getLatestHealth(workerName) {
+        if (!redisClient) {
+            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
+        }
+        try {
+            const key = getHealthKey(workerName);
+            // Get the most recent entry
+            const results = await redisClient.zrevrange(key, 0, 0);
+            if (results.length === 0) {
+                return null;
+            }
+            return JSON.parse(results[0]);
+        }
+        catch (error) {
+            Logger.error(`Error retrieving latest health for ${workerName}`, error);
+            return null;
+        }
+    },
+    /**
+     * Get metrics summary for all workers
+     */
+    async getAllWorkersSummary() {
+        if (!redisClient) {
+            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
+        }
+        try {
+            // Find all unique worker names from health keys
+            const pattern = `${HEALTH_PREFIX}*`;
+            const keys = await redisClient.keys(pattern);
+            const workerNames = keys.map((key) => key.replace(HEALTH_PREFIX, ''));
+            const summaries = await Promise.all(workerNames.map(async (workerName) => {
+                const now = new Date();
+                const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);
+                const [health, processed, errors] = await Promise.all([
+                    WorkerMetrics.getLatestHealth(workerName),
+                    WorkerMetrics.aggregate({
+                        workerName,
+                        metricType: 'processed',
+                        granularity: 'hourly',
+                        startDate: oneHourAgo,
+                        endDate: now,
+                    }),
+                    WorkerMetrics.aggregate({
+                        workerName,
+                        metricType: 'errors',
+                        granularity: 'hourly',
+                        startDate: oneHourAgo,
+                        endDate: now,
+                    }),
+                ]);
+                const totalJobs = processed.total + errors.total;
+                const errorRate = totalJobs > 0 ? errors.total / totalJobs : 0;
+                return {
+                    workerName,
+                    health,
+                    metrics: {
+                        processed: processed.total,
+                        errors: errors.total,
+                        errorRate,
+                    },
+                };
+            }));
+            return summaries;
+        }
+        catch (error) {
+            Logger.error('Error retrieving all workers summary', error);
+            return [];
+        }
+    },
+    /**
+     * Delete all metrics for a worker
+     */
+    async deleteWorkerMetrics(workerName) {
+        if (!redisClient) {
+            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
+        }
+        try {
+            const pattern = `${METRICS_PREFIX}${workerName}:*`;
+            const keys = await redisClient.keys(pattern);
+            if (keys.length > 0) {
+                await redisClient.del(...keys);
+            }
+            // Also delete health scores
+            const healthKey = getHealthKey(workerName);
+            await redisClient.del(healthKey);
+            Logger.info(`Deleted all metrics for worker "${workerName}"`);
+        }
+        catch (error) {
+            Logger.error(`Error deleting metrics for worker "${workerName}"`, error);
+            throw error;
+        }
+    },
+    /**
+     * Shutdown and disconnect
+     */
+    async shutdown() {
+        if (!redisClient) {
+            return;
+        }
+        Logger.info('WorkerMetrics shutting down...');
+        await redisClient.quit();
+        redisClient = null;
+        Logger.info('WorkerMetrics shutdown complete');
+    },
+});
+// Graceful shutdown handled by WorkerShutdown

package/dist/WorkerRegistry.d.ts

@@ -0,0 +1,145 @@
+/**
+ * Worker Registry
+ * Central registry for all background workers with lifecycle management
+ * Sealed namespace for immutability
+ */
+import { type WorkerConfig, type WorkerStatus } from '@zintrust/core';
+export type WorkerMetadata = {
+    name: string;
+    status: WorkerStatus;
+    version: string;
+    region: string;
+    queueName: string;
+    concurrency: number;
+    startedAt: Date | null;
+    stoppedAt: Date | null;
+    lastProcessedAt: Date | null;
+    restartCount: number;
+    processedCount: number;
+    errorCount: number;
+    lockKey: string | null;
+    priority: number;
+    memoryUsage: number;
+    cpuUsage: number;
+    circuitState: 'closed' | 'open' | 'half-open';
+    queues: ReadonlyArray<string>;
+    plugins: ReadonlyArray<string>;
+    datacenter: string;
+    canaryPercentage: number;
+    config: Partial<WorkerConfig>;
+};
+export type WorkerInstance = {
+    metadata: WorkerMetadata;
+    instance: unknown;
+    start: () => void;
+    stop: () => Promise<void>;
+    drain: () => Promise<void>;
+    sleep: () => Promise<void>;
+    wakeup: () => void;
+    getStatus: () => WorkerStatus;
+    getHealth: () => 'green' | 'yellow' | 'red';
+};
+export type RegisterWorkerOptions = {
+    name: string;
+    config: Partial<WorkerConfig>;
+    version?: string;
+    region?: string;
+    queues?: ReadonlyArray<string>;
+    factory: () => Promise<WorkerInstance>;
+};
+export type WorkerRegistrySnapshot = {
+    timestamp: Date;
+    totalWorkers: number;
+    runningWorkers: number;
+    stoppedWorkers: number;
+    sleepingWorkers: number;
+    unhealthyWorkers: number;
+    workers: ReadonlyArray<{
+        name: string;
+        status: WorkerStatus;
+        health: 'green' | 'yellow' | 'red';
+        uptime: number | null;
+        processedCount: number;
+        errorCount: number;
+    }>;
+};
+/**
+ * Worker Registry - Sealed namespace
+ */
+export declare const WorkerRegistry: Readonly<{
+    /**
+     * Register a worker with the registry
+     */
+    register(options: RegisterWorkerOptions): void;
+    /**
+     * Start a worker
+     */
+    start(name: string, version?: string): Promise<void>;
+    /**
+     * Stop a worker
+     */
+    stop(name: string): Promise<void>;
+    /**
+     * Restart a worker (stop + start)
+     */
+    restart(name: string): Promise<void>;
+    /**
+     * Sleep a worker (pause processing but keep lock)
+     */
+    sleep(name: string): Promise<void>;
+    /**
+     * Wakeup a worker (resume from sleep)
+     */
+    wakeup(name: string): Promise<void>;
+    /**
+     * Get worker status
+     */
+    status(name: string): WorkerMetadata | null;
+    /**
+     * List all registered workers
+     */
+    list(): ReadonlyArray<string>;
+    /**
+     * List all running workers
+     */
+    listRunning(): ReadonlyArray<string>;
+    /**
+     * Stop all running workers
+     */
+    stopAll(): Promise<void>;
+    /**
+     * Get worker metrics
+     */
+    getMetrics(name: string): Pick<WorkerMetadata, "processedCount" | "errorCount" | "memoryUsage" | "cpuUsage"> | null;
+    /**
+     * Get worker health status
+     */
+    getHealth(name: string): "green" | "yellow" | "red" | null;
+    /**
+     * Get registry snapshot
+     */
+    getSnapshot(): WorkerRegistrySnapshot;
+    /**
+     * Get worker topology (cluster view)
+     */
+    getTopology(): Record<string, {
+        workers: string[];
+        count: number;
+    }>;
+    /**
+     * Unregister a worker and clear its instance
+     */
+    unregister(name: string): void;
+    /**
+     * Check if worker is registered
+     */
+    isRegistered(name: string): boolean;
+    /**
+     * Check if worker is running
+     */
+    isRunning(name: string): boolean;
+    /**
+     * Get worker instance (internal use)
+     */
+    getInstance(name: string): WorkerInstance | null;
+}>;