@zintrust/workers 0.1.29 → 0.1.30

package/dist/WorkerMetrics.d.ts

@@ -53,7 +53,7 @@ export type WorkerHealthScore = {
 /**
  * Worker Metrics Manager - Sealed namespace
  */
-export declare const WorkerMetrics: Readonly<{
+declare const WorkerMetrics: Readonly<{
     /**
      * Initialize the metrics manager with Redis connection
      */
@@ -112,3 +112,4 @@ export declare const WorkerMetrics: Readonly<{
      */
     shutdown(): Promise<void>;
 }>;
+export { WorkerMetrics };

package/dist/WorkerMetrics.js

@@ -10,8 +10,31 @@ const RETENTION = {
     daily: 30 * 24 * 60 * 60, // 30 days
     monthly: 365 * 24 * 60 * 60, // 1 year
 };
+const runInBatches = async (items, handler, batchSize = 10) => {
+    for (let i = 0; i < items.length; i += batchSize) {
+        const batch = items.slice(i, i + batchSize);
+        // Batch processing is intentionally sequential to avoid overwhelming the system
+        // eslint-disable-next-line no-await-in-loop
+        await Promise.all(batch.map((item) => handler(item)));
+    }
+};
 // Internal state
 let redisClient = null;
+let cachedConfig = null;
+let keepLoggin = 0;
+/**
+ * Helper: Get valid Redis client
+ */
+const getValidClient = async () => {
+    if (!cachedConfig) {
+        throw ErrorFactory.createWorkerError('WorkerMetrics not initialized. Call initialize() first.');
+    }
+    // If no client, create one
+    if (!redisClient) {
+        redisClient = createRedisConnection(cachedConfig);
+    }
+    return redisClient;
+};
 /**
  * Helper: Get Redis key for metrics
  * Uses singleton RedisKeys for consistent key management
@@ -106,10 +129,84 @@ const calculateHealthScore = (metrics) => {
         },
     };
 };
+/**
+ * Helper: Create empty metrics result for error cases
+ */
+const createEmptyMetrics = (options, defaultStartDate) => ({
+    workerName: options.workerName,
+    metricType: options.metricType,
+    period: {
+        start: options.startDate ?? defaultStartDate ?? new Date(),
+        end: options.endDate ?? new Date(),
+    },
+    total: 0,
+    average: 0,
+    min: 0,
+    max: 0,
+    count: 0,
+});
+/**
+ * Helper: Handle uninitialized Redis client
+ */
+const handleUninitializedMetrics = (optionsList) => {
+    if (keepLoggin === 0) {
+        keepLoggin = 1;
+        Logger.warn(`[METRICS] WorkerMetrics not initialized globally. Make sure all workers running`);
+    }
+    return optionsList.map((options) => createEmptyMetrics(options));
+};
+/**
+ * Helper: Build Redis pipeline for batch metrics query
+ */
+const buildMetricsPipeline = (client, optionsList) => {
+    const pipeline = client.pipeline();
+    for (const options of optionsList) {
+        const { workerName, metricType, granularity, startDate, endDate, limit = 1000 } = options;
+        const key = getMetricsKey(workerName, metricType, granularity);
+        const minScore = startDate ? startDate.getTime() : '-inf';
+        const maxScore = endDate ? endDate.getTime() : '+inf';
+        pipeline.zrangebyscore(key, minScore, maxScore, 'LIMIT', 0, limit);
+    }
+    return pipeline;
+};
+/**
+ * Helper: Process batch results and calculate aggregations
+ */
+const processBatchResults = (optionsList, results) => {
+    return optionsList.map((options, index) => {
+        const [err, data] = results[index];
+        if (err) {
+            Logger.error(`Error querying metrics for ${options.workerName}/${options.metricType}`, err);
+            return createEmptyMetrics(options);
+        }
+        const points = data.map((d) => JSON.parse(d));
+        if (points.length === 0) {
+            return createEmptyMetrics(options, new Date(0));
+        }
+        const values = points.map((p) => p.value);
+        const total = values.reduce((sum, val) => sum + val, 0);
+        const average = total / values.length;
+        const min = Math.min(...values);
+        const max = Math.max(...values);
+        return {
+            workerName: options.workerName,
+            metricType: options.metricType,
+            period: {
+                start: points[0].timestamp,
+                end: points.at(-1)?.timestamp ?? new Date(),
+            },
+            total,
+            average,
+            min,
+            max,
+            count: values.length,
+        };
+    });
+};
 /**
  * Worker Metrics Manager - Sealed namespace
  */
-export const WorkerMetrics = Object.freeze({
+const WorkerMetrics = Object.freeze({
     /**
      * Initialize the metrics manager with Redis connection
      */
@@ -118,6 +215,7 @@ export const WorkerMetrics = Object.freeze({
             Logger.warn('WorkerMetrics already initialized');
             return;
         }
+        cachedConfig = config;
         redisClient = createRedisConnection(config);
         Logger.info('WorkerMetrics initialized');
     },
@@ -125,9 +223,7 @@ export const WorkerMetrics = Object.freeze({
      * Record a metric point
      */
     async record(workerName, metricType, value, metadata) {
-        if (!redisClient) {
-            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized. Call initialize() first.');
-        }
+        const client = await getValidClient();
         const now = new Date();
         // Record at all granularities
         const granularities = ['hourly', 'daily', 'monthly'];
@@ -142,10 +238,9 @@ export const WorkerMetrics = Object.freeze({
             // Store in sorted set with timestamp as score
             const score = roundedTimestamp.getTime();
             const data = JSON.stringify(point);
-            await redisClient?.zadd(key, score, data);
+            await client.zadd(key, score, data);
             // Cleanup old metrics (lightweight: ~1% based on time slice)
-            const client = redisClient;
-            if (client && Date.now() % 100 === 0) {
+            if (Date.now() % 100 === 0) {
                 cleanupOldMetrics(client, key, granularity).catch((err) => {
                     Logger.error('Failed to cleanup old metrics', err);
                 });
@@ -157,22 +252,31 @@ export const WorkerMetrics = Object.freeze({
      * Record multiple metrics at once (batch operation)
      */
     async recordBatch(workerName, metrics) {
-        await Promise.all(metrics.map(async (m) => WorkerMetrics.record(workerName, m.metricType, m.value, m.metadata)));
+        await runInBatches(metrics, async (m) => {
+            await WorkerMetrics.record(workerName, m.metricType, m.value, m.metadata);
+        });
     },
     /**
      * Query metrics for a time range
      */
     async query(options) {
-        if (!redisClient) {
-            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
+        if (!cachedConfig) {
+            Logger.warn(`[METRICS] WorkerMetrics not initialized for worker: ${options.workerName}. Please start the worker first to enable metrics collection.`);
+            return {
+                workerName: options.workerName,
+                metricType: options.metricType,
+                granularity: options.granularity,
+                points: [],
+            };
         }
         const { workerName, metricType, granularity, startDate, endDate, limit = 1000 } = options;
         const key = getMetricsKey(workerName, metricType, granularity);
         const minScore = startDate ? startDate.getTime() : '-inf';
         const maxScore = endDate ? endDate.getTime() : '+inf';
         try {
+            const client = await getValidClient();
             // Get data from sorted set
-            const results = await redisClient.zrangebyscore(key, minScore, maxScore, 'LIMIT', 0, limit);
+            const results = await client.zrangebyscore(key, minScore, maxScore, 'LIMIT', 0, limit);
             const points = results.map((data) => JSON.parse(data));
             return {
                 workerName,
@@ -226,78 +330,24 @@ export const WorkerMetrics = Object.freeze({
         };
     },
     async aggregateBatch(optionsList) {
-        if (!redisClient) {
-            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
+        if (!cachedConfig) {
+            return handleUninitializedMetrics(optionsList);
         }
         if (optionsList.length === 0)
             return [];
-        const pipeline = redisClient.pipeline();
-        for (const options of optionsList) {
-            const { workerName, metricType, granularity, startDate, endDate, limit = 1000 } = options;
-            const key = getMetricsKey(workerName, metricType, granularity);
-            const minScore = startDate ? startDate.getTime() : '-inf';
-            const maxScore = endDate ? endDate.getTime() : '+inf';
-            pipeline.zrangebyscore(key, minScore, maxScore, 'LIMIT', 0, limit);
-        }
+        const client = await getValidClient();
+        const pipeline = buildMetricsPipeline(client, optionsList);
         const results = await pipeline.exec();
         if (!results) {
             throw ErrorFactory.createWorkerError('Failed to execute metrics pipeline');
         }
-        return optionsList.map((options, index) => {
-            const [err, data] = results[index];
-            if (err) {
-                Logger.error(`Error querying metrics for ${options.workerName}/${options.metricType}`, err);
-                return {
-                    workerName: options.workerName,
-                    metricType: options.metricType,
-                    period: { start: options.startDate ?? new Date(), end: options.endDate ?? new Date() },
-                    total: 0,
-                    average: 0,
-                    min: 0,
-                    max: 0,
-                    count: 0,
-                };
-            }
-            const points = data.map((d) => JSON.parse(d));
-            if (points.length === 0) {
-                return {
-                    workerName: options.workerName,
-                    metricType: options.metricType,
-                    period: { start: options.startDate ?? new Date(0), end: options.endDate ?? new Date() },
-                    total: 0,
-                    average: 0,
-                    min: 0,
-                    max: 0,
-                    count: 0,
-                };
-            }
-            const values = points.map((p) => p.value);
-            const total = values.reduce((sum, val) => sum + val, 0);
-            const average = total / values.length;
-            const min = Math.min(...values);
-            const max = Math.max(...values);
-            return {
-                workerName: options.workerName,
-                metricType: options.metricType,
-                period: {
-                    start: points[0].timestamp,
-                    end: points.at(-1)?.timestamp ?? new Date(),
-                },
-                total,
-                average,
-                min,
-                max,
-                count: values.length,
-            };
-        });
+        return processBatchResults(optionsList, results);
     },
     /**
      * Calculate and store health score
      */
     async calculateHealth(workerName) {
-        if (!redisClient) {
-            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
-        }
+        const client = await getValidClient();
         const now = new Date();
         const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);
         try {
@@ -363,12 +413,12 @@ export const WorkerMetrics = Object.freeze({
             const key = getHealthKey(workerName);
             const score = now.getTime();
             const data = JSON.stringify(healthScore);
-            await redisClient.zadd(key, score, data);
+            await client.zadd(key, score, data);
             // Keep only last 24 hours
             const cutoff = now.getTime() - 24 * 60 * 60 * 1000;
-            await redisClient.zremrangebyscore(key, '-inf', cutoff);
+            await client.zremrangebyscore(key, '-inf', cutoff);
             // Set expiry (48 hours)
-            await redisClient.expire(key, 48 * 60 * 60);
+            await client.expire(key, 48 * 60 * 60);
             Logger.debug(`Health score for ${workerName}: ${healthScore.score} (${healthScore.status})`);
             return healthScore;
         }
@@ -381,14 +431,12 @@ export const WorkerMetrics = Object.freeze({
      * Get recent health scores
      */
     async getHealthHistory(workerName, hours = 24) {
-        if (!redisClient) {
-            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
-        }
         try {
+            const client = await getValidClient();
             const key = getHealthKey(workerName);
             const now = Date.now();
             const startTime = now - hours * 60 * 60 * 1000;
-            const results = await redisClient.zrangebyscore(key, startTime, now);
+            const results = await client.zrangebyscore(key, startTime, now);
             return results.map((data) => JSON.parse(data));
         }
         catch (error) {
@@ -400,13 +448,11 @@ export const WorkerMetrics = Object.freeze({
      * Get latest health score
      */
     async getLatestHealth(workerName) {
-        if (!redisClient) {
-            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
-        }
         try {
+            const client = await getValidClient();
             const key = getHealthKey(workerName);
             // Get the most recent entry
-            const results = await redisClient.zrevrange(key, 0, 0);
+            const results = await client.zrevrange(key, 0, 0);
             if (results.length === 0) {
                 return null;
             }
@@ -421,13 +467,11 @@ export const WorkerMetrics = Object.freeze({
      * Get metrics summary for all workers
      */
     async getAllWorkersSummary() {
-        if (!redisClient) {
-            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
-        }
         try {
+            const client = await getValidClient();
             // Find all unique worker names from health keys
             const pattern = `${RedisKeys.healthPrefix}*`;
-            const keys = await redisClient.keys(pattern);
+            const keys = await client.keys(pattern);
             const workerNames = keys.map((key) => key.replace(RedisKeys.healthPrefix, ''));
             const summaries = await Promise.all(workerNames.map(async (workerName) => {
                 const now = new Date();
@@ -472,18 +516,16 @@ export const WorkerMetrics = Object.freeze({
      * Delete all metrics for a worker
      */
     async deleteWorkerMetrics(workerName) {
-        if (!redisClient) {
-            throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
-        }
+        const client = await getValidClient();
         try {
             const pattern = `${RedisKeys.metricsPrefix}${workerName}:*`;
-            const keys = await redisClient.keys(pattern);
+            const keys = await client.keys(pattern);
             if (keys.length > 0) {
-                await redisClient.del(...keys);
+                await client.del(...keys);
             }
             // Also delete health scores
             const healthKey = getHealthKey(workerName);
-            await redisClient.del(healthKey);
+            await client.del(healthKey);
             Logger.info(`Deleted all metrics for worker "${workerName}"`);
         }
         catch (error) {
@@ -499,9 +541,26 @@ export const WorkerMetrics = Object.freeze({
             return;
         }
         Logger.info('WorkerMetrics shutting down...');
-        await redisClient.quit();
+        // Detach client immediately to allow re-initialization
+        const client = redisClient;
         redisClient = null;
+        try {
+            // Attempt graceful quit
+            await client.quit();
+        }
+        catch (error) {
+            // If graceful quit fails, force disconnect
+            Logger.warn('WorkerMetrics graceful shutdown failed, forcing disconnect', error);
+            try {
+                client.disconnect();
+            }
+            catch (disconnectError) {
+                Logger.error('WorkerMetrics forced disconnect failed', disconnectError);
+                // Ignore disconnect errors
+            }
+        }
         Logger.info('WorkerMetrics shutdown complete');
     },
 });
+export { WorkerMetrics };
 // Graceful shutdown handled by WorkerShutdown

package/dist/WorkerRegistry.d.ts

@@ -11,6 +11,7 @@ export type WorkerMetadata = {
     region: string;
     queueName: string;
     concurrency: number;
+    activeStatus?: boolean;
     startedAt: Date | null;
     stoppedAt: Date | null;
     lastProcessedAt: Date | null;
@@ -42,6 +43,7 @@ export type WorkerInstance = {
 export type RegisterWorkerOptions = {
     name: string;
     config: Partial<WorkerConfig>;
+    activeStatus?: boolean;
     version?: string;
     region?: string;
     queues?: ReadonlyArray<string>;
@@ -99,6 +101,10 @@ export declare const WorkerRegistry: Readonly<{
      * List all registered workers
      */
     list(): ReadonlyArray<string>;
+    /**
+     * Update active status for a registered worker
+     */
+    setActiveStatus(name: string, activeStatus: boolean): void;
     /**
      * List all running workers
      */

package/dist/WorkerRegistry.js

@@ -4,9 +4,52 @@
  * Sealed namespace for immutability
  */
 import { ErrorFactory, Logger } from '@zintrust/core';
+import { AnomalyDetection } from './AnomalyDetection';
 // Internal storage
 const workers = new Map();
 const registrations = new Map();
+// Cleanup configuration
+const STOPPED_WORKER_CLEANUP_DELAY = 5 * 60 * 1000; // 5 minutes
+const cleanupTimers = new Map();
+/**
+ * Helper: Schedule cleanup of stopped worker
+ */
+const scheduleStoppedWorkerCleanup = (name) => {
+    // Clear existing timer if any
+    const existingTimer = cleanupTimers.get(name);
+    if (existingTimer) {
+        clearTimeout(existingTimer);
+    }
+    // Schedule new cleanup with proper cleanup handling
+    // eslint-disable-next-line no-restricted-syntax
+    const timer = setTimeout(() => {
+        try {
+            const instance = workers.get(name);
+            if (instance && instance.metadata.status === 'stopped') {
+                Logger.info(`Auto-cleaning up stopped worker: ${name}`);
+                workers.delete(name);
+                registrations.delete(name);
+            }
+        }
+        catch (error) {
+            Logger.error(`Error during auto-cleanup of worker ${name}`, error);
+        }
+        finally {
+            cleanupTimers.delete(name);
+        }
+    }, STOPPED_WORKER_CLEANUP_DELAY);
+    cleanupTimers.set(name, timer);
+};
+/**
+ * Helper: Cancel cleanup timer
+ */
+const cancelCleanupTimer = (name) => {
+    const timer = cleanupTimers.get(name);
+    if (timer) {
+        clearTimeout(timer);
+        cleanupTimers.delete(name);
+    }
+};
 /**
  * Helper: Calculate uptime in seconds
  */
@@ -51,6 +94,9 @@ export const WorkerRegistry = Object.freeze({
         if (!registration) {
             throw ErrorFactory.createWorkerError(`Worker "${name}" is not registered`);
         }
+        if (registration.activeStatus === false) {
+            throw ErrorFactory.createWorkerError(`Worker "${name}" is inactive`);
+        }
         if (workers.has(name)) {
             const existing = workers.get(name);
             if (existing?.metadata.status === 'running') {
@@ -64,6 +110,8 @@ export const WorkerRegistry = Object.freeze({
             const instance = await registration.factory();
             instance.metadata.status = 'starting';
             instance.metadata.version = version ?? '1.0.0';
+            // Cancel any pending cleanup timer when worker restarts
+            cancelCleanupTimer(name);
             workers.set(name, instance);
             instance.start();
             instance.metadata.status = 'running';
@@ -96,6 +144,9 @@ export const WorkerRegistry = Object.freeze({
             await instance.stop();
             instance.metadata.status = 'stopped';
             instance.metadata.stoppedAt = new Date();
+            AnomalyDetection.cleanup(name);
+            // Schedule automatic cleanup for stopped worker
+            scheduleStoppedWorkerCleanup(name);
             Logger.info(`Worker "${name}" stopped successfully`);
         }
         catch (error) {
@@ -179,7 +230,22 @@ export const WorkerRegistry = Object.freeze({
      * List all registered workers
      */
     list() {
-        return Array.from(registrations.keys());
+        const names = [];
+        for (const [name, registration] of registrations.entries()) {
+            if (registration.activeStatus === false)
+                continue;
+            names.push(name);
+        }
+        return names;
+    },
+    /**
+     * Update active status for a registered worker
+     */
+    setActiveStatus(name, activeStatus) {
+        const registration = registrations.get(name);
+        if (!registration)
+            return;
+        registrations.set(name, { ...registration, activeStatus });
     },
     /**
      * List all running workers
@@ -293,8 +359,11 @@ export const WorkerRegistry = Object.freeze({
         if (instance?.metadata.status === 'running') {
             Logger.warn(`Worker "${name}" is still running during unregister`);
         }
+        // Cancel any pending cleanup timer
+        cancelCleanupTimer(name);
         workers.delete(name);
         registrations.delete(name);
+        AnomalyDetection.cleanup(name);
         Logger.info(`Worker "${name}" unregistered`);
     },
     /**

package/dist/WorkerShutdown.d.ts

@@ -24,6 +24,11 @@ interface IShutdownState {
     startedAt: Date | null;
     reason: string | null;
 }
+type DurableShutdownState = {
+    shuttingDown: boolean;
+    startedAt?: string;
+    reason?: string;
+};
 /**
  * Perform graceful shutdown of all worker modules
  */
@@ -32,6 +37,10 @@ declare function shutdown(options?: IShutdownOptions): Promise<void>;
  * Register process signal handlers for graceful shutdown
  */
 declare function registerShutdownHandlers(): void;
+/**
+ * Unregister process signal handlers (for hot reload/testing)
+ */
+declare function unregisterShutdownHandlers(): void;
 /**
  * Check if system is currently shutting down
  */
@@ -49,6 +58,10 @@ export declare const WorkerShutdown: Readonly<{
      * Register process signal handlers for graceful shutdown
      */
     registerShutdownHandlers: typeof registerShutdownHandlers;
+    /**
+     * Unregister process signal handlers (for hot reload/testing)
+     */
+    unregisterShutdownHandlers: typeof unregisterShutdownHandlers;
     /**
      * Check if system is currently shutting down
      */
@@ -57,5 +70,13 @@ export declare const WorkerShutdown: Readonly<{
      * Get current shutdown state
      */
     getShutdownState: typeof getShutdownState;
+    /**
+     * Request shutdown via Durable Object (Workers)
+     */
+    requestDurableShutdown: (reason?: string) => Promise<boolean>;
+    /**
+     * Read shutdown state from Durable Object (Workers)
+     */
+    getDurableShutdownState: () => Promise<DurableShutdownState | null>;
 }>;
 export {};