@zintrust/workers 0.1.28 → 0.1.30

package/src/WorkerInit.ts

@@ -187,7 +187,7 @@ async function initialize(options: IWorkerInitOptions = {}): Promise<void> {
 async function autoStartPersistedWorkers(): Promise<void> {
   // Check if auto-start is enabled globally via environment variable
   Logger.debug('Auto-start check', {
-    envAutoStart: process.env['WORKER_AUTO_START'],
+    envAutoStart: Env.getBool('WORKER_AUTO_START', false),
     configAutoStart: workersConfig.defaultWorker?.autoStart,
   });
 
@@ -204,6 +204,9 @@ async function autoStartPersistedWorkers(): Promise<void> {
     });
 
     const candidates = records.filter((record) => {
+      if (record.activeStatus === false) {
+        return false;
+      }
       // If autoStart is explicitly true, always include
       if (record.autoStart === true) {
         return true;
@@ -232,7 +235,8 @@ async function autoStartPersistedWorkers(): Promise<void> {
           await WorkerFactory.startFromPersisted(record.name);
           return { name: record.name, started: true, skipped: false };
         } catch (error) {
-          Logger.warn(`Auto-start failed for worker ${record.name}`, error as Error);
+          const message = error instanceof Error ? error.message : String(error);
+          Logger.warn(`Auto-start failed for worker ${record.name}: ${message}`);
           return { name: record.name, started: false, skipped: false };
         }
       })
@@ -246,7 +250,8 @@ async function autoStartPersistedWorkers(): Promise<void> {
       skipped: skippedCount,
     });
   } catch (error) {
-    Logger.warn('Auto-start persisted workers failed', error as Error);
+    const message = error instanceof Error ? error.message : String(error);
+    Logger.warn(`Auto-start persisted workers failed: ${message}`);
   }
 }
 

package/src/WorkerMetrics.ts

@@ -12,6 +12,7 @@ import {
   type RedisConfig,
 } from '@zintrust/core';
 import type IORedis from 'ioredis';
+import type { ChainableCommander } from 'ioredis';
 
 export type MetricType =
   | 'processed'
@@ -81,8 +82,39 @@ const RETENTION = {
   monthly: 365 * 24 * 60 * 60, // 1 year
 };
 
+const runInBatches = async <T>(
+  items: ReadonlyArray<T>,
+  handler: (item: T) => Promise<void>,
+  batchSize = 10
+): Promise<void> => {
+  for (let i = 0; i < items.length; i += batchSize) {
+    const batch = items.slice(i, i + batchSize);
+    // Batch processing is intentionally sequential to avoid overwhelming the system
+    // eslint-disable-next-line no-await-in-loop
+    await Promise.all(batch.map((item) => handler(item)));
+  }
+};
+
 // Internal state
 let redisClient: IORedis | null = null;
+let cachedConfig: RedisConfig | null = null;
+let keepLoggin = 0;
+
+/**
+ * Helper: Get valid Redis client
+ */
+const getValidClient = async (): Promise<IORedis> => {
+  if (!cachedConfig) {
+    throw ErrorFactory.createWorkerError('WorkerMetrics not initialized. Call initialize() first.');
+  }
+
+  // If no client, create one
+  if (!redisClient) {
+    redisClient = createRedisConnection(cachedConfig);
+  }
+
+  return redisClient;
+};
 
 /**
  * Helper: Get Redis key for metrics
@@ -213,10 +245,103 @@ const calculateHealthScore = (metrics: {
   };
 };
 
+/**
+ * Helper: Create empty metrics result for error cases
+ */
+const createEmptyMetrics = (
+  options: MetricQueryOptions,
+  defaultStartDate?: Date
+): AggregatedMetrics => ({
+  workerName: options.workerName,
+  metricType: options.metricType,
+  period: {
+    start: options.startDate ?? defaultStartDate ?? new Date(),
+    end: options.endDate ?? new Date(),
+  },
+  total: 0,
+  average: 0,
+  min: 0,
+  max: 0,
+  count: 0,
+});
+
+/**
+ * Helper: Handle uninitialized Redis client
+ */
+const handleUninitializedMetrics = (optionsList: MetricQueryOptions[]): AggregatedMetrics[] => {
+  if (keepLoggin === 0) {
+    keepLoggin = 1;
+    Logger.warn(`[METRICS] WorkerMetrics not initialized globally. Make sure all workers running`);
+  }
+  return optionsList.map((options) => createEmptyMetrics(options));
+};
+
+/**
+ * Helper: Build Redis pipeline for batch metrics query
+ */
+const buildMetricsPipeline = (
+  client: IORedis,
+  optionsList: MetricQueryOptions[]
+): ChainableCommander => {
+  const pipeline = client.pipeline();
+
+  for (const options of optionsList) {
+    const { workerName, metricType, granularity, startDate, endDate, limit = 1000 } = options;
+    const key = getMetricsKey(workerName, metricType, granularity);
+    const minScore = startDate ? startDate.getTime() : '-inf';
+    const maxScore = endDate ? endDate.getTime() : '+inf';
+    pipeline.zrangebyscore(key, minScore, maxScore, 'LIMIT', 0, limit);
+  }
+
+  return pipeline;
+};
+
+/**
+ * Helper: Process batch results and calculate aggregations
+ */
+const processBatchResults = (
+  optionsList: MetricQueryOptions[],
+  results: [Error | null, unknown][]
+): AggregatedMetrics[] => {
+  return optionsList.map((options, index) => {
+    const [err, data] = results[index];
+    if (err) {
+      Logger.error(`Error querying metrics for ${options.workerName}/${options.metricType}`, err);
+      return createEmptyMetrics(options);
+    }
+
+    const points: MetricPoint[] = (data as string[]).map((d) => JSON.parse(d) as MetricPoint);
+
+    if (points.length === 0) {
+      return createEmptyMetrics(options, new Date(0));
+    }
+
+    const values = points.map((p) => p.value);
+    const total = values.reduce((sum, val) => sum + val, 0);
+    const average = total / values.length;
+    const min = Math.min(...values);
+    const max = Math.max(...values);
+
+    return {
+      workerName: options.workerName,
+      metricType: options.metricType,
+      period: {
+        start: points[0].timestamp,
+        end: points.at(-1)?.timestamp ?? new Date(),
+      },
+      total,
+      average,
+      min,
+      max,
+      count: values.length,
+    };
+  });
+};
+
 /**
  * Worker Metrics Manager - Sealed namespace
  */
-export const WorkerMetrics = Object.freeze({
+const WorkerMetrics = Object.freeze({
   /**
    * Initialize the metrics manager with Redis connection
    */
@@ -226,6 +351,7 @@ export const WorkerMetrics = Object.freeze({
       return;
     }
 
+    cachedConfig = config;
     redisClient = createRedisConnection(config);
     Logger.info('WorkerMetrics initialized');
   },
@@ -239,11 +365,7 @@ export const WorkerMetrics = Object.freeze({
     value: number,
     metadata?: Record<string, unknown>
   ): Promise<void> {
-    if (!redisClient) {
-      throw ErrorFactory.createWorkerError(
-        'WorkerMetrics not initialized. Call initialize() first.'
-      );
-    }
+    const client = await getValidClient();
 
     const now = new Date();
 
@@ -265,11 +387,10 @@ export const WorkerMetrics = Object.freeze({
         const score = roundedTimestamp.getTime();
         const data = JSON.stringify(point);
 
-        await redisClient?.zadd(key, score, data);
+        await client.zadd(key, score, data);
 
         // Cleanup old metrics (lightweight: ~1% based on time slice)
-        const client = redisClient;
-        if (client && Date.now() % 100 === 0) {
+        if (Date.now() % 100 === 0) {
           cleanupOldMetrics(client, key, granularity).catch((err) => {
             Logger.error('Failed to cleanup old metrics', err);
           });
@@ -287,17 +408,25 @@ export const WorkerMetrics = Object.freeze({
     workerName: string,
     metrics: Array<{ metricType: MetricType; value: number; metadata?: Record<string, unknown> }>
   ): Promise<void> {
-    await Promise.all(
-      metrics.map(async (m) => WorkerMetrics.record(workerName, m.metricType, m.value, m.metadata))
-    );
+    await runInBatches(metrics, async (m) => {
+      await WorkerMetrics.record(workerName, m.metricType, m.value, m.metadata);
+    });
   },
 
   /**
    * Query metrics for a time range
    */
   async query(options: MetricQueryOptions): Promise<MetricEntry> {
-    if (!redisClient) {
-      throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
+    if (!cachedConfig) {
+      Logger.warn(
+        `[METRICS] WorkerMetrics not initialized for worker: ${options.workerName}. Please start the worker first to enable metrics collection.`
+      );
+      return {
+        workerName: options.workerName,
+        metricType: options.metricType,
+        granularity: options.granularity,
+        points: [],
+      };
     }
 
     const { workerName, metricType, granularity, startDate, endDate, limit = 1000 } = options;
@@ -307,8 +436,9 @@ export const WorkerMetrics = Object.freeze({
     const maxScore = endDate ? endDate.getTime() : '+inf';
 
     try {
+      const client = await getValidClient();
       // Get data from sorted set
-      const results = await redisClient.zrangebyscore(key, minScore, maxScore, 'LIMIT', 0, limit);
+      const results = await client.zrangebyscore(key, minScore, maxScore, 'LIMIT', 0, limit);
 
       const points: MetricPoint[] = results.map((data) => JSON.parse(data) as MetricPoint);
 
@@ -368,20 +498,13 @@ export const WorkerMetrics = Object.freeze({
   },
 
   async aggregateBatch(optionsList: MetricQueryOptions[]): Promise<AggregatedMetrics[]> {
-    if (!redisClient) {
-      throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
+    if (!cachedConfig) {
+      return handleUninitializedMetrics(optionsList);
     }
     if (optionsList.length === 0) return [];
 
-    const pipeline = redisClient.pipeline();
-
-    for (const options of optionsList) {
-      const { workerName, metricType, granularity, startDate, endDate, limit = 1000 } = options;
-      const key = getMetricsKey(workerName, metricType, granularity);
-      const minScore = startDate ? startDate.getTime() : '-inf';
-      const maxScore = endDate ? endDate.getTime() : '+inf';
-      pipeline.zrangebyscore(key, minScore, maxScore, 'LIMIT', 0, limit);
-    }
+    const client = await getValidClient();
+    const pipeline = buildMetricsPipeline(client, optionsList);
 
     const results = await pipeline.exec();
 
@@ -389,66 +512,14 @@ export const WorkerMetrics = Object.freeze({
       throw ErrorFactory.createWorkerError('Failed to execute metrics pipeline');
     }
 
-    return optionsList.map((options, index) => {
-      const [err, data] = results[index];
-      if (err) {
-        Logger.error(`Error querying metrics for ${options.workerName}/${options.metricType}`, err);
-        return {
-          workerName: options.workerName,
-          metricType: options.metricType,
-          period: { start: options.startDate ?? new Date(), end: options.endDate ?? new Date() },
-          total: 0,
-          average: 0,
-          min: 0,
-          max: 0,
-          count: 0,
-        };
-      }
-
-      const points: MetricPoint[] = (data as string[]).map((d) => JSON.parse(d) as MetricPoint);
-
-      if (points.length === 0) {
-        return {
-          workerName: options.workerName,
-          metricType: options.metricType,
-          period: { start: options.startDate ?? new Date(0), end: options.endDate ?? new Date() },
-          total: 0,
-          average: 0,
-          min: 0,
-          max: 0,
-          count: 0,
-        };
-      }
-
-      const values = points.map((p) => p.value);
-      const total = values.reduce((sum, val) => sum + val, 0);
-      const average = total / values.length;
-      const min = Math.min(...values);
-      const max = Math.max(...values);
-
-      return {
-        workerName: options.workerName,
-        metricType: options.metricType,
-        period: {
-          start: points[0].timestamp,
-          end: points.at(-1)?.timestamp ?? new Date(),
-        },
-        total,
-        average,
-        min,
-        max,
-        count: values.length,
-      };
-    });
+    return processBatchResults(optionsList, results);
   },
 
   /**
    * Calculate and store health score
    */
   async calculateHealth(workerName: string): Promise<WorkerHealthScore> {
-    if (!redisClient) {
-      throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
-    }
+    const client = await getValidClient();
 
     const now = new Date();
     const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);
@@ -521,14 +592,14 @@ export const WorkerMetrics = Object.freeze({
       const score = now.getTime();
       const data = JSON.stringify(healthScore);
 
-      await redisClient.zadd(key, score, data);
+      await client.zadd(key, score, data);
 
       // Keep only last 24 hours
       const cutoff = now.getTime() - 24 * 60 * 60 * 1000;
-      await redisClient.zremrangebyscore(key, '-inf', cutoff);
+      await client.zremrangebyscore(key, '-inf', cutoff);
 
       // Set expiry (48 hours)
-      await redisClient.expire(key, 48 * 60 * 60);
+      await client.expire(key, 48 * 60 * 60);
 
       Logger.debug(`Health score for ${workerName}: ${healthScore.score} (${healthScore.status})`);
 
@@ -546,16 +617,13 @@ export const WorkerMetrics = Object.freeze({
     workerName: string,
     hours = 24
   ): Promise<ReadonlyArray<WorkerHealthScore>> {
-    if (!redisClient) {
-      throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
-    }
-
     try {
+      const client = await getValidClient();
       const key = getHealthKey(workerName);
       const now = Date.now();
       const startTime = now - hours * 60 * 60 * 1000;
 
-      const results = await redisClient.zrangebyscore(key, startTime, now);
+      const results = await client.zrangebyscore(key, startTime, now);
 
       return results.map((data) => JSON.parse(data) as WorkerHealthScore);
     } catch (error) {
@@ -568,15 +636,12 @@ export const WorkerMetrics = Object.freeze({
    * Get latest health score
    */
   async getLatestHealth(workerName: string): Promise<WorkerHealthScore | null> {
-    if (!redisClient) {
-      throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
-    }
-
     try {
+      const client = await getValidClient();
       const key = getHealthKey(workerName);
 
       // Get the most recent entry
-      const results = await redisClient.zrevrange(key, 0, 0);
+      const results = await client.zrevrange(key, 0, 0);
 
       if (results.length === 0) {
         return null;
@@ -603,14 +668,12 @@ export const WorkerMetrics = Object.freeze({
       };
     }>
   > {
-    if (!redisClient) {
-      throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
-    }
-
     try {
+      const client = await getValidClient();
+
       // Find all unique worker names from health keys
       const pattern = `${RedisKeys.healthPrefix}*`;
-      const keys = await redisClient.keys(pattern);
+      const keys = await client.keys(pattern);
       const workerNames = keys.map((key) => key.replace(RedisKeys.healthPrefix, ''));
 
       const summaries = await Promise.all(
@@ -662,21 +725,19 @@ export const WorkerMetrics = Object.freeze({
    * Delete all metrics for a worker
    */
   async deleteWorkerMetrics(workerName: string): Promise<void> {
-    if (!redisClient) {
-      throw ErrorFactory.createWorkerError('WorkerMetrics not initialized');
-    }
+    const client = await getValidClient();
 
     try {
       const pattern = `${RedisKeys.metricsPrefix}${workerName}:*`;
-      const keys = await redisClient.keys(pattern);
+      const keys = await client.keys(pattern);
 
       if (keys.length > 0) {
-        await redisClient.del(...keys);
+        await client.del(...keys);
       }
 
       // Also delete health scores
       const healthKey = getHealthKey(workerName);
-      await redisClient.del(healthKey);
+      await client.del(healthKey);
 
       Logger.info(`Deleted all metrics for worker "${workerName}"`);
     } catch (error) {
@@ -695,11 +756,28 @@ export const WorkerMetrics = Object.freeze({
 
     Logger.info('WorkerMetrics shutting down...');
 
-    await redisClient.quit();
+    // Detach client immediately to allow re-initialization
+    const client = redisClient;
     redisClient = null;
 
+    try {
+      // Attempt graceful quit
+      await client.quit();
+    } catch (error) {
+      // If graceful quit fails, force disconnect
+      Logger.warn('WorkerMetrics graceful shutdown failed, forcing disconnect', error);
+      try {
+        client.disconnect();
+      } catch (disconnectError) {
+        Logger.error('WorkerMetrics forced disconnect failed', disconnectError);
+        // Ignore disconnect errors
+      }
+    }
+
     Logger.info('WorkerMetrics shutdown complete');
   },
 });
 
+export { WorkerMetrics };
+
 // Graceful shutdown handled by WorkerShutdown

package/src/WorkerRegistry.ts

@@ -5,6 +5,7 @@
  */
 
 import { ErrorFactory, Logger, type WorkerConfig, type WorkerStatus } from '@zintrust/core';
+import { AnomalyDetection } from './AnomalyDetection';
 
 export type WorkerMetadata = {
   name: string;
@@ -13,6 +14,7 @@ export type WorkerMetadata = {
   region: string;
   queueName: string;
   concurrency: number;
+  activeStatus?: boolean;
   startedAt: Date | null;
   stoppedAt: Date | null;
   lastProcessedAt: Date | null;
@@ -46,6 +48,7 @@ export type WorkerInstance = {
 export type RegisterWorkerOptions = {
   name: string;
   config: Partial<WorkerConfig>;
+  activeStatus?: boolean;
   version?: string;
   region?: string;
   queues?: ReadonlyArray<string>;
@@ -75,6 +78,51 @@ type Rego = { workers: string[]; count: number };
 const workers = new Map<string, WorkerInstance>();
 const registrations = new Map<string, RegisterWorkerOptions>();
 
+// Cleanup configuration
+const STOPPED_WORKER_CLEANUP_DELAY = 5 * 60 * 1000; // 5 minutes
+const cleanupTimers = new Map<string, NodeJS.Timeout>();
+
+/**
+ * Helper: Schedule cleanup of stopped worker
+ */
+const scheduleStoppedWorkerCleanup = (name: string): void => {
+  // Clear existing timer if any
+  const existingTimer = cleanupTimers.get(name);
+  if (existingTimer) {
+    clearTimeout(existingTimer);
+  }
+
+  // Schedule new cleanup with proper cleanup handling
+  // eslint-disable-next-line no-restricted-syntax
+  const timer = setTimeout(() => {
+    try {
+      const instance = workers.get(name);
+      if (instance && instance.metadata.status === 'stopped') {
+        Logger.info(`Auto-cleaning up stopped worker: ${name}`);
+        workers.delete(name);
+        registrations.delete(name);
+      }
+    } catch (error) {
+      Logger.error(`Error during auto-cleanup of worker ${name}`, error);
+    } finally {
+      cleanupTimers.delete(name);
+    }
+  }, STOPPED_WORKER_CLEANUP_DELAY);
+
+  cleanupTimers.set(name, timer);
+};
+
+/**
+ * Helper: Cancel cleanup timer
+ */
+const cancelCleanupTimer = (name: string): void => {
+  const timer = cleanupTimers.get(name);
+  if (timer) {
+    clearTimeout(timer);
+    cleanupTimers.delete(name);
+  }
+};
+
 /**
  * Helper: Calculate uptime in seconds
  */
@@ -127,6 +175,10 @@ export const WorkerRegistry = Object.freeze({
       throw ErrorFactory.createWorkerError(`Worker "${name}" is not registered`);
     }
 
+    if (registration.activeStatus === false) {
+      throw ErrorFactory.createWorkerError(`Worker "${name}" is inactive`);
+    }
+
     if (workers.has(name)) {
       const existing = workers.get(name);
       if (existing?.metadata.status === 'running') {
@@ -143,6 +195,9 @@ export const WorkerRegistry = Object.freeze({
       instance.metadata.status = 'starting';
       instance.metadata.version = version ?? '1.0.0';
 
+      // Cancel any pending cleanup timer when worker restarts
+      cancelCleanupTimer(name);
+
       workers.set(name, instance);
 
       instance.start();
@@ -183,6 +238,11 @@ export const WorkerRegistry = Object.freeze({
       instance.metadata.status = 'stopped';
       instance.metadata.stoppedAt = new Date();
 
+      AnomalyDetection.cleanup(name);
+
+      // Schedule automatic cleanup for stopped worker
+      scheduleStoppedWorkerCleanup(name);
+
       Logger.info(`Worker "${name}" stopped successfully`);
     } catch (error) {
       Logger.error(`Failed to stop worker "${name}"`, error);
@@ -280,7 +340,21 @@ export const WorkerRegistry = Object.freeze({
    * List all registered workers
    */
   list(): ReadonlyArray<string> {
-    return Array.from(registrations.keys());
+    const names: string[] = [];
+    for (const [name, registration] of registrations.entries()) {
+      if (registration.activeStatus === false) continue;
+      names.push(name);
+    }
+    return names;
+  },
+
+  /**
+   * Update active status for a registered worker
+   */
+  setActiveStatus(name: string, activeStatus: boolean): void {
+    const registration = registrations.get(name);
+    if (!registration) return;
+    registrations.set(name, { ...registration, activeStatus });
   },
 
   /**
@@ -413,9 +487,14 @@ export const WorkerRegistry = Object.freeze({
       Logger.warn(`Worker "${name}" is still running during unregister`);
     }
 
+    // Cancel any pending cleanup timer
+    cancelCleanupTimer(name);
+
     workers.delete(name);
     registrations.delete(name);
 
+    AnomalyDetection.cleanup(name);
+
     Logger.info(`Worker "${name}" unregistered`);
   },