@zintrust/workers 0.1.28 → 0.1.30

package/README.md

@@ -647,6 +647,11 @@ registerWorkerRoutes(Router);
 // ... and many more endpoints
 ```
 
+Processor specs can be file paths or URL specs (recommended for production). Remote processors
+must export a named `ZinTrustProcessor` function.
+
+Workers support `activeStatus` to pause without deletion; inactive workers do not auto-start.
+
 See the [API Reference](#api-reference) section for all available endpoints.
 
 ## Lifecycle Management
@@ -705,6 +710,16 @@ ENABLE_HEALTH_CHECKS=true
 
 # Observability
 OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+
+# Processor spec resolver
+REMOTE_PROCESSOR_ALLOWLIST=wk.zintrust.com
+PROCESSOR_FETCH_TIMEOUT=30000
+PROCESSOR_FETCH_MAX_SIZE=524288
+PROCESSOR_FETCH_RETRY_ATTEMPTS=3
+PROCESSOR_FETCH_RETRY_BACKOFF_MS=1000
+PROCESSOR_CACHE_DEFAULT_TTL=3600
+PROCESSOR_CACHE_MAX_TTL=604800
+PROCESSOR_CACHE_MAX_SIZE=52428800
 ```
 
 ## Testing
@@ -844,7 +859,7 @@ MIT © ZinTrust
 
 ## Support
 
-- Documentation: https://docs.zintrust.dev/workers
+- Documentation: https://doc.zintrust.com/workers
 - Issues: https://github.com/ZinTrust/zintrust/issues
 - Discord: https://discord.gg/zintrust
 

package/dist/AnomalyDetection.d.ts

@@ -62,6 +62,10 @@ export declare const AnomalyDetection: Readonly<{
      * Configure anomaly detection for a worker
      */
     configure(config: IAnomalyConfig): void;
+    /**
+     * Cleanup anomaly models for a worker
+     */
+    cleanup(workerName: string): void;
     /**
      * Train baseline model
      */

package/dist/AnomalyDetection.js

@@ -115,6 +115,14 @@ export const AnomalyDetection = Object.freeze({
         configs.set(config.workerName, { ...config });
         Logger.info(`Anomaly detection configured for ${config.workerName}`);
     },
+    /**
+     * Cleanup anomaly models for a worker
+     */
+    cleanup(workerName) {
+        configs.delete(workerName);
+        models.delete(workerName);
+        Logger.debug(`Anomaly detection cleanup completed for ${workerName}`);
+    },
     /**
      * Train baseline model
      */

package/dist/BroadcastWorker.d.ts

@@ -11,11 +11,13 @@ export declare const BroadcastWorker: Readonly<{
         queueName?: string;
         driverName?: string;
         maxItems?: number;
+        maxDurationMs?: number;
     }) => Promise<number>;
     startWorker: (opts?: {
         queueName?: string;
         driverName?: string;
         signal?: AbortSignal;
+        maxDurationMs?: number;
     }) => Promise<number>;
 }>;
 export default BroadcastWorker;

package/dist/CanaryController.js

@@ -103,7 +103,18 @@ const incrementTraffic = (workerName) => {
         }
         // eslint-disable-next-line no-restricted-syntax
         const timer = setTimeout(() => {
-            CanaryController.complete(workerName);
+            try {
+                CanaryController.complete(workerName);
+            }
+            catch (error) {
+                Logger.error('Error during canary completion callback', error);
+            }
+            finally {
+                const current = canaryTimers.get(`${workerName}:complete`);
+                if (current === timer) {
+                    canaryTimers.delete(`${workerName}:complete`);
+                }
+            }
         }, config.monitoringDuration * 1000);
         canaryTimers.set(`${workerName}:complete`, timer);
     }
@@ -116,7 +127,18 @@ const incrementTraffic = (workerName) => {
         }
         // eslint-disable-next-line no-restricted-syntax
         const timer = setTimeout(() => {
-            incrementTraffic(workerName);
+            try {
+                incrementTraffic(workerName);
+            }
+            catch (error) {
+                Logger.error('Error during canary increment callback', error);
+            }
+            finally {
+                const current = canaryTimers.get(workerName);
+                if (current === timer) {
+                    canaryTimers.delete(workerName);
+                }
+            }
         }, config.incrementInterval * 1000);
         canaryTimers.set(workerName, timer);
     }
@@ -124,7 +146,7 @@ const incrementTraffic = (workerName) => {
 const appendHistory = (deployment, entry) => {
     deployment.history.push(entry);
     if (deployment.history.length > MAX_HISTORY) {
-        deployment.history.shift();
+        deployment.history.splice(0, deployment.history.length - MAX_HISTORY);
     }
 };
 /**
@@ -176,7 +198,18 @@ export const CanaryController = Object.freeze({
         }
         // eslint-disable-next-line no-restricted-syntax
         const timer = setTimeout(() => {
-            incrementTraffic(workerName);
+            try {
+                incrementTraffic(workerName);
+            }
+            catch (error) {
+                Logger.error('Error during canary start callback', error);
+            }
+            finally {
+                const current = canaryTimers.get(workerName);
+                if (current === timer) {
+                    canaryTimers.delete(workerName);
+                }
+            }
         }, config.monitoringDuration * 1000);
         canaryTimers.set(workerName, timer);
     },
@@ -212,7 +245,18 @@ export const CanaryController = Object.freeze({
         }
         // eslint-disable-next-line no-restricted-syntax
         const timer = setTimeout(() => {
-            incrementTraffic(workerName);
+            try {
+                incrementTraffic(workerName);
+            }
+            catch (error) {
+                Logger.error('Error during canary resume callback', error);
+            }
+            finally {
+                const current = canaryTimers.get(workerName);
+                if (current === timer) {
+                    canaryTimers.delete(workerName);
+                }
+            }
         }, deployment.config.incrementInterval * 1000);
         canaryTimers.set(workerName, timer);
         Logger.info('Canary deployment resumed', { workerName });

package/dist/ChaosEngineering.js

@@ -9,6 +9,18 @@ import { Observability } from './Observability';
 import { ResourceMonitor } from './ResourceMonitor';
 import { WorkerRegistry } from './WorkerRegistry';
 const experiments = new Map();
+const EXPERIMENT_RETENTION_MS = 24 * 60 * 60 * 1000;
+const cleanupExpiredExperiments = () => {
+    const cutoff = Date.now() - EXPERIMENT_RETENTION_MS;
+    for (const [id, record] of experiments.entries()) {
+        if (record.status.state !== 'completed')
+            continue;
+        const endedAt = record.status.endedAt?.getTime() ?? 0;
+        if (endedAt > 0 && endedAt < cutoff) {
+            experiments.delete(id);
+        }
+    }
+};
 const getTargetWorkers = (config) => {
     const candidates = config.target.workers ?? WorkerRegistry.listRunning();
     if (candidates.length === 0)
@@ -151,6 +163,7 @@ export const ChaosEngineering = Object.freeze({
             id: experimentId,
             duration: record.config.duration,
         });
+        cleanupExpiredExperiments();
     },
     /**
      * Get experiment status

package/dist/ClusterLock.js

@@ -3,9 +3,20 @@
  * Distributed locking using Redis for multi-instance worker coordination
  * Sealed namespace for immutability
  */
-import { ErrorFactory, Logger, createRedisConnection, generateUuid, } from '@zintrust/core';
-// Generate unique instance ID for this process
-const INSTANCE_ID = `worker-${process.pid}-${Date.now()}-${generateUuid()}`;
+import { Cloudflare, ErrorFactory, Logger, createRedisConnection, generateUuid, } from '@zintrust/core';
+let INSTANCE_ID = '';
+const createInstanceId = () => {
+    const workers = Cloudflare.getWorkersEnv() !== null;
+    const pid = typeof process !== 'undefined' && typeof process.pid === 'number' ? process.pid : 0;
+    const prefix = workers ? 'worker-cf' : 'worker';
+    return `${prefix}-${pid}-${Date.now()}-${generateUuid()}`;
+};
+const getInstanceId = () => {
+    if (INSTANCE_ID !== '')
+        return INSTANCE_ID;
+    INSTANCE_ID = createInstanceId();
+    return INSTANCE_ID;
+};
 // Redis key prefixes
 const LOCK_PREFIX = 'worker:lock:';
 const AUDIT_PREFIX = 'worker:audit:lock:';
@@ -50,7 +61,7 @@ const auditLockOperation = async (client, entry) => {
 const extendLockTTL = async (client, lockKey, ttl) => {
     const redisKey = getLockKey(lockKey);
     const value = await client.get(redisKey);
-    if (value === null || value !== INSTANCE_ID) {
+    if (value === null || value !== getInstanceId()) {
         return false; // Lock not held by this instance
     }
     const result = await client.expire(redisKey, ttl);
@@ -80,7 +91,7 @@ const startHeartbeat = (client) => {
                             timestamp: now,
                             operation: 'extend',
                             lockKey,
-                            instanceId: INSTANCE_ID,
+                            instanceId: getInstanceId(),
                             success: true,
                         });
                     }
@@ -122,7 +133,7 @@ export const ClusterLock = Object.freeze({
         }
         redisClient = createRedisConnection(config);
         startHeartbeat(redisClient);
-        Logger.info('ClusterLock initialized', { instanceId: INSTANCE_ID });
+        Logger.info('ClusterLock initialized', { instanceId: getInstanceId() });
     },
     /**
      * Acquire a distributed lock
@@ -136,12 +147,12 @@ export const ClusterLock = Object.freeze({
         const now = new Date();
         try {
             // Try to acquire lock using SET NX EX (set if not exists with expiry)
-            const result = await redisClient.set(redisKey, INSTANCE_ID, 'EX', ttl, 'NX');
+            const result = await redisClient.set(redisKey, getInstanceId(), 'EX', ttl, 'NX');
             const success = result === 'OK';
             if (success) {
                 const lockInfo = {
                     lockKey,
-                    instanceId: INSTANCE_ID,
+                    instanceId: getInstanceId(),
                     acquiredAt: now,
                     expiresAt: new Date(now.getTime() + ttl * 1000),
                     region,
@@ -158,7 +169,7 @@ export const ClusterLock = Object.freeze({
                     timestamp: now,
                     operation: 'acquire',
                     lockKey,
-                    instanceId: INSTANCE_ID,
+                    instanceId: getInstanceId(),
                     userId,
                     success: true,
                 });
@@ -169,7 +180,7 @@ export const ClusterLock = Object.freeze({
                     timestamp: now,
                     operation: 'acquire',
                     lockKey,
-                    instanceId: INSTANCE_ID,
+                    instanceId: getInstanceId(),
                     userId,
                     success: false,
                 });

package/dist/DeadLetterQueue.js

@@ -4,9 +4,13 @@
  * Sealed namespace for immutability
  */
 import { ErrorFactory, Logger, createRedisConnection } from '@zintrust/core';
-// Redis key prefixes
-const DLQ_PREFIX = 'worker:dlq:';
-const AUDIT_PREFIX = 'worker:dlq:audit:';
+// Redis key prefixes - using workers package prefix system
+const getDLQPrefix = () => {
+    return 'worker:dlq:';
+};
+const getAuditPrefix = () => {
+    return 'worker:dlq:audit:';
+};
 // Internal state
 let redisClient = null;
 let retentionPolicy = null;
@@ -15,13 +19,13 @@ let cleanupInterval = null;
  * Helper: Get DLQ key
  */
 const getDLQKey = (queueName) => {
-    return `${DLQ_PREFIX}${queueName}`;
+    return `${getDLQPrefix()}${queueName}`;
 };
 /**
  * Helper: Get audit key
  */
 const getAuditKey = (failedJobId) => {
-    return `${AUDIT_PREFIX}${failedJobId}`;
+    return `${getAuditPrefix()}${failedJobId}`;
 };
 /**
  * Helper: Record audit entry
@@ -104,7 +108,7 @@ const cleanupOldEntries = async () => {
     try {
         const cutoffTimestamp = Date.now() - policy.autoDeleteAfterDays * 24 * 60 * 60 * 1000;
         // Find all DLQ keys
-        const pattern = `${DLQ_PREFIX}*`;
+        const pattern = `${getDLQPrefix()}*`;
         const keys = await client.keys(pattern);
         const cleanedCounts = await Promise.all(keys.map(async (key) => {
             const oldEntries = await client.zrangebyscore(key, '-inf', cutoffTimestamp);
@@ -424,10 +428,10 @@ export const DeadLetterQueue = Object.freeze({
         }
         try {
             const client = redisClient;
-            const pattern = `${DLQ_PREFIX}*`;
+            const pattern = `${getDLQPrefix()}*`;
             const keys = await client.keys(pattern);
             const entriesByQueue = await Promise.all(keys.map(async (key) => {
-                const queueName = key.replace(DLQ_PREFIX, '');
+                const queueName = key.replace(getDLQPrefix(), '');
                 const entries = await client.zrange(key, 0, -1);
                 return {
                     queueName,

package/dist/MultiQueueWorker.d.ts

@@ -37,7 +37,7 @@ export declare const MultiQueueWorker: Readonly<{
     /**
      * Create multi-queue worker
      */
-    create(config: MultiQueueWorkerConfig): void;
+    create(config: MultiQueueWorkerConfig): Promise<void>;
     /**
      * Start processing for a specific queue
      */

package/dist/MultiQueueWorker.js

@@ -11,8 +11,8 @@ const multiQueueWorkers = new Map();
 /**
  * Helper: Create worker for a queue
  */
-const createQueueWorker = (workerName, queueConfig, processor) => {
-    const queue = PriorityQueue.getQueueInstance(queueConfig.name);
+const createQueueWorker = async (workerName, queueConfig, processor) => {
+    const queue = await PriorityQueue.getQueueInstance(queueConfig.name);
     const connection = queue.opts.connection;
     const workerOptions = {
         connection,
@@ -80,7 +80,7 @@ export const MultiQueueWorker = Object.freeze({
     /**
      * Create multi-queue worker
      */
-    create(config) {
+    async create(config) {
         if (multiQueueWorkers.has(config.workerName)) {
             throw ErrorFactory.createWorkerError(`Multi-queue worker "${config.workerName}" already exists`);
         }
@@ -88,9 +88,14 @@ export const MultiQueueWorker = Object.freeze({
         const stats = new Map();
         // Sort queues by priority (higher first)
         const sortedQueues = [...config.queues].sort((a, b) => b.priority - a.priority);
-        // Create workers for each queue
-        for (const queueConfig of sortedQueues) {
-            const worker = createQueueWorker(config.workerName, queueConfig, config.processor);
+        // Create workers for each queue in parallel
+        const workerPromises = sortedQueues.map(async (queueConfig) => {
+            const worker = await createQueueWorker(config.workerName, queueConfig, config.processor);
+            return { queueConfig, worker };
+        });
+        const workerResults = await Promise.all(workerPromises);
+        // Store workers and stats
+        for (const { queueConfig, worker } of workerResults) {
             workers.set(queueConfig.name, worker);
             stats.set(queueConfig.name, initializeQueueStats(queueConfig.name, queueConfig.enabled));
         }
@@ -200,7 +205,7 @@ export const MultiQueueWorker = Object.freeze({
         queueConfig.concurrency = concurrency;
         // Update worker concurrency (requires restart in BullMQ)
         await worker.close();
-        const newWorker = createQueueWorker(workerName, queueConfig, mqw.config.processor);
+        const newWorker = await createQueueWorker(workerName, queueConfig, mqw.config.processor);
         mqw.workers.set(queueName, newWorker);
         Logger.info(`Queue concurrency updated: ${queueName}`, { workerName, concurrency });
     },

package/dist/NotificationWorker.d.ts

@@ -11,11 +11,13 @@ export declare const NotificationWorker: Readonly<{
         queueName?: string;
         driverName?: string;
         maxItems?: number;
+        maxDurationMs?: number;
     }) => Promise<number>;
     startWorker: (opts?: {
         queueName?: string;
         driverName?: string;
         signal?: AbortSignal;
+        maxDurationMs?: number;
     }) => Promise<number>;
 }>;
 export default NotificationWorker;

package/dist/PriorityQueue.d.ts

@@ -81,7 +81,7 @@ export declare const PriorityQueue: Readonly<{
     /**
      * Get all queue names
      */
-    getQueueNames(): string[];
+    getQueueNames(): Promise<string[]>;
     /**
      * Drain queue (remove all jobs)
      */
@@ -105,7 +105,7 @@ export declare const PriorityQueue: Readonly<{
     /**
      * Get queue instance (internal use)
      */
-    getQueueInstance(queueName: string): Queue;
+    getQueueInstance(queueName: string): Promise<Queue>;
     /**
      * Close a queue
      */

package/dist/PriorityQueue.js

@@ -3,7 +3,7 @@
  * BullMQ priority levels with datacenter affinity
  * Sealed namespace for immutability
  */
-import { ErrorFactory, Logger, NodeSingletons } from '@zintrust/core';
+import { ErrorFactory, Logger } from '@zintrust/core';
 // Priority mappings
 const PRIORITY_VALUES = {
     critical: 10,
@@ -11,14 +11,13 @@ const PRIORITY_VALUES = {
     normal: 1,
     low: 0,
 };
-const require = NodeSingletons.module.createRequire(import.meta.url);
 let queueRedisModule;
 let hasWarnedMissingQueueRedis = false;
-const loadQueueRedisModule = () => {
+const loadQueueRedisModule = async () => {
     if (queueRedisModule)
         return queueRedisModule;
     try {
-        queueRedisModule = require('@zintrust/queue-redis');
+        queueRedisModule = (await import('@zintrust/queue-redis'));
         return queueRedisModule;
     }
     catch (error) {
@@ -32,8 +31,8 @@ const loadQueueRedisModule = () => {
 /**
  * Helper: Get or create queue via shared driver
  */
-const getQueue = (queueName) => {
-    const queueRedis = loadQueueRedisModule();
+const getQueue = async (queueName) => {
+    const queueRedis = await loadQueueRedisModule();
     if (!queueRedis) {
         throw ErrorFactory.createWorkerError('Optional package "@zintrust/queue-redis" is required for PriorityQueue. Install it to use queue features.');
     }
@@ -98,7 +97,7 @@ export const PriorityQueue = Object.freeze({
      * Add a job to the queue with priority
      */
     async addJob(queueName, jobName, data, options) {
-        const queue = getQueue(queueName);
+        const queue = await getQueue(queueName);
         const jobOptions = buildJobOptions(options);
         try {
             const job = await queue.add(jobName, data, jobOptions);
@@ -121,7 +120,7 @@ export const PriorityQueue = Object.freeze({
      * Add multiple jobs in bulk
      */
     async addBulk(queueName, jobs) {
-        const queue = getQueue(queueName);
+        const queue = await getQueue(queueName);
         try {
             const bulkJobs = jobs.map((job) => ({
                 name: job.name,
@@ -146,14 +145,14 @@ export const PriorityQueue = Object.freeze({
      * Get job by ID
      */
     async getJob(queueName, jobId) {
-        const queue = getQueue(queueName);
+        const queue = await getQueue(queueName);
         return queue.getJob(jobId);
     },
     /**
      * Remove a job
      */
     async removeJob(queueName, jobId) {
-        const queue = getQueue(queueName);
+        const queue = await getQueue(queueName);
         const job = await queue.getJob(jobId);
         if (job) {
             await job.remove();
@@ -164,7 +163,7 @@ export const PriorityQueue = Object.freeze({
      * Pause a queue
      */
     async pause(queueName) {
-        const queue = getQueue(queueName);
+        const queue = await getQueue(queueName);
         await queue.pause();
         Logger.info(`Paused queue "${queueName}"`);
     },
@@ -172,7 +171,7 @@ export const PriorityQueue = Object.freeze({
      * Resume a queue
      */
     async resume(queueName) {
-        const queue = getQueue(queueName);
+        const queue = await getQueue(queueName);
         await queue.resume();
         Logger.info(`Resumed queue "${queueName}"`);
     },
@@ -180,7 +179,7 @@ export const PriorityQueue = Object.freeze({
      * Get queue information
      */
     async getQueueInfo(queueName) {
-        const queue = getQueue(queueName);
+        const queue = await getQueue(queueName);
         const isPaused = await queue.isPaused();
         const jobCounts = await queue.getJobCounts();
         return {
@@ -199,8 +198,8 @@ export const PriorityQueue = Object.freeze({
     /**
      * Get all queue names
      */
-    getQueueNames() {
-        const queueRedis = loadQueueRedisModule();
+    async getQueueNames() {
+        const queueRedis = await loadQueueRedisModule();
         if (!queueRedis)
             return [];
         return queueRedis.BullMQRedisQueue.getQueueNames();
@@ -209,7 +208,7 @@ export const PriorityQueue = Object.freeze({
      * Drain queue (remove all jobs)
      */
     async drain(queueName, delayed = false) {
-        const queue = getQueue(queueName);
+        const queue = await getQueue(queueName);
         await queue.drain(delayed);
         Logger.info(`Drained queue "${queueName}"`, { delayed });
     },
@@ -217,7 +216,7 @@ export const PriorityQueue = Object.freeze({
      * Clean old jobs from queue
      */
     async clean(queueName, grace, limit, type = 'completed') {
-        const queue = getQueue(queueName);
+        const queue = await getQueue(queueName);
         const jobs = await queue.clean(grace, limit, type);
         Logger.info(`Cleaned ${jobs.length} ${type} jobs from queue "${queueName}"`);
         return jobs;
@@ -226,9 +225,9 @@ export const PriorityQueue = Object.freeze({
      * Obliterate queue (remove all data including queue itself)
      */
     async obliterate(queueName, force = false) {
-        const queue = getQueue(queueName);
+        const queue = await getQueue(queueName);
         await queue.obliterate({ force });
-        const queueRedis = loadQueueRedisModule();
+        const queueRedis = await loadQueueRedisModule();
         if (queueRedis) {
             await queueRedis.BullMQRedisQueue.closeQueue(queueName);
         }
@@ -256,7 +255,7 @@ export const PriorityQueue = Object.freeze({
      * Close a queue
      */
     async closeQueue(queueName) {
-        const queueRedis = loadQueueRedisModule();
+        const queueRedis = await loadQueueRedisModule();
         if (!queueRedis)
             return;
         await queueRedis.BullMQRedisQueue.closeQueue(queueName);
@@ -267,7 +266,7 @@ export const PriorityQueue = Object.freeze({
      */
     async shutdown() {
         Logger.info('PriorityQueue shutting down via BullMQRedisQueue...');
-        const queueRedis = loadQueueRedisModule();
+        const queueRedis = await loadQueueRedisModule();
         if (!queueRedis)
             return;
         await queueRedis.BullMQRedisQueue.shutdown();