@morojs/moro 1.6.6 → 1.7.0

package/dist/core/jobs/job-executor.js

@@ -0,0 +1,418 @@
+// Production-grade Job Executor
+// Handles job execution with retry logic, timeout, circuit breaker, and memory monitoring
+import { EventEmitter } from 'events';
+import { CircuitBreaker } from '../utilities/circuit-breaker.js';
+// Reference to global AbortController
+const AbortControllerClass = globalThis.AbortController;
+/**
+ * JobExecutor - Executes jobs with production-grade resilience
+ * Features:
+ * - Configurable retry with exponential backoff + jitter
+ * - Timeout enforcement
+ * - Circuit breaker integration
+ * - Memory leak detection
+ * - Graceful cancellation
+ */
+export class JobExecutor extends EventEmitter {
+    options;
+    logger;
+    circuitBreakers = new Map();
+    activeExecutions = new Map();
+    isShuttingDown = false;
+    constructor(logger, options = {}) {
+        super();
+        this.logger = logger;
+        // Set defaults
+        this.options = {
+            maxRetries: options.maxRetries ?? 3,
+            retryDelay: options.retryDelay ?? 1000,
+            retryBackoff: options.retryBackoff ?? 'exponential',
+            retryBackoffMultiplier: options.retryBackoffMultiplier ?? 2,
+            maxRetryDelay: options.maxRetryDelay ?? 60000,
+            timeout: options.timeout ?? 300000, // 5 minutes default
+            enableCircuitBreaker: options.enableCircuitBreaker ?? true,
+            circuitBreakerThreshold: options.circuitBreakerThreshold ?? 5,
+            circuitBreakerResetTimeout: options.circuitBreakerResetTimeout ?? 60000,
+            enableMemoryMonitoring: options.enableMemoryMonitoring ?? true,
+            memoryThreshold: options.memoryThreshold ?? 512, // 512MB default
+        };
+        this.logger.debug('JobExecutor initialized', 'JobExecutor', { options: this.options });
+    }
+    /**
+     * Execute a job with full resilience features
+     */
+    async execute(jobId, executionId, jobFn, context) {
+        if (this.isShuttingDown) {
+            throw new Error('JobExecutor is shutting down');
+        }
+        const ctx = context || {
+            jobId,
+            executionId,
+            attempt: 1,
+            startTime: new Date(),
+        };
+        const startTime = Date.now();
+        const abortController = new AbortControllerClass();
+        this.activeExecutions.set(executionId, abortController);
+        let lastError;
+        let attempts = 0;
+        let circuitBreakerTripped = false;
+        let timedOut = false;
+        try {
+            // Check memory before execution
+            if (this.options.enableMemoryMonitoring) {
+                await this.checkMemoryUsage(jobId);
+            }
+            // Execute with retries
+            while (attempts <= this.options.maxRetries) {
+                attempts++;
+                ctx.attempt = attempts;
+                try {
+                    // Check circuit breaker
+                    if (this.options.enableCircuitBreaker) {
+                        const breaker = this.getCircuitBreaker(jobId);
+                        if (breaker.isOpen()) {
+                            circuitBreakerTripped = true;
+                            throw new Error(`Circuit breaker open for job ${jobId}`);
+                        }
+                    }
+                    // Execute with timeout
+                    const result = await this.executeWithTimeout(jobFn, ctx, abortController.signal);
+                    // Success - record in circuit breaker
+                    if (this.options.enableCircuitBreaker) {
+                        const breaker = this.getCircuitBreaker(jobId);
+                        breaker.recordSuccess();
+                    }
+                    const duration = Date.now() - startTime;
+                    const memoryUsed = this.getMemoryUsage();
+                    this.logger.debug(`Job executed successfully: ${jobId}`, 'JobExecutor', {
+                        executionId,
+                        attempts,
+                        duration,
+                        memoryUsed,
+                    });
+                    this.emit('execution:success', {
+                        jobId,
+                        executionId,
+                        attempts,
+                        duration,
+                        memoryUsed,
+                    });
+                    return {
+                        success: true,
+                        value: result,
+                        attempts,
+                        duration,
+                        memoryUsed,
+                        circuitBreakerTripped: false,
+                        timedOut: false,
+                    };
+                }
+                catch (error) {
+                    lastError = error instanceof Error ? error : new Error(String(error));
+                    // Check if it's a timeout
+                    if (lastError.name === 'TimeoutError' || lastError.message.includes('timeout')) {
+                        timedOut = true;
+                    }
+                    // Record failure in circuit breaker
+                    if (this.options.enableCircuitBreaker && !circuitBreakerTripped) {
+                        const breaker = this.getCircuitBreaker(jobId);
+                        breaker.recordFailure();
+                    }
+                    this.logger.warn(`Job execution failed: ${jobId} (attempt ${attempts}/${this.options.maxRetries + 1})`, 'JobExecutor', {
+                        executionId,
+                        error: lastError.message,
+                        timedOut,
+                        circuitBreakerTripped,
+                    });
+                    this.emit('execution:retry', {
+                        jobId,
+                        executionId,
+                        attempt: attempts,
+                        maxAttempts: this.options.maxRetries + 1,
+                        error: lastError,
+                        timedOut,
+                    });
+                    // Don't retry if circuit breaker tripped or shutting down
+                    if (circuitBreakerTripped || this.isShuttingDown) {
+                        break;
+                    }
+                    // Calculate retry delay with backoff and jitter
+                    if (attempts <= this.options.maxRetries) {
+                        const delay = this.calculateRetryDelay(attempts);
+                        await this.sleep(delay);
+                    }
+                }
+            }
+            // All retries exhausted
+            const duration = Date.now() - startTime;
+            this.logger.error(`Job execution failed after ${attempts} attempts: ${jobId}`, 'JobExecutor', {
+                executionId,
+                error: lastError?.message,
+                timedOut,
+                circuitBreakerTripped,
+            });
+            this.emit('execution:failed', {
+                jobId,
+                executionId,
+                attempts,
+                duration,
+                error: lastError,
+                timedOut,
+                circuitBreakerTripped,
+            });
+            return {
+                success: false,
+                error: lastError,
+                attempts,
+                duration,
+                circuitBreakerTripped,
+                timedOut,
+            };
+        }
+        finally {
+            this.activeExecutions.delete(executionId);
+            // Force GC if memory threshold exceeded
+            if (this.options.enableMemoryMonitoring) {
+                const memUsage = process.memoryUsage();
+                const heapUsedMB = memUsage.heapUsed / 1024 / 1024;
+                if (heapUsedMB > this.options.memoryThreshold * 0.9 && globalThis.gc) {
+                    this.logger.warn('High memory usage detected, triggering GC', 'JobExecutor', {
+                        heapUsedMB: Math.round(heapUsedMB),
+                        threshold: this.options.memoryThreshold,
+                    });
+                    globalThis.gc();
+                }
+            }
+        }
+    }
+    /**
+     * Execute job function with timeout
+     */
+    async executeWithTimeout(jobFn, context, signal) {
+        return new Promise((resolve, reject) => {
+            // Setup timeout
+            const timeoutId = setTimeout(() => {
+                const error = new Error(`Job execution timeout after ${this.options.timeout}ms`);
+                error.name = 'TimeoutError';
+                reject(error);
+            }, this.options.timeout);
+            // Don't keep process alive for job execution timeout
+            timeoutId.unref();
+            // Setup abort handler
+            const abortHandler = () => {
+                clearTimeout(timeoutId);
+                reject(new Error('Job execution cancelled'));
+            };
+            if (signal.aborted) {
+                clearTimeout(timeoutId);
+                reject(new Error('Job execution cancelled'));
+                return;
+            }
+            signal.addEventListener('abort', abortHandler, { once: true });
+            // Execute job
+            Promise.resolve()
+                .then(() => jobFn(context))
+                .then(result => {
+                clearTimeout(timeoutId);
+                signal.removeEventListener('abort', abortHandler);
+                resolve(result);
+            })
+                .catch(error => {
+                clearTimeout(timeoutId);
+                signal.removeEventListener('abort', abortHandler);
+                reject(error);
+            });
+        });
+    }
+    /**
+     * Calculate retry delay with backoff and jitter
+     */
+    calculateRetryDelay(attempt) {
+        let delay;
+        if (this.options.retryBackoff === 'exponential') {
+            delay = this.options.retryDelay * Math.pow(this.options.retryBackoffMultiplier, attempt - 1);
+        }
+        else {
+            delay = this.options.retryDelay * attempt;
+        }
+        // Cap at max delay
+        delay = Math.min(delay, this.options.maxRetryDelay);
+        // Add jitter (±20%)
+        const jitter = delay * 0.2 * (Math.random() * 2 - 1);
+        delay = Math.max(0, delay + jitter);
+        return Math.floor(delay);
+    }
+    /**
+     * Sleep helper
+     */
+    sleep(ms) {
+        return new Promise(resolve => {
+            const timer = setTimeout(resolve, ms);
+            timer.unref(); // Don't keep process alive
+        });
+    }
+    /**
+     * Get or create circuit breaker for job
+     */
+    getCircuitBreaker(jobId) {
+        let breaker = this.circuitBreakers.get(jobId);
+        if (!breaker) {
+            breaker = new CircuitBreaker({
+                failureThreshold: this.options.circuitBreakerThreshold,
+                resetTimeout: this.options.circuitBreakerResetTimeout,
+            });
+            // Log circuit breaker state changes
+            breaker.on('open', () => {
+                this.logger.error(`Circuit breaker opened for job: ${jobId}`);
+                this.emit('circuit-breaker:open', { jobId });
+            });
+            breaker.on('halfOpen', () => {
+                this.logger.info(`Circuit breaker half-open for job: ${jobId}`);
+                this.emit('circuit-breaker:half-open', { jobId });
+            });
+            breaker.on('closed', () => {
+                this.logger.info(`Circuit breaker closed for job: ${jobId}`);
+                this.emit('circuit-breaker:closed', { jobId });
+            });
+            this.circuitBreakers.set(jobId, breaker);
+        }
+        return breaker;
+    }
+    /**
+     * Check memory usage before execution
+     */
+    async checkMemoryUsage(jobId) {
+        const memUsage = process.memoryUsage();
+        const heapUsedMB = memUsage.heapUsed / 1024 / 1024;
+        if (heapUsedMB > this.options.memoryThreshold) {
+            const error = `Memory threshold exceeded: ${Math.round(heapUsedMB)}MB / ${this.options.memoryThreshold}MB`;
+            this.logger.error(error, 'JobExecutor', { jobId });
+            this.emit('memory:threshold-exceeded', {
+                jobId,
+                heapUsedMB,
+                threshold: this.options.memoryThreshold,
+            });
+            // Try to force GC if available
+            if (globalThis.gc) {
+                this.logger.warn('Forcing garbage collection', 'JobExecutor', { jobId });
+                globalThis.gc();
+                // Check again after GC
+                const newMemUsage = process.memoryUsage();
+                const newHeapUsedMB = newMemUsage.heapUsed / 1024 / 1024;
+                if (newHeapUsedMB > this.options.memoryThreshold) {
+                    throw new Error(error);
+                }
+                this.logger.info('Memory recovered after GC', 'JobExecutor', {
+                    before: Math.round(heapUsedMB),
+                    after: Math.round(newHeapUsedMB),
+                });
+            }
+            else {
+                throw new Error(error);
+            }
+        }
+    }
+    /**
+     * Get current memory usage in MB
+     */
+    getMemoryUsage() {
+        const memUsage = process.memoryUsage();
+        return Math.round(memUsage.heapUsed / 1024 / 1024);
+    }
+    /**
+     * Cancel a running execution
+     */
+    cancelExecution(executionId) {
+        const controller = this.activeExecutions.get(executionId);
+        if (!controller) {
+            return false;
+        }
+        controller.abort();
+        this.logger.info(`Execution cancelled: ${executionId}`);
+        this.emit('execution:cancelled', { executionId });
+        return true;
+    }
+    /**
+     * Cancel all running executions
+     */
+    cancelAllExecutions() {
+        const executionIds = Array.from(this.activeExecutions.keys());
+        this.logger.info(`Cancelling ${executionIds.length} running executions`);
+        for (const executionId of executionIds) {
+            this.cancelExecution(executionId);
+        }
+    }
+    /**
+     * Get active execution count
+     */
+    getActiveExecutionCount() {
+        return this.activeExecutions.size;
+    }
+    /**
+     * Get circuit breaker status for job
+     */
+    getCircuitBreakerStatus(jobId) {
+        const breaker = this.circuitBreakers.get(jobId);
+        if (!breaker) {
+            return { exists: false };
+        }
+        return {
+            exists: true,
+            state: breaker.getState().toLowerCase(),
+            failures: breaker.getFailures(),
+            threshold: this.options.circuitBreakerThreshold,
+        };
+    }
+    /**
+     * Reset circuit breaker for job
+     */
+    resetCircuitBreaker(jobId) {
+        const breaker = this.circuitBreakers.get(jobId);
+        if (!breaker) {
+            return false;
+        }
+        breaker.reset();
+        this.logger.info(`Circuit breaker reset for job: ${jobId}`);
+        return true;
+    }
+    /**
+     * Update executor options at runtime
+     */
+    updateOptions(options) {
+        Object.assign(this.options, options);
+        this.logger.info('JobExecutor options updated', 'JobExecutor', { options });
+    }
+    /**
+     * Get current options
+     */
+    getOptions() {
+        return { ...this.options };
+    }
+    /**
+     * Shutdown and cleanup
+     */
+    async shutdown(gracePeriod = 30000) {
+        this.logger.info('JobExecutor shutting down...', 'JobExecutor', {
+            activeExecutions: this.activeExecutions.size,
+            gracePeriod,
+        });
+        this.isShuttingDown = true;
+        // Wait for active executions to complete or grace period to expire
+        const startTime = Date.now();
+        while (this.activeExecutions.size > 0 && Date.now() - startTime < gracePeriod) {
+            this.logger.debug(`Waiting for ${this.activeExecutions.size} active executions to complete`);
+            await this.sleep(1000);
+        }
+        // Force cancel remaining executions
+        if (this.activeExecutions.size > 0) {
+            this.logger.warn(`Force cancelling ${this.activeExecutions.size} remaining executions`);
+            this.cancelAllExecutions();
+        }
+        // Cleanup circuit breakers
+        this.circuitBreakers.clear();
+        this.removeAllListeners();
+        this.logger.info('JobExecutor shutdown complete');
+    }
+}
+//# sourceMappingURL=job-executor.js.map

package/dist/core/jobs/job-executor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-executor.js","sourceRoot":"","sources":["../../../src/core/jobs/job-executor.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,0FAA0F;AAE1F,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEtC,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAqBjE,sCAAsC;AACtC,MAAM,oBAAoB,GAAG,UAAU,CAAC,eAAe,CAAC;AAuCxD;;;;;;;;GAQG;AACH,MAAM,OAAO,WAAY,SAAQ,YAAY;IACnC,OAAO,CAA+B;IACtC,MAAM,CAAS;IACf,eAAe,GAAG,IAAI,GAAG,EAA0B,CAAC;IACpD,gBAAgB,GAAG,IAAI,GAAG,EAA2B,CAAC;IACtD,cAAc,GAAG,KAAK,CAAC;IAE/B,YAAY,MAAc,EAAE,UAA8B,EAAE;QAC1D,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,eAAe;QACf,IAAI,CAAC,OAAO,GAAG;YACb,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,CAAC;YACnC,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,IAAI;YACtC,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,aAAa;YACnD,sBAAsB,EAAE,OAAO,CAAC,sBAAsB,IAAI,CAAC;YAC3D,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,KAAK;YAC7C,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,MAAM,EAAE,oBAAoB;YACxD,oBAAoB,EAAE,OAAO,CAAC,oBAAoB,IAAI,IAAI;YAC1D,uBAAuB,EAAE,OAAO,CAAC,uBAAuB,IAAI,CAAC;YAC7D,0BAA0B,EAAE,OAAO,CAAC,0BAA0B,IAAI,KAAK;YACvE,sBAAsB,EAAE,OAAO,CAAC,sBAAsB,IAAI,IAAI;YAC9D,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,GAAG,EAAE,gBAAgB;SAClE,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IACzF,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,OAAO,CAClB,KAAa,EACb,WAAmB,EACnB,KAAkB,EAClB,OAA0B;QAE1B,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,GAAG,GAAqB,OAAO,IAAI;YACvC,KAAK;YACL,WAAW;YACX,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,eAAe,GAAG,IAAI,oBAAoB,EAAE,CAAC;QACnD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;QAExD,IAAI,SAA4B,CAAC;QACjC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,qBAAqB,GAAG,KAAK,CAAC;QAClC,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,CAAC;YACH,gCAAgC;YAChC,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;gBACxC,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACrC,CAAC;YAED,uBAAuB;YACvB,OAAO,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;gBAC3C,QAAQ,EAAE,CAAC;gBACX,GAAG,CAAC,OAAO,GAAG,QAAQ,CAAC;gBAEvB,IAAI,CAAC;oBACH,wBAAwB;oBACxB,IAAI,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC;wBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;wBAE9C,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;4BACrB,qBAAqB,GAAG,IAAI,CAAC;4BAC7B,MAAM,IAAI,KAAK,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC;wBAC3D,CAAC;oBACH,CAAC;oBAED,uBAAuB;oBACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,GAAG,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;oBAEjF,sCAAsC;oBACtC,IAAI,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC;wBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;wBAC9C,OAAO,CAAC,aAAa,EAAE,CAAC;oBAC1B,CAAC;oBAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;oBACxC,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;oBAEzC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,KAAK,EAAE,EAAE,aAAa,EAAE;wBACtE,WAAW;wBACX,QAAQ;wBACR,QAAQ;wBACR,UAAU;qBACX,CAAC,CAAC;oBAEH,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;wBAC7B,KAAK;wBACL,WAAW;wBACX,QAAQ;wBACR,QAAQ;wBACR,UAAU;qBACX,CAAC,CAAC;oBAEH,OAAO;wBACL,OAAO,EAAE,IAAI;wBACb,KAAK,EAAE,MAAM;wBACb,QAAQ;wBACR,QAAQ;wBACR,UAAU;wBACV,qBAAqB,EAAE,KAAK;wBAC5B,QAAQ,EAAE,KAAK;qBAChB,CAAC;gBACJ,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;oBAEtE,0BAA0B;oBAC1B,IAAI,SAAS,CAAC,IAAI,KAAK,cAAc,IAAI,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC/E,QAAQ,GAAG,IAAI,CAAC;oBAClB,CAAC;oBAED,oCAAoC;oBACpC,IAAI,IAAI,CAAC,OAAO,CAAC,oBAAoB,IAAI,CAAC,qBAAqB,EAAE,CAAC;wBAChE,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;wBAC9C,OAAO,CAAC,aAAa,EAAE,CAAC;oBAC1B,CAAC;oBAED,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,yBAAyB,KAAK,aAAa,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,CAAC,GAAG,EACrF,aAAa,EACb;wBACE,WAAW;wBACX,KAAK,EAAE,SAAS,CAAC,OAAO;wBACxB,QAAQ;wBACR,qBAAqB;qBACtB,CACF,CAAC;oBAEF,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE;wBAC3B,KAAK;wBACL,WAAW;wBACX,OAAO,EAAE,QAAQ;wBACjB,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,CAAC;wBACxC,KAAK,EAAE,SAAS;wBAChB,QAAQ;qBACT,CAAC,CAAC;oBAEH,0DAA0D;oBAC1D,IAAI,qBAAqB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;wBACjD,MAAM;oBACR,CAAC;oBAED,gDAAgD;oBAChD,IAAI,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;wBACxC,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;wBACjD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC1B,CAAC;gBACH,CAAC;YACH,CAAC;YAED,wBAAwB;YACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAExC,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,8BAA8B,QAAQ,cAAc,KAAK,EAAE,EAC3D,aAAa,EACb;gBACE,WAAW;gBACX,KAAK,EAAE,SAAS,EAAE,OAAO;gBACzB,QAAQ;gBACR,qBAAqB;aACtB,CACF,CAAC;YAEF,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE;gBAC5B,KAAK;gBACL,WAAW;gBACX,QAAQ;gBACR,QAAQ;gBACR,KAAK,EAAE,SAAS;gBAChB,QAAQ;gBACR,qBAAqB;aACtB,CAAC,CAAC;YAEH,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,SAAS;gBAChB,QAAQ;gBACR,QAAQ;gBACR,qBAAqB;gBACrB,QAAQ;aACT,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YAE1C,wCAAwC;YACxC,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;gBACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;gBACvC,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC;gBAEnD,IAAI,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,GAAG,GAAG,IAAI,UAAU,CAAC,EAAE,EAAE,CAAC;oBACrE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2CAA2C,EAAE,aAAa,EAAE;wBAC3E,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;wBAClC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,eAAe;qBACxC,CAAC,CAAC;oBACH,UAAU,CAAC,EAAE,EAAE,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAC9B,KAAkB,EAClB,OAAyB,EACzB,MAAmB;QAEnB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,gBAAgB;YAChB,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;gBAChC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,+BAA+B,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;gBACjF,KAAK,CAAC,IAAI,GAAG,cAAc,CAAC;gBAC5B,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAEzB,qDAAqD;YACrD,SAAS,CAAC,KAAK,EAAE,CAAC;YAElB,sBAAsB;YACtB,MAAM,YAAY,GAAG,GAAG,EAAE;gBACxB,YAAY,CAAC,SAAS,CAAC,CAAC;gBACxB,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC,CAAC;YAC/C,CAAC,CAAC;YAEF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,YAAY,CAAC,SAAS,CAAC,CAAC;gBACxB,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC,CAAC;gBAC7C,OAAO;YACT,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAE/D,cAAc;YACd,OAAO,CAAC,OAAO,EAAE;iBACd,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;iBAC1B,IAAI,CAAC,MAAM,CAAC,EAAE;gBACb,YAAY,CAAC,SAAS,CAAC,CAAC;gBACxB,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;gBAClD,OAAO,CAAC,MAAM,CAAC,CAAC;YAClB,CAAC,CAAC;iBACD,KAAK,CAAC,KAAK,CAAC,EAAE;gBACb,YAAY,CAAC,SAAS,CAAC,CAAC;gBACxB,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;gBAClD,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,OAAe;QACzC,IAAI,KAAa,CAAC;QAElB,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,KAAK,aAAa,EAAE,CAAC;YAChD,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;QAC/F,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC;QAC5C,CAAC;QAED,mBAAmB;QACnB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QAEpD,oBAAoB;QACpB,MAAM,MAAM,GAAG,KAAK,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACrD,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC;QAEpC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;YAC3B,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACtC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,2BAA2B;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,KAAa;QACrC,IAAI,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAE9C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,IAAI,cAAc,CAAC;gBAC3B,gBAAgB,EAAE,IAAI,CAAC,OAAO,CAAC,uBAAuB;gBACtD,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,0BAA0B;aACtD,CAAC,CAAC;YAEH,oCAAoC;YACpC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,KAAK,EAAE,CAAC,CAAC;gBAC9D,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/C,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE;gBAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,KAAK,EAAE,CAAC,CAAC;gBAChE,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;gBACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,KAAK,EAAE,CAAC,CAAC;gBAC7D,IAAI,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,KAAa;QAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC;QAEnD,IAAI,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,8BAA8B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC;YAC3G,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAEnD,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBACrC,KAAK;gBACL,UAAU;gBACV,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,eAAe;aACxC,CAAC,CAAC;YAEH,+BAA+B;YAC/B,IAAI,UAAU,CAAC,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;gBACzE,UAAU,CAAC,EAAE,EAAE,CAAC;gBAEhB,uBAAuB;gBACvB,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,aAAa,GAAG,WAAW,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC;gBAEzD,IAAI,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;oBACjD,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;gBACzB,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE,aAAa,EAAE;oBAC3D,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;oBAC9B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC;iBACjC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,cAAc;QACpB,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QACvC,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;OAEG;IACI,eAAe,CAAC,WAAmB;QACxC,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC1D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,UAAU,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,WAAW,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QAElD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACI,mBAAmB;QACxB,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC;QAE9D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,YAAY,CAAC,MAAM,qBAAqB,CAAC,CAAC;QAEzE,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACvC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED;;OAEG;IACI,uBAAuB;QAC5B,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;IACpC,CAAC;IAED;;OAEG;IACI,uBAAuB,CAAC,KAAa;QAM1C,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEhD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAC3B,CAAC;QAED,OAAO;YACL,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE;YAC/B,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,uBAAuB;SAChD,CAAC;IACJ,CAAC;IAED;;OAEG;IACI,mBAAmB,CAAC,KAAa;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,CAAC,KAAK,EAAE,CAAC;QAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,KAAK,EAAE,CAAC,CAAC;QAC5D,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACI,aAAa,CAAC,OAAoC;QACvD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED;;OAEG;IACI,UAAU;QACf,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,QAAQ,CAAC,cAAsB,KAAK;QAC/C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8BAA8B,EAAE,aAAa,EAAE;YAC9D,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI;YAC5C,WAAW;SACZ,CAAC,CAAC;QAEH,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAE3B,mEAAmE;QACnE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,WAAW,EAAE,CAAC;YAC9E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,gBAAgB,CAAC,IAAI,gCAAgC,CAAC,CAAC;YAC7F,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;QAED,oCAAoC;QACpC,IAAI,IAAI,CAAC,gBAAgB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACnC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,IAAI,CAAC,gBAAgB,CAAC,IAAI,uBAAuB,CAAC,CAAC;YACxF,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC7B,CAAC;QAED,2BAA2B;QAC3B,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAE7B,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IACpD,CAAC;CACF"}

package/dist/core/jobs/job-scheduler.d.ts

@@ -0,0 +1,214 @@
+import { EventEmitter } from 'events';
+import { Logger } from '../../types/logger.js';
+import { JobExecution } from './job-state-manager.js';
+import { LeaderElectionOptions } from './leader-election.js';
+import { JobExecutorOptions, JobFunction } from './job-executor.js';
+export type JobScheduleType = 'cron' | 'interval' | 'oneTime';
+export interface JobSchedule {
+    type: JobScheduleType;
+    cron?: string;
+    interval?: number;
+    at?: Date;
+}
+export interface JobOptions {
+    name?: string;
+    schedule: JobSchedule;
+    enabled?: boolean;
+    priority?: number;
+    timezone?: string;
+    maxConcurrent?: number;
+    timeout?: number;
+    maxRetries?: number;
+    retryDelay?: number;
+    retryBackoff?: 'linear' | 'exponential';
+    enableCircuitBreaker?: boolean;
+    metadata?: Record<string, any>;
+    onStart?: (context: ExecutionContext) => void | Promise<void>;
+    onComplete?: (context: ExecutionContext, result: any) => void | Promise<void>;
+    onError?: (context: ExecutionContext, error: Error) => void | Promise<void>;
+}
+export interface ExecutionContext {
+    jobId: string;
+    executionId: string;
+    attempt: number;
+    startTime: Date;
+    metadata?: Record<string, any>;
+}
+export interface Job {
+    id: string;
+    name: string;
+    schedule: JobSchedule;
+    fn: JobFunction;
+    options: JobOptions;
+    nextRun?: Date;
+    enabled: boolean;
+    priority: number;
+    timer?: NodeJS.Timeout;
+    concurrentExecutions: number;
+    createdAt: Date;
+}
+export interface JobSchedulerOptions {
+    maxConcurrentJobs?: number;
+    enableLeaderElection?: boolean;
+    leaderElection?: LeaderElectionOptions;
+    executor?: JobExecutorOptions;
+    stateManager?: {
+        persistPath?: string;
+        historySize?: number;
+        persistInterval?: number;
+        enableAutoPersist?: boolean;
+        enableRecovery?: boolean;
+    };
+    gracefulShutdownTimeout?: number;
+}
+/**
+ * JobScheduler - Production-grade job scheduling system
+ * Features:
+ * - Cron, interval, and one-time scheduling
+ * - Leader election for distributed systems
+ * - Graceful shutdown with running job completion
+ * - Global and per-job concurrency control
+ * - Priority queue for job execution
+ * - Automatic crash recovery
+ * - Full observability with events and metrics
+ */
+export declare class JobScheduler extends EventEmitter {
+    private jobs;
+    private stateManager;
+    private leaderElection?;
+    private executor;
+    private logger;
+    private loggerContext;
+    private started;
+    private isShuttingDown;
+    private maxConcurrentJobs;
+    private currentConcurrentJobs;
+    private pendingQueue;
+    private gracefulShutdownTimeout;
+    private enableLeaderElection;
+    constructor(logger: Logger, options?: JobSchedulerOptions);
+    /**
+     * Start the job scheduler
+     */
+    start(): Promise<void>;
+    /**
+     * Register a new job
+     */
+    registerJob(name: string, schedule: JobSchedule, fn: JobFunction, options?: Partial<JobOptions>): string;
+    /**
+     * Unregister a job
+     */
+    unregisterJob(jobId: string): boolean;
+    /**
+     * Get job by ID
+     */
+    getJob(jobId: string): Job | undefined;
+    /**
+     * Get all jobs
+     */
+    getAllJobs(): Job[];
+    /**
+     * Enable/disable job
+     */
+    setJobEnabled(jobId: string, enabled: boolean): boolean;
+    /**
+     * Manually trigger job execution
+     */
+    triggerJob(jobId: string, metadata?: Record<string, any>): Promise<ExecutionResult>;
+    /**
+     * Validate schedule configuration
+     */
+    private validateSchedule;
+    /**
+     * Calculate next run time for job
+     */
+    private calculateNextRun;
+    /**
+     * Schedule all jobs
+     */
+    private scheduleAllJobs;
+    /**
+     * Schedule a single job
+     */
+    private scheduleJob;
+    /**
+     * Unschedule a job
+     */
+    private unscheduleJob;
+    /**
+     * Unschedule all jobs
+     */
+    private unscheduleAllJobs;
+    /**
+     * Queue job for execution (with concurrency control)
+     */
+    private queueJobExecution;
+    /**
+     * Execute job asynchronously
+     */
+    private executeJobAsync;
+    /**
+     * Execute job with full lifecycle management
+     */
+    private executeJob;
+    /**
+     * Process pending queue
+     */
+    private processPendingQueue;
+    /**
+     * Check if this instance is the leader
+     */
+    private isLeader;
+    /**
+     * Generate unique job ID
+     */
+    private generateJobId;
+    /**
+     * Generate unique execution ID
+     */
+    private generateExecutionId;
+    /**
+     * Get scheduler stats
+     */
+    getStats(): {
+        totalJobs: number;
+        enabledJobs: number;
+        runningJobs: number;
+        queuedJobs: number;
+        isLeader: boolean;
+        isStarted: boolean;
+    };
+    /**
+     * Get job state
+     */
+    getJobState(jobId: string): import("./job-state-manager.js").JobState | undefined;
+    /**
+     * Get job history
+     */
+    getJobHistory(jobId: string, limit?: number): JobExecution[];
+    /**
+     * Get job metrics
+     */
+    getJobMetrics(jobId: string): {
+        successRate: number;
+        failureRate: number;
+        averageDuration: number;
+        totalExecutions: number;
+        recentFailures: number;
+    } | null;
+    /**
+     * Graceful shutdown
+     */
+    shutdown(): Promise<void>;
+    private sleep;
+}
+interface ExecutionResult {
+    success: boolean;
+    value?: any;
+    error?: Error;
+    attempts: number;
+    duration: number;
+    circuitBreakerTripped?: boolean;
+    timedOut?: boolean;
+}
+export {};