@archlast/server 0.1.8 → 0.1.9

package/dist/jobs/bullmq-adapter.js

@@ -0,0 +1,688 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BullMQJobQueue = void 0;
+const bullmq_1 = require("bullmq");
+const cuid2_1 = require("@paralleldrive/cuid2");
+const logger_js_1 = require("../logging/logger.js");
+const collector_js_1 = require("../metrics/collector.js");
+const redis_connection_js_1 = require("./redis-connection.js");
+const worker_js_1 = require("./worker.js");
+/**
+ * Converts priority from system format (1-20, lower=higher priority)
+ * to BullMQ format (1-2097152, lower=higher priority)
+ */
+function toBullMQPriority(priority) {
+    // Map 1-20 to 2097152-1 (higher priority = lower number in BullMQ)
+    // 1 -> 2097152 (highest priority in BullMQ)
+    // 20 -> 104857 (lowest priority in BullMQ)
+    const normalizedPriority = Math.max(1, Math.min(20, priority));
+    return Math.floor((21 - normalizedPriority) * 104857.6);
+}
+/**
+ * Converts priority from BullMQ format back to system format
+ */
+function fromBullMQPriority(bullmqPriority) {
+    // Reverse the mapping
+    const systemPriority = 21 - Math.ceil(bullmqPriority / 104857.6);
+    return Math.max(1, Math.min(20, systemPriority));
+}
+/**
+ * BullMQ-based implementation of the JobQueue interface.
+ * Uses BullMQ Queue for job management and a separate DLQ queue for failed jobs.
+ */
+class BullMQJobQueue {
+    queue;
+    dlqQueue;
+    defaultMaxAttempts;
+    backoffBaseMs;
+    queueName;
+    disposed = false;
+    constructor(options) {
+        this.queueName = options.queueName ?? "jobs";
+        this.defaultMaxAttempts = options.defaultMaxAttempts ?? 5;
+        this.backoffBaseMs = options.backoffBaseMs ?? 30_000;
+        const connection = (0, redis_connection_js_1.createBullMQProducerConnection)();
+        this.queue = new bullmq_1.Queue(this.queueName, {
+            connection,
+            defaultJobOptions: {
+                removeOnComplete: false,
+                removeOnFail: false,
+                attempts: this.defaultMaxAttempts,
+                backoff: {
+                    type: "fixed",
+                    delay: this.backoffBaseMs,
+                },
+            },
+        });
+        this.dlqQueue = new bullmq_1.Queue(`${this.queueName}-dlq`, {
+            connection,
+            defaultJobOptions: {
+                removeOnComplete: false,
+                removeOnFail: false,
+            },
+        });
+    }
+    /**
+     * Enqueue a new job
+     */
+    async enqueue(name, payload, options) {
+        const now = Date.now();
+        const runAfterMs = options?.runAfterMs ?? 0;
+        const delay = runAfterMs > 0 ? runAfterMs : undefined;
+        const maxAttempts = options?.maxAttempts ?? this.defaultMaxAttempts;
+        const priority = options?.priority ?? 5;
+        const timeoutMs = options?.timeoutMs ?? 300000;
+        const jobData = {
+            name,
+            payload,
+            originalPriority: priority,
+            attempts: 0,
+            maxAttempts,
+            createdAt: now,
+            dependsOn: options?.dependsOn,
+            timeoutMs,
+        };
+        const bullmqJob = await this.queue.add(name, jobData, {
+            delay,
+            priority: toBullMQPriority(priority),
+            attempts: maxAttempts,
+            backoff: {
+                type: "fixed",
+                delay: this.backoffBaseMs,
+            },
+        });
+        const id = bullmqJob.id ?? (0, cuid2_1.createId)();
+        // Update metrics
+        collector_js_1.jobMetrics.jobsPending.inc({ priority_level: String(priority) });
+        logger_js_1.logger.log({
+            timestamp: Date.now(),
+            level: "info",
+            kind: "system",
+            message: `[BullMQJobQueue] Job enqueued: ${id} (${name})`,
+        });
+        return id;
+    }
+    /**
+     * Claim the next available job for execution.
+     * Note: In BullMQ, workers handle job claiming automatically.
+     * This method is kept for interface compatibility but returns null
+     * as BullMQ workers handle the actual claiming.
+     */
+    async claimNext() {
+        // BullMQ workers handle job claiming automatically
+        // This method exists for interface compatibility
+        // To get a job manually, we'd need to use queue.getNextJob() which requires
+        // a worker token - this is handled by the Worker class
+        logger_js_1.logger.log({
+            timestamp: Date.now(),
+            level: "info",
+            kind: "system",
+            message: `[BullMQJobQueue] claimNext() called - BullMQ workers handle this automatically`,
+        });
+        return null;
+    }
+    /**
+     * Mark a job as completed
+     */
+    async complete(id, jobName) {
+        // In BullMQ, jobs are marked complete by the worker
+        // This method updates metrics and logging for compatibility
+        if (jobName) {
+            collector_js_1.jobMetrics.jobsCompleted.inc({ job_name: jobName, status: "completed" });
+        }
+        collector_js_1.jobMetrics.jobsActive.dec();
+        logger_js_1.logger.log({
+            timestamp: Date.now(),
+            level: "info",
+            kind: "system",
+            message: `[BullMQJobQueue] Job completed: ${id}`,
+        });
+    }
+    /**
+     * Handle job failure - either retry or move to DLQ
+     */
+    async fail(job, error, backoffMs) {
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        const attempts = job.attempts;
+        const maxAttempts = job.maxAttempts;
+        const errorCategory = this.categorizeError(error);
+        if (attempts >= maxAttempts) {
+            // Move to DLQ
+            await this.moveToDLQ(job, error, errorCategory);
+        }
+        else {
+            // Schedule retry - BullMQ handles this automatically
+            // We just log and update metrics
+            const delay = backoffMs ?? this.backoffBaseMs * attempts;
+            collector_js_1.jobMetrics.jobsRetried.inc({ job_name: job.name });
+            collector_js_1.jobMetrics.jobsActive.dec();
+            logger_js_1.logger.log({
+                timestamp: Date.now(),
+                level: "warn",
+                kind: "system",
+                message: `[BullMQJobQueue] Job rescheduled: ${job.id} (attempt ${attempts}/${maxAttempts}, next run in ${delay}ms)`,
+            });
+        }
+    }
+    /**
+     * Move a job to the Dead Letter Queue
+     */
+    async moveToDLQ(job, error, errorCategory) {
+        const now = Date.now();
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        const dlqData = {
+            name: job.name,
+            payload: job.payload,
+            originalPriority: job.priority ?? 5,
+            attempts: job.attempts,
+            maxAttempts: job.maxAttempts,
+            createdAt: job.createdAt,
+            dependsOn: job.dependsOn,
+            timeoutMs: job.timeoutMs ?? 300000,
+            jobId: job.id,
+            originalJobId: job.id,
+            deadLetteredAt: now,
+            errorCategory,
+            diagnostics: this.extractDiagnostics(error),
+            reprocessCount: 0,
+            lastError: errorMsg,
+        };
+        await this.dlqQueue.add(job.name, dlqData);
+        // Update metrics
+        collector_js_1.jobMetrics.jobsFailed.inc({
+            job_name: job.name,
+            error_category: errorCategory,
+        });
+        collector_js_1.jobMetrics.jobsInDLQ.inc({ error_category: errorCategory });
+        collector_js_1.jobMetrics.jobsActive.dec();
+        logger_js_1.logger.log({
+            timestamp: Date.now(),
+            level: "error",
+            kind: "system",
+            message: `[BullMQJobQueue] Job moved to DLQ: ${job.id} (${job.name}) - ${errorCategory}: ${errorMsg}`,
+        });
+    }
+    /**
+     * Categorize error for DLQ classification
+     */
+    categorizeError(error) {
+        if (error instanceof worker_js_1.JobTimeoutError) {
+            return "timeout";
+        }
+        if (error instanceof Error) {
+            const msg = error.message.toLowerCase();
+            if (msg.includes("timeout") || msg.includes("timed out")) {
+                return "timeout";
+            }
+            if (msg.includes("validation") || msg.includes("invalid")) {
+                return "validation";
+            }
+            if (msg.includes("dependency") || msg.includes("depends on")) {
+                return "dependency";
+            }
+        }
+        return "system";
+    }
+    /**
+     * Extract diagnostic information from error
+     */
+    extractDiagnostics(error) {
+        const diagnostics = {
+            timestamp: Date.now(),
+        };
+        if (error instanceof Error) {
+            diagnostics.name = error.name;
+            diagnostics.message = error.message;
+            diagnostics.stack = error.stack;
+        }
+        return diagnostics;
+    }
+    /**
+     * Get a job by ID
+     */
+    async getJob(id) {
+        // BullMQ uses job IDs as numbers internally
+        const job = await this.queue.getJob(id);
+        if (!job)
+            return null;
+        return this.mapBullMQJobToJobRecord(job);
+    }
+    /**
+     * List jobs by status
+     */
+    async listJobs(status) {
+        let jobs = [];
+        switch (status) {
+            case "pending":
+                jobs = await this.queue.getWaiting();
+                break;
+            case "active":
+                jobs = await this.queue.getActive();
+                break;
+            case "completed":
+                jobs = await this.queue.getCompleted();
+                break;
+            case "failed":
+                jobs = await this.queue.getFailed();
+                break;
+            default:
+                // Get all jobs from all states
+                const [waiting, active, completed, failed] = await Promise.all([
+                    this.queue.getWaiting(),
+                    this.queue.getActive(),
+                    this.queue.getCompleted(),
+                    this.queue.getFailed(),
+                ]);
+                jobs = [...waiting, ...active, ...completed, ...failed];
+        }
+        return jobs.map((job) => this.mapBullMQJobToJobRecord(job));
+    }
+    /**
+     * Map BullMQ Job to JobRecord
+     */
+    mapBullMQJobToJobRecord(job) {
+        const data = job.data;
+        const now = Date.now();
+        // Determine status based on BullMQ job state
+        let status = "pending";
+        if (job.failedReason) {
+            status = "failed";
+        }
+        else if (job.returnvalue !== undefined) {
+            status = "completed";
+        }
+        else if (job.processedOn) {
+            status = "active";
+        }
+        return {
+            _id: job.id ?? (0, cuid2_1.createId)(),
+            id: job.id ?? (0, cuid2_1.createId)(),
+            name: data.name,
+            payload: data.payload,
+            status,
+            attempts: job.attemptsMade,
+            maxAttempts: data.maxAttempts ?? this.defaultMaxAttempts,
+            runAfter: job.delay ? data.createdAt + job.delay : data.createdAt,
+            createdAt: data.createdAt,
+            updatedAt: job.processedOn ?? now,
+            lastError: job.failedReason,
+            priority: data.originalPriority,
+            timeoutMs: data.timeoutMs,
+            dependsOn: data.dependsOn,
+        };
+    }
+    // ========== DLQ Management Methods ==========
+    /**
+     * List DLQ jobs with optional filtering and pagination
+     */
+    async getDLQJobs(options) {
+        const { limit = 100, offset = 0, errorCategory, jobName } = options ?? {};
+        // Get all DLQ jobs (BullMQ doesn't support offset/limit natively for getJobs)
+        const jobs = await this.dlqQueue.getJobs(["waiting", "active", "completed", "failed"]);
+        // Map and filter
+        let dlqJobs = jobs.map((job) => this.mapBullMQJobToDLQRecord(job));
+        if (errorCategory) {
+            dlqJobs = dlqJobs.filter((j) => j.errorCategory === errorCategory);
+        }
+        if (jobName) {
+            dlqJobs = dlqJobs.filter((j) => j.name === jobName);
+        }
+        // Sort by deadLetteredAt descending and apply pagination
+        dlqJobs.sort((a, b) => b.deadLetteredAt - a.deadLetteredAt);
+        logger_js_1.logger.log({
+            timestamp: Date.now(),
+            level: "info",
+            kind: "system",
+            message: `[BullMQJobQueue] Listed ${dlqJobs.length} DLQ jobs`,
+        });
+        return dlqJobs.slice(offset, offset + limit);
+    }
+    /**
+     * Map BullMQ Job to JobDLQRecord
+     */
+    mapBullMQJobToDLQRecord(job) {
+        const data = job.data;
+        const now = Date.now();
+        return {
+            _id: job.id ?? (0, cuid2_1.createId)(),
+            id: job.id ?? (0, cuid2_1.createId)(),
+            name: data.name,
+            payload: data.payload,
+            status: "failed",
+            attempts: job.attemptsMade,
+            maxAttempts: data.maxAttempts ?? this.defaultMaxAttempts,
+            runAfter: data.createdAt,
+            createdAt: data.createdAt,
+            updatedAt: job.processedOn ?? now,
+            lastError: data.lastError ?? job.failedReason,
+            priority: data.originalPriority,
+            timeoutMs: data.timeoutMs,
+            dependsOn: data.dependsOn,
+            originalJobId: data.originalJobId ?? job.id ?? (0, cuid2_1.createId)(),
+            deadLetteredAt: data.deadLetteredAt ?? now,
+            errorCategory: data.errorCategory ?? "system",
+            diagnostics: data.diagnostics ?? {},
+            reprocessCount: data.reprocessCount ?? 0,
+        };
+    }
+    /**
+     * Get specific DLQ job by ID
+     */
+    async getDLQJob(id) {
+        const job = await this.dlqQueue.getJob(id);
+        if (!job)
+            return null;
+        return this.mapBullMQJobToDLQRecord(job);
+    }
+    /**
+     * Get DLQ statistics
+     */
+    async getDLQStats() {
+        const jobs = await this.dlqQueue.getJobs(["waiting", "active", "completed", "failed"]);
+        // Map server error categories to client-expected categories
+        const categoryMapping = {
+            timeout: "timeout",
+            validation: "validation",
+            dependency: "unknown",
+            system: "execution",
+        };
+        const byErrorCategory = {};
+        const byStatus = {};
+        const byJobName = {};
+        const now = Date.now();
+        let oldestFailure = null;
+        let recentFailures24h = 0;
+        const twentyFourHoursAgo = now - 24 * 60 * 60 * 1000;
+        for (const job of jobs) {
+            const data = job.data;
+            // Map and count by error category
+            const rawCategory = data.errorCategory ?? "system";
+            const mappedCategory = categoryMapping[rawCategory] ?? "unknown";
+            byErrorCategory[mappedCategory] = (byErrorCategory[mappedCategory] ?? 0) + 1;
+            // Count by status (based on reprocessCount)
+            const status = (data.reprocessCount ?? 0) > 0 ? "retried" : "pending";
+            byStatus[status] = (byStatus[status] ?? 0) + 1;
+            // Count by job name
+            const name = data.name ?? "unnamed";
+            byJobName[name] = (byJobName[name] ?? 0) + 1;
+            // Track oldest failure timestamp
+            const deadLetteredAt = data.deadLetteredAt ?? now;
+            if (oldestFailure === null || deadLetteredAt < oldestFailure) {
+                oldestFailure = deadLetteredAt;
+            }
+            // Count failures in last 24 hours
+            if (deadLetteredAt >= twentyFourHoursAgo) {
+                recentFailures24h++;
+            }
+        }
+        return {
+            totalDLQJobs: jobs.length,
+            byStatus,
+            byErrorCategory,
+            byJobName,
+            oldestFailure,
+            recentFailures24h,
+        };
+    }
+    /**
+     * Reprocess a job from DLQ back to main queue
+     */
+    async reprocessFromDLQ(dlqJobId, options) {
+        // Retrieve DLQ job
+        const dlqJob = await this.getDLQJob(dlqJobId);
+        if (!dlqJob) {
+            throw new Error(`DLQ job not found: ${dlqJobId}`);
+        }
+        // Create new job in main queue with same payload
+        const newJobId = await this.enqueue(dlqJob.name, dlqJob.payload, {
+            maxAttempts: options?.newMaxAttempts ?? dlqJob.maxAttempts,
+            priority: options?.priority ?? dlqJob.priority,
+        });
+        // Update reprocess count in DLQ record
+        const existingJob = await this.dlqQueue.getJob(dlqJobId);
+        if (existingJob) {
+            const data = existingJob.data;
+            data.reprocessCount = (data.reprocessCount ?? 0) + 1;
+            // Note: BullMQ doesn't support updating job data directly,
+            // so we rely on the in-memory modification for subsequent reads
+        }
+        logger_js_1.logger.log({
+            timestamp: Date.now(),
+            level: "info",
+            kind: "system",
+            message: `[BullMQJobQueue] Reprocessed DLQ job ${dlqJobId} -> new job ${newJobId}`,
+        });
+        return newJobId;
+    }
+    /**
+     * Delete specific DLQ job
+     */
+    async deleteFromDLQ(id) {
+        const job = await this.dlqQueue.getJob(id);
+        if (!job) {
+            throw new Error(`DLQ job not found: ${id}`);
+        }
+        await job.remove();
+        logger_js_1.logger.log({
+            timestamp: Date.now(),
+            level: "info",
+            kind: "system",
+            message: `[BullMQJobQueue] Deleted DLQ job: ${id}`,
+        });
+    }
+    /**
+     * Purge DLQ jobs older than specified time
+     */
+    async purgeDLQ(olderThanMs = 30 * 24 * 60 * 60 * 1000) {
+        const now = Date.now();
+        const cutoffTime = now - olderThanMs;
+        // Get all DLQ jobs
+        const jobs = await this.dlqQueue.getJobs(["waiting", "active", "completed", "failed"]);
+        // Filter jobs older than cutoff
+        const toDelete = jobs.filter((job) => {
+            const data = job.data;
+            const deadLetteredAt = data.deadLetteredAt ?? now;
+            return deadLetteredAt < cutoffTime;
+        });
+        // Delete each job
+        for (const job of toDelete) {
+            await job.remove();
+        }
+        logger_js_1.logger.log({
+            timestamp: Date.now(),
+            level: "info",
+            kind: "system",
+            message: `[BullMQJobQueue] Purged ${toDelete.length} DLQ jobs older than ${olderThanMs}ms`,
+        });
+        return toDelete.length;
+    }
+    // ========== Statistics and Priority Methods ==========
+    /**
+     * Get queue statistics
+     */
+    async getQueueStats() {
+        const [waiting, active, completed, failed, dlqJobs] = await Promise.all([
+            this.queue.getWaiting(),
+            this.queue.getActive(),
+            this.queue.getCompleted(),
+            this.queue.getFailed(),
+            this.dlqQueue.getJobs(["waiting", "active", "completed", "failed"]),
+        ]);
+        const priorityDistribution = {};
+        let totalProcessingTime = 0;
+        let completedCount = 0;
+        // Count all jobs by priority
+        for (const job of [...waiting, ...active, ...completed, ...failed]) {
+            const priority = String(job.data.originalPriority ?? 5);
+            priorityDistribution[priority] = (priorityDistribution[priority] || 0) + 1;
+            // Calculate processing time for completed jobs
+            if (job.returnvalue !== undefined && job.processedOn && job.timestamp) {
+                const processingTime = job.processedOn - job.timestamp;
+                totalProcessingTime += processingTime;
+                completedCount++;
+            }
+        }
+        const avgProcessingTime = completedCount > 0 ? totalProcessingTime / completedCount : 0;
+        return {
+            totalJobs: waiting.length + active.length + completed.length + failed.length,
+            pendingJobs: waiting.length,
+            activeJobs: active.length,
+            completedJobs: completed.length,
+            failedJobs: failed.length,
+            dlqJobs: dlqJobs.length,
+            avgProcessingTime,
+            priorityDistribution,
+        };
+    }
+    /**
+     * Get queue depth by priority level
+     */
+    async getQueueDepthByPriority() {
+        const waiting = await this.queue.getWaiting();
+        const depthByPriority = {};
+        for (const job of waiting) {
+            const priority = job.data.originalPriority ?? 5;
+            depthByPriority[priority] = (depthByPriority[priority] || 0) + 1;
+        }
+        // Convert to array and sort by priority (descending - higher priority first)
+        return Object.entries(depthByPriority)
+            .map(([priority, count]) => ({
+            priority: parseInt(priority),
+            count,
+            label: this.getPriorityLabel(parseInt(priority)),
+        }))
+            .sort((a, b) => b.priority - a.priority);
+    }
+    getPriorityLabel(priority) {
+        if (priority >= 20)
+            return "urgent";
+        if (priority >= 10)
+            return "high";
+        if (priority >= 5)
+            return "normal";
+        return "low";
+    }
+    /**
+     * Update job priority
+     * Note: BullMQ doesn't support changing priority after job creation,
+     * so we remove and re-add the job with the new priority.
+     */
+    async updateJobPriority(jobId, priority) {
+        // Validate priority range (1-20)
+        if (priority < 1 || priority > 20) {
+            throw new Error("Priority must be between 1 and 20");
+        }
+        const job = await this.queue.getJob(jobId);
+        if (!job) {
+            throw new Error(`Job not found: ${jobId}`);
+        }
+        // BullMQ doesn't support updating job priority directly
+        // We need to remove and re-add the job
+        const data = job.data;
+        const delay = job.delay > 0 ? job.delay - (Date.now() - job.timestamp) : 0;
+        // Remove old job
+        await job.remove();
+        // Add new job with updated priority
+        await this.queue.add(data.name, { ...data, originalPriority: priority }, {
+            delay: delay > 0 ? delay : undefined,
+            priority: toBullMQPriority(priority),
+            attempts: data.maxAttempts - job.attemptsMade,
+            backoff: {
+                type: "fixed",
+                delay: this.backoffBaseMs,
+            },
+        });
+        logger_js_1.logger.log({
+            timestamp: Date.now(),
+            level: "info",
+            kind: "system",
+            message: `[BullMQJobQueue] Job priority updated: ${jobId} -> ${priority}`,
+        });
+    }
+    // ========== Stuck Job Management ==========
+    /**
+     * Find jobs that are stuck in "active" status (worker terminated)
+     */
+    async findStuckJobs(timeoutMs = 300000) {
+        const now = Date.now();
+        const staleThreshold = now - timeoutMs;
+        const active = await this.queue.getActive();
+        const stuckJobs = active
+            .filter((job) => {
+            // Job is stuck if processedOn is older than threshold
+            return job.processedOn && job.processedOn < staleThreshold;
+        })
+            .map((job) => this.mapBullMQJobToJobRecord(job));
+        return stuckJobs;
+    }
+    /**
+     * Reset a stuck job from "active" to "pending" status
+     */
+    async resetStuckJob(jobId) {
+        const job = await this.queue.getJob(jobId);
+        if (!job) {
+            throw new Error(`Job not found: ${jobId}`);
+        }
+        // In BullMQ, we move the job back to waiting state
+        // Note: This is a workaround since BullMQ doesn't have a direct "reset" method
+        const data = job.data;
+        // Remove the stuck job and re-add it
+        await job.remove();
+        await this.queue.add(data.name, data, {
+            priority: toBullMQPriority(data.originalPriority),
+            attempts: data.maxAttempts - job.attemptsMade,
+            backoff: {
+                type: "fixed",
+                delay: this.backoffBaseMs,
+            },
+        });
+        logger_js_1.logger.log({
+            timestamp: Date.now(),
+            level: "info",
+            kind: "system",
+            message: `[BullMQJobQueue] Reset stuck job: ${jobId} from active to pending`,
+        });
+    }
+    /**
+     * Bulk reset multiple stuck jobs
+     */
+    async resetStuckJobs(jobIds) {
+        let reset = 0;
+        let failed = 0;
+        for (const jobId of jobIds) {
+            try {
+                await this.resetStuckJob(jobId);
+                reset++;
+            }
+            catch (error) {
+                logger_js_1.logger.log({
+                    timestamp: Date.now(),
+                    level: "error",
+                    kind: "system",
+                    message: `[BullMQJobQueue] Failed to reset stuck job ${jobId}: ${error}`,
+                });
+                failed++;
+            }
+        }
+        return { reset, failed };
+    }
+    /**
+     * Dispose of the queue connections
+     */
+    dispose() {
+        if (this.disposed)
+            return;
+        this.queue.close();
+        this.dlqQueue.close();
+        this.disposed = true;
+        logger_js_1.logger.log({
+            timestamp: Date.now(),
+            level: "info",
+            kind: "system",
+            message: `[BullMQJobQueue] Disposed queues`,
+        });
+    }
+}
+exports.BullMQJobQueue = BullMQJobQueue;
+exports.default = BullMQJobQueue;
+//# sourceMappingURL=bullmq-adapter.js.map