@taicode/common-server 1.0.10 → 1.0.12

package/output/redis-queue/redis-queue-consumer.d.ts

@@ -0,0 +1,102 @@
+import type { RedisQueueConsumerConfig, TaskData, QueueStats } from './types';
+/**
+ * Redis 队列消费者
+ *
+ * 只负责消费任务,不包含入队逻辑
+ *
+ * @template T 任务数据类型
+ *
+ * @example
+ * ```ts
+ * interface EmailTask {
+ *   to: string
+ *   subject: string
+ * }
+ *
+ * const consumer = new RedisQueueConsumer<EmailTask>({
+ *   redisUrl: 'redis://localhost:6379',
+ *   queueKey: 'email-queue',
+ *   concurrency: 5,
+ *   handler: async (data) => {
+ *     await sendEmail(data.to, data.subject)
+ *     return catchIt(() => {})
+ *   }
+ * })
+ *
+ * await consumer.connect() // 自动开始消费
+ *
+ * // 获取统计信息
+ * const stats = await consumer.statistics()
+ *
+ * // 停止消费
+ * consumer.disconnect()
+ * ```
+ */
+export declare class RedisQueueConsumer<T extends TaskData = TaskData> {
+    private consumerRunning;
+    private redis;
+    private consumerInterval;
+    private recoveryInterval;
+    private processingTasks;
+    private readonly config;
+    private readonly failedQueue;
+    private readonly pendingQueue;
+    private readonly processingQueue;
+    private readonly completedQueue;
+    constructor(config: RedisQueueConsumerConfig<string, T>);
+    /**
+     * 连接 Redis 并自动启动消费者
+     */
+    connect(): Promise<void>;
+    /**
+     * 断开 Redis 连接并停止消费者
+     */
+    disconnect(): void;
+    /**
+     * 获取任务详情
+     */
+    private getTask;
+    /**
+     * 更新任务状态并移动到对应队列（原子操作）
+     */
+    private applyStatus;
+    /**
+     * 根据状态获取对应的队列键
+     */
+    private getQueueByStatus;
+    /**
+     * 恢复超时的任务
+     * 检查 processing 队列中的任务，将超时的任务直接标记为失败
+     * 使用 Lua 脚本批量处理以提高性能和原子性
+     */
+    private recoverStalledTasks;
+    /**
+     * 处理单个任务
+     */
+    private processTask;
+    /**
+     * 启动恢复机制（内部方法，自动调用）
+     */
+    private startRecovery;
+    /**
+     * 停止恢复机制（内部方法，自动调用）
+     */
+    private stopRecovery;
+    /**
+     * 启动消费者（内部方法，自动调用）
+     */
+    private startConsumer;
+    /**
+     * 停止消费者（内部方法，自动调用）
+     */
+    private stopConsumer;
+    /**
+     * 获取队列统计信息（O(1) 时间复杂度）
+     */
+    statistics(): Promise<QueueStats>;
+    /**
+     * 健康检查
+     */
+    health(): Promise<boolean>;
+}
+//# sourceMappingURL=redis-queue-consumer.d.ts.map

package/output/redis-queue/redis-queue-consumer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redis-queue-consumer.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-consumer.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,wBAAwB,EAAuB,QAAQ,EAAQ,UAAU,EAAE,MAAM,SAAS,CAAA;AAExG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,qBAAa,kBAAkB,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IAC3D,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAAI;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAOtB;IAGD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAQ;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;gBAE3B,MAAM,EAAE,wBAAwB,CAAC,MAAM,EAAE,CAAC,CAAC;IAgDvD;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAY9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAUlB;;OAEG;YACW,OAAO;IAWrB;;OAEG;YACW,WAAW;IA8BzB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;;;OAIG;YACW,mBAAmB;IA2EjC;;OAEG;YACW,WAAW;IA4EzB;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACH,OAAO,CAAC,aAAa;IA4ErB;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC;IAevC;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAMjC"}

package/output/redis-queue/redis-queue-consumer.js

@@ -0,0 +1,461 @@
+import { createClient } from 'redis';
+import { catchIt } from '@taicode/common-base';
+/**
+ * Redis 队列消费者
+ *
+ * 只负责消费任务,不包含入队逻辑
+ *
+ * @template T 任务数据类型
+ *
+ * @example
+ * ```ts
+ * interface EmailTask {
+ *   to: string
+ *   subject: string
+ * }
+ *
+ * const consumer = new RedisQueueConsumer<EmailTask>({
+ *   redisUrl: 'redis://localhost:6379',
+ *   queueKey: 'email-queue',
+ *   concurrency: 5,
+ *   handler: async (data) => {
+ *     await sendEmail(data.to, data.subject)
+ *     return catchIt(() => {})
+ *   }
+ * })
+ *
+ * await consumer.connect() // 自动开始消费
+ *
+ * // 获取统计信息
+ * const stats = await consumer.statistics()
+ *
+ * // 停止消费
+ * consumer.disconnect()
+ * ```
+ */
+export class RedisQueueConsumer {
+    consumerRunning = false;
+    redis = null;
+    consumerInterval = null;
+    recoveryInterval = null;
+    processingTasks = 0; // 当前正在处理的任务数
+    config;
+    // 不同状态队列的键名
+    failedQueue;
+    pendingQueue;
+    processingQueue;
+    completedQueue;
+    constructor(config) {
+        // 验证必填参数
+        if (!config.redisUrl && !config.redisClient) {
+            throw new Error('[RedisQueueConsumer] Either redisUrl or redisClient is required');
+        }
+        if (config.redisUrl && config.redisClient) {
+            throw new Error('[RedisQueueConsumer] Cannot specify both redisUrl and redisClient');
+        }
+        if (!config.queueKey) {
+            throw new Error('[RedisQueueConsumer] queueKey is required');
+        }
+        if (typeof config.queueKey === 'string' && config.queueKey.length < 6) {
+            throw new Error('[RedisQueueConsumer] queueKey must be at least 6 characters long');
+        }
+        if (!config.handler) {
+            throw new Error('[RedisQueueConsumer] handler is required');
+        }
+        this.config = {
+            queueKey: config.queueKey,
+            redisUrl: config.redisUrl,
+            redisClient: config.redisClient,
+            handler: config.handler,
+            cleanupDelay: config.cleanupDelay ?? 86400, // 24 hours
+            maxRetries: config.maxRetries ?? 3,
+            concurrency: config.concurrency ?? 1,
+            consumerInterval: config.consumerInterval ?? 1000,
+            processingTimeout: config.processingTimeout ?? 60000, // 60 seconds
+        };
+        // 初始化不同状态队列的键名
+        this.failedQueue = `${config.queueKey}:failed`;
+        this.pendingQueue = `${config.queueKey}:pending`;
+        this.completedQueue = `${config.queueKey}:completed`;
+        this.processingQueue = `${config.queueKey}:processing`;
+        // 使用外部客户端或创建新客户端
+        if (config.redisClient) {
+            this.redis = config.redisClient;
+        }
+        else {
+            this.redis = createClient({ url: this.config.redisUrl });
+            // 添加错误处理
+            this.redis.on('error', (err) => {
+                console.error('[RedisQueueConsumer] Redis Client Error:', err);
+            });
+        }
+    }
+    /**
+     * 连接 Redis 并自动启动消费者
+     */
+    async connect() {
+        if (this.redis && !this.redis.isOpen) {
+            await this.redis.connect().catch((error) => {
+                console.error('[RedisQueueConsumer] Failed to connect to Redis:', error);
+                throw error;
+            });
+            // 连接成功后启动消费者和恢复机制
+            this.startConsumer();
+            this.startRecovery();
+        }
+    }
+    /**
+     * 断开 Redis 连接并停止消费者
+     */
+    disconnect() {
+        this.stopConsumer();
+        this.stopRecovery();
+        if (this.redis && this.redis.isOpen) {
+            this.redis.disconnect().catch((error) => {
+                console.error('[RedisQueueConsumer] Failed to disconnect:', error);
+            });
+        }
+    }
+    /**
+     * 获取任务详情
+     */
+    async getTask(taskId) {
+        if (!this.redis)
+            return null;
+        const taskKey = `${this.config.queueKey}:task:${taskId}`;
+        const taskData = await this.redis.get(taskKey);
+        if (!taskData)
+            return null;
+        return JSON.parse(taskData);
+    }
+    /**
+     * 更新任务状态并移动到对应队列（原子操作）
+     */
+    async applyStatus(taskId, oldStatus, newStatus) {
+        if (!this.redis)
+            return;
+        const task = await this.getTask(taskId);
+        if (!task)
+            return;
+        task.status = newStatus;
+        const taskKey = `${this.config.queueKey}:task:${taskId}`;
+        const oldQueue = this.getQueueByStatus(oldStatus);
+        const newQueue = this.getQueueByStatus(newStatus);
+        if (oldQueue !== newQueue) {
+            // 使用 Lua 脚本确保原子性：更新任务 + 移动队列
+            const script = `
+        redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
+        redis.call('LREM', KEYS[2], 0, ARGV[3])
+        redis.call('RPUSH', KEYS[3], ARGV[3])
+        return 1
+      `;
+            await this.redis.eval(script, {
+                keys: [taskKey, oldQueue, newQueue],
+                arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
+            });
+        }
+        else {
+            // 只更新任务数据
+            await this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
+        }
+    }
+    /**
+     * 根据状态获取对应的队列键
+     */
+    getQueueByStatus(status) {
+        switch (status) {
+            case 'pending': return this.pendingQueue;
+            case 'processing': return this.processingQueue;
+            case 'completed': return this.completedQueue;
+            case 'failed': return this.failedQueue;
+            default: return this.pendingQueue;
+        }
+    }
+    /**
+     * 恢复超时的任务
+     * 检查 processing 队列中的任务，将超时的任务直接标记为失败
+     * 使用 Lua 脚本批量处理以提高性能和原子性
+     */
+    async recoverStalledTasks() {
+        if (!this.redis)
+            return;
+        try {
+            const processingTaskIds = await this.redis.lRange(this.processingQueue, 0, -1);
+            if (processingTaskIds.length === 0) {
+                return;
+            }
+            // 使用 Lua 脚本批量恢复超时任务
+            const batchRecoveryScript = `
+        local processingQueue = KEYS[1]
+        local pendingQueue = KEYS[2]
+        local failedQueue = KEYS[3]
+        local queueKeyPrefix = ARGV[1]
+        local ttl = tonumber(ARGV[2])
+        local processingTimeout = tonumber(ARGV[3])
+        local now = tonumber(ARGV[4])
+        
+        local retryCount = 0
+        local failedCount = 0
+        local cleanupCount = 0
+        
+        -- ARGV[5], ARGV[6], ARGV[7]... 是 taskId
+        for i = 5, #ARGV do
+          local taskId = ARGV[i]
+          local taskKey = queueKeyPrefix .. ':task:' .. taskId
+          local taskData = redis.call('GET', taskKey)
+          
+          if not taskData then
+            -- 任务不存在，从队列中清理
+            redis.call('LREM', processingQueue, 0, taskId)
+            cleanupCount = cleanupCount + 1
+          else
+            local task = cjson.decode(taskData)
+            local processingTime = now - (task.processingStartTime or now)
+            
+            -- 检查是否超时
+            if processingTime > processingTimeout then
+              -- 超时直接标记为失败
+              task.status = 'failed'
+              task.processingStartTime = nil
+              
+              redis.call('SETEX', taskKey, ttl, cjson.encode(task))
+              redis.call('LREM', processingQueue, 0, taskId)
+              redis.call('RPUSH', failedQueue, taskId)
+              failedCount = failedCount + 1
+            end
+          end
+        end
+        
+        return {retryCount, failedCount, cleanupCount}
+      `;
+            const result = await this.redis.eval(batchRecoveryScript, {
+                keys: [this.processingQueue, this.pendingQueue, this.failedQueue],
+                arguments: [
+                    this.config.queueKey,
+                    this.config.cleanupDelay.toString(),
+                    this.config.processingTimeout.toString(),
+                    Date.now().toString(),
+                    ...processingTaskIds,
+                ],
+            });
+            const [retryCount, failedCount, cleanupCount] = result;
+            if (failedCount > 0 || cleanupCount > 0) {
+                console.log(`[RedisQueueConsumer] Recovered tasks - timeout failed: ${failedCount}, cleaned: ${cleanupCount}`);
+            }
+        }
+        catch (error) {
+            console.error('[RedisQueueConsumer] Failed to recover stalled tasks:', error);
+        }
+    }
+    /**
+     * 处理单个任务
+     */
+    async processTask(taskId) {
+        this.processingTasks++;
+        try {
+            const task = await this.getTask(taskId);
+            if (!task) {
+                console.warn(`[RedisQueueConsumer] Task not found: ${taskId}`);
+                return;
+            }
+            // 任务应该是 pending 状态(Lua 脚本已确保延迟检查)
+            if (task.status !== 'pending') {
+                console.log(`[RedisQueueConsumer] Task ${taskId} has invalid status (${task.status}), marking as failed`);
+                await this.applyStatus(taskId, task.status, 'failed');
+                return;
+            }
+            try {
+                // 任务已在 processing 队列中(由 Lua 脚本完成),只需更新状态和开始时间
+                task.status = 'processing';
+                task.processingStartTime = Date.now();
+                const taskKey = `${this.config.queueKey}:task:${taskId}`;
+                await this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
+                // 执行任务处理器
+                await this.config.handler(task.data);
+                // 更新状态为完成
+                await this.applyStatus(taskId, 'processing', 'completed');
+                console.log(`[RedisQueueConsumer] Task completed: ${taskId}`);
+            }
+            catch (error) {
+                console.error(`[RedisQueueConsumer] Task failed: ${taskId}`, error);
+                // 检查是否需要重试
+                if (task.retryCount < task.maxRetries) {
+                    task.retryCount++;
+                    task.status = 'pending';
+                    task.processingStartTime = undefined;
+                    const taskKey = `${this.config.queueKey}:task:${taskId}`;
+                    // 使用 Lua 脚本确保原子性
+                    const script = `
+            redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
+            redis.call('LREM', KEYS[2], 0, ARGV[3])
+            redis.call('RPUSH', KEYS[3], ARGV[3])
+            return 1
+          `;
+                    await this.redis.eval(script, {
+                        keys: [taskKey, this.processingQueue, this.pendingQueue],
+                        arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
+                    });
+                    console.log(`[RedisQueueConsumer] Task ${taskId} will retry (${task.retryCount}/${task.maxRetries})`);
+                }
+                else {
+                    task.status = 'failed';
+                    task.processingStartTime = undefined;
+                    const taskKey = `${this.config.queueKey}:task:${taskId}`;
+                    // 使用 Lua 脚本确保原子性
+                    const script = `
+            redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
+            redis.call('LREM', KEYS[2], 0, ARGV[3])
+            redis.call('RPUSH', KEYS[3], ARGV[3])
+            return 1
+          `;
+                    await this.redis.eval(script, {
+                        keys: [taskKey, this.processingQueue, this.failedQueue],
+                        arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
+                    });
+                    console.error(`[RedisQueueConsumer] Task ${taskId} failed after ${task.maxRetries} retries`);
+                }
+            }
+        }
+        finally {
+            this.processingTasks--;
+        }
+    }
+    /**
+     * 启动恢复机制（内部方法，自动调用）
+     */
+    startRecovery() {
+        if (this.recoveryInterval || !this.redis) {
+            return;
+        }
+        // 立即执行一次恢复
+        this.recoverStalledTasks().catch(error => {
+            console.error('[RedisQueueConsumer] Initial recovery error:', error);
+        });
+        // 定期检查（每 10 秒检查一次）
+        this.recoveryInterval = setInterval(() => {
+            this.recoverStalledTasks().catch(error => {
+                console.error('[RedisQueueConsumer] Recovery error:', error);
+            });
+        }, 10000);
+        console.log('[RedisQueueConsumer] Recovery mechanism started');
+    }
+    /**
+     * 停止恢复机制（内部方法，自动调用）
+     */
+    stopRecovery() {
+        if (this.recoveryInterval) {
+            clearInterval(this.recoveryInterval);
+            this.recoveryInterval = null;
+            console.log('[RedisQueueConsumer] Recovery mechanism stopped');
+        }
+    }
+    /**
+     * 启动消费者（内部方法，自动调用）
+     */
+    startConsumer() {
+        if (this.consumerRunning || !this.redis) {
+            return;
+        }
+        this.consumerRunning = true;
+        console.log(`[RedisQueueConsumer] Consumer started with concurrency: ${this.config.concurrency}`);
+        // Lua 脚本: 原子化从 pending 取出任务并移到 processing(只取已到延迟时间的任务)
+        const popAndMoveScript = `
+      local pendingQueue = KEYS[1]
+      local processingQueue = KEYS[2]
+      local queueKeyPrefix = KEYS[3]
+      local count = tonumber(ARGV[1])
+      local currentTime = tonumber(ARGV[2])
+      
+      local taskIds = {}
+      local checkedCount = 0
+      local maxCheck = count * 3  -- 最多检查 count*3 个任务,避免无限循环
+      
+      while #taskIds < count and checkedCount < maxCheck do
+        local taskId = redis.call('LPOP', pendingQueue)
+        if not taskId then
+          break
+        end
+        
+        checkedCount = checkedCount + 1
+        
+        -- 获取任务详情检查延迟时间
+        local taskKey = queueKeyPrefix .. ':task:' .. taskId
+        local taskData = redis.call('GET', taskKey)
+        
+        if taskData then
+          local task = cjson.decode(taskData)
+          local delayUntil = task.delayUntil
+          
+          -- 如果没有延迟或已到延迟时间,则移到 processing
+          if not delayUntil or delayUntil <= currentTime then
+            redis.call('RPUSH', processingQueue, taskId)
+            table.insert(taskIds, taskId)
+          else
+            -- 未到延迟时间,放回队列尾部
+            redis.call('RPUSH', pendingQueue, taskId)
+          end
+        end
+      end
+      
+      return taskIds
+    `;
+        this.consumerInterval = setInterval(async () => {
+            try {
+                // 检查当前是否有可用的并发槽位
+                const availableSlots = this.config.concurrency - this.processingTasks;
+                if (availableSlots <= 0) {
+                    return; // 已达到并发限制，等待下一次轮询
+                }
+                // 使用 Lua 脚本原子化取出任务并移到 processing 队列
+                const taskIds = await this.redis.eval(popAndMoveScript, {
+                    keys: [this.pendingQueue, this.processingQueue, this.config.queueKey],
+                    arguments: [availableSlots.toString(), Date.now().toString()],
+                });
+                // 并发处理所有任务（不等待完成）
+                taskIds.forEach(taskId => {
+                    this.processTask(taskId).catch(error => {
+                        console.error(`[RedisQueueConsumer] Unhandled error in processTask: ${taskId}`, error);
+                    });
+                });
+            }
+            catch (error) {
+                console.error('[RedisQueueConsumer] Consumer error:', error);
+            }
+        }, this.config.consumerInterval);
+    }
+    /**
+     * 停止消费者（内部方法，自动调用）
+     */
+    stopConsumer() {
+        if (this.consumerInterval) {
+            clearInterval(this.consumerInterval);
+            this.consumerInterval = null;
+        }
+        this.consumerRunning = false;
+        console.log('[RedisQueueConsumer] Consumer stopped');
+    }
+    /**
+     * 获取队列统计信息（O(1) 时间复杂度）
+     */
+    async statistics() {
+        if (!this.redis) {
+            return { pending: 0, processing: 0, completed: 0, failed: 0 };
+        }
+        const [pending, processing, completed, failed] = await Promise.all([
+            this.redis.lLen(this.pendingQueue),
+            this.redis.lLen(this.processingQueue),
+            this.redis.lLen(this.completedQueue),
+            this.redis.lLen(this.failedQueue),
+        ]);
+        return { pending, processing, completed, failed };
+    }
+    /**
+     * 健康检查
+     */
+    async health() {
+        if (!this.redis)
+            return false;
+        const result = await catchIt(() => this.redis.ping());
+        return !result.isError() && result.value === 'PONG';
+    }
+}

package/output/redis-queue/redis-queue-consumer.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=redis-queue-consumer.test.d.ts.map

package/output/redis-queue/redis-queue-consumer.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redis-queue-consumer.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-consumer.test.ts"],"names":[],"mappings":""}