@taicode/common-server 1.0.11 → 1.0.12

package/output/redis-queue/batch-consumer.d.ts

@@ -0,0 +1,107 @@
+import type { BatchConsumerConfig, TaskData, QueueStats } from './types';
+/**
+ * Redis 批量队列消费者
+ *
+ * 批量消费任务,每次处理多条
+ *
+ * @template T 任务数据类型
+ *
+ * @example
+ * ```ts
+ * interface EmailTask {
+ *   to: string
+ * }
+ *
+ * const consumer = new RedisBatchConsumer<EmailTask>({
+ *   redisUrl: 'redis://localhost:6379',
+ *   queueKey: 'email-queue',
+ *   batchSize: 50,
+ *   handler: async (dataList) => {
+ *     await sendEmailsBatch(dataList.map(d => d.to))
+ *     return catchIt(() => {})
+ *   }
+ * })
+ *
+ * await consumer.connect() // 自动开始消费
+ *
+ * // 获取统计信息
+ * const stats = await consumer.statistics()
+ *
+ * // 停止消费
+ * consumer.disconnect()
+ * ```
+ */
+export declare class RedisBatchConsumer<T extends TaskData = TaskData> {
+    private consumerRunning;
+    private redis;
+    private consumerInterval;
+    private recoveryInterval;
+    private processingBatches;
+    private readonly config;
+    private readonly failedQueue;
+    private readonly pendingQueue;
+    private readonly processingQueue;
+    private readonly completedQueue;
+    constructor(config: BatchConsumerConfig<T>);
+    /**
+     * 连接 Redis 并自动启动消费者
+     */
+    connect(): Promise<void>;
+    /**
+     * 断开 Redis 连接并停止消费者
+     */
+    disconnect(): void;
+    /**
+     * 获取任务详情
+     */
+    private getTask;
+    /**
+     * 更新任务状态并移动到对应队列（原子操作）
+     */
+    private applyStatus;
+    /**
+     * 批量更新任务状态
+     */
+    private applyStatusBatch;
+    /**
+     * 根据状态获取对应的队列键
+     */
+    private getQueueByStatus;
+    /**
+     * 恢复超时的任务
+     */
+    private recoverStalledTasks;
+    /**
+     * 批量处理任务
+     */
+    private processBatch;
+    /**
+     * 启动恢复机制
+     */
+    private startRecovery;
+    /**
+     * 停止恢复机制
+     */
+    private stopRecovery;
+    /**
+     * 启动消费者
+     */
+    private startConsumer;
+    /**
+     * 停止消费者
+     */
+    private stopConsumer;
+    /**
+     * 获取队列统计信息（O(1) 时间复杂度）
+     */
+    statistics(): Promise<QueueStats>;
+    /**
+     * 清空所有队列和任务数据
+     */
+    clear(): Promise<void>;
+    /**
+     * 健康检查
+     */
+    health(): Promise<boolean>;
+}
+//# sourceMappingURL=batch-consumer.d.ts.map

package/output/redis-queue/batch-consumer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch-consumer.d.ts","sourceRoot":"","sources":["../../source/redis-queue/batch-consumer.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,mBAAmB,EAAU,QAAQ,EAAQ,UAAU,EAAE,MAAM,SAAS,CAAA;AAEtF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,kBAAkB,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IAC3D,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,iBAAiB,CAAI;IAE7B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkC;IAGzD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAQ;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;gBAE3B,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;IAyC1C;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAY9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAUlB;;OAEG;YACW,OAAO;IAWrB;;OAEG;YACW,WAAW;IA8BzB;;OAEG;YACW,gBAAgB;IAI9B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;OAEG;YACW,mBAAmB;IA6EjC;;OAEG;YACW,YAAY;IA4F1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACH,OAAO,CAAC,aAAa;IAyErB;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC;IAevC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA4B5B;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAMjC"}

package/output/redis-queue/batch-consumer.js

@@ -0,0 +1,492 @@
+import { createClient } from 'redis';
+import { catchIt } from '@taicode/common-base';
+/**
+ * Redis 批量队列消费者
+ *
+ * 批量消费任务,每次处理多条
+ *
+ * @template T 任务数据类型
+ *
+ * @example
+ * ```ts
+ * interface EmailTask {
+ *   to: string
+ * }
+ *
+ * const consumer = new RedisBatchConsumer<EmailTask>({
+ *   redisUrl: 'redis://localhost:6379',
+ *   queueKey: 'email-queue',
+ *   batchSize: 50,
+ *   handler: async (dataList) => {
+ *     await sendEmailsBatch(dataList.map(d => d.to))
+ *     return catchIt(() => {})
+ *   }
+ * })
+ *
+ * await consumer.connect() // 自动开始消费
+ *
+ * // 获取统计信息
+ * const stats = await consumer.statistics()
+ *
+ * // 停止消费
+ * consumer.disconnect()
+ * ```
+ */
+export class RedisBatchConsumer {
+    consumerRunning = false;
+    redis = null;
+    consumerInterval = null;
+    recoveryInterval = null;
+    processingBatches = 0; // 当前正在处理的批次数
+    config;
+    // 不同状态队列的键名
+    failedQueue;
+    pendingQueue;
+    processingQueue;
+    completedQueue;
+    constructor(config) {
+        // 验证必填参数
+        if (!config.redisUrl) {
+            throw new Error('[RedisBatchConsumer] redisUrl is required');
+        }
+        if (!config.queueKey) {
+            throw new Error('[RedisBatchConsumer] queueKey is required');
+        }
+        if (config.queueKey.length < 6) {
+            throw new Error('[RedisBatchConsumer] queueKey must be at least 6 characters long');
+        }
+        if (!config.handler) {
+            throw new Error('[RedisBatchConsumer] handler is required');
+        }
+        this.config = {
+            queueKey: config.queueKey,
+            redisUrl: config.redisUrl,
+            handler: config.handler,
+            batchSize: config.batchSize ?? 10,
+            maxRetries: config.maxRetries ?? 3,
+            concurrency: config.concurrency ?? 1,
+            cleanupDelay: config.cleanupDelay ?? 86400, // 24 hours
+            consumerInterval: config.consumerInterval ?? 1000,
+            processingTimeout: config.processingTimeout ?? 60000, // 60 seconds
+        };
+        // 初始化不同状态队列的键名
+        this.failedQueue = `${config.queueKey}:failed`;
+        this.pendingQueue = `${config.queueKey}:pending`;
+        this.completedQueue = `${config.queueKey}:completed`;
+        this.processingQueue = `${config.queueKey}:processing`;
+        this.redis = createClient({ url: this.config.redisUrl });
+        // 添加错误处理
+        this.redis.on('error', (err) => {
+            console.error('[RedisBatchConsumer] Redis Client Error:', err);
+        });
+    }
+    /**
+     * 连接 Redis 并自动启动消费者
+     */
+    async connect() {
+        if (this.redis && !this.redis.isOpen) {
+            await this.redis.connect().catch((error) => {
+                console.error('[RedisBatchConsumer] Failed to connect to Redis:', error);
+                throw error;
+            });
+            // 连接成功后启动消费者和恢复机制
+            this.startConsumer();
+            this.startRecovery();
+        }
+    }
+    /**
+     * 断开 Redis 连接并停止消费者
+     */
+    disconnect() {
+        this.stopConsumer();
+        this.stopRecovery();
+        if (this.redis && this.redis.isOpen) {
+            this.redis.disconnect().catch((error) => {
+                console.error('[RedisBatchConsumer] Failed to disconnect:', error);
+            });
+        }
+    }
+    /**
+     * 获取任务详情
+     */
+    async getTask(taskId) {
+        if (!this.redis)
+            return null;
+        const taskKey = `${this.config.queueKey}:task:${taskId}`;
+        const taskData = await this.redis.get(taskKey);
+        if (!taskData)
+            return null;
+        return JSON.parse(taskData);
+    }
+    /**
+     * 更新任务状态并移动到对应队列（原子操作）
+     */
+    async applyStatus(taskId, oldStatus, newStatus) {
+        if (!this.redis)
+            return;
+        const task = await this.getTask(taskId);
+        if (!task)
+            return;
+        task.status = newStatus;
+        const taskKey = `${this.config.queueKey}:task:${taskId}`;
+        const oldQueue = this.getQueueByStatus(oldStatus);
+        const newQueue = this.getQueueByStatus(newStatus);
+        if (oldQueue !== newQueue) {
+            // 使用 Lua 脚本确保原子性：更新任务 + 移动队列
+            const script = `
+        redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
+        redis.call('LREM', KEYS[2], 0, ARGV[3])
+        redis.call('RPUSH', KEYS[3], ARGV[3])
+        return 1
+      `;
+            await this.redis.eval(script, {
+                keys: [taskKey, oldQueue, newQueue],
+                arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
+            });
+        }
+        else {
+            // 只更新任务数据
+            await this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
+        }
+    }
+    /**
+     * 批量更新任务状态
+     */
+    async applyStatusBatch(taskIds, oldStatus, newStatus) {
+        await Promise.all(taskIds.map(taskId => this.applyStatus(taskId, oldStatus, newStatus)));
+    }
+    /**
+     * 根据状态获取对应的队列键
+     */
+    getQueueByStatus(status) {
+        switch (status) {
+            case 'pending': return this.pendingQueue;
+            case 'processing': return this.processingQueue;
+            case 'completed': return this.completedQueue;
+            case 'failed': return this.failedQueue;
+            default: return this.pendingQueue;
+        }
+    }
+    /**
+     * 恢复超时的任务
+     */
+    async recoverStalledTasks() {
+        if (!this.redis)
+            return;
+        try {
+            const processingTaskIds = await this.redis.lRange(this.processingQueue, 0, -1);
+            if (processingTaskIds.length === 0) {
+                return;
+            }
+            const now = Date.now();
+            let recoveredCount = 0;
+            for (const taskId of processingTaskIds) {
+                const task = await this.getTask(taskId);
+                if (!task) {
+                    // 任务不存在，从队列中移除
+                    await this.redis.lRem(this.processingQueue, 0, taskId);
+                    continue;
+                }
+                // 检查是否超时
+                const processingTime = now - (task.processingStartTime || now);
+                if (processingTime > this.config.processingTimeout) {
+                    console.log(`[RedisBatchConsumer] Task timeout: ${taskId} (processing time: ${processingTime}ms)`);
+                    const taskKey = `${this.config.queueKey}:task:${taskId}`;
+                    // 检查是否还可以重试
+                    if (task.retryCount < task.maxRetries) {
+                        // 重试：放回 pending 队列
+                        task.status = 'pending';
+                        task.retryCount++;
+                        task.processingStartTime = undefined;
+                        // 使用 Lua 脚本确保原子性
+                        const script = `
+              redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
+              redis.call('LREM', KEYS[2], 0, ARGV[3])
+              redis.call('RPUSH', KEYS[3], ARGV[3])
+              return 1
+            `;
+                        await this.redis.eval(script, {
+                            keys: [taskKey, this.processingQueue, this.pendingQueue],
+                            arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
+                        });
+                        console.log(`[RedisBatchConsumer] Task ${taskId} will retry (${task.retryCount}/${task.maxRetries})`);
+                    }
+                    else {
+                        // 已达到最大重试次数，标记为失败
+                        task.status = 'failed';
+                        task.processingStartTime = undefined;
+                        // 使用 Lua 脚本确保原子性
+                        const script = `
+              redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
+              redis.call('LREM', KEYS[2], 0, ARGV[3])
+              redis.call('RPUSH', KEYS[3], ARGV[3])
+              return 1
+            `;
+                        await this.redis.eval(script, {
+                            keys: [taskKey, this.processingQueue, this.failedQueue],
+                            arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
+                        });
+                        console.error(`[RedisBatchConsumer] Task ${taskId} failed after timeout`);
+                    }
+                    recoveredCount++;
+                }
+            }
+            if (recoveredCount > 0) {
+                console.log(`[RedisBatchConsumer] Recovered ${recoveredCount} timeout tasks`);
+            }
+        }
+        catch (error) {
+            console.error('[RedisBatchConsumer] Failed to recover stalled tasks:', error);
+        }
+    }
+    /**
+     * 批量处理任务
+     */
+    async processBatch(taskIds) {
+        this.processingBatches++;
+        try {
+            // 获取所有任务
+            const tasks = await Promise.all(taskIds.map(id => this.getTask(id)));
+            const validTasks = tasks.filter((task) => task !== null);
+            if (validTasks.length === 0) {
+                console.warn(`[RedisBatchConsumer] No valid tasks found in batch`);
+                return;
+            }
+            // 过滤出有效的待处理任务
+            const tasksToProcess = [];
+            for (const task of validTasks) {
+                // 检查状态
+                if (task.status !== 'pending') {
+                    console.log(`[RedisBatchConsumer] Task ${task.id} has invalid status (${task.status}), marking as failed`);
+                    await this.applyStatus(task.id, task.status, 'failed');
+                    continue;
+                }
+                tasksToProcess.push(task);
+            }
+            if (tasksToProcess.length === 0) {
+                return;
+            }
+            try {
+                // 任务已在 processing 队列中(由 Lua 脚本完成),只需更新状态和开始时间
+                const taskIdList = tasksToProcess.map(t => t.id);
+                const now = Date.now();
+                await Promise.all(tasksToProcess.map(task => {
+                    task.status = 'processing';
+                    task.processingStartTime = now;
+                    const taskKey = `${this.config.queueKey}:task:${task.id}`;
+                    return this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
+                }));
+                // 批量处理所有任务
+                const dataList = tasksToProcess.map(t => t.data);
+                await this.config.handler(dataList);
+                // 批量更新状态为完成
+                await this.applyStatusBatch(taskIdList, 'processing', 'completed');
+                console.log(`[RedisBatchConsumer] Batch completed: ${taskIdList.length} tasks`);
+            }
+            catch (error) {
+                console.error(`[RedisBatchConsumer] Batch failed:`, error);
+                // 批量重试或失败
+                for (const task of tasksToProcess) {
+                    if (task.retryCount < task.maxRetries) {
+                        task.retryCount++;
+                        task.status = 'pending';
+                        task.processingStartTime = undefined;
+                        const taskKey = `${this.config.queueKey}:task:${task.id}`;
+                        const script = `
+              redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
+              redis.call('LREM', KEYS[2], 0, ARGV[3])
+              redis.call('RPUSH', KEYS[3], ARGV[3])
+              return 1
+            `;
+                        await this.redis.eval(script, {
+                            keys: [taskKey, this.processingQueue, this.pendingQueue],
+                            arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
+                        });
+                        console.log(`[RedisBatchConsumer] Task ${task.id} will retry (${task.retryCount}/${task.maxRetries})`);
+                    }
+                    else {
+                        task.status = 'failed';
+                        task.processingStartTime = undefined;
+                        const taskKey = `${this.config.queueKey}:task:${task.id}`;
+                        const script = `
+              redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
+              redis.call('LREM', KEYS[2], 0, ARGV[3])
+              redis.call('RPUSH', KEYS[3], ARGV[3])
+              return 1
+            `;
+                        await this.redis.eval(script, {
+                            keys: [taskKey, this.processingQueue, this.failedQueue],
+                            arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
+                        });
+                        console.error(`[RedisBatchConsumer] Task ${task.id} failed after ${task.maxRetries} retries`);
+                    }
+                }
+            }
+        }
+        finally {
+            this.processingBatches--;
+        }
+    }
+    /**
+     * 启动恢复机制
+     */
+    startRecovery() {
+        if (this.recoveryInterval || !this.redis) {
+            return;
+        }
+        // 立即执行一次恢复
+        this.recoverStalledTasks().catch(error => {
+            console.error('[RedisBatchConsumer] Initial recovery error:', error);
+        });
+        // 定期检查（每 10 秒检查一次）
+        this.recoveryInterval = setInterval(() => {
+            this.recoverStalledTasks().catch(error => {
+                console.error('[RedisBatchConsumer] Recovery error:', error);
+            });
+        }, 10000);
+        console.log('[RedisBatchConsumer] Recovery mechanism started');
+    }
+    /**
+     * 停止恢复机制
+     */
+    stopRecovery() {
+        if (this.recoveryInterval) {
+            clearInterval(this.recoveryInterval);
+            this.recoveryInterval = null;
+            console.log('[RedisBatchConsumer] Recovery mechanism stopped');
+        }
+    }
+    /**
+     * 启动消费者
+     */
+    startConsumer() {
+        if (this.consumerRunning || !this.redis) {
+            return;
+        }
+        this.consumerRunning = true;
+        console.log(`[RedisBatchConsumer] Consumer started with batchSize: ${this.config.batchSize}, concurrency: ${this.config.concurrency}`);
+        // Lua 脚本: 批量原子化从 pending 取出任务并移到 processing
+        const popAndMoveBatchScript = `
+      local pendingQueue = KEYS[1]
+      local processingQueue = KEYS[2]
+      local queueKeyPrefix = KEYS[3]
+      local count = tonumber(ARGV[1])
+      local currentTime = tonumber(ARGV[2])
+      
+      local taskIds = {}
+      local checkedCount = 0
+      local maxCheck = count * 3
+      
+      while #taskIds < count and checkedCount < maxCheck do
+        local taskId = redis.call('LPOP', pendingQueue)
+        if not taskId then
+          break
+        end
+        
+        checkedCount = checkedCount + 1
+        
+        local taskKey = queueKeyPrefix .. ':task:' .. taskId
+        local taskData = redis.call('GET', taskKey)
+        
+        if taskData then
+          local task = cjson.decode(taskData)
+          local delayUntil = task.delayUntil
+          
+          if not delayUntil or delayUntil <= currentTime then
+            redis.call('RPUSH', processingQueue, taskId)
+            table.insert(taskIds, taskId)
+          else
+            redis.call('RPUSH', pendingQueue, taskId)
+          end
+        end
+      end
+      
+      return taskIds
+    `;
+        this.consumerInterval = setInterval(async () => {
+            try {
+                // 检查当前是否有可用的并发槽位
+                const availableSlots = this.config.concurrency - this.processingBatches;
+                if (availableSlots <= 0) {
+                    return; // 已达到并发限制
+                }
+                // 批量取出任务
+                const taskIds = await this.redis.eval(popAndMoveBatchScript, {
+                    keys: [this.pendingQueue, this.processingQueue, this.config.queueKey],
+                    arguments: [this.config.batchSize.toString(), Date.now().toString()],
+                });
+                if (taskIds.length > 0) {
+                    // 处理这批任务（不等待完成）
+                    this.processBatch(taskIds).catch(error => {
+                        console.error(`[RedisBatchConsumer] Unhandled error in processBatch:`, error);
+                    });
+                }
+            }
+            catch (error) {
+                console.error('[RedisBatchConsumer] Consumer error:', error);
+            }
+        }, this.config.consumerInterval);
+    }
+    /**
+     * 停止消费者
+     */
+    stopConsumer() {
+        if (this.consumerInterval) {
+            clearInterval(this.consumerInterval);
+            this.consumerInterval = null;
+        }
+        this.consumerRunning = false;
+        console.log('[RedisBatchConsumer] Consumer stopped');
+    }
+    /**
+     * 获取队列统计信息（O(1) 时间复杂度）
+     */
+    async statistics() {
+        if (!this.redis) {
+            return { pending: 0, processing: 0, completed: 0, failed: 0 };
+        }
+        const [pending, processing, completed, failed] = await Promise.all([
+            this.redis.lLen(this.pendingQueue),
+            this.redis.lLen(this.processingQueue),
+            this.redis.lLen(this.completedQueue),
+            this.redis.lLen(this.failedQueue),
+        ]);
+        return { pending, processing, completed, failed };
+    }
+    /**
+     * 清空所有队列和任务数据
+     */
+    async clear() {
+        if (!this.redis)
+            return;
+        // 获取所有队列中的任务 ID
+        const [pendingIds, processingIds, completedIds, failedIds] = await Promise.all([
+            this.redis.lRange(this.pendingQueue, 0, -1),
+            this.redis.lRange(this.processingQueue, 0, -1),
+            this.redis.lRange(this.completedQueue, 0, -1),
+            this.redis.lRange(this.failedQueue, 0, -1),
+        ]);
+        const allTaskIds = [...pendingIds, ...processingIds, ...completedIds, ...failedIds];
+        // 删除所有任务数据
+        const taskKeys = allTaskIds.map(id => `${this.config.queueKey}:task:${id}`);
+        if (taskKeys.length > 0) {
+            await this.redis.del(taskKeys);
+        }
+        // 删除所有队列
+        await this.redis.del([
+            this.pendingQueue,
+            this.processingQueue,
+            this.completedQueue,
+            this.failedQueue,
+        ]);
+    }
+    /**
+     * 健康检查
+     */
+    async health() {
+        if (!this.redis)
+            return false;
+        const result = await catchIt(() => this.redis.ping());
+        return !result.isError() && result.value === 'PONG';
+    }
+}

package/output/redis-queue/batch-consumer.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=batch-consumer.test.d.ts.map

package/output/redis-queue/batch-consumer.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch-consumer.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/batch-consumer.test.ts"],"names":[],"mappings":""}