@taicode/common-server 3.7.4 → 3.7.7

package/output/redis-queue/redis-queue-batch-consumer.js

@@ -1,341 +0,0 @@
-import { catchIt } from '@taicode/common-base';
-import { logger } from '@taicode/common-base';
-import { RedisQueueCommon } from './redis-queue-common.js';
-/**
- * Redis 批量队列消费者
- *
- * 批量消费任务,每次处理多条
- *
- * @template R Registry 类型,默认为 RedisQueueRegistry
- * @template K 队列键名类型
- *
- * @example
- * ```ts
- * // 方式 1: 使用默认 RedisQueueRegistry
- * const consumer = new RedisQueueBatchConsumer('email-queue', {
- *   redisUrl: 'redis://localhost:6379',
- *   batchSize: 50,
- *   handler: async (dataList) => {
- *     await sendEmailsBatch(dataList.map(d => d.to))
- *     return catchIt(() => {})
- *   }
- * })
- *
- * // 方式 2: 使用自定义 Registry
- * interface MyRegistry {
- *   'order-queue': { orderId: string; amount: number }
- * }
- *
- * const consumer = new RedisQueueBatchConsumer<MyRegistry, 'order-queue'>(
- *   'order-queue',
- *   {
- *     redisUrl: 'redis://localhost:6379',
- *     batchSize: 50,
- *     handler: async (dataList) => {
- *       const total = dataList.reduce((sum, d) => sum + d.amount, 0)
- *       logger.info('Total amount:', total)
- *       return catchIt(() => {})
- *     }
- *   }
- * )
- *
- * await consumer.connect() // 自动开始消费
- *
- * // 获取统计信息
- * const stats = await consumer.statistics()
- *
- * // 停止消费
- * consumer.disconnect()
- * ```
- */
-export class RedisQueueBatchConsumer extends RedisQueueCommon {
-    consumerRunning = false;
-    consumerInterval = null;
-    recoveryInterval = null;
-    processingBatches = 0; // 当前正在处理的批次数
-    processingPromises = new Set(); // 跟踪所有正在处理的批次
-    config;
-    constructor(queueKey, config) {
-        // 验证必填参数
-        if (!config.handler) {
-            throw new Error('[RedisQueueBatchConsumer] handler is required');
-        }
-        // 调用父类构造函数
-        super(queueKey, {
-            redisUrl: config.redisUrl,
-            redisClient: config.redisClient,
-            cleanupDelay: config.cleanupDelay ?? 86400,
-        });
-        this.config = {
-            handler: config.handler,
-            batchSize: config.batchSize ?? 10,
-            maxRetries: config.maxRetries ?? 3,
-            concurrency: config.concurrency ?? 1,
-            consumerInterval: config.consumerInterval ?? 1000,
-            processingTimeout: config.processingTimeout ?? 60000, // 60 seconds
-        };
-    }
-    getLogPrefix() {
-        return 'RedisQueueBatchConsumer';
-    }
-    /**
-     * 连接 Redis 并自动启动消费者
-     */
-    async connect() {
-        await super.connect();
-        // 连接成功后启动消费者和恢复机制
-        this.startConsumer();
-        this.startRecovery();
-    }
-    /**
-     * 断开 Redis 连接并停止消费者
-     * 会等待所有正在处理的批次完成后再断开连接
-     */
-    async disconnect() {
-        // 先停止消费者和恢复机制，不再接受新任务
-        this.stopConsumer();
-        this.stopRecovery();
-        // 等待所有正在处理的批次完成
-        if (this.processingPromises.size > 0) {
-            logger.info(`[RedisQueueBatchConsumer] Waiting for ${this.processingPromises.size} batches to complete...`);
-            await Promise.allSettled(this.processingPromises);
-            logger.info(`[RedisQueueBatchConsumer] All batches completed`);
-        }
-        // 最后断开 Redis 连接
-        super.disconnect();
-    }
-    /**
-     * 批量处理任务
-     */
-    async processBatch(taskIds) {
-        this.processingBatches++;
-        // 创建一个 Promise 来跟踪这个批次的处理过程
-        const batchPromise = (async () => {
-            try {
-                // 获取所有任务
-                const tasks = await Promise.all(taskIds.map(id => this.getTask(id)));
-                const validTasks = tasks.filter((task) => task !== null);
-                if (validTasks.length === 0) {
-                    logger.warn(`[RedisQueueBatchConsumer] No valid tasks found in batch`);
-                    return;
-                }
-                // 过滤出有效的待处理任务
-                const tasksToProcess = [];
-                for (const task of validTasks) {
-                    // 检查状态
-                    if (task.status !== 'pending') {
-                        logger.info(`[RedisQueueBatchConsumer] Task ${task.id} has invalid status (${task.status}), marking as failed`);
-                        await this.applyStatus(task.id, task.status, 'failed');
-                        continue;
-                    }
-                    tasksToProcess.push(task);
-                }
-                if (tasksToProcess.length === 0) {
-                    return;
-                }
-                try {
-                    // 任务已在 processing 队列中(由 Lua 脚本完成),只需更新状态和开始时间
-                    const taskIdList = tasksToProcess.map(t => t.id);
-                    const now = Date.now();
-                    await Promise.all(tasksToProcess.map(task => {
-                        task.status = 'processing';
-                        task.processingStartTime = now;
-                        const taskKey = `${this.queueKey}:task:${task.id}`;
-                        return this.redis.setEx(taskKey, this.cleanupDelay, JSON.stringify(task));
-                    }));
-                    // 批量处理所有任务
-                    const dataList = tasksToProcess.map(t => t.data);
-                    await this.config.handler(dataList);
-                    // 批量更新状态为完成
-                    const updatedCount = await this.applyStatusBatch(taskIdList, 'processing', 'completed');
-                    logger.info(`[RedisQueueBatchConsumer] Batch completed: ${updatedCount} tasks`);
-                }
-                catch (error) {
-                    logger.error(`[RedisQueueBatchConsumer] Batch failed:`, error);
-                    // 批量重试或失败
-                    for (const task of tasksToProcess) {
-                        if (task.retryCount < task.maxRetries) {
-                            task.retryCount++;
-                            task.status = 'pending';
-                            task.processingStartTime = undefined;
-                            const taskKey = `${this.queueKey}:task:${task.id}`;
-                            const script = `
-                redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
-                redis.call('LREM', KEYS[2], 0, ARGV[3])
-                redis.call('RPUSH', KEYS[3], ARGV[3])
-                return 1
-              `;
-                            await this.redis.eval(script, {
-                                keys: [taskKey, this.processingQueue, this.pendingQueue],
-                                arguments: [this.cleanupDelay.toString(), JSON.stringify(task), task.id],
-                            });
-                            logger.info(`[RedisQueueBatchConsumer] Task ${task.id} will retry (${task.retryCount}/${task.maxRetries})`);
-                        }
-                        else {
-                            task.status = 'failed';
-                            task.processingStartTime = undefined;
-                            const taskKey = `${this.queueKey}:task:${task.id}`;
-                            const script = `
-                redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
-                redis.call('LREM', KEYS[2], 0, ARGV[3])
-                redis.call('RPUSH', KEYS[3], ARGV[3])
-                return 1
-              `;
-                            await this.redis.eval(script, {
-                                keys: [taskKey, this.processingQueue, this.failedQueue],
-                                arguments: [this.cleanupDelay.toString(), JSON.stringify(task), task.id],
-                            });
-                            logger.error(`[RedisQueueBatchConsumer] Task ${task.id} failed after ${task.maxRetries} retries`);
-                        }
-                    }
-                }
-            }
-            finally {
-                this.processingBatches--;
-            }
-        })();
-        // 将批次 Promise 添加到集合中
-        this.processingPromises.add(batchPromise);
-        // 批次完成后从集合中移除
-        batchPromise.finally(() => {
-            this.processingPromises.delete(batchPromise);
-        });
-        // 返回批次 Promise
-        return batchPromise;
-    }
-    /**
-     * 启动恢复机制
-     */
-    startRecovery() {
-        if (this.recoveryInterval || !this.redis) {
-            return;
-        }
-        // 立即执行一次恢复
-        this.recoverStalled(this.config.processingTimeout).catch(error => {
-            logger.error('[RedisQueueBatchConsumer] Initial recovery error:', error);
-        });
-        // 定期检查（每 10 秒检查一次）
-        this.recoveryInterval = setInterval(() => {
-            this.recoverStalled(this.config.processingTimeout).catch(error => {
-                logger.error('[RedisQueueBatchConsumer] Recovery error:', error);
-            });
-        }, 10000);
-        logger.info('[RedisQueueBatchConsumer] Recovery mechanism started');
-    }
-    /**
-     * 停止恢复机制
-     */
-    stopRecovery() {
-        if (this.recoveryInterval) {
-            clearInterval(this.recoveryInterval);
-            this.recoveryInterval = null;
-            logger.info('[RedisQueueBatchConsumer] Recovery mechanism stopped');
-        }
-    }
-    /**
-     * 启动消费者
-     */
-    startConsumer() {
-        if (this.consumerRunning || !this.redis) {
-            return;
-        }
-        this.consumerRunning = true;
-        logger.info(`[RedisQueueBatchConsumer] Consumer started with batchSize: ${this.config.batchSize}, concurrency: ${this.config.concurrency}`);
-        // Lua 脚本: 批量原子化从 pending 取出任务并移到 processing
-        const popAndMoveBatchScript = `
-      local pendingQueue = KEYS[1]
-      local processingQueue = KEYS[2]
-      local queueKeyPrefix = KEYS[3]
-      local count = tonumber(ARGV[1])
-      local currentTime = tonumber(ARGV[2])
-      
-      local taskIds = {}
-      local checkedCount = 0
-      local maxCheck = count * 3
-      
-      while #taskIds < count and checkedCount < maxCheck do
-        local taskId = redis.call('LPOP', pendingQueue)
-        if not taskId then
-          break
-        end
-        
-        checkedCount = checkedCount + 1
-        
-        local taskKey = queueKeyPrefix .. ':task:' .. taskId
-        local taskData = redis.call('GET', taskKey)
-        
-        if taskData then
-          local task = cjson.decode(taskData)
-          local delayUntil = task.delayUntil
-          
-          if not delayUntil or delayUntil <= currentTime then
-            redis.call('RPUSH', processingQueue, taskId)
-            table.insert(taskIds, taskId)
-          else
-            redis.call('RPUSH', pendingQueue, taskId)
-          end
-        end
-      end
-      
-      return taskIds
-    `;
-        this.consumerInterval = setInterval(async () => {
-            try {
-                // 检查当前是否有可用的并发槽位
-                const availableSlots = this.config.concurrency - this.processingBatches;
-                if (availableSlots <= 0) {
-                    return; // 已达到并发限制
-                }
-                // 批量取出任务
-                const taskIds = await this.redis.eval(popAndMoveBatchScript, {
-                    keys: [this.pendingQueue, this.processingQueue, this.queueKey],
-                    arguments: [this.config.batchSize.toString(), Date.now().toString()],
-                });
-                if (taskIds.length > 0) {
-                    // 处理这批任务（不等待完成）
-                    this.processBatch(taskIds).catch(error => {
-                        logger.error(`[RedisQueueBatchConsumer] Unhandled error in processBatch:`, error);
-                    });
-                }
-            }
-            catch (error) {
-                logger.error('[RedisQueueBatchConsumer] Consumer error:', error);
-            }
-        }, this.config.consumerInterval);
-    }
-    /**
-     * 停止消费者
-     */
-    stopConsumer() {
-        if (this.consumerInterval) {
-            clearInterval(this.consumerInterval);
-            this.consumerInterval = null;
-        }
-        this.consumerRunning = false;
-        logger.info('[RedisQueueBatchConsumer] Consumer stopped');
-    }
-    /**
-     * 获取队列统计信息（O(1) 时间复杂度）
-     */
-    async statistics() {
-        if (!this.redis) {
-            return { pending: 0, processing: 0, completed: 0, failed: 0 };
-        }
-        const [pending, processing, completed, failed] = await Promise.all([
-            this.redis.lLen(this.pendingQueue),
-            this.redis.lLen(this.processingQueue),
-            this.redis.lLen(this.completedQueue),
-            this.redis.lLen(this.failedQueue),
-        ]);
-        return { pending, processing, completed, failed };
-    }
-    /**
-     * 健康检查
-     */
-    async health() {
-        if (!this.redis)
-            return false;
-        const result = await catchIt(() => this.redis.ping());
-        return !result.isError() && result.value === 'PONG';
-    }
-}

package/output/redis-queue/redis-queue-batch-consumer.test.d.ts

@@ -1,26 +0,0 @@
-type TestTaskData = {
-    id?: string;
-    value?: number | string;
-    test?: boolean | string;
-    index?: number;
-    payload?: string;
-    order?: number;
-    taskId?: number;
-    message?: string;
-    name?: string;
-    shouldFail?: boolean;
-    step?: number;
-    real?: boolean;
-    nested?: {
-        value: number;
-    };
-    example?: string;
-    data?: any;
-};
-declare module './types.js' {
-    interface RedisQueueRegistry {
-        [key: `test:batch:${string}`]: TestTaskData;
-    }
-}
-export {};
-//# sourceMappingURL=redis-queue-batch-consumer.test.d.ts.map

package/output/redis-queue/redis-queue-batch-consumer.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"redis-queue-batch-consumer.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-batch-consumer.test.ts"],"names":[],"mappings":"AAQA,KAAK,YAAY,GAAG;IAClB,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,IAAI,CAAC,EAAE,GAAG,CAAA;CACX,CAAA;AAGD,OAAO,QAAQ,YAAY,CAAC;IAC1B,UAAU,kBAAkB;QAC1B,CAAC,GAAG,EAAE,cAAc,MAAM,EAAE,GAAG,YAAY,CAAA;KAC5C;CACF"}