@taicode/common-server 1.0.11 → 1.0.13

package/output/redis-queue/batch-redis-queue.js

@@ -1,573 +0,0 @@
-import { randomUUID } from 'node:crypto';
-import { createClient } from 'redis';
-import { catchIt } from '@taicode/common-base';
-/**
- * 批量任务队列类（泛型）
- *
- * 提供基于 Redis 的批量任务队列功能，支持：
- * - 批量处理任务（每次处理多条）
- * - 任务入队和持久化
- * - 自动重试机制
- * - 任务状态追踪
- * - 分布式消费
- *
- * @template T 任务数据类型
- *
- * @example
- * ```ts
- * interface EmailTask {
- *   to: string
- * }
- *
- * const queue = new BatchRedisQueue<EmailTask>({
- *   redisUrl: 'redis://localhost:6379',
- *   queueKey: 'email-batch-queue',
- *   batchSize: 50,
- *   handler: async (dataList) => {
- *     await sendEmailsBatch(dataList.map(d => d.to))
- *     return catchIt(() => {})
- *   }
- * })
- *
- * // 连接 Redis（自动启动消费者）
- * await queue.connect()
- *
- * // 入队任务（自动开始处理）
- * await queue.enqueue([
- *   { to: 'user1@example.com' },
- *   { to: 'user2@example.com' },
- * ])
- *
- * // 获取队列统计信息（O(1) 时间复杂度）
- * const stats = await queue.statistics()
- * console.log(stats) // { pending: 100, processing: 50, completed: 200, failed: 5 }
- * ```
- */
-export class BatchRedisQueue {
-    consumerRunning = false;
-    redis = null;
-    consumerInterval = null;
-    recoveryInterval = null;
-    processingBatches = 0; // 当前正在处理的批次数
-    config;
-    handler;
-    // 不同状态队列的键名
-    failedQueue;
-    pendingQueue;
-    processingQueue;
-    completedQueue;
-    constructor(config) {
-        // 验证必填参数
-        if (!config.redisUrl) {
-            throw new Error('[BatchTaskQueue] redisUrl is required');
-        }
-        if (!config.queueKey) {
-            throw new Error('[BatchTaskQueue] queueKey is required');
-        }
-        if (config.queueKey.length < 6) {
-            throw new Error('[BatchTaskQueue] queueKey must be at least 6 characters long');
-        }
-        if (!config.handler) {
-            throw new Error('[BatchTaskQueue] handler is required');
-        }
-        this.handler = config.handler;
-        this.config = {
-            queueKey: config.queueKey,
-            redisUrl: config.redisUrl,
-            batchSize: config.batchSize || 10,
-            maxRetries: config.maxRetries || 3,
-            concurrency: config.concurrency || 1,
-            cleanupDelay: config.cleanupDelay || 86400, // 24 hours
-            processingDelay: config.processingDelay || 0, // 默认立即执行
-            consumerInterval: config.consumerInterval || 1000,
-            processingTimeout: config.processingTimeout || 60000, // 60 seconds
-        };
-        // 初始化不同状态队列的键名
-        this.failedQueue = `${config.queueKey}:failed`;
-        this.pendingQueue = `${config.queueKey}:pending`;
-        this.completedQueue = `${config.queueKey}:completed`;
-        this.processingQueue = `${config.queueKey}:processing`;
-        this.redis = createClient({ url: this.config.redisUrl });
-        // 添加错误处理
-        this.redis.on('error', (err) => {
-            console.error('[BatchTaskQueue] Redis Client Error:', err);
-        });
-    }
-    /**
-     * 连接 Redis 并自动启动消费者
-     */
-    async connect() {
-        if (this.redis && !this.redis.isOpen) {
-            await this.redis.connect().catch((error) => {
-                console.error('[BatchTaskQueue] Failed to connect to Redis:', error);
-                throw error;
-            });
-            // 连接成功后启动消费者和恢复机制
-            this.startConsumer();
-            this.startRecovery();
-        }
-    }
-    /**
-     * 断开 Redis 连接并停止消费者
-     */
-    disconnect() {
-        this.stopConsumer();
-        this.stopRecovery();
-        if (this.redis && this.redis.isOpen) {
-            this.redis.disconnect().catch((error) => {
-                console.error('[BatchTaskQueue] Failed to disconnect:', error);
-            });
-        }
-    }
-    async enqueue(data) {
-        if (!this.redis) {
-            console.warn('[BatchTaskQueue] Redis not available, skipping task enqueue');
-            return Array.isArray(data) ? [] : '';
-        }
-        // 统一处理为数组
-        const dataList = Array.isArray(data) ? data : [data];
-        const taskIds = [];
-        // 使用 Lua 脚本确保幂等入队的原子性
-        const enqueueScript = `
-      local taskKey = KEYS[1]
-      local pendingQueue = KEYS[2]
-      local taskData = ARGV[1]
-      local ttl = tonumber(ARGV[2])
-      local taskId = ARGV[3]
-      
-      -- 检查任务是否已存在
-      local exists = redis.call('EXISTS', taskKey)
-      if exists == 1 then
-        return 0  -- 任务已存在,跳过
-      end
-      
-      -- 原子化操作: 创建任务 + 推入队列
-      redis.call('SETEX', taskKey, ttl, taskData)
-      redis.call('RPUSH', pendingQueue, taskId)
-      return 1  -- 成功创建
-    `;
-        for (const item of dataList) {
-            // 检查 data 中是否包含自定义 id
-            const customId = item.id;
-            const taskId = customId || randomUUID();
-            const taskKey = `${this.config.queueKey}:task:${taskId}`;
-            const task = {
-                id: taskId,
-                data: item,
-                retryCount: 0,
-                maxRetries: this.config.maxRetries,
-                createdTime: new Date().toISOString(),
-                status: 'pending',
-                ...(this.config.processingDelay > 0 && { delayUntil: Date.now() + this.config.processingDelay }),
-            };
-            // 使用 Lua 脚本原子化入队
-            const result = await this.redis.eval(enqueueScript, {
-                keys: [taskKey, this.pendingQueue],
-                arguments: [JSON.stringify(task), this.config.cleanupDelay.toString(), taskId],
-            });
-            if (result === 0) {
-                console.log(`[BatchTaskQueue] Task already exists, skipping: ${taskId}`);
-            }
-            else {
-                console.log(`[BatchTaskQueue] Enqueued task: ${task.id}`);
-            }
-            taskIds.push(taskId);
-        }
-        // 返回单个 ID 或 ID 数组
-        return Array.isArray(data) ? taskIds : taskIds[0];
-    }
-    /**
-     * 获取任务详情
-     */
-    async getTask(taskId) {
-        if (!this.redis)
-            return null;
-        const taskKey = `${this.config.queueKey}:task:${taskId}`;
-        const taskData = await this.redis.get(taskKey);
-        if (!taskData)
-            return null;
-        return JSON.parse(taskData);
-    }
-    /**
-     * 更新任务状态并移动到对应队列（原子操作）
-     */
-    async applyStatus(taskId, oldStatus, newStatus) {
-        if (!this.redis)
-            return;
-        const task = await this.getTask(taskId);
-        if (!task)
-            return;
-        task.status = newStatus;
-        const taskKey = `${this.config.queueKey}:task:${taskId}`;
-        const oldQueue = this.getQueueByStatus(oldStatus);
-        const newQueue = this.getQueueByStatus(newStatus);
-        if (oldQueue !== newQueue) {
-            // 使用 Lua 脚本确保原子性：更新任务 + 移动队列
-            const script = `
-        redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
-        redis.call('LREM', KEYS[2], 0, ARGV[3])
-        redis.call('RPUSH', KEYS[3], ARGV[3])
-        return 1
-      `;
-            await this.redis.eval(script, {
-                keys: [taskKey, oldQueue, newQueue],
-                arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
-            });
-        }
-        else {
-            // 只更新任务数据
-            await this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
-        }
-    }
-    /**
-     * 批量更新任务状态
-     */
-    async applyStatusBatch(taskIds, oldStatus, newStatus) {
-        await Promise.all(taskIds.map(taskId => this.applyStatus(taskId, oldStatus, newStatus)));
-    }
-    /**
-     * 根据状态获取对应的队列键
-     */
-    getQueueByStatus(status) {
-        switch (status) {
-            case 'pending': return this.pendingQueue;
-            case 'processing': return this.processingQueue;
-            case 'completed': return this.completedQueue;
-            case 'failed': return this.failedQueue;
-            default: return this.pendingQueue;
-        }
-    }
-    /**
-     * 恢复超时的任务
-     * 检查 processing 队列中的任务，将超时的任务重试或标记为失败
-     */
-    async recoverStalledTasks() {
-        if (!this.redis)
-            return;
-        try {
-            const processingTaskIds = await this.redis.lRange(this.processingQueue, 0, -1);
-            if (processingTaskIds.length === 0) {
-                return;
-            }
-            const now = Date.now();
-            let recoveredCount = 0;
-            for (const taskId of processingTaskIds) {
-                const task = await this.getTask(taskId);
-                if (!task) {
-                    // 任务不存在，从队列中移除
-                    await this.redis.lRem(this.processingQueue, 0, taskId);
-                    continue;
-                }
-                // 检查是否超时
-                const processingTime = now - (task.processingStartTime || now);
-                if (processingTime > this.config.processingTimeout) {
-                    console.log(`[BatchTaskQueue] Task timeout: ${taskId} (processing time: ${processingTime}ms)`);
-                    const taskKey = `${this.config.queueKey}:task:${taskId}`;
-                    // 检查是否还可以重试
-                    if (task.retryCount < task.maxRetries) {
-                        // 重试：放回 pending 队列
-                        task.status = 'pending';
-                        task.retryCount++;
-                        task.processingStartTime = undefined;
-                        // 使用 Lua 脚本确保原子性
-                        const script = `
-              redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
-              redis.call('LREM', KEYS[2], 0, ARGV[3])
-              redis.call('RPUSH', KEYS[3], ARGV[3])
-              return 1
-            `;
-                        await this.redis.eval(script, {
-                            keys: [taskKey, this.processingQueue, this.pendingQueue],
-                            arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
-                        });
-                        console.log(`[BatchTaskQueue] Task ${taskId} will retry (${task.retryCount}/${task.maxRetries})`);
-                    }
-                    else {
-                        // 已达到最大重试次数，标记为失败
-                        task.status = 'failed';
-                        task.processingStartTime = undefined;
-                        // 使用 Lua 脚本确保原子性
-                        const script = `
-              redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
-              redis.call('LREM', KEYS[2], 0, ARGV[3])
-              redis.call('RPUSH', KEYS[3], ARGV[3])
-              return 1
-            `;
-                        await this.redis.eval(script, {
-                            keys: [taskKey, this.processingQueue, this.failedQueue],
-                            arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
-                        });
-                        console.error(`[BatchTaskQueue] Task ${taskId} failed after timeout`);
-                    }
-                    recoveredCount++;
-                }
-            }
-            if (recoveredCount > 0) {
-                console.log(`[BatchTaskQueue] Recovered ${recoveredCount} timeout tasks`);
-            }
-        }
-        catch (error) {
-            console.error('[BatchTaskQueue] Failed to recover stalled tasks:', error);
-        }
-    }
-    /**
-     * 批量处理任务
-     */
-    async processBatch(taskIds) {
-        this.processingBatches++;
-        try {
-            // 获取所有任务
-            const tasks = await Promise.all(taskIds.map(id => this.getTask(id)));
-            const validTasks = tasks.filter((task) => task !== null);
-            if (validTasks.length === 0) {
-                console.warn(`[BatchTaskQueue] No valid tasks found in batch`);
-                return;
-            }
-            // 过滤出有效的待处理任务
-            const tasksToProcess = [];
-            for (const task of validTasks) {
-                // 检查状态
-                if (task.status !== 'pending') {
-                    console.log(`[BatchTaskQueue] Task ${task.id} has invalid status (${task.status}), marking as failed`);
-                    await this.applyStatus(task.id, task.status, 'failed');
-                    continue;
-                }
-                tasksToProcess.push(task);
-            }
-            if (tasksToProcess.length === 0) {
-                return;
-            }
-            try {
-                // 任务已在 processing 队列中(由 Lua 脚本完成),只需更新状态和开始时间
-                const taskIdList = tasksToProcess.map(t => t.id);
-                const now = Date.now();
-                await Promise.all(tasksToProcess.map(task => {
-                    task.status = 'processing';
-                    task.processingStartTime = now;
-                    const taskKey = `${this.config.queueKey}:task:${task.id}`;
-                    return this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
-                }));
-                // 执行批量任务处理器
-                const dataList = tasksToProcess.map(t => t.data);
-                await this.handler(dataList);
-                // 更新状态为完成
-                await this.applyStatusBatch(taskIdList, 'processing', 'completed');
-                console.log(`[BatchTaskQueue] Batch completed: ${tasksToProcess.length} tasks`);
-            }
-            catch (error) {
-                console.error(`[BatchTaskQueue] Batch failed`, error);
-                // 检查每个任务是否需要重试
-                for (const task of tasksToProcess) {
-                    if (task.retryCount < task.maxRetries) {
-                        task.retryCount++;
-                        task.status = 'pending';
-                        task.processingStartTime = undefined;
-                        const taskKey = `${this.config.queueKey}:task:${task.id}`;
-                        // 使用 Lua 脚本确保原子性
-                        const script = `
-              redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
-              redis.call('LREM', KEYS[2], 0, ARGV[3])
-              redis.call('RPUSH', KEYS[3], ARGV[3])
-              return 1
-            `;
-                        await this.redis.eval(script, {
-                            keys: [taskKey, this.processingQueue, this.pendingQueue],
-                            arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
-                        });
-                        console.log(`[BatchTaskQueue] Task ${task.id} will retry (${task.retryCount}/${task.maxRetries})`);
-                    }
-                    else {
-                        task.status = 'failed';
-                        task.processingStartTime = undefined;
-                        const taskKey = `${this.config.queueKey}:task:${task.id}`;
-                        // 使用 Lua 脚本确保原子性
-                        const script = `
-              redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
-              redis.call('LREM', KEYS[2], 0, ARGV[3])
-              redis.call('RPUSH', KEYS[3], ARGV[3])
-              return 1
-            `;
-                        await this.redis.eval(script, {
-                            keys: [taskKey, this.processingQueue, this.failedQueue],
-                            arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
-                        });
-                        console.error(`[BatchTaskQueue] Task ${task.id} failed after ${task.maxRetries} retries`);
-                    }
-                }
-            }
-        }
-        finally {
-            this.processingBatches--;
-        }
-    }
-    /**
-     * 启动恢复机制（内部方法，自动调用）
-     */
-    startRecovery() {
-        if (this.recoveryInterval || !this.redis) {
-            return;
-        }
-        // 立即执行一次恢复
-        this.recoverStalledTasks().catch(error => {
-            console.error('[BatchTaskQueue] Initial recovery error:', error);
-        });
-        // 定期检查（每 10 秒检查一次）
-        this.recoveryInterval = setInterval(() => {
-            this.recoverStalledTasks().catch(error => {
-                console.error('[BatchTaskQueue] Recovery error:', error);
-            });
-        }, 10000);
-        console.log('[BatchTaskQueue] Recovery mechanism started');
-    }
-    /**
-     * 停止恢复机制（内部方法，自动调用）
-     */
-    stopRecovery() {
-        if (this.recoveryInterval) {
-            clearInterval(this.recoveryInterval);
-            this.recoveryInterval = null;
-            console.log('[BatchTaskQueue] Recovery mechanism stopped');
-        }
-    }
-    /**
-     * 启动消费者（内部方法，自动调用）
-     */
-    startConsumer() {
-        if (this.consumerRunning || !this.redis) {
-            return;
-        }
-        this.consumerRunning = true;
-        console.log(`[BatchTaskQueue] Consumer started with batchSize: ${this.config.batchSize}`);
-        // Lua 脚本: 原子化从 pending 取出任务并移到 processing(只取已到延迟时间的任务)
-        const popAndMoveScript = `
-      local pendingQueue = KEYS[1]
-      local processingQueue = KEYS[2]
-      local queueKeyPrefix = KEYS[3]
-      local count = tonumber(ARGV[1])
-      local currentTime = tonumber(ARGV[2])
-      
-      local taskIds = {}
-      local checkedCount = 0
-      local maxCheck = count * 3  -- 最多检查 count*3 个任务,避免无限循环
-      
-      while #taskIds < count and checkedCount < maxCheck do
-        local taskId = redis.call('LPOP', pendingQueue)
-        if not taskId then
-          break
-        end
-        
-        checkedCount = checkedCount + 1
-        
-        -- 获取任务详情检查延迟时间
-        local taskKey = queueKeyPrefix .. ':task:' .. taskId
-        local taskData = redis.call('GET', taskKey)
-        
-        if taskData then
-          local task = cjson.decode(taskData)
-          local delayUntil = task.delayUntil
-          
-          -- 如果没有延迟或已到延迟时间,则移到 processing
-          if not delayUntil or delayUntil <= currentTime then
-            redis.call('RPUSH', processingQueue, taskId)
-            table.insert(taskIds, taskId)
-          else
-            -- 未到延迟时间,放回队列尾部
-            redis.call('RPUSH', pendingQueue, taskId)
-          end
-        end
-      end
-      
-      return taskIds
-    `;
-        this.consumerInterval = setInterval(async () => {
-            try {
-                // 检查是否有正在处理的批次（批量队列通常不需要并发控制，一次处理一批）
-                if (this.processingBatches > 0) {
-                    return;
-                }
-                // 使用 Lua 脚本原子化取出任务并移到 processing 队列
-                const taskIds = await this.redis.eval(popAndMoveScript, {
-                    keys: [this.pendingQueue, this.processingQueue, this.config.queueKey],
-                    arguments: [this.config.batchSize.toString(), Date.now().toString()],
-                });
-                if (taskIds.length > 0) {
-                    // 批量处理任务（不等待完成）
-                    this.processBatch(taskIds).catch(error => {
-                        console.error(`[BatchTaskQueue] Unhandled error in processBatch`, error);
-                    });
-                }
-            }
-            catch (error) {
-                console.error('[BatchTaskQueue] Consumer error:', error);
-            }
-        }, this.config.consumerInterval);
-    }
-    /**
-     * 停止消费者（内部方法，自动调用）
-     */
-    stopConsumer() {
-        if (this.consumerInterval) {
-            clearInterval(this.consumerInterval);
-            this.consumerInterval = null;
-        }
-        this.consumerRunning = false;
-        console.log('[BatchTaskQueue] Consumer stopped');
-    }
-    /**
-     * 获取队列统计信息
-     * 返回队列中各种状态任务的详细数量
-     *
-     * 使用分离队列设计，O(1) 时间复杂度
-     */
-    async statistics() {
-        if (!this.redis) {
-            return {
-                failed: 0,
-                pending: 0,
-                completed: 0,
-                processing: 0,
-            };
-        }
-        // 并行获取各队列长度，O(1) 时间复杂度
-        const [pending, processing, completed, failed] = await Promise.all([
-            this.redis.lLen(this.pendingQueue),
-            this.redis.lLen(this.processingQueue),
-            this.redis.lLen(this.completedQueue),
-            this.redis.lLen(this.failedQueue),
-        ]);
-        return {
-            pending,
-            processing,
-            completed,
-            failed,
-        };
-    }
-    /**
-     * 清空所有队列
-     */
-    async clear() {
-        if (!this.redis)
-            return;
-        // 使用 Lua 脚本原子化清空所有队列
-        const clearScript = `
-      redis.call('DEL', KEYS[1])
-      redis.call('DEL', KEYS[2])
-      redis.call('DEL', KEYS[3])
-      redis.call('DEL', KEYS[4])
-      return 1
-    `;
-        await this.redis.eval(clearScript, {
-            keys: [this.failedQueue, this.pendingQueue, this.completedQueue, this.processingQueue],
-            arguments: [],
-        });
-    }
-    /**
-     * 健康检查
-     */
-    async health() {
-        if (!this.redis)
-            return false;
-        const result = await catchIt(() => this.redis.ping());
-        return !result.isError() && result.value === 'PONG';
-    }
-}

package/output/redis-queue/batch-redis-queue.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=batch-redis-queue.test.d.ts.map

package/output/redis-queue/batch-redis-queue.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"batch-redis-queue.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/batch-redis-queue.test.ts"],"names":[],"mappings":""}