@taicode/common-server 1.0.12 → 1.0.13

package/output/redis-queue/redis-queue.js

@@ -1,557 +0,0 @@
-import { randomUUID } from 'node:crypto';
-import { createClient } from 'redis';
-import { catchIt } from '@taicode/common-base';
-/**
- * 通用任务队列类（泛型）
- *
- * 提供基于 Redis 的任务队列功能，支持：
- * - 任务入队和持久化
- * - 自动重试机制
- * - 任务状态追踪
- * - 分布式消费
- *
- * @template T 任务数据类型
- *
- * @example
- * ```ts
- * interface EmailTask {
- *   to: string
- *   subject: string
- * }
- *
- * const queue = new RedisQueue<EmailTask>({
- *   redisUrl: 'redis://localhost:6379',
- *   queueKey: 'email-queue',
- *   concurrency: 5,
- *   handler: async (data) => {
- *     await sendEmail(data.to, data.subject)
- *     return catchIt(() => {})
- *   }
- * })
- *
- * // 连接 Redis（自动启动消费者）
- * await queue.connect()
- *
- * // 入队任务（自动开始处理）
- * await queue.enqueue({ to: 'user@example.com', subject: 'Hello' })
- *
- * // 获取队列统计信息（O(1) 时间复杂度）
- * const stats = await queue.statistics()
- * console.log(stats) // { pending: 5, processing: 2, completed: 2, failed: 1 }
- * ```
- */
-export class RedisQueue {
-    consumerRunning = false;
-    redis = null;
-    consumerInterval = null;
-    recoveryInterval = null;
-    processingTasks = 0; // 当前正在处理的任务数
-    config;
-    handler;
-    // 不同状态队列的键名
-    failedQueue;
-    pendingQueue;
-    processingQueue;
-    completedQueue;
-    constructor(config) {
-        // 验证必填参数
-        if (!config.redisUrl && !config.redisClient) {
-            throw new Error('[TaskQueue] Either redisUrl or redisClient is required');
-        }
-        if (config.redisUrl && config.redisClient) {
-            throw new Error('[TaskQueue] Cannot specify both redisUrl and redisClient');
-        }
-        if (!config.queueKey) {
-            throw new Error('[TaskQueue] queueKey is required');
-        }
-        if (config.queueKey.length < 6) {
-            throw new Error('[TaskQueue] queueKey must be at least 6 characters long');
-        }
-        if (!config.handler) {
-            throw new Error('[TaskQueue] handler is required');
-        }
-        this.handler = config.handler;
-        this.config = {
-            queueKey: config.queueKey,
-            redisUrl: config.redisUrl,
-            redisClient: config.redisClient,
-            cleanupDelay: config.cleanupDelay || 86400, // 24 hours
-            maxRetries: config.maxRetries || 3,
-            concurrency: config.concurrency || 1,
-            consumerInterval: config.consumerInterval || 1000,
-            processingDelay: config.processingDelay || 0, // 默认立即执行
-            processingTimeout: config.processingTimeout || 60000, // 60 seconds
-        };
-        // 初始化不同状态队列的键名
-        this.failedQueue = `${config.queueKey}:failed`;
-        this.pendingQueue = `${config.queueKey}:pending`;
-        this.completedQueue = `${config.queueKey}:completed`;
-        this.processingQueue = `${config.queueKey}:processing`;
-        // 使用外部客户端或创建新客户端
-        if (config.redisClient) {
-            this.redis = config.redisClient;
-        }
-        else {
-            this.redis = createClient({ url: this.config.redisUrl });
-            // 添加错误处理
-            this.redis.on('error', (err) => {
-                console.error('[TaskQueue] Redis Client Error:', err);
-            });
-        }
-    }
-    /**
-     * 连接 Redis 并自动启动消费者
-     */
-    async connect() {
-        if (this.redis && !this.redis.isOpen) {
-            await this.redis.connect().catch((error) => {
-                console.error('[TaskQueue] Failed to connect to Redis:', error);
-                throw error;
-            });
-            // 连接成功后启动消费者和恢复机制
-            this.startConsumer();
-            this.startRecovery();
-        }
-    }
-    /**
-     * 断开 Redis 连接并停止消费者
-     */
-    disconnect() {
-        this.stopConsumer();
-        this.stopRecovery();
-        if (this.redis && this.redis.isOpen) {
-            this.redis.disconnect().catch((error) => {
-                console.error('[TaskQueue] Failed to disconnect:', error);
-            });
-        }
-    }
-    async enqueue(data) {
-        if (!this.redis) {
-            console.warn('[TaskQueue] Redis not available, skipping task enqueue');
-            return Array.isArray(data) ? [] : '';
-        }
-        // 统一处理为数组
-        const dataList = Array.isArray(data) ? data : [data];
-        const taskIds = [];
-        // 使用 Lua 脚本确保幂等入队的原子性
-        const enqueueScript = `
-      local taskKey = KEYS[1]
-      local pendingQueue = KEYS[2]
-      local taskData = ARGV[1]
-      local ttl = tonumber(ARGV[2])
-      local taskId = ARGV[3]
-      
-      -- 检查任务是否已存在
-      local exists = redis.call('EXISTS', taskKey)
-      if exists == 1 then
-        return 0  -- 任务已存在,跳过
-      end
-      
-      -- 原子化操作: 创建任务 + 推入队列
-      redis.call('SETEX', taskKey, ttl, taskData)
-      redis.call('RPUSH', pendingQueue, taskId)
-      return 1  -- 成功创建
-    `;
-        for (const item of dataList) {
-            // 检查 data 中是否包含自定义 id
-            const customId = item.id;
-            const taskId = customId || randomUUID();
-            const taskKey = `${this.config.queueKey}:task:${taskId}`;
-            const task = {
-                id: taskId,
-                data: item,
-                retryCount: 0,
-                maxRetries: this.config.maxRetries,
-                createdTime: new Date().toISOString(),
-                status: 'pending',
-                ...(this.config.processingDelay > 0 && { delayUntil: Date.now() + this.config.processingDelay }),
-            };
-            // 使用 Lua 脚本原子化入队
-            const result = await this.redis.eval(enqueueScript, {
-                keys: [taskKey, this.pendingQueue],
-                arguments: [JSON.stringify(task), this.config.cleanupDelay.toString(), taskId],
-            });
-            if (result === 0) {
-                console.log(`[TaskQueue] Task already exists, skipping: ${taskId}`);
-            }
-            else {
-                console.log(`[TaskQueue] Enqueued task: ${task.id}`);
-            }
-            taskIds.push(taskId);
-        }
-        // 返回单个 ID 或 ID 数组
-        return Array.isArray(data) ? taskIds : taskIds[0];
-    }
-    /**
-     * 获取任务详情
-     */
-    async getTask(taskId) {
-        if (!this.redis)
-            return null;
-        const taskKey = `${this.config.queueKey}:task:${taskId}`;
-        const taskData = await this.redis.get(taskKey);
-        if (!taskData)
-            return null;
-        return JSON.parse(taskData);
-    }
-    /**
-     * 更新任务状态并移动到对应队列（原子操作）
-     */
-    async applyStatus(taskId, oldStatus, newStatus) {
-        if (!this.redis)
-            return;
-        const task = await this.getTask(taskId);
-        if (!task)
-            return;
-        task.status = newStatus;
-        const taskKey = `${this.config.queueKey}:task:${taskId}`;
-        const oldQueue = this.getQueueByStatus(oldStatus);
-        const newQueue = this.getQueueByStatus(newStatus);
-        if (oldQueue !== newQueue) {
-            // 使用 Lua 脚本确保原子性：更新任务 + 移动队列
-            const script = `
-        redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
-        redis.call('LREM', KEYS[2], 0, ARGV[3])
-        redis.call('RPUSH', KEYS[3], ARGV[3])
-        return 1
-      `;
-            await this.redis.eval(script, {
-                keys: [taskKey, oldQueue, newQueue],
-                arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
-            });
-        }
-        else {
-            // 只更新任务数据
-            await this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
-        }
-    }
-    /**
-     * 根据状态获取对应的队列键
-     */
-    getQueueByStatus(status) {
-        switch (status) {
-            case 'pending': return this.pendingQueue;
-            case 'processing': return this.processingQueue;
-            case 'completed': return this.completedQueue;
-            case 'failed': return this.failedQueue;
-            default: return this.pendingQueue;
-        }
-    }
-    /**
-     * 恢复超时的任务
-     * 检查 processing 队列中的任务，将超时的任务重试或标记为失败
-     */
-    async recoverStalledTasks() {
-        if (!this.redis)
-            return;
-        try {
-            const processingTaskIds = await this.redis.lRange(this.processingQueue, 0, -1);
-            if (processingTaskIds.length === 0) {
-                return;
-            }
-            const now = Date.now();
-            let recoveredCount = 0;
-            for (const taskId of processingTaskIds) {
-                const task = await this.getTask(taskId);
-                if (!task) {
-                    // 任务不存在，从队列中移除
-                    await this.redis.lRem(this.processingQueue, 0, taskId);
-                    continue;
-                }
-                // 检查是否超时
-                const processingTime = now - (task.processingStartTime || now);
-                if (processingTime > this.config.processingTimeout) {
-                    console.log(`[TaskQueue] Task timeout: ${taskId} (processing time: ${processingTime}ms)`);
-                    const taskKey = `${this.config.queueKey}:task:${taskId}`;
-                    // 检查是否还可以重试
-                    if (task.retryCount < task.maxRetries) {
-                        // 重试：放回 pending 队列
-                        task.status = 'pending';
-                        task.retryCount++;
-                        task.processingStartTime = undefined;
-                        // 使用 Lua 脚本确保原子性
-                        const script = `
-              redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
-              redis.call('LREM', KEYS[2], 0, ARGV[3])
-              redis.call('RPUSH', KEYS[3], ARGV[3])
-              return 1
-            `;
-                        await this.redis.eval(script, {
-                            keys: [taskKey, this.processingQueue, this.pendingQueue],
-                            arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
-                        });
-                        console.log(`[TaskQueue] Task ${taskId} will retry (${task.retryCount}/${task.maxRetries})`);
-                    }
-                    else {
-                        // 已达到最大重试次数，标记为失败
-                        task.status = 'failed';
-                        task.processingStartTime = undefined;
-                        // 使用 Lua 脚本确保原子性
-                        const script = `
-              redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
-              redis.call('LREM', KEYS[2], 0, ARGV[3])
-              redis.call('RPUSH', KEYS[3], ARGV[3])
-              return 1
-            `;
-                        await this.redis.eval(script, {
-                            keys: [taskKey, this.processingQueue, this.failedQueue],
-                            arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
-                        });
-                        console.error(`[TaskQueue] Task ${taskId} failed after timeout`);
-                    }
-                    recoveredCount++;
-                }
-            }
-            if (recoveredCount > 0) {
-                console.log(`[TaskQueue] Recovered ${recoveredCount} timeout tasks`);
-            }
-        }
-        catch (error) {
-            console.error('[TaskQueue] Failed to recover stalled tasks:', error);
-        }
-    }
-    /**
-     * 处理单个任务
-     */
-    async processTask(taskId) {
-        this.processingTasks++;
-        try {
-            const task = await this.getTask(taskId);
-            if (!task) {
-                console.warn(`[TaskQueue] Task not found: ${taskId}`);
-                return;
-            }
-            // 任务应该是 pending 状态(Lua 脚本已确保延迟检查)
-            if (task.status !== 'pending') {
-                console.log(`[TaskQueue] Task ${taskId} has invalid status (${task.status}), marking as failed`);
-                await this.applyStatus(taskId, task.status, 'failed');
-                return;
-            }
-            try {
-                // 任务已在 processing 队列中(由 Lua 脚本完成),只需更新状态和开始时间
-                task.status = 'processing';
-                task.processingStartTime = Date.now();
-                const taskKey = `${this.config.queueKey}:task:${taskId}`;
-                await this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
-                // 执行任务处理器
-                await this.handler(task.data);
-                // 更新状态为完成
-                await this.applyStatus(taskId, 'processing', 'completed');
-                console.log(`[TaskQueue] Task completed: ${taskId}`);
-            }
-            catch (error) {
-                console.error(`[TaskQueue] Task failed: ${taskId}`, error);
-                // 检查是否需要重试
-                if (task.retryCount < task.maxRetries) {
-                    task.retryCount++;
-                    task.status = 'pending';
-                    task.processingStartTime = undefined;
-                    const taskKey = `${this.config.queueKey}:task:${taskId}`;
-                    // 使用 Lua 脚本确保原子性
-                    const script = `
-            redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
-            redis.call('LREM', KEYS[2], 0, ARGV[3])
-            redis.call('RPUSH', KEYS[3], ARGV[3])
-            return 1
-          `;
-                    await this.redis.eval(script, {
-                        keys: [taskKey, this.processingQueue, this.pendingQueue],
-                        arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
-                    });
-                    console.log(`[TaskQueue] Task ${taskId} will retry (${task.retryCount}/${task.maxRetries})`);
-                }
-                else {
-                    task.status = 'failed';
-                    task.processingStartTime = undefined;
-                    const taskKey = `${this.config.queueKey}:task:${taskId}`;
-                    // 使用 Lua 脚本确保原子性
-                    const script = `
-            redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
-            redis.call('LREM', KEYS[2], 0, ARGV[3])
-            redis.call('RPUSH', KEYS[3], ARGV[3])
-            return 1
-          `;
-                    await this.redis.eval(script, {
-                        keys: [taskKey, this.processingQueue, this.failedQueue],
-                        arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
-                    });
-                    console.error(`[TaskQueue] Task ${taskId} failed after ${task.maxRetries} retries`);
-                }
-            }
-        }
-        finally {
-            this.processingTasks--;
-        }
-    }
-    /**
-     * 启动恢复机制（内部方法，自动调用）
-     */
-    startRecovery() {
-        if (this.recoveryInterval || !this.redis) {
-            return;
-        }
-        // 立即执行一次恢复
-        this.recoverStalledTasks().catch(error => {
-            console.error('[TaskQueue] Initial recovery error:', error);
-        });
-        // 定期检查（每 10 秒检查一次）
-        this.recoveryInterval = setInterval(() => {
-            this.recoverStalledTasks().catch(error => {
-                console.error('[TaskQueue] Recovery error:', error);
-            });
-        }, 10000);
-        console.log('[TaskQueue] Recovery mechanism started');
-    }
-    /**
-     * 停止恢复机制（内部方法，自动调用）
-     */
-    stopRecovery() {
-        if (this.recoveryInterval) {
-            clearInterval(this.recoveryInterval);
-            this.recoveryInterval = null;
-            console.log('[TaskQueue] Recovery mechanism stopped');
-        }
-    }
-    /**
-     * 启动消费者（内部方法，自动调用）
-     */
-    startConsumer() {
-        if (this.consumerRunning || !this.redis) {
-            return;
-        }
-        this.consumerRunning = true;
-        console.log(`[TaskQueue] Consumer started with concurrency: ${this.config.concurrency}`);
-        // Lua 脚本: 原子化从 pending 取出任务并移到 processing(只取已到延迟时间的任务)
-        const popAndMoveScript = `
-      local pendingQueue = KEYS[1]
-      local processingQueue = KEYS[2]
-      local queueKeyPrefix = KEYS[3]
-      local count = tonumber(ARGV[1])
-      local currentTime = tonumber(ARGV[2])
-      
-      local taskIds = {}
-      local checkedCount = 0
-      local maxCheck = count * 3  -- 最多检查 count*3 个任务,避免无限循环
-      
-      while #taskIds < count and checkedCount < maxCheck do
-        local taskId = redis.call('LPOP', pendingQueue)
-        if not taskId then
-          break
-        end
-        
-        checkedCount = checkedCount + 1
-        
-        -- 获取任务详情检查延迟时间
-        local taskKey = queueKeyPrefix .. ':task:' .. taskId
-        local taskData = redis.call('GET', taskKey)
-        
-        if taskData then
-          local task = cjson.decode(taskData)
-          local delayUntil = task.delayUntil
-          
-          -- 如果没有延迟或已到延迟时间,则移到 processing
-          if not delayUntil or delayUntil <= currentTime then
-            redis.call('RPUSH', processingQueue, taskId)
-            table.insert(taskIds, taskId)
-          else
-            -- 未到延迟时间,放回队列尾部
-            redis.call('RPUSH', pendingQueue, taskId)
-          end
-        end
-      end
-      
-      return taskIds
-    `;
-        this.consumerInterval = setInterval(async () => {
-            try {
-                // 检查当前是否有可用的并发槽位
-                const availableSlots = this.config.concurrency - this.processingTasks;
-                if (availableSlots <= 0) {
-                    return; // 已达到并发限制，等待下一次轮询
-                }
-                // 使用 Lua 脚本原子化取出任务并移到 processing 队列
-                const taskIds = await this.redis.eval(popAndMoveScript, {
-                    keys: [this.pendingQueue, this.processingQueue, this.config.queueKey],
-                    arguments: [availableSlots.toString(), Date.now().toString()],
-                });
-                // 并发处理所有任务（不等待完成）
-                taskIds.forEach(taskId => {
-                    this.processTask(taskId).catch(error => {
-                        console.error(`[TaskQueue] Unhandled error in processTask: ${taskId}`, error);
-                    });
-                });
-            }
-            catch (error) {
-                console.error('[TaskQueue] Consumer error:', error);
-            }
-        }, this.config.consumerInterval);
-    }
-    /**
-     * 停止消费者（内部方法，自动调用）
-     */
-    stopConsumer() {
-        if (this.consumerInterval) {
-            clearInterval(this.consumerInterval);
-            this.consumerInterval = null;
-        }
-        this.consumerRunning = false;
-        console.log('[TaskQueue] Consumer stopped');
-    }
-    /**
-     * 获取队列统计信息
-     * 返回队列中各种状态任务的详细数量
-     *
-     * 使用分离队列设计，O(1) 时间复杂度
-     */
-    async statistics() {
-        if (!this.redis) {
-            return {
-                failed: 0,
-                pending: 0,
-                completed: 0,
-                processing: 0,
-            };
-        }
-        // 并行获取各队列长度，O(1) 时间复杂度
-        const [pending, processing, completed, failed] = await Promise.all([
-            this.redis.lLen(this.pendingQueue),
-            this.redis.lLen(this.processingQueue),
-            this.redis.lLen(this.completedQueue),
-            this.redis.lLen(this.failedQueue),
-        ]);
-        return {
-            pending,
-            processing,
-            completed,
-            failed,
-        };
-    }
-    /**
-     * 清空所有队列
-     */
-    async clear() {
-        if (!this.redis)
-            return;
-        // 使用 Lua 脚本原子化清空所有队列
-        const clearScript = `
-      redis.call('DEL', KEYS[1])
-      redis.call('DEL', KEYS[2])
-      redis.call('DEL', KEYS[3])
-      redis.call('DEL', KEYS[4])
-      return 1
-    `;
-        await this.redis.eval(clearScript, {
-            keys: [this.failedQueue, this.pendingQueue, this.completedQueue, this.processingQueue],
-            arguments: [],
-        });
-    }
-    /**
-     * 健康检查
-     */
-    async health() {
-        if (!this.redis)
-            return false;
-        const result = await catchIt(() => this.redis.ping());
-        return !result.isError() && result.value === 'PONG';
-    }
-}

package/output/redis-queue/redis-queue.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=redis-queue.test.d.ts.map

package/output/redis-queue/redis-queue.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"redis-queue.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue.test.ts"],"names":[],"mappings":""}