@taicode/common-server 1.0.11 → 1.0.13

package/output/index.d.ts

@@ -1,2 +1,3 @@
 export * from './schema';
+export * from './redis-queue';
 //# sourceMappingURL=index.d.ts.map

package/output/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../source/index.ts"],"names":[],"mappings":"AACA,cAAc,UAAU,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../source/index.ts"],"names":[],"mappings":"AACA,cAAc,UAAU,CAAA;AACxB,cAAc,eAAe,CAAA"}

package/output/index.js

@@ -1,2 +1,3 @@
 // 根 index.ts，统一导出 mock 目录下所有能力
 export * from './schema';
+export * from './redis-queue';

package/output/redis-queue/index.d.ts

@@ -1,5 +1,7 @@
-export { RedisQueue } from './redis-queue';
-export { BatchRedisQueue } from './batch-redis-queue';
-export type { TaskQueueConfig, BatchTaskQueueConfig } from './types';
-export type { Status, TaskData, Task, TaskHandler, BatchTaskHandler, QueueStats } from './types';
+export { RedisQueueProvider } from './redis-queue-provider';
+export { RedisQueueConsumer } from './redis-queue-consumer';
+export { RedisBatchConsumer } from './redis-batch-consumer';
+export type { RedisQueueProviderConfig, RedisQueueConsumerConfig, BatchConsumerConfig } from './types';
+export type { TaskData } from './types';
+export type { RedisQueueRegistry } from './types';
 //# sourceMappingURL=index.d.ts.map

package/output/redis-queue/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../source/redis-queue/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AACrD,YAAY,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAA;AACpE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../source/redis-queue/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAG3D,YAAY,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAA;AAGtG,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAGvC,YAAY,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAA"}

package/output/redis-queue/index.js

@@ -1,2 +1,4 @@
-export { RedisQueue } from './redis-queue';
-export { BatchRedisQueue } from './batch-redis-queue';
+// 核心类导出
+export { RedisQueueProvider } from './redis-queue-provider';
+export { RedisQueueConsumer } from './redis-queue-consumer';
+export { RedisBatchConsumer } from './redis-batch-consumer';

package/output/redis-queue/redis-batch-consumer.d.ts

@@ -0,0 +1,80 @@
+import { RedisQueueCommon } from './redis-queue-common';
+import type { BatchConsumerConfig, QueueStats, RedisQueueRegistry } from './types';
+/**
+ * Redis 批量队列消费者
+ *
+ * 批量消费任务,每次处理多条
+ *
+ * @template K 队列键名类型
+ *
+ * @example
+ * ```ts
+ * interface EmailTask {
+ *   to: string
+ * }
+ *
+ * const consumer = new RedisBatchConsumer<EmailTask>({
+ *   redisUrl: 'redis://localhost:6379',
+ *   queueKey: 'email-queue',
+ *   batchSize: 50,
+ *   handler: async (dataList) => {
+ *     await sendEmailsBatch(dataList.map(d => d.to))
+ *     return catchIt(() => {})
+ *   }
+ * })
+ *
+ * await consumer.connect() // 自动开始消费
+ *
+ * // 获取统计信息
+ * const stats = await consumer.statistics()
+ *
+ * // 停止消费
+ * consumer.disconnect()
+ * ```
+ */
+export declare class RedisBatchConsumer<K extends keyof RedisQueueRegistry> extends RedisQueueCommon {
+    private consumerRunning;
+    private consumerInterval;
+    private recoveryInterval;
+    private processingBatches;
+    private readonly config;
+    constructor(config: BatchConsumerConfig<K>);
+    protected getLogPrefix(): string;
+    /**
+     * 连接 Redis 并自动启动消费者
+     */
+    connect(): Promise<void>;
+    /**
+     * 断开 Redis 连接并停止消费者
+     */
+    disconnect(): void;
+    /**
+     * 批量处理任务
+     */
+    private processBatch;
+    /**
+     * 启动恢复机制
+     */
+    private startRecovery;
+    /**
+     * 停止恢复机制
+     */
+    private stopRecovery;
+    /**
+     * 启动消费者
+     */
+    private startConsumer;
+    /**
+     * 停止消费者
+     */
+    private stopConsumer;
+    /**
+     * 获取队列统计信息（O(1) 时间复杂度）
+     */
+    statistics(): Promise<QueueStats>;
+    /**
+     * 健康检查
+     */
+    health(): Promise<boolean>;
+}
+//# sourceMappingURL=redis-batch-consumer.d.ts.map

package/output/redis-queue/redis-batch-consumer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redis-batch-consumer.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-batch-consumer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AACvD,OAAO,KAAK,EAAE,mBAAmB,EAA0B,UAAU,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAA;AAE1G;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,kBAAkB,CAAC,CAAC,SAAS,MAAM,kBAAkB,CAAE,SAAQ,gBAAgB;IAC1F,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,iBAAiB,CAAI;IAE7B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAOtB;gBAEW,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;IA2B1C,SAAS,CAAC,YAAY,IAAI,MAAM;IAIhC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAMlB;;OAEG;YACW,YAAY;IA4F1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACH,OAAO,CAAC,aAAa;IAyErB;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC;IAevC;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAMjC"}

package/output/redis-queue/redis-batch-consumer.js

@@ -0,0 +1,308 @@
+import { catchIt } from '@taicode/common-base';
+import { RedisQueueCommon } from './redis-queue-common';
+/**
+ * Redis 批量队列消费者
+ *
+ * 批量消费任务,每次处理多条
+ *
+ * @template K 队列键名类型
+ *
+ * @example
+ * ```ts
+ * interface EmailTask {
+ *   to: string
+ * }
+ *
+ * const consumer = new RedisBatchConsumer<EmailTask>({
+ *   redisUrl: 'redis://localhost:6379',
+ *   queueKey: 'email-queue',
+ *   batchSize: 50,
+ *   handler: async (dataList) => {
+ *     await sendEmailsBatch(dataList.map(d => d.to))
+ *     return catchIt(() => {})
+ *   }
+ * })
+ *
+ * await consumer.connect() // 自动开始消费
+ *
+ * // 获取统计信息
+ * const stats = await consumer.statistics()
+ *
+ * // 停止消费
+ * consumer.disconnect()
+ * ```
+ */
+export class RedisBatchConsumer extends RedisQueueCommon {
+    consumerRunning = false;
+    consumerInterval = null;
+    recoveryInterval = null;
+    processingBatches = 0; // 当前正在处理的批次数
+    config;
+    constructor(config) {
+        // 验证必填参数
+        if (!config.handler) {
+            throw new Error('[RedisBatchConsumer] handler is required');
+        }
+        // 调用父类构造函数
+        super({
+            redisUrl: config.redisUrl,
+            redisClient: config.redisClient,
+            queueKey: config.queueKey,
+            cleanupDelay: config.cleanupDelay ?? 86400,
+        });
+        this.config = {
+            handler: config.handler,
+            queueKey: config.queueKey,
+            redisUrl: config.redisUrl,
+            redisClient: config.redisClient,
+            batchSize: config.batchSize ?? 10,
+            maxRetries: config.maxRetries ?? 3,
+            concurrency: config.concurrency ?? 1,
+            consumerInterval: config.consumerInterval ?? 1000,
+            processingTimeout: config.processingTimeout ?? 60000, // 60 seconds
+        };
+    }
+    getLogPrefix() {
+        return 'RedisBatchConsumer';
+    }
+    /**
+     * 连接 Redis 并自动启动消费者
+     */
+    async connect() {
+        await super.connect();
+        // 连接成功后启动消费者和恢复机制
+        this.startConsumer();
+        this.startRecovery();
+    }
+    /**
+     * 断开 Redis 连接并停止消费者
+     */
+    disconnect() {
+        this.stopConsumer();
+        this.stopRecovery();
+        super.disconnect();
+    }
+    /**
+     * 批量处理任务
+     */
+    async processBatch(taskIds) {
+        this.processingBatches++;
+        try {
+            // 获取所有任务
+            const tasks = await Promise.all(taskIds.map(id => this.getTask(id)));
+            const validTasks = tasks.filter((task) => task !== null);
+            if (validTasks.length === 0) {
+                console.warn(`[RedisBatchConsumer] No valid tasks found in batch`);
+                return;
+            }
+            // 过滤出有效的待处理任务
+            const tasksToProcess = [];
+            for (const task of validTasks) {
+                // 检查状态
+                if (task.status !== 'pending') {
+                    console.log(`[RedisBatchConsumer] Task ${task.id} has invalid status (${task.status}), marking as failed`);
+                    await this.applyStatus(task.id, task.status, 'failed');
+                    continue;
+                }
+                tasksToProcess.push(task);
+            }
+            if (tasksToProcess.length === 0) {
+                return;
+            }
+            try {
+                // 任务已在 processing 队列中(由 Lua 脚本完成),只需更新状态和开始时间
+                const taskIdList = tasksToProcess.map(t => t.id);
+                const now = Date.now();
+                await Promise.all(tasksToProcess.map(task => {
+                    task.status = 'processing';
+                    task.processingStartTime = now;
+                    const taskKey = `${this.queueKey}:task:${task.id}`;
+                    return this.redis.setEx(taskKey, this.cleanupDelay, JSON.stringify(task));
+                }));
+                // 批量处理所有任务
+                const dataList = tasksToProcess.map(t => t.data);
+                await this.config.handler(dataList);
+                // 批量更新状态为完成
+                const updatedCount = await this.applyStatusBatch(taskIdList, 'processing', 'completed');
+                console.log(`[RedisBatchConsumer] Batch completed: ${updatedCount} tasks`);
+            }
+            catch (error) {
+                console.error(`[RedisBatchConsumer] Batch failed:`, error);
+                // 批量重试或失败
+                for (const task of tasksToProcess) {
+                    if (task.retryCount < task.maxRetries) {
+                        task.retryCount++;
+                        task.status = 'pending';
+                        task.processingStartTime = undefined;
+                        const taskKey = `${this.queueKey}:task:${task.id}`;
+                        const script = `
+              redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
+              redis.call('LREM', KEYS[2], 0, ARGV[3])
+              redis.call('RPUSH', KEYS[3], ARGV[3])
+              return 1
+            `;
+                        await this.redis.eval(script, {
+                            keys: [taskKey, this.processingQueue, this.pendingQueue],
+                            arguments: [this.cleanupDelay.toString(), JSON.stringify(task), task.id],
+                        });
+                        console.log(`[RedisBatchConsumer] Task ${task.id} will retry (${task.retryCount}/${task.maxRetries})`);
+                    }
+                    else {
+                        task.status = 'failed';
+                        task.processingStartTime = undefined;
+                        const taskKey = `${this.queueKey}:task:${task.id}`;
+                        const script = `
+              redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
+              redis.call('LREM', KEYS[2], 0, ARGV[3])
+              redis.call('RPUSH', KEYS[3], ARGV[3])
+              return 1
+            `;
+                        await this.redis.eval(script, {
+                            keys: [taskKey, this.processingQueue, this.failedQueue],
+                            arguments: [this.cleanupDelay.toString(), JSON.stringify(task), task.id],
+                        });
+                        console.error(`[RedisBatchConsumer] Task ${task.id} failed after ${task.maxRetries} retries`);
+                    }
+                }
+            }
+        }
+        finally {
+            this.processingBatches--;
+        }
+    }
+    /**
+     * 启动恢复机制
+     */
+    startRecovery() {
+        if (this.recoveryInterval || !this.redis) {
+            return;
+        }
+        // 立即执行一次恢复
+        this.recoverStalled(this.config.processingTimeout).catch(error => {
+            console.error('[RedisBatchConsumer] Initial recovery error:', error);
+        });
+        // 定期检查（每 10 秒检查一次）
+        this.recoveryInterval = setInterval(() => {
+            this.recoverStalled(this.config.processingTimeout).catch(error => {
+                console.error('[RedisBatchConsumer] Recovery error:', error);
+            });
+        }, 10000);
+        console.log('[RedisBatchConsumer] Recovery mechanism started');
+    }
+    /**
+     * 停止恢复机制
+     */
+    stopRecovery() {
+        if (this.recoveryInterval) {
+            clearInterval(this.recoveryInterval);
+            this.recoveryInterval = null;
+            console.log('[RedisBatchConsumer] Recovery mechanism stopped');
+        }
+    }
+    /**
+     * 启动消费者
+     */
+    startConsumer() {
+        if (this.consumerRunning || !this.redis) {
+            return;
+        }
+        this.consumerRunning = true;
+        console.log(`[RedisBatchConsumer] Consumer started with batchSize: ${this.config.batchSize}, concurrency: ${this.config.concurrency}`);
+        // Lua 脚本: 批量原子化从 pending 取出任务并移到 processing
+        const popAndMoveBatchScript = `
+      local pendingQueue = KEYS[1]
+      local processingQueue = KEYS[2]
+      local queueKeyPrefix = KEYS[3]
+      local count = tonumber(ARGV[1])
+      local currentTime = tonumber(ARGV[2])
+      
+      local taskIds = {}
+      local checkedCount = 0
+      local maxCheck = count * 3
+      
+      while #taskIds < count and checkedCount < maxCheck do
+        local taskId = redis.call('LPOP', pendingQueue)
+        if not taskId then
+          break
+        end
+        
+        checkedCount = checkedCount + 1
+        
+        local taskKey = queueKeyPrefix .. ':task:' .. taskId
+        local taskData = redis.call('GET', taskKey)
+        
+        if taskData then
+          local task = cjson.decode(taskData)
+          local delayUntil = task.delayUntil
+          
+          if not delayUntil or delayUntil <= currentTime then
+            redis.call('RPUSH', processingQueue, taskId)
+            table.insert(taskIds, taskId)
+          else
+            redis.call('RPUSH', pendingQueue, taskId)
+          end
+        end
+      end
+      
+      return taskIds
+    `;
+        this.consumerInterval = setInterval(async () => {
+            try {
+                // 检查当前是否有可用的并发槽位
+                const availableSlots = this.config.concurrency - this.processingBatches;
+                if (availableSlots <= 0) {
+                    return; // 已达到并发限制
+                }
+                // 批量取出任务
+                const taskIds = await this.redis.eval(popAndMoveBatchScript, {
+                    keys: [this.pendingQueue, this.processingQueue, this.queueKey],
+                    arguments: [this.config.batchSize.toString(), Date.now().toString()],
+                });
+                if (taskIds.length > 0) {
+                    // 处理这批任务（不等待完成）
+                    this.processBatch(taskIds).catch(error => {
+                        console.error(`[RedisBatchConsumer] Unhandled error in processBatch:`, error);
+                    });
+                }
+            }
+            catch (error) {
+                console.error('[RedisBatchConsumer] Consumer error:', error);
+            }
+        }, this.config.consumerInterval);
+    }
+    /**
+     * 停止消费者
+     */
+    stopConsumer() {
+        if (this.consumerInterval) {
+            clearInterval(this.consumerInterval);
+            this.consumerInterval = null;
+        }
+        this.consumerRunning = false;
+        console.log('[RedisBatchConsumer] Consumer stopped');
+    }
+    /**
+     * 获取队列统计信息（O(1) 时间复杂度）
+     */
+    async statistics() {
+        if (!this.redis) {
+            return { pending: 0, processing: 0, completed: 0, failed: 0 };
+        }
+        const [pending, processing, completed, failed] = await Promise.all([
+            this.redis.lLen(this.pendingQueue),
+            this.redis.lLen(this.processingQueue),
+            this.redis.lLen(this.completedQueue),
+            this.redis.lLen(this.failedQueue),
+        ]);
+        return { pending, processing, completed, failed };
+    }
+    /**
+     * 健康检查
+     */
+    async health() {
+        if (!this.redis)
+            return false;
+        const result = await catchIt(() => this.redis.ping());
+        return !result.isError() && result.value === 'PONG';
+    }
+}

package/output/redis-queue/redis-batch-consumer.test.d.ts

@@ -0,0 +1,7 @@
+declare module './types' {
+    interface RedisQueueRegistry {
+        'test-queue': Record<string, any>;
+    }
+}
+export {};
+//# sourceMappingURL=redis-batch-consumer.test.d.ts.map

package/output/redis-queue/redis-batch-consumer.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redis-batch-consumer.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-batch-consumer.test.ts"],"names":[],"mappings":"AAOA,OAAO,QAAQ,SAAS,CAAC;IACvB,UAAU,kBAAkB;QAC1B,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAClC;CACF"}