@taicode/common-server 1.0.8 → 1.0.10

package/output/redis-queue/batch-redis-queue.test.js

@@ -0,0 +1,398 @@
+import { describe, it, expect } from 'vitest';
+import { catchIt } from '@taicode/common-base';
+import { BatchRedisQueue } from './batch-redis-queue';
+const REDIS_URL = 'redis://localhost:6379';
+describe('BatchRedisQueue', () => {
+    describe('连接管理', () => {
+        it('应该能够成功连接和断开 Redis', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: 'test-connection',
+            });
+            await queue.connect();
+            const health = await queue.health();
+            expect(health).toBe(true);
+            queue.disconnect();
+        });
+        it('缺少必填参数应该抛出错误', () => {
+            expect(() => {
+                new BatchRedisQueue({
+                    redisUrl: '',
+                    queueKey: 'test',
+                });
+            }).toThrow('[BatchTaskQueue] redisUrl is required');
+            expect(() => {
+                new BatchRedisQueue({
+                    redisUrl: REDIS_URL,
+                    queueKey: '',
+                });
+            }).toThrow('[BatchTaskQueue] queueKey is required');
+            expect(() => {
+                new BatchRedisQueue({
+                    redisUrl: REDIS_URL,
+                    queueKey: 'short',
+                });
+            }).toThrow('[BatchTaskQueue] queueKey must be at least 6 characters long');
+        });
+    });
+    describe('批量任务处理', () => {
+        it('应该批量处理任务', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-batch-process-${Date.now()}`,
+                batchSize: 5,
+            });
+            await queue.connect();
+            const processedBatches = [];
+            queue.handle('batch-task', async (dataList) => {
+                processedBatches.push(dataList.map(d => d.value));
+                return catchIt(() => { });
+            });
+            // 入队 10 个任务
+            await queue.enqueue('batch-task', Array.from({ length: 10 }, (_, i) => ({ value: i })));
+            // 等待批量处理
+            await new Promise(resolve => setTimeout(resolve, 2500));
+            // 应该处理了 2 批，每批 5 个
+            expect(processedBatches.length).toBe(2);
+            expect(processedBatches[0].length).toBe(5);
+            expect(processedBatches[1].length).toBe(5);
+            queue.disconnect();
+        });
+        it('应该按类型分组批量处理', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-batch-by-type-${Date.now()}`,
+                batchSize: 10,
+            });
+            await queue.connect();
+            const emailBatches = [];
+            const smsBatches = [];
+            queue.handle('send-email', async (dataList) => {
+                emailBatches.push(dataList);
+                return catchIt(() => { });
+            });
+            queue.handle('send-sms', async (dataList) => {
+                smsBatches.push(dataList);
+                return catchIt(() => { });
+            });
+            // 混合入队不同类型的任务
+            await queue.enqueue('send-email', [
+                { to: 'user1@example.com' },
+                { to: 'user2@example.com' },
+            ]);
+            await queue.enqueue('send-sms', [
+                { phone: '123456' },
+                { phone: '789012' },
+            ]);
+            await queue.enqueue('send-email', { to: 'user3@example.com' });
+            // 等待处理
+            await new Promise(resolve => setTimeout(resolve, 2000));
+            // 应该分别处理 email 和 sms
+            expect(emailBatches.length).toBe(1);
+            expect(emailBatches[0].length).toBe(3);
+            expect(smsBatches.length).toBe(1);
+            expect(smsBatches[0].length).toBe(2);
+            queue.disconnect();
+        });
+        it('未注册处理器的任务应该标记为失败', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-no-handler-${Date.now()}`,
+                batchSize: 5,
+            });
+            await queue.connect();
+            await queue.enqueue('unknown-task', [{ value: 1 }, { value: 2 }]);
+            // 等待处理
+            await new Promise(resolve => setTimeout(resolve, 1500));
+            const stats = await queue.statistics();
+            expect(stats.failed).toBe(2);
+            queue.disconnect();
+        });
+    });
+    describe('错误处理和重试', () => {
+        it('失败的批次应该自动重试', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-batch-retry-${Date.now()}`,
+                batchSize: 3,
+                maxRetries: 2,
+            });
+            await queue.connect();
+            let attemptCount = 0;
+            queue.handle('retry-batch', async () => {
+                attemptCount++;
+                if (attemptCount < 2) {
+                    throw new Error('Simulated batch failure');
+                }
+                return catchIt(() => { });
+            });
+            await queue.enqueue('retry-batch', [{ value: 1 }, { value: 2 }]);
+            // 等待重试
+            await new Promise(resolve => setTimeout(resolve, 3000));
+            expect(attemptCount).toBe(2);
+            queue.disconnect();
+        });
+        it('超过最大重试次数应该标记为失败', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-max-retries-${Date.now()}`,
+                batchSize: 2,
+                maxRetries: 1,
+            });
+            await queue.connect();
+            queue.handle('always-fail', async () => {
+                throw new Error('Always fails');
+            });
+            await queue.enqueue('always-fail', [{ value: 1 }, { value: 2 }]);
+            // 等待处理和重试
+            await new Promise(resolve => setTimeout(resolve, 2500));
+            const stats = await queue.statistics();
+            expect(stats.failed).toBe(2);
+            queue.disconnect();
+        });
+    });
+    describe('幂等性', () => {
+        it('应该支持在 data 中指定 id 来实现幂等性', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-batch-idempotent-${Date.now()}`,
+                batchSize: 10,
+            });
+            await queue.connect();
+            let processCount = 0;
+            queue.handle('idempotent-batch', async (dataList) => {
+                processCount += dataList.length;
+                return catchIt(() => { });
+            });
+            // 使用相同的 id 多次入队
+            await queue.enqueue('idempotent-batch', { id: 'unique-1', value: 1 });
+            await queue.enqueue('idempotent-batch', { id: 'unique-1', value: 2 });
+            await queue.enqueue('idempotent-batch', { id: 'unique-2', value: 3 });
+            // 等待处理
+            await new Promise(resolve => setTimeout(resolve, 1500));
+            // 应该只处理 2 个任务（unique-1 和 unique-2）
+            expect(processCount).toBe(2);
+            const stats = await queue.statistics();
+            expect(stats.completed).toBe(2);
+            queue.disconnect();
+        });
+    });
+    describe('延迟处理', () => {
+        it('应该支持延迟批量处理', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-batch-delay-${Date.now()}`,
+                batchSize: 3,
+                processingDelay: 2000,
+            });
+            await queue.connect();
+            let processCount = 0;
+            const startTime = Date.now();
+            let processTime = 0;
+            queue.handle('delay-batch', async (dataList) => {
+                processCount += dataList.length;
+                processTime = Date.now() - startTime;
+                return catchIt(() => { });
+            });
+            await queue.enqueue('delay-batch', [{ value: 1 }, { value: 2 }, { value: 3 }]);
+            // 等待 1 秒，任务应该还未处理
+            await new Promise(resolve => setTimeout(resolve, 1000));
+            expect(processCount).toBe(0);
+            // 再等待 2.5 秒，任务应该已处理
+            await new Promise(resolve => setTimeout(resolve, 2500));
+            expect(processCount).toBe(3);
+            expect(processTime).toBeGreaterThanOrEqual(1900);
+            queue.disconnect();
+        });
+    });
+    describe('队列操作', () => {
+        it('应该正确返回队列统计信息', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-batch-stats-${Date.now()}`,
+                batchSize: 5,
+            });
+            await queue.connect();
+            queue.handle('stats-batch', async () => {
+                await new Promise(resolve => setTimeout(resolve, 100));
+                return catchIt(() => { });
+            });
+            // 入队 10 个任务
+            await queue.enqueue('stats-batch', Array.from({ length: 10 }, (_, i) => ({ value: i })));
+            // 立即检查统计
+            const stats1 = await queue.statistics();
+            expect(stats1.pending).toBeGreaterThan(0);
+            // 等待处理
+            await new Promise(resolve => setTimeout(resolve, 3000));
+            const stats2 = await queue.statistics();
+            expect(stats2.completed).toBe(10);
+            queue.disconnect();
+        });
+        it('应该能够清空队列', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-clear-batch-${Date.now()}`,
+            });
+            await queue.connect();
+            await queue.enqueue('clear-batch', [{ value: 1 }, { value: 2 }]);
+            await queue.clear();
+            const stats = await queue.statistics();
+            expect(stats.pending).toBe(0);
+            queue.disconnect();
+        });
+    });
+    describe('大批量场景', () => {
+        it('应该能够处理大量任务', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-large-batch-${Date.now()}`,
+                batchSize: 50,
+            });
+            await queue.connect();
+            let processedCount = 0;
+            queue.handle('large-batch', async (dataList) => {
+                processedCount += dataList.length;
+                return catchIt(() => { });
+            });
+            // 入队 200 个任务
+            await queue.enqueue('large-batch', Array.from({ length: 200 }, (_, i) => ({ value: i })));
+            // 等待批量处理
+            await new Promise(resolve => setTimeout(resolve, 6000));
+            expect(processedCount).toBe(200);
+            const stats = await queue.statistics();
+            expect(stats.completed).toBe(200);
+            queue.disconnect();
+        }, 10000); // 增加超时到 10 秒
+    });
+    describe('并发安全性', () => {
+        it('多个批量队列实例应该能安全地处理任务', async () => {
+            const queueKey = `test-batch-concurrent-${Date.now()}`;
+            const queue1 = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey,
+                batchSize: 5,
+            });
+            const queue2 = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey,
+                batchSize: 5,
+            });
+            await queue1.connect();
+            await queue2.connect();
+            let totalProcessed = 0;
+            const handler = async (dataList) => {
+                totalProcessed += dataList.length;
+                await new Promise(resolve => setTimeout(resolve, 100));
+                return catchIt(() => { });
+            };
+            queue1.handle('concurrent-batch', handler);
+            queue2.handle('concurrent-batch', handler);
+            // 添加 20 个任务
+            await queue1.enqueue('concurrent-batch', Array.from({ length: 20 }, (_, i) => ({ value: i })));
+            // 等待处理
+            await new Promise(resolve => setTimeout(resolve, 3000));
+            // 所有任务都应该被处理一次
+            expect(totalProcessed).toBe(20);
+            queue1.disconnect();
+            queue2.disconnect();
+        });
+    });
+    describe('批量数据边界测试', () => {
+        it('应该能够处理包含大量数据的批次', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-large-batch-data-${Date.now()}`,
+                batchSize: 10,
+            });
+            await queue.connect();
+            let receivedBatch = false;
+            queue.handle('large-data-batch', async (dataList) => {
+                expect(dataList.length).toBe(10);
+                dataList.forEach(data => {
+                    expect(data.content.length).toBe(1000);
+                });
+                receivedBatch = true;
+                return catchIt(() => { });
+            });
+            // 每个任务 1KB,10 个任务共 10KB
+            await queue.enqueue('large-data-batch', Array.from({ length: 10 }, () => ({
+                content: 'x'.repeat(1000)
+            })));
+            await new Promise(resolve => setTimeout(resolve, 2000));
+            expect(receivedBatch).toBe(true);
+            queue.disconnect();
+        });
+    });
+    describe('批量错误处理边界测试', () => {
+        it('批量处理失败应该重试所有任务', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-batch-retry-${Date.now()}`,
+                batchSize: 3,
+                maxRetries: 2,
+            });
+            await queue.connect();
+            let attemptCount = 0;
+            queue.handle('batch-retry', async (dataList) => {
+                attemptCount++;
+                if (attemptCount < 2) {
+                    throw new Error('Batch processing failed');
+                }
+                return catchIt(() => { });
+            });
+            await queue.enqueue('batch-retry', [{ value: 1 }, { value: 2 }, { value: 3 }]);
+            await new Promise(resolve => setTimeout(resolve, 3000));
+            // 第一次失败,第二次成功
+            expect(attemptCount).toBe(2);
+            const stats = await queue.statistics();
+            expect(stats.completed).toBe(3);
+            expect(stats.failed).toBe(0);
+            queue.disconnect();
+        });
+        it('批量处理超过最大重试次数应该标记为失败', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-batch-max-retry-${Date.now()}`,
+                batchSize: 3,
+                maxRetries: 1,
+            });
+            await queue.connect();
+            let attemptCount = 0;
+            queue.handle('batch-max-retry', async () => {
+                attemptCount++;
+                throw new Error('Always fails');
+            });
+            await queue.enqueue('batch-max-retry', [{ value: 1 }, { value: 2 }, { value: 3 }]);
+            await new Promise(resolve => setTimeout(resolve, 3000));
+            // 初始 + 1 次重试 = 2 次
+            expect(attemptCount).toBe(2);
+            const stats = await queue.statistics();
+            expect(stats.failed).toBe(3);
+            expect(stats.completed).toBe(0);
+            queue.disconnect();
+        });
+    });
+    describe('批量延迟处理边界测试', () => {
+        it('延迟时间为 0 的批量任务应该立即处理', async () => {
+            const queue = new BatchRedisQueue({
+                redisUrl: REDIS_URL,
+                queueKey: `test-batch-zero-delay-${Date.now()}`,
+                batchSize: 5,
+                processingDelay: 0,
+            });
+            await queue.connect();
+            const startTime = Date.now();
+            let processTime = 0;
+            queue.handle('zero-delay-batch', async () => {
+                processTime = Date.now() - startTime;
+                return catchIt(() => { });
+            });
+            await queue.enqueue('zero-delay-batch', Array.from({ length: 5 }, (_, i) => ({ value: i })));
+            await new Promise(resolve => setTimeout(resolve, 2000));
+            // 应该在 2 秒内处理
+            expect(processTime).toBeGreaterThan(0);
+            expect(processTime).toBeLessThan(2000);
+            queue.disconnect();
+        });
+    });
+});

package/output/redis-queue/index.d.ts

@@ -0,0 +1,5 @@
+export { RedisQueue } from './redis-queue';
+export { BatchRedisQueue } from './batch-redis-queue';
+export type { TaskQueueConfig, BatchTaskQueueConfig } from './types';
+export type { Status, TaskData, Task, TaskHandler, BatchTaskHandler, QueueStats } from './types';
+//# sourceMappingURL=index.d.ts.map

package/output/redis-queue/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../source/redis-queue/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AACrD,YAAY,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAA;AACpE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA"}

package/output/redis-queue/index.js

@@ -0,0 +1,2 @@
+export { RedisQueue } from './redis-queue';
+export { BatchRedisQueue } from './batch-redis-queue';

package/output/redis-queue/redis-queue.d.ts

@@ -0,0 +1,128 @@
+import type { TaskQueueConfig, TaskData, Task, TaskHandler, QueueStats } from './types';
+/**
+ * 通用任务队列类
+ *
+ * 提供基于 Redis 的任务队列功能，支持：
+ * - 任务入队和持久化
+ * - 自动重试机制
+ * - 任务状态追踪
+ * - 分布式消费
+ *
+ * @example
+ * ```ts
+ * const queue = new RedisQueue({
+ *   redisUrl: 'redis://localhost:6379',
+ *   queueKey: 'my-queue',
+ *   concurrency: 5, // 并发处理 5 个任务
+ * })
+ *
+ * // 注册任务处理器
+ * queue.handle('send-email', async (data) => {
+ *   await sendEmail(data.to, data.subject)
+ * })
+ *
+ * // 连接 Redis（自动启动消费者）
+ * await queue.connect()
+ *
+ * // 入队任务（自动开始处理）
+ * await queue.enqueue('send-email', { to: 'user@example.com', subject: 'Hello' })
+ *
+ * // 获取队列统计信息（O(1) 时间复杂度）
+ * const stats = await queue.statistics()
+ * console.log(stats) // { pending: 5, processing: 2, completed: 2, failed: 1 }
+ * ```
+ */
+export declare class RedisQueue {
+    private consumerRunning;
+    private redis;
+    private consumerInterval;
+    private recoveryInterval;
+    private processingTasks;
+    private readonly config;
+    private readonly handlers;
+    private readonly failedQueue;
+    private readonly pendingQueue;
+    private readonly processingQueue;
+    private readonly completedQueue;
+    constructor(config: TaskQueueConfig);
+    /**
+     * 连接 Redis 并自动启动消费者
+     */
+    connect(): Promise<void>;
+    /**
+     * 断开 Redis 连接并停止消费者
+     */
+    disconnect(): void;
+    /**
+     * 注册任务处理器
+     */
+    handle<T extends TaskData = TaskData>(type: string, handler: TaskHandler<T>): void;
+    /**
+     * 将任务推入队列(支持单个或批量)
+     * @param type 任务类型
+     * @param data 任务数据(单个或数组)。可以在 data 中包含 `id` 字段来实现幂等性
+     *
+     * @example
+     * // 普通任务,自动生成 ID
+     * await queue.enqueue('send-email', { to: 'user@example.com' })
+     *
+     * // 幂等任务,手动指定 ID,重复提交会被忽略
+     * await queue.enqueue('send-email', { id: 'email-123', to: 'user@example.com' })
+     * await queue.enqueue('send-email', { id: 'email-123', to: 'user@example.com' }) // 会被跳过
+     */
+    enqueue<T extends TaskData = TaskData>(type: string, data: T[]): Promise<string[]>;
+    enqueue<T extends TaskData = TaskData>(type: string, data: T): Promise<string>;
+    /**
+     * 获取任务详情
+     */
+    getTask(taskId: string): Promise<Task | null>;
+    /**
+     * 更新任务状态并移动到对应队列（原子操作）
+     */
+    private applyStatus;
+    /**
+     * 根据状态获取对应的队列键
+     */
+    private getQueueByStatus;
+    /**
+     * 恢复超时的任务
+     * 检查 processing 队列中的任务，将超时的任务重试或标记为失败
+     */
+    private recoverStalledTasks;
+    /**
+     * 处理单个任务
+     */
+    private processTask;
+    /**
+     * 启动恢复机制（内部方法，自动调用）
+     */
+    private startRecovery;
+    /**
+     * 停止恢复机制（内部方法，自动调用）
+     */
+    private stopRecovery;
+    /**
+     * 启动消费者（内部方法，自动调用）
+     */
+    private startConsumer;
+    /**
+     * 停止消费者（内部方法，自动调用）
+     */
+    private stopConsumer;
+    /**
+     * 获取队列统计信息
+     * 返回队列中各种状态任务的详细数量
+     *
+     * 使用分离队列设计，O(1) 时间复杂度
+     */
+    statistics(): Promise<QueueStats>;
+    /**
+     * 清空所有队列
+     */
+    clear(): Promise<void>;
+    /**
+     * 健康检查
+     */
+    health(): Promise<boolean>;
+}
+//# sourceMappingURL=redis-queue.d.ts.map

package/output/redis-queue/redis-queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redis-queue.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAU,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAE/F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAAI;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAClD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiC;IAG1D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAQ;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;gBAE3B,MAAM,EAAE,eAAe;IAqCnC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAY9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAUlB;;OAEG;IACH,MAAM,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,IAAI;IAIlF;;;;;;;;;;;;OAYG;IACG,OAAO,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAClF,OAAO,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAmEpF;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAWnD;;OAEG;YACW,WAAW;IA8BzB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;;OAGG;YACW,mBAAmB;IA6EjC;;OAEG;YACW,WAAW;IAmFzB;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACH,OAAO,CAAC,aAAa;IA4ErB;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC;IA0BvC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB5B;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAMjC"}