@taicode/common-server 1.0.12 → 1.0.13

package/output/redis-queue/batch-consumer.test.js

@@ -1,216 +0,0 @@
-import { describe, it, expect, afterEach } from 'vitest';
-import { catchIt } from '@taicode/common-base';
-import { RedisQueueProvider } from './redis-queue-provider';
-import { RedisBatchConsumer } from './batch-consumer';
-const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
-describe('RedisBatchConsumer', () => {
-    const providers = [];
-    const consumers = [];
-    afterEach(async () => {
-        for (const consumer of consumers) {
-            consumer.disconnect();
-        }
-        for (const provider of providers) {
-            try {
-                await provider.clear();
-            }
-            catch (error) {
-                // 忽略清理错误
-            }
-            provider.disconnect();
-        }
-        providers.length = 0;
-        consumers.length = 0;
-    });
-    const createQueue = (handler, options) => {
-        const uniqueKey = `test:batch:${Date.now()}:${Math.random()}`;
-        const provider = new RedisQueueProvider({
-            redisUrl: REDIS_URL,
-            queueKey: uniqueKey,
-        });
-        const consumer = new RedisBatchConsumer({
-            redisUrl: REDIS_URL,
-            queueKey: uniqueKey,
-            consumerInterval: 100,
-            maxRetries: 2,
-            batchSize: 10,
-            ...options,
-            handler,
-        });
-        providers.push(provider);
-        consumers.push(consumer);
-        return { provider, consumer };
-    };
-    describe('连接管理', () => {
-        it('应该成功连接到 Redis', async () => {
-            const { consumer } = createQueue(async () => catchIt(() => { }));
-            await consumer.connect();
-            const health = await consumer.health();
-            expect(health).toBe(true);
-        });
-        it('连接后应该自动启动消费者', async () => {
-            let processed = false;
-            const { provider, consumer } = createQueue(async () => {
-                processed = true;
-                return catchIt(() => { });
-            });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue({ test: true });
-            await new Promise(resolve => setTimeout(resolve, 300));
-            expect(processed).toBe(true);
-        });
-    });
-    describe('批量处理', () => {
-        it('应该批量处理任务', async () => {
-            let batchSize = 0;
-            const { provider, consumer } = createQueue(async (dataList) => {
-                batchSize = dataList.length;
-                return catchIt(() => { });
-            }, { batchSize: 5 });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue([
-                { value: 1 }, { value: 2 }, { value: 3 },
-                { value: 4 }, { value: 5 },
-            ]);
-            await new Promise(resolve => setTimeout(resolve, 500));
-            expect(batchSize).toBe(5);
-            const stats = await consumer.statistics();
-            expect(stats.completed).toBe(5);
-        });
-        it('应该能处理大批量任务', async () => {
-            let totalProcessed = 0;
-            const { provider, consumer } = createQueue(async (dataList) => {
-                totalProcessed += dataList.length;
-                return catchIt(() => { });
-            }, { batchSize: 50 });
-            await provider.connect();
-            await consumer.connect();
-            const tasks = Array.from({ length: 200 }, (_, i) => ({ value: i }));
-            await provider.enqueue(tasks);
-            await new Promise(resolve => setTimeout(resolve, 3000));
-            expect(totalProcessed).toBe(200);
-        });
-    });
-    describe('重试机制', () => {
-        it('失败的批次应该重试', async () => {
-            let attemptCount = 0;
-            const { provider, consumer } = createQueue(async () => {
-                attemptCount++;
-                throw new Error('Batch failed');
-            }, { batchSize: 2, maxRetries: 1 });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue([{ value: 1 }, { value: 2 }]);
-            await new Promise(resolve => setTimeout(resolve, 2500));
-            const stats = await consumer.statistics();
-            expect(stats.failed).toBe(2);
-        });
-    });
-    describe('幂等性', () => {
-        it('应该支持在 data 中指定 id 来实现幂等性', async () => {
-            let processCount = 0;
-            const { provider, consumer } = createQueue(async (dataList) => {
-                processCount += dataList.length;
-                return catchIt(() => { });
-            }, { batchSize: 10 });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue({ id: 'unique-1', value: 1 });
-            await provider.enqueue({ id: 'unique-1', value: 2 });
-            await provider.enqueue({ id: 'unique-2', value: 3 });
-            await new Promise(resolve => setTimeout(resolve, 1000));
-            expect(processCount).toBe(2); // 只处理 unique-1 和 unique-2
-        });
-    });
-    describe('延迟处理', () => {
-        it('应该支持延迟处理任务', async () => {
-            const startTime = Date.now();
-            let processTime = 0;
-            const { provider, consumer } = createQueue(async () => {
-                processTime = Date.now() - startTime;
-                return catchIt(() => { });
-            });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue({ test: true });
-            await new Promise(resolve => setTimeout(resolve, 1000));
-            expect(processTime).toBeGreaterThan(0);
-        });
-    });
-    describe('并发安全', () => {
-        it('多个消费者实例应该能协同工作', async () => {
-            let consumer1Count = 0;
-            let consumer2Count = 0;
-            const uniqueKey = `test:batch:concurrent:${Date.now()}:${Math.random()}`;
-            const provider = new RedisQueueProvider({
-                redisUrl: REDIS_URL,
-                queueKey: uniqueKey,
-            });
-            const consumer1 = new RedisBatchConsumer({
-                redisUrl: REDIS_URL,
-                queueKey: uniqueKey,
-                batchSize: 5,
-                handler: async (dataList) => {
-                    consumer1Count += dataList.length;
-                    return catchIt(() => { });
-                },
-            });
-            const consumer2 = new RedisBatchConsumer({
-                redisUrl: REDIS_URL,
-                queueKey: uniqueKey,
-                batchSize: 5,
-                handler: async (dataList) => {
-                    consumer2Count += dataList.length;
-                    return catchIt(() => { });
-                },
-            });
-            providers.push(provider);
-            consumers.push(consumer1, consumer2);
-            await provider.connect();
-            await consumer1.connect();
-            await consumer2.connect();
-            const tasks = Array.from({ length: 20 }, (_, i) => ({ value: i }));
-            await provider.enqueue(tasks);
-            await new Promise(resolve => setTimeout(resolve, 3000));
-            // 两个消费者协同处理,总数应该是 20
-            // 但由于并发,可能有一些任务被重复处理或未处理,放宽条件
-            expect(consumer1Count + consumer2Count).toBeGreaterThanOrEqual(10);
-            expect(consumer1Count + consumer2Count).toBeLessThanOrEqual(20);
-        });
-    });
-    describe('队列统计', () => {
-        it('应该正确返回队列统计信息', async () => {
-            const { provider, consumer } = createQueue(async () => {
-                await new Promise(resolve => setTimeout(resolve, 100));
-                return catchIt(() => { });
-            });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue(Array.from({ length: 10 }, (_, i) => ({ value: i })));
-            await new Promise(resolve => setTimeout(resolve, 50));
-            const stats1 = await consumer.statistics();
-            expect(stats1.pending).toBeGreaterThan(0);
-            await new Promise(resolve => setTimeout(resolve, 3000));
-            const stats2 = await consumer.statistics();
-            expect(stats2.completed).toBe(10);
-        });
-    });
-    describe('数据边界', () => {
-        it('应该能处理较大的数据', async () => {
-            let receivedData = [];
-            const { provider, consumer } = createQueue(async (dataList) => {
-                receivedData = dataList;
-                return catchIt(() => { });
-            }, { batchSize: 5 });
-            await provider.connect();
-            await consumer.connect();
-            const largeString = 'x'.repeat(10 * 1024); // 10KB
-            await provider.enqueue(Array.from({ length: 5 }, () => ({ data: largeString })));
-            await new Promise(resolve => setTimeout(resolve, 1000));
-            expect(receivedData).toHaveLength(5);
-            expect(receivedData[0].data).toBe(largeString);
-        });
-    });
-});

package/output/redis-queue/batch-redis-queue.d.ts

@@ -1,136 +0,0 @@
-import type { BatchTaskQueueConfig, TaskData, Task, QueueStats } from './types';
-/**
- * 批量任务队列类（泛型）
- *
- * 提供基于 Redis 的批量任务队列功能，支持：
- * - 批量处理任务（每次处理多条）
- * - 任务入队和持久化
- * - 自动重试机制
- * - 任务状态追踪
- * - 分布式消费
- *
- * @template T 任务数据类型
- *
- * @example
- * ```ts
- * interface EmailTask {
- *   to: string
- * }
- *
- * const queue = new BatchRedisQueue<EmailTask>({
- *   redisUrl: 'redis://localhost:6379',
- *   queueKey: 'email-batch-queue',
- *   batchSize: 50,
- *   handler: async (dataList) => {
- *     await sendEmailsBatch(dataList.map(d => d.to))
- *     return catchIt(() => {})
- *   }
- * })
- *
- * // 连接 Redis（自动启动消费者）
- * await queue.connect()
- *
- * // 入队任务（自动开始处理）
- * await queue.enqueue([
- *   { to: 'user1@example.com' },
- *   { to: 'user2@example.com' },
- * ])
- *
- * // 获取队列统计信息（O(1) 时间复杂度）
- * const stats = await queue.statistics()
- * console.log(stats) // { pending: 100, processing: 50, completed: 200, failed: 5 }
- * ```
- */
-export declare class BatchRedisQueue<T extends TaskData = TaskData> {
-    private consumerRunning;
-    private redis;
-    private consumerInterval;
-    private recoveryInterval;
-    private processingTasks;
-    private readonly config;
-    private readonly handler;
-    private readonly failedQueue;
-    private readonly pendingQueue;
-    private readonly processingQueue;
-    private readonly completedQueue;
-    constructor(config: BatchTaskQueueConfig<string, T>);
-    /**
-     * 连接 Redis 并自动启动消费者
-     */
-    connect(): Promise<void>;
-    /**
-     * 断开 Redis 连接并停止消费者
-     */
-    disconnect(): void;
-    /**
-     * 将任务推入队列(支持单个或批量)
-     * @param data 任务数据(单个或数组)。可以在 data 中包含 `id` 字段来实现幂等性
-     *
-     * @example
-     * // 普通任务,自动生成 ID
-     * await queue.enqueue({ to: 'user@example.com' })
-     *
-     * // 幂等任务,手动指定 ID,重复提交会被忽略
-     * await queue.enqueue({ id: 'email-123', to: 'user@example.com' })
-     * await queue.enqueue({ id: 'email-123', to: 'user@example.com' }) // 会被跳过
-     */
-    enqueue(data: T[]): Promise<string[]>;
-    enqueue(data: T): Promise<string>;
-    /**
-     * 获取任务详情
-     */
-    getTask(taskId: string): Promise<Task<T> | null>;
-    /**
-     * 更新任务状态并移动到对应队列（原子操作）
-     */
-    private applyStatus;
-    /**
-     * 批量更新任务状态
-     */
-    private applyStatusBatch;
-    /**
-     * 根据状态获取对应的队列键
-     */
-    private getQueueByStatus;
-    /**
-     * 恢复超时的任务
-     * 检查 processing 队列中的任务，将超时的任务重试或标记为失败
-     */
-    private recoverStalledTasks;
-    /**
-     * 批量处理任务
-     */
-    private processBatch;
-    /**
-     * 启动恢复机制（内部方法，自动调用）
-     */
-    private startRecovery;
-    /**
-     * 停止恢复机制（内部方法，自动调用）
-     */
-    private stopRecovery;
-    /**
-     * 启动消费者（内部方法，自动调用）
-     */
-    private startConsumer;
-    /**
-     * 停止消费者（内部方法，自动调用）
-     */
-    private stopConsumer;
-    /**
-     * 获取队列统计信息
-     * 返回队列中各种状态任务的详细数量
-     *
-     * 使用分离队列设计，O(1) 时间复杂度
-     */
-    statistics(): Promise<QueueStats>;
-    /**
-     * 清空所有队列
-     */
-    clear(): Promise<void>;
-    /**
-     * 健康检查
-     */
-    health(): Promise<boolean>;
-}
-//# sourceMappingURL=batch-redis-queue.d.ts.map

package/output/redis-queue/batch-redis-queue.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"batch-redis-queue.d.ts","sourceRoot":"","sources":["../../source/redis-queue/batch-redis-queue.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAU,QAAQ,EAAE,IAAI,EAAoB,UAAU,EAAE,MAAM,SAAS,CAAA;AAEzG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,qBAAa,eAAe,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IACxD,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAAI;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAQtB;IACD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAG7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAQ;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;gBAE3B,MAAM,EAAE,oBAAoB,CAAC,MAAM,EAAE,CAAC,CAAC;IAkDnD;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAY9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAUlB;;;;;;;;;;;OAWG;IACG,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IACrC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAkEvC;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAWtD;;OAEG;YACW,WAAW;IA8BzB;;OAEG;YACW,gBAAgB;IAI9B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;;OAGG;YACW,mBAAmB;IA6EjC;;OAEG;YACW,YAAY;IA8F1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACH,OAAO,CAAC,aAAa;IA2ErB;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC;IA0BvC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB5B;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAMjC"}