@taicode/common-server 3.7.4 → 3.7.7

package/output/redis-queue/redis-queue-batch-consumer.test.js

@@ -1,341 +0,0 @@
-import { describe, it, expect, afterEach } from 'vitest';
-import { catchIt } from '@taicode/common-base';
-import { RedisQueueProvider } from './redis-queue-provider.js';
-import { RedisQueueBatchConsumer } from './redis-queue-batch-consumer.js';
-import { dispatchQueueTask, waitQueueCompletion, clearQueue, getQueueTasks } from './test-helpers.js';
-const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
-describe('RedisQueueBatchConsumer', () => {
-    const providers = [];
-    const consumers = [];
-    afterEach(async () => {
-        for (const consumer of consumers) {
-            await clearQueue(consumer);
-            await consumer.disconnect();
-        }
-        for (const provider of providers) {
-            provider.disconnect();
-        }
-        providers.length = 0;
-        consumers.length = 0;
-    });
-    const createQueue = (handler, options) => {
-        const uniqueKey = `test:batch:${Date.now()}:${Math.random()}`;
-        const provider = new RedisQueueProvider(uniqueKey, {
-            redisUrl: REDIS_URL,
-        });
-        const consumer = new RedisQueueBatchConsumer(uniqueKey, {
-            redisUrl: REDIS_URL,
-            consumerInterval: 100,
-            maxRetries: 2,
-            batchSize: 10,
-            ...options,
-            handler,
-        });
-        providers.push(provider);
-        consumers.push(consumer);
-        return { provider, consumer };
-    };
-    describe('连接管理', () => {
-        it('应该成功连接到 Redis', async () => {
-            const { consumer } = createQueue(async () => catchIt(() => { }));
-            await consumer.connect();
-            const health = await consumer.health();
-            expect(health).toBe(true);
-        });
-        it('连接后应该自动启动消费者', async () => {
-            let processed = false;
-            const { provider, consumer } = createQueue(async () => {
-                processed = true;
-                return catchIt(() => { });
-            });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue({ test: true });
-            await new Promise(resolve => setTimeout(resolve, 300));
-            expect(processed).toBe(true);
-        });
-    });
-    describe('批量处理', () => {
-        it('应该批量处理任务', async () => {
-            let batchSize = 0;
-            const { provider, consumer } = createQueue(async (dataList) => {
-                batchSize = dataList.length;
-                return catchIt(() => { });
-            }, { batchSize: 5 });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue([
-                { value: 1 }, { value: 2 }, { value: 3 },
-                { value: 4 }, { value: 5 },
-            ]);
-            await new Promise(resolve => setTimeout(resolve, 500));
-            expect(batchSize).toBe(5);
-            const stats = await consumer.statistics();
-            expect(stats.completed).toBe(5);
-        });
-        it('应该能够使用 dispatchQueueTask 立即处理批量任务', async () => {
-            const processedBatches = [];
-            const { provider, consumer } = createQueue(async (dataList) => {
-                processedBatches.push([...dataList]);
-                return catchIt(() => { });
-            }, { batchSize: 3 });
-            await provider.connect();
-            await consumer.connect();
-            const taskIds = await provider.enqueue([
-                { value: 1 }, { value: 2 }, { value: 3 },
-                { value: 4 }, { value: 5 },
-            ]);
-            // 使用 dispatchQueueTask 立即处理任务
-            await dispatchQueueTask(consumer, taskIds);
-            // 应该有 2 个批次（3+2）
-            expect(processedBatches).toHaveLength(2);
-            expect(processedBatches[0]).toHaveLength(3);
-            expect(processedBatches[1]).toHaveLength(2);
-            const stats = await consumer.statistics();
-            expect(stats.completed).toBe(5);
-        });
-        it('应该能处理大批量任务', async () => {
-            let totalProcessed = 0;
-            const { provider, consumer } = createQueue(async (dataList) => {
-                totalProcessed += dataList.length;
-                return catchIt(() => { });
-            }, { batchSize: 50 });
-            await provider.connect();
-            await consumer.connect();
-            const tasks = Array.from({ length: 200 }, (_, i) => ({ value: i }));
-            await provider.enqueue(tasks);
-            await new Promise(resolve => setTimeout(resolve, 3000));
-            expect(totalProcessed).toBe(200);
-        });
-        it('应该能够使用 waitQueueCompletion 等待批量处理完成', async () => {
-            const processedBatches = [];
-            const { provider, consumer } = createQueue(async (dataList) => {
-                processedBatches.push(dataList.length);
-                await new Promise(resolve => setTimeout(resolve, 100));
-                return catchIt(() => { });
-            }, { batchSize: 5 });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue(Array.from({ length: 15 }, (_, i) => ({ index: i })));
-            // 等待所有任务完成
-            await waitQueueCompletion(consumer, stats => stats.pending === 0 && stats.processing === 0 && stats.completed === 15);
-            // 应该有 3 个批次
-            expect(processedBatches).toHaveLength(3);
-            expect(processedBatches).toEqual([5, 5, 5]);
-        });
-        it('应该能够使用 getQueueTasks 获取批量任务详情', async () => {
-            const { provider, consumer } = createQueue(async () => {
-                // 模拟失败
-                throw new Error('Batch failed');
-            }, { batchSize: 3, maxRetries: 0 });
-            await provider.connect();
-            await consumer.connect();
-            const taskIds = await provider.enqueue([
-                { name: 'task1' }, { name: 'task2' }, { name: 'task3' }
-            ]);
-            // 立即处理
-            await dispatchQueueTask(consumer, taskIds);
-            // 等待一下确保任务处理完成
-            await new Promise(resolve => setTimeout(resolve, 100));
-            // 获取失败任务
-            const failedTasks = await getQueueTasks(provider, 'failed');
-            expect(failedTasks.length).toBeGreaterThan(0);
-            expect(failedTasks.every(t => t.status === 'failed')).toBe(true);
-        });
-    });
-    describe('重试机制', () => {
-        it('失败的批次应该重试', async () => {
-            let attemptCount = 0;
-            const { provider, consumer } = createQueue(async () => {
-                attemptCount++;
-                throw new Error('Batch failed');
-            }, { batchSize: 2, maxRetries: 1 });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue([{ value: 1 }, { value: 2 }]);
-            await new Promise(resolve => setTimeout(resolve, 2500));
-            const stats = await consumer.statistics();
-            expect(stats.failed).toBe(2);
-        });
-        it('disconnect 应该等待正在处理的批次完成', async () => {
-            const processedBatches = [];
-            let processingBatch = null;
-            const { provider, consumer } = createQueue(async (dataList) => {
-                processingBatch = dataList.map(d => d.index || 0);
-                await new Promise(resolve => setTimeout(resolve, 500)); // 模拟长时间处理
-                processedBatches.push(dataList.map(d => d.index || 0));
-                processingBatch = null;
-                return catchIt(() => { });
-            }, { batchSize: 2 });
-            await provider.connect();
-            await consumer.connect();
-            // 添加一批任务
-            await provider.enqueue([{ index: 1 }, { index: 2 }]);
-            // 等待批次开始处理
-            await new Promise(resolve => setTimeout(resolve, 200));
-            expect(processingBatch).toEqual([1, 2]);
-            expect(processedBatches).toHaveLength(0);
-            // 调用 disconnect，应该等待批次完成
-            const disconnectPromise = consumer.disconnect();
-            // 此时批次应该还在处理中
-            expect(processingBatch).toEqual([1, 2]);
-            // 等待 disconnect 完成
-            await disconnectPromise;
-            // disconnect 完成后，批次应该已经处理完成
-            expect(processingBatch).toBe(null);
-            expect(processedBatches).toHaveLength(1);
-            expect(processedBatches[0]).toEqual([1, 2]);
-        });
-    });
-    describe('幂等性', () => {
-        it('应该支持在 data 中指定 id 来实现幂等性', async () => {
-            let processCount = 0;
-            const { provider, consumer } = createQueue(async (dataList) => {
-                processCount += dataList.length;
-                return catchIt(() => { });
-            }, { batchSize: 10 });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue({ id: 'unique-1', value: 1 });
-            await provider.enqueue({ id: 'unique-1', value: 2 });
-            await provider.enqueue({ id: 'unique-2', value: 3 });
-            await new Promise(resolve => setTimeout(resolve, 1000));
-            expect(processCount).toBe(2); // 只处理 unique-1 和 unique-2
-        });
-    });
-    describe('延迟处理', () => {
-        it('应该支持延迟处理任务', async () => {
-            const startTime = Date.now();
-            let processTime = 0;
-            const { provider, consumer } = createQueue(async () => {
-                processTime = Date.now() - startTime;
-                return catchIt(() => { });
-            });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue({ test: true });
-            await new Promise(resolve => setTimeout(resolve, 1000));
-            expect(processTime).toBeGreaterThan(0);
-        });
-    });
-    describe('并发安全', () => {
-        it('多个消费者实例应该能协同工作', async () => {
-            let consumer1Count = 0;
-            let consumer2Count = 0;
-            const uniqueKey = `test:batch:concurrent:${Date.now()}:${Math.random()}`;
-            const provider = new RedisQueueProvider(uniqueKey, {
-                redisUrl: REDIS_URL,
-            });
-            const consumer1 = new RedisQueueBatchConsumer(uniqueKey, {
-                redisUrl: REDIS_URL,
-                batchSize: 5,
-                handler: async (dataList) => {
-                    consumer1Count += dataList.length;
-                    return catchIt(() => { });
-                },
-            });
-            const consumer2 = new RedisQueueBatchConsumer(uniqueKey, {
-                redisUrl: REDIS_URL,
-                batchSize: 5,
-                handler: async (dataList) => {
-                    consumer2Count += dataList.length;
-                    return catchIt(() => { });
-                },
-            });
-            providers.push(provider);
-            consumers.push(consumer1, consumer2);
-            await provider.connect();
-            await consumer1.connect();
-            await consumer2.connect();
-            const tasks = Array.from({ length: 20 }, (_, i) => ({ value: i }));
-            await provider.enqueue(tasks);
-            await new Promise(resolve => setTimeout(resolve, 3000));
-            // 两个消费者协同处理,总数应该是 20
-            // 但由于并发,可能有一些任务被重复处理或未处理,放宽条件
-            expect(consumer1Count + consumer2Count).toBeGreaterThanOrEqual(10);
-            expect(consumer1Count + consumer2Count).toBeLessThanOrEqual(20);
-        });
-    });
-    describe('队列统计', () => {
-        it('应该正确返回队列统计信息', async () => {
-            const { provider, consumer } = createQueue(async () => {
-                await new Promise(resolve => setTimeout(resolve, 100));
-                return catchIt(() => { });
-            });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue(Array.from({ length: 10 }, (_, i) => ({ value: i })));
-            await new Promise(resolve => setTimeout(resolve, 50));
-            const stats1 = await consumer.statistics();
-            expect(stats1.pending).toBeGreaterThan(0);
-            await new Promise(resolve => setTimeout(resolve, 3000));
-            const stats2 = await consumer.statistics();
-            expect(stats2.completed).toBe(10);
-        });
-    });
-    describe('数据边界', () => {
-        it('应该能处理较大的数据', async () => {
-            let receivedData = [];
-            const { provider, consumer } = createQueue(async (dataList) => {
-                receivedData = dataList;
-                return catchIt(() => { });
-            }, { batchSize: 5 });
-            await provider.connect();
-            await consumer.connect();
-            const largeString = 'x'.repeat(10 * 1024); // 10KB
-            await provider.enqueue(Array.from({ length: 5 }, () => ({ data: largeString })));
-            await new Promise(resolve => setTimeout(resolve, 1000));
-            expect(receivedData).toHaveLength(5);
-            expect(receivedData[0].data).toBe(largeString);
-        });
-    });
-    describe('超时恢复', () => {
-        it('超时的任务应该直接标记为失败', { timeout: 15000 }, async () => {
-            let processCount = 0;
-            const { provider, consumer } = createQueue(async (dataList) => {
-                processCount++;
-                // 模拟处理卡住，永远不完成 (但超时机制会检测到)
-                await new Promise(resolve => setTimeout(resolve, 20000));
-                return catchIt(() => { });
-            }, {
-                batchSize: 1,
-                processingTimeout: 500, // 500ms 超时
-                consumerInterval: 100,
-            });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue([{ value: 1 }]);
-            // 等待任务被处理和超时恢复 (需要等待至少2个恢复周期: 10秒)
-            await new Promise(resolve => setTimeout(resolve, 12000));
-            const stats = await consumer.statistics();
-            // 超时后直接标记为失败,不应该重试
-            expect(stats.failed).toBe(1);
-            expect(stats.pending).toBe(0);
-            expect(stats.processing).toBe(0);
-            expect(stats.completed).toBe(0);
-            // 应该只处理一次,不会重试
-            expect(processCount).toBe(1);
-        });
-        it('未超时的失败任务应该正常重试', async () => {
-            let attemptCount = 0;
-            const { provider, consumer } = createQueue(async () => {
-                attemptCount++;
-                throw new Error('Task failed');
-            }, {
-                batchSize: 1,
-                maxRetries: 3,
-                processingTimeout: 5000, // 5秒超时,足够长不会触发
-                consumerInterval: 100,
-            });
-            await provider.connect();
-            await consumer.connect();
-            await provider.enqueue([{ value: 1 }]);
-            // 等待所有重试完成
-            await new Promise(resolve => setTimeout(resolve, 3000));
-            const stats = await consumer.statistics();
-            // 应该重试 3 次后失败
-            expect(stats.failed).toBe(1);
-            expect(attemptCount).toBe(4); // 初始 1 次 + 3 次重试
-        });
-    });
-});

package/output/redis-queue/redis-queue-common.d.ts

@@ -1,73 +0,0 @@
-import { type RedisClientType } from 'redis';
-import type { Status, Task, RedisQueueCommonConfig } from './types.js';
-/**
- * Redis 队列基础类
- * 封装 Provider 和 Consumer 的公共逻辑
- */
-export declare abstract class RedisQueueCommon {
-    protected redis: RedisClientType | null;
-    protected isExternalRedis: boolean;
-    protected readonly queueKey: string;
-    protected readonly redisUrl?: string;
-    protected readonly cleanupDelay: number;
-    protected readonly failedQueue: string;
-    protected readonly pendingQueue: string;
-    protected readonly processingQueue: string;
-    protected readonly completedQueue: string;
-    constructor(queueKey: string, config?: RedisQueueCommonConfig);
-    /**
-     * 获取日志前缀(子类实现)
-     */
-    protected abstract getLogPrefix(): string;
-    /**
-     * 连接 Redis
-     */
-    connect(): Promise<void>;
-    /**
-     * 断开 Redis 连接
-     */
-    disconnect(): void;
-    /**
-     * 获取任务详情
-     */
-    protected getTask(taskId: string): Promise<Task | null>;
-    /**
-     * 根据状态获取对应的队列键
-     */
-    protected getQueueByStatus(status: Status): string;
-    /**
-     * 更新任务状态并移动到对应队列（原子操作）
-     */
-    protected applyStatus(taskId: string, oldStatus: Status, newStatus: Status): Promise<void>;
-    /**
-     * 批量更新任务状态（使用 Lua 脚本保证原子性）
-     * 注意: 此方法会在 Lua 脚本中读取任务数据,不需要提前获取
-     */
-    protected applyStatusBatch(taskIds: string[], oldStatus: Status, newStatus: Status): Promise<number>;
-    /**
-     * 获取队列统计信息（O(1) 时间复杂度）
-     */
-    statistics(): Promise<{
-        pending: number;
-        processing: number;
-        completed: number;
-        failed: number;
-    }>;
-    /**
-     * 健康检查
-     */
-    health(): Promise<boolean>;
-    /**
-     * 恢复超时的任务
-     * 检查 processing 队列中的任务，将超时的任务直接标记为失败
-     * 使用 Lua 脚本批量处理以提高性能和原子性
-     *
-     * @param processingTimeout 处理超时时间（毫秒）
-     * @returns 恢复的任务统计 {timeoutFailed: number, cleaned: number}
-     */
-    protected recoverStalled(processingTimeout: number): Promise<{
-        timeoutFailed: number;
-        cleaned: number;
-    }>;
-}
-//# sourceMappingURL=redis-queue-common.d.ts.map

package/output/redis-queue/redis-queue-common.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"redis-queue-common.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,eAAe,EAAE,MAAM,OAAO,CAAA;AAE1D,OAAO,KAAK,EAAE,MAAM,EAAY,IAAI,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAA;AAEhF;;;GAGG;AACH,8BAAsB,gBAAgB;IACpC,SAAS,CAAC,KAAK,EAAE,eAAe,GAAG,IAAI,CAAO;IAC9C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAQ;IAE1C,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACnC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACpC,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAGvC,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IACtC,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IACvC,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAC1C,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;gBAE7B,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAE,sBAA2B;IAwCjE;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,YAAY,IAAI,MAAM;IAEzC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAS9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAalB;;OAEG;cACa,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAW7D;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAUlD;;OAEG;cACa,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAqChG;;;OAGG;cACa,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAqD1G;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAevG;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAUhC;;;;;;;OAOG;cACa,cAAc,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CA4E/G"}