@taicode/common-server 1.0.10 → 1.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/output/redis-queue/batch-consumer.d.ts +107 -0
  2. package/output/redis-queue/batch-consumer.d.ts.map +1 -0
  3. package/output/redis-queue/batch-consumer.js +492 -0
  4. package/output/redis-queue/batch-consumer.test.d.ts +2 -0
  5. package/output/redis-queue/batch-consumer.test.d.ts.map +1 -0
  6. package/output/redis-queue/batch-consumer.test.js +216 -0
  7. package/output/redis-queue/batch-redis-queue.d.ts +26 -26
  8. package/output/redis-queue/batch-redis-queue.d.ts.map +1 -1
  9. package/output/redis-queue/batch-redis-queue.js +104 -101
  10. package/output/redis-queue/batch-redis-queue.test.js +96 -251
  11. package/output/redis-queue/index.d.ts +5 -4
  12. package/output/redis-queue/index.d.ts.map +1 -1
  13. package/output/redis-queue/index.js +4 -2
  14. package/output/redis-queue/redis-batch-consumer.d.ts +85 -0
  15. package/output/redis-queue/redis-batch-consumer.d.ts.map +1 -0
  16. package/output/redis-queue/redis-batch-consumer.js +360 -0
  17. package/output/redis-queue/redis-batch-consumer.test.d.ts +2 -0
  18. package/output/redis-queue/redis-batch-consumer.test.d.ts.map +1 -0
  19. package/output/redis-queue/redis-batch-consumer.test.js +265 -0
  20. package/output/redis-queue/redis-queue-common.d.ts +61 -0
  21. package/output/redis-queue/redis-queue-common.d.ts.map +1 -0
  22. package/output/redis-queue/redis-queue-common.js +222 -0
  23. package/output/redis-queue/redis-queue-common.test.d.ts +2 -0
  24. package/output/redis-queue/redis-queue-common.test.d.ts.map +1 -0
  25. package/output/redis-queue/redis-queue-common.test.js +623 -0
  26. package/output/redis-queue/redis-queue-consumer.d.ts +102 -0
  27. package/output/redis-queue/redis-queue-consumer.d.ts.map +1 -0
  28. package/output/redis-queue/redis-queue-consumer.js +461 -0
  29. package/output/redis-queue/redis-queue-consumer.test.d.ts +2 -0
  30. package/output/redis-queue/redis-queue-consumer.test.d.ts.map +1 -0
  31. package/output/redis-queue/redis-queue-consumer.test.js +242 -0
  32. package/output/redis-queue/redis-queue-provider.d.ts +57 -0
  33. package/output/redis-queue/redis-queue-provider.d.ts.map +1 -0
  34. package/output/redis-queue/redis-queue-provider.js +188 -0
  35. package/output/redis-queue/redis-queue-provider.test.d.ts +2 -0
  36. package/output/redis-queue/redis-queue-provider.test.d.ts.map +1 -0
  37. package/output/redis-queue/redis-queue-provider.test.js +114 -0
  38. package/output/redis-queue/redis-queue.d.ts +26 -25
  39. package/output/redis-queue/redis-queue.d.ts.map +1 -1
  40. package/output/redis-queue/redis-queue.js +42 -35
  41. package/output/redis-queue/redis-queue.test.js +96 -698
  42. package/output/redis-queue/registry.d.ts +57 -0
  43. package/output/redis-queue/registry.d.ts.map +1 -0
  44. package/output/redis-queue/registry.js +30 -0
  45. package/output/redis-queue/types.d.ts +42 -13
  46. package/output/redis-queue/types.d.ts.map +1 -1
  47. package/package.json +1 -1
@@ -0,0 +1,216 @@
1
+ import { describe, it, expect, afterEach } from 'vitest';
2
+ import { catchIt } from '@taicode/common-base';
3
+ import { RedisQueueProvider } from './redis-queue-provider';
4
+ import { RedisBatchConsumer } from './batch-consumer';
5
+ const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
6
+ describe('RedisBatchConsumer', () => {
7
+ const providers = [];
8
+ const consumers = [];
9
+ afterEach(async () => {
10
+ for (const consumer of consumers) {
11
+ consumer.disconnect();
12
+ }
13
+ for (const provider of providers) {
14
+ try {
15
+ await provider.clear();
16
+ }
17
+ catch (error) {
18
+ // 忽略清理错误
19
+ }
20
+ provider.disconnect();
21
+ }
22
+ providers.length = 0;
23
+ consumers.length = 0;
24
+ });
25
+ const createQueue = (handler, options) => {
26
+ const uniqueKey = `test:batch:${Date.now()}:${Math.random()}`;
27
+ const provider = new RedisQueueProvider({
28
+ redisUrl: REDIS_URL,
29
+ queueKey: uniqueKey,
30
+ });
31
+ const consumer = new RedisBatchConsumer({
32
+ redisUrl: REDIS_URL,
33
+ queueKey: uniqueKey,
34
+ consumerInterval: 100,
35
+ maxRetries: 2,
36
+ batchSize: 10,
37
+ ...options,
38
+ handler,
39
+ });
40
+ providers.push(provider);
41
+ consumers.push(consumer);
42
+ return { provider, consumer };
43
+ };
44
+ describe('连接管理', () => {
45
+ it('应该成功连接到 Redis', async () => {
46
+ const { consumer } = createQueue(async () => catchIt(() => { }));
47
+ await consumer.connect();
48
+ const health = await consumer.health();
49
+ expect(health).toBe(true);
50
+ });
51
+ it('连接后应该自动启动消费者', async () => {
52
+ let processed = false;
53
+ const { provider, consumer } = createQueue(async () => {
54
+ processed = true;
55
+ return catchIt(() => { });
56
+ });
57
+ await provider.connect();
58
+ await consumer.connect();
59
+ await provider.enqueue({ test: true });
60
+ await new Promise(resolve => setTimeout(resolve, 300));
61
+ expect(processed).toBe(true);
62
+ });
63
+ });
64
+ describe('批量处理', () => {
65
+ it('应该批量处理任务', async () => {
66
+ let batchSize = 0;
67
+ const { provider, consumer } = createQueue(async (dataList) => {
68
+ batchSize = dataList.length;
69
+ return catchIt(() => { });
70
+ }, { batchSize: 5 });
71
+ await provider.connect();
72
+ await consumer.connect();
73
+ await provider.enqueue([
74
+ { value: 1 }, { value: 2 }, { value: 3 },
75
+ { value: 4 }, { value: 5 },
76
+ ]);
77
+ await new Promise(resolve => setTimeout(resolve, 500));
78
+ expect(batchSize).toBe(5);
79
+ const stats = await consumer.statistics();
80
+ expect(stats.completed).toBe(5);
81
+ });
82
+ it('应该能处理大批量任务', async () => {
83
+ let totalProcessed = 0;
84
+ const { provider, consumer } = createQueue(async (dataList) => {
85
+ totalProcessed += dataList.length;
86
+ return catchIt(() => { });
87
+ }, { batchSize: 50 });
88
+ await provider.connect();
89
+ await consumer.connect();
90
+ const tasks = Array.from({ length: 200 }, (_, i) => ({ value: i }));
91
+ await provider.enqueue(tasks);
92
+ await new Promise(resolve => setTimeout(resolve, 3000));
93
+ expect(totalProcessed).toBe(200);
94
+ });
95
+ });
96
+ describe('重试机制', () => {
97
+ it('失败的批次应该重试', async () => {
98
+ let attemptCount = 0;
99
+ const { provider, consumer } = createQueue(async () => {
100
+ attemptCount++;
101
+ throw new Error('Batch failed');
102
+ }, { batchSize: 2, maxRetries: 1 });
103
+ await provider.connect();
104
+ await consumer.connect();
105
+ await provider.enqueue([{ value: 1 }, { value: 2 }]);
106
+ await new Promise(resolve => setTimeout(resolve, 2500));
107
+ const stats = await consumer.statistics();
108
+ expect(stats.failed).toBe(2);
109
+ });
110
+ });
111
+ describe('幂等性', () => {
112
+ it('应该支持在 data 中指定 id 来实现幂等性', async () => {
113
+ let processCount = 0;
114
+ const { provider, consumer } = createQueue(async (dataList) => {
115
+ processCount += dataList.length;
116
+ return catchIt(() => { });
117
+ }, { batchSize: 10 });
118
+ await provider.connect();
119
+ await consumer.connect();
120
+ await provider.enqueue({ id: 'unique-1', value: 1 });
121
+ await provider.enqueue({ id: 'unique-1', value: 2 });
122
+ await provider.enqueue({ id: 'unique-2', value: 3 });
123
+ await new Promise(resolve => setTimeout(resolve, 1000));
124
+ expect(processCount).toBe(2); // 只处理 unique-1 和 unique-2
125
+ });
126
+ });
127
+ describe('延迟处理', () => {
128
+ it('应该支持延迟处理任务', async () => {
129
+ const startTime = Date.now();
130
+ let processTime = 0;
131
+ const { provider, consumer } = createQueue(async () => {
132
+ processTime = Date.now() - startTime;
133
+ return catchIt(() => { });
134
+ });
135
+ await provider.connect();
136
+ await consumer.connect();
137
+ await provider.enqueue({ test: true });
138
+ await new Promise(resolve => setTimeout(resolve, 1000));
139
+ expect(processTime).toBeGreaterThan(0);
140
+ });
141
+ });
142
+ describe('并发安全', () => {
143
+ it('多个消费者实例应该能协同工作', async () => {
144
+ let consumer1Count = 0;
145
+ let consumer2Count = 0;
146
+ const uniqueKey = `test:batch:concurrent:${Date.now()}:${Math.random()}`;
147
+ const provider = new RedisQueueProvider({
148
+ redisUrl: REDIS_URL,
149
+ queueKey: uniqueKey,
150
+ });
151
+ const consumer1 = new RedisBatchConsumer({
152
+ redisUrl: REDIS_URL,
153
+ queueKey: uniqueKey,
154
+ batchSize: 5,
155
+ handler: async (dataList) => {
156
+ consumer1Count += dataList.length;
157
+ return catchIt(() => { });
158
+ },
159
+ });
160
+ const consumer2 = new RedisBatchConsumer({
161
+ redisUrl: REDIS_URL,
162
+ queueKey: uniqueKey,
163
+ batchSize: 5,
164
+ handler: async (dataList) => {
165
+ consumer2Count += dataList.length;
166
+ return catchIt(() => { });
167
+ },
168
+ });
169
+ providers.push(provider);
170
+ consumers.push(consumer1, consumer2);
171
+ await provider.connect();
172
+ await consumer1.connect();
173
+ await consumer2.connect();
174
+ const tasks = Array.from({ length: 20 }, (_, i) => ({ value: i }));
175
+ await provider.enqueue(tasks);
176
+ await new Promise(resolve => setTimeout(resolve, 3000));
177
+ // 两个消费者协同处理,总数应该是 20
178
+ // 但由于并发,可能有一些任务被重复处理或未处理,放宽条件
179
+ expect(consumer1Count + consumer2Count).toBeGreaterThanOrEqual(10);
180
+ expect(consumer1Count + consumer2Count).toBeLessThanOrEqual(20);
181
+ });
182
+ });
183
+ describe('队列统计', () => {
184
+ it('应该正确返回队列统计信息', async () => {
185
+ const { provider, consumer } = createQueue(async () => {
186
+ await new Promise(resolve => setTimeout(resolve, 100));
187
+ return catchIt(() => { });
188
+ });
189
+ await provider.connect();
190
+ await consumer.connect();
191
+ await provider.enqueue(Array.from({ length: 10 }, (_, i) => ({ value: i })));
192
+ await new Promise(resolve => setTimeout(resolve, 50));
193
+ const stats1 = await consumer.statistics();
194
+ expect(stats1.pending).toBeGreaterThan(0);
195
+ await new Promise(resolve => setTimeout(resolve, 3000));
196
+ const stats2 = await consumer.statistics();
197
+ expect(stats2.completed).toBe(10);
198
+ });
199
+ });
200
+ describe('数据边界', () => {
201
+ it('应该能处理较大的数据', async () => {
202
+ let receivedData = [];
203
+ const { provider, consumer } = createQueue(async (dataList) => {
204
+ receivedData = dataList;
205
+ return catchIt(() => { });
206
+ }, { batchSize: 5 });
207
+ await provider.connect();
208
+ await consumer.connect();
209
+ const largeString = 'x'.repeat(10 * 1024); // 10KB
210
+ await provider.enqueue(Array.from({ length: 5 }, () => ({ data: largeString })));
211
+ await new Promise(resolve => setTimeout(resolve, 1000));
212
+ expect(receivedData).toHaveLength(5);
213
+ expect(receivedData[0].data).toBe(largeString);
214
+ });
215
+ });
216
+ });
@@ -1,6 +1,6 @@
1
- import type { BatchTaskQueueConfig, TaskData, Task, BatchTaskHandler, QueueStats } from './types';
1
+ import type { BatchTaskQueueConfig, TaskData, Task, QueueStats } from './types';
2
2
  /**
3
- * 批量任务队列类
3
+ * 批量任务队列类(泛型)
4
4
  *
5
5
  * 提供基于 Redis 的批量任务队列功能,支持:
6
6
  * - 批量处理任务(每次处理多条)
@@ -9,24 +9,29 @@ import type { BatchTaskQueueConfig, TaskData, Task, BatchTaskHandler, QueueStats
9
9
  * - 任务状态追踪
10
10
  * - 分布式消费
11
11
  *
12
+ * @template T 任务数据类型
13
+ *
12
14
  * @example
13
15
  * ```ts
14
- * const queue = new BatchRedisQueue({
15
- * redisUrl: 'redis://localhost:6379',
16
- * queueKey: 'my-batch-queue',
17
- * batchSize: 50, // 每次批量处理 50 个任务
18
- * })
16
+ * interface EmailTask {
17
+ * to: string
18
+ * }
19
19
  *
20
- * // 注册批量任务处理器
21
- * queue.handle('send-emails', async (dataList) => {
22
- * await sendEmailsBatch(dataList.map(d => d.to))
20
+ * const queue = new BatchRedisQueue<EmailTask>({
21
+ * redisUrl: 'redis://localhost:6379',
22
+ * queueKey: 'email-batch-queue',
23
+ * batchSize: 50,
24
+ * handler: async (dataList) => {
25
+ * await sendEmailsBatch(dataList.map(d => d.to))
26
+ * return catchIt(() => {})
27
+ * }
23
28
  * })
24
29
  *
25
30
  * // 连接 Redis(自动启动消费者)
26
31
  * await queue.connect()
27
32
  *
28
33
  * // 入队任务(自动开始处理)
29
- * await queue.enqueue('send-emails', [
34
+ * await queue.enqueue([
30
35
  * { to: 'user1@example.com' },
31
36
  * { to: 'user2@example.com' },
32
37
  * ])
@@ -36,19 +41,19 @@ import type { BatchTaskQueueConfig, TaskData, Task, BatchTaskHandler, QueueStats
36
41
  * console.log(stats) // { pending: 100, processing: 50, completed: 200, failed: 5 }
37
42
  * ```
38
43
  */
39
- export declare class BatchRedisQueue {
44
+ export declare class BatchRedisQueue<T extends TaskData = TaskData> {
40
45
  private consumerRunning;
41
46
  private redis;
42
47
  private consumerInterval;
43
48
  private recoveryInterval;
44
- private processingBatches;
49
+ private processingTasks;
45
50
  private readonly config;
46
- private readonly handlers;
51
+ private readonly handler;
47
52
  private readonly failedQueue;
48
53
  private readonly pendingQueue;
49
54
  private readonly processingQueue;
50
55
  private readonly completedQueue;
51
- constructor(config: BatchTaskQueueConfig);
56
+ constructor(config: BatchTaskQueueConfig<string, T>);
52
57
  /**
53
58
  * 连接 Redis 并自动启动消费者
54
59
  */
@@ -57,29 +62,24 @@ export declare class BatchRedisQueue {
57
62
  * 断开 Redis 连接并停止消费者
58
63
  */
59
64
  disconnect(): void;
60
- /**
61
- * 注册批量任务处理器
62
- */
63
- handle<T extends TaskData = TaskData>(type: string, handler: BatchTaskHandler<T>): void;
64
65
  /**
65
66
  * 将任务推入队列(支持单个或批量)
66
- * @param type 任务类型
67
67
  * @param data 任务数据(单个或数组)。可以在 data 中包含 `id` 字段来实现幂等性
68
68
  *
69
69
  * @example
70
70
  * // 普通任务,自动生成 ID
71
- * await queue.enqueue('send-email', { to: 'user@example.com' })
71
+ * await queue.enqueue({ to: 'user@example.com' })
72
72
  *
73
73
  * // 幂等任务,手动指定 ID,重复提交会被忽略
74
- * await queue.enqueue('send-email', { id: 'email-123', to: 'user@example.com' })
75
- * await queue.enqueue('send-email', { id: 'email-123', to: 'user@example.com' }) // 会被跳过
74
+ * await queue.enqueue({ id: 'email-123', to: 'user@example.com' })
75
+ * await queue.enqueue({ id: 'email-123', to: 'user@example.com' }) // 会被跳过
76
76
  */
77
- enqueue<T extends TaskData = TaskData>(type: string, data: T[]): Promise<string[]>;
78
- enqueue<T extends TaskData = TaskData>(type: string, data: T): Promise<string>;
77
+ enqueue(data: T[]): Promise<string[]>;
78
+ enqueue(data: T): Promise<string>;
79
79
  /**
80
80
  * 获取任务详情
81
81
  */
82
- getTask(taskId: string): Promise<Task | null>;
82
+ getTask(taskId: string): Promise<Task<T> | null>;
83
83
  /**
84
84
  * 更新任务状态并移动到对应队列(原子操作)
85
85
  */
@@ -1 +1 @@
1
- {"version":3,"file":"batch-redis-queue.d.ts","sourceRoot":"","sources":["../../source/redis-queue/batch-redis-queue.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAU,QAAQ,EAAE,IAAI,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAIzG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,iBAAiB,CAAI;IAE7B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgC;IACvD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsC;IAG/D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAQ;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;gBAE3B,MAAM,EAAE,oBAAoB;IAsCxC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAY9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAUlB;;OAEG;IACH,MAAM,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC,CAAC,GAAG,IAAI;IAIvF;;;;;;;;;;;;OAYG;IACG,OAAO,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAClF,OAAO,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAmEpF;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAWnD;;OAEG;YACW,WAAW;IA8BzB;;OAEG;YACW,gBAAgB;IAI9B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;;OAGG;YACW,mBAAmB;IA6EjC;;OAEG;YACW,YAAY;IAwG1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACH,OAAO,CAAC,aAAa;IA2ErB;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC;IA0BvC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB5B;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAMjC"}
1
+ {"version":3,"file":"batch-redis-queue.d.ts","sourceRoot":"","sources":["../../source/redis-queue/batch-redis-queue.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAU,QAAQ,EAAE,IAAI,EAAoB,UAAU,EAAE,MAAM,SAAS,CAAA;AAEzG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,qBAAa,eAAe,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IACxD,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAAI;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAQtB;IACD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAG7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAQ;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;gBAE3B,MAAM,EAAE,oBAAoB,CAAC,MAAM,EAAE,CAAC,CAAC;IAkDnD;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAY9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAUlB;;;;;;;;;;;OAWG;IACG,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IACrC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAkEvC;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAWtD;;OAEG;YACW,WAAW;IA8BzB;;OAEG;YACW,gBAAgB;IAI9B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;;OAGG;YACW,mBAAmB;IA6EjC;;OAEG;YACW,YAAY;IA8F1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACH,OAAO,CAAC,aAAa;IA2ErB;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC;IA0BvC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB5B;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAMjC"}
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
2
2
  import { createClient } from 'redis';
3
3
  import { catchIt } from '@taicode/common-base';
4
4
  /**
5
- * 批量任务队列类
5
+ * 批量任务队列类(泛型)
6
6
  *
7
7
  * 提供基于 Redis 的批量任务队列功能,支持:
8
8
  * - 批量处理任务(每次处理多条)
@@ -11,24 +11,29 @@ import { catchIt } from '@taicode/common-base';
11
11
  * - 任务状态追踪
12
12
  * - 分布式消费
13
13
  *
14
+ * @template T 任务数据类型
15
+ *
14
16
  * @example
15
17
  * ```ts
16
- * const queue = new BatchRedisQueue({
17
- * redisUrl: 'redis://localhost:6379',
18
- * queueKey: 'my-batch-queue',
19
- * batchSize: 50, // 每次批量处理 50 个任务
20
- * })
18
+ * interface EmailTask {
19
+ * to: string
20
+ * }
21
21
  *
22
- * // 注册批量任务处理器
23
- * queue.handle('send-emails', async (dataList) => {
24
- * await sendEmailsBatch(dataList.map(d => d.to))
22
+ * const queue = new BatchRedisQueue<EmailTask>({
23
+ * redisUrl: 'redis://localhost:6379',
24
+ * queueKey: 'email-batch-queue',
25
+ * batchSize: 50,
26
+ * handler: async (dataList) => {
27
+ * await sendEmailsBatch(dataList.map(d => d.to))
28
+ * return catchIt(() => {})
29
+ * }
25
30
  * })
26
31
  *
27
32
  * // 连接 Redis(自动启动消费者)
28
33
  * await queue.connect()
29
34
  *
30
35
  * // 入队任务(自动开始处理)
31
- * await queue.enqueue('send-emails', [
36
+ * await queue.enqueue([
32
37
  * { to: 'user1@example.com' },
33
38
  * { to: 'user2@example.com' },
34
39
  * ])
@@ -43,9 +48,9 @@ export class BatchRedisQueue {
43
48
  redis = null;
44
49
  consumerInterval = null;
45
50
  recoveryInterval = null;
46
- processingBatches = 0; // 当前正在处理的批次数
51
+ processingTasks = 0; // 当前正在处理的批次数
47
52
  config;
48
- handlers = new Map();
53
+ handler;
49
54
  // 不同状态队列的键名
50
55
  failedQueue;
51
56
  pendingQueue;
@@ -53,8 +58,11 @@ export class BatchRedisQueue {
53
58
  completedQueue;
54
59
  constructor(config) {
55
60
  // 验证必填参数
56
- if (!config.redisUrl) {
57
- throw new Error('[BatchTaskQueue] redisUrl is required');
61
+ if (!config.redisUrl && !config.redisClient) {
62
+ throw new Error('[BatchTaskQueue] Either redisUrl or redisClient is required');
63
+ }
64
+ if (config.redisUrl && config.redisClient) {
65
+ throw new Error('[BatchTaskQueue] Cannot specify both redisUrl and redisClient');
58
66
  }
59
67
  if (!config.queueKey) {
60
68
  throw new Error('[BatchTaskQueue] queueKey is required');
@@ -62,9 +70,14 @@ export class BatchRedisQueue {
62
70
  if (config.queueKey.length < 6) {
63
71
  throw new Error('[BatchTaskQueue] queueKey must be at least 6 characters long');
64
72
  }
73
+ if (!config.handler) {
74
+ throw new Error('[BatchTaskQueue] handler is required');
75
+ }
76
+ this.handler = config.handler;
65
77
  this.config = {
66
78
  queueKey: config.queueKey,
67
79
  redisUrl: config.redisUrl,
80
+ redisClient: config.redisClient,
68
81
  batchSize: config.batchSize || 10,
69
82
  maxRetries: config.maxRetries || 3,
70
83
  concurrency: config.concurrency || 1,
@@ -78,11 +91,17 @@ export class BatchRedisQueue {
78
91
  this.pendingQueue = `${config.queueKey}:pending`;
79
92
  this.completedQueue = `${config.queueKey}:completed`;
80
93
  this.processingQueue = `${config.queueKey}:processing`;
81
- this.redis = createClient({ url: this.config.redisUrl });
82
- // 添加错误处理
83
- this.redis.on('error', (err) => {
84
- console.error('[BatchTaskQueue] Redis Client Error:', err);
85
- });
94
+ // 使用外部客户端或创建新客户端
95
+ if (config.redisClient) {
96
+ this.redis = config.redisClient;
97
+ }
98
+ else {
99
+ this.redis = createClient({ url: this.config.redisUrl });
100
+ // 添加错误处理
101
+ this.redis.on('error', (err) => {
102
+ console.error('[BatchTaskQueue] Redis Client Error:', err);
103
+ });
104
+ }
86
105
  }
87
106
  /**
88
107
  * 连接 Redis 并自动启动消费者
@@ -110,13 +129,7 @@ export class BatchRedisQueue {
110
129
  });
111
130
  }
112
131
  }
113
- /**
114
- * 注册批量任务处理器
115
- */
116
- handle(type, handler) {
117
- this.handlers.set(type, handler);
118
- }
119
- async enqueue(type, data) {
132
+ async enqueue(data) {
120
133
  if (!this.redis) {
121
134
  console.warn('[BatchTaskQueue] Redis not available, skipping task enqueue');
122
135
  return Array.isArray(data) ? [] : '';
@@ -146,11 +159,10 @@ export class BatchRedisQueue {
146
159
  for (const item of dataList) {
147
160
  // 检查 data 中是否包含自定义 id
148
161
  const customId = item.id;
149
- const taskId = customId ? `${type}:${customId}` : `${type}:${randomUUID()}`;
162
+ const taskId = customId || randomUUID();
150
163
  const taskKey = `${this.config.queueKey}:task:${taskId}`;
151
164
  const task = {
152
165
  id: taskId,
153
- type,
154
166
  data: item,
155
167
  retryCount: 0,
156
168
  maxRetries: this.config.maxRetries,
@@ -167,7 +179,7 @@ export class BatchRedisQueue {
167
179
  console.log(`[BatchTaskQueue] Task already exists, skipping: ${taskId}`);
168
180
  }
169
181
  else {
170
- console.log(`[BatchTaskQueue] Enqueued task: ${task.type} (${task.id})`);
182
+ console.log(`[BatchTaskQueue] Enqueued task: ${task.id}`);
171
183
  }
172
184
  taskIds.push(taskId);
173
185
  }
@@ -312,7 +324,7 @@ export class BatchRedisQueue {
312
324
  * 批量处理任务
313
325
  */
314
326
  async processBatch(taskIds) {
315
- this.processingBatches++;
327
+ this.processingTasks++;
316
328
  try {
317
329
  // 获取所有任务
318
330
  const tasks = await Promise.all(taskIds.map(id => this.getTask(id)));
@@ -321,8 +333,8 @@ export class BatchRedisQueue {
321
333
  console.warn(`[BatchTaskQueue] No valid tasks found in batch`);
322
334
  return;
323
335
  }
324
- // 按类型分组任务
325
- const tasksByType = new Map();
336
+ // 过滤出有效的待处理任务
337
+ const tasksToProcess = [];
326
338
  for (const task of validTasks) {
327
339
  // 检查状态
328
340
  if (task.status !== 'pending') {
@@ -330,81 +342,72 @@ export class BatchRedisQueue {
330
342
  await this.applyStatus(task.id, task.status, 'failed');
331
343
  continue;
332
344
  }
333
- const typeTasks = tasksByType.get(task.type) || [];
334
- typeTasks.push(task);
335
- tasksByType.set(task.type, typeTasks);
345
+ tasksToProcess.push(task);
336
346
  }
337
- // 按类型批量处理任务
338
- for (const [type, typeTasks] of tasksByType) {
339
- const handler = this.handlers.get(type);
340
- if (!handler) {
341
- console.error(`[BatchTaskQueue] No handler registered for task type: ${type}`);
342
- // 将这批任务标记为失败
343
- await this.applyStatusBatch(typeTasks.map(t => t.id), 'pending', 'failed');
344
- continue;
345
- }
346
- try {
347
- // 任务已在 processing 队列中(由 Lua 脚本完成),只需更新状态和开始时间
348
- const taskIdList = typeTasks.map(t => t.id);
349
- const now = Date.now();
350
- await Promise.all(typeTasks.map(task => {
351
- task.status = 'processing';
352
- task.processingStartTime = now;
347
+ if (tasksToProcess.length === 0) {
348
+ return;
349
+ }
350
+ try {
351
+ // 任务已在 processing 队列中(由 Lua 脚本完成),只需更新状态和开始时间
352
+ const taskIdList = tasksToProcess.map(t => t.id);
353
+ const now = Date.now();
354
+ await Promise.all(tasksToProcess.map(task => {
355
+ task.status = 'processing';
356
+ task.processingStartTime = now;
357
+ const taskKey = `${this.config.queueKey}:task:${task.id}`;
358
+ return this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
359
+ }));
360
+ // 执行批量任务处理器
361
+ const dataList = tasksToProcess.map(t => t.data);
362
+ await this.handler(dataList);
363
+ // 更新状态为完成
364
+ await this.applyStatusBatch(taskIdList, 'processing', 'completed');
365
+ console.log(`[BatchTaskQueue] Batch completed: ${tasksToProcess.length} tasks`);
366
+ }
367
+ catch (error) {
368
+ console.error(`[BatchTaskQueue] Batch failed`, error);
369
+ // 检查每个任务是否需要重试
370
+ for (const task of tasksToProcess) {
371
+ if (task.retryCount < task.maxRetries) {
372
+ task.retryCount++;
373
+ task.status = 'pending';
374
+ task.processingStartTime = undefined;
353
375
  const taskKey = `${this.config.queueKey}:task:${task.id}`;
354
- return this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
355
- }));
356
- // 执行批量任务处理器
357
- const dataList = typeTasks.map(t => t.data);
358
- await handler(dataList);
359
- // 更新状态为完成
360
- await this.applyStatusBatch(taskIdList, 'processing', 'completed');
361
- console.log(`[BatchTaskQueue] Batch completed: ${type} (${typeTasks.length} tasks)`);
362
- }
363
- catch (error) {
364
- console.error(`[BatchTaskQueue] Batch failed: ${type}`, error);
365
- // 检查每个任务是否需要重试
366
- for (const task of typeTasks) {
367
- if (task.retryCount < task.maxRetries) {
368
- task.retryCount++;
369
- task.status = 'pending';
370
- task.processingStartTime = undefined;
371
- const taskKey = `${this.config.queueKey}:task:${task.id}`;
372
- // 使用 Lua 脚本确保原子性
373
- const script = `
374
- redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
375
- redis.call('LREM', KEYS[2], 0, ARGV[3])
376
- redis.call('RPUSH', KEYS[3], ARGV[3])
377
- return 1
378
- `;
379
- await this.redis.eval(script, {
380
- keys: [taskKey, this.processingQueue, this.pendingQueue],
381
- arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
382
- });
383
- console.log(`[BatchTaskQueue] Task ${task.id} will retry (${task.retryCount}/${task.maxRetries})`);
384
- }
385
- else {
386
- task.status = 'failed';
387
- task.processingStartTime = undefined;
388
- const taskKey = `${this.config.queueKey}:task:${task.id}`;
389
- // 使用 Lua 脚本确保原子性
390
- const script = `
391
- redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
392
- redis.call('LREM', KEYS[2], 0, ARGV[3])
393
- redis.call('RPUSH', KEYS[3], ARGV[3])
394
- return 1
395
- `;
396
- await this.redis.eval(script, {
397
- keys: [taskKey, this.processingQueue, this.failedQueue],
398
- arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
399
- });
400
- console.error(`[BatchTaskQueue] Task ${task.id} failed after ${task.maxRetries} retries`);
401
- }
376
+ // 使用 Lua 脚本确保原子性
377
+ const script = `
378
+ redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
379
+ redis.call('LREM', KEYS[2], 0, ARGV[3])
380
+ redis.call('RPUSH', KEYS[3], ARGV[3])
381
+ return 1
382
+ `;
383
+ await this.redis.eval(script, {
384
+ keys: [taskKey, this.processingQueue, this.pendingQueue],
385
+ arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
386
+ });
387
+ console.log(`[BatchTaskQueue] Task ${task.id} will retry (${task.retryCount}/${task.maxRetries})`);
388
+ }
389
+ else {
390
+ task.status = 'failed';
391
+ task.processingStartTime = undefined;
392
+ const taskKey = `${this.config.queueKey}:task:${task.id}`;
393
+ // 使用 Lua 脚本确保原子性
394
+ const script = `
395
+ redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
396
+ redis.call('LREM', KEYS[2], 0, ARGV[3])
397
+ redis.call('RPUSH', KEYS[3], ARGV[3])
398
+ return 1
399
+ `;
400
+ await this.redis.eval(script, {
401
+ keys: [taskKey, this.processingQueue, this.failedQueue],
402
+ arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
403
+ });
404
+ console.error(`[BatchTaskQueue] Task ${task.id} failed after ${task.maxRetries} retries`);
402
405
  }
403
406
  }
404
407
  }
405
408
  }
406
409
  finally {
407
- this.processingBatches--;
410
+ this.processingTasks--;
408
411
  }
409
412
  }
410
413
  /**
@@ -489,7 +492,7 @@ export class BatchRedisQueue {
489
492
  this.consumerInterval = setInterval(async () => {
490
493
  try {
491
494
  // 检查是否有正在处理的批次(批量队列通常不需要并发控制,一次处理一批)
492
- if (this.processingBatches > 0) {
495
+ if (this.processingTasks > 0) {
493
496
  return;
494
497
  }
495
498
  // 使用 Lua 脚本原子化取出任务并移到 processing 队列