@taicode/common-server 1.0.10 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/output/redis-queue/batch-consumer.d.ts +107 -0
- package/output/redis-queue/batch-consumer.d.ts.map +1 -0
- package/output/redis-queue/batch-consumer.js +492 -0
- package/output/redis-queue/batch-consumer.test.d.ts +2 -0
- package/output/redis-queue/batch-consumer.test.d.ts.map +1 -0
- package/output/redis-queue/batch-consumer.test.js +216 -0
- package/output/redis-queue/batch-redis-queue.d.ts +26 -26
- package/output/redis-queue/batch-redis-queue.d.ts.map +1 -1
- package/output/redis-queue/batch-redis-queue.js +104 -101
- package/output/redis-queue/batch-redis-queue.test.js +96 -251
- package/output/redis-queue/index.d.ts +5 -4
- package/output/redis-queue/index.d.ts.map +1 -1
- package/output/redis-queue/index.js +4 -2
- package/output/redis-queue/redis-batch-consumer.d.ts +85 -0
- package/output/redis-queue/redis-batch-consumer.d.ts.map +1 -0
- package/output/redis-queue/redis-batch-consumer.js +360 -0
- package/output/redis-queue/redis-batch-consumer.test.d.ts +2 -0
- package/output/redis-queue/redis-batch-consumer.test.d.ts.map +1 -0
- package/output/redis-queue/redis-batch-consumer.test.js +265 -0
- package/output/redis-queue/redis-queue-common.d.ts +61 -0
- package/output/redis-queue/redis-queue-common.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-common.js +222 -0
- package/output/redis-queue/redis-queue-common.test.d.ts +2 -0
- package/output/redis-queue/redis-queue-common.test.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-common.test.js +623 -0
- package/output/redis-queue/redis-queue-consumer.d.ts +102 -0
- package/output/redis-queue/redis-queue-consumer.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-consumer.js +461 -0
- package/output/redis-queue/redis-queue-consumer.test.d.ts +2 -0
- package/output/redis-queue/redis-queue-consumer.test.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-consumer.test.js +242 -0
- package/output/redis-queue/redis-queue-provider.d.ts +57 -0
- package/output/redis-queue/redis-queue-provider.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-provider.js +188 -0
- package/output/redis-queue/redis-queue-provider.test.d.ts +2 -0
- package/output/redis-queue/redis-queue-provider.test.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-provider.test.js +114 -0
- package/output/redis-queue/redis-queue.d.ts +26 -25
- package/output/redis-queue/redis-queue.d.ts.map +1 -1
- package/output/redis-queue/redis-queue.js +42 -35
- package/output/redis-queue/redis-queue.test.js +96 -698
- package/output/redis-queue/registry.d.ts +57 -0
- package/output/redis-queue/registry.d.ts.map +1 -0
- package/output/redis-queue/registry.js +30 -0
- package/output/redis-queue/types.d.ts +42 -13
- package/output/redis-queue/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { describe, it, expect, afterEach } from 'vitest';
|
|
2
|
+
import { catchIt } from '@taicode/common-base';
|
|
3
|
+
import { RedisQueueProvider } from './redis-queue-provider';
|
|
4
|
+
import { RedisBatchConsumer } from './batch-consumer';
|
|
5
|
+
const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
|
|
6
|
+
describe('RedisBatchConsumer', () => {
|
|
7
|
+
const providers = [];
|
|
8
|
+
const consumers = [];
|
|
9
|
+
afterEach(async () => {
|
|
10
|
+
for (const consumer of consumers) {
|
|
11
|
+
consumer.disconnect();
|
|
12
|
+
}
|
|
13
|
+
for (const provider of providers) {
|
|
14
|
+
try {
|
|
15
|
+
await provider.clear();
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
// 忽略清理错误
|
|
19
|
+
}
|
|
20
|
+
provider.disconnect();
|
|
21
|
+
}
|
|
22
|
+
providers.length = 0;
|
|
23
|
+
consumers.length = 0;
|
|
24
|
+
});
|
|
25
|
+
const createQueue = (handler, options) => {
|
|
26
|
+
const uniqueKey = `test:batch:${Date.now()}:${Math.random()}`;
|
|
27
|
+
const provider = new RedisQueueProvider({
|
|
28
|
+
redisUrl: REDIS_URL,
|
|
29
|
+
queueKey: uniqueKey,
|
|
30
|
+
});
|
|
31
|
+
const consumer = new RedisBatchConsumer({
|
|
32
|
+
redisUrl: REDIS_URL,
|
|
33
|
+
queueKey: uniqueKey,
|
|
34
|
+
consumerInterval: 100,
|
|
35
|
+
maxRetries: 2,
|
|
36
|
+
batchSize: 10,
|
|
37
|
+
...options,
|
|
38
|
+
handler,
|
|
39
|
+
});
|
|
40
|
+
providers.push(provider);
|
|
41
|
+
consumers.push(consumer);
|
|
42
|
+
return { provider, consumer };
|
|
43
|
+
};
|
|
44
|
+
describe('连接管理', () => {
|
|
45
|
+
it('应该成功连接到 Redis', async () => {
|
|
46
|
+
const { consumer } = createQueue(async () => catchIt(() => { }));
|
|
47
|
+
await consumer.connect();
|
|
48
|
+
const health = await consumer.health();
|
|
49
|
+
expect(health).toBe(true);
|
|
50
|
+
});
|
|
51
|
+
it('连接后应该自动启动消费者', async () => {
|
|
52
|
+
let processed = false;
|
|
53
|
+
const { provider, consumer } = createQueue(async () => {
|
|
54
|
+
processed = true;
|
|
55
|
+
return catchIt(() => { });
|
|
56
|
+
});
|
|
57
|
+
await provider.connect();
|
|
58
|
+
await consumer.connect();
|
|
59
|
+
await provider.enqueue({ test: true });
|
|
60
|
+
await new Promise(resolve => setTimeout(resolve, 300));
|
|
61
|
+
expect(processed).toBe(true);
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
describe('批量处理', () => {
|
|
65
|
+
it('应该批量处理任务', async () => {
|
|
66
|
+
let batchSize = 0;
|
|
67
|
+
const { provider, consumer } = createQueue(async (dataList) => {
|
|
68
|
+
batchSize = dataList.length;
|
|
69
|
+
return catchIt(() => { });
|
|
70
|
+
}, { batchSize: 5 });
|
|
71
|
+
await provider.connect();
|
|
72
|
+
await consumer.connect();
|
|
73
|
+
await provider.enqueue([
|
|
74
|
+
{ value: 1 }, { value: 2 }, { value: 3 },
|
|
75
|
+
{ value: 4 }, { value: 5 },
|
|
76
|
+
]);
|
|
77
|
+
await new Promise(resolve => setTimeout(resolve, 500));
|
|
78
|
+
expect(batchSize).toBe(5);
|
|
79
|
+
const stats = await consumer.statistics();
|
|
80
|
+
expect(stats.completed).toBe(5);
|
|
81
|
+
});
|
|
82
|
+
it('应该能处理大批量任务', async () => {
|
|
83
|
+
let totalProcessed = 0;
|
|
84
|
+
const { provider, consumer } = createQueue(async (dataList) => {
|
|
85
|
+
totalProcessed += dataList.length;
|
|
86
|
+
return catchIt(() => { });
|
|
87
|
+
}, { batchSize: 50 });
|
|
88
|
+
await provider.connect();
|
|
89
|
+
await consumer.connect();
|
|
90
|
+
const tasks = Array.from({ length: 200 }, (_, i) => ({ value: i }));
|
|
91
|
+
await provider.enqueue(tasks);
|
|
92
|
+
await new Promise(resolve => setTimeout(resolve, 3000));
|
|
93
|
+
expect(totalProcessed).toBe(200);
|
|
94
|
+
});
|
|
95
|
+
});
|
|
96
|
+
describe('重试机制', () => {
|
|
97
|
+
it('失败的批次应该重试', async () => {
|
|
98
|
+
let attemptCount = 0;
|
|
99
|
+
const { provider, consumer } = createQueue(async () => {
|
|
100
|
+
attemptCount++;
|
|
101
|
+
throw new Error('Batch failed');
|
|
102
|
+
}, { batchSize: 2, maxRetries: 1 });
|
|
103
|
+
await provider.connect();
|
|
104
|
+
await consumer.connect();
|
|
105
|
+
await provider.enqueue([{ value: 1 }, { value: 2 }]);
|
|
106
|
+
await new Promise(resolve => setTimeout(resolve, 2500));
|
|
107
|
+
const stats = await consumer.statistics();
|
|
108
|
+
expect(stats.failed).toBe(2);
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
describe('幂等性', () => {
|
|
112
|
+
it('应该支持在 data 中指定 id 来实现幂等性', async () => {
|
|
113
|
+
let processCount = 0;
|
|
114
|
+
const { provider, consumer } = createQueue(async (dataList) => {
|
|
115
|
+
processCount += dataList.length;
|
|
116
|
+
return catchIt(() => { });
|
|
117
|
+
}, { batchSize: 10 });
|
|
118
|
+
await provider.connect();
|
|
119
|
+
await consumer.connect();
|
|
120
|
+
await provider.enqueue({ id: 'unique-1', value: 1 });
|
|
121
|
+
await provider.enqueue({ id: 'unique-1', value: 2 });
|
|
122
|
+
await provider.enqueue({ id: 'unique-2', value: 3 });
|
|
123
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
124
|
+
expect(processCount).toBe(2); // 只处理 unique-1 和 unique-2
|
|
125
|
+
});
|
|
126
|
+
});
|
|
127
|
+
describe('延迟处理', () => {
|
|
128
|
+
it('应该支持延迟处理任务', async () => {
|
|
129
|
+
const startTime = Date.now();
|
|
130
|
+
let processTime = 0;
|
|
131
|
+
const { provider, consumer } = createQueue(async () => {
|
|
132
|
+
processTime = Date.now() - startTime;
|
|
133
|
+
return catchIt(() => { });
|
|
134
|
+
});
|
|
135
|
+
await provider.connect();
|
|
136
|
+
await consumer.connect();
|
|
137
|
+
await provider.enqueue({ test: true });
|
|
138
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
139
|
+
expect(processTime).toBeGreaterThan(0);
|
|
140
|
+
});
|
|
141
|
+
});
|
|
142
|
+
describe('并发安全', () => {
|
|
143
|
+
it('多个消费者实例应该能协同工作', async () => {
|
|
144
|
+
let consumer1Count = 0;
|
|
145
|
+
let consumer2Count = 0;
|
|
146
|
+
const uniqueKey = `test:batch:concurrent:${Date.now()}:${Math.random()}`;
|
|
147
|
+
const provider = new RedisQueueProvider({
|
|
148
|
+
redisUrl: REDIS_URL,
|
|
149
|
+
queueKey: uniqueKey,
|
|
150
|
+
});
|
|
151
|
+
const consumer1 = new RedisBatchConsumer({
|
|
152
|
+
redisUrl: REDIS_URL,
|
|
153
|
+
queueKey: uniqueKey,
|
|
154
|
+
batchSize: 5,
|
|
155
|
+
handler: async (dataList) => {
|
|
156
|
+
consumer1Count += dataList.length;
|
|
157
|
+
return catchIt(() => { });
|
|
158
|
+
},
|
|
159
|
+
});
|
|
160
|
+
const consumer2 = new RedisBatchConsumer({
|
|
161
|
+
redisUrl: REDIS_URL,
|
|
162
|
+
queueKey: uniqueKey,
|
|
163
|
+
batchSize: 5,
|
|
164
|
+
handler: async (dataList) => {
|
|
165
|
+
consumer2Count += dataList.length;
|
|
166
|
+
return catchIt(() => { });
|
|
167
|
+
},
|
|
168
|
+
});
|
|
169
|
+
providers.push(provider);
|
|
170
|
+
consumers.push(consumer1, consumer2);
|
|
171
|
+
await provider.connect();
|
|
172
|
+
await consumer1.connect();
|
|
173
|
+
await consumer2.connect();
|
|
174
|
+
const tasks = Array.from({ length: 20 }, (_, i) => ({ value: i }));
|
|
175
|
+
await provider.enqueue(tasks);
|
|
176
|
+
await new Promise(resolve => setTimeout(resolve, 3000));
|
|
177
|
+
// 两个消费者协同处理,总数应该是 20
|
|
178
|
+
// 但由于并发,可能有一些任务被重复处理或未处理,放宽条件
|
|
179
|
+
expect(consumer1Count + consumer2Count).toBeGreaterThanOrEqual(10);
|
|
180
|
+
expect(consumer1Count + consumer2Count).toBeLessThanOrEqual(20);
|
|
181
|
+
});
|
|
182
|
+
});
|
|
183
|
+
describe('队列统计', () => {
|
|
184
|
+
it('应该正确返回队列统计信息', async () => {
|
|
185
|
+
const { provider, consumer } = createQueue(async () => {
|
|
186
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
187
|
+
return catchIt(() => { });
|
|
188
|
+
});
|
|
189
|
+
await provider.connect();
|
|
190
|
+
await consumer.connect();
|
|
191
|
+
await provider.enqueue(Array.from({ length: 10 }, (_, i) => ({ value: i })));
|
|
192
|
+
await new Promise(resolve => setTimeout(resolve, 50));
|
|
193
|
+
const stats1 = await consumer.statistics();
|
|
194
|
+
expect(stats1.pending).toBeGreaterThan(0);
|
|
195
|
+
await new Promise(resolve => setTimeout(resolve, 3000));
|
|
196
|
+
const stats2 = await consumer.statistics();
|
|
197
|
+
expect(stats2.completed).toBe(10);
|
|
198
|
+
});
|
|
199
|
+
});
|
|
200
|
+
describe('数据边界', () => {
|
|
201
|
+
it('应该能处理较大的数据', async () => {
|
|
202
|
+
let receivedData = [];
|
|
203
|
+
const { provider, consumer } = createQueue(async (dataList) => {
|
|
204
|
+
receivedData = dataList;
|
|
205
|
+
return catchIt(() => { });
|
|
206
|
+
}, { batchSize: 5 });
|
|
207
|
+
await provider.connect();
|
|
208
|
+
await consumer.connect();
|
|
209
|
+
const largeString = 'x'.repeat(10 * 1024); // 10KB
|
|
210
|
+
await provider.enqueue(Array.from({ length: 5 }, () => ({ data: largeString })));
|
|
211
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
212
|
+
expect(receivedData).toHaveLength(5);
|
|
213
|
+
expect(receivedData[0].data).toBe(largeString);
|
|
214
|
+
});
|
|
215
|
+
});
|
|
216
|
+
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { BatchTaskQueueConfig, TaskData, Task,
|
|
1
|
+
import type { BatchTaskQueueConfig, TaskData, Task, QueueStats } from './types';
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
3
|
+
* 批量任务队列类(泛型)
|
|
4
4
|
*
|
|
5
5
|
* 提供基于 Redis 的批量任务队列功能,支持:
|
|
6
6
|
* - 批量处理任务(每次处理多条)
|
|
@@ -9,24 +9,29 @@ import type { BatchTaskQueueConfig, TaskData, Task, BatchTaskHandler, QueueStats
|
|
|
9
9
|
* - 任务状态追踪
|
|
10
10
|
* - 分布式消费
|
|
11
11
|
*
|
|
12
|
+
* @template T 任务数据类型
|
|
13
|
+
*
|
|
12
14
|
* @example
|
|
13
15
|
* ```ts
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* batchSize: 50, // 每次批量处理 50 个任务
|
|
18
|
-
* })
|
|
16
|
+
* interface EmailTask {
|
|
17
|
+
* to: string
|
|
18
|
+
* }
|
|
19
19
|
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
20
|
+
* const queue = new BatchRedisQueue<EmailTask>({
|
|
21
|
+
* redisUrl: 'redis://localhost:6379',
|
|
22
|
+
* queueKey: 'email-batch-queue',
|
|
23
|
+
* batchSize: 50,
|
|
24
|
+
* handler: async (dataList) => {
|
|
25
|
+
* await sendEmailsBatch(dataList.map(d => d.to))
|
|
26
|
+
* return catchIt(() => {})
|
|
27
|
+
* }
|
|
23
28
|
* })
|
|
24
29
|
*
|
|
25
30
|
* // 连接 Redis(自动启动消费者)
|
|
26
31
|
* await queue.connect()
|
|
27
32
|
*
|
|
28
33
|
* // 入队任务(自动开始处理)
|
|
29
|
-
* await queue.enqueue(
|
|
34
|
+
* await queue.enqueue([
|
|
30
35
|
* { to: 'user1@example.com' },
|
|
31
36
|
* { to: 'user2@example.com' },
|
|
32
37
|
* ])
|
|
@@ -36,19 +41,19 @@ import type { BatchTaskQueueConfig, TaskData, Task, BatchTaskHandler, QueueStats
|
|
|
36
41
|
* console.log(stats) // { pending: 100, processing: 50, completed: 200, failed: 5 }
|
|
37
42
|
* ```
|
|
38
43
|
*/
|
|
39
|
-
export declare class BatchRedisQueue {
|
|
44
|
+
export declare class BatchRedisQueue<T extends TaskData = TaskData> {
|
|
40
45
|
private consumerRunning;
|
|
41
46
|
private redis;
|
|
42
47
|
private consumerInterval;
|
|
43
48
|
private recoveryInterval;
|
|
44
|
-
private
|
|
49
|
+
private processingTasks;
|
|
45
50
|
private readonly config;
|
|
46
|
-
private readonly
|
|
51
|
+
private readonly handler;
|
|
47
52
|
private readonly failedQueue;
|
|
48
53
|
private readonly pendingQueue;
|
|
49
54
|
private readonly processingQueue;
|
|
50
55
|
private readonly completedQueue;
|
|
51
|
-
constructor(config: BatchTaskQueueConfig);
|
|
56
|
+
constructor(config: BatchTaskQueueConfig<string, T>);
|
|
52
57
|
/**
|
|
53
58
|
* 连接 Redis 并自动启动消费者
|
|
54
59
|
*/
|
|
@@ -57,29 +62,24 @@ export declare class BatchRedisQueue {
|
|
|
57
62
|
* 断开 Redis 连接并停止消费者
|
|
58
63
|
*/
|
|
59
64
|
disconnect(): void;
|
|
60
|
-
/**
|
|
61
|
-
* 注册批量任务处理器
|
|
62
|
-
*/
|
|
63
|
-
handle<T extends TaskData = TaskData>(type: string, handler: BatchTaskHandler<T>): void;
|
|
64
65
|
/**
|
|
65
66
|
* 将任务推入队列(支持单个或批量)
|
|
66
|
-
* @param type 任务类型
|
|
67
67
|
* @param data 任务数据(单个或数组)。可以在 data 中包含 `id` 字段来实现幂等性
|
|
68
68
|
*
|
|
69
69
|
* @example
|
|
70
70
|
* // 普通任务,自动生成 ID
|
|
71
|
-
* await queue.enqueue(
|
|
71
|
+
* await queue.enqueue({ to: 'user@example.com' })
|
|
72
72
|
*
|
|
73
73
|
* // 幂等任务,手动指定 ID,重复提交会被忽略
|
|
74
|
-
* await queue.enqueue(
|
|
75
|
-
* await queue.enqueue(
|
|
74
|
+
* await queue.enqueue({ id: 'email-123', to: 'user@example.com' })
|
|
75
|
+
* await queue.enqueue({ id: 'email-123', to: 'user@example.com' }) // 会被跳过
|
|
76
76
|
*/
|
|
77
|
-
enqueue
|
|
78
|
-
enqueue
|
|
77
|
+
enqueue(data: T[]): Promise<string[]>;
|
|
78
|
+
enqueue(data: T): Promise<string>;
|
|
79
79
|
/**
|
|
80
80
|
* 获取任务详情
|
|
81
81
|
*/
|
|
82
|
-
getTask(taskId: string): Promise<Task | null>;
|
|
82
|
+
getTask(taskId: string): Promise<Task<T> | null>;
|
|
83
83
|
/**
|
|
84
84
|
* 更新任务状态并移动到对应队列(原子操作)
|
|
85
85
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-redis-queue.d.ts","sourceRoot":"","sources":["../../source/redis-queue/batch-redis-queue.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAU,QAAQ,EAAE,IAAI,
|
|
1
|
+
{"version":3,"file":"batch-redis-queue.d.ts","sourceRoot":"","sources":["../../source/redis-queue/batch-redis-queue.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAU,QAAQ,EAAE,IAAI,EAAoB,UAAU,EAAE,MAAM,SAAS,CAAA;AAEzG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,qBAAa,eAAe,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IACxD,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAAI;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAQtB;IACD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAG7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAQ;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;gBAE3B,MAAM,EAAE,oBAAoB,CAAC,MAAM,EAAE,CAAC,CAAC;IAkDnD;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAY9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAUlB;;;;;;;;;;;OAWG;IACG,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IACrC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAkEvC;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAWtD;;OAEG;YACW,WAAW;IA8BzB;;OAEG;YACW,gBAAgB;IAI9B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;;OAGG;YACW,mBAAmB;IA6EjC;;OAEG;YACW,YAAY;IA8F1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACH,OAAO,CAAC,aAAa;IA2ErB;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC;IA0BvC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB5B;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAMjC"}
|
|
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { createClient } from 'redis';
|
|
3
3
|
import { catchIt } from '@taicode/common-base';
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* 批量任务队列类(泛型)
|
|
6
6
|
*
|
|
7
7
|
* 提供基于 Redis 的批量任务队列功能,支持:
|
|
8
8
|
* - 批量处理任务(每次处理多条)
|
|
@@ -11,24 +11,29 @@ import { catchIt } from '@taicode/common-base';
|
|
|
11
11
|
* - 任务状态追踪
|
|
12
12
|
* - 分布式消费
|
|
13
13
|
*
|
|
14
|
+
* @template T 任务数据类型
|
|
15
|
+
*
|
|
14
16
|
* @example
|
|
15
17
|
* ```ts
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* batchSize: 50, // 每次批量处理 50 个任务
|
|
20
|
-
* })
|
|
18
|
+
* interface EmailTask {
|
|
19
|
+
* to: string
|
|
20
|
+
* }
|
|
21
21
|
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
22
|
+
* const queue = new BatchRedisQueue<EmailTask>({
|
|
23
|
+
* redisUrl: 'redis://localhost:6379',
|
|
24
|
+
* queueKey: 'email-batch-queue',
|
|
25
|
+
* batchSize: 50,
|
|
26
|
+
* handler: async (dataList) => {
|
|
27
|
+
* await sendEmailsBatch(dataList.map(d => d.to))
|
|
28
|
+
* return catchIt(() => {})
|
|
29
|
+
* }
|
|
25
30
|
* })
|
|
26
31
|
*
|
|
27
32
|
* // 连接 Redis(自动启动消费者)
|
|
28
33
|
* await queue.connect()
|
|
29
34
|
*
|
|
30
35
|
* // 入队任务(自动开始处理)
|
|
31
|
-
* await queue.enqueue(
|
|
36
|
+
* await queue.enqueue([
|
|
32
37
|
* { to: 'user1@example.com' },
|
|
33
38
|
* { to: 'user2@example.com' },
|
|
34
39
|
* ])
|
|
@@ -43,9 +48,9 @@ export class BatchRedisQueue {
|
|
|
43
48
|
redis = null;
|
|
44
49
|
consumerInterval = null;
|
|
45
50
|
recoveryInterval = null;
|
|
46
|
-
|
|
51
|
+
processingTasks = 0; // 当前正在处理的批次数
|
|
47
52
|
config;
|
|
48
|
-
|
|
53
|
+
handler;
|
|
49
54
|
// 不同状态队列的键名
|
|
50
55
|
failedQueue;
|
|
51
56
|
pendingQueue;
|
|
@@ -53,8 +58,11 @@ export class BatchRedisQueue {
|
|
|
53
58
|
completedQueue;
|
|
54
59
|
constructor(config) {
|
|
55
60
|
// 验证必填参数
|
|
56
|
-
if (!config.redisUrl) {
|
|
57
|
-
throw new Error('[BatchTaskQueue] redisUrl is required');
|
|
61
|
+
if (!config.redisUrl && !config.redisClient) {
|
|
62
|
+
throw new Error('[BatchTaskQueue] Either redisUrl or redisClient is required');
|
|
63
|
+
}
|
|
64
|
+
if (config.redisUrl && config.redisClient) {
|
|
65
|
+
throw new Error('[BatchTaskQueue] Cannot specify both redisUrl and redisClient');
|
|
58
66
|
}
|
|
59
67
|
if (!config.queueKey) {
|
|
60
68
|
throw new Error('[BatchTaskQueue] queueKey is required');
|
|
@@ -62,9 +70,14 @@ export class BatchRedisQueue {
|
|
|
62
70
|
if (config.queueKey.length < 6) {
|
|
63
71
|
throw new Error('[BatchTaskQueue] queueKey must be at least 6 characters long');
|
|
64
72
|
}
|
|
73
|
+
if (!config.handler) {
|
|
74
|
+
throw new Error('[BatchTaskQueue] handler is required');
|
|
75
|
+
}
|
|
76
|
+
this.handler = config.handler;
|
|
65
77
|
this.config = {
|
|
66
78
|
queueKey: config.queueKey,
|
|
67
79
|
redisUrl: config.redisUrl,
|
|
80
|
+
redisClient: config.redisClient,
|
|
68
81
|
batchSize: config.batchSize || 10,
|
|
69
82
|
maxRetries: config.maxRetries || 3,
|
|
70
83
|
concurrency: config.concurrency || 1,
|
|
@@ -78,11 +91,17 @@ export class BatchRedisQueue {
|
|
|
78
91
|
this.pendingQueue = `${config.queueKey}:pending`;
|
|
79
92
|
this.completedQueue = `${config.queueKey}:completed`;
|
|
80
93
|
this.processingQueue = `${config.queueKey}:processing`;
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
94
|
+
// 使用外部客户端或创建新客户端
|
|
95
|
+
if (config.redisClient) {
|
|
96
|
+
this.redis = config.redisClient;
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
this.redis = createClient({ url: this.config.redisUrl });
|
|
100
|
+
// 添加错误处理
|
|
101
|
+
this.redis.on('error', (err) => {
|
|
102
|
+
console.error('[BatchTaskQueue] Redis Client Error:', err);
|
|
103
|
+
});
|
|
104
|
+
}
|
|
86
105
|
}
|
|
87
106
|
/**
|
|
88
107
|
* 连接 Redis 并自动启动消费者
|
|
@@ -110,13 +129,7 @@ export class BatchRedisQueue {
|
|
|
110
129
|
});
|
|
111
130
|
}
|
|
112
131
|
}
|
|
113
|
-
|
|
114
|
-
* 注册批量任务处理器
|
|
115
|
-
*/
|
|
116
|
-
handle(type, handler) {
|
|
117
|
-
this.handlers.set(type, handler);
|
|
118
|
-
}
|
|
119
|
-
async enqueue(type, data) {
|
|
132
|
+
async enqueue(data) {
|
|
120
133
|
if (!this.redis) {
|
|
121
134
|
console.warn('[BatchTaskQueue] Redis not available, skipping task enqueue');
|
|
122
135
|
return Array.isArray(data) ? [] : '';
|
|
@@ -146,11 +159,10 @@ export class BatchRedisQueue {
|
|
|
146
159
|
for (const item of dataList) {
|
|
147
160
|
// 检查 data 中是否包含自定义 id
|
|
148
161
|
const customId = item.id;
|
|
149
|
-
const taskId = customId
|
|
162
|
+
const taskId = customId || randomUUID();
|
|
150
163
|
const taskKey = `${this.config.queueKey}:task:${taskId}`;
|
|
151
164
|
const task = {
|
|
152
165
|
id: taskId,
|
|
153
|
-
type,
|
|
154
166
|
data: item,
|
|
155
167
|
retryCount: 0,
|
|
156
168
|
maxRetries: this.config.maxRetries,
|
|
@@ -167,7 +179,7 @@ export class BatchRedisQueue {
|
|
|
167
179
|
console.log(`[BatchTaskQueue] Task already exists, skipping: ${taskId}`);
|
|
168
180
|
}
|
|
169
181
|
else {
|
|
170
|
-
console.log(`[BatchTaskQueue] Enqueued task: ${task.
|
|
182
|
+
console.log(`[BatchTaskQueue] Enqueued task: ${task.id}`);
|
|
171
183
|
}
|
|
172
184
|
taskIds.push(taskId);
|
|
173
185
|
}
|
|
@@ -312,7 +324,7 @@ export class BatchRedisQueue {
|
|
|
312
324
|
* 批量处理任务
|
|
313
325
|
*/
|
|
314
326
|
async processBatch(taskIds) {
|
|
315
|
-
this.
|
|
327
|
+
this.processingTasks++;
|
|
316
328
|
try {
|
|
317
329
|
// 获取所有任务
|
|
318
330
|
const tasks = await Promise.all(taskIds.map(id => this.getTask(id)));
|
|
@@ -321,8 +333,8 @@ export class BatchRedisQueue {
|
|
|
321
333
|
console.warn(`[BatchTaskQueue] No valid tasks found in batch`);
|
|
322
334
|
return;
|
|
323
335
|
}
|
|
324
|
-
//
|
|
325
|
-
const
|
|
336
|
+
// 过滤出有效的待处理任务
|
|
337
|
+
const tasksToProcess = [];
|
|
326
338
|
for (const task of validTasks) {
|
|
327
339
|
// 检查状态
|
|
328
340
|
if (task.status !== 'pending') {
|
|
@@ -330,81 +342,72 @@ export class BatchRedisQueue {
|
|
|
330
342
|
await this.applyStatus(task.id, task.status, 'failed');
|
|
331
343
|
continue;
|
|
332
344
|
}
|
|
333
|
-
|
|
334
|
-
typeTasks.push(task);
|
|
335
|
-
tasksByType.set(task.type, typeTasks);
|
|
345
|
+
tasksToProcess.push(task);
|
|
336
346
|
}
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
347
|
+
if (tasksToProcess.length === 0) {
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
try {
|
|
351
|
+
// 任务已在 processing 队列中(由 Lua 脚本完成),只需更新状态和开始时间
|
|
352
|
+
const taskIdList = tasksToProcess.map(t => t.id);
|
|
353
|
+
const now = Date.now();
|
|
354
|
+
await Promise.all(tasksToProcess.map(task => {
|
|
355
|
+
task.status = 'processing';
|
|
356
|
+
task.processingStartTime = now;
|
|
357
|
+
const taskKey = `${this.config.queueKey}:task:${task.id}`;
|
|
358
|
+
return this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
|
|
359
|
+
}));
|
|
360
|
+
// 执行批量任务处理器
|
|
361
|
+
const dataList = tasksToProcess.map(t => t.data);
|
|
362
|
+
await this.handler(dataList);
|
|
363
|
+
// 更新状态为完成
|
|
364
|
+
await this.applyStatusBatch(taskIdList, 'processing', 'completed');
|
|
365
|
+
console.log(`[BatchTaskQueue] Batch completed: ${tasksToProcess.length} tasks`);
|
|
366
|
+
}
|
|
367
|
+
catch (error) {
|
|
368
|
+
console.error(`[BatchTaskQueue] Batch failed`, error);
|
|
369
|
+
// 检查每个任务是否需要重试
|
|
370
|
+
for (const task of tasksToProcess) {
|
|
371
|
+
if (task.retryCount < task.maxRetries) {
|
|
372
|
+
task.retryCount++;
|
|
373
|
+
task.status = 'pending';
|
|
374
|
+
task.processingStartTime = undefined;
|
|
353
375
|
const taskKey = `${this.config.queueKey}:task:${task.id}`;
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
console.log(`[BatchTaskQueue] Task ${task.id} will retry (${task.retryCount}/${task.maxRetries})`);
|
|
384
|
-
}
|
|
385
|
-
else {
|
|
386
|
-
task.status = 'failed';
|
|
387
|
-
task.processingStartTime = undefined;
|
|
388
|
-
const taskKey = `${this.config.queueKey}:task:${task.id}`;
|
|
389
|
-
// 使用 Lua 脚本确保原子性
|
|
390
|
-
const script = `
|
|
391
|
-
redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
|
|
392
|
-
redis.call('LREM', KEYS[2], 0, ARGV[3])
|
|
393
|
-
redis.call('RPUSH', KEYS[3], ARGV[3])
|
|
394
|
-
return 1
|
|
395
|
-
`;
|
|
396
|
-
await this.redis.eval(script, {
|
|
397
|
-
keys: [taskKey, this.processingQueue, this.failedQueue],
|
|
398
|
-
arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
|
|
399
|
-
});
|
|
400
|
-
console.error(`[BatchTaskQueue] Task ${task.id} failed after ${task.maxRetries} retries`);
|
|
401
|
-
}
|
|
376
|
+
// 使用 Lua 脚本确保原子性
|
|
377
|
+
const script = `
|
|
378
|
+
redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
|
|
379
|
+
redis.call('LREM', KEYS[2], 0, ARGV[3])
|
|
380
|
+
redis.call('RPUSH', KEYS[3], ARGV[3])
|
|
381
|
+
return 1
|
|
382
|
+
`;
|
|
383
|
+
await this.redis.eval(script, {
|
|
384
|
+
keys: [taskKey, this.processingQueue, this.pendingQueue],
|
|
385
|
+
arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
|
|
386
|
+
});
|
|
387
|
+
console.log(`[BatchTaskQueue] Task ${task.id} will retry (${task.retryCount}/${task.maxRetries})`);
|
|
388
|
+
}
|
|
389
|
+
else {
|
|
390
|
+
task.status = 'failed';
|
|
391
|
+
task.processingStartTime = undefined;
|
|
392
|
+
const taskKey = `${this.config.queueKey}:task:${task.id}`;
|
|
393
|
+
// 使用 Lua 脚本确保原子性
|
|
394
|
+
const script = `
|
|
395
|
+
redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
|
|
396
|
+
redis.call('LREM', KEYS[2], 0, ARGV[3])
|
|
397
|
+
redis.call('RPUSH', KEYS[3], ARGV[3])
|
|
398
|
+
return 1
|
|
399
|
+
`;
|
|
400
|
+
await this.redis.eval(script, {
|
|
401
|
+
keys: [taskKey, this.processingQueue, this.failedQueue],
|
|
402
|
+
arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
|
|
403
|
+
});
|
|
404
|
+
console.error(`[BatchTaskQueue] Task ${task.id} failed after ${task.maxRetries} retries`);
|
|
402
405
|
}
|
|
403
406
|
}
|
|
404
407
|
}
|
|
405
408
|
}
|
|
406
409
|
finally {
|
|
407
|
-
this.
|
|
410
|
+
this.processingTasks--;
|
|
408
411
|
}
|
|
409
412
|
}
|
|
410
413
|
/**
|
|
@@ -489,7 +492,7 @@ export class BatchRedisQueue {
|
|
|
489
492
|
this.consumerInterval = setInterval(async () => {
|
|
490
493
|
try {
|
|
491
494
|
// 检查是否有正在处理的批次(批量队列通常不需要并发控制,一次处理一批)
|
|
492
|
-
if (this.
|
|
495
|
+
if (this.processingTasks > 0) {
|
|
493
496
|
return;
|
|
494
497
|
}
|
|
495
498
|
// 使用 Lua 脚本原子化取出任务并移到 processing 队列
|