@taicode/common-server 1.0.13 → 1.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/output/redis-queue/index.d.ts +1 -1
- package/output/redis-queue/index.d.ts.map +1 -1
- package/output/redis-queue/index.js +1 -1
- package/output/redis-queue/redis-batch-consumer.d.ts +9 -0
- package/output/redis-queue/redis-batch-consumer.d.ts.map +1 -1
- package/output/redis-queue/redis-batch-consumer.js +63 -0
- package/output/redis-queue/redis-queue-batch-consumer.d.ts +77 -0
- package/output/redis-queue/redis-queue-batch-consumer.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-batch-consumer.js +320 -0
- package/output/redis-queue/redis-queue-batch-consumer.test.d.ts +26 -0
- package/output/redis-queue/redis-queue-batch-consumer.test.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-batch-consumer.test.js +341 -0
- package/output/redis-queue/redis-queue-common.d.ts +1 -1
- package/output/redis-queue/redis-queue-common.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-common.js +8 -8
- package/output/redis-queue/redis-queue-common.test.js +16 -54
- package/output/redis-queue/redis-queue-consumer.d.ts +5 -9
- package/output/redis-queue/redis-queue-consumer.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-consumer.js +80 -69
- package/output/redis-queue/redis-queue-consumer.test.d.ts +20 -1
- package/output/redis-queue/redis-queue-consumer.test.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-consumer.test.js +89 -15
- package/output/redis-queue/redis-queue-provider.d.ts +4 -4
- package/output/redis-queue/redis-queue-provider.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-provider.js +10 -6
- package/output/redis-queue/redis-queue-provider.test.d.ts +23 -2
- package/output/redis-queue/redis-queue-provider.test.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-provider.test.js +73 -38
- package/output/redis-queue/test-helpers.d.ts +112 -0
- package/output/redis-queue/test-helpers.d.ts.map +1 -0
- package/output/redis-queue/test-helpers.js +242 -0
- package/output/redis-queue/test-helpers.test.d.ts +28 -0
- package/output/redis-queue/test-helpers.test.d.ts.map +1 -0
- package/output/redis-queue/test-helpers.test.js +572 -0
- package/output/redis-queue/types.d.ts +0 -7
- package/output/redis-queue/types.d.ts.map +1 -1
- package/package.json +5 -3
|
@@ -0,0 +1,572 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { catchIt } from '@taicode/common-base';
|
|
3
|
+
import { RedisQueueProvider } from './redis-queue-provider';
|
|
4
|
+
import { RedisQueueConsumer } from './redis-queue-consumer';
|
|
5
|
+
import { RedisQueueBatchConsumer } from './redis-queue-batch-consumer';
|
|
6
|
+
import { dispatchQueueTask, waitQueueCompletion, clearQueue, getQueueTasks, } from './test-helpers';
|
|
7
|
+
const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
|
|
8
|
+
/**
|
|
9
|
+
* 测试辅助方法使用示例
|
|
10
|
+
*
|
|
11
|
+
* 展示如何使用所有测试辅助函数进行快速、可控的队列测试
|
|
12
|
+
*/
|
|
13
|
+
describe('Redis Queue Test Helpers', () => {
|
|
14
|
+
// 每次运行使用唯一的队列名,避免测试间干扰
|
|
15
|
+
let testQueueKey;
|
|
16
|
+
beforeEach(() => {
|
|
17
|
+
testQueueKey = `test:helpers:${Date.now()}:${Math.random().toString(36).slice(2, 9)}`;
|
|
18
|
+
});
|
|
19
|
+
describe('RedisQueueConsumer.dispatch', () => {
|
|
20
|
+
let provider;
|
|
21
|
+
let consumer;
|
|
22
|
+
let queueKey;
|
|
23
|
+
const processedTasks = [];
|
|
24
|
+
beforeEach(async () => {
|
|
25
|
+
processedTasks.length = 0;
|
|
26
|
+
// 每个测试套件使用独立的队列名
|
|
27
|
+
queueKey = `test:consumer:${Date.now()}:${Math.random().toString(36).slice(2, 9)}`;
|
|
28
|
+
provider = new RedisQueueProvider(queueKey, { redisUrl: REDIS_URL });
|
|
29
|
+
consumer = new RedisQueueConsumer(queueKey, { redisUrl: REDIS_URL, handler: async (data) => {
|
|
30
|
+
processedTasks.push(data);
|
|
31
|
+
return catchIt(() => { });
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
await provider.connect();
|
|
35
|
+
await consumer.connect();
|
|
36
|
+
});
|
|
37
|
+
afterEach(async () => {
|
|
38
|
+
await clearQueue(consumer);
|
|
39
|
+
provider.disconnect();
|
|
40
|
+
await consumer.disconnect();
|
|
41
|
+
});
|
|
42
|
+
it('应该立即派发单个任务', async () => {
|
|
43
|
+
// 添加任务
|
|
44
|
+
const taskId = await provider.enqueue({ message: 'test task' });
|
|
45
|
+
// 立即派发任务(不等待轮询)
|
|
46
|
+
await dispatchQueueTask(consumer, [taskId]);
|
|
47
|
+
// 验证任务已处理
|
|
48
|
+
expect(processedTasks).toHaveLength(1);
|
|
49
|
+
expect(processedTasks[0]).toEqual({ message: 'test task' });
|
|
50
|
+
// 验证队列状态
|
|
51
|
+
const stats = await consumer.statistics();
|
|
52
|
+
expect(stats.completed).toBe(1);
|
|
53
|
+
expect(stats.pending).toBe(0);
|
|
54
|
+
});
|
|
55
|
+
it('应该立即派发多个任务', async () => {
|
|
56
|
+
// 添加多个任务
|
|
57
|
+
const taskIds = (await provider.enqueue([
|
|
58
|
+
{ id: '1' },
|
|
59
|
+
{ id: '2' },
|
|
60
|
+
{ id: '3' },
|
|
61
|
+
]));
|
|
62
|
+
// 立即派发所有任务
|
|
63
|
+
await dispatchQueueTask(consumer, taskIds);
|
|
64
|
+
// 验证所有任务已处理
|
|
65
|
+
expect(processedTasks).toHaveLength(3);
|
|
66
|
+
expect(processedTasks.map(t => t.id).sort()).toEqual(['1', '2', '3']);
|
|
67
|
+
// 验证队列状态
|
|
68
|
+
const stats = await consumer.statistics();
|
|
69
|
+
expect(stats.completed).toBe(3);
|
|
70
|
+
});
|
|
71
|
+
it('应该处理任务失败和重试', async () => {
|
|
72
|
+
let attemptCount = 0;
|
|
73
|
+
const failingConsumer = new RedisQueueConsumer(queueKey, {
|
|
74
|
+
redisUrl: REDIS_URL,
|
|
75
|
+
maxRetries: 2,
|
|
76
|
+
handler: async (data) => {
|
|
77
|
+
attemptCount++;
|
|
78
|
+
if (attemptCount < 3) {
|
|
79
|
+
throw new Error('Simulated failure');
|
|
80
|
+
}
|
|
81
|
+
processedTasks.push(data);
|
|
82
|
+
return catchIt(() => { });
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
await failingConsumer.connect();
|
|
86
|
+
const taskId = await provider.enqueue({ test: 'retry' });
|
|
87
|
+
// 立即派发任务
|
|
88
|
+
await dispatchQueueTask(failingConsumer, [taskId]);
|
|
89
|
+
// 验证重试逻辑
|
|
90
|
+
expect(attemptCount).toBe(3); // 1次初始 + 2次重试
|
|
91
|
+
expect(processedTasks).toHaveLength(1);
|
|
92
|
+
await failingConsumer.disconnect();
|
|
93
|
+
});
|
|
94
|
+
it('应该忽略不存在的任务ID', async () => {
|
|
95
|
+
const realTaskId = await provider.enqueue({ real: true });
|
|
96
|
+
const fakeTaskId = 'non-existent-task-id';
|
|
97
|
+
// 混合真实和虚假任务ID
|
|
98
|
+
await dispatchQueueTask(consumer, [realTaskId, fakeTaskId]);
|
|
99
|
+
// 只有真实任务被处理
|
|
100
|
+
expect(processedTasks).toHaveLength(1);
|
|
101
|
+
expect(processedTasks[0]).toEqual({ real: true });
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
describe('RedisQueueBatchConsumer.dispatch', () => {
|
|
105
|
+
let provider;
|
|
106
|
+
let consumer;
|
|
107
|
+
let queueKey;
|
|
108
|
+
const processedBatches = [];
|
|
109
|
+
beforeEach(async () => {
|
|
110
|
+
processedBatches.length = 0;
|
|
111
|
+
// 每个测试套件使用独立的队列名
|
|
112
|
+
queueKey = `test:batch:${Date.now()}:${Math.random().toString(36).slice(2, 9)}`;
|
|
113
|
+
provider = new RedisQueueProvider(queueKey, { redisUrl: REDIS_URL });
|
|
114
|
+
consumer = new RedisQueueBatchConsumer(queueKey, { redisUrl: REDIS_URL, batchSize: 3,
|
|
115
|
+
handler: async (dataList) => {
|
|
116
|
+
processedBatches.push([...dataList]);
|
|
117
|
+
return catchIt(() => { });
|
|
118
|
+
},
|
|
119
|
+
});
|
|
120
|
+
await provider.connect();
|
|
121
|
+
await consumer.connect();
|
|
122
|
+
});
|
|
123
|
+
afterEach(async () => {
|
|
124
|
+
await clearQueue(consumer);
|
|
125
|
+
provider.disconnect();
|
|
126
|
+
await consumer.disconnect();
|
|
127
|
+
});
|
|
128
|
+
it('应该按批次大小立即派发任务', async () => {
|
|
129
|
+
// 添加 5 个任务(batchSize = 3)
|
|
130
|
+
const taskIds = (await provider.enqueue([
|
|
131
|
+
{ id: '1' },
|
|
132
|
+
{ id: '2' },
|
|
133
|
+
{ id: '3' },
|
|
134
|
+
{ id: '4' },
|
|
135
|
+
{ id: '5' },
|
|
136
|
+
]));
|
|
137
|
+
// 立即派发所有任务
|
|
138
|
+
await dispatchQueueTask(consumer, taskIds);
|
|
139
|
+
// 验证批次处理:应该有2个批次(3+2)
|
|
140
|
+
expect(processedBatches).toHaveLength(2);
|
|
141
|
+
expect(processedBatches[0]).toHaveLength(3);
|
|
142
|
+
expect(processedBatches[1]).toHaveLength(2);
|
|
143
|
+
// 验证所有任务都被处理
|
|
144
|
+
const allProcessed = processedBatches.flat();
|
|
145
|
+
expect(allProcessed).toHaveLength(5);
|
|
146
|
+
// 验证队列状态
|
|
147
|
+
const stats = await consumer.statistics();
|
|
148
|
+
expect(stats.completed).toBe(5);
|
|
149
|
+
});
|
|
150
|
+
it('应该处理单个批次', async () => {
|
|
151
|
+
const taskIds = (await provider.enqueue([
|
|
152
|
+
{ value: 'a' },
|
|
153
|
+
{ value: 'b' },
|
|
154
|
+
]));
|
|
155
|
+
await dispatchQueueTask(consumer, taskIds);
|
|
156
|
+
// 只有一个批次
|
|
157
|
+
expect(processedBatches).toHaveLength(1);
|
|
158
|
+
expect(processedBatches[0]).toHaveLength(2);
|
|
159
|
+
});
|
|
160
|
+
it('应该支持大批量任务测试', async () => {
|
|
161
|
+
// 添加 100 个任务
|
|
162
|
+
const data = Array.from({ length: 100 }, (_, i) => ({ index: i }));
|
|
163
|
+
const taskIds = (await provider.enqueue(data));
|
|
164
|
+
// 立即派发
|
|
165
|
+
await dispatchQueueTask(consumer, taskIds);
|
|
166
|
+
// 验证批次数量(100 / 3 = 34 批次)
|
|
167
|
+
const totalProcessed = processedBatches.flat().length;
|
|
168
|
+
expect(totalProcessed).toBe(100);
|
|
169
|
+
// 验证队列清空
|
|
170
|
+
const stats = await consumer.statistics();
|
|
171
|
+
expect(stats.pending).toBe(0);
|
|
172
|
+
expect(stats.completed).toBe(100);
|
|
173
|
+
});
|
|
174
|
+
it('应该处理批量任务失败', async () => {
|
|
175
|
+
let attemptCount = 0;
|
|
176
|
+
const failingConsumer = new RedisQueueBatchConsumer(queueKey, { redisUrl: REDIS_URL, batchSize: 2,
|
|
177
|
+
maxRetries: 0, // 不重试
|
|
178
|
+
handler: async (dataList) => {
|
|
179
|
+
attemptCount++;
|
|
180
|
+
throw new Error('Batch processing failed');
|
|
181
|
+
},
|
|
182
|
+
});
|
|
183
|
+
await failingConsumer.connect();
|
|
184
|
+
const taskIds = (await provider.enqueue([
|
|
185
|
+
{ id: '1' },
|
|
186
|
+
{ id: '2' },
|
|
187
|
+
]));
|
|
188
|
+
await dispatchQueueTask(failingConsumer, taskIds);
|
|
189
|
+
// 等待一下,确保任务处理完成
|
|
190
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
191
|
+
// 验证失败处理
|
|
192
|
+
const stats = await failingConsumer.statistics();
|
|
193
|
+
// 任务会被重试,因为 Provider 设置了 maxRetries: 3
|
|
194
|
+
// 所以最终会进入 failed 队列
|
|
195
|
+
expect(stats.failed).toBeGreaterThanOrEqual(0);
|
|
196
|
+
expect(stats.completed).toBe(0);
|
|
197
|
+
await clearQueue(failingConsumer);
|
|
198
|
+
await failingConsumer.disconnect();
|
|
199
|
+
});
|
|
200
|
+
});
|
|
201
|
+
describe('性能测试示例', () => {
|
|
202
|
+
it('应该快速完成大量任务测试', async () => {
|
|
203
|
+
const queueKey = `test:perf1:${Date.now()}:${Math.random().toString(36).slice(2, 9)}`;
|
|
204
|
+
const provider = new RedisQueueProvider(queueKey, { redisUrl: REDIS_URL });
|
|
205
|
+
let processedCount = 0;
|
|
206
|
+
const consumer = new RedisQueueConsumer(queueKey, { redisUrl: REDIS_URL, handler: async () => {
|
|
207
|
+
processedCount++;
|
|
208
|
+
return catchIt(() => { });
|
|
209
|
+
},
|
|
210
|
+
});
|
|
211
|
+
await provider.connect();
|
|
212
|
+
await consumer.connect();
|
|
213
|
+
// 添加 1000 个任务
|
|
214
|
+
const startAdd = Date.now();
|
|
215
|
+
const data = Array.from({ length: 1000 }, (_, i) => ({ index: i }));
|
|
216
|
+
const taskIds = (await provider.enqueue(data));
|
|
217
|
+
const addTime = Date.now() - startAdd;
|
|
218
|
+
// 立即派发所有任务
|
|
219
|
+
const startDispatch = Date.now();
|
|
220
|
+
await dispatchQueueTask(consumer, taskIds);
|
|
221
|
+
const dispatchTime = Date.now() - startDispatch;
|
|
222
|
+
// 验证处理完成
|
|
223
|
+
expect(processedCount).toBe(1000);
|
|
224
|
+
console.log(`Performance test: Add ${addTime}ms, Dispatch ${dispatchTime}ms`);
|
|
225
|
+
provider.disconnect();
|
|
226
|
+
await consumer.disconnect();
|
|
227
|
+
}, 30000); // 30秒超时
|
|
228
|
+
});
|
|
229
|
+
describe('waitQueueCompletion', () => {
|
|
230
|
+
let provider;
|
|
231
|
+
let consumer;
|
|
232
|
+
beforeEach(async () => {
|
|
233
|
+
const queueKey = `test:wait:${Date.now()}:${Math.random().toString(36).slice(2, 9)}`;
|
|
234
|
+
provider = new RedisQueueProvider(queueKey, { redisUrl: REDIS_URL });
|
|
235
|
+
consumer = new RedisQueueConsumer(queueKey, { redisUrl: REDIS_URL, handler: async (data) => catchIt(() => { }),
|
|
236
|
+
});
|
|
237
|
+
await provider.connect();
|
|
238
|
+
await consumer.connect();
|
|
239
|
+
});
|
|
240
|
+
afterEach(async () => {
|
|
241
|
+
await clearQueue(consumer);
|
|
242
|
+
provider.disconnect();
|
|
243
|
+
await consumer.disconnect();
|
|
244
|
+
});
|
|
245
|
+
it('应该等待队列完成所有任务', async () => {
|
|
246
|
+
// 添加任务
|
|
247
|
+
const taskIds = (await provider.enqueue([
|
|
248
|
+
{ id: '1' },
|
|
249
|
+
{ id: '2' },
|
|
250
|
+
{ id: '3' },
|
|
251
|
+
]));
|
|
252
|
+
// consumer.connect() 已经启动了轮询,等待所有任务完成
|
|
253
|
+
await waitQueueCompletion(consumer, stats => stats.pending === 0 && stats.processing === 0 && stats.completed === 3);
|
|
254
|
+
const stats = await consumer.statistics();
|
|
255
|
+
expect(stats.completed).toBe(3);
|
|
256
|
+
expect(stats.pending).toBe(0);
|
|
257
|
+
});
|
|
258
|
+
it('应该支持自定义超时时间', async () => {
|
|
259
|
+
// 添加一个任务但不启动消费
|
|
260
|
+
await provider.enqueue({ id: '1' });
|
|
261
|
+
// 使用短超时,预期会超时失败
|
|
262
|
+
await expect(waitQueueCompletion(consumer, stats => stats.completed === 1, { timeout: 100, interval: 10 })).rejects.toThrow(/timeout/);
|
|
263
|
+
});
|
|
264
|
+
it('应该支持自定义检查间隔', async () => {
|
|
265
|
+
const taskIds = (await provider.enqueue([
|
|
266
|
+
{ id: '1' },
|
|
267
|
+
{ id: '2' },
|
|
268
|
+
]));
|
|
269
|
+
const startTime = Date.now();
|
|
270
|
+
// 使用较长的检查间隔
|
|
271
|
+
await waitQueueCompletion(consumer, stats => stats.completed === 2, { interval: 500 });
|
|
272
|
+
// 验证使用了指定的间隔(至少等待了一些时间)
|
|
273
|
+
const elapsed = Date.now() - startTime;
|
|
274
|
+
expect(elapsed).toBeGreaterThan(0);
|
|
275
|
+
});
|
|
276
|
+
it('应该在条件立即满足时快速返回', async () => {
|
|
277
|
+
const startTime = Date.now();
|
|
278
|
+
// 条件已经满足(没有任务)
|
|
279
|
+
await waitQueueCompletion(consumer, stats => stats.pending === 0);
|
|
280
|
+
const elapsed = Date.now() - startTime;
|
|
281
|
+
expect(elapsed).toBeLessThan(200); // 应该很快返回
|
|
282
|
+
});
|
|
283
|
+
});
|
|
284
|
+
describe('clearQueue', () => {
|
|
285
|
+
let provider;
|
|
286
|
+
let consumer;
|
|
287
|
+
beforeEach(async () => {
|
|
288
|
+
const queueKey = `test:clear:${Date.now()}:${Math.random().toString(36).slice(2, 9)}`;
|
|
289
|
+
provider = new RedisQueueProvider(queueKey, { redisUrl: REDIS_URL });
|
|
290
|
+
consumer = new RedisQueueConsumer(queueKey, { redisUrl: REDIS_URL, handler: async (data) => catchIt(() => { }),
|
|
291
|
+
});
|
|
292
|
+
await provider.connect();
|
|
293
|
+
await consumer.connect();
|
|
294
|
+
});
|
|
295
|
+
afterEach(async () => {
|
|
296
|
+
provider.disconnect();
|
|
297
|
+
await consumer.disconnect();
|
|
298
|
+
});
|
|
299
|
+
it('应该清空队列中的所有任务', async () => {
|
|
300
|
+
// 添加多个任务
|
|
301
|
+
await provider.enqueue([
|
|
302
|
+
{ id: '1' },
|
|
303
|
+
{ id: '2' },
|
|
304
|
+
{ id: '3' },
|
|
305
|
+
]);
|
|
306
|
+
let stats = await consumer.statistics();
|
|
307
|
+
expect(stats.pending).toBe(3);
|
|
308
|
+
// 清空队列
|
|
309
|
+
await clearQueue(consumer);
|
|
310
|
+
stats = await consumer.statistics();
|
|
311
|
+
expect(stats.pending).toBe(0);
|
|
312
|
+
expect(stats.completed).toBe(0);
|
|
313
|
+
expect(stats.failed).toBe(0);
|
|
314
|
+
});
|
|
315
|
+
it('应该清空已完成的任务', async () => {
|
|
316
|
+
const taskIds = (await provider.enqueue([
|
|
317
|
+
{ id: '1' },
|
|
318
|
+
{ id: '2' },
|
|
319
|
+
]));
|
|
320
|
+
// 处理任务
|
|
321
|
+
await dispatchQueueTask(consumer, taskIds);
|
|
322
|
+
let stats = await consumer.statistics();
|
|
323
|
+
expect(stats.completed).toBe(2);
|
|
324
|
+
// 清空队列
|
|
325
|
+
await clearQueue(consumer);
|
|
326
|
+
stats = await consumer.statistics();
|
|
327
|
+
expect(stats.completed).toBe(0);
|
|
328
|
+
});
|
|
329
|
+
it('应该处理空队列的情况', async () => {
|
|
330
|
+
// 队列本身就是空的
|
|
331
|
+
await expect(clearQueue(consumer)).resolves.not.toThrow();
|
|
332
|
+
const stats = await consumer.statistics();
|
|
333
|
+
expect(stats.pending).toBe(0);
|
|
334
|
+
});
|
|
335
|
+
it('应该处理断开连接的情况', async () => {
|
|
336
|
+
await consumer.disconnect();
|
|
337
|
+
// 断开连接后清空应该不报错
|
|
338
|
+
await expect(clearQueue(consumer)).resolves.not.toThrow();
|
|
339
|
+
});
|
|
340
|
+
});
|
|
341
|
+
describe('getQueueTasks', () => {
|
|
342
|
+
let provider;
|
|
343
|
+
let consumer;
|
|
344
|
+
beforeEach(async () => {
|
|
345
|
+
const queueKey = `test:get:${Date.now()}:${Math.random().toString(36).slice(2, 9)}`;
|
|
346
|
+
provider = new RedisQueueProvider(queueKey, { redisUrl: REDIS_URL });
|
|
347
|
+
consumer = new RedisQueueConsumer(queueKey, { redisUrl: REDIS_URL, maxRetries: 0,
|
|
348
|
+
handler: async (data) => {
|
|
349
|
+
if (data.shouldFail) {
|
|
350
|
+
throw new Error('Task failed');
|
|
351
|
+
}
|
|
352
|
+
return catchIt(() => { });
|
|
353
|
+
},
|
|
354
|
+
});
|
|
355
|
+
await provider.connect();
|
|
356
|
+
await consumer.connect();
|
|
357
|
+
});
|
|
358
|
+
afterEach(async () => {
|
|
359
|
+
await clearQueue(consumer);
|
|
360
|
+
provider.disconnect();
|
|
361
|
+
await consumer.disconnect();
|
|
362
|
+
});
|
|
363
|
+
it('应该获取所有待处理的任务', async () => {
|
|
364
|
+
await provider.enqueue([
|
|
365
|
+
{ id: '1', name: 'task1' },
|
|
366
|
+
{ id: '2', name: 'task2' },
|
|
367
|
+
{ id: '3', name: 'task3' },
|
|
368
|
+
]);
|
|
369
|
+
const tasks = await getQueueTasks(provider, 'pending');
|
|
370
|
+
expect(tasks).toHaveLength(3);
|
|
371
|
+
expect(tasks.map(t => t.data.id).sort()).toEqual(['1', '2', '3']);
|
|
372
|
+
});
|
|
373
|
+
it('应该获取已完成的任务', async () => {
|
|
374
|
+
const taskIds = (await provider.enqueue([
|
|
375
|
+
{ id: '1' },
|
|
376
|
+
{ id: '2' },
|
|
377
|
+
]));
|
|
378
|
+
await dispatchQueueTask(consumer, taskIds);
|
|
379
|
+
const tasks = await getQueueTasks(provider, 'completed');
|
|
380
|
+
expect(tasks).toHaveLength(2);
|
|
381
|
+
expect(tasks.every(t => t.status === 'completed')).toBe(true);
|
|
382
|
+
});
|
|
383
|
+
it('应该获取失败的任务', async () => {
|
|
384
|
+
const taskIds = (await provider.enqueue([
|
|
385
|
+
{ id: '1', shouldFail: true },
|
|
386
|
+
{ id: '2', shouldFail: true },
|
|
387
|
+
]));
|
|
388
|
+
await dispatchQueueTask(consumer, taskIds);
|
|
389
|
+
const tasks = await getQueueTasks(provider, 'failed');
|
|
390
|
+
expect(tasks).toHaveLength(2);
|
|
391
|
+
expect(tasks.every(t => t.status === 'failed')).toBe(true);
|
|
392
|
+
});
|
|
393
|
+
it('应该获取所有任务(不指定状态)', async () => {
|
|
394
|
+
// 添加待处理任务
|
|
395
|
+
await provider.enqueue([{ id: '1' }, { id: '2' }]);
|
|
396
|
+
// 添加并完成一些任务
|
|
397
|
+
const taskIds = (await provider.enqueue([
|
|
398
|
+
{ id: '3' },
|
|
399
|
+
{ id: '4' },
|
|
400
|
+
]));
|
|
401
|
+
await dispatchQueueTask(consumer, taskIds);
|
|
402
|
+
// 获取所有任务
|
|
403
|
+
const allTasks = await getQueueTasks(provider);
|
|
404
|
+
expect(allTasks.length).toBeGreaterThanOrEqual(4);
|
|
405
|
+
});
|
|
406
|
+
it('应该验证任务数据完整性', async () => {
|
|
407
|
+
const testData = { id: '999', message: 'test', nested: { value: 42 } };
|
|
408
|
+
await provider.enqueue([testData]);
|
|
409
|
+
const tasks = await getQueueTasks(provider, 'pending');
|
|
410
|
+
expect(tasks).toHaveLength(1);
|
|
411
|
+
expect(tasks[0].data).toEqual(testData);
|
|
412
|
+
expect(tasks[0].id).toBeTruthy();
|
|
413
|
+
expect(tasks[0].retryCount).toBe(0);
|
|
414
|
+
});
|
|
415
|
+
it('应该处理空队列', async () => {
|
|
416
|
+
const tasks = await getQueueTasks(provider, 'pending');
|
|
417
|
+
expect(tasks).toEqual([]);
|
|
418
|
+
});
|
|
419
|
+
it('应该处理断开连接的情况', async () => {
|
|
420
|
+
provider.disconnect();
|
|
421
|
+
const tasks = await getQueueTasks(provider, 'pending');
|
|
422
|
+
expect(tasks).toEqual([]);
|
|
423
|
+
});
|
|
424
|
+
});
|
|
425
|
+
describe('dispatchQueueTask', () => {
|
|
426
|
+
let provider;
|
|
427
|
+
let consumer;
|
|
428
|
+
let queueKey;
|
|
429
|
+
const processedTasks = [];
|
|
430
|
+
beforeEach(async () => {
|
|
431
|
+
processedTasks.length = 0;
|
|
432
|
+
queueKey = `test:dispatch:${Date.now()}:${Math.random().toString(36).slice(2, 9)}`;
|
|
433
|
+
provider = new RedisQueueProvider(queueKey, { redisUrl: REDIS_URL });
|
|
434
|
+
consumer = new RedisQueueConsumer(queueKey, { redisUrl: REDIS_URL, handler: async (data) => {
|
|
435
|
+
processedTasks.push(data);
|
|
436
|
+
return catchIt(() => { });
|
|
437
|
+
},
|
|
438
|
+
});
|
|
439
|
+
await provider.connect();
|
|
440
|
+
await consumer.connect();
|
|
441
|
+
});
|
|
442
|
+
afterEach(async () => {
|
|
443
|
+
await clearQueue(consumer);
|
|
444
|
+
provider.disconnect();
|
|
445
|
+
await consumer.disconnect();
|
|
446
|
+
});
|
|
447
|
+
it('应该立即派发单个任务', async () => {
|
|
448
|
+
const taskId = await provider.enqueue({ message: 'test task' });
|
|
449
|
+
await dispatchQueueTask(consumer, [taskId]);
|
|
450
|
+
expect(processedTasks).toHaveLength(1);
|
|
451
|
+
expect(processedTasks[0]).toEqual({ message: 'test task' });
|
|
452
|
+
const stats = await consumer.statistics();
|
|
453
|
+
expect(stats.completed).toBe(1);
|
|
454
|
+
expect(stats.pending).toBe(0);
|
|
455
|
+
});
|
|
456
|
+
it('应该立即派发多个任务', async () => {
|
|
457
|
+
const taskIds = (await provider.enqueue([
|
|
458
|
+
{ id: '1' },
|
|
459
|
+
{ id: '2' },
|
|
460
|
+
{ id: '3' },
|
|
461
|
+
]));
|
|
462
|
+
await dispatchQueueTask(consumer, taskIds);
|
|
463
|
+
expect(processedTasks).toHaveLength(3);
|
|
464
|
+
expect(processedTasks.map(t => t.id).sort()).toEqual(['1', '2', '3']);
|
|
465
|
+
const stats = await consumer.statistics();
|
|
466
|
+
expect(stats.completed).toBe(3);
|
|
467
|
+
});
|
|
468
|
+
it('应该处理任务失败和重试', async () => {
|
|
469
|
+
let attemptCount = 0;
|
|
470
|
+
const failingConsumer = new RedisQueueConsumer(queueKey, { redisUrl: REDIS_URL, maxRetries: 2,
|
|
471
|
+
handler: async (data) => {
|
|
472
|
+
attemptCount++;
|
|
473
|
+
// 前2次失败,第3次成功
|
|
474
|
+
if (attemptCount < 3) {
|
|
475
|
+
throw new Error('Simulated failure');
|
|
476
|
+
}
|
|
477
|
+
processedTasks.push(data);
|
|
478
|
+
return catchIt(() => { });
|
|
479
|
+
},
|
|
480
|
+
});
|
|
481
|
+
await failingConsumer.connect();
|
|
482
|
+
const taskId = await provider.enqueue({ test: 'retry' });
|
|
483
|
+
await dispatchQueueTask(failingConsumer, [taskId]);
|
|
484
|
+
// maxRetries=2 意味着: 1次初始尝试 + 最多2次重试 = 最多3次尝试
|
|
485
|
+
// 第3次成功,所以 attemptCount 应该是 3
|
|
486
|
+
expect(attemptCount).toBe(3);
|
|
487
|
+
expect(processedTasks).toHaveLength(1);
|
|
488
|
+
await clearQueue(failingConsumer);
|
|
489
|
+
await failingConsumer.disconnect();
|
|
490
|
+
});
|
|
491
|
+
it('应该忽略不存在的任务ID', async () => {
|
|
492
|
+
const realTaskId = await provider.enqueue({ real: true });
|
|
493
|
+
const fakeTaskId = 'non-existent-task-id';
|
|
494
|
+
await dispatchQueueTask(consumer, [realTaskId, fakeTaskId]);
|
|
495
|
+
expect(processedTasks).toHaveLength(1);
|
|
496
|
+
expect(processedTasks[0]).toEqual({ real: true });
|
|
497
|
+
});
|
|
498
|
+
it('应该处理空任务列表', async () => {
|
|
499
|
+
await expect(dispatchQueueTask(consumer, [])).resolves.not.toThrow();
|
|
500
|
+
expect(processedTasks).toHaveLength(0);
|
|
501
|
+
});
|
|
502
|
+
});
|
|
503
|
+
describe('Integration: 组合使用测试工具', () => {
|
|
504
|
+
it('应该支持完整的测试工作流', async () => {
|
|
505
|
+
// 1. 创建唯一队列名
|
|
506
|
+
const testQueue = `test:integration:${Date.now()}:${Math.random().toString(36).slice(2, 9)}`;
|
|
507
|
+
// 2. 初始化 provider 和 consumer
|
|
508
|
+
const provider = new RedisQueueProvider(testQueue, { redisUrl: REDIS_URL });
|
|
509
|
+
const processedData = [];
|
|
510
|
+
const consumer = new RedisQueueConsumer(testQueue, { redisUrl: REDIS_URL, handler: async (data) => {
|
|
511
|
+
processedData.push(data);
|
|
512
|
+
return catchIt(() => { });
|
|
513
|
+
},
|
|
514
|
+
});
|
|
515
|
+
await provider.connect();
|
|
516
|
+
await consumer.connect();
|
|
517
|
+
try {
|
|
518
|
+
// 3. 添加任务
|
|
519
|
+
const taskIds = (await provider.enqueue([
|
|
520
|
+
{ step: 1 },
|
|
521
|
+
{ step: 2 },
|
|
522
|
+
{ step: 3 },
|
|
523
|
+
]));
|
|
524
|
+
// 4. 验证待处理任务
|
|
525
|
+
const pendingTasks = await getQueueTasks(provider, 'pending');
|
|
526
|
+
expect(pendingTasks).toHaveLength(3);
|
|
527
|
+
// 5. 立即派发任务
|
|
528
|
+
await dispatchQueueTask(consumer, taskIds);
|
|
529
|
+
// 6. 验证完成任务
|
|
530
|
+
const completedTasks = await getQueueTasks(provider, 'completed');
|
|
531
|
+
expect(completedTasks).toHaveLength(3);
|
|
532
|
+
expect(processedData).toHaveLength(3);
|
|
533
|
+
// 7. 清理队列
|
|
534
|
+
await clearQueue(consumer);
|
|
535
|
+
// 8. 验证清理结果
|
|
536
|
+
const stats = await consumer.statistics();
|
|
537
|
+
expect(stats.pending).toBe(0);
|
|
538
|
+
expect(stats.completed).toBe(0);
|
|
539
|
+
}
|
|
540
|
+
finally {
|
|
541
|
+
provider.disconnect();
|
|
542
|
+
await consumer.disconnect();
|
|
543
|
+
}
|
|
544
|
+
});
|
|
545
|
+
});
|
|
546
|
+
describe('性能测试示例', () => {
|
|
547
|
+
it('应该快速完成大量任务测试', async () => {
|
|
548
|
+
const queueKey = `test:perf2:${Date.now()}:${Math.random().toString(36).slice(2, 9)}`;
|
|
549
|
+
const provider = new RedisQueueProvider(queueKey, { redisUrl: REDIS_URL });
|
|
550
|
+
let processedCount = 0;
|
|
551
|
+
const consumer = new RedisQueueConsumer(queueKey, { redisUrl: REDIS_URL, handler: async () => {
|
|
552
|
+
processedCount++;
|
|
553
|
+
return catchIt(() => { });
|
|
554
|
+
},
|
|
555
|
+
});
|
|
556
|
+
await provider.connect();
|
|
557
|
+
await consumer.connect();
|
|
558
|
+
const startAdd = Date.now();
|
|
559
|
+
const data = Array.from({ length: 1000 }, (_, i) => ({ index: i }));
|
|
560
|
+
const taskIds = (await provider.enqueue(data));
|
|
561
|
+
const addTime = Date.now() - startAdd;
|
|
562
|
+
const startDispatch = Date.now();
|
|
563
|
+
await dispatchQueueTask(consumer, taskIds);
|
|
564
|
+
const dispatchTime = Date.now() - startDispatch;
|
|
565
|
+
expect(processedCount).toBe(1000);
|
|
566
|
+
console.log(`Performance test: Add ${addTime}ms, Dispatch ${dispatchTime}ms`);
|
|
567
|
+
await clearQueue(consumer);
|
|
568
|
+
provider.disconnect();
|
|
569
|
+
await consumer.disconnect();
|
|
570
|
+
}, 30000);
|
|
571
|
+
});
|
|
572
|
+
});
|
|
@@ -32,9 +32,6 @@ import type { RedisClientType } from 'redis';
|
|
|
32
32
|
* ```
|
|
33
33
|
*/
|
|
34
34
|
export interface RedisQueueRegistry {
|
|
35
|
-
'test': {
|
|
36
|
-
example: string;
|
|
37
|
-
};
|
|
38
35
|
}
|
|
39
36
|
/**
|
|
40
37
|
* 任务状态
|
|
@@ -75,7 +72,6 @@ export interface QueueStats {
|
|
|
75
72
|
export interface RedisQueueCommonConfig {
|
|
76
73
|
redisUrl?: string;
|
|
77
74
|
redisClient?: RedisClientType;
|
|
78
|
-
queueKey: keyof RedisQueueRegistry;
|
|
79
75
|
cleanupDelay?: number;
|
|
80
76
|
}
|
|
81
77
|
/**
|
|
@@ -84,7 +80,6 @@ export interface RedisQueueCommonConfig {
|
|
|
84
80
|
export interface RedisQueueProviderConfig<K extends keyof RedisQueueRegistry = keyof RedisQueueRegistry> {
|
|
85
81
|
redisUrl?: string;
|
|
86
82
|
redisClient?: RedisClientType;
|
|
87
|
-
queueKey: K;
|
|
88
83
|
cleanupDelay?: number;
|
|
89
84
|
processingDelay?: number;
|
|
90
85
|
}
|
|
@@ -94,7 +89,6 @@ export interface RedisQueueProviderConfig<K extends keyof RedisQueueRegistry = k
|
|
|
94
89
|
export interface RedisQueueConsumerConfig<K extends keyof RedisQueueRegistry> {
|
|
95
90
|
redisUrl?: string;
|
|
96
91
|
redisClient?: RedisClientType;
|
|
97
|
-
queueKey: K;
|
|
98
92
|
handler: TaskHandler<RedisQueueRegistry[K]>;
|
|
99
93
|
maxRetries?: number;
|
|
100
94
|
concurrency?: number;
|
|
@@ -112,7 +106,6 @@ export type TaskHandler<T extends TaskData = TaskData> = (data: T) => Promise<Ca
|
|
|
112
106
|
export interface BatchConsumerConfig<K extends keyof RedisQueueRegistry> {
|
|
113
107
|
redisUrl?: string;
|
|
114
108
|
redisClient?: RedisClientType;
|
|
115
|
-
queueKey: K;
|
|
116
109
|
handler: BatchTaskHandler<RedisQueueRegistry[K]>;
|
|
117
110
|
batchSize?: number;
|
|
118
111
|
maxRetries?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../source/redis-queue/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,OAAO,CAAA;AAE5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,MAAM,WAAW,kBAAkB;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../source/redis-queue/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,OAAO,CAAA;AAE5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,MAAM,WAAW,kBAAkB;CAElC;AAED;;GAEG;AACH,MAAM,MAAM,MAAM,GAAG,SAAS,GAAG,YAAY,GAAG,WAAW,GAAG,QAAQ,CAAA;AAEtE;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAE,CAAA;AAE1D;;GAEG;AACH,MAAM,WAAW,IAAI,CAAC,CAAC,GAAG,QAAQ;IAChC,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,CAAC,CAAA;IACP,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAA;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB,CAAC,CAAC,SAAS,MAAM,kBAAkB,GAAG,MAAM,kBAAkB;IACrG,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB,CAAC,CAAC,SAAS,MAAM,kBAAkB;IAC1E,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,OAAO,EAAE,WAAW,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,MAAM,WAAW,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAA;AAE3G;;GAEG;AACH,MAAM,WAAW,mBAAmB,CAAE,CAAC,SAAS,MAAM,kBAAkB;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,WAAW,CAAC,EAAE,eAAe,CAAA;IAC7B,OAAO,EAAE,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAA;IAChD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAC3B;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@taicode/common-server",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.14",
|
|
4
4
|
"author": "Alain",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"description": "",
|
|
@@ -29,14 +29,16 @@
|
|
|
29
29
|
],
|
|
30
30
|
"peerDependencies": {
|
|
31
31
|
"@taicode/common-base": "^1.7.0",
|
|
32
|
+
"bullmq": "^5.34.0",
|
|
32
33
|
"fastify": "^5.5.0",
|
|
33
|
-
"
|
|
34
|
+
"ioredis": "^5.4.2",
|
|
34
35
|
"zod": "^4.1.0"
|
|
35
36
|
},
|
|
36
37
|
"devDependencies": {
|
|
37
38
|
"@types/node": "^22.10.2",
|
|
39
|
+
"bullmq": "^5.65.0",
|
|
38
40
|
"fastify": "^5.5.0",
|
|
39
|
-
"
|
|
41
|
+
"ioredis": "^5.8.2",
|
|
40
42
|
"vitest": "^3.2.4",
|
|
41
43
|
"zod": "^4.1.0"
|
|
42
44
|
}
|