@taicode/common-server 1.0.10 → 1.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/output/redis-queue/batch-redis-queue.d.ts +25 -25
- package/output/redis-queue/batch-redis-queue.d.ts.map +1 -1
- package/output/redis-queue/batch-redis-queue.js +83 -90
- package/output/redis-queue/batch-redis-queue.test.js +96 -251
- package/output/redis-queue/redis-queue.d.ts +26 -25
- package/output/redis-queue/redis-queue.d.ts.map +1 -1
- package/output/redis-queue/redis-queue.js +25 -28
- package/output/redis-queue/redis-queue.test.js +96 -698
- package/output/redis-queue/types.d.ts +14 -5
- package/output/redis-queue/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { BatchTaskQueueConfig, TaskData, Task,
|
|
1
|
+
import type { BatchTaskQueueConfig, TaskData, Task, QueueStats } from './types';
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
3
|
+
* 批量任务队列类（泛型）
|
|
4
4
|
*
|
|
5
5
|
* 提供基于 Redis 的批量任务队列功能，支持：
|
|
6
6
|
* - 批量处理任务（每次处理多条）
|
|
@@ -9,24 +9,29 @@ import type { BatchTaskQueueConfig, TaskData, Task, BatchTaskHandler, QueueStats
|
|
|
9
9
|
* - 任务状态追踪
|
|
10
10
|
* - 分布式消费
|
|
11
11
|
*
|
|
12
|
+
* @template T 任务数据类型
|
|
13
|
+
*
|
|
12
14
|
* @example
|
|
13
15
|
* ```ts
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* batchSize: 50, // 每次批量处理 50 个任务
|
|
18
|
-
* })
|
|
16
|
+
* interface EmailTask {
|
|
17
|
+
* to: string
|
|
18
|
+
* }
|
|
19
19
|
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
20
|
+
* const queue = new BatchRedisQueue<EmailTask>({
|
|
21
|
+
* redisUrl: 'redis://localhost:6379',
|
|
22
|
+
* queueKey: 'email-batch-queue',
|
|
23
|
+
* batchSize: 50,
|
|
24
|
+
* handler: async (dataList) => {
|
|
25
|
+
* await sendEmailsBatch(dataList.map(d => d.to))
|
|
26
|
+
* return catchIt(() => {})
|
|
27
|
+
* }
|
|
23
28
|
* })
|
|
24
29
|
*
|
|
25
30
|
* // 连接 Redis（自动启动消费者）
|
|
26
31
|
* await queue.connect()
|
|
27
32
|
*
|
|
28
33
|
* // 入队任务（自动开始处理）
|
|
29
|
-
* await queue.enqueue(
|
|
34
|
+
* await queue.enqueue([
|
|
30
35
|
* { to: 'user1@example.com' },
|
|
31
36
|
* { to: 'user2@example.com' },
|
|
32
37
|
* ])
|
|
@@ -36,19 +41,19 @@ import type { BatchTaskQueueConfig, TaskData, Task, BatchTaskHandler, QueueStats
|
|
|
36
41
|
* console.log(stats) // { pending: 100, processing: 50, completed: 200, failed: 5 }
|
|
37
42
|
* ```
|
|
38
43
|
*/
|
|
39
|
-
export declare class BatchRedisQueue {
|
|
44
|
+
export declare class BatchRedisQueue<T extends TaskData = TaskData> {
|
|
40
45
|
private consumerRunning;
|
|
41
46
|
private redis;
|
|
42
47
|
private consumerInterval;
|
|
43
48
|
private recoveryInterval;
|
|
44
49
|
private processingBatches;
|
|
45
50
|
private readonly config;
|
|
46
|
-
private readonly
|
|
51
|
+
private readonly handler;
|
|
47
52
|
private readonly failedQueue;
|
|
48
53
|
private readonly pendingQueue;
|
|
49
54
|
private readonly processingQueue;
|
|
50
55
|
private readonly completedQueue;
|
|
51
|
-
constructor(config: BatchTaskQueueConfig);
|
|
56
|
+
constructor(config: BatchTaskQueueConfig<T>);
|
|
52
57
|
/**
|
|
53
58
|
* 连接 Redis 并自动启动消费者
|
|
54
59
|
*/
|
|
@@ -57,29 +62,24 @@ export declare class BatchRedisQueue {
|
|
|
57
62
|
* 断开 Redis 连接并停止消费者
|
|
58
63
|
*/
|
|
59
64
|
disconnect(): void;
|
|
60
|
-
/**
|
|
61
|
-
* 注册批量任务处理器
|
|
62
|
-
*/
|
|
63
|
-
handle<T extends TaskData = TaskData>(type: string, handler: BatchTaskHandler<T>): void;
|
|
64
65
|
/**
|
|
65
66
|
* 将任务推入队列(支持单个或批量)
|
|
66
|
-
* @param type 任务类型
|
|
67
67
|
* @param data 任务数据(单个或数组)。可以在 data 中包含 `id` 字段来实现幂等性
|
|
68
68
|
*
|
|
69
69
|
* @example
|
|
70
70
|
* // 普通任务,自动生成 ID
|
|
71
|
-
* await queue.enqueue(
|
|
71
|
+
* await queue.enqueue({ to: 'user@example.com' })
|
|
72
72
|
*
|
|
73
73
|
* // 幂等任务,手动指定 ID,重复提交会被忽略
|
|
74
|
-
* await queue.enqueue(
|
|
75
|
-
* await queue.enqueue(
|
|
74
|
+
* await queue.enqueue({ id: 'email-123', to: 'user@example.com' })
|
|
75
|
+
* await queue.enqueue({ id: 'email-123', to: 'user@example.com' }) // 会被跳过
|
|
76
76
|
*/
|
|
77
|
-
enqueue
|
|
78
|
-
enqueue
|
|
77
|
+
enqueue(data: T[]): Promise<string[]>;
|
|
78
|
+
enqueue(data: T): Promise<string>;
|
|
79
79
|
/**
|
|
80
80
|
* 获取任务详情
|
|
81
81
|
*/
|
|
82
|
-
getTask(taskId: string): Promise<Task | null>;
|
|
82
|
+
getTask(taskId: string): Promise<Task<T> | null>;
|
|
83
83
|
/**
|
|
84
84
|
* 更新任务状态并移动到对应队列（原子操作）
|
|
85
85
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-redis-queue.d.ts","sourceRoot":"","sources":["../../source/redis-queue/batch-redis-queue.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAU,QAAQ,EAAE,IAAI,
|
|
1
|
+
{"version":3,"file":"batch-redis-queue.d.ts","sourceRoot":"","sources":["../../source/redis-queue/batch-redis-queue.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAU,QAAQ,EAAE,IAAI,EAAoB,UAAU,EAAE,MAAM,SAAS,CAAA;AAEzG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,qBAAa,eAAe,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IACxD,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,iBAAiB,CAAI;IAE7B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoD;IAC3E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAG7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAQ;IACrC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAQ;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;gBAE3B,MAAM,EAAE,oBAAoB,CAAC,CAAC,CAAC;IA0C3C;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAY9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAUlB;;;;;;;;;;;OAWG;IACG,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IACrC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAkEvC;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAWtD;;OAEG;YACW,WAAW;IA8BzB;;OAEG;YACW,gBAAgB;IAI9B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;;OAGG;YACW,mBAAmB;IA6EjC;;OAEG;YACW,YAAY;IA8F1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACH,OAAO,CAAC,aAAa;IA2ErB;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC;IA0BvC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB5B;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAMjC"}
|
|
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { createClient } from 'redis';
|
|
3
3
|
import { catchIt } from '@taicode/common-base';
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* 批量任务队列类（泛型）
|
|
6
6
|
*
|
|
7
7
|
* 提供基于 Redis 的批量任务队列功能，支持：
|
|
8
8
|
* - 批量处理任务（每次处理多条）
|
|
@@ -11,24 +11,29 @@ import { catchIt } from '@taicode/common-base';
|
|
|
11
11
|
* - 任务状态追踪
|
|
12
12
|
* - 分布式消费
|
|
13
13
|
*
|
|
14
|
+
* @template T 任务数据类型
|
|
15
|
+
*
|
|
14
16
|
* @example
|
|
15
17
|
* ```ts
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* batchSize: 50, // 每次批量处理 50 个任务
|
|
20
|
-
* })
|
|
18
|
+
* interface EmailTask {
|
|
19
|
+
* to: string
|
|
20
|
+
* }
|
|
21
21
|
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
22
|
+
* const queue = new BatchRedisQueue<EmailTask>({
|
|
23
|
+
* redisUrl: 'redis://localhost:6379',
|
|
24
|
+
* queueKey: 'email-batch-queue',
|
|
25
|
+
* batchSize: 50,
|
|
26
|
+
* handler: async (dataList) => {
|
|
27
|
+
* await sendEmailsBatch(dataList.map(d => d.to))
|
|
28
|
+
* return catchIt(() => {})
|
|
29
|
+
* }
|
|
25
30
|
* })
|
|
26
31
|
*
|
|
27
32
|
* // 连接 Redis（自动启动消费者）
|
|
28
33
|
* await queue.connect()
|
|
29
34
|
*
|
|
30
35
|
* // 入队任务（自动开始处理）
|
|
31
|
-
* await queue.enqueue(
|
|
36
|
+
* await queue.enqueue([
|
|
32
37
|
* { to: 'user1@example.com' },
|
|
33
38
|
* { to: 'user2@example.com' },
|
|
34
39
|
* ])
|
|
@@ -45,7 +50,7 @@ export class BatchRedisQueue {
|
|
|
45
50
|
recoveryInterval = null;
|
|
46
51
|
processingBatches = 0; // 当前正在处理的批次数
|
|
47
52
|
config;
|
|
48
|
-
|
|
53
|
+
handler;
|
|
49
54
|
// 不同状态队列的键名
|
|
50
55
|
failedQueue;
|
|
51
56
|
pendingQueue;
|
|
@@ -62,6 +67,10 @@ export class BatchRedisQueue {
|
|
|
62
67
|
if (config.queueKey.length < 6) {
|
|
63
68
|
throw new Error('[BatchTaskQueue] queueKey must be at least 6 characters long');
|
|
64
69
|
}
|
|
70
|
+
if (!config.handler) {
|
|
71
|
+
throw new Error('[BatchTaskQueue] handler is required');
|
|
72
|
+
}
|
|
73
|
+
this.handler = config.handler;
|
|
65
74
|
this.config = {
|
|
66
75
|
queueKey: config.queueKey,
|
|
67
76
|
redisUrl: config.redisUrl,
|
|
@@ -110,13 +119,7 @@ export class BatchRedisQueue {
|
|
|
110
119
|
});
|
|
111
120
|
}
|
|
112
121
|
}
|
|
113
|
-
|
|
114
|
-
* 注册批量任务处理器
|
|
115
|
-
*/
|
|
116
|
-
handle(type, handler) {
|
|
117
|
-
this.handlers.set(type, handler);
|
|
118
|
-
}
|
|
119
|
-
async enqueue(type, data) {
|
|
122
|
+
async enqueue(data) {
|
|
120
123
|
if (!this.redis) {
|
|
121
124
|
console.warn('[BatchTaskQueue] Redis not available, skipping task enqueue');
|
|
122
125
|
return Array.isArray(data) ? [] : '';
|
|
@@ -146,11 +149,10 @@ export class BatchRedisQueue {
|
|
|
146
149
|
for (const item of dataList) {
|
|
147
150
|
// 检查 data 中是否包含自定义 id
|
|
148
151
|
const customId = item.id;
|
|
149
|
-
const taskId = customId
|
|
152
|
+
const taskId = customId || randomUUID();
|
|
150
153
|
const taskKey = `${this.config.queueKey}:task:${taskId}`;
|
|
151
154
|
const task = {
|
|
152
155
|
id: taskId,
|
|
153
|
-
type,
|
|
154
156
|
data: item,
|
|
155
157
|
retryCount: 0,
|
|
156
158
|
maxRetries: this.config.maxRetries,
|
|
@@ -167,7 +169,7 @@ export class BatchRedisQueue {
|
|
|
167
169
|
console.log(`[BatchTaskQueue] Task already exists, skipping: ${taskId}`);
|
|
168
170
|
}
|
|
169
171
|
else {
|
|
170
|
-
console.log(`[BatchTaskQueue] Enqueued task: ${task.
|
|
172
|
+
console.log(`[BatchTaskQueue] Enqueued task: ${task.id}`);
|
|
171
173
|
}
|
|
172
174
|
taskIds.push(taskId);
|
|
173
175
|
}
|
|
@@ -321,8 +323,8 @@ export class BatchRedisQueue {
|
|
|
321
323
|
console.warn(`[BatchTaskQueue] No valid tasks found in batch`);
|
|
322
324
|
return;
|
|
323
325
|
}
|
|
324
|
-
//
|
|
325
|
-
const
|
|
326
|
+
// 过滤出有效的待处理任务
|
|
327
|
+
const tasksToProcess = [];
|
|
326
328
|
for (const task of validTasks) {
|
|
327
329
|
// 检查状态
|
|
328
330
|
if (task.status !== 'pending') {
|
|
@@ -330,75 +332,66 @@ export class BatchRedisQueue {
|
|
|
330
332
|
await this.applyStatus(task.id, task.status, 'failed');
|
|
331
333
|
continue;
|
|
332
334
|
}
|
|
333
|
-
|
|
334
|
-
typeTasks.push(task);
|
|
335
|
-
tasksByType.set(task.type, typeTasks);
|
|
335
|
+
tasksToProcess.push(task);
|
|
336
336
|
}
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
337
|
+
if (tasksToProcess.length === 0) {
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
340
|
+
try {
|
|
341
|
+
// 任务已在 processing 队列中(由 Lua 脚本完成),只需更新状态和开始时间
|
|
342
|
+
const taskIdList = tasksToProcess.map(t => t.id);
|
|
343
|
+
const now = Date.now();
|
|
344
|
+
await Promise.all(tasksToProcess.map(task => {
|
|
345
|
+
task.status = 'processing';
|
|
346
|
+
task.processingStartTime = now;
|
|
347
|
+
const taskKey = `${this.config.queueKey}:task:${task.id}`;
|
|
348
|
+
return this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
|
|
349
|
+
}));
|
|
350
|
+
// 执行批量任务处理器
|
|
351
|
+
const dataList = tasksToProcess.map(t => t.data);
|
|
352
|
+
await this.handler(dataList);
|
|
353
|
+
// 更新状态为完成
|
|
354
|
+
await this.applyStatusBatch(taskIdList, 'processing', 'completed');
|
|
355
|
+
console.log(`[BatchTaskQueue] Batch completed: ${tasksToProcess.length} tasks`);
|
|
356
|
+
}
|
|
357
|
+
catch (error) {
|
|
358
|
+
console.error(`[BatchTaskQueue] Batch failed`, error);
|
|
359
|
+
// 检查每个任务是否需要重试
|
|
360
|
+
for (const task of tasksToProcess) {
|
|
361
|
+
if (task.retryCount < task.maxRetries) {
|
|
362
|
+
task.retryCount++;
|
|
363
|
+
task.status = 'pending';
|
|
364
|
+
task.processingStartTime = undefined;
|
|
353
365
|
const taskKey = `${this.config.queueKey}:task:${task.id}`;
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
console.log(`[BatchTaskQueue] Task ${task.id} will retry (${task.retryCount}/${task.maxRetries})`);
|
|
384
|
-
}
|
|
385
|
-
else {
|
|
386
|
-
task.status = 'failed';
|
|
387
|
-
task.processingStartTime = undefined;
|
|
388
|
-
const taskKey = `${this.config.queueKey}:task:${task.id}`;
|
|
389
|
-
// 使用 Lua 脚本确保原子性
|
|
390
|
-
const script = `
|
|
391
|
-
redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
|
|
392
|
-
redis.call('LREM', KEYS[2], 0, ARGV[3])
|
|
393
|
-
redis.call('RPUSH', KEYS[3], ARGV[3])
|
|
394
|
-
return 1
|
|
395
|
-
`;
|
|
396
|
-
await this.redis.eval(script, {
|
|
397
|
-
keys: [taskKey, this.processingQueue, this.failedQueue],
|
|
398
|
-
arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
|
|
399
|
-
});
|
|
400
|
-
console.error(`[BatchTaskQueue] Task ${task.id} failed after ${task.maxRetries} retries`);
|
|
401
|
-
}
|
|
366
|
+
// 使用 Lua 脚本确保原子性
|
|
367
|
+
const script = `
|
|
368
|
+
redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
|
|
369
|
+
redis.call('LREM', KEYS[2], 0, ARGV[3])
|
|
370
|
+
redis.call('RPUSH', KEYS[3], ARGV[3])
|
|
371
|
+
return 1
|
|
372
|
+
`;
|
|
373
|
+
await this.redis.eval(script, {
|
|
374
|
+
keys: [taskKey, this.processingQueue, this.pendingQueue],
|
|
375
|
+
arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
|
|
376
|
+
});
|
|
377
|
+
console.log(`[BatchTaskQueue] Task ${task.id} will retry (${task.retryCount}/${task.maxRetries})`);
|
|
378
|
+
}
|
|
379
|
+
else {
|
|
380
|
+
task.status = 'failed';
|
|
381
|
+
task.processingStartTime = undefined;
|
|
382
|
+
const taskKey = `${this.config.queueKey}:task:${task.id}`;
|
|
383
|
+
// 使用 Lua 脚本确保原子性
|
|
384
|
+
const script = `
|
|
385
|
+
redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
|
|
386
|
+
redis.call('LREM', KEYS[2], 0, ARGV[3])
|
|
387
|
+
redis.call('RPUSH', KEYS[3], ARGV[3])
|
|
388
|
+
return 1
|
|
389
|
+
`;
|
|
390
|
+
await this.redis.eval(script, {
|
|
391
|
+
keys: [taskKey, this.processingQueue, this.failedQueue],
|
|
392
|
+
arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), task.id],
|
|
393
|
+
});
|
|
394
|
+
console.error(`[BatchTaskQueue] Task ${task.id} failed after ${task.maxRetries} retries`);
|
|
402
395
|
}
|
|
403
396
|
}
|
|
404
397
|
}
|