@taicode/common-server 1.0.13 → 1.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/output/redis-queue/index.d.ts +1 -1
- package/output/redis-queue/index.d.ts.map +1 -1
- package/output/redis-queue/index.js +1 -1
- package/output/redis-queue/redis-batch-consumer.d.ts +9 -0
- package/output/redis-queue/redis-batch-consumer.d.ts.map +1 -1
- package/output/redis-queue/redis-batch-consumer.js +63 -0
- package/output/redis-queue/redis-queue-batch-consumer.d.ts +77 -0
- package/output/redis-queue/redis-queue-batch-consumer.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-batch-consumer.js +320 -0
- package/output/redis-queue/redis-queue-batch-consumer.test.d.ts +26 -0
- package/output/redis-queue/redis-queue-batch-consumer.test.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-batch-consumer.test.js +341 -0
- package/output/redis-queue/redis-queue-common.d.ts +1 -1
- package/output/redis-queue/redis-queue-common.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-common.js +8 -8
- package/output/redis-queue/redis-queue-common.test.js +16 -54
- package/output/redis-queue/redis-queue-consumer.d.ts +5 -9
- package/output/redis-queue/redis-queue-consumer.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-consumer.js +80 -69
- package/output/redis-queue/redis-queue-consumer.test.d.ts +20 -1
- package/output/redis-queue/redis-queue-consumer.test.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-consumer.test.js +89 -15
- package/output/redis-queue/redis-queue-provider.d.ts +4 -4
- package/output/redis-queue/redis-queue-provider.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-provider.js +10 -6
- package/output/redis-queue/redis-queue-provider.test.d.ts +23 -2
- package/output/redis-queue/redis-queue-provider.test.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-provider.test.js +73 -38
- package/output/redis-queue/test-helpers.d.ts +112 -0
- package/output/redis-queue/test-helpers.d.ts.map +1 -0
- package/output/redis-queue/test-helpers.js +242 -0
- package/output/redis-queue/test-helpers.test.d.ts +28 -0
- package/output/redis-queue/test-helpers.test.d.ts.map +1 -0
- package/output/redis-queue/test-helpers.test.js +572 -0
- package/output/redis-queue/types.d.ts +0 -7
- package/output/redis-queue/types.d.ts.map +1 -1
- package/package.json +5 -3
|
@@ -9,14 +9,8 @@ import { RedisQueueCommon } from './redis-queue-common';
|
|
|
9
9
|
*
|
|
10
10
|
* @example
|
|
11
11
|
* ```ts
|
|
12
|
-
*
|
|
13
|
-
* to: string
|
|
14
|
-
* subject: string
|
|
15
|
-
* }
|
|
16
|
-
*
|
|
17
|
-
* const consumer = new RedisQueueConsumer<EmailTask>({
|
|
12
|
+
* const consumer = new RedisQueueConsumer('email-queue', {
|
|
18
13
|
* redisUrl: 'redis://localhost:6379',
|
|
19
|
-
* queueKey: 'email-queue',
|
|
20
14
|
* concurrency: 5,
|
|
21
15
|
* handler: async (data) => {
|
|
22
16
|
* await sendEmail(data.to, data.subject)
|
|
@@ -38,24 +32,21 @@ export class RedisQueueConsumer extends RedisQueueCommon {
|
|
|
38
32
|
consumerInterval = null;
|
|
39
33
|
recoveryInterval = null;
|
|
40
34
|
processingTasks = 0; // 当前正在处理的任务数
|
|
35
|
+
processingPromises = new Set(); // 跟踪所有正在处理的任务
|
|
41
36
|
config;
|
|
42
|
-
constructor(config) {
|
|
37
|
+
constructor(queueKey, config) {
|
|
43
38
|
// 验证必填参数
|
|
44
39
|
if (!config.handler) {
|
|
45
40
|
throw new Error('[RedisQueueConsumer] handler is required');
|
|
46
41
|
}
|
|
47
42
|
// 调用父类构造函数
|
|
48
|
-
super({
|
|
43
|
+
super(queueKey, {
|
|
49
44
|
redisUrl: config.redisUrl,
|
|
50
45
|
redisClient: config.redisClient,
|
|
51
|
-
queueKey: config.queueKey,
|
|
52
46
|
cleanupDelay: config.cleanupDelay ?? 86400,
|
|
53
47
|
});
|
|
54
48
|
this.config = {
|
|
55
49
|
handler: config.handler,
|
|
56
|
-
queueKey: config.queueKey,
|
|
57
|
-
redisUrl: config.redisUrl,
|
|
58
|
-
redisClient: config.redisClient,
|
|
59
50
|
maxRetries: config.maxRetries ?? 3,
|
|
60
51
|
concurrency: config.concurrency ?? 1,
|
|
61
52
|
consumerInterval: config.consumerInterval ?? 1000,
|
|
@@ -77,10 +68,19 @@ export class RedisQueueConsumer extends RedisQueueCommon {
|
|
|
77
68
|
}
|
|
78
69
|
/**
|
|
79
70
|
* 断开 Redis 连接并停止消费者
|
|
71
|
+
* 会等待所有正在处理的任务完成后再断开连接
|
|
80
72
|
*/
|
|
81
|
-
disconnect() {
|
|
73
|
+
async disconnect() {
|
|
74
|
+
// 先停止消费者和恢复机制,不再接受新任务
|
|
82
75
|
this.stopConsumer();
|
|
83
76
|
this.stopRecovery();
|
|
77
|
+
// 等待所有正在处理的任务完成
|
|
78
|
+
if (this.processingPromises.size > 0) {
|
|
79
|
+
console.log(`[RedisQueueConsumer] Waiting for ${this.processingPromises.size} tasks to complete...`);
|
|
80
|
+
await Promise.allSettled(this.processingPromises);
|
|
81
|
+
console.log(`[RedisQueueConsumer] All tasks completed`);
|
|
82
|
+
}
|
|
83
|
+
// 最后断开 Redis 连接
|
|
84
84
|
super.disconnect();
|
|
85
85
|
}
|
|
86
86
|
/**
|
|
@@ -88,73 +88,84 @@ export class RedisQueueConsumer extends RedisQueueCommon {
|
|
|
88
88
|
*/
|
|
89
89
|
async processTask(taskId) {
|
|
90
90
|
this.processingTasks++;
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
if (!task) {
|
|
94
|
-
console.warn(`[RedisQueueConsumer] Task not found: ${taskId}`);
|
|
95
|
-
return;
|
|
96
|
-
}
|
|
97
|
-
// 任务应该是 pending 状态(Lua 脚本已确保延迟检查)
|
|
98
|
-
if (task.status !== 'pending') {
|
|
99
|
-
console.log(`[RedisQueueConsumer] Task ${taskId} has invalid status (${task.status}), marking as failed`);
|
|
100
|
-
await this.applyStatus(taskId, task.status, 'failed');
|
|
101
|
-
return;
|
|
102
|
-
}
|
|
91
|
+
// 创建一个 Promise 来跟踪这个任务的处理过程
|
|
92
|
+
const taskPromise = (async () => {
|
|
103
93
|
try {
|
|
104
|
-
|
|
105
|
-
task
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
//
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
task
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
//
|
|
124
|
-
|
|
94
|
+
const task = await this.getTask(taskId);
|
|
95
|
+
if (!task) {
|
|
96
|
+
console.warn(`[RedisQueueConsumer] Task not found: ${taskId}`);
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
// 任务应该是 pending 状态(Lua 脚本已确保延迟检查)
|
|
100
|
+
if (task.status !== 'pending') {
|
|
101
|
+
console.log(`[RedisQueueConsumer] Task ${taskId} has invalid status (${task.status}), marking as failed`);
|
|
102
|
+
await this.applyStatus(taskId, task.status, 'failed');
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
try {
|
|
106
|
+
// 任务已在 processing 队列中(由 Lua 脚本完成),只需更新状态和开始时间
|
|
107
|
+
task.status = 'processing';
|
|
108
|
+
task.processingStartTime = Date.now();
|
|
109
|
+
const taskKey = `${this.queueKey}:task:${taskId}`;
|
|
110
|
+
await this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
|
|
111
|
+
// 执行任务处理器
|
|
112
|
+
await this.config.handler(task.data);
|
|
113
|
+
// 更新状态为完成
|
|
114
|
+
await this.applyStatus(taskId, 'processing', 'completed');
|
|
115
|
+
console.log(`[RedisQueueConsumer] Task completed: ${taskId}`);
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
console.error(`[RedisQueueConsumer] Task failed: ${taskId}`, error);
|
|
119
|
+
// 检查是否需要重试
|
|
120
|
+
if (task.retryCount < task.maxRetries) {
|
|
121
|
+
task.retryCount++;
|
|
122
|
+
task.status = 'pending';
|
|
123
|
+
task.processingStartTime = undefined;
|
|
124
|
+
const taskKey = `${this.queueKey}:task:${taskId}`;
|
|
125
|
+
// 使用 Lua 脚本确保原子性
|
|
126
|
+
const script = `
|
|
125
127
|
redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
|
|
126
128
|
redis.call('LREM', KEYS[2], 0, ARGV[3])
|
|
127
129
|
redis.call('RPUSH', KEYS[3], ARGV[3])
|
|
128
130
|
return 1
|
|
129
131
|
`;
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
132
|
+
await this.redis.eval(script, {
|
|
133
|
+
keys: [taskKey, this.processingQueue, this.pendingQueue],
|
|
134
|
+
arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
|
|
135
|
+
});
|
|
136
|
+
console.log(`[RedisQueueConsumer] Task ${taskId} will retry (${task.retryCount}/${task.maxRetries})`);
|
|
137
|
+
}
|
|
138
|
+
else {
|
|
139
|
+
task.status = 'failed';
|
|
140
|
+
task.processingStartTime = undefined;
|
|
141
|
+
const taskKey = `${this.queueKey}:task:${taskId}`;
|
|
142
|
+
// 使用 Lua 脚本确保原子性
|
|
143
|
+
const script = `
|
|
142
144
|
redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
|
|
143
145
|
redis.call('LREM', KEYS[2], 0, ARGV[3])
|
|
144
146
|
redis.call('RPUSH', KEYS[3], ARGV[3])
|
|
145
147
|
return 1
|
|
146
148
|
`;
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
149
|
+
await this.redis.eval(script, {
|
|
150
|
+
keys: [taskKey, this.processingQueue, this.failedQueue],
|
|
151
|
+
arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
|
|
152
|
+
});
|
|
153
|
+
console.error(`[RedisQueueConsumer] Task ${taskId} failed after ${task.maxRetries} retries`);
|
|
154
|
+
}
|
|
152
155
|
}
|
|
153
156
|
}
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
}
|
|
157
|
+
finally {
|
|
158
|
+
this.processingTasks--;
|
|
159
|
+
}
|
|
160
|
+
})();
|
|
161
|
+
// 将任务 Promise 添加到集合中
|
|
162
|
+
this.processingPromises.add(taskPromise);
|
|
163
|
+
// 任务完成后从集合中移除
|
|
164
|
+
taskPromise.finally(() => {
|
|
165
|
+
this.processingPromises.delete(taskPromise);
|
|
166
|
+
});
|
|
167
|
+
// 返回任务 Promise
|
|
168
|
+
return taskPromise;
|
|
158
169
|
}
|
|
159
170
|
/**
|
|
160
171
|
* 启动恢复机制(内部方法,自动调用)
|
|
@@ -244,7 +255,7 @@ export class RedisQueueConsumer extends RedisQueueCommon {
|
|
|
244
255
|
}
|
|
245
256
|
// 使用 Lua 脚本原子化取出任务并移到 processing 队列
|
|
246
257
|
const taskIds = await this.redis.eval(popAndMoveScript, {
|
|
247
|
-
keys: [this.pendingQueue, this.processingQueue, this.
|
|
258
|
+
keys: [this.pendingQueue, this.processingQueue, this.queueKey],
|
|
248
259
|
arguments: [availableSlots.toString(), Date.now().toString()],
|
|
249
260
|
});
|
|
250
261
|
// 并发处理所有任务(不等待完成)
|
|
@@ -1,6 +1,25 @@
|
|
|
1
|
+
type TestTaskData = {
|
|
2
|
+
id?: string;
|
|
3
|
+
value?: number | string;
|
|
4
|
+
test?: boolean | string;
|
|
5
|
+
index?: number;
|
|
6
|
+
payload?: string;
|
|
7
|
+
order?: number;
|
|
8
|
+
taskId?: number;
|
|
9
|
+
message?: string;
|
|
10
|
+
name?: string;
|
|
11
|
+
shouldFail?: boolean;
|
|
12
|
+
step?: number;
|
|
13
|
+
real?: boolean;
|
|
14
|
+
nested?: {
|
|
15
|
+
value: number;
|
|
16
|
+
};
|
|
17
|
+
example?: string;
|
|
18
|
+
data?: any;
|
|
19
|
+
};
|
|
1
20
|
declare module './types' {
|
|
2
21
|
interface RedisQueueRegistry {
|
|
3
|
-
|
|
22
|
+
[key: `test:queue:${string}`]: TestTaskData;
|
|
4
23
|
}
|
|
5
24
|
}
|
|
6
25
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redis-queue-consumer.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-consumer.test.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"redis-queue-consumer.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-consumer.test.ts"],"names":[],"mappings":"AAQA,KAAK,YAAY,GAAG;IAClB,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,IAAI,CAAC,EAAE,GAAG,CAAA;CACX,CAAA;AAGD,OAAO,QAAQ,SAAS,CAAC;IACvB,UAAU,kBAAkB;QAC1B,CAAC,GAAG,EAAE,cAAc,MAAM,EAAE,GAAG,YAAY,CAAA;KAC5C;CACF"}
|
|
@@ -2,21 +2,17 @@ import { describe, it, expect, afterEach } from 'vitest';
|
|
|
2
2
|
import { catchIt } from '@taicode/common-base';
|
|
3
3
|
import { RedisQueueProvider } from './redis-queue-provider';
|
|
4
4
|
import { RedisQueueConsumer } from './redis-queue-consumer';
|
|
5
|
+
import { dispatchQueueTask, waitQueueCompletion, clearQueue, getQueueTasks } from './test-helpers';
|
|
5
6
|
const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
|
|
6
7
|
describe('RedisQueueConsumer', () => {
|
|
7
8
|
const providers = [];
|
|
8
9
|
const consumers = [];
|
|
9
10
|
afterEach(async () => {
|
|
10
11
|
for (const consumer of consumers) {
|
|
11
|
-
consumer
|
|
12
|
+
await clearQueue(consumer);
|
|
13
|
+
await consumer.disconnect();
|
|
12
14
|
}
|
|
13
15
|
for (const provider of providers) {
|
|
14
|
-
try {
|
|
15
|
-
await provider.clear();
|
|
16
|
-
}
|
|
17
|
-
catch (error) {
|
|
18
|
-
// 忽略清理错误
|
|
19
|
-
}
|
|
20
16
|
provider.disconnect();
|
|
21
17
|
}
|
|
22
18
|
providers.length = 0;
|
|
@@ -24,13 +20,11 @@ describe('RedisQueueConsumer', () => {
|
|
|
24
20
|
});
|
|
25
21
|
const createQueue = (handler, options) => {
|
|
26
22
|
const uniqueKey = `test:queue:${Date.now()}:${Math.random()}`;
|
|
27
|
-
const provider = new RedisQueueProvider({
|
|
23
|
+
const provider = new RedisQueueProvider(uniqueKey, {
|
|
28
24
|
redisUrl: REDIS_URL,
|
|
29
|
-
queueKey: uniqueKey,
|
|
30
25
|
});
|
|
31
|
-
const consumer = new RedisQueueConsumer({
|
|
26
|
+
const consumer = new RedisQueueConsumer(uniqueKey, {
|
|
32
27
|
redisUrl: REDIS_URL,
|
|
33
|
-
queueKey: uniqueKey,
|
|
34
28
|
consumerInterval: 100,
|
|
35
29
|
maxRetries: 2,
|
|
36
30
|
...options,
|
|
@@ -76,10 +70,28 @@ describe('RedisQueueConsumer', () => {
|
|
|
76
70
|
const stats = await consumer.statistics();
|
|
77
71
|
expect(stats.completed).toBe(1);
|
|
78
72
|
});
|
|
73
|
+
it('应该能够使用 dispatchQueueTask 立即处理任务', async () => {
|
|
74
|
+
const processedData = [];
|
|
75
|
+
const { provider, consumer } = createQueue(async (data) => {
|
|
76
|
+
processedData.push(data);
|
|
77
|
+
return catchIt(() => { });
|
|
78
|
+
});
|
|
79
|
+
await provider.connect();
|
|
80
|
+
await consumer.connect();
|
|
81
|
+
const taskIds = await provider.enqueue([{ value: 1 }, { value: 2 }, { value: 3 }]);
|
|
82
|
+
// 使用 dispatchQueueTask 立即处理任务
|
|
83
|
+
await dispatchQueueTask(consumer, taskIds);
|
|
84
|
+
expect(processedData).toHaveLength(3);
|
|
85
|
+
expect(processedData.map(d => d.value)).toEqual([1, 2, 3]);
|
|
86
|
+
const stats = await consumer.statistics();
|
|
87
|
+
expect(stats.completed).toBe(3);
|
|
88
|
+
});
|
|
79
89
|
it('应该按 FIFO 顺序处理任务', async () => {
|
|
80
90
|
const processOrder = [];
|
|
81
91
|
const { provider, consumer } = createQueue(async (data) => {
|
|
82
|
-
|
|
92
|
+
if (data.order !== undefined) {
|
|
93
|
+
processOrder.push(data.order);
|
|
94
|
+
}
|
|
83
95
|
return catchIt(() => { });
|
|
84
96
|
});
|
|
85
97
|
await provider.connect();
|
|
@@ -88,6 +100,35 @@ describe('RedisQueueConsumer', () => {
|
|
|
88
100
|
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
89
101
|
expect(processOrder).toEqual([1, 2, 3]);
|
|
90
102
|
});
|
|
103
|
+
it('应该能够使用 waitQueueCompletion 等待队列完成', async () => {
|
|
104
|
+
const { provider, consumer } = createQueue(async () => {
|
|
105
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
106
|
+
return catchIt(() => { });
|
|
107
|
+
});
|
|
108
|
+
await provider.connect();
|
|
109
|
+
await consumer.connect();
|
|
110
|
+
await provider.enqueue([{ test: 1 }, { test: 2 }, { test: 3 }]);
|
|
111
|
+
// 等待所有任务完成
|
|
112
|
+
await waitQueueCompletion(consumer, stats => stats.pending === 0 && stats.processing === 0 && stats.completed === 3);
|
|
113
|
+
const stats = await consumer.statistics();
|
|
114
|
+
expect(stats.completed).toBe(3);
|
|
115
|
+
});
|
|
116
|
+
it('应该能够使用 getQueueTasks 获取任务详情', async () => {
|
|
117
|
+
const { provider, consumer } = createQueue(async () => catchIt(() => { }));
|
|
118
|
+
await provider.connect();
|
|
119
|
+
await consumer.connect();
|
|
120
|
+
await provider.enqueue([{ name: 'task1' }, { name: 'task2' }]);
|
|
121
|
+
// 获取待处理任务
|
|
122
|
+
const pendingTasks = await getQueueTasks(provider, 'pending');
|
|
123
|
+
expect(pendingTasks).toHaveLength(2);
|
|
124
|
+
expect(pendingTasks.map(t => t.data.name).sort()).toEqual(['task1', 'task2']);
|
|
125
|
+
// 等待任务完成
|
|
126
|
+
await waitQueueCompletion(consumer, stats => stats.completed === 2);
|
|
127
|
+
// 获取已完成任务
|
|
128
|
+
const completedTasks = await getQueueTasks(provider, 'completed');
|
|
129
|
+
expect(completedTasks).toHaveLength(2);
|
|
130
|
+
expect(completedTasks.every(t => t.status === 'completed')).toBe(true);
|
|
131
|
+
});
|
|
91
132
|
});
|
|
92
133
|
describe('重试机制', () => {
|
|
93
134
|
it('失败的任务应该自动重试', async () => {
|
|
@@ -119,12 +160,41 @@ describe('RedisQueueConsumer', () => {
|
|
|
119
160
|
await provider.enqueue({ test: 1 });
|
|
120
161
|
await new Promise(resolve => setTimeout(resolve, 300));
|
|
121
162
|
expect(processCount).toBe(1);
|
|
122
|
-
consumer.disconnect();
|
|
163
|
+
await consumer.disconnect();
|
|
123
164
|
await provider.connect();
|
|
124
165
|
await provider.enqueue({ test: 2 });
|
|
125
166
|
await new Promise(resolve => setTimeout(resolve, 300));
|
|
126
167
|
expect(processCount).toBe(1); // 没有增加
|
|
127
168
|
});
|
|
169
|
+
it('disconnect 应该等待正在处理的任务完成', async () => {
|
|
170
|
+
const processedTasks = [];
|
|
171
|
+
let processingTask = null;
|
|
172
|
+
const { provider, consumer } = createQueue(async (data) => {
|
|
173
|
+
processingTask = data.taskId || 0;
|
|
174
|
+
await new Promise(resolve => setTimeout(resolve, 500)); // 模拟长时间处理
|
|
175
|
+
processedTasks.push(data.taskId || 0);
|
|
176
|
+
processingTask = null;
|
|
177
|
+
return catchIt(() => { });
|
|
178
|
+
});
|
|
179
|
+
await provider.connect();
|
|
180
|
+
await consumer.connect();
|
|
181
|
+
// 添加一个任务
|
|
182
|
+
await provider.enqueue({ taskId: 1 });
|
|
183
|
+
// 等待任务开始处理
|
|
184
|
+
await new Promise(resolve => setTimeout(resolve, 200));
|
|
185
|
+
expect(processingTask).toBe(1);
|
|
186
|
+
expect(processedTasks).toHaveLength(0);
|
|
187
|
+
// 调用 disconnect,应该等待任务完成
|
|
188
|
+
const disconnectPromise = consumer.disconnect();
|
|
189
|
+
// 此时任务应该还在处理中
|
|
190
|
+
expect(processingTask).toBe(1);
|
|
191
|
+
// 等待 disconnect 完成
|
|
192
|
+
await disconnectPromise;
|
|
193
|
+
// disconnect 完成后,任务应该已经处理完成
|
|
194
|
+
expect(processingTask).toBe(null);
|
|
195
|
+
expect(processedTasks).toHaveLength(1);
|
|
196
|
+
expect(processedTasks[0]).toBe(1);
|
|
197
|
+
});
|
|
128
198
|
});
|
|
129
199
|
describe('队列统计', () => {
|
|
130
200
|
it('应该正确返回队列统计信息', async () => {
|
|
@@ -145,9 +215,13 @@ describe('RedisQueueConsumer', () => {
|
|
|
145
215
|
const processing = [];
|
|
146
216
|
const completed = [];
|
|
147
217
|
const { provider, consumer } = createQueue(async (data) => {
|
|
148
|
-
|
|
218
|
+
if (data.taskId !== undefined) {
|
|
219
|
+
processing.push(data.taskId);
|
|
220
|
+
}
|
|
149
221
|
await new Promise(resolve => setTimeout(resolve, 200));
|
|
150
|
-
|
|
222
|
+
if (data.taskId !== undefined) {
|
|
223
|
+
completed.push(data.taskId);
|
|
224
|
+
}
|
|
151
225
|
return catchIt(() => { });
|
|
152
226
|
}, { concurrency: 3 });
|
|
153
227
|
await provider.connect();
|
|
@@ -10,8 +10,8 @@ import type { RedisQueueRegistry, RedisQueueProviderConfig, Task, QueueStats } f
|
|
|
10
10
|
* @example
|
|
11
11
|
* ```ts
|
|
12
12
|
* // 注册类型后使用
|
|
13
|
-
* const provider = new RedisQueueProvider({
|
|
14
|
-
*
|
|
13
|
+
* const provider = new RedisQueueProvider('email-queue', {
|
|
14
|
+
* redisUrl: 'redis://localhost:6379'
|
|
15
15
|
* })
|
|
16
16
|
*
|
|
17
17
|
* await provider.enqueue({
|
|
@@ -23,7 +23,7 @@ import type { RedisQueueRegistry, RedisQueueProviderConfig, Task, QueueStats } f
|
|
|
23
23
|
*/
|
|
24
24
|
export declare class RedisQueueProvider<K extends keyof RedisQueueRegistry> extends RedisQueueCommon {
|
|
25
25
|
private readonly processingDelay;
|
|
26
|
-
constructor(
|
|
26
|
+
constructor(queueKey: K, config?: RedisQueueProviderConfig<K>);
|
|
27
27
|
protected getLogPrefix(): string;
|
|
28
28
|
/**
|
|
29
29
|
* 将任务推入队列(支持单个或批量)
|
|
@@ -37,8 +37,8 @@ export declare class RedisQueueProvider<K extends keyof RedisQueueRegistry> exte
|
|
|
37
37
|
* await provider.enqueue({ id: 'email-123', to: 'user@example.com' })
|
|
38
38
|
* await provider.enqueue({ id: 'email-123', to: 'user@example.com' }) // 会被跳过
|
|
39
39
|
*/
|
|
40
|
+
enqueue(data: RedisQueueRegistry[K][]): Promise<string[]>;
|
|
40
41
|
enqueue(data: RedisQueueRegistry[K]): Promise<string>;
|
|
41
|
-
enqueue(data: RedisQueueRegistry[K]): Promise<string[]>;
|
|
42
42
|
/**
|
|
43
43
|
* 获取任务详情
|
|
44
44
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redis-queue-provider.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-provider.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AACvD,OAAO,KAAK,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAE7F;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,kBAAkB,CAAC,CAAC,SAAS,MAAM,kBAAkB,CAAE,SAAQ,gBAAgB;IAC1F,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAQ;gBAE5B,
|
|
1
|
+
{"version":3,"file":"redis-queue-provider.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-provider.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AACvD,OAAO,KAAK,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAE7F;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,kBAAkB,CAAC,CAAC,SAAS,MAAM,kBAAkB,CAAE,SAAQ,gBAAgB;IAC1F,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAQ;gBAE5B,QAAQ,EAAE,CAAC,EAAE,MAAM,GAAE,wBAAwB,CAAC,CAAC,CAAM;IAUjE,SAAS,CAAC,YAAY,IAAI,MAAM;IAMhC;;;;;;;;;;;OAWG;IACG,OAAO,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IACzD,OAAO,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IA0F3D;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAW1E;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC;IAevC;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAmD7B"}
|
|
@@ -10,8 +10,8 @@ import { RedisQueueCommon } from './redis-queue-common';
|
|
|
10
10
|
* @example
|
|
11
11
|
* ```ts
|
|
12
12
|
* // 注册类型后使用
|
|
13
|
-
* const provider = new RedisQueueProvider({
|
|
14
|
-
*
|
|
13
|
+
* const provider = new RedisQueueProvider('email-queue', {
|
|
14
|
+
* redisUrl: 'redis://localhost:6379'
|
|
15
15
|
* })
|
|
16
16
|
*
|
|
17
17
|
* await provider.enqueue({
|
|
@@ -23,11 +23,10 @@ import { RedisQueueCommon } from './redis-queue-common';
|
|
|
23
23
|
*/
|
|
24
24
|
export class RedisQueueProvider extends RedisQueueCommon {
|
|
25
25
|
processingDelay;
|
|
26
|
-
constructor(config) {
|
|
27
|
-
super({
|
|
26
|
+
constructor(queueKey, config = {}) {
|
|
27
|
+
super(queueKey, {
|
|
28
28
|
redisUrl: config.redisUrl,
|
|
29
29
|
redisClient: config.redisClient,
|
|
30
|
-
queueKey: config.queueKey,
|
|
31
30
|
cleanupDelay: config.cleanupDelay,
|
|
32
31
|
});
|
|
33
32
|
this.processingDelay = config.processingDelay ?? 0; // 默认立即执行
|
|
@@ -47,6 +46,11 @@ export class RedisQueueProvider extends RedisQueueCommon {
|
|
|
47
46
|
// 准备任务数据
|
|
48
47
|
for (const item of dataList) {
|
|
49
48
|
const customId = item.id;
|
|
49
|
+
// 严格检查 id 类型,只接受 string
|
|
50
|
+
if (customId !== undefined && typeof customId !== 'string') {
|
|
51
|
+
throw new TypeError(`[RedisQueueProvider] Task id must be a string, got ${typeof customId}. ` +
|
|
52
|
+
`Please use string type for task id field.`);
|
|
53
|
+
}
|
|
50
54
|
const taskId = customId || randomUUID();
|
|
51
55
|
const task = {
|
|
52
56
|
id: taskId,
|
|
@@ -86,7 +90,7 @@ export class RedisQueueProvider extends RedisQueueCommon {
|
|
|
86
90
|
|
|
87
91
|
return successCount
|
|
88
92
|
`;
|
|
89
|
-
// 构建参数数组
|
|
93
|
+
// 构建参数数组 - Redis 要求所有参数都是字符串
|
|
90
94
|
const args = [this.cleanupDelay.toString(), this.queueKey];
|
|
91
95
|
for (let i = 0; i < tasks.length; i++) {
|
|
92
96
|
args.push(taskIds[i]);
|
|
@@ -1,6 +1,27 @@
|
|
|
1
|
-
|
|
1
|
+
type TestTaskData = {
|
|
2
|
+
id?: string;
|
|
3
|
+
value?: number | string;
|
|
4
|
+
test?: boolean | string;
|
|
5
|
+
index?: number;
|
|
6
|
+
payload?: string;
|
|
7
|
+
order?: number;
|
|
8
|
+
taskId?: number;
|
|
9
|
+
message?: string;
|
|
10
|
+
name?: string;
|
|
11
|
+
shouldFail?: boolean;
|
|
12
|
+
step?: number;
|
|
13
|
+
real?: boolean;
|
|
14
|
+
nested?: {
|
|
15
|
+
value: number;
|
|
16
|
+
};
|
|
17
|
+
example?: string;
|
|
18
|
+
foo?: string;
|
|
19
|
+
number?: number;
|
|
20
|
+
data?: any;
|
|
21
|
+
};
|
|
22
|
+
declare module "./types" {
|
|
2
23
|
interface RedisQueueRegistry {
|
|
3
|
-
|
|
24
|
+
[key: `test:provider:${string}`]: TestTaskData;
|
|
4
25
|
}
|
|
5
26
|
}
|
|
6
27
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redis-queue-provider.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-provider.test.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"redis-queue-provider.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-provider.test.ts"],"names":[],"mappings":"AAMA,KAAK,YAAY,GAAG;IAClB,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,CAAC,EAAE,GAAG,CAAA;CACX,CAAA;AAGD,OAAO,QAAQ,SAAS,CAAC;IACvB,UAAU,kBAAkB;QAC1B,CAAC,GAAG,EAAE,iBAAiB,MAAM,EAAE,GAAG,YAAY,CAAA;KAC/C;CACF"}
|