@taicode/common-server 1.0.11 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/output/index.d.ts +1 -0
- package/output/index.d.ts.map +1 -1
- package/output/index.js +1 -0
- package/output/redis-queue/index.d.ts +6 -4
- package/output/redis-queue/index.d.ts.map +1 -1
- package/output/redis-queue/index.js +4 -2
- package/output/redis-queue/redis-batch-consumer.d.ts +80 -0
- package/output/redis-queue/redis-batch-consumer.d.ts.map +1 -0
- package/output/redis-queue/redis-batch-consumer.js +308 -0
- package/output/redis-queue/redis-batch-consumer.test.d.ts +7 -0
- package/output/redis-queue/redis-batch-consumer.test.d.ts.map +1 -0
- package/output/redis-queue/redis-batch-consumer.test.js +265 -0
- package/output/redis-queue/redis-queue-common.d.ts +73 -0
- package/output/redis-queue/redis-queue-common.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-common.js +302 -0
- package/output/redis-queue/redis-queue-common.test.d.ts +19 -0
- package/output/redis-queue/redis-queue-common.test.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-common.test.js +623 -0
- package/output/redis-queue/redis-queue-consumer.d.ts +81 -0
- package/output/redis-queue/redis-queue-consumer.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-consumer.js +297 -0
- package/output/redis-queue/redis-queue-consumer.test.d.ts +7 -0
- package/output/redis-queue/redis-queue-consumer.test.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-consumer.test.js +242 -0
- package/output/redis-queue/redis-queue-provider.d.ts +56 -0
- package/output/redis-queue/redis-queue-provider.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-provider.js +187 -0
- package/output/redis-queue/redis-queue-provider.test.d.ts +7 -0
- package/output/redis-queue/redis-queue-provider.test.d.ts.map +1 -0
- package/output/redis-queue/redis-queue-provider.test.js +114 -0
- package/output/redis-queue/types.d.ts +77 -19
- package/output/redis-queue/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/output/logger/logger.d.ts +0 -33
- package/output/logger/logger.d.ts.map +0 -1
- package/output/logger/logger.js +0 -65
- package/output/logger/logger.test.d.ts +0 -2
- package/output/logger/logger.test.d.ts.map +0 -1
- package/output/logger/logger.test.js +0 -87
- package/output/redis-queue/batch-redis-queue.d.ts +0 -136
- package/output/redis-queue/batch-redis-queue.d.ts.map +0 -1
- package/output/redis-queue/batch-redis-queue.js +0 -573
- package/output/redis-queue/batch-redis-queue.test.d.ts +0 -2
- package/output/redis-queue/batch-redis-queue.test.d.ts.map +0 -1
- package/output/redis-queue/batch-redis-queue.test.js +0 -243
- package/output/redis-queue/redis-queue.d.ts +0 -129
- package/output/redis-queue/redis-queue.d.ts.map +0 -1
- package/output/redis-queue/redis-queue.js +0 -547
- package/output/redis-queue/redis-queue.test.d.ts +0 -2
- package/output/redis-queue/redis-queue.test.d.ts.map +0 -1
- package/output/redis-queue/redis-queue.test.js +0 -234
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { RedisQueueCommon } from './redis-queue-common';
|
|
2
|
+
import type { RedisQueueConsumerConfig, RedisQueueRegistry, QueueStats } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* Redis 队列消费者
|
|
5
|
+
*
|
|
6
|
+
* 只负责消费任务,不包含入队逻辑
|
|
7
|
+
*
|
|
8
|
+
* @template K 队列键名类型
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```ts
|
|
12
|
+
* interface EmailTask {
|
|
13
|
+
* to: string
|
|
14
|
+
* subject: string
|
|
15
|
+
* }
|
|
16
|
+
*
|
|
17
|
+
* const consumer = new RedisQueueConsumer<EmailTask>({
|
|
18
|
+
* redisUrl: 'redis://localhost:6379',
|
|
19
|
+
* queueKey: 'email-queue',
|
|
20
|
+
* concurrency: 5,
|
|
21
|
+
* handler: async (data) => {
|
|
22
|
+
* await sendEmail(data.to, data.subject)
|
|
23
|
+
* return catchIt(() => {})
|
|
24
|
+
* }
|
|
25
|
+
* })
|
|
26
|
+
*
|
|
27
|
+
* await consumer.connect() // 自动开始消费
|
|
28
|
+
*
|
|
29
|
+
* // 获取统计信息
|
|
30
|
+
* const stats = await consumer.statistics()
|
|
31
|
+
*
|
|
32
|
+
* // 停止消费
|
|
33
|
+
* consumer.disconnect()
|
|
34
|
+
* ```
|
|
35
|
+
*/
|
|
36
|
+
export declare class RedisQueueConsumer<K extends keyof RedisQueueRegistry> extends RedisQueueCommon {
|
|
37
|
+
private consumerRunning;
|
|
38
|
+
private consumerInterval;
|
|
39
|
+
private recoveryInterval;
|
|
40
|
+
private processingTasks;
|
|
41
|
+
private readonly config;
|
|
42
|
+
constructor(config: RedisQueueConsumerConfig<K>);
|
|
43
|
+
protected getLogPrefix(): string;
|
|
44
|
+
/**
|
|
45
|
+
* 连接 Redis 并自动启动消费者
|
|
46
|
+
*/
|
|
47
|
+
connect(): Promise<void>;
|
|
48
|
+
/**
|
|
49
|
+
* 断开 Redis 连接并停止消费者
|
|
50
|
+
*/
|
|
51
|
+
disconnect(): void;
|
|
52
|
+
/**
|
|
53
|
+
* 处理单个任务
|
|
54
|
+
*/
|
|
55
|
+
private processTask;
|
|
56
|
+
/**
|
|
57
|
+
* 启动恢复机制(内部方法,自动调用)
|
|
58
|
+
*/
|
|
59
|
+
private startRecovery;
|
|
60
|
+
/**
|
|
61
|
+
* 停止恢复机制(内部方法,自动调用)
|
|
62
|
+
*/
|
|
63
|
+
private stopRecovery;
|
|
64
|
+
/**
|
|
65
|
+
* 启动消费者(内部方法,自动调用)
|
|
66
|
+
*/
|
|
67
|
+
private startConsumer;
|
|
68
|
+
/**
|
|
69
|
+
* 停止消费者(内部方法,自动调用)
|
|
70
|
+
*/
|
|
71
|
+
private stopConsumer;
|
|
72
|
+
/**
|
|
73
|
+
* 获取队列统计信息(O(1) 时间复杂度)
|
|
74
|
+
*/
|
|
75
|
+
statistics(): Promise<QueueStats>;
|
|
76
|
+
/**
|
|
77
|
+
* 健康检查
|
|
78
|
+
*/
|
|
79
|
+
health(): Promise<boolean>;
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=redis-queue-consumer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis-queue-consumer.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-consumer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AACvD,OAAO,KAAK,EAAE,wBAAwB,EAAe,kBAAkB,EAAgB,UAAU,EAAE,MAAM,SAAS,CAAA;AAElH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,qBAAa,kBAAkB,CAAC,CAAC,SAAS,MAAM,kBAAkB,CAAE,SAAQ,gBAAgB;IAC1F,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAAI;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAUtB;gBAEW,MAAM,EAAE,wBAAwB,CAAC,CAAC,CAAC;IA2B/C,SAAS,CAAC,YAAY,IAAI,MAAM;IAIhC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAMlB;;OAEG;YACW,WAAW;IA4EzB;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACH,OAAO,CAAC,aAAa;IA4ErB;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC;IAevC;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAMjC"}
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
import { catchIt } from '@taicode/common-base';
|
|
2
|
+
import { RedisQueueCommon } from './redis-queue-common';
|
|
3
|
+
/**
|
|
4
|
+
* Redis 队列消费者
|
|
5
|
+
*
|
|
6
|
+
* 只负责消费任务,不包含入队逻辑
|
|
7
|
+
*
|
|
8
|
+
* @template K 队列键名类型
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```ts
|
|
12
|
+
* interface EmailTask {
|
|
13
|
+
* to: string
|
|
14
|
+
* subject: string
|
|
15
|
+
* }
|
|
16
|
+
*
|
|
17
|
+
* const consumer = new RedisQueueConsumer<EmailTask>({
|
|
18
|
+
* redisUrl: 'redis://localhost:6379',
|
|
19
|
+
* queueKey: 'email-queue',
|
|
20
|
+
* concurrency: 5,
|
|
21
|
+
* handler: async (data) => {
|
|
22
|
+
* await sendEmail(data.to, data.subject)
|
|
23
|
+
* return catchIt(() => {})
|
|
24
|
+
* }
|
|
25
|
+
* })
|
|
26
|
+
*
|
|
27
|
+
* await consumer.connect() // 自动开始消费
|
|
28
|
+
*
|
|
29
|
+
* // 获取统计信息
|
|
30
|
+
* const stats = await consumer.statistics()
|
|
31
|
+
*
|
|
32
|
+
* // 停止消费
|
|
33
|
+
* consumer.disconnect()
|
|
34
|
+
* ```
|
|
35
|
+
*/
|
|
36
|
+
export class RedisQueueConsumer extends RedisQueueCommon {
|
|
37
|
+
consumerRunning = false;
|
|
38
|
+
consumerInterval = null;
|
|
39
|
+
recoveryInterval = null;
|
|
40
|
+
processingTasks = 0; // 当前正在处理的任务数
|
|
41
|
+
config;
|
|
42
|
+
constructor(config) {
|
|
43
|
+
// 验证必填参数
|
|
44
|
+
if (!config.handler) {
|
|
45
|
+
throw new Error('[RedisQueueConsumer] handler is required');
|
|
46
|
+
}
|
|
47
|
+
// 调用父类构造函数
|
|
48
|
+
super({
|
|
49
|
+
redisUrl: config.redisUrl,
|
|
50
|
+
redisClient: config.redisClient,
|
|
51
|
+
queueKey: config.queueKey,
|
|
52
|
+
cleanupDelay: config.cleanupDelay ?? 86400,
|
|
53
|
+
});
|
|
54
|
+
this.config = {
|
|
55
|
+
handler: config.handler,
|
|
56
|
+
queueKey: config.queueKey,
|
|
57
|
+
redisUrl: config.redisUrl,
|
|
58
|
+
redisClient: config.redisClient,
|
|
59
|
+
maxRetries: config.maxRetries ?? 3,
|
|
60
|
+
concurrency: config.concurrency ?? 1,
|
|
61
|
+
consumerInterval: config.consumerInterval ?? 1000,
|
|
62
|
+
processingTimeout: config.processingTimeout ?? 60000,
|
|
63
|
+
cleanupDelay: config.cleanupDelay ?? 86400,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
getLogPrefix() {
|
|
67
|
+
return 'RedisQueueConsumer';
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* 连接 Redis 并自动启动消费者
|
|
71
|
+
*/
|
|
72
|
+
async connect() {
|
|
73
|
+
await super.connect();
|
|
74
|
+
// 连接成功后启动消费者和恢复机制
|
|
75
|
+
this.startConsumer();
|
|
76
|
+
this.startRecovery();
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* 断开 Redis 连接并停止消费者
|
|
80
|
+
*/
|
|
81
|
+
disconnect() {
|
|
82
|
+
this.stopConsumer();
|
|
83
|
+
this.stopRecovery();
|
|
84
|
+
super.disconnect();
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* 处理单个任务
|
|
88
|
+
*/
|
|
89
|
+
async processTask(taskId) {
|
|
90
|
+
this.processingTasks++;
|
|
91
|
+
try {
|
|
92
|
+
const task = await this.getTask(taskId);
|
|
93
|
+
if (!task) {
|
|
94
|
+
console.warn(`[RedisQueueConsumer] Task not found: ${taskId}`);
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
// 任务应该是 pending 状态(Lua 脚本已确保延迟检查)
|
|
98
|
+
if (task.status !== 'pending') {
|
|
99
|
+
console.log(`[RedisQueueConsumer] Task ${taskId} has invalid status (${task.status}), marking as failed`);
|
|
100
|
+
await this.applyStatus(taskId, task.status, 'failed');
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
try {
|
|
104
|
+
// 任务已在 processing 队列中(由 Lua 脚本完成),只需更新状态和开始时间
|
|
105
|
+
task.status = 'processing';
|
|
106
|
+
task.processingStartTime = Date.now();
|
|
107
|
+
const taskKey = `${this.config.queueKey}:task:${taskId}`;
|
|
108
|
+
await this.redis.setEx(taskKey, this.config.cleanupDelay, JSON.stringify(task));
|
|
109
|
+
// 执行任务处理器
|
|
110
|
+
await this.config.handler(task.data);
|
|
111
|
+
// 更新状态为完成
|
|
112
|
+
await this.applyStatus(taskId, 'processing', 'completed');
|
|
113
|
+
console.log(`[RedisQueueConsumer] Task completed: ${taskId}`);
|
|
114
|
+
}
|
|
115
|
+
catch (error) {
|
|
116
|
+
console.error(`[RedisQueueConsumer] Task failed: ${taskId}`, error);
|
|
117
|
+
// 检查是否需要重试
|
|
118
|
+
if (task.retryCount < task.maxRetries) {
|
|
119
|
+
task.retryCount++;
|
|
120
|
+
task.status = 'pending';
|
|
121
|
+
task.processingStartTime = undefined;
|
|
122
|
+
const taskKey = `${this.config.queueKey}:task:${taskId}`;
|
|
123
|
+
// 使用 Lua 脚本确保原子性
|
|
124
|
+
const script = `
|
|
125
|
+
redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
|
|
126
|
+
redis.call('LREM', KEYS[2], 0, ARGV[3])
|
|
127
|
+
redis.call('RPUSH', KEYS[3], ARGV[3])
|
|
128
|
+
return 1
|
|
129
|
+
`;
|
|
130
|
+
await this.redis.eval(script, {
|
|
131
|
+
keys: [taskKey, this.processingQueue, this.pendingQueue],
|
|
132
|
+
arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
|
|
133
|
+
});
|
|
134
|
+
console.log(`[RedisQueueConsumer] Task ${taskId} will retry (${task.retryCount}/${task.maxRetries})`);
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
task.status = 'failed';
|
|
138
|
+
task.processingStartTime = undefined;
|
|
139
|
+
const taskKey = `${this.config.queueKey}:task:${taskId}`;
|
|
140
|
+
// 使用 Lua 脚本确保原子性
|
|
141
|
+
const script = `
|
|
142
|
+
redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
|
|
143
|
+
redis.call('LREM', KEYS[2], 0, ARGV[3])
|
|
144
|
+
redis.call('RPUSH', KEYS[3], ARGV[3])
|
|
145
|
+
return 1
|
|
146
|
+
`;
|
|
147
|
+
await this.redis.eval(script, {
|
|
148
|
+
keys: [taskKey, this.processingQueue, this.failedQueue],
|
|
149
|
+
arguments: [this.config.cleanupDelay.toString(), JSON.stringify(task), taskId],
|
|
150
|
+
});
|
|
151
|
+
console.error(`[RedisQueueConsumer] Task ${taskId} failed after ${task.maxRetries} retries`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
finally {
|
|
156
|
+
this.processingTasks--;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* 启动恢复机制(内部方法,自动调用)
|
|
161
|
+
*/
|
|
162
|
+
startRecovery() {
|
|
163
|
+
if (this.recoveryInterval || !this.redis) {
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
// 立即执行一次恢复
|
|
167
|
+
this.recoverStalled(this.config.processingTimeout).catch(error => {
|
|
168
|
+
console.error('[RedisQueueConsumer] Initial recovery error:', error);
|
|
169
|
+
});
|
|
170
|
+
// 定期检查(每 10 秒检查一次)
|
|
171
|
+
this.recoveryInterval = setInterval(() => {
|
|
172
|
+
this.recoverStalled(this.config.processingTimeout).catch(error => {
|
|
173
|
+
console.error('[RedisQueueConsumer] Recovery error:', error);
|
|
174
|
+
});
|
|
175
|
+
}, 10000);
|
|
176
|
+
console.log('[RedisQueueConsumer] Recovery mechanism started');
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* 停止恢复机制(内部方法,自动调用)
|
|
180
|
+
*/
|
|
181
|
+
stopRecovery() {
|
|
182
|
+
if (this.recoveryInterval) {
|
|
183
|
+
clearInterval(this.recoveryInterval);
|
|
184
|
+
this.recoveryInterval = null;
|
|
185
|
+
console.log('[RedisQueueConsumer] Recovery mechanism stopped');
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* 启动消费者(内部方法,自动调用)
|
|
190
|
+
*/
|
|
191
|
+
startConsumer() {
|
|
192
|
+
if (this.consumerRunning || !this.redis) {
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
this.consumerRunning = true;
|
|
196
|
+
console.log(`[RedisQueueConsumer] Consumer started with concurrency: ${this.config.concurrency}`);
|
|
197
|
+
// Lua 脚本: 原子化从 pending 取出任务并移到 processing(只取已到延迟时间的任务)
|
|
198
|
+
const popAndMoveScript = `
|
|
199
|
+
local pendingQueue = KEYS[1]
|
|
200
|
+
local processingQueue = KEYS[2]
|
|
201
|
+
local queueKeyPrefix = KEYS[3]
|
|
202
|
+
local count = tonumber(ARGV[1])
|
|
203
|
+
local currentTime = tonumber(ARGV[2])
|
|
204
|
+
|
|
205
|
+
local taskIds = {}
|
|
206
|
+
local checkedCount = 0
|
|
207
|
+
local maxCheck = count * 3 -- 最多检查 count*3 个任务,避免无限循环
|
|
208
|
+
|
|
209
|
+
while #taskIds < count and checkedCount < maxCheck do
|
|
210
|
+
local taskId = redis.call('LPOP', pendingQueue)
|
|
211
|
+
if not taskId then
|
|
212
|
+
break
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
checkedCount = checkedCount + 1
|
|
216
|
+
|
|
217
|
+
-- 获取任务详情检查延迟时间
|
|
218
|
+
local taskKey = queueKeyPrefix .. ':task:' .. taskId
|
|
219
|
+
local taskData = redis.call('GET', taskKey)
|
|
220
|
+
|
|
221
|
+
if taskData then
|
|
222
|
+
local task = cjson.decode(taskData)
|
|
223
|
+
local delayUntil = task.delayUntil
|
|
224
|
+
|
|
225
|
+
-- 如果没有延迟或已到延迟时间,则移到 processing
|
|
226
|
+
if not delayUntil or delayUntil <= currentTime then
|
|
227
|
+
redis.call('RPUSH', processingQueue, taskId)
|
|
228
|
+
table.insert(taskIds, taskId)
|
|
229
|
+
else
|
|
230
|
+
-- 未到延迟时间,放回队列尾部
|
|
231
|
+
redis.call('RPUSH', pendingQueue, taskId)
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
return taskIds
|
|
237
|
+
`;
|
|
238
|
+
this.consumerInterval = setInterval(async () => {
|
|
239
|
+
try {
|
|
240
|
+
// 检查当前是否有可用的并发槽位
|
|
241
|
+
const availableSlots = this.config.concurrency - this.processingTasks;
|
|
242
|
+
if (availableSlots <= 0) {
|
|
243
|
+
return; // 已达到并发限制,等待下一次轮询
|
|
244
|
+
}
|
|
245
|
+
// 使用 Lua 脚本原子化取出任务并移到 processing 队列
|
|
246
|
+
const taskIds = await this.redis.eval(popAndMoveScript, {
|
|
247
|
+
keys: [this.pendingQueue, this.processingQueue, this.config.queueKey],
|
|
248
|
+
arguments: [availableSlots.toString(), Date.now().toString()],
|
|
249
|
+
});
|
|
250
|
+
// 并发处理所有任务(不等待完成)
|
|
251
|
+
taskIds.forEach(taskId => {
|
|
252
|
+
this.processTask(taskId).catch(error => {
|
|
253
|
+
console.error(`[RedisQueueConsumer] Unhandled error in processTask: ${taskId}`, error);
|
|
254
|
+
});
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
catch (error) {
|
|
258
|
+
console.error('[RedisQueueConsumer] Consumer error:', error);
|
|
259
|
+
}
|
|
260
|
+
}, this.config.consumerInterval);
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* 停止消费者(内部方法,自动调用)
|
|
264
|
+
*/
|
|
265
|
+
stopConsumer() {
|
|
266
|
+
if (this.consumerInterval) {
|
|
267
|
+
clearInterval(this.consumerInterval);
|
|
268
|
+
this.consumerInterval = null;
|
|
269
|
+
}
|
|
270
|
+
this.consumerRunning = false;
|
|
271
|
+
console.log('[RedisQueueConsumer] Consumer stopped');
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* 获取队列统计信息(O(1) 时间复杂度)
|
|
275
|
+
*/
|
|
276
|
+
async statistics() {
|
|
277
|
+
if (!this.redis) {
|
|
278
|
+
return { pending: 0, processing: 0, completed: 0, failed: 0 };
|
|
279
|
+
}
|
|
280
|
+
const [pending, processing, completed, failed] = await Promise.all([
|
|
281
|
+
this.redis.lLen(this.pendingQueue),
|
|
282
|
+
this.redis.lLen(this.processingQueue),
|
|
283
|
+
this.redis.lLen(this.completedQueue),
|
|
284
|
+
this.redis.lLen(this.failedQueue),
|
|
285
|
+
]);
|
|
286
|
+
return { pending, processing, completed, failed };
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* 健康检查
|
|
290
|
+
*/
|
|
291
|
+
async health() {
|
|
292
|
+
if (!this.redis)
|
|
293
|
+
return false;
|
|
294
|
+
const result = await catchIt(() => this.redis.ping());
|
|
295
|
+
return !result.isError() && result.value === 'PONG';
|
|
296
|
+
}
|
|
297
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis-queue-consumer.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-consumer.test.ts"],"names":[],"mappings":"AAOA,OAAO,QAAQ,SAAS,CAAC;IACvB,UAAU,kBAAkB;QAC1B,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAClC;CACF"}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import { describe, it, expect, afterEach } from 'vitest';
|
|
2
|
+
import { catchIt } from '@taicode/common-base';
|
|
3
|
+
import { RedisQueueProvider } from './redis-queue-provider';
|
|
4
|
+
import { RedisQueueConsumer } from './redis-queue-consumer';
|
|
5
|
+
const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
|
|
6
|
+
describe('RedisQueueConsumer', () => {
|
|
7
|
+
const providers = [];
|
|
8
|
+
const consumers = [];
|
|
9
|
+
afterEach(async () => {
|
|
10
|
+
for (const consumer of consumers) {
|
|
11
|
+
consumer.disconnect();
|
|
12
|
+
}
|
|
13
|
+
for (const provider of providers) {
|
|
14
|
+
try {
|
|
15
|
+
await provider.clear();
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
// 忽略清理错误
|
|
19
|
+
}
|
|
20
|
+
provider.disconnect();
|
|
21
|
+
}
|
|
22
|
+
providers.length = 0;
|
|
23
|
+
consumers.length = 0;
|
|
24
|
+
});
|
|
25
|
+
const createQueue = (handler, options) => {
|
|
26
|
+
const uniqueKey = `test:queue:${Date.now()}:${Math.random()}`;
|
|
27
|
+
const provider = new RedisQueueProvider({
|
|
28
|
+
redisUrl: REDIS_URL,
|
|
29
|
+
queueKey: uniqueKey,
|
|
30
|
+
});
|
|
31
|
+
const consumer = new RedisQueueConsumer({
|
|
32
|
+
redisUrl: REDIS_URL,
|
|
33
|
+
queueKey: uniqueKey,
|
|
34
|
+
consumerInterval: 100,
|
|
35
|
+
maxRetries: 2,
|
|
36
|
+
...options,
|
|
37
|
+
handler,
|
|
38
|
+
});
|
|
39
|
+
providers.push(provider);
|
|
40
|
+
consumers.push(consumer);
|
|
41
|
+
return { provider, consumer };
|
|
42
|
+
};
|
|
43
|
+
describe('连接管理', () => {
|
|
44
|
+
it('应该成功连接到 Redis', async () => {
|
|
45
|
+
const { consumer } = createQueue(async () => catchIt(() => { }));
|
|
46
|
+
await consumer.connect();
|
|
47
|
+
const health = await consumer.health();
|
|
48
|
+
expect(health).toBe(true);
|
|
49
|
+
});
|
|
50
|
+
it('连接后应该自动启动消费者', async () => {
|
|
51
|
+
let processed = false;
|
|
52
|
+
const { provider, consumer } = createQueue(async () => {
|
|
53
|
+
processed = true;
|
|
54
|
+
return catchIt(() => { });
|
|
55
|
+
});
|
|
56
|
+
await provider.connect();
|
|
57
|
+
await consumer.connect();
|
|
58
|
+
await provider.enqueue({ test: true });
|
|
59
|
+
await new Promise(resolve => setTimeout(resolve, 300));
|
|
60
|
+
expect(processed).toBe(true);
|
|
61
|
+
});
|
|
62
|
+
});
|
|
63
|
+
describe('任务处理', () => {
|
|
64
|
+
it('应该成功处理任务', async () => {
|
|
65
|
+
const processedData = [];
|
|
66
|
+
const { provider, consumer } = createQueue(async (data) => {
|
|
67
|
+
processedData.push(data);
|
|
68
|
+
return catchIt(() => { });
|
|
69
|
+
});
|
|
70
|
+
await provider.connect();
|
|
71
|
+
await consumer.connect();
|
|
72
|
+
await provider.enqueue({ value: 'test' });
|
|
73
|
+
await new Promise(resolve => setTimeout(resolve, 500));
|
|
74
|
+
expect(processedData).toHaveLength(1);
|
|
75
|
+
expect(processedData[0]).toEqual({ value: 'test' });
|
|
76
|
+
const stats = await consumer.statistics();
|
|
77
|
+
expect(stats.completed).toBe(1);
|
|
78
|
+
});
|
|
79
|
+
it('应该按 FIFO 顺序处理任务', async () => {
|
|
80
|
+
const processOrder = [];
|
|
81
|
+
const { provider, consumer } = createQueue(async (data) => {
|
|
82
|
+
processOrder.push(data.order);
|
|
83
|
+
return catchIt(() => { });
|
|
84
|
+
});
|
|
85
|
+
await provider.connect();
|
|
86
|
+
await consumer.connect();
|
|
87
|
+
await provider.enqueue([{ order: 1 }, { order: 2 }, { order: 3 }]);
|
|
88
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
89
|
+
expect(processOrder).toEqual([1, 2, 3]);
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
describe('重试机制', () => {
|
|
93
|
+
it('失败的任务应该自动重试', async () => {
|
|
94
|
+
let attemptCount = 0;
|
|
95
|
+
const { provider, consumer } = createQueue(async () => {
|
|
96
|
+
attemptCount++;
|
|
97
|
+
throw new Error('Task failed');
|
|
98
|
+
}, { maxRetries: 2 });
|
|
99
|
+
await provider.connect();
|
|
100
|
+
await consumer.connect();
|
|
101
|
+
const taskId = await provider.enqueue({ test: 1 });
|
|
102
|
+
await new Promise(resolve => setTimeout(resolve, 3000));
|
|
103
|
+
// 最多 1 次初始 + 2 次重试 = 3 次,但允许恢复机制导致的额外尝试
|
|
104
|
+
expect(attemptCount).toBeGreaterThanOrEqual(3);
|
|
105
|
+
expect(attemptCount).toBeLessThanOrEqual(4);
|
|
106
|
+
const stats = await consumer.statistics();
|
|
107
|
+
expect(stats.failed).toBe(1);
|
|
108
|
+
});
|
|
109
|
+
});
|
|
110
|
+
describe('消费者管理', () => {
|
|
111
|
+
it('断开连接后消费者应该停止', async () => {
|
|
112
|
+
let processCount = 0;
|
|
113
|
+
const { provider, consumer } = createQueue(async () => {
|
|
114
|
+
processCount++;
|
|
115
|
+
return catchIt(() => { });
|
|
116
|
+
});
|
|
117
|
+
await provider.connect();
|
|
118
|
+
await consumer.connect();
|
|
119
|
+
await provider.enqueue({ test: 1 });
|
|
120
|
+
await new Promise(resolve => setTimeout(resolve, 300));
|
|
121
|
+
expect(processCount).toBe(1);
|
|
122
|
+
consumer.disconnect();
|
|
123
|
+
await provider.connect();
|
|
124
|
+
await provider.enqueue({ test: 2 });
|
|
125
|
+
await new Promise(resolve => setTimeout(resolve, 300));
|
|
126
|
+
expect(processCount).toBe(1); // 没有增加
|
|
127
|
+
});
|
|
128
|
+
});
|
|
129
|
+
describe('队列统计', () => {
|
|
130
|
+
it('应该正确返回队列统计信息', async () => {
|
|
131
|
+
const { provider, consumer } = createQueue(async () => {
|
|
132
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
133
|
+
return catchIt(() => { });
|
|
134
|
+
});
|
|
135
|
+
await provider.connect();
|
|
136
|
+
await consumer.connect();
|
|
137
|
+
await provider.enqueue([{ data: 1 }, { data: 2 }, { data: 3 }]);
|
|
138
|
+
await new Promise(resolve => setTimeout(resolve, 50));
|
|
139
|
+
const stats = await consumer.statistics();
|
|
140
|
+
expect(stats.pending + stats.processing).toBeGreaterThan(0);
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
describe('并发处理', () => {
|
|
144
|
+
it('应该支持并发处理多个任务', async () => {
|
|
145
|
+
const processing = [];
|
|
146
|
+
const completed = [];
|
|
147
|
+
const { provider, consumer } = createQueue(async (data) => {
|
|
148
|
+
processing.push(data.taskId);
|
|
149
|
+
await new Promise(resolve => setTimeout(resolve, 200));
|
|
150
|
+
completed.push(data.taskId);
|
|
151
|
+
return catchIt(() => { });
|
|
152
|
+
}, { concurrency: 3 });
|
|
153
|
+
await provider.connect();
|
|
154
|
+
await consumer.connect();
|
|
155
|
+
await provider.enqueue([
|
|
156
|
+
{ taskId: 1 }, { taskId: 2 }, { taskId: 3 }, { taskId: 4 }, { taskId: 5 },
|
|
157
|
+
]);
|
|
158
|
+
await new Promise(resolve => setTimeout(resolve, 250));
|
|
159
|
+
expect(processing.length).toBeGreaterThanOrEqual(2);
|
|
160
|
+
await new Promise(resolve => setTimeout(resolve, 1500));
|
|
161
|
+
expect(completed.length).toBe(5);
|
|
162
|
+
});
|
|
163
|
+
});
|
|
164
|
+
describe('幂等性', () => {
|
|
165
|
+
it('应该支持在 data 中指定 id 来实现幂等性', async () => {
|
|
166
|
+
let processCount = 0;
|
|
167
|
+
const { provider, consumer } = createQueue(async () => {
|
|
168
|
+
processCount++;
|
|
169
|
+
return catchIt(() => { });
|
|
170
|
+
});
|
|
171
|
+
await provider.connect();
|
|
172
|
+
await consumer.connect();
|
|
173
|
+
await provider.enqueue({ id: 'unique-1', value: 1 });
|
|
174
|
+
await provider.enqueue({ id: 'unique-1', value: 2 });
|
|
175
|
+
await provider.enqueue({ id: 'unique-2', value: 3 });
|
|
176
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
177
|
+
expect(processCount).toBe(2); // 只处理 unique-1 和 unique-2
|
|
178
|
+
});
|
|
179
|
+
});
|
|
180
|
+
describe('延迟处理', () => {
|
|
181
|
+
it('应该支持延迟处理任务', async () => {
|
|
182
|
+
const startTime = Date.now();
|
|
183
|
+
let processTime = 0;
|
|
184
|
+
const { provider, consumer } = createQueue(async () => {
|
|
185
|
+
processTime = Date.now() - startTime;
|
|
186
|
+
return catchIt(() => { });
|
|
187
|
+
});
|
|
188
|
+
await provider.connect();
|
|
189
|
+
await consumer.connect();
|
|
190
|
+
await provider.enqueue({ test: true });
|
|
191
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
192
|
+
expect(processTime).toBeGreaterThan(0);
|
|
193
|
+
});
|
|
194
|
+
});
|
|
195
|
+
describe('超时恢复', () => {
|
|
196
|
+
it('超时的任务应该直接标记为失败', { timeout: 15000 }, async () => {
|
|
197
|
+
let processCount = 0;
|
|
198
|
+
const { provider, consumer } = createQueue(async () => {
|
|
199
|
+
processCount++;
|
|
200
|
+
// 模拟处理卡住,永远不完成 (但超时机制会检测到)
|
|
201
|
+
await new Promise(resolve => setTimeout(resolve, 20000));
|
|
202
|
+
return catchIt(() => { });
|
|
203
|
+
}, {
|
|
204
|
+
processingTimeout: 500, // 500ms 超时
|
|
205
|
+
consumerInterval: 100,
|
|
206
|
+
});
|
|
207
|
+
await provider.connect();
|
|
208
|
+
await consumer.connect();
|
|
209
|
+
await provider.enqueue({ value: 1 });
|
|
210
|
+
// 等待任务被处理和超时恢复 (需要等待至少2个恢复周期: 10秒)
|
|
211
|
+
await new Promise(resolve => setTimeout(resolve, 12000));
|
|
212
|
+
const stats = await consumer.statistics();
|
|
213
|
+
// 超时后直接标记为失败,不应该重试
|
|
214
|
+
expect(stats.failed).toBe(1);
|
|
215
|
+
expect(stats.pending).toBe(0);
|
|
216
|
+
expect(stats.processing).toBe(0);
|
|
217
|
+
expect(stats.completed).toBe(0);
|
|
218
|
+
// 应该只处理一次,不会重试
|
|
219
|
+
expect(processCount).toBe(1);
|
|
220
|
+
});
|
|
221
|
+
it('未超时的失败任务应该正常重试', async () => {
|
|
222
|
+
let attemptCount = 0;
|
|
223
|
+
const { provider, consumer } = createQueue(async () => {
|
|
224
|
+
attemptCount++;
|
|
225
|
+
throw new Error('Task failed');
|
|
226
|
+
}, {
|
|
227
|
+
maxRetries: 3,
|
|
228
|
+
processingTimeout: 5000, // 5秒超时,足够长不会触发
|
|
229
|
+
consumerInterval: 100,
|
|
230
|
+
});
|
|
231
|
+
await provider.connect();
|
|
232
|
+
await consumer.connect();
|
|
233
|
+
await provider.enqueue({ value: 1 });
|
|
234
|
+
// 等待所有重试完成
|
|
235
|
+
await new Promise(resolve => setTimeout(resolve, 3000));
|
|
236
|
+
const stats = await consumer.statistics();
|
|
237
|
+
// 应该重试 3 次后失败
|
|
238
|
+
expect(stats.failed).toBe(1);
|
|
239
|
+
expect(attemptCount).toBe(4); // 初始 1 次 + 3 次重试
|
|
240
|
+
});
|
|
241
|
+
});
|
|
242
|
+
});
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { RedisQueueCommon } from './redis-queue-common';
|
|
2
|
+
import type { RedisQueueRegistry, RedisQueueProviderConfig, Task, QueueStats } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* Redis 队列生产者
|
|
5
|
+
*
|
|
6
|
+
* 只负责任务的入队和队列管理,不包含消费逻辑
|
|
7
|
+
*
|
|
8
|
+
* @template K 队列键类型
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```ts
|
|
12
|
+
* // 注册类型后使用
|
|
13
|
+
* const provider = new RedisQueueProvider({
|
|
14
|
+
* queueKey: 'email-queue', // 只能使用注册的键
|
|
15
|
+
* })
|
|
16
|
+
*
|
|
17
|
+
* await provider.enqueue({
|
|
18
|
+
* to: 'user@example.com', // 类型自动推断
|
|
19
|
+
* subject: 'Hello',
|
|
20
|
+
* body: 'World'
|
|
21
|
+
* })
|
|
22
|
+
* ```
|
|
23
|
+
*/
|
|
24
|
+
export declare class RedisQueueProvider<K extends keyof RedisQueueRegistry> extends RedisQueueCommon {
|
|
25
|
+
private readonly processingDelay;
|
|
26
|
+
constructor(config: RedisQueueProviderConfig<K>);
|
|
27
|
+
protected getLogPrefix(): string;
|
|
28
|
+
/**
|
|
29
|
+
* 将任务推入队列(支持单个或批量)
|
|
30
|
+
* @param data 任务数据(单个或数组)。可以在 data 中包含 `id` 字段来实现幂等性
|
|
31
|
+
*
|
|
32
|
+
* @example
|
|
33
|
+
* // 普通任务,自动生成 ID
|
|
34
|
+
* await provider.enqueue({ to: 'user@example.com' })
|
|
35
|
+
*
|
|
36
|
+
* // 幂等任务,手动指定 ID,重复提交会被忽略
|
|
37
|
+
* await provider.enqueue({ id: 'email-123', to: 'user@example.com' })
|
|
38
|
+
* await provider.enqueue({ id: 'email-123', to: 'user@example.com' }) // 会被跳过
|
|
39
|
+
*/
|
|
40
|
+
enqueue(data: RedisQueueRegistry[K]): Promise<string>;
|
|
41
|
+
enqueue(data: RedisQueueRegistry[K]): Promise<string[]>;
|
|
42
|
+
/**
|
|
43
|
+
* 获取任务详情
|
|
44
|
+
*/
|
|
45
|
+
getTask(taskId: string): Promise<Task<RedisQueueRegistry[K]> | null>;
|
|
46
|
+
/**
|
|
47
|
+
* 获取队列统计信息(O(1) 时间复杂度)
|
|
48
|
+
*/
|
|
49
|
+
statistics(): Promise<QueueStats>;
|
|
50
|
+
/**
|
|
51
|
+
* 清空所有队列和任务数据
|
|
52
|
+
* 使用 Lua 脚本确保原子性
|
|
53
|
+
*/
|
|
54
|
+
clear(): Promise<void>;
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=redis-queue-provider.d.ts.map
|