@taicode/common-server 1.0.12 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/output/index.d.ts +1 -0
- package/output/index.d.ts.map +1 -1
- package/output/index.js +1 -0
- package/output/redis-queue/index.d.ts +1 -0
- package/output/redis-queue/index.d.ts.map +1 -1
- package/output/redis-queue/redis-batch-consumer.d.ts +4 -9
- package/output/redis-queue/redis-batch-consumer.d.ts.map +1 -1
- package/output/redis-queue/redis-batch-consumer.js +3 -55
- package/output/redis-queue/redis-batch-consumer.test.d.ts +5 -0
- package/output/redis-queue/redis-batch-consumer.test.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-common.d.ts +15 -3
- package/output/redis-queue/redis-queue-common.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-common.js +80 -0
- package/output/redis-queue/redis-queue-common.test.d.ts +17 -0
- package/output/redis-queue/redis-queue-common.test.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-common.test.js +11 -11
- package/output/redis-queue/redis-queue-consumer.d.ts +6 -27
- package/output/redis-queue/redis-queue-consumer.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-consumer.js +23 -187
- package/output/redis-queue/redis-queue-consumer.test.d.ts +5 -0
- package/output/redis-queue/redis-queue-consumer.test.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-provider.d.ts +14 -15
- package/output/redis-queue/redis-queue-provider.d.ts.map +1 -1
- package/output/redis-queue/redis-queue-provider.js +8 -9
- package/output/redis-queue/redis-queue-provider.test.d.ts +5 -0
- package/output/redis-queue/redis-queue-provider.test.d.ts.map +1 -1
- package/output/redis-queue/types.d.ts +47 -9
- package/output/redis-queue/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/output/logger/logger.d.ts +0 -33
- package/output/logger/logger.d.ts.map +0 -1
- package/output/logger/logger.js +0 -65
- package/output/logger/logger.test.d.ts +0 -2
- package/output/logger/logger.test.d.ts.map +0 -1
- package/output/logger/logger.test.js +0 -87
- package/output/redis-queue/batch-consumer.d.ts +0 -107
- package/output/redis-queue/batch-consumer.d.ts.map +0 -1
- package/output/redis-queue/batch-consumer.js +0 -492
- package/output/redis-queue/batch-consumer.test.d.ts +0 -2
- package/output/redis-queue/batch-consumer.test.d.ts.map +0 -1
- package/output/redis-queue/batch-consumer.test.js +0 -216
- package/output/redis-queue/batch-redis-queue.d.ts +0 -136
- package/output/redis-queue/batch-redis-queue.d.ts.map +0 -1
- package/output/redis-queue/batch-redis-queue.js +0 -583
- package/output/redis-queue/batch-redis-queue.test.d.ts +0 -2
- package/output/redis-queue/batch-redis-queue.test.d.ts.map +0 -1
- package/output/redis-queue/batch-redis-queue.test.js +0 -243
- package/output/redis-queue/redis-queue.d.ts +0 -129
- package/output/redis-queue/redis-queue.d.ts.map +0 -1
- package/output/redis-queue/redis-queue.js +0 -557
- package/output/redis-queue/redis-queue.test.d.ts +0 -2
- package/output/redis-queue/redis-queue.test.d.ts.map +0 -1
- package/output/redis-queue/redis-queue.test.js +0 -234
- package/output/redis-queue/registry.d.ts +0 -57
- package/output/redis-queue/registry.d.ts.map +0 -1
- package/output/redis-queue/registry.js +0 -30
package/output/index.d.ts
CHANGED
package/output/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../source/index.ts"],"names":[],"mappings":"AACA,cAAc,UAAU,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../source/index.ts"],"names":[],"mappings":"AACA,cAAc,UAAU,CAAA;AACxB,cAAc,eAAe,CAAA"}
|
package/output/index.js
CHANGED
|
@@ -3,4 +3,5 @@ export { RedisQueueConsumer } from './redis-queue-consumer';
|
|
|
3
3
|
export { RedisBatchConsumer } from './redis-batch-consumer';
|
|
4
4
|
export type { RedisQueueProviderConfig, RedisQueueConsumerConfig, BatchConsumerConfig } from './types';
|
|
5
5
|
export type { TaskData } from './types';
|
|
6
|
+
export type { RedisQueueRegistry } from './types';
|
|
6
7
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../source/redis-queue/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAG3D,YAAY,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAA;AAGtG,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../source/redis-queue/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAG3D,YAAY,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAA;AAGtG,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAGvC,YAAY,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAA"}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { RedisQueueCommon } from './redis-queue-common';
|
|
2
|
-
import type { BatchConsumerConfig,
|
|
2
|
+
import type { BatchConsumerConfig, QueueStats, RedisQueueRegistry } from './types';
|
|
3
3
|
/**
|
|
4
4
|
* Redis 批量队列消费者
|
|
5
5
|
*
|
|
6
6
|
* 批量消费任务,每次处理多条
|
|
7
7
|
*
|
|
8
|
-
* @template
|
|
8
|
+
* @template K 队列键名类型
|
|
9
9
|
*
|
|
10
10
|
* @example
|
|
11
11
|
* ```ts
|
|
@@ -32,13 +32,13 @@ import type { BatchConsumerConfig, TaskData, QueueStats } from './types';
|
|
|
32
32
|
* consumer.disconnect()
|
|
33
33
|
* ```
|
|
34
34
|
*/
|
|
35
|
-
export declare class RedisBatchConsumer<
|
|
35
|
+
export declare class RedisBatchConsumer<K extends keyof RedisQueueRegistry> extends RedisQueueCommon {
|
|
36
36
|
private consumerRunning;
|
|
37
37
|
private consumerInterval;
|
|
38
38
|
private recoveryInterval;
|
|
39
39
|
private processingBatches;
|
|
40
40
|
private readonly config;
|
|
41
|
-
constructor(config: BatchConsumerConfig<
|
|
41
|
+
constructor(config: BatchConsumerConfig<K>);
|
|
42
42
|
protected getLogPrefix(): string;
|
|
43
43
|
/**
|
|
44
44
|
* 连接 Redis 并自动启动消费者
|
|
@@ -48,11 +48,6 @@ export declare class RedisBatchConsumer<T extends TaskData = TaskData> extends R
|
|
|
48
48
|
* 断开 Redis 连接并停止消费者
|
|
49
49
|
*/
|
|
50
50
|
disconnect(): void;
|
|
51
|
-
/**
|
|
52
|
-
* 恢复超时的任务
|
|
53
|
-
* 检查 processing 队列中的任务,将超时的任务直接标记为失败
|
|
54
|
-
*/
|
|
55
|
-
private recoverStalledTasks;
|
|
56
51
|
/**
|
|
57
52
|
* 批量处理任务
|
|
58
53
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redis-batch-consumer.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-batch-consumer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AACvD,OAAO,KAAK,EAAE,mBAAmB,
|
|
1
|
+
{"version":3,"file":"redis-batch-consumer.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-batch-consumer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AACvD,OAAO,KAAK,EAAE,mBAAmB,EAA0B,UAAU,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAA;AAE1G;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,kBAAkB,CAAC,CAAC,SAAS,MAAM,kBAAkB,CAAE,SAAQ,gBAAgB;IAC1F,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,iBAAiB,CAAI;IAE7B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAOtB;gBAEW,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;IA2B1C,SAAS,CAAC,YAAY,IAAI,MAAM;IAIhC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAMlB;;OAEG;YACW,YAAY;IA4F1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACH,OAAO,CAAC,aAAa;IAyErB;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC;IAevC;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAMjC"}
|
|
@@ -5,7 +5,7 @@ import { RedisQueueCommon } from './redis-queue-common';
|
|
|
5
5
|
*
|
|
6
6
|
* 批量消费任务,每次处理多条
|
|
7
7
|
*
|
|
8
|
-
* @template
|
|
8
|
+
* @template K 队列键名类型
|
|
9
9
|
*
|
|
10
10
|
* @example
|
|
11
11
|
* ```ts
|
|
@@ -82,58 +82,6 @@ export class RedisBatchConsumer extends RedisQueueCommon {
|
|
|
82
82
|
this.stopRecovery();
|
|
83
83
|
super.disconnect();
|
|
84
84
|
}
|
|
85
|
-
/**
|
|
86
|
-
* 恢复超时的任务
|
|
87
|
-
* 检查 processing 队列中的任务,将超时的任务直接标记为失败
|
|
88
|
-
*/
|
|
89
|
-
async recoverStalledTasks() {
|
|
90
|
-
if (!this.redis)
|
|
91
|
-
return;
|
|
92
|
-
try {
|
|
93
|
-
const processingTaskIds = await this.redis.lRange(this.processingQueue, 0, -1);
|
|
94
|
-
if (processingTaskIds.length === 0) {
|
|
95
|
-
return;
|
|
96
|
-
}
|
|
97
|
-
const now = Date.now();
|
|
98
|
-
let recoveredCount = 0;
|
|
99
|
-
for (const taskId of processingTaskIds) {
|
|
100
|
-
const task = await this.getTask(taskId);
|
|
101
|
-
if (!task) {
|
|
102
|
-
// 任务不存在,从队列中移除
|
|
103
|
-
await this.redis.lRem(this.processingQueue, 0, taskId);
|
|
104
|
-
continue;
|
|
105
|
-
}
|
|
106
|
-
// 检查是否超时
|
|
107
|
-
const processingTime = now - (task.processingStartTime || now);
|
|
108
|
-
if (processingTime > this.config.processingTimeout) {
|
|
109
|
-
console.log(`[RedisBatchConsumer] Task timeout: ${taskId} (processing time: ${processingTime}ms)`);
|
|
110
|
-
const taskKey = `${this.queueKey}:task:${taskId}`;
|
|
111
|
-
// 超时直接标记为失败
|
|
112
|
-
task.status = 'failed';
|
|
113
|
-
task.processingStartTime = undefined;
|
|
114
|
-
// 使用 Lua 脚本确保原子性
|
|
115
|
-
const script = `
|
|
116
|
-
redis.call('SETEX', KEYS[1], ARGV[1], ARGV[2])
|
|
117
|
-
redis.call('LREM', KEYS[2], 0, ARGV[3])
|
|
118
|
-
redis.call('RPUSH', KEYS[3], ARGV[3])
|
|
119
|
-
return 1
|
|
120
|
-
`;
|
|
121
|
-
await this.redis.eval(script, {
|
|
122
|
-
keys: [taskKey, this.processingQueue, this.failedQueue],
|
|
123
|
-
arguments: [this.cleanupDelay.toString(), JSON.stringify(task), taskId],
|
|
124
|
-
});
|
|
125
|
-
console.error(`[RedisBatchConsumer] Task ${taskId} failed after timeout`);
|
|
126
|
-
recoveredCount++;
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
if (recoveredCount > 0) {
|
|
130
|
-
console.log(`[RedisBatchConsumer] Recovered ${recoveredCount} timeout tasks`);
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
catch (error) {
|
|
134
|
-
console.error('[RedisBatchConsumer] Failed to recover stalled tasks:', error);
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
85
|
/**
|
|
138
86
|
* 批量处理任务
|
|
139
87
|
*/
|
|
@@ -230,12 +178,12 @@ export class RedisBatchConsumer extends RedisQueueCommon {
|
|
|
230
178
|
return;
|
|
231
179
|
}
|
|
232
180
|
// 立即执行一次恢复
|
|
233
|
-
this.
|
|
181
|
+
this.recoverStalled(this.config.processingTimeout).catch(error => {
|
|
234
182
|
console.error('[RedisBatchConsumer] Initial recovery error:', error);
|
|
235
183
|
});
|
|
236
184
|
// 定期检查(每 10 秒检查一次)
|
|
237
185
|
this.recoveryInterval = setInterval(() => {
|
|
238
|
-
this.
|
|
186
|
+
this.recoverStalled(this.config.processingTimeout).catch(error => {
|
|
239
187
|
console.error('[RedisBatchConsumer] Recovery error:', error);
|
|
240
188
|
});
|
|
241
189
|
}, 10000);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redis-batch-consumer.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-batch-consumer.test.ts"],"names":[],"mappings":""}
|
|
1
|
+
{"version":3,"file":"redis-batch-consumer.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-batch-consumer.test.ts"],"names":[],"mappings":"AAOA,OAAO,QAAQ,SAAS,CAAC;IACvB,UAAU,kBAAkB;QAC1B,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAClC;CACF"}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { RedisClientType } from 'redis';
|
|
2
|
-
import type { Status,
|
|
2
|
+
import type { Status, Task, RedisQueueCommonConfig } from './types';
|
|
3
3
|
/**
|
|
4
4
|
* Redis 队列基础类
|
|
5
5
|
* 封装 Provider 和 Consumer 的公共逻辑
|
|
6
6
|
*/
|
|
7
|
-
export declare abstract class RedisQueueCommon
|
|
7
|
+
export declare abstract class RedisQueueCommon {
|
|
8
8
|
protected redis: RedisClientType | null;
|
|
9
9
|
protected isExternalRedis: boolean;
|
|
10
10
|
protected readonly queueKey: string;
|
|
@@ -30,7 +30,7 @@ export declare abstract class RedisQueueCommon<T extends TaskData = TaskData> {
|
|
|
30
30
|
/**
|
|
31
31
|
* 获取任务详情
|
|
32
32
|
*/
|
|
33
|
-
protected getTask(taskId: string): Promise<Task
|
|
33
|
+
protected getTask(taskId: string): Promise<Task | null>;
|
|
34
34
|
/**
|
|
35
35
|
* 根据状态获取对应的队列键
|
|
36
36
|
*/
|
|
@@ -57,5 +57,17 @@ export declare abstract class RedisQueueCommon<T extends TaskData = TaskData> {
|
|
|
57
57
|
* 健康检查
|
|
58
58
|
*/
|
|
59
59
|
health(): Promise<boolean>;
|
|
60
|
+
/**
|
|
61
|
+
* 恢复超时的任务
|
|
62
|
+
* 检查 processing 队列中的任务,将超时的任务直接标记为失败
|
|
63
|
+
* 使用 Lua 脚本批量处理以提高性能和原子性
|
|
64
|
+
*
|
|
65
|
+
* @param processingTimeout 处理超时时间(毫秒)
|
|
66
|
+
* @returns 恢复的任务统计 {timeoutFailed: number, cleaned: number}
|
|
67
|
+
*/
|
|
68
|
+
protected recoverStalled(processingTimeout: number): Promise<{
|
|
69
|
+
timeoutFailed: number;
|
|
70
|
+
cleaned: number;
|
|
71
|
+
}>;
|
|
60
72
|
}
|
|
61
73
|
//# sourceMappingURL=redis-queue-common.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redis-queue-common.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,eAAe,EAAE,MAAM,OAAO,CAAA;AACrD,OAAO,KAAK,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"redis-queue-common.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,eAAe,EAAE,MAAM,OAAO,CAAA;AACrD,OAAO,KAAK,EAAE,MAAM,EAAY,IAAI,EAAE,sBAAsB,EAAE,MAAM,SAAS,CAAA;AAE7E;;;GAGG;AACH,8BAAsB,gBAAgB;IACpC,SAAS,CAAC,KAAK,EAAE,eAAe,GAAG,IAAI,CAAO;IAC9C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAQ;IAE1C,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACnC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACpC,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAGvC,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IACtC,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IACvC,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAC1C,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;gBAE7B,MAAM,EAAE,sBAAsB;IAwC1C;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,YAAY,IAAI,MAAM;IAEzC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAS9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAalB;;OAEG;cACa,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAW7D;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAUlD;;OAEG;cACa,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAqChG;;;OAGG;cACa,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAqD1G;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAevG;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAUhC;;;;;;;OAOG;cACa,cAAc,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CA4E/G"}
|
|
@@ -219,4 +219,84 @@ export class RedisQueueCommon {
|
|
|
219
219
|
return false;
|
|
220
220
|
}
|
|
221
221
|
}
|
|
222
|
+
/**
|
|
223
|
+
* 恢复超时的任务
|
|
224
|
+
* 检查 processing 队列中的任务,将超时的任务直接标记为失败
|
|
225
|
+
* 使用 Lua 脚本批量处理以提高性能和原子性
|
|
226
|
+
*
|
|
227
|
+
* @param processingTimeout 处理超时时间(毫秒)
|
|
228
|
+
* @returns 恢复的任务统计 {timeoutFailed: number, cleaned: number}
|
|
229
|
+
*/
|
|
230
|
+
async recoverStalled(processingTimeout) {
|
|
231
|
+
if (!this.redis || !this.redis.isOpen) {
|
|
232
|
+
return { timeoutFailed: 0, cleaned: 0 };
|
|
233
|
+
}
|
|
234
|
+
try {
|
|
235
|
+
const processingTaskIds = await this.redis.lRange(this.processingQueue, 0, -1);
|
|
236
|
+
if (processingTaskIds.length === 0) {
|
|
237
|
+
return { timeoutFailed: 0, cleaned: 0 };
|
|
238
|
+
}
|
|
239
|
+
// 使用 Lua 脚本批量恢复超时任务
|
|
240
|
+
const batchRecoveryScript = `
|
|
241
|
+
local processingQueue = KEYS[1]
|
|
242
|
+
local pendingQueue = KEYS[2]
|
|
243
|
+
local failedQueue = KEYS[3]
|
|
244
|
+
local queueKeyPrefix = ARGV[1]
|
|
245
|
+
local ttl = tonumber(ARGV[2])
|
|
246
|
+
local processingTimeout = tonumber(ARGV[3])
|
|
247
|
+
local now = tonumber(ARGV[4])
|
|
248
|
+
|
|
249
|
+
local retryCount = 0
|
|
250
|
+
local failedCount = 0
|
|
251
|
+
local cleanupCount = 0
|
|
252
|
+
|
|
253
|
+
-- ARGV[5], ARGV[6], ARGV[7]... 是 taskId
|
|
254
|
+
for i = 5, #ARGV do
|
|
255
|
+
local taskId = ARGV[i]
|
|
256
|
+
local taskKey = queueKeyPrefix .. ':task:' .. taskId
|
|
257
|
+
local taskData = redis.call('GET', taskKey)
|
|
258
|
+
|
|
259
|
+
if not taskData then
|
|
260
|
+
-- 任务不存在,从队列中清理
|
|
261
|
+
redis.call('LREM', processingQueue, 0, taskId)
|
|
262
|
+
cleanupCount = cleanupCount + 1
|
|
263
|
+
else
|
|
264
|
+
local task = cjson.decode(taskData)
|
|
265
|
+
local processingTime = now - (task.processingStartTime or now)
|
|
266
|
+
|
|
267
|
+
-- 检查是否超时
|
|
268
|
+
if processingTime > processingTimeout then
|
|
269
|
+
-- 超时直接标记为失败
|
|
270
|
+
task.status = 'failed'
|
|
271
|
+
task.processingStartTime = nil
|
|
272
|
+
|
|
273
|
+
redis.call('SETEX', taskKey, ttl, cjson.encode(task))
|
|
274
|
+
redis.call('LREM', processingQueue, 0, taskId)
|
|
275
|
+
redis.call('RPUSH', failedQueue, taskId)
|
|
276
|
+
failedCount = failedCount + 1
|
|
277
|
+
end
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
return {retryCount, failedCount, cleanupCount}
|
|
282
|
+
`;
|
|
283
|
+
const result = await this.redis.eval(batchRecoveryScript, {
|
|
284
|
+
keys: [this.processingQueue, this.pendingQueue, this.failedQueue],
|
|
285
|
+
arguments: [
|
|
286
|
+
this.queueKey,
|
|
287
|
+
this.cleanupDelay.toString(),
|
|
288
|
+
processingTimeout.toString(),
|
|
289
|
+
Date.now().toString(),
|
|
290
|
+
...processingTaskIds,
|
|
291
|
+
],
|
|
292
|
+
});
|
|
293
|
+
const [, failedCount, cleanupCount] = result;
|
|
294
|
+
return { timeoutFailed: failedCount, cleaned: cleanupCount };
|
|
295
|
+
}
|
|
296
|
+
catch (error) {
|
|
297
|
+
const logPrefix = this.getLogPrefix();
|
|
298
|
+
console.error(`[${logPrefix}] Failed to recover stalled tasks:`, error);
|
|
299
|
+
return { timeoutFailed: 0, cleaned: 0 };
|
|
300
|
+
}
|
|
301
|
+
}
|
|
222
302
|
}
|
|
@@ -1,2 +1,19 @@
|
|
|
1
|
+
declare module './types' {
|
|
2
|
+
interface RedisQueueRegistry {
|
|
3
|
+
'test-queue': Record<string, any>;
|
|
4
|
+
'test-queue-error': Record<string, any>;
|
|
5
|
+
'short': Record<string, any>;
|
|
6
|
+
'valid-key': Record<string, any>;
|
|
7
|
+
'test-default': Record<string, any>;
|
|
8
|
+
'test-custom': Record<string, any>;
|
|
9
|
+
'': Record<string, any>;
|
|
10
|
+
'my-queue': Record<string, any>;
|
|
11
|
+
'test-error': Record<string, any>;
|
|
12
|
+
'test-bad': Record<string, any>;
|
|
13
|
+
'queue-one': Record<string, any>;
|
|
14
|
+
'queue-two': Record<string, any>;
|
|
15
|
+
[key: `test:reconnect:${number}`]: Record<string, any>;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
1
18
|
export {};
|
|
2
19
|
//# sourceMappingURL=redis-queue-common.test.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redis-queue-common.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-common.test.ts"],"names":[],"mappings":""}
|
|
1
|
+
{"version":3,"file":"redis-queue-common.test.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-common.test.ts"],"names":[],"mappings":"AAOA,OAAO,QAAQ,SAAS,CAAC;IACvB,UAAU,kBAAkB;QAC1B,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QACjC,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QACvC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAC5B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAChC,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QACnC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAClC,EAAE,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QACvB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAC/B,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QACjC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAC/B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAChC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAChC,CAAC,GAAG,EAAE,kBAAkB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KACvD;CACF"}
|
|
@@ -8,26 +8,26 @@ class TestRedisQueueCommon extends RedisQueueCommon {
|
|
|
8
8
|
}
|
|
9
9
|
// 暴露 protected 方法用于测试
|
|
10
10
|
testGetTask(taskId) {
|
|
11
|
-
return this
|
|
11
|
+
return this.getTask(taskId);
|
|
12
12
|
}
|
|
13
13
|
testGetQueueByStatus(status) {
|
|
14
|
-
return this
|
|
14
|
+
return this.getQueueByStatus(status);
|
|
15
15
|
}
|
|
16
16
|
async testApplyStatus(taskId, oldStatus, newStatus) {
|
|
17
|
-
return this
|
|
17
|
+
return this.applyStatus(taskId, oldStatus, newStatus);
|
|
18
18
|
}
|
|
19
19
|
getRedisClient() {
|
|
20
|
-
return this
|
|
20
|
+
return this.redis;
|
|
21
21
|
}
|
|
22
22
|
getConfig() {
|
|
23
23
|
return {
|
|
24
|
-
queueKey: this
|
|
25
|
-
redisUrl: this
|
|
26
|
-
cleanupDelay: this
|
|
27
|
-
pendingQueue: this
|
|
28
|
-
processingQueue: this
|
|
29
|
-
completedQueue: this
|
|
30
|
-
failedQueue: this
|
|
24
|
+
queueKey: this.queueKey,
|
|
25
|
+
redisUrl: this.redisUrl,
|
|
26
|
+
cleanupDelay: this.cleanupDelay,
|
|
27
|
+
pendingQueue: this.pendingQueue,
|
|
28
|
+
processingQueue: this.processingQueue,
|
|
29
|
+
completedQueue: this.completedQueue,
|
|
30
|
+
failedQueue: this.failedQueue,
|
|
31
31
|
};
|
|
32
32
|
}
|
|
33
33
|
}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { RedisQueueCommon } from './redis-queue-common';
|
|
2
|
+
import type { RedisQueueConsumerConfig, RedisQueueRegistry, QueueStats } from './types';
|
|
2
3
|
/**
|
|
3
4
|
* Redis 队列消费者
|
|
4
5
|
*
|
|
5
6
|
* 只负责消费任务,不包含入队逻辑
|
|
6
7
|
*
|
|
7
|
-
* @template
|
|
8
|
+
* @template K 队列键名类型
|
|
8
9
|
*
|
|
9
10
|
* @example
|
|
10
11
|
* ```ts
|
|
@@ -32,18 +33,14 @@ import type { RedisQueueConsumerConfig, TaskData, QueueStats } from './types';
|
|
|
32
33
|
* consumer.disconnect()
|
|
33
34
|
* ```
|
|
34
35
|
*/
|
|
35
|
-
export declare class RedisQueueConsumer<
|
|
36
|
+
export declare class RedisQueueConsumer<K extends keyof RedisQueueRegistry> extends RedisQueueCommon {
|
|
36
37
|
private consumerRunning;
|
|
37
|
-
private redis;
|
|
38
38
|
private consumerInterval;
|
|
39
39
|
private recoveryInterval;
|
|
40
40
|
private processingTasks;
|
|
41
41
|
private readonly config;
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
private readonly processingQueue;
|
|
45
|
-
private readonly completedQueue;
|
|
46
|
-
constructor(config: RedisQueueConsumerConfig<string, T>);
|
|
42
|
+
constructor(config: RedisQueueConsumerConfig<K>);
|
|
43
|
+
protected getLogPrefix(): string;
|
|
47
44
|
/**
|
|
48
45
|
* 连接 Redis 并自动启动消费者
|
|
49
46
|
*/
|
|
@@ -52,24 +49,6 @@ export declare class RedisQueueConsumer<T extends TaskData = TaskData> {
|
|
|
52
49
|
* 断开 Redis 连接并停止消费者
|
|
53
50
|
*/
|
|
54
51
|
disconnect(): void;
|
|
55
|
-
/**
|
|
56
|
-
* 获取任务详情
|
|
57
|
-
*/
|
|
58
|
-
private getTask;
|
|
59
|
-
/**
|
|
60
|
-
* 更新任务状态并移动到对应队列(原子操作)
|
|
61
|
-
*/
|
|
62
|
-
private applyStatus;
|
|
63
|
-
/**
|
|
64
|
-
* 根据状态获取对应的队列键
|
|
65
|
-
*/
|
|
66
|
-
private getQueueByStatus;
|
|
67
|
-
/**
|
|
68
|
-
* 恢复超时的任务
|
|
69
|
-
* 检查 processing 队列中的任务,将超时的任务直接标记为失败
|
|
70
|
-
* 使用 Lua 脚本批量处理以提高性能和原子性
|
|
71
|
-
*/
|
|
72
|
-
private recoverStalledTasks;
|
|
73
52
|
/**
|
|
74
53
|
* 处理单个任务
|
|
75
54
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redis-queue-consumer.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-consumer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"redis-queue-consumer.d.ts","sourceRoot":"","sources":["../../source/redis-queue/redis-queue-consumer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AACvD,OAAO,KAAK,EAAE,wBAAwB,EAAe,kBAAkB,EAAgB,UAAU,EAAE,MAAM,SAAS,CAAA;AAElH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,qBAAa,kBAAkB,CAAC,CAAC,SAAS,MAAM,kBAAkB,CAAE,SAAQ,gBAAgB;IAC1F,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,gBAAgB,CAA8B;IACtD,OAAO,CAAC,eAAe,CAAI;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAUtB;gBAEW,MAAM,EAAE,wBAAwB,CAAC,CAAC,CAAC;IA2B/C,SAAS,CAAC,YAAY,IAAI,MAAM;IAIhC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;OAEG;IACH,UAAU,IAAI,IAAI;IAMlB;;OAEG;YACW,WAAW;IA4EzB;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACH,OAAO,CAAC,aAAa;IA4ErB;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC;IAevC;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAMjC"}
|