@zintrust/queue-redis 2.1.3 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -2
- package/dist/BullMQRedisQueue.d.ts +0 -2
- package/dist/BullMQRedisQueue.js +32 -57
- package/dist/QueueHttpGateway.d.ts +1 -1
- package/dist/QueueHttpGateway.js +56 -78
- package/dist/RedisRpcQueueDriver.d.ts +11 -0
- package/dist/RedisRpcQueueDriver.js +131 -0
- package/dist/build-manifest.json +24 -23
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/package.json +9 -3
- package/dist/RedisQueue.d.ts +0 -10
- package/dist/RedisQueue.js +0 -92
package/README.md
CHANGED
|
@@ -16,9 +16,16 @@ npm i @zintrust/queue-redis
|
|
|
16
16
|
import '@zintrust/queue-redis/register';
|
|
17
17
|
```
|
|
18
18
|
|
|
19
|
-
Then set `QUEUE_DRIVER=redis` and configure
|
|
19
|
+
Then set `QUEUE_DRIVER=redis` and configure your Redis connection.
|
|
20
20
|
|
|
21
|
-
For Cloudflare Workers
|
|
21
|
+
For Cloudflare Workers or any runtime without Redis TCP access, run Redis RPC from a Node.js backend and configure the Worker with both flags:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
USE_REDIS_PROXY=true
|
|
25
|
+
REDIS_RPC_URL=https://queues.example.com
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Start the backend with `zin redis-rpc` or `zin s redis-rpc`. See [`@zintrust/redis-rpc`](https://www.npmjs.com/package/@zintrust/redis-rpc).
|
|
22
29
|
|
|
23
30
|
## When to use
|
|
24
31
|
|
|
@@ -13,8 +13,6 @@ interface IBullMQRedisQueue extends IQueueDriver {
|
|
|
13
13
|
closeQueue(queueName: string): Promise<void>;
|
|
14
14
|
getQueueNames(): string[];
|
|
15
15
|
}
|
|
16
|
-
export declare const shouldUseHttpProxyDriver: () => boolean;
|
|
17
|
-
export declare const runWithDirectQueueDriver: <T>(fn: () => Promise<T>) => Promise<T>;
|
|
18
16
|
/**
|
|
19
17
|
* BullMQ Redis Queue Driver
|
|
20
18
|
*
|
package/dist/BullMQRedisQueue.js
CHANGED
|
@@ -1,29 +1,13 @@
|
|
|
1
1
|
import { Cloudflare } from '@zintrust/core/cloudflare';
|
|
2
2
|
import { Env, queueConfig } from '@zintrust/core/config';
|
|
3
3
|
import { ErrorFactory } from '@zintrust/core/errors';
|
|
4
|
+
import { isNullish, isUndefinedOrNull } from '@zintrust/core/helper';
|
|
4
5
|
import { Logger } from '@zintrust/core/logger';
|
|
5
6
|
import { createLockProvider, getLockProvider, registerLockProvider, resolveDeduplicationLockKey, resolveLockPrefix, } from '@zintrust/core/queue';
|
|
6
7
|
import { createRedisConnection, getBullMQSafeQueueName } from '@zintrust/core/redis';
|
|
7
8
|
import { generateUuid, ZintrustLang } from '@zintrust/core/utils';
|
|
8
9
|
import { Queue } from 'bullmq';
|
|
9
|
-
import {
|
|
10
|
-
export const shouldUseHttpProxyDriver = () => {
|
|
11
|
-
if (directModeDepth > 0)
|
|
12
|
-
return false;
|
|
13
|
-
const isCloudFlareWorkers = Cloudflare.getWorkersEnv() !== null;
|
|
14
|
-
const isProxy = isCloudFlareWorkers || Env.getBool('QUEUE_HTTP_PROXY_ENABLED', false);
|
|
15
|
-
return isProxy;
|
|
16
|
-
};
|
|
17
|
-
let directModeDepth = 0;
|
|
18
|
-
export const runWithDirectQueueDriver = async (fn) => {
|
|
19
|
-
directModeDepth += 1;
|
|
20
|
-
try {
|
|
21
|
-
return await fn();
|
|
22
|
-
}
|
|
23
|
-
finally {
|
|
24
|
-
directModeDepth = Math.max(0, directModeDepth - 1);
|
|
25
|
-
}
|
|
26
|
-
};
|
|
10
|
+
import { RedisRpcQueueDriver, shouldUseRedisRpcQueueDriver } from './RedisRpcQueueDriver.js';
|
|
27
11
|
/**
|
|
28
12
|
* BullMQ Redis Queue Driver
|
|
29
13
|
*
|
|
@@ -36,44 +20,38 @@ export const BullMQRedisQueue = (() => {
|
|
|
36
20
|
let lockProviderCache = null;
|
|
37
21
|
const PULL_WORKER_TOKEN = 'pull-worker';
|
|
38
22
|
const SHARED_CONNECTION_SHUTDOWN_TIMEOUT_MS = 100;
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
23
|
+
const resolveQueueRedisConfig = () => {
|
|
24
|
+
let workersHost = Cloudflare.getWorkersVar('WORKERS_REDIS_HOST');
|
|
25
|
+
let workersPortRaw = Cloudflare.getWorkersVar('WORKERS_REDIS_PORT');
|
|
26
|
+
let workersPassword = Cloudflare.getWorkersVar('WORKERS_REDIS_PASSWORD');
|
|
27
|
+
let workersDbRaw = Cloudflare.getWorkersVar('WORKERS_REDIS_QUEUE_DB');
|
|
28
|
+
if (isUndefinedOrNull(workersPassword) || isNullish(workersPassword)) {
|
|
29
|
+
workersPassword = Env.get('REDIS_PASSWORD', '');
|
|
45
30
|
}
|
|
46
|
-
if (
|
|
47
|
-
|
|
31
|
+
if (isUndefinedOrNull(workersPortRaw) || isNullish(workersPortRaw)) {
|
|
32
|
+
workersPortRaw = Env.get('REDIS_PORT', '6379');
|
|
33
|
+
}
|
|
34
|
+
if (isUndefinedOrNull(workersHost) || isNullish(workersHost)) {
|
|
35
|
+
workersHost = Env.get('REDIS_HOST', '127.0.0.1');
|
|
36
|
+
}
|
|
37
|
+
if (isUndefinedOrNull(workersDbRaw) || isNullish(workersDbRaw)) {
|
|
38
|
+
workersDbRaw = Env.get('REDIS_QUEUE_DB', '0');
|
|
48
39
|
}
|
|
49
|
-
};
|
|
50
|
-
const resolveQueueRedisConfig = () => {
|
|
51
|
-
const workersHost = Cloudflare.getWorkersVar('WORKERS_REDIS_HOST');
|
|
52
|
-
const workersPortRaw = Cloudflare.getWorkersVar('WORKERS_REDIS_PORT');
|
|
53
|
-
const workersPassword = Cloudflare.getWorkersVar('WORKERS_REDIS_PASSWORD');
|
|
54
|
-
const workersDbRaw = Cloudflare.getWorkersVar('WORKERS_REDIS_QUEUE_DB');
|
|
55
40
|
return {
|
|
56
|
-
host: workersHost
|
|
57
|
-
port:
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
password: workersPassword !== null && workersPassword.trim() !== ''
|
|
61
|
-
? workersPassword
|
|
62
|
-
: Env.REDIS_PASSWORD,
|
|
63
|
-
database: workersDbRaw !== null && Number.isFinite(Number.parseInt(workersDbRaw, 10))
|
|
64
|
-
? Number.parseInt(workersDbRaw, 10)
|
|
65
|
-
: Env.getInt('REDIS_QUEUE_DB', 0),
|
|
41
|
+
host: workersHost,
|
|
42
|
+
port: Number(workersPortRaw),
|
|
43
|
+
password: workersPassword,
|
|
44
|
+
database: Number(workersDbRaw),
|
|
66
45
|
};
|
|
67
46
|
};
|
|
68
47
|
const assertWorkersHostIsReachable = (isWorkersRuntime, redisConfig) => {
|
|
69
48
|
if (isWorkersRuntime &&
|
|
70
49
|
(redisConfig.host === 'localhost' || redisConfig.host === '127.0.0.1')) {
|
|
71
|
-
throw ErrorFactory.createConfigError('Redis host cannot be localhost in Cloudflare Workers. Use a public Redis host
|
|
50
|
+
throw ErrorFactory.createConfigError('Redis host cannot be localhost in Cloudflare Workers. Use a public Redis host.');
|
|
72
51
|
}
|
|
73
52
|
};
|
|
74
53
|
const createSharedBullMqConnection = () => {
|
|
75
54
|
const isWorkersRuntime = Cloudflare.getWorkersEnv() !== null;
|
|
76
|
-
assertProxyAndWorkersCompatibility(isWorkersRuntime);
|
|
77
55
|
const redisConfig = resolveQueueRedisConfig();
|
|
78
56
|
assertWorkersHostIsReachable(isWorkersRuntime, redisConfig);
|
|
79
57
|
return createRedisConnection({
|
|
@@ -201,9 +179,6 @@ export const BullMQRedisQueue = (() => {
|
|
|
201
179
|
}
|
|
202
180
|
};
|
|
203
181
|
const getQueue = (queueName) => {
|
|
204
|
-
if (shouldUseHttpProxyDriver()) {
|
|
205
|
-
throw ErrorFactory.createConfigError('BullMQ queue instance is not available when QUEUE_HTTP_PROXY mode is active.');
|
|
206
|
-
}
|
|
207
182
|
// Check if queue exists in cache
|
|
208
183
|
if (queues.has(queueName)) {
|
|
209
184
|
const existingQueue = queues.get(queueName);
|
|
@@ -405,8 +380,8 @@ export const BullMQRedisQueue = (() => {
|
|
|
405
380
|
closeQueue,
|
|
406
381
|
getQueueNames,
|
|
407
382
|
async enqueue(queue, payload) {
|
|
408
|
-
if (
|
|
409
|
-
return
|
|
383
|
+
if (shouldUseRedisRpcQueueDriver()) {
|
|
384
|
+
return RedisRpcQueueDriver.enqueue(queue, payload);
|
|
410
385
|
}
|
|
411
386
|
let requestedJobId;
|
|
412
387
|
try {
|
|
@@ -443,8 +418,8 @@ export const BullMQRedisQueue = (() => {
|
|
|
443
418
|
}
|
|
444
419
|
},
|
|
445
420
|
async dequeue(queue) {
|
|
446
|
-
if (
|
|
447
|
-
return
|
|
421
|
+
if (shouldUseRedisRpcQueueDriver()) {
|
|
422
|
+
return RedisRpcQueueDriver.dequeue(queue);
|
|
448
423
|
}
|
|
449
424
|
try {
|
|
450
425
|
const q = getQueue(queue);
|
|
@@ -474,8 +449,8 @@ export const BullMQRedisQueue = (() => {
|
|
|
474
449
|
}
|
|
475
450
|
},
|
|
476
451
|
async ack(queue, id) {
|
|
477
|
-
if (
|
|
478
|
-
await
|
|
452
|
+
if (shouldUseRedisRpcQueueDriver()) {
|
|
453
|
+
await RedisRpcQueueDriver.ack(queue, id);
|
|
479
454
|
return;
|
|
480
455
|
}
|
|
481
456
|
try {
|
|
@@ -494,8 +469,8 @@ export const BullMQRedisQueue = (() => {
|
|
|
494
469
|
}
|
|
495
470
|
},
|
|
496
471
|
async length(queue) {
|
|
497
|
-
if (
|
|
498
|
-
return
|
|
472
|
+
if (shouldUseRedisRpcQueueDriver()) {
|
|
473
|
+
return RedisRpcQueueDriver.length(queue);
|
|
499
474
|
}
|
|
500
475
|
try {
|
|
501
476
|
const q = getQueue(queue);
|
|
@@ -508,8 +483,8 @@ export const BullMQRedisQueue = (() => {
|
|
|
508
483
|
}
|
|
509
484
|
},
|
|
510
485
|
async drain(queue) {
|
|
511
|
-
if (
|
|
512
|
-
await
|
|
486
|
+
if (shouldUseRedisRpcQueueDriver()) {
|
|
487
|
+
await RedisRpcQueueDriver.drain(queue);
|
|
513
488
|
return;
|
|
514
489
|
}
|
|
515
490
|
try {
|
package/dist/QueueHttpGateway.js
CHANGED
|
@@ -3,7 +3,7 @@ import { ErrorFactory } from '@zintrust/core/errors';
|
|
|
3
3
|
import { Router } from '@zintrust/core/http';
|
|
4
4
|
import { Logger } from '@zintrust/core/logger';
|
|
5
5
|
import { SignedRequest } from '@zintrust/core/security';
|
|
6
|
-
import BullMQRedisQueue
|
|
6
|
+
import BullMQRedisQueue from './BullMQRedisQueue.js';
|
|
7
7
|
const nonces = new Map();
|
|
8
8
|
const nowMs = () => Date.now();
|
|
9
9
|
const normalizePath = (value) => {
|
|
@@ -31,9 +31,8 @@ const readSettings = () => {
|
|
|
31
31
|
const cleanupExpiredNonces = () => {
|
|
32
32
|
const current = nowMs();
|
|
33
33
|
for (const [nonceKey, expiresAt] of nonces.entries()) {
|
|
34
|
-
if (expiresAt <= current)
|
|
34
|
+
if (expiresAt <= current)
|
|
35
35
|
nonces.delete(nonceKey);
|
|
36
|
-
}
|
|
37
36
|
}
|
|
38
37
|
};
|
|
39
38
|
const storeNonce = async (keyId, nonce, ttlMs) => {
|
|
@@ -52,18 +51,14 @@ const getBodyRecord = (req) => {
|
|
|
52
51
|
return {};
|
|
53
52
|
};
|
|
54
53
|
const getRawBody = (req) => {
|
|
55
|
-
const rawText = req.context['rawBodyText'];
|
|
54
|
+
const rawText = req.context?.['rawBodyText'];
|
|
56
55
|
if (typeof rawText === 'string')
|
|
57
56
|
return rawText;
|
|
58
57
|
return JSON.stringify(getBodyRecord(req));
|
|
59
58
|
};
|
|
60
59
|
const toIncomingHeaders = (req) => {
|
|
61
60
|
const headers = req.getHeaders();
|
|
62
|
-
const normalize = (value) =>
|
|
63
|
-
if (Array.isArray(value))
|
|
64
|
-
return value.join(',');
|
|
65
|
-
return value;
|
|
66
|
-
};
|
|
61
|
+
const normalize = (value) => Array.isArray(value) ? value.join(',') : value;
|
|
67
62
|
return {
|
|
68
63
|
'x-zt-key-id': normalize(headers['x-zt-key-id']),
|
|
69
64
|
'x-zt-timestamp': normalize(headers['x-zt-timestamp']),
|
|
@@ -73,63 +68,51 @@ const toIncomingHeaders = (req) => {
|
|
|
73
68
|
};
|
|
74
69
|
};
|
|
75
70
|
const sendFailure = (res, requestId, status, code, message, details) => {
|
|
76
|
-
|
|
71
|
+
res.status(status).json({
|
|
77
72
|
ok: false,
|
|
78
73
|
requestId,
|
|
79
74
|
result: null,
|
|
80
75
|
error: { code, message, details },
|
|
81
|
-
};
|
|
82
|
-
res.status(status).json(payload);
|
|
76
|
+
});
|
|
83
77
|
};
|
|
84
78
|
const sendSuccess = (res, requestId, result) => {
|
|
85
|
-
|
|
86
|
-
ok: true,
|
|
87
|
-
requestId,
|
|
88
|
-
result,
|
|
89
|
-
error: null,
|
|
90
|
-
};
|
|
91
|
-
res.status(200).json(payload);
|
|
79
|
+
res.status(200).json({ ok: true, requestId, result, error: null });
|
|
92
80
|
};
|
|
93
81
|
const readQueueName = (payload) => {
|
|
94
|
-
const value = payload
|
|
95
|
-
if (typeof value !== 'string')
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
return
|
|
82
|
+
const value = payload['queue'];
|
|
83
|
+
if (typeof value !== 'string' || value.trim() === '') {
|
|
84
|
+
throw ErrorFactory.createValidationError('payload.queue is required');
|
|
85
|
+
}
|
|
86
|
+
return value.trim();
|
|
99
87
|
};
|
|
100
88
|
const executeAction = async (request) => {
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
case 'enqueue': {
|
|
108
|
-
const payload = request.payload.payload;
|
|
109
|
-
if (!payload || typeof payload !== 'object') {
|
|
110
|
-
throw ErrorFactory.createValidationError('payload.payload is required for enqueue');
|
|
111
|
-
}
|
|
112
|
-
return BullMQRedisQueue.enqueue(queueName, payload);
|
|
89
|
+
const queueName = readQueueName(request.payload);
|
|
90
|
+
switch (request.action) {
|
|
91
|
+
case 'enqueue': {
|
|
92
|
+
const payload = request.payload['payload'];
|
|
93
|
+
if (!payload || typeof payload !== 'object') {
|
|
94
|
+
throw ErrorFactory.createValidationError('payload.payload is required for enqueue');
|
|
113
95
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
return null;
|
|
96
|
+
return BullMQRedisQueue.enqueue(queueName, payload);
|
|
97
|
+
}
|
|
98
|
+
case 'dequeue':
|
|
99
|
+
return BullMQRedisQueue.dequeue(queueName);
|
|
100
|
+
case 'ack': {
|
|
101
|
+
const id = request.payload['id'];
|
|
102
|
+
if (typeof id !== 'string' || id.trim() === '') {
|
|
103
|
+
throw ErrorFactory.createValidationError('payload.id is required for ack');
|
|
123
104
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
case 'drain':
|
|
127
|
-
await BullMQRedisQueue.drain(queueName);
|
|
128
|
-
return null;
|
|
129
|
-
default:
|
|
130
|
-
throw ErrorFactory.createValidationError(`Unsupported action: ${String(request.action)}`);
|
|
105
|
+
await BullMQRedisQueue.ack(queueName, id);
|
|
106
|
+
return null;
|
|
131
107
|
}
|
|
132
|
-
|
|
108
|
+
case 'length':
|
|
109
|
+
return BullMQRedisQueue.length(queueName);
|
|
110
|
+
case 'drain':
|
|
111
|
+
await BullMQRedisQueue.drain(queueName);
|
|
112
|
+
return null;
|
|
113
|
+
default:
|
|
114
|
+
throw ErrorFactory.createValidationError(`Unsupported action: ${String(request.action)}`);
|
|
115
|
+
}
|
|
133
116
|
};
|
|
134
117
|
const verifyRequest = async (req, bodyText, settings) => {
|
|
135
118
|
if (settings.keyId.trim() === '' || settings.secret.trim() === '') {
|
|
@@ -140,30 +123,25 @@ const verifyRequest = async (req, bodyText, settings) => {
|
|
|
140
123
|
message: 'Queue HTTP gateway signing credentials are not configured',
|
|
141
124
|
};
|
|
142
125
|
}
|
|
143
|
-
const url = new URL(req.getPath(), 'http://localhost');
|
|
144
126
|
const verifyResult = await SignedRequest.verify({
|
|
145
127
|
method: req.getMethod(),
|
|
146
|
-
url,
|
|
128
|
+
url: new URL(req.getPath(), 'http://localhost'),
|
|
147
129
|
body: bodyText,
|
|
148
130
|
headers: toIncomingHeaders(req),
|
|
149
131
|
nowMs: nowMs(),
|
|
150
132
|
windowMs: settings.signingWindowMs,
|
|
151
133
|
verifyNonce: async (keyId, nonce) => storeNonce(keyId, nonce, settings.nonceTtlMs),
|
|
152
|
-
getSecretForKeyId: async (keyId) =>
|
|
153
|
-
if (keyId === settings.keyId)
|
|
154
|
-
return settings.secret;
|
|
155
|
-
return undefined;
|
|
156
|
-
},
|
|
134
|
+
getSecretForKeyId: async (keyId) => (keyId === settings.keyId ? settings.secret : undefined),
|
|
157
135
|
});
|
|
158
136
|
if (verifyResult.ok === true)
|
|
159
137
|
return { ok: true };
|
|
160
|
-
const
|
|
161
|
-
const
|
|
138
|
+
const code = 'code' in verifyResult ? verifyResult.code : 'INVALID_SIGNATURE';
|
|
139
|
+
const message = 'message' in verifyResult ? verifyResult.message : 'Invalid signature';
|
|
162
140
|
return {
|
|
163
141
|
ok: false,
|
|
164
|
-
code
|
|
165
|
-
status:
|
|
166
|
-
message
|
|
142
|
+
code,
|
|
143
|
+
status: code === 'EXPIRED' || code === 'REPLAYED' ? 401 : 403,
|
|
144
|
+
message,
|
|
167
145
|
};
|
|
168
146
|
};
|
|
169
147
|
const createHandler = (settings) => {
|
|
@@ -178,37 +156,37 @@ const createHandler = (settings) => {
|
|
|
178
156
|
sendFailure(res, requestId, auth.status, auth.code, auth.message);
|
|
179
157
|
return;
|
|
180
158
|
}
|
|
181
|
-
|
|
182
|
-
const payload = body['payload'];
|
|
183
|
-
if (typeof action !== 'string') {
|
|
159
|
+
if (typeof body['action'] !== 'string') {
|
|
184
160
|
sendFailure(res, requestId, 400, 'VALIDATION_ERROR', 'action is required');
|
|
185
161
|
return;
|
|
186
162
|
}
|
|
187
|
-
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
|
|
163
|
+
if (!body['payload'] || typeof body['payload'] !== 'object' || Array.isArray(body['payload'])) {
|
|
188
164
|
sendFailure(res, requestId, 400, 'VALIDATION_ERROR', 'payload must be an object');
|
|
189
165
|
return;
|
|
190
166
|
}
|
|
191
|
-
const normalizedRequest = {
|
|
192
|
-
action: action,
|
|
193
|
-
requestId,
|
|
194
|
-
payload: payload,
|
|
195
|
-
};
|
|
196
167
|
try {
|
|
197
|
-
const result = await executeAction(
|
|
168
|
+
const result = await executeAction({
|
|
169
|
+
action: body['action'],
|
|
170
|
+
requestId,
|
|
171
|
+
payload: body['payload'],
|
|
172
|
+
});
|
|
198
173
|
sendSuccess(res, requestId, result);
|
|
199
174
|
}
|
|
200
175
|
catch (error) {
|
|
201
176
|
Logger.error('Queue HTTP gateway action failed', error);
|
|
202
|
-
sendFailure(res, requestId, 500, 'QUEUE_ERROR', 'Queue operation failed',
|
|
177
|
+
sendFailure(res, requestId, 500, 'QUEUE_ERROR', 'Queue operation failed', {
|
|
178
|
+
message: error instanceof Error ? error.message : String(error),
|
|
179
|
+
});
|
|
203
180
|
}
|
|
204
181
|
};
|
|
205
182
|
};
|
|
206
183
|
export const QueueHttpGateway = Object.freeze({
|
|
207
184
|
create(config) {
|
|
185
|
+
const defaults = readSettings();
|
|
208
186
|
const settings = {
|
|
209
|
-
...
|
|
187
|
+
...defaults,
|
|
210
188
|
...config,
|
|
211
|
-
basePath: normalizePath(config?.basePath ??
|
|
189
|
+
basePath: normalizePath(config?.basePath ?? defaults.basePath),
|
|
212
190
|
};
|
|
213
191
|
const routeOptions = settings.middleware.length > 0 ? { middleware: settings.middleware } : undefined;
|
|
214
192
|
return {
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { BullMQPayload, QueueMessage } from '@zintrust/core/queue';
|
|
2
|
+
export interface IQueueDriver {
|
|
3
|
+
enqueue(queue: string, payload: BullMQPayload): Promise<string>;
|
|
4
|
+
dequeue<T = unknown>(queue: string): Promise<QueueMessage<T> | undefined>;
|
|
5
|
+
ack(queue: string, id: string): Promise<void>;
|
|
6
|
+
length(queue: string): Promise<number>;
|
|
7
|
+
drain(queue: string): Promise<void>;
|
|
8
|
+
}
|
|
9
|
+
export declare const shouldUseRedisRpcQueueDriver: () => boolean;
|
|
10
|
+
export declare const RedisRpcQueueDriver: IQueueDriver;
|
|
11
|
+
export default RedisRpcQueueDriver;
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { Env } from '@zintrust/core/config';
|
|
2
|
+
import { ErrorFactory } from '@zintrust/core/errors';
|
|
3
|
+
import { JobStateTracker, TimeoutManager } from '@zintrust/core/queue';
|
|
4
|
+
import { generateUuid } from '@zintrust/core/utils';
|
|
5
|
+
const REDIS_RPC_PACKAGE = '@zintrust/redis-rpc';
|
|
6
|
+
export const shouldUseRedisRpcQueueDriver = () => {
|
|
7
|
+
return Env.USE_REDIS_PROXY === true && Env.get('REDIS_RPC_URL', '').trim() !== '';
|
|
8
|
+
};
|
|
9
|
+
const resolveRpcBaseUrl = () => {
|
|
10
|
+
const configured = Env.get('REDIS_RPC_URL', '').trim();
|
|
11
|
+
if (configured.length > 0)
|
|
12
|
+
return configured;
|
|
13
|
+
const host = Env.get('REDIS_RPC_HOST', '127.0.0.1').trim() || '127.0.0.1';
|
|
14
|
+
const port = Env.getInt('REDIS_RPC_PORT', 8794);
|
|
15
|
+
return `http://${host}:${port}`;
|
|
16
|
+
};
|
|
17
|
+
const createRpcClient = async () => {
|
|
18
|
+
try {
|
|
19
|
+
const mod = (await import(REDIS_RPC_PACKAGE));
|
|
20
|
+
return mod.createRedisRpcClient({
|
|
21
|
+
baseUrl: resolveRpcBaseUrl(),
|
|
22
|
+
secret: Env.get('REDIS_RPC_SECRET', Env.get('REDIS_PROXY_SECRET', Env.APP_KEY)),
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
throw ErrorFactory.createConfigError('@zintrust/redis-rpc is required when USE_REDIS_PROXY=true and REDIS_RPC_URL is configured', error);
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
const resolveRequestedJobId = (payloadData) => {
|
|
30
|
+
if (typeof payloadData?.jobId === 'string' && payloadData.jobId.trim().length > 0) {
|
|
31
|
+
return payloadData.jobId.trim();
|
|
32
|
+
}
|
|
33
|
+
return generateUuid();
|
|
34
|
+
};
|
|
35
|
+
const createJobOptions = (payloadData) => ({
|
|
36
|
+
jobId: resolveRequestedJobId(payloadData),
|
|
37
|
+
delay: payloadData.delay,
|
|
38
|
+
attempts: payloadData.attempts,
|
|
39
|
+
priority: payloadData.priority,
|
|
40
|
+
removeOnComplete: payloadData.removeOnComplete || 100,
|
|
41
|
+
removeOnFail: payloadData.removeOnFail || 50,
|
|
42
|
+
backoff: payloadData.backoff || {
|
|
43
|
+
type: 'exponential',
|
|
44
|
+
delay: 2000,
|
|
45
|
+
},
|
|
46
|
+
repeat: payloadData.repeat,
|
|
47
|
+
lifo: payloadData.lifo ?? false,
|
|
48
|
+
});
|
|
49
|
+
const resolveJobId = (result, fallback) => {
|
|
50
|
+
if (typeof result === 'object' && result !== null) {
|
|
51
|
+
const id = result.id;
|
|
52
|
+
if (typeof id === 'string' && id.trim().length > 0)
|
|
53
|
+
return id;
|
|
54
|
+
if (typeof id === 'number' && Number.isFinite(id))
|
|
55
|
+
return String(id);
|
|
56
|
+
}
|
|
57
|
+
return fallback;
|
|
58
|
+
};
|
|
59
|
+
const markPendingRecoveryFallback = async (input) => {
|
|
60
|
+
const payload = input.payload;
|
|
61
|
+
const currentAttempts = typeof payload['_currentAttempts'] === 'number' && Number.isFinite(payload['_currentAttempts'])
|
|
62
|
+
? Math.max(0, Math.floor(payload['_currentAttempts']))
|
|
63
|
+
: 0;
|
|
64
|
+
const maxAttempts = typeof payload['attempts'] === 'number' && Number.isFinite(payload['attempts'])
|
|
65
|
+
? Math.max(1, Math.floor(payload['attempts']))
|
|
66
|
+
: undefined;
|
|
67
|
+
const idempotencyKey = typeof payload['uniqueId'] === 'string' && payload['uniqueId'].trim().length > 0
|
|
68
|
+
? payload['uniqueId'].trim()
|
|
69
|
+
: undefined;
|
|
70
|
+
await JobStateTracker.enqueued({
|
|
71
|
+
queueName: input.queue,
|
|
72
|
+
jobId: input.fallbackJobId,
|
|
73
|
+
payload: input.payload,
|
|
74
|
+
attempts: currentAttempts,
|
|
75
|
+
maxAttempts,
|
|
76
|
+
idempotencyKey,
|
|
77
|
+
});
|
|
78
|
+
const pendingRecoveryApi = JobStateTracker;
|
|
79
|
+
await pendingRecoveryApi.pendingRecovery?.({
|
|
80
|
+
queueName: input.queue,
|
|
81
|
+
jobId: input.fallbackJobId,
|
|
82
|
+
reason: 'Redis RPC enqueue failed; marked pending recovery',
|
|
83
|
+
error: input.error,
|
|
84
|
+
});
|
|
85
|
+
};
|
|
86
|
+
export const RedisRpcQueueDriver = Object.freeze({
|
|
87
|
+
async enqueue(queue, payload) {
|
|
88
|
+
const fallbackJobId = resolveRequestedJobId(payload);
|
|
89
|
+
const timeoutMs = Env.getInt('REDIS_RPC_TIMEOUT_MS', Env.getInt('QUEUE_HTTP_PROXY_TIMEOUT_MS', 10000));
|
|
90
|
+
const options = createJobOptions({ ...payload, jobId: fallbackJobId });
|
|
91
|
+
try {
|
|
92
|
+
return await TimeoutManager.withTimeoutRetry(async () => {
|
|
93
|
+
const client = await createRpcClient();
|
|
94
|
+
const result = await client.queue('add', {
|
|
95
|
+
target: queue,
|
|
96
|
+
args: [`${queue}-job`, payload, options],
|
|
97
|
+
});
|
|
98
|
+
return resolveJobId(result, fallbackJobId);
|
|
99
|
+
}, {
|
|
100
|
+
timeoutMs,
|
|
101
|
+
maxRetries: Math.max(0, Env.getInt('REDIS_RPC_RETRY_MAX', Env.getInt('QUEUE_HTTP_PROXY_RETRY_MAX', 2))),
|
|
102
|
+
retryDelayMs: Math.max(0, Env.getInt('REDIS_RPC_RETRY_DELAY_MS', Env.getInt('QUEUE_HTTP_PROXY_RETRY_DELAY_MS', 500))),
|
|
103
|
+
operationName: `redis-rpc-queue-enqueue:${queue}`,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
catch (error) {
|
|
107
|
+
await markPendingRecoveryFallback({ queue, fallbackJobId, payload, error });
|
|
108
|
+
return fallbackJobId;
|
|
109
|
+
}
|
|
110
|
+
},
|
|
111
|
+
async dequeue(queue) {
|
|
112
|
+
const client = await createRpcClient();
|
|
113
|
+
return client.queue('dequeue', {
|
|
114
|
+
target: queue,
|
|
115
|
+
visibilityTimeoutMs: Env.getInt('QUEUE_REDIS_VISIBILITY_TIMEOUT_MS', 30_000),
|
|
116
|
+
});
|
|
117
|
+
},
|
|
118
|
+
async ack(queue, id) {
|
|
119
|
+
const client = await createRpcClient();
|
|
120
|
+
await client.queue('ack', { target: queue, args: [id] });
|
|
121
|
+
},
|
|
122
|
+
async length(queue) {
|
|
123
|
+
const client = await createRpcClient();
|
|
124
|
+
return client.queue('length', { target: queue });
|
|
125
|
+
},
|
|
126
|
+
async drain(queue) {
|
|
127
|
+
const client = await createRpcClient();
|
|
128
|
+
await client.queue('drain', { target: queue });
|
|
129
|
+
},
|
|
130
|
+
});
|
|
131
|
+
export default RedisRpcQueueDriver;
|
package/dist/build-manifest.json
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zintrust/queue-redis",
|
|
3
|
-
"version": "2.1
|
|
4
|
-
"buildDate": "2026-05-
|
|
3
|
+
"version": "2.4.1",
|
|
4
|
+
"buildDate": "2026-05-31T11:38:17.470Z",
|
|
5
5
|
"buildEnvironment": {
|
|
6
6
|
"node": "v22.22.1",
|
|
7
7
|
"platform": "darwin",
|
|
8
8
|
"arch": "arm64"
|
|
9
9
|
},
|
|
10
10
|
"git": {
|
|
11
|
-
"commit": "
|
|
11
|
+
"commit": "e97b7b3d",
|
|
12
12
|
"branch": "release"
|
|
13
13
|
},
|
|
14
14
|
"package": {
|
|
@@ -20,17 +20,18 @@
|
|
|
20
20
|
"redis"
|
|
21
21
|
],
|
|
22
22
|
"peerDependencies": [
|
|
23
|
-
"@zintrust/core"
|
|
23
|
+
"@zintrust/core",
|
|
24
|
+
"@zintrust/redis-rpc"
|
|
24
25
|
]
|
|
25
26
|
},
|
|
26
27
|
"files": {
|
|
27
28
|
"BullMQRedisQueue.d.ts": {
|
|
28
|
-
"size":
|
|
29
|
-
"sha256": "
|
|
29
|
+
"size": 918,
|
|
30
|
+
"sha256": "fdcd271e7aa241cc3af89c979a4ee2538541d1442024bd6dda91c8b12f712c6f"
|
|
30
31
|
},
|
|
31
32
|
"BullMQRedisQueue.js": {
|
|
32
|
-
"size":
|
|
33
|
-
"sha256": "
|
|
33
|
+
"size": 21906,
|
|
34
|
+
"sha256": "87fd1eb1a320ea85b4cffde44e6f86ec487332cf8cd7052b11fd1b89caf8a1e5"
|
|
34
35
|
},
|
|
35
36
|
"HttpQueueDriver.d.ts": {
|
|
36
37
|
"size": 841,
|
|
@@ -42,11 +43,11 @@
|
|
|
42
43
|
},
|
|
43
44
|
"QueueHttpGateway.d.ts": {
|
|
44
45
|
"size": 437,
|
|
45
|
-
"sha256": "
|
|
46
|
+
"sha256": "9c45a8643136415b248828322442aac16cd4b2e0d22355c3b8ecf6188973ffe7"
|
|
46
47
|
},
|
|
47
48
|
"QueueHttpGateway.js": {
|
|
48
|
-
"size":
|
|
49
|
-
"sha256": "
|
|
49
|
+
"size": 7665,
|
|
50
|
+
"sha256": "352a1c2de6bbbaf7cd684bb428454bdd3293aff070956b3ee912403fe1394992"
|
|
50
51
|
},
|
|
51
52
|
"RedisPublishClient.d.ts": {
|
|
52
53
|
"size": 451,
|
|
@@ -56,25 +57,25 @@
|
|
|
56
57
|
"size": 5589,
|
|
57
58
|
"sha256": "aed9cee5023addfdbbf1026d0a45c86d29a8f3e65ffbd01637c1e0e7f994213b"
|
|
58
59
|
},
|
|
59
|
-
"
|
|
60
|
-
"size":
|
|
61
|
-
"sha256": "
|
|
60
|
+
"RedisRpcQueueDriver.d.ts": {
|
|
61
|
+
"size": 549,
|
|
62
|
+
"sha256": "19dc10ffc29e813a241a9c69184e988d94d1c16ffc82010c75cadc29a4833a93"
|
|
62
63
|
},
|
|
63
|
-
"
|
|
64
|
-
"size":
|
|
65
|
-
"sha256": "
|
|
64
|
+
"RedisRpcQueueDriver.js": {
|
|
65
|
+
"size": 5357,
|
|
66
|
+
"sha256": "24bb11e1a9f8a163ec79b08dc326e0c337297f312d393347dad91d38b692a9ea"
|
|
66
67
|
},
|
|
67
68
|
"build-manifest.json": {
|
|
68
|
-
"size":
|
|
69
|
-
"sha256": "
|
|
69
|
+
"size": 2273,
|
|
70
|
+
"sha256": "774ebf83d6b9fc3222555c6ace7c7318e4cdeba9e4f439bce38cfc99cef9cf66"
|
|
70
71
|
},
|
|
71
72
|
"index.d.ts": {
|
|
72
|
-
"size":
|
|
73
|
-
"sha256": "
|
|
73
|
+
"size": 609,
|
|
74
|
+
"sha256": "c74e26117ad588f7d28dda0c71fdd17588e6f58a0c68ede56af2fb2795352726"
|
|
74
75
|
},
|
|
75
76
|
"index.js": {
|
|
76
|
-
"size":
|
|
77
|
-
"sha256": "
|
|
77
|
+
"size": 714,
|
|
78
|
+
"sha256": "ea00335dbb0386bc3863d61def64aebcc330f8dcef348a663bae51055d1bcc20"
|
|
78
79
|
},
|
|
79
80
|
"register.d.ts": {
|
|
80
81
|
"size": 170,
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { BullMQRedisQueue } from './BullMQRedisQueue';
|
|
2
|
-
export { HttpQueueDriver } from './HttpQueueDriver';
|
|
3
2
|
export { QueueHttpGateway } from './QueueHttpGateway';
|
|
3
|
+
export { RedisRpcQueueDriver, shouldUseRedisRpcQueueDriver } from './RedisRpcQueueDriver';
|
|
4
4
|
export { createRedisPublishClient, resetPublishClient, type RedisPublishClient, } from './RedisPublishClient';
|
|
5
5
|
export type { QueueMessage } from '@zintrust/core/queue';
|
|
6
6
|
/**
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { BullMQRedisQueue } from './BullMQRedisQueue.js';
|
|
2
|
-
export { HttpQueueDriver } from './HttpQueueDriver.js';
|
|
3
2
|
export { QueueHttpGateway } from './QueueHttpGateway.js';
|
|
3
|
+
export { RedisRpcQueueDriver, shouldUseRedisRpcQueueDriver } from './RedisRpcQueueDriver.js';
|
|
4
4
|
export { createRedisPublishClient, resetPublishClient, } from './RedisPublishClient.js';
|
|
5
5
|
/**
|
|
6
6
|
* Package version and build metadata
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zintrust/queue-redis",
|
|
3
|
-
"version": "2.1
|
|
3
|
+
"version": "2.4.1",
|
|
4
4
|
"description": "Redis queue driver for ZinTrust.",
|
|
5
5
|
"private": false,
|
|
6
6
|
"type": "module",
|
|
@@ -23,7 +23,13 @@
|
|
|
23
23
|
"node": ">=20.0.0"
|
|
24
24
|
},
|
|
25
25
|
"peerDependencies": {
|
|
26
|
-
"@zintrust/core": "*"
|
|
26
|
+
"@zintrust/core": "*",
|
|
27
|
+
"@zintrust/redis-rpc": "*"
|
|
28
|
+
},
|
|
29
|
+
"peerDependenciesMeta": {
|
|
30
|
+
"@zintrust/redis-rpc": {
|
|
31
|
+
"optional": true
|
|
32
|
+
}
|
|
27
33
|
},
|
|
28
34
|
"publishConfig": {
|
|
29
35
|
"access": "public"
|
|
@@ -40,7 +46,7 @@
|
|
|
40
46
|
"prepublishOnly": "npm run build"
|
|
41
47
|
},
|
|
42
48
|
"dependencies": {
|
|
43
|
-
"ioredis": "^5.
|
|
49
|
+
"ioredis": "^5.11.0",
|
|
44
50
|
"redis": "^5.12.1"
|
|
45
51
|
}
|
|
46
52
|
}
|
package/dist/RedisQueue.d.ts
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import type { QueueMessage } from '@zintrust/core';
|
|
2
|
-
interface IQueueDriver {
|
|
3
|
-
enqueue<T = unknown>(queue: string, payload: T): Promise<string>;
|
|
4
|
-
dequeue<T = unknown>(queue: string): Promise<QueueMessage<T> | undefined>;
|
|
5
|
-
ack(queue: string, id: string): Promise<void>;
|
|
6
|
-
length(queue: string): Promise<number>;
|
|
7
|
-
drain(queue: string): Promise<void>;
|
|
8
|
-
}
|
|
9
|
-
export declare const RedisQueue: IQueueDriver;
|
|
10
|
-
export default RedisQueue;
|
package/dist/RedisQueue.js
DELETED
|
@@ -1,92 +0,0 @@
|
|
|
1
|
-
import { ErrorFactory, generateUuid, getRedisUrl, Logger } from '@zintrust/core';
|
|
2
|
-
export const RedisQueue = (() => {
|
|
3
|
-
let client = null;
|
|
4
|
-
let connected = false;
|
|
5
|
-
const ensureClient = async () => {
|
|
6
|
-
if (connected && client !== null)
|
|
7
|
-
return client;
|
|
8
|
-
const url = getRedisUrl();
|
|
9
|
-
if (url === null)
|
|
10
|
-
throw ErrorFactory.createConfigError('Redis queue driver requires REDIS_URL');
|
|
11
|
-
// Import lazily so package is optional for environments that don't use Redis
|
|
12
|
-
try {
|
|
13
|
-
// Prefer the redis package when available
|
|
14
|
-
try {
|
|
15
|
-
const mod = (await import('redis'));
|
|
16
|
-
const createClient = mod.createClient;
|
|
17
|
-
client = createClient({ url });
|
|
18
|
-
if (typeof client.connect === 'function') {
|
|
19
|
-
try {
|
|
20
|
-
await client.connect();
|
|
21
|
-
connected = true;
|
|
22
|
-
}
|
|
23
|
-
catch (connectionError) {
|
|
24
|
-
connected = false;
|
|
25
|
-
Logger.warn('Redis client connect failed:', String(connectionError));
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
else {
|
|
29
|
-
connected = true;
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
catch {
|
|
33
|
-
// Fallback to ioredis when available (used by queue-monitor)
|
|
34
|
-
const mod = (await import('ioredis'));
|
|
35
|
-
const redis = mod.default(url);
|
|
36
|
-
client = {
|
|
37
|
-
rPush: (queue, value) => redis.rpush(queue, value),
|
|
38
|
-
lPop: (queue) => redis.lpop(queue),
|
|
39
|
-
lLen: (queue) => redis.llen(queue),
|
|
40
|
-
del: (queue) => redis.del(queue),
|
|
41
|
-
};
|
|
42
|
-
connected = true;
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
catch (error) {
|
|
46
|
-
const globalFake = globalThis
|
|
47
|
-
.__fakeRedisClient;
|
|
48
|
-
if (globalFake === undefined) {
|
|
49
|
-
throw ErrorFactory.createConfigError("Redis queue driver requires the 'redis' or 'ioredis' package (run `zin add queue:redis` / `zin plugin install queue:redis`, or `npm install redis` / `npm install ioredis`) or a test fake client set in globalThis.__fakeRedisClient", error);
|
|
50
|
-
}
|
|
51
|
-
client = globalFake;
|
|
52
|
-
connected = true;
|
|
53
|
-
}
|
|
54
|
-
if (client === null)
|
|
55
|
-
throw ErrorFactory.createConfigError('Redis client could not be initialized');
|
|
56
|
-
return client;
|
|
57
|
-
};
|
|
58
|
-
return {
|
|
59
|
-
async enqueue(queue, payload) {
|
|
60
|
-
const cli = await ensureClient();
|
|
61
|
-
const id = generateUuid();
|
|
62
|
-
const msg = JSON.stringify({ id, payload, attempts: 0 });
|
|
63
|
-
await cli.rPush(queue, msg);
|
|
64
|
-
return id;
|
|
65
|
-
},
|
|
66
|
-
async dequeue(queue) {
|
|
67
|
-
const cli = await ensureClient();
|
|
68
|
-
const raw = await cli.lPop(queue);
|
|
69
|
-
if (raw === null)
|
|
70
|
-
return undefined;
|
|
71
|
-
try {
|
|
72
|
-
const parsed = JSON.parse(raw);
|
|
73
|
-
return parsed;
|
|
74
|
-
}
|
|
75
|
-
catch (err) {
|
|
76
|
-
throw ErrorFactory.createTryCatchError('Failed to parse queue message', err);
|
|
77
|
-
}
|
|
78
|
-
},
|
|
79
|
-
async ack(_queue, _id) {
|
|
80
|
-
return Promise.resolve(); // NOSONAR
|
|
81
|
-
},
|
|
82
|
-
async length(queue) {
|
|
83
|
-
const cli = await ensureClient();
|
|
84
|
-
return cli.lLen(queue);
|
|
85
|
-
},
|
|
86
|
-
async drain(queue) {
|
|
87
|
-
const cli = await ensureClient();
|
|
88
|
-
await cli.del(queue);
|
|
89
|
-
},
|
|
90
|
-
};
|
|
91
|
-
})();
|
|
92
|
-
export default RedisQueue;
|