bullmq 6.4.2 → 6.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/bun-redis-client.js +6 -6
- package/dist/cjs/classes/child-pool.js +1 -1
- package/dist/cjs/classes/child.js +8 -8
- package/dist/cjs/classes/flow-producer.js +10 -4
- package/dist/cjs/classes/job.js +2 -2
- package/dist/cjs/classes/main-worker.js +2 -2
- package/dist/cjs/classes/node-redis-client.js +6 -6
- package/dist/cjs/classes/queue-base.js +2 -2
- package/dist/cjs/classes/redis-connection.js +2 -2
- package/dist/cjs/classes/redis-queue-backend.js +40 -4
- package/dist/cjs/classes/redis-relay-backend.js +2 -2
- package/dist/cjs/classes/relay.js +8 -8
- package/dist/cjs/classes/valkey-glide-client.js +6 -6
- package/dist/cjs/classes/worker.js +4 -4
- package/dist/cjs/commands/addDelayedJob-6.lua +3 -1
- package/dist/cjs/commands/addParentJob-6.lua +3 -1
- package/dist/cjs/commands/addPrioritizedJob-9.lua +3 -1
- package/dist/cjs/commands/addStandardJob-9.lua +3 -1
- package/dist/cjs/commands/includes/handleDuplicatedJob.lua +8 -3
- package/dist/cjs/commands/includes/updateExistingJobsParent.lua +16 -4
- package/dist/cjs/commands/releaseParentIfNoPendingDependencies-2.lua +34 -0
- package/dist/cjs/postgres/migrations/index.js +9 -0
- package/dist/cjs/postgres/postgres-connection.js +2 -2
- package/dist/cjs/postgres/postgres-queue-backend.js +2 -2
- package/dist/cjs/postgres/postgres-relay-backend.js +4 -4
- package/dist/cjs/postgres/sql-scripts.js +1 -0
- package/dist/cjs/scripts/addDelayedJob-6.js +26 -8
- package/dist/cjs/scripts/addParentJob-6.js +26 -8
- package/dist/cjs/scripts/addPrioritizedJob-9.js +26 -8
- package/dist/cjs/scripts/addStandardJob-9.js +26 -8
- package/dist/cjs/scripts/index.js +1 -0
- package/dist/cjs/scripts/releaseParentIfNoPendingDependencies-2.js +167 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/utils/index.js +2 -2
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/bun-redis-client.js +3 -3
- package/dist/esm/classes/child-pool.js +1 -1
- package/dist/esm/classes/child.d.ts +3 -3
- package/dist/esm/classes/child.js +4 -4
- package/dist/esm/classes/flow-producer.d.ts +1 -1
- package/dist/esm/classes/flow-producer.js +9 -3
- package/dist/esm/classes/job.js +1 -1
- package/dist/esm/classes/main-worker.js +1 -1
- package/dist/esm/classes/node-redis-client.js +3 -3
- package/dist/esm/classes/queue-base.d.ts +1 -1
- package/dist/esm/classes/queue-base.js +1 -1
- package/dist/esm/classes/redis-connection.d.ts +1 -1
- package/dist/esm/classes/redis-connection.js +1 -1
- package/dist/esm/classes/redis-queue-backend.d.ts +1 -1
- package/dist/esm/classes/redis-queue-backend.js +39 -3
- package/dist/esm/classes/redis-relay-backend.d.ts +1 -1
- package/dist/esm/classes/redis-relay-backend.js +1 -1
- package/dist/esm/classes/relay.d.ts +1 -1
- package/dist/esm/classes/relay.js +3 -3
- package/dist/esm/classes/valkey-glide-client.js +3 -3
- package/dist/esm/classes/worker.d.ts +1 -1
- package/dist/esm/classes/worker.js +3 -3
- package/dist/esm/commands/addDelayedJob-6.lua +3 -1
- package/dist/esm/commands/addParentJob-6.lua +3 -1
- package/dist/esm/commands/addPrioritizedJob-9.lua +3 -1
- package/dist/esm/commands/addStandardJob-9.lua +3 -1
- package/dist/esm/commands/includes/handleDuplicatedJob.lua +8 -3
- package/dist/esm/commands/includes/updateExistingJobsParent.lua +16 -4
- package/dist/esm/commands/releaseParentIfNoPendingDependencies-2.lua +34 -0
- package/dist/esm/interfaces/connection.d.ts +1 -1
- package/dist/esm/interfaces/parent.d.ts +8 -0
- package/dist/esm/interfaces/redis-client.d.ts +1 -1
- package/dist/esm/interfaces/sandboxed-options.d.ts +2 -2
- package/dist/esm/postgres/migrations/index.js +9 -0
- package/dist/esm/postgres/postgres-connection.d.ts +1 -1
- package/dist/esm/postgres/postgres-connection.js +1 -1
- package/dist/esm/postgres/postgres-queue-backend.d.ts +1 -1
- package/dist/esm/postgres/postgres-queue-backend.js +1 -1
- package/dist/esm/postgres/postgres-relay-backend.d.ts +1 -1
- package/dist/esm/postgres/postgres-relay-backend.js +2 -2
- package/dist/esm/postgres/sql-scripts.js +1 -0
- package/dist/esm/scripts/addDelayedJob-6.js +26 -8
- package/dist/esm/scripts/addParentJob-6.js +26 -8
- package/dist/esm/scripts/addPrioritizedJob-9.js +26 -8
- package/dist/esm/scripts/addStandardJob-9.js +26 -8
- package/dist/esm/scripts/index.d.ts +1 -0
- package/dist/esm/scripts/index.js +1 -0
- package/dist/esm/scripts/releaseParentIfNoPendingDependencies-2.d.ts +5 -0
- package/dist/esm/scripts/releaseParentIfNoPendingDependencies-2.js +164 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/utils/index.d.ts +2 -2
- package/dist/esm/utils/index.js +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +5 -5
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.createBunRedisClient = createBunRedisClient;
|
|
4
|
-
const
|
|
5
|
-
const
|
|
6
|
-
const
|
|
4
|
+
const node_crypto_1 = require("node:crypto");
|
|
5
|
+
const node_events_1 = require("node:events");
|
|
6
|
+
const node_stream_1 = require("node:stream");
|
|
7
7
|
const connection_closed_error_1 = require("./errors/connection-closed-error");
|
|
8
8
|
function normalizeScriptArgs(args) {
|
|
9
9
|
return args.length === 1 && Array.isArray(args[0]) ? args[0] : args;
|
|
@@ -47,7 +47,7 @@ function createBunRedisClient(client, opts) {
|
|
|
47
47
|
* 3. Raw command execution: Bun's `send()` is the universal escape hatch
|
|
48
48
|
* for commands without convenience methods.
|
|
49
49
|
*/
|
|
50
|
-
class BunRedisAdapter extends
|
|
50
|
+
class BunRedisAdapter extends node_events_1.EventEmitter {
|
|
51
51
|
get status() {
|
|
52
52
|
var _a;
|
|
53
53
|
if (this.statusOverride) {
|
|
@@ -469,7 +469,7 @@ class BunRedisAdapter extends events_1.EventEmitter {
|
|
|
469
469
|
// Lua script engine
|
|
470
470
|
// ---------------------------------------------------------------
|
|
471
471
|
defineCommand(name, definition) {
|
|
472
|
-
const sha = (0,
|
|
472
|
+
const sha = (0, node_crypto_1.createHash)('sha1').update(definition.lua).digest('hex');
|
|
473
473
|
this.scripts.set(name, {
|
|
474
474
|
sha,
|
|
475
475
|
lua: definition.lua,
|
|
@@ -866,7 +866,7 @@ class BunRedisAdapter extends events_1.EventEmitter {
|
|
|
866
866
|
const adapter = this;
|
|
867
867
|
let cursor = '0';
|
|
868
868
|
let started = false;
|
|
869
|
-
const readable = new
|
|
869
|
+
const readable = new node_stream_1.Readable({
|
|
870
870
|
objectMode: true,
|
|
871
871
|
async read() {
|
|
872
872
|
if (started && cursor === '0') {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ChildPool = void 0;
|
|
4
|
-
const path = require("path");
|
|
4
|
+
const path = require("node:path");
|
|
5
5
|
const child_1 = require("./child");
|
|
6
6
|
const CHILD_KILL_TIMEOUT = 30000;
|
|
7
7
|
const supportCJS = () => {
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Child = void 0;
|
|
4
|
-
const
|
|
5
|
-
const
|
|
6
|
-
const
|
|
4
|
+
const node_child_process_1 = require("node:child_process");
|
|
5
|
+
const node_net_1 = require("node:net");
|
|
6
|
+
const node_worker_threads_1 = require("node:worker_threads");
|
|
7
7
|
const enums_1 = require("../enums");
|
|
8
|
-
const
|
|
8
|
+
const node_events_1 = require("node:events");
|
|
9
9
|
/**
|
|
10
10
|
* @see https://nodejs.org/api/process.html#process_exit_codes
|
|
11
11
|
*/
|
|
@@ -30,7 +30,7 @@ const exitCodesErrors = {
|
|
|
30
30
|
* isolated processes or threads for processing jobs.
|
|
31
31
|
*
|
|
32
32
|
*/
|
|
33
|
-
class Child extends
|
|
33
|
+
class Child extends node_events_1.EventEmitter {
|
|
34
34
|
constructor(mainFile, processFile, opts = {
|
|
35
35
|
useWorkerThreads: false,
|
|
36
36
|
}) {
|
|
@@ -72,12 +72,12 @@ class Child extends events_1.EventEmitter {
|
|
|
72
72
|
const execArgv = await convertExecArgv(process.execArgv);
|
|
73
73
|
let parent;
|
|
74
74
|
if (this.opts.useWorkerThreads) {
|
|
75
|
-
this.worker = parent = new
|
|
75
|
+
this.worker = parent = new node_worker_threads_1.Worker(this.mainFile, Object.assign({ execArgv, stdin: true, stdout: true, stderr: true }, (this.opts.workerThreadsOptions
|
|
76
76
|
? this.opts.workerThreadsOptions
|
|
77
77
|
: {})));
|
|
78
78
|
}
|
|
79
79
|
else {
|
|
80
|
-
this.childProcess = parent = (0,
|
|
80
|
+
this.childProcess = parent = (0, node_child_process_1.fork)(this.mainFile, [], Object.assign({ execArgv, stdio: 'pipe' }, (this.opts.workerForkOptions ? this.opts.workerForkOptions : {})));
|
|
81
81
|
}
|
|
82
82
|
parent.on('exit', (exitCode, signalCode) => {
|
|
83
83
|
this._exitCode = exitCode;
|
|
@@ -195,7 +195,7 @@ function onExitOnce(child) {
|
|
|
195
195
|
}
|
|
196
196
|
const getFreePort = async () => {
|
|
197
197
|
return new Promise(resolve => {
|
|
198
|
-
const server = (0,
|
|
198
|
+
const server = (0, node_net_1.createServer)();
|
|
199
199
|
server.listen(0, () => {
|
|
200
200
|
const { port } = server.address();
|
|
201
201
|
server.close(() => resolve(port));
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.FlowProducer = void 0;
|
|
4
|
-
const
|
|
4
|
+
const node_events_1 = require("node:events");
|
|
5
5
|
const utils_1 = require("../utils");
|
|
6
6
|
const create_backend_1 = require("../utils/create-backend");
|
|
7
7
|
const job_1 = require("./job");
|
|
@@ -14,7 +14,7 @@ const enums_1 = require("../enums");
|
|
|
14
14
|
* will be processed, being able to access the children's result data.
|
|
15
15
|
* All Jobs can be in different queues, either children or parents,
|
|
16
16
|
*/
|
|
17
|
-
class FlowProducer extends
|
|
17
|
+
class FlowProducer extends node_events_1.EventEmitter {
|
|
18
18
|
constructor(opts, backendFactory) {
|
|
19
19
|
super();
|
|
20
20
|
this.opts = Object.assign(opts === undefined ? { connection: {} } : {}, opts);
|
|
@@ -209,7 +209,7 @@ class FlowProducer extends events_1.EventEmitter {
|
|
|
209
209
|
const jobsOpts = (_a = queueOpts === null || queueOpts === void 0 ? void 0 : queueOpts.defaultJobOptions) !== null && _a !== void 0 ? _a : {};
|
|
210
210
|
const jobId = ((_b = node.opts) === null || _b === void 0 ? void 0 : _b.jobId) || (0, utils_1.randomUUID)();
|
|
211
211
|
return (0, utils_1.trace)(this.telemetry, enums_1.SpanKind.PRODUCER, node.queueName, 'addNode', node.queueName, async (span, srcPropagationMetadata) => {
|
|
212
|
-
var _a, _b;
|
|
212
|
+
var _a, _b, _c;
|
|
213
213
|
span === null || span === void 0 ? void 0 : span.setAttributes({
|
|
214
214
|
[enums_1.TelemetryAttributes.JobName]: node.name,
|
|
215
215
|
[enums_1.TelemetryAttributes.JobId]: jobId,
|
|
@@ -227,7 +227,11 @@ class FlowProducer extends events_1.EventEmitter {
|
|
|
227
227
|
};
|
|
228
228
|
}
|
|
229
229
|
}
|
|
230
|
-
const
|
|
230
|
+
const jobOpts = Object.assign(Object.assign(Object.assign({}, jobsOpts), opts), { parent: parent === null || parent === void 0 ? void 0 : parent.parentOpts, telemetry });
|
|
231
|
+
if (jobOpts.parent && ((_c = jobOpts.deduplication) === null || _c === void 0 ? void 0 : _c.keepLastIfActive)) {
|
|
232
|
+
throw new Error('keepLastIfActive deduplication cannot be used on flow nodes with a parent');
|
|
233
|
+
}
|
|
234
|
+
const job = new this.Job(queue, node.name, node.data, jobOpts, jobId);
|
|
231
235
|
const parentKey = (0, utils_1.getParentKey)(parent === null || parent === void 0 ? void 0 : parent.parentOpts);
|
|
232
236
|
if (node.children && node.children.length > 0) {
|
|
233
237
|
// Create the parent job, it will be a job in status "waiting-children".
|
|
@@ -334,6 +338,8 @@ class FlowProducer extends events_1.EventEmitter {
|
|
|
334
338
|
}
|
|
335
339
|
}
|
|
336
340
|
validateFlowJobs(nodes) {
|
|
341
|
+
// TODO: In the next breaking change, reject deduplication on every flow
|
|
342
|
+
// node. Parent IDs must be known before any add scripts run.
|
|
337
343
|
for (const node of nodes) {
|
|
338
344
|
const children = node.children;
|
|
339
345
|
if (children && children.length > 0) {
|
package/dist/cjs/classes/job.js
CHANGED
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Job = exports.PRIORITY_LIMIT = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
|
-
const
|
|
5
|
+
const node_util_1 = require("node:util");
|
|
6
6
|
const utils_1 = require("../utils");
|
|
7
7
|
const backoffs_1 = require("./backoffs");
|
|
8
8
|
const unrecoverable_error_1 = require("./errors/unrecoverable-error");
|
|
9
9
|
const enums_1 = require("../enums");
|
|
10
|
-
const logger = (0,
|
|
10
|
+
const logger = (0, node_util_1.debuglog)('bull');
|
|
11
11
|
exports.PRIORITY_LIMIT = 2 ** 21 - 1;
|
|
12
12
|
/**
|
|
13
13
|
* Job
|
|
@@ -4,6 +4,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
4
4
|
* Worker Thread wrapper for sandboxing
|
|
5
5
|
*
|
|
6
6
|
*/
|
|
7
|
-
const
|
|
7
|
+
const node_worker_threads_1 = require("node:worker_threads");
|
|
8
8
|
const main_base_1 = require("./main-base");
|
|
9
|
-
(0, main_base_1.default)(async (msg) =>
|
|
9
|
+
(0, main_base_1.default)(async (msg) => node_worker_threads_1.parentPort.postMessage(msg), node_worker_threads_1.parentPort);
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.createNodeRedisClient = createNodeRedisClient;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
|
-
const
|
|
6
|
-
const
|
|
7
|
-
const
|
|
5
|
+
const node_crypto_1 = require("node:crypto");
|
|
6
|
+
const node_events_1 = require("node:events");
|
|
7
|
+
const node_stream_1 = require("node:stream");
|
|
8
8
|
const connection_closed_error_1 = require("./errors/connection-closed-error");
|
|
9
9
|
function normalizeScriptArgs(args) {
|
|
10
10
|
return args.length === 1 && Array.isArray(args[0]) ? args[0] : args;
|
|
@@ -21,7 +21,7 @@ function createNodeRedisClient(client) {
|
|
|
21
21
|
* Full wrapper (not augmentation) because node-redis's API is structurally
|
|
22
22
|
* different from ioredis and cannot be patched in-place.
|
|
23
23
|
*/
|
|
24
|
-
class NodeRedisAdapter extends
|
|
24
|
+
class NodeRedisAdapter extends node_events_1.EventEmitter {
|
|
25
25
|
/**
|
|
26
26
|
* Expose connection status using the vocabulary that
|
|
27
27
|
* {@link RedisConnection.waitUntilReady} expects:
|
|
@@ -239,7 +239,7 @@ class NodeRedisAdapter extends events_1.EventEmitter {
|
|
|
239
239
|
// Lua script engine
|
|
240
240
|
// ---------------------------------------------------------------
|
|
241
241
|
defineCommand(name, definition) {
|
|
242
|
-
const sha = (0,
|
|
242
|
+
const sha = (0, node_crypto_1.createHash)('sha1').update(definition.lua).digest('hex');
|
|
243
243
|
this.scripts.set(name, {
|
|
244
244
|
sha,
|
|
245
245
|
lua: definition.lua,
|
|
@@ -550,7 +550,7 @@ class NodeRedisAdapter extends events_1.EventEmitter {
|
|
|
550
550
|
if (options.count) {
|
|
551
551
|
scanOpts.COUNT = options.count;
|
|
552
552
|
}
|
|
553
|
-
const readable = new
|
|
553
|
+
const readable = new node_stream_1.Readable({
|
|
554
554
|
objectMode: true,
|
|
555
555
|
async read() {
|
|
556
556
|
var _a, e_1, _b, _c;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.QueueBase = void 0;
|
|
4
|
-
const
|
|
4
|
+
const node_events_1 = require("node:events");
|
|
5
5
|
const utils_1 = require("../utils");
|
|
6
6
|
const create_backend_1 = require("../utils/create-backend");
|
|
7
7
|
const job_1 = require("./job");
|
|
@@ -10,7 +10,7 @@ const job_1 = require("./job");
|
|
|
10
10
|
* This class is normally not used directly, but extended by the other classes.
|
|
11
11
|
*
|
|
12
12
|
*/
|
|
13
|
-
class QueueBase extends
|
|
13
|
+
class QueueBase extends node_events_1.EventEmitter {
|
|
14
14
|
constructor(name, opts, backendFactory, hasBlockingConnection = false) {
|
|
15
15
|
super();
|
|
16
16
|
this.closed = false;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.RedisConnection = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
|
-
const
|
|
5
|
+
const node_events_1 = require("node:events");
|
|
6
6
|
const utils_1 = require("../utils");
|
|
7
7
|
const version_1 = require("../version");
|
|
8
8
|
const scripts = require("../scripts");
|
|
@@ -97,7 +97,7 @@ function wrapRedisInstance(instance) {
|
|
|
97
97
|
// Default: treat as an ioredis instance (backwards compatible).
|
|
98
98
|
return (0, ioredis_client_1.createIORedisClient)(instance);
|
|
99
99
|
}
|
|
100
|
-
class RedisConnection extends
|
|
100
|
+
class RedisConnection extends node_events_1.EventEmitter {
|
|
101
101
|
constructor(opts, extraOptions) {
|
|
102
102
|
super();
|
|
103
103
|
this.extraOptions = extraOptions;
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.RedisQueueBackend = void 0;
|
|
7
7
|
exports.raw2NextJobData = raw2NextJobData;
|
|
8
|
-
const
|
|
8
|
+
const node_events_1 = require("node:events");
|
|
9
9
|
const msgpackr_1 = require("msgpackr");
|
|
10
10
|
const packer = new msgpackr_1.Packr({
|
|
11
11
|
useRecords: false,
|
|
@@ -61,7 +61,7 @@ async function resetBlockedConnection(connection, client, reconnect) {
|
|
|
61
61
|
await reconnect();
|
|
62
62
|
}
|
|
63
63
|
}
|
|
64
|
-
class RedisQueueBackend extends
|
|
64
|
+
class RedisQueueBackend extends node_events_1.EventEmitter {
|
|
65
65
|
constructor(connection, name, keys, toKey, opts, blockingConnection, ownsConnection = true) {
|
|
66
66
|
var _a;
|
|
67
67
|
super();
|
|
@@ -452,6 +452,7 @@ class RedisQueueBackend extends events_1.EventEmitter {
|
|
|
452
452
|
parent,
|
|
453
453
|
job.repeatJobKey,
|
|
454
454
|
job.deduplicationId ? `${queueKeys.de}:${job.deduplicationId}` : null,
|
|
455
|
+
parentKeyOpts.deferParentRelease ? 1 : null,
|
|
455
456
|
];
|
|
456
457
|
const encodedOpts = pack(optsAsJSON(opts));
|
|
457
458
|
let result;
|
|
@@ -509,13 +510,48 @@ class RedisQueueBackend extends events_1.EventEmitter {
|
|
|
509
510
|
* queue's key map.
|
|
510
511
|
*/
|
|
511
512
|
async addFlow(entries) {
|
|
513
|
+
var _a;
|
|
512
514
|
const client = await this.queue.client;
|
|
513
515
|
const multi = client.multi();
|
|
516
|
+
// Parents touched by this flow, deduplicated by key. A parent is released
|
|
517
|
+
// from "waiting-children" only after the whole tree has been inserted (see
|
|
518
|
+
// below), so collect them while queuing the inserts.
|
|
519
|
+
const parents = new Map();
|
|
514
520
|
for (const entry of entries) {
|
|
515
521
|
const keys = new queue_keys_1.QueueKeys(entry.prefix).getKeys(entry.queueName);
|
|
516
|
-
await this.addJobToTransaction(multi, entry.jobData, entry.jobId, entry.parentKeyOpts, keys);
|
|
522
|
+
await this.addJobToTransaction(multi, entry.jobData, entry.jobId, Object.assign(Object.assign({}, entry.parentKeyOpts), { deferParentRelease: true }), keys);
|
|
523
|
+
const parent = entry.jobData.parent;
|
|
524
|
+
const parentKey = (_a = entry.parentKeyOpts) === null || _a === void 0 ? void 0 : _a.parentKey;
|
|
525
|
+
if (parent && parentKey) {
|
|
526
|
+
parents.set(parentKey, {
|
|
527
|
+
parentId: parent.id,
|
|
528
|
+
parentKey,
|
|
529
|
+
parentQueueKey: parent.queueKey,
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
// Now that every node of the flow is in the transaction, validate each
|
|
534
|
+
// parent: one that ended up without pending dependencies (e.g. all of its
|
|
535
|
+
// children already existed and were completed) is moved out of
|
|
536
|
+
// "waiting-children". The script is a no-op otherwise.
|
|
537
|
+
for (const { parentId, parentKey, parentQueueKey } of parents.values()) {
|
|
538
|
+
this.execCommand(multi, 'releaseParentIfNoPendingDependencies', [
|
|
539
|
+
parentKey,
|
|
540
|
+
`${parentKey}:dependencies`,
|
|
541
|
+
parentQueueKey,
|
|
542
|
+
parentId,
|
|
543
|
+
Date.now(),
|
|
544
|
+
]);
|
|
545
|
+
}
|
|
546
|
+
const results = (await multi.exec());
|
|
547
|
+
// Callers index the results by entry, so drop the trailing parent-release
|
|
548
|
+
// replies.
|
|
549
|
+
for (const [err] of results.slice(entries.length)) {
|
|
550
|
+
if (err) {
|
|
551
|
+
throw err;
|
|
552
|
+
}
|
|
517
553
|
}
|
|
518
|
-
return (
|
|
554
|
+
return results.slice(0, entries.length);
|
|
519
555
|
}
|
|
520
556
|
pauseArgs(pause, emitEvent = true) {
|
|
521
557
|
let src = 'wait', dst = 'paused';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.createRedisRelayBackend = exports.RedisRelayBackend = exports.RELAY_LAYOUT_VERSION = exports.RELAY_MINIMUM_REDIS_VERSION = void 0;
|
|
4
|
-
const
|
|
4
|
+
const node_events_1 = require("node:events");
|
|
5
5
|
const utils_1 = require("../utils");
|
|
6
6
|
const redis_connection_1 = require("./redis-connection");
|
|
7
7
|
const relay_errors_1 = require("./relay-errors");
|
|
@@ -54,7 +54,7 @@ exports.RELAY_LAYOUT_VERSION = '1';
|
|
|
54
54
|
* connection. Entries published while that connection is reconnecting are
|
|
55
55
|
* read once it's back.
|
|
56
56
|
*/
|
|
57
|
-
class RedisRelayBackend extends
|
|
57
|
+
class RedisRelayBackend extends node_events_1.EventEmitter {
|
|
58
58
|
constructor(opts) {
|
|
59
59
|
var _a, _b, _c, _d, _e;
|
|
60
60
|
super();
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Relay = void 0;
|
|
4
|
-
const
|
|
5
|
-
const
|
|
6
|
-
const
|
|
4
|
+
const node_events_1 = require("node:events");
|
|
5
|
+
const node_crypto_1 = require("node:crypto");
|
|
6
|
+
const node_os_1 = require("node:os");
|
|
7
7
|
const redis_relay_backend_1 = require("./redis-relay-backend");
|
|
8
8
|
const version_1 = require("../version");
|
|
9
9
|
const ID_RE = /^[A-Za-z0-9_-]{1,64}$/;
|
|
@@ -28,7 +28,7 @@ const MAX_NAME_LENGTH = 256;
|
|
|
28
28
|
* @experimental The API may change in minor releases while the relay is
|
|
29
29
|
* completed.
|
|
30
30
|
*/
|
|
31
|
-
class Relay extends
|
|
31
|
+
class Relay extends node_events_1.EventEmitter {
|
|
32
32
|
constructor(opts) {
|
|
33
33
|
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
34
34
|
super();
|
|
@@ -46,7 +46,7 @@ class Relay extends events_1.EventEmitter {
|
|
|
46
46
|
throw new Error('Relay requires a connection');
|
|
47
47
|
}
|
|
48
48
|
this.namespace = (_a = opts.namespace) !== null && _a !== void 0 ? _a : 'default';
|
|
49
|
-
this.nodeId = (_b = opts.nodeId) !== null && _b !== void 0 ? _b : (0,
|
|
49
|
+
this.nodeId = (_b = opts.nodeId) !== null && _b !== void 0 ? _b : (0, node_crypto_1.randomUUID)().replace(/-/g, '');
|
|
50
50
|
for (const [name, value] of [
|
|
51
51
|
['namespace', this.namespace],
|
|
52
52
|
['nodeId', this.nodeId],
|
|
@@ -92,7 +92,7 @@ class Relay extends events_1.EventEmitter {
|
|
|
92
92
|
inboxMaxLength: this.inboxMaxLength,
|
|
93
93
|
skipVersionCheck: opts.skipVersionCheck,
|
|
94
94
|
});
|
|
95
|
-
if (this.backend instanceof
|
|
95
|
+
if (this.backend instanceof node_events_1.EventEmitter) {
|
|
96
96
|
this.backend.on('error', err => this.emitError(err));
|
|
97
97
|
}
|
|
98
98
|
this.cursor = this.backend.initialInboxCursor;
|
|
@@ -110,7 +110,7 @@ class Relay extends events_1.EventEmitter {
|
|
|
110
110
|
var _a;
|
|
111
111
|
await this.ready;
|
|
112
112
|
this.assertOpen();
|
|
113
|
-
const endpointId = (0,
|
|
113
|
+
const endpointId = (0, node_crypto_1.randomUUID)().replace(/-/g, '');
|
|
114
114
|
const endpoint = {
|
|
115
115
|
pattern,
|
|
116
116
|
listener,
|
|
@@ -399,7 +399,7 @@ class Relay extends events_1.EventEmitter {
|
|
|
399
399
|
register() {
|
|
400
400
|
return this.backend.registerNode(this.nodeId, this.leaseDuration, {
|
|
401
401
|
name: this.opts.name,
|
|
402
|
-
host: (0,
|
|
402
|
+
host: (0, node_os_1.hostname)(),
|
|
403
403
|
pid: process.pid,
|
|
404
404
|
version: version_1.version,
|
|
405
405
|
}, this.inboxMaxLength);
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.createValkeyGlideClient = createValkeyGlideClient;
|
|
4
|
-
const
|
|
5
|
-
const
|
|
6
|
-
const
|
|
4
|
+
const node_crypto_1 = require("node:crypto");
|
|
5
|
+
const node_events_1 = require("node:events");
|
|
6
|
+
const node_stream_1 = require("node:stream");
|
|
7
7
|
const connection_closed_error_1 = require("./errors/connection-closed-error");
|
|
8
8
|
const GLIDE_STRING_DECODER = 1;
|
|
9
9
|
function createCompatibilityBatch() {
|
|
@@ -211,7 +211,7 @@ function isBlockingCommand(args) {
|
|
|
211
211
|
function createValkeyGlideClient(client) {
|
|
212
212
|
return new ValkeyGlideAdapter(client);
|
|
213
213
|
}
|
|
214
|
-
class ValkeyGlideAdapter extends
|
|
214
|
+
class ValkeyGlideAdapter extends node_events_1.EventEmitter {
|
|
215
215
|
constructor(rawOrPromise, connectionName) {
|
|
216
216
|
super();
|
|
217
217
|
this.connectionName = connectionName;
|
|
@@ -491,7 +491,7 @@ class ValkeyGlideAdapter extends events_1.EventEmitter {
|
|
|
491
491
|
return new ValkeyGlideAdapter(duplicated, options.connectionName);
|
|
492
492
|
}
|
|
493
493
|
defineCommand(name, definition) {
|
|
494
|
-
const sha = (0,
|
|
494
|
+
const sha = (0, node_crypto_1.createHash)('sha1').update(definition.lua).digest('hex');
|
|
495
495
|
const script = {
|
|
496
496
|
sha,
|
|
497
497
|
lua: definition.lua,
|
|
@@ -736,7 +736,7 @@ class ValkeyGlideAdapter extends events_1.EventEmitter {
|
|
|
736
736
|
scanStream(options) {
|
|
737
737
|
let cursor = '0';
|
|
738
738
|
let running = false;
|
|
739
|
-
const stream = new
|
|
739
|
+
const stream = new node_stream_1.Readable({
|
|
740
740
|
objectMode: true,
|
|
741
741
|
read: () => {
|
|
742
742
|
if (running) {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Worker = void 0;
|
|
4
|
-
const fs = require("fs");
|
|
5
|
-
const
|
|
6
|
-
const path = require("path");
|
|
4
|
+
const fs = require("node:fs");
|
|
5
|
+
const node_url_1 = require("node:url");
|
|
6
|
+
const path = require("node:path");
|
|
7
7
|
const abort_controller_1 = require("./abort-controller");
|
|
8
8
|
const utils_1 = require("../utils");
|
|
9
9
|
const queue_base_1 = require("./queue-base");
|
|
@@ -87,7 +87,7 @@ class Worker extends queue_base_1.QueueBase {
|
|
|
87
87
|
}
|
|
88
88
|
else {
|
|
89
89
|
// SANDBOXED
|
|
90
|
-
if (processor instanceof
|
|
90
|
+
if (processor instanceof node_url_1.URL) {
|
|
91
91
|
if (!fs.existsSync(processor)) {
|
|
92
92
|
throw new Error(`URL ${processor} does not exist in the local file system`);
|
|
93
93
|
}
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
[7] parent? {id, queueKey}
|
|
26
26
|
[8] repeat job key
|
|
27
27
|
[9] deduplication key
|
|
28
|
+
[10] defer parent release (flow inserts)
|
|
28
29
|
|
|
29
30
|
ARGV[2] Json stringified job data
|
|
30
31
|
ARGV[3] msgpacked options
|
|
@@ -52,6 +53,7 @@ local parentKey = args[5]
|
|
|
52
53
|
local parent = args[7]
|
|
53
54
|
local repeatJobKey = args[8]
|
|
54
55
|
local deduplicationKey = args[9]
|
|
56
|
+
local deferParentRelease = args[10]
|
|
55
57
|
local parentData
|
|
56
58
|
|
|
57
59
|
-- Includes
|
|
@@ -84,7 +86,7 @@ else
|
|
|
84
86
|
if rcall("EXISTS", jobIdKey) == 1 then
|
|
85
87
|
return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
|
|
86
88
|
parentData, parentDependenciesKey, completedKey, eventsKey,
|
|
87
|
-
maxEvents, timestamp)
|
|
89
|
+
maxEvents, timestamp, deferParentRelease)
|
|
88
90
|
end
|
|
89
91
|
end
|
|
90
92
|
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
[7] parent? {id, queueKey}
|
|
23
23
|
[8] repeat job key
|
|
24
24
|
[9] deduplication key
|
|
25
|
+
[10] defer parent release (flow inserts)
|
|
25
26
|
|
|
26
27
|
ARGV[2] Json stringified job data
|
|
27
28
|
ARGV[3] msgpacked options
|
|
@@ -50,6 +51,7 @@ local parentKey = args[5]
|
|
|
50
51
|
local parent = args[7]
|
|
51
52
|
local repeatJobKey = args[8]
|
|
52
53
|
local deduplicationKey = args[9]
|
|
54
|
+
local deferParentRelease = args[10]
|
|
53
55
|
local parentData
|
|
54
56
|
|
|
55
57
|
-- Includes
|
|
@@ -79,7 +81,7 @@ else
|
|
|
79
81
|
if rcall("EXISTS", jobIdKey) == 1 then
|
|
80
82
|
return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
|
|
81
83
|
parentData, parentDependenciesKey, completedKey, eventsKey,
|
|
82
|
-
maxEvents, timestamp)
|
|
84
|
+
maxEvents, timestamp, deferParentRelease)
|
|
83
85
|
end
|
|
84
86
|
end
|
|
85
87
|
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
[7] parent? {id, queueKey}
|
|
26
26
|
[8] repeat job key
|
|
27
27
|
[9] deduplication key
|
|
28
|
+
[10] defer parent release (flow inserts)
|
|
28
29
|
|
|
29
30
|
ARGV[2] Json stringified job data
|
|
30
31
|
ARGV[3] msgpacked options
|
|
@@ -55,6 +56,7 @@ local parentKey = args[5]
|
|
|
55
56
|
local parent = args[7]
|
|
56
57
|
local repeatJobKey = args[8]
|
|
57
58
|
local deduplicationKey = args[9]
|
|
59
|
+
local deferParentRelease = args[10]
|
|
58
60
|
local parentData
|
|
59
61
|
|
|
60
62
|
-- Includes
|
|
@@ -86,7 +88,7 @@ else
|
|
|
86
88
|
if rcall("EXISTS", jobIdKey) == 1 then
|
|
87
89
|
return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
|
|
88
90
|
parentData, parentDependenciesKey, completedKey, eventsKey,
|
|
89
|
-
maxEvents, timestamp)
|
|
91
|
+
maxEvents, timestamp, deferParentRelease)
|
|
90
92
|
end
|
|
91
93
|
end
|
|
92
94
|
|
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
[7] parent? {id, queueKey}
|
|
36
36
|
[8] repeat job key
|
|
37
37
|
[9] deduplication key
|
|
38
|
+
[10] defer parent release (flow inserts)
|
|
38
39
|
|
|
39
40
|
ARGV[2] Json stringified job data
|
|
40
41
|
ARGV[3] msgpacked options
|
|
@@ -61,6 +62,7 @@ local parentKey = args[5]
|
|
|
61
62
|
local parent = args[7]
|
|
62
63
|
local repeatJobKey = args[8]
|
|
63
64
|
local deduplicationKey = args[9]
|
|
65
|
+
local deferParentRelease = args[10]
|
|
64
66
|
local parentData
|
|
65
67
|
|
|
66
68
|
-- Includes
|
|
@@ -92,7 +94,7 @@ else
|
|
|
92
94
|
if rcall("EXISTS", jobIdKey) == 1 then
|
|
93
95
|
return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
|
|
94
96
|
parentData, parentDependenciesKey, KEYS[5], eventsKey,
|
|
95
|
-
maxEvents, timestamp)
|
|
97
|
+
maxEvents, timestamp, deferParentRelease)
|
|
96
98
|
end
|
|
97
99
|
end
|
|
98
100
|
|
|
@@ -6,16 +6,21 @@
|
|
|
6
6
|
--- @include "updateExistingJobsParent"
|
|
7
7
|
|
|
8
8
|
local function handleDuplicatedJob(jobKey, jobId, currentParentKey, currentParent,
|
|
9
|
-
parentData, parentDependenciesKey, completedKey, eventsKey, maxEvents, timestamp
|
|
9
|
+
parentData, parentDependenciesKey, completedKey, eventsKey, maxEvents, timestamp,
|
|
10
|
+
deferParentRelease)
|
|
10
11
|
local existedParentKey = rcall("HGET", jobKey, "parentKey")
|
|
11
12
|
|
|
12
|
-
if not existedParentKey or existedParentKey == currentParentKey
|
|
13
|
+
if not existedParentKey or existedParentKey == currentParentKey
|
|
14
|
+
or rcall("EXISTS", existedParentKey) == 0 then
|
|
13
15
|
updateExistingJobsParent(currentParentKey, currentParent, parentData,
|
|
14
16
|
parentDependenciesKey, completedKey, jobKey,
|
|
15
|
-
jobId, timestamp)
|
|
17
|
+
jobId, timestamp, deferParentRelease)
|
|
16
18
|
else
|
|
17
19
|
if currentParentKey ~= nil and currentParentKey ~= existedParentKey
|
|
18
20
|
and (rcall("EXISTS", existedParentKey) == 1) then
|
|
21
|
+
if deferParentRelease then
|
|
22
|
+
rcall("HSET", currentParentKey, "flowAttachmentFailed", "1")
|
|
23
|
+
end
|
|
19
24
|
return -7
|
|
20
25
|
end
|
|
21
26
|
end
|
|
@@ -3,6 +3,13 @@
|
|
|
3
3
|
is already completed and about to be ignored. The parent must get its
|
|
4
4
|
dependencies updated to avoid the parent job being stuck forever in
|
|
5
5
|
the waiting-children state.
|
|
6
|
+
|
|
7
|
+
When "deferParentRelease" is true (flow inserts), the parent is *not*
|
|
8
|
+
moved out of waiting-children here: the remaining children of the flow
|
|
9
|
+
have not been inserted yet, so the parent would be released too early.
|
|
10
|
+
Only the processed dependency is recorded; releasing the parent is
|
|
11
|
+
validated by "releaseParentIfNoPendingDependencies" once the whole flow
|
|
12
|
+
has been added.
|
|
6
13
|
]]
|
|
7
14
|
|
|
8
15
|
-- Includes
|
|
@@ -10,13 +17,18 @@
|
|
|
10
17
|
|
|
11
18
|
local function updateExistingJobsParent(parentKey, parent, parentData,
|
|
12
19
|
parentDependenciesKey, completedKey,
|
|
13
|
-
jobIdKey, jobId, timestamp
|
|
20
|
+
jobIdKey, jobId, timestamp,
|
|
21
|
+
deferParentRelease)
|
|
14
22
|
if parentKey ~= nil then
|
|
15
23
|
if rcall("ZSCORE", completedKey, jobId) then
|
|
16
24
|
local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
25
|
+
if deferParentRelease then
|
|
26
|
+
rcall("HSET", parentKey .. ":processed", jobIdKey, returnvalue)
|
|
27
|
+
else
|
|
28
|
+
updateParentDepsIfNeeded(parentKey, parent['queueKey'],
|
|
29
|
+
parentDependenciesKey, parent['id'],
|
|
30
|
+
jobIdKey, returnvalue, timestamp)
|
|
31
|
+
end
|
|
20
32
|
else
|
|
21
33
|
if parentDependenciesKey ~= nil then
|
|
22
34
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|