bullmq 6.3.5 → 6.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/bun-redis-client.js +41 -7
- package/dist/cjs/commands/includes/deduplicateJob.lua +11 -0
- package/dist/cjs/commands/includes/deduplicateJobWithoutReplace.lua +11 -0
- package/dist/cjs/commands/includes/recoverStaleDeduplicationKey.lua +28 -0
- package/dist/cjs/postgres/migrations/0003_dedup_stale_key.sql +147 -0
- package/dist/cjs/postgres/migrations/index.js +8 -0
- package/dist/cjs/scripts/addDelayedJob-6.js +42 -0
- package/dist/cjs/scripts/addParentJob-6.js +34 -0
- package/dist/cjs/scripts/addPrioritizedJob-9.js +42 -0
- package/dist/cjs/scripts/addStandardJob-9.js +42 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/bun-redis-client.js +41 -7
- package/dist/esm/commands/includes/deduplicateJob.lua +11 -0
- package/dist/esm/commands/includes/deduplicateJobWithoutReplace.lua +11 -0
- package/dist/esm/commands/includes/recoverStaleDeduplicationKey.lua +28 -0
- package/dist/esm/postgres/migrations/0003_dedup_stale_key.sql +147 -0
- package/dist/esm/postgres/migrations/index.js +8 -0
- package/dist/esm/scripts/addDelayedJob-6.js +42 -0
- package/dist/esm/scripts/addParentJob-6.js +34 -0
- package/dist/esm/scripts/addPrioritizedJob-9.js +42 -0
- package/dist/esm/scripts/addStandardJob-9.js +42 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +8 -8
|
@@ -218,11 +218,11 @@ class BunRedisAdapter extends events_1.EventEmitter {
|
|
|
218
218
|
async connect() {
|
|
219
219
|
var _a, _b;
|
|
220
220
|
// A duplicate created with a `rawFactory` builds its raw client lazily on
|
|
221
|
-
// the first connect (Bun's native `duplicate()` is async).
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
this.
|
|
221
|
+
// the first connect (Bun's native `duplicate()` is async). Concurrent
|
|
222
|
+
// connect()/_ensureRaw() share one in-flight materialization so we don't
|
|
223
|
+
// create two native clients and overwrite an unclosed raw.
|
|
224
|
+
if (!this.raw && (this.rawFactory || this.materializing)) {
|
|
225
|
+
await this._materializeRaw();
|
|
226
226
|
}
|
|
227
227
|
const replaceRaw = this.hasConnected && (this.closed || !this.raw.connected);
|
|
228
228
|
if (this.reconnectTimer) {
|
|
@@ -394,6 +394,36 @@ class BunRedisAdapter extends events_1.EventEmitter {
|
|
|
394
394
|
const Ctor = src.constructor;
|
|
395
395
|
return new Ctor(src.url);
|
|
396
396
|
}
|
|
397
|
+
/**
|
|
398
|
+
* Materialize a lazily-created raw client (duplicate `rawFactory`).
|
|
399
|
+
* Concurrent callers share the in-flight promise so the factory runs once.
|
|
400
|
+
*/
|
|
401
|
+
_materializeRaw() {
|
|
402
|
+
if (this.raw) {
|
|
403
|
+
return Promise.resolve(this.raw);
|
|
404
|
+
}
|
|
405
|
+
if (this.materializing) {
|
|
406
|
+
return this.materializing;
|
|
407
|
+
}
|
|
408
|
+
const factory = this.rawFactory;
|
|
409
|
+
if (!factory) {
|
|
410
|
+
return Promise.resolve(this.raw);
|
|
411
|
+
}
|
|
412
|
+
const materializing = factory()
|
|
413
|
+
.then(raw => {
|
|
414
|
+
this.raw = raw;
|
|
415
|
+
this.rawFactory = undefined;
|
|
416
|
+
this._setupCallbacks();
|
|
417
|
+
return raw;
|
|
418
|
+
})
|
|
419
|
+
.finally(() => {
|
|
420
|
+
if (this.materializing === materializing) {
|
|
421
|
+
this.materializing = undefined;
|
|
422
|
+
}
|
|
423
|
+
});
|
|
424
|
+
this.materializing = materializing;
|
|
425
|
+
return materializing;
|
|
426
|
+
}
|
|
397
427
|
/**
|
|
398
428
|
* Return the raw client, materializing it first when this adapter is a
|
|
399
429
|
* lazily-initialized duplicate (created via `duplicate()` with a
|
|
@@ -413,9 +443,13 @@ class BunRedisAdapter extends events_1.EventEmitter {
|
|
|
413
443
|
// on first connect. Rebuilding from `this.raw.url` is not possible because
|
|
414
444
|
// Bun never exposes the URL, which previously sent duplicates to the
|
|
415
445
|
// wrong (default) server (#4582).
|
|
416
|
-
|
|
446
|
+
//
|
|
447
|
+
// Do not close over `this.raw`: a duplicate may not have materialized its
|
|
448
|
+
// raw client yet, so nested `duplicate().duplicate()` would capture
|
|
449
|
+
// undefined and throw in `_duplicateRaw` (#4706). Resolve the parent raw
|
|
450
|
+
// lazily, matching reconnect's fallback to `rawFactory`.
|
|
417
451
|
const adapter = new BunRedisAdapter(undefined, {
|
|
418
|
-
rawFactory: () => this._duplicateRaw(
|
|
452
|
+
rawFactory: async () => this._duplicateRaw(await this._ensureRaw()),
|
|
419
453
|
});
|
|
420
454
|
// Copy registered scripts to the duplicate
|
|
421
455
|
for (const [name, script] of this.scripts) {
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
]]
|
|
4
4
|
-- Includes
|
|
5
5
|
--- @include "deduplicateJobWithoutReplace"
|
|
6
|
+
--- @include "recoverStaleDeduplicationKey"
|
|
6
7
|
--- @include "removeJobKeys"
|
|
7
8
|
--- @include "setDeduplicationKey"
|
|
8
9
|
--- @include "storeDeduplicatedNextJob"
|
|
@@ -33,6 +34,11 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
33
34
|
local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
34
35
|
currentDeduplicatedJobId, jobId, deduplicationId, prefix)
|
|
35
36
|
if isRemoved then
|
|
37
|
+
-- Discard any pending next-job payload stored while the replaced
|
|
38
|
+
-- job was active, otherwise it would be resurrected when the
|
|
39
|
+
-- incoming job finalizes.
|
|
40
|
+
rcall('DEL', prefix .. "dn:" .. deduplicationId)
|
|
41
|
+
|
|
36
42
|
if deduplicationOpts['keepLastIfActive'] then
|
|
37
43
|
rcall('SET', deduplicationKey, jobId)
|
|
38
44
|
else
|
|
@@ -45,6 +51,11 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
45
51
|
end
|
|
46
52
|
return
|
|
47
53
|
else
|
|
54
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
55
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
56
|
+
return
|
|
57
|
+
end
|
|
58
|
+
|
|
48
59
|
storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
49
60
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
50
61
|
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
--[[
|
|
2
2
|
Function to deduplicate a job.
|
|
3
3
|
]]
|
|
4
|
+
--- @include "recoverStaleDeduplicationKey"
|
|
4
5
|
--- @include "setDeduplicationKey"
|
|
5
6
|
--- @include "storeDeduplicatedNextJob"
|
|
6
7
|
|
|
@@ -9,10 +10,15 @@ local function deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
|
9
10
|
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
10
11
|
local ttl = deduplicationOpts['ttl']
|
|
11
12
|
local deduplicationKeyExists
|
|
13
|
+
|
|
12
14
|
if ttl and ttl > 0 then
|
|
13
15
|
if deduplicationOpts['extend'] then
|
|
14
16
|
local currentDeduplicatedJobId = rcall('GET', deduplicationKey)
|
|
15
17
|
if currentDeduplicatedJobId then
|
|
18
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
19
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
20
|
+
return
|
|
21
|
+
end
|
|
16
22
|
if storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
17
23
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
18
24
|
parentKey, parentData, parentDependenciesKey, repeatJobKey) then
|
|
@@ -48,6 +54,11 @@ local function deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
|
48
54
|
if deduplicationKeyExists then
|
|
49
55
|
local currentDeduplicatedJobId = rcall('GET', deduplicationKey)
|
|
50
56
|
|
|
57
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
58
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
59
|
+
return
|
|
60
|
+
end
|
|
61
|
+
|
|
51
62
|
if storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
52
63
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
53
64
|
parentKey, parentData, parentDependenciesKey, repeatJobKey) then
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
--[[
|
|
2
|
+
Function to recover a stale deduplication key.
|
|
3
|
+
When the existing deduplication key has no expiry (simple mode, or
|
|
4
|
+
keepLastIfActive after the key was persisted), it should disappear
|
|
5
|
+
when the winning job finishes. If that job key no longer exists
|
|
6
|
+
because of an outage, the deduplication key is stale, so we discard it
|
|
7
|
+
and start a new deduplication window with the incoming job.
|
|
8
|
+
Returns true if the stale key was recovered, false otherwise.
|
|
9
|
+
]]
|
|
10
|
+
--- @include "setDeduplicationKey"
|
|
11
|
+
|
|
12
|
+
local function recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId, jobId,
|
|
13
|
+
deduplicationId, deduplicationOpts)
|
|
14
|
+
if currentDeduplicatedJobId and rcall('PTTL', deduplicationKey) == -1 and
|
|
15
|
+
rcall('EXISTS', prefix .. currentDeduplicatedJobId) == 0 then
|
|
16
|
+
rcall('DEL', deduplicationKey)
|
|
17
|
+
-- Discard any pending next-job payload stored for the stale winner,
|
|
18
|
+
-- otherwise it would be resurrected when this job finalizes.
|
|
19
|
+
rcall('DEL', prefix .. "dn:" .. deduplicationId)
|
|
20
|
+
if deduplicationOpts['keepLastIfActive'] then
|
|
21
|
+
rcall('SET', deduplicationKey, jobId)
|
|
22
|
+
else
|
|
23
|
+
setDeduplicationKey(deduplicationKey, jobId, deduplicationOpts)
|
|
24
|
+
end
|
|
25
|
+
return true
|
|
26
|
+
end
|
|
27
|
+
return false
|
|
28
|
+
end
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
-- BullMQ PostgreSQL backend — deduplication stale-key recovery.
|
|
2
|
+
--
|
|
3
|
+
-- Replaces `deduplicate_job` (created in 0002_functions.sql) to fix two ways a
|
|
4
|
+
-- deduplication key could outlive the job it points at:
|
|
5
|
+
--
|
|
6
|
+
-- 1. Stale key: a key with no expiry is meant to disappear when its winner
|
|
7
|
+
-- finishes, but `clean`, `drain`, `obliterate` (and outages) delete the
|
|
8
|
+
-- job row without clearing the `dedup` row. Every later add then matched
|
|
9
|
+
-- that dead winner and was silently dropped, forever. Mirrors the Redis
|
|
10
|
+
-- `recoverStaleDeduplicationKey` include.
|
|
11
|
+
-- 2. Orphaned proto-next: replacing a *delayed* winner left behind any
|
|
12
|
+
-- `dedup_next` payload stashed while that winner was previously active,
|
|
13
|
+
-- so it got resurrected when the replacement finished.
|
|
14
|
+
--
|
|
15
|
+
-- Only the function body changes — no schema/DDL change — so the signature is
|
|
16
|
+
-- identical and older clients keep working against the updated schema.
|
|
17
|
+
|
|
18
|
+
CREATE OR REPLACE FUNCTION deduplicate_job(
|
|
19
|
+
p_queue text, p_dedup jsonb, p_job_id text, p_now bigint,
|
|
20
|
+
p_name text, p_data jsonb, p_opts jsonb
|
|
21
|
+
) RETURNS text
|
|
22
|
+
LANGUAGE plpgsql
|
|
23
|
+
SET search_path FROM CURRENT
|
|
24
|
+
AS $$
|
|
25
|
+
DECLARE
|
|
26
|
+
v_id text := p_dedup->>'id';
|
|
27
|
+
v_ttl bigint := NULLIF(p_dedup->>'ttl', '')::bigint;
|
|
28
|
+
v_extend boolean := COALESCE((p_dedup->>'extend')::boolean, false);
|
|
29
|
+
v_replace boolean := COALESCE((p_dedup->>'replace')::boolean, false);
|
|
30
|
+
v_keeplast boolean := COALESCE((p_dedup->>'keepLastIfActive')::boolean, false);
|
|
31
|
+
v_cur text;
|
|
32
|
+
v_exp bigint;
|
|
33
|
+
v_state job_state;
|
|
34
|
+
BEGIN
|
|
35
|
+
IF v_id IS NULL OR v_id = '' THEN
|
|
36
|
+
RETURN NULL;
|
|
37
|
+
END IF;
|
|
38
|
+
|
|
39
|
+
-- The current winner (only if the key is still live).
|
|
40
|
+
SELECT job_id, expire_at_ms INTO v_cur, v_exp
|
|
41
|
+
FROM dedup WHERE queue = p_queue AND dedup_id = v_id;
|
|
42
|
+
IF v_cur IS NULL OR (v_exp IS NOT NULL AND v_exp <= p_now) THEN
|
|
43
|
+
v_cur := NULL;
|
|
44
|
+
ELSIF v_exp IS NULL AND NOT EXISTS (
|
|
45
|
+
SELECT 1 FROM job WHERE queue = p_queue AND id = v_cur
|
|
46
|
+
) THEN
|
|
47
|
+
-- Stale key recovery (mirrors recoverStaleDeduplicationKey): a key with no
|
|
48
|
+
-- expiry is meant to disappear when its winner finishes. If the job row is
|
|
49
|
+
-- gone (clean, drain, obliterate, an outage…) the key is stale, so discard
|
|
50
|
+
-- it together with any pending proto-next (which would otherwise be
|
|
51
|
+
-- resurrected) and start a new window with the incoming job.
|
|
52
|
+
DELETE FROM dedup WHERE queue = p_queue AND dedup_id = v_id;
|
|
53
|
+
DELETE FROM dedup_next WHERE queue = p_queue AND dedup_id = v_id;
|
|
54
|
+
v_cur := NULL;
|
|
55
|
+
END IF;
|
|
56
|
+
|
|
57
|
+
IF v_replace THEN
|
|
58
|
+
IF v_cur IS NOT NULL THEN
|
|
59
|
+
SELECT state INTO v_state
|
|
60
|
+
FROM job WHERE queue = p_queue AND id = v_cur;
|
|
61
|
+
IF v_state = 'delayed' THEN
|
|
62
|
+
-- Drop the previous delayed job and take its place.
|
|
63
|
+
DELETE FROM job WHERE queue = p_queue AND id = v_cur;
|
|
64
|
+
PERFORM publish_event(p_queue, 'removed',
|
|
65
|
+
jsonb_build_object('jobId', v_cur, 'prev', 'delayed'));
|
|
66
|
+
PERFORM publish_event(p_queue, 'deduplicated',
|
|
67
|
+
jsonb_build_object('jobId', p_job_id, 'deduplicationId', v_id,
|
|
68
|
+
'deduplicatedJobId', v_cur));
|
|
69
|
+
-- Discard any pending proto-next stashed while the replaced job was
|
|
70
|
+
-- active, otherwise it would be resurrected when p_job_id finalizes.
|
|
71
|
+
DELETE FROM dedup_next WHERE queue = p_queue AND dedup_id = v_id;
|
|
72
|
+
IF v_keeplast THEN
|
|
73
|
+
UPDATE dedup SET job_id = p_job_id, expire_at_ms = NULL
|
|
74
|
+
WHERE queue = p_queue AND dedup_id = v_id;
|
|
75
|
+
ELSIF NOT v_extend AND COALESCE(v_ttl, 0) > 0 THEN
|
|
76
|
+
-- KEEPTTL: keep the existing window, just swap the winner.
|
|
77
|
+
UPDATE dedup SET job_id = p_job_id
|
|
78
|
+
WHERE queue = p_queue AND dedup_id = v_id;
|
|
79
|
+
ELSE
|
|
80
|
+
UPDATE dedup
|
|
81
|
+
SET job_id = p_job_id,
|
|
82
|
+
expire_at_ms = CASE WHEN COALESCE(v_ttl, 0) > 0
|
|
83
|
+
THEN p_now + v_ttl ELSE NULL END
|
|
84
|
+
WHERE queue = p_queue AND dedup_id = v_id;
|
|
85
|
+
END IF;
|
|
86
|
+
RETURN NULL;
|
|
87
|
+
ELSE
|
|
88
|
+
-- Winner is not a removable delayed job: stash proto-next if it is
|
|
89
|
+
-- active + keepLastIfActive, then deduplicate.
|
|
90
|
+
PERFORM dedup_store_next(p_queue, v_id, v_cur, p_job_id,
|
|
91
|
+
v_keeplast, p_name, p_data, p_opts);
|
|
92
|
+
RETURN v_cur;
|
|
93
|
+
END IF;
|
|
94
|
+
ELSE
|
|
95
|
+
INSERT INTO dedup (queue, dedup_id, job_id, expire_at_ms)
|
|
96
|
+
VALUES (p_queue, v_id, p_job_id,
|
|
97
|
+
CASE WHEN NOT v_keeplast AND COALESCE(v_ttl, 0) > 0
|
|
98
|
+
THEN p_now + v_ttl ELSE NULL END)
|
|
99
|
+
ON CONFLICT (queue, dedup_id) DO UPDATE
|
|
100
|
+
SET job_id = EXCLUDED.job_id, expire_at_ms = EXCLUDED.expire_at_ms;
|
|
101
|
+
RETURN NULL;
|
|
102
|
+
END IF;
|
|
103
|
+
END IF;
|
|
104
|
+
|
|
105
|
+
-- Without replace.
|
|
106
|
+
IF COALESCE(v_ttl, 0) > 0 AND v_extend THEN
|
|
107
|
+
IF v_cur IS NOT NULL THEN
|
|
108
|
+
-- Stash proto-next if active+keepLast; else extend the window (or
|
|
109
|
+
-- persist when keepLastIfActive). Either way keep the current winner.
|
|
110
|
+
IF NOT dedup_store_next(p_queue, v_id, v_cur, p_job_id,
|
|
111
|
+
v_keeplast, p_name, p_data, p_opts) THEN
|
|
112
|
+
UPDATE dedup
|
|
113
|
+
SET expire_at_ms = CASE WHEN v_keeplast THEN NULL ELSE p_now + v_ttl END
|
|
114
|
+
WHERE queue = p_queue AND dedup_id = v_id;
|
|
115
|
+
END IF;
|
|
116
|
+
PERFORM publish_event(p_queue, 'deduplicated',
|
|
117
|
+
jsonb_build_object('jobId', v_cur, 'deduplicationId', v_id,
|
|
118
|
+
'deduplicatedJobId', p_job_id));
|
|
119
|
+
RETURN v_cur;
|
|
120
|
+
END IF;
|
|
121
|
+
INSERT INTO dedup (queue, dedup_id, job_id, expire_at_ms)
|
|
122
|
+
VALUES (p_queue, v_id, p_job_id,
|
|
123
|
+
CASE WHEN v_keeplast THEN NULL ELSE p_now + v_ttl END)
|
|
124
|
+
ON CONFLICT (queue, dedup_id) DO UPDATE
|
|
125
|
+
SET job_id = EXCLUDED.job_id, expire_at_ms = EXCLUDED.expire_at_ms;
|
|
126
|
+
RETURN NULL;
|
|
127
|
+
END IF;
|
|
128
|
+
|
|
129
|
+
-- SET NX semantics (ttl>0 non-extend, or no ttl at all).
|
|
130
|
+
IF v_cur IS NOT NULL THEN
|
|
131
|
+
PERFORM dedup_store_next(p_queue, v_id, v_cur, p_job_id,
|
|
132
|
+
v_keeplast, p_name, p_data, p_opts);
|
|
133
|
+
PERFORM publish_event(p_queue, 'deduplicated',
|
|
134
|
+
jsonb_build_object('jobId', v_cur, 'deduplicationId', v_id,
|
|
135
|
+
'deduplicatedJobId', p_job_id));
|
|
136
|
+
RETURN v_cur;
|
|
137
|
+
END IF;
|
|
138
|
+
|
|
139
|
+
INSERT INTO dedup (queue, dedup_id, job_id, expire_at_ms)
|
|
140
|
+
VALUES (p_queue, v_id, p_job_id,
|
|
141
|
+
CASE WHEN NOT v_keeplast AND COALESCE(v_ttl, 0) > 0
|
|
142
|
+
THEN p_now + v_ttl ELSE NULL END)
|
|
143
|
+
ON CONFLICT (queue, dedup_id) DO UPDATE
|
|
144
|
+
SET job_id = EXCLUDED.job_id, expire_at_ms = EXCLUDED.expire_at_ms;
|
|
145
|
+
RETURN NULL;
|
|
146
|
+
END;
|
|
147
|
+
$$;
|
|
@@ -21,6 +21,14 @@ exports.MIGRATIONS = [
|
|
|
21
21
|
minClientVersion: 6,
|
|
22
22
|
load: () => (0, sql_loader_1.loadMigrationSql)('0002_functions.sql'),
|
|
23
23
|
},
|
|
24
|
+
{
|
|
25
|
+
version: 3,
|
|
26
|
+
name: '0003_dedup_stale_key',
|
|
27
|
+
// Function-body fix only (CREATE OR REPLACE, identical signature, no DDL),
|
|
28
|
+
// so clients from the same major keep working against the updated schema.
|
|
29
|
+
minClientVersion: 6,
|
|
30
|
+
load: () => (0, sql_loader_1.loadMigrationSql)('0003_dedup_stale_key.sql'),
|
|
31
|
+
},
|
|
24
32
|
];
|
|
25
33
|
/**
|
|
26
34
|
* The highest schema version this BullMQ build knows how to produce. Explicit
|
|
@@ -118,6 +118,15 @@ end
|
|
|
118
118
|
--[[
|
|
119
119
|
Function to deduplicate a job.
|
|
120
120
|
]]
|
|
121
|
+
--[[
|
|
122
|
+
Function to recover a stale deduplication key.
|
|
123
|
+
When the existing deduplication key has no expiry (simple mode, or
|
|
124
|
+
keepLastIfActive after the key was persisted), it should disappear
|
|
125
|
+
when the winning job finishes. If that job key no longer exists
|
|
126
|
+
because of an outage, the deduplication key is stale, so we discard it
|
|
127
|
+
and start a new deduplication window with the incoming job.
|
|
128
|
+
Returns true if the stale key was recovered, false otherwise.
|
|
129
|
+
]]
|
|
121
130
|
--[[
|
|
122
131
|
Function to set the deduplication key for a job.
|
|
123
132
|
Uses TTL from deduplication opts if provided.
|
|
@@ -130,6 +139,23 @@ local function setDeduplicationKey(deduplicationKey, jobId, deduplicationOpts)
|
|
|
130
139
|
rcall('SET', deduplicationKey, jobId)
|
|
131
140
|
end
|
|
132
141
|
end
|
|
142
|
+
local function recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId, jobId,
|
|
143
|
+
deduplicationId, deduplicationOpts)
|
|
144
|
+
if currentDeduplicatedJobId and rcall('PTTL', deduplicationKey) == -1 and
|
|
145
|
+
rcall('EXISTS', prefix .. currentDeduplicatedJobId) == 0 then
|
|
146
|
+
rcall('DEL', deduplicationKey)
|
|
147
|
+
-- Discard any pending next-job payload stored for the stale winner,
|
|
148
|
+
-- otherwise it would be resurrected when this job finalizes.
|
|
149
|
+
rcall('DEL', prefix .. "dn:" .. deduplicationId)
|
|
150
|
+
if deduplicationOpts['keepLastIfActive'] then
|
|
151
|
+
rcall('SET', deduplicationKey, jobId)
|
|
152
|
+
else
|
|
153
|
+
setDeduplicationKey(deduplicationKey, jobId, deduplicationOpts)
|
|
154
|
+
end
|
|
155
|
+
return true
|
|
156
|
+
end
|
|
157
|
+
return false
|
|
158
|
+
end
|
|
133
159
|
--[[
|
|
134
160
|
Function to store a deduplicated next job if the existing job is active
|
|
135
161
|
and keepLastIfActive is set. When the active job finishes, the stored
|
|
@@ -196,6 +222,10 @@ local function deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
|
196
222
|
if deduplicationOpts['extend'] then
|
|
197
223
|
local currentDeduplicatedJobId = rcall('GET', deduplicationKey)
|
|
198
224
|
if currentDeduplicatedJobId then
|
|
225
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
226
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
227
|
+
return
|
|
228
|
+
end
|
|
199
229
|
if storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
200
230
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
201
231
|
parentKey, parentData, parentDependenciesKey, repeatJobKey) then
|
|
@@ -229,6 +259,10 @@ local function deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
|
229
259
|
end
|
|
230
260
|
if deduplicationKeyExists then
|
|
231
261
|
local currentDeduplicatedJobId = rcall('GET', deduplicationKey)
|
|
262
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
263
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
264
|
+
return
|
|
265
|
+
end
|
|
232
266
|
if storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
233
267
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
234
268
|
parentKey, parentData, parentDependenciesKey, repeatJobKey) then
|
|
@@ -268,6 +302,10 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
268
302
|
local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
269
303
|
currentDeduplicatedJobId, jobId, deduplicationId, prefix)
|
|
270
304
|
if isRemoved then
|
|
305
|
+
-- Discard any pending next-job payload stored while the replaced
|
|
306
|
+
-- job was active, otherwise it would be resurrected when the
|
|
307
|
+
-- incoming job finalizes.
|
|
308
|
+
rcall('DEL', prefix .. "dn:" .. deduplicationId)
|
|
271
309
|
if deduplicationOpts['keepLastIfActive'] then
|
|
272
310
|
rcall('SET', deduplicationKey, jobId)
|
|
273
311
|
else
|
|
@@ -280,6 +318,10 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
280
318
|
end
|
|
281
319
|
return
|
|
282
320
|
else
|
|
321
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
322
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
323
|
+
return
|
|
324
|
+
end
|
|
283
325
|
storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
284
326
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
285
327
|
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
@@ -49,6 +49,15 @@ local parentData
|
|
|
49
49
|
--[[
|
|
50
50
|
Function to deduplicate a job.
|
|
51
51
|
]]
|
|
52
|
+
--[[
|
|
53
|
+
Function to recover a stale deduplication key.
|
|
54
|
+
When the existing deduplication key has no expiry (simple mode, or
|
|
55
|
+
keepLastIfActive after the key was persisted), it should disappear
|
|
56
|
+
when the winning job finishes. If that job key no longer exists
|
|
57
|
+
because of an outage, the deduplication key is stale, so we discard it
|
|
58
|
+
and start a new deduplication window with the incoming job.
|
|
59
|
+
Returns true if the stale key was recovered, false otherwise.
|
|
60
|
+
]]
|
|
52
61
|
--[[
|
|
53
62
|
Function to set the deduplication key for a job.
|
|
54
63
|
Uses TTL from deduplication opts if provided.
|
|
@@ -61,6 +70,23 @@ local function setDeduplicationKey(deduplicationKey, jobId, deduplicationOpts)
|
|
|
61
70
|
rcall('SET', deduplicationKey, jobId)
|
|
62
71
|
end
|
|
63
72
|
end
|
|
73
|
+
local function recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId, jobId,
|
|
74
|
+
deduplicationId, deduplicationOpts)
|
|
75
|
+
if currentDeduplicatedJobId and rcall('PTTL', deduplicationKey) == -1 and
|
|
76
|
+
rcall('EXISTS', prefix .. currentDeduplicatedJobId) == 0 then
|
|
77
|
+
rcall('DEL', deduplicationKey)
|
|
78
|
+
-- Discard any pending next-job payload stored for the stale winner,
|
|
79
|
+
-- otherwise it would be resurrected when this job finalizes.
|
|
80
|
+
rcall('DEL', prefix .. "dn:" .. deduplicationId)
|
|
81
|
+
if deduplicationOpts['keepLastIfActive'] then
|
|
82
|
+
rcall('SET', deduplicationKey, jobId)
|
|
83
|
+
else
|
|
84
|
+
setDeduplicationKey(deduplicationKey, jobId, deduplicationOpts)
|
|
85
|
+
end
|
|
86
|
+
return true
|
|
87
|
+
end
|
|
88
|
+
return false
|
|
89
|
+
end
|
|
64
90
|
--[[
|
|
65
91
|
Function to store a deduplicated next job if the existing job is active
|
|
66
92
|
and keepLastIfActive is set. When the active job finishes, the stored
|
|
@@ -127,6 +153,10 @@ local function deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
|
127
153
|
if deduplicationOpts['extend'] then
|
|
128
154
|
local currentDeduplicatedJobId = rcall('GET', deduplicationKey)
|
|
129
155
|
if currentDeduplicatedJobId then
|
|
156
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
157
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
158
|
+
return
|
|
159
|
+
end
|
|
130
160
|
if storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
131
161
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
132
162
|
parentKey, parentData, parentDependenciesKey, repeatJobKey) then
|
|
@@ -160,6 +190,10 @@ local function deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
|
160
190
|
end
|
|
161
191
|
if deduplicationKeyExists then
|
|
162
192
|
local currentDeduplicatedJobId = rcall('GET', deduplicationKey)
|
|
193
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
194
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
195
|
+
return
|
|
196
|
+
end
|
|
163
197
|
if storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
164
198
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
165
199
|
parentKey, parentData, parentDependenciesKey, repeatJobKey) then
|
|
@@ -83,6 +83,15 @@ end
|
|
|
83
83
|
--[[
|
|
84
84
|
Function to deduplicate a job.
|
|
85
85
|
]]
|
|
86
|
+
--[[
|
|
87
|
+
Function to recover a stale deduplication key.
|
|
88
|
+
When the existing deduplication key has no expiry (simple mode, or
|
|
89
|
+
keepLastIfActive after the key was persisted), it should disappear
|
|
90
|
+
when the winning job finishes. If that job key no longer exists
|
|
91
|
+
because of an outage, the deduplication key is stale, so we discard it
|
|
92
|
+
and start a new deduplication window with the incoming job.
|
|
93
|
+
Returns true if the stale key was recovered, false otherwise.
|
|
94
|
+
]]
|
|
86
95
|
--[[
|
|
87
96
|
Function to set the deduplication key for a job.
|
|
88
97
|
Uses TTL from deduplication opts if provided.
|
|
@@ -95,6 +104,23 @@ local function setDeduplicationKey(deduplicationKey, jobId, deduplicationOpts)
|
|
|
95
104
|
rcall('SET', deduplicationKey, jobId)
|
|
96
105
|
end
|
|
97
106
|
end
|
|
107
|
+
local function recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId, jobId,
|
|
108
|
+
deduplicationId, deduplicationOpts)
|
|
109
|
+
if currentDeduplicatedJobId and rcall('PTTL', deduplicationKey) == -1 and
|
|
110
|
+
rcall('EXISTS', prefix .. currentDeduplicatedJobId) == 0 then
|
|
111
|
+
rcall('DEL', deduplicationKey)
|
|
112
|
+
-- Discard any pending next-job payload stored for the stale winner,
|
|
113
|
+
-- otherwise it would be resurrected when this job finalizes.
|
|
114
|
+
rcall('DEL', prefix .. "dn:" .. deduplicationId)
|
|
115
|
+
if deduplicationOpts['keepLastIfActive'] then
|
|
116
|
+
rcall('SET', deduplicationKey, jobId)
|
|
117
|
+
else
|
|
118
|
+
setDeduplicationKey(deduplicationKey, jobId, deduplicationOpts)
|
|
119
|
+
end
|
|
120
|
+
return true
|
|
121
|
+
end
|
|
122
|
+
return false
|
|
123
|
+
end
|
|
98
124
|
--[[
|
|
99
125
|
Function to store a deduplicated next job if the existing job is active
|
|
100
126
|
and keepLastIfActive is set. When the active job finishes, the stored
|
|
@@ -161,6 +187,10 @@ local function deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
|
161
187
|
if deduplicationOpts['extend'] then
|
|
162
188
|
local currentDeduplicatedJobId = rcall('GET', deduplicationKey)
|
|
163
189
|
if currentDeduplicatedJobId then
|
|
190
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
191
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
192
|
+
return
|
|
193
|
+
end
|
|
164
194
|
if storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
165
195
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
166
196
|
parentKey, parentData, parentDependenciesKey, repeatJobKey) then
|
|
@@ -194,6 +224,10 @@ local function deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
|
194
224
|
end
|
|
195
225
|
if deduplicationKeyExists then
|
|
196
226
|
local currentDeduplicatedJobId = rcall('GET', deduplicationKey)
|
|
227
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
228
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
229
|
+
return
|
|
230
|
+
end
|
|
197
231
|
if storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
198
232
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
199
233
|
parentKey, parentData, parentDependenciesKey, repeatJobKey) then
|
|
@@ -233,6 +267,10 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
233
267
|
local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
234
268
|
currentDeduplicatedJobId, jobId, deduplicationId, prefix)
|
|
235
269
|
if isRemoved then
|
|
270
|
+
-- Discard any pending next-job payload stored while the replaced
|
|
271
|
+
-- job was active, otherwise it would be resurrected when the
|
|
272
|
+
-- incoming job finalizes.
|
|
273
|
+
rcall('DEL', prefix .. "dn:" .. deduplicationId)
|
|
236
274
|
if deduplicationOpts['keepLastIfActive'] then
|
|
237
275
|
rcall('SET', deduplicationKey, jobId)
|
|
238
276
|
else
|
|
@@ -245,6 +283,10 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
245
283
|
end
|
|
246
284
|
return
|
|
247
285
|
else
|
|
286
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
287
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
288
|
+
return
|
|
289
|
+
end
|
|
248
290
|
storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
249
291
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
250
292
|
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
@@ -80,6 +80,15 @@ end
|
|
|
80
80
|
--[[
|
|
81
81
|
Function to deduplicate a job.
|
|
82
82
|
]]
|
|
83
|
+
--[[
|
|
84
|
+
Function to recover a stale deduplication key.
|
|
85
|
+
When the existing deduplication key has no expiry (simple mode, or
|
|
86
|
+
keepLastIfActive after the key was persisted), it should disappear
|
|
87
|
+
when the winning job finishes. If that job key no longer exists
|
|
88
|
+
because of an outage, the deduplication key is stale, so we discard it
|
|
89
|
+
and start a new deduplication window with the incoming job.
|
|
90
|
+
Returns true if the stale key was recovered, false otherwise.
|
|
91
|
+
]]
|
|
83
92
|
--[[
|
|
84
93
|
Function to set the deduplication key for a job.
|
|
85
94
|
Uses TTL from deduplication opts if provided.
|
|
@@ -92,6 +101,23 @@ local function setDeduplicationKey(deduplicationKey, jobId, deduplicationOpts)
|
|
|
92
101
|
rcall('SET', deduplicationKey, jobId)
|
|
93
102
|
end
|
|
94
103
|
end
|
|
104
|
+
local function recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId, jobId,
|
|
105
|
+
deduplicationId, deduplicationOpts)
|
|
106
|
+
if currentDeduplicatedJobId and rcall('PTTL', deduplicationKey) == -1 and
|
|
107
|
+
rcall('EXISTS', prefix .. currentDeduplicatedJobId) == 0 then
|
|
108
|
+
rcall('DEL', deduplicationKey)
|
|
109
|
+
-- Discard any pending next-job payload stored for the stale winner,
|
|
110
|
+
-- otherwise it would be resurrected when this job finalizes.
|
|
111
|
+
rcall('DEL', prefix .. "dn:" .. deduplicationId)
|
|
112
|
+
if deduplicationOpts['keepLastIfActive'] then
|
|
113
|
+
rcall('SET', deduplicationKey, jobId)
|
|
114
|
+
else
|
|
115
|
+
setDeduplicationKey(deduplicationKey, jobId, deduplicationOpts)
|
|
116
|
+
end
|
|
117
|
+
return true
|
|
118
|
+
end
|
|
119
|
+
return false
|
|
120
|
+
end
|
|
95
121
|
--[[
|
|
96
122
|
Function to store a deduplicated next job if the existing job is active
|
|
97
123
|
and keepLastIfActive is set. When the active job finishes, the stored
|
|
@@ -158,6 +184,10 @@ local function deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
|
158
184
|
if deduplicationOpts['extend'] then
|
|
159
185
|
local currentDeduplicatedJobId = rcall('GET', deduplicationKey)
|
|
160
186
|
if currentDeduplicatedJobId then
|
|
187
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
188
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
189
|
+
return
|
|
190
|
+
end
|
|
161
191
|
if storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
162
192
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
163
193
|
parentKey, parentData, parentDependenciesKey, repeatJobKey) then
|
|
@@ -191,6 +221,10 @@ local function deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
|
191
221
|
end
|
|
192
222
|
if deduplicationKeyExists then
|
|
193
223
|
local currentDeduplicatedJobId = rcall('GET', deduplicationKey)
|
|
224
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
225
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
226
|
+
return
|
|
227
|
+
end
|
|
194
228
|
if storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
195
229
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
196
230
|
parentKey, parentData, parentDependenciesKey, repeatJobKey) then
|
|
@@ -230,6 +264,10 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
230
264
|
local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
231
265
|
currentDeduplicatedJobId, jobId, deduplicationId, prefix)
|
|
232
266
|
if isRemoved then
|
|
267
|
+
-- Discard any pending next-job payload stored while the replaced
|
|
268
|
+
-- job was active, otherwise it would be resurrected when the
|
|
269
|
+
-- incoming job finalizes.
|
|
270
|
+
rcall('DEL', prefix .. "dn:" .. deduplicationId)
|
|
233
271
|
if deduplicationOpts['keepLastIfActive'] then
|
|
234
272
|
rcall('SET', deduplicationKey, jobId)
|
|
235
273
|
else
|
|
@@ -242,6 +280,10 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
242
280
|
end
|
|
243
281
|
return
|
|
244
282
|
else
|
|
283
|
+
if recoverStaleDeduplicationKey(deduplicationKey, prefix, currentDeduplicatedJobId,
|
|
284
|
+
jobId, deduplicationId, deduplicationOpts) then
|
|
285
|
+
return
|
|
286
|
+
end
|
|
245
287
|
storeDeduplicatedNextJob(deduplicationOpts, currentDeduplicatedJobId, prefix,
|
|
246
288
|
deduplicationId, jobName, jobData, fullOpts, eventsKey, maxEvents, jobId,
|
|
247
289
|
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|