bullmq 6.5.3 → 6.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/commands/addDelayedJob-6.lua +1 -1
- package/dist/cjs/commands/addPrioritizedJob-9.lua +1 -1
- package/dist/cjs/commands/addStandardJob-9.lua +1 -1
- package/dist/cjs/commands/includes/deduplicateJob.lua +11 -2
- package/dist/cjs/postgres/migrations/index.js +36 -0
- package/dist/cjs/postgres/postgres-queue-backend.js +3 -2
- package/dist/cjs/postgres/sql-scripts.js +6 -0
- package/dist/cjs/scripts/addDelayedJob-6.js +11 -3
- package/dist/cjs/scripts/addPrioritizedJob-9.js +75 -27
- package/dist/cjs/scripts/addStandardJob-9.js +75 -27
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/commands/addDelayedJob-6.lua +1 -1
- package/dist/esm/commands/addPrioritizedJob-9.lua +1 -1
- package/dist/esm/commands/addStandardJob-9.lua +1 -1
- package/dist/esm/commands/includes/deduplicateJob.lua +11 -2
- package/dist/esm/postgres/migrations/index.js +36 -0
- package/dist/esm/postgres/postgres-queue-backend.js +3 -2
- package/dist/esm/postgres/sql-scripts.js +6 -0
- package/dist/esm/scripts/addDelayedJob-6.js +11 -3
- package/dist/esm/scripts/addPrioritizedJob-9.js +75 -27
- package/dist/esm/scripts/addStandardJob-9.js +75 -27
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/deduplication-options.d.ts +7 -0
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +8 -9
|
@@ -92,7 +92,7 @@ end
|
|
|
92
92
|
|
|
93
93
|
local deduplicationJobId = deduplicateJob(opts['de'], jobId, delayedKey, deduplicationKey,
|
|
94
94
|
eventsKey, maxEvents, args[1], args[3], ARGV[2], opts,
|
|
95
|
-
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
95
|
+
parentKey, parentData, parentDependenciesKey, repeatJobKey, timestamp)
|
|
96
96
|
if deduplicationJobId then
|
|
97
97
|
return deduplicationJobId
|
|
98
98
|
end
|
|
@@ -94,7 +94,7 @@ end
|
|
|
94
94
|
|
|
95
95
|
local deduplicationJobId = deduplicateJob(opts['de'], jobId, KEYS[5],
|
|
96
96
|
deduplicationKey, eventsKey, maxEvents, args[1], args[3], ARGV[2], opts,
|
|
97
|
-
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
97
|
+
parentKey, parentData, parentDependenciesKey, repeatJobKey, timestamp)
|
|
98
98
|
if deduplicationJobId then
|
|
99
99
|
return deduplicationJobId
|
|
100
100
|
end
|
|
@@ -100,7 +100,7 @@ end
|
|
|
100
100
|
|
|
101
101
|
local deduplicationJobId = deduplicateJob(opts['de'], jobId, KEYS[6],
|
|
102
102
|
deduplicationKey, eventsKey, maxEvents, args[1], args[3], ARGV[2], opts,
|
|
103
|
-
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
103
|
+
parentKey, parentData, parentDependenciesKey, repeatJobKey, timestamp)
|
|
104
104
|
if deduplicationJobId then
|
|
105
105
|
return deduplicationJobId
|
|
106
106
|
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Function to debounce a job.
|
|
3
3
|
]]
|
|
4
4
|
-- Includes
|
|
5
|
+
--- @include "addDelayedJob"
|
|
5
6
|
--- @include "deduplicateJobWithoutReplace"
|
|
6
7
|
--- @include "recoverStaleDeduplicationKey"
|
|
7
8
|
--- @include "removeJobKeys"
|
|
@@ -25,7 +26,7 @@ local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEven
|
|
|
25
26
|
end
|
|
26
27
|
|
|
27
28
|
local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
28
|
-
prefix, jobName, jobData, fullOpts, parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
29
|
+
prefix, jobName, jobData, fullOpts, parentKey, parentData, parentDependenciesKey, repeatJobKey, timestamp)
|
|
29
30
|
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
|
30
31
|
if deduplicationId then
|
|
31
32
|
if deduplicationOpts['replace'] then
|
|
@@ -70,9 +71,17 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
70
71
|
return
|
|
71
72
|
end
|
|
72
73
|
else
|
|
73
|
-
|
|
74
|
+
local currentDeduplicatedJobId = deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
74
75
|
jobId, deduplicationKey, eventsKey, maxEvents, prefix, jobName, jobData, fullOpts,
|
|
75
76
|
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
77
|
+
if currentDeduplicatedJobId and deduplicationOpts['reschedule'] and
|
|
78
|
+
rcall('ZSCORE', delayedKey, currentDeduplicatedJobId) then
|
|
79
|
+
local delay = tonumber(fullOpts['delay']) or 0
|
|
80
|
+
rcall('HSET', prefix .. currentDeduplicatedJobId, 'delay', delay)
|
|
81
|
+
addDelayedJob(currentDeduplicatedJobId, delayedKey, eventsKey, timestamp,
|
|
82
|
+
maxEvents, prefix .. 'marker', delay)
|
|
83
|
+
end
|
|
84
|
+
return currentDeduplicatedJobId
|
|
76
85
|
end
|
|
77
86
|
end
|
|
78
87
|
end
|
|
@@ -45,6 +45,42 @@ exports.MIGRATIONS = [
|
|
|
45
45
|
minClientVersion: 6,
|
|
46
46
|
load: () => (0, sql_loader_1.loadMigrationSql)('0005_flow_parent_release.sql'),
|
|
47
47
|
},
|
|
48
|
+
{
|
|
49
|
+
version: 6,
|
|
50
|
+
name: '0006_dedup_reschedule',
|
|
51
|
+
minClientVersion: 6,
|
|
52
|
+
load: () => (0, sql_loader_1.loadMigrationSql)('0006_dedup_reschedule.sql'),
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
version: 7,
|
|
56
|
+
name: '0007_obliterate_dedup_lock_order',
|
|
57
|
+
minClientVersion: 6,
|
|
58
|
+
load: () => (0, sql_loader_1.loadMigrationSql)('0007_obliterate_dedup_lock_order.sql'),
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
version: 8,
|
|
62
|
+
name: '0008_bulk_dedup_lock_order',
|
|
63
|
+
minClientVersion: 6,
|
|
64
|
+
load: () => (0, sql_loader_1.loadMigrationSql)('0008_bulk_dedup_lock_order.sql'),
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
version: 9,
|
|
68
|
+
name: '0009_obliterate_bounded_dedup_locks',
|
|
69
|
+
minClientVersion: 6,
|
|
70
|
+
load: () => (0, sql_loader_1.loadMigrationSql)('0009_obliterate_bounded_dedup_locks.sql'),
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
version: 10,
|
|
74
|
+
name: '0010_obliterate_finalization_lock',
|
|
75
|
+
minClientVersion: 6,
|
|
76
|
+
load: () => (0, sql_loader_1.loadMigrationSql)('0010_obliterate_finalization_lock.sql'),
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
version: 11,
|
|
80
|
+
name: '0011_obliterate_producer_lock',
|
|
81
|
+
minClientVersion: 6,
|
|
82
|
+
load: () => (0, sql_loader_1.loadMigrationSql)('0011_obliterate_producer_lock.sql'),
|
|
83
|
+
},
|
|
48
84
|
];
|
|
49
85
|
/**
|
|
50
86
|
* The highest schema version this BullMQ build knows how to produce. Explicit
|
|
@@ -411,6 +411,7 @@ class PostgresQueueBackend extends node_events_1.EventEmitter {
|
|
|
411
411
|
async addJob(job, jobId, parentKeyOpts = {}) {
|
|
412
412
|
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
|
|
413
413
|
const opts = ((_a = job.opts) !== null && _a !== void 0 ? _a : {});
|
|
414
|
+
const jobOpts = job.delay == null ? opts : Object.assign(Object.assign({}, opts), { delay: job.delay });
|
|
414
415
|
const parentKey = (_c = (_b = parentKeyOpts.parentKey) !== null && _b !== void 0 ? _b : job.parentKey) !== null && _c !== void 0 ? _c : null;
|
|
415
416
|
let rows;
|
|
416
417
|
try {
|
|
@@ -419,7 +420,7 @@ class PostgresQueueBackend extends node_events_1.EventEmitter {
|
|
|
419
420
|
jobId || job.id || '',
|
|
420
421
|
job.name,
|
|
421
422
|
(_d = job.data) !== null && _d !== void 0 ? _d : '{}',
|
|
422
|
-
JSON.stringify(
|
|
423
|
+
JSON.stringify(jobOpts !== null && jobOpts !== void 0 ? jobOpts : {}),
|
|
423
424
|
(_f = (_e = job.priority) !== null && _e !== void 0 ? _e : opts.priority) !== null && _f !== void 0 ? _f : 0,
|
|
424
425
|
(_h = (_g = job.delay) !== null && _g !== void 0 ? _g : opts.delay) !== null && _h !== void 0 ? _h : 0,
|
|
425
426
|
(_j = job.timestamp) !== null && _j !== void 0 ? _j : Date.now(),
|
|
@@ -475,7 +476,7 @@ class PostgresQueueBackend extends node_events_1.EventEmitter {
|
|
|
475
476
|
id: jobId || data.id || '',
|
|
476
477
|
name: data.name,
|
|
477
478
|
data: (_b = data.data) !== null && _b !== void 0 ? _b : '{}',
|
|
478
|
-
opts,
|
|
479
|
+
opts: data.delay == null ? opts : Object.assign(Object.assign({}, opts), { delay: data.delay }),
|
|
479
480
|
priority: (_d = (_c = data.priority) !== null && _c !== void 0 ? _c : opts.priority) !== null && _d !== void 0 ? _d : 0,
|
|
480
481
|
delay: (_f = (_e = data.delay) !== null && _e !== void 0 ? _e : opts.delay) !== null && _f !== void 0 ? _f : 0,
|
|
481
482
|
timestamp: (_g = data.timestamp) !== null && _g !== void 0 ? _g : Date.now(),
|
|
@@ -127,4 +127,10 @@ exports.SQL_MIGRATIONS = {
|
|
|
127
127
|
"0003_dedup_stale_key.sql": "-- BullMQ PostgreSQL backend — deduplication stale-key recovery.\n--\n-- Replaces `deduplicate_job` (created in 0002_functions.sql) to fix two ways a\n-- deduplication key could outlive the job it points at:\n--\n-- 1. Stale key: a key with no expiry is meant to disappear when its winner\n-- finishes, but `clean`, `drain`, `obliterate` (and outages) delete the\n-- job row without clearing the `dedup` row. Every later add then matched\n-- that dead winner and was silently dropped, forever. Mirrors the Redis\n-- `recoverStaleDeduplicationKey` include.\n-- 2. Orphaned proto-next: replacing a *delayed* winner left behind any\n-- `dedup_next` payload stashed while that winner was previously active,\n-- so it got resurrected when the replacement finished.\n--\n-- Only the function body changes — no schema/DDL change — so the signature is\n-- identical and older clients keep working against the updated schema.\n\nCREATE OR REPLACE FUNCTION deduplicate_job(\n p_queue text, p_dedup jsonb, p_job_id text, p_now bigint,\n p_name text, p_data jsonb, p_opts jsonb\n) RETURNS text\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nDECLARE\n v_id text := p_dedup->>'id';\n v_ttl bigint := NULLIF(p_dedup->>'ttl', '')::bigint;\n v_extend boolean := COALESCE((p_dedup->>'extend')::boolean, false);\n v_replace boolean := COALESCE((p_dedup->>'replace')::boolean, false);\n v_keeplast boolean := COALESCE((p_dedup->>'keepLastIfActive')::boolean, false);\n v_cur text;\n v_exp bigint;\n v_state job_state;\nBEGIN\n IF v_id IS NULL OR v_id = '' THEN\n RETURN NULL;\n END IF;\n\n -- The current winner (only if the key is still live).\n SELECT job_id, expire_at_ms INTO v_cur, v_exp\n FROM dedup WHERE queue = p_queue AND dedup_id = v_id;\n IF v_cur IS NULL OR (v_exp IS NOT NULL AND v_exp <= p_now) THEN\n v_cur := NULL;\n ELSIF v_exp IS NULL AND NOT EXISTS (\n SELECT 1 FROM job WHERE queue = p_queue AND id = v_cur\n ) THEN\n -- Stale key recovery (mirrors recoverStaleDeduplicationKey): a key with no\n -- expiry is meant to disappear when its winner finishes. If the job row is\n -- gone (clean, drain, obliterate, an outage…) the key is stale, so discard\n -- it together with any pending proto-next (which would otherwise be\n -- resurrected) and start a new window with the incoming job.\n DELETE FROM dedup WHERE queue = p_queue AND dedup_id = v_id;\n DELETE FROM dedup_next WHERE queue = p_queue AND dedup_id = v_id;\n v_cur := NULL;\n END IF;\n\n IF v_replace THEN\n IF v_cur IS NOT NULL THEN\n SELECT state INTO v_state\n FROM job WHERE queue = p_queue AND id = v_cur;\n IF v_state = 'delayed' THEN\n -- Drop the previous delayed job and take its place.\n DELETE FROM job WHERE queue = p_queue AND id = v_cur;\n PERFORM publish_event(p_queue, 'removed',\n jsonb_build_object('jobId', v_cur, 'prev', 'delayed'));\n PERFORM publish_event(p_queue, 'deduplicated',\n jsonb_build_object('jobId', p_job_id, 'deduplicationId', v_id,\n 'deduplicatedJobId', v_cur));\n -- Discard any pending proto-next stashed while the replaced job was\n -- active, otherwise it would be resurrected when p_job_id finalizes.\n DELETE FROM dedup_next WHERE queue = p_queue AND dedup_id = v_id;\n IF v_keeplast THEN\n UPDATE dedup SET job_id = p_job_id, expire_at_ms = NULL\n WHERE queue = p_queue AND dedup_id = v_id;\n ELSIF NOT v_extend AND COALESCE(v_ttl, 0) > 0 THEN\n -- KEEPTTL: keep the existing window, just swap the winner.\n UPDATE dedup SET job_id = p_job_id\n WHERE queue = p_queue AND dedup_id = v_id;\n ELSE\n UPDATE dedup\n SET job_id = p_job_id,\n expire_at_ms = CASE WHEN COALESCE(v_ttl, 0) > 0\n THEN p_now + v_ttl ELSE NULL END\n WHERE queue = p_queue AND dedup_id = v_id;\n END IF;\n RETURN NULL;\n ELSE\n -- Winner is not a removable delayed job: stash proto-next if it is\n -- active + keepLastIfActive, then deduplicate.\n PERFORM dedup_store_next(p_queue, v_id, v_cur, p_job_id,\n v_keeplast, p_name, p_data, p_opts);\n RETURN v_cur;\n END IF;\n ELSE\n INSERT INTO dedup (queue, dedup_id, job_id, expire_at_ms)\n VALUES (p_queue, v_id, p_job_id,\n CASE WHEN NOT v_keeplast AND COALESCE(v_ttl, 0) > 0\n THEN p_now + v_ttl ELSE NULL END)\n ON CONFLICT (queue, dedup_id) DO UPDATE\n SET job_id = EXCLUDED.job_id, expire_at_ms = EXCLUDED.expire_at_ms;\n RETURN NULL;\n END IF;\n END IF;\n\n -- Without replace.\n IF COALESCE(v_ttl, 0) > 0 AND v_extend THEN\n IF v_cur IS NOT NULL THEN\n -- Stash proto-next if active+keepLast; else extend the window (or\n -- persist when keepLastIfActive). Either way keep the current winner.\n IF NOT dedup_store_next(p_queue, v_id, v_cur, p_job_id,\n v_keeplast, p_name, p_data, p_opts) THEN\n UPDATE dedup\n SET expire_at_ms = CASE WHEN v_keeplast THEN NULL ELSE p_now + v_ttl END\n WHERE queue = p_queue AND dedup_id = v_id;\n END IF;\n PERFORM publish_event(p_queue, 'deduplicated',\n jsonb_build_object('jobId', v_cur, 'deduplicationId', v_id,\n 'deduplicatedJobId', p_job_id));\n RETURN v_cur;\n END IF;\n INSERT INTO dedup (queue, dedup_id, job_id, expire_at_ms)\n VALUES (p_queue, v_id, p_job_id,\n CASE WHEN v_keeplast THEN NULL ELSE p_now + v_ttl END)\n ON CONFLICT (queue, dedup_id) DO UPDATE\n SET job_id = EXCLUDED.job_id, expire_at_ms = EXCLUDED.expire_at_ms;\n RETURN NULL;\n END IF;\n\n -- SET NX semantics (ttl>0 non-extend, or no ttl at all).\n IF v_cur IS NOT NULL THEN\n PERFORM dedup_store_next(p_queue, v_id, v_cur, p_job_id,\n v_keeplast, p_name, p_data, p_opts);\n PERFORM publish_event(p_queue, 'deduplicated',\n jsonb_build_object('jobId', v_cur, 'deduplicationId', v_id,\n 'deduplicatedJobId', p_job_id));\n RETURN v_cur;\n END IF;\n\n INSERT INTO dedup (queue, dedup_id, job_id, expire_at_ms)\n VALUES (p_queue, v_id, p_job_id,\n CASE WHEN NOT v_keeplast AND COALESCE(v_ttl, 0) > 0\n THEN p_now + v_ttl ELSE NULL END)\n ON CONFLICT (queue, dedup_id) DO UPDATE\n SET job_id = EXCLUDED.job_id, expire_at_ms = EXCLUDED.expire_at_ms;\n RETURN NULL;\nEND;\n$$;\n",
|
|
128
128
|
"0004_relay.sql": "-- Relay: messaging between the nodes of a fleet (see the Relay class).\n--\n-- Additive migration: it only creates new relay_* objects and does not touch\n-- the queue schema, so clients that don't use the relay are unaffected.\n--\n-- All routing logic lives here (and in the equivalent Lua scripts), so every\n-- BullMQ runtime gets identical semantics. Error convention, as for the queue\n-- functions: RAISE ... USING ERRCODE = 'BM001', DETAIL = '<negative code>'.\n-- -1 invalid topic, -2 invalid pattern, -3 node not registered,\n-- -4 data too large, -5 invalid id.\n\nCREATE TABLE relay_node (\n ns text NOT NULL,\n node_id text NOT NULL,\n lease_until timestamptz NOT NULL,\n registered_at timestamptz NOT NULL DEFAULT clock_timestamp(),\n last_heartbeat timestamptz,\n -- As given by the node (name, host, pid, version...), for dashboards.\n info jsonb NOT NULL DEFAULT '{}',\n -- Unread inbox entries as of the last trim, and entries dropped by trims.\n backlog integer NOT NULL DEFAULT 0,\n dropped bigint NOT NULL DEFAULT 0,\n -- Ids of the inbox batch last fetched by the node (in flight until it is\n -- acknowledged): trims may remove them, but they are not dropped.\n in_flight bigint[] NOT NULL DEFAULT '{}',\n PRIMARY KEY (ns, node_id)\n);\n\n-- Sweeps remove expired nodes of every namespace, so an abandoned namespace\n-- is cleaned by any node using the same schema.\nCREATE INDEX relay_node_lease_idx ON relay_node (lease_until);\nCREATE INDEX relay_node_registered_idx ON relay_node (ns, registered_at);\n\nCREATE TABLE relay_subscription (\n ns text NOT NULL,\n node_id text NOT NULL,\n endpoint_id text NOT NULL,\n pattern text NOT NULL,\n -- Wildcard patterns only: their literal first segment, or '*' for wildcard\n -- roots. NULL for exact topics, which publishes find by pattern directly.\n root text,\n PRIMARY KEY (ns, pattern, node_id, endpoint_id),\n FOREIGN KEY (ns, node_id) REFERENCES relay_node (ns, node_id) ON DELETE CASCADE\n);\n\nCREATE INDEX relay_subscription_root_idx ON relay_subscription (ns, root);\nCREATE INDEX relay_subscription_endpoint_idx\n ON relay_subscription (ns, node_id, endpoint_id);\n\nCREATE TABLE relay_inbox (\n id bigserial PRIMARY KEY,\n ns text NOT NULL,\n node_id text NOT NULL,\n kind text NOT NULL,\n topic text NOT NULL,\n mid bigint NOT NULL,\n ts bigint NOT NULL,\n data text NOT NULL,\n endpoints text[] NOT NULL,\n FOREIGN KEY (ns, node_id) REFERENCES relay_node (ns, node_id) ON DELETE CASCADE\n);\n\nCREATE INDEX relay_inbox_node_idx ON relay_inbox (ns, node_id, id);\n\nCREATE TABLE relay_retained (\n ns text NOT NULL,\n topic text NOT NULL,\n mid bigint NOT NULL,\n ts bigint NOT NULL,\n data text NOT NULL,\n expires_at timestamptz NOT NULL,\n PRIMARY KEY (ns, topic)\n);\n\n-- Retained messages expire even if their namespace has no nodes left.\nCREATE INDEX relay_retained_expiry_idx ON relay_retained (expires_at);\n\nCREATE SEQUENCE relay_mid_seq;\n\n-- Returns the segments of a valid topic (or pattern), or NULL if invalid.\n-- Same grammar as includes/relayTopics.lua.\nCREATE FUNCTION relay_parse_topic(p_value text, p_allow_wildcards boolean)\nRETURNS text[]\nLANGUAGE plpgsql IMMUTABLE AS $$\nDECLARE\n v_segments text[];\n v_count integer;\n v_segment text;\nBEGIN\n IF p_value IS NULL OR octet_length(p_value) = 0\n OR octet_length(p_value) > 512 THEN\n RETURN NULL;\n END IF;\n v_segments := string_to_array(p_value, '.');\n v_count := cardinality(v_segments);\n IF v_count > 16 THEN\n RETURN NULL;\n END IF;\n FOR i IN 1..v_count LOOP\n v_segment := v_segments[i];\n IF p_allow_wildcards AND v_segment = '*' THEN\n CONTINUE;\n END IF;\n IF p_allow_wildcards AND v_segment = '>' THEN\n IF i <> v_count THEN\n RETURN NULL;\n END IF;\n CONTINUE;\n END IF;\n IF v_segment !~ '^[A-Za-z0-9_:%-]+$' THEN\n RETURN NULL;\n END IF;\n END LOOP;\n RETURN v_segments;\nEND;\n$$;\n\nCREATE FUNCTION relay_match_topic(p_pattern text[], p_topic text[])\nRETURNS boolean\nLANGUAGE plpgsql IMMUTABLE AS $$\nDECLARE\n v_topic_count integer := cardinality(p_topic);\n v_pattern_count integer := cardinality(p_pattern);\nBEGIN\n FOR i IN 1..v_pattern_count LOOP\n IF p_pattern[i] = '>' THEN\n RETURN v_topic_count >= i;\n END IF;\n IF i > v_topic_count THEN\n RETURN false;\n END IF;\n IF p_pattern[i] <> '*' AND p_pattern[i] <> p_topic[i] THEN\n RETURN false;\n END IF;\n END LOOP;\n RETURN v_pattern_count = v_topic_count;\nEND;\n$$;\n\nCREATE FUNCTION relay_is_valid_id(p_id text)\nRETURNS boolean\nLANGUAGE sql IMMUTABLE AS $$\n SELECT p_id IS NOT NULL AND p_id ~ '^[A-Za-z0-9_-]{1,64}$';\n$$;\n\n-- Registers (or refreshes) a node. Reusing an expired, unswept node id\n-- starts with a clean inbox and subscriptions; refreshing a live registration\n-- preserves both.\nCREATE FUNCTION relay_register_node(\n p_ns text, p_node text, p_lease_ms bigint, p_info jsonb\n)\nRETURNS void\nLANGUAGE plpgsql AS $$\nBEGIN\n IF NOT relay_is_valid_id(p_node) THEN\n RAISE EXCEPTION 'bullmq: relay invalid node id %', p_node\n USING ERRCODE = 'BM001', DETAIL = '-5';\n END IF;\n DELETE FROM relay_node\n WHERE ns = p_ns AND node_id = p_node\n AND lease_until <= clock_timestamp();\n INSERT INTO relay_node (ns, node_id, lease_until, last_heartbeat, info)\n VALUES (\n p_ns, p_node, clock_timestamp() + p_lease_ms * interval '1 millisecond',\n clock_timestamp(), p_info\n )\n ON CONFLICT (ns, node_id)\n DO UPDATE SET lease_until = EXCLUDED.lease_until,\n last_heartbeat = EXCLUDED.last_heartbeat,\n info = EXCLUDED.info;\nEND;\n$$;\n\n-- Renews a node's lease. Returns false when the node is not registered or\n-- its lease expired (even if not swept yet): the caller must register again\n-- and restore its subscriptions.\nCREATE FUNCTION relay_heartbeat(p_ns text, p_node text, p_lease_ms bigint)\nRETURNS boolean\nLANGUAGE plpgsql AS $$\nBEGIN\n UPDATE relay_node\n SET lease_until = clock_timestamp() + p_lease_ms * interval '1 millisecond',\n last_heartbeat = clock_timestamp()\n WHERE ns = p_ns AND node_id = p_node\n AND lease_until > clock_timestamp();\n RETURN FOUND;\nEND;\n$$;\n\nCREATE FUNCTION relay_unregister_node(p_ns text, p_node text)\nRETURNS boolean\nLANGUAGE plpgsql AS $$\nBEGIN\n DELETE FROM relay_node WHERE ns = p_ns AND node_id = p_node;\n RETURN FOUND;\nEND;\n$$;\n\n-- Removes up to p_limit expired nodes (their subscriptions and inbox go with\n-- them) and up to p_limit expired retained messages, of every namespace, so\n-- abandoned namespaces are cleaned too. Returns the removed nodes of p_ns.\nCREATE FUNCTION relay_sweep(p_ns text, p_limit integer)\nRETURNS SETOF text\nLANGUAGE plpgsql AS $$\nBEGIN\n WITH expired AS (\n SELECT r.ns, r.topic\n FROM relay_retained r\n WHERE r.expires_at < statement_timestamp()\n ORDER BY r.expires_at\n LIMIT p_limit\n FOR UPDATE SKIP LOCKED\n )\n DELETE FROM relay_retained r\n USING expired e\n WHERE r.ns = e.ns AND r.topic = e.topic;\n\n RETURN QUERY\n WITH expired AS (\n SELECT n.ns, n.node_id\n FROM relay_node n\n WHERE n.lease_until < clock_timestamp()\n ORDER BY n.lease_until\n LIMIT p_limit\n FOR UPDATE SKIP LOCKED\n ), removed AS (\n DELETE FROM relay_node n\n USING expired e\n WHERE n.ns = e.ns AND n.node_id = e.node_id\n RETURNING n.ns, n.node_id\n )\n SELECT removed.node_id FROM removed WHERE removed.ns = p_ns;\nEND;\n$$;\n\n-- Subscribes an endpoint to a pattern. For an exact topic the retained\n-- message, if any, is returned atomically with the subscription.\n--\n-- Exact subscriptions and publishes on the same topic are serialized by an\n-- advisory lock taken before either reads routing or retained state; each\n-- statement after the lock gets a fresh READ COMMITTED snapshot, including\n-- changes committed by the previous lock holder.\nCREATE FUNCTION relay_subscribe(\n p_ns text, p_node text, p_endpoint text, p_pattern text\n)\nRETURNS TABLE (added boolean, retained_mid bigint, retained_ts bigint, retained_data text)\nLANGUAGE plpgsql AS $$\nDECLARE\n v_segments text[];\n v_has_wildcards boolean;\nBEGIN\n IF NOT relay_is_valid_id(p_node) OR NOT relay_is_valid_id(p_endpoint) THEN\n RAISE EXCEPTION 'bullmq: relay invalid id'\n USING ERRCODE = 'BM001', DETAIL = '-5';\n END IF;\n v_segments := relay_parse_topic(p_pattern, true);\n IF v_segments IS NULL THEN\n RAISE EXCEPTION 'bullmq: relay invalid pattern %', p_pattern\n USING ERRCODE = 'BM001', DETAIL = '-2';\n END IF;\n v_has_wildcards := '*' = ANY (v_segments) OR '>' = ANY (v_segments);\n IF NOT v_has_wildcards THEN\n PERFORM pg_advisory_xact_lock(hashtext(p_ns), hashtext(p_pattern));\n END IF;\n\n -- Keep the live node registered until its subscription is stored.\n PERFORM 1 FROM relay_node\n WHERE ns = p_ns AND node_id = p_node\n AND lease_until > clock_timestamp()\n FOR KEY SHARE;\n IF NOT FOUND THEN\n RAISE EXCEPTION 'bullmq: relay node % not registered', p_node\n USING ERRCODE = 'BM001', DETAIL = '-3';\n END IF;\n\n INSERT INTO relay_subscription (ns, node_id, endpoint_id, pattern, root)\n VALUES (\n p_ns, p_node, p_endpoint, p_pattern,\n CASE\n WHEN NOT v_has_wildcards THEN NULL\n WHEN v_segments[1] IN ('*', '>') THEN '*'\n ELSE v_segments[1]\n END\n )\n ON CONFLICT DO NOTHING;\n added := FOUND;\n\n IF NOT v_has_wildcards THEN\n SELECT r.mid, r.ts, r.data\n INTO retained_mid, retained_ts, retained_data\n FROM relay_retained r\n WHERE r.ns = p_ns AND r.topic = p_pattern\n AND r.expires_at >= clock_timestamp();\n END IF;\n RETURN NEXT;\nEND;\n$$;\n\nCREATE FUNCTION relay_remove_endpoint(p_ns text, p_node text, p_endpoint text)\nRETURNS integer\nLANGUAGE plpgsql AS $$\nDECLARE\n v_count integer;\nBEGIN\n WITH removed AS (\n DELETE FROM relay_subscription\n WHERE ns = p_ns AND node_id = p_node AND endpoint_id = p_endpoint\n RETURNING 1\n )\n SELECT count(*) INTO v_count FROM removed;\n RETURN v_count;\nEND;\n$$;\n\n-- Publishes a message: one inbox row per live node with matching\n-- subscriptions, listing that node's endpoints (each endpoint once), and a\n-- NOTIFY per node to wake it up.\nCREATE FUNCTION relay_publish(\n p_ns text, p_topic text, p_data text, p_ts bigint,\n p_retain_ms bigint, p_max_size integer\n)\nRETURNS TABLE (message_id bigint, node_count integer, endpoint_count integer)\nLANGUAGE plpgsql AS $$\nDECLARE\n v_segments text[];\n v_row record;\nBEGIN\n v_segments := relay_parse_topic(p_topic, false);\n IF v_segments IS NULL THEN\n RAISE EXCEPTION 'bullmq: relay invalid topic %', p_topic\n USING ERRCODE = 'BM001', DETAIL = '-1';\n END IF;\n IF octet_length(p_data) > p_max_size THEN\n RAISE EXCEPTION 'bullmq: relay data too large'\n USING ERRCODE = 'BM001', DETAIL = '-4';\n END IF;\n\n -- See relay_subscribe.\n PERFORM pg_advisory_xact_lock(hashtext(p_ns), hashtext(p_topic));\n message_id := nextval('relay_mid_seq');\n node_count := 0;\n endpoint_count := 0;\n\n IF p_retain_ms > 0 THEN\n INSERT INTO relay_retained (ns, topic, mid, ts, data, expires_at)\n VALUES (\n p_ns, p_topic, message_id, p_ts, p_data,\n clock_timestamp() + p_retain_ms * interval '1 millisecond'\n )\n ON CONFLICT (ns, topic) DO UPDATE\n SET mid = EXCLUDED.mid, ts = EXCLUDED.ts, data = EXCLUDED.data,\n expires_at = EXCLUDED.expires_at;\n END IF;\n\n FOR v_row IN\n WITH matched AS (\n -- Exact subscriptions: a primary-key lookup, whatever their number.\n SELECT s.node_id, s.endpoint_id\n FROM relay_subscription s\n WHERE s.ns = p_ns AND s.pattern = p_topic\n UNION ALL\n -- Wildcard subscriptions: only those indexed under the topic's first\n -- segment (or a wildcard root) are matched one by one.\n SELECT s.node_id, s.endpoint_id\n FROM relay_subscription s\n WHERE s.ns = p_ns\n AND s.root IN (v_segments[1], '*')\n AND relay_match_topic(string_to_array(s.pattern, '.'), v_segments)\n ), live AS (\n -- Locks the target nodes so a concurrent sweep can't delete them before\n -- their inbox rows are inserted (the sweep skips locked nodes; a node it\n -- deleted first is simply not returned here).\n SELECT n.node_id\n FROM relay_node n\n WHERE n.ns = p_ns\n AND n.node_id IN (SELECT matched.node_id FROM matched)\n AND n.lease_until >= clock_timestamp()\n FOR KEY SHARE\n )\n INSERT INTO relay_inbox (ns, node_id, kind, topic, mid, ts, data, endpoints)\n SELECT p_ns, m.node_id, 'msg', p_topic, message_id, p_ts, p_data,\n array_agg(DISTINCT m.endpoint_id ORDER BY m.endpoint_id)\n FROM matched m\n JOIN live l ON l.node_id = m.node_id\n GROUP BY m.node_id\n RETURNING relay_inbox.node_id AS node_id,\n cardinality(relay_inbox.endpoints) AS endpoint_total\n LOOP\n node_count := node_count + 1;\n endpoint_count := endpoint_count + v_row.endpoint_total;\n PERFORM pg_notify('bullmq_relay', p_ns || ':' || v_row.node_id);\n END LOOP;\n\n RETURN NEXT;\nEND;\n$$;\n\n-- Acknowledges (deletes) the given inbox entries, then returns the oldest\n-- remaining ones. Acknowledging by id rather than reading after a cursor never\n-- skips a row whose transaction committed late. Returns nothing while the\n-- node's lease is expired: its pending entries belong to a registration that\n-- recovery replaces, so they must not be delivered after it. The returned\n-- ids are recorded as in flight on the node, atomically with the read (the\n-- node row lock serializes it with trims), so a trim that runs before the\n-- reader gets them does not count them as dropped.\nCREATE FUNCTION relay_read_inbox(\n p_ns text, p_node text, p_ack bigint[], p_limit integer\n)\nRETURNS SETOF relay_inbox\nLANGUAGE plpgsql AS $$\nDECLARE\n v_rows relay_inbox[];\nBEGIN\n -- Lock the node before its inbox rows, as relay_trim_inbox does.\n PERFORM 1 FROM relay_node\n WHERE ns = p_ns AND node_id = p_node\n FOR NO KEY UPDATE;\n IF cardinality(p_ack) > 0 THEN\n DELETE FROM relay_inbox\n WHERE ns = p_ns AND node_id = p_node AND id = ANY (p_ack);\n END IF;\n SELECT coalesce(array_agg(b ORDER BY b.id), '{}') INTO v_rows\n FROM (\n SELECT i.*\n FROM relay_inbox i\n WHERE i.ns = p_ns AND i.node_id = p_node\n AND EXISTS (\n SELECT 1\n FROM relay_node n\n WHERE n.ns = p_ns AND n.node_id = p_node\n AND n.lease_until > clock_timestamp()\n )\n ORDER BY i.id\n LIMIT p_limit\n ) b;\n UPDATE relay_node\n SET in_flight = ARRAY(SELECT r.id FROM unnest(v_rows) r)\n WHERE ns = p_ns AND node_id = p_node;\n RETURN QUERY SELECT r.* FROM unnest(v_rows) r ORDER BY r.id;\nEND;\n$$;\n\n-- Removes the consumed entries of a node's inbox (the ids of its last read\n-- batch) and the oldest entries beyond p_max_len, and returns the number of\n-- entries left: the messages the node hasn't read yet. The backlog and the\n-- number of dropped entries are recorded on the node. Entries of the batch in\n-- flight are delivered by the reader, so they are not counted as dropped.\nCREATE FUNCTION relay_trim_inbox(\n p_ns text, p_node text, p_ack bigint[], p_max_len integer\n)\nRETURNS integer\nLANGUAGE plpgsql AS $$\nDECLARE\n v_oldest_kept bigint;\n v_dropped integer := 0;\n v_count integer;\n v_in_flight bigint[];\nBEGIN\n -- Lock the node before its inbox rows, in the same order as unregistering\n -- and sweeping (which delete the node, then cascade to its inbox).\n SELECT in_flight INTO v_in_flight FROM relay_node\n WHERE ns = p_ns AND node_id = p_node\n FOR NO KEY UPDATE;\n IF NOT FOUND THEN\n RETURN 0;\n END IF;\n\n IF cardinality(p_ack) > 0 THEN\n DELETE FROM relay_inbox\n WHERE ns = p_ns AND node_id = p_node AND id = ANY (p_ack);\n END IF;\n\n SELECT id INTO v_oldest_kept\n FROM relay_inbox\n WHERE ns = p_ns AND node_id = p_node\n ORDER BY id DESC\n OFFSET p_max_len - 1\n LIMIT 1;\n IF v_oldest_kept IS NOT NULL THEN\n WITH removed AS (\n DELETE FROM relay_inbox\n WHERE ns = p_ns AND node_id = p_node AND id < v_oldest_kept\n RETURNING id\n )\n SELECT count(*)::integer INTO v_dropped\n FROM removed\n WHERE removed.id <> ALL (v_in_flight);\n END IF;\n\n SELECT count(*)::integer INTO v_count\n FROM relay_inbox\n WHERE ns = p_ns AND node_id = p_node;\n UPDATE relay_node\n SET backlog = v_count, dropped = dropped + v_dropped\n WHERE ns = p_ns AND node_id = p_node;\n RETURN v_count;\nEND;\n$$;\n\n-- Deletes the retained message of a topic; returns whether there was one.\nCREATE FUNCTION relay_delete_retained(p_ns text, p_topic text)\nRETURNS boolean\nLANGUAGE plpgsql AS $$\nBEGIN\n IF relay_parse_topic(p_topic, false) IS NULL THEN\n RAISE EXCEPTION 'bullmq: relay invalid topic %', p_topic\n USING ERRCODE = 'BM001', DETAIL = '-1';\n END IF;\n -- Same lock as publishes and exact subscriptions of the topic.\n PERFORM pg_advisory_xact_lock(hashtext(p_ns), hashtext(p_topic));\n DELETE FROM relay_retained\n WHERE ns = p_ns AND topic = p_topic\n AND expires_at >= clock_timestamp();\n RETURN FOUND;\nEND;\n$$;\n\n-- Read-only getters, for dashboards. Pages are inclusive [p_start, p_end]\n-- positions (p_end < 0: to the end); orders match the Redis backend (bytewise\n-- on the same composite values), so pagination behaves the same.\n\nCREATE FUNCTION relay_ms(p_time timestamptz)\nRETURNS bigint\nLANGUAGE sql STABLE AS $$\n SELECT floor(extract(epoch FROM p_time) * 1000)::bigint;\n$$;\n\nCREATE FUNCTION relay_page_limit(p_start integer, p_end integer)\nRETURNS bigint\nLANGUAGE sql IMMUTABLE AS $$\n SELECT CASE WHEN p_end < 0 THEN NULL ELSE greatest(p_end - p_start + 1, 0) END;\n$$;\n\nCREATE FUNCTION relay_get_counts(p_ns text)\nRETURNS TABLE (nodes integer, subscriptions integer, patterns integer, retained integer)\nLANGUAGE sql STABLE AS $$\n SELECT\n (SELECT count(*)::integer FROM relay_node WHERE ns = p_ns),\n (SELECT count(*)::integer FROM relay_subscription WHERE ns = p_ns),\n (SELECT count(DISTINCT pattern)::integer\n FROM relay_subscription WHERE ns = p_ns),\n (SELECT count(*)::integer FROM relay_retained\n WHERE ns = p_ns AND expires_at > clock_timestamp());\n$$;\n\n-- Nodes by registration. A node whose lease expired (not swept yet) is listed\n-- without its info and stats, as on Redis.\nCREATE FUNCTION relay_get_nodes(p_ns text, p_start integer, p_end integer)\nRETURNS TABLE (\n node_id text, registered_at bigint, lease_until bigint, info jsonb,\n last_heartbeat bigint, backlog integer, dropped bigint, subscriptions integer\n)\nLANGUAGE sql STABLE AS $$\n SELECT n.node_id, relay_ms(n.registered_at), relay_ms(n.lease_until),\n CASE WHEN n.lease_until > clock_timestamp() THEN n.info END,\n CASE WHEN n.lease_until > clock_timestamp()\n THEN relay_ms(n.last_heartbeat) END,\n CASE WHEN n.lease_until > clock_timestamp() THEN n.backlog ELSE 0 END,\n CASE WHEN n.lease_until > clock_timestamp() THEN n.dropped ELSE 0 END,\n (SELECT count(*)::integer FROM relay_subscription s\n WHERE s.ns = n.ns AND s.node_id = n.node_id)\n FROM relay_node n\n WHERE n.ns = p_ns\n ORDER BY n.registered_at, n.node_id COLLATE \"C\"\n OFFSET p_start\n LIMIT relay_page_limit(p_start, p_end);\n$$;\n\n-- All subscriptions (by node, endpoint, pattern), or those of a node (by\n-- endpoint, pattern) or of a pattern (by node, endpoint).\nCREATE FUNCTION relay_get_subscriptions(\n p_ns text, p_node text, p_pattern text, p_start integer, p_end integer\n)\nRETURNS TABLE (node_id text, endpoint_id text, pattern text)\nLANGUAGE sql STABLE AS $$\n SELECT s.node_id, s.endpoint_id, s.pattern\n FROM relay_subscription s\n WHERE s.ns = p_ns\n AND (p_node IS NULL OR s.node_id = p_node)\n AND (p_pattern IS NULL OR s.pattern = p_pattern)\n ORDER BY (CASE\n WHEN p_node IS NOT NULL THEN s.endpoint_id || '|' || s.pattern\n WHEN p_pattern IS NOT NULL THEN s.node_id || '|' || s.endpoint_id\n ELSE s.node_id || '|' || s.endpoint_id || '|' || s.pattern\n END) COLLATE \"C\"\n OFFSET p_start\n LIMIT relay_page_limit(p_start, p_end);\n$$;\n\nCREATE FUNCTION relay_get_patterns(p_ns text, p_start integer, p_end integer)\nRETURNS TABLE (pattern text, subscriptions integer)\nLANGUAGE sql STABLE AS $$\n SELECT s.pattern, count(*)::integer\n FROM relay_subscription s\n WHERE s.ns = p_ns\n GROUP BY s.pattern\n ORDER BY s.pattern COLLATE \"C\"\n OFFSET p_start\n LIMIT relay_page_limit(p_start, p_end);\n$$;\n\nCREATE FUNCTION relay_get_retained(p_ns text, p_start integer, p_end integer)\nRETURNS TABLE (topic text, expires_at bigint, mid bigint, ts bigint, data text)\nLANGUAGE sql STABLE AS $$\n SELECT r.topic, relay_ms(r.expires_at), r.mid, r.ts, r.data\n FROM relay_retained r\n WHERE r.ns = p_ns AND r.expires_at > clock_timestamp()\n ORDER BY r.expires_at, r.topic COLLATE \"C\"\n OFFSET p_start\n LIMIT relay_page_limit(p_start, p_end);\n$$;\n\n-- Namespaces with live nodes or retained messages.\nCREATE FUNCTION relay_get_namespaces()\nRETURNS SETOF text\nLANGUAGE sql STABLE AS $$\n SELECT u.ns FROM (\n SELECT ns FROM relay_node WHERE lease_until > clock_timestamp()\n UNION\n SELECT ns FROM relay_retained WHERE expires_at > clock_timestamp()\n ) u\n ORDER BY u.ns COLLATE \"C\";\n$$;\n",
|
|
129
129
|
"0005_flow_parent_release.sql": "-- BullMQ PostgreSQL backend — deferred flow parent release (schema version 5).\n--\n-- When a flow references a child job id that already exists and is already\n-- completed, `handle_duplicated_job` recorded the processed dependency and\n-- immediately released the parent if it had no pending dependencies left.\n-- Inside `add_flow` that check runs while the rest of the tree has not been\n-- inserted yet, so a parent could be released (and picked up by a worker)\n-- while later siblings were still being attached to it.\n--\n-- `handle_duplicated_job` gains a `p_defer_release` flag: when set it only\n-- records the processed dependency. `add_flow` sets it and, once the whole\n-- tree has been inserted, validates every parent touched by the flow and\n-- releases the ones that genuinely have no pending dependencies left.\n\n-- ──────────────────────────────────────────────────────────────────────────\n-- handle_duplicated_job: re-attach an already-existing job to a new parent.\n-- ──────────────────────────────────────────────────────────────────────────\nDROP FUNCTION IF EXISTS handle_duplicated_job(\n text, text, text, text, text, bigint);\n\nCREATE FUNCTION handle_duplicated_job(\n p_queue text,\n p_id text,\n p_parent_queue text,\n p_parent_id text,\n p_parent_key text,\n p_now bigint,\n p_defer_release boolean DEFAULT false\n) RETURNS integer\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nDECLARE\n v_ex_pq text;\n v_ex_pid text;\n v_state job_state;\n v_rv jsonb;\n v_remaining integer;\n v_added boolean;\nBEGIN\n -- No new parent to attach: still a duplicate add, so announce it.\n IF p_parent_id IS NULL OR p_parent_queue IS NULL THEN\n PERFORM publish_event(p_queue, 'duplicated',\n jsonb_build_object('jobId', p_id));\n RETURN 0;\n END IF;\n\n SELECT parent_queue, parent_id, state, return_value\n INTO v_ex_pq, v_ex_pid, v_state, v_rv\n FROM job WHERE queue = p_queue AND id = p_id;\n\n -- The existing job already belongs to a different (still-existing) parent.\n IF v_ex_pq IS NOT NULL\n AND (v_ex_pq IS DISTINCT FROM p_parent_queue\n OR v_ex_pid IS DISTINCT FROM p_parent_id)\n AND EXISTS (\n SELECT 1 FROM job WHERE queue = v_ex_pq AND id = v_ex_pid\n ) THEN\n RETURN -7;\n END IF;\n\n IF v_state = 'completed' THEN\n -- Already finished: record a processed dependency (no pending increment)\n -- and release the parent if this was its last outstanding dependency.\n INSERT INTO job_dependency (\n parent_queue, parent_id, child_queue, child_id, child_key, status, value\n ) VALUES (\n p_parent_queue, p_parent_id, p_queue, p_id,\n p_queue || ':' || p_id, 'processed', v_rv\n )\n ON CONFLICT (parent_queue, parent_id, child_key)\n DO UPDATE SET status = 'processed', value = v_rv;\n\n -- Within a flow insert the parent may still be waiting for siblings that\n -- have not been inserted yet, so the release is validated by the caller\n -- once the whole tree is in.\n IF NOT p_defer_release THEN\n SELECT pending_deps INTO v_remaining\n FROM job WHERE queue = p_parent_queue AND id = p_parent_id;\n IF COALESCE(v_remaining, 0) = 0 THEN\n PERFORM move_parent_to_wait(p_parent_queue, p_parent_id, p_now);\n END IF;\n END IF;\n ELSE\n -- Still pending: register a pending dependency and count it on the parent.\n INSERT INTO job_dependency (\n parent_queue, parent_id, child_queue, child_id, child_key, status\n ) VALUES (\n p_parent_queue, p_parent_id, p_queue, p_id,\n p_queue || ':' || p_id, 'pending'\n )\n ON CONFLICT (parent_queue, parent_id, child_key) DO NOTHING;\n GET DIAGNOSTICS v_added = ROW_COUNT;\n IF v_added THEN\n UPDATE job SET pending_deps = pending_deps + 1\n WHERE queue = p_parent_queue AND id = p_parent_id;\n END IF;\n END IF;\n\n -- Point the existing job at its new parent.\n UPDATE job\n SET parent_queue = p_parent_queue,\n parent_id = p_parent_id,\n parent_key = p_parent_key\n WHERE queue = p_queue AND id = p_id;\n\n PERFORM publish_event(p_queue, 'duplicated',\n jsonb_build_object('jobId', p_id));\n\n RETURN 0;\nEND;\n$$;\n\n-- ──────────────────────────────────────────────────────────────────────────\n-- add_flow: insert a whole tree atomically, releasing parents only at the end.\n-- ──────────────────────────────────────────────────────────────────────────\nCREATE OR REPLACE FUNCTION add_flow(p_entries jsonb) RETURNS SETOF text\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nDECLARE\n e jsonb;\n v_id text;\n v_seq bigint;\n v_state job_state;\n v_process_at bigint;\n v_queue text;\n v_delay bigint;\n v_timestamp bigint;\n v_lifo boolean;\n v_parent_queue text;\n v_parent_id text;\n v_awc boolean;\n v_dedup_id text;\n v_dedup jsonb;\n v_winner text;\n v_inserted boolean;\n v_code integer;\n -- Lifecycle-event accumulation. Rather than PERFORM publish_event per\n -- job (which turns a 1000-job addBulk into ~2000 extra statement executions —\n -- the dominant cost of a single-connection bulk add), events are collected\n -- here and flushed in ONE set-based INSERT after the loop. Appending to a\n -- plpgsql-local array is amortized O(1) (read-write \"expanded array\"), so this\n -- stays O(n).\n v_ev_queue text[] := '{}';\n v_ev_event text[] := '{}';\n v_ev_data jsonb[] := '{}';\n v_notify_jobs text[] := '{}';\n v_max integer;\n -- Parents touched by this flow; validated once the whole tree is inserted.\n v_parents_q text[] := '{}';\n v_parents_id text[] := '{}';\n v_failed_parents_q text[] := '{}';\n v_failed_parents_id text[] := '{}';\n v_pq text;\n v_pid text;\n v_remaining integer;\n v_now bigint;\nBEGIN\n FOR e IN SELECT value FROM jsonb_array_elements(p_entries)\n LOOP\n v_queue := e ->> 'queue';\n v_id := COALESCE(e ->> 'id', '');\n v_delay := COALESCE((e ->> 'delay')::bigint, 0);\n v_timestamp := COALESCE((e ->> 'timestamp')::bigint, 0);\n v_lifo := COALESCE((e ->> 'lifo')::boolean, false);\n v_parent_queue := e ->> 'parentQueue';\n v_parent_id := e ->> 'parentId';\n v_awc := COALESCE((e ->> 'addToWaitingChildren')::boolean, false);\n v_dedup_id := e ->> 'dedupId';\n\n -- Serialize dedup operations for this id (matches add_job and the finish\n -- paths: advisory lock taken before the id-counter INCR below).\n IF v_dedup_id IS NOT NULL AND v_dedup_id <> '' THEN\n PERFORM pg_advisory_xact_lock(\n hashtext('bullmq:dedup:' || v_queue || ':' || v_dedup_id));\n END IF;\n\n IF v_id IS NULL OR v_id = '' THEN\n v_id := next_job_id(v_queue);\n END IF;\n\n -- Root deduplication: if a live key already won, return its id and skip\n -- inserting this job (and, because it never exists, its descendants).\n IF v_dedup_id IS NOT NULL AND v_dedup_id <> '' THEN\n v_dedup := COALESCE(e -> 'opts' -> 'deduplication',\n e -> 'opts' -> 'debounce',\n jsonb_build_object('id', v_dedup_id));\n v_winner := deduplicate_job(v_queue, v_dedup, v_id, v_timestamp,\n e ->> 'name', COALESCE((e ->> 'data')::jsonb, '{}'::jsonb),\n COALESCE(e -> 'opts', '{}'::jsonb));\n IF v_winner IS NOT NULL THEN\n RETURN NEXT v_winner;\n CONTINUE;\n END IF;\n END IF;\n\n -- An entry whose parent does not exist (a child of a deduplicated root, or\n -- a genuinely missing parent) is not inserted; report -5 as addJob does,\n -- without aborting the rest of the flow.\n IF v_parent_id IS NOT NULL AND v_parent_queue IS NOT NULL\n AND NOT EXISTS (\n SELECT 1 FROM job\n WHERE queue = v_parent_queue AND id = v_parent_id\n ) THEN\n RETURN NEXT '-5';\n CONTINUE;\n END IF;\n\n v_seq := nextval('job_seq');\n IF v_lifo THEN\n v_seq := -v_seq;\n END IF;\n\n IF v_awc THEN\n v_state := 'waiting-children';\n v_process_at := NULL;\n ELSIF v_delay > 0 THEN\n v_state := 'delayed';\n v_process_at := v_timestamp + v_delay;\n ELSE\n v_state := 'waiting';\n v_process_at := NULL;\n END IF;\n\n INSERT INTO job (\n queue, id, seq, name, state,\n data, opts, priority, delay_ms, max_attempts,\n added_at_ms, process_at_ms,\n dedup_id, scheduler_id,\n parent_queue, parent_id, parent_key, pending_deps\n ) VALUES (\n v_queue, v_id, v_seq, e ->> 'name', v_state,\n COALESCE((e ->> 'data')::jsonb, '{}'::jsonb),\n COALESCE(e -> 'opts', '{}'::jsonb),\n COALESCE((e ->> 'priority')::integer, 0), v_delay,\n COALESCE((e ->> 'attempts')::integer, 1),\n v_timestamp, v_process_at,\n e ->> 'dedupId', e ->> 'schedulerId',\n v_parent_queue, v_parent_id, e ->> 'parentKey', 0\n )\n ON CONFLICT (queue, id) DO NOTHING;\n GET DIAGNOSTICS v_inserted = ROW_COUNT;\n\n IF v_parent_id IS NOT NULL AND v_parent_queue IS NOT NULL THEN\n v_parents_q := array_append(v_parents_q, v_parent_queue);\n v_parents_id := array_append(v_parents_id, v_parent_id);\n\n IF v_inserted THEN\n -- New job: register a pending dependency on its parent.\n INSERT INTO job_dependency (\n parent_queue, parent_id, child_queue, child_id, child_key, status\n ) VALUES (\n v_parent_queue, v_parent_id, v_queue, v_id,\n v_queue || ':' || v_id, 'pending'\n )\n ON CONFLICT (parent_queue, parent_id, child_key) DO NOTHING;\n\n UPDATE job\n SET pending_deps = pending_deps + 1\n WHERE queue = v_parent_queue AND id = v_parent_id;\n ELSE\n -- The job already existed: re-attach it to this new parent (mirrors\n -- handleDuplicatedJob). -7 means it already has a different parent.\n -- The parent is never released here: the rest of the tree is not in\n -- yet, so that check is deferred to the end of the flow.\n v_code := handle_duplicated_job(v_queue, v_id,\n v_parent_queue, v_parent_id, e ->> 'parentKey', v_timestamp, true);\n IF v_code = -7 THEN\n v_failed_parents_q := array_append(v_failed_parents_q, v_parent_queue);\n v_failed_parents_id := array_append(v_failed_parents_id, v_parent_id);\n RETURN NEXT '-7';\n CONTINUE;\n END IF;\n END IF;\n END IF;\n\n -- Accumulate lifecycle events for newly-inserted jobs; they are flushed in\n -- a single set-based INSERT after the loop. Accumulation order (per job:\n -- 'added' then the state event, in job order) is what the flush replays via\n -- WITH ORDINALITY, preserving the stream ordering the per-row publish gave.\n IF v_inserted THEN\n -- Every stored job announces itself (mirrors storeJob.lua's 'added').\n v_ev_queue := array_append(v_ev_queue, v_queue);\n v_ev_event := array_append(v_ev_event, 'added');\n v_ev_data := array_append(v_ev_data,\n jsonb_build_object('jobId', v_id, 'name', e ->> 'name'));\n IF v_state = 'waiting' THEN\n v_ev_queue := array_append(v_ev_queue, v_queue);\n v_ev_event := array_append(v_ev_event, 'waiting');\n v_ev_data := array_append(v_ev_data,\n jsonb_build_object('jobId', v_id));\n v_notify_jobs := array_append(v_notify_jobs, v_queue);\n ELSIF v_state = 'delayed' THEN\n v_ev_queue := array_append(v_ev_queue, v_queue);\n v_ev_event := array_append(v_ev_event, 'delayed');\n v_ev_data := array_append(v_ev_data,\n jsonb_build_object('jobId', v_id, 'delay', v_process_at));\n v_notify_jobs := array_append(v_notify_jobs, v_queue);\n ELSIF v_state = 'waiting-children' THEN\n v_ev_queue := array_append(v_ev_queue, v_queue);\n v_ev_event := array_append(v_ev_event, 'waiting-children');\n v_ev_data := array_append(v_ev_data,\n jsonb_build_object('jobId', v_id));\n END IF;\n END IF;\n\n RETURN NEXT v_id;\n END LOOP;\n\n -- ── Flush accumulated lifecycle events in one statement ──────────────────\n IF array_length(v_ev_queue, 1) > 0 THEN\n -- Single set-based append. The id DEFAULT nextval('event_seq') is\n -- assigned in ordinality order (the ORDER BY pins row-production order), so\n -- the stream keeps each job's 'added' → state-event ordering.\n INSERT INTO event (queue, event, data, created_at_ms)\n SELECT q, ev, dat, (extract(epoch FROM clock_timestamp()) * 1000)::bigint\n FROM unnest(v_ev_queue, v_ev_event, v_ev_data)\n WITH ORDINALITY AS t(q, ev, dat, ord)\n ORDER BY ord;\n\n -- One 'bullmq_events' wakeup per distinct queue, then trim that queue's\n -- stream once (not per publish). A non-blocking advisory lock keeps at most\n -- one trimmer per queue; publishers that miss it skip (the next flush covers\n -- the same rows). Matches publish_event's approximate-MAXLEN trim.\n FOR v_queue IN SELECT DISTINCT q FROM unnest(v_ev_queue) AS q LOOP\n PERFORM pg_notify('bullmq_events', v_queue);\n\n SELECT value::integer INTO v_max\n FROM meta\n WHERE queue = v_queue AND field = 'opts.maxLenEvents';\n IF v_max IS NULL THEN\n v_max := 10000;\n END IF;\n IF v_max > 0\n AND pg_try_advisory_xact_lock(hashtext('bullmq:evtrim:' || v_queue))\n THEN\n DELETE FROM event\n WHERE queue = v_queue\n AND id <= (SELECT max(id) FROM event WHERE queue = v_queue)\n - v_max;\n END IF;\n END LOOP;\n\n -- One 'bullmq_jobs' wakeup per distinct queue that gained a runnable job.\n FOR v_queue IN SELECT DISTINCT q FROM unnest(v_notify_jobs) AS q LOOP\n PERFORM pg_notify('bullmq_jobs', v_queue);\n END LOOP;\n END IF;\n\n -- ── Release parents left without pending dependencies ────────────────────\n -- Now that every node of the flow is in, a parent whose dependencies were\n -- all already-completed jobs (so nothing pending was ever registered) can\n -- safely be released. move_parent_to_wait is a no-op for any parent that is\n -- not in waiting-children, so this is safe for every parent of the flow.\n IF array_length(v_parents_q, 1) > 0 THEN\n v_now := (extract(epoch FROM clock_timestamp()) * 1000)::bigint;\n FOR v_pq, v_pid IN\n SELECT DISTINCT pq, pid\n FROM unnest(v_parents_q, v_parents_id) AS t(pq, pid)\n WHERE NOT EXISTS (\n SELECT 1\n FROM unnest(v_failed_parents_q, v_failed_parents_id) AS f(pq, pid)\n WHERE f.pq = t.pq AND f.pid = t.pid\n )\n LOOP\n SELECT pending_deps INTO v_remaining\n FROM job WHERE queue = v_pq AND id = v_pid;\n IF COALESCE(v_remaining, 0) = 0 THEN\n PERFORM move_parent_to_wait(v_pq, v_pid, v_now);\n END IF;\n END LOOP;\n END IF;\nEND;\n$$;\n",
|
|
130
|
+
"0006_dedup_reschedule.sql": "-- Reschedule delayed deduplication winners without replacing their payload.\nCREATE OR REPLACE FUNCTION deduplicate_job(\n p_queue text, p_dedup jsonb, p_job_id text, p_now bigint,\n p_name text, p_data jsonb, p_opts jsonb\n) RETURNS text\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nDECLARE\n v_id text := p_dedup->>'id';\n v_ttl bigint := NULLIF(p_dedup->>'ttl', '')::bigint;\n v_extend boolean := COALESCE((p_dedup->>'extend')::boolean, false);\n v_replace boolean := COALESCE((p_dedup->>'replace')::boolean, false);\n v_keeplast boolean := COALESCE((p_dedup->>'keepLastIfActive')::boolean, false);\n v_delay bigint;\n v_cur text;\n v_exp bigint;\n v_state job_state;\nBEGIN\n IF v_id IS NULL OR v_id = '' THEN\n RETURN NULL;\n END IF;\n\n -- Lock the dedup row before touching the winner job row, matching the\n -- dedup -> job lock order used by removal (dedup_on_removal, then DELETE job).\n SELECT job_id, expire_at_ms INTO v_cur, v_exp\n FROM dedup WHERE queue = p_queue AND dedup_id = v_id\n FOR UPDATE;\n IF v_cur IS NULL OR (v_exp IS NOT NULL AND v_exp <= p_now) THEN\n v_cur := NULL;\n ELSIF v_exp IS NULL AND NOT EXISTS (\n SELECT 1 FROM job WHERE queue = p_queue AND id = v_cur\n ) THEN\n DELETE FROM dedup WHERE queue = p_queue AND dedup_id = v_id;\n DELETE FROM dedup_next WHERE queue = p_queue AND dedup_id = v_id;\n v_cur := NULL;\n END IF;\n\n IF v_replace THEN\n IF v_cur IS NOT NULL THEN\n SELECT state INTO v_state\n FROM job WHERE queue = p_queue AND id = v_cur;\n IF v_state = 'delayed' THEN\n DELETE FROM job WHERE queue = p_queue AND id = v_cur;\n PERFORM publish_event(p_queue, 'removed',\n jsonb_build_object('jobId', v_cur, 'prev', 'delayed'));\n PERFORM publish_event(p_queue, 'deduplicated',\n jsonb_build_object('jobId', p_job_id, 'deduplicationId', v_id,\n 'deduplicatedJobId', v_cur));\n DELETE FROM dedup_next WHERE queue = p_queue AND dedup_id = v_id;\n IF v_keeplast THEN\n UPDATE dedup SET job_id = p_job_id, expire_at_ms = NULL\n WHERE queue = p_queue AND dedup_id = v_id;\n ELSIF NOT v_extend AND COALESCE(v_ttl, 0) > 0 THEN\n UPDATE dedup SET job_id = p_job_id\n WHERE queue = p_queue AND dedup_id = v_id;\n ELSE\n UPDATE dedup\n SET job_id = p_job_id,\n expire_at_ms = CASE WHEN COALESCE(v_ttl, 0) > 0\n THEN p_now + v_ttl ELSE NULL END\n WHERE queue = p_queue AND dedup_id = v_id;\n END IF;\n RETURN NULL;\n ELSE\n PERFORM dedup_store_next(p_queue, v_id, v_cur, p_job_id,\n v_keeplast, p_name, p_data, p_opts);\n RETURN v_cur;\n END IF;\n ELSE\n INSERT INTO dedup (queue, dedup_id, job_id, expire_at_ms)\n VALUES (p_queue, v_id, p_job_id,\n CASE WHEN NOT v_keeplast AND COALESCE(v_ttl, 0) > 0\n THEN p_now + v_ttl ELSE NULL END)\n ON CONFLICT (queue, dedup_id) DO UPDATE\n SET job_id = EXCLUDED.job_id, expire_at_ms = EXCLUDED.expire_at_ms;\n RETURN NULL;\n END IF;\n END IF;\n\n IF v_cur IS NOT NULL AND COALESCE((p_dedup->>'reschedule')::boolean, false) THEN\n v_delay := COALESCE((p_opts->>'delay')::bigint, 0);\n UPDATE job SET delay_ms = v_delay,\n process_at_ms = p_now + GREATEST(v_delay, 0)\n WHERE queue = p_queue AND id = v_cur AND state = 'delayed';\n IF FOUND THEN\n PERFORM publish_event(p_queue, 'delayed',\n jsonb_build_object('jobId', v_cur, 'delay', p_now + GREATEST(v_delay, 0)));\n PERFORM pg_notify('bullmq_jobs', p_queue);\n END IF;\n END IF;\n\n IF COALESCE(v_ttl, 0) > 0 AND v_extend THEN\n IF v_cur IS NOT NULL THEN\n IF NOT dedup_store_next(p_queue, v_id, v_cur, p_job_id,\n v_keeplast, p_name, p_data, p_opts) THEN\n UPDATE dedup\n SET expire_at_ms = CASE WHEN v_keeplast THEN NULL ELSE p_now + v_ttl END\n WHERE queue = p_queue AND dedup_id = v_id;\n END IF;\n PERFORM publish_event(p_queue, 'deduplicated',\n jsonb_build_object('jobId', v_cur, 'deduplicationId', v_id,\n 'deduplicatedJobId', p_job_id));\n RETURN v_cur;\n END IF;\n INSERT INTO dedup (queue, dedup_id, job_id, expire_at_ms)\n VALUES (p_queue, v_id, p_job_id,\n CASE WHEN v_keeplast THEN NULL ELSE p_now + v_ttl END)\n ON CONFLICT (queue, dedup_id) DO UPDATE\n SET job_id = EXCLUDED.job_id, expire_at_ms = EXCLUDED.expire_at_ms;\n RETURN NULL;\n END IF;\n\n IF v_cur IS NOT NULL THEN\n PERFORM dedup_store_next(p_queue, v_id, v_cur, p_job_id,\n v_keeplast, p_name, p_data, p_opts);\n PERFORM publish_event(p_queue, 'deduplicated',\n jsonb_build_object('jobId', v_cur, 'deduplicationId', v_id,\n 'deduplicatedJobId', p_job_id));\n RETURN v_cur;\n END IF;\n\n INSERT INTO dedup (queue, dedup_id, job_id, expire_at_ms)\n VALUES (p_queue, v_id, p_job_id,\n CASE WHEN NOT v_keeplast AND COALESCE(v_ttl, 0) > 0\n THEN p_now + v_ttl ELSE NULL END)\n ON CONFLICT (queue, dedup_id) DO UPDATE\n SET job_id = EXCLUDED.job_id, expire_at_ms = EXCLUDED.expire_at_ms;\n RETURN NULL;\nEND;\n$$;\n",
|
|
131
|
+
"0007_obliterate_dedup_lock_order.sql": "-- Match deduplicate_job's dedup -> job lock order in every obliteration batch.\nCREATE OR REPLACE FUNCTION obliterate(\n p_queue text, p_count integer, p_force boolean\n) RETURNS integer\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nDECLARE\n v_deleted integer;\nBEGIN\n IF NOT EXISTS (\n SELECT 1 FROM meta\n WHERE queue = p_queue AND field = 'paused' AND value = '1'\n ) THEN\n RETURN -1;\n END IF;\n\n IF NOT p_force AND EXISTS (\n SELECT 1 FROM job WHERE queue = p_queue AND state = 'active'\n ) THEN\n RETURN -2;\n END IF;\n\n -- Producers can still add while paused. Lock existing dedup rows before\n -- any job deletion, including non-final batches, to avoid waiting for a\n -- dedup row while its owner is waiting for a job this transaction deleted.\n -- Keep keys intact until final cleanup so intermediate batches do not\n -- prematurely end live deduplication windows.\n PERFORM dedup_id FROM dedup\n WHERE queue = p_queue\n ORDER BY dedup_id\n FOR UPDATE;\n\n WITH batch AS (\n SELECT id FROM job WHERE queue = p_queue LIMIT p_count\n )\n DELETE FROM job j\n USING batch\n WHERE j.queue = p_queue AND j.id = batch.id;\n GET DIAGNOSTICS v_deleted = ROW_COUNT;\n\n IF v_deleted >= p_count THEN\n RETURN 1;\n END IF;\n\n DELETE FROM scheduler WHERE queue = p_queue;\n DELETE FROM rate_limit WHERE queue = p_queue;\n DELETE FROM event WHERE queue = p_queue;\n DELETE FROM metrics WHERE queue = p_queue;\n DELETE FROM dedup WHERE queue = p_queue;\n DELETE FROM dedup_next WHERE queue = p_queue;\n DELETE FROM meta WHERE queue = p_queue;\n EXECUTE format(\n 'DROP SEQUENCE IF EXISTS %I', job_id_seq_name(p_queue));\n RETURN 0;\nEND;\n$$;\n",
|
|
132
|
+
"0008_bulk_dedup_lock_order.sql": "-- Acquire every batch dedup advisory lock before touching winner job rows.\nCREATE OR REPLACE FUNCTION add_flow(p_entries jsonb) RETURNS SETOF text\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nDECLARE\n e jsonb;\n v_id text;\n v_seq bigint;\n v_state job_state;\n v_process_at bigint;\n v_queue text;\n v_delay bigint;\n v_timestamp bigint;\n v_lifo boolean;\n v_parent_queue text;\n v_parent_id text;\n v_awc boolean;\n v_dedup_id text;\n v_dedup jsonb;\n v_winner text;\n v_inserted boolean;\n v_code integer;\n v_lock_key integer;\n v_ev_queue text[] := '{}';\n v_ev_event text[] := '{}';\n v_ev_data jsonb[] := '{}';\n v_notify_jobs text[] := '{}';\n v_max integer;\n v_parents_q text[] := '{}';\n v_parents_id text[] := '{}';\n v_failed_parents_q text[] := '{}';\n v_failed_parents_id text[] := '{}';\n v_pq text;\n v_pid text;\n v_remaining integer;\n v_now bigint;\nBEGIN\n -- Finish-and-fetch holds its dedup advisory lock while promoting delayed\n -- jobs. Take all batch locks before any job/sequence mutation, ordered by\n -- the actual lock key so reversed batches and hash collisions agree.\n FOR v_lock_key IN\n SELECT DISTINCT hashtext(\n 'bullmq:dedup:' || (value ->> 'queue') || ':' || (value ->> 'dedupId'))\n FROM jsonb_array_elements(p_entries)\n WHERE value ->> 'dedupId' IS NOT NULL AND value ->> 'dedupId' <> ''\n ORDER BY 1\n LOOP\n PERFORM pg_advisory_xact_lock(v_lock_key);\n END LOOP;\n\n FOR e IN SELECT value FROM jsonb_array_elements(p_entries)\n LOOP\n v_queue := e ->> 'queue';\n v_id := COALESCE(e ->> 'id', '');\n v_delay := COALESCE((e ->> 'delay')::bigint, 0);\n v_timestamp := COALESCE((e ->> 'timestamp')::bigint, 0);\n v_lifo := COALESCE((e ->> 'lifo')::boolean, false);\n v_parent_queue := e ->> 'parentQueue';\n v_parent_id := e ->> 'parentId';\n v_awc := COALESCE((e ->> 'addToWaitingChildren')::boolean, false);\n v_dedup_id := e ->> 'dedupId';\n\n IF v_id IS NULL OR v_id = '' THEN\n v_id := next_job_id(v_queue);\n END IF;\n\n IF v_dedup_id IS NOT NULL AND v_dedup_id <> '' THEN\n v_dedup := COALESCE(e -> 'opts' -> 'deduplication',\n e -> 'opts' -> 'debounce',\n jsonb_build_object('id', v_dedup_id));\n v_winner := deduplicate_job(v_queue, v_dedup, v_id, v_timestamp,\n e ->> 'name', COALESCE((e ->> 'data')::jsonb, '{}'::jsonb),\n COALESCE(e -> 'opts', '{}'::jsonb));\n IF v_winner IS NOT NULL THEN\n RETURN NEXT v_winner;\n CONTINUE;\n END IF;\n END IF;\n\n IF v_parent_id IS NOT NULL AND v_parent_queue IS NOT NULL\n AND NOT EXISTS (\n SELECT 1 FROM job\n WHERE queue = v_parent_queue AND id = v_parent_id\n ) THEN\n RETURN NEXT '-5';\n CONTINUE;\n END IF;\n\n v_seq := nextval('job_seq');\n IF v_lifo THEN\n v_seq := -v_seq;\n END IF;\n\n IF v_awc THEN\n v_state := 'waiting-children';\n v_process_at := NULL;\n ELSIF v_delay > 0 THEN\n v_state := 'delayed';\n v_process_at := v_timestamp + v_delay;\n ELSE\n v_state := 'waiting';\n v_process_at := NULL;\n END IF;\n\n INSERT INTO job (\n queue, id, seq, name, state,\n data, opts, priority, delay_ms, max_attempts,\n added_at_ms, process_at_ms,\n dedup_id, scheduler_id,\n parent_queue, parent_id, parent_key, pending_deps\n ) VALUES (\n v_queue, v_id, v_seq, e ->> 'name', v_state,\n COALESCE((e ->> 'data')::jsonb, '{}'::jsonb),\n COALESCE(e -> 'opts', '{}'::jsonb),\n COALESCE((e ->> 'priority')::integer, 0), v_delay,\n COALESCE((e ->> 'attempts')::integer, 1),\n v_timestamp, v_process_at,\n e ->> 'dedupId', e ->> 'schedulerId',\n v_parent_queue, v_parent_id, e ->> 'parentKey', 0\n )\n ON CONFLICT (queue, id) DO NOTHING;\n GET DIAGNOSTICS v_inserted = ROW_COUNT;\n\n IF v_parent_id IS NOT NULL AND v_parent_queue IS NOT NULL THEN\n v_parents_q := array_append(v_parents_q, v_parent_queue);\n v_parents_id := array_append(v_parents_id, v_parent_id);\n\n IF v_inserted THEN\n INSERT INTO job_dependency (\n parent_queue, parent_id, child_queue, child_id, child_key, status\n ) VALUES (\n v_parent_queue, v_parent_id, v_queue, v_id,\n v_queue || ':' || v_id, 'pending'\n )\n ON CONFLICT (parent_queue, parent_id, child_key) DO NOTHING;\n\n UPDATE job\n SET pending_deps = pending_deps + 1\n WHERE queue = v_parent_queue AND id = v_parent_id;\n ELSE\n v_code := handle_duplicated_job(v_queue, v_id,\n v_parent_queue, v_parent_id, e ->> 'parentKey', v_timestamp, true);\n IF v_code = -7 THEN\n v_failed_parents_q := array_append(v_failed_parents_q, v_parent_queue);\n v_failed_parents_id := array_append(v_failed_parents_id, v_parent_id);\n RETURN NEXT '-7';\n CONTINUE;\n END IF;\n END IF;\n END IF;\n\n IF v_inserted THEN\n v_ev_queue := array_append(v_ev_queue, v_queue);\n v_ev_event := array_append(v_ev_event, 'added');\n v_ev_data := array_append(v_ev_data,\n jsonb_build_object('jobId', v_id, 'name', e ->> 'name'));\n IF v_state = 'waiting' THEN\n v_ev_queue := array_append(v_ev_queue, v_queue);\n v_ev_event := array_append(v_ev_event, 'waiting');\n v_ev_data := array_append(v_ev_data,\n jsonb_build_object('jobId', v_id));\n v_notify_jobs := array_append(v_notify_jobs, v_queue);\n ELSIF v_state = 'delayed' THEN\n v_ev_queue := array_append(v_ev_queue, v_queue);\n v_ev_event := array_append(v_ev_event, 'delayed');\n v_ev_data := array_append(v_ev_data,\n jsonb_build_object('jobId', v_id, 'delay', v_process_at));\n v_notify_jobs := array_append(v_notify_jobs, v_queue);\n ELSIF v_state = 'waiting-children' THEN\n v_ev_queue := array_append(v_ev_queue, v_queue);\n v_ev_event := array_append(v_ev_event, 'waiting-children');\n v_ev_data := array_append(v_ev_data,\n jsonb_build_object('jobId', v_id));\n END IF;\n END IF;\n\n RETURN NEXT v_id;\n END LOOP;\n\n IF array_length(v_ev_queue, 1) > 0 THEN\n INSERT INTO event (queue, event, data, created_at_ms)\n SELECT q, ev, dat, (extract(epoch FROM clock_timestamp()) * 1000)::bigint\n FROM unnest(v_ev_queue, v_ev_event, v_ev_data)\n WITH ORDINALITY AS t(q, ev, dat, ord)\n ORDER BY ord;\n\n FOR v_queue IN SELECT DISTINCT q FROM unnest(v_ev_queue) AS q LOOP\n PERFORM pg_notify('bullmq_events', v_queue);\n\n SELECT value::integer INTO v_max\n FROM meta\n WHERE queue = v_queue AND field = 'opts.maxLenEvents';\n IF v_max IS NULL THEN\n v_max := 10000;\n END IF;\n IF v_max > 0\n AND pg_try_advisory_xact_lock(hashtext('bullmq:evtrim:' || v_queue))\n THEN\n DELETE FROM event\n WHERE queue = v_queue\n AND id <= (SELECT max(id) FROM event WHERE queue = v_queue)\n - v_max;\n END IF;\n END LOOP;\n\n FOR v_queue IN SELECT DISTINCT q FROM unnest(v_notify_jobs) AS q LOOP\n PERFORM pg_notify('bullmq_jobs', v_queue);\n END LOOP;\n END IF;\n\n IF array_length(v_parents_q, 1) > 0 THEN\n v_now := (extract(epoch FROM clock_timestamp()) * 1000)::bigint;\n FOR v_pq, v_pid IN\n SELECT DISTINCT pq, pid\n FROM unnest(v_parents_q, v_parents_id) AS t(pq, pid)\n WHERE NOT EXISTS (\n SELECT 1\n FROM unnest(v_failed_parents_q, v_failed_parents_id) AS f(pq, pid)\n WHERE f.pq = t.pq AND f.pid = t.pid\n )\n LOOP\n SELECT pending_deps INTO v_remaining\n FROM job WHERE queue = v_pq AND id = v_pid;\n IF COALESCE(v_remaining, 0) = 0 THEN\n PERFORM move_parent_to_wait(v_pq, v_pid, v_now);\n END IF;\n END LOOP;\n END IF;\nEND;\n$$;\n",
|
|
133
|
+
"0009_obliterate_bounded_dedup_locks.sql": "-- Look up every key pointing at a selected winner without scanning the queue.\nCREATE INDEX dedup_job_idx ON dedup (queue, job_id);\n\nCREATE OR REPLACE FUNCTION obliterate(\n p_queue text, p_count integer, p_force boolean\n) RETURNS integer\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nDECLARE\n v_ids text[];\n v_final boolean;\nBEGIN\n IF NOT EXISTS (\n SELECT 1 FROM meta\n WHERE queue = p_queue AND field = 'paused' AND value = '1'\n ) THEN\n RETURN -1;\n END IF;\n\n IF NOT p_force AND EXISTS (\n SELECT 1 FROM job WHERE queue = p_queue AND state = 'active'\n ) THEN\n RETURN -2;\n END IF;\n\n SELECT COALESCE(array_agg(id), ARRAY[]::text[]) INTO v_ids\n FROM (\n SELECT id FROM job WHERE queue = p_queue ORDER BY id LIMIT p_count\n ) AS batch;\n v_final := cardinality(v_ids) < p_count;\n\n -- Decide final cleanup before taking job locks. A full selected batch\n -- always asks for another batch, even if concurrent deletion removes some\n -- selected jobs, so whole-queue cleanup never introduces job -> dedup locks.\n IF v_final THEN\n PERFORM dedup_id FROM dedup\n WHERE queue = p_queue\n ORDER BY dedup_id\n FOR UPDATE;\n ELSE\n PERFORM dedup_id FROM dedup\n WHERE queue = p_queue AND job_id = ANY(v_ids)\n ORDER BY dedup_id\n FOR UPDATE;\n END IF;\n\n -- The only job-delete cascades are job_log and the parent's job_dependency\n -- rows. They do not delete child jobs or their dedup keys.\n DELETE FROM job WHERE queue = p_queue AND id = ANY(v_ids);\n\n IF NOT v_final THEN\n RETURN 1;\n END IF;\n\n DELETE FROM scheduler WHERE queue = p_queue;\n DELETE FROM rate_limit WHERE queue = p_queue;\n DELETE FROM event WHERE queue = p_queue;\n DELETE FROM metrics WHERE queue = p_queue;\n DELETE FROM dedup WHERE queue = p_queue;\n DELETE FROM dedup_next WHERE queue = p_queue;\n DELETE FROM meta WHERE queue = p_queue;\n EXECUTE format(\n 'DROP SEQUENCE IF EXISTS %I', job_id_seq_name(p_queue));\n RETURN 0;\nEND;\n$$;\n",
|
|
134
|
+
"0010_obliterate_finalization_lock.sql": "-- Finalization may also update a parent in another queue. Coordinate both\n-- queues before any row/parent/dedup locks, in actual advisory-key order.\nCREATE FUNCTION lock_finalization_queues(p_queue text, p_id text) RETURNS void\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nDECLARE\n v_key integer;\nBEGIN\n FOR v_key IN\n SELECT DISTINCT hashtext(q)\n FROM (\n SELECT p_queue AS q\n UNION ALL\n SELECT parent_queue FROM job WHERE queue = p_queue AND id = p_id\n ) AS queues\n WHERE q IS NOT NULL\n ORDER BY 1\n LOOP\n PERFORM pg_advisory_xact_lock_shared(hashtext('bullmq:finalization'), v_key);\n END LOOP;\nEND;\n$$;\n\nCREATE OR REPLACE FUNCTION move_to_completed(\n p_queue text, p_id text, p_token text, p_return_value jsonb,\n p_finished_on bigint, p_remove_all boolean, p_keep_age bigint,\n p_keep_count integer\n) RETURNS bigint\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nDECLARE\n v_state job_state;\n v_lock text;\n v_pq text;\n v_pid text;\n v_dedup_id text;\n v_remaining integer;\nBEGIN\n PERFORM lock_finalization_queues(p_queue, p_id);\n\n SELECT parent_queue, parent_id INTO v_pq, v_pid\n FROM job WHERE queue = p_queue AND id = p_id;\n IF v_pq IS NOT NULL AND v_pid IS NOT NULL THEN\n PERFORM pg_advisory_xact_lock(\n hashtext('bullmq:parent:' || v_pq || ':' || v_pid));\n END IF;\n\n SELECT state, lock_token, parent_queue, parent_id, dedup_id\n INTO v_state, v_lock, v_pq, v_pid, v_dedup_id\n FROM job WHERE queue = p_queue AND id = p_id FOR UPDATE;\n IF NOT FOUND THEN\n RAISE EXCEPTION 'bullmq: missing job %', p_id USING ERRCODE = 'BM001', DETAIL = '-1';\n END IF;\n IF v_state <> 'active' THEN\n RAISE EXCEPTION 'bullmq: job % not active', p_id USING ERRCODE = 'BM001', DETAIL = '-3';\n END IF;\n IF p_token <> '0' AND v_lock IS DISTINCT FROM p_token THEN\n RAISE EXCEPTION 'bullmq: job % lock mismatch', p_id USING ERRCODE = 'BM001', DETAIL = '-6';\n END IF;\n\n IF EXISTS (\n SELECT 1 FROM job_dependency\n WHERE parent_queue = p_queue AND parent_id = p_id AND status = 'pending'\n ) THEN\n RAISE EXCEPTION 'bullmq: job % has pending dependencies', p_id\n USING ERRCODE = 'BM001', DETAIL = '-4';\n END IF;\n IF EXISTS (\n SELECT 1 FROM job_dependency\n WHERE parent_queue = p_queue AND parent_id = p_id AND status = 'failed'\n ) THEN\n RAISE EXCEPTION 'bullmq: job % has failed dependencies', p_id\n USING ERRCODE = 'BM001', DETAIL = '-9';\n END IF;\n\n UPDATE job\n SET state = 'completed',\n return_value = p_return_value,\n finished_at_ms = p_finished_on,\n lock_token = NULL,\n locked_until_ms = NULL,\n attempts_made = attempts_made + 1\n WHERE queue = p_queue AND id = p_id;\n\n PERFORM publish_event(p_queue, 'completed',\n jsonb_build_object('jobId', p_id, 'returnvalue',\n COALESCE(p_return_value, 'null'::jsonb)::text, 'prev', 'active'));\n\n IF v_pid IS NOT NULL AND v_pq IS NOT NULL THEN\n UPDATE job_dependency\n SET status = 'processed', value = p_return_value\n WHERE parent_queue = v_pq AND parent_id = v_pid\n AND child_key = p_queue || ':' || p_id;\n\n UPDATE job\n SET pending_deps = GREATEST(pending_deps - 1, 0)\n WHERE queue = v_pq AND id = v_pid\n RETURNING pending_deps INTO v_remaining;\n\n IF v_remaining = 0 THEN\n PERFORM move_parent_to_wait(v_pq, v_pid, p_finished_on);\n END IF;\n END IF;\n\n PERFORM apply_retention(\n p_queue, p_id, 'completed', p_finished_on, p_remove_all, p_keep_age, p_keep_count\n );\n\n IF v_dedup_id IS NOT NULL THEN\n PERFORM pg_advisory_xact_lock(\n hashtext('bullmq:dedup:' || p_queue || ':' || v_dedup_id));\n END IF;\n PERFORM dedup_finalize(p_queue, v_dedup_id, p_id, p_finished_on);\n PERFORM requeue_dedup_next(p_queue, v_dedup_id, p_finished_on);\n\n IF NOT EXISTS (\n SELECT 1 FROM job WHERE queue = p_queue AND state = 'active'\n )\n AND (\n EXISTS (\n SELECT 1 FROM meta\n WHERE queue = p_queue AND field = 'paused' AND value = '1'\n )\n OR NOT EXISTS (\n SELECT 1 FROM job WHERE queue = p_queue AND state = 'waiting'\n )\n ) THEN\n PERFORM publish_event(p_queue, 'drained', '{}'::jsonb);\n END IF;\n\n RETURN p_finished_on;\nEND;\n$$;\n\nCREATE OR REPLACE FUNCTION move_to_failed(\n p_queue text, p_id text, p_token text, p_failed_reason text,\n p_stacktrace jsonb, p_finished_on bigint, p_remove_all boolean,\n p_keep_age bigint, p_keep_count integer\n) RETURNS bigint\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nDECLARE\n v_state job_state;\n v_lock text;\n v_dedup_id text;\n v_attempts integer;\nBEGIN\n PERFORM lock_finalization_queues(p_queue, p_id);\n\n PERFORM pg_advisory_xact_lock(\n hashtext('bullmq:parent:' || p_queue || ':' || p_id));\n\n SELECT state, lock_token, dedup_id INTO v_state, v_lock, v_dedup_id\n FROM job WHERE queue = p_queue AND id = p_id FOR UPDATE;\n IF NOT FOUND THEN\n RAISE EXCEPTION 'bullmq: missing job %', p_id USING ERRCODE = 'BM001', DETAIL = '-1';\n END IF;\n IF v_state <> 'active' THEN\n RAISE EXCEPTION 'bullmq: job % not active', p_id USING ERRCODE = 'BM001', DETAIL = '-3';\n END IF;\n IF p_token <> '0' AND v_lock IS DISTINCT FROM p_token THEN\n RAISE EXCEPTION 'bullmq: job % lock mismatch', p_id USING ERRCODE = 'BM001', DETAIL = '-6';\n END IF;\n\n UPDATE job\n SET state = 'failed',\n failed_reason = p_failed_reason,\n stacktrace = COALESCE(p_stacktrace, stacktrace),\n finished_at_ms = p_finished_on,\n lock_token = NULL,\n locked_until_ms = NULL,\n deferred_failure = NULL,\n attempts_made = attempts_made + 1\n WHERE queue = p_queue AND id = p_id\n RETURNING attempts_made INTO v_attempts;\n\n PERFORM publish_event(p_queue, 'failed',\n jsonb_build_object('jobId', p_id, 'failedReason', p_failed_reason, 'prev', 'active'));\n PERFORM publish_event(p_queue, 'retries-exhausted',\n jsonb_build_object('jobId', p_id, 'attemptsMade', v_attempts));\n PERFORM handle_child_failure(p_queue, p_id, p_failed_reason, p_finished_on);\n PERFORM apply_retention(\n p_queue, p_id, 'failed', p_finished_on, p_remove_all, p_keep_age, p_keep_count\n );\n\n IF v_dedup_id IS NOT NULL THEN\n PERFORM pg_advisory_xact_lock(\n hashtext('bullmq:dedup:' || p_queue || ':' || v_dedup_id));\n END IF;\n PERFORM dedup_finalize(p_queue, v_dedup_id, p_id, p_finished_on);\n PERFORM requeue_dedup_next(p_queue, v_dedup_id, p_finished_on);\n\n IF NOT EXISTS (\n SELECT 1 FROM job WHERE queue = p_queue AND state = 'active'\n )\n AND (\n EXISTS (\n SELECT 1 FROM meta\n WHERE queue = p_queue AND field = 'paused' AND value = '1'\n )\n OR NOT EXISTS (\n SELECT 1 FROM job WHERE queue = p_queue AND state = 'waiting'\n )\n ) THEN\n PERFORM publish_event(p_queue, 'drained', '{}'::jsonb);\n END IF;\n\n RETURN p_finished_on;\nEND;\n$$;\n\nCREATE OR REPLACE FUNCTION obliterate(\n p_queue text, p_count integer, p_force boolean\n) RETURNS integer\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nDECLARE\n v_ids text[];\n v_final boolean;\nBEGIN\n PERFORM pg_advisory_xact_lock(hashtext('bullmq:finalization'), hashtext(p_queue));\n\n IF NOT EXISTS (\n SELECT 1 FROM meta\n WHERE queue = p_queue AND field = 'paused' AND value = '1'\n ) THEN\n RETURN -1;\n END IF;\n\n IF NOT p_force AND EXISTS (\n SELECT 1 FROM job WHERE queue = p_queue AND state = 'active'\n ) THEN\n RETURN -2;\n END IF;\n\n SELECT COALESCE(array_agg(id), ARRAY[]::text[]) INTO v_ids\n FROM (\n SELECT id FROM job WHERE queue = p_queue ORDER BY id LIMIT p_count\n ) AS batch;\n v_final := cardinality(v_ids) < p_count;\n\n IF v_final THEN\n PERFORM dedup_id FROM dedup\n WHERE queue = p_queue\n ORDER BY dedup_id\n FOR UPDATE;\n ELSE\n PERFORM dedup_id FROM dedup\n WHERE queue = p_queue AND job_id = ANY(v_ids)\n ORDER BY dedup_id\n FOR UPDATE;\n END IF;\n\n DELETE FROM job WHERE queue = p_queue AND id = ANY(v_ids);\n\n IF NOT v_final THEN\n RETURN 1;\n END IF;\n\n DELETE FROM scheduler WHERE queue = p_queue;\n DELETE FROM rate_limit WHERE queue = p_queue;\n DELETE FROM event WHERE queue = p_queue;\n DELETE FROM metrics WHERE queue = p_queue;\n DELETE FROM dedup WHERE queue = p_queue;\n DELETE FROM dedup_next WHERE queue = p_queue;\n DELETE FROM meta WHERE queue = p_queue;\n EXECUTE format(\n 'DROP SEQUENCE IF EXISTS %I', job_id_seq_name(p_queue));\n RETURN 0;\nEND;\n$$;\n",
|
|
135
|
+
"0011_obliterate_producer_lock.sql": "CREATE FUNCTION lock_producer_queues(p_entries jsonb) RETURNS void\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nDECLARE\n v_key integer;\nBEGIN\n FOR v_key IN\n SELECT DISTINCT hashtext(q)\n FROM (\n SELECT entry ->> 'queue' AS q\n FROM jsonb_array_elements(p_entries) AS entries(entry)\n UNION ALL\n SELECT entry ->> 'parentQueue'\n FROM jsonb_array_elements(p_entries) AS entries(entry)\n ) AS queues\n WHERE q IS NOT NULL\n ORDER BY 1\n LOOP\n PERFORM pg_advisory_xact_lock_shared(\n hashtext('bullmq:finalization'), v_key);\n END LOOP;\nEND;\n$$;\n\nALTER FUNCTION add_job(\n text, text, text, jsonb, jsonb, integer, bigint, bigint, integer,\n text, text, text, text, text, boolean\n) RENAME TO add_job_unlocked;\n\nCREATE FUNCTION add_job(\n p_queue text,\n p_id text,\n p_name text,\n p_data jsonb,\n p_opts jsonb,\n p_priority integer,\n p_delay bigint,\n p_timestamp bigint,\n p_max_attempts integer,\n p_parent_queue text,\n p_parent_id text,\n p_parent_key text,\n p_dedup_id text,\n p_scheduler_id text,\n p_lifo boolean\n) RETURNS text\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nBEGIN\n PERFORM lock_producer_queues(jsonb_build_array(jsonb_build_object(\n 'queue', p_queue, 'parentQueue', p_parent_queue)));\n RETURN add_job_unlocked(\n p_queue, p_id, p_name, p_data, p_opts, p_priority, p_delay, p_timestamp,\n p_max_attempts, p_parent_queue, p_parent_id, p_parent_key, p_dedup_id,\n p_scheduler_id, p_lifo);\nEND;\n$$;\n\nALTER FUNCTION add_flow(jsonb) RENAME TO add_flow_unlocked;\n\nCREATE FUNCTION add_flow(p_entries jsonb) RETURNS SETOF text\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nBEGIN\n PERFORM lock_producer_queues(p_entries);\n RETURN QUERY SELECT * FROM add_flow_unlocked(p_entries);\nEND;\n$$;\n\nALTER FUNCTION add_jobs_bulk(text, jsonb) RENAME TO add_jobs_bulk_unlocked;\n\nCREATE FUNCTION add_jobs_bulk(p_queue text, p_entries jsonb) RETURNS SETOF text\nLANGUAGE plpgsql\nSET search_path FROM CURRENT\nAS $$\nBEGIN\n PERFORM lock_producer_queues(jsonb_build_array(\n jsonb_build_object('queue', p_queue)));\n RETURN QUERY SELECT * FROM add_jobs_bulk_unlocked(p_queue, p_entries);\nEND;\n$$;\n",
|
|
130
136
|
};
|
|
@@ -298,7 +298,7 @@ local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEven
|
|
|
298
298
|
return false
|
|
299
299
|
end
|
|
300
300
|
local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
301
|
-
prefix, jobName, jobData, fullOpts, parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
301
|
+
prefix, jobName, jobData, fullOpts, parentKey, parentData, parentDependenciesKey, repeatJobKey, timestamp)
|
|
302
302
|
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
|
303
303
|
if deduplicationId then
|
|
304
304
|
if deduplicationOpts['replace'] then
|
|
@@ -341,9 +341,17 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
341
341
|
return
|
|
342
342
|
end
|
|
343
343
|
else
|
|
344
|
-
|
|
344
|
+
local currentDeduplicatedJobId = deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
345
345
|
jobId, deduplicationKey, eventsKey, maxEvents, prefix, jobName, jobData, fullOpts,
|
|
346
346
|
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
347
|
+
if currentDeduplicatedJobId and deduplicationOpts['reschedule'] and
|
|
348
|
+
rcall('ZSCORE', delayedKey, currentDeduplicatedJobId) then
|
|
349
|
+
local delay = tonumber(fullOpts['delay']) or 0
|
|
350
|
+
rcall('HSET', prefix .. currentDeduplicatedJobId, 'delay', delay)
|
|
351
|
+
addDelayedJob(currentDeduplicatedJobId, delayedKey, eventsKey, timestamp,
|
|
352
|
+
maxEvents, prefix .. 'marker', delay)
|
|
353
|
+
end
|
|
354
|
+
return currentDeduplicatedJobId
|
|
347
355
|
end
|
|
348
356
|
end
|
|
349
357
|
end
|
|
@@ -590,7 +598,7 @@ else
|
|
|
590
598
|
end
|
|
591
599
|
local deduplicationJobId = deduplicateJob(opts['de'], jobId, delayedKey, deduplicationKey,
|
|
592
600
|
eventsKey, maxEvents, args[1], args[3], ARGV[2], opts,
|
|
593
|
-
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
601
|
+
parentKey, parentData, parentDependenciesKey, repeatJobKey, timestamp)
|
|
594
602
|
if deduplicationJobId then
|
|
595
603
|
return deduplicationJobId
|
|
596
604
|
end
|
|
@@ -82,6 +82,70 @@ end
|
|
|
82
82
|
Function to debounce a job.
|
|
83
83
|
]]
|
|
84
84
|
-- Includes
|
|
85
|
+
--[[
|
|
86
|
+
Adds a delayed job to the queue by doing the following:
|
|
87
|
+
- Creates a new job key with the job data.
|
|
88
|
+
- adds to delayed zset.
|
|
89
|
+
- Emits a global event 'delayed' if the job is delayed.
|
|
90
|
+
]]
|
|
91
|
+
-- Includes
|
|
92
|
+
--[[
|
|
93
|
+
Add delay marker if needed.
|
|
94
|
+
]]
|
|
95
|
+
-- Includes
|
|
96
|
+
--[[
|
|
97
|
+
Function to return the next delayed job timestamp.
|
|
98
|
+
]]
|
|
99
|
+
local function getNextDelayedTimestamp(delayedKey)
|
|
100
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
|
101
|
+
if #result then
|
|
102
|
+
local nextTimestamp = tonumber(result[2])
|
|
103
|
+
if nextTimestamp ~= nil then
|
|
104
|
+
return nextTimestamp / 0x1000
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
109
|
+
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
|
110
|
+
if nextTimestamp ~= nil then
|
|
111
|
+
-- Replace the score of the marker with the newest known
|
|
112
|
+
-- next timestamp.
|
|
113
|
+
rcall("ZADD", markerKey, nextTimestamp, "1")
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
--[[
|
|
117
|
+
Bake in the job id first 12 bits into the timestamp
|
|
118
|
+
to guarantee correct execution order of delayed jobs
|
|
119
|
+
(up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
|
|
120
|
+
WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
|
|
121
|
+
]]
|
|
122
|
+
local function getDelayedScore(delayedKey, timestamp, delay)
|
|
123
|
+
local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
|
|
124
|
+
local minScore = delayedTimestamp * 0x1000
|
|
125
|
+
local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
|
|
126
|
+
local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
|
|
127
|
+
minScore, "WITHSCORES","LIMIT", 0, 1)
|
|
128
|
+
if #result then
|
|
129
|
+
local currentMaxScore = tonumber(result[2])
|
|
130
|
+
if currentMaxScore ~= nil then
|
|
131
|
+
if currentMaxScore >= maxScore then
|
|
132
|
+
return maxScore, delayedTimestamp
|
|
133
|
+
else
|
|
134
|
+
return currentMaxScore + 1, delayedTimestamp
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
return minScore, delayedTimestamp
|
|
139
|
+
end
|
|
140
|
+
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
|
|
141
|
+
maxEvents, markerKey, delay)
|
|
142
|
+
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
|
|
143
|
+
rcall("ZADD", delayedKey, score, jobId)
|
|
144
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
|
145
|
+
"jobId", jobId, "delay", delayedTimestamp)
|
|
146
|
+
-- mark that a delayed job is available
|
|
147
|
+
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
148
|
+
end
|
|
85
149
|
--[[
|
|
86
150
|
Function to deduplicate a job.
|
|
87
151
|
]]
|
|
@@ -263,7 +327,7 @@ local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEven
|
|
|
263
327
|
return false
|
|
264
328
|
end
|
|
265
329
|
local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
266
|
-
prefix, jobName, jobData, fullOpts, parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
330
|
+
prefix, jobName, jobData, fullOpts, parentKey, parentData, parentDependenciesKey, repeatJobKey, timestamp)
|
|
267
331
|
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
|
268
332
|
if deduplicationId then
|
|
269
333
|
if deduplicationOpts['replace'] then
|
|
@@ -306,9 +370,17 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
306
370
|
return
|
|
307
371
|
end
|
|
308
372
|
else
|
|
309
|
-
|
|
373
|
+
local currentDeduplicatedJobId = deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
310
374
|
jobId, deduplicationKey, eventsKey, maxEvents, prefix, jobName, jobData, fullOpts,
|
|
311
375
|
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
376
|
+
if currentDeduplicatedJobId and deduplicationOpts['reschedule'] and
|
|
377
|
+
rcall('ZSCORE', delayedKey, currentDeduplicatedJobId) then
|
|
378
|
+
local delay = tonumber(fullOpts['delay']) or 0
|
|
379
|
+
rcall('HSET', prefix .. currentDeduplicatedJobId, 'delay', delay)
|
|
380
|
+
addDelayedJob(currentDeduplicatedJobId, delayedKey, eventsKey, timestamp,
|
|
381
|
+
maxEvents, prefix .. 'marker', delay)
|
|
382
|
+
end
|
|
383
|
+
return currentDeduplicatedJobId
|
|
312
384
|
end
|
|
313
385
|
end
|
|
314
386
|
end
|
|
@@ -387,30 +459,6 @@ end
|
|
|
387
459
|
Move parent to a wait status (wait, prioritized or delayed)
|
|
388
460
|
]]
|
|
389
461
|
-- Includes
|
|
390
|
-
--[[
|
|
391
|
-
Add delay marker if needed.
|
|
392
|
-
]]
|
|
393
|
-
-- Includes
|
|
394
|
-
--[[
|
|
395
|
-
Function to return the next delayed job timestamp.
|
|
396
|
-
]]
|
|
397
|
-
local function getNextDelayedTimestamp(delayedKey)
|
|
398
|
-
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
|
399
|
-
if #result then
|
|
400
|
-
local nextTimestamp = tonumber(result[2])
|
|
401
|
-
if nextTimestamp ~= nil then
|
|
402
|
-
return nextTimestamp / 0x1000
|
|
403
|
-
end
|
|
404
|
-
end
|
|
405
|
-
end
|
|
406
|
-
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
407
|
-
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
|
408
|
-
if nextTimestamp ~= nil then
|
|
409
|
-
-- Replace the score of the marker with the newest known
|
|
410
|
-
-- next timestamp.
|
|
411
|
-
rcall("ZADD", markerKey, nextTimestamp, "1")
|
|
412
|
-
end
|
|
413
|
-
end
|
|
414
462
|
--[[
|
|
415
463
|
Function to add job in target list and add marker if needed.
|
|
416
464
|
]]
|
|
@@ -553,7 +601,7 @@ else
|
|
|
553
601
|
end
|
|
554
602
|
local deduplicationJobId = deduplicateJob(opts['de'], jobId, KEYS[5],
|
|
555
603
|
deduplicationKey, eventsKey, maxEvents, args[1], args[3], ARGV[2], opts,
|
|
556
|
-
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
604
|
+
parentKey, parentData, parentDependenciesKey, repeatJobKey, timestamp)
|
|
557
605
|
if deduplicationJobId then
|
|
558
606
|
return deduplicationJobId
|
|
559
607
|
end
|
|
@@ -79,6 +79,70 @@ end
|
|
|
79
79
|
Function to debounce a job.
|
|
80
80
|
]]
|
|
81
81
|
-- Includes
|
|
82
|
+
--[[
|
|
83
|
+
Adds a delayed job to the queue by doing the following:
|
|
84
|
+
- Creates a new job key with the job data.
|
|
85
|
+
- adds to delayed zset.
|
|
86
|
+
- Emits a global event 'delayed' if the job is delayed.
|
|
87
|
+
]]
|
|
88
|
+
-- Includes
|
|
89
|
+
--[[
|
|
90
|
+
Add delay marker if needed.
|
|
91
|
+
]]
|
|
92
|
+
-- Includes
|
|
93
|
+
--[[
|
|
94
|
+
Function to return the next delayed job timestamp.
|
|
95
|
+
]]
|
|
96
|
+
local function getNextDelayedTimestamp(delayedKey)
|
|
97
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
|
98
|
+
if #result then
|
|
99
|
+
local nextTimestamp = tonumber(result[2])
|
|
100
|
+
if nextTimestamp ~= nil then
|
|
101
|
+
return nextTimestamp / 0x1000
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
106
|
+
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
|
107
|
+
if nextTimestamp ~= nil then
|
|
108
|
+
-- Replace the score of the marker with the newest known
|
|
109
|
+
-- next timestamp.
|
|
110
|
+
rcall("ZADD", markerKey, nextTimestamp, "1")
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
--[[
|
|
114
|
+
Bake in the job id first 12 bits into the timestamp
|
|
115
|
+
to guarantee correct execution order of delayed jobs
|
|
116
|
+
(up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
|
|
117
|
+
WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
|
|
118
|
+
]]
|
|
119
|
+
local function getDelayedScore(delayedKey, timestamp, delay)
|
|
120
|
+
local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
|
|
121
|
+
local minScore = delayedTimestamp * 0x1000
|
|
122
|
+
local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
|
|
123
|
+
local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
|
|
124
|
+
minScore, "WITHSCORES","LIMIT", 0, 1)
|
|
125
|
+
if #result then
|
|
126
|
+
local currentMaxScore = tonumber(result[2])
|
|
127
|
+
if currentMaxScore ~= nil then
|
|
128
|
+
if currentMaxScore >= maxScore then
|
|
129
|
+
return maxScore, delayedTimestamp
|
|
130
|
+
else
|
|
131
|
+
return currentMaxScore + 1, delayedTimestamp
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
return minScore, delayedTimestamp
|
|
136
|
+
end
|
|
137
|
+
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
|
|
138
|
+
maxEvents, markerKey, delay)
|
|
139
|
+
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
|
|
140
|
+
rcall("ZADD", delayedKey, score, jobId)
|
|
141
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
|
142
|
+
"jobId", jobId, "delay", delayedTimestamp)
|
|
143
|
+
-- mark that a delayed job is available
|
|
144
|
+
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
145
|
+
end
|
|
82
146
|
--[[
|
|
83
147
|
Function to deduplicate a job.
|
|
84
148
|
]]
|
|
@@ -260,7 +324,7 @@ local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEven
|
|
|
260
324
|
return false
|
|
261
325
|
end
|
|
262
326
|
local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
263
|
-
prefix, jobName, jobData, fullOpts, parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
327
|
+
prefix, jobName, jobData, fullOpts, parentKey, parentData, parentDependenciesKey, repeatJobKey, timestamp)
|
|
264
328
|
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
|
265
329
|
if deduplicationId then
|
|
266
330
|
if deduplicationOpts['replace'] then
|
|
@@ -303,9 +367,17 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
303
367
|
return
|
|
304
368
|
end
|
|
305
369
|
else
|
|
306
|
-
|
|
370
|
+
local currentDeduplicatedJobId = deduplicateJobWithoutReplace(deduplicationId, deduplicationOpts,
|
|
307
371
|
jobId, deduplicationKey, eventsKey, maxEvents, prefix, jobName, jobData, fullOpts,
|
|
308
372
|
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
373
|
+
if currentDeduplicatedJobId and deduplicationOpts['reschedule'] and
|
|
374
|
+
rcall('ZSCORE', delayedKey, currentDeduplicatedJobId) then
|
|
375
|
+
local delay = tonumber(fullOpts['delay']) or 0
|
|
376
|
+
rcall('HSET', prefix .. currentDeduplicatedJobId, 'delay', delay)
|
|
377
|
+
addDelayedJob(currentDeduplicatedJobId, delayedKey, eventsKey, timestamp,
|
|
378
|
+
maxEvents, prefix .. 'marker', delay)
|
|
379
|
+
end
|
|
380
|
+
return currentDeduplicatedJobId
|
|
309
381
|
end
|
|
310
382
|
end
|
|
311
383
|
end
|
|
@@ -354,30 +426,6 @@ end
|
|
|
354
426
|
Move parent to a wait status (wait, prioritized or delayed)
|
|
355
427
|
]]
|
|
356
428
|
-- Includes
|
|
357
|
-
--[[
|
|
358
|
-
Add delay marker if needed.
|
|
359
|
-
]]
|
|
360
|
-
-- Includes
|
|
361
|
-
--[[
|
|
362
|
-
Function to return the next delayed job timestamp.
|
|
363
|
-
]]
|
|
364
|
-
local function getNextDelayedTimestamp(delayedKey)
|
|
365
|
-
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
|
366
|
-
if #result then
|
|
367
|
-
local nextTimestamp = tonumber(result[2])
|
|
368
|
-
if nextTimestamp ~= nil then
|
|
369
|
-
return nextTimestamp / 0x1000
|
|
370
|
-
end
|
|
371
|
-
end
|
|
372
|
-
end
|
|
373
|
-
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
374
|
-
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
|
375
|
-
if nextTimestamp ~= nil then
|
|
376
|
-
-- Replace the score of the marker with the newest known
|
|
377
|
-
-- next timestamp.
|
|
378
|
-
rcall("ZADD", markerKey, nextTimestamp, "1")
|
|
379
|
-
end
|
|
380
|
-
end
|
|
381
429
|
--[[
|
|
382
430
|
Function to add job considering priority.
|
|
383
431
|
]]
|
|
@@ -559,7 +607,7 @@ else
|
|
|
559
607
|
end
|
|
560
608
|
local deduplicationJobId = deduplicateJob(opts['de'], jobId, KEYS[6],
|
|
561
609
|
deduplicationKey, eventsKey, maxEvents, args[1], args[3], ARGV[2], opts,
|
|
562
|
-
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
|
610
|
+
parentKey, parentData, parentDependenciesKey, repeatJobKey, timestamp)
|
|
563
611
|
if deduplicationJobId then
|
|
564
612
|
return deduplicationJobId
|
|
565
613
|
end
|