bullmq 6.3.11 → 6.3.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/queue-getters.js +3 -1
- package/dist/cjs/classes/queue.js +2 -1
- package/dist/cjs/classes/redis-connection.js +1 -1
- package/dist/cjs/classes/redis-queue-backend.js +1 -1
- package/dist/cjs/commands/includes/moveChildBackToParentDependencies.lua +20 -0
- package/dist/cjs/commands/moveJobsToWait-8.lua +3 -0
- package/dist/cjs/commands/reprocessJob-7.lua +2 -14
- package/dist/cjs/postgres/sql-loader.js +27 -41
- package/dist/cjs/postgres/sql-scripts.js +110 -0
- package/dist/cjs/scripts/moveJobsToWait-8.js +21 -0
- package/dist/cjs/scripts/reprocessJob-7.js +20 -13
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/queue-getters.d.ts +3 -1
- package/dist/esm/classes/queue-getters.js +3 -1
- package/dist/esm/classes/queue.d.ts +2 -1
- package/dist/esm/classes/queue.js +2 -1
- package/dist/esm/classes/redis-connection.d.ts +1 -1
- package/dist/esm/classes/redis-connection.js +1 -1
- package/dist/esm/classes/redis-queue-backend.d.ts +1 -1
- package/dist/esm/classes/redis-queue-backend.js +1 -1
- package/dist/esm/commands/includes/moveChildBackToParentDependencies.lua +20 -0
- package/dist/esm/commands/moveJobsToWait-8.lua +3 -0
- package/dist/esm/commands/reprocessJob-7.lua +2 -14
- package/dist/esm/postgres/sql-loader.d.ts +6 -1
- package/dist/esm/postgres/sql-loader.js +27 -41
- package/dist/esm/postgres/sql-scripts.d.ts +10 -0
- package/dist/esm/postgres/sql-scripts.js +107 -0
- package/dist/esm/scripts/moveJobsToWait-8.js +21 -0
- package/dist/esm/scripts/reprocessJob-7.js +20 -13
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +10 -9
- package/dist/cjs/postgres/commands/add_flow.sql +0 -3
- package/dist/cjs/postgres/commands/add_job.sql +0 -5
- package/dist/cjs/postgres/commands/add_job_scheduler.sql +0 -6
- package/dist/cjs/postgres/commands/add_jobs_bulk.sql +0 -3
- package/dist/cjs/postgres/commands/add_log.sql +0 -14
- package/dist/cjs/postgres/commands/change_delay.sql +0 -3
- package/dist/cjs/postgres/commands/change_priority.sql +0 -3
- package/dist/cjs/postgres/commands/clean.sql +0 -3
- package/dist/cjs/postgres/commands/clear_logs.sql +0 -9
- package/dist/cjs/postgres/commands/collect_metrics.sql +0 -4
- package/dist/cjs/postgres/commands/delete_deduplication_key.sql +0 -6
- package/dist/cjs/postgres/commands/drain.sql +0 -2
- package/dist/cjs/postgres/commands/extend_lock.sql +0 -3
- package/dist/cjs/postgres/commands/extend_locks.sql +0 -25
- package/dist/cjs/postgres/commands/get_client_list.sql +0 -11
- package/dist/cjs/postgres/commands/get_counts.sql +0 -13
- package/dist/cjs/postgres/commands/get_counts_per_priority.sql +0 -14
- package/dist/cjs/postgres/commands/get_deduplication_job_id.sql +0 -7
- package/dist/cjs/postgres/commands/get_dependencies.sql +0 -8
- package/dist/cjs/postgres/commands/get_dependencies_page.sql +0 -7
- package/dist/cjs/postgres/commands/get_dependency_counts.sql +0 -9
- package/dist/cjs/postgres/commands/get_ignored_children_failures.sql +0 -7
- package/dist/cjs/postgres/commands/get_job_data.sql +0 -2
- package/dist/cjs/postgres/commands/get_job_logs_asc.sql +0 -6
- package/dist/cjs/postgres/commands/get_job_logs_count.sql +0 -2
- package/dist/cjs/postgres/commands/get_job_logs_desc.sql +0 -6
- package/dist/cjs/postgres/commands/get_job_scheduler.sql +0 -6
- package/dist/cjs/postgres/commands/get_job_schedulers_count.sql +0 -2
- package/dist/cjs/postgres/commands/get_job_schedulers_range.sql +0 -10
- package/dist/cjs/postgres/commands/get_metrics.sql +0 -14
- package/dist/cjs/postgres/commands/get_processed_children_values.sql +0 -8
- package/dist/cjs/postgres/commands/get_queue_meta.sql +0 -2
- package/dist/cjs/postgres/commands/get_queue_meta_field.sql +0 -2
- package/dist/cjs/postgres/commands/get_queue_meta_fields.sql +0 -3
- package/dist/cjs/postgres/commands/get_range.sql +0 -3
- package/dist/cjs/postgres/commands/get_rate_limit_ttl.sql +0 -3
- package/dist/cjs/postgres/commands/get_state.sql +0 -2
- package/dist/cjs/postgres/commands/has_queue_meta_field.sql +0 -4
- package/dist/cjs/postgres/commands/has_waiting_job.sql +0 -11
- package/dist/cjs/postgres/commands/is_finished.sql +0 -4
- package/dist/cjs/postgres/commands/is_job_in_state.sql +0 -6
- package/dist/cjs/postgres/commands/is_job_in_wait.sql +0 -15
- package/dist/cjs/postgres/commands/is_job_prioritized.sql +0 -6
- package/dist/cjs/postgres/commands/is_job_scheduler.sql +0 -4
- package/dist/cjs/postgres/commands/is_maxed.sql +0 -10
- package/dist/cjs/postgres/commands/listen_events.sql +0 -3
- package/dist/cjs/postgres/commands/listen_jobs.sql +0 -4
- package/dist/cjs/postgres/commands/move_active_to_wait.sql +0 -4
- package/dist/cjs/postgres/commands/move_stalled_jobs_to_wait.sql +0 -4
- package/dist/cjs/postgres/commands/move_to_active.sql +0 -4
- package/dist/cjs/postgres/commands/move_to_completed.sql +0 -4
- package/dist/cjs/postgres/commands/move_to_completed_fetch.sql +0 -9
- package/dist/cjs/postgres/commands/move_to_delayed.sql +0 -4
- package/dist/cjs/postgres/commands/move_to_failed.sql +0 -4
- package/dist/cjs/postgres/commands/move_to_failed_fetch.sql +0 -9
- package/dist/cjs/postgres/commands/move_to_waiting_children.sql +0 -3
- package/dist/cjs/postgres/commands/next_delay.sql +0 -2
- package/dist/cjs/postgres/commands/next_signal.sql +0 -3
- package/dist/cjs/postgres/commands/obliterate.sql +0 -4
- package/dist/cjs/postgres/commands/paginate_dependencies.sql +0 -19
- package/dist/cjs/postgres/commands/pause.sql +0 -2
- package/dist/cjs/postgres/commands/promote.sql +0 -3
- package/dist/cjs/postgres/commands/promote_jobs.sql +0 -3
- package/dist/cjs/postgres/commands/publish_event.sql +0 -3
- package/dist/cjs/postgres/commands/read_events.sql +0 -6
- package/dist/cjs/postgres/commands/read_events_max.sql +0 -3
- package/dist/cjs/postgres/commands/remove.sql +0 -3
- package/dist/cjs/postgres/commands/remove_child_dependency.sql +0 -4
- package/dist/cjs/postgres/commands/remove_deduplication_key.sql +0 -9
- package/dist/cjs/postgres/commands/remove_job_scheduler.sql +0 -3
- package/dist/cjs/postgres/commands/remove_queue_meta_fields.sql +0 -2
- package/dist/cjs/postgres/commands/remove_rate_limit.sql +0 -6
- package/dist/cjs/postgres/commands/remove_unprocessed_children.sql +0 -3
- package/dist/cjs/postgres/commands/reprocess_job.sql +0 -4
- package/dist/cjs/postgres/commands/retry_job.sql +0 -3
- package/dist/cjs/postgres/commands/retry_jobs.sql +0 -3
- package/dist/cjs/postgres/commands/set_queue_meta.sql +0 -4
- package/dist/cjs/postgres/commands/set_rate_limit.sql +0 -3
- package/dist/cjs/postgres/commands/trim_logs.sql +0 -3
- package/dist/cjs/postgres/commands/update_data.sql +0 -3
- package/dist/cjs/postgres/commands/update_job_scheduler.sql +0 -5
- package/dist/cjs/postgres/commands/update_progress.sql +0 -3
- package/dist/cjs/postgres/migrations/0001_schema.sql +0 -348
- package/dist/cjs/postgres/migrations/0002_functions.sql +0 -3473
- package/dist/cjs/postgres/migrations/0003_dedup_stale_key.sql +0 -147
- package/dist/esm/postgres/commands/add_flow.sql +0 -3
- package/dist/esm/postgres/commands/add_job.sql +0 -5
- package/dist/esm/postgres/commands/add_job_scheduler.sql +0 -6
- package/dist/esm/postgres/commands/add_jobs_bulk.sql +0 -3
- package/dist/esm/postgres/commands/add_log.sql +0 -14
- package/dist/esm/postgres/commands/change_delay.sql +0 -3
- package/dist/esm/postgres/commands/change_priority.sql +0 -3
- package/dist/esm/postgres/commands/clean.sql +0 -3
- package/dist/esm/postgres/commands/clear_logs.sql +0 -9
- package/dist/esm/postgres/commands/collect_metrics.sql +0 -4
- package/dist/esm/postgres/commands/delete_deduplication_key.sql +0 -6
- package/dist/esm/postgres/commands/drain.sql +0 -2
- package/dist/esm/postgres/commands/extend_lock.sql +0 -3
- package/dist/esm/postgres/commands/extend_locks.sql +0 -25
- package/dist/esm/postgres/commands/get_client_list.sql +0 -11
- package/dist/esm/postgres/commands/get_counts.sql +0 -13
- package/dist/esm/postgres/commands/get_counts_per_priority.sql +0 -14
- package/dist/esm/postgres/commands/get_deduplication_job_id.sql +0 -7
- package/dist/esm/postgres/commands/get_dependencies.sql +0 -8
- package/dist/esm/postgres/commands/get_dependencies_page.sql +0 -7
- package/dist/esm/postgres/commands/get_dependency_counts.sql +0 -9
- package/dist/esm/postgres/commands/get_ignored_children_failures.sql +0 -7
- package/dist/esm/postgres/commands/get_job_data.sql +0 -2
- package/dist/esm/postgres/commands/get_job_logs_asc.sql +0 -6
- package/dist/esm/postgres/commands/get_job_logs_count.sql +0 -2
- package/dist/esm/postgres/commands/get_job_logs_desc.sql +0 -6
- package/dist/esm/postgres/commands/get_job_scheduler.sql +0 -6
- package/dist/esm/postgres/commands/get_job_schedulers_count.sql +0 -2
- package/dist/esm/postgres/commands/get_job_schedulers_range.sql +0 -10
- package/dist/esm/postgres/commands/get_metrics.sql +0 -14
- package/dist/esm/postgres/commands/get_processed_children_values.sql +0 -8
- package/dist/esm/postgres/commands/get_queue_meta.sql +0 -2
- package/dist/esm/postgres/commands/get_queue_meta_field.sql +0 -2
- package/dist/esm/postgres/commands/get_queue_meta_fields.sql +0 -3
- package/dist/esm/postgres/commands/get_range.sql +0 -3
- package/dist/esm/postgres/commands/get_rate_limit_ttl.sql +0 -3
- package/dist/esm/postgres/commands/get_state.sql +0 -2
- package/dist/esm/postgres/commands/has_queue_meta_field.sql +0 -4
- package/dist/esm/postgres/commands/has_waiting_job.sql +0 -11
- package/dist/esm/postgres/commands/is_finished.sql +0 -4
- package/dist/esm/postgres/commands/is_job_in_state.sql +0 -6
- package/dist/esm/postgres/commands/is_job_in_wait.sql +0 -15
- package/dist/esm/postgres/commands/is_job_prioritized.sql +0 -6
- package/dist/esm/postgres/commands/is_job_scheduler.sql +0 -4
- package/dist/esm/postgres/commands/is_maxed.sql +0 -10
- package/dist/esm/postgres/commands/listen_events.sql +0 -3
- package/dist/esm/postgres/commands/listen_jobs.sql +0 -4
- package/dist/esm/postgres/commands/move_active_to_wait.sql +0 -4
- package/dist/esm/postgres/commands/move_stalled_jobs_to_wait.sql +0 -4
- package/dist/esm/postgres/commands/move_to_active.sql +0 -4
- package/dist/esm/postgres/commands/move_to_completed.sql +0 -4
- package/dist/esm/postgres/commands/move_to_completed_fetch.sql +0 -9
- package/dist/esm/postgres/commands/move_to_delayed.sql +0 -4
- package/dist/esm/postgres/commands/move_to_failed.sql +0 -4
- package/dist/esm/postgres/commands/move_to_failed_fetch.sql +0 -9
- package/dist/esm/postgres/commands/move_to_waiting_children.sql +0 -3
- package/dist/esm/postgres/commands/next_delay.sql +0 -2
- package/dist/esm/postgres/commands/next_signal.sql +0 -3
- package/dist/esm/postgres/commands/obliterate.sql +0 -4
- package/dist/esm/postgres/commands/paginate_dependencies.sql +0 -19
- package/dist/esm/postgres/commands/pause.sql +0 -2
- package/dist/esm/postgres/commands/promote.sql +0 -3
- package/dist/esm/postgres/commands/promote_jobs.sql +0 -3
- package/dist/esm/postgres/commands/publish_event.sql +0 -3
- package/dist/esm/postgres/commands/read_events.sql +0 -6
- package/dist/esm/postgres/commands/read_events_max.sql +0 -3
- package/dist/esm/postgres/commands/remove.sql +0 -3
- package/dist/esm/postgres/commands/remove_child_dependency.sql +0 -4
- package/dist/esm/postgres/commands/remove_deduplication_key.sql +0 -9
- package/dist/esm/postgres/commands/remove_job_scheduler.sql +0 -3
- package/dist/esm/postgres/commands/remove_queue_meta_fields.sql +0 -2
- package/dist/esm/postgres/commands/remove_rate_limit.sql +0 -6
- package/dist/esm/postgres/commands/remove_unprocessed_children.sql +0 -3
- package/dist/esm/postgres/commands/reprocess_job.sql +0 -4
- package/dist/esm/postgres/commands/retry_job.sql +0 -3
- package/dist/esm/postgres/commands/retry_jobs.sql +0 -3
- package/dist/esm/postgres/commands/set_queue_meta.sql +0 -4
- package/dist/esm/postgres/commands/set_rate_limit.sql +0 -3
- package/dist/esm/postgres/commands/trim_logs.sql +0 -3
- package/dist/esm/postgres/commands/update_data.sql +0 -3
- package/dist/esm/postgres/commands/update_job_scheduler.sql +0 -5
- package/dist/esm/postgres/commands/update_progress.sql +0 -3
- package/dist/esm/postgres/migrations/0001_schema.sql +0 -348
- package/dist/esm/postgres/migrations/0002_functions.sql +0 -3473
- package/dist/esm/postgres/migrations/0003_dedup_stale_key.sql +0 -147
|
@@ -1,147 +0,0 @@
|
|
|
1
|
-
-- BullMQ PostgreSQL backend — deduplication stale-key recovery.
|
|
2
|
-
--
|
|
3
|
-
-- Replaces `deduplicate_job` (created in 0002_functions.sql) to fix two ways a
|
|
4
|
-
-- deduplication key could outlive the job it points at:
|
|
5
|
-
--
|
|
6
|
-
-- 1. Stale key: a key with no expiry is meant to disappear when its winner
|
|
7
|
-
-- finishes, but `clean`, `drain`, `obliterate` (and outages) delete the
|
|
8
|
-
-- job row without clearing the `dedup` row. Every later add then matched
|
|
9
|
-
-- that dead winner and was silently dropped, forever. Mirrors the Redis
|
|
10
|
-
-- `recoverStaleDeduplicationKey` include.
|
|
11
|
-
-- 2. Orphaned proto-next: replacing a *delayed* winner left behind any
|
|
12
|
-
-- `dedup_next` payload stashed while that winner was previously active,
|
|
13
|
-
-- so it got resurrected when the replacement finished.
|
|
14
|
-
--
|
|
15
|
-
-- Only the function body changes — no schema/DDL change — so the signature is
|
|
16
|
-
-- identical and older clients keep working against the updated schema.
|
|
17
|
-
|
|
18
|
-
CREATE OR REPLACE FUNCTION deduplicate_job(
|
|
19
|
-
p_queue text, p_dedup jsonb, p_job_id text, p_now bigint,
|
|
20
|
-
p_name text, p_data jsonb, p_opts jsonb
|
|
21
|
-
) RETURNS text
|
|
22
|
-
LANGUAGE plpgsql
|
|
23
|
-
SET search_path FROM CURRENT
|
|
24
|
-
AS $$
|
|
25
|
-
DECLARE
|
|
26
|
-
v_id text := p_dedup->>'id';
|
|
27
|
-
v_ttl bigint := NULLIF(p_dedup->>'ttl', '')::bigint;
|
|
28
|
-
v_extend boolean := COALESCE((p_dedup->>'extend')::boolean, false);
|
|
29
|
-
v_replace boolean := COALESCE((p_dedup->>'replace')::boolean, false);
|
|
30
|
-
v_keeplast boolean := COALESCE((p_dedup->>'keepLastIfActive')::boolean, false);
|
|
31
|
-
v_cur text;
|
|
32
|
-
v_exp bigint;
|
|
33
|
-
v_state job_state;
|
|
34
|
-
BEGIN
|
|
35
|
-
IF v_id IS NULL OR v_id = '' THEN
|
|
36
|
-
RETURN NULL;
|
|
37
|
-
END IF;
|
|
38
|
-
|
|
39
|
-
-- The current winner (only if the key is still live).
|
|
40
|
-
SELECT job_id, expire_at_ms INTO v_cur, v_exp
|
|
41
|
-
FROM dedup WHERE queue = p_queue AND dedup_id = v_id;
|
|
42
|
-
IF v_cur IS NULL OR (v_exp IS NOT NULL AND v_exp <= p_now) THEN
|
|
43
|
-
v_cur := NULL;
|
|
44
|
-
ELSIF v_exp IS NULL AND NOT EXISTS (
|
|
45
|
-
SELECT 1 FROM job WHERE queue = p_queue AND id = v_cur
|
|
46
|
-
) THEN
|
|
47
|
-
-- Stale key recovery (mirrors recoverStaleDeduplicationKey): a key with no
|
|
48
|
-
-- expiry is meant to disappear when its winner finishes. If the job row is
|
|
49
|
-
-- gone (clean, drain, obliterate, an outage…) the key is stale, so discard
|
|
50
|
-
-- it together with any pending proto-next (which would otherwise be
|
|
51
|
-
-- resurrected) and start a new window with the incoming job.
|
|
52
|
-
DELETE FROM dedup WHERE queue = p_queue AND dedup_id = v_id;
|
|
53
|
-
DELETE FROM dedup_next WHERE queue = p_queue AND dedup_id = v_id;
|
|
54
|
-
v_cur := NULL;
|
|
55
|
-
END IF;
|
|
56
|
-
|
|
57
|
-
IF v_replace THEN
|
|
58
|
-
IF v_cur IS NOT NULL THEN
|
|
59
|
-
SELECT state INTO v_state
|
|
60
|
-
FROM job WHERE queue = p_queue AND id = v_cur;
|
|
61
|
-
IF v_state = 'delayed' THEN
|
|
62
|
-
-- Drop the previous delayed job and take its place.
|
|
63
|
-
DELETE FROM job WHERE queue = p_queue AND id = v_cur;
|
|
64
|
-
PERFORM publish_event(p_queue, 'removed',
|
|
65
|
-
jsonb_build_object('jobId', v_cur, 'prev', 'delayed'));
|
|
66
|
-
PERFORM publish_event(p_queue, 'deduplicated',
|
|
67
|
-
jsonb_build_object('jobId', p_job_id, 'deduplicationId', v_id,
|
|
68
|
-
'deduplicatedJobId', v_cur));
|
|
69
|
-
-- Discard any pending proto-next stashed while the replaced job was
|
|
70
|
-
-- active, otherwise it would be resurrected when p_job_id finalizes.
|
|
71
|
-
DELETE FROM dedup_next WHERE queue = p_queue AND dedup_id = v_id;
|
|
72
|
-
IF v_keeplast THEN
|
|
73
|
-
UPDATE dedup SET job_id = p_job_id, expire_at_ms = NULL
|
|
74
|
-
WHERE queue = p_queue AND dedup_id = v_id;
|
|
75
|
-
ELSIF NOT v_extend AND COALESCE(v_ttl, 0) > 0 THEN
|
|
76
|
-
-- KEEPTTL: keep the existing window, just swap the winner.
|
|
77
|
-
UPDATE dedup SET job_id = p_job_id
|
|
78
|
-
WHERE queue = p_queue AND dedup_id = v_id;
|
|
79
|
-
ELSE
|
|
80
|
-
UPDATE dedup
|
|
81
|
-
SET job_id = p_job_id,
|
|
82
|
-
expire_at_ms = CASE WHEN COALESCE(v_ttl, 0) > 0
|
|
83
|
-
THEN p_now + v_ttl ELSE NULL END
|
|
84
|
-
WHERE queue = p_queue AND dedup_id = v_id;
|
|
85
|
-
END IF;
|
|
86
|
-
RETURN NULL;
|
|
87
|
-
ELSE
|
|
88
|
-
-- Winner is not a removable delayed job: stash proto-next if it is
|
|
89
|
-
-- active + keepLastIfActive, then deduplicate.
|
|
90
|
-
PERFORM dedup_store_next(p_queue, v_id, v_cur, p_job_id,
|
|
91
|
-
v_keeplast, p_name, p_data, p_opts);
|
|
92
|
-
RETURN v_cur;
|
|
93
|
-
END IF;
|
|
94
|
-
ELSE
|
|
95
|
-
INSERT INTO dedup (queue, dedup_id, job_id, expire_at_ms)
|
|
96
|
-
VALUES (p_queue, v_id, p_job_id,
|
|
97
|
-
CASE WHEN NOT v_keeplast AND COALESCE(v_ttl, 0) > 0
|
|
98
|
-
THEN p_now + v_ttl ELSE NULL END)
|
|
99
|
-
ON CONFLICT (queue, dedup_id) DO UPDATE
|
|
100
|
-
SET job_id = EXCLUDED.job_id, expire_at_ms = EXCLUDED.expire_at_ms;
|
|
101
|
-
RETURN NULL;
|
|
102
|
-
END IF;
|
|
103
|
-
END IF;
|
|
104
|
-
|
|
105
|
-
-- Without replace.
|
|
106
|
-
IF COALESCE(v_ttl, 0) > 0 AND v_extend THEN
|
|
107
|
-
IF v_cur IS NOT NULL THEN
|
|
108
|
-
-- Stash proto-next if active+keepLast; else extend the window (or
|
|
109
|
-
-- persist when keepLastIfActive). Either way keep the current winner.
|
|
110
|
-
IF NOT dedup_store_next(p_queue, v_id, v_cur, p_job_id,
|
|
111
|
-
v_keeplast, p_name, p_data, p_opts) THEN
|
|
112
|
-
UPDATE dedup
|
|
113
|
-
SET expire_at_ms = CASE WHEN v_keeplast THEN NULL ELSE p_now + v_ttl END
|
|
114
|
-
WHERE queue = p_queue AND dedup_id = v_id;
|
|
115
|
-
END IF;
|
|
116
|
-
PERFORM publish_event(p_queue, 'deduplicated',
|
|
117
|
-
jsonb_build_object('jobId', v_cur, 'deduplicationId', v_id,
|
|
118
|
-
'deduplicatedJobId', p_job_id));
|
|
119
|
-
RETURN v_cur;
|
|
120
|
-
END IF;
|
|
121
|
-
INSERT INTO dedup (queue, dedup_id, job_id, expire_at_ms)
|
|
122
|
-
VALUES (p_queue, v_id, p_job_id,
|
|
123
|
-
CASE WHEN v_keeplast THEN NULL ELSE p_now + v_ttl END)
|
|
124
|
-
ON CONFLICT (queue, dedup_id) DO UPDATE
|
|
125
|
-
SET job_id = EXCLUDED.job_id, expire_at_ms = EXCLUDED.expire_at_ms;
|
|
126
|
-
RETURN NULL;
|
|
127
|
-
END IF;
|
|
128
|
-
|
|
129
|
-
-- SET NX semantics (ttl>0 non-extend, or no ttl at all).
|
|
130
|
-
IF v_cur IS NOT NULL THEN
|
|
131
|
-
PERFORM dedup_store_next(p_queue, v_id, v_cur, p_job_id,
|
|
132
|
-
v_keeplast, p_name, p_data, p_opts);
|
|
133
|
-
PERFORM publish_event(p_queue, 'deduplicated',
|
|
134
|
-
jsonb_build_object('jobId', v_cur, 'deduplicationId', v_id,
|
|
135
|
-
'deduplicatedJobId', p_job_id));
|
|
136
|
-
RETURN v_cur;
|
|
137
|
-
END IF;
|
|
138
|
-
|
|
139
|
-
INSERT INTO dedup (queue, dedup_id, job_id, expire_at_ms)
|
|
140
|
-
VALUES (p_queue, v_id, p_job_id,
|
|
141
|
-
CASE WHEN NOT v_keeplast AND COALESCE(v_ttl, 0) > 0
|
|
142
|
-
THEN p_now + v_ttl ELSE NULL END)
|
|
143
|
-
ON CONFLICT (queue, dedup_id) DO UPDATE
|
|
144
|
-
SET job_id = EXCLUDED.job_id, expire_at_ms = EXCLUDED.expire_at_ms;
|
|
145
|
-
RETURN NULL;
|
|
146
|
-
END;
|
|
147
|
-
$$;
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
-- Register/update a scheduler and enqueue its next iteration; returns the
|
|
2
|
-
-- delayed job id and its delay. Params: $1 queue, $2 scheduler_id,
|
|
3
|
-
-- $3 next_millis, $4 template_data (jsonb), $5 template_opts (jsonb),
|
|
4
|
-
-- $6 opts (jsonb), $7 delayed_opts (jsonb), $8 now, $9 producer_id.
|
|
5
|
-
SELECT job_id, delay
|
|
6
|
-
FROM add_job_scheduler($1, $2, $3, $4, $5, $6, $7, $8, $9);
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
-- Append a log line at the next per-job ordinal; returns its index.
|
|
2
|
-
-- Params: $1 queue, $2 job_id, $3 row.
|
|
3
|
-
-- The unqualified table name resolves in the configured BullMQ schema via
|
|
4
|
-
-- search_path (default schema: bullmq).
|
|
5
|
-
INSERT INTO job_log (queue, job_id, idx, row)
|
|
6
|
-
VALUES (
|
|
7
|
-
$1, $2,
|
|
8
|
-
COALESCE(
|
|
9
|
-
(SELECT MAX(idx) + 1 FROM job_log WHERE queue = $1 AND job_id = $2),
|
|
10
|
-
0
|
|
11
|
-
),
|
|
12
|
-
$3
|
|
13
|
-
)
|
|
14
|
-
RETURNING idx;
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
-- Clear a job's logs, keeping the latest $3 entries (NULL/0 → remove all).
|
|
2
|
-
-- Params: $1 queue, $2 job_id, $3 keepLogs.
|
|
3
|
-
DELETE FROM job_log
|
|
4
|
-
WHERE queue = $1 AND job_id = $2
|
|
5
|
-
AND idx < COALESCE(
|
|
6
|
-
(SELECT MAX(idx) FROM job_log WHERE queue = $1 AND job_id = $2)
|
|
7
|
-
- $3 + 1,
|
|
8
|
-
idx + 1
|
|
9
|
-
);
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
-- Unconditionally remove a deduplication key (Queue.removeDeduplicationKey /
|
|
2
|
-
-- the deprecated removeDebounceKey). Params: $1 queue, $2 dedup_id. Returns the
|
|
3
|
-
-- removed key (0 or 1 row).
|
|
4
|
-
DELETE FROM dedup
|
|
5
|
-
WHERE queue = $1 AND dedup_id = $2
|
|
6
|
-
RETURNING dedup_id;
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
-- Refresh multiple active-job locks in one round-trip; returns the ids whose
|
|
2
|
-
-- locks could not be renewed. Params: $1 queue, $2 ids (text[]), $3 tokens
|
|
3
|
-
-- (text[]), $4 lock_ms, $5 now_ms.
|
|
4
|
-
WITH input AS (
|
|
5
|
-
SELECT ids.job_id, toks.token, ids.ord
|
|
6
|
-
FROM unnest($2::text[]) WITH ORDINALITY AS ids(job_id, ord)
|
|
7
|
-
JOIN unnest($3::text[]) WITH ORDINALITY AS toks(token, ord)
|
|
8
|
-
USING (ord)
|
|
9
|
-
),
|
|
10
|
-
updated AS (
|
|
11
|
-
UPDATE job
|
|
12
|
-
SET locked_until_ms = $5::bigint + $4::bigint
|
|
13
|
-
FROM input
|
|
14
|
-
WHERE queue = $1
|
|
15
|
-
AND id = input.job_id
|
|
16
|
-
AND state = 'active'
|
|
17
|
-
AND lock_token = input.token
|
|
18
|
-
RETURNING id
|
|
19
|
-
)
|
|
20
|
-
SELECT input.job_id AS id
|
|
21
|
-
FROM input
|
|
22
|
-
LEFT JOIN updated
|
|
23
|
-
ON updated.id = input.job_id
|
|
24
|
-
WHERE updated.id IS NULL
|
|
25
|
-
ORDER BY input.ord;
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
-- Connected BullMQ clients, mirroring Redis `CLIENT LIST` for worker/queue
|
|
2
|
-
-- discovery. Each long-lived worker / QueueEvents listener names its dedicated
|
|
3
|
-
-- connection via `application_name` (the PostgreSQL analogue of
|
|
4
|
-
-- `CLIENT SETNAME`), so listing the named sessions on this database reproduces
|
|
5
|
-
-- the same discovery surface. Pooled query connections leave `application_name`
|
|
6
|
-
-- empty and are excluded. One row per connection; the backend formats them into
|
|
7
|
-
-- `name=<application_name>` lines for the shared client-list parser.
|
|
8
|
-
SELECT application_name
|
|
9
|
-
FROM pg_stat_activity
|
|
10
|
-
WHERE datname = current_database()
|
|
11
|
-
AND application_name <> '';
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
-- Job counts by state for a queue. Param: $1 queue.
|
|
2
|
-
-- "prioritized" = waiting with priority > 0; "waiting" = waiting with priority 0.
|
|
3
|
-
SELECT
|
|
4
|
-
COUNT(*) FILTER (WHERE state = 'active') AS active,
|
|
5
|
-
COUNT(*) FILTER (WHERE state = 'completed') AS completed,
|
|
6
|
-
COUNT(*) FILTER (WHERE state = 'failed') AS failed,
|
|
7
|
-
COUNT(*) FILTER (WHERE state = 'delayed') AS delayed,
|
|
8
|
-
COUNT(*) FILTER (WHERE state = 'waiting' AND priority = 0) AS waiting,
|
|
9
|
-
COUNT(*) FILTER (WHERE state = 'waiting' AND priority > 0) AS prioritized,
|
|
10
|
-
COUNT(*) FILTER (WHERE state = 'waiting-children') AS "waiting-children",
|
|
11
|
-
(SELECT value FROM meta WHERE queue = $1 AND field = 'paused') AS paused
|
|
12
|
-
FROM job
|
|
13
|
-
WHERE queue = $1;
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
-- Waiting-job counts per priority. Params: $1 queue, $2 priorities (bigint[]).
|
|
2
|
-
-- Returns one row per requested priority, in the input array order, with the
|
|
3
|
-
-- number of waiting jobs at that priority. Pausing is an O(1) meta flag that
|
|
4
|
-
-- leaves jobs in the 'waiting' state, so the counts are identical whether or
|
|
5
|
-
-- not the queue is paused (mirrors Redis, where pausing does not touch the
|
|
6
|
-
-- wait/prioritized sets).
|
|
7
|
-
SELECT COUNT(j.id) AS cnt
|
|
8
|
-
FROM unnest($2::bigint[]) WITH ORDINALITY AS pr(priority, ord)
|
|
9
|
-
LEFT JOIN job j
|
|
10
|
-
ON j.queue = $1
|
|
11
|
-
AND j.state = 'waiting'
|
|
12
|
-
AND j.priority = pr.priority
|
|
13
|
-
GROUP BY pr.ord
|
|
14
|
-
ORDER BY pr.ord;
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
-- All of a parent job's child dependencies with their status and stored value.
|
|
2
|
-
-- Params: $1 parent_queue, $2 parent_id. Status maps to the public categories:
|
|
3
|
-
-- processed → processed (value = return value), pending → unprocessed,
|
|
4
|
-
-- ignored → ignored (value = reason), failed → failed.
|
|
5
|
-
SELECT status::text AS status, child_key, value
|
|
6
|
-
FROM job_dependency
|
|
7
|
-
WHERE parent_queue = $1 AND parent_id = $2
|
|
8
|
-
ORDER BY child_id;
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
-- One page of a parent's child dependencies in a single status category.
|
|
2
|
-
-- Params: $1 parent_queue, $2 parent_id, $3 status, $4 offset, $5 count.
|
|
3
|
-
SELECT child_key, value
|
|
4
|
-
FROM job_dependency
|
|
5
|
-
WHERE parent_queue = $1 AND parent_id = $2 AND status = $3::dep_status
|
|
6
|
-
ORDER BY child_id
|
|
7
|
-
OFFSET $4 LIMIT $5;
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
-- Child-dependency counts per status for a parent job.
|
|
2
|
-
-- Params: $1 parent_queue, $2 parent_id. ("unprocessed" = pending children.)
|
|
3
|
-
SELECT
|
|
4
|
-
COUNT(*) FILTER (WHERE status = 'processed') AS processed,
|
|
5
|
-
COUNT(*) FILTER (WHERE status = 'pending') AS unprocessed,
|
|
6
|
-
COUNT(*) FILTER (WHERE status = 'ignored') AS ignored,
|
|
7
|
-
COUNT(*) FILTER (WHERE status = 'failed') AS failed
|
|
8
|
-
FROM job_dependency
|
|
9
|
-
WHERE parent_queue = $1 AND parent_id = $2;
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
-- Ignored children of a parent as (child_key → failure reason) pairs. The reason
|
|
2
|
-
-- is stored as a JSON scalar string, so `#>> '{}'` extracts the raw text
|
|
3
|
-
-- (mirrors the Redis `<jobId>:failed` hash). Params: $1 parent_queue, $2 parent_id.
|
|
4
|
-
SELECT child_key, COALESCE(value #>> '{}', value::text) AS reason
|
|
5
|
-
FROM job_dependency
|
|
6
|
-
WHERE parent_queue = $1 AND parent_id = $2 AND status = 'ignored'
|
|
7
|
-
ORDER BY child_id;
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
-- Fetch a single scheduler's stored metadata and next-run score.
|
|
2
|
-
-- Params: $1 queue, $2 scheduler_id.
|
|
3
|
-
SELECT name, iteration_count, limit_count, start_date_ms, end_date_ms, tz,
|
|
4
|
-
pattern, every_ms, offset_ms, template_data, template_opts, next_run_ms
|
|
5
|
-
FROM scheduler
|
|
6
|
-
WHERE queue = $1 AND scheduler_id = $2;
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
-- Fetch a page of schedulers ordered by next-run time. Params: $1 queue,
|
|
2
|
-
-- $2 asc (boolean), $3 offset, $4 count (NULL = all remaining).
|
|
3
|
-
SELECT scheduler_id, next_run_ms
|
|
4
|
-
FROM scheduler
|
|
5
|
-
WHERE queue = $1
|
|
6
|
-
ORDER BY CASE WHEN $2 THEN next_run_ms END ASC,
|
|
7
|
-
CASE WHEN NOT $2 THEN next_run_ms END DESC,
|
|
8
|
-
scheduler_id
|
|
9
|
-
OFFSET $3
|
|
10
|
-
LIMIT $4;
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
-- Metrics for a queue/kind: the cumulative meta `count` and the per-minute data
|
|
2
|
-
-- points (newest first), sliced to [start, end] with Redis LRANGE semantics
|
|
3
|
-
-- (index 0 is the newest point; a negative end means "to the oldest").
|
|
4
|
-
-- Params: $1 queue, $2 kind ('completed' | 'failed'), $3 start, $4 end.
|
|
5
|
-
SELECT
|
|
6
|
-
COALESCE((SELECT count FROM metrics
|
|
7
|
-
WHERE queue = $1 AND kind = $2), 0)::bigint AS total,
|
|
8
|
-
COALESCE((
|
|
9
|
-
SELECT CASE
|
|
10
|
-
WHEN $4 < 0 THEN data[($3 + 1) : ]
|
|
11
|
-
ELSE data[($3 + 1) : ($4 + 1)]
|
|
12
|
-
END
|
|
13
|
-
FROM metrics WHERE queue = $1 AND kind = $2
|
|
14
|
-
), ARRAY[]::bigint[]) AS data;
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
-- Processed children of a parent as (child_key → serialized value) pairs. The
|
|
2
|
-
-- value is returned as JSON text so the caller can JSON.parse it (mirrors the
|
|
3
|
-
-- Redis `<jobId>:processed` hash of stringified values). Params: $1 parent_queue,
|
|
4
|
-
-- $2 parent_id.
|
|
5
|
-
SELECT child_key, value::text AS value
|
|
6
|
-
FROM job_dependency
|
|
7
|
-
WHERE parent_queue = $1 AND parent_id = $2 AND status = 'processed'
|
|
8
|
-
ORDER BY child_id;
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
-- Whether the queue has a claimable (waiting, non-paused) job right now. Used
|
|
2
|
-
-- by waitForJob to close the race where a NOTIFY fires before the listener is
|
|
3
|
-
-- established (mirrors the atomicity of Redis's blocking pop). Param: $1 queue.
|
|
4
|
-
SELECT
|
|
5
|
-
EXISTS(
|
|
6
|
-
SELECT 1 FROM job WHERE queue = $1 AND state = 'waiting'
|
|
7
|
-
)
|
|
8
|
-
AND NOT EXISTS(
|
|
9
|
-
SELECT 1 FROM meta
|
|
10
|
-
WHERE queue = $1 AND field = 'paused' AND value = '1'
|
|
11
|
-
) AS present;
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
-- Whether a (non-prioritized) waiting job is in the wait or paused list. The
|
|
2
|
-
-- queue has no separate paused list: a waiting job belongs to the paused list
|
|
3
|
-
-- when the queue is paused, and to the wait list otherwise. $3 selects which
|
|
4
|
-
-- list to test (true = paused, false = wait). Params: $1 queue, $2 id, $3 paused.
|
|
5
|
-
SELECT EXISTS(
|
|
6
|
-
SELECT 1 FROM job j
|
|
7
|
-
WHERE j.queue = $1 AND j.id = $2
|
|
8
|
-
AND j.state = 'waiting' AND j.priority = 0
|
|
9
|
-
AND (
|
|
10
|
-
EXISTS(
|
|
11
|
-
SELECT 1 FROM meta m
|
|
12
|
-
WHERE m.queue = $1 AND m.field = 'paused' AND m.value = '1'
|
|
13
|
-
)
|
|
14
|
-
) = $3
|
|
15
|
-
) AS present;
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
-- Whether the queue is "maxed": it has a global concurrency limit and the
|
|
2
|
-
-- number of active jobs has reached it (mirrors isQueueMaxed). Returns false
|
|
3
|
-
-- when no concurrency limit is configured.
|
|
4
|
-
-- Params: $1 queue.
|
|
5
|
-
SELECT COALESCE(
|
|
6
|
-
(SELECT count(*) FROM job WHERE queue = $1 AND state = 'active')
|
|
7
|
-
>= (SELECT value::integer FROM meta
|
|
8
|
-
WHERE queue = $1 AND field = 'concurrency'),
|
|
9
|
-
false
|
|
10
|
-
) AS maxed;
|
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
-- Move an active job back to wait (Job.moveToWait / dynamic rate limit).
|
|
2
|
-
-- Params: $1 queue, $2 id, $3 token ('0' bypasses the lock check), $4 now_ms.
|
|
3
|
-
-- Returns the limiter window ms (>=0), or -1 when the job no longer exists.
|
|
4
|
-
SELECT move_active_to_wait($1, $2, $3, $4) AS n;
|
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
-- Claim the next ready job for a worker (0 or 1 rows), honouring the limiter.
|
|
2
|
-
-- Params: $1 queue, $2 token, $3 lock_ms, $4 now_ms, $5 worker name,
|
|
3
|
-
-- $6 limiter max (worker option, NULL if none), $7 limiter duration ms.
|
|
4
|
-
SELECT * FROM move_to_active($1, $2, $3, $4, $5, $6, $7);
|
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
-- Finish an active job successfully; returns the finished-at timestamp.
|
|
2
|
-
-- Params: $1 queue, $2 id, $3 token, $4 return_value (jsonb), $5 finished_on,
|
|
3
|
-
-- $6 remove_all, $7 keep_age (s), $8 keep_count.
|
|
4
|
-
SELECT move_to_completed($1, $2, $3, $4::jsonb, $5, $6, $7, $8) AS finished_on;
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
-- Finish an active job successfully AND claim the next ready job in one
|
|
2
|
-
-- transaction (one commit) — the fused equivalent of move_to_completed followed
|
|
3
|
-
-- by move_to_active, mirroring Redis's moveToFinished. Returns 0 or 1 job rows.
|
|
4
|
-
-- Params: $1 queue, $2 id, $3 token, $4 return_value (jsonb), $5 finished_on,
|
|
5
|
-
-- $6 remove_all, $7 keep_age (s), $8 keep_count,
|
|
6
|
-
-- $9 lock_ms, $10 now_ms, $11 worker name,
|
|
7
|
-
-- $12 limiter max (worker option, NULL if none), $13 limiter duration ms.
|
|
8
|
-
SELECT * FROM move_to_completed_fetch(
|
|
9
|
-
$1, $2, $3, $4::jsonb, $5, $6, $7, $8, $9, $10, $11, $12, $13);
|
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
-- Re-queue an active job to the delayed state (retry-with-delay / manual delay).
|
|
2
|
-
-- Params: $1 queue, $2 id, $3 token, $4 process_at, $5 delay,
|
|
3
|
-
-- $6 skip_attempt, $7 failed_reason, $8 stacktrace (jsonb).
|
|
4
|
-
SELECT move_to_delayed($1, $2, $3, $4, $5, $6, $7, $8::jsonb) AS n;
|
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
-- Mark an active job failed; returns the finished-at timestamp.
|
|
2
|
-
-- Params: $1 queue, $2 id, $3 token, $4 failed_reason, $5 stacktrace (jsonb),
|
|
3
|
-
-- $6 finished_on, $7 remove_all, $8 keep_age (s), $9 keep_count.
|
|
4
|
-
SELECT move_to_failed($1, $2, $3, $4, $5::jsonb, $6, $7, $8, $9) AS finished_on;
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
-- Fail (or retry) an active job AND claim the next ready job in one transaction
|
|
2
|
-
-- (one commit) — the fused equivalent of move_to_failed followed by
|
|
3
|
-
-- move_to_active, mirroring Redis's moveToFinished. Returns 0 or 1 job rows.
|
|
4
|
-
-- Params: $1 queue, $2 id, $3 token, $4 failed_reason, $5 stacktrace (jsonb),
|
|
5
|
-
-- $6 finished_on, $7 remove_all, $8 keep_age (s), $9 keep_count,
|
|
6
|
-
-- $10 lock_ms, $11 now_ms, $12 worker name,
|
|
7
|
-
-- $13 limiter max (worker option, NULL if none), $14 limiter duration ms.
|
|
8
|
-
SELECT * FROM move_to_failed_fetch(
|
|
9
|
-
$1, $2, $3, $4, $5::jsonb, $6, $7, $8, $9, $10, $11, $12, $13, $14);
|