pgbus 0.11.4 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2166 @@
1
+ ------------------------------------------------------------
2
+ -- Schema, tables, records, privileges, indexes, etc
3
+ ------------------------------------------------------------
4
+ -- When installed as an extension, we don't need to create the `pgmq` schema
5
+ -- because it is automatically created by postgres due to being declared in
6
+ -- the extension control file
7
+ DO
8
+ $$
9
+ BEGIN
10
+ IF (SELECT NOT EXISTS( SELECT 1 FROM pg_extension WHERE extname = 'pgmq')) THEN
11
+ CREATE SCHEMA IF NOT EXISTS pgmq;
12
+ END IF;
13
+ END
14
+ $$;
15
+
16
+ -- Table where queues and metadata about them is stored
17
+ CREATE TABLE IF NOT EXISTS pgmq.meta (
18
+ queue_name VARCHAR UNIQUE NOT NULL,
19
+ is_partitioned BOOLEAN NOT NULL,
20
+ is_unlogged BOOLEAN NOT NULL,
21
+ created_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL
22
+ );
23
+
24
+ -- Grant permission to pg_monitor to all tables and sequences
25
+ -- These grants are intentionally placed here (after creating `pgmq.meta` but before creating other tables). This
26
+ -- allows the `pg_dump` output for a fresh installation to match the output for an installation that followed the
27
+ -- upgrade path.
28
+ GRANT USAGE ON SCHEMA pgmq TO pg_monitor;
29
+ GRANT SELECT ON ALL TABLES IN SCHEMA pgmq TO pg_monitor;
30
+ GRANT SELECT ON ALL SEQUENCES IN SCHEMA pgmq TO pg_monitor;
31
+ ALTER DEFAULT PRIVILEGES IN SCHEMA pgmq GRANT SELECT ON TABLES TO pg_monitor;
32
+ ALTER DEFAULT PRIVILEGES IN SCHEMA pgmq GRANT SELECT ON SEQUENCES TO pg_monitor;
33
+
34
+ -- Table to track notification throttling for queues
35
+ CREATE UNLOGGED TABLE IF NOT EXISTS pgmq.notify_insert_throttle (
36
+ queue_name VARCHAR UNIQUE NOT NULL -- Queue name (without 'q_' prefix)
37
+ CONSTRAINT notify_insert_throttle_meta_queue_name_fk
38
+ REFERENCES pgmq.meta (queue_name)
39
+ ON DELETE CASCADE,
40
+ throttle_interval_ms INTEGER NOT NULL DEFAULT 0, -- Min milliseconds between notifications (0 = no throttling)
41
+ last_notified_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT to_timestamp(0) -- Timestamp of last sent notification
42
+ );
43
+
44
+ CREATE INDEX IF NOT EXISTS idx_notify_throttle_active
45
+ ON pgmq.notify_insert_throttle (queue_name, last_notified_at)
46
+ WHERE throttle_interval_ms > 0;
47
+
48
+ CREATE TABLE IF NOT EXISTS pgmq.topic_bindings
49
+ (
50
+ pattern text NOT NULL, -- Wildcard pattern for routing key matching (* = one segment, # = zero or more segments)
51
+ queue_name text NOT NULL -- Name of the queue that receives messages when pattern matches
52
+ CONSTRAINT topic_bindings_meta_queue_name_fk
53
+ REFERENCES pgmq.meta (queue_name)
54
+ ON DELETE CASCADE,
55
+ bound_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL, -- Timestamp when the binding was created
56
+ compiled_regex text GENERATED ALWAYS AS (
57
+ -- Pre-compile the pattern to regex for faster matching
58
+ -- This avoids runtime compilation on every send_topic call
59
+ '^' ||
60
+ replace(
61
+ replace(
62
+ regexp_replace(pattern, '([.+?{}()|\[\]\\^$])', '\\\1', 'g'),
63
+ '*', '[^.]+'
64
+ ),
65
+ '#', '.*'
66
+ ) || '$'
67
+ ) STORED, -- Computed column: stores the compiled regex pattern
68
+ CONSTRAINT topic_bindings_unique_pattern_queue UNIQUE (pattern, queue_name)
69
+ );
70
+
71
+ -- Create covering index for better performance when scanning patterns
72
+ -- Includes queue_name and compiled_regex to allow index-only scans (no table access needed)
73
+ CREATE INDEX IF NOT EXISTS idx_topic_bindings_covering ON pgmq.topic_bindings (pattern) INCLUDE (queue_name, compiled_regex);
74
+
75
+ -- Allow the following `pgmq` tables to be dumped by `pg_dump` when pgmq is installed as an extension
76
+ DO
77
+ $$
78
+ BEGIN
79
+ IF EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pgmq') THEN
80
+ PERFORM pg_catalog.pg_extension_config_dump('pgmq.meta', '');
81
+ PERFORM pg_catalog.pg_extension_config_dump('pgmq.notify_insert_throttle', '');
82
+ PERFORM pg_catalog.pg_extension_config_dump('pgmq.topic_bindings', '');
83
+ END IF;
84
+ END
85
+ $$;
86
+
87
+ -- This type has the shape of a message in a queue, and is often returned by
88
+ -- pgmq functions that return messages.
89
+ -- Note: Changing the order of fields in this type is a breaking change -- our Rust Diesel client implementation
90
+ -- expects a specific order of fields.
91
+ CREATE TYPE pgmq.message_record AS (
92
+ msg_id BIGINT,
93
+ read_ct INTEGER,
94
+ enqueued_at TIMESTAMP WITH TIME ZONE,
95
+ last_read_at TIMESTAMP WITH TIME ZONE,
96
+ vt TIMESTAMP WITH TIME ZONE,
97
+ message JSONB,
98
+ headers JSONB
99
+ );
100
+
101
+ -- Note: Changing the order of fields in this type is a breaking change -- our Rust Diesel client implementation
102
+ -- expects a specific order of fields.
103
+ CREATE TYPE pgmq.queue_record AS (
104
+ queue_name VARCHAR,
105
+ is_partitioned BOOLEAN,
106
+ is_unlogged BOOLEAN,
107
+ created_at TIMESTAMP WITH TIME ZONE
108
+ );
109
+
110
+ ------------------------------------------------------------
111
+ -- Functions
112
+ ------------------------------------------------------------
113
+
114
+ -- prevents race conditions during queue creation by acquiring a transaction-level advisory lock
115
+ -- uses a transaction advisory lock maintain the lock until transaction commit
116
+ -- a race condition would still exist if lock was released before commit
117
+ CREATE FUNCTION pgmq.acquire_queue_lock(queue_name TEXT)
118
+ RETURNS void AS $$
119
+ BEGIN
120
+ PERFORM pg_advisory_xact_lock(hashtext('pgmq.queue_' || queue_name));
121
+ END;
122
+ $$ LANGUAGE plpgsql;
123
+
124
+ -- read_grouped_round_robin
125
+ -- reads messages while preserving FIFO within groups and interleaving across groups (layered round-robin)
126
+ CREATE FUNCTION pgmq.read_grouped_rr(
127
+ queue_name TEXT,
128
+ vt INTEGER,
129
+ qty INTEGER
130
+ )
131
+ RETURNS SETOF pgmq.message_record AS $$
132
+ DECLARE
133
+ sql TEXT;
134
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
135
+ BEGIN
136
+ sql := FORMAT(
137
+ $QUERY$
138
+ WITH fifo_groups AS (
139
+ -- Determine the absolute head (oldest) message id per FIFO group, regardless of visibility
140
+ SELECT
141
+ COALESCE(headers->>'x-pgmq-group', '_default_fifo_group') AS fifo_key,
142
+ MIN(msg_id) AS head_msg_id
143
+ FROM pgmq.%1$I
144
+ GROUP BY COALESCE(headers->>'x-pgmq-group', '_default_fifo_group')
145
+ ),
146
+ eligible_groups AS (
147
+ -- Only groups whose head message is currently visible
148
+ -- Acquire a transaction-level advisory lock per group to prevent concurrent selection
149
+ SELECT
150
+ g.fifo_key,
151
+ g.head_msg_id,
152
+ ROW_NUMBER() OVER (ORDER BY g.head_msg_id) AS group_priority
153
+ FROM fifo_groups g
154
+ JOIN pgmq.%2$I h ON h.msg_id = g.head_msg_id
155
+ WHERE h.vt <= clock_timestamp()
156
+ AND pg_try_advisory_xact_lock(pg_catalog.hashtextextended(g.fifo_key, 0))
157
+ ),
158
+ available_messages AS (
159
+ -- All currently visible messages starting at the head for each eligible group
160
+ SELECT
161
+ m.msg_id,
162
+ eg.group_priority,
163
+ ROW_NUMBER() OVER (
164
+ PARTITION BY eg.fifo_key
165
+ ORDER BY m.msg_id
166
+ ) AS msg_rank_in_group
167
+ FROM pgmq.%3$I m
168
+ JOIN eligible_groups eg
169
+ ON COALESCE(m.headers->>'x-pgmq-group', '_default_fifo_group') = eg.fifo_key
170
+ WHERE m.vt <= clock_timestamp()
171
+ AND m.msg_id >= eg.head_msg_id
172
+ ),
173
+ ordered_messages AS (
174
+ -- Layered round-robin: take rank 1 of all groups by group_priority, then rank 2, etc.
175
+ -- Assign selection order before locking
176
+ SELECT msg_id, ROW_NUMBER() OVER (ORDER BY msg_rank_in_group, group_priority) as selection_order
177
+ FROM available_messages
178
+ ),
179
+ selected_messages AS (
180
+ -- Lock the messages in the correct order, preserving selection_order
181
+ SELECT om.msg_id, om.selection_order
182
+ FROM ordered_messages om
183
+ JOIN pgmq.%4$I m ON m.msg_id = om.msg_id
184
+ WHERE om.selection_order <= $1
185
+ ORDER BY om.selection_order
186
+ FOR UPDATE OF m SKIP LOCKED
187
+ ),
188
+ updated_messages AS (
189
+ UPDATE pgmq.%5$I m
190
+ SET
191
+ vt = clock_timestamp() + %6$L,
192
+ read_ct = read_ct + 1,
193
+ last_read_at = clock_timestamp()
194
+ FROM selected_messages sm
195
+ WHERE m.msg_id = sm.msg_id
196
+ AND m.vt <= clock_timestamp() -- final guard to avoid duplicate reads under races
197
+ RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.last_read_at, m.vt, m.message, m.headers, sm.selection_order
198
+ )
199
+ SELECT msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers
200
+ FROM updated_messages
201
+ ORDER BY selection_order;
202
+ $QUERY$,
203
+ qtable, qtable, qtable, qtable, qtable, make_interval(secs => vt)
204
+ );
205
+ RETURN QUERY EXECUTE sql USING qty;
206
+ END;
207
+ $$ LANGUAGE plpgsql;
208
+
209
+ -- read_grouped_rr_with_poll
210
+ -- reads messages using round-robin layering across groups, with polling support
211
+ CREATE FUNCTION pgmq.read_grouped_rr_with_poll(
212
+ queue_name TEXT,
213
+ vt INTEGER,
214
+ qty INTEGER,
215
+ max_poll_seconds INTEGER DEFAULT 5,
216
+ poll_interval_ms INTEGER DEFAULT 100
217
+ )
218
+ RETURNS SETOF pgmq.message_record AS $$
219
+ DECLARE
220
+ r pgmq.message_record;
221
+ stop_at TIMESTAMP;
222
+ BEGIN
223
+ stop_at := clock_timestamp() + make_interval(secs => max_poll_seconds);
224
+ LOOP
225
+ IF (SELECT clock_timestamp() >= stop_at) THEN
226
+ RETURN;
227
+ END IF;
228
+
229
+ FOR r IN
230
+ SELECT * FROM pgmq.read_grouped_rr(queue_name, vt, qty)
231
+ LOOP
232
+ RETURN NEXT r;
233
+ END LOOP;
234
+ IF FOUND THEN
235
+ RETURN;
236
+ ELSE
237
+ PERFORM pg_sleep(poll_interval_ms::numeric / 1000);
238
+ END IF;
239
+ END LOOP;
240
+ END;
241
+ $$ LANGUAGE plpgsql;
242
+
243
+ -- read_grouped_head: read the head of N different FIFO groups in a single operation.
244
+ -- This supports horizontal scaling by processing groups in parallel while ensuring message ordering is preserved per group.
245
+ CREATE FUNCTION pgmq.read_grouped_head(
246
+ queue_name TEXT,
247
+ vt INTEGER,
248
+ qty INTEGER
249
+ )
250
+ RETURNS SETOF pgmq.message_record AS $$
251
+ DECLARE
252
+ sql TEXT;
253
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
254
+ BEGIN
255
+ sql := FORMAT(
256
+ $QUERY$
257
+ WITH fifo_groups AS (
258
+ -- Determine the absolute head (oldest) message id per FIFO group, regardless of visibility
259
+ SELECT
260
+ COALESCE(headers->>'x-pgmq-group', '_default_fifo_group') AS fifo_key,
261
+ MIN(msg_id) AS head_msg_id
262
+ FROM pgmq.%1$I
263
+ GROUP BY COALESCE(headers->>'x-pgmq-group', '_default_fifo_group')
264
+ ),
265
+ selected_messages AS (
266
+ -- Take at most 1 message per group
267
+ SELECT g.head_msg_id msg_id
268
+ FROM fifo_groups g
269
+ JOIN pgmq.%1$I q ON q.msg_id = g.head_msg_id
270
+ WHERE q.vt <= clock_timestamp()
271
+ ORDER BY q.msg_id
272
+ LIMIT $1
273
+ FOR UPDATE SKIP LOCKED
274
+ )
275
+ UPDATE pgmq.%1$I m
276
+ SET
277
+ vt = clock_timestamp() + %2$L,
278
+ read_ct = read_ct + 1,
279
+ last_read_at = clock_timestamp()
280
+ FROM selected_messages sm
281
+ WHERE m.msg_id = sm.msg_id
282
+ RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.last_read_at, m.vt, m.message, m.headers;
283
+ $QUERY$,
284
+ qtable, make_interval(secs => vt)
285
+ );
286
+ RETURN QUERY EXECUTE sql USING qty;
287
+ END;
288
+ $$ LANGUAGE plpgsql;
289
+
290
+ -- read_grouped_head_with_poll
291
+ -- reads the head of N different FIFO groups in a single operation, with polling support
292
+ CREATE FUNCTION pgmq.read_grouped_head_with_poll(
293
+ queue_name TEXT,
294
+ vt INTEGER,
295
+ qty INTEGER,
296
+ max_poll_seconds INTEGER DEFAULT 5,
297
+ poll_interval_ms INTEGER DEFAULT 100
298
+ )
299
+ RETURNS SETOF pgmq.message_record AS $$
300
+ DECLARE
301
+ r pgmq.message_record;
302
+ stop_at TIMESTAMPTZ;
303
+ BEGIN
304
+ stop_at := clock_timestamp() + make_interval(secs => max_poll_seconds);
305
+ LOOP
306
+ IF clock_timestamp() >= stop_at THEN
307
+ RETURN;
308
+ END IF;
309
+
310
+ FOR r IN
311
+ SELECT * FROM pgmq.read_grouped_head(queue_name, vt, qty)
312
+ LOOP
313
+ RETURN NEXT r;
314
+ END LOOP;
315
+ IF FOUND THEN
316
+ RETURN;
317
+ ELSE
318
+ PERFORM pg_sleep(poll_interval_ms::numeric / 1000);
319
+ END IF;
320
+ END LOOP;
321
+ END;
322
+ $$ LANGUAGE plpgsql;
323
+
324
+ -- a helper to format table names and check for invalid characters
325
+ CREATE FUNCTION pgmq.format_table_name(queue_name text, prefix text)
326
+ RETURNS TEXT AS $$
327
+ BEGIN
328
+ IF queue_name ~ '\$|;|--|'''
329
+ THEN
330
+ RAISE EXCEPTION 'queue name contains invalid characters: $, ;, --, or \''';
331
+ END IF;
332
+ RETURN lower(prefix || '_' || queue_name);
333
+ END;
334
+ $$ LANGUAGE plpgsql;
335
+
336
+ -- read
337
+ -- reads a number of messages from a queue, setting a visibility timeout on them
338
+ CREATE FUNCTION pgmq.read(
339
+ queue_name TEXT,
340
+ vt INTEGER,
341
+ qty INTEGER,
342
+ conditional JSONB DEFAULT '{}'
343
+ )
344
+ RETURNS SETOF pgmq.message_record AS $$
345
+ DECLARE
346
+ sql TEXT;
347
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
348
+ BEGIN
349
+ sql := FORMAT(
350
+ $QUERY$
351
+ WITH cte AS
352
+ (
353
+ SELECT msg_id
354
+ FROM pgmq.%I
355
+ WHERE vt <= clock_timestamp() AND CASE
356
+ WHEN %L != '{}'::jsonb THEN (message @> %2$L)::integer
357
+ ELSE 1
358
+ END = 1
359
+ ORDER BY msg_id ASC
360
+ LIMIT $1
361
+ FOR UPDATE SKIP LOCKED
362
+ )
363
+ UPDATE pgmq.%I m
364
+ SET
365
+ last_read_at = clock_timestamp(),
366
+ vt = clock_timestamp() + %L,
367
+ read_ct = read_ct + 1
368
+ FROM cte
369
+ WHERE m.msg_id = cte.msg_id
370
+ RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.last_read_at, m.vt, m.message, m.headers;
371
+ $QUERY$,
372
+ qtable, conditional, qtable, make_interval(secs => vt)
373
+ );
374
+ RETURN QUERY EXECUTE sql USING qty;
375
+ END;
376
+ $$ LANGUAGE plpgsql;
377
+
378
+ -- read_grouped
379
+ -- reads messages with AWS SQS FIFO-style batch retrieval behavior
380
+ -- attempts to return as many messages as possible from the same message group
381
+ CREATE FUNCTION pgmq.read_grouped(
382
+ queue_name TEXT,
383
+ vt INTEGER,
384
+ qty INTEGER
385
+ )
386
+ RETURNS SETOF pgmq.message_record AS $$
387
+ DECLARE
388
+ sql TEXT;
389
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
390
+ BEGIN
391
+ sql := FORMAT(
392
+ $QUERY$
393
+ WITH fifo_groups AS (
394
+ -- Find the minimum msg_id for each FIFO group that's ready to be processed
395
+ SELECT
396
+ COALESCE(headers->>'x-pgmq-group', '_default_fifo_group') as fifo_key,
397
+ MIN(msg_id) as min_msg_id
398
+ FROM pgmq.%I
399
+ WHERE vt <= clock_timestamp()
400
+ GROUP BY COALESCE(headers->>'x-pgmq-group', '_default_fifo_group')
401
+ ),
402
+ locked_groups AS (
403
+ -- Lock the first available message in each FIFO group
404
+ SELECT
405
+ m.msg_id,
406
+ fg.fifo_key
407
+ FROM pgmq.%I m
408
+ INNER JOIN fifo_groups fg ON
409
+ COALESCE(m.headers->>'x-pgmq-group', '_default_fifo_group') = fg.fifo_key
410
+ AND m.msg_id = fg.min_msg_id
411
+ WHERE m.vt <= clock_timestamp()
412
+ ORDER BY m.msg_id ASC
413
+ FOR UPDATE SKIP LOCKED
414
+ ),
415
+ group_priorities AS (
416
+ -- Assign priority to groups based on their oldest message
417
+ SELECT
418
+ fifo_key,
419
+ msg_id as min_msg_id,
420
+ ROW_NUMBER() OVER (ORDER BY msg_id) as group_priority
421
+ FROM locked_groups
422
+ ),
423
+ filtered_groups as (
424
+ SELECT * FROM group_priorities gp
425
+ WHERE NOT EXISTS (
426
+ -- Ensure no earlier message in this group is currently being processed
427
+ SELECT 1
428
+ FROM pgmq.%I m2
429
+ WHERE COALESCE(m2.headers->>'x-pgmq-group', '_default_fifo_group') = gp.fifo_key
430
+ AND m2.vt > clock_timestamp()
431
+ AND m2.msg_id < gp.min_msg_id
432
+ )
433
+ ),
434
+ available_messages as (
435
+ SELECT gp.fifo_key, t.msg_id,gp.group_priority,
436
+ ROW_NUMBER() OVER (PARTITION BY gp.fifo_key ORDER BY t.msg_id) as msg_rank_in_group
437
+ FROM filtered_groups gp
438
+ CROSS JOIN LATERAL (
439
+ SELECT *
440
+ FROM pgmq.%I t
441
+ WHERE COALESCE(t.headers->>'x-pgmq-group', '_default_fifo_group') = gp.fifo_key
442
+ AND t.vt <= clock_timestamp()
443
+ ORDER BY msg_id
444
+ LIMIT $1 -- tip to limit query impact, we know we need at most qty in each group
445
+ ) t
446
+ ORDER BY gp.group_priority
447
+ ),
448
+ batch_selection AS (
449
+ -- Select messages to fill batch, prioritizing earliest group
450
+ SELECT
451
+ msg_id,
452
+ ROW_NUMBER() OVER (ORDER BY group_priority, msg_rank_in_group) as overall_rank
453
+ FROM available_messages
454
+ ),
455
+ selected_messages AS (
456
+ -- Limit to requested quantity
457
+ SELECT msg_id
458
+ FROM batch_selection
459
+ WHERE overall_rank <= $1
460
+ ORDER BY msg_id
461
+ FOR UPDATE SKIP LOCKED
462
+ )
463
+ UPDATE pgmq.%I m
464
+ SET
465
+ vt = clock_timestamp() + %L,
466
+ read_ct = read_ct + 1,
467
+ last_read_at = clock_timestamp()
468
+ FROM selected_messages sm
469
+ WHERE m.msg_id = sm.msg_id
470
+ RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.last_read_at, m.vt, m.message, m.headers;
471
+ $QUERY$,
472
+ qtable, qtable, qtable, qtable, qtable, make_interval(secs => vt)
473
+ );
474
+ RETURN QUERY EXECUTE sql USING qty;
475
+ END;
476
+ $$ LANGUAGE plpgsql;
477
+
478
+ -- read_grouped_with_poll
479
+ -- reads messages with AWS SQS FIFO-style batch retrieval behavior, with polling support
480
+ CREATE FUNCTION pgmq.read_grouped_with_poll(
481
+ queue_name TEXT,
482
+ vt INTEGER,
483
+ qty INTEGER,
484
+ max_poll_seconds INTEGER DEFAULT 5,
485
+ poll_interval_ms INTEGER DEFAULT 100
486
+ )
487
+ RETURNS SETOF pgmq.message_record AS $$
488
+ DECLARE
489
+ r pgmq.message_record;
490
+ stop_at TIMESTAMP;
491
+ BEGIN
492
+ stop_at := clock_timestamp() + make_interval(secs => max_poll_seconds);
493
+ LOOP
494
+ IF (SELECT clock_timestamp() >= stop_at) THEN
495
+ RETURN;
496
+ END IF;
497
+
498
+ FOR r IN
499
+ SELECT * FROM pgmq.read_grouped(queue_name, vt, qty)
500
+ LOOP
501
+ RETURN NEXT r;
502
+ END LOOP;
503
+ IF FOUND THEN
504
+ RETURN;
505
+ ELSE
506
+ PERFORM pg_sleep(poll_interval_ms::numeric / 1000);
507
+ END IF;
508
+ END LOOP;
509
+ END;
510
+ $$ LANGUAGE plpgsql;
511
+
512
+ ---- read_with_poll
513
+ ---- reads a number of messages from a queue, setting a visibility timeout on them
514
+ CREATE FUNCTION pgmq.read_with_poll(
515
+ queue_name TEXT,
516
+ vt INTEGER,
517
+ qty INTEGER,
518
+ max_poll_seconds INTEGER DEFAULT 5,
519
+ poll_interval_ms INTEGER DEFAULT 100,
520
+ conditional JSONB DEFAULT '{}'
521
+ )
522
+ RETURNS SETOF pgmq.message_record AS $$
523
+ DECLARE
524
+ r pgmq.message_record;
525
+ stop_at TIMESTAMP;
526
+ sql TEXT;
527
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
528
+ BEGIN
529
+ stop_at := clock_timestamp() + make_interval(secs => max_poll_seconds);
530
+ LOOP
531
+ IF (SELECT clock_timestamp() >= stop_at) THEN
532
+ RETURN;
533
+ END IF;
534
+
535
+ sql := FORMAT(
536
+ $QUERY$
537
+ WITH cte AS
538
+ (
539
+ SELECT msg_id
540
+ FROM pgmq.%I
541
+ WHERE vt <= clock_timestamp() AND CASE
542
+ WHEN %L != '{}'::jsonb THEN (message @> %2$L)::integer
543
+ ELSE 1
544
+ END = 1
545
+ ORDER BY msg_id ASC
546
+ LIMIT $1
547
+ FOR UPDATE SKIP LOCKED
548
+ )
549
+ UPDATE pgmq.%I m
550
+ SET
551
+ last_read_at = clock_timestamp(),
552
+ vt = clock_timestamp() + %L,
553
+ read_ct = read_ct + 1
554
+ FROM cte
555
+ WHERE m.msg_id = cte.msg_id
556
+ RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.last_read_at, m.vt, m.message, m.headers;
557
+ $QUERY$,
558
+ qtable, conditional, qtable, make_interval(secs => vt)
559
+ );
560
+
561
+ FOR r IN
562
+ EXECUTE sql USING qty
563
+ LOOP
564
+ RETURN NEXT r;
565
+ END LOOP;
566
+ IF FOUND THEN
567
+ RETURN;
568
+ ELSE
569
+ PERFORM pg_sleep(poll_interval_ms::numeric / 1000);
570
+ END IF;
571
+ END LOOP;
572
+ END;
573
+ $$ LANGUAGE plpgsql;
574
+
575
+ ---- archive
576
+ ---- removes a message from the queue, and sends it to the archive, where its
577
+ ---- saved permanently.
578
+ CREATE FUNCTION pgmq.archive(
579
+ queue_name TEXT,
580
+ msg_id BIGINT
581
+ )
582
+ RETURNS BOOLEAN AS $$
583
+ DECLARE
584
+ sql TEXT;
585
+ result BIGINT;
586
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
587
+ atable TEXT := pgmq.format_table_name(queue_name, 'a');
588
+ BEGIN
589
+ sql := FORMAT(
590
+ $QUERY$
591
+ WITH archived AS (
592
+ DELETE FROM pgmq.%I
593
+ WHERE msg_id = $1
594
+ RETURNING msg_id, vt, read_ct, enqueued_at, last_read_at, message, headers
595
+ )
596
+ INSERT INTO pgmq.%I (msg_id, vt, read_ct, enqueued_at, last_read_at, message, headers)
597
+ SELECT msg_id, vt, read_ct, enqueued_at, last_read_at, message, headers
598
+ FROM archived
599
+ RETURNING msg_id;
600
+ $QUERY$,
601
+ qtable, atable
602
+ );
603
+ EXECUTE sql USING msg_id INTO result;
604
+ RETURN NOT (result IS NULL);
605
+ END;
606
+ $$ LANGUAGE plpgsql;
607
+
608
+ ---- archive
609
+ ---- removes an array of message ids from the queue, and sends it to the archive,
610
+ ---- where these messages will be saved permanently.
611
+ CREATE FUNCTION pgmq.archive(
612
+ queue_name TEXT,
613
+ msg_ids BIGINT[]
614
+ )
615
+ RETURNS SETOF BIGINT AS $$
616
+ DECLARE
617
+ sql TEXT;
618
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
619
+ atable TEXT := pgmq.format_table_name(queue_name, 'a');
620
+ BEGIN
621
+ sql := FORMAT(
622
+ $QUERY$
623
+ WITH archived AS (
624
+ DELETE FROM pgmq.%I
625
+ WHERE msg_id = ANY($1)
626
+ RETURNING msg_id, vt, read_ct, enqueued_at, last_read_at, message, headers
627
+ )
628
+ INSERT INTO pgmq.%I (msg_id, vt, read_ct, enqueued_at, last_read_at, message, headers)
629
+ SELECT msg_id, vt, read_ct, enqueued_at, last_read_at, message, headers
630
+ FROM archived
631
+ RETURNING msg_id;
632
+ $QUERY$,
633
+ qtable, atable
634
+ );
635
+ RETURN QUERY EXECUTE sql USING msg_ids;
636
+ END;
637
+ $$ LANGUAGE plpgsql;
638
+
639
+ ---- delete
640
+ ---- deletes a message id from the queue permanently
641
+ CREATE FUNCTION pgmq.delete(
642
+ queue_name TEXT,
643
+ msg_id BIGINT
644
+ )
645
+ RETURNS BOOLEAN AS $$
646
+ DECLARE
647
+ sql TEXT;
648
+ result BIGINT;
649
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
650
+ BEGIN
651
+ sql := FORMAT(
652
+ $QUERY$
653
+ DELETE FROM pgmq.%I
654
+ WHERE msg_id = $1
655
+ RETURNING msg_id
656
+ $QUERY$,
657
+ qtable
658
+ );
659
+ EXECUTE sql USING msg_id INTO result;
660
+ RETURN NOT (result IS NULL);
661
+ END;
662
+ $$ LANGUAGE plpgsql;
663
+
664
+ ---- delete
665
+ ---- deletes an array of message ids from the queue permanently
666
+ CREATE FUNCTION pgmq.delete(
667
+ queue_name TEXT,
668
+ msg_ids BIGINT[]
669
+ )
670
+ RETURNS SETOF BIGINT AS $$
671
+ DECLARE
672
+ sql TEXT;
673
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
674
+ BEGIN
675
+ sql := FORMAT(
676
+ $QUERY$
677
+ DELETE FROM pgmq.%I
678
+ WHERE msg_id = ANY($1)
679
+ RETURNING msg_id
680
+ $QUERY$,
681
+ qtable
682
+ );
683
+ RETURN QUERY EXECUTE sql USING msg_ids;
684
+ END;
685
+ $$ LANGUAGE plpgsql;
686
+
687
+ -- send: actual implementation
688
+ CREATE FUNCTION pgmq.send(
689
+ queue_name TEXT,
690
+ msg JSONB,
691
+ headers JSONB,
692
+ delay TIMESTAMP WITH TIME ZONE
693
+ ) RETURNS SETOF BIGINT AS $$
694
+ DECLARE
695
+ sql TEXT;
696
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
697
+ BEGIN
698
+ sql := FORMAT(
699
+ $QUERY$
700
+ INSERT INTO pgmq.%I (vt, message, headers)
701
+ VALUES ($2, $1, $3)
702
+ RETURNING msg_id;
703
+ $QUERY$,
704
+ qtable
705
+ );
706
+ RETURN QUERY EXECUTE sql USING msg, delay, headers;
707
+ END;
708
+ $$ LANGUAGE plpgsql;
709
+
710
+ -- send: 2 args, no delay or headers
711
+ CREATE FUNCTION pgmq.send(
712
+ queue_name TEXT,
713
+ msg JSONB
714
+ ) RETURNS SETOF BIGINT AS $$
715
+ SELECT * FROM pgmq.send(queue_name, msg, NULL, clock_timestamp());
716
+ $$ LANGUAGE sql;
717
+
718
+ -- send: 3 args with headers
719
+ CREATE FUNCTION pgmq.send(
720
+ queue_name TEXT,
721
+ msg JSONB,
722
+ headers JSONB
723
+ ) RETURNS SETOF BIGINT AS $$
724
+ SELECT * FROM pgmq.send(queue_name, msg, headers, clock_timestamp());
725
+ $$ LANGUAGE sql;
726
+
727
+ -- send: 3 args with integer delay
728
+ CREATE FUNCTION pgmq.send(
729
+ queue_name TEXT,
730
+ msg JSONB,
731
+ delay INTEGER
732
+ ) RETURNS SETOF BIGINT AS $$
733
+ SELECT * FROM pgmq.send(queue_name, msg, NULL, clock_timestamp() + make_interval(secs => delay));
734
+ $$ LANGUAGE sql;
735
+
736
+ -- send: 3 args with timestamp
737
+ CREATE FUNCTION pgmq.send(
738
+ queue_name TEXT,
739
+ msg JSONB,
740
+ delay TIMESTAMP WITH TIME ZONE
741
+ ) RETURNS SETOF BIGINT AS $$
742
+ SELECT * FROM pgmq.send(queue_name, msg, NULL, delay);
743
+ $$ LANGUAGE sql;
744
+
745
+ -- send: 4 args with integer delay
746
+ CREATE FUNCTION pgmq.send(
747
+ queue_name TEXT,
748
+ msg JSONB,
749
+ headers JSONB,
750
+ delay INTEGER
751
+ ) RETURNS SETOF BIGINT AS $$
752
+ SELECT * FROM pgmq.send(queue_name, msg, headers, clock_timestamp() + make_interval(secs => delay));
753
+ $$ LANGUAGE sql;
754
+
755
+ -- _validate_batch_params: Private function to validate batch parameters
756
+ CREATE FUNCTION pgmq._validate_batch_params(
757
+ msgs JSONB[],
758
+ headers JSONB[]
759
+ ) RETURNS void AS $$
760
+ BEGIN
761
+ -- Validate that msgs is not NULL or empty
762
+ IF msgs IS NULL OR array_length(msgs, 1) IS NULL THEN
763
+ RAISE EXCEPTION 'msgs cannot be NULL or empty';
764
+ END IF;
765
+
766
+ -- Validate that headers array length matches msgs array length if headers is provided
767
+ -- Note: array_length returns NULL for empty arrays, so we use COALESCE to treat empty arrays as length 0
768
+ IF headers IS NOT NULL AND COALESCE(array_length(headers, 1), 0) != COALESCE(array_length(msgs, 1), 0) THEN
769
+ RAISE EXCEPTION 'headers array length (%) must match msgs array length (%)',
770
+ COALESCE(array_length(headers, 1), 0), COALESCE(array_length(msgs, 1), 0);
771
+ END IF;
772
+ END;
773
+ $$ LANGUAGE plpgsql;
774
+
775
+ -- _send_batch: Private function that performs the actual batch insert without validation
776
+ CREATE FUNCTION pgmq._send_batch(
777
+ queue_name TEXT,
778
+ msgs JSONB[],
779
+ headers JSONB[],
780
+ delay TIMESTAMP WITH TIME ZONE
781
+ ) RETURNS SETOF BIGINT AS $$
782
+ DECLARE
783
+ sql TEXT;
784
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
785
+ BEGIN
786
+ sql := FORMAT(
787
+ $QUERY$
788
+ INSERT INTO pgmq.%I (vt, message, headers)
789
+ SELECT $2, unnest($1), unnest(coalesce($3, ARRAY[]::jsonb[]))
790
+ RETURNING msg_id;
791
+ $QUERY$,
792
+ qtable
793
+ );
794
+ RETURN QUERY EXECUTE sql USING msgs, delay, headers;
795
+ END;
796
+ $$ LANGUAGE plpgsql;
797
+
798
+ -- send_batch: Public function with validation
799
+ CREATE FUNCTION pgmq.send_batch(
800
+ queue_name TEXT,
801
+ msgs JSONB[],
802
+ headers JSONB[],
803
+ delay TIMESTAMP WITH TIME ZONE
804
+ ) RETURNS SETOF BIGINT AS $$
805
+ BEGIN
806
+ PERFORM pgmq._validate_batch_params(msgs, headers);
807
+ RETURN QUERY SELECT * FROM pgmq._send_batch(queue_name, msgs, headers, delay);
808
+ END;
809
+ $$ LANGUAGE plpgsql;
810
+
811
+ -- send batch: 2 args
812
+ CREATE FUNCTION pgmq.send_batch(
813
+ queue_name TEXT,
814
+ msgs JSONB[]
815
+ ) RETURNS SETOF BIGINT AS $$
816
+ SELECT * FROM pgmq.send_batch(queue_name, msgs, NULL, clock_timestamp());
817
+ $$ LANGUAGE sql;
818
+
819
+ -- send batch: 3 args with headers
820
+ CREATE FUNCTION pgmq.send_batch(
821
+ queue_name TEXT,
822
+ msgs JSONB[],
823
+ headers JSONB[]
824
+ ) RETURNS SETOF BIGINT AS $$
825
+ SELECT * FROM pgmq.send_batch(queue_name, msgs, headers, clock_timestamp());
826
+ $$ LANGUAGE sql;
827
+
828
+ -- send batch: 3 args with integer delay
829
+ CREATE FUNCTION pgmq.send_batch(
830
+ queue_name TEXT,
831
+ msgs JSONB[],
832
+ delay INTEGER
833
+ ) RETURNS SETOF BIGINT AS $$
834
+ SELECT * FROM pgmq.send_batch(queue_name, msgs, NULL, clock_timestamp() + make_interval(secs => delay));
835
+ $$ LANGUAGE sql;
836
+
837
+ -- send batch: 3 args with timestamp
838
+ CREATE FUNCTION pgmq.send_batch(
839
+ queue_name TEXT,
840
+ msgs JSONB[],
841
+ delay TIMESTAMP WITH TIME ZONE
842
+ ) RETURNS SETOF BIGINT AS $$
843
+ SELECT * FROM pgmq.send_batch(queue_name, msgs, NULL, delay);
844
+ $$ LANGUAGE sql;
845
+
846
+ -- send_batch: 4 args with integer delay
847
+ CREATE FUNCTION pgmq.send_batch(
848
+ queue_name TEXT,
849
+ msgs JSONB[],
850
+ headers JSONB[],
851
+ delay INTEGER
852
+ ) RETURNS SETOF BIGINT AS $$
853
+ SELECT * FROM pgmq.send_batch(queue_name, msgs, headers, clock_timestamp() + make_interval(secs => delay));
854
+ $$ LANGUAGE sql;
855
+
856
+ -- returned by pgmq.metrics() and pgmq.metrics_all
857
+ -- Note: Changing the order of fields in this type is a breaking change -- our Rust Diesel client implementation
858
+ -- expects a specific order of fields.
859
+ CREATE TYPE pgmq.metrics_result AS (
860
+ queue_name text,
861
+ queue_length bigint,
862
+ newest_msg_age_sec int,
863
+ oldest_msg_age_sec int,
864
+ total_messages bigint,
865
+ scrape_time timestamp with time zone,
866
+ queue_visible_length bigint
867
+ );
868
+
869
+ -- get metrics for a single queue
870
+ CREATE FUNCTION pgmq.metrics(queue_name TEXT)
871
+ RETURNS pgmq.metrics_result AS $$
872
+ DECLARE
873
+ result_row pgmq.metrics_result;
874
+ query TEXT;
875
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
876
+ BEGIN
877
+ query := FORMAT(
878
+ $QUERY$
879
+ WITH q_summary AS (
880
+ SELECT
881
+ count(*) as queue_length,
882
+ count(CASE WHEN vt <= NOW() THEN 1 END) as queue_visible_length,
883
+ EXTRACT(epoch FROM (NOW() - max(enqueued_at)))::int as newest_msg_age_sec,
884
+ EXTRACT(epoch FROM (NOW() - min(enqueued_at)))::int as oldest_msg_age_sec,
885
+ NOW() as scrape_time
886
+ FROM pgmq.%I
887
+ ),
888
+ all_metrics AS (
889
+ SELECT CASE
890
+ WHEN is_called THEN last_value ELSE 0
891
+ END as total_messages
892
+ FROM pgmq.%I
893
+ )
894
+ SELECT
895
+ %L as queue_name,
896
+ q_summary.queue_length,
897
+ q_summary.newest_msg_age_sec,
898
+ q_summary.oldest_msg_age_sec,
899
+ all_metrics.total_messages,
900
+ q_summary.scrape_time,
901
+ q_summary.queue_visible_length
902
+ FROM q_summary, all_metrics
903
+ $QUERY$,
904
+ qtable, qtable || '_msg_id_seq', queue_name
905
+ );
906
+ EXECUTE query INTO result_row;
907
+ RETURN result_row;
908
+ END;
909
+ $$ LANGUAGE plpgsql;
910
+
911
+ -- get metrics for all queues
912
+ CREATE FUNCTION pgmq."metrics_all"()
913
+ RETURNS SETOF pgmq.metrics_result AS $$
914
+ DECLARE
915
+ row_name RECORD;
916
+ result_row pgmq.metrics_result;
917
+ BEGIN
918
+ FOR row_name IN SELECT queue_name FROM pgmq.meta LOOP
919
+ result_row := pgmq.metrics(row_name.queue_name);
920
+ RETURN NEXT result_row;
921
+ END LOOP;
922
+ END;
923
+ $$ LANGUAGE plpgsql;
924
+
925
+ -- list queues
926
+ CREATE FUNCTION pgmq."list_queues"()
927
+ RETURNS SETOF pgmq.queue_record AS $$
928
+ BEGIN
929
+ RETURN QUERY SELECT * FROM pgmq.meta;
930
+ END
931
+ $$ LANGUAGE plpgsql;
932
+
933
+ -- purge queue, deleting all entries in it.
934
+ CREATE OR REPLACE FUNCTION pgmq."purge_queue"(queue_name TEXT)
935
+ RETURNS BIGINT AS $$
936
+ DECLARE
937
+ deleted_count INTEGER;
938
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
939
+ BEGIN
940
+ -- Get the row count before truncating
941
+ EXECUTE format('SELECT count(*) FROM pgmq.%I', qtable) INTO deleted_count;
942
+
943
+ -- Use TRUNCATE for better performance on large tables
944
+ EXECUTE format('TRUNCATE TABLE pgmq.%I', qtable);
945
+
946
+ -- Return the number of purged rows
947
+ RETURN deleted_count;
948
+ END
949
+ $$ LANGUAGE plpgsql;
950
+
951
+ -- unassign archive, so it can be kept when a queue is deleted
952
+ CREATE FUNCTION pgmq."detach_archive"(queue_name TEXT)
953
+ RETURNS VOID AS $$
954
+ DECLARE
955
+ atable TEXT := pgmq.format_table_name(queue_name, 'a');
956
+ BEGIN
957
+ RAISE WARNING 'detach_archive(queue_name) is deprecated and is a no-op. It will be removed in PGMQ v2.0. Archive tables are no longer member objects.';
958
+ END
959
+ $$ LANGUAGE plpgsql;
960
+
961
+ -- pop: implementation
962
+ CREATE FUNCTION pgmq.pop(queue_name TEXT, qty INTEGER DEFAULT 1)
963
+ RETURNS SETOF pgmq.message_record AS $$
964
+ DECLARE
965
+ sql TEXT;
966
+ result pgmq.message_record;
967
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
968
+ BEGIN
969
+ sql := FORMAT(
970
+ $QUERY$
971
+ WITH cte AS
972
+ (
973
+ SELECT msg_id
974
+ FROM pgmq.%I
975
+ WHERE vt <= clock_timestamp()
976
+ ORDER BY msg_id ASC
977
+ LIMIT $1
978
+ FOR UPDATE SKIP LOCKED
979
+ )
980
+ DELETE from pgmq.%I
981
+ WHERE msg_id IN (select msg_id from cte)
982
+ RETURNING msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers;
983
+ $QUERY$,
984
+ qtable, qtable
985
+ );
986
+ RETURN QUERY EXECUTE sql USING qty;
987
+ END;
988
+ $$ LANGUAGE plpgsql;
989
+
990
+ -- Sets timestamp vt of a message, returns it
991
+ CREATE FUNCTION pgmq.set_vt(queue_name TEXT, msg_id BIGINT, vt TIMESTAMP WITH TIME ZONE)
992
+ RETURNS SETOF pgmq.message_record AS $$
993
+ DECLARE
994
+ sql TEXT;
995
+ result pgmq.message_record;
996
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
997
+ BEGIN
998
+ sql := FORMAT(
999
+ $QUERY$
1000
+ UPDATE pgmq.%I
1001
+ SET vt = $1
1002
+ WHERE msg_id = $2
1003
+ RETURNING msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers;
1004
+ $QUERY$,
1005
+ qtable
1006
+ );
1007
+ RETURN QUERY EXECUTE sql USING vt, msg_id;
1008
+ END;
1009
+ $$ LANGUAGE plpgsql;
1010
+
1011
+ -- Sets integer vt of a message, returns it
1012
+ CREATE FUNCTION pgmq.set_vt(queue_name TEXT, msg_id BIGINT, vt INTEGER)
1013
+ RETURNS SETOF pgmq.message_record AS $$
1014
+ SELECT * FROM pgmq.set_vt(queue_name, msg_id, clock_timestamp() + make_interval(secs => vt));
1015
+ $$ LANGUAGE sql;
1016
+
1017
+ -- Sets timestamp vt of multiple messages, returns them
1018
+ CREATE FUNCTION pgmq.set_vt(
1019
+ queue_name TEXT,
1020
+ msg_ids BIGINT[],
1021
+ vt TIMESTAMP WITH TIME ZONE
1022
+ )
1023
+ RETURNS SETOF pgmq.message_record AS $$
1024
+ DECLARE
1025
+ sql TEXT;
1026
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
1027
+ BEGIN
1028
+ sql := FORMAT(
1029
+ $QUERY$
1030
+ UPDATE pgmq.%I
1031
+ SET vt = $1
1032
+ WHERE msg_id = ANY($2)
1033
+ RETURNING msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers;
1034
+ $QUERY$,
1035
+ qtable
1036
+ );
1037
+ RETURN QUERY EXECUTE sql USING vt, msg_ids;
1038
+ END;
1039
+ $$ LANGUAGE plpgsql;
1040
+
1041
+ -- Sets integer vt of multiple messages, returns them
1042
+ CREATE FUNCTION pgmq.set_vt(
1043
+ queue_name TEXT,
1044
+ msg_ids BIGINT[],
1045
+ vt INTEGER
1046
+ )
1047
+ RETURNS SETOF pgmq.message_record AS $$
1048
+ SELECT * FROM pgmq.set_vt(queue_name, msg_ids, clock_timestamp() + make_interval(secs => vt));
1049
+ $$ LANGUAGE sql;
1050
+
1051
+ CREATE FUNCTION pgmq._get_pg_partman_schema()
1052
+ RETURNS TEXT AS $$
1053
+ SELECT
1054
+ extnamespace::regnamespace::text
1055
+ FROM
1056
+ pg_extension
1057
+ WHERE
1058
+ extname = 'pg_partman';
1059
+ $$ LANGUAGE SQL;
1060
+
1061
+ CREATE FUNCTION pgmq.drop_queue(queue_name TEXT, partitioned BOOLEAN)
1062
+ RETURNS BOOLEAN AS $$
1063
+ DECLARE
1064
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
1065
+ fq_qtable TEXT := 'pgmq.' || qtable;
1066
+ atable TEXT := pgmq.format_table_name(queue_name, 'a');
1067
+ fq_atable TEXT := 'pgmq.' || atable;
1068
+ BEGIN
1069
+ RAISE WARNING 'drop_queue(queue_name, partitioned) is deprecated and will be removed in PGMQ v2.0. Use drop_queue(queue_name) instead';
1070
+
1071
+ PERFORM pgmq.drop_queue(queue_name);
1072
+
1073
+ RETURN TRUE;
1074
+ END;
1075
+ $$ LANGUAGE plpgsql;
1076
+
1077
+ CREATE FUNCTION pgmq.drop_queue(queue_name TEXT)
1078
+ RETURNS BOOLEAN AS $$
1079
+ DECLARE
1080
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
1081
+ qtable_seq TEXT := qtable || '_msg_id_seq';
1082
+ fq_qtable TEXT := 'pgmq.' || qtable;
1083
+ atable TEXT := pgmq.format_table_name(queue_name, 'a');
1084
+ fq_atable TEXT := 'pgmq.' || atable;
1085
+ partitioned BOOLEAN;
1086
+ BEGIN
1087
+ PERFORM pgmq.acquire_queue_lock(queue_name);
1088
+ EXECUTE FORMAT(
1089
+ $QUERY$
1090
+ SELECT is_partitioned FROM pgmq.meta WHERE queue_name = %L
1091
+ $QUERY$,
1092
+ queue_name
1093
+ ) INTO partitioned;
1094
+
1095
+ -- check if the queue exists
1096
+ IF NOT EXISTS (
1097
+ SELECT 1
1098
+ FROM information_schema.tables
1099
+ WHERE table_name = qtable and table_schema = 'pgmq'
1100
+ ) THEN
1101
+ RAISE NOTICE 'pgmq queue `%` does not exist', queue_name;
1102
+ RETURN FALSE;
1103
+ END IF;
1104
+
1105
+ EXECUTE FORMAT(
1106
+ $QUERY$
1107
+ DROP TABLE IF EXISTS pgmq.%I
1108
+ $QUERY$,
1109
+ qtable
1110
+ );
1111
+
1112
+ EXECUTE FORMAT(
1113
+ $QUERY$
1114
+ DROP TABLE IF EXISTS pgmq.%I
1115
+ $QUERY$,
1116
+ atable
1117
+ );
1118
+
1119
+ IF EXISTS (
1120
+ SELECT 1
1121
+ FROM information_schema.tables
1122
+ WHERE table_name = 'meta' and table_schema = 'pgmq'
1123
+ ) THEN
1124
+ EXECUTE FORMAT(
1125
+ $QUERY$
1126
+ DELETE FROM pgmq.meta WHERE queue_name = %L
1127
+ $QUERY$,
1128
+ queue_name
1129
+ );
1130
+ END IF;
1131
+
1132
+ IF partitioned THEN
1133
+ EXECUTE FORMAT(
1134
+ $QUERY$
1135
+ DELETE FROM %I.part_config where parent_table in (%L, %L)
1136
+ $QUERY$,
1137
+ pgmq._get_pg_partman_schema(), fq_qtable, fq_atable
1138
+ );
1139
+ END IF;
1140
+
1141
+ RETURN TRUE;
1142
+ END;
1143
+ $$ LANGUAGE plpgsql;
1144
+
1145
+ CREATE FUNCTION pgmq.validate_queue_name(queue_name TEXT)
1146
+ RETURNS void AS $$
1147
+ BEGIN
1148
+ IF length(queue_name) > 47 THEN
1149
+ -- complete table identifier must be <= 63
1150
+ -- https://www.postgresql.org/docs/17/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
1151
+ -- e.g. template_pgmq_q_my_queue is an identifier for my_queue when partitioned
1152
+ -- template_pgmq_q_ (16) + <a max length queue name> (47) = 63
1153
+ RAISE EXCEPTION 'queue name is too long, maximum length is 47 characters';
1154
+ END IF;
1155
+ END;
1156
+ $$ LANGUAGE plpgsql;
1157
+
1158
+ CREATE FUNCTION pgmq._belongs_to_pgmq(table_name TEXT)
1159
+ RETURNS BOOLEAN AS $$
1160
+ DECLARE
1161
+ sql TEXT;
1162
+ result BOOLEAN;
1163
+ BEGIN
1164
+ SELECT EXISTS (
1165
+ SELECT 1
1166
+ FROM pg_depend
1167
+ WHERE refobjid = (SELECT oid FROM pg_extension WHERE extname = 'pgmq')
1168
+ AND objid = (
1169
+ SELECT oid
1170
+ FROM pg_class
1171
+ WHERE relname = table_name
1172
+ )
1173
+ ) INTO result;
1174
+ RETURN result;
1175
+ END;
1176
+ $$ LANGUAGE plpgsql;
1177
+
1178
+ CREATE FUNCTION pgmq.create_non_partitioned(queue_name TEXT)
1179
+ RETURNS void AS $$
1180
+ DECLARE
1181
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
1182
+ qtable_seq TEXT := qtable || '_msg_id_seq';
1183
+ atable TEXT := pgmq.format_table_name(queue_name, 'a');
1184
+ BEGIN
1185
+ PERFORM pgmq.validate_queue_name(queue_name);
1186
+ PERFORM pgmq.acquire_queue_lock(queue_name);
1187
+
1188
+ EXECUTE FORMAT(
1189
+ $QUERY$
1190
+ CREATE TABLE IF NOT EXISTS pgmq.%I (
1191
+ msg_id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
1192
+ read_ct INT DEFAULT 0 NOT NULL,
1193
+ enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
1194
+ last_read_at TIMESTAMP WITH TIME ZONE,
1195
+ vt TIMESTAMP WITH TIME ZONE NOT NULL,
1196
+ message JSONB,
1197
+ headers JSONB
1198
+ )
1199
+ $QUERY$,
1200
+ qtable
1201
+ );
1202
+
1203
+ EXECUTE FORMAT(
1204
+ $QUERY$
1205
+ CREATE TABLE IF NOT EXISTS pgmq.%I (
1206
+ msg_id BIGINT PRIMARY KEY,
1207
+ read_ct INT DEFAULT 0 NOT NULL,
1208
+ enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
1209
+ last_read_at TIMESTAMP WITH TIME ZONE,
1210
+ archived_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
1211
+ vt TIMESTAMP WITH TIME ZONE NOT NULL,
1212
+ message JSONB,
1213
+ headers JSONB
1214
+ );
1215
+ $QUERY$,
1216
+ atable
1217
+ );
1218
+
1219
+ EXECUTE FORMAT(
1220
+ $QUERY$
1221
+ CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (vt ASC);
1222
+ $QUERY$,
1223
+ qtable || '_vt_idx', qtable
1224
+ );
1225
+
1226
+ EXECUTE FORMAT(
1227
+ $QUERY$
1228
+ CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (archived_at);
1229
+ $QUERY$,
1230
+ 'archived_at_idx_' || queue_name, atable
1231
+ );
1232
+
1233
+ EXECUTE FORMAT(
1234
+ $QUERY$
1235
+ INSERT INTO pgmq.meta (queue_name, is_partitioned, is_unlogged)
1236
+ VALUES (%L, false, false)
1237
+ ON CONFLICT
1238
+ DO NOTHING;
1239
+ $QUERY$,
1240
+ queue_name
1241
+ );
1242
+
1243
+ END;
1244
+ $$ LANGUAGE plpgsql;
1245
+
1246
+ CREATE FUNCTION pgmq.create_unlogged(queue_name TEXT)
1247
+ RETURNS void AS $$
1248
+ DECLARE
1249
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
1250
+ qtable_seq TEXT := qtable || '_msg_id_seq';
1251
+ atable TEXT := pgmq.format_table_name(queue_name, 'a');
1252
+ BEGIN
1253
+ PERFORM pgmq.validate_queue_name(queue_name);
1254
+ PERFORM pgmq.acquire_queue_lock(queue_name);
1255
+
1256
+ EXECUTE FORMAT(
1257
+ $QUERY$
1258
+ CREATE UNLOGGED TABLE IF NOT EXISTS pgmq.%I (
1259
+ msg_id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
1260
+ read_ct INT DEFAULT 0 NOT NULL,
1261
+ enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
1262
+ last_read_at TIMESTAMP WITH TIME ZONE,
1263
+ vt TIMESTAMP WITH TIME ZONE NOT NULL,
1264
+ message JSONB,
1265
+ headers JSONB
1266
+ )
1267
+ $QUERY$,
1268
+ qtable
1269
+ );
1270
+
1271
+ EXECUTE FORMAT(
1272
+ $QUERY$
1273
+ CREATE TABLE IF NOT EXISTS pgmq.%I (
1274
+ msg_id BIGINT PRIMARY KEY,
1275
+ read_ct INT DEFAULT 0 NOT NULL,
1276
+ enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
1277
+ last_read_at TIMESTAMP WITH TIME ZONE,
1278
+ archived_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
1279
+ vt TIMESTAMP WITH TIME ZONE NOT NULL,
1280
+ message JSONB,
1281
+ headers JSONB
1282
+ );
1283
+ $QUERY$,
1284
+ atable
1285
+ );
1286
+
1287
+ EXECUTE FORMAT(
1288
+ $QUERY$
1289
+ CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (vt ASC);
1290
+ $QUERY$,
1291
+ qtable || '_vt_idx', qtable
1292
+ );
1293
+
1294
+ EXECUTE FORMAT(
1295
+ $QUERY$
1296
+ CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (archived_at);
1297
+ $QUERY$,
1298
+ 'archived_at_idx_' || queue_name, atable
1299
+ );
1300
+
1301
+ EXECUTE FORMAT(
1302
+ $QUERY$
1303
+ INSERT INTO pgmq.meta (queue_name, is_partitioned, is_unlogged)
1304
+ VALUES (%L, false, true)
1305
+ ON CONFLICT
1306
+ DO NOTHING;
1307
+ $QUERY$,
1308
+ queue_name
1309
+ );
1310
+ END;
1311
+ $$ LANGUAGE plpgsql;
1312
+
1313
+ CREATE FUNCTION pgmq._get_partition_col(partition_interval TEXT)
1314
+ RETURNS TEXT AS $$
1315
+ DECLARE
1316
+ num INTEGER;
1317
+ BEGIN
1318
+ BEGIN
1319
+ num := partition_interval::INTEGER;
1320
+ RETURN 'msg_id';
1321
+ EXCEPTION
1322
+ WHEN others THEN
1323
+ RETURN 'enqueued_at';
1324
+ END;
1325
+ END;
1326
+ $$ LANGUAGE plpgsql;
1327
+
1328
+ CREATE FUNCTION pgmq._extension_exists(extension_name TEXT)
1329
+ RETURNS BOOLEAN
1330
+ LANGUAGE SQL
1331
+ AS $$
1332
+ SELECT EXISTS (
1333
+ SELECT 1
1334
+ FROM pg_extension
1335
+ WHERE extname = extension_name
1336
+ )
1337
+ $$;
1338
+
1339
+ CREATE FUNCTION pgmq._ensure_pg_partman_installed()
1340
+ RETURNS void AS $$
1341
+ BEGIN
1342
+ IF NOT pgmq._extension_exists('pg_partman') THEN
1343
+ RAISE EXCEPTION 'pg_partman is required for partitioned queues';
1344
+ END IF;
1345
+ END;
1346
+ $$ LANGUAGE plpgsql;
1347
+
1348
+ CREATE FUNCTION pgmq._get_pg_partman_major_version()
1349
+ RETURNS INT
1350
+ LANGUAGE SQL
1351
+ AS $$
1352
+ SELECT split_part(extversion, '.', 1)::INT
1353
+ FROM pg_extension
1354
+ WHERE extname = 'pg_partman'
1355
+ $$;
1356
+
1357
+ CREATE FUNCTION pgmq.create_partitioned(
1358
+ queue_name TEXT,
1359
+ partition_interval TEXT DEFAULT '10000',
1360
+ retention_interval TEXT DEFAULT '100000'
1361
+ )
1362
+ RETURNS void AS $$
1363
+ DECLARE
1364
+ partition_col TEXT;
1365
+ a_partition_col TEXT;
1366
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
1367
+ qtable_seq TEXT := qtable || '_msg_id_seq';
1368
+ atable TEXT := pgmq.format_table_name(queue_name, 'a');
1369
+ fq_qtable TEXT := 'pgmq.' || qtable;
1370
+ fq_atable TEXT := 'pgmq.' || atable;
1371
+ BEGIN
1372
+ PERFORM pgmq.validate_queue_name(queue_name);
1373
+ PERFORM pgmq.acquire_queue_lock(queue_name);
1374
+ PERFORM pgmq._ensure_pg_partman_installed();
1375
+ SELECT pgmq._get_partition_col(partition_interval) INTO partition_col;
1376
+
1377
+ EXECUTE FORMAT(
1378
+ $QUERY$
1379
+ CREATE TABLE IF NOT EXISTS pgmq.%I (
1380
+ msg_id BIGINT GENERATED ALWAYS AS IDENTITY,
1381
+ read_ct INT DEFAULT 0 NOT NULL,
1382
+ enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
1383
+ last_read_at TIMESTAMP WITH TIME ZONE,
1384
+ vt TIMESTAMP WITH TIME ZONE NOT NULL,
1385
+ message JSONB,
1386
+ headers JSONB
1387
+ ) PARTITION BY RANGE (%I)
1388
+ $QUERY$,
1389
+ qtable, partition_col
1390
+ );
1391
+
1392
+ -- https://github.com/pgpartman/pg_partman/blob/master/doc/pg_partman.md
1393
+ -- p_parent_table - the existing parent table. MUST be schema qualified, even if in public schema.
1394
+ EXECUTE FORMAT(
1395
+ $QUERY$
1396
+ SELECT %I.create_parent(
1397
+ p_parent_table := %L,
1398
+ p_control := %L,
1399
+ p_interval := %L,
1400
+ p_type := case
1401
+ when pgmq._get_pg_partman_major_version() = 5 then 'range'
1402
+ else 'native'
1403
+ end
1404
+ )
1405
+ $QUERY$,
1406
+ pgmq._get_pg_partman_schema(),
1407
+ fq_qtable,
1408
+ partition_col,
1409
+ partition_interval
1410
+ );
1411
+
1412
+ EXECUTE FORMAT(
1413
+ $QUERY$
1414
+ CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (%I);
1415
+ $QUERY$,
1416
+ qtable || '_part_idx', qtable, partition_col
1417
+ );
1418
+
1419
+ EXECUTE FORMAT(
1420
+ $QUERY$
1421
+ UPDATE %I.part_config
1422
+ SET
1423
+ retention = %L,
1424
+ retention_keep_table = false,
1425
+ retention_keep_index = true,
1426
+ automatic_maintenance = 'on'
1427
+ WHERE parent_table = %L;
1428
+ $QUERY$,
1429
+ pgmq._get_pg_partman_schema(),
1430
+ retention_interval,
1431
+ 'pgmq.' || qtable
1432
+ );
1433
+
1434
+ EXECUTE FORMAT(
1435
+ $QUERY$
1436
+ INSERT INTO pgmq.meta (queue_name, is_partitioned, is_unlogged)
1437
+ VALUES (%L, true, false)
1438
+ ON CONFLICT
1439
+ DO NOTHING;
1440
+ $QUERY$,
1441
+ queue_name
1442
+ );
1443
+
1444
+ IF partition_col = 'enqueued_at' THEN
1445
+ a_partition_col := 'archived_at';
1446
+ ELSE
1447
+ a_partition_col := partition_col;
1448
+ END IF;
1449
+
1450
+ EXECUTE FORMAT(
1451
+ $QUERY$
1452
+ CREATE TABLE IF NOT EXISTS pgmq.%I (
1453
+ msg_id BIGINT NOT NULL,
1454
+ read_ct INT DEFAULT 0 NOT NULL,
1455
+ enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
1456
+ last_read_at TIMESTAMP WITH TIME ZONE,
1457
+ archived_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
1458
+ vt TIMESTAMP WITH TIME ZONE NOT NULL,
1459
+ message JSONB,
1460
+ headers JSONB
1461
+ ) PARTITION BY RANGE (%I);
1462
+ $QUERY$,
1463
+ atable, a_partition_col
1464
+ );
1465
+
1466
+ -- https://github.com/pgpartman/pg_partman/blob/master/doc/pg_partman.md
1467
+ -- p_parent_table - the existing parent table. MUST be schema qualified, even if in public schema.
1468
+ EXECUTE FORMAT(
1469
+ $QUERY$
1470
+ SELECT %I.create_parent(
1471
+ p_parent_table := %L,
1472
+ p_control := %L,
1473
+ p_interval := %L,
1474
+ p_type := case
1475
+ when pgmq._get_pg_partman_major_version() = 5 then 'range'
1476
+ else 'native'
1477
+ end
1478
+ )
1479
+ $QUERY$,
1480
+ pgmq._get_pg_partman_schema(),
1481
+ fq_atable,
1482
+ a_partition_col,
1483
+ partition_interval
1484
+ );
1485
+
1486
+ EXECUTE FORMAT(
1487
+ $QUERY$
1488
+ UPDATE %I.part_config
1489
+ SET
1490
+ retention = %L,
1491
+ retention_keep_table = false,
1492
+ retention_keep_index = true,
1493
+ automatic_maintenance = 'on'
1494
+ WHERE parent_table = %L;
1495
+ $QUERY$,
1496
+ pgmq._get_pg_partman_schema(),
1497
+ retention_interval,
1498
+ 'pgmq.' || atable
1499
+ );
1500
+
1501
+ EXECUTE FORMAT(
1502
+ $QUERY$
1503
+ CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (archived_at);
1504
+ $QUERY$,
1505
+ 'archived_at_idx_' || queue_name, atable
1506
+ );
1507
+
1508
+ END;
1509
+ $$ LANGUAGE plpgsql;
1510
+
1511
+ CREATE FUNCTION pgmq.create(queue_name TEXT)
1512
+ RETURNS void AS $$
1513
+ BEGIN
1514
+ PERFORM pgmq.create_non_partitioned(queue_name);
1515
+ END;
1516
+ $$ LANGUAGE plpgsql;
1517
+
1518
+ -- _create_fifo_index_if_not_exists
1519
+ -- internal function to create GIN index on headers for better FIFO performance
1520
+ CREATE OR REPLACE FUNCTION pgmq._create_fifo_index_if_not_exists(queue_name TEXT)
1521
+ RETURNS void AS $$
1522
+ DECLARE
1523
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
1524
+ index_name TEXT := qtable || '_fifo_idx';
1525
+ BEGIN
1526
+ -- Create GIN index on headers for efficient FIFO key lookups
1527
+ EXECUTE FORMAT(
1528
+ $QUERY$
1529
+ CREATE INDEX IF NOT EXISTS %I ON pgmq.%I USING GIN (headers);
1530
+ $QUERY$,
1531
+ index_name, qtable
1532
+ );
1533
+ END;
1534
+ $$ LANGUAGE plpgsql;
1535
+
1536
+ -- create_fifo_index
1537
+ -- creates a GIN index on the headers column to improve FIFO read performance
1538
+ CREATE FUNCTION pgmq.create_fifo_index(queue_name TEXT)
1539
+ RETURNS void AS $$
1540
+ BEGIN
1541
+ PERFORM pgmq._create_fifo_index_if_not_exists(queue_name);
1542
+ END;
1543
+ $$ LANGUAGE plpgsql;
1544
+
1545
+ -- create_fifo_indexes_all
1546
+ -- creates FIFO indexes on all existing queues
1547
+ CREATE FUNCTION pgmq.create_fifo_indexes_all()
1548
+ RETURNS void AS $$
1549
+ DECLARE
1550
+ queue_record RECORD;
1551
+ BEGIN
1552
+ FOR queue_record IN SELECT queue_name FROM pgmq.meta LOOP
1553
+ PERFORM pgmq.create_fifo_index(queue_record.queue_name);
1554
+ END LOOP;
1555
+ END;
1556
+ $$ LANGUAGE plpgsql;
1557
+
1558
+ CREATE OR REPLACE FUNCTION pgmq.convert_archive_partitioned(
1559
+ table_name TEXT,
1560
+ partition_interval TEXT DEFAULT '10000',
1561
+ retention_interval TEXT DEFAULT '100000',
1562
+ leading_partition INT DEFAULT 10
1563
+ )
1564
+ RETURNS void AS $$
1565
+ DECLARE
1566
+ a_table_name TEXT := pgmq.format_table_name(table_name, 'a');
1567
+ a_table_name_old TEXT := pgmq.format_table_name(table_name, 'a') || '_old';
1568
+ qualified_a_table_name TEXT := format('pgmq.%I', a_table_name);
1569
+ partition_col TEXT;
1570
+ a_partition_col TEXT;
1571
+ BEGIN
1572
+
1573
+ PERFORM c.relkind
1574
+ FROM pg_class c
1575
+ JOIN pg_namespace n ON n.oid = c.relnamespace
1576
+ WHERE c.relname = a_table_name
1577
+ AND c.relkind = 'p';
1578
+
1579
+ IF FOUND THEN
1580
+ RAISE NOTICE 'Table %s is already partitioned', a_table_name;
1581
+ RETURN;
1582
+ END IF;
1583
+
1584
+ PERFORM c.relkind
1585
+ FROM pg_class c
1586
+ JOIN pg_namespace n ON n.oid = c.relnamespace
1587
+ WHERE c.relname = a_table_name
1588
+ AND c.relkind = 'r';
1589
+
1590
+ IF NOT FOUND THEN
1591
+ RAISE NOTICE 'Table %s does not exists', a_table_name;
1592
+ RETURN;
1593
+ END IF;
1594
+
1595
+ SELECT pgmq._get_partition_col(partition_interval) INTO partition_col;
1596
+
1597
+ -- For archive tables, use archived_at for time-based partitioning
1598
+ IF partition_col = 'enqueued_at' THEN
1599
+ a_partition_col := 'archived_at';
1600
+ ELSE
1601
+ a_partition_col := partition_col;
1602
+ END IF;
1603
+
1604
+ EXECUTE 'ALTER TABLE ' || qualified_a_table_name || ' RENAME TO ' || a_table_name_old;
1605
+
1606
+ -- When partitioning by time (archived_at), we need to exclude constraints and indexes
1607
+ -- because the existing PRIMARY KEY on msg_id alone is incompatible with partitioning by archived_at.
1608
+ -- When partitioning by msg_id, we can keep all constraints including PRIMARY KEY.
1609
+ IF a_partition_col = 'archived_at' THEN
1610
+ EXECUTE format( 'CREATE TABLE pgmq.%I (LIKE pgmq.%I including defaults including generated including storage including comments) PARTITION BY RANGE (%I)', a_table_name, a_table_name_old, a_partition_col );
1611
+ ELSE
1612
+ EXECUTE format( 'CREATE TABLE pgmq.%I (LIKE pgmq.%I including all) PARTITION BY RANGE (%I)', a_table_name, a_table_name_old, a_partition_col );
1613
+ END IF;
1614
+
1615
+ EXECUTE 'ALTER INDEX pgmq.archived_at_idx_' || table_name || ' RENAME TO archived_at_idx_' || table_name || '_old';
1616
+ EXECUTE 'CREATE INDEX archived_at_idx_'|| table_name || ' ON ' || qualified_a_table_name ||'(archived_at)';
1617
+
1618
+ -- https://github.com/pgpartman/pg_partman/blob/master/doc/pg_partman.md
1619
+ -- p_parent_table - the existing parent table. MUST be schema qualified, even if in public schema.
1620
+ EXECUTE FORMAT(
1621
+ $QUERY$
1622
+ SELECT %I.create_parent(
1623
+ p_parent_table := %L,
1624
+ p_control := %L,
1625
+ p_interval := %L,
1626
+ p_type := case
1627
+ when pgmq._get_pg_partman_major_version() = 5 then 'range'
1628
+ else 'native'
1629
+ end
1630
+ )
1631
+ $QUERY$,
1632
+ pgmq._get_pg_partman_schema(),
1633
+ qualified_a_table_name,
1634
+ a_partition_col,
1635
+ partition_interval
1636
+ );
1637
+
1638
+ EXECUTE FORMAT(
1639
+ $QUERY$
1640
+ UPDATE %I.part_config
1641
+ SET
1642
+ retention = %L,
1643
+ retention_keep_table = false,
1644
+ retention_keep_index = false,
1645
+ infinite_time_partitions = true
1646
+ WHERE
1647
+ parent_table = %L;
1648
+ $QUERY$,
1649
+ pgmq._get_pg_partman_schema(),
1650
+ retention_interval,
1651
+ qualified_a_table_name
1652
+ );
1653
+ END;
1654
+ $$ LANGUAGE plpgsql;
1655
+
1656
+ CREATE OR REPLACE FUNCTION pgmq.notify_queue_listeners()
1657
+ RETURNS TRIGGER AS $$
1658
+ DECLARE
1659
+ queue_name_extracted TEXT; -- Queue name extracted from trigger table name
1660
+ updated_count INTEGER; -- Number of rows updated (0 or 1)
1661
+ BEGIN
1662
+ queue_name_extracted := substring(TG_TABLE_NAME from 3);
1663
+
1664
+ UPDATE pgmq.notify_insert_throttle
1665
+ SET last_notified_at = clock_timestamp()
1666
+ WHERE queue_name = queue_name_extracted
1667
+ AND (
1668
+ throttle_interval_ms = 0 -- No throttling configured
1669
+ OR clock_timestamp() - last_notified_at >=
1670
+ (throttle_interval_ms * INTERVAL '1 millisecond') -- Throttle interval has elapsed
1671
+ );
1672
+
1673
+ -- Check how many rows were updated (will be 0 or 1)
1674
+ GET DIAGNOSTICS updated_count = ROW_COUNT;
1675
+
1676
+ IF updated_count > 0 THEN
1677
+ PERFORM PG_NOTIFY('pgmq.' || TG_TABLE_NAME || '.' || TG_OP, NULL);
1678
+ END IF;
1679
+
1680
+ RETURN NEW;
1681
+ END;
1682
+ $$ LANGUAGE plpgsql;
1683
+
1684
+ CREATE OR REPLACE FUNCTION pgmq.enable_notify_insert(queue_name TEXT, throttle_interval_ms INTEGER DEFAULT 250)
1685
+ RETURNS void AS $$
1686
+ DECLARE
1687
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
1688
+ v_queue_name TEXT := queue_name;
1689
+ v_throttle_interval_ms INTEGER := throttle_interval_ms;
1690
+ BEGIN
1691
+ -- Validate that throttle_interval_ms is non-negative
1692
+ IF v_throttle_interval_ms < 0 THEN
1693
+ RAISE EXCEPTION 'throttle_interval_ms must be non-negative';
1694
+ END IF;
1695
+
1696
+ -- Validate that the queue table exists
1697
+ IF NOT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'pgmq' AND table_name = qtable) THEN
1698
+ RAISE EXCEPTION 'Queue "%" does not exist. Create it first using pgmq.create()', v_queue_name;
1699
+ END IF;
1700
+
1701
+ PERFORM pgmq.disable_notify_insert(v_queue_name);
1702
+
1703
+ INSERT INTO pgmq.notify_insert_throttle (queue_name, throttle_interval_ms)
1704
+ VALUES (v_queue_name, v_throttle_interval_ms)
1705
+ ON CONFLICT ON CONSTRAINT notify_insert_throttle_queue_name_key DO UPDATE
1706
+ SET throttle_interval_ms = EXCLUDED.throttle_interval_ms,
1707
+ last_notified_at = to_timestamp(0);
1708
+
1709
+ EXECUTE FORMAT(
1710
+ $QUERY$
1711
+ CREATE CONSTRAINT TRIGGER trigger_notify_queue_insert_listeners
1712
+ AFTER INSERT ON pgmq.%I
1713
+ DEFERRABLE FOR EACH ROW
1714
+ EXECUTE PROCEDURE pgmq.notify_queue_listeners()
1715
+ $QUERY$,
1716
+ qtable
1717
+ );
1718
+ END;
1719
+ $$ LANGUAGE plpgsql;
1720
+
1721
+ CREATE OR REPLACE FUNCTION pgmq.disable_notify_insert(queue_name TEXT)
1722
+ RETURNS void AS $$
1723
+ DECLARE
1724
+ qtable TEXT := pgmq.format_table_name(queue_name, 'q');
1725
+ v_queue_name TEXT := queue_name;
1726
+ BEGIN
1727
+ EXECUTE FORMAT(
1728
+ $QUERY$
1729
+ DROP TRIGGER IF EXISTS trigger_notify_queue_insert_listeners ON pgmq.%I;
1730
+ $QUERY$,
1731
+ qtable
1732
+ );
1733
+
1734
+ DELETE FROM pgmq.notify_insert_throttle nit WHERE nit.queue_name = v_queue_name;
1735
+ END;
1736
+ $$ LANGUAGE plpgsql;
1737
+
1738
+ CREATE OR REPLACE FUNCTION pgmq.list_notify_insert_throttles()
1739
+ RETURNS TABLE
1740
+ (
1741
+ queue_name text,
1742
+ throttle_interval_ms integer,
1743
+ last_notified_at TIMESTAMP WITH TIME ZONE
1744
+ )
1745
+ LANGUAGE sql
1746
+ STABLE
1747
+ AS
1748
+ $$
1749
+ SELECT queue_name, throttle_interval_ms, last_notified_at
1750
+ FROM pgmq.notify_insert_throttle
1751
+ ORDER BY queue_name;
1752
+ $$;
1753
+
1754
+ CREATE OR REPLACE FUNCTION pgmq.update_notify_insert(queue_name text, throttle_interval_ms integer)
1755
+ RETURNS void
1756
+ LANGUAGE plpgsql
1757
+ AS
1758
+ $$
1759
+ BEGIN
1760
+ IF throttle_interval_ms < 0 THEN
1761
+ RAISE EXCEPTION 'throttle_interval_ms must be non-negative, got: %', throttle_interval_ms;
1762
+ END IF;
1763
+
1764
+ IF NOT EXISTS (SELECT 1 FROM pgmq.meta WHERE meta.queue_name = update_notify_insert.queue_name) THEN
1765
+ RAISE EXCEPTION 'Queue "%" does not exist. Create the queue first using pgmq.create()', queue_name;
1766
+ END IF;
1767
+
1768
+ IF NOT EXISTS (SELECT 1 FROM pgmq.notify_insert_throttle WHERE notify_insert_throttle.queue_name = update_notify_insert.queue_name) THEN
1769
+ RAISE EXCEPTION 'Queue "%" does not have notify_insert enabled. Enable it first using pgmq.enable_notify_insert()', queue_name;
1770
+ END IF;
1771
+
1772
+ UPDATE pgmq.notify_insert_throttle
1773
+ SET throttle_interval_ms = update_notify_insert.throttle_interval_ms,
1774
+ last_notified_at = to_timestamp(0)
1775
+ WHERE notify_insert_throttle.queue_name = update_notify_insert.queue_name;
1776
+ END;
1777
+ $$;
1778
+
1779
+ CREATE OR REPLACE FUNCTION pgmq.validate_routing_key(routing_key text)
1780
+ RETURNS boolean
1781
+ LANGUAGE plpgsql
1782
+ IMMUTABLE
1783
+ AS
1784
+ $$
1785
+ BEGIN
1786
+ -- Valid routing key examples:
1787
+ -- "logs.error"
1788
+ -- "app.user-service.auth"
1789
+ -- "system_events.db.connection_failed"
1790
+ --
1791
+ -- Invalid routing key examples:
1792
+ -- "" - empty
1793
+ -- ".logs.error" - starts with dot
1794
+ -- "logs.error." - ends with dot
1795
+ -- "logs..error" - consecutive dots
1796
+ -- "logs.error!" - invalid character
1797
+ -- "logs error" - space not allowed
1798
+ -- "logs.*" - wildcards not allowed in routing keys
1799
+
1800
+ IF routing_key IS NULL OR routing_key = '' THEN
1801
+ RAISE EXCEPTION 'routing_key cannot be NULL or empty';
1802
+ END IF;
1803
+
1804
+ IF length(routing_key) > 255 THEN
1805
+ RAISE EXCEPTION 'routing_key length cannot exceed 255 characters, got % characters', length(routing_key);
1806
+ END IF;
1807
+
1808
+ IF routing_key !~ '^[a-zA-Z0-9._-]+$' THEN
1809
+ RAISE EXCEPTION 'routing_key contains invalid characters. Only alphanumeric, dots, hyphens, and underscores are allowed. Got: %', routing_key;
1810
+ END IF;
1811
+
1812
+ IF routing_key ~ '^\.' THEN
1813
+ RAISE EXCEPTION 'routing_key cannot start with a dot. Got: %', routing_key;
1814
+ END IF;
1815
+
1816
+ IF routing_key ~ '\.$' THEN
1817
+ RAISE EXCEPTION 'routing_key cannot end with a dot. Got: %', routing_key;
1818
+ END IF;
1819
+
1820
+ IF routing_key ~ '\.\.' THEN
1821
+ RAISE EXCEPTION 'routing_key cannot contain consecutive dots. Got: %', routing_key;
1822
+ END IF;
1823
+
1824
+ RETURN true;
1825
+ END;
1826
+ $$;
1827
+
1828
+ CREATE OR REPLACE FUNCTION pgmq.validate_topic_pattern(pattern text)
1829
+ RETURNS boolean
1830
+ LANGUAGE plpgsql
1831
+ IMMUTABLE
1832
+ AS
1833
+ $$
1834
+ BEGIN
1835
+ -- Valid pattern examples:
1836
+ -- "logs.*" - matches one segment after logs. (e.g., logs.error, logs.info)
1837
+ -- "logs.#" - matches one or more segments after logs. (e.g., logs.error, logs.api.error)
1838
+ -- "*.error" - matches one segment before .error (e.g., app.error, db.error)
1839
+ -- "#.error" - matches one or more segments before .error (e.g., app.error, x.y.error)
1840
+ -- "app.*.#" - mixed wildcards (one segment then one or more)
1841
+ -- "#" - catch-all pattern, matches any routing key
1842
+ --
1843
+ -- Invalid pattern examples:
1844
+ -- ".logs.*" - starts with dot
1845
+ -- "logs.*." - ends with dot
1846
+ -- "logs..error" - consecutive dots
1847
+ -- "logs.**" - consecutive stars
1848
+ -- "logs.##" - consecutive hashes
1849
+ -- "logs.*#" - adjacent wildcards
1850
+ -- "logs.error!" - invalid character
1851
+
1852
+ IF pattern IS NULL OR pattern = '' THEN
1853
+ RAISE EXCEPTION 'pattern cannot be NULL or empty';
1854
+ END IF;
1855
+
1856
+ IF length(pattern) > 255 THEN
1857
+ RAISE EXCEPTION 'pattern length cannot exceed 255 characters, got % characters', length(pattern);
1858
+ END IF;
1859
+
1860
+ IF pattern !~ '^[a-zA-Z0-9._\-*#]+$' THEN
1861
+ RAISE EXCEPTION 'pattern contains invalid characters. Only alphanumeric, dots, hyphens, underscores, *, and # are allowed. Got: %', pattern;
1862
+ END IF;
1863
+
1864
+ IF pattern ~ '^\.' THEN
1865
+ RAISE EXCEPTION 'pattern cannot start with a dot. Got: %', pattern;
1866
+ END IF;
1867
+
1868
+ IF pattern ~ '\.$' THEN
1869
+ RAISE EXCEPTION 'pattern cannot end with a dot. Got: %', pattern;
1870
+ END IF;
1871
+
1872
+ IF pattern ~ '\.\.' THEN
1873
+ RAISE EXCEPTION 'pattern cannot contain consecutive dots. Got: %', pattern;
1874
+ END IF;
1875
+
1876
+ IF pattern ~ '\*\*' THEN
1877
+ RAISE EXCEPTION 'pattern cannot contain consecutive stars (**). Use # for multi-segment matching. Got: %', pattern;
1878
+ END IF;
1879
+
1880
+ IF pattern ~ '##' THEN
1881
+ RAISE EXCEPTION 'pattern cannot contain consecutive hashes (##). A single # already matches zero or more segments. Got: %', pattern;
1882
+ END IF;
1883
+
1884
+ IF pattern ~ '\*#' OR pattern ~ '#\*' THEN
1885
+ RAISE EXCEPTION 'pattern cannot contain adjacent wildcards (*# or #*). Separate wildcards with dots. Got: %', pattern;
1886
+ END IF;
1887
+
1888
+ RETURN true;
1889
+ END;
1890
+ $$;
1891
+
1892
+ CREATE OR REPLACE FUNCTION pgmq.bind_topic(pattern text, queue_name text)
1893
+ RETURNS void
1894
+ LANGUAGE plpgsql
1895
+ AS
1896
+ $$
1897
+ BEGIN
1898
+ PERFORM pgmq.validate_topic_pattern(pattern);
1899
+ IF queue_name IS NULL OR queue_name = '' THEN
1900
+ RAISE EXCEPTION 'queue_name cannot be NULL or empty';
1901
+ END IF;
1902
+
1903
+ IF NOT EXISTS (SELECT 1 FROM pgmq.meta WHERE meta.queue_name = bind_topic.queue_name) THEN
1904
+ RAISE EXCEPTION 'Queue "%" does not exist. Create the queue first using pgmq.create()', queue_name;
1905
+ END IF;
1906
+
1907
+ INSERT INTO pgmq.topic_bindings (pattern, queue_name)
1908
+ VALUES (pattern, queue_name)
1909
+ ON CONFLICT ON CONSTRAINT topic_bindings_unique_pattern_queue DO NOTHING;
1910
+ END;
1911
+ $$;
1912
+
1913
+ CREATE OR REPLACE FUNCTION pgmq.unbind_topic(pattern text, queue_name text)
1914
+ RETURNS boolean
1915
+ LANGUAGE plpgsql
1916
+ AS
1917
+ $$
1918
+ DECLARE
1919
+ rows_deleted integer;
1920
+ BEGIN
1921
+ IF pattern IS NULL OR pattern = '' THEN
1922
+ RAISE EXCEPTION 'pattern cannot be NULL or empty';
1923
+ END IF;
1924
+
1925
+ IF queue_name IS NULL OR queue_name = '' THEN
1926
+ RAISE EXCEPTION 'queue_name cannot be NULL or empty';
1927
+ END IF;
1928
+
1929
+ DELETE
1930
+ FROM pgmq.topic_bindings
1931
+ WHERE topic_bindings.pattern = unbind_topic.pattern
1932
+ AND topic_bindings.queue_name = unbind_topic.queue_name;
1933
+
1934
+ GET DIAGNOSTICS rows_deleted = ROW_COUNT;
1935
+
1936
+ IF rows_deleted > 0 THEN
1937
+ RETURN true;
1938
+ ELSE
1939
+ RETURN false;
1940
+ END IF;
1941
+ END;
1942
+ $$;
1943
+
1944
+ CREATE OR REPLACE FUNCTION pgmq.test_routing(routing_key text)
1945
+ RETURNS TABLE
1946
+ (
1947
+ pattern text,
1948
+ queue_name text,
1949
+ compiled_regex text
1950
+ )
1951
+ LANGUAGE plpgsql
1952
+ STABLE
1953
+ AS
1954
+ $$
1955
+ BEGIN
1956
+ PERFORM pgmq.validate_routing_key(routing_key);
1957
+ RETURN QUERY
1958
+ SELECT b.pattern,
1959
+ b.queue_name,
1960
+ b.compiled_regex
1961
+ FROM pgmq.topic_bindings b
1962
+ WHERE routing_key ~ b.compiled_regex
1963
+ ORDER BY b.pattern;
1964
+ END;
1965
+ $$;
1966
+
1967
+ CREATE OR REPLACE FUNCTION pgmq.send_topic(routing_key text, msg jsonb, headers jsonb, delay integer)
1968
+ RETURNS integer
1969
+ LANGUAGE plpgsql
1970
+ VOLATILE
1971
+ AS
1972
+ $$
1973
+ DECLARE
1974
+ b RECORD;
1975
+ matched_count integer := 0;
1976
+ BEGIN
1977
+ PERFORM pgmq.validate_routing_key(routing_key);
1978
+
1979
+ IF msg IS NULL THEN
1980
+ RAISE EXCEPTION 'msg cannot be NULL';
1981
+ END IF;
1982
+
1983
+ IF delay < 0 THEN
1984
+ RAISE EXCEPTION 'delay cannot be negative, got: %', delay;
1985
+ END IF;
1986
+
1987
+ -- Filter matching patterns in SQL for better performance (uses index)
1988
+ -- Any failure will rollback the entire transaction
1989
+ FOR b IN
1990
+ SELECT DISTINCT tb.queue_name
1991
+ FROM pgmq.topic_bindings tb
1992
+ WHERE routing_key ~ tb.compiled_regex
1993
+ ORDER BY tb.queue_name -- Deterministic ordering, deduplicated by queue_name
1994
+ LOOP
1995
+ PERFORM pgmq.send(b.queue_name, msg, headers, delay);
1996
+ matched_count := matched_count + 1;
1997
+ END LOOP;
1998
+
1999
+ RETURN matched_count;
2000
+ END;
2001
+ $$;
2002
+
2003
+ CREATE OR REPLACE FUNCTION pgmq.send_topic(routing_key text, msg jsonb)
2004
+ RETURNS integer
2005
+ LANGUAGE plpgsql
2006
+ VOLATILE
2007
+ AS
2008
+ $$
2009
+ BEGIN
2010
+ RETURN pgmq.send_topic(routing_key, msg, NULL, 0);
2011
+ END;
2012
+ $$;
2013
+
2014
+ CREATE OR REPLACE FUNCTION pgmq.send_topic(routing_key text, msg jsonb, delay integer)
2015
+ RETURNS integer
2016
+ LANGUAGE plpgsql
2017
+ VOLATILE
2018
+ AS
2019
+ $$
2020
+ BEGIN
2021
+ RETURN pgmq.send_topic(routing_key, msg, NULL, delay);
2022
+ END;
2023
+ $$;
2024
+
2025
+ CREATE OR REPLACE FUNCTION pgmq.list_topic_bindings()
2026
+ RETURNS TABLE
2027
+ (
2028
+ pattern text,
2029
+ queue_name text,
2030
+ bound_at TIMESTAMP WITH TIME ZONE,
2031
+ compiled_regex text
2032
+ )
2033
+ LANGUAGE sql
2034
+ STABLE
2035
+ AS
2036
+ $$
2037
+ SELECT pattern, queue_name, bound_at, compiled_regex
2038
+ FROM pgmq.topic_bindings
2039
+ ORDER BY bound_at DESC, pattern, queue_name;
2040
+ $$;
2041
+
2042
+ CREATE OR REPLACE FUNCTION pgmq.list_topic_bindings(queue_name text)
2043
+ RETURNS TABLE
2044
+ (
2045
+ pattern text,
2046
+ queue_name text,
2047
+ bound_at TIMESTAMP WITH TIME ZONE,
2048
+ compiled_regex text
2049
+ )
2050
+ LANGUAGE sql
2051
+ STABLE
2052
+ AS
2053
+ $$
2054
+ SELECT pattern, tb.queue_name, bound_at, compiled_regex
2055
+ FROM pgmq.topic_bindings tb
2056
+ WHERE tb.queue_name = list_topic_bindings.queue_name
2057
+ ORDER BY bound_at DESC, pattern;
2058
+ $$;
2059
+
2060
+ -- send_batch_topic: Base implementation with TIMESTAMP WITH TIME ZONE delay
2061
+ CREATE OR REPLACE FUNCTION pgmq.send_batch_topic(
2062
+ routing_key text,
2063
+ msgs jsonb[],
2064
+ headers jsonb[],
2065
+ delay TIMESTAMP WITH TIME ZONE
2066
+ )
2067
+ RETURNS TABLE(queue_name text, msg_id bigint)
2068
+ LANGUAGE plpgsql
2069
+ VOLATILE
2070
+ AS
2071
+ $$
2072
+ DECLARE
2073
+ b RECORD;
2074
+ BEGIN
2075
+ PERFORM pgmq.validate_routing_key(routing_key);
2076
+
2077
+ -- Validate batch parameters once (not per queue)
2078
+ PERFORM pgmq._validate_batch_params(msgs, headers);
2079
+
2080
+ -- Filter matching patterns in SQL for better performance (uses index)
2081
+ -- Any failure will rollback the entire transaction
2082
+ FOR b IN
2083
+ SELECT DISTINCT tb.queue_name
2084
+ FROM pgmq.topic_bindings tb
2085
+ WHERE routing_key ~ tb.compiled_regex
2086
+ ORDER BY tb.queue_name -- Deterministic ordering, deduplicated by queue_name
2087
+ LOOP
2088
+ -- Use private _send_batch to avoid redundant validation
2089
+ RETURN QUERY
2090
+ SELECT b.queue_name, batch_result.msg_id
2091
+ FROM pgmq._send_batch(b.queue_name, msgs, headers, delay) AS batch_result(msg_id);
2092
+ END LOOP;
2093
+
2094
+ RETURN;
2095
+ END;
2096
+ $$;
2097
+
2098
+ -- send_batch_topic: 2 args (routing_key, msgs)
2099
+ CREATE OR REPLACE FUNCTION pgmq.send_batch_topic(
2100
+ routing_key text,
2101
+ msgs jsonb[]
2102
+ )
2103
+ RETURNS TABLE(queue_name text, msg_id bigint)
2104
+ LANGUAGE sql
2105
+ VOLATILE
2106
+ AS
2107
+ $$
2108
+ SELECT * FROM pgmq.send_batch_topic(routing_key, msgs, NULL, clock_timestamp());
2109
+ $$;
2110
+
2111
+ -- send_batch_topic: 3 args with headers
2112
+ CREATE OR REPLACE FUNCTION pgmq.send_batch_topic(
2113
+ routing_key text,
2114
+ msgs jsonb[],
2115
+ headers jsonb[]
2116
+ )
2117
+ RETURNS TABLE(queue_name text, msg_id bigint)
2118
+ LANGUAGE sql
2119
+ VOLATILE
2120
+ AS
2121
+ $$
2122
+ SELECT * FROM pgmq.send_batch_topic(routing_key, msgs, headers, clock_timestamp());
2123
+ $$;
2124
+
2125
+ -- send_batch_topic: 3 args with integer delay
2126
+ CREATE OR REPLACE FUNCTION pgmq.send_batch_topic(
2127
+ routing_key text,
2128
+ msgs jsonb[],
2129
+ delay integer
2130
+ )
2131
+ RETURNS TABLE(queue_name text, msg_id bigint)
2132
+ LANGUAGE sql
2133
+ VOLATILE
2134
+ AS
2135
+ $$
2136
+ SELECT * FROM pgmq.send_batch_topic(routing_key, msgs, NULL, clock_timestamp() + make_interval(secs => delay));
2137
+ $$;
2138
+
2139
+ -- send_batch_topic: 3 args with timestamp delay
2140
+ CREATE OR REPLACE FUNCTION pgmq.send_batch_topic(
2141
+ routing_key text,
2142
+ msgs jsonb[],
2143
+ delay TIMESTAMP WITH TIME ZONE
2144
+ )
2145
+ RETURNS TABLE(queue_name text, msg_id bigint)
2146
+ LANGUAGE sql
2147
+ VOLATILE
2148
+ AS
2149
+ $$
2150
+ SELECT * FROM pgmq.send_batch_topic(routing_key, msgs, NULL, delay);
2151
+ $$;
2152
+
2153
+ -- send_batch_topic: 4 args with integer delay
2154
+ CREATE OR REPLACE FUNCTION pgmq.send_batch_topic(
2155
+ routing_key text,
2156
+ msgs jsonb[],
2157
+ headers jsonb[],
2158
+ delay integer
2159
+ )
2160
+ RETURNS TABLE(queue_name text, msg_id bigint)
2161
+ LANGUAGE sql
2162
+ VOLATILE
2163
+ AS
2164
+ $$
2165
+ SELECT * FROM pgmq.send_batch_topic(routing_key, msgs, headers, clock_timestamp() + make_interval(secs => delay));
2166
+ $$;