pgbus 0.0.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +37 -3
- data/Rakefile +98 -1
- data/app/controllers/pgbus/application_controller.rb +8 -0
- data/app/controllers/pgbus/recurring_tasks_controller.rb +36 -0
- data/app/helpers/pgbus/application_helper.rb +41 -0
- data/app/models/pgbus/application_record.rb +7 -0
- data/app/models/pgbus/batch_entry.rb +31 -0
- data/app/models/pgbus/blocked_execution.rb +40 -0
- data/app/models/pgbus/process_entry.rb +9 -0
- data/app/models/pgbus/processed_event.rb +9 -0
- data/app/models/pgbus/recurring_execution.rb +33 -0
- data/app/models/pgbus/recurring_task.rb +42 -0
- data/app/models/pgbus/semaphore.rb +29 -0
- data/app/views/layouts/pgbus/application.html.erb +1 -0
- data/app/views/pgbus/dashboard/_stats_cards.html.erb +9 -1
- data/app/views/pgbus/dead_letter/_messages_table.html.erb +55 -18
- data/app/views/pgbus/jobs/_enqueued_table.html.erb +46 -8
- data/app/views/pgbus/recurring_tasks/_tasks_table.html.erb +79 -0
- data/app/views/pgbus/recurring_tasks/index.html.erb +6 -0
- data/app/views/pgbus/recurring_tasks/show.html.erb +122 -0
- data/config/routes.rb +7 -0
- data/lib/active_job/queue_adapters/pgbus_adapter.rb +29 -0
- data/lib/generators/pgbus/add_recurring_generator.rb +56 -0
- data/lib/generators/pgbus/install_generator.rb +76 -2
- data/lib/generators/pgbus/templates/add_recurring_tables.rb.erb +31 -0
- data/lib/generators/pgbus/templates/migration.rb.erb +72 -4
- data/lib/generators/pgbus/templates/recurring.yml.erb +40 -0
- data/lib/generators/pgbus/templates/upgrade_pgmq.rb.erb +30 -0
- data/lib/generators/pgbus/upgrade_pgmq_generator.rb +60 -0
- data/lib/pgbus/active_job/adapter.rb +3 -6
- data/lib/pgbus/active_job/executor.rb +26 -12
- data/lib/pgbus/batch.rb +65 -72
- data/lib/pgbus/cli.rb +11 -16
- data/lib/pgbus/client.rb +32 -15
- data/lib/pgbus/concurrency/blocked_execution.rb +32 -37
- data/lib/pgbus/concurrency/semaphore.rb +11 -39
- data/lib/pgbus/concurrency.rb +10 -2
- data/lib/pgbus/configuration.rb +48 -0
- data/lib/pgbus/engine.rb +19 -1
- data/lib/pgbus/event_bus/handler.rb +10 -23
- data/lib/pgbus/instrumentation.rb +29 -0
- data/lib/pgbus/pgmq_schema/pgmq_v1.11.0.sql +2123 -0
- data/lib/pgbus/pgmq_schema.rb +159 -0
- data/lib/pgbus/process/consumer.rb +17 -9
- data/lib/pgbus/process/dispatcher.rb +33 -41
- data/lib/pgbus/process/heartbeat.rb +15 -23
- data/lib/pgbus/process/signal_handler.rb +23 -1
- data/lib/pgbus/process/supervisor.rb +79 -2
- data/lib/pgbus/process/worker.rb +42 -13
- data/lib/pgbus/recurring/already_recorded.rb +7 -0
- data/lib/pgbus/recurring/command_job.rb +28 -0
- data/lib/pgbus/recurring/config_loader.rb +35 -0
- data/lib/pgbus/recurring/schedule.rb +102 -0
- data/lib/pgbus/recurring/scheduler.rb +102 -0
- data/lib/pgbus/recurring/task.rb +111 -0
- data/lib/pgbus/serializer.rb +16 -6
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +217 -36
- data/lib/pgbus.rb +8 -0
- data/lib/tasks/pgbus_pgmq.rake +62 -0
- metadata +51 -24
- data/.bun-version +0 -1
- data/.claude/commands/architect.md +0 -100
- data/.claude/commands/github-review-comments.md +0 -237
- data/.claude/commands/lfg.md +0 -271
- data/.claude/commands/review-pr.md +0 -69
- data/.claude/commands/security.md +0 -122
- data/.claude/commands/tdd.md +0 -148
- data/.claude/rules/agents.md +0 -49
- data/.claude/rules/coding-style.md +0 -91
- data/.claude/rules/git-workflow.md +0 -56
- data/.claude/rules/performance.md +0 -73
- data/.claude/rules/testing.md +0 -67
- data/CLAUDE.md +0 -80
- data/CODE_OF_CONDUCT.md +0 -10
- data/bun.lock +0 -18
- data/docs/README.md +0 -28
- data/docs/switch_from_good_job.md +0 -279
- data/docs/switch_from_sidekiq.md +0 -226
- data/docs/switch_from_solid_queue.md +0 -247
- data/package.json +0 -9
- data/sig/pgbus.rbs +0 -4
|
@@ -0,0 +1,2123 @@
|
|
|
1
|
+
------------------------------------------------------------
|
|
2
|
+
-- Schema, tables, records, privileges, indexes, etc
|
|
3
|
+
------------------------------------------------------------
|
|
4
|
+
-- When installed as an extension, we don't need to create the `pgmq` schema
|
|
5
|
+
-- because it is automatically created by postgres due to being declared in
|
|
6
|
+
-- the extension control file
|
|
7
|
+
DO
|
|
8
|
+
$$
|
|
9
|
+
BEGIN
|
|
10
|
+
IF (SELECT NOT EXISTS( SELECT 1 FROM pg_extension WHERE extname = 'pgmq')) THEN
|
|
11
|
+
CREATE SCHEMA IF NOT EXISTS pgmq;
|
|
12
|
+
END IF;
|
|
13
|
+
END
|
|
14
|
+
$$;
|
|
15
|
+
|
|
16
|
+
-- Table where queues and metadata about them is stored
|
|
17
|
+
CREATE TABLE IF NOT EXISTS pgmq.meta (
|
|
18
|
+
queue_name VARCHAR UNIQUE NOT NULL,
|
|
19
|
+
is_partitioned BOOLEAN NOT NULL,
|
|
20
|
+
is_unlogged BOOLEAN NOT NULL,
|
|
21
|
+
created_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL
|
|
22
|
+
);
|
|
23
|
+
|
|
24
|
+
-- Table to track notification throttling for queues
|
|
25
|
+
CREATE UNLOGGED TABLE IF NOT EXISTS pgmq.notify_insert_throttle (
|
|
26
|
+
queue_name VARCHAR UNIQUE NOT NULL -- Queue name (without 'q_' prefix)
|
|
27
|
+
CONSTRAINT notify_insert_throttle_meta_queue_name_fk
|
|
28
|
+
REFERENCES pgmq.meta (queue_name)
|
|
29
|
+
ON DELETE CASCADE,
|
|
30
|
+
throttle_interval_ms INTEGER NOT NULL DEFAULT 0, -- Min milliseconds between notifications (0 = no throttling)
|
|
31
|
+
last_notified_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT to_timestamp(0) -- Timestamp of last sent notification
|
|
32
|
+
);
|
|
33
|
+
|
|
34
|
+
CREATE INDEX IF NOT EXISTS idx_notify_throttle_active
|
|
35
|
+
ON pgmq.notify_insert_throttle (queue_name, last_notified_at)
|
|
36
|
+
WHERE throttle_interval_ms > 0;
|
|
37
|
+
|
|
38
|
+
CREATE TABLE IF NOT EXISTS pgmq.topic_bindings
|
|
39
|
+
(
|
|
40
|
+
pattern text NOT NULL, -- Wildcard pattern for routing key matching (* = one segment, # = zero or more segments)
|
|
41
|
+
queue_name text NOT NULL -- Name of the queue that receives messages when pattern matches
|
|
42
|
+
CONSTRAINT topic_bindings_meta_queue_name_fk
|
|
43
|
+
REFERENCES pgmq.meta (queue_name)
|
|
44
|
+
ON DELETE CASCADE,
|
|
45
|
+
bound_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL, -- Timestamp when the binding was created
|
|
46
|
+
compiled_regex text GENERATED ALWAYS AS (
|
|
47
|
+
-- Pre-compile the pattern to regex for faster matching
|
|
48
|
+
-- This avoids runtime compilation on every send_topic call
|
|
49
|
+
'^' ||
|
|
50
|
+
replace(
|
|
51
|
+
replace(
|
|
52
|
+
regexp_replace(pattern, '([.+?{}()|\[\]\\^$])', '\\\1', 'g'),
|
|
53
|
+
'*', '[^.]+'
|
|
54
|
+
),
|
|
55
|
+
'#', '.*'
|
|
56
|
+
) || '$'
|
|
57
|
+
) STORED, -- Computed column: stores the compiled regex pattern
|
|
58
|
+
CONSTRAINT topic_bindings_unique_pattern_queue UNIQUE (pattern, queue_name)
|
|
59
|
+
);
|
|
60
|
+
|
|
61
|
+
-- Create covering index for better performance when scanning patterns
|
|
62
|
+
-- Includes queue_name and compiled_regex to allow index-only scans (no table access needed)
|
|
63
|
+
CREATE INDEX IF NOT EXISTS idx_topic_bindings_covering ON pgmq.topic_bindings (pattern) INCLUDE (queue_name, compiled_regex);
|
|
64
|
+
|
|
65
|
+
-- Allow pgmq.meta to be dumped by `pg_dump` when pgmq is installed as an extension
|
|
66
|
+
DO
|
|
67
|
+
$$
|
|
68
|
+
BEGIN
|
|
69
|
+
IF EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pgmq') THEN
|
|
70
|
+
PERFORM pg_catalog.pg_extension_config_dump('pgmq.meta', '');
|
|
71
|
+
PERFORM pg_catalog.pg_extension_config_dump('pgmq.notify_insert_throttle', '');
|
|
72
|
+
PERFORM pg_catalog.pg_extension_config_dump('pgmq.topic_bindings', '');
|
|
73
|
+
END IF;
|
|
74
|
+
END
|
|
75
|
+
$$;
|
|
76
|
+
|
|
77
|
+
-- Grant permission to pg_monitor to all tables and sequences
|
|
78
|
+
GRANT USAGE ON SCHEMA pgmq TO pg_monitor;
|
|
79
|
+
GRANT SELECT ON ALL TABLES IN SCHEMA pgmq TO pg_monitor;
|
|
80
|
+
GRANT SELECT ON ALL SEQUENCES IN SCHEMA pgmq TO pg_monitor;
|
|
81
|
+
ALTER DEFAULT PRIVILEGES IN SCHEMA pgmq GRANT SELECT ON TABLES TO pg_monitor;
|
|
82
|
+
ALTER DEFAULT PRIVILEGES IN SCHEMA pgmq GRANT SELECT ON SEQUENCES TO pg_monitor;
|
|
83
|
+
|
|
84
|
+
-- This type has the shape of a message in a queue, and is often returned by
|
|
85
|
+
-- pgmq functions that return messages
|
|
86
|
+
CREATE TYPE pgmq.message_record AS (
|
|
87
|
+
msg_id BIGINT,
|
|
88
|
+
read_ct INTEGER,
|
|
89
|
+
enqueued_at TIMESTAMP WITH TIME ZONE,
|
|
90
|
+
last_read_at TIMESTAMP WITH TIME ZONE,
|
|
91
|
+
vt TIMESTAMP WITH TIME ZONE,
|
|
92
|
+
message JSONB,
|
|
93
|
+
headers JSONB
|
|
94
|
+
);
|
|
95
|
+
|
|
96
|
+
CREATE TYPE pgmq.queue_record AS (
|
|
97
|
+
queue_name VARCHAR,
|
|
98
|
+
is_partitioned BOOLEAN,
|
|
99
|
+
is_unlogged BOOLEAN,
|
|
100
|
+
created_at TIMESTAMP WITH TIME ZONE
|
|
101
|
+
);
|
|
102
|
+
|
|
103
|
+
------------------------------------------------------------
|
|
104
|
+
-- Functions
|
|
105
|
+
------------------------------------------------------------
|
|
106
|
+
|
|
107
|
+
-- prevents race conditions during queue creation by acquiring a transaction-level advisory lock
|
|
108
|
+
-- uses a transaction advisory lock maintain the lock until transaction commit
|
|
109
|
+
-- a race condition would still exist if lock was released before commit
|
|
110
|
+
CREATE FUNCTION pgmq.acquire_queue_lock(queue_name TEXT)
|
|
111
|
+
RETURNS void AS $$
|
|
112
|
+
BEGIN
|
|
113
|
+
PERFORM pg_advisory_xact_lock(hashtext('pgmq.queue_' || queue_name));
|
|
114
|
+
END;
|
|
115
|
+
$$ LANGUAGE plpgsql;
|
|
116
|
+
|
|
117
|
+
-- read_grouped_round_robin
|
|
118
|
+
-- reads messages while preserving FIFO within groups and interleaving across groups (layered round-robin)
|
|
119
|
+
CREATE FUNCTION pgmq.read_grouped_rr(
|
|
120
|
+
queue_name TEXT,
|
|
121
|
+
vt INTEGER,
|
|
122
|
+
qty INTEGER
|
|
123
|
+
)
|
|
124
|
+
RETURNS SETOF pgmq.message_record AS $$
|
|
125
|
+
DECLARE
|
|
126
|
+
sql TEXT;
|
|
127
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
128
|
+
BEGIN
|
|
129
|
+
sql := FORMAT(
|
|
130
|
+
$QUERY$
|
|
131
|
+
WITH fifo_groups AS (
|
|
132
|
+
-- Determine the absolute head (oldest) message id per FIFO group, regardless of visibility
|
|
133
|
+
SELECT
|
|
134
|
+
COALESCE(headers->>'x-pgmq-group', '_default_fifo_group') AS fifo_key,
|
|
135
|
+
MIN(msg_id) AS head_msg_id
|
|
136
|
+
FROM pgmq.%1$I
|
|
137
|
+
GROUP BY COALESCE(headers->>'x-pgmq-group', '_default_fifo_group')
|
|
138
|
+
),
|
|
139
|
+
eligible_groups AS (
|
|
140
|
+
-- Only groups whose head message is currently visible
|
|
141
|
+
-- Acquire a transaction-level advisory lock per group to prevent concurrent selection
|
|
142
|
+
SELECT
|
|
143
|
+
g.fifo_key,
|
|
144
|
+
g.head_msg_id,
|
|
145
|
+
ROW_NUMBER() OVER (ORDER BY g.head_msg_id) AS group_priority
|
|
146
|
+
FROM fifo_groups g
|
|
147
|
+
JOIN pgmq.%2$I h ON h.msg_id = g.head_msg_id
|
|
148
|
+
WHERE h.vt <= clock_timestamp()
|
|
149
|
+
AND pg_try_advisory_xact_lock(pg_catalog.hashtextextended(g.fifo_key, 0))
|
|
150
|
+
),
|
|
151
|
+
available_messages AS (
|
|
152
|
+
-- All currently visible messages starting at the head for each eligible group
|
|
153
|
+
SELECT
|
|
154
|
+
m.msg_id,
|
|
155
|
+
eg.group_priority,
|
|
156
|
+
ROW_NUMBER() OVER (
|
|
157
|
+
PARTITION BY eg.fifo_key
|
|
158
|
+
ORDER BY m.msg_id
|
|
159
|
+
) AS msg_rank_in_group
|
|
160
|
+
FROM pgmq.%3$I m
|
|
161
|
+
JOIN eligible_groups eg
|
|
162
|
+
ON COALESCE(m.headers->>'x-pgmq-group', '_default_fifo_group') = eg.fifo_key
|
|
163
|
+
WHERE m.vt <= clock_timestamp()
|
|
164
|
+
AND m.msg_id >= eg.head_msg_id
|
|
165
|
+
),
|
|
166
|
+
ordered_messages AS (
|
|
167
|
+
-- Layered round-robin: take rank 1 of all groups by group_priority, then rank 2, etc.
|
|
168
|
+
-- Assign selection order before locking
|
|
169
|
+
SELECT msg_id, ROW_NUMBER() OVER (ORDER BY msg_rank_in_group, group_priority) as selection_order
|
|
170
|
+
FROM available_messages
|
|
171
|
+
),
|
|
172
|
+
selected_messages AS (
|
|
173
|
+
-- Lock the messages in the correct order, preserving selection_order
|
|
174
|
+
SELECT om.msg_id, om.selection_order
|
|
175
|
+
FROM ordered_messages om
|
|
176
|
+
JOIN pgmq.%4$I m ON m.msg_id = om.msg_id
|
|
177
|
+
WHERE om.selection_order <= $1
|
|
178
|
+
ORDER BY om.selection_order
|
|
179
|
+
FOR UPDATE OF m SKIP LOCKED
|
|
180
|
+
),
|
|
181
|
+
updated_messages AS (
|
|
182
|
+
UPDATE pgmq.%5$I m
|
|
183
|
+
SET
|
|
184
|
+
vt = clock_timestamp() + %6$L,
|
|
185
|
+
read_ct = read_ct + 1,
|
|
186
|
+
last_read_at = clock_timestamp()
|
|
187
|
+
FROM selected_messages sm
|
|
188
|
+
WHERE m.msg_id = sm.msg_id
|
|
189
|
+
AND m.vt <= clock_timestamp() -- final guard to avoid duplicate reads under races
|
|
190
|
+
RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.last_read_at, m.vt, m.message, m.headers, sm.selection_order
|
|
191
|
+
)
|
|
192
|
+
SELECT msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers
|
|
193
|
+
FROM updated_messages
|
|
194
|
+
ORDER BY selection_order;
|
|
195
|
+
$QUERY$,
|
|
196
|
+
qtable, qtable, qtable, qtable, qtable, make_interval(secs => vt)
|
|
197
|
+
);
|
|
198
|
+
RETURN QUERY EXECUTE sql USING qty;
|
|
199
|
+
END;
|
|
200
|
+
$$ LANGUAGE plpgsql;
|
|
201
|
+
|
|
202
|
+
-- read_grouped_rr_with_poll
|
|
203
|
+
-- reads messages using round-robin layering across groups, with polling support
|
|
204
|
+
CREATE FUNCTION pgmq.read_grouped_rr_with_poll(
|
|
205
|
+
queue_name TEXT,
|
|
206
|
+
vt INTEGER,
|
|
207
|
+
qty INTEGER,
|
|
208
|
+
max_poll_seconds INTEGER DEFAULT 5,
|
|
209
|
+
poll_interval_ms INTEGER DEFAULT 100
|
|
210
|
+
)
|
|
211
|
+
RETURNS SETOF pgmq.message_record AS $$
|
|
212
|
+
DECLARE
|
|
213
|
+
r pgmq.message_record;
|
|
214
|
+
stop_at TIMESTAMP;
|
|
215
|
+
BEGIN
|
|
216
|
+
stop_at := clock_timestamp() + make_interval(secs => max_poll_seconds);
|
|
217
|
+
LOOP
|
|
218
|
+
IF (SELECT clock_timestamp() >= stop_at) THEN
|
|
219
|
+
RETURN;
|
|
220
|
+
END IF;
|
|
221
|
+
|
|
222
|
+
FOR r IN
|
|
223
|
+
SELECT * FROM pgmq.read_grouped_rr(queue_name, vt, qty)
|
|
224
|
+
LOOP
|
|
225
|
+
RETURN NEXT r;
|
|
226
|
+
END LOOP;
|
|
227
|
+
IF FOUND THEN
|
|
228
|
+
RETURN;
|
|
229
|
+
ELSE
|
|
230
|
+
PERFORM pg_sleep(poll_interval_ms::numeric / 1000);
|
|
231
|
+
END IF;
|
|
232
|
+
END LOOP;
|
|
233
|
+
END;
|
|
234
|
+
$$ LANGUAGE plpgsql;
|
|
235
|
+
|
|
236
|
+
-- read_grouped_head: read the head of N different FIFO groups in a single operation.
|
|
237
|
+
-- This supports horizontal scaling by processing groups in parallel while ensuring message ordering is preserved per group.
|
|
238
|
+
CREATE FUNCTION pgmq.read_grouped_head(
|
|
239
|
+
queue_name TEXT,
|
|
240
|
+
vt INTEGER,
|
|
241
|
+
qty INTEGER
|
|
242
|
+
)
|
|
243
|
+
RETURNS SETOF pgmq.message_record AS $$
|
|
244
|
+
DECLARE
|
|
245
|
+
sql TEXT;
|
|
246
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
247
|
+
BEGIN
|
|
248
|
+
sql := FORMAT(
|
|
249
|
+
$QUERY$
|
|
250
|
+
WITH fifo_groups AS (
|
|
251
|
+
-- Determine the absolute head (oldest) message id per FIFO group, regardless of visibility
|
|
252
|
+
SELECT
|
|
253
|
+
COALESCE(headers->>'x-pgmq-group', '_default_fifo_group') AS fifo_key,
|
|
254
|
+
MIN(msg_id) AS head_msg_id
|
|
255
|
+
FROM pgmq.%1$I
|
|
256
|
+
GROUP BY COALESCE(headers->>'x-pgmq-group', '_default_fifo_group')
|
|
257
|
+
),
|
|
258
|
+
selected_messages AS (
|
|
259
|
+
-- Take at most 1 message per group
|
|
260
|
+
SELECT g.head_msg_id msg_id
|
|
261
|
+
FROM fifo_groups g
|
|
262
|
+
JOIN pgmq.%1$I q ON q.msg_id = g.head_msg_id
|
|
263
|
+
WHERE q.vt <= clock_timestamp()
|
|
264
|
+
ORDER BY q.msg_id
|
|
265
|
+
LIMIT $1
|
|
266
|
+
FOR UPDATE SKIP LOCKED
|
|
267
|
+
)
|
|
268
|
+
UPDATE pgmq.%1$I m
|
|
269
|
+
SET
|
|
270
|
+
vt = clock_timestamp() + %2$L,
|
|
271
|
+
read_ct = read_ct + 1,
|
|
272
|
+
last_read_at = clock_timestamp()
|
|
273
|
+
FROM selected_messages sm
|
|
274
|
+
WHERE m.msg_id = sm.msg_id
|
|
275
|
+
RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.last_read_at, m.vt, m.message, m.headers;
|
|
276
|
+
$QUERY$,
|
|
277
|
+
qtable, make_interval(secs => vt)
|
|
278
|
+
);
|
|
279
|
+
RETURN QUERY EXECUTE sql USING qty;
|
|
280
|
+
END;
|
|
281
|
+
$$ LANGUAGE plpgsql;
|
|
282
|
+
|
|
283
|
+
-- a helper to format table names and check for invalid characters
|
|
284
|
+
CREATE FUNCTION pgmq.format_table_name(queue_name text, prefix text)
|
|
285
|
+
RETURNS TEXT AS $$
|
|
286
|
+
BEGIN
|
|
287
|
+
IF queue_name ~ '\$|;|--|'''
|
|
288
|
+
THEN
|
|
289
|
+
RAISE EXCEPTION 'queue name contains invalid characters: $, ;, --, or \''';
|
|
290
|
+
END IF;
|
|
291
|
+
RETURN lower(prefix || '_' || queue_name);
|
|
292
|
+
END;
|
|
293
|
+
$$ LANGUAGE plpgsql;
|
|
294
|
+
|
|
295
|
+
-- read
|
|
296
|
+
-- reads a number of messages from a queue, setting a visibility timeout on them
|
|
297
|
+
CREATE FUNCTION pgmq.read(
|
|
298
|
+
queue_name TEXT,
|
|
299
|
+
vt INTEGER,
|
|
300
|
+
qty INTEGER,
|
|
301
|
+
conditional JSONB DEFAULT '{}'
|
|
302
|
+
)
|
|
303
|
+
RETURNS SETOF pgmq.message_record AS $$
|
|
304
|
+
DECLARE
|
|
305
|
+
sql TEXT;
|
|
306
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
307
|
+
BEGIN
|
|
308
|
+
sql := FORMAT(
|
|
309
|
+
$QUERY$
|
|
310
|
+
WITH cte AS
|
|
311
|
+
(
|
|
312
|
+
SELECT msg_id
|
|
313
|
+
FROM pgmq.%I
|
|
314
|
+
WHERE vt <= clock_timestamp() AND CASE
|
|
315
|
+
WHEN %L != '{}'::jsonb THEN (message @> %2$L)::integer
|
|
316
|
+
ELSE 1
|
|
317
|
+
END = 1
|
|
318
|
+
ORDER BY msg_id ASC
|
|
319
|
+
LIMIT $1
|
|
320
|
+
FOR UPDATE SKIP LOCKED
|
|
321
|
+
)
|
|
322
|
+
UPDATE pgmq.%I m
|
|
323
|
+
SET
|
|
324
|
+
last_read_at = clock_timestamp(),
|
|
325
|
+
vt = clock_timestamp() + %L,
|
|
326
|
+
read_ct = read_ct + 1
|
|
327
|
+
FROM cte
|
|
328
|
+
WHERE m.msg_id = cte.msg_id
|
|
329
|
+
RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.last_read_at, m.vt, m.message, m.headers;
|
|
330
|
+
$QUERY$,
|
|
331
|
+
qtable, conditional, qtable, make_interval(secs => vt)
|
|
332
|
+
);
|
|
333
|
+
RETURN QUERY EXECUTE sql USING qty;
|
|
334
|
+
END;
|
|
335
|
+
$$ LANGUAGE plpgsql;
|
|
336
|
+
|
|
337
|
+
-- read_grouped
|
|
338
|
+
-- reads messages with AWS SQS FIFO-style batch retrieval behavior
|
|
339
|
+
-- attempts to return as many messages as possible from the same message group
|
|
340
|
+
CREATE FUNCTION pgmq.read_grouped(
|
|
341
|
+
queue_name TEXT,
|
|
342
|
+
vt INTEGER,
|
|
343
|
+
qty INTEGER
|
|
344
|
+
)
|
|
345
|
+
RETURNS SETOF pgmq.message_record AS $$
|
|
346
|
+
DECLARE
|
|
347
|
+
sql TEXT;
|
|
348
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
349
|
+
BEGIN
|
|
350
|
+
sql := FORMAT(
|
|
351
|
+
$QUERY$
|
|
352
|
+
WITH fifo_groups AS (
|
|
353
|
+
-- Find the minimum msg_id for each FIFO group that's ready to be processed
|
|
354
|
+
SELECT
|
|
355
|
+
COALESCE(headers->>'x-pgmq-group', '_default_fifo_group') as fifo_key,
|
|
356
|
+
MIN(msg_id) as min_msg_id
|
|
357
|
+
FROM pgmq.%I
|
|
358
|
+
WHERE vt <= clock_timestamp()
|
|
359
|
+
GROUP BY COALESCE(headers->>'x-pgmq-group', '_default_fifo_group')
|
|
360
|
+
),
|
|
361
|
+
locked_groups AS (
|
|
362
|
+
-- Lock the first available message in each FIFO group
|
|
363
|
+
SELECT
|
|
364
|
+
m.msg_id,
|
|
365
|
+
fg.fifo_key
|
|
366
|
+
FROM pgmq.%I m
|
|
367
|
+
INNER JOIN fifo_groups fg ON
|
|
368
|
+
COALESCE(m.headers->>'x-pgmq-group', '_default_fifo_group') = fg.fifo_key
|
|
369
|
+
AND m.msg_id = fg.min_msg_id
|
|
370
|
+
WHERE m.vt <= clock_timestamp()
|
|
371
|
+
ORDER BY m.msg_id ASC
|
|
372
|
+
FOR UPDATE SKIP LOCKED
|
|
373
|
+
),
|
|
374
|
+
group_priorities AS (
|
|
375
|
+
-- Assign priority to groups based on their oldest message
|
|
376
|
+
SELECT
|
|
377
|
+
fifo_key,
|
|
378
|
+
msg_id as min_msg_id,
|
|
379
|
+
ROW_NUMBER() OVER (ORDER BY msg_id) as group_priority
|
|
380
|
+
FROM locked_groups
|
|
381
|
+
),
|
|
382
|
+
filtered_groups as (
|
|
383
|
+
SELECT * FROM group_priorities gp
|
|
384
|
+
WHERE NOT EXISTS (
|
|
385
|
+
-- Ensure no earlier message in this group is currently being processed
|
|
386
|
+
SELECT 1
|
|
387
|
+
FROM pgmq.%I m2
|
|
388
|
+
WHERE COALESCE(m2.headers->>'x-pgmq-group', '_default_fifo_group') = gp.fifo_key
|
|
389
|
+
AND m2.vt > clock_timestamp()
|
|
390
|
+
AND m2.msg_id < gp.min_msg_id
|
|
391
|
+
)
|
|
392
|
+
),
|
|
393
|
+
available_messages as (
|
|
394
|
+
SELECT gp.fifo_key, t.msg_id,gp.group_priority,
|
|
395
|
+
ROW_NUMBER() OVER (PARTITION BY gp.fifo_key ORDER BY t.msg_id) as msg_rank_in_group
|
|
396
|
+
FROM filtered_groups gp
|
|
397
|
+
CROSS JOIN LATERAL (
|
|
398
|
+
SELECT *
|
|
399
|
+
FROM pgmq.%I t
|
|
400
|
+
WHERE COALESCE(t.headers->>'x-pgmq-group', '_default_fifo_group') = gp.fifo_key
|
|
401
|
+
AND t.vt <= clock_timestamp()
|
|
402
|
+
ORDER BY msg_id
|
|
403
|
+
LIMIT $1 -- tip to limit query impact, we know we need at most qty in each group
|
|
404
|
+
) t
|
|
405
|
+
ORDER BY gp.group_priority
|
|
406
|
+
),
|
|
407
|
+
batch_selection AS (
|
|
408
|
+
-- Select messages to fill batch, prioritizing earliest group
|
|
409
|
+
SELECT
|
|
410
|
+
msg_id,
|
|
411
|
+
ROW_NUMBER() OVER (ORDER BY group_priority, msg_rank_in_group) as overall_rank
|
|
412
|
+
FROM available_messages
|
|
413
|
+
),
|
|
414
|
+
selected_messages AS (
|
|
415
|
+
-- Limit to requested quantity
|
|
416
|
+
SELECT msg_id
|
|
417
|
+
FROM batch_selection
|
|
418
|
+
WHERE overall_rank <= $1
|
|
419
|
+
ORDER BY msg_id
|
|
420
|
+
FOR UPDATE SKIP LOCKED
|
|
421
|
+
)
|
|
422
|
+
UPDATE pgmq.%I m
|
|
423
|
+
SET
|
|
424
|
+
vt = clock_timestamp() + %L,
|
|
425
|
+
read_ct = read_ct + 1,
|
|
426
|
+
last_read_at = clock_timestamp()
|
|
427
|
+
FROM selected_messages sm
|
|
428
|
+
WHERE m.msg_id = sm.msg_id
|
|
429
|
+
RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.last_read_at, m.vt, m.message, m.headers;
|
|
430
|
+
$QUERY$,
|
|
431
|
+
qtable, qtable, qtable, qtable, qtable, make_interval(secs => vt)
|
|
432
|
+
);
|
|
433
|
+
RETURN QUERY EXECUTE sql USING qty;
|
|
434
|
+
END;
|
|
435
|
+
$$ LANGUAGE plpgsql;
|
|
436
|
+
|
|
437
|
+
-- read_grouped_with_poll
|
|
438
|
+
-- reads messages with AWS SQS FIFO-style batch retrieval behavior, with polling support
|
|
439
|
+
CREATE FUNCTION pgmq.read_grouped_with_poll(
|
|
440
|
+
queue_name TEXT,
|
|
441
|
+
vt INTEGER,
|
|
442
|
+
qty INTEGER,
|
|
443
|
+
max_poll_seconds INTEGER DEFAULT 5,
|
|
444
|
+
poll_interval_ms INTEGER DEFAULT 100
|
|
445
|
+
)
|
|
446
|
+
RETURNS SETOF pgmq.message_record AS $$
|
|
447
|
+
DECLARE
|
|
448
|
+
r pgmq.message_record;
|
|
449
|
+
stop_at TIMESTAMP;
|
|
450
|
+
BEGIN
|
|
451
|
+
stop_at := clock_timestamp() + make_interval(secs => max_poll_seconds);
|
|
452
|
+
LOOP
|
|
453
|
+
IF (SELECT clock_timestamp() >= stop_at) THEN
|
|
454
|
+
RETURN;
|
|
455
|
+
END IF;
|
|
456
|
+
|
|
457
|
+
FOR r IN
|
|
458
|
+
SELECT * FROM pgmq.read_grouped(queue_name, vt, qty)
|
|
459
|
+
LOOP
|
|
460
|
+
RETURN NEXT r;
|
|
461
|
+
END LOOP;
|
|
462
|
+
IF FOUND THEN
|
|
463
|
+
RETURN;
|
|
464
|
+
ELSE
|
|
465
|
+
PERFORM pg_sleep(poll_interval_ms::numeric / 1000);
|
|
466
|
+
END IF;
|
|
467
|
+
END LOOP;
|
|
468
|
+
END;
|
|
469
|
+
$$ LANGUAGE plpgsql;
|
|
470
|
+
|
|
471
|
+
---- read_with_poll
|
|
472
|
+
---- reads a number of messages from a queue, setting a visibility timeout on them
|
|
473
|
+
CREATE FUNCTION pgmq.read_with_poll(
|
|
474
|
+
queue_name TEXT,
|
|
475
|
+
vt INTEGER,
|
|
476
|
+
qty INTEGER,
|
|
477
|
+
max_poll_seconds INTEGER DEFAULT 5,
|
|
478
|
+
poll_interval_ms INTEGER DEFAULT 100,
|
|
479
|
+
conditional JSONB DEFAULT '{}'
|
|
480
|
+
)
|
|
481
|
+
RETURNS SETOF pgmq.message_record AS $$
|
|
482
|
+
DECLARE
|
|
483
|
+
r pgmq.message_record;
|
|
484
|
+
stop_at TIMESTAMP;
|
|
485
|
+
sql TEXT;
|
|
486
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
487
|
+
BEGIN
|
|
488
|
+
stop_at := clock_timestamp() + make_interval(secs => max_poll_seconds);
|
|
489
|
+
LOOP
|
|
490
|
+
IF (SELECT clock_timestamp() >= stop_at) THEN
|
|
491
|
+
RETURN;
|
|
492
|
+
END IF;
|
|
493
|
+
|
|
494
|
+
sql := FORMAT(
|
|
495
|
+
$QUERY$
|
|
496
|
+
WITH cte AS
|
|
497
|
+
(
|
|
498
|
+
SELECT msg_id
|
|
499
|
+
FROM pgmq.%I
|
|
500
|
+
WHERE vt <= clock_timestamp() AND CASE
|
|
501
|
+
WHEN %L != '{}'::jsonb THEN (message @> %2$L)::integer
|
|
502
|
+
ELSE 1
|
|
503
|
+
END = 1
|
|
504
|
+
ORDER BY msg_id ASC
|
|
505
|
+
LIMIT $1
|
|
506
|
+
FOR UPDATE SKIP LOCKED
|
|
507
|
+
)
|
|
508
|
+
UPDATE pgmq.%I m
|
|
509
|
+
SET
|
|
510
|
+
last_read_at = clock_timestamp(),
|
|
511
|
+
vt = clock_timestamp() + %L,
|
|
512
|
+
read_ct = read_ct + 1
|
|
513
|
+
FROM cte
|
|
514
|
+
WHERE m.msg_id = cte.msg_id
|
|
515
|
+
RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.last_read_at, m.vt, m.message, m.headers;
|
|
516
|
+
$QUERY$,
|
|
517
|
+
qtable, conditional, qtable, make_interval(secs => vt)
|
|
518
|
+
);
|
|
519
|
+
|
|
520
|
+
FOR r IN
|
|
521
|
+
EXECUTE sql USING qty
|
|
522
|
+
LOOP
|
|
523
|
+
RETURN NEXT r;
|
|
524
|
+
END LOOP;
|
|
525
|
+
IF FOUND THEN
|
|
526
|
+
RETURN;
|
|
527
|
+
ELSE
|
|
528
|
+
PERFORM pg_sleep(poll_interval_ms::numeric / 1000);
|
|
529
|
+
END IF;
|
|
530
|
+
END LOOP;
|
|
531
|
+
END;
|
|
532
|
+
$$ LANGUAGE plpgsql;
|
|
533
|
+
|
|
534
|
+
---- archive
|
|
535
|
+
---- removes a message from the queue, and sends it to the archive, where its
|
|
536
|
+
---- saved permanently.
|
|
537
|
+
CREATE FUNCTION pgmq.archive(
|
|
538
|
+
queue_name TEXT,
|
|
539
|
+
msg_id BIGINT
|
|
540
|
+
)
|
|
541
|
+
RETURNS BOOLEAN AS $$
|
|
542
|
+
DECLARE
|
|
543
|
+
sql TEXT;
|
|
544
|
+
result BIGINT;
|
|
545
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
546
|
+
atable TEXT := pgmq.format_table_name(queue_name, 'a');
|
|
547
|
+
BEGIN
|
|
548
|
+
sql := FORMAT(
|
|
549
|
+
$QUERY$
|
|
550
|
+
WITH archived AS (
|
|
551
|
+
DELETE FROM pgmq.%I
|
|
552
|
+
WHERE msg_id = $1
|
|
553
|
+
RETURNING msg_id, vt, read_ct, enqueued_at, last_read_at, message, headers
|
|
554
|
+
)
|
|
555
|
+
INSERT INTO pgmq.%I (msg_id, vt, read_ct, enqueued_at, last_read_at, message, headers)
|
|
556
|
+
SELECT msg_id, vt, read_ct, enqueued_at, last_read_at, message, headers
|
|
557
|
+
FROM archived
|
|
558
|
+
RETURNING msg_id;
|
|
559
|
+
$QUERY$,
|
|
560
|
+
qtable, atable
|
|
561
|
+
);
|
|
562
|
+
EXECUTE sql USING msg_id INTO result;
|
|
563
|
+
RETURN NOT (result IS NULL);
|
|
564
|
+
END;
|
|
565
|
+
$$ LANGUAGE plpgsql;
|
|
566
|
+
|
|
567
|
+
---- archive
|
|
568
|
+
---- removes an array of message ids from the queue, and sends it to the archive,
|
|
569
|
+
---- where these messages will be saved permanently.
|
|
570
|
+
CREATE FUNCTION pgmq.archive(
|
|
571
|
+
queue_name TEXT,
|
|
572
|
+
msg_ids BIGINT[]
|
|
573
|
+
)
|
|
574
|
+
RETURNS SETOF BIGINT AS $$
|
|
575
|
+
DECLARE
|
|
576
|
+
sql TEXT;
|
|
577
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
578
|
+
atable TEXT := pgmq.format_table_name(queue_name, 'a');
|
|
579
|
+
BEGIN
|
|
580
|
+
sql := FORMAT(
|
|
581
|
+
$QUERY$
|
|
582
|
+
WITH archived AS (
|
|
583
|
+
DELETE FROM pgmq.%I
|
|
584
|
+
WHERE msg_id = ANY($1)
|
|
585
|
+
RETURNING msg_id, vt, read_ct, enqueued_at, last_read_at, message, headers
|
|
586
|
+
)
|
|
587
|
+
INSERT INTO pgmq.%I (msg_id, vt, read_ct, enqueued_at, last_read_at, message, headers)
|
|
588
|
+
SELECT msg_id, vt, read_ct, enqueued_at, last_read_at, message, headers
|
|
589
|
+
FROM archived
|
|
590
|
+
RETURNING msg_id;
|
|
591
|
+
$QUERY$,
|
|
592
|
+
qtable, atable
|
|
593
|
+
);
|
|
594
|
+
RETURN QUERY EXECUTE sql USING msg_ids;
|
|
595
|
+
END;
|
|
596
|
+
$$ LANGUAGE plpgsql;
|
|
597
|
+
|
|
598
|
+
---- delete
|
|
599
|
+
---- deletes a message id from the queue permanently
|
|
600
|
+
CREATE FUNCTION pgmq.delete(
|
|
601
|
+
queue_name TEXT,
|
|
602
|
+
msg_id BIGINT
|
|
603
|
+
)
|
|
604
|
+
RETURNS BOOLEAN AS $$
|
|
605
|
+
DECLARE
|
|
606
|
+
sql TEXT;
|
|
607
|
+
result BIGINT;
|
|
608
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
609
|
+
BEGIN
|
|
610
|
+
sql := FORMAT(
|
|
611
|
+
$QUERY$
|
|
612
|
+
DELETE FROM pgmq.%I
|
|
613
|
+
WHERE msg_id = $1
|
|
614
|
+
RETURNING msg_id
|
|
615
|
+
$QUERY$,
|
|
616
|
+
qtable
|
|
617
|
+
);
|
|
618
|
+
EXECUTE sql USING msg_id INTO result;
|
|
619
|
+
RETURN NOT (result IS NULL);
|
|
620
|
+
END;
|
|
621
|
+
$$ LANGUAGE plpgsql;
|
|
622
|
+
|
|
623
|
+
---- delete
|
|
624
|
+
---- deletes an array of message ids from the queue permanently
|
|
625
|
+
CREATE FUNCTION pgmq.delete(
|
|
626
|
+
queue_name TEXT,
|
|
627
|
+
msg_ids BIGINT[]
|
|
628
|
+
)
|
|
629
|
+
RETURNS SETOF BIGINT AS $$
|
|
630
|
+
DECLARE
|
|
631
|
+
sql TEXT;
|
|
632
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
633
|
+
BEGIN
|
|
634
|
+
sql := FORMAT(
|
|
635
|
+
$QUERY$
|
|
636
|
+
DELETE FROM pgmq.%I
|
|
637
|
+
WHERE msg_id = ANY($1)
|
|
638
|
+
RETURNING msg_id
|
|
639
|
+
$QUERY$,
|
|
640
|
+
qtable
|
|
641
|
+
);
|
|
642
|
+
RETURN QUERY EXECUTE sql USING msg_ids;
|
|
643
|
+
END;
|
|
644
|
+
$$ LANGUAGE plpgsql;
|
|
645
|
+
|
|
646
|
+
-- send: actual implementation
|
|
647
|
+
CREATE FUNCTION pgmq.send(
|
|
648
|
+
queue_name TEXT,
|
|
649
|
+
msg JSONB,
|
|
650
|
+
headers JSONB,
|
|
651
|
+
delay TIMESTAMP WITH TIME ZONE
|
|
652
|
+
) RETURNS SETOF BIGINT AS $$
|
|
653
|
+
DECLARE
|
|
654
|
+
sql TEXT;
|
|
655
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
656
|
+
BEGIN
|
|
657
|
+
sql := FORMAT(
|
|
658
|
+
$QUERY$
|
|
659
|
+
INSERT INTO pgmq.%I (vt, message, headers)
|
|
660
|
+
VALUES ($2, $1, $3)
|
|
661
|
+
RETURNING msg_id;
|
|
662
|
+
$QUERY$,
|
|
663
|
+
qtable
|
|
664
|
+
);
|
|
665
|
+
RETURN QUERY EXECUTE sql USING msg, delay, headers;
|
|
666
|
+
END;
|
|
667
|
+
$$ LANGUAGE plpgsql;
|
|
668
|
+
|
|
669
|
+
-- send: 2 args, no delay or headers
|
|
670
|
+
CREATE FUNCTION pgmq.send(
|
|
671
|
+
queue_name TEXT,
|
|
672
|
+
msg JSONB
|
|
673
|
+
) RETURNS SETOF BIGINT AS $$
|
|
674
|
+
SELECT * FROM pgmq.send(queue_name, msg, NULL, clock_timestamp());
|
|
675
|
+
$$ LANGUAGE sql;
|
|
676
|
+
|
|
677
|
+
-- send: 3 args with headers
|
|
678
|
+
CREATE FUNCTION pgmq.send(
|
|
679
|
+
queue_name TEXT,
|
|
680
|
+
msg JSONB,
|
|
681
|
+
headers JSONB
|
|
682
|
+
) RETURNS SETOF BIGINT AS $$
|
|
683
|
+
SELECT * FROM pgmq.send(queue_name, msg, headers, clock_timestamp());
|
|
684
|
+
$$ LANGUAGE sql;
|
|
685
|
+
|
|
686
|
+
-- send: 3 args with integer delay
|
|
687
|
+
CREATE FUNCTION pgmq.send(
|
|
688
|
+
queue_name TEXT,
|
|
689
|
+
msg JSONB,
|
|
690
|
+
delay INTEGER
|
|
691
|
+
) RETURNS SETOF BIGINT AS $$
|
|
692
|
+
SELECT * FROM pgmq.send(queue_name, msg, NULL, clock_timestamp() + make_interval(secs => delay));
|
|
693
|
+
$$ LANGUAGE sql;
|
|
694
|
+
|
|
695
|
+
-- send: 3 args with timestamp
|
|
696
|
+
CREATE FUNCTION pgmq.send(
|
|
697
|
+
queue_name TEXT,
|
|
698
|
+
msg JSONB,
|
|
699
|
+
delay TIMESTAMP WITH TIME ZONE
|
|
700
|
+
) RETURNS SETOF BIGINT AS $$
|
|
701
|
+
SELECT * FROM pgmq.send(queue_name, msg, NULL, delay);
|
|
702
|
+
$$ LANGUAGE sql;
|
|
703
|
+
|
|
704
|
+
-- send: 4 args with integer delay
|
|
705
|
+
CREATE FUNCTION pgmq.send(
|
|
706
|
+
queue_name TEXT,
|
|
707
|
+
msg JSONB,
|
|
708
|
+
headers JSONB,
|
|
709
|
+
delay INTEGER
|
|
710
|
+
) RETURNS SETOF BIGINT AS $$
|
|
711
|
+
SELECT * FROM pgmq.send(queue_name, msg, headers, clock_timestamp() + make_interval(secs => delay));
|
|
712
|
+
$$ LANGUAGE sql;
|
|
713
|
+
|
|
714
|
+
-- _validate_batch_params: Private function to validate batch parameters
|
|
715
|
+
CREATE FUNCTION pgmq._validate_batch_params(
|
|
716
|
+
msgs JSONB[],
|
|
717
|
+
headers JSONB[]
|
|
718
|
+
) RETURNS void AS $$
|
|
719
|
+
BEGIN
|
|
720
|
+
-- Validate that msgs is not NULL or empty
|
|
721
|
+
IF msgs IS NULL OR array_length(msgs, 1) IS NULL THEN
|
|
722
|
+
RAISE EXCEPTION 'msgs cannot be NULL or empty';
|
|
723
|
+
END IF;
|
|
724
|
+
|
|
725
|
+
-- Validate that headers array length matches msgs array length if headers is provided
|
|
726
|
+
-- Note: array_length returns NULL for empty arrays, so we use COALESCE to treat empty arrays as length 0
|
|
727
|
+
IF headers IS NOT NULL AND COALESCE(array_length(headers, 1), 0) != COALESCE(array_length(msgs, 1), 0) THEN
|
|
728
|
+
RAISE EXCEPTION 'headers array length (%) must match msgs array length (%)',
|
|
729
|
+
COALESCE(array_length(headers, 1), 0), COALESCE(array_length(msgs, 1), 0);
|
|
730
|
+
END IF;
|
|
731
|
+
END;
|
|
732
|
+
$$ LANGUAGE plpgsql;
|
|
733
|
+
|
|
734
|
+
-- _send_batch: Private function that performs the actual batch insert without validation
|
|
735
|
+
CREATE FUNCTION pgmq._send_batch(
|
|
736
|
+
queue_name TEXT,
|
|
737
|
+
msgs JSONB[],
|
|
738
|
+
headers JSONB[],
|
|
739
|
+
delay TIMESTAMP WITH TIME ZONE
|
|
740
|
+
) RETURNS SETOF BIGINT AS $$
|
|
741
|
+
DECLARE
|
|
742
|
+
sql TEXT;
|
|
743
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
744
|
+
BEGIN
|
|
745
|
+
sql := FORMAT(
|
|
746
|
+
$QUERY$
|
|
747
|
+
INSERT INTO pgmq.%I (vt, message, headers)
|
|
748
|
+
SELECT $2, unnest($1), unnest(coalesce($3, ARRAY[]::jsonb[]))
|
|
749
|
+
RETURNING msg_id;
|
|
750
|
+
$QUERY$,
|
|
751
|
+
qtable
|
|
752
|
+
);
|
|
753
|
+
RETURN QUERY EXECUTE sql USING msgs, delay, headers;
|
|
754
|
+
END;
|
|
755
|
+
$$ LANGUAGE plpgsql;
|
|
756
|
+
|
|
757
|
+
-- send_batch: Public function with validation
|
|
758
|
+
CREATE FUNCTION pgmq.send_batch(
|
|
759
|
+
queue_name TEXT,
|
|
760
|
+
msgs JSONB[],
|
|
761
|
+
headers JSONB[],
|
|
762
|
+
delay TIMESTAMP WITH TIME ZONE
|
|
763
|
+
) RETURNS SETOF BIGINT AS $$
|
|
764
|
+
BEGIN
|
|
765
|
+
PERFORM pgmq._validate_batch_params(msgs, headers);
|
|
766
|
+
RETURN QUERY SELECT * FROM pgmq._send_batch(queue_name, msgs, headers, delay);
|
|
767
|
+
END;
|
|
768
|
+
$$ LANGUAGE plpgsql;
|
|
769
|
+
|
|
770
|
+
-- send batch: 2 args
|
|
771
|
+
CREATE FUNCTION pgmq.send_batch(
|
|
772
|
+
queue_name TEXT,
|
|
773
|
+
msgs JSONB[]
|
|
774
|
+
) RETURNS SETOF BIGINT AS $$
|
|
775
|
+
SELECT * FROM pgmq.send_batch(queue_name, msgs, NULL, clock_timestamp());
|
|
776
|
+
$$ LANGUAGE sql;
|
|
777
|
+
|
|
778
|
+
-- send batch: 3 args with headers
|
|
779
|
+
CREATE FUNCTION pgmq.send_batch(
|
|
780
|
+
queue_name TEXT,
|
|
781
|
+
msgs JSONB[],
|
|
782
|
+
headers JSONB[]
|
|
783
|
+
) RETURNS SETOF BIGINT AS $$
|
|
784
|
+
SELECT * FROM pgmq.send_batch(queue_name, msgs, headers, clock_timestamp());
|
|
785
|
+
$$ LANGUAGE sql;
|
|
786
|
+
|
|
787
|
+
-- send batch: 3 args with integer delay
|
|
788
|
+
CREATE FUNCTION pgmq.send_batch(
|
|
789
|
+
queue_name TEXT,
|
|
790
|
+
msgs JSONB[],
|
|
791
|
+
delay INTEGER
|
|
792
|
+
) RETURNS SETOF BIGINT AS $$
|
|
793
|
+
SELECT * FROM pgmq.send_batch(queue_name, msgs, NULL, clock_timestamp() + make_interval(secs => delay));
|
|
794
|
+
$$ LANGUAGE sql;
|
|
795
|
+
|
|
796
|
+
-- send batch: 3 args with timestamp
|
|
797
|
+
CREATE FUNCTION pgmq.send_batch(
|
|
798
|
+
queue_name TEXT,
|
|
799
|
+
msgs JSONB[],
|
|
800
|
+
delay TIMESTAMP WITH TIME ZONE
|
|
801
|
+
) RETURNS SETOF BIGINT AS $$
|
|
802
|
+
SELECT * FROM pgmq.send_batch(queue_name, msgs, NULL, delay);
|
|
803
|
+
$$ LANGUAGE sql;
|
|
804
|
+
|
|
805
|
+
-- send_batch: 4 args with integer delay
|
|
806
|
+
CREATE FUNCTION pgmq.send_batch(
|
|
807
|
+
queue_name TEXT,
|
|
808
|
+
msgs JSONB[],
|
|
809
|
+
headers JSONB[],
|
|
810
|
+
delay INTEGER
|
|
811
|
+
) RETURNS SETOF BIGINT AS $$
|
|
812
|
+
SELECT * FROM pgmq.send_batch(queue_name, msgs, headers, clock_timestamp() + make_interval(secs => delay));
|
|
813
|
+
$$ LANGUAGE sql;
|
|
814
|
+
|
|
815
|
+
-- returned by pgmq.metrics() and pgmq.metrics_all
|
|
816
|
+
CREATE TYPE pgmq.metrics_result AS (
|
|
817
|
+
queue_name text,
|
|
818
|
+
queue_length bigint,
|
|
819
|
+
newest_msg_age_sec int,
|
|
820
|
+
oldest_msg_age_sec int,
|
|
821
|
+
total_messages bigint,
|
|
822
|
+
scrape_time timestamp with time zone,
|
|
823
|
+
queue_visible_length bigint
|
|
824
|
+
);
|
|
825
|
+
|
|
826
|
+
-- get metrics for a single queue
|
|
827
|
+
CREATE FUNCTION pgmq.metrics(queue_name TEXT)
|
|
828
|
+
RETURNS pgmq.metrics_result AS $$
|
|
829
|
+
DECLARE
|
|
830
|
+
result_row pgmq.metrics_result;
|
|
831
|
+
query TEXT;
|
|
832
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
833
|
+
BEGIN
|
|
834
|
+
query := FORMAT(
|
|
835
|
+
$QUERY$
|
|
836
|
+
WITH q_summary AS (
|
|
837
|
+
SELECT
|
|
838
|
+
count(*) as queue_length,
|
|
839
|
+
count(CASE WHEN vt <= NOW() THEN 1 END) as queue_visible_length,
|
|
840
|
+
EXTRACT(epoch FROM (NOW() - max(enqueued_at)))::int as newest_msg_age_sec,
|
|
841
|
+
EXTRACT(epoch FROM (NOW() - min(enqueued_at)))::int as oldest_msg_age_sec,
|
|
842
|
+
NOW() as scrape_time
|
|
843
|
+
FROM pgmq.%I
|
|
844
|
+
),
|
|
845
|
+
all_metrics AS (
|
|
846
|
+
SELECT CASE
|
|
847
|
+
WHEN is_called THEN last_value ELSE 0
|
|
848
|
+
END as total_messages
|
|
849
|
+
FROM pgmq.%I
|
|
850
|
+
)
|
|
851
|
+
SELECT
|
|
852
|
+
%L as queue_name,
|
|
853
|
+
q_summary.queue_length,
|
|
854
|
+
q_summary.newest_msg_age_sec,
|
|
855
|
+
q_summary.oldest_msg_age_sec,
|
|
856
|
+
all_metrics.total_messages,
|
|
857
|
+
q_summary.scrape_time,
|
|
858
|
+
q_summary.queue_visible_length
|
|
859
|
+
FROM q_summary, all_metrics
|
|
860
|
+
$QUERY$,
|
|
861
|
+
qtable, qtable || '_msg_id_seq', queue_name
|
|
862
|
+
);
|
|
863
|
+
EXECUTE query INTO result_row;
|
|
864
|
+
RETURN result_row;
|
|
865
|
+
END;
|
|
866
|
+
$$ LANGUAGE plpgsql;
|
|
867
|
+
|
|
868
|
+
-- get metrics for all queues
|
|
869
|
+
CREATE FUNCTION pgmq."metrics_all"()
|
|
870
|
+
RETURNS SETOF pgmq.metrics_result AS $$
|
|
871
|
+
DECLARE
|
|
872
|
+
row_name RECORD;
|
|
873
|
+
result_row pgmq.metrics_result;
|
|
874
|
+
BEGIN
|
|
875
|
+
FOR row_name IN SELECT queue_name FROM pgmq.meta LOOP
|
|
876
|
+
result_row := pgmq.metrics(row_name.queue_name);
|
|
877
|
+
RETURN NEXT result_row;
|
|
878
|
+
END LOOP;
|
|
879
|
+
END;
|
|
880
|
+
$$ LANGUAGE plpgsql;
|
|
881
|
+
|
|
882
|
+
-- list queues
|
|
883
|
+
CREATE FUNCTION pgmq."list_queues"()
|
|
884
|
+
RETURNS SETOF pgmq.queue_record AS $$
|
|
885
|
+
BEGIN
|
|
886
|
+
RETURN QUERY SELECT * FROM pgmq.meta;
|
|
887
|
+
END
|
|
888
|
+
$$ LANGUAGE plpgsql;
|
|
889
|
+
|
|
890
|
+
-- purge queue, deleting all entries in it.
|
|
891
|
+
CREATE OR REPLACE FUNCTION pgmq."purge_queue"(queue_name TEXT)
|
|
892
|
+
RETURNS BIGINT AS $$
|
|
893
|
+
DECLARE
|
|
894
|
+
deleted_count INTEGER;
|
|
895
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
896
|
+
BEGIN
|
|
897
|
+
-- Get the row count before truncating
|
|
898
|
+
EXECUTE format('SELECT count(*) FROM pgmq.%I', qtable) INTO deleted_count;
|
|
899
|
+
|
|
900
|
+
-- Use TRUNCATE for better performance on large tables
|
|
901
|
+
EXECUTE format('TRUNCATE TABLE pgmq.%I', qtable);
|
|
902
|
+
|
|
903
|
+
-- Return the number of purged rows
|
|
904
|
+
RETURN deleted_count;
|
|
905
|
+
END
|
|
906
|
+
$$ LANGUAGE plpgsql;
|
|
907
|
+
|
|
908
|
+
-- unassign archive, so it can be kept when a queue is deleted
|
|
909
|
+
CREATE FUNCTION pgmq."detach_archive"(queue_name TEXT)
|
|
910
|
+
RETURNS VOID AS $$
|
|
911
|
+
DECLARE
|
|
912
|
+
atable TEXT := pgmq.format_table_name(queue_name, 'a');
|
|
913
|
+
BEGIN
|
|
914
|
+
RAISE WARNING 'detach_archive(queue_name) is deprecated and is a no-op. It will be removed in PGMQ v2.0. Archive tables are no longer member objects.';
|
|
915
|
+
END
|
|
916
|
+
$$ LANGUAGE plpgsql;
|
|
917
|
+
|
|
918
|
+
-- pop: implementation
|
|
919
|
+
CREATE FUNCTION pgmq.pop(queue_name TEXT, qty INTEGER DEFAULT 1)
|
|
920
|
+
RETURNS SETOF pgmq.message_record AS $$
|
|
921
|
+
DECLARE
|
|
922
|
+
sql TEXT;
|
|
923
|
+
result pgmq.message_record;
|
|
924
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
925
|
+
BEGIN
|
|
926
|
+
sql := FORMAT(
|
|
927
|
+
$QUERY$
|
|
928
|
+
WITH cte AS
|
|
929
|
+
(
|
|
930
|
+
SELECT msg_id
|
|
931
|
+
FROM pgmq.%I
|
|
932
|
+
WHERE vt <= clock_timestamp()
|
|
933
|
+
ORDER BY msg_id ASC
|
|
934
|
+
LIMIT $1
|
|
935
|
+
FOR UPDATE SKIP LOCKED
|
|
936
|
+
)
|
|
937
|
+
DELETE from pgmq.%I
|
|
938
|
+
WHERE msg_id IN (select msg_id from cte)
|
|
939
|
+
RETURNING msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers;
|
|
940
|
+
$QUERY$,
|
|
941
|
+
qtable, qtable
|
|
942
|
+
);
|
|
943
|
+
RETURN QUERY EXECUTE sql USING qty;
|
|
944
|
+
END;
|
|
945
|
+
$$ LANGUAGE plpgsql;
|
|
946
|
+
|
|
947
|
+
-- Sets timestamp vt of a message, returns it
|
|
948
|
+
CREATE FUNCTION pgmq.set_vt(queue_name TEXT, msg_id BIGINT, vt TIMESTAMP WITH TIME ZONE)
|
|
949
|
+
RETURNS SETOF pgmq.message_record AS $$
|
|
950
|
+
DECLARE
|
|
951
|
+
sql TEXT;
|
|
952
|
+
result pgmq.message_record;
|
|
953
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
954
|
+
BEGIN
|
|
955
|
+
sql := FORMAT(
|
|
956
|
+
$QUERY$
|
|
957
|
+
UPDATE pgmq.%I
|
|
958
|
+
SET vt = $1
|
|
959
|
+
WHERE msg_id = $2
|
|
960
|
+
RETURNING msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers;
|
|
961
|
+
$QUERY$,
|
|
962
|
+
qtable
|
|
963
|
+
);
|
|
964
|
+
RETURN QUERY EXECUTE sql USING vt, msg_id;
|
|
965
|
+
END;
|
|
966
|
+
$$ LANGUAGE plpgsql;
|
|
967
|
+
|
|
968
|
+
-- Sets integer vt of a message, returns it
|
|
969
|
+
CREATE FUNCTION pgmq.set_vt(queue_name TEXT, msg_id BIGINT, vt INTEGER)
|
|
970
|
+
RETURNS SETOF pgmq.message_record AS $$
|
|
971
|
+
SELECT * FROM pgmq.set_vt(queue_name, msg_id, clock_timestamp() + make_interval(secs => vt));
|
|
972
|
+
$$ LANGUAGE sql;
|
|
973
|
+
|
|
974
|
+
-- Sets timestamp vt of multiple messages, returns them
|
|
975
|
+
CREATE FUNCTION pgmq.set_vt(
|
|
976
|
+
queue_name TEXT,
|
|
977
|
+
msg_ids BIGINT[],
|
|
978
|
+
vt TIMESTAMP WITH TIME ZONE
|
|
979
|
+
)
|
|
980
|
+
RETURNS SETOF pgmq.message_record AS $$
|
|
981
|
+
DECLARE
|
|
982
|
+
sql TEXT;
|
|
983
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
984
|
+
BEGIN
|
|
985
|
+
sql := FORMAT(
|
|
986
|
+
$QUERY$
|
|
987
|
+
UPDATE pgmq.%I
|
|
988
|
+
SET vt = $1
|
|
989
|
+
WHERE msg_id = ANY($2)
|
|
990
|
+
RETURNING msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers;
|
|
991
|
+
$QUERY$,
|
|
992
|
+
qtable
|
|
993
|
+
);
|
|
994
|
+
RETURN QUERY EXECUTE sql USING vt, msg_ids;
|
|
995
|
+
END;
|
|
996
|
+
$$ LANGUAGE plpgsql;
|
|
997
|
+
|
|
998
|
+
-- Sets integer vt of multiple messages, returns them
|
|
999
|
+
CREATE FUNCTION pgmq.set_vt(
|
|
1000
|
+
queue_name TEXT,
|
|
1001
|
+
msg_ids BIGINT[],
|
|
1002
|
+
vt INTEGER
|
|
1003
|
+
)
|
|
1004
|
+
RETURNS SETOF pgmq.message_record AS $$
|
|
1005
|
+
SELECT * FROM pgmq.set_vt(queue_name, msg_ids, clock_timestamp() + make_interval(secs => vt));
|
|
1006
|
+
$$ LANGUAGE sql;
|
|
1007
|
+
|
|
1008
|
+
CREATE FUNCTION pgmq._get_pg_partman_schema()
|
|
1009
|
+
RETURNS TEXT AS $$
|
|
1010
|
+
SELECT
|
|
1011
|
+
extnamespace::regnamespace::text
|
|
1012
|
+
FROM
|
|
1013
|
+
pg_extension
|
|
1014
|
+
WHERE
|
|
1015
|
+
extname = 'pg_partman';
|
|
1016
|
+
$$ LANGUAGE SQL;
|
|
1017
|
+
|
|
1018
|
+
CREATE FUNCTION pgmq.drop_queue(queue_name TEXT, partitioned BOOLEAN)
|
|
1019
|
+
RETURNS BOOLEAN AS $$
|
|
1020
|
+
DECLARE
|
|
1021
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
1022
|
+
fq_qtable TEXT := 'pgmq.' || qtable;
|
|
1023
|
+
atable TEXT := pgmq.format_table_name(queue_name, 'a');
|
|
1024
|
+
fq_atable TEXT := 'pgmq.' || atable;
|
|
1025
|
+
BEGIN
|
|
1026
|
+
RAISE WARNING 'drop_queue(queue_name, partitioned) is deprecated and will be removed in PGMQ v2.0. Use drop_queue(queue_name) instead';
|
|
1027
|
+
|
|
1028
|
+
PERFORM pgmq.drop_queue(queue_name);
|
|
1029
|
+
|
|
1030
|
+
RETURN TRUE;
|
|
1031
|
+
END;
|
|
1032
|
+
$$ LANGUAGE plpgsql;
|
|
1033
|
+
|
|
1034
|
+
CREATE FUNCTION pgmq.drop_queue(queue_name TEXT)
|
|
1035
|
+
RETURNS BOOLEAN AS $$
|
|
1036
|
+
DECLARE
|
|
1037
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
1038
|
+
qtable_seq TEXT := qtable || '_msg_id_seq';
|
|
1039
|
+
fq_qtable TEXT := 'pgmq.' || qtable;
|
|
1040
|
+
atable TEXT := pgmq.format_table_name(queue_name, 'a');
|
|
1041
|
+
fq_atable TEXT := 'pgmq.' || atable;
|
|
1042
|
+
partitioned BOOLEAN;
|
|
1043
|
+
BEGIN
|
|
1044
|
+
PERFORM pgmq.acquire_queue_lock(queue_name);
|
|
1045
|
+
EXECUTE FORMAT(
|
|
1046
|
+
$QUERY$
|
|
1047
|
+
SELECT is_partitioned FROM pgmq.meta WHERE queue_name = %L
|
|
1048
|
+
$QUERY$,
|
|
1049
|
+
queue_name
|
|
1050
|
+
) INTO partitioned;
|
|
1051
|
+
|
|
1052
|
+
-- check if the queue exists
|
|
1053
|
+
IF NOT EXISTS (
|
|
1054
|
+
SELECT 1
|
|
1055
|
+
FROM information_schema.tables
|
|
1056
|
+
WHERE table_name = qtable and table_schema = 'pgmq'
|
|
1057
|
+
) THEN
|
|
1058
|
+
RAISE NOTICE 'pgmq queue `%` does not exist', queue_name;
|
|
1059
|
+
RETURN FALSE;
|
|
1060
|
+
END IF;
|
|
1061
|
+
|
|
1062
|
+
EXECUTE FORMAT(
|
|
1063
|
+
$QUERY$
|
|
1064
|
+
DROP TABLE IF EXISTS pgmq.%I
|
|
1065
|
+
$QUERY$,
|
|
1066
|
+
qtable
|
|
1067
|
+
);
|
|
1068
|
+
|
|
1069
|
+
EXECUTE FORMAT(
|
|
1070
|
+
$QUERY$
|
|
1071
|
+
DROP TABLE IF EXISTS pgmq.%I
|
|
1072
|
+
$QUERY$,
|
|
1073
|
+
atable
|
|
1074
|
+
);
|
|
1075
|
+
|
|
1076
|
+
IF EXISTS (
|
|
1077
|
+
SELECT 1
|
|
1078
|
+
FROM information_schema.tables
|
|
1079
|
+
WHERE table_name = 'meta' and table_schema = 'pgmq'
|
|
1080
|
+
) THEN
|
|
1081
|
+
EXECUTE FORMAT(
|
|
1082
|
+
$QUERY$
|
|
1083
|
+
DELETE FROM pgmq.meta WHERE queue_name = %L
|
|
1084
|
+
$QUERY$,
|
|
1085
|
+
queue_name
|
|
1086
|
+
);
|
|
1087
|
+
END IF;
|
|
1088
|
+
|
|
1089
|
+
IF partitioned THEN
|
|
1090
|
+
EXECUTE FORMAT(
|
|
1091
|
+
$QUERY$
|
|
1092
|
+
DELETE FROM %I.part_config where parent_table in (%L, %L)
|
|
1093
|
+
$QUERY$,
|
|
1094
|
+
pgmq._get_pg_partman_schema(), fq_qtable, fq_atable
|
|
1095
|
+
);
|
|
1096
|
+
END IF;
|
|
1097
|
+
|
|
1098
|
+
RETURN TRUE;
|
|
1099
|
+
END;
|
|
1100
|
+
$$ LANGUAGE plpgsql;
|
|
1101
|
+
|
|
1102
|
+
CREATE FUNCTION pgmq.validate_queue_name(queue_name TEXT)
|
|
1103
|
+
RETURNS void AS $$
|
|
1104
|
+
BEGIN
|
|
1105
|
+
IF length(queue_name) > 47 THEN
|
|
1106
|
+
-- complete table identifier must be <= 63
|
|
1107
|
+
-- https://www.postgresql.org/docs/17/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
|
|
1108
|
+
-- e.g. template_pgmq_q_my_queue is an identifier for my_queue when partitioned
|
|
1109
|
+
-- template_pgmq_q_ (16) + <a max length queue name> (47) = 63
|
|
1110
|
+
RAISE EXCEPTION 'queue name is too long, maximum length is 47 characters';
|
|
1111
|
+
END IF;
|
|
1112
|
+
END;
|
|
1113
|
+
$$ LANGUAGE plpgsql;
|
|
1114
|
+
|
|
1115
|
+
CREATE FUNCTION pgmq._belongs_to_pgmq(table_name TEXT)
|
|
1116
|
+
RETURNS BOOLEAN AS $$
|
|
1117
|
+
DECLARE
|
|
1118
|
+
sql TEXT;
|
|
1119
|
+
result BOOLEAN;
|
|
1120
|
+
BEGIN
|
|
1121
|
+
SELECT EXISTS (
|
|
1122
|
+
SELECT 1
|
|
1123
|
+
FROM pg_depend
|
|
1124
|
+
WHERE refobjid = (SELECT oid FROM pg_extension WHERE extname = 'pgmq')
|
|
1125
|
+
AND objid = (
|
|
1126
|
+
SELECT oid
|
|
1127
|
+
FROM pg_class
|
|
1128
|
+
WHERE relname = table_name
|
|
1129
|
+
)
|
|
1130
|
+
) INTO result;
|
|
1131
|
+
RETURN result;
|
|
1132
|
+
END;
|
|
1133
|
+
$$ LANGUAGE plpgsql;
|
|
1134
|
+
|
|
1135
|
+
CREATE FUNCTION pgmq.create_non_partitioned(queue_name TEXT)
|
|
1136
|
+
RETURNS void AS $$
|
|
1137
|
+
DECLARE
|
|
1138
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
1139
|
+
qtable_seq TEXT := qtable || '_msg_id_seq';
|
|
1140
|
+
atable TEXT := pgmq.format_table_name(queue_name, 'a');
|
|
1141
|
+
BEGIN
|
|
1142
|
+
PERFORM pgmq.validate_queue_name(queue_name);
|
|
1143
|
+
PERFORM pgmq.acquire_queue_lock(queue_name);
|
|
1144
|
+
|
|
1145
|
+
EXECUTE FORMAT(
|
|
1146
|
+
$QUERY$
|
|
1147
|
+
CREATE TABLE IF NOT EXISTS pgmq.%I (
|
|
1148
|
+
msg_id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
|
|
1149
|
+
read_ct INT DEFAULT 0 NOT NULL,
|
|
1150
|
+
enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
|
|
1151
|
+
last_read_at TIMESTAMP WITH TIME ZONE,
|
|
1152
|
+
vt TIMESTAMP WITH TIME ZONE NOT NULL,
|
|
1153
|
+
message JSONB,
|
|
1154
|
+
headers JSONB
|
|
1155
|
+
)
|
|
1156
|
+
$QUERY$,
|
|
1157
|
+
qtable
|
|
1158
|
+
);
|
|
1159
|
+
|
|
1160
|
+
EXECUTE FORMAT(
|
|
1161
|
+
$QUERY$
|
|
1162
|
+
CREATE TABLE IF NOT EXISTS pgmq.%I (
|
|
1163
|
+
msg_id BIGINT PRIMARY KEY,
|
|
1164
|
+
read_ct INT DEFAULT 0 NOT NULL,
|
|
1165
|
+
enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
|
|
1166
|
+
last_read_at TIMESTAMP WITH TIME ZONE,
|
|
1167
|
+
archived_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
|
|
1168
|
+
vt TIMESTAMP WITH TIME ZONE NOT NULL,
|
|
1169
|
+
message JSONB,
|
|
1170
|
+
headers JSONB
|
|
1171
|
+
);
|
|
1172
|
+
$QUERY$,
|
|
1173
|
+
atable
|
|
1174
|
+
);
|
|
1175
|
+
|
|
1176
|
+
EXECUTE FORMAT(
|
|
1177
|
+
$QUERY$
|
|
1178
|
+
CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (vt ASC);
|
|
1179
|
+
$QUERY$,
|
|
1180
|
+
qtable || '_vt_idx', qtable
|
|
1181
|
+
);
|
|
1182
|
+
|
|
1183
|
+
EXECUTE FORMAT(
|
|
1184
|
+
$QUERY$
|
|
1185
|
+
CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (archived_at);
|
|
1186
|
+
$QUERY$,
|
|
1187
|
+
'archived_at_idx_' || queue_name, atable
|
|
1188
|
+
);
|
|
1189
|
+
|
|
1190
|
+
EXECUTE FORMAT(
|
|
1191
|
+
$QUERY$
|
|
1192
|
+
INSERT INTO pgmq.meta (queue_name, is_partitioned, is_unlogged)
|
|
1193
|
+
VALUES (%L, false, false)
|
|
1194
|
+
ON CONFLICT
|
|
1195
|
+
DO NOTHING;
|
|
1196
|
+
$QUERY$,
|
|
1197
|
+
queue_name
|
|
1198
|
+
);
|
|
1199
|
+
|
|
1200
|
+
END;
|
|
1201
|
+
$$ LANGUAGE plpgsql;
|
|
1202
|
+
|
|
1203
|
+
CREATE FUNCTION pgmq.create_unlogged(queue_name TEXT)
|
|
1204
|
+
RETURNS void AS $$
|
|
1205
|
+
DECLARE
|
|
1206
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
1207
|
+
qtable_seq TEXT := qtable || '_msg_id_seq';
|
|
1208
|
+
atable TEXT := pgmq.format_table_name(queue_name, 'a');
|
|
1209
|
+
BEGIN
|
|
1210
|
+
PERFORM pgmq.validate_queue_name(queue_name);
|
|
1211
|
+
PERFORM pgmq.acquire_queue_lock(queue_name);
|
|
1212
|
+
|
|
1213
|
+
EXECUTE FORMAT(
|
|
1214
|
+
$QUERY$
|
|
1215
|
+
CREATE UNLOGGED TABLE IF NOT EXISTS pgmq.%I (
|
|
1216
|
+
msg_id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
|
|
1217
|
+
read_ct INT DEFAULT 0 NOT NULL,
|
|
1218
|
+
enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
|
|
1219
|
+
last_read_at TIMESTAMP WITH TIME ZONE,
|
|
1220
|
+
vt TIMESTAMP WITH TIME ZONE NOT NULL,
|
|
1221
|
+
message JSONB,
|
|
1222
|
+
headers JSONB
|
|
1223
|
+
)
|
|
1224
|
+
$QUERY$,
|
|
1225
|
+
qtable
|
|
1226
|
+
);
|
|
1227
|
+
|
|
1228
|
+
EXECUTE FORMAT(
|
|
1229
|
+
$QUERY$
|
|
1230
|
+
CREATE TABLE IF NOT EXISTS pgmq.%I (
|
|
1231
|
+
msg_id BIGINT PRIMARY KEY,
|
|
1232
|
+
read_ct INT DEFAULT 0 NOT NULL,
|
|
1233
|
+
enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
|
|
1234
|
+
last_read_at TIMESTAMP WITH TIME ZONE,
|
|
1235
|
+
archived_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
|
|
1236
|
+
vt TIMESTAMP WITH TIME ZONE NOT NULL,
|
|
1237
|
+
message JSONB,
|
|
1238
|
+
headers JSONB
|
|
1239
|
+
);
|
|
1240
|
+
$QUERY$,
|
|
1241
|
+
atable
|
|
1242
|
+
);
|
|
1243
|
+
|
|
1244
|
+
EXECUTE FORMAT(
|
|
1245
|
+
$QUERY$
|
|
1246
|
+
CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (vt ASC);
|
|
1247
|
+
$QUERY$,
|
|
1248
|
+
qtable || '_vt_idx', qtable
|
|
1249
|
+
);
|
|
1250
|
+
|
|
1251
|
+
EXECUTE FORMAT(
|
|
1252
|
+
$QUERY$
|
|
1253
|
+
CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (archived_at);
|
|
1254
|
+
$QUERY$,
|
|
1255
|
+
'archived_at_idx_' || queue_name, atable
|
|
1256
|
+
);
|
|
1257
|
+
|
|
1258
|
+
EXECUTE FORMAT(
|
|
1259
|
+
$QUERY$
|
|
1260
|
+
INSERT INTO pgmq.meta (queue_name, is_partitioned, is_unlogged)
|
|
1261
|
+
VALUES (%L, false, true)
|
|
1262
|
+
ON CONFLICT
|
|
1263
|
+
DO NOTHING;
|
|
1264
|
+
$QUERY$,
|
|
1265
|
+
queue_name
|
|
1266
|
+
);
|
|
1267
|
+
END;
|
|
1268
|
+
$$ LANGUAGE plpgsql;
|
|
1269
|
+
|
|
1270
|
+
CREATE FUNCTION pgmq._get_partition_col(partition_interval TEXT)
|
|
1271
|
+
RETURNS TEXT AS $$
|
|
1272
|
+
DECLARE
|
|
1273
|
+
num INTEGER;
|
|
1274
|
+
BEGIN
|
|
1275
|
+
BEGIN
|
|
1276
|
+
num := partition_interval::INTEGER;
|
|
1277
|
+
RETURN 'msg_id';
|
|
1278
|
+
EXCEPTION
|
|
1279
|
+
WHEN others THEN
|
|
1280
|
+
RETURN 'enqueued_at';
|
|
1281
|
+
END;
|
|
1282
|
+
END;
|
|
1283
|
+
$$ LANGUAGE plpgsql;
|
|
1284
|
+
|
|
1285
|
+
CREATE FUNCTION pgmq._extension_exists(extension_name TEXT)
|
|
1286
|
+
RETURNS BOOLEAN
|
|
1287
|
+
LANGUAGE SQL
|
|
1288
|
+
AS $$
|
|
1289
|
+
SELECT EXISTS (
|
|
1290
|
+
SELECT 1
|
|
1291
|
+
FROM pg_extension
|
|
1292
|
+
WHERE extname = extension_name
|
|
1293
|
+
)
|
|
1294
|
+
$$;
|
|
1295
|
+
|
|
1296
|
+
CREATE FUNCTION pgmq._ensure_pg_partman_installed()
|
|
1297
|
+
RETURNS void AS $$
|
|
1298
|
+
BEGIN
|
|
1299
|
+
IF NOT pgmq._extension_exists('pg_partman') THEN
|
|
1300
|
+
RAISE EXCEPTION 'pg_partman is required for partitioned queues';
|
|
1301
|
+
END IF;
|
|
1302
|
+
END;
|
|
1303
|
+
$$ LANGUAGE plpgsql;
|
|
1304
|
+
|
|
1305
|
+
CREATE FUNCTION pgmq._get_pg_partman_major_version()
|
|
1306
|
+
RETURNS INT
|
|
1307
|
+
LANGUAGE SQL
|
|
1308
|
+
AS $$
|
|
1309
|
+
SELECT split_part(extversion, '.', 1)::INT
|
|
1310
|
+
FROM pg_extension
|
|
1311
|
+
WHERE extname = 'pg_partman'
|
|
1312
|
+
$$;
|
|
1313
|
+
|
|
1314
|
+
CREATE FUNCTION pgmq.create_partitioned(
|
|
1315
|
+
queue_name TEXT,
|
|
1316
|
+
partition_interval TEXT DEFAULT '10000',
|
|
1317
|
+
retention_interval TEXT DEFAULT '100000'
|
|
1318
|
+
)
|
|
1319
|
+
RETURNS void AS $$
|
|
1320
|
+
DECLARE
|
|
1321
|
+
partition_col TEXT;
|
|
1322
|
+
a_partition_col TEXT;
|
|
1323
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
1324
|
+
qtable_seq TEXT := qtable || '_msg_id_seq';
|
|
1325
|
+
atable TEXT := pgmq.format_table_name(queue_name, 'a');
|
|
1326
|
+
fq_qtable TEXT := 'pgmq.' || qtable;
|
|
1327
|
+
fq_atable TEXT := 'pgmq.' || atable;
|
|
1328
|
+
BEGIN
|
|
1329
|
+
PERFORM pgmq.validate_queue_name(queue_name);
|
|
1330
|
+
PERFORM pgmq.acquire_queue_lock(queue_name);
|
|
1331
|
+
PERFORM pgmq._ensure_pg_partman_installed();
|
|
1332
|
+
SELECT pgmq._get_partition_col(partition_interval) INTO partition_col;
|
|
1333
|
+
|
|
1334
|
+
EXECUTE FORMAT(
|
|
1335
|
+
$QUERY$
|
|
1336
|
+
CREATE TABLE IF NOT EXISTS pgmq.%I (
|
|
1337
|
+
msg_id BIGINT GENERATED ALWAYS AS IDENTITY,
|
|
1338
|
+
read_ct INT DEFAULT 0 NOT NULL,
|
|
1339
|
+
enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
|
|
1340
|
+
last_read_at TIMESTAMP WITH TIME ZONE,
|
|
1341
|
+
vt TIMESTAMP WITH TIME ZONE NOT NULL,
|
|
1342
|
+
message JSONB,
|
|
1343
|
+
headers JSONB
|
|
1344
|
+
) PARTITION BY RANGE (%I)
|
|
1345
|
+
$QUERY$,
|
|
1346
|
+
qtable, partition_col
|
|
1347
|
+
);
|
|
1348
|
+
|
|
1349
|
+
-- https://github.com/pgpartman/pg_partman/blob/master/doc/pg_partman.md
|
|
1350
|
+
-- p_parent_table - the existing parent table. MUST be schema qualified, even if in public schema.
|
|
1351
|
+
EXECUTE FORMAT(
|
|
1352
|
+
$QUERY$
|
|
1353
|
+
SELECT %I.create_parent(
|
|
1354
|
+
p_parent_table := %L,
|
|
1355
|
+
p_control := %L,
|
|
1356
|
+
p_interval := %L,
|
|
1357
|
+
p_type := case
|
|
1358
|
+
when pgmq._get_pg_partman_major_version() = 5 then 'range'
|
|
1359
|
+
else 'native'
|
|
1360
|
+
end
|
|
1361
|
+
)
|
|
1362
|
+
$QUERY$,
|
|
1363
|
+
pgmq._get_pg_partman_schema(),
|
|
1364
|
+
fq_qtable,
|
|
1365
|
+
partition_col,
|
|
1366
|
+
partition_interval
|
|
1367
|
+
);
|
|
1368
|
+
|
|
1369
|
+
EXECUTE FORMAT(
|
|
1370
|
+
$QUERY$
|
|
1371
|
+
CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (%I);
|
|
1372
|
+
$QUERY$,
|
|
1373
|
+
qtable || '_part_idx', qtable, partition_col
|
|
1374
|
+
);
|
|
1375
|
+
|
|
1376
|
+
EXECUTE FORMAT(
|
|
1377
|
+
$QUERY$
|
|
1378
|
+
UPDATE %I.part_config
|
|
1379
|
+
SET
|
|
1380
|
+
retention = %L,
|
|
1381
|
+
retention_keep_table = false,
|
|
1382
|
+
retention_keep_index = true,
|
|
1383
|
+
automatic_maintenance = 'on'
|
|
1384
|
+
WHERE parent_table = %L;
|
|
1385
|
+
$QUERY$,
|
|
1386
|
+
pgmq._get_pg_partman_schema(),
|
|
1387
|
+
retention_interval,
|
|
1388
|
+
'pgmq.' || qtable
|
|
1389
|
+
);
|
|
1390
|
+
|
|
1391
|
+
EXECUTE FORMAT(
|
|
1392
|
+
$QUERY$
|
|
1393
|
+
INSERT INTO pgmq.meta (queue_name, is_partitioned, is_unlogged)
|
|
1394
|
+
VALUES (%L, true, false)
|
|
1395
|
+
ON CONFLICT
|
|
1396
|
+
DO NOTHING;
|
|
1397
|
+
$QUERY$,
|
|
1398
|
+
queue_name
|
|
1399
|
+
);
|
|
1400
|
+
|
|
1401
|
+
IF partition_col = 'enqueued_at' THEN
|
|
1402
|
+
a_partition_col := 'archived_at';
|
|
1403
|
+
ELSE
|
|
1404
|
+
a_partition_col := partition_col;
|
|
1405
|
+
END IF;
|
|
1406
|
+
|
|
1407
|
+
EXECUTE FORMAT(
|
|
1408
|
+
$QUERY$
|
|
1409
|
+
CREATE TABLE IF NOT EXISTS pgmq.%I (
|
|
1410
|
+
msg_id BIGINT NOT NULL,
|
|
1411
|
+
read_ct INT DEFAULT 0 NOT NULL,
|
|
1412
|
+
enqueued_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
|
|
1413
|
+
last_read_at TIMESTAMP WITH TIME ZONE,
|
|
1414
|
+
archived_at TIMESTAMP WITH TIME ZONE DEFAULT now() NOT NULL,
|
|
1415
|
+
vt TIMESTAMP WITH TIME ZONE NOT NULL,
|
|
1416
|
+
message JSONB,
|
|
1417
|
+
headers JSONB
|
|
1418
|
+
) PARTITION BY RANGE (%I);
|
|
1419
|
+
$QUERY$,
|
|
1420
|
+
atable, a_partition_col
|
|
1421
|
+
);
|
|
1422
|
+
|
|
1423
|
+
-- https://github.com/pgpartman/pg_partman/blob/master/doc/pg_partman.md
|
|
1424
|
+
-- p_parent_table - the existing parent table. MUST be schema qualified, even if in public schema.
|
|
1425
|
+
EXECUTE FORMAT(
|
|
1426
|
+
$QUERY$
|
|
1427
|
+
SELECT %I.create_parent(
|
|
1428
|
+
p_parent_table := %L,
|
|
1429
|
+
p_control := %L,
|
|
1430
|
+
p_interval := %L,
|
|
1431
|
+
p_type := case
|
|
1432
|
+
when pgmq._get_pg_partman_major_version() = 5 then 'range'
|
|
1433
|
+
else 'native'
|
|
1434
|
+
end
|
|
1435
|
+
)
|
|
1436
|
+
$QUERY$,
|
|
1437
|
+
pgmq._get_pg_partman_schema(),
|
|
1438
|
+
fq_atable,
|
|
1439
|
+
a_partition_col,
|
|
1440
|
+
partition_interval
|
|
1441
|
+
);
|
|
1442
|
+
|
|
1443
|
+
EXECUTE FORMAT(
|
|
1444
|
+
$QUERY$
|
|
1445
|
+
UPDATE %I.part_config
|
|
1446
|
+
SET
|
|
1447
|
+
retention = %L,
|
|
1448
|
+
retention_keep_table = false,
|
|
1449
|
+
retention_keep_index = true,
|
|
1450
|
+
automatic_maintenance = 'on'
|
|
1451
|
+
WHERE parent_table = %L;
|
|
1452
|
+
$QUERY$,
|
|
1453
|
+
pgmq._get_pg_partman_schema(),
|
|
1454
|
+
retention_interval,
|
|
1455
|
+
'pgmq.' || atable
|
|
1456
|
+
);
|
|
1457
|
+
|
|
1458
|
+
EXECUTE FORMAT(
|
|
1459
|
+
$QUERY$
|
|
1460
|
+
CREATE INDEX IF NOT EXISTS %I ON pgmq.%I (archived_at);
|
|
1461
|
+
$QUERY$,
|
|
1462
|
+
'archived_at_idx_' || queue_name, atable
|
|
1463
|
+
);
|
|
1464
|
+
|
|
1465
|
+
END;
|
|
1466
|
+
$$ LANGUAGE plpgsql;
|
|
1467
|
+
|
|
1468
|
+
CREATE FUNCTION pgmq.create(queue_name TEXT)
|
|
1469
|
+
RETURNS void AS $$
|
|
1470
|
+
BEGIN
|
|
1471
|
+
PERFORM pgmq.create_non_partitioned(queue_name);
|
|
1472
|
+
END;
|
|
1473
|
+
$$ LANGUAGE plpgsql;
|
|
1474
|
+
|
|
1475
|
+
-- _create_fifo_index_if_not_exists
|
|
1476
|
+
-- internal function to create GIN index on headers for better FIFO performance
|
|
1477
|
+
CREATE OR REPLACE FUNCTION pgmq._create_fifo_index_if_not_exists(queue_name TEXT)
|
|
1478
|
+
RETURNS void AS $$
|
|
1479
|
+
DECLARE
|
|
1480
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
1481
|
+
index_name TEXT := qtable || '_fifo_idx';
|
|
1482
|
+
BEGIN
|
|
1483
|
+
-- Create GIN index on headers for efficient FIFO key lookups
|
|
1484
|
+
EXECUTE FORMAT(
|
|
1485
|
+
$QUERY$
|
|
1486
|
+
CREATE INDEX IF NOT EXISTS %I ON pgmq.%I USING GIN (headers);
|
|
1487
|
+
$QUERY$,
|
|
1488
|
+
index_name, qtable
|
|
1489
|
+
);
|
|
1490
|
+
END;
|
|
1491
|
+
$$ LANGUAGE plpgsql;
|
|
1492
|
+
|
|
1493
|
+
-- create_fifo_index
|
|
1494
|
+
-- creates a GIN index on the headers column to improve FIFO read performance
|
|
1495
|
+
CREATE FUNCTION pgmq.create_fifo_index(queue_name TEXT)
|
|
1496
|
+
RETURNS void AS $$
|
|
1497
|
+
BEGIN
|
|
1498
|
+
PERFORM pgmq._create_fifo_index_if_not_exists(queue_name);
|
|
1499
|
+
END;
|
|
1500
|
+
$$ LANGUAGE plpgsql;
|
|
1501
|
+
|
|
1502
|
+
-- create_fifo_indexes_all
|
|
1503
|
+
-- creates FIFO indexes on all existing queues
|
|
1504
|
+
CREATE FUNCTION pgmq.create_fifo_indexes_all()
|
|
1505
|
+
RETURNS void AS $$
|
|
1506
|
+
DECLARE
|
|
1507
|
+
queue_record RECORD;
|
|
1508
|
+
BEGIN
|
|
1509
|
+
FOR queue_record IN SELECT queue_name FROM pgmq.meta LOOP
|
|
1510
|
+
PERFORM pgmq.create_fifo_index(queue_record.queue_name);
|
|
1511
|
+
END LOOP;
|
|
1512
|
+
END;
|
|
1513
|
+
$$ LANGUAGE plpgsql;
|
|
1514
|
+
|
|
1515
|
+
CREATE OR REPLACE FUNCTION pgmq.convert_archive_partitioned(
|
|
1516
|
+
table_name TEXT,
|
|
1517
|
+
partition_interval TEXT DEFAULT '10000',
|
|
1518
|
+
retention_interval TEXT DEFAULT '100000',
|
|
1519
|
+
leading_partition INT DEFAULT 10
|
|
1520
|
+
)
|
|
1521
|
+
RETURNS void AS $$
|
|
1522
|
+
DECLARE
|
|
1523
|
+
a_table_name TEXT := pgmq.format_table_name(table_name, 'a');
|
|
1524
|
+
a_table_name_old TEXT := pgmq.format_table_name(table_name, 'a') || '_old';
|
|
1525
|
+
qualified_a_table_name TEXT := format('pgmq.%I', a_table_name);
|
|
1526
|
+
partition_col TEXT;
|
|
1527
|
+
a_partition_col TEXT;
|
|
1528
|
+
BEGIN
|
|
1529
|
+
|
|
1530
|
+
PERFORM c.relkind
|
|
1531
|
+
FROM pg_class c
|
|
1532
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
1533
|
+
WHERE c.relname = a_table_name
|
|
1534
|
+
AND c.relkind = 'p';
|
|
1535
|
+
|
|
1536
|
+
IF FOUND THEN
|
|
1537
|
+
RAISE NOTICE 'Table %s is already partitioned', a_table_name;
|
|
1538
|
+
RETURN;
|
|
1539
|
+
END IF;
|
|
1540
|
+
|
|
1541
|
+
PERFORM c.relkind
|
|
1542
|
+
FROM pg_class c
|
|
1543
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
1544
|
+
WHERE c.relname = a_table_name
|
|
1545
|
+
AND c.relkind = 'r';
|
|
1546
|
+
|
|
1547
|
+
IF NOT FOUND THEN
|
|
1548
|
+
RAISE NOTICE 'Table %s does not exists', a_table_name;
|
|
1549
|
+
RETURN;
|
|
1550
|
+
END IF;
|
|
1551
|
+
|
|
1552
|
+
SELECT pgmq._get_partition_col(partition_interval) INTO partition_col;
|
|
1553
|
+
|
|
1554
|
+
-- For archive tables, use archived_at for time-based partitioning
|
|
1555
|
+
IF partition_col = 'enqueued_at' THEN
|
|
1556
|
+
a_partition_col := 'archived_at';
|
|
1557
|
+
ELSE
|
|
1558
|
+
a_partition_col := partition_col;
|
|
1559
|
+
END IF;
|
|
1560
|
+
|
|
1561
|
+
EXECUTE 'ALTER TABLE ' || qualified_a_table_name || ' RENAME TO ' || a_table_name_old;
|
|
1562
|
+
|
|
1563
|
+
-- When partitioning by time (archived_at), we need to exclude constraints and indexes
|
|
1564
|
+
-- because the existing PRIMARY KEY on msg_id alone is incompatible with partitioning by archived_at.
|
|
1565
|
+
-- When partitioning by msg_id, we can keep all constraints including PRIMARY KEY.
|
|
1566
|
+
IF a_partition_col = 'archived_at' THEN
|
|
1567
|
+
EXECUTE format( 'CREATE TABLE pgmq.%I (LIKE pgmq.%I including defaults including generated including storage including comments) PARTITION BY RANGE (%I)', a_table_name, a_table_name_old, a_partition_col );
|
|
1568
|
+
ELSE
|
|
1569
|
+
EXECUTE format( 'CREATE TABLE pgmq.%I (LIKE pgmq.%I including all) PARTITION BY RANGE (%I)', a_table_name, a_table_name_old, a_partition_col );
|
|
1570
|
+
END IF;
|
|
1571
|
+
|
|
1572
|
+
EXECUTE 'ALTER INDEX pgmq.archived_at_idx_' || table_name || ' RENAME TO archived_at_idx_' || table_name || '_old';
|
|
1573
|
+
EXECUTE 'CREATE INDEX archived_at_idx_'|| table_name || ' ON ' || qualified_a_table_name ||'(archived_at)';
|
|
1574
|
+
|
|
1575
|
+
-- https://github.com/pgpartman/pg_partman/blob/master/doc/pg_partman.md
|
|
1576
|
+
-- p_parent_table - the existing parent table. MUST be schema qualified, even if in public schema.
|
|
1577
|
+
EXECUTE FORMAT(
|
|
1578
|
+
$QUERY$
|
|
1579
|
+
SELECT %I.create_parent(
|
|
1580
|
+
p_parent_table := %L,
|
|
1581
|
+
p_control := %L,
|
|
1582
|
+
p_interval := %L,
|
|
1583
|
+
p_type := case
|
|
1584
|
+
when pgmq._get_pg_partman_major_version() = 5 then 'range'
|
|
1585
|
+
else 'native'
|
|
1586
|
+
end
|
|
1587
|
+
)
|
|
1588
|
+
$QUERY$,
|
|
1589
|
+
pgmq._get_pg_partman_schema(),
|
|
1590
|
+
qualified_a_table_name,
|
|
1591
|
+
a_partition_col,
|
|
1592
|
+
partition_interval
|
|
1593
|
+
);
|
|
1594
|
+
|
|
1595
|
+
EXECUTE FORMAT(
|
|
1596
|
+
$QUERY$
|
|
1597
|
+
UPDATE %I.part_config
|
|
1598
|
+
SET
|
|
1599
|
+
retention = %L,
|
|
1600
|
+
retention_keep_table = false,
|
|
1601
|
+
retention_keep_index = false,
|
|
1602
|
+
infinite_time_partitions = true
|
|
1603
|
+
WHERE
|
|
1604
|
+
parent_table = %L;
|
|
1605
|
+
$QUERY$,
|
|
1606
|
+
pgmq._get_pg_partman_schema(),
|
|
1607
|
+
retention_interval,
|
|
1608
|
+
qualified_a_table_name
|
|
1609
|
+
);
|
|
1610
|
+
END;
|
|
1611
|
+
$$ LANGUAGE plpgsql;
|
|
1612
|
+
|
|
1613
|
+
CREATE OR REPLACE FUNCTION pgmq.notify_queue_listeners()
|
|
1614
|
+
RETURNS TRIGGER AS $$
|
|
1615
|
+
DECLARE
|
|
1616
|
+
queue_name_extracted TEXT; -- Queue name extracted from trigger table name
|
|
1617
|
+
updated_count INTEGER; -- Number of rows updated (0 or 1)
|
|
1618
|
+
BEGIN
|
|
1619
|
+
queue_name_extracted := substring(TG_TABLE_NAME from 3);
|
|
1620
|
+
|
|
1621
|
+
UPDATE pgmq.notify_insert_throttle
|
|
1622
|
+
SET last_notified_at = clock_timestamp()
|
|
1623
|
+
WHERE queue_name = queue_name_extracted
|
|
1624
|
+
AND (
|
|
1625
|
+
throttle_interval_ms = 0 -- No throttling configured
|
|
1626
|
+
OR clock_timestamp() - last_notified_at >=
|
|
1627
|
+
(throttle_interval_ms * INTERVAL '1 millisecond') -- Throttle interval has elapsed
|
|
1628
|
+
);
|
|
1629
|
+
|
|
1630
|
+
-- Check how many rows were updated (will be 0 or 1)
|
|
1631
|
+
GET DIAGNOSTICS updated_count = ROW_COUNT;
|
|
1632
|
+
|
|
1633
|
+
IF updated_count > 0 THEN
|
|
1634
|
+
PERFORM PG_NOTIFY('pgmq.' || TG_TABLE_NAME || '.' || TG_OP, NULL);
|
|
1635
|
+
END IF;
|
|
1636
|
+
|
|
1637
|
+
RETURN NEW;
|
|
1638
|
+
END;
|
|
1639
|
+
$$ LANGUAGE plpgsql;
|
|
1640
|
+
|
|
1641
|
+
CREATE OR REPLACE FUNCTION pgmq.enable_notify_insert(queue_name TEXT, throttle_interval_ms INTEGER DEFAULT 250)
|
|
1642
|
+
RETURNS void AS $$
|
|
1643
|
+
DECLARE
|
|
1644
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
1645
|
+
v_queue_name TEXT := queue_name;
|
|
1646
|
+
v_throttle_interval_ms INTEGER := throttle_interval_ms;
|
|
1647
|
+
BEGIN
|
|
1648
|
+
-- Validate that throttle_interval_ms is non-negative
|
|
1649
|
+
IF v_throttle_interval_ms < 0 THEN
|
|
1650
|
+
RAISE EXCEPTION 'throttle_interval_ms must be non-negative';
|
|
1651
|
+
END IF;
|
|
1652
|
+
|
|
1653
|
+
-- Validate that the queue table exists
|
|
1654
|
+
IF NOT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'pgmq' AND table_name = qtable) THEN
|
|
1655
|
+
RAISE EXCEPTION 'Queue "%" does not exist. Create it first using pgmq.create()', v_queue_name;
|
|
1656
|
+
END IF;
|
|
1657
|
+
|
|
1658
|
+
PERFORM pgmq.disable_notify_insert(v_queue_name);
|
|
1659
|
+
|
|
1660
|
+
INSERT INTO pgmq.notify_insert_throttle (queue_name, throttle_interval_ms)
|
|
1661
|
+
VALUES (v_queue_name, v_throttle_interval_ms)
|
|
1662
|
+
ON CONFLICT ON CONSTRAINT notify_insert_throttle_queue_name_key DO UPDATE
|
|
1663
|
+
SET throttle_interval_ms = EXCLUDED.throttle_interval_ms,
|
|
1664
|
+
last_notified_at = to_timestamp(0);
|
|
1665
|
+
|
|
1666
|
+
EXECUTE FORMAT(
|
|
1667
|
+
$QUERY$
|
|
1668
|
+
CREATE CONSTRAINT TRIGGER trigger_notify_queue_insert_listeners
|
|
1669
|
+
AFTER INSERT ON pgmq.%I
|
|
1670
|
+
DEFERRABLE FOR EACH ROW
|
|
1671
|
+
EXECUTE PROCEDURE pgmq.notify_queue_listeners()
|
|
1672
|
+
$QUERY$,
|
|
1673
|
+
qtable
|
|
1674
|
+
);
|
|
1675
|
+
END;
|
|
1676
|
+
$$ LANGUAGE plpgsql;
|
|
1677
|
+
|
|
1678
|
+
CREATE OR REPLACE FUNCTION pgmq.disable_notify_insert(queue_name TEXT)
|
|
1679
|
+
RETURNS void AS $$
|
|
1680
|
+
DECLARE
|
|
1681
|
+
qtable TEXT := pgmq.format_table_name(queue_name, 'q');
|
|
1682
|
+
v_queue_name TEXT := queue_name;
|
|
1683
|
+
BEGIN
|
|
1684
|
+
EXECUTE FORMAT(
|
|
1685
|
+
$QUERY$
|
|
1686
|
+
DROP TRIGGER IF EXISTS trigger_notify_queue_insert_listeners ON pgmq.%I;
|
|
1687
|
+
$QUERY$,
|
|
1688
|
+
qtable
|
|
1689
|
+
);
|
|
1690
|
+
|
|
1691
|
+
DELETE FROM pgmq.notify_insert_throttle nit WHERE nit.queue_name = v_queue_name;
|
|
1692
|
+
END;
|
|
1693
|
+
$$ LANGUAGE plpgsql;
|
|
1694
|
+
|
|
1695
|
+
CREATE OR REPLACE FUNCTION pgmq.list_notify_insert_throttles()
|
|
1696
|
+
RETURNS TABLE
|
|
1697
|
+
(
|
|
1698
|
+
queue_name text,
|
|
1699
|
+
throttle_interval_ms integer,
|
|
1700
|
+
last_notified_at TIMESTAMP WITH TIME ZONE
|
|
1701
|
+
)
|
|
1702
|
+
LANGUAGE sql
|
|
1703
|
+
STABLE
|
|
1704
|
+
AS
|
|
1705
|
+
$$
|
|
1706
|
+
SELECT queue_name, throttle_interval_ms, last_notified_at
|
|
1707
|
+
FROM pgmq.notify_insert_throttle
|
|
1708
|
+
ORDER BY queue_name;
|
|
1709
|
+
$$;
|
|
1710
|
+
|
|
1711
|
+
CREATE OR REPLACE FUNCTION pgmq.update_notify_insert(queue_name text, throttle_interval_ms integer)
|
|
1712
|
+
RETURNS void
|
|
1713
|
+
LANGUAGE plpgsql
|
|
1714
|
+
AS
|
|
1715
|
+
$$
|
|
1716
|
+
BEGIN
|
|
1717
|
+
IF throttle_interval_ms < 0 THEN
|
|
1718
|
+
RAISE EXCEPTION 'throttle_interval_ms must be non-negative, got: %', throttle_interval_ms;
|
|
1719
|
+
END IF;
|
|
1720
|
+
|
|
1721
|
+
IF NOT EXISTS (SELECT 1 FROM pgmq.meta WHERE meta.queue_name = update_notify_insert.queue_name) THEN
|
|
1722
|
+
RAISE EXCEPTION 'Queue "%" does not exist. Create the queue first using pgmq.create()', queue_name;
|
|
1723
|
+
END IF;
|
|
1724
|
+
|
|
1725
|
+
IF NOT EXISTS (SELECT 1 FROM pgmq.notify_insert_throttle WHERE notify_insert_throttle.queue_name = update_notify_insert.queue_name) THEN
|
|
1726
|
+
RAISE EXCEPTION 'Queue "%" does not have notify_insert enabled. Enable it first using pgmq.enable_notify_insert()', queue_name;
|
|
1727
|
+
END IF;
|
|
1728
|
+
|
|
1729
|
+
UPDATE pgmq.notify_insert_throttle
|
|
1730
|
+
SET throttle_interval_ms = update_notify_insert.throttle_interval_ms,
|
|
1731
|
+
last_notified_at = to_timestamp(0)
|
|
1732
|
+
WHERE notify_insert_throttle.queue_name = update_notify_insert.queue_name;
|
|
1733
|
+
END;
|
|
1734
|
+
$$;
|
|
1735
|
+
|
|
1736
|
+
CREATE OR REPLACE FUNCTION pgmq.validate_routing_key(routing_key text)
|
|
1737
|
+
RETURNS boolean
|
|
1738
|
+
LANGUAGE plpgsql
|
|
1739
|
+
IMMUTABLE
|
|
1740
|
+
AS
|
|
1741
|
+
$$
|
|
1742
|
+
BEGIN
|
|
1743
|
+
-- Valid routing key examples:
|
|
1744
|
+
-- "logs.error"
|
|
1745
|
+
-- "app.user-service.auth"
|
|
1746
|
+
-- "system_events.db.connection_failed"
|
|
1747
|
+
--
|
|
1748
|
+
-- Invalid routing key examples:
|
|
1749
|
+
-- "" - empty
|
|
1750
|
+
-- ".logs.error" - starts with dot
|
|
1751
|
+
-- "logs.error." - ends with dot
|
|
1752
|
+
-- "logs..error" - consecutive dots
|
|
1753
|
+
-- "logs.error!" - invalid character
|
|
1754
|
+
-- "logs error" - space not allowed
|
|
1755
|
+
-- "logs.*" - wildcards not allowed in routing keys
|
|
1756
|
+
|
|
1757
|
+
IF routing_key IS NULL OR routing_key = '' THEN
|
|
1758
|
+
RAISE EXCEPTION 'routing_key cannot be NULL or empty';
|
|
1759
|
+
END IF;
|
|
1760
|
+
|
|
1761
|
+
IF length(routing_key) > 255 THEN
|
|
1762
|
+
RAISE EXCEPTION 'routing_key length cannot exceed 255 characters, got % characters', length(routing_key);
|
|
1763
|
+
END IF;
|
|
1764
|
+
|
|
1765
|
+
IF routing_key !~ '^[a-zA-Z0-9._-]+$' THEN
|
|
1766
|
+
RAISE EXCEPTION 'routing_key contains invalid characters. Only alphanumeric, dots, hyphens, and underscores are allowed. Got: %', routing_key;
|
|
1767
|
+
END IF;
|
|
1768
|
+
|
|
1769
|
+
IF routing_key ~ '^\.' THEN
|
|
1770
|
+
RAISE EXCEPTION 'routing_key cannot start with a dot. Got: %', routing_key;
|
|
1771
|
+
END IF;
|
|
1772
|
+
|
|
1773
|
+
IF routing_key ~ '\.$' THEN
|
|
1774
|
+
RAISE EXCEPTION 'routing_key cannot end with a dot. Got: %', routing_key;
|
|
1775
|
+
END IF;
|
|
1776
|
+
|
|
1777
|
+
IF routing_key ~ '\.\.' THEN
|
|
1778
|
+
RAISE EXCEPTION 'routing_key cannot contain consecutive dots. Got: %', routing_key;
|
|
1779
|
+
END IF;
|
|
1780
|
+
|
|
1781
|
+
RETURN true;
|
|
1782
|
+
END;
|
|
1783
|
+
$$;
|
|
1784
|
+
|
|
1785
|
+
CREATE OR REPLACE FUNCTION pgmq.validate_topic_pattern(pattern text)
|
|
1786
|
+
RETURNS boolean
|
|
1787
|
+
LANGUAGE plpgsql
|
|
1788
|
+
IMMUTABLE
|
|
1789
|
+
AS
|
|
1790
|
+
$$
|
|
1791
|
+
BEGIN
|
|
1792
|
+
-- Valid pattern examples:
|
|
1793
|
+
-- "logs.*" - matches one segment after logs. (e.g., logs.error, logs.info)
|
|
1794
|
+
-- "logs.#" - matches one or more segments after logs. (e.g., logs.error, logs.api.error)
|
|
1795
|
+
-- "*.error" - matches one segment before .error (e.g., app.error, db.error)
|
|
1796
|
+
-- "#.error" - matches one or more segments before .error (e.g., app.error, x.y.error)
|
|
1797
|
+
-- "app.*.#" - mixed wildcards (one segment then one or more)
|
|
1798
|
+
-- "#" - catch-all pattern, matches any routing key
|
|
1799
|
+
--
|
|
1800
|
+
-- Invalid pattern examples:
|
|
1801
|
+
-- ".logs.*" - starts with dot
|
|
1802
|
+
-- "logs.*." - ends with dot
|
|
1803
|
+
-- "logs..error" - consecutive dots
|
|
1804
|
+
-- "logs.**" - consecutive stars
|
|
1805
|
+
-- "logs.##" - consecutive hashes
|
|
1806
|
+
-- "logs.*#" - adjacent wildcards
|
|
1807
|
+
-- "logs.error!" - invalid character
|
|
1808
|
+
|
|
1809
|
+
IF pattern IS NULL OR pattern = '' THEN
|
|
1810
|
+
RAISE EXCEPTION 'pattern cannot be NULL or empty';
|
|
1811
|
+
END IF;
|
|
1812
|
+
|
|
1813
|
+
IF length(pattern) > 255 THEN
|
|
1814
|
+
RAISE EXCEPTION 'pattern length cannot exceed 255 characters, got % characters', length(pattern);
|
|
1815
|
+
END IF;
|
|
1816
|
+
|
|
1817
|
+
IF pattern !~ '^[a-zA-Z0-9._\-*#]+$' THEN
|
|
1818
|
+
RAISE EXCEPTION 'pattern contains invalid characters. Only alphanumeric, dots, hyphens, underscores, *, and # are allowed. Got: %', pattern;
|
|
1819
|
+
END IF;
|
|
1820
|
+
|
|
1821
|
+
IF pattern ~ '^\.' THEN
|
|
1822
|
+
RAISE EXCEPTION 'pattern cannot start with a dot. Got: %', pattern;
|
|
1823
|
+
END IF;
|
|
1824
|
+
|
|
1825
|
+
IF pattern ~ '\.$' THEN
|
|
1826
|
+
RAISE EXCEPTION 'pattern cannot end with a dot. Got: %', pattern;
|
|
1827
|
+
END IF;
|
|
1828
|
+
|
|
1829
|
+
IF pattern ~ '\.\.' THEN
|
|
1830
|
+
RAISE EXCEPTION 'pattern cannot contain consecutive dots. Got: %', pattern;
|
|
1831
|
+
END IF;
|
|
1832
|
+
|
|
1833
|
+
IF pattern ~ '\*\*' THEN
|
|
1834
|
+
RAISE EXCEPTION 'pattern cannot contain consecutive stars (**). Use # for multi-segment matching. Got: %', pattern;
|
|
1835
|
+
END IF;
|
|
1836
|
+
|
|
1837
|
+
IF pattern ~ '##' THEN
|
|
1838
|
+
RAISE EXCEPTION 'pattern cannot contain consecutive hashes (##). A single # already matches zero or more segments. Got: %', pattern;
|
|
1839
|
+
END IF;
|
|
1840
|
+
|
|
1841
|
+
IF pattern ~ '\*#' OR pattern ~ '#\*' THEN
|
|
1842
|
+
RAISE EXCEPTION 'pattern cannot contain adjacent wildcards (*# or #*). Separate wildcards with dots. Got: %', pattern;
|
|
1843
|
+
END IF;
|
|
1844
|
+
|
|
1845
|
+
RETURN true;
|
|
1846
|
+
END;
|
|
1847
|
+
$$;
|
|
1848
|
+
|
|
1849
|
+
CREATE OR REPLACE FUNCTION pgmq.bind_topic(pattern text, queue_name text)
|
|
1850
|
+
RETURNS void
|
|
1851
|
+
LANGUAGE plpgsql
|
|
1852
|
+
AS
|
|
1853
|
+
$$
|
|
1854
|
+
BEGIN
|
|
1855
|
+
PERFORM pgmq.validate_topic_pattern(pattern);
|
|
1856
|
+
IF queue_name IS NULL OR queue_name = '' THEN
|
|
1857
|
+
RAISE EXCEPTION 'queue_name cannot be NULL or empty';
|
|
1858
|
+
END IF;
|
|
1859
|
+
|
|
1860
|
+
IF NOT EXISTS (SELECT 1 FROM pgmq.meta WHERE meta.queue_name = bind_topic.queue_name) THEN
|
|
1861
|
+
RAISE EXCEPTION 'Queue "%" does not exist. Create the queue first using pgmq.create()', queue_name;
|
|
1862
|
+
END IF;
|
|
1863
|
+
|
|
1864
|
+
INSERT INTO pgmq.topic_bindings (pattern, queue_name)
|
|
1865
|
+
VALUES (pattern, queue_name)
|
|
1866
|
+
ON CONFLICT ON CONSTRAINT topic_bindings_unique_pattern_queue DO NOTHING;
|
|
1867
|
+
END;
|
|
1868
|
+
$$;
|
|
1869
|
+
|
|
1870
|
+
CREATE OR REPLACE FUNCTION pgmq.unbind_topic(pattern text, queue_name text)
|
|
1871
|
+
RETURNS boolean
|
|
1872
|
+
LANGUAGE plpgsql
|
|
1873
|
+
AS
|
|
1874
|
+
$$
|
|
1875
|
+
DECLARE
|
|
1876
|
+
rows_deleted integer;
|
|
1877
|
+
BEGIN
|
|
1878
|
+
IF pattern IS NULL OR pattern = '' THEN
|
|
1879
|
+
RAISE EXCEPTION 'pattern cannot be NULL or empty';
|
|
1880
|
+
END IF;
|
|
1881
|
+
|
|
1882
|
+
IF queue_name IS NULL OR queue_name = '' THEN
|
|
1883
|
+
RAISE EXCEPTION 'queue_name cannot be NULL or empty';
|
|
1884
|
+
END IF;
|
|
1885
|
+
|
|
1886
|
+
DELETE
|
|
1887
|
+
FROM pgmq.topic_bindings
|
|
1888
|
+
WHERE topic_bindings.pattern = unbind_topic.pattern
|
|
1889
|
+
AND topic_bindings.queue_name = unbind_topic.queue_name;
|
|
1890
|
+
|
|
1891
|
+
GET DIAGNOSTICS rows_deleted = ROW_COUNT;
|
|
1892
|
+
|
|
1893
|
+
IF rows_deleted > 0 THEN
|
|
1894
|
+
RETURN true;
|
|
1895
|
+
ELSE
|
|
1896
|
+
RETURN false;
|
|
1897
|
+
END IF;
|
|
1898
|
+
END;
|
|
1899
|
+
$$;
|
|
1900
|
+
|
|
1901
|
+
CREATE OR REPLACE FUNCTION pgmq.test_routing(routing_key text)
|
|
1902
|
+
RETURNS TABLE
|
|
1903
|
+
(
|
|
1904
|
+
pattern text,
|
|
1905
|
+
queue_name text,
|
|
1906
|
+
compiled_regex text
|
|
1907
|
+
)
|
|
1908
|
+
LANGUAGE plpgsql
|
|
1909
|
+
STABLE
|
|
1910
|
+
AS
|
|
1911
|
+
$$
|
|
1912
|
+
BEGIN
|
|
1913
|
+
PERFORM pgmq.validate_routing_key(routing_key);
|
|
1914
|
+
RETURN QUERY
|
|
1915
|
+
SELECT b.pattern,
|
|
1916
|
+
b.queue_name,
|
|
1917
|
+
b.compiled_regex
|
|
1918
|
+
FROM pgmq.topic_bindings b
|
|
1919
|
+
WHERE routing_key ~ b.compiled_regex
|
|
1920
|
+
ORDER BY b.pattern;
|
|
1921
|
+
END;
|
|
1922
|
+
$$;
|
|
1923
|
+
|
|
1924
|
+
CREATE OR REPLACE FUNCTION pgmq.send_topic(routing_key text, msg jsonb, headers jsonb, delay integer)
|
|
1925
|
+
RETURNS integer
|
|
1926
|
+
LANGUAGE plpgsql
|
|
1927
|
+
VOLATILE
|
|
1928
|
+
AS
|
|
1929
|
+
$$
|
|
1930
|
+
DECLARE
|
|
1931
|
+
b RECORD;
|
|
1932
|
+
matched_count integer := 0;
|
|
1933
|
+
BEGIN
|
|
1934
|
+
PERFORM pgmq.validate_routing_key(routing_key);
|
|
1935
|
+
|
|
1936
|
+
IF msg IS NULL THEN
|
|
1937
|
+
RAISE EXCEPTION 'msg cannot be NULL';
|
|
1938
|
+
END IF;
|
|
1939
|
+
|
|
1940
|
+
IF delay < 0 THEN
|
|
1941
|
+
RAISE EXCEPTION 'delay cannot be negative, got: %', delay;
|
|
1942
|
+
END IF;
|
|
1943
|
+
|
|
1944
|
+
-- Filter matching patterns in SQL for better performance (uses index)
|
|
1945
|
+
-- Any failure will rollback the entire transaction
|
|
1946
|
+
FOR b IN
|
|
1947
|
+
SELECT DISTINCT tb.queue_name
|
|
1948
|
+
FROM pgmq.topic_bindings tb
|
|
1949
|
+
WHERE routing_key ~ tb.compiled_regex
|
|
1950
|
+
ORDER BY tb.queue_name -- Deterministic ordering, deduplicated by queue_name
|
|
1951
|
+
LOOP
|
|
1952
|
+
PERFORM pgmq.send(b.queue_name, msg, headers, delay);
|
|
1953
|
+
matched_count := matched_count + 1;
|
|
1954
|
+
END LOOP;
|
|
1955
|
+
|
|
1956
|
+
RETURN matched_count;
|
|
1957
|
+
END;
|
|
1958
|
+
$$;
|
|
1959
|
+
|
|
1960
|
+
CREATE OR REPLACE FUNCTION pgmq.send_topic(routing_key text, msg jsonb)
|
|
1961
|
+
RETURNS integer
|
|
1962
|
+
LANGUAGE plpgsql
|
|
1963
|
+
VOLATILE
|
|
1964
|
+
AS
|
|
1965
|
+
$$
|
|
1966
|
+
BEGIN
|
|
1967
|
+
RETURN pgmq.send_topic(routing_key, msg, NULL, 0);
|
|
1968
|
+
END;
|
|
1969
|
+
$$;
|
|
1970
|
+
|
|
1971
|
+
CREATE OR REPLACE FUNCTION pgmq.send_topic(routing_key text, msg jsonb, delay integer)
|
|
1972
|
+
RETURNS integer
|
|
1973
|
+
LANGUAGE plpgsql
|
|
1974
|
+
VOLATILE
|
|
1975
|
+
AS
|
|
1976
|
+
$$
|
|
1977
|
+
BEGIN
|
|
1978
|
+
RETURN pgmq.send_topic(routing_key, msg, NULL, delay);
|
|
1979
|
+
END;
|
|
1980
|
+
$$;
|
|
1981
|
+
|
|
1982
|
+
CREATE OR REPLACE FUNCTION pgmq.list_topic_bindings()
|
|
1983
|
+
RETURNS TABLE
|
|
1984
|
+
(
|
|
1985
|
+
pattern text,
|
|
1986
|
+
queue_name text,
|
|
1987
|
+
bound_at TIMESTAMP WITH TIME ZONE,
|
|
1988
|
+
compiled_regex text
|
|
1989
|
+
)
|
|
1990
|
+
LANGUAGE sql
|
|
1991
|
+
STABLE
|
|
1992
|
+
AS
|
|
1993
|
+
$$
|
|
1994
|
+
SELECT pattern, queue_name, bound_at, compiled_regex
|
|
1995
|
+
FROM pgmq.topic_bindings
|
|
1996
|
+
ORDER BY bound_at DESC, pattern, queue_name;
|
|
1997
|
+
$$;
|
|
1998
|
+
|
|
1999
|
+
CREATE OR REPLACE FUNCTION pgmq.list_topic_bindings(queue_name text)
|
|
2000
|
+
RETURNS TABLE
|
|
2001
|
+
(
|
|
2002
|
+
pattern text,
|
|
2003
|
+
queue_name text,
|
|
2004
|
+
bound_at TIMESTAMP WITH TIME ZONE,
|
|
2005
|
+
compiled_regex text
|
|
2006
|
+
)
|
|
2007
|
+
LANGUAGE sql
|
|
2008
|
+
STABLE
|
|
2009
|
+
AS
|
|
2010
|
+
$$
|
|
2011
|
+
SELECT pattern, tb.queue_name, bound_at, compiled_regex
|
|
2012
|
+
FROM pgmq.topic_bindings tb
|
|
2013
|
+
WHERE tb.queue_name = list_topic_bindings.queue_name
|
|
2014
|
+
ORDER BY bound_at DESC, pattern;
|
|
2015
|
+
$$;
|
|
2016
|
+
|
|
2017
|
+
-- send_batch_topic: Base implementation with TIMESTAMP WITH TIME ZONE delay
|
|
2018
|
+
CREATE OR REPLACE FUNCTION pgmq.send_batch_topic(
|
|
2019
|
+
routing_key text,
|
|
2020
|
+
msgs jsonb[],
|
|
2021
|
+
headers jsonb[],
|
|
2022
|
+
delay TIMESTAMP WITH TIME ZONE
|
|
2023
|
+
)
|
|
2024
|
+
RETURNS TABLE(queue_name text, msg_id bigint)
|
|
2025
|
+
LANGUAGE plpgsql
|
|
2026
|
+
VOLATILE
|
|
2027
|
+
AS
|
|
2028
|
+
$$
|
|
2029
|
+
DECLARE
|
|
2030
|
+
b RECORD;
|
|
2031
|
+
BEGIN
|
|
2032
|
+
PERFORM pgmq.validate_routing_key(routing_key);
|
|
2033
|
+
|
|
2034
|
+
-- Validate batch parameters once (not per queue)
|
|
2035
|
+
PERFORM pgmq._validate_batch_params(msgs, headers);
|
|
2036
|
+
|
|
2037
|
+
-- Filter matching patterns in SQL for better performance (uses index)
|
|
2038
|
+
-- Any failure will rollback the entire transaction
|
|
2039
|
+
FOR b IN
|
|
2040
|
+
SELECT DISTINCT tb.queue_name
|
|
2041
|
+
FROM pgmq.topic_bindings tb
|
|
2042
|
+
WHERE routing_key ~ tb.compiled_regex
|
|
2043
|
+
ORDER BY tb.queue_name -- Deterministic ordering, deduplicated by queue_name
|
|
2044
|
+
LOOP
|
|
2045
|
+
-- Use private _send_batch to avoid redundant validation
|
|
2046
|
+
RETURN QUERY
|
|
2047
|
+
SELECT b.queue_name, batch_result.msg_id
|
|
2048
|
+
FROM pgmq._send_batch(b.queue_name, msgs, headers, delay) AS batch_result(msg_id);
|
|
2049
|
+
END LOOP;
|
|
2050
|
+
|
|
2051
|
+
RETURN;
|
|
2052
|
+
END;
|
|
2053
|
+
$$;
|
|
2054
|
+
|
|
2055
|
+
-- send_batch_topic: 2 args (routing_key, msgs)
|
|
2056
|
+
CREATE OR REPLACE FUNCTION pgmq.send_batch_topic(
|
|
2057
|
+
routing_key text,
|
|
2058
|
+
msgs jsonb[]
|
|
2059
|
+
)
|
|
2060
|
+
RETURNS TABLE(queue_name text, msg_id bigint)
|
|
2061
|
+
LANGUAGE sql
|
|
2062
|
+
VOLATILE
|
|
2063
|
+
AS
|
|
2064
|
+
$$
|
|
2065
|
+
SELECT * FROM pgmq.send_batch_topic(routing_key, msgs, NULL, clock_timestamp());
|
|
2066
|
+
$$;
|
|
2067
|
+
|
|
2068
|
+
-- send_batch_topic: 3 args with headers
|
|
2069
|
+
CREATE OR REPLACE FUNCTION pgmq.send_batch_topic(
|
|
2070
|
+
routing_key text,
|
|
2071
|
+
msgs jsonb[],
|
|
2072
|
+
headers jsonb[]
|
|
2073
|
+
)
|
|
2074
|
+
RETURNS TABLE(queue_name text, msg_id bigint)
|
|
2075
|
+
LANGUAGE sql
|
|
2076
|
+
VOLATILE
|
|
2077
|
+
AS
|
|
2078
|
+
$$
|
|
2079
|
+
SELECT * FROM pgmq.send_batch_topic(routing_key, msgs, headers, clock_timestamp());
|
|
2080
|
+
$$;
|
|
2081
|
+
|
|
2082
|
+
-- send_batch_topic: 3 args with integer delay
|
|
2083
|
+
CREATE OR REPLACE FUNCTION pgmq.send_batch_topic(
|
|
2084
|
+
routing_key text,
|
|
2085
|
+
msgs jsonb[],
|
|
2086
|
+
delay integer
|
|
2087
|
+
)
|
|
2088
|
+
RETURNS TABLE(queue_name text, msg_id bigint)
|
|
2089
|
+
LANGUAGE sql
|
|
2090
|
+
VOLATILE
|
|
2091
|
+
AS
|
|
2092
|
+
$$
|
|
2093
|
+
SELECT * FROM pgmq.send_batch_topic(routing_key, msgs, NULL, clock_timestamp() + make_interval(secs => delay));
|
|
2094
|
+
$$;
|
|
2095
|
+
|
|
2096
|
+
-- send_batch_topic: 3 args with timestamp delay
|
|
2097
|
+
CREATE OR REPLACE FUNCTION pgmq.send_batch_topic(
|
|
2098
|
+
routing_key text,
|
|
2099
|
+
msgs jsonb[],
|
|
2100
|
+
delay TIMESTAMP WITH TIME ZONE
|
|
2101
|
+
)
|
|
2102
|
+
RETURNS TABLE(queue_name text, msg_id bigint)
|
|
2103
|
+
LANGUAGE sql
|
|
2104
|
+
VOLATILE
|
|
2105
|
+
AS
|
|
2106
|
+
$$
|
|
2107
|
+
SELECT * FROM pgmq.send_batch_topic(routing_key, msgs, NULL, delay);
|
|
2108
|
+
$$;
|
|
2109
|
+
|
|
2110
|
+
-- send_batch_topic: 4 args with integer delay
|
|
2111
|
+
CREATE OR REPLACE FUNCTION pgmq.send_batch_topic(
|
|
2112
|
+
routing_key text,
|
|
2113
|
+
msgs jsonb[],
|
|
2114
|
+
headers jsonb[],
|
|
2115
|
+
delay integer
|
|
2116
|
+
)
|
|
2117
|
+
RETURNS TABLE(queue_name text, msg_id bigint)
|
|
2118
|
+
LANGUAGE sql
|
|
2119
|
+
VOLATILE
|
|
2120
|
+
AS
|
|
2121
|
+
$$
|
|
2122
|
+
SELECT * FROM pgmq.send_batch_topic(routing_key, msgs, headers, clock_timestamp() + make_interval(secs => delay));
|
|
2123
|
+
$$;
|