pgmq-ruby 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +97 -2
- data/README.md +266 -20
- data/lib/pgmq/client/autovacuum.rb +180 -0
- data/lib/pgmq/client/consumer.rb +121 -7
- data/lib/pgmq/client/maintenance.rb +154 -5
- data/lib/pgmq/client/message_lifecycle.rb +6 -7
- data/lib/pgmq/client/metrics.rb +1 -2
- data/lib/pgmq/client/multi_queue.rb +9 -11
- data/lib/pgmq/client/producer.rb +37 -12
- data/lib/pgmq/client/queue_management.rb +87 -5
- data/lib/pgmq/client/topics.rb +22 -12
- data/lib/pgmq/client.rb +49 -34
- data/lib/pgmq/connection.rb +25 -38
- data/lib/pgmq/message.rb +4 -5
- data/lib/pgmq/notify_throttle.rb +25 -0
- data/lib/pgmq/queue_name.rb +153 -0
- data/lib/pgmq/transaction.rb +6 -7
- data/lib/pgmq/version.rb +1 -1
- data/lib/pgmq.rb +2 -3
- metadata +5 -2
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PGMQ
|
|
4
|
+
class Client
|
|
5
|
+
# Autovacuum and storage tuning for queue and archive tables.
|
|
6
|
+
#
|
|
7
|
+
# PGMQ tables churn in a way PostgreSQL's defaults are not tuned for. A hot queue inserts, updates, and deletes
|
|
8
|
+
# rows constantly: every read UPDATEs +vt+, +read_ct+, and +last_read_at+, and every read+archive cycle deletes
|
|
9
|
+
# from the queue table and inserts into the archive. Two defaults hurt under that load:
|
|
10
|
+
#
|
|
11
|
+
# - +autovacuum_vacuum_scale_factor+ defaults to 0.2, so autovacuum only runs after dead tuples reach 20% of the
|
|
12
|
+
# table - by which point a busy queue has bloated its heap and B-tree indexes, slowing every read and lock.
|
|
13
|
+
# - +fillfactor+ defaults to 100, so heap pages fill completely. Because +vt+ is indexed and changes on every
|
|
14
|
+
# read, those UPDATEs are not HOT-eligible; leaving page headroom reduces page density between vacuum passes.
|
|
15
|
+
#
|
|
16
|
+
# This module applies per-table storage parameters via +ALTER TABLE+ so autovacuum runs far more often (and
|
|
17
|
+
# fillfactor reserves churn headroom) on these specific tables, without touching cluster-wide settings. It is
|
|
18
|
+
# intentionally opt-in: the gem is a thin wrapper and does not mutate table storage parameters unless you ask it
|
|
19
|
+
# to (either by calling {#tune_autovacuum} or by passing +tune_autovacuum: true+ to a queue-creation method).
|
|
20
|
+
#
|
|
21
|
+
# @see https://www.postgresql.org/docs/current/runtime-config-autovacuum.html
|
|
22
|
+
# @see https://planetscale.com/blog/keeping-a-postgres-queue-healthy
|
|
23
|
+
module Autovacuum
|
|
24
|
+
# Default storage parameters for the active queue table. Aggressive: autovacuum and autoanalyze trigger at 1%/5%
|
|
25
|
+
# dead-or-changed tuples, a small +cost_delay+ keeps each vacuum pass quick, and +fillfactor+ reserves 30% of
|
|
26
|
+
# each page for the per-read UPDATE churn (the queue table is the only one that benefits from fillfactor).
|
|
27
|
+
DEFAULT_QUEUE_SETTINGS = {
|
|
28
|
+
autovacuum_vacuum_scale_factor: 0.01,
|
|
29
|
+
autovacuum_vacuum_threshold: 50,
|
|
30
|
+
autovacuum_vacuum_cost_delay: 2,
|
|
31
|
+
autovacuum_analyze_scale_factor: 0.05,
|
|
32
|
+
fillfactor: 70
|
|
33
|
+
}.freeze
|
|
34
|
+
|
|
35
|
+
# Default storage parameters for the archive table. Archives are append-heavy with periodic purge, so a looser
|
|
36
|
+
# scale factor and a slightly higher cost delay are enough while still beating the 0.2 default. No fillfactor:
|
|
37
|
+
# archive rows are inserted once and never updated, so there is no page-churn headroom to reserve.
|
|
38
|
+
DEFAULT_ARCHIVE_SETTINGS = {
|
|
39
|
+
autovacuum_vacuum_scale_factor: 0.05,
|
|
40
|
+
autovacuum_vacuum_threshold: 50,
|
|
41
|
+
autovacuum_vacuum_cost_delay: 5,
|
|
42
|
+
autovacuum_analyze_scale_factor: 0.05
|
|
43
|
+
}.freeze
|
|
44
|
+
|
|
45
|
+
# Storage parameters whose values are interpolated as Floats; everything else is interpolated as an Integer.
|
|
46
|
+
FLOAT_SETTINGS = %i[
|
|
47
|
+
autovacuum_vacuum_scale_factor
|
|
48
|
+
autovacuum_analyze_scale_factor
|
|
49
|
+
].freeze
|
|
50
|
+
private_constant :FLOAT_SETTINGS
|
|
51
|
+
|
|
52
|
+
# Tunes autovacuum and storage parameters on a queue's underlying tables.
|
|
53
|
+
#
|
|
54
|
+
# Applies {DEFAULT_QUEUE_SETTINGS} to the queue table (+pgmq.q_<name>+) and, unless +archive: false+,
|
|
55
|
+
# {DEFAULT_ARCHIVE_SETTINGS} to the archive table (+pgmq.a_<name>+). Pass +queue_settings:+ / +archive_settings:+
|
|
56
|
+
# to override or extend the parameters per table; the Hash you pass is merged onto the defaults, so you only
|
|
57
|
+
# name the keys you want to change. Any other PostgreSQL storage default is left untouched. Safe to call
|
|
58
|
+
# repeatedly - +ALTER TABLE ... SET+ is idempotent for a given value.
|
|
59
|
+
#
|
|
60
|
+
# @param queue_name [String] name of the queue
|
|
61
|
+
# @param queue_settings [Hash{Symbol=>Numeric}] storage params for the queue table, merged onto
|
|
62
|
+
# {DEFAULT_QUEUE_SETTINGS}
|
|
63
|
+
# @param archive [Boolean] also tune the archive table (default: true)
|
|
64
|
+
# @param archive_settings [Hash{Symbol=>Numeric}] storage params for the archive table, merged onto
|
|
65
|
+
# {DEFAULT_ARCHIVE_SETTINGS}
|
|
66
|
+
# @return [void]
|
|
67
|
+
# @raise [PGMQ::Errors::InvalidQueueNameError] if the queue name is invalid
|
|
68
|
+
# @raise [PGMQ::Errors::ConnectionError] if the database operation fails (e.g. the queue does not exist)
|
|
69
|
+
#
|
|
70
|
+
# @example Apply PGMQ-tuned defaults to an existing queue
|
|
71
|
+
# client.tune_autovacuum("orders")
|
|
72
|
+
#
|
|
73
|
+
# @example Override a couple of queue params, skip the archive
|
|
74
|
+
# client.tune_autovacuum("orders", queue_settings: { autovacuum_vacuum_scale_factor: 0.005, fillfactor: 80 },
|
|
75
|
+
# archive: false)
|
|
76
|
+
#
|
|
77
|
+
# @note On a partitioned queue or archive, the parameters are set on the parent table. PostgreSQL does not
|
|
78
|
+
# cascade storage parameters to existing partitions, so pre-existing partitions keep their own settings;
|
|
79
|
+
# set them per-partition if needed.
|
|
80
|
+
def tune_autovacuum(queue_name, queue_settings: {}, archive: true, archive_settings: {})
|
|
81
|
+
validate_queue_name!(queue_name)
|
|
82
|
+
|
|
83
|
+
with_connection do |conn|
|
|
84
|
+
tune_autovacuum_on(
|
|
85
|
+
conn,
|
|
86
|
+
queue_name,
|
|
87
|
+
queue_settings: queue_settings,
|
|
88
|
+
archive: archive,
|
|
89
|
+
archive_settings: archive_settings
|
|
90
|
+
)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
nil
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
private
|
|
97
|
+
|
|
98
|
+
# Resolves storage settings against the PGMQ-tuned defaults and applies them on an existing connection.
|
|
99
|
+
#
|
|
100
|
+
# Single source of truth for both entry points: {#tune_autovacuum} (which passes its keyword arguments through)
|
|
101
|
+
# and the +tune_autovacuum:+ creation flag (which passes a possibly-empty Hash). Keeping the default resolution
|
|
102
|
+
# and the archive-skip rule here means the two paths cannot drift - +archive+ is a single truthiness test in one
|
|
103
|
+
# place, and per-table overrides are merged onto the defaults in one place.
|
|
104
|
+
#
|
|
105
|
+
# @param conn [PG::Connection] the connection to run the ALTER TABLE statements on
|
|
106
|
+
# @param queue_name [String] name of the queue (already validated)
|
|
107
|
+
# @param queue_settings [Hash] queue-table overrides, merged onto {DEFAULT_QUEUE_SETTINGS}
|
|
108
|
+
# @param archive [Boolean] also tune the archive table
|
|
109
|
+
# @param archive_settings [Hash] archive-table overrides, merged onto {DEFAULT_ARCHIVE_SETTINGS}
|
|
110
|
+
# @return [void]
|
|
111
|
+
def tune_autovacuum_on(conn, queue_name, queue_settings: {}, archive: true, archive_settings: {})
|
|
112
|
+
alter_storage(conn, "q_#{queue_name}", DEFAULT_QUEUE_SETTINGS.merge(symbolize(queue_settings)))
|
|
113
|
+
|
|
114
|
+
return unless archive
|
|
115
|
+
|
|
116
|
+
alter_storage(conn, "a_#{queue_name}", DEFAULT_ARCHIVE_SETTINGS.merge(symbolize(archive_settings)))
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Issues a single ALTER TABLE setting the given storage parameters on one pgmq table.
|
|
120
|
+
#
|
|
121
|
+
# PGMQ folds queue names to lower case when it creates the backing tables (`pgmq.create('MyQueue')` produces
|
|
122
|
+
# `pgmq.q_myqueue`), so the table name is lower-cased to match the physical table before it is quoted. Without
|
|
123
|
+
# this, a mixed-case queue name would build `q_MyQueue`, and +quote_ident+ would preserve that case and target a
|
|
124
|
+
# non-existent relation.
|
|
125
|
+
#
|
|
126
|
+
# Identifiers cannot be passed as bind parameters, so the table name is quoted with the connection's
|
|
127
|
+
# +quote_ident+. The queue name has already passed {#validate_queue_name!} (strict identifier rules); the
|
|
128
|
+
# parameter names are validated against the known default keys; and the values are coerced to Float/Integer, so
|
|
129
|
+
# no untrusted text reaches the statement.
|
|
130
|
+
#
|
|
131
|
+
# @param conn [PG::Connection] database connection
|
|
132
|
+
# @param table [String] unqualified table name (e.g. "q_orders")
|
|
133
|
+
# @param settings [Hash{Symbol=>Numeric}] storage parameters to set
|
|
134
|
+
# @return [void]
|
|
135
|
+
def alter_storage(conn, table, settings)
|
|
136
|
+
qualified = "pgmq.#{conn.quote_ident(table.downcase)}"
|
|
137
|
+
clause = settings.map { |key, value| "#{storage_param_name(key)} = #{coerce_setting(key, value)}" }.join(", ")
|
|
138
|
+
|
|
139
|
+
conn.exec("ALTER TABLE #{qualified} SET (#{clause})")
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Validates a storage-parameter name against the known keys before it is interpolated into SQL.
|
|
143
|
+
#
|
|
144
|
+
# Parameter names cannot be bound; allow-listing them against the union of the queue and archive default keys
|
|
145
|
+
# keeps an arbitrary Symbol from being injected as a raw identifier.
|
|
146
|
+
#
|
|
147
|
+
# @param key [Symbol] storage parameter name
|
|
148
|
+
# @return [String] the validated parameter name
|
|
149
|
+
# @raise [ArgumentError] if the key is not a recognised storage parameter
|
|
150
|
+
def storage_param_name(key)
|
|
151
|
+
return key.to_s if known_storage_params.include?(key)
|
|
152
|
+
|
|
153
|
+
raise ArgumentError, "Unknown storage parameter #{key.inspect}; allowed: #{known_storage_params.to_a.sort}"
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# @return [Set<Symbol>] the union of queue and archive default keys
|
|
157
|
+
def known_storage_params
|
|
158
|
+
@known_storage_params ||= (DEFAULT_QUEUE_SETTINGS.keys + DEFAULT_ARCHIVE_SETTINGS.keys).to_set
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Coerces a setting value to its SQL form: Float for scale factors, Integer for everything else. Raising on
|
|
162
|
+
# non-numeric input is the injection guard for values.
|
|
163
|
+
#
|
|
164
|
+
# @param key [Symbol] storage parameter name
|
|
165
|
+
# @param value [Numeric, String] the value to coerce
|
|
166
|
+
# @return [Float, Integer]
|
|
167
|
+
def coerce_setting(key, value)
|
|
168
|
+
FLOAT_SETTINGS.include?(key) ? Float(value) : Integer(value)
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# Symbolizes top-level keys of a settings Hash so callers may pass either Symbol or String keys.
|
|
172
|
+
#
|
|
173
|
+
# @param settings [Hash]
|
|
174
|
+
# @return [Hash{Symbol=>Object}]
|
|
175
|
+
def symbolize(settings)
|
|
176
|
+
settings.to_h.transform_keys(&:to_sym)
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
data/lib/pgmq/client/consumer.rb
CHANGED
|
@@ -4,8 +4,8 @@ module PGMQ
|
|
|
4
4
|
class Client
|
|
5
5
|
# Single-queue message reading operations
|
|
6
6
|
#
|
|
7
|
-
# This module handles reading messages from a single queue, including basic reads,
|
|
8
|
-
#
|
|
7
|
+
# This module handles reading messages from a single queue, including basic reads, batch reads, and long-polling for
|
|
8
|
+
# efficient message consumption.
|
|
9
9
|
module Consumer
|
|
10
10
|
# Reads a message from the queue
|
|
11
11
|
#
|
|
@@ -151,11 +151,126 @@ module PGMQ
|
|
|
151
151
|
result.map { |row| Message.new(row) }
|
|
152
152
|
end
|
|
153
153
|
|
|
154
|
+
# Reads messages using SQS-style grouped ordering (throughput-optimised)
|
|
155
|
+
#
|
|
156
|
+
# Messages are grouped by the first key in their JSON payload. Unlike round-robin, this strategy fills the
|
|
157
|
+
# requested batch from the oldest group first, then moves on to the next group only when the first is exhausted.
|
|
158
|
+
# Maximises throughput for bursty workloads at the cost of fairness across groups.
|
|
159
|
+
#
|
|
160
|
+
# @param queue_name [String] name of the queue
|
|
161
|
+
# @param vt [Integer] visibility timeout in seconds
|
|
162
|
+
# @param qty [Integer] number of messages to read
|
|
163
|
+
# @return [Array<PGMQ::Message>] array of messages, oldest group first
|
|
164
|
+
#
|
|
165
|
+
# @example Throughput-first batch processing
|
|
166
|
+
# # Queue contains: user1_msg1, user1_msg2, user2_msg1
|
|
167
|
+
# messages = client.read_grouped("tasks", vt: 30, qty: 3)
|
|
168
|
+
# # Returns: user1_msg1, user1_msg2, user2_msg1 (drains user1 first)
|
|
169
|
+
#
|
|
170
|
+
# @example High-volume processing where fairness is not required
|
|
171
|
+
# loop do
|
|
172
|
+
# messages = client.read_grouped("jobs", vt: 30, qty: 20)
|
|
173
|
+
# break if messages.empty?
|
|
174
|
+
# messages.each { |msg| process(msg) }
|
|
175
|
+
# end
|
|
176
|
+
def read_grouped(queue_name, vt: DEFAULT_VT, qty: 1)
|
|
177
|
+
validate_queue_name!(queue_name)
|
|
178
|
+
|
|
179
|
+
result = with_connection do |conn|
|
|
180
|
+
conn.exec_params(
|
|
181
|
+
"SELECT * FROM pgmq.read_grouped($1::text, $2::integer, $3::integer)",
|
|
182
|
+
[queue_name, vt, qty]
|
|
183
|
+
)
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
result.map { |row| Message.new(row) }
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
# Reads messages using SQS-style grouped ordering with long-polling support
|
|
190
|
+
#
|
|
191
|
+
# Combines SQS-style throughput-first grouped ordering with long-polling. Blocks up to max_poll_seconds if the
|
|
192
|
+
# queue is empty, returning as soon as any message arrives.
|
|
193
|
+
#
|
|
194
|
+
# @param queue_name [String] name of the queue
|
|
195
|
+
# @param vt [Integer] visibility timeout in seconds
|
|
196
|
+
# @param qty [Integer] number of messages to read
|
|
197
|
+
# @param max_poll_seconds [Integer] maximum time to poll in seconds
|
|
198
|
+
# @param poll_interval_ms [Integer] interval between polls in milliseconds
|
|
199
|
+
# @return [Array<PGMQ::Message>] array of messages
|
|
200
|
+
#
|
|
201
|
+
# @example Poll with throughput-first grouped ordering
|
|
202
|
+
# messages = client.read_grouped_with_poll("jobs",
|
|
203
|
+
# vt: 30,
|
|
204
|
+
# qty: 10,
|
|
205
|
+
# max_poll_seconds: 5,
|
|
206
|
+
# poll_interval_ms: 100
|
|
207
|
+
# )
|
|
208
|
+
def read_grouped_with_poll(
|
|
209
|
+
queue_name,
|
|
210
|
+
vt: DEFAULT_VT,
|
|
211
|
+
qty: 1,
|
|
212
|
+
max_poll_seconds: 5,
|
|
213
|
+
poll_interval_ms: 100
|
|
214
|
+
)
|
|
215
|
+
validate_queue_name!(queue_name)
|
|
216
|
+
|
|
217
|
+
result = with_connection do |conn|
|
|
218
|
+
conn.exec_params(
|
|
219
|
+
"SELECT * FROM pgmq.read_grouped_with_poll($1::text, $2::integer, $3::integer, $4::integer, $5::integer)",
|
|
220
|
+
[queue_name, vt, qty, max_poll_seconds, poll_interval_ms]
|
|
221
|
+
)
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
result.map { |row| Message.new(row) }
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Reads one message per FIFO group from the head of each group
|
|
228
|
+
#
|
|
229
|
+
# Returns exactly one message - the oldest visible message - from each distinct FIFO group, up to qty groups.
|
|
230
|
+
# Groups are determined by the `x-pgmq-group` key in the message headers (set via the `headers:` param on
|
|
231
|
+
# `produce`). Messages without that header key all land in a single implicit default group, so only one of them is
|
|
232
|
+
# returned per call.
|
|
233
|
+
#
|
|
234
|
+
# Unlike `read_grouped` (which groups by the first payload key and drains one group fully before moving to the
|
|
235
|
+
# next), `read_grouped_head` surfaces the leading edge of every group in one call - useful for detecting
|
|
236
|
+
# head-of-line stalls or building per-group progress dashboards.
|
|
237
|
+
#
|
|
238
|
+
# @note Requires PGMQ v1.11.1+.
|
|
239
|
+
#
|
|
240
|
+
# @param queue_name [String] name of the queue
|
|
241
|
+
# @param vt [Integer] visibility timeout in seconds
|
|
242
|
+
# @param qty [Integer] maximum number of groups to sample
|
|
243
|
+
# @return [Array<PGMQ::Message>] one message per group, up to qty
|
|
244
|
+
#
|
|
245
|
+
# @example Sample the head of each group
|
|
246
|
+
# # Produce with x-pgmq-group headers so each tenant is a separate group
|
|
247
|
+
# client.produce("tasks", '{"job":"build"}', headers: '{"x-pgmq-group":"tenant_a"}')
|
|
248
|
+
# client.produce("tasks", '{"job":"test"}', headers: '{"x-pgmq-group":"tenant_b"}')
|
|
249
|
+
# messages = client.read_grouped_head("tasks", vt: 30, qty: 10)
|
|
250
|
+
# # Returns: one message from tenant_a and one from tenant_b
|
|
251
|
+
#
|
|
252
|
+
# @example Monitor for stuck groups
|
|
253
|
+
# heads = client.read_grouped_head("jobs", vt: 30, qty: 100)
|
|
254
|
+
# heads.each do |msg|
|
|
255
|
+
# alert_if_stuck(msg) if msg.enqueued_at < Time.now - 3600
|
|
256
|
+
# end
|
|
257
|
+
def read_grouped_head(queue_name, vt: DEFAULT_VT, qty: 1)
|
|
258
|
+
validate_queue_name!(queue_name)
|
|
259
|
+
|
|
260
|
+
result = with_connection do |conn|
|
|
261
|
+
conn.exec_params(
|
|
262
|
+
"SELECT * FROM pgmq.read_grouped_head($1::text, $2::integer, $3::integer)",
|
|
263
|
+
[queue_name, vt, qty]
|
|
264
|
+
)
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
result.map { |row| Message.new(row) }
|
|
268
|
+
end
|
|
269
|
+
|
|
154
270
|
# Reads messages using grouped round-robin ordering
|
|
155
271
|
#
|
|
156
|
-
# Messages are grouped by the first key in their JSON payload and returned
|
|
157
|
-
#
|
|
158
|
-
# messages from different entities (users, orders, etc.) are in the queue.
|
|
272
|
+
# Messages are grouped by the first key in their JSON payload and returned in round-robin order across groups.
|
|
273
|
+
# This ensures fair processing when messages from different entities (users, orders, etc.) are in the queue.
|
|
159
274
|
#
|
|
160
275
|
# @param queue_name [String] name of the queue
|
|
161
276
|
# @param vt [Integer] visibility timeout in seconds
|
|
@@ -188,8 +303,7 @@ module PGMQ
|
|
|
188
303
|
|
|
189
304
|
# Reads messages using grouped round-robin with long-polling support
|
|
190
305
|
#
|
|
191
|
-
# Combines grouped round-robin ordering with long-polling for efficient
|
|
192
|
-
# and fair message consumption.
|
|
306
|
+
# Combines grouped round-robin ordering with long-polling for efficient and fair message consumption.
|
|
193
307
|
#
|
|
194
308
|
# @param queue_name [String] name of the queue
|
|
195
309
|
# @param vt [Integer] visibility timeout in seconds
|
|
@@ -4,8 +4,7 @@ module PGMQ
|
|
|
4
4
|
class Client
|
|
5
5
|
# Queue maintenance operations
|
|
6
6
|
#
|
|
7
|
-
# This module handles queue maintenance tasks such as purging messages
|
|
8
|
-
# and detaching archive tables.
|
|
7
|
+
# This module handles queue maintenance tasks such as purging messages and detaching archive tables.
|
|
9
8
|
module Maintenance
|
|
10
9
|
# Purges all messages from a queue
|
|
11
10
|
#
|
|
@@ -27,9 +26,8 @@ module PGMQ
|
|
|
27
26
|
|
|
28
27
|
# Enables PostgreSQL NOTIFY when messages are inserted into a queue
|
|
29
28
|
#
|
|
30
|
-
# When enabled, PostgreSQL will send a NOTIFY event on message insert,
|
|
31
|
-
#
|
|
32
|
-
# prevents notification storms during high-volume inserts.
|
|
29
|
+
# When enabled, PostgreSQL will send a NOTIFY event on message insert, allowing clients to use LISTEN instead of
|
|
30
|
+
# polling. The throttle interval prevents notification storms during high-volume inserts.
|
|
33
31
|
#
|
|
34
32
|
# @param queue_name [String] name of the queue
|
|
35
33
|
# @param throttle_interval_ms [Integer] minimum ms between notifications (default: 250)
|
|
@@ -56,6 +54,105 @@ module PGMQ
|
|
|
56
54
|
nil
|
|
57
55
|
end
|
|
58
56
|
|
|
57
|
+
# Converts a standard queue's archive table to a pg_partman-managed partitioned table
|
|
58
|
+
#
|
|
59
|
+
# Provides a migration path for queues originally created with `create` or `create_unlogged` whose archive tables
|
|
60
|
+
# have grown large enough to benefit from partitioning. The queue message table is not affected - only the archive
|
|
61
|
+
# table (`pgmq.a_<queue_name>`) is converted.
|
|
62
|
+
#
|
|
63
|
+
# The operation renames the existing archive table to `pgmq.a_<queue_name>_old`, creates a new partitioned table
|
|
64
|
+
# with the same schema, and hands it over to pg_partman for lifecycle management. **Existing archived rows are
|
|
65
|
+
# left in the `_old` table and must be migrated manually** if visibility in the new partitioned archive is needed.
|
|
66
|
+
# If the archive table is already partitioned the function returns without error (idempotent). If the archive
|
|
67
|
+
# table does not exist it also returns without error.
|
|
68
|
+
#
|
|
69
|
+
# @note Requires the `pg_partman` PostgreSQL extension. If pg_partman is not installed and the archive table
|
|
70
|
+
# exists, the call raises `PGMQ::Errors::ConnectionError`. If the archive table does not exist the call
|
|
71
|
+
# succeeds (returns nil) without touching pg_partman, so no extension is needed in that case.
|
|
72
|
+
#
|
|
73
|
+
# @param queue_name [String] name of the queue whose archive table to convert
|
|
74
|
+
# @param partition_interval [String] partition interval passed to pg_partman (default: "10000" rows or a time
|
|
75
|
+
# expression such as "daily" / "1 month")
|
|
76
|
+
# @param retention_interval [String] retention interval passed to pg_partman (default: "100000")
|
|
77
|
+
# @param leading_partition [Integer] number of leading partitions pg_partman should pre-create (default: 10)
|
|
78
|
+
# @return [void]
|
|
79
|
+
# @raise [PGMQ::Errors::InvalidQueueNameError] if queue name is invalid
|
|
80
|
+
# @raise [PGMQ::Errors::ConnectionError] if the database operation fails (e.g. pg_partman not installed)
|
|
81
|
+
#
|
|
82
|
+
# @example Convert with default partitioning (row-count based)
|
|
83
|
+
# client.convert_archive_partitioned("orders")
|
|
84
|
+
#
|
|
85
|
+
# @example Convert with time-based daily partitioning
|
|
86
|
+
# client.convert_archive_partitioned("orders",
|
|
87
|
+
# partition_interval: "daily",
|
|
88
|
+
# retention_interval: "30 days"
|
|
89
|
+
# )
|
|
90
|
+
def convert_archive_partitioned(
|
|
91
|
+
queue_name,
|
|
92
|
+
partition_interval: "10000",
|
|
93
|
+
retention_interval: "100000",
|
|
94
|
+
leading_partition: 10
|
|
95
|
+
)
|
|
96
|
+
validate_queue_name!(queue_name)
|
|
97
|
+
|
|
98
|
+
with_connection do |conn|
|
|
99
|
+
conn.exec_params(
|
|
100
|
+
"SELECT pgmq.convert_archive_partitioned($1::text, $2::text, $3::text, $4::integer)",
|
|
101
|
+
[queue_name, partition_interval, retention_interval, leading_partition]
|
|
102
|
+
)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
nil
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Lists all queues that have a NOTIFY trigger enabled, with their throttle configuration
|
|
109
|
+
#
|
|
110
|
+
# Returns one {PGMQ::NotifyThrottle} per queue that has had {#enable_notify_insert} called on it.
|
|
111
|
+
# Useful for auditing notification configuration across all queues at once.
|
|
112
|
+
#
|
|
113
|
+
# @return [Array<PGMQ::NotifyThrottle>] throttle configs (empty array if none configured)
|
|
114
|
+
#
|
|
115
|
+
# @example
|
|
116
|
+
# throttles = client.list_notify_insert_throttles
|
|
117
|
+
# throttles.each do |t|
|
|
118
|
+
# puts "#{t.queue_name}: #{t.throttle_interval_ms}ms"
|
|
119
|
+
# end
|
|
120
|
+
def list_notify_insert_throttles
|
|
121
|
+
result = with_connection do |conn|
|
|
122
|
+
conn.exec("SELECT * FROM pgmq.list_notify_insert_throttles()")
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
result.map { |row| PGMQ::NotifyThrottle.new(row) }
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Updates the throttle interval for an already-enabled NOTIFY trigger
|
|
129
|
+
#
|
|
130
|
+
# Changes how frequently PostgreSQL is allowed to fire a NOTIFY event on message insert, without
|
|
131
|
+
# having to disable and re-enable the trigger. The queue must already have notifications enabled
|
|
132
|
+
# via {#enable_notify_insert}.
|
|
133
|
+
#
|
|
134
|
+
# @param queue_name [String] name of the queue
|
|
135
|
+
# @param throttle_interval_ms [Integer] new minimum ms between notifications
|
|
136
|
+
# @return [void]
|
|
137
|
+
#
|
|
138
|
+
# @example Tighten throttle to 100ms during high-throughput ingestion
|
|
139
|
+
# client.update_notify_insert("orders", throttle_interval_ms: 100)
|
|
140
|
+
#
|
|
141
|
+
# @example Remove throttling entirely
|
|
142
|
+
# client.update_notify_insert("orders", throttle_interval_ms: 0)
|
|
143
|
+
def update_notify_insert(queue_name, throttle_interval_ms:)
|
|
144
|
+
validate_queue_name!(queue_name)
|
|
145
|
+
|
|
146
|
+
with_connection do |conn|
|
|
147
|
+
conn.exec_params(
|
|
148
|
+
"SELECT pgmq.update_notify_insert($1::text, $2::integer)",
|
|
149
|
+
[queue_name, throttle_interval_ms]
|
|
150
|
+
)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
nil
|
|
154
|
+
end
|
|
155
|
+
|
|
59
156
|
# Disables PostgreSQL NOTIFY for a queue
|
|
60
157
|
#
|
|
61
158
|
# @param queue_name [String] name of the queue
|
|
@@ -72,6 +169,58 @@ module PGMQ
|
|
|
72
169
|
|
|
73
170
|
nil
|
|
74
171
|
end
|
|
172
|
+
|
|
173
|
+
# Blocks until a PostgreSQL NOTIFY arrives on the queue's channel or the timeout expires.
|
|
174
|
+
#
|
|
175
|
+
# PGMQ sends notifications on the channel `pgmq.q_<queue_name>.INSERT` whenever a message is inserted (requires
|
|
176
|
+
# {#enable_notify_insert} to be called first). This method issues `LISTEN`, waits for a notification via the `pg`
|
|
177
|
+
# gem's `wait_for_notify`, then issues `UNLISTEN` before returning the connection to the pool.
|
|
178
|
+
#
|
|
179
|
+
# Compared with {PGMQ::Client::Consumer#read_with_poll}, which holds the connection inside a PL/pgSQL loop for
|
|
180
|
+
# the full poll window, `wait_for_notify` releases the connection the moment a notification arrives (or the
|
|
181
|
+
# timeout expires). This makes it more efficient under low message rates where the poll window would otherwise
|
|
182
|
+
# burn idle time.
|
|
183
|
+
#
|
|
184
|
+
# The optional block receives `channel`, `backend_pid`, and `payload` when a notification arrives.
|
|
185
|
+
# The return value mirrors `PG::Connection#wait_for_notify`: the channel name string on success,
|
|
186
|
+
# or `nil` on timeout.
|
|
187
|
+
#
|
|
188
|
+
# @note Orchestration (retry loop, reconnect-on-drop, graceful shutdown) is the caller's responsibility.
|
|
189
|
+
# This method is a thin primitive — it listens once, waits, and returns.
|
|
190
|
+
#
|
|
191
|
+
# @param queue_name [String] name of the queue (must have notifications enabled via {#enable_notify_insert})
|
|
192
|
+
# @param timeout [Numeric, nil] seconds to wait; `nil` blocks indefinitely
|
|
193
|
+
# @return [String, nil] notification channel name, or `nil` if the timeout expired
|
|
194
|
+
#
|
|
195
|
+
# @example Basic usage (wake up when a message arrives, then read it)
|
|
196
|
+
# client.enable_notify_insert("orders")
|
|
197
|
+
#
|
|
198
|
+
# loop do
|
|
199
|
+
# next unless client.wait_for_notify("orders", timeout: 5)
|
|
200
|
+
# msg = client.read("orders", vt: 30)
|
|
201
|
+
# process(msg) if msg
|
|
202
|
+
# end
|
|
203
|
+
#
|
|
204
|
+
# @example Block form (inspect notification metadata)
|
|
205
|
+
# client.wait_for_notify("orders", timeout: 5) do |channel, pid, payload|
|
|
206
|
+
# puts "Notified on #{channel} by backend #{pid}"
|
|
207
|
+
# end
|
|
208
|
+
def wait_for_notify(queue_name, timeout: nil)
|
|
209
|
+
validate_queue_name!(queue_name)
|
|
210
|
+
# PGMQ trigger fires pg_notify('pgmq.q_<queue>.INSERT', NULL)
|
|
211
|
+
channel = "pgmq.q_#{queue_name}.INSERT"
|
|
212
|
+
|
|
213
|
+
with_connection do |conn|
|
|
214
|
+
conn.exec("LISTEN \"#{channel}\"")
|
|
215
|
+
begin
|
|
216
|
+
conn.wait_for_notify(timeout) do |ch, pid, payload|
|
|
217
|
+
yield ch, pid, payload if block_given?
|
|
218
|
+
end
|
|
219
|
+
ensure
|
|
220
|
+
conn.exec("UNLISTEN \"#{channel}\"")
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
end
|
|
75
224
|
end
|
|
76
225
|
end
|
|
77
226
|
end
|
|
@@ -4,8 +4,8 @@ module PGMQ
|
|
|
4
4
|
class Client
|
|
5
5
|
# Message lifecycle operations (pop, delete, archive, visibility timeout)
|
|
6
6
|
#
|
|
7
|
-
# This module handles message state transitions including popping (atomic read+delete),
|
|
8
|
-
#
|
|
7
|
+
# This module handles message state transitions including popping (atomic read+delete), deleting, archiving, and
|
|
8
|
+
# updating visibility timeouts.
|
|
9
9
|
module MessageLifecycle
|
|
10
10
|
# Pops a message (atomic read + delete)
|
|
11
11
|
#
|
|
@@ -104,8 +104,8 @@ module PGMQ
|
|
|
104
104
|
|
|
105
105
|
# Deletes specific messages from multiple queues in a single transaction
|
|
106
106
|
#
|
|
107
|
-
# Efficiently deletes messages across different queues atomically.
|
|
108
|
-
#
|
|
107
|
+
# Efficiently deletes messages across different queues atomically. Useful when processing related messages from
|
|
108
|
+
# different queues.
|
|
109
109
|
#
|
|
110
110
|
# @param deletions [Hash] hash of queue_name => array of msg_ids
|
|
111
111
|
# @return [Hash] hash of queue_name => array of deleted msg_ids
|
|
@@ -307,9 +307,8 @@ module PGMQ
|
|
|
307
307
|
|
|
308
308
|
# Updates visibility timeout for messages across multiple queues in a single transaction
|
|
309
309
|
#
|
|
310
|
-
# Efficiently updates visibility timeouts across different queues atomically.
|
|
311
|
-
#
|
|
312
|
-
# to extend their visibility timeouts together.
|
|
310
|
+
# Efficiently updates visibility timeouts across different queues atomically. Useful when processing related
|
|
311
|
+
# messages from different queues and needing to extend their visibility timeouts together.
|
|
313
312
|
#
|
|
314
313
|
# Supports two modes:
|
|
315
314
|
# - Integer offset (seconds from now): `vt: 60` - messages visible in 60 seconds
|
data/lib/pgmq/client/metrics.rb
CHANGED
|
@@ -4,8 +4,7 @@ module PGMQ
|
|
|
4
4
|
class Client
|
|
5
5
|
# Queue metrics and monitoring
|
|
6
6
|
#
|
|
7
|
-
# This module handles retrieving queue metrics such as queue length,
|
|
8
|
-
# message age, and total message counts.
|
|
7
|
+
# This module handles retrieving queue metrics such as queue length, message age, and total message counts.
|
|
9
8
|
module Metrics
|
|
10
9
|
# Gets metrics for a specific queue
|
|
11
10
|
#
|
|
@@ -4,13 +4,13 @@ module PGMQ
|
|
|
4
4
|
class Client
|
|
5
5
|
# Multi-queue operations
|
|
6
6
|
#
|
|
7
|
-
# This module handles efficient operations across multiple queues using single
|
|
8
|
-
#
|
|
7
|
+
# This module handles efficient operations across multiple queues using single database queries with UNION ALL for
|
|
8
|
+
# optimal performance.
|
|
9
9
|
module MultiQueue
|
|
10
10
|
# Reads from multiple queues in a single query
|
|
11
11
|
#
|
|
12
|
-
# This is the most efficient way to monitor multiple queues with a single
|
|
13
|
-
#
|
|
12
|
+
# This is the most efficient way to monitor multiple queues with a single database connection. Uses UNION ALL to
|
|
13
|
+
# read from all queues in one query.
|
|
14
14
|
#
|
|
15
15
|
# @param queue_names [Array<String>] array of queue names to read from
|
|
16
16
|
# @param vt [Integer] visibility timeout in seconds
|
|
@@ -53,8 +53,7 @@ module PGMQ
|
|
|
53
53
|
# Validate all queue names (prevents SQL injection)
|
|
54
54
|
queue_names.each { |qn| validate_queue_name!(qn) }
|
|
55
55
|
|
|
56
|
-
# Build UNION ALL query for all queues
|
|
57
|
-
# Note: Queue names are validated, so this is safe from SQL injection
|
|
56
|
+
# Build UNION ALL query for all queues. Note: Queue names are validated, so this is safe from SQL injection.
|
|
58
57
|
union_queries = queue_names.map do |queue_name|
|
|
59
58
|
# Escape single quotes in queue name (though validation should prevent this)
|
|
60
59
|
escaped_name = queue_name.gsub("'", "''")
|
|
@@ -76,9 +75,8 @@ module PGMQ
|
|
|
76
75
|
|
|
77
76
|
# Reads from multiple queues with long-polling (waits for messages)
|
|
78
77
|
#
|
|
79
|
-
# Efficiently polls multiple queues waiting for the first available message.
|
|
80
|
-
#
|
|
81
|
-
# or the timeout is reached.
|
|
78
|
+
# Efficiently polls multiple queues waiting for the first available message. This uses a single connection with
|
|
79
|
+
# periodic polling until a message arrives or the timeout is reached.
|
|
82
80
|
#
|
|
83
81
|
# @param queue_names [Array<String>] array of queue names to poll
|
|
84
82
|
# @param vt [Integer] visibility timeout in seconds
|
|
@@ -144,8 +142,8 @@ module PGMQ
|
|
|
144
142
|
|
|
145
143
|
# Pops a message from multiple queues (atomic read + delete)
|
|
146
144
|
#
|
|
147
|
-
# Efficiently gets and immediately deletes the first available message from
|
|
148
|
-
#
|
|
145
|
+
# Efficiently gets and immediately deletes the first available message from any of the specified queues. Uses a
|
|
146
|
+
# single query with UNION ALL.
|
|
149
147
|
#
|
|
150
148
|
# @param queue_names [Array<String>] array of queue names
|
|
151
149
|
# @return [PGMQ::Message, nil] message with queue_name attribute, or nil if no messages
|