pgmq-ruby 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +97 -2
- data/README.md +266 -20
- data/lib/pgmq/client/autovacuum.rb +180 -0
- data/lib/pgmq/client/consumer.rb +121 -7
- data/lib/pgmq/client/maintenance.rb +154 -5
- data/lib/pgmq/client/message_lifecycle.rb +6 -7
- data/lib/pgmq/client/metrics.rb +1 -2
- data/lib/pgmq/client/multi_queue.rb +9 -11
- data/lib/pgmq/client/producer.rb +37 -12
- data/lib/pgmq/client/queue_management.rb +87 -5
- data/lib/pgmq/client/topics.rb +22 -12
- data/lib/pgmq/client.rb +49 -34
- data/lib/pgmq/connection.rb +25 -38
- data/lib/pgmq/message.rb +4 -5
- data/lib/pgmq/notify_throttle.rb +25 -0
- data/lib/pgmq/queue_name.rb +153 -0
- data/lib/pgmq/transaction.rb +6 -7
- data/lib/pgmq/version.rb +1 -1
- data/lib/pgmq.rb +2 -3
- metadata +5 -2
data/lib/pgmq/client/producer.rb
CHANGED
|
@@ -4,31 +4,36 @@ module PGMQ
|
|
|
4
4
|
class Client
|
|
5
5
|
# Message producing operations
|
|
6
6
|
#
|
|
7
|
-
# This module handles producing messages to queues, both individual messages
|
|
8
|
-
#
|
|
7
|
+
# This module handles producing messages to queues, both individual messages and batches. Users must serialize
|
|
8
|
+
# messages to JSON strings themselves.
|
|
9
9
|
module Producer
|
|
10
10
|
# Produces a message to a queue
|
|
11
11
|
#
|
|
12
12
|
# @param queue_name [String] name of the queue
|
|
13
13
|
# @param message [String] message as JSON string (for PostgreSQL JSONB)
|
|
14
14
|
# @param headers [String, nil] optional headers as JSON string (for metadata, routing, tracing)
|
|
15
|
-
# @param delay [
|
|
15
|
+
# @param delay [Numeric, Time] delay in seconds before message becomes visible (integer or float),
|
|
16
|
+
# or an absolute Time at which the message becomes visible, including ActiveSupport::TimeWithZone
|
|
17
|
+
# (PGMQ v1.10.0+)
|
|
16
18
|
# @return [String] message ID as string
|
|
17
19
|
#
|
|
18
20
|
# @example Basic produce
|
|
19
21
|
# msg_id = client.produce("orders", '{"order_id":123,"total":99.99}')
|
|
20
22
|
#
|
|
21
|
-
# @example With delay
|
|
23
|
+
# @example With integer delay (seconds)
|
|
22
24
|
# msg_id = client.produce("orders", '{"data":"value"}', delay: 60)
|
|
23
25
|
#
|
|
26
|
+
# @example With absolute timestamp delay
|
|
27
|
+
# msg_id = client.produce("orders", '{"data":"value"}', delay: Time.now + 3600)
|
|
28
|
+
#
|
|
24
29
|
# @example With headers for routing/tracing
|
|
25
30
|
# msg_id = client.produce("orders", '{"order_id":123}',
|
|
26
31
|
# headers: '{"trace_id":"abc123","priority":"high"}')
|
|
27
32
|
#
|
|
28
|
-
# @example With headers and delay
|
|
33
|
+
# @example With headers and absolute timestamp delay
|
|
29
34
|
# msg_id = client.produce("orders", '{"order_id":123}',
|
|
30
35
|
# headers: '{"correlation_id":"req-456"}',
|
|
31
|
-
# delay:
|
|
36
|
+
# delay: Time.now + 300)
|
|
32
37
|
#
|
|
33
38
|
# @note Users must serialize to JSON themselves. Higher-level frameworks
|
|
34
39
|
# should handle serialization.
|
|
@@ -41,11 +46,21 @@ module PGMQ
|
|
|
41
46
|
validate_queue_name!(queue_name)
|
|
42
47
|
|
|
43
48
|
result = with_connection do |conn|
|
|
44
|
-
if headers
|
|
49
|
+
if headers && !delay.is_a?(Numeric)
|
|
50
|
+
conn.exec_params(
|
|
51
|
+
"SELECT * FROM pgmq.send($1::text, $2::jsonb, $3::jsonb, $4::timestamptz)",
|
|
52
|
+
[queue_name, message, headers, delay.to_time.utc.iso8601(6)]
|
|
53
|
+
)
|
|
54
|
+
elsif headers
|
|
45
55
|
conn.exec_params(
|
|
46
56
|
"SELECT * FROM pgmq.send($1::text, $2::jsonb, $3::jsonb, $4::integer)",
|
|
47
57
|
[queue_name, message, headers, delay]
|
|
48
58
|
)
|
|
59
|
+
elsif !delay.is_a?(Numeric)
|
|
60
|
+
conn.exec_params(
|
|
61
|
+
"SELECT * FROM pgmq.send($1::text, $2::jsonb, $3::timestamptz)",
|
|
62
|
+
[queue_name, message, delay.to_time.utc.iso8601(6)]
|
|
63
|
+
)
|
|
49
64
|
else
|
|
50
65
|
conn.exec_params(
|
|
51
66
|
"SELECT * FROM pgmq.send($1::text, $2::jsonb, $3::integer)",
|
|
@@ -62,7 +77,9 @@ module PGMQ
|
|
|
62
77
|
# @param queue_name [String] name of the queue
|
|
63
78
|
# @param messages [Array<String>] array of message payloads as JSON strings
|
|
64
79
|
# @param headers [Array<String>, nil] optional array of headers as JSON strings (must match messages length)
|
|
65
|
-
# @param delay [
|
|
80
|
+
# @param delay [Numeric, Time] delay in seconds before messages become visible (integer or float),
|
|
81
|
+
# or an absolute Time at which the messages become visible, including ActiveSupport::TimeWithZone
|
|
82
|
+
# (PGMQ v1.10.0+)
|
|
66
83
|
# @return [Array<String>] array of message IDs
|
|
67
84
|
# @raise [ArgumentError] if headers array length doesn't match messages length
|
|
68
85
|
#
|
|
@@ -97,19 +114,27 @@ module PGMQ
|
|
|
97
114
|
"headers array length (#{headers.length}) must match messages array length (#{messages.length})"
|
|
98
115
|
end
|
|
99
116
|
|
|
100
|
-
# Use PostgreSQL array parameter binding for security
|
|
101
|
-
# PG gem will properly encode the array values
|
|
102
117
|
result = with_connection do |conn|
|
|
103
|
-
# Create array encoder for proper PostgreSQL array formatting
|
|
104
118
|
encoder = PG::TextEncoder::Array.new
|
|
105
119
|
encoded_messages = encoder.encode(messages)
|
|
106
120
|
|
|
107
|
-
if headers
|
|
121
|
+
if headers && !delay.is_a?(Numeric)
|
|
122
|
+
encoded_headers = encoder.encode(headers)
|
|
123
|
+
conn.exec_params(
|
|
124
|
+
"SELECT * FROM pgmq.send_batch($1::text, $2::jsonb[], $3::jsonb[], $4::timestamptz)",
|
|
125
|
+
[queue_name, encoded_messages, encoded_headers, delay.to_time.utc.iso8601(6)]
|
|
126
|
+
)
|
|
127
|
+
elsif headers
|
|
108
128
|
encoded_headers = encoder.encode(headers)
|
|
109
129
|
conn.exec_params(
|
|
110
130
|
"SELECT * FROM pgmq.send_batch($1::text, $2::jsonb[], $3::jsonb[], $4::integer)",
|
|
111
131
|
[queue_name, encoded_messages, encoded_headers, delay]
|
|
112
132
|
)
|
|
133
|
+
elsif !delay.is_a?(Numeric)
|
|
134
|
+
conn.exec_params(
|
|
135
|
+
"SELECT * FROM pgmq.send_batch($1::text, $2::jsonb[], $3::timestamptz)",
|
|
136
|
+
[queue_name, encoded_messages, delay.to_time.utc.iso8601(6)]
|
|
137
|
+
)
|
|
113
138
|
else
|
|
114
139
|
conn.exec_params(
|
|
115
140
|
"SELECT * FROM pgmq.send_batch($1::text, $2::jsonb[], $3::integer)",
|
|
@@ -4,12 +4,15 @@ module PGMQ
|
|
|
4
4
|
class Client
|
|
5
5
|
# Queue management operations (create, drop, list)
|
|
6
6
|
#
|
|
7
|
-
# This module handles all queue lifecycle operations including creating queues
|
|
8
|
-
#
|
|
7
|
+
# This module handles all queue lifecycle operations including creating queues (standard, partitioned, unlogged),
|
|
8
|
+
# dropping queues, and listing existing queues.
|
|
9
9
|
module QueueManagement
|
|
10
10
|
# Creates a new queue
|
|
11
11
|
#
|
|
12
12
|
# @param queue_name [String] name of the queue to create
|
|
13
|
+
# @param tune_autovacuum [Boolean, Hash] when truthy, tune autovacuum on the new queue's tables after creation.
|
|
14
|
+
# Pass +true+ for PGMQ-tuned defaults, or a Hash of options forwarded to {Autovacuum#tune_autovacuum}
|
|
15
|
+
# (e.g. +{scale_factor: 0.005, archive: false}+). Defaults to +false+ (no change).
|
|
13
16
|
# @return [Boolean] true if queue was created, false if it already existed
|
|
14
17
|
# @raise [PGMQ::Errors::InvalidQueueNameError] if queue name is invalid
|
|
15
18
|
# @raise [PGMQ::Errors::ConnectionError] if database operation fails
|
|
@@ -17,12 +20,16 @@ module PGMQ
|
|
|
17
20
|
# @example
|
|
18
21
|
# client.create("orders") # => true (created)
|
|
19
22
|
# client.create("orders") # => false (already exists)
|
|
20
|
-
|
|
23
|
+
#
|
|
24
|
+
# @example Create with tuned autovacuum
|
|
25
|
+
# client.create("orders", tune_autovacuum: true)
|
|
26
|
+
def create(queue_name, tune_autovacuum: false)
|
|
21
27
|
validate_queue_name!(queue_name)
|
|
22
28
|
|
|
23
29
|
with_connection do |conn|
|
|
24
30
|
existed = queue_exists?(conn, queue_name)
|
|
25
31
|
conn.exec_params("SELECT pgmq.create($1::text)", [queue_name])
|
|
32
|
+
apply_tune_autovacuum_option(conn, queue_name, tune_autovacuum)
|
|
26
33
|
!existed
|
|
27
34
|
end
|
|
28
35
|
end
|
|
@@ -34,6 +41,9 @@ module PGMQ
|
|
|
34
41
|
# @param queue_name [String] name of the queue
|
|
35
42
|
# @param partition_interval [String] partition interval (e.g., "daily", "10000")
|
|
36
43
|
# @param retention_interval [String] retention interval (e.g., "7 days", "100000")
|
|
44
|
+
# @param tune_autovacuum [Boolean, Hash] when truthy, tune autovacuum on the new queue's tables after creation.
|
|
45
|
+
# Pass +true+ for PGMQ-tuned defaults, or a Hash forwarded to {Autovacuum#tune_autovacuum}. Defaults to
|
|
46
|
+
# +false+. Note: storage parameters are set on the partitioned parent and do not cascade to partitions.
|
|
37
47
|
# @return [Boolean] true if queue was created, false if it already existed
|
|
38
48
|
#
|
|
39
49
|
# @example
|
|
@@ -44,7 +54,8 @@ module PGMQ
|
|
|
44
54
|
def create_partitioned(
|
|
45
55
|
queue_name,
|
|
46
56
|
partition_interval: "10000",
|
|
47
|
-
retention_interval: "100000"
|
|
57
|
+
retention_interval: "100000",
|
|
58
|
+
tune_autovacuum: false
|
|
48
59
|
)
|
|
49
60
|
validate_queue_name!(queue_name)
|
|
50
61
|
|
|
@@ -54,6 +65,7 @@ module PGMQ
|
|
|
54
65
|
"SELECT pgmq.create_partitioned($1::text, $2::text, $3::text)",
|
|
55
66
|
[queue_name, partition_interval, retention_interval]
|
|
56
67
|
)
|
|
68
|
+
apply_tune_autovacuum_option(conn, queue_name, tune_autovacuum)
|
|
57
69
|
!existed
|
|
58
70
|
end
|
|
59
71
|
end
|
|
@@ -61,16 +73,20 @@ module PGMQ
|
|
|
61
73
|
# Creates an unlogged queue for higher throughput (no crash recovery)
|
|
62
74
|
#
|
|
63
75
|
# @param queue_name [String] name of the queue
|
|
76
|
+
# @param tune_autovacuum [Boolean, Hash] when truthy, tune autovacuum on the new queue's tables after creation.
|
|
77
|
+
# Pass +true+ for PGMQ-tuned defaults, or a Hash forwarded to {Autovacuum#tune_autovacuum}. Defaults to
|
|
78
|
+
# +false+.
|
|
64
79
|
# @return [Boolean] true if queue was created, false if it already existed
|
|
65
80
|
#
|
|
66
81
|
# @example
|
|
67
82
|
# client.create_unlogged("fast_queue") # => true
|
|
68
|
-
def create_unlogged(queue_name)
|
|
83
|
+
def create_unlogged(queue_name, tune_autovacuum: false)
|
|
69
84
|
validate_queue_name!(queue_name)
|
|
70
85
|
|
|
71
86
|
with_connection do |conn|
|
|
72
87
|
existed = queue_exists?(conn, queue_name)
|
|
73
88
|
conn.exec_params("SELECT pgmq.create_unlogged($1::text)", [queue_name])
|
|
89
|
+
apply_tune_autovacuum_option(conn, queue_name, tune_autovacuum)
|
|
74
90
|
!existed
|
|
75
91
|
end
|
|
76
92
|
end
|
|
@@ -94,6 +110,49 @@ module PGMQ
|
|
|
94
110
|
result[0]["drop_queue"] == "t"
|
|
95
111
|
end
|
|
96
112
|
|
|
113
|
+
# Creates the FIFO index on a queue's table required for grouped reads
|
|
114
|
+
#
|
|
115
|
+
# Grouped read operations (`read_grouped`, `read_grouped_rr`, `read_grouped_head`) rely on this index for correct
|
|
116
|
+
# ordering and acceptable query performance. Without it, grouped reads will work but may be slow or return
|
|
117
|
+
# incorrect ordering at scale. The operation is idempotent - calling it on a queue that already has the index is
|
|
118
|
+
# safe.
|
|
119
|
+
#
|
|
120
|
+
# @param queue_name [String] name of the queue
|
|
121
|
+
# @return [void]
|
|
122
|
+
# @raise [PGMQ::Errors::InvalidQueueNameError] if queue name is invalid
|
|
123
|
+
# @raise [PGMQ::Errors::ConnectionError] if database operation fails
|
|
124
|
+
#
|
|
125
|
+
# @example
|
|
126
|
+
# client.create("tasks")
|
|
127
|
+
# client.create_fifo_index("tasks")
|
|
128
|
+
def create_fifo_index(queue_name)
|
|
129
|
+
validate_queue_name!(queue_name)
|
|
130
|
+
|
|
131
|
+
with_connection do |conn|
|
|
132
|
+
conn.exec_params("SELECT pgmq.create_fifo_index($1::text)", [queue_name])
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
nil
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Creates FIFO indexes on all existing queues
|
|
139
|
+
#
|
|
140
|
+
# Convenience wrapper that calls `create_fifo_index` for every queue registered in `pgmq.meta`. Useful for
|
|
141
|
+
# one-time migrations when adding grouped reads to an existing deployment. The operation is idempotent.
|
|
142
|
+
#
|
|
143
|
+
# @return [void]
|
|
144
|
+
# @raise [PGMQ::Errors::ConnectionError] if database operation fails
|
|
145
|
+
#
|
|
146
|
+
# @example Migrate an existing deployment to use grouped reads
|
|
147
|
+
# client.create_fifo_indexes_all
|
|
148
|
+
def create_fifo_indexes_all
|
|
149
|
+
with_connection do |conn|
|
|
150
|
+
conn.exec("SELECT pgmq.create_fifo_indexes_all()")
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
nil
|
|
154
|
+
end
|
|
155
|
+
|
|
97
156
|
# Lists all queues
|
|
98
157
|
#
|
|
99
158
|
# @return [Array<PGMQ::QueueMetadata>] array of queue metadata objects
|
|
@@ -123,6 +182,29 @@ module PGMQ
|
|
|
123
182
|
)
|
|
124
183
|
result.ntuples.positive?
|
|
125
184
|
end
|
|
185
|
+
|
|
186
|
+
# Applies the +tune_autovacuum:+ creation option on the create connection.
|
|
187
|
+
#
|
|
188
|
+
# Accepts the same shapes the create methods document: +false+/+nil+ (no-op), +true+ (PGMQ-tuned defaults), or a
|
|
189
|
+
# Hash of {Autovacuum#tune_autovacuum} keyword options (+queue_settings:+, +archive:+, +archive_settings:+).
|
|
190
|
+
# Delegates resolution and the ALTER TABLE statements to {Autovacuum#tune_autovacuum_on}, the single source of
|
|
191
|
+
# truth shared with {Autovacuum#tune_autovacuum}, so defaults and the archive-skip rule cannot drift between the
|
|
192
|
+
# two paths. Runs on the connection that just created the queue, so the ALTER TABLE shares that checkout rather
|
|
193
|
+
# than acquiring a second one.
|
|
194
|
+
#
|
|
195
|
+
# @param conn [PG::Connection] the connection the queue was created on
|
|
196
|
+
# @param queue_name [String] name of the queue
|
|
197
|
+
# @param option [Boolean, Hash] the tune_autovacuum option as passed to the create method
|
|
198
|
+
# @option option [Hash] :queue_settings queue-table storage params, merged onto the defaults
|
|
199
|
+
# @option option [Boolean] :archive also tune the archive table (default: true)
|
|
200
|
+
# @option option [Hash] :archive_settings archive-table storage params, merged onto the defaults
|
|
201
|
+
# @return [void]
|
|
202
|
+
def apply_tune_autovacuum_option(conn, queue_name, option)
|
|
203
|
+
return unless option
|
|
204
|
+
|
|
205
|
+
opts = option.is_a?(Hash) ? option.transform_keys(&:to_sym) : {}
|
|
206
|
+
tune_autovacuum_on(conn, queue_name, **opts)
|
|
207
|
+
end
|
|
126
208
|
end
|
|
127
209
|
end
|
|
128
210
|
end
|
data/lib/pgmq/client/topics.rb
CHANGED
|
@@ -4,8 +4,8 @@ module PGMQ
|
|
|
4
4
|
class Client
|
|
5
5
|
# Topic routing operations (AMQP-like patterns)
|
|
6
6
|
#
|
|
7
|
-
# This module provides AMQP-style topic routing for PGMQ, allowing messages
|
|
8
|
-
#
|
|
7
|
+
# This module provides AMQP-style topic routing for PGMQ, allowing messages to be routed to multiple queues based on
|
|
8
|
+
# pattern matching.
|
|
9
9
|
#
|
|
10
10
|
# Topic patterns support wildcards:
|
|
11
11
|
# - `*` matches exactly one word between dots (e.g., `orders.*` matches `orders.new`)
|
|
@@ -15,8 +15,7 @@ module PGMQ
|
|
|
15
15
|
module Topics
|
|
16
16
|
# Binds a topic pattern to a queue
|
|
17
17
|
#
|
|
18
|
-
# Messages sent with routing keys matching this pattern will be delivered
|
|
19
|
-
# to the specified queue.
|
|
18
|
+
# Messages sent with routing keys matching this pattern will be delivered to the specified queue.
|
|
20
19
|
#
|
|
21
20
|
# @param pattern [String] topic pattern with optional wildcards (* or #)
|
|
22
21
|
# @param queue_name [String] name of the queue to bind
|
|
@@ -66,8 +65,7 @@ module PGMQ
|
|
|
66
65
|
|
|
67
66
|
# Sends a message via topic routing
|
|
68
67
|
#
|
|
69
|
-
# The message will be delivered to all queues whose bound patterns match
|
|
70
|
-
# the routing key.
|
|
68
|
+
# The message will be delivered to all queues whose bound patterns match the routing key.
|
|
71
69
|
#
|
|
72
70
|
# @param routing_key [String] dot-separated routing key (e.g., "orders.new.priority")
|
|
73
71
|
# @param message [String] message as JSON string
|
|
@@ -108,13 +106,14 @@ module PGMQ
|
|
|
108
106
|
|
|
109
107
|
# Sends multiple messages via topic routing
|
|
110
108
|
#
|
|
111
|
-
# All messages will be delivered to all queues whose bound patterns match
|
|
112
|
-
# the routing key.
|
|
109
|
+
# All messages will be delivered to all queues whose bound patterns match the routing key.
|
|
113
110
|
#
|
|
114
111
|
# @param routing_key [String] dot-separated routing key
|
|
115
112
|
# @param messages [Array<String>] array of message payloads as JSON strings
|
|
116
113
|
# @param headers [Array<String>, nil] optional array of headers as JSON strings
|
|
117
|
-
# @param delay [
|
|
114
|
+
# @param delay [Numeric, Time] delay in seconds before messages become visible (integer or float),
|
|
115
|
+
# or an absolute Time at which the messages become visible, including ActiveSupport::TimeWithZone
|
|
116
|
+
# (PGMQ v1.10.0+)
|
|
118
117
|
# @return [Array<Hash>] array of hashes with :queue_name and :msg_id
|
|
119
118
|
#
|
|
120
119
|
# @example Batch topic send
|
|
@@ -135,12 +134,23 @@ module PGMQ
|
|
|
135
134
|
encoder = PG::TextEncoder::Array.new
|
|
136
135
|
encoded_messages = encoder.encode(messages)
|
|
137
136
|
|
|
138
|
-
if headers
|
|
137
|
+
if headers && !delay.is_a?(Numeric)
|
|
138
|
+
encoded_headers = encoder.encode(headers)
|
|
139
|
+
conn.exec_params(
|
|
140
|
+
"SELECT * FROM pgmq.send_batch_topic($1::text, $2::jsonb[], $3::jsonb[], $4::timestamptz)",
|
|
141
|
+
[routing_key, encoded_messages, encoded_headers, delay.to_time.utc.iso8601(6)]
|
|
142
|
+
)
|
|
143
|
+
elsif headers
|
|
139
144
|
encoded_headers = encoder.encode(headers)
|
|
140
145
|
conn.exec_params(
|
|
141
146
|
"SELECT * FROM pgmq.send_batch_topic($1::text, $2::jsonb[], $3::jsonb[], $4::integer)",
|
|
142
147
|
[routing_key, encoded_messages, encoded_headers, delay]
|
|
143
148
|
)
|
|
149
|
+
elsif !delay.is_a?(Numeric)
|
|
150
|
+
conn.exec_params(
|
|
151
|
+
"SELECT * FROM pgmq.send_batch_topic($1::text, $2::jsonb[], $3::timestamptz)",
|
|
152
|
+
[routing_key, encoded_messages, delay.to_time.utc.iso8601(6)]
|
|
153
|
+
)
|
|
144
154
|
elsif delay > 0
|
|
145
155
|
conn.exec_params(
|
|
146
156
|
"SELECT * FROM pgmq.send_batch_topic($1::text, $2::jsonb[], $3::integer)",
|
|
@@ -217,8 +227,8 @@ module PGMQ
|
|
|
217
227
|
|
|
218
228
|
# Validates a routing key
|
|
219
229
|
#
|
|
220
|
-
# Routing keys are dot-separated words (no wildcards allowed).
|
|
221
|
-
#
|
|
230
|
+
# Routing keys are dot-separated words (no wildcards allowed). Returns false for invalid routing keys (PGMQ raises
|
|
231
|
+
# an error for invalid keys).
|
|
222
232
|
#
|
|
223
233
|
# @param routing_key [String] routing key to validate
|
|
224
234
|
# @return [Boolean] true if valid, false if invalid
|
data/lib/pgmq/client.rb
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
module PGMQ
|
|
4
4
|
# Low-level client for interacting with PGMQ (Postgres Message Queue)
|
|
5
5
|
#
|
|
6
|
-
# This is a thin wrapper around PGMQ SQL functions. For higher-level
|
|
7
|
-
#
|
|
6
|
+
# This is a thin wrapper around PGMQ SQL functions. For higher-level abstractions (job processing, retries, Rails
|
|
7
|
+
# integration), use pgmq-framework.
|
|
8
8
|
#
|
|
9
9
|
# @example Basic usage
|
|
10
10
|
# client = PGMQ::Client.new(
|
|
@@ -34,6 +34,7 @@ module PGMQ
|
|
|
34
34
|
include Maintenance # Queue maintenance (purge, notifications)
|
|
35
35
|
include Metrics # Monitoring and metrics
|
|
36
36
|
include Topics # Topic routing (AMQP-like patterns, PGMQ v1.11.0+)
|
|
37
|
+
include Autovacuum # Autovacuum tuning for queue/archive tables
|
|
37
38
|
|
|
38
39
|
# Default visibility timeout in seconds
|
|
39
40
|
DEFAULT_VT = 30
|
|
@@ -93,47 +94,61 @@ module PGMQ
|
|
|
93
94
|
@connection.stats
|
|
94
95
|
end
|
|
95
96
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
#
|
|
99
|
-
#
|
|
100
|
-
#
|
|
97
|
+
# Checks out a pooled connection and yields the raw +PG::Connection+ for arbitrary SQL.
|
|
98
|
+
#
|
|
99
|
+
# All PGMQ operations run through this method internally, so it carries the same guarantees: a connection is taken
|
|
100
|
+
# from the pool, health-checked when +auto_reconnect+ is enabled, and a single retry on a fresh connection is
|
|
101
|
+
# attempted if the checked-out connection turns out to be dead before any SQL is sent (see
|
|
102
|
+
# {PGMQ::Connection#with_connection}). The connection is returned to the pool when the block exits.
|
|
103
|
+
#
|
|
104
|
+
# Exposed so callers can issue PostgreSQL statements PGMQ does not wrap without standing up a second pool: ad-hoc
|
|
105
|
+
# +NOTIFY+, +LISTEN+, advisory locks, custom monitoring queries, or DDL that lives alongside your queue tables.
|
|
106
|
+
# Reusing the PGMQ pool keeps connection accounting in one place.
|
|
107
|
+
#
|
|
108
|
+
# @yield [PG::Connection] a pooled, health-checked database connection
|
|
109
|
+
# @return [Object] the return value of the block
|
|
110
|
+
# @raise [PGMQ::Errors::ConnectionError] if a connection cannot be obtained
|
|
111
|
+
#
|
|
112
|
+
# @example Fire a custom NOTIFY on the PGMQ pool
|
|
113
|
+
# client.with_connection do |conn|
|
|
114
|
+
# conn.exec_params("SELECT pg_notify($1, $2)", ["my_channel", payload])
|
|
115
|
+
# end
|
|
116
|
+
#
|
|
117
|
+
# @example Run a query PGMQ does not wrap
|
|
118
|
+
# depth = client.with_connection do |conn|
|
|
119
|
+
# conn.exec("SELECT count(*) FROM pgmq.q_orders")[0]["count"].to_i
|
|
120
|
+
# end
|
|
121
|
+
#
|
|
122
|
+
# @note You receive the raw +PG::Connection+. PGMQ performs no type mapping (results come back as strings) and does
|
|
123
|
+
# not wrap your statement in a transaction. Use {#transaction} when you need atomicity.
|
|
124
|
+
#
|
|
125
|
+
# @note Do not retain or use the yielded connection outside the block. Once the block returns, the connection
|
|
126
|
+
# goes back to the pool and another thread may check it out; +PG::Connection+ is not thread-safe, so using it
|
|
127
|
+
# after the block can corrupt libpq state (nil results, wrong data, segfaults).
|
|
128
|
+
#
|
|
129
|
+
# @note The connection is returned to the pool *without* resetting session state. Anything session-scoped you
|
|
130
|
+
# create - +LISTEN+, +SET+, a session-level advisory lock (+pg_advisory_lock+), a prepared statement, a temp
|
|
131
|
+
# table - survives check-in and leaks to the next pool user. Undo it before the block exits (+UNLISTEN+,
|
|
132
|
+
# +RESET+, +pg_advisory_unlock+, etc.). For LISTEN/NOTIFY consumption, prefer {#wait_for_notify}, which manages
|
|
133
|
+
# +LISTEN+/+UNLISTEN+ for you.
|
|
101
134
|
def with_connection(&)
|
|
102
135
|
@connection.with_connection(&)
|
|
103
136
|
end
|
|
104
137
|
|
|
138
|
+
private
|
|
139
|
+
|
|
105
140
|
# Validates a queue name
|
|
141
|
+
#
|
|
142
|
+
# Delegates to {PGMQ::QueueName.validate!}, the single source of truth for queue-name rules. See
|
|
143
|
+
# {PGMQ::QueueName} for the +normalize+ and +sanitize+ helpers if you need to derive a valid name from
|
|
144
|
+
# friendlier or untrusted input before calling a queue operation.
|
|
145
|
+
#
|
|
106
146
|
# @param queue_name [String] queue name to validate
|
|
107
147
|
# @return [void]
|
|
108
148
|
# @raise [PGMQ::Errors::InvalidQueueNameError] if name is invalid
|
|
109
149
|
def validate_queue_name!(queue_name)
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
Errors::InvalidQueueNameError,
|
|
113
|
-
"Queue name cannot be empty"
|
|
114
|
-
)
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
# PGMQ creates tables with prefixes (pgmq.q_<name>, pgmq.a_<name>)
|
|
118
|
-
# PostgreSQL has a 63-character limit for identifiers, but PGMQ enforces 48
|
|
119
|
-
# to account for prefixes and potential suffixes
|
|
120
|
-
if queue_name.to_s.length >= 48
|
|
121
|
-
raise(
|
|
122
|
-
Errors::InvalidQueueNameError,
|
|
123
|
-
"Queue name '#{queue_name}' exceeds maximum length of 48 characters " \
|
|
124
|
-
"(current length: #{queue_name.to_s.length})"
|
|
125
|
-
)
|
|
126
|
-
end
|
|
127
|
-
|
|
128
|
-
# PostgreSQL identifier rules: start with letter or underscore,
|
|
129
|
-
# contain only letters, digits, underscores
|
|
130
|
-
return if queue_name.to_s.match?(/\A[a-zA-Z_][a-zA-Z0-9_]*\z/)
|
|
131
|
-
|
|
132
|
-
raise(
|
|
133
|
-
Errors::InvalidQueueNameError,
|
|
134
|
-
"Invalid queue name '#{queue_name}': must start with a letter or underscore " \
|
|
135
|
-
"and contain only letters, digits, and underscores"
|
|
136
|
-
)
|
|
150
|
+
QueueName.validate!(queue_name)
|
|
151
|
+
nil
|
|
137
152
|
end
|
|
138
153
|
end
|
|
139
154
|
end
|
data/lib/pgmq/connection.rb
CHANGED
|
@@ -27,14 +27,11 @@ module PGMQ
|
|
|
27
27
|
DEFAULT_POOL_TIMEOUT = 5
|
|
28
28
|
|
|
29
29
|
class << self
|
|
30
|
-
# Additional error message patterns (String or Regexp) that mean the
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
# message. The built-in LOST_CONNECTION_MESSAGES are always checked
|
|
34
|
-
# first — this list is appended to them.
|
|
30
|
+
# Additional error message patterns (String or Regexp) that mean the connection is dead and a retry on a fresh
|
|
31
|
+
# socket is safe. Strings are matched as case-insensitive substrings; Regexps match the original message. The
|
|
32
|
+
# built-in LOST_CONNECTION_MESSAGES are always checked first - this list is appended to them.
|
|
35
33
|
#
|
|
36
|
-
# Thread-safe: reads are lock-free (frozen array swap); writes should
|
|
37
|
-
# be done at boot time before forking workers.
|
|
34
|
+
# Thread-safe: reads are lock-free (frozen array swap); writes should be done at boot time before forking workers.
|
|
38
35
|
#
|
|
39
36
|
# @return [Array<String, Regexp>]
|
|
40
37
|
# @example
|
|
@@ -42,9 +39,8 @@ module PGMQ
|
|
|
42
39
|
# PGMQ::Connection.reconnectable_error_patterns << /\Abroken pipe\b/i
|
|
43
40
|
attr_reader :reconnectable_error_patterns
|
|
44
41
|
|
|
45
|
-
# Additional exception classes that mean the connection is dead.
|
|
46
|
-
#
|
|
47
|
-
# list is appended to them. Subclasses also match.
|
|
42
|
+
# Additional exception classes that mean the connection is dead. `PG::ConnectionBad` and `PG::UnableToSend` are
|
|
43
|
+
# always matched - this list is appended to them. Subclasses also match.
|
|
48
44
|
#
|
|
49
45
|
# Thread-safe: reads are lock-free; writes should be done at boot time.
|
|
50
46
|
#
|
|
@@ -73,9 +69,8 @@ module PGMQ
|
|
|
73
69
|
|
|
74
70
|
# Normalizes user-supplied reconnectable error patterns.
|
|
75
71
|
#
|
|
76
|
-
# Strings are downcased once at configuration time so the hot path
|
|
77
|
-
#
|
|
78
|
-
# passed through unchanged.
|
|
72
|
+
# Strings are downcased once at configuration time so the hot path (`connection_lost_error?`) only does substring
|
|
73
|
+
# checks. Regexps are passed through unchanged.
|
|
79
74
|
#
|
|
80
75
|
# @param patterns [Array<String, Regexp>, String, Regexp, nil]
|
|
81
76
|
# @return [Array<String, Regexp>]
|
|
@@ -198,11 +193,9 @@ module PGMQ
|
|
|
198
193
|
|
|
199
194
|
private
|
|
200
195
|
|
|
201
|
-
# Messages libpq raises when the server/pooler has already torn down the
|
|
202
|
-
#
|
|
203
|
-
#
|
|
204
|
-
# libpq raises as `PG::ConnectionBad` or `PG::UnableToSend` without
|
|
205
|
-
# waiting for a new message to hit production.
|
|
196
|
+
# Messages libpq raises when the server/pooler has already torn down the socket. The list has grown organically with
|
|
197
|
+
# each pooler/TLS variant we see in the wild; the class check below catches future variants that libpq raises as
|
|
198
|
+
# `PG::ConnectionBad` or `PG::UnableToSend` without waiting for a new message to hit production.
|
|
206
199
|
LOST_CONNECTION_MESSAGES = [
|
|
207
200
|
"server closed the connection",
|
|
208
201
|
"connection not open",
|
|
@@ -220,13 +213,10 @@ module PGMQ
|
|
|
220
213
|
|
|
221
214
|
# Checks if the error indicates a lost connection.
|
|
222
215
|
#
|
|
223
|
-
# Matches in three steps: first by class (`PG::ConnectionBad` /
|
|
224
|
-
#
|
|
225
|
-
#
|
|
226
|
-
#
|
|
227
|
-
# libpq doesn't reach for the specific subclass, and finally by
|
|
228
|
-
# user-supplied patterns (strings matched as case-insensitive
|
|
229
|
-
# substrings, Regexps matched against the original message).
|
|
216
|
+
# Matches in three steps: first by class (`PG::ConnectionBad` / `PG::UnableToSend` are dedicated connection-failure
|
|
217
|
+
# classes libpq raises regardless of message, plus any user-supplied classes), then by built-in message substrings
|
|
218
|
+
# for the bare `PG::Error` cases where libpq doesn't reach for the specific subclass, and finally by user-supplied
|
|
219
|
+
# patterns (strings matched as case-insensitive substrings, Regexps matched against the original message).
|
|
230
220
|
#
|
|
231
221
|
# @param error [PG::Error] the error to check
|
|
232
222
|
# @return [Boolean] true if the connection was lost and a retry on a
|
|
@@ -252,11 +242,9 @@ module PGMQ
|
|
|
252
242
|
|
|
253
243
|
# Verifies a connection is alive and working.
|
|
254
244
|
#
|
|
255
|
-
# Also resets when the connection reports `PG::CONNECTION_BAD`, which
|
|
256
|
-
#
|
|
257
|
-
#
|
|
258
|
-
# still exists. `#finished?` alone only catches connections closed
|
|
259
|
-
# explicitly from the client side.
|
|
245
|
+
# Also resets when the connection reports `PG::CONNECTION_BAD`, which happens when the server (or an intermediate
|
|
246
|
+
# pooler such as PgBouncer) has closed the socket while the client-side `PG::Connection` object still exists.
|
|
247
|
+
# `#finished?` alone only catches connections closed explicitly from the client side.
|
|
260
248
|
#
|
|
261
249
|
# @param conn [PG::Connection] connection to verify
|
|
262
250
|
# @raise [PG::Error] if the reset itself fails
|
|
@@ -306,16 +294,15 @@ module PGMQ
|
|
|
306
294
|
ConnectionPool.new(size: @pool_size, timeout: @pool_timeout) do
|
|
307
295
|
conn = create_connection(params)
|
|
308
296
|
|
|
309
|
-
# Detect shared connections: if a callable returns the same PG::Connection
|
|
310
|
-
#
|
|
311
|
-
# (nil results, segfaults, wrong data). Fail fast with a clear message.
|
|
297
|
+
# Detect shared connections: if a callable returns the same PG::Connection object to multiple pool slots,
|
|
298
|
+
# concurrent use will corrupt libpq state (nil results, segfaults, wrong data). Fail fast with a clear message.
|
|
312
299
|
if conn.is_a?(PG::Connection)
|
|
313
300
|
seen_mutex.synchronize do
|
|
314
301
|
if seen_connections.key?(conn)
|
|
315
302
|
raise PGMQ::Errors::ConfigurationError,
|
|
316
303
|
"Connection callable returned the same PG::Connection object " \
|
|
317
304
|
"(object_id: #{conn.object_id}) to multiple pool slots. " \
|
|
318
|
-
"PG::Connection is NOT thread-safe
|
|
305
|
+
"PG::Connection is NOT thread-safe - concurrent use causes nil results, " \
|
|
319
306
|
"segfaults, and data corruption. Ensure your callable returns a unique " \
|
|
320
307
|
"PG::Connection instance on each invocation (for example, by calling " \
|
|
321
308
|
"PG.connect inside the callable)."
|
|
@@ -338,10 +325,10 @@ module PGMQ
|
|
|
338
325
|
# If we have a callable (e.g., for Rails), call it to get the connection
|
|
339
326
|
return params.call if params.respond_to?(:call)
|
|
340
327
|
|
|
341
|
-
# Create new connection from parameters
|
|
342
|
-
# Low-level library: return all values as strings from PostgreSQL
|
|
343
|
-
# No automatic type conversion - let higher-level frameworks handle parsing
|
|
344
|
-
# conn.type_map_for_results intentionally NOT set
|
|
328
|
+
# Create new connection from parameters.
|
|
329
|
+
# Low-level library: return all values as strings from PostgreSQL.
|
|
330
|
+
# No automatic type conversion - let higher-level frameworks handle parsing.
|
|
331
|
+
# conn.type_map_for_results intentionally NOT set.
|
|
345
332
|
PG.connect(params[:conninfo] || params)
|
|
346
333
|
rescue PG::Error => e
|
|
347
334
|
raise PGMQ::Errors::ConnectionError, "Failed to connect to database: #{e.message}"
|
data/lib/pgmq/message.rb
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
module PGMQ
|
|
4
4
|
# Represents a message read from a PGMQ queue
|
|
5
5
|
#
|
|
6
|
-
# Returns raw values from PostgreSQL without transformation.
|
|
7
|
-
#
|
|
6
|
+
# Returns raw values from PostgreSQL without transformation. Higher-level frameworks should handle parsing,
|
|
7
|
+
# deserialization, etc.
|
|
8
8
|
#
|
|
9
9
|
# @example Reading a message (raw values)
|
|
10
10
|
# msg = client.read("my_queue", vt: 30)
|
|
@@ -24,9 +24,8 @@ module PGMQ
|
|
|
24
24
|
# @param row [Hash] database row from PG result
|
|
25
25
|
# @return [Message]
|
|
26
26
|
def new(row, **)
|
|
27
|
-
# Return raw values as-is from PostgreSQL
|
|
28
|
-
#
|
|
29
|
-
# The pg gem returns JSONB as String by default
|
|
27
|
+
# Return raw values as-is from PostgreSQL. No parsing, no deserialization, no transformation.
|
|
28
|
+
# The pg gem returns JSONB as String by default.
|
|
30
29
|
super(
|
|
31
30
|
msg_id: row["msg_id"],
|
|
32
31
|
read_ct: row["read_ct"],
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PGMQ
|
|
4
|
+
# Represents the NOTIFY throttle configuration for a queue
|
|
5
|
+
#
|
|
6
|
+
# @example Inspecting notification configuration
|
|
7
|
+
# throttles = client.list_notify_insert_throttles
|
|
8
|
+
# throttles.each do |t|
|
|
9
|
+
# puts "#{t.queue_name}: #{t.throttle_interval_ms}ms (last notified: #{t.last_notified_at})"
|
|
10
|
+
# end
|
|
11
|
+
class NotifyThrottle < Data.define(:queue_name, :throttle_interval_ms, :last_notified_at)
|
|
12
|
+
class << self
|
|
13
|
+
# Creates a new NotifyThrottle from a database row
|
|
14
|
+
# @param row [Hash] database row from PG result
|
|
15
|
+
# @return [NotifyThrottle]
|
|
16
|
+
def new(row, **)
|
|
17
|
+
super(
|
|
18
|
+
queue_name: row["queue_name"],
|
|
19
|
+
throttle_interval_ms: row["throttle_interval_ms"],
|
|
20
|
+
last_notified_at: row["last_notified_at"]
|
|
21
|
+
)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|