pgmq-ruby 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +97 -0
- data/README.md +109 -6
- data/config/locales/errors.yml +46 -0
- data/lib/pgmq/client/autovacuum.rb +22 -33
- data/lib/pgmq/client/consumer.rb +3 -4
- data/lib/pgmq/client/maintenance.rb +11 -7
- data/lib/pgmq/client/message_lifecycle.rb +8 -5
- data/lib/pgmq/client/multi_queue.rb +13 -13
- data/lib/pgmq/client/producer.rb +32 -41
- data/lib/pgmq/client/queue_management.rb +25 -5
- data/lib/pgmq/client/topics.rb +23 -28
- data/lib/pgmq/client.rb +37 -6
- data/lib/pgmq/connection.rb +28 -55
- data/lib/pgmq/consumer.rb +222 -0
- data/lib/pgmq/contractable/contract.rb +193 -0
- data/lib/pgmq/contractable/result.rb +89 -0
- data/lib/pgmq/contractable/rule.rb +8 -0
- data/lib/pgmq/contractable.rb +12 -0
- data/lib/pgmq/contracts/autovacuum.rb +46 -0
- data/lib/pgmq/contracts/batch.rb +21 -0
- data/lib/pgmq/contracts/connection.rb +27 -0
- data/lib/pgmq/contracts/consumer.rb +45 -0
- data/lib/pgmq/contracts/message.rb +16 -0
- data/lib/pgmq/contracts/multi_queue.rb +28 -0
- data/lib/pgmq/contracts.rb +16 -0
- data/lib/pgmq/errors.rb +27 -1
- data/lib/pgmq/metrics.rb +11 -2
- data/lib/pgmq/queue_name.rb +2 -3
- data/lib/pgmq/transaction.rb +50 -3
- data/lib/pgmq/version.rb +1 -1
- data/lib/pgmq.rb +8 -1
- data/pgmq-ruby.gemspec +2 -2
- metadata +23 -4
|
@@ -17,6 +17,8 @@ module PGMQ
|
|
|
17
17
|
# @param qty [Integer] max messages to read per queue
|
|
18
18
|
# @param limit [Integer, nil] max total messages across all queues (nil = all)
|
|
19
19
|
# @return [Array<PGMQ::Message>] array of messages with queue_name attribute
|
|
20
|
+
# @raise [PGMQ::Errors::MultiQueueInvalidError] if queue_names is not an Array, is empty or has more than 50
|
|
21
|
+
# names
|
|
20
22
|
#
|
|
21
23
|
# @example Read from multiple queues (gets first available from any queue)
|
|
22
24
|
# msg = client.read_multi(['orders', 'notifications', 'emails'], vt: 30, limit: 1).first
|
|
@@ -25,7 +27,7 @@ module PGMQ
|
|
|
25
27
|
# @example Read up to 5 messages from any of the queues
|
|
26
28
|
# messages = client.read_multi(['queue1', 'queue2', 'queue3'], vt: 30, qty: 5, limit: 5)
|
|
27
29
|
# messages.each do |msg|
|
|
28
|
-
# puts "Processing #{msg.queue_name}: #{msg.
|
|
30
|
+
# puts "Processing #{msg.queue_name}: #{msg.message}"
|
|
29
31
|
# client.delete(msg.queue_name, msg.msg_id)
|
|
30
32
|
# end
|
|
31
33
|
#
|
|
@@ -46,9 +48,7 @@ module PGMQ
|
|
|
46
48
|
qty: 1,
|
|
47
49
|
limit: nil
|
|
48
50
|
)
|
|
49
|
-
|
|
50
|
-
raise ArgumentError, "queue_names cannot be empty" if queue_names.empty?
|
|
51
|
-
raise ArgumentError, "queue_names cannot exceed 50 queues" if queue_names.size > 50
|
|
51
|
+
MULTI_QUEUE_CONTRACT.validate!({ queue_names: queue_names }, Errors::MultiQueueInvalidError)
|
|
52
52
|
|
|
53
53
|
# Validate all queue names (prevents SQL injection)
|
|
54
54
|
queue_names.each { |qn| validate_queue_name!(qn) }
|
|
@@ -85,6 +85,8 @@ module PGMQ
|
|
|
85
85
|
# @param max_poll_seconds [Integer] maximum seconds to wait for messages
|
|
86
86
|
# @param poll_interval_ms [Integer] milliseconds between polls
|
|
87
87
|
# @return [Array<PGMQ::Message>] array of messages (empty if timeout)
|
|
88
|
+
# @raise [PGMQ::Errors::MultiQueueInvalidError] if queue_names is not an Array, is empty or has more than 50
|
|
89
|
+
# names
|
|
88
90
|
#
|
|
89
91
|
# @example Wait for first available message from any queue
|
|
90
92
|
# msg = client.read_multi_with_poll(
|
|
@@ -104,7 +106,7 @@ module PGMQ
|
|
|
104
106
|
# )
|
|
105
107
|
#
|
|
106
108
|
# messages.each do |msg|
|
|
107
|
-
# process(msg.queue_name, msg.
|
|
109
|
+
# process(msg.queue_name, msg.message)
|
|
108
110
|
# client.delete(msg.queue_name, msg.msg_id)
|
|
109
111
|
# end
|
|
110
112
|
# end
|
|
@@ -116,9 +118,7 @@ module PGMQ
|
|
|
116
118
|
max_poll_seconds: 5,
|
|
117
119
|
poll_interval_ms: 100
|
|
118
120
|
)
|
|
119
|
-
|
|
120
|
-
raise ArgumentError, "queue_names cannot be empty" if queue_names.empty?
|
|
121
|
-
raise ArgumentError, "queue_names cannot exceed 50 queues" if queue_names.size > 50
|
|
121
|
+
MULTI_QUEUE_CONTRACT.validate!({ queue_names: queue_names }, Errors::MultiQueueInvalidError)
|
|
122
122
|
|
|
123
123
|
start_time = Time.now
|
|
124
124
|
poll_interval_seconds = poll_interval_ms / 1000.0
|
|
@@ -147,12 +147,14 @@ module PGMQ
|
|
|
147
147
|
#
|
|
148
148
|
# @param queue_names [Array<String>] array of queue names
|
|
149
149
|
# @return [PGMQ::Message, nil] message with queue_name attribute, or nil if no messages
|
|
150
|
+
# @raise [PGMQ::Errors::MultiQueueInvalidError] if queue_names is not an Array, is empty or has more than 50
|
|
151
|
+
# names
|
|
150
152
|
#
|
|
151
153
|
# @example Get first available message from any queue and auto-delete
|
|
152
154
|
# msg = client.pop_multi(['orders', 'notifications', 'emails'])
|
|
153
155
|
# if msg
|
|
154
156
|
# puts "Got message from #{msg.queue_name}"
|
|
155
|
-
# process(msg.
|
|
157
|
+
# process(msg.message)
|
|
156
158
|
# # No need to delete - already deleted!
|
|
157
159
|
# end
|
|
158
160
|
#
|
|
@@ -160,12 +162,10 @@ module PGMQ
|
|
|
160
162
|
# loop do
|
|
161
163
|
# msg = client.pop_multi(['queue1', 'queue2', 'queue3'])
|
|
162
164
|
# break unless msg
|
|
163
|
-
# process(msg.queue_name, msg.
|
|
165
|
+
# process(msg.queue_name, msg.message)
|
|
164
166
|
# end
|
|
165
167
|
def pop_multi(queue_names)
|
|
166
|
-
|
|
167
|
-
raise ArgumentError, "queue_names cannot be empty" if queue_names.empty?
|
|
168
|
-
raise ArgumentError, "queue_names cannot exceed 50 queues" if queue_names.size > 50
|
|
168
|
+
MULTI_QUEUE_CONTRACT.validate!({ queue_names: queue_names }, Errors::MultiQueueInvalidError)
|
|
169
169
|
|
|
170
170
|
# Validate all queue names
|
|
171
171
|
queue_names.each { |qn| validate_queue_name!(qn) }
|
data/lib/pgmq/client/producer.rb
CHANGED
|
@@ -12,10 +12,12 @@ module PGMQ
|
|
|
12
12
|
# @param queue_name [String] name of the queue
|
|
13
13
|
# @param message [String] message as JSON string (for PostgreSQL JSONB)
|
|
14
14
|
# @param headers [String, nil] optional headers as JSON string (for metadata, routing, tracing)
|
|
15
|
-
# @param delay [Numeric, Time] delay in seconds before message becomes visible (integer or float
|
|
16
|
-
# or an absolute Time at which the message becomes visible,
|
|
17
|
-
# (PGMQ v1.10.0+)
|
|
15
|
+
# @param delay [Numeric, Time] delay in seconds before message becomes visible (integer or float;
|
|
16
|
+
# a float such as 1.5 needs PGMQ v1.10.0+), or an absolute Time at which the message becomes visible,
|
|
17
|
+
# including ActiveSupport::TimeWithZone (PGMQ v1.10.0+)
|
|
18
18
|
# @return [String] message ID as string
|
|
19
|
+
# @raise [PGMQ::Errors::MessageInvalidError] if message is not a String, or headers is neither a String nor
|
|
20
|
+
# nil
|
|
19
21
|
#
|
|
20
22
|
# @example Basic produce
|
|
21
23
|
# msg_id = client.produce("orders", '{"order_id":123,"total":99.99}')
|
|
@@ -44,27 +46,19 @@ module PGMQ
|
|
|
44
46
|
delay: 0
|
|
45
47
|
)
|
|
46
48
|
validate_queue_name!(queue_name)
|
|
49
|
+
MESSAGE_CONTRACT.validate!({ message: message, headers: headers }, Errors::MessageInvalidError)
|
|
50
|
+
delay_sql, delay_value = delay_argument(delay, headers ? 4 : 3)
|
|
47
51
|
|
|
48
52
|
result = with_connection do |conn|
|
|
49
|
-
if headers
|
|
53
|
+
if headers
|
|
50
54
|
conn.exec_params(
|
|
51
|
-
"SELECT * FROM pgmq.send($1::text, $2::jsonb, $3::jsonb,
|
|
52
|
-
[queue_name, message, headers,
|
|
53
|
-
)
|
|
54
|
-
elsif headers
|
|
55
|
-
conn.exec_params(
|
|
56
|
-
"SELECT * FROM pgmq.send($1::text, $2::jsonb, $3::jsonb, $4::integer)",
|
|
57
|
-
[queue_name, message, headers, delay]
|
|
58
|
-
)
|
|
59
|
-
elsif !delay.is_a?(Numeric)
|
|
60
|
-
conn.exec_params(
|
|
61
|
-
"SELECT * FROM pgmq.send($1::text, $2::jsonb, $3::timestamptz)",
|
|
62
|
-
[queue_name, message, delay.to_time.utc.iso8601(6)]
|
|
55
|
+
"SELECT * FROM pgmq.send($1::text, $2::jsonb, $3::jsonb, #{delay_sql})",
|
|
56
|
+
[queue_name, message, headers, delay_value]
|
|
63
57
|
)
|
|
64
58
|
else
|
|
65
59
|
conn.exec_params(
|
|
66
|
-
"SELECT * FROM pgmq.send($1::text, $2::jsonb,
|
|
67
|
-
[queue_name, message,
|
|
60
|
+
"SELECT * FROM pgmq.send($1::text, $2::jsonb, #{delay_sql})",
|
|
61
|
+
[queue_name, message, delay_value]
|
|
68
62
|
)
|
|
69
63
|
end
|
|
70
64
|
end
|
|
@@ -77,11 +71,13 @@ module PGMQ
|
|
|
77
71
|
# @param queue_name [String] name of the queue
|
|
78
72
|
# @param messages [Array<String>] array of message payloads as JSON strings
|
|
79
73
|
# @param headers [Array<String>, nil] optional array of headers as JSON strings (must match messages length)
|
|
80
|
-
# @param delay [Numeric, Time] delay in seconds before messages become visible (integer or float
|
|
81
|
-
# or an absolute Time at which the messages become visible,
|
|
82
|
-
# (PGMQ v1.10.0+)
|
|
74
|
+
# @param delay [Numeric, Time] delay in seconds before messages become visible (integer or float;
|
|
75
|
+
# a float such as 1.5 needs PGMQ v1.10.0+), or an absolute Time at which the messages become visible,
|
|
76
|
+
# including ActiveSupport::TimeWithZone (PGMQ v1.10.0+)
|
|
83
77
|
# @return [Array<String>] array of message IDs
|
|
84
|
-
# @raise [
|
|
78
|
+
# @raise [PGMQ::Errors::MessageInvalidError] if messages is not an Array, headers is neither an Array nor nil,
|
|
79
|
+
# headers length doesn't match messages length, a message is not a String, or a header is neither a String
|
|
80
|
+
# nor nil
|
|
85
81
|
#
|
|
86
82
|
# @example Basic batch produce
|
|
87
83
|
# ids = client.produce_batch("orders", [
|
|
@@ -107,38 +103,33 @@ module PGMQ
|
|
|
107
103
|
delay: 0
|
|
108
104
|
)
|
|
109
105
|
validate_queue_name!(queue_name)
|
|
106
|
+
BATCH_CONTRACT.validate!({ messages: messages, headers: headers }, Errors::MessageInvalidError)
|
|
110
107
|
return [] if messages.empty?
|
|
111
108
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
109
|
+
messages.each_with_index do |message, index|
|
|
110
|
+
MESSAGE_CONTRACT.validate!(
|
|
111
|
+
{ message: message, headers: headers && headers[index] },
|
|
112
|
+
Errors::MessageInvalidError,
|
|
113
|
+
scope: ["messages", index]
|
|
114
|
+
)
|
|
115
115
|
end
|
|
116
116
|
|
|
117
117
|
result = with_connection do |conn|
|
|
118
118
|
encoder = PG::TextEncoder::Array.new
|
|
119
119
|
encoded_messages = encoder.encode(messages)
|
|
120
120
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
"SELECT * FROM pgmq.send_batch($1::text, $2::jsonb[], $3::jsonb[], $4::timestamptz)",
|
|
125
|
-
[queue_name, encoded_messages, encoded_headers, delay.to_time.utc.iso8601(6)]
|
|
126
|
-
)
|
|
127
|
-
elsif headers
|
|
121
|
+
delay_sql, delay_value = delay_argument(delay, headers ? 4 : 3)
|
|
122
|
+
|
|
123
|
+
if headers
|
|
128
124
|
encoded_headers = encoder.encode(headers)
|
|
129
125
|
conn.exec_params(
|
|
130
|
-
"SELECT * FROM pgmq.send_batch($1::text, $2::jsonb[], $3::jsonb[],
|
|
131
|
-
[queue_name, encoded_messages, encoded_headers,
|
|
132
|
-
)
|
|
133
|
-
elsif !delay.is_a?(Numeric)
|
|
134
|
-
conn.exec_params(
|
|
135
|
-
"SELECT * FROM pgmq.send_batch($1::text, $2::jsonb[], $3::timestamptz)",
|
|
136
|
-
[queue_name, encoded_messages, delay.to_time.utc.iso8601(6)]
|
|
126
|
+
"SELECT * FROM pgmq.send_batch($1::text, $2::jsonb[], $3::jsonb[], #{delay_sql})",
|
|
127
|
+
[queue_name, encoded_messages, encoded_headers, delay_value]
|
|
137
128
|
)
|
|
138
129
|
else
|
|
139
130
|
conn.exec_params(
|
|
140
|
-
"SELECT * FROM pgmq.send_batch($1::text, $2::jsonb[],
|
|
141
|
-
[queue_name, encoded_messages,
|
|
131
|
+
"SELECT * FROM pgmq.send_batch($1::text, $2::jsonb[], #{delay_sql})",
|
|
132
|
+
[queue_name, encoded_messages, delay_value]
|
|
142
133
|
)
|
|
143
134
|
end
|
|
144
135
|
end
|
|
@@ -44,6 +44,9 @@ module PGMQ
|
|
|
44
44
|
# @param tune_autovacuum [Boolean, Hash] when truthy, tune autovacuum on the new queue's tables after creation.
|
|
45
45
|
# Pass +true+ for PGMQ-tuned defaults, or a Hash forwarded to {Autovacuum#tune_autovacuum}. Defaults to
|
|
46
46
|
# +false+. Note: storage parameters are set on the partitioned parent and do not cascade to partitions.
|
|
47
|
+
# @param premake [Integer, nil] how many partitions pg_partman creates ahead of the current one, for both the
|
|
48
|
+
# queue and the archive table. Must be at least 1 (PGMQ raises otherwise). Defaults to +nil+, which uses
|
|
49
|
+
# PGMQ's default (4). Requires PGMQ v1.13.0+ when given.
|
|
47
50
|
# @return [Boolean] true if queue was created, false if it already existed
|
|
48
51
|
#
|
|
49
52
|
# @example
|
|
@@ -51,20 +54,37 @@ module PGMQ
|
|
|
51
54
|
# partition_interval: "daily",
|
|
52
55
|
# retention_interval: "7 days"
|
|
53
56
|
# ) # => true
|
|
57
|
+
#
|
|
58
|
+
# @example Pre-make more partitions for a high message rate (PGMQ v1.13.0+)
|
|
59
|
+
# client.create_partitioned("events",
|
|
60
|
+
# partition_interval: "10000",
|
|
61
|
+
# retention_interval: "100000",
|
|
62
|
+
# premake: 12
|
|
63
|
+
# ) # => true
|
|
54
64
|
def create_partitioned(
|
|
55
65
|
queue_name,
|
|
56
66
|
partition_interval: "10000",
|
|
57
67
|
retention_interval: "100000",
|
|
58
|
-
tune_autovacuum: false
|
|
68
|
+
tune_autovacuum: false,
|
|
69
|
+
premake: nil
|
|
59
70
|
)
|
|
60
71
|
validate_queue_name!(queue_name)
|
|
61
72
|
|
|
62
73
|
with_connection do |conn|
|
|
63
74
|
existed = queue_exists?(conn, queue_name)
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
75
|
+
|
|
76
|
+
if premake.nil?
|
|
77
|
+
conn.exec_params(
|
|
78
|
+
"SELECT pgmq.create_partitioned($1::text, $2::text, $3::text)",
|
|
79
|
+
[queue_name, partition_interval, retention_interval]
|
|
80
|
+
)
|
|
81
|
+
else
|
|
82
|
+
conn.exec_params(
|
|
83
|
+
"SELECT pgmq.create_partitioned($1::text, $2::text, $3::text, $4::integer)",
|
|
84
|
+
[queue_name, partition_interval, retention_interval, premake]
|
|
85
|
+
)
|
|
86
|
+
end
|
|
87
|
+
|
|
68
88
|
apply_tune_autovacuum_option(conn, queue_name, tune_autovacuum)
|
|
69
89
|
!existed
|
|
70
90
|
end
|
data/lib/pgmq/client/topics.rb
CHANGED
|
@@ -70,7 +70,8 @@ module PGMQ
|
|
|
70
70
|
# @param routing_key [String] dot-separated routing key (e.g., "orders.new.priority")
|
|
71
71
|
# @param message [String] message as JSON string
|
|
72
72
|
# @param headers [String, nil] optional headers as JSON string
|
|
73
|
-
# @param delay [
|
|
73
|
+
# @param delay [Numeric] delay in seconds before message becomes visible (integer or float; a float such as
|
|
74
|
+
# 1.5 needs PGMQ v1.10.0+)
|
|
74
75
|
# @return [Integer] count of queues the message was delivered to
|
|
75
76
|
#
|
|
76
77
|
# @example Basic topic send
|
|
@@ -82,6 +83,11 @@ module PGMQ
|
|
|
82
83
|
# headers: '{"trace_id":"abc"}',
|
|
83
84
|
# delay: 30)
|
|
84
85
|
def produce_topic(routing_key, message, headers: nil, delay: 0)
|
|
86
|
+
# send_topic takes whole seconds only, so a fractional delay goes through send_batch_topic
|
|
87
|
+
if delay.is_a?(Numeric) && !delay.integer?
|
|
88
|
+
return produce_batch_topic(routing_key, [message], headers: headers && [headers], delay: delay).size
|
|
89
|
+
end
|
|
90
|
+
|
|
85
91
|
result = with_connection do |conn|
|
|
86
92
|
if headers
|
|
87
93
|
conn.exec_params(
|
|
@@ -111,10 +117,12 @@ module PGMQ
|
|
|
111
117
|
# @param routing_key [String] dot-separated routing key
|
|
112
118
|
# @param messages [Array<String>] array of message payloads as JSON strings
|
|
113
119
|
# @param headers [Array<String>, nil] optional array of headers as JSON strings
|
|
114
|
-
# @param delay [Numeric, Time] delay in seconds before messages become visible (integer or float
|
|
115
|
-
# or an absolute Time at which the messages become visible,
|
|
116
|
-
# (PGMQ v1.10.0+)
|
|
120
|
+
# @param delay [Numeric, Time] delay in seconds before messages become visible (integer or float;
|
|
121
|
+
# a float such as 1.5 needs PGMQ v1.10.0+), or an absolute Time at which the messages become visible,
|
|
122
|
+
# including ActiveSupport::TimeWithZone (PGMQ v1.10.0+)
|
|
117
123
|
# @return [Array<Hash>] array of hashes with :queue_name and :msg_id
|
|
124
|
+
# @raise [PGMQ::Errors::MessageInvalidError] if messages is not an Array, headers is neither an Array nor nil,
|
|
125
|
+
# or headers length doesn't match messages length
|
|
118
126
|
#
|
|
119
127
|
# @example Batch topic send
|
|
120
128
|
# results = client.produce_batch_topic("orders.new", [
|
|
@@ -123,43 +131,30 @@ module PGMQ
|
|
|
123
131
|
# ])
|
|
124
132
|
# # => [{ queue_name: "new_orders", msg_id: "1" }, ...]
|
|
125
133
|
def produce_batch_topic(routing_key, messages, headers: nil, delay: 0)
|
|
134
|
+
BATCH_CONTRACT.validate!({ messages: messages, headers: headers }, Errors::MessageInvalidError)
|
|
126
135
|
return [] if messages.empty?
|
|
127
136
|
|
|
128
|
-
if headers && headers.length != messages.length
|
|
129
|
-
raise ArgumentError,
|
|
130
|
-
"headers array length (#{headers.length}) must match messages array length (#{messages.length})"
|
|
131
|
-
end
|
|
132
|
-
|
|
133
137
|
result = with_connection do |conn|
|
|
134
138
|
encoder = PG::TextEncoder::Array.new
|
|
135
139
|
encoded_messages = encoder.encode(messages)
|
|
136
140
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
"SELECT * FROM pgmq.send_batch_topic($1::text, $2::jsonb[], $3::jsonb[], $4::timestamptz)",
|
|
141
|
-
[routing_key, encoded_messages, encoded_headers, delay.to_time.utc.iso8601(6)]
|
|
142
|
-
)
|
|
143
|
-
elsif headers
|
|
141
|
+
delay_sql, delay_value = delay_argument(delay, headers ? 4 : 3)
|
|
142
|
+
|
|
143
|
+
if headers
|
|
144
144
|
encoded_headers = encoder.encode(headers)
|
|
145
145
|
conn.exec_params(
|
|
146
|
-
"SELECT * FROM pgmq.send_batch_topic($1::text, $2::jsonb[], $3::jsonb[],
|
|
147
|
-
[routing_key, encoded_messages, encoded_headers,
|
|
148
|
-
)
|
|
149
|
-
elsif !delay.is_a?(Numeric)
|
|
150
|
-
conn.exec_params(
|
|
151
|
-
"SELECT * FROM pgmq.send_batch_topic($1::text, $2::jsonb[], $3::timestamptz)",
|
|
152
|
-
[routing_key, encoded_messages, delay.to_time.utc.iso8601(6)]
|
|
146
|
+
"SELECT * FROM pgmq.send_batch_topic($1::text, $2::jsonb[], $3::jsonb[], #{delay_sql})",
|
|
147
|
+
[routing_key, encoded_messages, encoded_headers, delay_value]
|
|
153
148
|
)
|
|
154
|
-
elsif delay
|
|
149
|
+
elsif delay.is_a?(Integer) && delay <= 0
|
|
155
150
|
conn.exec_params(
|
|
156
|
-
"SELECT * FROM pgmq.send_batch_topic($1::text, $2::jsonb[]
|
|
157
|
-
[routing_key, encoded_messages
|
|
151
|
+
"SELECT * FROM pgmq.send_batch_topic($1::text, $2::jsonb[])",
|
|
152
|
+
[routing_key, encoded_messages]
|
|
158
153
|
)
|
|
159
154
|
else
|
|
160
155
|
conn.exec_params(
|
|
161
|
-
"SELECT * FROM pgmq.send_batch_topic($1::text, $2::jsonb[])",
|
|
162
|
-
[routing_key, encoded_messages]
|
|
156
|
+
"SELECT * FROM pgmq.send_batch_topic($1::text, $2::jsonb[], #{delay_sql})",
|
|
157
|
+
[routing_key, encoded_messages, delay_value]
|
|
163
158
|
)
|
|
164
159
|
end
|
|
165
160
|
end
|
data/lib/pgmq/client.rb
CHANGED
|
@@ -24,6 +24,17 @@ module PGMQ
|
|
|
24
24
|
# @example With connection string
|
|
25
25
|
# client = PGMQ::Client.new('postgres://localhost/mydb')
|
|
26
26
|
class Client
|
|
27
|
+
# Validates message and headers types before they are bound
|
|
28
|
+
MESSAGE_CONTRACT = Contracts::Message.new
|
|
29
|
+
|
|
30
|
+
# Validates the messages and headers arrays of a batch
|
|
31
|
+
BATCH_CONTRACT = Contracts::Batch.new
|
|
32
|
+
|
|
33
|
+
# Validates the arguments of multi-queue operations
|
|
34
|
+
MULTI_QUEUE_CONTRACT = Contracts::MultiQueue.new
|
|
35
|
+
|
|
36
|
+
private_constant :MESSAGE_CONTRACT, :BATCH_CONTRACT, :MULTI_QUEUE_CONTRACT
|
|
37
|
+
|
|
27
38
|
# Include functional modules (order matters for discoverability)
|
|
28
39
|
include Transaction # Transaction support (already existed)
|
|
29
40
|
include QueueManagement # Queue lifecycle (create, drop, list)
|
|
@@ -84,11 +95,10 @@ module PGMQ
|
|
|
84
95
|
@connection.close
|
|
85
96
|
end
|
|
86
97
|
|
|
87
|
-
# Drops every pooled connection and rebuilds lazily on the next checkout,
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
#
|
|
91
|
-
# poisoned so it would re-hang on reuse. See {PGMQ::Connection#reload}.
|
|
98
|
+
# Drops every pooled connection and rebuilds lazily on the next checkout, leaving the pool usable (unlike {#close}).
|
|
99
|
+
# Use it to discard a connection that reports +CONNECTION_OK+ but is actually wedged - e.g. after a wall-clock
|
|
100
|
+
# timeout interrupted a query mid-flight and left the socket poisoned so it would re-hang on reuse. See
|
|
101
|
+
# {PGMQ::Connection#reload}.
|
|
92
102
|
#
|
|
93
103
|
# @return [void]
|
|
94
104
|
def reload
|
|
@@ -118,7 +128,8 @@ module PGMQ
|
|
|
118
128
|
#
|
|
119
129
|
# @yield [PG::Connection] a pooled, health-checked database connection
|
|
120
130
|
# @return [Object] the return value of the block
|
|
121
|
-
# @raise [PGMQ::Errors::ConnectionError] if
|
|
131
|
+
# @raise [PGMQ::Errors::ConnectionError] if the block raises any `PG::Error` (the original is in `#cause`), or a
|
|
132
|
+
# connection cannot be obtained
|
|
122
133
|
#
|
|
123
134
|
# @example Fire a custom NOTIFY on the PGMQ pool
|
|
124
135
|
# client.with_connection do |conn|
|
|
@@ -161,5 +172,25 @@ module PGMQ
|
|
|
161
172
|
QueueName.validate!(queue_name)
|
|
162
173
|
nil
|
|
163
174
|
end
|
|
175
|
+
|
|
176
|
+
# Builds the SQL argument and bind value for a +delay:+ given in seconds or as an absolute Time
|
|
177
|
+
#
|
|
178
|
+
# PostgreSQL rejects a Float such as +1.5+ for an +integer+ bind, so a non-Integer Numeric is passed to the
|
|
179
|
+
# +timestamptz+ overload as +clock_timestamp()+ plus the delay. This is how PGMQ itself turns an integer delay
|
|
180
|
+
# into a visibility time, so the database clock is used in both cases.
|
|
181
|
+
#
|
|
182
|
+
# @param delay [Numeric, Time] delay in seconds or an absolute Time
|
|
183
|
+
# @param index [Integer] position of the delay bind parameter in the query
|
|
184
|
+
# @return [Array(String, Object)] SQL argument and its bind value
|
|
185
|
+
def delay_argument(delay, index)
|
|
186
|
+
case delay
|
|
187
|
+
when Integer
|
|
188
|
+
["$#{index}::integer", delay]
|
|
189
|
+
when Numeric
|
|
190
|
+
["clock_timestamp() + make_interval(secs => $#{index}::double precision)", delay.to_f]
|
|
191
|
+
else
|
|
192
|
+
["$#{index}::timestamptz", delay.to_time.utc.iso8601(6)]
|
|
193
|
+
end
|
|
194
|
+
end
|
|
164
195
|
end
|
|
165
196
|
end
|
data/lib/pgmq/connection.rb
CHANGED
|
@@ -26,6 +26,11 @@ module PGMQ
|
|
|
26
26
|
# Default connection pool timeout in seconds
|
|
27
27
|
DEFAULT_POOL_TIMEOUT = 5
|
|
28
28
|
|
|
29
|
+
# Validates the connection configuration
|
|
30
|
+
CONTRACT = Contracts::Connection.new
|
|
31
|
+
|
|
32
|
+
private_constant :CONTRACT
|
|
33
|
+
|
|
29
34
|
class << self
|
|
30
35
|
# Additional error message patterns (String or Regexp) that mean the connection is dead and a retry on a fresh
|
|
31
36
|
# socket is safe. Strings are matched as case-insensitive substrings; Regexps match the original message. The
|
|
@@ -35,8 +40,10 @@ module PGMQ
|
|
|
35
40
|
#
|
|
36
41
|
# @return [Array<String, Regexp>]
|
|
37
42
|
# @example
|
|
38
|
-
# PGMQ::Connection.reconnectable_error_patterns
|
|
39
|
-
#
|
|
43
|
+
# PGMQ::Connection.reconnectable_error_patterns = [
|
|
44
|
+
# "connection reset by peer",
|
|
45
|
+
# /\Abroken pipe\b/i
|
|
46
|
+
# ]
|
|
40
47
|
attr_reader :reconnectable_error_patterns
|
|
41
48
|
|
|
42
49
|
# Additional exception classes that mean the connection is dead. `PG::ConnectionBad` and `PG::UnableToSend` are
|
|
@@ -46,7 +53,7 @@ module PGMQ
|
|
|
46
53
|
#
|
|
47
54
|
# @return [Array<Class>]
|
|
48
55
|
# @example
|
|
49
|
-
# PGMQ::Connection.reconnectable_error_classes
|
|
56
|
+
# PGMQ::Connection.reconnectable_error_classes = [PG::ConnectionRefused]
|
|
50
57
|
attr_reader :reconnectable_error_classes
|
|
51
58
|
|
|
52
59
|
# Replaces the extra reconnectable error patterns.
|
|
@@ -54,6 +61,8 @@ module PGMQ
|
|
|
54
61
|
# @param patterns [Array<String, Regexp>]
|
|
55
62
|
# @raise [PGMQ::Errors::ConfigurationError] if any element is invalid
|
|
56
63
|
def reconnectable_error_patterns=(patterns)
|
|
64
|
+
CONTRACT.validate!({ reconnectable_error_patterns: patterns }, Errors::ConfigurationError)
|
|
65
|
+
|
|
57
66
|
@reconnectable_error_patterns = normalize_patterns(patterns)
|
|
58
67
|
end
|
|
59
68
|
|
|
@@ -62,7 +71,9 @@ module PGMQ
|
|
|
62
71
|
# @param classes [Array<Class>]
|
|
63
72
|
# @raise [PGMQ::Errors::ConfigurationError] if any element is invalid
|
|
64
73
|
def reconnectable_error_classes=(classes)
|
|
65
|
-
|
|
74
|
+
CONTRACT.validate!({ reconnectable_error_classes: classes }, Errors::ConfigurationError)
|
|
75
|
+
|
|
76
|
+
@reconnectable_error_classes = Array(classes).dup.freeze
|
|
66
77
|
end
|
|
67
78
|
|
|
68
79
|
private
|
|
@@ -75,40 +86,11 @@ module PGMQ
|
|
|
75
86
|
# @param patterns [Array<String, Regexp>, String, Regexp, nil]
|
|
76
87
|
# @return [Array<String, Regexp>]
|
|
77
88
|
def normalize_patterns(patterns)
|
|
78
|
-
Array(patterns).map
|
|
79
|
-
case pattern
|
|
80
|
-
when Regexp
|
|
81
|
-
pattern
|
|
82
|
-
when String
|
|
83
|
-
pattern.downcase
|
|
84
|
-
else
|
|
85
|
-
raise(
|
|
86
|
-
PGMQ::Errors::ConfigurationError,
|
|
87
|
-
"reconnectable_error_patterns must contain Strings or Regexps, got #{pattern.class}"
|
|
88
|
-
)
|
|
89
|
-
end
|
|
90
|
-
end.freeze
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
# Normalizes user-supplied reconnectable error classes.
|
|
94
|
-
#
|
|
95
|
-
# @param classes [Array<Class>, Class, nil]
|
|
96
|
-
# @return [Array<Class>]
|
|
97
|
-
def normalize_classes(classes)
|
|
98
|
-
Array(classes).map do |klass|
|
|
99
|
-
unless klass.is_a?(Class) && klass <= Exception
|
|
100
|
-
raise(
|
|
101
|
-
PGMQ::Errors::ConfigurationError,
|
|
102
|
-
"reconnectable_error_classes must contain Exception subclasses, got #{klass.inspect}"
|
|
103
|
-
)
|
|
104
|
-
end
|
|
105
|
-
|
|
106
|
-
klass
|
|
107
|
-
end.freeze
|
|
89
|
+
Array(patterns).map { |pattern| pattern.is_a?(String) ? pattern.downcase : pattern }.freeze
|
|
108
90
|
end
|
|
109
91
|
end
|
|
110
92
|
|
|
111
|
-
# Initialize class-level defaults (empty arrays, users
|
|
93
|
+
# Initialize class-level defaults (empty frozen arrays, users replace them at boot via the setters)
|
|
112
94
|
@reconnectable_error_patterns = [].freeze
|
|
113
95
|
@reconnectable_error_classes = [].freeze
|
|
114
96
|
|
|
@@ -128,12 +110,7 @@ module PGMQ
|
|
|
128
110
|
pool_timeout: DEFAULT_POOL_TIMEOUT,
|
|
129
111
|
auto_reconnect: true
|
|
130
112
|
)
|
|
131
|
-
|
|
132
|
-
raise(
|
|
133
|
-
PGMQ::Errors::ConfigurationError,
|
|
134
|
-
"Connection parameters are required"
|
|
135
|
-
)
|
|
136
|
-
end
|
|
113
|
+
CONTRACT.validate!({ conn_params: conn_params }, Errors::ConfigurationError)
|
|
137
114
|
|
|
138
115
|
@conn_params = normalize_connection_params(conn_params)
|
|
139
116
|
@pool_size = pool_size
|
|
@@ -146,7 +123,8 @@ module PGMQ
|
|
|
146
123
|
#
|
|
147
124
|
# @yield [PG::Connection] database connection
|
|
148
125
|
# @return [Object] result of the block
|
|
149
|
-
# @raise [PGMQ::Errors::ConnectionError] if
|
|
126
|
+
# @raise [PGMQ::Errors::ConnectionError] if the block raises any `PG::Error` (the original is in `#cause`), or a
|
|
127
|
+
# connection cannot be obtained
|
|
150
128
|
def with_connection
|
|
151
129
|
retries = @auto_reconnect ? 1 : 0
|
|
152
130
|
attempts = 0
|
|
@@ -178,16 +156,13 @@ module PGMQ
|
|
|
178
156
|
@pool.shutdown { |conn| conn.close unless conn.finished? }
|
|
179
157
|
end
|
|
180
158
|
|
|
181
|
-
# Drops every connection currently in the pool and lets the pool build fresh
|
|
182
|
-
#
|
|
183
|
-
# down permanently), the pool stays usable afterwards.
|
|
159
|
+
# Drops every connection currently in the pool and lets the pool build fresh ones lazily on the next checkout.
|
|
160
|
+
# Unlike {#close} (which shuts the pool down permanently), the pool stays usable afterwards.
|
|
184
161
|
#
|
|
185
|
-
# Use this to recover from a connection that libpq still reports as
|
|
186
|
-
#
|
|
187
|
-
#
|
|
188
|
-
#
|
|
189
|
-
# that case (the connection does not report +CONNECTION_BAD+), so the caller
|
|
190
|
-
# must discard it explicitly; +reload+ is the safe, pool-wide way to do so.
|
|
162
|
+
# Use this to recover from a connection that libpq still reports as +CONNECTION_OK+ but which is in fact wedged -
|
|
163
|
+
# e.g. after a wall-clock timeout (+Timeout.timeout+) interrupted a query mid-flight, leaving the socket poisoned so
|
|
164
|
+
# it re-hangs on reuse. {#verify_connection!} cannot catch that case (the connection does not report
|
|
165
|
+
# +CONNECTION_BAD+), so the caller must discard it explicitly; +reload+ is the safe, pool-wide way to do so.
|
|
191
166
|
#
|
|
192
167
|
# @return [void]
|
|
193
168
|
# @example Discard the pool after a query had to be force-interrupted
|
|
@@ -281,13 +256,11 @@ module PGMQ
|
|
|
281
256
|
# Normalizes various connection parameter formats
|
|
282
257
|
# @param params [String, Hash, Proc]
|
|
283
258
|
# @return [Hash, Proc]
|
|
284
|
-
# @raise [PGMQ::Errors::ConfigurationError] if
|
|
259
|
+
# @raise [PGMQ::Errors::ConfigurationError] if the connection string is invalid
|
|
285
260
|
def normalize_connection_params(params)
|
|
286
|
-
return params if params.respond_to?(:call) # Callable (e.g., proc for Rails)
|
|
287
261
|
return parse_connection_string(params) if params.is_a?(String)
|
|
288
|
-
return params if params.is_a?(Hash) && !params.empty?
|
|
289
262
|
|
|
290
|
-
|
|
263
|
+
params
|
|
291
264
|
end
|
|
292
265
|
|
|
293
266
|
# Parses a PostgreSQL connection string
|