beetle 3.3.8 → 3.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/RELEASE_NOTES.rdoc +21 -0
- data/beetle.gemspec +1 -1
- data/lib/beetle/base.rb +3 -2
- data/lib/beetle/client.rb +2 -1
- data/lib/beetle/configuration.rb +8 -1
- data/lib/beetle/deduplication_store.rb +1 -1
- data/lib/beetle/queue_properties.rb +22 -5
- data/lib/beetle/subscriber.rb +1 -1
- data/lib/beetle/version.rb +1 -1
- data/test/beetle/client_test.rb +1 -1
- data/test/beetle/deduplication_store_test.rb +1 -1
- data/test/beetle/message_test.rb +6 -14
- data/test/beetle/queue_properties_test.rb +30 -0
- data/test/beetle/subscriber_test.rb +6 -13
- metadata +7 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 28836e3040ba8ef4415a6aae49f3ae805384e61b45cd76993a29d7589c23b7bf
|
4
|
+
data.tar.gz: bed46b58fe787d79a21b1fb3413a0359d948e79e70da0de92bfb717ad5e4e7be
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5b92b7588bcdd74a13a211d76997bae5f2255f729bb62b3c673ef00ec0fbda9a69e609055ad1a65797350c8151da6aa059093af3a27016069f22f1fa77c347d6
|
7
|
+
data.tar.gz: 7baa29a222dea24a7ce3f90fbeb90634d7b816197aa34c7fb5401e0d4754ab339f36c733d67aeccbeaf61524e9f74a59f07e697e931af96d94343d9977209dea
|
data/RELEASE_NOTES.rdoc
CHANGED
@@ -1,5 +1,26 @@
|
|
1
1
|
= Release Notes
|
2
2
|
|
3
|
+
== Version 3.4.0
|
4
|
+
* Require redis gem version 4.2.1. This version changes the exists to check for the
|
5
|
+
existence of multiple keys, return the number of keys in the list that exist. This
|
6
|
+
requires at least redis gem version 4.2.0, but 4.2.1 contains a bug fix for said
|
7
|
+
command.
|
8
|
+
|
9
|
+
== Version 3.3.12
|
10
|
+
* Support queue level declaration of dead letter queue message TTL.
|
11
|
+
|
12
|
+
== Version 3.3.11
|
13
|
+
* Fixed that dead lettering only works correctly with global config option.
|
14
|
+
|
15
|
+
== Version 3.3.10
|
16
|
+
* Support configuring RabbitMQ write timeout.
|
17
|
+
|
18
|
+
== Version 3.3.9
|
19
|
+
* Reduce the number of queue policies created on the servers by allowing
|
20
|
+
the spefication of a default broker policy. You need to install the
|
21
|
+
default policy with a priority of zero to match all queues ever
|
22
|
+
created. This feature is optional.
|
23
|
+
|
3
24
|
== Version 3.3.8
|
4
25
|
* Avoid needless put call when updating queue policies.
|
5
26
|
It seems the additional call to get the current definition of a policy
|
data/beetle.gemspec
CHANGED
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
|
|
24
24
|
|
25
25
|
s.specification_version = 3
|
26
26
|
s.add_runtime_dependency "bunny", "~> 0.7.12"
|
27
|
-
s.add_runtime_dependency "redis", ">=
|
27
|
+
s.add_runtime_dependency "redis", ">= 4.2.1"
|
28
28
|
s.add_runtime_dependency "hiredis", ">= 0.4.5"
|
29
29
|
s.add_runtime_dependency "amq-protocol", "= 2.3.0"
|
30
30
|
s.add_runtime_dependency "amqp", "= 1.8.0"
|
data/lib/beetle/base.rb
CHANGED
@@ -78,7 +78,8 @@ module Beetle
|
|
78
78
|
end
|
79
79
|
|
80
80
|
def bind_dead_letter_queue!(channel, target_queue, creation_keys = {})
|
81
|
-
policy_options = @client.queues[target_queue].slice(:dead_lettering, :lazy)
|
81
|
+
policy_options = @client.queues[target_queue].slice(:dead_lettering, :lazy, :dead_lettering_msg_ttl)
|
82
|
+
policy_options[:message_ttl] = policy_options.delete(:dead_lettering_msg_ttl)
|
82
83
|
dead_letter_queue_name = "#{target_queue}_dead_letter"
|
83
84
|
if policy_options[:dead_lettering]
|
84
85
|
logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_keys.inspect}")
|
@@ -88,7 +89,7 @@ module Beetle
|
|
88
89
|
:queue_name => target_queue,
|
89
90
|
:bindings => @client.bindings[target_queue],
|
90
91
|
:dead_letter_queue_name => dead_letter_queue_name,
|
91
|
-
:message_ttl =>
|
92
|
+
:message_ttl => policy_options[:message_ttl]
|
92
93
|
}.merge(policy_options)
|
93
94
|
end
|
94
95
|
|
data/lib/beetle/client.rb
CHANGED
@@ -96,7 +96,8 @@ module Beetle
|
|
96
96
|
raise ConfigurationError.new("queue #{name} already configured") if queues.include?(name)
|
97
97
|
opts = {
|
98
98
|
:exchange => name, :key => name, :auto_delete => false, :amqp_name => name,
|
99
|
-
:lazy => config.lazy_queues_enabled, :dead_lettering => config.dead_lettering_enabled
|
99
|
+
:lazy => config.lazy_queues_enabled, :dead_lettering => config.dead_lettering_enabled,
|
100
|
+
:dead_lettering_msg_ttl => config.dead_lettering_msg_ttl
|
100
101
|
}.merge!(options.symbolize_keys)
|
101
102
|
opts.merge! :durable => true, :passive => false, :exclusive => false
|
102
103
|
exchange = opts.delete(:exchange).to_s
|
data/lib/beetle/configuration.rb
CHANGED
@@ -14,6 +14,9 @@ module Beetle
|
|
14
14
|
# Name of the policy update routing key
|
15
15
|
attr_accessor :beetle_policy_updates_routing_key
|
16
16
|
# default logger (defaults to <tt>Logger.new(log_file)</tt>)
|
17
|
+
attr_accessor :broker_default_policy
|
18
|
+
# set this to whatever your brokers have installed as default policy. For example, if you have installed
|
19
|
+
# a policy that makes every queue lazy, it should be set to <tt>{"queue-moode" => "lazy"}</tt>.
|
17
20
|
attr_accessor :logger
|
18
21
|
# defaults to <tt>STDOUT</tt>
|
19
22
|
attr_accessor :redis_logger
|
@@ -104,8 +107,10 @@ module Beetle
|
|
104
107
|
# Whether to update quueue policies synchronously or asynchronously.
|
105
108
|
attr_accessor :update_queue_properties_synchronously
|
106
109
|
|
107
|
-
# Read timeout for http requests to
|
110
|
+
# Read timeout for http requests to RabbitMQ HTTP API
|
108
111
|
attr_accessor :rabbitmq_api_read_timeout
|
112
|
+
# Write timeout for http requests to RabbitMQ HTTP API
|
113
|
+
attr_accessor :rabbitmq_api_write_timeout
|
109
114
|
|
110
115
|
# Returns the port on which the Rabbit API is hosted
|
111
116
|
attr_accessor :api_port
|
@@ -146,6 +151,7 @@ module Beetle
|
|
146
151
|
self.beetle_policy_exchange_name = "beetle-policies"
|
147
152
|
self.beetle_policy_updates_queue_name = "beetle-policy-updates"
|
148
153
|
self.beetle_policy_updates_routing_key = "beetle.policy.update"
|
154
|
+
self.broker_default_policy = {}
|
149
155
|
|
150
156
|
self.gc_threshold = 1.hour.to_i
|
151
157
|
self.redis_server = "localhost:6379"
|
@@ -174,6 +180,7 @@ module Beetle
|
|
174
180
|
self.dead_lettering_enabled = false
|
175
181
|
self.dead_lettering_msg_ttl = 1000 # 1 second
|
176
182
|
self.rabbitmq_api_read_timeout = 60 # 60 seconds
|
183
|
+
self.rabbitmq_api_write_timeout = 60 # 60 seconds
|
177
184
|
|
178
185
|
self.lazy_queues_enabled = false
|
179
186
|
self.throttling_refresh_interval = 60 # seconds
|
@@ -120,7 +120,7 @@ module Beetle
|
|
120
120
|
|
121
121
|
# check whether key with given suffix exists for a given <tt>msg_id</tt>.
|
122
122
|
def exists(msg_id, suffix)
|
123
|
-
with_failover { redis.exists(key(msg_id, suffix)) }
|
123
|
+
with_failover { redis.exists?(key(msg_id, suffix)) }
|
124
124
|
end
|
125
125
|
|
126
126
|
# flush the configured redis database. useful for testing.
|
@@ -45,9 +45,11 @@ module Beetle
|
|
45
45
|
return unless options[:dead_lettering] || options[:lazy]
|
46
46
|
|
47
47
|
# no need to worry that the server has the port 5672. Net:HTTP will take care of this. See below.
|
48
|
-
|
48
|
+
policy_name = "#{queue_name}_policy"
|
49
|
+
request_url = URI("http://#{server}/api/policies/#{vhost}/#{policy_name}")
|
49
50
|
get_request = Net::HTTP::Get.new(request_url)
|
50
51
|
put_request = Net::HTTP::Put.new(request_url)
|
52
|
+
delete_request = Net::HTTP::Delete.new(request_url)
|
51
53
|
|
52
54
|
# set up queue policy
|
53
55
|
definition = {}
|
@@ -66,13 +68,26 @@ module Beetle
|
|
66
68
|
"definition" => definition,
|
67
69
|
}
|
68
70
|
|
71
|
+
is_default_policy = definition == config.broker_default_policy
|
72
|
+
|
69
73
|
get_response = run_rabbit_http_request(request_url, get_request) do |http|
|
70
74
|
http.request(get_request, nil)
|
71
75
|
end
|
72
76
|
|
73
|
-
|
77
|
+
case get_response.code
|
78
|
+
when "200"
|
74
79
|
response_body = JSON.parse(get_response.body) rescue {}
|
75
|
-
|
80
|
+
same_policy = put_request_body.all? { |k,v| response_body[k] == v }
|
81
|
+
if same_policy
|
82
|
+
if is_default_policy
|
83
|
+
run_rabbit_http_request(request_url, delete_request) do |http|
|
84
|
+
http.request(get_request, nil)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
return :ok
|
88
|
+
end
|
89
|
+
when "404"
|
90
|
+
return :ok if is_default_policy
|
76
91
|
end
|
77
92
|
|
78
93
|
put_response = run_rabbit_http_request(request_url, put_request) do |http|
|
@@ -141,13 +156,15 @@ module Beetle
|
|
141
156
|
|
142
157
|
def run_rabbit_http_request(uri, request, &block)
|
143
158
|
request.basic_auth(config.user, config.password)
|
144
|
-
|
159
|
+
case request.class::METHOD
|
160
|
+
when 'GET'
|
145
161
|
request["Accept"] = "application/json"
|
146
|
-
|
162
|
+
when 'PUT'
|
147
163
|
request["Content-Type"] = "application/json"
|
148
164
|
end
|
149
165
|
http = Net::HTTP.new(uri.hostname, config.api_port)
|
150
166
|
http.read_timeout = config.rabbitmq_api_read_timeout
|
167
|
+
http.write_timeout = config.rabbitmq_api_write_timeout if http.respond_to?(:write_timeout=)
|
151
168
|
# don't do this in production:
|
152
169
|
# http.set_debug_output(logger.instance_eval{ @logdev.dev })
|
153
170
|
http.start do |instance|
|
data/lib/beetle/subscriber.rb
CHANGED
@@ -180,7 +180,7 @@ module Beetle
|
|
180
180
|
processor = Handler.create(handler, opts)
|
181
181
|
result = m.process(processor)
|
182
182
|
if result.reject?
|
183
|
-
if @client.
|
183
|
+
if @client.queues[queue_name][:dead_lettering]
|
184
184
|
header.reject(:requeue => false)
|
185
185
|
else
|
186
186
|
sleep 1
|
data/lib/beetle/version.rb
CHANGED
data/test/beetle/client_test.rb
CHANGED
@@ -79,7 +79,7 @@ module Beetle
|
|
79
79
|
|
80
80
|
test "registering a queue should store it in the configuration with symbolized option keys and force durable=true and passive=false and set the amqp queue name" do
|
81
81
|
@client.register_queue("some_queue", "durable" => false, "exchange" => "some_exchange")
|
82
|
-
assert_equal({:durable => true, :passive => false, :lazy=>false, :dead_lettering=>false, :auto_delete => false, :exclusive => false, :amqp_name => "some_queue"}, @client.queues["some_queue"])
|
82
|
+
assert_equal({:durable => true, :passive => false, :lazy=>false, :dead_lettering=>false, :dead_lettering_msg_ttl=>1000, :auto_delete => false, :exclusive => false, :amqp_name => "some_queue"}, @client.queues["some_queue"])
|
83
83
|
end
|
84
84
|
|
85
85
|
test "registering a queue should add the queue to the list of queues of the queue's exchange" do
|
data/test/beetle/message_test.rb
CHANGED
@@ -141,9 +141,7 @@ module Beetle
|
|
141
141
|
|
142
142
|
message.process(@null_handler)
|
143
143
|
keys = @store.keys(message.msg_id)
|
144
|
-
|
145
|
-
assert !@store.redis.exists(key)
|
146
|
-
end
|
144
|
+
assert_equal 0, @store.redis.exists(*keys)
|
147
145
|
end
|
148
146
|
|
149
147
|
test "successful processing of a non redundant message should delete all keys from the database (except the status key, which should be set to expire)" do
|
@@ -159,11 +157,9 @@ module Beetle
|
|
159
157
|
message.process(@null_handler)
|
160
158
|
keys = @store.keys(message.msg_id)
|
161
159
|
status_key = keys.shift
|
162
|
-
assert @store.redis.exists(status_key)
|
160
|
+
assert @store.redis.exists?(status_key)
|
163
161
|
assert @store.redis.ttl(status_key) <= @config.redis_status_key_expiry_interval
|
164
|
-
|
165
|
-
assert !@store.redis.exists(key)
|
166
|
-
end
|
162
|
+
assert_equal 0, @store.redis.exists(*keys)
|
167
163
|
end
|
168
164
|
|
169
165
|
test "successful processing of a redundant message twice should delete all keys from the database" do
|
@@ -179,9 +175,7 @@ module Beetle
|
|
179
175
|
message.process(@null_handler)
|
180
176
|
|
181
177
|
keys = @store.keys(message.msg_id)
|
182
|
-
|
183
|
-
assert !@store.redis.exists(key)
|
184
|
-
end
|
178
|
+
assert_equal 0, @store.redis.exists(*keys)
|
185
179
|
end
|
186
180
|
|
187
181
|
test "successful processing of a redundant message twice should delete all keys from the database (except the status key, which should be set to expire)" do
|
@@ -199,11 +193,9 @@ module Beetle
|
|
199
193
|
|
200
194
|
keys = @store.keys(message.msg_id)
|
201
195
|
status_key = keys.shift
|
202
|
-
assert @store.redis.exists(status_key)
|
196
|
+
assert @store.redis.exists?(status_key)
|
203
197
|
assert @store.redis.ttl(status_key) <= @config.redis_status_key_expiry_interval
|
204
|
-
|
205
|
-
assert !@store.redis.exists(key)
|
206
|
-
end
|
198
|
+
assert_equal 0, @store.redis.exists(*keys)
|
207
199
|
end
|
208
200
|
|
209
201
|
test "successful processing of a redundant message once should insert all but the delay key and the exception count key into the database" do
|
@@ -78,6 +78,36 @@ module Beetle
|
|
78
78
|
@queue_properties.set_queue_policy!(@server, @queue_name, :lazy => false, :dead_lettering => true, :routing_key => "QUEUE_NAME_dead_letter")
|
79
79
|
end
|
80
80
|
|
81
|
+
test "deletes policy if its definition corresponds to the broker default policy" do
|
82
|
+
@config.broker_default_policy = { "queue-mode" => "lazy" }
|
83
|
+
stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
|
84
|
+
.with(basic_auth: ['guest', 'guest'])
|
85
|
+
.to_return(:status => 200,
|
86
|
+
:body => {
|
87
|
+
"vhost" => "/",
|
88
|
+
"name" => "QUEUE_NAME_policy",
|
89
|
+
"pattern" => "^QUEUE_NAME$",
|
90
|
+
"priority" => 1,
|
91
|
+
"apply-to" => "queues",
|
92
|
+
"definition" => {
|
93
|
+
"queue-mode" => "lazy",
|
94
|
+
}}.to_json)
|
95
|
+
stub_request(:delete, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
|
96
|
+
.with(basic_auth: ['guest', 'guest'])
|
97
|
+
.to_return(:status => 204)
|
98
|
+
|
99
|
+
@queue_properties.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => false, :routing_key => "QUEUE_NAME_dead_letter")
|
100
|
+
end
|
101
|
+
|
102
|
+
test "does nothing if its definition corresponds to the broker default policy and the policy does not exist on the server" do
|
103
|
+
@config.broker_default_policy = { "queue-mode" => "lazy" }
|
104
|
+
stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
|
105
|
+
.with(basic_auth: ['guest', 'guest'])
|
106
|
+
.to_return(:status => 404)
|
107
|
+
|
108
|
+
@queue_properties.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => false, :routing_key => "QUEUE_NAME_dead_letter")
|
109
|
+
end
|
110
|
+
|
81
111
|
test "creates a policy by posting to the rabbitmq if lazy queues are enabled" do
|
82
112
|
stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
|
83
113
|
.with(basic_auth: ['guest', 'guest'])
|
@@ -224,13 +224,11 @@ module Beetle
|
|
224
224
|
end
|
225
225
|
end
|
226
226
|
|
227
|
-
|
228
227
|
class DeadLetteringCallBackExecutionTest < Minitest::Test
|
229
228
|
def setup
|
230
229
|
@client = Client.new
|
231
|
-
@client.config.dead_lettering_enabled = true
|
232
230
|
@queue = "somequeue"
|
233
|
-
@client.register_queue(@queue)
|
231
|
+
@client.register_queue(@queue, :dead_lettering => true)
|
234
232
|
@sub = @client.send(:subscriber)
|
235
233
|
mq = mock("MQ")
|
236
234
|
mq.expects(:closing?).returns(false)
|
@@ -239,11 +237,7 @@ module Beetle
|
|
239
237
|
@handler = Handler.create(lambda{|*args| raise @exception})
|
240
238
|
# handler method 'processing_completed' should be called under all circumstances
|
241
239
|
@handler.expects(:processing_completed).once
|
242
|
-
@callback = @sub.send(:create_subscription_callback,
|
243
|
-
end
|
244
|
-
|
245
|
-
def teardown
|
246
|
-
@client.config.dead_lettering_enabled = false
|
240
|
+
@callback = @sub.send(:create_subscription_callback, @queue, @queue, @handler, :exceptions => 1)
|
247
241
|
end
|
248
242
|
|
249
243
|
test "should call reject on the message header when processing the handler returns true on reject? if dead lettering has been enabled" do
|
@@ -255,19 +249,18 @@ module Beetle
|
|
255
249
|
header.expects(:reject).with(:requeue => false)
|
256
250
|
@callback.call(header, 'foo')
|
257
251
|
end
|
258
|
-
|
259
252
|
end
|
260
253
|
|
261
254
|
class CallBackExecutionTest < Minitest::Test
|
262
255
|
def setup
|
263
|
-
client = Client.new
|
256
|
+
@client = Client.new
|
264
257
|
@queue = "somequeue"
|
265
|
-
client.register_queue(@queue)
|
266
|
-
@sub = client.send(:subscriber)
|
258
|
+
@client.register_queue(@queue)
|
259
|
+
@sub = @client.send(:subscriber)
|
267
260
|
@exception = Exception.new "murks"
|
268
261
|
@handler = Handler.create(lambda{|*args| raise @exception})
|
269
262
|
@handler.instance_eval { def post_process; raise "shoot"; end }
|
270
|
-
@callback = @sub.send(:create_subscription_callback,
|
263
|
+
@callback = @sub.send(:create_subscription_callback, @queue, @queue, @handler, :exceptions => 1)
|
271
264
|
end
|
272
265
|
|
273
266
|
test "exceptions raised from message processing should be ignored" do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: beetle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stefan Kaes
|
@@ -9,10 +9,10 @@ authors:
|
|
9
9
|
- Ali Jelveh
|
10
10
|
- Sebastian Roebke
|
11
11
|
- Larry Baltz
|
12
|
-
autorequire:
|
12
|
+
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2020-
|
15
|
+
date: 2020-06-21 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: bunny
|
@@ -34,14 +34,14 @@ dependencies:
|
|
34
34
|
requirements:
|
35
35
|
- - ">="
|
36
36
|
- !ruby/object:Gem::Version
|
37
|
-
version:
|
37
|
+
version: 4.2.1
|
38
38
|
type: :runtime
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
41
41
|
requirements:
|
42
42
|
- - ">="
|
43
43
|
- !ruby/object:Gem::Version
|
44
|
-
version:
|
44
|
+
version: 4.2.1
|
45
45
|
- !ruby/object:Gem::Dependency
|
46
46
|
name: hiredis
|
47
47
|
requirement: !ruby/object:Gem::Requirement
|
@@ -359,7 +359,7 @@ homepage: https://xing.github.com/beetle/
|
|
359
359
|
licenses: []
|
360
360
|
metadata:
|
361
361
|
changelog_uri: https://github.com/xing/beetle/blob/master/RELEASE_NOTES.rdoc
|
362
|
-
post_install_message:
|
362
|
+
post_install_message:
|
363
363
|
rdoc_options:
|
364
364
|
- "--charset=UTF-8"
|
365
365
|
require_paths:
|
@@ -376,7 +376,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
376
376
|
version: 1.3.7
|
377
377
|
requirements: []
|
378
378
|
rubygems_version: 3.0.8
|
379
|
-
signing_key:
|
379
|
+
signing_key:
|
380
380
|
specification_version: 3
|
381
381
|
summary: High Availability AMQP Messaging with Redundant Queues
|
382
382
|
test_files:
|