beetle 3.3.8 → 3.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b64b9ad7d4d36c450c8219f666db52323d1d30e6f741ccb5791469f9f9c7b7a0
4
- data.tar.gz: 860d8813af202e5463aa70d2b33466c0cb1d0f0ed8afe7490ae82e106c2cc3f7
3
+ metadata.gz: 28836e3040ba8ef4415a6aae49f3ae805384e61b45cd76993a29d7589c23b7bf
4
+ data.tar.gz: bed46b58fe787d79a21b1fb3413a0359d948e79e70da0de92bfb717ad5e4e7be
5
5
  SHA512:
6
- metadata.gz: b97d517694c27bfa3bcbb6d7714374b998a6e1268e0012e59a9d8e0c57e7fac0904cf978a9e5dee7c92c03f183c679ad238102704152e1f1115bedcc9280b70a
7
- data.tar.gz: 63875a03bf6cd2c0221ce925d9b14d49aae0fbf547a40177ef4eee77a5913894b64df82967c45a3aecc0eff8a7631a8038e4fae1e5b7618dea3a66be030a472d
6
+ metadata.gz: 5b92b7588bcdd74a13a211d76997bae5f2255f729bb62b3c673ef00ec0fbda9a69e609055ad1a65797350c8151da6aa059093af3a27016069f22f1fa77c347d6
7
+ data.tar.gz: 7baa29a222dea24a7ce3f90fbeb90634d7b816197aa34c7fb5401e0d4754ab339f36c733d67aeccbeaf61524e9f74a59f07e697e931af96d94343d9977209dea
@@ -1,5 +1,26 @@
1
1
  = Release Notes
2
2
 
3
+ == Version 3.4.0
4
+ * Require redis gem version 4.2.1. This version changes the exists to check for the
5
+ existence of multiple keys, return the number of keys in the list that exist. This
6
+ requires at least redis gem version 4.2.0, but 4.2.1 contains a bug fix for said
7
+ command.
8
+
9
+ == Version 3.3.12
10
+ * Support queue level declaration of dead letter queue message TTL.
11
+
12
+ == Version 3.3.11
13
+ * Fixed that dead lettering only works correctly with global config option.
14
+
15
+ == Version 3.3.10
16
+ * Support configuring RabbitMQ write timeout.
17
+
18
+ == Version 3.3.9
19
+ * Reduce the number of queue policies created on the servers by allowing
20
+ the spefication of a default broker policy. You need to install the
21
+ default policy with a priority of zero to match all queues ever
22
+ created. This feature is optional.
23
+
3
24
  == Version 3.3.8
4
25
  * Avoid needless put call when updating queue policies.
5
26
  It seems the additional call to get the current definition of a policy
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
24
24
 
25
25
  s.specification_version = 3
26
26
  s.add_runtime_dependency "bunny", "~> 0.7.12"
27
- s.add_runtime_dependency "redis", ">= 2.2.2"
27
+ s.add_runtime_dependency "redis", ">= 4.2.1"
28
28
  s.add_runtime_dependency "hiredis", ">= 0.4.5"
29
29
  s.add_runtime_dependency "amq-protocol", "= 2.3.0"
30
30
  s.add_runtime_dependency "amqp", "= 1.8.0"
@@ -78,7 +78,8 @@ module Beetle
78
78
  end
79
79
 
80
80
  def bind_dead_letter_queue!(channel, target_queue, creation_keys = {})
81
- policy_options = @client.queues[target_queue].slice(:dead_lettering, :lazy)
81
+ policy_options = @client.queues[target_queue].slice(:dead_lettering, :lazy, :dead_lettering_msg_ttl)
82
+ policy_options[:message_ttl] = policy_options.delete(:dead_lettering_msg_ttl)
82
83
  dead_letter_queue_name = "#{target_queue}_dead_letter"
83
84
  if policy_options[:dead_lettering]
84
85
  logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_keys.inspect}")
@@ -88,7 +89,7 @@ module Beetle
88
89
  :queue_name => target_queue,
89
90
  :bindings => @client.bindings[target_queue],
90
91
  :dead_letter_queue_name => dead_letter_queue_name,
91
- :message_ttl => @client.config.dead_lettering_msg_ttl,
92
+ :message_ttl => policy_options[:message_ttl]
92
93
  }.merge(policy_options)
93
94
  end
94
95
 
@@ -96,7 +96,8 @@ module Beetle
96
96
  raise ConfigurationError.new("queue #{name} already configured") if queues.include?(name)
97
97
  opts = {
98
98
  :exchange => name, :key => name, :auto_delete => false, :amqp_name => name,
99
- :lazy => config.lazy_queues_enabled, :dead_lettering => config.dead_lettering_enabled
99
+ :lazy => config.lazy_queues_enabled, :dead_lettering => config.dead_lettering_enabled,
100
+ :dead_lettering_msg_ttl => config.dead_lettering_msg_ttl
100
101
  }.merge!(options.symbolize_keys)
101
102
  opts.merge! :durable => true, :passive => false, :exclusive => false
102
103
  exchange = opts.delete(:exchange).to_s
@@ -14,6 +14,9 @@ module Beetle
14
14
  # Name of the policy update routing key
15
15
  attr_accessor :beetle_policy_updates_routing_key
16
16
  # default logger (defaults to <tt>Logger.new(log_file)</tt>)
17
+ attr_accessor :broker_default_policy
18
+ # set this to whatever your brokers have installed as default policy. For example, if you have installed
19
+ # a policy that makes every queue lazy, it should be set to <tt>{"queue-moode" => "lazy"}</tt>.
17
20
  attr_accessor :logger
18
21
  # defaults to <tt>STDOUT</tt>
19
22
  attr_accessor :redis_logger
@@ -104,8 +107,10 @@ module Beetle
104
107
  # Whether to update quueue policies synchronously or asynchronously.
105
108
  attr_accessor :update_queue_properties_synchronously
106
109
 
107
- # Read timeout for http requests to create dead letter bindings
110
+ # Read timeout for http requests to RabbitMQ HTTP API
108
111
  attr_accessor :rabbitmq_api_read_timeout
112
+ # Write timeout for http requests to RabbitMQ HTTP API
113
+ attr_accessor :rabbitmq_api_write_timeout
109
114
 
110
115
  # Returns the port on which the Rabbit API is hosted
111
116
  attr_accessor :api_port
@@ -146,6 +151,7 @@ module Beetle
146
151
  self.beetle_policy_exchange_name = "beetle-policies"
147
152
  self.beetle_policy_updates_queue_name = "beetle-policy-updates"
148
153
  self.beetle_policy_updates_routing_key = "beetle.policy.update"
154
+ self.broker_default_policy = {}
149
155
 
150
156
  self.gc_threshold = 1.hour.to_i
151
157
  self.redis_server = "localhost:6379"
@@ -174,6 +180,7 @@ module Beetle
174
180
  self.dead_lettering_enabled = false
175
181
  self.dead_lettering_msg_ttl = 1000 # 1 second
176
182
  self.rabbitmq_api_read_timeout = 60 # 60 seconds
183
+ self.rabbitmq_api_write_timeout = 60 # 60 seconds
177
184
 
178
185
  self.lazy_queues_enabled = false
179
186
  self.throttling_refresh_interval = 60 # seconds
@@ -120,7 +120,7 @@ module Beetle
120
120
 
121
121
  # check whether key with given suffix exists for a given <tt>msg_id</tt>.
122
122
  def exists(msg_id, suffix)
123
- with_failover { redis.exists(key(msg_id, suffix)) }
123
+ with_failover { redis.exists?(key(msg_id, suffix)) }
124
124
  end
125
125
 
126
126
  # flush the configured redis database. useful for testing.
@@ -45,9 +45,11 @@ module Beetle
45
45
  return unless options[:dead_lettering] || options[:lazy]
46
46
 
47
47
  # no need to worry that the server has the port 5672. Net:HTTP will take care of this. See below.
48
- request_url = URI("http://#{server}/api/policies/#{vhost}/#{queue_name}_policy")
48
+ policy_name = "#{queue_name}_policy"
49
+ request_url = URI("http://#{server}/api/policies/#{vhost}/#{policy_name}")
49
50
  get_request = Net::HTTP::Get.new(request_url)
50
51
  put_request = Net::HTTP::Put.new(request_url)
52
+ delete_request = Net::HTTP::Delete.new(request_url)
51
53
 
52
54
  # set up queue policy
53
55
  definition = {}
@@ -66,13 +68,26 @@ module Beetle
66
68
  "definition" => definition,
67
69
  }
68
70
 
71
+ is_default_policy = definition == config.broker_default_policy
72
+
69
73
  get_response = run_rabbit_http_request(request_url, get_request) do |http|
70
74
  http.request(get_request, nil)
71
75
  end
72
76
 
73
- if get_response.code == "200"
77
+ case get_response.code
78
+ when "200"
74
79
  response_body = JSON.parse(get_response.body) rescue {}
75
- return :ok if put_request_body.all? { |k,v| response_body[k] == v }
80
+ same_policy = put_request_body.all? { |k,v| response_body[k] == v }
81
+ if same_policy
82
+ if is_default_policy
83
+ run_rabbit_http_request(request_url, delete_request) do |http|
84
+ http.request(get_request, nil)
85
+ end
86
+ end
87
+ return :ok
88
+ end
89
+ when "404"
90
+ return :ok if is_default_policy
76
91
  end
77
92
 
78
93
  put_response = run_rabbit_http_request(request_url, put_request) do |http|
@@ -141,13 +156,15 @@ module Beetle
141
156
 
142
157
  def run_rabbit_http_request(uri, request, &block)
143
158
  request.basic_auth(config.user, config.password)
144
- if request.class::METHOD == 'GET'
159
+ case request.class::METHOD
160
+ when 'GET'
145
161
  request["Accept"] = "application/json"
146
- else
162
+ when 'PUT'
147
163
  request["Content-Type"] = "application/json"
148
164
  end
149
165
  http = Net::HTTP.new(uri.hostname, config.api_port)
150
166
  http.read_timeout = config.rabbitmq_api_read_timeout
167
+ http.write_timeout = config.rabbitmq_api_write_timeout if http.respond_to?(:write_timeout=)
151
168
  # don't do this in production:
152
169
  # http.set_debug_output(logger.instance_eval{ @logdev.dev })
153
170
  http.start do |instance|
@@ -180,7 +180,7 @@ module Beetle
180
180
  processor = Handler.create(handler, opts)
181
181
  result = m.process(processor)
182
182
  if result.reject?
183
- if @client.config.dead_lettering_enabled?
183
+ if @client.queues[queue_name][:dead_lettering]
184
184
  header.reject(:requeue => false)
185
185
  else
186
186
  sleep 1
@@ -1,3 +1,3 @@
1
1
  module Beetle
2
- VERSION = "3.3.8"
2
+ VERSION = "3.4.0"
3
3
  end
@@ -79,7 +79,7 @@ module Beetle
79
79
 
80
80
  test "registering a queue should store it in the configuration with symbolized option keys and force durable=true and passive=false and set the amqp queue name" do
81
81
  @client.register_queue("some_queue", "durable" => false, "exchange" => "some_exchange")
82
- assert_equal({:durable => true, :passive => false, :lazy=>false, :dead_lettering=>false, :auto_delete => false, :exclusive => false, :amqp_name => "some_queue"}, @client.queues["some_queue"])
82
+ assert_equal({:durable => true, :passive => false, :lazy=>false, :dead_lettering=>false, :dead_lettering_msg_ttl=>1000, :auto_delete => false, :exclusive => false, :amqp_name => "some_queue"}, @client.queues["some_queue"])
83
83
  end
84
84
 
85
85
  test "registering a queue should add the queue to the list of queues of the queue's exchange" do
@@ -9,7 +9,7 @@ module Beetle
9
9
  end
10
10
 
11
11
  test "trying to delete a non existent key doesn't throw an error" do
12
- assert !@r.exists("hahahaha")
12
+ assert !@r.exists?("hahahaha")
13
13
  assert_equal 0, @r.del("hahahaha")
14
14
  end
15
15
 
@@ -141,9 +141,7 @@ module Beetle
141
141
 
142
142
  message.process(@null_handler)
143
143
  keys = @store.keys(message.msg_id)
144
- keys.each do |key|
145
- assert !@store.redis.exists(key)
146
- end
144
+ assert_equal 0, @store.redis.exists(*keys)
147
145
  end
148
146
 
149
147
  test "successful processing of a non redundant message should delete all keys from the database (except the status key, which should be set to expire)" do
@@ -159,11 +157,9 @@ module Beetle
159
157
  message.process(@null_handler)
160
158
  keys = @store.keys(message.msg_id)
161
159
  status_key = keys.shift
162
- assert @store.redis.exists(status_key)
160
+ assert @store.redis.exists?(status_key)
163
161
  assert @store.redis.ttl(status_key) <= @config.redis_status_key_expiry_interval
164
- keys.each do |key|
165
- assert !@store.redis.exists(key)
166
- end
162
+ assert_equal 0, @store.redis.exists(*keys)
167
163
  end
168
164
 
169
165
  test "successful processing of a redundant message twice should delete all keys from the database" do
@@ -179,9 +175,7 @@ module Beetle
179
175
  message.process(@null_handler)
180
176
 
181
177
  keys = @store.keys(message.msg_id)
182
- keys.each do |key|
183
- assert !@store.redis.exists(key)
184
- end
178
+ assert_equal 0, @store.redis.exists(*keys)
185
179
  end
186
180
 
187
181
  test "successful processing of a redundant message twice should delete all keys from the database (except the status key, which should be set to expire)" do
@@ -199,11 +193,9 @@ module Beetle
199
193
 
200
194
  keys = @store.keys(message.msg_id)
201
195
  status_key = keys.shift
202
- assert @store.redis.exists(status_key)
196
+ assert @store.redis.exists?(status_key)
203
197
  assert @store.redis.ttl(status_key) <= @config.redis_status_key_expiry_interval
204
- keys.each do |key|
205
- assert !@store.redis.exists(key)
206
- end
198
+ assert_equal 0, @store.redis.exists(*keys)
207
199
  end
208
200
 
209
201
  test "successful processing of a redundant message once should insert all but the delay key and the exception count key into the database" do
@@ -78,6 +78,36 @@ module Beetle
78
78
  @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => false, :dead_lettering => true, :routing_key => "QUEUE_NAME_dead_letter")
79
79
  end
80
80
 
81
+ test "deletes policy if its definition corresponds to the broker default policy" do
82
+ @config.broker_default_policy = { "queue-mode" => "lazy" }
83
+ stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
84
+ .with(basic_auth: ['guest', 'guest'])
85
+ .to_return(:status => 200,
86
+ :body => {
87
+ "vhost" => "/",
88
+ "name" => "QUEUE_NAME_policy",
89
+ "pattern" => "^QUEUE_NAME$",
90
+ "priority" => 1,
91
+ "apply-to" => "queues",
92
+ "definition" => {
93
+ "queue-mode" => "lazy",
94
+ }}.to_json)
95
+ stub_request(:delete, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
96
+ .with(basic_auth: ['guest', 'guest'])
97
+ .to_return(:status => 204)
98
+
99
+ @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => false, :routing_key => "QUEUE_NAME_dead_letter")
100
+ end
101
+
102
+ test "does nothing if its definition corresponds to the broker default policy and the policy does not exist on the server" do
103
+ @config.broker_default_policy = { "queue-mode" => "lazy" }
104
+ stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
105
+ .with(basic_auth: ['guest', 'guest'])
106
+ .to_return(:status => 404)
107
+
108
+ @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => false, :routing_key => "QUEUE_NAME_dead_letter")
109
+ end
110
+
81
111
  test "creates a policy by posting to the rabbitmq if lazy queues are enabled" do
82
112
  stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
83
113
  .with(basic_auth: ['guest', 'guest'])
@@ -224,13 +224,11 @@ module Beetle
224
224
  end
225
225
  end
226
226
 
227
-
228
227
  class DeadLetteringCallBackExecutionTest < Minitest::Test
229
228
  def setup
230
229
  @client = Client.new
231
- @client.config.dead_lettering_enabled = true
232
230
  @queue = "somequeue"
233
- @client.register_queue(@queue)
231
+ @client.register_queue(@queue, :dead_lettering => true)
234
232
  @sub = @client.send(:subscriber)
235
233
  mq = mock("MQ")
236
234
  mq.expects(:closing?).returns(false)
@@ -239,11 +237,7 @@ module Beetle
239
237
  @handler = Handler.create(lambda{|*args| raise @exception})
240
238
  # handler method 'processing_completed' should be called under all circumstances
241
239
  @handler.expects(:processing_completed).once
242
- @callback = @sub.send(:create_subscription_callback, "my myessage", @queue, @handler, :exceptions => 1)
243
- end
244
-
245
- def teardown
246
- @client.config.dead_lettering_enabled = false
240
+ @callback = @sub.send(:create_subscription_callback, @queue, @queue, @handler, :exceptions => 1)
247
241
  end
248
242
 
249
243
  test "should call reject on the message header when processing the handler returns true on reject? if dead lettering has been enabled" do
@@ -255,19 +249,18 @@ module Beetle
255
249
  header.expects(:reject).with(:requeue => false)
256
250
  @callback.call(header, 'foo')
257
251
  end
258
-
259
252
  end
260
253
 
261
254
  class CallBackExecutionTest < Minitest::Test
262
255
  def setup
263
- client = Client.new
256
+ @client = Client.new
264
257
  @queue = "somequeue"
265
- client.register_queue(@queue)
266
- @sub = client.send(:subscriber)
258
+ @client.register_queue(@queue)
259
+ @sub = @client.send(:subscriber)
267
260
  @exception = Exception.new "murks"
268
261
  @handler = Handler.create(lambda{|*args| raise @exception})
269
262
  @handler.instance_eval { def post_process; raise "shoot"; end }
270
- @callback = @sub.send(:create_subscription_callback, "my myessage", @queue, @handler, :exceptions => 1)
263
+ @callback = @sub.send(:create_subscription_callback, @queue, @queue, @handler, :exceptions => 1)
271
264
  end
272
265
 
273
266
  test "exceptions raised from message processing should be ignored" do
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: beetle
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.3.8
4
+ version: 3.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Stefan Kaes
@@ -9,10 +9,10 @@ authors:
9
9
  - Ali Jelveh
10
10
  - Sebastian Roebke
11
11
  - Larry Baltz
12
- autorequire:
12
+ autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2020-05-18 00:00:00.000000000 Z
15
+ date: 2020-06-21 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: bunny
@@ -34,14 +34,14 @@ dependencies:
34
34
  requirements:
35
35
  - - ">="
36
36
  - !ruby/object:Gem::Version
37
- version: 2.2.2
37
+ version: 4.2.1
38
38
  type: :runtime
39
39
  prerelease: false
40
40
  version_requirements: !ruby/object:Gem::Requirement
41
41
  requirements:
42
42
  - - ">="
43
43
  - !ruby/object:Gem::Version
44
- version: 2.2.2
44
+ version: 4.2.1
45
45
  - !ruby/object:Gem::Dependency
46
46
  name: hiredis
47
47
  requirement: !ruby/object:Gem::Requirement
@@ -359,7 +359,7 @@ homepage: https://xing.github.com/beetle/
359
359
  licenses: []
360
360
  metadata:
361
361
  changelog_uri: https://github.com/xing/beetle/blob/master/RELEASE_NOTES.rdoc
362
- post_install_message:
362
+ post_install_message:
363
363
  rdoc_options:
364
364
  - "--charset=UTF-8"
365
365
  require_paths:
@@ -376,7 +376,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
376
376
  version: 1.3.7
377
377
  requirements: []
378
378
  rubygems_version: 3.0.8
379
- signing_key:
379
+ signing_key:
380
380
  specification_version: 3
381
381
  summary: High Availability AMQP Messaging with Redundant Queues
382
382
  test_files: