beetle 3.3.6 → 3.3.11

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6552c1aad5c0d816b274200e19198b55f459e4134726f532742a199d181d0e86
4
- data.tar.gz: 6d5f931794d753e0b4e9100e324007789ae8f62f4a20b539345a9bfc567efe05
3
+ metadata.gz: fd1ee5b22d344002924b2b648e036ce98584e270b05af6195115c5bc6aef6438
4
+ data.tar.gz: f32ef2c465d81cb73ac8453b677419c9117e1f2ab62126feb62dd2227cc1e924
5
5
  SHA512:
6
- metadata.gz: cc71d5c9dd604cb3cabbb40894289105aa1b840da347d1f06ddabc66f004c1e25bb34635c6902076f17465fccdd6fffe32c8b1e628949bdda83ef17a862d2b92
7
- data.tar.gz: 03a1e736d4f1c84528687b666efff22af82e885764c10df7e136a29ecb3cfc77beb234b1e97733c2d9078473a3889dc73b51c48d73863ffd940b76accd807c85
6
+ metadata.gz: 9521152d635cb12bd96dd87a8153cc20f4d8f9471d9a3d08c2d8d786a53bad2a76b783140ce1e8481b90c2b8dd1f6a391988985380fc4efcc93c30ba01ee013f
7
+ data.tar.gz: c29b033bcb301006b22bcbeeb4a9e62b6d72a5bcf54093e01aee405ea2c5a8d0f65c3a0e6a69bce0cbe0047cf52780069ae6c9a7d68c53161fea5de0c8b09b27
@@ -228,6 +228,6 @@ https://hub.docker.com/r/xingarchitects/gobeetle/.
228
228
 
229
229
  Run
230
230
 
231
- make tag push TAG=vX.X.X
231
+ make tag push TAG=X.X.X
232
232
 
233
233
  to tag and push the container with a specific version number.
@@ -1,5 +1,28 @@
1
1
  = Release Notes
2
2
 
3
+ == Version 3.3.11
4
+ * Fixed that dead lettering only works correctly with global config option.
5
+
6
+ == Version 3.3.10
7
+ * Support configuring RabbitMQ write timeout.
8
+
9
+ == Version 3.3.9
10
+ * Reduce the number of queue policies created on the servers by allowing
11
+ the spefication of a default broker policy. You need to install the
12
+ default policy with a priority of zero to match all queues ever
13
+ created. This feature is optional.
14
+
15
+ == Version 3.3.8
16
+ * Avoid needless put call when updating queue policies.
17
+ It seems the additional call to get the current definition of a policy
18
+ is to be preferred over relying on the idempotency of the PUT call.
19
+ This helps when adding a new fresh server to a beetle cluster: import
20
+ the definitions of one of the existing nodes on the fresh node before
21
+ actually adding it to the list of servers in the client's beetle config..
22
+
23
+ == Version 3.3.7
24
+ * Increased default http api read timeout to handle large server better.
25
+
3
26
  == Version 3.3.6
4
27
  * Fixed a redis connection leak in gobeetle.
5
28
 
@@ -14,6 +14,9 @@ module Beetle
14
14
  # Name of the policy update routing key
15
15
  attr_accessor :beetle_policy_updates_routing_key
16
16
  # default logger (defaults to <tt>Logger.new(log_file)</tt>)
17
+ attr_accessor :broker_default_policy
18
+ # set this to whatever your brokers have installed as default policy. For example, if you have installed
19
+ # a policy that makes every queue lazy, it should be set to <tt>{"queue-moode" => "lazy"}</tt>.
17
20
  attr_accessor :logger
18
21
  # defaults to <tt>STDOUT</tt>
19
22
  attr_accessor :redis_logger
@@ -104,8 +107,10 @@ module Beetle
104
107
  # Whether to update quueue policies synchronously or asynchronously.
105
108
  attr_accessor :update_queue_properties_synchronously
106
109
 
107
- # Read timeout for http requests to create dead letter bindings
110
+ # Read timeout for http requests to RabbitMQ HTTP API
108
111
  attr_accessor :rabbitmq_api_read_timeout
112
+ # Write timeout for http requests to RabbitMQ HTTP API
113
+ attr_accessor :rabbitmq_api_write_timeout
109
114
 
110
115
  # Returns the port on which the Rabbit API is hosted
111
116
  attr_accessor :api_port
@@ -146,6 +151,7 @@ module Beetle
146
151
  self.beetle_policy_exchange_name = "beetle-policies"
147
152
  self.beetle_policy_updates_queue_name = "beetle-policy-updates"
148
153
  self.beetle_policy_updates_routing_key = "beetle.policy.update"
154
+ self.broker_default_policy = {}
149
155
 
150
156
  self.gc_threshold = 1.hour.to_i
151
157
  self.redis_server = "localhost:6379"
@@ -173,7 +179,8 @@ module Beetle
173
179
 
174
180
  self.dead_lettering_enabled = false
175
181
  self.dead_lettering_msg_ttl = 1000 # 1 second
176
- self.rabbitmq_api_read_timeout = 5 # 5 seconds
182
+ self.rabbitmq_api_read_timeout = 60 # 60 seconds
183
+ self.rabbitmq_api_write_timeout = 60 # 60 seconds
177
184
 
178
185
  self.lazy_queues_enabled = false
179
186
  self.throttling_refresh_interval = 60 # seconds
@@ -45,8 +45,11 @@ module Beetle
45
45
  return unless options[:dead_lettering] || options[:lazy]
46
46
 
47
47
  # no need to worry that the server has the port 5672. Net:HTTP will take care of this. See below.
48
- request_url = URI("http://#{server}/api/policies/#{vhost}/#{queue_name}_policy")
49
- request = Net::HTTP::Put.new(request_url)
48
+ policy_name = "#{queue_name}_policy"
49
+ request_url = URI("http://#{server}/api/policies/#{vhost}/#{policy_name}")
50
+ get_request = Net::HTTP::Get.new(request_url)
51
+ put_request = Net::HTTP::Put.new(request_url)
52
+ delete_request = Net::HTTP::Delete.new(request_url)
50
53
 
51
54
  # set up queue policy
52
55
  definition = {}
@@ -58,19 +61,41 @@ module Beetle
58
61
 
59
62
  definition["queue-mode"] = "lazy" if options[:lazy]
60
63
 
61
- request_body = {
64
+ put_request_body = {
62
65
  "pattern" => "^#{queue_name}$",
63
66
  "priority" => 1,
64
67
  "apply-to" => "queues",
65
68
  "definition" => definition,
66
69
  }
67
70
 
68
- response = run_rabbit_http_request(request_url, request) do |http|
69
- http.request(request, request_body.to_json)
71
+ is_default_policy = definition == config.broker_default_policy
72
+
73
+ get_response = run_rabbit_http_request(request_url, get_request) do |http|
74
+ http.request(get_request, nil)
70
75
  end
71
76
 
72
- unless %w(200 201 204).include?(response.code)
73
- log_error("Failed to create policy for queue #{queue_name}", response)
77
+ case get_response.code
78
+ when "200"
79
+ response_body = JSON.parse(get_response.body) rescue {}
80
+ same_policy = put_request_body.all? { |k,v| response_body[k] == v }
81
+ if same_policy
82
+ if is_default_policy
83
+ run_rabbit_http_request(request_url, delete_request) do |http|
84
+ http.request(get_request, nil)
85
+ end
86
+ end
87
+ return :ok
88
+ end
89
+ when "404"
90
+ return :ok if is_default_policy
91
+ end
92
+
93
+ put_response = run_rabbit_http_request(request_url, put_request) do |http|
94
+ http.request(put_request, put_request_body.to_json)
95
+ end
96
+
97
+ unless %w(200 201 204).include?(put_response.code)
98
+ log_error("Failed to create policy for queue #{queue_name}", put_response)
74
99
  raise FailedRabbitRequest.new("Could not create policy")
75
100
  end
76
101
 
@@ -131,9 +156,15 @@ module Beetle
131
156
 
132
157
  def run_rabbit_http_request(uri, request, &block)
133
158
  request.basic_auth(config.user, config.password)
134
- request["Content-Type"] = "application/json"
159
+ case request.class::METHOD
160
+ when 'GET'
161
+ request["Accept"] = "application/json"
162
+ when 'PUT'
163
+ request["Content-Type"] = "application/json"
164
+ end
135
165
  http = Net::HTTP.new(uri.hostname, config.api_port)
136
166
  http.read_timeout = config.rabbitmq_api_read_timeout
167
+ http.write_timeout = config.rabbitmq_api_write_timeout if http.respond_to?(:write_timeout=)
137
168
  # don't do this in production:
138
169
  # http.set_debug_output(logger.instance_eval{ @logdev.dev })
139
170
  http.start do |instance|
@@ -180,7 +180,7 @@ module Beetle
180
180
  processor = Handler.create(handler, opts)
181
181
  result = m.process(processor)
182
182
  if result.reject?
183
- if @client.config.dead_lettering_enabled?
183
+ if @client.queues[queue_name][:dead_lettering]
184
184
  header.reject(:requeue => false)
185
185
  else
186
186
  sleep 1
@@ -1,3 +1,3 @@
1
1
  module Beetle
2
- VERSION = "3.3.6"
2
+ VERSION = "3.3.11"
3
3
  end
@@ -41,6 +41,10 @@ module Beetle
41
41
  end
42
42
 
43
43
  test "creates a policy by posting to the rabbitmq if dead lettering is enabled" do
44
+ stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
45
+ .with(basic_auth: ['guest', 'guest'])
46
+ .to_return(:status => 404)
47
+
44
48
  stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
45
49
  .with(basic_auth: ['guest', 'guest'])
46
50
  .with(:body => {
@@ -56,7 +60,59 @@ module Beetle
56
60
  @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => false, :dead_lettering => true, :routing_key => "QUEUE_NAME_dead_letter")
57
61
  end
58
62
 
63
+ test "skips the PUT call to rabbitmq if the policy is already defined as desired" do
64
+ stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
65
+ .with(basic_auth: ['guest', 'guest'])
66
+ .to_return(:status => 200,
67
+ :body => {
68
+ "vhost" => "/",
69
+ "name" => "QUEUE_NAME_policy",
70
+ "pattern" => "^QUEUE_NAME$",
71
+ "priority" => 1,
72
+ "apply-to" => "queues",
73
+ "definition" => {
74
+ "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
75
+ "dead-letter-exchange" => ""
76
+ }}.to_json)
77
+
78
+ @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => false, :dead_lettering => true, :routing_key => "QUEUE_NAME_dead_letter")
79
+ end
80
+
81
+ test "deletes policy if its definition corresponds to the broker default policy" do
82
+ @config.broker_default_policy = { "queue-mode" => "lazy" }
83
+ stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
84
+ .with(basic_auth: ['guest', 'guest'])
85
+ .to_return(:status => 200,
86
+ :body => {
87
+ "vhost" => "/",
88
+ "name" => "QUEUE_NAME_policy",
89
+ "pattern" => "^QUEUE_NAME$",
90
+ "priority" => 1,
91
+ "apply-to" => "queues",
92
+ "definition" => {
93
+ "queue-mode" => "lazy",
94
+ }}.to_json)
95
+ stub_request(:delete, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
96
+ .with(basic_auth: ['guest', 'guest'])
97
+ .to_return(:status => 204)
98
+
99
+ @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => false, :routing_key => "QUEUE_NAME_dead_letter")
100
+ end
101
+
102
+ test "does nothing if its definition corresponds to the broker default policy and the policy does not exist on the server" do
103
+ @config.broker_default_policy = { "queue-mode" => "lazy" }
104
+ stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
105
+ .with(basic_auth: ['guest', 'guest'])
106
+ .to_return(:status => 404)
107
+
108
+ @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => false, :routing_key => "QUEUE_NAME_dead_letter")
109
+ end
110
+
59
111
  test "creates a policy by posting to the rabbitmq if lazy queues are enabled" do
112
+ stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
113
+ .with(basic_auth: ['guest', 'guest'])
114
+ .to_return(:status => 404)
115
+
60
116
  stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
61
117
  .with(basic_auth: ['guest', 'guest'])
62
118
  .with(:body => {
@@ -72,6 +128,10 @@ module Beetle
72
128
  end
73
129
 
74
130
  test "raises exception when policy couldn't successfully be created" do
131
+ stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
132
+ .with(basic_auth: ['guest', 'guest'])
133
+ .to_return(:status => 404)
134
+
75
135
  stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
76
136
  .with(basic_auth: ['guest', 'guest'])
77
137
  .to_return(:status => [405])
@@ -82,6 +142,10 @@ module Beetle
82
142
  end
83
143
 
84
144
  test "can optionally specify a message ttl" do
145
+ stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
146
+ .with(basic_auth: ['guest', 'guest'])
147
+ .to_return(:status => 404)
148
+
85
149
  stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
86
150
  .with(basic_auth: ['guest', 'guest'])
87
151
  .with(:body => {
@@ -99,6 +163,10 @@ module Beetle
99
163
  end
100
164
 
101
165
  test "properly encodes the vhost from the configuration" do
166
+ stub_request(:get, "http://localhost:15672/api/policies/foo%2F/QUEUE_NAME_policy")
167
+ .with(basic_auth: ['guest', 'guest'])
168
+ .to_return(:status => 404)
169
+
102
170
  stub_request(:put, "http://localhost:15672/api/policies/foo%2F/QUEUE_NAME_policy")
103
171
  .with(basic_auth: ['guest', 'guest'])
104
172
  .with(:body => {
@@ -224,13 +224,11 @@ module Beetle
224
224
  end
225
225
  end
226
226
 
227
-
228
227
  class DeadLetteringCallBackExecutionTest < Minitest::Test
229
228
  def setup
230
229
  @client = Client.new
231
- @client.config.dead_lettering_enabled = true
232
230
  @queue = "somequeue"
233
- @client.register_queue(@queue)
231
+ @client.register_queue(@queue, :dead_lettering => true)
234
232
  @sub = @client.send(:subscriber)
235
233
  mq = mock("MQ")
236
234
  mq.expects(:closing?).returns(false)
@@ -239,11 +237,7 @@ module Beetle
239
237
  @handler = Handler.create(lambda{|*args| raise @exception})
240
238
  # handler method 'processing_completed' should be called under all circumstances
241
239
  @handler.expects(:processing_completed).once
242
- @callback = @sub.send(:create_subscription_callback, "my myessage", @queue, @handler, :exceptions => 1)
243
- end
244
-
245
- def teardown
246
- @client.config.dead_lettering_enabled = false
240
+ @callback = @sub.send(:create_subscription_callback, @queue, @queue, @handler, :exceptions => 1)
247
241
  end
248
242
 
249
243
  test "should call reject on the message header when processing the handler returns true on reject? if dead lettering has been enabled" do
@@ -255,19 +249,18 @@ module Beetle
255
249
  header.expects(:reject).with(:requeue => false)
256
250
  @callback.call(header, 'foo')
257
251
  end
258
-
259
252
  end
260
253
 
261
254
  class CallBackExecutionTest < Minitest::Test
262
255
  def setup
263
- client = Client.new
256
+ @client = Client.new
264
257
  @queue = "somequeue"
265
- client.register_queue(@queue)
266
- @sub = client.send(:subscriber)
258
+ @client.register_queue(@queue)
259
+ @sub = @client.send(:subscriber)
267
260
  @exception = Exception.new "murks"
268
261
  @handler = Handler.create(lambda{|*args| raise @exception})
269
262
  @handler.instance_eval { def post_process; raise "shoot"; end }
270
- @callback = @sub.send(:create_subscription_callback, "my myessage", @queue, @handler, :exceptions => 1)
263
+ @callback = @sub.send(:create_subscription_callback, @queue, @queue, @handler, :exceptions => 1)
271
264
  end
272
265
 
273
266
  test "exceptions raised from message processing should be ignored" do
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: beetle
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.3.6
4
+ version: 3.3.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - Stefan Kaes
@@ -9,10 +9,10 @@ authors:
9
9
  - Ali Jelveh
10
10
  - Sebastian Roebke
11
11
  - Larry Baltz
12
- autorequire:
12
+ autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2020-04-24 00:00:00.000000000 Z
15
+ date: 2020-06-18 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: bunny
@@ -359,7 +359,7 @@ homepage: https://xing.github.com/beetle/
359
359
  licenses: []
360
360
  metadata:
361
361
  changelog_uri: https://github.com/xing/beetle/blob/master/RELEASE_NOTES.rdoc
362
- post_install_message:
362
+ post_install_message:
363
363
  rdoc_options:
364
364
  - "--charset=UTF-8"
365
365
  require_paths:
@@ -376,7 +376,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
376
376
  version: 1.3.7
377
377
  requirements: []
378
378
  rubygems_version: 3.0.8
379
- signing_key:
379
+ signing_key:
380
380
  specification_version: 3
381
381
  summary: High Availability AMQP Messaging with Redundant Queues
382
382
  test_files: