beetle 3.3.6 → 3.3.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6552c1aad5c0d816b274200e19198b55f459e4134726f532742a199d181d0e86
-  data.tar.gz: 6d5f931794d753e0b4e9100e324007789ae8f62f4a20b539345a9bfc567efe05
+  metadata.gz: fd1ee5b22d344002924b2b648e036ce98584e270b05af6195115c5bc6aef6438
+  data.tar.gz: f32ef2c465d81cb73ac8453b677419c9117e1f2ab62126feb62dd2227cc1e924
 SHA512:
-  metadata.gz: cc71d5c9dd604cb3cabbb40894289105aa1b840da347d1f06ddabc66f004c1e25bb34635c6902076f17465fccdd6fffe32c8b1e628949bdda83ef17a862d2b92
-  data.tar.gz: 03a1e736d4f1c84528687b666efff22af82e885764c10df7e136a29ecb3cfc77beb234b1e97733c2d9078473a3889dc73b51c48d73863ffd940b76accd807c85
+  metadata.gz: 9521152d635cb12bd96dd87a8153cc20f4d8f9471d9a3d08c2d8d786a53bad2a76b783140ce1e8481b90c2b8dd1f6a391988985380fc4efcc93c30ba01ee013f
+  data.tar.gz: c29b033bcb301006b22bcbeeb4a9e62b6d72a5bcf54093e01aee405ea2c5a8d0f65c3a0e6a69bce0cbe0047cf52780069ae6c9a7d68c53161fea5de0c8b09b27

data/README.rdoc

@@ -228,6 +228,6 @@ https://hub.docker.com/r/xingarchitects/gobeetle/.
 
 Run
 
-    make tag push TAG=vX.X.X
+    make tag push TAG=X.X.X
 
 to tag and push the container with a specific version number.

data/RELEASE_NOTES.rdoc

@@ -1,5 +1,28 @@
 = Release Notes
 
+== Version 3.3.11
+* Fixed that dead lettering only works correctly with global config option.
+
+== Version 3.3.10
+* Support configuring RabbitMQ write timeout.
+
+== Version 3.3.9
+* Reduce the number of queue policies created on the servers by allowing
+  the spefication of a default broker policy. You need to install the
+  default policy with a priority of zero to match all queues ever
+  created. This feature is optional.
+
+== Version 3.3.8
+* Avoid needless put call when updating queue policies.
+  It seems the additional call to get the current definition of a policy
+  is to be preferred over relying on the idempotency of the PUT call.
+  This helps when adding a new fresh server to a beetle cluster: import
+  the definitions of one of the existing nodes on the fresh node before
+  actually adding it to the list of servers in the client's beetle config..
+
+== Version 3.3.7
+* Increased default http api read timeout to handle large server better.
+
 == Version 3.3.6
 * Fixed a redis connection leak in gobeetle.
 

data/lib/beetle/configuration.rb

@@ -14,6 +14,9 @@ module Beetle
     # Name of the policy update routing key
     attr_accessor :beetle_policy_updates_routing_key
     # default logger (defaults to <tt>Logger.new(log_file)</tt>)
+    attr_accessor :broker_default_policy
+    # set this to whatever your brokers have installed as default policy. For example, if you have installed
+    # a policy that makes every queue lazy, it should be set to <tt>{"queue-moode" => "lazy"}</tt>.
     attr_accessor :logger
     # defaults to <tt>STDOUT</tt>
     attr_accessor :redis_logger
@@ -104,8 +107,10 @@ module Beetle
     # Whether to update quueue policies synchronously or asynchronously.
     attr_accessor :update_queue_properties_synchronously
 
-    # Read timeout for http requests to create dead letter bindings
+    # Read timeout for http requests to RabbitMQ HTTP API
     attr_accessor :rabbitmq_api_read_timeout
+    # Write timeout for http requests to RabbitMQ HTTP API
+    attr_accessor :rabbitmq_api_write_timeout
 
     # Returns the port on which the Rabbit API is hosted
     attr_accessor :api_port
@@ -146,6 +151,7 @@ module Beetle
       self.beetle_policy_exchange_name = "beetle-policies"
       self.beetle_policy_updates_queue_name = "beetle-policy-updates"
       self.beetle_policy_updates_routing_key = "beetle.policy.update"
+      self.broker_default_policy = {}
 
       self.gc_threshold = 1.hour.to_i
       self.redis_server = "localhost:6379"
@@ -173,7 +179,8 @@ module Beetle
 
       self.dead_lettering_enabled = false
       self.dead_lettering_msg_ttl = 1000   # 1 second
-      self.rabbitmq_api_read_timeout = 5   # 5 seconds
+      self.rabbitmq_api_read_timeout = 60  # 60 seconds
+      self.rabbitmq_api_write_timeout = 60  # 60 seconds
 
       self.lazy_queues_enabled = false
       self.throttling_refresh_interval = 60 # seconds

data/lib/beetle/queue_properties.rb

@@ -45,8 +45,11 @@ module Beetle
       return unless options[:dead_lettering] || options[:lazy]
 
       # no need to worry that the server has the port 5672. Net:HTTP will take care of this. See below.
-      request_url = URI("http://#{server}/api/policies/#{vhost}/#{queue_name}_policy")
-      request = Net::HTTP::Put.new(request_url)
+      policy_name = "#{queue_name}_policy"
+      request_url = URI("http://#{server}/api/policies/#{vhost}/#{policy_name}")
+      get_request = Net::HTTP::Get.new(request_url)
+      put_request = Net::HTTP::Put.new(request_url)
+      delete_request = Net::HTTP::Delete.new(request_url)
 
       # set up queue policy
       definition = {}
@@ -58,19 +61,41 @@ module Beetle
 
       definition["queue-mode"] = "lazy" if options[:lazy]
 
-      request_body = {
+      put_request_body = {
         "pattern" => "^#{queue_name}$",
         "priority" => 1,
         "apply-to" => "queues",
         "definition" => definition,
       }
 
-      response = run_rabbit_http_request(request_url, request) do |http|
-        http.request(request, request_body.to_json)
+      is_default_policy = definition == config.broker_default_policy
+
+      get_response = run_rabbit_http_request(request_url, get_request) do |http|
+        http.request(get_request, nil)
       end
 
-      unless %w(200 201 204).include?(response.code)
-        log_error("Failed to create policy for queue #{queue_name}", response)
+      case get_response.code
+      when "200"
+        response_body = JSON.parse(get_response.body) rescue {}
+        same_policy = put_request_body.all? { |k,v| response_body[k] == v }
+        if same_policy
+          if is_default_policy
+            run_rabbit_http_request(request_url, delete_request) do |http|
+              http.request(get_request, nil)
+            end
+          end
+          return :ok
+        end
+      when "404"
+        return :ok if is_default_policy
+      end
+
+      put_response = run_rabbit_http_request(request_url, put_request) do |http|
+        http.request(put_request, put_request_body.to_json)
+      end
+
+      unless %w(200 201 204).include?(put_response.code)
+        log_error("Failed to create policy for queue #{queue_name}", put_response)
         raise FailedRabbitRequest.new("Could not create policy")
       end
 
@@ -131,9 +156,15 @@ module Beetle
 
     def run_rabbit_http_request(uri, request, &block)
       request.basic_auth(config.user, config.password)
-      request["Content-Type"] = "application/json"
+      case request.class::METHOD
+      when 'GET'
+        request["Accept"] = "application/json"
+      when 'PUT'
+        request["Content-Type"] = "application/json"
+      end
       http = Net::HTTP.new(uri.hostname, config.api_port)
       http.read_timeout = config.rabbitmq_api_read_timeout
+      http.write_timeout = config.rabbitmq_api_write_timeout if http.respond_to?(:write_timeout=)
       # don't do this in production:
       # http.set_debug_output(logger.instance_eval{ @logdev.dev })
       http.start do |instance|

data/lib/beetle/subscriber.rb

@@ -180,7 +180,7 @@ module Beetle
           processor = Handler.create(handler, opts)
           result = m.process(processor)
           if result.reject?
-            if @client.config.dead_lettering_enabled?
+            if @client.queues[queue_name][:dead_lettering]
               header.reject(:requeue => false)
             else
               sleep 1

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "3.3.6"
+  VERSION = "3.3.11"
 end

data/test/beetle/queue_properties_test.rb

@@ -41,6 +41,10 @@ module Beetle
     end
 
     test "creates a policy by posting to the rabbitmq if dead lettering is enabled" do
+      stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 404)
+
       stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
         .with(basic_auth: ['guest', 'guest'])
         .with(:body => {
@@ -56,7 +60,59 @@ module Beetle
       @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => false, :dead_lettering => true, :routing_key => "QUEUE_NAME_dead_letter")
     end
 
+    test "skips the PUT call to rabbitmq if the policy is already defined as desired" do
+      stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 200,
+                   :body => {
+                     "vhost" => "/",
+                     "name" => "QUEUE_NAME_policy",
+                     "pattern" => "^QUEUE_NAME$",
+                     "priority" => 1,
+                     "apply-to" => "queues",
+                     "definition" => {
+                       "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
+                       "dead-letter-exchange" => ""
+                     }}.to_json)
+
+      @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => false, :dead_lettering => true, :routing_key => "QUEUE_NAME_dead_letter")
+    end
+
+    test "deletes policy if its definition corresponds to the broker default policy" do
+      @config.broker_default_policy = { "queue-mode" => "lazy" }
+      stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 200,
+                   :body => {
+                     "vhost" => "/",
+                     "name" => "QUEUE_NAME_policy",
+                     "pattern" => "^QUEUE_NAME$",
+                     "priority" => 1,
+                     "apply-to" => "queues",
+                     "definition" => {
+                       "queue-mode" => "lazy",
+                     }}.to_json)
+      stub_request(:delete, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 204)
+
+      @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => false, :routing_key => "QUEUE_NAME_dead_letter")
+    end
+
+    test "does nothing if its definition corresponds to the broker default policy and the policy does not exist on the server" do
+      @config.broker_default_policy = { "queue-mode" => "lazy" }
+      stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 404)
+
+      @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => false, :routing_key => "QUEUE_NAME_dead_letter")
+    end
+
     test "creates a policy by posting to the rabbitmq if lazy queues are enabled" do
+      stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 404)
+
       stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
         .with(basic_auth: ['guest', 'guest'])
         .with(:body => {
@@ -72,6 +128,10 @@ module Beetle
     end
 
     test "raises exception when policy couldn't successfully be created" do
+      stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 404)
+
       stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
         .with(basic_auth: ['guest', 'guest'])
         .to_return(:status => [405])
@@ -82,6 +142,10 @@ module Beetle
     end
 
     test "can optionally specify a message ttl" do
+      stub_request(:get, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 404)
+
       stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
         .with(basic_auth: ['guest', 'guest'])
         .with(:body => {
@@ -99,6 +163,10 @@ module Beetle
     end
 
     test "properly encodes the vhost from the configuration" do
+      stub_request(:get, "http://localhost:15672/api/policies/foo%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 404)
+
       stub_request(:put, "http://localhost:15672/api/policies/foo%2F/QUEUE_NAME_policy")
         .with(basic_auth: ['guest', 'guest'])
         .with(:body => {

data/test/beetle/subscriber_test.rb

@@ -224,13 +224,11 @@ module Beetle
     end
   end
 
-
   class DeadLetteringCallBackExecutionTest < Minitest::Test
     def setup
       @client = Client.new
-      @client.config.dead_lettering_enabled = true
       @queue = "somequeue"
-      @client.register_queue(@queue)
+      @client.register_queue(@queue, :dead_lettering => true)
       @sub = @client.send(:subscriber)
       mq = mock("MQ")
       mq.expects(:closing?).returns(false)
@@ -239,11 +237,7 @@ module Beetle
       @handler = Handler.create(lambda{|*args| raise @exception})
       # handler method 'processing_completed' should be called under all circumstances
       @handler.expects(:processing_completed).once
-      @callback = @sub.send(:create_subscription_callback, "my myessage", @queue, @handler, :exceptions => 1)
-    end
-
-    def teardown
-      @client.config.dead_lettering_enabled = false
+      @callback = @sub.send(:create_subscription_callback, @queue, @queue, @handler, :exceptions => 1)
     end
 
     test "should call reject on the message header when processing the handler returns true on reject? if dead lettering has been enabled" do
@@ -255,19 +249,18 @@ module Beetle
       header.expects(:reject).with(:requeue => false)
       @callback.call(header, 'foo')
     end
-
   end
 
   class CallBackExecutionTest < Minitest::Test
     def setup
-      client = Client.new
+      @client = Client.new
       @queue = "somequeue"
-      client.register_queue(@queue)
-      @sub = client.send(:subscriber)
+      @client.register_queue(@queue)
+      @sub = @client.send(:subscriber)
       @exception = Exception.new "murks"
       @handler = Handler.create(lambda{|*args| raise @exception})
       @handler.instance_eval { def post_process; raise "shoot"; end }
-      @callback = @sub.send(:create_subscription_callback, "my myessage", @queue, @handler, :exceptions => 1)
+      @callback = @sub.send(:create_subscription_callback, @queue, @queue, @handler, :exceptions => 1)
     end
 
     test "exceptions raised from message processing should be ignored" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 3.3.6
+  version: 3.3.11
 platform: ruby
 authors:
 - Stefan Kaes
@@ -9,10 +9,10 @@ authors:
 - Ali Jelveh
 - Sebastian Roebke
 - Larry Baltz
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-24 00:00:00.000000000 Z
+date: 2020-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -359,7 +359,7 @@ homepage: https://xing.github.com/beetle/
 licenses: []
 metadata:
   changelog_uri: https://github.com/xing/beetle/blob/master/RELEASE_NOTES.rdoc
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--charset=UTF-8"
 require_paths:
@@ -376,7 +376,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.7
 requirements: []
 rubygems_version: 3.0.8
-signing_key: 
+signing_key:
 specification_version: 3
 summary: High Availability AMQP Messaging with Redundant Queues
 test_files: