beetle 3.0.0.rc5 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ae9b83119623fc200b35d2533bb1044cbad99b2b137235fc2bc03709584f244
-  data.tar.gz: a027e1eb96f2a87d0326091ecdf9e01bd0be7990479e24bcda32b3615d8638bb
+  metadata.gz: 8a90e706d339d2a9e458c0b82113b58d5be98243fa9c9adbeaafb12fadeeb3d1
+  data.tar.gz: d89db5e989cfa51558ebda9db9c6acd10170c27fda0135378157e252570be626
 SHA512:
-  metadata.gz: 045d2183037e6506e96b39df9b1a8b2e61e1bb6d9048270003903d4fd1dde6b2ff52be6cb095148f5b612d5cbb94ed7ffe55f913ba93c47034227f9fad61038a
-  data.tar.gz: 0ed9d1bc8fc16815bb4bf396aa47d3d47b52c9473d9a9696c53bfd5ae3076d646aaaf47c2177919ea1bd23cb5b49e3e3c919c44955502f663c1792b086f3644b
+  metadata.gz: ee25e895ba9f3f38afe6d4985297435d93cab432b0dd8d483b8890a25b58f89d9993fac429c0d710bfeed49568dd1a28c948a0241297c1352ad2200987e0aebc
+  data.tar.gz: d1dff384a6bd1da983b2c400e524970d086759ae7c24106089ef0e36a167b7cb9b8769f6938b96b9a5b78435abb945cddba432b14755716f67c96e508783ef51

data/RELEASE_NOTES.rdoc

@@ -1,36 +1,30 @@
 = Release Notes
 
-== Version 3.0.0.rc5
-* ake sure to clean dedup store when ack_count is larger than 2
+== Version 3.0.0
 
-== Version 3.0.0.rc4
-* fixed incorrect calculation of expiry time in garbage collector
-
-
-== Version 3.0.0.rc3
-* store redis gc stats in redis and display them in the configuration
-  server web UI
-
-== Version 3.0.0.rc2
+* provide client method to setup queues and queue policies.
+  Setting up queue policies on demand in publisher and subscriber is OK
+  for a small number of queues, publishers and subscribers. But the HTTP
+  API of RabbitMQ doesn't scale all that well, so that a large number
+  of HTTP calls to set up queue policies can in fact crash a server.
+* allow queue mode specification and dead lettering specification on a
+  per queue basis.
 * change policy setup to be asynchronous, on demand in publisher and
   consumer. Users have to run a processors to listen to messages and
   call setup_queue_policies! with the parses JSON paylod of the
   message.
-* added dump_expiries command to beetle to dump dediplication store
-  expiry times.
-* added delete_queue_keys command to beetle to allow deletion of
-  excess dedup store entries for a given queue.
+* store redis gc stats in redis and display them in the configuration
+  server web UI
 * config server: store current master server in consul, if consul
   config has been provided.
 * config server: server correctly updates configured client ids when
   they change in consul.
-
-== Version 3.0.0.rc1
-* provide client method to setup queues and queue policies.
-  Setting up queue policies on demand in publisher and subscriber is OK
-  for a small number of queues, publishers and subscribers. But the HTTP
-  API of RabbitMQ doesn't scale all that well, so that a large number
-  of HTTP calls to set up queue policies can in fact crash a server.
+* don't create dead letter queues when using the trace functionality
+* make sure to clean dedup store when ack_count is larger than 2
+* added dump_expiries command to beetle to dump dediplication store
+  expiry times.
+* added delete_queue_keys command to beetle to allow deletion of
+  excess dedup store entries for a given queue.
 
 == Version 2.3.2
 * config server: fixed a race condition when accessing server state.

data/Rakefile

@@ -85,7 +85,7 @@ end
 namespace :consul do
   desc "start consul agent in development mode"
   task :start do
-    system "consul agent -dev -node machine"
+    exec "consul agent -dev -node machine"
   end
 end
 

data/examples/throttling.rb

@@ -0,0 +1,68 @@
+# throttling.rb
+# this example shows you how throttling the publisher works
+#
+# ! check the examples/README.rdoc for information on starting your redis/rabbit !
+#
+# start it with ruby throttling.rb
+
+require "rubygems"
+require File.expand_path("../lib/beetle", File.dirname(__FILE__))
+
+# set Beetle log level to info, less noisy than debug
+Beetle.config.logger.level = Logger::INFO
+# override default of 60 seconds
+Beetle.config.throttling_refresh_interval = 5
+
+# setup clients
+client = Beetle::Client.new
+client.register_queue(:test)
+client.register_message(:test)
+client.throttle(:test => 50)
+
+consumer = Beetle::Client.new
+consumer.register_queue(:test)
+consumer.register_message(:test)
+
+# purge the test queue
+client.purge(:test)
+
+# empty the dedup store
+client.deduplication_store.flushdb
+
+# register our handler to the message, check out the message.rb for more stuff you can get from the message object
+messages_received = 0
+consumer.register_handler(:test) do |message|
+  sleep 0.2
+  messages_received += 1
+  puts "Received #{messages_received} messages"
+end
+
+interrupted = false
+
+t = Thread.new do
+  c = 0
+  loop do
+    break if c == 200 || interrupted
+    100.times do |i|
+      client.publish(:test, (c+=1).to_s)
+      puts "Published message #{c}"
+      sleep 0.1
+    end
+  end
+end
+
+trap('INT') do
+  interrupted = true
+  consumer.stop_listening
+end
+
+# start listening
+# this starts the event machine event loop using EM.run
+# the block passed to listen will be yielded as the last step of the setup process
+consumer.listen do
+  EM.add_periodic_timer(1) do
+    consumer.stop_listening if messages_received >= 200
+  end
+end
+
+t.join

data/lib/beetle.rb

@@ -50,7 +50,7 @@ module Beetle
   # use ruby's autoload mechanism for loading beetle classes
   lib_dir = File.expand_path(File.dirname(__FILE__) + '/beetle/')
   Dir["#{lib_dir}/*.rb"].each do |libfile|
-    autoload File.basename(libfile)[/^(.*)\.rb$/, 1].classify, libfile
+    autoload File.basename(libfile)[/^(.*)\.rb$/, 1].camelize, libfile
   end
 
   require "#{lib_dir}/redis_ext"

data/lib/beetle/base.rb

@@ -82,6 +82,7 @@ module Beetle
       end
       return {
         :queue_name => target_queue,
+        :bindings => @client.bindings[target_queue],
         :dead_letter_queue_name => dead_letter_queue_name,
         :message_ttl => @client.config.dead_lettering_msg_ttl,
       }.merge(policy_options)

data/lib/beetle/client.rb

@@ -55,7 +55,7 @@ module Beetle
       @messages = {}
       @bindings = {}
       @deduplication_store = DeduplicationStore.new(config)
-      @dead_lettering = DeadLettering.new(config)
+      @queue_properties = QueueProperties.new(config)
       load_brokers_from_config
       register_exchange(config.beetle_policy_exchange_name)
       # make sure dead lettering is false for the policy update queue
@@ -258,6 +258,27 @@ module Beetle
       subscriber.resume_listening(queues)
     end
 
+    # throttle publishing of messages based on queue lengths.
+    #
+    #    client.throttle(:foo => 10000, :bar => 10_000)
+    #
+    # throttle publisher to 1 message per second as long as queue foo has more
+    # than 10000 entries or queue bar has more than 100000 entries across all
+    # servers. Queue lenghts are periodically determined during publishing. You
+    # only want to use this feature if you plan to publish a huge amount of
+    # messages to slow consumers so as to not overload the broker or the redis
+    # deduplication store. You'll want to use this feature when running background
+    # jobs that publish huge amounts of messages to avoid overloading brokers
+    # and the message deduplication store.
+    def throttle(throttling_options)
+      publisher.throttle(throttling_options.stringify_keys)
+    end
+
+    # is the publisher currently throttled?
+    def throttled?
+      publisher.throttled?
+    end
+
     # traces queues without consuming them. useful for debugging message flow.
     def trace(queue_names=self.queues.keys, tracer=nil, &block)
       queues_to_trace = self.queues.slice(*queue_names)
@@ -275,7 +296,9 @@ module Beetle
           puts "DATA: #{msg.data}"
         end
       register_handler(queue_names){|msg| tracer.call msg }
+      @subscriber.tracing = true
       listen_queues(queue_names, &block)
+      @subscriber.tracing = false
     end
 
     # evaluate the ruby files matching the given +glob+ pattern in the context of the client instance.
@@ -299,8 +322,8 @@ module Beetle
       @subscriber = nil
     end
 
-    def set_queue_policies!(message_payload)
-      @dead_lettering.set_queue_policies!(message_payload)
+    def update_queue_properties!(message_payload)
+      @queue_properties.update_queue_properties!(message_payload)
     end
 
     private
@@ -350,7 +373,7 @@ module Beetle
     end
 
     def queue_name_for_tracing(queue)
-      "trace-#{queue}-#{Beetle.hostname}-#{$$}"
+      "trace-#{queue}-#{Beetle.hostname}"
     end
 
     def load_brokers_from_config

data/lib/beetle/configuration.rb

@@ -93,7 +93,7 @@ module Beetle
     attr_accessor :dead_lettering_msg_ttl
 
     # Read timeout for http requests to create dead letter bindings
-    attr_accessor :dead_lettering_read_timeout
+    attr_accessor :rabbitmq_api_read_timeout
 
     # Returns the port on which the Rabbit API is hosted
     attr_accessor :api_port
@@ -112,6 +112,9 @@ module Beetle
     # decreased parallelism.
     attr_accessor :prefetch_count
 
+    # refresh interval for determining queue lenghts for throttling.
+    attr_accessor :throttling_refresh_interval
+
     # directory to store large intermediate files (defaults '/tmp')
     attr_accessor :tmpdir
 
@@ -156,13 +159,14 @@ module Beetle
       self.prefetch_count = 1
 
       self.dead_lettering_enabled = false
-      self.dead_lettering_msg_ttl = 1000 #1 second
-      self.dead_lettering_read_timeout = 3 #3 seconds
+      self.dead_lettering_msg_ttl = 1000   # 1 second
+      self.rabbitmq_api_read_timeout = 5   # 5 seconds
 
       self.lazy_queues_enabled = false
+      self.throttling_refresh_interval = 60 # seconds
 
       self.publishing_timeout = 0
-      self.publisher_connect_timeout = 5
+      self.publisher_connect_timeout = 5   # seconds
       self.tmpdir = "/tmp"
 
       self.log_file = STDOUT

data/lib/beetle/publisher.rb

@@ -9,9 +9,24 @@ module Beetle
       @exchanges_with_bound_queues = {}
       @dead_servers = {}
       @bunnies = {}
+      @throttling_options = {}
+      @next_throttle_refresh = Time.now
+      @throttled = false
       at_exit { stop }
     end
 
+    def throttled?
+      @throttled
+    end
+
+    def throttling?
+      !@throttling_options.empty?
+    end
+
+    def throttling_status
+      @throttled ? 'throttled' : 'unthrottled'
+    end
+
     # list of exceptions potentially raised by bunny
     # these need to be lazy, because qrack exceptions are only defined after a connection has been established
     def bunny_exceptions
@@ -29,6 +44,7 @@ module Beetle
         exchange_name = opts.delete(:exchange)
         opts.delete(:queue)
         recycle_dead_servers unless @dead_servers.empty?
+        throttle!
         if opts[:redundant]
           publish_with_redundancy(exchange_name, message_name, data, opts)
         else
@@ -135,6 +151,16 @@ module Beetle
       [status, result]
     end
 
+    def throttle(queue_options)
+      @throttling_options = queue_options
+    end
+
+    def throttle!
+      return unless throttling?
+      refresh_throttling!
+      sleep 1 if throttled?
+    end
+
     def purge(queue_names) #:nodoc:
       each_server do
         queue_names.each do |name|
@@ -251,5 +277,30 @@ module Beetle
       @exchanges[@server] = {}
       @queues[@server] = {}
     end
+
+    def refresh_throttling!
+      t = Time.now
+      return if t < @next_throttle_refresh
+      @next_throttle_refresh = t + @client.config.throttling_refresh_interval
+      old_throttled = @throttled
+      @throttled = false
+      @throttling_options.each do |queue_name, max_length|
+        begin
+          len = 0
+          each_server do
+            len += queue(queue_name).status[:message_count]
+          end
+          # logger.debug "Beetle: queue '#{queue_name}' has size #{len}"
+          if len > max_length
+            @throttled = true
+            break
+          end
+        rescue => e
+          logger.warn "Beetle: could not fetch queue length for queue '#{queue_name}': #{e}"
+        end
+      end
+      logger.info "Beetle: publisher #{throttling_status}" if @throttled != old_throttled
+    end
+
   end
 end

data/lib/beetle/{dead_lettering.rb → queue_properties.rb}

@@ -2,7 +2,7 @@ require 'net/http'
 require 'json'
 
 module Beetle
-  class DeadLettering
+  class QueueProperties
     class FailedRabbitRequest < StandardError; end
 
     attr_reader :config
@@ -11,8 +11,12 @@ module Beetle
       @config = config
     end
 
-    def set_queue_policies!(options)
-      # logger.debug "Setting queue policies: #{options.inspect}"
+    def vhost
+      CGI.escape(@config.vhost)
+    end
+
+    def update_queue_properties!(options)
+      logger.info "Updating queue properties: #{options.inspect}"
       options = options.symbolize_keys
       server = options[:server]
       target_queue = options[:queue_name]
@@ -21,6 +25,7 @@ module Beetle
 
       target_queue_options = policy_options.merge(:routing_key => dead_letter_queue_name)
       set_queue_policy!(server, target_queue, target_queue_options)
+      remove_obsolete_bindings(server, target_queue, options[:bindings]) if options.has_key?(:bindings)
 
       dead_letter_queue_options = policy_options.merge(:routing_key => target_queue, :message_ttl => options[:message_ttl])
       set_queue_policy!(server, dead_letter_queue_name, dead_letter_queue_options)
@@ -34,7 +39,6 @@ module Beetle
 
       return unless options[:dead_lettering] || options[:lazy]
 
-      vhost = CGI.escape(config.vhost)
       # no need to worry that the server has the port 5672. Net:HTTP will take care of this. See below.
       request_url = URI("http://#{server}/api/policies/#{vhost}/#{queue_name}_policy")
       request = Net::HTTP::Put.new(request_url)
@@ -68,11 +72,63 @@ module Beetle
       :ok
     end
 
+    def remove_obsolete_bindings(server, queue_name, bindings)
+      logger.debug "Removing obsolete bindings"
+      raise ArgumentError.new("server missing")     if server.blank?
+      raise ArgumentError.new("queue name missing") if queue_name.blank?
+      raise ArgumentError.new("bindings missing")   if bindings.nil?
+
+      desired_bindings = bindings.each_with_object({}) do |b, desired|
+        desired[[b[:exchange], b[:key]]] = b.except(:exchange, :key)
+      end
+
+      server_bindings = retrieve_bindings(server, queue_name)
+      server_bindings.each do |b|
+        next unless b["destination_type"] == "queue" || b["destination"] == queue_name
+        next if b["source"] == ""
+        source_route = b.values_at("source", "routing_key")
+        unless desired_bindings.has_key?(source_route)
+          logger.info "Removing obsolete binding: #{b.inspect}"
+          remove_binding(server, queue_name, b["source"], b["properties_key"])
+        end
+      end
+    end
+
+    def retrieve_bindings(server, queue_name)
+      request_url = URI("http://#{server}/api/queues/#{vhost}/#{queue_name}/bindings")
+      request = Net::HTTP::Get.new(request_url)
+
+      response = run_rabbit_http_request(request_url, request) do |http|
+        http.request(request)
+      end
+
+      unless response.code == "200"
+        log_error("Failed to retrieve bindings for queue #{queue_name}", response)
+        raise FailedRabbitRequest.new("Could not retrieve queue bindings")
+      end
+
+      JSON.parse(response.body)
+    end
+
+    def remove_binding(server, queue_name, exchange, properties_key)
+      request_url = URI("http://#{server}/api/bindings/#{vhost}/e/#{exchange}/q/#{queue_name}/#{properties_key}")
+      request = Net::HTTP::Delete.new(request_url)
+
+      response = run_rabbit_http_request(request_url, request) do |http|
+        http.request(request)
+      end
+
+      unless %w(200 201 204).include?(response.code)
+        log_error("Failed to remove obsolete binding for queue #{queue_name}", response)
+        raise FailedRabbitRequest.new("Could not retrieve queue bindings")
+      end
+    end
+
     def run_rabbit_http_request(uri, request, &block)
       request.basic_auth(config.user, config.password)
       request["Content-Type"] = "application/json"
       http = Net::HTTP.new(uri.hostname, config.api_port)
-      http.read_timeout = config.dead_lettering_read_timeout
+      http.read_timeout = config.rabbitmq_api_read_timeout
       # don't do this in production:
       # http.set_debug_output(logger.instance_eval{ @logdev.dev })
       http.start do |instance|

data/lib/beetle/subscriber.rb

@@ -4,6 +4,11 @@ module Beetle
   # Manages subscriptions and message processing on the receiver side of things.
   class Subscriber < Base
 
+    attr_accessor :tracing
+    def tracing?
+      @tracing
+    end
+
     # create a new subscriber instance
     def initialize(client, options = {}) #:nodoc:
       super
@@ -14,6 +19,7 @@ module Beetle
       @subscriptions = {}
       @listened_queues = []
       @channels_closed = false
+      @tracing = false
     end
 
     # the client calls this method to subscribe to a list of queues.
@@ -206,8 +212,11 @@ module Beetle
 
     def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
       queue = channel.queue(queue_name, creation_keys)
-      policy_options = bind_dead_letter_queue!(channel, queue_name, creation_keys)
-      publish_policy_options(policy_options)
+      unless tracing?
+        # we don't want to create dead-letter queues for tracing
+        policy_options = bind_dead_letter_queue!(channel, queue_name, creation_keys)
+        publish_policy_options(policy_options)
+      end
       exchange = exchange(exchange_name)
       queue.bind(exchange, binding_keys)
       queue

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "3.0.0.rc5"
+  VERSION = "3.0.0"
 end

data/test/beetle/base_test.rb

@@ -68,6 +68,7 @@ module Beetle
       channel = stub('channel')
       expected_options = {
         :queue_name => "QUEUE_NAME",
+        :bindings=>[{:exchange=>"QUEUE_NAME", :key=>"QUEUE_NAME"}],
         :dead_letter_queue_name=>"QUEUE_NAME_dead_letter",
         :message_ttl => 1000,
         :dead_lettering => false,
@@ -84,6 +85,7 @@ module Beetle
 
       expected_options = {
         :queue_name => "QUEUE_NAME",
+        :bindings=>[{:exchange=>"QUEUE_NAME", :key=>"QUEUE_NAME"}],
         :dead_letter_queue_name=>"QUEUE_NAME_dead_letter",
         :message_ttl => 1000,
         :dead_lettering => true,

data/test/beetle/client_test.rb

@@ -356,10 +356,10 @@ module Beetle
       client.stop_listening
     end
 
-    test "should delegate set_queue_policies! to the dead lettering instance" do
+    test "should delegate update_queue_properties! to the queue policies instance" do
       client = Client.new
-      client.instance_variable_get(:@dead_lettering).expects(:set_queue_policies!)
-      client.set_queue_policies!({})
+      client.instance_variable_get(:@queue_properties).expects(:update_queue_properties!)
+      client.update_queue_properties!({})
     end
 
     test "should delegate pause_listening to the subscriber instance" do
@@ -430,6 +430,18 @@ module Beetle
       client.trace(["test"])
     end
 
+    test "delegates setting throttling options to the publisher instance and stringifies the queue names" do
+      client = Client.new
+      client.send(:publisher).expects(:throttle).with("test" => 1)
+      client.throttle(:test => 1)
+    end
+
+    test "delegates querying the throttling status to the publisher instance" do
+      client = Client.new
+      client.send(:publisher).expects(:throttled?)
+      client.throttled?
+    end
+
     def message_stub_for_tracing
       header_stub = stub_everything("header")
       header_stub.stubs(:method).returns(stub_everything("method"))

data/test/beetle/publisher_test.rb

@@ -80,6 +80,7 @@ module Beetle
       @pub.send(:mark_server_dead)
       publishing = sequence('publishing')
       @pub.expects(:recycle_dead_servers).in_sequence(publishing)
+      @pub.expects(:throttle!).in_sequence(publishing)
       @pub.expects(:publish_with_failover).with("mama-exchange", "mama", @data, @opts).in_sequence(publishing)
       @pub.publish("mama", @data)
     end
@@ -89,6 +90,7 @@ module Beetle
       @pub.send(:mark_server_dead)
       publishing = sequence('publishing')
       @pub.expects(:recycle_dead_servers).in_sequence(publishing)
+      @pub.expects(:throttle!).in_sequence(publishing)
       @pub.expects(:publish_with_redundancy).with("mama-exchange", "mama", @data, @opts.merge(:redundant => true)).in_sequence(publishing)
       @pub.publish("mama", @data, :redundant => true)
     end
@@ -339,6 +341,57 @@ module Beetle
       @pub.setup_queues_and_policies(["queue"])
     end
 
+    test "reports whether it has been throttled" do
+      assert !@pub.throttled?
+      @pub.instance_variable_set :@throttled, true
+      assert @pub.throttled?
+    end
+
+    test "sets throttling options" do
+      h = { "x" => 1, "y" => 2}
+      @pub.throttle(h)
+      assert_equal h, @pub.instance_variable_get(:@throttling_options)
+    end
+
+    test "throttle! sleeps appropriately when refreshing throttling" do
+      @pub.expects(:throttling?).returns(true).twice
+      @pub.expects(:refresh_throttling!).twice
+      @pub.expects(:throttled?).returns(true)
+      @pub.expects(:sleep).with(1)
+      @pub.throttle!
+      @pub.expects(:throttled?).returns(false)
+      @pub.expects(:sleep).never
+      @pub.throttle!
+    end
+
+    test "returns a throttling status" do
+      assert_equal 'unthrottled', @pub.throttling_status
+      @pub.instance_variable_set :@throttled, true
+      assert_equal 'throttled', @pub.throttling_status
+    end
+
+    test "refresh_throttling! does not recompute values when refresh interval has not passed" do
+      @pub.instance_variable_set :@next_throttle_refresh, Time.now + 1
+      options = {}
+      @pub.throttle(options)
+      options.expects(:each).never
+      @pub.__send__ :refresh_throttling!
+    end
+
+    test "refresh_throttling! throttles when queue length exceeds limit" do
+      assert !@pub.throttled?
+      @pub.instance_variable_set :@next_throttle_refresh, Time.now - 1
+      options = { "test" => 100 }
+      @pub.throttle(options)
+      @pub.expects(:each_server).yields
+      q = mock("queue")
+      q.expects(:status).returns(:message_count => 500)
+      @pub.expects(:queue).returns(q)
+      @pub.logger.expects(:info)
+      @pub.__send__ :refresh_throttling!
+      assert @pub.throttled?
+    end
+
   end
 
   class PublisherExchangeManagementTest < Minitest::Test

data/test/beetle/queue_properties_test.rb

@@ -0,0 +1,210 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+
+module Beetle
+  class SetDeadLetterPolicyTest < Minitest::Test
+    def setup
+      @server = "localhost:15672"
+      @queue_name = "QUEUE_NAME"
+      @config = Configuration.new
+      @config.logger = Logger.new("/dev/null")
+      @queue_properties = QueueProperties.new(@config)
+    end
+
+    test "raises exception when queue name wasn't specified" do
+      assert_raises ArgumentError do
+        @queue_properties.set_queue_policy!(@server, "")
+      end
+    end
+
+    test "raises exception when no server was specified" do
+      assert_raises ArgumentError do
+        @queue_properties.set_queue_policy!("", @queue_name)
+      end
+    end
+
+    test "update_queue_properties! calls set_queue_policy for both target queue and dead letter queue" do
+      options = {
+        :server => "server", :lazy => true, :dead_lettering => true,
+        :queue_name => "QUEUE_NAME", :dead_letter_queue_name => "QUEUE_NAME_dead_letter",
+        :message_ttl => 10000
+      }
+      @queue_properties.expects(:set_queue_policy!).with("server", "QUEUE_NAME",
+                                                       :lazy => true, :dead_lettering => true,
+                                                       :routing_key => "QUEUE_NAME_dead_letter")
+      @queue_properties.expects(:set_queue_policy!).with("server", "QUEUE_NAME_dead_letter",
+                                                       :lazy => true, :dead_lettering => true,
+                                                       :routing_key => "QUEUE_NAME",
+                                                       :message_ttl => 10000)
+      @queue_properties.update_queue_properties!(options)
+    end
+
+    test "creates a policy by posting to the rabbitmq if dead lettering is enabled" do
+      stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .with(:body => {
+               "pattern" => "^QUEUE_NAME$",
+               "priority" => 1,
+               "apply-to" => "queues",
+               "definition" => {
+                 "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
+                 "dead-letter-exchange" => ""
+               }}.to_json)
+        .to_return(:status => 204)
+
+      @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => false, :dead_lettering => true, :routing_key => "QUEUE_NAME_dead_letter")
+    end
+
+    test "creates a policy by posting to the rabbitmq if lazy queues are enabled" do
+      stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .with(:body => {
+               "pattern" => "^QUEUE_NAME$",
+               "priority" => 1,
+               "apply-to" => "queues",
+               "definition" => {
+                 "queue-mode" => "lazy"
+               }}.to_json)
+        .to_return(:status => 204)
+
+      @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => false)
+    end
+
+    test "raises exception when policy couldn't successfully be created" do
+      stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => [405])
+
+      assert_raises QueueProperties::FailedRabbitRequest do
+        @queue_properties.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => true)
+      end
+    end
+
+    test "can optionally specify a message ttl" do
+      stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .with(:body => {
+                "pattern" => "^QUEUE_NAME$",
+                "priority" => 1,
+                "apply-to" => "queues",
+                "definition" => {
+                  "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
+                  "dead-letter-exchange" => "",
+                  "message-ttl" => 10000
+                }}.to_json)
+        .to_return(:status => 204)
+
+      @queue_properties.set_queue_policy!(@server, @queue_name, :dead_lettering => true, :message_ttl => 10000, :routing_key => "QUEUE_NAME_dead_letter")
+    end
+
+    test "properly encodes the vhost from the configuration" do
+      stub_request(:put, "http://localhost:15672/api/policies/foo%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .with(:body => {
+               "pattern" => "^QUEUE_NAME$",
+               "priority" => 1,
+               "apply-to" => "queues",
+               "definition" => {
+                 "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
+                 "dead-letter-exchange" => ""
+               }}.to_json)
+        .to_return(:status => 204)
+
+      @config.vhost = "foo/"
+
+      @queue_properties.set_queue_policy!(@server, @queue_name, :dead_lettering => true, :routing_key => "QUEUE_NAME_dead_letter")
+    end
+
+    test "update_queue_properties! calls remove_obsolete_bindings if bindings are part of the options hash" do
+      bindings = [{:exchange => "foo", :key => "a.b.c"}]
+      options = {
+        :server => "server", :lazy => true, :dead_lettering => true,
+        :queue_name => "QUEUE_NAME", :dead_letter_queue_name => "QUEUE_NAME_dead_letter",
+        :message_ttl => 10000,
+        :bindings => bindings
+      }
+      @queue_properties.expects(:set_queue_policy!).twice
+      @queue_properties.expects(:remove_obsolete_bindings).with("server", "QUEUE_NAME", bindings)
+      @queue_properties.update_queue_properties!(options)
+    end
+
+    test "remove_obsolete_bindings removes an obsolete binding but does not remove the default binding" do
+      bindings = [{:exchange => "foo", :key => "QUEUE_NAME"}, {:exchange => "foo", :key => "a.b.c"}]
+      stub_request(:get, "http://localhost:15672/api/queues/%2F/QUEUE_NAME/bindings")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 200,
+                   :body =>[
+                     {
+                       "destination_type" => "queue",
+                       "source" => "",
+                       "routing_key" => "QUEUE_NAME",
+                       "destination" => "QUEUE_NAME",
+                       "vhost" => "/",
+                       "properties_key" => "QUEUE_NAME",
+                       "arguments" => {}
+                     },
+                     {
+                       "destination_type" => "queue",
+                       "source" => "foo",
+                       "routing_key" => "QUEUE_NAME",
+                       "destination" => "QUEUE_NAME",
+                       "vhost" => "/",
+                       "properties_key" => "QUEUE_NAME",
+                       "arguments" => {}
+                     },
+                     {
+                       "destination_type" => "queue",
+                       "source" => "foo",
+                       "routing_key" => "a.b.c",
+                       "destination" => "QUEUE_NAME",
+                       "vhost" => "/",
+                       "properties_key" => "a.b.c",
+                       "arguments" => {}
+                     },
+                     {
+                       "destination_type" => "queue",
+                       "source" => "foofoo",
+                       "routing_key" => "x.y.z",
+                       "destination" => "QUEUE_NAME",
+                       "vhost" => "/",
+                       "properties_key" => "x.y.z",
+                       "arguments" => {}
+                     }
+                   ].to_json)
+
+      stub_request(:delete, "http://localhost:15672/api/bindings/%2F/e/foofoo/q/QUEUE_NAME/x.y.z")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 200)
+
+      @queue_properties.remove_obsolete_bindings(@server, "QUEUE_NAME", bindings)
+    end
+
+    test "raises an error when bindings cannot be retrieved" do
+      stub_request(:get, "http://localhost:15672/api/queues/%2F/QUEUE_NAME/bindings")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 500)
+      assert_raises(Beetle::QueueProperties::FailedRabbitRequest) { @queue_properties.remove_obsolete_bindings(@server, "QUEUE_NAME", []) }
+    end
+
+    test "raises an error when bindings cannot be deleted" do
+      stub_request(:get, "http://localhost:15672/api/queues/%2F/QUEUE_NAME/bindings")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 200, :body => [
+                     {
+                       "destination_type" => "queue",
+                       "source" => "foofoo",
+                       "routing_key" => "x.y.z",
+                       "destination" => "QUEUE_NAME",
+                       "vhost" => "/",
+                       "properties_key" => "x.y.z",
+                       "arguments" => {}
+                     }
+                   ].to_json)
+      stub_request(:delete, "http://localhost:15672/api/bindings/%2F/e/foofoo/q/QUEUE_NAME/x.y.z")
+        .with(basic_auth: ['guest', 'guest'])
+        .to_return(:status => 500)
+
+      assert_raises(Beetle::QueueProperties::FailedRabbitRequest) { @queue_properties.remove_obsolete_bindings(@server, "QUEUE_NAME", []) }
+    end
+
+  end
+end

data/test/beetle/subscriber_test.rb

@@ -246,7 +246,7 @@ module Beetle
       @client.config.dead_lettering_enabled = false
     end
 
-    test "should call reject on the message header when processing the handler returns true on reject? if dead_lettering has been enabled" do
+    test "should call reject on the message header when processing the handler returns true on reject? if dead lettering has been enabled" do
       header = header_with_params({})
       result = mock("result")
       result.expects(:reject?).returns(true)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 3.0.0.rc5
+  version: 3.0.0
 platform: ruby
 authors:
 - Stefan Kaes
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-21 00:00:00.000000000 Z
+date: 2019-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -327,6 +327,7 @@ files:
 - examples/redundant.rb
 - examples/rpc.rb
 - examples/simple.rb
+- examples/throttling.rb
 - features/redis_auto_failover.feature
 - features/step_definitions/redis_auto_failover_steps.rb
 - features/support/beetle_handler
@@ -340,12 +341,12 @@ files:
 - lib/beetle/base.rb
 - lib/beetle/client.rb
 - lib/beetle/configuration.rb
-- lib/beetle/dead_lettering.rb
 - lib/beetle/deduplication_store.rb
 - lib/beetle/handler.rb
 - lib/beetle/logging.rb
 - lib/beetle/message.rb
 - lib/beetle/publisher.rb
+- lib/beetle/queue_properties.rb
 - lib/beetle/r_c.rb
 - lib/beetle/redis_ext.rb
 - lib/beetle/subscriber.rb
@@ -357,12 +358,12 @@ files:
 - test/beetle/beetle_test.rb
 - test/beetle/client_test.rb
 - test/beetle/configuration_test.rb
-- test/beetle/dead_lettering_test.rb
 - test/beetle/deduplication_store_test.rb
 - test/beetle/handler_test.rb
 - test/beetle/message/settings_test.rb
 - test/beetle/message_test.rb
 - test/beetle/publisher_test.rb
+- test/beetle/queue_properties_test.rb
 - test/beetle/r_c_test.rb
 - test/beetle/redis_ext_test.rb
 - test/beetle/subscriber_test.rb
@@ -397,13 +398,13 @@ test_files:
 - test/beetle/client_test.rb
 - test/beetle/amqp_gem_behavior_test.rb
 - test/beetle/deduplication_store_test.rb
+- test/beetle/queue_properties_test.rb
 - test/beetle/handler_test.rb
 - test/beetle/beetle_test.rb
 - test/beetle/configuration_test.rb
 - test/beetle/subscriber_test.rb
 - test/beetle/message/settings_test.rb
 - test/beetle/redis_ext_test.rb
-- test/beetle/dead_lettering_test.rb
 - test/beetle/message_test.rb
 - test/beetle/publisher_test.rb
 - test/beetle/r_c_test.rb

data/test/beetle/dead_lettering_test.rb

@@ -1,117 +0,0 @@
-require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
-
-module Beetle
-  class SetDeadLetterPolicyTest < Minitest::Test
-    def setup
-      @server = "localhost:15672"
-      @queue_name = "QUEUE_NAME"
-      @config = Configuration.new
-      @config.logger = Logger.new("/dev/null")
-      @dead_lettering = DeadLettering.new(@config)
-    end
-
-    test "raises exception when queue name wasn't specified" do
-      assert_raises ArgumentError do
-        @dead_lettering.set_queue_policy!(@server, "")
-      end
-    end
-
-    test "raises exception when no server was specified" do
-      assert_raises ArgumentError do
-        @dead_lettering.set_queue_policy!("", @queue_name)
-      end
-    end
-
-    test "set_queue_policies! calls set_queue_policy for both target queue and dead letter queue" do
-      options = {
-        :server => "server", :lazy => true, :dead_lettering => true,
-        :queue_name => "QUEUE_NAME", :dead_letter_queue_name => "QUEUE_NAME_dead_letter",
-        :message_ttl => 10000
-      }
-      @dead_lettering.expects(:set_queue_policy!).with("server", "QUEUE_NAME",
-                                                       :lazy => true, :dead_lettering => true,
-                                                       :routing_key => "QUEUE_NAME_dead_letter")
-      @dead_lettering.expects(:set_queue_policy!).with("server", "QUEUE_NAME_dead_letter",
-                                                       :lazy => true, :dead_lettering => true,
-                                                       :routing_key => "QUEUE_NAME",
-                                                       :message_ttl => 10000)
-      @dead_lettering.set_queue_policies!(options)
-    end
-
-    test "creates a policy by posting to the rabbitmq if dead lettering is enabled" do
-      stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
-        .with(basic_auth: ['guest', 'guest'])
-        .with(:body => {
-               "pattern" => "^QUEUE_NAME$",
-               "priority" => 1,
-               "apply-to" => "queues",
-               "definition" => {
-                 "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
-                 "dead-letter-exchange" => ""
-               }}.to_json)
-        .to_return(:status => 204)
-
-      @dead_lettering.set_queue_policy!(@server, @queue_name, :lazy => false, :dead_lettering => true, :routing_key => "QUEUE_NAME_dead_letter")
-    end
-
-    test "creates a policy by posting to the rabbitmq if lazy queues are enabled" do
-      stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
-        .with(basic_auth: ['guest', 'guest'])
-        .with(:body => {
-               "pattern" => "^QUEUE_NAME$",
-               "priority" => 1,
-               "apply-to" => "queues",
-               "definition" => {
-                 "queue-mode" => "lazy"
-               }}.to_json)
-        .to_return(:status => 204)
-
-      @dead_lettering.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => false)
-    end
-
-    test "raises exception when policy couldn't successfully be created" do
-      stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
-        .with(basic_auth: ['guest', 'guest'])
-        .to_return(:status => [405])
-
-      assert_raises DeadLettering::FailedRabbitRequest do
-        @dead_lettering.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => true)
-      end
-    end
-
-    test "can optionally specify a message ttl" do
-      stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
-        .with(basic_auth: ['guest', 'guest'])
-        .with(:body => {
-                "pattern" => "^QUEUE_NAME$",
-                "priority" => 1,
-                "apply-to" => "queues",
-                "definition" => {
-                  "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
-                  "dead-letter-exchange" => "",
-                  "message-ttl" => 10000
-                }}.to_json)
-        .to_return(:status => 204)
-
-      @dead_lettering.set_queue_policy!(@server, @queue_name, :dead_lettering => true, :message_ttl => 10000, :routing_key => "QUEUE_NAME_dead_letter")
-    end
-
-    test "properly encodes the vhost from the configuration" do
-      stub_request(:put, "http://localhost:15672/api/policies/foo%2F/QUEUE_NAME_policy")
-        .with(basic_auth: ['guest', 'guest'])
-        .with(:body => {
-               "pattern" => "^QUEUE_NAME$",
-               "priority" => 1,
-               "apply-to" => "queues",
-               "definition" => {
-                 "dead-letter-routing-key" => "QUEUE_NAME_dead_letter",
-                 "dead-letter-exchange" => ""
-               }}.to_json)
-        .to_return(:status => 204)
-
-      @config.vhost = "foo/"
-
-      @dead_lettering.set_queue_policy!(@server, @queue_name, :dead_lettering => true, :routing_key => "QUEUE_NAME_dead_letter")
-    end
-  end
-end