beetle 2.3.2 → 3.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 807f79dd4cc32720d7d4c4b1b642419296175f9382379dedb1291e03a89b3989
-  data.tar.gz: ac8fe19b561ff2717e2be0be1a0c391e76b115608dffb856fa4d619d6466c45a
+  metadata.gz: 299657b394d85d53b70417554db9a3ca53de5339dc0fc777e1555536237ccf1b
+  data.tar.gz: a4fda19185d645fbb26e563b7b7c72ffe67d03ef3cc621a581b7b1cee38b3178
 SHA512:
-  metadata.gz: 9b68750fe9385741dc7ae98cbbe65e8e5f48c1e628c0be596c6db2210e98976a8f2158875a80716b4987150cac08da2b937480e4bd7925be743192993a95a822
-  data.tar.gz: 2c0ee5d09050b3ea4ad319bcae89f4e5684d7318ba3f871ad33691ae6f86ec2d2a84b4dcb55b1daed3589702c7fb40e8b63559b807b3f5408d2c6a714721a141
+  metadata.gz: 3b6c98dd094e445db0424aab1c992eb9245df6fb4e402c45d087c8ca395c9bfd2f3ec57d95999a79ce5f4853d1c02e139a829b962f6cef896c73004a299743ad
+  data.tar.gz: 1324bd1753c97574cf0dd657f7096be41e26d76f271525c124aa198286056cbd8673bf5ff0894bc761b94351cdd14d1dbd3504e3925fcc6e11804f6ae99b19dd

data/README.rdoc

@@ -96,6 +96,7 @@ For development, you'll need
 * {mocha}[http://github.com/floehopper/mocha]
 * {cucumber}[http://github.com/aslakhellesoy/cucumber]
 * {daemon_controller}[http://github.com/FooBarWidget/daemon_controller]
+* {consul}[https://www.consul.io/downloads.html]
 
 For tests, you'll need
 * {activerecord}[https://github.com/rails/rails/tree/master/activerecord]
@@ -120,9 +121,11 @@ distribution.
 
 1. Fork it
 2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+3. Hack along and test your code using rake test and rake
+   cucumber. You must start consul in order to use cucumber.
+4. Commit your changes (`git commit -am 'Add some feature'`)
+5. Push to the branch (`git push origin my-new-feature`)
+6. Create new Pull Request
 
 Don't increase the gem version in your pull requests. It will be done after merging the request,
 to allow merging of pull requests in a flexible order.

data/RELEASE_NOTES.rdoc

@@ -1,5 +1,13 @@
 = Release Notes
 
+
+== Version 3.0.0.rc1
+* provide client method to setup queues and queue policies.
+  Setting up queue policies on demand in publisher and subscriber is OK
+  for a small number of queues, publishers and subscribers. But the HTTP
+  API of RabbitMQ doesn't scale all that well, so that a large number
+  of HTPP calls to set up queue policies can in fact crash a server.
+
 == Version 2.3.2
 * config server: fixed a race condition when accessing server state.
   HTTP requests run in threads separate from the server dispatcher

data/Rakefile

@@ -82,6 +82,13 @@ namespace :redis do
   end
 end
 
+namespace :consul do
+    desc "start consul agent in development mode"
+    task :start do
+      system "consul agent -dev -node machine"
+    end
+end
+
 Cucumber::Rake::Task.new(:cucumber) do |t|
   t.cucumber_opts = "features --format progress"
 end

data/features/redis_auto_failover.feature

@@ -5,6 +5,7 @@ Feature: Redis auto failover
   Background:
     Given a redis server "redis-1" exists as master
     And a redis server "redis-2" exists as slave of "redis-1"
+    And consul state has been cleared
 
   Scenario: Successful redis master switch
     Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists

data/features/step_definitions/redis_auto_failover_steps.rb

@@ -1,3 +1,10 @@
+Given /^consul state has been cleared$/ do
+  system "killall beetle beetle_handler >/dev/null 2>/dev/null"
+  system "curl --silent --request PUT http://localhost:8500/v1/kv/apps/beetle/config/ >/dev/null"
+  system "curl --silent --request PUT http://localhost:8500/v1/kv/shared/config/ >/dev/null"
+  system "curl --silent --request DELETE http://localhost:8500/v1/kv/apps/beetle/state/redis_master_file_content >/dev/null"
+end
+
 Given /^a redis server "([^\"]*)" exists as master$/ do |redis_name|
   TestDaemons::Redis[redis_name].restart
   TestDaemons::Redis[redis_name].master

data/features/support/test_daemons/redis_configuration_server.rb

@@ -27,7 +27,7 @@ module TestDaemons
       clients_parameter_string = @@redis_configuration_clients.blank? ? "" : "--client-ids #{@@redis_configuration_clients}"
       DaemonController.new(
          :identifier    => "Redis configuration test server",
-         :start_command => "./beetle configuration_server -v -d --redis-master-file #{redis_master_file} --redis-servers '#{@@redis_servers}' #{clients_parameter_string} --redis-master-retry-interval 1 --pid-file #{pid_file} --log-file #{log_file} --redis-failover-confidence-level #{@@confidence_level}",
+         :start_command => "./beetle configuration_server -v -d --redis-master-file #{redis_master_file} --redis-servers '#{@@redis_servers}' #{clients_parameter_string} --redis-master-retry-interval 1 --pid-file #{pid_file} --log-file #{log_file} --redis-failover-confidence-level #{@@confidence_level} --consul http://localhost:8500",
          :ping_command  => lambda{ answers_text_requests? },
          :pid_file      => pid_file,
          :log_file      => log_file,

data/lib/beetle/base.rb

@@ -12,7 +12,7 @@ module Beetle
       @server = @servers[rand @servers.size]
       @exchanges = {}
       @queues = {}
-      @dead_lettering = DeadLettering.new(@client.config)
+      @dead_lettering = DeadLettering.new(@client)
     end
 
     private
@@ -54,7 +54,13 @@ module Beetle
       @queues[@server] ||= {}
     end
 
-    def queue(name)
+    QueueInfo = Struct.new(:queue, :create_policies)
+
+    def queue(name, create_policies: false)
+      info = queues[name]
+      if info && create_policies && !info.create_policies
+        queues.delete(name)
+      end
       queues[name] ||=
         begin
           opts = @client.queues[name]
@@ -66,10 +72,11 @@ module Beetle
           @client.bindings[name].each do |binding_options|
             exchange_name = binding_options[:exchange]
             binding_options = binding_options.slice(*QUEUE_BINDING_KEYS)
-            the_queue = bind_queue!(queue_name, creation_options, exchange_name, binding_options)
+            the_queue = bind_queue!(queue_name, creation_options, exchange_name, binding_options, create_policies: create_policies)
           end
-          the_queue
+          info = QueueInfo.new(the_queue, create_policies)
         end
+      info.queue
     end
 
   end

data/lib/beetle/client.rb

@@ -73,12 +73,19 @@ module Beetle
     #   the name of the exchange this queue will be bound to (defaults to the name of the queue)
     # [<tt>:key</tt>]
     #   the binding key (defaults to the name of the queue)
+    # [<tt>:lazy</tt>]
+    #   whether the queue should use lazy mode (defaults to <tt>config.lazy_queues_enabled</tt>)
+    # [<tt>:dead_lettering</tt>]
+    #   whether the queue should use dead lettering (defaults to <tt>config.dead_lettering_enabled</tt>)
     # automatically registers the specified exchange if it hasn't been registered yet
 
     def register_queue(name, options={})
       name = name.to_s
       raise ConfigurationError.new("queue #{name} already configured") if queues.include?(name)
-      opts = {:exchange => name, :key => name, :auto_delete => false, :amqp_name => name}.merge!(options.symbolize_keys)
+      opts = {
+        :exchange => name, :key => name, :auto_delete => false, :amqp_name => name,
+        :lazy => config.lazy_queues_enabled, :dead_lettering => config.dead_lettering_enabled
+      }.merge!(options.symbolize_keys)
       opts.merge! :durable => true, :passive => false, :exclusive => false
       exchange = opts.delete(:exchange).to_s
       key = opts.delete(:key)
@@ -202,6 +209,11 @@ module Beetle
       publisher.purge(queues)
     end
 
+    # declares all queues, binds them and creates/updates all policies
+    def setup_queues_and_policies(queues)
+      publisher.setup_queues_and_policies(queues)
+    end
+
     # start listening to all registered queues. Calls #listen_queues internally
     # runs the given block before entering the eventmachine loop.
     def listen(_deprecated_messages=nil, &block)

data/lib/beetle/dead_lettering.rb

@@ -5,29 +5,34 @@ module Beetle
   class DeadLettering
     class FailedRabbitRequest < StandardError; end
 
-    def initialize(config)
-      @config = config
+    attr_reader :client, :config
+
+    def initialize(client)
+      @client = client
+      @config = client.config
     end
 
     def bind_dead_letter_queues!(channel, servers, target_queue, creation_keys = {})
-      if @config.dead_lettering_enabled?
+      policy_options = client.queues[target_queue].slice(:dead_lettering, :lazy)
+      if policy_options[:dead_lettering]
         dead_letter_queue_name = dead_letter_queue_name(target_queue)
-        logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_keys.inspect}")
+        logger.info("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_keys.inspect}")
         channel.queue(dead_letter_queue_name, creation_keys)
       end
 
-      if @config.dead_lettering_enabled?
-        logger.debug("Beetle: setting #{dead_letter_queue_name} as dead letter queue of #{target_queue} on all servers")
+      if policy_options[:dead_lettering]
+        logger.info("Beetle: setting #{dead_letter_queue_name} as dead letter queue of #{target_queue} on all servers")
       end
-      set_queue_policies!(servers, target_queue)
+      set_queue_policies!(servers, target_queue, policy_options)
 
-      if @config.dead_lettering_enabled?
-        logger.debug("Beetle: setting #{target_queue} as dead letter queue of #{dead_letter_queue_name} on all servers")
+      if policy_options[:dead_lettering]
+        logger.info("Beetle: setting #{target_queue} as dead letter queue of #{dead_letter_queue_name} on all servers")
         set_queue_policies!(
           servers,
           dead_letter_queue_name,
-          :message_ttl => @config.dead_lettering_msg_ttl,
-          :routing_key => target_queue
+          { :message_ttl => config.dead_lettering_msg_ttl,
+            :routing_key => target_queue
+          }.merge(policy_options)
         )
       end
     end
@@ -40,21 +45,21 @@ module Beetle
       raise ArgumentError.new("server missing")     if server.blank?
       raise ArgumentError.new("queue name missing") if queue_name.blank?
 
-      return unless @config.dead_lettering_enabled? || @config.lazy_queues_enabled?
+      return unless options[:dead_lettering] || options[:lazy]
 
-      vhost = CGI.escape(@config.vhost)
+      vhost = CGI.escape(config.vhost)
       request_url = URI("http://#{server}/api/policies/#{vhost}/#{queue_name}_policy")
       request = Net::HTTP::Put.new(request_url)
 
       # set up queue policy
       definition = {}
-      if @config.dead_lettering_enabled?
+      if options[:dead_lettering]
         definition["dead-letter-routing-key"] = dead_letter_routing_key(queue_name, options)
         definition["dead-letter-exchange"] = ""
         definition["message-ttl"] = options[:message_ttl] if options[:message_ttl]
       end
 
-      definition["queue-mode"] = "lazy" if @config.lazy_queues_enabled?
+      definition["queue-mode"] = "lazy" if options[:lazy]
 
       request_body = {
         "pattern" => "^#{queue_name}$",
@@ -84,9 +89,9 @@ module Beetle
     end
 
     def run_rabbit_http_request(uri, request, &block)
-      request.basic_auth(@config.user, @config.password)
+      request.basic_auth(config.user, config.password)
       request["Content-Type"] = "application/json"
-      Net::HTTP.start(uri.hostname, @config.api_port, :read_timeout => @config.dead_lettering_read_timeout) do |http|
+      Net::HTTP.start(uri.hostname, config.api_port, :read_timeout => config.dead_lettering_read_timeout) do |http|
         block.call(http) if block_given?
       end
     end
@@ -98,7 +103,7 @@ module Beetle
     end
 
     def logger
-      @config.logger
+      config.logger
     end
   end
 end

data/lib/beetle/deduplication_store.rb

@@ -154,7 +154,7 @@ module Beetle
       system_name = @config.system_name
       redis_master = ""
       text.each_line do |line|
-        parts = line.split('/')
+        parts = line.split('/', 2)
         case parts.size
         when 1
           redis_master = parts[0]

data/lib/beetle/publisher.rb

@@ -143,6 +143,14 @@ module Beetle
       end
     end
 
+    def setup_queues_and_policies(queue_names) #:nodoc:
+      each_server do
+        queue_names.each do |name|
+          queue(name, create_policies: true)
+        end
+      end
+    end
+
     def stop #:nodoc:
       each_server { stop! }
     end
@@ -214,10 +222,12 @@ module Beetle
     end
 
     # TODO: Refactor, fetch the keys and stuff itself
-    def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
+    def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys, create_policies: false)
       logger.debug("Beetle: creating queue with opts: #{creation_keys.inspect}")
       queue = bunny.queue(queue_name, creation_keys)
-      @dead_lettering.bind_dead_letter_queues!(bunny, @client.servers, queue_name, creation_keys)
+      if create_policies
+        @dead_lettering.bind_dead_letter_queues!(bunny, @client.servers, queue_name, creation_keys)
+      end
       logger.debug("Beetle: binding queue #{queue_name} to #{exchange_name} with opts: #{binding_keys.inspect}")
       queue.bind(exchange(exchange_name), binding_keys)
       queue

data/lib/beetle/subscriber.rb

@@ -144,19 +144,19 @@ module Beetle
       callback = create_subscription_callback(queue_name, amqp_queue_name, handler, opts)
       keys = opts.slice(*SUBSCRIPTION_KEYS).merge(:key => "#", :ack => true)
       logger.debug "Beetle: subscribing to queue #{amqp_queue_name} with key # on server #{@server}"
-      queues[queue_name].subscribe(keys, &callback)
+      queues[queue_name].queue.subscribe(keys, &callback)
       subscriptions[queue_name] = [keys, callback]
     end
 
     def pause(queue_name)
-      return unless queues[queue_name].subscribed?
-      queues[queue_name].unsubscribe
+      return unless queues[queue_name].queue.subscribed?
+      queues[queue_name].queue.unsubscribe
     end
 
     def resume(queue_name)
-      return if queues[queue_name].subscribed?
+      return if queues[queue_name].queue.subscribed?
       keys, callback = subscriptions[queue_name]
-      queues[queue_name].subscribe(keys, &callback)
+      queues[queue_name].queue.subscribe(keys, &callback)
     end
 
     def create_subscription_callback(queue_name, amqp_queue_name, handler, opts)
@@ -204,10 +204,12 @@ module Beetle
       channel.__send__(opts[:type], name, opts.slice(*EXCHANGE_CREATION_KEYS))
     end
 
-    def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
+    def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys, create_policies: false)
       queue = channel.queue(queue_name, creation_keys)
       target_servers = @client.servers + @client.additional_subscription_servers
-      @dead_lettering.bind_dead_letter_queues!(channel, target_servers, queue_name, creation_keys)
+      if create_policies
+        @dead_lettering.bind_dead_letter_queues!(channel, target_servers, queue_name, creation_keys)
+      end
       exchange = exchange(exchange_name)
       queue.bind(exchange, binding_keys)
       queue

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "2.3.2"
+  VERSION = "3.0.0.rc1"
 end

data/test/beetle/client_test.rb

@@ -79,7 +79,7 @@ module Beetle
 
     test "registering a queue should store it in the configuration with symbolized option keys and force durable=true and passive=false and set the amqp queue name" do
       @client.register_queue("some_queue", "durable" => false, "exchange" => "some_exchange")
-      assert_equal({:durable => true, :passive => false, :auto_delete => false, :exclusive => false, :amqp_name => "some_queue"}, @client.queues["some_queue"])
+      assert_equal({:durable => true, :passive => false, :lazy=>false, :dead_lettering=>false, :auto_delete => false, :exclusive => false, :amqp_name => "some_queue"}, @client.queues["some_queue"])
     end
 
     test "registering a queue should add the queue to the list of queues of the queue's exchange" do
@@ -308,6 +308,13 @@ module Beetle
       assert_equal "ha!", client.purge(:queue1, :queue2)
     end
 
+    test "should delegate queue setup to the publisher instance" do
+      client = Client.new
+      client.register_queue(:queue)
+      client.send(:publisher).expects(:setup_queues_and_policies).with(["queue"]).returns("ha!")
+      assert_equal "ha!", client.setup_queues_and_policies(["queue"])
+    end
+
     test "should delegate rpc calls to the publisher instance" do
       client = Client.new
       client.register_message("deadletter")

data/test/beetle/dead_lettering_test.rb

@@ -4,7 +4,8 @@ module Beetle
   class SetDeadLetteringsTest < Minitest::Test
     def setup
       @config = Configuration.new
-      @dead_lettering = DeadLettering.new(@config)
+      @client = Client.new @config
+      @dead_lettering = DeadLettering.new(@client)
       @config.dead_lettering_enabled = true
     end
 
@@ -25,9 +26,9 @@ module Beetle
       @server = "localhost:15672"
       @queue_name = "QUEUE_NAME"
       @config = Configuration.new
+      @client = Client.new @config
       @config.logger = Logger.new("/dev/null")
-      @dead_lettering = DeadLettering.new(@config)
-      @config.dead_lettering_enabled = true
+      @dead_lettering = DeadLettering.new(@client)
     end
 
     test "raises exception when queue name wasn't specified" do
@@ -55,12 +56,10 @@ module Beetle
                }}.to_json)
         .to_return(:status => 204)
 
-      @dead_lettering.set_queue_policy!(@server, @queue_name)
+      @dead_lettering.set_queue_policy!(@server, @queue_name, :lazy => false, :dead_lettering => true)
     end
 
     test "creates a policy by posting to the rabbitmq if lazy queues are enabled" do
-      @config.lazy_queues_enabled = true
-      @config.dead_lettering_enabled = false
       stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
         .with(basic_auth: ['guest', 'guest'])
         .with(:body => {
@@ -72,7 +71,7 @@ module Beetle
                }}.to_json)
         .to_return(:status => 204)
 
-      @dead_lettering.set_queue_policy!(@server, @queue_name)
+      @dead_lettering.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => false)
     end
 
     test "raises exception when policy couldn't successfully be created" do
@@ -81,7 +80,7 @@ module Beetle
         .to_return(:status => [405])
 
       assert_raises DeadLettering::FailedRabbitRequest do
-        @dead_lettering.set_queue_policy!(@server, @queue_name)
+        @dead_lettering.set_queue_policy!(@server, @queue_name, :lazy => true, :dead_lettering => true)
       end
     end
 
@@ -99,7 +98,7 @@ module Beetle
                 }}.to_json)
         .to_return(:status => 204)
 
-      @dead_lettering.set_queue_policy!(@server, @queue_name, :message_ttl => 10000)
+      @dead_lettering.set_queue_policy!(@server, @queue_name, :dead_lettering => true, :message_ttl => 10000)
     end
 
     test "properly encodes the vhost from the configuration" do
@@ -117,7 +116,7 @@ module Beetle
 
       @config.vhost = "foo/"
 
-      @dead_lettering.set_queue_policy!(@server, @queue_name)
+      @dead_lettering.set_queue_policy!(@server, @queue_name, :dead_lettering => true)
     end
   end
 
@@ -125,28 +124,31 @@ module Beetle
     def setup
       @queue_name = "QUEUE_NAME"
       @config = Configuration.new
+      @client = Client.new @config
       @config.logger = Logger.new("/dev/null")
-      @dead_lettering = DeadLettering.new(@config)
+      @dead_lettering = DeadLettering.new(@client)
       @servers = ["localhost:55672"]
     end
 
     test "is does not call out to rabbit if neither dead lettering nor lazy queues are enabled" do
-      @config.dead_lettering_enabled = false
+      @client.register_queue(@queue_name, :dead_lettering => false, :lazy => false)
       channel = stub('channel')
       @dead_lettering.expects(:run_rabbit_http_request).never
       @dead_lettering.bind_dead_letter_queues!(channel, @servers, @queue_name)
     end
 
     test "creates and connects the dead letter queue via policies when enabled" do
-      @config.dead_lettering_enabled = true
+      @client.register_queue(@queue_name, :dead_lettering => true, :lazy => false)
 
       channel = stub('channel')
 
       channel.expects(:queue).with("#{@queue_name}_dead_letter", {})
-      @dead_lettering.expects(:set_queue_policies!).with(@servers, @queue_name)
+      @dead_lettering.expects(:set_queue_policies!).with(@servers, @queue_name,  :dead_lettering => true, :lazy => false)
       @dead_lettering.expects(:set_queue_policies!).with(@servers, "#{@queue_name}_dead_letter",
         :routing_key => @queue_name,
-        :message_ttl => 1000
+        :message_ttl => 1000,
+        :dead_lettering => true,
+        :lazy => false
       )
 
       @dead_lettering.bind_dead_letter_queues!(channel, @servers, @queue_name)

data/test/beetle/publisher_test.rb

@@ -273,6 +273,19 @@ module Beetle
     end
 
     test "binding a queue should create it using the config and bind it to the exchange with the name specified" do
+      @client.register_queue("some_queue", :exchange => "some_exchange", :key => "haha.#", :arguments => {"foo" => "fighter"})
+      @pub.expects(:exchange).with("some_exchange").returns(:the_exchange)
+      q = mock("queue")
+      q.expects(:bind).with(:the_exchange, {:key => "haha.#"})
+      m = mock("Bunny")
+      m.expects(:queue).with("some_queue", :durable => true, :passive => false, :auto_delete => false, :exclusive => false, :arguments => {"foo" => "fighter"}).returns(q)
+      @pub.expects(:bunny).returns(m)
+
+      @pub.send(:queue, "some_queue")
+      assert_equal Base::QueueInfo.new(q, false), @pub.send(:queues)["some_queue"]
+    end
+
+    test "binding a queue should create it using the config and bind it to the exchange with the name specified and setup policies if requested" do
       @client.register_queue("some_queue", :exchange => "some_exchange", :key => "haha.#", :arguments => {"foo" => "fighter"})
       @pub.expects(:exchange).with("some_exchange").returns(:the_exchange)
       q = mock("queue")
@@ -281,8 +294,29 @@ module Beetle
       m.expects(:queue).with("some_queue", :durable => true, :passive => false, :auto_delete => false, :exclusive => false, :arguments => {"foo" => "fighter"}).returns(q)
       @pub.expects(:bunny).returns(m).twice
 
+      @pub.send(:queue, "some_queue", create_policies: true)
+      assert_equal Base::QueueInfo.new(q, true), @pub.send(:queues)["some_queue"]
+    end
+
+    test "binding a queue with create_plocies: true after having already declared it with create_plocies: false creates the policies" do
+      @client.register_queue("some_queue", :exchange => "some_exchange", :key => "haha.#", :arguments => {"foo" => "fighter"})
+      @pub.expects(:exchange).with("some_exchange").returns(:the_exchange)
+      q = mock("queue")
+      q.expects(:bind).with(:the_exchange, {:key => "haha.#"})
+      m = mock("Bunny")
+      m.expects(:queue).with("some_queue", :durable => true, :passive => false, :auto_delete => false, :exclusive => false, :arguments => {"foo" => "fighter"}).returns(q)
+      @pub.expects(:bunny).returns(m)
+
       @pub.send(:queue, "some_queue")
-      assert_equal q, @pub.send(:queues)["some_queue"]
+      assert_equal Base::QueueInfo.new(q, false), @pub.send(:queues)["some_queue"]
+
+      @pub.expects(:exchange).with("some_exchange").returns(:the_exchange)
+      q.expects(:bind).with(:the_exchange, {:key => "haha.#"})
+      m.expects(:queue).with("some_queue", :durable => true, :passive => false, :auto_delete => false, :exclusive => false, :arguments => {"foo" => "fighter"}).returns(q)
+      @pub.expects(:bunny).returns(m).twice
+
+      @pub.send(:queue, "some_queue", create_policies: true)
+      assert_equal Base::QueueInfo.new(q, true), @pub.send(:queues)["some_queue"]
     end
 
     test "should bind the defined queues for the used exchanges when publishing" do
@@ -325,6 +359,18 @@ module Beetle
       queue.expects(:purge).in_sequence(s)
       @pub.purge(["queue"])
     end
+
+    test "setting up queues and policies should iterate over all servers" do
+      @pub.servers = %w(a b)
+      queue = mock("queue")
+      s = sequence("setup")
+      @pub.expects(:set_current_server).with("a").in_sequence(s)
+      @pub.expects(:queue).with("queue", :create_policies => true).returns(queue).in_sequence(s)
+      @pub.expects(:set_current_server).with("b").in_sequence(s)
+      @pub.expects(:queue).with("queue", :create_policies => true).returns(queue).in_sequence(s)
+      @pub.setup_queues_and_policies(["queue"])
+    end
+
   end
 
   class PublisherExchangeManagementTest < Minitest::Test

data/test/beetle/subscriber_test.rb

@@ -89,7 +89,7 @@ module Beetle
       q = mock("queue a")
       q.expects(:subscribed?).returns(true)
       q.expects(:unsubscribe)
-      @sub.stubs(:queues).returns({"a" =>q})
+      @sub.stubs(:queues).returns({"a" => Base::QueueInfo.new(q,false)})
       @sub.__send__(:pause, "a")
     end
 
@@ -97,7 +97,7 @@ module Beetle
       q = mock("queue a")
       q.expects(:subscribed?).returns(false)
       q.expects(:unsubscribe).never
-      @sub.stubs(:queues).returns({"a" =>q})
+      @sub.stubs(:queues).returns({"a" => Base::QueueInfo.new(q,false)})
       @sub.__send__(:pause, "a")
     end
 
@@ -105,7 +105,7 @@ module Beetle
       q = mock("queue a")
       q.expects(:subscribed?).returns(false)
       q.expects(:subscribe)
-      @sub.stubs(:queues).returns({"a" =>q})
+      @sub.stubs(:queues).returns({"a" => Base::QueueInfo.new(q,false)})
       @sub.__send__(:resume, "a")
     end
 
@@ -113,7 +113,7 @@ module Beetle
       q = mock("queue a")
       q.expects(:subscribed?).returns(true)
       q.expects(:subscribe).never
-      @sub.stubs(:queues).returns({"a" =>q})
+      @sub.stubs(:queues).returns({"a" => Base::QueueInfo.new(q,false)})
       @sub.__send__(:resume, "a")
     end
 
@@ -150,10 +150,23 @@ module Beetle
       q.expects(:bind).with(:the_exchange, {:key => "haha.#"})
       m = mock("MQ")
       m.expects(:queue).with("some_queue", :durable => true, :passive => false, :auto_delete => false, :exclusive => false, :arguments => {"schmu" => 5}).returns(q)
-      @sub.expects(:channel).returns(m).twice
+      @sub.expects(:channel).returns(m)
 
       @sub.send(:queue, "some_queue")
-      assert_equal q, @sub.send(:queues)["some_queue"]
+      assert_equal q, @sub.send(:queues)["some_queue"].queue
+    end
+
+    test "binding a queue should create it using the config and bind it to the exchange with the name specified and create policies if requested" do
+      @client.register_queue("some_queue", "durable" => true, "exchange" => "some_exchange", "key" => "haha.#", "arguments" => {"schmu" => 5})
+      @sub.expects(:exchange).with("some_exchange").returns(:the_exchange)
+      q = mock("queue")
+      q.expects(:bind).with(:the_exchange, {:key => "haha.#"})
+      m = mock("MQ")
+      m.expects(:queue).with("some_queue", :durable => true, :passive => false, :auto_delete => false, :exclusive => false, :arguments => {"schmu" => 5}).returns(q)
+      @sub.expects(:channel).returns(m).twice
+
+      @sub.send(:queue, "some_queue", create_policies: true)
+      assert_equal q, @sub.send(:queues)["some_queue"].queue
     end
 
     test "binding queues should bind all queues" do
@@ -357,7 +370,7 @@ module Beetle
       q = mock("QUEUE")
       subscription_options = {:ack => true, :key => "#"}
       q.expects(:subscribe).with(subscription_options).yields(header, "foo")
-      @sub.expects(:queues).returns({"some_queue" => q}).once
+      @sub.expects(:queues).returns({"some_queue" => Base::QueueInfo.new(q,false)}).once
       @sub.send(:subscribe, "some_queue")
       assert block_called
       assert @sub.__send__(:has_subscription?, "some_queue")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 2.3.2
+  version: 3.0.0.rc1
 platform: ruby
 authors:
 - Stefan Kaes
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-26 00:00:00.000000000 Z
+date: 2019-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -322,7 +322,6 @@ files:
 - examples/headers.rb
 - examples/multiple_exchanges.rb
 - examples/multiple_queues.rb
-- examples/murks.rb
 - examples/nonexistent_server.rb
 - examples/pause_and_resume.rb
 - examples/publish_many_messages.rb

data/examples/murks.rb

@@ -1,30 +0,0 @@
-require 'bunny'
-
-conn = Bunny.new
-conn.start
-
-# consumer
-ch = conn.create_channel
-x = ch.topic("echo")
-q1 = ch.queue("echo").bind(x, :routing_key => "#")
-
-consumer = q1.subscribe do |delivery_info, properties, payload|
-  puts delivery_info.inspect, properties.inspect, payload.inspect
-  ch.default_exchange.publish(payload.to_s, :routing_key => properties[:reply_to])
-end
-
-# publisher
-q2 = ch.queue()
-puts "reply queue: #{q2.name}"
-x.publish("Hello, everybody!", :routing_key => "echo", :reply_to => q2.name)
-
-sleep 1
-
-delivery_info, properties, payload = q2.pop
-
-puts "This is the message: " + payload.inspect
-puts delivery_info.inspect, properties.inspect
-
-consumer.cancel
-ch.close
-conn.close