beetle 2.2.3 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a35a259d101d2b0623ae59f12e901b0872308a372ecaaecd43d95de13e38bec
-  data.tar.gz: bdc124f3b915978f1bee1265e9e331f3f48a908a5c9c5ef9792c8779037ca33a
+  metadata.gz: c8a5f1b489f657fd5e286f6461b13ee314b6b4abe3dae6452bb53ca81ce23255
+  data.tar.gz: 183a0d96adf42fe4648f946203527edd1b065b7c961b62b825e6f5c6483edf7e
 SHA512:
-  metadata.gz: 6b0b8d21c4b678de78c06b887075eb175d1ddcd902c8f40dc683fd5d2a3c160422e7ee94842db7f4620e1ffccf7ffa7051e74afe7f81d9d8206f206022034fff
-  data.tar.gz: 295f9b96e41cbde54221107736cf3bfaec30dade8e728ceb0dc0f7ea4a91bf0e0a5fdbcf6c3868a6ae54dc5964901451c2ec08a756c3e59582cf455eaa6f938b
+  metadata.gz: 3eaa2e3b642b8c3dd5d5062a4327d5499e87c3c02d27493b33f9f42fa1bb2964e2f6b5af386c88525b2343e075ba61e42fe9a75ce459d2a98ef949e418ffe7e1
+  data.tar.gz: cf80066e492795627ef460eec5c05a222a5af569d87e62084ec865939e715a6f4c6ca902f618d9039f7d650c9208b694a30e28dd81bd23571d71b1a1ff7ea287

data/README.rdoc

@@ -111,7 +111,7 @@ For tests, you'll need
 
 You can find out more about our work on our {dev blog}[http://devblog.xing.com].
 
-Copyright (c) 2010-2015 {XING AG}[http://www.xing.com/]
+Copyright (c) 2010-2019 {XING AG}[http://www.xing.com/]
 
 Released under the MIT license. For full details see MIT-LICENSE included in this
 distribution.

data/RELEASE_NOTES.rdoc

@@ -1,7 +1,27 @@
 = Release Notes
 
+== Version 2.3.0
+* redis failover: support multiple redis failover instances. This has
+  required a change in the redis master file format. The master file
+  content is now either the old format (host:port) for systems using a
+  single redis failover system or a mapping from system names to
+  host:port combinations. For example,
+  "system1/master1:6379\nsystem2/master" specifies to systems with
+  their corresponding redis masters. Beetle client uses the configured
+  system name to tind the master it should use.
+* support lazy queues: setting :lazy_quques_enabled on the beetle
+  cofiguration will enable queue_mode: "lazy" for all queues of
+  declared on the beetle client.
+* improved calculation of channel close an connection disconnect
+  timeouts for beetle publisher to avoid warnings in RabbitMQ logs.
+* use SecureRandom.uuid instead of UUID4R::uuid(4) if UUID4R cannot
+  be loaded.
+
+== Version 2.2.4
+* redis failover: prevent starting a new master switch while one is running
+
 == Version 2.2.3
-* redis failover server logs errors when redis oparations fail
+* redis failover: server logs errors when redis oparations fail
 
 == Version 2.2.2
 

data/beetle.gemspec

@@ -18,16 +18,19 @@ Gem::Specification.new do |s|
   s.rdoc_options       = ["--charset=UTF-8"]
   s.require_paths      = ["lib"]
   s.test_files         = Dir['test/**/*.rb']
+  s.metadata = {
+    "changelog_uri"    => "https://github.com/xing/beetle/blob/master/RELEASE_NOTES.rdoc"
+  }
 
   s.specification_version = 3
-  s.add_runtime_dependency "uuid4r",                  ">= 0.1.2"
-  s.add_runtime_dependency "bunny",                   "~> 0.7.10"
+  s.add_runtime_dependency "bunny",                   "~> 0.7.12"
   s.add_runtime_dependency "redis",                   ">= 2.2.2"
   s.add_runtime_dependency "hiredis",                 ">= 0.4.5"
   s.add_runtime_dependency "amq-protocol",            "= 2.0.1"
   s.add_runtime_dependency "amqp",                    "= 1.6.0"
   s.add_runtime_dependency "activesupport",           ">= 2.3.4"
 
+  s.add_development_dependency "uuid4r",              ">= 0.1.2"
   s.add_development_dependency "activerecord",        "~> 5.0"
   s.add_development_dependency "cucumber",            "~> 2.4.0"
   s.add_development_dependency "daemon_controller",   "~> 1.2.0"

data/examples/attempts.rb

@@ -16,6 +16,7 @@ Beetle.config.logger.level = Logger::INFO
 # setup client
 $client = Beetle::Client.new
 $client.config.dead_lettering_enabled = true
+$client.config.lazy_queues_enabled = true
 $client.configure(:key => "my.test.message") do
   message(:test)
   queue(:test)

data/features/redis_auto_failover.feature

@@ -23,6 +23,52 @@ Feature: Redis auto failover
     Given a redis server "redis-1" exists as master
     Then the role of redis server "redis-1" should be "slave"
 
+  Scenario: Successful single redis master switch with multiple failover sets
+    Given a redis server "redis-3" exists as master
+    And a redis server "redis-4" exists as slave of "redis-3"
+    Given a redis configuration server using redis servers "a/redis-1,redis-2;b/redis-3,redis-4" with clients "rc-client-1,rc-client-2" exists
+    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
+    And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
+    And a beetle handler using the redis-master file from "rc-client-1" exists
+    And redis server "redis-1" is down
+    And the retry timeout for the redis master check is reached
+    Then a system notification for "redis-1" not being available should be sent
+    And the role of redis server "redis-2" should be "master"
+    And the redis master file of the redis configuration server should contain "redis-2"
+    And the redis master file of the redis configuration server should contain "redis-3"
+    And the redis master of "rc-client-1" in system "a" should be "redis-2"
+    And the redis master of "rc-client-2" in system "a" should be "redis-2"
+    And the redis master of "rc-client-1" in system "b" should be "redis-3"
+    And the redis master of "rc-client-2" in system "b" should be "redis-3"
+    And the redis master of the beetle handler should be "redis-2"
+    And a system notification for switching from "redis-1" to "redis-2" should be sent
+    Given a redis server "redis-1" exists as master
+    Then the role of redis server "redis-1" should be "slave"
+
+  Scenario: Successful double redis master switch with multiple failover sets
+    Given a redis server "redis-3" exists as master
+    And a redis server "redis-4" exists as slave of "redis-3"
+    Given a redis configuration server using redis servers "a/redis-1,redis-2;b/redis-3,redis-4" with clients "rc-client-1,rc-client-2" exists
+    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
+    And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
+    And a beetle handler using the redis-master file from "rc-client-1" exists
+    And redis server "redis-1" is down
+    And redis server "redis-3" is down
+    And the retry timeout for the redis master check is reached
+    And the role of redis server "redis-2" should be "master"
+    And the role of redis server "redis-4" should be "master"
+    And the redis master file of the redis configuration server should contain "redis-2"
+    And the redis master file of the redis configuration server should contain "redis-4"
+    And the redis master of "rc-client-1" in system "a" should be "redis-2"
+    And the redis master of "rc-client-2" in system "a" should be "redis-2"
+    And the redis master of "rc-client-1" in system "b" should be "redis-4"
+    And the redis master of "rc-client-2" in system "b" should be "redis-4"
+    And the redis master of the beetle handler should be "redis-2"
+    Given a redis server "redis-1" exists as master
+    Then the role of redis server "redis-1" should be "slave"
+    Given a redis server "redis-3" exists as master
+    Then the role of redis server "redis-3" should be "slave"
+
   Scenario: Redis master only temporarily down (no switch necessary)
     Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
     And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists

data/features/step_definitions/redis_auto_failover_steps.rb

@@ -22,13 +22,22 @@ Given /^redis server "([^\"]*)" is slave of "([^\"]*)"$/ do |redis_name, redis_m
 end
 
 Given /^a redis configuration server using redis servers "([^\"]*)" with clients "([^\"]*)" (?:and confidence level "([^\"]*)" )?exists$/ do |redis_names, redis_configuration_client_names, confidence_level|
-  redis_servers = redis_names.split(",").map { |redis_name| TestDaemons::Redis[redis_name].ip_with_port }.join(",")
+  redis_servers = redis_names.split(";").map do |system_spec|
+    if system_spec.include?("/")
+      system_name, servers = system_spec.split("/", 2)
+    else
+      system_name, servers = nil, system_spec
+    end
+    servers = servers.split(",").map { |redis_name| TestDaemons::Redis[redis_name].ip_with_port }.join(",")
+    system_name.nil? ? servers : "#{system_name}/#{servers}"
+  end.join(";")
   TestDaemons::RedisConfigurationServer.start(redis_servers, redis_configuration_client_names, (confidence_level || 100).to_i)
 end
 
 Given /^a redis configuration client "([^\"]*)" using redis servers "([^\"]*)" exists$/ do |redis_configuration_client_name, redis_names|
-  redis_servers = redis_names.split(",").map do |redis_name|
-    TestDaemons::Redis[redis_name].ip_with_port
+  redis_names.split(";").each do |system_spec|
+    servers = system_spec.sub(/^.*\//, '')
+    servers.split(",").map { |redis_name| TestDaemons::Redis[redis_name].ip_with_port }
   end
   TestDaemons::RedisConfigurationClient[redis_configuration_client_name].start
 end
@@ -73,7 +82,7 @@ end
 Given /^an old redis master file for "([^\"]*)" with master "([^\"]*)" exists$/ do |redis_configuration_client_name, redis_name|
   master_file = redis_master_file(redis_configuration_client_name)
   File.open(master_file, 'w') do |f|
-    f.puts TestDaemons::Redis[redis_name].ip_with_port
+    f.puts "system/#{TestDaemons::Redis[redis_name].ip_with_port}"
   end
 end
 
@@ -87,22 +96,28 @@ Then /^the role of redis server "([^\"]*)" should be "(master|slave)"$/ do |redi
   assert expected_role, "#{redis_name} is not a #{role}"
 end
 
-Then /^the redis master of "([^\"]*)" should be "([^\"]*)"$/ do |redis_configuration_client_name, redis_name|
+Then /^the redis master of "([^\"]*)" (?:in system "([^"]*)" )?should be "([^\"]*)"$/ do |redis_configuration_client_name, system_name, redis_name|
+  system_name ||= "system"
   master_file = redis_master_file(redis_configuration_client_name)
   master = false
-  server_info = nil
+  server_info = ''
   10.times do
+    server_name = TestDaemons::Redis[redis_name].ip_with_port
     server_info = File.read(master_file).chomp if File.exist?(master_file)
-    master = true and break if TestDaemons::Redis[redis_name].ip_with_port == server_info
+    if server_info.include?("/")
+      master = true and break if server_info =~ /#{system_name}\/#{server_name}/m
+    else
+      master = true and break if server_info == server_name
+    end
     sleep 1
   end
   assert master, "#{redis_name} is not master of #{redis_configuration_client_name}, master file content: #{server_info.inspect}"
 end
 
-Then /^the redis master file of the redis configuration server should contain "([^"]*)"$/ do |redis_name|  # " for emacs :(
+Then /^the redis master file of the redis configuration server should contain "([^"]*)"$/ do |redis_name|
   master_file = TestDaemons::RedisConfigurationServer.redis_master_file
   file_contents = File.read(master_file).chomp
-  assert_equal TestDaemons::Redis[redis_name].ip_with_port, file_contents
+  assert_match /#{TestDaemons::Redis[redis_name].ip_with_port}/, file_contents
 end
 
 Then /^the redis master of "([^\"]*)" should be undefined$/ do |redis_configuration_client_name|
@@ -111,7 +126,7 @@ Then /^the redis master of "([^\"]*)" should be undefined$/ do |redis_configurat
   server_info = nil
   10.times do
     server_info = File.read(master_file).chomp if File.exist?(master_file)
-    empty = server_info == ""
+    empty = server_info !~ /:\d+/
     break if empty
     sleep 1
   end
@@ -125,7 +140,7 @@ Then /^the redis master of the beetle handler should be "([^\"]*)"$/ do |redis_n
     config.queue(:echo)
     config.message(:echo)
   end
-  assert_equal TestDaemons::Redis[redis_name].ip_with_port, client.rpc(:echo, 'nil').second
+  assert_match /#{TestDaemons::Redis[redis_name].ip_with_port}/, client.rpc(:echo, 'nil').second
 end
 
 Then /^a system notification for "([^\"]*)" not being available should be sent$/ do |redis_name|
@@ -157,7 +172,7 @@ end
 
 Given /^an immediate master switch is initiated and responds with (\d+)$/ do |response_code|
   response = TestDaemons::RedisConfigurationServer.initiate_master_switch
-  assert_equal response_code, response.code
+  assert_equal response_code, response.code, "unexpected response code #{response.code}, message: #{response.body}"
   sleep 1
 end
 

data/features/support/beetle_handler

@@ -24,9 +24,16 @@ Daemons.run_proc("beetle_handler", :log_output => true, :dir_mode => :normal, :d
   client = Beetle::Client.new.configure :auto_delete => true do |config|
     config.queue(:echo)
     config.message(:echo)
-    config.handler(:echo) {|message| client.deduplication_store.redis.server rescue "no redis master"}
+    config.handler(:echo) do |message|
+      begin
+        client.deduplication_store.redis.server
+      rescue
+        master_file_content = File.read(Beetle.config.redis_server)
+        "no redis master: exception: #{$!.class}(#{$!}), master_file: '#{master_file_content}'"
+      end
+    end
   end
   client.listen do
-    puts "Started beetle handler"
+    puts "Started beetle handler for system: #{Beetle.config.system_name}"
   end
 end

data/features/support/env.rb

@@ -7,6 +7,7 @@ World do
 end
 
 Before do
+  cleanup_master_files
   `ruby features/support/system_notification_logger start`
 end
 
@@ -23,8 +24,6 @@ def cleanup_test_env
   TestDaemons::RedisConfigurationServer.stop
 
   `ruby features/support/beetle_handler stop`
-  redis_master_files = tmp_path + "/redis-master-*"
-  `rm -f #{redis_master_files}`
 
   `ruby features/support/system_notification_logger stop`
   # `rm -f #{system_notification_log_path}`
@@ -32,6 +31,11 @@ def cleanup_test_env
   TestDaemons::Redis.stop_all
 end
 
+def cleanup_master_files
+  redis_master_files = tmp_path + "/redis-master-*"
+  `rm -f #{redis_master_files}`
+end
+
 def redis_master_file(client_name)
   tmp_path + "/redis-master-#{client_name}"
 end

data/features/support/test_daemons/redis_configuration_server.rb

@@ -12,10 +12,10 @@ module TestDaemons
     @@confidence_level = 100
 
     def self.start(redis_servers, redis_configuration_clients, confidence_level)
-      stop
       @@redis_servers = redis_servers
       @@redis_configuration_clients = redis_configuration_clients
       @@confidence_level = confidence_level
+      stop
       daemon_controller.start
     end
 
@@ -27,7 +27,7 @@ module TestDaemons
       clients_parameter_string = @@redis_configuration_clients.blank? ? "" : "--client-ids #{@@redis_configuration_clients}"
       DaemonController.new(
          :identifier    => "Redis configuration test server",
-         :start_command => "./beetle configuration_server -v -d --redis-master-file #{redis_master_file} --redis-servers #{@@redis_servers} #{clients_parameter_string} --redis-master-retry-interval 1 --pid-file #{pid_file} --log-file #{log_file} --redis-failover-confidence-level #{@@confidence_level}",
+         :start_command => "./beetle configuration_server -v -d --redis-master-file #{redis_master_file} --redis-servers '#{@@redis_servers}' #{clients_parameter_string} --redis-master-retry-interval 1 --pid-file #{pid_file} --log-file #{log_file} --redis-failover-confidence-level #{@@confidence_level}",
          :ping_command  => lambda{ answers_text_requests? },
          :pid_file      => pid_file,
          :log_file      => log_file,
@@ -86,9 +86,9 @@ module TestDaemons
       response
     end
 
-    def self.initiate_master_switch
+    def self.initiate_master_switch(system_name = "system")
       http = Net::HTTP.new('127.0.0.1', 9650)
-      response = http.post '/initiate_master_switch', ''
+      response = http.post "/initiate_master_switch?system_name=#{system_name}", ""
       response
     end
 

data/lib/beetle.rb

@@ -1,7 +1,6 @@
 $:.unshift(File.expand_path('..', __FILE__))
 require 'bunny'                    # which bunny picks up
 require 'qrack/errors'             # needed by the publisher
-require 'uuid4r'
 require 'redis/connection/hiredis' # require *before* redis as specified in the redis-rb gem docs
 require 'redis'
 require 'active_support/all'

data/lib/beetle/configuration.rb

@@ -62,6 +62,11 @@ module Beetle
     # in bunny.
     attr_accessor :channel_max
 
+    # Lazy queues have the advantage of consuming a lot less memory on the broker. For backwards
+    # compatibility, they are disabled by default.
+    attr_accessor :lazy_queues_enabled
+    alias_method :lazy_queues_enabled?, :lazy_queues_enabled
+
     # In contrast to RabbitMQ 2.x, RabbitMQ 3.x preserves message order when requeing a message. This can lead to
     # throughput degradation (when rejected messages block the processing of other messages
     # at the head of the queue) in some cases.
@@ -89,6 +94,11 @@ module Beetle
     # consider this a highly experimental feature for now.
     attr_accessor :publishing_timeout
 
+    # the connect/disconnect timeout in seconds for the publishing connection
+    # (defaults to <tt>5</tt>).
+    # consider this a highly experimental feature for now.
+    attr_accessor :publisher_connect_timeout
+
     # Prefetch count for subscribers (defaults to 1). Setting this higher
     # than 1 can potentially increase throughput, but comes at the cost of
     # decreased parallelism.
@@ -138,7 +148,10 @@ module Beetle
       self.dead_lettering_msg_ttl = 1000 #1 second
       self.dead_lettering_read_timeout = 3 #3 seconds
 
+      self.lazy_queues_enabled = false
+
       self.publishing_timeout = 0
+      self.publisher_connect_timeout = 5
       self.tmpdir = "/tmp"
 
       self.log_file = STDOUT

data/lib/beetle/dead_lettering.rb

@@ -10,54 +10,64 @@ module Beetle
     end
 
     def bind_dead_letter_queues!(channel, servers, target_queue, creation_keys = {})
-      return unless @config.dead_lettering_enabled?
-
-      dead_letter_queue_name = dead_letter_queue_name(target_queue)
-
-      logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_keys.inspect}")
-      channel.queue(dead_letter_queue_name, creation_keys)
-
-      logger.debug("Beetle: setting #{dead_letter_queue_name} as dead letter queue of #{target_queue} on all servers")
-      set_dead_letter_policies!(servers, target_queue)
+      if @config.dead_lettering_enabled?
+        dead_letter_queue_name = dead_letter_queue_name(target_queue)
+        logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_keys.inspect}")
+        channel.queue(dead_letter_queue_name, creation_keys)
+      end
 
-      logger.debug("Beetle: setting #{target_queue} as dead letter queue of #{dead_letter_queue_name} on all servers")
-      set_dead_letter_policies!(
-        servers,
-        dead_letter_queue_name,
-        :message_ttl => @config.dead_lettering_msg_ttl,
-        :routing_key => target_queue
-      )
+      if @config.dead_lettering_enabled?
+        logger.debug("Beetle: setting #{dead_letter_queue_name} as dead letter queue of #{target_queue} on all servers")
+      end
+      set_queue_policies!(servers, target_queue)
+
+      if @config.dead_lettering_enabled?
+        logger.debug("Beetle: setting #{target_queue} as dead letter queue of #{dead_letter_queue_name} on all servers")
+        set_queue_policies!(
+          servers,
+          dead_letter_queue_name,
+          :message_ttl => @config.dead_lettering_msg_ttl,
+          :routing_key => target_queue
+        )
+      end
     end
 
-    def set_dead_letter_policies!(servers, queue_name, options={})
-      servers.each { |server| set_dead_letter_policy!(server, queue_name, options) }
+    def set_queue_policies!(servers, queue_name, options={})
+      servers.each { |server| set_queue_policy!(server, queue_name, options) }
     end
 
-    def set_dead_letter_policy!(server, queue_name, options={})
+    def set_queue_policy!(server, queue_name, options={})
       raise ArgumentError.new("server missing")     if server.blank?
       raise ArgumentError.new("queue name missing") if queue_name.blank?
 
+      return unless @config.dead_lettering_enabled? || @config.lazy_queues_enabled?
+
       vhost = CGI.escape(@config.vhost)
       request_url = URI("http://#{server}/api/policies/#{vhost}/#{queue_name}_policy")
       request = Net::HTTP::Put.new(request_url)
 
+      # set up queue policy
+      definition = {}
+      if @config.dead_lettering_enabled?
+        definition["dead-letter-routing-key"] = dead_letter_routing_key(queue_name, options)
+        definition["dead-letter-exchange"] = ""
+        definition["message-ttl"] = options[:message_ttl] if options[:message_ttl]
+      end
+
+      definition["queue-mode"] = "lazy" if @config.lazy_queues_enabled?
+
       request_body = {
         "pattern" => "^#{queue_name}$",
         "priority" => 1,
         "apply-to" => "queues",
-        "definition" => {
-          "dead-letter-routing-key" => dead_letter_routing_key(queue_name, options),
-          "dead-letter-exchange" => ""
-        }
+        "definition" => definition,
       }
 
-      request_body["definition"].merge!("message-ttl" => options[:message_ttl]) if options[:message_ttl]
-
       response = run_rabbit_http_request(request_url, request) do |http|
         http.request(request, request_body.to_json)
       end
 
-      if response.code != "204"
+      unless %w(200 201 204).include?(response.code)
         log_error("Failed to create policy for queue #{queue_name}", response)
         raise FailedRabbitRequest.new("Could not create policy")
       end

data/lib/beetle/deduplication_store.rb

@@ -142,13 +142,30 @@ module Beetle
       @last_time_master_file_changed != File.mtime(@config.redis_server)
     end
 
-    # set current redis master from server:port string contained in the redis master file
+    # set current redis master from server:port string contained in the redis master for our system
     def set_current_redis_master_from_master_file
       @last_time_master_file_changed = File.mtime(@config.redis_server)
-      server_string = read_master_file
+      server_string = extract_redis_master(read_master_file)
       @current_master = !server_string.blank? ? Redis.from_server_string(server_string, :db => @config.redis_db) : nil
     end
 
+    # extract redis master from file content and return the server for our system
+    def extract_redis_master(text)
+      system_name = @config.system_name
+      redis_master = ""
+      text.each_line do |line|
+        parts = line.split('/')
+        case parts.size
+        when 1
+          redis_master = parts[0]
+        when 2
+          name, master = parts
+          redis_master = master if name == system_name
+        end
+      end
+      redis_master
+    end
+
     # server:port string from the redis master file
     def read_master_file
       File.read(@config.redis_server).chomp

data/lib/beetle/message.rb

@@ -146,8 +146,16 @@ module Beetle
     end
 
     # generate uuid for publishing
-    def self.generate_uuid
-      UUID4R::uuid(4)
+    begin
+      require "uuid4r"
+      def self.generate_uuid
+        UUID4R::uuid(4)
+      end
+    rescue LoadError
+      require "securerandom"
+      def self.generate_uuid
+        SecureRandom.uuid
+      end
     end
 
     # whether the publisher has tried sending this message to two servers

data/lib/beetle/publisher.rb

@@ -159,15 +159,16 @@ module Beetle
 
     def new_bunny
       b = Bunny.new(
-        :host           => current_host,
-        :port           => current_port,
-        :logging        => !!@options[:logging],
-        :user           => @client.config.user,
-        :pass           => @client.config.password,
-        :vhost          => @client.config.vhost,
-        :frame_max      => @client.config.frame_max,
-        :channel_max    => @client.config.channel_max,
-        :socket_timeout => @client.config.publishing_timeout,
+        :host               => current_host,
+        :port               => current_port,
+        :logging            => !!@options[:logging],
+        :user               => @client.config.user,
+        :pass               => @client.config.password,
+        :vhost              => @client.config.vhost,
+        :frame_max          => @client.config.frame_max,
+        :channel_max        => @client.config.channel_max,
+        :socket_timeout     => @client.config.publishing_timeout,
+        :connect_timeout    => @client.config.publisher_connect_timeout,
         :spec => '09')
       b.start
       b
@@ -224,7 +225,8 @@ module Beetle
 
     def stop!(exception=nil)
       return unless bunny?
-      Beetle::Timer.timeout(1) do
+      timeout = @client.config.publishing_timeout + @client.config.publisher_connect_timeout + 1
+      Beetle::Timer.timeout(timeout) do
         logger.debug "Beetle: closing connection from publisher to #{server}"
         if exception
           bunny.__send__ :close_socket

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "2.2.3"
+  VERSION = "2.3.0"
 end

data/script/console

@@ -10,16 +10,16 @@ OptionParser.new do |opt|
 end
 
 libs =  " -r irb/completion"
-libs << %( -r ubygems)
+libs << %( -r rubygems)
 libs << %( -r #{File.expand_path("../../lib/beetle.rb",__FILE__)})
 
 if options[:debugger]
   begin
-    require 'ruby-debug'
-    libs << " -r ruby-debug"
+    require 'byebug'
+    libs << " -r byebug"
     puts "=> Debugger enabled"
   rescue Exception
-    puts "You need to install ruby-debug to run the console in debugging mode. With gems, use 'gem install ruby-debug'"
+    puts "You need to install byebug to run the console in debugging mode. With gems, use 'gem install byebug'"
     exit
   end
 end

data/test/beetle/dead_lettering_test.rb

@@ -3,18 +3,20 @@ require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
 module Beetle
   class SetDeadLetteringsTest < Minitest::Test
     def setup
-      @dead_lettering = DeadLettering.new(Configuration.new)
+      @config = Configuration.new
+      @dead_lettering = DeadLettering.new(@config)
+      @config.dead_lettering_enabled = true
     end
 
     test "creates a dead letter queue for each server" do
       servers = %w(a b)
 
-      @dead_lettering.expects(:set_dead_letter_policy!).
+      @dead_lettering.expects(:set_queue_policy!).
         with("a", "QUEUE_NAME", :message_ttl => 10000)
-      @dead_lettering.expects(:set_dead_letter_policy!).
+      @dead_lettering.expects(:set_queue_policy!).
         with("b", "QUEUE_NAME", :message_ttl => 10000)
 
-      @dead_lettering.set_dead_letter_policies!(servers, "QUEUE_NAME", :message_ttl => 10000)
+      @dead_lettering.set_queue_policies!(servers, "QUEUE_NAME", :message_ttl => 10000)
     end
   end
 
@@ -25,21 +27,22 @@ module Beetle
       @config = Configuration.new
       @config.logger = Logger.new("/dev/null")
       @dead_lettering = DeadLettering.new(@config)
+      @config.dead_lettering_enabled = true
     end
 
     test "raises exception when queue name wasn't specified" do
       assert_raises ArgumentError do
-        @dead_lettering.set_dead_letter_policy!(@server, "")
+        @dead_lettering.set_queue_policy!(@server, "")
       end
     end
 
     test "raises exception when no server was specified" do
       assert_raises ArgumentError do
-        @dead_lettering.set_dead_letter_policy!("", @queue_name)
+        @dead_lettering.set_queue_policy!("", @queue_name)
       end
     end
 
-    test "creates a policy by posting to the rabbitmq" do
+    test "creates a policy by posting to the rabbitmq if dead lettering is enabled" do
       stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
         .with(basic_auth: ['guest', 'guest'])
         .with(:body => {
@@ -52,7 +55,24 @@ module Beetle
                }}.to_json)
         .to_return(:status => 204)
 
-      @dead_lettering.set_dead_letter_policy!(@server, @queue_name)
+      @dead_lettering.set_queue_policy!(@server, @queue_name)
+    end
+
+    test "creates a policy by posting to the rabbitmq if lazy queues are enabled" do
+      @config.lazy_queues_enabled = true
+      @config.dead_lettering_enabled = false
+      stub_request(:put, "http://localhost:15672/api/policies/%2F/QUEUE_NAME_policy")
+        .with(basic_auth: ['guest', 'guest'])
+        .with(:body => {
+               "pattern" => "^QUEUE_NAME$",
+               "priority" => 1,
+               "apply-to" => "queues",
+               "definition" => {
+                 "queue-mode" => "lazy"
+               }}.to_json)
+        .to_return(:status => 204)
+
+      @dead_lettering.set_queue_policy!(@server, @queue_name)
     end
 
     test "raises exception when policy couldn't successfully be created" do
@@ -61,7 +81,7 @@ module Beetle
         .to_return(:status => [405])
 
       assert_raises DeadLettering::FailedRabbitRequest do
-        @dead_lettering.set_dead_letter_policy!(@server, @queue_name)
+        @dead_lettering.set_queue_policy!(@server, @queue_name)
       end
     end
 
@@ -79,7 +99,7 @@ module Beetle
                 }}.to_json)
         .to_return(:status => 204)
 
-      @dead_lettering.set_dead_letter_policy!(@server, @queue_name, :message_ttl => 10000)
+      @dead_lettering.set_queue_policy!(@server, @queue_name, :message_ttl => 10000)
     end
 
     test "properly encodes the vhost from the configuration" do
@@ -97,7 +117,7 @@ module Beetle
 
       @config.vhost = "foo/"
 
-      @dead_lettering.set_dead_letter_policy!(@server, @queue_name)
+      @dead_lettering.set_queue_policy!(@server, @queue_name)
     end
   end
 
@@ -110,9 +130,10 @@ module Beetle
       @servers = ["localhost:55672"]
     end
 
-    test "is turned off by default" do
+    test "is does not call out to rabbit if neither dead lettering nor lazy queues are enabled" do
+      @config.dead_lettering_enabled = false
       channel = stub('channel')
-      @dead_lettering.expects(:set_dead_letter_policies!).never
+      @dead_lettering.expects(:run_rabbit_http_request).never
       @dead_lettering.bind_dead_letter_queues!(channel, @servers, @queue_name)
     end
 
@@ -122,8 +143,8 @@ module Beetle
       channel = stub('channel')
 
       channel.expects(:queue).with("#{@queue_name}_dead_letter", {})
-      @dead_lettering.expects(:set_dead_letter_policies!).with(@servers, @queue_name)
-      @dead_lettering.expects(:set_dead_letter_policies!).with(@servers, "#{@queue_name}_dead_letter",
+      @dead_lettering.expects(:set_queue_policies!).with(@servers, @queue_name)
+      @dead_lettering.expects(:set_queue_policies!).with(@servers, "#{@queue_name}_dead_letter",
         :routing_key => @queue_name,
         :message_ttl => 1000
       )

data/test/beetle/deduplication_store_test.rb

@@ -43,6 +43,7 @@ module Beetle
   class RedisServerFileTest < Minitest::Test
     def setup
       @original_redis_server = Beetle.config.redis_server
+      @original_system_name = Beetle.config.system_name
       @store = DeduplicationStore.new
       @server_string = "my_test_host_from_file:6379"
       Beetle.config.redis_server = redis_test_master_file(@server_string)
@@ -50,6 +51,7 @@ module Beetle
 
     def teardown
       Beetle.config.redis_server = @original_redis_server
+      Beetle.config.system_name = @original_system_name
     end
 
     test "redis should match the redis master file" do
@@ -71,6 +73,17 @@ module Beetle
       assert_raises(Errno::ENOENT) { @store.redis_master_from_master_file }
     end
 
+    test "should retrieve the redis master for the configured system name if the master file contains a mapping for it" do
+      redis_test_master_file("blabber/localhost:2\nblubber/localhost:1")
+      Beetle.config.system_name = "blubber"
+      assert_equal "localhost:1", @store.redis.server
+    end
+
+    test "should retrieve the redis master for the default system name if the master file contains a simple host:port entry" do
+      redis_test_master_file("localhost:2\nblubber/localhost:1")
+      assert_equal "localhost:2", @store.redis.server
+    end
+
     private
     def redis_test_master_file(server_string)
       tmp_dir = File.expand_path("../../../tmp", __FILE__)

data/test/beetle/publisher_test.rb

@@ -24,6 +24,7 @@ module Beetle
         :pass => "guest",
         :vhost => "/",
         :socket_timeout => 0,
+        :connect_timeout => 5,
         :frame_max => 131072,
         :channel_max => 2047,
         :spec => '09'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 2.2.3
+  version: 2.3.0
 platform: ruby
 authors:
 - Stefan Kaes
@@ -12,36 +12,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-18 00:00:00.000000000 Z
+date: 2019-02-28 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: uuid4r
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.1.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.1.2
 - !ruby/object:Gem::Dependency
   name: bunny
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.10
+        version: 0.7.12
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.10
+        version: 0.7.12
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -112,6 +98,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.3.4
+- !ruby/object:Gem::Dependency
+  name: uuid4r
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.2
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -328,7 +328,6 @@ files:
 - examples/redundant.rb
 - examples/rpc.rb
 - examples/simple.rb
-- features/#redis_auto_failover.feature#
 - features/README.rdoc
 - features/redis_auto_failover.feature
 - features/step_definitions/redis_auto_failover_steps.rb
@@ -351,7 +350,6 @@ files:
 - lib/beetle/publisher.rb
 - lib/beetle/r_c.rb
 - lib/beetle/redis_ext.rb
-- lib/beetle/redis_master_file.rb
 - lib/beetle/subscriber.rb
 - lib/beetle/version.rb
 - script/console
@@ -369,13 +367,13 @@ files:
 - test/beetle/publisher_test.rb
 - test/beetle/r_c_test.rb
 - test/beetle/redis_ext_test.rb
-- test/beetle/redis_master_file_test.rb
 - test/beetle/subscriber_test.rb
 - test/beetle_test.rb
 - test/test_helper.rb
 homepage: https://xing.github.com/beetle/
 licenses: []
-metadata: {}
+metadata:
+  changelog_uri: https://github.com/xing/beetle/blob/master/RELEASE_NOTES.rdoc
 post_install_message: 
 rdoc_options:
 - "--charset=UTF-8"
@@ -403,7 +401,6 @@ test_files:
 - test/beetle/deduplication_store_test.rb
 - test/beetle/handler_test.rb
 - test/beetle/beetle_test.rb
-- test/beetle/redis_master_file_test.rb
 - test/beetle/configuration_test.rb
 - test/beetle/subscriber_test.rb
 - test/beetle/message/settings_test.rb

data/features/#redis_auto_failover.feature#

@@ -1,165 +0,0 @@
-Feature: Redis auto failover
-  In order to eliminate a single point of failure
-  Beetle handlers should automatically switch to a new redis master in case of a redis master failure
-
-  Background:
-    Given a redis server "redis-1" exists as master
-    And a redis server "redis-2" exists as slave of "redis-1"
-
-  Scenario: Successful redis master switch
-    Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
-    And a beetle handler using the redis-master file from "rc-client-1" exists
-    And redis server "redis-1" is down
-    And the retry timeout for the redis master check is reached
-    Then a system notification for "redis-1" not being available should be sent
-    And the role of redis server "redis-2" should be "master"
-    And the redis master file of the redis configuration server should contain "redis-2"
-    And the redis master of "rc-client-1" should be "redis-2"
-    And the redis master of "rc-client-2" should be "redis-2"
-    And the redis master of the beetle handler should be "redis-2"
-    And a system notification for switching from "redis-1" to "redis-2" should be sent
-    Given a redis server "redis-1" exists as master
-    Then the role of redis server "redis-1" should be "slave"
-
-  Scenario: Not all redis configuration clients available (switch possible using confidence level 50)
-    Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" and confidence level "50" exists
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    And a beetle handler using the redis-master file from "rc-client-1" exists
-    And redis configuration client "rc-client-2" is down
-    And redis server "redis-1" is down
-    And the retry timeout for the redis master check is reached
-    Then a system notification for "redis-1" not being available should be sent
-    And the role of redis server "redis-2" should be "master"
-    And the redis master file of the redis configuration server should contain "redis-2"
-    And the redis master of "rc-client-1" should be "redis-2"
-    And the redis master of the beetle handler should be "redis-2"
-    And a system notification for switching from "redis-1" to "redis-2" should be sent
-    Given a redis server "redis-1" exists as master
-    Then the role of redis server "redis-1" should be "slave"
-
-  Scenario: Redis master only temporarily down (no switch necessary)
-    Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
-    And a beetle handler using the redis-master file from "rc-client-1" exists
-    And redis server "redis-1" is down for less seconds than the retry timeout for the redis master check
-    And the retry timeout for the redis master check is reached
-    Then the role of redis server "redis-1" should be "master"
-    Then the role of redis server "redis-2" should be "slave"
-    And the redis master of "rc-client-1" should be "redis-1"
-    And the redis master of "rc-client-2" should be "redis-1"
-    And the redis master of the beetle handler should be "redis-1"
-
-  Scenario: Not all redis configuration clients available (no switch possible)
-    Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
-    And redis server "redis-1" is down
-    And the retry timeout for the redis master check is reached
-    Then the role of redis server "redis-2" should be "slave"
-
-  Scenario: No redis slave available to become new master (no switch possible)
-    Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
-    And redis server "redis-1" is down
-    And redis server "redis-2" is down
-    And the retry timeout for the redis master check is reached
-    Then the redis master of "rc-client-1" should be "redis-1"
-    And the redis master of "rc-client-2" should be "redis-1"
-    And a system notification for no slave available to become new master should be sent
-
-  Scenario: Redis configuration client starts while no redis master available
-    Given redis server "redis-1" is down
-    And redis server "redis-2" is down
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    And the retry timeout for the redis master determination is reached
-    Then the redis master of "rc-client-1" should be undefined
-
-  Scenario: Redis configuration client starts while no redis master available but master file exists
-    Given redis server "redis-1" is down
-    And redis server "redis-2" is down
-    And an old redis master file for "rc-client-1" with master "redis-1" exists
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    And the retry timeout for the redis master determination is reached
-    Then the redis master of "rc-client-1" should be undefined
-
-  Scenario: Redis configuration client starts while both redis servers are master
-    Given redis server "redis-2" is master
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    Then the redis master of "rc-client-1" should be undefined
-
-  Scenario: Redis configuration client starts while both redis servers are master but master file exists
-    Given redis server "redis-2" is master
-    And an old redis master file for "rc-client-1" with master "redis-1" exists
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    Then the redis master of "rc-client-1" should be "redis-1"
-
-  Scenario: Redis configuration client starts while both redis servers are slave
-    Given a redis server "redis-3" exists as master
-    And redis server "redis-1" is slave of "redis-3"
-    And redis server "redis-2" is slave of "redis-3"
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    Then the redis master of "rc-client-1" should be undefined
-
-  Scenario: Redis configuration client starts while both redis servers are slave but master file exists
-    Given a redis server "redis-3" exists as master
-    And redis server "redis-1" is slave of "redis-3"
-    And redis server "redis-2" is slave of "redis-3"
-    And an old redis master file for "rc-client-1" with master "redis-1" exists
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    Then the redis master of "rc-client-1" should be undefined
-
-  Scenario: Redis configuration client starts while there is a redis master but no slave
-    Given redis server "redis-2" is down
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    Then the redis master of "rc-client-1" should be undefined
-
-  Scenario: Redis configuration client starts while there is a redis master but no slave but master file exists
-    Given redis server "redis-2" is down
-    And an old redis master file for "rc-client-1" with master "redis-1" exists
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    Then the redis master of "rc-client-1" should be "redis-1"
-
-  Scenario: Redis configuation server should embed a http server
-    Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
-    Then the redis configuration server should answer http requests
-
-  Scenario: Accelerated redis master switch when master is down
-    Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
-    And a beetle handler using the redis-master file from "rc-client-1" exists
-    And redis server "redis-1" is down
-    And an immediate master switch is initiated and responds with 201
-    Then a system notification for "redis-1" not being available should be sent
-    And the role of redis server "redis-2" should be "master"
-    And the redis master of "rc-client-1" should be "redis-2"
-    And the redis master of "rc-client-2" should be "redis-2"
-    And the redis master of the beetle handler should be "redis-2"
-    And a system notification for switching from "redis-1" to "redis-2" should be sent
-    Given a redis server "redis-1" exists as master
-    Then the role of redis server "redis-1" should be "slave"
-
-  Scenario: Accelerated redis master switch when master is up
-    Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
-    And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-    And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
-    And a beetle handler using the redis-master file from "rc-client-1" exists
-    And an immediate master switch is initiated and responds with 200
-    Then the role of redis server "redis-1" should be "master"
-    And the redis master of "rc-client-1" should be "redis-1"
-    And the redis master of "rc-client-2" should be "redis-1"
-    And the redis master of the beetle handler should be "redis-1"
-    And the role of redis server "redis-2" should be "slave"
-
-#   Scenario: Running the system for a few seconds to perform manual testing
-#     Given a redis configuration server using redis servers "redis-1,redis-2" with clients "rc-client-1,rc-client-2" exists
-#     And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
-#     And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
-#     And a beetle handler using the redis-master file from "rc-client-1" exists
-#     And the redis master of "rc-client-1" should be "redis-1"
-#     And the redis master of "rc-client-2" should be "redis-1"
-#     And the redis master of the beetle handler should be "redis-1"
-#     And the role of redis server "redis-2" should be "slave"
-#     Then the system can run for a while without dying

data/lib/beetle/redis_master_file.rb

@@ -1,30 +0,0 @@
-module Beetle
-  module RedisMasterFile #:nodoc:
-    private
-    def master_file_exists?
-      File.exist?(master_file)
-    end
-
-    def redis_master_from_master_file
-      redis.find(read_redis_master_file)
-    end
-
-    def clear_redis_master_file
-      write_redis_master_file("")
-    end
-
-    def read_redis_master_file
-      File.read(master_file).chomp
-    end
-
-    def write_redis_master_file(redis_server_string)
-      logger.warn "Writing '#{redis_server_string}' to redis master file '#{master_file}'"
-      File.open(master_file, "w"){|f| f.puts redis_server_string }
-    end
-
-    def master_file
-      config.redis_server
-    end
-
-  end
-end

data/test/beetle/redis_master_file_test.rb

@@ -1,39 +0,0 @@
-require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
-
-module Beetle
-
-  class RedisMasterFileTest < Minitest::Test
-    include Logging
-    include RedisMasterFile
-
-    def setup
-      File.open(master_file, "w"){|f| f.puts "localhost:6379"}
-    end
-
-    def teardown
-      File.unlink(master_file) if File.exist?(master_file)
-    end
-
-    test "should be able to check existence" do
-      assert master_file_exists?
-      File.unlink(master_file)
-      assert !master_file_exists?
-    end
-
-    test "should be able to read and write the master file"do
-      write_redis_master_file("localhost:6380")
-      assert_equal "localhost:6380", read_redis_master_file
-    end
-
-    test "should be able to clear the master file" do
-      logger.expects(:warn)
-      clear_redis_master_file
-      assert_equal "", read_redis_master_file
-    end
-
-    private
-    def master_file
-      "/tmp/mumu.txt"
-    end
-  end
-end