beetle 3.4.0 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 28836e3040ba8ef4415a6aae49f3ae805384e61b45cd76993a29d7589c23b7bf
-  data.tar.gz: bed46b58fe787d79a21b1fb3413a0359d948e79e70da0de92bfb717ad5e4e7be
+  metadata.gz: 2dcaf90eaf1b11b7f6350bb34e17d73591616790ede7b43b6e98cef363dd3770
+  data.tar.gz: 55348c9ce61cc2f43869f9c12cd26bbc5dc033b0e0a0506c0a821d6a6e24862c
 SHA512:
-  metadata.gz: 5b92b7588bcdd74a13a211d76997bae5f2255f729bb62b3c673ef00ec0fbda9a69e609055ad1a65797350c8151da6aa059093af3a27016069f22f1fa77c347d6
-  data.tar.gz: 7baa29a222dea24a7ce3f90fbeb90634d7b816197aa34c7fb5401e0d4754ab339f36c733d67aeccbeaf61524e9f74a59f07e697e931af96d94343d9977209dea
+  metadata.gz: 6daabfea72ab76b0fa6944d35cee52139678f9a02a272e86a32af6045750496ee81b96ff685eee1bbe9617030b981df3718b7569efabc98df59f894127da29d1
+  data.tar.gz: 04d599595b4c39e62a78b496b284a67bed1a545e759d9a094facc757e0334ed16e4bdd7bab4f7bfabb7b6f63da0e590351e86d0da55328ab6db898e9ceaaef7f

data/RELEASE_NOTES.rdoc

@@ -1,5 +1,22 @@
 = Release Notes
 
+== Version 3.5.1
+*  remove blank entries from server list strings
+*  make sure not to subscribe to any server twice
+
+== Version 3.5.0
+* expose publisher method to setup queues/policies ahead of use
+
+== Version 3.4.3
+* optimize declaration of queues with many bindings
+
+== Version 3.4.2
+* Updated amq-protocol gem to version 2.3.2.
+* Fixed a rare race condition on message handler timeouts.
+
+== Version 3.4.1
+* Updated amq-protocol gem to version 2.3.1.
+
 == Version 3.4.0
 * Require redis gem version 4.2.1. This version changes the exists to check for the
   existence of multiple keys, return the number of keys in the list that exist. This

data/beetle.gemspec

@@ -26,7 +26,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "bunny",                   "~> 0.7.12"
   s.add_runtime_dependency "redis",                   ">= 4.2.1"
   s.add_runtime_dependency "hiredis",                 ">= 0.4.5"
-  s.add_runtime_dependency "amq-protocol",            "= 2.3.0"
+  s.add_runtime_dependency "amq-protocol",            "= 2.3.2"
   s.add_runtime_dependency "amqp",                    "= 1.8.0"
   s.add_runtime_dependency "activesupport",           ">= 2.3.4"
 

data/lib/beetle/base.rb

@@ -67,23 +67,25 @@ module Beetle
           logger.debug("Beetle: binding queue #{name} with internal name #{opts[:amqp_name]} on server #{@server}")
           queue_name = opts[:amqp_name]
           creation_options = opts.slice(*QUEUE_CREATION_KEYS)
-          the_queue = nil
+
+          the_queue = declare_queue!(queue_name, creation_options)
           @client.bindings[name].each do |binding_options|
             exchange_name = binding_options[:exchange]
             binding_options = binding_options.slice(*QUEUE_BINDING_KEYS)
-            the_queue = bind_queue!(queue_name, creation_options, exchange_name, binding_options)
+            logger.debug("Beetle: binding queue #{queue_name} to #{exchange_name} with opts: #{binding_options.inspect}")
+            bind_queue!(the_queue, exchange_name, binding_options)
           end
           the_queue
         end
     end
 
-    def bind_dead_letter_queue!(channel, target_queue, creation_keys = {})
+    def bind_dead_letter_queue!(channel, target_queue, creation_options = {})
       policy_options = @client.queues[target_queue].slice(:dead_lettering, :lazy, :dead_lettering_msg_ttl)
       policy_options[:message_ttl] = policy_options.delete(:dead_lettering_msg_ttl)
       dead_letter_queue_name = "#{target_queue}_dead_letter"
       if policy_options[:dead_lettering]
-        logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_keys.inspect}")
-        channel.queue(dead_letter_queue_name, creation_keys)
+        logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_options.inspect}")
+        channel.queue(dead_letter_queue_name, creation_options)
       end
       return {
         :queue_name => target_queue,
@@ -93,7 +95,7 @@ module Beetle
       }.merge(policy_options)
     end
 
-    # called by <tt>bind_queue!</tt>
+    # called by <tt>declare_queue!</tt>
     def publish_policy_options(options)
       # avoid endless recursion
       return if options[:queue_name] == @client.config.beetle_policy_updates_queue_name

data/lib/beetle/client.rb

@@ -280,6 +280,13 @@ module Beetle
       publisher.throttled?
     end
 
+    # set up queues and policies for all configured queues. Otherwise this will
+    # happen on first use of an exchange, which can be undesired for latency
+    # sensitive endpoints. Only needs to be called once.
+    def setup_queues_and_policies
+      publisher.setup_queues_and_policies
+    end
+
     # traces queues without consuming them. useful for debugging message flow.
     def trace(queue_names=self.queues.keys, tracer=nil, &block)
       queues_to_trace = self.queues.slice(*queue_names)
@@ -378,8 +385,12 @@ module Beetle
     end
 
     def load_brokers_from_config
-      @servers = config.servers.split(/ *, */)
-      @additional_subscription_servers = config.additional_subscription_servers.split(/ *, */)
+      @servers = parse_server_list(config.servers)
+      @additional_subscription_servers = parse_server_list(config.additional_subscription_servers) - @servers
+    end
+
+    def parse_server_list(s)
+      s.split(/ *, */).uniq.reject(&:blank?)
     end
   end
 end

data/lib/beetle/message.rb

@@ -18,6 +18,8 @@ module Beetle
     # forcefully abort a running handler after this many seconds.
     # can be overriden when registering a handler.
     DEFAULT_HANDLER_TIMEOUT = 600.seconds
+    # How much extra time on top of the handler timeout we add before considering a handler timed out
+    TIMEOUT_GRACE_PERIOD = 10.seconds
     # how many times we should try to run a handler before giving up
     DEFAULT_HANDLER_EXECUTION_ATTEMPTS = 1
     # how many seconds we should wait before retrying handler execution
@@ -167,8 +169,8 @@ module Beetle
     end
 
     # handler timed out?
-    def timed_out?
-      (t = @store.get(msg_id, :timeout)) && t.to_i < now
+    def timed_out?(t = nil)
+      (t ||= @store.get(msg_id, :timeout)) && (t.to_i + TIMEOUT_GRACE_PERIOD) < now
     end
 
     # reset handler timeout in the deduplication store
@@ -187,8 +189,8 @@ module Beetle
     end
 
     # whether we should wait before running the handler
-    def delayed?
-      (t = @store.get(msg_id, :delay)) && t.to_i > now
+    def delayed?(t = nil)
+      (t ||= @store.get(msg_id, :delay)) && t.to_i > now
     end
 
     # store delay value in the deduplication store
@@ -207,8 +209,8 @@ module Beetle
     end
 
     # whether we have already tried running the handler as often as specified when the handler was registered
-    def attempts_limit_reached?
-      (limit = @store.get(msg_id, :attempts)) && limit.to_i >= attempts_limit
+    def attempts_limit_reached?(attempts = nil)
+      (attempts ||= @store.get(msg_id, :attempts)) && attempts.to_i >= attempts_limit
     end
 
     # increment number of exception occurences in the deduplication store
@@ -217,8 +219,8 @@ module Beetle
     end
 
     # whether the number of exceptions has exceeded the limit set when the handler was registered
-    def exceptions_limit_reached?
-      @store.get(msg_id, :exceptions).to_i > exceptions_limit
+    def exceptions_limit_reached?(exceptions = nil)
+      (exceptions ||= @store.get(msg_id, :exceptions)) && exceptions.to_i > exceptions_limit
     end
 
     def exception_accepted?
@@ -306,17 +308,17 @@ module Beetle
         if status == "completed"
           ack!
           RC::OK
-        elsif delay && delay.to_i > now
+        elsif delay && delayed?(delay)
           logger.warn "Beetle: ignored delayed message (#{msg_id})!"
           RC::Delayed
-        elsif !(timeout && timeout.to_i < now)
+        elsif !(timeout && timed_out?(timeout))
           RC::HandlerNotYetTimedOut
-        elsif attempts.to_i >= attempts_limit
+        elsif attempts && attempts_limit_reached?(attempts)
           completed!
           ack!
           logger.warn "Beetle: reached the handler execution attempts limit: #{attempts_limit} on #{msg_id}"
           RC::AttemptsLimitReached
-        elsif exceptions.to_i > exceptions_limit
+        elsif exceptions && exceptions_limit_reached?(exceptions)
           completed!
           ack!
           logger.warn "Beetle: reached the handler exceptions limit: #{exceptions_limit} on #{msg_id}"

data/lib/beetle/publisher.rb

@@ -169,10 +169,14 @@ module Beetle
       end
     end
 
-    def setup_queues_and_policies(queue_names) #:nodoc:
+    def setup_queues_and_policies
       each_server do
-        queue_names.each do |name|
-          queue(name, create_policies: true)
+        begin
+          @client.queues.keys.each do |name|
+            queue(name)
+          end
+        rescue => e
+          logger.warn "Beetle: failed setting up queues and policies on #{@server}: #{e}"
         end
       end
     end
@@ -247,17 +251,19 @@ module Beetle
       @exchanges_with_bound_queues[exchange_name] = true
     end
 
-    # TODO: Refactor, fetch the keys and stuff itself
-    def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
-      logger.debug("Beetle: creating queue with opts: #{creation_keys.inspect}")
-      queue = bunny.queue(queue_name, creation_keys)
-      logger.debug("Beetle: binding queue #{queue_name} to #{exchange_name} with opts: #{binding_keys.inspect}")
-      queue.bind(exchange(exchange_name), binding_keys)
-      policy_options = bind_dead_letter_queue!(bunny, queue_name, creation_keys)
+    def declare_queue!(queue_name, creation_options)
+      logger.debug("Beetle: creating queue with opts: #{creation_options.inspect}")
+      queue = bunny.queue(queue_name, creation_options)
+
+      policy_options = bind_dead_letter_queue!(bunny, queue_name, creation_options)
       publish_policy_options(policy_options)
       queue
     end
 
+    def bind_queue!(queue, exchange_name, binding_options)
+      queue.bind(exchange(exchange_name), binding_options)
+    end
+
     def stop!(exception=nil)
       return unless bunny?
       timeout = @client.config.publishing_timeout + @client.config.publisher_connect_timeout + 1

data/lib/beetle/subscriber.rb

@@ -214,18 +214,20 @@ module Beetle
       channel.__send__(opts[:type], name, opts.slice(*EXCHANGE_CREATION_KEYS))
     end
 
-    def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
-      queue = channel.queue(queue_name, creation_keys)
+    def declare_queue!(queue_name, creation_options)
+      queue = channel.queue(queue_name, creation_options)
       unless tracing?
         # we don't want to create dead-letter queues for tracing
-        policy_options = bind_dead_letter_queue!(channel, queue_name, creation_keys)
+        policy_options = bind_dead_letter_queue!(channel, queue_name, creation_options)
         publish_policy_options(policy_options)
       end
-      exchange = exchange(exchange_name)
-      queue.bind(exchange, binding_keys)
       queue
     end
 
+    def bind_queue!(queue, exchange_name, binding_options)
+      queue.bind(exchange(exchange_name), binding_options)
+    end
+
     def connection_settings
       {
         :host => current_host, :port => current_port, :logging => false,

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "3.4.0"
+  VERSION = "3.5.1"
 end

data/script/analyze-expiries

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'time'
+
+counts = Hash.new(0)
+expiries = Hash.new{|h,k| h[k] = Hash.new(0)}
+t = Time.now.to_i
+
+File.open(ARGV[0]).each_line do |l|
+  parts = l.split(':')
+  queue = parts[1]
+  counts[queue] += 1
+  expiry = parts[4].to_i
+  expires_in = ((expiry - t)/(3600.0)).ceil
+  expiries[queue][expires_in] += 1
+end
+
+counts.to_a.sort_by{|_,v| -v}.each do |q,v|
+  puts "------------------------------------------------------------------"
+  puts "#{q}: #{v}"
+  puts "------------------------------------------------------------------"
+  expiries[q].to_a.sort_by{|k,_| -k}.each do |expiry, count|
+    printf "%3dh: %6d\n", expiry, count
+  end
+end

data/script/analyze-redis-keys

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+queue_counts = Hash.new { |h,k| h[k] = 0 }
+File.open(ARGV[0]).each_line do |l|
+  next if l == "lastgc" || l == "clients-last-seen"
+  a = l.split(':')
+  if a[0] == "msgid"
+    queue_counts[a[1]] += 1
+  else
+    queue_counts["none"] += 1
+  end
+end
+sorted_queues = queue_counts.to_a.sort_by{|a| -a[1]}
+sorted_queues.each do |q,c|
+  puts "#{c}:#{q}"
+end

data/script/expire-keys

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+require "redis"
+
+redis = Redis.new(:host => "beetle-1.redis.ams2.xing.com", :port => 6379, :db => 0)
+
+File.open(ARGV[0]).each_line.each_slice(10) do |s|
+  redis.pipelined do
+    s.each do |l|
+      l.chomp!
+      next if l =~ /^beetle:.*$/
+      redis.expire(l, 600)
+    end
+  end
+end

data/test/beetle/client_test.rb

@@ -282,6 +282,12 @@ module Beetle
       client.stop_publishing
     end
 
+    test "should delegate setup_queues_and_policies to the publisher instance" do
+      client = Client.new
+      client.send(:publisher).expects(:setup_queues_and_policies)
+      client.setup_queues_and_policies
+    end
+
     test "should delegate queue purging to the publisher instance" do
       client = Client.new
       client.register_queue(:queue)
@@ -337,17 +343,17 @@ module Beetle
 
     test "trying to listen to an unknown queue should raise an exception" do
       client = Client.new
-      assert_raises(UnknownQueue) { Client.new.listen_queues([:foobar])}
+      assert_raises(UnknownQueue) { client.listen_queues([:foobar])}
     end
 
     test "trying to pause listening on an unknown queue should raise an exception" do
       client = Client.new
-      assert_raises(UnknownQueue) { Client.new.pause_listening(:foobar)}
+      assert_raises(UnknownQueue) { client.pause_listening(:foobar)}
     end
 
     test "trying to resume listening on an unknown queue should raise an exception" do
       client = Client.new
-      assert_raises(UnknownQueue) { Client.new.pause_listening(:foobar)}
+      assert_raises(UnknownQueue) { client.pause_listening(:foobar)}
     end
 
     test "should delegate stop_listening to the subscriber instance" do
@@ -451,4 +457,39 @@ module Beetle
       msg_stub
     end
   end
+
+  class ServerDeduplicationTest < Minitest::Test
+    def setup
+      @config = Configuration.new
+      @config.servers = "localhost:5672,localhost:5672"
+      @config.additional_subscription_servers = @config.servers + ",localhost:1234,localhost:1234"
+      @client = Client.new(@config)
+    end
+
+    test "duplicates are removed from the server list" do
+      assert_equal ["localhost:5672"], @client.servers
+    end
+
+    test "duplicates and servers in the server list are removed from the additional subscription server list" do
+      assert_equal ["localhost:1234"], @client.additional_subscription_servers
+    end
+  end
+
+  class EmptyServerStringTest < Minitest::Test
+    def setup
+      @config = Configuration.new
+      @config.servers = "localhost:5672,\t,localhost:5672, "
+      @config.additional_subscription_servers = @config.servers + ",localhost:1234, , ,,"
+      @client = Client.new(@config)
+    end
+
+    test "empty strings are removed from the server list" do
+      assert_equal ["localhost:5672"], @client.servers
+    end
+
+    test "empty strings are removed from the additional subscription server list" do
+      assert_equal ["localhost:1234"], @client.additional_subscription_servers
+    end
+  end
+
 end

data/test/beetle/message/settings_test.rb

@@ -75,9 +75,9 @@ module Beetle
       message.expects(:now).returns(1)
       message.set_timeout!
       assert_equal "2", @store.get(message.msg_id, :timeout)
-      message.expects(:now).returns(2)
+      message.expects(:now).returns(2 + Message::TIMEOUT_GRACE_PERIOD)
       assert !message.timed_out?
-      message.expects(:now).returns(3)
+      message.expects(:now).returns(3 + Message::TIMEOUT_GRACE_PERIOD)
       assert message.timed_out?
     end
 
@@ -86,9 +86,9 @@ module Beetle
       message.expects(:now).returns(0)
       message.set_timeout!
       assert_equal "#{Message::DEFAULT_HANDLER_TIMEOUT}", @store.get(message.msg_id, :timeout)
-      message.expects(:now).returns(message.timeout)
+      message.expects(:now).returns(message.timeout + Message::TIMEOUT_GRACE_PERIOD)
       assert !message.timed_out?
-      message.expects(:now).returns(Message::DEFAULT_HANDLER_TIMEOUT + 1)
+      message.expects(:now).returns(message.timeout + Message::TIMEOUT_GRACE_PERIOD + 1)
       assert message.timed_out?
     end
 

data/test/beetle/publisher_test.rb

@@ -304,7 +304,9 @@ module Beetle
       @client.register_queue('test_queue_1', :exchange => 'test_exchange')
       @client.register_queue('test_queue_2', :exchange => 'test_exchange')
       @client.register_queue('test_queue_3', :exchange => 'test_exchange_2')
-      @pub.expects(:bind_queue!).returns(1).times(3)
+      queue = mock("queue")
+      queue.expects(:bind).times(3)
+      @pub.expects(:declare_queue!).returns(queue).times(3)
       @pub.send(:bind_queues_for_exchange, 'test_exchange')
       @pub.send(:bind_queues_for_exchange, 'test_exchange_2')
     end
@@ -312,8 +314,19 @@ module Beetle
     test "should not rebind the defined queues for the used exchanges if they already have been bound" do
       @client.register_queue('test_queue_1', :exchange => 'test_exchange')
       @client.register_queue('test_queue_2', :exchange => 'test_exchange')
-      @pub.expects(:bind_queue!).twice
+      queue = mock("queue")
+      queue.expects(:bind).twice
+      @pub.expects(:declare_queue!).returns(queue).twice
+      @pub.send(:bind_queues_for_exchange, 'test_exchange')
       @pub.send(:bind_queues_for_exchange, 'test_exchange')
+    end
+
+    test "should declare queues only once even with many bindings" do
+      @client.register_queue('test_queue', :exchange => 'test_exchange')
+      @client.register_binding('test_queue', :exchange => 'test_exchange', :key => 'sir-message-a-lot')
+      queue = mock("queue")
+      queue.expects(:bind).twice
+      @pub.expects(:declare_queue!).returns(queue).once
       @pub.send(:bind_queues_for_exchange, 'test_exchange')
     end
 
@@ -342,14 +355,34 @@ module Beetle
     end
 
     test "setting up queues and policies should iterate over all servers" do
-      @pub.servers = %w(a b)
-      queue = mock("queue")
+      client = Client.new
+      client.register_queue("queue")
+      pub = Publisher.new(client)
+      pub.servers = %w(a b)
+
       s = sequence("setup")
-      @pub.expects(:set_current_server).with("a").in_sequence(s)
-      @pub.expects(:queue).with("queue", :create_policies => true).returns(queue).in_sequence(s)
-      @pub.expects(:set_current_server).with("b").in_sequence(s)
-      @pub.expects(:queue).with("queue", :create_policies => true).returns(queue).in_sequence(s)
-      @pub.setup_queues_and_policies(["queue"])
+      pub.expects(:set_current_server).with("a").in_sequence(s)
+      pub.expects(:queue).with(client.config.beetle_policy_updates_queue_name).in_sequence(s)
+      pub.expects(:queue).with("queue").in_sequence(s)
+      pub.expects(:set_current_server).with("b").in_sequence(s)
+      pub.expects(:queue).with(client.config.beetle_policy_updates_queue_name).in_sequence(s)
+      pub.expects(:queue).with("queue").in_sequence(s)
+
+      pub.setup_queues_and_policies()
+    end
+
+    test "setting up queues and policies should handle ephemeral errors" do
+      client = Client.new
+      pub = Publisher.new(client)
+      client.register_queue("queue")
+      pub.servers = %w(a b)
+      pub.stubs(:queue).raises(StandardError)
+
+      s = sequence("setup")
+      pub.expects(:set_current_server).with("a").in_sequence(s)
+      pub.expects(:set_current_server).with("b").in_sequence(s)
+
+      pub.setup_queues_and_policies()
     end
 
     test "reports whether it has been throttled" do

data/test/beetle/subscriber_test.rb

@@ -127,7 +127,7 @@ module Beetle
       @sub = @client.send(:subscriber)
     end
 
-    test "subscribers server list should contain addtional subcription hosts" do
+    test "subscribers server list should contain additional subcription hosts" do
       assert_equal ["localhost:5672", "localhost:1234"], @sub.servers
     end
   end
@@ -168,6 +168,17 @@ module Beetle
       @sub.send(:bind_queues, %W(x y))
     end
 
+    test "binding queues with many bindings should create it only once" do
+      @client.register_queue(:x, :exchange => 'test_exchange')
+      @client.register_binding(:x, :exchange => 'test_exchange', :key => 'sir-message-a-lot')
+      @client.register_handler(%w(x)){}
+      @sub.stubs(:exchange)
+      queue = mock("queue")
+      queue.expects(:bind).twice
+      @sub.expects(:declare_queue!).returns(queue).once
+      @sub.send(:bind_queues, %W(x))
+    end
+
     test "subscribing to queues should subscribe on all queues" do
       @client.register_queue(:x)
       @client.register_queue(:y)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 3.5.1
 platform: ruby
 authors:
 - Stefan Kaes
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-21 00:00:00.000000000 Z
+date: 2021-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -62,14 +62,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.3.2
 - !ruby/object:Gem::Dependency
   name: amqp
   requirement: !ruby/object:Gem::Requirement
@@ -337,7 +337,10 @@ files:
 - lib/beetle/redis_ext.rb
 - lib/beetle/subscriber.rb
 - lib/beetle/version.rb
+- script/analyze-expiries
+- script/analyze-redis-keys
 - script/console
+- script/expire-keys
 - script/start_rabbit
 - test/beetle/amqp_gem_behavior_test.rb
 - test/beetle/base_test.rb
@@ -375,7 +378,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.7
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.2.16
 signing_key:
 specification_version: 3
 summary: High Availability AMQP Messaging with Redundant Queues