beetle 3.3.12 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 61a15a00cb56b621b261ae91e7a2252e236f7264483e6fe08c43df35728f3af3
-  data.tar.gz: 89a3f62aae66f38c17af84816cd4057fe7a74ca1744b72cc9871354e6f17ffc2
+  metadata.gz: cb4bee36ae48b68aa27483c59aea588aa1a263c7c33858738ce6471246e1bb96
+  data.tar.gz: 93f380bb4596fcabfc48bec927ba7767b04b826750c307934ed937023c7a8152
 SHA512:
-  metadata.gz: 24786199730c4035f0dc4650443689482bb1899b478a96422cb4d001aef001ff43f2c8f22950ebe0a9a2ef2c26ae7fd7ae7cca5aa1fc9bc0b3d3de0d1016101d
-  data.tar.gz: 03a4187b2080bf994280b93f327ae239c2d9ed81ff076e33db604b9ee0954dfa78f230cea963e6e8d2f4ae2af0794b964a62fd6242a206ad525c5a1ac7862125
+  metadata.gz: 34d0e2001a1f93f8b88a86992a77bda65acddc1913c550798144c62fc11b77b7da0dcc0b0c270edd38aa7d27b7ea5f1d0f03fe2a955154db0f91759f5aaa2aee
+  data.tar.gz: 7ee1364e4fc0e6c6d8cefb1f10c5d4baa0fa6c7a1bd1434a710169538d104b4b83016faa3fb3bb6616bbae1c1db668953f24acd9c342d481b20b6e5d41c967de

data/RELEASE_NOTES.rdoc

@@ -1,5 +1,24 @@
 = Release Notes
 
+== Version 3.5.0
+* expose publisher method to setup queues/policies ahead of use
+
+== Version 3.4.3
+* optimize declaration of queues with many bindings
+
+== Version 3.4.2
+* Updated amq-protocol gem to version 2.3.2.
+* Fixed a rare race condition on message handler timeouts.
+
+== Version 3.4.1
+* Updated amq-protocol gem to version 2.3.1.
+
+== Version 3.4.0
+* Require redis gem version 4.2.1. This version changes the exists to check for the
+  existence of multiple keys, return the number of keys in the list that exist. This
+  requires at least redis gem version 4.2.0, but 4.2.1 contains a bug fix for said
+  command.
+
 == Version 3.3.12
 * Support queue level declaration of dead letter queue message TTL.
 

data/beetle.gemspec

@@ -24,9 +24,9 @@ Gem::Specification.new do |s|
 
   s.specification_version = 3
   s.add_runtime_dependency "bunny",                   "~> 0.7.12"
-  s.add_runtime_dependency "redis",                   ">= 2.2.2"
+  s.add_runtime_dependency "redis",                   ">= 4.2.1"
   s.add_runtime_dependency "hiredis",                 ">= 0.4.5"
-  s.add_runtime_dependency "amq-protocol",            "= 2.3.0"
+  s.add_runtime_dependency "amq-protocol",            "= 2.3.2"
   s.add_runtime_dependency "amqp",                    "= 1.8.0"
   s.add_runtime_dependency "activesupport",           ">= 2.3.4"
 

data/lib/beetle/base.rb

@@ -67,23 +67,25 @@ module Beetle
           logger.debug("Beetle: binding queue #{name} with internal name #{opts[:amqp_name]} on server #{@server}")
           queue_name = opts[:amqp_name]
           creation_options = opts.slice(*QUEUE_CREATION_KEYS)
-          the_queue = nil
+
+          the_queue = declare_queue!(queue_name, creation_options)
           @client.bindings[name].each do |binding_options|
             exchange_name = binding_options[:exchange]
             binding_options = binding_options.slice(*QUEUE_BINDING_KEYS)
-            the_queue = bind_queue!(queue_name, creation_options, exchange_name, binding_options)
+            logger.debug("Beetle: binding queue #{queue_name} to #{exchange_name} with opts: #{binding_options.inspect}")
+            bind_queue!(the_queue, exchange_name, binding_options)
           end
           the_queue
         end
     end
 
-    def bind_dead_letter_queue!(channel, target_queue, creation_keys = {})
+    def bind_dead_letter_queue!(channel, target_queue, creation_options = {})
       policy_options = @client.queues[target_queue].slice(:dead_lettering, :lazy, :dead_lettering_msg_ttl)
       policy_options[:message_ttl] = policy_options.delete(:dead_lettering_msg_ttl)
       dead_letter_queue_name = "#{target_queue}_dead_letter"
       if policy_options[:dead_lettering]
-        logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_keys.inspect}")
-        channel.queue(dead_letter_queue_name, creation_keys)
+        logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_options.inspect}")
+        channel.queue(dead_letter_queue_name, creation_options)
       end
       return {
         :queue_name => target_queue,
@@ -93,7 +95,7 @@ module Beetle
       }.merge(policy_options)
     end
 
-    # called by <tt>bind_queue!</tt>
+    # called by <tt>declare_queue!</tt>
     def publish_policy_options(options)
       # avoid endless recursion
       return if options[:queue_name] == @client.config.beetle_policy_updates_queue_name

data/lib/beetle/client.rb

@@ -280,6 +280,13 @@ module Beetle
       publisher.throttled?
     end
 
+    # set up queues and policies for all configured queues. Otherwise this will
+    # happen on first use of an exchange, which can be undesired for latency
+    # sensitive endpoints. Only needs to be called once.
+    def setup_queues_and_policies
+      publisher.setup_queues_and_policies
+    end
+
     # traces queues without consuming them. useful for debugging message flow.
     def trace(queue_names=self.queues.keys, tracer=nil, &block)
       queues_to_trace = self.queues.slice(*queue_names)

data/lib/beetle/deduplication_store.rb

@@ -120,7 +120,7 @@ module Beetle
 
     # check whether key with given suffix exists for a given <tt>msg_id</tt>.
     def exists(msg_id, suffix)
-      with_failover { redis.exists(key(msg_id, suffix)) }
+      with_failover { redis.exists?(key(msg_id, suffix)) }
     end
 
     # flush the configured redis database. useful for testing.

data/lib/beetle/message.rb

@@ -18,6 +18,8 @@ module Beetle
     # forcefully abort a running handler after this many seconds.
     # can be overriden when registering a handler.
     DEFAULT_HANDLER_TIMEOUT = 600.seconds
+    # How much extra time on top of the handler timeout we add before considering a handler timed out
+    TIMEOUT_GRACE_PERIOD = 10.seconds
     # how many times we should try to run a handler before giving up
     DEFAULT_HANDLER_EXECUTION_ATTEMPTS = 1
     # how many seconds we should wait before retrying handler execution
@@ -167,8 +169,8 @@ module Beetle
     end
 
     # handler timed out?
-    def timed_out?
-      (t = @store.get(msg_id, :timeout)) && t.to_i < now
+    def timed_out?(t = nil)
+      (t ||= @store.get(msg_id, :timeout)) && (t.to_i + TIMEOUT_GRACE_PERIOD) < now
     end
 
     # reset handler timeout in the deduplication store
@@ -187,8 +189,8 @@ module Beetle
     end
 
     # whether we should wait before running the handler
-    def delayed?
-      (t = @store.get(msg_id, :delay)) && t.to_i > now
+    def delayed?(t = nil)
+      (t ||= @store.get(msg_id, :delay)) && t.to_i > now
     end
 
     # store delay value in the deduplication store
@@ -207,8 +209,8 @@ module Beetle
     end
 
     # whether we have already tried running the handler as often as specified when the handler was registered
-    def attempts_limit_reached?
-      (limit = @store.get(msg_id, :attempts)) && limit.to_i >= attempts_limit
+    def attempts_limit_reached?(attempts = nil)
+      (attempts ||= @store.get(msg_id, :attempts)) && attempts.to_i >= attempts_limit
     end
 
     # increment number of exception occurences in the deduplication store
@@ -217,8 +219,8 @@ module Beetle
     end
 
     # whether the number of exceptions has exceeded the limit set when the handler was registered
-    def exceptions_limit_reached?
-      @store.get(msg_id, :exceptions).to_i > exceptions_limit
+    def exceptions_limit_reached?(exceptions = nil)
+      (exceptions ||= @store.get(msg_id, :exceptions)) && exceptions.to_i > exceptions_limit
     end
 
     def exception_accepted?
@@ -306,17 +308,17 @@ module Beetle
         if status == "completed"
           ack!
           RC::OK
-        elsif delay && delay.to_i > now
+        elsif delay && delayed?(delay)
           logger.warn "Beetle: ignored delayed message (#{msg_id})!"
           RC::Delayed
-        elsif !(timeout && timeout.to_i < now)
+        elsif !(timeout && timed_out?(timeout))
           RC::HandlerNotYetTimedOut
-        elsif attempts.to_i >= attempts_limit
+        elsif attempts && attempts_limit_reached?(attempts)
           completed!
           ack!
           logger.warn "Beetle: reached the handler execution attempts limit: #{attempts_limit} on #{msg_id}"
           RC::AttemptsLimitReached
-        elsif exceptions.to_i > exceptions_limit
+        elsif exceptions && exceptions_limit_reached?(exceptions)
           completed!
           ack!
           logger.warn "Beetle: reached the handler exceptions limit: #{exceptions_limit} on #{msg_id}"

data/lib/beetle/publisher.rb

@@ -169,10 +169,14 @@ module Beetle
       end
     end
 
-    def setup_queues_and_policies(queue_names) #:nodoc:
+    def setup_queues_and_policies
       each_server do
-        queue_names.each do |name|
-          queue(name, create_policies: true)
+        begin
+          @client.queues.keys.each do |name|
+            queue(name)
+          end
+        rescue => e
+          logger.warn "Beetle: failed setting up queues and policies on #{@server}: #{e}"
         end
       end
     end
@@ -247,17 +251,19 @@ module Beetle
       @exchanges_with_bound_queues[exchange_name] = true
     end
 
-    # TODO: Refactor, fetch the keys and stuff itself
-    def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
-      logger.debug("Beetle: creating queue with opts: #{creation_keys.inspect}")
-      queue = bunny.queue(queue_name, creation_keys)
-      logger.debug("Beetle: binding queue #{queue_name} to #{exchange_name} with opts: #{binding_keys.inspect}")
-      queue.bind(exchange(exchange_name), binding_keys)
-      policy_options = bind_dead_letter_queue!(bunny, queue_name, creation_keys)
+    def declare_queue!(queue_name, creation_options)
+      logger.debug("Beetle: creating queue with opts: #{creation_options.inspect}")
+      queue = bunny.queue(queue_name, creation_options)
+
+      policy_options = bind_dead_letter_queue!(bunny, queue_name, creation_options)
       publish_policy_options(policy_options)
       queue
     end
 
+    def bind_queue!(queue, exchange_name, binding_options)
+      queue.bind(exchange(exchange_name), binding_options)
+    end
+
     def stop!(exception=nil)
       return unless bunny?
       timeout = @client.config.publishing_timeout + @client.config.publisher_connect_timeout + 1

data/lib/beetle/subscriber.rb

@@ -214,18 +214,20 @@ module Beetle
       channel.__send__(opts[:type], name, opts.slice(*EXCHANGE_CREATION_KEYS))
     end
 
-    def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
-      queue = channel.queue(queue_name, creation_keys)
+    def declare_queue!(queue_name, creation_options)
+      queue = channel.queue(queue_name, creation_options)
       unless tracing?
         # we don't want to create dead-letter queues for tracing
-        policy_options = bind_dead_letter_queue!(channel, queue_name, creation_keys)
+        policy_options = bind_dead_letter_queue!(channel, queue_name, creation_options)
         publish_policy_options(policy_options)
       end
-      exchange = exchange(exchange_name)
-      queue.bind(exchange, binding_keys)
       queue
     end
 
+    def bind_queue!(queue, exchange_name, binding_options)
+      queue.bind(exchange(exchange_name), binding_options)
+    end
+
     def connection_settings
       {
         :host => current_host, :port => current_port, :logging => false,

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "3.3.12"
+  VERSION = "3.5.0"
 end

data/script/analyze-expiries

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'time'
+
+counts = Hash.new(0)
+expiries = Hash.new{|h,k| h[k] = Hash.new(0)}
+t = Time.now.to_i
+
+File.open(ARGV[0]).each_line do |l|
+  parts = l.split(':')
+  queue = parts[1]
+  counts[queue] += 1
+  expiry = parts[4].to_i
+  expires_in = ((expiry - t)/(3600.0)).ceil
+  expiries[queue][expires_in] += 1
+end
+
+counts.to_a.sort_by{|_,v| -v}.each do |q,v|
+  puts "------------------------------------------------------------------"
+  puts "#{q}: #{v}"
+  puts "------------------------------------------------------------------"
+  expiries[q].to_a.sort_by{|k,_| -k}.each do |expiry, count|
+    printf "%3dh: %6d\n", expiry, count
+  end
+end

data/script/analyze-redis-keys

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+queue_counts = Hash.new { |h,k| h[k] = 0 }
+File.open(ARGV[0]).each_line do |l|
+  next if l == "lastgc" || l == "clients-last-seen"
+  a = l.split(':')
+  if a[0] == "msgid"
+    queue_counts[a[1]] += 1
+  else
+    queue_counts["none"] += 1
+  end
+end
+sorted_queues = queue_counts.to_a.sort_by{|a| -a[1]}
+sorted_queues.each do |q,c|
+  puts "#{c}:#{q}"
+end

data/script/expire-keys

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+require "redis"
+
+redis = Redis.new(:host => "beetle-1.redis.ams2.xing.com", :port => 6379, :db => 0)
+
+File.open(ARGV[0]).each_line.each_slice(10) do |s|
+  redis.pipelined do
+    s.each do |l|
+      l.chomp!
+      next if l =~ /^beetle:.*$/
+      redis.expire(l, 600)
+    end
+  end
+end

data/test/beetle/client_test.rb

@@ -282,6 +282,12 @@ module Beetle
       client.stop_publishing
     end
 
+    test "should delegate setup_queues_and_policies to the publisher instance" do
+      client = Client.new
+      client.send(:publisher).expects(:setup_queues_and_policies)
+      client.setup_queues_and_policies
+    end
+
     test "should delegate queue purging to the publisher instance" do
       client = Client.new
       client.register_queue(:queue)

data/test/beetle/deduplication_store_test.rb

@@ -9,7 +9,7 @@ module Beetle
     end
 
     test "trying to delete a non existent key doesn't throw an error" do
-      assert !@r.exists("hahahaha")
+      assert !@r.exists?("hahahaha")
       assert_equal 0, @r.del("hahahaha")
     end
 

data/test/beetle/message/settings_test.rb

@@ -75,9 +75,9 @@ module Beetle
       message.expects(:now).returns(1)
       message.set_timeout!
       assert_equal "2", @store.get(message.msg_id, :timeout)
-      message.expects(:now).returns(2)
+      message.expects(:now).returns(2 + Message::TIMEOUT_GRACE_PERIOD)
       assert !message.timed_out?
-      message.expects(:now).returns(3)
+      message.expects(:now).returns(3 + Message::TIMEOUT_GRACE_PERIOD)
       assert message.timed_out?
     end
 
@@ -86,9 +86,9 @@ module Beetle
       message.expects(:now).returns(0)
       message.set_timeout!
       assert_equal "#{Message::DEFAULT_HANDLER_TIMEOUT}", @store.get(message.msg_id, :timeout)
-      message.expects(:now).returns(message.timeout)
+      message.expects(:now).returns(message.timeout + Message::TIMEOUT_GRACE_PERIOD)
       assert !message.timed_out?
-      message.expects(:now).returns(Message::DEFAULT_HANDLER_TIMEOUT + 1)
+      message.expects(:now).returns(message.timeout + Message::TIMEOUT_GRACE_PERIOD + 1)
       assert message.timed_out?
     end
 

data/test/beetle/message_test.rb

@@ -141,9 +141,7 @@ module Beetle
 
       message.process(@null_handler)
       keys = @store.keys(message.msg_id)
-      keys.each do |key|
-        assert !@store.redis.exists(key)
-      end
+      assert_equal 0, @store.redis.exists(*keys)
     end
 
     test "successful processing of a non redundant message should delete all keys from the database (except the status key, which should be set to expire)" do
@@ -159,11 +157,9 @@ module Beetle
       message.process(@null_handler)
       keys = @store.keys(message.msg_id)
       status_key = keys.shift
-      assert @store.redis.exists(status_key)
+      assert @store.redis.exists?(status_key)
       assert @store.redis.ttl(status_key) <= @config.redis_status_key_expiry_interval
-      keys.each do |key|
-        assert !@store.redis.exists(key)
-      end
+      assert_equal 0, @store.redis.exists(*keys)
     end
 
     test "successful processing of a redundant message twice should delete all keys from the database" do
@@ -179,9 +175,7 @@ module Beetle
       message.process(@null_handler)
 
       keys = @store.keys(message.msg_id)
-      keys.each do |key|
-        assert !@store.redis.exists(key)
-      end
+      assert_equal 0, @store.redis.exists(*keys)
     end
 
     test "successful processing of a redundant message twice should delete all keys from the database (except the status key, which should be set to expire)" do
@@ -199,11 +193,9 @@ module Beetle
 
       keys = @store.keys(message.msg_id)
       status_key = keys.shift
-      assert @store.redis.exists(status_key)
+      assert @store.redis.exists?(status_key)
       assert @store.redis.ttl(status_key) <= @config.redis_status_key_expiry_interval
-      keys.each do |key|
-        assert !@store.redis.exists(key)
-      end
+      assert_equal 0, @store.redis.exists(*keys)
     end
 
     test "successful processing of a redundant message once should insert all but the delay key and the exception count key into the database" do

data/test/beetle/publisher_test.rb

@@ -304,7 +304,9 @@ module Beetle
       @client.register_queue('test_queue_1', :exchange => 'test_exchange')
       @client.register_queue('test_queue_2', :exchange => 'test_exchange')
       @client.register_queue('test_queue_3', :exchange => 'test_exchange_2')
-      @pub.expects(:bind_queue!).returns(1).times(3)
+      queue = mock("queue")
+      queue.expects(:bind).times(3)
+      @pub.expects(:declare_queue!).returns(queue).times(3)
       @pub.send(:bind_queues_for_exchange, 'test_exchange')
       @pub.send(:bind_queues_for_exchange, 'test_exchange_2')
     end
@@ -312,8 +314,19 @@ module Beetle
     test "should not rebind the defined queues for the used exchanges if they already have been bound" do
       @client.register_queue('test_queue_1', :exchange => 'test_exchange')
       @client.register_queue('test_queue_2', :exchange => 'test_exchange')
-      @pub.expects(:bind_queue!).twice
+      queue = mock("queue")
+      queue.expects(:bind).twice
+      @pub.expects(:declare_queue!).returns(queue).twice
+      @pub.send(:bind_queues_for_exchange, 'test_exchange')
       @pub.send(:bind_queues_for_exchange, 'test_exchange')
+    end
+
+    test "should declare queues only once even with many bindings" do
+      @client.register_queue('test_queue', :exchange => 'test_exchange')
+      @client.register_binding('test_queue', :exchange => 'test_exchange', :key => 'sir-message-a-lot')
+      queue = mock("queue")
+      queue.expects(:bind).twice
+      @pub.expects(:declare_queue!).returns(queue).once
       @pub.send(:bind_queues_for_exchange, 'test_exchange')
     end
 
@@ -342,14 +355,34 @@ module Beetle
     end
 
     test "setting up queues and policies should iterate over all servers" do
-      @pub.servers = %w(a b)
-      queue = mock("queue")
+      client = Client.new
+      client.register_queue("queue")
+      pub = Publisher.new(client)
+      pub.servers = %w(a b)
+
       s = sequence("setup")
-      @pub.expects(:set_current_server).with("a").in_sequence(s)
-      @pub.expects(:queue).with("queue", :create_policies => true).returns(queue).in_sequence(s)
-      @pub.expects(:set_current_server).with("b").in_sequence(s)
-      @pub.expects(:queue).with("queue", :create_policies => true).returns(queue).in_sequence(s)
-      @pub.setup_queues_and_policies(["queue"])
+      pub.expects(:set_current_server).with("a").in_sequence(s)
+      pub.expects(:queue).with(client.config.beetle_policy_updates_queue_name).in_sequence(s)
+      pub.expects(:queue).with("queue").in_sequence(s)
+      pub.expects(:set_current_server).with("b").in_sequence(s)
+      pub.expects(:queue).with(client.config.beetle_policy_updates_queue_name).in_sequence(s)
+      pub.expects(:queue).with("queue").in_sequence(s)
+
+      pub.setup_queues_and_policies()
+    end
+
+    test "setting up queues and policies should handle ephemeral errors" do
+      client = Client.new
+      pub = Publisher.new(client)
+      client.register_queue("queue")
+      pub.servers = %w(a b)
+      pub.stubs(:queue).raises(StandardError)
+
+      s = sequence("setup")
+      pub.expects(:set_current_server).with("a").in_sequence(s)
+      pub.expects(:set_current_server).with("b").in_sequence(s)
+
+      pub.setup_queues_and_policies()
     end
 
     test "reports whether it has been throttled" do

data/test/beetle/subscriber_test.rb

@@ -168,6 +168,17 @@ module Beetle
       @sub.send(:bind_queues, %W(x y))
     end
 
+    test "binding queues with many bindings should create it only once" do
+      @client.register_queue(:x, :exchange => 'test_exchange')
+      @client.register_binding(:x, :exchange => 'test_exchange', :key => 'sir-message-a-lot')
+      @client.register_handler(%w(x)){}
+      @sub.stubs(:exchange)
+      queue = mock("queue")
+      queue.expects(:bind).twice
+      @sub.expects(:declare_queue!).returns(queue).once
+      @sub.send(:bind_queues, %W(x))
+    end
+
     test "subscribing to queues should subscribe on all queues" do
       @client.register_queue(:x)
       @client.register_queue(:y)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 3.3.12
+  version: 3.5.0
 platform: ruby
 authors:
 - Stefan Kaes
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-20 00:00:00.000000000 Z
+date: 2021-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -34,14 +34,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 4.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 4.2.1
 - !ruby/object:Gem::Dependency
   name: hiredis
   requirement: !ruby/object:Gem::Requirement
@@ -62,14 +62,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.3.2
 - !ruby/object:Gem::Dependency
   name: amqp
   requirement: !ruby/object:Gem::Requirement
@@ -337,7 +337,10 @@ files:
 - lib/beetle/redis_ext.rb
 - lib/beetle/subscriber.rb
 - lib/beetle/version.rb
+- script/analyze-expiries
+- script/analyze-redis-keys
 - script/console
+- script/expire-keys
 - script/start_rabbit
 - test/beetle/amqp_gem_behavior_test.rb
 - test/beetle/base_test.rb