beetle 3.3.11 → 3.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd1ee5b22d344002924b2b648e036ce98584e270b05af6195115c5bc6aef6438
-  data.tar.gz: f32ef2c465d81cb73ac8453b677419c9117e1f2ab62126feb62dd2227cc1e924
+  metadata.gz: 64be33377e4235036fac944de3c020f03fc53e426a1969b729ca0db016351cae
+  data.tar.gz: 7fde625b27bce1814ba5840b92898b7cb365f1d04b743584b59e15e6bb9d7da7
 SHA512:
-  metadata.gz: 9521152d635cb12bd96dd87a8153cc20f4d8f9471d9a3d08c2d8d786a53bad2a76b783140ce1e8481b90c2b8dd1f6a391988985380fc4efcc93c30ba01ee013f
-  data.tar.gz: c29b033bcb301006b22bcbeeb4a9e62b6d72a5bcf54093e01aee405ea2c5a8d0f65c3a0e6a69bce0cbe0047cf52780069ae6c9a7d68c53161fea5de0c8b09b27
+  metadata.gz: 990d179e32cad50f9d51ec3f3bf8e4e3d518b8f1925995277917ae5553dfd3fd72dd510558a3f2f5ddbc815b57f5a8719f3854b18b2680a620f76dda1bb411b5
+  data.tar.gz: 42e77ac9b05035ec28891cdf43eb8ba0706e835a52d4c1906843a7d9f1cf8a9a211c332c2d031c5b044b2f4cebbf40ca94a8dd20eda99abc0af0c8cf9ce9ce29

data/RELEASE_NOTES.rdoc

@@ -1,5 +1,24 @@
 = Release Notes
 
+== Version 3.4.3
+* optimize declaration of queues with many bindings
+
+== Version 3.4.2
+* Updated amq-protocol gem to version 2.3.2.
+* Fixed a rare race condition on message handler timeouts.
+
+== Version 3.4.1
+* Updated amq-protocol gem to version 2.3.1.
+
+== Version 3.4.0
+* Require redis gem version 4.2.1. This version changes the exists to check for the
+  existence of multiple keys, return the number of keys in the list that exist. This
+  requires at least redis gem version 4.2.0, but 4.2.1 contains a bug fix for said
+  command.
+
+== Version 3.3.12
+* Support queue level declaration of dead letter queue message TTL.
+
 == Version 3.3.11
 * Fixed that dead lettering only works correctly with global config option.
 

data/beetle.gemspec

@@ -24,9 +24,9 @@ Gem::Specification.new do |s|
 
   s.specification_version = 3
   s.add_runtime_dependency "bunny",                   "~> 0.7.12"
-  s.add_runtime_dependency "redis",                   ">= 2.2.2"
+  s.add_runtime_dependency "redis",                   ">= 4.2.1"
   s.add_runtime_dependency "hiredis",                 ">= 0.4.5"
-  s.add_runtime_dependency "amq-protocol",            "= 2.3.0"
+  s.add_runtime_dependency "amq-protocol",            "= 2.3.2"
   s.add_runtime_dependency "amqp",                    "= 1.8.0"
   s.add_runtime_dependency "activesupport",           ">= 2.3.4"
 

data/lib/beetle/base.rb

@@ -67,32 +67,35 @@ module Beetle
           logger.debug("Beetle: binding queue #{name} with internal name #{opts[:amqp_name]} on server #{@server}")
           queue_name = opts[:amqp_name]
           creation_options = opts.slice(*QUEUE_CREATION_KEYS)
-          the_queue = nil
+
+          the_queue = declare_queue!(queue_name, creation_options)
           @client.bindings[name].each do |binding_options|
             exchange_name = binding_options[:exchange]
             binding_options = binding_options.slice(*QUEUE_BINDING_KEYS)
-            the_queue = bind_queue!(queue_name, creation_options, exchange_name, binding_options)
+            logger.debug("Beetle: binding queue #{queue_name} to #{exchange_name} with opts: #{binding_options.inspect}")
+            bind_queue!(the_queue, exchange_name, binding_options)
           end
           the_queue
         end
     end
 
-    def bind_dead_letter_queue!(channel, target_queue, creation_keys = {})
-      policy_options = @client.queues[target_queue].slice(:dead_lettering, :lazy)
+    def bind_dead_letter_queue!(channel, target_queue, creation_options = {})
+      policy_options = @client.queues[target_queue].slice(:dead_lettering, :lazy, :dead_lettering_msg_ttl)
+      policy_options[:message_ttl] = policy_options.delete(:dead_lettering_msg_ttl)
       dead_letter_queue_name = "#{target_queue}_dead_letter"
       if policy_options[:dead_lettering]
-        logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_keys.inspect}")
-        channel.queue(dead_letter_queue_name, creation_keys)
+        logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_options.inspect}")
+        channel.queue(dead_letter_queue_name, creation_options)
       end
       return {
         :queue_name => target_queue,
         :bindings => @client.bindings[target_queue],
         :dead_letter_queue_name => dead_letter_queue_name,
-        :message_ttl => @client.config.dead_lettering_msg_ttl,
+        :message_ttl => policy_options[:message_ttl]
       }.merge(policy_options)
     end
 
-    # called by <tt>bind_queue!</tt>
+    # called by <tt>declare_queue!</tt>
     def publish_policy_options(options)
       # avoid endless recursion
       return if options[:queue_name] == @client.config.beetle_policy_updates_queue_name

data/lib/beetle/client.rb

@@ -96,7 +96,8 @@ module Beetle
       raise ConfigurationError.new("queue #{name} already configured") if queues.include?(name)
       opts = {
         :exchange => name, :key => name, :auto_delete => false, :amqp_name => name,
-        :lazy => config.lazy_queues_enabled, :dead_lettering => config.dead_lettering_enabled
+        :lazy => config.lazy_queues_enabled, :dead_lettering => config.dead_lettering_enabled,
+        :dead_lettering_msg_ttl => config.dead_lettering_msg_ttl
       }.merge!(options.symbolize_keys)
       opts.merge! :durable => true, :passive => false, :exclusive => false
       exchange = opts.delete(:exchange).to_s

data/lib/beetle/deduplication_store.rb

@@ -120,7 +120,7 @@ module Beetle
 
     # check whether key with given suffix exists for a given <tt>msg_id</tt>.
     def exists(msg_id, suffix)
-      with_failover { redis.exists(key(msg_id, suffix)) }
+      with_failover { redis.exists?(key(msg_id, suffix)) }
     end
 
     # flush the configured redis database. useful for testing.

data/lib/beetle/message.rb

@@ -18,6 +18,8 @@ module Beetle
     # forcefully abort a running handler after this many seconds.
     # can be overriden when registering a handler.
     DEFAULT_HANDLER_TIMEOUT = 600.seconds
+    # How much extra time on top of the handler timeout we add before considering a handler timed out
+    TIMEOUT_GRACE_PERIOD = 10.seconds
     # how many times we should try to run a handler before giving up
     DEFAULT_HANDLER_EXECUTION_ATTEMPTS = 1
     # how many seconds we should wait before retrying handler execution
@@ -167,8 +169,8 @@ module Beetle
     end
 
     # handler timed out?
-    def timed_out?
-      (t = @store.get(msg_id, :timeout)) && t.to_i < now
+    def timed_out?(t = nil)
+      (t ||= @store.get(msg_id, :timeout)) && (t.to_i + TIMEOUT_GRACE_PERIOD) < now
     end
 
     # reset handler timeout in the deduplication store
@@ -187,8 +189,8 @@ module Beetle
     end
 
     # whether we should wait before running the handler
-    def delayed?
-      (t = @store.get(msg_id, :delay)) && t.to_i > now
+    def delayed?(t = nil)
+      (t ||= @store.get(msg_id, :delay)) && t.to_i > now
     end
 
     # store delay value in the deduplication store
@@ -207,8 +209,8 @@ module Beetle
     end
 
     # whether we have already tried running the handler as often as specified when the handler was registered
-    def attempts_limit_reached?
-      (limit = @store.get(msg_id, :attempts)) && limit.to_i >= attempts_limit
+    def attempts_limit_reached?(attempts = nil)
+      (attempts ||= @store.get(msg_id, :attempts)) && attempts.to_i >= attempts_limit
     end
 
     # increment number of exception occurences in the deduplication store
@@ -217,8 +219,8 @@ module Beetle
     end
 
     # whether the number of exceptions has exceeded the limit set when the handler was registered
-    def exceptions_limit_reached?
-      @store.get(msg_id, :exceptions).to_i > exceptions_limit
+    def exceptions_limit_reached?(exceptions = nil)
+      (exceptions ||= @store.get(msg_id, :exceptions)) && exceptions.to_i > exceptions_limit
     end
 
     def exception_accepted?
@@ -306,17 +308,17 @@ module Beetle
         if status == "completed"
           ack!
           RC::OK
-        elsif delay && delay.to_i > now
+        elsif delay && delayed?(delay)
           logger.warn "Beetle: ignored delayed message (#{msg_id})!"
           RC::Delayed
-        elsif !(timeout && timeout.to_i < now)
+        elsif !(timeout && timed_out?(timeout))
           RC::HandlerNotYetTimedOut
-        elsif attempts.to_i >= attempts_limit
+        elsif attempts && attempts_limit_reached?(attempts)
           completed!
           ack!
           logger.warn "Beetle: reached the handler execution attempts limit: #{attempts_limit} on #{msg_id}"
           RC::AttemptsLimitReached
-        elsif exceptions.to_i > exceptions_limit
+        elsif exceptions && exceptions_limit_reached?(exceptions)
           completed!
           ack!
           logger.warn "Beetle: reached the handler exceptions limit: #{exceptions_limit} on #{msg_id}"

data/lib/beetle/publisher.rb

@@ -247,17 +247,19 @@ module Beetle
       @exchanges_with_bound_queues[exchange_name] = true
     end
 
-    # TODO: Refactor, fetch the keys and stuff itself
-    def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
-      logger.debug("Beetle: creating queue with opts: #{creation_keys.inspect}")
-      queue = bunny.queue(queue_name, creation_keys)
-      logger.debug("Beetle: binding queue #{queue_name} to #{exchange_name} with opts: #{binding_keys.inspect}")
-      queue.bind(exchange(exchange_name), binding_keys)
-      policy_options = bind_dead_letter_queue!(bunny, queue_name, creation_keys)
+    def declare_queue!(queue_name, creation_options)
+      logger.debug("Beetle: creating queue with opts: #{creation_options.inspect}")
+      queue = bunny.queue(queue_name, creation_options)
+
+      policy_options = bind_dead_letter_queue!(bunny, queue_name, creation_options)
       publish_policy_options(policy_options)
       queue
     end
 
+    def bind_queue!(queue, exchange_name, binding_options)
+      queue.bind(exchange(exchange_name), binding_options)
+    end
+
     def stop!(exception=nil)
       return unless bunny?
       timeout = @client.config.publishing_timeout + @client.config.publisher_connect_timeout + 1

data/lib/beetle/subscriber.rb

@@ -214,18 +214,20 @@ module Beetle
       channel.__send__(opts[:type], name, opts.slice(*EXCHANGE_CREATION_KEYS))
     end
 
-    def bind_queue!(queue_name, creation_keys, exchange_name, binding_keys)
-      queue = channel.queue(queue_name, creation_keys)
+    def declare_queue!(queue_name, creation_options)
+      queue = channel.queue(queue_name, creation_options)
       unless tracing?
         # we don't want to create dead-letter queues for tracing
-        policy_options = bind_dead_letter_queue!(channel, queue_name, creation_keys)
+        policy_options = bind_dead_letter_queue!(channel, queue_name, creation_options)
         publish_policy_options(policy_options)
       end
-      exchange = exchange(exchange_name)
-      queue.bind(exchange, binding_keys)
       queue
     end
 
+    def bind_queue!(queue, exchange_name, binding_options)
+      queue.bind(exchange(exchange_name), binding_options)
+    end
+
     def connection_settings
       {
         :host => current_host, :port => current_port, :logging => false,

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "3.3.11"
+  VERSION = "3.4.3"
 end

data/script/analyze-expiries

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'time'
+
+counts = Hash.new(0)
+expiries = Hash.new{|h,k| h[k] = Hash.new(0)}
+t = Time.now.to_i
+
+File.open(ARGV[0]).each_line do |l|
+  parts = l.split(':')
+  queue = parts[1]
+  counts[queue] += 1
+  expiry = parts[4].to_i
+  expires_in = ((expiry - t)/(3600.0)).ceil
+  expiries[queue][expires_in] += 1
+end
+
+counts.to_a.sort_by{|_,v| -v}.each do |q,v|
+  puts "------------------------------------------------------------------"
+  puts "#{q}: #{v}"
+  puts "------------------------------------------------------------------"
+  expiries[q].to_a.sort_by{|k,_| -k}.each do |expiry, count|
+    printf "%3dh: %6d\n", expiry, count
+  end
+end

data/script/analyze-redis-keys

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+queue_counts = Hash.new { |h,k| h[k] = 0 }
+File.open(ARGV[0]).each_line do |l|
+  next if l == "lastgc" || l == "clients-last-seen"
+  a = l.split(':')
+  if a[0] == "msgid"
+    queue_counts[a[1]] += 1
+  else
+    queue_counts["none"] += 1
+  end
+end
+sorted_queues = queue_counts.to_a.sort_by{|a| -a[1]}
+sorted_queues.each do |q,c|
+  puts "#{c}:#{q}"
+end

data/script/expire-keys

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+require "redis"
+
+redis = Redis.new(:host => "beetle-1.redis.ams2.xing.com", :port => 6379, :db => 0)
+
+File.open(ARGV[0]).each_line.each_slice(10) do |s|
+  redis.pipelined do
+    s.each do |l|
+      l.chomp!
+      next if l =~ /^beetle:.*$/
+      redis.expire(l, 600)
+    end
+  end
+end

data/test/beetle/client_test.rb

@@ -79,7 +79,7 @@ module Beetle
 
     test "registering a queue should store it in the configuration with symbolized option keys and force durable=true and passive=false and set the amqp queue name" do
       @client.register_queue("some_queue", "durable" => false, "exchange" => "some_exchange")
-      assert_equal({:durable => true, :passive => false, :lazy=>false, :dead_lettering=>false, :auto_delete => false, :exclusive => false, :amqp_name => "some_queue"}, @client.queues["some_queue"])
+      assert_equal({:durable => true, :passive => false, :lazy=>false, :dead_lettering=>false, :dead_lettering_msg_ttl=>1000, :auto_delete => false, :exclusive => false, :amqp_name => "some_queue"}, @client.queues["some_queue"])
     end
 
     test "registering a queue should add the queue to the list of queues of the queue's exchange" do

data/test/beetle/deduplication_store_test.rb

@@ -9,7 +9,7 @@ module Beetle
     end
 
     test "trying to delete a non existent key doesn't throw an error" do
-      assert !@r.exists("hahahaha")
+      assert !@r.exists?("hahahaha")
       assert_equal 0, @r.del("hahahaha")
     end
 

data/test/beetle/message/settings_test.rb

@@ -75,9 +75,9 @@ module Beetle
       message.expects(:now).returns(1)
       message.set_timeout!
       assert_equal "2", @store.get(message.msg_id, :timeout)
-      message.expects(:now).returns(2)
+      message.expects(:now).returns(2 + Message::TIMEOUT_GRACE_PERIOD)
       assert !message.timed_out?
-      message.expects(:now).returns(3)
+      message.expects(:now).returns(3 + Message::TIMEOUT_GRACE_PERIOD)
       assert message.timed_out?
     end
 
@@ -86,9 +86,9 @@ module Beetle
       message.expects(:now).returns(0)
       message.set_timeout!
       assert_equal "#{Message::DEFAULT_HANDLER_TIMEOUT}", @store.get(message.msg_id, :timeout)
-      message.expects(:now).returns(message.timeout)
+      message.expects(:now).returns(message.timeout + Message::TIMEOUT_GRACE_PERIOD)
       assert !message.timed_out?
-      message.expects(:now).returns(Message::DEFAULT_HANDLER_TIMEOUT + 1)
+      message.expects(:now).returns(message.timeout + Message::TIMEOUT_GRACE_PERIOD + 1)
       assert message.timed_out?
     end
 

data/test/beetle/message_test.rb

@@ -141,9 +141,7 @@ module Beetle
 
       message.process(@null_handler)
       keys = @store.keys(message.msg_id)
-      keys.each do |key|
-        assert !@store.redis.exists(key)
-      end
+      assert_equal 0, @store.redis.exists(*keys)
     end
 
     test "successful processing of a non redundant message should delete all keys from the database (except the status key, which should be set to expire)" do
@@ -159,11 +157,9 @@ module Beetle
       message.process(@null_handler)
       keys = @store.keys(message.msg_id)
       status_key = keys.shift
-      assert @store.redis.exists(status_key)
+      assert @store.redis.exists?(status_key)
       assert @store.redis.ttl(status_key) <= @config.redis_status_key_expiry_interval
-      keys.each do |key|
-        assert !@store.redis.exists(key)
-      end
+      assert_equal 0, @store.redis.exists(*keys)
     end
 
     test "successful processing of a redundant message twice should delete all keys from the database" do
@@ -179,9 +175,7 @@ module Beetle
       message.process(@null_handler)
 
       keys = @store.keys(message.msg_id)
-      keys.each do |key|
-        assert !@store.redis.exists(key)
-      end
+      assert_equal 0, @store.redis.exists(*keys)
     end
 
     test "successful processing of a redundant message twice should delete all keys from the database (except the status key, which should be set to expire)" do
@@ -199,11 +193,9 @@ module Beetle
 
       keys = @store.keys(message.msg_id)
       status_key = keys.shift
-      assert @store.redis.exists(status_key)
+      assert @store.redis.exists?(status_key)
       assert @store.redis.ttl(status_key) <= @config.redis_status_key_expiry_interval
-      keys.each do |key|
-        assert !@store.redis.exists(key)
-      end
+      assert_equal 0, @store.redis.exists(*keys)
     end
 
     test "successful processing of a redundant message once should insert all but the delay key and the exception count key into the database" do

data/test/beetle/publisher_test.rb

@@ -304,7 +304,9 @@ module Beetle
       @client.register_queue('test_queue_1', :exchange => 'test_exchange')
       @client.register_queue('test_queue_2', :exchange => 'test_exchange')
       @client.register_queue('test_queue_3', :exchange => 'test_exchange_2')
-      @pub.expects(:bind_queue!).returns(1).times(3)
+      queue = mock("queue")
+      queue.expects(:bind).times(3)
+      @pub.expects(:declare_queue!).returns(queue).times(3)
       @pub.send(:bind_queues_for_exchange, 'test_exchange')
       @pub.send(:bind_queues_for_exchange, 'test_exchange_2')
     end
@@ -312,8 +314,19 @@ module Beetle
     test "should not rebind the defined queues for the used exchanges if they already have been bound" do
       @client.register_queue('test_queue_1', :exchange => 'test_exchange')
       @client.register_queue('test_queue_2', :exchange => 'test_exchange')
-      @pub.expects(:bind_queue!).twice
+      queue = mock("queue")
+      queue.expects(:bind).twice
+      @pub.expects(:declare_queue!).returns(queue).twice
+      @pub.send(:bind_queues_for_exchange, 'test_exchange')
       @pub.send(:bind_queues_for_exchange, 'test_exchange')
+    end
+
+    test "should declare queues only once even with many bindings" do
+      @client.register_queue('test_queue', :exchange => 'test_exchange')
+      @client.register_binding('test_queue', :exchange => 'test_exchange', :key => 'sir-message-a-lot')
+      queue = mock("queue")
+      queue.expects(:bind).twice
+      @pub.expects(:declare_queue!).returns(queue).once
       @pub.send(:bind_queues_for_exchange, 'test_exchange')
     end
 

data/test/beetle/subscriber_test.rb

@@ -168,6 +168,17 @@ module Beetle
       @sub.send(:bind_queues, %W(x y))
     end
 
+    test "binding queues with many bindings should create it only once" do
+      @client.register_queue(:x, :exchange => 'test_exchange')
+      @client.register_binding(:x, :exchange => 'test_exchange', :key => 'sir-message-a-lot')
+      @client.register_handler(%w(x)){}
+      @sub.stubs(:exchange)
+      queue = mock("queue")
+      queue.expects(:bind).twice
+      @sub.expects(:declare_queue!).returns(queue).once
+      @sub.send(:bind_queues, %W(x))
+    end
+
     test "subscribing to queues should subscribe on all queues" do
       @client.register_queue(:x)
       @client.register_queue(:y)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 3.3.11
+  version: 3.4.3
 platform: ruby
 authors:
 - Stefan Kaes
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-18 00:00:00.000000000 Z
+date: 2021-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -34,14 +34,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 4.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 4.2.1
 - !ruby/object:Gem::Dependency
   name: hiredis
   requirement: !ruby/object:Gem::Requirement
@@ -62,14 +62,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.3.2
 - !ruby/object:Gem::Dependency
   name: amqp
   requirement: !ruby/object:Gem::Requirement
@@ -337,7 +337,10 @@ files:
 - lib/beetle/redis_ext.rb
 - lib/beetle/subscriber.rb
 - lib/beetle/version.rb
+- script/analyze-expiries
+- script/analyze-redis-keys
 - script/console
+- script/expire-keys
 - script/start_rabbit
 - test/beetle/amqp_gem_behavior_test.rb
 - test/beetle/base_test.rb