beetle 3.3.10 → 3.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64ab9d9173f45b670f0daa0297cf5df54335452df87813a1404dd6508115332b
-  data.tar.gz: 640e13316b247168fa96e5b5bf138ffd0b2c430defc0778d9b87d0ca4a2a9af3
+  metadata.gz: '08db25367467939ff6a625b0caf2b940832b398c7988189ac2a0e91d5244a78b'
+  data.tar.gz: d9ffc4a722b0d86db98e81e0f9080b76e2fbd01537cbb4a6839b00665e778f19
 SHA512:
-  metadata.gz: 3da413178c33e889be536a73ebe69b5155af9f6ba054a9349616c4d07c741b791810d6ca37e931c88c90b874f0e8034fcd797210ff8b4abda85ce96945c828de
-  data.tar.gz: 1178a3e403b9d390a52e854f0c48f6afa3c5970eee2798d4628b4280fd02402ed3c3a913b8cb136a908f078befe775715dbee4efa36108155dfd382914f2c1b0
+  metadata.gz: c7f986880a871a59a74ee9e3fc63b9b384097713eedc9f106c7b799cd6b8762714d05d181665ac5aadea0675fd10f30acfbaec8fcdf921a31721c609220b95ad
+  data.tar.gz: 3beb91290f71cf8191f9abe0a116732e0e880a3e0e57d2c774c2c417bde4fb697b7980e2d2f6af0bdfcd601b124c7c4286a796fada014771be3bb8b049130d12

data/RELEASE_NOTES.rdoc

@@ -1,5 +1,24 @@
 = Release Notes
 
+== Version 3.4.2
+* Updated amq-protocol gem to version 2.3.2.
+* Fixed a rare race condition on message handler timeouts.
+
+== Version 3.4.1
+* Updated amq-protocol gem to version 2.3.1.
+
+== Version 3.4.0
+* Require redis gem version 4.2.1. This version changes the exists to check for the
+  existence of multiple keys, return the number of keys in the list that exist. This
+  requires at least redis gem version 4.2.0, but 4.2.1 contains a bug fix for said
+  command.
+
+== Version 3.3.12
+* Support queue level declaration of dead letter queue message TTL.
+
+== Version 3.3.11
+* Fixed that dead lettering only works correctly with global config option.
+
 == Version 3.3.10
 * Support configuring RabbitMQ write timeout.
 

data/beetle.gemspec

@@ -24,9 +24,9 @@ Gem::Specification.new do |s|
 
   s.specification_version = 3
   s.add_runtime_dependency "bunny",                   "~> 0.7.12"
-  s.add_runtime_dependency "redis",                   ">= 2.2.2"
+  s.add_runtime_dependency "redis",                   ">= 4.2.1"
   s.add_runtime_dependency "hiredis",                 ">= 0.4.5"
-  s.add_runtime_dependency "amq-protocol",            "= 2.3.0"
+  s.add_runtime_dependency "amq-protocol",            "= 2.3.2"
   s.add_runtime_dependency "amqp",                    "= 1.8.0"
   s.add_runtime_dependency "activesupport",           ">= 2.3.4"
 

data/lib/beetle/base.rb

@@ -78,7 +78,8 @@ module Beetle
     end
 
     def bind_dead_letter_queue!(channel, target_queue, creation_keys = {})
-      policy_options = @client.queues[target_queue].slice(:dead_lettering, :lazy)
+      policy_options = @client.queues[target_queue].slice(:dead_lettering, :lazy, :dead_lettering_msg_ttl)
+      policy_options[:message_ttl] = policy_options.delete(:dead_lettering_msg_ttl)
       dead_letter_queue_name = "#{target_queue}_dead_letter"
       if policy_options[:dead_lettering]
         logger.debug("Beetle: creating dead letter queue #{dead_letter_queue_name} with opts: #{creation_keys.inspect}")
@@ -88,7 +89,7 @@ module Beetle
         :queue_name => target_queue,
         :bindings => @client.bindings[target_queue],
         :dead_letter_queue_name => dead_letter_queue_name,
-        :message_ttl => @client.config.dead_lettering_msg_ttl,
+        :message_ttl => policy_options[:message_ttl]
       }.merge(policy_options)
     end
 

data/lib/beetle/client.rb

@@ -96,7 +96,8 @@ module Beetle
       raise ConfigurationError.new("queue #{name} already configured") if queues.include?(name)
       opts = {
         :exchange => name, :key => name, :auto_delete => false, :amqp_name => name,
-        :lazy => config.lazy_queues_enabled, :dead_lettering => config.dead_lettering_enabled
+        :lazy => config.lazy_queues_enabled, :dead_lettering => config.dead_lettering_enabled,
+        :dead_lettering_msg_ttl => config.dead_lettering_msg_ttl
       }.merge!(options.symbolize_keys)
       opts.merge! :durable => true, :passive => false, :exclusive => false
       exchange = opts.delete(:exchange).to_s

data/lib/beetle/deduplication_store.rb

@@ -120,7 +120,7 @@ module Beetle
 
     # check whether key with given suffix exists for a given <tt>msg_id</tt>.
     def exists(msg_id, suffix)
-      with_failover { redis.exists(key(msg_id, suffix)) }
+      with_failover { redis.exists?(key(msg_id, suffix)) }
     end
 
     # flush the configured redis database. useful for testing.

data/lib/beetle/message.rb

@@ -18,6 +18,8 @@ module Beetle
     # forcefully abort a running handler after this many seconds.
     # can be overriden when registering a handler.
     DEFAULT_HANDLER_TIMEOUT = 600.seconds
+    # How much extra time on top of the handler timeout we add before considering a handler timed out
+    TIMEOUT_GRACE_PERIOD = 10.seconds
     # how many times we should try to run a handler before giving up
     DEFAULT_HANDLER_EXECUTION_ATTEMPTS = 1
     # how many seconds we should wait before retrying handler execution
@@ -167,8 +169,8 @@ module Beetle
     end
 
     # handler timed out?
-    def timed_out?
-      (t = @store.get(msg_id, :timeout)) && t.to_i < now
+    def timed_out?(t = nil)
+      (t ||= @store.get(msg_id, :timeout)) && (t.to_i + TIMEOUT_GRACE_PERIOD) < now
     end
 
     # reset handler timeout in the deduplication store
@@ -187,8 +189,8 @@ module Beetle
     end
 
     # whether we should wait before running the handler
-    def delayed?
-      (t = @store.get(msg_id, :delay)) && t.to_i > now
+    def delayed?(t = nil)
+      (t ||= @store.get(msg_id, :delay)) && t.to_i > now
     end
 
     # store delay value in the deduplication store
@@ -207,8 +209,8 @@ module Beetle
     end
 
     # whether we have already tried running the handler as often as specified when the handler was registered
-    def attempts_limit_reached?
-      (limit = @store.get(msg_id, :attempts)) && limit.to_i >= attempts_limit
+    def attempts_limit_reached?(attempts = nil)
+      (attempts ||= @store.get(msg_id, :attempts)) && attempts.to_i >= attempts_limit
     end
 
     # increment number of exception occurences in the deduplication store
@@ -217,8 +219,8 @@ module Beetle
     end
 
     # whether the number of exceptions has exceeded the limit set when the handler was registered
-    def exceptions_limit_reached?
-      @store.get(msg_id, :exceptions).to_i > exceptions_limit
+    def exceptions_limit_reached?(exceptions = nil)
+      (exceptions ||= @store.get(msg_id, :exceptions)) && exceptions.to_i > exceptions_limit
     end
 
     def exception_accepted?
@@ -306,17 +308,17 @@ module Beetle
         if status == "completed"
           ack!
           RC::OK
-        elsif delay && delay.to_i > now
+        elsif delay && delayed?(delay)
           logger.warn "Beetle: ignored delayed message (#{msg_id})!"
           RC::Delayed
-        elsif !(timeout && timeout.to_i < now)
+        elsif !(timeout && timed_out?(timeout))
           RC::HandlerNotYetTimedOut
-        elsif attempts.to_i >= attempts_limit
+        elsif attempts && attempts_limit_reached?(attempts)
           completed!
           ack!
           logger.warn "Beetle: reached the handler execution attempts limit: #{attempts_limit} on #{msg_id}"
           RC::AttemptsLimitReached
-        elsif exceptions.to_i > exceptions_limit
+        elsif exceptions && exceptions_limit_reached?(exceptions)
           completed!
           ack!
           logger.warn "Beetle: reached the handler exceptions limit: #{exceptions_limit} on #{msg_id}"

data/lib/beetle/queue_properties.rb

@@ -164,7 +164,7 @@ module Beetle
       end
       http = Net::HTTP.new(uri.hostname, config.api_port)
       http.read_timeout = config.rabbitmq_api_read_timeout
-      http.write_timeout = config.rabbitmq_api_write_timeout
+      http.write_timeout = config.rabbitmq_api_write_timeout if http.respond_to?(:write_timeout=)
       # don't do this in production:
       # http.set_debug_output(logger.instance_eval{ @logdev.dev })
       http.start do |instance|

data/lib/beetle/subscriber.rb

@@ -180,7 +180,7 @@ module Beetle
           processor = Handler.create(handler, opts)
           result = m.process(processor)
           if result.reject?
-            if @client.config.dead_lettering_enabled?
+            if @client.queues[queue_name][:dead_lettering]
               header.reject(:requeue => false)
             else
               sleep 1

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "3.3.10"
+  VERSION = "3.4.2"
 end

data/script/analyze-expiries

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'time'
+
+counts = Hash.new(0)
+expiries = Hash.new{|h,k| h[k] = Hash.new(0)}
+t = Time.now.to_i
+
+File.open(ARGV[0]).each_line do |l|
+  parts = l.split(':')
+  queue = parts[1]
+  counts[queue] += 1
+  expiry = parts[4].to_i
+  expires_in = ((expiry - t)/(3600.0)).ceil
+  expiries[queue][expires_in] += 1
+end
+
+counts.to_a.sort_by{|_,v| -v}.each do |q,v|
+  puts "------------------------------------------------------------------"
+  puts "#{q}: #{v}"
+  puts "------------------------------------------------------------------"
+  expiries[q].to_a.sort_by{|k,_| -k}.each do |expiry, count|
+    printf "%3dh: %6d\n", expiry, count
+  end
+end

data/script/analyze-redis-keys

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+queue_counts = Hash.new { |h,k| h[k] = 0 }
+File.open(ARGV[0]).each_line do |l|
+  next if l == "lastgc" || l == "clients-last-seen"
+  a = l.split(':')
+  if a[0] == "msgid"
+    queue_counts[a[1]] += 1
+  else
+    queue_counts["none"] += 1
+  end
+end
+sorted_queues = queue_counts.to_a.sort_by{|a| -a[1]}
+sorted_queues.each do |q,c|
+  puts "#{c}:#{q}"
+end

data/script/expire-keys

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+require "redis"
+
+redis = Redis.new(:host => "beetle-1.redis.ams2.xing.com", :port => 6379, :db => 0)
+
+File.open(ARGV[0]).each_line.each_slice(10) do |s|
+  redis.pipelined do
+    s.each do |l|
+      l.chomp!
+      next if l =~ /^beetle:.*$/
+      redis.expire(l, 600)
+    end
+  end
+end

data/test/beetle/client_test.rb

@@ -79,7 +79,7 @@ module Beetle
 
     test "registering a queue should store it in the configuration with symbolized option keys and force durable=true and passive=false and set the amqp queue name" do
       @client.register_queue("some_queue", "durable" => false, "exchange" => "some_exchange")
-      assert_equal({:durable => true, :passive => false, :lazy=>false, :dead_lettering=>false, :auto_delete => false, :exclusive => false, :amqp_name => "some_queue"}, @client.queues["some_queue"])
+      assert_equal({:durable => true, :passive => false, :lazy=>false, :dead_lettering=>false, :dead_lettering_msg_ttl=>1000, :auto_delete => false, :exclusive => false, :amqp_name => "some_queue"}, @client.queues["some_queue"])
     end
 
     test "registering a queue should add the queue to the list of queues of the queue's exchange" do

data/test/beetle/deduplication_store_test.rb

@@ -9,7 +9,7 @@ module Beetle
     end
 
     test "trying to delete a non existent key doesn't throw an error" do
-      assert !@r.exists("hahahaha")
+      assert !@r.exists?("hahahaha")
       assert_equal 0, @r.del("hahahaha")
     end
 

data/test/beetle/message/settings_test.rb

@@ -75,9 +75,9 @@ module Beetle
       message.expects(:now).returns(1)
       message.set_timeout!
       assert_equal "2", @store.get(message.msg_id, :timeout)
-      message.expects(:now).returns(2)
+      message.expects(:now).returns(2 + Message::TIMEOUT_GRACE_PERIOD)
       assert !message.timed_out?
-      message.expects(:now).returns(3)
+      message.expects(:now).returns(3 + Message::TIMEOUT_GRACE_PERIOD)
       assert message.timed_out?
     end
 
@@ -86,9 +86,9 @@ module Beetle
       message.expects(:now).returns(0)
       message.set_timeout!
       assert_equal "#{Message::DEFAULT_HANDLER_TIMEOUT}", @store.get(message.msg_id, :timeout)
-      message.expects(:now).returns(message.timeout)
+      message.expects(:now).returns(message.timeout + Message::TIMEOUT_GRACE_PERIOD)
       assert !message.timed_out?
-      message.expects(:now).returns(Message::DEFAULT_HANDLER_TIMEOUT + 1)
+      message.expects(:now).returns(message.timeout + Message::TIMEOUT_GRACE_PERIOD + 1)
       assert message.timed_out?
     end
 

data/test/beetle/message_test.rb

@@ -141,9 +141,7 @@ module Beetle
 
       message.process(@null_handler)
       keys = @store.keys(message.msg_id)
-      keys.each do |key|
-        assert !@store.redis.exists(key)
-      end
+      assert_equal 0, @store.redis.exists(*keys)
     end
 
     test "successful processing of a non redundant message should delete all keys from the database (except the status key, which should be set to expire)" do
@@ -159,11 +157,9 @@ module Beetle
       message.process(@null_handler)
       keys = @store.keys(message.msg_id)
       status_key = keys.shift
-      assert @store.redis.exists(status_key)
+      assert @store.redis.exists?(status_key)
       assert @store.redis.ttl(status_key) <= @config.redis_status_key_expiry_interval
-      keys.each do |key|
-        assert !@store.redis.exists(key)
-      end
+      assert_equal 0, @store.redis.exists(*keys)
     end
 
     test "successful processing of a redundant message twice should delete all keys from the database" do
@@ -179,9 +175,7 @@ module Beetle
       message.process(@null_handler)
 
       keys = @store.keys(message.msg_id)
-      keys.each do |key|
-        assert !@store.redis.exists(key)
-      end
+      assert_equal 0, @store.redis.exists(*keys)
     end
 
     test "successful processing of a redundant message twice should delete all keys from the database (except the status key, which should be set to expire)" do
@@ -199,11 +193,9 @@ module Beetle
 
       keys = @store.keys(message.msg_id)
       status_key = keys.shift
-      assert @store.redis.exists(status_key)
+      assert @store.redis.exists?(status_key)
       assert @store.redis.ttl(status_key) <= @config.redis_status_key_expiry_interval
-      keys.each do |key|
-        assert !@store.redis.exists(key)
-      end
+      assert_equal 0, @store.redis.exists(*keys)
     end
 
     test "successful processing of a redundant message once should insert all but the delay key and the exception count key into the database" do

data/test/beetle/subscriber_test.rb

@@ -224,13 +224,11 @@ module Beetle
     end
   end
 
-
   class DeadLetteringCallBackExecutionTest < Minitest::Test
     def setup
       @client = Client.new
-      @client.config.dead_lettering_enabled = true
       @queue = "somequeue"
-      @client.register_queue(@queue)
+      @client.register_queue(@queue, :dead_lettering => true)
       @sub = @client.send(:subscriber)
       mq = mock("MQ")
       mq.expects(:closing?).returns(false)
@@ -239,11 +237,7 @@ module Beetle
       @handler = Handler.create(lambda{|*args| raise @exception})
       # handler method 'processing_completed' should be called under all circumstances
       @handler.expects(:processing_completed).once
-      @callback = @sub.send(:create_subscription_callback, "my myessage", @queue, @handler, :exceptions => 1)
-    end
-
-    def teardown
-      @client.config.dead_lettering_enabled = false
+      @callback = @sub.send(:create_subscription_callback, @queue, @queue, @handler, :exceptions => 1)
     end
 
     test "should call reject on the message header when processing the handler returns true on reject? if dead lettering has been enabled" do
@@ -255,19 +249,18 @@ module Beetle
       header.expects(:reject).with(:requeue => false)
       @callback.call(header, 'foo')
     end
-
   end
 
   class CallBackExecutionTest < Minitest::Test
     def setup
-      client = Client.new
+      @client = Client.new
       @queue = "somequeue"
-      client.register_queue(@queue)
-      @sub = client.send(:subscriber)
+      @client.register_queue(@queue)
+      @sub = @client.send(:subscriber)
       @exception = Exception.new "murks"
       @handler = Handler.create(lambda{|*args| raise @exception})
       @handler.instance_eval { def post_process; raise "shoot"; end }
-      @callback = @sub.send(:create_subscription_callback, "my myessage", @queue, @handler, :exceptions => 1)
+      @callback = @sub.send(:create_subscription_callback, @queue, @queue, @handler, :exceptions => 1)
     end
 
     test "exceptions raised from message processing should be ignored" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 3.3.10
+  version: 3.4.2
 platform: ruby
 authors:
 - Stefan Kaes
@@ -9,10 +9,10 @@ authors:
 - Ali Jelveh
 - Sebastian Roebke
 - Larry Baltz
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-21 00:00:00.000000000 Z
+date: 2020-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -34,14 +34,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 4.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 4.2.1
 - !ruby/object:Gem::Dependency
   name: hiredis
   requirement: !ruby/object:Gem::Requirement
@@ -62,14 +62,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.3.2
 - !ruby/object:Gem::Dependency
   name: amqp
   requirement: !ruby/object:Gem::Requirement
@@ -337,7 +337,10 @@ files:
 - lib/beetle/redis_ext.rb
 - lib/beetle/subscriber.rb
 - lib/beetle/version.rb
+- script/analyze-expiries
+- script/analyze-redis-keys
 - script/console
+- script/expire-keys
 - script/start_rabbit
 - test/beetle/amqp_gem_behavior_test.rb
 - test/beetle/base_test.rb
@@ -359,7 +362,7 @@ homepage: https://xing.github.com/beetle/
 licenses: []
 metadata:
   changelog_uri: https://github.com/xing/beetle/blob/master/RELEASE_NOTES.rdoc
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--charset=UTF-8"
 require_paths:
@@ -376,7 +379,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.7
 requirements: []
 rubygems_version: 3.0.8
-signing_key: 
+signing_key:
 specification_version: 3
 summary: High Availability AMQP Messaging with Redundant Queues
 test_files: