beetle 3.2.0 → 3.3.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cfe005056abcea7bb999f5c1ffbc3300882e6625633034e0cc4ad7e8fc2b4d18
-  data.tar.gz: bc6a039a77d190c10878045420ce6fe57bb34d05cd343d71bbc8e9dc6832b74c
+  metadata.gz: df35415e0cf7a4d4915a56aa594fe4aa1e63be2d536eab5589a5b80c349fdb96
+  data.tar.gz: 6f621ab1501f71cdc7ee84d8989dd446ebd17676d53f9e23ba9d8278206908ed
 SHA512:
-  metadata.gz: 284dc26f2ccdcf11df046f977e083ac35ff0f35d759da350508cdaffecdcf3b56579a06381818f613711b9b5b2c57ff7d044d106562b39530b983e4a4148268e
-  data.tar.gz: 962ba73a9f2580748e53e846a36fefcbc1ee9a1054a88a8f60f3de77b3824ac547deaa2c69e94995155400e1aca1ce39cd06779efd6d8910420a99d7b682ec07
+  metadata.gz: 51e61e992d734f636b5a86b176ab1150530c8e7c18dbbaf8c4f8e685eaf48b2531133bb0770b9b8f50bea45e95588cad6b892866a6cd6644dc8d5a3e67e71b7d
+  data.tar.gz: 4ef44c5608cfe93adc0bf9fefea0cfdfda59f358323b5ed93158fb57e0122241c45a4deff76a8a098664a4e027ff1394e135541857a4682286ab135481c012a9

data/RELEASE_NOTES.rdoc

@@ -1,6 +1,14 @@
 = Release Notes
 
-== Version 3.1.1
+== Version 3.3.0.rc1
+* protect against duplicate handler execution by keeping the status of
+  handler exectution for both redundant and non-redundant messages in
+  the dedup store for a configurable time window. The config option
+  is named redis_status_key_expiry_interval. Note that this will
+  significantly increase the cpu load and memory usage of the Redis
+  server for applications using a large number of non-redundant messages.
+
+== Version 3.2.0
 * added currently processed message to the handler pre-processing step
 
 == Version 3.1.0

data/Rakefile

@@ -120,5 +120,5 @@ RDoc::Task.new do |rdoc|
 end
 
 task :clean do
-  system('rm -f tmp/*.output tmp/*.log tmp/master/* tmp/slave/* tmp/*lock tmp/*pid test.log')
+  sh "rm -f tmp/*.output tmp/*.log tmp/master-dir/* tmp/slave-dir/* tmp/*lock tmp/*pid test.log"
 end

data/features/redis_auto_failover.feature

@@ -27,7 +27,7 @@ Feature: Redis auto failover
   Scenario: Successful single redis master switch with multiple failover sets
     Given a redis server "redis-3" exists as master
     And a redis server "redis-4" exists as slave of "redis-3"
-    Given a redis configuration server using redis servers "a/redis-1,redis-2;b/redis-3,redis-4" with clients "rc-client-1,rc-client-2" exists
+    Given a redis configuration server using redis servers "system/redis-1,redis-2;system2/redis-3,redis-4" with clients "rc-client-1,rc-client-2" exists
     And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
     And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
     And a beetle handler using the redis-master file from "rc-client-1" exists
@@ -37,10 +37,10 @@ Feature: Redis auto failover
     And the role of redis server "redis-2" should be "master"
     And the redis master file of the redis configuration server should contain "redis-2"
     And the redis master file of the redis configuration server should contain "redis-3"
-    And the redis master of "rc-client-1" in system "a" should be "redis-2"
-    And the redis master of "rc-client-2" in system "a" should be "redis-2"
-    And the redis master of "rc-client-1" in system "b" should be "redis-3"
-    And the redis master of "rc-client-2" in system "b" should be "redis-3"
+    And the redis master of "rc-client-1" in system "system" should be "redis-2"
+    And the redis master of "rc-client-2" in system "system" should be "redis-2"
+    And the redis master of "rc-client-1" in system "system2" should be "redis-3"
+    And the redis master of "rc-client-2" in system "system2" should be "redis-3"
     And the redis master of the beetle handler should be "redis-2"
     And a system notification for switching from "redis-1" to "redis-2" should be sent
     Given a redis server "redis-1" exists as master
@@ -49,7 +49,7 @@ Feature: Redis auto failover
   Scenario: Successful double redis master switch with multiple failover sets
     Given a redis server "redis-3" exists as master
     And a redis server "redis-4" exists as slave of "redis-3"
-    Given a redis configuration server using redis servers "a/redis-1,redis-2;b/redis-3,redis-4" with clients "rc-client-1,rc-client-2" exists
+    Given a redis configuration server using redis servers "system/redis-1,redis-2;system2/redis-3,redis-4" with clients "rc-client-1,rc-client-2" exists
     And a redis configuration client "rc-client-1" using redis servers "redis-1,redis-2" exists
     And a redis configuration client "rc-client-2" using redis servers "redis-1,redis-2" exists
     And a beetle handler using the redis-master file from "rc-client-1" exists
@@ -60,10 +60,10 @@ Feature: Redis auto failover
     And the role of redis server "redis-4" should be "master"
     And the redis master file of the redis configuration server should contain "redis-2"
     And the redis master file of the redis configuration server should contain "redis-4"
-    And the redis master of "rc-client-1" in system "a" should be "redis-2"
-    And the redis master of "rc-client-2" in system "a" should be "redis-2"
-    And the redis master of "rc-client-1" in system "b" should be "redis-4"
-    And the redis master of "rc-client-2" in system "b" should be "redis-4"
+    And the redis master of "rc-client-1" in system "system" should be "redis-2"
+    And the redis master of "rc-client-2" in system "system" should be "redis-2"
+    And the redis master of "rc-client-1" in system "system2" should be "redis-4"
+    And the redis master of "rc-client-2" in system "system2" should be "redis-4"
     And the redis master of the beetle handler should be "redis-2"
     Given a redis server "redis-1" exists as master
     Then the role of redis server "redis-1" should be "slave"

data/features/support/beetle_handler

@@ -36,7 +36,7 @@ Daemons.run_proc("beetle_handler", :log_output => true, :dir_mode => :normal, :d
     config.handler(Beetle.config.beetle_policy_updates_queue_name) do |message|
       begin
         Beetle.config.logger.info "received beetle policy update message': #{message.data}"
-        client.set_queue_policies!(JSON.parse(message.data))
+        client.update_queue_properties!(JSON.parse(message.data))
       rescue => e
         Beetle.config.logger.error("#{e}:#{e.backtrace.join("\n")}")
       end

data/lib/beetle/configuration.rb

@@ -16,6 +16,8 @@ module Beetle
     # default logger (defaults to <tt>Logger.new(log_file)</tt>)
     attr_accessor :logger
     # defaults to <tt>STDOUT</tt>
+    attr_accessor :redis_logger
+    # set this to a logger instance if you want redis operations to be logged. defaults to <tt>nil</tt>.
     attr_accessor :log_file
     # number of seconds after which keys are removed from the message deduplication store (defaults to <tt>1.hour</tt>)
     attr_accessor :gc_threshold
@@ -35,6 +37,15 @@ module Beetle
     # handler timeout.
     attr_accessor :redis_failover_timeout
 
+    # how long we want status keys to survive after we have seen the second message of a
+    # redundant message pair. Defaults to 5 minutes. Setting this to a high value (hours)
+    # will reduce the likelihood of executing handler logic more than once, but can cause
+    # a higher redis database size with all associated problems. A handler can be
+    # executed more than once if the ack to the RabbitMQ server gets lost between the consumer
+    # and the RabbitMQ instance. This happens extremely seldom in our environment, but we cannot
+    # rule it out and we also don't quite understand (yet) why it happens.
+    attr_accessor :redis_status_key_expiry_interval
+
     # how often heartbeat messages are exchanged between failover
     # daemons. defaults to 10 seconds.
     attr_accessor :redis_failover_client_heartbeat_interval
@@ -140,6 +151,7 @@ module Beetle
       self.redis_servers = ""
       self.redis_db = 4
       self.redis_failover_timeout = 180.seconds
+      self.redis_status_key_expiry_interval = 5.minutes
       self.redis_failover_client_heartbeat_interval = 10.seconds
       self.redis_failover_client_dead_interval = 60.seconds
 

data/lib/beetle/deduplication_store.rb

@@ -30,7 +30,8 @@ module Beetle
       redis
     end
 
-    # list of key suffixes to use for storing values in Redis.
+    # list of key suffixes to use for storing values in Redis. 'status'
+    # always needs to be the first element of the array.
     KEY_SUFFIXES = [:status, :ack_count, :timeout, :delay, :attempts, :exceptions, :mutex, :expires]
 
     # build a Redis key out of a message id and a given suffix
@@ -58,6 +59,19 @@ module Beetle
       with_failover { redis.setnx(key(msg_id, suffix), value) }
     end
 
+    # store completion status for given <tt>msg_id</tt> if it doesn't exist yet. Returns whether the
+    # operation was successful.
+    def setnx_completed!(msg_id)
+      with_failover do
+        redis.set(
+          key(msg_id, :status),
+          "completed",
+          :nx => true,
+          :ex => @config.redis_status_key_expiry_interval,
+        )
+      end
+    end
+
     # store some key/value pairs
     def mset(msg_id, values)
       values = values.inject([]){|a,(k,v)| a.concat([key(msg_id, k), v])}
@@ -93,7 +107,12 @@ module Beetle
 
     # delete all keys associated with the given <tt>msg_id</tt>.
     def del_keys(msg_id)
-      with_failover { redis.del(*keys(msg_id)) }
+      keys = keys(msg_id)
+      status_key = keys.shift
+      with_failover do
+        redis.del(*keys)
+        redis.expire(status_key, @config.redis_status_key_expiry_interval)
+      end
     end
 
     # check whether key with given suffix exists for a given <tt>msg_id</tt>.
@@ -128,7 +147,7 @@ module Beetle
 
     # set current redis master instance (as specified in the Beetle::Configuration)
     def redis_master_from_server_string
-      @current_master ||= Redis.from_server_string(@config.redis_server, :db => @config.redis_db)
+      @current_master ||= Redis.from_server_string(@config.redis_server, :db => @config.redis_db, :logger => @config.redis_logger)
     end
 
     # set current redis master from master file
@@ -146,7 +165,7 @@ module Beetle
     def set_current_redis_master_from_master_file
       @last_time_master_file_changed = File.mtime(@config.redis_server)
       server_string = extract_redis_master(read_master_file)
-      @current_master = !server_string.blank? ? Redis.from_server_string(server_string, :db => @config.redis_db) : nil
+      @current_master = !server_string.blank? ? Redis.from_server_string(server_string, :db => @config.redis_db, :logger => @config.redis_logger) : nil
     end
 
     # extract redis master from file content and return the server for our system

data/lib/beetle/message.rb

@@ -265,7 +265,7 @@ module Beetle
         Beetle::reraise_expectation_errors!
         logger.error "Beetle: preprocessing error #{@pre_exception.class}(#{@pre_exception}) for #{msg_id}"
       end
-      logger.debug "Beetle: processing message #{msg_id}(#{timestamp})"
+      logger.debug "Beetle: processing message #{msg_id}(#{timestamp}) redelivered: #{header.redelivered?}"
       begin
         result = process_internal(handler)
         handler.process_exception(@exception || @pre_exception) if (@exception || @pre_exception)
@@ -294,7 +294,11 @@ module Beetle
         RC::Ancient
       elsif simple?
         ack!
-        run_handler(handler) == RC::HandlerCrash ? RC::AttemptsLimitReached : RC::OK
+        if @store.setnx_completed!(msg_id)
+          run_handler(handler) == RC::HandlerCrash ? RC::AttemptsLimitReached : RC::OK
+        else
+          RC::OK
+        end
       elsif !key_exists?
         run_handler!(handler)
       else
@@ -383,7 +387,7 @@ module Beetle
       #:doc:
       logger.debug "Beetle: ack! for message #{msg_id}"
       header.ack
-      return if simple? # simple messages don't use the deduplication store
+      return if simple?
       if !redundant? || @store.incr(msg_id, :ack_count) >= 2
         # we test for >= 2 here, because we retry increments in the
         # dedup store so the counter could be greater than two

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "3.2.0"
+  VERSION = "3.3.0.rc1"
 end

data/test/beetle/message_test.rb

@@ -115,7 +115,8 @@ module Beetle
 
   class KeyManagementTest < Minitest::Test
     def setup
-      @store = DeduplicationStore.new
+      @config = Configuration.new
+      @store = DeduplicationStore.new(@config)
       @store.flushdb
       @null_handler = Handler.create(lambda {|*args|})
     end
@@ -128,33 +129,42 @@ module Beetle
       end
     end
 
-    test "successful processing of a non redundant message should delete all keys from the database" do
+    test "successful processing of a non redundant message should delete all keys from the database (except the status key, which should be set to expire)" do
       header = header_with_params({})
       header.expects(:ack)
       message = Message.new("somequeue", header, 'foo', :store => @store)
+      message.stubs(:simple?).returns(false)
 
       assert !message.expired?
       assert !message.redundant?
 
       message.process(@null_handler)
-
-      @store.keys(message.msg_id).each do |key|
+      keys = @store.keys(message.msg_id)
+      status_key = keys.shift
+      assert @store.redis.exists(status_key)
+      assert @store.redis.ttl(status_key) <= @config.redis_status_key_expiry_interval
+      keys.each do |key|
         assert !@store.redis.exists(key)
       end
     end
 
-    test "succesful processing of a redundant message twice should delete all keys from the database" do
+    test "successful processing of a redundant message twice should delete all keys from the database (except the status key, which should be set to expire)" do
       header = header_with_params({:redundant => true})
       header.expects(:ack).twice
       message = Message.new("somequeue", header, 'foo', :store => @store)
 
       assert !message.expired?
       assert message.redundant?
+      assert !message.simple?
 
       message.process(@null_handler)
       message.process(@null_handler)
 
-      @store.keys(message.msg_id).each do |key|
+      keys = @store.keys(message.msg_id)
+      status_key = keys.shift
+      assert @store.redis.exists(status_key)
+      assert @store.redis.ttl(status_key) <= @config.redis_status_key_expiry_interval
+      keys.each do |key|
         assert !@store.redis.exists(key)
       end
     end
@@ -294,10 +304,9 @@ module Beetle
     def setup
       @store = DeduplicationStore.new
       @store.flushdb
-      @store.expects(:redis).never
     end
 
-    test "when processing a simple message, ack should precede calling the handler" do
+    test "when processing a simple message, ack should follow calling the handler" do
       header = header_with_params({})
       message = Message.new("somequeue", header, 'foo', :attempts => 1, :store => @store)
 

data/test/test_helper.rb

@@ -35,6 +35,7 @@ def header_with_params(opts = {})
   beetle_headers = Beetle::Message.publishing_options(opts)
   header = mock("header")
   header.stubs(:attributes).returns(beetle_headers)
+  header.stubs(:redelivered?).returns(false)
   header
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.0.rc1
 platform: ruby
 authors:
 - Stefan Kaes
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-09 00:00:00.000000000 Z
+date: 2019-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny