beetle 3.3.0.rc1 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df35415e0cf7a4d4915a56aa594fe4aa1e63be2d536eab5589a5b80c349fdb96
-  data.tar.gz: 6f621ab1501f71cdc7ee84d8989dd446ebd17676d53f9e23ba9d8278206908ed
+  metadata.gz: 3e1e8a0378a54fd5b151e8a85fe404d74b06bc0e33458e632676bb5b60b7b38d
+  data.tar.gz: 546a4383cdc3727eb9709ea6ef30647cfc647c803e5be9853b77b50b70c915c0
 SHA512:
-  metadata.gz: 51e61e992d734f636b5a86b176ab1150530c8e7c18dbbaf8c4f8e685eaf48b2531133bb0770b9b8f50bea45e95588cad6b892866a6cd6644dc8d5a3e67e71b7d
-  data.tar.gz: 4ef44c5608cfe93adc0bf9fefea0cfdfda59f358323b5ed93158fb57e0122241c45a4deff76a8a098664a4e027ff1394e135541857a4682286ab135481c012a9
+  metadata.gz: d307e9c99eaafc64463e2fc5b21b3b9b4a04a831b3eb74d0330f86411412dc01f4f3e0a3e059695d5cfbd2b94b527352549fbaf4e5ebd4c8f28acdce79b25dcb
+  data.tar.gz: a285335c8023bda35630fbe01162d5cd5e196fc60576c827f3f05e099d9f5c3e35608d7819d996de849c9f25e3619a33de90a2cca3704fab11636f3ec6645d0f

data/RELEASE_NOTES.rdoc

@@ -1,12 +1,14 @@
 = Release Notes
 
-== Version 3.3.0.rc1
+== Version 3.3.0
 * protect against duplicate handler execution by keeping the status of
   handler exectution for both redundant and non-redundant messages in
   the dedup store for a configurable time window. The config option
   is named redis_status_key_expiry_interval. Note that this will
   significantly increase the cpu load and memory usage of the Redis
-  server for applications using a large number of non-redundant messages.
+  server for applications using a large number of non-redundant
+  messages. This feature is turned off by default, but we will
+  activate it with the next major release (4.0.0).
 
 == Version 3.2.0
 * added currently processed message to the handler pre-processing step

data/lib/beetle/configuration.rb

@@ -37,13 +37,11 @@ module Beetle
     # handler timeout.
     attr_accessor :redis_failover_timeout
 
-    # how long we want status keys to survive after we have seen the second message of a
-    # redundant message pair. Defaults to 5 minutes. Setting this to a high value (hours)
-    # will reduce the likelihood of executing handler logic more than once, but can cause
-    # a higher redis database size with all associated problems. A handler can be
-    # executed more than once if the ack to the RabbitMQ server gets lost between the consumer
-    # and the RabbitMQ instance. This happens extremely seldom in our environment, but we cannot
-    # rule it out and we also don't quite understand (yet) why it happens.
+    # how long we want status keys to survive after we have seen the second message of a redundant
+    # message pair. Defaults to 0 seconds, but will be set to something non-zero with the next
+    # major beetle release. A recommended value would be 5 minutes (300 seconds). Setting this to a
+    # high value (hours) will reduce the likelihood of executing handler logic more than once, but
+    # can cause a higher redis database size with all associated problems.
     attr_accessor :redis_status_key_expiry_interval
 
     # how often heartbeat messages are exchanged between failover
@@ -151,7 +149,7 @@ module Beetle
       self.redis_servers = ""
       self.redis_db = 4
       self.redis_failover_timeout = 180.seconds
-      self.redis_status_key_expiry_interval = 5.minutes
+      self.redis_status_key_expiry_interval = 0.seconds
       self.redis_failover_client_heartbeat_interval = 10.seconds
       self.redis_failover_client_dead_interval = 60.seconds
 

data/lib/beetle/deduplication_store.rb

@@ -62,12 +62,14 @@ module Beetle
     # store completion status for given <tt>msg_id</tt> if it doesn't exist yet. Returns whether the
     # operation was successful.
     def setnx_completed!(msg_id)
+      expiry = @config.redis_status_key_expiry_interval
+      return true if expiry == 0
       with_failover do
         redis.set(
           key(msg_id, :status),
           "completed",
           :nx => true,
-          :ex => @config.redis_status_key_expiry_interval,
+          :ex => expiry,
         )
       end
     end
@@ -107,11 +109,12 @@ module Beetle
 
     # delete all keys associated with the given <tt>msg_id</tt>.
     def del_keys(msg_id)
+      expiry = @config.redis_status_key_expiry_interval
       keys = keys(msg_id)
-      status_key = keys.shift
+      status_key = keys.shift if expiry > 0
       with_failover do
         redis.del(*keys)
-        redis.expire(status_key, @config.redis_status_key_expiry_interval)
+        redis.expire(status_key, expiry) if expiry > 0
       end
     end
 

data/lib/beetle/version.rb

@@ -1,3 +1,3 @@
 module Beetle
-  VERSION = "3.3.0.rc1"
+  VERSION = "3.3.0"
 end

data/test/beetle/message_test.rb

@@ -129,7 +129,24 @@ module Beetle
       end
     end
 
+    test "successful processing of a non redundant message should delete all keys from the database" do
+      header = header_with_params({})
+      header.expects(:ack)
+      message = Message.new("somequeue", header, 'foo', :store => @store)
+      message.stubs(:simple?).returns(false)
+
+      assert !message.expired?
+      assert !message.redundant?
+
+      message.process(@null_handler)
+      keys = @store.keys(message.msg_id)
+      keys.each do |key|
+        assert !@store.redis.exists(key)
+      end
+    end
+
     test "successful processing of a non redundant message should delete all keys from the database (except the status key, which should be set to expire)" do
+      @config.redis_status_key_expiry_interval = 1.minutes
       header = header_with_params({})
       header.expects(:ack)
       message = Message.new("somequeue", header, 'foo', :store => @store)
@@ -148,7 +165,26 @@ module Beetle
       end
     end
 
+    test "successful processing of a redundant message twice should delete all keys from the database" do
+      header = header_with_params({:redundant => true})
+      header.expects(:ack).twice
+      message = Message.new("somequeue", header, 'foo', :store => @store)
+
+      assert !message.expired?
+      assert message.redundant?
+      assert !message.simple?
+
+      message.process(@null_handler)
+      message.process(@null_handler)
+
+      keys = @store.keys(message.msg_id)
+      keys.each do |key|
+        assert !@store.redis.exists(key)
+      end
+    end
+
     test "successful processing of a redundant message twice should delete all keys from the database (except the status key, which should be set to expire)" do
+      @config.redis_status_key_expiry_interval = 1.minutes
       header = header_with_params({:redundant => true})
       header.expects(:ack).twice
       message = Message.new("somequeue", header, 'foo', :store => @store)
@@ -302,7 +338,8 @@ module Beetle
 
   class SimpleMessageTest < Minitest::Test
     def setup
-      @store = DeduplicationStore.new
+      @config = Configuration.new
+      @store = DeduplicationStore.new(@config)
       @store.flushdb
     end
 
@@ -333,6 +370,26 @@ module Beetle
       assert_equal RC::AttemptsLimitReached, message.process(handler)
     end
 
+    test "when processing a simple message, the handler should be executed only once if status keys are used" do
+      @config.redis_status_key_expiry_interval = 1.minute
+      header = header_with_params({})
+      message = Message.new("somequeue", header, 'foo', :attempts => 1, :store => @store)
+
+      handler = mock("handler")
+      s = sequence("s")
+      handler.expects(:pre_process).with(message).in_sequence(s)
+      header.expects(:ack).in_sequence(s)
+      handler.expects(:call).in_sequence(s)
+      assert_equal RC::OK, message.process(handler)
+
+      handler2 = mock("handler")
+      s2 = sequence("s2")
+      handler2.expects(:pre_process).with(message).in_sequence(s2)
+      header.expects(:ack).in_sequence(s2)
+      handler2.expects(:call).in_sequence(s2).never
+      assert_equal RC::OK, message.process(handler2)
+    end
+
   end
 
   class HandlerCrashTest < Minitest::Test

data/test/beetle/publisher_test.rb

@@ -53,6 +53,17 @@ module Beetle
       assert_nil @pub.instance_variable_get(:@bunnies)[@pub.server]
     end
 
+    test "stop!(exception) should close the bunny socket if an exception is not nil" do
+      b = mock("bunny")
+      b.expects(:close_socket)
+      @pub.expects(:bunny?).returns(true)
+      @pub.expects(:bunny).returns(b)
+      @pub.send(:stop!, Exception.new)
+      assert_equal({}, @pub.send(:exchanges))
+      assert_equal({}, @pub.send(:queues))
+      assert_nil @pub.instance_variable_get(:@bunnies)[@pub.server]
+    end
+
     test "stop! should not create a new bunny " do
       @pub.expects(:bunny?).returns(false)
       @pub.expects(:bunny).never
@@ -392,6 +403,17 @@ module Beetle
       assert @pub.throttled?
     end
 
+    test "refresh_throttling! logs a warning if an exception is raised during throttling" do
+      assert !@pub.throttled?
+      @pub.instance_variable_set :@next_throttle_refresh, Time.now - 1
+      options = { "test" => 100 }
+      @pub.throttle(options)
+      @pub.expects(:each_server).raises(StandardError.new("foo"))
+      @pub.logger.expects(:warn)
+      @pub.__send__ :refresh_throttling!
+      assert !@pub.throttled?
+    end
+
   end
 
   class PublisherExchangeManagementTest < Minitest::Test

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beetle
 version: !ruby/object:Gem::Version
-  version: 3.3.0.rc1
+  version: 3.3.0
 platform: ruby
 authors:
 - Stefan Kaes
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-17 00:00:00.000000000 Z
+date: 2019-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny