eventq 2.0.3 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bbf0fcaf93e6ca42d1fa190d4880b3416f1e32481950b1817cf3e97b87f7e465
-  data.tar.gz: be23ab444cbc536d1aa0f3996f2209ee5409e19c1d8b5e98f8a79ad7736d531d
+  metadata.gz: 946df3b90ab1e50b46c3e3c1732b0e82c7adfe67468e1a49b08d6fa761a0cce9
+  data.tar.gz: 3fcf13525865b2896f73c3088de352d6658b2b9dbf0754acc06412f8d5d9fcba
 SHA512:
-  metadata.gz: af21be2b6e181e44241efc06733c025c7fae3ba65062e39a7ae34e5c07e16a7887751c9bfc8c6853eec973030fdb23059c74804f2ad4e8dbb40c64adf5ebeb87
-  data.tar.gz: df647d0e20c6adb7c99499fbdd21e106a127d6912e01f9a71a1eb5931191b4976b6e67d0a49604200dd999ec8fa38ca9a98eae2a240e5147d0f72fe8259bdec4
+  metadata.gz: ed166219545af071b020b6bd8268c0a2a0da4f8727b2751fbd4b72d3350cfe5503c0ac885dd177aef4272f837d869cfe8a7bfbdc7931a2fc196ce15072e4bc87
+  data.tar.gz: 002dce7368948f01ef23a9cfa9756c7de082e31978811bdfde91a1ba4a68501d0620237160254f85af9d2d8acc6ea7042c502af0253163400a7ea50761998ca1

data/README.md

@@ -54,7 +54,7 @@ A subscription queue should be defined to receive any events raised for the subs
 **Attributes**
 
  - **allow_retry** [Bool] [Optional] [Default=false] This determines if the queue should allow processing failures to be retried.
- - **allow_retry_backoff** [Bool] [Optional] [Default=false] This is used to specify if failed messages that retry should incrementally backoff.
+ - **allow_retry_back_off** [Bool] [Optional] [Default=false] This is used to specify if failed messages that retry should incrementally backoff.
  - **retry_back_off_grace** [Int] [Optional] [Default=0] This is the number of times to allow retries without applying retry back off if enabled.
  - **dlq** [EventQ::Queue] [Optional] [Default=nil] A queue that will receive the messages which were not successfully processed after maximum number of receives by consumers. This is created at the same time as the parent queue.
  - **max_retry_attempts** [Int] [Optional] [Default=5] This is used to specify the max number of times an event should be allowed to retry before failing.
@@ -63,6 +63,7 @@ A subscription queue should be defined to receive any events raised for the subs
  - **name** [String] [Required] This is the name of the queue, it must be unique.
  - **require_signature** [Bool] [Optional] [Default=false] This is used to specify if messages within this queue must be signed.
  - **retry_delay** [Int] [Optional] [Default=30000] This is used to specify the time delay in milliseconds before a failed message is re-added to the subscription queue.
+ - **retry_back_off_weight** [Int] [Optional] [Default=1] Additional multiplier for the timeout backoff. Normally used when `retry_delay` is too small (eg: 30ms) in order to get meaningful backoff values.
 
 **Example**
 
@@ -254,7 +255,7 @@ This is used to specify the signature secret that should be used for message sig
 
 ### NonceManager
 
-The NonceManager is used to prevent duplicate messages from being processed. Each event message that is raised is given a unique identifier, most message queue providers guarantee at least once delivery which may result in the message being delivered more than once. If your use case needs to enforce once only processing then 
+The NonceManager is used to prevent duplicate messages from being processed. Each event message that is raised is given a unique identifier, most message queue providers guarantee at least once delivery which may result in the message being delivered more than once. If your use case needs to enforce once only processing then
 the NonceManager can be configured to prevent duplicate messages from being processed. (It is a distributed store that currently uses redis locks to ensure accuracy between scaled out workers)
 
 #### configure

data/lib/eventq/aws.rb

@@ -1,6 +1,7 @@
 require 'aws-sdk-core'
 require 'eventq'
 
+require_relative './eventq_aws/aws_calculate_visibility_timeout'
 require_relative './eventq_aws/aws_eventq_client'
 require_relative './eventq_aws/sns'
 require_relative './eventq_aws/sqs'

data/lib/eventq/eventq_aws/aws_calculate_visibility_timeout.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module EventQ
+  module Amazon
+    # Class responsible to know how to calculate message Visibility Timeout for Amazon SQS
+    class CalculateVisibilityTimeout
+      def initialize(max_timeout:, logger: EventQ.logger)
+        @max_timeout = max_timeout
+        @logger      = logger
+      end
+
+      # Calculate Visibility Timeout
+      #
+      # @param retry_attempts [Integer] Current retry
+      # @param queue_settings [Hash] Queue settings
+      # @option allow_retry_back_off [Bool] Enables/Disables backoff strategy
+      # @option max_retry_delay [Integer] Maximum amount of time a retry will take in ms
+      # @option retry_back_off_grace [Integer] Amount of retries to wait before starting to backoff
+      # @option retry_back_off_weight [Integer] Multiplier for the backoff retry
+      # @option retry_delay [Integer] Amount of time to wait until retry in ms
+      # @return [Integer] the calculated visibility timeout in seconds
+      def call(retry_attempts:, queue_settings:)
+        @retry_attempts       = retry_attempts
+
+        @allow_retry_back_off = queue_settings.fetch(:allow_retry_back_off)
+        @max_retry_delay      = queue_settings.fetch(:max_retry_delay)
+        @retry_back_off_grace = queue_settings.fetch(:retry_back_off_grace)
+        @retry_back_off_weight= queue_settings.fetch(:retry_back_off_weight)
+        @retry_delay          = queue_settings.fetch(:retry_delay)
+
+        if @allow_retry_back_off && retry_past_grace_period?
+          visibility_timeout = timeout_with_back_off
+          visibility_timeout = check_for_max_timeout(visibility_timeout)
+        else
+          visibility_timeout = timeout_without_back_off
+        end
+
+        visibility_timeout
+      end
+
+      private
+
+      attr_reader :logger
+
+      def retry_past_grace_period?
+        @retry_attempts > @retry_back_off_grace
+      end
+
+      def timeout_without_back_off
+        ms_to_seconds(@retry_delay)
+      end
+
+      def timeout_with_back_off
+        factor = @retry_attempts - @retry_back_off_grace
+
+        visibility_timeout = ms_to_seconds(@retry_delay * factor * @retry_back_off_weight)
+        max_retry_delay = ms_to_seconds(@max_retry_delay)
+
+        if visibility_timeout > max_retry_delay
+          logger.debug { "[#{self.class}] - Max message back off retry delay reached: #{max_retry_delay}" }
+          visibility_timeout = max_retry_delay
+        end
+
+        visibility_timeout
+      end
+
+      def ms_to_seconds(value)
+        value / 1000
+      end
+
+      def check_for_max_timeout(visibility_timeout)
+        if visibility_timeout > @max_timeout
+          logger.debug { "[#{self.class}] - AWS max visibility timeout of 12 hours has been exceeded. Setting message retry delay to 12 hours." }
+          visibility_timeout = @max_timeout
+        end
+        visibility_timeout
+      end
+    end
+  end
+end

data/lib/eventq/eventq_aws/aws_queue_worker.rb

@@ -9,12 +9,16 @@ module EventQ
 
       APPROXIMATE_RECEIVE_COUNT = 'ApproximateReceiveCount'
       MESSAGE = 'Message'
+      AWS_MAX_VISIBILITY_TIMEOUT = 43_200 # 12h
 
       attr_accessor :context
 
       def initialize
         @serialization_provider_manager = EventQ::SerializationProviders::Manager.new
         @signature_provider_manager = EventQ::SignatureProviders::Manager.new
+        @calculate_visibility_timeout = Amazon::CalculateVisibilityTimeout.new(
+          max_timeout: AWS_MAX_VISIBILITY_TIMEOUT
+        )
       end
 
       def pre_process(context, options)
@@ -114,23 +118,16 @@ module EventQ
 
           EventQ.logger.info("[#{self.class}] - Message rejected requesting retry. Attempts: #{retry_attempts}")
 
-          retry_attempts = retry_attempts - queue.retry_back_off_grace
-          retry_attempts = 1 if retry_attempts < 1
-
-          if queue.allow_retry_back_off == true
-            visibility_timeout = (queue.retry_delay * retry_attempts) / 1000
-            if visibility_timeout > (queue.max_retry_delay / 1000)
-              EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
-              visibility_timeout = queue.max_retry_delay / 1000
-            end
-          else
-            visibility_timeout = queue.retry_delay / 1000
-          end
-
-          if visibility_timeout > 43200
-            EventQ.logger.debug { "[#{self.class}] - AWS max visibility timeout of 12 hours has been exceeded. Setting message retry delay to 12 hours." }
-            visibility_timeout = 43200
-          end
+          visibility_timeout = @calculate_visibility_timeout.call(
+            retry_attempts:       retry_attempts,
+            queue_settings: {
+              allow_retry_back_off:  queue.allow_retry_back_off,
+              max_retry_delay:       queue.max_retry_delay,
+              retry_back_off_grace:  queue.retry_back_off_grace,
+              retry_back_off_weight: queue.retry_back_off_weight,
+              retry_delay:           queue.retry_delay
+            }
+          )
 
           EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{visibility_timeout}" }
           poller.change_message_visibility_timeout(msg, visibility_timeout)

data/lib/eventq/eventq_base/queue.rb

@@ -10,6 +10,7 @@ module EventQ
     attr_accessor :require_signature
     attr_accessor :retry_delay
     attr_accessor :retry_back_off_grace
+    attr_accessor :retry_back_off_weight
 
     def initialize
       @allow_retry = false
@@ -25,6 +26,8 @@ module EventQ
       @retry_delay = 30000
       # This is the amount of times to allow retry to occurr before back off is implemented
       @retry_back_off_grace = 0
+      # Multiplier for the backoff retry in case retry_delay is too small
+      @retry_back_off_weight = 1
     end
   end
 end

data/lib/eventq/eventq_base/serialization_providers/oj_serialization_provider.rb

@@ -3,7 +3,6 @@ module EventQ
     class OjSerializationProvider
 
       def initialize
-        require 'oj'
         @json_serializer = EventQ::SerializationProviders::JsonSerializationProvider.new
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventq
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.1.0
 platform: ruby
 authors:
 - SageOne
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-30 00:00:00.000000000 Z
+date: 2018-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -205,6 +205,7 @@ files:
 - bin/setup
 - lib/eventq.rb
 - lib/eventq/aws.rb
+- lib/eventq/eventq_aws/aws_calculate_visibility_timeout.rb
 - lib/eventq/eventq_aws/aws_eventq_client.rb
 - lib/eventq/eventq_aws/aws_queue_client.rb
 - lib/eventq/eventq_aws/aws_queue_manager.rb