eventq 3.3.0 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9bffbe50f80916c6ade1874a58559bb95050135c8d5c68d57492a9a9374d57b0
-  data.tar.gz: 8ced4aed2def13b2a2f972984cf818ba160d9e62bfc939f0409e03755705cbe5
+  metadata.gz: 830a90e777c5896d7d7d7f7d02bf400ec53b2e46d37235956f489719742724f1
+  data.tar.gz: 9d874da06b6190367a382ddac2cfc0dafe937bb622136474bf2430d32805af34
 SHA512:
-  metadata.gz: 38d00948a38f7ee013deb2a8dc779bffb4b764d613cad904cf2a6db66b7ff73dff4b75fa1bde5efedf7c1cfaf047a36cd1981f8f0975ce99b13b0b70dacfac84
-  data.tar.gz: 4ca6a478a61c8e7fcdcc7fc617537d7f6db73e1ff1cbf68c1d6444588aa18f3049c21957a332d67134e354d0b07139fcae89e59f3f3be665c1e8430f5590cffe
+  metadata.gz: 5ef60aecf44e22fe3ed07efd0e7d86205e3191c75c020e9a722715334598d284445fee1548cab1873868128f28556bc30fe9f5810628cb388d17add6c20bc4d9
+  data.tar.gz: 2157ca99b534f9d34156f444da0343814bc7f415649b7aeda6ae9d6c4e97d3b2ce131549bff9ebf57973a6d859fa95d03b89c2dbbd22d18dc3c4c84c04c8a370

data/README.md

@@ -54,6 +54,7 @@ A subscription queue should be defined to receive any events raised for the subs
 
  - **allow_retry** [Bool] [Optional] [Default=false] This determines if the queue should allow processing failures to be retried.
  - **allow_retry_back_off** [Bool] [Optional] [Default=false] This is used to specify if failed messages that retry should incrementally backoff.
+ - **allow_exponential_back_off** [Bool] [Optional] [Default=false] This is used to specify if failed messages that retry should expontentially backoff.
  - **retry_back_off_grace** [Int] [Optional] [Default=0] This is the number of times to allow retries without applying retry back off if enabled.
  - **dlq** [EventQ::Queue] [Optional] [Default=nil] A queue that will receive the messages which were not successfully processed after maximum number of receives by consumers. This is created at the same time as the parent queue.
  - **max_retry_attempts** [Int] [Optional] [Default=5] This is used to specify the max number of times an event should be allowed to retry before failing.
@@ -63,6 +64,7 @@ A subscription queue should be defined to receive any events raised for the subs
  - **require_signature** [Bool] [Optional] [Default=false] This is used to specify if messages within this queue must be signed.
  - **retry_delay** [Int] [Optional] [Default=30000] This is used to specify the time delay in milliseconds before a failed message is re-added to the subscription queue.
  - **retry_back_off_weight** [Int] [Optional] [Default=1] Additional multiplier for the timeout backoff. Normally used when `retry_delay` is too small (eg: 30ms) in order to get meaningful backoff values.
+ - **retry_jitter_ratio** [Int] [Optional] [Default=0] Amount of randomness for retry delays in percent to avoid a bulk of retries hitting again at the same time. 0% means no randomness, while 100% means full randomness. With full randomness, a random number between 0 and the calculated retry delay will be chosen for the delay.
 
 **Example**
 
@@ -72,7 +74,7 @@ class DataChangeAddressQueue < Queue
   def initialize
     @name = 'Data.Change.Address'
     @allow_retry = true
-    @retry_delay = 20000
+    @retry_delay = 20_000
     @max_retry_attempts = 3
   end
 end
@@ -109,10 +111,10 @@ This method is called to unsubscribe a queue.
 
 **Example**
 
-    #create an instant of the queue definition
+    #create an instance of the queue definition
 	queue = DateChangeAddressQueue.new
 
-    #subscribe the queue definition to an event type
+    #unsubscribe the queue definition
     subscription_manager.unsubscribe(queue)
 
 
@@ -330,8 +332,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 Run the setup script of eventq to build the environment. This will create the `eventq` image.
 
-    $ cd script
-    $ ./setup.sh
+    $ ./script/setup.sh
 
 ### Running the tests
 
@@ -345,13 +346,11 @@ You will also need to comment out the AWS_* environment variables in the `docker
 
 Run the whole test suite:
 
-    $ cd script
-    $ ./test.sh
+    $ ./script/test.sh
 
 You can run the specs that don't depend on an AWS account with:
 
-    $ cd script
-    $ ./test.sh --tag ~integration
+    $ ./script/test.sh --tag ~integration
 
 ## Contributing
 

data/lib/eventq/eventq_aws/aws_calculate_visibility_timeout.rb

@@ -5,7 +5,7 @@ module EventQ
     # Class responsible to know how to calculate message Visibility Timeout for Amazon SQS
     class CalculateVisibilityTimeout
       def initialize(max_timeout:, logger: EventQ.logger)
-        @max_timeout = max_timeout
+        @max_timeout = seconds_to_ms(max_timeout)
         @logger      = logger
       end
 
@@ -14,28 +14,33 @@ module EventQ
       # @param retry_attempts [Integer] Current retry
       # @param queue_settings [Hash] Queue settings
       # @option allow_retry_back_off [Bool] Enables/Disables backoff strategy
+      # @option allow_exponential_back_off [Bool] Enables/Disables exponential backoff strategy
       # @option max_retry_delay [Integer] Maximum amount of time a retry will take in ms
       # @option retry_back_off_grace [Integer] Amount of retries to wait before starting to backoff
       # @option retry_back_off_weight [Integer] Multiplier for the backoff retry
+      # @option retry_jitter_ratio [Integer] Ratio of how much jitter to apply to the backoff retry
       # @option retry_delay [Integer] Amount of time to wait until retry in ms
       # @return [Integer] the calculated visibility timeout in seconds
       def call(retry_attempts:, queue_settings:)
-        @retry_attempts       = retry_attempts
+        @retry_attempts = retry_attempts
 
-        @allow_retry_back_off = queue_settings.fetch(:allow_retry_back_off)
-        @max_retry_delay      = queue_settings.fetch(:max_retry_delay)
-        @retry_back_off_grace = queue_settings.fetch(:retry_back_off_grace)
-        @retry_back_off_weight= queue_settings.fetch(:retry_back_off_weight)
-        @retry_delay          = queue_settings.fetch(:retry_delay)
+        @allow_retry_back_off       = queue_settings.fetch(:allow_retry_back_off)
+        @allow_exponential_back_off = queue_settings.fetch(:allow_exponential_back_off)
+        @max_retry_delay            = queue_settings.fetch(:max_retry_delay)
+        @retry_back_off_grace       = queue_settings.fetch(:retry_back_off_grace)
+        @retry_back_off_weight      = queue_settings.fetch(:retry_back_off_weight)
+        @retry_jitter_ratio         = queue_settings.fetch(:retry_jitter_ratio)
+        @retry_delay                = queue_settings.fetch(:retry_delay)
 
-        if @allow_retry_back_off && retry_past_grace_period?
-          visibility_timeout = timeout_with_back_off
-          visibility_timeout = check_for_max_timeout(visibility_timeout)
+        visibility_timeout = if @allow_retry_back_off && retry_past_grace_period?
+          timeout_with_back_off
         else
-          visibility_timeout = timeout_without_back_off
+          timeout_without_back_off
         end
 
-        visibility_timeout
+        visibility_timeout = apply_jitter(visibility_timeout) if @retry_jitter_ratio > 0
+
+        ms_to_seconds(visibility_timeout)
       end
 
       private
@@ -47,34 +52,57 @@ module EventQ
       end
 
       def timeout_without_back_off
-        ms_to_seconds(@retry_delay)
+        @retry_delay
       end
 
       def timeout_with_back_off
         factor = @retry_attempts - @retry_back_off_grace
+        weighted_retry_delay = @retry_delay * @retry_back_off_weight
 
-        visibility_timeout = ms_to_seconds(@retry_delay * factor * @retry_back_off_weight)
-        max_retry_delay = ms_to_seconds(@max_retry_delay)
-
-        if visibility_timeout > max_retry_delay
-          logger.debug { "[#{self.class}] - Max message back off retry delay reached: #{max_retry_delay}" }
-          visibility_timeout = max_retry_delay
+        visibility_timeout = if @allow_exponential_back_off
+          weighted_retry_delay * 2 ** (factor - 1)
+        else
+          weighted_retry_delay * factor
         end
 
-        visibility_timeout
+        visibility_timeout = check_for_max_retry_delay(visibility_timeout)
+        check_for_max_timeout(visibility_timeout)
+      end
+
+      def apply_jitter(visibility_timeout)
+        ratio = @retry_jitter_ratio / 100.0
+        min_visibility_timeout = (visibility_timeout * (1 - ratio)).to_i
+        rand(min_visibility_timeout..visibility_timeout)
       end
 
       def ms_to_seconds(value)
         value / 1000
       end
 
+      def seconds_to_ms(value)
+        value * 1000
+      end
+
+      def check_for_max_retry_delay(visibility_timeout)
+        return visibility_timeout if visibility_timeout <= @max_retry_delay
+
+        logger.debug do
+          "[#{self.class}] - Max message back off retry delay reached: #{ms_to_seconds(@max_retry_delay)}"
+        end
+
+        @max_retry_delay
+      end
+
       def check_for_max_timeout(visibility_timeout)
-        if visibility_timeout > @max_timeout
-          logger.debug { "[#{self.class}] - AWS max visibility timeout of 12 hours has been exceeded. Setting message retry delay to 12 hours." }
-          visibility_timeout = @max_timeout
+        return visibility_timeout if visibility_timeout <= @max_timeout
+
+        logger.debug do
+          "[#{self.class}] - AWS max visibility timeout of 12 hours has been exceeded. "\
+          "Setting message retry delay to 12 hours."
         end
-        visibility_timeout
+
+        @max_timeout
       end
     end
   end
-end
+end

data/lib/eventq/eventq_aws/aws_eventq_client.rb

@@ -53,10 +53,9 @@ module EventQ
       end
 
       def raise_event(event_type, event, context = {}, region = nil)
-        register_event(event_type, region)
+        topic_arn = register_event(event_type, region)
 
         with_prepared_message(event_type, event, context) do |message|
-          topic_arn = topic_arn(event_type, region)
           response = @client.sns(region).publish(
             topic_arn: topic_arn,
             message: message,
@@ -125,10 +124,6 @@ module EventQ
         serialization_provider.serialize(queue_message)
       end
 
-      def topic_arn(event_type, region = nil)
-        @client.sns_helper(region).get_topic_arn(event_type, region)
-      end
-
       def sqs_message_body_for(payload_message)
         JSON.dump(EventQ::Amazon::QueueWorker::MESSAGE => payload_message)
       end

data/lib/eventq/eventq_aws/aws_queue_worker.rb

@@ -119,16 +119,7 @@ module EventQ
 
           EventQ.logger.warn("[#{self.class}] - Message Id: #{args.id}. Rejected requesting retry. Attempts: #{retry_attempts}")
 
-          visibility_timeout = @calculate_visibility_timeout.call(
-            retry_attempts:       retry_attempts,
-            queue_settings: {
-              allow_retry_back_off:  queue.allow_retry_back_off,
-              max_retry_delay:       queue.max_retry_delay,
-              retry_back_off_grace:  queue.retry_back_off_grace,
-              retry_back_off_weight: queue.retry_back_off_weight,
-              retry_delay:           queue.retry_delay
-            }
-          )
+          visibility_timeout = calculate_visibility_timeout(retry_attempts, queue)
 
           EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{visibility_timeout}" }
           poller.change_message_visibility_timeout(msg, visibility_timeout)
@@ -136,6 +127,21 @@ module EventQ
           context.call_on_retry_block(message)
         end
       end
+
+      def calculate_visibility_timeout(retry_attempts, queue)
+        @calculate_visibility_timeout.call(
+          retry_attempts: retry_attempts,
+          queue_settings: {
+            allow_retry_back_off:       queue.allow_retry_back_off,
+            allow_exponential_back_off: queue.allow_exponential_back_off,
+            max_retry_delay:            queue.max_retry_delay,
+            retry_back_off_grace:       queue.retry_back_off_grace,
+            retry_back_off_weight:      queue.retry_back_off_weight,
+            retry_jitter_ratio:         queue.retry_jitter_ratio,
+            retry_delay:                queue.retry_delay
+          }
+        )
+      end
     end
   end
 end

data/lib/eventq/eventq_aws/sns.rb

@@ -21,7 +21,7 @@ module EventQ
         _event_type = EventQ.create_event_type(event_type)
         topic_key = "#{region}:#{_event_type}"
 
-        arn = get_topic_arn(event_type, region)
+        arn = @@topic_arns[topic_key]
         unless arn
           response = sns.create_topic(name: aws_safe_name(_event_type))
           arn = response.topic_arn

data/lib/eventq/eventq_base/queue.rb

@@ -2,6 +2,7 @@ module EventQ
   class Queue
     attr_accessor :allow_retry
     attr_accessor :allow_retry_back_off
+    attr_accessor :allow_exponential_back_off
     attr_accessor :dlq
     attr_accessor :max_retry_attempts
     attr_accessor :max_retry_delay
@@ -11,6 +12,7 @@ module EventQ
     attr_accessor :retry_delay
     attr_accessor :retry_back_off_grace
     attr_accessor :retry_back_off_weight
+    attr_accessor :retry_jitter_ratio
     # Character delimiter between namespace and queue name.  Default = '-'
     attr_accessor :namespace_delimiter
     # Flag to control that the queue runs in isolation of auto creating the topic it belongs to
@@ -20,6 +22,8 @@ module EventQ
       @allow_retry = false
       # Default retry back off settings
       @allow_retry_back_off = false
+      # Default exponential back off settings
+      @allow_exponential_back_off = false
       # Default max receive count is 30
       @max_receive_count = 30
       # Default max retry attempts is 5
@@ -34,6 +38,8 @@ module EventQ
       @retry_back_off_grace = 0
       # Multiplier for the backoff retry in case retry_delay is too small
       @retry_back_off_weight = 1
+      # Ratio of how much jitter to apply to the retry delay
+      @retry_jitter_ratio = 0
       @isolated = false
     end
   end

data/lib/eventq/eventq_rabbitmq/rabbitmq_queue_worker.rb

@@ -78,16 +78,7 @@ module EventQ
           message.retry_attempts += 1
           retry_attempts = message.retry_attempts - queue.retry_back_off_grace
           retry_attempts = 1 if retry_attempts < 1
-
-          if queue.allow_retry_back_off == true
-            message_ttl = retry_attempts * queue.retry_delay
-            if (retry_attempts * queue.retry_delay) > queue.max_retry_delay
-              EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
-              message_ttl = queue.max_retry_delay
-            end
-          else
-            message_ttl = queue.retry_delay
-          end
+          message_ttl = retry_delay(queue, retry_attempts)
 
           EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{message_ttl}" }
           retry_exchange.publish(serialize_message(message), :expiration => message_ttl)
@@ -128,6 +119,31 @@ module EventQ
             raise "Unrecognized status: #{status}"
         end
       end
+
+      def retry_delay(queue, retry_attempts)
+        return queue.retry_delay unless queue.allow_retry_back_off
+
+        message_ttl = if queue.allow_exponential_back_off
+          queue.retry_delay * 2 ** (retry_attempts - 1)
+        else
+          queue.retry_delay * retry_attempts
+        end
+
+        if message_ttl > queue.max_retry_delay
+          EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
+          message_ttl = queue.max_retry_delay
+        end
+
+        apply_jitter(queue.retry_jitter_ratio, message_ttl)
+      end
+
+      def apply_jitter(retry_jitter_ratio, message_ttl)
+        return message_ttl if retry_jitter_ratio == 0
+
+        ratio = retry_jitter_ratio / 100.0
+        min_message_ttl = (message_ttl * (1 - ratio)).to_i
+        rand(min_message_ttl..message_ttl)
+      end
     end
   end
 end

data/lib/eventq.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'securerandom'
 require 'redlock'
 require 'class_kit'
@@ -22,6 +24,3 @@ require_relative 'eventq/eventq_base/nonce_manager'
 require_relative 'eventq/eventq_base/signature_providers'
 require_relative 'eventq/eventq_base/exceptions'
 require_relative 'eventq/queue_worker'
-
-
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventq
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 4.1.0
 platform: ruby
 authors:
 - SageOne
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-01-27 00:00:00.000000000 Z
+date: 2024-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -123,33 +123,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.18.0
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-sqs
+  name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sns
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bunny
   requirement: !ruby/object:Gem::Requirement
@@ -310,7 +324,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.5
 signing_key: 
 specification_version: 4
 summary: EventQ is a pub/sub system that uses async notifications and message queues