nagare-redis 0.4.0 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e9e2f9eff170361ad7bd5cf8177e204901661a1c2ead971a7c691cc56074a61
-  data.tar.gz: b7da8fbdfc9ef0f4df7d9d69edfc75fe00af96e2fb90e60639c76e473388e124
+  metadata.gz: 7378a63f1de40e7088716424831aff64be7a83e0c24aa9941d30cbf11de3a812
+  data.tar.gz: 26082b6b4770ebbe95815675bb237f0104fb00d8ff269e2c69ec0ffed0211108
 SHA512:
-  metadata.gz: 348df3c8aca37b15300f8e9bd167a5b70462467c220f9ac6b6e25bf074c886eeb9425e9aeb00dba1d4fbad3fb345691424e325fed1677f6f94a1bbc106bf2601
-  data.tar.gz: a42b40b705890d5b485c73af9c63afd786723552af98a7d728e9b224a4601e52e4ea6073e924b0d007d9ec771040404bb04d2395d1439568159727773774bb52
+  metadata.gz: dddaa8eb7e38f9bc7d521b2e88a656638a25a654436c89a3d6aec2e33c7ea16333112245ea803a44ffd90570dbc3e1f4432a50a1cf8be31f362566fbaa4f382c
+  data.tar.gz: 97801de9da2eefc3492b371e2c1e297876078ea6aaf227d4b5c51088d1725fae2d9dc2efc92de25b59fd6fcb4c0f5f266001d5abb50230a95bcf9b65259f645e

data/Gemfile

@@ -4,5 +4,4 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'rake', '~> 12.0'
-gem 'redis', github: 'vavato-be/redis-rb'
 gem 'rspec', '~> 3.0'

data/Gemfile.lock

@@ -1,14 +1,8 @@
-GIT
-  remote: https://github.com/vavato-be/redis-rb.git
-  revision: 79f36a58e69beb72f3bcef2ae584b30324fa2f07
-  specs:
-    redis (6.2.0)
-
 PATH
   remote: .
   specs:
-    nagare-redis (0.4.0)
-      redis (~> 6.2, >= 6.2.0)
+    nagare-redis (0.5.2)
+      redis (~> 4.4, >= 4.4.0)
 
 GEM
   remote: https://rubygems.org/
@@ -20,6 +14,7 @@ GEM
       ast (~> 2.4.1)
     rainbow (3.0.0)
     rake (12.3.2)
+    redis (4.4.0)
     regexp_parser (2.1.1)
     rexml (3.2.5)
     rspec (3.9.0)
@@ -58,7 +53,6 @@ PLATFORMS
 DEPENDENCIES
   nagare-redis!
   rake (~> 12.0)
-  redis!
   rspec (~> 3.0)
   rubocop (~> 1.18.3, >= 1.18.3)
   rubocop-rspec (~> 2.4.0, >= 2.4.0)

data/README.md

@@ -67,6 +67,19 @@ Nagare.configure do |config|
   # and in the background
   # Default: 3 threads
   config.threads = 3
+
+  # Nagare can execute a proc for error handling. This enables you to
+  # use APM tools like New Relic or Appsignal with it.
+  # The proc takes 2 parameters, message and error.
+  # By default, nagare logs the error to stderr.
+  config.error_handler = proc do |message, error|
+    Appsignal.set_error(error);
+  end
+
+  # After exceeding the maximum number of retries, nagare moves the
+  # failing messages to a Dead Letter Queue stream. 
+  # By default this stream is named 'dlq', but you can customize it.
+  config.dlq_stream = 'its_dead_jim'
 end
 ```
 

data/lib/nagare/config.rb

@@ -5,13 +5,14 @@ module Nagare
   # See the README for possible values and what they do
   class Config
     class << self
-      attr_accessor :group_name, :redis_url, :threads, :suffix, :min_idle_time
+      attr_accessor :group_name, :redis_url, :threads, :suffix, :min_idle_time,
+                    :error_handler, :dlq_stream, :max_retries
 
       # Runs code in the block passed in to configure Nagare and sets defaults
       # when values are not set.
       #
       # returns [Nagare::Config] self
-      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
       def configure
         yield(self)
         @dead_consumer_timeout ||= 5000
@@ -20,9 +21,16 @@ module Nagare
         @threads ||= 1
         @suffix ||= nil
         @min_idle_time ||= 600_000
+        @error_handler ||= proc do |message, error|
+          Nagare.logger.error "Failed to process message #{message}"
+          Nagare.logger.error error.message
+          Nagare.logger.error error.backtrace.join("\n")
+        end
+        @dlq_stream ||= 'dlq'
+        @max_retries ||= 10
         self
       end
-      # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
     end
   end
 end

data/lib/nagare/listener_pool.rb

@@ -83,10 +83,8 @@ module Nagare
         listeners.each do |listener|
           invoke_listener(stream, message, listener)
         rescue StandardError => e
-          logger.error e.message
-          logger.error e.backtrace.join("\n")
           listener_failed = true
-          # TODO: Notify Appsignal
+          Nagare::Config.error_handler.call(message, e)
         end
 
         return if listener_failed

data/lib/nagare/redis_streams.rb

@@ -12,6 +12,7 @@ module Nagare
   #
   # Important: Groups are always assumed to be named `<stream>-<group>`.
   #            Consumers are always created using the hostname + thread id
+  # rubocop:disable Metrics/ClassLength
   class RedisStreams
     class << self
       ##
@@ -83,20 +84,57 @@ module Nagare
       # Claums the next message of the consumer group that is stuck
       # (pending and past min_idle_time since being picked up)
       #
-      # @param stream [String] name of the stream
+      # @param stream_prefix [String] name of the stream
       # @param group [String] name of the consumer group
       #
       # @return [Array[Hash]] array containing the 1 message or empty
-      def claim_next_stuck_message(stream, group)
-        stream = stream_name(stream)
+      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def claim_next_stuck_message(stream_prefix, group)
+        stream = stream_name(stream_prefix)
         result = connection.xautoclaim(stream,
                                        "#{stream}-#{group}",
                                        "#{hostname}-#{thread_id}",
                                        Nagare::Config.min_idle_time,
                                        '0-0',
                                        count: 1)
+
+        # Move message to DLQ if retried too much and get next one
+        if result['entries'].any?
+          message_id = result['entries'].first.first
+          if retry_count(stream_prefix, group,
+                         message_id) > Nagare::Config.max_retries
+            move_to_dlq(stream_prefix, group, result['entries'].first)
+            return claim_next_stuck_message(stream, group)
+          end
+        end
+
         result['entries'] || []
       end
+      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+
+      ##
+      # Uses XPENDING to verify the number of times the message was
+      # delivered
+      def retry_count(stream, group, message_id)
+        stream = stream_name(stream)
+        result = connection.xpending(stream,
+                                     "#{stream}-#{group}",
+                                     message_id,
+                                     message_id,
+                                     1)
+        return 0 unless result.any?
+
+        result.first['count']
+      end
+
+      ##
+      # Moves a message to the dead letter queue stream
+      def move_to_dlq(stream, group, message)
+        Nagare.logger.warn "Moving message to DLQ #{message} \
+                            from stream #{stream}"
+        publish(Nagare::Config.dlq_stream, stream, message.to_json)
+        mark_processed(stream, group, message.first)
+      end
 
       ##
       # Reads the next messages from the consumer group in redis.
@@ -189,3 +227,4 @@ module Nagare
     end
   end
 end
+# rubocop:enable Metrics/ClassLength

data/lib/nagare/version.rb

@@ -6,6 +6,6 @@ module Nagare
   # see https://github.com/googleapis/release-please/blob/master/src/updaters/version-rb.ts
   #
   # rubocop:disable Style/StringLiterals
-  VERSION = "0.4.0"
+  VERSION = "0.5.3"
   # rubocop:enable Style/StringLiterals
 end

data/nagare.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.metadata['source_code_uri'] = 'https://github.com/vavato-be/nagare.git'
   spec.metadata['changelog_uri'] = 'https://github.com/vavato-be/nagare/CHANGELOG.md'
 
-  spec.add_dependency 'redis', '~> 6.2', '>= 6.2.0'
+  spec.add_dependency 'redis', '~> 4.4', '>= 4.4.0'
   spec.add_development_dependency 'rubocop', '~> 1.18.3', '>= 1.18.3'
   spec.add_development_dependency 'rubocop-rspec', '~> 2.4.0', '>= 2.4.0'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nagare-redis
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.3
 platform: ruby
 authors:
 - Alex Reis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-07-13 00:00:00.000000000 Z
+date: 2021-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -16,20 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.2.0
+        version: 4.4.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '4.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.2.0
+        version: 4.4.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '4.4'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -124,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Persistent and resilient pub/sub using Redis Streams