nagare-redis 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e9e2f9eff170361ad7bd5cf8177e204901661a1c2ead971a7c691cc56074a61
-  data.tar.gz: b7da8fbdfc9ef0f4df7d9d69edfc75fe00af96e2fb90e60639c76e473388e124
+  metadata.gz: a74be4520fe261310cc782a2232ad5270ed572c4afbd98072241ef6dc1bc118c
+  data.tar.gz: f3d9391aae29591f9652e342034ce79959498a020596b01a4d4c3d40bf35d33a
 SHA512:
-  metadata.gz: 348df3c8aca37b15300f8e9bd167a5b70462467c220f9ac6b6e25bf074c886eeb9425e9aeb00dba1d4fbad3fb345691424e325fed1677f6f94a1bbc106bf2601
-  data.tar.gz: a42b40b705890d5b485c73af9c63afd786723552af98a7d728e9b224a4601e52e4ea6073e924b0d007d9ec771040404bb04d2395d1439568159727773774bb52
+  metadata.gz: b9c5c5bfa26746f5a0d5c84446991aad2d3d3faf802e8c9222d61fb79b6f2aaecdf55b81654ec71571c5d978d1350bb8306c054db2e816106c9a730383c2b9e6
+  data.tar.gz: a973dc7583907a0b75d0c8e09dda57e799721eab302a3d1786e0c83293c1475baa37cbea8ccc3dbb36fd39a45e4c906bac33dea2b4980a36c7a733b7a919f96f

data/Gemfile.lock

@@ -7,7 +7,7 @@ GIT
 PATH
   remote: .
   specs:
-    nagare-redis (0.4.0)
+    nagare-redis (0.3.0)
       redis (~> 6.2, >= 6.2.0)
 
 GEM

data/README.md

@@ -67,6 +67,19 @@ Nagare.configure do |config|
   # and in the background
   # Default: 3 threads
   config.threads = 3
+
+  # Nagare can execute a proc for error handling. This enables you to
+  # use APM tools like New Relic or Appsignal with it.
+  # The proc takes 2 parameters, message and error.
+  # By default, nagare logs the error to stderr.
+  config.error_handler = proc do |message, error|
+    Appsignal.set_error(error);
+  end
+
+  # After exceeding the maximum number of retries, nagare moves the
+  # failing messages to a Dead Letter Queue stream. 
+  # By default this stream is named 'dlq', but you can customize it.
+  config.dlq_stream = 'its_dead_jim'
 end
 ```
 

data/lib/nagare/config.rb

@@ -5,13 +5,14 @@ module Nagare
   # See the README for possible values and what they do
   class Config
     class << self
-      attr_accessor :group_name, :redis_url, :threads, :suffix, :min_idle_time
+      attr_accessor :group_name, :redis_url, :threads, :suffix, :min_idle_time,
+                    :error_handler, :dlq_stream, :max_retries
 
       # Runs code in the block passed in to configure Nagare and sets defaults
       # when values are not set.
       #
       # returns [Nagare::Config] self
-      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
       def configure
         yield(self)
         @dead_consumer_timeout ||= 5000
@@ -20,9 +21,16 @@ module Nagare
         @threads ||= 1
         @suffix ||= nil
         @min_idle_time ||= 600_000
+        @error_handler ||= proc do |message, error|
+          Nagare.logger.error "Failed to process message #{message}"
+          Nagare.logger.error error.message
+          Nagare.logger.error error.backtrace.join("\n")
+        end
+        @dlq_stream ||= 'dlq'
+        @max_retries ||= 10
         self
       end
-      # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
     end
   end
 end

data/lib/nagare/listener_pool.rb

@@ -83,10 +83,8 @@ module Nagare
         listeners.each do |listener|
           invoke_listener(stream, message, listener)
         rescue StandardError => e
-          logger.error e.message
-          logger.error e.backtrace.join("\n")
           listener_failed = true
-          # TODO: Notify Appsignal
+          Nagare::Config.error_handler.call(message, e)
         end
 
         return if listener_failed

data/lib/nagare/redis_streams.rb

@@ -76,27 +76,64 @@ module Nagare
       # @return [String] message id
       def publish(stream, event_name, data)
         stream = stream_name(stream)
-        connection.xadd(stream, { "#{event_name}": data })
+        connection.xadd(stream, { "#{event_name}": data.to_json })
       end
 
       ##
       # Claums the next message of the consumer group that is stuck
       # (pending and past min_idle_time since being picked up)
       #
-      # @param stream [String] name of the stream
+      # @param stream_prefix [String] name of the stream
       # @param group [String] name of the consumer group
       #
       # @return [Array[Hash]] array containing the 1 message or empty
-      def claim_next_stuck_message(stream, group)
-        stream = stream_name(stream)
+      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def claim_next_stuck_message(stream_prefix, group)
+        stream = stream_name(stream_prefix)
         result = connection.xautoclaim(stream,
                                        "#{stream}-#{group}",
                                        "#{hostname}-#{thread_id}",
                                        Nagare::Config.min_idle_time,
                                        '0-0',
                                        count: 1)
+
+        # Move message to DLQ if retried too much and get next one
+        if result['entries'].any?
+          message_id = result['entries'].first.first
+          if retry_count(stream_prefix, group,
+                         message_id) > Nagare::Config.max_retries
+            move_to_dlq(stream_prefix, group, result['entries'].first)
+            return claim_next_stuck_message(stream, group)
+          end
+        end
+
         result['entries'] || []
       end
+      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+
+      ##
+      # Uses XPENDING to verify the number of times the message was
+      # delivered
+      def retry_count(stream, group, message_id)
+        stream = stream_name(stream)
+        result = connection.xpending(stream,
+                                     "#{stream}-#{group}",
+                                     message_id,
+                                     message_id,
+                                     1)
+        return 0 unless result.any?
+
+        result.first['count']
+      end
+
+      ##
+      # Moves a message to the dead letter queue stream
+      def move_to_dlq(stream, group, message)
+        Nagare.logger.warn "Moving message to DLQ #{message} \
+                            from stream #{stream}"
+        publish(Nagare::Config.dlq_stream, stream, message)
+        mark_processed(stream, group, message.first)
+      end
 
       ##
       # Reads the next messages from the consumer group in redis.

data/lib/nagare/version.rb

@@ -6,6 +6,6 @@ module Nagare
   # see https://github.com/googleapis/release-please/blob/master/src/updaters/version-rb.ts
   #
   # rubocop:disable Style/StringLiterals
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
   # rubocop:enable Style/StringLiterals
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nagare-redis
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Alex Reis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-07-13 00:00:00.000000000 Z
+date: 2021-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -124,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Persistent and resilient pub/sub using Redis Streams