nagare-redis 0.1.5 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7c85971f2e4fda73c0c03700cc54c07e2b2b40dcf508b6f6f6f95f9e595aba4
-  data.tar.gz: d0b1617d063bd24878e6a59abac5d5a64f2e8a6311992844d01ec4646bc67adb
+  metadata.gz: 47ac2065d72b1abf37d0855641328d767503d4ff5cde242aa046132102870271
+  data.tar.gz: '0974078409071d939db89e48f22831e7b9c495ad40a7ff1e780a0e075b608f92'
 SHA512:
-  metadata.gz: 37907d60c4a97ae9d16f2bc2aa5f4054b769fe33ce91145c63aba78b5d09d3c2efe21240154de6f69056667f11c0f9482c4fdc68df7dda1abeea1facaf3e5238
-  data.tar.gz: cfcace1bce2ab24afaad1568616b23ad91aac10e5780bd33f2c82d938844720234e4a31acb12d3f12f4fcc157c812f06ae99e31d6510b552fd85ff502f8a8d37
+  metadata.gz: cbd3d90cc9f37efd7300cd4cb155271c90bc20417e4c27712b31fb21ae8a7204bfcf49f261f7e28e9ae00f233ada2516ac5fd9b7236fc1848dfa51d8914b6201
+  data.tar.gz: 85f6cecb070588aa4ea9d19a6886378099c87faf8e5ad13c4121b2026176ca54cf089372635810d8dfc581c1537f9453e8143f80c30cbc31abf1f4a3de27f27b

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [0.4.0](https://www.github.com/vavato-be/nagare/compare/v0.3.0...v0.4.0) (2021-07-13)
+
+
+### Features
+
+* Increment version number automaticaly on release ([fe5f181](https://www.github.com/vavato-be/nagare/commit/fe5f1816f65869d1ecba04ee34c4eb290e2823d4))
+
 ## [0.3.0](https://www.github.com/vavato-be/nagare/compare/v0.2.0...v0.3.0) (2021-07-13)
 
 

data/Gemfile

@@ -4,5 +4,4 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'rake', '~> 12.0'
-gem 'redis', github: 'vavato-be/redis-rb'
 gem 'rspec', '~> 3.0'

data/Gemfile.lock

@@ -1,14 +1,8 @@
-GIT
-  remote: https://github.com/vavato-be/redis-rb.git
-  revision: 79f36a58e69beb72f3bcef2ae584b30324fa2f07
-  specs:
-    redis (6.2.0)
-
 PATH
   remote: .
   specs:
-    nagare-redis (0.1.5)
-      redis (~> 6.2, >= 6.2.0)
+    nagare-redis (0.5.1)
+      redis (~> 4.4, >= 4.4.0)
 
 GEM
   remote: https://rubygems.org/
@@ -20,6 +14,7 @@ GEM
       ast (~> 2.4.1)
     rainbow (3.0.0)
     rake (12.3.2)
+    redis (4.4.0)
     regexp_parser (2.1.1)
     rexml (3.2.5)
     rspec (3.9.0)
@@ -58,7 +53,6 @@ PLATFORMS
 DEPENDENCIES
   nagare-redis!
   rake (~> 12.0)
-  redis!
   rspec (~> 3.0)
   rubocop (~> 1.18.3, >= 1.18.3)
   rubocop-rspec (~> 2.4.0, >= 2.4.0)

data/README.md

@@ -67,6 +67,19 @@ Nagare.configure do |config|
   # and in the background
   # Default: 3 threads
   config.threads = 3
+
+  # Nagare can execute a proc for error handling. This enables you to
+  # use APM tools like New Relic or Appsignal with it.
+  # The proc takes 2 parameters, message and error.
+  # By default, nagare logs the error to stderr.
+  config.error_handler = proc do |message, error|
+    Appsignal.set_error(error);
+  end
+
+  # After exceeding the maximum number of retries, nagare moves the
+  # failing messages to a Dead Letter Queue stream. 
+  # By default this stream is named 'dlq', but you can customize it.
+  config.dlq_stream = 'its_dead_jim'
 end
 ```
 

data/lib/nagare/config.rb

@@ -5,13 +5,14 @@ module Nagare
   # See the README for possible values and what they do
   class Config
     class << self
-      attr_accessor :group_name, :redis_url, :threads, :suffix, :min_idle_time
+      attr_accessor :group_name, :redis_url, :threads, :suffix, :min_idle_time,
+                    :error_handler, :dlq_stream, :max_retries
 
       # Runs code in the block passed in to configure Nagare and sets defaults
       # when values are not set.
       #
       # returns [Nagare::Config] self
-      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
       def configure
         yield(self)
         @dead_consumer_timeout ||= 5000
@@ -20,9 +21,16 @@ module Nagare
         @threads ||= 1
         @suffix ||= nil
         @min_idle_time ||= 600_000
+        @error_handler ||= proc do |message, error|
+          Nagare.logger.error "Failed to process message #{message}"
+          Nagare.logger.error error.message
+          Nagare.logger.error error.backtrace.join("\n")
+        end
+        @dlq_stream ||= 'dlq'
+        @max_retries ||= 10
         self
       end
-      # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
     end
   end
 end

data/lib/nagare/listener_pool.rb

@@ -83,10 +83,8 @@ module Nagare
         listeners.each do |listener|
           invoke_listener(stream, message, listener)
         rescue StandardError => e
-          logger.error e.message
-          logger.error e.backtrace.join("\n")
           listener_failed = true
-          # TODO: Notify Appsignal
+          Nagare::Config.error_handler.call(message, e)
         end
 
         return if listener_failed

data/lib/nagare/redis_streams.rb

@@ -12,6 +12,7 @@ module Nagare
   #
   # Important: Groups are always assumed to be named `<stream>-<group>`.
   #            Consumers are always created using the hostname + thread id
+  # rubocop:disable Metrics/ClassLength
   class RedisStreams
     class << self
       ##
@@ -83,20 +84,57 @@ module Nagare
       # Claums the next message of the consumer group that is stuck
       # (pending and past min_idle_time since being picked up)
       #
-      # @param stream [String] name of the stream
+      # @param stream_prefix [String] name of the stream
       # @param group [String] name of the consumer group
       #
       # @return [Array[Hash]] array containing the 1 message or empty
-      def claim_next_stuck_message(stream, group)
-        stream = stream_name(stream)
+      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def claim_next_stuck_message(stream_prefix, group)
+        stream = stream_name(stream_prefix)
         result = connection.xautoclaim(stream,
                                        "#{stream}-#{group}",
                                        "#{hostname}-#{thread_id}",
                                        Nagare::Config.min_idle_time,
                                        '0-0',
                                        count: 1)
+
+        # Move message to DLQ if retried too much and get next one
+        if result['entries'].any?
+          message_id = result['entries'].first.first
+          if retry_count(stream_prefix, group,
+                         message_id) > Nagare::Config.max_retries
+            move_to_dlq(stream_prefix, group, result['entries'].first)
+            return claim_next_stuck_message(stream, group)
+          end
+        end
+
         result['entries'] || []
       end
+      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+
+      ##
+      # Uses XPENDING to verify the number of times the message was
+      # delivered
+      def retry_count(stream, group, message_id)
+        stream = stream_name(stream)
+        result = connection.xpending(stream,
+                                     "#{stream}-#{group}",
+                                     message_id,
+                                     message_id,
+                                     1)
+        return 0 unless result.any?
+
+        result.first['count']
+      end
+
+      ##
+      # Moves a message to the dead letter queue stream
+      def move_to_dlq(stream, group, message)
+        Nagare.logger.warn "Moving message to DLQ #{message} \
+                            from stream #{stream}"
+        publish(Nagare::Config.dlq_stream, stream, message)
+        mark_processed(stream, group, message.first)
+      end
 
       ##
       # Reads the next messages from the consumer group in redis.
@@ -189,3 +227,4 @@ module Nagare
     end
   end
 end
+# rubocop:enable Metrics/ClassLength

data/lib/nagare/version.rb

@@ -1,5 +1,11 @@
 # frozen_string_literal: true
 
+# rubocop:disable
 module Nagare
-  VERSION = '0.1.5'
+  # Needs to be double quotes for release-please ruby updater
+  # see https://github.com/googleapis/release-please/blob/master/src/updaters/version-rb.ts
+  #
+  # rubocop:disable Style/StringLiterals
+  VERSION = "0.5.2"
+  # rubocop:enable Style/StringLiterals
 end

data/nagare.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.metadata['source_code_uri'] = 'https://github.com/vavato-be/nagare.git'
   spec.metadata['changelog_uri'] = 'https://github.com/vavato-be/nagare/CHANGELOG.md'
 
-  spec.add_dependency 'redis', '~> 6.2', '>= 6.2.0'
+  spec.add_dependency 'redis', '~> 4.4', '>= 4.4.0'
   spec.add_development_dependency 'rubocop', '~> 1.18.3', '>= 1.18.3'
   spec.add_development_dependency 'rubocop-rspec', '~> 2.4.0', '>= 2.4.0'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nagare-redis
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.5.2
 platform: ruby
 authors:
 - Alex Reis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-07-13 00:00:00.000000000 Z
+date: 2021-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -16,20 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.2.0
+        version: 4.4.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '4.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.2.0
+        version: 4.4.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '4.4'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -124,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Persistent and resilient pub/sub using Redis Streams