nagare-redis 0.1.3 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97ba30f65c114767427c1fe42b8558585e3bd4e767aa58b21300472a12787e06
-  data.tar.gz: 32e945230c09fa0a903ac34afd08a5e624ad579c2386841bbe0081b415a0f7ea
+  metadata.gz: a74be4520fe261310cc782a2232ad5270ed572c4afbd98072241ef6dc1bc118c
+  data.tar.gz: f3d9391aae29591f9652e342034ce79959498a020596b01a4d4c3d40bf35d33a
 SHA512:
-  metadata.gz: f283a8b1348c26c58e6d78f3dab2ad03ff6c70da1424826dab9fe539fb4b633b9205942ae10eb702181ac82ad2b7cf08d3da05134189ec9e508623acd0e586ad
-  data.tar.gz: 5e9ce639ab272a002955bf414b51034703238a7f364bb958bf2ff0c8e5dffccb607a8920f06df76f0afa593c77fc9ca7312ea95c1b1c651b0c052ad1941f22a6
+  metadata.gz: b9c5c5bfa26746f5a0d5c84446991aad2d3d3faf802e8c9222d61fb79b6f2aaecdf55b81654ec71571c5d978d1350bb8306c054db2e816106c9a730383c2b9e6
+  data.tar.gz: a973dc7583907a0b75d0c8e09dda57e799721eab302a3d1786e0c83293c1475baa37cbea8ccc3dbb36fd39a45e4c906bac33dea2b4980a36c7a733b7a919f96f

data/.github/workflows/release.yml

@@ -0,0 +1,43 @@
+name: release
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  release-please:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: GoogleCloudPlatform/release-please-action@v2
+        id: release
+        with:
+          release-type: ruby
+          package-name: nagare-redis
+          bump-minor-pre-major: true
+          version-file: "lib/nagare/version.rb"
+      # Checkout code if release was created
+      - uses: actions/checkout@v2
+        if: ${{ steps.release.outputs.release_created }}
+      # Setup ruby if a release was created
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.6.8
+        if: ${{ steps.release.outputs.release_created }}
+      # Bundle install
+      - run: bundle install
+        if: ${{ steps.release.outputs.release_created }}
+      # Publish
+      - name: publish gem
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem
+        env:
+          # Make sure to update the secret name
+          # if yours isn't named RUBYGEMS_AUTH_TOKEN
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
+        if: ${{ steps.release.outputs.release_created }}

data/.github/workflows/run-tests.yml

@@ -15,9 +15,9 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Set up Ruby 2.6
-        uses: actions/setup-ruby@v1
+        uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.6.6
+          ruby-version: 2.6.8
 
       - uses: actions/cache@v1
         with:

data/CHANGELOG.md

@@ -0,0 +1,22 @@
+# Changelog
+
+## [0.4.0](https://www.github.com/vavato-be/nagare/compare/v0.3.0...v0.4.0) (2021-07-13)
+
+
+### Features
+
+* Increment version number automaticaly on release ([fe5f181](https://www.github.com/vavato-be/nagare/commit/fe5f1816f65869d1ecba04ee34c4eb290e2823d4))
+
+## [0.3.0](https://www.github.com/vavato-be/nagare/compare/v0.2.0...v0.3.0) (2021-07-13)
+
+
+### Features
+
+* document release process ([eb973e5](https://www.github.com/vavato-be/nagare/commit/eb973e526156dfa89344671060044e6d8560e0e6))
+
+## [0.2.0](https://www.github.com/vavato-be/nagare/compare/v0.1.2...v0.2.0) (2021-07-13)
+
+
+### Features
+
+* create PR for release after merge on main ([f562d62](https://www.github.com/vavato-be/nagare/commit/f562d6216e45b63d1594a425b04faca3acad8857))

data/Gemfile

@@ -4,4 +4,5 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'rake', '~> 12.0'
+gem 'redis', github: 'vavato-be/redis-rb'
 gem 'rspec', '~> 3.0'

data/Gemfile.lock

@@ -1,22 +1,27 @@
+GIT
+  remote: https://github.com/vavato-be/redis-rb.git
+  revision: 79f36a58e69beb72f3bcef2ae584b30324fa2f07
+  specs:
+    redis (6.2.0)
+
 PATH
   remote: .
   specs:
-    nagare-redis (0.1.3)
-      redis (~> 4.1, >= 4.1.0)
+    nagare-redis (0.3.0)
+      redis (~> 6.2, >= 6.2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.1)
+    ast (2.4.2)
     diff-lcs (1.3)
-    parallel (1.19.2)
-    parser (2.7.1.4)
+    parallel (1.20.1)
+    parser (3.0.2.0)
       ast (~> 2.4.1)
     rainbow (3.0.0)
     rake (12.3.2)
-    redis (4.2.1)
-    regexp_parser (1.7.1)
-    rexml (3.2.4)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -30,21 +35,22 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
-    rubocop (0.88.0)
+    rubocop (1.18.3)
       parallel (~> 1.10)
-      parser (>= 2.7.1.1)
+      parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.7)
+      regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 0.1.0, < 1.0)
+      rubocop-ast (>= 1.7.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.2.0)
-      parser (>= 2.7.0.1)
-    rubocop-rspec (1.42.0)
-      rubocop (>= 0.87.0)
-    ruby-progressbar (1.10.1)
-    unicode-display_width (1.7.0)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.7.0)
+      parser (>= 3.0.1.1)
+    rubocop-rspec (2.4.0)
+      rubocop (~> 1.0)
+      rubocop-ast (>= 1.1.0)
+    ruby-progressbar (1.11.0)
+    unicode-display_width (2.0.0)
 
 PLATFORMS
   ruby
@@ -52,9 +58,10 @@ PLATFORMS
 DEPENDENCIES
   nagare-redis!
   rake (~> 12.0)
+  redis!
   rspec (~> 3.0)
-  rubocop (~> 0.88, >= 0.88)
-  rubocop-rspec (~> 1.42, >= 1.42.0)
+  rubocop (~> 1.18.3, >= 1.18.3)
+  rubocop-rspec (~> 2.4.0, >= 2.4.0)
 
 BUNDLED WITH
    2.1.0

data/README.md

@@ -40,20 +40,25 @@ To use with rails, add nagare to the initializers:
 #### config/initializers/nagare.rb
 ```ruby
 Nagare.configure do |config|
-  # After x seconds a consumer is considered dead and its messages
-  # are assigned to a different consumer in the group. Configuring
+  # After x milisseconds a pending message is considered failed and
+  # gets retried by a different consumer in the group. Configuring
   # it too low might cause double processing of messages as a consumer
   # "steals" the load of another while the first one is still processing
   # it and hasn't had the chance to ACK, configuring it too high will 
   # introduce latency in your processing.
-  # Default: 300 (5 minutes)
-  config.dead_consumer_timeout = 600
+  # Default: 600.000 (10 minutes)
+  config.min_idle_time = 600_0000
 
   # This is the consumer group name that will be used or created in
   # Redis. Use a different group for every microservice / application
   # Default: Rails.env
   config.group_name = :monolith
 
+  # A suffix is supported in order to separate different environments
+  # within the same redis database. Useful for development/test. This
+  # gets added automatically to stream names and consumer group names.
+  config.suffix = ''
+
   # URL to connect to redis. Defaults to redis://localhost:6379 uses 
   # ENV['REDIS_URL'] if present.
   config.redis_url = 'redis://10.1.1.1:6379'
@@ -62,6 +67,19 @@ Nagare.configure do |config|
   # and in the background
   # Default: 3 threads
   config.threads = 3
+
+  # Nagare can execute a proc for error handling. This enables you to
+  # use APM tools like New Relic or Appsignal with it.
+  # The proc takes 2 parameters, message and error.
+  # By default, nagare logs the error to stderr.
+  config.error_handler = proc do |message, error|
+    Appsignal.set_error(error);
+  end
+
+  # After exceeding the maximum number of retries, nagare moves the
+  # failing messages to a Dead Letter Queue stream. 
+  # By default this stream is named 'dlq', but you can customize it.
+  config.dlq_stream = 'its_dead_jim'
 end
 ```
 
@@ -106,7 +124,7 @@ end
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). A merge to the `main` branch will automatically open a PR to release a new version of nagare. Merging that PR will release to rubygems.
 
 ## Contributing
 

data/exe/nagare

@@ -2,8 +2,8 @@
 # frozen_string_literal: true
 
 if File.exist?('config/environment.rb')
-  # Load rails env if inside a rails appa
-  require_relative '../config/environment'
+  # Load rails env if inside a rails app
+  require_relative File.join(Dir.pwd, 'config', 'environment')
 elsif File.exist?('Gemfile.lock')
   # Load bundler context if using bundler
   require 'bundler/setup'

data/lib/nagare/config.rb

@@ -5,13 +5,14 @@ module Nagare
   # See the README for possible values and what they do
   class Config
     class << self
-      attr_accessor :dead_consumer_timeout, :group_name, :redis_url, :threads,
-                    :suffix
+      attr_accessor :group_name, :redis_url, :threads, :suffix, :min_idle_time,
+                    :error_handler, :dlq_stream, :max_retries
 
       # Runs code in the block passed in to configure Nagare and sets defaults
       # when values are not set.
       #
-      # returns Nagare::Config self
+      # returns [Nagare::Config] self
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
       def configure
         yield(self)
         @dead_consumer_timeout ||= 5000
@@ -19,8 +20,17 @@ module Nagare
         @redis_url = redis_url || ENV['REDIS_URL'] || 'redis://localhost:6379'
         @threads ||= 1
         @suffix ||= nil
+        @min_idle_time ||= 600_000
+        @error_handler ||= proc do |message, error|
+          Nagare.logger.error "Failed to process message #{message}"
+          Nagare.logger.error error.message
+          Nagare.logger.error error.backtrace.join("\n")
+        end
+        @dlq_stream ||= 'dlq'
+        @max_retries ||= 10
         self
       end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
     end
   end
 end

data/lib/nagare/listener.rb

@@ -43,6 +43,7 @@ module Nagare
     # The ClassMethods module is automatically loaded into child classes
     # effectively adding the `stream` class method to the child class.`
     def self.inherited(subclass)
+      super
       subclass.extend(ClassMethods)
     end
 

data/lib/nagare/listener_pool.rb

@@ -59,8 +59,13 @@ module Nagare
       private
 
       def poll_stream(stream, listeners)
-        # TODO: Use thread pool
-        messages = Nagare::RedisStreams.read_next_messages(stream, group)
+        return unless Nagare::RedisStreams.group_exists?(stream, group)
+
+        messages = Nagare::RedisStreams.claim_next_stuck_message(stream, group)
+
+        if messages.nil? || messages.empty?
+          messages = Nagare::RedisStreams.read_next_messages(stream, group)
+        end
         return unless messages.any?
 
         messages.each do |message|
@@ -68,17 +73,18 @@ module Nagare
         end
       end
 
+      def claim_pending_messages(stream)
+        return nil unless Nagare::RedisStreams.group_exists?(stream, group)
+      end
+
       def deliver_message(stream, message, listeners)
         listener_failed = false
 
         listeners.each do |listener|
           invoke_listener(stream, message, listener)
         rescue StandardError => e
-          # TODO: Retry logic
-          logger.error e.message
-          logger.error e.backtrace.join("\n")
           listener_failed = true
-          # TODO: Notify Appsignal
+          Nagare::Config.error_handler.call(message, e)
         end
 
         return if listener_failed

data/lib/nagare/redis_streams.rb

@@ -30,7 +30,6 @@ module Nagare
       # @param group [String] name of the group
       #
       # @return [Boolean] true if the group exists, otherwise false
-      # rubocop:disable Metrics/AbcSize
       def group_exists?(stream, group)
         stream = stream_name(stream)
         info = connection.xinfo(:groups, stream.to_s)
@@ -41,7 +40,6 @@ module Nagare
         logger.info e.backtrace.join("\n")
         false
       end
-      # rubocop:enable Metrics/AbcSize
 
       ##
       # Creates a group in redis for the stream using xgroup
@@ -78,7 +76,63 @@ module Nagare
       # @return [String] message id
       def publish(stream, event_name, data)
         stream = stream_name(stream)
-        connection.xadd(stream, { "#{event_name}": data })
+        connection.xadd(stream, { "#{event_name}": data.to_json })
+      end
+
+      ##
+      # Claums the next message of the consumer group that is stuck
+      # (pending and past min_idle_time since being picked up)
+      #
+      # @param stream_prefix [String] name of the stream
+      # @param group [String] name of the consumer group
+      #
+      # @return [Array[Hash]] array containing the 1 message or empty
+      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def claim_next_stuck_message(stream_prefix, group)
+        stream = stream_name(stream_prefix)
+        result = connection.xautoclaim(stream,
+                                       "#{stream}-#{group}",
+                                       "#{hostname}-#{thread_id}",
+                                       Nagare::Config.min_idle_time,
+                                       '0-0',
+                                       count: 1)
+
+        # Move message to DLQ if retried too much and get next one
+        if result['entries'].any?
+          message_id = result['entries'].first.first
+          if retry_count(stream_prefix, group,
+                         message_id) > Nagare::Config.max_retries
+            move_to_dlq(stream_prefix, group, result['entries'].first)
+            return claim_next_stuck_message(stream, group)
+          end
+        end
+
+        result['entries'] || []
+      end
+      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+
+      ##
+      # Uses XPENDING to verify the number of times the message was
+      # delivered
+      def retry_count(stream, group, message_id)
+        stream = stream_name(stream)
+        result = connection.xpending(stream,
+                                     "#{stream}-#{group}",
+                                     message_id,
+                                     message_id,
+                                     1)
+        return 0 unless result.any?
+
+        result.first['count']
+      end
+
+      ##
+      # Moves a message to the dead letter queue stream
+      def move_to_dlq(stream, group, message)
+        Nagare.logger.warn "Moving message to DLQ #{message} \
+                            from stream #{stream}"
+        publish(Nagare::Config.dlq_stream, stream, message)
+        mark_processed(stream, group, message.first)
       end
 
       ##

data/lib/nagare/version.rb

@@ -1,5 +1,11 @@
 # frozen_string_literal: true
 
+# rubocop:disable
 module Nagare
-  VERSION = '0.1.3'
+  # Needs to be double quotes for release-please ruby updater
+  # see https://github.com/googleapis/release-please/blob/master/src/updaters/version-rb.ts
+  #
+  # rubocop:disable Style/StringLiterals
+  VERSION = "0.5.0"
+  # rubocop:enable Style/StringLiterals
 end

data/nagare.gemspec

@@ -20,14 +20,16 @@ Gem::Specification.new do |spec|
   spec.metadata['source_code_uri'] = 'https://github.com/vavato-be/nagare.git'
   spec.metadata['changelog_uri'] = 'https://github.com/vavato-be/nagare/CHANGELOG.md'
 
-  spec.add_dependency 'redis', '~> 4.1', '>= 4.1.0'
-  spec.add_development_dependency 'rubocop', '~> 0.88', '>= 0.88'
-  spec.add_development_dependency 'rubocop-rspec', '~> 1.42', '>= 1.42.0'
+  spec.add_dependency 'redis', '~> 6.2', '>= 6.2.0'
+  spec.add_development_dependency 'rubocop', '~> 1.18.3', '>= 1.18.3'
+  spec.add_development_dependency 'rubocop-rspec', '~> 2.4.0', '>= 2.4.0'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features)/})
+    end
   end
   spec.bindir = 'exe'
   spec.executables << 'nagare'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nagare-redis
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.5.0
 platform: ruby
 authors:
 - Alex Reis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-10-13 00:00:00.000000000 Z
+date: 2021-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -16,60 +16,60 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.1.0
+        version: 6.2.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '6.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.1.0
+        version: 6.2.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '6.2'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: 1.18.3
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: 1.18.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: 1.18.3
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: 1.18.3
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.42.0
+        version: 2.4.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.42'
+        version: 2.4.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.42.0
+        version: 2.4.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.42'
+        version: 2.4.0
 description: Nagare is a wrapper around Redis Streams that enables event-driven architectures
   and pub/sub messaging withdurable subscribers
 email:
@@ -79,10 +79,12 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/release.yml"
 - ".github/workflows/run-tests.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock