nagare-redis 0.1.1 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce1c0837c43e0622c3472886a4e994ba6148dffe7d7d5db96dc0b63aff0db0bb
-  data.tar.gz: b885c670da1a7c41e25cf92bfc3be24bdfde8cbac52d8c693dca48c76a6bdfd5
+  metadata.gz: f7c85971f2e4fda73c0c03700cc54c07e2b2b40dcf508b6f6f6f95f9e595aba4
+  data.tar.gz: d0b1617d063bd24878e6a59abac5d5a64f2e8a6311992844d01ec4646bc67adb
 SHA512:
-  metadata.gz: a1c6b90dc57c92dc9f36fe64d84e14a7ada92becb47cec9359d004833a10377757f1de74ba88bd6b28cdce80891b0df9503ec1921631ab5cb8693b35b8c6df12
-  data.tar.gz: 6434761604072b94a3f9d0634b03a4a68dc8e31b973bd46771c78fc8e918092b4cc9e3c56e2caa22b496439705c7ff63968f7907e9241a80045c74c8ce68dd88
+  metadata.gz: 37907d60c4a97ae9d16f2bc2aa5f4054b769fe33ce91145c63aba78b5d09d3c2efe21240154de6f69056667f11c0f9482c4fdc68df7dda1abeea1facaf3e5238
+  data.tar.gz: cfcace1bce2ab24afaad1568616b23ad91aac10e5780bd33f2c82d938844720234e4a31acb12d3f12f4fcc157c812f06ae99e31d6510b552fd85ff502f8a8d37

data/.github/workflows/release.yml

@@ -0,0 +1,43 @@
+name: release
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  release-please:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: GoogleCloudPlatform/release-please-action@v2
+        id: release
+        with:
+          release-type: ruby
+          package-name: nagare-redis
+          bump-minor-pre-major: true
+          version-file: "lib/nagare/version.rb"
+      # Checkout code if release was created
+      - uses: actions/checkout@v2
+        if: ${{ steps.release.outputs.release_created }}
+      # Setup ruby if a release was created
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.6.8
+        if: ${{ steps.release.outputs.release_created }}
+      # Bundle install
+      - run: bundle install
+        if: ${{ steps.release.outputs.release_created }}
+      # Publish
+      - name: publish gem
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem
+        env:
+          # Make sure to update the secret name
+          # if yours isn't named RUBYGEMS_AUTH_TOKEN
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
+        if: ${{ steps.release.outputs.release_created }}

data/.github/workflows/run-tests.yml

@@ -15,9 +15,9 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Set up Ruby 2.6
-        uses: actions/setup-ruby@v1
+        uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.6.6
+          ruby-version: 2.6.8
 
       - uses: actions/cache@v1
         with:

data/CHANGELOG.md

@@ -0,0 +1,15 @@
+# Changelog
+
+## [0.3.0](https://www.github.com/vavato-be/nagare/compare/v0.2.0...v0.3.0) (2021-07-13)
+
+
+### Features
+
+* document release process ([eb973e5](https://www.github.com/vavato-be/nagare/commit/eb973e526156dfa89344671060044e6d8560e0e6))
+
+## [0.2.0](https://www.github.com/vavato-be/nagare/compare/v0.1.2...v0.2.0) (2021-07-13)
+
+
+### Features
+
+* create PR for release after merge on main ([f562d62](https://www.github.com/vavato-be/nagare/commit/f562d6216e45b63d1594a425b04faca3acad8857))

data/Gemfile

@@ -4,4 +4,5 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'rake', '~> 12.0'
+gem 'redis', github: 'vavato-be/redis-rb'
 gem 'rspec', '~> 3.0'

data/Gemfile.lock

@@ -1,22 +1,27 @@
+GIT
+  remote: https://github.com/vavato-be/redis-rb.git
+  revision: 79f36a58e69beb72f3bcef2ae584b30324fa2f07
+  specs:
+    redis (6.2.0)
+
 PATH
   remote: .
   specs:
-    nagare-redis (0.1.1)
-      redis (~> 4.2, >= 4.1.0)
+    nagare-redis (0.1.5)
+      redis (~> 6.2, >= 6.2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.1)
+    ast (2.4.2)
     diff-lcs (1.3)
-    parallel (1.19.2)
-    parser (2.7.1.4)
+    parallel (1.20.1)
+    parser (3.0.2.0)
       ast (~> 2.4.1)
     rainbow (3.0.0)
     rake (12.3.2)
-    redis (4.2.1)
-    regexp_parser (1.7.1)
-    rexml (3.2.4)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -30,21 +35,22 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
-    rubocop (0.88.0)
+    rubocop (1.18.3)
       parallel (~> 1.10)
-      parser (>= 2.7.1.1)
+      parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.7)
+      regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 0.1.0, < 1.0)
+      rubocop-ast (>= 1.7.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.2.0)
-      parser (>= 2.7.0.1)
-    rubocop-rspec (1.42.0)
-      rubocop (>= 0.87.0)
-    ruby-progressbar (1.10.1)
-    unicode-display_width (1.7.0)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.7.0)
+      parser (>= 3.0.1.1)
+    rubocop-rspec (2.4.0)
+      rubocop (~> 1.0)
+      rubocop-ast (>= 1.1.0)
+    ruby-progressbar (1.11.0)
+    unicode-display_width (2.0.0)
 
 PLATFORMS
   ruby
@@ -52,9 +58,10 @@ PLATFORMS
 DEPENDENCIES
   nagare-redis!
   rake (~> 12.0)
+  redis!
   rspec (~> 3.0)
-  rubocop (~> 0.88, >= 0.88)
-  rubocop-rspec (~> 1.42, >= 1.42.0)
+  rubocop (~> 1.18.3, >= 1.18.3)
+  rubocop-rspec (~> 2.4.0, >= 2.4.0)
 
 BUNDLED WITH
    2.1.0

data/README.md

@@ -25,7 +25,7 @@ information on how this works see
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'nagare'
+gem 'nagare-redis'
 ```
 
 And then execute:
@@ -40,20 +40,25 @@ To use with rails, add nagare to the initializers:
 #### config/initializers/nagare.rb
 ```ruby
 Nagare.configure do |config|
-  # After x seconds a consumer is considered dead and its messages
-  # are assigned to a different consumer in the group. Configuring
+  # After x milisseconds a pending message is considered failed and
+  # gets retried by a different consumer in the group. Configuring
   # it too low might cause double processing of messages as a consumer
   # "steals" the load of another while the first one is still processing
   # it and hasn't had the chance to ACK, configuring it too high will 
   # introduce latency in your processing.
-  # Default: 300 (5 minutes)
-  config.dead_consumer_timeout = 600
+  # Default: 600.000 (10 minutes)
+  config.min_idle_time = 600_0000
 
   # This is the consumer group name that will be used or created in
   # Redis. Use a different group for every microservice / application
   # Default: Rails.env
   config.group_name = :monolith
 
+  # A suffix is supported in order to separate different environments
+  # within the same redis database. Useful for development/test. This
+  # gets added automatically to stream names and consumer group names.
+  config.suffix = ''
+
   # URL to connect to redis. Defaults to redis://localhost:6379 uses 
   # ENV['REDIS_URL'] if present.
   config.redis_url = 'redis://10.1.1.1:6379'
@@ -106,7 +111,7 @@ end
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). A merge to the `main` branch will automatically open a PR to release a new version of nagare. Merging that PR will release to rubygems.
 
 ## Contributing
 

data/exe/nagare

@@ -2,8 +2,8 @@
 # frozen_string_literal: true
 
 if File.exist?('config/environment.rb')
-  # Load rails env if inside a rails appa
-  require_relative '../config/environment'
+  # Load rails env if inside a rails app
+  require_relative File.join(Dir.pwd, 'config', 'environment')
 elsif File.exist?('Gemfile.lock')
   # Load bundler context if using bundler
   require 'bundler/setup'

data/lib/nagare.rb

@@ -2,6 +2,7 @@
 
 require 'logger'
 require 'json'
+require 'redis'
 require 'nagare/version'
 require 'nagare/config'
 require 'nagare/redis_streams'

data/lib/nagare/config.rb

@@ -5,13 +5,13 @@ module Nagare
   # See the README for possible values and what they do
   class Config
     class << self
-      attr_accessor :dead_consumer_timeout, :group_name, :redis_url, :threads,
-                    :suffix
+      attr_accessor :group_name, :redis_url, :threads, :suffix, :min_idle_time
 
       # Runs code in the block passed in to configure Nagare and sets defaults
       # when values are not set.
       #
-      # returns Nagare::Config self
+      # returns [Nagare::Config] self
+      # rubocop:disable Metrics/CyclomaticComplexity
       def configure
         yield(self)
         @dead_consumer_timeout ||= 5000
@@ -19,8 +19,10 @@ module Nagare
         @redis_url = redis_url || ENV['REDIS_URL'] || 'redis://localhost:6379'
         @threads ||= 1
         @suffix ||= nil
+        @min_idle_time ||= 600_000
         self
       end
+      # rubocop:enable Metrics/CyclomaticComplexity
     end
   end
 end

data/lib/nagare/listener.rb

@@ -43,6 +43,7 @@ module Nagare
     # The ClassMethods module is automatically loaded into child classes
     # effectively adding the `stream` class method to the child class.`
     def self.inherited(subclass)
+      super
       subclass.extend(ClassMethods)
     end
 

data/lib/nagare/listener_pool.rb

@@ -59,8 +59,13 @@ module Nagare
       private
 
       def poll_stream(stream, listeners)
-        # TODO: Use thread pool
-        messages = Nagare::RedisStreams.read_next_messages(stream, group)
+        return unless Nagare::RedisStreams.group_exists?(stream, group)
+
+        messages = Nagare::RedisStreams.claim_next_stuck_message(stream, group)
+
+        if messages.nil? || messages.empty?
+          messages = Nagare::RedisStreams.read_next_messages(stream, group)
+        end
         return unless messages.any?
 
         messages.each do |message|
@@ -68,13 +73,16 @@ module Nagare
         end
       end
 
+      def claim_pending_messages(stream)
+        return nil unless Nagare::RedisStreams.group_exists?(stream, group)
+      end
+
       def deliver_message(stream, message, listeners)
         listener_failed = false
 
         listeners.each do |listener|
           invoke_listener(stream, message, listener)
         rescue StandardError => e
-          # TODO: Retry logic
           logger.error e.message
           logger.error e.backtrace.join("\n")
           listener_failed = true

data/lib/nagare/redis_streams.rb

@@ -30,7 +30,6 @@ module Nagare
       # @param group [String] name of the group
       #
       # @return [Boolean] true if the group exists, otherwise false
-      # rubocop:disable Metrics/AbcSize
       def group_exists?(stream, group)
         stream = stream_name(stream)
         info = connection.xinfo(:groups, stream.to_s)
@@ -41,7 +40,6 @@ module Nagare
         logger.info e.backtrace.join("\n")
         false
       end
-      # rubocop:enable Metrics/AbcSize
 
       ##
       # Creates a group in redis for the stream using xgroup
@@ -81,6 +79,25 @@ module Nagare
         connection.xadd(stream, { "#{event_name}": data })
       end
 
+      ##
+      # Claums the next message of the consumer group that is stuck
+      # (pending and past min_idle_time since being picked up)
+      #
+      # @param stream [String] name of the stream
+      # @param group [String] name of the consumer group
+      #
+      # @return [Array[Hash]] array containing the 1 message or empty
+      def claim_next_stuck_message(stream, group)
+        stream = stream_name(stream)
+        result = connection.xautoclaim(stream,
+                                       "#{stream}-#{group}",
+                                       "#{hostname}-#{thread_id}",
+                                       Nagare::Config.min_idle_time,
+                                       '0-0',
+                                       count: 1)
+        result['entries'] || []
+      end
+
       ##
       # Reads the next messages from the consumer group in redis.
       #

data/lib/nagare/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Nagare
-  VERSION = '0.1.1'
+  VERSION = '0.1.5'
 end

data/nagare.gemspec

@@ -20,14 +20,16 @@ Gem::Specification.new do |spec|
   spec.metadata['source_code_uri'] = 'https://github.com/vavato-be/nagare.git'
   spec.metadata['changelog_uri'] = 'https://github.com/vavato-be/nagare/CHANGELOG.md'
 
-  spec.add_dependency 'redis', '~> 4.2', '>= 4.1.0'
-  spec.add_development_dependency 'rubocop', '~> 0.88', '>= 0.88'
-  spec.add_development_dependency 'rubocop-rspec', '~> 1.42', '>= 1.42.0'
+  spec.add_dependency 'redis', '~> 6.2', '>= 6.2.0'
+  spec.add_development_dependency 'rubocop', '~> 1.18.3', '>= 1.18.3'
+  spec.add_development_dependency 'rubocop-rspec', '~> 2.4.0', '>= 2.4.0'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features)/})
+    end
   end
   spec.bindir = 'exe'
   spec.executables << 'nagare'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nagare-redis
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.5
 platform: ruby
 authors:
 - Alex Reis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-30 00:00:00.000000000 Z
+date: 2021-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -16,60 +16,60 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.1.0
+        version: 6.2.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '6.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.1.0
+        version: 6.2.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '6.2'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: 1.18.3
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: 1.18.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: 1.18.3
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: 1.18.3
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.42.0
+        version: 2.4.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.42'
+        version: 2.4.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.42.0
+        version: 2.4.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.42'
+        version: 2.4.0
 description: Nagare is a wrapper around Redis Streams that enables event-driven architectures
   and pub/sub messaging withdurable subscribers
 email:
@@ -79,10 +79,12 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/release.yml"
 - ".github/workflows/run-tests.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -122,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Persistent and resilient pub/sub using Redis Streams