nagare-redis 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0145d9edce24fc677b9f622e4223ee40fe447c2a7775d021ee9ac420698e6232
+  data.tar.gz: a594d0a160818770f60f4cc6be42dd3188b7877716f3d7cb29847c0da9707211
+SHA512:
+  metadata.gz: 00a32389749cd48b1743fa8fda3e603540b3cd36871c925902cecdc51b21293823c1c457e1214a6b81a70aae674b172a0c1ca8338bc2b82d646f445bc87af66a
+  data.tar.gz: eca633d905ba7c3ac3eace599814db941f5c6bdad1c8a0fb1338486c915b4e179a9f73d44dbe1cb76c51eb2b5da28807f80e584551068604165dd0f96d35609d

data/.github/workflows/run-tests.yml

@@ -0,0 +1,53 @@
+---
+name: CI
+on: [push, pull_request]
+env:
+  CI: true
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        ports:
+        - 6379/tcp
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Ruby 2.6
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: 2.6.6
+
+      - uses: actions/cache@v1
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-
+
+      - name: Fix broken apt list
+        if: matrix == null || matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-18.04'
+        run: sudo perl -p -i -e 's#16\.04/prod xenial#18.04/prod bionic#' /etc/apt/sources.list.d/microsoft-prod.list{,.save}
+
+      - name: Bundle Install and Create DB
+        env:
+          REDIS_URL: redis://localhost:${{ job.services.redis.ports[6379] }}
+        run: |
+          gem install bundler --no-document
+          bundle config path vendor/bundle
+          bundle install --jobs 4 --retry 3 --path vendor/bundle
+
+      #- name: Perform overcommit hooks	
+        #run: |	
+          #bundle exec overcommit --install
+          #bundle exec overcommit --sign
+          #bundle exec overcommit --sign pre-commit
+          #SKIP=AuthorEmail,AuthorName bundle exec overcommit --run
+
+      - name: Run tests
+        env:
+          REDIS_URL: redis://localhost:${{ job.services.redis.ports[6379] }}
+        run: |
+          bundle exec rubocop
+          bundle exec rspec

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,80 @@
+AllCops:
+  TargetRubyVersion: 2.6
+  Exclude:
+    - 'nagare.gemspec'
+    - 'vendor/**/*'
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+
+Style/FrozenStringLiteralComment:
+  Exclude:
+    - 'Gemfile'
+    - 'Rakefile'
+    - 'bin/console'
+
+Layout/LineLength:
+  Max: 80
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+
+Style/ClassVars:
+  Exclude:
+    - 'lib/nagare/listener.rb'
+    - 'lib/nagare/publisher.rb'
+
+# Cops from latest versions
+Layout/EmptyLinesAroundAttributeAccessor:
+  Enabled: true
+Layout/SpaceAroundMethodCallOperator:
+  Enabled: true
+Lint/DeprecatedOpenSSLConstant:
+  Enabled: true
+Lint/DuplicateElsifCondition:
+  Enabled: true
+Lint/MixedRegexpCaptureTypes:
+  Enabled: true
+Lint/RaiseException:
+  Enabled: true
+Lint/StructNewOverride:
+  Enabled: true
+Style/AccessorGrouping:
+  Enabled: true
+Style/ArrayCoercion:
+  Enabled: true
+Style/BisectedAttrAccessor:
+  Enabled: true
+Style/CaseLikeIf:
+  Enabled: true
+Style/ExponentialNotation:
+  Enabled: true
+Style/HashAsLastArrayItem:
+  Enabled: true
+Style/HashEachMethods:
+  Enabled: true
+Style/HashLikeCase:
+  Enabled: true
+Style/HashTransformKeys:
+  Enabled: true
+Style/HashTransformValues:
+  Enabled: true
+Style/RedundantAssignment:
+  Enabled: true
+Style/RedundantFetchBlock:
+  Enabled: true
+Style/RedundantFileExtensionInRequire:
+  Enabled: true
+Style/RedundantRegexpCharacterClass:
+  Enabled: true
+Style/RedundantRegexpEscape:
+  Enabled: true
+Style/SlicingWithRange:
+  Enabled: true

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at alex@alexmreis.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [https://contributor-covenant.org/version/1/4][version]
+
+[homepage]: https://contributor-covenant.org
+[version]: https://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in nagare.gemspec
+gemspec
+
+gem 'rake', '~> 12.0'
+gem 'rspec', '~> 3.0'

data/Gemfile.lock

@@ -0,0 +1,60 @@
+PATH
+  remote: .
+  specs:
+    nagare-redis (0.1.0)
+      redis (~> 4.2, >= 4.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.1)
+    diff-lcs (1.3)
+    parallel (1.19.2)
+    parser (2.7.1.4)
+      ast (~> 2.4.1)
+    rainbow (3.0.0)
+    rake (12.3.2)
+    redis (4.2.1)
+    regexp_parser (1.7.1)
+    rexml (3.2.4)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    rubocop (0.88.0)
+      parallel (~> 1.10)
+      parser (>= 2.7.1.1)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.7)
+      rexml
+      rubocop-ast (>= 0.1.0, < 1.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (0.2.0)
+      parser (>= 2.7.0.1)
+    rubocop-rspec (1.42.0)
+      rubocop (>= 0.87.0)
+    ruby-progressbar (1.10.1)
+    unicode-display_width (1.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  nagare-redis!
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  rubocop (~> 0.88, >= 0.88)
+  rubocop-rspec (~> 1.42, >= 1.42.0)
+
+BUNDLED WITH
+   2.1.0

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Vavato BVBA
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,122 @@
+# nagare - A publish/subscribe library backed by Redis Streams
+
+Nagare (flow in japanese) makes it easy to work with **Redis Streams** events 
+in Ruby and Rails. This enables publish/subscribe patterns in your applications
+and can be handy to enable event-driven architectures. It may also assist in 
+the decomposition and decoupling of Rails monoliths into microservices.
+
+
+## Guarantees & Behaviour
+Nagare guarantees through the use of Redis Streams Groups exactly-once delivery
+of messages to listeners. Nagare is infinitely horizontally scalable, adding new
+servers running Nagare will add more consumers to the listener group in redis.
+
+By hooking into ActiveRecord transactions, Nagare automatically ACK's messages
+only on successful transactions, and automatically retries failed ones according
+to configuration.
+
+Nagare ensures that if a listener is removed or dies, messages are redistributed
+to other listeners as soon as they become available, based on a timeout. For more
+information on how this works see 
+[Recovering from permanent failures](https://redis.io/topics/streams-intro#recovering-from-permanent-failures)
+
+### Configuration
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'nagare'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install nagare
+
+To use with rails, add nagare to the initializers:
+#### config/initializers/nagare.rb
+```ruby
+Nagare.configure do |config|
+  # After x seconds a consumer is considered dead and its messages
+  # are assigned to a different consumer in the group. Configuring
+  # it too low might cause double processing of messages as a consumer
+  # "steals" the load of another while the first one is still processing
+  # it and hasn't had the chance to ACK, configuring it too high will 
+  # introduce latency in your processing.
+  # Default: 300 (5 minutes)
+  config.dead_consumer_timeout = 600
+
+  # This is the consumer group name that will be used or created in
+  # Redis. Use a different group for every microservice / application
+  # Default: Rails.env
+  config.group_name = :monolith
+
+  # URL to connect to redis. Defaults to redis://localhost:6379 uses 
+  # ENV['REDIS_URL'] if present.
+  config.redis_url = 'redis://10.1.1.1:6379'
+
+  # Nagare uses ruby's threading model to run listeners in parallel 
+  # and in the background
+  # Default: 3 threads
+  config.threads = 3
+end
+```
+
+## Usage
+
+### Concepts
+#### Publishers
+**Publisher** is a mixin you can add into controllers, models and services to
+produce events to be consumed by other parts of your application or other
+microservices in your landscape.
+
+##### Usage
+```ruby
+class User < ApplicationModel
+  include Nagare::Publisher
+  stream 'users'
+
+  after_commit :publish_event
+
+  def publish_event
+    publish(user_updated: self.id)
+  end
+end
+```
+
+#### Listeners
+**Listener** is a new first class citizen in the Rails world like models and 
+controllers. They receive events from Redis Stream Groups and process them.
+##### Usage
+```ruby
+class UserListener < Nagare::Listener
+  stream 'users'
+
+  def user_updated(event)
+    user = User.find(event.data)
+    Mailchimp.update_user(user)
+  end
+end
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/vavato-be/nagare. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/vavato-be/nagare/blob/master/CODE_OF_CONDUCT.md).
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Nagare project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/vavato-be/nagare/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'nagare'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/nagare

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+if File.exist?('config/environment.rb')
+  # Load rails env if inside a rails appa
+  require_relative '../config/environment'
+elsif File.exist?('Gemfile.lock')
+  # Load bundler context if using bundler
+  require 'bundler/setup'
+end
+
+require 'nagare'
+
+Nagare.logger = nil
+Nagare.logger.level = :info
+
+# TODO: Capture interrupt and wait for listeners to not be busy before shutting
+#       down
+Nagare::ListenerPool.start_listening.join

data/lib/nagare.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'json'
+require 'nagare/version'
+require 'nagare/config'
+require 'nagare/redis_streams'
+require 'nagare/listener'
+require 'nagare/publisher'
+require 'nagare/listener_pool'
+
+#
+# Nagare: Redis Streams wrapper for pub/sub with durable consumers
+# see https://github.com/vavato-be/nagare
+module Nagare
+  class << self
+    attr_writer :logger
+
+    def logger
+      @logger ||= Logger.new($stdout).tap do |log|
+        log.progname = name
+      end
+    end
+  end
+end

data/lib/nagare/config.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Nagare
+  # Configuration class for Nagare.
+  # See the README for possible values and what they do
+  class Config
+    class << self
+      attr_accessor :dead_consumer_timeout, :group_name, :redis_url, :threads,
+                    :suffix
+
+      # Runs code in the block passed in to configure Nagare and sets defaults
+      # when values are not set.
+      #
+      # returns Nagare::Config self
+      def configure
+        yield(self)
+        @dead_consumer_timeout ||= 5000
+        @group_name ||= 'nagare'
+        @redis_url = redis_url || ENV['REDIS_URL'] || 'redis://localhost:6379'
+        @threads ||= 1
+        @suffix ||= nil
+        self
+      end
+    end
+  end
+end

data/lib/nagare/listener.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative './listener_pool'
+module Nagare
+  ##
+  # Listener is a base class for your own listeners.
+  #
+  # It defines default behaviour for #handle_event, invoking a method on the
+  # listener with the same name as the event if such method exists.
+  #
+  # It also adds the `stream` class method, that when used causes the listener
+  # to register itself with the listener pool for receiveing messages.
+  class Listener
+    ##
+    # Class methods that automatically get added to inheriting classes
+    module ClassMethods
+      ##
+      # Defines the name of the stream this listener listens to.
+      #
+      # This method causes the listener to register itself with
+      # the listener pool, creating automatically a consumer group
+      # if none exists for the stream, and the stream itself if not
+      # initialized.
+      #
+      # Defining a stream is required for every listener, failing
+      # to do so will cause the listener never to be invoked.
+      #
+      # @param name [String] name of the stream the listener should listen to.
+      def stream(name)
+        class_variable_set(:@@stream_name, name)
+
+        # Force consumer group creation
+        Nagare::ListenerPool.listener_pool
+        name
+      end
+
+      def stream_name
+        class_variable_get(:@@stream_name)
+      end
+    end
+
+    ##
+    # The ClassMethods module is automatically loaded into child classes
+    # effectively adding the `stream` class method to the child class.`
+    def self.inherited(subclass)
+      subclass.extend(ClassMethods)
+    end
+
+    ##
+    # This method gets called by the ListenerPool when messages are received
+    # from redis. You may override it in your own listener if you so wish.
+    #
+    # The default implementation works based on the following convention:
+    # Listeners define methods with the name of the event they handle.
+    #
+    # Events in nagare are always stored in redis as { event_name: data }
+    def handle_event(event)
+      event_name = event.keys.first
+      Nagare.logger.debug("Received #{event}")
+      return unless respond_to?(event_name)
+
+      send(event_name, JSON.parse(event[event_name], symbolize_names: true))
+    end
+  end
+end

data/lib/nagare/listener_pool.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Nagare
+  ##
+  # ListenerPool acts both as a registry of all listeners in the application
+  # and as the polling mechanism that retrieves messages from redis using
+  # consumer groups and deivers them to registered listenersone at a time.
+  class ListenerPool
+    class << self
+      ##
+      # A registry of listeners in the format { stream: [listeners...]}
+      #
+      # @return [Hash] listeners
+      def listener_pool
+        listeners.each_with_object({}) do |listener, hash|
+          stream = listener.stream_name
+
+          unless hash.key?(listener.stream_name)
+            logger.debug "Assigned stream #{stream} - listener #{listener.name}"
+            create_and_subscribe_to_stream(stream)
+            hash[stream] = []
+          end
+          hash[stream] << listener
+          hash
+        end
+      end
+
+      def listeners
+        ObjectSpace.each_object(Class).select do |klass|
+          klass < Nagare::Listener
+        end
+      end
+
+      ##
+      # Initiates polling of redis and distribution of messages to
+      # listeners in a thread
+      #
+      # @return [Thread] the listening thread
+      def start_listening
+        logger.info 'Starting Nagare thread'
+        Thread.new do
+          loop do
+            poll
+            sleep 1
+          end
+        end
+      end
+
+      ##
+      # Polls redis for new messages on all registered streams and delivers
+      # messages to the registered listeners. If the listener does not raise any
+      # errors, automatically ACKs the message to the redis consumer group.
+      def poll
+        listener_pool.each do |stream, listeners|
+          poll_stream(stream, listeners)
+        end
+      end
+
+      private
+
+      def poll_stream(stream, listeners)
+        # TODO: Use thread pool
+        messages = Nagare::RedisStreams.read_next_messages(stream, group)
+        return unless messages.any?
+
+        messages.each do |message|
+          deliver_message(stream, message, listeners)
+        end
+      end
+
+      def deliver_message(stream, message, listeners)
+        listener_failed = false
+
+        listeners.each do |listener|
+          invoke_listener(stream, message, listener)
+        rescue StandardError => e
+          # TODO: Retry logic
+          logger.error e.message
+          logger.error e.backtrace.join("\n")
+          listener_failed = true
+          # TODO: Notify Appsignal
+        end
+
+        return if listener_failed
+
+        Nagare::RedisStreams.mark_processed(stream, group, message[0])
+      end
+
+      def invoke_listener(stream, message, listener)
+        # TODO: Transactions
+        logger.info "Invoking listener #{listener.name} for stream #{stream} "\
+          "with message #{message}"
+        listener.new.handle_event(message[1])
+      end
+
+      def logger
+        Nagare.logger
+      end
+
+      def group
+        Nagare::Config.group_name
+      end
+
+      def create_and_subscribe_to_stream(stream)
+        unless Nagare::RedisStreams.group_exists?(stream, group)
+          logger.info("Creating listener group #{group} for stream #{stream}")
+          Nagare::RedisStreams.create_group(stream, group)
+          return true
+        end
+        false
+      end
+    end
+  end
+end

data/lib/nagare/publisher.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Nagare
+  ##
+  # Publisher is a mixin that allows classes to easily publish events
+  # to a redis stream.
+  module Publisher
+    ##
+    # Class methods that get injected into a class or module that
+    # extends Publisher
+    module ClassMethods
+      attr_accessor :redis_publisher_stream
+
+      ##
+      # Defines which stream to use for publish when none is specified
+      #
+      # The stream is automatically created by Redis if it doesn't exist
+      # when a message is first published to it.
+      #
+      # Defaults to the name of the class publishing the message
+      #
+      # @param [String] name name of the stream
+      def stream(name)
+        self.redis_publisher_stream = name.to_s
+      end
+    end
+
+    ##
+    # Publishes a message to the configured stream for this class.
+    #
+    # The message is always in the format { event_name: data }
+    # hence the 2 separate parameters for this method.
+    #
+    # Event name will be used on the listener side to determine
+    # which method of the listener to invoke.
+    #
+    # @param event_name [String] event_name name of the event. If it
+    # matches a method on a listener on this stream, that method will
+    # be invoked upon receiving the message
+    #
+    # @param data       [Object] an object representing the data
+    # @param stream     [String] name of the stream to publish to
+    def publish(event_name, data, stream = nil)
+      stream ||= stream_name
+      Nagare.logger.info "Publishing to stream #{stream}: "\
+        "#{event_name}: #{data}"
+      Nagare::RedisStreams.publish(stream, event_name, data.to_json)
+    end
+
+    ##
+    # Returns the name of the configured or default stream for this
+    # publisher class.
+    #
+    # @return [String] stream name
+    def stream_name
+      own_class = self.class
+      own_class.redis_publisher_stream || own_class.name.downcase
+    end
+
+    class << self
+      def included(base)
+        base.extend ClassMethods
+      end
+    end
+  end
+end

data/lib/nagare/redis_streams.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require 'socket'
+
+module Nagare
+  ##
+  # Abstraction layer for dealing with the basic RedisStreams X... commands
+  # for interacting with streams, groups and consumers.
+  #
+  # This module may be mocked during testing if necessary, or replaced with
+  # an implementation using other technology, like kafka, AciveMQ or others.
+  #
+  # Important: Groups are always assumed to be named `<stream>-<group>`.
+  #            Consumers are always created using the hostname + thread id
+  class RedisStreams
+    class << self
+      ##
+      # Returns a connection to redis. Currently not pooled
+      #
+      # @return [Redis] a connection to redis from the redis-rb gem
+      def connection
+        # FIXME: Connection pool should come in handy
+        @connection ||= Redis.new(url: Nagare::Config.redis_url)
+      end
+
+      ##
+      # Determines wether a group already exists in redis or not using xinfo
+      #
+      # @param stream [String] name of the stream
+      # @param group [String] name of the group
+      #
+      # @return [Boolean] true if the group exists, otherwise false
+      # rubocop:disable Metrics/AbcSize
+      def group_exists?(stream, group)
+        stream = stream_name(stream)
+        info = connection.xinfo(:groups, stream.to_s)
+        info.any? { |line| line['name'] == "#{stream}-#{group}" }
+      rescue Redis::CommandError => e
+        logger.info "Seems the group doesn't exist"
+        logger.info e.message
+        logger.info e.backtrace.join("\n")
+        false
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      ##
+      # Creates a group in redis for the stream using xgroup
+      #
+      # @param stream [String] name of the stream
+      # @param group [String] name of the group
+      #
+      # @return [String] OK
+      def create_group(stream, group)
+        stream = stream_name(stream)
+        connection.xgroup(:create, stream, "#{stream}-#{group}", '$',
+                          mkstream: true)
+      end
+
+      ##
+      # Deletes a group in redis for the stream using xgroup
+      #
+      # @param stream [String] name of the stream
+      # @param group [String] name of the group
+      #
+      # @return [String] OK
+      def delete_group(stream, group)
+        stream = stream_name(stream)
+        connection.xgroup(:destroy, stream, "#{stream}-#{group}")
+      end
+
+      ##
+      # Publishes an eevent to the specified stream
+      #
+      # @param stream [String] name of the stream
+      # @param event_name [String] key of the event
+      # @param data [String] data for the event, usually in JSON format.
+      #
+      # @return [String] message id
+      def publish(stream, event_name, data)
+        stream = stream_name(stream)
+        connection.xadd(stream, { "#{event_name}": data })
+      end
+
+      ##
+      # Reads the next messages from the consumer group in redis.
+      #
+      # @returns [Array] Array of tuples with [message-id, data_as_hash]
+      def read_next_messages(stream, group)
+        stream = stream_name(stream)
+        result = connection.xreadgroup("#{stream}-#{group}",
+                                       "#{hostname}-#{thread_id}", stream, '>')
+        result[stream] || []
+      end
+
+      ##
+      # Acknowledges a message as processed in the consumer group
+      #
+      # @param stream [String] name of the stream
+      # @param group [String] name of the group
+      # @param message_id [String] the id of the message
+      #
+      # @return [Integer] number of messages processed
+      def mark_processed(stream, group, message_id)
+        stream = stream_name(stream)
+        group = "#{stream}-#{group}"
+
+        count = connection.xack(stream, group, message_id)
+        return if count == 1
+
+        raise "Message could not be ACKed in Redis: #{stream} #{group} "\
+          "#{message_id}. Return value: #{count}"
+      end
+
+      ##
+      # Reads the last message on the stream without using a consumer group
+      #
+      # @param stream [String] name of the stream
+      #
+      # @return [Array] tuple of [message-id, event]
+      def read_one(stream)
+        stream = stream_name(stream)
+        result = connection.xread(stream, [0], count: 1)
+        result[stream]&.first
+      end
+
+      ##
+      # Empties a stream for all readers, not only the consumer group
+      #
+      # @return [Integer] the number of entries actually deleted
+      def truncate(stream)
+        stream = stream_name(stream)
+        connection.xtrim(stream, 0)
+      end
+
+      def stream_name(stream)
+        suffix = Nagare::Config.suffix
+        if suffix.nil?
+          stream
+        else
+          "#{stream}-#{suffix}"
+        end
+      end
+
+      ##
+      # Query pending messages for a consumer group
+      #
+      # @return [Hash] {
+      #   "size"=>0,
+      #   "min_entry_id"=>nil,
+      #   "max_entry_id"=>nil,
+      #   "consumers"=>{}
+      # }
+      def pending(stream, group)
+        stream = stream_name(stream)
+        group = "#{stream}-#{group}"
+        connection.xpending(stream, group)
+      end
+
+      private
+
+      def logger
+        Nagare.logger
+      end
+
+      def hostname
+        Socket.gethostname
+      end
+
+      def thread_id
+        Thread.current.object_id
+      end
+    end
+  end
+end

data/lib/nagare/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Nagare
+  VERSION = '0.1.0'
+end

data/nagare.gemspec

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative 'lib/nagare/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'nagare-redis'
+  spec.version       = Nagare::VERSION
+  spec.authors       = ['Alex Reis']
+  spec.email         = ['alex@alexmreis.com']
+
+  spec.summary       = 'Persistent and resilient pub/sub using Redis Streams'
+  spec.description   = 'Nagare is a wrapper around Redis Streams that enables '\
+                       'event-driven architectures and pub/sub messaging with'\
+                       'durable subscribers'
+  spec.homepage      = 'https://github.com/vavato-be/nagare'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.6.0')
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/vavato-be/nagare.git'
+  spec.metadata['changelog_uri'] = 'https://github.com/vavato-be/nagare/CHANGELOG.md'
+
+  spec.add_dependency 'redis', '~> 4.2', '>= 4.1.0'
+  spec.add_development_dependency 'rubocop', '~> 0.88', '>= 0.88'
+  spec.add_development_dependency 'rubocop-rspec', '~> 1.42', '>= 1.42.0'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir = 'exe'
+  spec.executables << 'nagare'
+  spec.require_paths = ['lib']
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: nagare-redis
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Alex Reis
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-07-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.88'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.88'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.88'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.88'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.42.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.42'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.42.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.42'
+description: Nagare is a wrapper around Redis Streams that enables event-driven architectures
+  and pub/sub messaging withdurable subscribers
+email:
+- alex@alexmreis.com
+executables:
+- nagare
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/run-tests.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/nagare
+- lib/nagare.rb
+- lib/nagare/config.rb
+- lib/nagare/listener.rb
+- lib/nagare/listener_pool.rb
+- lib/nagare/publisher.rb
+- lib/nagare/redis_streams.rb
+- lib/nagare/version.rb
+- nagare.gemspec
+homepage: https://github.com/vavato-be/nagare
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/vavato-be/nagare
+  source_code_uri: https://github.com/vavato-be/nagare.git
+  changelog_uri: https://github.com/vavato-be/nagare/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Persistent and resilient pub/sub using Redis Streams
+test_files: []