nagare-redis 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 0145d9edce24fc677b9f622e4223ee40fe447c2a7775d021ee9ac420698e6232
4
+ data.tar.gz: a594d0a160818770f60f4cc6be42dd3188b7877716f3d7cb29847c0da9707211
5
+ SHA512:
6
+ metadata.gz: 00a32389749cd48b1743fa8fda3e603540b3cd36871c925902cecdc51b21293823c1c457e1214a6b81a70aae674b172a0c1ca8338bc2b82d646f445bc87af66a
7
+ data.tar.gz: eca633d905ba7c3ac3eace599814db941f5c6bdad1c8a0fb1338486c915b4e179a9f73d44dbe1cb76c51eb2b5da28807f80e584551068604165dd0f96d35609d
@@ -0,0 +1,53 @@
1
+ ---
2
+ name: CI
3
+ on: [push, pull_request]
4
+ env:
5
+ CI: true
6
+ jobs:
7
+ test:
8
+ runs-on: ubuntu-latest
9
+ services:
10
+ redis:
11
+ image: redis
12
+ ports:
13
+ - 6379/tcp
14
+ steps:
15
+ - uses: actions/checkout@v2
16
+
17
+ - name: Set up Ruby 2.6
18
+ uses: actions/setup-ruby@v1
19
+ with:
20
+ ruby-version: 2.6.6
21
+
22
+ - uses: actions/cache@v1
23
+ with:
24
+ path: vendor/bundle
25
+ key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }}
26
+ restore-keys: |
27
+ ${{ runner.os }}-gems-
28
+
29
+ - name: Fix broken apt list
30
+ if: matrix == null || matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-18.04'
31
+ run: sudo perl -p -i -e 's#16\.04/prod xenial#18.04/prod bionic#' /etc/apt/sources.list.d/microsoft-prod.list{,.save}
32
+
33
+ - name: Bundle Install and Create DB
34
+ env:
35
+ REDIS_URL: redis://localhost:${{ job.services.redis.ports[6379] }}
36
+ run: |
37
+ gem install bundler --no-document
38
+ bundle config path vendor/bundle
39
+ bundle install --jobs 4 --retry 3 --path vendor/bundle
40
+
41
+ #- name: Perform overcommit hooks
42
+ #run: |
43
+ #bundle exec overcommit --install
44
+ #bundle exec overcommit --sign
45
+ #bundle exec overcommit --sign pre-commit
46
+ #SKIP=AuthorEmail,AuthorName bundle exec overcommit --run
47
+
48
+ - name: Run tests
49
+ env:
50
+ REDIS_URL: redis://localhost:${{ job.services.redis.ports[6379] }}
51
+ run: |
52
+ bundle exec rubocop
53
+ bundle exec rspec
@@ -0,0 +1,11 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
9
+
10
+ # rspec failure tracking
11
+ .rspec_status
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
@@ -0,0 +1,80 @@
1
+ AllCops:
2
+ TargetRubyVersion: 2.6
3
+ Exclude:
4
+ - 'nagare.gemspec'
5
+ - 'vendor/**/*'
6
+
7
+ Metrics/BlockLength:
8
+ Exclude:
9
+ - 'spec/**/*'
10
+ - 'test/**/*'
11
+
12
+ Style/Documentation:
13
+ Exclude:
14
+ - 'spec/**/*'
15
+ - 'test/**/*'
16
+
17
+ Style/FrozenStringLiteralComment:
18
+ Exclude:
19
+ - 'Gemfile'
20
+ - 'Rakefile'
21
+ - 'bin/console'
22
+
23
+ Layout/LineLength:
24
+ Max: 80
25
+ Exclude:
26
+ - 'spec/**/*'
27
+ - 'test/**/*'
28
+
29
+ Style/ClassVars:
30
+ Exclude:
31
+ - 'lib/nagare/listener.rb'
32
+ - 'lib/nagare/publisher.rb'
33
+
34
+ # Cops from latest versions
35
+ Layout/EmptyLinesAroundAttributeAccessor:
36
+ Enabled: true
37
+ Layout/SpaceAroundMethodCallOperator:
38
+ Enabled: true
39
+ Lint/DeprecatedOpenSSLConstant:
40
+ Enabled: true
41
+ Lint/DuplicateElsifCondition:
42
+ Enabled: true
43
+ Lint/MixedRegexpCaptureTypes:
44
+ Enabled: true
45
+ Lint/RaiseException:
46
+ Enabled: true
47
+ Lint/StructNewOverride:
48
+ Enabled: true
49
+ Style/AccessorGrouping:
50
+ Enabled: true
51
+ Style/ArrayCoercion:
52
+ Enabled: true
53
+ Style/BisectedAttrAccessor:
54
+ Enabled: true
55
+ Style/CaseLikeIf:
56
+ Enabled: true
57
+ Style/ExponentialNotation:
58
+ Enabled: true
59
+ Style/HashAsLastArrayItem:
60
+ Enabled: true
61
+ Style/HashEachMethods:
62
+ Enabled: true
63
+ Style/HashLikeCase:
64
+ Enabled: true
65
+ Style/HashTransformKeys:
66
+ Enabled: true
67
+ Style/HashTransformValues:
68
+ Enabled: true
69
+ Style/RedundantAssignment:
70
+ Enabled: true
71
+ Style/RedundantFetchBlock:
72
+ Enabled: true
73
+ Style/RedundantFileExtensionInRequire:
74
+ Enabled: true
75
+ Style/RedundantRegexpCharacterClass:
76
+ Enabled: true
77
+ Style/RedundantRegexpEscape:
78
+ Enabled: true
79
+ Style/SlicingWithRange:
80
+ Enabled: true
@@ -0,0 +1,74 @@
1
+ # Contributor Covenant Code of Conduct
2
+
3
+ ## Our Pledge
4
+
5
+ In the interest of fostering an open and welcoming environment, we as
6
+ contributors and maintainers pledge to making participation in our project and
7
+ our community a harassment-free experience for everyone, regardless of age, body
8
+ size, disability, ethnicity, gender identity and expression, level of experience,
9
+ nationality, personal appearance, race, religion, or sexual identity and
10
+ orientation.
11
+
12
+ ## Our Standards
13
+
14
+ Examples of behavior that contributes to creating a positive environment
15
+ include:
16
+
17
+ * Using welcoming and inclusive language
18
+ * Being respectful of differing viewpoints and experiences
19
+ * Gracefully accepting constructive criticism
20
+ * Focusing on what is best for the community
21
+ * Showing empathy towards other community members
22
+
23
+ Examples of unacceptable behavior by participants include:
24
+
25
+ * The use of sexualized language or imagery and unwelcome sexual attention or
26
+ advances
27
+ * Trolling, insulting/derogatory comments, and personal or political attacks
28
+ * Public or private harassment
29
+ * Publishing others' private information, such as a physical or electronic
30
+ address, without explicit permission
31
+ * Other conduct which could reasonably be considered inappropriate in a
32
+ professional setting
33
+
34
+ ## Our Responsibilities
35
+
36
+ Project maintainers are responsible for clarifying the standards of acceptable
37
+ behavior and are expected to take appropriate and fair corrective action in
38
+ response to any instances of unacceptable behavior.
39
+
40
+ Project maintainers have the right and responsibility to remove, edit, or
41
+ reject comments, commits, code, wiki edits, issues, and other contributions
42
+ that are not aligned to this Code of Conduct, or to ban temporarily or
43
+ permanently any contributor for other behaviors that they deem inappropriate,
44
+ threatening, offensive, or harmful.
45
+
46
+ ## Scope
47
+
48
+ This Code of Conduct applies both within project spaces and in public spaces
49
+ when an individual is representing the project or its community. Examples of
50
+ representing a project or community include using an official project e-mail
51
+ address, posting via an official social media account, or acting as an appointed
52
+ representative at an online or offline event. Representation of a project may be
53
+ further defined and clarified by project maintainers.
54
+
55
+ ## Enforcement
56
+
57
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be
58
+ reported by contacting the project team at alex@alexmreis.com. All
59
+ complaints will be reviewed and investigated and will result in a response that
60
+ is deemed necessary and appropriate to the circumstances. The project team is
61
+ obligated to maintain confidentiality with regard to the reporter of an incident.
62
+ Further details of specific enforcement policies may be posted separately.
63
+
64
+ Project maintainers who do not follow or enforce the Code of Conduct in good
65
+ faith may face temporary or permanent repercussions as determined by other
66
+ members of the project's leadership.
67
+
68
+ ## Attribution
69
+
70
+ This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
71
+ available at [https://contributor-covenant.org/version/1/4][version]
72
+
73
+ [homepage]: https://contributor-covenant.org
74
+ [version]: https://contributor-covenant.org/version/1/4/
data/Gemfile ADDED
@@ -0,0 +1,7 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in nagare.gemspec
4
+ gemspec
5
+
6
+ gem 'rake', '~> 12.0'
7
+ gem 'rspec', '~> 3.0'
@@ -0,0 +1,60 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ nagare-redis (0.1.0)
5
+ redis (~> 4.2, >= 4.1.0)
6
+
7
+ GEM
8
+ remote: https://rubygems.org/
9
+ specs:
10
+ ast (2.4.1)
11
+ diff-lcs (1.3)
12
+ parallel (1.19.2)
13
+ parser (2.7.1.4)
14
+ ast (~> 2.4.1)
15
+ rainbow (3.0.0)
16
+ rake (12.3.2)
17
+ redis (4.2.1)
18
+ regexp_parser (1.7.1)
19
+ rexml (3.2.4)
20
+ rspec (3.9.0)
21
+ rspec-core (~> 3.9.0)
22
+ rspec-expectations (~> 3.9.0)
23
+ rspec-mocks (~> 3.9.0)
24
+ rspec-core (3.9.2)
25
+ rspec-support (~> 3.9.3)
26
+ rspec-expectations (3.9.2)
27
+ diff-lcs (>= 1.2.0, < 2.0)
28
+ rspec-support (~> 3.9.0)
29
+ rspec-mocks (3.9.1)
30
+ diff-lcs (>= 1.2.0, < 2.0)
31
+ rspec-support (~> 3.9.0)
32
+ rspec-support (3.9.3)
33
+ rubocop (0.88.0)
34
+ parallel (~> 1.10)
35
+ parser (>= 2.7.1.1)
36
+ rainbow (>= 2.2.2, < 4.0)
37
+ regexp_parser (>= 1.7)
38
+ rexml
39
+ rubocop-ast (>= 0.1.0, < 1.0)
40
+ ruby-progressbar (~> 1.7)
41
+ unicode-display_width (>= 1.4.0, < 2.0)
42
+ rubocop-ast (0.2.0)
43
+ parser (>= 2.7.0.1)
44
+ rubocop-rspec (1.42.0)
45
+ rubocop (>= 0.87.0)
46
+ ruby-progressbar (1.10.1)
47
+ unicode-display_width (1.7.0)
48
+
49
+ PLATFORMS
50
+ ruby
51
+
52
+ DEPENDENCIES
53
+ nagare-redis!
54
+ rake (~> 12.0)
55
+ rspec (~> 3.0)
56
+ rubocop (~> 0.88, >= 0.88)
57
+ rubocop-rspec (~> 1.42, >= 1.42.0)
58
+
59
+ BUNDLED WITH
60
+ 2.1.0
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2020 Vavato BVBA
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,122 @@
1
+ # nagare - A publish/subscribe library backed by Redis Streams
2
+
3
+ Nagare (flow in japanese) makes it easy to work with **Redis Streams** events
4
+ in Ruby and Rails. This enables publish/subscribe patterns in your applications
5
+ and can be handy to enable event-driven architectures. It may also assist in
6
+ the decomposition and decoupling of Rails monoliths into microservices.
7
+
8
+
9
+ ## Guarantees & Behaviour
10
+ Nagare guarantees through the use of Redis Streams Groups exactly-once delivery
11
+ of messages to listeners. Nagare is infinitely horizontally scalable, adding new
12
+ servers running Nagare will add more consumers to the listener group in redis.
13
+
14
+ By hooking into ActiveRecord transactions, Nagare automatically ACK's messages
15
+ only on successful transactions, and automatically retries failed ones according
16
+ to configuration.
17
+
18
+ Nagare ensures that if a listener is removed or dies, messages are redistributed
19
+ to other listeners as soon as they become available, based on a timeout. For more
20
+ information on how this works see
21
+ [Recovering from permanent failures](https://redis.io/topics/streams-intro#recovering-from-permanent-failures)
22
+
23
+ ### Configuration
24
+
25
+ Add this line to your application's Gemfile:
26
+
27
+ ```ruby
28
+ gem 'nagare'
29
+ ```
30
+
31
+ And then execute:
32
+
33
+ $ bundle install
34
+
35
+ Or install it yourself as:
36
+
37
+ $ gem install nagare
38
+
39
+ To use with rails, add nagare to the initializers:
40
+ #### config/initializers/nagare.rb
41
+ ```ruby
42
+ Nagare.configure do |config|
43
+ # After x seconds a consumer is considered dead and its messages
44
+ # are assigned to a different consumer in the group. Configuring
45
+ # it too low might cause double processing of messages as a consumer
46
+ # "steals" the load of another while the first one is still processing
47
+ # it and hasn't had the chance to ACK, configuring it too high will
48
+ # introduce latency in your processing.
49
+ # Default: 300 (5 minutes)
50
+ config.dead_consumer_timeout = 600
51
+
52
+ # This is the consumer group name that will be used or created in
53
+ # Redis. Use a different group for every microservice / application
54
+ # Default: Rails.env
55
+ config.group_name = :monolith
56
+
57
+ # URL to connect to redis. Defaults to redis://localhost:6379 uses
58
+ # ENV['REDIS_URL'] if present.
59
+ config.redis_url = 'redis://10.1.1.1:6379'
60
+
61
+ # Nagare uses ruby's threading model to run listeners in parallel
62
+ # and in the background
63
+ # Default: 3 threads
64
+ config.threads = 3
65
+ end
66
+ ```
67
+
68
+ ## Usage
69
+
70
+ ### Concepts
71
+ #### Publishers
72
+ **Publisher** is a mixin you can add into controllers, models and services to
73
+ produce events to be consumed by other parts of your application or other
74
+ microservices in your landscape.
75
+
76
+ ##### Usage
77
+ ```ruby
78
+ class User < ApplicationModel
79
+ include Nagare::Publisher
80
+ stream 'users'
81
+
82
+ after_commit :publish_event
83
+
84
+ def publish_event
85
+ publish(user_updated: self.id)
86
+ end
87
+ end
88
+ ```
89
+
90
+ #### Listeners
91
+ **Listener** is a new first class citizen in the Rails world like models and
92
+ controllers. They receive events from Redis Stream Groups and process them.
93
+ ##### Usage
94
+ ```ruby
95
+ class UserListener < Nagare::Listener
96
+ stream 'users'
97
+
98
+ def user_updated(event)
99
+ user = User.find(event.data)
100
+ Mailchimp.update_user(user)
101
+ end
102
+ end
103
+ ```
104
+
105
+ ## Development
106
+
107
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
108
+
109
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
110
+
111
+ ## Contributing
112
+
113
+ Bug reports and pull requests are welcome on GitHub at https://github.com/vavato-be/nagare. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/vavato-be/nagare/blob/master/CODE_OF_CONDUCT.md).
114
+
115
+
116
+ ## License
117
+
118
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
119
+
120
+ ## Code of Conduct
121
+
122
+ Everyone interacting in the Nagare project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/vavato-be/nagare/blob/master/CODE_OF_CONDUCT.md).
@@ -0,0 +1,6 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rspec/core/rake_task'
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task default: :spec
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bundler/setup'
4
+ require 'nagare'
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require 'irb'
14
+ IRB.start(__FILE__)
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ if File.exist?('config/environment.rb')
5
+ # Load rails env if inside a rails appa
6
+ require_relative '../config/environment'
7
+ elsif File.exist?('Gemfile.lock')
8
+ # Load bundler context if using bundler
9
+ require 'bundler/setup'
10
+ end
11
+
12
+ require 'nagare'
13
+
14
+ Nagare.logger = nil
15
+ Nagare.logger.level = :info
16
+
17
+ # TODO: Capture interrupt and wait for listeners to not be busy before shutting
18
+ # down
19
+ Nagare::ListenerPool.start_listening.join
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'logger'
4
+ require 'json'
5
+ require 'nagare/version'
6
+ require 'nagare/config'
7
+ require 'nagare/redis_streams'
8
+ require 'nagare/listener'
9
+ require 'nagare/publisher'
10
+ require 'nagare/listener_pool'
11
+
12
+ #
13
+ # Nagare: Redis Streams wrapper for pub/sub with durable consumers
14
+ # see https://github.com/vavato-be/nagare
15
+ module Nagare
16
+ class << self
17
+ attr_writer :logger
18
+
19
+ def logger
20
+ @logger ||= Logger.new($stdout).tap do |log|
21
+ log.progname = name
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nagare
4
+ # Configuration class for Nagare.
5
+ # See the README for possible values and what they do
6
+ class Config
7
+ class << self
8
+ attr_accessor :dead_consumer_timeout, :group_name, :redis_url, :threads,
9
+ :suffix
10
+
11
+ # Runs code in the block passed in to configure Nagare and sets defaults
12
+ # when values are not set.
13
+ #
14
+ # returns Nagare::Config self
15
+ def configure
16
+ yield(self)
17
+ @dead_consumer_timeout ||= 5000
18
+ @group_name ||= 'nagare'
19
+ @redis_url = redis_url || ENV['REDIS_URL'] || 'redis://localhost:6379'
20
+ @threads ||= 1
21
+ @suffix ||= nil
22
+ self
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative './listener_pool'
4
+ module Nagare
5
+ ##
6
+ # Listener is a base class for your own listeners.
7
+ #
8
+ # It defines default behaviour for #handle_event, invoking a method on the
9
+ # listener with the same name as the event if such method exists.
10
+ #
11
+ # It also adds the `stream` class method, that when used causes the listener
12
+ # to register itself with the listener pool for receiveing messages.
13
+ class Listener
14
+ ##
15
+ # Class methods that automatically get added to inheriting classes
16
+ module ClassMethods
17
+ ##
18
+ # Defines the name of the stream this listener listens to.
19
+ #
20
+ # This method causes the listener to register itself with
21
+ # the listener pool, creating automatically a consumer group
22
+ # if none exists for the stream, and the stream itself if not
23
+ # initialized.
24
+ #
25
+ # Defining a stream is required for every listener, failing
26
+ # to do so will cause the listener never to be invoked.
27
+ #
28
+ # @param name [String] name of the stream the listener should listen to.
29
+ def stream(name)
30
+ class_variable_set(:@@stream_name, name)
31
+
32
+ # Force consumer group creation
33
+ Nagare::ListenerPool.listener_pool
34
+ name
35
+ end
36
+
37
+ def stream_name
38
+ class_variable_get(:@@stream_name)
39
+ end
40
+ end
41
+
42
+ ##
43
+ # The ClassMethods module is automatically loaded into child classes
44
+ # effectively adding the `stream` class method to the child class.`
45
+ def self.inherited(subclass)
46
+ subclass.extend(ClassMethods)
47
+ end
48
+
49
+ ##
50
+ # This method gets called by the ListenerPool when messages are received
51
+ # from redis. You may override it in your own listener if you so wish.
52
+ #
53
+ # The default implementation works based on the following convention:
54
+ # Listeners define methods with the name of the event they handle.
55
+ #
56
+ # Events in nagare are always stored in redis as { event_name: data }
57
+ def handle_event(event)
58
+ event_name = event.keys.first
59
+ Nagare.logger.debug("Received #{event}")
60
+ return unless respond_to?(event_name)
61
+
62
+ send(event_name, JSON.parse(event[event_name], symbolize_names: true))
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,114 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nagare
4
+ ##
5
+ # ListenerPool acts both as a registry of all listeners in the application
6
+ # and as the polling mechanism that retrieves messages from redis using
7
+ # consumer groups and deivers them to registered listenersone at a time.
8
+ class ListenerPool
9
+ class << self
10
+ ##
11
+ # A registry of listeners in the format { stream: [listeners...]}
12
+ #
13
+ # @return [Hash] listeners
14
+ def listener_pool
15
+ listeners.each_with_object({}) do |listener, hash|
16
+ stream = listener.stream_name
17
+
18
+ unless hash.key?(listener.stream_name)
19
+ logger.debug "Assigned stream #{stream} - listener #{listener.name}"
20
+ create_and_subscribe_to_stream(stream)
21
+ hash[stream] = []
22
+ end
23
+ hash[stream] << listener
24
+ hash
25
+ end
26
+ end
27
+
28
+ def listeners
29
+ ObjectSpace.each_object(Class).select do |klass|
30
+ klass < Nagare::Listener
31
+ end
32
+ end
33
+
34
+ ##
35
+ # Initiates polling of redis and distribution of messages to
36
+ # listeners in a thread
37
+ #
38
+ # @return [Thread] the listening thread
39
+ def start_listening
40
+ logger.info 'Starting Nagare thread'
41
+ Thread.new do
42
+ loop do
43
+ poll
44
+ sleep 1
45
+ end
46
+ end
47
+ end
48
+
49
+ ##
50
+ # Polls redis for new messages on all registered streams and delivers
51
+ # messages to the registered listeners. If the listener does not raise any
52
+ # errors, automatically ACKs the message to the redis consumer group.
53
+ def poll
54
+ listener_pool.each do |stream, listeners|
55
+ poll_stream(stream, listeners)
56
+ end
57
+ end
58
+
59
+ private
60
+
61
+ def poll_stream(stream, listeners)
62
+ # TODO: Use thread pool
63
+ messages = Nagare::RedisStreams.read_next_messages(stream, group)
64
+ return unless messages.any?
65
+
66
+ messages.each do |message|
67
+ deliver_message(stream, message, listeners)
68
+ end
69
+ end
70
+
71
+ def deliver_message(stream, message, listeners)
72
+ listener_failed = false
73
+
74
+ listeners.each do |listener|
75
+ invoke_listener(stream, message, listener)
76
+ rescue StandardError => e
77
+ # TODO: Retry logic
78
+ logger.error e.message
79
+ logger.error e.backtrace.join("\n")
80
+ listener_failed = true
81
+ # TODO: Notify Appsignal
82
+ end
83
+
84
+ return if listener_failed
85
+
86
+ Nagare::RedisStreams.mark_processed(stream, group, message[0])
87
+ end
88
+
89
+ def invoke_listener(stream, message, listener)
90
+ # TODO: Transactions
91
+ logger.info "Invoking listener #{listener.name} for stream #{stream} "\
92
+ "with message #{message}"
93
+ listener.new.handle_event(message[1])
94
+ end
95
+
96
+ def logger
97
+ Nagare.logger
98
+ end
99
+
100
+ def group
101
+ Nagare::Config.group_name
102
+ end
103
+
104
+ def create_and_subscribe_to_stream(stream)
105
+ unless Nagare::RedisStreams.group_exists?(stream, group)
106
+ logger.info("Creating listener group #{group} for stream #{stream}")
107
+ Nagare::RedisStreams.create_group(stream, group)
108
+ return true
109
+ end
110
+ false
111
+ end
112
+ end
113
+ end
114
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nagare
4
+ ##
5
+ # Publisher is a mixin that allows classes to easily publish events
6
+ # to a redis stream.
7
+ module Publisher
8
+ ##
9
+ # Class methods that get injected into a class or module that
10
+ # extends Publisher
11
+ module ClassMethods
12
+ attr_accessor :redis_publisher_stream
13
+
14
+ ##
15
+ # Defines which stream to use for publish when none is specified
16
+ #
17
+ # The stream is automatically created by Redis if it doesn't exist
18
+ # when a message is first published to it.
19
+ #
20
+ # Defaults to the name of the class publishing the message
21
+ #
22
+ # @param [String] name name of the stream
23
+ def stream(name)
24
+ self.redis_publisher_stream = name.to_s
25
+ end
26
+ end
27
+
28
+ ##
29
+ # Publishes a message to the configured stream for this class.
30
+ #
31
+ # The message is always in the format { event_name: data }
32
+ # hence the 2 separate parameters for this method.
33
+ #
34
+ # Event name will be used on the listener side to determine
35
+ # which method of the listener to invoke.
36
+ #
37
+ # @param event_name [String] event_name name of the event. If it
38
+ # matches a method on a listener on this stream, that method will
39
+ # be invoked upon receiving the message
40
+ #
41
+ # @param data [Object] an object representing the data
42
+ # @param stream [String] name of the stream to publish to
43
+ def publish(event_name, data, stream = nil)
44
+ stream ||= stream_name
45
+ Nagare.logger.info "Publishing to stream #{stream}: "\
46
+ "#{event_name}: #{data}"
47
+ Nagare::RedisStreams.publish(stream, event_name, data.to_json)
48
+ end
49
+
50
+ ##
51
+ # Returns the name of the configured or default stream for this
52
+ # publisher class.
53
+ #
54
+ # @return [String] stream name
55
+ def stream_name
56
+ own_class = self.class
57
+ own_class.redis_publisher_stream || own_class.name.downcase
58
+ end
59
+
60
+ class << self
61
+ def included(base)
62
+ base.extend ClassMethods
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,174 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'socket'
4
+
5
+ module Nagare
6
+ ##
7
+ # Abstraction layer for dealing with the basic RedisStreams X... commands
8
+ # for interacting with streams, groups and consumers.
9
+ #
10
+ # This module may be mocked during testing if necessary, or replaced with
11
+ # an implementation using other technology, like kafka, AciveMQ or others.
12
+ #
13
+ # Important: Groups are always assumed to be named `<stream>-<group>`.
14
+ # Consumers are always created using the hostname + thread id
15
+ class RedisStreams
16
+ class << self
17
+ ##
18
+ # Returns a connection to redis. Currently not pooled
19
+ #
20
+ # @return [Redis] a connection to redis from the redis-rb gem
21
+ def connection
22
+ # FIXME: Connection pool should come in handy
23
+ @connection ||= Redis.new(url: Nagare::Config.redis_url)
24
+ end
25
+
26
+ ##
27
+ # Determines wether a group already exists in redis or not using xinfo
28
+ #
29
+ # @param stream [String] name of the stream
30
+ # @param group [String] name of the group
31
+ #
32
+ # @return [Boolean] true if the group exists, otherwise false
33
+ # rubocop:disable Metrics/AbcSize
34
+ def group_exists?(stream, group)
35
+ stream = stream_name(stream)
36
+ info = connection.xinfo(:groups, stream.to_s)
37
+ info.any? { |line| line['name'] == "#{stream}-#{group}" }
38
+ rescue Redis::CommandError => e
39
+ logger.info "Seems the group doesn't exist"
40
+ logger.info e.message
41
+ logger.info e.backtrace.join("\n")
42
+ false
43
+ end
44
+ # rubocop:enable Metrics/AbcSize
45
+
46
+ ##
47
+ # Creates a group in redis for the stream using xgroup
48
+ #
49
+ # @param stream [String] name of the stream
50
+ # @param group [String] name of the group
51
+ #
52
+ # @return [String] OK
53
+ def create_group(stream, group)
54
+ stream = stream_name(stream)
55
+ connection.xgroup(:create, stream, "#{stream}-#{group}", '$',
56
+ mkstream: true)
57
+ end
58
+
59
+ ##
60
+ # Deletes a group in redis for the stream using xgroup
61
+ #
62
+ # @param stream [String] name of the stream
63
+ # @param group [String] name of the group
64
+ #
65
+ # @return [String] OK
66
+ def delete_group(stream, group)
67
+ stream = stream_name(stream)
68
+ connection.xgroup(:destroy, stream, "#{stream}-#{group}")
69
+ end
70
+
71
+ ##
72
+ # Publishes an eevent to the specified stream
73
+ #
74
+ # @param stream [String] name of the stream
75
+ # @param event_name [String] key of the event
76
+ # @param data [String] data for the event, usually in JSON format.
77
+ #
78
+ # @return [String] message id
79
+ def publish(stream, event_name, data)
80
+ stream = stream_name(stream)
81
+ connection.xadd(stream, { "#{event_name}": data })
82
+ end
83
+
84
+ ##
85
+ # Reads the next messages from the consumer group in redis.
86
+ #
87
+ # @returns [Array] Array of tuples with [message-id, data_as_hash]
88
+ def read_next_messages(stream, group)
89
+ stream = stream_name(stream)
90
+ result = connection.xreadgroup("#{stream}-#{group}",
91
+ "#{hostname}-#{thread_id}", stream, '>')
92
+ result[stream] || []
93
+ end
94
+
95
+ ##
96
+ # Acknowledges a message as processed in the consumer group
97
+ #
98
+ # @param stream [String] name of the stream
99
+ # @param group [String] name of the group
100
+ # @param message_id [String] the id of the message
101
+ #
102
+ # @return [Integer] number of messages processed
103
+ def mark_processed(stream, group, message_id)
104
+ stream = stream_name(stream)
105
+ group = "#{stream}-#{group}"
106
+
107
+ count = connection.xack(stream, group, message_id)
108
+ return if count == 1
109
+
110
+ raise "Message could not be ACKed in Redis: #{stream} #{group} "\
111
+ "#{message_id}. Return value: #{count}"
112
+ end
113
+
114
+ ##
115
+ # Reads the last message on the stream without using a consumer group
116
+ #
117
+ # @param stream [String] name of the stream
118
+ #
119
+ # @return [Array] tuple of [message-id, event]
120
+ def read_one(stream)
121
+ stream = stream_name(stream)
122
+ result = connection.xread(stream, [0], count: 1)
123
+ result[stream]&.first
124
+ end
125
+
126
+ ##
127
+ # Empties a stream for all readers, not only the consumer group
128
+ #
129
+ # @return [Integer] the number of entries actually deleted
130
+ def truncate(stream)
131
+ stream = stream_name(stream)
132
+ connection.xtrim(stream, 0)
133
+ end
134
+
135
+ def stream_name(stream)
136
+ suffix = Nagare::Config.suffix
137
+ if suffix.nil?
138
+ stream
139
+ else
140
+ "#{stream}-#{suffix}"
141
+ end
142
+ end
143
+
144
+ ##
145
+ # Query pending messages for a consumer group
146
+ #
147
+ # @return [Hash] {
148
+ # "size"=>0,
149
+ # "min_entry_id"=>nil,
150
+ # "max_entry_id"=>nil,
151
+ # "consumers"=>{}
152
+ # }
153
+ def pending(stream, group)
154
+ stream = stream_name(stream)
155
+ group = "#{stream}-#{group}"
156
+ connection.xpending(stream, group)
157
+ end
158
+
159
+ private
160
+
161
+ def logger
162
+ Nagare.logger
163
+ end
164
+
165
+ def hostname
166
+ Socket.gethostname
167
+ end
168
+
169
+ def thread_id
170
+ Thread.current.object_id
171
+ end
172
+ end
173
+ end
174
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nagare
4
+ VERSION = '0.1.0'
5
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'lib/nagare/version'
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = 'nagare-redis'
7
+ spec.version = Nagare::VERSION
8
+ spec.authors = ['Alex Reis']
9
+ spec.email = ['alex@alexmreis.com']
10
+
11
+ spec.summary = 'Persistent and resilient pub/sub using Redis Streams'
12
+ spec.description = 'Nagare is a wrapper around Redis Streams that enables '\
13
+ 'event-driven architectures and pub/sub messaging with'\
14
+ 'durable subscribers'
15
+ spec.homepage = 'https://github.com/vavato-be/nagare'
16
+ spec.license = 'MIT'
17
+ spec.required_ruby_version = Gem::Requirement.new('>= 2.6.0')
18
+
19
+ spec.metadata['homepage_uri'] = spec.homepage
20
+ spec.metadata['source_code_uri'] = 'https://github.com/vavato-be/nagare.git'
21
+ spec.metadata['changelog_uri'] = 'https://github.com/vavato-be/nagare/CHANGELOG.md'
22
+
23
+ spec.add_dependency 'redis', '~> 4.2', '>= 4.1.0'
24
+ spec.add_development_dependency 'rubocop', '~> 0.88', '>= 0.88'
25
+ spec.add_development_dependency 'rubocop-rspec', '~> 1.42', '>= 1.42.0'
26
+
27
+ # Specify which files should be added to the gem when it is released.
28
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
29
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
30
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
31
+ end
32
+ spec.bindir = 'exe'
33
+ spec.executables << 'nagare'
34
+ spec.require_paths = ['lib']
35
+ end
metadata ADDED
@@ -0,0 +1,129 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: nagare-redis
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Alex Reis
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2020-07-30 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: redis
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: 4.1.0
20
+ - - "~>"
21
+ - !ruby/object:Gem::Version
22
+ version: '4.2'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: 4.1.0
30
+ - - "~>"
31
+ - !ruby/object:Gem::Version
32
+ version: '4.2'
33
+ - !ruby/object:Gem::Dependency
34
+ name: rubocop
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ">="
38
+ - !ruby/object:Gem::Version
39
+ version: '0.88'
40
+ - - "~>"
41
+ - !ruby/object:Gem::Version
42
+ version: '0.88'
43
+ type: :development
44
+ prerelease: false
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - ">="
48
+ - !ruby/object:Gem::Version
49
+ version: '0.88'
50
+ - - "~>"
51
+ - !ruby/object:Gem::Version
52
+ version: '0.88'
53
+ - !ruby/object:Gem::Dependency
54
+ name: rubocop-rspec
55
+ requirement: !ruby/object:Gem::Requirement
56
+ requirements:
57
+ - - ">="
58
+ - !ruby/object:Gem::Version
59
+ version: 1.42.0
60
+ - - "~>"
61
+ - !ruby/object:Gem::Version
62
+ version: '1.42'
63
+ type: :development
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - ">="
68
+ - !ruby/object:Gem::Version
69
+ version: 1.42.0
70
+ - - "~>"
71
+ - !ruby/object:Gem::Version
72
+ version: '1.42'
73
+ description: Nagare is a wrapper around Redis Streams that enables event-driven architectures
74
+ and pub/sub messaging withdurable subscribers
75
+ email:
76
+ - alex@alexmreis.com
77
+ executables:
78
+ - nagare
79
+ extensions: []
80
+ extra_rdoc_files: []
81
+ files:
82
+ - ".github/workflows/run-tests.yml"
83
+ - ".gitignore"
84
+ - ".rspec"
85
+ - ".rubocop.yml"
86
+ - CODE_OF_CONDUCT.md
87
+ - Gemfile
88
+ - Gemfile.lock
89
+ - LICENSE.txt
90
+ - README.md
91
+ - Rakefile
92
+ - bin/console
93
+ - bin/setup
94
+ - exe/nagare
95
+ - lib/nagare.rb
96
+ - lib/nagare/config.rb
97
+ - lib/nagare/listener.rb
98
+ - lib/nagare/listener_pool.rb
99
+ - lib/nagare/publisher.rb
100
+ - lib/nagare/redis_streams.rb
101
+ - lib/nagare/version.rb
102
+ - nagare.gemspec
103
+ homepage: https://github.com/vavato-be/nagare
104
+ licenses:
105
+ - MIT
106
+ metadata:
107
+ homepage_uri: https://github.com/vavato-be/nagare
108
+ source_code_uri: https://github.com/vavato-be/nagare.git
109
+ changelog_uri: https://github.com/vavato-be/nagare/CHANGELOG.md
110
+ post_install_message:
111
+ rdoc_options: []
112
+ require_paths:
113
+ - lib
114
+ required_ruby_version: !ruby/object:Gem::Requirement
115
+ requirements:
116
+ - - ">="
117
+ - !ruby/object:Gem::Version
118
+ version: 2.6.0
119
+ required_rubygems_version: !ruby/object:Gem::Requirement
120
+ requirements:
121
+ - - ">="
122
+ - !ruby/object:Gem::Version
123
+ version: '0'
124
+ requirements: []
125
+ rubygems_version: 3.0.3
126
+ signing_key:
127
+ specification_version: 4
128
+ summary: Persistent and resilient pub/sub using Redis Streams
129
+ test_files: []