eventsimple 1.3.3 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 872c921ebca70dcc352f005afe39e0cf48ae4bb2d009f7f55acc95f0b1250f04
-  data.tar.gz: 0445db43a4641baceb36926e29e9b9eaa9c41aff5c69029d05bc2af9c457afa3
+  metadata.gz: 124fba5b3a01b411d35423265f33d4de64cf5a5824cb57491e502454557de90c
+  data.tar.gz: 9352b0c064d692bfc5f3623bf3c3a487a9c430a08f3526da2cb03ecaf36bd40c
 SHA512:
-  metadata.gz: fa5257d5083fa3a2f1f6a6e6d57e249e0a78dd8a8c4d1a257a1726a1b6939a857dde9832e1a7e25e113c8697a869de435737d836449bd5a4ccb0358da8d6d7d7
-  data.tar.gz: b89aa88f3b19f9286148631207176c828be018c78cf68291235e267be2194f01f08bf85d56be04d9fd6045f02e98f008ade400ac0e7eb2c3858059fd96b552e7
+  metadata.gz: 669a73959571590b6cc4ac32835c761d2aeb93a3b4c57e6043d79b94f5d271f523020f839845ae9f8aef0274757bb8de837d59a1822f6b51790fd582ff1e344a
+  data.tar.gz: d0740368d330aa08bfbd8e65fb7253800b1c8f055c29d9d92202b08812a79d64b74a08c8d557ab563efad57a021830a9a70fc4986b4bfd6aa652cb5e526b56db

data/CHANGELOG.md

@@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## 1.4.0 - 2024-04-02
+### Changed
+- Production ready release of the outbox consumer
+- Multiple consumers on an event stream are supported
+- The outbox processor is instantiated once and takes the event as an argument.
+- Added proper SIGTERM event handling and fixed shutdown behaviour.
+
 ## 1.3.3 - 2024-04-02
 ### Changed
 - add `parent_record` configuration so it can be easily overwritten

data/Gemfile

@@ -2,5 +2,9 @@
 
 source 'https://rubygems.org'
 
+group :test do
+  gem 'factory_bot_rails'
+end
+
 # Specify your gem's dependencies in eventsimple.gemspec
 gemspec

data/Gemfile.lock

@@ -1,12 +1,13 @@
 PATH
   remote: .
   specs:
-    eventsimple (1.3.3)
+    eventsimple (1.4.0)
       dry-struct (~> 1.6)
       dry-types (~> 1.7)
       pg (~> 1.4)
       rails (~> 7.0)
       retriable (~> 3.1)
+      with_advisory_lock (>= 5.1)
 
 GEM
   remote: https://rubygems.org/
@@ -176,7 +177,7 @@ GEM
       net-imap
       net-pop
       net-smtp
-    marcel (1.0.3)
+    marcel (1.0.4)
     method_source (1.0.0)
     mini_mime (1.1.5)
     minitest (5.22.2)
@@ -189,7 +190,7 @@ GEM
       net-protocol
     net-protocol (0.2.2)
       timeout
-    net-smtp (0.4.0.1)
+    net-smtp (0.5.0)
       net-protocol
     nio4r (2.7.0)
     nokogiri (1.16.2-arm64-darwin)
@@ -205,7 +206,7 @@ GEM
     parser (3.3.0.5)
       ast (~> 2.4.1)
       racc
-    pg (1.5.5)
+    pg (1.5.6)
     polyglot (0.3.5)
     pry (0.14.2)
       coderay (~> 1.1)
@@ -348,6 +349,9 @@ GEM
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
+    with_advisory_lock (5.1.0)
+      activerecord (>= 6.1)
+      zeitwerk (>= 2.6)
     ws-style (7.4.3)
       rubocop-rspec (>= 2.2.0)
       rubocop-vendor (>= 0.11)

data/README.md

@@ -276,14 +276,11 @@ end
 
 ## Configuring an outbox consumer
 
-For many use cases, async reactors are sufficient to handle workflows like making an API call or publishing to a message broker.
-However since reactors use ActiveJob, order is not guaranteed.
+For many use cases, async reactors are sufficient to handle workflows like making an API call or publishing to a message broker. However as reactors use ActiveJob, order is not guaranteed. For use cases requiring order, eventsimple provides an ordered outbox implementation.
 
-Eventsimple provides an outbox implementation with order and eventual consistency guarantees.
+**Caveat**: The current implementation leverages a single advisory lock to guarantee write order. This will impact write throughput on the model. On a db.rg6.large Aurora instance for example, write throughput is limited to ~300 events per second.
 
-**Caveat**: The current implementation leverages a single advisory lock to guarantee write order. This will significantly impact write throughput on the model. On a standard Aurora instance for example, write throughput is limited to ~300 events per second.
-
-For more information on why an advisory lock is required:
+For an explaination of why an advisory lock is required:
 https://github.com/pawelpacana/account-basics
 
 ### Setup an ordered outbox
@@ -297,8 +294,6 @@ Generate migration to setup the outbox cursor table. This table is used to track
 Create a consummer and processor class for the outbox.
 Note: The presence of the consumer class moves all writes to the respective events table to be written using an advisory lock.
 
-Only a single outbox consumer per events table is supported. **DO NOT** create multiple consumers for the same events table.
-
 ```ruby
 require 'eventsimple/outbox/consumer'
 
@@ -306,6 +301,7 @@ module UserComponent
   class Consumer
     extend Eventsimple::Outbox::Consumer
 
+    identitfier 'UserComponent::Consumer'
     consumes_event UserEvent
     processor EventProcessor
   end
@@ -315,12 +311,7 @@ end
 ```ruby
 module UserComponent
   class EventProcessor
-    def initialize(event)
-      @event = event
-    end
-    attr_reader :event
-
-    def call
+    def call(event)
       Rails.logger.info("PROCESSING EVENT: #{event.id}")
     end
   end
@@ -339,6 +330,12 @@ Create a rake task to run the consumer
   end
 ```
 
+To set the cursor position to the latest event:
+
+```ruby
+  Eventsimple::Outbox::Cursor.set('UserComponent::Consumer', UserEvent.last.id)
+```
+
 ## Helper methods
 Some convenience methods are provided to help with common use cases.
 

data/eventsimple.gemspec

@@ -28,9 +28,9 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'pg', '~> 1.4'
   spec.add_runtime_dependency 'rails', '~> 7.0'
   spec.add_runtime_dependency 'retriable', '~> 3.1'
+  spec.add_runtime_dependency 'with_advisory_lock', '>= 5.1'
 
   spec.add_development_dependency 'bundle-audit'
-  spec.add_development_dependency 'factory_bot_rails'
   spec.add_development_dependency 'fuubar'
   spec.add_development_dependency 'git'
   spec.add_development_dependency 'guard-rspec'

data/lib/eventsimple/event.rb

@@ -14,8 +14,8 @@ module Eventsimple
       class_attribute :_aggregate_id
       self._aggregate_id = aggregate_id
 
-      class_attribute :_outbox_mode
-      class_attribute :_outbox_concurrency
+      class_attribute :_outbox_enabled
+      class_attribute :_consumer_group_size
 
       class_attribute :_on_invalid_transition
       self._on_invalid_transition = ->(error) { raise error }
@@ -161,7 +161,7 @@ module Eventsimple
       end
 
       def with_locks(&block)
-        if _outbox_mode
+        if _outbox_enabled
           base_class.with_advisory_lock(base_class.name, { transaction: true }, &block)
         else
           yield

data/lib/eventsimple/outbox/consumer.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'with_advisory_lock'
 require 'eventsimple/outbox/models/cursor'
 
 module Eventsimple
@@ -9,39 +10,55 @@ module Eventsimple
         klass.class_exec do
           class_attribute :_event_klass
           class_attribute :_processor_klass
+          class_attribute :_processor
           class_attribute :stop_consumer, default: false
-
-          Signal.trap('SIGINT') do
-            self.stop_consumer = true
-            $stdout.puts('SIGINT received, stopping consumer')
-          end
+          class_attribute :_identifier, default: name.to_s
         end
       end
 
-      def consumes_event(event_klass, concurrency: 1)
-        event_klass._outbox_mode = true
-        event_klass._outbox_concurrency = concurrency
+      def identifier(name = nil)
+        self._identifier = name
+      end
+
+      def consumes_event(event_klass, group_size: 1)
+        event_klass._outbox_enabled = true
+        event_klass._consumer_group_size = group_size
 
         self._event_klass = event_klass
       end
 
       def processor(processor_klass)
         self._processor_klass = processor_klass
+        self._processor = processor_klass.new
+      end
+
+      def start(group_number: 0) # rubocop:disable Metrics/AbcSize
+        Signal.trap('INT') do
+          self.stop_consumer = true
+          $stdout.puts('INT received, stopping consumer')
+        end
+        Signal.trap('TERM') do
+          self.stop_consumer = true
+          $stdout.puts('TERM received, stopping consumer')
+        end
+
+        run_consumer(group_number: group_number)
       end
 
-      def start # rubocop:disable Metrics/AbcSize
-        cursor = Outbox::Cursor.fetch(_event_klass, 0)
+      def run_consumer(group_number:)
+        cursor = Outbox::Cursor.fetch(_identifier, group_number: group_number)
 
         until stop_consumer
           _event_klass.unscoped.in_batches(start: cursor + 1, load: true).each do |batch|
             batch.each do |event|
-              _processor_klass.new(event).call
+              _processor.call(event)
 
+              cursor = event.id
               break if stop_consumer
             end
 
-            cursor = batch.last.id
-            Outbox::Cursor.set(_event_klass, 0, cursor)
+            Outbox::Cursor.set(_identifier, cursor, group_number: group_number)
+            break if stop_consumer
           end
 
           sleep(1)

data/lib/eventsimple/outbox/models/cursor.rb

@@ -5,19 +5,19 @@ module Eventsimple
     class Cursor < Eventsimple.configuration.parent_record_klass
       self.table_name = 'eventsimple_outbox_cursors'
 
-      def self.fetch(event_klass, group_number)
-        existing = find_by(event_klass: event_klass.to_s, group_number: group_number)
+      def self.fetch(identifier, group_number: 0)
+        existing = find_by(identifier: identifier.to_s, group_number: group_number)
         existing ? existing.cursor : 0
       end
 
-      def self.set(event_klass, group_number, cursor)
+      def self.set(identifier, cursor, group_number: 0)
         upsert(
           {
-            event_klass: event_klass.to_s,
+            identifier: identifier.to_s,
             group_number: group_number,
             cursor: cursor,
           },
-          unique_by: [:event_klass, :group_number],
+          unique_by: [:identifier, :group_number],
         )
       end
     end

data/lib/eventsimple/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Eventsimple
-  VERSION = '1.3.3'
+  VERSION = '1.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventsimple
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 1.4.0
 platform: ruby
 authors:
 - Zulfiqar Ali
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-04-03 00:00:00.000000000 Z
+date: 2024-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-struct
@@ -81,21 +81,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.1'
 - !ruby/object:Gem::Dependency
-  name: bundle-audit
+  name: with_advisory_lock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: '5.1'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.1'
 - !ruby/object:Gem::Dependency
-  name: factory_bot_rails
+  name: bundle-audit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="