eventsimple 1.4.2 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4e4684f9e4b569bd20e0b09e70a043d44d1f650530e192d8a4eb9e60fcc35c7
-  data.tar.gz: 40ee62f907e02a421db4e83653c7c64953df65402c64613597fafa75ae2d8370
+  metadata.gz: 0e34181920a4294d6f99a84819ea4861377c3e760a33bde393821ba4de8a8165
+  data.tar.gz: 426eeaa345d28493854e989c08891b40766fd595d5b822876dbac030e65ce9d8
 SHA512:
-  metadata.gz: 4ba23518b78c46ca08a7b5669a2870a2e993054d95995c4bf7a564db63b0bcf5b8a141d380a3006fc2259f41689d6570a1ed9d39b8483badb84d1fb5f7d02105
-  data.tar.gz: 9f27f387e7a7cff1631b6b4f427c167a692b1e2664783bd1a4ded614a3abb7266921c2f3195a521fe4e348064f3cce808e2eb9a481b58bdba4154f45af64179a
+  metadata.gz: a9f364c4e825147973326bcb9c87f0201720b6773957a6670a2c466b7febd606c5ba3868f81557f06b8a95a2e49035b866c83c188a2bc5e11a8c2af6e0de49f2
+  data.tar.gz: d761164e0bf06957c696f1d720c9d292b71a43a50827d44a0564a67280d36d0c98805af8bee7a042104cb8011da7ab0876c0b8e8e74e8eda61cdd7d20ca27b17

data/.ruby-version

@@ -1 +1 @@
-3.2.2
+3.2.3

data/CHANGELOG.md

@@ -6,6 +6,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## 1.5.0 - 2024-05-07
+### Changed
+- The outbox consumer processes event batches concurrently
+- Removes unused group_number/group_size config
+
+## 1.4.3 - 2024-05-06
+### Changed
+- The order of execution for synchronous reactors is now guaranteed to be the order in which they were registered.
+- The shared examples for reactors `'an event which (a)synchronously dispatches'` now accept 1 or multiple arguments. The synchronous version checks that reactors are given in the order in which they are registered.
+
 ## 1.4.2 - 2024-04-30
 ### Changed
 - Raise error on missing consumer config

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    eventsimple (1.4.2)
+    eventsimple (1.5.0)
+      concurrent-ruby (>= 1.2.3)
       dry-struct (~> 1.6)
       dry-types (~> 1.7)
       pg (~> 1.4)

data/README.md

@@ -276,9 +276,9 @@ end
 
 ## Configuring an outbox consumer
 
-For many use cases, async reactors are sufficient to handle workflows like making an API call or publishing to a message broker. However as reactors use ActiveJob, order is not guaranteed. For use cases requiring order, eventsimple provides an ordered outbox implementation.
+For many use cases, async reactors are sufficient to handle workflows like making an API call or publishing to a message broker. However as reactors use ActiveJob, order is not guaranteed. For use cases requiring order, eventsimple provides an simple ordered outbox implementation.
 
-**Caveat**: The current implementation leverages a single advisory lock to guarantee write order. This will impact write throughput on the model. On a db.rg6.large Aurora instance for example, write throughput is limited to ~300 events per second.
+The current implementation leverages a single advisory lock to guarantee write order. This will impact write throughput on the model. On a db.rg6.large Aurora instance for example, write throughput to the table is ~300 events per second.
 
 For an explaination of why an advisory lock is required:
 https://github.com/pawelpacana/account-basics
@@ -292,7 +292,6 @@ Generate migration to setup the outbox cursor table. This table is used to track
 ```
 
 Create a consummer and processor class for the outbox.
-Note: The presence of the consumer class moves all writes to the respective events table to be written using an advisory lock.
 
 ```ruby
 require 'eventsimple/outbox/consumer'
@@ -304,6 +303,7 @@ module UserComponent
     identitfier 'UserComponent::Consumer'
     consumes_event UserEvent
     processor EventProcessor
+    concurrency 5 # default is 5
   end
 end
 ```

data/eventsimple.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'rails', '~> 7.0'
   spec.add_runtime_dependency 'retriable', '~> 3.1'
   spec.add_runtime_dependency 'with_advisory_lock', '>= 5.1'
+  spec.add_runtime_dependency 'concurrent-ruby', '>= 1.2.3'
 
   spec.add_development_dependency 'bundle-audit'
   spec.add_development_dependency 'fuubar'

data/lib/eventsimple/engine.rb

@@ -12,6 +12,7 @@ module Eventsimple
 
     config.after_initialize do
       require 'eventsimple/reactor'
+      require 'eventsimple/outbox/models/cursor'
 
       verify_dispatchers!
 

data/lib/eventsimple/event.rb

@@ -15,7 +15,6 @@ module Eventsimple
       self._aggregate_id = aggregate_id
 
       class_attribute :_outbox_enabled
-      class_attribute :_consumer_group_size
 
       class_attribute :_on_invalid_transition
       self._on_invalid_transition = ->(error) { raise error }

data/lib/eventsimple/event_dispatcher.rb

@@ -55,6 +55,7 @@ module Eventsimple
       end
 
       # Return a ReactorSet containing all Reactors matching an Event
+      # Reactors will be in the order in which they were registered
       def for(event)
         reactors = ReactorSet.new
 
@@ -77,16 +78,16 @@ module Eventsimple
       attr_reader :sync, :async
 
       def initialize
-        @sync = Set.new
-        @async = Set.new
+        @sync = []
+        @async = []
       end
 
       def add_sync(reactors)
-        @sync += reactors
+        @sync |= reactors
       end
 
       def add_async(reactors)
-        @async += reactors
+        @async |= reactors
       end
     end
   end

data/lib/eventsimple/generators/outbox/templates/create_outbox_cursor.erb

@@ -4,10 +4,9 @@ class CreateEventsimpleOutboxCursor < ActiveRecord::Migration<%= migration_versi
   def change
     create_table :eventsimple_outbox_cursors do |t|
       t.string :event_klass, null: false
-      t.integer :group_number, null: false
       t.bigint :cursor, null: false
 
-      t.index [:event_klass, :group_number], unique: true
+      t.index [:event_klass], unique: true
     end
   end
 end

data/lib/eventsimple/outbox/consumer.rb

@@ -6,13 +6,17 @@ require 'eventsimple/outbox/models/cursor'
 module Eventsimple
   module Outbox
     module Consumer
+      class ExitError < StandardError; end
+
       def self.extended(klass)
         klass.class_exec do
           class_attribute :_event_klass
+          class_attribute :_identifier
           class_attribute :_processor_klass
-          class_attribute :_processor
+          class_attribute :_processor_pool
+          class_attribute :_concurrency, default: 5
+          class_attribute :_batch_size, default: 1000
           class_attribute :stop_consumer, default: false
-          class_attribute :_identifier
         end
       end
 
@@ -20,19 +24,22 @@ module Eventsimple
         self._identifier = name
       end
 
-      def consumes_event(event_klass, group_size: 1)
+      def consumes_event(event_klass)
         event_klass._outbox_enabled = true
-        event_klass._consumer_group_size = group_size
 
         self._event_klass = event_klass
       end
 
       def processor(processor_klass)
         self._processor_klass = processor_klass
-        self._processor = processor_klass.new
+        self._processor_pool = _concurrency.times.map { processor_klass.new }
       end
 
-      def start(group_number: 0) # rubocop:disable Metrics/AbcSize
+      def concurrency(concurrency)
+        self._concurrency = concurrency
+      end
+
+      def start # rubocop:disable Metrics/AbcSize
         Signal.trap('INT') do
           self.stop_consumer = true
           $stdout.puts('INT received, stopping consumer')
@@ -42,33 +49,42 @@ module Eventsimple
           $stdout.puts('TERM received, stopping consumer')
         end
 
-        run_consumer(group_number: group_number)
+        run_consumer
       end
 
-      def run_consumer(group_number:)
+      def run_consumer
         raise 'Eventsimple: No event class defined' unless _event_klass
-        raise 'Eventsimple: No processor defined' unless _processor
+        raise 'Eventsimple: No processor defined' unless _processor_klass
         raise 'Eventsimple: No identifier defined' unless _identifier
+        raise 'Eventsimple: No concurrency defined' unless _concurrency.is_a?(Integer)
 
-        Rails.logger.info("Starting consumer for #{_identifier}, processing #{_event_klass} events with group number #{group_number}")
+        $stdout.puts("Starting consumer for #{_identifier}")
 
-        cursor = Outbox::Cursor.fetch(_identifier, group_number: group_number)
+        cursor = Outbox::Cursor.fetch(_identifier)
 
         until stop_consumer
-          _event_klass.unscoped.in_batches(start: cursor + 1, load: true).each do |batch|
-            batch.each do |event|
-              _processor.call(event)
+          _event_klass.unscoped.in_batches(start: cursor + 1, load: true, of: _batch_size).each do |batch|
+            grouped_events = batch.group_by { |event| event.aggregate_id.unpack1('L') % _concurrency }
+
+            promises = grouped_events.map { |index, events|
+              Concurrent::Promises.future {
+                events.each do |event|
+                  _processor_pool[index].call(event)
+                  raise ExitError if stop_consumer
+                end
+              }
+            }
 
-              cursor = event.id
-              break if stop_consumer
-            end
+            Concurrent::Promises.zip(*promises).value!
 
-            Outbox::Cursor.set(_identifier, cursor, group_number: group_number)
-            break if stop_consumer
+            cursor = batch.last.id
+            Outbox::Cursor.set(_identifier, cursor)
           end
 
           sleep(1)
         end
+      rescue ExitError
+        $stdout.puts("Stopping consumer for #{_identifier}")
       end
     end
   end

data/lib/eventsimple/outbox/models/cursor.rb

@@ -5,19 +5,18 @@ module Eventsimple
     class Cursor < Eventsimple.configuration.parent_record_klass
       self.table_name = 'eventsimple_outbox_cursors'
 
-      def self.fetch(identifier, group_number: 0)
-        existing = find_by(identifier: identifier.to_s, group_number: group_number)
+      def self.fetch(identifier)
+        existing = find_by(identifier: identifier)
         existing ? existing.cursor : 0
       end
 
-      def self.set(identifier, cursor, group_number: 0)
+      def self.set(identifier, cursor)
         upsert(
           {
-            identifier: identifier.to_s,
-            group_number: group_number,
+            identifier: identifier,
             cursor: cursor,
           },
-          unique_by: [:identifier, :group_number],
+          unique_by: [:identifier],
         )
       end
     end

data/lib/eventsimple/support/spec_helpers.rb

@@ -1,18 +1,20 @@
 # frozen_string_literal: true
 
-RSpec.shared_examples 'an event which synchronously dispatches' do |dispatcher_klass|
+RSpec.shared_examples 'an event which synchronously dispatches' do |*dispatcher_klasses|
   specify do
     reactors = Eventsimple::EventDispatcher.rules.for(described_class.new)
 
-    expect(reactors.sync).to include(dispatcher_klass)
+    # Order is important here since the synchronous reactors are executed sequentially
+    expect(reactors.sync & dispatcher_klasses).to eq(dispatcher_klasses)
   end
 end
 
-RSpec.shared_examples 'an event which asynchronously dispatches' do |dispatcher_klass|
+RSpec.shared_examples 'an event which asynchronously dispatches' do |*dispatcher_klasses|
   specify do
     reactors = Eventsimple::EventDispatcher.rules.for(described_class.new)
 
-    expect(reactors.async).to include(dispatcher_klass)
+    # Order is _not_ important here since async reactors have no order guarantee
+    expect(reactors.async).to include(*dispatcher_klasses)
   end
 end
 

data/lib/eventsimple/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Eventsimple
-  VERSION = '1.4.2'
+  VERSION = '1.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventsimple
 version: !ruby/object:Gem::Version
-  version: 1.4.2
+  version: 1.5.0
 platform: ruby
 authors:
 - Zulfiqar Ali
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-04-30 00:00:00.000000000 Z
+date: 2024-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-struct
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5.1'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.3
 - !ruby/object:Gem::Dependency
   name: bundle-audit
   requirement: !ruby/object:Gem::Requirement
@@ -298,7 +312,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.4.19
 signing_key: 
 specification_version: 4
 summary: Event sourcing toolkit using Rails and ActiveJob