eventsimple 1.4.3 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89459aafe1f0b6343f5893f9921d43e2b41a77bdd051160f8206837589941dc7
-  data.tar.gz: 9920a2bd4454bc8f97fbf9d9fc036ddc3801508321602a27cf1236de005a3633
+  metadata.gz: 0e34181920a4294d6f99a84819ea4861377c3e760a33bde393821ba4de8a8165
+  data.tar.gz: 426eeaa345d28493854e989c08891b40766fd595d5b822876dbac030e65ce9d8
 SHA512:
-  metadata.gz: fa543c41fa1318ab325ad2be11586ba642f07ba4210cb9163b309023da7c56cffb9220ffa92af85b57f43011e02a38a3f60e416fa5c96e6c6a3da8ef5f465541
-  data.tar.gz: c6a1e7bda23440825c00effa10ae939690f9b0963dd5269b048df08de4ea7de72785a6f26a168a0b2ddc7e1e3a038e8d20a35fccc55465130688058ee04b342c
+  metadata.gz: a9f364c4e825147973326bcb9c87f0201720b6773957a6670a2c466b7febd606c5ba3868f81557f06b8a95a2e49035b866c83c188a2bc5e11a8c2af6e0de49f2
+  data.tar.gz: d761164e0bf06957c696f1d720c9d292b71a43a50827d44a0564a67280d36d0c98805af8bee7a042104cb8011da7ab0876c0b8e8e74e8eda61cdd7d20ca27b17

data/CHANGELOG.md

@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## 1.5.0 - 2024-05-07
+### Changed
+- The outbox consumer processes event batches concurrently
+- Removes unused group_number/group_size config
+
 ## 1.4.3 - 2024-05-06
 ### Changed
 - The order of execution for synchronous reactors is now guaranteed to be the order in which they were registered.

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    eventsimple (1.4.3)
+    eventsimple (1.5.0)
+      concurrent-ruby (>= 1.2.3)
       dry-struct (~> 1.6)
       dry-types (~> 1.7)
       pg (~> 1.4)

data/README.md

@@ -276,9 +276,9 @@ end
 
 ## Configuring an outbox consumer
 
-For many use cases, async reactors are sufficient to handle workflows like making an API call or publishing to a message broker. However as reactors use ActiveJob, order is not guaranteed. For use cases requiring order, eventsimple provides an ordered outbox implementation.
+For many use cases, async reactors are sufficient to handle workflows like making an API call or publishing to a message broker. However as reactors use ActiveJob, order is not guaranteed. For use cases requiring order, eventsimple provides an simple ordered outbox implementation.
 
-**Caveat**: The current implementation leverages a single advisory lock to guarantee write order. This will impact write throughput on the model. On a db.rg6.large Aurora instance for example, write throughput is limited to ~300 events per second.
+The current implementation leverages a single advisory lock to guarantee write order. This will impact write throughput on the model. On a db.rg6.large Aurora instance for example, write throughput to the table is ~300 events per second.
 
 For an explaination of why an advisory lock is required:
 https://github.com/pawelpacana/account-basics
@@ -292,7 +292,6 @@ Generate migration to setup the outbox cursor table. This table is used to track
 ```
 
 Create a consummer and processor class for the outbox.
-Note: The presence of the consumer class moves all writes to the respective events table to be written using an advisory lock.
 
 ```ruby
 require 'eventsimple/outbox/consumer'
@@ -304,6 +303,7 @@ module UserComponent
     identitfier 'UserComponent::Consumer'
     consumes_event UserEvent
     processor EventProcessor
+    concurrency 5 # default is 5
   end
 end
 ```

data/eventsimple.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'rails', '~> 7.0'
   spec.add_runtime_dependency 'retriable', '~> 3.1'
   spec.add_runtime_dependency 'with_advisory_lock', '>= 5.1'
+  spec.add_runtime_dependency 'concurrent-ruby', '>= 1.2.3'
 
   spec.add_development_dependency 'bundle-audit'
   spec.add_development_dependency 'fuubar'

data/lib/eventsimple/engine.rb

@@ -12,6 +12,7 @@ module Eventsimple
 
     config.after_initialize do
       require 'eventsimple/reactor'
+      require 'eventsimple/outbox/models/cursor'
 
       verify_dispatchers!
 

data/lib/eventsimple/event.rb

@@ -15,7 +15,6 @@ module Eventsimple
       self._aggregate_id = aggregate_id
 
       class_attribute :_outbox_enabled
-      class_attribute :_consumer_group_size
 
       class_attribute :_on_invalid_transition
       self._on_invalid_transition = ->(error) { raise error }

data/lib/eventsimple/generators/outbox/templates/create_outbox_cursor.erb

@@ -4,10 +4,9 @@ class CreateEventsimpleOutboxCursor < ActiveRecord::Migration<%= migration_versi
   def change
     create_table :eventsimple_outbox_cursors do |t|
       t.string :event_klass, null: false
-      t.integer :group_number, null: false
       t.bigint :cursor, null: false
 
-      t.index [:event_klass, :group_number], unique: true
+      t.index [:event_klass], unique: true
     end
   end
 end

data/lib/eventsimple/outbox/consumer.rb

@@ -6,13 +6,17 @@ require 'eventsimple/outbox/models/cursor'
 module Eventsimple
   module Outbox
     module Consumer
+      class ExitError < StandardError; end
+
       def self.extended(klass)
         klass.class_exec do
           class_attribute :_event_klass
+          class_attribute :_identifier
           class_attribute :_processor_klass
-          class_attribute :_processor
+          class_attribute :_processor_pool
+          class_attribute :_concurrency, default: 5
+          class_attribute :_batch_size, default: 1000
           class_attribute :stop_consumer, default: false
-          class_attribute :_identifier
         end
       end
 
@@ -20,19 +24,22 @@ module Eventsimple
         self._identifier = name
       end
 
-      def consumes_event(event_klass, group_size: 1)
+      def consumes_event(event_klass)
         event_klass._outbox_enabled = true
-        event_klass._consumer_group_size = group_size
 
         self._event_klass = event_klass
       end
 
       def processor(processor_klass)
         self._processor_klass = processor_klass
-        self._processor = processor_klass.new
+        self._processor_pool = _concurrency.times.map { processor_klass.new }
       end
 
-      def start(group_number: 0) # rubocop:disable Metrics/AbcSize
+      def concurrency(concurrency)
+        self._concurrency = concurrency
+      end
+
+      def start # rubocop:disable Metrics/AbcSize
         Signal.trap('INT') do
           self.stop_consumer = true
           $stdout.puts('INT received, stopping consumer')
@@ -42,33 +49,42 @@ module Eventsimple
           $stdout.puts('TERM received, stopping consumer')
         end
 
-        run_consumer(group_number: group_number)
+        run_consumer
       end
 
-      def run_consumer(group_number:)
+      def run_consumer
         raise 'Eventsimple: No event class defined' unless _event_klass
-        raise 'Eventsimple: No processor defined' unless _processor
+        raise 'Eventsimple: No processor defined' unless _processor_klass
         raise 'Eventsimple: No identifier defined' unless _identifier
+        raise 'Eventsimple: No concurrency defined' unless _concurrency.is_a?(Integer)
 
-        Rails.logger.info("Starting consumer for #{_identifier}, processing #{_event_klass} events with group number #{group_number}")
+        $stdout.puts("Starting consumer for #{_identifier}")
 
-        cursor = Outbox::Cursor.fetch(_identifier, group_number: group_number)
+        cursor = Outbox::Cursor.fetch(_identifier)
 
         until stop_consumer
-          _event_klass.unscoped.in_batches(start: cursor + 1, load: true).each do |batch|
-            batch.each do |event|
-              _processor.call(event)
+          _event_klass.unscoped.in_batches(start: cursor + 1, load: true, of: _batch_size).each do |batch|
+            grouped_events = batch.group_by { |event| event.aggregate_id.unpack1('L') % _concurrency }
+
+            promises = grouped_events.map { |index, events|
+              Concurrent::Promises.future {
+                events.each do |event|
+                  _processor_pool[index].call(event)
+                  raise ExitError if stop_consumer
+                end
+              }
+            }
 
-              cursor = event.id
-              break if stop_consumer
-            end
+            Concurrent::Promises.zip(*promises).value!
 
-            Outbox::Cursor.set(_identifier, cursor, group_number: group_number)
-            break if stop_consumer
+            cursor = batch.last.id
+            Outbox::Cursor.set(_identifier, cursor)
           end
 
           sleep(1)
         end
+      rescue ExitError
+        $stdout.puts("Stopping consumer for #{_identifier}")
       end
     end
   end

data/lib/eventsimple/outbox/models/cursor.rb

@@ -5,19 +5,18 @@ module Eventsimple
     class Cursor < Eventsimple.configuration.parent_record_klass
       self.table_name = 'eventsimple_outbox_cursors'
 
-      def self.fetch(identifier, group_number: 0)
-        existing = find_by(identifier: identifier.to_s, group_number: group_number)
+      def self.fetch(identifier)
+        existing = find_by(identifier: identifier)
         existing ? existing.cursor : 0
       end
 
-      def self.set(identifier, cursor, group_number: 0)
+      def self.set(identifier, cursor)
         upsert(
           {
-            identifier: identifier.to_s,
-            group_number: group_number,
+            identifier: identifier,
             cursor: cursor,
           },
-          unique_by: [:identifier, :group_number],
+          unique_by: [:identifier],
         )
       end
     end

data/lib/eventsimple/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Eventsimple
-  VERSION = '1.4.3'
+  VERSION = '1.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventsimple
 version: !ruby/object:Gem::Version
-  version: 1.4.3
+  version: 1.5.0
 platform: ruby
 authors:
 - Zulfiqar Ali
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-05-06 00:00:00.000000000 Z
+date: 2024-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-struct
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5.1'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.3
 - !ruby/object:Gem::Dependency
   name: bundle-audit
   requirement: !ruby/object:Gem::Requirement