eventsimple 1.4.3 → 1.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89459aafe1f0b6343f5893f9921d43e2b41a77bdd051160f8206837589941dc7
-  data.tar.gz: 9920a2bd4454bc8f97fbf9d9fc036ddc3801508321602a27cf1236de005a3633
+  metadata.gz: 42e3b62f3cfbc6f95ff097ea4e23917365589ff637aa69b8fe4a9a65fb13d41b
+  data.tar.gz: 796eb8680f793866df0b3d6689b8eb42bff643c6fcd70e8f90794e1c26a341a0
 SHA512:
-  metadata.gz: fa543c41fa1318ab325ad2be11586ba642f07ba4210cb9163b309023da7c56cffb9220ffa92af85b57f43011e02a38a3f60e416fa5c96e6c6a3da8ef5f465541
-  data.tar.gz: c6a1e7bda23440825c00effa10ae939690f9b0963dd5269b048df08de4ea7de72785a6f26a168a0b2ddc7e1e3a038e8d20a35fccc55465130688058ee04b342c
+  metadata.gz: 6d0c5d23e6963904e2ea48400d02f3b4878e00ab4a1b4b8def5972795d7dad5e6753169fbff261f0b207c21b03c37bd28300e1b6bdf9ec924eb45afa1d8fcd38
+  data.tar.gz: 67165eb3d7c2e324328fcef9c406e95118d8d1cc4e8c4ec7ef6b3e4b49a5ca5126907d42c2a2721e8b8fdfff1d288dd7f906d3f37971e8606c5548e85684f8f9

data/CHANGELOG.md

@@ -6,6 +6,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## 1.5.3 - 2024-09-09
+### Changed
+- Pass self to `enable_writes!` block
+
+## 1.5.2 - 2024-05-22
+### Changed
+- Add created_at index in events migration generation
+
+## 1.5.1 - 2024-05-13
+### Changed
+- Fix bug where outbox concurrency was not being configured correctly.
+
+## 1.5.0 - 2024-05-07
+### Changed
+- The outbox consumer processes event batches concurrently
+- Removes unused group_number/group_size config
+
 ## 1.4.3 - 2024-05-06
 ### Changed
 - The order of execution for synchronous reactors is now guaranteed to be the order in which they were registered.

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    eventsimple (1.4.3)
+    eventsimple (1.5.3)
+      concurrent-ruby (>= 1.2.3)
       dry-struct (~> 1.6)
       dry-types (~> 1.7)
       pg (~> 1.4)
@@ -108,7 +109,7 @@ GEM
     dry-core (1.0.1)
       concurrent-ruby (~> 1.0)
       zeitwerk (~> 2.6)
-    dry-inflector (1.0.0)
+    dry-inflector (1.1.0)
     dry-logic (1.5.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 1.0, < 2)
@@ -183,7 +184,7 @@ GEM
     minitest (5.22.2)
     mutex_m (0.2.0)
     nenv (0.3.0)
-    net-imap (0.4.10)
+    net-imap (0.4.16)
       date
       net-protocol
     net-pop (0.1.2)
@@ -193,9 +194,9 @@ GEM
     net-smtp (0.5.0)
       net-protocol
     nio4r (2.7.0)
-    nokogiri (1.16.2-arm64-darwin)
+    nokogiri (1.16.5-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.16.2-x86_64-linux)
+    nokogiri (1.16.5-x86_64-linux)
       racc (~> 1.4)
     notiffany (0.1.3)
       nenv (~> 0.1)
@@ -206,7 +207,7 @@ GEM
     parser (3.3.0.5)
       ast (~> 2.4.1)
       racc
-    pg (1.5.6)
+    pg (1.5.8)
     polyglot (0.3.5)
     pry (0.14.2)
       coderay (~> 1.1)
@@ -260,13 +261,14 @@ GEM
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     rchardet (1.8.0)
-    rdoc (6.6.2)
+    rdoc (6.6.3.1)
       psych (>= 4.0.0)
     regexp_parser (2.9.0)
     reline (0.4.3)
       io-console (~> 0.5)
     retriable (3.1.2)
-    rexml (3.2.6)
+    rexml (3.2.8)
+      strscan (>= 3.0.9)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
@@ -338,6 +340,7 @@ GEM
       lint_roller (~> 1.0)
       rubocop-rails (~> 2.23.1)
     stringio (3.1.0)
+    strscan (3.1.0)
     thor (1.3.1)
     timeout (0.4.1)
     treetop (1.6.12)

data/README.md

@@ -128,6 +128,7 @@ create_table :user_events do |t|
   t.timestamps
 
   t.index :idempotency_key, unique: true
+  t.index :created_at
 end
 
 add_column :users, :lock_version, :integer
@@ -276,9 +277,9 @@ end
 
 ## Configuring an outbox consumer
 
-For many use cases, async reactors are sufficient to handle workflows like making an API call or publishing to a message broker. However as reactors use ActiveJob, order is not guaranteed. For use cases requiring order, eventsimple provides an ordered outbox implementation.
+For many use cases, async reactors are sufficient to handle workflows like making an API call or publishing to a message broker. However as reactors use ActiveJob, order is not guaranteed. For use cases requiring order, eventsimple provides an simple ordered outbox implementation.
 
-**Caveat**: The current implementation leverages a single advisory lock to guarantee write order. This will impact write throughput on the model. On a db.rg6.large Aurora instance for example, write throughput is limited to ~300 events per second.
+The current implementation leverages a single advisory lock to guarantee write order. This will impact write throughput on the model. On a db.rg6.large Aurora instance for example, write throughput to the table is ~300 events per second.
 
 For an explaination of why an advisory lock is required:
 https://github.com/pawelpacana/account-basics
@@ -292,7 +293,6 @@ Generate migration to setup the outbox cursor table. This table is used to track
 ```
 
 Create a consummer and processor class for the outbox.
-Note: The presence of the consumer class moves all writes to the respective events table to be written using an advisory lock.
 
 ```ruby
 require 'eventsimple/outbox/consumer'
@@ -303,7 +303,7 @@ module UserComponent
 
     identitfier 'UserComponent::Consumer'
     consumes_event UserEvent
-    processor EventProcessor
+    processor EventProcessor, concurrency: 5
   end
 end
 ```

data/eventsimple.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'rails', '~> 7.0'
   spec.add_runtime_dependency 'retriable', '~> 3.1'
   spec.add_runtime_dependency 'with_advisory_lock', '>= 5.1'
+  spec.add_runtime_dependency 'concurrent-ruby', '>= 1.2.3'
 
   spec.add_development_dependency 'bundle-audit'
   spec.add_development_dependency 'fuubar'

data/lib/eventsimple/engine.rb

@@ -12,6 +12,7 @@ module Eventsimple
 
     config.after_initialize do
       require 'eventsimple/reactor'
+      require 'eventsimple/outbox/models/cursor'
 
       verify_dispatchers!
 

data/lib/eventsimple/entity.rb

@@ -39,7 +39,7 @@ module Eventsimple
         @readonly = false
 
         if block
-          yield
+          yield self
           @readonly = true if was_readonly
         end
       end

data/lib/eventsimple/event.rb

@@ -15,7 +15,6 @@ module Eventsimple
       self._aggregate_id = aggregate_id
 
       class_attribute :_outbox_enabled
-      class_attribute :_consumer_group_size
 
       class_attribute :_on_invalid_transition
       self._on_invalid_transition = ->(error) { raise error }

data/lib/eventsimple/generators/outbox/templates/create_outbox_cursor.erb

@@ -4,10 +4,9 @@ class CreateEventsimpleOutboxCursor < ActiveRecord::Migration<%= migration_versi
   def change
     create_table :eventsimple_outbox_cursors do |t|
       t.string :event_klass, null: false
-      t.integer :group_number, null: false
       t.bigint :cursor, null: false
 
-      t.index [:event_klass, :group_number], unique: true
+      t.index [:event_klass], unique: true
     end
   end
 end

data/lib/eventsimple/generators/templates/create_events.erb

@@ -13,6 +13,7 @@ class Create<%= model_name.camelize %>Events < ActiveRecord::Migration[7.0]
       t.timestamps
 
       t.index :idempotency_key, unique: true
+      t.index :created_at
     end
 
     # Enables optimistic locking on the evented table

data/lib/eventsimple/outbox/consumer.rb

@@ -6,13 +6,17 @@ require 'eventsimple/outbox/models/cursor'
 module Eventsimple
   module Outbox
     module Consumer
+      class ExitError < StandardError; end
+
       def self.extended(klass)
         klass.class_exec do
           class_attribute :_event_klass
+          class_attribute :_identifier
           class_attribute :_processor_klass
-          class_attribute :_processor
+          class_attribute :_processor_pool
+          class_attribute :_concurrency, default: 5
+          class_attribute :_batch_size, default: 1000
           class_attribute :stop_consumer, default: false
-          class_attribute :_identifier
         end
       end
 
@@ -20,19 +24,19 @@ module Eventsimple
         self._identifier = name
       end
 
-      def consumes_event(event_klass, group_size: 1)
+      def consumes_event(event_klass)
         event_klass._outbox_enabled = true
-        event_klass._consumer_group_size = group_size
 
         self._event_klass = event_klass
       end
 
-      def processor(processor_klass)
+      def processor(processor_klass, concurrency: 5)
+        self._concurrency = concurrency
         self._processor_klass = processor_klass
-        self._processor = processor_klass.new
+        self._processor_pool = _concurrency.times.map { processor_klass.new }
       end
 
-      def start(group_number: 0) # rubocop:disable Metrics/AbcSize
+      def start # rubocop:disable Metrics/AbcSize
         Signal.trap('INT') do
           self.stop_consumer = true
           $stdout.puts('INT received, stopping consumer')
@@ -42,33 +46,42 @@ module Eventsimple
           $stdout.puts('TERM received, stopping consumer')
         end
 
-        run_consumer(group_number: group_number)
+        run_consumer
       end
 
-      def run_consumer(group_number:)
+      def run_consumer
         raise 'Eventsimple: No event class defined' unless _event_klass
-        raise 'Eventsimple: No processor defined' unless _processor
+        raise 'Eventsimple: No processor defined' unless _processor_klass
         raise 'Eventsimple: No identifier defined' unless _identifier
+        raise 'Eventsimple: No concurrency defined' unless _concurrency.is_a?(Integer)
 
-        Rails.logger.info("Starting consumer for #{_identifier}, processing #{_event_klass} events with group number #{group_number}")
+        $stdout.puts("Starting consumer for #{_identifier}")
 
-        cursor = Outbox::Cursor.fetch(_identifier, group_number: group_number)
+        cursor = Outbox::Cursor.fetch(_identifier)
 
         until stop_consumer
-          _event_klass.unscoped.in_batches(start: cursor + 1, load: true).each do |batch|
-            batch.each do |event|
-              _processor.call(event)
+          _event_klass.unscoped.in_batches(start: cursor + 1, load: true, of: _batch_size).each do |batch|
+            grouped_events = batch.group_by { |event| event.aggregate_id.unpack1('L') % _concurrency }
+
+            promises = grouped_events.map { |index, events|
+              Concurrent::Promises.future {
+                events.each do |event|
+                  _processor_pool[index].call(event)
+                  raise ExitError if stop_consumer
+                end
+              }
+            }
 
-              cursor = event.id
-              break if stop_consumer
-            end
+            Concurrent::Promises.zip(*promises).value!
 
-            Outbox::Cursor.set(_identifier, cursor, group_number: group_number)
-            break if stop_consumer
+            cursor = batch.last.id
+            Outbox::Cursor.set(_identifier, cursor)
           end
 
           sleep(1)
         end
+      rescue ExitError
+        $stdout.puts("Stopping consumer for #{_identifier}")
       end
     end
   end

data/lib/eventsimple/outbox/models/cursor.rb

@@ -5,19 +5,18 @@ module Eventsimple
     class Cursor < Eventsimple.configuration.parent_record_klass
       self.table_name = 'eventsimple_outbox_cursors'
 
-      def self.fetch(identifier, group_number: 0)
-        existing = find_by(identifier: identifier.to_s, group_number: group_number)
+      def self.fetch(identifier)
+        existing = find_by(identifier: identifier)
         existing ? existing.cursor : 0
       end
 
-      def self.set(identifier, cursor, group_number: 0)
+      def self.set(identifier, cursor)
         upsert(
           {
-            identifier: identifier.to_s,
-            group_number: group_number,
+            identifier: identifier,
             cursor: cursor,
           },
-          unique_by: [:identifier, :group_number],
+          unique_by: [:identifier],
         )
       end
     end

data/lib/eventsimple/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Eventsimple
-  VERSION = '1.4.3'
+  VERSION = '1.5.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventsimple
 version: !ruby/object:Gem::Version
-  version: 1.4.3
+  version: 1.5.3
 platform: ruby
 authors:
 - Zulfiqar Ali
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-05-06 00:00:00.000000000 Z
+date: 2024-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-struct
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5.1'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.3
 - !ruby/object:Gem::Dependency
   name: bundle-audit
   requirement: !ruby/object:Gem::Requirement