tobox 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15d95687a102c98fcc33859b3a369dc9026f04fa58f7f785bcad1d9ccb40010f
-  data.tar.gz: 656b296f9d72d67877945317d4a6d5505de63044f74e08c506970b2c841599f4
+  metadata.gz: b8f6e85f173d4636a64617ede574141d4b2365799b774a6e56a76b70a59db6ba
+  data.tar.gz: 49bb80cc99034ba4db846964988c8d1fc2a1d8e062decbe57d5b786e892afb8b
 SHA512:
-  metadata.gz: 172b54fd340f865dbc26a30266f815fe556424cd150c8205c872da02d3725def786648710097b0880de41cc16940a39bd34c8592049d22a907d527ff84b26a3a
-  data.tar.gz: 1dafbf50e8d18c4eb7f4dfd31539322230bd1d11d403f250e4612a0579e878c366fe6b2d44eb24ae17f32476bd25994457563d3fd5ef1a46d7499dc92a93ee1d
+  metadata.gz: e1e3f588dbe62ceaf50d6d1e67e3fbcecf0798027728907961d2bd92d18ad5cf779d215c6cad614ace6bdf0c6bc00e8c32a2e3f0ddd30ae1c7f6c2f6524b3133
+  data.tar.gz: 6170813875d41ed03012ded3a418290446abfeac34a80fb5a7e21df3741032df23b05261d7b87c4c9ff3c2d230738ff20e93c7d178969606e092617d6f91357c

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## [Unreleased]
 
+## [0.5.2] - 2024-10-22
+
+## Bugfixes
+
+* prevent `:max_connections` from being overidden.
+
+### Improvements
+
 ## [0.5.1] - 2024-09-26
 
 ### Improvements

data/README.md

@@ -219,6 +219,20 @@ exponential_retry_factor 2
 
 **Note**: the new attempt will be retried in `n ** 4`, where `n` is the number of past attempts for that event.
 
+### `batch_size`
+
+Number of events fetched in each outbox loop.
+
+```ruby
+batch_size 10
+```
+
+**Note**: event handlers will receive all events covered by the given callback in its arguments:
+
+```ruby
+on(:event1, :event2) do |*events| # at most 10 events, may contain events of type 1 and 2
+```
+
 ### `concurrency`
 
 Number of workers processing events.
@@ -229,6 +243,21 @@ concurrency 4
 
 **Note**: the default concurrency is adapted and different for each worker pool type, so make sure you understand how this tweak may affect you.
 
+### `max_connections`
+
+Number of database connections the workers will share to do their work.
+
+In the (default) threaded worker mode, it'll default to the number of workers (set by the `concurrency` configuration).
+
+```ruby
+# 10 workers sharing 4 database connections
+max_connections 4
+concurrency 10
+```
+
+This can be useful when the database presents overhead in managing write intensive workload (such as the one an outbox generates), and you want to be able to scale without putting more work on the database.
+
+
 ### `worker`
 
 Type of the worker used to process events. Can be `:thread` (default), `:fiber`, or a class implementing the `Tobox::Pool` protocol (TBD: define what this protocol is).

data/lib/tobox/configuration.rb

@@ -20,6 +20,7 @@ module Tobox
       database_options: nil,
       table: :outbox,
       created_at_column: nil,
+      batch_size: 1,
       max_attempts: 10,
       exponential_retry_factor: 2,
       wait_for_events_delay: 5,
@@ -64,7 +65,7 @@ module Tobox
 
       @database = if @config[:database_uri]
                     database_opts = @config[:database_options] || {}
-                    database_opts[:max_connections] = @config[:concurrency] if @config[:worker] == :thread
+                    database_opts[:max_connections] ||= (@config[:concurrency] if @config[:worker] == :thread)
                     db = Sequel.connect(@config[:database_uri].to_s, database_opts)
                     Array(@lifecycle_events[:database_connect]).each { |cb| cb.call(db) }
                     db
@@ -83,9 +84,10 @@ module Tobox
       freeze
     end
 
-    def on(*events, &callback)
-      events.each do |event|
-        (@handlers[event.to_sym] ||= []) << callback
+    def on(*event_types, &callback)
+      callback_events = (@handlers[callback] ||= [])
+      event_types.each do |event_type|
+        callback_events << event_type.to_sym
       end
       self
     end

data/lib/tobox/fetcher.rb

@@ -28,6 +28,8 @@ module Tobox
                           .where(run_at_conds)
                           .order(Sequel.desc(:run_at, nulls: :first), :id)
 
+      @batch_size = configuration[:batch_size]
+
       @mark_as_fetched_params = { attempts: Sequel[@table][:attempts] + 1, last_error: nil }
 
       @before_event_handlers = Array(@configuration.lifecycle_events[:before_event])
@@ -38,18 +40,19 @@ module Tobox
     def fetch_events(&blk)
       num_events = 0
       events_tr do
-        event_id = nil
+        event_ids = nil
+        # @type var event_ids: Array[Integer]
 
         event_id_tr do
-          event_id = fetch_event_id
-          mark_as_fetched(event_id) if event_id
+          event_ids = fetch_event_ids
+          mark_as_fetched(event_ids) unless event_ids.empty?
         end
 
-        if event_id
-          with_event(event_id) do |event|
-            num_events = 1
+        if event_ids && !event_ids.empty?
+          with_events(event_ids) do |events|
+            num_events = events.size
 
-            prepare_event(event, &blk)
+            prepare_events(events, &blk)
           end
         end
       end
@@ -59,20 +62,23 @@ module Tobox
 
     private
 
-    def prepare_event(event)
-      event[:metadata] = try_json_parse(event[:metadata])
-      handle_before_event(event)
-      yield(to_message(event))
+    def prepare_events(events)
+      prepared_events = events.map do |event|
+        event[:metadata] = try_json_parse(event[:metadata]) if event[:metadata]
+        handle_before_event(event)
+        to_message(event)
+      end
+      yield(prepared_events)
     end
 
-    def fetch_event_id
+    def fetch_event_ids
       @pick_next_sql.for_update
                     .skip_locked
-                    .limit(1).select_map(:id).first # lock starts here
+                    .limit(@batch_size).select_map(:id) # lock starts here
     end
 
-    def mark_as_fetched(event_id)
-      @ds.where(id: event_id).update(@mark_as_fetched_params)
+    def mark_as_fetched(event_ids)
+      @ds.where(id: event_ids).update(@mark_as_fetched_params)
     end
 
     def events_tr(&block)
@@ -83,32 +89,34 @@ module Tobox
       yield
     end
 
-    def with_event(event_id, &blk)
-      event, error = yield_event(event_id, &blk)
+    def with_events(event_ids, &blk)
+      events, error = yield_events(event_ids, &blk)
 
-      if error
-        event.merge!(mark_as_error(event, error))
-        handle_error_event(event, error)
-      else
-        handle_after_event(event)
+      events.each do |event|
+        if error
+          event.merge!(mark_as_error(event, error))
+          handle_error_event(event, error)
+        else
+          handle_after_event(event)
+        end
       end
     end
 
-    def yield_event(event_id)
-      events_ds = @ds.where(id: event_id)
-      event = error = nil
+    def yield_events(event_ids)
+      events_ds = @ds.where(id: event_ids)
+      events = error = nil
 
       begin
-        event = events_ds.first
+        events = events_ds.all
 
-        yield event
+        yield events
 
-        events_ds.delete
+        events_ds.delete unless events.empty?
       rescue StandardError => e
         error = e
       end
 
-      [event, error]
+      [events, error]
     end
 
     def log_message(msg)
@@ -141,15 +149,13 @@ module Tobox
       {
         id: event[:id],
         type: event[:type],
-        before: try_json_parse(event[:data_before]),
-        after: try_json_parse(event[:data_after]),
+        before: (try_json_parse(event[:data_before]) if event[:data_before]),
+        after: (try_json_parse(event[:data_after]) if event[:data_after]),
         at: event[:created_at]
       }
     end
 
     def try_json_parse(data)
-      return unless data
-
       data = JSON.parse(data.to_s) unless data.respond_to?(:to_hash)
 
       data

data/lib/tobox/plugins/event_grouping.rb

@@ -16,7 +16,7 @@ module Tobox
 
         private
 
-        def fetch_event_id
+        def fetch_event_ids
           group = @pick_next_sql.for_update
                                 .skip_locked
                                 .limit(1)
@@ -36,8 +36,8 @@ module Tobox
             event_ids = []
           end
 
-          # lock all, process 1
-          event_ids.first
+          # lock all
+          event_ids.first(@batch_size)
         end
       end
     end

data/lib/tobox/plugins/inbox.rb

@@ -20,24 +20,47 @@ module Tobox
 
         private
 
-        def prepare_event(event, &blk)
-          try_insert_inbox(event) { super }
+        def prepare_events(events)
+          try_insert_inbox(events) do |deduped_events|
+            super(deduped_events)
+          end
         end
 
-        def try_insert_inbox(event)
+        def try_insert_inbox(events)
+          inboxed = nil
+
           if @inbox_ds.respond_to?(:supports_insert_conflict?) && @inbox_ds.supports_insert_conflict?
-            ret = @inbox_ds.insert_conflict.insert(@inbox_column => event[@inbox_column])
+            if @inbox_ds.supports_returning?(:insert)
+              inboxed = @inbox_ds.insert_conflict
+                                 .returning(@inbox_column)
+                                 .multi_insert(events.map { |event| { @inbox_column => event[@inbox_column] } })
+            else
+              ret = @inbox_ds.insert_conflict.multi_insert(events.map do |event|
+                                                             { @inbox_column => event[@inbox_column] }
+                                                           end)
 
-            return event unless ret
+              if ret
+                inboxed = @inbox_ds.where(@inbox_column => events.map do |ev|
+                                                             ev[@inbox_column]
+                                                           end).select_map(@inbox_column)
+              end
+            end
           else
-            begin
+            inboxed = []
+            events.each do |event|
               @inbox_ds.insert(@inbox_column => event[@inbox_column])
+              inboxed << event[@inbox_column]
             rescue Sequel::UniqueConstraintViolation
-              return event
+              # ignore
             end
+            return events if inboxed.empty?
           end
 
-          yield
+          return events if inboxed && inboxed.empty?
+
+          yield events.select { |ev| inboxed.include?(ev[@inbox_column]) }
+
+          events
         end
       end
     end

data/lib/tobox/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tobox
-  VERSION = "0.5.1"
+  VERSION = "0.5.2"
 end

data/lib/tobox/worker.rb

@@ -33,15 +33,16 @@ module Tobox
     def do_work
       return if @finished
 
-      sum_fetched_events = @fetcher.fetch_events do |event|
-        event_type = event[:type].to_sym
-        args = message_to_arguments(event)
-
-        if @handlers.key?(event_type)
-          @handlers[event_type].each do |handler|
-            handler.call(args)
+      sum_fetched_events = @fetcher.fetch_events do |events|
+        cs = @handlers.each_with_object({}) do |(callback, event_types), bucket|
+          events.each do |event|
+            (bucket[callback] ||= []) << message_to_arguments(event) if event_types.include?(event[:type].to_sym)
           end
         end
+
+        cs.each do |callback, evs|
+          callback.call(*evs)
+        end
       end
 
       return if @finished

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: tobox
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - HoneyryderChuck
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-09-26 00:00:00.000000000 Z
+date: 2024-10-24 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement