tobox 0.5.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8f6e85f173d4636a64617ede574141d4b2365799b774a6e56a76b70a59db6ba
-  data.tar.gz: 49bb80cc99034ba4db846964988c8d1fc2a1d8e062decbe57d5b786e892afb8b
+  metadata.gz: 1d1d0e33f4747993a500d22dddbda2eaf2118e0351342a9d09acdf9b6516f86c
+  data.tar.gz: e75a3699edf170a1711cc749a8df6ab6b6a4a829057246e1fd690065d89224c5
 SHA512:
-  metadata.gz: e1e3f588dbe62ceaf50d6d1e67e3fbcecf0798027728907961d2bd92d18ad5cf779d215c6cad614ace6bdf0c6bc00e8c32a2e3f0ddd30ae1c7f6c2f6524b3133
-  data.tar.gz: 6170813875d41ed03012ded3a418290446abfeac34a80fb5a7e21df3741032df23b05261d7b87c4c9ff3c2d230738ff20e93c7d178969606e092617d6f91357c
+  metadata.gz: d097b80a68d2ec806f521f0a36113bf74dac00fd0e7080e95496ff0b93a2f8e2e1f594b9d13a4b02e6d094ff260ed839d76bd1116f774c8672c01fda5761f5d6
+  data.tar.gz: cf138ca863803ae62e165cbd43846c6e35f6b500b344b62fbb1d0a906839cd4d8cc00f720ead1de220dc34802d70baca3a3bae730d01b99fb7d7f550015a6f1e

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 ## [Unreleased]
 
+## [0.6.0] - 2024-10-25
+
+### Features
+
+#### Batch Events handling
+
+It's now possible to handle N events at a time.
+
+```ruby
+# tobox.rb
+
+batch_size 10 # fetching 10 events at a time
+
+on("user_created", "user_updated") do |*events| # 10 events at most
+  if events.size == 1
+    DataLakeService.user_created(events.first)
+  else
+    DataLakeService.batch_users_created(events)
+  end
+end
+```
+
+This also supports raising errors only for a subset of the events which failed processing (more info in the README).
+
 ## [0.5.2] - 2024-10-22
 
 ## Bugfixes

data/README.md

@@ -21,6 +21,8 @@ Simple, data-first events processing framework based on the [transactional outbo
   - [Sentry](#sentry)
   - [Datadog](#datadog)
   - [Stats](#stats)
+- [Advanced](#advanced)
+  - [Batch Events Handling](#batch-events)
 - [Supported Rubies](#supported-rubies)
 - [Rails support](#rails-support)
 - [Why?](#why)
@@ -101,8 +103,8 @@ database Sequel.connect("postgres://user:pass@dbhost/database")
 # concurrency 8
 on("user_created") do |event|
   puts "created user #{event[:after]["id"]}"
-  DataLakeService.user_created(user_data_hash)
-  BillingService.bill_user_account(user_data_hash)
+  DataLakeService.user_created(event)
+  BillingService.bill_user_account(event)
 end
 on("user_updated") do |event|
   # ...
@@ -660,6 +662,56 @@ c.on_stats(5) do |stats_collector, db|
 end
 ```
 
+<a id="markdown-advanced" name="advanced"></a>
+## Advanced
+
+<a id="markdown-batch-events" name="batch-events"></a>
+### Batch Events Handling
+
+You may start hitting a scale where the workload generated by `tobox` puts the master replica under water. Particularly with PostgreSQL, which isn't optimized for writes, this manifests in CPU usage spiking due to index bypasses, or locks on accessing shared buffers.
+
+A way to aleviate this is by hnadling events in batches. By handling N events at a time, the database can drain events more efficiently, while you can either still handle them one by one, or batch them, if possible. For instance, the AWS SDK contains batching alternatives of several APIs, including the SNS publish API.
+
+You can do so by setting a batch size in your configuration, and spread the arguments in the event handler:
+
+```ruby
+# tobox.rb
+
+batch_size 10 # fetching 10 events at a time
+
+on("user_created", "user_updated") do |*events| # 10 events at most
+  if events.size == 1
+    DataLakeService.user_created(events.first)
+  else
+    DataLakeService.batch_users_created(events)
+  end
+end
+```
+
+In case you're using a batch API solution which may fail for a subset of events, you are able to communicate which events from the batch failed by using `Tobox.raise_batch_errors` API:
+
+```ruby
+on("user_created", "user_updated") do |*events| # 10 events at most
+  if events.size == 1
+    DataLakeService.user_created(events.first)
+  else
+    success, failed_events_with_errors = DataLakeService.batch_users_created(events)
+
+    # handle success first
+
+    batch_errors = failed_events_with_errors.to_h do |event, exception|
+      [
+        events.index(event),
+        exception
+      ]
+    end
+
+    # events identified by the batch index will be retried.
+    Tobox.raise_batch_errors(batch_errors)
+  end
+end
+```
+
 <a id="markdown-supported-rubies" name="supported-rubies"></a>
 ## Supported Rubies
 

data/lib/tobox/fetcher.rb

@@ -93,9 +93,10 @@ module Tobox
       events, error = yield_events(event_ids, &blk)
 
       events.each do |event|
-        if error
-          event.merge!(mark_as_error(event, error))
-          handle_error_event(event, error)
+        event_error = error || event[:error]
+        if event_error
+          event.merge!(mark_as_error(event, event_error))
+          handle_error_event(event, event_error)
         else
           handle_after_event(event)
         end
@@ -109,9 +110,28 @@ module Tobox
       begin
         events = events_ds.all
 
-        yield events
+        unless events.empty?
+          errors_by_id = catch(:tobox_batch_errors) do
+            yield events
+            nil
+          end
+
+          # some events from batch errored
+          if errors_by_id
+            failed = events.values_at(*errors_by_id.keys)
+            successful = events - failed
+
+            # fill in with batch error
+            failed.each do |ev|
+              ev[:error] = errors_by_id[events.index(ev)]
+            end
 
-        events_ds.delete unless events.empty?
+            # delete successful
+            @ds.where(id: successful.map { |ev| ev[:id] }).delete unless successful.empty?
+          else
+            events_ds.delete
+          end
+        end
       rescue StandardError => e
         error = e
       end
@@ -129,7 +149,7 @@ module Tobox
                                 seconds: @exponential_retry_factor**(event[:attempts] - 1)),
         # run_at: Sequel.date_add(Sequel::CURRENT_TIMESTAMP,
         #                         seconds: Sequel.function(:POWER, Sequel[@table][:attempts] + 1,  4)),
-        last_error: "#{error.message}\n#{error.backtrace.join("\n")}"
+        last_error: error.full_message(highlight: false)
       }
 
       set_event_retry_attempts(event, update_params)

data/lib/tobox/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tobox
-  VERSION = "0.5.2"
+  VERSION = "0.6.0"
 end

data/lib/tobox.rb

@@ -34,6 +34,32 @@ module Tobox
       h.synchronize { h[name] = mod }
     end
   end
+
+  # when using batch sizes higher than 1, this method can be used to signal multiple errors
+  # for a subset of the events which may have failed processing; these events are identified
+  # by the index inside the batch.
+  #
+  #     on(:event_type) do |*events|
+  #       successful, failed = handle_event_batch(events)
+  #
+  #       deal_with_success(successful)
+  #
+  #       batch_errors = failed.to_h do |failed_event|
+  #         [
+  #            events.index(failed_event),
+  #            MyException.new("failed handling process batch")
+  #         ]
+  #       end
+  #
+  #       Tobox.raise_batch_error(batch_errors)
+  #     end
+  def self.raise_batch_errors(batch_errors)
+    unless batch_errors.respond_to?(:to_hash) && batch_errors.all? { |k, v| k.is_a?(Integer) && v.is_a?(Exception) }
+      raise "batch errors must be an array of index-to-exception tuples"
+    end
+
+    throw(:tobox_batch_errors, batch_errors.to_h)
+  end
 end
 
 require_relative "tobox/fetcher"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tobox
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.0
 platform: ruby
 authors:
 - HoneyryderChuck
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-24 00:00:00.000000000 Z
+date: 2024-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logger