tobox 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d1d0e33f4747993a500d22dddbda2eaf2118e0351342a9d09acdf9b6516f86c
-  data.tar.gz: e75a3699edf170a1711cc749a8df6ab6b6a4a829057246e1fd690065d89224c5
+  metadata.gz: 7fcb3a283fa1bb64036319c64c537891a3c4640331ef96058c0a80f311ff6d9a
+  data.tar.gz: e6f6a8cd2b0ec8f38cc5e52added655c1922cc79be890fa79c72df3a56418990
 SHA512:
-  metadata.gz: d097b80a68d2ec806f521f0a36113bf74dac00fd0e7080e95496ff0b93a2f8e2e1f594b9d13a4b02e6d094ff260ed839d76bd1116f774c8672c01fda5761f5d6
-  data.tar.gz: cf138ca863803ae62e165cbd43846c6e35f6b500b344b62fbb1d0a906839cd4d8cc00f720ead1de220dc34802d70baca3a3bae730d01b99fb7d7f550015a6f1e
+  metadata.gz: 9b48f7d89bfecba3313b4807fb7cc3139aac28f43c43755657feedddfca7c7fb9048e1b6898becbc3fe05ee9ba28b8a1e57948661743173870ea7b04785a2ddc
+  data.tar.gz: 24f01a9794d77d90e406b71be0045c309e7280e46b98ad8aad2375935d47647963780c834833b76a0a4ed838ba5dd95aa184c17e3707059e51cd4ed2f334c932

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## [Unreleased]
 
+## [0.6.1] - 2024-10-30
+
+### Improvements
+
+When possible, use `UPDATE FROM outbox RETURNING WHERE (SELECT id .... FOR UPDATE SKIP LOCKED` to fetch-and-update events in one go.
+
 ## [0.6.0] - 2024-10-25
 
 ### Features

data/lib/tobox/fetcher.rb

@@ -30,8 +30,6 @@ module Tobox
 
       @batch_size = configuration[:batch_size]
 
-      @mark_as_fetched_params = { attempts: Sequel[@table][:attempts] + 1, last_error: nil }
-
       @before_event_handlers = Array(@configuration.lifecycle_events[:before_event])
       @after_event_handlers = Array(@configuration.lifecycle_events[:after_event])
       @error_event_handlers = Array(@configuration.lifecycle_events[:error_event])
@@ -40,16 +38,15 @@ module Tobox
     def fetch_events(&blk)
       num_events = 0
       events_tr do
-        event_ids = nil
-        # @type var event_ids: Array[Integer]
+        events = nil
+        # @type var events: Array[event]?
 
         event_id_tr do
-          event_ids = fetch_event_ids
-          mark_as_fetched(event_ids) unless event_ids.empty?
+          events = do_fetch_events
         end
 
-        if event_ids && !event_ids.empty?
-          with_events(event_ids) do |events|
+        if events && !events.empty?
+          with_events(events) do |events|
             num_events = events.size
 
             prepare_events(events, &blk)
@@ -74,11 +71,11 @@ module Tobox
     def fetch_event_ids
       @pick_next_sql.for_update
                     .skip_locked
-                    .limit(@batch_size).select_map(:id) # lock starts here
+                    .limit(@batch_size).select(:id) # lock starts here
     end
 
-    def mark_as_fetched(event_ids)
-      @ds.where(id: event_ids).update(@mark_as_fetched_params)
+    def do_fetch_events
+      @ds.where(id: fetch_event_ids).all
     end
 
     def events_tr(&block)
@@ -89,11 +86,11 @@ module Tobox
       yield
     end
 
-    def with_events(event_ids, &blk)
-      events, error = yield_events(event_ids, &blk)
+    def with_events(events, &blk)
+      yield_events(events, &blk)
 
       events.each do |event|
-        event_error = error || event[:error]
+        event_error = event[:error]
         if event_error
           event.merge!(mark_as_error(event, event_error))
           handle_error_event(event, event_error)
@@ -103,40 +100,33 @@ module Tobox
       end
     end
 
-    def yield_events(event_ids)
-      events_ds = @ds.where(id: event_ids)
-      events = error = nil
+    def yield_events(events)
+      unless events.empty?
+        errors_by_id = catch(:tobox_batch_errors) do
+          yield events
+          nil
+        end
 
-      begin
-        events = events_ds.all
+        # some events from batch errored
+        if errors_by_id
+          failed = events.values_at(*errors_by_id.keys)
+          successful = events - failed
 
-        unless events.empty?
-          errors_by_id = catch(:tobox_batch_errors) do
-            yield events
-            nil
+          # fill in with batch error
+          failed.each do |ev|
+            ev[:error] = errors_by_id[events.index(ev)]
           end
 
-          # some events from batch errored
-          if errors_by_id
-            failed = events.values_at(*errors_by_id.keys)
-            successful = events - failed
-
-            # fill in with batch error
-            failed.each do |ev|
-              ev[:error] = errors_by_id[events.index(ev)]
-            end
-
-            # delete successful
-            @ds.where(id: successful.map { |ev| ev[:id] }).delete unless successful.empty?
-          else
-            events_ds.delete
-          end
+          # delete successful
+          @ds.where(id: successful.map { |ev| ev[:id] }).delete unless successful.empty?
+        else
+          @ds.where(id: events.map { |ev| ev[:id] }).delete
         end
-      rescue StandardError => e
-        error = e
       end
-
-      [events, error]
+    rescue StandardError => e
+      events.each do |event|
+        event[:error] = e
+      end
     end
 
     def log_message(msg)
@@ -145,10 +135,8 @@ module Tobox
 
     def mark_as_error(event, error)
       update_params = {
-        run_at: Sequel.date_add(Sequel::CURRENT_TIMESTAMP,
-                                seconds: @exponential_retry_factor**(event[:attempts] - 1)),
-        # run_at: Sequel.date_add(Sequel::CURRENT_TIMESTAMP,
-        #                         seconds: Sequel.function(:POWER, Sequel[@table][:attempts] + 1,  4)),
+        run_at: calculate_event_retry_interval(event[:attempts]),
+        attempts: Sequel[@table][:attempts] + 1,
         last_error: error.full_message(highlight: false)
       }
 
@@ -165,6 +153,12 @@ module Tobox
       end
     end
 
+    def calculate_event_retry_interval(attempts)
+      # Sequel.date_add(Sequel::CURRENT_TIMESTAMP,
+      #                 seconds: Sequel.function(:POWER, Sequel[@table][:attempts] + 1,  4)
+      Sequel.date_add(Sequel::CURRENT_TIMESTAMP, seconds: @exponential_retry_factor**attempts)
+    end
+
     def to_message(event)
       {
         id: event[:id],

data/lib/tobox/plugins/progress.rb

@@ -10,13 +10,34 @@ module Tobox
       module FetcherMethods
         private
 
-        def initialize(_, configuration)
-          super
+        def do_fetch_events
+          # mark events as invisible by using run_at as a visibility timeout
+          mark_as_fetched_params = {
+            run_at: Sequel.date_add(
+              Sequel::CURRENT_TIMESTAMP,
+              seconds: @configuration[:visibility_timeout]
+            ),
+            attempts: Sequel[@table][:attempts] + 1,
+            last_error: nil
+          }
+
+          if @ds.supports_returning?(:update)
+            @ds.where(id: fetch_event_ids).returning.update(mark_as_fetched_params)
+          else
+            event_ids = fetch_event_ids.select_map(:id)
+            events_ds = @ds.where(id: event_ids)
+            events_ds.update(mark_as_fetched_params)
+            events_ds.first(@batch_size)
+          end
+        end
 
-          @mark_as_fetched_params[:run_at] = Sequel.date_add(
-            Sequel::CURRENT_TIMESTAMP,
-            seconds: configuration[:visibility_timeout]
-          )
+        def calculate_event_retry_interval(attempts)
+          super(attempts - 1)
+        end
+
+        def set_event_retry_attempts(event, update_params)
+          update_params.delete(:attempts)
+          super
         end
 
         def events_tr

data/lib/tobox/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tobox
-  VERSION = "0.6.0"
+  VERSION = "0.6.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tobox
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - HoneyryderChuck
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-25 00:00:00.000000000 Z
+date: 2024-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logger