sbmt-outbox 5.0.4 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 838bf63855d71b86ca0ce7be93fa36c7b35496d8a450485bd1b8f41c29e76d6e
-  data.tar.gz: b37e5777fad8e3d532bf8265513edf89354ffa5b3ce312a5da43b1dfa294165e
+  metadata.gz: 818de79692d37138c630bf2fce625a54becf3a8fa71eafe3c5435d573dbcf665
+  data.tar.gz: 2516c414d14a60851d53fd9f821845179ac12cfb88176a18c1f60d8ba8e38bc9
 SHA512:
-  metadata.gz: 10ecf87fd36a86a85a29e003d4335cee56751a30720ff58fa7aa52395d3118eb1bb168c9685422da2f7dee758c34dc47b8ee32e8ecd0ec8bd103b83d8e041716
-  data.tar.gz: 756e0a98911120354c14c27ca6732272c51c470a8fd2393d9318d60b60de126c959cf297c84a8b8c4ee75a26a2e48ae74396fd57f8d24a138cc849841da2be1f
+  metadata.gz: 0e1c0f1f55ef75e5c73352f0789dc51559656f5868a64f2ca8b0c97c10b2865cd37189dc6b3a5d0346673251800ed1c652df951734a5fe77029ca87dbb0fc1a3
+  data.tar.gz: f0aba9307dcfac3c8e330f1a8a1e40e33439a624c4c40c34ea9223c53c25acff2540ad03c72b00a71c1e963885ecee4e6befc5e5e94d4d78e13ecf6acac4c69e

data/README.md

@@ -117,8 +117,8 @@ create_table :my_outbox_items do |t|
 end
 
 add_index :my_outbox_items, :uuid, unique: true
-add_index :my_outbox_items, [:status, :bucket]
-add_index :my_outbox_items, [:event_name, :event_key]
+add_index :my_outbox_items, [:status, :bucket, :errors_count]
+add_index :my_outbox_items, [:event_name, :event_key, :id]
 add_index :my_outbox_items, :created_at
 ```
 
@@ -222,13 +222,51 @@ Rails.application.config.outbox.tap do |config|
     x[:batch_size] = 200
   end
 
-  # optional
+  # optional (worker v1: DEPRECATED)
   config.worker.tap do |worker|
     # number of batches that one thread will process per rate interval
     worker[:rate_limit] = 10
     # rate interval in seconds
     worker[:rate_interval] = 60
   end
+  
+  # optional (worker v2: default)
+  c.poller = ActiveSupport::OrderedOptions.new.tap do |pc|
+    # max parallel threads (per box-item, globally)
+    pc.concurrency = 6
+    # max threads count (per worker process)
+    pc.threads_count = 1
+    # maximum processing time of the batch, after which the batch will be considered hung and processing will be aborted
+    pc.general_timeout = 60
+    # poll buffer consists of regular items (errors_count = 0, i.e. without any processing errors) and retryable items (errors_count > 0)
+    # max poll buffer size = regular_items_batch_size + retryable_items_batch_size
+    pc.regular_items_batch_size = 200
+    pc.retryable_items_batch_size = 100
+
+    # poll tactic: default is optimal for most cases: rate limit + redis job-queue size threshold
+    # poll tactic: aggressive is for high-intencity data: without rate limits + redis job-queue size threshold
+    # poll tactic: low-priority is for low-intencity data: rate limits + redis job-queue size threshold + + redis job-queue lag threshold
+    pc.tactic = "default"
+    # number of batches that one thread will process per rate interval
+    pc.rate_limit = 60
+    # rate interval in seconds
+    pc.rate_interval = 60
+    # mix / max redis job queue thresholds per box-item for default / aggressive / low-priority poll tactics
+    pc.min_queue_size = 10
+    pc.max_queue_size = 100
+    # min redis job queue time lag threshold per box-item for low-priority poll tactic (in seconds)
+    pc.min_queue_timelag = 5
+    # throttling delay for default / aggressive / low-priority poll tactics (in seconds)
+    pc.queue_delay = 0.1
+  end
+  c.processor = ActiveSupport::OrderedOptions.new.tap do |pc|
+    # max threads count (per worker process)
+    pc.threads_count = 4
+    # maximum processing time of the batch, after which the batch will be considered hung and processing will be aborted
+    pc.general_timeout = 120
+    # BRPOP delay (in seconds) for polling redis job queue per box-item
+    pc.brpop_delay = 2
+  end
 end
 ```
 
@@ -292,7 +330,7 @@ outbox_items:
       - exponential_backoff
 ```
 
-#### Compacted log
+#### Latest available
 
 This strategy ensures idempotency. In short, if a message fails and a later message with the same event_key has already been delivered, then you most likely do not want to re-deliver the first one when it is retried.
 
@@ -303,10 +341,10 @@ outbox_items:
     ...
     retry_strategies:
       - exponential_backoff
-      - compacted_log
+      - latest_available
 ```
 
-The exponential backoff strategy should be used in conjunction with the compact log strategy, and it should come last to minimize the number of database queries.
+The exponential backoff strategy should be used in conjunction with the latest available strategy, and it should come last to minimize the number of database queries.
 
 ### Partition strategies
 
@@ -428,13 +466,24 @@ end
 
 The gem is optionally integrated with OpenTelemetry. If your main application has `opentelemetry-*` gems, the tracing will be configured automatically.
 
-## CLI Arguments
+## CLI Arguments (v1: DEPRECATED)
 
 | Key                   | Description                                                               |
 |-----------------------|---------------------------------------------------------------------------|
 | `--boxes or -b`       | Outbox/Inbox processors to start`                                         |
 | `--concurrency or -c` | Number of threads. Default 10.                                            |
 
+## CLI Arguments (v2: default)
+
+| Key                        | Description                                                          |
+|----------------------------|----------------------------------------------------------------------|
+| `--boxes or -b`            | Outbox/Inbox processors to start`                                    |
+| `--concurrency or -c`      | Number of process threads. Default 4.                                |
+| `--poll-concurrency or -p` | Number of poller partitions. Default 6.                              |
+| `--poll-threads or -n`     | Number of poll threads. Default 1.                                   |
+| `--poll-tactic or -t`      | Poll tactic. Default "default".                                      |
+| `--worker-version or -w`   | Worker version. Default 2.                                           |
+
 ## Development & Test
 
 ### Installation

data/app/interactors/sbmt/outbox/process_item.rb

@@ -5,6 +5,7 @@ module Sbmt
     class ProcessItem < Sbmt::Outbox::DryInteractor
       param :item_class, reader: :private
       param :item_id, reader: :private
+      option :worker_version, reader: :private, optional: true, default: -> { 1 }
 
       METRICS_COUNTERS = %i[error_counter retry_counter sent_counter fetch_error_counter discarded_counter].freeze
 
@@ -254,7 +255,7 @@ module Sbmt
       end
 
       def labels_for(item)
-        {type: box_type, name: box_name, owner: owner, partition: item&.partition}
+        {worker_version: worker_version, type: box_type, name: box_name, owner: owner, partition: item&.partition}
       end
 
       def counters

data/app/interactors/sbmt/outbox/retry_strategies/base.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module RetryStrategies
+      class Base < Outbox::DryInteractor
+        param :item
+
+        def call
+          raise NotImplementedError, "Implement #call for Sbmt::Outbox::RetryStrategies::Base"
+        end
+      end
+    end
+  end
+end

data/app/interactors/sbmt/outbox/retry_strategies/compacted_log.rb

@@ -3,38 +3,8 @@
 module Sbmt
   module Outbox
     module RetryStrategies
-      class CompactedLog < Outbox::DryInteractor
-        param :outbox_item
-
-        def call
-          unless outbox_item.has_attribute?(:event_key)
-            return Failure(:missing_event_key)
-          end
-
-          if outbox_item.event_key.nil?
-            return Failure(:empty_event_key)
-          end
-
-          if delivered_later?
-            Failure(:discard_item)
-          else
-            Success()
-          end
-        end
-
-        private
-
-        def delivered_later?
-          scope = outbox_item.class
-            .where("id > ?", outbox_item)
-            .where(event_key: outbox_item.event_key, status: Sbmt::Outbox::BaseItem.statuses[:delivered])
-
-          if outbox_item.has_attribute?(:event_name) && outbox_item.event_name.present?
-            scope = scope.where(event_name: outbox_item.event_name)
-          end
-
-          scope.exists?
-        end
+      class CompactedLog < LatestAvailable
+        # exists only as alias for backward compatibility
       end
     end
   end

data/app/interactors/sbmt/outbox/retry_strategies/exponential_backoff.rb

@@ -3,13 +3,11 @@
 module Sbmt
   module Outbox
     module RetryStrategies
-      class ExponentialBackoff < Outbox::DryInteractor
-        param :outbox_item
-
+      class ExponentialBackoff < Base
         def call
-          delay = backoff(outbox_item.config).interval_at(outbox_item.errors_count - 1)
+          delay = backoff(item.config).interval_at(item.errors_count - 1)
 
-          still_early = outbox_item.processed_at + delay.seconds > Time.current
+          still_early = item.processed_at + delay.seconds > Time.current
 
           if still_early
             Failure(:skip_processing)

data/app/interactors/sbmt/outbox/retry_strategies/latest_available.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module RetryStrategies
+      class LatestAvailable < Base
+        def call
+          unless item.has_attribute?(:event_key)
+            return Failure(:missing_event_key)
+          end
+
+          if item.event_key.nil?
+            return Failure(:empty_event_key)
+          end
+
+          if delivered_later?
+            Failure(:discard_item)
+          else
+            Success()
+          end
+        end
+
+        private
+
+        def delivered_later?
+          scope = item.class
+            .where("id > ?", item)
+            .where(event_key: item.event_key)
+
+          if item.has_attribute?(:event_name) && item.event_name.present?
+            scope = scope.where(event_name: item.event_name)
+          end
+
+          scope.exists?
+        end
+      end
+    end
+  end
+end

data/app/interactors/sbmt/outbox/retry_strategies/no_delay.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module RetryStrategies
+      class NoDelay < Base
+        def call
+          Success()
+        end
+      end
+    end
+  end
+end

data/app/models/sbmt/outbox/base_item.rb

@@ -20,15 +20,16 @@ module Sbmt
           @config ||= lookup_config.new(box_name)
         end
 
+        def calc_bucket_partitions(count)
+          (0...count).to_a
+            .each_with_object({}) do |x, m|
+            m[x] = (0...config.bucket_size).to_a
+              .select { |p| p % count == x }
+          end
+        end
+
         def partition_buckets
-          @partition_buckets ||=
-            (0...config.partition_size)
-              .to_a
-              .each_with_object({}) do |x, m|
-                m[x] = (0...config.bucket_size)
-                  .to_a
-                  .select { |p| p % config.partition_size == x }
-              end
+          @partition_buckets ||= calc_bucket_partitions(config.partition_size)
         end
 
         def bucket_partitions

data/app/models/sbmt/outbox/base_item_config.rb

@@ -23,7 +23,11 @@ module Sbmt
       end
 
       def partition_size
-        @partition_size ||= (options[:partition_size] || 1).to_i
+        @partition_size ||= (partition_size_raw || 1).to_i
+      end
+
+      def partition_size_raw
+        @partition_size_raw ||= options[:partition_size]
       end
 
       def retention
@@ -47,7 +51,12 @@ module Sbmt
       end
 
       def retry_strategies
-        @retry_strategies ||= Array.wrap(options[:retry_strategies]).map do |str_name|
+        return @retry_strategies if defined?(@retry_strategies)
+
+        configured_strategies = options[:retry_strategies]
+        strategies = configured_strategies.presence || %w[exponential_backoff latest_available]
+
+        @retry_strategies ||= Array.wrap(strategies).map do |str_name|
           "Sbmt::Outbox::RetryStrategies::#{str_name.camelize}".constantize
         end
       end

data/config/initializers/yabeda.rb

@@ -3,6 +3,8 @@
 Yabeda.configure do
   # error_counter retry_counter sent_counter fetch_error_counter discarded_counter
   group :outbox do
+    default_tag(:worker_version, 1)
+
     counter :created_counter,
       tags: %i[type name partition owner],
       comment: "The total number of created messages"
@@ -44,28 +46,53 @@ Yabeda.configure do
   end
 
   group :box_worker do
+    default_tag(:worker_version, 1)
+    default_tag(:worker_name, "worker")
+
     counter :job_counter,
-      tags: %i[type name partition worker_number state],
+      tags: %i[type name partition state],
       comment: "The total number of processed jobs"
 
     counter :job_timeout_counter,
-      tags: %i[type name partition_key worker_number],
+      tags: %i[type name partition_key],
       comment: "Requeue of a job that occurred while processing the batch"
 
     counter :job_items_counter,
-      tags: %i[type name partition worker_number],
+      tags: %i[type name partition],
       comment: "The total number of processed items in jobs"
 
     histogram :job_execution_runtime,
       comment: "A histogram of the job execution time",
       unit: :seconds,
-      tags: %i[type name partition worker_number],
+      tags: %i[type name partition],
       buckets: [0.5, 1, 2.5, 5, 10, 15, 30, 45, 60, 90, 120, 180, 240, 300, 600]
 
     histogram :item_execution_runtime,
       comment: "A histogram of the item execution time",
       unit: :seconds,
-      tags: %i[type name partition worker_number],
+      tags: %i[type name partition],
+      buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300]
+
+    counter :batches_per_poll_counter,
+      tags: %i[type name partition],
+      comment: "The total number of poll batches per poll"
+
+    gauge :redis_job_queue_size,
+      tags: %i[type name partition],
+      comment: "The total size of redis job queue"
+
+    gauge :redis_job_queue_time_lag,
+      tags: %i[type name partition],
+      comment: "The total time lag of redis job queue"
+
+    counter :poll_throttling_counter,
+      tags: %i[type name partition throttler status],
+      comment: "The total number of poll throttlings"
+
+    histogram :poll_throttling_runtime,
+      comment: "A histogram of the poll throttling time",
+      unit: :seconds,
+      tags: %i[type name partition throttler],
       buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300]
   end
 end

data/lib/generators/helpers/migration.rb

@@ -44,8 +44,8 @@ module Outbox
             end
 
             add_index :#{table_name}, :uuid, unique: true
-            add_index :#{table_name}, [:status, :bucket]
-            add_index :#{table_name}, :event_key
+            add_index :#{table_name}, [:status, :bucket, :errors_count]
+            add_index :#{table_name}, [:event_key, :id]
             add_index :#{table_name}, :created_at
           RUBY
 

data/lib/sbmt/outbox/cli.rb

@@ -20,15 +20,47 @@ module Sbmt
       option :concurrency,
         aliases: "-c",
         type: :numeric,
-        default: 10,
-        desc: "Number of threads"
+        desc: "Number of threads (processor)"
+      option :poll_concurrency,
+        aliases: "-p",
+        type: :numeric,
+        desc: "Number of poller partitions"
+      option :poll_threads,
+        aliases: "-n",
+        type: :numeric,
+        desc: "Number of threads (poller)"
+      option :poll_tactic,
+        aliases: "-t",
+        type: :string,
+        desc: "Poll tactic: [default, low-priority, aggressive]"
+      option :worker_version,
+        aliases: "-w",
+        type: :numeric,
+        desc: "Worker version: [1 | 2]"
       def start
         load_environment
 
-        worker = Sbmt::Outbox::Worker.new(
-          boxes: format_boxes(options[:box]),
-          concurrency: options[:concurrency]
-        )
+        version = options[:worker_version] || Outbox.default_worker_version
+
+        boxes = format_boxes(options[:box])
+        check_deprecations!(boxes, version)
+
+        worker = if version == 1
+          Sbmt::Outbox::V1::Worker.new(
+            boxes: boxes,
+            concurrency: options[:concurrency] || 10
+          )
+        elsif version == 2
+          Sbmt::Outbox::V2::Worker.new(
+            boxes: boxes,
+            poll_tactic: options[:poll_tactic],
+            poller_threads_count: options[:poll_threads],
+            poller_partitions_count: options[:poll_concurrency],
+            processor_concurrency: options[:concurrency] || 4
+          )
+        else
+          raise "Worker version #{version} is invalid, available versions: 1|2"
+        end
 
         Sbmt::Outbox.current_worker = worker
 
@@ -45,11 +77,22 @@ module Sbmt
 
       private
 
+      def check_deprecations!(boxes, version)
+        return unless version == 2
+
+        boxes.each do |item_class|
+          next if item_class.config.partition_size_raw.blank?
+
+          raise "partition_size option is invalid and cannot be used with worker v2, please remove it from config/outbox.yml for #{item_class.name.underscore}"
+        end
+      end
+
       def load_environment
         load(lookup_outboxfile)
 
         require "sbmt/outbox"
-        require "sbmt/outbox/worker"
+        require "sbmt/outbox/v1/worker"
+        require "sbmt/outbox/v2/worker"
       end
 
       def lookup_outboxfile

data/lib/sbmt/outbox/engine.rb

@@ -26,6 +26,31 @@ module Sbmt
           c.rate_interval = 60
           c.shuffle_jobs = true
         end
+        c.default_worker_version = 2
+
+        # worker v2
+        c.poller = ActiveSupport::OrderedOptions.new.tap do |pc|
+          pc.concurrency = 6
+          pc.threads_count = 2
+          pc.general_timeout = 60
+          pc.regular_items_batch_size = 200
+          pc.retryable_items_batch_size = 100
+
+          pc.tactic = "default"
+          pc.rate_limit = 60
+          pc.rate_interval = 60
+          pc.min_queue_size = 10
+          pc.max_queue_size = 100
+          pc.min_queue_timelag = 5
+          pc.queue_delay = 0.1
+        end
+        c.processor = ActiveSupport::OrderedOptions.new.tap do |pc|
+          pc.threads_count = 4
+          pc.general_timeout = 120
+          pc.cutoff_timeout = 60
+          pc.brpop_delay = 2
+        end
+
         c.database_switcher = "Sbmt::Outbox::DatabaseSwitcher"
         c.batch_process_middlewares = []
         c.item_process_middlewares = []

data/lib/sbmt/outbox/logger.rb

@@ -5,6 +5,12 @@ module Sbmt
     class Logger
       delegate :logger, to: :Rails
 
+      def log_debug(message, **params)
+        with_tags(**params) do
+          logger.debug(message)
+        end
+      end
+
       def log_info(message, **params)
         with_tags(**params) do
           logger.info(message)

data/lib/sbmt/outbox/v1/thread_pool.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/v1/throttler"
+
+module Sbmt
+  module Outbox
+    module V1
+      class ThreadPool
+        BREAK = Object.new.freeze
+        SKIPPED = Object.new.freeze
+        PROCESSED = Object.new.freeze
+
+        def initialize(&block)
+          self.task_source = block
+          self.task_mutex = Mutex.new
+          self.stopped = true
+        end
+
+        def next_task
+          task_mutex.synchronize do
+            return if stopped
+            item = task_source.call
+
+            if item == BREAK
+              self.stopped = true
+              return
+            end
+
+            item
+          end
+        end
+
+        def start(concurrency:)
+          self.stopped = false
+          result = run_threads(count: concurrency) do |item|
+            yield worker_number, item
+          end
+
+          raise result if result.is_a?(Exception)
+          nil
+        ensure
+          self.stopped = true
+        end
+
+        def stop
+          self.stopped = true
+        end
+
+        def worker_number
+          Thread.current["thread_pool_worker_number:#{object_id}"]
+        end
+
+        private
+
+        attr_accessor :task_source, :task_mutex, :stopped
+
+        def run_threads(count:)
+          exception = nil
+
+          in_threads(count: count) do |worker_num|
+            self.worker_number = worker_num
+            # We don't want to start all threads at the same time
+            random_sleep = rand * (worker_num + 1)
+
+            throttler = Throttler.new(
+              limit: Outbox.config.worker.rate_limit,
+              interval: Outbox.config.worker.rate_interval + random_sleep
+            )
+
+            sleep(random_sleep)
+
+            last_result = nil
+            until exception
+              throttler.wait if last_result == PROCESSED
+              item = next_task
+              break unless item
+
+              begin
+                last_result = yield item
+              rescue Exception => e # rubocop:disable Lint/RescueException
+                exception = e
+              end
+            end
+          end
+
+          exception
+        end
+
+        def in_threads(count:)
+          threads = []
+
+          Thread.handle_interrupt(Exception => :never) do
+            Thread.handle_interrupt(Exception => :immediate) do
+              count.times do |i|
+                threads << Thread.new { yield(i) }
+              end
+              threads.map(&:value)
+            end
+          ensure
+            threads.each(&:kill)
+          end
+        end
+
+        def worker_number=(num)
+          Thread.current["thread_pool_worker_number:#{object_id}"] = num
+        end
+      end
+    end
+  end
+end