sbmt-outbox 5.0.1 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a240a2348f45c36bf76377a8edbc3af77cea2caf1c3240f85b68072fbb99cd32
-  data.tar.gz: 33a71dec75089581641a2ddc1bde47e35994c3e9b5f383ace8865077e3620df1
+  metadata.gz: 818de79692d37138c630bf2fce625a54becf3a8fa71eafe3c5435d573dbcf665
+  data.tar.gz: 2516c414d14a60851d53fd9f821845179ac12cfb88176a18c1f60d8ba8e38bc9
 SHA512:
-  metadata.gz: cd32451e9af08be2e0c5d3793b34a9c3f23997482304e9783e8dc0bab5b9c01d46e265ae4cb5ed8649429972f28bb51aa8421fdc5ce418aebf4f41a21d99594e
-  data.tar.gz: a57430bf42f65f452bf20e6d3f468e2c750085b8fc50e558d6ad99f3d39cb317bcfd0a0b3ac13ba8c04eb167001c0be027a32de9abb64c32daa04a1b99dcb604
+  metadata.gz: 0e1c0f1f55ef75e5c73352f0789dc51559656f5868a64f2ca8b0c97c10b2865cd37189dc6b3a5d0346673251800ed1c652df951734a5fe77029ca87dbb0fc1a3
+  data.tar.gz: f0aba9307dcfac3c8e330f1a8a1e40e33439a624c4c40c34ea9223c53c25acff2540ad03c72b00a71c1e963885ecee4e6befc5e5e94d4d78e13ecf6acac4c69e

data/README.md

@@ -23,6 +23,10 @@ And then execute:
 bundle install
 ```
 
+## Demo
+
+Learn how to use this gem and how it works with Ruby on Rails at here https://github.com/SberMarket-Tech/outbox-example-apps
+
 ## Auto configuration
 
 We recommend going through the configuration and files creation process using the following Rails generators:
@@ -113,8 +117,8 @@ create_table :my_outbox_items do |t|
 end
 
 add_index :my_outbox_items, :uuid, unique: true
-add_index :my_outbox_items, [:status, :bucket]
-add_index :my_outbox_items, [:event_name, :event_key]
+add_index :my_outbox_items, [:status, :bucket, :errors_count]
+add_index :my_outbox_items, [:event_name, :event_key, :id]
 add_index :my_outbox_items, :created_at
 ```
 
@@ -146,7 +150,11 @@ default: &default
       max_retries: 3 # default 0, the number of retries before the item will be marked as failed
       transports: # transports section
         produce_message: # underscored transport class name
-          topic: "my-topic-name" # default transport arguments
+          # transport reserved options
+          class: produce_message # optional; default is inferred from transport name
+          disposable: false # optional; default false; if true, the transport class will be instantiated only once
+          # ProduceMessage instance arguments
+          topic: "my-topic-name"
 
 development:
   <<: *default
@@ -182,9 +190,11 @@ Transports are defined as follows when `event_name` is used:
 outbox_items:
   my_outbox_item:
     transports:
+        # transport reserved options
       - class: produce_message
         event_name: "order_created" # event name marker
-        topic: "order_created_topic" # some transport default argument
+        # ProduceMessage instance arguments
+        topic: "order_created_topic" # some transport argument
       - class: produce_message
         event_name: "orders_completed"
         topic: "orders_completed_topic"
@@ -212,13 +222,51 @@ Rails.application.config.outbox.tap do |config|
     x[:batch_size] = 200
   end
 
-  # optional
+  # optional (worker v1: DEPRECATED)
   config.worker.tap do |worker|
     # number of batches that one thread will process per rate interval
     worker[:rate_limit] = 10
     # rate interval in seconds
     worker[:rate_interval] = 60
   end
+  
+  # optional (worker v2: default)
+  c.poller = ActiveSupport::OrderedOptions.new.tap do |pc|
+    # max parallel threads (per box-item, globally)
+    pc.concurrency = 6
+    # max threads count (per worker process)
+    pc.threads_count = 1
+    # maximum processing time of the batch, after which the batch will be considered hung and processing will be aborted
+    pc.general_timeout = 60
+    # poll buffer consists of regular items (errors_count = 0, i.e. without any processing errors) and retryable items (errors_count > 0)
+    # max poll buffer size = regular_items_batch_size + retryable_items_batch_size
+    pc.regular_items_batch_size = 200
+    pc.retryable_items_batch_size = 100
+
+    # poll tactic: default is optimal for most cases: rate limit + redis job-queue size threshold
+    # poll tactic: aggressive is for high-intencity data: without rate limits + redis job-queue size threshold
+    # poll tactic: low-priority is for low-intencity data: rate limits + redis job-queue size threshold + + redis job-queue lag threshold
+    pc.tactic = "default"
+    # number of batches that one thread will process per rate interval
+    pc.rate_limit = 60
+    # rate interval in seconds
+    pc.rate_interval = 60
+    # mix / max redis job queue thresholds per box-item for default / aggressive / low-priority poll tactics
+    pc.min_queue_size = 10
+    pc.max_queue_size = 100
+    # min redis job queue time lag threshold per box-item for low-priority poll tactic (in seconds)
+    pc.min_queue_timelag = 5
+    # throttling delay for default / aggressive / low-priority poll tactics (in seconds)
+    pc.queue_delay = 0.1
+  end
+  c.processor = ActiveSupport::OrderedOptions.new.tap do |pc|
+    # max threads count (per worker process)
+    pc.threads_count = 4
+    # maximum processing time of the batch, after which the batch will be considered hung and processing will be aborted
+    pc.general_timeout = 120
+    # BRPOP delay (in seconds) for polling redis job queue per box-item
+    pc.brpop_delay = 2
+  end
 end
 ```
 
@@ -282,7 +330,7 @@ outbox_items:
       - exponential_backoff
 ```
 
-#### Compacted log
+#### Latest available
 
 This strategy ensures idempotency. In short, if a message fails and a later message with the same event_key has already been delivered, then you most likely do not want to re-deliver the first one when it is retried.
 
@@ -293,10 +341,10 @@ outbox_items:
     ...
     retry_strategies:
       - exponential_backoff
-      - compacted_log
+      - latest_available
 ```
 
-The exponential backoff strategy should be used in conjunction with the compact log strategy, and it should come last to minimize the number of database queries.
+The exponential backoff strategy should be used in conjunction with the latest available strategy, and it should come last to minimize the number of database queries.
 
 ### Partition strategies
 
@@ -418,13 +466,24 @@ end
 
 The gem is optionally integrated with OpenTelemetry. If your main application has `opentelemetry-*` gems, the tracing will be configured automatically.
 
-## CLI Arguments
+## CLI Arguments (v1: DEPRECATED)
 
 | Key                   | Description                                                               |
 |-----------------------|---------------------------------------------------------------------------|
 | `--boxes or -b`       | Outbox/Inbox processors to start`                                         |
 | `--concurrency or -c` | Number of threads. Default 10.                                            |
 
+## CLI Arguments (v2: default)
+
+| Key                        | Description                                                          |
+|----------------------------|----------------------------------------------------------------------|
+| `--boxes or -b`            | Outbox/Inbox processors to start`                                    |
+| `--concurrency or -c`      | Number of process threads. Default 4.                                |
+| `--poll-concurrency or -p` | Number of poller partitions. Default 6.                              |
+| `--poll-threads or -n`     | Number of poll threads. Default 1.                                   |
+| `--poll-tactic or -t`      | Poll tactic. Default "default".                                      |
+| `--worker-version or -w`   | Worker version. Default 2.                                           |
+
 ## Development & Test
 
 ### Installation

data/app/interactors/sbmt/outbox/process_item.rb

@@ -5,6 +5,7 @@ module Sbmt
     class ProcessItem < Sbmt::Outbox::DryInteractor
       param :item_class, reader: :private
       param :item_id, reader: :private
+      option :worker_version, reader: :private, optional: true, default: -> { 1 }
 
       METRICS_COUNTERS = %i[error_counter retry_counter sent_counter fetch_error_counter discarded_counter].freeze
 
@@ -254,7 +255,7 @@ module Sbmt
       end
 
       def labels_for(item)
-        {type: box_type, name: box_name, owner: owner, partition: item&.partition}
+        {worker_version: worker_version, type: box_type, name: box_name, owner: owner, partition: item&.partition}
       end
 
       def counters

data/app/interactors/sbmt/outbox/retry_strategies/base.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module RetryStrategies
+      class Base < Outbox::DryInteractor
+        param :item
+
+        def call
+          raise NotImplementedError, "Implement #call for Sbmt::Outbox::RetryStrategies::Base"
+        end
+      end
+    end
+  end
+end

data/app/interactors/sbmt/outbox/retry_strategies/compacted_log.rb

@@ -3,38 +3,8 @@
 module Sbmt
   module Outbox
     module RetryStrategies
-      class CompactedLog < Outbox::DryInteractor
-        param :outbox_item
-
-        def call
-          unless outbox_item.has_attribute?(:event_key)
-            return Failure(:missing_event_key)
-          end
-
-          if outbox_item.event_key.nil?
-            return Failure(:empty_event_key)
-          end
-
-          if delivered_later?
-            Failure(:discard_item)
-          else
-            Success()
-          end
-        end
-
-        private
-
-        def delivered_later?
-          scope = outbox_item.class
-            .where("id > ?", outbox_item)
-            .where(event_key: outbox_item.event_key, status: Sbmt::Outbox::BaseItem.statuses[:delivered])
-
-          if outbox_item.has_attribute?(:event_name) && outbox_item.event_name.present?
-            scope = scope.where(event_name: outbox_item.event_name)
-          end
-
-          scope.exists?
-        end
+      class CompactedLog < LatestAvailable
+        # exists only as alias for backward compatibility
       end
     end
   end

data/app/interactors/sbmt/outbox/retry_strategies/exponential_backoff.rb

@@ -3,13 +3,11 @@
 module Sbmt
   module Outbox
     module RetryStrategies
-      class ExponentialBackoff < Outbox::DryInteractor
-        param :outbox_item
-
+      class ExponentialBackoff < Base
         def call
-          delay = backoff(outbox_item.config).interval_at(outbox_item.errors_count - 1)
+          delay = backoff(item.config).interval_at(item.errors_count - 1)
 
-          still_early = outbox_item.processed_at + delay.seconds > Time.current
+          still_early = item.processed_at + delay.seconds > Time.current
 
           if still_early
             Failure(:skip_processing)

data/app/interactors/sbmt/outbox/retry_strategies/latest_available.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module RetryStrategies
+      class LatestAvailable < Base
+        def call
+          unless item.has_attribute?(:event_key)
+            return Failure(:missing_event_key)
+          end
+
+          if item.event_key.nil?
+            return Failure(:empty_event_key)
+          end
+
+          if delivered_later?
+            Failure(:discard_item)
+          else
+            Success()
+          end
+        end
+
+        private
+
+        def delivered_later?
+          scope = item.class
+            .where("id > ?", item)
+            .where(event_key: item.event_key)
+
+          if item.has_attribute?(:event_name) && item.event_name.present?
+            scope = scope.where(event_name: item.event_name)
+          end
+
+          scope.exists?
+        end
+      end
+    end
+  end
+end

data/app/interactors/sbmt/outbox/retry_strategies/no_delay.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module RetryStrategies
+      class NoDelay < Base
+        def call
+          Success()
+        end
+      end
+    end
+  end
+end

data/app/models/sbmt/outbox/base_item.rb

@@ -20,15 +20,16 @@ module Sbmt
           @config ||= lookup_config.new(box_name)
         end
 
+        def calc_bucket_partitions(count)
+          (0...count).to_a
+            .each_with_object({}) do |x, m|
+            m[x] = (0...config.bucket_size).to_a
+              .select { |p| p % count == x }
+          end
+        end
+
         def partition_buckets
-          @partition_buckets ||=
-            (0...config.partition_size)
-              .to_a
-              .each_with_object({}) do |x, m|
-                m[x] = (0...config.bucket_size)
-                  .to_a
-                  .select { |p| p % config.partition_size == x }
-              end
+          @partition_buckets ||= calc_bucket_partitions(config.partition_size)
         end
 
         def bucket_partitions

data/app/models/sbmt/outbox/base_item_config.rb

@@ -23,7 +23,11 @@ module Sbmt
       end
 
       def partition_size
-        @partition_size ||= (options[:partition_size] || 1).to_i
+        @partition_size ||= (partition_size_raw || 1).to_i
+      end
+
+      def partition_size_raw
+        @partition_size_raw ||= options[:partition_size]
       end
 
       def retention
@@ -47,7 +51,12 @@ module Sbmt
       end
 
       def retry_strategies
-        @retry_strategies ||= Array.wrap(options[:retry_strategies]).map do |str_name|
+        return @retry_strategies if defined?(@retry_strategies)
+
+        configured_strategies = options[:retry_strategies]
+        strategies = configured_strategies.presence || %w[exponential_backoff latest_available]
+
+        @retry_strategies ||= Array.wrap(strategies).map do |str_name|
           "Sbmt::Outbox::RetryStrategies::#{str_name.camelize}".constantize
         end
       end
@@ -76,12 +85,22 @@ module Sbmt
           memo[event_name] ||= []
           namespace = params.delete(:class)&.camelize
           raise ArgumentError, "Transport name cannot be blank" if namespace.blank?
+          disposable = params.key?(:disposable) ? params.delete(:disposable) : Outbox.config.disposable_transports
 
           factory = "#{namespace}::OutboxTransportFactory".safe_constantize
           memo[event_name] << if factory
-            factory.build(**params.symbolize_keys)
+            if disposable
+              ->(*args) { factory.build(**params).call(*args) }
+            else
+              factory.build(**params)
+            end
           else
-            namespace.constantize.new(**params.symbolize_keys)
+            klass = namespace.constantize
+            if disposable
+              ->(*args) { klass.new(**params).call(*args) }
+            else
+              klass.new(**params)
+            end
           end
         end
       end

data/config/initializers/yabeda.rb

@@ -3,6 +3,8 @@
 Yabeda.configure do
   # error_counter retry_counter sent_counter fetch_error_counter discarded_counter
   group :outbox do
+    default_tag(:worker_version, 1)
+
     counter :created_counter,
       tags: %i[type name partition owner],
       comment: "The total number of created messages"
@@ -44,28 +46,53 @@ Yabeda.configure do
   end
 
   group :box_worker do
+    default_tag(:worker_version, 1)
+    default_tag(:worker_name, "worker")
+
     counter :job_counter,
-      tags: %i[type name partition worker_number state],
+      tags: %i[type name partition state],
       comment: "The total number of processed jobs"
 
     counter :job_timeout_counter,
-      tags: %i[type name partition_key worker_number],
+      tags: %i[type name partition_key],
       comment: "Requeue of a job that occurred while processing the batch"
 
     counter :job_items_counter,
-      tags: %i[type name partition worker_number],
+      tags: %i[type name partition],
       comment: "The total number of processed items in jobs"
 
     histogram :job_execution_runtime,
       comment: "A histogram of the job execution time",
       unit: :seconds,
-      tags: %i[type name partition worker_number],
+      tags: %i[type name partition],
       buckets: [0.5, 1, 2.5, 5, 10, 15, 30, 45, 60, 90, 120, 180, 240, 300, 600]
 
     histogram :item_execution_runtime,
       comment: "A histogram of the item execution time",
       unit: :seconds,
-      tags: %i[type name partition worker_number],
+      tags: %i[type name partition],
+      buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300]
+
+    counter :batches_per_poll_counter,
+      tags: %i[type name partition],
+      comment: "The total number of poll batches per poll"
+
+    gauge :redis_job_queue_size,
+      tags: %i[type name partition],
+      comment: "The total size of redis job queue"
+
+    gauge :redis_job_queue_time_lag,
+      tags: %i[type name partition],
+      comment: "The total time lag of redis job queue"
+
+    counter :poll_throttling_counter,
+      tags: %i[type name partition throttler status],
+      comment: "The total number of poll throttlings"
+
+    histogram :poll_throttling_runtime,
+      comment: "A histogram of the poll throttling time",
+      unit: :seconds,
+      tags: %i[type name partition throttler],
       buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300]
   end
 end

data/lib/generators/helpers/migration.rb

@@ -44,8 +44,8 @@ module Outbox
             end
 
             add_index :#{table_name}, :uuid, unique: true
-            add_index :#{table_name}, [:status, :bucket]
-            add_index :#{table_name}, :event_key
+            add_index :#{table_name}, [:status, :bucket, :errors_count]
+            add_index :#{table_name}, [:event_key, :id]
             add_index :#{table_name}, :created_at
           RUBY
 

data/lib/sbmt/outbox/cli.rb

@@ -20,15 +20,47 @@ module Sbmt
       option :concurrency,
         aliases: "-c",
         type: :numeric,
-        default: 10,
-        desc: "Number of threads"
+        desc: "Number of threads (processor)"
+      option :poll_concurrency,
+        aliases: "-p",
+        type: :numeric,
+        desc: "Number of poller partitions"
+      option :poll_threads,
+        aliases: "-n",
+        type: :numeric,
+        desc: "Number of threads (poller)"
+      option :poll_tactic,
+        aliases: "-t",
+        type: :string,
+        desc: "Poll tactic: [default, low-priority, aggressive]"
+      option :worker_version,
+        aliases: "-w",
+        type: :numeric,
+        desc: "Worker version: [1 | 2]"
       def start
         load_environment
 
-        worker = Sbmt::Outbox::Worker.new(
-          boxes: format_boxes(options[:box]),
-          concurrency: options[:concurrency]
-        )
+        version = options[:worker_version] || Outbox.default_worker_version
+
+        boxes = format_boxes(options[:box])
+        check_deprecations!(boxes, version)
+
+        worker = if version == 1
+          Sbmt::Outbox::V1::Worker.new(
+            boxes: boxes,
+            concurrency: options[:concurrency] || 10
+          )
+        elsif version == 2
+          Sbmt::Outbox::V2::Worker.new(
+            boxes: boxes,
+            poll_tactic: options[:poll_tactic],
+            poller_threads_count: options[:poll_threads],
+            poller_partitions_count: options[:poll_concurrency],
+            processor_concurrency: options[:concurrency] || 4
+          )
+        else
+          raise "Worker version #{version} is invalid, available versions: 1|2"
+        end
 
         Sbmt::Outbox.current_worker = worker
 
@@ -45,11 +77,22 @@ module Sbmt
 
       private
 
+      def check_deprecations!(boxes, version)
+        return unless version == 2
+
+        boxes.each do |item_class|
+          next if item_class.config.partition_size_raw.blank?
+
+          raise "partition_size option is invalid and cannot be used with worker v2, please remove it from config/outbox.yml for #{item_class.name.underscore}"
+        end
+      end
+
       def load_environment
         load(lookup_outboxfile)
 
         require "sbmt/outbox"
-        require "sbmt/outbox/worker"
+        require "sbmt/outbox/v1/worker"
+        require "sbmt/outbox/v2/worker"
       end
 
       def lookup_outboxfile

data/lib/sbmt/outbox/engine.rb

@@ -14,6 +14,7 @@ module Sbmt
         c.outbox_item_classes = []
         c.inbox_item_classes = []
         c.paths = []
+        c.disposable_transports = false
         c.redis = {url: ENV.fetch("REDIS_URL", "redis://127.0.0.1:6379")}
         c.process_items = ActiveSupport::OrderedOptions.new.tap do |c|
           c.general_timeout = 120
@@ -25,6 +26,31 @@ module Sbmt
           c.rate_interval = 60
           c.shuffle_jobs = true
         end
+        c.default_worker_version = 2
+
+        # worker v2
+        c.poller = ActiveSupport::OrderedOptions.new.tap do |pc|
+          pc.concurrency = 6
+          pc.threads_count = 2
+          pc.general_timeout = 60
+          pc.regular_items_batch_size = 200
+          pc.retryable_items_batch_size = 100
+
+          pc.tactic = "default"
+          pc.rate_limit = 60
+          pc.rate_interval = 60
+          pc.min_queue_size = 10
+          pc.max_queue_size = 100
+          pc.min_queue_timelag = 5
+          pc.queue_delay = 0.1
+        end
+        c.processor = ActiveSupport::OrderedOptions.new.tap do |pc|
+          pc.threads_count = 4
+          pc.general_timeout = 120
+          pc.cutoff_timeout = 60
+          pc.brpop_delay = 2
+        end
+
         c.database_switcher = "Sbmt::Outbox::DatabaseSwitcher"
         c.batch_process_middlewares = []
         c.item_process_middlewares = []

data/lib/sbmt/outbox/logger.rb

@@ -5,6 +5,12 @@ module Sbmt
     class Logger
       delegate :logger, to: :Rails
 
+      def log_debug(message, **params)
+        with_tags(**params) do
+          logger.debug(message)
+        end
+      end
+
       def log_info(message, **params)
         with_tags(**params) do
           logger.info(message)