sbmt-outbox 6.20.0 → 7.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b5eb57696c5cf8629d009f7da69a2f74722bc1425dc79853b49416d815204e87
-  data.tar.gz: 33bc6c09ff276884a35e351bbce3f732fad2eb22c7e5e8a3d384ee4b765e551e
+  metadata.gz: 23edf4cf26107ce12a000cc13fd63f2488e82c2b2f633f41ac8a8a81ab3f1dfb
+  data.tar.gz: c6f2e7936814535ff29759b3c72c2396128cae67310feca766060ac23d3d413c
 SHA512:
-  metadata.gz: 8316b5867875b12c96d47a399eefde7962e5d16876c347c9cb7ed376054e79fb8d1ba14ae27dec9188311afbeac87a04dc32729a6e9c99c5274b49dc074d30d1
-  data.tar.gz: 4df45b21bb48f1dc053b6c6625c518a4daad8e6dcef609ad433b9802c91730bf9e090adc16b35397e97d316d0782f66e0a686a3832045f969bb125b2c51755bf
+  metadata.gz: cb10853b53332370d2ef41d11b7ac4bb1415f0461e79588ce26f2afedcd6c7d2d88e02c11a3bc4f488489af0f46b1537b3ce20b35f991549fad80b18135a41cc
+  data.tar.gz: 283e97a0d0129a7ea9813bcd8f911487ed55436c9c9d6588071ae261a14d56d82aef172b192823a3036b42da8b414503ed5e81ce4f9548397cf1b8d4d792b61d

data/README.md

@@ -154,7 +154,7 @@ Rails.application.config.outbox.tap do |config|
   config.redis = {url: ENV.fetch("REDIS_URL")} # Redis is used as a coordinator service
   config.paths << Rails.root.join("config/outbox.yml").to_s # optional; configuration file paths, deep merged at the application start, useful with Rails engines
 
-  # optional (worker v2: default)
+  # optional
   config.poller = ActiveSupport::OrderedOptions.new.tap do |pc|
     # max parallel threads (per box-item, globally)
     pc.concurrency = 6
@@ -184,7 +184,7 @@ Rails.application.config.outbox.tap do |config|
     pc.queue_delay = 0.1
   end
 
-  # optional (worker v2: default)
+  # optional
   config.processor = ActiveSupport::OrderedOptions.new.tap do |pc|
     # max threads count (per worker process)
     pc.threads_count = 4
@@ -193,26 +193,6 @@ Rails.application.config.outbox.tap do |config|
     # BRPOP delay (in seconds) for polling redis job queue per box-item
     pc.brpop_delay = 2
   end
-
-  # optional (worker v1: DEPRECATED)
-  config.process_items.tap do |x|
-    # maximum processing time of the batch, after which the batch will be considered hung and processing will be aborted
-    x.general_timeout = 180
-    # maximum batch processing time, after which the processing of the batch will be aborted in the current thread,
-    # and the next thread that picks up the batch will start processing from the same place
-    x.cutoff_timeout = 60
-    # batch size
-    x.batch_size = 200
-  end
-
-  # optional (worker v1: DEPRECATED)
-  config.worker.tap do |worker|
-    # number of batches that one thread will process per rate interval
-    worker.rate_limit = 10
-    # rate interval in seconds
-    worker.rate_interval = 60
-  end
-end
 ```
 
 ### Outbox pattern
@@ -642,7 +622,7 @@ end
 
 We would like to see more features added to the web UI. If you have any suggestions, please feel free to submit a pull request 🤗.
 
-## CLI Arguments (v2: default)
+## CLI Arguments
 
 | Key                        | Description                                                          |
 |----------------------------|----------------------------------------------------------------------|
@@ -651,14 +631,6 @@ We would like to see more features added to the web UI. If you have any suggesti
 | `--poll-concurrency or -p` | Number of poller partitions. Default 6.                              |
 | `--poll-threads or -n`     | Number of poll threads. Default 1.                                   |
 | `--poll-tactic or -t`      | Poll tactic. Default "default".                                      |
-| `--worker-version or -w`   | Worker version. Default 2.                                           |
-
-## CLI Arguments (v1: DEPRECATED)
-
-| Key                   | Description                                                               |
-|-----------------------|---------------------------------------------------------------------------|
-| `--boxes or -b`       | Outbox/Inbox processors to start`                                         |
-| `--concurrency or -c` | Number of threads. Default 10.                                            |
 
 ## Development & Test
 

data/lib/sbmt/outbox/cli.rb

@@ -33,34 +33,19 @@ module Sbmt
         aliases: "-t",
         type: :string,
         desc: "Poll tactic: [default, low-priority, aggressive]"
-      option :worker_version,
-        aliases: "-w",
-        type: :numeric,
-        desc: "Worker version: [1 | 2]"
       def start
         load_environment
 
-        version = options[:worker_version] || Outbox.default_worker_version
-
         boxes = format_boxes(options[:box])
-        check_deprecations!(boxes, version)
-
-        worker = if version == 1
-          Sbmt::Outbox::V1::Worker.new(
-            boxes: boxes,
-            concurrency: options[:concurrency] || 10
-          )
-        elsif version == 2
-          Sbmt::Outbox::V2::Worker.new(
-            boxes: boxes,
-            poll_tactic: options[:poll_tactic],
-            poller_threads_count: options[:poll_threads],
-            poller_partitions_count: options[:poll_concurrency],
-            processor_concurrency: options[:concurrency] || 4
-          )
-        else
-          raise "Worker version #{version} is invalid, available versions: 1|2"
-        end
+        check_deprecations!(boxes)
+
+        worker = Sbmt::Outbox::V2::Worker.new(
+          boxes: boxes,
+          poll_tactic: options[:poll_tactic],
+          poller_threads_count: options[:poll_threads],
+          poller_partitions_count: options[:poll_concurrency],
+          processor_concurrency: options[:concurrency] || 4
+        )
 
         Sbmt::Outbox.current_worker = worker
 
@@ -77,9 +62,7 @@ module Sbmt
 
       private
 
-      def check_deprecations!(boxes, version)
-        return unless version == 2
-
+      def check_deprecations!(boxes)
         boxes.each do |item_class|
           next if item_class.config.partition_size_raw.blank?
 
@@ -91,7 +74,6 @@ module Sbmt
         load(lookup_outboxfile)
 
         require "sbmt/outbox"
-        require "sbmt/outbox/v1/worker"
         require "sbmt/outbox/v2/worker"
       end
 

data/lib/sbmt/outbox/engine.rb

@@ -29,12 +29,6 @@ module Sbmt
           c.cutoff_timeout = 90
           c.batch_size = 200
         end
-        c.worker = ActiveSupport::OrderedOptions.new.tap do |c|
-          c.rate_limit = 20
-          c.rate_interval = 60
-          c.shuffle_jobs = true
-        end
-        c.default_worker_version = 2
 
         # worker v2
         c.poller = ActiveSupport::OrderedOptions.new.tap do |pc|

data/lib/sbmt/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module Outbox
-    VERSION = "6.20.0"
+    VERSION = "7.0.0"
   end
 end

data/lib/sbmt/outbox.rb

@@ -65,10 +65,6 @@ module Sbmt
       @processor_config ||= config.processor
     end
 
-    def default_worker_version
-      @default_worker_version ||= config.default_worker_version&.to_i || 2
-    end
-
     def active_record_base_class
       @active_record_base_class ||= config.active_record_base_class.safe_constantize || ::ActiveRecord::Base
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-outbox
 version: !ruby/object:Gem::Version
-  version: 6.20.0
+  version: 7.0.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-04-02 00:00:00.000000000 Z
+date: 2025-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -585,9 +585,6 @@ files:
 - lib/sbmt/outbox/tasks/delete_items.rake
 - lib/sbmt/outbox/tasks/retry_failed_items.rake
 - lib/sbmt/outbox/tasks/update_status_items.rake
-- lib/sbmt/outbox/v1/thread_pool.rb
-- lib/sbmt/outbox/v1/throttler.rb
-- lib/sbmt/outbox/v1/worker.rb
 - lib/sbmt/outbox/v2/box_processor.rb
 - lib/sbmt/outbox/v2/poll_throttler.rb
 - lib/sbmt/outbox/v2/poll_throttler/base.rb

data/lib/sbmt/outbox/v1/thread_pool.rb

@@ -1,110 +0,0 @@
-# frozen_string_literal: true
-
-require "sbmt/outbox/v1/throttler"
-
-module Sbmt
-  module Outbox
-    module V1
-      class ThreadPool
-        BREAK = Object.new.freeze
-        SKIPPED = Object.new.freeze
-        PROCESSED = Object.new.freeze
-
-        def initialize(&block)
-          self.task_source = block
-          self.task_mutex = Mutex.new
-          self.stopped = true
-        end
-
-        def next_task
-          task_mutex.synchronize do
-            return if stopped
-            item = task_source.call
-
-            if item == BREAK
-              self.stopped = true
-              return
-            end
-
-            item
-          end
-        end
-
-        def start(concurrency:)
-          self.stopped = false
-          result = run_threads(count: concurrency) do |item|
-            yield worker_number, item
-          end
-
-          raise result if result.is_a?(Exception)
-          nil
-        ensure
-          self.stopped = true
-        end
-
-        def stop
-          self.stopped = true
-        end
-
-        def worker_number
-          Thread.current["thread_pool_worker_number:#{object_id}"]
-        end
-
-        private
-
-        attr_accessor :task_source, :task_mutex, :stopped
-
-        def run_threads(count:)
-          exception = nil
-
-          in_threads(count: count) do |worker_num|
-            self.worker_number = worker_num
-            # We don't want to start all threads at the same time
-            random_sleep = rand * (worker_num + 1)
-
-            throttler = Throttler.new(
-              limit: Outbox.config.worker.rate_limit,
-              interval: Outbox.config.worker.rate_interval + random_sleep
-            )
-
-            sleep(random_sleep)
-
-            last_result = nil
-            until exception
-              throttler.wait if last_result == PROCESSED
-              item = next_task
-              break unless item
-
-              begin
-                last_result = yield item
-              rescue Exception => e # rubocop:disable Lint/RescueException
-                exception = e
-              end
-            end
-          end
-
-          exception
-        end
-
-        def in_threads(count:)
-          threads = []
-
-          Thread.handle_interrupt(Exception => :never) do
-            Thread.handle_interrupt(Exception => :immediate) do
-              count.times do |i|
-                threads << Thread.new { yield(i) }
-              end
-              threads.map(&:value)
-            end
-          ensure
-            threads.each(&:kill)
-          end
-        end
-
-        def worker_number=(num)
-          Thread.current["thread_pool_worker_number:#{object_id}"] = num
-        end
-      end
-    end
-  end
-end

data/lib/sbmt/outbox/v1/throttler.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-module Sbmt
-  module Outbox
-    module V1
-      # Based on https://github.com/Shopify/limiter/blob/master/lib/limiter/rate_queue.rb
-      # We cannot use that gem because we have to support Ruby 2.5,
-      # but Shopify's limiter requires minimum Ruby 2.6
-      class Throttler
-        def initialize(limit: nil, interval: nil)
-          @limit = limit
-          @interval = limit
-          @map = (0...@limit).map { |i| base_time + (gap * i) }
-          @index = 0
-          @mutex = Mutex.new
-        end
-
-        def wait
-          time = nil
-
-          @mutex.synchronize do
-            time = @map[@index]
-
-            sleep_until(time + @interval)
-
-            @map[@index] = now
-            @index = (@index + 1) % @limit
-          end
-
-          time
-        end
-
-        private
-
-        def sleep_until(time)
-          period = time - now
-          sleep(period) if period > 0
-        end
-
-        def base_time
-          now - @interval
-        end
-
-        def gap
-          @interval.to_f / @limit.to_f
-        end
-
-        def now
-          Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        end
-      end
-    end
-  end
-end

data/lib/sbmt/outbox/v1/worker.rb

@@ -1,233 +0,0 @@
-# frozen_string_literal: true
-
-require "redlock"
-require "sbmt/outbox/v1/thread_pool"
-require "sbmt/outbox/metrics/utils"
-
-module Sbmt
-  module Outbox
-    module V1
-      class Worker
-        Job = Struct.new(
-          :item_class,
-          :partition,
-          :buckets,
-          :log_tags,
-          :yabeda_labels,
-          :resource_key,
-          :resource_path,
-          keyword_init: true
-        )
-
-        delegate :config,
-          :logger,
-          :batch_process_middlewares,
-          to: "Sbmt::Outbox"
-        delegate :stop, to: :thread_pool
-        delegate :general_timeout, :cutoff_timeout, :batch_size, to: "Sbmt::Outbox.config.process_items"
-        delegate :job_counter,
-          :job_execution_runtime,
-          :item_execution_runtime,
-          :job_items_counter,
-          :job_timeout_counter,
-          to: "Yabeda.box_worker"
-
-        def initialize(boxes:, concurrency: 10)
-          self.queue = Queue.new
-          build_jobs(boxes).each { |job| queue << job }
-          self.thread_pool = ThreadPool.new { queue.pop }
-          self.concurrency = [concurrency, queue.size].min
-          self.thread_workers = {}
-          init_redis
-        end
-
-        def start
-          raise "Outbox is already started" if started
-          self.started = true
-          self.thread_workers = {}
-
-          thread_pool.start(concurrency: concurrency) do |worker_number, job|
-            touch_thread_worker!
-            result = ThreadPool::PROCESSED
-            logger.with_tags(**job.log_tags.merge(worker: worker_number)) do
-              lock_manager.lock("#{job.resource_path}:lock", general_timeout * 1000) do |locked|
-                labels = job.yabeda_labels
-
-                if locked
-                  job_execution_runtime.measure(labels) do
-                    ::Rails.application.executor.wrap do
-                      safe_process_job(job, worker_number, labels)
-                    end
-                  end
-                else
-                  result = ThreadPool::SKIPPED
-                  logger.log_info("Skip processing already locked #{job.resource_key}")
-                end
-
-                job_counter.increment(labels.merge(state: locked ? "processed" : "skipped"), by: 1)
-              end
-            end
-
-            result
-          ensure
-            queue << job
-          end
-        rescue => e
-          Outbox.error_tracker.error(e)
-          raise
-        ensure
-          self.started = false
-        end
-
-        def ready?
-          started && thread_workers.any?
-        end
-
-        def alive?
-          return false unless started
-
-          deadline = Time.current - general_timeout
-          thread_workers.all? do |_worker_number, time|
-            deadline < time
-          end
-        end
-
-        private
-
-        attr_accessor :queue, :thread_pool, :concurrency, :lock_manager, :redis, :thread_workers, :started
-
-        def init_redis
-          self.redis = ConnectionPool::Wrapper.new(size: concurrency) { RedisClientFactory.build(config.redis) }
-
-          client = if Gem::Version.new(Redlock::VERSION) >= Gem::Version.new("2.0.0")
-            redis
-          else
-            ConnectionPool::Wrapper.new(size: concurrency) { Redis.new(config.redis) }
-          end
-
-          self.lock_manager = Redlock::Client.new([client], retry_count: 0)
-        end
-
-        def build_jobs(boxes)
-          res = boxes.map do |item_class|
-            partitions = (0...item_class.config.partition_size).to_a
-            partitions.map do |partition|
-              buckets = item_class.partition_buckets.fetch(partition)
-              resource_key = "#{item_class.box_name}/#{partition}"
-
-              Job.new(
-                item_class: item_class,
-                partition: partition,
-                buckets: buckets,
-                log_tags: {
-                  box_type: item_class.box_type,
-                  box_name: item_class.box_name,
-                  box_partition: partition,
-                  trace_id: nil
-                },
-                yabeda_labels: {
-                  type: item_class.box_type,
-                  name: Sbmt::Outbox::Metrics::Utils.metric_safe(item_class.box_name),
-                  partition: partition,
-                  owner: item_class.owner
-                },
-                resource_key: resource_key,
-                resource_path: "sbmt/outbox/worker/#{resource_key}"
-              )
-            end
-          end.flatten
-
-          res.shuffle! if Outbox.config.worker.shuffle_jobs
-          res
-        end
-
-        def touch_thread_worker!
-          thread_workers[thread_pool.worker_number] = Time.current
-        end
-
-        def safe_process_job(job, worker_number, labels)
-          middlewares = Middleware::Builder.new(batch_process_middlewares)
-
-          middlewares.call(job) do
-            start_id ||= redis.call("GETDEL", "#{job.resource_path}:last_id").to_i + 1
-            logger.log_info("Start processing #{job.resource_key} from id #{start_id}")
-            process_job_with_timeouts(job, start_id, labels)
-          end
-        rescue => e
-          log_fatal(e, job, worker_number)
-          track_fatal(e, job)
-        end
-
-        def process_job_with_timeouts(job, start_id, labels)
-          count = 0
-          last_id = nil
-          lock_timer = Cutoff.new(general_timeout)
-          requeue_timer = Cutoff.new(cutoff_timeout)
-
-          process_job(job, start_id, labels) do |item|
-            job_items_counter.increment(labels, by: 1)
-            last_id = item.id
-            count += 1
-            lock_timer.checkpoint!
-            requeue_timer.checkpoint!
-          end
-
-          logger.log_info("Finish processing #{job.resource_key} at id #{last_id}")
-        rescue Cutoff::CutoffExceededError
-          job_timeout_counter.increment(labels, by: 1)
-
-          msg = if lock_timer.exceeded?
-            "Lock timeout"
-          elsif requeue_timer.exceeded?
-            redis.call("SET", "#{job.resource_path}:last_id", last_id, "EX", general_timeout) if last_id
-            "Requeue timeout"
-          end
-          raise "Unknown timer has been timed out" unless msg
-
-          logger.log_info("#{msg} while processing #{job.resource_key} at id #{last_id}")
-        end
-
-        def process_job(job, start_id, labels)
-          Outbox.database_switcher.use_slave do
-            item_class = job.item_class
-
-            scope = item_class
-              .for_processing
-              .select(:id)
-
-            if item_class.has_attribute?(:bucket)
-              scope = scope.where(bucket: job.buckets)
-            elsif job.partition > 0
-              raise "Could not filter by partition #{job.resource_key}"
-            end
-
-            scope.find_each(start: start_id, batch_size: batch_size) do |item|
-              touch_thread_worker!
-              item_execution_runtime.measure(labels) do
-                Outbox.database_switcher.use_master do
-                  ProcessItem.call(job.item_class, item.id)
-                end
-                yield item
-              end
-            end
-          end
-        end
-
-        def log_fatal(e, job, worker_number)
-          backtrace = e.backtrace.join("\n") if e.respond_to?(:backtrace)
-
-          logger.log_error(
-            "Failed processing #{job.resource_key} with error: #{e.class} #{e.message}",
-            stacktrace: backtrace
-          )
-        end
-
-        def track_fatal(e, job)
-          job_counter.increment(job.yabeda_labels.merge(state: "failed"))
-
-          Outbox.error_tracker.error(e, **job.log_tags)
-        end
-      end
-    end
-  end
-end