sbmt-outbox 5.0.1 → 6.0.0

data/lib/sbmt/outbox/v2/thread_pool.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module V2
+      class ThreadPool
+        delegate :logger, to: "Sbmt::Outbox"
+
+        BREAK = Object.new.freeze
+        SKIPPED = Object.new.freeze
+        PROCESSED = Object.new.freeze
+
+        def initialize(concurrency:, name: "thread_pool", random_startup_delay: true, start_async: true, &block)
+          self.concurrency = concurrency
+          self.name = name
+          self.random_startup_delay = random_startup_delay
+          self.start_async = start_async
+          self.task_source = block
+          self.task_mutex = Mutex.new
+          self.stopped = true
+          self.threads = Concurrent::Array.new
+        end
+
+        def next_task
+          task_mutex.synchronize do
+            return if stopped
+            task = task_source.call
+
+            if task == BREAK
+              self.stopped = true
+              return
+            end
+
+            task
+          end
+        end
+
+        def start
+          self.stopped = false
+
+          mode = start_async ? "async" : "sync"
+          logger.log_info("#{name}: starting #{concurrency} threads in #{mode} mode")
+
+          result = run_threads do |task|
+            logger.with_tags(worker: worker_number) do
+              yield worker_number, task
+            end
+          end
+
+          logger.log_info("#{name}: threads started")
+
+          raise result if result.is_a?(Exception)
+        end
+
+        def stop
+          self.stopped = true
+
+          threads.map(&:join) if start_async
+        ensure
+          stop_threads
+        end
+
+        def running?
+          return false if stopped
+
+          true
+        end
+
+        def alive?(timeout)
+          return false if stopped
+
+          deadline = Time.current - timeout
+          threads.all? do |thread|
+            last_active_at = last_active_at(thread)
+            return false unless last_active_at
+
+            deadline < last_active_at
+          end
+        end
+
+        private
+
+        attr_accessor :concurrency, :name, :random_startup_delay, :task_source, :task_mutex, :stopped, :start_async, :threads
+
+        def touch_worker!
+          self.last_active_at = Time.current
+        end
+
+        def worker_number(thread = Thread.current)
+          thread.thread_variable_get("#{name}_worker_number:#{object_id}")
+        end
+
+        def last_active_at(thread = Thread.current)
+          thread.thread_variable_get("#{name}_last_active_at:#{object_id}")
+        end
+
+        def run_threads
+          exception = nil
+
+          in_threads do |worker_num|
+            self.worker_number = worker_num
+            # We don't want to start all threads at the same time
+            sleep(rand * (worker_num + 1)) if random_startup_delay
+
+            touch_worker!
+
+            until exception
+              task = next_task
+              break unless task
+
+              touch_worker!
+
+              begin
+                yield task
+              rescue Exception => e # rubocop:disable Lint/RescueException
+                exception = e
+              end
+            end
+          end
+
+          exception
+        end
+
+        def in_threads
+          Thread.handle_interrupt(Exception => :never) do
+            Thread.handle_interrupt(Exception => :immediate) do
+              concurrency.times do |i|
+                threads << Thread.new { yield(i) }
+              end
+              threads.map(&:value) unless start_async
+            end
+          ensure
+            stop_threads unless start_async
+          end
+        end
+
+        def stop_threads
+          threads.each(&:kill)
+          threads.clear
+        end
+
+        def worker_number=(num)
+          Thread.current.thread_variable_set("#{name}_worker_number:#{object_id}", num)
+        end
+
+        def last_active_at=(at)
+          Thread.current.thread_variable_set("#{name}_last_active_at:#{object_id}", at)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/throttler.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module V2
+      module Throttler
+        THROTTLE_STATUS = "throttle"
+        SKIP_STATUS = "skip"
+        NOOP_STATUS = "noop"
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/worker.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "redlock"
+require "sbmt/outbox/v2/poller"
+require "sbmt/outbox/v2/processor"
+
+module Sbmt
+  module Outbox
+    module V2
+      class Worker
+        def initialize(boxes:, poll_tactic: nil, processor_concurrency: nil, poller_partitions_count: nil, poller_threads_count: nil)
+          @poller = Poller.new(boxes, throttler_tactic: poll_tactic, threads_count: poller_threads_count, partitions_count: poller_partitions_count)
+          @processor = Processor.new(boxes, threads_count: processor_concurrency)
+        end
+
+        def start
+          start_async
+
+          loop do
+            sleep 0.1
+            break unless @poller.started && @processor.started
+          end
+        end
+
+        def start_async
+          @poller.start
+          @processor.start
+
+          loop do
+            sleep(0.1)
+            break if ready?
+          end
+        end
+
+        def stop
+          @poller.stop
+          @processor.stop
+        end
+
+        def ready?
+          @poller.ready? && @processor.ready?
+        end
+
+        def alive?
+          return false unless ready?
+
+          @poller.alive?(@poller.lock_timeout) && @processor.alive?(@processor.lock_timeout)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module Outbox
-    VERSION = "5.0.1"
+    VERSION = "6.0.0"
   end
 end

data/lib/sbmt/outbox.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "rails"
+require "active_job"
+require "active_record"
 require "dry-initializer"
 require "dry-monads"
 require "dry/monads/do"
@@ -53,12 +55,24 @@ module Sbmt
       @logger ||= Sbmt::Outbox::Logger.new
     end
 
+    def poller_config
+      @poller_config ||= config.poller
+    end
+
+    def processor_config
+      @processor_config ||= config.processor
+    end
+
+    def default_worker_version
+      @default_worker_version ||= config.default_worker_version&.to_i || 2
+    end
+
     def active_record_base_class
-      @active_record_base_class ||= config.active_record_base_class.safe_constantize || ActiveRecord::Base
+      @active_record_base_class ||= config.active_record_base_class.safe_constantize || ::ActiveRecord::Base
     end
 
     def active_job_base_class
-      @active_job_base_class ||= config.active_job_base_class.safe_constantize || ActiveJob::Base
+      @active_job_base_class ||= config.active_job_base_class.safe_constantize || ::ActiveJob::Base
     end
 
     def error_tracker

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-outbox
 version: !ruby/object:Gem::Version
-  version: 5.0.1
+  version: 6.0.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-02-29 00:00:00.000000000 Z
+date: 2024-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -188,6 +188,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: ruby-limiter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -506,8 +520,11 @@ files:
 - app/interactors/sbmt/outbox/partition_strategies/hash_partitioning.rb
 - app/interactors/sbmt/outbox/partition_strategies/number_partitioning.rb
 - app/interactors/sbmt/outbox/process_item.rb
+- app/interactors/sbmt/outbox/retry_strategies/base.rb
 - app/interactors/sbmt/outbox/retry_strategies/compacted_log.rb
 - app/interactors/sbmt/outbox/retry_strategies/exponential_backoff.rb
+- app/interactors/sbmt/outbox/retry_strategies/latest_available.rb
+- app/interactors/sbmt/outbox/retry_strategies/no_delay.rb
 - app/jobs/sbmt/outbox/base_delete_stale_items_job.rb
 - app/jobs/sbmt/outbox/delete_stale_inbox_items_job.rb
 - app/jobs/sbmt/outbox/delete_stale_outbox_items_job.rb
@@ -563,10 +580,29 @@ files:
 - lib/sbmt/outbox/redis_client_factory.rb
 - lib/sbmt/outbox/tasks/delete_failed_items.rake
 - lib/sbmt/outbox/tasks/retry_failed_items.rake
-- lib/sbmt/outbox/thread_pool.rb
-- lib/sbmt/outbox/throttler.rb
+- lib/sbmt/outbox/v1/thread_pool.rb
+- lib/sbmt/outbox/v1/throttler.rb
+- lib/sbmt/outbox/v1/worker.rb
+- lib/sbmt/outbox/v2/box_processor.rb
+- lib/sbmt/outbox/v2/poll_throttler.rb
+- lib/sbmt/outbox/v2/poll_throttler/base.rb
+- lib/sbmt/outbox/v2/poll_throttler/composite.rb
+- lib/sbmt/outbox/v2/poll_throttler/fixed_delay.rb
+- lib/sbmt/outbox/v2/poll_throttler/noop.rb
+- lib/sbmt/outbox/v2/poll_throttler/rate_limited.rb
+- lib/sbmt/outbox/v2/poll_throttler/redis_queue_size.rb
+- lib/sbmt/outbox/v2/poll_throttler/redis_queue_time_lag.rb
+- lib/sbmt/outbox/v2/poller.rb
+- lib/sbmt/outbox/v2/processor.rb
+- lib/sbmt/outbox/v2/redis_job.rb
+- lib/sbmt/outbox/v2/tasks/base.rb
+- lib/sbmt/outbox/v2/tasks/default.rb
+- lib/sbmt/outbox/v2/tasks/poll.rb
+- lib/sbmt/outbox/v2/tasks/process.rb
+- lib/sbmt/outbox/v2/thread_pool.rb
+- lib/sbmt/outbox/v2/throttler.rb
+- lib/sbmt/outbox/v2/worker.rb
 - lib/sbmt/outbox/version.rb
-- lib/sbmt/outbox/worker.rb
 homepage: https://github.com/SberMarket-Tech/sbmt-outbox
 licenses:
 - MIT

data/lib/sbmt/outbox/thread_pool.rb

@@ -1,108 +0,0 @@
-# frozen_string_literal: true
-
-require "sbmt/outbox/throttler"
-
-module Sbmt
-  module Outbox
-    class ThreadPool
-      BREAK = Object.new.freeze
-      SKIPPED = Object.new.freeze
-      PROCESSED = Object.new.freeze
-
-      def initialize(&block)
-        self.task_source = block
-        self.task_mutex = Mutex.new
-        self.stopped = true
-      end
-
-      def next_task
-        task_mutex.synchronize do
-          return if stopped
-          item = task_source.call
-
-          if item == BREAK
-            self.stopped = true
-            return
-          end
-
-          item
-        end
-      end
-
-      def start(concurrency:)
-        self.stopped = false
-        result = run_threads(count: concurrency) do |item|
-          yield worker_number, item
-        end
-
-        raise result if result.is_a?(Exception)
-        nil
-      ensure
-        self.stopped = true
-      end
-
-      def stop
-        self.stopped = true
-      end
-
-      def worker_number
-        Thread.current["thread_pool_worker_number:#{object_id}"]
-      end
-
-      private
-
-      attr_accessor :task_source, :task_mutex, :stopped
-
-      def run_threads(count:)
-        exception = nil
-
-        in_threads(count: count) do |worker_num|
-          self.worker_number = worker_num
-          # We don't want to start all threads at the same time
-          random_sleep = rand * (worker_num + 1)
-
-          throttler = Throttler.new(
-            limit: Outbox.config.worker.rate_limit,
-            interval: Outbox.config.worker.rate_interval + random_sleep
-          )
-
-          sleep(random_sleep)
-
-          last_result = nil
-          until exception
-            throttler.wait if last_result == PROCESSED
-            item = next_task
-            break unless item
-
-            begin
-              last_result = yield item
-            rescue Exception => e # rubocop:disable Lint/RescueException
-              exception = e
-            end
-          end
-        end
-
-        exception
-      end
-
-      def in_threads(count:)
-        threads = []
-
-        Thread.handle_interrupt(Exception => :never) do
-          Thread.handle_interrupt(Exception => :immediate) do
-            count.times do |i|
-              threads << Thread.new { yield(i) }
-            end
-            threads.map(&:value)
-          end
-        ensure
-          threads.each(&:kill)
-        end
-      end
-
-      def worker_number=(num)
-        Thread.current["thread_pool_worker_number:#{object_id}"] = num
-      end
-    end
-  end
-end

data/lib/sbmt/outbox/throttler.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module Sbmt
-  module Outbox
-    # Based on https://github.com/Shopify/limiter/blob/master/lib/limiter/rate_queue.rb
-    # We cannot use that gem because we have to support Ruby 2.5,
-    # but Shopify's limiter requires minimum Ruby 2.6
-    class Throttler
-      def initialize(limit: nil, interval: nil)
-        @limit = limit
-        @interval = limit
-        @map = (0...@limit).map { |i| base_time + (gap * i) }
-        @index = 0
-        @mutex = Mutex.new
-      end
-
-      def wait
-        time = nil
-
-        @mutex.synchronize do
-          time = @map[@index]
-
-          sleep_until(time + @interval)
-
-          @map[@index] = now
-          @index = (@index + 1) % @limit
-        end
-
-        time
-      end
-
-      private
-
-      def sleep_until(time)
-        period = time - now
-        sleep(period) if period > 0
-      end
-
-      def base_time
-        now - @interval
-      end
-
-      def gap
-        @interval.to_f / @limit.to_f
-      end
-
-      def now
-        Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      end
-    end
-  end
-end