sbmt-outbox 5.0.4 → 6.0.1

data/lib/sbmt/outbox/v2/worker.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "redlock"
+require "sbmt/outbox/v2/poller"
+require "sbmt/outbox/v2/processor"
+
+module Sbmt
+  module Outbox
+    module V2
+      class Worker
+        def initialize(boxes:, poll_tactic: nil, processor_concurrency: nil, poller_partitions_count: nil, poller_threads_count: nil)
+          @poller = Poller.new(boxes, throttler_tactic: poll_tactic, threads_count: poller_threads_count, partitions_count: poller_partitions_count)
+          @processor = Processor.new(boxes, threads_count: processor_concurrency)
+        end
+
+        def start
+          start_async
+
+          loop do
+            sleep 0.1
+            break unless @poller.started && @processor.started
+          end
+        end
+
+        def start_async
+          @poller.start
+          @processor.start
+
+          loop do
+            sleep(0.1)
+            break if ready?
+          end
+        end
+
+        def stop
+          @poller.stop
+          @processor.stop
+        end
+
+        def ready?
+          @poller.ready? && @processor.ready?
+        end
+
+        def alive?
+          return false unless ready?
+
+          @poller.alive?(@poller.lock_timeout) && @processor.alive?(@processor.lock_timeout)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module Outbox
-    VERSION = "5.0.4"
+    VERSION = "6.0.1"
   end
 end

data/lib/sbmt/outbox.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "rails"
+require "active_job"
+require "active_record"
 require "dry-initializer"
 require "dry-monads"
 require "dry/monads/do"
@@ -53,12 +55,24 @@ module Sbmt
       @logger ||= Sbmt::Outbox::Logger.new
     end
 
+    def poller_config
+      @poller_config ||= config.poller
+    end
+
+    def processor_config
+      @processor_config ||= config.processor
+    end
+
+    def default_worker_version
+      @default_worker_version ||= config.default_worker_version&.to_i || 2
+    end
+
     def active_record_base_class
-      @active_record_base_class ||= config.active_record_base_class.safe_constantize || ActiveRecord::Base
+      @active_record_base_class ||= config.active_record_base_class.safe_constantize || ::ActiveRecord::Base
     end
 
     def active_job_base_class
-      @active_job_base_class ||= config.active_job_base_class.safe_constantize || ActiveJob::Base
+      @active_job_base_class ||= config.active_job_base_class.safe_constantize || ::ActiveJob::Base
     end
 
     def error_tracker

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-outbox
 version: !ruby/object:Gem::Version
-  version: 5.0.4
+  version: 6.0.1
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
@@ -188,6 +188,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: ruby-limiter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -506,8 +520,11 @@ files:
 - app/interactors/sbmt/outbox/partition_strategies/hash_partitioning.rb
 - app/interactors/sbmt/outbox/partition_strategies/number_partitioning.rb
 - app/interactors/sbmt/outbox/process_item.rb
+- app/interactors/sbmt/outbox/retry_strategies/base.rb
 - app/interactors/sbmt/outbox/retry_strategies/compacted_log.rb
 - app/interactors/sbmt/outbox/retry_strategies/exponential_backoff.rb
+- app/interactors/sbmt/outbox/retry_strategies/latest_available.rb
+- app/interactors/sbmt/outbox/retry_strategies/no_delay.rb
 - app/jobs/sbmt/outbox/base_delete_stale_items_job.rb
 - app/jobs/sbmt/outbox/delete_stale_inbox_items_job.rb
 - app/jobs/sbmt/outbox/delete_stale_outbox_items_job.rb
@@ -563,10 +580,29 @@ files:
 - lib/sbmt/outbox/redis_client_factory.rb
 - lib/sbmt/outbox/tasks/delete_failed_items.rake
 - lib/sbmt/outbox/tasks/retry_failed_items.rake
-- lib/sbmt/outbox/thread_pool.rb
-- lib/sbmt/outbox/throttler.rb
+- lib/sbmt/outbox/v1/thread_pool.rb
+- lib/sbmt/outbox/v1/throttler.rb
+- lib/sbmt/outbox/v1/worker.rb
+- lib/sbmt/outbox/v2/box_processor.rb
+- lib/sbmt/outbox/v2/poll_throttler.rb
+- lib/sbmt/outbox/v2/poll_throttler/base.rb
+- lib/sbmt/outbox/v2/poll_throttler/composite.rb
+- lib/sbmt/outbox/v2/poll_throttler/fixed_delay.rb
+- lib/sbmt/outbox/v2/poll_throttler/noop.rb
+- lib/sbmt/outbox/v2/poll_throttler/rate_limited.rb
+- lib/sbmt/outbox/v2/poll_throttler/redis_queue_size.rb
+- lib/sbmt/outbox/v2/poll_throttler/redis_queue_time_lag.rb
+- lib/sbmt/outbox/v2/poller.rb
+- lib/sbmt/outbox/v2/processor.rb
+- lib/sbmt/outbox/v2/redis_job.rb
+- lib/sbmt/outbox/v2/tasks/base.rb
+- lib/sbmt/outbox/v2/tasks/default.rb
+- lib/sbmt/outbox/v2/tasks/poll.rb
+- lib/sbmt/outbox/v2/tasks/process.rb
+- lib/sbmt/outbox/v2/thread_pool.rb
+- lib/sbmt/outbox/v2/throttler.rb
+- lib/sbmt/outbox/v2/worker.rb
 - lib/sbmt/outbox/version.rb
-- lib/sbmt/outbox/worker.rb
 homepage: https://github.com/SberMarket-Tech/sbmt-outbox
 licenses:
 - MIT

data/lib/sbmt/outbox/thread_pool.rb

@@ -1,108 +0,0 @@
-# frozen_string_literal: true
-
-require "sbmt/outbox/throttler"
-
-module Sbmt
-  module Outbox
-    class ThreadPool
-      BREAK = Object.new.freeze
-      SKIPPED = Object.new.freeze
-      PROCESSED = Object.new.freeze
-
-      def initialize(&block)
-        self.task_source = block
-        self.task_mutex = Mutex.new
-        self.stopped = true
-      end
-
-      def next_task
-        task_mutex.synchronize do
-          return if stopped
-          item = task_source.call
-
-          if item == BREAK
-            self.stopped = true
-            return
-          end
-
-          item
-        end
-      end
-
-      def start(concurrency:)
-        self.stopped = false
-        result = run_threads(count: concurrency) do |item|
-          yield worker_number, item
-        end
-
-        raise result if result.is_a?(Exception)
-        nil
-      ensure
-        self.stopped = true
-      end
-
-      def stop
-        self.stopped = true
-      end
-
-      def worker_number
-        Thread.current["thread_pool_worker_number:#{object_id}"]
-      end
-
-      private
-
-      attr_accessor :task_source, :task_mutex, :stopped
-
-      def run_threads(count:)
-        exception = nil
-
-        in_threads(count: count) do |worker_num|
-          self.worker_number = worker_num
-          # We don't want to start all threads at the same time
-          random_sleep = rand * (worker_num + 1)
-
-          throttler = Throttler.new(
-            limit: Outbox.config.worker.rate_limit,
-            interval: Outbox.config.worker.rate_interval + random_sleep
-          )
-
-          sleep(random_sleep)
-
-          last_result = nil
-          until exception
-            throttler.wait if last_result == PROCESSED
-            item = next_task
-            break unless item
-
-            begin
-              last_result = yield item
-            rescue Exception => e # rubocop:disable Lint/RescueException
-              exception = e
-            end
-          end
-        end
-
-        exception
-      end
-
-      def in_threads(count:)
-        threads = []
-
-        Thread.handle_interrupt(Exception => :never) do
-          Thread.handle_interrupt(Exception => :immediate) do
-            count.times do |i|
-              threads << Thread.new { yield(i) }
-            end
-            threads.map(&:value)
-          end
-        ensure
-          threads.each(&:kill)
-        end
-      end
-
-      def worker_number=(num)
-        Thread.current["thread_pool_worker_number:#{object_id}"] = num
-      end
-    end
-  end
-end

data/lib/sbmt/outbox/throttler.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module Sbmt
-  module Outbox
-    # Based on https://github.com/Shopify/limiter/blob/master/lib/limiter/rate_queue.rb
-    # We cannot use that gem because we have to support Ruby 2.5,
-    # but Shopify's limiter requires minimum Ruby 2.6
-    class Throttler
-      def initialize(limit: nil, interval: nil)
-        @limit = limit
-        @interval = limit
-        @map = (0...@limit).map { |i| base_time + (gap * i) }
-        @index = 0
-        @mutex = Mutex.new
-      end
-
-      def wait
-        time = nil
-
-        @mutex.synchronize do
-          time = @map[@index]
-
-          sleep_until(time + @interval)
-
-          @map[@index] = now
-          @index = (@index + 1) % @limit
-        end
-
-        time
-      end
-
-      private
-
-      def sleep_until(time)
-        period = time - now
-        sleep(period) if period > 0
-      end
-
-      def base_time
-        now - @interval
-      end
-
-      def gap
-        @interval.to_f / @limit.to_f
-      end
-
-      def now
-        Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      end
-    end
-  end
-end

data/lib/sbmt/outbox/worker.rb

@@ -1,233 +0,0 @@
-# frozen_string_literal: true
-
-require "redlock"
-require "sbmt/outbox/thread_pool"
-
-module Sbmt
-  module Outbox
-    class Worker
-      Job = Struct.new(
-        :item_class,
-        :partition,
-        :buckets,
-        :log_tags,
-        :yabeda_labels,
-        :resource_key,
-        :resource_path,
-        keyword_init: true
-      )
-
-      delegate :config,
-        :logger,
-        :batch_process_middlewares,
-        to: "Sbmt::Outbox"
-      delegate :stop, to: :thread_pool
-      delegate :general_timeout, :cutoff_timeout, :batch_size, to: "Sbmt::Outbox.config.process_items"
-      delegate :job_counter,
-        :job_execution_runtime,
-        :item_execution_runtime,
-        :job_items_counter,
-        :job_timeout_counter,
-        to: "Yabeda.box_worker"
-
-      def initialize(boxes:, concurrency: 10)
-        self.queue = Queue.new
-        build_jobs(boxes).each { |job| queue << job }
-        self.thread_pool = ThreadPool.new { queue.pop }
-        self.concurrency = [concurrency, queue.size].min
-        self.thread_workers = {}
-        init_redis
-      end
-
-      def start
-        raise "Outbox is already started" if started
-        self.started = true
-        self.thread_workers = {}
-
-        thread_pool.start(concurrency: concurrency) do |worker_number, job|
-          touch_thread_worker!
-          result = ThreadPool::PROCESSED
-          logger.with_tags(**job.log_tags.merge(worker: worker_number)) do
-            lock_manager.lock("#{job.resource_path}:lock", general_timeout * 1000) do |locked|
-              labels = job.yabeda_labels.merge(worker_number: worker_number)
-
-              if locked
-                job_execution_runtime.measure(labels) do
-                  ::Rails.application.executor.wrap do
-                    safe_process_job(job, worker_number, labels)
-                  end
-                end
-              else
-                result = ThreadPool::SKIPPED
-                logger.log_info("Skip processing already locked #{job.resource_key}")
-              end
-
-              job_counter.increment(labels.merge(state: locked ? "processed" : "skipped"), by: 1)
-            end
-          end
-
-          result
-        ensure
-          queue << job
-        end
-      rescue => e
-        Outbox.error_tracker.error(e)
-        raise
-      ensure
-        self.started = false
-      end
-
-      def ready?
-        started && thread_workers.any?
-      end
-
-      def alive?
-        return false unless started
-
-        deadline = Time.current - general_timeout
-        thread_workers.all? do |_worker_number, time|
-          deadline < time
-        end
-      end
-
-      private
-
-      attr_accessor :queue, :thread_pool, :concurrency, :lock_manager, :redis, :thread_workers, :started
-
-      def init_redis
-        self.redis = ConnectionPool::Wrapper.new(size: concurrency) { RedisClientFactory.build(config.redis) }
-
-        client = if Gem::Version.new(Redlock::VERSION) >= Gem::Version.new("2.0.0")
-          redis
-        else
-          ConnectionPool::Wrapper.new(size: concurrency) { Redis.new(config.redis) }
-        end
-
-        self.lock_manager = Redlock::Client.new([client], retry_count: 0)
-      end
-
-      def build_jobs(boxes)
-        res = boxes.map do |item_class|
-          partitions = (0...item_class.config.partition_size).to_a
-          partitions.map do |partition|
-            buckets = item_class.partition_buckets.fetch(partition)
-            resource_key = "#{item_class.box_name}/#{partition}"
-
-            Job.new(
-              item_class: item_class,
-              partition: partition,
-              buckets: buckets,
-              log_tags: {
-                box_type: item_class.box_type,
-                box_name: item_class.box_name,
-                box_partition: partition,
-                trace_id: nil
-              },
-              yabeda_labels: {
-                type: item_class.box_type,
-                name: item_class.box_name,
-                partition: partition
-              },
-              resource_key: resource_key,
-              resource_path: "sbmt/outbox/worker/#{resource_key}"
-            )
-          end
-        end.flatten
-
-        res.shuffle! if Outbox.config.worker.shuffle_jobs
-        res
-      end
-
-      def touch_thread_worker!
-        thread_workers[thread_pool.worker_number] = Time.current
-      end
-
-      def safe_process_job(job, worker_number, labels)
-        middlewares = Middleware::Builder.new(batch_process_middlewares)
-
-        middlewares.call(job) do
-          start_id ||= redis.call("GETDEL", "#{job.resource_path}:last_id").to_i + 1
-          logger.log_info("Start processing #{job.resource_key} from id #{start_id}")
-          process_job_with_timeouts(job, start_id, labels)
-        end
-      rescue => e
-        log_fatal(e, job, worker_number)
-        track_fatal(e, job, worker_number)
-      end
-
-      def process_job_with_timeouts(job, start_id, labels)
-        count = 0
-        last_id = nil
-        lock_timer = Cutoff.new(general_timeout)
-        requeue_timer = Cutoff.new(cutoff_timeout)
-
-        process_job(job, start_id, labels) do |item|
-          job_items_counter.increment(labels, by: 1)
-          last_id = item.id
-          count += 1
-          lock_timer.checkpoint!
-          requeue_timer.checkpoint!
-        end
-
-        logger.log_info("Finish processing #{job.resource_key} at id #{last_id}")
-      rescue Cutoff::CutoffExceededError
-        job_timeout_counter.increment(labels, by: 1)
-
-        msg = if lock_timer.exceeded?
-          "Lock timeout"
-        elsif requeue_timer.exceeded?
-          redis.call("SET", "#{job.resource_path}:last_id", last_id, "EX", general_timeout) if last_id
-          "Requeue timeout"
-        end
-        raise "Unknown timer has been timed out" unless msg
-
-        logger.log_info("#{msg} while processing #{job.resource_key} at id #{last_id}")
-      end
-
-      def process_job(job, start_id, labels)
-        Outbox.database_switcher.use_slave do
-          item_class = job.item_class
-
-          scope = item_class
-            .for_processing
-            .select(:id)
-
-          if item_class.has_attribute?(:bucket)
-            scope = scope.where(bucket: job.buckets)
-          elsif job.partition > 0
-            raise "Could not filter by partition #{job.resource_key}"
-          end
-
-          scope.find_each(start: start_id, batch_size: batch_size) do |item|
-            touch_thread_worker!
-            item_execution_runtime.measure(labels) do
-              Outbox.database_switcher.use_master do
-                ProcessItem.call(job.item_class, item.id)
-              end
-              yield item
-            end
-          end
-        end
-      end
-
-      def log_fatal(e, job, worker_number)
-        backtrace = e.backtrace.join("\n") if e.respond_to?(:backtrace)
-
-        logger.log_error(
-          "Failed processing #{job.resource_key} with error: #{e.class} #{e.message}",
-          backtrace: backtrace
-        )
-      end
-
-      def track_fatal(e, job, worker_number)
-        job_counter
-          .increment(
-            job.yabeda_labels.merge(worker_number: worker_number, state: "failed"),
-            by: 1
-          )
-
-        Outbox.error_tracker.error(e, **job.log_tags)
-      end
-    end
-  end
-end