sbmt-outbox 5.0.4 → 6.0.0

data/lib/sbmt/outbox/v1/throttler.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module V1
+      # Based on https://github.com/Shopify/limiter/blob/master/lib/limiter/rate_queue.rb
+      # We cannot use that gem because we have to support Ruby 2.5,
+      # but Shopify's limiter requires minimum Ruby 2.6
+      class Throttler
+        def initialize(limit: nil, interval: nil)
+          @limit = limit
+          @interval = limit
+          @map = (0...@limit).map { |i| base_time + (gap * i) }
+          @index = 0
+          @mutex = Mutex.new
+        end
+
+        def wait
+          time = nil
+
+          @mutex.synchronize do
+            time = @map[@index]
+
+            sleep_until(time + @interval)
+
+            @map[@index] = now
+            @index = (@index + 1) % @limit
+          end
+
+          time
+        end
+
+        private
+
+        def sleep_until(time)
+          period = time - now
+          sleep(period) if period > 0
+        end
+
+        def base_time
+          now - @interval
+        end
+
+        def gap
+          @interval.to_f / @limit.to_f
+        end
+
+        def now
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v1/worker.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require "redlock"
+require "sbmt/outbox/v1/thread_pool"
+
+module Sbmt
+  module Outbox
+    module V1
+      class Worker
+        Job = Struct.new(
+          :item_class,
+          :partition,
+          :buckets,
+          :log_tags,
+          :yabeda_labels,
+          :resource_key,
+          :resource_path,
+          keyword_init: true
+        )
+
+        delegate :config,
+          :logger,
+          :batch_process_middlewares,
+          to: "Sbmt::Outbox"
+        delegate :stop, to: :thread_pool
+        delegate :general_timeout, :cutoff_timeout, :batch_size, to: "Sbmt::Outbox.config.process_items"
+        delegate :job_counter,
+          :job_execution_runtime,
+          :item_execution_runtime,
+          :job_items_counter,
+          :job_timeout_counter,
+          to: "Yabeda.box_worker"
+
+        def initialize(boxes:, concurrency: 10)
+          self.queue = Queue.new
+          build_jobs(boxes).each { |job| queue << job }
+          self.thread_pool = ThreadPool.new { queue.pop }
+          self.concurrency = [concurrency, queue.size].min
+          self.thread_workers = {}
+          init_redis
+        end
+
+        def start
+          raise "Outbox is already started" if started
+          self.started = true
+          self.thread_workers = {}
+
+          thread_pool.start(concurrency: concurrency) do |worker_number, job|
+            touch_thread_worker!
+            result = ThreadPool::PROCESSED
+            logger.with_tags(**job.log_tags.merge(worker: worker_number)) do
+              lock_manager.lock("#{job.resource_path}:lock", general_timeout * 1000) do |locked|
+                labels = job.yabeda_labels
+
+                if locked
+                  job_execution_runtime.measure(labels) do
+                    ::Rails.application.executor.wrap do
+                      safe_process_job(job, worker_number, labels)
+                    end
+                  end
+                else
+                  result = ThreadPool::SKIPPED
+                  logger.log_info("Skip processing already locked #{job.resource_key}")
+                end
+
+                job_counter.increment(labels.merge(state: locked ? "processed" : "skipped"), by: 1)
+              end
+            end
+
+            result
+          ensure
+            queue << job
+          end
+        rescue => e
+          Outbox.error_tracker.error(e)
+          raise
+        ensure
+          self.started = false
+        end
+
+        def ready?
+          started && thread_workers.any?
+        end
+
+        def alive?
+          return false unless started
+
+          deadline = Time.current - general_timeout
+          thread_workers.all? do |_worker_number, time|
+            deadline < time
+          end
+        end
+
+        private
+
+        attr_accessor :queue, :thread_pool, :concurrency, :lock_manager, :redis, :thread_workers, :started
+
+        def init_redis
+          self.redis = ConnectionPool::Wrapper.new(size: concurrency) { RedisClientFactory.build(config.redis) }
+
+          client = if Gem::Version.new(Redlock::VERSION) >= Gem::Version.new("2.0.0")
+            redis
+          else
+            ConnectionPool::Wrapper.new(size: concurrency) { Redis.new(config.redis) }
+          end
+
+          self.lock_manager = Redlock::Client.new([client], retry_count: 0)
+        end
+
+        def build_jobs(boxes)
+          res = boxes.map do |item_class|
+            partitions = (0...item_class.config.partition_size).to_a
+            partitions.map do |partition|
+              buckets = item_class.partition_buckets.fetch(partition)
+              resource_key = "#{item_class.box_name}/#{partition}"
+
+              Job.new(
+                item_class: item_class,
+                partition: partition,
+                buckets: buckets,
+                log_tags: {
+                  box_type: item_class.box_type,
+                  box_name: item_class.box_name,
+                  box_partition: partition,
+                  trace_id: nil
+                },
+                yabeda_labels: {
+                  type: item_class.box_type,
+                  name: item_class.box_name,
+                  partition: partition
+                },
+                resource_key: resource_key,
+                resource_path: "sbmt/outbox/worker/#{resource_key}"
+              )
+            end
+          end.flatten
+
+          res.shuffle! if Outbox.config.worker.shuffle_jobs
+          res
+        end
+
+        def touch_thread_worker!
+          thread_workers[thread_pool.worker_number] = Time.current
+        end
+
+        def safe_process_job(job, worker_number, labels)
+          middlewares = Middleware::Builder.new(batch_process_middlewares)
+
+          middlewares.call(job) do
+            start_id ||= redis.call("GETDEL", "#{job.resource_path}:last_id").to_i + 1
+            logger.log_info("Start processing #{job.resource_key} from id #{start_id}")
+            process_job_with_timeouts(job, start_id, labels)
+          end
+        rescue => e
+          log_fatal(e, job, worker_number)
+          track_fatal(e, job)
+        end
+
+        def process_job_with_timeouts(job, start_id, labels)
+          count = 0
+          last_id = nil
+          lock_timer = Cutoff.new(general_timeout)
+          requeue_timer = Cutoff.new(cutoff_timeout)
+
+          process_job(job, start_id, labels) do |item|
+            job_items_counter.increment(labels, by: 1)
+            last_id = item.id
+            count += 1
+            lock_timer.checkpoint!
+            requeue_timer.checkpoint!
+          end
+
+          logger.log_info("Finish processing #{job.resource_key} at id #{last_id}")
+        rescue Cutoff::CutoffExceededError
+          job_timeout_counter.increment(labels, by: 1)
+
+          msg = if lock_timer.exceeded?
+            "Lock timeout"
+          elsif requeue_timer.exceeded?
+            redis.call("SET", "#{job.resource_path}:last_id", last_id, "EX", general_timeout) if last_id
+            "Requeue timeout"
+          end
+          raise "Unknown timer has been timed out" unless msg
+
+          logger.log_info("#{msg} while processing #{job.resource_key} at id #{last_id}")
+        end
+
+        def process_job(job, start_id, labels)
+          Outbox.database_switcher.use_slave do
+            item_class = job.item_class
+
+            scope = item_class
+              .for_processing
+              .select(:id)
+
+            if item_class.has_attribute?(:bucket)
+              scope = scope.where(bucket: job.buckets)
+            elsif job.partition > 0
+              raise "Could not filter by partition #{job.resource_key}"
+            end
+
+            scope.find_each(start: start_id, batch_size: batch_size) do |item|
+              touch_thread_worker!
+              item_execution_runtime.measure(labels) do
+                Outbox.database_switcher.use_master do
+                  ProcessItem.call(job.item_class, item.id)
+                end
+                yield item
+              end
+            end
+          end
+        end
+
+        def log_fatal(e, job, worker_number)
+          backtrace = e.backtrace.join("\n") if e.respond_to?(:backtrace)
+
+          logger.log_error(
+            "Failed processing #{job.resource_key} with error: #{e.class} #{e.message}",
+            backtrace: backtrace
+          )
+        end
+
+        def track_fatal(e, job)
+          job_counter.increment(job.yabeda_labels.merge(state: "failed"))
+
+          Outbox.error_tracker.error(e, **job.log_tags)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/box_processor.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/v2/throttler"
+require "sbmt/outbox/v2/thread_pool"
+require "sbmt/outbox/v2/tasks/default"
+
+module Sbmt
+  module Outbox
+    module V2
+      class BoxProcessor
+        delegate :config, :logger, to: "Sbmt::Outbox"
+        delegate :box_worker, to: "Yabeda"
+        attr_reader :started, :threads_count, :worker_name
+
+        def initialize(boxes:, threads_count:, name: "abstract_worker", redis: nil)
+          @threads_count = threads_count
+          @worker_name = name
+
+          @queue = build_task_queue(boxes)
+          @thread_pool = ThreadPool.new(
+            concurrency: threads_count,
+            name: "#{name}_thread_pool"
+          ) do
+            queue.pop
+          end
+
+          @started = false
+
+          init_redis(redis)
+        end
+
+        def start
+          raise "#{worker_name} is already started" if started
+          @started = true
+
+          thread_pool.start do |worker_number, scheduled_task|
+            result = ThreadPool::PROCESSED
+            last_result = Thread.current[:last_polling_result]
+
+            throttling_res = throttle(worker_number, scheduled_task, last_result)
+            next ThreadPool::SKIPPED if throttling_res&.value_or(nil) == Sbmt::Outbox::V2::Throttler::SKIP_STATUS
+
+            lock_task(scheduled_task) do |locked_task|
+              base_labels = scheduled_task.yabeda_labels.merge(worker_name: worker_name)
+              if locked_task
+                labels = base_labels.merge(locked_task.yabeda_labels)
+                box_worker.job_execution_runtime.measure(labels) do
+                  ::Rails.application.executor.wrap do
+                    logger.with_tags(**locked_task.log_tags) do
+                      result = safe_process_task(worker_number, locked_task)
+                    end
+                  end
+                end
+              else
+                result = ThreadPool::SKIPPED
+              end
+
+              box_worker.job_counter.increment(base_labels.merge(state: locked_task ? "processed" : "skipped"), by: 1)
+            end
+
+            Thread.current[:last_polling_result] = result || ThreadPool::PROCESSED
+          ensure
+            queue << scheduled_task
+          end
+        rescue => e
+          Outbox.error_tracker.error(e)
+          raise
+        end
+
+        def stop
+          @started = false
+          @thread_pool.stop
+        end
+
+        def ready?
+          started && @thread_pool.running?
+        end
+
+        def alive?(timeout)
+          return false unless ready?
+
+          @thread_pool.alive?(timeout)
+        end
+
+        def safe_process_task(worker_number, task)
+          process_task(worker_number, task)
+        rescue => e
+          log_fatal(e, task)
+          track_fatal(e, task)
+        end
+
+        def throttle(_worker_number, _scheduled_task, _result)
+          # noop by default
+          # IMPORTANT: method is called from thread-pool, i.e. code must be thread-safe
+        end
+
+        def process_task(_worker_number, _task)
+          raise NotImplementedError, "Implement #process_task for Sbmt::Outbox::V2::BoxProcessor"
+        end
+
+        private
+
+        attr_accessor :queue, :thread_pool, :redis, :lock_manager
+
+        def init_redis(redis)
+          self.redis = redis || ConnectionPool::Wrapper.new(size: threads_count) { RedisClientFactory.build(config.redis) }
+
+          client = if Gem::Version.new(Redlock::VERSION) >= Gem::Version.new("2.0.0")
+            self.redis
+          else
+            ConnectionPool::Wrapper.new(size: threads_count) { Redis.new(config.redis) }
+          end
+
+          self.lock_manager = Redlock::Client.new([client], retry_count: 0)
+        end
+
+        def lock_task(scheduled_task)
+          # by default there's no locking
+          yield scheduled_task
+        end
+
+        def build_task_queue(boxes)
+          scheduled_tasks = boxes.map do |item_class|
+            Tasks::Default.new(item_class: item_class, worker_name: worker_name)
+          end
+
+          scheduled_tasks.shuffle!
+
+          Queue.new.tap { |queue| scheduled_tasks.each { |task| queue << task } }
+        end
+
+        def log_fatal(e, task)
+          backtrace = e.backtrace.join("\n") if e.respond_to?(:backtrace)
+
+          logger.log_error(
+            "Failed processing #{task} with error: #{e.class} #{e.message}",
+            backtrace: backtrace
+          )
+        end
+
+        def track_fatal(e, task)
+          box_worker.job_counter.increment(task.yabeda_labels.merge(state: "failed"))
+          Outbox.error_tracker.error(e, **task.log_tags)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/poll_throttler/base.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/v2/throttler"
+
+module Sbmt
+  module Outbox
+    module V2
+      module PollThrottler
+        class Base < Outbox::DryInteractor
+          delegate :poll_throttling_counter, :poll_throttling_runtime, to: "Yabeda.box_worker"
+
+          def call(worker_num, poll_task, task_result)
+            with_metrics(poll_task) do
+              wait(worker_num, poll_task, task_result)
+            end
+          end
+
+          def wait(_worker_num, _poll_task, _task_result)
+            raise NotImplementedError, "Implement #wait for Sbmt::Outbox::PollThrottler::Base"
+          end
+
+          private
+
+          def with_metrics(poll_task, &block)
+            tags = metric_tags(poll_task)
+            result = nil
+
+            poll_throttling_runtime.measure(tags) do
+              result = yield
+              poll_throttling_counter.increment(tags.merge(status: result.value_or(result.failure)), by: 1)
+            end
+
+            result
+          end
+
+          def metric_tags(poll_task)
+            poll_task.yabeda_labels.merge(throttler: self.class.name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/poll_throttler/composite.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/v2/poll_throttler/base"
+
+module Sbmt
+  module Outbox
+    module V2
+      module PollThrottler
+        class Composite < Base
+          attr_reader :throttlers
+
+          def initialize(throttlers:)
+            super()
+
+            @throttlers = throttlers
+          end
+
+          def call(worker_num, poll_task, task_result)
+            # each throttler delays polling thread by it's own rules
+            # i.e. resulting delay is a sum of each throttler's ones
+            results = @throttlers.map do |t|
+              res = t.call(worker_num, poll_task, task_result)
+
+              return res if res.success? && res.value! == Sbmt::Outbox::V2::Throttler::SKIP_STATUS
+              return res if res.failure?
+
+              res
+            end
+
+            throttled(results) || Success(Sbmt::Outbox::V2::Throttler::NOOP_STATUS)
+          end
+
+          private
+
+          def throttled(results)
+            results.find { |res| res.success? && res.value! == Sbmt::Outbox::V2::Throttler::THROTTLE_STATUS }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/poll_throttler/fixed_delay.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/v2/poll_throttler/base"
+require "sbmt/outbox/v2/thread_pool"
+
+module Sbmt
+  module Outbox
+    module V2
+      module PollThrottler
+        class FixedDelay < Base
+          def initialize(delay:)
+            super()
+
+            @delay = delay
+          end
+
+          def wait(worker_num, poll_task, task_result)
+            return Success(Sbmt::Outbox::V2::Throttler::NOOP_STATUS) unless task_result == Sbmt::Outbox::V2::ThreadPool::PROCESSED
+
+            sleep(@delay)
+
+            Success(Sbmt::Outbox::V2::Throttler::THROTTLE_STATUS)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/poll_throttler/noop.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/v2/poll_throttler/base"
+
+module Sbmt
+  module Outbox
+    module V2
+      module PollThrottler
+        class Noop < Base
+          def wait(worker_num, poll_task, _task_result)
+            Success(Sbmt::Outbox::V2::Throttler::NOOP_STATUS)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/poll_throttler/rate_limited.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/v2/poll_throttler/base"
+require "ruby-limiter"
+
+module Sbmt
+  module Outbox
+    module V2
+      module PollThrottler
+        class RateLimited < Base
+          def initialize(limit: nil, interval: nil, balanced: true)
+            @queue = Limiter::RateQueue.new(limit, interval: interval, balanced: balanced)
+          end
+
+          def wait(_worker_num, _poll_task, _task_result)
+            @queue.shift
+
+            Success(Sbmt::Outbox::V2::Throttler::THROTTLE_STATUS)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/poll_throttler/redis_queue_size.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/v2/poll_throttler/base"
+
+module Sbmt
+  module Outbox
+    module V2
+      module PollThrottler
+        class RedisQueueSize < Base
+          delegate :redis_job_queue_size, to: "Yabeda.box_worker"
+
+          def initialize(redis:, min_size: -1, max_size: 100, delay: 0)
+            super()
+
+            @redis = redis
+            @min_size = min_size
+            @max_size = max_size
+            @delay = delay
+          end
+
+          def wait(worker_num, poll_task, _task_result)
+            # LLEN is O(1)
+            queue_size = @redis.call("LLEN", poll_task.redis_queue).to_i
+            redis_job_queue_size.set(metric_tags(poll_task), queue_size)
+
+            if queue_size < @min_size
+              # just throttle (not skip) to wait for job queue size becomes acceptable
+              sleep(@delay)
+              return Success(Sbmt::Outbox::V2::Throttler::THROTTLE_STATUS)
+            end
+
+            if queue_size > @max_size
+              # completely skip poll-cycle if job queue is oversized
+              sleep(@delay)
+              return Success(Sbmt::Outbox::V2::Throttler::SKIP_STATUS)
+            end
+
+            Success(Sbmt::Outbox::V2::Throttler::NOOP_STATUS)
+          rescue => ex
+            Failure(ex.message)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/poll_throttler/redis_queue_time_lag.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/v2/poll_throttler/base"
+require "sbmt/outbox/v2/redis_job"
+
+module Sbmt
+  module Outbox
+    module V2
+      module PollThrottler
+        class RedisQueueTimeLag < Base
+          delegate :redis_job_queue_time_lag, to: "Yabeda.box_worker"
+
+          def initialize(redis:, min_lag: 5, delay: 0)
+            super()
+
+            @redis = redis
+            @min_lag = min_lag
+            @delay = delay
+          end
+
+          def wait(worker_num, poll_task, _task_result)
+            # LINDEX is O(1) for first/last element
+            oldest_job = @redis.call("LINDEX", poll_task.redis_queue, -1)
+            return Success(Sbmt::Outbox::V2::Throttler::NOOP_STATUS) if oldest_job.nil?
+
+            job = RedisJob.deserialize!(oldest_job)
+            time_lag = Time.current.to_i - job.timestamp
+
+            redis_job_queue_time_lag.set(metric_tags(poll_task), time_lag)
+
+            if time_lag <= @min_lag
+              sleep(@delay)
+              return Success(Sbmt::Outbox::V2::Throttler::SKIP_STATUS)
+            end
+
+            Success(Sbmt::Outbox::V2::Throttler::NOOP_STATUS)
+          rescue => ex
+            # noop, just skip any redis / serialization errors
+            Failure(ex.message)
+          end
+        end
+      end
+    end
+  end
+end