sbmt-outbox 5.0.4 → 6.0.1

data/lib/sbmt/outbox/v2/poll_throttler.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/v2/poll_throttler/base"
+require "sbmt/outbox/v2/poll_throttler/composite"
+require "sbmt/outbox/v2/poll_throttler/rate_limited"
+require "sbmt/outbox/v2/poll_throttler/fixed_delay"
+require "sbmt/outbox/v2/poll_throttler/noop"
+require "sbmt/outbox/v2/poll_throttler/redis_queue_size"
+require "sbmt/outbox/v2/poll_throttler/redis_queue_time_lag"
+
+module Sbmt
+  module Outbox
+    module V2
+      module PollThrottler
+        POLL_TACTICS = %w[noop default low-priority aggressive]
+
+        def self.build(tactic, redis, poller_config)
+          raise "WARN: invalid poller poll tactic provided: #{tactic}, available options: #{POLL_TACTICS}" unless POLL_TACTICS.include?(tactic)
+
+          if tactic == "default"
+            # composite of RateLimited & RedisQueueSize (upper bound only)
+            # optimal polling performance for most cases
+            Composite.new(throttlers: [
+              RedisQueueSize.new(redis: redis, max_size: poller_config.max_queue_size, delay: poller_config.queue_delay),
+              RateLimited.new(limit: poller_config.rate_limit, interval: poller_config.rate_interval)
+            ])
+          elsif tactic == "low-priority"
+            # composite of RateLimited & RedisQueueSize (with lower & upper bounds) & RedisQueueTimeLag,
+            # delays polling depending on min job queue size threshold
+            # and also by min redis queue oldest item lag
+            # optimal polling performance for low-intensity data flow
+            Composite.new(throttlers: [
+              RedisQueueSize.new(redis: redis, min_size: poller_config.min_queue_size, max_size: poller_config.max_queue_size, delay: poller_config.queue_delay),
+              RedisQueueTimeLag.new(redis: redis, min_lag: poller_config.min_queue_timelag, delay: poller_config.queue_delay),
+              RateLimited.new(limit: poller_config.rate_limit, interval: poller_config.rate_interval)
+            ])
+          elsif tactic == "aggressive"
+            # throttles only by max job queue size, max polling performance
+            # optimal polling performance for high-intensity data flow
+            RedisQueueSize.new(redis: redis, max_size: poller_config.max_queue_size, delay: poller_config.queue_delay)
+          elsif tactic == "noop"
+            # no-op, for testing purposes
+            Noop.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/poller.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+require "redlock"
+require "sbmt/outbox/v2/box_processor"
+require "sbmt/outbox/v2/redis_job"
+require "sbmt/outbox/v2/poll_throttler"
+require "sbmt/outbox/v2/tasks/poll"
+
+module Sbmt
+  module Outbox
+    module V2
+      class Poller < BoxProcessor
+        delegate :poller_config, :logger, to: "Sbmt::Outbox"
+        delegate :box_worker, to: "Yabeda"
+        attr_reader :partitions_count, :lock_timeout, :regular_items_batch_size, :retryable_items_batch_size, :max_buffer_size, :max_batch_size, :throttler
+
+        def initialize(
+          boxes,
+          partitions_count: nil,
+          threads_count: nil,
+          lock_timeout: nil,
+          regular_items_batch_size: nil,
+          retryable_items_batch_size: nil,
+          throttler_tactic: nil,
+          redis: nil
+        )
+          @partitions_count = partitions_count || poller_config.concurrency
+          @lock_timeout = lock_timeout || poller_config.general_timeout
+
+          @regular_items_batch_size = regular_items_batch_size || poller_config.regular_items_batch_size
+          @retryable_items_batch_size = retryable_items_batch_size || poller_config.retryable_items_batch_size
+          @max_buffer_size = @regular_items_batch_size + @retryable_items_batch_size
+          @max_batch_size = @regular_items_batch_size
+
+          super(boxes: boxes, threads_count: threads_count || poller_config.threads_count, name: "poller", redis: redis)
+
+          @throttler = PollThrottler.build(throttler_tactic || poller_config.tactic || "default", self.redis, poller_config)
+        end
+
+        def throttle(worker_number, poll_task, result)
+          throttler.call(worker_number, poll_task, result)
+        end
+
+        def process_task(_worker_number, task)
+          poll(task)
+        end
+
+        private
+
+        def build_task_queue(boxes)
+          scheduled_tasks = boxes.map do |item_class|
+            schedule_concurrency = (0...partitions_count).to_a
+            schedule_concurrency.map do |partition|
+              buckets = item_class.calc_bucket_partitions(partitions_count).fetch(partition)
+
+              Tasks::Poll.new(
+                item_class: item_class,
+                worker_name: worker_name,
+                partition: partition,
+                buckets: buckets
+              )
+            end
+          end.flatten
+
+          scheduled_tasks.shuffle!
+          Queue.new.tap { |queue| scheduled_tasks.each { |task| queue << task } }
+        end
+
+        def lock_task(poll_task)
+          lock_manager.lock("#{poll_task.resource_path}:lock", lock_timeout * 1000) do |locked|
+            lock_status = locked ? "locked" : "skipped"
+            logger.log_debug("poller: lock for #{poll_task}: #{lock_status}")
+
+            yield(locked ? poll_task : nil)
+          end
+        end
+
+        def poll(task)
+          lock_timer = Cutoff.new(lock_timeout)
+          last_id = 0
+
+          box_worker.item_execution_runtime.measure(task.yabeda_labels) do
+            Outbox.database_switcher.use_slave do
+              result = fetch_items(task) do |item|
+                box_worker.job_items_counter.increment(task.yabeda_labels)
+
+                last_id = item.id
+                lock_timer.checkpoint!
+              end
+
+              logger.log_debug("poll task #{task}: fetched buckets:#{result.keys.count}, items:#{result.values.sum(0) { |ids| ids.count }}")
+
+              push_to_redis(task, result) if result.present?
+            end
+          end
+        rescue Cutoff::CutoffExceededError
+          box_worker.job_timeout_counter.increment(labels)
+          logger.log_info("Lock timeout while processing #{task.resource_key} at id #{last_id}")
+        end
+
+        def fetch_items(task)
+          regular_count = 0
+          retryable_count = 0
+
+          # single buffer to preserve item's positions
+          poll_buffer = {}
+
+          fetch_items_with_retries(task, max_batch_size).each do |item|
+            if item.errors_count > 0
+              # skip if retryable buffer capacity limit reached
+              next if retryable_count >= retryable_items_batch_size
+
+              poll_buffer[item.bucket] ||= []
+              poll_buffer[item.bucket] << item.id
+
+              retryable_count += 1
+            else
+              poll_buffer[item.bucket] ||= []
+              poll_buffer[item.bucket] << item.id
+
+              regular_count += 1
+            end
+
+            yield(item)
+          end
+
+          box_worker.batches_per_poll_counter.increment(task.yabeda_labels)
+
+          return {} if poll_buffer.blank?
+
+          # regular items have priority over retryable ones
+          return poll_buffer if regular_count >= regular_items_batch_size
+
+          # additionally poll regular items only when retryable buffer capacity limit reached
+          # and no regular items were found
+          if retryable_count >= retryable_items_batch_size && regular_count == 0
+            fetch_regular_items(task, regular_items_batch_size).each do |item|
+              poll_buffer[item.bucket] ||= []
+              poll_buffer[item.bucket] << item.id
+
+              yield(item)
+            end
+            box_worker.batches_per_poll_counter.increment(task.yabeda_labels)
+          end
+
+          poll_buffer
+        end
+
+        def fetch_items_with_retries(task, limit)
+          task.item_class
+            .for_processing
+            .where(bucket: task.buckets)
+            .order(id: :asc)
+            .limit(limit)
+            .select(:id, :bucket, :errors_count)
+        end
+
+        def fetch_regular_items(task, limit)
+          task.item_class
+            .for_processing
+            .where(bucket: task.buckets, errors_count: 0)
+            .order(id: :asc)
+            .limit(limit)
+            .select(:id, :bucket)
+        end
+
+        def push_to_redis(poll_task, ids_per_bucket)
+          redis.pipelined do |conn|
+            ids_per_bucket.each do |bucket, ids|
+              redis_job = RedisJob.new(bucket, ids)
+
+              logger.log_debug("pushing job to redis, items count: #{ids.count}: #{redis_job}")
+              conn.call("LPUSH", poll_task.redis_queue, redis_job.serialize)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/processor.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "redlock"
+require "sbmt/outbox/v2/box_processor"
+require "sbmt/outbox/v2/redis_job"
+require "sbmt/outbox/v2/tasks/process"
+
+module Sbmt
+  module Outbox
+    module V2
+      class Processor < BoxProcessor
+        delegate :processor_config, :batch_process_middlewares, :logger, to: "Sbmt::Outbox"
+        attr_reader :lock_timeout, :brpop_delay
+
+        REDIS_BRPOP_MIN_DELAY = 0.1
+
+        def initialize(
+          boxes,
+          threads_count: nil,
+          lock_timeout: nil,
+          brpop_delay: nil,
+          redis: nil
+        )
+          @lock_timeout = lock_timeout || processor_config.general_timeout
+          @brpop_delay = brpop_delay || redis_brpop_delay(boxes.count, processor_config.brpop_delay)
+
+          super(boxes: boxes, threads_count: threads_count || processor_config.threads_count, name: "processor", redis: redis)
+        end
+
+        def process_task(_worker_number, task)
+          middlewares = Middleware::Builder.new(batch_process_middlewares)
+          middlewares.call(task) { process(task) }
+        end
+
+        private
+
+        def build_task_queue(boxes)
+          # queue size is: boxes_count * threads_count
+          # to simplify scheduling per box
+          tasks = boxes.map do |item_class|
+            (0...threads_count)
+              .to_a
+              .map { Tasks::Base.new(item_class: item_class, worker_name: worker_name) }
+          end.flatten
+
+          tasks.shuffle!
+          Queue.new.tap { |queue| tasks.each { |task| queue << task } }
+        end
+
+        def lock_task(scheduled_task)
+          redis_job = fetch_redis_job(scheduled_task)
+          return yield(nil) if redis_job.blank?
+
+          processor_task = Tasks::Process.new(
+            item_class: scheduled_task.item_class,
+            worker_name: worker_name,
+            bucket: redis_job.bucket,
+            ids: redis_job.ids
+          )
+          lock_manager.lock("#{processor_task.resource_path}:lock", lock_timeout * 1000) do |locked|
+            lock_status = locked ? "locked" : "skipped"
+            logger.log_debug("processor: lock for #{processor_task}: #{lock_status}")
+
+            yield(locked ? processor_task : nil)
+          end
+        end
+
+        def process(task)
+          lock_timer = Cutoff.new(lock_timeout)
+          last_id = 0
+
+          box_worker.item_execution_runtime.measure(task.yabeda_labels) do
+            Outbox.database_switcher.use_master do
+              task.ids.each do |id|
+                ProcessItem.call(task.item_class, id, worker_version: task.yabeda_labels[:worker_version])
+
+                box_worker.job_items_counter.increment(task.yabeda_labels)
+                last_id = id
+                lock_timer.checkpoint!
+              end
+            end
+          end
+        rescue Cutoff::CutoffExceededError
+          box_worker.job_timeout_counter.increment(task.yabeda_labels)
+          logger.log_info("Lock timeout while processing #{task.resource_key} at id #{last_id}")
+        end
+
+        def fetch_redis_job(scheduled_task)
+          _queue, result = redis.blocking_call(redis_block_timeout, "BRPOP", "#{scheduled_task.item_class.box_name}:job_queue", brpop_delay)
+          return if result.blank?
+
+          RedisJob.deserialize!(result)
+        rescue => ex
+          logger.log_error("error while fetching redis job: #{ex.message}")
+        end
+
+        def redis_block_timeout
+          redis.read_timeout + brpop_delay
+        end
+
+        def redis_brpop_delay(boxes_count, default_delay)
+          return default_delay if boxes_count == 1
+
+          REDIS_BRPOP_MIN_DELAY
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/redis_job.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module V2
+      class RedisJob
+        attr_reader :bucket, :timestamp, :ids
+
+        GENERIC_SEPARATOR = ":"
+        IDS_SEPARATOR = ","
+
+        def initialize(bucket, ids, timestamp = Time.current.to_i)
+          @bucket = bucket
+          @ids = ids
+          @timestamp = timestamp
+        end
+
+        def to_s
+          serialize
+        end
+
+        def serialize
+          [bucket, timestamp, ids.join(IDS_SEPARATOR)].join(GENERIC_SEPARATOR)
+        end
+
+        def self.deserialize!(value)
+          raise "invalid data type: string is required" unless value.is_a?(String)
+
+          bucket, ts_utc, ids_str, _ = value.split(GENERIC_SEPARATOR)
+          raise "invalid data format: bucket or ids are not valid" if bucket.blank? || ts_utc.blank? || ids_str.blank?
+
+          ts = ts_utc.to_i
+
+          ids = ids_str.split(IDS_SEPARATOR).map(&:to_i)
+          raise "invalid data format: IDs are empty" if ids.blank?
+
+          new(bucket, ids, ts)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/tasks/base.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module V2
+      module Tasks
+        class Base
+          attr_reader :item_class, :worker_name, :worker_version, :log_tags, :yabeda_labels
+
+          def initialize(item_class:, worker_name:, worker_version: 2)
+            @item_class = item_class
+            @worker_name = worker_name
+            @worker_version = worker_version
+
+            @log_tags = {
+              box_type: item_class.box_type,
+              box_name: item_class.box_name,
+              worker_name: worker_name,
+              worker_version: worker_version
+            }
+
+            @yabeda_labels = {
+              type: item_class.box_type,
+              name: metric_safe(item_class.box_name),
+              worker_version: 2,
+              worker_name: worker_name
+            }
+          end
+
+          def to_h
+            result = {}
+            instance_variables.each do |iv|
+              iv = iv.to_s[1..]
+              result[iv.to_sym] = instance_variable_get(:"@#{iv}")
+            end
+            result
+          end
+
+          private
+
+          def metric_safe(str)
+            str.tr("/", "-")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/tasks/default.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/v2/tasks/base"
+
+module Sbmt
+  module Outbox
+    module V2
+      module Tasks
+        class Default < Base
+          def to_s
+            "#{item_class.box_type}/#{item_class.box_name}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/tasks/poll.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/v2/tasks/base"
+
+module Sbmt
+  module Outbox
+    module V2
+      module Tasks
+        class Poll < Base
+          attr_reader :partition, :buckets, :resource_key, :resource_path, :redis_queue
+
+          def initialize(item_class:, worker_name:, partition:, buckets:)
+            super(item_class: item_class, worker_name: worker_name)
+
+            @partition = partition
+            @buckets = buckets
+
+            @resource_key = "#{item_class.box_name}:#{partition}"
+            @resource_path = "sbmt:outbox:#{worker_name}:#{resource_key}"
+            @redis_queue = "#{item_class.box_name}:job_queue"
+
+            @log_tags = log_tags.merge(box_partition: partition)
+
+            @yabeda_labels = yabeda_labels.merge(partition: partition)
+          end
+
+          def to_s
+            resource_path
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/tasks/process.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/v2/tasks/base"
+
+module Sbmt
+  module Outbox
+    module V2
+      module Tasks
+        class Process < Base
+          attr_reader :partition, :bucket, :ids, :resource_key, :resource_path
+
+          def initialize(item_class:, worker_name:, bucket:, ids:)
+            super(item_class: item_class, worker_name: worker_name)
+
+            @bucket = bucket
+            @ids = ids
+
+            @resource_key = "#{item_class.box_name}:#{bucket}"
+            @resource_path = "sbmt:outbox:#{worker_name}:#{resource_key}"
+
+            @log_tags = log_tags.merge(bucket: bucket)
+          end
+
+          def to_s
+            resource_path
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/thread_pool.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module V2
+      class ThreadPool
+        delegate :logger, to: "Sbmt::Outbox"
+
+        BREAK = Object.new.freeze
+        SKIPPED = Object.new.freeze
+        PROCESSED = Object.new.freeze
+
+        def initialize(concurrency:, name: "thread_pool", random_startup_delay: true, start_async: true, &block)
+          self.concurrency = concurrency
+          self.name = name
+          self.random_startup_delay = random_startup_delay
+          self.start_async = start_async
+          self.task_source = block
+          self.task_mutex = Mutex.new
+          self.stopped = true
+          self.threads = Concurrent::Array.new
+        end
+
+        def next_task
+          task_mutex.synchronize do
+            return if stopped
+            task = task_source.call
+
+            if task == BREAK
+              self.stopped = true
+              return
+            end
+
+            task
+          end
+        end
+
+        def start
+          self.stopped = false
+
+          mode = start_async ? "async" : "sync"
+          logger.log_info("#{name}: starting #{concurrency} threads in #{mode} mode")
+
+          result = run_threads do |task|
+            logger.with_tags(worker: worker_number) do
+              yield worker_number, task
+            end
+          end
+
+          logger.log_info("#{name}: threads started")
+
+          raise result if result.is_a?(Exception)
+        end
+
+        def stop
+          self.stopped = true
+
+          threads.map(&:join) if start_async
+        ensure
+          stop_threads
+        end
+
+        def running?
+          return false if stopped
+
+          true
+        end
+
+        def alive?(timeout)
+          return false if stopped
+
+          deadline = Time.current - timeout
+          threads.all? do |thread|
+            last_active_at = last_active_at(thread)
+            return false unless last_active_at
+
+            deadline < last_active_at
+          end
+        end
+
+        private
+
+        attr_accessor :concurrency, :name, :random_startup_delay, :task_source, :task_mutex, :stopped, :start_async, :threads
+
+        def touch_worker!
+          self.last_active_at = Time.current
+        end
+
+        def worker_number(thread = Thread.current)
+          thread.thread_variable_get("#{name}_worker_number:#{object_id}")
+        end
+
+        def last_active_at(thread = Thread.current)
+          thread.thread_variable_get("#{name}_last_active_at:#{object_id}")
+        end
+
+        def run_threads
+          exception = nil
+
+          in_threads do |worker_num|
+            self.worker_number = worker_num
+            # We don't want to start all threads at the same time
+            sleep(rand * (worker_num + 1)) if random_startup_delay
+
+            touch_worker!
+
+            until exception
+              task = next_task
+              break unless task
+
+              touch_worker!
+
+              begin
+                yield task
+              rescue Exception => e # rubocop:disable Lint/RescueException
+                exception = e
+              end
+            end
+          end
+
+          exception
+        end
+
+        def in_threads
+          Thread.handle_interrupt(Exception => :never) do
+            Thread.handle_interrupt(Exception => :immediate) do
+              concurrency.times do |i|
+                threads << Thread.new { yield(i) }
+              end
+              threads.map(&:value) unless start_async
+            end
+          ensure
+            stop_threads unless start_async
+          end
+        end
+
+        def stop_threads
+          threads.each(&:kill)
+          threads.clear
+        end
+
+        def worker_number=(num)
+          Thread.current.thread_variable_set("#{name}_worker_number:#{object_id}", num)
+        end
+
+        def last_active_at=(at)
+          Thread.current.thread_variable_set("#{name}_last_active_at:#{object_id}", at)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/v2/throttler.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module V2
+      module Throttler
+        THROTTLE_STATUS = "throttle"
+        SKIP_STATUS = "skip"
+        NOOP_STATUS = "noop"
+      end
+    end
+  end
+end