sbmt-outbox 6.16.0 → 6.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 86044914d6babd961a882ecfd9394875e0b67f74a2b92f44db48480bb5096448
-  data.tar.gz: 3ae09e5801c2dcacc9096bae5c2428f2c57445394dd5a3d036c58b1fa63121d2
+  metadata.gz: 3145b7891faf0d1c657121122757608c910d34812d5084674dd77b1d925d8a43
+  data.tar.gz: de2766b4beb880123d46224b6d978c8bf2b031ad8dfcb22722acca72d3d94d0b
 SHA512:
-  metadata.gz: 9f180114bce2540c91f6c942195fa59c51bcfea67bd984ee5b7db19fd95653933492ce30f7890833e879469b2bb08ca5bcfad2b5a8efa8221eae904b2839a42f
-  data.tar.gz: e9aeadbc87d76a0f86c7b7c9ddad157c45851e3941901bf4ff249c116a7d32827645157ed644709ba59756569af0d9637e0b66c0c9b0c10937ba6b5878ac740d
+  metadata.gz: a129a09ceb3b19931a6eb698dabbf4438628b209297f1d32d4e80e7d141a2ea3dd5f70e5a34cfe999da793731c836c94e55e93d6127a9e3e11a9021d62163c6a
+  data.tar.gz: 0b9a632efe465d1554ddedc11197d752996f6dc621b51c831e69c1e9b1acd528f3180be1107bfcc43dd796cb48f9459fc135f79ae46bd46a30b1362135e9045d

data/README.md

@@ -273,6 +273,7 @@ default: &default
       delivered_min_retention_period: PT1H #optional, default: PT1H, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
       deletion_batch_size: 1_000 #optional, default: 1_000
       deletion_sleep_time: 0.5 #optional, default: 0.5
+      deletion_time_window: PT4H #optional, default: PT4H, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
       max_retries: 3 # default 0, the number of retries before the item will be marked as failed
       strict_order: false # optional, default
       transports: # transports section
@@ -353,6 +354,7 @@ inbox_items: # inbox items section
     delivered_min_retention_period: PT1H #optional, default: PT1H, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
     deletion_batch_size: 1_000 #optional, default: 1_000
     deletion_sleep_time: 0.5 #optional, default: 0.5
+    deletion_time_window: PT4H #optional, default: PT4H, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
     max_retries: 3 # default 0, the number of retries before the item will be marked as failed
     transports: # transports section
       import_order: # underscored transport class name

data/app/interactors/sbmt/outbox/process_item.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "sbmt/outbox/metrics/utils"
+require "sbmt/outbox/v2/redis_item_meta"
 
 module Sbmt
   module Outbox
@@ -8,14 +9,16 @@ module Sbmt
       param :item_class, reader: :private
       param :item_id, reader: :private
       option :worker_version, reader: :private, optional: true, default: -> { 1 }
+      option :cache_ttl_sec, reader: :private, optional: true, default: -> { 5 * 60 }
+      option :redis, reader: :private, optional: true, default: -> {}
 
       METRICS_COUNTERS = %i[error_counter retry_counter sent_counter fetch_error_counter discarded_counter].freeze
 
-      delegate :log_success, :log_info, :log_failure, to: "Sbmt::Outbox.logger"
+      delegate :log_success, :log_info, :log_failure, :log_debug, to: "Sbmt::Outbox.logger"
       delegate :item_process_middlewares, to: "Sbmt::Outbox"
       delegate :box_type, :box_name, :owner, to: :item_class
 
-      attr_accessor :process_latency
+      attr_accessor :process_latency, :retry_latency
 
       def call
         log_success(
@@ -26,9 +29,23 @@ module Sbmt
         item = nil
 
         item_class.transaction do
-          item = yield fetch_item
+          item = yield fetch_item_and_lock_for_update
+
+          cached_item = fetch_redis_item_meta(redis_item_key(item_id))
+          if cached_retries_exceeded?(cached_item)
+            msg = "max retries exceeded: marking item as failed based on cached data: #{cached_item}"
+            item.set_errors_count(cached_item.errors_count)
+            track_failed(msg, item)
+            next Failure(msg)
+          end
+
+          if cached_greater_errors_count?(item, cached_item)
+            log_failure("inconsistent item: cached_errors_count:#{cached_item.errors_count} > db_errors_count:#{item.errors_count}: setting errors_count based on cached data:#{cached_item}")
+            item.set_errors_count(cached_item.errors_count)
+          end
 
           if item.processed_at?
+            self.retry_latency = Time.current - item.created_at
             item.config.retry_strategies.each do |retry_strategy|
               yield check_retry_strategy(item, retry_strategy)
             end
@@ -62,7 +79,48 @@ module Sbmt
 
       private
 
-      def fetch_item
+      def cached_retries_exceeded?(cached_item)
+        return false unless cached_item
+
+        item_class.max_retries_exceeded?(cached_item.errors_count)
+      end
+
+      def cached_greater_errors_count?(db_item, cached_item)
+        return false unless cached_item
+
+        cached_item.errors_count > db_item.errors_count
+      end
+
+      def fetch_redis_item_meta(redis_key)
+        return if worker_version < 2
+
+        data = redis.call("GET", redis_key)
+        return if data.blank?
+
+        Sbmt::Outbox::V2::RedisItemMeta.deserialize!(data)
+      rescue => ex
+        log_debug("error while fetching redis meta: #{ex.message}")
+        nil
+      end
+
+      def set_redis_item_meta(item, ex)
+        return if worker_version < 2
+        return if item.nil?
+
+        redis_key = redis_item_key(item.id)
+        error_msg = format_exception_error(ex, extract_cause: false)
+        data = Sbmt::Outbox::V2::RedisItemMeta.new(errors_count: item.errors_count, error_msg: error_msg)
+        redis.call("SET", redis_key, data.to_s, "EX", cache_ttl_sec)
+      rescue => ex
+        log_debug("error while fetching redis meta: #{ex.message}")
+        nil
+      end
+
+      def redis_item_key(item_id)
+        "#{box_type}:#{item_class.box_name}:#{item_id}"
+      end
+
+      def fetch_item_and_lock_for_update
         item = item_class
           .lock("FOR UPDATE")
           .find_by(id: item_id)
@@ -171,6 +229,7 @@ module Sbmt
           item.pending!
         end
       rescue => e
+        set_redis_item_meta(item, e)
         log_error_handling_error(e, item)
       end
 
@@ -259,6 +318,7 @@ module Sbmt
         end
 
         track_process_latency(labels) if process_latency
+        track_retry_latency(labels) if retry_latency
 
         return unless counters[:sent_counter].positive?
 
@@ -279,6 +339,10 @@ module Sbmt
       def track_process_latency(labels)
         Yabeda.outbox.process_latency.measure(labels, process_latency.round(3))
       end
+
+      def track_retry_latency(labels)
+        Yabeda.outbox.retry_latency.measure(labels, retry_latency.round(3))
+      end
     end
   end
 end

data/app/jobs/sbmt/outbox/base_delete_stale_items_job.rb

@@ -102,45 +102,50 @@ module Sbmt
       #     SELECT "items"."id"
       #     FROM "items"
       #     WHERE (
-      #       "items"."status" = 1 AND "items"."created_at" < '2023-05-01 00:00:00'
+      #       "items"."status" IN (2) AND "items"."created_at" BETWEEN "2025-01-29 12:18:32.917836" AND "2025-01-29 12:18:32.927596" LIMIT 1000
       #     )
-      #     LIMIT 1000
       #   )
       def postgres_delete_in_batches(waterline_failed, waterline_delivered)
-        table = item_class.arel_table
-
         status_delivered = item_class.statuses[:delivered]
         status_failed_discarded = item_class.statuses.values_at(:failed, :discarded)
 
-        delete_items_in_batches(table, table[:status].eq(status_delivered).and(table[:created_at].lt(waterline_delivered)))
-        delete_items_in_batches(table, table[:status].in(status_failed_discarded).and(table[:created_at].lt(waterline_failed)))
+        delete_items_in_batches_with_between(waterline_delivered, status_delivered)
+        delete_items_in_batches_with_between(waterline_failed, status_failed_discarded)
       end
 
-      def delete_items_in_batches(table, condition)
-        subquery = table
-          .project(table[:id])
-          .where(condition)
-          .take(item_class.config.deletion_batch_size)
-
-        delete_statement = Arel::Nodes::DeleteStatement.new
-        delete_statement.relation = table
-        delete_statement.wheres = [table[:id].in(subquery)]
+      def delete_items_in_batches_with_between(waterline, statuses)
+        table = item_class.arel_table
+        batch_size = item_class.config.deletion_batch_size
+        time_window = item_class.config.deletion_time_window
+        min_date = item_class.where(table[:status].in(statuses)).minimum(:created_at)
         deleted_count = nil
 
-        loop do
-          track_deleted_latency do
-            deleted_count = item_class
-              .connection
-              .execute(delete_statement.to_sql)
-              .cmd_tuples
-          end
+        while min_date && min_date < waterline
+          max_date = [min_date + time_window, waterline].min
+
+          loop do
+            subquery = table
+              .project(table[:id])
+              .where(table[:status].in(statuses))
+              .where(table[:created_at].between(min_date..max_date))
+              .take(batch_size)
+
+            delete_statement = Arel::Nodes::DeleteStatement.new
+            delete_statement.relation = table
+            delete_statement.wheres = [table[:id].in(subquery)]
+
+            track_deleted_latency do
+              deleted_count = item_class.connection.execute(delete_statement.to_sql).cmd_tuples
+            end
 
-          track_deleted_counter(deleted_count)
+            track_deleted_counter(deleted_count)
 
-          logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
-          break if deleted_count == 0
-          lock_timer.checkpoint!
-          sleep(item_class.config.deletion_sleep_time)
+            logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} between #{min_date} and #{max_date}")
+            break if deleted_count < batch_size
+            lock_timer.checkpoint!
+            sleep(item_class.config.deletion_sleep_time) if deleted_count > 0
+          end
+          min_date = max_date
         end
       end
 
@@ -154,37 +159,43 @@ module Sbmt
       # This approach doesn't require a subquery, making it more straightforward.
       #
       # Example SQL generated for deletion:
-      #   DELETE FROM `items`
+      #   DELETE FROM "items"
       #   WHERE (
-      #     `items`.`status` = 1 AND `items`.`created_at` < '2023-05-01 00:00:00'
+      #     "items"."status" IN (2) AND "items"."created_at" BETWEEN "2024-12-29 18:34:25.369234" AND "2024-12-29 22:34:25.369234" LIMIT 1000
       #   )
-      #   LIMIT 1000
       def mysql_delete_in_batches(waterline_failed, waterline_delivered)
         status_delivered = item_class.statuses[:delivered]
         status_failed_discarded = [item_class.statuses.values_at(:failed, :discarded)]
 
-        delete_items_in_batches_mysql(
-          item_class.where(status: status_delivered, created_at: ...waterline_delivered)
-        )
-        delete_items_in_batches_mysql(
-          item_class.where(status: status_failed_discarded).where(created_at: ...waterline_failed)
-        )
+        delete_items_in_batches_with_between_mysql(waterline_delivered, status_delivered)
+        delete_items_in_batches_with_between_mysql(waterline_failed, status_failed_discarded)
       end
 
-      def delete_items_in_batches_mysql(query)
+      def delete_items_in_batches_with_between_mysql(waterline, statuses)
+        batch_size = item_class.config.deletion_batch_size
+        time_window = item_class.config.deletion_time_window
+        min_date = item_class.where(status: statuses).minimum(:created_at)
         deleted_count = nil
 
-        loop do
-          track_deleted_latency do
-            deleted_count = query.limit(item_class.config.deletion_batch_size).delete_all
-          end
+        while min_date && min_date < waterline
+          max_date = [min_date + time_window, waterline].min
+
+          loop do
+            track_deleted_latency do
+              deleted_count = item_class
+                .where(status: statuses, created_at: min_date..max_date)
+                .limit(batch_size)
+                .delete_all
+            end
 
-          track_deleted_counter(deleted_count)
+            track_deleted_counter(deleted_count)
 
-          logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
-          break if deleted_count == 0
-          lock_timer.checkpoint!
-          sleep(item_class.config.deletion_sleep_time)
+            logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} between #{min_date} and #{max_date}")
+            break if deleted_count < batch_size
+            lock_timer.checkpoint!
+            sleep(item_class.config.deletion_sleep_time) if deleted_count > 0
+          end
+          min_date = max_date
         end
       end
 

data/app/models/sbmt/outbox/base_item.rb

@@ -49,6 +49,17 @@ module Sbmt
               end
             end
         end
+
+        def max_retries_exceeded?(count)
+          return false if config.strict_order
+          return true unless retriable?
+
+          count > config.max_retries
+        end
+
+        def retriable?
+          config.max_retries > 0
+        end
       end
 
       enum :status, {
@@ -135,20 +146,21 @@ module Sbmt
       end
 
       def retriable?
-        config.max_retries > 0
+        self.class.retriable?
       end
 
       def max_retries_exceeded?
-        return false if config.strict_order
-        return true unless retriable?
-
-        errors_count > config.max_retries
+        self.class.max_retries_exceeded?(errors_count)
       end
 
       def increment_errors_counter
         increment(:errors_count)
       end
 
+      def set_errors_count(count)
+        self.errors_count = count
+      end
+
       def add_error(ex_or_msg)
         increment_errors_counter
 

data/app/models/sbmt/outbox/base_item_config.rb

@@ -60,6 +60,10 @@ module Sbmt
         @delivered_min_retention_period ||= ActiveSupport::Duration.parse(options[:delivered_min_retention_period] || "PT1H")
       end
 
+      def deletion_time_window
+        @deletion_time_window ||= ActiveSupport::Duration.parse(options[:deletion_time_window] || "PT4H")
+      end
+
       def max_retries
         @max_retries ||= (options[:max_retries] || 0).to_i
       end

data/config/initializers/yabeda.rb

@@ -50,6 +50,12 @@ Yabeda.configure do
       buckets: [0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 30].freeze,
       comment: "A histogram for outbox/inbox deletion latency"
 
+    histogram :retry_latency,
+      tags: %i[type name partition owner],
+      unit: :seconds,
+      buckets: [1, 10, 20, 50, 120, 300, 900, 1800, 3600].freeze,
+      comment: "A histogram outbox retry latency"
+
     counter :deleted_counter,
       tags: %i[box_type box_name],
       comment: "A counter for the number of deleted outbox/inbox items"

data/lib/sbmt/outbox/engine.rb

@@ -25,8 +25,8 @@ module Sbmt
           c.cdn_url = "https://cdn.jsdelivr.net/npm/sbmt-outbox-ui@0.0.8/dist/assets/index.js"
         end
         c.process_items = ActiveSupport::OrderedOptions.new.tap do |c|
-          c.general_timeout = 120
-          c.cutoff_timeout = 60
+          c.general_timeout = 180
+          c.cutoff_timeout = 90
           c.batch_size = 200
         end
         c.worker = ActiveSupport::OrderedOptions.new.tap do |c|
@@ -54,8 +54,8 @@ module Sbmt
         end
         c.processor = ActiveSupport::OrderedOptions.new.tap do |pc|
           pc.threads_count = 4
-          pc.general_timeout = 120
-          pc.cutoff_timeout = 60
+          pc.general_timeout = 180
+          pc.cutoff_timeout = 90
           pc.brpop_delay = 1
         end
 

data/lib/sbmt/outbox/v2/processor.rb

@@ -10,7 +10,7 @@ module Sbmt
     module V2
       class Processor < BoxProcessor
         delegate :processor_config, :batch_process_middlewares, :logger, to: "Sbmt::Outbox"
-        attr_reader :lock_timeout, :brpop_delay
+        attr_reader :lock_timeout, :cache_ttl, :cutoff_timeout, :brpop_delay
 
         REDIS_BRPOP_MIN_DELAY = 0.1
 
@@ -18,11 +18,16 @@ module Sbmt
           boxes,
           threads_count: nil,
           lock_timeout: nil,
+          cache_ttl: nil,
+          cutoff_timeout: nil,
           brpop_delay: nil,
           redis: nil
         )
           @lock_timeout = lock_timeout || processor_config.general_timeout
+          @cache_ttl = cache_ttl || @lock_timeout * 10
+          @cutoff_timeout = cutoff_timeout || processor_config.cutoff_timeout
           @brpop_delay = brpop_delay || redis_brpop_delay(boxes.count, processor_config.brpop_delay)
+          @redis = redis
 
           super(boxes: boxes, threads_count: threads_count || processor_config.threads_count, name: "processor", redis: redis)
         end
@@ -66,14 +71,19 @@ module Sbmt
         end
 
         def process(task)
-          lock_timer = Cutoff.new(lock_timeout)
+          lock_timer = Cutoff.new(cutoff_timeout)
           last_id = 0
           strict_order = task.item_class.config.strict_order
 
           box_worker.item_execution_runtime.measure(task.yabeda_labels) do
             Outbox.database_switcher.use_master do
               task.ids.each do |id|
-                result = ProcessItem.call(task.item_class, id, worker_version: task.yabeda_labels[:worker_version])
+                result = ProcessItem.call(
+                  task.item_class, id,
+                  worker_version: task.yabeda_labels[:worker_version],
+                  cache_ttl_sec: cache_ttl,
+                  redis: @redis
+                )
 
                 box_worker.job_items_counter.increment(task.yabeda_labels)
                 last_id = id

data/lib/sbmt/outbox/v2/redis_item_meta.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module V2
+      class RedisItemMeta
+        attr_reader :version, :timestamp, :errors_count, :error_msg
+
+        CURRENT_VERSION = 1
+        MAX_ERROR_LEN = 200
+
+        def initialize(errors_count:, error_msg:, timestamp: Time.current.to_i, version: CURRENT_VERSION)
+          @errors_count = errors_count
+          @error_msg = error_msg
+          @timestamp = timestamp
+          @version = version
+        end
+
+        def to_s
+          serialize
+        end
+
+        def serialize
+          JSON.generate({
+            version: version,
+            timestamp: timestamp,
+            errors_count: errors_count,
+            error_msg: error_msg.slice(0, MAX_ERROR_LEN)
+          })
+        end
+
+        def self.deserialize!(value)
+          raise "invalid data type: string is required" unless value.is_a?(String)
+
+          data = JSON.parse!(value, max_nesting: 1)
+          new(
+            version: data["version"],
+            timestamp: data["timestamp"].to_i,
+            errors_count: data["errors_count"].to_i,
+            error_msg: data["error_msg"]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module Outbox
-    VERSION = "6.16.0"
+    VERSION = "6.18.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-outbox
 version: !ruby/object:Gem::Version
-  version: 6.16.0
+  version: 6.18.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-01-28 00:00:00.000000000 Z
+date: 2025-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -600,6 +600,7 @@ files:
 - lib/sbmt/outbox/v2/poll_throttler/redis_queue_time_lag.rb
 - lib/sbmt/outbox/v2/poller.rb
 - lib/sbmt/outbox/v2/processor.rb
+- lib/sbmt/outbox/v2/redis_item_meta.rb
 - lib/sbmt/outbox/v2/redis_job.rb
 - lib/sbmt/outbox/v2/tasks/base.rb
 - lib/sbmt/outbox/v2/tasks/default.rb
@@ -629,7 +630,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Outbox service