sbmt-outbox 6.10.5 → 6.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44e7fd575f780f405d0aea05415627bab8f9a566bd0417d8a1564bd784564be5
-  data.tar.gz: fff1b10a605751abd0e0f1c87da31710176ea39df22d1527f794aef4ca78dd4b
+  metadata.gz: 0b78c4c7a59f68f29949ef958639b1dd6ae9e30026ff30dd9c431f8e17551c7f
+  data.tar.gz: a2ead04d61fdb3f0ddeb2926f7019566e2c519d89496075c1b2c86b13890a006
 SHA512:
-  metadata.gz: 5730bdd718b6e87fb6f0b44b30d3d6e37fa9baf851eaab8c61c4a2c9fd430ed165e3272c11b06647538e09281cd0ad30d247959f24f40ec79370a6b5b05c80fc
-  data.tar.gz: 18c56bd8ca7a5004875d309cd2178ab2ddb91eaacd7b5535c0b989ad1ba427711f99129ef47ff59d53c54ad21df4a6bb0391488bde2e0b212615eb48819198ab
+  metadata.gz: 4d57153ed214e2caceaaf46162fc045213884f7d40ada35880d47c7084bce3b67b0725422838baebc750be024a519ba14217dc28ed8db5944118d9c940bd9cfd
+  data.tar.gz: ec87e24359bd4aed28bbef3185e0eedb1704dbd032ce4ab12a0c321742a3eecf50d534b6e6866d53c498785cbfc547b73780c2ec7baf5c253fa4e0070856dfd2

data/README.md

@@ -267,7 +267,8 @@ default: &default
   outbox_items: # outbox items section
     my_outbox_item: # underscored model class name
       owner: my_outbox_item_team # optional, used in Yabeda metrics
-      retention: P1W # retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      retention: P1W # for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      retention_delivered_items: PT6H # for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
       max_retries: 3 # default 0, the number of retries before the item will be marked as failed
       strict_order: false # optional, default
       transports: # transports section
@@ -342,7 +343,8 @@ end
 inbox_items: # inbox items section
   my_inbox_item: # underscored model class name
     owner: my_inbox_item_team # optional, used in Yabeda metrics
-    retention: P1W # retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    retention: P1W # for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    retention_delivered_items: PT6H # for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
     max_retries: 3 # default 0, the number of retries before the item will be marked as failed
     transports: # transports section
       import_order: # underscored transport class name

data/app/jobs/sbmt/outbox/base_delete_stale_items_job.rb

@@ -13,7 +13,8 @@ module Sbmt
       class << self
         def enqueue
           item_classes.each do |item_class|
-            perform_later(item_class.to_s)
+            delay = rand(15).seconds
+            set(wait: delay).perform_later(item_class.to_s)
           end
         end
 
@@ -41,12 +42,13 @@ module Sbmt
 
         lock_manager.lock("#{self.class.name}:#{item_class_name}:lock", LOCK_TTL) do |locked|
           if locked
-            duration = item_class.config.retention
+            duration_failed = item_class.config.retention
+            duration_delivered = item_class.config.retention_delivered_items
 
-            validate_retention!(duration)
+            validate_retention!(duration_failed)
 
             logger.with_tags(box_type: box_type, box_name: box_name) do
-              delete_stale_items(Time.current - duration)
+              delete_stale_items(Time.current - duration_failed, Time.current - duration_delivered)
             end
           else
             logger.log_info("Failed to acquire lock #{self.class.name}:#{item_class_name}")
@@ -58,25 +60,25 @@ module Sbmt
 
       private
 
-      def validate_retention!(duration)
-        return if duration >= MIN_RETENTION_PERIOD
+      def validate_retention!(duration_failed)
+        return if duration_failed >= MIN_RETENTION_PERIOD
 
         raise "Retention period for #{box_name} must be longer than #{MIN_RETENTION_PERIOD.inspect}"
       end
 
-      def delete_stale_items(waterline)
-        logger.log_info("Start deleting #{box_type} items for #{box_name} older than #{waterline}")
+      def delete_stale_items(waterline_failed, waterline_delivered)
+        logger.log_info("Start deleting #{box_type} items for #{box_name} older than: failed and discarded items #{waterline_failed} and delivered items #{waterline_delivered}")
 
         case database_type
         when :postgresql
-          postgres_delete_in_batches(waterline)
+          postgres_delete_in_batches(waterline_failed, waterline_delivered)
         when :mysql
-          mysql_delete_in_batches(waterline)
+          mysql_delete_in_batches(waterline_failed, waterline_delivered)
         else
           raise "Unsupported database type"
         end
 
-        logger.log_info("Successfully deleted #{box_type} items for #{box_name} older than #{waterline}")
+        logger.log_info("Successfully deleted #{box_type} items for #{box_name} older than: failed and discarded items #{waterline_failed} and delivered items #{waterline_delivered}")
       end
 
       # Deletes stale items from PostgreSQL database in batches
@@ -90,12 +92,22 @@ module Sbmt
       #   WHERE "items"."id" IN (
       #     SELECT "items"."id"
       #     FROM "items"
-      #     WHERE "items"."created_at" < '2023-05-01 00:00:00'
+      #     WHERE (
+      #       "items"."status" = 1 AND "items"."created_at" < '2023-05-01 00:00:00'
+      #     )
       #     LIMIT 1000
       #   )
-      def postgres_delete_in_batches(waterline)
+      def postgres_delete_in_batches(waterline_failed, waterline_delivered)
         table = item_class.arel_table
-        condition = table[:created_at].lt(waterline)
+
+        status_delivered = item_class.statuses[:delivered]
+        status_failed_discarded = item_class.statuses.values_at(:failed, :discarded)
+
+        delete_items_in_batches(table, table[:status].eq(status_delivered).and(table[:created_at].lt(waterline_delivered)))
+        delete_items_in_batches(table, table[:status].in(status_failed_discarded).and(table[:created_at].lt(waterline_failed)))
+      end
+
+      def delete_items_in_batches(table, condition)
         subquery = table
           .project(table[:id])
           .where(condition)
@@ -129,14 +141,25 @@ module Sbmt
       #
       # Example SQL generated for deletion:
       #   DELETE FROM `items`
-      #   WHERE `items`.`created_at` < '2023-05-01 00:00:00'
+      #   WHERE (
+      #     `items`.`status` = 1 AND `items`.`created_at` < '2023-05-01 00:00:00'
+      #   )
       #   LIMIT 1000
-      def mysql_delete_in_batches(waterline)
+      def mysql_delete_in_batches(waterline_failed, waterline_delivered)
+        status_delivered = item_class.statuses[:delivered]
+        status_failed_discarded = [item_class.statuses.values_at(:failed, :discarded)]
+
+        delete_items_in_batches_mysql(
+          item_class.where(status: status_delivered, created_at: ...waterline_delivered)
+        )
+        delete_items_in_batches_mysql(
+          item_class.where(status: status_failed_discarded).where(created_at: ...waterline_failed)
+        )
+      end
+
+      def delete_items_in_batches_mysql(query)
         loop do
-          deleted_count = item_class
-            .where("created_at < ?", waterline)
-            .limit(BATCH_SIZE)
-            .delete_all
+          deleted_count = query.limit(BATCH_SIZE).delete_all
 
           logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
           break if deleted_count == 0

data/app/models/sbmt/outbox/base_item.rb

@@ -1,8 +1,14 @@
 # frozen_string_literal: true
 
+require_relative "../../../../lib/sbmt/outbox/enum_refinement"
+
 module Sbmt
   module Outbox
     class BaseItem < Outbox.active_record_base_class
+      # For compatibility with rails < 7
+      # Remove when drop support of Rails < 7
+      using EnumRefinement
+
       self.abstract_class = true
 
       class << self
@@ -26,8 +32,8 @@ module Sbmt
 
         def calc_bucket_partitions(count)
           (0...count).to_a
-            .each_with_object({}) do |x, m|
-            m[x] = (0...config.bucket_size).to_a
+            .index_with do |x|
+            (0...config.bucket_size).to_a
               .select { |p| p % count == x }
           end
         end
@@ -46,7 +52,7 @@ module Sbmt
         end
       end
 
-      enum status: {
+      enum :status, {
         pending: 0,
         failed: 1,
         delivered: 2,

data/app/models/sbmt/outbox/base_item_config.rb

@@ -37,6 +37,13 @@ module Sbmt
         @retention ||= ActiveSupport::Duration.parse(options[:retention] || "P1W")
       end
 
+      def retention_delivered_items
+        @retention_delivered_items ||= begin
+          value = options[:retention_delivered_items] || retention
+          value.is_a?(String) ? ActiveSupport::Duration.parse(value) : value
+        end
+      end
+
       def max_retries
         @max_retries ||= (options[:max_retries] || 0).to_i
       end

data/lib/sbmt/outbox/enum_refinement.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module EnumRefinement
+      refine ActiveRecord::Base.singleton_class do
+        def enum(name, values = nil)
+          if Rails::VERSION::MAJOR >= 7
+            super
+          else
+            super(name => values)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module Outbox
-    VERSION = "6.10.5"
+    VERSION = "6.12.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-outbox
 version: !ruby/object:Gem::Version
-  version: 6.10.5
+  version: 6.12.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-11-05 00:00:00.000000000 Z
+date: 2025-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -86,20 +86,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8'
+        version: '8.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '8'
+        version: '8.1'
 - !ruby/object:Gem::Dependency
   name: yabeda
   requirement: !ruby/object:Gem::Requirement
@@ -563,6 +563,7 @@ files:
 - lib/sbmt/outbox/cli.rb
 - lib/sbmt/outbox/database_switcher.rb
 - lib/sbmt/outbox/engine.rb
+- lib/sbmt/outbox/enum_refinement.rb
 - lib/sbmt/outbox/error_tracker.rb
 - lib/sbmt/outbox/errors.rb
 - lib/sbmt/outbox/instrumentation/open_telemetry_loader.rb
@@ -626,7 +627,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.21
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Outbox service