sbmt-outbox 6.11.0 → 6.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e410e7a4507b5470d10986ac8d9949c4c9460a6773a97063b0287d9e777c4d9
-  data.tar.gz: ef17da6564faeb5f17cee5073cbecebeca083a555a9c046d0794f210f1a1b414
+  metadata.gz: 0b78c4c7a59f68f29949ef958639b1dd6ae9e30026ff30dd9c431f8e17551c7f
+  data.tar.gz: a2ead04d61fdb3f0ddeb2926f7019566e2c519d89496075c1b2c86b13890a006
 SHA512:
-  metadata.gz: 1e5fa1c14bab5bee90e4f5c664c22cdc5e8bea8fcfca391a3eb967ddf1c3b57b96080ab147cf366e5cbc9d2add8d486b8b2ca5ebfe8c9e96d0e989ab27ae685d
-  data.tar.gz: c47678da77b2f59787dd64cd162974b7cd4f6954fc2fafae54614b930fcf44068498491aaddae77d99b86f50dc67637b2090ae7d155a844ca193b9f011cf7189
+  metadata.gz: 4d57153ed214e2caceaaf46162fc045213884f7d40ada35880d47c7084bce3b67b0725422838baebc750be024a519ba14217dc28ed8db5944118d9c940bd9cfd
+  data.tar.gz: ec87e24359bd4aed28bbef3185e0eedb1704dbd032ce4ab12a0c321742a3eecf50d534b6e6866d53c498785cbfc547b73780c2ec7baf5c253fa4e0070856dfd2

data/README.md

@@ -267,7 +267,8 @@ default: &default
   outbox_items: # outbox items section
     my_outbox_item: # underscored model class name
       owner: my_outbox_item_team # optional, used in Yabeda metrics
-      retention: P1W # retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      retention: P1W # for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      retention_delivered_items: PT6H # for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
       max_retries: 3 # default 0, the number of retries before the item will be marked as failed
       strict_order: false # optional, default
       transports: # transports section
@@ -342,7 +343,8 @@ end
 inbox_items: # inbox items section
   my_inbox_item: # underscored model class name
     owner: my_inbox_item_team # optional, used in Yabeda metrics
-    retention: P1W # retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    retention: P1W # for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    retention_delivered_items: PT6H # for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
     max_retries: 3 # default 0, the number of retries before the item will be marked as failed
     transports: # transports section
       import_order: # underscored transport class name

data/app/jobs/sbmt/outbox/base_delete_stale_items_job.rb

@@ -13,7 +13,8 @@ module Sbmt
       class << self
         def enqueue
           item_classes.each do |item_class|
-            perform_later(item_class.to_s)
+            delay = rand(15).seconds
+            set(wait: delay).perform_later(item_class.to_s)
           end
         end
 
@@ -41,12 +42,13 @@ module Sbmt
 
         lock_manager.lock("#{self.class.name}:#{item_class_name}:lock", LOCK_TTL) do |locked|
           if locked
-            duration = item_class.config.retention
+            duration_failed = item_class.config.retention
+            duration_delivered = item_class.config.retention_delivered_items
 
-            validate_retention!(duration)
+            validate_retention!(duration_failed)
 
             logger.with_tags(box_type: box_type, box_name: box_name) do
-              delete_stale_items(Time.current - duration)
+              delete_stale_items(Time.current - duration_failed, Time.current - duration_delivered)
             end
           else
             logger.log_info("Failed to acquire lock #{self.class.name}:#{item_class_name}")
@@ -58,25 +60,25 @@ module Sbmt
 
       private
 
-      def validate_retention!(duration)
-        return if duration >= MIN_RETENTION_PERIOD
+      def validate_retention!(duration_failed)
+        return if duration_failed >= MIN_RETENTION_PERIOD
 
         raise "Retention period for #{box_name} must be longer than #{MIN_RETENTION_PERIOD.inspect}"
       end
 
-      def delete_stale_items(waterline)
-        logger.log_info("Start deleting #{box_type} items for #{box_name} older than #{waterline}")
+      def delete_stale_items(waterline_failed, waterline_delivered)
+        logger.log_info("Start deleting #{box_type} items for #{box_name} older than: failed and discarded items #{waterline_failed} and delivered items #{waterline_delivered}")
 
         case database_type
         when :postgresql
-          postgres_delete_in_batches(waterline)
+          postgres_delete_in_batches(waterline_failed, waterline_delivered)
         when :mysql
-          mysql_delete_in_batches(waterline)
+          mysql_delete_in_batches(waterline_failed, waterline_delivered)
         else
           raise "Unsupported database type"
         end
 
-        logger.log_info("Successfully deleted #{box_type} items for #{box_name} older than #{waterline}")
+        logger.log_info("Successfully deleted #{box_type} items for #{box_name} older than: failed and discarded items #{waterline_failed} and delivered items #{waterline_delivered}")
       end
 
       # Deletes stale items from PostgreSQL database in batches
@@ -90,12 +92,22 @@ module Sbmt
       #   WHERE "items"."id" IN (
       #     SELECT "items"."id"
       #     FROM "items"
-      #     WHERE "items"."created_at" < '2023-05-01 00:00:00'
+      #     WHERE (
+      #       "items"."status" = 1 AND "items"."created_at" < '2023-05-01 00:00:00'
+      #     )
       #     LIMIT 1000
       #   )
-      def postgres_delete_in_batches(waterline)
+      def postgres_delete_in_batches(waterline_failed, waterline_delivered)
         table = item_class.arel_table
-        condition = table[:created_at].lt(waterline)
+
+        status_delivered = item_class.statuses[:delivered]
+        status_failed_discarded = item_class.statuses.values_at(:failed, :discarded)
+
+        delete_items_in_batches(table, table[:status].eq(status_delivered).and(table[:created_at].lt(waterline_delivered)))
+        delete_items_in_batches(table, table[:status].in(status_failed_discarded).and(table[:created_at].lt(waterline_failed)))
+      end
+
+      def delete_items_in_batches(table, condition)
         subquery = table
           .project(table[:id])
           .where(condition)
@@ -129,14 +141,25 @@ module Sbmt
       #
       # Example SQL generated for deletion:
       #   DELETE FROM `items`
-      #   WHERE `items`.`created_at` < '2023-05-01 00:00:00'
+      #   WHERE (
+      #     `items`.`status` = 1 AND `items`.`created_at` < '2023-05-01 00:00:00'
+      #   )
       #   LIMIT 1000
-      def mysql_delete_in_batches(waterline)
+      def mysql_delete_in_batches(waterline_failed, waterline_delivered)
+        status_delivered = item_class.statuses[:delivered]
+        status_failed_discarded = [item_class.statuses.values_at(:failed, :discarded)]
+
+        delete_items_in_batches_mysql(
+          item_class.where(status: status_delivered, created_at: ...waterline_delivered)
+        )
+        delete_items_in_batches_mysql(
+          item_class.where(status: status_failed_discarded).where(created_at: ...waterline_failed)
+        )
+      end
+
+      def delete_items_in_batches_mysql(query)
         loop do
-          deleted_count = item_class
-            .where(created_at: ...waterline)
-            .limit(BATCH_SIZE)
-            .delete_all
+          deleted_count = query.limit(BATCH_SIZE).delete_all
 
           logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
           break if deleted_count == 0

data/app/models/sbmt/outbox/base_item_config.rb

@@ -37,6 +37,13 @@ module Sbmt
         @retention ||= ActiveSupport::Duration.parse(options[:retention] || "P1W")
       end
 
+      def retention_delivered_items
+        @retention_delivered_items ||= begin
+          value = options[:retention_delivered_items] || retention
+          value.is_a?(String) ? ActiveSupport::Duration.parse(value) : value
+        end
+      end
+
       def max_retries
         @max_retries ||= (options[:max_retries] || 0).to_i
       end

data/lib/sbmt/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module Outbox
-    VERSION = "6.11.0"
+    VERSION = "6.12.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-outbox
 version: !ruby/object:Gem::Version
-  version: 6.11.0
+  version: 6.12.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-12-27 00:00:00.000000000 Z
+date: 2025-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool