sbmt-outbox 6.13.1 → 6.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bcee17041bce58afa1aae1d66f2eaa9fa6d5caf1fd7a120702784e5edf5a397d
-  data.tar.gz: 0decf4919d3b431b25aab5e05a80c18cf7abb4c30cf74e0ac13f3f96c794b294
+  metadata.gz: 2616bf630410714a263f3eea95a7c98af5568018386d0792b8af8ac8de9572b2
+  data.tar.gz: 711b190d7c2d45ae1e66f07e60d9742fea6d2f9d0eae07431756da99bbc71e83
 SHA512:
-  metadata.gz: c6dc93a04ec8754f9c2a268a954d3cef3418992fa44d27a0c141c68243bbeb411340080154c51dbff2a633554b576546b6d63bb2f66b9ec44eb8dc8b1b910d8a
-  data.tar.gz: 1b12263dd0ac7b43b01290640ee7fcb6c7dd1b44c8188a68bb77b30ffe5e03dae7f465a133beb09851e2504aceb620143a1e36bd6eb482847d79e134d3ad1b22
+  metadata.gz: 8af881dffaa9802491a6764f471a6fedaac2ea5766e1f2c6749a1b6f5523f74f5f2239025b6d1e48fd985243556569b1670ed80d5aeb92f0a40ae67c2254a777
+  data.tar.gz: 543ad28b03629931934332ef0f62a6190a90159c0aa45ef3800c09472a824bfd474f270a5c9d27df2879ace34d2f03bbc92206381476568313254cf8db34a70a

data/README.md

@@ -267,8 +267,12 @@ default: &default
   outbox_items: # outbox items section
     my_outbox_item: # underscored model class name
       owner: my_outbox_item_team # optional, used in Yabeda metrics
-      retention: P1W # for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
-      retention_delivered_items: PT6H # for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      retention: P1W #optional, default: P1W, for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      min_retention_period: P1D #optional, default: P1D, for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      retention_delivered_items: PT6H #optional, default: P1W, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      delivered_min_retention_period: PT1H #optional, default: PT1H, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      deletion_batch_size: 1_000 #optional, default: 1_000
+      deletion_sleep_time: 0.5 #optional, default: 0.5
       max_retries: 3 # default 0, the number of retries before the item will be marked as failed
       strict_order: false # optional, default
       transports: # transports section
@@ -343,8 +347,12 @@ end
 inbox_items: # inbox items section
   my_inbox_item: # underscored model class name
     owner: my_inbox_item_team # optional, used in Yabeda metrics
-    retention: P1W # for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
-    retention_delivered_items: PT6H # for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    retention: P1W #optional, default: P1W, for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    min_retention_period: P1D #optional, default: P1D, for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    retention_delivered_items: PT6H #optional, default: P1W, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    delivered_min_retention_period: PT1H #optional, default: PT1H, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    deletion_batch_size: 1_000 #optional, default: 1_000
+    deletion_sleep_time: 0.5 #optional, default: 0.5
     max_retries: 3 # default 0, the number of retries before the item will be marked as failed
     transports: # transports section
       import_order: # underscored transport class name

data/app/jobs/sbmt/outbox/base_delete_stale_items_job.rb

@@ -5,10 +5,7 @@ require "redlock"
 module Sbmt
   module Outbox
     class BaseDeleteStaleItemsJob < Outbox.active_job_base_class
-      MIN_RETENTION_PERIOD = 1.day
       LOCK_TTL = 10_800_000
-      BATCH_SIZE = 1_000
-      SLEEP_TIME = 0.5
 
       class << self
         def enqueue
@@ -45,7 +42,7 @@ module Sbmt
             duration_failed = item_class.config.retention
             duration_delivered = item_class.config.retention_delivered_items
 
-            validate_retention!(duration_failed)
+            validate_retention!(duration_delivered, duration_failed)
 
             logger.with_tags(box_type: box_type, box_name: box_name) do
               delete_stale_items(Time.current - duration_failed, Time.current - duration_delivered)
@@ -60,10 +57,22 @@ module Sbmt
 
       private
 
-      def validate_retention!(duration_failed)
-        return if duration_failed >= MIN_RETENTION_PERIOD
+      def validate_retention!(duration_delivered, duration_failed)
+        validate_retention_for!(
+          duration: duration_delivered,
+          min_period: item_class.config.delivered_min_retention_period,
+          error_message: "Retention period for #{box_name} must be longer than #{item_class.config.delivered_min_retention_period.inspect}"
+        )
+
+        validate_retention_for!(
+          duration: duration_failed,
+          min_period: item_class.config.min_retention_period,
+          error_message: "Retention period for #{box_name} must be longer than #{item_class.config.min_retention_period.inspect}"
+        )
+      end
 
-        raise "Retention period for #{box_name} must be longer than #{MIN_RETENTION_PERIOD.inspect}"
+      def validate_retention_for!(duration:, min_period:, error_message:)
+        raise error_message if duration < min_period
       end
 
       def delete_stale_items(waterline_failed, waterline_delivered)
@@ -111,7 +120,7 @@ module Sbmt
         subquery = table
           .project(table[:id])
           .where(condition)
-          .take(BATCH_SIZE)
+          .take(item_class.config.deletion_batch_size)
 
         delete_statement = Arel::Nodes::DeleteStatement.new
         delete_statement.relation = table
@@ -131,7 +140,7 @@ module Sbmt
           logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
           break if deleted_count == 0
           lock_timer.checkpoint!
-          sleep(SLEEP_TIME)
+          sleep(item_class.config.deletion_sleep_time)
         end
       end
 
@@ -167,7 +176,7 @@ module Sbmt
 
         loop do
           track_deleted_latency do
-            deleted_count = query.limit(BATCH_SIZE).delete_all
+            deleted_count = query.limit(item_class.config.deletion_batch_size).delete_all
           end
 
           track_deleted_counter(deleted_count)
@@ -175,7 +184,7 @@ module Sbmt
           logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
           break if deleted_count == 0
           lock_timer.checkpoint!
-          sleep(SLEEP_TIME)
+          sleep(item_class.config.deletion_sleep_time)
         end
       end
 

data/app/models/sbmt/outbox/base_item_config.rb

@@ -44,6 +44,22 @@ module Sbmt
         end
       end
 
+      def deletion_batch_size
+        @deletion_batch_size ||= (options[:deletion_batch_size] || 1_000).to_i
+      end
+
+      def deletion_sleep_time
+        @deletion_sleep_time ||= (options[:deletion_sleep_time] || 0.5).to_f
+      end
+
+      def min_retention_period
+        @min_retention_period ||= ActiveSupport::Duration.parse(options[:min_retention_period] || "P1D")
+      end
+
+      def delivered_min_retention_period
+        @delivered_min_retention_period ||= ActiveSupport::Duration.parse(options[:delivered_min_retention_period] || "PT1H")
+      end
+
       def max_retries
         @max_retries ||= (options[:max_retries] || 0).to_i
       end

data/lib/generators/outbox/install/templates/outbox.yml

@@ -17,6 +17,11 @@ default: &default
 #      partition_size: 2
 #      partition_strategy: number
 #      retention: P1W
+#      min_retention_period: P1D
+#      retention_delivered_items: PT6H 
+#      delivered_min_retention_period: PT1H 
+#      deletion_batch_size: 1_000
+#      deletion_sleep_time: 0.5
 #      retry_strategies:
 #        - exponential_backoff
 #        - compacted_log
@@ -32,6 +37,11 @@ default: &default
 #      partition_size: 2
 #      partition_strategy: number
 #      retention: P1W
+#      min_retention_period: P1D
+#      retention_delivered_items: PT6H 
+#      delivered_min_retention_period: PT1H 
+#      deletion_batch_size: 1_000
+#      deletion_sleep_time: 0.5
 #      retry_strategies:
 #        - exponential_backoff
 #      transports:

data/lib/sbmt/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module Outbox
-    VERSION = "6.13.1"
+    VERSION = "6.14.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-outbox
 version: !ruby/object:Gem::Version
-  version: 6.13.1
+  version: 6.14.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-01-15 00:00:00.000000000 Z
+date: 2025-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool