sbmt-outbox 6.13.1 → 6.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bcee17041bce58afa1aae1d66f2eaa9fa6d5caf1fd7a120702784e5edf5a397d
-  data.tar.gz: 0decf4919d3b431b25aab5e05a80c18cf7abb4c30cf74e0ac13f3f96c794b294
+  metadata.gz: 89c324ebe083d7dece71ae19ee9011a8da91843b31997ddbde85722e9f8209a5
+  data.tar.gz: fedd73ddecdfc54da1728bf19cdce13632e40ada5c94047a783d15a7e82370f1
 SHA512:
-  metadata.gz: c6dc93a04ec8754f9c2a268a954d3cef3418992fa44d27a0c141c68243bbeb411340080154c51dbff2a633554b576546b6d63bb2f66b9ec44eb8dc8b1b910d8a
-  data.tar.gz: 1b12263dd0ac7b43b01290640ee7fcb6c7dd1b44c8188a68bb77b30ffe5e03dae7f465a133beb09851e2504aceb620143a1e36bd6eb482847d79e134d3ad1b22
+  metadata.gz: 671f178a2b39285be8adb846ce823652cf1c6f2a68f4ce7e7e327015ea69b22858f95bd8403a637bd38c8ebd981e1423341d420bd308f2fae0d8f9bf151550cb
+  data.tar.gz: bea9bbb10ced61b77814e708528deada7058cf7afc8c90726d3d2c4068c619a1ddecb39e5caa836b9620dcfa9fda28cc20e3d4a083e8a83163f6146d9b28b00b

data/README.md

@@ -267,8 +267,12 @@ default: &default
   outbox_items: # outbox items section
     my_outbox_item: # underscored model class name
       owner: my_outbox_item_team # optional, used in Yabeda metrics
-      retention: P1W # for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
-      retention_delivered_items: PT6H # for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      retention: P1W #optional, default: P1W, for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      min_retention_period: P1D #optional, default: P1D, for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      retention_delivered_items: PT6H #optional, default: P1W, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      delivered_min_retention_period: PT1H #optional, default: PT1H, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
+      deletion_batch_size: 1_000 #optional, default: 1_000
+      deletion_sleep_time: 0.5 #optional, default: 0.5
       max_retries: 3 # default 0, the number of retries before the item will be marked as failed
       strict_order: false # optional, default
       transports: # transports section
@@ -343,8 +347,12 @@ end
 inbox_items: # inbox items section
   my_inbox_item: # underscored model class name
     owner: my_inbox_item_team # optional, used in Yabeda metrics
-    retention: P1W # for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
-    retention_delivered_items: PT6H # for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    retention: P1W #optional, default: P1W, for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    min_retention_period: P1D #optional, default: P1D, for statuses: failed and discarded, retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    retention_delivered_items: PT6H #optional, default: P1W, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    delivered_min_retention_period: PT1H #optional, default: PT1H, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
+    deletion_batch_size: 1_000 #optional, default: 1_000
+    deletion_sleep_time: 0.5 #optional, default: 0.5
     max_retries: 3 # default 0, the number of retries before the item will be marked as failed
     transports: # transports section
       import_order: # underscored transport class name
@@ -431,6 +439,42 @@ outbox_items:
     partition_strategy: hash
 ```
 
+## Rake tasks
+
+```shell
+rake outbox:delete_items
+rake outbox:update_status_items
+```
+
+Example run:
+```shell
+rake outbox:delete_items[OutboxItem,1] # Mandatory parameters box class and status
+rake outbox:update_status_items[OutboxItem,0,3] # Mandatory parameters box class, current status and new status
+
+```
+
+Both tasks have optional parameters:
+```ruby
+- start_time # boxes are younger than the specified time, by default nil, time is specified in the format "2025-01-05T23:59:59"
+- end_time # boxes are older than the specified time, by default 6.hours.ago, time is specified in the format "2025-01-05T23:59:59"
+- batch_size # batch size, by default 1_000
+- sleep_time # sleep time between batches, by default 0.5
+```
+
+Example with optional parameters:
+  - format optional parameters:
+  ```shell
+    rake outbox:delete_items[klass_name,status,start_time,end_time,batch_size,sleep_time]
+
+    rake outbox:update_status_items[klass_name,status,new_status,start_time,end_time,batch_size,sleep_time]
+  ```
+  - example:
+  ```shell
+    rake outbox:delete_items[OutboxItem,1,"2025-01-05T23:59:59","2025-01-05T00:00:00",10_000,5]
+
+    rake outbox:update_status_items[OutboxItem,0,3,"2025-01-05T23:59:59","2025-01-05T00:00:00",10_000,5]
+  ```
+
 ## Concurrency
 
 The worker process consists of a poller and a processor, each of which has its own thread pool.

data/app/jobs/sbmt/outbox/base_delete_stale_items_job.rb

@@ -5,10 +5,7 @@ require "redlock"
 module Sbmt
   module Outbox
     class BaseDeleteStaleItemsJob < Outbox.active_job_base_class
-      MIN_RETENTION_PERIOD = 1.day
       LOCK_TTL = 10_800_000
-      BATCH_SIZE = 1_000
-      SLEEP_TIME = 0.5
 
       class << self
         def enqueue
@@ -45,7 +42,7 @@ module Sbmt
             duration_failed = item_class.config.retention
             duration_delivered = item_class.config.retention_delivered_items
 
-            validate_retention!(duration_failed)
+            validate_retention!(duration_delivered, duration_failed)
 
             logger.with_tags(box_type: box_type, box_name: box_name) do
               delete_stale_items(Time.current - duration_failed, Time.current - duration_delivered)
@@ -60,10 +57,22 @@ module Sbmt
 
       private
 
-      def validate_retention!(duration_failed)
-        return if duration_failed >= MIN_RETENTION_PERIOD
+      def validate_retention!(duration_delivered, duration_failed)
+        validate_retention_for!(
+          duration: duration_delivered,
+          min_period: item_class.config.delivered_min_retention_period,
+          error_message: "Retention period for #{box_name} must be longer than #{item_class.config.delivered_min_retention_period.inspect}"
+        )
+
+        validate_retention_for!(
+          duration: duration_failed,
+          min_period: item_class.config.min_retention_period,
+          error_message: "Retention period for #{box_name} must be longer than #{item_class.config.min_retention_period.inspect}"
+        )
+      end
 
-        raise "Retention period for #{box_name} must be longer than #{MIN_RETENTION_PERIOD.inspect}"
+      def validate_retention_for!(duration:, min_period:, error_message:)
+        raise error_message if duration < min_period
       end
 
       def delete_stale_items(waterline_failed, waterline_delivered)
@@ -111,7 +120,7 @@ module Sbmt
         subquery = table
           .project(table[:id])
           .where(condition)
-          .take(BATCH_SIZE)
+          .take(item_class.config.deletion_batch_size)
 
         delete_statement = Arel::Nodes::DeleteStatement.new
         delete_statement.relation = table
@@ -131,7 +140,7 @@ module Sbmt
           logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
           break if deleted_count == 0
           lock_timer.checkpoint!
-          sleep(SLEEP_TIME)
+          sleep(item_class.config.deletion_sleep_time)
         end
       end
 
@@ -167,7 +176,7 @@ module Sbmt
 
         loop do
           track_deleted_latency do
-            deleted_count = query.limit(BATCH_SIZE).delete_all
+            deleted_count = query.limit(item_class.config.deletion_batch_size).delete_all
           end
 
           track_deleted_counter(deleted_count)
@@ -175,7 +184,7 @@ module Sbmt
           logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
           break if deleted_count == 0
           lock_timer.checkpoint!
-          sleep(SLEEP_TIME)
+          sleep(item_class.config.deletion_sleep_time)
         end
       end
 

data/app/models/sbmt/outbox/base_item_config.rb

@@ -44,6 +44,22 @@ module Sbmt
         end
       end
 
+      def deletion_batch_size
+        @deletion_batch_size ||= (options[:deletion_batch_size] || 1_000).to_i
+      end
+
+      def deletion_sleep_time
+        @deletion_sleep_time ||= (options[:deletion_sleep_time] || 0.5).to_f
+      end
+
+      def min_retention_period
+        @min_retention_period ||= ActiveSupport::Duration.parse(options[:min_retention_period] || "P1D")
+      end
+
+      def delivered_min_retention_period
+        @delivered_min_retention_period ||= ActiveSupport::Duration.parse(options[:delivered_min_retention_period] || "PT1H")
+      end
+
       def max_retries
         @max_retries ||= (options[:max_retries] || 0).to_i
       end

data/lib/generators/outbox/install/templates/outbox.yml

@@ -17,6 +17,11 @@ default: &default
 #      partition_size: 2
 #      partition_strategy: number
 #      retention: P1W
+#      min_retention_period: P1D
+#      retention_delivered_items: PT6H 
+#      delivered_min_retention_period: PT1H 
+#      deletion_batch_size: 1_000
+#      deletion_sleep_time: 0.5
 #      retry_strategies:
 #        - exponential_backoff
 #        - compacted_log
@@ -32,6 +37,11 @@ default: &default
 #      partition_size: 2
 #      partition_strategy: number
 #      retention: P1W
+#      min_retention_period: P1D
+#      retention_delivered_items: PT6H 
+#      delivered_min_retention_period: PT1H 
+#      deletion_batch_size: 1_000
+#      deletion_sleep_time: 0.5
 #      retry_strategies:
 #        - exponential_backoff
 #      transports:

data/lib/sbmt/outbox/engine.rb

@@ -79,6 +79,8 @@ module Sbmt
       rake_tasks do
         load "sbmt/outbox/tasks/retry_failed_items.rake"
         load "sbmt/outbox/tasks/delete_failed_items.rake"
+        load "sbmt/outbox/tasks/delete_items.rake"
+        load "sbmt/outbox/tasks/update_status_items.rake"
       end
     end
   end

data/lib/sbmt/outbox/tasks/delete_items.rake

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+namespace :outbox do
+  desc "Delete outbox/inbox items"
+  task :delete_items, [:klass_name, :status, :start_time, :end_time, :batch_size, :sleep_time] => :environment do |_, args|
+    args.with_defaults(start_time: nil, end_time: 6.hours.ago, batch_size: 1000, sleep_time: 0.5)
+
+    klass_name = args[:klass_name]
+    status = args[:status]
+    start_time = args[:start_time]
+    end_time = args[:end_time]
+    batch_size = args[:batch_size]
+    sleep_time = args[:sleep_time]
+
+    unless klass_name && status
+      raise "Error: Class and status must be specified. Example: rake outbox:delete_items[OutboxItem,1]"
+    end
+
+    klass_name = klass_name.constantize
+    query = klass_name.where(status: status)
+
+    if start_time && end_time
+      query = query.where(created_at: start_time..end_time)
+    elsif start_time
+      query = query.where(created_at: start_time..)
+    elsif end_time
+      query = query.where(created_at: ..end_time)
+    end
+
+    total_deleted = 0
+    query.in_batches(of: batch_size) do |batch|
+      deleted_count = batch.delete_all
+
+      Rails.logger.info("Batch items deleted: #{deleted_count}")
+
+      total_deleted += deleted_count
+
+      sleep sleep_time
+    end
+
+    Rails.logger.info("Total items deleted: #{total_deleted}")
+  end
+end

data/lib/sbmt/outbox/tasks/update_status_items.rake

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+namespace :outbox do
+  desc "Update status of outbox/inbox items"
+  task :update_status_items, [:klass_name, :status, :new_status, :start_time, :end_time, :batch_size, :sleep_time] => :environment do |_, args|
+    args.with_defaults(start_time: nil, end_time: 6.hours.ago, batch_size: 1000, sleep_time: 0.5)
+
+    klass_name = args[:klass_name]
+    status = args[:status]
+    new_status = args[:new_status]
+    start_time = args[:start_time]
+    end_time = args[:end_time]
+    batch_size = args[:batch_size]
+    sleep_time = args[:sleep_time]
+
+    unless klass_name && status && new_status
+      raise "Error: Class, current status, and new status must be specified. Example: rake outbox:update_status_items[OutboxItem,0,3]"
+    end
+
+    klass_name = klass_name.constantize
+    query = klass_name.where(status: status)
+
+    if start_time && end_time
+      query = query.where(created_at: start_time..end_time)
+    elsif start_time
+      query = query.where(created_at: start_time..)
+    elsif end_time
+      query = query.where(created_at: ..end_time)
+    end
+
+    total_updated = 0
+    query.in_batches(of: batch_size) do |batch|
+      updated_count = batch.update_all(status: new_status)
+
+      Rails.logger.info("Batch items updated: #{updated_count}")
+
+      total_updated += updated_count
+      sleep sleep_time
+    end
+
+    Rails.logger.info("Total items updated: #{total_updated}")
+  end
+end

data/lib/sbmt/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module Outbox
-    VERSION = "6.13.1"
+    VERSION = "6.15.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-outbox
 version: !ruby/object:Gem::Version
-  version: 6.13.1
+  version: 6.15.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-01-15 00:00:00.000000000 Z
+date: 2025-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -582,7 +582,9 @@ files:
 - lib/sbmt/outbox/probes/probe.rb
 - lib/sbmt/outbox/redis_client_factory.rb
 - lib/sbmt/outbox/tasks/delete_failed_items.rake
+- lib/sbmt/outbox/tasks/delete_items.rake
 - lib/sbmt/outbox/tasks/retry_failed_items.rake
+- lib/sbmt/outbox/tasks/update_status_items.rake
 - lib/sbmt/outbox/v1/thread_pool.rb
 - lib/sbmt/outbox/v1/throttler.rb
 - lib/sbmt/outbox/v1/worker.rb