sbmt-outbox 6.10.2 → 6.10.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b9f32bdc55d20994a9ccd6a7c448b68b1732e9c0667ed06494b594635188dad
-  data.tar.gz: da53b24ea40c840c5324ee3176f8341b4e0f96e19d6905dcd1e1df93f49913b0
+  metadata.gz: fabf75c06b0dea9c159918fe4496d3ffd47aa71caa1c364a6bfbef8c16a93494
+  data.tar.gz: d3c9b3a1bdfd55a0331b7e4fb971ac629c91b66cb952f4abb659e2b046a50187
 SHA512:
-  metadata.gz: 721fcc1d9eb752ec086cb51407f6f387c33bde2d11c0ef57046154dc1ccf7a9d2eb34b61ac4a3668dcff0893ae4c394cea216605d2bd9780e417facb2ff2b215
-  data.tar.gz: 2eac4d443109be1ba47d0d2acff62eb3151cc7f9425d7eb71478fec2b5f478fe4824b8270ed0d763fc1f2aa93390c5f6cfd17f43fe2a851f7eca851050c459fa
+  metadata.gz: ff0b3cce600d2b1b82750036da4819d21715d05956491b0b9d7f056bb62800fc1e49b72ae069b978196c06671cd7157f0b0e78cb133e80819d16e191da5d533d
+  data.tar.gz: 99754cf0cb25199388a3dd3457c5d8225ff3a1cc6b8a16da2c67c69efd999268597be2aa2a2fba13a24e484e262ea59f382af2699da048143432cd97a74a3516

data/app/jobs/sbmt/outbox/base_delete_stale_items_job.rb

@@ -7,8 +7,8 @@ module Sbmt
     class BaseDeleteStaleItemsJob < Outbox.active_job_base_class
       MIN_RETENTION_PERIOD = 1.day
       LOCK_TTL = 10_800_000
-      BATCH_SIZE = 1000
-      SLEEP_TIME = 1
+      BATCH_SIZE = 1_000
+      SLEEP_TIME = 0.5
 
       class << self
         def enqueue
@@ -25,7 +25,7 @@ module Sbmt
       delegate :config, :logger, to: "Sbmt::Outbox"
       delegate :box_type, :box_name, to: :item_class
 
-      attr_accessor :item_class
+      attr_accessor :item_class, :lock_timer
 
       def perform(item_class_name)
         self.item_class = item_class_name.constantize
@@ -36,6 +36,7 @@ module Sbmt
           Redis.new(config.redis)
         end
 
+        self.lock_timer = Cutoff.new(LOCK_TTL / 1000)
         lock_manager = Redlock::Client.new([client], retry_count: 0)
 
         lock_manager.lock("#{self.class.name}:#{item_class_name}:lock", LOCK_TTL) do |locked|
@@ -51,6 +52,8 @@ module Sbmt
             logger.log_info("Failed to acquire lock #{self.class.name}:#{item_class_name}")
           end
         end
+      rescue Cutoff::CutoffExceededError
+        logger.log_info("Lock timeout while processing #{item_class_name}")
       end
 
       private
@@ -64,17 +67,94 @@ module Sbmt
       def delete_stale_items(waterline)
         logger.log_info("Start deleting #{box_type} items for #{box_name} older than #{waterline}")
 
+        case database_type
+        when :postgresql
+          postgres_delete_in_batches(waterline)
+        when :mysql
+          mysql_delete_in_batches(waterline)
+        else
+          raise "Unsupported database type"
+        end
+
+        logger.log_info("Successfully deleted #{box_type} items for #{box_name} older than #{waterline}")
+      end
+
+      # Deletes stale items from PostgreSQL database in batches
+      #
+      # This method efficiently deletes items older than the given waterline
+      # using a subquery approach to avoid locking large portions of the table.
+      #
+      #
+      # Example SQL generated for deletion:
+      #   DELETE FROM "items"
+      #   WHERE "items"."id" IN (
+      #     SELECT "items"."id"
+      #     FROM "items"
+      #     WHERE "items"."created_at" < '2023-05-01 00:00:00'
+      #     LIMIT 1000
+      #   )
+      def postgres_delete_in_batches(waterline)
+        table = item_class.arel_table
+        condition = table[:created_at].lt(waterline)
+        subquery = table
+          .project(table[:id])
+          .where(condition)
+          .take(BATCH_SIZE)
+
+        delete_statement = Arel::Nodes::DeleteStatement.new
+        delete_statement.relation = table
+        delete_statement.wheres = [table[:id].in(subquery)]
+
         loop do
-          ids = Outbox.database_switcher.use_slave do
-            item_class.where("created_at < ?", waterline).limit(BATCH_SIZE).ids
-          end
-          break if ids.empty?
+          deleted_count = item_class
+            .connection
+            .execute(delete_statement.to_sql)
+            .cmd_tuples
+
+          logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
+          break if deleted_count == 0
+          lock_timer.checkpoint!
+          sleep(SLEEP_TIME)
+        end
+      end
 
-          item_class.where(id: ids).delete_all
-          sleep SLEEP_TIME
+      # Deletes stale items from MySQL database in batches
+      #
+      # This method efficiently deletes items older than the given waterline
+      # using MySQL's built-in LIMIT clause for DELETE statements.
+      #
+      # The main difference from the PostgreSQL method is that MySQL allows
+      # direct use of LIMIT in DELETE statements, simplifying the query.
+      # This approach doesn't require a subquery, making it more straightforward.
+      #
+      # Example SQL generated for deletion:
+      #   DELETE FROM `items`
+      #   WHERE `items`.`created_at` < '2023-05-01 00:00:00'
+      #   LIMIT 1000
+      def mysql_delete_in_batches(waterline)
+        loop do
+          deleted_count = item_class
+            .where("created_at < ?", waterline)
+            .limit(BATCH_SIZE)
+            .delete_all
+
+          logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
+          break if deleted_count == 0
+          lock_timer.checkpoint!
+          sleep(SLEEP_TIME)
         end
+      end
 
-        logger.log_info("Successfully deleted #{box_type} items for #{box_name} older than #{waterline}")
+      def database_type
+        adapter_name = item_class.connection.adapter_name.downcase
+        case adapter_name
+        when "postgresql"
+          :postgresql
+        when "mysql2"
+          :mysql
+        else
+          :unknown
+        end
       end
     end
   end

data/lib/sbmt/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module Outbox
-    VERSION = "6.10.2"
+    VERSION = "6.10.3"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-outbox
 version: !ruby/object:Gem::Version
-  version: 6.10.2
+  version: 6.10.3
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-01 00:00:00.000000000 Z
+date: 2024-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -348,56 +348,42 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.5'
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop-performance
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.11'
 - !ruby/object:Gem::Dependency
   name: standard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.12'
 - !ruby/object:Gem::Dependency
   name: schked
   requirement: !ruby/object:Gem::Requirement
@@ -640,7 +626,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.5.21
 signing_key:
 specification_version: 4
 summary: Outbox service