sbmt-outbox 6.15.0 → 6.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89c324ebe083d7dece71ae19ee9011a8da91843b31997ddbde85722e9f8209a5
-  data.tar.gz: fedd73ddecdfc54da1728bf19cdce13632e40ada5c94047a783d15a7e82370f1
+  metadata.gz: ad4860726f28975383df9bf2d522e994f80644dfe9a3d3476cf517ee90443bf4
+  data.tar.gz: 16925e325aeab7dc040a46940e34c7318f526bb39c633fb3dd7be53d187d6e96
 SHA512:
-  metadata.gz: 671f178a2b39285be8adb846ce823652cf1c6f2a68f4ce7e7e327015ea69b22858f95bd8403a637bd38c8ebd981e1423341d420bd308f2fae0d8f9bf151550cb
-  data.tar.gz: bea9bbb10ced61b77814e708528deada7058cf7afc8c90726d3d2c4068c619a1ddecb39e5caa836b9620dcfa9fda28cc20e3d4a083e8a83163f6146d9b28b00b
+  metadata.gz: c4a2fb0a085133946fa0a741545ab76f173b096d2c817deaac12511207f695e93e15c40fdda475ac747ccaf79c8bb9895c73ce5594afc7e71db7fc0a4427ebb4
+  data.tar.gz: 6baa8c4f01d13859ba3879de9c8f9a749251331045b0df35aee4a0dc52660e136a46653db416e5d9eaf78711864a79da6e3f788711b5474b46be1f62cb505d15

data/README.md

@@ -273,6 +273,7 @@ default: &default
       delivered_min_retention_period: PT1H #optional, default: PT1H, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
       deletion_batch_size: 1_000 #optional, default: 1_000
       deletion_sleep_time: 0.5 #optional, default: 0.5
+      deletion_time_window: PT4H #optional, default: PT4H, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
       max_retries: 3 # default 0, the number of retries before the item will be marked as failed
       strict_order: false # optional, default
       transports: # transports section
@@ -353,6 +354,7 @@ inbox_items: # inbox items section
     delivered_min_retention_period: PT1H #optional, default: PT1H, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
     deletion_batch_size: 1_000 #optional, default: 1_000
     deletion_sleep_time: 0.5 #optional, default: 0.5
+    deletion_time_window: PT4H #optional, default: PT4H, for statuses: delivered, retention period for delivered items, https://en.wikipedia.org/wiki/ISO_8601#Durations
     max_retries: 3 # default 0, the number of retries before the item will be marked as failed
     transports: # transports section
       import_order: # underscored transport class name
@@ -493,6 +495,7 @@ You can wrap item processing within middlewares. There are three types:
 - server middlewares – triggered inside a daemon; divided into two types:
   - batch middlewares – executed alongside a batch being fetched from the database
   - item middlewares – execute alongside an item during processing
+  - polling middlewares - execute with element during pooling
 
 The order of execution depends on the order specified in the outbox configuration:
 
@@ -580,6 +583,25 @@ class MyItemMiddleware
 end
 ```
 
+Example of an polling middleware:
+
+```ruby
+# config/initializers/outbox.rb
+Rails.application.config.outbox.tap do |config|
+  config.polling_item_middlewares.push(
+    'MyItemMiddleware'
+  )
+end
+
+# my_create_polling_middleware.rb
+class MyPollingItemMiddleware
+  def call(item)
+    # your code
+    yield
+    # your code
+  end
+end
+
 ## Tracing
 
 The gem is optionally integrated with OpenTelemetry. If your main application has `opentelemetry-*` gems, the tracing will be configured automatically.

data/app/jobs/sbmt/outbox/base_delete_stale_items_job.rb

@@ -102,45 +102,50 @@ module Sbmt
       #     SELECT "items"."id"
       #     FROM "items"
       #     WHERE (
-      #       "items"."status" = 1 AND "items"."created_at" < '2023-05-01 00:00:00'
+      #       "items"."status" IN (2) AND "items"."created_at" BETWEEN "2025-01-29 12:18:32.917836" AND "2025-01-29 12:18:32.927596" LIMIT 1000
       #     )
-      #     LIMIT 1000
       #   )
       def postgres_delete_in_batches(waterline_failed, waterline_delivered)
-        table = item_class.arel_table
-
         status_delivered = item_class.statuses[:delivered]
         status_failed_discarded = item_class.statuses.values_at(:failed, :discarded)
 
-        delete_items_in_batches(table, table[:status].eq(status_delivered).and(table[:created_at].lt(waterline_delivered)))
-        delete_items_in_batches(table, table[:status].in(status_failed_discarded).and(table[:created_at].lt(waterline_failed)))
+        delete_items_in_batches_with_between(waterline_delivered, status_delivered)
+        delete_items_in_batches_with_between(waterline_failed, status_failed_discarded)
       end
 
-      def delete_items_in_batches(table, condition)
-        subquery = table
-          .project(table[:id])
-          .where(condition)
-          .take(item_class.config.deletion_batch_size)
-
-        delete_statement = Arel::Nodes::DeleteStatement.new
-        delete_statement.relation = table
-        delete_statement.wheres = [table[:id].in(subquery)]
+      def delete_items_in_batches_with_between(waterline, statuses)
+        table = item_class.arel_table
+        batch_size = item_class.config.deletion_batch_size
+        time_window = item_class.config.deletion_time_window
+        min_date = item_class.where(table[:status].in(statuses)).minimum(:created_at)
         deleted_count = nil
 
-        loop do
-          track_deleted_latency do
-            deleted_count = item_class
-              .connection
-              .execute(delete_statement.to_sql)
-              .cmd_tuples
-          end
+        while min_date && min_date < waterline
+          max_date = [min_date + time_window, waterline].min
+
+          loop do
+            subquery = table
+              .project(table[:id])
+              .where(table[:status].in(statuses))
+              .where(table[:created_at].between(min_date..max_date))
+              .take(batch_size)
+
+            delete_statement = Arel::Nodes::DeleteStatement.new
+            delete_statement.relation = table
+            delete_statement.wheres = [table[:id].in(subquery)]
+
+            track_deleted_latency do
+              deleted_count = item_class.connection.execute(delete_statement.to_sql).cmd_tuples
+            end
 
-          track_deleted_counter(deleted_count)
+            track_deleted_counter(deleted_count)
 
-          logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
-          break if deleted_count == 0
-          lock_timer.checkpoint!
-          sleep(item_class.config.deletion_sleep_time)
+            logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} between #{min_date} and #{max_date}")
+            break if deleted_count < batch_size
+            lock_timer.checkpoint!
+            sleep(item_class.config.deletion_sleep_time) if deleted_count > 0
+          end
+          min_date = max_date
         end
       end
 
@@ -154,37 +159,43 @@ module Sbmt
       # This approach doesn't require a subquery, making it more straightforward.
       #
       # Example SQL generated for deletion:
-      #   DELETE FROM `items`
+      #   DELETE FROM "items"
       #   WHERE (
-      #     `items`.`status` = 1 AND `items`.`created_at` < '2023-05-01 00:00:00'
+      #     "items"."status" IN (2) AND "items"."created_at" BETWEEN "2024-12-29 18:34:25.369234" AND "2024-12-29 22:34:25.369234" LIMIT 1000
       #   )
-      #   LIMIT 1000
       def mysql_delete_in_batches(waterline_failed, waterline_delivered)
         status_delivered = item_class.statuses[:delivered]
         status_failed_discarded = [item_class.statuses.values_at(:failed, :discarded)]
 
-        delete_items_in_batches_mysql(
-          item_class.where(status: status_delivered, created_at: ...waterline_delivered)
-        )
-        delete_items_in_batches_mysql(
-          item_class.where(status: status_failed_discarded).where(created_at: ...waterline_failed)
-        )
+        delete_items_in_batches_with_between_mysql(waterline_delivered, status_delivered)
+        delete_items_in_batches_with_between_mysql(waterline_failed, status_failed_discarded)
       end
 
-      def delete_items_in_batches_mysql(query)
+      def delete_items_in_batches_with_between_mysql(waterline, statuses)
+        batch_size = item_class.config.deletion_batch_size
+        time_window = item_class.config.deletion_time_window
+        min_date = item_class.where(status: statuses).minimum(:created_at)
         deleted_count = nil
 
-        loop do
-          track_deleted_latency do
-            deleted_count = query.limit(item_class.config.deletion_batch_size).delete_all
-          end
+        while min_date && min_date < waterline
+          max_date = [min_date + time_window, waterline].min
+
+          loop do
+            track_deleted_latency do
+              deleted_count = item_class
+                .where(status: statuses, created_at: min_date..max_date)
+                .limit(batch_size)
+                .delete_all
+            end
 
-          track_deleted_counter(deleted_count)
+            track_deleted_counter(deleted_count)
 
-          logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} items")
-          break if deleted_count == 0
-          lock_timer.checkpoint!
-          sleep(item_class.config.deletion_sleep_time)
+            logger.log_info("Deleted #{deleted_count} #{box_type} items for #{box_name} between #{min_date} and #{max_date}")
+            break if deleted_count < batch_size
+            lock_timer.checkpoint!
+            sleep(item_class.config.deletion_sleep_time) if deleted_count > 0
+          end
+          min_date = max_date
         end
       end
 

data/app/models/sbmt/outbox/base_item_config.rb

@@ -60,6 +60,10 @@ module Sbmt
         @delivered_min_retention_period ||= ActiveSupport::Duration.parse(options[:delivered_min_retention_period] || "PT1H")
       end
 
+      def deletion_time_window
+        @deletion_time_window ||= ActiveSupport::Duration.parse(options[:deletion_time_window] || "PT4H")
+      end
+
       def max_retries
         @max_retries ||= (options[:max_retries] || 0).to_i
       end

data/lib/sbmt/outbox/engine.rb

@@ -64,6 +64,7 @@ module Sbmt
         c.item_process_middlewares = []
         c.create_item_middlewares = []
         c.create_batch_middlewares = []
+        c.polling_item_middlewares = []
 
         if defined?(::Sentry)
           c.batch_process_middlewares.push("Sbmt::Outbox::Middleware::Sentry::TracingBatchProcessMiddleware")

data/lib/sbmt/outbox/v2/poller.rb

@@ -10,9 +10,9 @@ module Sbmt
   module Outbox
     module V2
       class Poller < BoxProcessor
-        delegate :poller_config, :logger, to: "Sbmt::Outbox"
+        delegate :poller_config, :polling_item_middlewares, :logger, to: "Sbmt::Outbox"
         delegate :box_worker, to: "Yabeda"
-        attr_reader :partitions_count, :lock_timeout, :regular_items_batch_size, :retryable_items_batch_size, :max_buffer_size, :max_batch_size, :throttler
+        attr_reader :partitions_count, :lock_timeout, :regular_items_batch_size, :retryable_items_batch_size, :max_buffer_size, :max_batch_size, :throttler, :middleware_builder
 
         def initialize(
           boxes,
@@ -35,6 +35,7 @@ module Sbmt
           super(boxes: boxes, threads_count: threads_count || poller_config.threads_count, name: "poller", redis: redis)
 
           @throttler = PollThrottler.build(throttler_tactic || poller_config.tactic || "default", self.redis, poller_config)
+          @middleware_builder = Middleware::Builder.new(polling_item_middlewares)
         end
 
         def throttle(worker_number, poll_task, result)
@@ -42,7 +43,7 @@ module Sbmt
         end
 
         def process_task(_worker_number, task)
-          poll(task)
+          middleware_builder.call(task) { poll(task) }
         end
 
         private

data/lib/sbmt/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module Outbox
-    VERSION = "6.15.0"
+    VERSION = "6.17.0"
   end
 end

data/lib/sbmt/outbox.rb

@@ -170,5 +170,9 @@ module Sbmt
     def create_batch_middlewares
       @create_batch_middlewares ||= config.create_batch_middlewares.map(&:constantize)
     end
+
+    def polling_item_middlewares
+      @polling_item_middlewares ||= config.polling_item_middlewares.map(&:constantize)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-outbox
 version: !ruby/object:Gem::Version
-  version: 6.15.0
+  version: 6.17.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-01-23 00:00:00.000000000 Z
+date: 2025-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -629,7 +629,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.5.21
 signing_key:
 specification_version: 4
 summary: Outbox service