sbmt-outbox 6.14.0 → 6.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2616bf630410714a263f3eea95a7c98af5568018386d0792b8af8ac8de9572b2
-  data.tar.gz: 711b190d7c2d45ae1e66f07e60d9742fea6d2f9d0eae07431756da99bbc71e83
+  metadata.gz: 86044914d6babd961a882ecfd9394875e0b67f74a2b92f44db48480bb5096448
+  data.tar.gz: 3ae09e5801c2dcacc9096bae5c2428f2c57445394dd5a3d036c58b1fa63121d2
 SHA512:
-  metadata.gz: 8af881dffaa9802491a6764f471a6fedaac2ea5766e1f2c6749a1b6f5523f74f5f2239025b6d1e48fd985243556569b1670ed80d5aeb92f0a40ae67c2254a777
-  data.tar.gz: 543ad28b03629931934332ef0f62a6190a90159c0aa45ef3800c09472a824bfd474f270a5c9d27df2879ace34d2f03bbc92206381476568313254cf8db34a70a
+  metadata.gz: 9f180114bce2540c91f6c942195fa59c51bcfea67bd984ee5b7db19fd95653933492ce30f7890833e879469b2bb08ca5bcfad2b5a8efa8221eae904b2839a42f
+  data.tar.gz: e9aeadbc87d76a0f86c7b7c9ddad157c45851e3941901bf4ff249c116a7d32827645157ed644709ba59756569af0d9637e0b66c0c9b0c10937ba6b5878ac740d

data/README.md

@@ -439,6 +439,42 @@ outbox_items:
     partition_strategy: hash
 ```
 
+## Rake tasks
+
+```shell
+rake outbox:delete_items
+rake outbox:update_status_items
+```
+
+Example run:
+```shell
+rake outbox:delete_items[OutboxItem,1] # Mandatory parameters box class and status
+rake outbox:update_status_items[OutboxItem,0,3] # Mandatory parameters box class, current status and new status
+
+```
+
+Both tasks have optional parameters:
+```ruby
+- start_time # boxes are younger than the specified time, by default nil, time is specified in the format "2025-01-05T23:59:59"
+- end_time # boxes are older than the specified time, by default 6.hours.ago, time is specified in the format "2025-01-05T23:59:59"
+- batch_size # batch size, by default 1_000
+- sleep_time # sleep time between batches, by default 0.5
+```
+
+Example with optional parameters:
+  - format optional parameters:
+  ```shell
+    rake outbox:delete_items[klass_name,status,start_time,end_time,batch_size,sleep_time]
+
+    rake outbox:update_status_items[klass_name,status,new_status,start_time,end_time,batch_size,sleep_time]
+  ```
+  - example:
+  ```shell
+    rake outbox:delete_items[OutboxItem,1,"2025-01-05T23:59:59","2025-01-05T00:00:00",10_000,5]
+
+    rake outbox:update_status_items[OutboxItem,0,3,"2025-01-05T23:59:59","2025-01-05T00:00:00",10_000,5]
+  ```
+
 ## Concurrency
 
 The worker process consists of a poller and a processor, each of which has its own thread pool.
@@ -457,6 +493,7 @@ You can wrap item processing within middlewares. There are three types:
 - server middlewares – triggered inside a daemon; divided into two types:
   - batch middlewares – executed alongside a batch being fetched from the database
   - item middlewares – execute alongside an item during processing
+  - polling middlewares - execute with element during pooling
 
 The order of execution depends on the order specified in the outbox configuration:
 
@@ -544,6 +581,25 @@ class MyItemMiddleware
 end
 ```
 
+Example of an polling middleware:
+
+```ruby
+# config/initializers/outbox.rb
+Rails.application.config.outbox.tap do |config|
+  config.polling_item_middlewares.push(
+    'MyItemMiddleware'
+  )
+end
+
+# my_create_polling_middleware.rb
+class MyPollingItemMiddleware
+  def call(item)
+    # your code
+    yield
+    # your code
+  end
+end
+
 ## Tracing
 
 The gem is optionally integrated with OpenTelemetry. If your main application has `opentelemetry-*` gems, the tracing will be configured automatically.

data/lib/sbmt/outbox/engine.rb

@@ -64,6 +64,7 @@ module Sbmt
         c.item_process_middlewares = []
         c.create_item_middlewares = []
         c.create_batch_middlewares = []
+        c.polling_item_middlewares = []
 
         if defined?(::Sentry)
           c.batch_process_middlewares.push("Sbmt::Outbox::Middleware::Sentry::TracingBatchProcessMiddleware")
@@ -79,6 +80,8 @@ module Sbmt
       rake_tasks do
         load "sbmt/outbox/tasks/retry_failed_items.rake"
         load "sbmt/outbox/tasks/delete_failed_items.rake"
+        load "sbmt/outbox/tasks/delete_items.rake"
+        load "sbmt/outbox/tasks/update_status_items.rake"
       end
     end
   end

data/lib/sbmt/outbox/tasks/delete_items.rake

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+namespace :outbox do
+  desc "Delete outbox/inbox items"
+  task :delete_items, [:klass_name, :status, :start_time, :end_time, :batch_size, :sleep_time] => :environment do |_, args|
+    args.with_defaults(start_time: nil, end_time: 6.hours.ago, batch_size: 1000, sleep_time: 0.5)
+
+    klass_name = args[:klass_name]
+    status = args[:status]
+    start_time = args[:start_time]
+    end_time = args[:end_time]
+    batch_size = args[:batch_size]
+    sleep_time = args[:sleep_time]
+
+    unless klass_name && status
+      raise "Error: Class and status must be specified. Example: rake outbox:delete_items[OutboxItem,1]"
+    end
+
+    klass_name = klass_name.constantize
+    query = klass_name.where(status: status)
+
+    if start_time && end_time
+      query = query.where(created_at: start_time..end_time)
+    elsif start_time
+      query = query.where(created_at: start_time..)
+    elsif end_time
+      query = query.where(created_at: ..end_time)
+    end
+
+    total_deleted = 0
+    query.in_batches(of: batch_size) do |batch|
+      deleted_count = batch.delete_all
+
+      Rails.logger.info("Batch items deleted: #{deleted_count}")
+
+      total_deleted += deleted_count
+
+      sleep sleep_time
+    end
+
+    Rails.logger.info("Total items deleted: #{total_deleted}")
+  end
+end

data/lib/sbmt/outbox/tasks/update_status_items.rake

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+namespace :outbox do
+  desc "Update status of outbox/inbox items"
+  task :update_status_items, [:klass_name, :status, :new_status, :start_time, :end_time, :batch_size, :sleep_time] => :environment do |_, args|
+    args.with_defaults(start_time: nil, end_time: 6.hours.ago, batch_size: 1000, sleep_time: 0.5)
+
+    klass_name = args[:klass_name]
+    status = args[:status]
+    new_status = args[:new_status]
+    start_time = args[:start_time]
+    end_time = args[:end_time]
+    batch_size = args[:batch_size]
+    sleep_time = args[:sleep_time]
+
+    unless klass_name && status && new_status
+      raise "Error: Class, current status, and new status must be specified. Example: rake outbox:update_status_items[OutboxItem,0,3]"
+    end
+
+    klass_name = klass_name.constantize
+    query = klass_name.where(status: status)
+
+    if start_time && end_time
+      query = query.where(created_at: start_time..end_time)
+    elsif start_time
+      query = query.where(created_at: start_time..)
+    elsif end_time
+      query = query.where(created_at: ..end_time)
+    end
+
+    total_updated = 0
+    query.in_batches(of: batch_size) do |batch|
+      updated_count = batch.update_all(status: new_status)
+
+      Rails.logger.info("Batch items updated: #{updated_count}")
+
+      total_updated += updated_count
+      sleep sleep_time
+    end
+
+    Rails.logger.info("Total items updated: #{total_updated}")
+  end
+end

data/lib/sbmt/outbox/v2/poller.rb

@@ -10,9 +10,9 @@ module Sbmt
   module Outbox
     module V2
       class Poller < BoxProcessor
-        delegate :poller_config, :logger, to: "Sbmt::Outbox"
+        delegate :poller_config, :polling_item_middlewares, :logger, to: "Sbmt::Outbox"
         delegate :box_worker, to: "Yabeda"
-        attr_reader :partitions_count, :lock_timeout, :regular_items_batch_size, :retryable_items_batch_size, :max_buffer_size, :max_batch_size, :throttler
+        attr_reader :partitions_count, :lock_timeout, :regular_items_batch_size, :retryable_items_batch_size, :max_buffer_size, :max_batch_size, :throttler, :middleware_builder
 
         def initialize(
           boxes,
@@ -35,6 +35,7 @@ module Sbmt
           super(boxes: boxes, threads_count: threads_count || poller_config.threads_count, name: "poller", redis: redis)
 
           @throttler = PollThrottler.build(throttler_tactic || poller_config.tactic || "default", self.redis, poller_config)
+          @middleware_builder = Middleware::Builder.new(polling_item_middlewares)
         end
 
         def throttle(worker_number, poll_task, result)
@@ -42,7 +43,7 @@ module Sbmt
         end
 
         def process_task(_worker_number, task)
-          poll(task)
+          middleware_builder.call(task) { poll(task) }
         end
 
         private

data/lib/sbmt/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module Outbox
-    VERSION = "6.14.0"
+    VERSION = "6.16.0"
   end
 end

data/lib/sbmt/outbox.rb

@@ -170,5 +170,9 @@ module Sbmt
     def create_batch_middlewares
       @create_batch_middlewares ||= config.create_batch_middlewares.map(&:constantize)
     end
+
+    def polling_item_middlewares
+      @polling_item_middlewares ||= config.polling_item_middlewares.map(&:constantize)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-outbox
 version: !ruby/object:Gem::Version
-  version: 6.14.0
+  version: 6.16.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-01-20 00:00:00.000000000 Z
+date: 2025-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -582,7 +582,9 @@ files:
 - lib/sbmt/outbox/probes/probe.rb
 - lib/sbmt/outbox/redis_client_factory.rb
 - lib/sbmt/outbox/tasks/delete_failed_items.rake
+- lib/sbmt/outbox/tasks/delete_items.rake
 - lib/sbmt/outbox/tasks/retry_failed_items.rake
+- lib/sbmt/outbox/tasks/update_status_items.rake
 - lib/sbmt/outbox/v1/thread_pool.rb
 - lib/sbmt/outbox/v1/throttler.rb
 - lib/sbmt/outbox/v1/worker.rb