sbmt-outbox 6.0.1 → 6.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7ed66a6f328704ae2838f6c0d4b8f6c42788d546d227ef3035d6774bd84e38c
-  data.tar.gz: 5e7eac918594e2268757d25aba26d18da783d50d31f5cf61e03bb921c2390e6a
+  metadata.gz: 7a8ef5d8d2289f5e84b0218df4d0ae6ebf4c3887891f4a694c53f5b3fb2a617c
+  data.tar.gz: 5df6474dd32187affba9667f86e3638dfb51349519626e247e0928d9b44bdb4b
 SHA512:
-  metadata.gz: af1f6fa94feb668716256f9a9e4c462585da9341da9823b2d960c5af2e6b392fc4736eb3cfbfb80cc8a255399ba80c01b9b6cdf6f28644dbdbd1538e8342dd94
-  data.tar.gz: 35957d9cc3e507b708f6b74cb1da20a687dfd0b251321a03f22245e9de6ebd73ea40cfa5620ffe4a7b6a2822b8411582385ae5681b8fe809165ef2da88f11882
+  metadata.gz: 4635896765ebef22224ea23123cbcd6d47facd6c88793f5ecd419cab54fa7afc281527cd43a1fe7466a098c53008d6d692903a85e1d23cd53080713f770ae010
+  data.tar.gz: 0b632a652cf286f792fe57dfa373409804782ccc5a8bfd6f7859edde9a6067de8ebdd9f5b653e84e61840e1c252607c2fd943746bd7ef409a415707a8c5a0e98

data/README.md

@@ -146,7 +146,6 @@ default: &default
     my_outbox_item: # underscored model class name
       owner: my_outbox_item_team # optional, used in Yabeda metrics
       retention: P1W # retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
-      partition_size: 2 # default 1, partitions count
       max_retries: 3 # default 0, the number of retries before the item will be marked as failed
       transports: # transports section
         produce_message: # underscored transport class name
@@ -287,7 +286,6 @@ inbox_items: # inbox items section
   my_inbox_item: # underscored model class name
     owner: my_inbox_item_team # optional, used in Yabeda metrics
     retention: P1W # retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
-    partition_size: 2 # default 1, partitions count
     max_retries: 3 # default 0, the number of retries before the item will be marked as failed
     transports: # transports section
       import_order: # underscored transport class name
@@ -376,9 +374,14 @@ outbox_items:
 
 ## Concurrency
 
-The Outbox executable CLI uses Ruby threads for concurrent processing of messages pulled from the database table. The number of threads is configured using the `--concurrency` option. By default, it's 10 unless a parameter is provided. You can run multiple daemons at once. The number of partitions for each outbox item class is set by the `partition_size` configuration option. Each partition is processed one at a time by a daemon. Each batch of partitions serves several buckets. A bucket is a number in a row in the `bucket` column generated by the partitioning strategy based on the `event_key` column when a message was committed to the database within the range of zero to `bucket_size`. Therefore, each outbox table has many partitions that contain several buckets. Note that you must not have `partition_size` greater than `bucket_size.` This architecture was designed to allow the daemons to scale without stopping the entire system in order to avoid mixing messages chronologically.So, if you need more partitions, just stop the daemons, set `partition_size` correctly, and start them again.
-
-**Example:** Suppose you have a Kafka topic with 18 partitions. The `bucket_size` is 256. If we expect a slow payload generation, we can set `partition_size` to 16. Therefore, we should run 4 daemons with 4 threads each in order to maximize utilization of the partitions.
+The worker process consists of a poller and a processor, each of which has its own thread pool.
+The poller is responsible for fetching messages ready for processing from the database table.
+The processor, in turn, is used for their consistent processing (while preserving the order of messages and the partitioning key). 
+Each bunch of buckets (i.e. buckets partition) is consistently fetched by poller one at a time. Each bucket is processed one at a time by a processor.
+A bucket is a number in a row in the `bucket` column generated by the partitioning strategy based on the `event_key` column when a message was committed to the database within the range of zero to `bucket_size`.
+The number of bucket partitions, which poller uses is 6 by default. The number of poller threads is 2 by default and is not intended for customization.
+The default number of processor threads is 4 and can be configured with the --concurrency option, thereby allowing you to customize message processing performance.
+This architecture was designed to allow the daemons to scale without stopping the entire system in order to avoid mixing messages chronologically.
 
 ### Middlewares
 

data/config/initializers/yabeda.rb

@@ -41,7 +41,7 @@ Yabeda.configure do
     histogram :process_latency,
       tags: %i[type name partition owner],
       unit: :seconds,
-      buckets: [1, 2.5, 5, 10, 15, 30, 45, 60, 90, 120, 180, 240, 300, 600, 1200].freeze,
+      buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 300].freeze,
       comment: "A histogram outbox process latency"
   end
 
@@ -50,7 +50,7 @@ Yabeda.configure do
     default_tag(:worker_name, "worker")
 
     counter :job_counter,
-      tags: %i[type name partition state],
+      tags: %i[type name partition state owner],
       comment: "The total number of processed jobs"
 
     counter :job_timeout_counter,
@@ -65,13 +65,13 @@ Yabeda.configure do
       comment: "A histogram of the job execution time",
       unit: :seconds,
       tags: %i[type name partition],
-      buckets: [0.5, 1, 2.5, 5, 10, 15, 30, 45, 60, 90, 120, 180, 240, 300, 600]
+      buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 300]
 
     histogram :item_execution_runtime,
       comment: "A histogram of the item execution time",
       unit: :seconds,
       tags: %i[type name partition],
-      buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300]
+      buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 300]
 
     counter :batches_per_poll_counter,
       tags: %i[type name partition],
@@ -93,6 +93,6 @@ Yabeda.configure do
       comment: "A histogram of the poll throttling time",
       unit: :seconds,
       tags: %i[type name partition throttler],
-      buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300]
+      buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 300]
   end
 end

data/lib/sbmt/outbox/v1/worker.rb

@@ -127,7 +127,8 @@ module Sbmt
                 yabeda_labels: {
                   type: item_class.box_type,
                   name: item_class.box_name,
-                  partition: partition
+                  partition: partition,
+                  owner: item_class.owner
                 },
                 resource_key: resource_key,
                 resource_path: "sbmt/outbox/worker/#{resource_key}"

data/lib/sbmt/outbox/v2/tasks/base.rb

@@ -7,6 +7,8 @@ module Sbmt
         class Base
           attr_reader :item_class, :worker_name, :worker_version, :log_tags, :yabeda_labels
 
+          delegate :owner, to: :item_class
+
           def initialize(item_class:, worker_name:, worker_version: 2)
             @item_class = item_class
             @worker_name = worker_name
@@ -22,6 +24,7 @@ module Sbmt
             @yabeda_labels = {
               type: item_class.box_type,
               name: metric_safe(item_class.box_name),
+              owner: owner,
               worker_version: 2,
               worker_name: worker_name
             }

data/lib/sbmt/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module Outbox
-    VERSION = "6.0.1"
+    VERSION = "6.2.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-outbox
 version: !ruby/object:Gem::Version
-  version: 6.0.1
+  version: 6.2.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-03-29 00:00:00.000000000 Z
+date: 2024-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool