sbmt-outbox 6.0.1 → 6.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d7ed66a6f328704ae2838f6c0d4b8f6c42788d546d227ef3035d6774bd84e38c
4
- data.tar.gz: 5e7eac918594e2268757d25aba26d18da783d50d31f5cf61e03bb921c2390e6a
3
+ metadata.gz: 7a8ef5d8d2289f5e84b0218df4d0ae6ebf4c3887891f4a694c53f5b3fb2a617c
4
+ data.tar.gz: 5df6474dd32187affba9667f86e3638dfb51349519626e247e0928d9b44bdb4b
5
5
  SHA512:
6
- metadata.gz: af1f6fa94feb668716256f9a9e4c462585da9341da9823b2d960c5af2e6b392fc4736eb3cfbfb80cc8a255399ba80c01b9b6cdf6f28644dbdbd1538e8342dd94
7
- data.tar.gz: 35957d9cc3e507b708f6b74cb1da20a687dfd0b251321a03f22245e9de6ebd73ea40cfa5620ffe4a7b6a2822b8411582385ae5681b8fe809165ef2da88f11882
6
+ metadata.gz: 4635896765ebef22224ea23123cbcd6d47facd6c88793f5ecd419cab54fa7afc281527cd43a1fe7466a098c53008d6d692903a85e1d23cd53080713f770ae010
7
+ data.tar.gz: 0b632a652cf286f792fe57dfa373409804782ccc5a8bfd6f7859edde9a6067de8ebdd9f5b653e84e61840e1c252607c2fd943746bd7ef409a415707a8c5a0e98
data/README.md CHANGED
@@ -146,7 +146,6 @@ default: &default
146
146
  my_outbox_item: # underscored model class name
147
147
  owner: my_outbox_item_team # optional, used in Yabeda metrics
148
148
  retention: P1W # retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
149
- partition_size: 2 # default 1, partitions count
150
149
  max_retries: 3 # default 0, the number of retries before the item will be marked as failed
151
150
  transports: # transports section
152
151
  produce_message: # underscored transport class name
@@ -287,7 +286,6 @@ inbox_items: # inbox items section
287
286
  my_inbox_item: # underscored model class name
288
287
  owner: my_inbox_item_team # optional, used in Yabeda metrics
289
288
  retention: P1W # retention period, https://en.wikipedia.org/wiki/ISO_8601#Durations
290
- partition_size: 2 # default 1, partitions count
291
289
  max_retries: 3 # default 0, the number of retries before the item will be marked as failed
292
290
  transports: # transports section
293
291
  import_order: # underscored transport class name
@@ -376,9 +374,14 @@ outbox_items:
376
374
 
377
375
  ## Concurrency
378
376
 
379
- The Outbox executable CLI uses Ruby threads for concurrent processing of messages pulled from the database table. The number of threads is configured using the `--concurrency` option. By default, it's 10 unless a parameter is provided. You can run multiple daemons at once. The number of partitions for each outbox item class is set by the `partition_size` configuration option. Each partition is processed one at a time by a daemon. Each batch of partitions serves several buckets. A bucket is a number in a row in the `bucket` column generated by the partitioning strategy based on the `event_key` column when a message was committed to the database within the range of zero to `bucket_size`. Therefore, each outbox table has many partitions that contain several buckets. Note that you must not have `partition_size` greater than `bucket_size.` This architecture was designed to allow the daemons to scale without stopping the entire system in order to avoid mixing messages chronologically.So, if you need more partitions, just stop the daemons, set `partition_size` correctly, and start them again.
380
-
381
- **Example:** Suppose you have a Kafka topic with 18 partitions. The `bucket_size` is 256. If we expect a slow payload generation, we can set `partition_size` to 16. Therefore, we should run 4 daemons with 4 threads each in order to maximize utilization of the partitions.
377
+ The worker process consists of a poller and a processor, each of which has its own thread pool.
378
+ The poller is responsible for fetching messages ready for processing from the database table.
379
+ The processor, in turn, is used for their consistent processing (while preserving the order of messages and the partitioning key).
380
+ Each bunch of buckets (i.e. buckets partition) is consistently fetched by poller one at a time. Each bucket is processed one at a time by a processor.
381
+ A bucket is a number in a row in the `bucket` column generated by the partitioning strategy based on the `event_key` column when a message was committed to the database within the range of zero to `bucket_size`.
382
+ The number of bucket partitions, which poller uses is 6 by default. The number of poller threads is 2 by default and is not intended for customization.
383
+ The default number of processor threads is 4 and can be configured with the --concurrency option, thereby allowing you to customize message processing performance.
384
+ This architecture was designed to allow the daemons to scale without stopping the entire system in order to avoid mixing messages chronologically.
382
385
 
383
386
  ### Middlewares
384
387
 
@@ -41,7 +41,7 @@ Yabeda.configure do
41
41
  histogram :process_latency,
42
42
  tags: %i[type name partition owner],
43
43
  unit: :seconds,
44
- buckets: [1, 2.5, 5, 10, 15, 30, 45, 60, 90, 120, 180, 240, 300, 600, 1200].freeze,
44
+ buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 300].freeze,
45
45
  comment: "A histogram outbox process latency"
46
46
  end
47
47
 
@@ -50,7 +50,7 @@ Yabeda.configure do
50
50
  default_tag(:worker_name, "worker")
51
51
 
52
52
  counter :job_counter,
53
- tags: %i[type name partition state],
53
+ tags: %i[type name partition state owner],
54
54
  comment: "The total number of processed jobs"
55
55
 
56
56
  counter :job_timeout_counter,
@@ -65,13 +65,13 @@ Yabeda.configure do
65
65
  comment: "A histogram of the job execution time",
66
66
  unit: :seconds,
67
67
  tags: %i[type name partition],
68
- buckets: [0.5, 1, 2.5, 5, 10, 15, 30, 45, 60, 90, 120, 180, 240, 300, 600]
68
+ buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 300]
69
69
 
70
70
  histogram :item_execution_runtime,
71
71
  comment: "A histogram of the item execution time",
72
72
  unit: :seconds,
73
73
  tags: %i[type name partition],
74
- buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300]
74
+ buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 300]
75
75
 
76
76
  counter :batches_per_poll_counter,
77
77
  tags: %i[type name partition],
@@ -93,6 +93,6 @@ Yabeda.configure do
93
93
  comment: "A histogram of the poll throttling time",
94
94
  unit: :seconds,
95
95
  tags: %i[type name partition throttler],
96
- buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300]
96
+ buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 300]
97
97
  end
98
98
  end
@@ -127,7 +127,8 @@ module Sbmt
127
127
  yabeda_labels: {
128
128
  type: item_class.box_type,
129
129
  name: item_class.box_name,
130
- partition: partition
130
+ partition: partition,
131
+ owner: item_class.owner
131
132
  },
132
133
  resource_key: resource_key,
133
134
  resource_path: "sbmt/outbox/worker/#{resource_key}"
@@ -7,6 +7,8 @@ module Sbmt
7
7
  class Base
8
8
  attr_reader :item_class, :worker_name, :worker_version, :log_tags, :yabeda_labels
9
9
 
10
+ delegate :owner, to: :item_class
11
+
10
12
  def initialize(item_class:, worker_name:, worker_version: 2)
11
13
  @item_class = item_class
12
14
  @worker_name = worker_name
@@ -22,6 +24,7 @@ module Sbmt
22
24
  @yabeda_labels = {
23
25
  type: item_class.box_type,
24
26
  name: metric_safe(item_class.box_name),
27
+ owner: owner,
25
28
  worker_version: 2,
26
29
  worker_name: worker_name
27
30
  }
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Sbmt
4
4
  module Outbox
5
- VERSION = "6.0.1"
5
+ VERSION = "6.2.0"
6
6
  end
7
7
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sbmt-outbox
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.0.1
4
+ version: 6.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sbermarket Ruby-Platform Team
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-03-29 00:00:00.000000000 Z
11
+ date: 2024-04-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: connection_pool