sbmt-outbox 5.0.1 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +68 -9
  3. data/app/interactors/sbmt/outbox/process_item.rb +2 -1
  4. data/app/interactors/sbmt/outbox/retry_strategies/base.rb +15 -0
  5. data/app/interactors/sbmt/outbox/retry_strategies/compacted_log.rb +2 -32
  6. data/app/interactors/sbmt/outbox/retry_strategies/exponential_backoff.rb +3 -5
  7. data/app/interactors/sbmt/outbox/retry_strategies/latest_available.rb +39 -0
  8. data/app/interactors/sbmt/outbox/retry_strategies/no_delay.rb +13 -0
  9. data/app/models/sbmt/outbox/base_item.rb +9 -8
  10. data/app/models/sbmt/outbox/base_item_config.rb +23 -4
  11. data/config/initializers/yabeda.rb +32 -5
  12. data/lib/generators/helpers/migration.rb +2 -2
  13. data/lib/sbmt/outbox/cli.rb +50 -7
  14. data/lib/sbmt/outbox/engine.rb +26 -0
  15. data/lib/sbmt/outbox/logger.rb +6 -0
  16. data/lib/sbmt/outbox/v1/thread_pool.rb +110 -0
  17. data/lib/sbmt/outbox/v1/throttler.rb +54 -0
  18. data/lib/sbmt/outbox/v1/worker.rb +231 -0
  19. data/lib/sbmt/outbox/v2/box_processor.rb +148 -0
  20. data/lib/sbmt/outbox/v2/poll_throttler/base.rb +43 -0
  21. data/lib/sbmt/outbox/v2/poll_throttler/composite.rb +42 -0
  22. data/lib/sbmt/outbox/v2/poll_throttler/fixed_delay.rb +28 -0
  23. data/lib/sbmt/outbox/v2/poll_throttler/noop.rb +17 -0
  24. data/lib/sbmt/outbox/v2/poll_throttler/rate_limited.rb +24 -0
  25. data/lib/sbmt/outbox/v2/poll_throttler/redis_queue_size.rb +46 -0
  26. data/lib/sbmt/outbox/v2/poll_throttler/redis_queue_time_lag.rb +45 -0
  27. data/lib/sbmt/outbox/v2/poll_throttler.rb +49 -0
  28. data/lib/sbmt/outbox/v2/poller.rb +180 -0
  29. data/lib/sbmt/outbox/v2/processor.rb +101 -0
  30. data/lib/sbmt/outbox/v2/redis_job.rb +42 -0
  31. data/lib/sbmt/outbox/v2/tasks/base.rb +48 -0
  32. data/lib/sbmt/outbox/v2/tasks/default.rb +17 -0
  33. data/lib/sbmt/outbox/v2/tasks/poll.rb +34 -0
  34. data/lib/sbmt/outbox/v2/tasks/process.rb +31 -0
  35. data/lib/sbmt/outbox/v2/thread_pool.rb +152 -0
  36. data/lib/sbmt/outbox/v2/throttler.rb +13 -0
  37. data/lib/sbmt/outbox/v2/worker.rb +52 -0
  38. data/lib/sbmt/outbox/version.rb +1 -1
  39. data/lib/sbmt/outbox.rb +16 -2
  40. metadata +41 -5
  41. data/lib/sbmt/outbox/thread_pool.rb +0 -108
  42. data/lib/sbmt/outbox/throttler.rb +0 -52
  43. data/lib/sbmt/outbox/worker.rb +0 -233
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a240a2348f45c36bf76377a8edbc3af77cea2caf1c3240f85b68072fbb99cd32
4
- data.tar.gz: 33a71dec75089581641a2ddc1bde47e35994c3e9b5f383ace8865077e3620df1
3
+ metadata.gz: 818de79692d37138c630bf2fce625a54becf3a8fa71eafe3c5435d573dbcf665
4
+ data.tar.gz: 2516c414d14a60851d53fd9f821845179ac12cfb88176a18c1f60d8ba8e38bc9
5
5
  SHA512:
6
- metadata.gz: cd32451e9af08be2e0c5d3793b34a9c3f23997482304e9783e8dc0bab5b9c01d46e265ae4cb5ed8649429972f28bb51aa8421fdc5ce418aebf4f41a21d99594e
7
- data.tar.gz: a57430bf42f65f452bf20e6d3f468e2c750085b8fc50e558d6ad99f3d39cb317bcfd0a0b3ac13ba8c04eb167001c0be027a32de9abb64c32daa04a1b99dcb604
6
+ metadata.gz: 0e1c0f1f55ef75e5c73352f0789dc51559656f5868a64f2ca8b0c97c10b2865cd37189dc6b3a5d0346673251800ed1c652df951734a5fe77029ca87dbb0fc1a3
7
+ data.tar.gz: f0aba9307dcfac3c8e330f1a8a1e40e33439a624c4c40c34ea9223c53c25acff2540ad03c72b00a71c1e963885ecee4e6befc5e5e94d4d78e13ecf6acac4c69e
data/README.md CHANGED
@@ -23,6 +23,10 @@ And then execute:
23
23
  bundle install
24
24
  ```
25
25
 
26
+ ## Demo
27
+
28
+ Learn how to use this gem and how it works with Ruby on Rails at here https://github.com/SberMarket-Tech/outbox-example-apps
29
+
26
30
  ## Auto configuration
27
31
 
28
32
  We recommend going through the configuration and files creation process using the following Rails generators:
@@ -113,8 +117,8 @@ create_table :my_outbox_items do |t|
113
117
  end
114
118
 
115
119
  add_index :my_outbox_items, :uuid, unique: true
116
- add_index :my_outbox_items, [:status, :bucket]
117
- add_index :my_outbox_items, [:event_name, :event_key]
120
+ add_index :my_outbox_items, [:status, :bucket, :errors_count]
121
+ add_index :my_outbox_items, [:event_name, :event_key, :id]
118
122
  add_index :my_outbox_items, :created_at
119
123
  ```
120
124
 
@@ -146,7 +150,11 @@ default: &default
146
150
  max_retries: 3 # default 0, the number of retries before the item will be marked as failed
147
151
  transports: # transports section
148
152
  produce_message: # underscored transport class name
149
- topic: "my-topic-name" # default transport arguments
153
+ # transport reserved options
154
+ class: produce_message # optional; default is inferred from transport name
155
+ disposable: false # optional; default false; if true, the transport class will be instantiated only once
156
+ # ProduceMessage instance arguments
157
+ topic: "my-topic-name"
150
158
 
151
159
  development:
152
160
  <<: *default
@@ -182,9 +190,11 @@ Transports are defined as follows when `event_name` is used:
182
190
  outbox_items:
183
191
  my_outbox_item:
184
192
  transports:
193
+ # transport reserved options
185
194
  - class: produce_message
186
195
  event_name: "order_created" # event name marker
187
- topic: "order_created_topic" # some transport default argument
196
+ # ProduceMessage instance arguments
197
+ topic: "order_created_topic" # some transport argument
188
198
  - class: produce_message
189
199
  event_name: "orders_completed"
190
200
  topic: "orders_completed_topic"
@@ -212,13 +222,51 @@ Rails.application.config.outbox.tap do |config|
212
222
  x[:batch_size] = 200
213
223
  end
214
224
 
215
- # optional
225
+ # optional (worker v1: DEPRECATED)
216
226
  config.worker.tap do |worker|
217
227
  # number of batches that one thread will process per rate interval
218
228
  worker[:rate_limit] = 10
219
229
  # rate interval in seconds
220
230
  worker[:rate_interval] = 60
221
231
  end
232
+
233
+ # optional (worker v2: default)
234
+ c.poller = ActiveSupport::OrderedOptions.new.tap do |pc|
235
+ # max parallel threads (per box-item, globally)
236
+ pc.concurrency = 6
237
+ # max threads count (per worker process)
238
+ pc.threads_count = 1
239
+ # maximum processing time of the batch, after which the batch will be considered hung and processing will be aborted
240
+ pc.general_timeout = 60
241
+ # poll buffer consists of regular items (errors_count = 0, i.e. without any processing errors) and retryable items (errors_count > 0)
242
+ # max poll buffer size = regular_items_batch_size + retryable_items_batch_size
243
+ pc.regular_items_batch_size = 200
244
+ pc.retryable_items_batch_size = 100
245
+
246
+ # poll tactic: default is optimal for most cases: rate limit + redis job-queue size threshold
247
+ # poll tactic: aggressive is for high-intencity data: without rate limits + redis job-queue size threshold
248
+ # poll tactic: low-priority is for low-intencity data: rate limits + redis job-queue size threshold + + redis job-queue lag threshold
249
+ pc.tactic = "default"
250
+ # number of batches that one thread will process per rate interval
251
+ pc.rate_limit = 60
252
+ # rate interval in seconds
253
+ pc.rate_interval = 60
254
+ # mix / max redis job queue thresholds per box-item for default / aggressive / low-priority poll tactics
255
+ pc.min_queue_size = 10
256
+ pc.max_queue_size = 100
257
+ # min redis job queue time lag threshold per box-item for low-priority poll tactic (in seconds)
258
+ pc.min_queue_timelag = 5
259
+ # throttling delay for default / aggressive / low-priority poll tactics (in seconds)
260
+ pc.queue_delay = 0.1
261
+ end
262
+ c.processor = ActiveSupport::OrderedOptions.new.tap do |pc|
263
+ # max threads count (per worker process)
264
+ pc.threads_count = 4
265
+ # maximum processing time of the batch, after which the batch will be considered hung and processing will be aborted
266
+ pc.general_timeout = 120
267
+ # BRPOP delay (in seconds) for polling redis job queue per box-item
268
+ pc.brpop_delay = 2
269
+ end
222
270
  end
223
271
  ```
224
272
 
@@ -282,7 +330,7 @@ outbox_items:
282
330
  - exponential_backoff
283
331
  ```
284
332
 
285
- #### Compacted log
333
+ #### Latest available
286
334
 
287
335
  This strategy ensures idempotency. In short, if a message fails and a later message with the same event_key has already been delivered, then you most likely do not want to re-deliver the first one when it is retried.
288
336
 
@@ -293,10 +341,10 @@ outbox_items:
293
341
  ...
294
342
  retry_strategies:
295
343
  - exponential_backoff
296
- - compacted_log
344
+ - latest_available
297
345
  ```
298
346
 
299
- The exponential backoff strategy should be used in conjunction with the compact log strategy, and it should come last to minimize the number of database queries.
347
+ The exponential backoff strategy should be used in conjunction with the latest available strategy, and it should come last to minimize the number of database queries.
300
348
 
301
349
  ### Partition strategies
302
350
 
@@ -418,13 +466,24 @@ end
418
466
 
419
467
  The gem is optionally integrated with OpenTelemetry. If your main application has `opentelemetry-*` gems, the tracing will be configured automatically.
420
468
 
421
- ## CLI Arguments
469
+ ## CLI Arguments (v1: DEPRECATED)
422
470
 
423
471
  | Key | Description |
424
472
  |-----------------------|---------------------------------------------------------------------------|
425
473
  | `--boxes or -b` | Outbox/Inbox processors to start` |
426
474
  | `--concurrency or -c` | Number of threads. Default 10. |
427
475
 
476
+ ## CLI Arguments (v2: default)
477
+
478
+ | Key | Description |
479
+ |----------------------------|----------------------------------------------------------------------|
480
+ | `--boxes or -b` | Outbox/Inbox processors to start` |
481
+ | `--concurrency or -c` | Number of process threads. Default 4. |
482
+ | `--poll-concurrency or -p` | Number of poller partitions. Default 6. |
483
+ | `--poll-threads or -n` | Number of poll threads. Default 1. |
484
+ | `--poll-tactic or -t` | Poll tactic. Default "default". |
485
+ | `--worker-version or -w` | Worker version. Default 2. |
486
+
428
487
  ## Development & Test
429
488
 
430
489
  ### Installation
@@ -5,6 +5,7 @@ module Sbmt
5
5
  class ProcessItem < Sbmt::Outbox::DryInteractor
6
6
  param :item_class, reader: :private
7
7
  param :item_id, reader: :private
8
+ option :worker_version, reader: :private, optional: true, default: -> { 1 }
8
9
 
9
10
  METRICS_COUNTERS = %i[error_counter retry_counter sent_counter fetch_error_counter discarded_counter].freeze
10
11
 
@@ -254,7 +255,7 @@ module Sbmt
254
255
  end
255
256
 
256
257
  def labels_for(item)
257
- {type: box_type, name: box_name, owner: owner, partition: item&.partition}
258
+ {worker_version: worker_version, type: box_type, name: box_name, owner: owner, partition: item&.partition}
258
259
  end
259
260
 
260
261
  def counters
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sbmt
4
+ module Outbox
5
+ module RetryStrategies
6
+ class Base < Outbox::DryInteractor
7
+ param :item
8
+
9
+ def call
10
+ raise NotImplementedError, "Implement #call for Sbmt::Outbox::RetryStrategies::Base"
11
+ end
12
+ end
13
+ end
14
+ end
15
+ end
@@ -3,38 +3,8 @@
3
3
  module Sbmt
4
4
  module Outbox
5
5
  module RetryStrategies
6
- class CompactedLog < Outbox::DryInteractor
7
- param :outbox_item
8
-
9
- def call
10
- unless outbox_item.has_attribute?(:event_key)
11
- return Failure(:missing_event_key)
12
- end
13
-
14
- if outbox_item.event_key.nil?
15
- return Failure(:empty_event_key)
16
- end
17
-
18
- if delivered_later?
19
- Failure(:discard_item)
20
- else
21
- Success()
22
- end
23
- end
24
-
25
- private
26
-
27
- def delivered_later?
28
- scope = outbox_item.class
29
- .where("id > ?", outbox_item)
30
- .where(event_key: outbox_item.event_key, status: Sbmt::Outbox::BaseItem.statuses[:delivered])
31
-
32
- if outbox_item.has_attribute?(:event_name) && outbox_item.event_name.present?
33
- scope = scope.where(event_name: outbox_item.event_name)
34
- end
35
-
36
- scope.exists?
37
- end
6
+ class CompactedLog < LatestAvailable
7
+ # exists only as alias for backward compatibility
38
8
  end
39
9
  end
40
10
  end
@@ -3,13 +3,11 @@
3
3
  module Sbmt
4
4
  module Outbox
5
5
  module RetryStrategies
6
- class ExponentialBackoff < Outbox::DryInteractor
7
- param :outbox_item
8
-
6
+ class ExponentialBackoff < Base
9
7
  def call
10
- delay = backoff(outbox_item.config).interval_at(outbox_item.errors_count - 1)
8
+ delay = backoff(item.config).interval_at(item.errors_count - 1)
11
9
 
12
- still_early = outbox_item.processed_at + delay.seconds > Time.current
10
+ still_early = item.processed_at + delay.seconds > Time.current
13
11
 
14
12
  if still_early
15
13
  Failure(:skip_processing)
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sbmt
4
+ module Outbox
5
+ module RetryStrategies
6
+ class LatestAvailable < Base
7
+ def call
8
+ unless item.has_attribute?(:event_key)
9
+ return Failure(:missing_event_key)
10
+ end
11
+
12
+ if item.event_key.nil?
13
+ return Failure(:empty_event_key)
14
+ end
15
+
16
+ if delivered_later?
17
+ Failure(:discard_item)
18
+ else
19
+ Success()
20
+ end
21
+ end
22
+
23
+ private
24
+
25
+ def delivered_later?
26
+ scope = item.class
27
+ .where("id > ?", item)
28
+ .where(event_key: item.event_key)
29
+
30
+ if item.has_attribute?(:event_name) && item.event_name.present?
31
+ scope = scope.where(event_name: item.event_name)
32
+ end
33
+
34
+ scope.exists?
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sbmt
4
+ module Outbox
5
+ module RetryStrategies
6
+ class NoDelay < Base
7
+ def call
8
+ Success()
9
+ end
10
+ end
11
+ end
12
+ end
13
+ end
@@ -20,15 +20,16 @@ module Sbmt
20
20
  @config ||= lookup_config.new(box_name)
21
21
  end
22
22
 
23
+ def calc_bucket_partitions(count)
24
+ (0...count).to_a
25
+ .each_with_object({}) do |x, m|
26
+ m[x] = (0...config.bucket_size).to_a
27
+ .select { |p| p % count == x }
28
+ end
29
+ end
30
+
23
31
  def partition_buckets
24
- @partition_buckets ||=
25
- (0...config.partition_size)
26
- .to_a
27
- .each_with_object({}) do |x, m|
28
- m[x] = (0...config.bucket_size)
29
- .to_a
30
- .select { |p| p % config.partition_size == x }
31
- end
32
+ @partition_buckets ||= calc_bucket_partitions(config.partition_size)
32
33
  end
33
34
 
34
35
  def bucket_partitions
@@ -23,7 +23,11 @@ module Sbmt
23
23
  end
24
24
 
25
25
  def partition_size
26
- @partition_size ||= (options[:partition_size] || 1).to_i
26
+ @partition_size ||= (partition_size_raw || 1).to_i
27
+ end
28
+
29
+ def partition_size_raw
30
+ @partition_size_raw ||= options[:partition_size]
27
31
  end
28
32
 
29
33
  def retention
@@ -47,7 +51,12 @@ module Sbmt
47
51
  end
48
52
 
49
53
  def retry_strategies
50
- @retry_strategies ||= Array.wrap(options[:retry_strategies]).map do |str_name|
54
+ return @retry_strategies if defined?(@retry_strategies)
55
+
56
+ configured_strategies = options[:retry_strategies]
57
+ strategies = configured_strategies.presence || %w[exponential_backoff latest_available]
58
+
59
+ @retry_strategies ||= Array.wrap(strategies).map do |str_name|
51
60
  "Sbmt::Outbox::RetryStrategies::#{str_name.camelize}".constantize
52
61
  end
53
62
  end
@@ -76,12 +85,22 @@ module Sbmt
76
85
  memo[event_name] ||= []
77
86
  namespace = params.delete(:class)&.camelize
78
87
  raise ArgumentError, "Transport name cannot be blank" if namespace.blank?
88
+ disposable = params.key?(:disposable) ? params.delete(:disposable) : Outbox.config.disposable_transports
79
89
 
80
90
  factory = "#{namespace}::OutboxTransportFactory".safe_constantize
81
91
  memo[event_name] << if factory
82
- factory.build(**params.symbolize_keys)
92
+ if disposable
93
+ ->(*args) { factory.build(**params).call(*args) }
94
+ else
95
+ factory.build(**params)
96
+ end
83
97
  else
84
- namespace.constantize.new(**params.symbolize_keys)
98
+ klass = namespace.constantize
99
+ if disposable
100
+ ->(*args) { klass.new(**params).call(*args) }
101
+ else
102
+ klass.new(**params)
103
+ end
85
104
  end
86
105
  end
87
106
  end
@@ -3,6 +3,8 @@
3
3
  Yabeda.configure do
4
4
  # error_counter retry_counter sent_counter fetch_error_counter discarded_counter
5
5
  group :outbox do
6
+ default_tag(:worker_version, 1)
7
+
6
8
  counter :created_counter,
7
9
  tags: %i[type name partition owner],
8
10
  comment: "The total number of created messages"
@@ -44,28 +46,53 @@ Yabeda.configure do
44
46
  end
45
47
 
46
48
  group :box_worker do
49
+ default_tag(:worker_version, 1)
50
+ default_tag(:worker_name, "worker")
51
+
47
52
  counter :job_counter,
48
- tags: %i[type name partition worker_number state],
53
+ tags: %i[type name partition state],
49
54
  comment: "The total number of processed jobs"
50
55
 
51
56
  counter :job_timeout_counter,
52
- tags: %i[type name partition_key worker_number],
57
+ tags: %i[type name partition_key],
53
58
  comment: "Requeue of a job that occurred while processing the batch"
54
59
 
55
60
  counter :job_items_counter,
56
- tags: %i[type name partition worker_number],
61
+ tags: %i[type name partition],
57
62
  comment: "The total number of processed items in jobs"
58
63
 
59
64
  histogram :job_execution_runtime,
60
65
  comment: "A histogram of the job execution time",
61
66
  unit: :seconds,
62
- tags: %i[type name partition worker_number],
67
+ tags: %i[type name partition],
63
68
  buckets: [0.5, 1, 2.5, 5, 10, 15, 30, 45, 60, 90, 120, 180, 240, 300, 600]
64
69
 
65
70
  histogram :item_execution_runtime,
66
71
  comment: "A histogram of the item execution time",
67
72
  unit: :seconds,
68
- tags: %i[type name partition worker_number],
73
+ tags: %i[type name partition],
74
+ buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300]
75
+
76
+ counter :batches_per_poll_counter,
77
+ tags: %i[type name partition],
78
+ comment: "The total number of poll batches per poll"
79
+
80
+ gauge :redis_job_queue_size,
81
+ tags: %i[type name partition],
82
+ comment: "The total size of redis job queue"
83
+
84
+ gauge :redis_job_queue_time_lag,
85
+ tags: %i[type name partition],
86
+ comment: "The total time lag of redis job queue"
87
+
88
+ counter :poll_throttling_counter,
89
+ tags: %i[type name partition throttler status],
90
+ comment: "The total number of poll throttlings"
91
+
92
+ histogram :poll_throttling_runtime,
93
+ comment: "A histogram of the poll throttling time",
94
+ unit: :seconds,
95
+ tags: %i[type name partition throttler],
69
96
  buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300]
70
97
  end
71
98
  end
@@ -44,8 +44,8 @@ module Outbox
44
44
  end
45
45
 
46
46
  add_index :#{table_name}, :uuid, unique: true
47
- add_index :#{table_name}, [:status, :bucket]
48
- add_index :#{table_name}, :event_key
47
+ add_index :#{table_name}, [:status, :bucket, :errors_count]
48
+ add_index :#{table_name}, [:event_key, :id]
49
49
  add_index :#{table_name}, :created_at
50
50
  RUBY
51
51
 
@@ -20,15 +20,47 @@ module Sbmt
20
20
  option :concurrency,
21
21
  aliases: "-c",
22
22
  type: :numeric,
23
- default: 10,
24
- desc: "Number of threads"
23
+ desc: "Number of threads (processor)"
24
+ option :poll_concurrency,
25
+ aliases: "-p",
26
+ type: :numeric,
27
+ desc: "Number of poller partitions"
28
+ option :poll_threads,
29
+ aliases: "-n",
30
+ type: :numeric,
31
+ desc: "Number of threads (poller)"
32
+ option :poll_tactic,
33
+ aliases: "-t",
34
+ type: :string,
35
+ desc: "Poll tactic: [default, low-priority, aggressive]"
36
+ option :worker_version,
37
+ aliases: "-w",
38
+ type: :numeric,
39
+ desc: "Worker version: [1 | 2]"
25
40
  def start
26
41
  load_environment
27
42
 
28
- worker = Sbmt::Outbox::Worker.new(
29
- boxes: format_boxes(options[:box]),
30
- concurrency: options[:concurrency]
31
- )
43
+ version = options[:worker_version] || Outbox.default_worker_version
44
+
45
+ boxes = format_boxes(options[:box])
46
+ check_deprecations!(boxes, version)
47
+
48
+ worker = if version == 1
49
+ Sbmt::Outbox::V1::Worker.new(
50
+ boxes: boxes,
51
+ concurrency: options[:concurrency] || 10
52
+ )
53
+ elsif version == 2
54
+ Sbmt::Outbox::V2::Worker.new(
55
+ boxes: boxes,
56
+ poll_tactic: options[:poll_tactic],
57
+ poller_threads_count: options[:poll_threads],
58
+ poller_partitions_count: options[:poll_concurrency],
59
+ processor_concurrency: options[:concurrency] || 4
60
+ )
61
+ else
62
+ raise "Worker version #{version} is invalid, available versions: 1|2"
63
+ end
32
64
 
33
65
  Sbmt::Outbox.current_worker = worker
34
66
 
@@ -45,11 +77,22 @@ module Sbmt
45
77
 
46
78
  private
47
79
 
80
+ def check_deprecations!(boxes, version)
81
+ return unless version == 2
82
+
83
+ boxes.each do |item_class|
84
+ next if item_class.config.partition_size_raw.blank?
85
+
86
+ raise "partition_size option is invalid and cannot be used with worker v2, please remove it from config/outbox.yml for #{item_class.name.underscore}"
87
+ end
88
+ end
89
+
48
90
  def load_environment
49
91
  load(lookup_outboxfile)
50
92
 
51
93
  require "sbmt/outbox"
52
- require "sbmt/outbox/worker"
94
+ require "sbmt/outbox/v1/worker"
95
+ require "sbmt/outbox/v2/worker"
53
96
  end
54
97
 
55
98
  def lookup_outboxfile
@@ -14,6 +14,7 @@ module Sbmt
14
14
  c.outbox_item_classes = []
15
15
  c.inbox_item_classes = []
16
16
  c.paths = []
17
+ c.disposable_transports = false
17
18
  c.redis = {url: ENV.fetch("REDIS_URL", "redis://127.0.0.1:6379")}
18
19
  c.process_items = ActiveSupport::OrderedOptions.new.tap do |c|
19
20
  c.general_timeout = 120
@@ -25,6 +26,31 @@ module Sbmt
25
26
  c.rate_interval = 60
26
27
  c.shuffle_jobs = true
27
28
  end
29
+ c.default_worker_version = 2
30
+
31
+ # worker v2
32
+ c.poller = ActiveSupport::OrderedOptions.new.tap do |pc|
33
+ pc.concurrency = 6
34
+ pc.threads_count = 2
35
+ pc.general_timeout = 60
36
+ pc.regular_items_batch_size = 200
37
+ pc.retryable_items_batch_size = 100
38
+
39
+ pc.tactic = "default"
40
+ pc.rate_limit = 60
41
+ pc.rate_interval = 60
42
+ pc.min_queue_size = 10
43
+ pc.max_queue_size = 100
44
+ pc.min_queue_timelag = 5
45
+ pc.queue_delay = 0.1
46
+ end
47
+ c.processor = ActiveSupport::OrderedOptions.new.tap do |pc|
48
+ pc.threads_count = 4
49
+ pc.general_timeout = 120
50
+ pc.cutoff_timeout = 60
51
+ pc.brpop_delay = 2
52
+ end
53
+
28
54
  c.database_switcher = "Sbmt::Outbox::DatabaseSwitcher"
29
55
  c.batch_process_middlewares = []
30
56
  c.item_process_middlewares = []
@@ -5,6 +5,12 @@ module Sbmt
5
5
  class Logger
6
6
  delegate :logger, to: :Rails
7
7
 
8
+ def log_debug(message, **params)
9
+ with_tags(**params) do
10
+ logger.debug(message)
11
+ end
12
+ end
13
+
8
14
  def log_info(message, **params)
9
15
  with_tags(**params) do
10
16
  logger.info(message)