sbmt-outbox 5.0.1 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +68 -9
- data/app/interactors/sbmt/outbox/process_item.rb +2 -1
- data/app/interactors/sbmt/outbox/retry_strategies/base.rb +15 -0
- data/app/interactors/sbmt/outbox/retry_strategies/compacted_log.rb +2 -32
- data/app/interactors/sbmt/outbox/retry_strategies/exponential_backoff.rb +3 -5
- data/app/interactors/sbmt/outbox/retry_strategies/latest_available.rb +39 -0
- data/app/interactors/sbmt/outbox/retry_strategies/no_delay.rb +13 -0
- data/app/models/sbmt/outbox/base_item.rb +9 -8
- data/app/models/sbmt/outbox/base_item_config.rb +23 -4
- data/config/initializers/yabeda.rb +32 -5
- data/lib/generators/helpers/migration.rb +2 -2
- data/lib/sbmt/outbox/cli.rb +50 -7
- data/lib/sbmt/outbox/engine.rb +26 -0
- data/lib/sbmt/outbox/logger.rb +6 -0
- data/lib/sbmt/outbox/v1/thread_pool.rb +110 -0
- data/lib/sbmt/outbox/v1/throttler.rb +54 -0
- data/lib/sbmt/outbox/v1/worker.rb +231 -0
- data/lib/sbmt/outbox/v2/box_processor.rb +148 -0
- data/lib/sbmt/outbox/v2/poll_throttler/base.rb +43 -0
- data/lib/sbmt/outbox/v2/poll_throttler/composite.rb +42 -0
- data/lib/sbmt/outbox/v2/poll_throttler/fixed_delay.rb +28 -0
- data/lib/sbmt/outbox/v2/poll_throttler/noop.rb +17 -0
- data/lib/sbmt/outbox/v2/poll_throttler/rate_limited.rb +24 -0
- data/lib/sbmt/outbox/v2/poll_throttler/redis_queue_size.rb +46 -0
- data/lib/sbmt/outbox/v2/poll_throttler/redis_queue_time_lag.rb +45 -0
- data/lib/sbmt/outbox/v2/poll_throttler.rb +49 -0
- data/lib/sbmt/outbox/v2/poller.rb +180 -0
- data/lib/sbmt/outbox/v2/processor.rb +101 -0
- data/lib/sbmt/outbox/v2/redis_job.rb +42 -0
- data/lib/sbmt/outbox/v2/tasks/base.rb +48 -0
- data/lib/sbmt/outbox/v2/tasks/default.rb +17 -0
- data/lib/sbmt/outbox/v2/tasks/poll.rb +34 -0
- data/lib/sbmt/outbox/v2/tasks/process.rb +31 -0
- data/lib/sbmt/outbox/v2/thread_pool.rb +152 -0
- data/lib/sbmt/outbox/v2/throttler.rb +13 -0
- data/lib/sbmt/outbox/v2/worker.rb +52 -0
- data/lib/sbmt/outbox/version.rb +1 -1
- data/lib/sbmt/outbox.rb +16 -2
- metadata +41 -5
- data/lib/sbmt/outbox/thread_pool.rb +0 -108
- data/lib/sbmt/outbox/throttler.rb +0 -52
- data/lib/sbmt/outbox/worker.rb +0 -233
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 818de79692d37138c630bf2fce625a54becf3a8fa71eafe3c5435d573dbcf665
|
4
|
+
data.tar.gz: 2516c414d14a60851d53fd9f821845179ac12cfb88176a18c1f60d8ba8e38bc9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0e1c0f1f55ef75e5c73352f0789dc51559656f5868a64f2ca8b0c97c10b2865cd37189dc6b3a5d0346673251800ed1c652df951734a5fe77029ca87dbb0fc1a3
|
7
|
+
data.tar.gz: f0aba9307dcfac3c8e330f1a8a1e40e33439a624c4c40c34ea9223c53c25acff2540ad03c72b00a71c1e963885ecee4e6befc5e5e94d4d78e13ecf6acac4c69e
|
data/README.md
CHANGED
@@ -23,6 +23,10 @@ And then execute:
|
|
23
23
|
bundle install
|
24
24
|
```
|
25
25
|
|
26
|
+
## Demo
|
27
|
+
|
28
|
+
Learn how to use this gem and how it works with Ruby on Rails at here https://github.com/SberMarket-Tech/outbox-example-apps
|
29
|
+
|
26
30
|
## Auto configuration
|
27
31
|
|
28
32
|
We recommend going through the configuration and files creation process using the following Rails generators:
|
@@ -113,8 +117,8 @@ create_table :my_outbox_items do |t|
|
|
113
117
|
end
|
114
118
|
|
115
119
|
add_index :my_outbox_items, :uuid, unique: true
|
116
|
-
add_index :my_outbox_items, [:status, :bucket]
|
117
|
-
add_index :my_outbox_items, [:event_name, :event_key]
|
120
|
+
add_index :my_outbox_items, [:status, :bucket, :errors_count]
|
121
|
+
add_index :my_outbox_items, [:event_name, :event_key, :id]
|
118
122
|
add_index :my_outbox_items, :created_at
|
119
123
|
```
|
120
124
|
|
@@ -146,7 +150,11 @@ default: &default
|
|
146
150
|
max_retries: 3 # default 0, the number of retries before the item will be marked as failed
|
147
151
|
transports: # transports section
|
148
152
|
produce_message: # underscored transport class name
|
149
|
-
|
153
|
+
# transport reserved options
|
154
|
+
class: produce_message # optional; default is inferred from transport name
|
155
|
+
disposable: false # optional; default false; if true, the transport class will be instantiated only once
|
156
|
+
# ProduceMessage instance arguments
|
157
|
+
topic: "my-topic-name"
|
150
158
|
|
151
159
|
development:
|
152
160
|
<<: *default
|
@@ -182,9 +190,11 @@ Transports are defined as follows when `event_name` is used:
|
|
182
190
|
outbox_items:
|
183
191
|
my_outbox_item:
|
184
192
|
transports:
|
193
|
+
# transport reserved options
|
185
194
|
- class: produce_message
|
186
195
|
event_name: "order_created" # event name marker
|
187
|
-
|
196
|
+
# ProduceMessage instance arguments
|
197
|
+
topic: "order_created_topic" # some transport argument
|
188
198
|
- class: produce_message
|
189
199
|
event_name: "orders_completed"
|
190
200
|
topic: "orders_completed_topic"
|
@@ -212,13 +222,51 @@ Rails.application.config.outbox.tap do |config|
|
|
212
222
|
x[:batch_size] = 200
|
213
223
|
end
|
214
224
|
|
215
|
-
# optional
|
225
|
+
# optional (worker v1: DEPRECATED)
|
216
226
|
config.worker.tap do |worker|
|
217
227
|
# number of batches that one thread will process per rate interval
|
218
228
|
worker[:rate_limit] = 10
|
219
229
|
# rate interval in seconds
|
220
230
|
worker[:rate_interval] = 60
|
221
231
|
end
|
232
|
+
|
233
|
+
# optional (worker v2: default)
|
234
|
+
c.poller = ActiveSupport::OrderedOptions.new.tap do |pc|
|
235
|
+
# max parallel threads (per box-item, globally)
|
236
|
+
pc.concurrency = 6
|
237
|
+
# max threads count (per worker process)
|
238
|
+
pc.threads_count = 1
|
239
|
+
# maximum processing time of the batch, after which the batch will be considered hung and processing will be aborted
|
240
|
+
pc.general_timeout = 60
|
241
|
+
# poll buffer consists of regular items (errors_count = 0, i.e. without any processing errors) and retryable items (errors_count > 0)
|
242
|
+
# max poll buffer size = regular_items_batch_size + retryable_items_batch_size
|
243
|
+
pc.regular_items_batch_size = 200
|
244
|
+
pc.retryable_items_batch_size = 100
|
245
|
+
|
246
|
+
# poll tactic: default is optimal for most cases: rate limit + redis job-queue size threshold
|
247
|
+
# poll tactic: aggressive is for high-intencity data: without rate limits + redis job-queue size threshold
|
248
|
+
# poll tactic: low-priority is for low-intencity data: rate limits + redis job-queue size threshold + + redis job-queue lag threshold
|
249
|
+
pc.tactic = "default"
|
250
|
+
# number of batches that one thread will process per rate interval
|
251
|
+
pc.rate_limit = 60
|
252
|
+
# rate interval in seconds
|
253
|
+
pc.rate_interval = 60
|
254
|
+
# mix / max redis job queue thresholds per box-item for default / aggressive / low-priority poll tactics
|
255
|
+
pc.min_queue_size = 10
|
256
|
+
pc.max_queue_size = 100
|
257
|
+
# min redis job queue time lag threshold per box-item for low-priority poll tactic (in seconds)
|
258
|
+
pc.min_queue_timelag = 5
|
259
|
+
# throttling delay for default / aggressive / low-priority poll tactics (in seconds)
|
260
|
+
pc.queue_delay = 0.1
|
261
|
+
end
|
262
|
+
c.processor = ActiveSupport::OrderedOptions.new.tap do |pc|
|
263
|
+
# max threads count (per worker process)
|
264
|
+
pc.threads_count = 4
|
265
|
+
# maximum processing time of the batch, after which the batch will be considered hung and processing will be aborted
|
266
|
+
pc.general_timeout = 120
|
267
|
+
# BRPOP delay (in seconds) for polling redis job queue per box-item
|
268
|
+
pc.brpop_delay = 2
|
269
|
+
end
|
222
270
|
end
|
223
271
|
```
|
224
272
|
|
@@ -282,7 +330,7 @@ outbox_items:
|
|
282
330
|
- exponential_backoff
|
283
331
|
```
|
284
332
|
|
285
|
-
####
|
333
|
+
#### Latest available
|
286
334
|
|
287
335
|
This strategy ensures idempotency. In short, if a message fails and a later message with the same event_key has already been delivered, then you most likely do not want to re-deliver the first one when it is retried.
|
288
336
|
|
@@ -293,10 +341,10 @@ outbox_items:
|
|
293
341
|
...
|
294
342
|
retry_strategies:
|
295
343
|
- exponential_backoff
|
296
|
-
-
|
344
|
+
- latest_available
|
297
345
|
```
|
298
346
|
|
299
|
-
The exponential backoff strategy should be used in conjunction with the
|
347
|
+
The exponential backoff strategy should be used in conjunction with the latest available strategy, and it should come last to minimize the number of database queries.
|
300
348
|
|
301
349
|
### Partition strategies
|
302
350
|
|
@@ -418,13 +466,24 @@ end
|
|
418
466
|
|
419
467
|
The gem is optionally integrated with OpenTelemetry. If your main application has `opentelemetry-*` gems, the tracing will be configured automatically.
|
420
468
|
|
421
|
-
## CLI Arguments
|
469
|
+
## CLI Arguments (v1: DEPRECATED)
|
422
470
|
|
423
471
|
| Key | Description |
|
424
472
|
|-----------------------|---------------------------------------------------------------------------|
|
425
473
|
| `--boxes or -b` | Outbox/Inbox processors to start` |
|
426
474
|
| `--concurrency or -c` | Number of threads. Default 10. |
|
427
475
|
|
476
|
+
## CLI Arguments (v2: default)
|
477
|
+
|
478
|
+
| Key | Description |
|
479
|
+
|----------------------------|----------------------------------------------------------------------|
|
480
|
+
| `--boxes or -b` | Outbox/Inbox processors to start` |
|
481
|
+
| `--concurrency or -c` | Number of process threads. Default 4. |
|
482
|
+
| `--poll-concurrency or -p` | Number of poller partitions. Default 6. |
|
483
|
+
| `--poll-threads or -n` | Number of poll threads. Default 1. |
|
484
|
+
| `--poll-tactic or -t` | Poll tactic. Default "default". |
|
485
|
+
| `--worker-version or -w` | Worker version. Default 2. |
|
486
|
+
|
428
487
|
## Development & Test
|
429
488
|
|
430
489
|
### Installation
|
@@ -5,6 +5,7 @@ module Sbmt
|
|
5
5
|
class ProcessItem < Sbmt::Outbox::DryInteractor
|
6
6
|
param :item_class, reader: :private
|
7
7
|
param :item_id, reader: :private
|
8
|
+
option :worker_version, reader: :private, optional: true, default: -> { 1 }
|
8
9
|
|
9
10
|
METRICS_COUNTERS = %i[error_counter retry_counter sent_counter fetch_error_counter discarded_counter].freeze
|
10
11
|
|
@@ -254,7 +255,7 @@ module Sbmt
|
|
254
255
|
end
|
255
256
|
|
256
257
|
def labels_for(item)
|
257
|
-
{type: box_type, name: box_name, owner: owner, partition: item&.partition}
|
258
|
+
{worker_version: worker_version, type: box_type, name: box_name, owner: owner, partition: item&.partition}
|
258
259
|
end
|
259
260
|
|
260
261
|
def counters
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sbmt
|
4
|
+
module Outbox
|
5
|
+
module RetryStrategies
|
6
|
+
class Base < Outbox::DryInteractor
|
7
|
+
param :item
|
8
|
+
|
9
|
+
def call
|
10
|
+
raise NotImplementedError, "Implement #call for Sbmt::Outbox::RetryStrategies::Base"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -3,38 +3,8 @@
|
|
3
3
|
module Sbmt
|
4
4
|
module Outbox
|
5
5
|
module RetryStrategies
|
6
|
-
class CompactedLog <
|
7
|
-
|
8
|
-
|
9
|
-
def call
|
10
|
-
unless outbox_item.has_attribute?(:event_key)
|
11
|
-
return Failure(:missing_event_key)
|
12
|
-
end
|
13
|
-
|
14
|
-
if outbox_item.event_key.nil?
|
15
|
-
return Failure(:empty_event_key)
|
16
|
-
end
|
17
|
-
|
18
|
-
if delivered_later?
|
19
|
-
Failure(:discard_item)
|
20
|
-
else
|
21
|
-
Success()
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
private
|
26
|
-
|
27
|
-
def delivered_later?
|
28
|
-
scope = outbox_item.class
|
29
|
-
.where("id > ?", outbox_item)
|
30
|
-
.where(event_key: outbox_item.event_key, status: Sbmt::Outbox::BaseItem.statuses[:delivered])
|
31
|
-
|
32
|
-
if outbox_item.has_attribute?(:event_name) && outbox_item.event_name.present?
|
33
|
-
scope = scope.where(event_name: outbox_item.event_name)
|
34
|
-
end
|
35
|
-
|
36
|
-
scope.exists?
|
37
|
-
end
|
6
|
+
class CompactedLog < LatestAvailable
|
7
|
+
# exists only as alias for backward compatibility
|
38
8
|
end
|
39
9
|
end
|
40
10
|
end
|
@@ -3,13 +3,11 @@
|
|
3
3
|
module Sbmt
|
4
4
|
module Outbox
|
5
5
|
module RetryStrategies
|
6
|
-
class ExponentialBackoff <
|
7
|
-
param :outbox_item
|
8
|
-
|
6
|
+
class ExponentialBackoff < Base
|
9
7
|
def call
|
10
|
-
delay = backoff(
|
8
|
+
delay = backoff(item.config).interval_at(item.errors_count - 1)
|
11
9
|
|
12
|
-
still_early =
|
10
|
+
still_early = item.processed_at + delay.seconds > Time.current
|
13
11
|
|
14
12
|
if still_early
|
15
13
|
Failure(:skip_processing)
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sbmt
|
4
|
+
module Outbox
|
5
|
+
module RetryStrategies
|
6
|
+
class LatestAvailable < Base
|
7
|
+
def call
|
8
|
+
unless item.has_attribute?(:event_key)
|
9
|
+
return Failure(:missing_event_key)
|
10
|
+
end
|
11
|
+
|
12
|
+
if item.event_key.nil?
|
13
|
+
return Failure(:empty_event_key)
|
14
|
+
end
|
15
|
+
|
16
|
+
if delivered_later?
|
17
|
+
Failure(:discard_item)
|
18
|
+
else
|
19
|
+
Success()
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def delivered_later?
|
26
|
+
scope = item.class
|
27
|
+
.where("id > ?", item)
|
28
|
+
.where(event_key: item.event_key)
|
29
|
+
|
30
|
+
if item.has_attribute?(:event_name) && item.event_name.present?
|
31
|
+
scope = scope.where(event_name: item.event_name)
|
32
|
+
end
|
33
|
+
|
34
|
+
scope.exists?
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -20,15 +20,16 @@ module Sbmt
|
|
20
20
|
@config ||= lookup_config.new(box_name)
|
21
21
|
end
|
22
22
|
|
23
|
+
def calc_bucket_partitions(count)
|
24
|
+
(0...count).to_a
|
25
|
+
.each_with_object({}) do |x, m|
|
26
|
+
m[x] = (0...config.bucket_size).to_a
|
27
|
+
.select { |p| p % count == x }
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
23
31
|
def partition_buckets
|
24
|
-
@partition_buckets ||=
|
25
|
-
(0...config.partition_size)
|
26
|
-
.to_a
|
27
|
-
.each_with_object({}) do |x, m|
|
28
|
-
m[x] = (0...config.bucket_size)
|
29
|
-
.to_a
|
30
|
-
.select { |p| p % config.partition_size == x }
|
31
|
-
end
|
32
|
+
@partition_buckets ||= calc_bucket_partitions(config.partition_size)
|
32
33
|
end
|
33
34
|
|
34
35
|
def bucket_partitions
|
@@ -23,7 +23,11 @@ module Sbmt
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def partition_size
|
26
|
-
@partition_size ||= (
|
26
|
+
@partition_size ||= (partition_size_raw || 1).to_i
|
27
|
+
end
|
28
|
+
|
29
|
+
def partition_size_raw
|
30
|
+
@partition_size_raw ||= options[:partition_size]
|
27
31
|
end
|
28
32
|
|
29
33
|
def retention
|
@@ -47,7 +51,12 @@ module Sbmt
|
|
47
51
|
end
|
48
52
|
|
49
53
|
def retry_strategies
|
50
|
-
@retry_strategies
|
54
|
+
return @retry_strategies if defined?(@retry_strategies)
|
55
|
+
|
56
|
+
configured_strategies = options[:retry_strategies]
|
57
|
+
strategies = configured_strategies.presence || %w[exponential_backoff latest_available]
|
58
|
+
|
59
|
+
@retry_strategies ||= Array.wrap(strategies).map do |str_name|
|
51
60
|
"Sbmt::Outbox::RetryStrategies::#{str_name.camelize}".constantize
|
52
61
|
end
|
53
62
|
end
|
@@ -76,12 +85,22 @@ module Sbmt
|
|
76
85
|
memo[event_name] ||= []
|
77
86
|
namespace = params.delete(:class)&.camelize
|
78
87
|
raise ArgumentError, "Transport name cannot be blank" if namespace.blank?
|
88
|
+
disposable = params.key?(:disposable) ? params.delete(:disposable) : Outbox.config.disposable_transports
|
79
89
|
|
80
90
|
factory = "#{namespace}::OutboxTransportFactory".safe_constantize
|
81
91
|
memo[event_name] << if factory
|
82
|
-
|
92
|
+
if disposable
|
93
|
+
->(*args) { factory.build(**params).call(*args) }
|
94
|
+
else
|
95
|
+
factory.build(**params)
|
96
|
+
end
|
83
97
|
else
|
84
|
-
namespace.constantize
|
98
|
+
klass = namespace.constantize
|
99
|
+
if disposable
|
100
|
+
->(*args) { klass.new(**params).call(*args) }
|
101
|
+
else
|
102
|
+
klass.new(**params)
|
103
|
+
end
|
85
104
|
end
|
86
105
|
end
|
87
106
|
end
|
@@ -3,6 +3,8 @@
|
|
3
3
|
Yabeda.configure do
|
4
4
|
# error_counter retry_counter sent_counter fetch_error_counter discarded_counter
|
5
5
|
group :outbox do
|
6
|
+
default_tag(:worker_version, 1)
|
7
|
+
|
6
8
|
counter :created_counter,
|
7
9
|
tags: %i[type name partition owner],
|
8
10
|
comment: "The total number of created messages"
|
@@ -44,28 +46,53 @@ Yabeda.configure do
|
|
44
46
|
end
|
45
47
|
|
46
48
|
group :box_worker do
|
49
|
+
default_tag(:worker_version, 1)
|
50
|
+
default_tag(:worker_name, "worker")
|
51
|
+
|
47
52
|
counter :job_counter,
|
48
|
-
tags: %i[type name partition
|
53
|
+
tags: %i[type name partition state],
|
49
54
|
comment: "The total number of processed jobs"
|
50
55
|
|
51
56
|
counter :job_timeout_counter,
|
52
|
-
tags: %i[type name partition_key
|
57
|
+
tags: %i[type name partition_key],
|
53
58
|
comment: "Requeue of a job that occurred while processing the batch"
|
54
59
|
|
55
60
|
counter :job_items_counter,
|
56
|
-
tags: %i[type name partition
|
61
|
+
tags: %i[type name partition],
|
57
62
|
comment: "The total number of processed items in jobs"
|
58
63
|
|
59
64
|
histogram :job_execution_runtime,
|
60
65
|
comment: "A histogram of the job execution time",
|
61
66
|
unit: :seconds,
|
62
|
-
tags: %i[type name partition
|
67
|
+
tags: %i[type name partition],
|
63
68
|
buckets: [0.5, 1, 2.5, 5, 10, 15, 30, 45, 60, 90, 120, 180, 240, 300, 600]
|
64
69
|
|
65
70
|
histogram :item_execution_runtime,
|
66
71
|
comment: "A histogram of the item execution time",
|
67
72
|
unit: :seconds,
|
68
|
-
tags: %i[type name partition
|
73
|
+
tags: %i[type name partition],
|
74
|
+
buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300]
|
75
|
+
|
76
|
+
counter :batches_per_poll_counter,
|
77
|
+
tags: %i[type name partition],
|
78
|
+
comment: "The total number of poll batches per poll"
|
79
|
+
|
80
|
+
gauge :redis_job_queue_size,
|
81
|
+
tags: %i[type name partition],
|
82
|
+
comment: "The total size of redis job queue"
|
83
|
+
|
84
|
+
gauge :redis_job_queue_time_lag,
|
85
|
+
tags: %i[type name partition],
|
86
|
+
comment: "The total time lag of redis job queue"
|
87
|
+
|
88
|
+
counter :poll_throttling_counter,
|
89
|
+
tags: %i[type name partition throttler status],
|
90
|
+
comment: "The total number of poll throttlings"
|
91
|
+
|
92
|
+
histogram :poll_throttling_runtime,
|
93
|
+
comment: "A histogram of the poll throttling time",
|
94
|
+
unit: :seconds,
|
95
|
+
tags: %i[type name partition throttler],
|
69
96
|
buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300]
|
70
97
|
end
|
71
98
|
end
|
@@ -44,8 +44,8 @@ module Outbox
|
|
44
44
|
end
|
45
45
|
|
46
46
|
add_index :#{table_name}, :uuid, unique: true
|
47
|
-
add_index :#{table_name}, [:status, :bucket]
|
48
|
-
add_index :#{table_name}, :event_key
|
47
|
+
add_index :#{table_name}, [:status, :bucket, :errors_count]
|
48
|
+
add_index :#{table_name}, [:event_key, :id]
|
49
49
|
add_index :#{table_name}, :created_at
|
50
50
|
RUBY
|
51
51
|
|
data/lib/sbmt/outbox/cli.rb
CHANGED
@@ -20,15 +20,47 @@ module Sbmt
|
|
20
20
|
option :concurrency,
|
21
21
|
aliases: "-c",
|
22
22
|
type: :numeric,
|
23
|
-
|
24
|
-
|
23
|
+
desc: "Number of threads (processor)"
|
24
|
+
option :poll_concurrency,
|
25
|
+
aliases: "-p",
|
26
|
+
type: :numeric,
|
27
|
+
desc: "Number of poller partitions"
|
28
|
+
option :poll_threads,
|
29
|
+
aliases: "-n",
|
30
|
+
type: :numeric,
|
31
|
+
desc: "Number of threads (poller)"
|
32
|
+
option :poll_tactic,
|
33
|
+
aliases: "-t",
|
34
|
+
type: :string,
|
35
|
+
desc: "Poll tactic: [default, low-priority, aggressive]"
|
36
|
+
option :worker_version,
|
37
|
+
aliases: "-w",
|
38
|
+
type: :numeric,
|
39
|
+
desc: "Worker version: [1 | 2]"
|
25
40
|
def start
|
26
41
|
load_environment
|
27
42
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
)
|
43
|
+
version = options[:worker_version] || Outbox.default_worker_version
|
44
|
+
|
45
|
+
boxes = format_boxes(options[:box])
|
46
|
+
check_deprecations!(boxes, version)
|
47
|
+
|
48
|
+
worker = if version == 1
|
49
|
+
Sbmt::Outbox::V1::Worker.new(
|
50
|
+
boxes: boxes,
|
51
|
+
concurrency: options[:concurrency] || 10
|
52
|
+
)
|
53
|
+
elsif version == 2
|
54
|
+
Sbmt::Outbox::V2::Worker.new(
|
55
|
+
boxes: boxes,
|
56
|
+
poll_tactic: options[:poll_tactic],
|
57
|
+
poller_threads_count: options[:poll_threads],
|
58
|
+
poller_partitions_count: options[:poll_concurrency],
|
59
|
+
processor_concurrency: options[:concurrency] || 4
|
60
|
+
)
|
61
|
+
else
|
62
|
+
raise "Worker version #{version} is invalid, available versions: 1|2"
|
63
|
+
end
|
32
64
|
|
33
65
|
Sbmt::Outbox.current_worker = worker
|
34
66
|
|
@@ -45,11 +77,22 @@ module Sbmt
|
|
45
77
|
|
46
78
|
private
|
47
79
|
|
80
|
+
def check_deprecations!(boxes, version)
|
81
|
+
return unless version == 2
|
82
|
+
|
83
|
+
boxes.each do |item_class|
|
84
|
+
next if item_class.config.partition_size_raw.blank?
|
85
|
+
|
86
|
+
raise "partition_size option is invalid and cannot be used with worker v2, please remove it from config/outbox.yml for #{item_class.name.underscore}"
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
48
90
|
def load_environment
|
49
91
|
load(lookup_outboxfile)
|
50
92
|
|
51
93
|
require "sbmt/outbox"
|
52
|
-
require "sbmt/outbox/worker"
|
94
|
+
require "sbmt/outbox/v1/worker"
|
95
|
+
require "sbmt/outbox/v2/worker"
|
53
96
|
end
|
54
97
|
|
55
98
|
def lookup_outboxfile
|
data/lib/sbmt/outbox/engine.rb
CHANGED
@@ -14,6 +14,7 @@ module Sbmt
|
|
14
14
|
c.outbox_item_classes = []
|
15
15
|
c.inbox_item_classes = []
|
16
16
|
c.paths = []
|
17
|
+
c.disposable_transports = false
|
17
18
|
c.redis = {url: ENV.fetch("REDIS_URL", "redis://127.0.0.1:6379")}
|
18
19
|
c.process_items = ActiveSupport::OrderedOptions.new.tap do |c|
|
19
20
|
c.general_timeout = 120
|
@@ -25,6 +26,31 @@ module Sbmt
|
|
25
26
|
c.rate_interval = 60
|
26
27
|
c.shuffle_jobs = true
|
27
28
|
end
|
29
|
+
c.default_worker_version = 2
|
30
|
+
|
31
|
+
# worker v2
|
32
|
+
c.poller = ActiveSupport::OrderedOptions.new.tap do |pc|
|
33
|
+
pc.concurrency = 6
|
34
|
+
pc.threads_count = 2
|
35
|
+
pc.general_timeout = 60
|
36
|
+
pc.regular_items_batch_size = 200
|
37
|
+
pc.retryable_items_batch_size = 100
|
38
|
+
|
39
|
+
pc.tactic = "default"
|
40
|
+
pc.rate_limit = 60
|
41
|
+
pc.rate_interval = 60
|
42
|
+
pc.min_queue_size = 10
|
43
|
+
pc.max_queue_size = 100
|
44
|
+
pc.min_queue_timelag = 5
|
45
|
+
pc.queue_delay = 0.1
|
46
|
+
end
|
47
|
+
c.processor = ActiveSupport::OrderedOptions.new.tap do |pc|
|
48
|
+
pc.threads_count = 4
|
49
|
+
pc.general_timeout = 120
|
50
|
+
pc.cutoff_timeout = 60
|
51
|
+
pc.brpop_delay = 2
|
52
|
+
end
|
53
|
+
|
28
54
|
c.database_switcher = "Sbmt::Outbox::DatabaseSwitcher"
|
29
55
|
c.batch_process_middlewares = []
|
30
56
|
c.item_process_middlewares = []
|
data/lib/sbmt/outbox/logger.rb
CHANGED