rails-transactional-outbox 0.4.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +1 -1
- data/.github/workflows/ci.yml +2 -2
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +5 -0
- data/Gemfile.lock +54 -15
- data/README.md +34 -1
- data/lib/rails_transactional_outbox/configuration.rb +12 -2
- data/lib/rails_transactional_outbox/datadog_latency_reporter.rb +28 -0
- data/lib/rails_transactional_outbox/datadog_latency_reporter_job.rb +13 -0
- data/lib/rails_transactional_outbox/datadog_latency_reporter_scheduler.rb +49 -0
- data/lib/rails_transactional_outbox/health_check.rb +1 -1
- data/lib/rails_transactional_outbox/latency_tracker.rb +45 -0
- data/lib/rails_transactional_outbox/outbox_entries_processor.rb +2 -2
- data/lib/rails_transactional_outbox/outbox_entries_processors/base_processor.rb +3 -3
- data/lib/rails_transactional_outbox/outbox_entries_processors/non_ordered_processor.rb +2 -2
- data/lib/rails_transactional_outbox/outbox_entries_processors/ordered_by_causality_key_processor.rb +2 -2
- data/lib/rails_transactional_outbox/outbox_model.rb +10 -1
- data/lib/rails_transactional_outbox/reliable_model.rb +8 -8
- data/lib/rails_transactional_outbox/version.rb +1 -1
- data/lib/rails_transactional_outbox.rb +3 -0
- data/lib/tasks/rails_transactional_outbox.rake +1 -1
- data/rails-transactional-outbox.gemspec +7 -1
- metadata +78 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f10edf27ea7703dbbc4e77cbf20d4d76d5af6a37eb2c757e48abeae1e61e5424
|
4
|
+
data.tar.gz: 6b542b7a5b030b5162a93a9cf6618122ccece186fc10d71b0713dba59fbe2a43
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e6b0ccdc12074b4bbf0c6b231cc6d6d392bd7a4b7781918c34d429623e3c03e32a505fa898ca9d7472821d9929003ee655df9acc1a6158bc6d6c3d255d63268f
|
7
|
+
data.tar.gz: b3d670696cab6f7093d15a1913489c19f8f63e6a5160a80080e20c4ffbc2b0704221a667c0cbd08e7097d8d4f53e70e450a297f569a1075e1da142d8ff2d22eb
|
data/.circleci/config.yml
CHANGED
data/.github/workflows/ci.yml
CHANGED
@@ -9,14 +9,14 @@ jobs:
|
|
9
9
|
- uses: actions/checkout@v2
|
10
10
|
- uses: ruby/setup-ruby@v1
|
11
11
|
with:
|
12
|
-
ruby-version:
|
12
|
+
ruby-version: 3.1
|
13
13
|
bundler-cache: true
|
14
14
|
- run: bundle exec rubocop
|
15
15
|
rspec:
|
16
16
|
strategy:
|
17
17
|
fail-fast: false
|
18
18
|
matrix:
|
19
|
-
ruby: [
|
19
|
+
ruby: ['3.1', '3.2', '3.3']
|
20
20
|
runs-on: ubuntu-latest
|
21
21
|
env:
|
22
22
|
DATABASE_URL: "postgresql://postgres:postgres@127.0.0.1:5432/rails-transactional-outbox-test"
|
data/.rubocop.yml
CHANGED
@@ -9,7 +9,7 @@ inherit_mode:
|
|
9
9
|
|
10
10
|
AllCops:
|
11
11
|
NewCops: enable
|
12
|
-
TargetRubyVersion:
|
12
|
+
TargetRubyVersion: 3.1
|
13
13
|
Exclude:
|
14
14
|
- "db/**/*"
|
15
15
|
- "bin/**/*"
|
@@ -147,6 +147,7 @@ Lint/ConstantDefinitionInBlock:
|
|
147
147
|
RSpec/AnyInstance:
|
148
148
|
Exclude:
|
149
149
|
- 'spec/rails_transactional_outbox/outbox_model_spec.rb'
|
150
|
+
- 'spec/rails_transactional_outbox/datadog_latency_reporter_job_spec.rb'
|
150
151
|
|
151
152
|
RSpec/VerifiedDoubles:
|
152
153
|
Exclude:
|
data/CHANGELOG.md
CHANGED
data/Gemfile.lock
CHANGED
@@ -9,12 +9,15 @@ GIT
|
|
9
9
|
PATH
|
10
10
|
remote: .
|
11
11
|
specs:
|
12
|
-
rails-transactional-outbox (0.
|
12
|
+
rails-transactional-outbox (1.0.0)
|
13
13
|
activerecord (>= 5)
|
14
14
|
activesupport (>= 3.2)
|
15
15
|
concurrent-ruby
|
16
|
+
dogstatsd-ruby
|
16
17
|
dry-monitor
|
17
18
|
file-based-healthcheck
|
19
|
+
sidekiq (>= 5.0)
|
20
|
+
sidekiq-cron
|
18
21
|
sigurd
|
19
22
|
zeitwerk
|
20
23
|
|
@@ -33,17 +36,22 @@ GEM
|
|
33
36
|
tzinfo (~> 2.0)
|
34
37
|
ast (2.4.2)
|
35
38
|
concurrent-ruby (1.1.10)
|
36
|
-
|
37
|
-
|
38
|
-
libddprof (~> 0.6.0.1.0)
|
39
|
-
libddwaf (~> 1.3.0.2.0)
|
39
|
+
connection_pool (2.4.1)
|
40
|
+
datadog-ci (0.8.3)
|
40
41
|
msgpack
|
41
|
-
|
42
|
+
ddtrace (1.23.3)
|
43
|
+
datadog-ci (~> 0.8.1)
|
44
|
+
debase-ruby_core_source (= 3.3.1)
|
45
|
+
libdatadog (~> 7.0.0.1.0)
|
46
|
+
libddwaf (~> 1.14.0.0.0)
|
47
|
+
msgpack
|
48
|
+
debase-ruby_core_source (3.3.1)
|
42
49
|
diff-lcs (1.5.0)
|
43
|
-
|
50
|
+
dogstatsd-ruby (4.8.3)
|
51
|
+
dry-configurable (1.2.0)
|
44
52
|
dry-core (~> 1.0, < 2)
|
45
53
|
zeitwerk (~> 2.6)
|
46
|
-
dry-core (1.0.
|
54
|
+
dry-core (1.0.1)
|
47
55
|
concurrent-ruby (~> 1.0)
|
48
56
|
zeitwerk (~> 2.6)
|
49
57
|
dry-events (1.0.1)
|
@@ -53,28 +61,43 @@ GEM
|
|
53
61
|
dry-configurable (~> 1.0, < 2)
|
54
62
|
dry-core (~> 1.0, < 2)
|
55
63
|
dry-events (~> 1.0, < 2)
|
64
|
+
et-orbi (1.2.11)
|
65
|
+
tzinfo
|
56
66
|
exponential-backoff (0.0.4)
|
57
|
-
ffi (1.
|
67
|
+
ffi (1.17.0-arm64-darwin)
|
68
|
+
ffi (1.17.0-x86_64-darwin)
|
69
|
+
ffi (1.17.0-x86_64-linux-gnu)
|
58
70
|
file-based-healthcheck (0.2.0)
|
59
71
|
activesupport (>= 3.2)
|
72
|
+
fugit (1.11.1)
|
73
|
+
et-orbi (~> 1, >= 1.2.11)
|
74
|
+
raabro (~> 1.4)
|
75
|
+
globalid (1.2.1)
|
76
|
+
activesupport (>= 6.1)
|
60
77
|
i18n (1.12.0)
|
61
78
|
concurrent-ruby (~> 1.0)
|
62
79
|
json (2.6.2)
|
63
|
-
|
64
|
-
|
65
|
-
libddwaf (1.
|
80
|
+
libdatadog (7.0.0.1.0)
|
81
|
+
libdatadog (7.0.0.1.0-x86_64-linux)
|
82
|
+
libddwaf (1.14.0.0.0-arm64-darwin)
|
83
|
+
ffi (~> 1.0)
|
84
|
+
libddwaf (1.14.0.0.0-x86_64-darwin)
|
66
85
|
ffi (~> 1.0)
|
67
|
-
libddwaf (1.
|
86
|
+
libddwaf (1.14.0.0.0-x86_64-linux)
|
68
87
|
ffi (~> 1.0)
|
69
88
|
minitest (5.16.3)
|
70
|
-
msgpack (1.
|
89
|
+
msgpack (1.7.2)
|
71
90
|
parallel (1.22.1)
|
72
91
|
parser (3.1.2.1)
|
73
92
|
ast (~> 2.4.1)
|
74
93
|
pg (1.4.3)
|
94
|
+
raabro (1.4.0)
|
95
|
+
rack (2.2.9)
|
75
96
|
rainbow (3.1.1)
|
76
97
|
rake (13.0.6)
|
77
98
|
redis (4.8.0)
|
99
|
+
redis-namespace (1.11.0)
|
100
|
+
redis (>= 4)
|
78
101
|
redlock (1.3.0)
|
79
102
|
redis (>= 3.0.0, < 6.0)
|
80
103
|
regexp_parser (2.5.0)
|
@@ -91,6 +114,11 @@ GEM
|
|
91
114
|
rspec-mocks (3.11.1)
|
92
115
|
diff-lcs (>= 1.2.0, < 2.0)
|
93
116
|
rspec-support (~> 3.11.0)
|
117
|
+
rspec-sidekiq (5.0.0)
|
118
|
+
rspec-core (~> 3.0)
|
119
|
+
rspec-expectations (~> 3.0)
|
120
|
+
rspec-mocks (~> 3.0)
|
121
|
+
sidekiq (>= 5, < 8)
|
94
122
|
rspec-support (3.11.0)
|
95
123
|
rubocop (1.35.1)
|
96
124
|
json (~> 2.3)
|
@@ -114,6 +142,14 @@ GEM
|
|
114
142
|
ruby-progressbar (1.11.0)
|
115
143
|
sentry-ruby (5.4.1)
|
116
144
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
145
|
+
sidekiq (6.5.12)
|
146
|
+
connection_pool (>= 2.2.5, < 3)
|
147
|
+
rack (~> 2.0)
|
148
|
+
redis (>= 4.5.0, < 5)
|
149
|
+
sidekiq-cron (1.12.0)
|
150
|
+
fugit (~> 1.8)
|
151
|
+
globalid (>= 1.0.1)
|
152
|
+
sidekiq (>= 6)
|
117
153
|
sigurd (0.1.0)
|
118
154
|
concurrent-ruby (~> 1)
|
119
155
|
exponential-backoff
|
@@ -121,9 +157,10 @@ GEM
|
|
121
157
|
tzinfo (2.0.5)
|
122
158
|
concurrent-ruby (~> 1.0)
|
123
159
|
unicode-display_width (2.2.0)
|
124
|
-
zeitwerk (2.6.
|
160
|
+
zeitwerk (2.6.17)
|
125
161
|
|
126
162
|
PLATFORMS
|
163
|
+
arm64-darwin-22
|
127
164
|
x86_64-darwin-18
|
128
165
|
x86_64-darwin-21
|
129
166
|
x86_64-linux
|
@@ -134,8 +171,10 @@ DEPENDENCIES
|
|
134
171
|
pg
|
135
172
|
rails-transactional-outbox!
|
136
173
|
rake (~> 13.0)
|
174
|
+
redis-namespace
|
137
175
|
redlock
|
138
176
|
rspec (~> 3.0)
|
177
|
+
rspec-sidekiq
|
139
178
|
rubocop
|
140
179
|
rubocop-performance
|
141
180
|
rubocop-rake
|
data/README.md
CHANGED
@@ -46,6 +46,9 @@ Rails.application.config.to_prepare do
|
|
46
46
|
config.outbox_entries_processor = `RailsTransactionalOutbox::OutboxEntriesProcessors::OrderedByCausalityKeyProcessor`.new # not required, defaults to RailsTransactionalOutbox::OutboxEntriesProcessors::NonOrderedProcessor.new
|
47
47
|
config.outbox_entry_causality_key_resolver = ->(model) { model.tenant_id } # not required, defaults to a lambda returning nil. Needed when using `outbox_entry_causality_key_resolver`
|
48
48
|
config.unprocessed_causality_keys_limit = 100_000 # not required, defaults to 10_000. Might be a good idea to decrease the value when you start experiencing OOMs - they are likely to be caused by fetching too many causality keys. It is likely to happen when you have huge amount of records to process.
|
49
|
+
|
50
|
+
config.datadog_statsd_client = Datadog::Statsd.new("localhost", 8125, namespace: "application_name.production") # needed only for latency tracking, defaults to `nil`
|
51
|
+
config.high_priority_sidekiq_queue = :critical # not required, defaults to `:rails_transactional_outbox_high_priority`
|
49
52
|
end
|
50
53
|
end
|
51
54
|
```
|
@@ -247,10 +250,40 @@ tartarus.register do |item|
|
|
247
250
|
end
|
248
251
|
```
|
249
252
|
|
253
|
+
### Outbox Processing Latency Tracking
|
254
|
+
|
255
|
+
It's highly recommended to tracking latency of processing outbox records defined as the difference between the `processed` and `created_at` timestamps.
|
256
|
+
|
257
|
+
``` rb
|
258
|
+
RailsTransactionalOutbox.configure do |config|
|
259
|
+
config.datadog_statsd_client = Datadog::Statsd.new("localhost", 8125, namespace: "application_name.production") # required for latency tracking, defaults to `nil`
|
260
|
+
config.high_priority_sidekiq_queue = :critical # not required, defaults to `:rails_transactional_outbox_high_priority`
|
261
|
+
end
|
262
|
+
```
|
263
|
+
|
264
|
+
You also need to add a job to the sidekiq-cron schedule that will run every 1 minute:
|
265
|
+
|
266
|
+
``` rb
|
267
|
+
Sidekiq.configure_server do |config|
|
268
|
+
config.on(:startup) do
|
269
|
+
RailsTransactionalOutbox::DatadogLatencyReporterScheduler.new.add_to_schedule
|
270
|
+
end
|
271
|
+
end
|
272
|
+
```
|
273
|
+
|
274
|
+
With this setup, you will have the following metrics available on DataDog:
|
275
|
+
|
276
|
+
- `"#{namespace}.rails_transactional_outbox.latency.minimum"`
|
277
|
+
- `"#{namespace}.rails_transactional_outbox.latency.maximum"`
|
278
|
+
- `"#{namespace}.rails_transactional_outbox.latency.average"`
|
279
|
+
- `"#{namespace}.rails_transactional_outbox.latency.highest_since_creation_date`
|
280
|
+
|
281
|
+
|
282
|
+
|
250
283
|
### Health Checks
|
251
284
|
|
252
285
|
|
253
|
-
|
286
|
+
You need to explicitly enable the health check (e.g. in the initializer):
|
254
287
|
|
255
288
|
``` rb
|
256
289
|
RailsTransactionalOutbox.enable_outbox_worker_healthcheck
|
@@ -2,11 +2,15 @@
|
|
2
2
|
|
3
3
|
class RailsTransactionalOutbox
|
4
4
|
class Configuration
|
5
|
-
attr_accessor :database_connection_provider, :logger, :outbox_model, :transaction_provider
|
5
|
+
attr_accessor :database_connection_provider, :logger, :outbox_model, :transaction_provider, :datadog_statsd_client
|
6
6
|
attr_writer :error_handler, :transactional_outbox_worker_sleep_seconds,
|
7
7
|
:transactional_outbox_worker_idle_delay_multiplier, :outbox_batch_size, :outbox_entries_processor,
|
8
8
|
:lock_client, :lock_expiry_time, :outbox_entry_causality_key_resolver,
|
9
|
-
:raise_not_found_model_error, :unprocessed_causality_keys_limit
|
9
|
+
:raise_not_found_model_error, :unprocessed_causality_keys_limit, :high_priority_sidekiq_queue
|
10
|
+
|
11
|
+
def self.high_priority_sidekiq_queue
|
12
|
+
:rails_transactional_outbox_high_priority
|
13
|
+
end
|
10
14
|
|
11
15
|
def error_handler
|
12
16
|
@error_handler || RailsTransactionalOutbox::ErrorHandlers::NullErrorHandler
|
@@ -61,5 +65,11 @@ class RailsTransactionalOutbox
|
|
61
65
|
|
62
66
|
10_000
|
63
67
|
end
|
68
|
+
|
69
|
+
def high_priority_sidekiq_queue
|
70
|
+
return @high_priority_sidekiq_queue if defined?(@high_priority_sidekiq_queue)
|
71
|
+
|
72
|
+
self.class.high_priority_sidekiq_queue
|
73
|
+
end
|
64
74
|
end
|
65
75
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class RailsTransactionalOutbox
|
4
|
+
class DatadogLatencyReporter
|
5
|
+
attr_reader :config
|
6
|
+
private :config
|
7
|
+
|
8
|
+
def initialize(config: RailsTransactionalOutbox.configuration)
|
9
|
+
@config = config
|
10
|
+
end
|
11
|
+
|
12
|
+
def report(latency: generate_latency)
|
13
|
+
datadog_statsd_client.gauge("rails_transactional_outbox.latency.minimum", latency.minimum)
|
14
|
+
datadog_statsd_client.gauge("rails_transactional_outbox.latency.maximum", latency.maximum)
|
15
|
+
datadog_statsd_client.gauge("rails_transactional_outbox.latency.average", latency.average)
|
16
|
+
datadog_statsd_client.gauge("rails_transactional_outbox.latency.highest_since_creation_date",
|
17
|
+
latency.highest_since_creation_date)
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
delegate :datadog_statsd_client, :datadog_statsd_prefix, to: :config
|
23
|
+
|
24
|
+
def generate_latency
|
25
|
+
RailsTransactionalOutbox::LatencyTracker.new.calculate
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class RailsTransactionalOutbox
|
4
|
+
class DatadogLatencyReporterJob
|
5
|
+
include Sidekiq::Worker
|
6
|
+
|
7
|
+
sidekiq_options queue: RailsTransactionalOutbox::Configuration.high_priority_sidekiq_queue
|
8
|
+
|
9
|
+
def perform
|
10
|
+
RailsTransactionalOutbox::DatadogLatencyReporter.new.report
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class RailsTransactionalOutbox
|
4
|
+
class DatadogLatencyReporterScheduler
|
5
|
+
JOB_NAME = "rails_transactional_outbox_datadog_latency_reporter_job"
|
6
|
+
EVERY_MINUTE_IN_CRON_SYNTAX = "* * * * *"
|
7
|
+
JOB_CLASS_NAME = "RailsTransactionalOutbox::DatadogLatencyReporterJob"
|
8
|
+
JOB_DESCRIPTION = "Collect latency metrics from rails-transactional-outbox and send them to Datadog"
|
9
|
+
|
10
|
+
private_constant :JOB_NAME, :EVERY_MINUTE_IN_CRON_SYNTAX, :JOB_CLASS_NAME, :JOB_DESCRIPTION
|
11
|
+
|
12
|
+
attr_reader :config
|
13
|
+
private :config
|
14
|
+
|
15
|
+
def initialize(config: RailsTransactionalOutbox.configuration)
|
16
|
+
@config = config
|
17
|
+
end
|
18
|
+
|
19
|
+
def add_to_schedule
|
20
|
+
find || create
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def find
|
26
|
+
Sidekiq::Cron::Job.find(name: JOB_NAME)
|
27
|
+
end
|
28
|
+
|
29
|
+
def create
|
30
|
+
Sidekiq::Cron::Job.create(create_job_arguments)
|
31
|
+
end
|
32
|
+
|
33
|
+
def create_job_arguments
|
34
|
+
{
|
35
|
+
name: JOB_NAME,
|
36
|
+
cron: EVERY_MINUTE_IN_CRON_SYNTAX,
|
37
|
+
class: JOB_CLASS_NAME,
|
38
|
+
queue: config.high_priority_sidekiq_queue,
|
39
|
+
active_job: false,
|
40
|
+
description: JOB_DESCRIPTION,
|
41
|
+
date_as_argument: false
|
42
|
+
}
|
43
|
+
end
|
44
|
+
|
45
|
+
def every_minute_to_cron_syntax
|
46
|
+
EVERY_MINUTE_IN_CRON_SYNTAX
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -7,7 +7,7 @@ class RailsTransactionalOutbox
|
|
7
7
|
private_constant :KEY_PREFIX, :TMP_DIR
|
8
8
|
|
9
9
|
def self.check(hostname: ENV.fetch("HOSTNAME", nil), expiry_time_in_seconds: 120)
|
10
|
-
new(hostname
|
10
|
+
new(hostname:, expiry_time_in_seconds:).check
|
11
11
|
end
|
12
12
|
|
13
13
|
attr_reader :hostname, :expiry_time_in_seconds
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class RailsTransactionalOutbox
|
4
|
+
class LatencyTracker
|
5
|
+
LatencyTrackerResult = Struct.new(:minimum, :maximum, :average, :highest_since_creation_date)
|
6
|
+
private_constant :LatencyTrackerResult
|
7
|
+
|
8
|
+
attr_reader :config, :clock
|
9
|
+
private :config, :clock
|
10
|
+
|
11
|
+
def initialize(config: RailsTransactionalOutbox.configuration, clock: Time)
|
12
|
+
@config = config
|
13
|
+
@clock = clock
|
14
|
+
end
|
15
|
+
|
16
|
+
def calculate(interval: 1.minute)
|
17
|
+
records = outbox_model.processed_since(interval.ago)
|
18
|
+
latencies = records.map(&:processing_latency)
|
19
|
+
|
20
|
+
LatencyTrackerResult.new(
|
21
|
+
latencies.min.to_d,
|
22
|
+
latencies.max.to_d,
|
23
|
+
calculate_average(latencies),
|
24
|
+
calculate_highest_since_creation_date.to_d
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
delegate :outbox_model, to: :config
|
31
|
+
|
32
|
+
def calculate_average(latencies)
|
33
|
+
if latencies.any?
|
34
|
+
latencies.sum.to_d / latencies.size
|
35
|
+
else
|
36
|
+
0
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def calculate_highest_since_creation_date
|
41
|
+
minimum_created_at_from_not_processed = outbox_model.not_processed.minimum(:created_at) or return 0
|
42
|
+
clock.current - minimum_created_at_from_not_processed
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -10,17 +10,17 @@ class RailsTransactionalOutbox
|
|
10
10
|
@config = config
|
11
11
|
end
|
12
12
|
|
13
|
-
def call(&
|
13
|
+
def call(&)
|
14
14
|
return [] unless outbox_model.any_records_to_process?
|
15
15
|
|
16
|
-
execute(&
|
16
|
+
execute(&)
|
17
17
|
end
|
18
18
|
|
19
19
|
private
|
20
20
|
|
21
21
|
delegate :outbox_model, :outbox_batch_size, to: :config
|
22
22
|
|
23
|
-
def execute(&
|
23
|
+
def execute(&)
|
24
24
|
raise "implement me"
|
25
25
|
end
|
26
26
|
|
@@ -8,10 +8,10 @@ class RailsTransactionalOutbox
|
|
8
8
|
delegate :transaction_provider, to: :config
|
9
9
|
delegate :transaction, to: :transaction_provider
|
10
10
|
|
11
|
-
def execute(&
|
11
|
+
def execute(&)
|
12
12
|
transaction do
|
13
13
|
outbox_model.fetch_processable(outbox_batch_size).to_a.tap do |records_to_process|
|
14
|
-
process_records(records_to_process, &
|
14
|
+
process_records(records_to_process, &)
|
15
15
|
end
|
16
16
|
end
|
17
17
|
end
|
data/lib/rails_transactional_outbox/outbox_entries_processors/ordered_by_causality_key_processor.rb
CHANGED
@@ -7,12 +7,12 @@ class RailsTransactionalOutbox
|
|
7
7
|
|
8
8
|
delegate :lock_client, :lock_expiry_time, to: :config
|
9
9
|
|
10
|
-
def execute(&
|
10
|
+
def execute(&)
|
11
11
|
unprocessed_causality_keys.each_with_object([]) do |causality_key, processed_records|
|
12
12
|
lock_client.lock(lock_name(causality_key), lock_expiry_time) do |locked|
|
13
13
|
next unless locked
|
14
14
|
|
15
|
-
processed_records.concat(fetch_records(causality_key).tap { |records| process_records(records, &
|
15
|
+
processed_records.concat(fetch_records(causality_key).tap { |records| process_records(records, &) })
|
16
16
|
end
|
17
17
|
end
|
18
18
|
end
|
@@ -14,7 +14,7 @@ class RailsTransactionalOutbox
|
|
14
14
|
|
15
15
|
scope :fetch_processable_for_causality_key, lambda { |batch_size, causality_key|
|
16
16
|
processable_now
|
17
|
-
.where(causality_key:
|
17
|
+
.where(causality_key:)
|
18
18
|
.order(created_at: :asc)
|
19
19
|
.limit(batch_size)
|
20
20
|
}
|
@@ -24,6 +24,9 @@ class RailsTransactionalOutbox
|
|
24
24
|
.where("retry_at IS NULL OR retry_at <= ?", Time.current)
|
25
25
|
}
|
26
26
|
|
27
|
+
scope :processed_since, ->(time) { where("processed_at >= ?", time) }
|
28
|
+
scope :not_processed, -> { where(processed_at: nil) }
|
29
|
+
|
27
30
|
def self.unprocessed_causality_keys(limit: RailsTransactionalOutbox.configuration.unprocessed_causality_keys_limit)
|
28
31
|
processable_now
|
29
32
|
.select("causality_key")
|
@@ -96,5 +99,11 @@ class RailsTransactionalOutbox
|
|
96
99
|
rescue ActiveRecord::RecordNotFound
|
97
100
|
model_klass.new(id: resource_id) if RailsTransactionalOutbox::EventType.new(event_type).destroy?
|
98
101
|
end
|
102
|
+
|
103
|
+
def processing_latency
|
104
|
+
return unless processed?
|
105
|
+
|
106
|
+
processed_at - created_at
|
107
|
+
end
|
99
108
|
end
|
100
109
|
end
|
@@ -29,20 +29,20 @@ class RailsTransactionalOutbox
|
|
29
29
|
reliable_after_commit_callbacks << ReliableCallback.new(callback_proc, final_options)
|
30
30
|
end
|
31
31
|
|
32
|
-
def self.reliable_after_create_commit(method_name = NOT_PROVIDED, options = {}, &
|
33
|
-
reliable_after_commit(method_name, options.merge(on: :create), &
|
32
|
+
def self.reliable_after_create_commit(method_name = NOT_PROVIDED, options = {}, &)
|
33
|
+
reliable_after_commit(method_name, options.merge(on: :create), &)
|
34
34
|
end
|
35
35
|
|
36
|
-
def self.reliable_after_update_commit(method_name = NOT_PROVIDED, options = {}, &
|
37
|
-
reliable_after_commit(method_name, options.merge(on: :update), &
|
36
|
+
def self.reliable_after_update_commit(method_name = NOT_PROVIDED, options = {}, &)
|
37
|
+
reliable_after_commit(method_name, options.merge(on: :update), &)
|
38
38
|
end
|
39
39
|
|
40
|
-
def self.reliable_after_destroy_commit(method_name = NOT_PROVIDED, options = {}, &
|
41
|
-
reliable_after_commit(method_name, options.merge(on: :destroy), &
|
40
|
+
def self.reliable_after_destroy_commit(method_name = NOT_PROVIDED, options = {}, &)
|
41
|
+
reliable_after_commit(method_name, options.merge(on: :destroy), &)
|
42
42
|
end
|
43
43
|
|
44
|
-
def self.reliable_after_save_commit(method_name = NOT_PROVIDED, options = {}, &
|
45
|
-
reliable_after_commit(method_name, options.merge(on: %i[create update]), &
|
44
|
+
def self.reliable_after_save_commit(method_name = NOT_PROVIDED, options = {}, &)
|
45
|
+
reliable_after_commit(method_name, options.merge(on: %i[create update]), &)
|
46
46
|
end
|
47
47
|
|
48
48
|
alias_method :original_previous_changes, :previous_changes
|
@@ -6,6 +6,6 @@ namespace :rails_transactional_outbox do
|
|
6
6
|
$stdout.sync = true
|
7
7
|
Rails.logger.info("Running rails_transactional_outbox:worker rake task.")
|
8
8
|
threads_number = ENV.fetch("RAILS_TRANSACTIONAL_OUTBOX_THREADS_NUMBER", 1).to_i
|
9
|
-
RailsTransactionalOutbox.start_outbox_worker(threads_number:
|
9
|
+
RailsTransactionalOutbox.start_outbox_worker(threads_number:)
|
10
10
|
end
|
11
11
|
end
|
@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
|
|
12
12
|
spec.description = "An implementation of transactional outbox pattern to be used with Rails."
|
13
13
|
spec.homepage = "https://github.com/BookingSync/rails-transactional-outbox"
|
14
14
|
spec.license = "MIT"
|
15
|
-
spec.required_ruby_version = Gem::Requirement.new(">=
|
15
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 3.1.0")
|
16
16
|
|
17
17
|
# spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
|
18
18
|
|
@@ -33,11 +33,17 @@ Gem::Specification.new do |spec|
|
|
33
33
|
spec.add_dependency "activerecord", ">= 5"
|
34
34
|
spec.add_dependency "activesupport", ">= 3.2"
|
35
35
|
spec.add_dependency "concurrent-ruby"
|
36
|
+
spec.add_dependency "dogstatsd-ruby"
|
36
37
|
spec.add_dependency "dry-monitor"
|
37
38
|
spec.add_dependency "file-based-healthcheck"
|
39
|
+
spec.add_dependency "sidekiq", ">= 5.0"
|
40
|
+
spec.add_dependency "sidekiq-cron"
|
38
41
|
spec.add_dependency "sigurd"
|
39
42
|
spec.add_dependency "zeitwerk"
|
40
43
|
|
44
|
+
spec.add_development_dependency "redis-namespace"
|
45
|
+
spec.add_development_dependency "rspec-sidekiq"
|
46
|
+
|
41
47
|
# For more information and examples about making a new gem, checkout our
|
42
48
|
# guide at: https://bundler.io/guides/creating_gem.html
|
43
49
|
spec.metadata["rubygems_mfa_required"] = "true"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rails-transactional-outbox
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Karol Galanciak
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-08-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: dogstatsd-ruby
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: dry-monitor
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -80,6 +94,34 @@ dependencies:
|
|
80
94
|
- - ">="
|
81
95
|
- !ruby/object:Gem::Version
|
82
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: sidekiq
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '5.0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '5.0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: sidekiq-cron
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
83
125
|
- !ruby/object:Gem::Dependency
|
84
126
|
name: sigurd
|
85
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -108,6 +150,34 @@ dependencies:
|
|
108
150
|
- - ">="
|
109
151
|
- !ruby/object:Gem::Version
|
110
152
|
version: '0'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: redis-namespace
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - ">="
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '0'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: rspec-sidekiq
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - ">="
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
174
|
+
type: :development
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - ">="
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '0'
|
111
181
|
description: An implementation of transactional outbox pattern to be used with Rails.
|
112
182
|
email:
|
113
183
|
- karol.galanciak@gmail.com
|
@@ -133,11 +203,15 @@ files:
|
|
133
203
|
- lib/rails-transactional-outbox.rb
|
134
204
|
- lib/rails_transactional_outbox.rb
|
135
205
|
- lib/rails_transactional_outbox/configuration.rb
|
206
|
+
- lib/rails_transactional_outbox/datadog_latency_reporter.rb
|
207
|
+
- lib/rails_transactional_outbox/datadog_latency_reporter_job.rb
|
208
|
+
- lib/rails_transactional_outbox/datadog_latency_reporter_scheduler.rb
|
136
209
|
- lib/rails_transactional_outbox/error_handlers.rb
|
137
210
|
- lib/rails_transactional_outbox/error_handlers/null_error_handler.rb
|
138
211
|
- lib/rails_transactional_outbox/event_type.rb
|
139
212
|
- lib/rails_transactional_outbox/exponential_backoff.rb
|
140
213
|
- lib/rails_transactional_outbox/health_check.rb
|
214
|
+
- lib/rails_transactional_outbox/latency_tracker.rb
|
141
215
|
- lib/rails_transactional_outbox/monitor.rb
|
142
216
|
- lib/rails_transactional_outbox/null_lock_client.rb
|
143
217
|
- lib/rails_transactional_outbox/outbox_entries_processor.rb
|
@@ -179,14 +253,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
179
253
|
requirements:
|
180
254
|
- - ">="
|
181
255
|
- !ruby/object:Gem::Version
|
182
|
-
version:
|
256
|
+
version: 3.1.0
|
183
257
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
184
258
|
requirements:
|
185
259
|
- - ">="
|
186
260
|
- !ruby/object:Gem::Version
|
187
261
|
version: '0'
|
188
262
|
requirements: []
|
189
|
-
rubygems_version: 3.
|
263
|
+
rubygems_version: 3.4.19
|
190
264
|
signing_key:
|
191
265
|
specification_version: 4
|
192
266
|
summary: An implementation of transactional outbox pattern to be used with Rails.
|