ruby_event_store-outbox 0.0.25 → 0.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/generators/ruby_event_store/outbox/migration_generator.rb +2 -2
- data/lib/generators/ruby_event_store/outbox/templates/{create_event_store_outbox_template.rb → create_event_store_outbox_template.erb} +4 -4
- data/lib/ruby_event_store/outbox/batch_result.rb +26 -0
- data/lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb +2 -0
- data/lib/ruby_event_store/outbox/cleanup_strategies/none.rb +2 -0
- data/lib/ruby_event_store/outbox/cleanup_strategies.rb +23 -0
- data/lib/ruby_event_store/outbox/cli.rb +10 -6
- data/lib/ruby_event_store/outbox/configuration.rb +50 -0
- data/lib/ruby_event_store/outbox/consumer.rb +39 -105
- data/lib/ruby_event_store/outbox/fetch_specification.rb +3 -3
- data/lib/ruby_event_store/outbox/metrics/influx.rb +2 -0
- data/lib/ruby_event_store/outbox/metrics/null.rb +2 -0
- data/lib/ruby_event_store/outbox/metrics/test.rb +2 -0
- data/lib/ruby_event_store/outbox/metrics.rb +2 -0
- data/lib/ruby_event_store/outbox/repository.rb +12 -12
- data/lib/ruby_event_store/outbox/runner.rb +43 -0
- data/lib/ruby_event_store/outbox/sidekiq_processor.rb +11 -5
- data/lib/ruby_event_store/outbox/sidekiq_producer.rb +9 -1
- data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb +1 -1
- data/lib/ruby_event_store/outbox/tempo.rb +25 -0
- data/lib/ruby_event_store/outbox/version.rb +1 -1
- data/lib/ruby_event_store/outbox.rb +5 -1
- metadata +14 -11
- data/lib/ruby_event_store/outbox/cli.rb.orig +0 -132
- data/lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb +0 -25
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a230a2cd8a85d32c26cb83c6dc879e7bee7f6a98c33c46f6376277a821805b0e
|
4
|
+
data.tar.gz: 184638665971b91e31e83ab937290e789639063d2e00942ed28a92907ca92545
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a49ccb2ed6e8c47683baa4a61bf88614adcbf17ede978409e7cb4c4ebe5294008a5c8ec1e20e555aa76108c5ed120f48853988ea1c510f8085f5a2dbfe4fc8b2
|
7
|
+
data.tar.gz: 62c952a369da38606b402c19cef8d7946ce9c9f8c62073615c111dbe68a7be598104c92ef07491a6841572cc85da7e3063cba7edeec73ae8a0d9946cf17ffaef
|
@@ -12,13 +12,13 @@ if defined?(Rails::Generators::Base)
|
|
12
12
|
source_root File.expand_path(File.join(File.dirname(__FILE__), "./templates"))
|
13
13
|
|
14
14
|
def create_migration
|
15
|
-
template "create_event_store_outbox_template.
|
15
|
+
template "create_event_store_outbox_template.erb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
|
16
16
|
end
|
17
17
|
|
18
18
|
private
|
19
19
|
|
20
20
|
def migration_version
|
21
|
-
|
21
|
+
::ActiveRecord::Migration.current_version
|
22
22
|
end
|
23
23
|
|
24
24
|
def timestamp
|
@@ -1,13 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
class CreateEventStoreOutbox < ActiveRecord::Migration<%= migration_version %>
|
3
|
+
class CreateEventStoreOutbox < ActiveRecord::Migration[<%= migration_version %>]
|
4
4
|
def change
|
5
5
|
create_table(:event_store_outbox, force: false) do |t|
|
6
6
|
t.string :split_key, null: true
|
7
7
|
t.string :format, null: false
|
8
8
|
t.binary :payload, null: false
|
9
|
-
t.datetime :created_at, null: false
|
10
|
-
t.datetime :enqueued_at, null: true
|
9
|
+
t.datetime :created_at, null: false, precision: 6
|
10
|
+
t.datetime :enqueued_at, null: true, precision: 6
|
11
11
|
end
|
12
12
|
add_index :event_store_outbox, [:format, :enqueued_at, :split_key], name: "index_event_store_outbox_for_pool"
|
13
13
|
add_index :event_store_outbox, [:created_at, :enqueued_at], name: "index_event_store_outbox_for_clear"
|
@@ -15,7 +15,7 @@ class CreateEventStoreOutbox < ActiveRecord::Migration<%= migration_version %>
|
|
15
15
|
create_table(:event_store_outbox_locks, force: false) do |t|
|
16
16
|
t.string :format, null: false
|
17
17
|
t.string :split_key, null: false
|
18
|
-
t.datetime :locked_at, null: true
|
18
|
+
t.datetime :locked_at, null: true, precision: 6
|
19
19
|
t.string :locked_by, null: true, limit: 36
|
20
20
|
end
|
21
21
|
add_index :event_store_outbox_locks, [:format, :split_key], name: "index_event_store_outbox_locks_for_locking", unique: true
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RubyEventStore
|
4
|
+
module Outbox
|
5
|
+
class BatchResult
|
6
|
+
def self.empty
|
7
|
+
new
|
8
|
+
end
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@success_count = 0
|
12
|
+
@failed_count = 0
|
13
|
+
end
|
14
|
+
|
15
|
+
attr_reader :success_count, :failed_count
|
16
|
+
|
17
|
+
def count_success!
|
18
|
+
@success_count += 1
|
19
|
+
end
|
20
|
+
|
21
|
+
def count_failed!
|
22
|
+
@failed_count += 1
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "cleanup_strategies/none"
|
4
|
+
require_relative "cleanup_strategies/clean_old_enqueued"
|
5
|
+
|
6
|
+
module RubyEventStore
|
7
|
+
module Outbox
|
8
|
+
module CleanupStrategies
|
9
|
+
def self.build(configuration, repository)
|
10
|
+
case configuration.cleanup
|
11
|
+
when :none
|
12
|
+
None.new
|
13
|
+
else
|
14
|
+
CleanOldEnqueued.new(
|
15
|
+
repository,
|
16
|
+
ActiveSupport::Duration.parse(configuration.cleanup),
|
17
|
+
configuration.cleanup_limit
|
18
|
+
)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -1,7 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "optparse"
|
2
4
|
require_relative "version"
|
3
5
|
require_relative "consumer"
|
6
|
+
require_relative "runner"
|
4
7
|
require_relative "metrics"
|
8
|
+
require_relative "configuration"
|
5
9
|
|
6
10
|
module RubyEventStore
|
7
11
|
module Outbox
|
@@ -92,15 +96,14 @@ module RubyEventStore
|
|
92
96
|
|
93
97
|
def run(argv)
|
94
98
|
options = Parser.parse(argv)
|
95
|
-
|
96
|
-
|
97
|
-
outbox_consumer.run
|
99
|
+
build_runner(options)
|
100
|
+
.run
|
98
101
|
end
|
99
102
|
|
100
|
-
def
|
103
|
+
def build_runner(options)
|
101
104
|
consumer_uuid = SecureRandom.uuid
|
102
105
|
logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox #{consumer_uuid}")
|
103
|
-
consumer_configuration =
|
106
|
+
consumer_configuration = Configuration.new(
|
104
107
|
split_keys: options.split_keys,
|
105
108
|
message_format: options.message_format,
|
106
109
|
batch_size: options.batch_size,
|
@@ -112,7 +115,8 @@ module RubyEventStore
|
|
112
115
|
)
|
113
116
|
metrics = Metrics.from_url(options.metrics_url)
|
114
117
|
outbox_consumer =
|
115
|
-
|
118
|
+
Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
|
119
|
+
Runner.new(outbox_consumer, consumer_configuration, logger: logger)
|
116
120
|
end
|
117
121
|
end
|
118
122
|
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RubyEventStore
|
4
|
+
module Outbox
|
5
|
+
class Configuration
|
6
|
+
def initialize(
|
7
|
+
split_keys:,
|
8
|
+
message_format:,
|
9
|
+
batch_size:,
|
10
|
+
database_url:,
|
11
|
+
redis_url:,
|
12
|
+
cleanup:,
|
13
|
+
cleanup_limit:,
|
14
|
+
sleep_on_empty:
|
15
|
+
)
|
16
|
+
@split_keys = split_keys
|
17
|
+
@message_format = message_format
|
18
|
+
@batch_size = batch_size || 100
|
19
|
+
@database_url = database_url
|
20
|
+
@redis_url = redis_url
|
21
|
+
@cleanup = cleanup
|
22
|
+
@cleanup_limit = cleanup_limit
|
23
|
+
@sleep_on_empty = sleep_on_empty
|
24
|
+
freeze
|
25
|
+
end
|
26
|
+
|
27
|
+
def with(overriden_options)
|
28
|
+
self.class.new(
|
29
|
+
split_keys: overriden_options.fetch(:split_keys, split_keys),
|
30
|
+
message_format: overriden_options.fetch(:message_format, message_format),
|
31
|
+
batch_size: overriden_options.fetch(:batch_size, batch_size),
|
32
|
+
database_url: overriden_options.fetch(:database_url, database_url),
|
33
|
+
redis_url: overriden_options.fetch(:redis_url, redis_url),
|
34
|
+
cleanup: overriden_options.fetch(:cleanup, cleanup),
|
35
|
+
cleanup_limit: overriden_options.fetch(:cleanup_limit, cleanup_limit),
|
36
|
+
sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
|
37
|
+
)
|
38
|
+
end
|
39
|
+
|
40
|
+
attr_reader :split_keys,
|
41
|
+
:message_format,
|
42
|
+
:batch_size,
|
43
|
+
:database_url,
|
44
|
+
:redis_url,
|
45
|
+
:cleanup,
|
46
|
+
:cleanup_limit,
|
47
|
+
:sleep_on_empty
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -1,110 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "logger"
|
2
|
-
require "redis"
|
4
|
+
require "redis-client"
|
3
5
|
require "active_record"
|
4
6
|
require_relative "repository"
|
5
7
|
require_relative "sidekiq5_format"
|
8
|
+
require_relative "tempo"
|
6
9
|
require_relative "sidekiq_processor"
|
7
10
|
require_relative "fetch_specification"
|
8
|
-
require_relative "cleanup_strategies
|
9
|
-
require_relative "cleanup_strategies/clean_old_enqueued"
|
11
|
+
require_relative "cleanup_strategies"
|
10
12
|
|
11
13
|
module RubyEventStore
|
12
14
|
module Outbox
|
13
15
|
class Consumer
|
14
16
|
MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK = 10
|
15
17
|
|
16
|
-
class Configuration
|
17
|
-
def initialize(
|
18
|
-
split_keys:,
|
19
|
-
message_format:,
|
20
|
-
batch_size:,
|
21
|
-
database_url:,
|
22
|
-
redis_url:,
|
23
|
-
cleanup:,
|
24
|
-
cleanup_limit:,
|
25
|
-
sleep_on_empty:
|
26
|
-
)
|
27
|
-
@split_keys = split_keys
|
28
|
-
@message_format = message_format
|
29
|
-
@batch_size = batch_size || 100
|
30
|
-
@database_url = database_url
|
31
|
-
@redis_url = redis_url
|
32
|
-
@cleanup = cleanup
|
33
|
-
@cleanup_limit = cleanup_limit
|
34
|
-
@sleep_on_empty = sleep_on_empty
|
35
|
-
freeze
|
36
|
-
end
|
37
|
-
|
38
|
-
def with(overriden_options)
|
39
|
-
self.class.new(
|
40
|
-
split_keys: overriden_options.fetch(:split_keys, split_keys),
|
41
|
-
message_format: overriden_options.fetch(:message_format, message_format),
|
42
|
-
batch_size: overriden_options.fetch(:batch_size, batch_size),
|
43
|
-
database_url: overriden_options.fetch(:database_url, database_url),
|
44
|
-
redis_url: overriden_options.fetch(:redis_url, redis_url),
|
45
|
-
cleanup: overriden_options.fetch(:cleanup, cleanup),
|
46
|
-
cleanup_limit: overriden_options.fetch(:cleanup_limit, cleanup_limit),
|
47
|
-
sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
|
48
|
-
)
|
49
|
-
end
|
50
|
-
|
51
|
-
attr_reader :split_keys,
|
52
|
-
:message_format,
|
53
|
-
:batch_size,
|
54
|
-
:database_url,
|
55
|
-
:redis_url,
|
56
|
-
:cleanup,
|
57
|
-
:cleanup_limit,
|
58
|
-
:sleep_on_empty
|
59
|
-
end
|
60
|
-
|
61
18
|
def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
|
62
19
|
@split_keys = configuration.split_keys
|
63
20
|
@clock = clock
|
64
21
|
@logger = logger
|
65
22
|
@metrics = metrics
|
66
|
-
@
|
67
|
-
@sleep_on_empty = configuration.sleep_on_empty
|
23
|
+
@tempo = Tempo.new(configuration.batch_size)
|
68
24
|
@consumer_uuid = consumer_uuid
|
69
25
|
|
70
26
|
raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
|
71
|
-
|
72
|
-
|
73
|
-
@gracefully_shutting_down = false
|
74
|
-
prepare_traps
|
27
|
+
redis_config = RedisClient.config(url: configuration.redis_url)
|
28
|
+
@processor = SidekiqProcessor.new(redis_config.new_client)
|
75
29
|
|
76
30
|
@repository = Repository.new(configuration.database_url)
|
77
|
-
@cleanup_strategy =
|
78
|
-
case configuration.cleanup
|
79
|
-
when :none
|
80
|
-
CleanupStrategies::None.new
|
81
|
-
else
|
82
|
-
CleanupStrategies::CleanOldEnqueued.new(
|
83
|
-
repository,
|
84
|
-
ActiveSupport::Duration.parse(configuration.cleanup),
|
85
|
-
configuration.cleanup_limit
|
86
|
-
)
|
87
|
-
end
|
31
|
+
@cleanup_strategy = CleanupStrategies.build(configuration, repository)
|
88
32
|
end
|
89
33
|
|
90
|
-
def
|
91
|
-
|
92
|
-
logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
|
93
|
-
end
|
94
|
-
|
95
|
-
def run
|
96
|
-
while !@gracefully_shutting_down
|
97
|
-
was_something_changed = one_loop
|
98
|
-
if !was_something_changed
|
99
|
-
STDOUT.flush
|
100
|
-
sleep sleep_on_empty
|
101
|
-
end
|
102
|
-
end
|
103
|
-
logger.info "Gracefully shutting down"
|
104
|
-
end
|
105
|
-
|
106
|
-
def one_loop
|
107
|
-
remaining_split_keys = @split_keys.dup
|
34
|
+
def process
|
35
|
+
remaining_split_keys = split_keys.dup
|
108
36
|
|
109
37
|
was_something_changed = false
|
110
38
|
while (split_key = remaining_split_keys.shift)
|
@@ -123,31 +51,27 @@ module RubyEventStore
|
|
123
51
|
batch = retrieve_batch(fetch_specification)
|
124
52
|
break if batch.empty?
|
125
53
|
|
126
|
-
|
127
|
-
updated_record_ids = []
|
54
|
+
batch_result = BatchResult.empty
|
128
55
|
batch.each do |record|
|
129
|
-
|
56
|
+
handle_failure(batch_result) do
|
130
57
|
now = @clock.now.utc
|
131
58
|
processor.process(record, now)
|
132
59
|
|
133
60
|
repository.mark_as_enqueued(record, now)
|
134
61
|
something_processed |= true
|
135
|
-
|
136
|
-
rescue => e
|
137
|
-
failed_record_ids << record.id
|
138
|
-
e.full_message.split($/).each { |line| logger.error(line) }
|
62
|
+
batch_result.count_success!
|
139
63
|
end
|
140
64
|
end
|
141
65
|
|
142
66
|
metrics.write_point_queue(
|
143
|
-
enqueued:
|
144
|
-
failed:
|
67
|
+
enqueued: batch_result.success_count,
|
68
|
+
failed: batch_result.failed_count,
|
145
69
|
format: fetch_specification.message_format,
|
146
70
|
split_key: fetch_specification.split_key,
|
147
71
|
remaining: get_remaining_count(fetch_specification)
|
148
72
|
)
|
149
73
|
|
150
|
-
logger.info "Sent #{
|
74
|
+
logger.info "Sent #{batch_result.success_count} messages from outbox table"
|
151
75
|
|
152
76
|
refresh_successful = refresh_lock_for_process(obtained_lock)
|
153
77
|
break unless refresh_successful
|
@@ -174,13 +98,32 @@ module RubyEventStore
|
|
174
98
|
|
175
99
|
attr_reader :split_keys,
|
176
100
|
:logger,
|
177
|
-
:batch_size,
|
178
101
|
:metrics,
|
179
102
|
:processor,
|
180
103
|
:consumer_uuid,
|
181
104
|
:repository,
|
182
105
|
:cleanup_strategy,
|
183
|
-
:
|
106
|
+
:tempo
|
107
|
+
|
108
|
+
def handle_failure(batch_result)
|
109
|
+
retried = false
|
110
|
+
yield
|
111
|
+
rescue RetriableError => error
|
112
|
+
if retried
|
113
|
+
batch_result.count_failed!
|
114
|
+
log_error(error)
|
115
|
+
else
|
116
|
+
retried = true
|
117
|
+
retry
|
118
|
+
end
|
119
|
+
rescue => error
|
120
|
+
batch_result.count_failed!
|
121
|
+
log_error(error)
|
122
|
+
end
|
123
|
+
|
124
|
+
def log_error(e)
|
125
|
+
e.full_message.split($/).each { |line| logger.error(line) }
|
126
|
+
end
|
184
127
|
|
185
128
|
def obtain_lock_for_process(fetch_specification)
|
186
129
|
result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
|
@@ -251,17 +194,8 @@ module RubyEventStore
|
|
251
194
|
end
|
252
195
|
end
|
253
196
|
|
254
|
-
def prepare_traps
|
255
|
-
Signal.trap("INT") { initiate_graceful_shutdown }
|
256
|
-
Signal.trap("TERM") { initiate_graceful_shutdown }
|
257
|
-
end
|
258
|
-
|
259
|
-
def initiate_graceful_shutdown
|
260
|
-
@gracefully_shutting_down = true
|
261
|
-
end
|
262
|
-
|
263
197
|
def retrieve_batch(fetch_specification)
|
264
|
-
repository.retrieve_batch(fetch_specification, batch_size)
|
198
|
+
repository.retrieve_batch(fetch_specification, tempo.batch_size)
|
265
199
|
end
|
266
200
|
|
267
201
|
def get_remaining_count(fetch_specification)
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module RubyEventStore
|
2
4
|
module Outbox
|
3
5
|
class FetchSpecification
|
@@ -13,10 +15,8 @@ module RubyEventStore
|
|
13
15
|
other.instance_of?(self.class) && other.message_format.eql?(message_format) && other.split_key.eql?(split_key)
|
14
16
|
end
|
15
17
|
|
16
|
-
BIG_VALUE = 0b111111100100010000010010110010101011011101110101001100100110000
|
17
|
-
|
18
18
|
def hash
|
19
|
-
[
|
19
|
+
[message_format, split_key].hash ^ self.class.hash
|
20
20
|
end
|
21
21
|
|
22
22
|
alias_method :eql?, :==
|
@@ -43,9 +43,9 @@ module RubyEventStore
|
|
43
43
|
l
|
44
44
|
end
|
45
45
|
end
|
46
|
-
rescue ActiveRecord::Deadlocked
|
46
|
+
rescue ::ActiveRecord::Deadlocked
|
47
47
|
:deadlocked
|
48
|
-
rescue ActiveRecord::LockWaitTimeout
|
48
|
+
rescue ::ActiveRecord::LockWaitTimeout
|
49
49
|
:lock_timeout
|
50
50
|
end
|
51
51
|
|
@@ -60,9 +60,9 @@ module RubyEventStore
|
|
60
60
|
:stolen
|
61
61
|
end
|
62
62
|
end
|
63
|
-
rescue ActiveRecord::Deadlocked
|
63
|
+
rescue ::ActiveRecord::Deadlocked
|
64
64
|
:deadlocked
|
65
|
-
rescue ActiveRecord::LockWaitTimeout
|
65
|
+
rescue ::ActiveRecord::LockWaitTimeout
|
66
66
|
:lock_timeout
|
67
67
|
end
|
68
68
|
|
@@ -76,9 +76,9 @@ module RubyEventStore
|
|
76
76
|
:ok
|
77
77
|
end
|
78
78
|
end
|
79
|
-
rescue ActiveRecord::Deadlocked
|
79
|
+
rescue ::ActiveRecord::Deadlocked
|
80
80
|
:deadlocked
|
81
|
-
rescue ActiveRecord::LockWaitTimeout
|
81
|
+
rescue ::ActiveRecord::LockWaitTimeout
|
82
82
|
:lock_timeout
|
83
83
|
end
|
84
84
|
|
@@ -105,7 +105,7 @@ module RubyEventStore
|
|
105
105
|
if l.nil?
|
106
106
|
begin
|
107
107
|
l = create!(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
|
108
|
-
rescue ActiveRecord::RecordNotUnique
|
108
|
+
rescue ::ActiveRecord::RecordNotUnique
|
109
109
|
l = lock_for_split_key(fetch_specification)
|
110
110
|
end
|
111
111
|
end
|
@@ -114,9 +114,9 @@ module RubyEventStore
|
|
114
114
|
end
|
115
115
|
|
116
116
|
def initialize(database_url)
|
117
|
-
ActiveRecord::Base.establish_connection(database_url) unless ActiveRecord::Base.connected?
|
118
|
-
if ActiveRecord::Base.connection.adapter_name == "Mysql2"
|
119
|
-
ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
|
117
|
+
::ActiveRecord::Base.establish_connection(database_url) unless ::ActiveRecord::Base.connected?
|
118
|
+
if ::ActiveRecord::Base.connection.adapter_name == "Mysql2"
|
119
|
+
::ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
|
120
120
|
end
|
121
121
|
end
|
122
122
|
|
@@ -145,9 +145,9 @@ module RubyEventStore
|
|
145
145
|
scope = scope.limit(limit).order(:id) unless limit == :all
|
146
146
|
scope.delete_all
|
147
147
|
:ok
|
148
|
-
rescue ActiveRecord::Deadlocked
|
148
|
+
rescue ::ActiveRecord::Deadlocked
|
149
149
|
:deadlocked
|
150
|
-
rescue ActiveRecord::LockWaitTimeout
|
150
|
+
rescue ::ActiveRecord::LockWaitTimeout
|
151
151
|
:lock_timeout
|
152
152
|
end
|
153
153
|
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RubyEventStore
|
4
|
+
module Outbox
|
5
|
+
class Runner
|
6
|
+
def initialize(consumer, configuration, logger:)
|
7
|
+
@consumer = consumer
|
8
|
+
@logger = logger
|
9
|
+
@sleep_on_empty = configuration.sleep_on_empty
|
10
|
+
@split_keys = configuration.split_keys
|
11
|
+
@gracefully_shutting_down = false
|
12
|
+
prepare_traps
|
13
|
+
end
|
14
|
+
|
15
|
+
def run
|
16
|
+
logger.info("Initiated RubyEventStore::Outbox v#{VERSION}")
|
17
|
+
logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
|
18
|
+
|
19
|
+
while !@gracefully_shutting_down
|
20
|
+
was_something_changed = consumer.process
|
21
|
+
if !was_something_changed
|
22
|
+
STDOUT.flush
|
23
|
+
sleep sleep_on_empty
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
logger.info "Gracefully shutting down"
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
attr_reader :consumer, :logger, :sleep_on_empty, :split_keys
|
32
|
+
|
33
|
+
def prepare_traps
|
34
|
+
Signal.trap("INT") { initiate_graceful_shutdown }
|
35
|
+
Signal.trap("TERM") { initiate_graceful_shutdown }
|
36
|
+
end
|
37
|
+
|
38
|
+
def initiate_graceful_shutdown
|
39
|
+
@gracefully_shutting_down = true
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -19,16 +19,15 @@ module RubyEventStore
|
|
19
19
|
raise InvalidPayload.new("Missing queue") if queue.nil? || queue.empty?
|
20
20
|
payload = JSON.generate(parsed_record.merge({ "enqueued_at" => now.to_f }))
|
21
21
|
|
22
|
-
redis.
|
22
|
+
redis.call("LPUSH", "queue:#{queue}", payload)
|
23
23
|
|
24
24
|
@recently_used_queues << queue
|
25
|
+
rescue RedisClient::TimeoutError, RedisClient::ConnectionError
|
26
|
+
raise RetriableError
|
25
27
|
end
|
26
28
|
|
27
29
|
def after_batch
|
28
|
-
|
29
|
-
redis.sadd("queues", @recently_used_queues.to_a)
|
30
|
-
@recently_used_queues.clear
|
31
|
-
end
|
30
|
+
ensure_that_sidekiq_knows_about_all_queues
|
32
31
|
end
|
33
32
|
|
34
33
|
def message_format
|
@@ -37,6 +36,13 @@ module RubyEventStore
|
|
37
36
|
|
38
37
|
private
|
39
38
|
|
39
|
+
def ensure_that_sidekiq_knows_about_all_queues
|
40
|
+
if !@recently_used_queues.empty?
|
41
|
+
redis.call("SADD", "queues", @recently_used_queues.to_a)
|
42
|
+
@recently_used_queues.clear
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
40
46
|
attr_reader :redis
|
41
47
|
end
|
42
48
|
end
|
@@ -8,7 +8,6 @@ module RubyEventStore
|
|
8
8
|
module Outbox
|
9
9
|
class SidekiqProducer
|
10
10
|
def call(klass, args)
|
11
|
-
sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
|
12
11
|
item = { "args" => args.map(&:to_h).map { |h| h.transform_keys(&:to_s) }, "class" => klass }
|
13
12
|
normalized_item = sidekiq_client.__send__(:normalize_item, item)
|
14
13
|
payload =
|
@@ -29,6 +28,15 @@ module RubyEventStore
|
|
29
28
|
private
|
30
29
|
|
31
30
|
attr_reader :repository
|
31
|
+
|
32
|
+
def sidekiq_client
|
33
|
+
@sidekiq_client ||=
|
34
|
+
if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("7.0.0")
|
35
|
+
Sidekiq::Client.new(Sidekiq.redis_pool)
|
36
|
+
else
|
37
|
+
Sidekiq::Client.new(pool: Sidekiq.redis_pool)
|
38
|
+
end
|
39
|
+
end
|
32
40
|
end
|
33
41
|
end
|
34
42
|
end
|
@@ -5,7 +5,7 @@ require_relative "sidekiq_producer"
|
|
5
5
|
module RubyEventStore
|
6
6
|
module Outbox
|
7
7
|
class SidekiqScheduler
|
8
|
-
def initialize(serializer:
|
8
|
+
def initialize(serializer: Serializers::YAML)
|
9
9
|
@serializer = serializer
|
10
10
|
@sidekiq_producer = SidekiqProducer.new
|
11
11
|
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RubyEventStore
|
4
|
+
module Outbox
|
5
|
+
class Tempo
|
6
|
+
EXPONENTIAL_MULTIPLIER = 2
|
7
|
+
|
8
|
+
def initialize(max_batch_size)
|
9
|
+
raise ArgumentError if max_batch_size < 1
|
10
|
+
@max_batch_size = max_batch_size
|
11
|
+
end
|
12
|
+
|
13
|
+
def batch_size
|
14
|
+
@batch_size = next_batch_size
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def next_batch_size
|
20
|
+
return 1 if @batch_size.nil?
|
21
|
+
[@batch_size * EXPONENTIAL_MULTIPLIER, @max_batch_size].min
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -2,11 +2,15 @@
|
|
2
2
|
|
3
3
|
module RubyEventStore
|
4
4
|
module Outbox
|
5
|
+
Error = Class.new(StandardError)
|
6
|
+
RetriableError = Class.new(Error)
|
5
7
|
end
|
6
8
|
end
|
7
9
|
|
8
10
|
require_relative "outbox/fetch_specification"
|
9
11
|
require_relative "outbox/repository"
|
10
12
|
require_relative "outbox/sidekiq_scheduler"
|
11
|
-
require_relative "outbox/legacy_sidekiq_scheduler"
|
12
13
|
require_relative "outbox/version"
|
14
|
+
require_relative "outbox/tempo"
|
15
|
+
require_relative "outbox/batch_result"
|
16
|
+
require_relative "outbox/cleanup_strategies"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby_event_store-outbox
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.27
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arkency
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-04-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ruby_event_store
|
@@ -38,7 +38,7 @@ dependencies:
|
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '6.0'
|
41
|
-
description:
|
41
|
+
description:
|
42
42
|
email: dev@arkency.com
|
43
43
|
executables:
|
44
44
|
- res_outbox
|
@@ -49,24 +49,27 @@ files:
|
|
49
49
|
- README.md
|
50
50
|
- bin/res_outbox
|
51
51
|
- lib/generators/ruby_event_store/outbox/migration_generator.rb
|
52
|
-
- lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.
|
52
|
+
- lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.erb
|
53
53
|
- lib/ruby_event_store/outbox.rb
|
54
|
+
- lib/ruby_event_store/outbox/batch_result.rb
|
55
|
+
- lib/ruby_event_store/outbox/cleanup_strategies.rb
|
54
56
|
- lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb
|
55
57
|
- lib/ruby_event_store/outbox/cleanup_strategies/none.rb
|
56
58
|
- lib/ruby_event_store/outbox/cli.rb
|
57
|
-
- lib/ruby_event_store/outbox/
|
59
|
+
- lib/ruby_event_store/outbox/configuration.rb
|
58
60
|
- lib/ruby_event_store/outbox/consumer.rb
|
59
61
|
- lib/ruby_event_store/outbox/fetch_specification.rb
|
60
|
-
- lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb
|
61
62
|
- lib/ruby_event_store/outbox/metrics.rb
|
62
63
|
- lib/ruby_event_store/outbox/metrics/influx.rb
|
63
64
|
- lib/ruby_event_store/outbox/metrics/null.rb
|
64
65
|
- lib/ruby_event_store/outbox/metrics/test.rb
|
65
66
|
- lib/ruby_event_store/outbox/repository.rb
|
67
|
+
- lib/ruby_event_store/outbox/runner.rb
|
66
68
|
- lib/ruby_event_store/outbox/sidekiq5_format.rb
|
67
69
|
- lib/ruby_event_store/outbox/sidekiq_processor.rb
|
68
70
|
- lib/ruby_event_store/outbox/sidekiq_producer.rb
|
69
71
|
- lib/ruby_event_store/outbox/sidekiq_scheduler.rb
|
72
|
+
- lib/ruby_event_store/outbox/tempo.rb
|
70
73
|
- lib/ruby_event_store/outbox/version.rb
|
71
74
|
homepage: https://railseventstore.org
|
72
75
|
licenses:
|
@@ -76,7 +79,7 @@ metadata:
|
|
76
79
|
source_code_uri: https://github.com/RailsEventStore/rails_event_store
|
77
80
|
bug_tracker_uri: https://github.com/RailsEventStore/rails_event_store/issues
|
78
81
|
rubygems_mfa_required: 'true'
|
79
|
-
post_install_message:
|
82
|
+
post_install_message:
|
80
83
|
rdoc_options: []
|
81
84
|
require_paths:
|
82
85
|
- lib
|
@@ -84,15 +87,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
84
87
|
requirements:
|
85
88
|
- - ">="
|
86
89
|
- !ruby/object:Gem::Version
|
87
|
-
version: '2.
|
90
|
+
version: '2.7'
|
88
91
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
89
92
|
requirements:
|
90
93
|
- - ">="
|
91
94
|
- !ruby/object:Gem::Version
|
92
95
|
version: '0'
|
93
96
|
requirements: []
|
94
|
-
rubygems_version: 3.
|
95
|
-
signing_key:
|
97
|
+
rubygems_version: 3.4.10
|
98
|
+
signing_key:
|
96
99
|
specification_version: 4
|
97
100
|
summary: Active Record based outbox for Ruby Event Store
|
98
101
|
test_files: []
|
@@ -1,132 +0,0 @@
|
|
1
|
-
require "optparse"
|
2
|
-
require_relative "version"
|
3
|
-
require_relative "consumer"
|
4
|
-
require_relative "metrics"
|
5
|
-
|
6
|
-
module RubyEventStore
|
7
|
-
module Outbox
|
8
|
-
class CLI
|
9
|
-
DEFAULTS = {
|
10
|
-
database_url: nil,
|
11
|
-
redis_url: nil,
|
12
|
-
log_level: :warn,
|
13
|
-
split_keys: nil,
|
14
|
-
message_format: "sidekiq5",
|
15
|
-
batch_size: 100,
|
16
|
-
metrics_url: nil,
|
17
|
-
cleanup_strategy: :none,
|
18
|
-
cleanup_limit: :all,
|
19
|
-
sleep_on_empty: 0.5
|
20
|
-
}
|
21
|
-
Options = Struct.new(*DEFAULTS.keys)
|
22
|
-
|
23
|
-
class Parser
|
24
|
-
def self.parse(argv)
|
25
|
-
options = Options.new(*DEFAULTS.values)
|
26
|
-
OptionParser
|
27
|
-
.new do |option_parser|
|
28
|
-
option_parser.banner = "Usage: res_outbox [options]"
|
29
|
-
|
30
|
-
option_parser.on(
|
31
|
-
"--database-url=DATABASE_URL",
|
32
|
-
"Database where outbox table is stored"
|
33
|
-
) { |database_url| options.database_url = database_url }
|
34
|
-
|
35
|
-
option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
|
36
|
-
options.redis_url = redis_url
|
37
|
-
end
|
38
|
-
|
39
|
-
option_parser.on(
|
40
|
-
"--log-level=LOG_LEVEL",
|
41
|
-
%i[fatal error warn info debug],
|
42
|
-
"Logging level, one of: fatal, error, warn, info, debug. Default: warn"
|
43
|
-
) { |log_level| options.log_level = log_level.to_sym }
|
44
|
-
|
45
|
-
option_parser.on(
|
46
|
-
"--message-format=FORMAT",
|
47
|
-
["sidekiq5"],
|
48
|
-
"Message format, supported: sidekiq5. Default: sidekiq5"
|
49
|
-
) { |message_format| options.message_format = message_format }
|
50
|
-
|
51
|
-
option_parser.on(
|
52
|
-
"--split-keys=SPLIT_KEYS",
|
53
|
-
Array,
|
54
|
-
"Split keys which should be handled, all if not specified"
|
55
|
-
) { |split_keys| options.split_keys = split_keys if !split_keys.empty? }
|
56
|
-
|
57
|
-
option_parser.on(
|
58
|
-
"--batch-size=BATCH_SIZE",
|
59
|
-
Integer,
|
60
|
-
"Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100"
|
61
|
-
) { |batch_size| options.batch_size = batch_size }
|
62
|
-
|
63
|
-
option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
|
64
|
-
options.metrics_url = metrics_url
|
65
|
-
end
|
66
|
-
|
67
|
-
option_parser.on(
|
68
|
-
"--cleanup=STRATEGY",
|
69
|
-
"A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none"
|
70
|
-
) { |cleanup_strategy| options.cleanup_strategy = cleanup_strategy }
|
71
|
-
|
72
|
-
option_parser.on(
|
73
|
-
"--cleanup-limit=LIMIT",
|
74
|
-
"Amount of records removed in single cleanup run. One of: all or number of records that should be removed. Default: all"
|
75
|
-
) { |cleanup_limit| options.cleanup_limit = cleanup_limit }
|
76
|
-
|
77
|
-
option_parser.on(
|
78
|
-
"--sleep-on-empty=SLEEP_TIME",
|
79
|
-
Float,
|
80
|
-
"How long to sleep before next check when there was nothing to do. Default: 0.5"
|
81
|
-
) { |sleep_on_empty| options.sleep_on_empty = sleep_on_empty }
|
82
|
-
|
83
|
-
option_parser.on_tail("--version", "Show version") do
|
84
|
-
puts VERSION
|
85
|
-
exit
|
86
|
-
end
|
87
|
-
end
|
88
|
-
.parse(argv)
|
89
|
-
return options
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
|
-
def run(argv)
|
94
|
-
options = Parser.parse(argv)
|
95
|
-
outbox_consumer = build_consumer(options)
|
96
|
-
outbox_consumer.init
|
97
|
-
outbox_consumer.run
|
98
|
-
end
|
99
|
-
|
100
|
-
def build_consumer(options)
|
101
|
-
consumer_uuid = SecureRandom.uuid
|
102
|
-
logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox #{consumer_uuid}")
|
103
|
-
<<<<<<< HEAD
|
104
|
-
consumer_configuration =
|
105
|
-
Consumer::Configuration.new(
|
106
|
-
split_keys: options.split_keys,
|
107
|
-
message_format: options.message_format,
|
108
|
-
batch_size: options.batch_size,
|
109
|
-
database_url: options.database_url,
|
110
|
-
redis_url: options.redis_url,
|
111
|
-
cleanup: options.cleanup_strategy,
|
112
|
-
sleep_on_empty: options.sleep_on_empty
|
113
|
-
)
|
114
|
-
=======
|
115
|
-
consumer_configuration = Consumer::Configuration.new(
|
116
|
-
split_keys: options.split_keys,
|
117
|
-
message_format: options.message_format,
|
118
|
-
batch_size: options.batch_size,
|
119
|
-
database_url: options.database_url,
|
120
|
-
redis_url: options.redis_url,
|
121
|
-
cleanup: options.cleanup_strategy,
|
122
|
-
cleanup_limit: options.cleanup_limit,
|
123
|
-
sleep_on_empty: options.sleep_on_empty,
|
124
|
-
)
|
125
|
-
>>>>>>> 7f077fa2... fix error with passing `--cleanup-limit` from CLI to consumer
|
126
|
-
metrics = Metrics.from_url(options.metrics_url)
|
127
|
-
outbox_consumer =
|
128
|
-
RubyEventStore::Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
|
129
|
-
end
|
130
|
-
end
|
131
|
-
end
|
132
|
-
end
|
@@ -1,25 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require_relative "sidekiq_producer"
|
4
|
-
|
5
|
-
module RubyEventStore
|
6
|
-
module Outbox
|
7
|
-
class LegacySidekiqScheduler
|
8
|
-
def initialize
|
9
|
-
@sidekiq_producer = SidekiqProducer.new
|
10
|
-
end
|
11
|
-
|
12
|
-
def call(klass, serialized_record)
|
13
|
-
sidekiq_producer.call(klass, [serialized_record])
|
14
|
-
end
|
15
|
-
|
16
|
-
def verify(subscriber)
|
17
|
-
Class === subscriber && subscriber.respond_to?(:through_outbox?) && subscriber.through_outbox?
|
18
|
-
end
|
19
|
-
|
20
|
-
private
|
21
|
-
|
22
|
-
attr_reader :sidekiq_producer
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|