ruby_event_store-outbox 0.0.24 → 0.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/lib/generators/ruby_event_store/outbox/migration_generator.rb +17 -15
- data/lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb +1 -0
- data/lib/ruby_event_store/outbox/cleanup_strategies/none.rb +1 -2
- data/lib/ruby_event_store/outbox/cli.rb +54 -43
- data/lib/ruby_event_store/outbox/cli.rb.orig +132 -0
- data/lib/ruby_event_store/outbox/consumer.rb +41 -24
- data/lib/ruby_event_store/outbox/fetch_specification.rb +2 -8
- data/lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb +1 -0
- data/lib/ruby_event_store/outbox/metrics/influx.rb +19 -25
- data/lib/ruby_event_store/outbox/metrics/null.rb +2 -4
- data/lib/ruby_event_store/outbox/metrics/test.rb +8 -2
- data/lib/ruby_event_store/outbox/repository.rb +8 -12
- data/lib/ruby_event_store/outbox/sidekiq_processor.rb +2 -3
- data/lib/ruby_event_store/outbox/sidekiq_producer.rb +11 -8
- data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb +1 -0
- data/lib/ruby_event_store/outbox/version.rb +1 -1
- data/lib/ruby_event_store/outbox.rb +5 -5
- metadata +5 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d9f4e4b0d475d2ad92997aee74165ffacbabbe19282b5d1b266b5ec7b7d3b59b
|
4
|
+
data.tar.gz: adccd92a63810fb4f285c5ea5e4ad37cc22ad671d4550b1d73b289f179d5831b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4a858aff2550a901615bad83ffa58dc6e30f669a8aa8a422d047343b74a32317d7476e27e95127f9750640eab2e538385b3284840f4c0a6532eff6ee8a20e96e
|
7
|
+
data.tar.gz: b81515fba19988345a3f45397de2d674f94e331475140246ccb5a7266cb19f49f6f2d59445b99fe139b6770e59310a53090f7e9eec2e6503767cf57e5801abb8
|
data/README.md
CHANGED
@@ -34,11 +34,12 @@ Additionally, your handler's `through_outbox?` method should return `true`, for
|
|
34
34
|
|
35
35
|
```ruby
|
36
36
|
class SomeHandler
|
37
|
-
def self.through_outbox
|
37
|
+
def self.through_outbox?
|
38
|
+
true
|
39
|
+
end
|
38
40
|
end
|
39
41
|
```
|
40
42
|
|
41
|
-
|
42
43
|
## Installation (outbox process)
|
43
44
|
|
44
45
|
Run following process in any way you prefer:
|
@@ -65,7 +66,6 @@ res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" \
|
|
65
66
|
--metrics-url=http://user:password@localhost:8086/dbname"
|
66
67
|
```
|
67
68
|
|
68
|
-
|
69
69
|
## Contributing
|
70
70
|
|
71
71
|
Bug reports and pull requests are welcome on GitHub at https://github.com/RailsEventStore/rails_event_store.
|
@@ -1,28 +1,30 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
begin
|
4
|
-
require
|
4
|
+
require "rails/generators"
|
5
5
|
rescue LoadError
|
6
6
|
end
|
7
7
|
|
8
|
-
|
9
|
-
module
|
10
|
-
|
11
|
-
|
8
|
+
if defined?(Rails::Generators::Base)
|
9
|
+
module RubyEventStore
|
10
|
+
module Outbox
|
11
|
+
class MigrationGenerator < Rails::Generators::Base
|
12
|
+
source_root File.expand_path(File.join(File.dirname(__FILE__), "./templates"))
|
12
13
|
|
13
|
-
|
14
|
-
|
15
|
-
|
14
|
+
def create_migration
|
15
|
+
template "create_event_store_outbox_template.rb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
|
16
|
+
end
|
16
17
|
|
17
|
-
|
18
|
+
private
|
18
19
|
|
19
|
-
|
20
|
-
|
21
|
-
|
20
|
+
def migration_version
|
21
|
+
"[4.2]"
|
22
|
+
end
|
22
23
|
|
23
|
-
|
24
|
-
|
24
|
+
def timestamp
|
25
|
+
Time.now.strftime("%Y%m%d%H%M%S")
|
26
|
+
end
|
25
27
|
end
|
26
28
|
end
|
27
29
|
end
|
28
|
-
end
|
30
|
+
end
|
@@ -11,7 +11,7 @@ module RubyEventStore
|
|
11
11
|
redis_url: nil,
|
12
12
|
log_level: :warn,
|
13
13
|
split_keys: nil,
|
14
|
-
message_format:
|
14
|
+
message_format: "sidekiq5",
|
15
15
|
batch_size: 100,
|
16
16
|
metrics_url: nil,
|
17
17
|
cleanup_strategy: :none,
|
@@ -23,54 +23,69 @@ module RubyEventStore
|
|
23
23
|
class Parser
|
24
24
|
def self.parse(argv)
|
25
25
|
options = Options.new(*DEFAULTS.values)
|
26
|
-
OptionParser
|
27
|
-
|
26
|
+
OptionParser
|
27
|
+
.new do |option_parser|
|
28
|
+
option_parser.banner = "Usage: res_outbox [options]"
|
28
29
|
|
29
|
-
|
30
|
-
|
31
|
-
|
30
|
+
option_parser.on(
|
31
|
+
"--database-url=DATABASE_URL",
|
32
|
+
"Database where outbox table is stored"
|
33
|
+
) { |database_url| options.database_url = database_url }
|
32
34
|
|
33
|
-
|
34
|
-
|
35
|
-
|
35
|
+
option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
|
36
|
+
options.redis_url = redis_url
|
37
|
+
end
|
36
38
|
|
37
|
-
|
38
|
-
|
39
|
-
|
39
|
+
option_parser.on(
|
40
|
+
"--log-level=LOG_LEVEL",
|
41
|
+
%i[fatal error warn info debug],
|
42
|
+
"Logging level, one of: fatal, error, warn, info, debug. Default: warn"
|
43
|
+
) { |log_level| options.log_level = log_level.to_sym }
|
40
44
|
|
41
|
-
|
42
|
-
|
43
|
-
|
45
|
+
option_parser.on(
|
46
|
+
"--message-format=FORMAT",
|
47
|
+
["sidekiq5"],
|
48
|
+
"Message format, supported: sidekiq5. Default: sidekiq5"
|
49
|
+
) { |message_format| options.message_format = message_format }
|
44
50
|
|
45
|
-
|
46
|
-
|
47
|
-
|
51
|
+
option_parser.on(
|
52
|
+
"--split-keys=SPLIT_KEYS",
|
53
|
+
Array,
|
54
|
+
"Split keys which should be handled, all if not specified"
|
55
|
+
) { |split_keys| options.split_keys = split_keys if !split_keys.empty? }
|
48
56
|
|
49
|
-
|
50
|
-
|
51
|
-
|
57
|
+
option_parser.on(
|
58
|
+
"--batch-size=BATCH_SIZE",
|
59
|
+
Integer,
|
60
|
+
"Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100"
|
61
|
+
) { |batch_size| options.batch_size = batch_size }
|
52
62
|
|
53
|
-
|
54
|
-
|
55
|
-
|
63
|
+
option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
|
64
|
+
options.metrics_url = metrics_url
|
65
|
+
end
|
56
66
|
|
57
|
-
|
58
|
-
|
59
|
-
|
67
|
+
option_parser.on(
|
68
|
+
"--cleanup=STRATEGY",
|
69
|
+
"A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none"
|
70
|
+
) { |cleanup_strategy| options.cleanup_strategy = cleanup_strategy }
|
60
71
|
|
61
|
-
|
62
|
-
|
63
|
-
|
72
|
+
option_parser.on(
|
73
|
+
"--cleanup-limit=LIMIT",
|
74
|
+
"Amount of records removed in single cleanup run. One of: all or number of records that should be removed. Default: all"
|
75
|
+
) { |cleanup_limit| options.cleanup_limit = cleanup_limit }
|
64
76
|
|
65
|
-
|
66
|
-
|
67
|
-
|
77
|
+
option_parser.on(
|
78
|
+
"--sleep-on-empty=SLEEP_TIME",
|
79
|
+
Float,
|
80
|
+
"How long to sleep before next check when there was nothing to do. Default: 0.5"
|
81
|
+
) { |sleep_on_empty| options.sleep_on_empty = sleep_on_empty }
|
68
82
|
|
69
|
-
|
70
|
-
|
71
|
-
|
83
|
+
option_parser.on_tail("--version", "Show version") do
|
84
|
+
puts VERSION
|
85
|
+
exit
|
86
|
+
end
|
72
87
|
end
|
73
|
-
|
88
|
+
.parse(argv)
|
74
89
|
return options
|
75
90
|
end
|
76
91
|
end
|
@@ -96,12 +111,8 @@ module RubyEventStore
|
|
96
111
|
sleep_on_empty: options.sleep_on_empty,
|
97
112
|
)
|
98
113
|
metrics = Metrics.from_url(options.metrics_url)
|
99
|
-
outbox_consumer =
|
100
|
-
consumer_uuid,
|
101
|
-
consumer_configuration,
|
102
|
-
logger: logger,
|
103
|
-
metrics: metrics,
|
104
|
-
)
|
114
|
+
outbox_consumer =
|
115
|
+
RubyEventStore::Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
|
105
116
|
end
|
106
117
|
end
|
107
118
|
end
|
@@ -0,0 +1,132 @@
|
|
1
|
+
require "optparse"
|
2
|
+
require_relative "version"
|
3
|
+
require_relative "consumer"
|
4
|
+
require_relative "metrics"
|
5
|
+
|
6
|
+
module RubyEventStore
|
7
|
+
module Outbox
|
8
|
+
class CLI
|
9
|
+
DEFAULTS = {
|
10
|
+
database_url: nil,
|
11
|
+
redis_url: nil,
|
12
|
+
log_level: :warn,
|
13
|
+
split_keys: nil,
|
14
|
+
message_format: "sidekiq5",
|
15
|
+
batch_size: 100,
|
16
|
+
metrics_url: nil,
|
17
|
+
cleanup_strategy: :none,
|
18
|
+
cleanup_limit: :all,
|
19
|
+
sleep_on_empty: 0.5
|
20
|
+
}
|
21
|
+
Options = Struct.new(*DEFAULTS.keys)
|
22
|
+
|
23
|
+
class Parser
|
24
|
+
def self.parse(argv)
|
25
|
+
options = Options.new(*DEFAULTS.values)
|
26
|
+
OptionParser
|
27
|
+
.new do |option_parser|
|
28
|
+
option_parser.banner = "Usage: res_outbox [options]"
|
29
|
+
|
30
|
+
option_parser.on(
|
31
|
+
"--database-url=DATABASE_URL",
|
32
|
+
"Database where outbox table is stored"
|
33
|
+
) { |database_url| options.database_url = database_url }
|
34
|
+
|
35
|
+
option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
|
36
|
+
options.redis_url = redis_url
|
37
|
+
end
|
38
|
+
|
39
|
+
option_parser.on(
|
40
|
+
"--log-level=LOG_LEVEL",
|
41
|
+
%i[fatal error warn info debug],
|
42
|
+
"Logging level, one of: fatal, error, warn, info, debug. Default: warn"
|
43
|
+
) { |log_level| options.log_level = log_level.to_sym }
|
44
|
+
|
45
|
+
option_parser.on(
|
46
|
+
"--message-format=FORMAT",
|
47
|
+
["sidekiq5"],
|
48
|
+
"Message format, supported: sidekiq5. Default: sidekiq5"
|
49
|
+
) { |message_format| options.message_format = message_format }
|
50
|
+
|
51
|
+
option_parser.on(
|
52
|
+
"--split-keys=SPLIT_KEYS",
|
53
|
+
Array,
|
54
|
+
"Split keys which should be handled, all if not specified"
|
55
|
+
) { |split_keys| options.split_keys = split_keys if !split_keys.empty? }
|
56
|
+
|
57
|
+
option_parser.on(
|
58
|
+
"--batch-size=BATCH_SIZE",
|
59
|
+
Integer,
|
60
|
+
"Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100"
|
61
|
+
) { |batch_size| options.batch_size = batch_size }
|
62
|
+
|
63
|
+
option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
|
64
|
+
options.metrics_url = metrics_url
|
65
|
+
end
|
66
|
+
|
67
|
+
option_parser.on(
|
68
|
+
"--cleanup=STRATEGY",
|
69
|
+
"A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none"
|
70
|
+
) { |cleanup_strategy| options.cleanup_strategy = cleanup_strategy }
|
71
|
+
|
72
|
+
option_parser.on(
|
73
|
+
"--cleanup-limit=LIMIT",
|
74
|
+
"Amount of records removed in single cleanup run. One of: all or number of records that should be removed. Default: all"
|
75
|
+
) { |cleanup_limit| options.cleanup_limit = cleanup_limit }
|
76
|
+
|
77
|
+
option_parser.on(
|
78
|
+
"--sleep-on-empty=SLEEP_TIME",
|
79
|
+
Float,
|
80
|
+
"How long to sleep before next check when there was nothing to do. Default: 0.5"
|
81
|
+
) { |sleep_on_empty| options.sleep_on_empty = sleep_on_empty }
|
82
|
+
|
83
|
+
option_parser.on_tail("--version", "Show version") do
|
84
|
+
puts VERSION
|
85
|
+
exit
|
86
|
+
end
|
87
|
+
end
|
88
|
+
.parse(argv)
|
89
|
+
return options
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def run(argv)
|
94
|
+
options = Parser.parse(argv)
|
95
|
+
outbox_consumer = build_consumer(options)
|
96
|
+
outbox_consumer.init
|
97
|
+
outbox_consumer.run
|
98
|
+
end
|
99
|
+
|
100
|
+
def build_consumer(options)
|
101
|
+
consumer_uuid = SecureRandom.uuid
|
102
|
+
logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox #{consumer_uuid}")
|
103
|
+
<<<<<<< HEAD
|
104
|
+
consumer_configuration =
|
105
|
+
Consumer::Configuration.new(
|
106
|
+
split_keys: options.split_keys,
|
107
|
+
message_format: options.message_format,
|
108
|
+
batch_size: options.batch_size,
|
109
|
+
database_url: options.database_url,
|
110
|
+
redis_url: options.redis_url,
|
111
|
+
cleanup: options.cleanup_strategy,
|
112
|
+
sleep_on_empty: options.sleep_on_empty
|
113
|
+
)
|
114
|
+
=======
|
115
|
+
consumer_configuration = Consumer::Configuration.new(
|
116
|
+
split_keys: options.split_keys,
|
117
|
+
message_format: options.message_format,
|
118
|
+
batch_size: options.batch_size,
|
119
|
+
database_url: options.database_url,
|
120
|
+
redis_url: options.redis_url,
|
121
|
+
cleanup: options.cleanup_strategy,
|
122
|
+
cleanup_limit: options.cleanup_limit,
|
123
|
+
sleep_on_empty: options.sleep_on_empty,
|
124
|
+
)
|
125
|
+
>>>>>>> 7f077fa2... fix error with passing `--cleanup-limit` from CLI to consumer
|
126
|
+
metrics = Metrics.from_url(options.metrics_url)
|
127
|
+
outbox_consumer =
|
128
|
+
RubyEventStore::Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
@@ -48,7 +48,14 @@ module RubyEventStore
|
|
48
48
|
)
|
49
49
|
end
|
50
50
|
|
51
|
-
attr_reader :split_keys,
|
51
|
+
attr_reader :split_keys,
|
52
|
+
:message_format,
|
53
|
+
:batch_size,
|
54
|
+
:database_url,
|
55
|
+
:redis_url,
|
56
|
+
:cleanup,
|
57
|
+
:cleanup_limit,
|
58
|
+
:sleep_on_empty
|
52
59
|
end
|
53
60
|
|
54
61
|
def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
|
@@ -67,12 +74,17 @@ module RubyEventStore
|
|
67
74
|
prepare_traps
|
68
75
|
|
69
76
|
@repository = Repository.new(configuration.database_url)
|
70
|
-
@cleanup_strategy =
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
77
|
+
@cleanup_strategy =
|
78
|
+
case configuration.cleanup
|
79
|
+
when :none
|
80
|
+
CleanupStrategies::None.new
|
81
|
+
else
|
82
|
+
CleanupStrategies::CleanOldEnqueued.new(
|
83
|
+
repository,
|
84
|
+
ActiveSupport::Duration.parse(configuration.cleanup),
|
85
|
+
configuration.cleanup_limit
|
86
|
+
)
|
87
|
+
end
|
76
88
|
end
|
77
89
|
|
78
90
|
def init
|
@@ -81,7 +93,7 @@ module RubyEventStore
|
|
81
93
|
end
|
82
94
|
|
83
95
|
def run
|
84
|
-
while !@gracefully_shutting_down
|
96
|
+
while !@gracefully_shutting_down
|
85
97
|
was_something_changed = one_loop
|
86
98
|
if !was_something_changed
|
87
99
|
STDOUT.flush
|
@@ -109,9 +121,7 @@ module RubyEventStore
|
|
109
121
|
|
110
122
|
MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK.times do
|
111
123
|
batch = retrieve_batch(fetch_specification)
|
112
|
-
if batch.empty?
|
113
|
-
break
|
114
|
-
end
|
124
|
+
break if batch.empty?
|
115
125
|
|
116
126
|
failed_record_ids = []
|
117
127
|
updated_record_ids = []
|
@@ -125,7 +135,7 @@ module RubyEventStore
|
|
125
135
|
updated_record_ids << record.id
|
126
136
|
rescue => e
|
127
137
|
failed_record_ids << record.id
|
128
|
-
e.full_message.split($/).each {|line| logger.error(line) }
|
138
|
+
e.full_message.split($/).each { |line| logger.error(line) }
|
129
139
|
end
|
130
140
|
end
|
131
141
|
|
@@ -143,11 +153,13 @@ module RubyEventStore
|
|
143
153
|
break unless refresh_successful
|
144
154
|
end
|
145
155
|
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
156
|
+
unless something_processed
|
157
|
+
metrics.write_point_queue(
|
158
|
+
format: fetch_specification.message_format,
|
159
|
+
split_key: fetch_specification.split_key,
|
160
|
+
remaining: get_remaining_count(fetch_specification)
|
161
|
+
)
|
162
|
+
end
|
151
163
|
|
152
164
|
release_lock_for_process(fetch_specification)
|
153
165
|
|
@@ -159,7 +171,16 @@ module RubyEventStore
|
|
159
171
|
end
|
160
172
|
|
161
173
|
private
|
162
|
-
|
174
|
+
|
175
|
+
attr_reader :split_keys,
|
176
|
+
:logger,
|
177
|
+
:batch_size,
|
178
|
+
:metrics,
|
179
|
+
:processor,
|
180
|
+
:consumer_uuid,
|
181
|
+
:repository,
|
182
|
+
:cleanup_strategy,
|
183
|
+
:sleep_on_empty
|
163
184
|
|
164
185
|
def obtain_lock_for_process(fetch_specification)
|
165
186
|
result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
|
@@ -231,12 +252,8 @@ module RubyEventStore
|
|
231
252
|
end
|
232
253
|
|
233
254
|
def prepare_traps
|
234
|
-
Signal.trap("INT")
|
235
|
-
|
236
|
-
end
|
237
|
-
Signal.trap("TERM") do
|
238
|
-
initiate_graceful_shutdown
|
239
|
-
end
|
255
|
+
Signal.trap("INT") { initiate_graceful_shutdown }
|
256
|
+
Signal.trap("TERM") { initiate_graceful_shutdown }
|
240
257
|
end
|
241
258
|
|
242
259
|
def initiate_graceful_shutdown
|
@@ -10,19 +10,13 @@ module RubyEventStore
|
|
10
10
|
attr_reader :message_format, :split_key
|
11
11
|
|
12
12
|
def ==(other)
|
13
|
-
other.instance_of?(self.class) &&
|
14
|
-
other.message_format.eql?(message_format) &&
|
15
|
-
other.split_key.eql?(split_key)
|
13
|
+
other.instance_of?(self.class) && other.message_format.eql?(message_format) && other.split_key.eql?(split_key)
|
16
14
|
end
|
17
15
|
|
18
16
|
BIG_VALUE = 0b111111100100010000010010110010101011011101110101001100100110000
|
19
17
|
|
20
18
|
def hash
|
21
|
-
[
|
22
|
-
self.class,
|
23
|
-
message_format,
|
24
|
-
split_key,
|
25
|
-
].hash ^ BIG_VALUE
|
19
|
+
[self.class, message_format, split_key].hash ^ BIG_VALUE
|
26
20
|
end
|
27
21
|
|
28
22
|
alias_method :eql?, :==
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "influxdb"
|
2
2
|
|
3
3
|
module RubyEventStore
|
4
4
|
module Outbox
|
@@ -6,38 +6,32 @@ module RubyEventStore
|
|
6
6
|
class Influx
|
7
7
|
def initialize(url)
|
8
8
|
uri = URI.parse(url)
|
9
|
-
options = {
|
10
|
-
url: url,
|
11
|
-
async: true,
|
12
|
-
time_precision: 'ns',
|
13
|
-
}
|
9
|
+
options = { url: url, async: true, time_precision: "ns" }
|
14
10
|
@influxdb_client = InfluxDB::Client.new(**options)
|
15
11
|
end
|
16
12
|
|
17
13
|
def write_operation_result(operation, result)
|
18
|
-
write_point(
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
tags: {
|
23
|
-
operation: operation,
|
24
|
-
result: result,
|
25
|
-
}
|
26
|
-
})
|
14
|
+
write_point(
|
15
|
+
"ruby_event_store.outbox.lock",
|
16
|
+
{ values: { value: 1 }, tags: { operation: operation, result: result } }
|
17
|
+
)
|
27
18
|
end
|
28
19
|
|
29
20
|
def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
|
30
|
-
write_point(
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
21
|
+
write_point(
|
22
|
+
"ruby_event_store.outbox.queue",
|
23
|
+
{
|
24
|
+
values: {
|
25
|
+
enqueued: enqueued,
|
26
|
+
failed: failed,
|
27
|
+
remaining: remaining
|
28
|
+
},
|
29
|
+
tags: {
|
30
|
+
format: format,
|
31
|
+
split_key: split_key
|
32
|
+
}
|
39
33
|
}
|
40
|
-
|
34
|
+
)
|
41
35
|
end
|
42
36
|
|
43
37
|
def write_point(series, data)
|
@@ -2,11 +2,9 @@ module RubyEventStore
|
|
2
2
|
module Outbox
|
3
3
|
module Metrics
|
4
4
|
class Null
|
5
|
-
def write_operation_result(operation, result)
|
6
|
-
end
|
5
|
+
def write_operation_result(operation, result); end
|
7
6
|
|
8
|
-
def write_point_queue(**kwargs)
|
9
|
-
end
|
7
|
+
def write_point_queue(**kwargs); end
|
10
8
|
end
|
11
9
|
end
|
12
10
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "influxdb"
|
2
2
|
|
3
3
|
module RubyEventStore
|
4
4
|
module Outbox
|
@@ -14,7 +14,13 @@ module RubyEventStore
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
|
17
|
-
@queue_stats << {
|
17
|
+
@queue_stats << {
|
18
|
+
enqueued: enqueued,
|
19
|
+
failed: failed,
|
20
|
+
remaining: remaining,
|
21
|
+
format: format,
|
22
|
+
split_key: split_key
|
23
|
+
}
|
18
24
|
end
|
19
25
|
|
20
26
|
attr_reader :operation_results, :queue_stats
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require "active_record"
|
4
|
+
require "active_support/core_ext/numeric/time.rb"
|
5
5
|
|
6
6
|
module RubyEventStore
|
7
7
|
module Outbox
|
@@ -10,7 +10,7 @@ module RubyEventStore
|
|
10
10
|
|
11
11
|
class Record < ::ActiveRecord::Base
|
12
12
|
self.primary_key = :id
|
13
|
-
self.table_name =
|
13
|
+
self.table_name = "event_store_outbox"
|
14
14
|
|
15
15
|
def self.remaining_for(fetch_specification)
|
16
16
|
where(format: fetch_specification.message_format, split_key: fetch_specification.split_key, enqueued_at: nil)
|
@@ -30,7 +30,7 @@ module RubyEventStore
|
|
30
30
|
end
|
31
31
|
|
32
32
|
class Lock < ::ActiveRecord::Base
|
33
|
-
self.table_name =
|
33
|
+
self.table_name = "event_store_outbox_locks"
|
34
34
|
|
35
35
|
def self.obtain(fetch_specification, process_uuid, clock:)
|
36
36
|
transaction do
|
@@ -39,10 +39,7 @@ module RubyEventStore
|
|
39
39
|
if l.recently_locked?(clock: clock)
|
40
40
|
:taken
|
41
41
|
else
|
42
|
-
l.update!(
|
43
|
-
locked_by: process_uuid,
|
44
|
-
locked_at: clock.now,
|
45
|
-
)
|
42
|
+
l.update!(locked_by: process_uuid, locked_at: clock.now)
|
46
43
|
l
|
47
44
|
end
|
48
45
|
end
|
@@ -98,6 +95,7 @@ module RubyEventStore
|
|
98
95
|
end
|
99
96
|
|
100
97
|
private
|
98
|
+
|
101
99
|
def self.lock_for_split_key(fetch_specification)
|
102
100
|
lock.find_by(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
|
103
101
|
end
|
@@ -143,10 +141,8 @@ module RubyEventStore
|
|
143
141
|
end
|
144
142
|
|
145
143
|
def delete_enqueued_older_than(fetch_specification, duration, limit)
|
146
|
-
scope = Record
|
147
|
-
|
148
|
-
.where("enqueued_at < ?", duration.ago)
|
149
|
-
scope = scope.limit(limit) unless limit == :all
|
144
|
+
scope = Record.for_fetch_specification(fetch_specification).where("enqueued_at < ?", duration.ago)
|
145
|
+
scope = scope.limit(limit).order(:id) unless limit == :all
|
150
146
|
scope.delete_all
|
151
147
|
:ok
|
152
148
|
rescue ActiveRecord::Deadlocked
|
@@ -17,9 +17,7 @@ module RubyEventStore
|
|
17
17
|
|
18
18
|
queue = parsed_record["queue"]
|
19
19
|
raise InvalidPayload.new("Missing queue") if queue.nil? || queue.empty?
|
20
|
-
payload = JSON.generate(parsed_record.merge({
|
21
|
-
"enqueued_at" => now.to_f,
|
22
|
-
}))
|
20
|
+
payload = JSON.generate(parsed_record.merge({ "enqueued_at" => now.to_f }))
|
23
21
|
|
24
22
|
redis.lpush("queue:#{queue}", payload)
|
25
23
|
|
@@ -38,6 +36,7 @@ module RubyEventStore
|
|
38
36
|
end
|
39
37
|
|
40
38
|
private
|
39
|
+
|
41
40
|
attr_reader :redis
|
42
41
|
end
|
43
42
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require_relative
|
3
|
+
require "sidekiq"
|
4
|
+
require_relative "sidekiq5_format"
|
5
5
|
require_relative "repository"
|
6
6
|
|
7
7
|
module RubyEventStore
|
@@ -9,22 +9,25 @@ module RubyEventStore
|
|
9
9
|
class SidekiqProducer
|
10
10
|
def call(klass, args)
|
11
11
|
sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
|
12
|
-
item = {
|
13
|
-
'args' => args.map(&:to_h).map {|h| h.transform_keys(&:to_s)},
|
14
|
-
'class' => klass,
|
15
|
-
}
|
12
|
+
item = { "args" => args.map(&:to_h).map { |h| h.transform_keys(&:to_s) }, "class" => klass }
|
16
13
|
normalized_item = sidekiq_client.__send__(:normalize_item, item)
|
17
|
-
payload =
|
14
|
+
payload =
|
15
|
+
sidekiq_client
|
16
|
+
.middleware
|
17
|
+
.invoke(normalized_item["class"], normalized_item, normalized_item["queue"], Sidekiq.redis_pool) do
|
18
|
+
normalized_item
|
19
|
+
end
|
18
20
|
if payload
|
19
21
|
Repository::Record.create!(
|
20
22
|
format: SIDEKIQ5_FORMAT,
|
21
|
-
split_key: payload.fetch(
|
23
|
+
split_key: payload.fetch("queue"),
|
22
24
|
payload: payload.to_json
|
23
25
|
)
|
24
26
|
end
|
25
27
|
end
|
26
28
|
|
27
29
|
private
|
30
|
+
|
28
31
|
attr_reader :repository
|
29
32
|
end
|
30
33
|
end
|
@@ -5,8 +5,8 @@ module RubyEventStore
|
|
5
5
|
end
|
6
6
|
end
|
7
7
|
|
8
|
-
require_relative
|
9
|
-
require_relative
|
10
|
-
require_relative
|
11
|
-
require_relative
|
12
|
-
require_relative
|
8
|
+
require_relative "outbox/fetch_specification"
|
9
|
+
require_relative "outbox/repository"
|
10
|
+
require_relative "outbox/sidekiq_scheduler"
|
11
|
+
require_relative "outbox/legacy_sidekiq_scheduler"
|
12
|
+
require_relative "outbox/version"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby_event_store-outbox
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.25
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arkency
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-05-
|
11
|
+
date: 2022-05-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ruby_event_store
|
@@ -30,14 +30,14 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '6.0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '6.0'
|
41
41
|
description:
|
42
42
|
email: dev@arkency.com
|
43
43
|
executables:
|
@@ -54,6 +54,7 @@ files:
|
|
54
54
|
- lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb
|
55
55
|
- lib/ruby_event_store/outbox/cleanup_strategies/none.rb
|
56
56
|
- lib/ruby_event_store/outbox/cli.rb
|
57
|
+
- lib/ruby_event_store/outbox/cli.rb.orig
|
57
58
|
- lib/ruby_event_store/outbox/consumer.rb
|
58
59
|
- lib/ruby_event_store/outbox/fetch_specification.rb
|
59
60
|
- lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb
|