ruby_event_store-outbox 0.0.24 → 0.0.25
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/lib/generators/ruby_event_store/outbox/migration_generator.rb +17 -15
- data/lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb +1 -0
- data/lib/ruby_event_store/outbox/cleanup_strategies/none.rb +1 -2
- data/lib/ruby_event_store/outbox/cli.rb +54 -43
- data/lib/ruby_event_store/outbox/cli.rb.orig +132 -0
- data/lib/ruby_event_store/outbox/consumer.rb +41 -24
- data/lib/ruby_event_store/outbox/fetch_specification.rb +2 -8
- data/lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb +1 -0
- data/lib/ruby_event_store/outbox/metrics/influx.rb +19 -25
- data/lib/ruby_event_store/outbox/metrics/null.rb +2 -4
- data/lib/ruby_event_store/outbox/metrics/test.rb +8 -2
- data/lib/ruby_event_store/outbox/repository.rb +8 -12
- data/lib/ruby_event_store/outbox/sidekiq_processor.rb +2 -3
- data/lib/ruby_event_store/outbox/sidekiq_producer.rb +11 -8
- data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb +1 -0
- data/lib/ruby_event_store/outbox/version.rb +1 -1
- data/lib/ruby_event_store/outbox.rb +5 -5
- metadata +5 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d9f4e4b0d475d2ad92997aee74165ffacbabbe19282b5d1b266b5ec7b7d3b59b
|
4
|
+
data.tar.gz: adccd92a63810fb4f285c5ea5e4ad37cc22ad671d4550b1d73b289f179d5831b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4a858aff2550a901615bad83ffa58dc6e30f669a8aa8a422d047343b74a32317d7476e27e95127f9750640eab2e538385b3284840f4c0a6532eff6ee8a20e96e
|
7
|
+
data.tar.gz: b81515fba19988345a3f45397de2d674f94e331475140246ccb5a7266cb19f49f6f2d59445b99fe139b6770e59310a53090f7e9eec2e6503767cf57e5801abb8
|
data/README.md
CHANGED
@@ -34,11 +34,12 @@ Additionally, your handler's `through_outbox?` method should return `true`, for
|
|
34
34
|
|
35
35
|
```ruby
|
36
36
|
class SomeHandler
|
37
|
-
def self.through_outbox
|
37
|
+
def self.through_outbox?
|
38
|
+
true
|
39
|
+
end
|
38
40
|
end
|
39
41
|
```
|
40
42
|
|
41
|
-
|
42
43
|
## Installation (outbox process)
|
43
44
|
|
44
45
|
Run following process in any way you prefer:
|
@@ -65,7 +66,6 @@ res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" \
|
|
65
66
|
--metrics-url=http://user:password@localhost:8086/dbname"
|
66
67
|
```
|
67
68
|
|
68
|
-
|
69
69
|
## Contributing
|
70
70
|
|
71
71
|
Bug reports and pull requests are welcome on GitHub at https://github.com/RailsEventStore/rails_event_store.
|
@@ -1,28 +1,30 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
begin
|
4
|
-
require
|
4
|
+
require "rails/generators"
|
5
5
|
rescue LoadError
|
6
6
|
end
|
7
7
|
|
8
|
-
|
9
|
-
module
|
10
|
-
|
11
|
-
|
8
|
+
if defined?(Rails::Generators::Base)
|
9
|
+
module RubyEventStore
|
10
|
+
module Outbox
|
11
|
+
class MigrationGenerator < Rails::Generators::Base
|
12
|
+
source_root File.expand_path(File.join(File.dirname(__FILE__), "./templates"))
|
12
13
|
|
13
|
-
|
14
|
-
|
15
|
-
|
14
|
+
def create_migration
|
15
|
+
template "create_event_store_outbox_template.rb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
|
16
|
+
end
|
16
17
|
|
17
|
-
|
18
|
+
private
|
18
19
|
|
19
|
-
|
20
|
-
|
21
|
-
|
20
|
+
def migration_version
|
21
|
+
"[4.2]"
|
22
|
+
end
|
22
23
|
|
23
|
-
|
24
|
-
|
24
|
+
def timestamp
|
25
|
+
Time.now.strftime("%Y%m%d%H%M%S")
|
26
|
+
end
|
25
27
|
end
|
26
28
|
end
|
27
29
|
end
|
28
|
-
end
|
30
|
+
end
|
@@ -11,7 +11,7 @@ module RubyEventStore
|
|
11
11
|
redis_url: nil,
|
12
12
|
log_level: :warn,
|
13
13
|
split_keys: nil,
|
14
|
-
message_format:
|
14
|
+
message_format: "sidekiq5",
|
15
15
|
batch_size: 100,
|
16
16
|
metrics_url: nil,
|
17
17
|
cleanup_strategy: :none,
|
@@ -23,54 +23,69 @@ module RubyEventStore
|
|
23
23
|
class Parser
|
24
24
|
def self.parse(argv)
|
25
25
|
options = Options.new(*DEFAULTS.values)
|
26
|
-
OptionParser
|
27
|
-
|
26
|
+
OptionParser
|
27
|
+
.new do |option_parser|
|
28
|
+
option_parser.banner = "Usage: res_outbox [options]"
|
28
29
|
|
29
|
-
|
30
|
-
|
31
|
-
|
30
|
+
option_parser.on(
|
31
|
+
"--database-url=DATABASE_URL",
|
32
|
+
"Database where outbox table is stored"
|
33
|
+
) { |database_url| options.database_url = database_url }
|
32
34
|
|
33
|
-
|
34
|
-
|
35
|
-
|
35
|
+
option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
|
36
|
+
options.redis_url = redis_url
|
37
|
+
end
|
36
38
|
|
37
|
-
|
38
|
-
|
39
|
-
|
39
|
+
option_parser.on(
|
40
|
+
"--log-level=LOG_LEVEL",
|
41
|
+
%i[fatal error warn info debug],
|
42
|
+
"Logging level, one of: fatal, error, warn, info, debug. Default: warn"
|
43
|
+
) { |log_level| options.log_level = log_level.to_sym }
|
40
44
|
|
41
|
-
|
42
|
-
|
43
|
-
|
45
|
+
option_parser.on(
|
46
|
+
"--message-format=FORMAT",
|
47
|
+
["sidekiq5"],
|
48
|
+
"Message format, supported: sidekiq5. Default: sidekiq5"
|
49
|
+
) { |message_format| options.message_format = message_format }
|
44
50
|
|
45
|
-
|
46
|
-
|
47
|
-
|
51
|
+
option_parser.on(
|
52
|
+
"--split-keys=SPLIT_KEYS",
|
53
|
+
Array,
|
54
|
+
"Split keys which should be handled, all if not specified"
|
55
|
+
) { |split_keys| options.split_keys = split_keys if !split_keys.empty? }
|
48
56
|
|
49
|
-
|
50
|
-
|
51
|
-
|
57
|
+
option_parser.on(
|
58
|
+
"--batch-size=BATCH_SIZE",
|
59
|
+
Integer,
|
60
|
+
"Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100"
|
61
|
+
) { |batch_size| options.batch_size = batch_size }
|
52
62
|
|
53
|
-
|
54
|
-
|
55
|
-
|
63
|
+
option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
|
64
|
+
options.metrics_url = metrics_url
|
65
|
+
end
|
56
66
|
|
57
|
-
|
58
|
-
|
59
|
-
|
67
|
+
option_parser.on(
|
68
|
+
"--cleanup=STRATEGY",
|
69
|
+
"A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none"
|
70
|
+
) { |cleanup_strategy| options.cleanup_strategy = cleanup_strategy }
|
60
71
|
|
61
|
-
|
62
|
-
|
63
|
-
|
72
|
+
option_parser.on(
|
73
|
+
"--cleanup-limit=LIMIT",
|
74
|
+
"Amount of records removed in single cleanup run. One of: all or number of records that should be removed. Default: all"
|
75
|
+
) { |cleanup_limit| options.cleanup_limit = cleanup_limit }
|
64
76
|
|
65
|
-
|
66
|
-
|
67
|
-
|
77
|
+
option_parser.on(
|
78
|
+
"--sleep-on-empty=SLEEP_TIME",
|
79
|
+
Float,
|
80
|
+
"How long to sleep before next check when there was nothing to do. Default: 0.5"
|
81
|
+
) { |sleep_on_empty| options.sleep_on_empty = sleep_on_empty }
|
68
82
|
|
69
|
-
|
70
|
-
|
71
|
-
|
83
|
+
option_parser.on_tail("--version", "Show version") do
|
84
|
+
puts VERSION
|
85
|
+
exit
|
86
|
+
end
|
72
87
|
end
|
73
|
-
|
88
|
+
.parse(argv)
|
74
89
|
return options
|
75
90
|
end
|
76
91
|
end
|
@@ -96,12 +111,8 @@ module RubyEventStore
|
|
96
111
|
sleep_on_empty: options.sleep_on_empty,
|
97
112
|
)
|
98
113
|
metrics = Metrics.from_url(options.metrics_url)
|
99
|
-
outbox_consumer =
|
100
|
-
consumer_uuid,
|
101
|
-
consumer_configuration,
|
102
|
-
logger: logger,
|
103
|
-
metrics: metrics,
|
104
|
-
)
|
114
|
+
outbox_consumer =
|
115
|
+
RubyEventStore::Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
|
105
116
|
end
|
106
117
|
end
|
107
118
|
end
|
@@ -0,0 +1,132 @@
|
|
1
|
+
require "optparse"
|
2
|
+
require_relative "version"
|
3
|
+
require_relative "consumer"
|
4
|
+
require_relative "metrics"
|
5
|
+
|
6
|
+
module RubyEventStore
|
7
|
+
module Outbox
|
8
|
+
class CLI
|
9
|
+
DEFAULTS = {
|
10
|
+
database_url: nil,
|
11
|
+
redis_url: nil,
|
12
|
+
log_level: :warn,
|
13
|
+
split_keys: nil,
|
14
|
+
message_format: "sidekiq5",
|
15
|
+
batch_size: 100,
|
16
|
+
metrics_url: nil,
|
17
|
+
cleanup_strategy: :none,
|
18
|
+
cleanup_limit: :all,
|
19
|
+
sleep_on_empty: 0.5
|
20
|
+
}
|
21
|
+
Options = Struct.new(*DEFAULTS.keys)
|
22
|
+
|
23
|
+
class Parser
|
24
|
+
def self.parse(argv)
|
25
|
+
options = Options.new(*DEFAULTS.values)
|
26
|
+
OptionParser
|
27
|
+
.new do |option_parser|
|
28
|
+
option_parser.banner = "Usage: res_outbox [options]"
|
29
|
+
|
30
|
+
option_parser.on(
|
31
|
+
"--database-url=DATABASE_URL",
|
32
|
+
"Database where outbox table is stored"
|
33
|
+
) { |database_url| options.database_url = database_url }
|
34
|
+
|
35
|
+
option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
|
36
|
+
options.redis_url = redis_url
|
37
|
+
end
|
38
|
+
|
39
|
+
option_parser.on(
|
40
|
+
"--log-level=LOG_LEVEL",
|
41
|
+
%i[fatal error warn info debug],
|
42
|
+
"Logging level, one of: fatal, error, warn, info, debug. Default: warn"
|
43
|
+
) { |log_level| options.log_level = log_level.to_sym }
|
44
|
+
|
45
|
+
option_parser.on(
|
46
|
+
"--message-format=FORMAT",
|
47
|
+
["sidekiq5"],
|
48
|
+
"Message format, supported: sidekiq5. Default: sidekiq5"
|
49
|
+
) { |message_format| options.message_format = message_format }
|
50
|
+
|
51
|
+
option_parser.on(
|
52
|
+
"--split-keys=SPLIT_KEYS",
|
53
|
+
Array,
|
54
|
+
"Split keys which should be handled, all if not specified"
|
55
|
+
) { |split_keys| options.split_keys = split_keys if !split_keys.empty? }
|
56
|
+
|
57
|
+
option_parser.on(
|
58
|
+
"--batch-size=BATCH_SIZE",
|
59
|
+
Integer,
|
60
|
+
"Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100"
|
61
|
+
) { |batch_size| options.batch_size = batch_size }
|
62
|
+
|
63
|
+
option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
|
64
|
+
options.metrics_url = metrics_url
|
65
|
+
end
|
66
|
+
|
67
|
+
option_parser.on(
|
68
|
+
"--cleanup=STRATEGY",
|
69
|
+
"A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none"
|
70
|
+
) { |cleanup_strategy| options.cleanup_strategy = cleanup_strategy }
|
71
|
+
|
72
|
+
option_parser.on(
|
73
|
+
"--cleanup-limit=LIMIT",
|
74
|
+
"Amount of records removed in single cleanup run. One of: all or number of records that should be removed. Default: all"
|
75
|
+
) { |cleanup_limit| options.cleanup_limit = cleanup_limit }
|
76
|
+
|
77
|
+
option_parser.on(
|
78
|
+
"--sleep-on-empty=SLEEP_TIME",
|
79
|
+
Float,
|
80
|
+
"How long to sleep before next check when there was nothing to do. Default: 0.5"
|
81
|
+
) { |sleep_on_empty| options.sleep_on_empty = sleep_on_empty }
|
82
|
+
|
83
|
+
option_parser.on_tail("--version", "Show version") do
|
84
|
+
puts VERSION
|
85
|
+
exit
|
86
|
+
end
|
87
|
+
end
|
88
|
+
.parse(argv)
|
89
|
+
return options
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def run(argv)
|
94
|
+
options = Parser.parse(argv)
|
95
|
+
outbox_consumer = build_consumer(options)
|
96
|
+
outbox_consumer.init
|
97
|
+
outbox_consumer.run
|
98
|
+
end
|
99
|
+
|
100
|
+
def build_consumer(options)
|
101
|
+
consumer_uuid = SecureRandom.uuid
|
102
|
+
logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox #{consumer_uuid}")
|
103
|
+
<<<<<<< HEAD
|
104
|
+
consumer_configuration =
|
105
|
+
Consumer::Configuration.new(
|
106
|
+
split_keys: options.split_keys,
|
107
|
+
message_format: options.message_format,
|
108
|
+
batch_size: options.batch_size,
|
109
|
+
database_url: options.database_url,
|
110
|
+
redis_url: options.redis_url,
|
111
|
+
cleanup: options.cleanup_strategy,
|
112
|
+
sleep_on_empty: options.sleep_on_empty
|
113
|
+
)
|
114
|
+
=======
|
115
|
+
consumer_configuration = Consumer::Configuration.new(
|
116
|
+
split_keys: options.split_keys,
|
117
|
+
message_format: options.message_format,
|
118
|
+
batch_size: options.batch_size,
|
119
|
+
database_url: options.database_url,
|
120
|
+
redis_url: options.redis_url,
|
121
|
+
cleanup: options.cleanup_strategy,
|
122
|
+
cleanup_limit: options.cleanup_limit,
|
123
|
+
sleep_on_empty: options.sleep_on_empty,
|
124
|
+
)
|
125
|
+
>>>>>>> 7f077fa2... fix error with passing `--cleanup-limit` from CLI to consumer
|
126
|
+
metrics = Metrics.from_url(options.metrics_url)
|
127
|
+
outbox_consumer =
|
128
|
+
RubyEventStore::Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
@@ -48,7 +48,14 @@ module RubyEventStore
|
|
48
48
|
)
|
49
49
|
end
|
50
50
|
|
51
|
-
attr_reader :split_keys,
|
51
|
+
attr_reader :split_keys,
|
52
|
+
:message_format,
|
53
|
+
:batch_size,
|
54
|
+
:database_url,
|
55
|
+
:redis_url,
|
56
|
+
:cleanup,
|
57
|
+
:cleanup_limit,
|
58
|
+
:sleep_on_empty
|
52
59
|
end
|
53
60
|
|
54
61
|
def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
|
@@ -67,12 +74,17 @@ module RubyEventStore
|
|
67
74
|
prepare_traps
|
68
75
|
|
69
76
|
@repository = Repository.new(configuration.database_url)
|
70
|
-
@cleanup_strategy =
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
77
|
+
@cleanup_strategy =
|
78
|
+
case configuration.cleanup
|
79
|
+
when :none
|
80
|
+
CleanupStrategies::None.new
|
81
|
+
else
|
82
|
+
CleanupStrategies::CleanOldEnqueued.new(
|
83
|
+
repository,
|
84
|
+
ActiveSupport::Duration.parse(configuration.cleanup),
|
85
|
+
configuration.cleanup_limit
|
86
|
+
)
|
87
|
+
end
|
76
88
|
end
|
77
89
|
|
78
90
|
def init
|
@@ -81,7 +93,7 @@ module RubyEventStore
|
|
81
93
|
end
|
82
94
|
|
83
95
|
def run
|
84
|
-
while !@gracefully_shutting_down
|
96
|
+
while !@gracefully_shutting_down
|
85
97
|
was_something_changed = one_loop
|
86
98
|
if !was_something_changed
|
87
99
|
STDOUT.flush
|
@@ -109,9 +121,7 @@ module RubyEventStore
|
|
109
121
|
|
110
122
|
MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK.times do
|
111
123
|
batch = retrieve_batch(fetch_specification)
|
112
|
-
if batch.empty?
|
113
|
-
break
|
114
|
-
end
|
124
|
+
break if batch.empty?
|
115
125
|
|
116
126
|
failed_record_ids = []
|
117
127
|
updated_record_ids = []
|
@@ -125,7 +135,7 @@ module RubyEventStore
|
|
125
135
|
updated_record_ids << record.id
|
126
136
|
rescue => e
|
127
137
|
failed_record_ids << record.id
|
128
|
-
e.full_message.split($/).each {|line| logger.error(line) }
|
138
|
+
e.full_message.split($/).each { |line| logger.error(line) }
|
129
139
|
end
|
130
140
|
end
|
131
141
|
|
@@ -143,11 +153,13 @@ module RubyEventStore
|
|
143
153
|
break unless refresh_successful
|
144
154
|
end
|
145
155
|
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
156
|
+
unless something_processed
|
157
|
+
metrics.write_point_queue(
|
158
|
+
format: fetch_specification.message_format,
|
159
|
+
split_key: fetch_specification.split_key,
|
160
|
+
remaining: get_remaining_count(fetch_specification)
|
161
|
+
)
|
162
|
+
end
|
151
163
|
|
152
164
|
release_lock_for_process(fetch_specification)
|
153
165
|
|
@@ -159,7 +171,16 @@ module RubyEventStore
|
|
159
171
|
end
|
160
172
|
|
161
173
|
private
|
162
|
-
|
174
|
+
|
175
|
+
attr_reader :split_keys,
|
176
|
+
:logger,
|
177
|
+
:batch_size,
|
178
|
+
:metrics,
|
179
|
+
:processor,
|
180
|
+
:consumer_uuid,
|
181
|
+
:repository,
|
182
|
+
:cleanup_strategy,
|
183
|
+
:sleep_on_empty
|
163
184
|
|
164
185
|
def obtain_lock_for_process(fetch_specification)
|
165
186
|
result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
|
@@ -231,12 +252,8 @@ module RubyEventStore
|
|
231
252
|
end
|
232
253
|
|
233
254
|
def prepare_traps
|
234
|
-
Signal.trap("INT")
|
235
|
-
|
236
|
-
end
|
237
|
-
Signal.trap("TERM") do
|
238
|
-
initiate_graceful_shutdown
|
239
|
-
end
|
255
|
+
Signal.trap("INT") { initiate_graceful_shutdown }
|
256
|
+
Signal.trap("TERM") { initiate_graceful_shutdown }
|
240
257
|
end
|
241
258
|
|
242
259
|
def initiate_graceful_shutdown
|
@@ -10,19 +10,13 @@ module RubyEventStore
|
|
10
10
|
attr_reader :message_format, :split_key
|
11
11
|
|
12
12
|
def ==(other)
|
13
|
-
other.instance_of?(self.class) &&
|
14
|
-
other.message_format.eql?(message_format) &&
|
15
|
-
other.split_key.eql?(split_key)
|
13
|
+
other.instance_of?(self.class) && other.message_format.eql?(message_format) && other.split_key.eql?(split_key)
|
16
14
|
end
|
17
15
|
|
18
16
|
BIG_VALUE = 0b111111100100010000010010110010101011011101110101001100100110000
|
19
17
|
|
20
18
|
def hash
|
21
|
-
[
|
22
|
-
self.class,
|
23
|
-
message_format,
|
24
|
-
split_key,
|
25
|
-
].hash ^ BIG_VALUE
|
19
|
+
[self.class, message_format, split_key].hash ^ BIG_VALUE
|
26
20
|
end
|
27
21
|
|
28
22
|
alias_method :eql?, :==
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "influxdb"
|
2
2
|
|
3
3
|
module RubyEventStore
|
4
4
|
module Outbox
|
@@ -6,38 +6,32 @@ module RubyEventStore
|
|
6
6
|
class Influx
|
7
7
|
def initialize(url)
|
8
8
|
uri = URI.parse(url)
|
9
|
-
options = {
|
10
|
-
url: url,
|
11
|
-
async: true,
|
12
|
-
time_precision: 'ns',
|
13
|
-
}
|
9
|
+
options = { url: url, async: true, time_precision: "ns" }
|
14
10
|
@influxdb_client = InfluxDB::Client.new(**options)
|
15
11
|
end
|
16
12
|
|
17
13
|
def write_operation_result(operation, result)
|
18
|
-
write_point(
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
tags: {
|
23
|
-
operation: operation,
|
24
|
-
result: result,
|
25
|
-
}
|
26
|
-
})
|
14
|
+
write_point(
|
15
|
+
"ruby_event_store.outbox.lock",
|
16
|
+
{ values: { value: 1 }, tags: { operation: operation, result: result } }
|
17
|
+
)
|
27
18
|
end
|
28
19
|
|
29
20
|
def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
|
30
|
-
write_point(
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
21
|
+
write_point(
|
22
|
+
"ruby_event_store.outbox.queue",
|
23
|
+
{
|
24
|
+
values: {
|
25
|
+
enqueued: enqueued,
|
26
|
+
failed: failed,
|
27
|
+
remaining: remaining
|
28
|
+
},
|
29
|
+
tags: {
|
30
|
+
format: format,
|
31
|
+
split_key: split_key
|
32
|
+
}
|
39
33
|
}
|
40
|
-
|
34
|
+
)
|
41
35
|
end
|
42
36
|
|
43
37
|
def write_point(series, data)
|
@@ -2,11 +2,9 @@ module RubyEventStore
|
|
2
2
|
module Outbox
|
3
3
|
module Metrics
|
4
4
|
class Null
|
5
|
-
def write_operation_result(operation, result)
|
6
|
-
end
|
5
|
+
def write_operation_result(operation, result); end
|
7
6
|
|
8
|
-
def write_point_queue(**kwargs)
|
9
|
-
end
|
7
|
+
def write_point_queue(**kwargs); end
|
10
8
|
end
|
11
9
|
end
|
12
10
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "influxdb"
|
2
2
|
|
3
3
|
module RubyEventStore
|
4
4
|
module Outbox
|
@@ -14,7 +14,13 @@ module RubyEventStore
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
|
17
|
-
@queue_stats << {
|
17
|
+
@queue_stats << {
|
18
|
+
enqueued: enqueued,
|
19
|
+
failed: failed,
|
20
|
+
remaining: remaining,
|
21
|
+
format: format,
|
22
|
+
split_key: split_key
|
23
|
+
}
|
18
24
|
end
|
19
25
|
|
20
26
|
attr_reader :operation_results, :queue_stats
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require "active_record"
|
4
|
+
require "active_support/core_ext/numeric/time.rb"
|
5
5
|
|
6
6
|
module RubyEventStore
|
7
7
|
module Outbox
|
@@ -10,7 +10,7 @@ module RubyEventStore
|
|
10
10
|
|
11
11
|
class Record < ::ActiveRecord::Base
|
12
12
|
self.primary_key = :id
|
13
|
-
self.table_name =
|
13
|
+
self.table_name = "event_store_outbox"
|
14
14
|
|
15
15
|
def self.remaining_for(fetch_specification)
|
16
16
|
where(format: fetch_specification.message_format, split_key: fetch_specification.split_key, enqueued_at: nil)
|
@@ -30,7 +30,7 @@ module RubyEventStore
|
|
30
30
|
end
|
31
31
|
|
32
32
|
class Lock < ::ActiveRecord::Base
|
33
|
-
self.table_name =
|
33
|
+
self.table_name = "event_store_outbox_locks"
|
34
34
|
|
35
35
|
def self.obtain(fetch_specification, process_uuid, clock:)
|
36
36
|
transaction do
|
@@ -39,10 +39,7 @@ module RubyEventStore
|
|
39
39
|
if l.recently_locked?(clock: clock)
|
40
40
|
:taken
|
41
41
|
else
|
42
|
-
l.update!(
|
43
|
-
locked_by: process_uuid,
|
44
|
-
locked_at: clock.now,
|
45
|
-
)
|
42
|
+
l.update!(locked_by: process_uuid, locked_at: clock.now)
|
46
43
|
l
|
47
44
|
end
|
48
45
|
end
|
@@ -98,6 +95,7 @@ module RubyEventStore
|
|
98
95
|
end
|
99
96
|
|
100
97
|
private
|
98
|
+
|
101
99
|
def self.lock_for_split_key(fetch_specification)
|
102
100
|
lock.find_by(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
|
103
101
|
end
|
@@ -143,10 +141,8 @@ module RubyEventStore
|
|
143
141
|
end
|
144
142
|
|
145
143
|
def delete_enqueued_older_than(fetch_specification, duration, limit)
|
146
|
-
scope = Record
|
147
|
-
|
148
|
-
.where("enqueued_at < ?", duration.ago)
|
149
|
-
scope = scope.limit(limit) unless limit == :all
|
144
|
+
scope = Record.for_fetch_specification(fetch_specification).where("enqueued_at < ?", duration.ago)
|
145
|
+
scope = scope.limit(limit).order(:id) unless limit == :all
|
150
146
|
scope.delete_all
|
151
147
|
:ok
|
152
148
|
rescue ActiveRecord::Deadlocked
|
@@ -17,9 +17,7 @@ module RubyEventStore
|
|
17
17
|
|
18
18
|
queue = parsed_record["queue"]
|
19
19
|
raise InvalidPayload.new("Missing queue") if queue.nil? || queue.empty?
|
20
|
-
payload = JSON.generate(parsed_record.merge({
|
21
|
-
"enqueued_at" => now.to_f,
|
22
|
-
}))
|
20
|
+
payload = JSON.generate(parsed_record.merge({ "enqueued_at" => now.to_f }))
|
23
21
|
|
24
22
|
redis.lpush("queue:#{queue}", payload)
|
25
23
|
|
@@ -38,6 +36,7 @@ module RubyEventStore
|
|
38
36
|
end
|
39
37
|
|
40
38
|
private
|
39
|
+
|
41
40
|
attr_reader :redis
|
42
41
|
end
|
43
42
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require_relative
|
3
|
+
require "sidekiq"
|
4
|
+
require_relative "sidekiq5_format"
|
5
5
|
require_relative "repository"
|
6
6
|
|
7
7
|
module RubyEventStore
|
@@ -9,22 +9,25 @@ module RubyEventStore
|
|
9
9
|
class SidekiqProducer
|
10
10
|
def call(klass, args)
|
11
11
|
sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
|
12
|
-
item = {
|
13
|
-
'args' => args.map(&:to_h).map {|h| h.transform_keys(&:to_s)},
|
14
|
-
'class' => klass,
|
15
|
-
}
|
12
|
+
item = { "args" => args.map(&:to_h).map { |h| h.transform_keys(&:to_s) }, "class" => klass }
|
16
13
|
normalized_item = sidekiq_client.__send__(:normalize_item, item)
|
17
|
-
payload =
|
14
|
+
payload =
|
15
|
+
sidekiq_client
|
16
|
+
.middleware
|
17
|
+
.invoke(normalized_item["class"], normalized_item, normalized_item["queue"], Sidekiq.redis_pool) do
|
18
|
+
normalized_item
|
19
|
+
end
|
18
20
|
if payload
|
19
21
|
Repository::Record.create!(
|
20
22
|
format: SIDEKIQ5_FORMAT,
|
21
|
-
split_key: payload.fetch(
|
23
|
+
split_key: payload.fetch("queue"),
|
22
24
|
payload: payload.to_json
|
23
25
|
)
|
24
26
|
end
|
25
27
|
end
|
26
28
|
|
27
29
|
private
|
30
|
+
|
28
31
|
attr_reader :repository
|
29
32
|
end
|
30
33
|
end
|
@@ -5,8 +5,8 @@ module RubyEventStore
|
|
5
5
|
end
|
6
6
|
end
|
7
7
|
|
8
|
-
require_relative
|
9
|
-
require_relative
|
10
|
-
require_relative
|
11
|
-
require_relative
|
12
|
-
require_relative
|
8
|
+
require_relative "outbox/fetch_specification"
|
9
|
+
require_relative "outbox/repository"
|
10
|
+
require_relative "outbox/sidekiq_scheduler"
|
11
|
+
require_relative "outbox/legacy_sidekiq_scheduler"
|
12
|
+
require_relative "outbox/version"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby_event_store-outbox
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.25
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arkency
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-05-
|
11
|
+
date: 2022-05-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ruby_event_store
|
@@ -30,14 +30,14 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '6.0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '6.0'
|
41
41
|
description:
|
42
42
|
email: dev@arkency.com
|
43
43
|
executables:
|
@@ -54,6 +54,7 @@ files:
|
|
54
54
|
- lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb
|
55
55
|
- lib/ruby_event_store/outbox/cleanup_strategies/none.rb
|
56
56
|
- lib/ruby_event_store/outbox/cli.rb
|
57
|
+
- lib/ruby_event_store/outbox/cli.rb.orig
|
57
58
|
- lib/ruby_event_store/outbox/consumer.rb
|
58
59
|
- lib/ruby_event_store/outbox/fetch_specification.rb
|
59
60
|
- lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb
|