ruby_event_store-outbox 0.0.22 → 0.0.25

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cf41fff7a72941537c0247985765f8ca110bc17f14f1e886fca032f172ec918b
4
- data.tar.gz: cef6f36936589755f6f636f07ef4edbc823b47ddb2a616700ec63558c75361b8
3
+ metadata.gz: d9f4e4b0d475d2ad92997aee74165ffacbabbe19282b5d1b266b5ec7b7d3b59b
4
+ data.tar.gz: adccd92a63810fb4f285c5ea5e4ad37cc22ad671d4550b1d73b289f179d5831b
5
5
  SHA512:
6
- metadata.gz: 30f86dcda05b796a4adce6cfa7643a2c2aa7b492ce6efe72bed540aa14d5477cc8f81dbd6bbf918500c007490f3857bad8cd268351b05b39d16dfa46ba6213e9
7
- data.tar.gz: 25ecce3dc414e61b775cc618ada4c2449f7be1217ed90637a485b5e372f1a1fd9fab3a8dd681ded98409ddb3537c3d2c156e65dae9527940d847fc52b893a998
6
+ metadata.gz: 4a858aff2550a901615bad83ffa58dc6e30f669a8aa8a422d047343b74a32317d7476e27e95127f9750640eab2e538385b3284840f4c0a6532eff6ee8a20e96e
7
+ data.tar.gz: b81515fba19988345a3f45397de2d674f94e331475140246ccb5a7266cb19f49f6f2d59445b99fe139b6770e59310a53090f7e9eec2e6503767cf57e5801abb8
data/README.md CHANGED
@@ -34,11 +34,12 @@ Additionally, your handler's `through_outbox?` method should return `true`, for
34
34
 
35
35
  ```ruby
36
36
  class SomeHandler
37
- def self.through_outbox?; true; end
37
+ def self.through_outbox?
38
+ true
39
+ end
38
40
  end
39
41
  ```
40
42
 
41
-
42
43
  ## Installation (outbox process)
43
44
 
44
45
  Run following process in any way you prefer:
@@ -65,7 +66,6 @@ res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" \
65
66
  --metrics-url=http://user:password@localhost:8086/dbname"
66
67
  ```
67
68
 
68
-
69
69
  ## Contributing
70
70
 
71
71
  Bug reports and pull requests are welcome on GitHub at https://github.com/RailsEventStore/rails_event_store.
@@ -1,28 +1,30 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  begin
4
- require 'rails/generators'
4
+ require "rails/generators"
5
5
  rescue LoadError
6
6
  end
7
7
 
8
- module RubyEventStore
9
- module Outbox
10
- class MigrationGenerator < Rails::Generators::Base
11
- source_root File.expand_path(File.join(File.dirname(__FILE__), './templates'))
8
+ if defined?(Rails::Generators::Base)
9
+ module RubyEventStore
10
+ module Outbox
11
+ class MigrationGenerator < Rails::Generators::Base
12
+ source_root File.expand_path(File.join(File.dirname(__FILE__), "./templates"))
12
13
 
13
- def create_migration
14
- template "create_event_store_outbox_template.rb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
15
- end
14
+ def create_migration
15
+ template "create_event_store_outbox_template.rb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
16
+ end
16
17
 
17
- private
18
+ private
18
19
 
19
- def migration_version
20
- "[4.2]"
21
- end
20
+ def migration_version
21
+ "[4.2]"
22
+ end
22
23
 
23
- def timestamp
24
- Time.now.strftime("%Y%m%d%H%M%S")
24
+ def timestamp
25
+ Time.now.strftime("%Y%m%d%H%M%S")
26
+ end
25
27
  end
26
28
  end
27
29
  end
28
- end if defined?(Rails::Generators::Base)
30
+ end
@@ -2,17 +2,19 @@ module RubyEventStore
2
2
  module Outbox
3
3
  module CleanupStrategies
4
4
  class CleanOldEnqueued
5
- def initialize(repository, duration)
5
+ def initialize(repository, duration, limit)
6
6
  @repository = repository
7
7
  @duration = duration
8
+ @limit = limit
8
9
  end
9
10
 
10
11
  def call(fetch_specification)
11
- repository.delete_enqueued_older_than(fetch_specification, duration)
12
+ repository.delete_enqueued_older_than(fetch_specification, duration, limit)
12
13
  end
13
14
 
14
15
  private
15
- attr_reader :repository, :duration
16
+
17
+ attr_reader :repository, :duration, :limit
16
18
  end
17
19
  end
18
20
  end
@@ -2,8 +2,7 @@ module RubyEventStore
2
2
  module Outbox
3
3
  module CleanupStrategies
4
4
  class None
5
- def initialize
6
- end
5
+ def initialize; end
7
6
 
8
7
  def call(_fetch_specification)
9
8
  :ok
@@ -11,10 +11,11 @@ module RubyEventStore
11
11
  redis_url: nil,
12
12
  log_level: :warn,
13
13
  split_keys: nil,
14
- message_format: 'sidekiq5',
14
+ message_format: "sidekiq5",
15
15
  batch_size: 100,
16
16
  metrics_url: nil,
17
17
  cleanup_strategy: :none,
18
+ cleanup_limit: :all,
18
19
  sleep_on_empty: 0.5
19
20
  }
20
21
  Options = Struct.new(*DEFAULTS.keys)
@@ -22,50 +23,69 @@ module RubyEventStore
22
23
  class Parser
23
24
  def self.parse(argv)
24
25
  options = Options.new(*DEFAULTS.values)
25
- OptionParser.new do |option_parser|
26
- option_parser.banner = "Usage: res_outbox [options]"
26
+ OptionParser
27
+ .new do |option_parser|
28
+ option_parser.banner = "Usage: res_outbox [options]"
27
29
 
28
- option_parser.on("--database-url=DATABASE_URL", "Database where outbox table is stored") do |database_url|
29
- options.database_url = database_url
30
- end
30
+ option_parser.on(
31
+ "--database-url=DATABASE_URL",
32
+ "Database where outbox table is stored"
33
+ ) { |database_url| options.database_url = database_url }
31
34
 
32
- option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
33
- options.redis_url = redis_url
34
- end
35
+ option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
36
+ options.redis_url = redis_url
37
+ end
35
38
 
36
- option_parser.on("--log-level=LOG_LEVEL", [:fatal, :error, :warn, :info, :debug], "Logging level, one of: fatal, error, warn, info, debug. Default: warn") do |log_level|
37
- options.log_level = log_level.to_sym
38
- end
39
+ option_parser.on(
40
+ "--log-level=LOG_LEVEL",
41
+ %i[fatal error warn info debug],
42
+ "Logging level, one of: fatal, error, warn, info, debug. Default: warn"
43
+ ) { |log_level| options.log_level = log_level.to_sym }
39
44
 
40
- option_parser.on("--message-format=FORMAT", ["sidekiq5"], "Message format, supported: sidekiq5. Default: sidekiq5") do |message_format|
41
- options.message_format = message_format
42
- end
45
+ option_parser.on(
46
+ "--message-format=FORMAT",
47
+ ["sidekiq5"],
48
+ "Message format, supported: sidekiq5. Default: sidekiq5"
49
+ ) { |message_format| options.message_format = message_format }
43
50
 
44
- option_parser.on("--split-keys=SPLIT_KEYS", Array, "Split keys which should be handled, all if not specified") do |split_keys|
45
- options.split_keys = split_keys if !split_keys.empty?
46
- end
51
+ option_parser.on(
52
+ "--split-keys=SPLIT_KEYS",
53
+ Array,
54
+ "Split keys which should be handled, all if not specified"
55
+ ) { |split_keys| options.split_keys = split_keys if !split_keys.empty? }
47
56
 
48
- option_parser.on("--batch-size=BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100") do |batch_size|
49
- options.batch_size = batch_size
50
- end
57
+ option_parser.on(
58
+ "--batch-size=BATCH_SIZE",
59
+ Integer,
60
+ "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100"
61
+ ) { |batch_size| options.batch_size = batch_size }
51
62
 
52
- option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
53
- options.metrics_url = metrics_url
54
- end
63
+ option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
64
+ options.metrics_url = metrics_url
65
+ end
55
66
 
56
- option_parser.on("--cleanup=STRATEGY", "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none") do |cleanup_strategy|
57
- options.cleanup_strategy = cleanup_strategy
58
- end
67
+ option_parser.on(
68
+ "--cleanup=STRATEGY",
69
+ "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none"
70
+ ) { |cleanup_strategy| options.cleanup_strategy = cleanup_strategy }
59
71
 
60
- option_parser.on("--sleep-on-empty=SLEEP_TIME", Float, "How long to sleep before next check when there was nothing to do. Default: 0.5") do |sleep_on_empty|
61
- options.sleep_on_empty = sleep_on_empty
62
- end
72
+ option_parser.on(
73
+ "--cleanup-limit=LIMIT",
74
+ "Amount of records removed in single cleanup run. One of: all or number of records that should be removed. Default: all"
75
+ ) { |cleanup_limit| options.cleanup_limit = cleanup_limit }
63
76
 
64
- option_parser.on_tail("--version", "Show version") do
65
- puts VERSION
66
- exit
77
+ option_parser.on(
78
+ "--sleep-on-empty=SLEEP_TIME",
79
+ Float,
80
+ "How long to sleep before next check when there was nothing to do. Default: 0.5"
81
+ ) { |sleep_on_empty| options.sleep_on_empty = sleep_on_empty }
82
+
83
+ option_parser.on_tail("--version", "Show version") do
84
+ puts VERSION
85
+ exit
86
+ end
67
87
  end
68
- end.parse(argv)
88
+ .parse(argv)
69
89
  return options
70
90
  end
71
91
  end
@@ -87,15 +107,12 @@ module RubyEventStore
87
107
  database_url: options.database_url,
88
108
  redis_url: options.redis_url,
89
109
  cleanup: options.cleanup_strategy,
110
+ cleanup_limit: options.cleanup_limit,
90
111
  sleep_on_empty: options.sleep_on_empty,
91
112
  )
92
113
  metrics = Metrics.from_url(options.metrics_url)
93
- outbox_consumer = RubyEventStore::Outbox::Consumer.new(
94
- consumer_uuid,
95
- consumer_configuration,
96
- logger: logger,
97
- metrics: metrics,
98
- )
114
+ outbox_consumer =
115
+ RubyEventStore::Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
99
116
  end
100
117
  end
101
118
  end
@@ -0,0 +1,132 @@
1
+ require "optparse"
2
+ require_relative "version"
3
+ require_relative "consumer"
4
+ require_relative "metrics"
5
+
6
+ module RubyEventStore
7
+ module Outbox
8
+ class CLI
9
+ DEFAULTS = {
10
+ database_url: nil,
11
+ redis_url: nil,
12
+ log_level: :warn,
13
+ split_keys: nil,
14
+ message_format: "sidekiq5",
15
+ batch_size: 100,
16
+ metrics_url: nil,
17
+ cleanup_strategy: :none,
18
+ cleanup_limit: :all,
19
+ sleep_on_empty: 0.5
20
+ }
21
+ Options = Struct.new(*DEFAULTS.keys)
22
+
23
+ class Parser
24
+ def self.parse(argv)
25
+ options = Options.new(*DEFAULTS.values)
26
+ OptionParser
27
+ .new do |option_parser|
28
+ option_parser.banner = "Usage: res_outbox [options]"
29
+
30
+ option_parser.on(
31
+ "--database-url=DATABASE_URL",
32
+ "Database where outbox table is stored"
33
+ ) { |database_url| options.database_url = database_url }
34
+
35
+ option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
36
+ options.redis_url = redis_url
37
+ end
38
+
39
+ option_parser.on(
40
+ "--log-level=LOG_LEVEL",
41
+ %i[fatal error warn info debug],
42
+ "Logging level, one of: fatal, error, warn, info, debug. Default: warn"
43
+ ) { |log_level| options.log_level = log_level.to_sym }
44
+
45
+ option_parser.on(
46
+ "--message-format=FORMAT",
47
+ ["sidekiq5"],
48
+ "Message format, supported: sidekiq5. Default: sidekiq5"
49
+ ) { |message_format| options.message_format = message_format }
50
+
51
+ option_parser.on(
52
+ "--split-keys=SPLIT_KEYS",
53
+ Array,
54
+ "Split keys which should be handled, all if not specified"
55
+ ) { |split_keys| options.split_keys = split_keys if !split_keys.empty? }
56
+
57
+ option_parser.on(
58
+ "--batch-size=BATCH_SIZE",
59
+ Integer,
60
+ "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100"
61
+ ) { |batch_size| options.batch_size = batch_size }
62
+
63
+ option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
64
+ options.metrics_url = metrics_url
65
+ end
66
+
67
+ option_parser.on(
68
+ "--cleanup=STRATEGY",
69
+ "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none"
70
+ ) { |cleanup_strategy| options.cleanup_strategy = cleanup_strategy }
71
+
72
+ option_parser.on(
73
+ "--cleanup-limit=LIMIT",
74
+ "Amount of records removed in single cleanup run. One of: all or number of records that should be removed. Default: all"
75
+ ) { |cleanup_limit| options.cleanup_limit = cleanup_limit }
76
+
77
+ option_parser.on(
78
+ "--sleep-on-empty=SLEEP_TIME",
79
+ Float,
80
+ "How long to sleep before next check when there was nothing to do. Default: 0.5"
81
+ ) { |sleep_on_empty| options.sleep_on_empty = sleep_on_empty }
82
+
83
+ option_parser.on_tail("--version", "Show version") do
84
+ puts VERSION
85
+ exit
86
+ end
87
+ end
88
+ .parse(argv)
89
+ return options
90
+ end
91
+ end
92
+
93
+ def run(argv)
94
+ options = Parser.parse(argv)
95
+ outbox_consumer = build_consumer(options)
96
+ outbox_consumer.init
97
+ outbox_consumer.run
98
+ end
99
+
100
+ def build_consumer(options)
101
+ consumer_uuid = SecureRandom.uuid
102
+ logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox #{consumer_uuid}")
103
+ <<<<<<< HEAD
104
+ consumer_configuration =
105
+ Consumer::Configuration.new(
106
+ split_keys: options.split_keys,
107
+ message_format: options.message_format,
108
+ batch_size: options.batch_size,
109
+ database_url: options.database_url,
110
+ redis_url: options.redis_url,
111
+ cleanup: options.cleanup_strategy,
112
+ sleep_on_empty: options.sleep_on_empty
113
+ )
114
+ =======
115
+ consumer_configuration = Consumer::Configuration.new(
116
+ split_keys: options.split_keys,
117
+ message_format: options.message_format,
118
+ batch_size: options.batch_size,
119
+ database_url: options.database_url,
120
+ redis_url: options.redis_url,
121
+ cleanup: options.cleanup_strategy,
122
+ cleanup_limit: options.cleanup_limit,
123
+ sleep_on_empty: options.sleep_on_empty,
124
+ )
125
+ >>>>>>> 7f077fa2... fix error with passing `--cleanup-limit` from CLI to consumer
126
+ metrics = Metrics.from_url(options.metrics_url)
127
+ outbox_consumer =
128
+ RubyEventStore::Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
129
+ end
130
+ end
131
+ end
132
+ end
@@ -21,6 +21,7 @@ module RubyEventStore
21
21
  database_url:,
22
22
  redis_url:,
23
23
  cleanup:,
24
+ cleanup_limit:,
24
25
  sleep_on_empty:
25
26
  )
26
27
  @split_keys = split_keys
@@ -29,6 +30,7 @@ module RubyEventStore
29
30
  @database_url = database_url
30
31
  @redis_url = redis_url
31
32
  @cleanup = cleanup
33
+ @cleanup_limit = cleanup_limit
32
34
  @sleep_on_empty = sleep_on_empty
33
35
  freeze
34
36
  end
@@ -41,11 +43,19 @@ module RubyEventStore
41
43
  database_url: overriden_options.fetch(:database_url, database_url),
42
44
  redis_url: overriden_options.fetch(:redis_url, redis_url),
43
45
  cleanup: overriden_options.fetch(:cleanup, cleanup),
46
+ cleanup_limit: overriden_options.fetch(:cleanup_limit, cleanup_limit),
44
47
  sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
45
48
  )
46
49
  end
47
50
 
48
- attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url, :cleanup, :sleep_on_empty
51
+ attr_reader :split_keys,
52
+ :message_format,
53
+ :batch_size,
54
+ :database_url,
55
+ :redis_url,
56
+ :cleanup,
57
+ :cleanup_limit,
58
+ :sleep_on_empty
49
59
  end
50
60
 
51
61
  def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
@@ -64,12 +74,17 @@ module RubyEventStore
64
74
  prepare_traps
65
75
 
66
76
  @repository = Repository.new(configuration.database_url)
67
- @cleanup_strategy = case configuration.cleanup
68
- when :none
69
- CleanupStrategies::None.new
70
- else
71
- CleanupStrategies::CleanOldEnqueued.new(repository, ActiveSupport::Duration.parse(configuration.cleanup))
72
- end
77
+ @cleanup_strategy =
78
+ case configuration.cleanup
79
+ when :none
80
+ CleanupStrategies::None.new
81
+ else
82
+ CleanupStrategies::CleanOldEnqueued.new(
83
+ repository,
84
+ ActiveSupport::Duration.parse(configuration.cleanup),
85
+ configuration.cleanup_limit
86
+ )
87
+ end
73
88
  end
74
89
 
75
90
  def init
@@ -78,7 +93,7 @@ module RubyEventStore
78
93
  end
79
94
 
80
95
  def run
81
- while !@gracefully_shutting_down do
96
+ while !@gracefully_shutting_down
82
97
  was_something_changed = one_loop
83
98
  if !was_something_changed
84
99
  STDOUT.flush
@@ -106,9 +121,7 @@ module RubyEventStore
106
121
 
107
122
  MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK.times do
108
123
  batch = retrieve_batch(fetch_specification)
109
- if batch.empty?
110
- break
111
- end
124
+ break if batch.empty?
112
125
 
113
126
  failed_record_ids = []
114
127
  updated_record_ids = []
@@ -122,7 +135,7 @@ module RubyEventStore
122
135
  updated_record_ids << record.id
123
136
  rescue => e
124
137
  failed_record_ids << record.id
125
- e.full_message.split($/).each {|line| logger.error(line) }
138
+ e.full_message.split($/).each { |line| logger.error(line) }
126
139
  end
127
140
  end
128
141
 
@@ -140,11 +153,13 @@ module RubyEventStore
140
153
  break unless refresh_successful
141
154
  end
142
155
 
143
- metrics.write_point_queue(
144
- format: fetch_specification.message_format,
145
- split_key: fetch_specification.split_key,
146
- remaining: get_remaining_count(fetch_specification)
147
- ) unless something_processed
156
+ unless something_processed
157
+ metrics.write_point_queue(
158
+ format: fetch_specification.message_format,
159
+ split_key: fetch_specification.split_key,
160
+ remaining: get_remaining_count(fetch_specification)
161
+ )
162
+ end
148
163
 
149
164
  release_lock_for_process(fetch_specification)
150
165
 
@@ -156,7 +171,16 @@ module RubyEventStore
156
171
  end
157
172
 
158
173
  private
159
- attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid, :repository, :cleanup_strategy, :sleep_on_empty
174
+
175
+ attr_reader :split_keys,
176
+ :logger,
177
+ :batch_size,
178
+ :metrics,
179
+ :processor,
180
+ :consumer_uuid,
181
+ :repository,
182
+ :cleanup_strategy,
183
+ :sleep_on_empty
160
184
 
161
185
  def obtain_lock_for_process(fetch_specification)
162
186
  result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
@@ -228,12 +252,8 @@ module RubyEventStore
228
252
  end
229
253
 
230
254
  def prepare_traps
231
- Signal.trap("INT") do
232
- initiate_graceful_shutdown
233
- end
234
- Signal.trap("TERM") do
235
- initiate_graceful_shutdown
236
- end
255
+ Signal.trap("INT") { initiate_graceful_shutdown }
256
+ Signal.trap("TERM") { initiate_graceful_shutdown }
237
257
  end
238
258
 
239
259
  def initiate_graceful_shutdown
@@ -10,19 +10,13 @@ module RubyEventStore
10
10
  attr_reader :message_format, :split_key
11
11
 
12
12
  def ==(other)
13
- other.instance_of?(self.class) &&
14
- other.message_format.eql?(message_format) &&
15
- other.split_key.eql?(split_key)
13
+ other.instance_of?(self.class) && other.message_format.eql?(message_format) && other.split_key.eql?(split_key)
16
14
  end
17
15
 
18
16
  BIG_VALUE = 0b111111100100010000010010110010101011011101110101001100100110000
19
17
 
20
18
  def hash
21
- [
22
- self.class,
23
- message_format,
24
- split_key,
25
- ].hash ^ BIG_VALUE
19
+ [self.class, message_format, split_key].hash ^ BIG_VALUE
26
20
  end
27
21
 
28
22
  alias_method :eql?, :==
@@ -18,6 +18,7 @@ module RubyEventStore
18
18
  end
19
19
 
20
20
  private
21
+
21
22
  attr_reader :sidekiq_producer
22
23
  end
23
24
  end
@@ -1,4 +1,4 @@
1
- require 'influxdb'
1
+ require "influxdb"
2
2
 
3
3
  module RubyEventStore
4
4
  module Outbox
@@ -6,38 +6,32 @@ module RubyEventStore
6
6
  class Influx
7
7
  def initialize(url)
8
8
  uri = URI.parse(url)
9
- options = {
10
- url: url,
11
- async: true,
12
- time_precision: 'ns',
13
- }
9
+ options = { url: url, async: true, time_precision: "ns" }
14
10
  @influxdb_client = InfluxDB::Client.new(**options)
15
11
  end
16
12
 
17
13
  def write_operation_result(operation, result)
18
- write_point("ruby_event_store.outbox.lock", {
19
- values: {
20
- value: 1,
21
- },
22
- tags: {
23
- operation: operation,
24
- result: result,
25
- }
26
- })
14
+ write_point(
15
+ "ruby_event_store.outbox.lock",
16
+ { values: { value: 1 }, tags: { operation: operation, result: result } }
17
+ )
27
18
  end
28
19
 
29
20
  def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
30
- write_point("ruby_event_store.outbox.queue", {
31
- values: {
32
- enqueued: enqueued,
33
- failed: failed,
34
- remaining: remaining,
35
- },
36
- tags: {
37
- format: format,
38
- split_key: split_key,
21
+ write_point(
22
+ "ruby_event_store.outbox.queue",
23
+ {
24
+ values: {
25
+ enqueued: enqueued,
26
+ failed: failed,
27
+ remaining: remaining
28
+ },
29
+ tags: {
30
+ format: format,
31
+ split_key: split_key
32
+ }
39
33
  }
40
- })
34
+ )
41
35
  end
42
36
 
43
37
  def write_point(series, data)
@@ -2,11 +2,9 @@ module RubyEventStore
2
2
  module Outbox
3
3
  module Metrics
4
4
  class Null
5
- def write_operation_result(operation, result)
6
- end
5
+ def write_operation_result(operation, result); end
7
6
 
8
- def write_point_queue(**kwargs)
9
- end
7
+ def write_point_queue(**kwargs); end
10
8
  end
11
9
  end
12
10
  end
@@ -1,4 +1,4 @@
1
- require 'influxdb'
1
+ require "influxdb"
2
2
 
3
3
  module RubyEventStore
4
4
  module Outbox
@@ -14,7 +14,13 @@ module RubyEventStore
14
14
  end
15
15
 
16
16
  def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
17
- @queue_stats << { enqueued: enqueued, failed: failed, remaining: remaining, format: format, split_key: split_key}
17
+ @queue_stats << {
18
+ enqueued: enqueued,
19
+ failed: failed,
20
+ remaining: remaining,
21
+ format: format,
22
+ split_key: split_key
23
+ }
18
24
  end
19
25
 
20
26
  attr_reader :operation_results, :queue_stats
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'active_record'
4
- require 'active_support/core_ext/numeric/time.rb'
3
+ require "active_record"
4
+ require "active_support/core_ext/numeric/time.rb"
5
5
 
6
6
  module RubyEventStore
7
7
  module Outbox
@@ -10,7 +10,7 @@ module RubyEventStore
10
10
 
11
11
  class Record < ::ActiveRecord::Base
12
12
  self.primary_key = :id
13
- self.table_name = 'event_store_outbox'
13
+ self.table_name = "event_store_outbox"
14
14
 
15
15
  def self.remaining_for(fetch_specification)
16
16
  where(format: fetch_specification.message_format, split_key: fetch_specification.split_key, enqueued_at: nil)
@@ -30,7 +30,7 @@ module RubyEventStore
30
30
  end
31
31
 
32
32
  class Lock < ::ActiveRecord::Base
33
- self.table_name = 'event_store_outbox_locks'
33
+ self.table_name = "event_store_outbox_locks"
34
34
 
35
35
  def self.obtain(fetch_specification, process_uuid, clock:)
36
36
  transaction do
@@ -39,10 +39,7 @@ module RubyEventStore
39
39
  if l.recently_locked?(clock: clock)
40
40
  :taken
41
41
  else
42
- l.update!(
43
- locked_by: process_uuid,
44
- locked_at: clock.now,
45
- )
42
+ l.update!(locked_by: process_uuid, locked_at: clock.now)
46
43
  l
47
44
  end
48
45
  end
@@ -98,6 +95,7 @@ module RubyEventStore
98
95
  end
99
96
 
100
97
  private
98
+
101
99
  def self.lock_for_split_key(fetch_specification)
102
100
  lock.find_by(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
103
101
  end
@@ -142,11 +140,10 @@ module RubyEventStore
142
140
  record.update_column(:enqueued_at, now)
143
141
  end
144
142
 
145
- def delete_enqueued_older_than(fetch_specification, duration)
146
- Record
147
- .for_fetch_specification(fetch_specification)
148
- .where("enqueued_at < ?", duration.ago)
149
- .delete_all
143
+ def delete_enqueued_older_than(fetch_specification, duration, limit)
144
+ scope = Record.for_fetch_specification(fetch_specification).where("enqueued_at < ?", duration.ago)
145
+ scope = scope.limit(limit).order(:id) unless limit == :all
146
+ scope.delete_all
150
147
  :ok
151
148
  rescue ActiveRecord::Deadlocked
152
149
  :deadlocked
@@ -17,9 +17,7 @@ module RubyEventStore
17
17
 
18
18
  queue = parsed_record["queue"]
19
19
  raise InvalidPayload.new("Missing queue") if queue.nil? || queue.empty?
20
- payload = JSON.generate(parsed_record.merge({
21
- "enqueued_at" => now.to_f,
22
- }))
20
+ payload = JSON.generate(parsed_record.merge({ "enqueued_at" => now.to_f }))
23
21
 
24
22
  redis.lpush("queue:#{queue}", payload)
25
23
 
@@ -38,6 +36,7 @@ module RubyEventStore
38
36
  end
39
37
 
40
38
  private
39
+
41
40
  attr_reader :redis
42
41
  end
43
42
  end
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'sidekiq'
4
- require_relative 'sidekiq5_format'
3
+ require "sidekiq"
4
+ require_relative "sidekiq5_format"
5
5
  require_relative "repository"
6
6
 
7
7
  module RubyEventStore
@@ -9,22 +9,25 @@ module RubyEventStore
9
9
  class SidekiqProducer
10
10
  def call(klass, args)
11
11
  sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
12
- item = {
13
- 'args' => args.map(&:to_h).map {|h| h.transform_keys(&:to_s)},
14
- 'class' => klass,
15
- }
12
+ item = { "args" => args.map(&:to_h).map { |h| h.transform_keys(&:to_s) }, "class" => klass }
16
13
  normalized_item = sidekiq_client.__send__(:normalize_item, item)
17
- payload = sidekiq_client.middleware.invoke(normalized_item['class'], normalized_item, normalized_item['queue'], Sidekiq.redis_pool) { normalized_item }
14
+ payload =
15
+ sidekiq_client
16
+ .middleware
17
+ .invoke(normalized_item["class"], normalized_item, normalized_item["queue"], Sidekiq.redis_pool) do
18
+ normalized_item
19
+ end
18
20
  if payload
19
21
  Repository::Record.create!(
20
22
  format: SIDEKIQ5_FORMAT,
21
- split_key: payload.fetch('queue'),
23
+ split_key: payload.fetch("queue"),
22
24
  payload: payload.to_json
23
25
  )
24
26
  end
25
27
  end
26
28
 
27
29
  private
30
+
28
31
  attr_reader :repository
29
32
  end
30
33
  end
@@ -19,6 +19,7 @@ module RubyEventStore
19
19
  end
20
20
 
21
21
  private
22
+
22
23
  attr_reader :serializer, :sidekiq_producer
23
24
  end
24
25
  end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module RubyEventStore
4
4
  module Outbox
5
- VERSION = "0.0.22"
5
+ VERSION = "0.0.25"
6
6
  end
7
7
  end
@@ -5,8 +5,8 @@ module RubyEventStore
5
5
  end
6
6
  end
7
7
 
8
- require_relative 'outbox/fetch_specification'
9
- require_relative 'outbox/repository'
10
- require_relative 'outbox/sidekiq_scheduler'
11
- require_relative 'outbox/legacy_sidekiq_scheduler'
12
- require_relative 'outbox/version'
8
+ require_relative "outbox/fetch_specification"
9
+ require_relative "outbox/repository"
10
+ require_relative "outbox/sidekiq_scheduler"
11
+ require_relative "outbox/legacy_sidekiq_scheduler"
12
+ require_relative "outbox/version"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ruby_event_store-outbox
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.22
4
+ version: 0.0.25
5
5
  platform: ruby
6
6
  authors:
7
7
  - Arkency
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-04-20 00:00:00.000000000 Z
11
+ date: 2022-05-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ruby_event_store
@@ -30,14 +30,14 @@ dependencies:
30
30
  requirements:
31
31
  - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: '5.2'
33
+ version: '6.0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: '5.2'
40
+ version: '6.0'
41
41
  description:
42
42
  email: dev@arkency.com
43
43
  executables:
@@ -54,6 +54,7 @@ files:
54
54
  - lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb
55
55
  - lib/ruby_event_store/outbox/cleanup_strategies/none.rb
56
56
  - lib/ruby_event_store/outbox/cli.rb
57
+ - lib/ruby_event_store/outbox/cli.rb.orig
57
58
  - lib/ruby_event_store/outbox/consumer.rb
58
59
  - lib/ruby_event_store/outbox/fetch_specification.rb
59
60
  - lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb