ruby_event_store-outbox 0.0.25 → 0.0.26

Sign up to get free protection for your applications and to get access to all the features.
Files changed (26) hide show
  1. checksums.yaml +4 -4
  2. data/lib/generators/ruby_event_store/outbox/migration_generator.rb +2 -2
  3. data/lib/generators/ruby_event_store/outbox/templates/{create_event_store_outbox_template.rb → create_event_store_outbox_template.erb} +4 -4
  4. data/lib/ruby_event_store/outbox/batch_result.rb +26 -0
  5. data/lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb +2 -0
  6. data/lib/ruby_event_store/outbox/cleanup_strategies/none.rb +2 -0
  7. data/lib/ruby_event_store/outbox/cleanup_strategies.rb +20 -0
  8. data/lib/ruby_event_store/outbox/cli.rb +11 -6
  9. data/lib/ruby_event_store/outbox/configuration.rb +50 -0
  10. data/lib/ruby_event_store/outbox/consumer.rb +37 -103
  11. data/lib/ruby_event_store/outbox/fetch_specification.rb +3 -3
  12. data/lib/ruby_event_store/outbox/metrics/influx.rb +2 -0
  13. data/lib/ruby_event_store/outbox/metrics/null.rb +2 -0
  14. data/lib/ruby_event_store/outbox/metrics/test.rb +2 -0
  15. data/lib/ruby_event_store/outbox/metrics.rb +2 -0
  16. data/lib/ruby_event_store/outbox/repository.rb +12 -12
  17. data/lib/ruby_event_store/outbox/runner.rb +43 -0
  18. data/lib/ruby_event_store/outbox/sidekiq_processor.rb +11 -5
  19. data/lib/ruby_event_store/outbox/sidekiq_producer.rb +9 -1
  20. data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb +1 -1
  21. data/lib/ruby_event_store/outbox/tempo.rb +25 -0
  22. data/lib/ruby_event_store/outbox/version.rb +1 -1
  23. data/lib/ruby_event_store/outbox.rb +5 -1
  24. metadata +14 -11
  25. data/lib/ruby_event_store/outbox/cli.rb.orig +0 -132
  26. data/lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb +0 -25
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d9f4e4b0d475d2ad92997aee74165ffacbabbe19282b5d1b266b5ec7b7d3b59b
4
- data.tar.gz: adccd92a63810fb4f285c5ea5e4ad37cc22ad671d4550b1d73b289f179d5831b
3
+ metadata.gz: e26efaae4895206e93089d60869e01dd1a0f8182e6f41e49edef000a2833b5bb
4
+ data.tar.gz: 1ab71712967fa31437d538c787721f6bd27f0a65d1b0474e95cd1428945d9b62
5
5
  SHA512:
6
- metadata.gz: 4a858aff2550a901615bad83ffa58dc6e30f669a8aa8a422d047343b74a32317d7476e27e95127f9750640eab2e538385b3284840f4c0a6532eff6ee8a20e96e
7
- data.tar.gz: b81515fba19988345a3f45397de2d674f94e331475140246ccb5a7266cb19f49f6f2d59445b99fe139b6770e59310a53090f7e9eec2e6503767cf57e5801abb8
6
+ metadata.gz: 3189203160620f90690f35b511d2e60471c6652e13ef510464952aa40ba2b35c650d66477f1ab5e3360e6bf31ab6a1fdf40aadd8916ff9aa802dbe069034a013
7
+ data.tar.gz: 765725def129ea5368be1fd5fcb2ae013e45efe17ed48b9d58eb4d1c1356582fb76fcafd9885d9fc527556a6351993709250c434b8ee586708f9b8514d617e2f
@@ -12,13 +12,13 @@ if defined?(Rails::Generators::Base)
12
12
  source_root File.expand_path(File.join(File.dirname(__FILE__), "./templates"))
13
13
 
14
14
  def create_migration
15
- template "create_event_store_outbox_template.rb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
15
+ template "create_event_store_outbox_template.erb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
16
16
  end
17
17
 
18
18
  private
19
19
 
20
20
  def migration_version
21
- "[4.2]"
21
+ ::ActiveRecord::Migration.current_version
22
22
  end
23
23
 
24
24
  def timestamp
@@ -1,13 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- class CreateEventStoreOutbox < ActiveRecord::Migration<%= migration_version %>
3
+ class CreateEventStoreOutbox < ActiveRecord::Migration[<%= migration_version %>]
4
4
  def change
5
5
  create_table(:event_store_outbox, force: false) do |t|
6
6
  t.string :split_key, null: true
7
7
  t.string :format, null: false
8
8
  t.binary :payload, null: false
9
- t.datetime :created_at, null: false
10
- t.datetime :enqueued_at, null: true
9
+ t.datetime :created_at, null: false, precision: 6
10
+ t.datetime :enqueued_at, null: true, precision: 6
11
11
  end
12
12
  add_index :event_store_outbox, [:format, :enqueued_at, :split_key], name: "index_event_store_outbox_for_pool"
13
13
  add_index :event_store_outbox, [:created_at, :enqueued_at], name: "index_event_store_outbox_for_clear"
@@ -15,7 +15,7 @@ class CreateEventStoreOutbox < ActiveRecord::Migration<%= migration_version %>
15
15
  create_table(:event_store_outbox_locks, force: false) do |t|
16
16
  t.string :format, null: false
17
17
  t.string :split_key, null: false
18
- t.datetime :locked_at, null: true
18
+ t.datetime :locked_at, null: true, precision: 6
19
19
  t.string :locked_by, null: true, limit: 36
20
20
  end
21
21
  add_index :event_store_outbox_locks, [:format, :split_key], name: "index_event_store_outbox_locks_for_locking", unique: true
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ class BatchResult
6
+ def self.empty
7
+ new
8
+ end
9
+
10
+ def initialize
11
+ @success_count = 0
12
+ @failed_count = 0
13
+ end
14
+
15
+ attr_reader :success_count, :failed_count
16
+
17
+ def count_success!
18
+ @success_count += 1
19
+ end
20
+
21
+ def count_failed!
22
+ @failed_count += 1
23
+ end
24
+ end
25
+ end
26
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  module CleanupStrategies
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  module CleanupStrategies
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ module CleanupStrategies
6
+ def self.build(configuration, repository)
7
+ case configuration.cleanup
8
+ when :none
9
+ None.new
10
+ else
11
+ CleanOldEnqueued.new(
12
+ repository,
13
+ ActiveSupport::Duration.parse(configuration.cleanup),
14
+ configuration.cleanup_limit
15
+ )
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -1,7 +1,11 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "optparse"
2
4
  require_relative "version"
3
5
  require_relative "consumer"
6
+ require_relative "runner"
4
7
  require_relative "metrics"
8
+ require_relative "configuration"
5
9
 
6
10
  module RubyEventStore
7
11
  module Outbox
@@ -92,15 +96,15 @@ module RubyEventStore
92
96
 
93
97
  def run(argv)
94
98
  options = Parser.parse(argv)
95
- outbox_consumer = build_consumer(options)
96
- outbox_consumer.init
97
- outbox_consumer.run
99
+ build_runner(options)
100
+ .init
101
+ .run
98
102
  end
99
103
 
100
- def build_consumer(options)
104
+ def build_runner(options)
101
105
  consumer_uuid = SecureRandom.uuid
102
106
  logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox #{consumer_uuid}")
103
- consumer_configuration = Consumer::Configuration.new(
107
+ consumer_configuration = Configuration.new(
104
108
  split_keys: options.split_keys,
105
109
  message_format: options.message_format,
106
110
  batch_size: options.batch_size,
@@ -112,7 +116,8 @@ module RubyEventStore
112
116
  )
113
117
  metrics = Metrics.from_url(options.metrics_url)
114
118
  outbox_consumer =
115
- RubyEventStore::Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
119
+ Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
120
+ Runner.new(outbox_consumer, consumer_configuration, logger: logger)
116
121
  end
117
122
  end
118
123
  end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ class Configuration
6
+ def initialize(
7
+ split_keys:,
8
+ message_format:,
9
+ batch_size:,
10
+ database_url:,
11
+ redis_url:,
12
+ cleanup:,
13
+ cleanup_limit:,
14
+ sleep_on_empty:
15
+ )
16
+ @split_keys = split_keys
17
+ @message_format = message_format
18
+ @batch_size = batch_size || 100
19
+ @database_url = database_url
20
+ @redis_url = redis_url
21
+ @cleanup = cleanup
22
+ @cleanup_limit = cleanup_limit
23
+ @sleep_on_empty = sleep_on_empty
24
+ freeze
25
+ end
26
+
27
+ def with(overriden_options)
28
+ self.class.new(
29
+ split_keys: overriden_options.fetch(:split_keys, split_keys),
30
+ message_format: overriden_options.fetch(:message_format, message_format),
31
+ batch_size: overriden_options.fetch(:batch_size, batch_size),
32
+ database_url: overriden_options.fetch(:database_url, database_url),
33
+ redis_url: overriden_options.fetch(:redis_url, redis_url),
34
+ cleanup: overriden_options.fetch(:cleanup, cleanup),
35
+ cleanup_limit: overriden_options.fetch(:cleanup_limit, cleanup_limit),
36
+ sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
37
+ )
38
+ end
39
+
40
+ attr_reader :split_keys,
41
+ :message_format,
42
+ :batch_size,
43
+ :database_url,
44
+ :redis_url,
45
+ :cleanup,
46
+ :cleanup_limit,
47
+ :sleep_on_empty
48
+ end
49
+ end
50
+ end
@@ -1,5 +1,7 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "logger"
2
- require "redis"
4
+ require "redis-client"
3
5
  require "active_record"
4
6
  require_relative "repository"
5
7
  require_relative "sidekiq5_format"
@@ -13,98 +15,24 @@ module RubyEventStore
13
15
  class Consumer
14
16
  MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK = 10
15
17
 
16
- class Configuration
17
- def initialize(
18
- split_keys:,
19
- message_format:,
20
- batch_size:,
21
- database_url:,
22
- redis_url:,
23
- cleanup:,
24
- cleanup_limit:,
25
- sleep_on_empty:
26
- )
27
- @split_keys = split_keys
28
- @message_format = message_format
29
- @batch_size = batch_size || 100
30
- @database_url = database_url
31
- @redis_url = redis_url
32
- @cleanup = cleanup
33
- @cleanup_limit = cleanup_limit
34
- @sleep_on_empty = sleep_on_empty
35
- freeze
36
- end
37
-
38
- def with(overriden_options)
39
- self.class.new(
40
- split_keys: overriden_options.fetch(:split_keys, split_keys),
41
- message_format: overriden_options.fetch(:message_format, message_format),
42
- batch_size: overriden_options.fetch(:batch_size, batch_size),
43
- database_url: overriden_options.fetch(:database_url, database_url),
44
- redis_url: overriden_options.fetch(:redis_url, redis_url),
45
- cleanup: overriden_options.fetch(:cleanup, cleanup),
46
- cleanup_limit: overriden_options.fetch(:cleanup_limit, cleanup_limit),
47
- sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
48
- )
49
- end
50
-
51
- attr_reader :split_keys,
52
- :message_format,
53
- :batch_size,
54
- :database_url,
55
- :redis_url,
56
- :cleanup,
57
- :cleanup_limit,
58
- :sleep_on_empty
59
- end
60
-
61
18
  def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
62
19
  @split_keys = configuration.split_keys
63
20
  @clock = clock
64
21
  @logger = logger
65
22
  @metrics = metrics
66
- @batch_size = configuration.batch_size
67
- @sleep_on_empty = configuration.sleep_on_empty
23
+ @tempo = Tempo.new(configuration.batch_size)
68
24
  @consumer_uuid = consumer_uuid
69
25
 
70
26
  raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
71
- @processor = SidekiqProcessor.new(Redis.new(url: configuration.redis_url))
72
-
73
- @gracefully_shutting_down = false
74
- prepare_traps
27
+ redis_config = RedisClient.config(url: configuration.redis_url)
28
+ @processor = SidekiqProcessor.new(redis_config.new_client)
75
29
 
76
30
  @repository = Repository.new(configuration.database_url)
77
- @cleanup_strategy =
78
- case configuration.cleanup
79
- when :none
80
- CleanupStrategies::None.new
81
- else
82
- CleanupStrategies::CleanOldEnqueued.new(
83
- repository,
84
- ActiveSupport::Duration.parse(configuration.cleanup),
85
- configuration.cleanup_limit
86
- )
87
- end
31
+ @cleanup_strategy = CleanupStrategies.build(configuration, repository)
88
32
  end
89
33
 
90
- def init
91
- logger.info("Initiated RubyEventStore::Outbox v#{VERSION}")
92
- logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
93
- end
94
-
95
- def run
96
- while !@gracefully_shutting_down
97
- was_something_changed = one_loop
98
- if !was_something_changed
99
- STDOUT.flush
100
- sleep sleep_on_empty
101
- end
102
- end
103
- logger.info "Gracefully shutting down"
104
- end
105
-
106
- def one_loop
107
- remaining_split_keys = @split_keys.dup
34
+ def process
35
+ remaining_split_keys = split_keys.dup
108
36
 
109
37
  was_something_changed = false
110
38
  while (split_key = remaining_split_keys.shift)
@@ -123,31 +51,27 @@ module RubyEventStore
123
51
  batch = retrieve_batch(fetch_specification)
124
52
  break if batch.empty?
125
53
 
126
- failed_record_ids = []
127
- updated_record_ids = []
54
+ batch_result = BatchResult.empty
128
55
  batch.each do |record|
129
- begin
56
+ handle_failure(batch_result) do
130
57
  now = @clock.now.utc
131
58
  processor.process(record, now)
132
59
 
133
60
  repository.mark_as_enqueued(record, now)
134
61
  something_processed |= true
135
- updated_record_ids << record.id
136
- rescue => e
137
- failed_record_ids << record.id
138
- e.full_message.split($/).each { |line| logger.error(line) }
62
+ batch_result.count_success!
139
63
  end
140
64
  end
141
65
 
142
66
  metrics.write_point_queue(
143
- enqueued: updated_record_ids.size,
144
- failed: failed_record_ids.size,
67
+ enqueued: batch_result.success_count,
68
+ failed: batch_result.failed_count,
145
69
  format: fetch_specification.message_format,
146
70
  split_key: fetch_specification.split_key,
147
71
  remaining: get_remaining_count(fetch_specification)
148
72
  )
149
73
 
150
- logger.info "Sent #{updated_record_ids.size} messages from outbox table"
74
+ logger.info "Sent #{batch_result.success_count} messages from outbox table"
151
75
 
152
76
  refresh_successful = refresh_lock_for_process(obtained_lock)
153
77
  break unless refresh_successful
@@ -174,13 +98,32 @@ module RubyEventStore
174
98
 
175
99
  attr_reader :split_keys,
176
100
  :logger,
177
- :batch_size,
178
101
  :metrics,
179
102
  :processor,
180
103
  :consumer_uuid,
181
104
  :repository,
182
105
  :cleanup_strategy,
183
- :sleep_on_empty
106
+ :tempo
107
+
108
+ def handle_failure(batch_result)
109
+ retried = false
110
+ yield
111
+ rescue RetriableError => error
112
+ if retried
113
+ batch_result.count_failed!
114
+ log_error(error)
115
+ else
116
+ retried = true
117
+ retry
118
+ end
119
+ rescue => error
120
+ batch_result.count_failed!
121
+ log_error(error)
122
+ end
123
+
124
+ def log_error(e)
125
+ e.full_message.split($/).each { |line| logger.error(line) }
126
+ end
184
127
 
185
128
  def obtain_lock_for_process(fetch_specification)
186
129
  result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
@@ -251,17 +194,8 @@ module RubyEventStore
251
194
  end
252
195
  end
253
196
 
254
- def prepare_traps
255
- Signal.trap("INT") { initiate_graceful_shutdown }
256
- Signal.trap("TERM") { initiate_graceful_shutdown }
257
- end
258
-
259
- def initiate_graceful_shutdown
260
- @gracefully_shutting_down = true
261
- end
262
-
263
197
  def retrieve_batch(fetch_specification)
264
- repository.retrieve_batch(fetch_specification, batch_size)
198
+ repository.retrieve_batch(fetch_specification, tempo.batch_size)
265
199
  end
266
200
 
267
201
  def get_remaining_count(fetch_specification)
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  class FetchSpecification
@@ -13,10 +15,8 @@ module RubyEventStore
13
15
  other.instance_of?(self.class) && other.message_format.eql?(message_format) && other.split_key.eql?(split_key)
14
16
  end
15
17
 
16
- BIG_VALUE = 0b111111100100010000010010110010101011011101110101001100100110000
17
-
18
18
  def hash
19
- [self.class, message_format, split_key].hash ^ BIG_VALUE
19
+ [message_format, split_key].hash ^ self.class.hash
20
20
  end
21
21
 
22
22
  alias_method :eql?, :==
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "influxdb"
2
4
 
3
5
  module RubyEventStore
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  module Metrics
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "influxdb"
2
4
 
3
5
  module RubyEventStore
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  module Metrics
@@ -43,9 +43,9 @@ module RubyEventStore
43
43
  l
44
44
  end
45
45
  end
46
- rescue ActiveRecord::Deadlocked
46
+ rescue ::ActiveRecord::Deadlocked
47
47
  :deadlocked
48
- rescue ActiveRecord::LockWaitTimeout
48
+ rescue ::ActiveRecord::LockWaitTimeout
49
49
  :lock_timeout
50
50
  end
51
51
 
@@ -60,9 +60,9 @@ module RubyEventStore
60
60
  :stolen
61
61
  end
62
62
  end
63
- rescue ActiveRecord::Deadlocked
63
+ rescue ::ActiveRecord::Deadlocked
64
64
  :deadlocked
65
- rescue ActiveRecord::LockWaitTimeout
65
+ rescue ::ActiveRecord::LockWaitTimeout
66
66
  :lock_timeout
67
67
  end
68
68
 
@@ -76,9 +76,9 @@ module RubyEventStore
76
76
  :ok
77
77
  end
78
78
  end
79
- rescue ActiveRecord::Deadlocked
79
+ rescue ::ActiveRecord::Deadlocked
80
80
  :deadlocked
81
- rescue ActiveRecord::LockWaitTimeout
81
+ rescue ::ActiveRecord::LockWaitTimeout
82
82
  :lock_timeout
83
83
  end
84
84
 
@@ -105,7 +105,7 @@ module RubyEventStore
105
105
  if l.nil?
106
106
  begin
107
107
  l = create!(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
108
- rescue ActiveRecord::RecordNotUnique
108
+ rescue ::ActiveRecord::RecordNotUnique
109
109
  l = lock_for_split_key(fetch_specification)
110
110
  end
111
111
  end
@@ -114,9 +114,9 @@ module RubyEventStore
114
114
  end
115
115
 
116
116
  def initialize(database_url)
117
- ActiveRecord::Base.establish_connection(database_url) unless ActiveRecord::Base.connected?
118
- if ActiveRecord::Base.connection.adapter_name == "Mysql2"
119
- ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
117
+ ::ActiveRecord::Base.establish_connection(database_url) unless ::ActiveRecord::Base.connected?
118
+ if ::ActiveRecord::Base.connection.adapter_name == "Mysql2"
119
+ ::ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
120
120
  end
121
121
  end
122
122
 
@@ -145,9 +145,9 @@ module RubyEventStore
145
145
  scope = scope.limit(limit).order(:id) unless limit == :all
146
146
  scope.delete_all
147
147
  :ok
148
- rescue ActiveRecord::Deadlocked
148
+ rescue ::ActiveRecord::Deadlocked
149
149
  :deadlocked
150
- rescue ActiveRecord::LockWaitTimeout
150
+ rescue ::ActiveRecord::LockWaitTimeout
151
151
  :lock_timeout
152
152
  end
153
153
  end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ class Runner
6
+ def initialize(consumer, configuration, logger:)
7
+ @consumer = consumer
8
+ @logger = logger
9
+ @sleep_on_empty = configuration.sleep_on_empty
10
+ @split_keys = configuration.split_keys
11
+ @gracefully_shutting_down = false
12
+ prepare_traps
13
+ end
14
+
15
+ def run
16
+ logger.info("Initiated RubyEventStore::Outbox v#{VERSION}")
17
+ logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
18
+
19
+ while !@gracefully_shutting_down
20
+ was_something_changed = consumer.process
21
+ if !was_something_changed
22
+ STDOUT.flush
23
+ sleep sleep_on_empty
24
+ end
25
+ end
26
+
27
+ logger.info "Gracefully shutting down"
28
+ end
29
+
30
+ private
31
+ attr_reader :consumer, :logger, :sleep_on_empty, :split_keys
32
+
33
+ def prepare_traps
34
+ Signal.trap("INT") { initiate_graceful_shutdown }
35
+ Signal.trap("TERM") { initiate_graceful_shutdown }
36
+ end
37
+
38
+ def initiate_graceful_shutdown
39
+ @gracefully_shutting_down = true
40
+ end
41
+ end
42
+ end
43
+ end
@@ -19,16 +19,15 @@ module RubyEventStore
19
19
  raise InvalidPayload.new("Missing queue") if queue.nil? || queue.empty?
20
20
  payload = JSON.generate(parsed_record.merge({ "enqueued_at" => now.to_f }))
21
21
 
22
- redis.lpush("queue:#{queue}", payload)
22
+ redis.call("LPUSH", "queue:#{queue}", payload)
23
23
 
24
24
  @recently_used_queues << queue
25
+ rescue RedisClient::TimeoutError, RedisClient::ConnectionError
26
+ raise RetriableError
25
27
  end
26
28
 
27
29
  def after_batch
28
- if !@recently_used_queues.empty?
29
- redis.sadd("queues", @recently_used_queues.to_a)
30
- @recently_used_queues.clear
31
- end
30
+ ensure_that_sidekiq_knows_about_all_queues
32
31
  end
33
32
 
34
33
  def message_format
@@ -37,6 +36,13 @@ module RubyEventStore
37
36
 
38
37
  private
39
38
 
39
+ def ensure_that_sidekiq_knows_about_all_queues
40
+ if !@recently_used_queues.empty?
41
+ redis.call("SADD", "queues", @recently_used_queues.to_a)
42
+ @recently_used_queues.clear
43
+ end
44
+ end
45
+
40
46
  attr_reader :redis
41
47
  end
42
48
  end
@@ -8,7 +8,6 @@ module RubyEventStore
8
8
  module Outbox
9
9
  class SidekiqProducer
10
10
  def call(klass, args)
11
- sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
12
11
  item = { "args" => args.map(&:to_h).map { |h| h.transform_keys(&:to_s) }, "class" => klass }
13
12
  normalized_item = sidekiq_client.__send__(:normalize_item, item)
14
13
  payload =
@@ -29,6 +28,15 @@ module RubyEventStore
29
28
  private
30
29
 
31
30
  attr_reader :repository
31
+
32
+ def sidekiq_client
33
+ @sidekiq_client ||=
34
+ if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("7.0.0")
35
+ Sidekiq::Client.new(Sidekiq.redis_pool)
36
+ else
37
+ Sidekiq::Client.new(pool: Sidekiq.redis_pool)
38
+ end
39
+ end
32
40
  end
33
41
  end
34
42
  end
@@ -5,7 +5,7 @@ require_relative "sidekiq_producer"
5
5
  module RubyEventStore
6
6
  module Outbox
7
7
  class SidekiqScheduler
8
- def initialize(serializer: RubyEventStore::Serializers::YAML)
8
+ def initialize(serializer: Serializers::YAML)
9
9
  @serializer = serializer
10
10
  @sidekiq_producer = SidekiqProducer.new
11
11
  end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ class Tempo
6
+ EXPONENTIAL_MULTIPLIER = 2
7
+
8
+ def initialize(max_batch_size)
9
+ raise ArgumentError if max_batch_size < 1
10
+ @max_batch_size = max_batch_size
11
+ end
12
+
13
+ def batch_size
14
+ @batch_size = next_batch_size
15
+ end
16
+
17
+ private
18
+
19
+ def next_batch_size
20
+ return 1 if @batch_size.nil?
21
+ [@batch_size * EXPONENTIAL_MULTIPLIER, @max_batch_size].min
22
+ end
23
+ end
24
+ end
25
+ end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module RubyEventStore
4
4
  module Outbox
5
- VERSION = "0.0.25"
5
+ VERSION = "0.0.26"
6
6
  end
7
7
  end
@@ -2,11 +2,15 @@
2
2
 
3
3
  module RubyEventStore
4
4
  module Outbox
5
+ Error = Class.new(StandardError)
6
+ RetriableError = Class.new(Error)
5
7
  end
6
8
  end
7
9
 
8
10
  require_relative "outbox/fetch_specification"
9
11
  require_relative "outbox/repository"
10
12
  require_relative "outbox/sidekiq_scheduler"
11
- require_relative "outbox/legacy_sidekiq_scheduler"
12
13
  require_relative "outbox/version"
14
+ require_relative "outbox/tempo"
15
+ require_relative "outbox/batch_result"
16
+ require_relative "outbox/cleanup_strategies"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ruby_event_store-outbox
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.25
4
+ version: 0.0.26
5
5
  platform: ruby
6
6
  authors:
7
7
  - Arkency
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-05-27 00:00:00.000000000 Z
11
+ date: 2024-04-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ruby_event_store
@@ -38,7 +38,7 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '6.0'
41
- description:
41
+ description:
42
42
  email: dev@arkency.com
43
43
  executables:
44
44
  - res_outbox
@@ -49,24 +49,27 @@ files:
49
49
  - README.md
50
50
  - bin/res_outbox
51
51
  - lib/generators/ruby_event_store/outbox/migration_generator.rb
52
- - lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.rb
52
+ - lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.erb
53
53
  - lib/ruby_event_store/outbox.rb
54
+ - lib/ruby_event_store/outbox/batch_result.rb
55
+ - lib/ruby_event_store/outbox/cleanup_strategies.rb
54
56
  - lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb
55
57
  - lib/ruby_event_store/outbox/cleanup_strategies/none.rb
56
58
  - lib/ruby_event_store/outbox/cli.rb
57
- - lib/ruby_event_store/outbox/cli.rb.orig
59
+ - lib/ruby_event_store/outbox/configuration.rb
58
60
  - lib/ruby_event_store/outbox/consumer.rb
59
61
  - lib/ruby_event_store/outbox/fetch_specification.rb
60
- - lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb
61
62
  - lib/ruby_event_store/outbox/metrics.rb
62
63
  - lib/ruby_event_store/outbox/metrics/influx.rb
63
64
  - lib/ruby_event_store/outbox/metrics/null.rb
64
65
  - lib/ruby_event_store/outbox/metrics/test.rb
65
66
  - lib/ruby_event_store/outbox/repository.rb
67
+ - lib/ruby_event_store/outbox/runner.rb
66
68
  - lib/ruby_event_store/outbox/sidekiq5_format.rb
67
69
  - lib/ruby_event_store/outbox/sidekiq_processor.rb
68
70
  - lib/ruby_event_store/outbox/sidekiq_producer.rb
69
71
  - lib/ruby_event_store/outbox/sidekiq_scheduler.rb
72
+ - lib/ruby_event_store/outbox/tempo.rb
70
73
  - lib/ruby_event_store/outbox/version.rb
71
74
  homepage: https://railseventstore.org
72
75
  licenses:
@@ -76,7 +79,7 @@ metadata:
76
79
  source_code_uri: https://github.com/RailsEventStore/rails_event_store
77
80
  bug_tracker_uri: https://github.com/RailsEventStore/rails_event_store/issues
78
81
  rubygems_mfa_required: 'true'
79
- post_install_message:
82
+ post_install_message:
80
83
  rdoc_options: []
81
84
  require_paths:
82
85
  - lib
@@ -84,15 +87,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
84
87
  requirements:
85
88
  - - ">="
86
89
  - !ruby/object:Gem::Version
87
- version: '2.6'
90
+ version: '2.7'
88
91
  required_rubygems_version: !ruby/object:Gem::Requirement
89
92
  requirements:
90
93
  - - ">="
91
94
  - !ruby/object:Gem::Version
92
95
  version: '0'
93
96
  requirements: []
94
- rubygems_version: 3.3.9
95
- signing_key:
97
+ rubygems_version: 3.4.10
98
+ signing_key:
96
99
  specification_version: 4
97
100
  summary: Active Record based outbox for Ruby Event Store
98
101
  test_files: []
@@ -1,132 +0,0 @@
1
- require "optparse"
2
- require_relative "version"
3
- require_relative "consumer"
4
- require_relative "metrics"
5
-
6
- module RubyEventStore
7
- module Outbox
8
- class CLI
9
- DEFAULTS = {
10
- database_url: nil,
11
- redis_url: nil,
12
- log_level: :warn,
13
- split_keys: nil,
14
- message_format: "sidekiq5",
15
- batch_size: 100,
16
- metrics_url: nil,
17
- cleanup_strategy: :none,
18
- cleanup_limit: :all,
19
- sleep_on_empty: 0.5
20
- }
21
- Options = Struct.new(*DEFAULTS.keys)
22
-
23
- class Parser
24
- def self.parse(argv)
25
- options = Options.new(*DEFAULTS.values)
26
- OptionParser
27
- .new do |option_parser|
28
- option_parser.banner = "Usage: res_outbox [options]"
29
-
30
- option_parser.on(
31
- "--database-url=DATABASE_URL",
32
- "Database where outbox table is stored"
33
- ) { |database_url| options.database_url = database_url }
34
-
35
- option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
36
- options.redis_url = redis_url
37
- end
38
-
39
- option_parser.on(
40
- "--log-level=LOG_LEVEL",
41
- %i[fatal error warn info debug],
42
- "Logging level, one of: fatal, error, warn, info, debug. Default: warn"
43
- ) { |log_level| options.log_level = log_level.to_sym }
44
-
45
- option_parser.on(
46
- "--message-format=FORMAT",
47
- ["sidekiq5"],
48
- "Message format, supported: sidekiq5. Default: sidekiq5"
49
- ) { |message_format| options.message_format = message_format }
50
-
51
- option_parser.on(
52
- "--split-keys=SPLIT_KEYS",
53
- Array,
54
- "Split keys which should be handled, all if not specified"
55
- ) { |split_keys| options.split_keys = split_keys if !split_keys.empty? }
56
-
57
- option_parser.on(
58
- "--batch-size=BATCH_SIZE",
59
- Integer,
60
- "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100"
61
- ) { |batch_size| options.batch_size = batch_size }
62
-
63
- option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
64
- options.metrics_url = metrics_url
65
- end
66
-
67
- option_parser.on(
68
- "--cleanup=STRATEGY",
69
- "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none"
70
- ) { |cleanup_strategy| options.cleanup_strategy = cleanup_strategy }
71
-
72
- option_parser.on(
73
- "--cleanup-limit=LIMIT",
74
- "Amount of records removed in single cleanup run. One of: all or number of records that should be removed. Default: all"
75
- ) { |cleanup_limit| options.cleanup_limit = cleanup_limit }
76
-
77
- option_parser.on(
78
- "--sleep-on-empty=SLEEP_TIME",
79
- Float,
80
- "How long to sleep before next check when there was nothing to do. Default: 0.5"
81
- ) { |sleep_on_empty| options.sleep_on_empty = sleep_on_empty }
82
-
83
- option_parser.on_tail("--version", "Show version") do
84
- puts VERSION
85
- exit
86
- end
87
- end
88
- .parse(argv)
89
- return options
90
- end
91
- end
92
-
93
- def run(argv)
94
- options = Parser.parse(argv)
95
- outbox_consumer = build_consumer(options)
96
- outbox_consumer.init
97
- outbox_consumer.run
98
- end
99
-
100
- def build_consumer(options)
101
- consumer_uuid = SecureRandom.uuid
102
- logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox #{consumer_uuid}")
103
- <<<<<<< HEAD
104
- consumer_configuration =
105
- Consumer::Configuration.new(
106
- split_keys: options.split_keys,
107
- message_format: options.message_format,
108
- batch_size: options.batch_size,
109
- database_url: options.database_url,
110
- redis_url: options.redis_url,
111
- cleanup: options.cleanup_strategy,
112
- sleep_on_empty: options.sleep_on_empty
113
- )
114
- =======
115
- consumer_configuration = Consumer::Configuration.new(
116
- split_keys: options.split_keys,
117
- message_format: options.message_format,
118
- batch_size: options.batch_size,
119
- database_url: options.database_url,
120
- redis_url: options.redis_url,
121
- cleanup: options.cleanup_strategy,
122
- cleanup_limit: options.cleanup_limit,
123
- sleep_on_empty: options.sleep_on_empty,
124
- )
125
- >>>>>>> 7f077fa2... fix error with passing `--cleanup-limit` from CLI to consumer
126
- metrics = Metrics.from_url(options.metrics_url)
127
- outbox_consumer =
128
- RubyEventStore::Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
129
- end
130
- end
131
- end
132
- end
@@ -1,25 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative "sidekiq_producer"
4
-
5
- module RubyEventStore
6
- module Outbox
7
- class LegacySidekiqScheduler
8
- def initialize
9
- @sidekiq_producer = SidekiqProducer.new
10
- end
11
-
12
- def call(klass, serialized_record)
13
- sidekiq_producer.call(klass, [serialized_record])
14
- end
15
-
16
- def verify(subscriber)
17
- Class === subscriber && subscriber.respond_to?(:through_outbox?) && subscriber.through_outbox?
18
- end
19
-
20
- private
21
-
22
- attr_reader :sidekiq_producer
23
- end
24
- end
25
- end