ruby_event_store-outbox 0.0.25 → 0.0.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. checksums.yaml +4 -4
  2. data/lib/generators/ruby_event_store/outbox/migration_generator.rb +2 -2
  3. data/lib/generators/ruby_event_store/outbox/templates/{create_event_store_outbox_template.rb → create_event_store_outbox_template.erb} +4 -4
  4. data/lib/ruby_event_store/outbox/batch_result.rb +26 -0
  5. data/lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb +2 -0
  6. data/lib/ruby_event_store/outbox/cleanup_strategies/none.rb +2 -0
  7. data/lib/ruby_event_store/outbox/cleanup_strategies.rb +23 -0
  8. data/lib/ruby_event_store/outbox/cli.rb +10 -6
  9. data/lib/ruby_event_store/outbox/configuration.rb +50 -0
  10. data/lib/ruby_event_store/outbox/consumer.rb +39 -105
  11. data/lib/ruby_event_store/outbox/fetch_specification.rb +3 -3
  12. data/lib/ruby_event_store/outbox/metrics/influx.rb +2 -0
  13. data/lib/ruby_event_store/outbox/metrics/null.rb +2 -0
  14. data/lib/ruby_event_store/outbox/metrics/test.rb +2 -0
  15. data/lib/ruby_event_store/outbox/metrics.rb +2 -0
  16. data/lib/ruby_event_store/outbox/repository.rb +12 -12
  17. data/lib/ruby_event_store/outbox/runner.rb +43 -0
  18. data/lib/ruby_event_store/outbox/sidekiq_processor.rb +11 -5
  19. data/lib/ruby_event_store/outbox/sidekiq_producer.rb +9 -1
  20. data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb +1 -1
  21. data/lib/ruby_event_store/outbox/tempo.rb +25 -0
  22. data/lib/ruby_event_store/outbox/version.rb +1 -1
  23. data/lib/ruby_event_store/outbox.rb +5 -1
  24. metadata +14 -11
  25. data/lib/ruby_event_store/outbox/cli.rb.orig +0 -132
  26. data/lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb +0 -25
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d9f4e4b0d475d2ad92997aee74165ffacbabbe19282b5d1b266b5ec7b7d3b59b
4
- data.tar.gz: adccd92a63810fb4f285c5ea5e4ad37cc22ad671d4550b1d73b289f179d5831b
3
+ metadata.gz: a230a2cd8a85d32c26cb83c6dc879e7bee7f6a98c33c46f6376277a821805b0e
4
+ data.tar.gz: 184638665971b91e31e83ab937290e789639063d2e00942ed28a92907ca92545
5
5
  SHA512:
6
- metadata.gz: 4a858aff2550a901615bad83ffa58dc6e30f669a8aa8a422d047343b74a32317d7476e27e95127f9750640eab2e538385b3284840f4c0a6532eff6ee8a20e96e
7
- data.tar.gz: b81515fba19988345a3f45397de2d674f94e331475140246ccb5a7266cb19f49f6f2d59445b99fe139b6770e59310a53090f7e9eec2e6503767cf57e5801abb8
6
+ metadata.gz: a49ccb2ed6e8c47683baa4a61bf88614adcbf17ede978409e7cb4c4ebe5294008a5c8ec1e20e555aa76108c5ed120f48853988ea1c510f8085f5a2dbfe4fc8b2
7
+ data.tar.gz: 62c952a369da38606b402c19cef8d7946ce9c9f8c62073615c111dbe68a7be598104c92ef07491a6841572cc85da7e3063cba7edeec73ae8a0d9946cf17ffaef
@@ -12,13 +12,13 @@ if defined?(Rails::Generators::Base)
12
12
  source_root File.expand_path(File.join(File.dirname(__FILE__), "./templates"))
13
13
 
14
14
  def create_migration
15
- template "create_event_store_outbox_template.rb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
15
+ template "create_event_store_outbox_template.erb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
16
16
  end
17
17
 
18
18
  private
19
19
 
20
20
  def migration_version
21
- "[4.2]"
21
+ ::ActiveRecord::Migration.current_version
22
22
  end
23
23
 
24
24
  def timestamp
@@ -1,13 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- class CreateEventStoreOutbox < ActiveRecord::Migration<%= migration_version %>
3
+ class CreateEventStoreOutbox < ActiveRecord::Migration[<%= migration_version %>]
4
4
  def change
5
5
  create_table(:event_store_outbox, force: false) do |t|
6
6
  t.string :split_key, null: true
7
7
  t.string :format, null: false
8
8
  t.binary :payload, null: false
9
- t.datetime :created_at, null: false
10
- t.datetime :enqueued_at, null: true
9
+ t.datetime :created_at, null: false, precision: 6
10
+ t.datetime :enqueued_at, null: true, precision: 6
11
11
  end
12
12
  add_index :event_store_outbox, [:format, :enqueued_at, :split_key], name: "index_event_store_outbox_for_pool"
13
13
  add_index :event_store_outbox, [:created_at, :enqueued_at], name: "index_event_store_outbox_for_clear"
@@ -15,7 +15,7 @@ class CreateEventStoreOutbox < ActiveRecord::Migration<%= migration_version %>
15
15
  create_table(:event_store_outbox_locks, force: false) do |t|
16
16
  t.string :format, null: false
17
17
  t.string :split_key, null: false
18
- t.datetime :locked_at, null: true
18
+ t.datetime :locked_at, null: true, precision: 6
19
19
  t.string :locked_by, null: true, limit: 36
20
20
  end
21
21
  add_index :event_store_outbox_locks, [:format, :split_key], name: "index_event_store_outbox_locks_for_locking", unique: true
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ class BatchResult
6
+ def self.empty
7
+ new
8
+ end
9
+
10
+ def initialize
11
+ @success_count = 0
12
+ @failed_count = 0
13
+ end
14
+
15
+ attr_reader :success_count, :failed_count
16
+
17
+ def count_success!
18
+ @success_count += 1
19
+ end
20
+
21
+ def count_failed!
22
+ @failed_count += 1
23
+ end
24
+ end
25
+ end
26
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  module CleanupStrategies
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  module CleanupStrategies
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "cleanup_strategies/none"
4
+ require_relative "cleanup_strategies/clean_old_enqueued"
5
+
6
+ module RubyEventStore
7
+ module Outbox
8
+ module CleanupStrategies
9
+ def self.build(configuration, repository)
10
+ case configuration.cleanup
11
+ when :none
12
+ None.new
13
+ else
14
+ CleanOldEnqueued.new(
15
+ repository,
16
+ ActiveSupport::Duration.parse(configuration.cleanup),
17
+ configuration.cleanup_limit
18
+ )
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -1,7 +1,11 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "optparse"
2
4
  require_relative "version"
3
5
  require_relative "consumer"
6
+ require_relative "runner"
4
7
  require_relative "metrics"
8
+ require_relative "configuration"
5
9
 
6
10
  module RubyEventStore
7
11
  module Outbox
@@ -92,15 +96,14 @@ module RubyEventStore
92
96
 
93
97
  def run(argv)
94
98
  options = Parser.parse(argv)
95
- outbox_consumer = build_consumer(options)
96
- outbox_consumer.init
97
- outbox_consumer.run
99
+ build_runner(options)
100
+ .run
98
101
  end
99
102
 
100
- def build_consumer(options)
103
+ def build_runner(options)
101
104
  consumer_uuid = SecureRandom.uuid
102
105
  logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox #{consumer_uuid}")
103
- consumer_configuration = Consumer::Configuration.new(
106
+ consumer_configuration = Configuration.new(
104
107
  split_keys: options.split_keys,
105
108
  message_format: options.message_format,
106
109
  batch_size: options.batch_size,
@@ -112,7 +115,8 @@ module RubyEventStore
112
115
  )
113
116
  metrics = Metrics.from_url(options.metrics_url)
114
117
  outbox_consumer =
115
- RubyEventStore::Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
118
+ Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
119
+ Runner.new(outbox_consumer, consumer_configuration, logger: logger)
116
120
  end
117
121
  end
118
122
  end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ class Configuration
6
+ def initialize(
7
+ split_keys:,
8
+ message_format:,
9
+ batch_size:,
10
+ database_url:,
11
+ redis_url:,
12
+ cleanup:,
13
+ cleanup_limit:,
14
+ sleep_on_empty:
15
+ )
16
+ @split_keys = split_keys
17
+ @message_format = message_format
18
+ @batch_size = batch_size || 100
19
+ @database_url = database_url
20
+ @redis_url = redis_url
21
+ @cleanup = cleanup
22
+ @cleanup_limit = cleanup_limit
23
+ @sleep_on_empty = sleep_on_empty
24
+ freeze
25
+ end
26
+
27
+ def with(overriden_options)
28
+ self.class.new(
29
+ split_keys: overriden_options.fetch(:split_keys, split_keys),
30
+ message_format: overriden_options.fetch(:message_format, message_format),
31
+ batch_size: overriden_options.fetch(:batch_size, batch_size),
32
+ database_url: overriden_options.fetch(:database_url, database_url),
33
+ redis_url: overriden_options.fetch(:redis_url, redis_url),
34
+ cleanup: overriden_options.fetch(:cleanup, cleanup),
35
+ cleanup_limit: overriden_options.fetch(:cleanup_limit, cleanup_limit),
36
+ sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
37
+ )
38
+ end
39
+
40
+ attr_reader :split_keys,
41
+ :message_format,
42
+ :batch_size,
43
+ :database_url,
44
+ :redis_url,
45
+ :cleanup,
46
+ :cleanup_limit,
47
+ :sleep_on_empty
48
+ end
49
+ end
50
+ end
@@ -1,110 +1,38 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "logger"
2
- require "redis"
4
+ require "redis-client"
3
5
  require "active_record"
4
6
  require_relative "repository"
5
7
  require_relative "sidekiq5_format"
8
+ require_relative "tempo"
6
9
  require_relative "sidekiq_processor"
7
10
  require_relative "fetch_specification"
8
- require_relative "cleanup_strategies/none"
9
- require_relative "cleanup_strategies/clean_old_enqueued"
11
+ require_relative "cleanup_strategies"
10
12
 
11
13
  module RubyEventStore
12
14
  module Outbox
13
15
  class Consumer
14
16
  MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK = 10
15
17
 
16
- class Configuration
17
- def initialize(
18
- split_keys:,
19
- message_format:,
20
- batch_size:,
21
- database_url:,
22
- redis_url:,
23
- cleanup:,
24
- cleanup_limit:,
25
- sleep_on_empty:
26
- )
27
- @split_keys = split_keys
28
- @message_format = message_format
29
- @batch_size = batch_size || 100
30
- @database_url = database_url
31
- @redis_url = redis_url
32
- @cleanup = cleanup
33
- @cleanup_limit = cleanup_limit
34
- @sleep_on_empty = sleep_on_empty
35
- freeze
36
- end
37
-
38
- def with(overriden_options)
39
- self.class.new(
40
- split_keys: overriden_options.fetch(:split_keys, split_keys),
41
- message_format: overriden_options.fetch(:message_format, message_format),
42
- batch_size: overriden_options.fetch(:batch_size, batch_size),
43
- database_url: overriden_options.fetch(:database_url, database_url),
44
- redis_url: overriden_options.fetch(:redis_url, redis_url),
45
- cleanup: overriden_options.fetch(:cleanup, cleanup),
46
- cleanup_limit: overriden_options.fetch(:cleanup_limit, cleanup_limit),
47
- sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
48
- )
49
- end
50
-
51
- attr_reader :split_keys,
52
- :message_format,
53
- :batch_size,
54
- :database_url,
55
- :redis_url,
56
- :cleanup,
57
- :cleanup_limit,
58
- :sleep_on_empty
59
- end
60
-
61
18
  def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
62
19
  @split_keys = configuration.split_keys
63
20
  @clock = clock
64
21
  @logger = logger
65
22
  @metrics = metrics
66
- @batch_size = configuration.batch_size
67
- @sleep_on_empty = configuration.sleep_on_empty
23
+ @tempo = Tempo.new(configuration.batch_size)
68
24
  @consumer_uuid = consumer_uuid
69
25
 
70
26
  raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
71
- @processor = SidekiqProcessor.new(Redis.new(url: configuration.redis_url))
72
-
73
- @gracefully_shutting_down = false
74
- prepare_traps
27
+ redis_config = RedisClient.config(url: configuration.redis_url)
28
+ @processor = SidekiqProcessor.new(redis_config.new_client)
75
29
 
76
30
  @repository = Repository.new(configuration.database_url)
77
- @cleanup_strategy =
78
- case configuration.cleanup
79
- when :none
80
- CleanupStrategies::None.new
81
- else
82
- CleanupStrategies::CleanOldEnqueued.new(
83
- repository,
84
- ActiveSupport::Duration.parse(configuration.cleanup),
85
- configuration.cleanup_limit
86
- )
87
- end
31
+ @cleanup_strategy = CleanupStrategies.build(configuration, repository)
88
32
  end
89
33
 
90
- def init
91
- logger.info("Initiated RubyEventStore::Outbox v#{VERSION}")
92
- logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
93
- end
94
-
95
- def run
96
- while !@gracefully_shutting_down
97
- was_something_changed = one_loop
98
- if !was_something_changed
99
- STDOUT.flush
100
- sleep sleep_on_empty
101
- end
102
- end
103
- logger.info "Gracefully shutting down"
104
- end
105
-
106
- def one_loop
107
- remaining_split_keys = @split_keys.dup
34
+ def process
35
+ remaining_split_keys = split_keys.dup
108
36
 
109
37
  was_something_changed = false
110
38
  while (split_key = remaining_split_keys.shift)
@@ -123,31 +51,27 @@ module RubyEventStore
123
51
  batch = retrieve_batch(fetch_specification)
124
52
  break if batch.empty?
125
53
 
126
- failed_record_ids = []
127
- updated_record_ids = []
54
+ batch_result = BatchResult.empty
128
55
  batch.each do |record|
129
- begin
56
+ handle_failure(batch_result) do
130
57
  now = @clock.now.utc
131
58
  processor.process(record, now)
132
59
 
133
60
  repository.mark_as_enqueued(record, now)
134
61
  something_processed |= true
135
- updated_record_ids << record.id
136
- rescue => e
137
- failed_record_ids << record.id
138
- e.full_message.split($/).each { |line| logger.error(line) }
62
+ batch_result.count_success!
139
63
  end
140
64
  end
141
65
 
142
66
  metrics.write_point_queue(
143
- enqueued: updated_record_ids.size,
144
- failed: failed_record_ids.size,
67
+ enqueued: batch_result.success_count,
68
+ failed: batch_result.failed_count,
145
69
  format: fetch_specification.message_format,
146
70
  split_key: fetch_specification.split_key,
147
71
  remaining: get_remaining_count(fetch_specification)
148
72
  )
149
73
 
150
- logger.info "Sent #{updated_record_ids.size} messages from outbox table"
74
+ logger.info "Sent #{batch_result.success_count} messages from outbox table"
151
75
 
152
76
  refresh_successful = refresh_lock_for_process(obtained_lock)
153
77
  break unless refresh_successful
@@ -174,13 +98,32 @@ module RubyEventStore
174
98
 
175
99
  attr_reader :split_keys,
176
100
  :logger,
177
- :batch_size,
178
101
  :metrics,
179
102
  :processor,
180
103
  :consumer_uuid,
181
104
  :repository,
182
105
  :cleanup_strategy,
183
- :sleep_on_empty
106
+ :tempo
107
+
108
+ def handle_failure(batch_result)
109
+ retried = false
110
+ yield
111
+ rescue RetriableError => error
112
+ if retried
113
+ batch_result.count_failed!
114
+ log_error(error)
115
+ else
116
+ retried = true
117
+ retry
118
+ end
119
+ rescue => error
120
+ batch_result.count_failed!
121
+ log_error(error)
122
+ end
123
+
124
+ def log_error(e)
125
+ e.full_message.split($/).each { |line| logger.error(line) }
126
+ end
184
127
 
185
128
  def obtain_lock_for_process(fetch_specification)
186
129
  result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
@@ -251,17 +194,8 @@ module RubyEventStore
251
194
  end
252
195
  end
253
196
 
254
- def prepare_traps
255
- Signal.trap("INT") { initiate_graceful_shutdown }
256
- Signal.trap("TERM") { initiate_graceful_shutdown }
257
- end
258
-
259
- def initiate_graceful_shutdown
260
- @gracefully_shutting_down = true
261
- end
262
-
263
197
  def retrieve_batch(fetch_specification)
264
- repository.retrieve_batch(fetch_specification, batch_size)
198
+ repository.retrieve_batch(fetch_specification, tempo.batch_size)
265
199
  end
266
200
 
267
201
  def get_remaining_count(fetch_specification)
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  class FetchSpecification
@@ -13,10 +15,8 @@ module RubyEventStore
13
15
  other.instance_of?(self.class) && other.message_format.eql?(message_format) && other.split_key.eql?(split_key)
14
16
  end
15
17
 
16
- BIG_VALUE = 0b111111100100010000010010110010101011011101110101001100100110000
17
-
18
18
  def hash
19
- [self.class, message_format, split_key].hash ^ BIG_VALUE
19
+ [message_format, split_key].hash ^ self.class.hash
20
20
  end
21
21
 
22
22
  alias_method :eql?, :==
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "influxdb"
2
4
 
3
5
  module RubyEventStore
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  module Metrics
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "influxdb"
2
4
 
3
5
  module RubyEventStore
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  module Metrics
@@ -43,9 +43,9 @@ module RubyEventStore
43
43
  l
44
44
  end
45
45
  end
46
- rescue ActiveRecord::Deadlocked
46
+ rescue ::ActiveRecord::Deadlocked
47
47
  :deadlocked
48
- rescue ActiveRecord::LockWaitTimeout
48
+ rescue ::ActiveRecord::LockWaitTimeout
49
49
  :lock_timeout
50
50
  end
51
51
 
@@ -60,9 +60,9 @@ module RubyEventStore
60
60
  :stolen
61
61
  end
62
62
  end
63
- rescue ActiveRecord::Deadlocked
63
+ rescue ::ActiveRecord::Deadlocked
64
64
  :deadlocked
65
- rescue ActiveRecord::LockWaitTimeout
65
+ rescue ::ActiveRecord::LockWaitTimeout
66
66
  :lock_timeout
67
67
  end
68
68
 
@@ -76,9 +76,9 @@ module RubyEventStore
76
76
  :ok
77
77
  end
78
78
  end
79
- rescue ActiveRecord::Deadlocked
79
+ rescue ::ActiveRecord::Deadlocked
80
80
  :deadlocked
81
- rescue ActiveRecord::LockWaitTimeout
81
+ rescue ::ActiveRecord::LockWaitTimeout
82
82
  :lock_timeout
83
83
  end
84
84
 
@@ -105,7 +105,7 @@ module RubyEventStore
105
105
  if l.nil?
106
106
  begin
107
107
  l = create!(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
108
- rescue ActiveRecord::RecordNotUnique
108
+ rescue ::ActiveRecord::RecordNotUnique
109
109
  l = lock_for_split_key(fetch_specification)
110
110
  end
111
111
  end
@@ -114,9 +114,9 @@ module RubyEventStore
114
114
  end
115
115
 
116
116
  def initialize(database_url)
117
- ActiveRecord::Base.establish_connection(database_url) unless ActiveRecord::Base.connected?
118
- if ActiveRecord::Base.connection.adapter_name == "Mysql2"
119
- ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
117
+ ::ActiveRecord::Base.establish_connection(database_url) unless ::ActiveRecord::Base.connected?
118
+ if ::ActiveRecord::Base.connection.adapter_name == "Mysql2"
119
+ ::ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
120
120
  end
121
121
  end
122
122
 
@@ -145,9 +145,9 @@ module RubyEventStore
145
145
  scope = scope.limit(limit).order(:id) unless limit == :all
146
146
  scope.delete_all
147
147
  :ok
148
- rescue ActiveRecord::Deadlocked
148
+ rescue ::ActiveRecord::Deadlocked
149
149
  :deadlocked
150
- rescue ActiveRecord::LockWaitTimeout
150
+ rescue ::ActiveRecord::LockWaitTimeout
151
151
  :lock_timeout
152
152
  end
153
153
  end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ class Runner
6
+ def initialize(consumer, configuration, logger:)
7
+ @consumer = consumer
8
+ @logger = logger
9
+ @sleep_on_empty = configuration.sleep_on_empty
10
+ @split_keys = configuration.split_keys
11
+ @gracefully_shutting_down = false
12
+ prepare_traps
13
+ end
14
+
15
+ def run
16
+ logger.info("Initiated RubyEventStore::Outbox v#{VERSION}")
17
+ logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
18
+
19
+ while !@gracefully_shutting_down
20
+ was_something_changed = consumer.process
21
+ if !was_something_changed
22
+ STDOUT.flush
23
+ sleep sleep_on_empty
24
+ end
25
+ end
26
+
27
+ logger.info "Gracefully shutting down"
28
+ end
29
+
30
+ private
31
+ attr_reader :consumer, :logger, :sleep_on_empty, :split_keys
32
+
33
+ def prepare_traps
34
+ Signal.trap("INT") { initiate_graceful_shutdown }
35
+ Signal.trap("TERM") { initiate_graceful_shutdown }
36
+ end
37
+
38
+ def initiate_graceful_shutdown
39
+ @gracefully_shutting_down = true
40
+ end
41
+ end
42
+ end
43
+ end
@@ -19,16 +19,15 @@ module RubyEventStore
19
19
  raise InvalidPayload.new("Missing queue") if queue.nil? || queue.empty?
20
20
  payload = JSON.generate(parsed_record.merge({ "enqueued_at" => now.to_f }))
21
21
 
22
- redis.lpush("queue:#{queue}", payload)
22
+ redis.call("LPUSH", "queue:#{queue}", payload)
23
23
 
24
24
  @recently_used_queues << queue
25
+ rescue RedisClient::TimeoutError, RedisClient::ConnectionError
26
+ raise RetriableError
25
27
  end
26
28
 
27
29
  def after_batch
28
- if !@recently_used_queues.empty?
29
- redis.sadd("queues", @recently_used_queues.to_a)
30
- @recently_used_queues.clear
31
- end
30
+ ensure_that_sidekiq_knows_about_all_queues
32
31
  end
33
32
 
34
33
  def message_format
@@ -37,6 +36,13 @@ module RubyEventStore
37
36
 
38
37
  private
39
38
 
39
+ def ensure_that_sidekiq_knows_about_all_queues
40
+ if !@recently_used_queues.empty?
41
+ redis.call("SADD", "queues", @recently_used_queues.to_a)
42
+ @recently_used_queues.clear
43
+ end
44
+ end
45
+
40
46
  attr_reader :redis
41
47
  end
42
48
  end
@@ -8,7 +8,6 @@ module RubyEventStore
8
8
  module Outbox
9
9
  class SidekiqProducer
10
10
  def call(klass, args)
11
- sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
12
11
  item = { "args" => args.map(&:to_h).map { |h| h.transform_keys(&:to_s) }, "class" => klass }
13
12
  normalized_item = sidekiq_client.__send__(:normalize_item, item)
14
13
  payload =
@@ -29,6 +28,15 @@ module RubyEventStore
29
28
  private
30
29
 
31
30
  attr_reader :repository
31
+
32
+ def sidekiq_client
33
+ @sidekiq_client ||=
34
+ if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("7.0.0")
35
+ Sidekiq::Client.new(Sidekiq.redis_pool)
36
+ else
37
+ Sidekiq::Client.new(pool: Sidekiq.redis_pool)
38
+ end
39
+ end
32
40
  end
33
41
  end
34
42
  end
@@ -5,7 +5,7 @@ require_relative "sidekiq_producer"
5
5
  module RubyEventStore
6
6
  module Outbox
7
7
  class SidekiqScheduler
8
- def initialize(serializer: RubyEventStore::Serializers::YAML)
8
+ def initialize(serializer: Serializers::YAML)
9
9
  @serializer = serializer
10
10
  @sidekiq_producer = SidekiqProducer.new
11
11
  end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ class Tempo
6
+ EXPONENTIAL_MULTIPLIER = 2
7
+
8
+ def initialize(max_batch_size)
9
+ raise ArgumentError if max_batch_size < 1
10
+ @max_batch_size = max_batch_size
11
+ end
12
+
13
+ def batch_size
14
+ @batch_size = next_batch_size
15
+ end
16
+
17
+ private
18
+
19
+ def next_batch_size
20
+ return 1 if @batch_size.nil?
21
+ [@batch_size * EXPONENTIAL_MULTIPLIER, @max_batch_size].min
22
+ end
23
+ end
24
+ end
25
+ end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module RubyEventStore
4
4
  module Outbox
5
- VERSION = "0.0.25"
5
+ VERSION = "0.0.27"
6
6
  end
7
7
  end
@@ -2,11 +2,15 @@
2
2
 
3
3
  module RubyEventStore
4
4
  module Outbox
5
+ Error = Class.new(StandardError)
6
+ RetriableError = Class.new(Error)
5
7
  end
6
8
  end
7
9
 
8
10
  require_relative "outbox/fetch_specification"
9
11
  require_relative "outbox/repository"
10
12
  require_relative "outbox/sidekiq_scheduler"
11
- require_relative "outbox/legacy_sidekiq_scheduler"
12
13
  require_relative "outbox/version"
14
+ require_relative "outbox/tempo"
15
+ require_relative "outbox/batch_result"
16
+ require_relative "outbox/cleanup_strategies"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ruby_event_store-outbox
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.25
4
+ version: 0.0.27
5
5
  platform: ruby
6
6
  authors:
7
7
  - Arkency
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-05-27 00:00:00.000000000 Z
11
+ date: 2024-04-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ruby_event_store
@@ -38,7 +38,7 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '6.0'
41
- description:
41
+ description:
42
42
  email: dev@arkency.com
43
43
  executables:
44
44
  - res_outbox
@@ -49,24 +49,27 @@ files:
49
49
  - README.md
50
50
  - bin/res_outbox
51
51
  - lib/generators/ruby_event_store/outbox/migration_generator.rb
52
- - lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.rb
52
+ - lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.erb
53
53
  - lib/ruby_event_store/outbox.rb
54
+ - lib/ruby_event_store/outbox/batch_result.rb
55
+ - lib/ruby_event_store/outbox/cleanup_strategies.rb
54
56
  - lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb
55
57
  - lib/ruby_event_store/outbox/cleanup_strategies/none.rb
56
58
  - lib/ruby_event_store/outbox/cli.rb
57
- - lib/ruby_event_store/outbox/cli.rb.orig
59
+ - lib/ruby_event_store/outbox/configuration.rb
58
60
  - lib/ruby_event_store/outbox/consumer.rb
59
61
  - lib/ruby_event_store/outbox/fetch_specification.rb
60
- - lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb
61
62
  - lib/ruby_event_store/outbox/metrics.rb
62
63
  - lib/ruby_event_store/outbox/metrics/influx.rb
63
64
  - lib/ruby_event_store/outbox/metrics/null.rb
64
65
  - lib/ruby_event_store/outbox/metrics/test.rb
65
66
  - lib/ruby_event_store/outbox/repository.rb
67
+ - lib/ruby_event_store/outbox/runner.rb
66
68
  - lib/ruby_event_store/outbox/sidekiq5_format.rb
67
69
  - lib/ruby_event_store/outbox/sidekiq_processor.rb
68
70
  - lib/ruby_event_store/outbox/sidekiq_producer.rb
69
71
  - lib/ruby_event_store/outbox/sidekiq_scheduler.rb
72
+ - lib/ruby_event_store/outbox/tempo.rb
70
73
  - lib/ruby_event_store/outbox/version.rb
71
74
  homepage: https://railseventstore.org
72
75
  licenses:
@@ -76,7 +79,7 @@ metadata:
76
79
  source_code_uri: https://github.com/RailsEventStore/rails_event_store
77
80
  bug_tracker_uri: https://github.com/RailsEventStore/rails_event_store/issues
78
81
  rubygems_mfa_required: 'true'
79
- post_install_message:
82
+ post_install_message:
80
83
  rdoc_options: []
81
84
  require_paths:
82
85
  - lib
@@ -84,15 +87,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
84
87
  requirements:
85
88
  - - ">="
86
89
  - !ruby/object:Gem::Version
87
- version: '2.6'
90
+ version: '2.7'
88
91
  required_rubygems_version: !ruby/object:Gem::Requirement
89
92
  requirements:
90
93
  - - ">="
91
94
  - !ruby/object:Gem::Version
92
95
  version: '0'
93
96
  requirements: []
94
- rubygems_version: 3.3.9
95
- signing_key:
97
+ rubygems_version: 3.4.10
98
+ signing_key:
96
99
  specification_version: 4
97
100
  summary: Active Record based outbox for Ruby Event Store
98
101
  test_files: []
@@ -1,132 +0,0 @@
1
- require "optparse"
2
- require_relative "version"
3
- require_relative "consumer"
4
- require_relative "metrics"
5
-
6
- module RubyEventStore
7
- module Outbox
8
- class CLI
9
- DEFAULTS = {
10
- database_url: nil,
11
- redis_url: nil,
12
- log_level: :warn,
13
- split_keys: nil,
14
- message_format: "sidekiq5",
15
- batch_size: 100,
16
- metrics_url: nil,
17
- cleanup_strategy: :none,
18
- cleanup_limit: :all,
19
- sleep_on_empty: 0.5
20
- }
21
- Options = Struct.new(*DEFAULTS.keys)
22
-
23
- class Parser
24
- def self.parse(argv)
25
- options = Options.new(*DEFAULTS.values)
26
- OptionParser
27
- .new do |option_parser|
28
- option_parser.banner = "Usage: res_outbox [options]"
29
-
30
- option_parser.on(
31
- "--database-url=DATABASE_URL",
32
- "Database where outbox table is stored"
33
- ) { |database_url| options.database_url = database_url }
34
-
35
- option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
36
- options.redis_url = redis_url
37
- end
38
-
39
- option_parser.on(
40
- "--log-level=LOG_LEVEL",
41
- %i[fatal error warn info debug],
42
- "Logging level, one of: fatal, error, warn, info, debug. Default: warn"
43
- ) { |log_level| options.log_level = log_level.to_sym }
44
-
45
- option_parser.on(
46
- "--message-format=FORMAT",
47
- ["sidekiq5"],
48
- "Message format, supported: sidekiq5. Default: sidekiq5"
49
- ) { |message_format| options.message_format = message_format }
50
-
51
- option_parser.on(
52
- "--split-keys=SPLIT_KEYS",
53
- Array,
54
- "Split keys which should be handled, all if not specified"
55
- ) { |split_keys| options.split_keys = split_keys if !split_keys.empty? }
56
-
57
- option_parser.on(
58
- "--batch-size=BATCH_SIZE",
59
- Integer,
60
- "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100"
61
- ) { |batch_size| options.batch_size = batch_size }
62
-
63
- option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
64
- options.metrics_url = metrics_url
65
- end
66
-
67
- option_parser.on(
68
- "--cleanup=STRATEGY",
69
- "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none"
70
- ) { |cleanup_strategy| options.cleanup_strategy = cleanup_strategy }
71
-
72
- option_parser.on(
73
- "--cleanup-limit=LIMIT",
74
- "Amount of records removed in single cleanup run. One of: all or number of records that should be removed. Default: all"
75
- ) { |cleanup_limit| options.cleanup_limit = cleanup_limit }
76
-
77
- option_parser.on(
78
- "--sleep-on-empty=SLEEP_TIME",
79
- Float,
80
- "How long to sleep before next check when there was nothing to do. Default: 0.5"
81
- ) { |sleep_on_empty| options.sleep_on_empty = sleep_on_empty }
82
-
83
- option_parser.on_tail("--version", "Show version") do
84
- puts VERSION
85
- exit
86
- end
87
- end
88
- .parse(argv)
89
- return options
90
- end
91
- end
92
-
93
- def run(argv)
94
- options = Parser.parse(argv)
95
- outbox_consumer = build_consumer(options)
96
- outbox_consumer.init
97
- outbox_consumer.run
98
- end
99
-
100
- def build_consumer(options)
101
- consumer_uuid = SecureRandom.uuid
102
- logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox #{consumer_uuid}")
103
- <<<<<<< HEAD
104
- consumer_configuration =
105
- Consumer::Configuration.new(
106
- split_keys: options.split_keys,
107
- message_format: options.message_format,
108
- batch_size: options.batch_size,
109
- database_url: options.database_url,
110
- redis_url: options.redis_url,
111
- cleanup: options.cleanup_strategy,
112
- sleep_on_empty: options.sleep_on_empty
113
- )
114
- =======
115
- consumer_configuration = Consumer::Configuration.new(
116
- split_keys: options.split_keys,
117
- message_format: options.message_format,
118
- batch_size: options.batch_size,
119
- database_url: options.database_url,
120
- redis_url: options.redis_url,
121
- cleanup: options.cleanup_strategy,
122
- cleanup_limit: options.cleanup_limit,
123
- sleep_on_empty: options.sleep_on_empty,
124
- )
125
- >>>>>>> 7f077fa2... fix error with passing `--cleanup-limit` from CLI to consumer
126
- metrics = Metrics.from_url(options.metrics_url)
127
- outbox_consumer =
128
- RubyEventStore::Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
129
- end
130
- end
131
- end
132
- end
@@ -1,25 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative "sidekiq_producer"
4
-
5
- module RubyEventStore
6
- module Outbox
7
- class LegacySidekiqScheduler
8
- def initialize
9
- @sidekiq_producer = SidekiqProducer.new
10
- end
11
-
12
- def call(klass, serialized_record)
13
- sidekiq_producer.call(klass, [serialized_record])
14
- end
15
-
16
- def verify(subscriber)
17
- Class === subscriber && subscriber.respond_to?(:through_outbox?) && subscriber.through_outbox?
18
- end
19
-
20
- private
21
-
22
- attr_reader :sidekiq_producer
23
- end
24
- end
25
- end