ruby_event_store-outbox 0.0.24 → 0.0.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +3 -3
  3. data/lib/generators/ruby_event_store/outbox/migration_generator.rb +17 -15
  4. data/lib/generators/ruby_event_store/outbox/templates/{create_event_store_outbox_template.rb → create_event_store_outbox_template.erb} +4 -4
  5. data/lib/ruby_event_store/outbox/batch_result.rb +26 -0
  6. data/lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb +3 -0
  7. data/lib/ruby_event_store/outbox/cleanup_strategies/none.rb +3 -2
  8. data/lib/ruby_event_store/outbox/cleanup_strategies.rb +20 -0
  9. data/lib/ruby_event_store/outbox/cli.rb +64 -48
  10. data/lib/ruby_event_store/outbox/configuration.rb +50 -0
  11. data/lib/ruby_event_store/outbox/consumer.rb +53 -102
  12. data/lib/ruby_event_store/outbox/fetch_specification.rb +4 -10
  13. data/lib/ruby_event_store/outbox/metrics/influx.rb +21 -25
  14. data/lib/ruby_event_store/outbox/metrics/null.rb +4 -4
  15. data/lib/ruby_event_store/outbox/metrics/test.rb +10 -2
  16. data/lib/ruby_event_store/outbox/metrics.rb +2 -0
  17. data/lib/ruby_event_store/outbox/repository.rb +20 -24
  18. data/lib/ruby_event_store/outbox/runner.rb +43 -0
  19. data/lib/ruby_event_store/outbox/sidekiq_processor.rb +13 -8
  20. data/lib/ruby_event_store/outbox/sidekiq_producer.rb +20 -9
  21. data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb +2 -1
  22. data/lib/ruby_event_store/outbox/tempo.rb +25 -0
  23. data/lib/ruby_event_store/outbox/version.rb +1 -1
  24. data/lib/ruby_event_store/outbox.rb +9 -5
  25. metadata +16 -12
  26. data/lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb +0 -24
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7c7c7068c9bd49735031384ece752500cc641f5035bc1f7953104e64a3d22dd8
4
- data.tar.gz: 2975fdb8c2dfe6fc9843ce5efd00042a2c16d2d9ec2e6524598507a4fcb36dc3
3
+ metadata.gz: e26efaae4895206e93089d60869e01dd1a0f8182e6f41e49edef000a2833b5bb
4
+ data.tar.gz: 1ab71712967fa31437d538c787721f6bd27f0a65d1b0474e95cd1428945d9b62
5
5
  SHA512:
6
- metadata.gz: 3a1118657e1c164df15f9eb7d117adb80a4f7a693eed4a3974dc4a9b0f3bdbd15504f441fe50dbd0fdf5318fae6205dd2c4cadeebd0c4897a61fb4be4ea2e382
7
- data.tar.gz: 7a84d03ea98a5d2b0019f2eedf3c76b509f5bb25c44fb2db7dc9a952c1752cf4db17ee284fd9ee8bce4295267743a29222d26bee1709d554375f95c64ebb1203
6
+ metadata.gz: 3189203160620f90690f35b511d2e60471c6652e13ef510464952aa40ba2b35c650d66477f1ab5e3360e6bf31ab6a1fdf40aadd8916ff9aa802dbe069034a013
7
+ data.tar.gz: 765725def129ea5368be1fd5fcb2ae013e45efe17ed48b9d58eb4d1c1356582fb76fcafd9885d9fc527556a6351993709250c434b8ee586708f9b8514d617e2f
data/README.md CHANGED
@@ -34,11 +34,12 @@ Additionally, your handler's `through_outbox?` method should return `true`, for
34
34
 
35
35
  ```ruby
36
36
  class SomeHandler
37
- def self.through_outbox?; true; end
37
+ def self.through_outbox?
38
+ true
39
+ end
38
40
  end
39
41
  ```
40
42
 
41
-
42
43
  ## Installation (outbox process)
43
44
 
44
45
  Run following process in any way you prefer:
@@ -65,7 +66,6 @@ res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" \
65
66
  --metrics-url=http://user:password@localhost:8086/dbname"
66
67
  ```
67
68
 
68
-
69
69
  ## Contributing
70
70
 
71
71
  Bug reports and pull requests are welcome on GitHub at https://github.com/RailsEventStore/rails_event_store.
@@ -1,28 +1,30 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  begin
4
- require 'rails/generators'
4
+ require "rails/generators"
5
5
  rescue LoadError
6
6
  end
7
7
 
8
- module RubyEventStore
9
- module Outbox
10
- class MigrationGenerator < Rails::Generators::Base
11
- source_root File.expand_path(File.join(File.dirname(__FILE__), './templates'))
8
+ if defined?(Rails::Generators::Base)
9
+ module RubyEventStore
10
+ module Outbox
11
+ class MigrationGenerator < Rails::Generators::Base
12
+ source_root File.expand_path(File.join(File.dirname(__FILE__), "./templates"))
12
13
 
13
- def create_migration
14
- template "create_event_store_outbox_template.rb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
15
- end
14
+ def create_migration
15
+ template "create_event_store_outbox_template.erb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
16
+ end
16
17
 
17
- private
18
+ private
18
19
 
19
- def migration_version
20
- "[4.2]"
21
- end
20
+ def migration_version
21
+ ::ActiveRecord::Migration.current_version
22
+ end
22
23
 
23
- def timestamp
24
- Time.now.strftime("%Y%m%d%H%M%S")
24
+ def timestamp
25
+ Time.now.strftime("%Y%m%d%H%M%S")
26
+ end
25
27
  end
26
28
  end
27
29
  end
28
- end if defined?(Rails::Generators::Base)
30
+ end
@@ -1,13 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- class CreateEventStoreOutbox < ActiveRecord::Migration<%= migration_version %>
3
+ class CreateEventStoreOutbox < ActiveRecord::Migration[<%= migration_version %>]
4
4
  def change
5
5
  create_table(:event_store_outbox, force: false) do |t|
6
6
  t.string :split_key, null: true
7
7
  t.string :format, null: false
8
8
  t.binary :payload, null: false
9
- t.datetime :created_at, null: false
10
- t.datetime :enqueued_at, null: true
9
+ t.datetime :created_at, null: false, precision: 6
10
+ t.datetime :enqueued_at, null: true, precision: 6
11
11
  end
12
12
  add_index :event_store_outbox, [:format, :enqueued_at, :split_key], name: "index_event_store_outbox_for_pool"
13
13
  add_index :event_store_outbox, [:created_at, :enqueued_at], name: "index_event_store_outbox_for_clear"
@@ -15,7 +15,7 @@ class CreateEventStoreOutbox < ActiveRecord::Migration<%= migration_version %>
15
15
  create_table(:event_store_outbox_locks, force: false) do |t|
16
16
  t.string :format, null: false
17
17
  t.string :split_key, null: false
18
- t.datetime :locked_at, null: true
18
+ t.datetime :locked_at, null: true, precision: 6
19
19
  t.string :locked_by, null: true, limit: 36
20
20
  end
21
21
  add_index :event_store_outbox_locks, [:format, :split_key], name: "index_event_store_outbox_locks_for_locking", unique: true
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ class BatchResult
6
+ def self.empty
7
+ new
8
+ end
9
+
10
+ def initialize
11
+ @success_count = 0
12
+ @failed_count = 0
13
+ end
14
+
15
+ attr_reader :success_count, :failed_count
16
+
17
+ def count_success!
18
+ @success_count += 1
19
+ end
20
+
21
+ def count_failed!
22
+ @failed_count += 1
23
+ end
24
+ end
25
+ end
26
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  module CleanupStrategies
@@ -13,6 +15,7 @@ module RubyEventStore
13
15
  end
14
16
 
15
17
  private
18
+
16
19
  attr_reader :repository, :duration, :limit
17
20
  end
18
21
  end
@@ -1,9 +1,10 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  module CleanupStrategies
4
6
  class None
5
- def initialize
6
- end
7
+ def initialize; end
7
8
 
8
9
  def call(_fetch_specification)
9
10
  :ok
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ module CleanupStrategies
6
+ def self.build(configuration, repository)
7
+ case configuration.cleanup
8
+ when :none
9
+ None.new
10
+ else
11
+ CleanOldEnqueued.new(
12
+ repository,
13
+ ActiveSupport::Duration.parse(configuration.cleanup),
14
+ configuration.cleanup_limit
15
+ )
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -1,7 +1,11 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "optparse"
2
4
  require_relative "version"
3
5
  require_relative "consumer"
6
+ require_relative "runner"
4
7
  require_relative "metrics"
8
+ require_relative "configuration"
5
9
 
6
10
  module RubyEventStore
7
11
  module Outbox
@@ -11,7 +15,7 @@ module RubyEventStore
11
15
  redis_url: nil,
12
16
  log_level: :warn,
13
17
  split_keys: nil,
14
- message_format: 'sidekiq5',
18
+ message_format: "sidekiq5",
15
19
  batch_size: 100,
16
20
  metrics_url: nil,
17
21
  cleanup_strategy: :none,
@@ -23,69 +27,84 @@ module RubyEventStore
23
27
  class Parser
24
28
  def self.parse(argv)
25
29
  options = Options.new(*DEFAULTS.values)
26
- OptionParser.new do |option_parser|
27
- option_parser.banner = "Usage: res_outbox [options]"
30
+ OptionParser
31
+ .new do |option_parser|
32
+ option_parser.banner = "Usage: res_outbox [options]"
28
33
 
29
- option_parser.on("--database-url=DATABASE_URL", "Database where outbox table is stored") do |database_url|
30
- options.database_url = database_url
31
- end
34
+ option_parser.on(
35
+ "--database-url=DATABASE_URL",
36
+ "Database where outbox table is stored"
37
+ ) { |database_url| options.database_url = database_url }
32
38
 
33
- option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
34
- options.redis_url = redis_url
35
- end
39
+ option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
40
+ options.redis_url = redis_url
41
+ end
36
42
 
37
- option_parser.on("--log-level=LOG_LEVEL", [:fatal, :error, :warn, :info, :debug], "Logging level, one of: fatal, error, warn, info, debug. Default: warn") do |log_level|
38
- options.log_level = log_level.to_sym
39
- end
43
+ option_parser.on(
44
+ "--log-level=LOG_LEVEL",
45
+ %i[fatal error warn info debug],
46
+ "Logging level, one of: fatal, error, warn, info, debug. Default: warn"
47
+ ) { |log_level| options.log_level = log_level.to_sym }
40
48
 
41
- option_parser.on("--message-format=FORMAT", ["sidekiq5"], "Message format, supported: sidekiq5. Default: sidekiq5") do |message_format|
42
- options.message_format = message_format
43
- end
49
+ option_parser.on(
50
+ "--message-format=FORMAT",
51
+ ["sidekiq5"],
52
+ "Message format, supported: sidekiq5. Default: sidekiq5"
53
+ ) { |message_format| options.message_format = message_format }
44
54
 
45
- option_parser.on("--split-keys=SPLIT_KEYS", Array, "Split keys which should be handled, all if not specified") do |split_keys|
46
- options.split_keys = split_keys if !split_keys.empty?
47
- end
55
+ option_parser.on(
56
+ "--split-keys=SPLIT_KEYS",
57
+ Array,
58
+ "Split keys which should be handled, all if not specified"
59
+ ) { |split_keys| options.split_keys = split_keys if !split_keys.empty? }
48
60
 
49
- option_parser.on("--batch-size=BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100") do |batch_size|
50
- options.batch_size = batch_size
51
- end
61
+ option_parser.on(
62
+ "--batch-size=BATCH_SIZE",
63
+ Integer,
64
+ "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100"
65
+ ) { |batch_size| options.batch_size = batch_size }
52
66
 
53
- option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
54
- options.metrics_url = metrics_url
55
- end
67
+ option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
68
+ options.metrics_url = metrics_url
69
+ end
56
70
 
57
- option_parser.on("--cleanup=STRATEGY", "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none") do |cleanup_strategy|
58
- options.cleanup_strategy = cleanup_strategy
59
- end
71
+ option_parser.on(
72
+ "--cleanup=STRATEGY",
73
+ "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none"
74
+ ) { |cleanup_strategy| options.cleanup_strategy = cleanup_strategy }
60
75
 
61
- option_parser.on("--cleanup-limit=LIMIT", "Amount of records removed in single cleanup run. One of: all or number of records that should be removed. Default: all") do |cleanup_limit|
62
- options.cleanup_limit = cleanup_limit
63
- end
76
+ option_parser.on(
77
+ "--cleanup-limit=LIMIT",
78
+ "Amount of records removed in single cleanup run. One of: all or number of records that should be removed. Default: all"
79
+ ) { |cleanup_limit| options.cleanup_limit = cleanup_limit }
64
80
 
65
- option_parser.on("--sleep-on-empty=SLEEP_TIME", Float, "How long to sleep before next check when there was nothing to do. Default: 0.5") do |sleep_on_empty|
66
- options.sleep_on_empty = sleep_on_empty
67
- end
81
+ option_parser.on(
82
+ "--sleep-on-empty=SLEEP_TIME",
83
+ Float,
84
+ "How long to sleep before next check when there was nothing to do. Default: 0.5"
85
+ ) { |sleep_on_empty| options.sleep_on_empty = sleep_on_empty }
68
86
 
69
- option_parser.on_tail("--version", "Show version") do
70
- puts VERSION
71
- exit
87
+ option_parser.on_tail("--version", "Show version") do
88
+ puts VERSION
89
+ exit
90
+ end
72
91
  end
73
- end.parse(argv)
92
+ .parse(argv)
74
93
  return options
75
94
  end
76
95
  end
77
96
 
78
97
  def run(argv)
79
98
  options = Parser.parse(argv)
80
- outbox_consumer = build_consumer(options)
81
- outbox_consumer.init
82
- outbox_consumer.run
99
+ build_runner(options)
100
+ .init
101
+ .run
83
102
  end
84
103
 
85
- def build_consumer(options)
104
+ def build_runner(options)
86
105
  consumer_uuid = SecureRandom.uuid
87
106
  logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox #{consumer_uuid}")
88
- consumer_configuration = Consumer::Configuration.new(
107
+ consumer_configuration = Configuration.new(
89
108
  split_keys: options.split_keys,
90
109
  message_format: options.message_format,
91
110
  batch_size: options.batch_size,
@@ -96,12 +115,9 @@ module RubyEventStore
96
115
  sleep_on_empty: options.sleep_on_empty,
97
116
  )
98
117
  metrics = Metrics.from_url(options.metrics_url)
99
- outbox_consumer = RubyEventStore::Outbox::Consumer.new(
100
- consumer_uuid,
101
- consumer_configuration,
102
- logger: logger,
103
- metrics: metrics,
104
- )
118
+ outbox_consumer =
119
+ Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
120
+ Runner.new(outbox_consumer, consumer_configuration, logger: logger)
105
121
  end
106
122
  end
107
123
  end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ class Configuration
6
+ def initialize(
7
+ split_keys:,
8
+ message_format:,
9
+ batch_size:,
10
+ database_url:,
11
+ redis_url:,
12
+ cleanup:,
13
+ cleanup_limit:,
14
+ sleep_on_empty:
15
+ )
16
+ @split_keys = split_keys
17
+ @message_format = message_format
18
+ @batch_size = batch_size || 100
19
+ @database_url = database_url
20
+ @redis_url = redis_url
21
+ @cleanup = cleanup
22
+ @cleanup_limit = cleanup_limit
23
+ @sleep_on_empty = sleep_on_empty
24
+ freeze
25
+ end
26
+
27
+ def with(overriden_options)
28
+ self.class.new(
29
+ split_keys: overriden_options.fetch(:split_keys, split_keys),
30
+ message_format: overriden_options.fetch(:message_format, message_format),
31
+ batch_size: overriden_options.fetch(:batch_size, batch_size),
32
+ database_url: overriden_options.fetch(:database_url, database_url),
33
+ redis_url: overriden_options.fetch(:redis_url, redis_url),
34
+ cleanup: overriden_options.fetch(:cleanup, cleanup),
35
+ cleanup_limit: overriden_options.fetch(:cleanup_limit, cleanup_limit),
36
+ sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
37
+ )
38
+ end
39
+
40
+ attr_reader :split_keys,
41
+ :message_format,
42
+ :batch_size,
43
+ :database_url,
44
+ :redis_url,
45
+ :cleanup,
46
+ :cleanup_limit,
47
+ :sleep_on_empty
48
+ end
49
+ end
50
+ end
@@ -1,5 +1,7 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "logger"
2
- require "redis"
4
+ require "redis-client"
3
5
  require "active_record"
4
6
  require_relative "repository"
5
7
  require_relative "sidekiq5_format"
@@ -13,86 +15,24 @@ module RubyEventStore
13
15
  class Consumer
14
16
  MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK = 10
15
17
 
16
- class Configuration
17
- def initialize(
18
- split_keys:,
19
- message_format:,
20
- batch_size:,
21
- database_url:,
22
- redis_url:,
23
- cleanup:,
24
- cleanup_limit:,
25
- sleep_on_empty:
26
- )
27
- @split_keys = split_keys
28
- @message_format = message_format
29
- @batch_size = batch_size || 100
30
- @database_url = database_url
31
- @redis_url = redis_url
32
- @cleanup = cleanup
33
- @cleanup_limit = cleanup_limit
34
- @sleep_on_empty = sleep_on_empty
35
- freeze
36
- end
37
-
38
- def with(overriden_options)
39
- self.class.new(
40
- split_keys: overriden_options.fetch(:split_keys, split_keys),
41
- message_format: overriden_options.fetch(:message_format, message_format),
42
- batch_size: overriden_options.fetch(:batch_size, batch_size),
43
- database_url: overriden_options.fetch(:database_url, database_url),
44
- redis_url: overriden_options.fetch(:redis_url, redis_url),
45
- cleanup: overriden_options.fetch(:cleanup, cleanup),
46
- cleanup_limit: overriden_options.fetch(:cleanup_limit, cleanup_limit),
47
- sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
48
- )
49
- end
50
-
51
- attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url, :cleanup, :cleanup_limit, :sleep_on_empty
52
- end
53
-
54
18
  def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
55
19
  @split_keys = configuration.split_keys
56
20
  @clock = clock
57
21
  @logger = logger
58
22
  @metrics = metrics
59
- @batch_size = configuration.batch_size
60
- @sleep_on_empty = configuration.sleep_on_empty
23
+ @tempo = Tempo.new(configuration.batch_size)
61
24
  @consumer_uuid = consumer_uuid
62
25
 
63
26
  raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
64
- @processor = SidekiqProcessor.new(Redis.new(url: configuration.redis_url))
65
-
66
- @gracefully_shutting_down = false
67
- prepare_traps
27
+ redis_config = RedisClient.config(url: configuration.redis_url)
28
+ @processor = SidekiqProcessor.new(redis_config.new_client)
68
29
 
69
30
  @repository = Repository.new(configuration.database_url)
70
- @cleanup_strategy = case configuration.cleanup
71
- when :none
72
- CleanupStrategies::None.new
73
- else
74
- CleanupStrategies::CleanOldEnqueued.new(repository, ActiveSupport::Duration.parse(configuration.cleanup), configuration.cleanup_limit)
75
- end
31
+ @cleanup_strategy = CleanupStrategies.build(configuration, repository)
76
32
  end
77
33
 
78
- def init
79
- logger.info("Initiated RubyEventStore::Outbox v#{VERSION}")
80
- logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
81
- end
82
-
83
- def run
84
- while !@gracefully_shutting_down do
85
- was_something_changed = one_loop
86
- if !was_something_changed
87
- STDOUT.flush
88
- sleep sleep_on_empty
89
- end
90
- end
91
- logger.info "Gracefully shutting down"
92
- end
93
-
94
- def one_loop
95
- remaining_split_keys = @split_keys.dup
34
+ def process
35
+ remaining_split_keys = split_keys.dup
96
36
 
97
37
  was_something_changed = false
98
38
  while (split_key = remaining_split_keys.shift)
@@ -109,45 +49,41 @@ module RubyEventStore
109
49
 
110
50
  MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK.times do
111
51
  batch = retrieve_batch(fetch_specification)
112
- if batch.empty?
113
- break
114
- end
52
+ break if batch.empty?
115
53
 
116
- failed_record_ids = []
117
- updated_record_ids = []
54
+ batch_result = BatchResult.empty
118
55
  batch.each do |record|
119
- begin
56
+ handle_failure(batch_result) do
120
57
  now = @clock.now.utc
121
58
  processor.process(record, now)
122
59
 
123
60
  repository.mark_as_enqueued(record, now)
124
61
  something_processed |= true
125
- updated_record_ids << record.id
126
- rescue => e
127
- failed_record_ids << record.id
128
- e.full_message.split($/).each {|line| logger.error(line) }
62
+ batch_result.count_success!
129
63
  end
130
64
  end
131
65
 
132
66
  metrics.write_point_queue(
133
- enqueued: updated_record_ids.size,
134
- failed: failed_record_ids.size,
67
+ enqueued: batch_result.success_count,
68
+ failed: batch_result.failed_count,
135
69
  format: fetch_specification.message_format,
136
70
  split_key: fetch_specification.split_key,
137
71
  remaining: get_remaining_count(fetch_specification)
138
72
  )
139
73
 
140
- logger.info "Sent #{updated_record_ids.size} messages from outbox table"
74
+ logger.info "Sent #{batch_result.success_count} messages from outbox table"
141
75
 
142
76
  refresh_successful = refresh_lock_for_process(obtained_lock)
143
77
  break unless refresh_successful
144
78
  end
145
79
 
146
- metrics.write_point_queue(
147
- format: fetch_specification.message_format,
148
- split_key: fetch_specification.split_key,
149
- remaining: get_remaining_count(fetch_specification)
150
- ) unless something_processed
80
+ unless something_processed
81
+ metrics.write_point_queue(
82
+ format: fetch_specification.message_format,
83
+ split_key: fetch_specification.split_key,
84
+ remaining: get_remaining_count(fetch_specification)
85
+ )
86
+ end
151
87
 
152
88
  release_lock_for_process(fetch_specification)
153
89
 
@@ -159,7 +95,35 @@ module RubyEventStore
159
95
  end
160
96
 
161
97
  private
162
- attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid, :repository, :cleanup_strategy, :sleep_on_empty
98
+
99
+ attr_reader :split_keys,
100
+ :logger,
101
+ :metrics,
102
+ :processor,
103
+ :consumer_uuid,
104
+ :repository,
105
+ :cleanup_strategy,
106
+ :tempo
107
+
108
+ def handle_failure(batch_result)
109
+ retried = false
110
+ yield
111
+ rescue RetriableError => error
112
+ if retried
113
+ batch_result.count_failed!
114
+ log_error(error)
115
+ else
116
+ retried = true
117
+ retry
118
+ end
119
+ rescue => error
120
+ batch_result.count_failed!
121
+ log_error(error)
122
+ end
123
+
124
+ def log_error(e)
125
+ e.full_message.split($/).each { |line| logger.error(line) }
126
+ end
163
127
 
164
128
  def obtain_lock_for_process(fetch_specification)
165
129
  result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
@@ -230,21 +194,8 @@ module RubyEventStore
230
194
  end
231
195
  end
232
196
 
233
- def prepare_traps
234
- Signal.trap("INT") do
235
- initiate_graceful_shutdown
236
- end
237
- Signal.trap("TERM") do
238
- initiate_graceful_shutdown
239
- end
240
- end
241
-
242
- def initiate_graceful_shutdown
243
- @gracefully_shutting_down = true
244
- end
245
-
246
197
  def retrieve_batch(fetch_specification)
247
- repository.retrieve_batch(fetch_specification, batch_size)
198
+ repository.retrieve_batch(fetch_specification, tempo.batch_size)
248
199
  end
249
200
 
250
201
  def get_remaining_count(fetch_specification)
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  class FetchSpecification
@@ -10,19 +12,11 @@ module RubyEventStore
10
12
  attr_reader :message_format, :split_key
11
13
 
12
14
  def ==(other)
13
- other.instance_of?(self.class) &&
14
- other.message_format.eql?(message_format) &&
15
- other.split_key.eql?(split_key)
15
+ other.instance_of?(self.class) && other.message_format.eql?(message_format) && other.split_key.eql?(split_key)
16
16
  end
17
17
 
18
- BIG_VALUE = 0b111111100100010000010010110010101011011101110101001100100110000
19
-
20
18
  def hash
21
- [
22
- self.class,
23
- message_format,
24
- split_key,
25
- ].hash ^ BIG_VALUE
19
+ [message_format, split_key].hash ^ self.class.hash
26
20
  end
27
21
 
28
22
  alias_method :eql?, :==
@@ -1,4 +1,6 @@
1
- require 'influxdb'
1
+ # frozen_string_literal: true
2
+
3
+ require "influxdb"
2
4
 
3
5
  module RubyEventStore
4
6
  module Outbox
@@ -6,38 +8,32 @@ module RubyEventStore
6
8
  class Influx
7
9
  def initialize(url)
8
10
  uri = URI.parse(url)
9
- options = {
10
- url: url,
11
- async: true,
12
- time_precision: 'ns',
13
- }
11
+ options = { url: url, async: true, time_precision: "ns" }
14
12
  @influxdb_client = InfluxDB::Client.new(**options)
15
13
  end
16
14
 
17
15
  def write_operation_result(operation, result)
18
- write_point("ruby_event_store.outbox.lock", {
19
- values: {
20
- value: 1,
21
- },
22
- tags: {
23
- operation: operation,
24
- result: result,
25
- }
26
- })
16
+ write_point(
17
+ "ruby_event_store.outbox.lock",
18
+ { values: { value: 1 }, tags: { operation: operation, result: result } }
19
+ )
27
20
  end
28
21
 
29
22
  def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
30
- write_point("ruby_event_store.outbox.queue", {
31
- values: {
32
- enqueued: enqueued,
33
- failed: failed,
34
- remaining: remaining,
35
- },
36
- tags: {
37
- format: format,
38
- split_key: split_key,
23
+ write_point(
24
+ "ruby_event_store.outbox.queue",
25
+ {
26
+ values: {
27
+ enqueued: enqueued,
28
+ failed: failed,
29
+ remaining: remaining
30
+ },
31
+ tags: {
32
+ format: format,
33
+ split_key: split_key
34
+ }
39
35
  }
40
- })
36
+ )
41
37
  end
42
38
 
43
39
  def write_point(series, data)
@@ -1,12 +1,12 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  module Metrics
4
6
  class Null
5
- def write_operation_result(operation, result)
6
- end
7
+ def write_operation_result(operation, result); end
7
8
 
8
- def write_point_queue(**kwargs)
9
- end
9
+ def write_point_queue(**kwargs); end
10
10
  end
11
11
  end
12
12
  end
@@ -1,4 +1,6 @@
1
- require 'influxdb'
1
+ # frozen_string_literal: true
2
+
3
+ require "influxdb"
2
4
 
3
5
  module RubyEventStore
4
6
  module Outbox
@@ -14,7 +16,13 @@ module RubyEventStore
14
16
  end
15
17
 
16
18
  def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
17
- @queue_stats << { enqueued: enqueued, failed: failed, remaining: remaining, format: format, split_key: split_key}
19
+ @queue_stats << {
20
+ enqueued: enqueued,
21
+ failed: failed,
22
+ remaining: remaining,
23
+ format: format,
24
+ split_key: split_key
25
+ }
18
26
  end
19
27
 
20
28
  attr_reader :operation_results, :queue_stats
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RubyEventStore
2
4
  module Outbox
3
5
  module Metrics
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'active_record'
4
- require 'active_support/core_ext/numeric/time.rb'
3
+ require "active_record"
4
+ require "active_support/core_ext/numeric/time.rb"
5
5
 
6
6
  module RubyEventStore
7
7
  module Outbox
@@ -10,7 +10,7 @@ module RubyEventStore
10
10
 
11
11
  class Record < ::ActiveRecord::Base
12
12
  self.primary_key = :id
13
- self.table_name = 'event_store_outbox'
13
+ self.table_name = "event_store_outbox"
14
14
 
15
15
  def self.remaining_for(fetch_specification)
16
16
  where(format: fetch_specification.message_format, split_key: fetch_specification.split_key, enqueued_at: nil)
@@ -30,7 +30,7 @@ module RubyEventStore
30
30
  end
31
31
 
32
32
  class Lock < ::ActiveRecord::Base
33
- self.table_name = 'event_store_outbox_locks'
33
+ self.table_name = "event_store_outbox_locks"
34
34
 
35
35
  def self.obtain(fetch_specification, process_uuid, clock:)
36
36
  transaction do
@@ -39,16 +39,13 @@ module RubyEventStore
39
39
  if l.recently_locked?(clock: clock)
40
40
  :taken
41
41
  else
42
- l.update!(
43
- locked_by: process_uuid,
44
- locked_at: clock.now,
45
- )
42
+ l.update!(locked_by: process_uuid, locked_at: clock.now)
46
43
  l
47
44
  end
48
45
  end
49
- rescue ActiveRecord::Deadlocked
46
+ rescue ::ActiveRecord::Deadlocked
50
47
  :deadlocked
51
- rescue ActiveRecord::LockWaitTimeout
48
+ rescue ::ActiveRecord::LockWaitTimeout
52
49
  :lock_timeout
53
50
  end
54
51
 
@@ -63,9 +60,9 @@ module RubyEventStore
63
60
  :stolen
64
61
  end
65
62
  end
66
- rescue ActiveRecord::Deadlocked
63
+ rescue ::ActiveRecord::Deadlocked
67
64
  :deadlocked
68
- rescue ActiveRecord::LockWaitTimeout
65
+ rescue ::ActiveRecord::LockWaitTimeout
69
66
  :lock_timeout
70
67
  end
71
68
 
@@ -79,9 +76,9 @@ module RubyEventStore
79
76
  :ok
80
77
  end
81
78
  end
82
- rescue ActiveRecord::Deadlocked
79
+ rescue ::ActiveRecord::Deadlocked
83
80
  :deadlocked
84
- rescue ActiveRecord::LockWaitTimeout
81
+ rescue ::ActiveRecord::LockWaitTimeout
85
82
  :lock_timeout
86
83
  end
87
84
 
@@ -98,6 +95,7 @@ module RubyEventStore
98
95
  end
99
96
 
100
97
  private
98
+
101
99
  def self.lock_for_split_key(fetch_specification)
102
100
  lock.find_by(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
103
101
  end
@@ -107,7 +105,7 @@ module RubyEventStore
107
105
  if l.nil?
108
106
  begin
109
107
  l = create!(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
110
- rescue ActiveRecord::RecordNotUnique
108
+ rescue ::ActiveRecord::RecordNotUnique
111
109
  l = lock_for_split_key(fetch_specification)
112
110
  end
113
111
  end
@@ -116,9 +114,9 @@ module RubyEventStore
116
114
  end
117
115
 
118
116
  def initialize(database_url)
119
- ActiveRecord::Base.establish_connection(database_url) unless ActiveRecord::Base.connected?
120
- if ActiveRecord::Base.connection.adapter_name == "Mysql2"
121
- ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
117
+ ::ActiveRecord::Base.establish_connection(database_url) unless ::ActiveRecord::Base.connected?
118
+ if ::ActiveRecord::Base.connection.adapter_name == "Mysql2"
119
+ ::ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
122
120
  end
123
121
  end
124
122
 
@@ -143,15 +141,13 @@ module RubyEventStore
143
141
  end
144
142
 
145
143
  def delete_enqueued_older_than(fetch_specification, duration, limit)
146
- scope = Record
147
- .for_fetch_specification(fetch_specification)
148
- .where("enqueued_at < ?", duration.ago)
149
- scope = scope.limit(limit) unless limit == :all
144
+ scope = Record.for_fetch_specification(fetch_specification).where("enqueued_at < ?", duration.ago)
145
+ scope = scope.limit(limit).order(:id) unless limit == :all
150
146
  scope.delete_all
151
147
  :ok
152
- rescue ActiveRecord::Deadlocked
148
+ rescue ::ActiveRecord::Deadlocked
153
149
  :deadlocked
154
- rescue ActiveRecord::LockWaitTimeout
150
+ rescue ::ActiveRecord::LockWaitTimeout
155
151
  :lock_timeout
156
152
  end
157
153
  end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ class Runner
6
+ def initialize(consumer, configuration, logger:)
7
+ @consumer = consumer
8
+ @logger = logger
9
+ @sleep_on_empty = configuration.sleep_on_empty
10
+ @split_keys = configuration.split_keys
11
+ @gracefully_shutting_down = false
12
+ prepare_traps
13
+ end
14
+
15
+ def run
16
+ logger.info("Initiated RubyEventStore::Outbox v#{VERSION}")
17
+ logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
18
+
19
+ while !@gracefully_shutting_down
20
+ was_something_changed = consumer.process
21
+ if !was_something_changed
22
+ STDOUT.flush
23
+ sleep sleep_on_empty
24
+ end
25
+ end
26
+
27
+ logger.info "Gracefully shutting down"
28
+ end
29
+
30
+ private
31
+ attr_reader :consumer, :logger, :sleep_on_empty, :split_keys
32
+
33
+ def prepare_traps
34
+ Signal.trap("INT") { initiate_graceful_shutdown }
35
+ Signal.trap("TERM") { initiate_graceful_shutdown }
36
+ end
37
+
38
+ def initiate_graceful_shutdown
39
+ @gracefully_shutting_down = true
40
+ end
41
+ end
42
+ end
43
+ end
@@ -17,20 +17,17 @@ module RubyEventStore
17
17
 
18
18
  queue = parsed_record["queue"]
19
19
  raise InvalidPayload.new("Missing queue") if queue.nil? || queue.empty?
20
- payload = JSON.generate(parsed_record.merge({
21
- "enqueued_at" => now.to_f,
22
- }))
20
+ payload = JSON.generate(parsed_record.merge({ "enqueued_at" => now.to_f }))
23
21
 
24
- redis.lpush("queue:#{queue}", payload)
22
+ redis.call("LPUSH", "queue:#{queue}", payload)
25
23
 
26
24
  @recently_used_queues << queue
25
+ rescue RedisClient::TimeoutError, RedisClient::ConnectionError
26
+ raise RetriableError
27
27
  end
28
28
 
29
29
  def after_batch
30
- if !@recently_used_queues.empty?
31
- redis.sadd("queues", @recently_used_queues.to_a)
32
- @recently_used_queues.clear
33
- end
30
+ ensure_that_sidekiq_knows_about_all_queues
34
31
  end
35
32
 
36
33
  def message_format
@@ -38,6 +35,14 @@ module RubyEventStore
38
35
  end
39
36
 
40
37
  private
38
+
39
+ def ensure_that_sidekiq_knows_about_all_queues
40
+ if !@recently_used_queues.empty?
41
+ redis.call("SADD", "queues", @recently_used_queues.to_a)
42
+ @recently_used_queues.clear
43
+ end
44
+ end
45
+
41
46
  attr_reader :redis
42
47
  end
43
48
  end
@@ -1,31 +1,42 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'sidekiq'
4
- require_relative 'sidekiq5_format'
3
+ require "sidekiq"
4
+ require_relative "sidekiq5_format"
5
5
  require_relative "repository"
6
6
 
7
7
  module RubyEventStore
8
8
  module Outbox
9
9
  class SidekiqProducer
10
10
  def call(klass, args)
11
- sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
12
- item = {
13
- 'args' => args.map(&:to_h).map {|h| h.transform_keys(&:to_s)},
14
- 'class' => klass,
15
- }
11
+ item = { "args" => args.map(&:to_h).map { |h| h.transform_keys(&:to_s) }, "class" => klass }
16
12
  normalized_item = sidekiq_client.__send__(:normalize_item, item)
17
- payload = sidekiq_client.middleware.invoke(normalized_item['class'], normalized_item, normalized_item['queue'], Sidekiq.redis_pool) { normalized_item }
13
+ payload =
14
+ sidekiq_client
15
+ .middleware
16
+ .invoke(normalized_item["class"], normalized_item, normalized_item["queue"], Sidekiq.redis_pool) do
17
+ normalized_item
18
+ end
18
19
  if payload
19
20
  Repository::Record.create!(
20
21
  format: SIDEKIQ5_FORMAT,
21
- split_key: payload.fetch('queue'),
22
+ split_key: payload.fetch("queue"),
22
23
  payload: payload.to_json
23
24
  )
24
25
  end
25
26
  end
26
27
 
27
28
  private
29
+
28
30
  attr_reader :repository
31
+
32
+ def sidekiq_client
33
+ @sidekiq_client ||=
34
+ if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("7.0.0")
35
+ Sidekiq::Client.new(Sidekiq.redis_pool)
36
+ else
37
+ Sidekiq::Client.new(pool: Sidekiq.redis_pool)
38
+ end
39
+ end
29
40
  end
30
41
  end
31
42
  end
@@ -5,7 +5,7 @@ require_relative "sidekiq_producer"
5
5
  module RubyEventStore
6
6
  module Outbox
7
7
  class SidekiqScheduler
8
- def initialize(serializer: RubyEventStore::Serializers::YAML)
8
+ def initialize(serializer: Serializers::YAML)
9
9
  @serializer = serializer
10
10
  @sidekiq_producer = SidekiqProducer.new
11
11
  end
@@ -19,6 +19,7 @@ module RubyEventStore
19
19
  end
20
20
 
21
21
  private
22
+
22
23
  attr_reader :serializer, :sidekiq_producer
23
24
  end
24
25
  end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ class Tempo
6
+ EXPONENTIAL_MULTIPLIER = 2
7
+
8
+ def initialize(max_batch_size)
9
+ raise ArgumentError if max_batch_size < 1
10
+ @max_batch_size = max_batch_size
11
+ end
12
+
13
+ def batch_size
14
+ @batch_size = next_batch_size
15
+ end
16
+
17
+ private
18
+
19
+ def next_batch_size
20
+ return 1 if @batch_size.nil?
21
+ [@batch_size * EXPONENTIAL_MULTIPLIER, @max_batch_size].min
22
+ end
23
+ end
24
+ end
25
+ end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module RubyEventStore
4
4
  module Outbox
5
- VERSION = "0.0.24"
5
+ VERSION = "0.0.26"
6
6
  end
7
7
  end
@@ -2,11 +2,15 @@
2
2
 
3
3
  module RubyEventStore
4
4
  module Outbox
5
+ Error = Class.new(StandardError)
6
+ RetriableError = Class.new(Error)
5
7
  end
6
8
  end
7
9
 
8
- require_relative 'outbox/fetch_specification'
9
- require_relative 'outbox/repository'
10
- require_relative 'outbox/sidekiq_scheduler'
11
- require_relative 'outbox/legacy_sidekiq_scheduler'
12
- require_relative 'outbox/version'
10
+ require_relative "outbox/fetch_specification"
11
+ require_relative "outbox/repository"
12
+ require_relative "outbox/sidekiq_scheduler"
13
+ require_relative "outbox/version"
14
+ require_relative "outbox/tempo"
15
+ require_relative "outbox/batch_result"
16
+ require_relative "outbox/cleanup_strategies"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ruby_event_store-outbox
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.24
4
+ version: 0.0.26
5
5
  platform: ruby
6
6
  authors:
7
7
  - Arkency
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-05-26 00:00:00.000000000 Z
11
+ date: 2024-04-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ruby_event_store
@@ -30,15 +30,15 @@ dependencies:
30
30
  requirements:
31
31
  - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: '5.2'
33
+ version: '6.0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: '5.2'
41
- description:
40
+ version: '6.0'
41
+ description:
42
42
  email: dev@arkency.com
43
43
  executables:
44
44
  - res_outbox
@@ -49,23 +49,27 @@ files:
49
49
  - README.md
50
50
  - bin/res_outbox
51
51
  - lib/generators/ruby_event_store/outbox/migration_generator.rb
52
- - lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.rb
52
+ - lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.erb
53
53
  - lib/ruby_event_store/outbox.rb
54
+ - lib/ruby_event_store/outbox/batch_result.rb
55
+ - lib/ruby_event_store/outbox/cleanup_strategies.rb
54
56
  - lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb
55
57
  - lib/ruby_event_store/outbox/cleanup_strategies/none.rb
56
58
  - lib/ruby_event_store/outbox/cli.rb
59
+ - lib/ruby_event_store/outbox/configuration.rb
57
60
  - lib/ruby_event_store/outbox/consumer.rb
58
61
  - lib/ruby_event_store/outbox/fetch_specification.rb
59
- - lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb
60
62
  - lib/ruby_event_store/outbox/metrics.rb
61
63
  - lib/ruby_event_store/outbox/metrics/influx.rb
62
64
  - lib/ruby_event_store/outbox/metrics/null.rb
63
65
  - lib/ruby_event_store/outbox/metrics/test.rb
64
66
  - lib/ruby_event_store/outbox/repository.rb
67
+ - lib/ruby_event_store/outbox/runner.rb
65
68
  - lib/ruby_event_store/outbox/sidekiq5_format.rb
66
69
  - lib/ruby_event_store/outbox/sidekiq_processor.rb
67
70
  - lib/ruby_event_store/outbox/sidekiq_producer.rb
68
71
  - lib/ruby_event_store/outbox/sidekiq_scheduler.rb
72
+ - lib/ruby_event_store/outbox/tempo.rb
69
73
  - lib/ruby_event_store/outbox/version.rb
70
74
  homepage: https://railseventstore.org
71
75
  licenses:
@@ -75,7 +79,7 @@ metadata:
75
79
  source_code_uri: https://github.com/RailsEventStore/rails_event_store
76
80
  bug_tracker_uri: https://github.com/RailsEventStore/rails_event_store/issues
77
81
  rubygems_mfa_required: 'true'
78
- post_install_message:
82
+ post_install_message:
79
83
  rdoc_options: []
80
84
  require_paths:
81
85
  - lib
@@ -83,15 +87,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
83
87
  requirements:
84
88
  - - ">="
85
89
  - !ruby/object:Gem::Version
86
- version: '2.6'
90
+ version: '2.7'
87
91
  required_rubygems_version: !ruby/object:Gem::Requirement
88
92
  requirements:
89
93
  - - ">="
90
94
  - !ruby/object:Gem::Version
91
95
  version: '0'
92
96
  requirements: []
93
- rubygems_version: 3.3.9
94
- signing_key:
97
+ rubygems_version: 3.4.10
98
+ signing_key:
95
99
  specification_version: 4
96
100
  summary: Active Record based outbox for Ruby Event Store
97
101
  test_files: []
@@ -1,24 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative "sidekiq_producer"
4
-
5
- module RubyEventStore
6
- module Outbox
7
- class LegacySidekiqScheduler
8
- def initialize
9
- @sidekiq_producer = SidekiqProducer.new
10
- end
11
-
12
- def call(klass, serialized_record)
13
- sidekiq_producer.call(klass, [serialized_record])
14
- end
15
-
16
- def verify(subscriber)
17
- Class === subscriber && subscriber.respond_to?(:through_outbox?) && subscriber.through_outbox?
18
- end
19
-
20
- private
21
- attr_reader :sidekiq_producer
22
- end
23
- end
24
- end