ruby_event_store-outbox 0.0.13 → 0.0.17

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1b73edc767b3c0caa9ff76cd06a58468e18aa2556f25a7c170bd9f0da6bdce9a
4
- data.tar.gz: b96f01c515c3975694b7ba54ab9b8057bb506534d22c313823488f0150b98ce3
3
+ metadata.gz: 2dbabda53c709b7e11850772272b736226ed33e441392e9f7a1b7b5e3ead47b7
4
+ data.tar.gz: 02eec2972f701e1ecc09bd65c38cecafef53d4dc45f95f69a869457289a1edcc
5
5
  SHA512:
6
- metadata.gz: d9c3e4f4d5733cb1afbea5ffa9ae91af3c1e4ef17f6d371a9dff483e95a4034f57c0bbe1166a87b5e44f562773f683d5442eaa60b0a704439d974c0718a74a49
7
- data.tar.gz: 5a1e56fd33a9f296246ed9c429dc9abf35e7fed714c9a2b30cb6aaf926babf505a09c8a093a3aba77076aa6f6727f3568d4677035bec47289181811878534a0a
6
+ metadata.gz: aec65f023d21c3a86204b31b22ae1b21157f46c39869a22e522efe7b13e320cd3d4f75ef139c6e421d40b203a3d33bcb368cf16cb98552ff803380a36469290a
7
+ data.tar.gz: 6fa822fb40b763717f31297944423bcb708400ba3bfad9852cde4edc99f79308ad3bb837ffc593d2cae6942961735a6d5c6c9bf25088bafbaea9b48f2d278842
data/README.md CHANGED
@@ -1,7 +1,10 @@
1
1
  # Ruby Event Store Outbox
2
2
 
3
- Very much work in progress.
3
+ ![Ruby Event Store Outbox](https://github.com/RailsEventStore/rails_event_store/workflows/ruby_event_store-outbox/badge.svg)
4
4
 
5
+ **Experimental feature of RES ecosystem.**
6
+
7
+ This repository includes a process and a Rails Event Store scheduler, which can be used to transactionally enqueue background jobs into your background jobs tool of choice. The scheduler included in this repo adds the jobs into the RDBMS into specific table instead of redis inside your transaction, and the process is enqueuing the jobs from that table to the background jobs tool.
5
8
 
6
9
  ## Installation (app)
7
10
 
@@ -11,6 +14,12 @@ Add to your gemfile in application:
11
14
  gem "ruby_event_store-outbox"
12
15
  ```
13
16
 
17
+ Generate and execute the migration adding necessary tables. If it's needed, change the type of the `payload` column to `mediumbinary` or `longbinary`.
18
+
19
+ ```
20
+ bin/rails generate ruby_event_store:outbox:migration
21
+ ```
22
+
14
23
  In your event store configuration, as a dispatcher use `RubyEventStore::ImmediateAsyncDispatcher` with `RubyEventStore::Outbox::SidekiqScheduler`, for example:
15
24
 
16
25
  ```ruby
@@ -35,10 +44,36 @@ end
35
44
  Run following process in any way you prefer:
36
45
 
37
46
  ```
38
- res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" --redis-url="redis://localhost:6379/0" --log-level=info
47
+ res_outbox \
48
+ --database-url="mysql2://root@0.0.0.0:3306/my_database" \
49
+ --redis-url="redis://localhost:6379/0" \
50
+ --log-level=info \
51
+ --split-keys=sidekiq_queue1,sidekiq_queue2
52
+ ```
53
+
54
+ It is possible to run as many instances as you prefer, but it does not make sense to run more instances than there are different split keys (sidekiq queues), as one process is operating at one moment only one split key.
55
+
56
+ ### Metrics
57
+
58
+ It is possible for the outbox process to send metrics to InfluxDB (this requires influxdb gem in version at least 0.8.1). In order to do that, specify a `--metrics-url` parameter, for example:
59
+
60
+ ```
61
+ res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" \
62
+ --redis-url="redis://localhost:6379/0" \
63
+ --log-level=info \
64
+ --split-keys=sidekiq_queue1,sidekiq_queue2 \
65
+ --metrics-url=http://user:password@localhost:8086/dbname"
39
66
  ```
40
67
 
41
68
 
42
69
  ## Contributing
43
70
 
44
71
  Bug reports and pull requests are welcome on GitHub at https://github.com/RailsEventStore/rails_event_store.
72
+
73
+ ## Releasing
74
+
75
+ 1. Bump version
76
+ 2. `make build`
77
+ 3. `make push`
78
+ 4. `make docker-build`
79
+ 5. `make docker-push`
@@ -16,12 +16,7 @@ module RubyEventStore
16
16
 
17
17
  private
18
18
 
19
- def rails_version
20
- Rails::VERSION::STRING
21
- end
22
-
23
19
  def migration_version
24
- return nil if Gem::Version.new(rails_version) < Gem::Version.new("5.0.0")
25
20
  "[4.2]"
26
21
  end
27
22
 
@@ -0,0 +1,19 @@
1
+ module RubyEventStore
2
+ module Outbox
3
+ module CleanupStrategies
4
+ class CleanOldEnqueued
5
+ def initialize(repository, duration)
6
+ @repository = repository
7
+ @duration = duration
8
+ end
9
+
10
+ def call(fetch_specification)
11
+ repository.delete_enqueued_older_than(fetch_specification, duration)
12
+ end
13
+
14
+ private
15
+ attr_reader :repository, :duration
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,13 @@
1
+ module RubyEventStore
2
+ module Outbox
3
+ module CleanupStrategies
4
+ class None
5
+ def initialize
6
+ end
7
+
8
+ def call(_fetch_specification)
9
+ end
10
+ end
11
+ end
12
+ end
13
+ end
@@ -1,47 +1,66 @@
1
1
  require "optparse"
2
- require "ruby_event_store/outbox/version"
3
- require "ruby_event_store/outbox/consumer"
4
- require "ruby_event_store/outbox/metrics"
2
+ require_relative "version"
3
+ require_relative "consumer"
4
+ require_relative "metrics"
5
5
 
6
6
  module RubyEventStore
7
7
  module Outbox
8
8
  class CLI
9
- Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format, :batch_size, :metrics_url)
9
+ DEFAULTS = {
10
+ database_url: nil,
11
+ redis_url: nil,
12
+ log_level: :warn,
13
+ split_keys: nil,
14
+ message_format: 'sidekiq5',
15
+ batch_size: 100,
16
+ metrics_url: nil,
17
+ cleanup_strategy: :none,
18
+ sleep_on_empty: 0.5
19
+ }
20
+ Options = Struct.new(*DEFAULTS.keys)
10
21
 
11
22
  class Parser
12
23
  def self.parse(argv)
13
- options = Options.new(nil, nil, :warn, nil, nil, 100)
24
+ options = Options.new(*DEFAULTS.values)
14
25
  OptionParser.new do |option_parser|
15
26
  option_parser.banner = "Usage: res_outbox [options]"
16
27
 
17
- option_parser.on("--database-url DATABASE_URL", "Database where outbox table is stored") do |database_url|
28
+ option_parser.on("--database-url=DATABASE_URL", "Database where outbox table is stored") do |database_url|
18
29
  options.database_url = database_url
19
30
  end
20
31
 
21
- option_parser.on("--redis-url REDIS_URL", "URL to redis database") do |redis_url|
32
+ option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
22
33
  options.redis_url = redis_url
23
34
  end
24
35
 
25
- option_parser.on("--log-level LOG_LEVEL", [:fatal, :error, :warn, :info, :debug], "Logging level, one of: fatal, error, warn, info, debug") do |log_level|
36
+ option_parser.on("--log-level=LOG_LEVEL", [:fatal, :error, :warn, :info, :debug], "Logging level, one of: fatal, error, warn, info, debug. Default: warn") do |log_level|
26
37
  options.log_level = log_level.to_sym
27
38
  end
28
39
 
29
- option_parser.on("--message-format FORMAT", ["sidekiq5"], "Message format, supported: sidekiq5") do |message_format|
40
+ option_parser.on("--message-format=FORMAT", ["sidekiq5"], "Message format, supported: sidekiq5. Default: sidekiq5") do |message_format|
30
41
  options.message_format = message_format
31
42
  end
32
43
 
33
- option_parser.on("--split-keys=split_keys", Array, "Split keys which should be handled, all if not specified") do |split_keys|
44
+ option_parser.on("--split-keys=SPLIT_KEYS", Array, "Split keys which should be handled, all if not specified") do |split_keys|
34
45
  options.split_keys = split_keys if !split_keys.empty?
35
46
  end
36
47
 
37
- option_parser.on("--batch-size BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur.") do |batch_size|
48
+ option_parser.on("--batch-size=BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100") do |batch_size|
38
49
  options.batch_size = batch_size
39
50
  end
40
51
 
41
- option_parser.on("--metrics-url METRICS_URL", "URI to metrics collector") do |metrics_url|
52
+ option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
42
53
  options.metrics_url = metrics_url
43
54
  end
44
55
 
56
+ option_parser.on("--cleanup=STRATEGY", "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none") do |cleanup_strategy|
57
+ options.cleanup_strategy = cleanup_strategy
58
+ end
59
+
60
+ option_parser.on("--sleep-on-empty=SLEEP_TIME", Float, "How long to sleep before next check when there was nothing to do. Default: 0.5") do |sleep_on_empty|
61
+ options.sleep_on_empty = sleep_on_empty
62
+ end
63
+
45
64
  option_parser.on_tail("--version", "Show version") do
46
65
  puts VERSION
47
66
  exit
@@ -67,11 +86,13 @@ module RubyEventStore
67
86
  batch_size: options.batch_size,
68
87
  database_url: options.database_url,
69
88
  redis_url: options.redis_url,
89
+ cleanup: options.cleanup_strategy,
90
+ sleep_on_empty: options.sleep_on_empty,
70
91
  )
71
92
  metrics = Metrics.from_url(options.metrics_url)
72
93
  outbox_consumer = RubyEventStore::Outbox::Consumer.new(
73
94
  consumer_uuid,
74
- options,
95
+ consumer_configuration,
75
96
  logger: logger,
76
97
  metrics: metrics,
77
98
  )
@@ -1,15 +1,16 @@
1
1
  require "logger"
2
2
  require "redis"
3
3
  require "active_record"
4
- require "ruby_event_store/outbox/record"
5
- require "ruby_event_store/outbox/sidekiq5_format"
6
- require "ruby_event_store/outbox/sidekiq_processor"
7
- require "ruby_event_store/outbox/fetch_specification"
4
+ require_relative "repository"
5
+ require_relative "sidekiq5_format"
6
+ require_relative "sidekiq_processor"
7
+ require_relative "fetch_specification"
8
+ require_relative "cleanup_strategies/none"
9
+ require_relative "cleanup_strategies/clean_old_enqueued"
8
10
 
9
11
  module RubyEventStore
10
12
  module Outbox
11
13
  class Consumer
12
- SLEEP_TIME_WHEN_NOTHING_TO_DO = 0.1
13
14
  MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK = 10
14
15
 
15
16
  class Configuration
@@ -18,13 +19,17 @@ module RubyEventStore
18
19
  message_format:,
19
20
  batch_size:,
20
21
  database_url:,
21
- redis_url:
22
+ redis_url:,
23
+ cleanup:,
24
+ sleep_on_empty:
22
25
  )
23
26
  @split_keys = split_keys
24
27
  @message_format = message_format
25
28
  @batch_size = batch_size || 100
26
29
  @database_url = database_url
27
30
  @redis_url = redis_url
31
+ @cleanup = cleanup
32
+ @sleep_on_empty = sleep_on_empty
28
33
  freeze
29
34
  end
30
35
 
@@ -35,10 +40,12 @@ module RubyEventStore
35
40
  batch_size: overriden_options.fetch(:batch_size, batch_size),
36
41
  database_url: overriden_options.fetch(:database_url, database_url),
37
42
  redis_url: overriden_options.fetch(:redis_url, redis_url),
43
+ cleanup: overriden_options.fetch(:cleanup, cleanup),
44
+ sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
38
45
  )
39
46
  end
40
47
 
41
- attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url
48
+ attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url, :cleanup, :sleep_on_empty
42
49
  end
43
50
 
44
51
  def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
@@ -47,17 +54,22 @@ module RubyEventStore
47
54
  @logger = logger
48
55
  @metrics = metrics
49
56
  @batch_size = configuration.batch_size
57
+ @sleep_on_empty = configuration.sleep_on_empty
50
58
  @consumer_uuid = consumer_uuid
51
- ActiveRecord::Base.establish_connection(configuration.database_url) unless ActiveRecord::Base.connected?
52
- if ActiveRecord::Base.connection.adapter_name == "Mysql2"
53
- ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
54
- end
55
59
 
56
60
  raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
57
61
  @processor = SidekiqProcessor.new(Redis.new(url: configuration.redis_url))
58
62
 
59
63
  @gracefully_shutting_down = false
60
64
  prepare_traps
65
+
66
+ @repository = Repository.new(configuration.database_url)
67
+ @cleanup_strategy = case configuration.cleanup
68
+ when :none
69
+ CleanupStrategies::None.new
70
+ else
71
+ CleanupStrategies::CleanOldEnqueued.new(repository, ActiveSupport::Duration.parse(configuration.cleanup))
72
+ end
61
73
  end
62
74
 
63
75
  def init
@@ -70,7 +82,7 @@ module RubyEventStore
70
82
  was_something_changed = one_loop
71
83
  if !was_something_changed
72
84
  STDOUT.flush
73
- sleep SLEEP_TIME_WHEN_NOTHING_TO_DO
85
+ sleep sleep_on_empty
74
86
  end
75
87
  end
76
88
  logger.info "Gracefully shutting down"
@@ -105,7 +117,7 @@ module RubyEventStore
105
117
  now = @clock.now.utc
106
118
  processor.process(record, now)
107
119
 
108
- record.update_column(:enqueued_at, now)
120
+ repository.mark_as_enqueued(record, now)
109
121
  something_processed |= true
110
122
  updated_record_ids << record.id
111
123
  rescue => e
@@ -124,8 +136,8 @@ module RubyEventStore
124
136
 
125
137
  logger.info "Sent #{updated_record_ids.size} messages from outbox table"
126
138
 
127
- obtained_lock = refresh_lock_for_process(obtained_lock)
128
- break unless obtained_lock
139
+ refresh_successful = refresh_lock_for_process(obtained_lock)
140
+ break unless refresh_successful
129
141
  end
130
142
 
131
143
  metrics.write_point_queue(
@@ -136,16 +148,18 @@ module RubyEventStore
136
148
 
137
149
  release_lock_for_process(fetch_specification)
138
150
 
151
+ cleanup_strategy.call(fetch_specification)
152
+
139
153
  processor.after_batch
140
154
 
141
155
  something_processed
142
156
  end
143
157
 
144
158
  private
145
- attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid
159
+ attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid, :repository, :cleanup_strategy, :sleep_on_empty
146
160
 
147
161
  def obtain_lock_for_process(fetch_specification)
148
- result = Lock.obtain(fetch_specification, consumer_uuid, clock: @clock)
162
+ result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
149
163
  case result
150
164
  when :deadlocked
151
165
  logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
@@ -165,7 +179,7 @@ module RubyEventStore
165
179
  end
166
180
 
167
181
  def release_lock_for_process(fetch_specification)
168
- result = Lock.release(fetch_specification, consumer_uuid)
182
+ result = repository.release_lock_for_process(fetch_specification, consumer_uuid)
169
183
  case result
170
184
  when :ok
171
185
  when :deadlocked
@@ -185,6 +199,8 @@ module RubyEventStore
185
199
  def refresh_lock_for_process(lock)
186
200
  result = lock.refresh(clock: @clock)
187
201
  case result
202
+ when :ok
203
+ return true
188
204
  when :deadlocked
189
205
  logger.warn "Refreshing lock for split_key '#{lock.split_key}' failed (deadlock)"
190
206
  metrics.write_operation_result("refresh", "deadlocked")
@@ -198,7 +214,7 @@ module RubyEventStore
198
214
  metrics.write_operation_result("refresh", "stolen")
199
215
  return false
200
216
  else
201
- return result
217
+ raise "Unexpected result #{result}"
202
218
  end
203
219
  end
204
220
 
@@ -216,11 +232,11 @@ module RubyEventStore
216
232
  end
217
233
 
218
234
  def retrieve_batch(fetch_specification)
219
- Record.remaining_for(fetch_specification).order("id ASC").limit(batch_size).to_a
235
+ repository.retrieve_batch(fetch_specification, batch_size)
220
236
  end
221
237
 
222
238
  def get_remaining_count(fetch_specification)
223
- Record.remaining_for(fetch_specification).count
239
+ repository.get_remaining_count(fetch_specification)
224
240
  end
225
241
  end
226
242
  end
@@ -8,6 +8,24 @@ module RubyEventStore
8
8
  end
9
9
 
10
10
  attr_reader :message_format, :split_key
11
+
12
+ def ==(other)
13
+ other.instance_of?(self.class) &&
14
+ other.message_format.eql?(message_format) &&
15
+ other.split_key.eql?(split_key)
16
+ end
17
+
18
+ BIG_VALUE = 0b111111100100010000010010110010101011011101110101001100100110000
19
+
20
+ def hash
21
+ [
22
+ self.class,
23
+ message_format,
24
+ split_key,
25
+ ].hash ^ BIG_VALUE
26
+ end
27
+
28
+ alias_method :eql?, :==
11
29
  end
12
30
  end
13
31
  end
@@ -6,14 +6,11 @@ module RubyEventStore
6
6
  class Influx
7
7
  def initialize(url)
8
8
  uri = URI.parse(url)
9
- params = CGI.parse(uri.query || "")
10
9
  options = {
11
10
  url: url,
12
11
  async: true,
13
12
  time_precision: 'ns',
14
13
  }
15
- options[:username] = params.fetch("username").first if params.key?("username")
16
- options[:password] = params.fetch("password").first if params.key?("password")
17
14
  @influxdb_client = InfluxDB::Client.new(**options)
18
15
  end
19
16
 
@@ -3,10 +3,10 @@ module RubyEventStore
3
3
  module Metrics
4
4
  def self.from_url(metrics_url)
5
5
  if metrics_url.nil?
6
- require "ruby_event_store/outbox/metrics/null"
6
+ require_relative "metrics/null"
7
7
  Null.new
8
8
  else
9
- require "ruby_event_store/outbox/metrics/influx"
9
+ require_relative "metrics/influx"
10
10
  Influx.new(metrics_url)
11
11
  end
12
12
  end
@@ -0,0 +1,153 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'active_record'
4
+ require 'active_support/core_ext/numeric/time.rb'
5
+
6
+ module RubyEventStore
7
+ module Outbox
8
+ class Repository
9
+ RECENTLY_LOCKED_DURATION = 10.minutes
10
+
11
+ class Record < ::ActiveRecord::Base
12
+ self.primary_key = :id
13
+ self.table_name = 'event_store_outbox'
14
+
15
+ def self.remaining_for(fetch_specification)
16
+ where(format: fetch_specification.message_format, split_key: fetch_specification.split_key, enqueued_at: nil)
17
+ end
18
+
19
+ def self.for_fetch_specification(fetch_specification)
20
+ where(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
21
+ end
22
+
23
+ def hash_payload
24
+ JSON.parse(payload).deep_symbolize_keys
25
+ end
26
+
27
+ def enqueued?
28
+ !enqueued_at.nil?
29
+ end
30
+ end
31
+
32
+ class Lock < ::ActiveRecord::Base
33
+ self.table_name = 'event_store_outbox_locks'
34
+
35
+ def self.obtain(fetch_specification, process_uuid, clock:)
36
+ transaction do
37
+ l = get_lock_record(fetch_specification)
38
+
39
+ if l.recently_locked?
40
+ :taken
41
+ else
42
+ l.update!(
43
+ locked_by: process_uuid,
44
+ locked_at: clock.now,
45
+ )
46
+ l
47
+ end
48
+ end
49
+ rescue ActiveRecord::Deadlocked
50
+ :deadlocked
51
+ rescue ActiveRecord::LockWaitTimeout
52
+ :lock_timeout
53
+ end
54
+
55
+ def refresh(clock:)
56
+ transaction do
57
+ current_process_uuid = locked_by
58
+ lock!
59
+ if locked_by == current_process_uuid
60
+ update!(locked_at: clock.now)
61
+ :ok
62
+ else
63
+ :stolen
64
+ end
65
+ end
66
+ rescue ActiveRecord::Deadlocked
67
+ :deadlocked
68
+ rescue ActiveRecord::LockWaitTimeout
69
+ :lock_timeout
70
+ end
71
+
72
+ def self.release(fetch_specification, process_uuid)
73
+ transaction do
74
+ l = get_lock_record(fetch_specification)
75
+ if !l.locked_by?(process_uuid)
76
+ :not_taken_by_this_process
77
+ else
78
+ l.update!(locked_by: nil, locked_at: nil)
79
+ :ok
80
+ end
81
+ end
82
+ rescue ActiveRecord::Deadlocked
83
+ :deadlocked
84
+ rescue ActiveRecord::LockWaitTimeout
85
+ :lock_timeout
86
+ end
87
+
88
+ def locked_by?(process_uuid)
89
+ locked_by.eql?(process_uuid)
90
+ end
91
+
92
+ def recently_locked?
93
+ locked_by && locked_at > RECENTLY_LOCKED_DURATION.ago
94
+ end
95
+
96
+ def fetch_specification
97
+ FetchSpecification.new(format, split_key)
98
+ end
99
+
100
+ private
101
+ def self.lock_for_split_key(fetch_specification)
102
+ lock.find_by(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
103
+ end
104
+
105
+ def self.get_lock_record(fetch_specification)
106
+ l = lock_for_split_key(fetch_specification)
107
+ if l.nil?
108
+ begin
109
+ l = create!(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
110
+ rescue ActiveRecord::RecordNotUnique
111
+ l = lock_for_split_key(fetch_specification)
112
+ end
113
+ end
114
+ l
115
+ end
116
+ end
117
+
118
+ def initialize(database_url)
119
+ ActiveRecord::Base.establish_connection(database_url) unless ActiveRecord::Base.connected?
120
+ if ActiveRecord::Base.connection.adapter_name == "Mysql2"
121
+ ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
122
+ end
123
+ end
124
+
125
+ def retrieve_batch(fetch_specification, batch_size)
126
+ Record.remaining_for(fetch_specification).order("id ASC").limit(batch_size).to_a
127
+ end
128
+
129
+ def get_remaining_count(fetch_specification)
130
+ Record.remaining_for(fetch_specification).count
131
+ end
132
+
133
+ def obtain_lock_for_process(fetch_specification, process_uuid, clock:)
134
+ Lock.obtain(fetch_specification, process_uuid, clock: clock)
135
+ end
136
+
137
+ def release_lock_for_process(fetch_specification, process_uuid)
138
+ Lock.release(fetch_specification, process_uuid)
139
+ end
140
+
141
+ def mark_as_enqueued(record, now)
142
+ record.update_column(:enqueued_at, now)
143
+ end
144
+
145
+ def delete_enqueued_older_than(fetch_specification, duration)
146
+ Record
147
+ .for_fetch_specification(fetch_specification)
148
+ .where("enqueued_at < ?", duration.ago)
149
+ .delete_all
150
+ end
151
+ end
152
+ end
153
+ end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "ruby_event_store/outbox/sidekiq5_format"
3
+ require_relative "sidekiq5_format"
4
4
 
5
5
  module RubyEventStore
6
6
  module Outbox
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'sidekiq'
4
- require "ruby_event_store/outbox/sidekiq5_format"
4
+ require_relative "repository"
5
5
 
6
6
  module RubyEventStore
7
7
  module Outbox
@@ -15,13 +15,16 @@ module RubyEventStore
15
15
  normalized_item = sidekiq_client.__send__(:normalize_item, item)
16
16
  payload = sidekiq_client.__send__(:process_single, normalized_item.fetch('class'), normalized_item)
17
17
  if payload
18
- Record.create!(
18
+ Repository::Record.create!(
19
19
  format: SIDEKIQ5_FORMAT,
20
20
  split_key: payload.fetch('queue'),
21
21
  payload: payload.to_json
22
22
  )
23
23
  end
24
24
  end
25
+
26
+ private
27
+ attr_reader :repository
25
28
  end
26
29
  end
27
30
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "ruby_event_store/outbox/sidekiq_producer"
3
+ require_relative "sidekiq_producer"
4
4
 
5
5
  module RubyEventStore
6
6
  module Outbox
@@ -9,8 +9,8 @@ module RubyEventStore
9
9
  @sidekiq_producer = SidekiqProducer.new
10
10
  end
11
11
 
12
- def call(klass, serialized_event)
13
- sidekiq_producer.call(klass, [serialized_event.to_h])
12
+ def call(klass, serialized_record)
13
+ sidekiq_producer.call(klass, [serialized_record.to_h])
14
14
  end
15
15
 
16
16
  def verify(subscriber)
@@ -2,6 +2,6 @@
2
2
 
3
3
  module RubyEventStore
4
4
  module Outbox
5
- VERSION = "0.0.13"
5
+ VERSION = "0.0.17"
6
6
  end
7
7
  end
@@ -6,7 +6,7 @@ module RubyEventStore
6
6
  end
7
7
 
8
8
  require_relative 'outbox/fetch_specification'
9
- require_relative 'outbox/record'
9
+ require_relative 'outbox/repository'
10
10
  require_relative 'outbox/sidekiq_scheduler'
11
11
  require_relative 'outbox/consumer'
12
12
  require_relative 'outbox/version'
metadata CHANGED
@@ -1,27 +1,27 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ruby_event_store-outbox
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.13
4
+ version: 0.0.17
5
5
  platform: ruby
6
6
  authors:
7
7
  - Arkency
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-08-30 00:00:00.000000000 Z
11
+ date: 2021-09-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ruby_event_store
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - '='
17
+ - - ">="
18
18
  - !ruby/object:Gem::Version
19
19
  version: 1.0.0
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - '='
24
+ - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: 1.0.0
27
27
  - !ruby/object:Gem::Dependency
@@ -30,37 +30,37 @@ dependencies:
30
30
  requirements:
31
31
  - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: '3.0'
33
+ version: '5.2'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: '3.0'
40
+ version: '5.2'
41
41
  description:
42
- email:
43
- - dev@arkency.com
42
+ email: dev@arkency.com
44
43
  executables:
45
44
  - res_outbox
46
45
  extensions: []
47
- extra_rdoc_files: []
46
+ extra_rdoc_files:
47
+ - README.md
48
48
  files:
49
49
  - README.md
50
50
  - bin/res_outbox
51
51
  - lib/generators/ruby_event_store/outbox/migration_generator.rb
52
52
  - lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.rb
53
53
  - lib/ruby_event_store/outbox.rb
54
+ - lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb
55
+ - lib/ruby_event_store/outbox/cleanup_strategies/none.rb
54
56
  - lib/ruby_event_store/outbox/cli.rb
55
57
  - lib/ruby_event_store/outbox/consumer.rb
56
- - lib/ruby_event_store/outbox/consumer_process.rb
57
58
  - lib/ruby_event_store/outbox/fetch_specification.rb
58
59
  - lib/ruby_event_store/outbox/metrics.rb
59
60
  - lib/ruby_event_store/outbox/metrics/influx.rb
60
61
  - lib/ruby_event_store/outbox/metrics/null.rb
61
- - lib/ruby_event_store/outbox/record.rb
62
+ - lib/ruby_event_store/outbox/repository.rb
62
63
  - lib/ruby_event_store/outbox/sidekiq5_format.rb
63
- - lib/ruby_event_store/outbox/sidekiq_message_handler.rb
64
64
  - lib/ruby_event_store/outbox/sidekiq_processor.rb
65
65
  - lib/ruby_event_store/outbox/sidekiq_producer.rb
66
66
  - lib/ruby_event_store/outbox/sidekiq_scheduler.rb
@@ -69,8 +69,7 @@ homepage: https://railseventstore.org
69
69
  licenses:
70
70
  - MIT
71
71
  metadata:
72
- homepage_uri: https://railseventstore.org/
73
- changelog_uri: https://github.com/RailsEventStore/rails_event_store/releases
72
+ homepage_uri: https://railseventstore.org
74
73
  source_code_uri: https://github.com/RailsEventStore/rails_event_store
75
74
  bug_tracker_uri: https://github.com/RailsEventStore/rails_event_store/issues
76
75
  post_install_message:
@@ -81,14 +80,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
81
80
  requirements:
82
81
  - - ">="
83
82
  - !ruby/object:Gem::Version
84
- version: '0'
83
+ version: '2.6'
85
84
  required_rubygems_version: !ruby/object:Gem::Requirement
86
85
  requirements:
87
86
  - - ">="
88
87
  - !ruby/object:Gem::Version
89
88
  version: '0'
90
89
  requirements: []
91
- rubygems_version: 3.0.3
90
+ rubygems_version: 3.0.6
92
91
  signing_key:
93
92
  specification_version: 4
94
93
  summary: Active Record based outbox for Ruby Event Store
@@ -1,6 +0,0 @@
1
- module RubyEventStore
2
- module Outbox
3
- class ConsumerProcess
4
- end
5
- end
6
- end
@@ -1,104 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'active_record'
4
-
5
- module RubyEventStore
6
- module Outbox
7
- class Record < ::ActiveRecord::Base
8
- self.primary_key = :id
9
- self.table_name = 'event_store_outbox'
10
-
11
- def self.remaining_for(fetch_specification)
12
- where(format: fetch_specification.message_format, split_key: fetch_specification.split_key, enqueued_at: nil)
13
- end
14
-
15
- def hash_payload
16
- JSON.parse(payload).deep_symbolize_keys
17
- end
18
-
19
- def enqueued?
20
- !enqueued_at.nil?
21
- end
22
- end
23
-
24
- class Lock < ::ActiveRecord::Base
25
- self.primary_key = :split_key
26
- self.table_name = 'event_store_outbox_locks'
27
-
28
- def self.obtain(fetch_specification, process_uuid, clock:)
29
- l = nil
30
- transaction do
31
- l = get_lock_record(fetch_specification)
32
-
33
- return :taken if l.recently_locked?
34
-
35
- l.update!(
36
- locked_by: process_uuid,
37
- locked_at: clock.now,
38
- )
39
- end
40
- l
41
- rescue ActiveRecord::Deadlocked
42
- :deadlocked
43
- rescue ActiveRecord::LockWaitTimeout
44
- :lock_timeout
45
- end
46
-
47
- def refresh(clock:)
48
- transaction do
49
- current_process_uuid = locked_by
50
- lock!
51
- if locked_by == current_process_uuid
52
- update!(locked_at: clock.now)
53
- return self
54
- else
55
- return :stolen
56
- end
57
- end
58
- rescue ActiveRecord::Deadlocked
59
- :deadlocked
60
- rescue ActiveRecord::LockWaitTimeout
61
- :lock_timeout
62
- end
63
-
64
- def self.release(fetch_specification, process_uuid)
65
- transaction do
66
- l = get_lock_record(fetch_specification)
67
- return :not_taken_by_this_process if !l.locked_by?(process_uuid)
68
-
69
- l.update!(locked_by: nil, locked_at: nil)
70
- end
71
- :ok
72
- rescue ActiveRecord::Deadlocked
73
- :deadlocked
74
- rescue ActiveRecord::LockWaitTimeout
75
- :lock_timeout
76
- end
77
-
78
- def locked_by?(process_uuid)
79
- locked_by.eql?(process_uuid)
80
- end
81
-
82
- def recently_locked?
83
- locked_by && locked_at > 10.minutes.ago
84
- end
85
-
86
- private
87
- def self.lock_for_split_key(fetch_specification)
88
- lock.find_by(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
89
- end
90
-
91
- def self.get_lock_record(fetch_specification)
92
- l = lock_for_split_key(fetch_specification)
93
- if l.nil?
94
- begin
95
- l = create!(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
96
- rescue ActiveRecord::RecordNotUnique
97
- l = lock_for_split_key(fetch_specification)
98
- end
99
- end
100
- l
101
- end
102
- end
103
- end
104
- end
@@ -1,19 +0,0 @@
1
- require "sidekiq"
2
-
3
- module RubyEventStore
4
- module Outbox
5
- class SidekiqMessageHandler
6
- def initialize
7
- @sidekiq = Sidekiq::Client.new(Sidekiq.redis_pool)
8
- end
9
-
10
- def init
11
- end
12
-
13
- def handle_one_record(now, record)
14
- hash_payload = JSON.parse(record.payload)
15
- @sidekiq.__send__(:raw_push, [hash_payload])
16
- end
17
- end
18
- end
19
- end