ruby_event_store-outbox 0.0.4 → 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: db60ea5281225f61b150abd5c7e6cc8e7379f09f20c58ed5f10b234534e43612
4
- data.tar.gz: 6a4d1015845ce306d2d9663d3b70fe284fa75ea783960e3c345d5d315d79cc61
3
+ metadata.gz: da07b2263a96dbbfaad959ab5dd5636d5bb3398d810894080ab6e3f80750c29f
4
+ data.tar.gz: 53cd5a2de183f9c821eacf6c05689f42e1fdc39120aac25aa450f90601bbd964
5
5
  SHA512:
6
- metadata.gz: f3629cebca8b80cf62979c5a1cdcb27e53c63552cbdc73e3b0296d8a4fed1b2715e9a8de5f2eafa69fd3f68a3cb2b77aa2e6209be26f3fa724fc70a59575031a
7
- data.tar.gz: 52d9e07cad234988c58453a700fe80a9325e22d68265821afd66ced82d2cbae27d2e0ab0d22b14b761a9faf908e49e766dedb99eca82aa825712e3362b5181a6
6
+ metadata.gz: d6e7cb309a64e4743ae5e6d182987c7b1d35378b0a371ce784c2f0695de026903a92d58106e98fdbab73687bf0c35f89912a614982f8347e5b4d0f0ab0daabf4
7
+ data.tar.gz: 1d7fe76b631564b4205dab9f928da9487e6db08cf93cee9b52a975b7cad65dddd07de97babd586851855c262f1428f902a3612185b9bc0fc7f3a4fa848c0024c
@@ -1,15 +1,16 @@
1
1
  require "optparse"
2
2
  require "ruby_event_store/outbox/version"
3
3
  require "ruby_event_store/outbox/consumer"
4
+ require "ruby_event_store/outbox/metrics"
4
5
 
5
6
  module RubyEventStore
6
7
  module Outbox
7
8
  class CLI
8
- Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format)
9
+ Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format, :batch_size, :metrics_url)
9
10
 
10
11
  class Parser
11
12
  def self.parse(argv)
12
- options = Options.new(nil, nil, :warn, nil, nil)
13
+ options = Options.new(nil, nil, :warn, nil, nil, 100)
13
14
  OptionParser.new do |option_parser|
14
15
  option_parser.banner = "Usage: res_outbox [options]"
15
16
 
@@ -33,6 +34,14 @@ module RubyEventStore
33
34
  options.split_keys = split_keys if !split_keys.empty?
34
35
  end
35
36
 
37
+ option_parser.on("--batch-size BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur.") do |batch_size|
38
+ options.batch_size = batch_size
39
+ end
40
+
41
+ option_parser.on("--metrics-url METRICS_URL", "URI to metrics collector") do |metrics_url|
42
+ options.metrics_url = metrics_url
43
+ end
44
+
36
45
  option_parser.on_tail("--version", "Show version") do
37
46
  puts VERSION
38
47
  exit
@@ -44,16 +53,26 @@ module RubyEventStore
44
53
 
45
54
  def run(argv)
46
55
  options = Parser.parse(argv)
56
+ outbox_consumer = build_consumer(options)
57
+ outbox_consumer.init
58
+ outbox_consumer.run
59
+ end
60
+
61
+ def build_consumer(options)
47
62
  logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox")
48
- outbox_consumer = RubyEventStore::Outbox::Consumer.new(
49
- options.message_format,
50
- options.split_keys,
63
+ consumer_configuration = Consumer::Configuration.new(
64
+ split_keys: options.split_keys,
65
+ message_format: options.message_format,
66
+ batch_size: options.batch_size,
51
67
  database_url: options.database_url,
52
68
  redis_url: options.redis_url,
69
+ )
70
+ metrics = Metrics.from_url(options.metrics_url)
71
+ outbox_consumer = RubyEventStore::Outbox::Consumer.new(
72
+ options,
53
73
  logger: logger,
74
+ metrics: metrics,
54
75
  )
55
- outbox_consumer.init
56
- outbox_consumer.run
57
76
  end
58
77
  end
59
78
  end
@@ -9,14 +9,49 @@ module RubyEventStore
9
9
  class Consumer
10
10
  SLEEP_TIME_WHEN_NOTHING_TO_DO = 0.1
11
11
 
12
- def initialize(format, split_keys, database_url:, redis_url:, clock: Time, logger:)
13
- @split_keys = split_keys
12
+ class Configuration
13
+ def initialize(
14
+ split_keys:,
15
+ message_format:,
16
+ batch_size:,
17
+ database_url:,
18
+ redis_url:
19
+ )
20
+ @split_keys = split_keys
21
+ @message_format = message_format
22
+ @batch_size = batch_size || 100
23
+ @database_url = database_url
24
+ @redis_url = redis_url
25
+ freeze
26
+ end
27
+
28
+ def with(overriden_options)
29
+ self.class.new(
30
+ split_keys: overriden_options.fetch(:split_keys, split_keys),
31
+ message_format: overriden_options.fetch(:message_format, message_format),
32
+ batch_size: overriden_options.fetch(:batch_size, batch_size),
33
+ database_url: overriden_options.fetch(:database_url, database_url),
34
+ redis_url: overriden_options.fetch(:redis_url, redis_url),
35
+ )
36
+ end
37
+
38
+ attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url
39
+ end
40
+
41
+ def initialize(configuration, clock: Time, logger:, metrics:)
42
+ @split_keys = configuration.split_keys
14
43
  @clock = clock
15
- @redis = Redis.new(url: redis_url)
44
+ @redis = Redis.new(url: configuration.redis_url)
16
45
  @logger = logger
17
- ActiveRecord::Base.establish_connection(database_url) unless ActiveRecord::Base.connected?
46
+ @metrics = metrics
47
+ @batch_size = configuration.batch_size
48
+ ActiveRecord::Base.establish_connection(configuration.database_url) unless ActiveRecord::Base.connected?
49
+ if ActiveRecord::Base.connection.adapter_name == "Mysql2"
50
+ ActiveRecord::Base.connection.execute("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED;")
51
+ ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
52
+ end
18
53
 
19
- raise "Unknown format" if format != SIDEKIQ5_FORMAT
54
+ raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
20
55
  @message_format = SIDEKIQ5_FORMAT
21
56
 
22
57
  @gracefully_shutting_down = false
@@ -44,8 +79,11 @@ module RubyEventStore
44
79
  Record.transaction do
45
80
  records_scope = Record.lock.where(format: message_format, enqueued_at: nil)
46
81
  records_scope = records_scope.where(split_key: split_keys) if !split_keys.nil?
47
- records = records_scope.order("id ASC").limit(100).to_a
48
- return false if records.empty?
82
+ records = records_scope.order("id ASC").limit(batch_size).to_a
83
+ if records.empty?
84
+ metrics.write_point_queue(status: "ok")
85
+ return false
86
+ end
49
87
 
50
88
  now = @clock.now.utc
51
89
  failed_record_ids = []
@@ -58,18 +96,25 @@ module RubyEventStore
58
96
  end
59
97
  end
60
98
 
61
- Record.where(id: records.map(&:id) - failed_record_ids).update_all(enqueued_at: now)
99
+ updated_record_ids = records.map(&:id) - failed_record_ids
100
+ Record.where(id: updated_record_ids).update_all(enqueued_at: now)
101
+ metrics.write_point_queue(status: "ok", enqueued: updated_record_ids.size, failed: failed_record_ids.size)
62
102
 
63
103
  logger.info "Sent #{records.size} messages from outbox table"
64
- return true
104
+ true
65
105
  end
66
106
  rescue ActiveRecord::Deadlocked
67
107
  logger.warn "Outbox fetch deadlocked"
108
+ metrics.write_point_queue(status: "deadlocked")
109
+ false
110
+ rescue ActiveRecord::LockWaitTimeout
111
+ logger.warn "Outbox fetch lock timeout"
112
+ metrics.write_point_queue(status: "lock_timeout")
68
113
  false
69
114
  end
70
115
 
71
116
  private
72
- attr_reader :split_keys, :logger, :message_format
117
+ attr_reader :split_keys, :logger, :message_format, :batch_size, :metrics
73
118
 
74
119
  def handle_one_record(now, record)
75
120
  hash_payload = JSON.parse(record.payload)
@@ -0,0 +1,16 @@
1
+ require "ruby_event_store/outbox/metrics/null"
2
+ require "ruby_event_store/outbox/metrics/influx"
3
+
4
+ module RubyEventStore
5
+ module Outbox
6
+ module Metrics
7
+ def self.from_url(metrics_url)
8
+ if metrics_url.nil?
9
+ Null.new
10
+ else
11
+ Influx.new(metrics_url)
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,41 @@
1
+ require 'influxdb'
2
+
3
+ module RubyEventStore
4
+ module Outbox
5
+ module Metrics
6
+ class Influx
7
+ def initialize(url)
8
+ uri = URI.parse(url)
9
+ params = CGI.parse(uri.query || "")
10
+ options = {
11
+ url: url,
12
+ async: true,
13
+ time_precision: 'ns',
14
+ }
15
+ options[:username] = params.fetch("username").first if params.key?("username")
16
+ options[:password] = params.fetch("password").first if params.key?("password")
17
+ @influxdb_client = InfluxDB::Client.new(**options)
18
+ end
19
+
20
+ def write_point_queue(status:, enqueued: 0, failed: 0)
21
+ write_point("ruby_event_store.outbox.queue", {
22
+ values: {
23
+ enqueued: enqueued,
24
+ failed: failed,
25
+ },
26
+ tags: {
27
+ status: status,
28
+ }
29
+ })
30
+ end
31
+
32
+ def write_point(series, data)
33
+ data[:timestamp] = Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond)
34
+ influxdb_client.write_point(series, data)
35
+ end
36
+
37
+ attr_reader :influxdb_client
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,10 @@
1
+ module RubyEventStore
2
+ module Outbox
3
+ module Metrics
4
+ class Null
5
+ def write_point_queue(**kwargs)
6
+ end
7
+ end
8
+ end
9
+ end
10
+ end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module RubyEventStore
4
4
  module Outbox
5
- VERSION = "0.0.4"
5
+ VERSION = "0.0.9"
6
6
  end
7
7
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ruby_event_store-outbox
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.0.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - Arkency
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-06-24 00:00:00.000000000 Z
11
+ date: 2020-06-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ruby_event_store
@@ -38,6 +38,20 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '3.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: influxdb
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: 0.8.0
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: 0.8.0
41
55
  description:
42
56
  email:
43
57
  - dev@arkency.com
@@ -53,6 +67,9 @@ files:
53
67
  - lib/ruby_event_store/outbox.rb
54
68
  - lib/ruby_event_store/outbox/cli.rb
55
69
  - lib/ruby_event_store/outbox/consumer.rb
70
+ - lib/ruby_event_store/outbox/metrics.rb
71
+ - lib/ruby_event_store/outbox/metrics/influx.rb
72
+ - lib/ruby_event_store/outbox/metrics/null.rb
56
73
  - lib/ruby_event_store/outbox/record.rb
57
74
  - lib/ruby_event_store/outbox/sidekiq5_format.rb
58
75
  - lib/ruby_event_store/outbox/sidekiq_message_handler.rb