ruby_event_store-outbox 0.0.2 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08a5fc74b45a962065d79e01a6347919882bd97903ed30d5c722069d3e7ee617'
-  data.tar.gz: 4238ad6ce88f09744ee98feb3b513ec6e3903db92579494109f0575098f23be5
+  metadata.gz: c8e2e739e68444fa47e69b09357b61b1d7b7136db13663028b97eaff9c8f3375
+  data.tar.gz: fbf657458847ca98c43698c4b40acea4993e0a9c752f788e112490e190d2415b
 SHA512:
-  metadata.gz: 7939074f5b2fc85d9f4795103e0aaf38a59ca8e10b27f746a265b611fa4eda84a23806b1009845ce918dc61669bce4f5887d71c0139f152e4b04a9e7102a1fd9
-  data.tar.gz: 7c6f7168c15b2c81cc462e63dd528ef1415b690eae8b3f49344bd2fdb8681338721ef2a2c0b511d75c265c1590031d799d268417ab69694714624fe0246e1e8a
+  metadata.gz: a2a08df4a60ba2e339e2914c20fe4a5638f18b20a55a580e2b048758bf273d20fbff1a05fe0eb183d38d625ce6ce94a2e8cc034a1136320ade6bd0e3adeec165
+  data.tar.gz: 8c6114269f96e992a134b5b731a7be3ba870c4a57d75bfc74129cd385b5100c93d450a9574a53fd3d1bf0489801a0839fc9d3fc9e35237d96672dc4c62baf0da

data/lib/ruby_event_store/outbox/cli.rb

@@ -1,15 +1,16 @@
 require "optparse"
 require "ruby_event_store/outbox/version"
 require "ruby_event_store/outbox/consumer"
+require "ruby_event_store/outbox/metrics"
 
 module RubyEventStore
   module Outbox
     class CLI
-      Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format)
+      Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format, :batch_size, :metrics_url)
 
       class Parser
         def self.parse(argv)
-          options = Options.new(nil, nil, :warn, nil, nil)
+          options = Options.new(nil, nil, :warn, nil, nil, 100)
           OptionParser.new do |option_parser|
             option_parser.banner = "Usage: res_outbox [options]"
 
@@ -33,6 +34,14 @@ module RubyEventStore
               options.split_keys = split_keys if !split_keys.empty?
             end
 
+            option_parser.on("--batch-size BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur.") do |batch_size|
+              options.batch_size = batch_size
+            end
+
+            option_parser.on("--metrics-url METRICS_URL", "URI to metrics collector") do |metrics_url|
+              options.metrics_url = metrics_url
+            end
+
             option_parser.on_tail("--version", "Show version") do
               puts VERSION
               exit
@@ -44,16 +53,26 @@ module RubyEventStore
 
       def run(argv)
         options = Parser.parse(argv)
+        outbox_consumer = build_consumer(options)
+        outbox_consumer.init
+        outbox_consumer.run
+      end
+
+      def build_consumer(options)
         logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox")
-        outbox_consumer = RubyEventStore::Outbox::Consumer.new(
-          options.message_format,
-          options.split_keys,
+        consumer_configuration = Consumer::Configuration.new(
+          split_keys: options.split_keys,
+          message_format: options.message_format,
+          batch_size: options.batch_size,
           database_url: options.database_url,
           redis_url: options.redis_url,
+        )
+        metrics = Metrics.from_url(options.metrics_url)
+        outbox_consumer = RubyEventStore::Outbox::Consumer.new(
+          options,
           logger: logger,
+          metrics: metrics,
         )
-        outbox_consumer.init
-        outbox_consumer.run
       end
     end
   end

data/lib/ruby_event_store/outbox/consumer.rb

@@ -9,14 +9,45 @@ module RubyEventStore
     class Consumer
       SLEEP_TIME_WHEN_NOTHING_TO_DO = 0.1
 
-      def initialize(format, split_keys, database_url:, redis_url:, clock: Time, logger:)
-        @split_keys = split_keys
+      class Configuration
+        def initialize(
+          split_keys:,
+          message_format:,
+          batch_size:,
+          database_url:,
+          redis_url:
+        )
+          @split_keys = split_keys
+          @message_format = message_format
+          @batch_size = batch_size || 100
+          @database_url = database_url
+          @redis_url = redis_url
+          freeze
+        end
+
+        def with(overriden_options)
+          self.class.new(
+            split_keys: overriden_options.fetch(:split_keys, split_keys),
+            message_format: overriden_options.fetch(:message_format, message_format),
+            batch_size: overriden_options.fetch(:batch_size, batch_size),
+            database_url: overriden_options.fetch(:database_url, database_url),
+            redis_url: overriden_options.fetch(:redis_url, redis_url),
+          )
+        end
+
+        attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url
+      end
+
+      def initialize(configuration, clock: Time, logger:, metrics:)
+        @split_keys = configuration.split_keys
         @clock = clock
-        @redis = Redis.new(url: redis_url)
+        @redis = Redis.new(url: configuration.redis_url)
         @logger = logger
-        ActiveRecord::Base.establish_connection(database_url) unless ActiveRecord::Base.connected?
+        @metrics = metrics
+        @batch_size = configuration.batch_size
+        ActiveRecord::Base.establish_connection(configuration.database_url) unless ActiveRecord::Base.connected?
 
-        raise "Unknown format" if format != SIDEKIQ5_FORMAT
+        raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
         @message_format = SIDEKIQ5_FORMAT
 
         @gracefully_shutting_down = false
@@ -32,7 +63,10 @@ module RubyEventStore
       def run
         while !@gracefully_shutting_down do
           was_something_changed = one_loop
-          sleep SLEEP_TIME_WHEN_NOTHING_TO_DO if !was_something_changed
+          if !was_something_changed
+            STDOUT.flush
+            sleep SLEEP_TIME_WHEN_NOTHING_TO_DO
+          end
         end
         logger.info "Gracefully shutting down"
       end
@@ -41,8 +75,11 @@ module RubyEventStore
         Record.transaction do
           records_scope = Record.lock.where(format: message_format, enqueued_at: nil)
           records_scope = records_scope.where(split_key: split_keys) if !split_keys.nil?
-          records = records_scope.order("id ASC").limit(100)
-          return false if records.empty?
+          records = records_scope.order("id ASC").limit(batch_size).to_a
+          if records.empty?
+            metrics.write_point_queue(status: "ok")
+            return false
+          end
 
           now = @clock.now.utc
           failed_record_ids = []
@@ -55,20 +92,25 @@ module RubyEventStore
             end
           end
 
-          update_scope = if failed_record_ids.empty?
-            records
-          else
-            records.where("id NOT IN (?)", failed_record_ids)
-          end
-          update_scope.update_all(enqueued_at: now)
+          updated_record_ids = records.map(&:id) - failed_record_ids
+          Record.where(id: updated_record_ids).update_all(enqueued_at: now)
+          metrics.write_point_queue(status: "ok", enqueued: updated_record_ids.size, failed: failed_record_ids.size)
 
           logger.info "Sent #{records.size} messages from outbox table"
-          return true
+          true
         end
+      rescue ActiveRecord::Deadlocked
+        logger.warn "Outbox fetch deadlocked"
+        metrics.write_point_queue(status: "deadlocked")
+        false
+      rescue ActiveRecord::LockWaitTimeout
+        logger.warn "Outbox fetch lock timeout"
+        metrics.write_point_queue(status: "lock_timeout")
+        false
       end
 
       private
-      attr_reader :split_keys, :logger, :message_format
+      attr_reader :split_keys, :logger, :message_format, :batch_size, :metrics
 
       def handle_one_record(now, record)
         hash_payload = JSON.parse(record.payload)

data/lib/ruby_event_store/outbox/metrics.rb

@@ -0,0 +1,16 @@
+require "ruby_event_store/outbox/metrics/null"
+require "ruby_event_store/outbox/metrics/influx"
+
+module RubyEventStore
+  module Outbox
+    module Metrics
+      def self.from_url(metrics_url)
+        if metrics_url.nil?
+          Null.new
+        else
+          Influx.new(metrics_url)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/metrics/influx.rb

@@ -0,0 +1,41 @@
+require 'influxdb'
+
+module RubyEventStore
+  module Outbox
+    module Metrics
+      class Influx
+        def initialize(url)
+          uri = URI.parse(url)
+          params = CGI.parse(uri.query || "")
+          options = {
+            url: url,
+            async: true,
+            time_precision: 'ns',
+          }
+          options[:username] = params["username"]&.first if params["username"].present?
+          options[:password] = params["password"]&.first if params["password"].present?
+          @influxdb_client = InfluxDB::Client.new(**options)
+        end
+
+        def write_point_queue(status:, enqueued: 0, failed: 0)
+          write_point("ruby_event_store.outbox.queue", {
+            values: {
+              enqueued: enqueued,
+              failed: failed,
+            },
+            tags: {
+              status: status,
+            }
+          })
+        end
+
+        def write_point(series, data)
+          data[:timestamp] ||= Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond).to_i
+          @influxdb_client.write_point(series, data)
+        end
+
+        attr_reader :influxdb_client
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/metrics/null.rb

@@ -0,0 +1,10 @@
+module RubyEventStore
+  module Outbox
+    module Metrics
+      class Null
+        def write_point_queue(status:, enqueued: 0, failed: 0)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/sidekiq_message_handler.rb

@@ -0,0 +1,19 @@
+require "sidekiq"
+
+module RubyEventStore
+  module Outbox
+    class SidekiqMessageHandler
+      def initialize
+        @sidekiq = Sidekiq::Client.new(Sidekiq.redis_pool)
+      end
+
+      def init
+      end
+
+      def handle_one_record(now, record)
+        hash_payload = JSON.parse(record.payload)
+        @sidekiq.__send__(:raw_push, [hash_payload])
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module RubyEventStore
   module Outbox
-    VERSION = "0.0.2"
+    VERSION = "0.0.7"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_event_store-outbox
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.7
 platform: ruby
 authors:
 - Arkency
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-12 00:00:00.000000000 Z
+date: 2020-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby_event_store
@@ -53,8 +53,12 @@ files:
 - lib/ruby_event_store/outbox.rb
 - lib/ruby_event_store/outbox/cli.rb
 - lib/ruby_event_store/outbox/consumer.rb
+- lib/ruby_event_store/outbox/metrics.rb
+- lib/ruby_event_store/outbox/metrics/influx.rb
+- lib/ruby_event_store/outbox/metrics/null.rb
 - lib/ruby_event_store/outbox/record.rb
 - lib/ruby_event_store/outbox/sidekiq5_format.rb
+- lib/ruby_event_store/outbox/sidekiq_message_handler.rb
 - lib/ruby_event_store/outbox/sidekiq_scheduler.rb
 - lib/ruby_event_store/outbox/version.rb
 homepage: https://railseventstore.org