ruby_event_store-outbox 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db60ea5281225f61b150abd5c7e6cc8e7379f09f20c58ed5f10b234534e43612
-  data.tar.gz: 6a4d1015845ce306d2d9663d3b70fe284fa75ea783960e3c345d5d315d79cc61
+  metadata.gz: 4a53d0609414ba7c53da1a8602042de223631437392a9a29b0e3dbb3d2015a61
+  data.tar.gz: fa4dde249eecb5701f311d91a772fc12394b5412c6af18649d5ba604b11c9567
 SHA512:
-  metadata.gz: f3629cebca8b80cf62979c5a1cdcb27e53c63552cbdc73e3b0296d8a4fed1b2715e9a8de5f2eafa69fd3f68a3cb2b77aa2e6209be26f3fa724fc70a59575031a
-  data.tar.gz: 52d9e07cad234988c58453a700fe80a9325e22d68265821afd66ced82d2cbae27d2e0ab0d22b14b761a9faf908e49e766dedb99eca82aa825712e3362b5181a6
+  metadata.gz: f8d236f0e1cb2236f3740e7f92ea19662df5af2350e729266c50635208e3d714ccef6dfdd1f9080ac9237467a88be351e07f656cd5ae2361223f4eb6c96fb3cc
+  data.tar.gz: eb6718caa012a1d6f5ed52374108c0a74f2f7faf73ddb8208d17c2f0bdb636712ee30249865b8eb39d410c1f273721e0495fa53d2b3f32d06a8f2be98eb4459f

data/lib/ruby_event_store/outbox/cli.rb

@@ -1,15 +1,16 @@
 require "optparse"
 require "ruby_event_store/outbox/version"
 require "ruby_event_store/outbox/consumer"
+require "ruby_event_store/outbox/metrics"
 
 module RubyEventStore
   module Outbox
     class CLI
-      Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format)
+      Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format, :batch_size, :metrics_url)
 
       class Parser
         def self.parse(argv)
-          options = Options.new(nil, nil, :warn, nil, nil)
+          options = Options.new(nil, nil, :warn, nil, nil, 100)
           OptionParser.new do |option_parser|
             option_parser.banner = "Usage: res_outbox [options]"
 
@@ -33,6 +34,14 @@ module RubyEventStore
               options.split_keys = split_keys if !split_keys.empty?
             end
 
+            option_parser.on("--batch-size BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur.") do |batch_size|
+              options.batch_size = batch_size
+            end
+
+            option_parser.on("--metrics-url METRICS_URL", "URI to metrics collector") do |metrics_url|
+              options.metrics_url = metrics_url
+            end
+
             option_parser.on_tail("--version", "Show version") do
               puts VERSION
               exit
@@ -44,16 +53,26 @@ module RubyEventStore
 
       def run(argv)
         options = Parser.parse(argv)
+        outbox_consumer = build_consumer(options)
+        outbox_consumer.init
+        outbox_consumer.run
+      end
+
+      def build_consumer(options)
         logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox")
-        outbox_consumer = RubyEventStore::Outbox::Consumer.new(
-          options.message_format,
-          options.split_keys,
+        consumer_configuration = Consumer::Configuration.new(
+          split_keys: options.split_keys,
+          message_format: options.message_format,
+          batch_size: options.batch_size,
           database_url: options.database_url,
           redis_url: options.redis_url,
+        )
+        metrics = Metrics.from_url(options.metrics_url)
+        outbox_consumer = RubyEventStore::Outbox::Consumer.new(
+          options,
           logger: logger,
+          metrics: metrics,
         )
-        outbox_consumer.init
-        outbox_consumer.run
       end
     end
   end

data/lib/ruby_event_store/outbox/consumer.rb

@@ -9,14 +9,45 @@ module RubyEventStore
     class Consumer
       SLEEP_TIME_WHEN_NOTHING_TO_DO = 0.1
 
-      def initialize(format, split_keys, database_url:, redis_url:, clock: Time, logger:)
-        @split_keys = split_keys
+      class Configuration
+        def initialize(
+          split_keys:,
+          message_format:,
+          batch_size:,
+          database_url:,
+          redis_url:
+        )
+          @split_keys = split_keys
+          @message_format = message_format
+          @batch_size = batch_size || 100
+          @database_url = database_url
+          @redis_url = redis_url
+          freeze
+        end
+
+        def with(overriden_options)
+          self.class.new(
+            split_keys: overriden_options.fetch(:split_keys, split_keys),
+            message_format: overriden_options.fetch(:message_format, message_format),
+            batch_size: overriden_options.fetch(:batch_size, batch_size),
+            database_url: overriden_options.fetch(:database_url, database_url),
+            redis_url: overriden_options.fetch(:redis_url, redis_url),
+          )
+        end
+
+        attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url
+      end
+
+      def initialize(configuration, clock: Time, logger:, metrics:)
+        @split_keys = configuration.split_keys
         @clock = clock
-        @redis = Redis.new(url: redis_url)
+        @redis = Redis.new(url: configuration.redis_url)
         @logger = logger
-        ActiveRecord::Base.establish_connection(database_url) unless ActiveRecord::Base.connected?
+        @metrics = metrics
+        @batch_size = configuration.batch_size
+        ActiveRecord::Base.establish_connection(configuration.database_url) unless ActiveRecord::Base.connected?
 
-        raise "Unknown format" if format != SIDEKIQ5_FORMAT
+        raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
         @message_format = SIDEKIQ5_FORMAT
 
         @gracefully_shutting_down = false
@@ -44,8 +75,11 @@ module RubyEventStore
         Record.transaction do
           records_scope = Record.lock.where(format: message_format, enqueued_at: nil)
           records_scope = records_scope.where(split_key: split_keys) if !split_keys.nil?
-          records = records_scope.order("id ASC").limit(100).to_a
-          return false if records.empty?
+          records = records_scope.order("id ASC").limit(batch_size).to_a
+          if records.empty?
+            metrics.write_point_queue(status: "ok")
+            return false
+          end
 
           now = @clock.now.utc
           failed_record_ids = []
@@ -58,18 +92,25 @@ module RubyEventStore
             end
           end
 
-          Record.where(id: records.map(&:id) - failed_record_ids).update_all(enqueued_at: now)
+          updated_record_ids = records.map(&:id) - failed_record_ids
+          Record.where(id: updated_record_ids).update_all(enqueued_at: now)
+          metrics.write_point_queue(status: "ok", enqueued: updated_record_ids.size, failed: failed_record_ids.size)
 
           logger.info "Sent #{records.size} messages from outbox table"
-          return true
+          true
         end
       rescue ActiveRecord::Deadlocked
         logger.warn "Outbox fetch deadlocked"
+        metrics.write_point_queue(status: "deadlocked")
+        false
+      rescue ActiveRecord::LockWaitTimeout
+        logger.warn "Outbox fetch lock timeout"
+        metrics.write_point_queue(status: "lock_timeout")
         false
       end
 
       private
-      attr_reader :split_keys, :logger, :message_format
+      attr_reader :split_keys, :logger, :message_format, :batch_size, :metrics
 
       def handle_one_record(now, record)
         hash_payload = JSON.parse(record.payload)

data/lib/ruby_event_store/outbox/metrics.rb

@@ -0,0 +1,16 @@
+require "ruby_event_store/outbox/metrics/null"
+require "ruby_event_store/outbox/metrics/influx"
+
+module RubyEventStore
+  module Outbox
+    module Metrics
+      def self.from_url(metrics_url)
+        if metrics_url.nil?
+          Null.new
+        else
+          Influx.new(metrics_url)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/metrics/influx.rb

@@ -0,0 +1,33 @@
+require 'influxdb'
+
+module RubyEventStore
+  module Outbox
+    module Metrics
+      class Influx
+        def initialize(url)
+          @influxdb_client = InfluxDB::Client.new(url: url, async: true, time_precision: 'ns')
+        end
+
+        def write_point_queue(status:, enqueued: 0, failed: 0)
+          write_point("ruby_event_store.outbox.queue", {
+            values: {
+              enqueued: enqueued,
+              failed: failed,
+            },
+            tags: {
+              status: status,
+            }
+          })
+        end
+
+        def write_point(series, data)
+          data[:timestamp] ||= Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond).to_i
+          @influxdb_client.write_point(series, data)
+        end
+
+        private
+        attr_reader :influxdb_client
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/metrics/null.rb

@@ -0,0 +1,10 @@
+module RubyEventStore
+  module Outbox
+    module Metrics
+      class Null
+        def write_point_queue(status:, enqueued: 0, failed: 0)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module RubyEventStore
   module Outbox
-    VERSION = "0.0.4"
+    VERSION = "0.0.5"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_event_store-outbox
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Arkency
@@ -53,6 +53,9 @@ files:
 - lib/ruby_event_store/outbox.rb
 - lib/ruby_event_store/outbox/cli.rb
 - lib/ruby_event_store/outbox/consumer.rb
+- lib/ruby_event_store/outbox/metrics.rb
+- lib/ruby_event_store/outbox/metrics/influx.rb
+- lib/ruby_event_store/outbox/metrics/null.rb
 - lib/ruby_event_store/outbox/record.rb
 - lib/ruby_event_store/outbox/sidekiq5_format.rb
 - lib/ruby_event_store/outbox/sidekiq_message_handler.rb