ruby_event_store-outbox 0.0.6 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4bb8606ceba1dbd647b6c8716ab8a46fea986f38e8ac62c3fcd52cad1d2d0539
-  data.tar.gz: b7053afc49a662a60e227c74677885e2d3cade60fde340d8373622bed6cb72a1
+  metadata.gz: e6e7bf65a4d5106581d89918d0bb21bf6f19ee4e08a6f525d27665b41c07b3b0
+  data.tar.gz: eb1350270156bfd886bc11b94dfe858beda55692aa9c72832ca23fcadafe5fc0
 SHA512:
-  metadata.gz: a5793ac3db528b2960a33632d726f91410ef7dba24e9ca0b4de3a61dd6fb0a5a6ca6aa07d2462caeeec1538a2de948481717f8e86453c976dc71e75b9697febf
-  data.tar.gz: b03737a37ab5468b64dd6d167fe45116856e1bfcb45a5748cca749fd28806d259f5e51b58ee471d64f1e21449bc756c0fd560913afc7c7e6e96e74e43e2d8295
+  metadata.gz: eda134f0e4a489546c9b05fd541d27c83f24bdac1e60da244398df09e3b913c650180b0c3472d5d6c0afd502870f025de01d0e94389c90cd491c7504e9693697
+  data.tar.gz: c73fcda82e6635636ae1eccbaeb3c52d4b2c438a7a55e075fccc254d18e118b5ea79266b48926074ec38bdf51b2b81cd787c6f7a61bf6facb5086ab1a5dd166d

data/lib/ruby_event_store/outbox/consumer.rb

@@ -46,6 +46,10 @@ module RubyEventStore
         @metrics = metrics
         @batch_size = configuration.batch_size
         ActiveRecord::Base.establish_connection(configuration.database_url) unless ActiveRecord::Base.connected?
+        if ActiveRecord::Base.connection.adapter_name == "Mysql2"
+          ActiveRecord::Base.connection.execute("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED;")
+          ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
+        end
 
         raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
         @message_format = SIDEKIQ5_FORMAT
@@ -83,11 +87,12 @@ module RubyEventStore
 
           now = @clock.now.utc
           failed_record_ids = []
-          records.each do |record|
+          records.group_by(&:split_key).each do |split_key, records2|
             begin
-              handle_one_record(now, record)
+              failed = handle_group_of_records(now, split_key, records2)
+              failed_record_ids.concat(failed.map(&:id))
             rescue => e
-              failed_record_ids << record.id
+              failed_record_ids.concat(records2.map(&:id))
               e.full_message.split($/).each {|line| logger.error(line) }
             end
           end
@@ -96,7 +101,7 @@ module RubyEventStore
           Record.where(id: updated_record_ids).update_all(enqueued_at: now)
           metrics.write_point_queue(status: "ok", enqueued: updated_record_ids.size, failed: failed_record_ids.size)
 
-          logger.info "Sent #{records.size} messages from outbox table"
+          logger.info "Sent #{updated_record_ids.size} messages from outbox table"
           true
         end
       rescue ActiveRecord::Deadlocked
@@ -112,11 +117,21 @@ module RubyEventStore
       private
       attr_reader :split_keys, :logger, :message_format, :batch_size, :metrics
 
-      def handle_one_record(now, record)
-        hash_payload = JSON.parse(record.payload)
-        @redis.lpush("queue:#{hash_payload.fetch("queue")}", JSON.generate(JSON.parse(record.payload).merge({
-          "enqueued_at" => now.to_f,
-        })))
+      def handle_group_of_records(now, split_key, records)
+        failed = []
+        elements = []
+        records.each do |record|
+          begin
+            elements << JSON.generate(JSON.parse(record.payload).merge({
+              "enqueued_at" => now.to_f,
+            }))
+          rescue => e
+            failed << record
+            e.full_message.split($/).each {|line| logger.error(line) }
+          end
+        end
+        @redis.lpush("queue:#{split_key}", elements)
+        failed
       end
 
       def prepare_traps

data/lib/ruby_event_store/outbox/metrics.rb

@@ -1,13 +1,12 @@
-require "ruby_event_store/outbox/metrics/null"
-require "ruby_event_store/outbox/metrics/influx"
-
 module RubyEventStore
   module Outbox
     module Metrics
       def self.from_url(metrics_url)
         if metrics_url.nil?
+          require "ruby_event_store/outbox/metrics/null"
           Null.new
         else
+          require "ruby_event_store/outbox/metrics/influx"
           Influx.new(metrics_url)
         end
       end

data/lib/ruby_event_store/outbox/metrics/influx.rb

@@ -5,7 +5,16 @@ module RubyEventStore
     module Metrics
       class Influx
         def initialize(url)
-          @influxdb_client = InfluxDB::Client.new(url: url, async: true, time_precision: 'ns')
+          uri = URI.parse(url)
+          params = CGI.parse(uri.query || "")
+          options = {
+            url: url,
+            async: true,
+            time_precision: 'ns',
+          }
+          options[:username] = params.fetch("username").first if params.key?("username")
+          options[:password] = params.fetch("password").first if params.key?("password")
+          @influxdb_client = InfluxDB::Client.new(**options)
         end
 
         def write_point_queue(status:, enqueued: 0, failed: 0)
@@ -21,11 +30,10 @@ module RubyEventStore
         end
 
         def write_point(series, data)
-          data[:timestamp] ||= Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond).to_i
-          @influxdb_client.write_point(series, data)
+          data[:timestamp] = Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond)
+          influxdb_client.write_point(series, data)
         end
 
-        private
         attr_reader :influxdb_client
       end
     end

data/lib/ruby_event_store/outbox/metrics/null.rb

@@ -2,7 +2,7 @@ module RubyEventStore
   module Outbox
     module Metrics
       class Null
-        def write_point_queue(status:, enqueued: 0, failed: 0)
+        def write_point_queue(**kwargs)
         end
       end
     end

data/lib/ruby_event_store/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module RubyEventStore
   module Outbox
-    VERSION = "0.0.6"
+    VERSION = "0.0.11"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_event_store-outbox
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.11
 platform: ruby
 authors:
 - Arkency
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-24 00:00:00.000000000 Z
+date: 2020-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby_event_store