ruby_event_store-outbox 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6e7bf65a4d5106581d89918d0bb21bf6f19ee4e08a6f525d27665b41c07b3b0
-  data.tar.gz: eb1350270156bfd886bc11b94dfe858beda55692aa9c72832ca23fcadafe5fc0
+  metadata.gz: c1d9945d7b34ce9254c8150dc671453cdf3c72a5799bc39b11f1e86e109baa8c
+  data.tar.gz: 58fb2365dd712495a8681fb78ed444662422cfddf3b2b165fda5e4299082dc91
 SHA512:
-  metadata.gz: eda134f0e4a489546c9b05fd541d27c83f24bdac1e60da244398df09e3b913c650180b0c3472d5d6c0afd502870f025de01d0e94389c90cd491c7504e9693697
-  data.tar.gz: c73fcda82e6635636ae1eccbaeb3c52d4b2c438a7a55e075fccc254d18e118b5ea79266b48926074ec38bdf51b2b81cd787c6f7a61bf6facb5086ab1a5dd166d
+  metadata.gz: 01fbff71356a4095838f197b96e9673762d02f9677d0d0df3075757861c273131b636b6e181db0ce165fa7d04f2d45b6450e4cd19a49261eb6f9e87d6782b8f7
+  data.tar.gz: 53bea6ab8e0e3c2bfad41018fbc65d7f2de33af168b05d5578cf6349a56614be4c4e00448b3954ed59ca42f637908e635701f5c2eb2ef7d4958f1078c8eac18d

data/lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.rb

@@ -11,5 +11,13 @@ class CreateEventStoreOutbox < ActiveRecord::Migration<%= migration_version %>
     end
     add_index :event_store_outbox, [:format, :enqueued_at, :split_key], name: "index_event_store_outbox_for_pool"
     add_index :event_store_outbox, [:created_at, :enqueued_at], name: "index_event_store_outbox_for_clear"
+
+    create_table(:event_store_outbox_locks, force: false) do |t|
+      t.string :format, null: false
+      t.string :split_key, null: false
+      t.datetime :locked_at, null: true
+      t.string :locked_by, null: true, limit: 36
+    end
+    add_index :event_store_outbox_locks, [:format, :split_key], name: "index_event_store_outbox_locks_for_locking", unique: true
   end
 end

data/lib/ruby_event_store/outbox.rb

@@ -5,6 +5,7 @@ module RubyEventStore
   end
 end
 
+require_relative 'outbox/fetch_specification'
 require_relative 'outbox/record'
 require_relative 'outbox/sidekiq_scheduler'
 require_relative 'outbox/consumer'

data/lib/ruby_event_store/outbox/cli.rb

@@ -59,7 +59,8 @@ module RubyEventStore
       end
 
       def build_consumer(options)
-        logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox")
+        consumer_uuid = SecureRandom.uuid
+        logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox #{consumer_uuid}")
         consumer_configuration = Consumer::Configuration.new(
           split_keys: options.split_keys,
           message_format: options.message_format,
@@ -69,6 +70,7 @@ module RubyEventStore
         )
         metrics = Metrics.from_url(options.metrics_url)
         outbox_consumer = RubyEventStore::Outbox::Consumer.new(
+          consumer_uuid,
           options,
           logger: logger,
           metrics: metrics,

data/lib/ruby_event_store/outbox/consumer.rb

@@ -3,11 +3,13 @@ require "redis"
 require "active_record"
 require "ruby_event_store/outbox/record"
 require "ruby_event_store/outbox/sidekiq5_format"
+require "ruby_event_store/outbox/sidekiq_processor"
 
 module RubyEventStore
   module Outbox
     class Consumer
       SLEEP_TIME_WHEN_NOTHING_TO_DO = 0.1
+      MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK = 10
 
       class Configuration
         def initialize(
@@ -38,28 +40,26 @@ module RubyEventStore
         attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url
       end
 
-      def initialize(configuration, clock: Time, logger:, metrics:)
+      def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
         @split_keys = configuration.split_keys
         @clock = clock
-        @redis = Redis.new(url: configuration.redis_url)
         @logger = logger
         @metrics = metrics
         @batch_size = configuration.batch_size
+        @consumer_uuid = consumer_uuid
         ActiveRecord::Base.establish_connection(configuration.database_url) unless ActiveRecord::Base.connected?
         if ActiveRecord::Base.connection.adapter_name == "Mysql2"
-          ActiveRecord::Base.connection.execute("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED;")
           ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
         end
 
         raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
-        @message_format = SIDEKIQ5_FORMAT
+        @processor = SidekiqProcessor.new(Redis.new(url: configuration.redis_url))
 
         @gracefully_shutting_down = false
         prepare_traps
       end
 
       def init
-        @redis.sadd("queues", split_keys)
         logger.info("Initiated RubyEventStore::Outbox v#{VERSION}")
         logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
       end
@@ -76,62 +76,129 @@ module RubyEventStore
       end
 
       def one_loop
-        Record.transaction do
-          records_scope = Record.lock.where(format: message_format, enqueued_at: nil)
-          records_scope = records_scope.where(split_key: split_keys) if !split_keys.nil?
-          records = records_scope.order("id ASC").limit(batch_size).to_a
-          if records.empty?
-            metrics.write_point_queue(status: "ok")
-            return false
+        remaining_split_keys = @split_keys.dup
+
+        was_something_changed = false
+        while (split_key = remaining_split_keys.shift)
+          was_something_changed |= handle_split(FetchSpecification.new(processor.message_format, split_key))
+        end
+        was_something_changed
+      end
+
+      def handle_split(fetch_specification)
+        obtained_lock = obtain_lock_for_process(fetch_specification)
+        return false unless obtained_lock
+
+        something_processed = false
+
+        MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK.times do
+          batch = retrieve_batch(fetch_specification)
+          if batch.empty?
+            break
           end
 
-          now = @clock.now.utc
           failed_record_ids = []
-          records.group_by(&:split_key).each do |split_key, records2|
+          updated_record_ids = []
+          batch.each do |record|
             begin
-              failed = handle_group_of_records(now, split_key, records2)
-              failed_record_ids.concat(failed.map(&:id))
+              now = @clock.now.utc
+              processor.process(record, now)
+
+              record.update_column(:enqueued_at, now)
+              something_processed |= true
+              updated_record_ids << record.id
             rescue => e
-              failed_record_ids.concat(records2.map(&:id))
+              failed_record_ids << record.id
               e.full_message.split($/).each {|line| logger.error(line) }
             end
           end
 
-          updated_record_ids = records.map(&:id) - failed_record_ids
-          Record.where(id: updated_record_ids).update_all(enqueued_at: now)
-          metrics.write_point_queue(status: "ok", enqueued: updated_record_ids.size, failed: failed_record_ids.size)
+          metrics.write_point_queue(
+            enqueued: updated_record_ids.size,
+            failed: failed_record_ids.size,
+            format: fetch_specification.message_format,
+            split_key: fetch_specification.split_key,
+            remaining: get_remaining_count(fetch_specification)
+          )
 
           logger.info "Sent #{updated_record_ids.size} messages from outbox table"
-          true
+
+          obtained_lock = refresh_lock_for_process(obtained_lock)
+          break unless obtained_lock
         end
-      rescue ActiveRecord::Deadlocked
-        logger.warn "Outbox fetch deadlocked"
-        metrics.write_point_queue(status: "deadlocked")
-        false
-      rescue ActiveRecord::LockWaitTimeout
-        logger.warn "Outbox fetch lock timeout"
-        metrics.write_point_queue(status: "lock_timeout")
-        false
+
+        metrics.write_point_queue(
+          format: fetch_specification.message_format,
+          split_key: fetch_specification.split_key,
+          remaining: get_remaining_count(fetch_specification)
+        ) unless something_processed
+
+        release_lock_for_process(fetch_specification)
+
+        processor.after_batch
+
+        something_processed
       end
 
       private
-      attr_reader :split_keys, :logger, :message_format, :batch_size, :metrics
-
-      def handle_group_of_records(now, split_key, records)
-        failed = []
-        elements = []
-        records.each do |record|
-          begin
-            elements << JSON.generate(JSON.parse(record.payload).merge({
-              "enqueued_at" => now.to_f,
-            }))
-          rescue => e
-            failed << record
-            e.full_message.split($/).each {|line| logger.error(line) }
-          end
+      attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid
+
+      def obtain_lock_for_process(fetch_specification)
+        result = Lock.obtain(fetch_specification, consumer_uuid, clock: @clock)
+        case result
+        when :deadlocked
+          logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
+          metrics.write_operation_result("obtain", "deadlocked")
+          return false
+        when :lock_timeout
+          logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (lock timeout)"
+          metrics.write_operation_result("obtain", "lock_timeout")
+          return false
+        when :taken
+          logger.debug "Obtaining lock for split_key '#{fetch_specification.split_key}' unsuccessful (taken)"
+          metrics.write_operation_result("obtain", "taken")
+          return false
+        else
+          return result
+        end
+      end
+
+      def release_lock_for_process(fetch_specification)
+        result = Lock.release(fetch_specification, consumer_uuid)
+        case result
+        when :ok
+        when :deadlocked
+          logger.warn "Releasing lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
+          metrics.write_operation_result("release", "deadlocked")
+        when :lock_timeout
+          logger.warn "Releasing lock for split_key '#{fetch_specification.split_key}' failed (lock timeout)"
+          metrics.write_operation_result("release", "lock_timeout")
+        when :not_taken_by_this_process
+          logger.debug "Releasing lock for split_key '#{fetch_specification.split_key}' failed (not taken by this process)"
+          metrics.write_operation_result("release", "not_taken_by_this_process")
+        else
+          raise "Unexpected result #{result}"
+        end
+      end
+
+      def refresh_lock_for_process(lock)
+        result = lock.refresh(clock: @clock)
+        case result
+        when :deadlocked
+          logger.warn "Refreshing lock for split_key '#{lock.split_key}' failed (deadlock)"
+          metrics.write_operation_result("refresh", "deadlocked")
+          return false
+        when :lock_timeout
+          logger.warn "Refreshing lock for split_key '#{lock.split_key}' failed (lock timeout)"
+          metrics.write_operation_result("refresh", "lock_timeout")
+          return false
+        when :stolen
+          logger.debug "Refreshing lock for split_key '#{lock.split_key}' unsuccessful (stolen)"
+          metrics.write_operation_result("refresh", "stolen")
+          return false
+        else
+          return result
         end
-        @redis.lpush("queue:#{split_key}", elements)
-        failed
       end
 
       def prepare_traps
@@ -146,6 +213,14 @@ module RubyEventStore
       def initiate_graceful_shutdown
         @gracefully_shutting_down = true
       end
+
+      def retrieve_batch(fetch_specification)
+        Record.remaining_for(fetch_specification).order("id ASC").limit(batch_size).to_a
+      end
+
+      def get_remaining_count(fetch_specification)
+        Record.remaining_for(fetch_specification).count
+      end
     end
   end
 end

data/lib/ruby_event_store/outbox/consumer_process.rb

@@ -0,0 +1,6 @@
+module RubyEventStore
+  module Outbox
+    class ConsumerProcess
+    end
+  end
+end

data/lib/ruby_event_store/outbox/fetch_specification.rb

@@ -0,0 +1,13 @@
+module RubyEventStore
+  module Outbox
+    class FetchSpecification
+      def initialize(message_format, split_key)
+        @message_format = message_format
+        @split_key = split_key
+        freeze
+      end
+
+      attr_reader :message_format, :split_key
+    end
+  end
+end

data/lib/ruby_event_store/outbox/metrics/influx.rb

@@ -17,14 +17,28 @@ module RubyEventStore
           @influxdb_client = InfluxDB::Client.new(**options)
         end
 
-        def write_point_queue(status:, enqueued: 0, failed: 0)
+        def write_operation_result(operation, result)
+          write_point("ruby_event_store.outbox.lock", {
+            values: {
+              value: 1,
+            },
+            tags: {
+              operation: operation,
+              result: result,
+            }
+          })
+        end
+
+        def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
           write_point("ruby_event_store.outbox.queue", {
             values: {
               enqueued: enqueued,
               failed: failed,
+              remaining: remaining,
             },
             tags: {
-              status: status,
+              format: format,
+              split_key: split_key,
             }
           })
         end

data/lib/ruby_event_store/outbox/metrics/null.rb

@@ -2,6 +2,9 @@ module RubyEventStore
   module Outbox
     module Metrics
       class Null
+        def write_operation_result(operation, result)
+        end
+
         def write_point_queue(**kwargs)
         end
       end

data/lib/ruby_event_store/outbox/record.rb

@@ -8,9 +8,97 @@ module RubyEventStore
       self.primary_key = :id
       self.table_name = 'event_store_outbox'
 
+      def self.remaining_for(fetch_specification)
+        where(format: fetch_specification.message_format, split_key: fetch_specification.split_key, enqueued_at: nil)
+      end
+
       def hash_payload
         JSON.parse(payload).deep_symbolize_keys
       end
+
+      def enqueued?
+        !enqueued_at.nil?
+      end
+    end
+
+    class Lock < ::ActiveRecord::Base
+      self.primary_key = :split_key
+      self.table_name = 'event_store_outbox_locks'
+
+      def self.obtain(fetch_specification, process_uuid, clock:)
+        l = nil
+        transaction do
+          l = get_lock_record(fetch_specification)
+
+          return :taken if l.recently_locked?
+
+          l.update!(
+            locked_by: process_uuid,
+            locked_at: clock.now,
+          )
+        end
+        l
+      rescue ActiveRecord::Deadlocked
+        :deadlocked
+      rescue ActiveRecord::LockWaitTimeout
+        :lock_timeout
+      end
+
+      def refresh(clock:)
+        transaction do
+          current_process_uuid = locked_by
+          lock!
+          if locked_by == current_process_uuid
+            update!(locked_at: clock.now)
+            return self
+          else
+            return :stolen
+          end
+        end
+      rescue ActiveRecord::Deadlocked
+        :deadlocked
+      rescue ActiveRecord::LockWaitTimeout
+        :lock_timeout
+      end
+
+      def self.release(fetch_specification, process_uuid)
+        transaction do
+          l = get_lock_record(fetch_specification)
+          return :not_taken_by_this_process if !l.locked_by?(process_uuid)
+
+          l.update!(locked_by: nil, locked_at: nil)
+        end
+        :ok
+      rescue ActiveRecord::Deadlocked
+        :deadlocked
+      rescue ActiveRecord::LockWaitTimeout
+        :lock_timeout
+      end
+
+      def locked_by?(process_uuid)
+        locked_by.eql?(process_uuid)
+      end
+
+      def recently_locked?
+        locked_by && locked_at > 10.minutes.ago
+      end
+
+      private
+      def self.lock_for_split_key(fetch_specification)
+        lock.find_by(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
+      end
+
+      def self.get_lock_record(fetch_specification)
+        l = lock_for_split_key(fetch_specification)
+        if l.nil?
+          begin
+            l = create!(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
+          rescue ActiveRecord::RecordNotUnique
+            l = lock_for_split_key(fetch_specification)
+          end
+        end
+        l
+      end
     end
   end
 end

data/lib/ruby_event_store/outbox/sidekiq_processor.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "ruby_event_store/outbox/sidekiq5_format"
+
+module RubyEventStore
+  module Outbox
+    class SidekiqProcessor
+      InvalidPayload = Class.new(StandardError)
+
+      def initialize(redis)
+        @redis = redis
+        @recently_used_queues = Set.new
+      end
+
+      def process(record, now)
+        parsed_record = JSON.parse(record.payload)
+
+        queue = parsed_record["queue"]
+        raise InvalidPayload.new("Missing queue") if queue.nil? || queue.empty?
+        payload = JSON.generate(parsed_record.merge({
+          "enqueued_at" => now.to_f,
+        }))
+
+        redis.lpush("queue:#{queue}", payload)
+
+        @recently_used_queues << queue
+      end
+
+      def after_batch
+        if !@recently_used_queues.empty?
+          redis.sadd("queues", @recently_used_queues.to_a)
+          @recently_used_queues.clear
+        end
+      end
+
+      def message_format
+        SIDEKIQ5_FORMAT
+      end
+
+      private
+      attr_reader :redis
+    end
+  end
+end

data/lib/ruby_event_store/outbox/sidekiq_producer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+require "ruby_event_store/outbox/sidekiq5_format"
+
+module RubyEventStore
+  module Outbox
+    class SidekiqProducer
+      def call(klass, args)
+        sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
+        item = {
+          'class' => klass,
+          'args' => args,
+        }
+        normalized_item = sidekiq_client.__send__(:normalize_item, item)
+        payload = sidekiq_client.__send__(:process_single, normalized_item.fetch('class'), normalized_item)
+        if payload
+          Record.create!(
+            format: SIDEKIQ5_FORMAT,
+            split_key: payload.fetch('queue'),
+            payload: payload.to_json
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb

@@ -1,31 +1,24 @@
 # frozen_string_literal: true
 
-require 'sidekiq'
-require "ruby_event_store/outbox/sidekiq5_format"
+require "ruby_event_store/outbox/sidekiq_producer"
 
 module RubyEventStore
   module Outbox
     class SidekiqScheduler
+      def initialize
+        @sidekiq_producer = SidekiqProducer.new
+      end
+
       def call(klass, serialized_event)
-        sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
-        item = {
-          'class' => klass,
-          'args' => [serialized_event.to_h],
-        }
-        normalized_item = sidekiq_client.__send__(:normalize_item, item)
-        payload = sidekiq_client.__send__(:process_single, normalized_item.fetch('class'), normalized_item)
-        if payload
-          Record.create!(
-            format: SIDEKIQ5_FORMAT,
-            split_key: payload.fetch('queue'),
-            payload: payload.to_json
-          )
-        end
+        sidekiq_producer.call(klass, [serialized_event.to_h])
       end
 
       def verify(subscriber)
         Class === subscriber && subscriber.respond_to?(:through_outbox?) && subscriber.through_outbox?
       end
+
+      private
+      attr_reader :sidekiq_producer
     end
   end
 end

data/lib/ruby_event_store/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module RubyEventStore
   module Outbox
-    VERSION = "0.0.11"
+    VERSION = "0.0.12"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_event_store-outbox
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12
 platform: ruby
 authors:
 - Arkency
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-26 00:00:00.000000000 Z
+date: 2020-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby_event_store
@@ -53,12 +53,16 @@ files:
 - lib/ruby_event_store/outbox.rb
 - lib/ruby_event_store/outbox/cli.rb
 - lib/ruby_event_store/outbox/consumer.rb
+- lib/ruby_event_store/outbox/consumer_process.rb
+- lib/ruby_event_store/outbox/fetch_specification.rb
 - lib/ruby_event_store/outbox/metrics.rb
 - lib/ruby_event_store/outbox/metrics/influx.rb
 - lib/ruby_event_store/outbox/metrics/null.rb
 - lib/ruby_event_store/outbox/record.rb
 - lib/ruby_event_store/outbox/sidekiq5_format.rb
 - lib/ruby_event_store/outbox/sidekiq_message_handler.rb
+- lib/ruby_event_store/outbox/sidekiq_processor.rb
+- lib/ruby_event_store/outbox/sidekiq_producer.rb
 - lib/ruby_event_store/outbox/sidekiq_scheduler.rb
 - lib/ruby_event_store/outbox/version.rb
 homepage: https://railseventstore.org