ruby_event_store-outbox 0.0.15 → 0.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4c373da7338563c51f8bc7d3bae7e56ceb99f9b75d8104196d003f5316cdb6a
-  data.tar.gz: fbb6dad9bdaee2fea028a2cea8541eb6a9f964e1d10d4e704ebe3f715a88d623
+  metadata.gz: '008b96c4b43e3cbfc23d0aa8c9ebb160f0adbfcfc82a976967f1c56338346cc0'
+  data.tar.gz: e65157c0638ecec0a4967a251e820979d18eb92e55212f107be95c23d5a4315b
 SHA512:
-  metadata.gz: e6f7ac3498d139a98ef14b8cbb7abe7bd4af8b794b41f90c6b65150523410a25a59ff6e863301d87f3a90bb5be80cc422d7ae94dd8a16fedd3b88a1db334eee0
-  data.tar.gz: 1c30a1e0c60bee39d1064e0aa7c535416a3cca7d4aa25328ab684741bf4a6bc65373e412388e65d5e57183a4dbf25e4fff385c456cd562c1afc2fe9e04e1d5d8
+  metadata.gz: f0181744b8c621bc1f0b20d6d4a65ac779db7ba48a80c9f1a02f39208762aa82e5579e745f20cd35f2b54992876065f803e04e24c1c239abe22641f927327070
+  data.tar.gz: 453ce565584436b1b7dd145c738a0d50e95821cb8b8f5d9c00f0d70d2a8ea88e585402de534388459d04853d293c0bcbf7027285d31bbd00aeaef09471cf666d

data/README.md

@@ -1,7 +1,10 @@
 # Ruby Event Store Outbox
 
-Very much work in progress.
+![Ruby Event Store Outbox](https://github.com/RailsEventStore/rails_event_store/workflows/ruby_event_store-outbox/badge.svg)
 
+**Experimental feature of RES ecosystem.**
+
+This repository includes a process and a Rails Event Store scheduler, which can be used to transactionally enqueue background jobs into your background jobs tool of choice. The scheduler included in this repo adds the jobs into the RDBMS into specific table instead of redis inside your transaction, and the process is enqueuing the jobs from that table to the background jobs tool.
 
 ## Installation (app)
 
@@ -35,7 +38,21 @@ end
 Run following process in any way you prefer:
 
 ```
-res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" --redis-url="redis://localhost:6379/0" --log-level=info
+res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" --redis-url="redis://localhost:6379/0" --log-level=info --split-keys=sidekiq_queue1,sidekiq_queue2
+```
+
+It is possible to run as many instances as you prefer, but it does not make sense to run more instances than there are different split keys (sidekiq queues), as one process is operating at one moment only one split key.
+
+### Metrics
+
+It is possible for the outbox process to send metrics to InfluxDB. In order to do that, specify a `--metrics-url` parameter, for example:
+
+```
+res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" \
+  --redis-url="redis://localhost:6379/0" \
+  --log-level=info \
+  --split-keys=sidekiq_queue1,sidekiq_queue2 \
+  --metrics-url=http://user:password@localhost:8086/dbname"
 ```
 
 

data/lib/generators/ruby_event_store/outbox/migration_generator.rb

@@ -16,12 +16,7 @@ module RubyEventStore
 
       private
 
-      def rails_version
-        Rails::VERSION::STRING
-      end
-
       def migration_version
-        return nil if Gem::Version.new(rails_version) < Gem::Version.new("5.0.0")
         "[4.2]"
       end
 

data/lib/ruby_event_store/outbox.rb

@@ -6,7 +6,7 @@ module RubyEventStore
 end
 
 require_relative 'outbox/fetch_specification'
-require_relative 'outbox/record'
+require_relative 'outbox/repository'
 require_relative 'outbox/sidekiq_scheduler'
 require_relative 'outbox/consumer'
 require_relative 'outbox/version'

data/lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb

@@ -0,0 +1,19 @@
+module RubyEventStore
+  module Outbox
+    module CleanupStrategies
+      class CleanOldEnqueued
+        def initialize(repository, duration)
+          @repository = repository
+          @duration = duration
+        end
+
+        def call(fetch_specification)
+          repository.delete_enqueued_older_than(fetch_specification, duration)
+        end
+
+        private
+        attr_reader :repository, :duration
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/cleanup_strategies/none.rb

@@ -0,0 +1,13 @@
+module RubyEventStore
+  module Outbox
+    module CleanupStrategies
+      class None
+        def initialize
+        end
+
+        def call(_fetch_specification)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/cli.rb

@@ -6,11 +6,11 @@ require "ruby_event_store/outbox/metrics"
 module RubyEventStore
   module Outbox
     class CLI
-      Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format, :batch_size, :metrics_url)
+      Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format, :batch_size, :metrics_url, :cleanup_strategy)
 
       class Parser
         def self.parse(argv)
-          options = Options.new(nil, nil, :warn, nil, nil, 100)
+          options = Options.new(nil, nil, :warn, nil, nil, 100, nil, :none)
           OptionParser.new do |option_parser|
             option_parser.banner = "Usage: res_outbox [options]"
 
@@ -42,6 +42,10 @@ module RubyEventStore
               options.metrics_url = metrics_url
             end
 
+            option_parser.on("--cleanup CLEANUP_STRATEGY", "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed") do |cleanup_strategy|
+              options.cleanup_strategy = cleanup_strategy
+            end
+
             option_parser.on_tail("--version", "Show version") do
               puts VERSION
               exit
@@ -67,11 +71,12 @@ module RubyEventStore
           batch_size: options.batch_size,
           database_url: options.database_url,
           redis_url: options.redis_url,
+          cleanup: options.cleanup_strategy,
         )
         metrics = Metrics.from_url(options.metrics_url)
         outbox_consumer = RubyEventStore::Outbox::Consumer.new(
           consumer_uuid,
-          options,
+          consumer_configuration,
           logger: logger,
           metrics: metrics,
         )

data/lib/ruby_event_store/outbox/consumer.rb

@@ -1,10 +1,12 @@
 require "logger"
 require "redis"
 require "active_record"
-require "ruby_event_store/outbox/record"
+require "ruby_event_store/outbox/repository"
 require "ruby_event_store/outbox/sidekiq5_format"
 require "ruby_event_store/outbox/sidekiq_processor"
 require "ruby_event_store/outbox/fetch_specification"
+require "ruby_event_store/outbox/cleanup_strategies/none"
+require "ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued"
 
 module RubyEventStore
   module Outbox
@@ -18,13 +20,15 @@ module RubyEventStore
           message_format:,
           batch_size:,
           database_url:,
-          redis_url:
+          redis_url:,
+          cleanup:
         )
           @split_keys = split_keys
           @message_format = message_format
           @batch_size = batch_size || 100
           @database_url = database_url
           @redis_url = redis_url
+          @cleanup = cleanup
           freeze
         end
 
@@ -35,10 +39,11 @@ module RubyEventStore
             batch_size: overriden_options.fetch(:batch_size, batch_size),
             database_url: overriden_options.fetch(:database_url, database_url),
             redis_url: overriden_options.fetch(:redis_url, redis_url),
+            cleanup: overriden_options.fetch(:cleanup, cleanup)
           )
         end
 
-        attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url
+        attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url, :cleanup
       end
 
       def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
@@ -48,16 +53,20 @@ module RubyEventStore
         @metrics = metrics
         @batch_size = configuration.batch_size
         @consumer_uuid = consumer_uuid
-        ActiveRecord::Base.establish_connection(configuration.database_url) unless ActiveRecord::Base.connected?
-        if ActiveRecord::Base.connection.adapter_name == "Mysql2"
-          ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
-        end
 
         raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
         @processor = SidekiqProcessor.new(Redis.new(url: configuration.redis_url))
 
         @gracefully_shutting_down = false
         prepare_traps
+
+        @repository = Repository.new(configuration.database_url)
+        @cleanup_strategy = case configuration.cleanup
+        when :none
+          CleanupStrategies::None.new
+        else
+          CleanupStrategies::CleanOldEnqueued.new(repository, ActiveSupport::Duration.parse(configuration.cleanup))
+        end
       end
 
       def init
@@ -105,7 +114,7 @@ module RubyEventStore
               now = @clock.now.utc
               processor.process(record, now)
 
-              record.update_column(:enqueued_at, now)
+              repository.mark_as_enqueued(record, now)
               something_processed |= true
               updated_record_ids << record.id
             rescue => e
@@ -124,8 +133,8 @@ module RubyEventStore
 
           logger.info "Sent #{updated_record_ids.size} messages from outbox table"
 
-          obtained_lock = refresh_lock_for_process(obtained_lock)
-          break unless obtained_lock
+          refresh_successful = refresh_lock_for_process(obtained_lock)
+          break unless refresh_successful
         end
 
         metrics.write_point_queue(
@@ -136,16 +145,18 @@ module RubyEventStore
 
         release_lock_for_process(fetch_specification)
 
+        cleanup_strategy.call(fetch_specification)
+
         processor.after_batch
 
         something_processed
       end
 
       private
-      attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid
+      attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid, :repository, :cleanup_strategy
 
       def obtain_lock_for_process(fetch_specification)
-        result = Lock.obtain(fetch_specification, consumer_uuid, clock: @clock)
+        result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
         case result
         when :deadlocked
           logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
@@ -165,7 +176,7 @@ module RubyEventStore
       end
 
       def release_lock_for_process(fetch_specification)
-        result = Lock.release(fetch_specification, consumer_uuid)
+        result = repository.release_lock_for_process(fetch_specification, consumer_uuid)
         case result
         when :ok
         when :deadlocked
@@ -185,6 +196,8 @@ module RubyEventStore
       def refresh_lock_for_process(lock)
         result = lock.refresh(clock: @clock)
         case result
+        when :ok
+          return true
         when :deadlocked
           logger.warn "Refreshing lock for split_key '#{lock.split_key}' failed (deadlock)"
           metrics.write_operation_result("refresh", "deadlocked")
@@ -198,7 +211,7 @@ module RubyEventStore
           metrics.write_operation_result("refresh", "stolen")
           return false
         else
-          return result
+          raise "Unexpected result #{result}"
         end
       end
 
@@ -216,11 +229,11 @@ module RubyEventStore
       end
 
       def retrieve_batch(fetch_specification)
-        Record.remaining_for(fetch_specification).order("id ASC").limit(batch_size).to_a
+        repository.retrieve_batch(fetch_specification, batch_size)
       end
 
       def get_remaining_count(fetch_specification)
-        Record.remaining_for(fetch_specification).count
+        repository.get_remaining_count(fetch_specification)
       end
     end
   end

data/lib/ruby_event_store/outbox/fetch_specification.rb

@@ -8,6 +8,24 @@ module RubyEventStore
       end
 
       attr_reader :message_format, :split_key
+
+      def ==(other)
+        other.instance_of?(self.class) &&
+          other.message_format.eql?(message_format) &&
+          other.split_key.eql?(split_key)
+      end
+
+      BIG_VALUE = 0b111111100100010000010010110010101011011101110101001100100110000
+
+      def hash
+        [
+          self.class,
+          message_format,
+          split_key,
+        ].hash ^ BIG_VALUE
+      end
+
+      alias_method :eql?, :==
     end
   end
 end

data/lib/ruby_event_store/outbox/repository.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require 'active_support/core_ext/numeric/time.rb'
+
+module RubyEventStore
+  module Outbox
+    class Repository
+      RECENTLY_LOCKED_DURATION = 10.minutes
+
+      class Record < ::ActiveRecord::Base
+        self.primary_key = :id
+        self.table_name = 'event_store_outbox'
+
+        def self.remaining_for(fetch_specification)
+          where(format: fetch_specification.message_format, split_key: fetch_specification.split_key, enqueued_at: nil)
+        end
+
+        def self.for_fetch_specification(fetch_specification)
+          where(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
+        end
+
+        def hash_payload
+          JSON.parse(payload).deep_symbolize_keys
+        end
+
+        def enqueued?
+          !enqueued_at.nil?
+        end
+      end
+
+      class Lock < ::ActiveRecord::Base
+        self.table_name = 'event_store_outbox_locks'
+
+        def self.obtain(fetch_specification, process_uuid, clock:)
+          transaction do
+            l = get_lock_record(fetch_specification)
+
+            if l.recently_locked?
+              :taken
+            else
+              l.update!(
+                locked_by: process_uuid,
+                locked_at: clock.now,
+              )
+              l
+            end
+          end
+        rescue ActiveRecord::Deadlocked
+          :deadlocked
+        rescue ActiveRecord::LockWaitTimeout
+          :lock_timeout
+        end
+
+        def refresh(clock:)
+          transaction do
+            current_process_uuid = locked_by
+            lock!
+            if locked_by == current_process_uuid
+              update!(locked_at: clock.now)
+              :ok
+            else
+              :stolen
+            end
+          end
+        rescue ActiveRecord::Deadlocked
+          :deadlocked
+        rescue ActiveRecord::LockWaitTimeout
+          :lock_timeout
+        end
+
+        def self.release(fetch_specification, process_uuid)
+          transaction do
+            l = get_lock_record(fetch_specification)
+            if !l.locked_by?(process_uuid)
+              :not_taken_by_this_process
+            else
+              l.update!(locked_by: nil, locked_at: nil)
+              :ok
+            end
+          end
+        rescue ActiveRecord::Deadlocked
+          :deadlocked
+        rescue ActiveRecord::LockWaitTimeout
+          :lock_timeout
+        end
+
+        def locked_by?(process_uuid)
+          locked_by.eql?(process_uuid)
+        end
+
+        def recently_locked?
+          locked_by && locked_at > RECENTLY_LOCKED_DURATION.ago
+        end
+
+        def fetch_specification
+          FetchSpecification.new(format, split_key)
+        end
+
+        private
+        def self.lock_for_split_key(fetch_specification)
+          lock.find_by(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
+        end
+
+        def self.get_lock_record(fetch_specification)
+          l = lock_for_split_key(fetch_specification)
+          if l.nil?
+            begin
+              l = create!(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
+            rescue ActiveRecord::RecordNotUnique
+              l = lock_for_split_key(fetch_specification)
+            end
+          end
+          l
+        end
+      end
+
+      def initialize(database_url)
+        ActiveRecord::Base.establish_connection(database_url) unless ActiveRecord::Base.connected?
+        if ActiveRecord::Base.connection.adapter_name == "Mysql2"
+          ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
+        end
+      end
+
+      def retrieve_batch(fetch_specification, batch_size)
+        Record.remaining_for(fetch_specification).order("id ASC").limit(batch_size).to_a
+      end
+
+      def get_remaining_count(fetch_specification)
+        Record.remaining_for(fetch_specification).count
+      end
+
+      def obtain_lock_for_process(fetch_specification, process_uuid, clock:)
+        Lock.obtain(fetch_specification, process_uuid, clock: clock)
+      end
+
+      def release_lock_for_process(fetch_specification, process_uuid)
+        Lock.release(fetch_specification, process_uuid)
+      end
+
+      def mark_as_enqueued(record, now)
+        record.update_column(:enqueued_at, now)
+      end
+
+      def delete_enqueued_older_than(fetch_specification, duration)
+        Record
+          .for_fetch_specification(fetch_specification)
+          .where("enqueued_at < ?", duration.ago)
+          .delete_all
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/sidekiq_producer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'sidekiq'
-require "ruby_event_store/outbox/sidekiq5_format"
+require "ruby_event_store/outbox/repository"
 
 module RubyEventStore
   module Outbox
@@ -15,13 +15,16 @@ module RubyEventStore
         normalized_item = sidekiq_client.__send__(:normalize_item, item)
         payload = sidekiq_client.__send__(:process_single, normalized_item.fetch('class'), normalized_item)
         if payload
-          Record.create!(
+          Repository::Record.create!(
             format: SIDEKIQ5_FORMAT,
             split_key: payload.fetch('queue'),
             payload: payload.to_json
           )
         end
       end
+
+      private
+      attr_reader :repository
     end
   end
 end

data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb

@@ -9,8 +9,8 @@ module RubyEventStore
         @sidekiq_producer = SidekiqProducer.new
       end
 
-      def call(klass, serialized_event)
-        sidekiq_producer.call(klass, [serialized_event.to_h])
+      def call(klass, serialized_record)
+        sidekiq_producer.call(klass, [serialized_record.to_h])
       end
 
       def verify(subscriber)

data/lib/ruby_event_store/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module RubyEventStore
   module Outbox
-    VERSION = "0.0.15"
+    VERSION = "0.0.16"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_event_store-outbox
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.16
 platform: ruby
 authors:
 - Arkency
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-30 00:00:00.000000000 Z
+date: 2021-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby_event_store
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '5.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '5.2'
 description: 
 email:
 - dev@arkency.com
@@ -51,6 +51,8 @@ files:
 - lib/generators/ruby_event_store/outbox/migration_generator.rb
 - lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.rb
 - lib/ruby_event_store/outbox.rb
+- lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb
+- lib/ruby_event_store/outbox/cleanup_strategies/none.rb
 - lib/ruby_event_store/outbox/cli.rb
 - lib/ruby_event_store/outbox/consumer.rb
 - lib/ruby_event_store/outbox/consumer_process.rb
@@ -58,7 +60,7 @@ files:
 - lib/ruby_event_store/outbox/metrics.rb
 - lib/ruby_event_store/outbox/metrics/influx.rb
 - lib/ruby_event_store/outbox/metrics/null.rb
-- lib/ruby_event_store/outbox/record.rb
+- lib/ruby_event_store/outbox/repository.rb
 - lib/ruby_event_store/outbox/sidekiq5_format.rb
 - lib/ruby_event_store/outbox/sidekiq_message_handler.rb
 - lib/ruby_event_store/outbox/sidekiq_processor.rb

data/lib/ruby_event_store/outbox/record.rb

@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-require 'active_record'
-
-module RubyEventStore
-  module Outbox
-    class Record < ::ActiveRecord::Base
-      self.primary_key = :id
-      self.table_name = 'event_store_outbox'
-
-      def self.remaining_for(fetch_specification)
-        where(format: fetch_specification.message_format, split_key: fetch_specification.split_key, enqueued_at: nil)
-      end
-
-      def hash_payload
-        JSON.parse(payload).deep_symbolize_keys
-      end
-
-      def enqueued?
-        !enqueued_at.nil?
-      end
-    end
-
-    class Lock < ::ActiveRecord::Base
-      self.table_name = 'event_store_outbox_locks'
-
-      def self.obtain(fetch_specification, process_uuid, clock:)
-        l = nil
-        transaction do
-          l = get_lock_record(fetch_specification)
-
-          return :taken if l.recently_locked?
-
-          l.update!(
-            locked_by: process_uuid,
-            locked_at: clock.now,
-          )
-        end
-        l
-      rescue ActiveRecord::Deadlocked
-        :deadlocked
-      rescue ActiveRecord::LockWaitTimeout
-        :lock_timeout
-      end
-
-      def refresh(clock:)
-        transaction do
-          current_process_uuid = locked_by
-          lock!
-          if locked_by == current_process_uuid
-            update!(locked_at: clock.now)
-            return self
-          else
-            return :stolen
-          end
-        end
-      rescue ActiveRecord::Deadlocked
-        :deadlocked
-      rescue ActiveRecord::LockWaitTimeout
-        :lock_timeout
-      end
-
-      def self.release(fetch_specification, process_uuid)
-        transaction do
-          l = get_lock_record(fetch_specification)
-          return :not_taken_by_this_process if !l.locked_by?(process_uuid)
-
-          l.update!(locked_by: nil, locked_at: nil)
-        end
-        :ok
-      rescue ActiveRecord::Deadlocked
-        :deadlocked
-      rescue ActiveRecord::LockWaitTimeout
-        :lock_timeout
-      end
-
-      def locked_by?(process_uuid)
-        locked_by.eql?(process_uuid)
-      end
-
-      def recently_locked?
-        locked_by && locked_at > 10.minutes.ago
-      end
-
-      private
-      def self.lock_for_split_key(fetch_specification)
-        lock.find_by(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
-      end
-
-      def self.get_lock_record(fetch_specification)
-        l = lock_for_split_key(fetch_specification)
-        if l.nil?
-          begin
-            l = create!(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
-          rescue ActiveRecord::RecordNotUnique
-            l = lock_for_split_key(fetch_specification)
-          end
-        end
-        l
-      end
-    end
-  end
-end