ruby_event_store-outbox 0.0.15 → 0.0.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4c373da7338563c51f8bc7d3bae7e56ceb99f9b75d8104196d003f5316cdb6a
-  data.tar.gz: fbb6dad9bdaee2fea028a2cea8541eb6a9f964e1d10d4e704ebe3f715a88d623
+  metadata.gz: b78f20ac9a815274e65c78322746557d961af6b05982d2ebd828f65186002982
+  data.tar.gz: 4a88357afd1a360398d8c1b3fe1fe6081f6e5c05169ba892d16cd701dfd38b53
 SHA512:
-  metadata.gz: e6f7ac3498d139a98ef14b8cbb7abe7bd4af8b794b41f90c6b65150523410a25a59ff6e863301d87f3a90bb5be80cc422d7ae94dd8a16fedd3b88a1db334eee0
-  data.tar.gz: 1c30a1e0c60bee39d1064e0aa7c535416a3cca7d4aa25328ab684741bf4a6bc65373e412388e65d5e57183a4dbf25e4fff385c456cd562c1afc2fe9e04e1d5d8
+  metadata.gz: 3d82d744889b4f595ecfb9c16a6ab57817fd3d652e3c30cbe4f4367bf794de41245b82c35ae02e01e73907ae745f829b1fe3486378a8b0685eaed47d25a359fd
+  data.tar.gz: '058fec9678d6d6db03bc58c1e982ebce51547714dccf657ab5dd2161f2de55af0d07cd44b15e286e9186626c3cb4ad087fd100b118f235fdd7650163ca2a9f68'

data/README.md

@@ -1,7 +1,10 @@
 # Ruby Event Store Outbox
 
-Very much work in progress.
+![Ruby Event Store Outbox](https://github.com/RailsEventStore/rails_event_store/workflows/ruby_event_store-outbox/badge.svg)
 
+**Experimental feature of RES ecosystem.**
+
+This repository includes a process and a Rails Event Store scheduler, which can be used to transactionally enqueue background jobs into your background jobs tool of choice. The scheduler included in this repo adds the jobs into the RDBMS into specific table instead of redis inside your transaction, and the process is enqueuing the jobs from that table to the background jobs tool.
 
 ## Installation (app)
 
@@ -11,6 +14,12 @@ Add to your gemfile in application:
 gem "ruby_event_store-outbox"
 ```
 
+Generate and execute the migration adding necessary tables. If it's needed, change the type of the `payload` column to `mediumbinary` or `longbinary`.
+
+```
+bin/rails generate ruby_event_store:outbox:migration
+```
+
 In your event store configuration, as a dispatcher use `RubyEventStore::ImmediateAsyncDispatcher` with `RubyEventStore::Outbox::SidekiqScheduler`, for example:
 
 ```ruby
@@ -35,10 +44,36 @@ end
 Run following process in any way you prefer:
 
 ```
-res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" --redis-url="redis://localhost:6379/0" --log-level=info
+res_outbox \
+  --database-url="mysql2://root@0.0.0.0:3306/my_database" \
+  --redis-url="redis://localhost:6379/0" \
+  --log-level=info \
+  --split-keys=sidekiq_queue1,sidekiq_queue2
+```
+
+It is possible to run as many instances as you prefer, but it does not make sense to run more instances than there are different split keys (sidekiq queues), as one process is operating at one moment only one split key.
+
+### Metrics
+
+It is possible for the outbox process to send metrics to InfluxDB (this requires influxdb gem in version at least 0.8.1). In order to do that, specify a `--metrics-url` parameter, for example:
+
+```
+res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" \
+  --redis-url="redis://localhost:6379/0" \
+  --log-level=info \
+  --split-keys=sidekiq_queue1,sidekiq_queue2 \
+  --metrics-url=http://user:password@localhost:8086/dbname"
 ```
 
 
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/RailsEventStore/rails_event_store.
+
+## Releasing
+
+1. Bump version
+2. `make build`
+3. `make push`
+4. `make docker-build`
+5. `make docker-push`

data/lib/generators/ruby_event_store/outbox/migration_generator.rb

@@ -16,12 +16,7 @@ module RubyEventStore
 
       private
 
-      def rails_version
-        Rails::VERSION::STRING
-      end
-
       def migration_version
-        return nil if Gem::Version.new(rails_version) < Gem::Version.new("5.0.0")
         "[4.2]"
       end
 

data/lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb

@@ -0,0 +1,19 @@
+module RubyEventStore
+  module Outbox
+    module CleanupStrategies
+      class CleanOldEnqueued
+        def initialize(repository, duration)
+          @repository = repository
+          @duration = duration
+        end
+
+        def call(fetch_specification)
+          repository.delete_enqueued_older_than(fetch_specification, duration)
+        end
+
+        private
+        attr_reader :repository, :duration
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/cleanup_strategies/none.rb

@@ -0,0 +1,14 @@
+module RubyEventStore
+  module Outbox
+    module CleanupStrategies
+      class None
+        def initialize
+        end
+
+        def call(_fetch_specification)
+          :ok
+        end
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/cli.rb

@@ -1,47 +1,66 @@
 require "optparse"
-require "ruby_event_store/outbox/version"
-require "ruby_event_store/outbox/consumer"
-require "ruby_event_store/outbox/metrics"
+require_relative "version"
+require_relative "consumer"
+require_relative "metrics"
 
 module RubyEventStore
   module Outbox
     class CLI
-      Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format, :batch_size, :metrics_url)
+      DEFAULTS = {
+        database_url: nil,
+        redis_url: nil,
+        log_level: :warn,
+        split_keys: nil,
+        message_format: 'sidekiq5',
+        batch_size: 100,
+        metrics_url: nil,
+        cleanup_strategy: :none,
+        sleep_on_empty: 0.5
+      }
+      Options = Struct.new(*DEFAULTS.keys)
 
       class Parser
         def self.parse(argv)
-          options = Options.new(nil, nil, :warn, nil, nil, 100)
+          options = Options.new(*DEFAULTS.values)
           OptionParser.new do |option_parser|
             option_parser.banner = "Usage: res_outbox [options]"
 
-            option_parser.on("--database-url DATABASE_URL", "Database where outbox table is stored") do |database_url|
+            option_parser.on("--database-url=DATABASE_URL", "Database where outbox table is stored") do |database_url|
               options.database_url = database_url
             end
 
-            option_parser.on("--redis-url REDIS_URL", "URL to redis database") do |redis_url|
+            option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
               options.redis_url = redis_url
             end
 
-            option_parser.on("--log-level LOG_LEVEL", [:fatal, :error, :warn, :info, :debug], "Logging level, one of: fatal, error, warn, info, debug") do |log_level|
+            option_parser.on("--log-level=LOG_LEVEL", [:fatal, :error, :warn, :info, :debug], "Logging level, one of: fatal, error, warn, info, debug. Default: warn") do |log_level|
               options.log_level = log_level.to_sym
             end
 
-            option_parser.on("--message-format FORMAT", ["sidekiq5"], "Message format, supported: sidekiq5") do |message_format|
+            option_parser.on("--message-format=FORMAT", ["sidekiq5"], "Message format, supported: sidekiq5. Default: sidekiq5") do |message_format|
               options.message_format = message_format
             end
 
-            option_parser.on("--split-keys=split_keys", Array, "Split keys which should be handled, all if not specified") do |split_keys|
+            option_parser.on("--split-keys=SPLIT_KEYS", Array, "Split keys which should be handled, all if not specified") do |split_keys|
               options.split_keys = split_keys if !split_keys.empty?
             end
 
-            option_parser.on("--batch-size BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur.") do |batch_size|
+            option_parser.on("--batch-size=BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100") do |batch_size|
               options.batch_size = batch_size
             end
 
-            option_parser.on("--metrics-url METRICS_URL", "URI to metrics collector") do |metrics_url|
+            option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
               options.metrics_url = metrics_url
             end
 
+            option_parser.on("--cleanup=STRATEGY", "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none") do |cleanup_strategy|
+              options.cleanup_strategy = cleanup_strategy
+            end
+
+            option_parser.on("--sleep-on-empty=SLEEP_TIME", Float, "How long to sleep before next check when there was nothing to do. Default: 0.5") do |sleep_on_empty|
+              options.sleep_on_empty = sleep_on_empty
+            end
+
             option_parser.on_tail("--version", "Show version") do
               puts VERSION
               exit
@@ -67,11 +86,13 @@ module RubyEventStore
           batch_size: options.batch_size,
           database_url: options.database_url,
           redis_url: options.redis_url,
+          cleanup: options.cleanup_strategy,
+          sleep_on_empty: options.sleep_on_empty,
         )
         metrics = Metrics.from_url(options.metrics_url)
         outbox_consumer = RubyEventStore::Outbox::Consumer.new(
           consumer_uuid,
-          options,
+          consumer_configuration,
           logger: logger,
           metrics: metrics,
         )

data/lib/ruby_event_store/outbox/consumer.rb

@@ -1,15 +1,16 @@
 require "logger"
 require "redis"
 require "active_record"
-require "ruby_event_store/outbox/record"
-require "ruby_event_store/outbox/sidekiq5_format"
-require "ruby_event_store/outbox/sidekiq_processor"
-require "ruby_event_store/outbox/fetch_specification"
+require_relative "repository"
+require_relative "sidekiq5_format"
+require_relative "sidekiq_processor"
+require_relative "fetch_specification"
+require_relative "cleanup_strategies/none"
+require_relative "cleanup_strategies/clean_old_enqueued"
 
 module RubyEventStore
   module Outbox
     class Consumer
-      SLEEP_TIME_WHEN_NOTHING_TO_DO = 0.5
       MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK = 10
 
       class Configuration
@@ -18,13 +19,17 @@ module RubyEventStore
           message_format:,
           batch_size:,
           database_url:,
-          redis_url:
+          redis_url:,
+          cleanup:,
+          sleep_on_empty:
         )
           @split_keys = split_keys
           @message_format = message_format
           @batch_size = batch_size || 100
           @database_url = database_url
           @redis_url = redis_url
+          @cleanup = cleanup
+          @sleep_on_empty = sleep_on_empty
           freeze
         end
 
@@ -35,10 +40,12 @@ module RubyEventStore
             batch_size: overriden_options.fetch(:batch_size, batch_size),
             database_url: overriden_options.fetch(:database_url, database_url),
             redis_url: overriden_options.fetch(:redis_url, redis_url),
+            cleanup: overriden_options.fetch(:cleanup, cleanup),
+            sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
           )
         end
 
-        attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url
+        attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url, :cleanup, :sleep_on_empty
       end
 
       def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
@@ -47,17 +54,22 @@ module RubyEventStore
         @logger = logger
         @metrics = metrics
         @batch_size = configuration.batch_size
+        @sleep_on_empty = configuration.sleep_on_empty
         @consumer_uuid = consumer_uuid
-        ActiveRecord::Base.establish_connection(configuration.database_url) unless ActiveRecord::Base.connected?
-        if ActiveRecord::Base.connection.adapter_name == "Mysql2"
-          ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
-        end
 
         raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
         @processor = SidekiqProcessor.new(Redis.new(url: configuration.redis_url))
 
         @gracefully_shutting_down = false
         prepare_traps
+
+        @repository = Repository.new(configuration.database_url)
+        @cleanup_strategy = case configuration.cleanup
+        when :none
+          CleanupStrategies::None.new
+        else
+          CleanupStrategies::CleanOldEnqueued.new(repository, ActiveSupport::Duration.parse(configuration.cleanup))
+        end
       end
 
       def init
@@ -70,7 +82,7 @@ module RubyEventStore
           was_something_changed = one_loop
           if !was_something_changed
             STDOUT.flush
-            sleep SLEEP_TIME_WHEN_NOTHING_TO_DO
+            sleep sleep_on_empty
           end
         end
         logger.info "Gracefully shutting down"
@@ -105,7 +117,7 @@ module RubyEventStore
               now = @clock.now.utc
               processor.process(record, now)
 
-              record.update_column(:enqueued_at, now)
+              repository.mark_as_enqueued(record, now)
               something_processed |= true
               updated_record_ids << record.id
             rescue => e
@@ -124,8 +136,8 @@ module RubyEventStore
 
           logger.info "Sent #{updated_record_ids.size} messages from outbox table"
 
-          obtained_lock = refresh_lock_for_process(obtained_lock)
-          break unless obtained_lock
+          refresh_successful = refresh_lock_for_process(obtained_lock)
+          break unless refresh_successful
         end
 
         metrics.write_point_queue(
@@ -136,38 +148,39 @@ module RubyEventStore
 
         release_lock_for_process(fetch_specification)
 
+        cleanup(fetch_specification)
+
         processor.after_batch
 
         something_processed
       end
 
       private
-      attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid
+      attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid, :repository, :cleanup_strategy, :sleep_on_empty
 
       def obtain_lock_for_process(fetch_specification)
-        result = Lock.obtain(fetch_specification, consumer_uuid, clock: @clock)
+        result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
         case result
         when :deadlocked
           logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
           metrics.write_operation_result("obtain", "deadlocked")
-          return false
+          false
         when :lock_timeout
           logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (lock timeout)"
           metrics.write_operation_result("obtain", "lock_timeout")
-          return false
+          false
         when :taken
           logger.debug "Obtaining lock for split_key '#{fetch_specification.split_key}' unsuccessful (taken)"
           metrics.write_operation_result("obtain", "taken")
-          return false
+          false
         else
-          return result
+          result
         end
       end
 
       def release_lock_for_process(fetch_specification)
-        result = Lock.release(fetch_specification, consumer_uuid)
+        result = repository.release_lock_for_process(fetch_specification, consumer_uuid)
         case result
-        when :ok
         when :deadlocked
           logger.warn "Releasing lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
           metrics.write_operation_result("release", "deadlocked")
@@ -177,14 +190,14 @@ module RubyEventStore
         when :not_taken_by_this_process
           logger.debug "Releasing lock for split_key '#{fetch_specification.split_key}' failed (not taken by this process)"
           metrics.write_operation_result("release", "not_taken_by_this_process")
-        else
-          raise "Unexpected result #{result}"
         end
       end
 
       def refresh_lock_for_process(lock)
         result = lock.refresh(clock: @clock)
         case result
+        when :ok
+          return true
         when :deadlocked
           logger.warn "Refreshing lock for split_key '#{lock.split_key}' failed (deadlock)"
           metrics.write_operation_result("refresh", "deadlocked")
@@ -198,7 +211,19 @@ module RubyEventStore
           metrics.write_operation_result("refresh", "stolen")
           return false
         else
-          return result
+          raise "Unexpected result #{result}"
+        end
+      end
+
+      def cleanup(fetch_specification)
+        result = cleanup_strategy.call(fetch_specification)
+        case result
+        when :deadlocked
+          logger.warn "Cleanup for split_key '#{fetch_specification.split_key}' failed (deadlock)"
+          metrics.write_operation_result("cleanup", "deadlocked")
+        when :lock_timeout
+          logger.warn "Cleanup for split_key '#{fetch_specification.split_key}' failed (lock timeout)"
+          metrics.write_operation_result("cleanup", "lock_timeout")
         end
       end
 
@@ -216,11 +241,11 @@ module RubyEventStore
       end
 
       def retrieve_batch(fetch_specification)
-        Record.remaining_for(fetch_specification).order("id ASC").limit(batch_size).to_a
+        repository.retrieve_batch(fetch_specification, batch_size)
       end
 
       def get_remaining_count(fetch_specification)
-        Record.remaining_for(fetch_specification).count
+        repository.get_remaining_count(fetch_specification)
       end
     end
   end

data/lib/ruby_event_store/outbox/fetch_specification.rb

@@ -8,6 +8,24 @@ module RubyEventStore
       end
 
       attr_reader :message_format, :split_key
+
+      def ==(other)
+        other.instance_of?(self.class) &&
+          other.message_format.eql?(message_format) &&
+          other.split_key.eql?(split_key)
+      end
+
+      BIG_VALUE = 0b111111100100010000010010110010101011011101110101001100100110000
+
+      def hash
+        [
+          self.class,
+          message_format,
+          split_key,
+        ].hash ^ BIG_VALUE
+      end
+
+      alias_method :eql?, :==
     end
   end
 end

data/lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative "sidekiq_producer"
+
+module RubyEventStore
+  module Outbox
+    class LegacySidekiqScheduler
+      def initialize
+        @sidekiq_producer = SidekiqProducer.new
+      end
+
+      def call(klass, serialized_record)
+        sidekiq_producer.call(klass, [serialized_record])
+      end
+
+      def verify(subscriber)
+        Class === subscriber && subscriber.respond_to?(:through_outbox?) && subscriber.through_outbox?
+      end
+
+      private
+      attr_reader :sidekiq_producer
+    end
+  end
+end

data/lib/ruby_event_store/outbox/metrics/influx.rb

@@ -6,14 +6,11 @@ module RubyEventStore
       class Influx
         def initialize(url)
           uri = URI.parse(url)
-          params = CGI.parse(uri.query || "")
           options = {
             url: url,
             async: true,
             time_precision: 'ns',
           }
-          options[:username] = params.fetch("username").first if params.key?("username")
-          options[:password] = params.fetch("password").first if params.key?("password")
           @influxdb_client = InfluxDB::Client.new(**options)
         end
 

data/lib/ruby_event_store/outbox/metrics/test.rb

@@ -0,0 +1,24 @@
+require 'influxdb'
+
+module RubyEventStore
+  module Outbox
+    module Metrics
+      class Test
+        def initialize
+          @operation_results = []
+          @queue_stats = []
+        end
+
+        def write_operation_result(operation, result)
+          @operation_results << { operation: operation, result: result }
+        end
+
+        def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
+          @queue_stats << { enqueued: enqueued, failed: failed, remaining: remaining, format: format, split_key: split_key}
+        end
+
+        attr_reader :operation_results, :queue_stats
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/metrics.rb

@@ -3,10 +3,10 @@ module RubyEventStore
     module Metrics
       def self.from_url(metrics_url)
         if metrics_url.nil?
-          require "ruby_event_store/outbox/metrics/null"
+          require_relative "metrics/null"
           Null.new
         else
-          require "ruby_event_store/outbox/metrics/influx"
+          require_relative "metrics/influx"
           Influx.new(metrics_url)
         end
       end

data/lib/ruby_event_store/outbox/repository.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require 'active_support/core_ext/numeric/time.rb'
+
+module RubyEventStore
+  module Outbox
+    class Repository
+      RECENTLY_LOCKED_DURATION = 10.minutes
+
+      class Record < ::ActiveRecord::Base
+        self.primary_key = :id
+        self.table_name = 'event_store_outbox'
+
+        def self.remaining_for(fetch_specification)
+          where(format: fetch_specification.message_format, split_key: fetch_specification.split_key, enqueued_at: nil)
+        end
+
+        def self.for_fetch_specification(fetch_specification)
+          where(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
+        end
+
+        def hash_payload
+          JSON.parse(payload).deep_symbolize_keys
+        end
+
+        def enqueued?
+          !enqueued_at.nil?
+        end
+      end
+
+      class Lock < ::ActiveRecord::Base
+        self.table_name = 'event_store_outbox_locks'
+
+        def self.obtain(fetch_specification, process_uuid, clock:)
+          transaction do
+            l = get_lock_record(fetch_specification)
+
+            if l.recently_locked?(clock: clock)
+              :taken
+            else
+              l.update!(
+                locked_by: process_uuid,
+                locked_at: clock.now,
+              )
+              l
+            end
+          end
+        rescue ActiveRecord::Deadlocked
+          :deadlocked
+        rescue ActiveRecord::LockWaitTimeout
+          :lock_timeout
+        end
+
+        def refresh(clock:)
+          transaction do
+            current_process_uuid = locked_by
+            lock!
+            if locked_by == current_process_uuid
+              update!(locked_at: clock.now)
+              :ok
+            else
+              :stolen
+            end
+          end
+        rescue ActiveRecord::Deadlocked
+          :deadlocked
+        rescue ActiveRecord::LockWaitTimeout
+          :lock_timeout
+        end
+
+        def self.release(fetch_specification, process_uuid)
+          transaction do
+            l = get_lock_record(fetch_specification)
+            if !l.locked_by?(process_uuid)
+              :not_taken_by_this_process
+            else
+              l.update!(locked_by: nil, locked_at: nil)
+              :ok
+            end
+          end
+        rescue ActiveRecord::Deadlocked
+          :deadlocked
+        rescue ActiveRecord::LockWaitTimeout
+          :lock_timeout
+        end
+
+        def locked_by?(process_uuid)
+          locked_by.eql?(process_uuid)
+        end
+
+        def recently_locked?(clock:)
+          locked_by && locked_at > RECENTLY_LOCKED_DURATION.ago(clock.now)
+        end
+
+        def fetch_specification
+          FetchSpecification.new(format, split_key)
+        end
+
+        private
+        def self.lock_for_split_key(fetch_specification)
+          lock.find_by(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
+        end
+
+        def self.get_lock_record(fetch_specification)
+          l = lock_for_split_key(fetch_specification)
+          if l.nil?
+            begin
+              l = create!(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
+            rescue ActiveRecord::RecordNotUnique
+              l = lock_for_split_key(fetch_specification)
+            end
+          end
+          l
+        end
+      end
+
+      def initialize(database_url)
+        ActiveRecord::Base.establish_connection(database_url) unless ActiveRecord::Base.connected?
+        if ActiveRecord::Base.connection.adapter_name == "Mysql2"
+          ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
+        end
+      end
+
+      def retrieve_batch(fetch_specification, batch_size)
+        Record.remaining_for(fetch_specification).order("id ASC").limit(batch_size).to_a
+      end
+
+      def get_remaining_count(fetch_specification)
+        Record.remaining_for(fetch_specification).count
+      end
+
+      def obtain_lock_for_process(fetch_specification, process_uuid, clock:)
+        Lock.obtain(fetch_specification, process_uuid, clock: clock)
+      end
+
+      def release_lock_for_process(fetch_specification, process_uuid)
+        Lock.release(fetch_specification, process_uuid)
+      end
+
+      def mark_as_enqueued(record, now)
+        record.update_column(:enqueued_at, now)
+      end
+
+      def delete_enqueued_older_than(fetch_specification, duration)
+        Record
+          .for_fetch_specification(fetch_specification)
+          .where("enqueued_at < ?", duration.ago)
+          .delete_all
+        :ok
+      rescue ActiveRecord::Deadlocked
+        :deadlocked
+      rescue ActiveRecord::LockWaitTimeout
+        :lock_timeout
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/sidekiq_processor.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "ruby_event_store/outbox/sidekiq5_format"
+require_relative "sidekiq5_format"
 
 module RubyEventStore
   module Outbox

data/lib/ruby_event_store/outbox/sidekiq_producer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'sidekiq'
-require "ruby_event_store/outbox/sidekiq5_format"
+require_relative "repository"
 
 module RubyEventStore
   module Outbox
@@ -10,18 +10,21 @@ module RubyEventStore
         sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
         item = {
           'class' => klass,
-          'args' => args,
+          'args' => args.map(&:to_h),
         }
         normalized_item = sidekiq_client.__send__(:normalize_item, item)
         payload = sidekiq_client.__send__(:process_single, normalized_item.fetch('class'), normalized_item)
         if payload
-          Record.create!(
+          Repository::Record.create!(
             format: SIDEKIQ5_FORMAT,
             split_key: payload.fetch('queue'),
             payload: payload.to_json
           )
         end
       end
+
+      private
+      attr_reader :repository
     end
   end
 end

data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb

@@ -1,16 +1,17 @@
 # frozen_string_literal: true
 
-require "ruby_event_store/outbox/sidekiq_producer"
+require_relative "sidekiq_producer"
 
 module RubyEventStore
   module Outbox
     class SidekiqScheduler
-      def initialize
+      def initialize(serializer: YAML)
+        @serializer = serializer
         @sidekiq_producer = SidekiqProducer.new
       end
 
-      def call(klass, serialized_event)
-        sidekiq_producer.call(klass, [serialized_event.to_h])
+      def call(klass, record)
+        sidekiq_producer.call(klass, [record.serialize(serializer)])
       end
 
       def verify(subscriber)
@@ -18,7 +19,7 @@ module RubyEventStore
       end
 
       private
-      attr_reader :sidekiq_producer
+      attr_reader :serializer, :sidekiq_producer
     end
   end
 end

data/lib/ruby_event_store/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module RubyEventStore
   module Outbox
-    VERSION = "0.0.15"
+    VERSION = "0.0.19"
   end
 end

data/lib/ruby_event_store/outbox.rb

@@ -6,7 +6,7 @@ module RubyEventStore
 end
 
 require_relative 'outbox/fetch_specification'
-require_relative 'outbox/record'
+require_relative 'outbox/repository'
 require_relative 'outbox/sidekiq_scheduler'
-require_relative 'outbox/consumer'
+require_relative 'outbox/legacy_sidekiq_scheduler'
 require_relative 'outbox/version'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_event_store-outbox
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.19
 platform: ruby
 authors:
 - Arkency
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-30 00:00:00.000000000 Z
+date: 2021-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby_event_store
@@ -30,37 +30,39 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '5.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
-description: 
-email:
-- dev@arkency.com
+        version: '5.2'
+description:
+email: dev@arkency.com
 executables:
 - res_outbox
 extensions: []
-extra_rdoc_files: []
+extra_rdoc_files:
+- README.md
 files:
 - README.md
 - bin/res_outbox
 - lib/generators/ruby_event_store/outbox/migration_generator.rb
 - lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.rb
 - lib/ruby_event_store/outbox.rb
+- lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb
+- lib/ruby_event_store/outbox/cleanup_strategies/none.rb
 - lib/ruby_event_store/outbox/cli.rb
 - lib/ruby_event_store/outbox/consumer.rb
-- lib/ruby_event_store/outbox/consumer_process.rb
 - lib/ruby_event_store/outbox/fetch_specification.rb
+- lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb
 - lib/ruby_event_store/outbox/metrics.rb
 - lib/ruby_event_store/outbox/metrics/influx.rb
 - lib/ruby_event_store/outbox/metrics/null.rb
-- lib/ruby_event_store/outbox/record.rb
+- lib/ruby_event_store/outbox/metrics/test.rb
+- lib/ruby_event_store/outbox/repository.rb
 - lib/ruby_event_store/outbox/sidekiq5_format.rb
-- lib/ruby_event_store/outbox/sidekiq_message_handler.rb
 - lib/ruby_event_store/outbox/sidekiq_processor.rb
 - lib/ruby_event_store/outbox/sidekiq_producer.rb
 - lib/ruby_event_store/outbox/sidekiq_scheduler.rb
@@ -69,11 +71,11 @@ homepage: https://railseventstore.org
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://railseventstore.org/
-  changelog_uri: https://github.com/RailsEventStore/rails_event_store/releases
+  homepage_uri: https://railseventstore.org
   source_code_uri: https://github.com/RailsEventStore/rails_event_store
   bug_tracker_uri: https://github.com/RailsEventStore/rails_event_store/issues
-post_install_message: 
+  rubygems_mfa_required: 'true'
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -81,15 +83,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: Active Record based outbox for Ruby Event Store
 test_files: []

data/lib/ruby_event_store/outbox/consumer_process.rb

@@ -1,6 +0,0 @@
-module RubyEventStore
-  module Outbox
-    class ConsumerProcess
-    end
-  end
-end

data/lib/ruby_event_store/outbox/record.rb

@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-require 'active_record'
-
-module RubyEventStore
-  module Outbox
-    class Record < ::ActiveRecord::Base
-      self.primary_key = :id
-      self.table_name = 'event_store_outbox'
-
-      def self.remaining_for(fetch_specification)
-        where(format: fetch_specification.message_format, split_key: fetch_specification.split_key, enqueued_at: nil)
-      end
-
-      def hash_payload
-        JSON.parse(payload).deep_symbolize_keys
-      end
-
-      def enqueued?
-        !enqueued_at.nil?
-      end
-    end
-
-    class Lock < ::ActiveRecord::Base
-      self.table_name = 'event_store_outbox_locks'
-
-      def self.obtain(fetch_specification, process_uuid, clock:)
-        l = nil
-        transaction do
-          l = get_lock_record(fetch_specification)
-
-          return :taken if l.recently_locked?
-
-          l.update!(
-            locked_by: process_uuid,
-            locked_at: clock.now,
-          )
-        end
-        l
-      rescue ActiveRecord::Deadlocked
-        :deadlocked
-      rescue ActiveRecord::LockWaitTimeout
-        :lock_timeout
-      end
-
-      def refresh(clock:)
-        transaction do
-          current_process_uuid = locked_by
-          lock!
-          if locked_by == current_process_uuid
-            update!(locked_at: clock.now)
-            return self
-          else
-            return :stolen
-          end
-        end
-      rescue ActiveRecord::Deadlocked
-        :deadlocked
-      rescue ActiveRecord::LockWaitTimeout
-        :lock_timeout
-      end
-
-      def self.release(fetch_specification, process_uuid)
-        transaction do
-          l = get_lock_record(fetch_specification)
-          return :not_taken_by_this_process if !l.locked_by?(process_uuid)
-
-          l.update!(locked_by: nil, locked_at: nil)
-        end
-        :ok
-      rescue ActiveRecord::Deadlocked
-        :deadlocked
-      rescue ActiveRecord::LockWaitTimeout
-        :lock_timeout
-      end
-
-      def locked_by?(process_uuid)
-        locked_by.eql?(process_uuid)
-      end
-
-      def recently_locked?
-        locked_by && locked_at > 10.minutes.ago
-      end
-
-      private
-      def self.lock_for_split_key(fetch_specification)
-        lock.find_by(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
-      end
-
-      def self.get_lock_record(fetch_specification)
-        l = lock_for_split_key(fetch_specification)
-        if l.nil?
-          begin
-            l = create!(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
-          rescue ActiveRecord::RecordNotUnique
-            l = lock_for_split_key(fetch_specification)
-          end
-        end
-        l
-      end
-    end
-  end
-end

data/lib/ruby_event_store/outbox/sidekiq_message_handler.rb

@@ -1,19 +0,0 @@
-require "sidekiq"
-
-module RubyEventStore
-  module Outbox
-    class SidekiqMessageHandler
-      def initialize
-        @sidekiq = Sidekiq::Client.new(Sidekiq.redis_pool)
-      end
-
-      def init
-      end
-
-      def handle_one_record(now, record)
-        hash_payload = JSON.parse(record.payload)
-        @sidekiq.__send__(:raw_push, [hash_payload])
-      end
-    end
-  end
-end