ruby_event_store-outbox 0.0.11 → 0.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6e7bf65a4d5106581d89918d0bb21bf6f19ee4e08a6f525d27665b41c07b3b0
-  data.tar.gz: eb1350270156bfd886bc11b94dfe858beda55692aa9c72832ca23fcadafe5fc0
+  metadata.gz: '008b96c4b43e3cbfc23d0aa8c9ebb160f0adbfcfc82a976967f1c56338346cc0'
+  data.tar.gz: e65157c0638ecec0a4967a251e820979d18eb92e55212f107be95c23d5a4315b
 SHA512:
-  metadata.gz: eda134f0e4a489546c9b05fd541d27c83f24bdac1e60da244398df09e3b913c650180b0c3472d5d6c0afd502870f025de01d0e94389c90cd491c7504e9693697
-  data.tar.gz: c73fcda82e6635636ae1eccbaeb3c52d4b2c438a7a55e075fccc254d18e118b5ea79266b48926074ec38bdf51b2b81cd787c6f7a61bf6facb5086ab1a5dd166d
+  metadata.gz: f0181744b8c621bc1f0b20d6d4a65ac779db7ba48a80c9f1a02f39208762aa82e5579e745f20cd35f2b54992876065f803e04e24c1c239abe22641f927327070
+  data.tar.gz: 453ce565584436b1b7dd145c738a0d50e95821cb8b8f5d9c00f0d70d2a8ea88e585402de534388459d04853d293c0bcbf7027285d31bbd00aeaef09471cf666d

data/README.md

@@ -1,7 +1,10 @@
 # Ruby Event Store Outbox
 
-Very much work in progress.
+![Ruby Event Store Outbox](https://github.com/RailsEventStore/rails_event_store/workflows/ruby_event_store-outbox/badge.svg)
 
+**Experimental feature of RES ecosystem.**
+
+This repository includes a process and a Rails Event Store scheduler, which can be used to transactionally enqueue background jobs into your background jobs tool of choice. The scheduler included in this repo adds the jobs into the RDBMS into specific table instead of redis inside your transaction, and the process is enqueuing the jobs from that table to the background jobs tool.
 
 ## Installation (app)
 
@@ -35,7 +38,21 @@ end
 Run following process in any way you prefer:
 
 ```
-res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" --redis-url="redis://localhost:6379/0" --log-level=info
+res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" --redis-url="redis://localhost:6379/0" --log-level=info --split-keys=sidekiq_queue1,sidekiq_queue2
+```
+
+It is possible to run as many instances as you prefer, but it does not make sense to run more instances than there are different split keys (sidekiq queues), as one process is operating at one moment only one split key.
+
+### Metrics
+
+It is possible for the outbox process to send metrics to InfluxDB. In order to do that, specify a `--metrics-url` parameter, for example:
+
+```
+res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" \
+  --redis-url="redis://localhost:6379/0" \
+  --log-level=info \
+  --split-keys=sidekiq_queue1,sidekiq_queue2 \
+  --metrics-url=http://user:password@localhost:8086/dbname"
 ```
 
 

data/lib/generators/ruby_event_store/outbox/migration_generator.rb

@@ -16,12 +16,7 @@ module RubyEventStore
 
       private
 
-      def rails_version
-        Rails::VERSION::STRING
-      end
-
       def migration_version
-        return nil if Gem::Version.new(rails_version) < Gem::Version.new("5.0.0")
         "[4.2]"
       end
 

data/lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.rb

@@ -11,5 +11,13 @@ class CreateEventStoreOutbox < ActiveRecord::Migration<%= migration_version %>
     end
     add_index :event_store_outbox, [:format, :enqueued_at, :split_key], name: "index_event_store_outbox_for_pool"
     add_index :event_store_outbox, [:created_at, :enqueued_at], name: "index_event_store_outbox_for_clear"
+
+    create_table(:event_store_outbox_locks, force: false) do |t|
+      t.string :format, null: false
+      t.string :split_key, null: false
+      t.datetime :locked_at, null: true
+      t.string :locked_by, null: true, limit: 36
+    end
+    add_index :event_store_outbox_locks, [:format, :split_key], name: "index_event_store_outbox_locks_for_locking", unique: true
   end
 end

data/lib/ruby_event_store/outbox.rb

@@ -5,7 +5,8 @@ module RubyEventStore
   end
 end
 
-require_relative 'outbox/record'
+require_relative 'outbox/fetch_specification'
+require_relative 'outbox/repository'
 require_relative 'outbox/sidekiq_scheduler'
 require_relative 'outbox/consumer'
 require_relative 'outbox/version'

data/lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb

@@ -0,0 +1,19 @@
+module RubyEventStore
+  module Outbox
+    module CleanupStrategies
+      class CleanOldEnqueued
+        def initialize(repository, duration)
+          @repository = repository
+          @duration = duration
+        end
+
+        def call(fetch_specification)
+          repository.delete_enqueued_older_than(fetch_specification, duration)
+        end
+
+        private
+        attr_reader :repository, :duration
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/cleanup_strategies/none.rb

@@ -0,0 +1,13 @@
+module RubyEventStore
+  module Outbox
+    module CleanupStrategies
+      class None
+        def initialize
+        end
+
+        def call(_fetch_specification)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/cli.rb

@@ -6,11 +6,11 @@ require "ruby_event_store/outbox/metrics"
 module RubyEventStore
   module Outbox
     class CLI
-      Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format, :batch_size, :metrics_url)
+      Options = Struct.new(:database_url, :redis_url, :log_level, :split_keys, :message_format, :batch_size, :metrics_url, :cleanup_strategy)
 
       class Parser
         def self.parse(argv)
-          options = Options.new(nil, nil, :warn, nil, nil, 100)
+          options = Options.new(nil, nil, :warn, nil, nil, 100, nil, :none)
           OptionParser.new do |option_parser|
             option_parser.banner = "Usage: res_outbox [options]"
 
@@ -42,6 +42,10 @@ module RubyEventStore
               options.metrics_url = metrics_url
             end
 
+            option_parser.on("--cleanup CLEANUP_STRATEGY", "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed") do |cleanup_strategy|
+              options.cleanup_strategy = cleanup_strategy
+            end
+
             option_parser.on_tail("--version", "Show version") do
               puts VERSION
               exit
@@ -59,17 +63,20 @@ module RubyEventStore
       end
 
       def build_consumer(options)
-        logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox")
+        consumer_uuid = SecureRandom.uuid
+        logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox #{consumer_uuid}")
         consumer_configuration = Consumer::Configuration.new(
           split_keys: options.split_keys,
           message_format: options.message_format,
           batch_size: options.batch_size,
           database_url: options.database_url,
           redis_url: options.redis_url,
+          cleanup: options.cleanup_strategy,
         )
         metrics = Metrics.from_url(options.metrics_url)
         outbox_consumer = RubyEventStore::Outbox::Consumer.new(
-          options,
+          consumer_uuid,
+          consumer_configuration,
           logger: logger,
           metrics: metrics,
         )

data/lib/ruby_event_store/outbox/consumer.rb

@@ -1,13 +1,18 @@
 require "logger"
 require "redis"
 require "active_record"
-require "ruby_event_store/outbox/record"
+require "ruby_event_store/outbox/repository"
 require "ruby_event_store/outbox/sidekiq5_format"
+require "ruby_event_store/outbox/sidekiq_processor"
+require "ruby_event_store/outbox/fetch_specification"
+require "ruby_event_store/outbox/cleanup_strategies/none"
+require "ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued"
 
 module RubyEventStore
   module Outbox
     class Consumer
-      SLEEP_TIME_WHEN_NOTHING_TO_DO = 0.1
+      SLEEP_TIME_WHEN_NOTHING_TO_DO = 0.5
+      MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK = 10
 
       class Configuration
         def initialize(
@@ -15,13 +20,15 @@ module RubyEventStore
           message_format:,
           batch_size:,
           database_url:,
-          redis_url:
+          redis_url:,
+          cleanup:
         )
           @split_keys = split_keys
           @message_format = message_format
           @batch_size = batch_size || 100
           @database_url = database_url
           @redis_url = redis_url
+          @cleanup = cleanup
           freeze
         end
 
@@ -32,34 +39,37 @@ module RubyEventStore
             batch_size: overriden_options.fetch(:batch_size, batch_size),
             database_url: overriden_options.fetch(:database_url, database_url),
             redis_url: overriden_options.fetch(:redis_url, redis_url),
+            cleanup: overriden_options.fetch(:cleanup, cleanup)
           )
         end
 
-        attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url
+        attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url, :cleanup
       end
 
-      def initialize(configuration, clock: Time, logger:, metrics:)
+      def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
         @split_keys = configuration.split_keys
         @clock = clock
-        @redis = Redis.new(url: configuration.redis_url)
         @logger = logger
         @metrics = metrics
         @batch_size = configuration.batch_size
-        ActiveRecord::Base.establish_connection(configuration.database_url) unless ActiveRecord::Base.connected?
-        if ActiveRecord::Base.connection.adapter_name == "Mysql2"
-          ActiveRecord::Base.connection.execute("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED;")
-          ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
-        end
+        @consumer_uuid = consumer_uuid
 
         raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
-        @message_format = SIDEKIQ5_FORMAT
+        @processor = SidekiqProcessor.new(Redis.new(url: configuration.redis_url))
 
         @gracefully_shutting_down = false
         prepare_traps
+
+        @repository = Repository.new(configuration.database_url)
+        @cleanup_strategy = case configuration.cleanup
+        when :none
+          CleanupStrategies::None.new
+        else
+          CleanupStrategies::CleanOldEnqueued.new(repository, ActiveSupport::Duration.parse(configuration.cleanup))
+        end
       end
 
       def init
-        @redis.sadd("queues", split_keys)
         logger.info("Initiated RubyEventStore::Outbox v#{VERSION}")
         logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
       end
@@ -76,62 +86,133 @@ module RubyEventStore
       end
 
       def one_loop
-        Record.transaction do
-          records_scope = Record.lock.where(format: message_format, enqueued_at: nil)
-          records_scope = records_scope.where(split_key: split_keys) if !split_keys.nil?
-          records = records_scope.order("id ASC").limit(batch_size).to_a
-          if records.empty?
-            metrics.write_point_queue(status: "ok")
-            return false
+        remaining_split_keys = @split_keys.dup
+
+        was_something_changed = false
+        while (split_key = remaining_split_keys.shift)
+          was_something_changed |= handle_split(FetchSpecification.new(processor.message_format, split_key))
+        end
+        was_something_changed
+      end
+
+      def handle_split(fetch_specification)
+        obtained_lock = obtain_lock_for_process(fetch_specification)
+        return false unless obtained_lock
+
+        something_processed = false
+
+        MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK.times do
+          batch = retrieve_batch(fetch_specification)
+          if batch.empty?
+            break
           end
 
-          now = @clock.now.utc
           failed_record_ids = []
-          records.group_by(&:split_key).each do |split_key, records2|
+          updated_record_ids = []
+          batch.each do |record|
             begin
-              failed = handle_group_of_records(now, split_key, records2)
-              failed_record_ids.concat(failed.map(&:id))
+              now = @clock.now.utc
+              processor.process(record, now)
+
+              repository.mark_as_enqueued(record, now)
+              something_processed |= true
+              updated_record_ids << record.id
             rescue => e
-              failed_record_ids.concat(records2.map(&:id))
+              failed_record_ids << record.id
               e.full_message.split($/).each {|line| logger.error(line) }
             end
           end
 
-          updated_record_ids = records.map(&:id) - failed_record_ids
-          Record.where(id: updated_record_ids).update_all(enqueued_at: now)
-          metrics.write_point_queue(status: "ok", enqueued: updated_record_ids.size, failed: failed_record_ids.size)
+          metrics.write_point_queue(
+            enqueued: updated_record_ids.size,
+            failed: failed_record_ids.size,
+            format: fetch_specification.message_format,
+            split_key: fetch_specification.split_key,
+            remaining: get_remaining_count(fetch_specification)
+          )
 
           logger.info "Sent #{updated_record_ids.size} messages from outbox table"
-          true
+
+          refresh_successful = refresh_lock_for_process(obtained_lock)
+          break unless refresh_successful
         end
-      rescue ActiveRecord::Deadlocked
-        logger.warn "Outbox fetch deadlocked"
-        metrics.write_point_queue(status: "deadlocked")
-        false
-      rescue ActiveRecord::LockWaitTimeout
-        logger.warn "Outbox fetch lock timeout"
-        metrics.write_point_queue(status: "lock_timeout")
-        false
+
+        metrics.write_point_queue(
+          format: fetch_specification.message_format,
+          split_key: fetch_specification.split_key,
+          remaining: get_remaining_count(fetch_specification)
+        ) unless something_processed
+
+        release_lock_for_process(fetch_specification)
+
+        cleanup_strategy.call(fetch_specification)
+
+        processor.after_batch
+
+        something_processed
       end
 
       private
-      attr_reader :split_keys, :logger, :message_format, :batch_size, :metrics
-
-      def handle_group_of_records(now, split_key, records)
-        failed = []
-        elements = []
-        records.each do |record|
-          begin
-            elements << JSON.generate(JSON.parse(record.payload).merge({
-              "enqueued_at" => now.to_f,
-            }))
-          rescue => e
-            failed << record
-            e.full_message.split($/).each {|line| logger.error(line) }
-          end
+      attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid, :repository, :cleanup_strategy
+
+      def obtain_lock_for_process(fetch_specification)
+        result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
+        case result
+        when :deadlocked
+          logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
+          metrics.write_operation_result("obtain", "deadlocked")
+          return false
+        when :lock_timeout
+          logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (lock timeout)"
+          metrics.write_operation_result("obtain", "lock_timeout")
+          return false
+        when :taken
+          logger.debug "Obtaining lock for split_key '#{fetch_specification.split_key}' unsuccessful (taken)"
+          metrics.write_operation_result("obtain", "taken")
+          return false
+        else
+          return result
+        end
+      end
+
+      def release_lock_for_process(fetch_specification)
+        result = repository.release_lock_for_process(fetch_specification, consumer_uuid)
+        case result
+        when :ok
+        when :deadlocked
+          logger.warn "Releasing lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
+          metrics.write_operation_result("release", "deadlocked")
+        when :lock_timeout
+          logger.warn "Releasing lock for split_key '#{fetch_specification.split_key}' failed (lock timeout)"
+          metrics.write_operation_result("release", "lock_timeout")
+        when :not_taken_by_this_process
+          logger.debug "Releasing lock for split_key '#{fetch_specification.split_key}' failed (not taken by this process)"
+          metrics.write_operation_result("release", "not_taken_by_this_process")
+        else
+          raise "Unexpected result #{result}"
+        end
+      end
+
+      def refresh_lock_for_process(lock)
+        result = lock.refresh(clock: @clock)
+        case result
+        when :ok
+          return true
+        when :deadlocked
+          logger.warn "Refreshing lock for split_key '#{lock.split_key}' failed (deadlock)"
+          metrics.write_operation_result("refresh", "deadlocked")
+          return false
+        when :lock_timeout
+          logger.warn "Refreshing lock for split_key '#{lock.split_key}' failed (lock timeout)"
+          metrics.write_operation_result("refresh", "lock_timeout")
+          return false
+        when :stolen
+          logger.debug "Refreshing lock for split_key '#{lock.split_key}' unsuccessful (stolen)"
+          metrics.write_operation_result("refresh", "stolen")
+          return false
+        else
+          raise "Unexpected result #{result}"
         end
-        @redis.lpush("queue:#{split_key}", elements)
-        failed
       end
 
       def prepare_traps
@@ -146,6 +227,14 @@ module RubyEventStore
       def initiate_graceful_shutdown
         @gracefully_shutting_down = true
       end
+
+      def retrieve_batch(fetch_specification)
+        repository.retrieve_batch(fetch_specification, batch_size)
+      end
+
+      def get_remaining_count(fetch_specification)
+        repository.get_remaining_count(fetch_specification)
+      end
     end
   end
 end

data/lib/ruby_event_store/outbox/consumer_process.rb

@@ -0,0 +1,6 @@
+module RubyEventStore
+  module Outbox
+    class ConsumerProcess
+    end
+  end
+end

data/lib/ruby_event_store/outbox/fetch_specification.rb

@@ -0,0 +1,31 @@
+module RubyEventStore
+  module Outbox
+    class FetchSpecification
+      def initialize(message_format, split_key)
+        @message_format = message_format
+        @split_key = split_key
+        freeze
+      end
+
+      attr_reader :message_format, :split_key
+
+      def ==(other)
+        other.instance_of?(self.class) &&
+          other.message_format.eql?(message_format) &&
+          other.split_key.eql?(split_key)
+      end
+
+      BIG_VALUE = 0b111111100100010000010010110010101011011101110101001100100110000
+
+      def hash
+        [
+          self.class,
+          message_format,
+          split_key,
+        ].hash ^ BIG_VALUE
+      end
+
+      alias_method :eql?, :==
+    end
+  end
+end

data/lib/ruby_event_store/outbox/metrics/influx.rb

@@ -17,14 +17,28 @@ module RubyEventStore
           @influxdb_client = InfluxDB::Client.new(**options)
         end
 
-        def write_point_queue(status:, enqueued: 0, failed: 0)
+        def write_operation_result(operation, result)
+          write_point("ruby_event_store.outbox.lock", {
+            values: {
+              value: 1,
+            },
+            tags: {
+              operation: operation,
+              result: result,
+            }
+          })
+        end
+
+        def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
           write_point("ruby_event_store.outbox.queue", {
             values: {
               enqueued: enqueued,
               failed: failed,
+              remaining: remaining,
             },
             tags: {
-              status: status,
+              format: format,
+              split_key: split_key,
             }
           })
         end

data/lib/ruby_event_store/outbox/metrics/null.rb

@@ -2,6 +2,9 @@ module RubyEventStore
   module Outbox
     module Metrics
       class Null
+        def write_operation_result(operation, result)
+        end
+
         def write_point_queue(**kwargs)
         end
       end

data/lib/ruby_event_store/outbox/repository.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require 'active_support/core_ext/numeric/time.rb'
+
+module RubyEventStore
+  module Outbox
+    class Repository
+      RECENTLY_LOCKED_DURATION = 10.minutes
+
+      class Record < ::ActiveRecord::Base
+        self.primary_key = :id
+        self.table_name = 'event_store_outbox'
+
+        def self.remaining_for(fetch_specification)
+          where(format: fetch_specification.message_format, split_key: fetch_specification.split_key, enqueued_at: nil)
+        end
+
+        def self.for_fetch_specification(fetch_specification)
+          where(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
+        end
+
+        def hash_payload
+          JSON.parse(payload).deep_symbolize_keys
+        end
+
+        def enqueued?
+          !enqueued_at.nil?
+        end
+      end
+
+      class Lock < ::ActiveRecord::Base
+        self.table_name = 'event_store_outbox_locks'
+
+        def self.obtain(fetch_specification, process_uuid, clock:)
+          transaction do
+            l = get_lock_record(fetch_specification)
+
+            if l.recently_locked?
+              :taken
+            else
+              l.update!(
+                locked_by: process_uuid,
+                locked_at: clock.now,
+              )
+              l
+            end
+          end
+        rescue ActiveRecord::Deadlocked
+          :deadlocked
+        rescue ActiveRecord::LockWaitTimeout
+          :lock_timeout
+        end
+
+        def refresh(clock:)
+          transaction do
+            current_process_uuid = locked_by
+            lock!
+            if locked_by == current_process_uuid
+              update!(locked_at: clock.now)
+              :ok
+            else
+              :stolen
+            end
+          end
+        rescue ActiveRecord::Deadlocked
+          :deadlocked
+        rescue ActiveRecord::LockWaitTimeout
+          :lock_timeout
+        end
+
+        def self.release(fetch_specification, process_uuid)
+          transaction do
+            l = get_lock_record(fetch_specification)
+            if !l.locked_by?(process_uuid)
+              :not_taken_by_this_process
+            else
+              l.update!(locked_by: nil, locked_at: nil)
+              :ok
+            end
+          end
+        rescue ActiveRecord::Deadlocked
+          :deadlocked
+        rescue ActiveRecord::LockWaitTimeout
+          :lock_timeout
+        end
+
+        def locked_by?(process_uuid)
+          locked_by.eql?(process_uuid)
+        end
+
+        def recently_locked?
+          locked_by && locked_at > RECENTLY_LOCKED_DURATION.ago
+        end
+
+        def fetch_specification
+          FetchSpecification.new(format, split_key)
+        end
+
+        private
+        def self.lock_for_split_key(fetch_specification)
+          lock.find_by(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
+        end
+
+        def self.get_lock_record(fetch_specification)
+          l = lock_for_split_key(fetch_specification)
+          if l.nil?
+            begin
+              l = create!(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
+            rescue ActiveRecord::RecordNotUnique
+              l = lock_for_split_key(fetch_specification)
+            end
+          end
+          l
+        end
+      end
+
+      def initialize(database_url)
+        ActiveRecord::Base.establish_connection(database_url) unless ActiveRecord::Base.connected?
+        if ActiveRecord::Base.connection.adapter_name == "Mysql2"
+          ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
+        end
+      end
+
+      def retrieve_batch(fetch_specification, batch_size)
+        Record.remaining_for(fetch_specification).order("id ASC").limit(batch_size).to_a
+      end
+
+      def get_remaining_count(fetch_specification)
+        Record.remaining_for(fetch_specification).count
+      end
+
+      def obtain_lock_for_process(fetch_specification, process_uuid, clock:)
+        Lock.obtain(fetch_specification, process_uuid, clock: clock)
+      end
+
+      def release_lock_for_process(fetch_specification, process_uuid)
+        Lock.release(fetch_specification, process_uuid)
+      end
+
+      def mark_as_enqueued(record, now)
+        record.update_column(:enqueued_at, now)
+      end
+
+      def delete_enqueued_older_than(fetch_specification, duration)
+        Record
+          .for_fetch_specification(fetch_specification)
+          .where("enqueued_at < ?", duration.ago)
+          .delete_all
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/sidekiq_processor.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "ruby_event_store/outbox/sidekiq5_format"
+
+module RubyEventStore
+  module Outbox
+    class SidekiqProcessor
+      InvalidPayload = Class.new(StandardError)
+
+      def initialize(redis)
+        @redis = redis
+        @recently_used_queues = Set.new
+      end
+
+      def process(record, now)
+        parsed_record = JSON.parse(record.payload)
+
+        queue = parsed_record["queue"]
+        raise InvalidPayload.new("Missing queue") if queue.nil? || queue.empty?
+        payload = JSON.generate(parsed_record.merge({
+          "enqueued_at" => now.to_f,
+        }))
+
+        redis.lpush("queue:#{queue}", payload)
+
+        @recently_used_queues << queue
+      end
+
+      def after_batch
+        if !@recently_used_queues.empty?
+          redis.sadd("queues", @recently_used_queues.to_a)
+          @recently_used_queues.clear
+        end
+      end
+
+      def message_format
+        SIDEKIQ5_FORMAT
+      end
+
+      private
+      attr_reader :redis
+    end
+  end
+end

data/lib/ruby_event_store/outbox/sidekiq_producer.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+require "ruby_event_store/outbox/repository"
+
+module RubyEventStore
+  module Outbox
+    class SidekiqProducer
+      def call(klass, args)
+        sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
+        item = {
+          'class' => klass,
+          'args' => args,
+        }
+        normalized_item = sidekiq_client.__send__(:normalize_item, item)
+        payload = sidekiq_client.__send__(:process_single, normalized_item.fetch('class'), normalized_item)
+        if payload
+          Repository::Record.create!(
+            format: SIDEKIQ5_FORMAT,
+            split_key: payload.fetch('queue'),
+            payload: payload.to_json
+          )
+        end
+      end
+
+      private
+      attr_reader :repository
+    end
+  end
+end

data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb

@@ -1,31 +1,24 @@
 # frozen_string_literal: true
 
-require 'sidekiq'
-require "ruby_event_store/outbox/sidekiq5_format"
+require "ruby_event_store/outbox/sidekiq_producer"
 
 module RubyEventStore
   module Outbox
     class SidekiqScheduler
-      def call(klass, serialized_event)
-        sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
-        item = {
-          'class' => klass,
-          'args' => [serialized_event.to_h],
-        }
-        normalized_item = sidekiq_client.__send__(:normalize_item, item)
-        payload = sidekiq_client.__send__(:process_single, normalized_item.fetch('class'), normalized_item)
-        if payload
-          Record.create!(
-            format: SIDEKIQ5_FORMAT,
-            split_key: payload.fetch('queue'),
-            payload: payload.to_json
-          )
-        end
+      def initialize
+        @sidekiq_producer = SidekiqProducer.new
+      end
+
+      def call(klass, serialized_record)
+        sidekiq_producer.call(klass, [serialized_record.to_h])
       end
 
       def verify(subscriber)
         Class === subscriber && subscriber.respond_to?(:through_outbox?) && subscriber.through_outbox?
       end
+
+      private
+      attr_reader :sidekiq_producer
     end
   end
 end

data/lib/ruby_event_store/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module RubyEventStore
   module Outbox
-    VERSION = "0.0.11"
+    VERSION = "0.0.16"
   end
 end

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: ruby_event_store-outbox
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.16
 platform: ruby
 authors:
 - Arkency
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-26 00:00:00.000000000 Z
+date: 2021-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby_event_store
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
 - !ruby/object:Gem::Dependency
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '5.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '5.2'
 description: 
 email:
 - dev@arkency.com
@@ -51,14 +51,20 @@ files:
 - lib/generators/ruby_event_store/outbox/migration_generator.rb
 - lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.rb
 - lib/ruby_event_store/outbox.rb
+- lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb
+- lib/ruby_event_store/outbox/cleanup_strategies/none.rb
 - lib/ruby_event_store/outbox/cli.rb
 - lib/ruby_event_store/outbox/consumer.rb
+- lib/ruby_event_store/outbox/consumer_process.rb
+- lib/ruby_event_store/outbox/fetch_specification.rb
 - lib/ruby_event_store/outbox/metrics.rb
 - lib/ruby_event_store/outbox/metrics/influx.rb
 - lib/ruby_event_store/outbox/metrics/null.rb
-- lib/ruby_event_store/outbox/record.rb
+- lib/ruby_event_store/outbox/repository.rb
 - lib/ruby_event_store/outbox/sidekiq5_format.rb
 - lib/ruby_event_store/outbox/sidekiq_message_handler.rb
+- lib/ruby_event_store/outbox/sidekiq_processor.rb
+- lib/ruby_event_store/outbox/sidekiq_producer.rb
 - lib/ruby_event_store/outbox/sidekiq_scheduler.rb
 - lib/ruby_event_store/outbox/version.rb
 homepage: https://railseventstore.org

data/lib/ruby_event_store/outbox/record.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-require 'active_record'
-
-module RubyEventStore
-  module Outbox
-    class Record < ::ActiveRecord::Base
-      self.primary_key = :id
-      self.table_name = 'event_store_outbox'
-
-      def hash_payload
-        JSON.parse(payload).deep_symbolize_keys
-      end
-    end
-  end
-end