ruby_event_store-outbox 0.0.24 → 0.0.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7c7c7068c9bd49735031384ece752500cc641f5035bc1f7953104e64a3d22dd8
-  data.tar.gz: 2975fdb8c2dfe6fc9843ce5efd00042a2c16d2d9ec2e6524598507a4fcb36dc3
+  metadata.gz: e26efaae4895206e93089d60869e01dd1a0f8182e6f41e49edef000a2833b5bb
+  data.tar.gz: 1ab71712967fa31437d538c787721f6bd27f0a65d1b0474e95cd1428945d9b62
 SHA512:
-  metadata.gz: 3a1118657e1c164df15f9eb7d117adb80a4f7a693eed4a3974dc4a9b0f3bdbd15504f441fe50dbd0fdf5318fae6205dd2c4cadeebd0c4897a61fb4be4ea2e382
-  data.tar.gz: 7a84d03ea98a5d2b0019f2eedf3c76b509f5bb25c44fb2db7dc9a952c1752cf4db17ee284fd9ee8bce4295267743a29222d26bee1709d554375f95c64ebb1203
+  metadata.gz: 3189203160620f90690f35b511d2e60471c6652e13ef510464952aa40ba2b35c650d66477f1ab5e3360e6bf31ab6a1fdf40aadd8916ff9aa802dbe069034a013
+  data.tar.gz: 765725def129ea5368be1fd5fcb2ae013e45efe17ed48b9d58eb4d1c1356582fb76fcafd9885d9fc527556a6351993709250c434b8ee586708f9b8514d617e2f

data/README.md

@@ -34,11 +34,12 @@ Additionally, your handler's `through_outbox?` method should return `true`, for
 
 ```ruby
 class SomeHandler
-  def self.through_outbox?; true; end
+  def self.through_outbox?
+    true
+  end
 end
 ```
 
-
 ## Installation (outbox process)
 
 Run following process in any way you prefer:
@@ -65,7 +66,6 @@ res_outbox --database-url="mysql2://root@0.0.0.0:3306/my_database" \
   --metrics-url=http://user:password@localhost:8086/dbname"
 ```
 
-
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/RailsEventStore/rails_event_store.

data/lib/generators/ruby_event_store/outbox/migration_generator.rb

@@ -1,28 +1,30 @@
 # frozen_string_literal: true
 
 begin
-  require 'rails/generators'
+  require "rails/generators"
 rescue LoadError
 end
 
-module RubyEventStore
-  module Outbox
-    class MigrationGenerator < Rails::Generators::Base
-      source_root File.expand_path(File.join(File.dirname(__FILE__), './templates'))
+if defined?(Rails::Generators::Base)
+  module RubyEventStore
+    module Outbox
+      class MigrationGenerator < Rails::Generators::Base
+        source_root File.expand_path(File.join(File.dirname(__FILE__), "./templates"))
 
-      def create_migration
-        template "create_event_store_outbox_template.rb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
-      end
+        def create_migration
+          template "create_event_store_outbox_template.erb", "db/migrate/#{timestamp}_create_event_store_outbox.rb"
+        end
 
-      private
+        private
 
-      def migration_version
-        "[4.2]"
-      end
+        def migration_version
+          ::ActiveRecord::Migration.current_version
+        end
 
-      def timestamp
-        Time.now.strftime("%Y%m%d%H%M%S")
+        def timestamp
+          Time.now.strftime("%Y%m%d%H%M%S")
+        end
       end
     end
   end
-end if defined?(Rails::Generators::Base)
+end

data/lib/generators/ruby_event_store/outbox/templates/{create_event_store_outbox_template.rb → create_event_store_outbox_template.erb}

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
-class CreateEventStoreOutbox < ActiveRecord::Migration<%= migration_version %>
+class CreateEventStoreOutbox < ActiveRecord::Migration[<%= migration_version %>]
   def change
     create_table(:event_store_outbox, force: false) do |t|
       t.string :split_key, null: true
       t.string :format, null: false
       t.binary :payload, null: false
-      t.datetime :created_at, null: false
-      t.datetime :enqueued_at, null: true
+      t.datetime :created_at, null: false, precision: 6
+      t.datetime :enqueued_at, null: true, precision: 6
     end
     add_index :event_store_outbox, [:format, :enqueued_at, :split_key], name: "index_event_store_outbox_for_pool"
     add_index :event_store_outbox, [:created_at, :enqueued_at], name: "index_event_store_outbox_for_clear"
@@ -15,7 +15,7 @@ class CreateEventStoreOutbox < ActiveRecord::Migration<%= migration_version %>
     create_table(:event_store_outbox_locks, force: false) do |t|
       t.string :format, null: false
       t.string :split_key, null: false
-      t.datetime :locked_at, null: true
+      t.datetime :locked_at, null: true, precision: 6
       t.string :locked_by, null: true, limit: 36
     end
     add_index :event_store_outbox_locks, [:format, :split_key], name: "index_event_store_outbox_locks_for_locking", unique: true

data/lib/ruby_event_store/outbox/batch_result.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module RubyEventStore
+  module Outbox
+    class BatchResult
+      def self.empty
+        new
+      end
+
+      def initialize
+        @success_count = 0
+        @failed_count = 0
+      end
+
+      attr_reader :success_count, :failed_count
+
+      def count_success!
+        @success_count += 1
+      end
+
+      def count_failed!
+        @failed_count += 1
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module RubyEventStore
   module Outbox
     module CleanupStrategies
@@ -13,6 +15,7 @@ module RubyEventStore
         end
 
         private
+
         attr_reader :repository, :duration, :limit
       end
     end

data/lib/ruby_event_store/outbox/cleanup_strategies/none.rb

@@ -1,9 +1,10 @@
+# frozen_string_literal: true
+
 module RubyEventStore
   module Outbox
     module CleanupStrategies
       class None
-        def initialize
-        end
+        def initialize; end
 
         def call(_fetch_specification)
           :ok

data/lib/ruby_event_store/outbox/cleanup_strategies.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module RubyEventStore
+  module Outbox
+    module CleanupStrategies
+      def self.build(configuration, repository)
+        case configuration.cleanup
+        when :none
+          None.new
+        else
+          CleanOldEnqueued.new(
+            repository,
+            ActiveSupport::Duration.parse(configuration.cleanup),
+            configuration.cleanup_limit
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/cli.rb

@@ -1,7 +1,11 @@
+# frozen_string_literal: true
+
 require "optparse"
 require_relative "version"
 require_relative "consumer"
+require_relative "runner"
 require_relative "metrics"
+require_relative "configuration"
 
 module RubyEventStore
   module Outbox
@@ -11,7 +15,7 @@ module RubyEventStore
         redis_url: nil,
         log_level: :warn,
         split_keys: nil,
-        message_format: 'sidekiq5',
+        message_format: "sidekiq5",
         batch_size: 100,
         metrics_url: nil,
         cleanup_strategy: :none,
@@ -23,69 +27,84 @@ module RubyEventStore
       class Parser
         def self.parse(argv)
           options = Options.new(*DEFAULTS.values)
-          OptionParser.new do |option_parser|
-            option_parser.banner = "Usage: res_outbox [options]"
+          OptionParser
+            .new do |option_parser|
+              option_parser.banner = "Usage: res_outbox [options]"
 
-            option_parser.on("--database-url=DATABASE_URL", "Database where outbox table is stored") do |database_url|
-              options.database_url = database_url
-            end
+              option_parser.on(
+                "--database-url=DATABASE_URL",
+                "Database where outbox table is stored"
+              ) { |database_url| options.database_url = database_url }
 
-            option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
-              options.redis_url = redis_url
-            end
+              option_parser.on("--redis-url=REDIS_URL", "URL to redis database") do |redis_url|
+                options.redis_url = redis_url
+              end
 
-            option_parser.on("--log-level=LOG_LEVEL", [:fatal, :error, :warn, :info, :debug], "Logging level, one of: fatal, error, warn, info, debug. Default: warn") do |log_level|
-              options.log_level = log_level.to_sym
-            end
+              option_parser.on(
+                "--log-level=LOG_LEVEL",
+                %i[fatal error warn info debug],
+                "Logging level, one of: fatal, error, warn, info, debug. Default: warn"
+              ) { |log_level| options.log_level = log_level.to_sym }
 
-            option_parser.on("--message-format=FORMAT", ["sidekiq5"], "Message format, supported: sidekiq5. Default: sidekiq5") do |message_format|
-              options.message_format = message_format
-            end
+              option_parser.on(
+                "--message-format=FORMAT",
+                ["sidekiq5"],
+                "Message format, supported: sidekiq5. Default: sidekiq5"
+              ) { |message_format| options.message_format = message_format }
 
-            option_parser.on("--split-keys=SPLIT_KEYS", Array, "Split keys which should be handled, all if not specified") do |split_keys|
-              options.split_keys = split_keys if !split_keys.empty?
-            end
+              option_parser.on(
+                "--split-keys=SPLIT_KEYS",
+                Array,
+                "Split keys which should be handled, all if not specified"
+              ) { |split_keys| options.split_keys = split_keys if !split_keys.empty? }
 
-            option_parser.on("--batch-size=BATCH_SIZE", Integer, "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100") do |batch_size|
-              options.batch_size = batch_size
-            end
+              option_parser.on(
+                "--batch-size=BATCH_SIZE",
+                Integer,
+                "Amount of records fetched in one fetch. Bigger value means more duplicated messages when network problems occur. Default: 100"
+              ) { |batch_size| options.batch_size = batch_size }
 
-            option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
-              options.metrics_url = metrics_url
-            end
+              option_parser.on("--metrics-url=METRICS_URL", "URI to metrics collector, optional") do |metrics_url|
+                options.metrics_url = metrics_url
+              end
 
-            option_parser.on("--cleanup=STRATEGY", "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none") do |cleanup_strategy|
-              options.cleanup_strategy = cleanup_strategy
-            end
+              option_parser.on(
+                "--cleanup=STRATEGY",
+                "A strategy for cleaning old records. One of: none or iso8601 duration format how old enqueued records should be removed. Default: none"
+              ) { |cleanup_strategy| options.cleanup_strategy = cleanup_strategy }
 
-            option_parser.on("--cleanup-limit=LIMIT", "Amount of records removed in single cleanup run. One of: all or number of records that should be removed. Default: all") do |cleanup_limit|
-              options.cleanup_limit = cleanup_limit
-            end
+              option_parser.on(
+                "--cleanup-limit=LIMIT",
+                "Amount of records removed in single cleanup run. One of: all or number of records that should be removed. Default: all"
+              ) { |cleanup_limit| options.cleanup_limit = cleanup_limit }
 
-            option_parser.on("--sleep-on-empty=SLEEP_TIME", Float, "How long to sleep before next check when there was nothing to do. Default: 0.5") do |sleep_on_empty|
-              options.sleep_on_empty = sleep_on_empty
-            end
+              option_parser.on(
+                "--sleep-on-empty=SLEEP_TIME",
+                Float,
+                "How long to sleep before next check when there was nothing to do. Default: 0.5"
+              ) { |sleep_on_empty| options.sleep_on_empty = sleep_on_empty }
 
-            option_parser.on_tail("--version", "Show version") do
-              puts VERSION
-              exit
+              option_parser.on_tail("--version", "Show version") do
+                puts VERSION
+                exit
+              end
             end
-          end.parse(argv)
+            .parse(argv)
           return options
         end
       end
 
       def run(argv)
         options = Parser.parse(argv)
-        outbox_consumer = build_consumer(options)
-        outbox_consumer.init
-        outbox_consumer.run
+        build_runner(options)
+          .init
+          .run
       end
 
-      def build_consumer(options)
+      def build_runner(options)
         consumer_uuid = SecureRandom.uuid
         logger = Logger.new(STDOUT, level: options.log_level, progname: "RES-Outbox #{consumer_uuid}")
-        consumer_configuration = Consumer::Configuration.new(
+        consumer_configuration = Configuration.new(
           split_keys: options.split_keys,
           message_format: options.message_format,
           batch_size: options.batch_size,
@@ -96,12 +115,9 @@ module RubyEventStore
           sleep_on_empty: options.sleep_on_empty,
         )
         metrics = Metrics.from_url(options.metrics_url)
-        outbox_consumer = RubyEventStore::Outbox::Consumer.new(
-          consumer_uuid,
-          consumer_configuration,
-          logger: logger,
-          metrics: metrics,
-        )
+        outbox_consumer =
+          Outbox::Consumer.new(consumer_uuid, consumer_configuration, logger: logger, metrics: metrics)
+        Runner.new(outbox_consumer, consumer_configuration, logger: logger)
       end
     end
   end

data/lib/ruby_event_store/outbox/configuration.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module RubyEventStore
+  module Outbox
+    class Configuration
+      def initialize(
+        split_keys:,
+        message_format:,
+        batch_size:,
+        database_url:,
+        redis_url:,
+        cleanup:,
+        cleanup_limit:,
+        sleep_on_empty:
+      )
+        @split_keys = split_keys
+        @message_format = message_format
+        @batch_size = batch_size || 100
+        @database_url = database_url
+        @redis_url = redis_url
+        @cleanup = cleanup
+        @cleanup_limit = cleanup_limit
+        @sleep_on_empty = sleep_on_empty
+        freeze
+      end
+
+      def with(overriden_options)
+        self.class.new(
+          split_keys: overriden_options.fetch(:split_keys, split_keys),
+          message_format: overriden_options.fetch(:message_format, message_format),
+          batch_size: overriden_options.fetch(:batch_size, batch_size),
+          database_url: overriden_options.fetch(:database_url, database_url),
+          redis_url: overriden_options.fetch(:redis_url, redis_url),
+          cleanup: overriden_options.fetch(:cleanup, cleanup),
+          cleanup_limit: overriden_options.fetch(:cleanup_limit, cleanup_limit),
+          sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
+        )
+      end
+
+      attr_reader :split_keys,
+        :message_format,
+        :batch_size,
+        :database_url,
+        :redis_url,
+        :cleanup,
+        :cleanup_limit,
+        :sleep_on_empty
+    end
+  end
+end

data/lib/ruby_event_store/outbox/consumer.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 require "logger"
-require "redis"
+require "redis-client"
 require "active_record"
 require_relative "repository"
 require_relative "sidekiq5_format"
@@ -13,86 +15,24 @@ module RubyEventStore
     class Consumer
       MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK = 10
 
-      class Configuration
-        def initialize(
-          split_keys:,
-          message_format:,
-          batch_size:,
-          database_url:,
-          redis_url:,
-          cleanup:,
-          cleanup_limit:,
-          sleep_on_empty:
-        )
-          @split_keys = split_keys
-          @message_format = message_format
-          @batch_size = batch_size || 100
-          @database_url = database_url
-          @redis_url = redis_url
-          @cleanup = cleanup
-          @cleanup_limit = cleanup_limit
-          @sleep_on_empty = sleep_on_empty
-          freeze
-        end
-
-        def with(overriden_options)
-          self.class.new(
-            split_keys: overriden_options.fetch(:split_keys, split_keys),
-            message_format: overriden_options.fetch(:message_format, message_format),
-            batch_size: overriden_options.fetch(:batch_size, batch_size),
-            database_url: overriden_options.fetch(:database_url, database_url),
-            redis_url: overriden_options.fetch(:redis_url, redis_url),
-            cleanup: overriden_options.fetch(:cleanup, cleanup),
-            cleanup_limit: overriden_options.fetch(:cleanup_limit, cleanup_limit),
-            sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
-          )
-        end
-
-        attr_reader :split_keys, :message_format, :batch_size, :database_url, :redis_url, :cleanup, :cleanup_limit, :sleep_on_empty
-      end
-
       def initialize(consumer_uuid, configuration, clock: Time, logger:, metrics:)
         @split_keys = configuration.split_keys
         @clock = clock
         @logger = logger
         @metrics = metrics
-        @batch_size = configuration.batch_size
-        @sleep_on_empty = configuration.sleep_on_empty
+        @tempo = Tempo.new(configuration.batch_size)
         @consumer_uuid = consumer_uuid
 
         raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
-        @processor = SidekiqProcessor.new(Redis.new(url: configuration.redis_url))
-
-        @gracefully_shutting_down = false
-        prepare_traps
+        redis_config = RedisClient.config(url: configuration.redis_url)
+        @processor = SidekiqProcessor.new(redis_config.new_client)
 
         @repository = Repository.new(configuration.database_url)
-        @cleanup_strategy = case configuration.cleanup
-        when :none
-          CleanupStrategies::None.new
-        else
-          CleanupStrategies::CleanOldEnqueued.new(repository, ActiveSupport::Duration.parse(configuration.cleanup), configuration.cleanup_limit)
-        end
+        @cleanup_strategy = CleanupStrategies.build(configuration, repository)
       end
 
-      def init
-        logger.info("Initiated RubyEventStore::Outbox v#{VERSION}")
-        logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
-      end
-
-      def run
-        while !@gracefully_shutting_down do
-          was_something_changed = one_loop
-          if !was_something_changed
-            STDOUT.flush
-            sleep sleep_on_empty
-          end
-        end
-        logger.info "Gracefully shutting down"
-      end
-
-      def one_loop
-        remaining_split_keys = @split_keys.dup
+      def process
+        remaining_split_keys = split_keys.dup
 
         was_something_changed = false
         while (split_key = remaining_split_keys.shift)
@@ -109,45 +49,41 @@ module RubyEventStore
 
         MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK.times do
           batch = retrieve_batch(fetch_specification)
-          if batch.empty?
-            break
-          end
+          break if batch.empty?
 
-          failed_record_ids = []
-          updated_record_ids = []
+          batch_result = BatchResult.empty
           batch.each do |record|
-            begin
+            handle_failure(batch_result) do
               now = @clock.now.utc
               processor.process(record, now)
 
               repository.mark_as_enqueued(record, now)
               something_processed |= true
-              updated_record_ids << record.id
-            rescue => e
-              failed_record_ids << record.id
-              e.full_message.split($/).each {|line| logger.error(line) }
+              batch_result.count_success!
             end
           end
 
           metrics.write_point_queue(
-            enqueued: updated_record_ids.size,
-            failed: failed_record_ids.size,
+            enqueued: batch_result.success_count,
+            failed: batch_result.failed_count,
             format: fetch_specification.message_format,
             split_key: fetch_specification.split_key,
             remaining: get_remaining_count(fetch_specification)
           )
 
-          logger.info "Sent #{updated_record_ids.size} messages from outbox table"
+          logger.info "Sent #{batch_result.success_count} messages from outbox table"
 
           refresh_successful = refresh_lock_for_process(obtained_lock)
           break unless refresh_successful
         end
 
-        metrics.write_point_queue(
-          format: fetch_specification.message_format,
-          split_key: fetch_specification.split_key,
-          remaining: get_remaining_count(fetch_specification)
-        ) unless something_processed
+        unless something_processed
+          metrics.write_point_queue(
+            format: fetch_specification.message_format,
+            split_key: fetch_specification.split_key,
+            remaining: get_remaining_count(fetch_specification)
+          )
+        end
 
         release_lock_for_process(fetch_specification)
 
@@ -159,7 +95,35 @@ module RubyEventStore
       end
 
       private
-      attr_reader :split_keys, :logger, :batch_size, :metrics, :processor, :consumer_uuid, :repository, :cleanup_strategy, :sleep_on_empty
+
+      attr_reader :split_keys,
+                  :logger,
+                  :metrics,
+                  :processor,
+                  :consumer_uuid,
+                  :repository,
+                  :cleanup_strategy,
+                  :tempo
+
+      def handle_failure(batch_result)
+        retried = false
+        yield
+      rescue RetriableError => error
+        if retried
+          batch_result.count_failed!
+          log_error(error)
+        else
+          retried = true
+          retry
+        end
+      rescue => error
+        batch_result.count_failed!
+        log_error(error)
+      end
+
+      def log_error(e)
+        e.full_message.split($/).each { |line| logger.error(line) }
+      end
 
       def obtain_lock_for_process(fetch_specification)
         result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
@@ -230,21 +194,8 @@ module RubyEventStore
         end
       end
 
-      def prepare_traps
-        Signal.trap("INT") do
-          initiate_graceful_shutdown
-        end
-        Signal.trap("TERM") do
-          initiate_graceful_shutdown
-        end
-      end
-
-      def initiate_graceful_shutdown
-        @gracefully_shutting_down = true
-      end
-
       def retrieve_batch(fetch_specification)
-        repository.retrieve_batch(fetch_specification, batch_size)
+        repository.retrieve_batch(fetch_specification, tempo.batch_size)
       end
 
       def get_remaining_count(fetch_specification)

data/lib/ruby_event_store/outbox/fetch_specification.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module RubyEventStore
   module Outbox
     class FetchSpecification
@@ -10,19 +12,11 @@ module RubyEventStore
       attr_reader :message_format, :split_key
 
       def ==(other)
-        other.instance_of?(self.class) &&
-          other.message_format.eql?(message_format) &&
-          other.split_key.eql?(split_key)
+        other.instance_of?(self.class) && other.message_format.eql?(message_format) && other.split_key.eql?(split_key)
       end
 
-      BIG_VALUE = 0b111111100100010000010010110010101011011101110101001100100110000
-
       def hash
-        [
-          self.class,
-          message_format,
-          split_key,
-        ].hash ^ BIG_VALUE
+        [message_format, split_key].hash ^ self.class.hash
       end
 
       alias_method :eql?, :==

data/lib/ruby_event_store/outbox/metrics/influx.rb

@@ -1,4 +1,6 @@
-require 'influxdb'
+# frozen_string_literal: true
+
+require "influxdb"
 
 module RubyEventStore
   module Outbox
@@ -6,38 +8,32 @@ module RubyEventStore
       class Influx
         def initialize(url)
           uri = URI.parse(url)
-          options = {
-            url: url,
-            async: true,
-            time_precision: 'ns',
-          }
+          options = { url: url, async: true, time_precision: "ns" }
           @influxdb_client = InfluxDB::Client.new(**options)
         end
 
         def write_operation_result(operation, result)
-          write_point("ruby_event_store.outbox.lock", {
-            values: {
-              value: 1,
-            },
-            tags: {
-              operation: operation,
-              result: result,
-            }
-          })
+          write_point(
+            "ruby_event_store.outbox.lock",
+            { values: { value: 1 }, tags: { operation: operation, result: result } }
+          )
         end
 
         def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
-          write_point("ruby_event_store.outbox.queue", {
-            values: {
-              enqueued: enqueued,
-              failed: failed,
-              remaining: remaining,
-            },
-            tags: {
-              format: format,
-              split_key: split_key,
+          write_point(
+            "ruby_event_store.outbox.queue",
+            {
+              values: {
+                enqueued: enqueued,
+                failed: failed,
+                remaining: remaining
+              },
+              tags: {
+                format: format,
+                split_key: split_key
+              }
             }
-          })
+          )
         end
 
         def write_point(series, data)

data/lib/ruby_event_store/outbox/metrics/null.rb

@@ -1,12 +1,12 @@
+# frozen_string_literal: true
+
 module RubyEventStore
   module Outbox
     module Metrics
       class Null
-        def write_operation_result(operation, result)
-        end
+        def write_operation_result(operation, result); end
 
-        def write_point_queue(**kwargs)
-        end
+        def write_point_queue(**kwargs); end
       end
     end
   end

data/lib/ruby_event_store/outbox/metrics/test.rb

@@ -1,4 +1,6 @@
-require 'influxdb'
+# frozen_string_literal: true
+
+require "influxdb"
 
 module RubyEventStore
   module Outbox
@@ -14,7 +16,13 @@ module RubyEventStore
         end
 
         def write_point_queue(enqueued: 0, failed: 0, remaining: 0, format: nil, split_key: nil)
-          @queue_stats << { enqueued: enqueued, failed: failed, remaining: remaining, format: format, split_key: split_key}
+          @queue_stats << {
+            enqueued: enqueued,
+            failed: failed,
+            remaining: remaining,
+            format: format,
+            split_key: split_key
+          }
         end
 
         attr_reader :operation_results, :queue_stats

data/lib/ruby_event_store/outbox/metrics.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module RubyEventStore
   module Outbox
     module Metrics

data/lib/ruby_event_store/outbox/repository.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'active_record'
-require 'active_support/core_ext/numeric/time.rb'
+require "active_record"
+require "active_support/core_ext/numeric/time.rb"
 
 module RubyEventStore
   module Outbox
@@ -10,7 +10,7 @@ module RubyEventStore
 
       class Record < ::ActiveRecord::Base
         self.primary_key = :id
-        self.table_name = 'event_store_outbox'
+        self.table_name = "event_store_outbox"
 
         def self.remaining_for(fetch_specification)
           where(format: fetch_specification.message_format, split_key: fetch_specification.split_key, enqueued_at: nil)
@@ -30,7 +30,7 @@ module RubyEventStore
       end
 
       class Lock < ::ActiveRecord::Base
-        self.table_name = 'event_store_outbox_locks'
+        self.table_name = "event_store_outbox_locks"
 
         def self.obtain(fetch_specification, process_uuid, clock:)
           transaction do
@@ -39,16 +39,13 @@ module RubyEventStore
             if l.recently_locked?(clock: clock)
               :taken
             else
-              l.update!(
-                locked_by: process_uuid,
-                locked_at: clock.now,
-              )
+              l.update!(locked_by: process_uuid, locked_at: clock.now)
               l
             end
           end
-        rescue ActiveRecord::Deadlocked
+        rescue ::ActiveRecord::Deadlocked
           :deadlocked
-        rescue ActiveRecord::LockWaitTimeout
+        rescue ::ActiveRecord::LockWaitTimeout
           :lock_timeout
         end
 
@@ -63,9 +60,9 @@ module RubyEventStore
               :stolen
             end
           end
-        rescue ActiveRecord::Deadlocked
+        rescue ::ActiveRecord::Deadlocked
           :deadlocked
-        rescue ActiveRecord::LockWaitTimeout
+        rescue ::ActiveRecord::LockWaitTimeout
           :lock_timeout
         end
 
@@ -79,9 +76,9 @@ module RubyEventStore
               :ok
             end
           end
-        rescue ActiveRecord::Deadlocked
+        rescue ::ActiveRecord::Deadlocked
           :deadlocked
-        rescue ActiveRecord::LockWaitTimeout
+        rescue ::ActiveRecord::LockWaitTimeout
           :lock_timeout
         end
 
@@ -98,6 +95,7 @@ module RubyEventStore
         end
 
         private
+
         def self.lock_for_split_key(fetch_specification)
           lock.find_by(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
         end
@@ -107,7 +105,7 @@ module RubyEventStore
           if l.nil?
             begin
               l = create!(format: fetch_specification.message_format, split_key: fetch_specification.split_key)
-            rescue ActiveRecord::RecordNotUnique
+            rescue ::ActiveRecord::RecordNotUnique
               l = lock_for_split_key(fetch_specification)
             end
           end
@@ -116,9 +114,9 @@ module RubyEventStore
       end
 
       def initialize(database_url)
-        ActiveRecord::Base.establish_connection(database_url) unless ActiveRecord::Base.connected?
-        if ActiveRecord::Base.connection.adapter_name == "Mysql2"
-          ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
+        ::ActiveRecord::Base.establish_connection(database_url) unless ::ActiveRecord::Base.connected?
+        if ::ActiveRecord::Base.connection.adapter_name == "Mysql2"
+          ::ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
         end
       end
 
@@ -143,15 +141,13 @@ module RubyEventStore
       end
 
       def delete_enqueued_older_than(fetch_specification, duration, limit)
-        scope = Record
-          .for_fetch_specification(fetch_specification)
-          .where("enqueued_at < ?", duration.ago)
-        scope = scope.limit(limit) unless limit == :all
+        scope = Record.for_fetch_specification(fetch_specification).where("enqueued_at < ?", duration.ago)
+        scope = scope.limit(limit).order(:id) unless limit == :all
         scope.delete_all
         :ok
-      rescue ActiveRecord::Deadlocked
+      rescue ::ActiveRecord::Deadlocked
         :deadlocked
-      rescue ActiveRecord::LockWaitTimeout
+      rescue ::ActiveRecord::LockWaitTimeout
         :lock_timeout
       end
     end

data/lib/ruby_event_store/outbox/runner.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module RubyEventStore
+  module Outbox
+    class Runner
+      def initialize(consumer, configuration, logger:)
+        @consumer = consumer
+        @logger = logger
+        @sleep_on_empty = configuration.sleep_on_empty
+        @split_keys = configuration.split_keys
+        @gracefully_shutting_down = false
+        prepare_traps
+      end
+
+      def run
+        logger.info("Initiated RubyEventStore::Outbox v#{VERSION}")
+        logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
+
+        while !@gracefully_shutting_down
+          was_something_changed = consumer.process
+          if !was_something_changed
+            STDOUT.flush
+            sleep sleep_on_empty
+          end
+        end
+
+        logger.info "Gracefully shutting down"
+      end
+
+      private
+      attr_reader :consumer, :logger, :sleep_on_empty, :split_keys
+
+      def prepare_traps
+        Signal.trap("INT") { initiate_graceful_shutdown }
+        Signal.trap("TERM") { initiate_graceful_shutdown }
+      end
+
+      def initiate_graceful_shutdown
+        @gracefully_shutting_down = true
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/sidekiq_processor.rb

@@ -17,20 +17,17 @@ module RubyEventStore
 
         queue = parsed_record["queue"]
         raise InvalidPayload.new("Missing queue") if queue.nil? || queue.empty?
-        payload = JSON.generate(parsed_record.merge({
-          "enqueued_at" => now.to_f,
-        }))
+        payload = JSON.generate(parsed_record.merge({ "enqueued_at" => now.to_f }))
 
-        redis.lpush("queue:#{queue}", payload)
+        redis.call("LPUSH", "queue:#{queue}", payload)
 
         @recently_used_queues << queue
+      rescue RedisClient::TimeoutError, RedisClient::ConnectionError
+        raise RetriableError
       end
 
       def after_batch
-        if !@recently_used_queues.empty?
-          redis.sadd("queues", @recently_used_queues.to_a)
-          @recently_used_queues.clear
-        end
+        ensure_that_sidekiq_knows_about_all_queues
       end
 
       def message_format
@@ -38,6 +35,14 @@ module RubyEventStore
       end
 
       private
+
+      def ensure_that_sidekiq_knows_about_all_queues
+        if !@recently_used_queues.empty?
+          redis.call("SADD", "queues", @recently_used_queues.to_a)
+          @recently_used_queues.clear
+        end
+      end
+
       attr_reader :redis
     end
   end

data/lib/ruby_event_store/outbox/sidekiq_producer.rb

@@ -1,31 +1,42 @@
 # frozen_string_literal: true
 
-require 'sidekiq'
-require_relative 'sidekiq5_format'
+require "sidekiq"
+require_relative "sidekiq5_format"
 require_relative "repository"
 
 module RubyEventStore
   module Outbox
     class SidekiqProducer
       def call(klass, args)
-        sidekiq_client = Sidekiq::Client.new(Sidekiq.redis_pool)
-        item = {
-          'args' => args.map(&:to_h).map {|h| h.transform_keys(&:to_s)},
-          'class' => klass,
-        }
+        item = { "args" => args.map(&:to_h).map { |h| h.transform_keys(&:to_s) }, "class" => klass }
         normalized_item = sidekiq_client.__send__(:normalize_item, item)
-        payload = sidekiq_client.middleware.invoke(normalized_item['class'], normalized_item, normalized_item['queue'], Sidekiq.redis_pool) { normalized_item }
+        payload =
+          sidekiq_client
+            .middleware
+            .invoke(normalized_item["class"], normalized_item, normalized_item["queue"], Sidekiq.redis_pool) do
+              normalized_item
+            end
         if payload
           Repository::Record.create!(
             format: SIDEKIQ5_FORMAT,
-            split_key: payload.fetch('queue'),
+            split_key: payload.fetch("queue"),
             payload: payload.to_json
           )
         end
       end
 
       private
+
       attr_reader :repository
+
+      def sidekiq_client
+        @sidekiq_client ||=
+          if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("7.0.0")
+            Sidekiq::Client.new(Sidekiq.redis_pool)
+          else
+            Sidekiq::Client.new(pool: Sidekiq.redis_pool)
+          end
+      end
     end
   end
 end

data/lib/ruby_event_store/outbox/sidekiq_scheduler.rb

@@ -5,7 +5,7 @@ require_relative "sidekiq_producer"
 module RubyEventStore
   module Outbox
     class SidekiqScheduler
-      def initialize(serializer: RubyEventStore::Serializers::YAML)
+      def initialize(serializer: Serializers::YAML)
         @serializer = serializer
         @sidekiq_producer = SidekiqProducer.new
       end
@@ -19,6 +19,7 @@ module RubyEventStore
       end
 
       private
+
       attr_reader :serializer, :sidekiq_producer
     end
   end

data/lib/ruby_event_store/outbox/tempo.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module RubyEventStore
+  module Outbox
+    class Tempo
+      EXPONENTIAL_MULTIPLIER = 2
+
+      def initialize(max_batch_size)
+        raise ArgumentError if max_batch_size < 1
+        @max_batch_size = max_batch_size
+      end
+
+      def batch_size
+        @batch_size = next_batch_size
+      end
+
+      private
+
+      def next_batch_size
+        return 1 if @batch_size.nil?
+        [@batch_size * EXPONENTIAL_MULTIPLIER, @max_batch_size].min
+      end
+    end
+  end
+end

data/lib/ruby_event_store/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module RubyEventStore
   module Outbox
-    VERSION = "0.0.24"
+    VERSION = "0.0.26"
   end
 end

data/lib/ruby_event_store/outbox.rb

@@ -2,11 +2,15 @@
 
 module RubyEventStore
   module Outbox
+    Error = Class.new(StandardError)
+    RetriableError = Class.new(Error)
   end
 end
 
-require_relative 'outbox/fetch_specification'
-require_relative 'outbox/repository'
-require_relative 'outbox/sidekiq_scheduler'
-require_relative 'outbox/legacy_sidekiq_scheduler'
-require_relative 'outbox/version'
+require_relative "outbox/fetch_specification"
+require_relative "outbox/repository"
+require_relative "outbox/sidekiq_scheduler"
+require_relative "outbox/version"
+require_relative "outbox/tempo"
+require_relative "outbox/batch_result"
+require_relative "outbox/cleanup_strategies"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_event_store-outbox
 version: !ruby/object:Gem::Version
-  version: 0.0.24
+  version: 0.0.26
 platform: ruby
 authors:
 - Arkency
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-26 00:00:00.000000000 Z
+date: 2024-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby_event_store
@@ -30,15 +30,15 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
-description: 
+        version: '6.0'
+description:
 email: dev@arkency.com
 executables:
 - res_outbox
@@ -49,23 +49,27 @@ files:
 - README.md
 - bin/res_outbox
 - lib/generators/ruby_event_store/outbox/migration_generator.rb
-- lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.rb
+- lib/generators/ruby_event_store/outbox/templates/create_event_store_outbox_template.erb
 - lib/ruby_event_store/outbox.rb
+- lib/ruby_event_store/outbox/batch_result.rb
+- lib/ruby_event_store/outbox/cleanup_strategies.rb
 - lib/ruby_event_store/outbox/cleanup_strategies/clean_old_enqueued.rb
 - lib/ruby_event_store/outbox/cleanup_strategies/none.rb
 - lib/ruby_event_store/outbox/cli.rb
+- lib/ruby_event_store/outbox/configuration.rb
 - lib/ruby_event_store/outbox/consumer.rb
 - lib/ruby_event_store/outbox/fetch_specification.rb
-- lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb
 - lib/ruby_event_store/outbox/metrics.rb
 - lib/ruby_event_store/outbox/metrics/influx.rb
 - lib/ruby_event_store/outbox/metrics/null.rb
 - lib/ruby_event_store/outbox/metrics/test.rb
 - lib/ruby_event_store/outbox/repository.rb
+- lib/ruby_event_store/outbox/runner.rb
 - lib/ruby_event_store/outbox/sidekiq5_format.rb
 - lib/ruby_event_store/outbox/sidekiq_processor.rb
 - lib/ruby_event_store/outbox/sidekiq_producer.rb
 - lib/ruby_event_store/outbox/sidekiq_scheduler.rb
+- lib/ruby_event_store/outbox/tempo.rb
 - lib/ruby_event_store/outbox/version.rb
 homepage: https://railseventstore.org
 licenses:
@@ -75,7 +79,7 @@ metadata:
   source_code_uri: https://github.com/RailsEventStore/rails_event_store
   bug_tracker_uri: https://github.com/RailsEventStore/rails_event_store/issues
   rubygems_mfa_required: 'true'
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -83,15 +87,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.9
-signing_key: 
+rubygems_version: 3.4.10
+signing_key:
 specification_version: 4
 summary: Active Record based outbox for Ruby Event Store
 test_files: []

data/lib/ruby_event_store/outbox/legacy_sidekiq_scheduler.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "sidekiq_producer"
-
-module RubyEventStore
-  module Outbox
-    class LegacySidekiqScheduler
-      def initialize
-        @sidekiq_producer = SidekiqProducer.new
-      end
-
-      def call(klass, serialized_record)
-        sidekiq_producer.call(klass, [serialized_record])
-      end
-
-      def verify(subscriber)
-        Class === subscriber && subscriber.respond_to?(:through_outbox?) && subscriber.through_outbox?
-      end
-
-      private
-      attr_reader :sidekiq_producer
-    end
-  end
-end