ruby_event_store-outbox 0.0.30 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b41d9195dccdaa450d5e52678e65e238a0e6e36ea9147cb8ec6c397605690baf
-  data.tar.gz: faab545ddeddc537089d2e0227c70cc493a9b4196ae39777dfb2f28da45aa9f6
+  metadata.gz: 72cede48801be6a8a32295f690fc6a7a4091ddda2613791765b0d6127ea9eaec
+  data.tar.gz: 7a20be4c5a340e2305a760ecb104547dd7cf728d68e97717c38fde97ef404dac
 SHA512:
-  metadata.gz: a46cd984375895708527aa817da14e198b829d330f3063cccd838925ba6a5c3a12a2cb72bc06cc4725fc65d83bcb36bd7e0469f5a3d4cbc742b12f974a3c0bee
-  data.tar.gz: 67ccdf300534a01a646711cf924ba744904d7c7fdca3850b3a7cc97b76b6d5881d153844cea9b6e87ce116fb31788e2c63a9caa529dd2141a076567177cb086e
+  metadata.gz: e5412c791d66024632f3ff638d0bc72b7d4b6fe7535c5f736ecaff420e19d8059d8b97e8e0c34de5032433320df2fd329affb7a8c6356f60ed406d8dfaeb0953
+  data.tar.gz: f5375242631ef96645463bf23d1507b491a9259231d3082158a5ddcb4b2a35e2afbe473aea2dd7cec7c73629f735525a972ab673a1e6779d6e7a88b53ec7c378

data/lib/ruby_event_store/outbox/cli.rb

@@ -20,7 +20,8 @@ module RubyEventStore
         metrics_url: nil,
         cleanup_strategy: :none,
         cleanup_limit: :all,
-        sleep_on_empty: 0.5
+        sleep_on_empty: 0.5,
+        locking: true,
       }
       Options = Struct.new(*DEFAULTS.keys)
 
@@ -84,6 +85,10 @@ module RubyEventStore
                 "How long to sleep before next check when there was nothing to do. Default: 0.5"
               ) { |sleep_on_empty| options.sleep_on_empty = sleep_on_empty }
 
+              option_parser.on("-l", "--[no-]lock", "Lock split key in consumer") do |locking|
+                options.locking = locking
+              end
+
               option_parser.on_tail("--version", "Show version") do
                 puts VERSION
                 exit
@@ -112,6 +117,7 @@ module RubyEventStore
           cleanup: options.cleanup_strategy,
           cleanup_limit: options.cleanup_limit,
           sleep_on_empty: options.sleep_on_empty,
+          locking: options.locking
         )
         metrics = Metrics.from_url(options.metrics_url)
         outbox_consumer =

data/lib/ruby_event_store/outbox/configuration.rb

@@ -11,7 +11,8 @@ module RubyEventStore
         redis_url:,
         cleanup:,
         cleanup_limit:,
-        sleep_on_empty:
+        sleep_on_empty:,
+        locking:
       )
         @split_keys = split_keys
         @message_format = message_format
@@ -21,6 +22,7 @@ module RubyEventStore
         @cleanup = cleanup
         @cleanup_limit = cleanup_limit
         @sleep_on_empty = sleep_on_empty
+        @locking = locking
         freeze
       end
 
@@ -33,7 +35,8 @@ module RubyEventStore
           redis_url: overriden_options.fetch(:redis_url, redis_url),
           cleanup: overriden_options.fetch(:cleanup, cleanup),
           cleanup_limit: overriden_options.fetch(:cleanup_limit, cleanup_limit),
-          sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty)
+          sleep_on_empty: overriden_options.fetch(:sleep_on_empty, sleep_on_empty),
+          locking: overriden_options.fetch(:locking, locking),
         )
       end
 
@@ -44,7 +47,8 @@ module RubyEventStore
         :redis_url,
         :cleanup,
         :cleanup_limit,
-        :sleep_on_empty
+        :sleep_on_empty,
+        :locking
     end
   end
 end

data/lib/ruby_event_store/outbox/consumer.rb

@@ -23,12 +23,13 @@ module RubyEventStore
         @metrics = metrics
         @tempo = Tempo.new(configuration.batch_size)
         @consumer_uuid = consumer_uuid
+        @locking = configuration.locking
 
         raise "Unknown format" if configuration.message_format != SIDEKIQ5_FORMAT
         redis_config = RedisClient.config(url: configuration.redis_url)
         @processor = SidekiqProcessor.new(redis_config.new_client)
 
-        @repository = Repository.new(configuration.database_url)
+        @repository = Repository.new(configuration.database_url, logger, metrics)
         @cleanup_strategy = CleanupStrategies.build(configuration, repository)
       end
 
@@ -43,56 +44,14 @@ module RubyEventStore
       end
 
       def handle_split(fetch_specification)
-        obtained_lock = obtain_lock_for_process(fetch_specification)
-        return false unless obtained_lock
-
-        something_processed = false
-
-        MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK.times do
-          batch = retrieve_batch(fetch_specification)
-          break if batch.empty?
-
-          batch_result = BatchResult.empty
-          batch.each do |record|
-            handle_failure(batch_result) do
-              now = @clock.now.utc
-              processor.process(record, now)
-
-              repository.mark_as_enqueued(record, now)
-              something_processed |= true
-              batch_result.count_success!
-            end
-          end
-
-          metrics.write_point_queue(
-            enqueued: batch_result.success_count,
-            failed: batch_result.failed_count,
-            format: fetch_specification.message_format,
-            split_key: fetch_specification.split_key,
-            remaining: get_remaining_count(fetch_specification)
-          )
-
-          logger.info "Sent #{batch_result.success_count} messages from outbox table"
-
-          refresh_successful = refresh_lock_for_process(obtained_lock)
-          break unless refresh_successful
-        end
-
-        unless something_processed
-          metrics.write_point_queue(
-            format: fetch_specification.message_format,
-            split_key: fetch_specification.split_key,
-            remaining: get_remaining_count(fetch_specification)
-          )
-        end
-
-        release_lock_for_process(fetch_specification)
-
-        cleanup(fetch_specification)
-
-        processor.after_batch
-
-        something_processed
+        repository.with_next_batch(fetch_specification, tempo.batch_size, consumer_uuid, locking, @clock) do |record|
+          now = @clock.now.utc
+          processor.process(record, now)
+          repository.mark_as_enqueued(record, now)
+        end.tap do
+          cleanup(fetch_specification)
+          processor.after_batch
+        end.success_count > 0
       end
 
       private
@@ -103,86 +62,10 @@ module RubyEventStore
                   :processor,
                   :consumer_uuid,
                   :repository,
+                  :locking,
                   :cleanup_strategy,
                   :tempo
 
-      def handle_failure(batch_result)
-        retried = false
-        yield
-      rescue RetriableRedisError => error
-        if retried
-          batch_result.count_failed!
-          log_error(error)
-        else
-          retried = true
-          retry
-        end
-      rescue => error
-        batch_result.count_failed!
-        log_error(error)
-      end
-
-      def log_error(e)
-        e.full_message.split($/).each { |line| logger.error(line) }
-      end
-
-      def obtain_lock_for_process(fetch_specification)
-        result = repository.obtain_lock_for_process(fetch_specification, consumer_uuid, clock: @clock)
-        case result
-        when :deadlocked
-          logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
-          metrics.write_operation_result("obtain", "deadlocked")
-          false
-        when :lock_timeout
-          logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (lock timeout)"
-          metrics.write_operation_result("obtain", "lock_timeout")
-          false
-        when :taken
-          logger.debug "Obtaining lock for split_key '#{fetch_specification.split_key}' unsuccessful (taken)"
-          metrics.write_operation_result("obtain", "taken")
-          false
-        else
-          result
-        end
-      end
-
-      def release_lock_for_process(fetch_specification)
-        result = repository.release_lock_for_process(fetch_specification, consumer_uuid)
-        case result
-        when :deadlocked
-          logger.warn "Releasing lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
-          metrics.write_operation_result("release", "deadlocked")
-        when :lock_timeout
-          logger.warn "Releasing lock for split_key '#{fetch_specification.split_key}' failed (lock timeout)"
-          metrics.write_operation_result("release", "lock_timeout")
-        when :not_taken_by_this_process
-          logger.debug "Releasing lock for split_key '#{fetch_specification.split_key}' failed (not taken by this process)"
-          metrics.write_operation_result("release", "not_taken_by_this_process")
-        end
-      end
-
-      def refresh_lock_for_process(lock)
-        result = lock.refresh(clock: @clock)
-        case result
-        when :ok
-          return true
-        when :deadlocked
-          logger.warn "Refreshing lock for split_key '#{lock.split_key}' failed (deadlock)"
-          metrics.write_operation_result("refresh", "deadlocked")
-          return false
-        when :lock_timeout
-          logger.warn "Refreshing lock for split_key '#{lock.split_key}' failed (lock timeout)"
-          metrics.write_operation_result("refresh", "lock_timeout")
-          return false
-        when :stolen
-          logger.debug "Refreshing lock for split_key '#{lock.split_key}' unsuccessful (stolen)"
-          metrics.write_operation_result("refresh", "stolen")
-          return false
-        else
-          raise "Unexpected result #{result}"
-        end
-      end
-
       def cleanup(fetch_specification)
         result = cleanup_strategy.call(fetch_specification)
         case result
@@ -194,14 +77,6 @@ module RubyEventStore
           metrics.write_operation_result("cleanup", "lock_timeout")
         end
       end
-
-      def retrieve_batch(fetch_specification)
-        repository.retrieve_batch(fetch_specification, tempo.batch_size)
-      end
-
-      def get_remaining_count(fetch_specification)
-        repository.get_remaining_count(fetch_specification)
-      end
     end
   end
 end

data/lib/ruby_event_store/outbox/repository.rb

@@ -113,27 +113,21 @@ module RubyEventStore
         end
       end
 
-      def initialize(database_url)
+      def initialize(database_url, logger, metrics)
+        @logger = logger
+        @metrics = metrics
         ::ActiveRecord::Base.establish_connection(database_url) unless ::ActiveRecord::Base.connected?
         if ::ActiveRecord::Base.connection.adapter_name == "Mysql2"
           ::ActiveRecord::Base.connection.execute("SET SESSION innodb_lock_wait_timeout = 1;")
         end
       end
 
-      def retrieve_batch(fetch_specification, batch_size)
-        Record.remaining_for(fetch_specification).order("id ASC").limit(batch_size).to_a
-      end
-
-      def get_remaining_count(fetch_specification)
-        Record.remaining_for(fetch_specification).count
-      end
-
-      def obtain_lock_for_process(fetch_specification, process_uuid, clock:)
-        Lock.obtain(fetch_specification, process_uuid, clock: clock)
-      end
-
-      def release_lock_for_process(fetch_specification, process_uuid)
-        Lock.release(fetch_specification, process_uuid)
+      def with_next_batch(fetch_specification, batch_size, consumer_uuid, locking, clock, &block)
+        if locking
+          with_next_locking_batch(fetch_specification, batch_size, consumer_uuid, clock, &block)
+        else
+          with_next_non_locking_batch(fetch_specification, batch_size, &block)
+        end
       end
 
       def mark_as_enqueued(record, now)
@@ -150,6 +144,131 @@ module RubyEventStore
       rescue ::ActiveRecord::LockWaitTimeout
         :lock_timeout
       end
+
+      private
+
+      def with_next_locking_batch(fetch_specification, batch_size, consumer_uuid, clock, &block)
+        BatchResult.empty.tap do |result|
+          obtained_lock = obtain_lock_for_process(fetch_specification, consumer_uuid, clock: clock)
+          case obtained_lock
+            when :deadlocked
+              logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
+              metrics.write_operation_result("obtain", "deadlocked")
+              return BatchResult.empty
+            when :lock_timeout
+              logger.warn "Obtaining lock for split_key '#{fetch_specification.split_key}' failed (lock timeout)"
+              metrics.write_operation_result("obtain", "lock_timeout")
+              return BatchResult.empty
+            when :taken
+              logger.debug "Obtaining lock for split_key '#{fetch_specification.split_key}' unsuccessful (taken)"
+              metrics.write_operation_result("obtain", "taken")
+              return BatchResult.empty
+          end
+
+          Consumer::MAXIMUM_BATCH_FETCHES_IN_ONE_LOCK.times do
+            batch = retrieve_batch(fetch_specification, batch_size).to_a
+            break if batch.empty?
+            batch.each do |record|
+              handle_execution(result) do
+                block.call(record)
+              end
+            end
+            case (refresh_result = obtained_lock.refresh(clock: clock))
+              when :ok
+              when :deadlocked
+                logger.warn "Refreshing lock for split_key '#{lock.split_key}' failed (deadlock)"
+                metrics.write_operation_result("refresh", "deadlocked")
+                break
+              when :lock_timeout
+                logger.warn "Refreshing lock for split_key '#{lock.split_key}' failed (lock timeout)"
+                metrics.write_operation_result("refresh", "lock_timeout")
+                break
+              when :stolen
+                logger.debug "Refreshing lock for split_key '#{lock.split_key}' unsuccessful (stolen)"
+                metrics.write_operation_result("refresh", "stolen")
+                break
+              else
+                raise "Unexpected result #{refresh_result}"
+            end
+          end
+
+          case release_lock_for_process(fetch_specification, consumer_uuid)
+            when :deadlocked
+              logger.warn "Releasing lock for split_key '#{fetch_specification.split_key}' failed (deadlock)"
+              metrics.write_operation_result("release", "deadlocked")
+            when :lock_timeout
+              logger.warn "Releasing lock for split_key '#{fetch_specification.split_key}' failed (lock timeout)"
+              metrics.write_operation_result("release", "lock_timeout")
+            when :not_taken_by_this_process
+              logger.debug "Releasing lock for split_key '#{fetch_specification.split_key}' failed (not taken by this process)"
+              metrics.write_operation_result("release", "not_taken_by_this_process")
+          end
+          instrument_batch_result(fetch_specification, result)
+        end
+      end
+
+      def with_next_non_locking_batch(fetch_specification, batch_size, &block)
+        BatchResult.empty.tap do |result|
+          Record.transaction do
+            batch = retrieve_batch(fetch_specification, batch_size).lock("FOR UPDATE SKIP LOCKED")
+            break if batch.empty?
+            batch.each do |record|
+              handle_execution(result) do
+                block.call(record)
+              end
+            end
+          end
+
+          instrument_batch_result(fetch_specification, result)
+        end
+      end
+
+      def instrument_batch_result(fetch_specification, result)
+        metrics.write_point_queue(
+          enqueued: result.success_count,
+          failed: result.failed_count,
+          format: fetch_specification.message_format,
+          split_key: fetch_specification.split_key,
+          remaining: Record.remaining_for(fetch_specification).count
+        )
+
+        logger.info "Sent #{result.success_count} messages from outbox table"
+      end
+
+      def handle_execution(batch_result)
+        retried = false
+        yield
+        batch_result.count_success!
+      rescue RetriableRedisError => error
+        if retried
+          batch_result.count_failed!
+          log_error(error)
+        else
+          retried = true
+          retry
+        end
+      rescue => error
+        batch_result.count_failed!
+        log_error(error)
+      end
+
+      def log_error(e)
+        e.full_message.split($/).each { |line| logger.error(line) }
+      end
+
+      def retrieve_batch(fetch_specification, batch_size)
+        Record.remaining_for(fetch_specification).order("id ASC").limit(batch_size)
+      end
+
+      def obtain_lock_for_process(fetch_specification, process_uuid, clock:)
+        Lock.obtain(fetch_specification, process_uuid, clock: clock)
+      end
+
+      def release_lock_for_process(fetch_specification, process_uuid)
+        Lock.release(fetch_specification, process_uuid)
+      end
+
+      attr_reader :logger, :metrics
     end
   end
 end

data/lib/ruby_event_store/outbox/runner.rb

@@ -8,12 +8,14 @@ module RubyEventStore
         @logger = logger
         @sleep_on_empty = configuration.sleep_on_empty
         @split_keys = configuration.split_keys
+        @locking = configuration.locking
         @gracefully_shutting_down = false
         prepare_traps
       end
 
       def run
         logger.info("Initiated RubyEventStore::Outbox v#{VERSION}")
+        logger.info("Using #{@locking ? "locking" : "non-locking"} mode")
         logger.info("Handling split keys: #{split_keys ? split_keys.join(", ") : "(all of them)"}")
 
         while !@gracefully_shutting_down

data/lib/ruby_event_store/outbox/version.rb

@@ -2,6 +2,6 @@
 
 module RubyEventStore
   module Outbox
-    VERSION = "0.0.30"
+    VERSION = "0.1.0"
   end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: ruby_event_store-outbox
 version: !ruby/object:Gem::Version
-  version: 0.0.30
+  version: 0.1.0
 platform: ruby
 authors:
 - Arkency
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-03-25 00:00:00.000000000 Z
+date: 2025-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby_event_store
@@ -38,7 +37,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '6.0'
-description:
 email: dev@arkency.com
 executables:
 - res_outbox
@@ -79,7 +77,6 @@ metadata:
   source_code_uri: https://github.com/RailsEventStore/rails_event_store
   bug_tracker_uri: https://github.com/RailsEventStore/rails_event_store/issues
   rubygems_mfa_required: 'true'
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -94,8 +91,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
-signing_key:
+rubygems_version: 3.6.6
 specification_version: 4
 summary: Active Record based outbox for Ruby Event Store
 test_files: []