rails-transactional-outbox 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7981d03fa1319ccf83fe5f66ac64869ef024e8add13de49f6af43b26ff39ac3e
-  data.tar.gz: e2f5048f5c4f5700b95b247064cc4e2fe9d8601b6df63d39b1e06896aeb1e915
+  metadata.gz: 20d98c7e305f01ecd8e54b92170a117c01a412d050acab60899835b99a0d7d0c
+  data.tar.gz: 1658aadc54b3a01dd1e48274055befa210638df3b1fdc60bdd5a0bb2f858e1ae
 SHA512:
-  metadata.gz: e903bcb6ee6c1b58d77b7c9b0dff41297e44b81c45457df0888caf6ecbdf2fa0f7254f9424cac14db2d31933e6f9790438cf2a9e232cb6af06405f9985221ceb
-  data.tar.gz: 8296fbab0324e8b0fa6410187e0fb59392121aecfccab2301fecff8fc1b51d815eaa71af78fdbd1d7c67f11573fc06d9598ec9e719d01738fc72d92646a33e0a
+  metadata.gz: 75c6ef520d685476d27fc71f3c3bcac986248ab5d62addd2840238ed020136622cea9c4d20f0ade4fe6d693ffa3545531152d4a57465b72a6cb7a91112a2d5d0
+  data.tar.gz: 9efa9ef99b4c610f89c9ae22d2b5bd0592c0535667d46636e69a8d219cb3e92dda648d315ccf56ca8cfe01f3ae9ccb8588d4f51742072a6309829a68da33614e

data/.rubocop.yml

@@ -151,3 +151,6 @@ RSpec/AnyInstance:
 RSpec/VerifiedDoubles:
   Exclude:
     - 'spec/rails_transactional_outbox/runner_spec.rb'
+
+Layout/LineLength:
+  Max: 125

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## [Unreleased]
 
+## [0.4.0] - 2024-01-25
+
+- add config option to specify causality keys limit
+
+## [0.3.1] - 2023-05-24
+
+- add config option whether to raise error when outbox entry record is not found
+
 ## [0.3.0] - 2022-12-20
 
 - Move to file-based healthchecks, instead of using Redis-based ones.

data/Gemfile.lock

@@ -9,7 +9,7 @@ GIT
 PATH
   remote: .
   specs:
-    rails-transactional-outbox (0.3.0)
+    rails-transactional-outbox (0.4.0)
       activerecord (>= 5)
       activesupport (>= 3.2)
       concurrent-ruby
@@ -55,7 +55,7 @@ GEM
       dry-events (~> 1.0, < 2)
     exponential-backoff (0.0.4)
     ffi (1.15.5)
-    file-based-healthcheck (0.1.1)
+    file-based-healthcheck (0.2.0)
       activesupport (>= 3.2)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
@@ -121,7 +121,7 @@ GEM
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.2.0)
-    zeitwerk (2.6.6)
+    zeitwerk (2.6.12)
 
 PLATFORMS
   x86_64-darwin-18

data/README.md

@@ -39,11 +39,13 @@ Rails.application.config.to_prepare do
     config.transactional_outbox_worker_idle_delay_multiplier = 5 # optional, defaults to 1, if there are no outbox entries to be processed, then the sleep time for the thread will be equal to transactional_outbox_worker_idle_delay_multiplier * transactional_outbox_worker_sleep_seconds
     config.outbox_batch_size = 100 # optional, defaults to 100
     config.add_record_processor(MyCustomOperationProcerssor) # optional, by default it contains only one processor for ActiveRecord, but you could add more
+    config.raise_not_found_model_error = true # optional, defaults to true. Should the error be raised if outbox entry model is not found
 
     config.lock_client = Redlock::Client.new([ENV["REDIS_URL"]]) # required if you want to use RailsTransactionalOutbox::OutboxEntriesProcessors::OrderedByCausalityKeyProcessor, defaults to RailsTransactionalOutbox::NullLockClient. Check its interface and the interface of `redlock` gem. To cut the long story short, when the lock is acquired, a hash with the structure outlined in RailsTransactionalOutbox::NullLockClient should be yielded, if the lock is not acquired, a nil should be yielded.
     config.lock_expiry_time  = 10_000 # not required, defaults to 10_000, the unit is milliseconds
     config.outbox_entries_processor = `RailsTransactionalOutbox::OutboxEntriesProcessors::OrderedByCausalityKeyProcessor`.new # not required, defaults to RailsTransactionalOutbox::OutboxEntriesProcessors::NonOrderedProcessor.new
     config.outbox_entry_causality_key_resolver = ->(model) { model.tenant_id } # not required, defaults to a lambda returning nil. Needed when using `outbox_entry_causality_key_resolver`
+    config.unprocessed_causality_keys_limit = 100_000 # not required, defaults to 10_000. Might be a good idea to decrease the value when you start experiencing OOMs - they are likely to be caused by fetching too many causality keys. It is likely to happen when you have huge amount of records to process.
   end
 end
 ```

data/lib/rails_transactional_outbox/configuration.rb

@@ -5,7 +5,8 @@ class RailsTransactionalOutbox
     attr_accessor :database_connection_provider, :logger, :outbox_model, :transaction_provider
     attr_writer :error_handler, :transactional_outbox_worker_sleep_seconds,
       :transactional_outbox_worker_idle_delay_multiplier, :outbox_batch_size, :outbox_entries_processor,
-      :lock_client, :lock_expiry_time, :outbox_entry_causality_key_resolver
+      :lock_client, :lock_expiry_time, :outbox_entry_causality_key_resolver,
+      :raise_not_found_model_error, :unprocessed_causality_keys_limit
 
     def error_handler
       @error_handler || RailsTransactionalOutbox::ErrorHandlers::NullErrorHandler
@@ -35,6 +36,14 @@ class RailsTransactionalOutbox
       @outbox_entries_processor ||= RailsTransactionalOutbox::OutboxEntriesProcessors::NonOrderedProcessor.new
     end
 
+    def raise_not_found_model_error
+      return @raise_not_found_model_error if defined?(@raise_not_found_model_error)
+
+      true
+    end
+
+    alias_method :raise_not_found_model_error?, :raise_not_found_model_error
+
     def lock_client
       @lock_client || RailsTransactionalOutbox::NullLockClient
     end
@@ -46,5 +55,11 @@ class RailsTransactionalOutbox
     def outbox_entry_causality_key_resolver
       @outbox_entry_causality_key_resolver || ->(_model) {}
     end
+
+    def unprocessed_causality_keys_limit
+      return @unprocessed_causality_keys_limit.to_i if defined?(@unprocessed_causality_keys_limit)
+
+      10_000
+    end
   end
 end

data/lib/rails_transactional_outbox/outbox_model.rb

@@ -24,9 +24,10 @@ class RailsTransactionalOutbox
           .where("retry_at IS NULL OR retry_at <= ?", Time.current)
       }
 
-      def self.unprocessed_causality_keys
+      def self.unprocessed_causality_keys(limit: RailsTransactionalOutbox.configuration.unprocessed_causality_keys_limit)
         processable_now
           .select("causality_key")
+          .limit(limit)
           .distinct
           .pluck(:causality_key)
       end

data/lib/rails_transactional_outbox/record_processors/active_record_processor.rb

@@ -15,7 +15,12 @@ class RailsTransactionalOutbox
       end
 
       def call(record)
-        model = record.infer_model or raise CouldNotFindModelError.new(record)
+        model = record.infer_model
+        if model.nil?
+          raise CouldNotFindModelError.new(record) if RailsTransactionalOutbox.configuration.raise_not_found_model_error?
+
+          return
+        end
         model.previous_changes = record.transformed_changeset.with_indifferent_access
         model.reliable_after_commit_callbacks.for_event_type(record.event_type).each do |callback|
           callback.call(model)

data/lib/rails_transactional_outbox/version.rb

@@ -3,5 +3,5 @@
 class RailsTransactionalOutbox
   module Version
   end
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails-transactional-outbox
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Karol Galanciak
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-12-20 00:00:00.000000000 Z
+date: 2024-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord