rails-transactional-outbox 0.1.0

data/lib/rails_transactional_outbox/outbox_model.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  module OutboxModel
+    extend ActiveSupport::Concern
+
+    included do
+      scope :fetch_processable, lambda { |batch_size|
+        where(processed_at: nil)
+          .lock("FOR UPDATE SKIP LOCKED")
+          .where("retry_at IS NULL OR retry_at <= ?", Time.current)
+          .order(created_at: :asc)
+          .limit(batch_size)
+      }
+
+      def self.any_records_to_process?
+        where(processed_at: nil)
+          .where("retry_at IS NULL OR retry_at <= ?", Time.current)
+          .exists?
+      end
+
+      def self.outbox_encrypt_json_for(*encryptable_json_attributes)
+        encryptable_json_attributes.each do |attribute|
+          define_method "#{attribute}=" do |payload|
+            super(payload.to_json)
+          end
+
+          define_method "transformed_#{attribute}" do
+            JSON.parse(public_send(attribute)).symbolize_keys
+          end
+        end
+      end
+    end
+
+    def transformed_arguments
+      arguments.to_h.symbolize_keys
+    end
+
+    def transformed_changeset
+      changeset.to_h.symbolize_keys
+    end
+
+    def processed?
+      processed_at.present?
+    end
+
+    def failed?
+      failed_at.present?
+    end
+
+    def handle_error(raised_error, clock: Time, backoff_multiplier: 5)
+      @error = raised_error
+      self.error_class = raised_error.class
+      self.error_message = raised_error.message
+      self.failed_at = clock.current
+      self.attempts ||= 0
+      self.attempts += 1
+      self.retry_at = clock.current.advance(
+        seconds: RailsTransactionalOutbox::ExponentialBackoff.backoff_for(backoff_multiplier, attempts)
+      )
+    end
+
+    def error
+      @error || error_class.constantize.new(error_message)
+    end
+
+    def event_type
+      RailsTransactionalOutbox::EventType.resolve_from_event_name(event_name).to_sym
+    end
+
+    def infer_model
+      model_klass = resource_class.constantize
+      model_klass.find(resource_id)
+    rescue ActiveRecord::RecordNotFound
+      model_klass.new(id: resource_id) if RailsTransactionalOutbox::EventType.new(event_type).destroy?
+    end
+  end
+end

data/lib/rails_transactional_outbox/railtie.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  class Railtie < Rails::Railtie
+    railtie_name :rails_transactional_outbox
+
+    rake_tasks do
+      load "tasks/rails_transactional_outbox.rake"
+    end
+  end
+end

data/lib/rails_transactional_outbox/record_processor.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  class RecordProcessor
+    attr_reader :config
+    private     :config
+
+    def initialize(config: RailsTransactionalOutbox.configuration)
+      @config = config
+    end
+
+    def call(record)
+      applicable_record_processors = record_processors.select { |processor| processor.applies?(record) }
+      applicable_record_processors.any? or raise ProcessorNotFoundError.new(record)
+
+      applicable_record_processors.each { |processor| processor.call(record) }
+    end
+
+    delegate :record_processors, to: :config
+
+    class ProcessorNotFoundError < StandardError
+      attr_reader :record
+      private     :record
+
+      def initialize(record)
+        super()
+        @record = record
+      end
+
+      def to_s
+        "no processor was found for record with ID: #{record.id}, context: #{record.context}"
+      end
+    end
+  end
+end

data/lib/rails_transactional_outbox/record_processors/active_record_processor.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  class RecordProcessors
+    class ActiveRecordProcessor < RailsTransactionalOutbox::RecordProcessors::BaseProcessor
+      ACTIVE_RECORD_CONTEXT = "active_record"
+      private_constant :ACTIVE_RECORD_CONTEXT
+
+      def self.context
+        ACTIVE_RECORD_CONTEXT
+      end
+
+      def applies?(record)
+        record.context == ACTIVE_RECORD_CONTEXT
+      end
+
+      def call(record)
+        model = record.infer_model or raise CouldNotFindModelError.new(record)
+        model.previous_changes = record.transformed_changeset.with_indifferent_access
+        model.reliable_after_commit_callbacks.for_event_type(record.event_type).each do |callback|
+          callback.call(model)
+        end
+      end
+
+      class CouldNotFindModelError < StandardError
+        attr_reader :record
+
+        def initialize(record)
+          super()
+          @record = record
+        end
+
+        def to_s
+          "could not find model for outbox record: #{record.id}"
+        end
+      end
+    end
+  end
+end

data/lib/rails_transactional_outbox/record_processors/base_processor.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  class RecordProcessors
+    class BaseProcessor
+      def applies?(_record)
+        raise "implement me"
+      end
+
+      def call(_record)
+        raise "implement me"
+      end
+    end
+  end
+end

data/lib/rails_transactional_outbox/record_processors.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  class RecordProcessors
+  end
+end

data/lib/rails_transactional_outbox/reliable_model/reliable_callback.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  module ReliableModel
+    class ReliableCallback
+      attr_reader :callback, :options
+      private     :callback, :options
+
+      def initialize(callback, options)
+        @callback = callback
+        @options = options
+      end
+
+      def for_event?(event_type)
+        on.include?(event_type.to_sym)
+      end
+
+      def call(model)
+        return unless execute?(model)
+
+        model.instance_exec(&callback)
+      end
+
+      private
+
+      def on
+        Array(options.fetch(:on, [])).map(&:to_sym)
+      end
+
+      def execute?(model)
+        if options.key?(:if)
+          model.instance_exec(&options[:if])
+        elsif options.key?(:unless)
+          !model.instance_exec(&options[:unless])
+        else
+          true
+        end
+      end
+    end
+  end
+end

data/lib/rails_transactional_outbox/reliable_model/reliable_callbacks_registry.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  module ReliableModel
+    class ReliableCallbacksRegistry
+      include Enumerable
+
+      delegate :each, to: :registry
+
+      attr_reader :registry
+      private     :registry
+
+      def initialize
+        @registry = []
+      end
+
+      def <<(item)
+        registry << item
+      end
+
+      def for_event_type(event_type)
+        registry.select { |cb| cb.for_event?(event_type) }
+      end
+    end
+  end
+end

data/lib/rails_transactional_outbox/reliable_model.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  module ReliableModel
+    extend ActiveSupport::Concern
+
+    NOT_PROVIDED = Object.new.freeze
+    private_constant :NOT_PROVIDED
+
+    included do
+      after_create :transactional_outbox_insert_model_created
+      after_update :transactional_outbox_insert_model_updated
+      after_destroy :transactional_outbox_insert_model_destroyed
+
+      def self.reliable_after_commit_callbacks
+        @reliable_after_commit_callbacks ||= RailsTransactionalOutbox::ReliableModel::ReliableCallbacksRegistry.new
+      end
+
+      def self.reliable_after_commit(method_name = NOT_PROVIDED, options = {}, &block)
+        if block
+          callback_proc = block
+        else
+          raise ArgumentError.new("You must provide a block or a method name") unless method_name.is_a?(Symbol)
+
+          callback_proc = -> { send(method_name) }
+        end
+
+        final_options = options.reverse_merge(on: %i[create update destroy])
+        reliable_after_commit_callbacks << ReliableCallback.new(callback_proc, final_options)
+      end
+
+      def self.reliable_after_create_commit(method_name = NOT_PROVIDED, options = {}, &block)
+        reliable_after_commit(method_name, options.merge(on: :create), &block)
+      end
+
+      def self.reliable_after_update_commit(method_name = NOT_PROVIDED, options = {}, &block)
+        reliable_after_commit(method_name, options.merge(on: :update), &block)
+      end
+
+      def self.reliable_after_destroy_commit(method_name = NOT_PROVIDED, options = {}, &block)
+        reliable_after_commit(method_name, options.merge(on: :destroy), &block)
+      end
+
+      def self.reliable_after_save_commit(method_name = NOT_PROVIDED, options = {}, &block)
+        reliable_after_commit(method_name, options.merge(on: %i[create update]), &block)
+      end
+
+      alias_method :original_previous_changes, :previous_changes
+
+      def previous_changes
+        @previous_changes || original_previous_changes
+      end
+
+      def previous_changes=(changeset)
+        @previous_changes = changeset
+      end
+
+      private
+
+      def transactional_outbox_insert_model_created
+        transactional_outbox_entry_factory.build(self, :create).save
+      end
+
+      def transactional_outbox_insert_model_updated
+        transactional_outbox_entry_factory.build(self, :update).save!
+      end
+
+      def transactional_outbox_insert_model_destroyed
+        transactional_outbox_entry_factory.build(self, :destroy).save!
+      end
+
+      def transactional_outbox_entry_factory
+        @transactional_outbox_entry_factory ||= RailsTransactionalOutbox::OutboxEntryFactory.new
+      end
+    end
+
+    def reliable_after_commit_callbacks
+      self.class.reliable_after_commit_callbacks
+    end
+  end
+end

data/lib/rails_transactional_outbox/runner.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  class Runner
+    attr_reader :config, :id
+    private     :config
+
+    def initialize(config: RailsTransactionalOutbox.configuration)
+      @id = SecureRandom.uuid
+      @config = config
+      logger.push_tags("RailsTransactionalOutbox::Runner #{id}") if logger.respond_to?(:push_tags)
+    end
+
+    def start
+      log("started")
+      instrument("rails_transactional_outbox.started")
+      @should_stop = false
+      ensure_database_connection!
+      loop do
+        if @should_stop
+          instrument("rails_transactional_outbox.shutting_down")
+          log("shutting down")
+          break
+        end
+        entries = process_entries
+        instrument("rails_transactional_outbox.heartbeat")
+        sleep sleep_interval_for(entries)
+      end
+    rescue => e
+      error_handler.capture_exception(e)
+      log("error: #{e} #{e.message}")
+      instrument("rails_transactional_outbox.error", error: e, error_message: e.message)
+      raise e
+    end
+
+    def stop
+      log("Rails Transactional Outbox Worker stopping")
+      instrument("rails_transactional_outbox.stopped")
+      @should_stop = true
+    end
+
+    private
+
+    delegate :error_handler, :transactional_outbox_worker_sleep_seconds,
+      :transactional_outbox_worker_idle_delay_multiplier, :database_connection_provider, :logger, to: :config
+    delegate :monitor, to: RailsTransactionalOutbox
+
+    def process_entries
+      tracer.trace("rails_transactional_outbox_entries_processor") do
+        outbox_entries_processor.call do |record|
+          if record.failed?
+            instrument("rails_transactional_outbox.record_processing_failed", outbox_record: record)
+            error("failed to process #{record.inspect}")
+            error_handler.capture_exception(record.error)
+          else
+            debug("processed #{record.inspect}")
+            instrument("rails_transactional_outbox.record_processed", outbox_record: record)
+          end
+        end
+      end
+    end
+
+    def ensure_database_connection!
+      database_connection_provider.connection.reconnect!
+    end
+
+    def outbox_entries_processor
+      @outbox_entries_processor ||= RailsTransactionalOutbox::OutboxEntriesProcessor.new
+    end
+
+    def log(message)
+      logger.info("#{log_prefix} #{message}")
+    end
+
+    def debug(message)
+      logger.debug("#{log_prefix} #{message}")
+    end
+
+    def error(message)
+      logger.error("#{log_prefix} #{message}")
+    end
+
+    def log_prefix
+      "[Rails Transactional Outbox Worker] "
+    end
+
+    def instrument(*args, **kwargs)
+      monitor.instrument(*args, **kwargs) do
+        yield if block_given?
+      end
+    end
+
+    def tracer
+      @tracer ||= if Object.const_defined?(:Datadog)
+        RailsTransactionalOutbox::Tracers::DatadogTracer.new
+      else
+        RailsTransactionalOutbox::Tracers::NullTracer
+      end
+    end
+
+    def sleep_interval_for(entries)
+      RailsTransactionalOutbox::RunnerSleepInterval.interval_for(entries, transactional_outbox_worker_sleep_seconds,
+        transactional_outbox_worker_idle_delay_multiplier)
+    end
+  end
+end

data/lib/rails_transactional_outbox/runner_sleep_interval.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  class RunnerSleepInterval
+    # TODO: maybe apply some backoff or longer pause if there were no entries to be processed?
+    def self.interval_for(processed_entries, sleep_seconds, idle_delay_multiplier)
+      if processed_entries.any?
+        sleep_seconds
+      else
+        sleep_seconds * idle_delay_multiplier
+      end
+    end
+  end
+end

data/lib/rails_transactional_outbox/tracers/datadog_tracer.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  module Tracers
+    class DatadogTracer
+      SERVICE_NAME = "rails_transactional_outbox_worker"
+      private_constant :SERVICE_NAME
+
+      def self.service_name
+        SERVICE_NAME
+      end
+
+      def trace(event_name)
+        tracer.trace(event_name, span_type: "worker", service: self.class.service_name,
+          on_error: error_handler) do |_span|
+          yield
+        end
+      end
+
+      private
+
+      def tracer
+        if Datadog.respond_to?(:tracer)
+          Datadog.tracer
+        else
+          Datadog::Tracing
+        end
+      end
+
+      def error_handler
+        ->(span, error) { span.set_error(error) }
+      end
+    end
+  end
+end

data/lib/rails_transactional_outbox/tracers/null_tracer.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  module Tracers
+    class NullTracer
+      def self.trace(_event_name); end
+    end
+  end
+end

data/lib/rails_transactional_outbox/tracers.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  module Tracers
+    autoload :DatadogTracer, "rails_transactional_outbox/tracers/datadog_tracer"
+  end
+end

data/lib/rails_transactional_outbox/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class RailsTransactionalOutbox
+  module Version
+  end
+  VERSION = "0.1.0"
+end

data/lib/rails_transactional_outbox.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+require "logger"
+require "dry-monitor"
+require "sigurd"
+require "concurrent-ruby"
+
+class RailsTransactionalOutbox
+  def self.loader
+    @loader ||= Zeitwerk::Loader.for_gem.tap do |loader|
+      loader.ignore(
+        "#{__dir__}/rails-transactional-outbox.rb",
+        "#{__dir__}/tracers/datadog_tracer.rb"
+      )
+    end
+  end
+
+  def self.configuration
+    @configuration ||= RailsTransactionalOutbox::Configuration.new
+  end
+
+  def self.configure
+    yield configuration
+  end
+
+  def self.monitor
+    @monitor ||= RailsTransactionalOutbox::Monitor.new
+  end
+
+  def self.reset
+    @configuration = nil
+  end
+
+  def self.outbox_worker_health_check
+    @outbox_worker_health_check ||= RailsTransactionalOutbox::HealthCheck.new
+  end
+
+  def self.enable_outbox_worker_healthcheck
+    monitor.subscribe("rails_transactional_outbox.started") { outbox_worker_health_check.register_heartbeat }
+    monitor.subscribe("rails_transactional_outbox.stopped") { outbox_worker_health_check.worker_stopped }
+    monitor.subscribe("rails_transactional_outbox.heartbeat") { outbox_worker_health_check.register_heartbeat }
+  end
+
+  def self.start_outbox_worker(threads_number: 1)
+    runners = (1..threads_number).map { RailsTransactionalOutbox::Runner.new(config: configuration) }
+    executor = Sigurd::Executor.new(runners, sleep_seconds: 5, logger: configuration.logger)
+    signal_handler = Sigurd::SignalHandler.new(executor)
+    signal_handler.run!
+  end
+end
+
+RailsTransactionalOutbox.loader.setup
+require "rails_transactional_outbox/railtie" if defined?(Rails)

data/lib/tasks/rails_transactional_outbox.rake

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+namespace :rails_transactional_outbox do
+  desc "Starts the RailsTransactionalOutbox worker"
+  task worker: :environment do
+    $stdout.sync = true
+    Rails.logger.info("Running rails_transactional_outbox:worker rake task.")
+    threads_number = ENV.fetch("RAILS_TRANSACTIONAL_OUTBOX_THREADS_NUMBER", 1).to_i
+    RailsTransactionalOutbox.start_outbox_worker(threads_number: threads_number)
+  end
+end

data/rails-transactional-outbox.gemspec

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative "lib/rails_transactional_outbox/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "rails-transactional-outbox"
+  spec.version       = RailsTransactionalOutbox::VERSION
+  spec.authors       = ["Karol Galanciak"]
+  spec.email         = ["karol.galanciak@gmail.com"]
+
+  spec.summary       = "An implementation of transactional outbox pattern to be used with Rails."
+  spec.description   = "An implementation of transactional outbox pattern to be used with Rails."
+  spec.homepage      = "https://github.com/BookingSync/rails-transactional-outbox"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
+
+  # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/BookingSync/rails-transactional-outbox"
+  spec.metadata["changelog_uri"] = "https://github.com/BookingSync/rails-transactional-outbox/blob/master/CHANGELOG.md"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    %x(git ls-files -z).split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "bin"
+  spec.executables   = %w[rails_transactional_outbox_health_check]
+  spec.require_paths = ["lib"]
+
+  # Uncomment to register a new dependency of your gem
+  spec.add_dependency "activerecord", ">= 5"
+  spec.add_dependency "activesupport", ">= 3.2"
+  spec.add_dependency "concurrent-ruby"
+  spec.add_dependency "dry-monitor"
+  spec.add_dependency "redis"
+  spec.add_dependency "sigurd"
+  spec.add_dependency "zeitwerk"
+
+  # For more information and examples about making a new gem, checkout our
+  # guide at: https://bundler.io/guides/creating_gem.html
+  spec.metadata["rubygems_mfa_required"] = "true"
+end