sbmt-outbox 5.0.0

data/app/interactors/sbmt/outbox/process_item.rb

@@ -0,0 +1,269 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    class ProcessItem < Sbmt::Outbox::DryInteractor
+      param :item_class, reader: :private
+      param :item_id, reader: :private
+
+      METRICS_COUNTERS = %i[error_counter retry_counter sent_counter fetch_error_counter discarded_counter].freeze
+
+      delegate :log_success, :log_failure, to: "Sbmt::Outbox.logger"
+      delegate :item_process_middlewares, to: "Sbmt::Outbox"
+      delegate :box_type, :box_name, :owner, to: :item_class
+
+      attr_accessor :process_latency
+
+      def call
+        log_success(
+          "Start processing #{box_type} item.\n" \
+          "Record: #{item_class.name}##{item_id}"
+        )
+
+        item = nil
+
+        item_class.transaction do
+          item = yield fetch_item
+
+          if item.processed_at?
+            item.config.retry_strategies.each do |retry_strategy|
+              yield check_retry_strategy(item, retry_strategy)
+            end
+          else
+            self.process_latency = Time.current - item.created_at
+          end
+
+          middlewares = Middleware::Builder.new(item_process_middlewares)
+          payload = yield build_payload(item)
+          transports = yield fetch_transports(item)
+
+          middlewares.call(item) do
+            transports.each do |transport|
+              yield process_item(transport, item, payload)
+            end
+
+            track_successed(item)
+
+            Success(item)
+          end
+        rescue Dry::Monads::Do::Halt => e
+          e.result
+        rescue => e
+          track_failed(e, item)
+          Failure(e.message)
+        end
+      ensure
+        report_metrics(item)
+      end
+      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+
+      private
+
+      def fetch_item
+        item = item_class
+          .lock("FOR UPDATE")
+          .find_by(id: item_id)
+
+        unless item
+          track_failed("not found")
+          return Failure(:not_found)
+        end
+
+        unless item.for_processing?
+          log_error("already processed")
+          counters[:fetch_error_counter] += 1
+          return Failure(:already_processed)
+        end
+
+        Success(item)
+      end
+
+      def check_retry_strategy(item, retry_strategy)
+        result = retry_strategy.call(item)
+
+        return Success() if result.success?
+
+        case result.failure
+        when :skip_processing
+          Failure(:skip_processing)
+        when :discard_item
+          track_discarded(item)
+          Failure(:discard_item)
+        else
+          track_failed("retry stratagy returned unknown failure: #{result.failure}")
+          Failure(:retry_strategy_failure)
+        end
+      end
+
+      def build_payload(item)
+        builder = item.payload_builder
+
+        if builder
+          payload = item.payload_builder.call(item)
+          return payload if payload.success?
+
+          track_failed("payload builder returned failure: #{payload.failure}", item)
+          Failure(:payload_failure)
+        else
+          Success(item.payload)
+        end
+      end
+
+      def fetch_transports(item)
+        transports = item.transports
+        return Success(transports) if transports.present?
+
+        track_failed("missing transports", item)
+        Failure(:missing_transports)
+      end
+
+      # rubocop:disable Metrics/MethodLength
+      def process_item(transport, item, payload)
+        transport_error = nil
+
+        result = item_class.transaction(requires_new: true) do
+          transport.call(item, payload)
+        rescue => e
+          transport_error = e
+          raise ActiveRecord::Rollback
+        end
+
+        if transport_error
+          track_failed(transport_error, item)
+          return Failure(:transport_failure)
+        end
+
+        case result
+        when Dry::Monads::Result
+          if result.failure?
+            track_failed("transport #{transport} returned failure: #{result.failure}", item)
+            Failure(:transport_failure)
+          else
+            Success()
+          end
+        when false
+          track_failed("transport #{transport} returned #{result.inspect}", item)
+          Failure(:transport_failure)
+        else
+          Success()
+        end
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      def track_failed(ex_or_msg, item = nil)
+        log_error(ex_or_msg, item)
+
+        item&.touch_processed_at
+        item&.add_error(ex_or_msg)
+
+        if item.nil?
+          report_error(ex_or_msg)
+          counters[:fetch_error_counter] += 1
+        elsif item.max_retries_exceeded?
+          report_error(ex_or_msg, item)
+          counters[:error_counter] += 1
+          item.failed!
+        else
+          counters[:retry_counter] += 1
+          item.pending!
+        end
+      end
+
+      def track_successed(item)
+        msg = "Successfully delivered #{box_type} item.\n" \
+              "Record: #{item_class.name}##{item_id}.\n" \
+              "#{item.log_details.to_json}"
+        log_success(msg)
+
+        item.touch_processed_at
+        item.delivered!
+
+        counters[:sent_counter] += 1
+      end
+
+      def track_discarded(item)
+        msg = "Skipped and discarded #{box_type} item.\n" \
+              "Record: #{item_class.name}##{item_id}.\n" \
+              "#{item.log_details.to_json}"
+        log_success(msg)
+
+        item.touch_processed_at
+        item.discarded!
+
+        counters[:discarded_counter] += 1
+      end
+
+      def log_error(ex_or_msg, item = nil)
+        text = format_exception_error(ex_or_msg)
+
+        msg = "Failed processing #{box_type} item with error: #{text}.\n" \
+              "Record: #{item_class.name}##{item_id}.\n" \
+              "#{item&.log_details&.to_json}"
+
+        log_failure(msg, backtrace: format_backtrace(ex_or_msg))
+      end
+
+      def format_exception_error(e)
+        text = if e.respond_to?(:cause) && !e.cause.nil?
+          "#{format_exception_error(e.cause)}. "
+        else
+          ""
+        end
+
+        if e.respond_to?(:message)
+          "#{text}#{e.class.name} #{e.message}"
+        else
+          "#{text}#{e}"
+        end
+      end
+
+      def format_backtrace(e)
+        if e.respond_to?(:backtrace) && !e.backtrace.nil?
+          e.backtrace.join("\n")
+        end
+      end
+
+      def report_error(ex_or_msg, item = nil)
+        Outbox.error_tracker.error(
+          ex_or_msg,
+          box_name: item_class.box_name,
+          item_class: item_class.name,
+          item_id: item_id,
+          item_details: item&.log_details&.to_json
+        )
+      end
+
+      def report_metrics(item)
+        labels = labels_for(item)
+
+        METRICS_COUNTERS.each do |counter_name|
+          Yabeda
+            .outbox
+            .send(counter_name)
+            .increment(labels, by: counters[counter_name])
+        end
+
+        track_process_latency(labels) if process_latency
+
+        return unless counters[:sent_counter].positive?
+
+        Yabeda
+          .outbox
+          .last_sent_event_id
+          .set(labels, item_id)
+      end
+
+      def labels_for(item)
+        {type: box_type, name: box_name, owner: owner, partition: item&.partition}
+      end
+
+      def counters
+        @counters ||= Hash.new(0)
+      end
+
+      def track_process_latency(labels)
+        Yabeda.outbox.process_latency.measure(labels, process_latency.round(3))
+      end
+    end
+  end
+end

data/app/interactors/sbmt/outbox/retry_strategies/compacted_log.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module RetryStrategies
+      class CompactedLog < Outbox::DryInteractor
+        param :outbox_item
+
+        def call
+          unless outbox_item.has_attribute?(:event_key)
+            return Failure(:missing_event_key)
+          end
+
+          if outbox_item.event_key.nil?
+            return Failure(:empty_event_key)
+          end
+
+          if delivered_later?
+            Failure(:discard_item)
+          else
+            Success()
+          end
+        end
+
+        private
+
+        def delivered_later?
+          scope = outbox_item.class
+            .where("id > ?", outbox_item)
+            .where(event_key: outbox_item.event_key, status: Sbmt::Outbox::BaseItem.statuses[:delivered])
+
+          if outbox_item.has_attribute?(:event_name) && outbox_item.event_name.present?
+            scope = scope.where(event_name: outbox_item.event_name)
+          end
+
+          scope.exists?
+        end
+      end
+    end
+  end
+end

data/app/interactors/sbmt/outbox/retry_strategies/exponential_backoff.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module RetryStrategies
+      class ExponentialBackoff < Outbox::DryInteractor
+        param :outbox_item
+
+        def call
+          delay = backoff(outbox_item.config).interval_at(outbox_item.errors_count - 1)
+
+          still_early = outbox_item.processed_at + delay.seconds > Time.current
+
+          if still_early
+            Failure(:skip_processing)
+          else
+            Success()
+          end
+        end
+
+        private
+
+        def backoff(config)
+          @backoff ||= ::ExponentialBackoff.new([
+            config.minimal_retry_interval,
+            config.maximal_retry_interval
+          ]).tap do |x|
+            x.multiplier = config.multiplier_retry_interval
+          end
+        end
+      end
+    end
+  end
+end

data/app/jobs/sbmt/outbox/base_delete_stale_items_job.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "redlock"
+
+module Sbmt
+  module Outbox
+    class BaseDeleteStaleItemsJob < Outbox.active_job_base_class
+      MIN_RETENTION_PERIOD = 1.day
+      LOCK_TTL = 10_800_000
+      BATCH_SIZE = 1000
+      SLEEP_TIME = 1
+
+      class << self
+        def enqueue
+          item_classes.each do |item_class|
+            perform_later(item_class.to_s)
+          end
+        end
+
+        def item_classes
+          raise NotImplementedError
+        end
+      end
+
+      delegate :config, :logger, to: "Sbmt::Outbox"
+      delegate :box_type, :box_name, to: :item_class
+
+      attr_accessor :item_class
+
+      def perform(item_class_name)
+        self.item_class = item_class_name.constantize
+
+        client = if Gem::Version.new(Redlock::VERSION) >= Gem::Version.new("2.0.0")
+          RedisClientFactory.build(config.redis)
+        else
+          Redis.new(config.redis)
+        end
+
+        lock_manager = Redlock::Client.new([client], retry_count: 0)
+
+        lock_manager.lock("#{self.class.name}:#{item_class_name}:lock", LOCK_TTL) do |locked|
+          if locked
+            duration = item_class.config.retention
+
+            validate_retention!(duration)
+
+            logger.with_tags(box_type: box_type, box_name: box_name) do
+              delete_stale_items(Time.current - duration)
+            end
+          else
+            logger.log_info("Failed to acquire lock #{self.class.name}:#{item_class_name}")
+          end
+        end
+      end
+
+      private
+
+      def validate_retention!(duration)
+        return if duration >= MIN_RETENTION_PERIOD
+
+        raise "Retention period for #{box_name} must be longer than #{MIN_RETENTION_PERIOD.inspect}"
+      end
+
+      def delete_stale_items(waterline)
+        logger.log_info("Start deleting #{box_type} items for #{box_name} older than #{waterline}")
+
+        scope = item_class.where("created_at < ?", waterline)
+
+        while (ids = scope.limit(BATCH_SIZE).ids).present?
+          item_class.where(id: ids).delete_all
+          sleep SLEEP_TIME
+        end
+
+        logger.log_info("Successfully deleted #{box_type} items for #{box_name} older than #{waterline}")
+      end
+    end
+  end
+end

data/app/jobs/sbmt/outbox/delete_stale_inbox_items_job.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    class DeleteStaleInboxItemsJob < BaseDeleteStaleItemsJob
+      queue_as :inbox
+
+      class << self
+        def item_classes
+          Outbox.inbox_item_classes
+        end
+      end
+    end
+  end
+end

data/app/jobs/sbmt/outbox/delete_stale_outbox_items_job.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    class DeleteStaleOutboxItemsJob < BaseDeleteStaleItemsJob
+      queue_as :outbox
+
+      class << self
+        def item_classes
+          Outbox.outbox_item_classes
+        end
+      end
+    end
+  end
+end

data/app/models/sbmt/outbox/base_item.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    class BaseItem < Outbox.active_record_base_class
+      self.abstract_class = true
+
+      class << self
+        delegate :owner, to: :config
+
+        def box_type
+          raise NotImplementedError
+        end
+
+        def box_name
+          @box_name ||= name.underscore
+        end
+
+        def config
+          @config ||= lookup_config.new(box_name)
+        end
+
+        def partition_buckets
+          @partition_buckets ||=
+            (0...config.partition_size)
+              .to_a
+              .each_with_object({}) do |x, m|
+                m[x] = (0...config.bucket_size)
+                  .to_a
+                  .select { |p| p % config.partition_size == x }
+              end
+        end
+
+        def bucket_partitions
+          @bucket_partitions ||=
+            partition_buckets.each_with_object({}) do |(partition, buckets), m|
+              buckets.each do |bucket|
+                m[bucket] = partition
+              end
+            end
+        end
+      end
+
+      enum status: {
+        pending: 0,
+        failed: 1,
+        delivered: 2,
+        discarded: 3
+      }
+
+      scope :for_processing, -> { where(status: :pending) }
+
+      validates :uuid, :event_key, :bucket, :payload, presence: true
+
+      delegate :box_name, :config, to: "self.class"
+
+      after_initialize do
+        self.uuid ||= SecureRandom.uuid if has_attribute?(:uuid)
+      end
+
+      def payload
+        if has_attribute?(:proto_payload)
+          proto_payload
+        else
+          self[:payload]
+        end
+      end
+
+      def payload=(value)
+        if has_attribute?(:proto_payload)
+          self.proto_payload = value
+        else
+          self[:payload] = value
+        end
+      end
+
+      def for_processing?
+        pending?
+      end
+
+      def options
+        options = (self[:options] || {}).symbolize_keys
+        options = default_options.deep_merge(extra_options).deep_merge(options)
+        options.symbolize_keys
+      end
+
+      def transports
+        if config.transports.empty?
+          raise Error, "Transports are not defined"
+        end
+
+        if has_attribute?(:event_name)
+          config.transports.fetch(event_name)
+        else
+          config.transports.fetch(:_all_)
+        end
+      end
+
+      def log_details
+        default_log_details.deep_merge(extra_log_details)
+      end
+
+      def payload_builder
+        nil
+      end
+
+      def touch_processed_at
+        self.processed_at = Time.current
+      end
+
+      def retriable?
+        config.max_retries > 0
+      end
+
+      def max_retries_exceeded?
+        return true unless retriable?
+
+        errors_count > config.max_retries
+      end
+
+      def increment_errors_counter
+        increment(:errors_count)
+      end
+
+      def add_error(ex_or_msg)
+        increment_errors_counter
+
+        return unless has_attribute?(:error_log)
+
+        self.error_log = "-----\n#{Time.zone.now} \n #{ex_or_msg}\n #{add_backtrace(ex_or_msg)}"
+      end
+
+      def partition
+        self.class.bucket_partitions.fetch(bucket)
+      end
+
+      private
+
+      def default_options
+        raise NotImplementedError
+      end
+
+      # Override in descendants
+      def extra_options
+        {}
+      end
+
+      def default_log_details
+        raise NotImplementedError
+      end
+
+      # Override in descendants
+      def extra_log_details
+        {}
+      end
+
+      def add_backtrace(ex)
+        return unless ex.respond_to?(:backtrace)
+        return if ex.backtrace.nil?
+
+        ex.backtrace.first(30).join("\n")
+      end
+    end
+  end
+end