sbmt-outbox 5.0.0

data/lib/sbmt/outbox/middleware/open_telemetry/tracing_create_item_middleware.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module Middleware
+      module OpenTelemetry
+        class TracingCreateItemMiddleware
+          def call(item_class, item_attributes)
+            return yield unless defined?(::OpenTelemetry)
+
+            span_attributes = {
+              "messaging.system" => "outbox",
+              "messaging.outbox.box_type" => item_class.box_type.to_s,
+              "messaging.outbox.box_name" => item_class.box_name,
+              "messaging.outbox.owner" => item_class.owner,
+              "messaging.destination" => item_class.name,
+              "messaging.destination_kind" => "database"
+            }
+
+            options = item_attributes[:options] ||= {}
+            headers = options[:headers] || options["headers"]
+            headers ||= options[:headers] ||= {}
+            tracer.in_span(span_name(item_class), attributes: span_attributes.compact, kind: :producer) do
+              ::OpenTelemetry.propagation.inject(headers)
+              yield
+            end
+          end
+
+          private
+
+          def tracer
+            ::Sbmt::Outbox::Instrumentation::OpenTelemetryLoader.instance.tracer
+          end
+
+          def span_name(item_class)
+            "#{item_class.box_type}/#{item_class.box_name} create item"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/middleware/open_telemetry/tracing_item_process_middleware.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module Middleware
+      module OpenTelemetry
+        class TracingItemProcessMiddleware
+          def call(item)
+            return yield unless defined?(::OpenTelemetry)
+
+            item_class = item.class
+            item_options = item.options || {}
+            item_options_headers = item_options[:headers] || item_options["headers"]
+
+            return yield unless item_class && item_options_headers
+
+            span_attributes = {
+              "messaging.system" => "outbox",
+              "messaging.outbox.item_id" => item.id.to_s,
+              "messaging.outbox.box_type" => item_class.box_type.to_s,
+              "messaging.outbox.box_name" => item_class.box_name,
+              "messaging.outbox.owner" => item_class.owner,
+              "messaging.destination" => item_class.name,
+              "messaging.destination_kind" => "database",
+              "messaging.operation" => "process"
+            }
+
+            extracted_context = ::OpenTelemetry.propagation.extract(item_options_headers)
+            ::OpenTelemetry::Context.with_current(extracted_context) do
+              tracer.in_span(span_name(item_class), attributes: span_attributes.compact, kind: :consumer) do
+                yield
+              end
+            end
+          end
+
+          private
+
+          def tracer
+            ::Sbmt::Outbox::Instrumentation::OpenTelemetryLoader.instance.tracer
+          end
+
+          def span_name(item_class)
+            "#{item_class.box_type}/#{item_class.box_name} process item"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/middleware/runner.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module Middleware
+      class Runner
+        attr_reader :stack
+
+        def initialize(stack)
+          @stack = stack
+        end
+
+        def call(*args)
+          return yield if stack.empty?
+
+          chain = stack.map { |i| i.new }
+          traverse_chain = proc do
+            if chain.empty?
+              yield
+            else
+              chain.shift.call(*args, &traverse_chain)
+            end
+          end
+          traverse_chain.call
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/middleware/sentry/tracing_batch_process_middleware.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/middleware/sentry/transaction"
+
+module Sbmt
+  module Outbox
+    module Middleware
+      module Sentry
+        class TracingBatchProcessMiddleware
+          include Transaction
+
+          def call(job)
+            return yield unless ::Sentry.initialized?
+
+            scope = ::Sentry.get_current_scope
+
+            # transaction will be nil if sentry tracing is not enabled
+            transaction = start_transaction(scope, job)
+            job.log_tags[:trace_id] = scope&.tags&.[](:trace_id)
+
+            begin
+              yield
+            rescue
+              finish_sentry_transaction(scope, transaction, 500)
+              raise
+            end
+
+            finish_sentry_transaction(scope, transaction, 200)
+          end
+
+          private
+
+          def start_transaction(scope, job)
+            start_sentry_transaction(scope, op(job), transaction_name(job), job.log_tags)
+          end
+
+          def op(job)
+            "sbmt.#{job.log_tags[:box_type]&.downcase}.batch_process"
+          end
+
+          def transaction_name(job)
+            "Sbmt.#{job.log_tags[:box_type]&.capitalize}.#{job.log_tags[:box_name]&.capitalize}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/middleware/sentry/tracing_item_process_middleware.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/middleware/sentry/transaction"
+
+module Sbmt
+  module Outbox
+    module Middleware
+      module Sentry
+        class TracingItemProcessMiddleware
+          include Transaction
+
+          attr_reader :new_transaction
+
+          def call(item)
+            return yield unless ::Sentry.initialized?
+
+            scope = ::Sentry.get_current_scope
+
+            # transaction will be nil if sentry tracing is not enabled
+            transaction = scope&.get_transaction || start_transaction(scope, item.class)
+            span = transaction&.start_child(op: op(item.class), description: "Starting item processing")
+            span&.set_data(:item_id, item.id)
+
+            begin
+              yield
+            rescue
+              finish_span(span, 500)
+              finish_sentry_transaction(scope, transaction, 500) if new_transaction
+              raise
+            end
+
+            finish_span(span, 200)
+            finish_sentry_transaction(scope, transaction, 200) if new_transaction
+          end
+
+          private
+
+          def finish_span(span, status)
+            return unless span
+
+            span.set_data(:status, status)
+            span.finish
+          end
+
+          def start_transaction(scope, item_class)
+            @new_transaction = true
+            start_sentry_transaction(scope, op(item_class), transaction_name(item_class), tags(item_class))
+          end
+
+          def transaction_name(item_class)
+            "Sbmt.#{item_class.box_type.capitalize}.#{item_class.box_name.capitalize}"
+          end
+
+          def tags(item_class)
+            {box_type: item_class.box_type, box_name: item_class.box_name}
+          end
+
+          def op(item_class)
+            "sbmt.#{item_class.box_type.downcase}.item_process"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/middleware/sentry/transaction.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module Middleware
+      module Sentry
+        module Transaction
+          def start_sentry_transaction(scope, op, name, tags = {})
+            trace_id = SecureRandom.base58
+            scope&.set_tags(tags.merge(trace_id: trace_id))
+            transaction = ::Sentry.start_transaction(op: op, name: name)
+            scope&.set_span(transaction) if transaction
+
+            transaction
+          end
+
+          def finish_sentry_transaction(scope, transaction, status)
+            return unless transaction
+
+            transaction.set_http_status(status)
+            transaction.finish
+            scope.clear
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/probes/probe.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module Probes
+      class Probe
+        DEFAULT_PROBE_PORT = 5555
+        class << self
+          def run_probes
+            ::HttpHealthCheck.run_server_async(
+              port: probe_port,
+              rack_app: HttpHealthCheck::RackApp.configure do |c|
+                c.logger Rails.logger
+                c.probe "/readiness/outbox" do |_env|
+                  code = Sbmt::Outbox.current_worker.ready? ? 200 : 500
+                  [code, {}, ["Outbox version: #{Sbmt::Outbox::VERSION}"]]
+                end
+
+                c.probe "/liveness/outbox" do |_env|
+                  code = Sbmt::Outbox.current_worker.alive? ? 200 : 500
+                  [code, {}, ["Outbox version: #{Sbmt::Outbox::VERSION}"]]
+                end
+              end
+            )
+          end
+
+          private
+
+          def probe_port
+            return DEFAULT_PROBE_PORT if Outbox.yaml_config["probes"].nil?
+
+            Sbmt::Outbox.yaml_config.fetch(:probes).fetch(:port)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/redis_client_factory.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module RedisClientFactory
+      def self.build(options)
+        options = options.deep_symbolize_keys
+
+        unless options.key?(:reconnect_attempts)
+          options[:reconnect_attempts] = 3
+        end
+
+        if options.key?(:sentinels)
+          if (url = options.delete(:url))
+            uri = URI.parse(url)
+            if !options.key?(:name) && uri.host
+              options[:name] = uri.host
+            end
+
+            if !options.key?(:password) && uri.password && !uri.password.empty?
+              options[:password] = uri.password
+            end
+
+            if !options.key?(:username) && uri.user && !uri.user.empty?
+              options[:username] = uri.user
+            end
+          end
+
+          RedisClient.sentinel(**options).new_client
+        else
+          RedisClient.config(**options).new_client
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/tasks/delete_failed_items.rake

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+namespace :outbox do
+  desc "Delete messages that we were unable to deliver"
+  # rake 'outbox:delete_failed_items[some/box_item]'
+  # rake 'outbox:delete_failed_items[some/box_item,1,2,3,4,5]'
+  task :delete_failed_items, %i[] => :environment do |_, args|
+    item_class_name, *ids = args.extras
+    item_class_name = item_class_name.classify
+    raise "Invalid item name" unless Sbmt::Outbox.item_classes.map(&:to_s).include?(item_class_name)
+    item_class = item_class_name.constantize
+
+    scope = item_class.failed
+    scope = scope.where(id: ids) unless ids.empty?
+    scope.in_batches.delete_all
+  end
+end

data/lib/sbmt/outbox/tasks/retry_failed_items.rake

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+namespace :outbox do
+  desc "Retry messages that we were unable to deliver"
+  # rake 'outbox:retry_failed_items[some/box_item]'
+  # rake 'outbox:retry_failed_items[some/box_item,1,2,3,4,5]'
+  task :retry_failed_items, %i[] => :environment do |_, args|
+    item_class_name, *ids = args.extras
+    item_class_name = item_class_name.classify
+    raise "Invalid item name" unless Sbmt::Outbox.item_classes.map(&:to_s).include?(item_class_name)
+    item_class = item_class_name.constantize
+
+    scope = item_class.failed
+    scope = scope.where(id: ids) unless ids.empty?
+    scope.in_batches.update_all(
+      status: Sbmt::Outbox::BaseItem.statuses[:pending],
+      errors_count: 0
+    )
+  end
+end

data/lib/sbmt/outbox/thread_pool.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "sbmt/outbox/throttler"
+
+module Sbmt
+  module Outbox
+    class ThreadPool
+      BREAK = Object.new.freeze
+      SKIPPED = Object.new.freeze
+      PROCESSED = Object.new.freeze
+
+      def initialize(&block)
+        self.task_source = block
+        self.task_mutex = Mutex.new
+        self.stopped = true
+      end
+
+      def next_task
+        task_mutex.synchronize do
+          return if stopped
+          item = task_source.call
+
+          if item == BREAK
+            self.stopped = true
+            return
+          end
+
+          item
+        end
+      end
+
+      def start(concurrency:)
+        self.stopped = false
+        result = run_threads(count: concurrency) do |item|
+          yield worker_number, item
+        end
+
+        raise result if result.is_a?(Exception)
+        nil
+      ensure
+        self.stopped = true
+      end
+
+      def stop
+        self.stopped = true
+      end
+
+      def worker_number
+        Thread.current["thread_pool_worker_number:#{object_id}"]
+      end
+
+      private
+
+      attr_accessor :task_source, :task_mutex, :stopped
+
+      def run_threads(count:)
+        exception = nil
+
+        in_threads(count: count) do |worker_num|
+          self.worker_number = worker_num
+          # We don't want to start all threads at the same time
+          random_sleep = rand * (worker_num + 1)
+
+          throttler = Throttler.new(
+            limit: Outbox.config.worker.rate_limit,
+            interval: Outbox.config.worker.rate_interval + random_sleep
+          )
+
+          sleep(random_sleep)
+
+          last_result = nil
+          until exception
+            throttler.wait if last_result == PROCESSED
+            item = next_task
+            break unless item
+
+            begin
+              last_result = yield item
+            rescue Exception => e # rubocop:disable Lint/RescueException
+              exception = e
+            end
+          end
+        end
+
+        exception
+      end
+
+      def in_threads(count:)
+        threads = []
+
+        Thread.handle_interrupt(Exception => :never) do
+          Thread.handle_interrupt(Exception => :immediate) do
+            count.times do |i|
+              threads << Thread.new { yield(i) }
+            end
+            threads.map(&:value)
+          end
+        ensure
+          threads.each(&:kill)
+        end
+      end
+
+      def worker_number=(num)
+        Thread.current["thread_pool_worker_number:#{object_id}"] = num
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/throttler.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    # Based on https://github.com/Shopify/limiter/blob/master/lib/limiter/rate_queue.rb
+    # We cannot use that gem because we have to support Ruby 2.5,
+    # but Shopify's limiter requires minimum Ruby 2.6
+    class Throttler
+      def initialize(limit: nil, interval: nil)
+        @limit = limit
+        @interval = limit
+        @map = (0...@limit).map { |i| base_time + (gap * i) }
+        @index = 0
+        @mutex = Mutex.new
+      end
+
+      def wait
+        time = nil
+
+        @mutex.synchronize do
+          time = @map[@index]
+
+          sleep_until(time + @interval)
+
+          @map[@index] = now
+          @index = (@index + 1) % @limit
+        end
+
+        time
+      end
+
+      private
+
+      def sleep_until(time)
+        period = time - now
+        sleep(period) if period > 0
+      end
+
+      def base_time
+        now - @interval
+      end
+
+      def gap
+        @interval.to_f / @limit.to_f
+      end
+
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    VERSION = "5.0.0"
+  end
+end