transactional_outbox 1.0.0beta1

data/lib/transactional_outbox/event.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative "event/contextable"
+require_relative "event/payloadable"
+
+module TransactionalOutbox
+  class Event
+    extend Dry::Configurable
+    extend Contextable
+    extend Payloadable
+
+    setting :schema
+    setting :aggregate_type
+    setting :event_type
+    setting :topic
+    setting :event_builder
+
+    config.values.keys.each do |attr|
+      define_singleton_method(attr) do |value|
+        config[attr] = value
+      end
+    end
+
+    def create!(context)
+      validate_context!(context)
+
+      event = build_event(context)
+
+      save([event])
+    end
+
+    def bulk_create!(contexts)
+      validate_contexts!(contexts)
+
+      events = build_events(contexts)
+
+      save(events)
+    end
+
+    private
+
+    def config = @config ||= self.class.config
+
+    def event_builder
+      @event_builder ||= begin
+        klass = config.event_builder || TransactionalOutbox.config.default_event_builder
+
+        Object.const_get(klass)
+      end
+    end
+
+    def save(rows) = TransactionalOutbox::Database.new.insert_events(rows)
+
+    def validate_event(schema, payload)
+      return true if config.schema.nil?
+
+      errors = JSON::Validator.fully_validate(schema, payload)
+
+      return true if errors.empty?
+
+      raise TransactionalOutbox::InvalidPayloadError, errors.join(", ")
+    end
+
+    def build_event(context)
+      payload = build_payload(context)
+
+      validate_event(config.schema, payload)
+
+      event_builder.build(self.class.config, payload, context)
+    end
+
+    def build_events(contexts) = contexts.map { build_event(_1) }
+
+    def validate_contexts!(contexts) = contexts.each { |context| validate_context!(context) }
+  end
+end

data/lib/transactional_outbox/exceptions.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  module Exceptions
+    class UnsupportedEventTypeError < StandardError; end
+    class InvalidPayloadError < StandardError; end
+    class UnknownAdapterError < StandardError; end
+    class AdapterAlreadyExistsError < StandardError; end
+    class MigrationFileNotExistsError < StandardError; end
+    class InvalidContextError < StandardError; end
+  end
+end

data/lib/transactional_outbox/exponential_backoff.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  class ExponentialBackoff
+    def self.calculate_retry_delay(retry_num) = 2**retry_num
+  end
+end

data/lib/transactional_outbox/producer/adapters/interface.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  class Producer
+    class Adapters
+      class Interface
+        def initialize(client)
+          @client = client
+        end
+
+        def produce_batch(_topic, _batch) = raise NotImplementedError
+
+        private
+
+        attr_reader :client
+      end
+    end
+  end
+end

data/lib/transactional_outbox/producer/adapters/kafka.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  class Producer
+    class Adapters
+      class Kafka < Interface
+        def produce_batch(topic, events)
+          buffer_events(topic, events)
+
+          producer.deliver_messages
+        end
+
+        private
+
+        def producer = @producer ||= client.producer
+
+        def buffer_events(topic, events)
+          events.each do |event|
+            producer.produce(event.to_json, topic:)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/transactional_outbox/producer/adapters/null.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  class Producer
+    class Adapters
+      class Null < Interface
+        def messages = @messages ||= {}
+        def clear_store = @messages = {}
+
+        def produce_batch(topic, events)
+          msg = messages[topic] ||= []
+
+          msg.concat(events)
+
+          TransactionalOutbox.config.logger.info("Batch produced")
+        end
+      end
+    end
+  end
+end

data/lib/transactional_outbox/producer/adapters.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  class Producer
+    class Adapters < TransactionalOutbox::AdaptersContainer
+    end
+  end
+end

data/lib/transactional_outbox/producer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  class Producer
+    extend Forwardable
+
+    def_delegators :@adapter, :produce_batch
+
+    attr_reader :adapter
+
+    def initialize
+      @adapter = TransactionalOutbox::Producer::Adapters.resolve(fetch_adapter).new(fetch_client)
+    end
+
+    private
+
+    def config = @config ||= TransactionalOutbox.config
+
+    def fetch_adapter
+      return :null if config.test_environment
+
+      config.producer.adapter.to_sym
+    end
+
+    def fetch_client
+      return if config.test_environment
+
+      config.producer.client
+    end
+  end
+end

data/lib/transactional_outbox/railtie.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  class Railtie < ::Rails::Railtie
+    generators do
+      require_relative "../generators/transactional_outbox/migration/migration_generator"
+    end
+
+    rake_tasks do
+      Dir[File.join(File.dirname(__FILE__), "tasks/*.rake")].each { |f| load f }
+    end
+  end
+end

data/lib/transactional_outbox/relay/event_processor.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  class Relay
+    class EventProcessor
+      attr_reader :topic, :db, :producer
+
+      def initialize(topic)
+        @topic = topic
+        @db = TransactionalOutbox::Database.new
+        @producer = TransactionalOutbox::Producer.new
+      end
+
+      def call
+        events = db.fetch_events(topic, config.relay.batch_size)
+
+        return if events.empty?
+
+        process_events(events)
+      rescue StandardError => e
+        TransactionalOutbox::Relay.monitor.publish(
+          TransactionalOutbox::WORKER_EXCEPTIONS_TOTAL_MONITOR_EVENT,
+          { topic:, exception: e.class.to_s }
+        )
+
+        resolve_failover.call(e, events)
+      end
+
+      private
+
+      def config = @config ||= TransactionalOutbox.config
+
+      def resolve_failover
+        configured = config.relay.failover
+
+        return Object.get_const(configured) if configured.is_a?(String)
+
+        configured
+      end
+
+      def process_events(events)
+        producer.produce_batch(topic, events)
+
+        db.delete_events(events.map { |x| x[:id] })
+
+        TransactionalOutbox::Relay.monitor.publish(
+          TransactionalOutbox::WORKER_EVENTS_PROCESSED_MONITOR_EVENT, { topic:, count: events.size }
+        )
+
+        config.logger.info("Events have sent to topic #{topic}: #{events.size}")
+      end
+    end
+  end
+end

data/lib/transactional_outbox/relay/failover.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  class Relay
+    class Failover
+      def self.call(exception, _events) = raise exception
+    end
+  end
+end

data/lib/transactional_outbox/relay/graceful_shutdown.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  class Relay
+    class GracefulShutdown
+      class << self
+        def call(worker_set)
+          worker_set.stop_workers
+
+          wait_workers(worker_set)
+        end
+
+        private
+
+        def config = @config ||= TransactionalOutbox.config
+
+        def wait_workers(worker_set)
+          shutdown_time = Time.now.utc
+
+          while shutdown_time + config.shutdown_waiting_time_seconds > Time.now.utc
+            if worker_set.all_stopped?
+              TransactionalOutbox::Relay.monitor.publish(TransactionalOutbox::RUNNER_STOPPED_MONITOR_EVENT)
+
+              config.logger.info("All workers have stopped.")
+
+              return true
+            end
+
+            sleep(0.1)
+          end
+
+          false
+        end
+      end
+    end
+  end
+end

data/lib/transactional_outbox/relay/monitor.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  class Relay
+    class Monitor
+      extend Forwardable
+
+      def_delegators :@notifications, :publish, :subscribe
+
+      def initialize
+        @notifications = Dry::Monitor::Notifications.new(:transactional_outbox).tap { |n| register_events(n) }
+      end
+
+      private
+
+      attr_reader :notifications
+
+      def register_events(notifications)
+        notifications.register_event(TransactionalOutbox::RUNNER_INIT_MONITOR_EVENT)
+        notifications.register_event(TransactionalOutbox::RUNNER_STOPPED_MONITOR_EVENT)
+        notifications.register_event(TransactionalOutbox::WORKER_RUN_MONITOR_EVENT)
+        notifications.register_event(TransactionalOutbox::WORKER_EVENTS_PROCESSED_MONITOR_EVENT)
+        notifications.register_event(TransactionalOutbox::WORKER_STOPPED_MONITOR_EVENT)
+        notifications.register_event(TransactionalOutbox::WORKER_EXCEPTIONS_TOTAL_MONITOR_EVENT)
+      end
+    end
+  end
+end

data/lib/transactional_outbox/relay/runner.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative "worker_set"
+require_relative "worker_set/processor"
+require_relative "graceful_shutdown"
+
+module TransactionalOutbox
+  class Relay
+    class Runner
+      class << self
+        def start
+          TransactionalOutbox::Relay.monitor.publish(TransactionalOutbox::RUNNER_INIT_MONITOR_EVENT)
+
+          worker_set = TransactionalOutbox::Relay::WorkerSet.new
+
+          start_relay(worker_set)
+        rescue StandardError => e
+          shutdown(e, worker_set, 1)
+        rescue SignalException => e
+          shutdown(e, worker_set, 0)
+        end
+
+        private
+
+        def start_relay(worker_set) = TransactionalOutbox::Relay::WorkerSet::Processor.new(worker_set).call
+        def config = @config ||= TransactionalOutbox.config
+
+        def start_graceful_shutdown(worker_set)
+          return unless worker_set
+
+          TransactionalOutbox::Relay::GracefulShutdown.call(worker_set)
+        end
+
+        def call_exit(code)
+          return true if config.test_environment
+
+          exit(code)
+        end
+
+        def shutdown(exception, worker_set, exit_code)
+          config.logger.info("Received exception: #{exception}. Shutting down...")
+
+          start_graceful_shutdown(worker_set)
+
+          call_exit(exit_code)
+        end
+      end
+    end
+  end
+end

data/lib/transactional_outbox/relay/worker_set/processor.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  class Relay
+    class WorkerSet
+      class Processor
+        def initialize(worker_set)
+          @worker_set = worker_set
+          @db = TransactionalOutbox::Database.new
+        end
+
+        def call # rubocop:disable Metrics/MethodLength
+          @retry_counter = 0
+
+          loop do
+            topics = db.fetch_topics
+
+            process_topics(topics)
+
+            break if config.test_environment
+
+            @retry_counter = 0
+          rescue StandardError => e
+            config.logger.error("Exception: #{e}, trying to retry...")
+
+            raise e if @retry_counter >= config.relay.max_runner_retries_count
+
+            @retry_counter += 1
+
+            sleep(calculate_retry_delay)
+
+            retry
+          end
+        end
+
+        private
+
+        attr_reader :worker_set, :db
+
+        def config = @config ||= TransactionalOutbox.config
+        def calculate_retry_delay = TransactionalOutbox::ExponentialBackoff.calculate_retry_delay(@retry_counter)
+
+        def process_topics(topics)
+          topics.each do |topic|
+            worker = worker_set.get_worker(topic)
+
+            worker ? worker_set.try_to_recover_worker(topic) : worker_set.add_worker(topic)
+          end
+
+          sleep(config.relay.delay_between_worker_set_processor_cycles)
+        end
+      end
+    end
+  end
+end

data/lib/transactional_outbox/relay/worker_set/worker.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require_relative "../event_processor"
+
+module TransactionalOutbox
+  class Relay
+    class WorkerSet
+      class Worker
+        attr_reader :topic, :db, :producer
+
+        def initialize(topic)
+          @topic = topic
+        end
+
+        def run
+          @thread = spawn_thread
+        end
+
+        def shutdown
+          return true if thread[:stopped]
+
+          thread[:shutdown] = true
+        end
+
+        def shutting_down? = !thread.nil? && !!thread[:shutdown]
+        def stopped? = !thread.nil? && !!thread[:stopped]
+
+        private
+
+        attr_reader :thread
+
+        def config = @config ||= TransactionalOutbox.config
+
+        def spawn_thread # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+          TransactionalOutbox::Relay.monitor.publish(TransactionalOutbox::WORKER_RUN_MONITOR_EVENT, { topic: })
+
+          Thread.new do
+            loop do
+              TransactionalOutbox::Relay::EventProcessor.new(topic).call
+
+              if Thread.current[:shutdown]
+                config.logger.info("Thread for topic #{topic} successfully shutted down")
+
+                mark_thread_as_stopped
+
+                break
+              end
+
+              break if config.test_environment
+
+              sleep(config.relay.wait_between_batches_seconds)
+            end
+          rescue StandardError => e
+            mark_thread_as_stopped
+
+            raise e
+          end
+        end
+
+        def mark_thread_as_stopped
+          Thread.current[:stopped] = true
+
+          TransactionalOutbox::Relay.monitor.publish(TransactionalOutbox::WORKER_STOPPED_MONITOR_EVENT, { topic: })
+        end
+      end
+    end
+  end
+end

data/lib/transactional_outbox/relay/worker_set.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "worker_set/worker"
+
+module TransactionalOutbox
+  class Relay
+    class WorkerSet
+      def initialize
+        @workers = {}
+      end
+
+      def get_worker(topic) = workers[topic]
+
+      def add_worker(topic)
+        return if workers.key?(topic)
+
+        create_worker(topic)
+      end
+
+      def try_to_recover_worker(topic)
+        worker = get_worker(topic)
+
+        return if !worker.nil? && (worker.shutting_down? || !worker.stopped?)
+
+        create_worker(topic)
+      end
+
+      def stop_workers = workers.each_value(&:shutdown)
+      def all_stopped? = workers.values.all?(&:stopped?)
+
+      private
+
+      attr_reader :workers
+
+      def create_worker(topic) = workers[topic] = Worker.new(topic).tap(&:run)
+    end
+  end
+end

data/lib/transactional_outbox/relay.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative "relay/monitor"
+
+module TransactionalOutbox
+  class Relay
+    def self.monitor = @monitor ||= TransactionalOutbox::Relay::Monitor.new
+  end
+end

data/lib/transactional_outbox/tasks/relay.rake

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+namespace :relay do
+  desc "Runs transactional outbox event relay"
+  task run: :environment do
+    TransactionalOutbox::Relay::Runner.start
+  end
+end

data/lib/transactional_outbox/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module TransactionalOutbox
+  VERSION = "1.0.0beta1"
+end

data/lib/transactional_outbox.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+require "dry-container"
+require "dry-configurable"
+require "dry-monitor"
+require "dry-validation"
+require "json-schema"
+require "securerandom"
+
+require_relative "transactional_outbox/version"
+require_relative "transactional_outbox/adapters_container"
+require_relative "transactional_outbox/database/adapters"
+require_relative "transactional_outbox/database/adapters/interface"
+require_relative "transactional_outbox/database/adapters/null"
+require_relative "transactional_outbox/database/adapters/active_record"
+require_relative "transactional_outbox/database/adapters/sequel"
+require_relative "transactional_outbox/producer/adapters"
+require_relative "transactional_outbox/producer/adapters/interface"
+require_relative "transactional_outbox/producer/adapters/null"
+require_relative "transactional_outbox/producer/adapters/kafka"
+require_relative "transactional_outbox/constants"
+require_relative "transactional_outbox/database"
+require_relative "transactional_outbox/event"
+require_relative "transactional_outbox/event/builder"
+require_relative "transactional_outbox/exceptions"
+require_relative "transactional_outbox/exponential_backoff"
+require_relative "transactional_outbox/producer"
+
+require_relative "transactional_outbox/relay"
+require_relative "transactional_outbox/relay/failover"
+require_relative "transactional_outbox/relay/runner"
+
+require_relative "transactional_outbox/railtie" if defined?(Rails::Railtie)
+
+TransactionalOutbox::Database::Adapters.register(:null, TransactionalOutbox::Database::Adapters::Null)
+TransactionalOutbox::Database::Adapters.register(:sequel, TransactionalOutbox::Database::Adapters::Sequel)
+TransactionalOutbox::Database::Adapters.register(:active_record, TransactionalOutbox::Database::Adapters::ActiveRecord)
+
+TransactionalOutbox::Producer::Adapters.register(:null, TransactionalOutbox::Producer::Adapters::Null)
+TransactionalOutbox::Producer::Adapters.register(:kafka, TransactionalOutbox::Producer::Adapters::Kafka)
+
+module TransactionalOutbox
+  include Constants
+  include Exceptions
+
+  extend Dry::Configurable
+
+  setting :logger
+  setting :outbox_table_name, default: "outbox_events"
+  setting :default_event_builder, default: TransactionalOutbox::Event::Builder
+  setting :shutdown_waiting_time_seconds, default: 10
+  setting :migrations_directory
+  setting :test_environment, default: false
+
+  setting :relay do
+    setting :batch_size, default: 20
+    setting :wait_between_batches_seconds, default: 0.1
+    setting :failover, default: TransactionalOutbox::Relay::Failover
+    setting :max_runner_retries_count, default: 5
+    setting :delay_between_worker_set_processor_cycles, default: 1
+  end
+
+  setting :db do
+    setting :adapter
+    setting :connection_data
+  end
+
+  setting :producer do
+    setting :adapter
+    setting :client
+  end
+
+  class << self
+    def transaction(event)
+      Database.new.transaction do
+        yield(event.new)
+      end
+    end
+  end
+end